summaryrefslogtreecommitdiff
path: root/system/easy-kernel/0100-linux.5.15.44.patch
diff options
context:
space:
mode:
Diffstat (limited to 'system/easy-kernel/0100-linux.5.15.44.patch')
-rw-r--r--system/easy-kernel/0100-linux.5.15.44.patch270104
1 files changed, 270104 insertions, 0 deletions
diff --git a/system/easy-kernel/0100-linux.5.15.44.patch b/system/easy-kernel/0100-linux.5.15.44.patch
new file mode 100644
index 000000000..11060bfba
--- /dev/null
+++ b/system/easy-kernel/0100-linux.5.15.44.patch
@@ -0,0 +1,270104 @@
+diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
+index f627e705e663b..48d41b6696270 100644
+--- a/Documentation/ABI/testing/sysfs-fs-f2fs
++++ b/Documentation/ABI/testing/sysfs-fs-f2fs
+@@ -425,6 +425,7 @@ Description: Show status of f2fs superblock in real time.
+ 0x800 SBI_QUOTA_SKIP_FLUSH skip flushing quota in current CP
+ 0x1000 SBI_QUOTA_NEED_REPAIR quota file may be corrupted
+ 0x2000 SBI_IS_RESIZEFS resizefs is in process
++ 0x4000 SBI_IS_FREEZING freefs is in process
+ ====== ===================== =================================
+
+ What: /sys/fs/f2fs/<disk>/ckpt_thread_ioprio
+diff --git a/Documentation/accounting/psi.rst b/Documentation/accounting/psi.rst
+index f2b3439edcc2c..860fe651d6453 100644
+--- a/Documentation/accounting/psi.rst
++++ b/Documentation/accounting/psi.rst
+@@ -92,7 +92,8 @@ Triggers can be set on more than one psi metric and more than one trigger
+ for the same psi metric can be specified. However for each trigger a separate
+ file descriptor is required to be able to poll it separately from others,
+ therefore for each trigger a separate open() syscall should be made even
+-when opening the same psi interface file.
++when opening the same psi interface file. Write operations to a file descriptor
++with an already existing psi trigger will fail with EBUSY.
+
+ Monitors activate only when system enters stall state for the monitored
+ psi metric and deactivates upon exit from the stall state. While system is
+diff --git a/Documentation/admin-guide/cifs/usage.rst b/Documentation/admin-guide/cifs/usage.rst
+index f170d88202588..3766bf8a1c20e 100644
+--- a/Documentation/admin-guide/cifs/usage.rst
++++ b/Documentation/admin-guide/cifs/usage.rst
+@@ -734,10 +734,9 @@ SecurityFlags Flags which control security negotiation and
+ using weaker password hashes is 0x37037 (lanman,
+ plaintext, ntlm, ntlmv2, signing allowed). Some
+ SecurityFlags require the corresponding menuconfig
+- options to be enabled (lanman and plaintext require
+- CONFIG_CIFS_WEAK_PW_HASH for example). Enabling
+- plaintext authentication currently requires also
+- enabling lanman authentication in the security flags
++ options to be enabled. Enabling plaintext
++ authentication currently requires also enabling
++ lanman authentication in the security flags
+ because the cifs module only supports sending
+ laintext passwords using the older lanman dialect
+ form of the session setup SMB. (e.g. for authentication
+diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt
+index 922c23bb4372a..c07dc0ee860e7 100644
+--- a/Documentation/admin-guide/devices.txt
++++ b/Documentation/admin-guide/devices.txt
+@@ -2339,13 +2339,7 @@
+ disks (see major number 3) except that the limit on
+ partitions is 31.
+
+- 162 char Raw block device interface
+- 0 = /dev/rawctl Raw I/O control device
+- 1 = /dev/raw/raw1 First raw I/O device
+- 2 = /dev/raw/raw2 Second raw I/O device
+- ...
+- max minor number of raw device is set by kernel config
+- MAX_RAW_DEVS or raw module parameter 'max_raw_devs'
++ 162 char Used for (now removed) raw block device interface
+
+ 163 char
+
+diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst
+index e05e581af5cfe..6bd97cd50d625 100644
+--- a/Documentation/admin-guide/hw-vuln/spectre.rst
++++ b/Documentation/admin-guide/hw-vuln/spectre.rst
+@@ -60,8 +60,8 @@ privileged data touched during the speculative execution.
+ Spectre variant 1 attacks take advantage of speculative execution of
+ conditional branches, while Spectre variant 2 attacks use speculative
+ execution of indirect branches to leak privileged memory.
+-See :ref:`[1] <spec_ref1>` :ref:`[5] <spec_ref5>` :ref:`[7] <spec_ref7>`
+-:ref:`[10] <spec_ref10>` :ref:`[11] <spec_ref11>`.
++See :ref:`[1] <spec_ref1>` :ref:`[5] <spec_ref5>` :ref:`[6] <spec_ref6>`
++:ref:`[7] <spec_ref7>` :ref:`[10] <spec_ref10>` :ref:`[11] <spec_ref11>`.
+
+ Spectre variant 1 (Bounds Check Bypass)
+ ---------------------------------------
+@@ -131,6 +131,19 @@ steer its indirect branch speculations to gadget code, and measure the
+ speculative execution's side effects left in level 1 cache to infer the
+ victim's data.
+
++Yet another variant 2 attack vector is for the attacker to poison the
++Branch History Buffer (BHB) to speculatively steer an indirect branch
++to a specific Branch Target Buffer (BTB) entry, even if the entry isn't
++associated with the source address of the indirect branch. Specifically,
++the BHB might be shared across privilege levels even in the presence of
++Enhanced IBRS.
++
++Currently the only known real-world BHB attack vector is via
++unprivileged eBPF. Therefore, it's highly recommended to not enable
++unprivileged eBPF, especially when eIBRS is used (without retpolines).
++For a full mitigation against BHB attacks, it's recommended to use
++retpolines (or eIBRS combined with retpolines).
++
+ Attack scenarios
+ ----------------
+
+@@ -364,13 +377,15 @@ The possible values in this file are:
+
+ - Kernel status:
+
+- ==================================== =================================
+- 'Not affected' The processor is not vulnerable
+- 'Vulnerable' Vulnerable, no mitigation
+- 'Mitigation: Full generic retpoline' Software-focused mitigation
+- 'Mitigation: Full AMD retpoline' AMD-specific software mitigation
+- 'Mitigation: Enhanced IBRS' Hardware-focused mitigation
+- ==================================== =================================
++ ======================================== =================================
++ 'Not affected' The processor is not vulnerable
++ 'Mitigation: None' Vulnerable, no mitigation
++ 'Mitigation: Retpolines' Use Retpoline thunks
++ 'Mitigation: LFENCE' Use LFENCE instructions
++ 'Mitigation: Enhanced IBRS' Hardware-focused mitigation
++ 'Mitigation: Enhanced IBRS + Retpolines' Hardware-focused + Retpolines
++ 'Mitigation: Enhanced IBRS + LFENCE' Hardware-focused + LFENCE
++ ======================================== =================================
+
+ - Firmware status: Show if Indirect Branch Restricted Speculation (IBRS) is
+ used to protect against Spectre variant 2 attacks when calling firmware (x86 only).
+@@ -468,7 +483,7 @@ Spectre variant 2
+ before invoking any firmware code to prevent Spectre variant 2 exploits
+ using the firmware.
+
+- Using kernel address space randomization (CONFIG_RANDOMIZE_SLAB=y
++ Using kernel address space randomization (CONFIG_RANDOMIZE_BASE=y
+ and CONFIG_SLAB_FREELIST_RANDOM=y in the kernel configuration) makes
+ attacks on the kernel generally more difficult.
+
+@@ -584,12 +599,13 @@ kernel command line.
+
+ Specific mitigations can also be selected manually:
+
+- retpoline
+- replace indirect branches
+- retpoline,generic
+- google's original retpoline
+- retpoline,amd
+- AMD-specific minimal thunk
++ retpoline auto pick between generic,lfence
++ retpoline,generic Retpolines
++ retpoline,lfence LFENCE; indirect branch
++ retpoline,amd alias for retpoline,lfence
++ eibrs enhanced IBRS
++ eibrs,retpoline enhanced IBRS + Retpolines
++ eibrs,lfence enhanced IBRS + LFENCE
+
+ Not specifying this option is equivalent to
+ spectre_v2=auto.
+@@ -730,7 +746,7 @@ AMD white papers:
+
+ .. _spec_ref6:
+
+-[6] `Software techniques for managing speculation on AMD processors <https://developer.amd.com/wp-content/resources/90343-B_SoftwareTechniquesforManagingSpeculation_WP_7-18Update_FNL.pdf>`_.
++[6] `Software techniques for managing speculation on AMD processors <https://developer.amd.com/wp-content/resources/Managing-Speculation-on-AMD-Processors.pdf>`_.
+
+ ARM white papers:
+
+diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
+index 43dc35fe5bc03..fd3d14dabadcb 100644
+--- a/Documentation/admin-guide/kernel-parameters.txt
++++ b/Documentation/admin-guide/kernel-parameters.txt
+@@ -1690,6 +1690,8 @@
+ architectures force reset to be always executed
+ i8042.unlock [HW] Unlock (ignore) the keylock
+ i8042.kbdreset [HW] Reset device connected to KBD port
++ i8042.probe_defer
++ [HW] Allow deferred probing upon i8042 probe errors
+
+ i810= [HW,DRM]
+
+@@ -2403,8 +2405,12 @@
+ Default is 1 (enabled)
+
+ kvm-intel.emulate_invalid_guest_state=
+- [KVM,Intel] Enable emulation of invalid guest states
+- Default is 0 (disabled)
++ [KVM,Intel] Disable emulation of invalid guest state.
++ Ignored if kvm-intel.enable_unrestricted_guest=1, as
++ guest state is never invalid for unrestricted guests.
++ This param doesn't apply to nested guests (L2), as KVM
++ never emulates invalid L2 guest state.
++ Default is 1 (enabled)
+
+ kvm-intel.flexpriority=
+ [KVM,Intel] Disable FlexPriority feature (TPR shadow).
+@@ -3446,8 +3452,7 @@
+ difficult since unequal pointers can no longer be
+ compared. However, if this command-line option is
+ specified, then all normal pointers will have their true
+- value printed. Pointers printed via %pK may still be
+- hashed. This option should only be specified when
++ value printed. This option should only be specified when
+ debugging the kernel. Please do not use on production
+ kernels.
+
+@@ -4302,6 +4307,12 @@
+ fully seed the kernel's CRNG. Default is controlled
+ by CONFIG_RANDOM_TRUST_CPU.
+
++ random.trust_bootloader={on,off}
++ [KNL] Enable or disable trusting the use of a
++ seed passed by the bootloader (if available) to
++ fully seed the kernel's CRNG. Default is controlled
++ by CONFIG_RANDOM_TRUST_BOOTLOADER.
++
+ randomize_kstack_offset=
+ [KNL] Enable or disable kernel stack offset
+ randomization, which provides roughly 5 bits of
+@@ -5261,8 +5272,12 @@
+ Specific mitigations can also be selected manually:
+
+ retpoline - replace indirect branches
+- retpoline,generic - google's original retpoline
+- retpoline,amd - AMD-specific minimal thunk
++ retpoline,generic - Retpolines
++ retpoline,lfence - LFENCE; indirect branch
++ retpoline,amd - alias for retpoline,lfence
++ eibrs - enhanced IBRS
++ eibrs,retpoline - enhanced IBRS + Retpolines
++ eibrs,lfence - enhanced IBRS + LFENCE
+
+ Not specifying this option is equivalent to
+ spectre_v2=auto.
+@@ -6349,6 +6364,13 @@
+ improve timer resolution at the expense of processing
+ more timer interrupts.
+
++ xen.balloon_boot_timeout= [XEN]
++ The time (in seconds) to wait before giving up to boot
++ in case initial ballooning fails to free enough memory.
++ Applies only when running as HVM or PVH guest and
++ started with less memory configured than allowed at
++ max. Default is 180.
++
+ xen.event_eoi_delay= [XEN]
+ How long to delay EOI handling in case of event
+ storms (jiffies). Default is 10.
+diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst
+index fb578fbbb76ca..49857ce1cd03e 100644
+--- a/Documentation/admin-guide/mm/pagemap.rst
++++ b/Documentation/admin-guide/mm/pagemap.rst
+@@ -23,7 +23,7 @@ There are four components to pagemap:
+ * Bit 56 page exclusively mapped (since 4.2)
+ * Bit 57 pte is uffd-wp write-protected (since 5.13) (see
+ :ref:`Documentation/admin-guide/mm/userfaultfd.rst <userfaultfd>`)
+- * Bits 57-60 zero
++ * Bits 58-60 zero
+ * Bit 61 page is file-page or shared-anon (since 3.5)
+ * Bit 62 page swapped
+ * Bit 63 page present
+diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
+index 426162009ce99..609b891754081 100644
+--- a/Documentation/admin-guide/sysctl/kernel.rst
++++ b/Documentation/admin-guide/sysctl/kernel.rst
+@@ -795,6 +795,7 @@ bit 1 print system memory info
+ bit 2 print timer info
+ bit 3 print locks info if ``CONFIG_LOCKDEP`` is on
+ bit 4 print ftrace buffer
++bit 5 print all printk messages in buffer
+ ===== ============================================
+
+ So for example to print tasks and memory info on panic, user can::
+@@ -1013,28 +1014,22 @@ This is a directory, with the following entries:
+ * ``boot_id``: a UUID generated the first time this is retrieved, and
+ unvarying after that;
+
++* ``uuid``: a UUID generated every time this is retrieved (this can
++ thus be used to generate UUIDs at will);
++
+ * ``entropy_avail``: the pool's entropy count, in bits;
+
+ * ``poolsize``: the entropy pool size, in bits;
+
+ * ``urandom_min_reseed_secs``: obsolete (used to determine the minimum
+- number of seconds between urandom pool reseeding).
+-
+-* ``uuid``: a UUID generated every time this is retrieved (this can
+- thus be used to generate UUIDs at will);
++ number of seconds between urandom pool reseeding). This file is
++ writable for compatibility purposes, but writing to it has no effect
++ on any RNG behavior;
+
+ * ``write_wakeup_threshold``: when the entropy count drops below this
+ (as a number of bits), processes waiting to write to ``/dev/random``
+- are woken up.
+-
+-If ``drivers/char/random.c`` is built with ``ADD_INTERRUPT_BENCH``
+-defined, these additional entries are present:
+-
+-* ``add_interrupt_avg_cycles``: the average number of cycles between
+- interrupts used to feed the pool;
+-
+-* ``add_interrupt_avg_deviation``: the standard deviation seen on the
+- number of cycles between interrupts used to feed the pool.
++ are woken up. This file is writable for compatibility purposes, but
++ writing to it has no effect on any RNG behavior.
+
+
+ randomize_va_space
+@@ -1099,7 +1094,7 @@ task_delayacct
+ ===============
+
+ Enables/disables task delay accounting (see
+-:doc:`accounting/delay-accounting.rst`). Enabling this feature incurs
++Documentation/accounting/delay-accounting.rst. Enabling this feature incurs
+ a small amount of overhead in the scheduler but is useful for debugging
+ and performance tuning. It is required by some tools such as iotop.
+
+diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst
+index 328e0c454fbd4..749ae970c3195 100644
+--- a/Documentation/arm64/cpu-feature-registers.rst
++++ b/Documentation/arm64/cpu-feature-registers.rst
+@@ -235,7 +235,15 @@ infrastructure:
+ | DPB | [3-0] | y |
+ +------------------------------+---------+---------+
+
+- 6) ID_AA64MMFR2_EL1 - Memory model feature register 2
++ 6) ID_AA64MMFR0_EL1 - Memory model feature register 0
++
++ +------------------------------+---------+---------+
++ | Name | bits | visible |
++ +------------------------------+---------+---------+
++ | ECV | [63-60] | y |
++ +------------------------------+---------+---------+
++
++ 7) ID_AA64MMFR2_EL1 - Memory model feature register 2
+
+ +------------------------------+---------+---------+
+ | Name | bits | visible |
+@@ -243,7 +251,7 @@ infrastructure:
+ | AT | [35-32] | y |
+ +------------------------------+---------+---------+
+
+- 7) ID_AA64ZFR0_EL1 - SVE feature ID register 0
++ 8) ID_AA64ZFR0_EL1 - SVE feature ID register 0
+
+ +------------------------------+---------+---------+
+ | Name | bits | visible |
+@@ -267,6 +275,23 @@ infrastructure:
+ | SVEVer | [3-0] | y |
+ +------------------------------+---------+---------+
+
++ 8) ID_AA64MMFR1_EL1 - Memory model feature register 1
++
++ +------------------------------+---------+---------+
++ | Name | bits | visible |
++ +------------------------------+---------+---------+
++ | AFP | [47-44] | y |
++ +------------------------------+---------+---------+
++
++ 9) ID_AA64ISAR2_EL1 - Instruction set attribute register 2
++
++ +------------------------------+---------+---------+
++ | Name | bits | visible |
++ +------------------------------+---------+---------+
++ | RPRES | [7-4] | y |
++ +------------------------------+---------+---------+
++
++
+ Appendix I: Example
+ -------------------
+
+diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
+index ec1a5a63c1d09..b72ff17d600ae 100644
+--- a/Documentation/arm64/elf_hwcaps.rst
++++ b/Documentation/arm64/elf_hwcaps.rst
+@@ -247,6 +247,18 @@ HWCAP2_MTE
+ Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described
+ by Documentation/arm64/memory-tagging-extension.rst.
+
++HWCAP2_ECV
++
++ Functionality implied by ID_AA64MMFR0_EL1.ECV == 0b0001.
++
++HWCAP2_AFP
++
++ Functionality implied by ID_AA64MFR1_EL1.AFP == 0b0001.
++
++HWCAP2_RPRES
++
++ Functionality implied by ID_AA64ISAR2_EL1.RPRES == 0b0001.
++
+ 4. Unused AT_HWCAP bits
+ -----------------------
+
+diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst
+index d410a47ffa57a..7c1750bcc5bd8 100644
+--- a/Documentation/arm64/silicon-errata.rst
++++ b/Documentation/arm64/silicon-errata.rst
+@@ -163,6 +163,9 @@ stable kernels.
+ +----------------+-----------------+-----------------+-----------------------------+
+ | Qualcomm Tech. | Kryo4xx Silver | N/A | ARM64_ERRATUM_1024718 |
+ +----------------+-----------------+-----------------+-----------------------------+
++| Qualcomm Tech. | Kryo4xx Gold | N/A | ARM64_ERRATUM_1286807 |
+++----------------+-----------------+-----------------+-----------------------------+
++
+ +----------------+-----------------+-----------------+-----------------------------+
+ | Fujitsu | A64FX | E#010001 | FUJITSU_ERRATUM_010001 |
+ +----------------+-----------------+-----------------+-----------------------------+
+diff --git a/Documentation/dev-tools/kfence.rst b/Documentation/dev-tools/kfence.rst
+index 0fbe3308bf37f..48244d32780f6 100644
+--- a/Documentation/dev-tools/kfence.rst
++++ b/Documentation/dev-tools/kfence.rst
+@@ -231,10 +231,14 @@ Guarded allocations are set up based on the sample interval. After expiration
+ of the sample interval, the next allocation through the main allocator (SLAB or
+ SLUB) returns a guarded allocation from the KFENCE object pool (allocation
+ sizes up to PAGE_SIZE are supported). At this point, the timer is reset, and
+-the next allocation is set up after the expiration of the interval. To "gate" a
+-KFENCE allocation through the main allocator's fast-path without overhead,
+-KFENCE relies on static branches via the static keys infrastructure. The static
+-branch is toggled to redirect the allocation to KFENCE.
++the next allocation is set up after the expiration of the interval.
++
++When using ``CONFIG_KFENCE_STATIC_KEYS=y``, KFENCE allocations are "gated"
++through the main allocator's fast-path by relying on static branches via the
++static keys infrastructure. The static branch is toggled to redirect the
++allocation to KFENCE. Depending on sample interval, target workloads, and
++system architecture, this may perform better than the simple dynamic branch.
++Careful benchmarking is recommended.
+
+ KFENCE objects each reside on a dedicated page, at either the left or right
+ page boundaries selected at random. The pages to the left and right of the
+diff --git a/Documentation/devicetree/bindings/arm/omap/omap.txt b/Documentation/devicetree/bindings/arm/omap/omap.txt
+index e77635c5422c6..fa8b31660cadd 100644
+--- a/Documentation/devicetree/bindings/arm/omap/omap.txt
++++ b/Documentation/devicetree/bindings/arm/omap/omap.txt
+@@ -119,6 +119,9 @@ Boards (incomplete list of examples):
+ - OMAP3 BeagleBoard : Low cost community board
+ compatible = "ti,omap3-beagle", "ti,omap3430", "ti,omap3"
+
++- OMAP3 BeagleBoard A to B4 : Early BeagleBoard revisions A to B4 with a timer quirk
++ compatible = "ti,omap3-beagle-ab4", "ti,omap3-beagle", "ti,omap3430", "ti,omap3"
++
+ - OMAP3 Tobi with Overo : Commercial expansion board with daughter board
+ compatible = "gumstix,omap3-overo-tobi", "gumstix,omap3-overo", "ti,omap3430", "ti,omap3"
+
+diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml
+index cf5a208f2f105..343598c9f473b 100644
+--- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml
++++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml
+@@ -10,6 +10,9 @@ title: Amlogic specific extensions to the Synopsys Designware HDMI Controller
+ maintainers:
+ - Neil Armstrong <narmstrong@baylibre.com>
+
++allOf:
++ - $ref: /schemas/sound/name-prefix.yaml#
++
+ description: |
+ The Amlogic Meson Synopsys Designware Integration is composed of
+ - A Synopsys DesignWare HDMI Controller IP
+@@ -99,6 +102,8 @@ properties:
+ "#sound-dai-cells":
+ const: 0
+
++ sound-name-prefix: true
++
+ required:
+ - compatible
+ - reg
+diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml
+index 851cb07812173..047fd69e03770 100644
+--- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml
++++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml
+@@ -78,6 +78,10 @@ properties:
+ interrupts:
+ maxItems: 1
+
++ amlogic,canvas:
++ description: should point to a canvas provider node
++ $ref: /schemas/types.yaml#/definitions/phandle
++
+ power-domains:
+ maxItems: 1
+ description: phandle to the associated power domain
+@@ -106,6 +110,7 @@ required:
+ - port@1
+ - "#address-cells"
+ - "#size-cells"
++ - amlogic,canvas
+
+ additionalProperties: false
+
+@@ -118,6 +123,7 @@ examples:
+ interrupts = <3>;
+ #address-cells = <1>;
+ #size-cells = <0>;
++ amlogic,canvas = <&canvas>;
+
+ /* CVBS VDAC output port */
+ port@0 {
+diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5766.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5766.yaml
+index d5c54813ce872..a8f7720d1e3e2 100644
+--- a/Documentation/devicetree/bindings/iio/dac/adi,ad5766.yaml
++++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5766.yaml
+@@ -54,7 +54,7 @@ examples:
+
+ ad5766@0 {
+ compatible = "adi,ad5766";
+- output-range-microvolts = <(-5000) 5000>;
++ output-range-microvolts = <(-5000000) 5000000>;
+ reg = <0>;
+ spi-cpol;
+ spi-max-frequency = <1000000>;
+diff --git a/Documentation/devicetree/bindings/input/hid-over-i2c.txt b/Documentation/devicetree/bindings/input/hid-over-i2c.txt
+index c76bafaf98d2f..34c43d3bddfd1 100644
+--- a/Documentation/devicetree/bindings/input/hid-over-i2c.txt
++++ b/Documentation/devicetree/bindings/input/hid-over-i2c.txt
+@@ -32,6 +32,8 @@ device-specific compatible properties, which should be used in addition to the
+ - vdd-supply: phandle of the regulator that provides the supply voltage.
+ - post-power-on-delay-ms: time required by the device after enabling its regulators
+ or powering it on, before it is ready for communication.
++- touchscreen-inverted-x: See touchscreen.txt
++- touchscreen-inverted-y: See touchscreen.txt
+
+ Example:
+
+diff --git a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml
+index 877183cf42787..1ef849dc74d7e 100644
+--- a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml
++++ b/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml
+@@ -79,6 +79,8 @@ properties:
+
+ properties:
+ data-lanes:
++ description:
++ Note that 'fsl,imx7-mipi-csi2' only supports up to 2 data lines.
+ items:
+ minItems: 1
+ maxItems: 4
+@@ -91,18 +93,6 @@ properties:
+ required:
+ - data-lanes
+
+- allOf:
+- - if:
+- properties:
+- compatible:
+- contains:
+- const: fsl,imx7-mipi-csi2
+- then:
+- properties:
+- data-lanes:
+- items:
+- maxItems: 2
+-
+ port@1:
+ $ref: /schemas/graph.yaml#/properties/port
+ description:
+diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
+index 2353f6cf3c805..750720e0123a0 100644
+--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
++++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
+@@ -106,7 +106,6 @@ allOf:
+ - mediatek,mt2701-smi-larb
+ - mediatek,mt2712-smi-larb
+ - mediatek,mt6779-smi-larb
+- - mediatek,mt8167-smi-larb
+ - mediatek,mt8192-smi-larb
+
+ then:
+diff --git a/Documentation/devicetree/bindings/mtd/nand-controller.yaml b/Documentation/devicetree/bindings/mtd/nand-controller.yaml
+index bd217e6f5018a..5cd144a9ec992 100644
+--- a/Documentation/devicetree/bindings/mtd/nand-controller.yaml
++++ b/Documentation/devicetree/bindings/mtd/nand-controller.yaml
+@@ -55,7 +55,7 @@ patternProperties:
+ properties:
+ reg:
+ description:
+- Contains the native Ready/Busy IDs.
++ Contains the chip-select IDs.
+
+ nand-ecc-engine:
+ allOf:
+@@ -184,7 +184,7 @@ examples:
+ nand-use-soft-ecc-engine;
+ nand-ecc-algo = "bch";
+
+- /* controller specific properties */
++ /* NAND chip specific properties */
+ };
+
+ nand@1 {
+diff --git a/Documentation/devicetree/bindings/net/can/tcan4x5x.txt b/Documentation/devicetree/bindings/net/can/tcan4x5x.txt
+index 0968b40aef1e8..e3501bfa22e90 100644
+--- a/Documentation/devicetree/bindings/net/can/tcan4x5x.txt
++++ b/Documentation/devicetree/bindings/net/can/tcan4x5x.txt
+@@ -31,7 +31,7 @@ tcan4x5x: tcan4x5x@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ spi-max-frequency = <10000000>;
+- bosch,mram-cfg = <0x0 0 0 32 0 0 1 1>;
++ bosch,mram-cfg = <0x0 0 0 16 0 0 1 1>;
+ interrupt-parent = <&gpio1>;
+ interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+ device-state-gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>;
+diff --git a/Documentation/devicetree/bindings/net/ethernet-phy.yaml b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
+index 2766fe45bb98b..ee42328a109dc 100644
+--- a/Documentation/devicetree/bindings/net/ethernet-phy.yaml
++++ b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
+@@ -91,6 +91,14 @@ properties:
+ compensate for the board being designed with the lanes
+ swapped.
+
++ enet-phy-lane-no-swap:
++ $ref: /schemas/types.yaml#/definitions/flag
++ description:
++ If set, indicates that PHY will disable swap of the
++ TX/RX lanes. This property allows the PHY to work correcly after
++ e.g. wrong bootstrap configuration caused by issues in PCB
++ layout design.
++
+ eee-broken-100tx:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description:
+diff --git a/Documentation/devicetree/bindings/net/qcom,ipa.yaml b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
+index b8a0b392b24ea..c52ec1ee7df6e 100644
+--- a/Documentation/devicetree/bindings/net/qcom,ipa.yaml
++++ b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
+@@ -106,6 +106,10 @@ properties:
+ - const: imem
+ - const: config
+
++ qcom,qmp:
++ $ref: /schemas/types.yaml#/definitions/phandle
++ description: phandle to the AOSS side-channel message RAM
++
+ qcom,smem-states:
+ $ref: /schemas/types.yaml#/definitions/phandle-array
+ description: State bits used in by the AP to signal the modem.
+@@ -221,6 +225,8 @@ examples:
+ "imem",
+ "config";
+
++ qcom,qmp = <&aoss_qmp>;
++
+ qcom,smem-states = <&ipa_smp2p_out 0>,
+ <&ipa_smp2p_out 1>;
+ qcom,smem-state-names = "ipa-clock-enabled-valid",
+diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
+index c115c95ee584e..5b8db76b6cdd7 100644
+--- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml
++++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
+@@ -53,20 +53,18 @@ properties:
+ - allwinner,sun8i-r40-emac
+ - allwinner,sun8i-v3s-emac
+ - allwinner,sun50i-a64-emac
+- - loongson,ls2k-dwmac
+- - loongson,ls7a-dwmac
+ - amlogic,meson6-dwmac
+ - amlogic,meson8b-dwmac
+ - amlogic,meson8m2-dwmac
+ - amlogic,meson-gxbb-dwmac
+ - amlogic,meson-axg-dwmac
+- - loongson,ls2k-dwmac
+- - loongson,ls7a-dwmac
+ - ingenic,jz4775-mac
+ - ingenic,x1000-mac
+ - ingenic,x1600-mac
+ - ingenic,x1830-mac
+ - ingenic,x2000-mac
++ - loongson,ls2k-dwmac
++ - loongson,ls7a-dwmac
+ - rockchip,px30-gmac
+ - rockchip,rk3128-gmac
+ - rockchip,rk3228-gmac
+diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml
+index ad2866c997383..fcd82df3aebbd 100644
+--- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml
++++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml
+@@ -58,7 +58,7 @@ patternProperties:
+ $ref: "/schemas/types.yaml#/definitions/string"
+ enum: [ ADC0, ADC1, ADC10, ADC11, ADC12, ADC13, ADC14, ADC15, ADC2,
+ ADC3, ADC4, ADC5, ADC6, ADC7, ADC8, ADC9, BMCINT, EMMCG1, EMMCG4,
+- EMMCG8, ESPI, ESPIALT, FSI1, FSI2, FWSPIABR, FWSPID, FWQSPID, FWSPIWP,
++ EMMCG8, ESPI, ESPIALT, FSI1, FSI2, FWSPIABR, FWSPID, FWSPIWP,
+ GPIT0, GPIT1, GPIT2, GPIT3, GPIT4, GPIT5, GPIT6, GPIT7, GPIU0, GPIU1,
+ GPIU2, GPIU3, GPIU4, GPIU5, GPIU6, GPIU7, HVI3C3, HVI3C4, I2C1, I2C10,
+ I2C11, I2C12, I2C13, I2C14, I2C15, I2C16, I2C2, I2C3, I2C4, I2C5,
+diff --git a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml
+index 4fe35e650909c..8dcdd32c2e015 100644
+--- a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml
++++ b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml
+@@ -138,7 +138,7 @@ examples:
+ clocks = <&sys_clk>;
+ pinctrl-0 = <&sgpio2_pins>;
+ pinctrl-names = "default";
+- reg = <0x1101059c 0x100>;
++ reg = <0x1101059c 0x118>;
+ microchip,sgpio-port-ranges = <0 0>, <16 18>, <28 31>;
+ bus-frequency = <25000000>;
+ sgpio_in2: gpio@0 {
+diff --git a/Documentation/devicetree/bindings/regulator/samsung,s5m8767.txt b/Documentation/devicetree/bindings/regulator/samsung,s5m8767.txt
+index 093edda0c8dfc..6cd83d920155f 100644
+--- a/Documentation/devicetree/bindings/regulator/samsung,s5m8767.txt
++++ b/Documentation/devicetree/bindings/regulator/samsung,s5m8767.txt
+@@ -13,6 +13,14 @@ common regulator binding documented in:
+
+
+ Required properties of the main device node (the parent!):
++ - s5m8767,pmic-buck-ds-gpios: GPIO specifiers for three host gpio's used
++ for selecting GPIO DVS lines. It is one-to-one mapped to dvs gpio lines.
++
++ [1] If either of the 's5m8767,pmic-buck[2/3/4]-uses-gpio-dvs' optional
++ property is specified, then all the eight voltage values for the
++ 's5m8767,pmic-buck[2/3/4]-dvs-voltage' should be specified.
++
++Optional properties of the main device node (the parent!):
+ - s5m8767,pmic-buck2-dvs-voltage: A set of 8 voltage values in micro-volt (uV)
+ units for buck2 when changing voltage using gpio dvs. Refer to [1] below
+ for additional information.
+@@ -25,26 +33,13 @@ Required properties of the main device node (the parent!):
+ units for buck4 when changing voltage using gpio dvs. Refer to [1] below
+ for additional information.
+
+- - s5m8767,pmic-buck-ds-gpios: GPIO specifiers for three host gpio's used
+- for selecting GPIO DVS lines. It is one-to-one mapped to dvs gpio lines.
+-
+- [1] If none of the 's5m8767,pmic-buck[2/3/4]-uses-gpio-dvs' optional
+- property is specified, the 's5m8767,pmic-buck[2/3/4]-dvs-voltage'
+- property should specify atleast one voltage level (which would be a
+- safe operating voltage).
+-
+- If either of the 's5m8767,pmic-buck[2/3/4]-uses-gpio-dvs' optional
+- property is specified, then all the eight voltage values for the
+- 's5m8767,pmic-buck[2/3/4]-dvs-voltage' should be specified.
+-
+-Optional properties of the main device node (the parent!):
+ - s5m8767,pmic-buck2-uses-gpio-dvs: 'buck2' can be controlled by gpio dvs.
+ - s5m8767,pmic-buck3-uses-gpio-dvs: 'buck3' can be controlled by gpio dvs.
+ - s5m8767,pmic-buck4-uses-gpio-dvs: 'buck4' can be controlled by gpio dvs.
+
+ Additional properties required if either of the optional properties are used:
+
+- - s5m8767,pmic-buck234-default-dvs-idx: Default voltage setting selected from
++ - s5m8767,pmic-buck-default-dvs-idx: Default voltage setting selected from
+ the possible 8 options selectable by the dvs gpios. The value of this
+ property should be between 0 and 7. If not specified or if out of range, the
+ default value of this property is set to 0.
+diff --git a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml
+index 35a8045b2c70d..53627c6e2ae32 100644
+--- a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml
++++ b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml
+@@ -106,7 +106,7 @@ examples:
+ dma-names = "rx", "tx";
+
+ flash@0 {
+- compatible = "spi-nor";
++ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <104000000>;
+ spi-tx-bus-width = <2>;
+diff --git a/Documentation/devicetree/bindings/spi/spi-mxic.txt b/Documentation/devicetree/bindings/spi/spi-mxic.txt
+index 529f2dab2648a..7bcbb229b78bb 100644
+--- a/Documentation/devicetree/bindings/spi/spi-mxic.txt
++++ b/Documentation/devicetree/bindings/spi/spi-mxic.txt
+@@ -8,11 +8,13 @@ Required properties:
+ - reg: should contain 2 entries, one for the registers and one for the direct
+ mapping area
+ - reg-names: should contain "regs" and "dirmap"
+-- interrupts: interrupt line connected to the SPI controller
+ - clock-names: should contain "ps_clk", "send_clk" and "send_dly_clk"
+ - clocks: should contain 3 entries for the "ps_clk", "send_clk" and
+ "send_dly_clk" clocks
+
++Optional properties:
++- interrupts: interrupt line connected to the SPI controller
++
+ Example:
+
+ spi@43c30000 {
+diff --git a/Documentation/devicetree/bindings/thermal/thermal-zones.yaml b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml
+index a07de5ed0ca6a..2d34f3ccb2572 100644
+--- a/Documentation/devicetree/bindings/thermal/thermal-zones.yaml
++++ b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml
+@@ -199,12 +199,11 @@ patternProperties:
+
+ contribution:
+ $ref: /schemas/types.yaml#/definitions/uint32
+- minimum: 0
+- maximum: 100
+ description:
+- The percentage contribution of the cooling devices at the
+- specific trip temperature referenced in this map
+- to this thermal zone
++ The cooling contribution to the thermal zone of the referred
++ cooling device at the referred trip point. The contribution is
++ a ratio of the sum of all cooling contributions within a
++ thermal zone.
+
+ required:
+ - trip
+diff --git a/Documentation/devicetree/bindings/usb/usb-hcd.yaml b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
+index 56853c17af667..1dc3d5d7b44fe 100644
+--- a/Documentation/devicetree/bindings/usb/usb-hcd.yaml
++++ b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
+@@ -33,7 +33,7 @@ patternProperties:
+ "^.*@[0-9a-f]{1,2}$":
+ description: The hard wired USB devices
+ type: object
+- $ref: /usb/usb-device.yaml
++ $ref: /schemas/usb/usb-device.yaml
+
+ additionalProperties: true
+
+diff --git a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
+index 76cb9586ee00c..93cd77a6e92c0 100644
+--- a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
++++ b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
+@@ -39,8 +39,8 @@ properties:
+ samsung,syscon-phandle:
+ $ref: /schemas/types.yaml#/definitions/phandle
+ description:
+- Phandle to the PMU system controller node (in case of Exynos5250
+- and Exynos5420).
++ Phandle to the PMU system controller node (in case of Exynos5250,
++ Exynos5420 and Exynos7).
+
+ required:
+ - compatible
+@@ -58,6 +58,7 @@ allOf:
+ enum:
+ - samsung,exynos5250-wdt
+ - samsung,exynos5420-wdt
++ - samsung,exynos7-wdt
+ then:
+ required:
+ - samsung,syscon-phandle
+diff --git a/Documentation/driver-api/dmaengine/dmatest.rst b/Documentation/driver-api/dmaengine/dmatest.rst
+index ee268d445d38b..d2e1d8b58e7dc 100644
+--- a/Documentation/driver-api/dmaengine/dmatest.rst
++++ b/Documentation/driver-api/dmaengine/dmatest.rst
+@@ -143,13 +143,14 @@ Part 5 - Handling channel allocation
+ Allocating Channels
+ -------------------
+
+-Channels are required to be configured prior to starting the test run.
+-Attempting to run the test without configuring the channels will fail.
++Channels do not need to be configured prior to starting a test run. Attempting
++to run the test without configuring the channels will result in testing any
++channels that are available.
+
+ Example::
+
+ % echo 1 > /sys/module/dmatest/parameters/run
+- dmatest: Could not start test, no channels configured
++ dmatest: No channels configured, continue with any
+
+ Channels are registered using the "channel" parameter. Channels can be requested by their
+ name, once requested, the channel is registered and a pending thread is added to the test list.
+diff --git a/Documentation/driver-api/firewire.rst b/Documentation/driver-api/firewire.rst
+index 94a2d7f01d999..d3cfa73cbb2b4 100644
+--- a/Documentation/driver-api/firewire.rst
++++ b/Documentation/driver-api/firewire.rst
+@@ -19,7 +19,7 @@ of kernel interfaces is available via exported symbols in `firewire-core` module
+ Firewire char device data structures
+ ====================================
+
+-.. include:: /ABI/stable/firewire-cdev
++.. include:: ../ABI/stable/firewire-cdev
+ :literal:
+
+ .. kernel-doc:: include/uapi/linux/firewire-cdev.h
+@@ -28,7 +28,7 @@ Firewire char device data structures
+ Firewire device probing and sysfs interfaces
+ ============================================
+
+-.. include:: /ABI/stable/sysfs-bus-firewire
++.. include:: ../ABI/stable/sysfs-bus-firewire
+ :literal:
+
+ .. kernel-doc:: drivers/firewire/core-device.c
+diff --git a/Documentation/filesystems/ext4/attributes.rst b/Documentation/filesystems/ext4/attributes.rst
+index 54386a010a8d7..871d2da7a0a91 100644
+--- a/Documentation/filesystems/ext4/attributes.rst
++++ b/Documentation/filesystems/ext4/attributes.rst
+@@ -76,7 +76,7 @@ The beginning of an extended attribute block is in
+ - Checksum of the extended attribute block.
+ * - 0x14
+ - \_\_u32
+- - h\_reserved[2]
++ - h\_reserved[3]
+ - Zero.
+
+ The checksum is calculated against the FS UUID, the 64-bit block number
+diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst
+index 0eb799d9d05a2..7940a45d39522 100644
+--- a/Documentation/filesystems/fscrypt.rst
++++ b/Documentation/filesystems/fscrypt.rst
+@@ -176,11 +176,11 @@ Master Keys
+
+ Each encrypted directory tree is protected by a *master key*. Master
+ keys can be up to 64 bytes long, and must be at least as long as the
+-greater of the key length needed by the contents and filenames
+-encryption modes being used. For example, if AES-256-XTS is used for
+-contents encryption, the master key must be 64 bytes (512 bits). Note
+-that the XTS mode is defined to require a key twice as long as that
+-required by the underlying block cipher.
++greater of the security strength of the contents and filenames
++encryption modes being used. For example, if any AES-256 mode is
++used, the master key must be at least 256 bits, i.e. 32 bytes. A
++stricter requirement applies if the key is used by a v1 encryption
++policy and AES-256-XTS is used; such keys must be 64 bytes.
+
+ To "unlock" an encrypted directory tree, userspace must provide the
+ appropriate master key. There can be any number of master keys, each
+diff --git a/Documentation/firmware-guide/acpi/dsd/data-node-references.rst b/Documentation/firmware-guide/acpi/dsd/data-node-references.rst
+index b7ad47df49de0..8b65b32e6e40e 100644
+--- a/Documentation/firmware-guide/acpi/dsd/data-node-references.rst
++++ b/Documentation/firmware-guide/acpi/dsd/data-node-references.rst
+@@ -5,7 +5,7 @@
+ Referencing hierarchical data nodes
+ ===================================
+
+-:Copyright: |copy| 2018 Intel Corporation
++:Copyright: |copy| 2018, 2021 Intel Corporation
+ :Author: Sakari Ailus <sakari.ailus@linux.intel.com>
+
+ ACPI in general allows referring to device objects in the tree only.
+@@ -52,12 +52,14 @@ the ANOD object which is also the final target node of the reference.
+ Name (NOD0, Package() {
+ ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"),
+ Package () {
++ Package () { "reg", 0 },
+ Package () { "random-property", 3 },
+ }
+ })
+ Name (NOD1, Package() {
+ ToUUID("dbb8e3e6-5886-4ba6-8795-1319f52a966b"),
+ Package () {
++ Package () { "reg", 1 },
+ Package () { "anothernode", "ANOD" },
+ }
+ })
+@@ -74,7 +76,11 @@ the ANOD object which is also the final target node of the reference.
+ Name (_DSD, Package () {
+ ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"),
+ Package () {
+- Package () { "reference", ^DEV0, "node@1", "anothernode" },
++ Package () {
++ "reference", Package () {
++ ^DEV0, "node@1", "anothernode"
++ }
++ },
+ }
+ })
+ }
+diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
+index 204ebdaadb45a..03021dfa0dd81 100644
+--- a/Documentation/gpu/i915.rst
++++ b/Documentation/gpu/i915.rst
+@@ -183,25 +183,25 @@ Frame Buffer Compression (FBC)
+ Display Refresh Rate Switching (DRRS)
+ -------------------------------------
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :doc: Display Refresh Rate Switching (DRRS)
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_dp_set_drrs_state
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_edp_drrs_enable
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_edp_drrs_disable
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_edp_drrs_invalidate
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_edp_drrs_flush
+
+-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp.c
++.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+ :functions: intel_dp_drrs_init
+
+ DPIO
+diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
+index 12e61869939e8..67de1e94fdf76 100644
+--- a/Documentation/gpu/todo.rst
++++ b/Documentation/gpu/todo.rst
+@@ -311,27 +311,6 @@ Contact: Daniel Vetter, Noralf Tronnes
+
+ Level: Advanced
+
+-Garbage collect fbdev scrolling acceleration
+---------------------------------------------
+-
+-Scroll acceleration is disabled in fbcon by hard-wiring p->scrollmode =
+-SCROLL_REDRAW. There's a ton of code this will allow us to remove:
+-
+-- lots of code in fbcon.c
+-
+-- a bunch of the hooks in fbcon_ops, maybe the remaining hooks could be called
+- directly instead of the function table (with a switch on p->rotate)
+-
+-- fb_copyarea is unused after this, and can be deleted from all drivers
+-
+-Note that not all acceleration code can be deleted, since clearing and cursor
+-support is still accelerated, which might be good candidates for further
+-deletion projects.
+-
+-Contact: Daniel Vetter
+-
+-Level: Intermediate
+-
+ idr_init_base()
+ ---------------
+
+diff --git a/Documentation/hwmon/lm90.rst b/Documentation/hwmon/lm90.rst
+index 3da8c6e06a365..05391fb4042d9 100644
+--- a/Documentation/hwmon/lm90.rst
++++ b/Documentation/hwmon/lm90.rst
+@@ -265,6 +265,16 @@ Supported chips:
+
+ https://www.ti.com/litv/pdf/sbos686
+
++ * Texas Instruments TMP461
++
++ Prefix: 'tmp461'
++
++ Addresses scanned: I2C 0x48 through 0x4F
++
++ Datasheet: Publicly available at TI website
++
++ https://www.ti.com/lit/gpn/tmp461
++
+ Author: Jean Delvare <jdelvare@suse.de>
+
+
+diff --git a/Documentation/locking/locktypes.rst b/Documentation/locking/locktypes.rst
+index ddada4a537493..4fd7b70fcde19 100644
+--- a/Documentation/locking/locktypes.rst
++++ b/Documentation/locking/locktypes.rst
+@@ -439,11 +439,9 @@ preemption. The following substitution works on both kernels::
+ spin_lock(&p->lock);
+ p->count += this_cpu_read(var2);
+
+-On a non-PREEMPT_RT kernel migrate_disable() maps to preempt_disable()
+-which makes the above code fully equivalent. On a PREEMPT_RT kernel
+ migrate_disable() ensures that the task is pinned on the current CPU which
+ in turn guarantees that the per-CPU access to var1 and var2 are staying on
+-the same CPU.
++the same CPU while the task remains preemptible.
+
+ The migrate_disable() substitution is not valid for the following
+ scenario::
+@@ -456,9 +454,8 @@ scenario::
+ p = this_cpu_ptr(&var1);
+ p->val = func2();
+
+-While correct on a non-PREEMPT_RT kernel, this breaks on PREEMPT_RT because
+-here migrate_disable() does not protect against reentrancy from a
+-preempting task. A correct substitution for this case is::
++This breaks because migrate_disable() does not protect against reentrancy from
++a preempting task. A correct substitution for this case is::
+
+ func()
+ {
+diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst
+index 31cfd7d674a6c..c0a789b008063 100644
+--- a/Documentation/networking/bonding.rst
++++ b/Documentation/networking/bonding.rst
+@@ -196,11 +196,12 @@ ad_actor_sys_prio
+ ad_actor_system
+
+ In an AD system, this specifies the mac-address for the actor in
+- protocol packet exchanges (LACPDUs). The value cannot be NULL or
+- multicast. It is preferred to have the local-admin bit set for this
+- mac but driver does not enforce it. If the value is not given then
+- system defaults to using the masters' mac address as actors' system
+- address.
++ protocol packet exchanges (LACPDUs). The value cannot be a multicast
++ address. If the all-zeroes MAC is specified, bonding will internally
++ use the MAC of the bond itself. It is preferred to have the
++ local-admin bit set for this mac but driver does not enforce it. If
++ the value is not given then system defaults to using the masters'
++ mac address as actors' system address.
+
+ This parameter has effect only in 802.3ad mode and is available through
+ SysFs interface.
+diff --git a/Documentation/networking/device_drivers/ethernet/intel/ixgbe.rst b/Documentation/networking/device_drivers/ethernet/intel/ixgbe.rst
+index f1d5233e5e510..0a233b17c664e 100644
+--- a/Documentation/networking/device_drivers/ethernet/intel/ixgbe.rst
++++ b/Documentation/networking/device_drivers/ethernet/intel/ixgbe.rst
+@@ -440,6 +440,22 @@ NOTE: For 82599-based network connections, if you are enabling jumbo frames in
+ a virtual function (VF), jumbo frames must first be enabled in the physical
+ function (PF). The VF MTU setting cannot be larger than the PF MTU.
+
++NBASE-T Support
++---------------
++The ixgbe driver supports NBASE-T on some devices. However, the advertisement
++of NBASE-T speeds is suppressed by default, to accommodate broken network
++switches which cannot cope with advertised NBASE-T speeds. Use the ethtool
++command to enable advertising NBASE-T speeds on devices which support it::
++
++ ethtool -s eth? advertise 0x1800000001028
++
++On Linux systems with INTERFACES(5), this can be specified as a pre-up command
++in /etc/network/interfaces so that the interface is always brought up with
++NBASE-T support, e.g.::
++
++ iface eth? inet dhcp
++ pre-up ethtool -s eth? advertise 0x1800000001028 || true
++
+ Generic Receive Offload, aka GRO
+ --------------------------------
+ The driver supports the in-kernel software implementation of GRO. GRO has
+diff --git a/Documentation/networking/ipvs-sysctl.rst b/Documentation/networking/ipvs-sysctl.rst
+index 2afccc63856ee..1cfbf1add2fc9 100644
+--- a/Documentation/networking/ipvs-sysctl.rst
++++ b/Documentation/networking/ipvs-sysctl.rst
+@@ -37,8 +37,7 @@ conn_reuse_mode - INTEGER
+
+ 0: disable any special handling on port reuse. The new
+ connection will be delivered to the same real server that was
+- servicing the previous connection. This will effectively
+- disable expire_nodest_conn.
++ servicing the previous connection.
+
+ bit 1: enable rescheduling of new connections when it is safe.
+ That is, whenever expire_nodest_conn and for TCP sockets, when
+diff --git a/Documentation/process/stable-kernel-rules.rst b/Documentation/process/stable-kernel-rules.rst
+index 003c865e9c212..fbcb48bc2a903 100644
+--- a/Documentation/process/stable-kernel-rules.rst
++++ b/Documentation/process/stable-kernel-rules.rst
+@@ -168,7 +168,16 @@ Trees
+ - The finalized and tagged releases of all stable kernels can be found
+ in separate branches per version at:
+
+- https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
++ https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
++
++ - The release candidate of all stable kernel versions can be found at:
++
++ https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git/
++
++ .. warning::
++ The -stable-rc tree is a snapshot in time of the stable-queue tree and
++ will change frequently, hence will be rebased often. It should only be
++ used for testing purposes (e.g. to be consumed by CI systems).
+
+
+ Review committee
+diff --git a/Documentation/sound/hd-audio/models.rst b/Documentation/sound/hd-audio/models.rst
+index 0ea967d345838..9b52f50a68542 100644
+--- a/Documentation/sound/hd-audio/models.rst
++++ b/Documentation/sound/hd-audio/models.rst
+@@ -261,6 +261,10 @@ alc-sense-combo
+ huawei-mbx-stereo
+ Enable initialization verbs for Huawei MBX stereo speakers;
+ might be risky, try this at your own risk
++alc298-samsung-headphone
++ Samsung laptops with ALC298
++alc256-samsung-headphone
++ Samsung laptops with ALC256
+
+ ALC66x/67x/892
+ ==============
+@@ -326,6 +330,8 @@ usi-headset
+ Headset support on USI machines
+ dual-codecs
+ Lenovo laptops with dual codecs
++alc285-hp-amp-init
++ HP laptops which require speaker amplifier initialization (ALC285)
+
+ ALC680
+ ======
+diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt
+index 9a35f50798a65..2c573541ab712 100644
+--- a/Documentation/sphinx/requirements.txt
++++ b/Documentation/sphinx/requirements.txt
+@@ -1,2 +1,4 @@
++# jinja2>=3.1 is not compatible with Sphinx<4.0
++jinja2<3.1
+ sphinx_rtd_theme
+ Sphinx==2.4.4
+diff --git a/Documentation/trace/coresight/coresight-config.rst b/Documentation/trace/coresight/coresight-config.rst
+index a4e3ef2952401..6ed13398ca2ce 100644
+--- a/Documentation/trace/coresight/coresight-config.rst
++++ b/Documentation/trace/coresight/coresight-config.rst
+@@ -211,19 +211,13 @@ also declared in the perf 'cs_etm' event infrastructure so that they can
+ be selected when running trace under perf::
+
+ $ ls /sys/devices/cs_etm
+- configurations format perf_event_mux_interval_ms sinks type
+- events nr_addr_filters power
++ cpu0 cpu2 events nr_addr_filters power subsystem uevent
++ cpu1 cpu3 format perf_event_mux_interval_ms sinks type
+
+-Key directories here are 'configurations' - which lists the loaded
+-configurations, and 'events' - a generic perf directory which allows
+-selection on the perf command line.::
++The key directory here is 'events' - a generic perf directory which allows
++selection on the perf command line. As with the sinks entries, this provides
++a hash of the configuration name.
+
+- $ ls configurations/
+- autofdo
+- $ cat configurations/autofdo
+- 0xa7c3dddd
+-
+-As with the sinks entries, this provides a hash of the configuration name.
+ The entry in the 'events' directory uses perfs built in syntax generator
+ to substitute the syntax for the name when evaluating the command::
+
+diff --git a/Documentation/trace/events.rst b/Documentation/trace/events.rst
+index 8ddb9b09451c8..c47f381d0c002 100644
+--- a/Documentation/trace/events.rst
++++ b/Documentation/trace/events.rst
+@@ -198,6 +198,15 @@ The glob (~) accepts a wild card character (\*,?) and character classes
+ prev_comm ~ "*sh*"
+ prev_comm ~ "ba*sh"
+
++If the field is a pointer that points into user space (for example
++"filename" from sys_enter_openat), then you have to append ".ustring" to the
++field name::
++
++ filename.ustring ~ "password"
++
++As the kernel will have to know how to retrieve the memory that the pointer
++is at from user space.
++
+ 5.2 Setting filters
+ -------------------
+
+@@ -230,6 +239,16 @@ Currently the caret ('^') for an error always appears at the beginning of
+ the filter string; the error message should still be useful though
+ even without more accurate position info.
+
++5.2.1 Filter limitations
++------------------------
++
++If a filter is placed on a string pointer ``(char *)`` that does not point
++to a string on the ring buffer, but instead points to kernel or user space
++memory, then, for safety reasons, at most 1024 bytes of the content is
++copied onto a temporary buffer to do the compare. If the copy of the memory
++faults (the pointer points to memory that should not be accessed), then the
++string compare will be treated as not matching.
++
+ 5.3 Clearing filters
+ --------------------
+
+diff --git a/MAINTAINERS b/MAINTAINERS
+index 3b79fd441dde8..942e0b173f2cc 100644
+--- a/MAINTAINERS
++++ b/MAINTAINERS
+@@ -7024,7 +7024,6 @@ F: drivers/net/mdio/fwnode_mdio.c
+ F: drivers/net/mdio/of_mdio.c
+ F: drivers/net/pcs/
+ F: drivers/net/phy/
+-F: drivers/of/of_net.c
+ F: include/dt-bindings/net/qca-ar803x.h
+ F: include/linux/*mdio*.h
+ F: include/linux/mdio/*.h
+@@ -7036,6 +7035,7 @@ F: include/linux/platform_data/mdio-gpio.h
+ F: include/trace/events/mdio.h
+ F: include/uapi/linux/mdio.h
+ F: include/uapi/linux/mii.h
++F: net/core/of_net.c
+
+ EXFAT FILE SYSTEM
+ M: Namjae Jeon <linkinjeon@kernel.org>
+@@ -15720,6 +15720,8 @@ F: arch/mips/generic/board-ranchu.c
+
+ RANDOM NUMBER DRIVER
+ M: "Theodore Ts'o" <tytso@mit.edu>
++M: Jason A. Donenfeld <Jason@zx2c4.com>
++T: git https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git
+ S: Maintained
+ F: drivers/char/random.c
+
+diff --git a/Makefile b/Makefile
+index ed6e7ec60eff6..b8ce2ba174862 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,7 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0
+ VERSION = 5
+ PATCHLEVEL = 15
+-SUBLEVEL = 0
++SUBLEVEL = 44
+ EXTRAVERSION =
+ NAME = Trick or Treat
+
+diff --git a/arch/Kconfig b/arch/Kconfig
+index 8df1c71026435..191589f26b1a8 100644
+--- a/arch/Kconfig
++++ b/arch/Kconfig
+@@ -1141,6 +1141,7 @@ config HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET
+ config RANDOMIZE_KSTACK_OFFSET_DEFAULT
+ bool "Randomize kernel stack offset on syscall entry"
+ depends on HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET
++ depends on INIT_STACK_NONE || !CC_IS_CLANG || CLANG_VERSION >= 140000
+ help
+ The kernel stack offset can be randomized (after pt_regs) by
+ roughly 5 bits of entropy, frustrating memory corruption
+@@ -1234,6 +1235,9 @@ config RELR
+ config ARCH_HAS_MEM_ENCRYPT
+ bool
+
++config ARCH_HAS_CC_PLATFORM
++ bool
++
+ config HAVE_SPARSE_SYSCALL_NR
+ bool
+ help
+diff --git a/arch/alpha/include/asm/timex.h b/arch/alpha/include/asm/timex.h
+index b565cc6f408e9..f89798da8a147 100644
+--- a/arch/alpha/include/asm/timex.h
++++ b/arch/alpha/include/asm/timex.h
+@@ -28,5 +28,6 @@ static inline cycles_t get_cycles (void)
+ __asm__ __volatile__ ("rpcc %0" : "=r"(ret));
+ return ret;
+ }
++#define get_cycles get_cycles
+
+ #endif
+diff --git a/arch/alpha/kernel/rtc.c b/arch/alpha/kernel/rtc.c
+index ce3077946e1d9..fb3025396ac96 100644
+--- a/arch/alpha/kernel/rtc.c
++++ b/arch/alpha/kernel/rtc.c
+@@ -80,7 +80,12 @@ init_rtc_epoch(void)
+ static int
+ alpha_rtc_read_time(struct device *dev, struct rtc_time *tm)
+ {
+- mc146818_get_time(tm);
++ int ret = mc146818_get_time(tm);
++
++ if (ret < 0) {
++ dev_err_ratelimited(dev, "unable to read current time\n");
++ return ret;
++ }
+
+ /* Adjust for non-default epochs. It's easier to depend on the
+ generic __get_rtc_time and adjust the epoch here than create
+diff --git a/arch/arc/kernel/entry.S b/arch/arc/kernel/entry.S
+index dd77a0c8f740b..66ba549b520fc 100644
+--- a/arch/arc/kernel/entry.S
++++ b/arch/arc/kernel/entry.S
+@@ -196,6 +196,7 @@ tracesys_exit:
+ st r0, [sp, PT_r0] ; sys call return value in pt_regs
+
+ ;POST Sys Call Ptrace Hook
++ mov r0, sp ; pt_regs needed
+ bl @syscall_trace_exit
+ b ret_from_exception ; NOT ret_from_system_call at is saves r0 which
+ ; we'd done before calling post hook above
+diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c
+index 3793876f42d9b..5f7f5aab361f1 100644
+--- a/arch/arc/kernel/process.c
++++ b/arch/arc/kernel/process.c
+@@ -43,7 +43,7 @@ SYSCALL_DEFINE0(arc_gettls)
+ return task_thread_info(current)->thr_ptr;
+ }
+
+-SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new)
++SYSCALL_DEFINE3(arc_usr_cmpxchg, int __user *, uaddr, int, expected, int, new)
+ {
+ struct pt_regs *regs = current_pt_regs();
+ u32 uval;
+@@ -294,7 +294,7 @@ int elf_check_arch(const struct elf32_hdr *x)
+ eflags = x->e_flags;
+ if ((eflags & EF_ARC_OSABI_MSK) != EF_ARC_OSABI_CURRENT) {
+ pr_err("ABI mismatch - you need newer toolchain\n");
+- force_sigsegv(SIGSEGV);
++ force_fatal_sig(SIGSEGV);
+ return 0;
+ }
+
+diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
+index dcf2df6da98f0..4ebd512043be5 100644
+--- a/arch/arm/Kconfig
++++ b/arch/arm/Kconfig
+@@ -1455,6 +1455,7 @@ config HIGHMEM
+ bool "High Memory Support"
+ depends on MMU
+ select KMAP_LOCAL
++ select KMAP_LOCAL_NON_LINEAR_PTE_ARRAY
+ help
+ The address space of ARM processors is only 4 Gigabytes large
+ and it has to accommodate user address space, kernel address
+diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
+index 98436702e0c7e..644875d73ba15 100644
+--- a/arch/arm/Kconfig.debug
++++ b/arch/arm/Kconfig.debug
+@@ -410,12 +410,12 @@ choice
+ Say Y here if you want kernel low-level debugging support
+ on i.MX25.
+
+- config DEBUG_IMX21_IMX27_UART
+- bool "i.MX21 and i.MX27 Debug UART"
+- depends on SOC_IMX21 || SOC_IMX27
++ config DEBUG_IMX27_UART
++ bool "i.MX27 Debug UART"
++ depends on SOC_IMX27
+ help
+ Say Y here if you want kernel low-level debugging support
+- on i.MX21 or i.MX27.
++ on i.MX27.
+
+ config DEBUG_IMX28_UART
+ bool "i.MX28 Debug UART"
+@@ -1481,7 +1481,7 @@ config DEBUG_IMX_UART_PORT
+ int "i.MX Debug UART Port Selection"
+ depends on DEBUG_IMX1_UART || \
+ DEBUG_IMX25_UART || \
+- DEBUG_IMX21_IMX27_UART || \
++ DEBUG_IMX27_UART || \
+ DEBUG_IMX31_UART || \
+ DEBUG_IMX35_UART || \
+ DEBUG_IMX50_UART || \
+@@ -1540,12 +1540,12 @@ config DEBUG_LL_INCLUDE
+ default "debug/icedcc.S" if DEBUG_ICEDCC
+ default "debug/imx.S" if DEBUG_IMX1_UART || \
+ DEBUG_IMX25_UART || \
+- DEBUG_IMX21_IMX27_UART || \
++ DEBUG_IMX27_UART || \
+ DEBUG_IMX31_UART || \
+ DEBUG_IMX35_UART || \
+ DEBUG_IMX50_UART || \
+ DEBUG_IMX51_UART || \
+- DEBUG_IMX53_UART ||\
++ DEBUG_IMX53_UART || \
+ DEBUG_IMX6Q_UART || \
+ DEBUG_IMX6SL_UART || \
+ DEBUG_IMX6SX_UART || \
+diff --git a/arch/arm/Makefile b/arch/arm/Makefile
+index 847c31e7c3687..fa45837b8065c 100644
+--- a/arch/arm/Makefile
++++ b/arch/arm/Makefile
+@@ -60,15 +60,15 @@ KBUILD_CFLAGS += $(call cc-option,-fno-ipa-sra)
+ # Note that GCC does not numerically define an architecture version
+ # macro, but instead defines a whole series of macros which makes
+ # testing for a specific architecture or later rather impossible.
+-arch-$(CONFIG_CPU_32v7M) =-D__LINUX_ARM_ARCH__=7 -march=armv7-m -Wa,-march=armv7-m
+-arch-$(CONFIG_CPU_32v7) =-D__LINUX_ARM_ARCH__=7 $(call cc-option,-march=armv7-a,-march=armv5t -Wa$(comma)-march=armv7-a)
+-arch-$(CONFIG_CPU_32v6) =-D__LINUX_ARM_ARCH__=6 $(call cc-option,-march=armv6,-march=armv5t -Wa$(comma)-march=armv6)
++arch-$(CONFIG_CPU_32v7M) =-D__LINUX_ARM_ARCH__=7 -march=armv7-m
++arch-$(CONFIG_CPU_32v7) =-D__LINUX_ARM_ARCH__=7 -march=armv7-a
++arch-$(CONFIG_CPU_32v6) =-D__LINUX_ARM_ARCH__=6 -march=armv6
+ # Only override the compiler option if ARMv6. The ARMv6K extensions are
+ # always available in ARMv7
+ ifeq ($(CONFIG_CPU_32v6),y)
+-arch-$(CONFIG_CPU_32v6K) =-D__LINUX_ARM_ARCH__=6 $(call cc-option,-march=armv6k,-march=armv5t -Wa$(comma)-march=armv6k)
++arch-$(CONFIG_CPU_32v6K) =-D__LINUX_ARM_ARCH__=6 -march=armv6k
+ endif
+-arch-$(CONFIG_CPU_32v5) =-D__LINUX_ARM_ARCH__=5 $(call cc-option,-march=armv5te,-march=armv4t)
++arch-$(CONFIG_CPU_32v5) =-D__LINUX_ARM_ARCH__=5 -march=armv5te
+ arch-$(CONFIG_CPU_32v4T) =-D__LINUX_ARM_ARCH__=4 -march=armv4t
+ arch-$(CONFIG_CPU_32v4) =-D__LINUX_ARM_ARCH__=4 -march=armv4
+ arch-$(CONFIG_CPU_32v3) =-D__LINUX_ARM_ARCH__=3 -march=armv3m
+@@ -82,7 +82,7 @@ tune-$(CONFIG_CPU_ARM720T) =-mtune=arm7tdmi
+ tune-$(CONFIG_CPU_ARM740T) =-mtune=arm7tdmi
+ tune-$(CONFIG_CPU_ARM9TDMI) =-mtune=arm9tdmi
+ tune-$(CONFIG_CPU_ARM940T) =-mtune=arm9tdmi
+-tune-$(CONFIG_CPU_ARM946E) =$(call cc-option,-mtune=arm9e,-mtune=arm9tdmi)
++tune-$(CONFIG_CPU_ARM946E) =-mtune=arm9e
+ tune-$(CONFIG_CPU_ARM920T) =-mtune=arm9tdmi
+ tune-$(CONFIG_CPU_ARM922T) =-mtune=arm9tdmi
+ tune-$(CONFIG_CPU_ARM925T) =-mtune=arm9tdmi
+@@ -90,11 +90,11 @@ tune-$(CONFIG_CPU_ARM926T) =-mtune=arm9tdmi
+ tune-$(CONFIG_CPU_FA526) =-mtune=arm9tdmi
+ tune-$(CONFIG_CPU_SA110) =-mtune=strongarm110
+ tune-$(CONFIG_CPU_SA1100) =-mtune=strongarm1100
+-tune-$(CONFIG_CPU_XSCALE) =$(call cc-option,-mtune=xscale,-mtune=strongarm110) -Wa,-mcpu=xscale
+-tune-$(CONFIG_CPU_XSC3) =$(call cc-option,-mtune=xscale,-mtune=strongarm110) -Wa,-mcpu=xscale
+-tune-$(CONFIG_CPU_FEROCEON) =$(call cc-option,-mtune=marvell-f,-mtune=xscale)
+-tune-$(CONFIG_CPU_V6) =$(call cc-option,-mtune=arm1136j-s,-mtune=strongarm)
+-tune-$(CONFIG_CPU_V6K) =$(call cc-option,-mtune=arm1136j-s,-mtune=strongarm)
++tune-$(CONFIG_CPU_XSCALE) =-mtune=xscale
++tune-$(CONFIG_CPU_XSC3) =-mtune=xscale
++tune-$(CONFIG_CPU_FEROCEON) =-mtune=xscale
++tune-$(CONFIG_CPU_V6) =-mtune=arm1136j-s
++tune-$(CONFIG_CPU_V6K) =-mtune=arm1136j-s
+
+ # Evaluate tune cc-option calls now
+ tune-y := $(tune-y)
+diff --git a/arch/arm/boot/compressed/efi-header.S b/arch/arm/boot/compressed/efi-header.S
+index c0e7a745103e2..230030c130853 100644
+--- a/arch/arm/boot/compressed/efi-header.S
++++ b/arch/arm/boot/compressed/efi-header.S
+@@ -9,16 +9,22 @@
+ #include <linux/sizes.h>
+
+ .macro __nop
+-#ifdef CONFIG_EFI_STUB
+- @ This is almost but not quite a NOP, since it does clobber the
+- @ condition flags. But it is the best we can do for EFI, since
+- @ PE/COFF expects the magic string "MZ" at offset 0, while the
+- @ ARM/Linux boot protocol expects an executable instruction
+- @ there.
+- .inst MZ_MAGIC | (0x1310 << 16) @ tstne r0, #0x4d000
+-#else
+ AR_CLASS( mov r0, r0 )
+ M_CLASS( nop.w )
++ .endm
++
++ .macro __initial_nops
++#ifdef CONFIG_EFI_STUB
++ @ This is a two-instruction NOP, which happens to bear the
++ @ PE/COFF signature "MZ" in the first two bytes, so the kernel
++ @ is accepted as an EFI binary. Booting via the UEFI stub
++ @ will not execute those instructions, but the ARM/Linux
++ @ boot protocol does, so we need some NOPs here.
++ .inst MZ_MAGIC | (0xe225 << 16) @ eor r5, r5, 0x4d000
++ eor r5, r5, 0x4d000 @ undo previous insn
++#else
++ __nop
++ __nop
+ #endif
+ .endm
+
+diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
+index b1cb1972361b8..bf79f2f78d232 100644
+--- a/arch/arm/boot/compressed/head.S
++++ b/arch/arm/boot/compressed/head.S
+@@ -203,7 +203,8 @@ start:
+ * were patching the initial instructions of the kernel, i.e
+ * had started to exploit this "patch area".
+ */
+- .rept 7
++ __initial_nops
++ .rept 5
+ __nop
+ .endr
+ #ifndef CONFIG_THUMB2_KERNEL
+diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
+index 7e0934180724d..27ca1ca6e827c 100644
+--- a/arch/arm/boot/dts/Makefile
++++ b/arch/arm/boot/dts/Makefile
+@@ -779,6 +779,7 @@ dtb-$(CONFIG_ARCH_OMAP3) += \
+ logicpd-som-lv-37xx-devkit.dtb \
+ omap3430-sdp.dtb \
+ omap3-beagle.dtb \
++ omap3-beagle-ab4.dtb \
+ omap3-beagle-xm.dtb \
+ omap3-beagle-xm-ab.dtb \
+ omap3-cm-t3517.dtb \
+diff --git a/arch/arm/boot/dts/am3517-evm.dts b/arch/arm/boot/dts/am3517-evm.dts
+index 0d2fac98ce7d2..c8b80f156ec98 100644
+--- a/arch/arm/boot/dts/am3517-evm.dts
++++ b/arch/arm/boot/dts/am3517-evm.dts
+@@ -161,6 +161,8 @@
+
+ /* HS USB Host PHY on PORT 1 */
+ hsusb1_phy: hsusb1_phy {
++ pinctrl-names = "default";
++ pinctrl-0 = <&hsusb1_rst_pins>;
+ compatible = "usb-nop-xceiv";
+ reset-gpios = <&gpio2 25 GPIO_ACTIVE_LOW>; /* gpio_57 */
+ #phy-cells = <0>;
+@@ -168,7 +170,9 @@
+ };
+
+ &davinci_emac {
+- status = "okay";
++ pinctrl-names = "default";
++ pinctrl-0 = <&ethernet_pins>;
++ status = "okay";
+ };
+
+ &davinci_mdio {
+@@ -193,6 +197,8 @@
+ };
+
+ &i2c2 {
++ pinctrl-names = "default";
++ pinctrl-0 = <&i2c2_pins>;
+ clock-frequency = <400000>;
+ /* User DIP swithes [1:8] / User LEDS [1:2] */
+ tca6416: gpio@21 {
+@@ -205,6 +211,8 @@
+ };
+
+ &i2c3 {
++ pinctrl-names = "default";
++ pinctrl-0 = <&i2c3_pins>;
+ clock-frequency = <400000>;
+ };
+
+@@ -223,6 +231,8 @@
+ };
+
+ &usbhshost {
++ pinctrl-names = "default";
++ pinctrl-0 = <&hsusb1_pins>;
+ port1-mode = "ehci-phy";
+ };
+
+@@ -231,8 +241,35 @@
+ };
+
+ &omap3_pmx_core {
+- pinctrl-names = "default";
+- pinctrl-0 = <&hsusb1_rst_pins>;
++
++ ethernet_pins: pinmux_ethernet_pins {
++ pinctrl-single,pins = <
++ OMAP3_CORE1_IOPAD(0x21fe, PIN_INPUT | MUX_MODE0) /* rmii_mdio_data */
++ OMAP3_CORE1_IOPAD(0x2200, MUX_MODE0) /* rmii_mdio_clk */
++ OMAP3_CORE1_IOPAD(0x2202, PIN_INPUT_PULLDOWN | MUX_MODE0) /* rmii_rxd0 */
++ OMAP3_CORE1_IOPAD(0x2204, PIN_INPUT_PULLDOWN | MUX_MODE0) /* rmii_rxd1 */
++ OMAP3_CORE1_IOPAD(0x2206, PIN_INPUT_PULLDOWN | MUX_MODE0) /* rmii_crs_dv */
++ OMAP3_CORE1_IOPAD(0x2208, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* rmii_rxer */
++ OMAP3_CORE1_IOPAD(0x220a, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* rmii_txd0 */
++ OMAP3_CORE1_IOPAD(0x220c, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* rmii_txd1 */
++ OMAP3_CORE1_IOPAD(0x220e, PIN_OUTPUT_PULLDOWN |MUX_MODE0) /* rmii_txen */
++ OMAP3_CORE1_IOPAD(0x2210, PIN_INPUT_PULLDOWN | MUX_MODE0) /* rmii_50mhz_clk */
++ >;
++ };
++
++ i2c2_pins: pinmux_i2c2_pins {
++ pinctrl-single,pins = <
++ OMAP3_CORE1_IOPAD(0x21be, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c2_scl */
++ OMAP3_CORE1_IOPAD(0x21c0, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c2_sda */
++ >;
++ };
++
++ i2c3_pins: pinmux_i2c3_pins {
++ pinctrl-single,pins = <
++ OMAP3_CORE1_IOPAD(0x21c2, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c3_scl */
++ OMAP3_CORE1_IOPAD(0x21c4, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c3_sda */
++ >;
++ };
+
+ leds_pins: pinmux_leds_pins {
+ pinctrl-single,pins = <
+@@ -300,8 +337,6 @@
+ };
+
+ &omap3_pmx_core2 {
+- pinctrl-names = "default";
+- pinctrl-0 = <&hsusb1_pins>;
+
+ hsusb1_pins: pinmux_hsusb1_pins {
+ pinctrl-single,pins = <
+diff --git a/arch/arm/boot/dts/am3517-som.dtsi b/arch/arm/boot/dts/am3517-som.dtsi
+index 8b669e2eafec4..f7b680f6c48ad 100644
+--- a/arch/arm/boot/dts/am3517-som.dtsi
++++ b/arch/arm/boot/dts/am3517-som.dtsi
+@@ -69,6 +69,8 @@
+ };
+
+ &i2c1 {
++ pinctrl-names = "default";
++ pinctrl-0 = <&i2c1_pins>;
+ clock-frequency = <400000>;
+
+ s35390a: s35390a@30 {
+@@ -179,6 +181,13 @@
+
+ &omap3_pmx_core {
+
++ i2c1_pins: pinmux_i2c1_pins {
++ pinctrl-single,pins = <
++ OMAP3_CORE1_IOPAD(0x21ba, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c1_scl */
++ OMAP3_CORE1_IOPAD(0x21bc, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c1_sda */
++ >;
++ };
++
+ wl12xx_buffer_pins: pinmux_wl12xx_buffer_pins {
+ pinctrl-single,pins = <
+ OMAP3_CORE1_IOPAD(0x2156, PIN_OUTPUT | MUX_MODE4) /* mmc1_dat7.gpio_129 */
+diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi
+index 9b1a24cc5e91f..df3c8d1d8f641 100644
+--- a/arch/arm/boot/dts/armada-38x.dtsi
++++ b/arch/arm/boot/dts/armada-38x.dtsi
+@@ -168,7 +168,7 @@
+ };
+
+ uart0: serial@12000 {
+- compatible = "marvell,armada-38x-uart";
++ compatible = "marvell,armada-38x-uart", "ns16550a";
+ reg = <0x12000 0x100>;
+ reg-shift = <2>;
+ interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+@@ -178,7 +178,7 @@
+ };
+
+ uart1: serial@12100 {
+- compatible = "marvell,armada-38x-uart";
++ compatible = "marvell,armada-38x-uart", "ns16550a";
+ reg = <0x12100 0x100>;
+ reg-shift = <2>;
+ interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+diff --git a/arch/arm/boot/dts/aspeed-bmc-ibm-everest.dts b/arch/arm/boot/dts/aspeed-bmc-ibm-everest.dts
+index 2efd70666738c..af7ea7cab8cfa 100644
+--- a/arch/arm/boot/dts/aspeed-bmc-ibm-everest.dts
++++ b/arch/arm/boot/dts/aspeed-bmc-ibm-everest.dts
+@@ -231,6 +231,21 @@
+ gpios = <&gpio0 ASPEED_GPIO(P, 4) GPIO_ACTIVE_LOW>;
+ };
+ };
++
++ iio-hwmon {
++ compatible = "iio-hwmon";
++ io-channels = <&adc1 7>;
++ };
++};
++
++&adc1 {
++ status = "okay";
++ aspeed,int-vref-microvolt = <2500000>;
++ pinctrl-names = "default";
++ pinctrl-0 = <&pinctrl_adc8_default &pinctrl_adc9_default
++ &pinctrl_adc10_default &pinctrl_adc11_default
++ &pinctrl_adc12_default &pinctrl_adc13_default
++ &pinctrl_adc14_default &pinctrl_adc15_default>;
+ };
+
+ &gpio0 {
+diff --git a/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts b/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts
+index 6419c9762c0b6..6c9f34396a3ae 100644
+--- a/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts
++++ b/arch/arm/boot/dts/aspeed-bmc-ibm-rainier.dts
+@@ -246,6 +246,21 @@
+ linux,code = <11>;
+ };
+ };
++
++ iio-hwmon {
++ compatible = "iio-hwmon";
++ io-channels = <&adc1 7>;
++ };
++};
++
++&adc1 {
++ status = "okay";
++ aspeed,int-vref-microvolt = <2500000>;
++ pinctrl-names = "default";
++ pinctrl-0 = <&pinctrl_adc8_default &pinctrl_adc9_default
++ &pinctrl_adc10_default &pinctrl_adc11_default
++ &pinctrl_adc12_default &pinctrl_adc13_default
++ &pinctrl_adc14_default &pinctrl_adc15_default>;
+ };
+
+ &ehci1 {
+diff --git a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi
+index 6dde51c2aed3f..ac07c240419a2 100644
+--- a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi
++++ b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi
+@@ -117,11 +117,6 @@
+ groups = "FWSPID";
+ };
+
+- pinctrl_fwqspid_default: fwqspid_default {
+- function = "FWQSPID";
+- groups = "FWQSPID";
+- };
+-
+ pinctrl_fwspiwp_default: fwspiwp_default {
+ function = "FWSPIWP";
+ groups = "FWSPIWP";
+@@ -653,12 +648,12 @@
+ };
+
+ pinctrl_qspi1_default: qspi1_default {
+- function = "QSPI1";
++ function = "SPI1";
+ groups = "QSPI1";
+ };
+
+ pinctrl_qspi2_default: qspi2_default {
+- function = "QSPI2";
++ function = "SPI2";
+ groups = "QSPI2";
+ };
+
+diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi
+index 1b47be1704f83..e5724b1a2e20d 100644
+--- a/arch/arm/boot/dts/aspeed-g6.dtsi
++++ b/arch/arm/boot/dts/aspeed-g6.dtsi
+@@ -364,6 +364,41 @@
+ status = "disabled";
+ };
+
++ adc0: adc@1e6e9000 {
++ compatible = "aspeed,ast2600-adc0";
++ reg = <0x1e6e9000 0x100>;
++ clocks = <&syscon ASPEED_CLK_APB2>;
++ resets = <&syscon ASPEED_RESET_ADC>;
++ interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
++ #io-channel-cells = <1>;
++ status = "disabled";
++ };
++
++ adc1: adc@1e6e9100 {
++ compatible = "aspeed,ast2600-adc1";
++ reg = <0x1e6e9100 0x100>;
++ clocks = <&syscon ASPEED_CLK_APB2>;
++ resets = <&syscon ASPEED_RESET_ADC>;
++ interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
++ #io-channel-cells = <1>;
++ status = "disabled";
++ };
++
++ sbc: secure-boot-controller@1e6f2000 {
++ compatible = "aspeed,ast2600-sbc";
++ reg = <0x1e6f2000 0x1000>;
++ };
++
++ video: video@1e700000 {
++ compatible = "aspeed,ast2600-video-engine";
++ reg = <0x1e700000 0x1000>;
++ clocks = <&syscon ASPEED_CLK_GATE_VCLK>,
++ <&syscon ASPEED_CLK_GATE_ECLK>;
++ clock-names = "vclk", "eclk";
++ interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
++ status = "disabled";
++ };
++
+ gpio0: gpio@1e780000 {
+ #gpio-cells = <2>;
+ gpio-controller;
+diff --git a/arch/arm/boot/dts/at91-sama5d3_xplained.dts b/arch/arm/boot/dts/at91-sama5d3_xplained.dts
+index d72c042f28507..a49c2966b41e2 100644
+--- a/arch/arm/boot/dts/at91-sama5d3_xplained.dts
++++ b/arch/arm/boot/dts/at91-sama5d3_xplained.dts
+@@ -57,8 +57,8 @@
+ };
+
+ spi0: spi@f0004000 {
+- pinctrl-names = "default";
+- pinctrl-0 = <&pinctrl_spi0_cs>;
++ pinctrl-names = "default", "cs";
++ pinctrl-1 = <&pinctrl_spi0_cs>;
+ cs-gpios = <&pioD 13 0>, <0>, <0>, <&pioD 16 0>;
+ status = "okay";
+ };
+@@ -171,8 +171,8 @@
+ };
+
+ spi1: spi@f8008000 {
+- pinctrl-names = "default";
+- pinctrl-0 = <&pinctrl_spi1_cs>;
++ pinctrl-names = "default", "cs";
++ pinctrl-1 = <&pinctrl_spi1_cs>;
+ cs-gpios = <&pioC 25 0>;
+ status = "okay";
+ };
+diff --git a/arch/arm/boot/dts/at91-sama5d4_xplained.dts b/arch/arm/boot/dts/at91-sama5d4_xplained.dts
+index d241c24f0d836..e519d27479362 100644
+--- a/arch/arm/boot/dts/at91-sama5d4_xplained.dts
++++ b/arch/arm/boot/dts/at91-sama5d4_xplained.dts
+@@ -81,8 +81,8 @@
+ };
+
+ spi1: spi@fc018000 {
+- pinctrl-names = "default";
+- pinctrl-0 = <&pinctrl_spi0_cs>;
++ pinctrl-names = "default", "cs";
++ pinctrl-1 = <&pinctrl_spi1_cs>;
+ cs-gpios = <&pioB 21 0>;
+ status = "okay";
+ };
+@@ -140,7 +140,7 @@
+ atmel,pins =
+ <AT91_PIOE 1 AT91_PERIPH_GPIO AT91_PINCTRL_PULL_UP_DEGLITCH>;
+ };
+- pinctrl_spi0_cs: spi0_cs_default {
++ pinctrl_spi1_cs: spi1_cs_default {
+ atmel,pins =
+ <AT91_PIOB 21 AT91_PERIPH_GPIO AT91_PINCTRL_NONE>;
+ };
+diff --git a/arch/arm/boot/dts/at91-sama7g5ek.dts b/arch/arm/boot/dts/at91-sama7g5ek.dts
+index f3d6aaa3a78dc..bac0e49cc5770 100644
+--- a/arch/arm/boot/dts/at91-sama7g5ek.dts
++++ b/arch/arm/boot/dts/at91-sama7g5ek.dts
+@@ -403,7 +403,7 @@
+ pinctrl_flx3_default: flx3_default {
+ pinmux = <PIN_PD16__FLEXCOM3_IO0>,
+ <PIN_PD17__FLEXCOM3_IO1>;
+- bias-disable;
++ bias-pull-up;
+ };
+
+ pinctrl_flx4_default: flx4_default {
+diff --git a/arch/arm/boot/dts/at91-tse850-3.dts b/arch/arm/boot/dts/at91-tse850-3.dts
+index 3ca97b47c69ce..7e5c598e7e68f 100644
+--- a/arch/arm/boot/dts/at91-tse850-3.dts
++++ b/arch/arm/boot/dts/at91-tse850-3.dts
+@@ -262,7 +262,7 @@
+ &macb1 {
+ status = "okay";
+
+- phy-mode = "rgmii";
++ phy-mode = "rmii";
+
+ #address-cells = <1>;
+ #size-cells = <0>;
+diff --git a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
+index 87bb39060e8be..ca03685f0f086 100644
+--- a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
++++ b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
+@@ -219,6 +219,12 @@
+ wm8731: wm8731@1b {
+ compatible = "wm8731";
+ reg = <0x1b>;
++
++ /* PCK0 at 12MHz */
++ clocks = <&pmc PMC_TYPE_SYSTEM 8>;
++ clock-names = "mclk";
++ assigned-clocks = <&pmc PMC_TYPE_SYSTEM 8>;
++ assigned-clock-rates = <12000000>;
+ };
+ };
+
+diff --git a/arch/arm/boot/dts/bcm-nsp.dtsi b/arch/arm/boot/dts/bcm-nsp.dtsi
+index 748df7955ae67..e96ddb2e26e2c 100644
+--- a/arch/arm/boot/dts/bcm-nsp.dtsi
++++ b/arch/arm/boot/dts/bcm-nsp.dtsi
+@@ -77,7 +77,7 @@
+ interrupt-affinity = <&cpu0>, <&cpu1>;
+ };
+
+- mpcore@19000000 {
++ mpcore-bus@19000000 {
+ compatible = "simple-bus";
+ ranges = <0x00000000 0x19000000 0x00023000>;
+ #address-cells = <1>;
+@@ -219,7 +219,7 @@
+ status = "disabled";
+ };
+
+- sdio: sdhci@21000 {
++ sdio: mmc@21000 {
+ compatible = "brcm,sdhci-iproc-cygnus";
+ reg = <0x21000 0x100>;
+ interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
+diff --git a/arch/arm/boot/dts/bcm2711.dtsi b/arch/arm/boot/dts/bcm2711.dtsi
+index 3b60297af7f60..89af57482bc8f 100644
+--- a/arch/arm/boot/dts/bcm2711.dtsi
++++ b/arch/arm/boot/dts/bcm2711.dtsi
+@@ -290,6 +290,7 @@
+
+ hvs: hvs@7e400000 {
+ compatible = "brcm,bcm2711-hvs";
++ reg = <0x7e400000 0x8000>;
+ interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+ };
+
+@@ -458,12 +459,26 @@
+ #size-cells = <0>;
+ enable-method = "brcm,bcm2836-smp"; // for ARM 32-bit
+
++ /* Source for d/i-cache-line-size and d/i-cache-sets
++ * https://developer.arm.com/documentation/100095/0003
++ * /Level-1-Memory-System/About-the-L1-memory-system?lang=en
++ * Source for d/i-cache-size
++ * https://www.raspberrypi.com/documentation/computers
++ * /processors.html#bcm2711
++ */
+ cpu0: cpu@0 {
+ device_type = "cpu";
+ compatible = "arm,cortex-a72";
+ reg = <0>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000d8>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ i-cache-size = <0xc000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 48KiB(size)/64(line-size)=768ways/3-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu1: cpu@1 {
+@@ -472,6 +487,13 @@
+ reg = <1>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000e0>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ i-cache-size = <0xc000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 48KiB(size)/64(line-size)=768ways/3-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu2: cpu@2 {
+@@ -480,6 +502,13 @@
+ reg = <2>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000e8>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ i-cache-size = <0xc000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 48KiB(size)/64(line-size)=768ways/3-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu3: cpu@3 {
+@@ -488,6 +517,28 @@
+ reg = <3>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000f0>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ i-cache-size = <0xc000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 48KiB(size)/64(line-size)=768ways/3-way set
++ next-level-cache = <&l2>;
++ };
++
++ /* Source for d/i-cache-line-size and d/i-cache-sets
++ * https://developer.arm.com/documentation/100095/0003
++ * /Level-2-Memory-System/About-the-L2-memory-system?lang=en
++ * Source for d/i-cache-size
++ * https://www.raspberrypi.com/documentation/computers
++ * /processors.html#bcm2711
++ */
++ l2: l2-cache0 {
++ compatible = "cache";
++ cache-size = <0x100000>;
++ cache-line-size = <64>;
++ cache-sets = <1024>; // 1MiB(size)/64(line-size)=16384ways/16-way set
++ cache-level = <2>;
+ };
+ };
+
+@@ -506,11 +557,17 @@
+ #address-cells = <3>;
+ #interrupt-cells = <1>;
+ #size-cells = <2>;
+- interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
++ interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "pcie", "msi";
+ interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+ interrupt-map = <0 0 0 1 &gicv2 GIC_SPI 143
++ IRQ_TYPE_LEVEL_HIGH>,
++ <0 0 0 2 &gicv2 GIC_SPI 144
++ IRQ_TYPE_LEVEL_HIGH>,
++ <0 0 0 3 &gicv2 GIC_SPI 145
++ IRQ_TYPE_LEVEL_HIGH>,
++ <0 0 0 4 &gicv2 GIC_SPI 146
+ IRQ_TYPE_LEVEL_HIGH>;
+ msi-controller;
+ msi-parent = <&pcie0>;
+@@ -576,6 +633,8 @@
+ <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+
++ gpio-ranges = <&gpio 0 0 58>;
++
+ gpclk0_gpio49: gpclk0_gpio49 {
+ pin-gpclk {
+ pins = "gpio49";
+diff --git a/arch/arm/boot/dts/bcm2837.dtsi b/arch/arm/boot/dts/bcm2837.dtsi
+index 0199ec98cd616..5dbdebc462594 100644
+--- a/arch/arm/boot/dts/bcm2837.dtsi
++++ b/arch/arm/boot/dts/bcm2837.dtsi
+@@ -40,12 +40,26 @@
+ #size-cells = <0>;
+ enable-method = "brcm,bcm2836-smp"; // for ARM 32-bit
+
++ /* Source for d/i-cache-line-size and d/i-cache-sets
++ * https://developer.arm.com/documentation/ddi0500/e/level-1-memory-system
++ * /about-the-l1-memory-system?lang=en
++ *
++ * Source for d/i-cache-size
++ * https://magpi.raspberrypi.com/articles/raspberry-pi-3-specs-benchmarks
++ */
+ cpu0: cpu@0 {
+ device_type = "cpu";
+ compatible = "arm,cortex-a53";
+ reg = <0>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000d8>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <128>; // 32KiB(size)/64(line-size)=512ways/4-way set
++ i-cache-size = <0x8000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu1: cpu@1 {
+@@ -54,6 +68,13 @@
+ reg = <1>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000e0>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <128>; // 32KiB(size)/64(line-size)=512ways/4-way set
++ i-cache-size = <0x8000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu2: cpu@2 {
+@@ -62,6 +83,13 @@
+ reg = <2>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000e8>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <128>; // 32KiB(size)/64(line-size)=512ways/4-way set
++ i-cache-size = <0x8000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ next-level-cache = <&l2>;
+ };
+
+ cpu3: cpu@3 {
+@@ -70,6 +98,27 @@
+ reg = <3>;
+ enable-method = "spin-table";
+ cpu-release-addr = <0x0 0x000000f0>;
++ d-cache-size = <0x8000>;
++ d-cache-line-size = <64>;
++ d-cache-sets = <128>; // 32KiB(size)/64(line-size)=512ways/4-way set
++ i-cache-size = <0x8000>;
++ i-cache-line-size = <64>;
++ i-cache-sets = <256>; // 32KiB(size)/64(line-size)=512ways/2-way set
++ next-level-cache = <&l2>;
++ };
++
++ /* Source for cache-line-size + cache-sets
++ * https://developer.arm.com/documentation/ddi0500
++ * /e/level-2-memory-system/about-the-l2-memory-system?lang=en
++ * Source for cache-size
++ * https://datasheets.raspberrypi.com/cm/cm1-and-cm3-datasheet.pdf
++ */
++ l2: l2-cache0 {
++ compatible = "cache";
++ cache-size = <0x80000>;
++ cache-line-size = <64>;
++ cache-sets = <512>; // 512KiB(size)/64(line-size)=8192ways/16-way set
++ cache-level = <2>;
+ };
+ };
+ };
+diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi
+index a3e06b6809476..c113661a6668f 100644
+--- a/arch/arm/boot/dts/bcm283x.dtsi
++++ b/arch/arm/boot/dts/bcm283x.dtsi
+@@ -126,6 +126,8 @@
+ interrupt-controller;
+ #interrupt-cells = <2>;
+
++ gpio-ranges = <&gpio 0 0 54>;
++
+ /* Defines common pin muxing groups
+ *
+ * While each pin can have its mux selected
+diff --git a/arch/arm/boot/dts/bcm4708-netgear-r6250.dts b/arch/arm/boot/dts/bcm4708-netgear-r6250.dts
+index 61c7b137607e5..7900aac4f35a9 100644
+--- a/arch/arm/boot/dts/bcm4708-netgear-r6250.dts
++++ b/arch/arm/boot/dts/bcm4708-netgear-r6250.dts
+@@ -20,7 +20,7 @@
+ bootargs = "console=ttyS0,115200 earlycon";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x08000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
+index 6c6bb7b17d27a..7546c8d07bcd7 100644
+--- a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
++++ b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
+@@ -19,7 +19,7 @@
+ bootargs = "console=ttyS0,115200";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x08000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
+index d29e7f80ea6aa..beae9eab9cb8c 100644
+--- a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
++++ b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
+@@ -19,7 +19,7 @@
+ bootargs = "console=ttyS0,115200";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x18000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
+index 9b6887d477d86..7879f7d7d9c33 100644
+--- a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
++++ b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
+@@ -16,7 +16,7 @@
+ bootargs = "console=ttyS0,115200";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x08000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
+index 7989a53597d4f..56d309dbc6b0d 100644
+--- a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
++++ b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
+@@ -19,7 +19,7 @@
+ bootargs = "console=ttyS0,115200";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x08000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
+index 87b655be674c5..184e3039aa864 100644
+--- a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
++++ b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
+@@ -30,7 +30,7 @@
+ bootargs = "console=ttyS0,115200";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x08000000>;
+diff --git a/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
+index f806be5da7237..c2a266a439d05 100644
+--- a/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
++++ b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
+@@ -15,7 +15,7 @@
+ bootargs = "console=ttyS0,115200 earlycon";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>;
+ };
+diff --git a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
+index 05d4f2931772b..9bef6b9bfa8d9 100644
+--- a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
++++ b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
+@@ -129,7 +129,7 @@
+ };
+ };
+
+- mdio-bus-mux@18003000 {
++ mdio-mux@18003000 {
+
+ /* BIT(9) = 1 => external mdio */
+ mdio@200 {
+diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts b/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts
+index 452b8d0ab180e..b0d8a688141d3 100644
+--- a/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts
++++ b/arch/arm/boot/dts/bcm47094-luxul-xwc-2000.dts
+@@ -16,7 +16,7 @@
+ bootargs = "earlycon";
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>,
+ <0x88000000 0x18000000>;
+diff --git a/arch/arm/boot/dts/bcm53016-meraki-mr32.dts b/arch/arm/boot/dts/bcm53016-meraki-mr32.dts
+index 3b978dc8997a4..577a4dc604d93 100644
+--- a/arch/arm/boot/dts/bcm53016-meraki-mr32.dts
++++ b/arch/arm/boot/dts/bcm53016-meraki-mr32.dts
+@@ -20,7 +20,7 @@
+ bootargs = " console=ttyS0,115200n8 earlycon";
+ };
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x08000000>;
+ device_type = "memory";
+ };
+@@ -195,3 +195,25 @@
+ };
+ };
+ };
++
++&srab {
++ status = "okay";
++
++ ports {
++ port@0 {
++ reg = <0>;
++ label = "poe";
++ };
++
++ port@5 {
++ reg = <5>;
++ label = "cpu";
++ ethernet = <&gmac0>;
++
++ fixed-link {
++ speed = <1000>;
++ duplex-full;
++ };
++ };
++ };
++};
+diff --git a/arch/arm/boot/dts/bcm5301x.dtsi b/arch/arm/boot/dts/bcm5301x.dtsi
+index f92089290ccd5..f69d2af3c1fa4 100644
+--- a/arch/arm/boot/dts/bcm5301x.dtsi
++++ b/arch/arm/boot/dts/bcm5301x.dtsi
+@@ -19,7 +19,7 @@
+ #size-cells = <1>;
+ interrupt-parent = <&gic>;
+
+- chipcommonA@18000000 {
++ chipcommon-a-bus@18000000 {
+ compatible = "simple-bus";
+ ranges = <0x00000000 0x18000000 0x00001000>;
+ #address-cells = <1>;
+@@ -44,7 +44,7 @@
+ };
+ };
+
+- mpcore@19000000 {
++ mpcore-bus@19000000 {
+ compatible = "simple-bus";
+ ranges = <0x00000000 0x19000000 0x00023000>;
+ #address-cells = <1>;
+@@ -242,6 +242,8 @@
+
+ gpio-controller;
+ #gpio-cells = <2>;
++ interrupt-controller;
++ #interrupt-cells = <2>;
+ };
+
+ pcie0: pcie@12000 {
+@@ -369,8 +371,8 @@
+ #address-cells = <1>;
+ };
+
+- mdio-bus-mux@18003000 {
+- compatible = "mdio-mux-mmioreg";
++ mdio-mux@18003000 {
++ compatible = "mdio-mux-mmioreg", "mdio-mux";
+ mdio-parent-bus = <&mdio>;
+ #address-cells = <1>;
+ #size-cells = <0>;
+@@ -408,14 +410,14 @@
+ i2c0: i2c@18009000 {
+ compatible = "brcm,iproc-i2c";
+ reg = <0x18009000 0x50>;
+- interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
++ interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+ #address-cells = <1>;
+ #size-cells = <0>;
+ clock-frequency = <100000>;
+ status = "disabled";
+ };
+
+- dmu@1800c000 {
++ dmu-bus@1800c000 {
+ compatible = "simple-bus";
+ ranges = <0 0x1800c000 0x1000>;
+ #address-cells = <1>;
+diff --git a/arch/arm/boot/dts/bcm94708.dts b/arch/arm/boot/dts/bcm94708.dts
+index 3d13e46c69494..d9eb2040b9631 100644
+--- a/arch/arm/boot/dts/bcm94708.dts
++++ b/arch/arm/boot/dts/bcm94708.dts
+@@ -38,7 +38,7 @@
+ model = "NorthStar SVK (BCM94708)";
+ compatible = "brcm,bcm94708", "brcm,bcm4708";
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>;
+ };
+diff --git a/arch/arm/boot/dts/bcm94709.dts b/arch/arm/boot/dts/bcm94709.dts
+index 5017b7b259cbe..618c812eef73e 100644
+--- a/arch/arm/boot/dts/bcm94709.dts
++++ b/arch/arm/boot/dts/bcm94709.dts
+@@ -38,7 +38,7 @@
+ model = "NorthStar SVK (BCM94709)";
+ compatible = "brcm,bcm94709", "brcm,bcm4709", "brcm,bcm4708";
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>;
+ };
+diff --git a/arch/arm/boot/dts/dra7-l4.dtsi b/arch/arm/boot/dts/dra7-l4.dtsi
+index 956a26d52a4c3..5733e3a4ea8e7 100644
+--- a/arch/arm/boot/dts/dra7-l4.dtsi
++++ b/arch/arm/boot/dts/dra7-l4.dtsi
+@@ -3482,8 +3482,7 @@
+ ti,timer-pwm;
+ };
+ };
+-
+- target-module@2c000 { /* 0x4882c000, ap 17 02.0 */
++ timer15_target: target-module@2c000 { /* 0x4882c000, ap 17 02.0 */
+ compatible = "ti,sysc-omap4-timer", "ti,sysc";
+ reg = <0x2c000 0x4>,
+ <0x2c010 0x4>;
+@@ -3511,7 +3510,7 @@
+ };
+ };
+
+- target-module@2e000 { /* 0x4882e000, ap 19 14.0 */
++ timer16_target: target-module@2e000 { /* 0x4882e000, ap 19 14.0 */
+ compatible = "ti,sysc-omap4-timer", "ti,sysc";
+ reg = <0x2e000 0x4>,
+ <0x2e010 0x4>;
+@@ -4189,11 +4188,11 @@
+ reg = <0x1d0010 0x4>;
+ reg-names = "sysc";
+ ti,sysc-midle = <SYSC_IDLE_FORCE>,
+- <SYSC_IDLE_NO>,
+- <SYSC_IDLE_SMART>;
++ <SYSC_IDLE_NO>;
+ ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+ <SYSC_IDLE_NO>,
+ <SYSC_IDLE_SMART>;
++ power-domains = <&prm_vpe>;
+ clocks = <&vpe_clkctrl DRA7_VPE_VPE_CLKCTRL 0>;
+ clock-names = "fck";
+ #address-cells = <1>;
+diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
+index dfc1ef8ef6aea..61a3fb3e2a2f9 100644
+--- a/arch/arm/boot/dts/dra7.dtsi
++++ b/arch/arm/boot/dts/dra7.dtsi
+@@ -1320,20 +1320,20 @@
+ };
+
+ /* Local timers, see ARM architected timer wrap erratum i940 */
+-&timer3_target {
++&timer15_target {
+ ti,no-reset-on-init;
+ ti,no-idle;
+ timer@0 {
+- assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER3_CLKCTRL 24>;
++ assigned-clocks = <&l4per3_clkctrl DRA7_L4PER3_TIMER15_CLKCTRL 24>;
+ assigned-clock-parents = <&timer_sys_clk_div>;
+ };
+ };
+
+-&timer4_target {
++&timer16_target {
+ ti,no-reset-on-init;
+ ti,no-idle;
+ timer@0 {
+- assigned-clocks = <&l4per_clkctrl DRA7_L4PER_TIMER4_CLKCTRL 24>;
++ assigned-clocks = <&l4per3_clkctrl DRA7_L4PER3_TIMER16_CLKCTRL 24>;
+ assigned-clock-parents = <&timer_sys_clk_div>;
+ };
+ };
+diff --git a/arch/arm/boot/dts/exynos4210-i9100.dts b/arch/arm/boot/dts/exynos4210-i9100.dts
+index 55922176807e6..5f5d9b1357365 100644
+--- a/arch/arm/boot/dts/exynos4210-i9100.dts
++++ b/arch/arm/boot/dts/exynos4210-i9100.dts
+@@ -827,7 +827,7 @@
+ compatible = "brcm,bcm4330-bt";
+
+ shutdown-gpios = <&gpl0 4 GPIO_ACTIVE_HIGH>;
+- reset-gpios = <&gpl1 0 GPIO_ACTIVE_HIGH>;
++ reset-gpios = <&gpl1 0 GPIO_ACTIVE_LOW>;
+ device-wakeup-gpios = <&gpx3 1 GPIO_ACTIVE_HIGH>;
+ host-wakeup-gpios = <&gpx2 6 GPIO_ACTIVE_HIGH>;
+ };
+diff --git a/arch/arm/boot/dts/exynos5250-pinctrl.dtsi b/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
+index d31a68672bfac..d7d756614edd1 100644
+--- a/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
++++ b/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
+@@ -260,7 +260,7 @@
+ };
+
+ uart3_data: uart3-data {
+- samsung,pins = "gpa1-4", "gpa1-4";
++ samsung,pins = "gpa1-4", "gpa1-5";
+ samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
+ samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts
+index 39bbe18145cf2..f042954bdfa5d 100644
+--- a/arch/arm/boot/dts/exynos5250-smdk5250.dts
++++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts
+@@ -118,6 +118,9 @@
+ status = "okay";
+ ddc = <&i2c_2>;
+ hpd-gpios = <&gpx3 7 GPIO_ACTIVE_HIGH>;
++ vdd-supply = <&ldo8_reg>;
++ vdd_osc-supply = <&ldo10_reg>;
++ vdd_pll-supply = <&ldo8_reg>;
+ };
+
+ &i2c_0 {
+diff --git a/arch/arm/boot/dts/exynos5420-smdk5420.dts b/arch/arm/boot/dts/exynos5420-smdk5420.dts
+index a4f0e3ffedbd3..07f65213aae65 100644
+--- a/arch/arm/boot/dts/exynos5420-smdk5420.dts
++++ b/arch/arm/boot/dts/exynos5420-smdk5420.dts
+@@ -124,6 +124,9 @@
+ hpd-gpios = <&gpx3 7 GPIO_ACTIVE_HIGH>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&hdmi_hpd_irq>;
++ vdd-supply = <&ldo6_reg>;
++ vdd_osc-supply = <&ldo7_reg>;
++ vdd_pll-supply = <&ldo6_reg>;
+ };
+
+ &hsi2c_4 {
+diff --git a/arch/arm/boot/dts/gemini-nas4220b.dts b/arch/arm/boot/dts/gemini-nas4220b.dts
+index 13112a8a5dd88..6544c730340fa 100644
+--- a/arch/arm/boot/dts/gemini-nas4220b.dts
++++ b/arch/arm/boot/dts/gemini-nas4220b.dts
+@@ -84,7 +84,7 @@
+ partitions {
+ compatible = "redboot-fis";
+ /* Eraseblock at 0xfe0000 */
+- fis-index-block = <0x1fc>;
++ fis-index-block = <0x7f>;
+ };
+ };
+
+diff --git a/arch/arm/boot/dts/imx23-evk.dts b/arch/arm/boot/dts/imx23-evk.dts
+index 8cbaf1c811745..3b609d987d883 100644
+--- a/arch/arm/boot/dts/imx23-evk.dts
++++ b/arch/arm/boot/dts/imx23-evk.dts
+@@ -79,7 +79,6 @@
+ MX23_PAD_LCD_RESET__GPIO_1_18
+ MX23_PAD_PWM3__GPIO_1_29
+ MX23_PAD_PWM4__GPIO_1_30
+- MX23_PAD_SSP1_DETECT__SSP1_DETECT
+ >;
+ fsl,drive-strength = <MXS_DRIVE_4mA>;
+ fsl,voltage = <MXS_VOLTAGE_HIGH>;
+diff --git a/arch/arm/boot/dts/imx53-m53menlo.dts b/arch/arm/boot/dts/imx53-m53menlo.dts
+index 4f88e96d81ddb..d5c68d1ea707c 100644
+--- a/arch/arm/boot/dts/imx53-m53menlo.dts
++++ b/arch/arm/boot/dts/imx53-m53menlo.dts
+@@ -53,6 +53,31 @@
+ };
+ };
+
++ lvds-decoder {
++ compatible = "ti,ds90cf364a", "lvds-decoder";
++
++ ports {
++ #address-cells = <1>;
++ #size-cells = <0>;
++
++ port@0 {
++ reg = <0>;
++
++ lvds_decoder_in: endpoint {
++ remote-endpoint = <&lvds0_out>;
++ };
++ };
++
++ port@1 {
++ reg = <1>;
++
++ lvds_decoder_out: endpoint {
++ remote-endpoint = <&panel_in>;
++ };
++ };
++ };
++ };
++
+ panel {
+ compatible = "edt,etm0700g0dh6";
+ pinctrl-0 = <&pinctrl_display_gpio>;
+@@ -61,7 +86,7 @@
+
+ port {
+ panel_in: endpoint {
+- remote-endpoint = <&lvds0_out>;
++ remote-endpoint = <&lvds_decoder_out>;
+ };
+ };
+ };
+@@ -450,7 +475,7 @@
+ reg = <2>;
+
+ lvds0_out: endpoint {
+- remote-endpoint = <&panel_in>;
++ remote-endpoint = <&lvds_decoder_in>;
+ };
+ };
+ };
+diff --git a/arch/arm/boot/dts/imx6qdl-apalis.dtsi b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
+index 30fa349f9d054..a696873dc1abe 100644
+--- a/arch/arm/boot/dts/imx6qdl-apalis.dtsi
++++ b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
+@@ -286,6 +286,8 @@
+ codec: sgtl5000@a {
+ compatible = "fsl,sgtl5000";
+ reg = <0x0a>;
++ pinctrl-names = "default";
++ pinctrl-0 = <&pinctrl_sgtl5000>;
+ clocks = <&clks IMX6QDL_CLK_CKO>;
+ VDDA-supply = <&reg_module_3v3_audio>;
+ VDDIO-supply = <&reg_module_3v3>;
+@@ -516,8 +518,6 @@
+ MX6QDL_PAD_DISP0_DAT21__AUD4_TXD 0x130b0
+ MX6QDL_PAD_DISP0_DAT22__AUD4_TXFS 0x130b0
+ MX6QDL_PAD_DISP0_DAT23__AUD4_RXD 0x130b0
+- /* SGTL5000 sys_mclk */
+- MX6QDL_PAD_GPIO_5__CCM_CLKO1 0x130b0
+ >;
+ };
+
+@@ -810,6 +810,12 @@
+ >;
+ };
+
++ pinctrl_sgtl5000: sgtl5000grp {
++ fsl,pins = <
++ MX6QDL_PAD_GPIO_5__CCM_CLKO1 0x130b0
++ >;
++ };
++
+ pinctrl_spdif: spdifgrp {
+ fsl,pins = <
+ MX6QDL_PAD_GPIO_16__SPDIF_IN 0x1b0b0
+diff --git a/arch/arm/boot/dts/imx6qdl-udoo.dtsi b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
+index d07d8f83456d2..ccfa8e320be62 100644
+--- a/arch/arm/boot/dts/imx6qdl-udoo.dtsi
++++ b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
+@@ -5,6 +5,8 @@
+ * Author: Fabio Estevam <fabio.estevam@freescale.com>
+ */
+
++#include <dt-bindings/gpio/gpio.h>
++
+ / {
+ aliases {
+ backlight = &backlight;
+@@ -226,6 +228,7 @@
+ MX6QDL_PAD_SD3_DAT1__SD3_DATA1 0x17059
+ MX6QDL_PAD_SD3_DAT2__SD3_DATA2 0x17059
+ MX6QDL_PAD_SD3_DAT3__SD3_DATA3 0x17059
++ MX6QDL_PAD_SD3_DAT5__GPIO7_IO00 0x1b0b0
+ >;
+ };
+
+@@ -304,7 +307,7 @@
+ &usdhc3 {
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_usdhc3>;
+- non-removable;
++ cd-gpios = <&gpio7 0 GPIO_ACTIVE_LOW>;
+ status = "okay";
+ };
+
+diff --git a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
+index b62a0dbb033ff..ec6fba5ee8fde 100644
+--- a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
++++ b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
+@@ -309,6 +309,7 @@
+
+ ethphy: ethernet-phy@1 {
+ reg = <1>;
++ qca,clk-out-frequency = <125000000>;
+ };
+ };
+ };
+diff --git a/arch/arm/boot/dts/imx6ull-colibri.dtsi b/arch/arm/boot/dts/imx6ull-colibri.dtsi
+index 0cdbf7b6e7285..b6fc879e9dbe6 100644
+--- a/arch/arm/boot/dts/imx6ull-colibri.dtsi
++++ b/arch/arm/boot/dts/imx6ull-colibri.dtsi
+@@ -37,7 +37,7 @@
+
+ reg_sd1_vmmc: regulator-sd1-vmmc {
+ compatible = "regulator-gpio";
+- gpio = <&gpio5 9 GPIO_ACTIVE_HIGH>;
++ gpios = <&gpio5 9 GPIO_ACTIVE_HIGH>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_snvs_reg_sd>;
+ regulator-always-on;
+diff --git a/arch/arm/boot/dts/imx6ull-pinfunc.h b/arch/arm/boot/dts/imx6ull-pinfunc.h
+index eb025a9d47592..7328d4ef8559f 100644
+--- a/arch/arm/boot/dts/imx6ull-pinfunc.h
++++ b/arch/arm/boot/dts/imx6ull-pinfunc.h
+@@ -82,6 +82,6 @@
+ #define MX6ULL_PAD_CSI_DATA04__ESAI_TX_FS 0x01F4 0x0480 0x0000 0x9 0x0
+ #define MX6ULL_PAD_CSI_DATA05__ESAI_TX_CLK 0x01F8 0x0484 0x0000 0x9 0x0
+ #define MX6ULL_PAD_CSI_DATA06__ESAI_TX5_RX0 0x01FC 0x0488 0x0000 0x9 0x0
+-#define MX6ULL_PAD_CSI_DATA07__ESAI_T0 0x0200 0x048C 0x0000 0x9 0x0
++#define MX6ULL_PAD_CSI_DATA07__ESAI_TX0 0x0200 0x048C 0x0000 0x9 0x0
+
+ #endif /* __DTS_IMX6ULL_PINFUNC_H */
+diff --git a/arch/arm/boot/dts/imx7-colibri.dtsi b/arch/arm/boot/dts/imx7-colibri.dtsi
+index 62b771c1d5a9a..f1c60b0cb143e 100644
+--- a/arch/arm/boot/dts/imx7-colibri.dtsi
++++ b/arch/arm/boot/dts/imx7-colibri.dtsi
+@@ -40,7 +40,7 @@
+
+ dailink_master: simple-audio-card,codec {
+ sound-dai = <&codec>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ };
+ };
+ };
+@@ -293,7 +293,7 @@
+ compatible = "fsl,sgtl5000";
+ #sound-dai-cells = <0>;
+ reg = <0x0a>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_sai1_mclk>;
+ VDDA-supply = <&reg_module_3v3_avdd>;
+diff --git a/arch/arm/boot/dts/imx7-mba7.dtsi b/arch/arm/boot/dts/imx7-mba7.dtsi
+index 5e6bef230dc75..b55a7792a8391 100644
+--- a/arch/arm/boot/dts/imx7-mba7.dtsi
++++ b/arch/arm/boot/dts/imx7-mba7.dtsi
+@@ -264,7 +264,7 @@
+ tlv320aic32x4: audio-codec@18 {
+ compatible = "ti,tlv320aic32x4";
+ reg = <0x18>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ clock-names = "mclk";
+ ldoin-supply = <&reg_audio_3v3>;
+ iov-supply = <&reg_audio_3v3>;
+diff --git a/arch/arm/boot/dts/imx7d-nitrogen7.dts b/arch/arm/boot/dts/imx7d-nitrogen7.dts
+index e0751e6ba3c0f..a31de900139d6 100644
+--- a/arch/arm/boot/dts/imx7d-nitrogen7.dts
++++ b/arch/arm/boot/dts/imx7d-nitrogen7.dts
+@@ -288,7 +288,7 @@
+ codec: wm8960@1a {
+ compatible = "wlf,wm8960";
+ reg = <0x1a>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ clock-names = "mclk";
+ wlf,shared-lrclk;
+ };
+diff --git a/arch/arm/boot/dts/imx7d-pico-hobbit.dts b/arch/arm/boot/dts/imx7d-pico-hobbit.dts
+index 7b2198a9372c6..d917dc4f2f227 100644
+--- a/arch/arm/boot/dts/imx7d-pico-hobbit.dts
++++ b/arch/arm/boot/dts/imx7d-pico-hobbit.dts
+@@ -31,7 +31,7 @@
+
+ dailink_master: simple-audio-card,codec {
+ sound-dai = <&sgtl5000>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ };
+ };
+ };
+@@ -41,7 +41,7 @@
+ #sound-dai-cells = <0>;
+ reg = <0x0a>;
+ compatible = "fsl,sgtl5000";
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ VDDA-supply = <&reg_2p5v>;
+ VDDIO-supply = <&reg_vref_1v8>;
+ };
+diff --git a/arch/arm/boot/dts/imx7d-pico-pi.dts b/arch/arm/boot/dts/imx7d-pico-pi.dts
+index 70bea95c06d83..f263e391e24cb 100644
+--- a/arch/arm/boot/dts/imx7d-pico-pi.dts
++++ b/arch/arm/boot/dts/imx7d-pico-pi.dts
+@@ -31,7 +31,7 @@
+
+ dailink_master: simple-audio-card,codec {
+ sound-dai = <&sgtl5000>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ };
+ };
+ };
+@@ -41,7 +41,7 @@
+ #sound-dai-cells = <0>;
+ reg = <0x0a>;
+ compatible = "fsl,sgtl5000";
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ VDDA-supply = <&reg_2p5v>;
+ VDDIO-supply = <&reg_vref_1v8>;
+ };
+diff --git a/arch/arm/boot/dts/imx7d-sdb.dts b/arch/arm/boot/dts/imx7d-sdb.dts
+index 4a0d83784d7d1..e5f1bdbe79921 100644
+--- a/arch/arm/boot/dts/imx7d-sdb.dts
++++ b/arch/arm/boot/dts/imx7d-sdb.dts
+@@ -385,14 +385,14 @@
+ codec: wm8960@1a {
+ compatible = "wlf,wm8960";
+ reg = <0x1a>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ clock-names = "mclk";
+ wlf,shared-lrclk;
+ wlf,hp-cfg = <2 2 3>;
+ wlf,gpio-cfg = <1 3>;
+ assigned-clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_SRC>,
+ <&clks IMX7D_PLL_AUDIO_POST_DIV>,
+- <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+ assigned-clock-rates = <0>, <884736000>, <12288000>;
+ };
+diff --git a/arch/arm/boot/dts/imx7s-warp.dts b/arch/arm/boot/dts/imx7s-warp.dts
+index 569bbd84e371a..558b064da743c 100644
+--- a/arch/arm/boot/dts/imx7s-warp.dts
++++ b/arch/arm/boot/dts/imx7s-warp.dts
+@@ -75,7 +75,7 @@
+
+ dailink_master: simple-audio-card,codec {
+ sound-dai = <&codec>;
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ };
+ };
+ };
+@@ -232,7 +232,7 @@
+ #sound-dai-cells = <0>;
+ reg = <0x0a>;
+ compatible = "fsl,sgtl5000";
+- clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
++ clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_sai1_mclk>;
+ VDDA-supply = <&vgen4_reg>;
+diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi
+index b7ea37ad4e55c..bcec98b964114 100644
+--- a/arch/arm/boot/dts/imx7ulp.dtsi
++++ b/arch/arm/boot/dts/imx7ulp.dtsi
+@@ -259,7 +259,7 @@
+ interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&pcc2 IMX7ULP_CLK_WDG1>;
+ assigned-clocks = <&pcc2 IMX7ULP_CLK_WDG1>;
+- assigned-clocks-parents = <&scg1 IMX7ULP_CLK_FIRC_BUS_CLK>;
++ assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC_BUS_CLK>;
+ timeout-sec = <40>;
+ };
+
+diff --git a/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
+index 2a0a98fe67f06..3240c67e0c392 100644
+--- a/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
++++ b/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
+@@ -11,3 +11,18 @@
+ model = "LogicPD Zoom OMAP35xx SOM-LV Development Kit";
+ compatible = "logicpd,dm3730-som-lv-devkit", "ti,omap3430", "ti,omap3";
+ };
++
++&omap3_pmx_core2 {
++ pinctrl-names = "default";
++ pinctrl-0 = <&hsusb2_2_pins>;
++ hsusb2_2_pins: pinmux_hsusb2_2_pins {
++ pinctrl-single,pins = <
++ OMAP3430_CORE2_IOPAD(0x25f0, PIN_OUTPUT | MUX_MODE3) /* etk_d10.hsusb2_clk */
++ OMAP3430_CORE2_IOPAD(0x25f2, PIN_OUTPUT | MUX_MODE3) /* etk_d11.hsusb2_stp */
++ OMAP3430_CORE2_IOPAD(0x25f4, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d12.hsusb2_dir */
++ OMAP3430_CORE2_IOPAD(0x25f6, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d13.hsusb2_nxt */
++ OMAP3430_CORE2_IOPAD(0x25f8, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d14.hsusb2_data0 */
++ OMAP3430_CORE2_IOPAD(0x25fa, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d15.hsusb2_data1 */
++ >;
++ };
++};
+diff --git a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
+index a604d92221a4f..c757f0d7781c1 100644
+--- a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
++++ b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
+@@ -11,3 +11,18 @@
+ model = "LogicPD Zoom DM3730 SOM-LV Development Kit";
+ compatible = "logicpd,dm3730-som-lv-devkit", "ti,omap3630", "ti,omap3";
+ };
++
++&omap3_pmx_core2 {
++ pinctrl-names = "default";
++ pinctrl-0 = <&hsusb2_2_pins>;
++ hsusb2_2_pins: pinmux_hsusb2_2_pins {
++ pinctrl-single,pins = <
++ OMAP3630_CORE2_IOPAD(0x25f0, PIN_OUTPUT | MUX_MODE3) /* etk_d10.hsusb2_clk */
++ OMAP3630_CORE2_IOPAD(0x25f2, PIN_OUTPUT | MUX_MODE3) /* etk_d11.hsusb2_stp */
++ OMAP3630_CORE2_IOPAD(0x25f4, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d12.hsusb2_dir */
++ OMAP3630_CORE2_IOPAD(0x25f6, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d13.hsusb2_nxt */
++ OMAP3630_CORE2_IOPAD(0x25f8, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d14.hsusb2_data0 */
++ OMAP3630_CORE2_IOPAD(0x25fa, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d15.hsusb2_data1 */
++ >;
++ };
++};
+diff --git a/arch/arm/boot/dts/logicpd-som-lv.dtsi b/arch/arm/boot/dts/logicpd-som-lv.dtsi
+index b56524cc7fe27..55b619c99e24d 100644
+--- a/arch/arm/boot/dts/logicpd-som-lv.dtsi
++++ b/arch/arm/boot/dts/logicpd-som-lv.dtsi
+@@ -265,21 +265,6 @@
+ };
+ };
+
+-&omap3_pmx_core2 {
+- pinctrl-names = "default";
+- pinctrl-0 = <&hsusb2_2_pins>;
+- hsusb2_2_pins: pinmux_hsusb2_2_pins {
+- pinctrl-single,pins = <
+- OMAP3630_CORE2_IOPAD(0x25f0, PIN_OUTPUT | MUX_MODE3) /* etk_d10.hsusb2_clk */
+- OMAP3630_CORE2_IOPAD(0x25f2, PIN_OUTPUT | MUX_MODE3) /* etk_d11.hsusb2_stp */
+- OMAP3630_CORE2_IOPAD(0x25f4, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d12.hsusb2_dir */
+- OMAP3630_CORE2_IOPAD(0x25f6, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d13.hsusb2_nxt */
+- OMAP3630_CORE2_IOPAD(0x25f8, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d14.hsusb2_data0 */
+- OMAP3630_CORE2_IOPAD(0x25fa, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d15.hsusb2_data1 */
+- >;
+- };
+-};
+-
+ &uart2 {
+ interrupts-extended = <&intc 73 &omap3_pmx_core OMAP3_UART2_RX>;
+ pinctrl-names = "default";
+diff --git a/arch/arm/boot/dts/ls1021a-tsn.dts b/arch/arm/boot/dts/ls1021a-tsn.dts
+index 9d8f0c2a8aba3..aca78b5eddf20 100644
+--- a/arch/arm/boot/dts/ls1021a-tsn.dts
++++ b/arch/arm/boot/dts/ls1021a-tsn.dts
+@@ -251,7 +251,7 @@
+
+ flash@0 {
+ /* Rev. A uses 64MB flash, Rev. B & C use 32MB flash */
+- compatible = "jedec,spi-nor", "s25fl256s1", "s25fl512s";
++ compatible = "jedec,spi-nor";
+ spi-max-frequency = <20000000>;
+ #address-cells = <1>;
+ #size-cells = <1>;
+diff --git a/arch/arm/boot/dts/ls1021a.dtsi b/arch/arm/boot/dts/ls1021a.dtsi
+index 4fce81422943b..f3b8540750b61 100644
+--- a/arch/arm/boot/dts/ls1021a.dtsi
++++ b/arch/arm/boot/dts/ls1021a.dtsi
+@@ -329,39 +329,6 @@
+ #thermal-sensor-cells = <1>;
+ };
+
+- thermal-zones {
+- cpu_thermal: cpu-thermal {
+- polling-delay-passive = <1000>;
+- polling-delay = <5000>;
+-
+- thermal-sensors = <&tmu 0>;
+-
+- trips {
+- cpu_alert: cpu-alert {
+- temperature = <85000>;
+- hysteresis = <2000>;
+- type = "passive";
+- };
+- cpu_crit: cpu-crit {
+- temperature = <95000>;
+- hysteresis = <2000>;
+- type = "critical";
+- };
+- };
+-
+- cooling-maps {
+- map0 {
+- trip = <&cpu_alert>;
+- cooling-device =
+- <&cpu0 THERMAL_NO_LIMIT
+- THERMAL_NO_LIMIT>,
+- <&cpu1 THERMAL_NO_LIMIT
+- THERMAL_NO_LIMIT>;
+- };
+- };
+- };
+- };
+-
+ dspi0: spi@2100000 {
+ compatible = "fsl,ls1021a-v1.0-dspi";
+ #address-cells = <1>;
+@@ -1016,4 +983,37 @@
+ big-endian;
+ };
+ };
++
++ thermal-zones {
++ cpu_thermal: cpu-thermal {
++ polling-delay-passive = <1000>;
++ polling-delay = <5000>;
++
++ thermal-sensors = <&tmu 0>;
++
++ trips {
++ cpu_alert: cpu-alert {
++ temperature = <85000>;
++ hysteresis = <2000>;
++ type = "passive";
++ };
++ cpu_crit: cpu-crit {
++ temperature = <95000>;
++ hysteresis = <2000>;
++ type = "critical";
++ };
++ };
++
++ cooling-maps {
++ map0 {
++ trip = <&cpu_alert>;
++ cooling-device =
++ <&cpu0 THERMAL_NO_LIMIT
++ THERMAL_NO_LIMIT>,
++ <&cpu1 THERMAL_NO_LIMIT
++ THERMAL_NO_LIMIT>;
++ };
++ };
++ };
++ };
+ };
+diff --git a/arch/arm/boot/dts/meson.dtsi b/arch/arm/boot/dts/meson.dtsi
+index 3be7cba603d5a..26eaba3fa96f3 100644
+--- a/arch/arm/boot/dts/meson.dtsi
++++ b/arch/arm/boot/dts/meson.dtsi
+@@ -59,7 +59,7 @@
+ };
+
+ uart_A: serial@84c0 {
+- compatible = "amlogic,meson6-uart", "amlogic,meson-uart";
++ compatible = "amlogic,meson6-uart";
+ reg = <0x84c0 0x18>;
+ interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>;
+ fifo-size = <128>;
+@@ -67,7 +67,7 @@
+ };
+
+ uart_B: serial@84dc {
+- compatible = "amlogic,meson6-uart", "amlogic,meson-uart";
++ compatible = "amlogic,meson6-uart";
+ reg = <0x84dc 0x18>;
+ interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>;
+ status = "disabled";
+@@ -105,7 +105,7 @@
+ };
+
+ uart_C: serial@8700 {
+- compatible = "amlogic,meson6-uart", "amlogic,meson-uart";
++ compatible = "amlogic,meson6-uart";
+ reg = <0x8700 0x18>;
+ interrupts = <GIC_SPI 93 IRQ_TYPE_EDGE_RISING>;
+ status = "disabled";
+@@ -228,7 +228,7 @@
+ };
+
+ uart_AO: serial@4c0 {
+- compatible = "amlogic,meson6-uart", "amlogic,meson-ao-uart", "amlogic,meson-uart";
++ compatible = "amlogic,meson6-uart", "amlogic,meson-ao-uart";
+ reg = <0x4c0 0x18>;
+ interrupts = <GIC_SPI 90 IRQ_TYPE_EDGE_RISING>;
+ status = "disabled";
+diff --git a/arch/arm/boot/dts/meson8.dtsi b/arch/arm/boot/dts/meson8.dtsi
+index f80ddc98d3a2b..9997a5d0333a3 100644
+--- a/arch/arm/boot/dts/meson8.dtsi
++++ b/arch/arm/boot/dts/meson8.dtsi
+@@ -736,27 +736,27 @@
+ };
+
+ &uart_AO {
+- compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_CLK81>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8-uart", "amlogic,meson-ao-uart";
++ clocks = <&xtal>, <&clkc CLKID_CLK81>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_A {
+- compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART0>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_B {
+- compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART1>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_C {
+- compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART2>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &usb0 {
+diff --git a/arch/arm/boot/dts/meson8b.dtsi b/arch/arm/boot/dts/meson8b.dtsi
+index b49b7cbaed4ee..94f1c03deccef 100644
+--- a/arch/arm/boot/dts/meson8b.dtsi
++++ b/arch/arm/boot/dts/meson8b.dtsi
+@@ -724,27 +724,27 @@
+ };
+
+ &uart_AO {
+- compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_CLK81>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8b-uart", "amlogic,meson-ao-uart";
++ clocks = <&xtal>, <&clkc CLKID_CLK81>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_A {
+- compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART0>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8b-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_B {
+- compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART1>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8b-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &uart_C {
+- compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+- clocks = <&clkc CLKID_CLK81>, <&xtal>, <&clkc CLKID_UART2>;
+- clock-names = "baud", "xtal", "pclk";
++ compatible = "amlogic,meson8b-uart";
++ clocks = <&xtal>, <&clkc CLKID_UART0>, <&clkc CLKID_CLK81>;
++ clock-names = "xtal", "pclk", "baud";
+ };
+
+ &usb0 {
+diff --git a/arch/arm/boot/dts/omap-gpmc-smsc9221.dtsi b/arch/arm/boot/dts/omap-gpmc-smsc9221.dtsi
+index 7f6aefd134514..e7534fe9c53cf 100644
+--- a/arch/arm/boot/dts/omap-gpmc-smsc9221.dtsi
++++ b/arch/arm/boot/dts/omap-gpmc-smsc9221.dtsi
+@@ -29,7 +29,7 @@
+ compatible = "smsc,lan9221","smsc,lan9115";
+ bank-width = <2>;
+
+- gpmc,mux-add-data;
++ gpmc,mux-add-data = <0>;
+ gpmc,cs-on-ns = <0>;
+ gpmc,cs-rd-off-ns = <42>;
+ gpmc,cs-wr-off-ns = <36>;
+diff --git a/arch/arm/boot/dts/omap3-beagle-ab4.dts b/arch/arm/boot/dts/omap3-beagle-ab4.dts
+new file mode 100644
+index 0000000000000..990ff2d846868
+--- /dev/null
++++ b/arch/arm/boot/dts/omap3-beagle-ab4.dts
+@@ -0,0 +1,47 @@
++// SPDX-License-Identifier: GPL-2.0-only
++/dts-v1/;
++
++#include "omap3-beagle.dts"
++
++/ {
++ model = "TI OMAP3 BeagleBoard A to B4";
++ compatible = "ti,omap3-beagle-ab4", "ti,omap3-beagle", "ti,omap3430", "ti,omap3";
++};
++
++/*
++ * Workaround for capacitor C70 issue, see "Boards revision A and < B5"
++ * section at https://elinux.org/BeagleBoard_Community
++ */
++
++/* Unusable as clocksource because of unreliable oscillator */
++&counter32k {
++ status = "disabled";
++};
++
++/* Unusable as clockevent because of unreliable oscillator, allow to idle */
++&timer1_target {
++ /delete-property/ti,no-reset-on-init;
++ /delete-property/ti,no-idle;
++ timer@0 {
++ /delete-property/ti,timer-alwon;
++ };
++};
++
++/* Preferred always-on timer for clocksource */
++&timer12_target {
++ ti,no-reset-on-init;
++ ti,no-idle;
++ timer@0 {
++ /* Always clocked by secure_32k_fck */
++ };
++};
++
++/* Preferred timer for clockevent */
++&timer2_target {
++ ti,no-reset-on-init;
++ ti,no-idle;
++ timer@0 {
++ assigned-clocks = <&gpt2_fck>;
++ assigned-clock-parents = <&sys_ck>;
++ };
++};
+diff --git a/arch/arm/boot/dts/omap3-beagle.dts b/arch/arm/boot/dts/omap3-beagle.dts
+index f9f34b8458e91..0548b391334fd 100644
+--- a/arch/arm/boot/dts/omap3-beagle.dts
++++ b/arch/arm/boot/dts/omap3-beagle.dts
+@@ -304,39 +304,6 @@
+ phys = <0 &hsusb2_phy>;
+ };
+
+-/* Unusable as clocksource because of unreliable oscillator */
+-&counter32k {
+- status = "disabled";
+-};
+-
+-/* Unusable as clockevent because if unreliable oscillator, allow to idle */
+-&timer1_target {
+- /delete-property/ti,no-reset-on-init;
+- /delete-property/ti,no-idle;
+- timer@0 {
+- /delete-property/ti,timer-alwon;
+- };
+-};
+-
+-/* Preferred always-on timer for clocksource */
+-&timer12_target {
+- ti,no-reset-on-init;
+- ti,no-idle;
+- timer@0 {
+- /* Always clocked by secure_32k_fck */
+- };
+-};
+-
+-/* Preferred timer for clockevent */
+-&timer2_target {
+- ti,no-reset-on-init;
+- ti,no-idle;
+- timer@0 {
+- assigned-clocks = <&gpt2_fck>;
+- assigned-clock-parents = <&sys_ck>;
+- };
+-};
+-
+ &twl_gpio {
+ ti,use-leds;
+ /* pullups: BIT(1) */
+diff --git a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
+index 2c19d6e255bdc..6883ccb45600b 100644
+--- a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
++++ b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
+@@ -158,6 +158,24 @@
+ status = "disabled";
+ };
+
++/* Unusable as clockevent because if unreliable oscillator, allow to idle */
++&timer1_target {
++ /delete-property/ti,no-reset-on-init;
++ /delete-property/ti,no-idle;
++ timer@0 {
++ /delete-property/ti,timer-alwon;
++ };
++};
++
++/* Preferred timer for clockevent */
++&timer12_target {
++ ti,no-reset-on-init;
++ ti,no-idle;
++ timer@0 {
++ /* Always clocked by secure_32k_fck */
++ };
++};
++
+ &twl_gpio {
+ ti,use-leds;
+ /*
+diff --git a/arch/arm/boot/dts/omap3-devkit8000.dts b/arch/arm/boot/dts/omap3-devkit8000.dts
+index c2995a280729d..162d0726b0080 100644
+--- a/arch/arm/boot/dts/omap3-devkit8000.dts
++++ b/arch/arm/boot/dts/omap3-devkit8000.dts
+@@ -14,36 +14,3 @@
+ display2 = &tv0;
+ };
+ };
+-
+-/* Unusable as clocksource because of unreliable oscillator */
+-&counter32k {
+- status = "disabled";
+-};
+-
+-/* Unusable as clockevent because if unreliable oscillator, allow to idle */
+-&timer1_target {
+- /delete-property/ti,no-reset-on-init;
+- /delete-property/ti,no-idle;
+- timer@0 {
+- /delete-property/ti,timer-alwon;
+- };
+-};
+-
+-/* Preferred always-on timer for clocksource */
+-&timer12_target {
+- ti,no-reset-on-init;
+- ti,no-idle;
+- timer@0 {
+- /* Always clocked by secure_32k_fck */
+- };
+-};
+-
+-/* Preferred timer for clockevent */
+-&timer2_target {
+- ti,no-reset-on-init;
+- ti,no-idle;
+- timer@0 {
+- assigned-clocks = <&gpt2_fck>;
+- assigned-clock-parents = <&sys_ck>;
+- };
+-};
+diff --git a/arch/arm/boot/dts/omap3-gta04.dtsi b/arch/arm/boot/dts/omap3-gta04.dtsi
+index 938cc691bb2fe..3923b38e798d0 100644
+--- a/arch/arm/boot/dts/omap3-gta04.dtsi
++++ b/arch/arm/boot/dts/omap3-gta04.dtsi
+@@ -31,6 +31,8 @@
+ aliases {
+ display0 = &lcd;
+ display1 = &tv0;
++ /delete-property/ mmc2;
++ /delete-property/ mmc3;
+ };
+
+ ldo_3v3: fixedregulator {
+@@ -515,7 +517,7 @@
+ compatible = "bosch,bma180";
+ reg = <0x41>;
+ pinctrl-names = "default";
+- pintcrl-0 = <&bma180_pins>;
++ pinctrl-0 = <&bma180_pins>;
+ interrupt-parent = <&gpio4>;
+ interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; /* GPIO_115 */
+ };
+diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts
+index 32335d4ce478b..d40c3d2c4914e 100644
+--- a/arch/arm/boot/dts/omap3-n900.dts
++++ b/arch/arm/boot/dts/omap3-n900.dts
+@@ -8,6 +8,7 @@
+
+ #include "omap34xx.dtsi"
+ #include <dt-bindings/input/input.h>
++#include <dt-bindings/leds/common.h>
+
+ /*
+ * Default secure signed bootloader (Nokia X-Loader) does not enable L3 firewall
+@@ -630,63 +631,92 @@
+ };
+
+ lp5523: lp5523@32 {
++ #address-cells = <1>;
++ #size-cells = <0>;
+ compatible = "national,lp5523";
+ reg = <0x32>;
+ clock-mode = /bits/ 8 <0>; /* LP55XX_CLOCK_AUTO */
+- enable-gpio = <&gpio2 9 GPIO_ACTIVE_HIGH>; /* 41 */
++ enable-gpios = <&gpio2 9 GPIO_ACTIVE_HIGH>; /* 41 */
+
+- chan0 {
++ led@0 {
++ reg = <0>;
+ chan-name = "lp5523:kb1";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+
+- chan1 {
++ led@1 {
++ reg = <1>;
+ chan-name = "lp5523:kb2";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+
+- chan2 {
++ led@2 {
++ reg = <2>;
+ chan-name = "lp5523:kb3";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+
+- chan3 {
++ led@3 {
++ reg = <3>;
+ chan-name = "lp5523:kb4";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+
+- chan4 {
++ led@4 {
++ reg = <4>;
+ chan-name = "lp5523:b";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_BLUE>;
++ function = LED_FUNCTION_STATUS;
+ };
+
+- chan5 {
++ led@5 {
++ reg = <5>;
+ chan-name = "lp5523:g";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_GREEN>;
++ function = LED_FUNCTION_STATUS;
+ };
+
+- chan6 {
++ led@6 {
++ reg = <6>;
+ chan-name = "lp5523:r";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_RED>;
++ function = LED_FUNCTION_STATUS;
+ };
+
+- chan7 {
++ led@7 {
++ reg = <7>;
+ chan-name = "lp5523:kb5";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+
+- chan8 {
++ led@8 {
++ reg = <8>;
+ chan-name = "lp5523:kb6";
+ led-cur = /bits/ 8 <50>;
+ max-cur = /bits/ 8 <100>;
++ color = <LED_COLOR_ID_WHITE>;
++ function = LED_FUNCTION_KBD_BACKLIGHT;
+ };
+ };
+
+diff --git a/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi b/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi
+index e5da3bc6f1050..218a10c0d8159 100644
+--- a/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi
++++ b/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi
+@@ -22,7 +22,7 @@
+ compatible = "smsc,lan9221","smsc,lan9115";
+ bank-width = <2>;
+
+- gpmc,mux-add-data;
++ gpmc,mux-add-data = <0>;
+ gpmc,cs-on-ns = <0>;
+ gpmc,cs-rd-off-ns = <42>;
+ gpmc,cs-wr-off-ns = <36>;
+diff --git a/arch/arm/boot/dts/openbmc-flash-layout-64.dtsi b/arch/arm/boot/dts/openbmc-flash-layout-64.dtsi
+index 31f59de5190b8..7af41361c4800 100644
+--- a/arch/arm/boot/dts/openbmc-flash-layout-64.dtsi
++++ b/arch/arm/boot/dts/openbmc-flash-layout-64.dtsi
+@@ -28,7 +28,7 @@ partitions {
+ label = "rofs";
+ };
+
+- rwfs@6000000 {
++ rwfs@2a00000 {
+ reg = <0x2a00000 0x1600000>; // 22MB
+ label = "rwfs";
+ };
+diff --git a/arch/arm/boot/dts/openbmc-flash-layout.dtsi b/arch/arm/boot/dts/openbmc-flash-layout.dtsi
+index 6c26524e93e11..b47e14063c380 100644
+--- a/arch/arm/boot/dts/openbmc-flash-layout.dtsi
++++ b/arch/arm/boot/dts/openbmc-flash-layout.dtsi
+@@ -20,7 +20,7 @@ partitions {
+ label = "kernel";
+ };
+
+- rofs@c0000 {
++ rofs@4c0000 {
+ reg = <0x4c0000 0x1740000>;
+ label = "rofs";
+ };
+diff --git a/arch/arm/boot/dts/qcom-ipq4019.dtsi b/arch/arm/boot/dts/qcom-ipq4019.dtsi
+index ff1bdb10ad198..08bc5f46649dd 100644
+--- a/arch/arm/boot/dts/qcom-ipq4019.dtsi
++++ b/arch/arm/boot/dts/qcom-ipq4019.dtsi
+@@ -142,7 +142,8 @@
+ clocks {
+ sleep_clk: sleep_clk {
+ compatible = "fixed-clock";
+- clock-frequency = <32768>;
++ clock-frequency = <32000>;
++ clock-output-names = "gcc_sleep_clk_src";
+ #clock-cells = <0>;
+ };
+
+diff --git a/arch/arm/boot/dts/qcom-ipq8064-rb3011.dts b/arch/arm/boot/dts/qcom-ipq8064-rb3011.dts
+index f7ea2e5dd1914..971d2e2292600 100644
+--- a/arch/arm/boot/dts/qcom-ipq8064-rb3011.dts
++++ b/arch/arm/boot/dts/qcom-ipq8064-rb3011.dts
+@@ -19,12 +19,12 @@
+ stdout-path = "serial0:115200n8";
+ };
+
+- memory@0 {
++ memory@42000000 {
+ reg = <0x42000000 0x3e000000>;
+ device_type = "memory";
+ };
+
+- mdio0: mdio@0 {
++ mdio0: mdio-0 {
+ status = "okay";
+ compatible = "virtual,mdio-gpio";
+ gpios = <&qcom_pinmux 1 GPIO_ACTIVE_HIGH>,
+@@ -91,7 +91,7 @@
+ };
+ };
+
+- mdio1: mdio@1 {
++ mdio1: mdio-1 {
+ status = "okay";
+ compatible = "virtual,mdio-gpio";
+ gpios = <&qcom_pinmux 11 GPIO_ACTIVE_HIGH>,
+diff --git a/arch/arm/boot/dts/qcom-msm8960.dtsi b/arch/arm/boot/dts/qcom-msm8960.dtsi
+index 172ea3c70eac2..c197927e7435f 100644
+--- a/arch/arm/boot/dts/qcom-msm8960.dtsi
++++ b/arch/arm/boot/dts/qcom-msm8960.dtsi
+@@ -146,7 +146,9 @@
+ reg = <0x108000 0x1000>;
+ qcom,ipc = <&l2cc 0x8 2>;
+
+- interrupts = <0 19 0>, <0 21 0>, <0 22 0>;
++ interrupts = <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>,
++ <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
++ <GIC_SPI 22 IRQ_TYPE_EDGE_RISING>;
+ interrupt-names = "ack", "err", "wakeup";
+
+ regulators {
+@@ -192,7 +194,7 @@
+ compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
+ reg = <0x16440000 0x1000>,
+ <0x16400000 0x1000>;
+- interrupts = <0 154 0x0>;
++ interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>;
+ clock-names = "core", "iface";
+ status = "disabled";
+@@ -318,7 +320,7 @@
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0x16080000 0x1000>;
+- interrupts = <0 147 0>;
++ interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
+ spi-max-frequency = <24000000>;
+ cs-gpios = <&msmgpio 8 0>;
+
+diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi
+index 78ec496d5bc30..2b01bc29ddf23 100644
+--- a/arch/arm/boot/dts/qcom-msm8974.dtsi
++++ b/arch/arm/boot/dts/qcom-msm8974.dtsi
+@@ -1589,8 +1589,8 @@
+ #phy-cells = <0>;
+ qcom,dsi-phy-index = <0>;
+
+- clocks = <&mmcc MDSS_AHB_CLK>;
+- clock-names = "iface";
++ clocks = <&mmcc MDSS_AHB_CLK>, <&xo_board>;
++ clock-names = "iface", "ref";
+ };
+ };
+
+diff --git a/arch/arm/boot/dts/qcom-sdx55.dtsi b/arch/arm/boot/dts/qcom-sdx55.dtsi
+index 1e6ce035f76a9..b5b784c5c65e4 100644
+--- a/arch/arm/boot/dts/qcom-sdx55.dtsi
++++ b/arch/arm/boot/dts/qcom-sdx55.dtsi
+@@ -334,12 +334,10 @@
+ clocks = <&rpmhcc RPMH_IPA_CLK>;
+ clock-names = "core";
+
+- interconnects = <&system_noc MASTER_IPA &system_noc SLAVE_SNOC_MEM_NOC_GC>,
+- <&mem_noc MASTER_SNOC_GC_MEM_NOC &mc_virt SLAVE_EBI_CH0>,
++ interconnects = <&system_noc MASTER_IPA &mc_virt SLAVE_EBI_CH0>,
+ <&system_noc MASTER_IPA &system_noc SLAVE_OCIMEM>,
+ <&mem_noc MASTER_AMPSS_M0 &system_noc SLAVE_IPA_CFG>;
+- interconnect-names = "memory-a",
+- "memory-b",
++ interconnect-names = "memory",
+ "imem",
+ "config";
+
+diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi
+index 75af99c76d7ea..f31cf1df892b2 100644
+--- a/arch/arm/boot/dts/rk322x.dtsi
++++ b/arch/arm/boot/dts/rk322x.dtsi
+@@ -718,8 +718,8 @@
+ interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+ assigned-clocks = <&cru SCLK_HDMI_PHY>;
+ assigned-clock-parents = <&hdmi_phy>;
+- clocks = <&cru SCLK_HDMI_HDCP>, <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_CEC>;
+- clock-names = "isfr", "iahb", "cec";
++ clocks = <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_HDCP>, <&cru SCLK_HDMI_CEC>;
++ clock-names = "iahb", "isfr", "cec";
+ pinctrl-names = "default";
+ pinctrl-0 = <&hdmii2c_xfer &hdmi_hpd &hdmi_cec>;
+ resets = <&cru SRST_HDMI_P>;
+diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
+index 4dcdcf17c9776..66ff5db53c5a9 100644
+--- a/arch/arm/boot/dts/rk3288.dtsi
++++ b/arch/arm/boot/dts/rk3288.dtsi
+@@ -971,7 +971,7 @@
+ status = "disabled";
+ };
+
+- crypto: cypto-controller@ff8a0000 {
++ crypto: crypto@ff8a0000 {
+ compatible = "rockchip,rk3288-crypto";
+ reg = <0x0 0xff8a0000 0x0 0x4000>;
+ interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi
+index 801969c113d64..de88eb4847185 100644
+--- a/arch/arm/boot/dts/sama5d2.dtsi
++++ b/arch/arm/boot/dts/sama5d2.dtsi
+@@ -413,7 +413,7 @@
+ pmecc: ecc-engine@f8014070 {
+ compatible = "atmel,sama5d2-pmecc";
+ reg = <0xf8014070 0x490>,
+- <0xf8014500 0x100>;
++ <0xf8014500 0x200>;
+ };
+ };
+
+diff --git a/arch/arm/boot/dts/sama7g5-pinfunc.h b/arch/arm/boot/dts/sama7g5-pinfunc.h
+index 22fe9e522a97b..4eb30445d2057 100644
+--- a/arch/arm/boot/dts/sama7g5-pinfunc.h
++++ b/arch/arm/boot/dts/sama7g5-pinfunc.h
+@@ -765,7 +765,7 @@
+ #define PIN_PD20__PCK0 PINMUX_PIN(PIN_PD20, 1, 3)
+ #define PIN_PD20__FLEXCOM2_IO3 PINMUX_PIN(PIN_PD20, 2, 2)
+ #define PIN_PD20__PWMH3 PINMUX_PIN(PIN_PD20, 3, 4)
+-#define PIN_PD20__CANTX4 PINMUX_PIN(PIN_PD20, 5, 2)
++#define PIN_PD20__CANTX4 PINMUX_PIN(PIN_PD20, 4, 2)
+ #define PIN_PD20__FLEXCOM5_IO0 PINMUX_PIN(PIN_PD20, 6, 5)
+ #define PIN_PD21 117
+ #define PIN_PD21__GPIO PINMUX_PIN(PIN_PD21, 0, 0)
+diff --git a/arch/arm/boot/dts/sama7g5.dtsi b/arch/arm/boot/dts/sama7g5.dtsi
+index 6c58c151c6d9e..ac84d2e37f9b2 100644
+--- a/arch/arm/boot/dts/sama7g5.dtsi
++++ b/arch/arm/boot/dts/sama7g5.dtsi
+@@ -319,8 +319,6 @@
+ dmas = <&dma0 AT91_XDMAC_DT_PERID(7)>,
+ <&dma0 AT91_XDMAC_DT_PERID(8)>;
+ dma-names = "rx", "tx";
+- atmel,use-dma-rx;
+- atmel,use-dma-tx;
+ status = "disabled";
+ };
+ };
+@@ -485,8 +483,6 @@
+ dmas = <&dma0 AT91_XDMAC_DT_PERID(21)>,
+ <&dma0 AT91_XDMAC_DT_PERID(22)>;
+ dma-names = "rx", "tx";
+- atmel,use-dma-rx;
+- atmel,use-dma-tx;
+ status = "disabled";
+ };
+ };
+@@ -511,8 +507,6 @@
+ dmas = <&dma0 AT91_XDMAC_DT_PERID(23)>,
+ <&dma0 AT91_XDMAC_DT_PERID(24)>;
+ dma-names = "rx", "tx";
+- atmel,use-dma-rx;
+- atmel,use-dma-tx;
+ status = "disabled";
+ };
+ };
+diff --git a/arch/arm/boot/dts/socfpga.dtsi b/arch/arm/boot/dts/socfpga.dtsi
+index 0b021eef0b538..7c1d6423d7f8c 100644
+--- a/arch/arm/boot/dts/socfpga.dtsi
++++ b/arch/arm/boot/dts/socfpga.dtsi
+@@ -782,7 +782,7 @@
+ };
+
+ qspi: spi@ff705000 {
+- compatible = "cdns,qspi-nor";
++ compatible = "intel,socfpga-qspi", "cdns,qspi-nor";
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0xff705000 0x1000>,
+diff --git a/arch/arm/boot/dts/socfpga_arria10.dtsi b/arch/arm/boot/dts/socfpga_arria10.dtsi
+index a574ea91d9d3f..3ba431dfa8c94 100644
+--- a/arch/arm/boot/dts/socfpga_arria10.dtsi
++++ b/arch/arm/boot/dts/socfpga_arria10.dtsi
+@@ -756,7 +756,7 @@
+ };
+
+ qspi: spi@ff809000 {
+- compatible = "cdns,qspi-nor";
++ compatible = "intel,socfpga-qspi", "cdns,qspi-nor";
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0xff809000 0x100>,
+diff --git a/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts b/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts
+index 2b645642b9352..2a745522404d6 100644
+--- a/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts
++++ b/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts
+@@ -12,7 +12,7 @@
+ flash0: n25q00@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q00aa";
++ compatible = "micron,mt25qu02g", "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <100000000>;
+
+diff --git a/arch/arm/boot/dts/socfpga_arria5_socdk.dts b/arch/arm/boot/dts/socfpga_arria5_socdk.dts
+index 90e676e7019f2..1b02d46496a85 100644
+--- a/arch/arm/boot/dts/socfpga_arria5_socdk.dts
++++ b/arch/arm/boot/dts/socfpga_arria5_socdk.dts
+@@ -119,7 +119,7 @@
+ flash: flash@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q256a";
++ compatible = "micron,n25q256a", "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <100000000>;
+
+diff --git a/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts b/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
+index 6f138b2b26163..51bb436784e24 100644
+--- a/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
++++ b/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
+@@ -124,7 +124,7 @@
+ flash0: n25q00@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q00";
++ compatible = "micron,mt25qu02g", "jedec,spi-nor";
+ reg = <0>; /* chip select */
+ spi-max-frequency = <100000000>;
+
+diff --git a/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts b/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
+index c155ff02eb6e0..cae9ddd5ed38b 100644
+--- a/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
++++ b/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
+@@ -169,7 +169,7 @@
+ flash: flash@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q00";
++ compatible = "micron,mt25qu02g", "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <100000000>;
+
+diff --git a/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts b/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
+index 8d5d3996f6f27..ca18b959e6559 100644
+--- a/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
++++ b/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
+@@ -80,7 +80,7 @@
+ flash: flash@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q256a";
++ compatible = "micron,n25q256a", "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <100000000>;
+ m25p,fast-read;
+diff --git a/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts b/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts
+index 99a71757cdf46..3f7aa7bf0863a 100644
+--- a/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts
++++ b/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts
+@@ -116,7 +116,7 @@
+ flash0: n25q512a@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q512a";
++ compatible = "micron,n25q512a", "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <100000000>;
+
+diff --git a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
+index a060718758b67..25874e1b9c829 100644
+--- a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
++++ b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
+@@ -224,7 +224,7 @@
+ n25q128@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q128";
++ compatible = "micron,n25q128", "jedec,spi-nor";
+ reg = <0>; /* chip select */
+ spi-max-frequency = <100000000>;
+ m25p,fast-read;
+@@ -241,7 +241,7 @@
+ n25q00@1 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+- compatible = "n25q00";
++ compatible = "micron,mt25qu02g", "jedec,spi-nor";
+ reg = <1>; /* chip select */
+ spi-max-frequency = <100000000>;
+ m25p,fast-read;
+diff --git a/arch/arm/boot/dts/spear1340.dtsi b/arch/arm/boot/dts/spear1340.dtsi
+index 1a8f5e8b10e3a..66cd473ecb617 100644
+--- a/arch/arm/boot/dts/spear1340.dtsi
++++ b/arch/arm/boot/dts/spear1340.dtsi
+@@ -136,9 +136,9 @@
+ reg = <0xb4100000 0x1000>;
+ interrupts = <0 105 0x4>;
+ status = "disabled";
+- dmas = <&dwdma0 12 0 1>,
+- <&dwdma0 13 1 0>;
+- dma-names = "tx", "rx";
++ dmas = <&dwdma0 13 0 1>,
++ <&dwdma0 12 1 0>;
++ dma-names = "rx", "tx";
+ };
+
+ thermal@e07008c4 {
+diff --git a/arch/arm/boot/dts/spear13xx.dtsi b/arch/arm/boot/dts/spear13xx.dtsi
+index c87b881b2c8bb..9135533676879 100644
+--- a/arch/arm/boot/dts/spear13xx.dtsi
++++ b/arch/arm/boot/dts/spear13xx.dtsi
+@@ -284,9 +284,9 @@
+ #size-cells = <0>;
+ interrupts = <0 31 0x4>;
+ status = "disabled";
+- dmas = <&dwdma0 4 0 0>,
+- <&dwdma0 5 0 0>;
+- dma-names = "tx", "rx";
++ dmas = <&dwdma0 5 0 0>,
++ <&dwdma0 4 0 0>;
++ dma-names = "rx", "tx";
+ };
+
+ rtc@e0580000 {
+diff --git a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts
+index 264f3e9b5fce5..7fab746e0570e 100644
+--- a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts
++++ b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts
+@@ -181,10 +181,6 @@
+ cap-sd-highspeed;
+ cap-mmc-highspeed;
+ /* All direction control is used */
+- st,sig-dir-cmd;
+- st,sig-dir-dat0;
+- st,sig-dir-dat2;
+- st,sig-dir-dat31;
+ st,sig-pin-fbclk;
+ full-pwr-cycle;
+ vmmc-supply = <&ab8500_ldo_aux3_reg>;
+@@ -292,10 +288,10 @@
+ };
+
+ ab8500_ldo_aux2 {
+- /* Supplies the Cypress TMA140 touchscreen only with 3.3V */
++ /* Supplies the Cypress TMA140 touchscreen only with 3.0V */
+ regulator-name = "AUX2";
+- regulator-min-microvolt = <3300000>;
+- regulator-max-microvolt = <3300000>;
++ regulator-min-microvolt = <3000000>;
++ regulator-max-microvolt = <3000000>;
+ };
+
+ ab8500_ldo_aux3 {
+@@ -314,9 +310,9 @@
+
+ ab8500_ldo_aux5 {
+ regulator-name = "AUX5";
++ /* Intended for 1V8 for touchscreen but actually left unused */
+ regulator-min-microvolt = <1050000>;
+ regulator-max-microvolt = <2790000>;
+- regulator-always-on;
+ };
+
+ ab8500_ldo_aux6 {
+diff --git a/arch/arm/boot/dts/stm32f429-disco.dts b/arch/arm/boot/dts/stm32f429-disco.dts
+index 075ac57d0bf4a..6435e099c6326 100644
+--- a/arch/arm/boot/dts/stm32f429-disco.dts
++++ b/arch/arm/boot/dts/stm32f429-disco.dts
+@@ -192,7 +192,7 @@
+
+ display: display@1{
+ /* Connect panel-ilitek-9341 to ltdc */
+- compatible = "st,sf-tc240t-9370-t";
++ compatible = "st,sf-tc240t-9370-t", "ilitek,ili9341";
+ reg = <1>;
+ spi-3wire;
+ spi-max-frequency = <10000000>;
+diff --git a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
+index 5b60ecbd718f0..2ebafe27a865b 100644
+--- a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
++++ b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
+@@ -1179,7 +1179,7 @@
+ };
+ };
+
+- sai2a_pins_c: sai2a-4 {
++ sai2a_pins_c: sai2a-2 {
+ pins {
+ pinmux = <STM32_PINMUX('D', 13, AF10)>, /* SAI2_SCK_A */
+ <STM32_PINMUX('D', 11, AF10)>, /* SAI2_SD_A */
+@@ -1190,7 +1190,7 @@
+ };
+ };
+
+- sai2a_sleep_pins_c: sai2a-5 {
++ sai2a_sleep_pins_c: sai2a-2 {
+ pins {
+ pinmux = <STM32_PINMUX('D', 13, ANALOG)>, /* SAI2_SCK_A */
+ <STM32_PINMUX('D', 11, ANALOG)>, /* SAI2_SD_A */
+@@ -1235,14 +1235,14 @@
+ };
+ };
+
+- sai2b_pins_c: sai2a-4 {
++ sai2b_pins_c: sai2b-2 {
+ pins1 {
+ pinmux = <STM32_PINMUX('F', 11, AF10)>; /* SAI2_SD_B */
+ bias-disable;
+ };
+ };
+
+- sai2b_sleep_pins_c: sai2a-sleep-5 {
++ sai2b_sleep_pins_c: sai2b-sleep-2 {
+ pins {
+ pinmux = <STM32_PINMUX('F', 11, ANALOG)>; /* SAI2_SD_B */
+ };
+diff --git a/arch/arm/boot/dts/stm32mp151.dtsi b/arch/arm/boot/dts/stm32mp151.dtsi
+index bd289bf5d2690..6992a4b0ba79b 100644
+--- a/arch/arm/boot/dts/stm32mp151.dtsi
++++ b/arch/arm/boot/dts/stm32mp151.dtsi
+@@ -824,7 +824,7 @@
+ #sound-dai-cells = <0>;
+
+ compatible = "st,stm32-sai-sub-a";
+- reg = <0x4 0x1c>;
++ reg = <0x4 0x20>;
+ clocks = <&rcc SAI1_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 87 0x400 0x01>;
+@@ -834,7 +834,7 @@
+ sai1b: audio-controller@4400a024 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-b";
+- reg = <0x24 0x1c>;
++ reg = <0x24 0x20>;
+ clocks = <&rcc SAI1_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 88 0x400 0x01>;
+@@ -855,7 +855,7 @@
+ sai2a: audio-controller@4400b004 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-a";
+- reg = <0x4 0x1c>;
++ reg = <0x4 0x20>;
+ clocks = <&rcc SAI2_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 89 0x400 0x01>;
+@@ -865,7 +865,7 @@
+ sai2b: audio-controller@4400b024 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-b";
+- reg = <0x24 0x1c>;
++ reg = <0x24 0x20>;
+ clocks = <&rcc SAI2_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 90 0x400 0x01>;
+@@ -886,7 +886,7 @@
+ sai3a: audio-controller@4400c004 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-a";
+- reg = <0x04 0x1c>;
++ reg = <0x04 0x20>;
+ clocks = <&rcc SAI3_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 113 0x400 0x01>;
+@@ -896,7 +896,7 @@
+ sai3b: audio-controller@4400c024 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-b";
+- reg = <0x24 0x1c>;
++ reg = <0x24 0x20>;
+ clocks = <&rcc SAI3_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 114 0x400 0x01>;
+@@ -1271,7 +1271,7 @@
+ sai4a: audio-controller@50027004 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-a";
+- reg = <0x04 0x1c>;
++ reg = <0x04 0x20>;
+ clocks = <&rcc SAI4_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 99 0x400 0x01>;
+@@ -1281,7 +1281,7 @@
+ sai4b: audio-controller@50027024 {
+ #sound-dai-cells = <0>;
+ compatible = "st,stm32-sai-sub-b";
+- reg = <0x24 0x1c>;
++ reg = <0x24 0x20>;
+ clocks = <&rcc SAI4_K>;
+ clock-names = "sai_ck";
+ dmas = <&dmamux1 100 0x400 0x01>;
+diff --git a/arch/arm/boot/dts/stm32mp15xx-dhcor-som.dtsi b/arch/arm/boot/dts/stm32mp15xx-dhcor-som.dtsi
+index 2b0ac605549d7..44ecc47085871 100644
+--- a/arch/arm/boot/dts/stm32mp15xx-dhcor-som.dtsi
++++ b/arch/arm/boot/dts/stm32mp15xx-dhcor-som.dtsi
+@@ -202,7 +202,7 @@
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-rx-bus-width = <4>;
+- spi-max-frequency = <108000000>;
++ spi-max-frequency = <50000000>;
+ #address-cells = <1>;
+ #size-cells = <1>;
+ };
+diff --git a/arch/arm/boot/dts/stm32mp15xx-dkx.dtsi b/arch/arm/boot/dts/stm32mp15xx-dkx.dtsi
+index 899bfe04aeb91..48beed0f1f30a 100644
+--- a/arch/arm/boot/dts/stm32mp15xx-dkx.dtsi
++++ b/arch/arm/boot/dts/stm32mp15xx-dkx.dtsi
+@@ -249,7 +249,7 @@
+ stusb1600@28 {
+ compatible = "st,stusb1600";
+ reg = <0x28>;
+- interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
++ interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+ interrupt-parent = <&gpioi>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&stusb1600_pins_a>;
+diff --git a/arch/arm/boot/dts/sun8i-a33.dtsi b/arch/arm/boot/dts/sun8i-a33.dtsi
+index 2beddbb3c5183..b3d1bdfb5118e 100644
+--- a/arch/arm/boot/dts/sun8i-a33.dtsi
++++ b/arch/arm/boot/dts/sun8i-a33.dtsi
+@@ -46,7 +46,7 @@
+ #include <dt-bindings/thermal/thermal.h>
+
+ / {
+- cpu0_opp_table: opp_table0 {
++ cpu0_opp_table: opp-table-cpu {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+@@ -164,7 +164,7 @@
+ io-channels = <&ths>;
+ };
+
+- mali_opp_table: gpu-opp-table {
++ mali_opp_table: opp-table-gpu {
+ compatible = "operating-points-v2";
+
+ opp-144000000 {
+diff --git a/arch/arm/boot/dts/sun8i-a83t.dtsi b/arch/arm/boot/dts/sun8i-a83t.dtsi
+index ac97eac91349b..82fdb04122caa 100644
+--- a/arch/arm/boot/dts/sun8i-a83t.dtsi
++++ b/arch/arm/boot/dts/sun8i-a83t.dtsi
+@@ -200,7 +200,7 @@
+ status = "disabled";
+ };
+
+- cpu0_opp_table: opp_table0 {
++ cpu0_opp_table: opp-table-cluster0 {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+@@ -253,7 +253,7 @@
+ };
+ };
+
+- cpu1_opp_table: opp_table1 {
++ cpu1_opp_table: opp-table-cluster1 {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+diff --git a/arch/arm/boot/dts/sun8i-h3.dtsi b/arch/arm/boot/dts/sun8i-h3.dtsi
+index 4e89701df91f8..ae4f933abb895 100644
+--- a/arch/arm/boot/dts/sun8i-h3.dtsi
++++ b/arch/arm/boot/dts/sun8i-h3.dtsi
+@@ -44,7 +44,7 @@
+ #include <dt-bindings/thermal/thermal.h>
+
+ / {
+- cpu0_opp_table: opp_table0 {
++ cpu0_opp_table: opp-table-cpu {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+@@ -112,7 +112,7 @@
+ };
+ };
+
+- gpu_opp_table: gpu-opp-table {
++ gpu_opp_table: opp-table-gpu {
+ compatible = "operating-points-v2";
+
+ opp-120000000 {
+diff --git a/arch/arm/boot/dts/sun8i-v3s.dtsi b/arch/arm/boot/dts/sun8i-v3s.dtsi
+index b30bc1a25ebb9..084323d5c61cb 100644
+--- a/arch/arm/boot/dts/sun8i-v3s.dtsi
++++ b/arch/arm/boot/dts/sun8i-v3s.dtsi
+@@ -593,6 +593,17 @@
+ #size-cells = <0>;
+ };
+
++ gic: interrupt-controller@1c81000 {
++ compatible = "arm,gic-400";
++ reg = <0x01c81000 0x1000>,
++ <0x01c82000 0x2000>,
++ <0x01c84000 0x2000>,
++ <0x01c86000 0x2000>;
++ interrupt-controller;
++ #interrupt-cells = <3>;
++ interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
++ };
++
+ csi1: camera@1cb4000 {
+ compatible = "allwinner,sun8i-v3s-csi";
+ reg = <0x01cb4000 0x3000>;
+@@ -604,16 +615,5 @@
+ resets = <&ccu RST_BUS_CSI>;
+ status = "disabled";
+ };
+-
+- gic: interrupt-controller@1c81000 {
+- compatible = "arm,gic-400";
+- reg = <0x01c81000 0x1000>,
+- <0x01c82000 0x2000>,
+- <0x01c84000 0x2000>,
+- <0x01c86000 0x2000>;
+- interrupt-controller;
+- #interrupt-cells = <3>;
+- interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+- };
+ };
+ };
+diff --git a/arch/arm/boot/dts/tegra124-nyan-big.dts b/arch/arm/boot/dts/tegra124-nyan-big.dts
+index 1d2aac2cb6d03..fdc1d64dfff9d 100644
+--- a/arch/arm/boot/dts/tegra124-nyan-big.dts
++++ b/arch/arm/boot/dts/tegra124-nyan-big.dts
+@@ -13,12 +13,15 @@
+ "google,nyan-big-rev1", "google,nyan-big-rev0",
+ "google,nyan-big", "google,nyan", "nvidia,tegra124";
+
+- panel: panel {
+- compatible = "auo,b133xtn01";
+-
+- power-supply = <&vdd_3v3_panel>;
+- backlight = <&backlight>;
+- ddc-i2c-bus = <&dpaux>;
++ host1x@50000000 {
++ dpaux@545c0000 {
++ aux-bus {
++ panel: panel {
++ compatible = "auo,b133xtn01";
++ backlight = <&backlight>;
++ };
++ };
++ };
+ };
+
+ mmc@700b0400 { /* SD Card on this bus */
+diff --git a/arch/arm/boot/dts/tegra124-nyan-blaze.dts b/arch/arm/boot/dts/tegra124-nyan-blaze.dts
+index 677babde6460e..abdf4456826f8 100644
+--- a/arch/arm/boot/dts/tegra124-nyan-blaze.dts
++++ b/arch/arm/boot/dts/tegra124-nyan-blaze.dts
+@@ -15,12 +15,15 @@
+ "google,nyan-blaze-rev0", "google,nyan-blaze",
+ "google,nyan", "nvidia,tegra124";
+
+- panel: panel {
+- compatible = "samsung,ltn140at29-301";
+-
+- power-supply = <&vdd_3v3_panel>;
+- backlight = <&backlight>;
+- ddc-i2c-bus = <&dpaux>;
++ host1x@50000000 {
++ dpaux@545c0000 {
++ aux-bus {
++ panel: panel {
++ compatible = "samsung,ltn140at29-301";
++ backlight = <&backlight>;
++ };
++ };
++ };
+ };
+
+ sound {
+diff --git a/arch/arm/boot/dts/tegra124-venice2.dts b/arch/arm/boot/dts/tegra124-venice2.dts
+index e6b54ac1ebd1a..84e2d24065e9a 100644
+--- a/arch/arm/boot/dts/tegra124-venice2.dts
++++ b/arch/arm/boot/dts/tegra124-venice2.dts
+@@ -48,6 +48,13 @@
+ dpaux@545c0000 {
+ vdd-supply = <&vdd_3v3_panel>;
+ status = "okay";
++
++ aux-bus {
++ panel: panel {
++ compatible = "lg,lp129qe";
++ backlight = <&backlight>;
++ };
++ };
+ };
+ };
+
+@@ -1079,13 +1086,6 @@
+ };
+ };
+
+- panel: panel {
+- compatible = "lg,lp129qe";
+- power-supply = <&vdd_3v3_panel>;
+- backlight = <&backlight>;
+- ddc-i2c-bus = <&dpaux>;
+- };
+-
+ vdd_mux: regulator@0 {
+ compatible = "regulator-fixed";
+ regulator-name = "+VDD_MUX";
+diff --git a/arch/arm/boot/dts/tegra20-tamonten.dtsi b/arch/arm/boot/dts/tegra20-tamonten.dtsi
+index dd4d506683de7..7f14f0d005c3e 100644
+--- a/arch/arm/boot/dts/tegra20-tamonten.dtsi
++++ b/arch/arm/boot/dts/tegra20-tamonten.dtsi
+@@ -183,8 +183,8 @@
+ };
+ conf_ata {
+ nvidia,pins = "ata", "atb", "atc", "atd", "ate",
+- "cdev1", "cdev2", "dap1", "dtb", "gma",
+- "gmb", "gmc", "gmd", "gme", "gpu7",
++ "cdev1", "cdev2", "dap1", "dtb", "dtf",
++ "gma", "gmb", "gmc", "gmd", "gme", "gpu7",
+ "gpv", "i2cp", "irrx", "irtx", "pta",
+ "rm", "slxa", "slxk", "spia", "spib",
+ "uac";
+@@ -203,7 +203,7 @@
+ };
+ conf_crtp {
+ nvidia,pins = "crtp", "dap2", "dap3", "dap4",
+- "dtc", "dte", "dtf", "gpu", "sdio1",
++ "dtc", "dte", "gpu", "sdio1",
+ "slxc", "slxd", "spdi", "spdo", "spig",
+ "uda";
+ nvidia,pull = <TEGRA_PIN_PULL_NONE>;
+diff --git a/arch/arm/configs/cm_x300_defconfig b/arch/arm/configs/cm_x300_defconfig
+index 502a9d870ca44..45769d0ddd4ef 100644
+--- a/arch/arm/configs/cm_x300_defconfig
++++ b/arch/arm/configs/cm_x300_defconfig
+@@ -146,7 +146,6 @@ CONFIG_NFS_V3_ACL=y
+ CONFIG_NFS_V4=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_PARTITION_ADVANCED=y
+ CONFIG_NLS_CODEPAGE_437=m
+ CONFIG_NLS_ISO8859_1=m
+diff --git a/arch/arm/configs/ezx_defconfig b/arch/arm/configs/ezx_defconfig
+index a49e699e52de3..ec84d80096b1c 100644
+--- a/arch/arm/configs/ezx_defconfig
++++ b/arch/arm/configs/ezx_defconfig
+@@ -314,7 +314,6 @@ CONFIG_NFSD_V3_ACL=y
+ CONFIG_SMB_FS=m
+ CONFIG_CIFS=m
+ CONFIG_CIFS_STATS=y
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/arm/configs/imote2_defconfig b/arch/arm/configs/imote2_defconfig
+index 118c4c927f264..6db871d4e0775 100644
+--- a/arch/arm/configs/imote2_defconfig
++++ b/arch/arm/configs/imote2_defconfig
+@@ -288,7 +288,6 @@ CONFIG_NFSD_V3_ACL=y
+ CONFIG_SMB_FS=m
+ CONFIG_CIFS=m
+ CONFIG_CIFS_STATS=y
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
+index 80a3ae02d7594..e883cb6dc6459 100644
+--- a/arch/arm/configs/multi_v5_defconfig
++++ b/arch/arm/configs/multi_v5_defconfig
+@@ -188,6 +188,7 @@ CONFIG_REGULATOR=y
+ CONFIG_REGULATOR_FIXED_VOLTAGE=y
+ CONFIG_MEDIA_SUPPORT=y
+ CONFIG_MEDIA_CAMERA_SUPPORT=y
++CONFIG_MEDIA_PLATFORM_SUPPORT=y
+ CONFIG_V4L_PLATFORM_DRIVERS=y
+ CONFIG_VIDEO_ASPEED=m
+ CONFIG_VIDEO_ATMEL_ISI=m
+@@ -195,6 +196,7 @@ CONFIG_DRM=y
+ CONFIG_DRM_ATMEL_HLCDC=m
+ CONFIG_DRM_PANEL_SIMPLE=y
+ CONFIG_DRM_ASPEED_GFX=m
++CONFIG_FB=y
+ CONFIG_FB_IMX=y
+ CONFIG_FB_ATMEL=y
+ CONFIG_BACKLIGHT_ATMEL_LCDC=y
+diff --git a/arch/arm/configs/nhk8815_defconfig b/arch/arm/configs/nhk8815_defconfig
+index 23595fc5a29a9..907d6512821ad 100644
+--- a/arch/arm/configs/nhk8815_defconfig
++++ b/arch/arm/configs/nhk8815_defconfig
+@@ -127,7 +127,6 @@ CONFIG_NFS_FS=y
+ CONFIG_NFS_V3_ACL=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_NLS_CODEPAGE_437=y
+ CONFIG_NLS_ASCII=y
+ CONFIG_NLS_ISO8859_1=y
+diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
+index 58f4834289e63..dedaaae3d0d8a 100644
+--- a/arch/arm/configs/pxa_defconfig
++++ b/arch/arm/configs/pxa_defconfig
+@@ -699,7 +699,6 @@ CONFIG_NFSD_V3_ACL=y
+ CONFIG_NFSD_V4=y
+ CONFIG_CIFS=m
+ CONFIG_CIFS_STATS=y
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_DEFAULT="utf8"
+diff --git a/arch/arm/configs/spear13xx_defconfig b/arch/arm/configs/spear13xx_defconfig
+index 3b206a31902ff..065553326b391 100644
+--- a/arch/arm/configs/spear13xx_defconfig
++++ b/arch/arm/configs/spear13xx_defconfig
+@@ -61,7 +61,6 @@ CONFIG_SERIAL_AMBA_PL011=y
+ CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
+ # CONFIG_HW_RANDOM is not set
+ CONFIG_RAW_DRIVER=y
+-CONFIG_MAX_RAW_DEVS=8192
+ CONFIG_I2C=y
+ CONFIG_I2C_DESIGNWARE_PLATFORM=y
+ CONFIG_SPI=y
+diff --git a/arch/arm/configs/spear3xx_defconfig b/arch/arm/configs/spear3xx_defconfig
+index fc5f71c765edc..afca722d6605c 100644
+--- a/arch/arm/configs/spear3xx_defconfig
++++ b/arch/arm/configs/spear3xx_defconfig
+@@ -41,7 +41,6 @@ CONFIG_SERIAL_AMBA_PL011=y
+ CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
+ # CONFIG_HW_RANDOM is not set
+ CONFIG_RAW_DRIVER=y
+-CONFIG_MAX_RAW_DEVS=8192
+ CONFIG_I2C=y
+ CONFIG_I2C_DESIGNWARE_PLATFORM=y
+ CONFIG_SPI=y
+diff --git a/arch/arm/configs/spear6xx_defconfig b/arch/arm/configs/spear6xx_defconfig
+index 52a56b8ce6a71..bc32c02cb86b1 100644
+--- a/arch/arm/configs/spear6xx_defconfig
++++ b/arch/arm/configs/spear6xx_defconfig
+@@ -36,7 +36,6 @@ CONFIG_INPUT_FF_MEMLESS=y
+ CONFIG_SERIAL_AMBA_PL011=y
+ CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
+ CONFIG_RAW_DRIVER=y
+-CONFIG_MAX_RAW_DEVS=8192
+ CONFIG_I2C=y
+ CONFIG_I2C_DESIGNWARE_PLATFORM=y
+ CONFIG_SPI=y
+diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
+index 2b575792363e5..e4dba5461cb3e 100644
+--- a/arch/arm/crypto/Kconfig
++++ b/arch/arm/crypto/Kconfig
+@@ -102,6 +102,8 @@ config CRYPTO_AES_ARM_BS
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_SKCIPHER
+ select CRYPTO_LIB_AES
++ select CRYPTO_AES
++ select CRYPTO_CBC
+ select CRYPTO_SIMD
+ help
+ Use a faster and more secure NEON based implementation of AES in CBC,
+diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
+index eafa898ba6a73..0274f81cc8ea0 100644
+--- a/arch/arm/crypto/Makefile
++++ b/arch/arm/crypto/Makefile
+@@ -10,6 +10,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
+ obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
+ obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
+ obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += blake2s-arm.o
++obj-$(if $(CONFIG_CRYPTO_BLAKE2S_ARM),y) += libblake2s-arm.o
+ obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o
+ obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o
+ obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o
+@@ -31,7 +32,8 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o
+ sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y)
+ sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o
+ sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y)
+-blake2s-arm-y := blake2s-core.o blake2s-glue.o
++blake2s-arm-y := blake2s-shash.o
++libblake2s-arm-y:= blake2s-core.o blake2s-glue.o
+ blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o
+ sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o
+ sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o
+diff --git a/arch/arm/crypto/blake2s-core.S b/arch/arm/crypto/blake2s-core.S
+index 86345751bbf3a..df40e46601f10 100644
+--- a/arch/arm/crypto/blake2s-core.S
++++ b/arch/arm/crypto/blake2s-core.S
+@@ -167,8 +167,8 @@
+ .endm
+
+ //
+-// void blake2s_compress_arch(struct blake2s_state *state,
+-// const u8 *block, size_t nblocks, u32 inc);
++// void blake2s_compress(struct blake2s_state *state,
++// const u8 *block, size_t nblocks, u32 inc);
+ //
+ // Only the first three fields of struct blake2s_state are used:
+ // u32 h[8]; (inout)
+@@ -176,7 +176,7 @@
+ // u32 f[2]; (in)
+ //
+ .align 5
+-ENTRY(blake2s_compress_arch)
++ENTRY(blake2s_compress)
+ push {r0-r2,r4-r11,lr} // keep this an even number
+
+ .Lnext_block:
+@@ -303,4 +303,4 @@ ENTRY(blake2s_compress_arch)
+ str r3, [r12], #4
+ bne 1b
+ b .Lcopy_block_done
+-ENDPROC(blake2s_compress_arch)
++ENDPROC(blake2s_compress)
+diff --git a/arch/arm/crypto/blake2s-glue.c b/arch/arm/crypto/blake2s-glue.c
+index f2cc1e5fc9ec1..0238a70d9581e 100644
+--- a/arch/arm/crypto/blake2s-glue.c
++++ b/arch/arm/crypto/blake2s-glue.c
+@@ -1,78 +1,7 @@
+ // SPDX-License-Identifier: GPL-2.0-or-later
+-/*
+- * BLAKE2s digest algorithm, ARM scalar implementation
+- *
+- * Copyright 2020 Google LLC
+- */
+
+ #include <crypto/internal/blake2s.h>
+-#include <crypto/internal/hash.h>
+-
+ #include <linux/module.h>
+
+ /* defined in blake2s-core.S */
+-EXPORT_SYMBOL(blake2s_compress_arch);
+-
+-static int crypto_blake2s_update_arm(struct shash_desc *desc,
+- const u8 *in, unsigned int inlen)
+-{
+- return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch);
+-}
+-
+-static int crypto_blake2s_final_arm(struct shash_desc *desc, u8 *out)
+-{
+- return crypto_blake2s_final(desc, out, blake2s_compress_arch);
+-}
+-
+-#define BLAKE2S_ALG(name, driver_name, digest_size) \
+- { \
+- .base.cra_name = name, \
+- .base.cra_driver_name = driver_name, \
+- .base.cra_priority = 200, \
+- .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
+- .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \
+- .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \
+- .base.cra_module = THIS_MODULE, \
+- .digestsize = digest_size, \
+- .setkey = crypto_blake2s_setkey, \
+- .init = crypto_blake2s_init, \
+- .update = crypto_blake2s_update_arm, \
+- .final = crypto_blake2s_final_arm, \
+- .descsize = sizeof(struct blake2s_state), \
+- }
+-
+-static struct shash_alg blake2s_arm_algs[] = {
+- BLAKE2S_ALG("blake2s-128", "blake2s-128-arm", BLAKE2S_128_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-160", "blake2s-160-arm", BLAKE2S_160_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-224", "blake2s-224-arm", BLAKE2S_224_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-256", "blake2s-256-arm", BLAKE2S_256_HASH_SIZE),
+-};
+-
+-static int __init blake2s_arm_mod_init(void)
+-{
+- return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
+- crypto_register_shashes(blake2s_arm_algs,
+- ARRAY_SIZE(blake2s_arm_algs)) : 0;
+-}
+-
+-static void __exit blake2s_arm_mod_exit(void)
+-{
+- if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
+- crypto_unregister_shashes(blake2s_arm_algs,
+- ARRAY_SIZE(blake2s_arm_algs));
+-}
+-
+-module_init(blake2s_arm_mod_init);
+-module_exit(blake2s_arm_mod_exit);
+-
+-MODULE_DESCRIPTION("BLAKE2s digest algorithm, ARM scalar implementation");
+-MODULE_LICENSE("GPL");
+-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
+-MODULE_ALIAS_CRYPTO("blake2s-128");
+-MODULE_ALIAS_CRYPTO("blake2s-128-arm");
+-MODULE_ALIAS_CRYPTO("blake2s-160");
+-MODULE_ALIAS_CRYPTO("blake2s-160-arm");
+-MODULE_ALIAS_CRYPTO("blake2s-224");
+-MODULE_ALIAS_CRYPTO("blake2s-224-arm");
+-MODULE_ALIAS_CRYPTO("blake2s-256");
+-MODULE_ALIAS_CRYPTO("blake2s-256-arm");
++EXPORT_SYMBOL(blake2s_compress);
+diff --git a/arch/arm/crypto/blake2s-shash.c b/arch/arm/crypto/blake2s-shash.c
+new file mode 100644
+index 0000000000000..763c73beea2d0
+--- /dev/null
++++ b/arch/arm/crypto/blake2s-shash.c
+@@ -0,0 +1,75 @@
++// SPDX-License-Identifier: GPL-2.0-or-later
++/*
++ * BLAKE2s digest algorithm, ARM scalar implementation
++ *
++ * Copyright 2020 Google LLC
++ */
++
++#include <crypto/internal/blake2s.h>
++#include <crypto/internal/hash.h>
++
++#include <linux/module.h>
++
++static int crypto_blake2s_update_arm(struct shash_desc *desc,
++ const u8 *in, unsigned int inlen)
++{
++ return crypto_blake2s_update(desc, in, inlen, false);
++}
++
++static int crypto_blake2s_final_arm(struct shash_desc *desc, u8 *out)
++{
++ return crypto_blake2s_final(desc, out, false);
++}
++
++#define BLAKE2S_ALG(name, driver_name, digest_size) \
++ { \
++ .base.cra_name = name, \
++ .base.cra_driver_name = driver_name, \
++ .base.cra_priority = 200, \
++ .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
++ .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \
++ .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \
++ .base.cra_module = THIS_MODULE, \
++ .digestsize = digest_size, \
++ .setkey = crypto_blake2s_setkey, \
++ .init = crypto_blake2s_init, \
++ .update = crypto_blake2s_update_arm, \
++ .final = crypto_blake2s_final_arm, \
++ .descsize = sizeof(struct blake2s_state), \
++ }
++
++static struct shash_alg blake2s_arm_algs[] = {
++ BLAKE2S_ALG("blake2s-128", "blake2s-128-arm", BLAKE2S_128_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-160", "blake2s-160-arm", BLAKE2S_160_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-224", "blake2s-224-arm", BLAKE2S_224_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-256", "blake2s-256-arm", BLAKE2S_256_HASH_SIZE),
++};
++
++static int __init blake2s_arm_mod_init(void)
++{
++ return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
++ crypto_register_shashes(blake2s_arm_algs,
++ ARRAY_SIZE(blake2s_arm_algs)) : 0;
++}
++
++static void __exit blake2s_arm_mod_exit(void)
++{
++ if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
++ crypto_unregister_shashes(blake2s_arm_algs,
++ ARRAY_SIZE(blake2s_arm_algs));
++}
++
++module_init(blake2s_arm_mod_init);
++module_exit(blake2s_arm_mod_exit);
++
++MODULE_DESCRIPTION("BLAKE2s digest algorithm, ARM scalar implementation");
++MODULE_LICENSE("GPL");
++MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
++MODULE_ALIAS_CRYPTO("blake2s-128");
++MODULE_ALIAS_CRYPTO("blake2s-128-arm");
++MODULE_ALIAS_CRYPTO("blake2s-160");
++MODULE_ALIAS_CRYPTO("blake2s-160-arm");
++MODULE_ALIAS_CRYPTO("blake2s-224");
++MODULE_ALIAS_CRYPTO("blake2s-224-arm");
++MODULE_ALIAS_CRYPTO("blake2s-256");
++MODULE_ALIAS_CRYPTO("blake2s-256-arm");
+diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
+index e2b1fd558bf3c..09c241280ed9c 100644
+--- a/arch/arm/include/asm/assembler.h
++++ b/arch/arm/include/asm/assembler.h
+@@ -107,6 +107,16 @@
+ .endm
+ #endif
+
++#if __LINUX_ARM_ARCH__ < 7
++ .macro dsb, args
++ mcr p15, 0, r0, c7, c10, 4
++ .endm
++
++ .macro isb, args
++ mcr p15, 0, r0, c7, c5, 4
++ .endm
++#endif
++
+ .macro asm_trace_hardirqs_off, save=1
+ #if defined(CONFIG_TRACE_IRQFLAGS)
+ .if \save
+@@ -259,6 +269,7 @@
+ */
+ #define ALT_UP(instr...) \
+ .pushsection ".alt.smp.init", "a" ;\
++ .align 2 ;\
+ .long 9998b - . ;\
+ 9997: instr ;\
+ .if . - 9997b == 2 ;\
+@@ -270,6 +281,7 @@
+ .popsection
+ #define ALT_UP_B(label) \
+ .pushsection ".alt.smp.init", "a" ;\
++ .align 2 ;\
+ .long 9998b - . ;\
+ W(b) . + (label - 9998b) ;\
+ .popsection
+diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h
+index a6f3b179e8a94..27218eabbf9a0 100644
+--- a/arch/arm/include/asm/efi.h
++++ b/arch/arm/include/asm/efi.h
+@@ -17,7 +17,6 @@
+
+ #ifdef CONFIG_EFI
+ void efi_init(void);
+-extern void efifb_setup_from_dmi(struct screen_info *si, const char *opt);
+
+ int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md);
+ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md);
+diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h
+index f74944c6fe8d3..79d246ac93ab6 100644
+--- a/arch/arm/include/asm/io.h
++++ b/arch/arm/include/asm/io.h
+@@ -436,6 +436,9 @@ extern void pci_iounmap(struct pci_dev *dev, void __iomem *addr);
+ #define ARCH_HAS_VALID_PHYS_ADDR_RANGE
+ extern int valid_phys_addr_range(phys_addr_t addr, size_t size);
+ extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size);
++extern bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size,
++ unsigned long flags);
++#define arch_memremap_can_ram_remap arch_memremap_can_ram_remap
+ #endif
+
+ /*
+diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h
+index 9e6b972863077..8aeff55aebfaf 100644
+--- a/arch/arm/include/asm/processor.h
++++ b/arch/arm/include/asm/processor.h
+@@ -96,6 +96,7 @@ unsigned long get_wchan(struct task_struct *p);
+ #define __ALT_SMP_ASM(smp, up) \
+ "9998: " smp "\n" \
+ " .pushsection \".alt.smp.init\", \"a\"\n" \
++ " .align 2\n" \
+ " .long 9998b - .\n" \
+ " " up "\n" \
+ " .popsection\n"
+diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h
+new file mode 100644
+index 0000000000000..85f9e538fb325
+--- /dev/null
++++ b/arch/arm/include/asm/spectre.h
+@@ -0,0 +1,38 @@
++/* SPDX-License-Identifier: GPL-2.0-only */
++
++#ifndef __ASM_SPECTRE_H
++#define __ASM_SPECTRE_H
++
++enum {
++ SPECTRE_UNAFFECTED,
++ SPECTRE_MITIGATED,
++ SPECTRE_VULNERABLE,
++};
++
++enum {
++ __SPECTRE_V2_METHOD_BPIALL,
++ __SPECTRE_V2_METHOD_ICIALLU,
++ __SPECTRE_V2_METHOD_SMC,
++ __SPECTRE_V2_METHOD_HVC,
++ __SPECTRE_V2_METHOD_LOOP8,
++};
++
++enum {
++ SPECTRE_V2_METHOD_BPIALL = BIT(__SPECTRE_V2_METHOD_BPIALL),
++ SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU),
++ SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC),
++ SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC),
++ SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8),
++};
++
++#ifdef CONFIG_GENERIC_CPU_VULNERABILITIES
++void spectre_v2_update_state(unsigned int state, unsigned int methods);
++#else
++static inline void spectre_v2_update_state(unsigned int state,
++ unsigned int methods)
++{}
++#endif
++
++int spectre_bhb_update_vectors(unsigned int method);
++
++#endif
+diff --git a/arch/arm/include/asm/timex.h b/arch/arm/include/asm/timex.h
+index 7c3b3671d6c25..6d1337c169cd3 100644
+--- a/arch/arm/include/asm/timex.h
++++ b/arch/arm/include/asm/timex.h
+@@ -11,5 +11,6 @@
+
+ typedef unsigned long cycles_t;
+ #define get_cycles() ({ cycles_t c; read_current_timer(&c) ? 0 : c; })
++#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
+
+ #endif
+diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
+index 36fbc33292526..32dbfd81f42a4 100644
+--- a/arch/arm/include/asm/uaccess.h
++++ b/arch/arm/include/asm/uaccess.h
+@@ -11,6 +11,7 @@
+ #include <linux/string.h>
+ #include <asm/memory.h>
+ #include <asm/domain.h>
++#include <asm/unaligned.h>
+ #include <asm/unified.h>
+ #include <asm/compiler.h>
+
+@@ -497,7 +498,10 @@ do { \
+ } \
+ default: __err = __get_user_bad(); break; \
+ } \
+- *(type *)(dst) = __val; \
++ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) \
++ put_unaligned(__val, (type *)(dst)); \
++ else \
++ *(type *)(dst) = __val; /* aligned by caller */ \
+ if (__err) \
+ goto err_label; \
+ } while (0)
+@@ -507,7 +511,9 @@ do { \
+ const type *__pk_ptr = (dst); \
+ unsigned long __dst = (unsigned long)__pk_ptr; \
+ int __err = 0; \
+- type __val = *(type *)src; \
++ type __val = IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) \
++ ? get_unaligned((type *)(src)) \
++ : *(type *)(src); /* aligned by caller */ \
+ switch (sizeof(type)) { \
+ case 1: __put_user_asm_byte(__val, __dst, __err, ""); break; \
+ case 2: __put_user_asm_half(__val, __dst, __err, ""); break; \
+diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h
+index 4a91428c324db..fad45c884e988 100644
+--- a/arch/arm/include/asm/vmlinux.lds.h
++++ b/arch/arm/include/asm/vmlinux.lds.h
+@@ -26,6 +26,19 @@
+ #define ARM_MMU_DISCARD(x) x
+ #endif
+
++/*
++ * ld.lld does not support NOCROSSREFS:
++ * https://github.com/ClangBuiltLinux/linux/issues/1609
++ */
++#ifdef CONFIG_LD_IS_LLD
++#define NOCROSSREFS
++#endif
++
++/* Set start/end symbol names to the LMA for the section */
++#define ARM_LMA(sym, section) \
++ sym##_start = LOADADDR(section); \
++ sym##_end = LOADADDR(section) + SIZEOF(section)
++
+ #define PROC_INFO \
+ . = ALIGN(4); \
+ __proc_info_begin = .; \
+@@ -110,19 +123,31 @@
+ * only thing that matters is their relative offsets
+ */
+ #define ARM_VECTORS \
+- __vectors_start = .; \
+- .vectors 0xffff0000 : AT(__vectors_start) { \
+- *(.vectors) \
++ __vectors_lma = .; \
++ OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { \
++ .vectors { \
++ *(.vectors) \
++ } \
++ .vectors.bhb.loop8 { \
++ *(.vectors.bhb.loop8) \
++ } \
++ .vectors.bhb.bpiall { \
++ *(.vectors.bhb.bpiall) \
++ } \
+ } \
+- . = __vectors_start + SIZEOF(.vectors); \
+- __vectors_end = .; \
++ ARM_LMA(__vectors, .vectors); \
++ ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); \
++ ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); \
++ . = __vectors_lma + SIZEOF(.vectors) + \
++ SIZEOF(.vectors.bhb.loop8) + \
++ SIZEOF(.vectors.bhb.bpiall); \
+ \
+- __stubs_start = .; \
+- .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \
++ __stubs_lma = .; \
++ .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { \
+ *(.stubs) \
+ } \
+- . = __stubs_start + SIZEOF(.stubs); \
+- __stubs_end = .; \
++ ARM_LMA(__stubs, .stubs); \
++ . = __stubs_lma + SIZEOF(.stubs); \
+ \
+ PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors));
+
+diff --git a/arch/arm/include/debug/imx-uart.h b/arch/arm/include/debug/imx-uart.h
+index c8eb83d4b8964..3edbb3c5b42bf 100644
+--- a/arch/arm/include/debug/imx-uart.h
++++ b/arch/arm/include/debug/imx-uart.h
+@@ -11,13 +11,6 @@
+ #define IMX1_UART_BASE_ADDR(n) IMX1_UART##n##_BASE_ADDR
+ #define IMX1_UART_BASE(n) IMX1_UART_BASE_ADDR(n)
+
+-#define IMX21_UART1_BASE_ADDR 0x1000a000
+-#define IMX21_UART2_BASE_ADDR 0x1000b000
+-#define IMX21_UART3_BASE_ADDR 0x1000c000
+-#define IMX21_UART4_BASE_ADDR 0x1000d000
+-#define IMX21_UART_BASE_ADDR(n) IMX21_UART##n##_BASE_ADDR
+-#define IMX21_UART_BASE(n) IMX21_UART_BASE_ADDR(n)
+-
+ #define IMX25_UART1_BASE_ADDR 0x43f90000
+ #define IMX25_UART2_BASE_ADDR 0x43f94000
+ #define IMX25_UART3_BASE_ADDR 0x5000c000
+@@ -26,6 +19,13 @@
+ #define IMX25_UART_BASE_ADDR(n) IMX25_UART##n##_BASE_ADDR
+ #define IMX25_UART_BASE(n) IMX25_UART_BASE_ADDR(n)
+
++#define IMX27_UART1_BASE_ADDR 0x1000a000
++#define IMX27_UART2_BASE_ADDR 0x1000b000
++#define IMX27_UART3_BASE_ADDR 0x1000c000
++#define IMX27_UART4_BASE_ADDR 0x1000d000
++#define IMX27_UART_BASE_ADDR(n) IMX27_UART##n##_BASE_ADDR
++#define IMX27_UART_BASE(n) IMX27_UART_BASE_ADDR(n)
++
+ #define IMX31_UART1_BASE_ADDR 0x43f90000
+ #define IMX31_UART2_BASE_ADDR 0x43f94000
+ #define IMX31_UART3_BASE_ADDR 0x5000c000
+@@ -112,10 +112,10 @@
+
+ #ifdef CONFIG_DEBUG_IMX1_UART
+ #define UART_PADDR IMX_DEBUG_UART_BASE(IMX1)
+-#elif defined(CONFIG_DEBUG_IMX21_IMX27_UART)
+-#define UART_PADDR IMX_DEBUG_UART_BASE(IMX21)
+ #elif defined(CONFIG_DEBUG_IMX25_UART)
+ #define UART_PADDR IMX_DEBUG_UART_BASE(IMX25)
++#elif defined(CONFIG_DEBUG_IMX27_UART)
++#define UART_PADDR IMX_DEBUG_UART_BASE(IMX27)
+ #elif defined(CONFIG_DEBUG_IMX31_UART)
+ #define UART_PADDR IMX_DEBUG_UART_BASE(IMX31)
+ #elif defined(CONFIG_DEBUG_IMX35_UART)
+diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
+index ae295a3bcfefd..6ef3b535b7bf7 100644
+--- a/arch/arm/kernel/Makefile
++++ b/arch/arm/kernel/Makefile
+@@ -106,4 +106,6 @@ endif
+
+ obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o
+
++obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o
++
+ extra-y := $(head-y) vmlinux.lds
+diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
+index 241b73d64df73..68261a83b7ad8 100644
+--- a/arch/arm/kernel/entry-armv.S
++++ b/arch/arm/kernel/entry-armv.S
+@@ -597,11 +597,9 @@ call_fpe:
+ tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2
+ reteq lr
+ and r8, r0, #0x00000f00 @ mask out CP number
+- THUMB( lsr r8, r8, #8 )
+ mov r7, #1
+- add r6, r10, #TI_USED_CP
+- ARM( strb r7, [r6, r8, lsr #8] ) @ set appropriate used_cp[]
+- THUMB( strb r7, [r6, r8] ) @ set appropriate used_cp[]
++ add r6, r10, r8, lsr #8 @ add used_cp[] array offset first
++ strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[]
+ #ifdef CONFIG_IWMMXT
+ @ Test if we need to give access to iWMMXt coprocessors
+ ldr r5, [r10, #TI_FLAGS]
+@@ -610,7 +608,7 @@ call_fpe:
+ bcs iwmmxt_task_enable
+ #endif
+ ARM( add pc, pc, r8, lsr #6 )
+- THUMB( lsl r8, r8, #2 )
++ THUMB( lsr r8, r8, #6 )
+ THUMB( add pc, r8 )
+ nop
+
+@@ -1002,12 +1000,11 @@ vector_\name:
+ sub lr, lr, #\correction
+ .endif
+
+- @
+- @ Save r0, lr_<exception> (parent PC) and spsr_<exception>
+- @ (parent CPSR)
+- @
++ @ Save r0, lr_<exception> (parent PC)
+ stmia sp, {r0, lr} @ save r0, lr
+- mrs lr, spsr
++
++ @ Save spsr_<exception> (parent CPSR)
++2: mrs lr, spsr
+ str lr, [sp, #8] @ save spsr
+
+ @
+@@ -1028,6 +1025,44 @@ vector_\name:
+ movs pc, lr @ branch to handler in SVC mode
+ ENDPROC(vector_\name)
+
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++ .subsection 1
++ .align 5
++vector_bhb_loop8_\name:
++ .if \correction
++ sub lr, lr, #\correction
++ .endif
++
++ @ Save r0, lr_<exception> (parent PC)
++ stmia sp, {r0, lr}
++
++ @ bhb workaround
++ mov r0, #8
++3: W(b) . + 4
++ subs r0, r0, #1
++ bne 3b
++ dsb
++ isb
++ b 2b
++ENDPROC(vector_bhb_loop8_\name)
++
++vector_bhb_bpiall_\name:
++ .if \correction
++ sub lr, lr, #\correction
++ .endif
++
++ @ Save r0, lr_<exception> (parent PC)
++ stmia sp, {r0, lr}
++
++ @ bhb workaround
++ mcr p15, 0, r0, c7, c5, 6 @ BPIALL
++ @ isb not needed due to "movs pc, lr" in the vector stub
++ @ which gives a "context synchronisation".
++ b 2b
++ENDPROC(vector_bhb_bpiall_\name)
++ .previous
++#endif
++
+ .align 2
+ @ handler addresses follow this label
+ 1:
+@@ -1036,6 +1071,10 @@ ENDPROC(vector_\name)
+ .section .stubs, "ax", %progbits
+ @ This must be the first word
+ .word vector_swi
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++ .word vector_bhb_loop8_swi
++ .word vector_bhb_bpiall_swi
++#endif
+
+ vector_rst:
+ ARM( swi SYS_ERROR0 )
+@@ -1150,8 +1189,10 @@ vector_addrexcptn:
+ * FIQ "NMI" handler
+ *-----------------------------------------------------------------------------
+ * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
+- * systems.
++ * systems. This must be the last vector stub, so lets place it in its own
++ * subsection.
+ */
++ .subsection 2
+ vector_stub fiq, FIQ_MODE, 4
+
+ .long __fiq_usr @ 0 (USR_26 / USR_32)
+@@ -1184,6 +1225,30 @@ vector_addrexcptn:
+ W(b) vector_irq
+ W(b) vector_fiq
+
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++ .section .vectors.bhb.loop8, "ax", %progbits
++.L__vectors_bhb_loop8_start:
++ W(b) vector_rst
++ W(b) vector_bhb_loop8_und
++ W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004
++ W(b) vector_bhb_loop8_pabt
++ W(b) vector_bhb_loop8_dabt
++ W(b) vector_addrexcptn
++ W(b) vector_bhb_loop8_irq
++ W(b) vector_bhb_loop8_fiq
++
++ .section .vectors.bhb.bpiall, "ax", %progbits
++.L__vectors_bhb_bpiall_start:
++ W(b) vector_rst
++ W(b) vector_bhb_bpiall_und
++ W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008
++ W(b) vector_bhb_bpiall_pabt
++ W(b) vector_bhb_bpiall_dabt
++ W(b) vector_addrexcptn
++ W(b) vector_bhb_bpiall_irq
++ W(b) vector_bhb_bpiall_fiq
++#endif
++
+ .data
+ .align 2
+
+diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
+index d9c99db50243f..fde7ac271b147 100644
+--- a/arch/arm/kernel/entry-common.S
++++ b/arch/arm/kernel/entry-common.S
+@@ -153,6 +153,29 @@ ENDPROC(ret_from_fork)
+ *-----------------------------------------------------------------------------
+ */
+
++ .align 5
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++ENTRY(vector_bhb_loop8_swi)
++ sub sp, sp, #PT_REGS_SIZE
++ stmia sp, {r0 - r12}
++ mov r8, #8
++1: b 2f
++2: subs r8, r8, #1
++ bne 1b
++ dsb
++ isb
++ b 3f
++ENDPROC(vector_bhb_loop8_swi)
++
++ .align 5
++ENTRY(vector_bhb_bpiall_swi)
++ sub sp, sp, #PT_REGS_SIZE
++ stmia sp, {r0 - r12}
++ mcr p15, 0, r8, c7, c5, 6 @ BPIALL
++ isb
++ b 3f
++ENDPROC(vector_bhb_bpiall_swi)
++#endif
+ .align 5
+ ENTRY(vector_swi)
+ #ifdef CONFIG_CPU_V7M
+@@ -160,6 +183,7 @@ ENTRY(vector_swi)
+ #else
+ sub sp, sp, #PT_REGS_SIZE
+ stmia sp, {r0 - r12} @ Calling r0 - r12
++3:
+ ARM( add r8, sp, #S_PC )
+ ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr
+ THUMB( mov r8, sp )
+diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S
+index a74289ebc8036..5f1b1ce10473a 100644
+--- a/arch/arm/kernel/entry-ftrace.S
++++ b/arch/arm/kernel/entry-ftrace.S
+@@ -22,10 +22,7 @@
+ * mcount can be thought of as a function called in the middle of a subroutine
+ * call. As such, it needs to be transparent for both the caller and the
+ * callee: the original lr needs to be restored when leaving mcount, and no
+- * registers should be clobbered. (In the __gnu_mcount_nc implementation, we
+- * clobber the ip register. This is OK because the ARM calling convention
+- * allows it to be clobbered in subroutines and doesn't use it to hold
+- * parameters.)
++ * registers should be clobbered.
+ *
+ * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}"
+ * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
+@@ -70,26 +67,25 @@
+
+ .macro __ftrace_regs_caller
+
+- sub sp, sp, #8 @ space for PC and CPSR OLD_R0,
++ str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0,
+ @ OLD_R0 will overwrite previous LR
+
+- add ip, sp, #12 @ move in IP the value of SP as it was
+- @ before the push {lr} of the mcount mechanism
++ ldr lr, [sp, #8] @ get previous LR
+
+- str lr, [sp, #0] @ store LR instead of PC
++ str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR
+
+- ldr lr, [sp, #8] @ get previous LR
++ str lr, [sp, #-4]! @ store previous LR as LR
+
+- str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR
++ add lr, sp, #16 @ move in LR the value of SP as it was
++ @ before the push {lr} of the mcount mechanism
+
+- stmdb sp!, {ip, lr}
+- stmdb sp!, {r0-r11, lr}
++ push {r0-r11, ip, lr}
+
+ @ stack content at this point:
+ @ 0 4 48 52 56 60 64 68 72
+- @ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 |
++ @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 |
+
+- mov r3, sp @ struct pt_regs*
++ mov r3, sp @ struct pt_regs*
+
+ ldr r2, =function_trace_op
+ ldr r2, [r2] @ pointer to the current
+@@ -112,11 +108,9 @@ ftrace_graph_regs_call:
+ #endif
+
+ @ pop saved regs
+- ldmia sp!, {r0-r12} @ restore r0 through r12
+- ldr ip, [sp, #8] @ restore PC
+- ldr lr, [sp, #4] @ restore LR
+- ldr sp, [sp, #0] @ restore SP
+- mov pc, ip @ return
++ pop {r0-r11, ip, lr} @ restore r0 through r12
++ ldr lr, [sp], #4 @ restore LR
++ ldr pc, [sp], #12
+ .endm
+
+ #ifdef CONFIG_FUNCTION_GRAPH_TRACER
+@@ -132,11 +126,9 @@ ftrace_graph_regs_call:
+ bl prepare_ftrace_return
+
+ @ pop registers saved in ftrace_regs_caller
+- ldmia sp!, {r0-r12} @ restore r0 through r12
+- ldr ip, [sp, #8] @ restore PC
+- ldr lr, [sp, #4] @ restore LR
+- ldr sp, [sp, #0] @ restore SP
+- mov pc, ip @ return
++ pop {r0-r11, ip, lr} @ restore r0 through r12
++ ldr lr, [sp], #4 @ restore LR
++ ldr pc, [sp], #12
+
+ .endm
+ #endif
+@@ -202,16 +194,17 @@ ftrace_graph_call\suffix:
+ .endm
+
+ .macro mcount_exit
+- ldmia sp!, {r0-r3, ip, lr}
+- ret ip
++ ldmia sp!, {r0-r3}
++ ldr lr, [sp, #4]
++ ldr pc, [sp], #8
+ .endm
+
+ ENTRY(__gnu_mcount_nc)
+ UNWIND(.fnstart)
+ #ifdef CONFIG_DYNAMIC_FTRACE
+- mov ip, lr
+- ldmia sp!, {lr}
+- ret ip
++ push {lr}
++ ldr lr, [sp, #4]
++ ldr pc, [sp], #8
+ #else
+ __mcount
+ #endif
+diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S
+index 0fc814bbc34b1..8796a69c78e00 100644
+--- a/arch/arm/kernel/head-nommu.S
++++ b/arch/arm/kernel/head-nommu.S
+@@ -114,6 +114,7 @@ ENTRY(secondary_startup)
+ add r12, r12, r10
+ ret r12
+ 1: bl __after_proc_init
++ ldr r7, __secondary_data @ reload r7
+ ldr sp, [r7, #12] @ set up the stack pointer
+ mov fp, #0
+ b secondary_start_kernel
+diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c
+index 7bd30c0a4280d..22f937e6f3ffb 100644
+--- a/arch/arm/kernel/kgdb.c
++++ b/arch/arm/kernel/kgdb.c
+@@ -154,22 +154,38 @@ static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int instr)
+ return 0;
+ }
+
+-static struct undef_hook kgdb_brkpt_hook = {
++static struct undef_hook kgdb_brkpt_arm_hook = {
+ .instr_mask = 0xffffffff,
+ .instr_val = KGDB_BREAKINST,
+- .cpsr_mask = MODE_MASK,
++ .cpsr_mask = PSR_T_BIT | MODE_MASK,
+ .cpsr_val = SVC_MODE,
+ .fn = kgdb_brk_fn
+ };
+
+-static struct undef_hook kgdb_compiled_brkpt_hook = {
++static struct undef_hook kgdb_brkpt_thumb_hook = {
++ .instr_mask = 0xffff,
++ .instr_val = KGDB_BREAKINST & 0xffff,
++ .cpsr_mask = PSR_T_BIT | MODE_MASK,
++ .cpsr_val = PSR_T_BIT | SVC_MODE,
++ .fn = kgdb_brk_fn
++};
++
++static struct undef_hook kgdb_compiled_brkpt_arm_hook = {
+ .instr_mask = 0xffffffff,
+ .instr_val = KGDB_COMPILED_BREAK,
+- .cpsr_mask = MODE_MASK,
++ .cpsr_mask = PSR_T_BIT | MODE_MASK,
+ .cpsr_val = SVC_MODE,
+ .fn = kgdb_compiled_brk_fn
+ };
+
++static struct undef_hook kgdb_compiled_brkpt_thumb_hook = {
++ .instr_mask = 0xffff,
++ .instr_val = KGDB_COMPILED_BREAK & 0xffff,
++ .cpsr_mask = PSR_T_BIT | MODE_MASK,
++ .cpsr_val = PSR_T_BIT | SVC_MODE,
++ .fn = kgdb_compiled_brk_fn
++};
++
+ static int __kgdb_notify(struct die_args *args, unsigned long cmd)
+ {
+ struct pt_regs *regs = args->regs;
+@@ -210,8 +226,10 @@ int kgdb_arch_init(void)
+ if (ret != 0)
+ return ret;
+
+- register_undef_hook(&kgdb_brkpt_hook);
+- register_undef_hook(&kgdb_compiled_brkpt_hook);
++ register_undef_hook(&kgdb_brkpt_arm_hook);
++ register_undef_hook(&kgdb_brkpt_thumb_hook);
++ register_undef_hook(&kgdb_compiled_brkpt_arm_hook);
++ register_undef_hook(&kgdb_compiled_brkpt_thumb_hook);
+
+ return 0;
+ }
+@@ -224,8 +242,10 @@ int kgdb_arch_init(void)
+ */
+ void kgdb_arch_exit(void)
+ {
+- unregister_undef_hook(&kgdb_brkpt_hook);
+- unregister_undef_hook(&kgdb_compiled_brkpt_hook);
++ unregister_undef_hook(&kgdb_brkpt_arm_hook);
++ unregister_undef_hook(&kgdb_brkpt_thumb_hook);
++ unregister_undef_hook(&kgdb_compiled_brkpt_arm_hook);
++ unregister_undef_hook(&kgdb_compiled_brkpt_thumb_hook);
+ unregister_die_notifier(&kgdb_notifier);
+ }
+
+diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c
+index 3b69a76d341e7..1626dfc6f6ce6 100644
+--- a/arch/arm/kernel/perf_callchain.c
++++ b/arch/arm/kernel/perf_callchain.c
+@@ -62,9 +62,10 @@ user_backtrace(struct frame_tail __user *tail,
+ void
+ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct frame_tail __user *tail;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -98,9 +99,10 @@ callchain_trace(struct stackframe *fr,
+ void
+ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct stackframe fr;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -111,18 +113,21 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re
+
+ unsigned long perf_instruction_pointer(struct pt_regs *regs)
+ {
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+- return perf_guest_cbs->get_guest_ip();
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
++ if (guest_cbs && guest_cbs->is_in_guest())
++ return guest_cbs->get_guest_ip();
+
+ return instruction_pointer(regs);
+ }
+
+ unsigned long perf_misc_flags(struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ int misc = 0;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+- if (perf_guest_cbs->is_user_mode())
++ if (guest_cbs && guest_cbs->is_in_guest()) {
++ if (guest_cbs->is_user_mode())
+ misc |= PERF_RECORD_MISC_GUEST_USER;
+ else
+ misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c
+new file mode 100644
+index 0000000000000..0dcefc36fb7a0
+--- /dev/null
++++ b/arch/arm/kernel/spectre.c
+@@ -0,0 +1,71 @@
++// SPDX-License-Identifier: GPL-2.0-only
++#include <linux/bpf.h>
++#include <linux/cpu.h>
++#include <linux/device.h>
++
++#include <asm/spectre.h>
++
++static bool _unprivileged_ebpf_enabled(void)
++{
++#ifdef CONFIG_BPF_SYSCALL
++ return !sysctl_unprivileged_bpf_disabled;
++#else
++ return false;
++#endif
++}
++
++ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
++ char *buf)
++{
++ return sprintf(buf, "Mitigation: __user pointer sanitization\n");
++}
++
++static unsigned int spectre_v2_state;
++static unsigned int spectre_v2_methods;
++
++void spectre_v2_update_state(unsigned int state, unsigned int method)
++{
++ if (state > spectre_v2_state)
++ spectre_v2_state = state;
++ spectre_v2_methods |= method;
++}
++
++ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
++ char *buf)
++{
++ const char *method;
++
++ if (spectre_v2_state == SPECTRE_UNAFFECTED)
++ return sprintf(buf, "%s\n", "Not affected");
++
++ if (spectre_v2_state != SPECTRE_MITIGATED)
++ return sprintf(buf, "%s\n", "Vulnerable");
++
++ if (_unprivileged_ebpf_enabled())
++ return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
++
++ switch (spectre_v2_methods) {
++ case SPECTRE_V2_METHOD_BPIALL:
++ method = "Branch predictor hardening";
++ break;
++
++ case SPECTRE_V2_METHOD_ICIALLU:
++ method = "I-cache invalidation";
++ break;
++
++ case SPECTRE_V2_METHOD_SMC:
++ case SPECTRE_V2_METHOD_HVC:
++ method = "Firmware call";
++ break;
++
++ case SPECTRE_V2_METHOD_LOOP8:
++ method = "History overwrite";
++ break;
++
++ default:
++ method = "Multiple mitigations";
++ break;
++ }
++
++ return sprintf(buf, "Mitigation: %s\n", method);
++}
+diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c
+index 76ea4178a55cb..8247749998259 100644
+--- a/arch/arm/kernel/stacktrace.c
++++ b/arch/arm/kernel/stacktrace.c
+@@ -53,18 +53,17 @@ int notrace unwind_frame(struct stackframe *frame)
+ return -EINVAL;
+
+ frame->sp = frame->fp;
+- frame->fp = *(unsigned long *)(fp);
+- frame->pc = frame->lr;
+- frame->lr = *(unsigned long *)(fp + 4);
++ frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
++ frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4));
+ #else
+ /* check current frame pointer is within bounds */
+ if (fp < low + 12 || fp > high - 4)
+ return -EINVAL;
+
+ /* restore the registers from the stack frame */
+- frame->fp = *(unsigned long *)(fp - 12);
+- frame->sp = *(unsigned long *)(fp - 8);
+- frame->pc = *(unsigned long *)(fp - 4);
++ frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 12));
++ frame->sp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 8));
++ frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 4));
+ #endif
+
+ return 0;
+diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c
+index 6166ba38bf994..b74bfcf94fb1a 100644
+--- a/arch/arm/kernel/swp_emulate.c
++++ b/arch/arm/kernel/swp_emulate.c
+@@ -195,7 +195,7 @@ static int swp_handler(struct pt_regs *regs, unsigned int instr)
+ destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data);
+
+ /* Check access in reasonable access range for both SWP and SWPB */
+- if (!access_ok((address & ~3), 4)) {
++ if (!access_ok((void __user *)(address & ~3), 4)) {
+ pr_debug("SWP{B} emulation: access to %p not allowed!\n",
+ (void *)address);
+ res = -EFAULT;
+diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
+index 195dff58bafc7..54abd8720ddef 100644
+--- a/arch/arm/kernel/traps.c
++++ b/arch/arm/kernel/traps.c
+@@ -30,6 +30,7 @@
+ #include <linux/atomic.h>
+ #include <asm/cacheflush.h>
+ #include <asm/exception.h>
++#include <asm/spectre.h>
+ #include <asm/unistd.h>
+ #include <asm/traps.h>
+ #include <asm/ptrace.h>
+@@ -574,7 +575,7 @@ do_cache_op(unsigned long start, unsigned long end, int flags)
+ if (end < start || flags)
+ return -EINVAL;
+
+- if (!access_ok(start, end - start))
++ if (!access_ok((void __user *)start, end - start))
+ return -EFAULT;
+
+ return __do_cache_op(start, end);
+@@ -787,10 +788,59 @@ static inline void __init kuser_init(void *vectors)
+ }
+ #endif
+
++#ifndef CONFIG_CPU_V7M
++static void copy_from_lma(void *vma, void *lma_start, void *lma_end)
++{
++ memcpy(vma, lma_start, lma_end - lma_start);
++}
++
++static void flush_vectors(void *vma, size_t offset, size_t size)
++{
++ unsigned long start = (unsigned long)vma + offset;
++ unsigned long end = start + size;
++
++ flush_icache_range(start, end);
++}
++
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++int spectre_bhb_update_vectors(unsigned int method)
++{
++ extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[];
++ extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[];
++ void *vec_start, *vec_end;
++
++ if (system_state > SYSTEM_SCHEDULING) {
++ pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n",
++ smp_processor_id());
++ return SPECTRE_VULNERABLE;
++ }
++
++ switch (method) {
++ case SPECTRE_V2_METHOD_LOOP8:
++ vec_start = __vectors_bhb_loop8_start;
++ vec_end = __vectors_bhb_loop8_end;
++ break;
++
++ case SPECTRE_V2_METHOD_BPIALL:
++ vec_start = __vectors_bhb_bpiall_start;
++ vec_end = __vectors_bhb_bpiall_end;
++ break;
++
++ default:
++ pr_err("CPU%u: unknown Spectre BHB state %d\n",
++ smp_processor_id(), method);
++ return SPECTRE_VULNERABLE;
++ }
++
++ copy_from_lma(vectors_page, vec_start, vec_end);
++ flush_vectors(vectors_page, 0, vec_end - vec_start);
++
++ return SPECTRE_MITIGATED;
++}
++#endif
++
+ void __init early_trap_init(void *vectors_base)
+ {
+-#ifndef CONFIG_CPU_V7M
+- unsigned long vectors = (unsigned long)vectors_base;
+ extern char __stubs_start[], __stubs_end[];
+ extern char __vectors_start[], __vectors_end[];
+ unsigned i;
+@@ -811,17 +861,20 @@ void __init early_trap_init(void *vectors_base)
+ * into the vector page, mapped at 0xffff0000, and ensure these
+ * are visible to the instruction stream.
+ */
+- memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start);
+- memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start);
++ copy_from_lma(vectors_base, __vectors_start, __vectors_end);
++ copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end);
+
+ kuser_init(vectors_base);
+
+- flush_icache_range(vectors, vectors + PAGE_SIZE * 2);
++ flush_vectors(vectors_base, 0, PAGE_SIZE * 2);
++}
+ #else /* ifndef CONFIG_CPU_V7M */
++void __init early_trap_init(void *vectors_base)
++{
+ /*
+ * on V7-M there is no need to copy the vector table to a dedicated
+ * memory area. The address is configurable and so a table in the kernel
+ * image can be used.
+ */
+-#endif
+ }
++#endif
+diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
+index 428012687a802..7f7f6bae21c2d 100644
+--- a/arch/arm/mach-davinci/board-da850-evm.c
++++ b/arch/arm/mach-davinci/board-da850-evm.c
+@@ -1101,11 +1101,13 @@ static int __init da850_evm_config_emac(void)
+ int ret;
+ u32 val;
+ struct davinci_soc_info *soc_info = &davinci_soc_info;
+- u8 rmii_en = soc_info->emac_pdata->rmii_en;
++ u8 rmii_en;
+
+ if (!machine_is_davinci_da850_evm())
+ return 0;
+
++ rmii_en = soc_info->emac_pdata->rmii_en;
++
+ cfg_chip3_base = DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP3_REG);
+
+ val = __raw_readl(cfg_chip3_base);
+diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig
+index 5a48abac6af49..4b554cc8fa58a 100644
+--- a/arch/arm/mach-exynos/Kconfig
++++ b/arch/arm/mach-exynos/Kconfig
+@@ -18,7 +18,6 @@ menuconfig ARCH_EXYNOS
+ select EXYNOS_PMU
+ select EXYNOS_SROM
+ select EXYNOS_PM_DOMAINS if PM_GENERIC_DOMAINS
+- select GPIOLIB
+ select HAVE_ARM_ARCH_TIMER if ARCH_EXYNOS5
+ select HAVE_ARM_SCU if SMP
+ select HAVE_S3C2410_I2C if I2C
+diff --git a/arch/arm/mach-iop32x/include/mach/entry-macro.S b/arch/arm/mach-iop32x/include/mach/entry-macro.S
+index 8e6766d4621eb..341e5d9a6616d 100644
+--- a/arch/arm/mach-iop32x/include/mach/entry-macro.S
++++ b/arch/arm/mach-iop32x/include/mach/entry-macro.S
+@@ -20,7 +20,7 @@
+ mrc p6, 0, \irqstat, c8, c0, 0 @ Read IINTSRC
+ cmp \irqstat, #0
+ clzne \irqnr, \irqstat
+- rsbne \irqnr, \irqnr, #31
++ rsbne \irqnr, \irqnr, #32
+ .endm
+
+ .macro arch_ret_to_user, tmp1, tmp2
+diff --git a/arch/arm/mach-iop32x/include/mach/irqs.h b/arch/arm/mach-iop32x/include/mach/irqs.h
+index c4e78df428e86..e09ae5f48aec5 100644
+--- a/arch/arm/mach-iop32x/include/mach/irqs.h
++++ b/arch/arm/mach-iop32x/include/mach/irqs.h
+@@ -9,6 +9,6 @@
+ #ifndef __IRQS_H
+ #define __IRQS_H
+
+-#define NR_IRQS 32
++#define NR_IRQS 33
+
+ #endif
+diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c
+index 2d48bf1398c10..d1e8824cbd824 100644
+--- a/arch/arm/mach-iop32x/irq.c
++++ b/arch/arm/mach-iop32x/irq.c
+@@ -32,14 +32,14 @@ static void intstr_write(u32 val)
+ static void
+ iop32x_irq_mask(struct irq_data *d)
+ {
+- iop32x_mask &= ~(1 << d->irq);
++ iop32x_mask &= ~(1 << (d->irq - 1));
+ intctl_write(iop32x_mask);
+ }
+
+ static void
+ iop32x_irq_unmask(struct irq_data *d)
+ {
+- iop32x_mask |= 1 << d->irq;
++ iop32x_mask |= 1 << (d->irq - 1);
+ intctl_write(iop32x_mask);
+ }
+
+@@ -65,7 +65,7 @@ void __init iop32x_init_irq(void)
+ machine_is_em7210())
+ *IOP3XX_PCIIRSR = 0x0f;
+
+- for (i = 0; i < NR_IRQS; i++) {
++ for (i = 1; i < NR_IRQS; i++) {
+ irq_set_chip_and_handler(i, &ext_chip, handle_level_irq);
+ irq_clear_status_flags(i, IRQ_NOREQUEST | IRQ_NOPROBE);
+ }
+diff --git a/arch/arm/mach-iop32x/irqs.h b/arch/arm/mach-iop32x/irqs.h
+index 69858e4e905d1..e1dfc8b4e7d7e 100644
+--- a/arch/arm/mach-iop32x/irqs.h
++++ b/arch/arm/mach-iop32x/irqs.h
+@@ -7,36 +7,40 @@
+ #ifndef __IOP32X_IRQS_H
+ #define __IOP32X_IRQS_H
+
++/* Interrupts in Linux start at 1, hardware starts at 0 */
++
++#define IOP_IRQ(x) ((x) + 1)
++
+ /*
+ * IOP80321 chipset interrupts
+ */
+-#define IRQ_IOP32X_DMA0_EOT 0
+-#define IRQ_IOP32X_DMA0_EOC 1
+-#define IRQ_IOP32X_DMA1_EOT 2
+-#define IRQ_IOP32X_DMA1_EOC 3
+-#define IRQ_IOP32X_AA_EOT 6
+-#define IRQ_IOP32X_AA_EOC 7
+-#define IRQ_IOP32X_CORE_PMON 8
+-#define IRQ_IOP32X_TIMER0 9
+-#define IRQ_IOP32X_TIMER1 10
+-#define IRQ_IOP32X_I2C_0 11
+-#define IRQ_IOP32X_I2C_1 12
+-#define IRQ_IOP32X_MESSAGING 13
+-#define IRQ_IOP32X_ATU_BIST 14
+-#define IRQ_IOP32X_PERFMON 15
+-#define IRQ_IOP32X_CORE_PMU 16
+-#define IRQ_IOP32X_BIU_ERR 17
+-#define IRQ_IOP32X_ATU_ERR 18
+-#define IRQ_IOP32X_MCU_ERR 19
+-#define IRQ_IOP32X_DMA0_ERR 20
+-#define IRQ_IOP32X_DMA1_ERR 21
+-#define IRQ_IOP32X_AA_ERR 23
+-#define IRQ_IOP32X_MSG_ERR 24
+-#define IRQ_IOP32X_SSP 25
+-#define IRQ_IOP32X_XINT0 27
+-#define IRQ_IOP32X_XINT1 28
+-#define IRQ_IOP32X_XINT2 29
+-#define IRQ_IOP32X_XINT3 30
+-#define IRQ_IOP32X_HPI 31
++#define IRQ_IOP32X_DMA0_EOT IOP_IRQ(0)
++#define IRQ_IOP32X_DMA0_EOC IOP_IRQ(1)
++#define IRQ_IOP32X_DMA1_EOT IOP_IRQ(2)
++#define IRQ_IOP32X_DMA1_EOC IOP_IRQ(3)
++#define IRQ_IOP32X_AA_EOT IOP_IRQ(6)
++#define IRQ_IOP32X_AA_EOC IOP_IRQ(7)
++#define IRQ_IOP32X_CORE_PMON IOP_IRQ(8)
++#define IRQ_IOP32X_TIMER0 IOP_IRQ(9)
++#define IRQ_IOP32X_TIMER1 IOP_IRQ(10)
++#define IRQ_IOP32X_I2C_0 IOP_IRQ(11)
++#define IRQ_IOP32X_I2C_1 IOP_IRQ(12)
++#define IRQ_IOP32X_MESSAGING IOP_IRQ(13)
++#define IRQ_IOP32X_ATU_BIST IOP_IRQ(14)
++#define IRQ_IOP32X_PERFMON IOP_IRQ(15)
++#define IRQ_IOP32X_CORE_PMU IOP_IRQ(16)
++#define IRQ_IOP32X_BIU_ERR IOP_IRQ(17)
++#define IRQ_IOP32X_ATU_ERR IOP_IRQ(18)
++#define IRQ_IOP32X_MCU_ERR IOP_IRQ(19)
++#define IRQ_IOP32X_DMA0_ERR IOP_IRQ(20)
++#define IRQ_IOP32X_DMA1_ERR IOP_IRQ(21)
++#define IRQ_IOP32X_AA_ERR IOP_IRQ(23)
++#define IRQ_IOP32X_MSG_ERR IOP_IRQ(24)
++#define IRQ_IOP32X_SSP IOP_IRQ(25)
++#define IRQ_IOP32X_XINT0 IOP_IRQ(27)
++#define IRQ_IOP32X_XINT1 IOP_IRQ(28)
++#define IRQ_IOP32X_XINT2 IOP_IRQ(29)
++#define IRQ_IOP32X_XINT3 IOP_IRQ(30)
++#define IRQ_IOP32X_HPI IOP_IRQ(31)
+
+ #endif
+diff --git a/arch/arm/mach-mmp/sram.c b/arch/arm/mach-mmp/sram.c
+index 6794e2db1ad5f..ecc46c31004f6 100644
+--- a/arch/arm/mach-mmp/sram.c
++++ b/arch/arm/mach-mmp/sram.c
+@@ -72,6 +72,8 @@ static int sram_probe(struct platform_device *pdev)
+ if (!info)
+ return -ENOMEM;
+
++ platform_set_drvdata(pdev, info);
++
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (res == NULL) {
+ dev_err(&pdev->dev, "no memory resource defined\n");
+@@ -107,8 +109,6 @@ static int sram_probe(struct platform_device *pdev)
+ list_add(&info->node, &sram_bank_list);
+ mutex_unlock(&sram_lock);
+
+- platform_set_drvdata(pdev, info);
+-
+ dev_info(&pdev->dev, "initialized\n");
+ return 0;
+
+@@ -127,17 +127,19 @@ static int sram_remove(struct platform_device *pdev)
+ struct sram_bank_info *info;
+
+ info = platform_get_drvdata(pdev);
+- if (info == NULL)
+- return -ENODEV;
+
+- mutex_lock(&sram_lock);
+- list_del(&info->node);
+- mutex_unlock(&sram_lock);
++ if (info->sram_size) {
++ mutex_lock(&sram_lock);
++ list_del(&info->node);
++ mutex_unlock(&sram_lock);
++
++ gen_pool_destroy(info->gpool);
++ iounmap(info->sram_virt);
++ kfree(info->pool_name);
++ }
+
+- gen_pool_destroy(info->gpool);
+- iounmap(info->sram_virt);
+- kfree(info->pool_name);
+ kfree(info);
++
+ return 0;
+ }
+
+diff --git a/arch/arm/mach-mstar/Kconfig b/arch/arm/mach-mstar/Kconfig
+index cd300eeedc206..0bf4d312bcfd9 100644
+--- a/arch/arm/mach-mstar/Kconfig
++++ b/arch/arm/mach-mstar/Kconfig
+@@ -3,6 +3,7 @@ menuconfig ARCH_MSTARV7
+ depends on ARCH_MULTI_V7
+ select ARM_GIC
+ select ARM_HEAVY_MB
++ select HAVE_ARM_ARCH_TIMER
+ select MST_IRQ
+ select MSTAR_MSC313_MPLL
+ help
+diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
+index 6daaa645ae5d9..21413a9b7b6c6 100644
+--- a/arch/arm/mach-omap2/display.c
++++ b/arch/arm/mach-omap2/display.c
+@@ -263,9 +263,9 @@ static int __init omapdss_init_of(void)
+ }
+
+ r = of_platform_populate(node, NULL, NULL, &pdev->dev);
++ put_device(&pdev->dev);
+ if (r) {
+ pr_err("Unable to populate DSS submodule devices\n");
+- put_device(&pdev->dev);
+ return r;
+ }
+
+diff --git a/arch/arm/mach-omap2/omap4-common.c b/arch/arm/mach-omap2/omap4-common.c
+index 5c3845730dbf5..0b80f8bcd3047 100644
+--- a/arch/arm/mach-omap2/omap4-common.c
++++ b/arch/arm/mach-omap2/omap4-common.c
+@@ -314,10 +314,12 @@ void __init omap_gic_of_init(void)
+
+ np = of_find_compatible_node(NULL, NULL, "arm,cortex-a9-gic");
+ gic_dist_base_addr = of_iomap(np, 0);
++ of_node_put(np);
+ WARN_ON(!gic_dist_base_addr);
+
+ np = of_find_compatible_node(NULL, NULL, "arm,cortex-a9-twd-timer");
+ twd_base = of_iomap(np, 0);
++ of_node_put(np);
+ WARN_ON(!twd_base);
+
+ skip_errata_init:
+diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
+index 0c2936c7a3799..a5e9cffcac10c 100644
+--- a/arch/arm/mach-omap2/omap_hwmod.c
++++ b/arch/arm/mach-omap2/omap_hwmod.c
+@@ -752,8 +752,10 @@ static int __init _init_clkctrl_providers(void)
+
+ for_each_matching_node(np, ti_clkctrl_match_table) {
+ ret = _setup_clkctrl_provider(np);
+- if (ret)
++ if (ret) {
++ of_node_put(np);
+ break;
++ }
+ }
+
+ return ret;
+diff --git a/arch/arm/mach-s3c/irq-s3c24xx.c b/arch/arm/mach-s3c/irq-s3c24xx.c
+index 3edc5f614eefc..c1c2f041ad3b1 100644
+--- a/arch/arm/mach-s3c/irq-s3c24xx.c
++++ b/arch/arm/mach-s3c/irq-s3c24xx.c
+@@ -361,11 +361,25 @@ static inline int s3c24xx_handle_intc(struct s3c_irq_intc *intc,
+ static asmlinkage void __exception_irq_entry s3c24xx_handle_irq(struct pt_regs *regs)
+ {
+ do {
+- if (likely(s3c_intc[0]))
+- if (s3c24xx_handle_intc(s3c_intc[0], regs, 0))
+- continue;
++ /*
++ * For platform based machines, neither ERR nor NULL can happen here.
++ * The s3c24xx_handle_irq() will be set as IRQ handler iff this succeeds:
++ *
++ * s3c_intc[0] = s3c24xx_init_intc()
++ *
++ * If this fails, the next calls to s3c24xx_init_intc() won't be executed.
++ *
++ * For DT machine, s3c_init_intc_of() could set the IRQ handler without
++ * setting s3c_intc[0] only if it was called with num_ctrl=0. There is no
++ * such code path, so again the s3c_intc[0] will have a valid pointer if
++ * set_handle_irq() is called.
++ *
++ * Therefore in s3c24xx_handle_irq(), the s3c_intc[0] is always something.
++ */
++ if (s3c24xx_handle_intc(s3c_intc[0], regs, 0))
++ continue;
+
+- if (s3c_intc[2])
++ if (!IS_ERR_OR_NULL(s3c_intc[2]))
+ if (s3c24xx_handle_intc(s3c_intc[2], regs, 64))
+ continue;
+
+diff --git a/arch/arm/mach-s3c/mach-jive.c b/arch/arm/mach-s3c/mach-jive.c
+index 0785638a9069b..7d15b84ae217e 100644
+--- a/arch/arm/mach-s3c/mach-jive.c
++++ b/arch/arm/mach-s3c/mach-jive.c
+@@ -236,11 +236,11 @@ static int __init jive_mtdset(char *options)
+ unsigned long set;
+
+ if (options == NULL || options[0] == '\0')
+- return 0;
++ return 1;
+
+ if (kstrtoul(options, 10, &set)) {
+ printk(KERN_ERR "failed to parse mtdset=%s\n", options);
+- return 0;
++ return 1;
+ }
+
+ switch (set) {
+@@ -255,7 +255,7 @@ static int __init jive_mtdset(char *options)
+ "using default.", set);
+ }
+
+- return 0;
++ return 1;
+ }
+
+ /* parse the mtdset= option given to the kernel command line */
+diff --git a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
+index ee949255ced3f..09ef73b99dd86 100644
+--- a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
++++ b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
+@@ -154,8 +154,10 @@ static int __init rcar_gen2_regulator_quirk(void)
+ return -ENODEV;
+
+ for_each_matching_node_and_match(np, rcar_gen2_quirk_match, &id) {
+- if (!of_device_is_available(np))
++ if (!of_device_is_available(np)) {
++ of_node_put(np);
+ break;
++ }
+
+ ret = of_property_read_u32(np, "reg", &addr);
+ if (ret) /* Skip invalid entry and continue */
+@@ -164,6 +166,7 @@ static int __init rcar_gen2_regulator_quirk(void)
+ quirk = kzalloc(sizeof(*quirk), GFP_KERNEL);
+ if (!quirk) {
+ ret = -ENOMEM;
++ of_node_put(np);
+ goto err_mem;
+ }
+
+diff --git a/arch/arm/mach-socfpga/Kconfig b/arch/arm/mach-socfpga/Kconfig
+index 43ddec677c0b3..594edf9bbea44 100644
+--- a/arch/arm/mach-socfpga/Kconfig
++++ b/arch/arm/mach-socfpga/Kconfig
+@@ -2,6 +2,7 @@
+ menuconfig ARCH_INTEL_SOCFPGA
+ bool "Altera SOCFPGA family"
+ depends on ARCH_MULTI_V7
++ select ARCH_HAS_RESET_CONTROLLER
+ select ARCH_SUPPORTS_BIG_ENDIAN
+ select ARM_AMBA
+ select ARM_GIC
+@@ -18,6 +19,7 @@ menuconfig ARCH_INTEL_SOCFPGA
+ select PL310_ERRATA_727915
+ select PL310_ERRATA_753970 if PL310
+ select PL310_ERRATA_769419
++ select RESET_CONTROLLER
+
+ if ARCH_INTEL_SOCFPGA
+ config SOCFPGA_SUSPEND
+diff --git a/arch/arm/mach-socfpga/core.h b/arch/arm/mach-socfpga/core.h
+index fc2608b18a0d0..18f01190dcfd4 100644
+--- a/arch/arm/mach-socfpga/core.h
++++ b/arch/arm/mach-socfpga/core.h
+@@ -33,7 +33,7 @@ extern void __iomem *sdr_ctl_base_addr;
+ u32 socfpga_sdram_self_refresh(u32 sdr_base);
+ extern unsigned int socfpga_sdram_self_refresh_sz;
+
+-extern char secondary_trampoline, secondary_trampoline_end;
++extern char secondary_trampoline[], secondary_trampoline_end[];
+
+ extern unsigned long socfpga_cpu1start_addr;
+
+diff --git a/arch/arm/mach-socfpga/platsmp.c b/arch/arm/mach-socfpga/platsmp.c
+index fbb80b883e5dd..201191cf68f32 100644
+--- a/arch/arm/mach-socfpga/platsmp.c
++++ b/arch/arm/mach-socfpga/platsmp.c
+@@ -20,14 +20,14 @@
+
+ static int socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle)
+ {
+- int trampoline_size = &secondary_trampoline_end - &secondary_trampoline;
++ int trampoline_size = secondary_trampoline_end - secondary_trampoline;
+
+ if (socfpga_cpu1start_addr) {
+ /* This will put CPU #1 into reset. */
+ writel(RSTMGR_MPUMODRST_CPU1,
+ rst_manager_base_addr + SOCFPGA_RSTMGR_MODMPURST);
+
+- memcpy(phys_to_virt(0), &secondary_trampoline, trampoline_size);
++ memcpy(phys_to_virt(0), secondary_trampoline, trampoline_size);
+
+ writel(__pa_symbol(secondary_startup),
+ sys_manager_base_addr + (socfpga_cpu1start_addr & 0x000000ff));
+@@ -45,12 +45,12 @@ static int socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle)
+
+ static int socfpga_a10_boot_secondary(unsigned int cpu, struct task_struct *idle)
+ {
+- int trampoline_size = &secondary_trampoline_end - &secondary_trampoline;
++ int trampoline_size = secondary_trampoline_end - secondary_trampoline;
+
+ if (socfpga_cpu1start_addr) {
+ writel(RSTMGR_MPUMODRST_CPU1, rst_manager_base_addr +
+ SOCFPGA_A10_RSTMGR_MODMPURST);
+- memcpy(phys_to_virt(0), &secondary_trampoline, trampoline_size);
++ memcpy(phys_to_virt(0), secondary_trampoline, trampoline_size);
+
+ writel(__pa_symbol(secondary_startup),
+ sys_manager_base_addr + (socfpga_cpu1start_addr & 0x00000fff));
+diff --git a/arch/arm/mach-vexpress/spc.c b/arch/arm/mach-vexpress/spc.c
+index 1da11bdb1dfbd..1c6500c4e6a17 100644
+--- a/arch/arm/mach-vexpress/spc.c
++++ b/arch/arm/mach-vexpress/spc.c
+@@ -580,7 +580,7 @@ static int __init ve_spc_clk_init(void)
+ }
+
+ cluster = topology_physical_package_id(cpu_dev->id);
+- if (init_opp_table[cluster])
++ if (cluster < 0 || init_opp_table[cluster])
+ continue;
+
+ if (ve_init_opp_table(cpu_dev))
+diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
+index 8355c38958942..f43cdc1cfbaed 100644
+--- a/arch/arm/mm/Kconfig
++++ b/arch/arm/mm/Kconfig
+@@ -750,7 +750,7 @@ config CPU_BIG_ENDIAN
+ config CPU_ENDIAN_BE8
+ bool
+ depends on CPU_BIG_ENDIAN
+- default CPU_V6 || CPU_V6K || CPU_V7
++ default CPU_V6 || CPU_V6K || CPU_V7 || CPU_V7M
+ help
+ Support for the BE-8 (big-endian) mode on ARMv6 and ARMv7 processors.
+
+@@ -830,6 +830,7 @@ config CPU_BPREDICT_DISABLE
+
+ config CPU_SPECTRE
+ bool
++ select GENERIC_CPU_VULNERABILITIES
+
+ config HARDEN_BRANCH_PREDICTOR
+ bool "Harden the branch predictor against aliasing attacks" if EXPERT
+@@ -850,6 +851,16 @@ config HARDEN_BRANCH_PREDICTOR
+
+ If unsure, say Y.
+
++config HARDEN_BRANCH_HISTORY
++ bool "Harden Spectre style attacks against branch history" if EXPERT
++ depends on CPU_SPECTRE
++ default y
++ help
++ Speculation attacks against some high-performance processors can
++ make use of branch history to influence future speculation. When
++ taking an exception, a sequence of branches overwrites the branch
++ history, or branch history is invalidated.
++
+ config TLS_REG_EMUL
+ bool
+ select NEED_KUSER_HELPERS
+diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
+index 80fb5a4a5c050..2660bdfcad4d0 100644
+--- a/arch/arm/mm/ioremap.c
++++ b/arch/arm/mm/ioremap.c
+@@ -479,3 +479,11 @@ void __init early_ioremap_init(void)
+ {
+ early_ioremap_setup();
+ }
++
++bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size,
++ unsigned long flags)
++{
++ unsigned long pfn = PHYS_PFN(offset);
++
++ return memblock_is_map_memory(pfn);
++}
+diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c
+index 9c348042a7244..4b1619584b23c 100644
+--- a/arch/arm/mm/kasan_init.c
++++ b/arch/arm/mm/kasan_init.c
+@@ -226,7 +226,7 @@ void __init kasan_init(void)
+ BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
+ pgd_index(KASAN_SHADOW_END));
+ memcpy(tmp_pmd_table,
+- pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
++ (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
+ sizeof(tmp_pmd_table));
+ set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
+ __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
+diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
+index a4e0060051070..5e2be37a198e2 100644
+--- a/arch/arm/mm/mmu.c
++++ b/arch/arm/mm/mmu.c
+@@ -212,12 +212,14 @@ early_param("ecc", early_ecc);
+ static int __init early_cachepolicy(char *p)
+ {
+ pr_warn("cachepolicy kernel parameter not supported without cp15\n");
++ return 0;
+ }
+ early_param("cachepolicy", early_cachepolicy);
+
+ static int __init noalign_setup(char *__unused)
+ {
+ pr_warn("noalign kernel parameter not supported without cp15\n");
++ return 1;
+ }
+ __setup("noalign", noalign_setup);
+
+@@ -390,9 +392,9 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
+ BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) < FIXADDR_START);
+ BUG_ON(idx >= __end_of_fixed_addresses);
+
+- /* we only support device mappings until pgprot_kernel has been set */
++ /* We support only device mappings before pgprot_kernel is set. */
+ if (WARN_ON(pgprot_val(prot) != pgprot_val(FIXMAP_PAGE_IO) &&
+- pgprot_val(pgprot_kernel) == 0))
++ pgprot_val(prot) && pgprot_val(pgprot_kernel) == 0))
+ return;
+
+ if (pgprot_val(prot))
+diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c
+index 114c05ab4dd91..fb9f3eb6bf483 100644
+--- a/arch/arm/mm/proc-v7-bugs.c
++++ b/arch/arm/mm/proc-v7-bugs.c
+@@ -6,8 +6,35 @@
+ #include <asm/cp15.h>
+ #include <asm/cputype.h>
+ #include <asm/proc-fns.h>
++#include <asm/spectre.h>
+ #include <asm/system_misc.h>
+
++#ifdef CONFIG_ARM_PSCI
++static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
++{
++ struct arm_smccc_res res;
++
++ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
++ ARM_SMCCC_ARCH_WORKAROUND_1, &res);
++
++ switch ((int)res.a0) {
++ case SMCCC_RET_SUCCESS:
++ return SPECTRE_MITIGATED;
++
++ case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
++ return SPECTRE_UNAFFECTED;
++
++ default:
++ return SPECTRE_VULNERABLE;
++ }
++}
++#else
++static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void)
++{
++ return SPECTRE_VULNERABLE;
++}
++#endif
++
+ #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
+
+@@ -36,13 +63,61 @@ static void __maybe_unused call_hvc_arch_workaround_1(void)
+ arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+ }
+
+-static void cpu_v7_spectre_init(void)
++static unsigned int spectre_v2_install_workaround(unsigned int method)
+ {
+ const char *spectre_v2_method = NULL;
+ int cpu = smp_processor_id();
+
+ if (per_cpu(harden_branch_predictor_fn, cpu))
+- return;
++ return SPECTRE_MITIGATED;
++
++ switch (method) {
++ case SPECTRE_V2_METHOD_BPIALL:
++ per_cpu(harden_branch_predictor_fn, cpu) =
++ harden_branch_predictor_bpiall;
++ spectre_v2_method = "BPIALL";
++ break;
++
++ case SPECTRE_V2_METHOD_ICIALLU:
++ per_cpu(harden_branch_predictor_fn, cpu) =
++ harden_branch_predictor_iciallu;
++ spectre_v2_method = "ICIALLU";
++ break;
++
++ case SPECTRE_V2_METHOD_HVC:
++ per_cpu(harden_branch_predictor_fn, cpu) =
++ call_hvc_arch_workaround_1;
++ cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
++ spectre_v2_method = "hypervisor";
++ break;
++
++ case SPECTRE_V2_METHOD_SMC:
++ per_cpu(harden_branch_predictor_fn, cpu) =
++ call_smc_arch_workaround_1;
++ cpu_do_switch_mm = cpu_v7_smc_switch_mm;
++ spectre_v2_method = "firmware";
++ break;
++ }
++
++ if (spectre_v2_method)
++ pr_info("CPU%u: Spectre v2: using %s workaround\n",
++ smp_processor_id(), spectre_v2_method);
++
++ return SPECTRE_MITIGATED;
++}
++#else
++static unsigned int spectre_v2_install_workaround(unsigned int method)
++{
++ pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n",
++ smp_processor_id());
++
++ return SPECTRE_VULNERABLE;
++}
++#endif
++
++static void cpu_v7_spectre_v2_init(void)
++{
++ unsigned int state, method = 0;
+
+ switch (read_cpuid_part()) {
+ case ARM_CPU_PART_CORTEX_A8:
+@@ -51,69 +126,133 @@ static void cpu_v7_spectre_init(void)
+ case ARM_CPU_PART_CORTEX_A17:
+ case ARM_CPU_PART_CORTEX_A73:
+ case ARM_CPU_PART_CORTEX_A75:
+- per_cpu(harden_branch_predictor_fn, cpu) =
+- harden_branch_predictor_bpiall;
+- spectre_v2_method = "BPIALL";
++ state = SPECTRE_MITIGATED;
++ method = SPECTRE_V2_METHOD_BPIALL;
+ break;
+
+ case ARM_CPU_PART_CORTEX_A15:
+ case ARM_CPU_PART_BRAHMA_B15:
+- per_cpu(harden_branch_predictor_fn, cpu) =
+- harden_branch_predictor_iciallu;
+- spectre_v2_method = "ICIALLU";
++ state = SPECTRE_MITIGATED;
++ method = SPECTRE_V2_METHOD_ICIALLU;
+ break;
+
+-#ifdef CONFIG_ARM_PSCI
+ case ARM_CPU_PART_BRAHMA_B53:
+ /* Requires no workaround */
++ state = SPECTRE_UNAFFECTED;
+ break;
++
+ default:
+ /* Other ARM CPUs require no workaround */
+- if (read_cpuid_implementor() == ARM_CPU_IMP_ARM)
++ if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) {
++ state = SPECTRE_UNAFFECTED;
+ break;
++ }
++
+ fallthrough;
+- /* Cortex A57/A72 require firmware workaround */
+- case ARM_CPU_PART_CORTEX_A57:
+- case ARM_CPU_PART_CORTEX_A72: {
+- struct arm_smccc_res res;
+
+- arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+- ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+- if ((int)res.a0 != 0)
+- return;
++ /* Cortex A57/A72 require firmware workaround */
++ case ARM_CPU_PART_CORTEX_A57:
++ case ARM_CPU_PART_CORTEX_A72:
++ state = spectre_v2_get_cpu_fw_mitigation_state();
++ if (state != SPECTRE_MITIGATED)
++ break;
+
+ switch (arm_smccc_1_1_get_conduit()) {
+ case SMCCC_CONDUIT_HVC:
+- per_cpu(harden_branch_predictor_fn, cpu) =
+- call_hvc_arch_workaround_1;
+- cpu_do_switch_mm = cpu_v7_hvc_switch_mm;
+- spectre_v2_method = "hypervisor";
++ method = SPECTRE_V2_METHOD_HVC;
+ break;
+
+ case SMCCC_CONDUIT_SMC:
+- per_cpu(harden_branch_predictor_fn, cpu) =
+- call_smc_arch_workaround_1;
+- cpu_do_switch_mm = cpu_v7_smc_switch_mm;
+- spectre_v2_method = "firmware";
++ method = SPECTRE_V2_METHOD_SMC;
+ break;
+
+ default:
++ state = SPECTRE_VULNERABLE;
+ break;
+ }
+ }
+-#endif
++
++ if (state == SPECTRE_MITIGATED)
++ state = spectre_v2_install_workaround(method);
++
++ spectre_v2_update_state(state, method);
++}
++
++#ifdef CONFIG_HARDEN_BRANCH_HISTORY
++static int spectre_bhb_method;
++
++static const char *spectre_bhb_method_name(int method)
++{
++ switch (method) {
++ case SPECTRE_V2_METHOD_LOOP8:
++ return "loop";
++
++ case SPECTRE_V2_METHOD_BPIALL:
++ return "BPIALL";
++
++ default:
++ return "unknown";
+ }
++}
+
+- if (spectre_v2_method)
+- pr_info("CPU%u: Spectre v2: using %s workaround\n",
+- smp_processor_id(), spectre_v2_method);
++static int spectre_bhb_install_workaround(int method)
++{
++ if (spectre_bhb_method != method) {
++ if (spectre_bhb_method) {
++ pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n",
++ smp_processor_id());
++
++ return SPECTRE_VULNERABLE;
++ }
++
++ if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE)
++ return SPECTRE_VULNERABLE;
++
++ spectre_bhb_method = method;
++ }
++
++ pr_info("CPU%u: Spectre BHB: using %s workaround\n",
++ smp_processor_id(), spectre_bhb_method_name(method));
++
++ return SPECTRE_MITIGATED;
+ }
+ #else
+-static void cpu_v7_spectre_init(void)
++static int spectre_bhb_install_workaround(int method)
+ {
++ return SPECTRE_VULNERABLE;
+ }
+ #endif
+
++static void cpu_v7_spectre_bhb_init(void)
++{
++ unsigned int state, method = 0;
++
++ switch (read_cpuid_part()) {
++ case ARM_CPU_PART_CORTEX_A15:
++ case ARM_CPU_PART_BRAHMA_B15:
++ case ARM_CPU_PART_CORTEX_A57:
++ case ARM_CPU_PART_CORTEX_A72:
++ state = SPECTRE_MITIGATED;
++ method = SPECTRE_V2_METHOD_LOOP8;
++ break;
++
++ case ARM_CPU_PART_CORTEX_A73:
++ case ARM_CPU_PART_CORTEX_A75:
++ state = SPECTRE_MITIGATED;
++ method = SPECTRE_V2_METHOD_BPIALL;
++ break;
++
++ default:
++ state = SPECTRE_UNAFFECTED;
++ break;
++ }
++
++ if (state == SPECTRE_MITIGATED)
++ state = spectre_bhb_install_workaround(method);
++
++ spectre_v2_update_state(state, method);
++}
++
+ static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
+ u32 mask, const char *msg)
+ {
+@@ -142,16 +281,18 @@ static bool check_spectre_auxcr(bool *warned, u32 bit)
+ void cpu_v7_ca8_ibe(void)
+ {
+ if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
+- cpu_v7_spectre_init();
++ cpu_v7_spectre_v2_init();
+ }
+
+ void cpu_v7_ca15_ibe(void)
+ {
+ if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
+- cpu_v7_spectre_init();
++ cpu_v7_spectre_v2_init();
++ cpu_v7_spectre_bhb_init();
+ }
+
+ void cpu_v7_bugs_init(void)
+ {
+- cpu_v7_spectre_init();
++ cpu_v7_spectre_v2_init();
++ cpu_v7_spectre_bhb_init();
+ }
+diff --git a/arch/arm/probes/kprobes/Makefile b/arch/arm/probes/kprobes/Makefile
+index 14db56f49f0a3..6159010dac4a6 100644
+--- a/arch/arm/probes/kprobes/Makefile
++++ b/arch/arm/probes/kprobes/Makefile
+@@ -1,4 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0
++KASAN_SANITIZE_actions-common.o := n
++KASAN_SANITIZE_actions-arm.o := n
++KASAN_SANITIZE_actions-thumb.o := n
+ obj-$(CONFIG_KPROBES) += core.o actions-common.o checkers-common.o
+ obj-$(CONFIG_ARM_KPROBES_TEST) += test-kprobes.o
+ test-kprobes-objs := test-core.o
+diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
+index fee914c716aa2..8b6f090e0364c 100644
+--- a/arch/arm64/Kconfig
++++ b/arch/arm64/Kconfig
+@@ -154,7 +154,6 @@ config ARM64
+ select HAVE_ARCH_KGDB
+ select HAVE_ARCH_MMAP_RND_BITS
+ select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
+- select HAVE_ARCH_PFN_VALID
+ select HAVE_ARCH_PREL32_RELOCATIONS
+ select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET
+ select HAVE_ARCH_SECCOMP_FILTER
+@@ -1053,9 +1052,6 @@ config HW_PERF_EVENTS
+ def_bool y
+ depends on ARM_PMU
+
+-config ARCH_HAS_FILTER_PGPROT
+- def_bool y
+-
+ # Supported by clang >= 7.0
+ config CC_HAVE_SHADOW_CALL_STACK
+ def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+@@ -1184,6 +1180,15 @@ config UNMAP_KERNEL_AT_EL0
+
+ If unsure, say Y.
+
++config MITIGATE_SPECTRE_BRANCH_HISTORY
++ bool "Mitigate Spectre style attacks against branch history" if EXPERT
++ default y
++ help
++ Speculation attacks against some high-performance processors can
++ make use of branch history to influence future speculation.
++ When taking an exception from user-space, a sequence of branches
++ or a firmware call overwrites the branch history.
++
+ config RODATA_FULL_DEFAULT_ENABLED
+ bool "Apply r/o permissions of VM areas also to their linear aliases"
+ default y
+@@ -1264,7 +1269,8 @@ config KUSER_HELPERS
+
+ config COMPAT_VDSO
+ bool "Enable vDSO for 32-bit applications"
+- depends on !CPU_BIG_ENDIAN && "$(CROSS_COMPILE_COMPAT)" != ""
++ depends on !CPU_BIG_ENDIAN
++ depends on (CC_IS_CLANG && LD_IS_LLD) || "$(CROSS_COMPILE_COMPAT)" != ""
+ select GENERIC_COMPAT_VDSO
+ default y
+ help
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
+index cc321c04f1219..f6d7d7f7fdabe 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
++++ b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
+@@ -343,19 +343,19 @@
+ };
+
+ thermal-zones {
+- cpu-thermal-zone {
++ cpu-thermal {
+ polling-delay-passive = <0>;
+ polling-delay = <0>;
+ thermal-sensors = <&ths 0>;
+ };
+
+- ddr-thermal-zone {
++ ddr-thermal {
+ polling-delay-passive = <0>;
+ polling-delay = <0>;
+ thermal-sensors = <&ths 2>;
+ };
+
+- gpu-thermal-zone {
++ gpu-thermal {
+ polling-delay-passive = <0>;
+ polling-delay = <0>;
+ thermal-sensors = <&ths 1>;
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-cpu-opp.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-cpu-opp.dtsi
+index 578c37490d901..e39db51eb4489 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-cpu-opp.dtsi
++++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-cpu-opp.dtsi
+@@ -4,7 +4,7 @@
+ */
+
+ / {
+- cpu0_opp_table: opp_table0 {
++ cpu0_opp_table: opp-table-cpu {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-cpu-opp.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h5-cpu-opp.dtsi
+index b2657201957eb..1afad8b437d72 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-cpu-opp.dtsi
++++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-cpu-opp.dtsi
+@@ -2,7 +2,7 @@
+ // Copyright (C) 2020 Chen-Yu Tsai <wens@csie.org>
+
+ / {
+- cpu_opp_table: cpu-opp-table {
++ cpu_opp_table: opp-table-cpu {
+ compatible = "operating-points-v2";
+ opp-shared;
+
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus.dts
+index d13980ed7a79a..7ec5ac850a0dc 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus.dts
++++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus.dts
+@@ -69,7 +69,7 @@
+ pinctrl-0 = <&emac_rgmii_pins>;
+ phy-supply = <&reg_gmac_3v3>;
+ phy-handle = <&ext_rgmii_phy>;
+- phy-mode = "rgmii";
++ phy-mode = "rgmii-id";
+ status = "okay";
+ };
+
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
+index 578a63dedf466..9988e87ea7b3d 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
++++ b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
+@@ -217,7 +217,7 @@
+ };
+ };
+
+- gpu_thermal {
++ gpu-thermal {
+ polling-delay-passive = <0>;
+ polling-delay = <0>;
+ thermal-sensors = <&ths 1>;
+diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-cpu-opp.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6-cpu-opp.dtsi
+index 8c6e8536b69fa..0baf0f8e4d272 100644
+--- a/arch/arm64/boot/dts/allwinner/sun50i-h6-cpu-opp.dtsi
++++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-cpu-opp.dtsi
+@@ -3,7 +3,7 @@
+ // Copyright (C) 2020 Clément Péron <peron.clem@gmail.com>
+
+ / {
+- cpu_opp_table: cpu-opp-table {
++ cpu_opp_table: opp-table-cpu {
+ compatible = "allwinner,sun50i-h6-operating-points";
+ nvmem-cells = <&cpu_speed_grade>;
+ opp-shared;
+diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
+index d301ac0d406bf..3ec301bd08a91 100644
+--- a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
++++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
+@@ -594,7 +594,7 @@
+ };
+
+ qspi: spi@ff8d2000 {
+- compatible = "cdns,qspi-nor";
++ compatible = "intel,socfpga-qspi", "cdns,qspi-nor";
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0xff8d2000 0x100>,
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi
+index 00c6f53290d43..a3a1ea0f21340 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi
+@@ -58,7 +58,7 @@
+ secure-monitor = <&sm>;
+ };
+
+- gpu_opp_table: gpu-opp-table {
++ gpu_opp_table: opp-table-gpu {
+ compatible = "operating-points-v2";
+
+ opp-124999998 {
+@@ -107,6 +107,12 @@
+ no-map;
+ };
+
++ /* 32 MiB reserved for ARM Trusted Firmware (BL32) */
++ secmon_reserved_bl32: secmon@5300000 {
++ reg = <0x0 0x05300000 0x0 0x2000000>;
++ no-map;
++ };
++
+ linux,cma {
+ compatible = "shared-dma-pool";
+ reusable;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts
+index 81269ccc24968..4fb31c2ba31c4 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-g12a-sei510.dts
+@@ -139,7 +139,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&dc_in>;
++ pwm-supply = <&dc_in>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -157,14 +157,6 @@
+ regulator-always-on;
+ };
+
+- reserved-memory {
+- /* TEE Reserved Memory */
+- bl32_reserved: bl32@5000000 {
+- reg = <0x0 0x05300000 0x0 0x2000000>;
+- no-map;
+- };
+- };
+-
+ sdio_pwrseq: sdio-pwrseq {
+ compatible = "mmc-pwrseq-simple";
+ reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-u200.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-u200.dts
+index a26bfe72550fe..4b5d11e56364d 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12a-u200.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-g12a-u200.dts
+@@ -139,7 +139,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
+index 579f3d02d613e..b4e86196e3468 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
+@@ -139,7 +139,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&dc_in>;
++ pwm-supply = <&dc_in>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-a311d.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-a311d.dtsi
+index d61f43052a344..8e9ad1e51d665 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12b-a311d.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12b-a311d.dtsi
+@@ -11,26 +11,6 @@
+ compatible = "operating-points-v2";
+ opp-shared;
+
+- opp-100000000 {
+- opp-hz = /bits/ 64 <100000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-250000000 {
+- opp-hz = /bits/ 64 <250000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-500000000 {
+- opp-hz = /bits/ 64 <500000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-667000000 {
+- opp-hz = /bits/ 64 <667000000>;
+- opp-microvolt = <731000>;
+- };
+-
+ opp-1000000000 {
+ opp-hz = /bits/ 64 <1000000000>;
+ opp-microvolt = <761000>;
+@@ -71,26 +51,6 @@
+ compatible = "operating-points-v2";
+ opp-shared;
+
+- opp-100000000 {
+- opp-hz = /bits/ 64 <100000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-250000000 {
+- opp-hz = /bits/ 64 <250000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-500000000 {
+- opp-hz = /bits/ 64 <500000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-667000000 {
+- opp-hz = /bits/ 64 <667000000>;
+- opp-microvolt = <731000>;
+- };
+-
+ opp-1000000000 {
+ opp-hz = /bits/ 64 <1000000000>;
+ opp-microvolt = <731000>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-khadas-vim3.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-khadas-vim3.dtsi
+index f42cf4b8af2d4..16dd409051b40 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12b-khadas-vim3.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12b-khadas-vim3.dtsi
+@@ -18,7 +18,7 @@
+ regulator-min-microvolt = <690000>;
+ regulator-max-microvolt = <1050000>;
+
+- vin-supply = <&dc_in>;
++ pwm-supply = <&dc_in>;
+
+ pwms = <&pwm_ab 0 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -37,7 +37,7 @@
+ regulator-min-microvolt = <690000>;
+ regulator-max-microvolt = <1050000>;
+
+- vin-supply = <&vsys_3v3>;
++ pwm-supply = <&vsys_3v3>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi
+index 344573e157a7b..d33e54b5e1969 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-n2.dtsi
+@@ -17,7 +17,7 @@
+ rtc1 = &vrtc;
+ };
+
+- dioo2133: audio-amplifier-0 {
++ dio2133: audio-amplifier-0 {
+ compatible = "simple-audio-amplifier";
+ enable-gpios = <&gpio_ao GPIOAO_2 GPIO_ACTIVE_HIGH>;
+ VCC-supply = <&vcc_5v>;
+@@ -130,7 +130,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_ab 0 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -149,7 +149,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -217,7 +217,7 @@
+ audio-widgets = "Line", "Lineout";
+ audio-aux-devs = <&tdmout_b>, <&tdmout_c>, <&tdmin_a>,
+ <&tdmin_b>, <&tdmin_c>, <&tdmin_lb>,
+- <&dioo2133>;
++ <&dio2133>;
+ audio-routing = "TDMOUT_B IN 0", "FRDDR_A OUT 1",
+ "TDMOUT_B IN 1", "FRDDR_B OUT 1",
+ "TDMOUT_B IN 2", "FRDDR_C OUT 1",
+@@ -607,7 +607,7 @@
+ pinctrl-0 = <&nor_pins>;
+ pinctrl-names = "default";
+
+- mx25u64: spi-flash@0 {
++ mx25u64: flash@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ compatible = "mxicy,mx25u6435f", "jedec,spi-nor";
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-s922x.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-s922x.dtsi
+index 1e5d0ee5d541b..44c23c984034c 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12b-s922x.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12b-s922x.dtsi
+@@ -11,26 +11,6 @@
+ compatible = "operating-points-v2";
+ opp-shared;
+
+- opp-100000000 {
+- opp-hz = /bits/ 64 <100000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-250000000 {
+- opp-hz = /bits/ 64 <250000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-500000000 {
+- opp-hz = /bits/ 64 <500000000>;
+- opp-microvolt = <731000>;
+- };
+-
+- opp-667000000 {
+- opp-hz = /bits/ 64 <667000000>;
+- opp-microvolt = <731000>;
+- };
+-
+ opp-1000000000 {
+ opp-hz = /bits/ 64 <1000000000>;
+ opp-microvolt = <731000>;
+@@ -76,26 +56,6 @@
+ compatible = "operating-points-v2";
+ opp-shared;
+
+- opp-100000000 {
+- opp-hz = /bits/ 64 <100000000>;
+- opp-microvolt = <751000>;
+- };
+-
+- opp-250000000 {
+- opp-hz = /bits/ 64 <250000000>;
+- opp-microvolt = <751000>;
+- };
+-
+- opp-500000000 {
+- opp-hz = /bits/ 64 <500000000>;
+- opp-microvolt = <751000>;
+- };
+-
+- opp-667000000 {
+- opp-hz = /bits/ 64 <667000000>;
+- opp-microvolt = <751000>;
+- };
+-
+ opp-1000000000 {
+ opp-hz = /bits/ 64 <1000000000>;
+ opp-microvolt = <771000>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-w400.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12b-w400.dtsi
+index feb0885047400..b40d2c1002c92 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-g12b-w400.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-g12b-w400.dtsi
+@@ -96,7 +96,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_ab 0 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -115,7 +115,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
+index 6b457b2c30a4b..aa14ea017a613 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
+@@ -49,6 +49,12 @@
+ no-map;
+ };
+
++ /* 32 MiB reserved for ARM Trusted Firmware (BL32) */
++ secmon_reserved_bl32: secmon@5300000 {
++ reg = <0x0 0x05300000 0x0 0x2000000>;
++ no-map;
++ };
++
+ linux,cma {
+ compatible = "shared-dma-pool";
+ reusable;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi
+index a350fee1264d7..a4d34398da358 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi
+@@ -6,6 +6,7 @@
+ */
+
+ #include "meson-gxbb.dtsi"
++#include <dt-bindings/gpio/gpio.h>
+
+ / {
+ aliases {
+@@ -64,6 +65,7 @@
+ regulator-name = "VDDIO_AO18";
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <1800000>;
++ regulator-always-on;
+ };
+
+ vcc_3v3: regulator-vcc_3v3 {
+@@ -161,6 +163,7 @@
+ status = "okay";
+ pinctrl-0 = <&hdmi_hpd_pins>, <&hdmi_i2c_pins>;
+ pinctrl-names = "default";
++ hdmi-supply = <&vddio_ao18>;
+ };
+
+ &hdmi_tx_tmds_port {
+diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts
+index effaa138b5f98..cadba194b149b 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-sm1-bananapi-m5.dts
+@@ -123,7 +123,7 @@
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+
+- enable-gpio = <&gpio GPIOE_2 GPIO_ACTIVE_HIGH>;
++ enable-gpio = <&gpio_ao GPIOE_2 GPIO_ACTIVE_HIGH>;
+ enable-active-high;
+ regulator-always-on;
+
+@@ -173,7 +173,7 @@
+ regulator-min-microvolt = <690000>;
+ regulator-max-microvolt = <1050000>;
+
+- vin-supply = <&dc_in>;
++ pwm-supply = <&dc_in>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -437,6 +437,7 @@
+ "",
+ "eMMC_RST#", /* BOOT_12 */
+ "eMMC_DS", /* BOOT_13 */
++ "", "",
+ /* GPIOC */
+ "SD_D0_B", /* GPIOC_0 */
+ "SD_D1_B", /* GPIOC_1 */
+diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-khadas-vim3l.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-khadas-vim3l.dts
+index f2c0981435944..9c0b544e22098 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-sm1-khadas-vim3l.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-sm1-khadas-vim3l.dts
+@@ -24,7 +24,7 @@
+ regulator-min-microvolt = <690000>;
+ regulator-max-microvolt = <1050000>;
+
+- vin-supply = <&vsys_3v3>;
++ pwm-supply = <&vsys_3v3>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi
+index fd0ad85c165ba..76ad052fbf0c9 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi
+@@ -48,7 +48,7 @@
+ regulator-max-microvolt = <3300000>;
+ vin-supply = <&vcc_5v>;
+
+- enable-gpio = <&gpio GPIOE_2 GPIO_ACTIVE_HIGH>;
++ enable-gpio = <&gpio_ao GPIOE_2 GPIO_OPEN_DRAIN>;
+ enable-active-high;
+ regulator-always-on;
+
+@@ -116,7 +116,7 @@
+ regulator-min-microvolt = <721000>;
+ regulator-max-microvolt = <1022000>;
+
+- vin-supply = <&main_12v>;
++ pwm-supply = <&main_12v>;
+
+ pwms = <&pwm_AO_cd 1 1250 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -263,6 +263,10 @@
+ reg = <0>;
+ max-speed = <1000>;
+
++ reset-assert-us = <10000>;
++ reset-deassert-us = <80000>;
++ reset-gpios = <&gpio GPIOZ_15 (GPIO_ACTIVE_LOW | GPIO_OPEN_DRAIN)>;
++
+ interrupt-parent = <&gpio_intc>;
+ /* MAC_INTR on GPIOZ_14 */
+ interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
+index 2194a778973f1..a5d79f2f7c196 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
++++ b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
+@@ -185,7 +185,7 @@
+ regulator-min-microvolt = <690000>;
+ regulator-max-microvolt = <1050000>;
+
+- vin-supply = <&dc_in>;
++ pwm-supply = <&dc_in>;
+
+ pwms = <&pwm_AO_cd 1 1500 0>;
+ pwm-dutycycle-range = <100 0>;
+@@ -203,14 +203,6 @@
+ regulator-always-on;
+ };
+
+- reserved-memory {
+- /* TEE Reserved Memory */
+- bl32_reserved: bl32@5000000 {
+- reg = <0x0 0x05300000 0x0 0x2000000>;
+- no-map;
+- };
+- };
+-
+ sdio_pwrseq: sdio-pwrseq {
+ compatible = "mmc-pwrseq-simple";
+ reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1.dtsi
+index 3d8b1f4f2001b..78bdbd2ccc9de 100644
+--- a/arch/arm64/boot/dts/amlogic/meson-sm1.dtsi
++++ b/arch/arm64/boot/dts/amlogic/meson-sm1.dtsi
+@@ -95,26 +95,6 @@
+ compatible = "operating-points-v2";
+ opp-shared;
+
+- opp-100000000 {
+- opp-hz = /bits/ 64 <100000000>;
+- opp-microvolt = <730000>;
+- };
+-
+- opp-250000000 {
+- opp-hz = /bits/ 64 <250000000>;
+- opp-microvolt = <730000>;
+- };
+-
+- opp-500000000 {
+- opp-hz = /bits/ 64 <500000000>;
+- opp-microvolt = <730000>;
+- };
+-
+- opp-667000000 {
+- opp-hz = /bits/ 64 <666666666>;
+- opp-microvolt = <750000>;
+- };
+-
+ opp-1000000000 {
+ opp-hz = /bits/ 64 <1000000000>;
+ opp-microvolt = <770000>;
+diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi
+index 6288e104a0893..a2635b14da309 100644
+--- a/arch/arm64/boot/dts/arm/juno-base.dtsi
++++ b/arch/arm64/boot/dts/arm/juno-base.dtsi
+@@ -543,8 +543,7 @@
+ <0x02000000 0x00 0x50000000 0x00 0x50000000 0x0 0x08000000>,
+ <0x42000000 0x40 0x00000000 0x40 0x00000000 0x1 0x00000000>;
+ /* Standard AXI Translation entries as programmed by EDK2 */
+- dma-ranges = <0x02000000 0x0 0x2c1c0000 0x0 0x2c1c0000 0x0 0x00040000>,
+- <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x80000000>,
++ dma-ranges = <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x80000000>,
+ <0x43000000 0x8 0x00000000 0x8 0x00000000 0x2 0x00000000>;
+ #interrupt-cells = <1>;
+ interrupt-map-mask = <0 0 0 7>;
+diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi
+index a5a64d17d9ea6..e8907d3fe2d11 100644
+--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi
++++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi
+@@ -273,9 +273,9 @@
+ #size-cells = <1>;
+ ranges = <0x00 0x00 0xff800000 0x3000>;
+
+- timer: timer@400 {
+- compatible = "brcm,bcm6328-timer", "syscon";
+- reg = <0x400 0x3c>;
++ twd: timer-mfd@400 {
++ compatible = "brcm,bcm4908-twd", "simple-mfd", "syscon";
++ reg = <0x400 0x4c>;
+ };
+
+ gpio0: gpio-controller@500 {
+@@ -292,7 +292,7 @@
+ reg = <0x640 0x18>;
+ interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&periph_clk>;
+- clock-names = "periph";
++ clock-names = "refclk";
+ status = "okay";
+ };
+
+@@ -326,12 +326,12 @@
+ #reset-cells = <1>;
+ };
+ };
++ };
+
+- reboot {
+- compatible = "syscon-reboot";
+- regmap = <&timer>;
+- offset = <0x34>;
+- mask = <1>;
+- };
++ reboot {
++ compatible = "syscon-reboot";
++ regmap = <&twd>;
++ offset = <0x34>;
++ mask = <1>;
+ };
+ };
+diff --git a/arch/arm64/boot/dts/broadcom/northstar2/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/northstar2/ns2-svk.dts
+index ec19fbf928a14..12a4b1c03390c 100644
+--- a/arch/arm64/boot/dts/broadcom/northstar2/ns2-svk.dts
++++ b/arch/arm64/boot/dts/broadcom/northstar2/ns2-svk.dts
+@@ -111,8 +111,8 @@
+ compatible = "silabs,si3226x";
+ reg = <0>;
+ spi-max-frequency = <5000000>;
+- spi-cpha = <1>;
+- spi-cpol = <1>;
++ spi-cpha;
++ spi-cpol;
+ pl022,hierarchy = <0>;
+ pl022,interface = <0>;
+ pl022,slave-tx-disable = <0>;
+@@ -135,8 +135,8 @@
+ at25,byte-len = <0x8000>;
+ at25,addr-mode = <2>;
+ at25,page-size = <64>;
+- spi-cpha = <1>;
+- spi-cpol = <1>;
++ spi-cpha;
++ spi-cpol;
+ pl022,hierarchy = <0>;
+ pl022,interface = <0>;
+ pl022,slave-tx-disable = <0>;
+diff --git a/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi b/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi
+index 2cfeaf3b0a876..8c218689fef70 100644
+--- a/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi
++++ b/arch/arm64/boot/dts/broadcom/northstar2/ns2.dtsi
+@@ -687,7 +687,7 @@
+ };
+ };
+
+- sata: ahci@663f2000 {
++ sata: sata@663f2000 {
+ compatible = "brcm,iproc-ahci", "generic-ahci";
+ reg = <0x663f2000 0x1000>;
+ dma-coherent;
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1012a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1012a-rdb.dts
+index 79f155dedb2d0..e662677a6e28f 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1012a-rdb.dts
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1012a-rdb.dts
+@@ -15,6 +15,7 @@
+ compatible = "fsl,ls1012a-rdb", "fsl,ls1012a";
+
+ aliases {
++ serial0 = &duart0;
+ mmc0 = &esdhc0;
+ mmc1 = &esdhc1;
+ };
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts
+index bfd14b64567e4..2f92e62ecafe9 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts
+@@ -272,11 +272,6 @@
+ vcc-supply = <&sb_3v3>;
+ };
+
+- rtc@51 {
+- compatible = "nxp,pcf2129";
+- reg = <0x51>;
+- };
+-
+ eeprom@56 {
+ compatible = "atmel,24c512";
+ reg = <0x56>;
+@@ -318,6 +313,15 @@
+
+ };
+
++&i2c1 {
++ status = "okay";
++
++ rtc@51 {
++ compatible = "nxp,pcf2129";
++ reg = <0x51>;
++ };
++};
++
+ &enetc_port1 {
+ phy-handle = <&qds_phy1>;
+ phy-connection-type = "rgmii-id";
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
+index 01b01e3204118..35d1939e690b0 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
+@@ -536,9 +536,9 @@
+ clock-names = "i2c";
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(1)>;
+- dmas = <&edma0 1 39>,
+- <&edma0 1 38>;
+- dma-names = "tx", "rx";
++ dmas = <&edma0 1 38>,
++ <&edma0 1 39>;
++ dma-names = "rx", "tx";
+ status = "disabled";
+ };
+
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
+index 687fea6d8afa4..4e7bd04d97984 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
+@@ -499,9 +499,9 @@
+ interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(2)>;
+- dmas = <&edma0 1 39>,
+- <&edma0 1 38>;
+- dma-names = "tx", "rx";
++ dmas = <&edma0 1 38>,
++ <&edma0 1 39>;
++ dma-names = "rx", "tx";
+ status = "disabled";
+ };
+
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a-ten64.dts b/arch/arm64/boot/dts/freescale/fsl-ls1088a-ten64.dts
+index 3063851c2fb91..d3f03dcbb8c38 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1088a-ten64.dts
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a-ten64.dts
+@@ -38,7 +38,6 @@
+ powerdn {
+ label = "External Power Down";
+ gpios = <&gpio1 17 GPIO_ACTIVE_LOW>;
+- interrupts = <&gpio1 17 IRQ_TYPE_EDGE_FALLING>;
+ linux,code = <KEY_POWER>;
+ };
+
+@@ -46,7 +45,6 @@
+ admin {
+ label = "ADMIN button";
+ gpios = <&gpio3 8 GPIO_ACTIVE_HIGH>;
+- interrupts = <&gpio3 8 IRQ_TYPE_EDGE_RISING>;
+ linux,code = <KEY_WPS_BUTTON>;
+ };
+ };
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
+index f85e437f80b73..6050723172436 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
++++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
+@@ -847,7 +847,7 @@
+ };
+
+ cluster1_core0_watchdog: wdt@c000000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc000000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -857,7 +857,7 @@
+ };
+
+ cluster1_core1_watchdog: wdt@c010000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc010000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -867,7 +867,7 @@
+ };
+
+ cluster1_core2_watchdog: wdt@c020000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc020000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -877,7 +877,7 @@
+ };
+
+ cluster1_core3_watchdog: wdt@c030000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc030000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -887,7 +887,7 @@
+ };
+
+ cluster2_core0_watchdog: wdt@c100000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc100000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -897,7 +897,7 @@
+ };
+
+ cluster2_core1_watchdog: wdt@c110000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc110000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -907,7 +907,7 @@
+ };
+
+ cluster2_core2_watchdog: wdt@c120000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc120000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+@@ -917,7 +917,7 @@
+ };
+
+ cluster2_core3_watchdog: wdt@c130000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc130000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>,
+diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
+index 801ba9612d361..1282b61da8a55 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
++++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
+@@ -387,7 +387,7 @@
+ };
+
+ cluster1_core0_watchdog: wdt@c000000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc000000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -397,7 +397,7 @@
+ };
+
+ cluster1_core1_watchdog: wdt@c010000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc010000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -407,7 +407,7 @@
+ };
+
+ cluster2_core0_watchdog: wdt@c100000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc100000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -417,7 +417,7 @@
+ };
+
+ cluster2_core1_watchdog: wdt@c110000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc110000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -427,7 +427,7 @@
+ };
+
+ cluster3_core0_watchdog: wdt@c200000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc200000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -437,7 +437,7 @@
+ };
+
+ cluster3_core1_watchdog: wdt@c210000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc210000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -447,7 +447,7 @@
+ };
+
+ cluster4_core0_watchdog: wdt@c300000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc300000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+@@ -457,7 +457,7 @@
+ };
+
+ cluster4_core1_watchdog: wdt@c310000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xc310000 0x0 0x1000>;
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(4)>,
+diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi
+index c4b1a59ba424b..51c4f61007cdb 100644
+--- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi
++++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi
+@@ -719,7 +719,7 @@
+ clock-names = "i2c";
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>;
+- scl-gpio = <&gpio2 15 GPIO_ACTIVE_HIGH>;
++ scl-gpios = <&gpio2 15 GPIO_ACTIVE_HIGH>;
+ status = "disabled";
+ };
+
+@@ -768,7 +768,7 @@
+ clock-names = "i2c";
+ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+ QORIQ_CLK_PLL_DIV(16)>;
+- scl-gpio = <&gpio2 16 GPIO_ACTIVE_HIGH>;
++ scl-gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>;
+ status = "disabled";
+ };
+
+diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts
+index e99e7644ff392..49d7470812eef 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts
++++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts
+@@ -123,8 +123,8 @@
+
+ ethphy: ethernet-phy@0 {
+ reg = <0>;
+- reset-assert-us = <100>;
+- reset-deassert-us = <100>;
++ reset-assert-us = <1>;
++ reset-deassert-us = <15000>;
+ reset-gpios = <&gpio4 27 GPIO_ACTIVE_LOW>;
+ };
+ };
+diff --git a/arch/arm64/boot/dts/freescale/imx8mm-var-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-var-som.dtsi
+index 1dc9d187601c5..a0bd540f27d3d 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mm-var-som.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mm-var-som.dtsi
+@@ -89,12 +89,12 @@
+ pendown-gpio = <&gpio1 3 GPIO_ACTIVE_LOW>;
+
+ ti,x-min = /bits/ 16 <125>;
+- touchscreen-size-x = /bits/ 16 <4008>;
++ touchscreen-size-x = <4008>;
+ ti,y-min = /bits/ 16 <282>;
+- touchscreen-size-y = /bits/ 16 <3864>;
++ touchscreen-size-y = <3864>;
+ ti,x-plate-ohms = /bits/ 16 <180>;
+- touchscreen-max-pressure = /bits/ 16 <255>;
+- touchscreen-average-samples = /bits/ 16 <10>;
++ touchscreen-max-pressure = <255>;
++ touchscreen-average-samples = <10>;
+ ti,debounce-tol = /bits/ 16 <3>;
+ ti,debounce-rep = /bits/ 16 <1>;
+ ti,settle-delay-usec = /bits/ 16 <150>;
+diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi
+index 8e4a0ce99790b..7ea909a4c1d5e 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi
+@@ -103,12 +103,14 @@
+
+ &usbotg1 {
+ dr_mode = "otg";
++ over-current-active-low;
+ vbus-supply = <&reg_usb_otg1_vbus>;
+ status = "okay";
+ };
+
+ &usbotg2 {
+ dr_mode = "host";
++ disable-over-current;
+ status = "okay";
+ };
+
+@@ -166,7 +168,7 @@
+ fsl,pins = <
+ MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
+ MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI 0xd6
+- MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
++ MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO 0xd6
+ MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13 0xd6
+ >;
+ };
+diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi
+index b7c91bdc21dd9..806ee21651d1f 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi
+@@ -139,12 +139,14 @@
+
+ &usbotg1 {
+ dr_mode = "otg";
++ over-current-active-low;
+ vbus-supply = <&reg_usb_otg1_vbus>;
+ status = "okay";
+ };
+
+ &usbotg2 {
+ dr_mode = "host";
++ disable-over-current;
+ vbus-supply = <&reg_usb_otg2_vbus>;
+ status = "okay";
+ };
+@@ -231,7 +233,7 @@
+ fsl,pins = <
+ MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
+ MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI 0xd6
+- MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
++ MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO 0xd6
+ MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13 0xd6
+ >;
+ };
+diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi
+index d2ffd62a3bd46..942fed2eed643 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi
+@@ -166,12 +166,14 @@
+
+ &usbotg1 {
+ dr_mode = "otg";
++ over-current-active-low;
+ vbus-supply = <&reg_usb_otg1_vbus>;
+ status = "okay";
+ };
+
+ &usbotg2 {
+ dr_mode = "host";
++ disable-over-current;
+ vbus-supply = <&reg_usb_otg2_vbus>;
+ status = "okay";
+ };
+@@ -280,7 +282,7 @@
+ fsl,pins = <
+ MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
+ MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI 0xd6
+- MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0xd6
++ MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO 0xd6
+ MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13 0xd6
+ >;
+ };
+diff --git a/arch/arm64/boot/dts/freescale/imx8mn-ddr4-evk.dts b/arch/arm64/boot/dts/freescale/imx8mn-ddr4-evk.dts
+index 7dfee715a2c4d..d8ce217c60166 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mn-ddr4-evk.dts
++++ b/arch/arm64/boot/dts/freescale/imx8mn-ddr4-evk.dts
+@@ -59,6 +59,10 @@
+ interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+ rohm,reset-snvs-powered;
+
++ #clock-cells = <0>;
++ clocks = <&osc_32k 0>;
++ clock-output-names = "clk-32k-out";
++
+ regulators {
+ buck1_reg: BUCK1 {
+ regulator-name = "buck1";
+diff --git a/arch/arm64/boot/dts/freescale/imx8mn-var-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-var-som.dtsi
+index b16c7caf34c11..87b5e23c766f7 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mn-var-som.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mn-var-som.dtsi
+@@ -70,12 +70,12 @@
+ pendown-gpio = <&gpio1 3 GPIO_ACTIVE_LOW>;
+
+ ti,x-min = /bits/ 16 <125>;
+- touchscreen-size-x = /bits/ 16 <4008>;
++ touchscreen-size-x = <4008>;
+ ti,y-min = /bits/ 16 <282>;
+- touchscreen-size-y = /bits/ 16 <3864>;
++ touchscreen-size-y = <3864>;
+ ti,x-plate-ohms = /bits/ 16 <180>;
+- touchscreen-max-pressure = /bits/ 16 <255>;
+- touchscreen-average-samples = /bits/ 16 <10>;
++ touchscreen-max-pressure = <255>;
++ touchscreen-average-samples = <10>;
+ ti,debounce-tol = /bits/ 16 <3>;
+ ti,debounce-rep = /bits/ 16 <1>;
+ ti,settle-delay-usec = /bits/ 16 <150>;
+diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi
+index da6c942fb7f9d..6d6cbd4c83b8f 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi
+@@ -263,7 +263,7 @@
+ ranges;
+
+ sai2: sai@30020000 {
+- compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
++ compatible = "fsl,imx8mn-sai", "fsl,imx8mq-sai";
+ reg = <0x30020000 0x10000>;
+ interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clk IMX8MN_CLK_SAI2_IPG>,
+@@ -277,7 +277,7 @@
+ };
+
+ sai3: sai@30030000 {
+- compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
++ compatible = "fsl,imx8mn-sai", "fsl,imx8mq-sai";
+ reg = <0x30030000 0x10000>;
+ interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clk IMX8MN_CLK_SAI3_IPG>,
+@@ -291,7 +291,7 @@
+ };
+
+ sai5: sai@30050000 {
+- compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
++ compatible = "fsl,imx8mn-sai", "fsl,imx8mq-sai";
+ reg = <0x30050000 0x10000>;
+ interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clk IMX8MN_CLK_SAI5_IPG>,
+@@ -307,7 +307,7 @@
+ };
+
+ sai6: sai@30060000 {
+- compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
++ compatible = "fsl,imx8mn-sai", "fsl,imx8mq-sai";
+ reg = <0x30060000 0x10000>;
+ interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clk IMX8MN_CLK_SAI6_IPG>,
+@@ -364,7 +364,7 @@
+ };
+
+ sai7: sai@300b0000 {
+- compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
++ compatible = "fsl,imx8mn-sai", "fsl,imx8mq-sai";
+ reg = <0x300b0000 0x10000>;
+ interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&clk IMX8MN_CLK_SAI7_IPG>,
+diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+index 4066b16126552..fd38092bb247e 100644
+--- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+@@ -524,11 +524,9 @@
+ <&clk IMX8MQ_VIDEO_PLL1>,
+ <&clk IMX8MQ_VIDEO_PLL1_OUT>;
+ assigned-clock-rates = <0>, <0>, <0>, <594000000>;
+- interconnects = <&noc IMX8MQ_ICM_LCDIF &noc IMX8MQ_ICS_DRAM>;
+- interconnect-names = "dram";
+ status = "disabled";
+
+- port@0 {
++ port {
+ lcdif_mipi_dsi: endpoint {
+ remote-endpoint = <&mipi_dsi_lcdif_in>;
+ };
+@@ -1125,8 +1123,8 @@
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+- port@0 {
+- reg = <0>;
++ port@1 {
++ reg = <1>;
+
+ csi1_mipi_ep: endpoint {
+ remote-endpoint = <&csi1_ep>;
+@@ -1177,8 +1175,8 @@
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+- port@0 {
+- reg = <0>;
++ port@1 {
++ reg = <1>;
+
+ csi2_mipi_ep: endpoint {
+ remote-endpoint = <&csi2_ep>;
+diff --git a/arch/arm64/boot/dts/freescale/imx8qm.dtsi b/arch/arm64/boot/dts/freescale/imx8qm.dtsi
+index aebbe2b84aa13..a143f38bc78bd 100644
+--- a/arch/arm64/boot/dts/freescale/imx8qm.dtsi
++++ b/arch/arm64/boot/dts/freescale/imx8qm.dtsi
+@@ -155,7 +155,7 @@
+ };
+
+ clk: clock-controller {
+- compatible = "fsl,imx8qxp-clk", "fsl,scu-clk";
++ compatible = "fsl,imx8qm-clk", "fsl,scu-clk";
+ #clock-cells = <2>;
+ };
+
+diff --git a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
+index 2d5c1a348716a..6eabec2602e23 100644
+--- a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
++++ b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
+@@ -1087,7 +1087,7 @@
+ };
+
+ watchdog0: watchdog@e8a06000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xe8a06000 0x0 0x1000>;
+ interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&crg_ctrl HI3660_OSC32K>,
+@@ -1096,7 +1096,7 @@
+ };
+
+ watchdog1: watchdog@e8a07000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xe8a07000 0x0 0x1000>;
+ interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&crg_ctrl HI3660_OSC32K>,
+diff --git a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
+index dde9371dc5451..e4860b8a638ec 100644
+--- a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
++++ b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
+@@ -840,7 +840,7 @@
+ };
+
+ watchdog0: watchdog@f8005000 {
+- compatible = "arm,sp805-wdt", "arm,primecell";
++ compatible = "arm,sp805", "arm,primecell";
+ reg = <0x0 0xf8005000 0x0 0x1000>;
+ interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+ clocks = <&ao_ctrl HI6220_WDT0_PCLK>,
+diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
+index 163f33b46e4f7..f4270cf189962 100644
+--- a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
++++ b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
+@@ -502,7 +502,7 @@
+ };
+
+ usb0: usb@ffb00000 {
+- compatible = "snps,dwc2";
++ compatible = "intel,socfpga-agilex-hsotg", "snps,dwc2";
+ reg = <0xffb00000 0x40000>;
+ interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
+ phys = <&usbphy0>;
+@@ -515,7 +515,7 @@
+ };
+
+ usb1: usb@ffb40000 {
+- compatible = "snps,dwc2";
++ compatible = "intel,socfpga-agilex-hsotg", "snps,dwc2";
+ reg = <0xffb40000 0x40000>;
+ interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+ phys = <&usbphy0>;
+@@ -628,7 +628,7 @@
+ };
+
+ qspi: spi@ff8d2000 {
+- compatible = "cdns,qspi-nor";
++ compatible = "intel,socfpga-qspi", "cdns,qspi-nor";
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0xff8d2000 0x100>,
+diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts
+index 04da07ae44208..1cee26479bfec 100644
+--- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts
++++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts
+@@ -18,6 +18,7 @@
+
+ aliases {
+ spi0 = &spi0;
++ ethernet0 = &eth0;
+ ethernet1 = &eth1;
+ mmc0 = &sdhci0;
+ mmc1 = &sdhci1;
+@@ -138,7 +139,9 @@
+ /*
+ * U-Boot port for Turris Mox has a bug which always expects that "ranges" DT property
+ * contains exactly 2 ranges with 3 (child) address cells, 2 (parent) address cells and
+- * 2 size cells and also expects that the second range starts at 16 MB offset. If these
++ * 2 size cells and also expects that the second range starts at 16 MB offset. Also it
++ * expects that first range uses same address for PCI (child) and CPU (parent) cells (so
++ * no remapping) and that this address is the lowest from all specified ranges. If these
+ * conditions are not met then U-Boot crashes during loading kernel DTB file. PCIe address
+ * space is 128 MB long, so the best split between MEM and IO is to use fixed 16 MB window
+ * for IO and the rest 112 MB (64+32+16) for MEM, despite that maximal IO size is just 64 kB.
+@@ -147,6 +150,9 @@
+ * https://source.denx.de/u-boot/u-boot/-/commit/cb2ddb291ee6fcbddd6d8f4ff49089dfe580f5d7
+ * https://source.denx.de/u-boot/u-boot/-/commit/c64ac3b3185aeb3846297ad7391fc6df8ecd73bf
+ * https://source.denx.de/u-boot/u-boot/-/commit/4a82fca8e330157081fc132a591ebd99ba02ee33
++ * Bug related to requirement of same child and parent addresses for first range is fixed
++ * in U-Boot version 2022.04 by following commit:
++ * https://source.denx.de/u-boot/u-boot/-/commit/1fd54253bca7d43d046bba4853fe5fafd034bc17
+ */
+ #address-cells = <3>;
+ #size-cells = <2>;
+diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+index 9acc5d2b5a002..0adc194e46d15 100644
+--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
++++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+@@ -497,7 +497,7 @@
+ * (totaling 127 MiB) for MEM.
+ */
+ ranges = <0x82000000 0 0xe8000000 0 0xe8000000 0 0x07f00000 /* Port 0 MEM */
+- 0x81000000 0 0xefff0000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */
++ 0x81000000 0 0x00000000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */
+ interrupt-map-mask = <0 0 0 7>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+diff --git a/arch/arm64/boot/dts/marvell/cn9130.dtsi b/arch/arm64/boot/dts/marvell/cn9130.dtsi
+index a2b7e5ec979d3..327b04134134f 100644
+--- a/arch/arm64/boot/dts/marvell/cn9130.dtsi
++++ b/arch/arm64/boot/dts/marvell/cn9130.dtsi
+@@ -11,6 +11,13 @@
+ model = "Marvell Armada CN9130 SoC";
+ compatible = "marvell,cn9130", "marvell,armada-ap807-quad",
+ "marvell,armada-ap807";
++
++ aliases {
++ gpio1 = &cp0_gpio1;
++ gpio2 = &cp0_gpio2;
++ spi1 = &cp0_spi0;
++ spi2 = &cp0_spi1;
++ };
+ };
+
+ /*
+@@ -35,3 +42,11 @@
+ #undef CP11X_PCIE0_BASE
+ #undef CP11X_PCIE1_BASE
+ #undef CP11X_PCIE2_BASE
++
++&cp0_gpio1 {
++ status = "okay";
++};
++
++&cp0_gpio2 {
++ status = "okay";
++};
+diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
+index e94f8add1a400..062e87e893316 100644
+--- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi
++++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
+@@ -1079,7 +1079,7 @@
+
+ ccplex@e000000 {
+ compatible = "nvidia,tegra186-ccplex-cluster";
+- reg = <0x0 0x0e000000 0x0 0x3fffff>;
++ reg = <0x0 0x0e000000 0x0 0x400000>;
+
+ nvidia,bpmp = <&bpmp>;
+ };
+diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
+index c8250a3f7891f..510d2974470cd 100644
+--- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi
++++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
+@@ -818,9 +818,8 @@
+ <&bpmp TEGRA194_CLK_HDA2CODEC_2X>;
+ clock-names = "hda", "hda2hdmi", "hda2codec_2x";
+ resets = <&bpmp TEGRA194_RESET_HDA>,
+- <&bpmp TEGRA194_RESET_HDA2HDMICODEC>,
+- <&bpmp TEGRA194_RESET_HDA2CODEC_2X>;
+- reset-names = "hda", "hda2hdmi", "hda2codec_2x";
++ <&bpmp TEGRA194_RESET_HDA2HDMICODEC>;
++ reset-names = "hda", "hda2hdmi";
+ power-domains = <&bpmp TEGRA194_POWER_DOMAIN_DISP>;
+ interconnects = <&mc TEGRA194_MEMORY_CLIENT_HDAR &emc>,
+ <&mc TEGRA194_MEMORY_CLIENT_HDAW &emc>;
+diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
+index d2fe58e0eb7aa..ce4c2b4a5fc07 100644
+--- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi
++++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
+@@ -200,7 +200,7 @@
+ clock-names = "bam_clk";
+ #dma-cells = <1>;
+ qcom,ee = <1>;
+- qcom,controlled-remotely = <1>;
++ qcom,controlled-remotely;
+ qcom,config-pipe-trust-reg = <0>;
+ };
+
+@@ -221,7 +221,7 @@
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+- gpio-ranges = <&tlmm 0 80>;
++ gpio-ranges = <&tlmm 0 0 80>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+
+diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
+index db333001df4d6..97f99663c132e 100644
+--- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi
++++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
+@@ -220,7 +220,7 @@
+ clock-names = "bam_clk";
+ #dma-cells = <1>;
+ qcom,ee = <1>;
+- qcom,controlled-remotely = <1>;
++ qcom,controlled-remotely;
+ status = "disabled";
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
+index 3f85e34a8ce6f..8b27242724641 100644
+--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
++++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
+@@ -19,8 +19,8 @@
+ #size-cells = <2>;
+
+ aliases {
+- sdhc1 = &sdhc_1; /* SDC1 eMMC slot */
+- sdhc2 = &sdhc_2; /* SDC2 SD card slot */
++ mmc0 = &sdhc_1; /* SDC1 eMMC slot */
++ mmc1 = &sdhc_2; /* SDC2 SD card slot */
+ };
+
+ chosen { };
+@@ -445,7 +445,7 @@
+ };
+ };
+
+- rpm_msg_ram: memory@60000 {
++ rpm_msg_ram: sram@60000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x00060000 0x8000>;
+ };
+@@ -1384,11 +1384,17 @@
+ lpass: audio-controller@7708000 {
+ status = "disabled";
+ compatible = "qcom,lpass-cpu-apq8016";
++
++ /*
++ * Note: Unlike the name would suggest, the SEC_I2S_CLK
++ * is actually only used by Tertiary MI2S while
++ * Primary/Secondary MI2S both use the PRI_I2S_CLK.
++ */
+ clocks = <&gcc GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK>,
+ <&gcc GCC_ULTAUDIO_PCNOC_MPORT_CLK>,
+ <&gcc GCC_ULTAUDIO_PCNOC_SWAY_CLK>,
+ <&gcc GCC_ULTAUDIO_LPAIF_PRI_I2S_CLK>,
+- <&gcc GCC_ULTAUDIO_LPAIF_SEC_I2S_CLK>,
++ <&gcc GCC_ULTAUDIO_LPAIF_PRI_I2S_CLK>,
+ <&gcc GCC_ULTAUDIO_LPAIF_SEC_I2S_CLK>,
+ <&gcc GCC_ULTAUDIO_LPAIF_AUX_I2S_CLK>;
+
+diff --git a/arch/arm64/boot/dts/qcom/msm8994.dtsi b/arch/arm64/boot/dts/qcom/msm8994.dtsi
+index 986fe60dec5fb..5a9a5ed0565f6 100644
+--- a/arch/arm64/boot/dts/qcom/msm8994.dtsi
++++ b/arch/arm64/boot/dts/qcom/msm8994.dtsi
+@@ -715,7 +715,7 @@
+ reg = <0xfc400000 0x2000>;
+ };
+
+- rpm_msg_ram: memory@fc428000 {
++ rpm_msg_ram: sram@fc428000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0xfc428000 0x4000>;
+ };
+diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
+index 52df22ab3f6ae..6077c36019514 100644
+--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
++++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
+@@ -638,7 +638,7 @@
+ };
+ };
+
+- rpm_msg_ram: memory@68000 {
++ rpm_msg_ram: sram@68000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x00068000 0x6000>;
+ };
+@@ -965,9 +965,6 @@
+ nvmem-cells = <&speedbin_efuse>;
+ nvmem-cell-names = "speed_bin";
+
+- qcom,gpu-quirk-two-pass-use-wfi;
+- qcom,gpu-quirk-fault-detect-mask;
+-
+ operating-points-v2 = <&gpu_opp_table>;
+
+ status = "disabled";
+diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
+index 34039b5c80175..228339f81c327 100644
+--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
++++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
+@@ -308,38 +308,42 @@
+ LITTLE_CPU_SLEEP_0: cpu-sleep-0-0 {
+ compatible = "arm,idle-state";
+ idle-state-name = "little-retention";
++ /* CPU Retention (C2D), L2 Active */
+ arm,psci-suspend-param = <0x00000002>;
+ entry-latency-us = <81>;
+ exit-latency-us = <86>;
+- min-residency-us = <200>;
++ min-residency-us = <504>;
+ };
+
+ LITTLE_CPU_SLEEP_1: cpu-sleep-0-1 {
+ compatible = "arm,idle-state";
+ idle-state-name = "little-power-collapse";
++ /* CPU + L2 Power Collapse (C3, D4) */
+ arm,psci-suspend-param = <0x40000003>;
+- entry-latency-us = <273>;
+- exit-latency-us = <612>;
+- min-residency-us = <1000>;
++ entry-latency-us = <814>;
++ exit-latency-us = <4562>;
++ min-residency-us = <9183>;
+ local-timer-stop;
+ };
+
+ BIG_CPU_SLEEP_0: cpu-sleep-1-0 {
+ compatible = "arm,idle-state";
+ idle-state-name = "big-retention";
++ /* CPU Retention (C2D), L2 Active */
+ arm,psci-suspend-param = <0x00000002>;
+ entry-latency-us = <79>;
+ exit-latency-us = <82>;
+- min-residency-us = <200>;
++ min-residency-us = <1302>;
+ };
+
+ BIG_CPU_SLEEP_1: cpu-sleep-1-1 {
+ compatible = "arm,idle-state";
+ idle-state-name = "big-power-collapse";
++ /* CPU + L2 Power Collapse (C3, D4) */
+ arm,psci-suspend-param = <0x40000003>;
+- entry-latency-us = <336>;
+- exit-latency-us = <525>;
+- min-residency-us = <1000>;
++ entry-latency-us = <724>;
++ exit-latency-us = <2027>;
++ min-residency-us = <9419>;
+ local-timer-stop;
+ };
+ };
+@@ -857,7 +861,7 @@
+ reg = <0x00100000 0xb0000>;
+ };
+
+- rpm_msg_ram: memory@778000 {
++ rpm_msg_ram: sram@778000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x00778000 0x7000>;
+ };
+diff --git a/arch/arm64/boot/dts/qcom/pm8916.dtsi b/arch/arm64/boot/dts/qcom/pm8916.dtsi
+index f931cb0de231f..42180f1b5dbbb 100644
+--- a/arch/arm64/boot/dts/qcom/pm8916.dtsi
++++ b/arch/arm64/boot/dts/qcom/pm8916.dtsi
+@@ -86,7 +86,6 @@
+ rtc@6000 {
+ compatible = "qcom,pm8941-rtc";
+ reg = <0x6000>;
+- reg-names = "rtc", "alarm";
+ interrupts = <0x0 0x61 0x1 IRQ_TYPE_EDGE_RISING>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/pmi8994.dtsi b/arch/arm64/boot/dts/qcom/pmi8994.dtsi
+index b4ac900ab115f..a06ea9adae810 100644
+--- a/arch/arm64/boot/dts/qcom/pmi8994.dtsi
++++ b/arch/arm64/boot/dts/qcom/pmi8994.dtsi
+@@ -42,7 +42,7 @@
+ /* Yes, all four strings *have to* be defined or things won't work. */
+ qcom,enabled-strings = <0 1 2 3>;
+ qcom,cabc;
+- qcom,eternal-pfet;
++ qcom,external-pfet;
+ status = "disabled";
+ };
+ };
+diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi
+index 339790ba585de..ca5be16479809 100644
+--- a/arch/arm64/boot/dts/qcom/qcs404.dtsi
++++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi
+@@ -318,7 +318,7 @@
+ status = "disabled";
+ };
+
+- rpm_msg_ram: memory@60000 {
++ rpm_msg_ram: sram@60000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x00060000 0x6000>;
+ };
+diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi
+index a758e4d226122..81098aa9687ba 100644
+--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi
+@@ -33,7 +33,7 @@ ap_h1_spi: &spi0 {};
+ polling-delay = <0>;
+
+ thermal-sensors = <&pm6150_adc_tm 1>;
+- sustainable-power = <814>;
++ sustainable-power = <965>;
+
+ trips {
+ skin_temp_alert0: trip-point0 {
+diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi
+index a246dbd74cc11..b7b5264888b7c 100644
+--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi
+@@ -44,7 +44,7 @@ ap_h1_spi: &spi0 {};
+ };
+
+ &cpu6_thermal {
+- sustainable-power = <948>;
++ sustainable-power = <1124>;
+ };
+
+ &cpu7_alert0 {
+@@ -56,7 +56,7 @@ ap_h1_spi: &spi0 {};
+ };
+
+ &cpu7_thermal {
+- sustainable-power = <948>;
++ sustainable-power = <1124>;
+ };
+
+ &cpu8_alert0 {
+@@ -68,7 +68,7 @@ ap_h1_spi: &spi0 {};
+ };
+
+ &cpu8_thermal {
+- sustainable-power = <948>;
++ sustainable-power = <1124>;
+ };
+
+ &cpu9_alert0 {
+@@ -80,7 +80,7 @@ ap_h1_spi: &spi0 {};
+ };
+
+ &cpu9_thermal {
+- sustainable-power = <948>;
++ sustainable-power = <1124>;
+ };
+
+ &gpio_keys {
+diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
+index c8921e2d6480f..de86ae3a7fd27 100644
+--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
+@@ -137,8 +137,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+ <&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
+@@ -162,8 +162,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ next-level-cache = <&L2_100>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -184,8 +184,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ next-level-cache = <&L2_200>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -206,8 +206,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ next-level-cache = <&L2_300>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -228,8 +228,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ next-level-cache = <&L2_400>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -250,8 +250,8 @@
+ cpu-idle-states = <&LITTLE_CPU_SLEEP_0
+ &LITTLE_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1024>;
+- dynamic-power-coefficient = <100>;
++ capacity-dmips-mhz = <415>;
++ dynamic-power-coefficient = <137>;
+ next-level-cache = <&L2_500>;
+ operating-points-v2 = <&cpu0_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -272,8 +272,8 @@
+ cpu-idle-states = <&BIG_CPU_SLEEP_0
+ &BIG_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1740>;
+- dynamic-power-coefficient = <405>;
++ capacity-dmips-mhz = <1024>;
++ dynamic-power-coefficient = <480>;
+ next-level-cache = <&L2_600>;
+ operating-points-v2 = <&cpu6_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -294,8 +294,8 @@
+ cpu-idle-states = <&BIG_CPU_SLEEP_0
+ &BIG_CPU_SLEEP_1
+ &CLUSTER_SLEEP_0>;
+- capacity-dmips-mhz = <1740>;
+- dynamic-power-coefficient = <405>;
++ capacity-dmips-mhz = <1024>;
++ dynamic-power-coefficient = <480>;
+ next-level-cache = <&L2_700>;
+ operating-points-v2 = <&cpu6_opp_table>;
+ interconnects = <&gem_noc MASTER_APPSS_PROC 3 &mc_virt SLAVE_EBI1 3>,
+@@ -1460,6 +1460,8 @@
+ "imem",
+ "config";
+
++ qcom,qmp = <&aoss_qmp>;
++
+ qcom,smem-states = <&ipa_smp2p_out 0>,
+ <&ipa_smp2p_out 1>;
+ qcom,smem-state-names = "ipa-clock-enabled-valid",
+@@ -3616,7 +3618,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 1>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu0_alert0: trip-point0 {
+@@ -3665,7 +3667,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 2>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu1_alert0: trip-point0 {
+@@ -3714,7 +3716,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 3>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu2_alert0: trip-point0 {
+@@ -3763,7 +3765,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 4>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu3_alert0: trip-point0 {
+@@ -3812,7 +3814,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 5>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu4_alert0: trip-point0 {
+@@ -3861,7 +3863,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 6>;
+- sustainable-power = <768>;
++ sustainable-power = <1052>;
+
+ trips {
+ cpu5_alert0: trip-point0 {
+@@ -3910,7 +3912,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 9>;
+- sustainable-power = <1202>;
++ sustainable-power = <1425>;
+
+ trips {
+ cpu6_alert0: trip-point0 {
+@@ -3951,7 +3953,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 10>;
+- sustainable-power = <1202>;
++ sustainable-power = <1425>;
+
+ trips {
+ cpu7_alert0: trip-point0 {
+@@ -3992,7 +3994,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 11>;
+- sustainable-power = <1202>;
++ sustainable-power = <1425>;
+
+ trips {
+ cpu8_alert0: trip-point0 {
+@@ -4033,7 +4035,7 @@
+ polling-delay = <0>;
+
+ thermal-sensors = <&tsens0 12>;
+- sustainable-power = <1202>;
++ sustainable-power = <1425>;
+
+ trips {
+ cpu9_alert0: trip-point0 {
+diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi
+index fd78f16181ddd..b795a9993cc1b 100644
+--- a/arch/arm64/boot/dts/qcom/sc7280.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi
+@@ -429,7 +429,7 @@
+ <&rpmhcc RPMH_CXO_CLK_A>, <&sleep_clk>,
+ <0>, <0>, <0>, <0>, <0>, <0>;
+ clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk",
+- "pcie_0_pipe_clk", "pcie_1_pipe-clk",
++ "pcie_0_pipe_clk", "pcie_1_pipe_clk",
+ "ufs_phy_rx_symbol_0_clk", "ufs_phy_rx_symbol_1_clk",
+ "ufs_phy_tx_symbol_0_clk",
+ "usb3_phy_wrapper_gcc_usb30_pipe_clk";
+@@ -615,6 +615,8 @@
+ interconnect-names = "memory",
+ "config";
+
++ qcom,qmp = <&aoss_qmp>;
++
+ qcom,smem-states = <&ipa_smp2p_out 0>,
+ <&ipa_smp2p_out 1>;
+ qcom,smem-state-names = "ipa-clock-enabled-valid",
+@@ -1258,15 +1260,11 @@
+ dp_phy: dp-phy@88ea200 {
+ reg = <0 0x088ea200 0 0x200>,
+ <0 0x088ea400 0 0x200>,
+- <0 0x088eac00 0 0x400>,
++ <0 0x088eaa00 0 0x200>,
+ <0 0x088ea600 0 0x200>,
+- <0 0x088ea800 0 0x200>,
+- <0 0x088eaa00 0 0x100>;
++ <0 0x088ea800 0 0x200>;
+ #phy-cells = <0>;
+ #clock-cells = <1>;
+- clocks = <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>;
+- clock-names = "pipe0";
+- clock-output-names = "usb3_phy_pipe_clk_src";
+ };
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi
+index 9c7f87e42fccd..a8724fd60645f 100644
+--- a/arch/arm64/boot/dts/qcom/sdm630.dtsi
++++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi
+@@ -541,7 +541,7 @@
+ <&sleep_clk>;
+ };
+
+- rpm_msg_ram: memory@778000 {
++ rpm_msg_ram: sram@778000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x00778000 0x7000>;
+ };
+diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
+index b3b9119261844..d20eacfc10176 100644
+--- a/arch/arm64/boot/dts/qcom/sdm845.dtsi
++++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
+@@ -2316,11 +2316,11 @@
+ compatible = "qcom,bam-v1.7.0";
+ reg = <0 0x01dc4000 0 0x24000>;
+ interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+- clocks = <&rpmhcc 15>;
++ clocks = <&rpmhcc RPMH_CE_CLK>;
+ clock-names = "bam_clk";
+ #dma-cells = <1>;
+ qcom,ee = <0>;
+- qcom,controlled-remotely = <1>;
++ qcom,controlled-remotely;
+ iommus = <&apps_smmu 0x704 0x1>,
+ <&apps_smmu 0x706 0x1>,
+ <&apps_smmu 0x714 0x1>,
+@@ -2331,8 +2331,8 @@
+ compatible = "qcom,crypto-v5.4";
+ reg = <0 0x01dfa000 0 0x6000>;
+ clocks = <&gcc GCC_CE1_AHB_CLK>,
+- <&gcc GCC_CE1_AHB_CLK>,
+- <&rpmhcc 15>;
++ <&gcc GCC_CE1_AXI_CLK>,
++ <&rpmhcc RPMH_CE_CLK>;
+ clock-names = "iface", "bus", "core";
+ dmas = <&cryptobam 6>, <&cryptobam 7>;
+ dma-names = "rx", "tx";
+@@ -3608,10 +3608,10 @@
+ #clock-cells = <0>;
+ clock-frequency = <9600000>;
+ clock-output-names = "mclk";
+- qcom,micbias1-millivolt = <1800>;
+- qcom,micbias2-millivolt = <1800>;
+- qcom,micbias3-millivolt = <1800>;
+- qcom,micbias4-millivolt = <1800>;
++ qcom,micbias1-microvolt = <1800000>;
++ qcom,micbias2-microvolt = <1800000>;
++ qcom,micbias3-microvolt = <1800000>;
++ qcom,micbias4-microvolt = <1800000>;
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
+index 2ba23aa582a18..617a634ac9051 100644
+--- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
++++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
+@@ -518,6 +518,10 @@
+ dai@1 {
+ reg = <1>;
+ };
++
++ dai@2 {
++ reg = <2>;
++ };
+ };
+
+ &sound {
+@@ -530,6 +534,7 @@
+ "SpkrLeft IN", "SPK1 OUT",
+ "SpkrRight IN", "SPK2 OUT",
+ "MM_DL1", "MultiMedia1 Playback",
++ "MM_DL3", "MultiMedia3 Playback",
+ "MultiMedia2 Capture", "MM_UL2";
+
+ mm1-dai-link {
+@@ -546,6 +551,13 @@
+ };
+ };
+
++ mm3-dai-link {
++ link-name = "MultiMedia3";
++ cpu {
++ sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA3>;
++ };
++ };
++
+ slim-dai-link {
+ link-name = "SLIM Playback";
+ cpu {
+@@ -575,6 +587,21 @@
+ sound-dai = <&wcd9340 1>;
+ };
+ };
++
++ slim-wcd-dai-link {
++ link-name = "SLIM WCD Playback";
++ cpu {
++ sound-dai = <&q6afedai SLIMBUS_1_RX>;
++ };
++
++ platform {
++ sound-dai = <&q6routing>;
++ };
++
++ codec {
++ sound-dai = <&wcd9340 2>;
++ };
++ };
+ };
+
+ &tlmm {
+diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi
+index 2b37ce6a9f9c5..9f476e3d0720b 100644
+--- a/arch/arm64/boot/dts/qcom/sm6125.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi
+@@ -380,7 +380,7 @@
+ status = "disabled";
+ };
+
+- rpm_msg_ram: memory@45f0000 {
++ rpm_msg_ram: sram@45f0000 {
+ compatible = "qcom,rpm-msg-ram";
+ reg = <0x045f0000 0x7000>;
+ };
+diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
+index ef0232c2cf45b..f347f752d536d 100644
+--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
+@@ -3434,9 +3434,9 @@
+ qcom,tcs-offset = <0xd00>;
+ qcom,drv-id = <2>;
+ qcom,tcs-config = <ACTIVE_TCS 2>,
+- <SLEEP_TCS 1>,
+- <WAKE_TCS 1>,
+- <CONTROL_TCS 0>;
++ <SLEEP_TCS 3>,
++ <WAKE_TCS 3>,
++ <CONTROL_TCS 1>;
+
+ rpmhcc: clock-controller {
+ compatible = "qcom,sm8150-rpmh-clk";
+diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
+index d12e4cbfc8527..2786e2c8e5659 100644
+--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
+@@ -1434,8 +1434,8 @@
+ phys = <&pcie0_lane>;
+ phy-names = "pciephy";
+
+- perst-gpio = <&tlmm 79 GPIO_ACTIVE_LOW>;
+- enable-gpio = <&tlmm 81 GPIO_ACTIVE_HIGH>;
++ perst-gpios = <&tlmm 79 GPIO_ACTIVE_LOW>;
++ wake-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+
+ pinctrl-names = "default";
+ pinctrl-0 = <&pcie0_default_state>;
+@@ -1495,7 +1495,7 @@
+ ranges = <0x01000000 0x0 0x40200000 0x0 0x40200000 0x0 0x100000>,
+ <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>;
+
+- interrupts = <GIC_SPI 306 IRQ_TYPE_EDGE_RISING>;
++ interrupts = <GIC_SPI 307 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "msi";
+ #interrupt-cells = <1>;
+ interrupt-map-mask = <0 0 0 0x7>;
+@@ -1538,8 +1538,8 @@
+ phys = <&pcie1_lane>;
+ phy-names = "pciephy";
+
+- perst-gpio = <&tlmm 82 GPIO_ACTIVE_LOW>;
+- enable-gpio = <&tlmm 84 GPIO_ACTIVE_HIGH>;
++ perst-gpios = <&tlmm 82 GPIO_ACTIVE_LOW>;
++ wake-gpios = <&tlmm 84 GPIO_ACTIVE_HIGH>;
+
+ pinctrl-names = "default";
+ pinctrl-0 = <&pcie1_default_state>;
+@@ -1601,7 +1601,7 @@
+ ranges = <0x01000000 0x0 0x64200000 0x0 0x64200000 0x0 0x100000>,
+ <0x02000000 0x0 0x64300000 0x0 0x64300000 0x0 0x3d00000>;
+
+- interrupts = <GIC_SPI 236 IRQ_TYPE_EDGE_RISING>;
++ interrupts = <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "msi";
+ #interrupt-cells = <1>;
+ interrupt-map-mask = <0 0 0 0x7>;
+@@ -1644,8 +1644,8 @@
+ phys = <&pcie2_lane>;
+ phy-names = "pciephy";
+
+- perst-gpio = <&tlmm 85 GPIO_ACTIVE_LOW>;
+- enable-gpio = <&tlmm 87 GPIO_ACTIVE_HIGH>;
++ perst-gpios = <&tlmm 85 GPIO_ACTIVE_LOW>;
++ wake-gpios = <&tlmm 87 GPIO_ACTIVE_HIGH>;
+
+ pinctrl-names = "default";
+ pinctrl-0 = <&pcie2_default_state>;
+diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
+index e91cd8a5e5356..9ffb7355850c7 100644
+--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
+@@ -35,6 +35,24 @@
+ clock-frequency = <32000>;
+ #clock-cells = <0>;
+ };
++
++ ufs_phy_rx_symbol_0_clk: ufs-phy-rx-symbol-0 {
++ compatible = "fixed-clock";
++ clock-frequency = <1000>;
++ #clock-cells = <0>;
++ };
++
++ ufs_phy_rx_symbol_1_clk: ufs-phy-rx-symbol-1 {
++ compatible = "fixed-clock";
++ clock-frequency = <1000>;
++ #clock-cells = <0>;
++ };
++
++ ufs_phy_tx_symbol_0_clk: ufs-phy-tx-symbol-0 {
++ compatible = "fixed-clock";
++ clock-frequency = <1000>;
++ #clock-cells = <0>;
++ };
+ };
+
+ cpus {
+@@ -443,8 +461,30 @@
+ #clock-cells = <1>;
+ #reset-cells = <1>;
+ #power-domain-cells = <1>;
+- clock-names = "bi_tcxo", "sleep_clk";
+- clocks = <&rpmhcc RPMH_CXO_CLK>, <&sleep_clk>;
++ clock-names = "bi_tcxo",
++ "sleep_clk",
++ "pcie_0_pipe_clk",
++ "pcie_1_pipe_clk",
++ "ufs_card_rx_symbol_0_clk",
++ "ufs_card_rx_symbol_1_clk",
++ "ufs_card_tx_symbol_0_clk",
++ "ufs_phy_rx_symbol_0_clk",
++ "ufs_phy_rx_symbol_1_clk",
++ "ufs_phy_tx_symbol_0_clk",
++ "usb3_phy_wrapper_gcc_usb30_pipe_clk",
++ "usb3_uni_phy_sec_gcc_usb30_pipe_clk";
++ clocks = <&rpmhcc RPMH_CXO_CLK>,
++ <&sleep_clk>,
++ <0>,
++ <0>,
++ <0>,
++ <0>,
++ <0>,
++ <&ufs_phy_rx_symbol_0_clk>,
++ <&ufs_phy_rx_symbol_1_clk>,
++ <&ufs_phy_tx_symbol_0_clk>,
++ <0>,
++ <0>;
+ };
+
+ ipcc: mailbox@408000 {
+@@ -696,6 +736,8 @@
+ interconnect-names = "memory",
+ "config";
+
++ qcom,qmp = <&aoss_qmp>;
++
+ qcom,smem-states = <&ipa_smp2p_out 0>,
+ <&ipa_smp2p_out 1>;
+ qcom,smem-state-names = "ipa-clock-enabled-valid",
+@@ -939,7 +981,7 @@
+ qcom,tcs-offset = <0xd00>;
+ qcom,drv-id = <2>;
+ qcom,tcs-config = <ACTIVE_TCS 2>, <SLEEP_TCS 3>,
+- <WAKE_TCS 3>, <CONTROL_TCS 1>;
++ <WAKE_TCS 3>, <CONTROL_TCS 0>;
+
+ rpmhcc: clock-controller {
+ compatible = "qcom,sm8350-rpmh-clk";
+@@ -1060,8 +1102,8 @@
+ <75000000 300000000>,
+ <0 0>,
+ <0 0>,
+- <75000000 300000000>,
+- <75000000 300000000>;
++ <0 0>,
++ <0 0>;
+ status = "disabled";
+ };
+
+@@ -2185,7 +2227,7 @@
+ };
+ };
+
+- camera-thermal-bottom {
++ cam-thermal-bottom {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+
+diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
+index 090dc9c4f57b5..937d17a426b66 100644
+--- a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
++++ b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
+@@ -50,6 +50,7 @@
+ &avb {
+ pinctrl-0 = <&avb_pins>;
+ pinctrl-names = "default";
++ phy-mode = "rgmii-rxid";
+ phy-handle = <&phy0>;
+ rx-internal-delay-ps = <1800>;
+ tx-internal-delay-ps = <2000>;
+diff --git a/arch/arm64/boot/dts/renesas/cat875.dtsi b/arch/arm64/boot/dts/renesas/cat875.dtsi
+index 801ea54b027c4..20f8adc635e72 100644
+--- a/arch/arm64/boot/dts/renesas/cat875.dtsi
++++ b/arch/arm64/boot/dts/renesas/cat875.dtsi
+@@ -18,6 +18,7 @@
+ pinctrl-names = "default";
+ renesas,no-ether-link;
+ phy-handle = <&phy0>;
++ phy-mode = "rgmii-id";
+ status = "okay";
+
+ phy0: ethernet-phy@0 {
+diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
+index 6f4fffacfca21..e70aa5a087402 100644
+--- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
+@@ -2784,7 +2784,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2799,7 +2799,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -2814,7 +2814,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
+index 0f7bdfc90a0dc..6c5694fa66900 100644
+--- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
+@@ -2629,7 +2629,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2644,7 +2644,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -2659,7 +2659,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
+index 379a1300272ba..62209ab6deb9a 100644
+--- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
+@@ -2904,7 +2904,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2919,7 +2919,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -2934,7 +2934,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
+index 1768a3e6bb8da..193d81be40fc4 100644
+--- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
+@@ -3375,7 +3375,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -3390,7 +3390,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -3405,7 +3405,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+index 2bd8169735d35..b526e4f0ee6a8 100644
+--- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+@@ -2972,7 +2972,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2987,7 +2987,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -3002,7 +3002,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a77961.dtsi b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
+index 041473aa5cd09..21fc95397c3c2 100644
+--- a/arch/arm64/boot/dts/renesas/r8a77961.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
+@@ -2719,7 +2719,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2734,7 +2734,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -2749,7 +2749,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+index 08df75606430b..f9679a4dd85fa 100644
+--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+@@ -2784,7 +2784,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -2799,7 +2799,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -2814,7 +2814,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+index 6347d15e66b64..21fe602bd25af 100644
+--- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+@@ -1580,7 +1580,7 @@
+ };
+
+ thermal-zones {
+- thermal-sensor-1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -1599,7 +1599,7 @@
+ };
+ };
+
+- thermal-sensor-2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
+index 631d520cebee5..26899fb768a73 100644
+--- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
++++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
+@@ -1149,7 +1149,7 @@
+ };
+
+ thermal-zones {
+- sensor_thermal1: sensor-thermal1 {
++ sensor1_thermal: sensor1-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 0>;
+@@ -1163,7 +1163,7 @@
+ };
+ };
+
+- sensor_thermal2: sensor-thermal2 {
++ sensor2_thermal: sensor2-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 1>;
+@@ -1177,7 +1177,7 @@
+ };
+ };
+
+- sensor_thermal3: sensor-thermal3 {
++ sensor3_thermal: sensor3-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 2>;
+@@ -1191,7 +1191,7 @@
+ };
+ };
+
+- sensor_thermal4: sensor-thermal4 {
++ sensor4_thermal: sensor4-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 3>;
+@@ -1205,7 +1205,7 @@
+ };
+ };
+
+- sensor_thermal5: sensor-thermal5 {
++ sensor5_thermal: sensor5-thermal {
+ polling-delay-passive = <250>;
+ polling-delay = <1000>;
+ thermal-sensors = <&tsc 4>;
+diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi
+index 248ebb61aa790..5200d0bbd9e9c 100644
+--- a/arch/arm64/boot/dts/rockchip/px30.dtsi
++++ b/arch/arm64/boot/dts/rockchip/px30.dtsi
+@@ -711,7 +711,7 @@
+ clock-names = "pclk", "timer";
+ };
+
+- dmac: dmac@ff240000 {
++ dmac: dma-controller@ff240000 {
+ compatible = "arm,pl330", "arm,primecell";
+ reg = <0x0 0xff240000 0x0 0x4000>;
+ interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+diff --git a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts
+index 665b2e69455dd..ea6820902ede0 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts
+@@ -97,7 +97,7 @@
+ regulator-max-microvolt = <3300000>;
+ regulator-always-on;
+ regulator-boot-on;
+- vim-supply = <&vcc_io>;
++ vin-supply = <&vcc_io>;
+ };
+
+ vdd_core: vdd-core {
+diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
+index 8c821acb21ffb..3cbe83e6fb9a4 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
+@@ -489,7 +489,7 @@
+ status = "disabled";
+ };
+
+- dmac: dmac@ff1f0000 {
++ dmac: dma-controller@ff1f0000 {
+ compatible = "arm,pl330", "arm,primecell";
+ reg = <0x0 0xff1f0000 0x0 0x4000>;
+ interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+@@ -599,7 +599,7 @@
+
+ gpu: gpu@ff300000 {
+ compatible = "rockchip,rk3328-mali", "arm,mali-450";
+- reg = <0x0 0xff300000 0x0 0x40000>;
++ reg = <0x0 0xff300000 0x0 0x30000>;
+ interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
+index c4dd2a6b48368..f81ce3240342c 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
+@@ -770,8 +770,8 @@
+ sd-uhs-sdr104;
+
+ /* Power supply */
+- vqmmc-supply = &vcc1v8_s3; /* IO line */
+- vmmc-supply = &vcc_sdio; /* card's power */
++ vqmmc-supply = <&vcc1v8_s3>; /* IO line */
++ vmmc-supply = <&vcc_sdio>; /* card's power */
+
+ #address-cells = <1>;
+ #size-cells = <0>;
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
+index c1bcc8ca3769d..2f8e117109699 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
+@@ -286,7 +286,7 @@
+
+ sound: sound {
+ compatible = "rockchip,rk3399-gru-sound";
+- rockchip,cpu = <&i2s0 &i2s2>;
++ rockchip,cpu = <&i2s0 &spdif>;
+ };
+ };
+
+@@ -437,10 +437,6 @@ ap_i2c_audio: &i2c8 {
+ status = "okay";
+ };
+
+-&i2s2 {
+- status = "okay";
+-};
+-
+ &io_domains {
+ status = "okay";
+
+@@ -537,6 +533,17 @@ ap_i2c_audio: &i2c8 {
+ vqmmc-supply = <&ppvar_sd_card_io>;
+ };
+
++&spdif {
++ status = "okay";
++
++ /*
++ * SPDIF is routed internally to DP; we either don't use these pins, or
++ * mux them to something else.
++ */
++ /delete-property/ pinctrl-0;
++ /delete-property/ pinctrl-names;
++};
++
+ &spi1 {
+ status = "okay";
+
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-khadas-edge.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-khadas-edge.dtsi
+index d5c7648c841dc..f1fcc6b5b402c 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-khadas-edge.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3399-khadas-edge.dtsi
+@@ -705,7 +705,6 @@
+ &sdhci {
+ bus-width = <8>;
+ mmc-hs400-1_8v;
+- mmc-hs400-enhanced-strobe;
+ non-removable;
+ status = "okay";
+ };
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts
+index 738cfd21df3ef..354f54767bad8 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts
+@@ -269,6 +269,7 @@
+ clock-output-names = "xin32k", "rk808-clkout2";
+ pinctrl-names = "default";
+ pinctrl-0 = <&pmic_int_l>;
++ rockchip,system-power-controller;
+ vcc1-supply = <&vcc5v0_sys>;
+ vcc2-supply = <&vcc5v0_sys>;
+ vcc3-supply = <&vcc5v0_sys>;
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-leez-p710.dts b/arch/arm64/boot/dts/rockchip/rk3399-leez-p710.dts
+index 7c93f840bc64f..e890166e7fd43 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-leez-p710.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3399-leez-p710.dts
+@@ -55,7 +55,7 @@
+ regulator-boot-on;
+ regulator-min-microvolt = <3300000>;
+ regulator-max-microvolt = <3300000>;
+- vim-supply = <&vcc3v3_sys>;
++ vin-supply = <&vcc3v3_sys>;
+ };
+
+ vcc3v3_sys: vcc3v3-sys {
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
+index 2b5f001ff4a61..9e5d07f5712e6 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
+@@ -385,10 +385,6 @@
+ };
+ };
+
+-&cdn_dp {
+- status = "okay";
+-};
+-
+ &cpu_b0 {
+ cpu-supply = <&vdd_cpu_b>;
+ };
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
+index 292bb7e80cf35..3ae5d727e3674 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
++++ b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
+@@ -232,6 +232,7 @@
+
+ &usbdrd_dwc3_0 {
+ dr_mode = "otg";
++ extcon = <&extcon_usb3>;
+ status = "okay";
+ };
+
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
+index fb67db4619ea0..08fa00364b42f 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
+@@ -25,6 +25,13 @@
+ };
+ };
+
++ extcon_usb3: extcon-usb3 {
++ compatible = "linux,extcon-usb-gpio";
++ id-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_HIGH>;
++ pinctrl-names = "default";
++ pinctrl-0 = <&usb3_id>;
++ };
++
+ clkin_gmac: external-gmac-clock {
+ compatible = "fixed-clock";
+ clock-frequency = <125000000>;
+@@ -422,9 +429,22 @@
+ <4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
+ };
+ };
++
++ usb3 {
++ usb3_id: usb3-id {
++ rockchip,pins =
++ <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
++ };
++ };
+ };
+
+ &sdhci {
++ /*
++ * Signal integrity isn't great at 200MHz but 100MHz has proven stable
++ * enough.
++ */
++ max-frequency = <100000000>;
++
+ bus-width = <8>;
+ mmc-hs400-1_8v;
+ mmc-hs400-enhanced-strobe;
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
+index b28888ea9262e..100a769165ef9 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
+@@ -457,7 +457,7 @@
+ status = "okay";
+
+ bt656-supply = <&vcc_3v0>;
+- audio-supply = <&vcc_3v0>;
++ audio-supply = <&vcc1v8_codec>;
+ sdmmc-supply = <&vcc_sdio>;
+ gpio1830-supply = <&vcc_3v0>;
+ };
+diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+index 3871c7fd83b00..00f1d036dfe08 100644
+--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
++++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+@@ -1802,10 +1802,10 @@
+ interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH 0>;
+ clocks = <&cru PCLK_HDMI_CTRL>,
+ <&cru SCLK_HDMI_SFR>,
+- <&cru PLL_VPLL>,
++ <&cru SCLK_HDMI_CEC>,
+ <&cru PCLK_VIO_GRF>,
+- <&cru SCLK_HDMI_CEC>;
+- clock-names = "iahb", "isfr", "vpll", "grf", "cec";
++ <&cru PLL_VPLL>;
++ clock-names = "iahb", "isfr", "cec", "grf", "vpll";
+ power-domains = <&power RK3399_PD_HDCP>;
+ reg-io-width = <4>;
+ rockchip,grf = <&grf>;
+diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
+index 42d1d219a3fd2..86291f3469f15 100644
+--- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
+@@ -59,7 +59,10 @@
+ #interrupt-cells = <3>;
+ interrupt-controller;
+ reg = <0x00 0x01800000 0x00 0x10000>, /* GICD */
+- <0x00 0x01840000 0x00 0xC0000>; /* GICR */
++ <0x00 0x01840000 0x00 0xC0000>, /* GICR */
++ <0x01 0x00000000 0x00 0x2000>, /* GICC */
++ <0x01 0x00010000 0x00 0x1000>, /* GICH */
++ <0x01 0x00020000 0x00 0x2000>; /* GICV */
+ /*
+ * vcpumntirq:
+ * virtual CPU interface maintenance interrupt
+diff --git a/arch/arm64/boot/dts/ti/k3-am64.dtsi b/arch/arm64/boot/dts/ti/k3-am64.dtsi
+index de6805b0c72c1..e589c58f60885 100644
+--- a/arch/arm64/boot/dts/ti/k3-am64.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-am64.dtsi
+@@ -85,6 +85,7 @@
+ <0x00 0x68000000 0x00 0x68000000 0x00 0x08000000>, /* PCIe DAT0 */
+ <0x00 0x70000000 0x00 0x70000000 0x00 0x00200000>, /* OC SRAM */
+ <0x00 0x78000000 0x00 0x78000000 0x00 0x00800000>, /* Main R5FSS */
++ <0x01 0x00000000 0x01 0x00000000 0x00 0x00310000>, /* A53 PERIPHBASE */
+ <0x06 0x00000000 0x06 0x00000000 0x01 0x00000000>, /* PCIe DAT1 */
+ <0x05 0x00000000 0x05 0x00000000 0x01 0x00000000>, /* FSS0 DAT3 */
+
+diff --git a/arch/arm64/boot/dts/ti/k3-am642.dtsi b/arch/arm64/boot/dts/ti/k3-am642.dtsi
+index e2b397c884018..8a76f4821b11b 100644
+--- a/arch/arm64/boot/dts/ti/k3-am642.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-am642.dtsi
+@@ -60,6 +60,6 @@
+ cache-level = <2>;
+ cache-size = <0x40000>;
+ cache-line-size = <64>;
+- cache-sets = <512>;
++ cache-sets = <256>;
+ };
+ };
+diff --git a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
+index ba4e5d3e1ed7a..82be00069bcd5 100644
+--- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
+@@ -35,7 +35,10 @@
+ #interrupt-cells = <3>;
+ interrupt-controller;
+ reg = <0x00 0x01800000 0x00 0x10000>, /* GICD */
+- <0x00 0x01880000 0x00 0x90000>; /* GICR */
++ <0x00 0x01880000 0x00 0x90000>, /* GICR */
++ <0x00 0x6f000000 0x00 0x2000>, /* GICC */
++ <0x00 0x6f010000 0x00 0x1000>, /* GICH */
++ <0x00 0x6f020000 0x00 0x2000>; /* GICV */
+ /*
+ * vcpumntirq:
+ * virtual CPU interface maintenance interrupt
+diff --git a/arch/arm64/boot/dts/ti/k3-am65.dtsi b/arch/arm64/boot/dts/ti/k3-am65.dtsi
+index a9fc1af03f27f..1607db9b32dd2 100644
+--- a/arch/arm64/boot/dts/ti/k3-am65.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-am65.dtsi
+@@ -84,6 +84,7 @@
+ <0x00 0x46000000 0x00 0x46000000 0x00 0x00200000>,
+ <0x00 0x47000000 0x00 0x47000000 0x00 0x00068400>,
+ <0x00 0x50000000 0x00 0x50000000 0x00 0x8000000>,
++ <0x00 0x6f000000 0x00 0x6f000000 0x00 0x00310000>, /* A53 PERIPHBASE */
+ <0x00 0x70000000 0x00 0x70000000 0x00 0x200000>,
+ <0x05 0x00000000 0x05 0x00000000 0x01 0x0000000>,
+ <0x07 0x00000000 0x07 0x00000000 0x01 0x0000000>;
+diff --git a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
+index e8a41d09b45f2..000b5732ea0c8 100644
+--- a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
+@@ -32,7 +32,7 @@
+ #size-cells = <1>;
+ ranges = <0x00 0x00 0x00100000 0x1c000>;
+
+- serdes_ln_ctrl: serdes-ln-ctrl@4080 {
++ serdes_ln_ctrl: mux-controller@4080 {
+ compatible = "mmio-mux";
+ #mux-control-cells = <1>;
+ mux-reg-masks = <0x4080 0x3>, <0x4084 0x3>, /* SERDES0 lane0/1 select */
+@@ -54,7 +54,10 @@
+ #interrupt-cells = <3>;
+ interrupt-controller;
+ reg = <0x00 0x01800000 0x00 0x10000>, /* GICD */
+- <0x00 0x01900000 0x00 0x100000>; /* GICR */
++ <0x00 0x01900000 0x00 0x100000>, /* GICR */
++ <0x00 0x6f000000 0x00 0x2000>, /* GICC */
++ <0x00 0x6f010000 0x00 0x1000>, /* GICH */
++ <0x00 0x6f020000 0x00 0x2000>; /* GICV */
+
+ /* vcpumntirq: virtual CPU interface maintenance interrupt */
+ interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+@@ -606,10 +609,10 @@
+ clock-names = "fck";
+ #address-cells = <3>;
+ #size-cells = <2>;
+- bus-range = <0x0 0xf>;
++ bus-range = <0x0 0xff>;
+ cdns,no-bar-match-nbits = <64>;
+- vendor-id = /bits/ 16 <0x104c>;
+- device-id = /bits/ 16 <0xb00f>;
++ vendor-id = <0x104c>;
++ device-id = <0xb00f>;
+ msi-map = <0x0 &gic_its 0x0 0x10000>;
+ dma-coherent;
+ ranges = <0x01000000 0x0 0x18001000 0x00 0x18001000 0x0 0x0010000>,
+diff --git a/arch/arm64/boot/dts/ti/k3-j7200.dtsi b/arch/arm64/boot/dts/ti/k3-j7200.dtsi
+index b7005b8031495..afe99f3920ccd 100644
+--- a/arch/arm64/boot/dts/ti/k3-j7200.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-j7200.dtsi
+@@ -60,7 +60,7 @@
+ i-cache-sets = <256>;
+ d-cache-size = <0x8000>;
+ d-cache-line-size = <64>;
+- d-cache-sets = <128>;
++ d-cache-sets = <256>;
+ next-level-cache = <&L2_0>;
+ };
+
+@@ -74,7 +74,7 @@
+ i-cache-sets = <256>;
+ d-cache-size = <0x8000>;
+ d-cache-line-size = <64>;
+- d-cache-sets = <128>;
++ d-cache-sets = <256>;
+ next-level-cache = <&L2_0>;
+ };
+ };
+@@ -84,7 +84,7 @@
+ cache-level = <2>;
+ cache-size = <0x100000>;
+ cache-line-size = <64>;
+- cache-sets = <2048>;
++ cache-sets = <1024>;
+ next-level-cache = <&msmc_l3>;
+ };
+
+@@ -127,6 +127,7 @@
+ <0x00 0x00a40000 0x00 0x00a40000 0x00 0x00000800>, /* timesync router */
+ <0x00 0x01000000 0x00 0x01000000 0x00 0x0d000000>, /* Most peripherals */
+ <0x00 0x30000000 0x00 0x30000000 0x00 0x0c400000>, /* MAIN NAVSS */
++ <0x00 0x6f000000 0x00 0x6f000000 0x00 0x00310000>, /* A72 PERIPHBASE */
+ <0x00 0x70000000 0x00 0x70000000 0x00 0x00800000>, /* MSMC RAM */
+ <0x00 0x18000000 0x00 0x18000000 0x00 0x08000000>, /* PCIe1 DAT0 */
+ <0x41 0x00000000 0x41 0x00000000 0x01 0x00000000>, /* PCIe1 DAT1 */
+diff --git a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
+index cf3482376c1e6..6c81997ee28ad 100644
+--- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
+@@ -42,7 +42,7 @@
+ #size-cells = <1>;
+ ranges = <0x0 0x0 0x00100000 0x1c000>;
+
+- serdes_ln_ctrl: mux@4080 {
++ serdes_ln_ctrl: mux-controller@4080 {
+ compatible = "mmio-mux";
+ reg = <0x00004080 0x50>;
+ #mux-control-cells = <1>;
+@@ -76,7 +76,10 @@
+ #interrupt-cells = <3>;
+ interrupt-controller;
+ reg = <0x00 0x01800000 0x00 0x10000>, /* GICD */
+- <0x00 0x01900000 0x00 0x100000>; /* GICR */
++ <0x00 0x01900000 0x00 0x100000>, /* GICR */
++ <0x00 0x6f000000 0x00 0x2000>, /* GICC */
++ <0x00 0x6f010000 0x00 0x1000>, /* GICH */
++ <0x00 0x6f020000 0x00 0x2000>; /* GICV */
+
+ /* vcpumntirq: virtual CPU interface maintenance interrupt */
+ interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+@@ -610,7 +613,7 @@
+ clock-names = "fck";
+ #address-cells = <3>;
+ #size-cells = <2>;
+- bus-range = <0x0 0xf>;
++ bus-range = <0x0 0xff>;
+ vendor-id = <0x104c>;
+ device-id = <0xb00d>;
+ msi-map = <0x0 &gic_its 0x0 0x10000>;
+@@ -636,7 +639,7 @@
+ clocks = <&k3_clks 239 1>;
+ clock-names = "fck";
+ max-functions = /bits/ 8 <6>;
+- max-virtual-functions = /bits/ 16 <4 4 4 4 0 0>;
++ max-virtual-functions = /bits/ 8 <4 4 4 4 0 0>;
+ dma-coherent;
+ };
+
+@@ -658,7 +661,7 @@
+ clock-names = "fck";
+ #address-cells = <3>;
+ #size-cells = <2>;
+- bus-range = <0x0 0xf>;
++ bus-range = <0x0 0xff>;
+ vendor-id = <0x104c>;
+ device-id = <0xb00d>;
+ msi-map = <0x0 &gic_its 0x10000 0x10000>;
+@@ -684,7 +687,7 @@
+ clocks = <&k3_clks 240 1>;
+ clock-names = "fck";
+ max-functions = /bits/ 8 <6>;
+- max-virtual-functions = /bits/ 16 <4 4 4 4 0 0>;
++ max-virtual-functions = /bits/ 8 <4 4 4 4 0 0>;
+ dma-coherent;
+ };
+
+@@ -706,7 +709,7 @@
+ clock-names = "fck";
+ #address-cells = <3>;
+ #size-cells = <2>;
+- bus-range = <0x0 0xf>;
++ bus-range = <0x0 0xff>;
+ vendor-id = <0x104c>;
+ device-id = <0xb00d>;
+ msi-map = <0x0 &gic_its 0x20000 0x10000>;
+@@ -732,7 +735,7 @@
+ clocks = <&k3_clks 241 1>;
+ clock-names = "fck";
+ max-functions = /bits/ 8 <6>;
+- max-virtual-functions = /bits/ 16 <4 4 4 4 0 0>;
++ max-virtual-functions = /bits/ 8 <4 4 4 4 0 0>;
+ dma-coherent;
+ };
+
+@@ -754,7 +757,7 @@
+ clock-names = "fck";
+ #address-cells = <3>;
+ #size-cells = <2>;
+- bus-range = <0x0 0xf>;
++ bus-range = <0x0 0xff>;
+ vendor-id = <0x104c>;
+ device-id = <0xb00d>;
+ msi-map = <0x0 &gic_its 0x30000 0x10000>;
+@@ -780,7 +783,7 @@
+ clocks = <&k3_clks 242 1>;
+ clock-names = "fck";
+ max-functions = /bits/ 8 <6>;
+- max-virtual-functions = /bits/ 16 <4 4 4 4 0 0>;
++ max-virtual-functions = /bits/ 8 <4 4 4 4 0 0>;
+ dma-coherent;
+ #address-cells = <2>;
+ #size-cells = <2>;
+diff --git a/arch/arm64/boot/dts/ti/k3-j721e.dtsi b/arch/arm64/boot/dts/ti/k3-j721e.dtsi
+index f0587fde147e6..2cd8883de5b53 100644
+--- a/arch/arm64/boot/dts/ti/k3-j721e.dtsi
++++ b/arch/arm64/boot/dts/ti/k3-j721e.dtsi
+@@ -61,7 +61,7 @@
+ i-cache-sets = <256>;
+ d-cache-size = <0x8000>;
+ d-cache-line-size = <64>;
+- d-cache-sets = <128>;
++ d-cache-sets = <256>;
+ next-level-cache = <&L2_0>;
+ };
+
+@@ -75,7 +75,7 @@
+ i-cache-sets = <256>;
+ d-cache-size = <0x8000>;
+ d-cache-line-size = <64>;
+- d-cache-sets = <128>;
++ d-cache-sets = <256>;
+ next-level-cache = <&L2_0>;
+ };
+ };
+@@ -85,7 +85,7 @@
+ cache-level = <2>;
+ cache-size = <0x100000>;
+ cache-line-size = <64>;
+- cache-sets = <2048>;
++ cache-sets = <1024>;
+ next-level-cache = <&msmc_l3>;
+ };
+
+@@ -136,6 +136,7 @@
+ <0x00 0x0e000000 0x00 0x0e000000 0x00 0x01800000>, /* PCIe Core*/
+ <0x00 0x10000000 0x00 0x10000000 0x00 0x10000000>, /* PCIe DAT */
+ <0x00 0x64800000 0x00 0x64800000 0x00 0x00800000>, /* C71 */
++ <0x00 0x6f000000 0x00 0x6f000000 0x00 0x00310000>, /* A72 PERIPHBASE */
+ <0x44 0x00000000 0x44 0x00000000 0x00 0x08000000>, /* PCIe2 DAT */
+ <0x44 0x10000000 0x44 0x10000000 0x00 0x08000000>, /* PCIe3 DAT */
+ <0x4d 0x80800000 0x4d 0x80800000 0x00 0x00800000>, /* C66_0 */
+diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
+index 4a86efa32d687..f7124e15f0ff6 100644
+--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
++++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
+@@ -131,7 +131,7 @@
+ reg = <0>;
+
+ partition@0 {
+- label = "data";
++ label = "spi0-data";
+ reg = <0x0 0x100000>;
+ };
+ };
+@@ -149,7 +149,7 @@
+ reg = <0>;
+
+ partition@0 {
+- label = "data";
++ label = "spi1-data";
+ reg = <0x0 0x84000>;
+ };
+ };
+diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
+index 28dccb891a535..8278876ad33fa 100644
+--- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
++++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
+@@ -792,7 +792,7 @@
+ };
+
+ uart0: serial@ff000000 {
+- compatible = "cdns,uart-r1p12", "xlnx,xuartps";
++ compatible = "xlnx,zynqmp-uart", "cdns,uart-r1p12";
+ status = "disabled";
+ interrupt-parent = <&gic>;
+ interrupts = <0 21 4>;
+@@ -802,7 +802,7 @@
+ };
+
+ uart1: serial@ff010000 {
+- compatible = "cdns,uart-r1p12", "xlnx,xuartps";
++ compatible = "xlnx,zynqmp-uart", "cdns,uart-r1p12";
+ status = "disabled";
+ interrupt-parent = <&gic>;
+ interrupts = <0 22 4>;
+diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
+index 545197bc05013..4972a81d40d60 100644
+--- a/arch/arm64/configs/defconfig
++++ b/arch/arm64/configs/defconfig
+@@ -921,7 +921,7 @@ CONFIG_DMADEVICES=y
+ CONFIG_DMA_BCM2835=y
+ CONFIG_DMA_SUN6I=m
+ CONFIG_FSL_EDMA=y
+-CONFIG_IMX_SDMA=y
++CONFIG_IMX_SDMA=m
+ CONFIG_K3_DMA=y
+ CONFIG_MV_XOR=y
+ CONFIG_MV_XOR_V2=y
+diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
+index bfa58409a4d4d..448a575db8e8e 100644
+--- a/arch/arm64/include/asm/assembler.h
++++ b/arch/arm64/include/asm/assembler.h
+@@ -107,6 +107,13 @@
+ hint #20
+ .endm
+
++/*
++ * Clear Branch History instruction
++ */
++ .macro clearbhb
++ hint #22
++ .endm
++
+ /*
+ * Speculation barrier
+ */
+@@ -830,4 +837,50 @@ alternative_endif
+
+ #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
+
++ .macro __mitigate_spectre_bhb_loop tmp
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++alternative_cb spectre_bhb_patch_loop_iter
++ mov \tmp, #32 // Patched to correct the immediate
++alternative_cb_end
++.Lspectre_bhb_loop\@:
++ b . + 4
++ subs \tmp, \tmp, #1
++ b.ne .Lspectre_bhb_loop\@
++ sb
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++ .endm
++
++ .macro mitigate_spectre_bhb_loop tmp
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++alternative_cb spectre_bhb_patch_loop_mitigation_enable
++ b .L_spectre_bhb_loop_done\@ // Patched to NOP
++alternative_cb_end
++ __mitigate_spectre_bhb_loop \tmp
++.L_spectre_bhb_loop_done\@:
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++ .endm
++
++ /* Save/restores x0-x3 to the stack */
++ .macro __mitigate_spectre_bhb_fw
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++ stp x0, x1, [sp, #-16]!
++ stp x2, x3, [sp, #-16]!
++ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
++alternative_cb smccc_patch_fw_mitigation_conduit
++ nop // Patched to SMC/HVC #0
++alternative_cb_end
++ ldp x2, x3, [sp], #16
++ ldp x0, x1, [sp], #16
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++ .endm
++
++ .macro mitigate_spectre_bhb_clear_insn
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++alternative_cb spectre_bhb_patch_clearbhb
++ /* Patched to NOP when not supported */
++ clearbhb
++ isb
++alternative_cb_end
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++ .endm
+ #endif /* __ASM_ASSEMBLER_H */
+diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h
+index 0f6d16faa5402..a58e366f0b074 100644
+--- a/arch/arm64/include/asm/cpu.h
++++ b/arch/arm64/include/asm/cpu.h
+@@ -51,6 +51,7 @@ struct cpuinfo_arm64 {
+ u64 reg_id_aa64dfr1;
+ u64 reg_id_aa64isar0;
+ u64 reg_id_aa64isar1;
++ u64 reg_id_aa64isar2;
+ u64 reg_id_aa64mmfr0;
+ u64 reg_id_aa64mmfr1;
+ u64 reg_id_aa64mmfr2;
+diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
+index ef6be92b1921a..a77b5f49b3a6c 100644
+--- a/arch/arm64/include/asm/cpufeature.h
++++ b/arch/arm64/include/asm/cpufeature.h
+@@ -637,6 +637,35 @@ static inline bool cpu_supports_mixed_endian_el0(void)
+ return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
+ }
+
++
++static inline bool supports_csv2p3(int scope)
++{
++ u64 pfr0;
++ u8 csv2_val;
++
++ if (scope == SCOPE_LOCAL_CPU)
++ pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
++ else
++ pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
++
++ csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
++ ID_AA64PFR0_CSV2_SHIFT);
++ return csv2_val == 3;
++}
++
++static inline bool supports_clearbhb(int scope)
++{
++ u64 isar2;
++
++ if (scope == SCOPE_LOCAL_CPU)
++ isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
++ else
++ isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
++
++ return cpuid_feature_extract_unsigned_field(isar2,
++ ID_AA64ISAR2_CLEARBHB_SHIFT);
++}
++
+ const struct cpumask *system_32bit_el0_cpumask(void);
+ DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
+
+diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
+index 6231e1f0abe7e..39f5c1672f480 100644
+--- a/arch/arm64/include/asm/cputype.h
++++ b/arch/arm64/include/asm/cputype.h
+@@ -73,6 +73,15 @@
+ #define ARM_CPU_PART_CORTEX_A76 0xD0B
+ #define ARM_CPU_PART_NEOVERSE_N1 0xD0C
+ #define ARM_CPU_PART_CORTEX_A77 0xD0D
++#define ARM_CPU_PART_NEOVERSE_V1 0xD40
++#define ARM_CPU_PART_CORTEX_A78 0xD41
++#define ARM_CPU_PART_CORTEX_A78AE 0xD42
++#define ARM_CPU_PART_CORTEX_X1 0xD44
++#define ARM_CPU_PART_CORTEX_A510 0xD46
++#define ARM_CPU_PART_CORTEX_A710 0xD47
++#define ARM_CPU_PART_CORTEX_X2 0xD48
++#define ARM_CPU_PART_NEOVERSE_N2 0xD49
++#define ARM_CPU_PART_CORTEX_A78C 0xD4B
+
+ #define APM_CPU_PART_POTENZA 0x000
+
+@@ -113,6 +122,15 @@
+ #define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)
+ #define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)
+ #define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)
++#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1)
++#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78)
++#define MIDR_CORTEX_A78AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78AE)
++#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
++#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
++#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
++#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
++#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
++#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
+ #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+ #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
+ #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
+diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
+index d3e1825337be3..ad55079abe476 100644
+--- a/arch/arm64/include/asm/efi.h
++++ b/arch/arm64/include/asm/efi.h
+@@ -14,7 +14,6 @@
+
+ #ifdef CONFIG_EFI
+ extern void efi_init(void);
+-extern void efifb_setup_from_dmi(struct screen_info *si, const char *opt);
+ #else
+ #define efi_init()
+ #endif
+diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
+index 3198acb2aad8c..7f3c87f7a0cec 100644
+--- a/arch/arm64/include/asm/el2_setup.h
++++ b/arch/arm64/include/asm/el2_setup.h
+@@ -106,7 +106,7 @@
+ msr_s SYS_ICC_SRE_EL2, x0
+ isb // Make sure SRE is now set
+ mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back,
+- tbz x0, #0, 1f // and check that it sticks
++ tbz x0, #0, .Lskip_gicv3_\@ // and check that it sticks
+ msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
+ .Lskip_gicv3_\@:
+ .endm
+diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
+index 29f97eb3dad41..8f59bbeba7a7e 100644
+--- a/arch/arm64/include/asm/esr.h
++++ b/arch/arm64/include/asm/esr.h
+@@ -68,6 +68,7 @@
+ #define ESR_ELx_EC_MAX (0x3F)
+
+ #define ESR_ELx_EC_SHIFT (26)
++#define ESR_ELx_EC_WIDTH (6)
+ #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
+ #define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT)
+
+diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
+index b15eb4a3e6b20..840a35ed92ec8 100644
+--- a/arch/arm64/include/asm/extable.h
++++ b/arch/arm64/include/asm/extable.h
+@@ -22,15 +22,6 @@ struct exception_table_entry
+
+ #define ARCH_HAS_RELATIVE_EXTABLE
+
+-static inline bool in_bpf_jit(struct pt_regs *regs)
+-{
+- if (!IS_ENABLED(CONFIG_BPF_JIT))
+- return false;
+-
+- return regs->pc >= BPF_JIT_REGION_START &&
+- regs->pc < BPF_JIT_REGION_END;
+-}
+-
+ #ifdef CONFIG_BPF_JIT
+ int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
+ struct pt_regs *regs);
+diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
+index 4335800201c97..daff882883f92 100644
+--- a/arch/arm64/include/asm/fixmap.h
++++ b/arch/arm64/include/asm/fixmap.h
+@@ -62,9 +62,11 @@ enum fixed_addresses {
+ #endif /* CONFIG_ACPI_APEI_GHES */
+
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
++ FIX_ENTRY_TRAMP_TEXT3,
++ FIX_ENTRY_TRAMP_TEXT2,
++ FIX_ENTRY_TRAMP_TEXT1,
+ FIX_ENTRY_TRAMP_DATA,
+- FIX_ENTRY_TRAMP_TEXT,
+-#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
++#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
+ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+ __end_of_permanent_fixed_addresses,
+
+diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
+index 8c129db8232a6..f68fbb2074730 100644
+--- a/arch/arm64/include/asm/hwcap.h
++++ b/arch/arm64/include/asm/hwcap.h
+@@ -105,6 +105,9 @@
+ #define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
+ #define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
+ #define KERNEL_HWCAP_MTE __khwcap2_feature(MTE)
++#define KERNEL_HWCAP_ECV __khwcap2_feature(ECV)
++#define KERNEL_HWCAP_AFP __khwcap2_feature(AFP)
++#define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES)
+
+ /*
+ * This yields a mask that user programs can use to figure out what
+diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
+index 6b776c8667b20..b02f0c328c8e4 100644
+--- a/arch/arm64/include/asm/insn.h
++++ b/arch/arm64/include/asm/insn.h
+@@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op {
+ AARCH64_INSN_HINT_PSB = 0x11 << 5,
+ AARCH64_INSN_HINT_TSB = 0x12 << 5,
+ AARCH64_INSN_HINT_CSDB = 0x14 << 5,
++ AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5,
+
+ AARCH64_INSN_HINT_BTI = 0x20 << 5,
+ AARCH64_INSN_HINT_BTIC = 0x22 << 5,
+diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
+index 7fd836bea7eb4..3995652daf81a 100644
+--- a/arch/arm64/include/asm/io.h
++++ b/arch/arm64/include/asm/io.h
+@@ -192,4 +192,8 @@ extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size);
+ extern int valid_phys_addr_range(phys_addr_t addr, size_t size);
+ extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size);
+
++extern bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size,
++ unsigned long flags);
++#define arch_memremap_can_ram_remap arch_memremap_can_ram_remap
++
+ #endif /* __ASM_IO_H */
+diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
+index 327120c0089fe..f67a561e0935e 100644
+--- a/arch/arm64/include/asm/kvm_arm.h
++++ b/arch/arm64/include/asm/kvm_arm.h
+@@ -91,7 +91,7 @@
+ #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
+
+ /* TCR_EL2 Registers bits */
+-#define TCR_EL2_RES1 ((1 << 31) | (1 << 23))
++#define TCR_EL2_RES1 ((1U << 31) | (1 << 23))
+ #define TCR_EL2_TBI (1 << 20)
+ #define TCR_EL2_PS_SHIFT 16
+ #define TCR_EL2_PS_MASK (7 << TCR_EL2_PS_SHIFT)
+@@ -276,7 +276,7 @@
+ #define CPTR_EL2_TFP_SHIFT 10
+
+ /* Hyp Coprocessor Trap Register */
+-#define CPTR_EL2_TCPAC (1 << 31)
++#define CPTR_EL2_TCPAC (1U << 31)
+ #define CPTR_EL2_TAM (1 << 30)
+ #define CPTR_EL2_TTA (1 << 20)
+ #define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT)
+diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
+index f8be56d5342ba..fc6ee6c5972d0 100644
+--- a/arch/arm64/include/asm/kvm_host.h
++++ b/arch/arm64/include/asm/kvm_host.h
+@@ -711,6 +711,11 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt)
+ ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();
+ }
+
++static inline bool kvm_system_needs_idmapped_vectors(void)
++{
++ return cpus_have_const_cap(ARM64_SPECTRE_V3A);
++}
++
+ void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu);
+
+ static inline void kvm_arch_hardware_unsetup(void) {}
+diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
+index f1745a8434144..05886322c300c 100644
+--- a/arch/arm64/include/asm/memory.h
++++ b/arch/arm64/include/asm/memory.h
+@@ -44,11 +44,8 @@
+ #define _PAGE_OFFSET(va) (-(UL(1) << (va)))
+ #define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS))
+ #define KIMAGE_VADDR (MODULES_END)
+-#define BPF_JIT_REGION_START (_PAGE_END(VA_BITS_MIN))
+-#define BPF_JIT_REGION_SIZE (SZ_128M)
+-#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+ #define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
+-#define MODULES_VADDR (BPF_JIT_REGION_END)
++#define MODULES_VADDR (_PAGE_END(VA_BITS_MIN))
+ #define MODULES_VSIZE (SZ_128M)
+ #define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT)))
+ #define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE)
+diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h
+index a11ccadd47d29..094701ec5500b 100644
+--- a/arch/arm64/include/asm/module.lds.h
++++ b/arch/arm64/include/asm/module.lds.h
+@@ -1,8 +1,8 @@
+ SECTIONS {
+ #ifdef CONFIG_ARM64_MODULE_PLTS
+- .plt 0 (NOLOAD) : { BYTE(0) }
+- .init.plt 0 (NOLOAD) : { BYTE(0) }
+- .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) }
++ .plt 0 : { BYTE(0) }
++ .init.plt 0 : { BYTE(0) }
++ .text.ftrace_trampoline 0 : { BYTE(0) }
+ #endif
+
+ #ifdef CONFIG_KASAN_SW_TAGS
+diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
+index 22420e1f8c037..592aabb25b0e7 100644
+--- a/arch/arm64/include/asm/mte-kasan.h
++++ b/arch/arm64/include/asm/mte-kasan.h
+@@ -5,6 +5,7 @@
+ #ifndef __ASM_MTE_KASAN_H
+ #define __ASM_MTE_KASAN_H
+
++#include <asm/compiler.h>
+ #include <asm/mte-def.h>
+
+ #ifndef __ASSEMBLY__
+@@ -84,10 +85,12 @@ static inline void __dc_gzva(u64 p)
+ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
+ bool init)
+ {
+- u64 curr, mask, dczid_bs, end1, end2, end3;
++ u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3;
+
+ /* Read DC G(Z)VA block size from the system register. */
+- dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);
++ dczid = read_cpuid(DCZID_EL0);
++ dczid_bs = 4ul << (dczid & 0xf);
++ dczid_dzp = (dczid >> 4) & 1;
+
+ curr = (u64)__tag_set(addr, tag);
+ mask = dczid_bs - 1;
+@@ -106,7 +109,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
+ */
+ #define SET_MEMTAG_RANGE(stg_post, dc_gva) \
+ do { \
+- if (size >= 2 * dczid_bs) { \
++ if (!dczid_dzp && size >= 2 * dczid_bs) {\
+ do { \
+ curr = stg_post(curr); \
+ } while (curr < end1); \
+diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
+index f98c91bbd7c17..993a27ea6f543 100644
+--- a/arch/arm64/include/asm/page.h
++++ b/arch/arm64/include/asm/page.h
+@@ -41,7 +41,6 @@ void tag_clear_highpage(struct page *to);
+
+ typedef struct page *pgtable_t;
+
+-int pfn_valid(unsigned long pfn);
+ int pfn_is_map_memory(unsigned long pfn);
+
+ #include <asm/memory.h>
+diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
+index 8433a2058eb15..237224484d0f6 100644
+--- a/arch/arm64/include/asm/pgalloc.h
++++ b/arch/arm64/include/asm/pgalloc.h
+@@ -76,7 +76,7 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep,
+ static inline void
+ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
+ {
+- VM_BUG_ON(mm != &init_mm);
++ VM_BUG_ON(mm && mm != &init_mm);
+ __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN);
+ }
+
+diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
+index 7032f04c8ac6e..b1e1b74d993c3 100644
+--- a/arch/arm64/include/asm/pgtable-prot.h
++++ b/arch/arm64/include/asm/pgtable-prot.h
+@@ -92,7 +92,7 @@ extern bool arm64_use_ng_mappings;
+ #define __P001 PAGE_READONLY
+ #define __P010 PAGE_READONLY
+ #define __P011 PAGE_READONLY
+-#define __P100 PAGE_EXECONLY
++#define __P100 PAGE_READONLY_EXEC /* PAGE_EXECONLY if Enhanced PAN */
+ #define __P101 PAGE_READONLY_EXEC
+ #define __P110 PAGE_READONLY_EXEC
+ #define __P111 PAGE_READONLY_EXEC
+@@ -101,7 +101,7 @@ extern bool arm64_use_ng_mappings;
+ #define __S001 PAGE_READONLY
+ #define __S010 PAGE_SHARED
+ #define __S011 PAGE_SHARED
+-#define __S100 PAGE_EXECONLY
++#define __S100 PAGE_READONLY_EXEC /* PAGE_EXECONLY if Enhanced PAN */
+ #define __S101 PAGE_READONLY_EXEC
+ #define __S110 PAGE_SHARED_EXEC
+ #define __S111 PAGE_SHARED_EXEC
+diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
+index dfa76afa0ccff..ed57717cd0040 100644
+--- a/arch/arm64/include/asm/pgtable.h
++++ b/arch/arm64/include/asm/pgtable.h
+@@ -67,9 +67,15 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
+ * page table entry, taking care of 52-bit addresses.
+ */
+ #ifdef CONFIG_ARM64_PA_BITS_52
+-#define __pte_to_phys(pte) \
+- ((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
+-#define __phys_to_pte_val(phys) (((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
++static inline phys_addr_t __pte_to_phys(pte_t pte)
++{
++ return (pte_val(pte) & PTE_ADDR_LOW) |
++ ((pte_val(pte) & PTE_ADDR_HIGH) << 36);
++}
++static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
++{
++ return (phys | (phys >> 36)) & PTE_ADDR_MASK;
++}
+ #else
+ #define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
+ #define __phys_to_pte_val(phys) (phys)
+@@ -529,7 +535,7 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
+ PMD_TYPE_TABLE)
+ #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
+ PMD_TYPE_SECT)
+-#define pmd_leaf(pmd) pmd_sect(pmd)
++#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd))
+ #define pmd_bad(pmd) (!pmd_table(pmd))
+
+ #define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
+@@ -619,7 +625,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
+ #define pud_none(pud) (!pud_val(pud))
+ #define pud_bad(pud) (!pud_table(pud))
+ #define pud_present(pud) pte_present(pud_pte(pud))
+-#define pud_leaf(pud) pud_sect(pud)
++#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud))
+ #define pud_valid(pud) pte_valid(pud_pte(pud))
+
+ static inline void set_pud(pud_t *pudp, pud_t pud)
+@@ -1011,18 +1017,6 @@ static inline bool arch_wants_old_prefaulted_pte(void)
+ }
+ #define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte
+
+-static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
+-{
+- if (cpus_have_const_cap(ARM64_HAS_EPAN))
+- return prot;
+-
+- if (pgprot_val(prot) != pgprot_val(PAGE_EXECONLY))
+- return prot;
+-
+- return PAGE_READONLY_EXEC;
+-}
+-
+-
+ #endif /* !__ASSEMBLY__ */
+
+ #endif /* __ASM_PGTABLE_H */
+diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h
+index 1bce62fa908a3..56f7b1d4d54b9 100644
+--- a/arch/arm64/include/asm/rwonce.h
++++ b/arch/arm64/include/asm/rwonce.h
+@@ -5,7 +5,7 @@
+ #ifndef __ASM_RWONCE_H
+ #define __ASM_RWONCE_H
+
+-#ifdef CONFIG_LTO
++#if defined(CONFIG_LTO) && !defined(__ASSEMBLY__)
+
+ #include <linux/compiler_types.h>
+ #include <asm/alternative-macros.h>
+@@ -66,7 +66,7 @@
+ })
+
+ #endif /* !BUILD_VDSO */
+-#endif /* CONFIG_LTO */
++#endif /* CONFIG_LTO && !__ASSEMBLY__ */
+
+ #include <asm-generic/rwonce.h>
+
+diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
+index e4ad9db53af1d..552891e626e53 100644
+--- a/arch/arm64/include/asm/sections.h
++++ b/arch/arm64/include/asm/sections.h
+@@ -22,4 +22,9 @@ extern char __irqentry_text_start[], __irqentry_text_end[];
+ extern char __mmuoff_data_start[], __mmuoff_data_end[];
+ extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
+
++static inline size_t entry_tramp_text_size(void)
++{
++ return __entry_tramp_text_end - __entry_tramp_text_start;
++}
++
+ #endif /* __ASM_SECTIONS_H */
+diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h
+index f62ca39da6c5a..aa3d3607d5c8d 100644
+--- a/arch/arm64/include/asm/spectre.h
++++ b/arch/arm64/include/asm/spectre.h
+@@ -67,7 +67,8 @@ struct bp_hardening_data {
+
+ DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
+
+-static inline void arm64_apply_bp_hardening(void)
++/* Called during entry so must be __always_inline */
++static __always_inline void arm64_apply_bp_hardening(void)
+ {
+ struct bp_hardening_data *d;
+
+@@ -93,5 +94,9 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk);
+
+ enum mitigation_state arm64_get_meltdown_state(void);
+
++enum mitigation_state arm64_get_spectre_bhb_state(void);
++bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
++u8 spectre_bhb_loop_affected(int scope);
++void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+ #endif /* __ASSEMBLY__ */
+ #endif /* __ASM_SPECTRE_H */
+diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
+index b268082d67edd..394fc5998a4b2 100644
+--- a/arch/arm64/include/asm/sysreg.h
++++ b/arch/arm64/include/asm/sysreg.h
+@@ -180,6 +180,7 @@
+
+ #define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)
+ #define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)
++#define SYS_ID_AA64ISAR2_EL1 sys_reg(3, 0, 0, 6, 2)
+
+ #define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0)
+ #define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
+@@ -764,6 +765,21 @@
+ #define ID_AA64ISAR1_GPI_NI 0x0
+ #define ID_AA64ISAR1_GPI_IMP_DEF 0x1
+
++/* id_aa64isar2 */
++#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
++#define ID_AA64ISAR2_RPRES_SHIFT 4
++#define ID_AA64ISAR2_WFXT_SHIFT 0
++
++#define ID_AA64ISAR2_RPRES_8BIT 0x0
++#define ID_AA64ISAR2_RPRES_12BIT 0x1
++/*
++ * Value 0x1 has been removed from the architecture, and is
++ * reserved, but has not yet been removed from the ARM ARM
++ * as of ARM DDI 0487G.b.
++ */
++#define ID_AA64ISAR2_WFXT_NI 0x0
++#define ID_AA64ISAR2_WFXT_SUPPORTED 0x2
++
+ /* id_aa64pfr0 */
+ #define ID_AA64PFR0_CSV3_SHIFT 60
+ #define ID_AA64PFR0_CSV2_SHIFT 56
+@@ -881,6 +897,8 @@
+ #endif
+
+ /* id_aa64mmfr1 */
++#define ID_AA64MMFR1_ECBHB_SHIFT 60
++#define ID_AA64MMFR1_AFP_SHIFT 44
+ #define ID_AA64MMFR1_ETS_SHIFT 36
+ #define ID_AA64MMFR1_TWED_SHIFT 32
+ #define ID_AA64MMFR1_XNX_SHIFT 28
+diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
+index 190b494e22ab9..0fd6056ba412b 100644
+--- a/arch/arm64/include/asm/uaccess.h
++++ b/arch/arm64/include/asm/uaccess.h
+@@ -292,12 +292,22 @@ do { \
+ (x) = (__force __typeof__(*(ptr)))__gu_val; \
+ } while (0)
+
++/*
++ * We must not call into the scheduler between uaccess_ttbr0_enable() and
++ * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,
++ * we must evaluate these outside of the critical section.
++ */
+ #define __raw_get_user(x, ptr, err) \
+ do { \
++ __typeof__(*(ptr)) __user *__rgu_ptr = (ptr); \
++ __typeof__(x) __rgu_val; \
+ __chk_user_ptr(ptr); \
++ \
+ uaccess_ttbr0_enable(); \
+- __raw_get_mem("ldtr", x, ptr, err); \
++ __raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err); \
+ uaccess_ttbr0_disable(); \
++ \
++ (x) = __rgu_val; \
+ } while (0)
+
+ #define __get_user_error(x, ptr, err) \
+@@ -321,14 +331,22 @@ do { \
+
+ #define get_user __get_user
+
++/*
++ * We must not call into the scheduler between __uaccess_enable_tco_async() and
++ * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
++ * functions, we must evaluate these outside of the critical section.
++ */
+ #define __get_kernel_nofault(dst, src, type, err_label) \
+ do { \
++ __typeof__(dst) __gkn_dst = (dst); \
++ __typeof__(src) __gkn_src = (src); \
+ int __gkn_err = 0; \
+ \
+ __uaccess_enable_tco_async(); \
+- __raw_get_mem("ldr", *((type *)(dst)), \
+- (__force type *)(src), __gkn_err); \
++ __raw_get_mem("ldr", *((type *)(__gkn_dst)), \
++ (__force type *)(__gkn_src), __gkn_err); \
+ __uaccess_disable_tco_async(); \
++ \
+ if (unlikely(__gkn_err)) \
+ goto err_label; \
+ } while (0)
+@@ -367,11 +385,19 @@ do { \
+ } \
+ } while (0)
+
++/*
++ * We must not call into the scheduler between uaccess_ttbr0_enable() and
++ * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,
++ * we must evaluate these outside of the critical section.
++ */
+ #define __raw_put_user(x, ptr, err) \
+ do { \
+- __chk_user_ptr(ptr); \
++ __typeof__(*(ptr)) __user *__rpu_ptr = (ptr); \
++ __typeof__(*(ptr)) __rpu_val = (x); \
++ __chk_user_ptr(__rpu_ptr); \
++ \
+ uaccess_ttbr0_enable(); \
+- __raw_put_mem("sttr", x, ptr, err); \
++ __raw_put_mem("sttr", __rpu_val, __rpu_ptr, err); \
+ uaccess_ttbr0_disable(); \
+ } while (0)
+
+@@ -396,14 +422,22 @@ do { \
+
+ #define put_user __put_user
+
++/*
++ * We must not call into the scheduler between __uaccess_enable_tco_async() and
++ * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
++ * functions, we must evaluate these outside of the critical section.
++ */
+ #define __put_kernel_nofault(dst, src, type, err_label) \
+ do { \
++ __typeof__(dst) __pkn_dst = (dst); \
++ __typeof__(src) __pkn_src = (src); \
+ int __pkn_err = 0; \
+ \
+ __uaccess_enable_tco_async(); \
+- __raw_put_mem("str", *((type *)(src)), \
+- (__force type *)(dst), __pkn_err); \
++ __raw_put_mem("str", *((type *)(__pkn_src)), \
++ (__force type *)(__pkn_dst), __pkn_err); \
+ __uaccess_disable_tco_async(); \
++ \
+ if (unlikely(__pkn_err)) \
+ goto err_label; \
+ } while(0)
+diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h
+new file mode 100644
+index 0000000000000..bc9a2145f4194
+--- /dev/null
++++ b/arch/arm64/include/asm/vectors.h
+@@ -0,0 +1,73 @@
++/* SPDX-License-Identifier: GPL-2.0-only */
++/*
++ * Copyright (C) 2022 ARM Ltd.
++ */
++#ifndef __ASM_VECTORS_H
++#define __ASM_VECTORS_H
++
++#include <linux/bug.h>
++#include <linux/percpu.h>
++
++#include <asm/fixmap.h>
++
++extern char vectors[];
++extern char tramp_vectors[];
++extern char __bp_harden_el1_vectors[];
++
++/*
++ * Note: the order of this enum corresponds to two arrays in entry.S:
++ * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
++ * 'full fat' vectors are used directly.
++ */
++enum arm64_bp_harden_el1_vectors {
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++ /*
++ * Perform the BHB loop mitigation, before branching to the canonical
++ * vectors.
++ */
++ EL1_VECTOR_BHB_LOOP,
++
++ /*
++ * Make the SMC call for firmware mitigation, before branching to the
++ * canonical vectors.
++ */
++ EL1_VECTOR_BHB_FW,
++
++ /*
++ * Use the ClearBHB instruction, before branching to the canonical
++ * vectors.
++ */
++ EL1_VECTOR_BHB_CLEAR_INSN,
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++
++ /*
++ * Remap the kernel before branching to the canonical vectors.
++ */
++ EL1_VECTOR_KPTI,
++};
++
++#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++#define EL1_VECTOR_BHB_LOOP -1
++#define EL1_VECTOR_BHB_FW -1
++#define EL1_VECTOR_BHB_CLEAR_INSN -1
++#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++
++/* The vectors to use on return from EL0. e.g. to remap the kernel */
++DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
++
++#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
++#define TRAMP_VALIAS 0ul
++#endif
++
++static inline const char *
++arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
++{
++ if (arm64_kernel_unmapped_at_el0())
++ return (char *)(TRAMP_VALIAS + SZ_2K * slot);
++
++ WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
++
++ return __bp_harden_el1_vectors + SZ_2K * slot;
++}
++
++#endif /* __ASM_VECTORS_H */
+diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
+index b8f41aa234ee1..f03731847d9df 100644
+--- a/arch/arm64/include/uapi/asm/hwcap.h
++++ b/arch/arm64/include/uapi/asm/hwcap.h
+@@ -75,5 +75,8 @@
+ #define HWCAP2_RNG (1 << 16)
+ #define HWCAP2_BTI (1 << 17)
+ #define HWCAP2_MTE (1 << 18)
++#define HWCAP2_ECV (1 << 19)
++#define HWCAP2_AFP (1 << 20)
++#define HWCAP2_RPRES (1 << 21)
+
+ #endif /* _UAPI__ASM_HWCAP_H */
+diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
+index b3edde68bc3e0..323e251ed37bc 100644
+--- a/arch/arm64/include/uapi/asm/kvm.h
++++ b/arch/arm64/include/uapi/asm/kvm.h
+@@ -281,6 +281,11 @@ struct kvm_arm_copy_mte_tags {
+ #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3
+ #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4)
+
++#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 KVM_REG_ARM_FW_REG(3)
++#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL 0
++#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL 1
++#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED 2
++
+ /* SVE registers */
+ #define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT)
+
+diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
+index 3f1490bfb938a..749e31475e413 100644
+--- a/arch/arm64/kernel/Makefile
++++ b/arch/arm64/kernel/Makefile
+@@ -74,6 +74,10 @@ obj-$(CONFIG_ARM64_MTE) += mte.o
+ obj-y += vdso-wrap.o
+ obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o
+
++# Force dependency (vdso*-wrap.S includes vdso.so through incbin)
++$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so
++$(obj)/vdso32-wrap.o: $(obj)/vdso32/vdso.so
++
+ obj-y += probes/
+ head-y := head.o
+ extra-y += $(head-y) vmlinux.lds
+diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
+index 3fb79b76e9d96..7bbf5104b7b7b 100644
+--- a/arch/arm64/kernel/alternative.c
++++ b/arch/arm64/kernel/alternative.c
+@@ -42,7 +42,7 @@ bool alternative_is_applied(u16 cpufeature)
+ /*
+ * Check if the target PC is within an alternative block.
+ */
+-static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
++static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
+ {
+ unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);
+ return !(pc >= replptr && pc <= (replptr + alt->alt_len));
+@@ -50,7 +50,7 @@ static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
+
+ #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
+
+-static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
++static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
+ {
+ u32 insn;
+
+@@ -95,7 +95,7 @@ static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnp
+ return insn;
+ }
+
+-static void patch_alternative(struct alt_instr *alt,
++static noinstr void patch_alternative(struct alt_instr *alt,
+ __le32 *origptr, __le32 *updptr, int nr_inst)
+ {
+ __le32 *replptr;
+diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
+index e2c20c036442f..c67c19d701597 100644
+--- a/arch/arm64/kernel/cpu_errata.c
++++ b/arch/arm64/kernel/cpu_errata.c
+@@ -208,6 +208,8 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = {
+ #ifdef CONFIG_ARM64_ERRATUM_1286807
+ {
+ ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
++ /* Kryo4xx Gold (rcpe to rfpe) => (r0p0 to r3p0) */
++ ERRATA_MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xe),
+ },
+ #endif
+ {},
+@@ -464,6 +466,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
+ .matches = has_spectre_v4,
+ .cpu_enable = spectre_v4_enable_mitigation,
+ },
++ {
++ .desc = "Spectre-BHB",
++ .capability = ARM64_SPECTRE_BHB,
++ .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
++ .matches = is_spectre_bhb_affected,
++ .cpu_enable = spectre_bhb_enable_mitigation,
++ },
+ #ifdef CONFIG_ARM64_ERRATUM_1418040
+ {
+ .desc = "ARM erratum 1418040",
+diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
+index 6ec7036ef7e18..e71c9cfb46e8c 100644
+--- a/arch/arm64/kernel/cpufeature.c
++++ b/arch/arm64/kernel/cpufeature.c
+@@ -73,6 +73,8 @@
+ #include <linux/mm.h>
+ #include <linux/cpu.h>
+ #include <linux/kasan.h>
++#include <linux/percpu.h>
++
+ #include <asm/cpu.h>
+ #include <asm/cpufeature.h>
+ #include <asm/cpu_ops.h>
+@@ -85,6 +87,7 @@
+ #include <asm/smp.h>
+ #include <asm/sysreg.h>
+ #include <asm/traps.h>
++#include <asm/vectors.h>
+ #include <asm/virt.h>
+
+ /* Kernel representation of AT_HWCAP and AT_HWCAP2 */
+@@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+ bool arm64_use_ng_mappings = false;
+ EXPORT_SYMBOL(arm64_use_ng_mappings);
+
++DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
++
+ /*
+ * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
+ * support it?
+@@ -225,6 +230,12 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
+ ARM64_FTR_END,
+ };
+
++static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
++ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
++ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
++ ARM64_FTR_END,
++};
++
+ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
+@@ -279,7 +290,7 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
+ };
+
+ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
+- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
++ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0),
+ /*
+@@ -325,6 +336,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
+ };
+
+ static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
++ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_AFP_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_ETS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TWED_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_XNX_SHIFT, 4, 0),
+@@ -573,15 +585,19 @@ static const struct arm64_ftr_bits ftr_raz[] = {
+ ARM64_FTR_END,
+ };
+
+-#define ARM64_FTR_REG_OVERRIDE(id, table, ovr) { \
++#define __ARM64_FTR_REG_OVERRIDE(id_str, id, table, ovr) { \
+ .sys_id = id, \
+ .reg = &(struct arm64_ftr_reg){ \
+- .name = #id, \
++ .name = id_str, \
+ .override = (ovr), \
+ .ftr_bits = &((table)[0]), \
+ }}
+
+-#define ARM64_FTR_REG(id, table) ARM64_FTR_REG_OVERRIDE(id, table, &no_override)
++#define ARM64_FTR_REG_OVERRIDE(id, table, ovr) \
++ __ARM64_FTR_REG_OVERRIDE(#id, id, table, ovr)
++
++#define ARM64_FTR_REG(id, table) \
++ __ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
+
+ struct arm64_ftr_override __ro_after_init id_aa64mmfr1_override;
+ struct arm64_ftr_override __ro_after_init id_aa64pfr1_override;
+@@ -633,6 +649,7 @@ static const struct __ftr_reg_entry {
+ ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
+ ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1,
+ &id_aa64isar1_override),
++ ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2),
+
+ /* Op1 = 0, CRn = 0, CRm = 7 */
+ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
+@@ -929,6 +946,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
+ init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
+ init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
+ init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
++ init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
+ init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
+ init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
+ init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
+@@ -1147,6 +1165,8 @@ void update_cpu_features(int cpu,
+ info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
+ taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
+ info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
++ taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
++ info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
+
+ /*
+ * Differing PARange support is fine as long as all peripherals and
+@@ -1268,6 +1288,7 @@ u64 __read_sysreg_by_encoding(u32 sys_id)
+ read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
+ read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
+ read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
++ read_sysreg_case(SYS_ID_AA64ISAR2_EL1);
+
+ read_sysreg_case(SYS_CNTFRQ_EL0);
+ read_sysreg_case(SYS_CTR_EL0);
+@@ -1575,6 +1596,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
+
+ int cpu = smp_processor_id();
+
++ if (__this_cpu_read(this_cpu_vector) == vectors) {
++ const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
++
++ __this_cpu_write(this_cpu_vector, v);
++ }
++
+ /*
+ * We don't need to rewrite the page-tables if either we've done
+ * it already or we have KASLR enabled and therefore have not
+@@ -2451,6 +2478,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
+ #ifdef CONFIG_ARM64_MTE
+ HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE),
+ #endif /* CONFIG_ARM64_MTE */
++ HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
++ HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
++ HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
+ {},
+ };
+
+diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c
+index 03991eeff6430..3006f43248084 100644
+--- a/arch/arm64/kernel/cpuidle.c
++++ b/arch/arm64/kernel/cpuidle.c
+@@ -54,6 +54,9 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu)
+ struct acpi_lpi_state *lpi;
+ struct acpi_processor *pr = per_cpu(processors, cpu);
+
++ if (unlikely(!pr || !pr->flags.has_lpi))
++ return -EINVAL;
++
+ /*
+ * If the PSCI cpu_suspend function hook has not been initialized
+ * idle states must not be enabled, so bail out
+@@ -61,9 +64,6 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu)
+ if (!psci_ops.cpu_suspend)
+ return -EOPNOTSUPP;
+
+- if (unlikely(!pr || !pr->flags.has_lpi))
+- return -EINVAL;
+-
+ count = pr->power.count - 1;
+ if (count <= 0)
+ return -ENODEV;
+diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
+index 87731fea5e418..591c18a889a56 100644
+--- a/arch/arm64/kernel/cpuinfo.c
++++ b/arch/arm64/kernel/cpuinfo.c
+@@ -94,6 +94,9 @@ static const char *const hwcap_str[] = {
+ [KERNEL_HWCAP_RNG] = "rng",
+ [KERNEL_HWCAP_BTI] = "bti",
+ [KERNEL_HWCAP_MTE] = "mte",
++ [KERNEL_HWCAP_ECV] = "ecv",
++ [KERNEL_HWCAP_AFP] = "afp",
++ [KERNEL_HWCAP_RPRES] = "rpres",
+ };
+
+ #ifdef CONFIG_COMPAT
+@@ -390,6 +393,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
+ info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1);
+ info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1);
+ info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
++ info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1);
+ info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
+ info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
+ info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
+diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
+index b3e4f9a088b1a..8cf970d219f5d 100644
+--- a/arch/arm64/kernel/entry-ftrace.S
++++ b/arch/arm64/kernel/entry-ftrace.S
+@@ -77,11 +77,17 @@
+ .endm
+
+ SYM_CODE_START(ftrace_regs_caller)
++#ifdef BTI_C
++ BTI_C
++#endif
+ ftrace_regs_entry 1
+ b ftrace_common
+ SYM_CODE_END(ftrace_regs_caller)
+
+ SYM_CODE_START(ftrace_caller)
++#ifdef BTI_C
++ BTI_C
++#endif
+ ftrace_regs_entry 0
+ b ftrace_common
+ SYM_CODE_END(ftrace_caller)
+diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
+index bc6d5a970a131..34e38eb00f056 100644
+--- a/arch/arm64/kernel/entry.S
++++ b/arch/arm64/kernel/entry.S
+@@ -37,18 +37,21 @@
+
+ .macro kernel_ventry, el:req, ht:req, regsize:req, label:req
+ .align 7
+-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
++.Lventry_start\@:
+ .if \el == 0
+-alternative_if ARM64_UNMAP_KERNEL_AT_EL0
++ /*
++ * This must be the first instruction of the EL0 vector entries. It is
++ * skipped by the trampoline vectors, to trigger the cleanup.
++ */
++ b .Lskip_tramp_vectors_cleanup\@
+ .if \regsize == 64
+ mrs x30, tpidrro_el0
+ msr tpidrro_el0, xzr
+ .else
+ mov x30, xzr
+ .endif
+-alternative_else_nop_endif
++.Lskip_tramp_vectors_cleanup\@:
+ .endif
+-#endif
+
+ sub sp, sp, #PT_REGS_SIZE
+ #ifdef CONFIG_VMAP_STACK
+@@ -95,11 +98,15 @@ alternative_else_nop_endif
+ mrs x0, tpidrro_el0
+ #endif
+ b el\el\ht\()_\regsize\()_\label
++.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
+ .endm
+
+- .macro tramp_alias, dst, sym
++ .macro tramp_alias, dst, sym, tmp
+ mov_q \dst, TRAMP_VALIAS
+- add \dst, \dst, #(\sym - .entry.tramp.text)
++ adr_l \tmp, \sym
++ add \dst, \dst, \tmp
++ adr_l \tmp, .entry.tramp.text
++ sub \dst, \dst, \tmp
+ .endm
+
+ /*
+@@ -116,7 +123,7 @@ alternative_cb_end
+ tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
+ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
+ mov w1, #\state
+-alternative_cb spectre_v4_patch_fw_mitigation_conduit
++alternative_cb smccc_patch_fw_mitigation_conduit
+ nop // Patched to SMC/HVC #0
+ alternative_cb_end
+ .L__asm_ssbd_skip\@:
+@@ -413,21 +420,26 @@ alternative_else_nop_endif
+ ldp x24, x25, [sp, #16 * 12]
+ ldp x26, x27, [sp, #16 * 13]
+ ldp x28, x29, [sp, #16 * 14]
+- ldr lr, [sp, #S_LR]
+- add sp, sp, #PT_REGS_SIZE // restore sp
+
+ .if \el == 0
+-alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
++alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
++ ldr lr, [sp, #S_LR]
++ add sp, sp, #PT_REGS_SIZE // restore sp
++ eret
++alternative_else_nop_endif
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ bne 4f
+- msr far_el1, x30
+- tramp_alias x30, tramp_exit_native
++ msr far_el1, x29
++ tramp_alias x30, tramp_exit_native, x29
+ br x30
+ 4:
+- tramp_alias x30, tramp_exit_compat
++ tramp_alias x30, tramp_exit_compat, x29
+ br x30
+ #endif
+ .else
++ ldr lr, [sp, #S_LR]
++ add sp, sp, #PT_REGS_SIZE // restore sp
++
+ /* Ensure any device/NC reads complete */
+ alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412
+
+@@ -594,12 +606,6 @@ SYM_CODE_END(ret_to_user)
+
+ .popsection // .entry.text
+
+-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+-/*
+- * Exception vectors trampoline.
+- */
+- .pushsection ".entry.tramp.text", "ax"
+-
+ // Move from tramp_pg_dir to swapper_pg_dir
+ .macro tramp_map_kernel, tmp
+ mrs \tmp, ttbr1_el1
+@@ -633,12 +639,47 @@ alternative_else_nop_endif
+ */
+ .endm
+
+- .macro tramp_ventry, regsize = 64
++ .macro tramp_data_page dst
++ adr_l \dst, .entry.tramp.text
++ sub \dst, \dst, PAGE_SIZE
++ .endm
++
++ .macro tramp_data_read_var dst, var
++#ifdef CONFIG_RANDOMIZE_BASE
++ tramp_data_page \dst
++ add \dst, \dst, #:lo12:__entry_tramp_data_\var
++ ldr \dst, [\dst]
++#else
++ ldr \dst, =\var
++#endif
++ .endm
++
++#define BHB_MITIGATION_NONE 0
++#define BHB_MITIGATION_LOOP 1
++#define BHB_MITIGATION_FW 2
++#define BHB_MITIGATION_INSN 3
++
++ .macro tramp_ventry, vector_start, regsize, kpti, bhb
+ .align 7
+ 1:
+ .if \regsize == 64
+ msr tpidrro_el0, x30 // Restored in kernel_ventry
+ .endif
++
++ .if \bhb == BHB_MITIGATION_LOOP
++ /*
++ * This sequence must appear before the first indirect branch. i.e. the
++ * ret out of tramp_ventry. It appears here because x30 is free.
++ */
++ __mitigate_spectre_bhb_loop x30
++ .endif // \bhb == BHB_MITIGATION_LOOP
++
++ .if \bhb == BHB_MITIGATION_INSN
++ clearbhb
++ isb
++ .endif // \bhb == BHB_MITIGATION_INSN
++
++ .if \kpti == 1
+ /*
+ * Defend against branch aliasing attacks by pushing a dummy
+ * entry onto the return stack and using a RET instruction to
+@@ -648,46 +689,75 @@ alternative_else_nop_endif
+ b .
+ 2:
+ tramp_map_kernel x30
+-#ifdef CONFIG_RANDOMIZE_BASE
+- adr x30, tramp_vectors + PAGE_SIZE
+ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
+- ldr x30, [x30]
+-#else
+- ldr x30, =vectors
+-#endif
++ tramp_data_read_var x30, vectors
+ alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
+- prfm plil1strm, [x30, #(1b - tramp_vectors)]
++ prfm plil1strm, [x30, #(1b - \vector_start)]
+ alternative_else_nop_endif
++
+ msr vbar_el1, x30
+- add x30, x30, #(1b - tramp_vectors)
+ isb
++ .else
++ ldr x30, =vectors
++ .endif // \kpti == 1
++
++ .if \bhb == BHB_MITIGATION_FW
++ /*
++ * The firmware sequence must appear before the first indirect branch.
++ * i.e. the ret out of tramp_ventry. But it also needs the stack to be
++ * mapped to save/restore the registers the SMC clobbers.
++ */
++ __mitigate_spectre_bhb_fw
++ .endif // \bhb == BHB_MITIGATION_FW
++
++ add x30, x30, #(1b - \vector_start + 4)
+ ret
++.org 1b + 128 // Did we overflow the ventry slot?
+ .endm
+
+ .macro tramp_exit, regsize = 64
+- adr x30, tramp_vectors
++ tramp_data_read_var x30, this_cpu_vector
++ get_this_cpu_offset x29
++ ldr x30, [x30, x29]
++
+ msr vbar_el1, x30
+- tramp_unmap_kernel x30
++ ldr lr, [sp, #S_LR]
++ tramp_unmap_kernel x29
+ .if \regsize == 64
+- mrs x30, far_el1
++ mrs x29, far_el1
+ .endif
++ add sp, sp, #PT_REGS_SIZE // restore sp
+ eret
+ sb
+ .endm
+
+- .align 11
+-SYM_CODE_START_NOALIGN(tramp_vectors)
++ .macro generate_tramp_vector, kpti, bhb
++.Lvector_start\@:
+ .space 0x400
+
+- tramp_ventry
+- tramp_ventry
+- tramp_ventry
+- tramp_ventry
++ .rept 4
++ tramp_ventry .Lvector_start\@, 64, \kpti, \bhb
++ .endr
++ .rept 4
++ tramp_ventry .Lvector_start\@, 32, \kpti, \bhb
++ .endr
++ .endm
+
+- tramp_ventry 32
+- tramp_ventry 32
+- tramp_ventry 32
+- tramp_ventry 32
++#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
++/*
++ * Exception vectors trampoline.
++ * The order must match __bp_harden_el1_vectors and the
++ * arm64_bp_harden_el1_vectors enum.
++ */
++ .pushsection ".entry.tramp.text", "ax"
++ .align 11
++SYM_CODE_START_NOALIGN(tramp_vectors)
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
++ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
++ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
+ SYM_CODE_END(tramp_vectors)
+
+ SYM_CODE_START(tramp_exit_native)
+@@ -704,12 +774,56 @@ SYM_CODE_END(tramp_exit_compat)
+ .pushsection ".rodata", "a"
+ .align PAGE_SHIFT
+ SYM_DATA_START(__entry_tramp_data_start)
++__entry_tramp_data_vectors:
+ .quad vectors
++#ifdef CONFIG_ARM_SDE_INTERFACE
++__entry_tramp_data___sdei_asm_handler:
++ .quad __sdei_asm_handler
++#endif /* CONFIG_ARM_SDE_INTERFACE */
++__entry_tramp_data_this_cpu_vector:
++ .quad this_cpu_vector
+ SYM_DATA_END(__entry_tramp_data_start)
+ .popsection // .rodata
+ #endif /* CONFIG_RANDOMIZE_BASE */
+ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
++/*
++ * Exception vectors for spectre mitigations on entry from EL1 when
++ * kpti is not in use.
++ */
++ .macro generate_el1_vector, bhb
++.Lvector_start\@:
++ kernel_ventry 1, t, 64, sync // Synchronous EL1t
++ kernel_ventry 1, t, 64, irq // IRQ EL1t
++ kernel_ventry 1, t, 64, fiq // FIQ EL1h
++ kernel_ventry 1, t, 64, error // Error EL1t
++
++ kernel_ventry 1, h, 64, sync // Synchronous EL1h
++ kernel_ventry 1, h, 64, irq // IRQ EL1h
++ kernel_ventry 1, h, 64, fiq // FIQ EL1h
++ kernel_ventry 1, h, 64, error // Error EL1h
++
++ .rept 4
++ tramp_ventry .Lvector_start\@, 64, 0, \bhb
++ .endr
++ .rept 4
++ tramp_ventry .Lvector_start\@, 32, 0, \bhb
++ .endr
++ .endm
++
++/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
++ .pushsection ".entry.text", "ax"
++ .align 11
++SYM_CODE_START(__bp_harden_el1_vectors)
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++ generate_el1_vector bhb=BHB_MITIGATION_LOOP
++ generate_el1_vector bhb=BHB_MITIGATION_FW
++ generate_el1_vector bhb=BHB_MITIGATION_INSN
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++SYM_CODE_END(__bp_harden_el1_vectors)
++ .popsection
++
++
+ /*
+ * Register switch for AArch64. The callee-saved registers need to be saved
+ * and restored. On entry:
+@@ -835,14 +949,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline)
+ * Remember whether to unmap the kernel on exit.
+ */
+ 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]
+-
+-#ifdef CONFIG_RANDOMIZE_BASE
+- adr x4, tramp_vectors + PAGE_SIZE
+- add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
+- ldr x4, [x4]
+-#else
+- ldr x4, =__sdei_asm_handler
+-#endif
++ tramp_data_read_var x4, __sdei_asm_handler
+ br x4
+ SYM_CODE_END(__sdei_asm_entry_trampoline)
+ NOKPROBE(__sdei_asm_entry_trampoline)
+@@ -865,13 +972,6 @@ SYM_CODE_END(__sdei_asm_exit_trampoline)
+ NOKPROBE(__sdei_asm_exit_trampoline)
+ .ltorg
+ .popsection // .entry.tramp.text
+-#ifdef CONFIG_RANDOMIZE_BASE
+-.pushsection ".rodata", "a"
+-SYM_DATA_START(__sdei_asm_trampoline_next_handler)
+- .quad __sdei_asm_handler
+-SYM_DATA_END(__sdei_asm_trampoline_next_handler)
+-.popsection // .rodata
+-#endif /* CONFIG_RANDOMIZE_BASE */
+ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
+ /*
+@@ -979,7 +1079,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
+ alternative_else_nop_endif
+
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+- tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline
++ tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
+ br x5
+ #endif
+ SYM_CODE_END(__sdei_asm_handler)
+diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
+index c96a9a0043bf4..e03e60f9482b4 100644
+--- a/arch/arm64/kernel/image-vars.h
++++ b/arch/arm64/kernel/image-vars.h
+@@ -66,6 +66,10 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch);
+ KVM_NVHE_ALIAS(kvm_update_va_mask);
+ KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
+ KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
++KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
++KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
++KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
++KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
+
+ /* Global kernel state accessed by nVHE hyp code. */
+ KVM_NVHE_ALIAS(kvm_vgic_global_state);
+diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c
+index 63634b4d72c15..59c648d518488 100644
+--- a/arch/arm64/kernel/machine_kexec_file.c
++++ b/arch/arm64/kernel/machine_kexec_file.c
+@@ -149,6 +149,7 @@ int load_other_segments(struct kimage *image,
+ initrd_len, cmdline, 0);
+ if (!dtb) {
+ pr_err("Preparing for new dtb failed\n");
++ ret = -EINVAL;
+ goto out_err;
+ }
+
+diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
+index b5ec010c481f3..309a27553c875 100644
+--- a/arch/arm64/kernel/module.c
++++ b/arch/arm64/kernel/module.c
+@@ -36,7 +36,7 @@ void *module_alloc(unsigned long size)
+ module_alloc_end = MODULES_END;
+
+ p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base,
+- module_alloc_end, gfp_mask, PAGE_KERNEL, 0,
++ module_alloc_end, gfp_mask, PAGE_KERNEL, VM_DEFER_KMEMLEAK,
+ NUMA_NO_NODE, __builtin_return_address(0));
+
+ if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
+@@ -58,7 +58,7 @@ void *module_alloc(unsigned long size)
+ PAGE_KERNEL, 0, NUMA_NO_NODE,
+ __builtin_return_address(0));
+
+- if (p && (kasan_module_alloc(p, size) < 0)) {
++ if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) {
+ vfree(p);
+ return NULL;
+ }
+diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
+index e5e801bc53122..7c1c82c8115cc 100644
+--- a/arch/arm64/kernel/mte.c
++++ b/arch/arm64/kernel/mte.c
+@@ -73,6 +73,9 @@ void mte_sync_tags(pte_t old_pte, pte_t pte)
+ mte_sync_page_tags(page, old_pte, check_swap,
+ pte_is_tagged);
+ }
++
++ /* ensure the tags are visible before the PTE is set */
++ smp_wmb();
+ }
+
+ int memcmp_pages(struct page *page1, struct page *page2)
+diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c
+index 75fed4460407d..57c7c211f8c71 100644
+--- a/arch/arm64/kernel/paravirt.c
++++ b/arch/arm64/kernel/paravirt.c
+@@ -35,7 +35,7 @@ static u64 native_steal_clock(int cpu)
+ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
+
+ struct pv_time_stolen_time_region {
+- struct pvclock_vcpu_stolen_time *kaddr;
++ struct pvclock_vcpu_stolen_time __rcu *kaddr;
+ };
+
+ static DEFINE_PER_CPU(struct pv_time_stolen_time_region, stolen_time_region);
+@@ -52,7 +52,9 @@ early_param("no-steal-acc", parse_no_stealacc);
+ /* return stolen time in ns by asking the hypervisor */
+ static u64 para_steal_clock(int cpu)
+ {
++ struct pvclock_vcpu_stolen_time *kaddr = NULL;
+ struct pv_time_stolen_time_region *reg;
++ u64 ret = 0;
+
+ reg = per_cpu_ptr(&stolen_time_region, cpu);
+
+@@ -61,28 +63,37 @@ static u64 para_steal_clock(int cpu)
+ * online notification callback runs. Until the callback
+ * has run we just return zero.
+ */
+- if (!reg->kaddr)
++ rcu_read_lock();
++ kaddr = rcu_dereference(reg->kaddr);
++ if (!kaddr) {
++ rcu_read_unlock();
+ return 0;
++ }
+
+- return le64_to_cpu(READ_ONCE(reg->kaddr->stolen_time));
++ ret = le64_to_cpu(READ_ONCE(kaddr->stolen_time));
++ rcu_read_unlock();
++ return ret;
+ }
+
+ static int stolen_time_cpu_down_prepare(unsigned int cpu)
+ {
++ struct pvclock_vcpu_stolen_time *kaddr = NULL;
+ struct pv_time_stolen_time_region *reg;
+
+ reg = this_cpu_ptr(&stolen_time_region);
+ if (!reg->kaddr)
+ return 0;
+
+- memunmap(reg->kaddr);
+- memset(reg, 0, sizeof(*reg));
++ kaddr = rcu_replace_pointer(reg->kaddr, NULL, true);
++ synchronize_rcu();
++ memunmap(kaddr);
+
+ return 0;
+ }
+
+ static int stolen_time_cpu_online(unsigned int cpu)
+ {
++ struct pvclock_vcpu_stolen_time *kaddr = NULL;
+ struct pv_time_stolen_time_region *reg;
+ struct arm_smccc_res res;
+
+@@ -93,17 +104,19 @@ static int stolen_time_cpu_online(unsigned int cpu)
+ if (res.a0 == SMCCC_RET_NOT_SUPPORTED)
+ return -EINVAL;
+
+- reg->kaddr = memremap(res.a0,
++ kaddr = memremap(res.a0,
+ sizeof(struct pvclock_vcpu_stolen_time),
+ MEMREMAP_WB);
+
++ rcu_assign_pointer(reg->kaddr, kaddr);
++
+ if (!reg->kaddr) {
+ pr_warn("Failed to map stolen time data structure\n");
+ return -ENOMEM;
+ }
+
+- if (le32_to_cpu(reg->kaddr->revision) != 0 ||
+- le32_to_cpu(reg->kaddr->attributes) != 0) {
++ if (le32_to_cpu(kaddr->revision) != 0 ||
++ le32_to_cpu(kaddr->attributes) != 0) {
+ pr_warn_once("Unexpected revision or attributes in stolen time data\n");
+ return -ENXIO;
+ }
+diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c
+index 771f543464e06..33e0fabc0b79b 100644
+--- a/arch/arm64/kernel/patching.c
++++ b/arch/arm64/kernel/patching.c
+@@ -117,8 +117,8 @@ static int __kprobes aarch64_insn_patch_text_cb(void *arg)
+ int i, ret = 0;
+ struct aarch64_insn_patch *pp = arg;
+
+- /* The first CPU becomes master */
+- if (atomic_inc_return(&pp->cpu_count) == 1) {
++ /* The last CPU becomes master */
++ if (atomic_inc_return(&pp->cpu_count) == num_online_cpus()) {
+ for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
+ ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
+ pp->new_insns[i]);
+diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c
+index 4a72c27273097..86d9f20131723 100644
+--- a/arch/arm64/kernel/perf_callchain.c
++++ b/arch/arm64/kernel/perf_callchain.c
+@@ -102,7 +102,9 @@ compat_user_backtrace(struct compat_frame_tail __user *tail,
+ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -147,9 +149,10 @@ static bool callchain_trace(void *data, unsigned long pc)
+ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct stackframe frame;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -160,18 +163,21 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+
+ unsigned long perf_instruction_pointer(struct pt_regs *regs)
+ {
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+- return perf_guest_cbs->get_guest_ip();
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
++ if (guest_cbs && guest_cbs->is_in_guest())
++ return guest_cbs->get_guest_ip();
+
+ return instruction_pointer(regs);
+ }
+
+ unsigned long perf_misc_flags(struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ int misc = 0;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+- if (perf_guest_cbs->is_user_mode())
++ if (guest_cbs && guest_cbs->is_in_guest()) {
++ if (guest_cbs->is_user_mode())
+ misc |= PERF_RECORD_MISC_GUEST_USER;
+ else
+ misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
+index 40adb8cdbf5af..23efabcb00b85 100644
+--- a/arch/arm64/kernel/process.c
++++ b/arch/arm64/kernel/process.c
+@@ -439,34 +439,26 @@ static void entry_task_switch(struct task_struct *next)
+
+ /*
+ * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
+- * Assuming the virtual counter is enabled at the beginning of times:
+- *
+- * - disable access when switching from a 64bit task to a 32bit task
+- * - enable access when switching from a 32bit task to a 64bit task
++ * Ensure access is disabled when switching to a 32bit task, ensure
++ * access is enabled when switching to a 64bit task.
+ */
+-static void erratum_1418040_thread_switch(struct task_struct *prev,
+- struct task_struct *next)
++static void erratum_1418040_thread_switch(struct task_struct *next)
+ {
+- bool prev32, next32;
+- u64 val;
+-
+- if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040))
+- return;
+-
+- prev32 = is_compat_thread(task_thread_info(prev));
+- next32 = is_compat_thread(task_thread_info(next));
+-
+- if (prev32 == next32 || !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
++ if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) ||
++ !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
+ return;
+
+- val = read_sysreg(cntkctl_el1);
+-
+- if (!next32)
+- val |= ARCH_TIMER_USR_VCT_ACCESS_EN;
++ if (is_compat_thread(task_thread_info(next)))
++ sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0);
+ else
+- val &= ~ARCH_TIMER_USR_VCT_ACCESS_EN;
++ sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);
++}
+
+- write_sysreg(val, cntkctl_el1);
++static void erratum_1418040_new_exec(void)
++{
++ preempt_disable();
++ erratum_1418040_thread_switch(current);
++ preempt_enable();
+ }
+
+ /*
+@@ -501,7 +493,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
+ contextidr_thread_switch(next);
+ entry_task_switch(next);
+ ssbs_thread_switch(next);
+- erratum_1418040_thread_switch(prev, next);
++ erratum_1418040_thread_switch(next);
+ ptrauth_thread_switch_user(next);
+
+ /*
+@@ -613,6 +605,7 @@ void arch_setup_new_exec(void)
+ current->mm->context.flags = mmflags;
+ ptrauth_thread_init_user();
+ mte_thread_init_user();
++ erratum_1418040_new_exec();
+
+ if (task_spec_ssb_noexec(current)) {
+ arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
+diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
+index 902e4084c4775..40be3a7c2c531 100644
+--- a/arch/arm64/kernel/proton-pack.c
++++ b/arch/arm64/kernel/proton-pack.c
+@@ -18,15 +18,18 @@
+ */
+
+ #include <linux/arm-smccc.h>
++#include <linux/bpf.h>
+ #include <linux/cpu.h>
+ #include <linux/device.h>
+ #include <linux/nospec.h>
+ #include <linux/prctl.h>
+ #include <linux/sched/task_stack.h>
+
++#include <asm/debug-monitors.h>
+ #include <asm/insn.h>
+ #include <asm/spectre.h>
+ #include <asm/traps.h>
++#include <asm/vectors.h>
+ #include <asm/virt.h>
+
+ /*
+@@ -96,14 +99,51 @@ static bool spectre_v2_mitigations_off(void)
+ return ret;
+ }
+
++static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
++{
++ switch (bhb_state) {
++ case SPECTRE_UNAFFECTED:
++ return "";
++ default:
++ case SPECTRE_VULNERABLE:
++ return ", but not BHB";
++ case SPECTRE_MITIGATED:
++ return ", BHB";
++ }
++}
++
++static bool _unprivileged_ebpf_enabled(void)
++{
++#ifdef CONFIG_BPF_SYSCALL
++ return !sysctl_unprivileged_bpf_disabled;
++#else
++ return false;
++#endif
++}
++
+ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
+ char *buf)
+ {
++ enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
++ const char *bhb_str = get_bhb_affected_string(bhb_state);
++ const char *v2_str = "Branch predictor hardening";
++
+ switch (spectre_v2_state) {
+ case SPECTRE_UNAFFECTED:
+- return sprintf(buf, "Not affected\n");
++ if (bhb_state == SPECTRE_UNAFFECTED)
++ return sprintf(buf, "Not affected\n");
++
++ /*
++ * Platforms affected by Spectre-BHB can't report
++ * "Not affected" for Spectre-v2.
++ */
++ v2_str = "CSV2";
++ fallthrough;
+ case SPECTRE_MITIGATED:
+- return sprintf(buf, "Mitigation: Branch predictor hardening\n");
++ if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
++ return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
++
++ return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
+ case SPECTRE_VULNERABLE:
+ fallthrough;
+ default:
+@@ -193,17 +233,20 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn)
+ __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT);
+ }
+
+-static void call_smc_arch_workaround_1(void)
++/* Called during entry so must be noinstr */
++static noinstr void call_smc_arch_workaround_1(void)
+ {
+ arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+ }
+
+-static void call_hvc_arch_workaround_1(void)
++/* Called during entry so must be noinstr */
++static noinstr void call_hvc_arch_workaround_1(void)
+ {
+ arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+ }
+
+-static void qcom_link_stack_sanitisation(void)
++/* Called during entry so must be noinstr */
++static noinstr void qcom_link_stack_sanitisation(void)
+ {
+ u64 tmp;
+
+@@ -554,9 +597,9 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
+ * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
+ * to call into firmware to adjust the mitigation state.
+ */
+-void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt,
+- __le32 *origptr,
+- __le32 *updptr, int nr_inst)
++void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
++ __le32 *origptr,
++ __le32 *updptr, int nr_inst)
+ {
+ u32 insn;
+
+@@ -770,3 +813,345 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
+ return -ENODEV;
+ }
+ }
++
++/*
++ * Spectre BHB.
++ *
++ * A CPU is either:
++ * - Mitigated by a branchy loop a CPU specific number of times, and listed
++ * in our "loop mitigated list".
++ * - Mitigated in software by the firmware Spectre v2 call.
++ * - Has the ClearBHB instruction to perform the mitigation.
++ * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
++ * software mitigation in the vectors is needed.
++ * - Has CSV2.3, so is unaffected.
++ */
++static enum mitigation_state spectre_bhb_state;
++
++enum mitigation_state arm64_get_spectre_bhb_state(void)
++{
++ return spectre_bhb_state;
++}
++
++enum bhb_mitigation_bits {
++ BHB_LOOP,
++ BHB_FW,
++ BHB_HW,
++ BHB_INSN,
++};
++static unsigned long system_bhb_mitigations;
++
++/*
++ * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
++ * SCOPE_SYSTEM call will give the right answer.
++ */
++u8 spectre_bhb_loop_affected(int scope)
++{
++ u8 k = 0;
++ static u8 max_bhb_k;
++
++ if (scope == SCOPE_LOCAL_CPU) {
++ static const struct midr_range spectre_bhb_k32_list[] = {
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
++ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
++ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
++ {},
++ };
++ static const struct midr_range spectre_bhb_k24_list[] = {
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
++ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
++ {},
++ };
++ static const struct midr_range spectre_bhb_k8_list[] = {
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
++ {},
++ };
++
++ if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list))
++ k = 32;
++ else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
++ k = 24;
++ else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
++ k = 8;
++
++ max_bhb_k = max(max_bhb_k, k);
++ } else {
++ k = max_bhb_k;
++ }
++
++ return k;
++}
++
++static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
++{
++ int ret;
++ struct arm_smccc_res res;
++
++ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
++ ARM_SMCCC_ARCH_WORKAROUND_3, &res);
++
++ ret = res.a0;
++ switch (ret) {
++ case SMCCC_RET_SUCCESS:
++ return SPECTRE_MITIGATED;
++ case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
++ return SPECTRE_UNAFFECTED;
++ default:
++ fallthrough;
++ case SMCCC_RET_NOT_SUPPORTED:
++ return SPECTRE_VULNERABLE;
++ }
++}
++
++static bool is_spectre_bhb_fw_affected(int scope)
++{
++ static bool system_affected;
++ enum mitigation_state fw_state;
++ bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
++ static const struct midr_range spectre_bhb_firmware_mitigated_list[] = {
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
++ MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
++ {},
++ };
++ bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(),
++ spectre_bhb_firmware_mitigated_list);
++
++ if (scope != SCOPE_LOCAL_CPU)
++ return system_affected;
++
++ fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
++ if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) {
++ system_affected = true;
++ return true;
++ }
++
++ return false;
++}
++
++static bool supports_ecbhb(int scope)
++{
++ u64 mmfr1;
++
++ if (scope == SCOPE_LOCAL_CPU)
++ mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
++ else
++ mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
++
++ return cpuid_feature_extract_unsigned_field(mmfr1,
++ ID_AA64MMFR1_ECBHB_SHIFT);
++}
++
++bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
++ int scope)
++{
++ WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
++
++ if (supports_csv2p3(scope))
++ return false;
++
++ if (supports_clearbhb(scope))
++ return true;
++
++ if (spectre_bhb_loop_affected(scope))
++ return true;
++
++ if (is_spectre_bhb_fw_affected(scope))
++ return true;
++
++ return false;
++}
++
++static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
++{
++ const char *v = arm64_get_bp_hardening_vector(slot);
++
++ if (slot < 0)
++ return;
++
++ __this_cpu_write(this_cpu_vector, v);
++
++ /*
++ * When KPTI is in use, the vectors are switched when exiting to
++ * user-space.
++ */
++ if (arm64_kernel_unmapped_at_el0())
++ return;
++
++ write_sysreg(v, vbar_el1);
++ isb();
++}
++
++void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
++{
++ bp_hardening_cb_t cpu_cb;
++ enum mitigation_state fw_state, state = SPECTRE_VULNERABLE;
++ struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
++
++ if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
++ return;
++
++ if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
++ /* No point mitigating Spectre-BHB alone. */
++ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
++ pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
++ } else if (cpu_mitigations_off()) {
++ pr_info_once("spectre-bhb mitigation disabled by command line option\n");
++ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
++ state = SPECTRE_MITIGATED;
++ set_bit(BHB_HW, &system_bhb_mitigations);
++ } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
++ /*
++ * Ensure KVM uses the indirect vector which will have ClearBHB
++ * added.
++ */
++ if (!data->slot)
++ data->slot = HYP_VECTOR_INDIRECT;
++
++ this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
++ state = SPECTRE_MITIGATED;
++ set_bit(BHB_INSN, &system_bhb_mitigations);
++ } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
++ /*
++ * Ensure KVM uses the indirect vector which will have the
++ * branchy-loop added. A57/A72-r0 will already have selected
++ * the spectre-indirect vector, which is sufficient for BHB
++ * too.
++ */
++ if (!data->slot)
++ data->slot = HYP_VECTOR_INDIRECT;
++
++ this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
++ state = SPECTRE_MITIGATED;
++ set_bit(BHB_LOOP, &system_bhb_mitigations);
++ } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) {
++ fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
++ if (fw_state == SPECTRE_MITIGATED) {
++ /*
++ * Ensure KVM uses one of the spectre bp_hardening
++ * vectors. The indirect vector doesn't include the EL3
++ * call, so needs upgrading to
++ * HYP_VECTOR_SPECTRE_INDIRECT.
++ */
++ if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
++ data->slot += 1;
++
++ this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
++
++ /*
++ * The WA3 call in the vectors supersedes the WA1 call
++ * made during context-switch. Uninstall any firmware
++ * bp_hardening callback.
++ */
++ cpu_cb = spectre_v2_get_sw_mitigation_cb();
++ if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
++ __this_cpu_write(bp_hardening_data.fn, NULL);
++
++ state = SPECTRE_MITIGATED;
++ set_bit(BHB_FW, &system_bhb_mitigations);
++ }
++ }
++
++ update_mitigation_state(&spectre_bhb_state, state);
++}
++
++/* Patched to NOP when enabled */
++void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
++ __le32 *origptr,
++ __le32 *updptr, int nr_inst)
++{
++ BUG_ON(nr_inst != 1);
++
++ if (test_bit(BHB_LOOP, &system_bhb_mitigations))
++ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
++}
++
++/* Patched to NOP when enabled */
++void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
++ __le32 *origptr,
++ __le32 *updptr, int nr_inst)
++{
++ BUG_ON(nr_inst != 1);
++
++ if (test_bit(BHB_FW, &system_bhb_mitigations))
++ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
++}
++
++/* Patched to correct the immediate */
++void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
++ __le32 *origptr, __le32 *updptr, int nr_inst)
++{
++ u8 rd;
++ u32 insn;
++ u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM);
++
++ BUG_ON(nr_inst != 1); /* MOV -> MOV */
++
++ if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
++ return;
++
++ insn = le32_to_cpu(*origptr);
++ rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
++ insn = aarch64_insn_gen_movewide(rd, loop_count, 0,
++ AARCH64_INSN_VARIANT_64BIT,
++ AARCH64_INSN_MOVEWIDE_ZERO);
++ *updptr++ = cpu_to_le32(insn);
++}
++
++/* Patched to mov WA3 when supported */
++void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
++ __le32 *origptr, __le32 *updptr, int nr_inst)
++{
++ u8 rd;
++ u32 insn;
++
++ BUG_ON(nr_inst != 1); /* MOV -> MOV */
++
++ if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
++ !test_bit(BHB_FW, &system_bhb_mitigations))
++ return;
++
++ insn = le32_to_cpu(*origptr);
++ rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
++
++ insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
++ AARCH64_INSN_VARIANT_32BIT,
++ AARCH64_INSN_REG_ZR, rd,
++ ARM_SMCCC_ARCH_WORKAROUND_3);
++ if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
++ return;
++
++ *updptr++ = cpu_to_le32(insn);
++}
++
++/* Patched to NOP when not supported */
++void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
++ __le32 *origptr, __le32 *updptr, int nr_inst)
++{
++ BUG_ON(nr_inst != 2);
++
++ if (test_bit(BHB_INSN, &system_bhb_mitigations))
++ return;
++
++ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
++ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
++}
++
++#ifdef CONFIG_BPF_SYSCALL
++#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
++void unpriv_ebpf_notify(int new_state)
++{
++ if (spectre_v2_state == SPECTRE_VULNERABLE ||
++ spectre_bhb_state != SPECTRE_MITIGATED)
++ return;
++
++ if (!new_state)
++ pr_err("WARNING: %s", EBPF_WARN);
++}
++#endif
+diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
+index c287b9407f287..981f0c4157c2f 100644
+--- a/arch/arm64/kernel/signal.c
++++ b/arch/arm64/kernel/signal.c
+@@ -577,10 +577,12 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user,
+ {
+ int err;
+
+- err = sigframe_alloc(user, &user->fpsimd_offset,
+- sizeof(struct fpsimd_context));
+- if (err)
+- return err;
++ if (system_supports_fpsimd()) {
++ err = sigframe_alloc(user, &user->fpsimd_offset,
++ sizeof(struct fpsimd_context));
++ if (err)
++ return err;
++ }
+
+ /* fault information, if valid */
+ if (add_all || current->thread.fault_code) {
+diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
+index 6f6ff072acbde..3beaa6640ab35 100644
+--- a/arch/arm64/kernel/smp.c
++++ b/arch/arm64/kernel/smp.c
+@@ -234,6 +234,7 @@ asmlinkage notrace void secondary_start_kernel(void)
+ * Log the CPU info before it is marked online and might get read.
+ */
+ cpuinfo_store_cpu();
++ store_cpu_topology(cpu);
+
+ /*
+ * Enable GIC and timers.
+@@ -242,7 +243,6 @@ asmlinkage notrace void secondary_start_kernel(void)
+
+ ipi_setup(cpu);
+
+- store_cpu_topology(cpu);
+ numa_add_cpu(cpu);
+
+ /*
+diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
+index 8982a2b78acfc..3b8dc538a4c42 100644
+--- a/arch/arm64/kernel/stacktrace.c
++++ b/arch/arm64/kernel/stacktrace.c
+@@ -33,7 +33,7 @@
+ */
+
+
+-void start_backtrace(struct stackframe *frame, unsigned long fp,
++notrace void start_backtrace(struct stackframe *frame, unsigned long fp,
+ unsigned long pc)
+ {
+ frame->fp = fp;
+@@ -55,6 +55,7 @@ void start_backtrace(struct stackframe *frame, unsigned long fp,
+ frame->prev_fp = 0;
+ frame->prev_type = STACK_TYPE_UNKNOWN;
+ }
++NOKPROBE_SYMBOL(start_backtrace);
+
+ /*
+ * Unwind from one frame record (A) to the next frame record (B).
+diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
+index b03e383d944ab..fe0cd0568813e 100644
+--- a/arch/arm64/kernel/traps.c
++++ b/arch/arm64/kernel/traps.c
+@@ -988,7 +988,7 @@ static struct break_hook bug_break_hook = {
+ static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr)
+ {
+ pr_err("%s generated an invalid instruction at %pS!\n",
+- in_bpf_jit(regs) ? "BPF JIT" : "Kernel text patching",
++ "Kernel text patching",
+ (void *)instruction_pointer(regs));
+
+ /* We cannot handle this */
+diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
+index 945e6bb326e3e..b5d8f72e8b32e 100644
+--- a/arch/arm64/kernel/vdso/Makefile
++++ b/arch/arm64/kernel/vdso/Makefile
+@@ -48,9 +48,6 @@ GCOV_PROFILE := n
+ targets += vdso.lds
+ CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
+
+-# Force dependency (incbin is bad)
+-$(obj)/vdso.o : $(obj)/vdso.so
+-
+ # Link rule for the .so file, .lds has to be first
+ $(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE
+ $(call if_changed,vdsold_and_vdso_check)
+diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile
+index 3dba0c4f8f42b..83e9399e38368 100644
+--- a/arch/arm64/kernel/vdso32/Makefile
++++ b/arch/arm64/kernel/vdso32/Makefile
+@@ -10,18 +10,15 @@ include $(srctree)/lib/vdso/Makefile
+
+ # Same as cc-*option, but using CC_COMPAT instead of CC
+ ifeq ($(CONFIG_CC_IS_CLANG), y)
+-CC_COMPAT_CLANG_FLAGS := --target=$(notdir $(CROSS_COMPILE_COMPAT:%-=%))
+-
+ CC_COMPAT ?= $(CC)
+-CC_COMPAT += $(CC_COMPAT_CLANG_FLAGS)
+-
+-ifneq ($(LLVM),)
+-LD_COMPAT ?= $(LD)
++CC_COMPAT += --target=arm-linux-gnueabi
+ else
+-LD_COMPAT ?= $(CROSS_COMPILE_COMPAT)ld
++CC_COMPAT ?= $(CROSS_COMPILE_COMPAT)gcc
+ endif
++
++ifeq ($(CONFIG_LD_IS_LLD), y)
++LD_COMPAT ?= $(LD)
+ else
+-CC_COMPAT ?= $(CROSS_COMPILE_COMPAT)gcc
+ LD_COMPAT ?= $(CROSS_COMPILE_COMPAT)ld
+ endif
+
+@@ -40,16 +37,13 @@ cc32-as-instr = $(call try-run,\
+ # As a result we set our own flags here.
+
+ # KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile
+-VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)
++VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc
++VDSO_CPPFLAGS += -isystem $(shell $(CC_COMPAT) -print-file-name=include 2>/dev/null)
+ VDSO_CPPFLAGS += $(LINUXINCLUDE)
+
+ # Common C and assembly flags
+ # From top-level Makefile
+ VDSO_CAFLAGS := $(VDSO_CPPFLAGS)
+-ifneq ($(shell $(CC_COMPAT) --version 2>&1 | head -n 1 | grep clang),)
+-VDSO_CAFLAGS += --target=$(notdir $(CROSS_COMPILE_COMPAT:%-=%))
+-endif
+-
+ VDSO_CAFLAGS += $(call cc32-option,-fno-PIE)
+ ifdef CONFIG_DEBUG_INFO
+ VDSO_CAFLAGS += -g
+@@ -150,9 +144,6 @@ obj-vdso := $(c-obj-vdso) $(c-obj-vdso-gettimeofday) $(asm-obj-vdso)
+ targets += vdso.lds
+ CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
+
+-# Force dependency (vdso.s includes vdso.so through incbin)
+-$(obj)/vdso.o: $(obj)/vdso.so
+-
+ include/generated/vdso32-offsets.h: $(obj)/vdso.so.dbg FORCE
+ $(call if_changed,vdsosym)
+
+diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
+index f6b1a88245db2..184abd7c4206e 100644
+--- a/arch/arm64/kernel/vmlinux.lds.S
++++ b/arch/arm64/kernel/vmlinux.lds.S
+@@ -330,7 +330,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
+ <= SZ_4K, "Hibernate exit text too big or misaligned")
+ #endif
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+-ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
++ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
+ "Entry trampoline text too big")
+ #endif
+ #ifdef CONFIG_KVM
+diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
+index fe102cd2e5183..0b2f684cd8ca5 100644
+--- a/arch/arm64/kvm/arm.c
++++ b/arch/arm64/kvm/arm.c
+@@ -755,6 +755,24 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
+ xfer_to_guest_mode_work_pending();
+ }
+
++/*
++ * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
++ * the vCPU is running.
++ *
++ * This must be noinstr as instrumentation may make use of RCU, and this is not
++ * safe during the EQS.
++ */
++static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
++{
++ int ret;
++
++ guest_state_enter_irqoff();
++ ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
++ guest_state_exit_irqoff();
++
++ return ret;
++}
++
+ /**
+ * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
+ * @vcpu: The VCPU pointer
+@@ -845,9 +863,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
+ * Enter the guest
+ */
+ trace_kvm_entry(*vcpu_pc(vcpu));
+- guest_enter_irqoff();
++ guest_timing_enter_irqoff();
+
+- ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
++ ret = kvm_arm_vcpu_enter_exit(vcpu);
+
+ vcpu->mode = OUTSIDE_GUEST_MODE;
+ vcpu->stat.exits++;
+@@ -882,26 +900,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
+ kvm_arch_vcpu_ctxsync_fp(vcpu);
+
+ /*
+- * We may have taken a host interrupt in HYP mode (ie
+- * while executing the guest). This interrupt is still
+- * pending, as we haven't serviced it yet!
++ * We must ensure that any pending interrupts are taken before
++ * we exit guest timing so that timer ticks are accounted as
++ * guest time. Transiently unmask interrupts so that any
++ * pending interrupts are taken.
+ *
+- * We're now back in SVC mode, with interrupts
+- * disabled. Enabling the interrupts now will have
+- * the effect of taking the interrupt again, in SVC
+- * mode this time.
++ * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
++ * context synchronization event) is necessary to ensure that
++ * pending interrupts are taken.
+ */
+ local_irq_enable();
++ isb();
++ local_irq_disable();
++
++ guest_timing_exit_irqoff();
++
++ local_irq_enable();
+
+- /*
+- * We do local_irq_enable() before calling guest_exit() so
+- * that if a timer interrupt hits while running the guest we
+- * account that tick as being spent in the guest. We enable
+- * preemption after calling guest_exit() so that if we get
+- * preempted we make sure ticks after that is not counted as
+- * guest time.
+- */
+- guest_exit();
+ trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
+
+ /* Exit types that need handling before we can be preempted */
+@@ -1443,10 +1458,7 @@ static int kvm_init_vector_slots(void)
+ base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
+ kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
+
+- if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
+- return 0;
+-
+- if (!has_vhe()) {
++ if (kvm_system_needs_idmapped_vectors() && !has_vhe()) {
+ err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
+ __BP_HARDEN_HYP_VECS_SZ, &base);
+ if (err)
+@@ -1971,9 +1983,25 @@ out_err:
+ return err;
+ }
+
+-static void _kvm_host_prot_finalize(void *discard)
++static void _kvm_host_prot_finalize(void *arg)
++{
++ int *err = arg;
++
++ if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)))
++ WRITE_ONCE(*err, -EINVAL);
++}
++
++static int pkvm_drop_host_privileges(void)
+ {
+- WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize));
++ int ret = 0;
++
++ /*
++ * Flip the static key upfront as that may no longer be possible
++ * once the host stage 2 is installed.
++ */
++ static_branch_enable(&kvm_protected_mode_initialized);
++ on_each_cpu(_kvm_host_prot_finalize, &ret, 1);
++ return ret;
+ }
+
+ static int finalize_hyp_mode(void)
+@@ -1987,15 +2015,7 @@ static int finalize_hyp_mode(void)
+ * None of other sections should ever be introspected.
+ */
+ kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
+-
+- /*
+- * Flip the static key upfront as that may no longer be possible
+- * once the host stage 2 is installed.
+- */
+- static_branch_enable(&kvm_protected_mode_initialized);
+- on_each_cpu(_kvm_host_prot_finalize, NULL, 1);
+-
+- return 0;
++ return pkvm_drop_host_privileges();
+ }
+
+ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
+diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
+index 275a27368a04c..a5ab5215094ee 100644
+--- a/arch/arm64/kvm/handle_exit.c
++++ b/arch/arm64/kvm/handle_exit.c
+@@ -226,6 +226,14 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index)
+ {
+ struct kvm_run *run = vcpu->run;
+
++ if (ARM_SERROR_PENDING(exception_index)) {
++ /*
++ * The SError is handled by handle_exit_early(). If the guest
++ * survives it will re-execute the original instruction.
++ */
++ return 1;
++ }
++
+ exception_index = ARM_EXCEPTION_CODE(exception_index);
+
+ switch (exception_index) {
+diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c
+index 0418399e0a201..c5d0097154020 100644
+--- a/arch/arm64/kvm/hyp/exception.c
++++ b/arch/arm64/kvm/hyp/exception.c
+@@ -38,7 +38,10 @@ static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
+
+ static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val)
+ {
+- write_sysreg_el1(val, SYS_SPSR);
++ if (has_vhe())
++ write_sysreg_el1(val, SYS_SPSR);
++ else
++ __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
+ }
+
+ static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
+diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
+index 9aa9b73475c95..7839d075729b1 100644
+--- a/arch/arm64/kvm/hyp/hyp-entry.S
++++ b/arch/arm64/kvm/hyp/hyp-entry.S
+@@ -44,7 +44,7 @@
+ el1_sync: // Guest trapped into EL2
+
+ mrs x0, esr_el2
+- lsr x0, x0, #ESR_ELx_EC_SHIFT
++ ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH
+ cmp x0, #ESR_ELx_EC_HVC64
+ ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
+ b.ne el1_trap
+@@ -62,6 +62,10 @@ el1_sync: // Guest trapped into EL2
+ /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
+ eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
+ ARM_SMCCC_ARCH_WORKAROUND_2)
++ cbz w1, wa_epilogue
++
++ eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
++ ARM_SMCCC_ARCH_WORKAROUND_3)
+ cbnz w1, el1_trap
+
+ wa_epilogue:
+@@ -192,7 +196,10 @@ SYM_CODE_END(__kvm_hyp_vector)
+ sub sp, sp, #(8 * 4)
+ stp x2, x3, [sp, #(8 * 0)]
+ stp x0, x1, [sp, #(8 * 2)]
++ alternative_cb spectre_bhb_patch_wa3
++ /* Patched to mov WA3 when supported */
+ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
++ alternative_cb_end
+ smc #0
+ ldp x2, x3, [sp, #(8 * 0)]
+ add sp, sp, #(8 * 2)
+@@ -205,6 +212,8 @@ SYM_CODE_END(__kvm_hyp_vector)
+ spectrev2_smccc_wa1_smc
+ .else
+ stp x0, x1, [sp, #-16]!
++ mitigate_spectre_bhb_loop x0
++ mitigate_spectre_bhb_clear_insn
+ .endif
+ .if \indirect != 0
+ alternative_cb kvm_patch_vector_branch
+diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
+index a0e78a6027be0..ecd41844eda09 100644
+--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
++++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
+@@ -416,10 +416,17 @@ static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu)
+ */
+ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
+ {
++ /*
++ * Save PSTATE early so that we can evaluate the vcpu mode
++ * early on.
++ */
++ vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR);
++
+ if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)
+ vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);
+
+- if (ARM_SERROR_PENDING(*exit_code)) {
++ if (ARM_SERROR_PENDING(*exit_code) &&
++ ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) {
+ u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
+
+ /*
+diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+index de7e14c862e6c..7ecca8b078519 100644
+--- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
++++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+@@ -70,7 +70,12 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
+ static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
+ {
+ ctxt->regs.pc = read_sysreg_el2(SYS_ELR);
+- ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
++ /*
++ * Guest PSTATE gets saved at guest fixup time in all
++ * cases. We still need to handle the nVHE host side here.
++ */
++ if (!has_vhe() && ctxt->__hyp_running_vcpu)
++ ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
+
+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
+ ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
+diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
+index 4b652ffb591d4..d310d2b2c8b40 100644
+--- a/arch/arm64/kvm/hyp/nvhe/host.S
++++ b/arch/arm64/kvm/hyp/nvhe/host.S
+@@ -115,7 +115,7 @@ SYM_FUNC_END(__hyp_do_panic)
+ .L__vect_start\@:
+ stp x0, x1, [sp, #-16]!
+ mrs x0, esr_el2
+- lsr x0, x0, #ESR_ELx_EC_SHIFT
++ ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH
+ cmp x0, #ESR_ELx_EC_HVC64
+ b.ne __host_exit
+
+diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
+index 2fabeceb889a9..5146fb1705054 100644
+--- a/arch/arm64/kvm/hyp/nvhe/mm.c
++++ b/arch/arm64/kvm/hyp/nvhe/mm.c
+@@ -146,8 +146,10 @@ int hyp_map_vectors(void)
+ phys_addr_t phys;
+ void *bp_base;
+
+- if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
++ if (!kvm_system_needs_idmapped_vectors()) {
++ __hyp_bp_vect_base = __bp_harden_hyp_vecs;
+ return 0;
++ }
+
+ phys = __hyp_pa(__bp_harden_hyp_vecs);
+ bp_base = (void *)__pkvm_create_private_mapping(phys,
+diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
+index 57c27846320f4..58ad9c5ba3112 100644
+--- a/arch/arm64/kvm/hyp/nvhe/setup.c
++++ b/arch/arm64/kvm/hyp/nvhe/setup.c
+@@ -177,7 +177,7 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level,
+
+ phys = kvm_pte_to_phys(pte);
+ if (!addr_is_memory(phys))
+- return 0;
++ return -EINVAL;
+
+ /*
+ * Adjust the host stage-2 mappings to match the ownership attributes
+@@ -206,8 +206,18 @@ static int finalize_host_mappings(void)
+ .cb = finalize_host_mappings_walker,
+ .flags = KVM_PGTABLE_WALK_LEAF,
+ };
++ int i, ret;
++
++ for (i = 0; i < hyp_memblock_nr; i++) {
++ struct memblock_region *reg = &hyp_memory[i];
++ u64 start = (u64)hyp_phys_to_virt(reg->base);
++
++ ret = kvm_pgtable_walk(&pkvm_pgtable, start, reg->size, &walker);
++ if (ret)
++ return ret;
++ }
+
+- return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits), &walker);
++ return 0;
+ }
+
+ void __noreturn __pkvm_init_finalise(void)
+diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
+index f8ceebe4982eb..4c77ff556f0ae 100644
+--- a/arch/arm64/kvm/hyp/pgtable.c
++++ b/arch/arm64/kvm/hyp/pgtable.c
+@@ -921,13 +921,9 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
+ */
+ stage2_put_pte(ptep, mmu, addr, level, mm_ops);
+
+- if (need_flush) {
+- kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
+-
+- dcache_clean_inval_poc((unsigned long)pte_follow,
+- (unsigned long)pte_follow +
+- kvm_granule_size(level));
+- }
++ if (need_flush && mm_ops->dcache_clean_inval_poc)
++ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
++ kvm_granule_size(level));
+
+ if (childp)
+ mm_ops->put_page(childp);
+@@ -1089,15 +1085,13 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
+ struct kvm_pgtable *pgt = arg;
+ struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
+ kvm_pte_t pte = *ptep;
+- kvm_pte_t *pte_follow;
+
+ if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))
+ return 0;
+
+- pte_follow = kvm_pte_follow(pte, mm_ops);
+- dcache_clean_inval_poc((unsigned long)pte_follow,
+- (unsigned long)pte_follow +
+- kvm_granule_size(level));
++ if (mm_ops->dcache_clean_inval_poc)
++ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
++ kvm_granule_size(level));
+ return 0;
+ }
+
+diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
+index ded2c66675f06..d88d3c143a735 100644
+--- a/arch/arm64/kvm/hyp/vhe/switch.c
++++ b/arch/arm64/kvm/hyp/vhe/switch.c
+@@ -10,6 +10,7 @@
+ #include <linux/kvm_host.h>
+ #include <linux/types.h>
+ #include <linux/jump_label.h>
++#include <linux/percpu.h>
+ #include <uapi/linux/psci.h>
+
+ #include <kvm/arm_psci.h>
+@@ -25,6 +26,7 @@
+ #include <asm/debug-monitors.h>
+ #include <asm/processor.h>
+ #include <asm/thread_info.h>
++#include <asm/vectors.h>
+
+ /* VHE specific context */
+ DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
+@@ -68,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps);
+
+ static void __deactivate_traps(struct kvm_vcpu *vcpu)
+ {
+- extern char vectors[]; /* kernel exception vectors */
++ const char *host_vectors = vectors;
+
+ ___deactivate_traps(vcpu);
+
+@@ -82,7 +84,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
+ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
+
+ write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
+- write_sysreg(vectors, vbar_el1);
++
++ if (!arm64_kernel_unmapped_at_el0())
++ host_vectors = __this_cpu_read(this_cpu_vector);
++ write_sysreg(host_vectors, vbar_el1);
+ }
+ NOKPROBE_SYMBOL(__deactivate_traps);
+
+diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c
+index 30da78f72b3b3..202b8c455724b 100644
+--- a/arch/arm64/kvm/hypercalls.c
++++ b/arch/arm64/kvm/hypercalls.c
+@@ -107,6 +107,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
+ break;
+ }
+ break;
++ case ARM_SMCCC_ARCH_WORKAROUND_3:
++ switch (arm64_get_spectre_bhb_state()) {
++ case SPECTRE_VULNERABLE:
++ break;
++ case SPECTRE_MITIGATED:
++ val[0] = SMCCC_RET_SUCCESS;
++ break;
++ case SPECTRE_UNAFFECTED:
++ val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
++ break;
++ }
++ break;
+ case ARM_SMCCC_HV_PV_TIME_FEATURES:
+ val[0] = SMCCC_RET_SUCCESS;
+ break;
+diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c
+index 74c47d4202534..44efe12dfc066 100644
+--- a/arch/arm64/kvm/psci.c
++++ b/arch/arm64/kvm/psci.c
+@@ -406,7 +406,7 @@ int kvm_psci_call(struct kvm_vcpu *vcpu)
+
+ int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu)
+ {
+- return 3; /* PSCI version and two workaround registers */
++ return 4; /* PSCI version and three workaround registers */
+ }
+
+ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
+@@ -420,6 +420,9 @@ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
+ if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++))
+ return -EFAULT;
+
++ if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++))
++ return -EFAULT;
++
+ return 0;
+ }
+
+@@ -459,6 +462,17 @@ static int get_kernel_wa_level(u64 regid)
+ case SPECTRE_VULNERABLE:
+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
+ }
++ break;
++ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
++ switch (arm64_get_spectre_bhb_state()) {
++ case SPECTRE_VULNERABLE:
++ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
++ case SPECTRE_MITIGATED:
++ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL;
++ case SPECTRE_UNAFFECTED:
++ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED;
++ }
++ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
+ }
+
+ return -EINVAL;
+@@ -475,6 +489,7 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+ break;
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
++ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
+ val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
+ break;
+ default:
+@@ -520,6 +535,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+ }
+
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
++ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
+ if (val & ~KVM_REG_FEATURE_LEVEL_MASK)
+ return -EINVAL;
+
+diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
+index 1d46e185f31e1..7c18e429b4493 100644
+--- a/arch/arm64/kvm/sys_regs.c
++++ b/arch/arm64/kvm/sys_regs.c
+@@ -1518,7 +1518,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
+ /* CRm=6 */
+ ID_SANITISED(ID_AA64ISAR0_EL1),
+ ID_SANITISED(ID_AA64ISAR1_EL1),
+- ID_UNALLOCATED(6,2),
++ ID_SANITISED(ID_AA64ISAR2_EL1),
+ ID_UNALLOCATED(6,3),
+ ID_UNALLOCATED(6,4),
+ ID_UNALLOCATED(6,5),
+diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c
+index 48c6067fc5ecb..f972992682746 100644
+--- a/arch/arm64/kvm/vgic/vgic-mmio.c
++++ b/arch/arm64/kvm/vgic/vgic-mmio.c
+@@ -248,6 +248,8 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
+ IRQCHIP_STATE_PENDING,
+ &val);
+ WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
++ } else if (vgic_irq_is_mapped_level(irq)) {
++ val = vgic_get_phys_line_level(irq);
+ } else {
+ val = irq_is_pending(irq);
+ }
+diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S
+index b84b179edba3a..1fd5d790ab800 100644
+--- a/arch/arm64/lib/clear_page.S
++++ b/arch/arm64/lib/clear_page.S
+@@ -16,6 +16,7 @@
+ */
+ SYM_FUNC_START_PI(clear_page)
+ mrs x1, dczid_el0
++ tbnz x1, #4, 2f /* Branch if DC ZVA is prohibited */
+ and w1, w1, #0xf
+ mov x2, #4
+ lsl x1, x2, x1
+@@ -25,5 +26,14 @@ SYM_FUNC_START_PI(clear_page)
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+ ret
++
++2: stnp xzr, xzr, [x0]
++ stnp xzr, xzr, [x0, #16]
++ stnp xzr, xzr, [x0, #32]
++ stnp xzr, xzr, [x0, #48]
++ add x0, x0, #64
++ tst x0, #(PAGE_SIZE - 1)
++ b.ne 2b
++ ret
+ SYM_FUNC_END_PI(clear_page)
+ EXPORT_SYMBOL(clear_page)
+diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
+index e83643b3995f4..f531dcb95174a 100644
+--- a/arch/arm64/lib/mte.S
++++ b/arch/arm64/lib/mte.S
+@@ -43,17 +43,23 @@ SYM_FUNC_END(mte_clear_page_tags)
+ * x0 - address to the beginning of the page
+ */
+ SYM_FUNC_START(mte_zero_clear_page_tags)
++ and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
+ mrs x1, dczid_el0
++ tbnz x1, #4, 2f // Branch if DC GZVA is prohibited
+ and w1, w1, #0xf
+ mov x2, #4
+ lsl x1, x2, x1
+- and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
+
+ 1: dc gzva, x0
+ add x0, x0, x1
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+ ret
++
++2: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2)
++ tst x0, #(PAGE_SIZE - 1)
++ b.ne 2b
++ ret
+ SYM_FUNC_END(mte_zero_clear_page_tags)
+
+ /*
+diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
+index aa0060178343a..60a8b6a8a42b5 100644
+--- a/arch/arm64/mm/extable.c
++++ b/arch/arm64/mm/extable.c
+@@ -9,14 +9,19 @@
+ int fixup_exception(struct pt_regs *regs)
+ {
+ const struct exception_table_entry *fixup;
++ unsigned long addr;
+
+- fixup = search_exception_tables(instruction_pointer(regs));
+- if (!fixup)
+- return 0;
++ addr = instruction_pointer(regs);
+
+- if (in_bpf_jit(regs))
++ /* Search the BPF tables first, these are formatted differently */
++ fixup = search_bpf_extables(addr);
++ if (fixup)
+ return arm64_bpf_fixup_exception(fixup, regs);
+
++ fixup = search_exception_tables(addr);
++ if (!fixup)
++ return 0;
++
+ regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
+ return 1;
+ }
+diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
+index 37a81754d9b61..3b269c7567984 100644
+--- a/arch/arm64/mm/init.c
++++ b/arch/arm64/mm/init.c
+@@ -61,8 +61,34 @@ EXPORT_SYMBOL(memstart_addr);
+ * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4).
+ * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory,
+ * otherwise it is empty.
++ *
++ * Memory reservation for crash kernel either done early or deferred
++ * depending on DMA memory zones configs (ZONE_DMA) --
++ *
++ * In absence of ZONE_DMA configs arm64_dma_phys_limit initialized
++ * here instead of max_zone_phys(). This lets early reservation of
++ * crash kernel memory which has a dependency on arm64_dma_phys_limit.
++ * Reserving memory early for crash kernel allows linear creation of block
++ * mappings (greater than page-granularity) for all the memory bank rangs.
++ * In this scheme a comparatively quicker boot is observed.
++ *
++ * If ZONE_DMA configs are defined, crash kernel memory reservation
++ * is delayed until DMA zone memory range size initilazation performed in
++ * zone_sizes_init(). The defer is necessary to steer clear of DMA zone
++ * memory range to avoid overlap allocation. So crash kernel memory boundaries
++ * are not known when mapping all bank memory ranges, which otherwise means
++ * not possible to exclude crash kernel range from creating block mappings
++ * so page-granularity mappings are created for the entire memory range.
++ * Hence a slightly slower boot is observed.
++ *
++ * Note: Page-granularity mapppings are necessary for crash kernel memory
++ * range for shrinking its size via /sys/kernel/kexec_crash_size interface.
+ */
+-phys_addr_t arm64_dma_phys_limit __ro_after_init;
++#if IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32)
++phys_addr_t __ro_after_init arm64_dma_phys_limit;
++#else
++phys_addr_t __ro_after_init arm64_dma_phys_limit = PHYS_MASK + 1;
++#endif
+
+ #ifdef CONFIG_KEXEC_CORE
+ /*
+@@ -153,50 +179,11 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
+ if (!arm64_dma_phys_limit)
+ arm64_dma_phys_limit = dma32_phys_limit;
+ #endif
+- if (!arm64_dma_phys_limit)
+- arm64_dma_phys_limit = PHYS_MASK + 1;
+ max_zone_pfns[ZONE_NORMAL] = max;
+
+ free_area_init(max_zone_pfns);
+ }
+
+-int pfn_valid(unsigned long pfn)
+-{
+- phys_addr_t addr = PFN_PHYS(pfn);
+- struct mem_section *ms;
+-
+- /*
+- * Ensure the upper PAGE_SHIFT bits are clear in the
+- * pfn. Else it might lead to false positives when
+- * some of the upper bits are set, but the lower bits
+- * match a valid pfn.
+- */
+- if (PHYS_PFN(addr) != pfn)
+- return 0;
+-
+- if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
+- return 0;
+-
+- ms = __pfn_to_section(pfn);
+- if (!valid_section(ms))
+- return 0;
+-
+- /*
+- * ZONE_DEVICE memory does not have the memblock entries.
+- * memblock_is_map_memory() check for ZONE_DEVICE based
+- * addresses will always fail. Even the normal hotplugged
+- * memory will never have MEMBLOCK_NOMAP flag set in their
+- * memblock entries. Skip memblock search for all non early
+- * memory sections covering all of hotplug memory including
+- * both normal and ZONE_DEVICE based.
+- */
+- if (!early_section(ms))
+- return pfn_section_valid(ms, pfn);
+-
+- return memblock_is_memory(addr);
+-}
+-EXPORT_SYMBOL(pfn_valid);
+-
+ int pfn_is_map_memory(unsigned long pfn)
+ {
+ phys_addr_t addr = PFN_PHYS(pfn);
+@@ -352,6 +339,9 @@ void __init arm64_memblock_init(void)
+
+ early_init_fdt_scan_reserved_mem();
+
++ if (!IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32))
++ reserve_crashkernel();
++
+ high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
+ }
+
+@@ -398,7 +388,8 @@ void __init bootmem_init(void)
+ * request_standard_resources() depends on crashkernel's memory being
+ * reserved, so do it here.
+ */
+- reserve_crashkernel();
++ if (IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32))
++ reserve_crashkernel();
+
+ memblock_dump_all();
+ }
+diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
+index b7c81dacabf07..b21f91cd830db 100644
+--- a/arch/arm64/mm/ioremap.c
++++ b/arch/arm64/mm/ioremap.c
+@@ -99,3 +99,11 @@ void __init early_ioremap_init(void)
+ {
+ early_ioremap_setup();
+ }
++
++bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size,
++ unsigned long flags)
++{
++ unsigned long pfn = PHYS_PFN(offset);
++
++ return pfn_is_map_memory(pfn);
++}
+diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c
+index a38f54cd638c2..77ada00280d93 100644
+--- a/arch/arm64/mm/mmap.c
++++ b/arch/arm64/mm/mmap.c
+@@ -7,8 +7,10 @@
+
+ #include <linux/io.h>
+ #include <linux/memblock.h>
++#include <linux/mm.h>
+ #include <linux/types.h>
+
++#include <asm/cpufeature.h>
+ #include <asm/page.h>
+
+ /*
+@@ -38,3 +40,18 @@ int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
+ {
+ return !(((pfn << PAGE_SHIFT) + size) & ~PHYS_MASK);
+ }
++
++static int __init adjust_protection_map(void)
++{
++ /*
++ * With Enhanced PAN we can honour the execute-only permissions as
++ * there is no PAN override with such mappings.
++ */
++ if (cpus_have_const_cap(ARM64_HAS_EPAN)) {
++ protection_map[VM_EXEC] = PAGE_EXECONLY;
++ protection_map[VM_EXEC | VM_SHARED] = PAGE_EXECONLY;
++ }
++
++ return 0;
++}
++arch_initcall(adjust_protection_map);
+diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
+index cfd9deb347c38..6680689242df3 100644
+--- a/arch/arm64/mm/mmu.c
++++ b/arch/arm64/mm/mmu.c
+@@ -63,6 +63,7 @@ static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
+ static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+
+ static DEFINE_SPINLOCK(swapper_pgdir_lock);
++static DEFINE_MUTEX(fixmap_lock);
+
+ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+ {
+@@ -328,6 +329,12 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ }
+ BUG_ON(p4d_bad(p4d));
+
++ /*
++ * No need for locking during early boot. And it doesn't work as
++ * expected with KASLR enabled.
++ */
++ if (system_state != SYSTEM_BOOTING)
++ mutex_lock(&fixmap_lock);
+ pudp = pud_set_fixmap_offset(p4dp, addr);
+ do {
+ pud_t old_pud = READ_ONCE(*pudp);
+@@ -358,6 +365,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ } while (pudp++, addr = next, addr != end);
+
+ pud_clear_fixmap();
++ if (system_state != SYSTEM_BOOTING)
++ mutex_unlock(&fixmap_lock);
+ }
+
+ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
+@@ -516,7 +525,7 @@ static void __init map_mem(pgd_t *pgdp)
+ */
+ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
+
+- if (can_set_direct_map() || crash_mem_map || IS_ENABLED(CONFIG_KFENCE))
++ if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE))
+ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+
+ /*
+@@ -527,6 +536,17 @@ static void __init map_mem(pgd_t *pgdp)
+ */
+ memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
+
++#ifdef CONFIG_KEXEC_CORE
++ if (crash_mem_map) {
++ if (IS_ENABLED(CONFIG_ZONE_DMA) ||
++ IS_ENABLED(CONFIG_ZONE_DMA32))
++ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
++ else if (crashk_res.end)
++ memblock_mark_nomap(crashk_res.start,
++ resource_size(&crashk_res));
++ }
++#endif
++
+ /* map all the memory banks */
+ for_each_mem_range(i, &start, &end) {
+ if (start >= end)
+@@ -553,6 +573,25 @@ static void __init map_mem(pgd_t *pgdp)
+ __map_memblock(pgdp, kernel_start, kernel_end,
+ PAGE_KERNEL, NO_CONT_MAPPINGS);
+ memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
++
++ /*
++ * Use page-level mappings here so that we can shrink the region
++ * in page granularity and put back unused memory to buddy system
++ * through /sys/kernel/kexec_crash_size interface.
++ */
++#ifdef CONFIG_KEXEC_CORE
++ if (crash_mem_map &&
++ !IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32)) {
++ if (crashk_res.end) {
++ __map_memblock(pgdp, crashk_res.start,
++ crashk_res.end + 1,
++ PAGE_KERNEL,
++ NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
++ memblock_clear_nomap(crashk_res.start,
++ resource_size(&crashk_res));
++ }
++ }
++#endif
+ }
+
+ void mark_rodata_ro(void)
+@@ -616,6 +655,8 @@ early_param("rodata", parse_rodata);
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ static int __init map_entry_trampoline(void)
+ {
++ int i;
++
+ pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+ phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+
+@@ -624,11 +665,15 @@ static int __init map_entry_trampoline(void)
+
+ /* Map only the text into the trampoline page table */
+ memset(tramp_pg_dir, 0, PGD_SIZE);
+- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
+- prot, __pgd_pgtable_alloc, 0);
++ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
++ entry_tramp_text_size(), prot,
++ __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
+
+ /* Map both the text and data into the kernel page table */
+- __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
++ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
++ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
++ pa_start + i * PAGE_SIZE, prot);
++
+ if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
+ extern char __entry_tramp_data_start[];
+
+@@ -1499,6 +1544,11 @@ int arch_add_memory(int nid, u64 start, u64 size,
+ if (ret)
+ __remove_pgd_mapping(swapper_pg_dir,
+ __phys_to_virt(start), size);
++ else {
++ max_pfn = PFN_UP(start + size);
++ max_low_pfn = max_pfn;
++ }
++
+ return ret;
+ }
+
+diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c
+index 1c403536c9bb0..9bc4066c5bf33 100644
+--- a/arch/arm64/mm/ptdump.c
++++ b/arch/arm64/mm/ptdump.c
+@@ -41,8 +41,6 @@ static struct addr_marker address_markers[] = {
+ { 0 /* KASAN_SHADOW_START */, "Kasan shadow start" },
+ { KASAN_SHADOW_END, "Kasan shadow end" },
+ #endif
+- { BPF_JIT_REGION_START, "BPF start" },
+- { BPF_JIT_REGION_END, "BPF end" },
+ { MODULES_VADDR, "Modules start" },
+ { MODULES_END, "Modules end" },
+ { VMALLOC_START, "vmalloc() area" },
+diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
+index 803e7773fa869..b56e7bd96594c 100644
+--- a/arch/arm64/net/bpf_jit_comp.c
++++ b/arch/arm64/net/bpf_jit_comp.c
+@@ -1042,15 +1042,18 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+ goto out_off;
+ }
+
+- /* 1. Initial fake pass to compute ctx->idx. */
+-
+- /* Fake pass to fill in ctx->offset. */
+- if (build_body(&ctx, extra_pass)) {
++ /*
++ * 1. Initial fake pass to compute ctx->idx and ctx->offset.
++ *
++ * BPF line info needs ctx->offset[i] to be the offset of
++ * instruction[i] in jited image, so build prologue first.
++ */
++ if (build_prologue(&ctx, was_classic)) {
+ prog = orig_prog;
+ goto out_off;
+ }
+
+- if (build_prologue(&ctx, was_classic)) {
++ if (build_body(&ctx, extra_pass)) {
+ prog = orig_prog;
+ goto out_off;
+ }
+@@ -1123,6 +1126,11 @@ skip_init_ctx:
+ prog->jited_len = prog_size;
+
+ if (!prog->is_func || extra_pass) {
++ int i;
++
++ /* offset[prog->len] is the size of program */
++ for (i = 0; i <= prog->len; i++)
++ ctx.offset[i] *= AARCH64_INSN_SIZE;
+ bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
+ out_off:
+ kfree(ctx.offset);
+@@ -1138,15 +1146,12 @@ out:
+
+ u64 bpf_jit_alloc_exec_limit(void)
+ {
+- return BPF_JIT_REGION_SIZE;
++ return VMALLOC_END - VMALLOC_START;
+ }
+
+ void *bpf_jit_alloc_exec(unsigned long size)
+ {
+- return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
+- BPF_JIT_REGION_END, GFP_KERNEL,
+- PAGE_KERNEL, 0, NUMA_NO_NODE,
+- __builtin_return_address(0));
++ return vmalloc(size);
+ }
+
+ void bpf_jit_free_exec(void *addr)
+diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
+index 49305c2e6dfd3..b71c6cbb23095 100644
+--- a/arch/arm64/tools/cpucaps
++++ b/arch/arm64/tools/cpucaps
+@@ -42,6 +42,7 @@ MTE
+ SPECTRE_V2
+ SPECTRE_V3A
+ SPECTRE_V4
++SPECTRE_BHB
+ SSBS
+ SVE
+ UNMAP_KERNEL_AT_EL0
+diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h
+index c40f06ee8d3ef..ac5a54f57d407 100644
+--- a/arch/csky/include/asm/uaccess.h
++++ b/arch/csky/include/asm/uaccess.h
+@@ -3,14 +3,13 @@
+ #ifndef __ASM_CSKY_UACCESS_H
+ #define __ASM_CSKY_UACCESS_H
+
+-#define user_addr_max() \
+- (uaccess_kernel() ? KERNEL_DS.seg : get_fs().seg)
++#define user_addr_max() (current_thread_info()->addr_limit.seg)
+
+ static inline int __access_ok(unsigned long addr, unsigned long size)
+ {
+- unsigned long limit = current_thread_info()->addr_limit.seg;
++ unsigned long limit = user_addr_max();
+
+- return ((addr < limit) && ((addr + size) < limit));
++ return (size <= limit) && (addr <= (limit - size));
+ }
+ #define __access_ok __access_ok
+
+diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c
+index ab55e98ee8f62..75e1f9df5f604 100644
+--- a/arch/csky/kernel/perf_callchain.c
++++ b/arch/csky/kernel/perf_callchain.c
+@@ -49,7 +49,7 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+ {
+ struct stackframe buftail;
+ unsigned long lr = 0;
+- unsigned long *user_frame_tail = (unsigned long *)fp;
++ unsigned long __user *user_frame_tail = (unsigned long __user *)fp;
+
+ /* Check accessibility of one struct frame_tail beyond */
+ if (!access_ok(user_frame_tail, sizeof(buftail)))
+@@ -86,10 +86,11 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ unsigned long fp = 0;
+
+ /* C-SKY does not support virtualization. */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
++ if (guest_cbs && guest_cbs->is_in_guest())
+ return;
+
+ fp = regs->regs[4];
+@@ -110,10 +111,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct stackframe fr;
+
+ /* C-SKY does not support virtualization. */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ pr_warn("C-SKY does not support perf in guest mode!");
+ return;
+ }
+diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c
+index c7b763d2f526e..8867ddf3e6c77 100644
+--- a/arch/csky/kernel/signal.c
++++ b/arch/csky/kernel/signal.c
+@@ -136,7 +136,7 @@ static inline void __user *get_sigframe(struct ksignal *ksig,
+ static int
+ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
+ {
+- struct rt_sigframe *frame;
++ struct rt_sigframe __user *frame;
+ int err = 0;
+
+ frame = get_sigframe(ksig, regs, sizeof(*frame));
+diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c
+index e5fbf8653a215..2020af88b6361 100644
+--- a/arch/csky/kernel/traps.c
++++ b/arch/csky/kernel/traps.c
+@@ -209,7 +209,7 @@ asmlinkage void do_trap_illinsn(struct pt_regs *regs)
+
+ asmlinkage void do_trap_fpe(struct pt_regs *regs)
+ {
+-#ifdef CONFIG_CPU_HAS_FP
++#ifdef CONFIG_CPU_HAS_FPU
+ return fpu_fpe(regs);
+ #else
+ do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->pc,
+@@ -219,7 +219,7 @@ asmlinkage void do_trap_fpe(struct pt_regs *regs)
+
+ asmlinkage void do_trap_priv(struct pt_regs *regs)
+ {
+-#ifdef CONFIG_CPU_HAS_FP
++#ifdef CONFIG_CPU_HAS_FPU
+ if (user_mode(regs) && fpu_libc_helper(regs))
+ return;
+ #endif
+diff --git a/arch/hexagon/include/asm/timer-regs.h b/arch/hexagon/include/asm/timer-regs.h
+deleted file mode 100644
+index ee6c61423a058..0000000000000
+--- a/arch/hexagon/include/asm/timer-regs.h
++++ /dev/null
+@@ -1,26 +0,0 @@
+-/* SPDX-License-Identifier: GPL-2.0-only */
+-/*
+- * Timer support for Hexagon
+- *
+- * Copyright (c) 2010-2011, The Linux Foundation. All rights reserved.
+- */
+-
+-#ifndef _ASM_TIMER_REGS_H
+-#define _ASM_TIMER_REGS_H
+-
+-/* This stuff should go into a platform specific file */
+-#define TCX0_CLK_RATE 19200
+-#define TIMER_ENABLE 0
+-#define TIMER_CLR_ON_MATCH 1
+-
+-/*
+- * 8x50 HDD Specs 5-8. Simulator co-sim not fixed until
+- * release 1.1, and then it's "adjustable" and probably not defaulted.
+- */
+-#define RTOS_TIMER_INT 3
+-#ifdef CONFIG_HEXAGON_COMET
+-#define RTOS_TIMER_REGS_ADDR 0xAB000000UL
+-#endif
+-#define SLEEP_CLK_RATE 32000
+-
+-#endif
+diff --git a/arch/hexagon/include/asm/timex.h b/arch/hexagon/include/asm/timex.h
+index 8d4ec76fceb45..dfe69e118b2be 100644
+--- a/arch/hexagon/include/asm/timex.h
++++ b/arch/hexagon/include/asm/timex.h
+@@ -7,11 +7,10 @@
+ #define _ASM_TIMEX_H
+
+ #include <asm-generic/timex.h>
+-#include <asm/timer-regs.h>
+ #include <asm/hexagon_vm.h>
+
+ /* Using TCX0 as our clock. CLOCK_TICK_RATE scheduled to be removed. */
+-#define CLOCK_TICK_RATE TCX0_CLK_RATE
++#define CLOCK_TICK_RATE 19200
+
+ #define ARCH_HAS_READ_CURRENT_TIMER
+
+diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h
+index ef5bfef8d490c..719ba3f3c45cd 100644
+--- a/arch/hexagon/include/asm/uaccess.h
++++ b/arch/hexagon/include/asm/uaccess.h
+@@ -25,17 +25,17 @@
+ * Returns true (nonzero) if the memory block *may* be valid, false (zero)
+ * if it is definitely invalid.
+ *
+- * User address space in Hexagon, like x86, goes to 0xbfffffff, so the
+- * simple MSB-based tests used by MIPS won't work. Some further
+- * optimization is probably possible here, but for now, keep it
+- * reasonably simple and not *too* slow. After all, we've got the
+- * MMU for backup.
+ */
++#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg)
++#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE)
+
+-#define __access_ok(addr, size) \
+- ((get_fs().seg == KERNEL_DS.seg) || \
+- (((unsigned long)addr < get_fs().seg) && \
+- (unsigned long)size < (get_fs().seg - (unsigned long)addr)))
++static inline int __access_ok(unsigned long addr, unsigned long size)
++{
++ unsigned long limit = TASK_SIZE;
++
++ return (size <= limit) && (addr <= (limit - size));
++}
++#define __access_ok __access_ok
+
+ /*
+ * When a kernel-mode page fault is taken, the faulting instruction
+diff --git a/arch/hexagon/kernel/time.c b/arch/hexagon/kernel/time.c
+index feffe527ac929..febc95714d756 100644
+--- a/arch/hexagon/kernel/time.c
++++ b/arch/hexagon/kernel/time.c
+@@ -17,9 +17,10 @@
+ #include <linux/of_irq.h>
+ #include <linux/module.h>
+
+-#include <asm/timer-regs.h>
+ #include <asm/hexagon_vm.h>
+
++#define TIMER_ENABLE BIT(0)
++
+ /*
+ * For the clocksource we need:
+ * pcycle frequency (600MHz)
+@@ -33,6 +34,13 @@ cycles_t pcycle_freq_mhz;
+ cycles_t thread_freq_mhz;
+ cycles_t sleep_clk_freq;
+
++/*
++ * 8x50 HDD Specs 5-8. Simulator co-sim not fixed until
++ * release 1.1, and then it's "adjustable" and probably not defaulted.
++ */
++#define RTOS_TIMER_INT 3
++#define RTOS_TIMER_REGS_ADDR 0xAB000000UL
++
+ static struct resource rtos_timer_resources[] = {
+ {
+ .start = RTOS_TIMER_REGS_ADDR,
+@@ -80,7 +88,7 @@ static int set_next_event(unsigned long delta, struct clock_event_device *evt)
+ iowrite32(0, &rtos_timer->clear);
+
+ iowrite32(delta, &rtos_timer->match);
+- iowrite32(1 << TIMER_ENABLE, &rtos_timer->enable);
++ iowrite32(TIMER_ENABLE, &rtos_timer->enable);
+ return 0;
+ }
+
+diff --git a/arch/hexagon/lib/io.c b/arch/hexagon/lib/io.c
+index d35d69d6588c4..55f75392857b0 100644
+--- a/arch/hexagon/lib/io.c
++++ b/arch/hexagon/lib/io.c
+@@ -27,6 +27,7 @@ void __raw_readsw(const void __iomem *addr, void *data, int len)
+ *dst++ = *src;
+
+ }
++EXPORT_SYMBOL(__raw_readsw);
+
+ /*
+ * __raw_writesw - read words a short at a time
+@@ -47,6 +48,7 @@ void __raw_writesw(void __iomem *addr, const void *data, int len)
+
+
+ }
++EXPORT_SYMBOL(__raw_writesw);
+
+ /* Pretty sure len is pre-adjusted for the length of the access already */
+ void __raw_readsl(const void __iomem *addr, void *data, int len)
+@@ -62,6 +64,7 @@ void __raw_readsl(const void __iomem *addr, void *data, int len)
+
+
+ }
++EXPORT_SYMBOL(__raw_readsl);
+
+ void __raw_writesl(void __iomem *addr, const void *data, int len)
+ {
+@@ -76,3 +79,4 @@ void __raw_writesl(void __iomem *addr, const void *data, int len)
+
+
+ }
++EXPORT_SYMBOL(__raw_writesl);
+diff --git a/arch/ia64/Kconfig.debug b/arch/ia64/Kconfig.debug
+index 40ca23bd228d6..2ce008e2d1644 100644
+--- a/arch/ia64/Kconfig.debug
++++ b/arch/ia64/Kconfig.debug
+@@ -39,7 +39,7 @@ config DISABLE_VHPT
+
+ config IA64_DEBUG_CMPXCHG
+ bool "Turn on compare-and-exchange bug checking (slow!)"
+- depends on DEBUG_KERNEL
++ depends on DEBUG_KERNEL && PRINTK
+ help
+ Selecting this option turns on bug checking for the IA-64
+ compare-and-exchange instructions. This is slow! Itaniums
+diff --git a/arch/ia64/include/asm/timex.h b/arch/ia64/include/asm/timex.h
+index 869a3ac6bf23a..7ccc077a60bed 100644
+--- a/arch/ia64/include/asm/timex.h
++++ b/arch/ia64/include/asm/timex.h
+@@ -39,6 +39,7 @@ get_cycles (void)
+ ret = ia64_getreg(_IA64_REG_AR_ITC);
+ return ret;
+ }
++#define get_cycles get_cycles
+
+ extern void ia64_cpu_local_tick (void);
+ extern unsigned long long ia64_native_sched_clock (void);
+diff --git a/arch/ia64/kernel/kprobes.c b/arch/ia64/kernel/kprobes.c
+index 441ed04b10378..d4048518a1d7d 100644
+--- a/arch/ia64/kernel/kprobes.c
++++ b/arch/ia64/kernel/kprobes.c
+@@ -398,7 +398,8 @@ static void kretprobe_trampoline(void)
+
+ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
+ {
+- regs->cr_iip = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL);
++ regs->cr_iip = __kretprobe_trampoline_handler(regs,
++ dereference_function_descriptor(kretprobe_trampoline), NULL);
+ /*
+ * By returning a non-zero value, we are telling
+ * kprobe_handler() that we don't want the post_handler
+@@ -414,7 +415,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
+ ri->fp = NULL;
+
+ /* Replace the return addr with trampoline addr */
+- regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;
++ regs->b0 = (unsigned long)dereference_function_descriptor(kretprobe_trampoline);
+ }
+
+ /* Check the instruction in the slot is break */
+@@ -902,14 +903,14 @@ static struct kprobe trampoline_p = {
+ int __init arch_init_kprobes(void)
+ {
+ trampoline_p.addr =
+- (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip;
++ dereference_function_descriptor(kretprobe_trampoline);
+ return register_kprobe(&trampoline_p);
+ }
+
+ int __kprobes arch_trampoline_kprobe(struct kprobe *p)
+ {
+ if (p->addr ==
+- (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip)
++ dereference_function_descriptor(kretprobe_trampoline))
+ return 1;
+
+ return 0;
+diff --git a/arch/ia64/pci/fixup.c b/arch/ia64/pci/fixup.c
+index acb55a41260dd..2bcdd7d3a1ada 100644
+--- a/arch/ia64/pci/fixup.c
++++ b/arch/ia64/pci/fixup.c
+@@ -76,5 +76,5 @@ static void pci_fixup_video(struct pci_dev *pdev)
+ }
+ }
+ }
+-DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID,
+- PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_video);
++DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_ANY_ID, PCI_ANY_ID,
++ PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_video);
+diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine
+index 36fa0c3ef1296..eeab4f3e6c197 100644
+--- a/arch/m68k/Kconfig.machine
++++ b/arch/m68k/Kconfig.machine
+@@ -203,6 +203,7 @@ config INIT_LCD
+ config MEMORY_RESERVE
+ int "Memory reservation (MiB)"
+ depends on (UCSIMM || UCDIMM)
++ default 0
+ help
+ Reserve certain memory regions on 68x328 based boards.
+
+diff --git a/arch/m68k/coldfire/device.c b/arch/m68k/coldfire/device.c
+index 0386252e9d043..4218750414bbf 100644
+--- a/arch/m68k/coldfire/device.c
++++ b/arch/m68k/coldfire/device.c
+@@ -480,7 +480,7 @@ static struct platform_device mcf_i2c5 = {
+ #endif /* MCFI2C_BASE5 */
+ #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
+
+-#if IS_ENABLED(CONFIG_MCF_EDMA)
++#ifdef MCFEDMA_BASE
+
+ static const struct dma_slave_map mcf_edma_map[] = {
+ { "dreq0", "rx-tx", MCF_EDMA_FILTER_PARAM(0) },
+@@ -552,7 +552,7 @@ static struct platform_device mcf_edma = {
+ .platform_data = &mcf_edma_data,
+ }
+ };
+-#endif /* IS_ENABLED(CONFIG_MCF_EDMA) */
++#endif /* MCFEDMA_BASE */
+
+ #ifdef MCFSDHC_BASE
+ static struct mcf_esdhc_platform_data mcf_esdhc_data = {
+@@ -651,7 +651,7 @@ static struct platform_device *mcf_devices[] __initdata = {
+ &mcf_i2c5,
+ #endif
+ #endif
+-#if IS_ENABLED(CONFIG_MCF_EDMA)
++#ifdef MCFEDMA_BASE
+ &mcf_edma,
+ #endif
+ #ifdef MCFSDHC_BASE
+diff --git a/arch/m68k/include/asm/timex.h b/arch/m68k/include/asm/timex.h
+index 6a21d93582805..f4a7a340f4cae 100644
+--- a/arch/m68k/include/asm/timex.h
++++ b/arch/m68k/include/asm/timex.h
+@@ -35,7 +35,7 @@ static inline unsigned long random_get_entropy(void)
+ {
+ if (mach_random_get_entropy)
+ return mach_random_get_entropy();
+- return 0;
++ return random_get_entropy_fallback();
+ }
+ #define random_get_entropy random_get_entropy
+
+diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h
+index ba670523885c8..60b786eb2254e 100644
+--- a/arch/m68k/include/asm/uaccess.h
++++ b/arch/m68k/include/asm/uaccess.h
+@@ -12,14 +12,17 @@
+ #include <asm/extable.h>
+
+ /* We let the MMU do all checking */
+-static inline int access_ok(const void __user *addr,
++static inline int access_ok(const void __user *ptr,
+ unsigned long size)
+ {
+- /*
+- * XXX: for !CONFIG_CPU_HAS_ADDRESS_SPACES this really needs to check
+- * for TASK_SIZE!
+- */
+- return 1;
++ unsigned long limit = TASK_SIZE;
++ unsigned long addr = (unsigned long)ptr;
++
++ if (IS_ENABLED(CONFIG_CPU_HAS_ADDRESS_SPACES) ||
++ !IS_ENABLED(CONFIG_MMU))
++ return 1;
++
++ return (size <= limit) && (addr <= (limit - size));
+ }
+
+ /*
+diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c
+index 9718ce94cc845..34d6458340b0f 100644
+--- a/arch/m68k/kernel/traps.c
++++ b/arch/m68k/kernel/traps.c
+@@ -1145,7 +1145,7 @@ asmlinkage void set_esp0(unsigned long ssp)
+ */
+ asmlinkage void fpsp040_die(void)
+ {
+- force_sigsegv(SIGSEGV);
++ force_exit_sig(SIGSEGV);
+ }
+
+ #ifdef CONFIG_M68KFPU_EMU
+diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h
+index d2a8ef9f89787..3fe96979d2c62 100644
+--- a/arch/microblaze/include/asm/uaccess.h
++++ b/arch/microblaze/include/asm/uaccess.h
+@@ -39,24 +39,13 @@
+
+ # define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg)
+
+-static inline int access_ok(const void __user *addr, unsigned long size)
++static inline int __access_ok(unsigned long addr, unsigned long size)
+ {
+- if (!size)
+- goto ok;
++ unsigned long limit = user_addr_max();
+
+- if ((get_fs().seg < ((unsigned long)addr)) ||
+- (get_fs().seg < ((unsigned long)addr + size - 1))) {
+- pr_devel("ACCESS fail at 0x%08x (size 0x%x), seg 0x%08x\n",
+- (__force u32)addr, (u32)size,
+- (u32)get_fs().seg);
+- return 0;
+- }
+-ok:
+- pr_devel("ACCESS OK at 0x%08x (size 0x%x), seg 0x%08x\n",
+- (__force u32)addr, (u32)size,
+- (u32)get_fs().seg);
+- return 1;
++ return (size <= limit) && (addr <= (limit - size));
+ }
++#define access_ok(addr, size) __access_ok((unsigned long)addr, size)
+
+ # define __FIXUP_SECTION ".section .fixup,\"ax\"\n"
+ # define __EX_TABLE_SECTION ".section __ex_table,\"a\"\n"
+@@ -141,27 +130,27 @@ extern long __user_bad(void);
+
+ #define __get_user(x, ptr) \
+ ({ \
+- unsigned long __gu_val = 0; \
+ long __gu_err; \
+ switch (sizeof(*(ptr))) { \
+ case 1: \
+- __get_user_asm("lbu", (ptr), __gu_val, __gu_err); \
++ __get_user_asm("lbu", (ptr), x, __gu_err); \
+ break; \
+ case 2: \
+- __get_user_asm("lhu", (ptr), __gu_val, __gu_err); \
++ __get_user_asm("lhu", (ptr), x, __gu_err); \
+ break; \
+ case 4: \
+- __get_user_asm("lw", (ptr), __gu_val, __gu_err); \
++ __get_user_asm("lw", (ptr), x, __gu_err); \
+ break; \
+- case 8: \
+- __gu_err = __copy_from_user(&__gu_val, ptr, 8); \
+- if (__gu_err) \
+- __gu_err = -EFAULT; \
++ case 8: { \
++ __u64 __x = 0; \
++ __gu_err = raw_copy_from_user(&__x, ptr, 8) ? \
++ -EFAULT : 0; \
++ (x) = (typeof(x))(typeof((x) - (x)))__x; \
+ break; \
++ } \
+ default: \
+ /* __gu_val = 0; __gu_err = -EINVAL;*/ __gu_err = __user_bad();\
+ } \
+- x = (__force __typeof__(*(ptr))) __gu_val; \
+ __gu_err; \
+ })
+
+diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms
+index 584081df89c28..6e3f36c841e5d 100644
+--- a/arch/mips/Kbuild.platforms
++++ b/arch/mips/Kbuild.platforms
+@@ -38,4 +38,4 @@ platform-$(CONFIG_MACH_TX49XX) += txx9/
+ platform-$(CONFIG_MACH_VR41XX) += vr41xx/
+
+ # include the platform specific files
+-include $(patsubst %, $(srctree)/arch/mips/%/Platform, $(platform-y))
++include $(patsubst %/, $(srctree)/arch/mips/%/Platform, $(platform-y))
+diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
+index 6b8f591c5054c..393eb2133243f 100644
+--- a/arch/mips/Kconfig
++++ b/arch/mips/Kconfig
+@@ -332,6 +332,9 @@ config BCM63XX
+ select SYS_SUPPORTS_32BIT_KERNEL
+ select SYS_SUPPORTS_BIG_ENDIAN
+ select SYS_HAS_EARLY_PRINTK
++ select SYS_HAS_CPU_BMIPS32_3300
++ select SYS_HAS_CPU_BMIPS4350
++ select SYS_HAS_CPU_BMIPS4380
+ select SWAP_IO_SPACE
+ select GPIOLIB
+ select MIPS_L1_CACHE_SHIFT_4
+@@ -1379,6 +1382,7 @@ config CPU_LOONGSON64
+ select MIPS_ASID_BITS_VARIABLE
+ select MIPS_PGD_C0_CONTEXT
+ select MIPS_L1_CACHE_SHIFT_6
++ select MIPS_FP_SUPPORT
+ select GPIOLIB
+ select SWIOTLB
+ select HAVE_KVM
+@@ -1989,6 +1993,10 @@ config SYS_HAS_CPU_MIPS64_R1
+ config SYS_HAS_CPU_MIPS64_R2
+ bool
+
++config SYS_HAS_CPU_MIPS64_R5
++ bool
++ select ARCH_HAS_SYNC_DMA_FOR_CPU if DMA_NONCOHERENT
++
+ config SYS_HAS_CPU_MIPS64_R6
+ bool
+ select ARCH_HAS_SYNC_DMA_FOR_CPU if DMA_NONCOHERENT
+@@ -2153,7 +2161,7 @@ config CPU_SUPPORTS_ADDRWINCFG
+ bool
+ config CPU_SUPPORTS_HUGEPAGES
+ bool
+- depends on !(32BIT && (ARCH_PHYS_ADDR_T_64BIT || EVA))
++ depends on !(32BIT && (PHYS_ADDR_T_64BIT || EVA))
+ config MIPS_PGD_C0_CONTEXT
+ bool
+ depends on 64BIT
+@@ -3185,7 +3193,7 @@ config STACKTRACE_SUPPORT
+ config PGTABLE_LEVELS
+ int
+ default 4 if PAGE_SIZE_4KB && MIPS_VA_BITS_48
+- default 3 if 64BIT && !PAGE_SIZE_64KB
++ default 3 if 64BIT && (!PAGE_SIZE_64KB || MIPS_VA_BITS_48)
+ default 2
+
+ config MIPS_AUTO_PFN_OFFSET
+diff --git a/arch/mips/Makefile b/arch/mips/Makefile
+index ea3cd080a1c7d..f7b58da2f3889 100644
+--- a/arch/mips/Makefile
++++ b/arch/mips/Makefile
+@@ -254,7 +254,9 @@ endif
+ #
+ # Board-dependent options and extra files
+ #
++ifdef need-compiler
+ include $(srctree)/arch/mips/Kbuild.platforms
++endif
+
+ ifdef CONFIG_PHYSICAL_START
+ load-y = $(CONFIG_PHYSICAL_START)
+diff --git a/arch/mips/bcm63xx/clk.c b/arch/mips/bcm63xx/clk.c
+index 5a3e325275d0d..6e6756e8fa0a9 100644
+--- a/arch/mips/bcm63xx/clk.c
++++ b/arch/mips/bcm63xx/clk.c
+@@ -381,6 +381,18 @@ void clk_disable(struct clk *clk)
+
+ EXPORT_SYMBOL(clk_disable);
+
++struct clk *clk_get_parent(struct clk *clk)
++{
++ return NULL;
++}
++EXPORT_SYMBOL(clk_get_parent);
++
++int clk_set_parent(struct clk *clk, struct clk *parent)
++{
++ return 0;
++}
++EXPORT_SYMBOL(clk_set_parent);
++
+ unsigned long clk_get_rate(struct clk *clk)
+ {
+ if (!clk)
+diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
+index 3548b3b452699..705b9e7f8035a 100644
+--- a/arch/mips/boot/compressed/Makefile
++++ b/arch/mips/boot/compressed/Makefile
+@@ -56,6 +56,8 @@ $(obj)/uart-ath79.c: $(srctree)/arch/mips/ath79/early_printk.c
+
+ vmlinuzobjs-$(CONFIG_KERNEL_XZ) += $(obj)/ashldi3.o
+
++vmlinuzobjs-$(CONFIG_KERNEL_ZSTD) += $(obj)/bswapdi.o $(obj)/ashldi3.o $(obj)/clz_ctz.o
++
+ extra-y += ashldi3.c
+ $(obj)/ashldi3.c: $(obj)/%.c: $(srctree)/lib/%.c FORCE
+ $(call if_changed,shipped)
+@@ -64,6 +66,10 @@ extra-y += bswapsi.c
+ $(obj)/bswapsi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c FORCE
+ $(call if_changed,shipped)
+
++extra-y += bswapdi.c
++$(obj)/bswapdi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c FORCE
++ $(call if_changed,shipped)
++
+ targets := $(notdir $(vmlinuzobjs-y))
+
+ targets += vmlinux.bin
+diff --git a/arch/mips/boot/compressed/clz_ctz.c b/arch/mips/boot/compressed/clz_ctz.c
+new file mode 100644
+index 0000000000000..b4a1b6eb2f8ad
+--- /dev/null
++++ b/arch/mips/boot/compressed/clz_ctz.c
+@@ -0,0 +1,2 @@
++// SPDX-License-Identifier: GPL-2.0-only
++#include "../../../../lib/clz_ctz.c"
+diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi
+index 9e34f433b9b58..efbbddaf0fde5 100644
+--- a/arch/mips/boot/dts/ingenic/jz4780.dtsi
++++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi
+@@ -450,7 +450,7 @@
+ #address-cells = <1>;
+ #size-cells = <1>;
+
+- eth0_addr: eth-mac-addr@0x22 {
++ eth0_addr: eth-mac-addr@22 {
+ reg = <0x22 0x6>;
+ };
+ };
+diff --git a/arch/mips/cavium-octeon/octeon-memcpy.S b/arch/mips/cavium-octeon/octeon-memcpy.S
+index 0a515cde1c183..25860fba6218d 100644
+--- a/arch/mips/cavium-octeon/octeon-memcpy.S
++++ b/arch/mips/cavium-octeon/octeon-memcpy.S
+@@ -74,7 +74,7 @@
+ #define EXC(inst_reg,addr,handler) \
+ 9: inst_reg, addr; \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous
+
+ /*
+diff --git a/arch/mips/cavium-octeon/octeon-platform.c b/arch/mips/cavium-octeon/octeon-platform.c
+index d56e9b9d2e434..a994022e32c9f 100644
+--- a/arch/mips/cavium-octeon/octeon-platform.c
++++ b/arch/mips/cavium-octeon/octeon-platform.c
+@@ -328,6 +328,7 @@ static int __init octeon_ehci_device_init(void)
+
+ pd->dev.platform_data = &octeon_ehci_pdata;
+ octeon_ehci_hw_start(&pd->dev);
++ put_device(&pd->dev);
+
+ return ret;
+ }
+@@ -391,6 +392,7 @@ static int __init octeon_ohci_device_init(void)
+
+ pd->dev.platform_data = &octeon_ohci_pdata;
+ octeon_ohci_hw_start(&pd->dev);
++ put_device(&pd->dev);
+
+ return ret;
+ }
+diff --git a/arch/mips/cavium-octeon/octeon-usb.c b/arch/mips/cavium-octeon/octeon-usb.c
+index 6e4d3619137af..4df919d26b082 100644
+--- a/arch/mips/cavium-octeon/octeon-usb.c
++++ b/arch/mips/cavium-octeon/octeon-usb.c
+@@ -537,6 +537,7 @@ static int __init dwc3_octeon_device_init(void)
+ devm_iounmap(&pdev->dev, base);
+ devm_release_mem_region(&pdev->dev, res->start,
+ resource_size(res));
++ put_device(&pdev->dev);
+ }
+ } while (node != NULL);
+
+diff --git a/arch/mips/configs/fuloong2e_defconfig b/arch/mips/configs/fuloong2e_defconfig
+index 5c24ac7fdf56d..ba47c5e929b7f 100644
+--- a/arch/mips/configs/fuloong2e_defconfig
++++ b/arch/mips/configs/fuloong2e_defconfig
+@@ -206,7 +206,6 @@ CONFIG_NFSD_V3_ACL=y
+ CONFIG_NFSD_V4=y
+ CONFIG_CIFS=m
+ CONFIG_CIFS_STATS2=y
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_CIFS_DEBUG2=y
+diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig
+index 614af02d83e6e..6fb9bc29f4a03 100644
+--- a/arch/mips/configs/malta_qemu_32r6_defconfig
++++ b/arch/mips/configs/malta_qemu_32r6_defconfig
+@@ -165,7 +165,6 @@ CONFIG_TMPFS=y
+ CONFIG_NFS_FS=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig
+index 9c051f8fd3300..eb72df528243a 100644
+--- a/arch/mips/configs/maltaaprp_defconfig
++++ b/arch/mips/configs/maltaaprp_defconfig
+@@ -166,7 +166,6 @@ CONFIG_TMPFS=y
+ CONFIG_NFS_FS=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig
+index 2e90d97551d6f..1fb40d310f49c 100644
+--- a/arch/mips/configs/maltasmvp_defconfig
++++ b/arch/mips/configs/maltasmvp_defconfig
+@@ -167,7 +167,6 @@ CONFIG_TMPFS=y
+ CONFIG_NFS_FS=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig
+index d1f7fdb27284b..75cb778c61496 100644
+--- a/arch/mips/configs/maltasmvp_eva_defconfig
++++ b/arch/mips/configs/maltasmvp_eva_defconfig
+@@ -169,7 +169,6 @@ CONFIG_TMPFS=y
+ CONFIG_NFS_FS=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig
+index 48e5bd4924522..7b4f247dc60cc 100644
+--- a/arch/mips/configs/maltaup_defconfig
++++ b/arch/mips/configs/maltaup_defconfig
+@@ -165,7 +165,6 @@ CONFIG_TMPFS=y
+ CONFIG_NFS_FS=y
+ CONFIG_ROOT_NFS=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+ CONFIG_NLS_CODEPAGE_437=m
+diff --git a/arch/mips/dec/int-handler.S b/arch/mips/dec/int-handler.S
+index ea5b5a83f1e11..011d1d678840a 100644
+--- a/arch/mips/dec/int-handler.S
++++ b/arch/mips/dec/int-handler.S
+@@ -131,7 +131,7 @@
+ */
+ mfc0 t0,CP0_CAUSE # get pending interrupts
+ mfc0 t1,CP0_STATUS
+-#ifdef CONFIG_32BIT
++#if defined(CONFIG_32BIT) && defined(CONFIG_MIPS_FP_SUPPORT)
+ lw t2,cpu_fpu_mask
+ #endif
+ andi t0,ST0_IM # CAUSE.CE may be non-zero!
+@@ -139,7 +139,7 @@
+
+ beqz t0,spurious
+
+-#ifdef CONFIG_32BIT
++#if defined(CONFIG_32BIT) && defined(CONFIG_MIPS_FP_SUPPORT)
+ and t2,t0
+ bnez t2,fpu # handle FPU immediately
+ #endif
+@@ -280,7 +280,7 @@ handle_it:
+ j dec_irq_dispatch
+ nop
+
+-#ifdef CONFIG_32BIT
++#if defined(CONFIG_32BIT) && defined(CONFIG_MIPS_FP_SUPPORT)
+ fpu:
+ lw t0,fpu_kstat_irq
+ nop
+diff --git a/arch/mips/dec/prom/Makefile b/arch/mips/dec/prom/Makefile
+index d95016016b42b..2bad87551203b 100644
+--- a/arch/mips/dec/prom/Makefile
++++ b/arch/mips/dec/prom/Makefile
+@@ -6,4 +6,4 @@
+
+ lib-y += init.o memory.o cmdline.o identify.o console.o
+
+-lib-$(CONFIG_32BIT) += locore.o
++lib-$(CONFIG_CPU_R3000) += locore.o
+diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c
+index eaad0ed4b523b..99b9b29750db3 100644
+--- a/arch/mips/dec/setup.c
++++ b/arch/mips/dec/setup.c
+@@ -746,7 +746,8 @@ void __init arch_init_irq(void)
+ dec_interrupt[DEC_IRQ_HALT] = -1;
+
+ /* Register board interrupts: FPU and cascade. */
+- if (dec_interrupt[DEC_IRQ_FPU] >= 0 && cpu_has_fpu) {
++ if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT) &&
++ dec_interrupt[DEC_IRQ_FPU] >= 0 && cpu_has_fpu) {
+ struct irq_desc *desc_fpu;
+ int irq_fpu;
+
+diff --git a/arch/mips/generic/yamon-dt.c b/arch/mips/generic/yamon-dt.c
+index a3aa22c77cadc..a07a5edbcda78 100644
+--- a/arch/mips/generic/yamon-dt.c
++++ b/arch/mips/generic/yamon-dt.c
+@@ -75,7 +75,7 @@ static unsigned int __init gen_fdt_mem_array(
+ __init int yamon_dt_append_memory(void *fdt,
+ const struct yamon_mem_region *regions)
+ {
+- unsigned long phys_memsize, memsize;
++ unsigned long phys_memsize = 0, memsize;
+ __be32 mem_array[2 * MAX_MEM_ARRAY_ENTRIES];
+ unsigned int mem_entries;
+ int i, err, mem_off;
+diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h
+index 2f8ce94ebaafe..cc69f1deb1ca8 100644
+--- a/arch/mips/include/asm/asm.h
++++ b/arch/mips/include/asm/asm.h
+@@ -276,7 +276,7 @@ symbol = value
+
+ #define PTR_SCALESHIFT 2
+
+-#define PTR .word
++#define PTR_WD .word
+ #define PTRSIZE 4
+ #define PTRLOG 2
+ #endif
+@@ -301,7 +301,7 @@ symbol = value
+
+ #define PTR_SCALESHIFT 3
+
+-#define PTR .dword
++#define PTR_WD .dword
+ #define PTRSIZE 8
+ #define PTRLOG 3
+ #endif
+diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h
+index 0b983800f48b7..66a8b293fd80b 100644
+--- a/arch/mips/include/asm/cmpxchg.h
++++ b/arch/mips/include/asm/cmpxchg.h
+@@ -249,6 +249,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr,
+ /* Load 64 bits from ptr */
+ " " __SYNC(full, loongson3_war) " \n"
+ "1: lld %L0, %3 # __cmpxchg64 \n"
++ " .set pop \n"
+ /*
+ * Split the 64 bit value we loaded into the 2 registers that hold the
+ * ret variable.
+@@ -276,12 +277,14 @@ static inline unsigned long __cmpxchg64(volatile void *ptr,
+ " or %L1, %L1, $at \n"
+ " .set at \n"
+ # endif
++ " .set push \n"
++ " .set " MIPS_ISA_ARCH_LEVEL " \n"
+ /* Attempt to store new at ptr */
+ " scd %L1, %2 \n"
+ /* If we failed, loop! */
+ "\t" __SC_BEQZ "%L1, 1b \n"
+- " .set pop \n"
+ "2: " __SYNC(full, loongson3_war) " \n"
++ " .set pop \n"
+ : "=&r"(ret),
+ "=&r"(tmp),
+ "=" GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr)
+diff --git a/arch/mips/include/asm/dec/prom.h b/arch/mips/include/asm/dec/prom.h
+index 62c7dfb90e06c..1e1247add1cf8 100644
+--- a/arch/mips/include/asm/dec/prom.h
++++ b/arch/mips/include/asm/dec/prom.h
+@@ -43,16 +43,11 @@
+ */
+ #define REX_PROM_MAGIC 0x30464354
+
+-#ifdef CONFIG_64BIT
+-
+-#define prom_is_rex(magic) 1 /* KN04 and KN05 are REX PROMs. */
+-
+-#else /* !CONFIG_64BIT */
+-
+-#define prom_is_rex(magic) ((magic) == REX_PROM_MAGIC)
+-
+-#endif /* !CONFIG_64BIT */
+-
++/* KN04 and KN05 are REX PROMs, so only do the check for R3k systems. */
++static inline bool prom_is_rex(u32 magic)
++{
++ return !IS_ENABLED(CONFIG_CPU_R3000) || magic == REX_PROM_MAGIC;
++}
+
+ /*
+ * 3MIN/MAXINE PROM entry points for DS5000/1xx's, DS5000/xx's and
+diff --git a/arch/mips/include/asm/ftrace.h b/arch/mips/include/asm/ftrace.h
+index b463f2aa5a613..db497a8167da2 100644
+--- a/arch/mips/include/asm/ftrace.h
++++ b/arch/mips/include/asm/ftrace.h
+@@ -32,7 +32,7 @@ do { \
+ ".previous\n" \
+ \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR) "\t1b, 3b\n\t" \
++ STR(PTR_WD) "\t1b, 3b\n\t" \
+ ".previous\n" \
+ \
+ : [tmp_dst] "=&r" (dst), [tmp_err] "=r" (error)\
+@@ -54,7 +54,7 @@ do { \
+ ".previous\n" \
+ \
+ ".section\t__ex_table,\"a\"\n\t"\
+- STR(PTR) "\t1b, 3b\n\t" \
++ STR(PTR_WD) "\t1b, 3b\n\t" \
+ ".previous\n" \
+ \
+ : [tmp_err] "=r" (error) \
+diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h
+index ecda7295ddcd1..3fa6340903882 100644
+--- a/arch/mips/include/asm/local.h
++++ b/arch/mips/include/asm/local.h
+@@ -5,6 +5,7 @@
+ #include <linux/percpu.h>
+ #include <linux/bitops.h>
+ #include <linux/atomic.h>
++#include <asm/asm.h>
+ #include <asm/cmpxchg.h>
+ #include <asm/compiler.h>
+ #include <asm/war.h>
+@@ -39,7 +40,7 @@ static __inline__ long local_add_return(long i, local_t * l)
+ " .set arch=r4000 \n"
+ __SYNC(full, loongson3_war) " \n"
+ "1:" __LL "%1, %2 # local_add_return \n"
+- " addu %0, %1, %3 \n"
++ __stringify(LONG_ADDU) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqzl %0, 1b \n"
+ " addu %0, %1, %3 \n"
+@@ -55,7 +56,7 @@ static __inline__ long local_add_return(long i, local_t * l)
+ " .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __SYNC(full, loongson3_war) " \n"
+ "1:" __LL "%1, %2 # local_add_return \n"
+- " addu %0, %1, %3 \n"
++ __stringify(LONG_ADDU) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqz %0, 1b \n"
+ " addu %0, %1, %3 \n"
+@@ -88,7 +89,7 @@ static __inline__ long local_sub_return(long i, local_t * l)
+ " .set arch=r4000 \n"
+ __SYNC(full, loongson3_war) " \n"
+ "1:" __LL "%1, %2 # local_sub_return \n"
+- " subu %0, %1, %3 \n"
++ __stringify(LONG_SUBU) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqzl %0, 1b \n"
+ " subu %0, %1, %3 \n"
+@@ -104,7 +105,7 @@ static __inline__ long local_sub_return(long i, local_t * l)
+ " .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __SYNC(full, loongson3_war) " \n"
+ "1:" __LL "%1, %2 # local_sub_return \n"
+- " subu %0, %1, %3 \n"
++ __stringify(LONG_SUBU) " %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqz %0, 1b \n"
+ " subu %0, %1, %3 \n"
+diff --git a/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h b/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h
+index 13373c5144f89..efb41b3519747 100644
+--- a/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h
++++ b/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h
+@@ -32,7 +32,7 @@
+ nop
+ /* Loongson-3A R2/R3 */
+ andi t0, (PRID_IMP_MASK | PRID_REV_MASK)
+- slti t0, (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)
++ slti t0, t0, (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)
+ bnez t0, 2f
+ nop
+ 1:
+@@ -63,7 +63,7 @@
+ nop
+ /* Loongson-3A R2/R3 */
+ andi t0, (PRID_IMP_MASK | PRID_REV_MASK)
+- slti t0, (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)
++ slti t0, t0, (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)
+ bnez t0, 2f
+ nop
+ 1:
+diff --git a/arch/mips/include/asm/mips-cm.h b/arch/mips/include/asm/mips-cm.h
+index aeae2effa123d..23c67c0871b17 100644
+--- a/arch/mips/include/asm/mips-cm.h
++++ b/arch/mips/include/asm/mips-cm.h
+@@ -11,6 +11,7 @@
+ #ifndef __MIPS_ASM_MIPS_CM_H__
+ #define __MIPS_ASM_MIPS_CM_H__
+
++#include <linux/bitfield.h>
+ #include <linux/bitops.h>
+ #include <linux/errno.h>
+
+@@ -153,8 +154,8 @@ GCR_ACCESSOR_RO(32, 0x030, rev)
+ #define CM_GCR_REV_MINOR GENMASK(7, 0)
+
+ #define CM_ENCODE_REV(major, minor) \
+- (((major) << __ffs(CM_GCR_REV_MAJOR)) | \
+- ((minor) << __ffs(CM_GCR_REV_MINOR)))
++ (FIELD_PREP(CM_GCR_REV_MAJOR, major) | \
++ FIELD_PREP(CM_GCR_REV_MINOR, minor))
+
+ #define CM_REV_CM2 CM_ENCODE_REV(6, 0)
+ #define CM_REV_CM2_5 CM_ENCODE_REV(7, 0)
+@@ -362,10 +363,10 @@ static inline int mips_cm_revision(void)
+ static inline unsigned int mips_cm_max_vp_width(void)
+ {
+ extern int smp_num_siblings;
+- uint32_t cfg;
+
+ if (mips_cm_revision() >= CM_REV_CM3)
+- return read_gcr_sys_config2() & CM_GCR_SYS_CONFIG2_MAXVPW;
++ return FIELD_GET(CM_GCR_SYS_CONFIG2_MAXVPW,
++ read_gcr_sys_config2());
+
+ if (mips_cm_present()) {
+ /*
+@@ -373,8 +374,7 @@ static inline unsigned int mips_cm_max_vp_width(void)
+ * number of VP(E)s, and if that ever changes then this will
+ * need revisiting.
+ */
+- cfg = read_gcr_cl_config() & CM_GCR_Cx_CONFIG_PVPE;
+- return (cfg >> __ffs(CM_GCR_Cx_CONFIG_PVPE)) + 1;
++ return FIELD_GET(CM_GCR_Cx_CONFIG_PVPE, read_gcr_cl_config()) + 1;
+ }
+
+ if (IS_ENABLED(CONFIG_SMP))
+diff --git a/arch/mips/include/asm/octeon/cvmx-bootinfo.h b/arch/mips/include/asm/octeon/cvmx-bootinfo.h
+index 0e6bf220db618..6c61e0a639249 100644
+--- a/arch/mips/include/asm/octeon/cvmx-bootinfo.h
++++ b/arch/mips/include/asm/octeon/cvmx-bootinfo.h
+@@ -318,7 +318,7 @@ enum cvmx_chip_types_enum {
+
+ /* Functions to return string based on type */
+ #define ENUM_BRD_TYPE_CASE(x) \
+- case x: return(#x + 16); /* Skip CVMX_BOARD_TYPE_ */
++ case x: return (&#x[16]); /* Skip CVMX_BOARD_TYPE_ */
+ static inline const char *cvmx_board_type_to_string(enum
+ cvmx_board_types_enum type)
+ {
+@@ -410,7 +410,7 @@ static inline const char *cvmx_board_type_to_string(enum
+ }
+
+ #define ENUM_CHIP_TYPE_CASE(x) \
+- case x: return(#x + 15); /* Skip CVMX_CHIP_TYPE */
++ case x: return (&#x[15]); /* Skip CVMX_CHIP_TYPE */
+ static inline const char *cvmx_chip_type_to_string(enum
+ cvmx_chip_types_enum type)
+ {
+diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
+index c7925d0e98746..867e9c3db76e9 100644
+--- a/arch/mips/include/asm/pgalloc.h
++++ b/arch/mips/include/asm/pgalloc.h
+@@ -15,6 +15,7 @@
+
+ #define __HAVE_ARCH_PMD_ALLOC_ONE
+ #define __HAVE_ARCH_PUD_ALLOC_ONE
++#define __HAVE_ARCH_PGD_FREE
+ #include <asm-generic/pgalloc.h>
+
+ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
+@@ -48,6 +49,11 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+ extern void pgd_init(unsigned long page);
+ extern pgd_t *pgd_alloc(struct mm_struct *mm);
+
++static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
++{
++ free_pages((unsigned long)pgd, PGD_ORDER);
++}
++
+ #define __pte_free_tlb(tlb,pte,address) \
+ do { \
+ pgtable_pte_page_dtor(pte); \
+diff --git a/arch/mips/include/asm/r4kcache.h b/arch/mips/include/asm/r4kcache.h
+index af3788589ee6d..431a1c9d53fc7 100644
+--- a/arch/mips/include/asm/r4kcache.h
++++ b/arch/mips/include/asm/r4kcache.h
+@@ -119,7 +119,7 @@ static inline void flush_scache_line(unsigned long addr)
+ " j 2b \n" \
+ " .previous \n" \
+ " .section __ex_table,\"a\" \n" \
+- " "STR(PTR)" 1b, 3b \n" \
++ " "STR(PTR_WD)" 1b, 3b \n" \
+ " .previous" \
+ : "+r" (__err) \
+ : "i" (op), "r" (addr), "i" (-EFAULT)); \
+@@ -142,7 +142,7 @@ static inline void flush_scache_line(unsigned long addr)
+ " j 2b \n" \
+ " .previous \n" \
+ " .section __ex_table,\"a\" \n" \
+- " "STR(PTR)" 1b, 3b \n" \
++ " "STR(PTR_WD)" 1b, 3b \n" \
+ " .previous" \
+ : "+r" (__err) \
+ : "i" (op), "r" (addr), "i" (-EFAULT)); \
+diff --git a/arch/mips/include/asm/setup.h b/arch/mips/include/asm/setup.h
+index bb36a400203df..8c56b862fd9c2 100644
+--- a/arch/mips/include/asm/setup.h
++++ b/arch/mips/include/asm/setup.h
+@@ -16,7 +16,7 @@ static inline void setup_8250_early_printk_port(unsigned long base,
+ unsigned int reg_shift, unsigned int timeout) {}
+ #endif
+
+-extern void set_handler(unsigned long offset, void *addr, unsigned long len);
++void set_handler(unsigned long offset, const void *addr, unsigned long len);
+ extern void set_uncached_handler(unsigned long offset, void *addr, unsigned long len);
+
+ typedef void (*vi_handler_t)(void);
+diff --git a/arch/mips/include/asm/timex.h b/arch/mips/include/asm/timex.h
+index b05bb70a2e46f..2e107886f97ac 100644
+--- a/arch/mips/include/asm/timex.h
++++ b/arch/mips/include/asm/timex.h
+@@ -40,9 +40,9 @@
+ typedef unsigned int cycles_t;
+
+ /*
+- * On R4000/R4400 before version 5.0 an erratum exists such that if the
+- * cycle counter is read in the exact moment that it is matching the
+- * compare register, no interrupt will be generated.
++ * On R4000/R4400 an erratum exists such that if the cycle counter is
++ * read in the exact moment that it is matching the compare register,
++ * no interrupt will be generated.
+ *
+ * There is a suggested workaround and also the erratum can't strike if
+ * the compare interrupt isn't being used as the clock source device.
+@@ -63,7 +63,7 @@ static inline int can_use_mips_counter(unsigned int prid)
+ if (!__builtin_constant_p(cpu_has_counter))
+ asm volatile("" : "=m" (cpu_data[0].options));
+ if (likely(cpu_has_counter &&
+- prid >= (PRID_IMP_R4000 | PRID_REV_ENCODE_44(5, 0))))
++ prid > (PRID_IMP_R4000 | PRID_REV_ENCODE_44(15, 15))))
+ return 1;
+ else
+ return 0;
+@@ -76,25 +76,24 @@ static inline cycles_t get_cycles(void)
+ else
+ return 0; /* no usable counter */
+ }
++#define get_cycles get_cycles
+
+ /*
+ * Like get_cycles - but where c0_count is not available we desperately
+ * use c0_random in an attempt to get at least a little bit of entropy.
+- *
+- * R6000 and R6000A neither have a count register nor a random register.
+- * That leaves no entropy source in the CPU itself.
+ */
+ static inline unsigned long random_get_entropy(void)
+ {
+- unsigned int prid = read_c0_prid();
+- unsigned int imp = prid & PRID_IMP_MASK;
++ unsigned int c0_random;
+
+- if (can_use_mips_counter(prid))
++ if (can_use_mips_counter(read_c0_prid()))
+ return read_c0_count();
+- else if (likely(imp != PRID_IMP_R6000 && imp != PRID_IMP_R6000A))
+- return read_c0_random();
++
++ if (cpu_has_3kex)
++ c0_random = (read_c0_random() >> 8) & 0x3f;
+ else
+- return 0; /* no usable register */
++ c0_random = read_c0_random() & 0x3f;
++ return (random_get_entropy_fallback() << 6) | (0x3f - c0_random);
+ }
+ #define random_get_entropy random_get_entropy
+
+diff --git a/arch/mips/include/asm/unaligned-emul.h b/arch/mips/include/asm/unaligned-emul.h
+index 2022b18944b97..9af0f4d3d288c 100644
+--- a/arch/mips/include/asm/unaligned-emul.h
++++ b/arch/mips/include/asm/unaligned-emul.h
+@@ -20,8 +20,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -41,8 +41,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -74,10 +74,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -102,8 +102,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -125,8 +125,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -145,8 +145,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -178,10 +178,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -223,14 +223,14 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
+- STR(PTR)"\t5b, 11b\n\t" \
+- STR(PTR)"\t6b, 11b\n\t" \
+- STR(PTR)"\t7b, 11b\n\t" \
+- STR(PTR)"\t8b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t5b, 11b\n\t" \
++ STR(PTR_WD)"\t6b, 11b\n\t" \
++ STR(PTR_WD)"\t7b, 11b\n\t" \
++ STR(PTR_WD)"\t8b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -255,8 +255,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT));\
+@@ -276,8 +276,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT)); \
+@@ -296,8 +296,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT)); \
+@@ -325,10 +325,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT) \
+@@ -365,14 +365,14 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
+- STR(PTR)"\t5b, 11b\n\t" \
+- STR(PTR)"\t6b, 11b\n\t" \
+- STR(PTR)"\t7b, 11b\n\t" \
+- STR(PTR)"\t8b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t5b, 11b\n\t" \
++ STR(PTR_WD)"\t6b, 11b\n\t" \
++ STR(PTR_WD)"\t7b, 11b\n\t" \
++ STR(PTR_WD)"\t8b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT) \
+@@ -398,8 +398,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -419,8 +419,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -452,10 +452,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -481,8 +481,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -504,8 +504,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -524,8 +524,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -557,10 +557,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -602,14 +602,14 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
+- STR(PTR)"\t5b, 11b\n\t" \
+- STR(PTR)"\t6b, 11b\n\t" \
+- STR(PTR)"\t7b, 11b\n\t" \
+- STR(PTR)"\t8b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t5b, 11b\n\t" \
++ STR(PTR_WD)"\t6b, 11b\n\t" \
++ STR(PTR_WD)"\t7b, 11b\n\t" \
++ STR(PTR_WD)"\t8b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (value), "=r" (res) \
+ : "r" (addr), "i" (-EFAULT)); \
+@@ -632,8 +632,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT));\
+@@ -653,8 +653,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT)); \
+@@ -673,8 +673,8 @@ do { \
+ "j\t3b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 4b\n\t" \
+- STR(PTR)"\t2b, 4b\n\t" \
++ STR(PTR_WD)"\t1b, 4b\n\t" \
++ STR(PTR_WD)"\t2b, 4b\n\t" \
+ ".previous" \
+ : "=r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT)); \
+@@ -703,10 +703,10 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT) \
+@@ -743,14 +743,14 @@ do { \
+ "j\t10b\n\t" \
+ ".previous\n\t" \
+ ".section\t__ex_table,\"a\"\n\t" \
+- STR(PTR)"\t1b, 11b\n\t" \
+- STR(PTR)"\t2b, 11b\n\t" \
+- STR(PTR)"\t3b, 11b\n\t" \
+- STR(PTR)"\t4b, 11b\n\t" \
+- STR(PTR)"\t5b, 11b\n\t" \
+- STR(PTR)"\t6b, 11b\n\t" \
+- STR(PTR)"\t7b, 11b\n\t" \
+- STR(PTR)"\t8b, 11b\n\t" \
++ STR(PTR_WD)"\t1b, 11b\n\t" \
++ STR(PTR_WD)"\t2b, 11b\n\t" \
++ STR(PTR_WD)"\t3b, 11b\n\t" \
++ STR(PTR_WD)"\t4b, 11b\n\t" \
++ STR(PTR_WD)"\t5b, 11b\n\t" \
++ STR(PTR_WD)"\t6b, 11b\n\t" \
++ STR(PTR_WD)"\t7b, 11b\n\t" \
++ STR(PTR_WD)"\t8b, 11b\n\t" \
+ ".previous" \
+ : "=&r" (res) \
+ : "r" (value), "r" (addr), "i" (-EFAULT) \
+diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
+index 630fcb4cb30e7..7c861e6a89529 100644
+--- a/arch/mips/kernel/cpu-probe.c
++++ b/arch/mips/kernel/cpu-probe.c
+@@ -1734,8 +1734,6 @@ static inline void decode_cpucfg(struct cpuinfo_mips *c)
+
+ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu)
+ {
+- decode_configs(c);
+-
+ /* All Loongson processors covered here define ExcCode 16 as GSExc. */
+ c->options |= MIPS_CPU_GSEXCEX;
+
+@@ -1796,6 +1794,8 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu)
+ panic("Unknown Loongson Processor ID!");
+ break;
+ }
++
++ decode_configs(c);
+ }
+ #else
+ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { }
+diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c
+index 90f1c3df1f0e4..b4f7d950c8468 100644
+--- a/arch/mips/kernel/mips-cm.c
++++ b/arch/mips/kernel/mips-cm.c
+@@ -221,8 +221,7 @@ static void mips_cm_probe_l2sync(void)
+ phys_addr_t addr;
+
+ /* L2-only sync was introduced with CM major revision 6 */
+- major_rev = (read_gcr_rev() & CM_GCR_REV_MAJOR) >>
+- __ffs(CM_GCR_REV_MAJOR);
++ major_rev = FIELD_GET(CM_GCR_REV_MAJOR, read_gcr_rev());
+ if (major_rev < 6)
+ return;
+
+@@ -306,13 +305,13 @@ void mips_cm_lock_other(unsigned int cluster, unsigned int core,
+ preempt_disable();
+
+ if (cm_rev >= CM_REV_CM3) {
+- val = core << __ffs(CM3_GCR_Cx_OTHER_CORE);
+- val |= vp << __ffs(CM3_GCR_Cx_OTHER_VP);
++ val = FIELD_PREP(CM3_GCR_Cx_OTHER_CORE, core) |
++ FIELD_PREP(CM3_GCR_Cx_OTHER_VP, vp);
+
+ if (cm_rev >= CM_REV_CM3_5) {
+ val |= CM_GCR_Cx_OTHER_CLUSTER_EN;
+- val |= cluster << __ffs(CM_GCR_Cx_OTHER_CLUSTER);
+- val |= block << __ffs(CM_GCR_Cx_OTHER_BLOCK);
++ val |= FIELD_PREP(CM_GCR_Cx_OTHER_CLUSTER, cluster);
++ val |= FIELD_PREP(CM_GCR_Cx_OTHER_BLOCK, block);
+ } else {
+ WARN_ON(cluster != 0);
+ WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL);
+@@ -342,7 +341,7 @@ void mips_cm_lock_other(unsigned int cluster, unsigned int core,
+ spin_lock_irqsave(&per_cpu(cm_core_lock, curr_core),
+ per_cpu(cm_core_lock_flags, curr_core));
+
+- val = core << __ffs(CM_GCR_Cx_OTHER_CORENUM);
++ val = FIELD_PREP(CM_GCR_Cx_OTHER_CORENUM, core);
+ }
+
+ write_gcr_cl_other(val);
+@@ -386,8 +385,8 @@ void mips_cm_error_report(void)
+ cm_other = read_gcr_error_mult();
+
+ if (revision < CM_REV_CM3) { /* CM2 */
+- cause = cm_error >> __ffs(CM_GCR_ERROR_CAUSE_ERRTYPE);
+- ocause = cm_other >> __ffs(CM_GCR_ERROR_MULT_ERR2ND);
++ cause = FIELD_GET(CM_GCR_ERROR_CAUSE_ERRTYPE, cm_error);
++ ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other);
+
+ if (!cause)
+ return;
+@@ -445,8 +444,8 @@ void mips_cm_error_report(void)
+ ulong core_id_bits, vp_id_bits, cmd_bits, cmd_group_bits;
+ ulong cm3_cca_bits, mcp_bits, cm3_tr_bits, sched_bit;
+
+- cause = cm_error >> __ffs64(CM3_GCR_ERROR_CAUSE_ERRTYPE);
+- ocause = cm_other >> __ffs(CM_GCR_ERROR_MULT_ERR2ND);
++ cause = FIELD_GET(CM3_GCR_ERROR_CAUSE_ERRTYPE, cm_error);
++ ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other);
+
+ if (!cause)
+ return;
+diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c
+index a39ec755e4c24..750fe569862b6 100644
+--- a/arch/mips/kernel/mips-r2-to-r6-emul.c
++++ b/arch/mips/kernel/mips-r2-to-r6-emul.c
+@@ -1258,10 +1258,10 @@ fpu_emul:
+ " j 10b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1333,10 +1333,10 @@ fpu_emul:
+ " j 10b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1404,10 +1404,10 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1474,10 +1474,10 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1589,14 +1589,14 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
+- STR(PTR) " 5b,8b\n"
+- STR(PTR) " 6b,8b\n"
+- STR(PTR) " 7b,8b\n"
+- STR(PTR) " 0b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
++ STR(PTR_WD) " 5b,8b\n"
++ STR(PTR_WD) " 6b,8b\n"
++ STR(PTR_WD) " 7b,8b\n"
++ STR(PTR_WD) " 0b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1708,14 +1708,14 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
+- STR(PTR) " 5b,8b\n"
+- STR(PTR) " 6b,8b\n"
+- STR(PTR) " 7b,8b\n"
+- STR(PTR) " 0b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
++ STR(PTR_WD) " 5b,8b\n"
++ STR(PTR_WD) " 6b,8b\n"
++ STR(PTR_WD) " 7b,8b\n"
++ STR(PTR_WD) " 0b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1827,14 +1827,14 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
+- STR(PTR) " 5b,8b\n"
+- STR(PTR) " 6b,8b\n"
+- STR(PTR) " 7b,8b\n"
+- STR(PTR) " 0b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
++ STR(PTR_WD) " 5b,8b\n"
++ STR(PTR_WD) " 6b,8b\n"
++ STR(PTR_WD) " 7b,8b\n"
++ STR(PTR_WD) " 0b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -1945,14 +1945,14 @@ fpu_emul:
+ " j 9b\n"
+ " .previous\n"
+ " .section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,8b\n"
+- STR(PTR) " 2b,8b\n"
+- STR(PTR) " 3b,8b\n"
+- STR(PTR) " 4b,8b\n"
+- STR(PTR) " 5b,8b\n"
+- STR(PTR) " 6b,8b\n"
+- STR(PTR) " 7b,8b\n"
+- STR(PTR) " 0b,8b\n"
++ STR(PTR_WD) " 1b,8b\n"
++ STR(PTR_WD) " 2b,8b\n"
++ STR(PTR_WD) " 3b,8b\n"
++ STR(PTR_WD) " 4b,8b\n"
++ STR(PTR_WD) " 5b,8b\n"
++ STR(PTR_WD) " 6b,8b\n"
++ STR(PTR_WD) " 7b,8b\n"
++ STR(PTR_WD) " 0b,8b\n"
+ " .previous\n"
+ " .set pop\n"
+ : "+&r"(rt), "=&r"(rs),
+@@ -2007,7 +2007,7 @@ fpu_emul:
+ "j 2b\n"
+ ".previous\n"
+ ".section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,3b\n"
++ STR(PTR_WD) " 1b,3b\n"
+ ".previous\n"
+ : "=&r"(res), "+&r"(err)
+ : "r"(vaddr), "i"(SIGSEGV)
+@@ -2065,7 +2065,7 @@ fpu_emul:
+ "j 2b\n"
+ ".previous\n"
+ ".section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,3b\n"
++ STR(PTR_WD) " 1b,3b\n"
+ ".previous\n"
+ : "+&r"(res), "+&r"(err)
+ : "r"(vaddr), "i"(SIGSEGV));
+@@ -2126,7 +2126,7 @@ fpu_emul:
+ "j 2b\n"
+ ".previous\n"
+ ".section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,3b\n"
++ STR(PTR_WD) " 1b,3b\n"
+ ".previous\n"
+ : "=&r"(res), "+&r"(err)
+ : "r"(vaddr), "i"(SIGSEGV)
+@@ -2189,7 +2189,7 @@ fpu_emul:
+ "j 2b\n"
+ ".previous\n"
+ ".section __ex_table,\"a\"\n"
+- STR(PTR) " 1b,3b\n"
++ STR(PTR_WD) " 1b,3b\n"
+ ".previous\n"
+ : "+&r"(res), "+&r"(err)
+ : "r"(vaddr), "i"(SIGSEGV));
+diff --git a/arch/mips/kernel/r2300_fpu.S b/arch/mips/kernel/r2300_fpu.S
+index 12e58053544fc..2748c55820c24 100644
+--- a/arch/mips/kernel/r2300_fpu.S
++++ b/arch/mips/kernel/r2300_fpu.S
+@@ -23,14 +23,14 @@
+ #define EX(a,b) \
+ 9: a,##b; \
+ .section __ex_table,"a"; \
+- PTR 9b,fault; \
++ PTR_WD 9b,fault; \
+ .previous
+
+ #define EX2(a,b) \
+ 9: a,##b; \
+ .section __ex_table,"a"; \
+- PTR 9b,bad_stack; \
+- PTR 9b+4,bad_stack; \
++ PTR_WD 9b,fault; \
++ PTR_WD 9b+4,fault; \
+ .previous
+
+ .set mips1
+diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S
+index b91e911064756..2e687c60bc4f1 100644
+--- a/arch/mips/kernel/r4k_fpu.S
++++ b/arch/mips/kernel/r4k_fpu.S
+@@ -31,7 +31,7 @@
+ .ex\@: \insn \reg, \src
+ .set pop
+ .section __ex_table,"a"
+- PTR .ex\@, fault
++ PTR_WD .ex\@, fault
+ .previous
+ .endm
+
+diff --git a/arch/mips/kernel/relocate_kernel.S b/arch/mips/kernel/relocate_kernel.S
+index f3c908abdbb80..cfde14b48fd8d 100644
+--- a/arch/mips/kernel/relocate_kernel.S
++++ b/arch/mips/kernel/relocate_kernel.S
+@@ -147,10 +147,10 @@ LEAF(kexec_smp_wait)
+
+ kexec_args:
+ EXPORT(kexec_args)
+-arg0: PTR 0x0
+-arg1: PTR 0x0
+-arg2: PTR 0x0
+-arg3: PTR 0x0
++arg0: PTR_WD 0x0
++arg1: PTR_WD 0x0
++arg2: PTR_WD 0x0
++arg3: PTR_WD 0x0
+ .size kexec_args,PTRSIZE*4
+
+ #ifdef CONFIG_SMP
+@@ -161,10 +161,10 @@ arg3: PTR 0x0
+ */
+ secondary_kexec_args:
+ EXPORT(secondary_kexec_args)
+-s_arg0: PTR 0x0
+-s_arg1: PTR 0x0
+-s_arg2: PTR 0x0
+-s_arg3: PTR 0x0
++s_arg0: PTR_WD 0x0
++s_arg1: PTR_WD 0x0
++s_arg2: PTR_WD 0x0
++s_arg3: PTR_WD 0x0
+ .size secondary_kexec_args,PTRSIZE*4
+ kexec_flag:
+ LONG 0x1
+@@ -173,17 +173,17 @@ kexec_flag:
+
+ kexec_start_address:
+ EXPORT(kexec_start_address)
+- PTR 0x0
++ PTR_WD 0x0
+ .size kexec_start_address, PTRSIZE
+
+ kexec_indirection_page:
+ EXPORT(kexec_indirection_page)
+- PTR 0
++ PTR_WD 0
+ .size kexec_indirection_page, PTRSIZE
+
+ relocate_new_kernel_end:
+
+ relocate_new_kernel_size:
+ EXPORT(relocate_new_kernel_size)
+- PTR relocate_new_kernel_end - relocate_new_kernel
++ PTR_WD relocate_new_kernel_end - relocate_new_kernel
+ .size relocate_new_kernel_size, PTRSIZE
+diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S
+index b1b2e106f7118..9bfce5f75f601 100644
+--- a/arch/mips/kernel/scall32-o32.S
++++ b/arch/mips/kernel/scall32-o32.S
+@@ -72,10 +72,10 @@ loads_done:
+ .set pop
+
+ .section __ex_table,"a"
+- PTR load_a4, bad_stack_a4
+- PTR load_a5, bad_stack_a5
+- PTR load_a6, bad_stack_a6
+- PTR load_a7, bad_stack_a7
++ PTR_WD load_a4, bad_stack_a4
++ PTR_WD load_a5, bad_stack_a5
++ PTR_WD load_a6, bad_stack_a6
++ PTR_WD load_a7, bad_stack_a7
+ .previous
+
+ lw t0, TI_FLAGS($28) # syscall tracing enabled?
+@@ -216,7 +216,7 @@ einval: li v0, -ENOSYS
+ #endif /* CONFIG_MIPS_MT_FPAFF */
+
+ #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native)
+-#define __SYSCALL(nr, entry) PTR entry
++#define __SYSCALL(nr, entry) PTR_WD entry
+ .align 2
+ .type sys_call_table, @object
+ EXPORT(sys_call_table)
+diff --git a/arch/mips/kernel/scall64-n32.S b/arch/mips/kernel/scall64-n32.S
+index f650c55a17dc5..97456b2ca7dc3 100644
+--- a/arch/mips/kernel/scall64-n32.S
++++ b/arch/mips/kernel/scall64-n32.S
+@@ -101,7 +101,7 @@ not_n32_scall:
+
+ END(handle_sysn32)
+
+-#define __SYSCALL(nr, entry) PTR entry
++#define __SYSCALL(nr, entry) PTR_WD entry
+ .type sysn32_call_table, @object
+ EXPORT(sysn32_call_table)
+ #include <asm/syscall_table_n32.h>
+diff --git a/arch/mips/kernel/scall64-n64.S b/arch/mips/kernel/scall64-n64.S
+index 5d7bfc65e4d0b..5f6ed4b4c3993 100644
+--- a/arch/mips/kernel/scall64-n64.S
++++ b/arch/mips/kernel/scall64-n64.S
+@@ -109,7 +109,7 @@ illegal_syscall:
+ j n64_syscall_exit
+ END(handle_sys64)
+
+-#define __SYSCALL(nr, entry) PTR entry
++#define __SYSCALL(nr, entry) PTR_WD entry
+ .align 3
+ .type sys_call_table, @object
+ EXPORT(sys_call_table)
+diff --git a/arch/mips/kernel/scall64-o32.S b/arch/mips/kernel/scall64-o32.S
+index cedc8bd888046..d3c2616cba226 100644
+--- a/arch/mips/kernel/scall64-o32.S
++++ b/arch/mips/kernel/scall64-o32.S
+@@ -73,10 +73,10 @@ load_a7: lw a7, 28(t0) # argument #8 from usp
+ loads_done:
+
+ .section __ex_table,"a"
+- PTR load_a4, bad_stack_a4
+- PTR load_a5, bad_stack_a5
+- PTR load_a6, bad_stack_a6
+- PTR load_a7, bad_stack_a7
++ PTR_WD load_a4, bad_stack_a4
++ PTR_WD load_a5, bad_stack_a5
++ PTR_WD load_a6, bad_stack_a6
++ PTR_WD load_a7, bad_stack_a7
+ .previous
+
+ li t1, _TIF_WORK_SYSCALL_ENTRY
+@@ -214,7 +214,7 @@ einval: li v0, -ENOSYS
+ END(sys32_syscall)
+
+ #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat)
+-#define __SYSCALL(nr, entry) PTR entry
++#define __SYSCALL(nr, entry) PTR_WD entry
+ .align 3
+ .type sys32_call_table,@object
+ EXPORT(sys32_call_table)
+diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
+index f979adfd4fc20..ef73ba1e0ec10 100644
+--- a/arch/mips/kernel/setup.c
++++ b/arch/mips/kernel/setup.c
+@@ -803,7 +803,7 @@ early_param("coherentio", setcoherentio);
+
+ static int __init setnocoherentio(char *str)
+ {
+- dma_default_coherent = true;
++ dma_default_coherent = false;
+ pr_info("Software DMA cache coherency (command line)\n");
+ return 0;
+ }
+diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
+index d542fb7af3ba2..1986d13094100 100644
+--- a/arch/mips/kernel/smp.c
++++ b/arch/mips/kernel/smp.c
+@@ -351,6 +351,9 @@ asmlinkage void start_secondary(void)
+ cpu = smp_processor_id();
+ cpu_data[cpu].udelay_val = loops_per_jiffy;
+
++ set_cpu_sibling_map(cpu);
++ set_cpu_core_map(cpu);
++
+ cpumask_set_cpu(cpu, &cpu_coherent_mask);
+ notify_cpu_starting(cpu);
+
+@@ -362,9 +365,6 @@ asmlinkage void start_secondary(void)
+ /* The CPU is running and counters synchronised, now mark it online */
+ set_cpu_online(cpu, true);
+
+- set_cpu_sibling_map(cpu);
+- set_cpu_core_map(cpu);
+-
+ calculate_cpu_foreign_map();
+
+ /*
+diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c
+index 2afa3eef486a9..ae93a607ddf7e 100644
+--- a/arch/mips/kernel/syscall.c
++++ b/arch/mips/kernel/syscall.c
+@@ -122,8 +122,8 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
+ " j 3b \n"
+ " .previous \n"
+ " .section __ex_table,\"a\" \n"
+- " "STR(PTR)" 1b, 4b \n"
+- " "STR(PTR)" 2b, 4b \n"
++ " "STR(PTR_WD)" 1b, 4b \n"
++ " "STR(PTR_WD)" 2b, 4b \n"
+ " .previous \n"
+ " .set pop \n"
+ : [old] "=&r" (old),
+@@ -152,8 +152,8 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
+ " j 3b \n"
+ " .previous \n"
+ " .section __ex_table,\"a\" \n"
+- " "STR(PTR)" 1b, 5b \n"
+- " "STR(PTR)" 2b, 5b \n"
++ " "STR(PTR_WD)" 1b, 5b \n"
++ " "STR(PTR_WD)" 2b, 5b \n"
+ " .previous \n"
+ " .set pop \n"
+ : [old] "=&r" (old),
+@@ -240,12 +240,3 @@ SYSCALL_DEFINE3(cachectl, char *, addr, int, nbytes, int, op)
+ {
+ return -ENOSYS;
+ }
+-
+-/*
+- * If we ever come here the user sp is bad. Zap the process right away.
+- * Due to the bad stack signaling wouldn't work.
+- */
+-asmlinkage void bad_stack(void)
+-{
+- do_exit(SIGSEGV);
+-}
+diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c
+index caa01457dce60..ed339d7979f3f 100644
+--- a/arch/mips/kernel/time.c
++++ b/arch/mips/kernel/time.c
+@@ -141,15 +141,10 @@ static __init int cpu_has_mfc0_count_bug(void)
+ case CPU_R4400MC:
+ /*
+ * The published errata for the R4400 up to 3.0 say the CPU
+- * has the mfc0 from count bug.
++ * has the mfc0 from count bug. This seems the last version
++ * produced.
+ */
+- if ((current_cpu_data.processor_id & 0xff) <= 0x30)
+- return 1;
+-
+- /*
+- * we assume newer revisions are ok
+- */
+- return 0;
++ return 1;
+ }
+
+ return 0;
+diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
+index 6f07362de5cec..edd93430b954a 100644
+--- a/arch/mips/kernel/traps.c
++++ b/arch/mips/kernel/traps.c
+@@ -2085,19 +2085,19 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs)
+ * If no shadow set is selected then use the default handler
+ * that does normal register saving and standard interrupt exit
+ */
+- extern char except_vec_vi, except_vec_vi_lui;
+- extern char except_vec_vi_ori, except_vec_vi_end;
+- extern char rollback_except_vec_vi;
+- char *vec_start = using_rollback_handler() ?
+- &rollback_except_vec_vi : &except_vec_vi;
++ extern const u8 except_vec_vi[], except_vec_vi_lui[];
++ extern const u8 except_vec_vi_ori[], except_vec_vi_end[];
++ extern const u8 rollback_except_vec_vi[];
++ const u8 *vec_start = using_rollback_handler() ?
++ rollback_except_vec_vi : except_vec_vi;
+ #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN)
+- const int lui_offset = &except_vec_vi_lui - vec_start + 2;
+- const int ori_offset = &except_vec_vi_ori - vec_start + 2;
++ const int lui_offset = except_vec_vi_lui - vec_start + 2;
++ const int ori_offset = except_vec_vi_ori - vec_start + 2;
+ #else
+- const int lui_offset = &except_vec_vi_lui - vec_start;
+- const int ori_offset = &except_vec_vi_ori - vec_start;
++ const int lui_offset = except_vec_vi_lui - vec_start;
++ const int ori_offset = except_vec_vi_ori - vec_start;
+ #endif
+- const int handler_len = &except_vec_vi_end - vec_start;
++ const int handler_len = except_vec_vi_end - vec_start;
+
+ if (handler_len > VECTORSPACING) {
+ /*
+@@ -2305,7 +2305,7 @@ void per_cpu_trap_init(bool is_boot_cpu)
+ }
+
+ /* Install CPU exception handler */
+-void set_handler(unsigned long offset, void *addr, unsigned long size)
++void set_handler(unsigned long offset, const void *addr, unsigned long size)
+ {
+ #ifdef CONFIG_CPU_MICROMIPS
+ memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size);
+diff --git a/arch/mips/lantiq/clk.c b/arch/mips/lantiq/clk.c
+index dd819e31fcbbf..7a623684d9b5e 100644
+--- a/arch/mips/lantiq/clk.c
++++ b/arch/mips/lantiq/clk.c
+@@ -158,6 +158,18 @@ void clk_deactivate(struct clk *clk)
+ }
+ EXPORT_SYMBOL(clk_deactivate);
+
++struct clk *clk_get_parent(struct clk *clk)
++{
++ return NULL;
++}
++EXPORT_SYMBOL(clk_get_parent);
++
++int clk_set_parent(struct clk *clk, struct clk *parent)
++{
++ return 0;
++}
++EXPORT_SYMBOL(clk_set_parent);
++
+ static inline u32 get_counter_resolution(void)
+ {
+ u32 res;
+diff --git a/arch/mips/lantiq/falcon/sysctrl.c b/arch/mips/lantiq/falcon/sysctrl.c
+index 42222f849bd25..446a2536999bf 100644
+--- a/arch/mips/lantiq/falcon/sysctrl.c
++++ b/arch/mips/lantiq/falcon/sysctrl.c
+@@ -167,6 +167,8 @@ static inline void clkdev_add_sys(const char *dev, unsigned int module,
+ {
+ struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL);
+
++ if (!clk)
++ return;
+ clk->cl.dev_id = dev;
+ clk->cl.con_id = NULL;
+ clk->cl.clk = clk;
+diff --git a/arch/mips/lantiq/xway/dma.c b/arch/mips/lantiq/xway/dma.c
+index 63dccb2ed08b2..53fcc672a2944 100644
+--- a/arch/mips/lantiq/xway/dma.c
++++ b/arch/mips/lantiq/xway/dma.c
+@@ -11,6 +11,7 @@
+ #include <linux/export.h>
+ #include <linux/spinlock.h>
+ #include <linux/clk.h>
++#include <linux/delay.h>
+ #include <linux/err.h>
+ #include <linux/of.h>
+
+@@ -30,6 +31,7 @@
+ #define LTQ_DMA_PCTRL 0x44
+ #define LTQ_DMA_IRNEN 0xf4
+
++#define DMA_ID_CHNR GENMASK(26, 20) /* channel number */
+ #define DMA_DESCPT BIT(3) /* descriptor complete irq */
+ #define DMA_TX BIT(8) /* TX channel direction */
+ #define DMA_CHAN_ON BIT(0) /* channel on / off bit */
+@@ -39,8 +41,11 @@
+ #define DMA_IRQ_ACK 0x7e /* IRQ status register */
+ #define DMA_POLL BIT(31) /* turn on channel polling */
+ #define DMA_CLK_DIV4 BIT(6) /* polling clock divider */
+-#define DMA_2W_BURST BIT(1) /* 2 word burst length */
+-#define DMA_MAX_CHANNEL 20 /* the soc has 20 channels */
++#define DMA_PCTRL_2W_BURST 0x1 /* 2 word burst length */
++#define DMA_PCTRL_4W_BURST 0x2 /* 4 word burst length */
++#define DMA_PCTRL_8W_BURST 0x3 /* 8 word burst length */
++#define DMA_TX_BURST_SHIFT 4 /* tx burst shift */
++#define DMA_RX_BURST_SHIFT 2 /* rx burst shift */
+ #define DMA_ETOP_ENDIANNESS (0xf << 8) /* endianness swap etop channels */
+ #define DMA_WEIGHT (BIT(17) | BIT(16)) /* default channel wheight */
+
+@@ -191,7 +196,8 @@ ltq_dma_init_port(int p)
+ break;
+
+ case DMA_PORT_DEU:
+- ltq_dma_w32((DMA_2W_BURST << 4) | (DMA_2W_BURST << 2),
++ ltq_dma_w32((DMA_PCTRL_2W_BURST << DMA_TX_BURST_SHIFT) |
++ (DMA_PCTRL_2W_BURST << DMA_RX_BURST_SHIFT),
+ LTQ_DMA_PCTRL);
+ break;
+
+@@ -206,7 +212,7 @@ ltq_dma_init(struct platform_device *pdev)
+ {
+ struct clk *clk;
+ struct resource *res;
+- unsigned id;
++ unsigned int id, nchannels;
+ int i;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+@@ -222,21 +228,24 @@ ltq_dma_init(struct platform_device *pdev)
+ clk_enable(clk);
+ ltq_dma_w32_mask(0, DMA_RESET, LTQ_DMA_CTRL);
+
++ usleep_range(1, 10);
++
+ /* disable all interrupts */
+ ltq_dma_w32(0, LTQ_DMA_IRNEN);
+
+ /* reset/configure each channel */
+- for (i = 0; i < DMA_MAX_CHANNEL; i++) {
++ id = ltq_dma_r32(LTQ_DMA_ID);
++ nchannels = ((id & DMA_ID_CHNR) >> 20);
++ for (i = 0; i < nchannels; i++) {
+ ltq_dma_w32(i, LTQ_DMA_CS);
+ ltq_dma_w32(DMA_CHAN_RST, LTQ_DMA_CCTRL);
+ ltq_dma_w32(DMA_POLL | DMA_CLK_DIV4, LTQ_DMA_CPOLL);
+ ltq_dma_w32_mask(DMA_CHAN_ON, 0, LTQ_DMA_CCTRL);
+ }
+
+- id = ltq_dma_r32(LTQ_DMA_ID);
+ dev_info(&pdev->dev,
+ "Init done - hw rev: %X, ports: %d, channels: %d\n",
+- id & 0x1f, (id >> 16) & 0xf, id >> 20);
++ id & 0x1f, (id >> 16) & 0xf, nchannels);
+
+ return 0;
+ }
+diff --git a/arch/mips/lantiq/xway/gptu.c b/arch/mips/lantiq/xway/gptu.c
+index 3d5683e75cf1e..200fe9ff641d6 100644
+--- a/arch/mips/lantiq/xway/gptu.c
++++ b/arch/mips/lantiq/xway/gptu.c
+@@ -122,6 +122,8 @@ static inline void clkdev_add_gptu(struct device *dev, const char *con,
+ {
+ struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL);
+
++ if (!clk)
++ return;
+ clk->cl.dev_id = dev_name(dev);
+ clk->cl.con_id = con;
+ clk->cl.clk = clk;
+diff --git a/arch/mips/lantiq/xway/sysctrl.c b/arch/mips/lantiq/xway/sysctrl.c
+index 917fac1636b71..084f6caba5f23 100644
+--- a/arch/mips/lantiq/xway/sysctrl.c
++++ b/arch/mips/lantiq/xway/sysctrl.c
+@@ -315,6 +315,8 @@ static void clkdev_add_pmu(const char *dev, const char *con, bool deactivate,
+ {
+ struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL);
+
++ if (!clk)
++ return;
+ clk->cl.dev_id = dev;
+ clk->cl.con_id = con;
+ clk->cl.clk = clk;
+@@ -338,6 +340,8 @@ static void clkdev_add_cgu(const char *dev, const char *con,
+ {
+ struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL);
+
++ if (!clk)
++ return;
+ clk->cl.dev_id = dev;
+ clk->cl.con_id = con;
+ clk->cl.clk = clk;
+@@ -356,24 +360,28 @@ static void clkdev_add_pci(void)
+ struct clk *clk_ext = kzalloc(sizeof(struct clk), GFP_KERNEL);
+
+ /* main pci clock */
+- clk->cl.dev_id = "17000000.pci";
+- clk->cl.con_id = NULL;
+- clk->cl.clk = clk;
+- clk->rate = CLOCK_33M;
+- clk->rates = valid_pci_rates;
+- clk->enable = pci_enable;
+- clk->disable = pmu_disable;
+- clk->module = 0;
+- clk->bits = PMU_PCI;
+- clkdev_add(&clk->cl);
++ if (clk) {
++ clk->cl.dev_id = "17000000.pci";
++ clk->cl.con_id = NULL;
++ clk->cl.clk = clk;
++ clk->rate = CLOCK_33M;
++ clk->rates = valid_pci_rates;
++ clk->enable = pci_enable;
++ clk->disable = pmu_disable;
++ clk->module = 0;
++ clk->bits = PMU_PCI;
++ clkdev_add(&clk->cl);
++ }
+
+ /* use internal/external bus clock */
+- clk_ext->cl.dev_id = "17000000.pci";
+- clk_ext->cl.con_id = "external";
+- clk_ext->cl.clk = clk_ext;
+- clk_ext->enable = pci_ext_enable;
+- clk_ext->disable = pci_ext_disable;
+- clkdev_add(&clk_ext->cl);
++ if (clk_ext) {
++ clk_ext->cl.dev_id = "17000000.pci";
++ clk_ext->cl.con_id = "external";
++ clk_ext->cl.clk = clk_ext;
++ clk_ext->enable = pci_ext_enable;
++ clk_ext->disable = pci_ext_disable;
++ clkdev_add(&clk_ext->cl);
++ }
+ }
+
+ /* xway socs can generate clocks on gpio pins */
+@@ -393,9 +401,15 @@ static void clkdev_add_clkout(void)
+ char *name;
+
+ name = kzalloc(sizeof("clkout0"), GFP_KERNEL);
++ if (!name)
++ continue;
+ sprintf(name, "clkout%d", i);
+
+ clk = kzalloc(sizeof(struct clk), GFP_KERNEL);
++ if (!clk) {
++ kfree(name);
++ continue;
++ }
+ clk->cl.dev_id = "1f103000.cgu";
+ clk->cl.con_id = name;
+ clk->cl.clk = clk;
+diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
+index a46db08071953..7767137c3e49a 100644
+--- a/arch/mips/lib/csum_partial.S
++++ b/arch/mips/lib/csum_partial.S
+@@ -347,7 +347,7 @@ EXPORT_SYMBOL(csum_partial)
+ .if \mode == LEGACY_MODE; \
+ 9: insn reg, addr; \
+ .section __ex_table,"a"; \
+- PTR 9b, .L_exc; \
++ PTR_WD 9b, .L_exc; \
+ .previous; \
+ /* This is enabled in EVA mode */ \
+ .else; \
+@@ -356,7 +356,7 @@ EXPORT_SYMBOL(csum_partial)
+ ((\to == USEROP) && (type == ST_INSN)); \
+ 9: __BUILD_EVA_INSN(insn##e, reg, addr); \
+ .section __ex_table,"a"; \
+- PTR 9b, .L_exc; \
++ PTR_WD 9b, .L_exc; \
+ .previous; \
+ .else; \
+ /* EVA without exception */ \
+diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
+index 277c32296636d..18a43f2e29c81 100644
+--- a/arch/mips/lib/memcpy.S
++++ b/arch/mips/lib/memcpy.S
+@@ -116,7 +116,7 @@
+ .if \mode == LEGACY_MODE; \
+ 9: insn reg, addr; \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous; \
+ /* This is assembled in EVA mode */ \
+ .else; \
+@@ -125,7 +125,7 @@
+ ((\to == USEROP) && (type == ST_INSN)); \
+ 9: __BUILD_EVA_INSN(insn##e, reg, addr); \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous; \
+ .else; \
+ /* \
+diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S
+index b0baa3c79fad0..0b342bae9a98c 100644
+--- a/arch/mips/lib/memset.S
++++ b/arch/mips/lib/memset.S
+@@ -52,7 +52,7 @@
+ 9: ___BUILD_EVA_INSN(insn, reg, addr); \
+ .endif; \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous
+
+ .macro f_fill64 dst, offset, val, fixup, mode
+diff --git a/arch/mips/lib/strncpy_user.S b/arch/mips/lib/strncpy_user.S
+index 556acf684d7be..13aaa9927ad12 100644
+--- a/arch/mips/lib/strncpy_user.S
++++ b/arch/mips/lib/strncpy_user.S
+@@ -15,7 +15,7 @@
+ #define EX(insn,reg,addr,handler) \
+ 9: insn reg, addr; \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous
+
+ /*
+@@ -59,7 +59,7 @@ LEAF(__strncpy_from_user_asm)
+ jr ra
+
+ .section __ex_table,"a"
+- PTR 1b, .Lfault
++ PTR_WD 1b, .Lfault
+ .previous
+
+ EXPORT_SYMBOL(__strncpy_from_user_asm)
+diff --git a/arch/mips/lib/strnlen_user.S b/arch/mips/lib/strnlen_user.S
+index 92b63f20ec05f..6de31b616f9c1 100644
+--- a/arch/mips/lib/strnlen_user.S
++++ b/arch/mips/lib/strnlen_user.S
+@@ -14,7 +14,7 @@
+ #define EX(insn,reg,addr,handler) \
+ 9: insn reg, addr; \
+ .section __ex_table,"a"; \
+- PTR 9b, handler; \
++ PTR_WD 9b, handler; \
+ .previous
+
+ /*
+diff --git a/arch/mips/loongson64/vbios_quirk.c b/arch/mips/loongson64/vbios_quirk.c
+index 9a29e94d3db1d..3115d4de982c5 100644
+--- a/arch/mips/loongson64/vbios_quirk.c
++++ b/arch/mips/loongson64/vbios_quirk.c
+@@ -3,7 +3,7 @@
+ #include <linux/pci.h>
+ #include <loongson.h>
+
+-static void pci_fixup_radeon(struct pci_dev *pdev)
++static void pci_fixup_video(struct pci_dev *pdev)
+ {
+ struct resource *res = &pdev->resource[PCI_ROM_RESOURCE];
+
+@@ -22,8 +22,7 @@ static void pci_fixup_radeon(struct pci_dev *pdev)
+ res->flags = IORESOURCE_MEM | IORESOURCE_ROM_SHADOW |
+ IORESOURCE_PCI_FIXED;
+
+- dev_info(&pdev->dev, "BAR %d: assigned %pR for Radeon ROM\n",
+- PCI_ROM_RESOURCE, res);
++ dev_info(&pdev->dev, "Video device with shadowed ROM at %pR\n", res);
+ }
+-DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_ATI, 0x9615,
+- PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_radeon);
++DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_ATI, 0x9615,
++ PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_video);
+diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
+index 9adad24c2e65e..046d51a454afc 100644
+--- a/arch/mips/mm/tlbex.c
++++ b/arch/mips/mm/tlbex.c
+@@ -2167,16 +2167,14 @@ static void build_r4000_tlb_load_handler(void)
+ uasm_i_tlbr(&p);
+
+ switch (current_cpu_type()) {
+- default:
+- if (cpu_has_mips_r2_exec_hazard) {
+- uasm_i_ehb(&p);
+- fallthrough;
+-
+ case CPU_CAVIUM_OCTEON:
+ case CPU_CAVIUM_OCTEON_PLUS:
+ case CPU_CAVIUM_OCTEON2:
+- break;
+- }
++ break;
++ default:
++ if (cpu_has_mips_r2_exec_hazard)
++ uasm_i_ehb(&p);
++ break;
+ }
+
+ /* Examine entrylo 0 or 1 based on ptr. */
+@@ -2243,15 +2241,14 @@ static void build_r4000_tlb_load_handler(void)
+ uasm_i_tlbr(&p);
+
+ switch (current_cpu_type()) {
+- default:
+- if (cpu_has_mips_r2_exec_hazard) {
+- uasm_i_ehb(&p);
+-
+ case CPU_CAVIUM_OCTEON:
+ case CPU_CAVIUM_OCTEON_PLUS:
+ case CPU_CAVIUM_OCTEON2:
+- break;
+- }
++ break;
++ default:
++ if (cpu_has_mips_r2_exec_hazard)
++ uasm_i_ehb(&p);
++ break;
+ }
+
+ /* Examine entrylo 0 or 1 based on ptr. */
+diff --git a/arch/mips/ralink/ill_acc.c b/arch/mips/ralink/ill_acc.c
+index bdf53807d7c2b..bea857c9da8b7 100644
+--- a/arch/mips/ralink/ill_acc.c
++++ b/arch/mips/ralink/ill_acc.c
+@@ -61,6 +61,7 @@ static int __init ill_acc_of_setup(void)
+ pdev = of_find_device_by_node(np);
+ if (!pdev) {
+ pr_err("%pOFn: failed to lookup pdev\n", np);
++ of_node_put(np);
+ return -EINVAL;
+ }
+
+diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c
+index bd71f5b142383..4c83786612193 100644
+--- a/arch/mips/ralink/mt7621.c
++++ b/arch/mips/ralink/mt7621.c
+@@ -20,31 +20,41 @@
+
+ #include "common.h"
+
+-static void *detect_magic __initdata = detect_memory_region;
++#define MT7621_MEM_TEST_PATTERN 0xaa5555aa
++
++static u32 detect_magic __initdata;
+
+ phys_addr_t mips_cpc_default_phys_base(void)
+ {
+ panic("Cannot detect cpc address");
+ }
+
++static bool __init mt7621_addr_wraparound_test(phys_addr_t size)
++{
++ void *dm = (void *)KSEG1ADDR(&detect_magic);
++
++ if (CPHYSADDR(dm + size) >= MT7621_LOWMEM_MAX_SIZE)
++ return true;
++ __raw_writel(MT7621_MEM_TEST_PATTERN, dm);
++ if (__raw_readl(dm) != __raw_readl(dm + size))
++ return false;
++ __raw_writel(~MT7621_MEM_TEST_PATTERN, dm);
++ return __raw_readl(dm) == __raw_readl(dm + size);
++}
++
+ static void __init mt7621_memory_detect(void)
+ {
+- void *dm = &detect_magic;
+ phys_addr_t size;
+
+- for (size = 32 * SZ_1M; size < 256 * SZ_1M; size <<= 1) {
+- if (!__builtin_memcmp(dm, dm + size, sizeof(detect_magic)))
+- break;
++ for (size = 32 * SZ_1M; size <= 256 * SZ_1M; size <<= 1) {
++ if (mt7621_addr_wraparound_test(size)) {
++ memblock_add(MT7621_LOWMEM_BASE, size);
++ return;
++ }
+ }
+
+- if ((size == 256 * SZ_1M) &&
+- (CPHYSADDR(dm + size) < MT7621_LOWMEM_MAX_SIZE) &&
+- __builtin_memcmp(dm, dm + size, sizeof(detect_magic))) {
+- memblock_add(MT7621_LOWMEM_BASE, MT7621_LOWMEM_MAX_SIZE);
+- memblock_add(MT7621_HIGHMEM_BASE, MT7621_HIGHMEM_SIZE);
+- } else {
+- memblock_add(MT7621_LOWMEM_BASE, size);
+- }
++ memblock_add(MT7621_LOWMEM_BASE, MT7621_LOWMEM_MAX_SIZE);
++ memblock_add(MT7621_HIGHMEM_BASE, MT7621_HIGHMEM_SIZE);
+ }
+
+ void __init ralink_of_remap(void)
+diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c
+index 04684990e28ef..b7f6f782d9a13 100644
+--- a/arch/mips/rb532/devices.c
++++ b/arch/mips/rb532/devices.c
+@@ -301,11 +301,9 @@ static int __init plat_setup_devices(void)
+ static int __init setup_kmac(char *s)
+ {
+ printk(KERN_INFO "korina mac = %s\n", s);
+- if (!mac_pton(s, korina_dev0_data.mac)) {
++ if (!mac_pton(s, korina_dev0_data.mac))
+ printk(KERN_ERR "Invalid mac\n");
+- return -EINVAL;
+- }
+- return 0;
++ return 1;
+ }
+
+ __setup("kmac=", setup_kmac);
+diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c
+index 240bb68ec2478..ff3ba7e778901 100644
+--- a/arch/mips/sni/time.c
++++ b/arch/mips/sni/time.c
+@@ -18,14 +18,14 @@ static int a20r_set_periodic(struct clock_event_device *evt)
+ {
+ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0x34;
+ wmb();
+- *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV;
++ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV & 0xff;
+ wmb();
+ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV >> 8;
+ wmb();
+
+ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0xb4;
+ wmb();
+- *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV;
++ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV & 0xff;
+ wmb();
+ *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV >> 8;
+ wmb();
+diff --git a/arch/nds32/include/asm/uaccess.h b/arch/nds32/include/asm/uaccess.h
+index d4cbf069dc224..37a40981deb3b 100644
+--- a/arch/nds32/include/asm/uaccess.h
++++ b/arch/nds32/include/asm/uaccess.h
+@@ -70,9 +70,7 @@ static inline void set_fs(mm_segment_t fs)
+ * versions are void (ie, don't return a value as such).
+ */
+
+-#define get_user __get_user \
+-
+-#define __get_user(x, ptr) \
++#define get_user(x, ptr) \
+ ({ \
+ long __gu_err = 0; \
+ __get_user_check((x), (ptr), __gu_err); \
+@@ -85,6 +83,14 @@ static inline void set_fs(mm_segment_t fs)
+ (void)0; \
+ })
+
++#define __get_user(x, ptr) \
++({ \
++ long __gu_err = 0; \
++ const __typeof__(*(ptr)) __user *__p = (ptr); \
++ __get_user_err((x), __p, (__gu_err)); \
++ __gu_err; \
++})
++
+ #define __get_user_check(x, ptr, err) \
+ ({ \
+ const __typeof__(*(ptr)) __user *__p = (ptr); \
+@@ -165,12 +171,18 @@ do { \
+ : "r"(addr), "i"(-EFAULT) \
+ : "cc")
+
+-#define put_user __put_user \
++#define put_user(x, ptr) \
++({ \
++ long __pu_err = 0; \
++ __put_user_check((x), (ptr), __pu_err); \
++ __pu_err; \
++})
+
+ #define __put_user(x, ptr) \
+ ({ \
+ long __pu_err = 0; \
+- __put_user_err((x), (ptr), __pu_err); \
++ __typeof__(*(ptr)) __user *__p = (ptr); \
++ __put_user_err((x), __p, __pu_err); \
+ __pu_err; \
+ })
+
+diff --git a/arch/nds32/kernel/perf_event_cpu.c b/arch/nds32/kernel/perf_event_cpu.c
+index 0ce6f9f307e6a..f387919607813 100644
+--- a/arch/nds32/kernel/perf_event_cpu.c
++++ b/arch/nds32/kernel/perf_event_cpu.c
+@@ -1363,6 +1363,7 @@ void
+ perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ unsigned long fp = 0;
+ unsigned long gp = 0;
+ unsigned long lp = 0;
+@@ -1371,7 +1372,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+
+ leaf_fp = 0;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -1479,9 +1480,10 @@ void
+ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct stackframe fr;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+@@ -1493,20 +1495,23 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+
+ unsigned long perf_instruction_pointer(struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
+ /* However, NDS32 does not support virtualization */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+- return perf_guest_cbs->get_guest_ip();
++ if (guest_cbs && guest_cbs->is_in_guest())
++ return guest_cbs->get_guest_ip();
+
+ return instruction_pointer(regs);
+ }
+
+ unsigned long perf_misc_flags(struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ int misc = 0;
+
+ /* However, NDS32 does not support virtualization */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+- if (perf_guest_cbs->is_user_mode())
++ if (guest_cbs && guest_cbs->is_in_guest()) {
++ if (guest_cbs->is_user_mode())
+ misc |= PERF_RECORD_MISC_GUEST_USER;
+ else
+ misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+diff --git a/arch/nios2/include/asm/timex.h b/arch/nios2/include/asm/timex.h
+index a769f871b28d9..40a1adc9bd03e 100644
+--- a/arch/nios2/include/asm/timex.h
++++ b/arch/nios2/include/asm/timex.h
+@@ -8,5 +8,8 @@
+ typedef unsigned long cycles_t;
+
+ extern cycles_t get_cycles(void);
++#define get_cycles get_cycles
++
++#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
+
+ #endif
+diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h
+index ba9340e96fd4c..ca9285a915efa 100644
+--- a/arch/nios2/include/asm/uaccess.h
++++ b/arch/nios2/include/asm/uaccess.h
+@@ -88,6 +88,7 @@ extern __must_check long strnlen_user(const char __user *s, long n);
+ /* Optimized macros */
+ #define __get_user_asm(val, insn, addr, err) \
+ { \
++ unsigned long __gu_val; \
+ __asm__ __volatile__( \
+ " movi %0, %3\n" \
+ "1: " insn " %1, 0(%2)\n" \
+@@ -96,14 +97,20 @@ extern __must_check long strnlen_user(const char __user *s, long n);
+ " .section __ex_table,\"a\"\n" \
+ " .word 1b, 2b\n" \
+ " .previous" \
+- : "=&r" (err), "=r" (val) \
++ : "=&r" (err), "=r" (__gu_val) \
+ : "r" (addr), "i" (-EFAULT)); \
++ val = (__force __typeof__(*(addr)))__gu_val; \
+ }
+
+-#define __get_user_unknown(val, size, ptr, err) do { \
++extern void __get_user_unknown(void);
++
++#define __get_user_8(val, ptr, err) do { \
++ u64 __val = 0; \
+ err = 0; \
+- if (__copy_from_user(&(val), ptr, size)) { \
++ if (raw_copy_from_user(&(__val), ptr, sizeof(val))) { \
+ err = -EFAULT; \
++ } else { \
++ val = (typeof(val))(typeof((val) - (val)))__val; \
+ } \
+ } while (0)
+
+@@ -119,8 +126,11 @@ do { \
+ case 4: \
+ __get_user_asm(val, "ldw", ptr, err); \
+ break; \
++ case 8: \
++ __get_user_8(val, ptr, err); \
++ break; \
+ default: \
+- __get_user_unknown(val, size, ptr, err); \
++ __get_user_unknown(); \
+ break; \
+ } \
+ } while (0)
+@@ -129,9 +139,7 @@ do { \
+ ({ \
+ long __gu_err = -EFAULT; \
+ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \
+- unsigned long __gu_val = 0; \
+- __get_user_common(__gu_val, sizeof(*(ptr)), __gu_ptr, __gu_err);\
+- (x) = (__force __typeof__(x))__gu_val; \
++ __get_user_common(x, sizeof(*(ptr)), __gu_ptr, __gu_err); \
+ __gu_err; \
+ })
+
+@@ -139,11 +147,9 @@ do { \
+ ({ \
+ long __gu_err = -EFAULT; \
+ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \
+- unsigned long __gu_val = 0; \
+ if (access_ok( __gu_ptr, sizeof(*__gu_ptr))) \
+- __get_user_common(__gu_val, sizeof(*__gu_ptr), \
++ __get_user_common(x, sizeof(*__gu_ptr), \
+ __gu_ptr, __gu_err); \
+- (x) = (__force __typeof__(x))__gu_val; \
+ __gu_err; \
+ })
+
+diff --git a/arch/nios2/kernel/signal.c b/arch/nios2/kernel/signal.c
+index 2009ae2d3c3bb..386e46443b605 100644
+--- a/arch/nios2/kernel/signal.c
++++ b/arch/nios2/kernel/signal.c
+@@ -36,10 +36,10 @@ struct rt_sigframe {
+
+ static inline int rt_restore_ucontext(struct pt_regs *regs,
+ struct switch_stack *sw,
+- struct ucontext *uc, int *pr2)
++ struct ucontext __user *uc, int *pr2)
+ {
+ int temp;
+- unsigned long *gregs = uc->uc_mcontext.gregs;
++ unsigned long __user *gregs = uc->uc_mcontext.gregs;
+ int err;
+
+ /* Always make any pending restarted system calls return -EINTR */
+@@ -102,10 +102,11 @@ asmlinkage int do_rt_sigreturn(struct switch_stack *sw)
+ {
+ struct pt_regs *regs = (struct pt_regs *)(sw + 1);
+ /* Verify, can we follow the stack back */
+- struct rt_sigframe *frame = (struct rt_sigframe *) regs->sp;
++ struct rt_sigframe __user *frame;
+ sigset_t set;
+ int rval;
+
++ frame = (struct rt_sigframe __user *) regs->sp;
+ if (!access_ok(frame, sizeof(*frame)))
+ goto badframe;
+
+@@ -124,10 +125,10 @@ badframe:
+ return 0;
+ }
+
+-static inline int rt_setup_ucontext(struct ucontext *uc, struct pt_regs *regs)
++static inline int rt_setup_ucontext(struct ucontext __user *uc, struct pt_regs *regs)
+ {
+ struct switch_stack *sw = (struct switch_stack *)regs - 1;
+- unsigned long *gregs = uc->uc_mcontext.gregs;
++ unsigned long __user *gregs = uc->uc_mcontext.gregs;
+ int err = 0;
+
+ err |= __put_user(MCONTEXT_VERSION, &uc->uc_mcontext.version);
+@@ -162,8 +163,9 @@ static inline int rt_setup_ucontext(struct ucontext *uc, struct pt_regs *regs)
+ return err;
+ }
+
+-static inline void *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
+- size_t frame_size)
++static inline void __user *get_sigframe(struct ksignal *ksig,
++ struct pt_regs *regs,
++ size_t frame_size)
+ {
+ unsigned long usp;
+
+@@ -174,13 +176,13 @@ static inline void *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
+ usp = sigsp(usp, ksig);
+
+ /* Verify, is it 32 or 64 bit aligned */
+- return (void *)((usp - frame_size) & -8UL);
++ return (void __user *)((usp - frame_size) & -8UL);
+ }
+
+ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+ struct pt_regs *regs)
+ {
+- struct rt_sigframe *frame;
++ struct rt_sigframe __user *frame;
+ int err = 0;
+
+ frame = get_sigframe(ksig, regs, sizeof(*frame));
+diff --git a/arch/openrisc/include/asm/syscalls.h b/arch/openrisc/include/asm/syscalls.h
+index 3a7eeae6f56a8..aa1c7e98722e3 100644
+--- a/arch/openrisc/include/asm/syscalls.h
++++ b/arch/openrisc/include/asm/syscalls.h
+@@ -22,9 +22,11 @@ asmlinkage long sys_or1k_atomic(unsigned long type, unsigned long *v1,
+
+ asmlinkage long __sys_clone(unsigned long clone_flags, unsigned long newsp,
+ void __user *parent_tid, void __user *child_tid, int tls);
++asmlinkage long __sys_clone3(struct clone_args __user *uargs, size_t size);
+ asmlinkage long __sys_fork(void);
+
+ #define sys_clone __sys_clone
++#define sys_clone3 __sys_clone3
+ #define sys_fork __sys_fork
+
+ #endif /* __ASM_OPENRISC_SYSCALLS_H */
+diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c
+index 1b16d97e7da7f..a82b2caaa560d 100644
+--- a/arch/openrisc/kernel/dma.c
++++ b/arch/openrisc/kernel/dma.c
+@@ -33,7 +33,7 @@ page_set_nocache(pte_t *pte, unsigned long addr,
+ * Flush the page out of the TLB so that the new page flags get
+ * picked up next time there's an access
+ */
+- flush_tlb_page(NULL, addr);
++ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+
+ /* Flush page out of dcache */
+ for (cl = __pa(addr); cl < __pa(next); cl += cpuinfo->dcache_block_size)
+@@ -56,7 +56,7 @@ page_clear_nocache(pte_t *pte, unsigned long addr,
+ * Flush the page out of the TLB so that the new page flags get
+ * picked up next time there's an access
+ */
+- flush_tlb_page(NULL, addr);
++ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+
+ return 0;
+ }
+diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S
+index edaa775a648e6..c68f3349c1741 100644
+--- a/arch/openrisc/kernel/entry.S
++++ b/arch/openrisc/kernel/entry.S
+@@ -1170,6 +1170,11 @@ ENTRY(__sys_clone)
+ l.j _fork_save_extra_regs_and_call
+ l.nop
+
++ENTRY(__sys_clone3)
++ l.movhi r29,hi(sys_clone3)
++ l.j _fork_save_extra_regs_and_call
++ l.ori r29,r29,lo(sys_clone3)
++
+ ENTRY(__sys_fork)
+ l.movhi r29,hi(sys_fork)
+ l.ori r29,r29,lo(sys_fork)
+diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c
+index 415e209732a3d..ba78766cf00b5 100644
+--- a/arch/openrisc/kernel/smp.c
++++ b/arch/openrisc/kernel/smp.c
+@@ -272,7 +272,7 @@ static inline void ipi_flush_tlb_range(void *info)
+ local_flush_tlb_range(NULL, fd->addr1, fd->addr2);
+ }
+
+-static void smp_flush_tlb_range(struct cpumask *cmask, unsigned long start,
++static void smp_flush_tlb_range(const struct cpumask *cmask, unsigned long start,
+ unsigned long end)
+ {
+ unsigned int cpuid;
+@@ -320,7 +320,9 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
+ void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+ {
+- smp_flush_tlb_range(mm_cpumask(vma->vm_mm), start, end);
++ const struct cpumask *cmask = vma ? mm_cpumask(vma->vm_mm)
++ : cpu_online_mask;
++ smp_flush_tlb_range(cmask, start, end);
+ }
+
+ /* Instruction cache invalidate - performed on each cpu */
+diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
+index fcde3ffa02213..fadb098de1545 100644
+--- a/arch/parisc/Makefile
++++ b/arch/parisc/Makefile
+@@ -17,7 +17,12 @@
+ # Mike Shaver, Helge Deller and Martin K. Petersen
+ #
+
++ifdef CONFIG_PARISC_SELF_EXTRACT
++boot := arch/parisc/boot
++KBUILD_IMAGE := $(boot)/bzImage
++else
+ KBUILD_IMAGE := vmlinuz
++endif
+
+ NM = sh $(srctree)/arch/parisc/nm
+ CHECKFLAGS += -D__hppa__=1
+diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h
+index aa4e883431c1a..5779d463b341f 100644
+--- a/arch/parisc/include/asm/bitops.h
++++ b/arch/parisc/include/asm/bitops.h
+@@ -12,6 +12,14 @@
+ #include <asm/barrier.h>
+ #include <linux/atomic.h>
+
++/* compiler build environment sanity checks: */
++#if !defined(CONFIG_64BIT) && defined(__LP64__)
++#error "Please use 'ARCH=parisc' to build the 32-bit kernel."
++#endif
++#if defined(CONFIG_64BIT) && !defined(__LP64__)
++#error "Please use 'ARCH=parisc64' to build the 64-bit kernel."
++#endif
++
+ /* See http://marc.theaimsgroup.com/?t=108826637900003 for discussion
+ * on use of volatile and __*_bit() (set/clear/change):
+ * *_bit() want use of volatile.
+diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h
+index fceb9cf02fb3a..71aa0921d6c72 100644
+--- a/arch/parisc/include/asm/futex.h
++++ b/arch/parisc/include/asm/futex.h
+@@ -16,7 +16,7 @@ static inline void
+ _futex_spin_lock_irqsave(u32 __user *uaddr, unsigned long int *flags)
+ {
+ extern u32 lws_lock_start[];
+- long index = ((long)uaddr & 0x3f8) >> 1;
++ long index = ((long)uaddr & 0x7f8) >> 1;
+ arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
+ local_irq_save(*flags);
+ arch_spin_lock(s);
+@@ -26,7 +26,7 @@ static inline void
+ _futex_spin_unlock_irqrestore(u32 __user *uaddr, unsigned long int *flags)
+ {
+ extern u32 lws_lock_start[];
+- long index = ((long)uaddr & 0x3f8) >> 1;
++ long index = ((long)uaddr & 0x7f8) >> 1;
+ arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
+ arch_spin_unlock(s);
+ local_irq_restore(*flags);
+diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h
+index 7badd872f05ac..3e7cf882639fb 100644
+--- a/arch/parisc/include/asm/pgtable.h
++++ b/arch/parisc/include/asm/pgtable.h
+@@ -76,6 +76,8 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
+ purge_tlb_end(flags);
+ }
+
++extern void __update_cache(pte_t pte);
++
+ /* Certain architectures need to do special things when PTEs
+ * within a page table are directly modified. Thus, the following
+ * hook is made available.
+@@ -83,11 +85,14 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
+ #define set_pte(pteptr, pteval) \
+ do { \
+ *(pteptr) = (pteval); \
+- barrier(); \
++ mb(); \
+ } while(0)
+
+ #define set_pte_at(mm, addr, pteptr, pteval) \
+ do { \
++ if (pte_present(pteval) && \
++ pte_user(pteval)) \
++ __update_cache(pteval); \
+ *(pteptr) = (pteval); \
+ purge_tlb_entries(mm, addr); \
+ } while (0)
+@@ -303,6 +308,7 @@ extern unsigned long *empty_zero_page;
+
+ #define pte_none(x) (pte_val(x) == 0)
+ #define pte_present(x) (pte_val(x) & _PAGE_PRESENT)
++#define pte_user(x) (pte_val(x) & _PAGE_USER)
+ #define pte_clear(mm, addr, xp) set_pte_at(mm, addr, xp, __pte(0))
+
+ #define pmd_flag(x) (pmd_val(x) & PxD_FLAG_MASK)
+@@ -410,7 +416,7 @@ extern void paging_init (void);
+
+ #define PG_dcache_dirty PG_arch_1
+
+-extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
++#define update_mmu_cache(vms,addr,ptep) __update_cache(*ptep)
+
+ /* Encode and de-code a swap entry */
+
+diff --git a/arch/parisc/include/asm/rt_sigframe.h b/arch/parisc/include/asm/rt_sigframe.h
+index 4b9e3d707571b..2b3010ade00e7 100644
+--- a/arch/parisc/include/asm/rt_sigframe.h
++++ b/arch/parisc/include/asm/rt_sigframe.h
+@@ -2,7 +2,7 @@
+ #ifndef _ASM_PARISC_RT_SIGFRAME_H
+ #define _ASM_PARISC_RT_SIGFRAME_H
+
+-#define SIGRETURN_TRAMP 3
++#define SIGRETURN_TRAMP 4
+ #define SIGRESTARTBLOCK_TRAMP 5
+ #define TRAMP_SIZE (SIGRETURN_TRAMP + SIGRESTARTBLOCK_TRAMP)
+
+diff --git a/arch/parisc/include/asm/special_insns.h b/arch/parisc/include/asm/special_insns.h
+index a303ae9a77f41..16ee41e77174f 100644
+--- a/arch/parisc/include/asm/special_insns.h
++++ b/arch/parisc/include/asm/special_insns.h
+@@ -2,28 +2,32 @@
+ #ifndef __PARISC_SPECIAL_INSNS_H
+ #define __PARISC_SPECIAL_INSNS_H
+
+-#define lpa(va) ({ \
+- unsigned long pa; \
+- __asm__ __volatile__( \
+- "copy %%r0,%0\n\t" \
+- "lpa %%r0(%1),%0" \
+- : "=r" (pa) \
+- : "r" (va) \
+- : "memory" \
+- ); \
+- pa; \
++#define lpa(va) ({ \
++ unsigned long pa; \
++ __asm__ __volatile__( \
++ "copy %%r0,%0\n" \
++ "8:\tlpa %%r0(%1),%0\n" \
++ "9:\n" \
++ ASM_EXCEPTIONTABLE_ENTRY(8b, 9b) \
++ : "=&r" (pa) \
++ : "r" (va) \
++ : "memory" \
++ ); \
++ pa; \
+ })
+
+-#define lpa_user(va) ({ \
+- unsigned long pa; \
+- __asm__ __volatile__( \
+- "copy %%r0,%0\n\t" \
+- "lpa %%r0(%%sr3,%1),%0" \
+- : "=r" (pa) \
+- : "r" (va) \
+- : "memory" \
+- ); \
+- pa; \
++#define lpa_user(va) ({ \
++ unsigned long pa; \
++ __asm__ __volatile__( \
++ "copy %%r0,%0\n" \
++ "8:\tlpa %%r0(%%sr3,%1),%0\n" \
++ "9:\n" \
++ ASM_EXCEPTIONTABLE_ENTRY(8b, 9b) \
++ : "=&r" (pa) \
++ : "r" (va) \
++ : "memory" \
++ ); \
++ pa; \
+ })
+
+ #define mfctl(reg) ({ \
+diff --git a/arch/parisc/include/asm/timex.h b/arch/parisc/include/asm/timex.h
+index 06b510f8172e3..b4622cb06a75e 100644
+--- a/arch/parisc/include/asm/timex.h
++++ b/arch/parisc/include/asm/timex.h
+@@ -13,9 +13,10 @@
+
+ typedef unsigned long cycles_t;
+
+-static inline cycles_t get_cycles (void)
++static inline cycles_t get_cycles(void)
+ {
+ return mfctl(16);
+ }
++#define get_cycles get_cycles
+
+ #endif
+diff --git a/arch/parisc/include/asm/traps.h b/arch/parisc/include/asm/traps.h
+index 8ecc1f0c0483d..d0e090a2c000d 100644
+--- a/arch/parisc/include/asm/traps.h
++++ b/arch/parisc/include/asm/traps.h
+@@ -17,6 +17,7 @@ void die_if_kernel(char *str, struct pt_regs *regs, long err);
+ const char *trap_name(unsigned long code);
+ void do_page_fault(struct pt_regs *regs, unsigned long code,
+ unsigned long address);
++int handle_nadtlb_fault(struct pt_regs *regs);
+ #endif
+
+ #endif
+diff --git a/arch/parisc/install.sh b/arch/parisc/install.sh
+index 056d588befdd6..70d3cffb02515 100644
+--- a/arch/parisc/install.sh
++++ b/arch/parisc/install.sh
+@@ -39,6 +39,7 @@ verify "$3"
+ if [ -n "${INSTALLKERNEL}" ]; then
+ if [ -x ~/bin/${INSTALLKERNEL} ]; then exec ~/bin/${INSTALLKERNEL} "$@"; fi
+ if [ -x /sbin/${INSTALLKERNEL} ]; then exec /sbin/${INSTALLKERNEL} "$@"; fi
++ if [ -x /usr/sbin/${INSTALLKERNEL} ]; then exec /usr/sbin/${INSTALLKERNEL} "$@"; fi
+ fi
+
+ # Default install
+diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
+index 39e02227e2310..4ed4942d8312b 100644
+--- a/arch/parisc/kernel/cache.c
++++ b/arch/parisc/kernel/cache.c
+@@ -83,9 +83,9 @@ EXPORT_SYMBOL(flush_cache_all_local);
+ #define pfn_va(pfn) __va(PFN_PHYS(pfn))
+
+ void
+-update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
++__update_cache(pte_t pte)
+ {
+- unsigned long pfn = pte_pfn(*ptep);
++ unsigned long pfn = pte_pfn(pte);
+ struct page *page;
+
+ /* We don't have pte special. As a result, we can be called with
+diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
+index 9f939afe6b88c..437c8d31f3907 100644
+--- a/arch/parisc/kernel/entry.S
++++ b/arch/parisc/kernel/entry.S
+@@ -1834,8 +1834,8 @@ syscall_restore:
+ LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
+
+ /* Are we being ptraced? */
+- ldw TASK_FLAGS(%r1),%r19
+- ldi _TIF_SYSCALL_TRACE_MASK,%r2
++ LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19
++ ldi _TIF_SINGLESTEP|_TIF_BLOCKSTEP,%r2
+ and,COND(=) %r19,%r2,%r0
+ b,n syscall_restore_rfi
+
+diff --git a/arch/parisc/kernel/patch.c b/arch/parisc/kernel/patch.c
+index 80a0ab372802d..e59574f65e641 100644
+--- a/arch/parisc/kernel/patch.c
++++ b/arch/parisc/kernel/patch.c
+@@ -40,10 +40,7 @@ static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags,
+
+ *need_unmap = 1;
+ set_fixmap(fixmap, page_to_phys(page));
+- if (flags)
+- raw_spin_lock_irqsave(&patch_lock, *flags);
+- else
+- __acquire(&patch_lock);
++ raw_spin_lock_irqsave(&patch_lock, *flags);
+
+ return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
+ }
+@@ -52,10 +49,7 @@ static void __kprobes patch_unmap(int fixmap, unsigned long *flags)
+ {
+ clear_fixmap(fixmap);
+
+- if (flags)
+- raw_spin_unlock_irqrestore(&patch_lock, *flags);
+- else
+- __release(&patch_lock);
++ raw_spin_unlock_irqrestore(&patch_lock, *flags);
+ }
+
+ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
+@@ -67,8 +61,9 @@ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
+ int mapped;
+
+ /* Make sure we don't have any aliases in cache */
+- flush_kernel_vmap_range(addr, len);
+- flush_icache_range(start, end);
++ flush_kernel_dcache_range_asm(start, end);
++ flush_kernel_icache_range_asm(start, end);
++ flush_tlb_kernel_range(start, end);
+
+ p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags, &mapped);
+
+@@ -81,8 +76,10 @@ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
+ * We're crossing a page boundary, so
+ * need to remap
+ */
+- flush_kernel_vmap_range((void *)fixmap,
+- (p-fixmap) * sizeof(*p));
++ flush_kernel_dcache_range_asm((unsigned long)fixmap,
++ (unsigned long)p);
++ flush_tlb_kernel_range((unsigned long)fixmap,
++ (unsigned long)p);
+ if (mapped)
+ patch_unmap(FIX_TEXT_POKE0, &flags);
+ p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags,
+@@ -90,10 +87,10 @@ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
+ }
+ }
+
+- flush_kernel_vmap_range((void *)fixmap, (p-fixmap) * sizeof(*p));
++ flush_kernel_dcache_range_asm((unsigned long)fixmap, (unsigned long)p);
++ flush_tlb_kernel_range((unsigned long)fixmap, (unsigned long)p);
+ if (mapped)
+ patch_unmap(FIX_TEXT_POKE0, &flags);
+- flush_icache_range(start, end);
+ }
+
+ void __kprobes __patch_text(void *addr, u32 insn)
+diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c
+index 1b6129e7d776b..b861bbbc87178 100644
+--- a/arch/parisc/kernel/processor.c
++++ b/arch/parisc/kernel/processor.c
+@@ -418,8 +418,7 @@ show_cpuinfo (struct seq_file *m, void *v)
+ }
+ seq_printf(m, " (0x%02lx)\n", boot_cpu_data.pdc.capabilities);
+
+- seq_printf(m, "model\t\t: %s\n"
+- "model name\t: %s\n",
++ seq_printf(m, "model\t\t: %s - %s\n",
+ boot_cpu_data.pdc.sys_model_name,
+ cpuinfo->dev ?
+ cpuinfo->dev->name : "Unknown");
+diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
+index cceb09855e03f..3fb86ee507dd5 100644
+--- a/arch/parisc/kernel/setup.c
++++ b/arch/parisc/kernel/setup.c
+@@ -150,6 +150,8 @@ void __init setup_arch(char **cmdline_p)
+ #ifdef CONFIG_PA11
+ dma_ops_init();
+ #endif
++
++ clear_sched_clock_stable();
+ }
+
+ /*
+diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
+index bbfe23c40c016..0fb06d87b3a5c 100644
+--- a/arch/parisc/kernel/signal.c
++++ b/arch/parisc/kernel/signal.c
+@@ -288,21 +288,22 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs,
+ already in userspace. The first words of tramp are used to
+ save the previous sigrestartblock trampoline that might be
+ on the stack. We start the sigreturn trampoline at
+- SIGRESTARTBLOCK_TRAMP. */
++ SIGRESTARTBLOCK_TRAMP+X. */
+ err |= __put_user(in_syscall ? INSN_LDI_R25_1 : INSN_LDI_R25_0,
+ &frame->tramp[SIGRESTARTBLOCK_TRAMP+0]);
+- err |= __put_user(INSN_BLE_SR2_R0,
+- &frame->tramp[SIGRESTARTBLOCK_TRAMP+1]);
+ err |= __put_user(INSN_LDI_R20,
++ &frame->tramp[SIGRESTARTBLOCK_TRAMP+1]);
++ err |= __put_user(INSN_BLE_SR2_R0,
+ &frame->tramp[SIGRESTARTBLOCK_TRAMP+2]);
++ err |= __put_user(INSN_NOP, &frame->tramp[SIGRESTARTBLOCK_TRAMP+3]);
+
+- start = (unsigned long) &frame->tramp[SIGRESTARTBLOCK_TRAMP+0];
+- end = (unsigned long) &frame->tramp[SIGRESTARTBLOCK_TRAMP+3];
++ start = (unsigned long) &frame->tramp[0];
++ end = (unsigned long) &frame->tramp[TRAMP_SIZE];
+ flush_user_dcache_range_asm(start, end);
+ flush_user_icache_range_asm(start, end);
+
+ /* TRAMP Words 0-4, Length 5 = SIGRESTARTBLOCK_TRAMP
+- * TRAMP Words 5-7, Length 3 = SIGRETURN_TRAMP
++ * TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP
+ * So the SIGRETURN_TRAMP is at the end of SIGRESTARTBLOCK_TRAMP
+ */
+ rp = (unsigned long) &frame->tramp[SIGRESTARTBLOCK_TRAMP];
+diff --git a/arch/parisc/kernel/signal32.h b/arch/parisc/kernel/signal32.h
+index a5bdbb5678b72..f166250f2d064 100644
+--- a/arch/parisc/kernel/signal32.h
++++ b/arch/parisc/kernel/signal32.h
+@@ -36,7 +36,7 @@ struct compat_regfile {
+ compat_int_t rf_sar;
+ };
+
+-#define COMPAT_SIGRETURN_TRAMP 3
++#define COMPAT_SIGRETURN_TRAMP 4
+ #define COMPAT_SIGRESTARTBLOCK_TRAMP 5
+ #define COMPAT_TRAMP_SIZE (COMPAT_SIGRETURN_TRAMP + \
+ COMPAT_SIGRESTARTBLOCK_TRAMP)
+diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
+index 1405b603b91b6..cf92ece20b757 100644
+--- a/arch/parisc/kernel/smp.c
++++ b/arch/parisc/kernel/smp.c
+@@ -29,6 +29,7 @@
+ #include <linux/bitops.h>
+ #include <linux/ftrace.h>
+ #include <linux/cpu.h>
++#include <linux/kgdb.h>
+
+ #include <linux/atomic.h>
+ #include <asm/current.h>
+@@ -69,7 +70,10 @@ enum ipi_message_type {
+ IPI_CALL_FUNC,
+ IPI_CPU_START,
+ IPI_CPU_STOP,
+- IPI_CPU_TEST
++ IPI_CPU_TEST,
++#ifdef CONFIG_KGDB
++ IPI_ENTER_KGDB,
++#endif
+ };
+
+
+@@ -167,7 +171,12 @@ ipi_interrupt(int irq, void *dev_id)
+ case IPI_CPU_TEST:
+ smp_debug(100, KERN_DEBUG "CPU%d is alive!\n", this_cpu);
+ break;
+-
++#ifdef CONFIG_KGDB
++ case IPI_ENTER_KGDB:
++ smp_debug(100, KERN_DEBUG "CPU%d ENTER_KGDB\n", this_cpu);
++ kgdb_nmicallback(raw_smp_processor_id(), get_irq_regs());
++ break;
++#endif
+ default:
+ printk(KERN_CRIT "Unknown IPI num on CPU%d: %lu\n",
+ this_cpu, which);
+@@ -226,6 +235,12 @@ send_IPI_allbutself(enum ipi_message_type op)
+ }
+ }
+
++#ifdef CONFIG_KGDB
++void kgdb_roundup_cpus(void)
++{
++ send_IPI_allbutself(IPI_ENTER_KGDB);
++}
++#endif
+
+ inline void
+ smp_send_stop(void) { send_IPI_allbutself(IPI_CPU_STOP); }
+diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S
+index 3f24a0af1e047..9842dcb2041e5 100644
+--- a/arch/parisc/kernel/syscall.S
++++ b/arch/parisc/kernel/syscall.S
+@@ -478,7 +478,7 @@ lws_start:
+ extrd,u %r1,PSW_W_BIT,1,%r1
+ /* sp must be aligned on 4, so deposit the W bit setting into
+ * the bottom of sp temporarily */
+- or,ev %r1,%r30,%r30
++ or,od %r1,%r30,%r30
+
+ /* Clip LWS number to a 32-bit value for 32-bit processes */
+ depdi 0, 31, 32, %r20
+diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c
+index 9fb1e794831b0..d8e59a1000ab7 100644
+--- a/arch/parisc/kernel/time.c
++++ b/arch/parisc/kernel/time.c
+@@ -249,30 +249,12 @@ void __init time_init(void)
+ static int __init init_cr16_clocksource(void)
+ {
+ /*
+- * The cr16 interval timers are not syncronized across CPUs on
+- * different sockets, so mark them unstable and lower rating on
+- * multi-socket SMP systems.
++ * The cr16 interval timers are not synchronized across CPUs.
+ */
+ if (num_online_cpus() > 1 && !running_on_qemu) {
+- int cpu;
+- unsigned long cpu0_loc;
+- cpu0_loc = per_cpu(cpu_data, 0).cpu_loc;
+-
+- for_each_online_cpu(cpu) {
+- if (cpu == 0)
+- continue;
+- if ((cpu0_loc != 0) &&
+- (cpu0_loc == per_cpu(cpu_data, cpu).cpu_loc))
+- continue;
+-
+- /* mark sched_clock unstable */
+- clear_sched_clock_stable();
+-
+- clocksource_cr16.name = "cr16_unstable";
+- clocksource_cr16.flags = CLOCK_SOURCE_UNSTABLE;
+- clocksource_cr16.rating = 0;
+- break;
+- }
++ clocksource_cr16.name = "cr16_unstable";
++ clocksource_cr16.flags = CLOCK_SOURCE_UNSTABLE;
++ clocksource_cr16.rating = 0;
+ }
+
+ /* register at clocksource framework */
+diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c
+index 747c328fb8862..6fe5a3e98edc2 100644
+--- a/arch/parisc/kernel/traps.c
++++ b/arch/parisc/kernel/traps.c
+@@ -661,6 +661,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
+ by hand. Technically we need to emulate:
+ fdc,fdce,pdc,"fic,4f",prober,probeir,probew, probeiw
+ */
++ if (code == 17 && handle_nadtlb_fault(regs))
++ return;
+ fault_address = regs->ior;
+ fault_space = regs->isr;
+ break;
+@@ -729,6 +731,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
+ }
+ mmap_read_unlock(current->mm);
+ }
++ /* CPU could not fetch instruction, so clear stale IIR value. */
++ regs->iir = 0xbaadf00d;
+ fallthrough;
+ case 27:
+ /* Data memory protection ID trap */
+@@ -782,7 +786,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
+ * unless pagefault_disable() was called before.
+ */
+
+- if (fault_space == 0 && !faulthandler_disabled())
++ if (faulthandler_disabled() || fault_space == 0)
+ {
+ /* Clean up and return if in exception table. */
+ if (fixup_exception(regs))
+diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c
+index 237d20dd5622d..286cec4d86d7b 100644
+--- a/arch/parisc/kernel/unaligned.c
++++ b/arch/parisc/kernel/unaligned.c
+@@ -340,7 +340,7 @@ static int emulate_stw(struct pt_regs *regs, int frreg, int flop)
+ : "r" (val), "r" (regs->ior), "r" (regs->isr)
+ : "r19", "r20", "r21", "r22", "r1", FIXUP_BRANCH_CLOBBER );
+
+- return 0;
++ return ret;
+ }
+ static int emulate_std(struct pt_regs *regs, int frreg, int flop)
+ {
+@@ -397,7 +397,7 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop)
+ __asm__ __volatile__ (
+ " mtsp %4, %%sr1\n"
+ " zdep %2, 29, 2, %%r19\n"
+-" dep %%r0, 31, 2, %2\n"
++" dep %%r0, 31, 2, %3\n"
+ " mtsar %%r19\n"
+ " zvdepi -2, 32, %%r19\n"
+ "1: ldw 0(%%sr1,%3),%%r20\n"
+@@ -409,7 +409,7 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop)
+ " andcm %%r21, %%r19, %%r21\n"
+ " or %1, %%r20, %1\n"
+ " or %2, %%r21, %2\n"
+-"3: stw %1,0(%%sr1,%1)\n"
++"3: stw %1,0(%%sr1,%3)\n"
+ "4: stw %%r1,4(%%sr1,%3)\n"
+ "5: stw %2,8(%%sr1,%3)\n"
+ " copy %%r0, %0\n"
+@@ -596,7 +596,6 @@ void handle_unaligned(struct pt_regs *regs)
+ ret = ERR_NOTHANDLED; /* "undefined", but lets kill them. */
+ break;
+ }
+-#ifdef CONFIG_PA20
+ switch (regs->iir & OPCODE2_MASK)
+ {
+ case OPCODE_FLDD_L:
+@@ -607,22 +606,23 @@ void handle_unaligned(struct pt_regs *regs)
+ flop=1;
+ ret = emulate_std(regs, R2(regs->iir),1);
+ break;
++#ifdef CONFIG_PA20
+ case OPCODE_LDD_L:
+ ret = emulate_ldd(regs, R2(regs->iir),0);
+ break;
+ case OPCODE_STD_L:
+ ret = emulate_std(regs, R2(regs->iir),0);
+ break;
+- }
+ #endif
++ }
+ switch (regs->iir & OPCODE3_MASK)
+ {
+ case OPCODE_FLDW_L:
+ flop=1;
+- ret = emulate_ldw(regs, R2(regs->iir),0);
++ ret = emulate_ldw(regs, R2(regs->iir), 1);
+ break;
+ case OPCODE_LDW_M:
+- ret = emulate_ldw(regs, R2(regs->iir),1);
++ ret = emulate_ldw(regs, R2(regs->iir), 0);
+ break;
+
+ case OPCODE_FSTW_L:
+diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c
+index 87ae476d1c4f5..86a57fb0e6fae 100644
+--- a/arch/parisc/kernel/unwind.c
++++ b/arch/parisc/kernel/unwind.c
+@@ -21,6 +21,8 @@
+ #include <asm/ptrace.h>
+
+ #include <asm/unwind.h>
++#include <asm/switch_to.h>
++#include <asm/sections.h>
+
+ /* #define DEBUG 1 */
+ #ifdef DEBUG
+@@ -203,6 +205,11 @@ int __init unwind_init(void)
+ return 0;
+ }
+
++static bool pc_is_kernel_fn(unsigned long pc, void *fn)
++{
++ return (unsigned long)dereference_kernel_function_descriptor(fn) == pc;
++}
++
+ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size)
+ {
+ /*
+@@ -221,7 +228,7 @@ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int
+ extern void * const _call_on_stack;
+ #endif /* CONFIG_IRQSTACKS */
+
+- if (pc == (unsigned long) &handle_interruption) {
++ if (pc_is_kernel_fn(pc, handle_interruption)) {
+ struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN);
+ dbg("Unwinding through handle_interruption()\n");
+ info->prev_sp = regs->gr[30];
+@@ -229,13 +236,13 @@ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int
+ return 1;
+ }
+
+- if (pc == (unsigned long) &ret_from_kernel_thread ||
+- pc == (unsigned long) &syscall_exit) {
++ if (pc_is_kernel_fn(pc, ret_from_kernel_thread) ||
++ pc_is_kernel_fn(pc, syscall_exit)) {
+ info->prev_sp = info->prev_ip = 0;
+ return 1;
+ }
+
+- if (pc == (unsigned long) &intr_return) {
++ if (pc_is_kernel_fn(pc, intr_return)) {
+ struct pt_regs *regs;
+
+ dbg("Found intr_return()\n");
+@@ -246,20 +253,20 @@ static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int
+ return 1;
+ }
+
+- if (pc == (unsigned long) &_switch_to_ret) {
++ if (pc_is_kernel_fn(pc, _switch_to) ||
++ pc_is_kernel_fn(pc, _switch_to_ret)) {
+ info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE;
+ info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET);
+ return 1;
+ }
+
+ #ifdef CONFIG_IRQSTACKS
+- if (pc == (unsigned long) &_call_on_stack) {
++ if (pc_is_kernel_fn(pc, _call_on_stack)) {
+ info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ);
+ info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET);
+ return 1;
+ }
+ #endif
+-
+ return 0;
+ }
+
+diff --git a/arch/parisc/lib/iomap.c b/arch/parisc/lib/iomap.c
+index 367f6397bda7a..8603850580857 100644
+--- a/arch/parisc/lib/iomap.c
++++ b/arch/parisc/lib/iomap.c
+@@ -346,6 +346,16 @@ u64 ioread64be(const void __iomem *addr)
+ return *((u64 *)addr);
+ }
+
++u64 ioread64_lo_hi(const void __iomem *addr)
++{
++ u32 low, high;
++
++ low = ioread32(addr);
++ high = ioread32(addr + sizeof(u32));
++
++ return low + ((u64)high << 32);
++}
++
+ u64 ioread64_hi_lo(const void __iomem *addr)
+ {
+ u32 low, high;
+@@ -419,6 +429,12 @@ void iowrite64be(u64 datum, void __iomem *addr)
+ }
+ }
+
++void iowrite64_lo_hi(u64 val, void __iomem *addr)
++{
++ iowrite32(val, addr);
++ iowrite32(val >> 32, addr + sizeof(u32));
++}
++
+ void iowrite64_hi_lo(u64 val, void __iomem *addr)
+ {
+ iowrite32(val >> 32, addr + sizeof(u32));
+@@ -530,6 +546,7 @@ EXPORT_SYMBOL(ioread32);
+ EXPORT_SYMBOL(ioread32be);
+ EXPORT_SYMBOL(ioread64);
+ EXPORT_SYMBOL(ioread64be);
++EXPORT_SYMBOL(ioread64_lo_hi);
+ EXPORT_SYMBOL(ioread64_hi_lo);
+ EXPORT_SYMBOL(iowrite8);
+ EXPORT_SYMBOL(iowrite16);
+@@ -538,6 +555,7 @@ EXPORT_SYMBOL(iowrite32);
+ EXPORT_SYMBOL(iowrite32be);
+ EXPORT_SYMBOL(iowrite64);
+ EXPORT_SYMBOL(iowrite64be);
++EXPORT_SYMBOL(iowrite64_lo_hi);
+ EXPORT_SYMBOL(iowrite64_hi_lo);
+ EXPORT_SYMBOL(ioread8_rep);
+ EXPORT_SYMBOL(ioread16_rep);
+diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
+index 716960f5d92ea..5faa3cff47387 100644
+--- a/arch/parisc/mm/fault.c
++++ b/arch/parisc/mm/fault.c
+@@ -424,3 +424,92 @@ no_context:
+ goto no_context;
+ pagefault_out_of_memory();
+ }
++
++/* Handle non-access data TLB miss faults.
++ *
++ * For probe instructions, accesses to userspace are considered allowed
++ * if they lie in a valid VMA and the access type matches. We are not
++ * allowed to handle MM faults here so there may be situations where an
++ * actual access would fail even though a probe was successful.
++ */
++int
++handle_nadtlb_fault(struct pt_regs *regs)
++{
++ unsigned long insn = regs->iir;
++ int breg, treg, xreg, val = 0;
++ struct vm_area_struct *vma, *prev_vma;
++ struct task_struct *tsk;
++ struct mm_struct *mm;
++ unsigned long address;
++ unsigned long acc_type;
++
++ switch (insn & 0x380) {
++ case 0x280:
++ /* FDC instruction */
++ fallthrough;
++ case 0x380:
++ /* PDC and FIC instructions */
++ if (printk_ratelimit()) {
++ pr_warn("BUG: nullifying cache flush/purge instruction\n");
++ show_regs(regs);
++ }
++ if (insn & 0x20) {
++ /* Base modification */
++ breg = (insn >> 21) & 0x1f;
++ xreg = (insn >> 16) & 0x1f;
++ if (breg && xreg)
++ regs->gr[breg] += regs->gr[xreg];
++ }
++ regs->gr[0] |= PSW_N;
++ return 1;
++
++ case 0x180:
++ /* PROBE instruction */
++ treg = insn & 0x1f;
++ if (regs->isr) {
++ tsk = current;
++ mm = tsk->mm;
++ if (mm) {
++ /* Search for VMA */
++ address = regs->ior;
++ mmap_read_lock(mm);
++ vma = find_vma_prev(mm, address, &prev_vma);
++ mmap_read_unlock(mm);
++
++ /*
++ * Check if access to the VMA is okay.
++ * We don't allow for stack expansion.
++ */
++ acc_type = (insn & 0x40) ? VM_WRITE : VM_READ;
++ if (vma
++ && address >= vma->vm_start
++ && (vma->vm_flags & acc_type) == acc_type)
++ val = 1;
++ }
++ }
++ if (treg)
++ regs->gr[treg] = val;
++ regs->gr[0] |= PSW_N;
++ return 1;
++
++ case 0x300:
++ /* LPA instruction */
++ if (insn & 0x20) {
++ /* Base modification */
++ breg = (insn >> 21) & 0x1f;
++ xreg = (insn >> 16) & 0x1f;
++ if (breg && xreg)
++ regs->gr[breg] += regs->gr[xreg];
++ }
++ treg = insn & 0x1f;
++ if (treg)
++ regs->gr[treg] = 0;
++ regs->gr[0] |= PSW_N;
++ return 1;
++
++ default:
++ break;
++ }
++
++ return 0;
++}
+diff --git a/arch/parisc/mm/fixmap.c b/arch/parisc/mm/fixmap.c
+index 24426a7e1a5e5..cc15d737fda64 100644
+--- a/arch/parisc/mm/fixmap.c
++++ b/arch/parisc/mm/fixmap.c
+@@ -20,12 +20,9 @@ void notrace set_fixmap(enum fixed_addresses idx, phys_addr_t phys)
+ pte_t *pte;
+
+ if (pmd_none(*pmd))
+- pmd = pmd_alloc(NULL, pud, vaddr);
+-
+- pte = pte_offset_kernel(pmd, vaddr);
+- if (pte_none(*pte))
+ pte = pte_alloc_kernel(pmd, vaddr);
+
++ pte = pte_offset_kernel(pmd, vaddr);
+ set_pte_at(&init_mm, vaddr, pte, __mk_pte(phys, PAGE_KERNEL_RWX));
+ flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE);
+ }
+diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
+index 3f7d6d5b56ac8..e5c18313b5d4f 100644
+--- a/arch/parisc/mm/init.c
++++ b/arch/parisc/mm/init.c
+@@ -341,9 +341,9 @@ static void __init setup_bootmem(void)
+
+ static bool kernel_set_to_readonly;
+
+-static void __init map_pages(unsigned long start_vaddr,
+- unsigned long start_paddr, unsigned long size,
+- pgprot_t pgprot, int force)
++static void __ref map_pages(unsigned long start_vaddr,
++ unsigned long start_paddr, unsigned long size,
++ pgprot_t pgprot, int force)
+ {
+ pmd_t *pmd;
+ pte_t *pg_table;
+@@ -453,7 +453,7 @@ void __init set_kernel_text_rw(int enable_read_write)
+ flush_tlb_all();
+ }
+
+-void __ref free_initmem(void)
++void free_initmem(void)
+ {
+ unsigned long init_begin = (unsigned long)__init_begin;
+ unsigned long init_end = (unsigned long)__init_end;
+@@ -467,7 +467,6 @@ void __ref free_initmem(void)
+ /* The init text pages are marked R-X. We have to
+ * flush the icache and mark them RW-
+ *
+- * This is tricky, because map_pages is in the init section.
+ * Do a dummy remap of the data section first (the data
+ * section is already PAGE_KERNEL) to pull in the TLB entries
+ * for map_kernel */
+@@ -842,9 +841,9 @@ void flush_tlb_all(void)
+ {
+ int do_recycle;
+
+- __inc_irq_stat(irq_tlb_count);
+ do_recycle = 0;
+ spin_lock(&sid_lock);
++ __inc_irq_stat(irq_tlb_count);
+ if (dirty_space_ids > RECYCLE_THRESHOLD) {
+ BUG_ON(recycle_inuse); /* FIXME: Use a semaphore/wait queue here */
+ get_dirty_sids(&recycle_ndirty,recycle_dirty_array);
+@@ -863,8 +862,8 @@ void flush_tlb_all(void)
+ #else
+ void flush_tlb_all(void)
+ {
+- __inc_irq_stat(irq_tlb_count);
+ spin_lock(&sid_lock);
++ __inc_irq_stat(irq_tlb_count);
+ flush_tlb_all_local(NULL);
+ recycle_sids();
+ spin_unlock(&sid_lock);
+diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
+index ba5b661893588..6b9f523882c58 100644
+--- a/arch/powerpc/Kconfig
++++ b/arch/powerpc/Kconfig
+@@ -138,7 +138,7 @@ config PPC
+ select ARCH_HAS_PTE_SPECIAL
+ select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64
+ select ARCH_HAS_SET_MEMORY
+- select ARCH_HAS_STRICT_KERNEL_RWX if ((PPC_BOOK3S_64 || PPC32) && !HIBERNATION)
++ select ARCH_HAS_STRICT_KERNEL_RWX if (PPC_BOOK3S || PPC_8xx || 40x) && !HIBERNATION
+ select ARCH_HAS_STRICT_MODULE_RWX if ARCH_HAS_STRICT_KERNEL_RWX && !PPC_BOOK3S_32
+ select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
+ select ARCH_HAS_UACCESS_FLUSHCACHE
+@@ -150,7 +150,7 @@ config PPC
+ select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
+ select ARCH_STACKWALK
+ select ARCH_SUPPORTS_ATOMIC_RMW
+- select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC32 || PPC_BOOK3S_64
++ select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x
+ select ARCH_USE_BUILTIN_BSWAP
+ select ARCH_USE_CMPXCHG_LOCKREF if PPC64
+ select ARCH_USE_MEMTEST
+@@ -190,7 +190,7 @@ config PPC
+ select HAVE_ARCH_JUMP_LABEL_RELATIVE
+ select HAVE_ARCH_KASAN if PPC32 && PPC_PAGE_SHIFT <= 14
+ select HAVE_ARCH_KASAN_VMALLOC if PPC32 && PPC_PAGE_SHIFT <= 14
+- select HAVE_ARCH_KFENCE if PPC32
++ select HAVE_ARCH_KFENCE if PPC_BOOK3S_32 || PPC_8xx || 40x
+ select HAVE_ARCH_KGDB
+ select HAVE_ARCH_MMAP_RND_BITS
+ select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
+diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
+index aa6808e706470..72610e2d61765 100644
+--- a/arch/powerpc/Makefile
++++ b/arch/powerpc/Makefile
+@@ -173,7 +173,7 @@ else
+ CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power7,$(call cc-option,-mtune=power5))
+ CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mcpu=power5,-mcpu=power4)
+ endif
+-else
++else ifdef CONFIG_PPC_BOOK3E_64
+ CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=powerpc64
+ endif
+
+diff --git a/arch/powerpc/boot/dts/charon.dts b/arch/powerpc/boot/dts/charon.dts
+index 408b486b13dff..cd589539f313f 100644
+--- a/arch/powerpc/boot/dts/charon.dts
++++ b/arch/powerpc/boot/dts/charon.dts
+@@ -35,7 +35,7 @@
+ };
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x08000000>; // 128MB
+ };
+diff --git a/arch/powerpc/boot/dts/digsy_mtc.dts b/arch/powerpc/boot/dts/digsy_mtc.dts
+index 0e5e9d3acf79f..19a14e62e65f4 100644
+--- a/arch/powerpc/boot/dts/digsy_mtc.dts
++++ b/arch/powerpc/boot/dts/digsy_mtc.dts
+@@ -16,7 +16,7 @@
+ model = "intercontrol,digsy-mtc";
+ compatible = "intercontrol,digsy-mtc";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x02000000>; // 32MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi
+index c90702b04a530..48e5cd61599c6 100644
+--- a/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi
++++ b/arch/powerpc/boot/dts/fsl/qoriq-fman3l-0.dtsi
+@@ -79,6 +79,7 @@ fman0: fman@400000 {
+ #size-cells = <0>;
+ compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio";
+ reg = <0xfc000 0x1000>;
++ fsl,erratum-a009885;
+ };
+
+ xmdio0: mdio@fd000 {
+@@ -86,6 +87,7 @@ fman0: fman@400000 {
+ #size-cells = <0>;
+ compatible = "fsl,fman-memac-mdio", "fsl,fman-xmdio";
+ reg = <0xfd000 0x1000>;
++ fsl,erratum-a009885;
+ };
+ };
+
+diff --git a/arch/powerpc/boot/dts/fsl/t1040rdb-rev-a.dts b/arch/powerpc/boot/dts/fsl/t1040rdb-rev-a.dts
+new file mode 100644
+index 0000000000000..73f8c998c64df
+--- /dev/null
++++ b/arch/powerpc/boot/dts/fsl/t1040rdb-rev-a.dts
+@@ -0,0 +1,30 @@
++// SPDX-License-Identifier: GPL-2.0-or-later
++/*
++ * T1040RDB-REV-A Device Tree Source
++ *
++ * Copyright 2014 - 2015 Freescale Semiconductor Inc.
++ *
++ */
++
++#include "t1040rdb.dts"
++
++/ {
++ model = "fsl,T1040RDB-REV-A";
++ compatible = "fsl,T1040RDB-REV-A";
++};
++
++&seville_port0 {
++ label = "ETH5";
++};
++
++&seville_port2 {
++ label = "ETH7";
++};
++
++&seville_port4 {
++ label = "ETH9";
++};
++
++&seville_port6 {
++ label = "ETH11";
++};
+diff --git a/arch/powerpc/boot/dts/fsl/t1040rdb.dts b/arch/powerpc/boot/dts/fsl/t1040rdb.dts
+index af0c8a6f56138..b6733e7e65805 100644
+--- a/arch/powerpc/boot/dts/fsl/t1040rdb.dts
++++ b/arch/powerpc/boot/dts/fsl/t1040rdb.dts
+@@ -119,7 +119,7 @@
+ managed = "in-band-status";
+ phy-handle = <&phy_qsgmii_0>;
+ phy-mode = "qsgmii";
+- label = "ETH5";
++ label = "ETH3";
+ status = "okay";
+ };
+
+@@ -135,7 +135,7 @@
+ managed = "in-band-status";
+ phy-handle = <&phy_qsgmii_2>;
+ phy-mode = "qsgmii";
+- label = "ETH7";
++ label = "ETH5";
+ status = "okay";
+ };
+
+@@ -151,7 +151,7 @@
+ managed = "in-band-status";
+ phy-handle = <&phy_qsgmii_4>;
+ phy-mode = "qsgmii";
+- label = "ETH9";
++ label = "ETH7";
+ status = "okay";
+ };
+
+@@ -167,7 +167,7 @@
+ managed = "in-band-status";
+ phy-handle = <&phy_qsgmii_6>;
+ phy-mode = "qsgmii";
+- label = "ETH11";
++ label = "ETH9";
+ status = "okay";
+ };
+
+diff --git a/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi b/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
+index 099a598c74c00..bfe1ed5be3374 100644
+--- a/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
++++ b/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
+@@ -139,12 +139,12 @@
+ fman@400000 {
+ ethernet@e6000 {
+ phy-handle = <&phy_rgmii_0>;
+- phy-connection-type = "rgmii";
++ phy-connection-type = "rgmii-id";
+ };
+
+ ethernet@e8000 {
+ phy-handle = <&phy_rgmii_1>;
+- phy-connection-type = "rgmii";
++ phy-connection-type = "rgmii-id";
+ };
+
+ mdio0: mdio@fc000 {
+diff --git a/arch/powerpc/boot/dts/lite5200.dts b/arch/powerpc/boot/dts/lite5200.dts
+index cb2782dd6132c..e7b194775d783 100644
+--- a/arch/powerpc/boot/dts/lite5200.dts
++++ b/arch/powerpc/boot/dts/lite5200.dts
+@@ -32,7 +32,7 @@
+ };
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x04000000>; // 64MB
+ };
+diff --git a/arch/powerpc/boot/dts/lite5200b.dts b/arch/powerpc/boot/dts/lite5200b.dts
+index 2b86c81f90485..547cbe726ff23 100644
+--- a/arch/powerpc/boot/dts/lite5200b.dts
++++ b/arch/powerpc/boot/dts/lite5200b.dts
+@@ -31,7 +31,7 @@
+ led4 { gpios = <&gpio_simple 2 1>; };
+ };
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x10000000>; // 256MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/media5200.dts b/arch/powerpc/boot/dts/media5200.dts
+index 61cae9dcddef4..f3188018faceb 100644
+--- a/arch/powerpc/boot/dts/media5200.dts
++++ b/arch/powerpc/boot/dts/media5200.dts
+@@ -32,7 +32,7 @@
+ };
+ };
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x08000000>; // 128MB RAM
+ };
+
+diff --git a/arch/powerpc/boot/dts/mpc5200b.dtsi b/arch/powerpc/boot/dts/mpc5200b.dtsi
+index 648fe31795f49..8b796f3b11da7 100644
+--- a/arch/powerpc/boot/dts/mpc5200b.dtsi
++++ b/arch/powerpc/boot/dts/mpc5200b.dtsi
+@@ -33,7 +33,7 @@
+ };
+ };
+
+- memory: memory {
++ memory: memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x04000000>; // 64MB
+ };
+diff --git a/arch/powerpc/boot/dts/o2d.dts b/arch/powerpc/boot/dts/o2d.dts
+index 24a46f65e5299..e0a8d3034417f 100644
+--- a/arch/powerpc/boot/dts/o2d.dts
++++ b/arch/powerpc/boot/dts/o2d.dts
+@@ -12,7 +12,7 @@
+ model = "ifm,o2d";
+ compatible = "ifm,o2d";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x08000000>; // 128MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/o2d.dtsi b/arch/powerpc/boot/dts/o2d.dtsi
+index 6661955a2be47..b55a9e5bd828c 100644
+--- a/arch/powerpc/boot/dts/o2d.dtsi
++++ b/arch/powerpc/boot/dts/o2d.dtsi
+@@ -19,7 +19,7 @@
+ model = "ifm,o2d";
+ compatible = "ifm,o2d";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x04000000>; // 64MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/o2dnt2.dts b/arch/powerpc/boot/dts/o2dnt2.dts
+index eeba7f5507d5d..c2eedbd1f5fcb 100644
+--- a/arch/powerpc/boot/dts/o2dnt2.dts
++++ b/arch/powerpc/boot/dts/o2dnt2.dts
+@@ -12,7 +12,7 @@
+ model = "ifm,o2dnt2";
+ compatible = "ifm,o2d";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x08000000>; // 128MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/o3dnt.dts b/arch/powerpc/boot/dts/o3dnt.dts
+index fd00396b0593e..e4c1bdd412716 100644
+--- a/arch/powerpc/boot/dts/o3dnt.dts
++++ b/arch/powerpc/boot/dts/o3dnt.dts
+@@ -12,7 +12,7 @@
+ model = "ifm,o3dnt";
+ compatible = "ifm,o2d";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x04000000>; // 64MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/pcm032.dts b/arch/powerpc/boot/dts/pcm032.dts
+index 780e13d99e7b8..1895bc95900cc 100644
+--- a/arch/powerpc/boot/dts/pcm032.dts
++++ b/arch/powerpc/boot/dts/pcm032.dts
+@@ -20,7 +20,7 @@
+ model = "phytec,pcm032";
+ compatible = "phytec,pcm032";
+
+- memory {
++ memory@0 {
+ reg = <0x00000000 0x08000000>; // 128MB
+ };
+
+diff --git a/arch/powerpc/boot/dts/tqm5200.dts b/arch/powerpc/boot/dts/tqm5200.dts
+index 9ed0bc78967e1..5bb25a9e40a01 100644
+--- a/arch/powerpc/boot/dts/tqm5200.dts
++++ b/arch/powerpc/boot/dts/tqm5200.dts
+@@ -32,7 +32,7 @@
+ };
+ };
+
+- memory {
++ memory@0 {
+ device_type = "memory";
+ reg = <0x00000000 0x04000000>; // 64MB
+ };
+diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
+index 6697c5e6682f1..bb549cb1c3e33 100644
+--- a/arch/powerpc/configs/ppc6xx_defconfig
++++ b/arch/powerpc/configs/ppc6xx_defconfig
+@@ -1022,7 +1022,6 @@ CONFIG_NFSD=m
+ CONFIG_NFSD_V3_ACL=y
+ CONFIG_NFSD_V4=y
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_CIFS_UPCALL=y
+ CONFIG_CIFS_XATTR=y
+ CONFIG_CIFS_POSIX=y
+diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig
+index b183629f1bcfb..d0494fbb49617 100644
+--- a/arch/powerpc/configs/pseries_defconfig
++++ b/arch/powerpc/configs/pseries_defconfig
+@@ -190,7 +190,6 @@ CONFIG_HVCS=m
+ CONFIG_VIRTIO_CONSOLE=m
+ CONFIG_IBM_BSR=m
+ CONFIG_RAW_DRIVER=y
+-CONFIG_MAX_RAW_DEVS=1024
+ CONFIG_I2C_CHARDEV=y
+ CONFIG_FB=y
+ CONFIG_FIRMWARE_EDID=y
+diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
+index f5be185cbdf8d..94ad7acfd0565 100644
+--- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h
++++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
+@@ -143,6 +143,8 @@ static __always_inline void update_user_segments(u32 val)
+ update_user_segment(15, val);
+ }
+
++int __init find_free_bat(void);
++unsigned int bat_block_size(unsigned long base, unsigned long top);
+ #endif /* !__ASSEMBLY__ */
+
+ /* We happily ignore the smaller BATs on 601, we don't actually use
+diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
+index 609c80f671943..f8b94f78403f1 100644
+--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
++++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
+@@ -178,6 +178,7 @@ static inline bool pte_user(pte_t pte)
+ #ifndef __ASSEMBLY__
+
+ int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot);
++void unmap_kernel_page(unsigned long va);
+
+ #endif /* !__ASSEMBLY__ */
+
+diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
+index 5d34a8646f081..6866d860d4f30 100644
+--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
++++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
+@@ -1082,6 +1082,8 @@ static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t p
+ return hash__map_kernel_page(ea, pa, prot);
+ }
+
++void unmap_kernel_page(unsigned long va);
++
+ static inline int __meminit vmemmap_create_mapping(unsigned long start,
+ unsigned long page_size,
+ unsigned long phys)
+diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h
+index 947b5b9c44241..a832aeafe5601 100644
+--- a/arch/powerpc/include/asm/fixmap.h
++++ b/arch/powerpc/include/asm/fixmap.h
+@@ -111,8 +111,10 @@ static inline void __set_fixmap(enum fixed_addresses idx,
+ BUILD_BUG_ON(idx >= __end_of_fixed_addresses);
+ else if (WARN_ON(idx >= __end_of_fixed_addresses))
+ return;
+-
+- map_kernel_page(__fix_to_virt(idx), phys, flags);
++ if (pgprot_val(flags))
++ map_kernel_page(__fix_to_virt(idx), phys, flags);
++ else
++ unmap_kernel_page(__fix_to_virt(idx));
+ }
+
+ #define __early_set_fixmap __set_fixmap
+diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
+index 21cc571ea9c2d..5c98a950eca0d 100644
+--- a/arch/powerpc/include/asm/hw_irq.h
++++ b/arch/powerpc/include/asm/hw_irq.h
+@@ -224,6 +224,42 @@ static inline bool arch_irqs_disabled(void)
+ return arch_irqs_disabled_flags(arch_local_save_flags());
+ }
+
++static inline void set_pmi_irq_pending(void)
++{
++ /*
++ * Invoked from PMU callback functions to set PMI bit in the paca.
++ * This has to be called with irq's disabled (via hard_irq_disable()).
++ */
++ if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
++ WARN_ON_ONCE(mfmsr() & MSR_EE);
++
++ get_paca()->irq_happened |= PACA_IRQ_PMI;
++}
++
++static inline void clear_pmi_irq_pending(void)
++{
++ /*
++ * Invoked from PMU callback functions to clear the pending PMI bit
++ * in the paca.
++ */
++ if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
++ WARN_ON_ONCE(mfmsr() & MSR_EE);
++
++ get_paca()->irq_happened &= ~PACA_IRQ_PMI;
++}
++
++static inline bool pmi_irq_pending(void)
++{
++ /*
++ * Invoked from PMU callback functions to check if there is a pending
++ * PMI bit in the paca.
++ */
++ if (get_paca()->irq_happened & PACA_IRQ_PMI)
++ return true;
++
++ return false;
++}
++
+ #ifdef CONFIG_PPC_BOOK3S
+ /*
+ * To support disabling and enabling of irq with PMI, set of
+@@ -408,6 +444,10 @@ static inline void do_hard_irq_enable(void)
+ BUILD_BUG();
+ }
+
++static inline void clear_pmi_irq_pending(void) { }
++static inline void set_pmi_irq_pending(void) { }
++static inline bool pmi_irq_pending(void) { return false; }
++
+ static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
+ {
+ }
+diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
+index a1d238255f077..a07960066b5fa 100644
+--- a/arch/powerpc/include/asm/interrupt.h
++++ b/arch/powerpc/include/asm/interrupt.h
+@@ -567,7 +567,7 @@ DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault);
+ DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault);
+
+ /* hash_utils.c */
+-DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault);
++DECLARE_INTERRUPT_HANDLER(do_hash_fault);
+
+ /* fault.c */
+ DECLARE_INTERRUPT_HANDLER(do_page_fault);
+diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h
+index f130783c8301d..a4fe1292909e6 100644
+--- a/arch/powerpc/include/asm/io.h
++++ b/arch/powerpc/include/asm/io.h
+@@ -359,25 +359,37 @@ static inline void __raw_writeq_be(unsigned long v, volatile void __iomem *addr)
+ */
+ static inline void __raw_rm_writeb(u8 val, volatile void __iomem *paddr)
+ {
+- __asm__ __volatile__("stbcix %0,0,%1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ stbcix %0,0,%1; \
++ .machine pop;"
+ : : "r" (val), "r" (paddr) : "memory");
+ }
+
+ static inline void __raw_rm_writew(u16 val, volatile void __iomem *paddr)
+ {
+- __asm__ __volatile__("sthcix %0,0,%1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ sthcix %0,0,%1; \
++ .machine pop;"
+ : : "r" (val), "r" (paddr) : "memory");
+ }
+
+ static inline void __raw_rm_writel(u32 val, volatile void __iomem *paddr)
+ {
+- __asm__ __volatile__("stwcix %0,0,%1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ stwcix %0,0,%1; \
++ .machine pop;"
+ : : "r" (val), "r" (paddr) : "memory");
+ }
+
+ static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
+ {
+- __asm__ __volatile__("stdcix %0,0,%1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ stdcix %0,0,%1; \
++ .machine pop;"
+ : : "r" (val), "r" (paddr) : "memory");
+ }
+
+@@ -389,7 +401,10 @@ static inline void __raw_rm_writeq_be(u64 val, volatile void __iomem *paddr)
+ static inline u8 __raw_rm_readb(volatile void __iomem *paddr)
+ {
+ u8 ret;
+- __asm__ __volatile__("lbzcix %0,0, %1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ lbzcix %0,0, %1; \
++ .machine pop;"
+ : "=r" (ret) : "r" (paddr) : "memory");
+ return ret;
+ }
+@@ -397,7 +412,10 @@ static inline u8 __raw_rm_readb(volatile void __iomem *paddr)
+ static inline u16 __raw_rm_readw(volatile void __iomem *paddr)
+ {
+ u16 ret;
+- __asm__ __volatile__("lhzcix %0,0, %1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ lhzcix %0,0, %1; \
++ .machine pop;"
+ : "=r" (ret) : "r" (paddr) : "memory");
+ return ret;
+ }
+@@ -405,7 +423,10 @@ static inline u16 __raw_rm_readw(volatile void __iomem *paddr)
+ static inline u32 __raw_rm_readl(volatile void __iomem *paddr)
+ {
+ u32 ret;
+- __asm__ __volatile__("lwzcix %0,0, %1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ lwzcix %0,0, %1; \
++ .machine pop;"
+ : "=r" (ret) : "r" (paddr) : "memory");
+ return ret;
+ }
+@@ -413,7 +434,10 @@ static inline u32 __raw_rm_readl(volatile void __iomem *paddr)
+ static inline u64 __raw_rm_readq(volatile void __iomem *paddr)
+ {
+ u64 ret;
+- __asm__ __volatile__("ldcix %0,0, %1"
++ __asm__ __volatile__(".machine push; \
++ .machine power6; \
++ ldcix %0,0, %1; \
++ .machine pop;"
+ : "=r" (ret) : "r" (paddr) : "memory");
+ return ret;
+ }
+diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
+index 19b6942c6969a..eaf3a562bf1ed 100644
+--- a/arch/powerpc/include/asm/kvm_book3s_64.h
++++ b/arch/powerpc/include/asm/kvm_book3s_64.h
+@@ -39,7 +39,6 @@ struct kvm_nested_guest {
+ pgd_t *shadow_pgtable; /* our page table for this guest */
+ u64 l1_gr_to_hr; /* L1's addr of part'n-scoped table */
+ u64 process_table; /* process table entry for this guest */
+- u64 hfscr; /* HFSCR that the L1 requested for this nested guest */
+ long refcnt; /* number of pointers to this struct */
+ struct mutex tlb_lock; /* serialize page faults and tlbies */
+ struct kvm_nested_guest *next;
+diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
+index 080a7feb77318..0d81a9bf37650 100644
+--- a/arch/powerpc/include/asm/kvm_host.h
++++ b/arch/powerpc/include/asm/kvm_host.h
+@@ -814,6 +814,7 @@ struct kvm_vcpu_arch {
+
+ /* For support of nested guests */
+ struct kvm_nested_guest *nested;
++ u64 nested_hfscr; /* HFSCR that the L1 requested for the nested guest */
+ u32 nested_vcpu_id;
+ gpa_t nested_io_gpr;
+ #endif
+diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
+index f06ae00f2a65e..63ea4693ccea6 100644
+--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
++++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
+@@ -64,6 +64,7 @@ extern int icache_44x_need_flush;
+ #ifndef __ASSEMBLY__
+
+ int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot);
++void unmap_kernel_page(unsigned long va);
+
+ #endif /* !__ASSEMBLY__ */
+
+@@ -193,10 +194,12 @@ static inline pte_t pte_wrprotect(pte_t pte)
+ }
+ #endif
+
++#ifndef pte_mkexec
+ static inline pte_t pte_mkexec(pte_t pte)
+ {
+ return __pte(pte_val(pte) | _PAGE_EXEC);
+ }
++#endif
+
+ #define pmd_none(pmd) (!pmd_val(pmd))
+ #define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD)
+@@ -306,30 +309,29 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+ }
+
+ #define __HAVE_ARCH_PTEP_SET_WRPROTECT
++#ifndef ptep_set_wrprotect
+ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
+ pte_t *ptep)
+ {
+- unsigned long clr = ~pte_val(pte_wrprotect(__pte(~0)));
+- unsigned long set = pte_val(pte_wrprotect(__pte(0)));
+-
+- pte_update(mm, addr, ptep, clr, set, 0);
++ pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
+ }
++#endif
+
++#ifndef __ptep_set_access_flags
+ static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
+ pte_t *ptep, pte_t entry,
+ unsigned long address,
+ int psize)
+ {
+- pte_t pte_set = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte_mkexec(__pte(0)))));
+- pte_t pte_clr = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte_mkexec(__pte(~0)))));
+- unsigned long set = pte_val(entry) & pte_val(pte_set);
+- unsigned long clr = ~pte_val(entry) & ~pte_val(pte_clr);
++ unsigned long set = pte_val(entry) &
++ (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
+ int huge = psize > mmu_virtual_psize ? 1 : 0;
+
+- pte_update(vma->vm_mm, address, ptep, clr, set, huge);
++ pte_update(vma->vm_mm, address, ptep, 0, set, huge);
+
+ flush_tlb_page(vma, address);
+ }
++#endif
+
+ static inline int pte_young(pte_t pte)
+ {
+diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+index fcc48d590d888..1a89ebdc3acc9 100644
+--- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h
++++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+@@ -136,6 +136,28 @@ static inline pte_t pte_mkhuge(pte_t pte)
+
+ #define pte_mkhuge pte_mkhuge
+
++static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,
++ unsigned long clr, unsigned long set, int huge);
++
++static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
++{
++ pte_update(mm, addr, ptep, 0, _PAGE_RO, 0);
++}
++#define ptep_set_wrprotect ptep_set_wrprotect
++
++static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
++ pte_t entry, unsigned long address, int psize)
++{
++ unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_EXEC);
++ unsigned long clr = ~pte_val(entry) & _PAGE_RO;
++ int huge = psize > mmu_virtual_psize ? 1 : 0;
++
++ pte_update(vma->vm_mm, address, ptep, clr, set, huge);
++
++ flush_tlb_page(vma, address);
++}
++#define __ptep_set_access_flags __ptep_set_access_flags
++
+ static inline unsigned long pgd_leaf_size(pgd_t pgd)
+ {
+ if (pgd_val(pgd) & _PMD_PAGE_8M)
+diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
+index d081704b13fb9..2225991c69b55 100644
+--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
++++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
+@@ -118,11 +118,6 @@ static inline pte_t pte_wrprotect(pte_t pte)
+ return __pte(pte_val(pte) & ~_PAGE_RW);
+ }
+
+-static inline pte_t pte_mkexec(pte_t pte)
+-{
+- return __pte(pte_val(pte) | _PAGE_EXEC);
+-}
+-
+ #define PMD_BAD_BITS (PTE_TABLE_SIZE-1)
+ #define PUD_BAD_BITS (PMD_TABLE_SIZE-1)
+
+@@ -313,6 +308,7 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
+ #define __swp_entry_to_pte(x) __pte((x).val)
+
+ int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot);
++void unmap_kernel_page(unsigned long va);
+ extern int __meminit vmemmap_create_mapping(unsigned long start,
+ unsigned long page_size,
+ unsigned long phys);
+diff --git a/arch/powerpc/include/asm/nohash/pte-book3e.h b/arch/powerpc/include/asm/nohash/pte-book3e.h
+index 813918f407653..f798640422c2d 100644
+--- a/arch/powerpc/include/asm/nohash/pte-book3e.h
++++ b/arch/powerpc/include/asm/nohash/pte-book3e.h
+@@ -48,7 +48,7 @@
+ #define _PAGE_WRITETHRU 0x800000 /* W: cache write-through */
+
+ /* "Higher level" linux bit combinations */
+-#define _PAGE_EXEC _PAGE_BAP_UX /* .. and was cache cleaned */
++#define _PAGE_EXEC (_PAGE_BAP_SX | _PAGE_BAP_UX) /* .. and was cache cleaned */
+ #define _PAGE_RW (_PAGE_BAP_SW | _PAGE_BAP_UW) /* User write permission */
+ #define _PAGE_KERNEL_RW (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY)
+ #define _PAGE_KERNEL_RO (_PAGE_BAP_SR)
+@@ -93,11 +93,11 @@
+ /* Permission masks used to generate the __P and __S table */
+ #define PAGE_NONE __pgprot(_PAGE_BASE)
+ #define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW)
+-#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC)
++#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_BAP_UX)
+ #define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER)
+-#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC)
++#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_BAP_UX)
+ #define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER)
+-#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC)
++#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_BAP_UX)
+
+ #ifndef __ASSEMBLY__
+ static inline pte_t pte_mkprivileged(pte_t pte)
+@@ -113,6 +113,16 @@ static inline pte_t pte_mkuser(pte_t pte)
+ }
+
+ #define pte_mkuser pte_mkuser
++
++static inline pte_t pte_mkexec(pte_t pte)
++{
++ if (pte_val(pte) & _PAGE_BAP_UR)
++ return __pte((pte_val(pte) & ~_PAGE_BAP_SX) | _PAGE_BAP_UX);
++ else
++ return __pte((pte_val(pte) & ~_PAGE_BAP_UX) | _PAGE_BAP_SX);
++}
++#define pte_mkexec pte_mkexec
++
+ #endif /* __ASSEMBLY__ */
+
+ #endif /* __KERNEL__ */
+diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h
+index 254687258f42b..f2c5c26869f1a 100644
+--- a/arch/powerpc/include/asm/page.h
++++ b/arch/powerpc/include/asm/page.h
+@@ -132,7 +132,11 @@ static inline bool pfn_valid(unsigned long pfn)
+ #define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr))
+ #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
+
+-#define virt_addr_valid(kaddr) pfn_valid(virt_to_pfn(kaddr))
++#define virt_addr_valid(vaddr) ({ \
++ unsigned long _addr = (unsigned long)vaddr; \
++ _addr >= PAGE_OFFSET && _addr < (unsigned long)high_memory && \
++ pfn_valid(virt_to_pfn(_addr)); \
++})
+
+ /*
+ * On Book-E parts we need __va to parse the device tree and we can't
+diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h
+index bcb7b5f917be6..b325022ffa2b0 100644
+--- a/arch/powerpc/include/asm/paravirt.h
++++ b/arch/powerpc/include/asm/paravirt.h
+@@ -97,7 +97,23 @@ static inline bool vcpu_is_preempted(int cpu)
+
+ #ifdef CONFIG_PPC_SPLPAR
+ if (!is_kvm_guest()) {
+- int first_cpu = cpu_first_thread_sibling(smp_processor_id());
++ int first_cpu;
++
++ /*
++ * The result of vcpu_is_preempted() is used in a
++ * speculative way, and is always subject to invalidation
++ * by events internal and external to Linux. While we can
++ * be called in preemptable context (in the Linux sense),
++ * we're not accessing per-cpu resources in a way that can
++ * race destructively with Linux scheduler preemption and
++ * migration, and callers can tolerate the potential for
++ * error introduced by sampling the CPU index without
++ * pinning the task to it. So it is permissible to use
++ * raw_smp_processor_id() here to defeat the preempt debug
++ * warnings that can arise from using smp_processor_id()
++ * in arbitrary contexts.
++ */
++ first_cpu = cpu_first_thread_sibling(raw_smp_processor_id());
+
+ /*
+ * Preemption can only happen at core granularity. This CPU
+diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
+index baea657bc8687..bca31a61e57f8 100644
+--- a/arch/powerpc/include/asm/ppc-opcode.h
++++ b/arch/powerpc/include/asm/ppc-opcode.h
+@@ -498,6 +498,7 @@
+ #define PPC_RAW_LDX(r, base, b) (0x7c00002a | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+ #define PPC_RAW_LHZ(r, base, i) (0xa0000000 | ___PPC_RT(r) | ___PPC_RA(base) | IMM_L(i))
+ #define PPC_RAW_LHBRX(r, base, b) (0x7c00062c | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
++#define PPC_RAW_LWBRX(r, base, b) (0x7c00042c | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+ #define PPC_RAW_LDBRX(r, base, b) (0x7c000428 | ___PPC_RT(r) | ___PPC_RA(base) | ___PPC_RB(b))
+ #define PPC_RAW_STWCX(s, a, b) (0x7c00012d | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+ #define PPC_RAW_CMPWI(a, i) (0x2c000000 | ___PPC_RA(a) | IMM_L(i))
+diff --git a/arch/powerpc/include/asm/set_memory.h b/arch/powerpc/include/asm/set_memory.h
+index b040094f79202..7ebc807aa8cc8 100644
+--- a/arch/powerpc/include/asm/set_memory.h
++++ b/arch/powerpc/include/asm/set_memory.h
+@@ -6,6 +6,8 @@
+ #define SET_MEMORY_RW 1
+ #define SET_MEMORY_NX 2
+ #define SET_MEMORY_X 3
++#define SET_MEMORY_NP 4 /* Set memory non present */
++#define SET_MEMORY_P 5 /* Set memory present */
+
+ int change_memory_attr(unsigned long addr, int numpages, long action);
+
+@@ -29,6 +31,14 @@ static inline int set_memory_x(unsigned long addr, int numpages)
+ return change_memory_attr(addr, numpages, SET_MEMORY_X);
+ }
+
+-int set_memory_attr(unsigned long addr, int numpages, pgprot_t prot);
++static inline int set_memory_np(unsigned long addr, int numpages)
++{
++ return change_memory_attr(addr, numpages, SET_MEMORY_NP);
++}
++
++static inline int set_memory_p(unsigned long addr, int numpages)
++{
++ return change_memory_attr(addr, numpages, SET_MEMORY_P);
++}
+
+ #endif
+diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h
+index c60ebd04b2ed9..61b968d9fba7c 100644
+--- a/arch/powerpc/include/asm/syscall.h
++++ b/arch/powerpc/include/asm/syscall.h
+@@ -90,7 +90,7 @@ static inline void syscall_get_arguments(struct task_struct *task,
+ unsigned long val, mask = -1UL;
+ unsigned int n = 6;
+
+- if (is_32bit_task())
++ if (is_tsk_32bit_task(task))
+ mask = 0xffffffff;
+
+ while (n--) {
+@@ -115,7 +115,7 @@ static inline void syscall_set_arguments(struct task_struct *task,
+
+ static inline int syscall_get_arch(struct task_struct *task)
+ {
+- if (is_32bit_task())
++ if (is_tsk_32bit_task(task))
+ return AUDIT_ARCH_PPC;
+ else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
+ return AUDIT_ARCH_PPC64LE;
+diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
+index b4ec6c7dd72ee..2a4ea0e213a92 100644
+--- a/arch/powerpc/include/asm/thread_info.h
++++ b/arch/powerpc/include/asm/thread_info.h
+@@ -165,8 +165,10 @@ static inline bool test_thread_local_flags(unsigned int flags)
+
+ #ifdef CONFIG_COMPAT
+ #define is_32bit_task() (test_thread_flag(TIF_32BIT))
++#define is_tsk_32bit_task(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT))
+ #else
+ #define is_32bit_task() (IS_ENABLED(CONFIG_PPC32))
++#define is_tsk_32bit_task(tsk) (IS_ENABLED(CONFIG_PPC32))
+ #endif
+
+ #if defined(CONFIG_PPC64)
+diff --git a/arch/powerpc/include/asm/timex.h b/arch/powerpc/include/asm/timex.h
+index fa2e76e4093a3..14b4489de52c5 100644
+--- a/arch/powerpc/include/asm/timex.h
++++ b/arch/powerpc/include/asm/timex.h
+@@ -19,6 +19,7 @@ static inline cycles_t get_cycles(void)
+ {
+ return mftb();
+ }
++#define get_cycles get_cycles
+
+ #endif /* __KERNEL__ */
+ #endif /* _ASM_POWERPC_TIMEX_H */
+diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
+index 22c79ab400060..b2680070d65d6 100644
+--- a/arch/powerpc/include/asm/uaccess.h
++++ b/arch/powerpc/include/asm/uaccess.h
+@@ -125,8 +125,11 @@ do { \
+ */
+ #define __get_user_atomic_128_aligned(kaddr, uaddr, err) \
+ __asm__ __volatile__( \
++ ".machine push\n" \
++ ".machine altivec\n" \
+ "1: lvx 0,0,%1 # get user\n" \
+ " stvx 0,0,%2 # put kernel\n" \
++ ".machine pop\n" \
+ "2:\n" \
+ ".section .fixup,\"ax\"\n" \
+ "3: li %0,%3\n" \
+diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
+index 7be36c1e1db6d..b1b23b4d56ba2 100644
+--- a/arch/powerpc/kernel/Makefile
++++ b/arch/powerpc/kernel/Makefile
+@@ -11,6 +11,7 @@ CFLAGS_prom_init.o += -fPIC
+ CFLAGS_btext.o += -fPIC
+ endif
+
++CFLAGS_early_32.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
+ CFLAGS_cputable.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
+ CFLAGS_prom_init.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
+ CFLAGS_btext.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
+@@ -196,3 +197,6 @@ clean-files := vmlinux.lds
+ # Force dependency (incbin is bad)
+ $(obj)/vdso32_wrapper.o : $(obj)/vdso32/vdso32.so.dbg
+ $(obj)/vdso64_wrapper.o : $(obj)/vdso64/vdso64.so.dbg
++
++# for cleaning
++subdir- += vdso32 vdso64
+diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c
+index 803c2a45b22ac..1cffb5e7c38d6 100644
+--- a/arch/powerpc/kernel/btext.c
++++ b/arch/powerpc/kernel/btext.c
+@@ -241,8 +241,10 @@ int __init btext_find_display(int allow_nonstdout)
+ rc = btext_initialize(np);
+ printk("result: %d\n", rc);
+ }
+- if (rc == 0)
++ if (rc == 0) {
++ of_node_put(np);
+ break;
++ }
+ }
+ return rc;
+ }
+diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c
+index b7ceb041743c9..60f5fc14aa235 100644
+--- a/arch/powerpc/kernel/fadump.c
++++ b/arch/powerpc/kernel/fadump.c
+@@ -1641,6 +1641,14 @@ int __init setup_fadump(void)
+ else if (fw_dump.reserve_dump_area_size)
+ fw_dump.ops->fadump_init_mem_struct(&fw_dump);
+
++ /*
++ * In case of panic, fadump is triggered via ppc_panic_event()
++ * panic notifier. Setting crash_kexec_post_notifiers to 'true'
++ * lets panic() function take crash friendly path before panic
++ * notifiers are invoked.
++ */
++ crash_kexec_post_notifiers = true;
++
+ return 1;
+ }
+ subsys_initcall(setup_fadump);
+diff --git a/arch/powerpc/kernel/firmware.c b/arch/powerpc/kernel/firmware.c
+index c7022c41cc314..20328f72f9f2b 100644
+--- a/arch/powerpc/kernel/firmware.c
++++ b/arch/powerpc/kernel/firmware.c
+@@ -31,11 +31,10 @@ int __init check_kvm_guest(void)
+ if (!hyper_node)
+ return 0;
+
+- if (!of_device_is_compatible(hyper_node, "linux,kvm"))
+- return 0;
+-
+- static_branch_enable(&kvm_guest);
++ if (of_device_is_compatible(hyper_node, "linux,kvm"))
++ static_branch_enable(&kvm_guest);
+
++ of_node_put(hyper_node);
+ return 0;
+ }
+ core_initcall(check_kvm_guest); // before kvm_guest_init()
+diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
+index 6b1ec9e3541b9..349c4a820231b 100644
+--- a/arch/powerpc/kernel/head_32.h
++++ b/arch/powerpc/kernel/head_32.h
+@@ -202,11 +202,11 @@ vmap_stack_overflow:
+ mfspr r1, SPRN_SPRG_THREAD
+ lwz r1, TASK_CPU - THREAD(r1)
+ slwi r1, r1, 3
+- addis r1, r1, emergency_ctx@ha
++ addis r1, r1, emergency_ctx-PAGE_OFFSET@ha
+ #else
+- lis r1, emergency_ctx@ha
++ lis r1, emergency_ctx-PAGE_OFFSET@ha
+ #endif
+- lwz r1, emergency_ctx@l(r1)
++ lwz r1, emergency_ctx-PAGE_OFFSET@l(r1)
+ addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
+ EXCEPTION_PROLOG_2 0 vmap_stack_overflow
+ prepare_transfer_to_handler
+diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
+index 7d72ee5ab387c..e783860bea838 100644
+--- a/arch/powerpc/kernel/head_40x.S
++++ b/arch/powerpc/kernel/head_40x.S
+@@ -27,6 +27,7 @@
+
+ #include <linux/init.h>
+ #include <linux/pgtable.h>
++#include <linux/sizes.h>
+ #include <asm/processor.h>
+ #include <asm/page.h>
+ #include <asm/mmu.h>
+@@ -650,7 +651,7 @@ start_here:
+ b . /* prevent prefetch past rfi */
+
+ /* Set up the initial MMU state so we can do the first level of
+- * kernel initialization. This maps the first 16 MBytes of memory 1:1
++ * kernel initialization. This maps the first 32 MBytes of memory 1:1
+ * virtual to physical and more importantly sets the cache mode.
+ */
+ initial_mmu:
+@@ -687,6 +688,12 @@ initial_mmu:
+ tlbwe r4,r0,TLB_DATA /* Load the data portion of the entry */
+ tlbwe r3,r0,TLB_TAG /* Load the tag portion of the entry */
+
++ li r0,62 /* TLB slot 62 */
++ addis r4,r4,SZ_16M@h
++ addis r3,r3,SZ_16M@h
++ tlbwe r4,r0,TLB_DATA /* Load the data portion of the entry */
++ tlbwe r3,r0,TLB_TAG /* Load the tag portion of the entry */
++
+ isync
+
+ /* Establish the exception vector base
+diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
+index 9bdb95f5694f7..0d073b9fd52c5 100644
+--- a/arch/powerpc/kernel/head_8xx.S
++++ b/arch/powerpc/kernel/head_8xx.S
+@@ -733,6 +733,7 @@ _GLOBAL(mmu_pin_tlb)
+ #ifdef CONFIG_PIN_TLB_DATA
+ LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET)
+ LOAD_REG_IMMEDIATE(r7, MI_SVALID | MI_PS8MEG | _PMD_ACCESSED)
++ li r8, 0
+ #ifdef CONFIG_PIN_TLB_IMMR
+ li r0, 3
+ #else
+@@ -741,26 +742,26 @@ _GLOBAL(mmu_pin_tlb)
+ mtctr r0
+ cmpwi r4, 0
+ beq 4f
+- LOAD_REG_IMMEDIATE(r8, 0xf0 | _PAGE_RO | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT)
+ LOAD_REG_ADDR(r9, _sinittext)
+
+ 2: ori r0, r6, MD_EVALID
++ ori r12, r8, 0xf0 | _PAGE_RO | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT
+ mtspr SPRN_MD_CTR, r5
+ mtspr SPRN_MD_EPN, r0
+ mtspr SPRN_MD_TWC, r7
+- mtspr SPRN_MD_RPN, r8
++ mtspr SPRN_MD_RPN, r12
+ addi r5, r5, 0x100
+ addis r6, r6, SZ_8M@h
+ addis r8, r8, SZ_8M@h
+ cmplw r6, r9
+ bdnzt lt, 2b
+-
+-4: LOAD_REG_IMMEDIATE(r8, 0xf0 | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT)
++4:
+ 2: ori r0, r6, MD_EVALID
++ ori r12, r8, 0xf0 | _PAGE_DIRTY | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT
+ mtspr SPRN_MD_CTR, r5
+ mtspr SPRN_MD_EPN, r0
+ mtspr SPRN_MD_TWC, r7
+- mtspr SPRN_MD_RPN, r8
++ mtspr SPRN_MD_RPN, r12
+ addi r5, r5, 0x100
+ addis r6, r6, SZ_8M@h
+ addis r8, r8, SZ_8M@h
+@@ -781,7 +782,7 @@ _GLOBAL(mmu_pin_tlb)
+ #endif
+ #if defined(CONFIG_PIN_TLB_IMMR) || defined(CONFIG_PIN_TLB_DATA)
+ lis r0, (MD_RSV4I | MD_TWAM)@h
+- mtspr SPRN_MI_CTR, r0
++ mtspr SPRN_MD_CTR, r0
+ #endif
+ mtspr SPRN_SRR1, r10
+ mtspr SPRN_SRR0, r11
+diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
+index 68e5c0a7e99d1..2e2a8211b17be 100644
+--- a/arch/powerpc/kernel/head_book3s_32.S
++++ b/arch/powerpc/kernel/head_book3s_32.S
+@@ -421,14 +421,14 @@ InstructionTLBMiss:
+ */
+ /* Get PTE (linux-style) and check access */
+ mfspr r3,SPRN_IMISS
+-#ifdef CONFIG_MODULES
++#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
+ lis r1, TASK_SIZE@h /* check if kernel address */
+ cmplw 0,r1,r3
+ #endif
+ mfspr r2, SPRN_SDR1
+ li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC | _PAGE_USER
+ rlwinm r2, r2, 28, 0xfffff000
+-#ifdef CONFIG_MODULES
++#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
+ bgt- 112f
+ lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
+ li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
+diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
+index e5503420b6c6d..ef8d1b1c234e7 100644
+--- a/arch/powerpc/kernel/head_booke.h
++++ b/arch/powerpc/kernel/head_booke.h
+@@ -465,12 +465,21 @@ label:
+ bl do_page_fault; \
+ b interrupt_return
+
++/*
++ * Instruction TLB Error interrupt handlers may call InstructionStorage
++ * directly without clearing ESR, so the ESR at this point may be left over
++ * from a prior interrupt.
++ *
++ * In any case, do_page_fault for BOOK3E does not use ESR and always expects
++ * dsisr to be 0. ESR_DST from a prior store in particular would confuse fault
++ * handling.
++ */
+ #define INSTRUCTION_STORAGE_EXCEPTION \
+ START_EXCEPTION(InstructionStorage) \
+- NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \
+- mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
++ NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \
++ li r5,0; /* Store 0 in regs->esr (dsisr) */ \
+ stw r5,_ESR(r11); \
+- stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
++ stw r12, _DEAR(r11); /* Set regs->dear (dar) to SRR0 */ \
+ prepare_transfer_to_handler; \
+ bl do_page_fault; \
+ b interrupt_return
+diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
+index de10a26972581..df048e331cbfe 100644
+--- a/arch/powerpc/kernel/interrupt.c
++++ b/arch/powerpc/kernel/interrupt.c
+@@ -148,7 +148,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
+ */
+ if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
+ unlikely(MSR_TM_TRANSACTIONAL(regs->msr)))
+- current_thread_info()->flags |= _TIF_RESTOREALL;
++ set_bits(_TIF_RESTOREALL, &current_thread_info()->flags);
+
+ /*
+ * If the system call was made with a transaction active, doom it and
+@@ -266,7 +266,7 @@ static void check_return_regs_valid(struct pt_regs *regs)
+ if (trap_is_scv(regs))
+ return;
+
+- trap = regs->trap;
++ trap = TRAP(regs);
+ // EE in HV mode sets HSRRs like 0xea0
+ if (cpu_has_feature(CPU_FTR_HVMODE) && trap == INTERRUPT_EXTERNAL)
+ trap = 0xea0;
+diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
+index ec950b08a8dcc..4c6d1a8dcefed 100644
+--- a/arch/powerpc/kernel/interrupt_64.S
++++ b/arch/powerpc/kernel/interrupt_64.S
+@@ -30,21 +30,25 @@ COMPAT_SYS_CALL_TABLE:
+ .ifc \srr,srr
+ mfspr r11,SPRN_SRR0
+ ld r12,_NIP(r1)
++ clrrdi r11,r11,2
++ clrrdi r12,r12,2
+ 100: tdne r11,r12
+- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
++ EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_SRR1
+ ld r12,_MSR(r1)
+ 100: tdne r11,r12
+- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
++ EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .else
+ mfspr r11,SPRN_HSRR0
+ ld r12,_NIP(r1)
++ clrrdi r11,r11,2
++ clrrdi r12,r12,2
+ 100: tdne r11,r12
+- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
++ EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_HSRR1
+ ld r12,_MSR(r1)
+ 100: tdne r11,r12
+- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
++ EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .endif
+ #endif
+ .endm
+diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c
+index 617eba82531cb..6568823cf3063 100644
+--- a/arch/powerpc/kernel/kvm.c
++++ b/arch/powerpc/kernel/kvm.c
+@@ -669,7 +669,8 @@ static void __init kvm_use_magic_page(void)
+ on_each_cpu(kvm_map_magic_page, &features, 1);
+
+ /* Quick self-test to see if the mapping works */
+- if (!fault_in_pages_readable((const char *)KVM_MAGIC_PAGE, sizeof(u32))) {
++ if (fault_in_readable((const char __user *)KVM_MAGIC_PAGE,
++ sizeof(u32))) {
+ kvm_patching_worked = false;
+ return;
+ }
+diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c
+index ed04a3ba66fe8..40a583e9d3c70 100644
+--- a/arch/powerpc/kernel/module.c
++++ b/arch/powerpc/kernel/module.c
+@@ -90,16 +90,17 @@ int module_finalize(const Elf_Ehdr *hdr,
+ }
+
+ static __always_inline void *
+-__module_alloc(unsigned long size, unsigned long start, unsigned long end)
++__module_alloc(unsigned long size, unsigned long start, unsigned long end, bool nowarn)
+ {
+ pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC;
++ gfp_t gfp = GFP_KERNEL | (nowarn ? __GFP_NOWARN : 0);
+
+ /*
+ * Don't do huge page allocations for modules yet until more testing
+ * is done. STRICT_MODULE_RWX may require extra work to support this
+ * too.
+ */
+- return __vmalloc_node_range(size, 1, start, end, GFP_KERNEL, prot,
++ return __vmalloc_node_range(size, 1, start, end, gfp, prot,
+ VM_FLUSH_RESET_PERMS | VM_NO_HUGE_VMAP,
+ NUMA_NO_NODE, __builtin_return_address(0));
+ }
+@@ -114,13 +115,13 @@ void *module_alloc(unsigned long size)
+
+ /* First try within 32M limit from _etext to avoid branch trampolines */
+ if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit)
+- ptr = __module_alloc(size, limit, MODULES_END);
++ ptr = __module_alloc(size, limit, MODULES_END, true);
+
+ if (!ptr)
+- ptr = __module_alloc(size, MODULES_VADDR, MODULES_END);
++ ptr = __module_alloc(size, MODULES_VADDR, MODULES_END, false);
+
+ return ptr;
+ #else
+- return __module_alloc(size, VMALLOC_START, VMALLOC_END);
++ return __module_alloc(size, VMALLOC_START, VMALLOC_END, false);
+ #endif
+ }
+diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c
+index 6baa676e7cb60..5d77d3f5fbb56 100644
+--- a/arch/powerpc/kernel/module_64.c
++++ b/arch/powerpc/kernel/module_64.c
+@@ -422,11 +422,17 @@ static inline int create_stub(const Elf64_Shdr *sechdrs,
+ const char *name)
+ {
+ long reladdr;
++ func_desc_t desc;
++ int i;
+
+ if (is_mprofile_ftrace_call(name))
+ return create_ftrace_stub(entry, addr, me);
+
+- memcpy(entry->jump, ppc64_stub_insns, sizeof(ppc64_stub_insns));
++ for (i = 0; i < sizeof(ppc64_stub_insns) / sizeof(u32); i++) {
++ if (patch_instruction(&entry->jump[i],
++ ppc_inst(ppc64_stub_insns[i])))
++ return 0;
++ }
+
+ /* Stub uses address relative to r2. */
+ reladdr = (unsigned long)entry - my_r2(sechdrs, me);
+@@ -437,10 +443,24 @@ static inline int create_stub(const Elf64_Shdr *sechdrs,
+ }
+ pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
+
+- entry->jump[0] |= PPC_HA(reladdr);
+- entry->jump[1] |= PPC_LO(reladdr);
+- entry->funcdata = func_desc(addr);
+- entry->magic = STUB_MAGIC;
++ if (patch_instruction(&entry->jump[0],
++ ppc_inst(entry->jump[0] | PPC_HA(reladdr))))
++ return 0;
++
++ if (patch_instruction(&entry->jump[1],
++ ppc_inst(entry->jump[1] | PPC_LO(reladdr))))
++ return 0;
++
++ // func_desc_t is 8 bytes if ABIv2, else 16 bytes
++ desc = func_desc(addr);
++ for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) {
++ if (patch_instruction(((u32 *)&entry->funcdata) + i,
++ ppc_inst(((u32 *)(&desc))[i])))
++ return 0;
++ }
++
++ if (patch_instruction(&entry->magic, ppc_inst(STUB_MAGIC)))
++ return 0;
+
+ return 1;
+ }
+@@ -495,8 +515,11 @@ static int restore_r2(const char *name, u32 *instruction, struct module *me)
+ me->name, *instruction, instruction);
+ return 0;
+ }
++
+ /* ld r2,R2_STACK_OFFSET(r1) */
+- *instruction = PPC_INST_LD_TOC;
++ if (patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC)))
++ return 0;
++
+ return 1;
+ }
+
+@@ -636,9 +659,12 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
+ }
+
+ /* Only replace bits 2 through 26 */
+- *(uint32_t *)location
+- = (*(uint32_t *)location & ~0x03fffffc)
++ value = (*(uint32_t *)location & ~0x03fffffc)
+ | (value & 0x03fffffc);
++
++ if (patch_instruction((u32 *)location, ppc_inst(value)))
++ return -EFAULT;
++
+ break;
+
+ case R_PPC64_REL64:
+diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
+index 18b04b08b9833..f845065c860e3 100644
+--- a/arch/powerpc/kernel/prom_init.c
++++ b/arch/powerpc/kernel/prom_init.c
+@@ -2991,7 +2991,7 @@ static void __init fixup_device_tree_efika_add_phy(void)
+
+ /* Check if the phy-handle property exists - bail if it does */
+ rv = prom_getprop(node, "phy-handle", prop, sizeof(prop));
+- if (!rv)
++ if (rv <= 0)
+ return;
+
+ /*
+diff --git a/arch/powerpc/kernel/reloc_64.S b/arch/powerpc/kernel/reloc_64.S
+index 02d4719bf43a8..232e4549defe1 100644
+--- a/arch/powerpc/kernel/reloc_64.S
++++ b/arch/powerpc/kernel/reloc_64.S
+@@ -8,8 +8,10 @@
+ #include <asm/ppc_asm.h>
+
+ RELA = 7
+-RELACOUNT = 0x6ffffff9
++RELASZ = 8
++RELAENT = 9
+ R_PPC64_RELATIVE = 22
++R_PPC64_UADDR64 = 43
+
+ /*
+ * r3 = desired final address of kernel
+@@ -25,29 +27,38 @@ _GLOBAL(relocate)
+ add r9,r9,r12 /* r9 has runtime addr of .rela.dyn section */
+ ld r10,(p_st - 0b)(r12)
+ add r10,r10,r12 /* r10 has runtime addr of _stext */
++ ld r13,(p_sym - 0b)(r12)
++ add r13,r13,r12 /* r13 has runtime addr of .dynsym */
+
+ /*
+- * Scan the dynamic section for the RELA and RELACOUNT entries.
++ * Scan the dynamic section for the RELA, RELASZ and RELAENT entries.
+ */
+ li r7,0
+ li r8,0
+-1: ld r6,0(r11) /* get tag */
++.Ltags:
++ ld r6,0(r11) /* get tag */
+ cmpdi r6,0
+- beq 4f /* end of list */
++ beq .Lend_of_list /* end of list */
+ cmpdi r6,RELA
+ bne 2f
+ ld r7,8(r11) /* get RELA pointer in r7 */
+- b 3f
+-2: addis r6,r6,(-RELACOUNT)@ha
+- cmpdi r6,RELACOUNT@l
++ b 4f
++2: cmpdi r6,RELASZ
+ bne 3f
+- ld r8,8(r11) /* get RELACOUNT value in r8 */
+-3: addi r11,r11,16
+- b 1b
+-4: cmpdi r7,0 /* check we have both RELA and RELACOUNT */
++ ld r8,8(r11) /* get RELASZ value in r8 */
++ b 4f
++3: cmpdi r6,RELAENT
++ bne 4f
++ ld r12,8(r11) /* get RELAENT value in r12 */
++4: addi r11,r11,16
++ b .Ltags
++.Lend_of_list:
++ cmpdi r7,0 /* check we have RELA, RELASZ, RELAENT */
+ cmpdi cr1,r8,0
+- beq 6f
+- beq cr1,6f
++ beq .Lout
++ beq cr1,.Lout
++ cmpdi r12,0
++ beq .Lout
+
+ /*
+ * Work out linktime address of _stext and hence the
+@@ -62,23 +73,39 @@ _GLOBAL(relocate)
+
+ /*
+ * Run through the list of relocations and process the
+- * R_PPC64_RELATIVE ones.
++ * R_PPC64_RELATIVE and R_PPC64_UADDR64 ones.
+ */
++ divd r8,r8,r12 /* RELASZ / RELAENT */
+ mtctr r8
+-5: ld r0,8(9) /* ELF64_R_TYPE(reloc->r_info) */
++.Lrels: ld r0,8(r9) /* ELF64_R_TYPE(reloc->r_info) */
+ cmpdi r0,R_PPC64_RELATIVE
+- bne 6f
++ bne .Luaddr64
+ ld r6,0(r9) /* reloc->r_offset */
+ ld r0,16(r9) /* reloc->r_addend */
++ b .Lstore
++.Luaddr64:
++ srdi r14,r0,32 /* ELF64_R_SYM(reloc->r_info) */
++ clrldi r0,r0,32
++ cmpdi r0,R_PPC64_UADDR64
++ bne .Lnext
++ ld r6,0(r9)
++ ld r0,16(r9)
++ mulli r14,r14,24 /* 24 == sizeof(elf64_sym) */
++ add r14,r14,r13 /* elf64_sym[ELF64_R_SYM] */
++ ld r14,8(r14)
++ add r0,r0,r14
++.Lstore:
+ add r0,r0,r3
+ stdx r0,r7,r6
+- addi r9,r9,24
+- bdnz 5b
+-
+-6: blr
++.Lnext:
++ add r9,r9,r12
++ bdnz .Lrels
++.Lout:
++ blr
+
+ .balign 8
+ p_dyn: .8byte __dynamic_start - 0b
+ p_rela: .8byte __rela_dyn_start - 0b
++p_sym: .8byte __dynamic_symtab - 0b
+ p_st: .8byte _stext - 0b
+
+diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
+index ff80bbad22a58..e18a725a8e5d3 100644
+--- a/arch/powerpc/kernel/rtas.c
++++ b/arch/powerpc/kernel/rtas.c
+@@ -1235,6 +1235,12 @@ int __init early_init_dt_scan_rtas(unsigned long node,
+ entryp = of_get_flat_dt_prop(node, "linux,rtas-entry", NULL);
+ sizep = of_get_flat_dt_prop(node, "rtas-size", NULL);
+
++#ifdef CONFIG_PPC64
++ /* need this feature to decide the crashkernel offset */
++ if (of_get_flat_dt_prop(node, "ibm,hypertas-functions", NULL))
++ powerpc_firmware_features |= FW_FEATURE_LPAR;
++#endif
++
+ if (basep && entryp && sizep) {
+ rtas.base = *basep;
+ rtas.entry = *entryp;
+diff --git a/arch/powerpc/kernel/secvar-sysfs.c b/arch/powerpc/kernel/secvar-sysfs.c
+index a0a78aba2083e..1ee4640a26413 100644
+--- a/arch/powerpc/kernel/secvar-sysfs.c
++++ b/arch/powerpc/kernel/secvar-sysfs.c
+@@ -26,15 +26,18 @@ static ssize_t format_show(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *format;
+
+ node = of_find_compatible_node(NULL, NULL, "ibm,secvar-backend");
+- if (!of_device_is_available(node))
+- return -ENODEV;
++ if (!of_device_is_available(node)) {
++ rc = -ENODEV;
++ goto out;
++ }
+
+ rc = of_property_read_string(node, "format", &format);
+ if (rc)
+- return rc;
++ goto out;
+
+ rc = sprintf(buf, "%s\n", format);
+
++out:
+ of_node_put(node);
+
+ return rc;
+diff --git a/arch/powerpc/kernel/signal.h b/arch/powerpc/kernel/signal.h
+index 1f07317964e49..618aeccdf6918 100644
+--- a/arch/powerpc/kernel/signal.h
++++ b/arch/powerpc/kernel/signal.h
+@@ -25,8 +25,14 @@ static inline int __get_user_sigset(sigset_t *dst, const sigset_t __user *src)
+
+ return __get_user(dst->sig[0], (u64 __user *)&src->sig[0]);
+ }
+-#define unsafe_get_user_sigset(dst, src, label) \
+- unsafe_get_user((dst)->sig[0], (u64 __user *)&(src)->sig[0], label)
++#define unsafe_get_user_sigset(dst, src, label) do { \
++ sigset_t *__dst = dst; \
++ const sigset_t __user *__src = src; \
++ int i; \
++ \
++ for (i = 0; i < _NSIG_WORDS; i++) \
++ unsafe_get_user(__dst->sig[i], &__src->sig[i], label); \
++} while (0)
+
+ #ifdef CONFIG_VSX
+ extern unsigned long copy_vsx_to_user(void __user *to,
+diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c
+index 0608581967f09..3e053e2fd6b69 100644
+--- a/arch/powerpc/kernel/signal_32.c
++++ b/arch/powerpc/kernel/signal_32.c
+@@ -1048,7 +1048,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+ if (new_ctx == NULL)
+ return 0;
+ if (!access_ok(new_ctx, ctx_size) ||
+- fault_in_pages_readable((u8 __user *)new_ctx, ctx_size))
++ fault_in_readable((char __user *)new_ctx, ctx_size))
+ return -EFAULT;
+
+ /*
+@@ -1062,8 +1062,10 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+ * or if another thread unmaps the region containing the context.
+ * We kill the task with a SIGSEGV in this situation.
+ */
+- if (do_setcontext(new_ctx, regs, 0))
+- do_exit(SIGSEGV);
++ if (do_setcontext(new_ctx, regs, 0)) {
++ force_exit_sig(SIGSEGV);
++ return -EFAULT;
++ }
+
+ set_thread_flag(TIF_RESTOREALL);
+ return 0;
+@@ -1237,7 +1239,7 @@ SYSCALL_DEFINE3(debug_setcontext, struct ucontext __user *, ctx,
+ #endif
+
+ if (!access_ok(ctx, sizeof(*ctx)) ||
+- fault_in_pages_readable((u8 __user *)ctx, sizeof(*ctx)))
++ fault_in_readable((char __user *)ctx, sizeof(*ctx)))
+ return -EFAULT;
+
+ /*
+diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
+index 1831bba0582e1..d1e1fc0acbea3 100644
+--- a/arch/powerpc/kernel/signal_64.c
++++ b/arch/powerpc/kernel/signal_64.c
+@@ -688,7 +688,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+ if (new_ctx == NULL)
+ return 0;
+ if (!access_ok(new_ctx, ctx_size) ||
+- fault_in_pages_readable((u8 __user *)new_ctx, ctx_size))
++ fault_in_readable((char __user *)new_ctx, ctx_size))
+ return -EFAULT;
+
+ /*
+@@ -703,15 +703,18 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+ * We kill the task with a SIGSEGV in this situation.
+ */
+
+- if (__get_user_sigset(&set, &new_ctx->uc_sigmask))
+- do_exit(SIGSEGV);
++ if (__get_user_sigset(&set, &new_ctx->uc_sigmask)) {
++ force_exit_sig(SIGSEGV);
++ return -EFAULT;
++ }
+ set_current_blocked(&set);
+
+ if (!user_read_access_begin(new_ctx, ctx_size))
+ return -EFAULT;
+ if (__unsafe_restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext)) {
+ user_read_access_end();
+- do_exit(SIGSEGV);
++ force_exit_sig(SIGSEGV);
++ return -EFAULT;
+ }
+ user_read_access_end();
+
+diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
+index 605bab448f847..fb95f92dcfac6 100644
+--- a/arch/powerpc/kernel/smp.c
++++ b/arch/powerpc/kernel/smp.c
+@@ -61,6 +61,7 @@
+ #include <asm/cpu_has_feature.h>
+ #include <asm/ftrace.h>
+ #include <asm/kup.h>
++#include <asm/fadump.h>
+
+ #ifdef DEBUG
+ #include <asm/udbg.h>
+@@ -620,6 +621,45 @@ void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
+ }
+ #endif
+
++#ifdef CONFIG_NMI_IPI
++static void crash_stop_this_cpu(struct pt_regs *regs)
++#else
++static void crash_stop_this_cpu(void *dummy)
++#endif
++{
++ /*
++ * Just busy wait here and avoid marking CPU as offline to ensure
++ * register data is captured appropriately.
++ */
++ while (1)
++ cpu_relax();
++}
++
++void crash_smp_send_stop(void)
++{
++ static bool stopped = false;
++
++ /*
++ * In case of fadump, register data for all CPUs is captured by f/w
++ * on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before
++ * this rtas call to avoid tricky post processing of those CPUs'
++ * backtraces.
++ */
++ if (should_fadump_crash())
++ return;
++
++ if (stopped)
++ return;
++
++ stopped = true;
++
++#ifdef CONFIG_NMI_IPI
++ smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_stop_this_cpu, 1000000);
++#else
++ smp_call_function(crash_stop_this_cpu, NULL, 0);
++#endif /* CONFIG_NMI_IPI */
++}
++
+ #ifdef CONFIG_NMI_IPI
+ static void nmi_stop_this_cpu(struct pt_regs *regs)
+ {
+@@ -1640,10 +1680,12 @@ void start_secondary(void *unused)
+ BUG();
+ }
+
++#ifdef CONFIG_PROFILING
+ int setup_profiling_timer(unsigned int multiplier)
+ {
+ return 0;
+ }
++#endif
+
+ static void fixup_topology(void)
+ {
+diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
+index 40bdefe9caa73..1a63e37f336ab 100644
+--- a/arch/powerpc/kernel/vmlinux.lds.S
++++ b/arch/powerpc/kernel/vmlinux.lds.S
+@@ -275,9 +275,7 @@ SECTIONS
+ . = ALIGN(8);
+ .dynsym : AT(ADDR(.dynsym) - LOAD_OFFSET)
+ {
+-#ifdef CONFIG_PPC32
+ __dynamic_symtab = .;
+-#endif
+ *(.dynsym)
+ }
+ .dynstr : AT(ADDR(.dynstr) - LOAD_OFFSET) { *(.dynstr) }
+diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
+index f9ea0e5357f92..ad94a2c6b7337 100644
+--- a/arch/powerpc/kernel/watchdog.c
++++ b/arch/powerpc/kernel/watchdog.c
+@@ -135,6 +135,10 @@ static void set_cpumask_stuck(const struct cpumask *cpumask, u64 tb)
+ {
+ cpumask_or(&wd_smp_cpus_stuck, &wd_smp_cpus_stuck, cpumask);
+ cpumask_andnot(&wd_smp_cpus_pending, &wd_smp_cpus_pending, cpumask);
++ /*
++ * See wd_smp_clear_cpu_pending()
++ */
++ smp_mb();
+ if (cpumask_empty(&wd_smp_cpus_pending)) {
+ wd_smp_last_reset_tb = tb;
+ cpumask_andnot(&wd_smp_cpus_pending,
+@@ -187,6 +191,12 @@ static void watchdog_smp_panic(int cpu, u64 tb)
+ if (sysctl_hardlockup_all_cpu_backtrace)
+ trigger_allbutself_cpu_backtrace();
+
++ /*
++ * Force flush any remote buffers that might be stuck in IRQ context
++ * and therefore could not run their irq_work.
++ */
++ printk_trigger_flush();
++
+ if (hardlockup_panic)
+ nmi_panic(NULL, "Hard LOCKUP");
+
+@@ -215,13 +225,44 @@ static void wd_smp_clear_cpu_pending(int cpu, u64 tb)
+
+ cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck);
+ wd_smp_unlock(&flags);
++ } else {
++ /*
++ * The last CPU to clear pending should have reset the
++ * watchdog so we generally should not find it empty
++ * here if our CPU was clear. However it could happen
++ * due to a rare race with another CPU taking the
++ * last CPU out of the mask concurrently.
++ *
++ * We can't add a warning for it. But just in case
++ * there is a problem with the watchdog that is causing
++ * the mask to not be reset, try to kick it along here.
++ */
++ if (unlikely(cpumask_empty(&wd_smp_cpus_pending)))
++ goto none_pending;
+ }
+ return;
+ }
++
+ cpumask_clear_cpu(cpu, &wd_smp_cpus_pending);
++
++ /*
++ * Order the store to clear pending with the load(s) to check all
++ * words in the pending mask to check they are all empty. This orders
++ * with the same barrier on another CPU. This prevents two CPUs
++ * clearing the last 2 pending bits, but neither seeing the other's
++ * store when checking if the mask is empty, and missing an empty
++ * mask, which ends with a false positive.
++ */
++ smp_mb();
+ if (cpumask_empty(&wd_smp_cpus_pending)) {
+ unsigned long flags;
+
++none_pending:
++ /*
++ * Double check under lock because more than one CPU could see
++ * a clear mask with the lockless check after clearing their
++ * pending bits.
++ */
+ wd_smp_lock(&flags);
+ if (cpumask_empty(&wd_smp_cpus_pending)) {
+ wd_smp_last_reset_tb = tb;
+@@ -312,8 +353,12 @@ void arch_touch_nmi_watchdog(void)
+ {
+ unsigned long ticks = tb_ticks_per_usec * wd_timer_period_ms * 1000;
+ int cpu = smp_processor_id();
+- u64 tb = get_tb();
++ u64 tb;
+
++ if (!cpumask_test_cpu(cpu, &watchdog_cpumask))
++ return;
++
++ tb = get_tb();
+ if (tb - per_cpu(wd_timer_tb, cpu) >= ticks) {
+ per_cpu(wd_timer_tb, cpu) = tb;
+ wd_smp_clear_cpu_pending(cpu, tb);
+diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c
+index 48525e8b57300..71b1bfdadd76a 100644
+--- a/arch/powerpc/kexec/core.c
++++ b/arch/powerpc/kexec/core.c
+@@ -147,11 +147,18 @@ void __init reserve_crashkernel(void)
+ if (!crashk_res.start) {
+ #ifdef CONFIG_PPC64
+ /*
+- * On 64bit we split the RMO in half but cap it at half of
+- * a small SLB (128MB) since the crash kernel needs to place
+- * itself and some stacks to be in the first segment.
++ * On the LPAR platform place the crash kernel to mid of
++ * RMA size (512MB or more) to ensure the crash kernel
++ * gets enough space to place itself and some stack to be
++ * in the first segment. At the same time normal kernel
++ * also get enough space to allocate memory for essential
++ * system resource in the first segment. Keep the crash
++ * kernel starts at 128MB offset on other platforms.
+ */
+- crashk_res.start = min(0x8000000ULL, (ppc64_rma_size / 2));
++ if (firmware_has_feature(FW_FEATURE_LPAR))
++ crashk_res.start = ppc64_rma_size / 2;
++ else
++ crashk_res.start = min(0x8000000ULL, (ppc64_rma_size / 2));
+ #else
+ crashk_res.start = KDUMP_KERNELBASE;
+ #endif
+diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
+index e3ab9df6cf199..6cfcd20d46686 100644
+--- a/arch/powerpc/kvm/book3s_32_sr.S
++++ b/arch/powerpc/kvm/book3s_32_sr.S
+@@ -122,11 +122,27 @@
+
+ /* 0x0 - 0xb */
+
+- /* 'current->mm' needs to be in r4 */
+- tophys(r4, r2)
+- lwz r4, MM(r4)
+- tophys(r4, r4)
+- /* This only clobbers r0, r3, r4 and r5 */
++ /* switch_mmu_context() needs paging, let's enable it */
++ mfmsr r9
++ ori r11, r9, MSR_DR
++ mtmsr r11
++ sync
++
++ /* switch_mmu_context() clobbers r12, rescue it */
++ SAVE_GPR(12, r1)
++
++ /* Calling switch_mmu_context(<inv>, current->mm, <inv>); */
++ lwz r4, MM(r2)
+ bl switch_mmu_context
+
++ /* restore r12 */
++ REST_GPR(12, r1)
++
++ /* Disable paging again */
++ mfmsr r9
++ li r6, MSR_DR
++ andc r9, r9, r6
++ mtmsr r9
++ sync
++
+ .endm
+diff --git a/arch/powerpc/kvm/book3s_64_entry.S b/arch/powerpc/kvm/book3s_64_entry.S
+index 983b8c18bc31e..a644003603da1 100644
+--- a/arch/powerpc/kvm/book3s_64_entry.S
++++ b/arch/powerpc/kvm/book3s_64_entry.S
+@@ -407,10 +407,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
+ */
+ ld r10,HSTATE_SCRATCH0(r13)
+ cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK
+- beq machine_check_common
++ beq .Lcall_machine_check_common
+
+ cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET
+- beq system_reset_common
++ beq .Lcall_system_reset_common
+
+ b .
++
++.Lcall_machine_check_common:
++ b machine_check_common
++
++.Lcall_system_reset_common:
++ b system_reset_common
+ #endif
+diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
+index 6365087f31602..3cb2e05a7ee83 100644
+--- a/arch/powerpc/kvm/book3s_64_vio.c
++++ b/arch/powerpc/kvm/book3s_64_vio.c
+@@ -421,13 +421,19 @@ static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt,
+ tbl[idx % TCES_PER_PAGE] = tce;
+ }
+
+-static void kvmppc_clear_tce(struct mm_struct *mm, struct iommu_table *tbl,
+- unsigned long entry)
++static void kvmppc_clear_tce(struct mm_struct *mm, struct kvmppc_spapr_tce_table *stt,
++ struct iommu_table *tbl, unsigned long entry)
+ {
+- unsigned long hpa = 0;
+- enum dma_data_direction dir = DMA_NONE;
++ unsigned long i;
++ unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
++ unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);
++
++ for (i = 0; i < subpages; ++i) {
++ unsigned long hpa = 0;
++ enum dma_data_direction dir = DMA_NONE;
+
+- iommu_tce_xchg_no_kill(mm, tbl, entry, &hpa, &dir);
++ iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir);
++ }
+ }
+
+ static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
+@@ -486,6 +492,8 @@ static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
+ break;
+ }
+
++ iommu_tce_kill(tbl, io_entry, subpages);
++
+ return ret;
+ }
+
+@@ -545,6 +553,8 @@ static long kvmppc_tce_iommu_map(struct kvm *kvm,
+ break;
+ }
+
++ iommu_tce_kill(tbl, io_entry, subpages);
++
+ return ret;
+ }
+
+@@ -591,10 +601,9 @@ long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
+ ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,
+ entry, ua, dir);
+
+- iommu_tce_kill(stit->tbl, entry, 1);
+
+ if (ret != H_SUCCESS) {
+- kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry);
++ kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);
+ goto unlock_exit;
+ }
+ }
+@@ -670,13 +679,13 @@ long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
+ */
+ if (get_user(tce, tces + i)) {
+ ret = H_TOO_HARD;
+- goto invalidate_exit;
++ goto unlock_exit;
+ }
+ tce = be64_to_cpu(tce);
+
+ if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
+ ret = H_PARAMETER;
+- goto invalidate_exit;
++ goto unlock_exit;
+ }
+
+ list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
+@@ -685,19 +694,15 @@ long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
+ iommu_tce_direction(tce));
+
+ if (ret != H_SUCCESS) {
+- kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl,
+- entry);
+- goto invalidate_exit;
++ kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,
++ entry + i);
++ goto unlock_exit;
+ }
+ }
+
+ kvmppc_tce_put(stt, entry + i, tce);
+ }
+
+-invalidate_exit:
+- list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
+- iommu_tce_kill(stit->tbl, entry, npages);
+-
+ unlock_exit:
+ srcu_read_unlock(&vcpu->kvm->srcu, idx);
+
+@@ -736,20 +741,16 @@ long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
+ continue;
+
+ if (ret == H_TOO_HARD)
+- goto invalidate_exit;
++ return ret;
+
+ WARN_ON_ONCE(1);
+- kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry);
++ kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);
+ }
+ }
+
+ for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
+ kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);
+
+-invalidate_exit:
+- list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
+- iommu_tce_kill(stit->tbl, ioba >> stt->page_shift, npages);
+-
+ return ret;
+ }
+ EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);
+diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
+index 870b7f0c7ea56..fdeda6a9cff44 100644
+--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
++++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
+@@ -247,13 +247,19 @@ static void iommu_tce_kill_rm(struct iommu_table *tbl,
+ tbl->it_ops->tce_kill(tbl, entry, pages, true);
+ }
+
+-static void kvmppc_rm_clear_tce(struct kvm *kvm, struct iommu_table *tbl,
+- unsigned long entry)
++static void kvmppc_rm_clear_tce(struct kvm *kvm, struct kvmppc_spapr_tce_table *stt,
++ struct iommu_table *tbl, unsigned long entry)
+ {
+- unsigned long hpa = 0;
+- enum dma_data_direction dir = DMA_NONE;
++ unsigned long i;
++ unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
++ unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);
++
++ for (i = 0; i < subpages; ++i) {
++ unsigned long hpa = 0;
++ enum dma_data_direction dir = DMA_NONE;
+
+- iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, entry, &hpa, &dir);
++ iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, io_entry + i, &hpa, &dir);
++ }
+ }
+
+ static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm,
+@@ -316,6 +322,8 @@ static long kvmppc_rm_tce_iommu_unmap(struct kvm *kvm,
+ break;
+ }
+
++ iommu_tce_kill_rm(tbl, io_entry, subpages);
++
+ return ret;
+ }
+
+@@ -379,6 +387,8 @@ static long kvmppc_rm_tce_iommu_map(struct kvm *kvm,
+ break;
+ }
+
++ iommu_tce_kill_rm(tbl, io_entry, subpages);
++
+ return ret;
+ }
+
+@@ -420,10 +430,8 @@ long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
+ ret = kvmppc_rm_tce_iommu_map(vcpu->kvm, stt,
+ stit->tbl, entry, ua, dir);
+
+- iommu_tce_kill_rm(stit->tbl, entry, 1);
+-
+ if (ret != H_SUCCESS) {
+- kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry);
++ kvmppc_rm_clear_tce(vcpu->kvm, stt, stit->tbl, entry);
+ return ret;
+ }
+ }
+@@ -561,7 +569,7 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
+ ua = 0;
+ if (kvmppc_rm_tce_to_ua(vcpu->kvm, tce, &ua)) {
+ ret = H_PARAMETER;
+- goto invalidate_exit;
++ goto unlock_exit;
+ }
+
+ list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
+@@ -570,19 +578,15 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
+ iommu_tce_direction(tce));
+
+ if (ret != H_SUCCESS) {
+- kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl,
+- entry);
+- goto invalidate_exit;
++ kvmppc_rm_clear_tce(vcpu->kvm, stt, stit->tbl,
++ entry + i);
++ goto unlock_exit;
+ }
+ }
+
+ kvmppc_rm_tce_put(stt, entry + i, tce);
+ }
+
+-invalidate_exit:
+- list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
+- iommu_tce_kill_rm(stit->tbl, entry, npages);
+-
+ unlock_exit:
+ if (!prereg)
+ arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock);
+@@ -620,20 +624,16 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu,
+ continue;
+
+ if (ret == H_TOO_HARD)
+- goto invalidate_exit;
++ return ret;
+
+ WARN_ON_ONCE_RM(1);
+- kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry);
++ kvmppc_rm_clear_tce(vcpu->kvm, stt, stit->tbl, entry + i);
+ }
+ }
+
+ for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
+ kvmppc_rm_tce_put(stt, ioba >> stt->page_shift, tce_value);
+
+-invalidate_exit:
+- list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
+- iommu_tce_kill_rm(stit->tbl, ioba >> stt->page_shift, npages);
+-
+ return ret;
+ }
+
+diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
+index 2acb1c96cfafd..7fa6857116690 100644
+--- a/arch/powerpc/kvm/book3s_hv.c
++++ b/arch/powerpc/kvm/book3s_hv.c
+@@ -1731,7 +1731,6 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
+
+ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
+ {
+- struct kvm_nested_guest *nested = vcpu->arch.nested;
+ int r;
+ int srcu_idx;
+
+@@ -1831,7 +1830,7 @@ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
+ * it into a HEAI.
+ */
+ if (!(vcpu->arch.hfscr_permitted & (1UL << cause)) ||
+- (nested->hfscr & (1UL << cause))) {
++ (vcpu->arch.nested_hfscr & (1UL << cause))) {
+ vcpu->arch.trap = BOOK3S_INTERRUPT_H_EMUL_ASSIST;
+
+ /*
+@@ -3726,7 +3725,20 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
+
+ kvmppc_set_host_core(pcpu);
+
+- guest_exit_irqoff();
++ context_tracking_guest_exit();
++ if (!vtime_accounting_enabled_this_cpu()) {
++ local_irq_enable();
++ /*
++ * Service IRQs here before vtime_account_guest_exit() so any
++ * ticks that occurred while running the guest are accounted to
++ * the guest. If vtime accounting is enabled, accounting uses
++ * TB rather than ticks, so it can be done without enabling
++ * interrupts here, which has the problem that it accounts
++ * interrupt processing overhead to the host.
++ */
++ local_irq_disable();
++ }
++ vtime_account_guest_exit();
+
+ local_irq_enable();
+
+@@ -4510,7 +4522,20 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
+
+ kvmppc_set_host_core(pcpu);
+
+- guest_exit_irqoff();
++ context_tracking_guest_exit();
++ if (!vtime_accounting_enabled_this_cpu()) {
++ local_irq_enable();
++ /*
++ * Service IRQs here before vtime_account_guest_exit() so any
++ * ticks that occurred while running the guest are accounted to
++ * the guest. If vtime accounting is enabled, accounting uses
++ * TB rather than ticks, so it can be done without enabling
++ * interrupts here, which has the problem that it accounts
++ * interrupt processing overhead to the host.
++ */
++ local_irq_disable();
++ }
++ vtime_account_guest_exit();
+
+ local_irq_enable();
+
+@@ -4835,8 +4860,12 @@ static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
+ unsigned long npages = mem->memory_size >> PAGE_SHIFT;
+
+ if (change == KVM_MR_CREATE) {
+- slot->arch.rmap = vzalloc(array_size(npages,
+- sizeof(*slot->arch.rmap)));
++ unsigned long size = array_size(npages, sizeof(*slot->arch.rmap));
++
++ if ((size >> PAGE_SHIFT) > totalram_pages())
++ return -ENOMEM;
++
++ slot->arch.rmap = vzalloc(size);
+ if (!slot->arch.rmap)
+ return -ENOMEM;
+ }
+@@ -6072,8 +6101,11 @@ static int kvmppc_book3s_init_hv(void)
+ if (r)
+ return r;
+
+- if (kvmppc_radix_possible())
++ if (kvmppc_radix_possible()) {
+ r = kvmppc_radix_init();
++ if (r)
++ return r;
++ }
+
+ r = kvmppc_uvmem_init();
+ if (r < 0)
+diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
+index fcf4760a3a0ea..70b7a8f971538 100644
+--- a/arch/powerpc/kvm/book3s_hv_builtin.c
++++ b/arch/powerpc/kvm/book3s_hv_builtin.c
+@@ -695,6 +695,7 @@ static void flush_guest_tlb(struct kvm *kvm)
+ "r" (0) : "memory");
+ }
+ asm volatile("ptesync": : :"memory");
++ // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
+ asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
+ } else {
+ for (set = 0; set < kvm->arch.tlb_sets; ++set) {
+@@ -705,7 +706,9 @@ static void flush_guest_tlb(struct kvm *kvm)
+ rb += PPC_BIT(51); /* increment set number */
+ }
+ asm volatile("ptesync": : :"memory");
+- asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
++ // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
++ if (cpu_has_feature(CPU_FTR_ARCH_300))
++ asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
+ }
+ }
+
+diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c
+index ed8a2c9f56299..6c4e0e93105ff 100644
+--- a/arch/powerpc/kvm/book3s_hv_nested.c
++++ b/arch/powerpc/kvm/book3s_hv_nested.c
+@@ -362,7 +362,7 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
+ /* set L1 state to L2 state */
+ vcpu->arch.nested = l2;
+ vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
+- l2->hfscr = l2_hv.hfscr;
++ vcpu->arch.nested_hfscr = l2_hv.hfscr;
+ vcpu->arch.regs = l2_regs;
+
+ /* Guest must always run with ME enabled, HV disabled. */
+@@ -582,7 +582,7 @@ long kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu *vcpu)
+ if (eaddr & (0xFFFUL << 52))
+ return H_PARAMETER;
+
+- buf = kzalloc(n, GFP_KERNEL);
++ buf = kzalloc(n, GFP_KERNEL | __GFP_NOWARN);
+ if (!buf)
+ return H_NO_MEM;
+
+diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+index eb776d0c5d8e9..32a4b4d412b92 100644
+--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
++++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+@@ -2005,7 +2005,7 @@ hcall_real_table:
+ .globl hcall_real_table_end
+ hcall_real_table_end:
+
+-_GLOBAL(kvmppc_h_set_xdabr)
++_GLOBAL_TOC(kvmppc_h_set_xdabr)
+ EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
+ andi. r0, r5, DABRX_USER | DABRX_KERNEL
+ beq 6f
+@@ -2015,7 +2015,7 @@ EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
+ 6: li r3, H_PARAMETER
+ blr
+
+-_GLOBAL(kvmppc_h_set_dabr)
++_GLOBAL_TOC(kvmppc_h_set_dabr)
+ EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr)
+ li r5, DABRX_USER | DABRX_KERNEL
+ 3:
+diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
+index 977801c83aff8..8c15c90dd3a97 100644
+--- a/arch/powerpc/kvm/booke.c
++++ b/arch/powerpc/kvm/booke.c
+@@ -1042,7 +1042,21 @@ int kvmppc_handle_exit(struct kvm_vcpu *vcpu, unsigned int exit_nr)
+ }
+
+ trace_kvm_exit(exit_nr, vcpu);
+- guest_exit_irqoff();
++
++ context_tracking_guest_exit();
++ if (!vtime_accounting_enabled_this_cpu()) {
++ local_irq_enable();
++ /*
++ * Service IRQs here before vtime_account_guest_exit() so any
++ * ticks that occurred while running the guest are accounted to
++ * the guest. If vtime accounting is enabled, accounting uses
++ * TB rather than ticks, so it can be done without enabling
++ * interrupts here, which has the problem that it accounts
++ * interrupt processing overhead to the host.
++ */
++ local_irq_disable();
++ }
++ vtime_account_guest_exit();
+
+ local_irq_enable();
+
+diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
+index b4e6f70b97b94..ee305455bd8db 100644
+--- a/arch/powerpc/kvm/powerpc.c
++++ b/arch/powerpc/kvm/powerpc.c
+@@ -1507,7 +1507,7 @@ int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu,
+ {
+ enum emulation_result emulated = EMULATE_DONE;
+
+- if (vcpu->arch.mmio_vsx_copy_nums > 2)
++ if (vcpu->arch.mmio_vmx_copy_nums > 2)
+ return EMULATE_FAIL;
+
+ while (vcpu->arch.mmio_vmx_copy_nums) {
+@@ -1604,7 +1604,7 @@ int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu,
+ unsigned int index = rs & KVM_MMIO_REG_MASK;
+ enum emulation_result emulated = EMULATE_DONE;
+
+- if (vcpu->arch.mmio_vsx_copy_nums > 2)
++ if (vcpu->arch.mmio_vmx_copy_nums > 2)
+ return EMULATE_FAIL;
+
+ vcpu->arch.io_gpr = rs;
+diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
+index 99a7c9132422c..54be64203b2ab 100644
+--- a/arch/powerpc/lib/Makefile
++++ b/arch/powerpc/lib/Makefile
+@@ -19,6 +19,9 @@ CFLAGS_code-patching.o += -DDISABLE_BRANCH_PROFILING
+ CFLAGS_feature-fixups.o += -DDISABLE_BRANCH_PROFILING
+ endif
+
++CFLAGS_code-patching.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
++CFLAGS_feature-fixups.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
++
+ obj-y += alloc.o code-patching.o feature-fixups.o pmem.o test_code-patching.o
+
+ ifndef CONFIG_KASAN
+diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
+index cda17bee5afea..c3e06922468b3 100644
+--- a/arch/powerpc/lib/feature-fixups.c
++++ b/arch/powerpc/lib/feature-fixups.c
+@@ -228,6 +228,7 @@ static void do_stf_exit_barrier_fixups(enum stf_barrier_type types)
+
+ static bool stf_exit_reentrant = false;
+ static bool rfi_exit_reentrant = false;
++static DEFINE_MUTEX(exit_flush_lock);
+
+ static int __do_stf_barrier_fixups(void *data)
+ {
+@@ -253,6 +254,9 @@ void do_stf_barrier_fixups(enum stf_barrier_type types)
+ * low level interrupt exit code before patching. After the patching,
+ * if allowed, then flip the branch to allow fast exits.
+ */
++
++ // Prevent static key update races with do_rfi_flush_fixups()
++ mutex_lock(&exit_flush_lock);
+ static_branch_enable(&interrupt_exit_not_reentrant);
+
+ stop_machine(__do_stf_barrier_fixups, &types, NULL);
+@@ -264,6 +268,8 @@ void do_stf_barrier_fixups(enum stf_barrier_type types)
+
+ if (stf_exit_reentrant && rfi_exit_reentrant)
+ static_branch_disable(&interrupt_exit_not_reentrant);
++
++ mutex_unlock(&exit_flush_lock);
+ }
+
+ void do_uaccess_flush_fixups(enum l1d_flush_type types)
+@@ -486,6 +492,9 @@ void do_rfi_flush_fixups(enum l1d_flush_type types)
+ * without stop_machine, so this could be achieved with a broadcast
+ * IPI instead, but this matches the stf sequence.
+ */
++
++ // Prevent static key update races with do_stf_barrier_fixups()
++ mutex_lock(&exit_flush_lock);
+ static_branch_enable(&interrupt_exit_not_reentrant);
+
+ stop_machine(__do_rfi_flush_fixups, &types, NULL);
+@@ -497,6 +506,8 @@ void do_rfi_flush_fixups(enum l1d_flush_type types)
+
+ if (stf_exit_reentrant && rfi_exit_reentrant)
+ static_branch_disable(&interrupt_exit_not_reentrant);
++
++ mutex_unlock(&exit_flush_lock);
+ }
+
+ void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end)
+diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
+index d8d5f901cee1c..2d39b7c246e30 100644
+--- a/arch/powerpc/lib/sstep.c
++++ b/arch/powerpc/lib/sstep.c
+@@ -112,9 +112,9 @@ static nokprobe_inline long address_ok(struct pt_regs *regs,
+ {
+ if (!user_mode(regs))
+ return 1;
+- if (__access_ok(ea, nb))
++ if (access_ok((void __user *)ea, nb))
+ return 1;
+- if (__access_ok(ea, 1))
++ if (access_ok((void __user *)ea, 1))
+ /* Access overlaps the end of the user region */
+ regs->dar = TASK_SIZE_MAX - 1;
+ else
+@@ -1014,7 +1014,10 @@ NOKPROBE_SYMBOL(emulate_dcbz);
+
+ #define __put_user_asmx(x, addr, err, op, cr) \
+ __asm__ __volatile__( \
++ ".machine push\n" \
++ ".machine power8\n" \
+ "1: " op " %2,0,%3\n" \
++ ".machine pop\n" \
+ " mfcr %1\n" \
+ "2:\n" \
+ ".section .fixup,\"ax\"\n" \
+@@ -1027,7 +1030,10 @@ NOKPROBE_SYMBOL(emulate_dcbz);
+
+ #define __get_user_asmx(x, addr, err, op) \
+ __asm__ __volatile__( \
++ ".machine push\n" \
++ ".machine power8\n" \
+ "1: "op" %1,0,%2\n" \
++ ".machine pop\n" \
+ "2:\n" \
+ ".section .fixup,\"ax\"\n" \
+ "3: li %0,%3\n" \
+@@ -3181,12 +3187,14 @@ void emulate_update_regs(struct pt_regs *regs, struct instruction_op *op)
+ case BARRIER_EIEIO:
+ eieio();
+ break;
++#ifdef CONFIG_PPC64
+ case BARRIER_LWSYNC:
+ asm volatile("lwsync" : : : "memory");
+ break;
+ case BARRIER_PTESYNC:
+ asm volatile("ptesync" : : : "memory");
+ break;
++#endif
+ }
+ break;
+
+@@ -3304,7 +3312,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
+ __put_user_asmx(op->val, ea, err, "stbcx.", cr);
+ break;
+ case 2:
+- __put_user_asmx(op->val, ea, err, "stbcx.", cr);
++ __put_user_asmx(op->val, ea, err, "sthcx.", cr);
+ break;
+ #endif
+ case 4:
+diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c
+index 27061583a0107..203735caf6915 100644
+--- a/arch/powerpc/mm/book3s32/mmu.c
++++ b/arch/powerpc/mm/book3s32/mmu.c
+@@ -76,7 +76,7 @@ unsigned long p_block_mapped(phys_addr_t pa)
+ return 0;
+ }
+
+-static int find_free_bat(void)
++int __init find_free_bat(void)
+ {
+ int b;
+ int n = mmu_has_feature(MMU_FTR_USE_HIGH_BATS) ? 8 : 4;
+@@ -100,7 +100,7 @@ static int find_free_bat(void)
+ * - block size has to be a power of two. This is calculated by finding the
+ * highest bit set to 1.
+ */
+-static unsigned int block_size(unsigned long base, unsigned long top)
++unsigned int bat_block_size(unsigned long base, unsigned long top)
+ {
+ unsigned int max_size = SZ_256M;
+ unsigned int base_shift = (ffs(base) - 1) & 31;
+@@ -145,7 +145,7 @@ static unsigned long __init __mmu_mapin_ram(unsigned long base, unsigned long to
+ int idx;
+
+ while ((idx = find_free_bat()) != -1 && base != top) {
+- unsigned int size = block_size(base, top);
++ unsigned int size = bat_block_size(base, top);
+
+ if (size < 128 << 10)
+ break;
+@@ -196,18 +196,17 @@ void mmu_mark_initmem_nx(void)
+ int nb = mmu_has_feature(MMU_FTR_USE_HIGH_BATS) ? 8 : 4;
+ int i;
+ unsigned long base = (unsigned long)_stext - PAGE_OFFSET;
+- unsigned long top = (unsigned long)_etext - PAGE_OFFSET;
++ unsigned long top = ALIGN((unsigned long)_etext - PAGE_OFFSET, SZ_128K);
+ unsigned long border = (unsigned long)__init_begin - PAGE_OFFSET;
+ unsigned long size;
+
+- for (i = 0; i < nb - 1 && base < top && top - base > (128 << 10);) {
+- size = block_size(base, top);
++ for (i = 0; i < nb - 1 && base < top;) {
++ size = bat_block_size(base, top);
+ setibat(i++, PAGE_OFFSET + base, base, size, PAGE_KERNEL_TEXT);
+ base += size;
+ }
+ if (base < top) {
+- size = block_size(base, top);
+- size = max(size, 128UL << 10);
++ size = bat_block_size(base, top);
+ if ((top - base) > size) {
+ size <<= 1;
+ if (strict_kernel_rwx_enabled() && base + size > border)
+diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
+index c145776d3ae5e..7bfd88c4b5470 100644
+--- a/arch/powerpc/mm/book3s64/hash_utils.c
++++ b/arch/powerpc/mm/book3s64/hash_utils.c
+@@ -1522,8 +1522,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
+ }
+ EXPORT_SYMBOL_GPL(hash_page);
+
+-DECLARE_INTERRUPT_HANDLER(__do_hash_fault);
+-DEFINE_INTERRUPT_HANDLER(__do_hash_fault)
++DEFINE_INTERRUPT_HANDLER(do_hash_fault)
+ {
+ unsigned long ea = regs->dar;
+ unsigned long dsisr = regs->dsisr;
+@@ -1582,35 +1581,6 @@ DEFINE_INTERRUPT_HANDLER(__do_hash_fault)
+ }
+ }
+
+-/*
+- * The _RAW interrupt entry checks for the in_nmi() case before
+- * running the full handler.
+- */
+-DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault)
+-{
+- /*
+- * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
+- * don't call hash_page, just fail the fault. This is required to
+- * prevent re-entrancy problems in the hash code, namely perf
+- * interrupts hitting while something holds H_PAGE_BUSY, and taking a
+- * hash fault. See the comment in hash_preload().
+- *
+- * We come here as a result of a DSI at a point where we don't want
+- * to call hash_page, such as when we are accessing memory (possibly
+- * user memory) inside a PMU interrupt that occurred while interrupts
+- * were soft-disabled. We want to invoke the exception handler for
+- * the access, or panic if there isn't a handler.
+- */
+- if (unlikely(in_nmi())) {
+- do_bad_page_fault_segv(regs);
+- return 0;
+- }
+-
+- __do_hash_fault(regs);
+-
+- return 0;
+-}
+-
+ #ifdef CONFIG_PPC_MM_SLICES
+ static bool should_hash_preload(struct mm_struct *mm, unsigned long ea)
+ {
+@@ -1677,26 +1647,18 @@ static void hash_preload(struct mm_struct *mm, pte_t *ptep, unsigned long ea,
+ #endif /* CONFIG_PPC_64K_PAGES */
+
+ /*
+- * __hash_page_* must run with interrupts off, as it sets the
+- * H_PAGE_BUSY bit. It's possible for perf interrupts to hit at any
+- * time and may take a hash fault reading the user stack, see
+- * read_user_stack_slow() in the powerpc/perf code.
+- *
+- * If that takes a hash fault on the same page as we lock here, it
+- * will bail out when seeing H_PAGE_BUSY set, and retry the access
+- * leading to an infinite loop.
++ * __hash_page_* must run with interrupts off, including PMI interrupts
++ * off, as it sets the H_PAGE_BUSY bit.
+ *
+- * Disabling interrupts here does not prevent perf interrupts, but it
+- * will prevent them taking hash faults (see the NMI test in
+- * do_hash_page), then read_user_stack's copy_from_user_nofault will
+- * fail and perf will fall back to read_user_stack_slow(), which
+- * walks the Linux page tables.
++ * It's otherwise possible for perf interrupts to hit at any time and
++ * may take a hash fault reading the user stack, which could take a
++ * hash miss and deadlock on the same H_PAGE_BUSY bit.
+ *
+ * Interrupts must also be off for the duration of the
+ * mm_is_thread_local test and update, to prevent preempt running the
+ * mm on another CPU (XXX: this may be racy vs kthread_use_mm).
+ */
+- local_irq_save(flags);
++ powerpc_local_irq_pmu_save(flags);
+
+ /* Is that local to this CPU ? */
+ if (mm_is_thread_local(mm))
+@@ -1721,7 +1683,7 @@ static void hash_preload(struct mm_struct *mm, pte_t *ptep, unsigned long ea,
+ mm_ctx_user_psize(&mm->context),
+ pte_val(*ptep));
+
+- local_irq_restore(flags);
++ powerpc_local_irq_pmu_restore(flags);
+ }
+
+ /*
+diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
+index ae20add7954a0..795d18a84f556 100644
+--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
++++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
+@@ -1093,7 +1093,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
+
+ int pud_clear_huge(pud_t *pud)
+ {
+- if (pud_huge(*pud)) {
++ if (pud_is_leaf(*pud)) {
+ pud_clear(pud);
+ return 1;
+ }
+@@ -1140,7 +1140,7 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
+
+ int pmd_clear_huge(pmd_t *pmd)
+ {
+- if (pmd_huge(*pmd)) {
++ if (pmd_is_leaf(*pmd)) {
+ pmd_clear(pmd);
+ return 1;
+ }
+diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
+index a8d0ce85d39ad..4a15172dfef29 100644
+--- a/arch/powerpc/mm/fault.c
++++ b/arch/powerpc/mm/fault.c
+@@ -568,18 +568,24 @@ NOKPROBE_SYMBOL(hash__do_page_fault);
+ static void __bad_page_fault(struct pt_regs *regs, int sig)
+ {
+ int is_write = page_fault_is_write(regs->dsisr);
++ const char *msg;
+
+ /* kernel has accessed a bad area */
+
++ if (regs->dar < PAGE_SIZE)
++ msg = "Kernel NULL pointer dereference";
++ else
++ msg = "Unable to handle kernel data access";
++
+ switch (TRAP(regs)) {
+ case INTERRUPT_DATA_STORAGE:
+- case INTERRUPT_DATA_SEGMENT:
+ case INTERRUPT_H_DATA_STORAGE:
+- pr_alert("BUG: %s on %s at 0x%08lx\n",
+- regs->dar < PAGE_SIZE ? "Kernel NULL pointer dereference" :
+- "Unable to handle kernel data access",
++ pr_alert("BUG: %s on %s at 0x%08lx\n", msg,
+ is_write ? "write" : "read", regs->dar);
+ break;
++ case INTERRUPT_DATA_SEGMENT:
++ pr_alert("BUG: %s at 0x%08lx\n", msg, regs->dar);
++ break;
+ case INTERRUPT_INST_STORAGE:
+ case INTERRUPT_INST_SEGMENT:
+ pr_alert("BUG: Unable to handle kernel instruction fetch%s",
+diff --git a/arch/powerpc/mm/kasan/book3s_32.c b/arch/powerpc/mm/kasan/book3s_32.c
+index 202bd260a0095..450a67ef0bbe1 100644
+--- a/arch/powerpc/mm/kasan/book3s_32.c
++++ b/arch/powerpc/mm/kasan/book3s_32.c
+@@ -10,47 +10,51 @@ int __init kasan_init_region(void *start, size_t size)
+ {
+ unsigned long k_start = (unsigned long)kasan_mem_to_shadow(start);
+ unsigned long k_end = (unsigned long)kasan_mem_to_shadow(start + size);
+- unsigned long k_cur = k_start;
+- int k_size = k_end - k_start;
+- int k_size_base = 1 << (ffs(k_size) - 1);
++ unsigned long k_nobat = k_start;
++ unsigned long k_cur;
++ phys_addr_t phys;
+ int ret;
+- void *block;
+
+- block = memblock_alloc(k_size, k_size_base);
+-
+- if (block && k_size_base >= SZ_128K && k_start == ALIGN(k_start, k_size_base)) {
+- int k_size_more = 1 << (ffs(k_size - k_size_base) - 1);
+-
+- setbat(-1, k_start, __pa(block), k_size_base, PAGE_KERNEL);
+- if (k_size_more >= SZ_128K)
+- setbat(-1, k_start + k_size_base, __pa(block) + k_size_base,
+- k_size_more, PAGE_KERNEL);
+- if (v_block_mapped(k_start))
+- k_cur = k_start + k_size_base;
+- if (v_block_mapped(k_start + k_size_base))
+- k_cur = k_start + k_size_base + k_size_more;
+-
+- update_bats();
++ while (k_nobat < k_end) {
++ unsigned int k_size = bat_block_size(k_nobat, k_end);
++ int idx = find_free_bat();
++
++ if (idx == -1)
++ break;
++ if (k_size < SZ_128K)
++ break;
++ phys = memblock_phys_alloc_range(k_size, k_size, 0,
++ MEMBLOCK_ALLOC_ANYWHERE);
++ if (!phys)
++ break;
++
++ setbat(idx, k_nobat, phys, k_size, PAGE_KERNEL);
++ k_nobat += k_size;
+ }
++ if (k_nobat != k_start)
++ update_bats();
+
+- if (!block)
+- block = memblock_alloc(k_size, PAGE_SIZE);
+- if (!block)
+- return -ENOMEM;
++ if (k_nobat < k_end) {
++ phys = memblock_phys_alloc_range(k_end - k_nobat, PAGE_SIZE, 0,
++ MEMBLOCK_ALLOC_ANYWHERE);
++ if (!phys)
++ return -ENOMEM;
++ }
+
+ ret = kasan_init_shadow_page_tables(k_start, k_end);
+ if (ret)
+ return ret;
+
+- kasan_update_early_region(k_start, k_cur, __pte(0));
++ kasan_update_early_region(k_start, k_nobat, __pte(0));
+
+- for (; k_cur < k_end; k_cur += PAGE_SIZE) {
++ for (k_cur = k_nobat; k_cur < k_end; k_cur += PAGE_SIZE) {
+ pmd_t *pmd = pmd_off_k(k_cur);
+- void *va = block + k_cur - k_start;
+- pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL);
++ pte_t pte = pfn_pte(PHYS_PFN(phys + k_cur - k_nobat), PAGE_KERNEL);
+
+ __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, k_cur), pte, 0);
+ }
+ flush_tlb_kernel_range(k_start, k_end);
++ memset(kasan_mem_to_shadow(start), 0, k_end - k_start);
++
+ return 0;
+ }
+diff --git a/arch/powerpc/mm/kasan/kasan_init_32.c b/arch/powerpc/mm/kasan/kasan_init_32.c
+index cf8770b1a692e..f3e4d069e0ba7 100644
+--- a/arch/powerpc/mm/kasan/kasan_init_32.c
++++ b/arch/powerpc/mm/kasan/kasan_init_32.c
+@@ -83,13 +83,12 @@ void __init
+ kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte)
+ {
+ unsigned long k_cur;
+- phys_addr_t pa = __pa(kasan_early_shadow_page);
+
+ for (k_cur = k_start; k_cur != k_end; k_cur += PAGE_SIZE) {
+ pmd_t *pmd = pmd_off_k(k_cur);
+ pte_t *ptep = pte_offset_kernel(pmd, k_cur);
+
+- if ((pte_val(*ptep) & PTE_RPN_MASK) != pa)
++ if (pte_page(*ptep) != virt_to_page(lm_alias(kasan_early_shadow_page)))
+ continue;
+
+ __set_pte_at(&init_mm, k_cur, ptep, pte, 0);
+diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
+index c3c4e31462eca..05b9c3f31456c 100644
+--- a/arch/powerpc/mm/mem.c
++++ b/arch/powerpc/mm/mem.c
+@@ -20,8 +20,8 @@
+ #include <asm/machdep.h>
+ #include <asm/rtas.h>
+ #include <asm/kasan.h>
+-#include <asm/sparsemem.h>
+ #include <asm/svm.h>
++#include <asm/mmzone.h>
+
+ #include <mm/mmu_decl.h>
+
+diff --git a/arch/powerpc/mm/nohash/tlb_low_64e.S b/arch/powerpc/mm/nohash/tlb_low_64e.S
+index bf24451f3e71f..9235e720e3572 100644
+--- a/arch/powerpc/mm/nohash/tlb_low_64e.S
++++ b/arch/powerpc/mm/nohash/tlb_low_64e.S
+@@ -222,7 +222,7 @@ tlb_miss_kernel_bolted:
+
+ tlb_miss_fault_bolted:
+ /* We need to check if it was an instruction miss */
+- andi. r10,r11,_PAGE_EXEC|_PAGE_BAP_SX
++ andi. r10,r11,_PAGE_BAP_UX|_PAGE_BAP_SX
+ bne itlb_miss_fault_bolted
+ dtlb_miss_fault_bolted:
+ tlb_epilog_bolted
+@@ -239,7 +239,7 @@ itlb_miss_fault_bolted:
+ srdi r15,r16,60 /* get region */
+ bne- itlb_miss_fault_bolted
+
+- li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
++ li r11,_PAGE_PRESENT|_PAGE_BAP_UX /* Base perm */
+
+ /* We do the user/kernel test for the PID here along with the RW test
+ */
+@@ -614,7 +614,7 @@ itlb_miss_fault_e6500:
+
+ /* We do the user/kernel test for the PID here along with the RW test
+ */
+- li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
++ li r11,_PAGE_PRESENT|_PAGE_BAP_UX /* Base perm */
+ oris r11,r11,_PAGE_ACCESSED@h
+
+ cmpldi cr0,r15,0 /* Check for user region */
+@@ -734,7 +734,7 @@ normal_tlb_miss_done:
+
+ normal_tlb_miss_access_fault:
+ /* We need to check if it was an instruction miss */
+- andi. r10,r11,_PAGE_EXEC
++ andi. r10,r11,_PAGE_BAP_UX
+ bne 1f
+ ld r14,EX_TLB_DEAR(r12)
+ ld r15,EX_TLB_ESR(r12)
+diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
+index 6f14c8fb6359d..5fb829256b59d 100644
+--- a/arch/powerpc/mm/numa.c
++++ b/arch/powerpc/mm/numa.c
+@@ -376,9 +376,9 @@ static void initialize_form2_numa_distance_lookup_table(void)
+ {
+ int i, j;
+ struct device_node *root;
+- const __u8 *numa_dist_table;
++ const __u8 *form2_distances;
+ const __be32 *numa_lookup_index;
+- int numa_dist_table_length;
++ int form2_distances_length;
+ int max_numa_index, distance_index;
+
+ if (firmware_has_feature(FW_FEATURE_OPAL))
+@@ -392,45 +392,41 @@ static void initialize_form2_numa_distance_lookup_table(void)
+ max_numa_index = of_read_number(&numa_lookup_index[0], 1);
+
+ /* first element of the array is the size and is encode-int */
+- numa_dist_table = of_get_property(root, "ibm,numa-distance-table", NULL);
+- numa_dist_table_length = of_read_number((const __be32 *)&numa_dist_table[0], 1);
++ form2_distances = of_get_property(root, "ibm,numa-distance-table", NULL);
++ form2_distances_length = of_read_number((const __be32 *)&form2_distances[0], 1);
+ /* Skip the size which is encoded int */
+- numa_dist_table += sizeof(__be32);
++ form2_distances += sizeof(__be32);
+
+- pr_debug("numa_dist_table_len = %d, numa_dist_indexes_len = %d\n",
+- numa_dist_table_length, max_numa_index);
++ pr_debug("form2_distances_len = %d, numa_dist_indexes_len = %d\n",
++ form2_distances_length, max_numa_index);
+
+ for (i = 0; i < max_numa_index; i++)
+ /* +1 skip the max_numa_index in the property */
+ numa_id_index_table[i] = of_read_number(&numa_lookup_index[i + 1], 1);
+
+
+- if (numa_dist_table_length != max_numa_index * max_numa_index) {
++ if (form2_distances_length != max_numa_index * max_numa_index) {
+ WARN(1, "Wrong NUMA distance information\n");
+- /* consider everybody else just remote. */
+- for (i = 0; i < max_numa_index; i++) {
+- for (j = 0; j < max_numa_index; j++) {
+- int nodeA = numa_id_index_table[i];
+- int nodeB = numa_id_index_table[j];
+-
+- if (nodeA == nodeB)
+- numa_distance_table[nodeA][nodeB] = LOCAL_DISTANCE;
+- else
+- numa_distance_table[nodeA][nodeB] = REMOTE_DISTANCE;
+- }
+- }
++ form2_distances = NULL; // don't use it
+ }
+-
+ distance_index = 0;
+ for (i = 0; i < max_numa_index; i++) {
+ for (j = 0; j < max_numa_index; j++) {
+ int nodeA = numa_id_index_table[i];
+ int nodeB = numa_id_index_table[j];
+-
+- numa_distance_table[nodeA][nodeB] = numa_dist_table[distance_index++];
+- pr_debug("dist[%d][%d]=%d ", nodeA, nodeB, numa_distance_table[nodeA][nodeB]);
++ int dist;
++
++ if (form2_distances)
++ dist = form2_distances[distance_index++];
++ else if (nodeA == nodeB)
++ dist = LOCAL_DISTANCE;
++ else
++ dist = REMOTE_DISTANCE;
++ numa_distance_table[nodeA][nodeB] = dist;
++ pr_debug("dist[%d][%d]=%d ", nodeA, nodeB, dist);
+ }
+ }
++
+ of_node_put(root);
+ }
+
+@@ -960,7 +956,9 @@ static int __init parse_numa_properties(void)
+ of_node_put(cpu);
+ }
+
+- node_set_online(nid);
++ /* node_set_online() is an UB if 'nid' is negative */
++ if (likely(nid >= 0))
++ node_set_online(nid);
+ }
+
+ get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
+diff --git a/arch/powerpc/mm/pageattr.c b/arch/powerpc/mm/pageattr.c
+index edea388e9d3fb..85753e32a4de9 100644
+--- a/arch/powerpc/mm/pageattr.c
++++ b/arch/powerpc/mm/pageattr.c
+@@ -15,12 +15,14 @@
+ #include <asm/pgtable.h>
+
+
++static pte_basic_t pte_update_delta(pte_t *ptep, unsigned long addr,
++ unsigned long old, unsigned long new)
++{
++ return pte_update(&init_mm, addr, ptep, old & ~new, new & ~old, 0);
++}
++
+ /*
+- * Updates the attributes of a page in three steps:
+- *
+- * 1. take the page_table_lock
+- * 2. install the new entry with the updated attributes
+- * 3. flush the TLB
++ * Updates the attributes of a page atomically.
+ *
+ * This sequence is safe against concurrent updates, and also allows updating the
+ * attributes of a page currently being executed or accessed.
+@@ -28,41 +30,39 @@
+ static int change_page_attr(pte_t *ptep, unsigned long addr, void *data)
+ {
+ long action = (long)data;
+- pte_t pte;
+
+- spin_lock(&init_mm.page_table_lock);
+-
+- pte = ptep_get(ptep);
+-
+- /* modify the PTE bits as desired, then apply */
++ /* modify the PTE bits as desired */
+ switch (action) {
+ case SET_MEMORY_RO:
+- pte = pte_wrprotect(pte);
++ /* Don't clear DIRTY bit */
++ pte_update_delta(ptep, addr, _PAGE_KERNEL_RW & ~_PAGE_DIRTY, _PAGE_KERNEL_RO);
+ break;
+ case SET_MEMORY_RW:
+- pte = pte_mkwrite(pte_mkdirty(pte));
++ pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_RW);
+ break;
+ case SET_MEMORY_NX:
+- pte = pte_exprotect(pte);
++ pte_update_delta(ptep, addr, _PAGE_KERNEL_ROX, _PAGE_KERNEL_RO);
+ break;
+ case SET_MEMORY_X:
+- pte = pte_mkexec(pte);
++ pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_ROX);
++ break;
++ case SET_MEMORY_NP:
++ pte_update(&init_mm, addr, ptep, _PAGE_PRESENT, 0, 0);
++ break;
++ case SET_MEMORY_P:
++ pte_update(&init_mm, addr, ptep, 0, _PAGE_PRESENT, 0);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+- pte_update(&init_mm, addr, ptep, ~0UL, pte_val(pte), 0);
+-
+ /* See ptesync comment in radix__set_pte_at() */
+ if (radix_enabled())
+ asm volatile("ptesync": : :"memory");
+
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+
+- spin_unlock(&init_mm.page_table_lock);
+-
+ return 0;
+ }
+
+@@ -96,36 +96,3 @@ int change_memory_attr(unsigned long addr, int numpages, long action)
+ return apply_to_existing_page_range(&init_mm, start, size,
+ change_page_attr, (void *)action);
+ }
+-
+-/*
+- * Set the attributes of a page:
+- *
+- * This function is used by PPC32 at the end of init to set final kernel memory
+- * protection. It includes changing the maping of the page it is executing from
+- * and data pages it is using.
+- */
+-static int set_page_attr(pte_t *ptep, unsigned long addr, void *data)
+-{
+- pgprot_t prot = __pgprot((unsigned long)data);
+-
+- spin_lock(&init_mm.page_table_lock);
+-
+- set_pte_at(&init_mm, addr, ptep, pte_modify(*ptep, prot));
+- flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+-
+- spin_unlock(&init_mm.page_table_lock);
+-
+- return 0;
+-}
+-
+-int set_memory_attr(unsigned long addr, int numpages, pgprot_t prot)
+-{
+- unsigned long start = ALIGN_DOWN(addr, PAGE_SIZE);
+- unsigned long sz = numpages * PAGE_SIZE;
+-
+- if (numpages <= 0)
+- return 0;
+-
+- return apply_to_existing_page_range(&init_mm, start, sz, set_page_attr,
+- (void *)pgprot_val(prot));
+-}
+diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
+index cd16b407f47e1..9a93c1a5aa1d1 100644
+--- a/arch/powerpc/mm/pgtable.c
++++ b/arch/powerpc/mm/pgtable.c
+@@ -203,6 +203,15 @@ void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
+ __set_pte_at(mm, addr, ptep, pte, 0);
+ }
+
++void unmap_kernel_page(unsigned long va)
++{
++ pmd_t *pmdp = pmd_off_k(va);
++ pte_t *ptep = pte_offset_kernel(pmdp, va);
++
++ pte_clear(&init_mm, va, ptep);
++ flush_tlb_kernel_range(va, va + PAGE_SIZE);
++}
++
+ /*
+ * This is called when relaxing access to a PTE. It's also called in the page
+ * fault path when we don't hit any of the major fault cases, ie, a minor
+diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
+index dcf5ecca19d99..f288597714407 100644
+--- a/arch/powerpc/mm/pgtable_32.c
++++ b/arch/powerpc/mm/pgtable_32.c
+@@ -138,10 +138,12 @@ void mark_initmem_nx(void)
+ unsigned long numpages = PFN_UP((unsigned long)_einittext) -
+ PFN_DOWN((unsigned long)_sinittext);
+
+- if (v_block_mapped((unsigned long)_sinittext))
++ if (v_block_mapped((unsigned long)_sinittext)) {
+ mmu_mark_initmem_nx();
+- else
+- set_memory_attr((unsigned long)_sinittext, numpages, PAGE_KERNEL);
++ } else {
++ set_memory_nx((unsigned long)_sinittext, numpages);
++ set_memory_rw((unsigned long)_sinittext, numpages);
++ }
+ }
+
+ #ifdef CONFIG_STRICT_KERNEL_RWX
+@@ -155,25 +157,21 @@ void mark_rodata_ro(void)
+ return;
+ }
+
+- numpages = PFN_UP((unsigned long)_etext) -
+- PFN_DOWN((unsigned long)_stext);
+-
+- set_memory_attr((unsigned long)_stext, numpages, PAGE_KERNEL_ROX);
+ /*
+- * mark .rodata as read only. Use __init_begin rather than __end_rodata
+- * to cover NOTES and EXCEPTION_TABLE.
++ * mark .text and .rodata as read only. Use __init_begin rather than
++ * __end_rodata to cover NOTES and EXCEPTION_TABLE.
+ */
+ numpages = PFN_UP((unsigned long)__init_begin) -
+- PFN_DOWN((unsigned long)__start_rodata);
++ PFN_DOWN((unsigned long)_stext);
+
+- set_memory_attr((unsigned long)__start_rodata, numpages, PAGE_KERNEL_RO);
++ set_memory_ro((unsigned long)_stext, numpages);
+
+ // mark_initmem_nx() should have already run by now
+ ptdump_check_wx();
+ }
+ #endif
+
+-#ifdef CONFIG_DEBUG_PAGEALLOC
++#if defined(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && defined(CONFIG_DEBUG_PAGEALLOC)
+ void __kernel_map_pages(struct page *page, int numpages, int enable)
+ {
+ unsigned long addr = (unsigned long)page_address(page);
+@@ -182,8 +180,8 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)
+ return;
+
+ if (enable)
+- set_memory_attr(addr, numpages, PAGE_KERNEL);
++ set_memory_p(addr, numpages);
+ else
+- set_memory_attr(addr, numpages, __pgprot(0));
++ set_memory_np(addr, numpages);
+ }
+ #endif /* CONFIG_DEBUG_PAGEALLOC */
+diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
+index 78c8cf01db5f9..175aabf101e87 100644
+--- a/arch/powerpc/mm/pgtable_64.c
++++ b/arch/powerpc/mm/pgtable_64.c
+@@ -102,7 +102,8 @@ EXPORT_SYMBOL(__pte_frag_size_shift);
+ struct page *p4d_page(p4d_t p4d)
+ {
+ if (p4d_is_leaf(p4d)) {
+- VM_WARN_ON(!p4d_huge(p4d));
++ if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
++ VM_WARN_ON(!p4d_huge(p4d));
+ return pte_page(p4d_pte(p4d));
+ }
+ return virt_to_page(p4d_pgtable(p4d));
+@@ -112,7 +113,8 @@ struct page *p4d_page(p4d_t p4d)
+ struct page *pud_page(pud_t pud)
+ {
+ if (pud_is_leaf(pud)) {
+- VM_WARN_ON(!pud_huge(pud));
++ if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
++ VM_WARN_ON(!pud_huge(pud));
+ return pte_page(pud_pte(pud));
+ }
+ return virt_to_page(pud_pgtable(pud));
+@@ -125,7 +127,13 @@ struct page *pud_page(pud_t pud)
+ struct page *pmd_page(pmd_t pmd)
+ {
+ if (pmd_is_leaf(pmd)) {
+- VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd)));
++ /*
++ * vmalloc_to_page may be called on any vmap address (not only
++ * vmalloc), and it uses pmd_page() etc., when huge vmap is
++ * enabled so these checks can't be used.
++ */
++ if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
++ VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd)));
+ return pte_page(pmd_pte(pmd));
+ }
+ return virt_to_page(pmd_page_vaddr(pmd));
+diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c
+index bf251191e78d9..32bfb215c4858 100644
+--- a/arch/powerpc/mm/ptdump/ptdump.c
++++ b/arch/powerpc/mm/ptdump/ptdump.c
+@@ -183,7 +183,7 @@ static void note_prot_wx(struct pg_state *st, unsigned long addr)
+ {
+ pte_t pte = __pte(st->current_flags);
+
+- if (!IS_ENABLED(CONFIG_PPC_DEBUG_WX) || !st->check_wx)
++ if (!IS_ENABLED(CONFIG_DEBUG_WX) || !st->check_wx)
+ return;
+
+ if (!pte_write(pte) || !pte_exec(pte))
+diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
+index fcbf7a917c566..8acf8a611a265 100644
+--- a/arch/powerpc/net/bpf_jit_comp.c
++++ b/arch/powerpc/net/bpf_jit_comp.c
+@@ -23,15 +23,15 @@ static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
+ memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
+ }
+
+-/* Fix the branch target addresses for subprog calls */
+-static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image,
+- struct codegen_context *ctx, u32 *addrs)
++/* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */
++static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,
++ struct codegen_context *ctx, u32 *addrs)
+ {
+ const struct bpf_insn *insn = fp->insnsi;
+ bool func_addr_fixed;
+ u64 func_addr;
+ u32 tmp_idx;
+- int i, ret;
++ int i, j, ret;
+
+ for (i = 0; i < fp->len; i++) {
+ /*
+@@ -66,6 +66,23 @@ static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image,
+ * of the JITed sequence remains unchanged.
+ */
+ ctx->idx = tmp_idx;
++ } else if (insn[i].code == (BPF_LD | BPF_IMM | BPF_DW)) {
++ tmp_idx = ctx->idx;
++ ctx->idx = addrs[i] / 4;
++#ifdef CONFIG_PPC32
++ PPC_LI32(ctx->b2p[insn[i].dst_reg] - 1, (u32)insn[i + 1].imm);
++ PPC_LI32(ctx->b2p[insn[i].dst_reg], (u32)insn[i].imm);
++ for (j = ctx->idx - addrs[i] / 4; j < 4; j++)
++ EMIT(PPC_RAW_NOP());
++#else
++ func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32);
++ PPC_LI64(b2p[insn[i].dst_reg], func_addr);
++ /* overwrite rest with nops */
++ for (j = ctx->idx - addrs[i] / 4; j < 5; j++)
++ EMIT(PPC_RAW_NOP());
++#endif
++ ctx->idx = tmp_idx;
++ i++;
+ }
+ }
+
+@@ -193,13 +210,13 @@ skip_init_ctx:
+ /*
+ * Do not touch the prologue and epilogue as they will remain
+ * unchanged. Only fix the branch target address for subprog
+- * calls in the body.
++ * calls in the body, and ldimm64 instructions.
+ *
+ * This does not change the offsets and lengths of the subprog
+ * call instruction sequences and hence, the size of the JITed
+ * image as well.
+ */
+- bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
++ bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs);
+
+ /* There is no need to perform the usual passes. */
+ goto skip_codegen_passes;
+@@ -241,8 +258,8 @@ skip_codegen_passes:
+ fp->jited_len = alloclen;
+
+ bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
+- bpf_jit_binary_lock_ro(bpf_hdr);
+ if (!fp->is_func || extra_pass) {
++ bpf_jit_binary_lock_ro(bpf_hdr);
+ bpf_prog_fill_jited_linfo(fp, addrs);
+ out_addrs:
+ kfree(addrs);
+diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
+index 0da31d41d4131..bce5eda85170f 100644
+--- a/arch/powerpc/net/bpf_jit_comp32.c
++++ b/arch/powerpc/net/bpf_jit_comp32.c
+@@ -191,6 +191,9 @@ void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 fun
+
+ if (image && rel < 0x2000000 && rel >= -0x2000000) {
+ PPC_BL_ABS(func);
++ EMIT(PPC_RAW_NOP());
++ EMIT(PPC_RAW_NOP());
++ EMIT(PPC_RAW_NOP());
+ } else {
+ /* Load function address into r0 */
+ EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
+@@ -289,6 +292,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
+ bool func_addr_fixed;
+ u64 func_addr;
+ u32 true_cond;
++ u32 tmp_idx;
++ int j;
+
+ /*
+ * addrs[] maps a BPF bytecode address into a real offset from
+@@ -836,8 +841,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
+ * 16 byte instruction that uses two 'struct bpf_insn'
+ */
+ case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
++ tmp_idx = ctx->idx;
+ PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
+ PPC_LI32(dst_reg, (u32)insn[i].imm);
++ /* padding to allow full 4 instructions for later patching */
++ for (j = ctx->idx - tmp_idx; j < 4; j++)
++ EMIT(PPC_RAW_NOP());
+ /* Adjust for two bpf instructions */
+ addrs[++i] = ctx->idx * 4;
+ break;
+diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
+index 8b5157ccfebae..57e1b6680365c 100644
+--- a/arch/powerpc/net/bpf_jit_comp64.c
++++ b/arch/powerpc/net/bpf_jit_comp64.c
+@@ -318,6 +318,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
+ u64 imm64;
+ u32 true_cond;
+ u32 tmp_idx;
++ int j;
+
+ /*
+ * addrs[] maps a BPF bytecode address into a real offset from
+@@ -632,17 +633,21 @@ bpf_alu32_trunc:
+ EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
+ break;
+ case 64:
+- /*
+- * Way easier and faster(?) to store the value
+- * into stack and then use ldbrx
+- *
+- * ctx->seen will be reliable in pass2, but
+- * the instructions generated will remain the
+- * same across all passes
+- */
++ /* Store the value to stack and then use byte-reverse loads */
+ PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
+ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
+- EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
++ if (cpu_has_feature(CPU_FTR_ARCH_206)) {
++ EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
++ } else {
++ EMIT(PPC_RAW_LWBRX(dst_reg, 0, b2p[TMP_REG_1]));
++ if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
++ EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
++ EMIT(PPC_RAW_LI(b2p[TMP_REG_2], 4));
++ EMIT(PPC_RAW_LWBRX(b2p[TMP_REG_2], b2p[TMP_REG_2], b2p[TMP_REG_1]));
++ if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
++ EMIT(PPC_RAW_SLDI(b2p[TMP_REG_2], b2p[TMP_REG_2], 32));
++ EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_2]));
++ }
+ break;
+ }
+ break;
+@@ -806,9 +811,13 @@ emit_clear:
+ case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
+ imm64 = ((u64)(u32) insn[i].imm) |
+ (((u64)(u32) insn[i+1].imm) << 32);
++ tmp_idx = ctx->idx;
++ PPC_LI64(dst_reg, imm64);
++ /* padding to allow full 5 instructions for later patching */
++ for (j = ctx->idx - tmp_idx; j < 5; j++)
++ EMIT(PPC_RAW_NOP());
+ /* Adjust for two bpf instructions */
+ addrs[++i] = ctx->idx * 4;
+- PPC_LI64(dst_reg, imm64);
+ break;
+
+ /*
+diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile
+index 2f46e31c76129..4f53d0b97539b 100644
+--- a/arch/powerpc/perf/Makefile
++++ b/arch/powerpc/perf/Makefile
+@@ -3,11 +3,11 @@
+ obj-y += callchain.o callchain_$(BITS).o perf_regs.o
+ obj-$(CONFIG_COMPAT) += callchain_32.o
+
+-obj-$(CONFIG_PPC_PERF_CTRS) += core-book3s.o bhrb.o
++obj-$(CONFIG_PPC_PERF_CTRS) += core-book3s.o
+ obj64-$(CONFIG_PPC_PERF_CTRS) += ppc970-pmu.o power5-pmu.o \
+ power5+-pmu.o power6-pmu.o power7-pmu.o \
+ isa207-common.o power8-pmu.o power9-pmu.o \
+- generic-compat-pmu.o power10-pmu.o
++ generic-compat-pmu.o power10-pmu.o bhrb.o
+ obj32-$(CONFIG_PPC_PERF_CTRS) += mpc7450-pmu.o
+
+ obj-$(CONFIG_PPC_POWERNV) += imc-pmu.o
+diff --git a/arch/powerpc/perf/callchain.h b/arch/powerpc/perf/callchain.h
+index d6fa6e25234f4..19a8d051ddf10 100644
+--- a/arch/powerpc/perf/callchain.h
++++ b/arch/powerpc/perf/callchain.h
+@@ -2,7 +2,6 @@
+ #ifndef _POWERPC_PERF_CALLCHAIN_H
+ #define _POWERPC_PERF_CALLCHAIN_H
+
+-int read_user_stack_slow(const void __user *ptr, void *buf, int nb);
+ void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs);
+ void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry,
+@@ -26,17 +25,11 @@ static inline int __read_user_stack(const void __user *ptr, void *ret,
+ size_t size)
+ {
+ unsigned long addr = (unsigned long)ptr;
+- int rc;
+
+ if (addr > TASK_SIZE - size || (addr & (size - 1)))
+ return -EFAULT;
+
+- rc = copy_from_user_nofault(ret, ptr, size);
+-
+- if (IS_ENABLED(CONFIG_PPC64) && !radix_enabled() && rc)
+- return read_user_stack_slow(ptr, ret, size);
+-
+- return rc;
++ return copy_from_user_nofault(ret, ptr, size);
+ }
+
+ #endif /* _POWERPC_PERF_CALLCHAIN_H */
+diff --git a/arch/powerpc/perf/callchain_64.c b/arch/powerpc/perf/callchain_64.c
+index 8d0df4226328d..488e8a21a11ea 100644
+--- a/arch/powerpc/perf/callchain_64.c
++++ b/arch/powerpc/perf/callchain_64.c
+@@ -18,33 +18,6 @@
+
+ #include "callchain.h"
+
+-/*
+- * On 64-bit we don't want to invoke hash_page on user addresses from
+- * interrupt context, so if the access faults, we read the page tables
+- * to find which page (if any) is mapped and access it directly. Radix
+- * has no need for this so it doesn't use read_user_stack_slow.
+- */
+-int read_user_stack_slow(const void __user *ptr, void *buf, int nb)
+-{
+-
+- unsigned long addr = (unsigned long) ptr;
+- unsigned long offset;
+- struct page *page;
+- void *kaddr;
+-
+- if (get_user_page_fast_only(addr, FOLL_WRITE, &page)) {
+- kaddr = page_address(page);
+-
+- /* align address to page boundary */
+- offset = addr & ~PAGE_MASK;
+-
+- memcpy(buf, kaddr + offset, nb);
+- put_page(page);
+- return 0;
+- }
+- return -EFAULT;
+-}
+-
+ static int read_user_stack_64(const unsigned long __user *ptr, unsigned long *ret)
+ {
+ return __read_user_stack(ptr, ret, sizeof(*ret));
+diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
+index 73e62e9b179bc..e78de70509472 100644
+--- a/arch/powerpc/perf/core-book3s.c
++++ b/arch/powerpc/perf/core-book3s.c
+@@ -857,6 +857,19 @@ static void write_pmc(int idx, unsigned long val)
+ }
+ }
+
++static int any_pmc_overflown(struct cpu_hw_events *cpuhw)
++{
++ int i, idx;
++
++ for (i = 0; i < cpuhw->n_events; i++) {
++ idx = cpuhw->event[i]->hw.idx;
++ if ((idx) && ((int)read_pmc(idx) < 0))
++ return idx;
++ }
++
++ return 0;
++}
++
+ /* Called from sysrq_handle_showregs() */
+ void perf_event_print_debug(void)
+ {
+@@ -1281,11 +1294,13 @@ static void power_pmu_disable(struct pmu *pmu)
+
+ /*
+ * Set the 'freeze counters' bit, clear EBE/BHRBA/PMCC/PMAO/FC56
++ * Also clear PMXE to disable PMI's getting triggered in some
++ * corner cases during PMU disable.
+ */
+ val = mmcr0 = mfspr(SPRN_MMCR0);
+ val |= MMCR0_FC;
+ val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO |
+- MMCR0_FC56);
++ MMCR0_PMXE | MMCR0_FC56);
+ /* Set mmcr0 PMCCEXT for p10 */
+ if (ppmu->flags & PPMU_ARCH_31)
+ val |= MMCR0_PMCCEXT;
+@@ -1299,6 +1314,34 @@ static void power_pmu_disable(struct pmu *pmu)
+ mb();
+ isync();
+
++ /*
++ * Some corner cases could clear the PMU counter overflow
++ * while a masked PMI is pending. One such case is when
++ * a PMI happens during interrupt replay and perf counter
++ * values are cleared by PMU callbacks before replay.
++ *
++ * If any PMC corresponding to the active PMU events are
++ * overflown, disable the interrupt by clearing the paca
++ * bit for PMI since we are disabling the PMU now.
++ * Otherwise provide a warning if there is PMI pending, but
++ * no counter is found overflown.
++ */
++ if (any_pmc_overflown(cpuhw)) {
++ /*
++ * Since power_pmu_disable runs under local_irq_save, it
++ * could happen that code hits a PMC overflow without PMI
++ * pending in paca. Hence only clear PMI pending if it was
++ * set.
++ *
++ * If a PMI is pending, then MSR[EE] must be disabled (because
++ * the masked PMI handler disabling EE). So it is safe to
++ * call clear_pmi_irq_pending().
++ */
++ if (pmi_irq_pending())
++ clear_pmi_irq_pending();
++ } else
++ WARN_ON(pmi_irq_pending());
++
+ val = mmcra = cpuhw->mmcr.mmcra;
+
+ /*
+@@ -1390,6 +1433,15 @@ static void power_pmu_enable(struct pmu *pmu)
+ * (possibly updated for removal of events).
+ */
+ if (!cpuhw->n_added) {
++ /*
++ * If there is any active event with an overflown PMC
++ * value, set back PACA_IRQ_PMI which would have been
++ * cleared in power_pmu_disable().
++ */
++ hard_irq_disable();
++ if (any_pmc_overflown(cpuhw))
++ set_pmi_irq_pending();
++
+ mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
+ mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
+ if (ppmu->flags & PPMU_ARCH_31)
+@@ -2337,6 +2389,14 @@ static void __perf_event_interrupt(struct pt_regs *regs)
+ break;
+ }
+ }
++
++ /*
++ * Clear PACA_IRQ_PMI in case it was set by
++ * set_pmi_irq_pending() when PMU was enabled
++ * after accounting for interrupts.
++ */
++ clear_pmi_irq_pending();
++
+ if (!active)
+ /* reset non active counters that have overflowed */
+ write_pmc(i + 1, 0);
+@@ -2356,6 +2416,13 @@ static void __perf_event_interrupt(struct pt_regs *regs)
+ }
+ }
+ }
++
++ /*
++ * During system wide profling or while specific CPU is monitored for an
++ * event, some corner cases could cause PMC to overflow in idle path. This
++ * will trigger a PMI after waking up from idle. Since counter values are _not_
++ * saved/restored in idle path, can lead to below "Can't find PMC" message.
++ */
+ if (unlikely(!found) && !arch_irq_disabled_regs(regs))
+ printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n");
+
+diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c
+index e106909ff9c37..e7583fbcc8fa1 100644
+--- a/arch/powerpc/perf/imc-pmu.c
++++ b/arch/powerpc/perf/imc-pmu.c
+@@ -1457,7 +1457,11 @@ static int trace_imc_event_init(struct perf_event *event)
+
+ event->hw.idx = -1;
+
+- event->pmu->task_ctx_nr = perf_hw_context;
++ /*
++ * There can only be a single PMU for perf_hw_context events which is assigned to
++ * core PMU. Hence use "perf_sw_context" for trace_imc.
++ */
++ event->pmu->task_ctx_nr = perf_sw_context;
+ event->destroy = reset_global_refc;
+ return 0;
+ }
+diff --git a/arch/powerpc/perf/power10-events-list.h b/arch/powerpc/perf/power10-events-list.h
+index 93be7197d2502..564f14097f07b 100644
+--- a/arch/powerpc/perf/power10-events-list.h
++++ b/arch/powerpc/perf/power10-events-list.h
+@@ -9,10 +9,10 @@
+ /*
+ * Power10 event codes.
+ */
+-EVENT(PM_RUN_CYC, 0x600f4);
++EVENT(PM_CYC, 0x600f4);
+ EVENT(PM_DISP_STALL_CYC, 0x100f8);
+ EVENT(PM_EXEC_STALL, 0x30008);
+-EVENT(PM_RUN_INST_CMPL, 0x500fa);
++EVENT(PM_INST_CMPL, 0x500fa);
+ EVENT(PM_BR_CMPL, 0x4d05e);
+ EVENT(PM_BR_MPRED_CMPL, 0x400f6);
+ EVENT(PM_BR_FIN, 0x2f04a);
+@@ -50,8 +50,8 @@ EVENT(PM_DTLB_MISS, 0x300fc);
+ /* ITLB Reloaded */
+ EVENT(PM_ITLB_MISS, 0x400fc);
+
+-EVENT(PM_RUN_CYC_ALT, 0x0001e);
+-EVENT(PM_RUN_INST_CMPL_ALT, 0x00002);
++EVENT(PM_CYC_ALT, 0x0001e);
++EVENT(PM_INST_CMPL_ALT, 0x00002);
+
+ /*
+ * Memory Access Events
+diff --git a/arch/powerpc/perf/power10-pmu.c b/arch/powerpc/perf/power10-pmu.c
+index f9d64c63bb4a7..07ca62d084d9d 100644
+--- a/arch/powerpc/perf/power10-pmu.c
++++ b/arch/powerpc/perf/power10-pmu.c
+@@ -91,8 +91,8 @@ extern u64 PERF_REG_EXTENDED_MASK;
+
+ /* Table of alternatives, sorted by column 0 */
+ static const unsigned int power10_event_alternatives[][MAX_ALT] = {
+- { PM_RUN_CYC_ALT, PM_RUN_CYC },
+- { PM_RUN_INST_CMPL_ALT, PM_RUN_INST_CMPL },
++ { PM_INST_CMPL_ALT, PM_INST_CMPL },
++ { PM_CYC_ALT, PM_CYC },
+ };
+
+ static int power10_get_alternatives(u64 event, unsigned int flags, u64 alt[])
+@@ -118,8 +118,8 @@ static int power10_check_attr_config(struct perf_event *ev)
+ return 0;
+ }
+
+-GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC);
+-GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL);
++GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC);
++GENERIC_EVENT_ATTR(instructions, PM_INST_CMPL);
+ GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);
+ GENERIC_EVENT_ATTR(branch-misses, PM_BR_MPRED_CMPL);
+ GENERIC_EVENT_ATTR(cache-references, PM_LD_REF_L1);
+@@ -148,8 +148,8 @@ CACHE_EVENT_ATTR(dTLB-load-misses, PM_DTLB_MISS);
+ CACHE_EVENT_ATTR(iTLB-load-misses, PM_ITLB_MISS);
+
+ static struct attribute *power10_events_attr_dd1[] = {
+- GENERIC_EVENT_PTR(PM_RUN_CYC),
+- GENERIC_EVENT_PTR(PM_RUN_INST_CMPL),
++ GENERIC_EVENT_PTR(PM_CYC),
++ GENERIC_EVENT_PTR(PM_INST_CMPL),
+ GENERIC_EVENT_PTR(PM_BR_CMPL),
+ GENERIC_EVENT_PTR(PM_BR_MPRED_CMPL),
+ GENERIC_EVENT_PTR(PM_LD_REF_L1),
+@@ -173,8 +173,8 @@ static struct attribute *power10_events_attr_dd1[] = {
+ };
+
+ static struct attribute *power10_events_attr[] = {
+- GENERIC_EVENT_PTR(PM_RUN_CYC),
+- GENERIC_EVENT_PTR(PM_RUN_INST_CMPL),
++ GENERIC_EVENT_PTR(PM_CYC),
++ GENERIC_EVENT_PTR(PM_INST_CMPL),
+ GENERIC_EVENT_PTR(PM_BR_FIN),
+ GENERIC_EVENT_PTR(PM_MPRED_BR_FIN),
+ GENERIC_EVENT_PTR(PM_LD_REF_L1),
+@@ -271,8 +271,8 @@ static const struct attribute_group *power10_pmu_attr_groups[] = {
+ };
+
+ static int power10_generic_events_dd1[] = {
+- [PERF_COUNT_HW_CPU_CYCLES] = PM_RUN_CYC,
+- [PERF_COUNT_HW_INSTRUCTIONS] = PM_RUN_INST_CMPL,
++ [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC,
++ [PERF_COUNT_HW_INSTRUCTIONS] = PM_INST_CMPL,
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_CMPL,
+ [PERF_COUNT_HW_BRANCH_MISSES] = PM_BR_MPRED_CMPL,
+ [PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1,
+@@ -280,8 +280,8 @@ static int power10_generic_events_dd1[] = {
+ };
+
+ static int power10_generic_events[] = {
+- [PERF_COUNT_HW_CPU_CYCLES] = PM_RUN_CYC,
+- [PERF_COUNT_HW_INSTRUCTIONS] = PM_RUN_INST_CMPL,
++ [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC,
++ [PERF_COUNT_HW_INSTRUCTIONS] = PM_INST_CMPL,
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_FIN,
+ [PERF_COUNT_HW_BRANCH_MISSES] = PM_MPRED_BR_FIN,
+ [PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1,
+@@ -548,6 +548,24 @@ static u64 power10_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
+
+ #undef C
+
++/*
++ * Set the MMCR0[CC56RUN] bit to enable counting for
++ * PMC5 and PMC6 regardless of the state of CTRL[RUN],
++ * so that we can use counters 5 and 6 as PM_INST_CMPL and
++ * PM_CYC.
++ */
++static int power10_compute_mmcr(u64 event[], int n_ev,
++ unsigned int hwc[], struct mmcr_regs *mmcr,
++ struct perf_event *pevents[], u32 flags)
++{
++ int ret;
++
++ ret = isa207_compute_mmcr(event, n_ev, hwc, mmcr, pevents, flags);
++ if (!ret)
++ mmcr->mmcr0 |= MMCR0_C56RUN;
++ return ret;
++}
++
+ static struct power_pmu power10_pmu = {
+ .name = "POWER10",
+ .n_counter = MAX_PMU_COUNTERS,
+@@ -555,7 +573,7 @@ static struct power_pmu power10_pmu = {
+ .test_adder = ISA207_TEST_ADDER,
+ .group_constraint_mask = CNST_CACHE_PMC4_MASK,
+ .group_constraint_val = CNST_CACHE_PMC4_VAL,
+- .compute_mmcr = isa207_compute_mmcr,
++ .compute_mmcr = power10_compute_mmcr,
+ .config_bhrb = power10_config_bhrb,
+ .bhrb_filter_map = power10_bhrb_filter_map,
+ .get_constraint = isa207_get_constraint,
+diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c
+index ff3382140d7e6..cbdd074ee2a70 100644
+--- a/arch/powerpc/perf/power9-pmu.c
++++ b/arch/powerpc/perf/power9-pmu.c
+@@ -133,11 +133,11 @@ int p9_dd22_bl_ev[] = {
+
+ /* Table of alternatives, sorted by column 0 */
+ static const unsigned int power9_event_alternatives[][MAX_ALT] = {
+- { PM_INST_DISP, PM_INST_DISP_ALT },
+- { PM_RUN_CYC_ALT, PM_RUN_CYC },
+- { PM_RUN_INST_CMPL_ALT, PM_RUN_INST_CMPL },
+- { PM_LD_MISS_L1, PM_LD_MISS_L1_ALT },
+ { PM_BR_2PATH, PM_BR_2PATH_ALT },
++ { PM_INST_DISP, PM_INST_DISP_ALT },
++ { PM_RUN_CYC_ALT, PM_RUN_CYC },
++ { PM_LD_MISS_L1, PM_LD_MISS_L1_ALT },
++ { PM_RUN_INST_CMPL_ALT, PM_RUN_INST_CMPL },
+ };
+
+ static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[])
+diff --git a/arch/powerpc/platforms/44x/fsp2.c b/arch/powerpc/platforms/44x/fsp2.c
+index b299e43f5ef94..823397c802def 100644
+--- a/arch/powerpc/platforms/44x/fsp2.c
++++ b/arch/powerpc/platforms/44x/fsp2.c
+@@ -208,6 +208,7 @@ static void node_irq_request(const char *compat, irq_handler_t errirq_handler)
+ if (irq == NO_IRQ) {
+ pr_err("device tree node %pOFn is missing a interrupt",
+ np);
++ of_node_put(np);
+ return;
+ }
+
+@@ -215,6 +216,7 @@ static void node_irq_request(const char *compat, irq_handler_t errirq_handler)
+ if (rc) {
+ pr_err("fsp_of_probe: request_irq failed: np=%pOF rc=%d",
+ np, rc);
++ of_node_put(np);
+ return;
+ }
+ }
+diff --git a/arch/powerpc/platforms/85xx/Makefile b/arch/powerpc/platforms/85xx/Makefile
+index 60e4e97a929db..260fbad7967b2 100644
+--- a/arch/powerpc/platforms/85xx/Makefile
++++ b/arch/powerpc/platforms/85xx/Makefile
+@@ -3,7 +3,9 @@
+ # Makefile for the PowerPC 85xx linux kernel.
+ #
+ obj-$(CONFIG_SMP) += smp.o
+-obj-$(CONFIG_FSL_PMC) += mpc85xx_pm_ops.o
++ifneq ($(CONFIG_FSL_CORENET_RCPM),y)
++obj-$(CONFIG_SMP) += mpc85xx_pm_ops.o
++endif
+
+ obj-y += common.o
+
+diff --git a/arch/powerpc/platforms/85xx/mpc85xx_pm_ops.c b/arch/powerpc/platforms/85xx/mpc85xx_pm_ops.c
+index 7c0133f558d02..4a8af80011a6f 100644
+--- a/arch/powerpc/platforms/85xx/mpc85xx_pm_ops.c
++++ b/arch/powerpc/platforms/85xx/mpc85xx_pm_ops.c
+@@ -17,6 +17,7 @@
+
+ static struct ccsr_guts __iomem *guts;
+
++#ifdef CONFIG_FSL_PMC
+ static void mpc85xx_irq_mask(int cpu)
+ {
+
+@@ -49,6 +50,7 @@ static void mpc85xx_cpu_up_prepare(int cpu)
+ {
+
+ }
++#endif
+
+ static void mpc85xx_freeze_time_base(bool freeze)
+ {
+@@ -76,10 +78,12 @@ static const struct of_device_id mpc85xx_smp_guts_ids[] = {
+
+ static const struct fsl_pm_ops mpc85xx_pm_ops = {
+ .freeze_time_base = mpc85xx_freeze_time_base,
++#ifdef CONFIG_FSL_PMC
+ .irq_mask = mpc85xx_irq_mask,
+ .irq_unmask = mpc85xx_irq_unmask,
+ .cpu_die = mpc85xx_cpu_die,
+ .cpu_up_prepare = mpc85xx_cpu_up_prepare,
++#endif
+ };
+
+ int __init mpc85xx_setup_pmc(void)
+@@ -94,9 +98,8 @@ int __init mpc85xx_setup_pmc(void)
+ pr_err("Could not map guts node address\n");
+ return -ENOMEM;
+ }
++ qoriq_pm_ops = &mpc85xx_pm_ops;
+ }
+
+- qoriq_pm_ops = &mpc85xx_pm_ops;
+-
+ return 0;
+ }
+diff --git a/arch/powerpc/platforms/85xx/smp.c b/arch/powerpc/platforms/85xx/smp.c
+index c6df294054fe9..d7081e9af65c7 100644
+--- a/arch/powerpc/platforms/85xx/smp.c
++++ b/arch/powerpc/platforms/85xx/smp.c
+@@ -40,7 +40,6 @@ struct epapr_spin_table {
+ u32 pir;
+ };
+
+-#ifdef CONFIG_HOTPLUG_CPU
+ static u64 timebase;
+ static int tb_req;
+ static int tb_valid;
+@@ -112,6 +111,7 @@ static void mpc85xx_take_timebase(void)
+ local_irq_restore(flags);
+ }
+
++#ifdef CONFIG_HOTPLUG_CPU
+ static void smp_85xx_cpu_offline_self(void)
+ {
+ unsigned int cpu = smp_processor_id();
+@@ -220,7 +220,7 @@ static int smp_85xx_start_cpu(int cpu)
+ local_irq_save(flags);
+ hard_irq_disable();
+
+- if (qoriq_pm_ops)
++ if (qoriq_pm_ops && qoriq_pm_ops->cpu_up_prepare)
+ qoriq_pm_ops->cpu_up_prepare(cpu);
+
+ /* if cpu is not spinning, reset it */
+@@ -292,7 +292,7 @@ static int smp_85xx_kick_cpu(int nr)
+ booting_thread_hwid = cpu_thread_in_core(nr);
+ primary = cpu_first_thread_sibling(nr);
+
+- if (qoriq_pm_ops)
++ if (qoriq_pm_ops && qoriq_pm_ops->cpu_up_prepare)
+ qoriq_pm_ops->cpu_up_prepare(nr);
+
+ /*
+@@ -495,21 +495,21 @@ void __init mpc85xx_smp_init(void)
+ smp_85xx_ops.probe = NULL;
+ }
+
+-#ifdef CONFIG_HOTPLUG_CPU
+ #ifdef CONFIG_FSL_CORENET_RCPM
++ /* Assign a value to qoriq_pm_ops on PPC_E500MC */
+ fsl_rcpm_init();
+-#endif
+-
+-#ifdef CONFIG_FSL_PMC
++#else
++ /* Assign a value to qoriq_pm_ops on !PPC_E500MC */
+ mpc85xx_setup_pmc();
+ #endif
+ if (qoriq_pm_ops) {
+ smp_85xx_ops.give_timebase = mpc85xx_give_timebase;
+ smp_85xx_ops.take_timebase = mpc85xx_take_timebase;
++#ifdef CONFIG_HOTPLUG_CPU
+ smp_85xx_ops.cpu_offline_self = smp_85xx_cpu_offline_self;
+ smp_85xx_ops.cpu_die = qoriq_cpu_kill;
+- }
+ #endif
++ }
+ smp_ops = &smp_85xx_ops;
+
+ #ifdef CONFIG_KEXEC_CORE
+diff --git a/arch/powerpc/platforms/8xx/pic.c b/arch/powerpc/platforms/8xx/pic.c
+index f2ba837249d69..04a6abf14c295 100644
+--- a/arch/powerpc/platforms/8xx/pic.c
++++ b/arch/powerpc/platforms/8xx/pic.c
+@@ -153,6 +153,7 @@ int __init mpc8xx_pic_init(void)
+ if (mpc8xx_pic_host == NULL) {
+ printk(KERN_ERR "MPC8xx PIC: failed to allocate irq host!\n");
+ ret = -ENOMEM;
++ goto out;
+ }
+
+ ret = 0;
+diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
+index a208997ade88b..87a95cbff2f3f 100644
+--- a/arch/powerpc/platforms/Kconfig.cputype
++++ b/arch/powerpc/platforms/Kconfig.cputype
+@@ -111,6 +111,7 @@ config PPC_BOOK3S_64
+
+ config PPC_BOOK3E_64
+ bool "Embedded processors"
++ select PPC_FSL_BOOK3E
+ select PPC_FPU # Make it a choice ?
+ select PPC_SMP_MUXED_IPI
+ select PPC_DOORBELL
+@@ -287,7 +288,7 @@ config FSL_BOOKE
+ config PPC_FSL_BOOK3E
+ bool
+ select ARCH_SUPPORTS_HUGETLBFS if PHYS_64BIT || PPC64
+- select FSL_EMB_PERFMON
++ imply FSL_EMB_PERFMON
+ select PPC_SMP_MUXED_IPI
+ select PPC_DOORBELL
+ default y if FSL_BOOKE
+diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c
+index 30172e52e16b7..4d82c92ddd523 100644
+--- a/arch/powerpc/platforms/book3s/vas-api.c
++++ b/arch/powerpc/platforms/book3s/vas-api.c
+@@ -303,7 +303,7 @@ static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
+ return -EINVAL;
+ }
+
+- if (!cp_inst->coproc->vops && !cp_inst->coproc->vops->open_win) {
++ if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) {
+ pr_err("VAS API is not registered\n");
+ return -EACCES;
+ }
+@@ -373,7 +373,7 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
+ return -EINVAL;
+ }
+
+- if (!cp_inst->coproc->vops && !cp_inst->coproc->vops->paste_addr) {
++ if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) {
+ pr_err("%s(): VAS API is not registered\n", __func__);
+ return -EACCES;
+ }
+diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c
+index fa08699aedeb8..d32f24de84798 100644
+--- a/arch/powerpc/platforms/cell/iommu.c
++++ b/arch/powerpc/platforms/cell/iommu.c
+@@ -977,6 +977,7 @@ static int __init cell_iommu_fixed_mapping_init(void)
+ if (hbase < dbase || (hend > (dbase + dsize))) {
+ pr_debug("iommu: hash window doesn't fit in"
+ "real DMA window\n");
++ of_node_put(np);
+ return -1;
+ }
+ }
+diff --git a/arch/powerpc/platforms/cell/pervasive.c b/arch/powerpc/platforms/cell/pervasive.c
+index 5b9a7e9f144b3..dff8d5e7ab82b 100644
+--- a/arch/powerpc/platforms/cell/pervasive.c
++++ b/arch/powerpc/platforms/cell/pervasive.c
+@@ -78,6 +78,7 @@ static int cbe_system_reset_exception(struct pt_regs *regs)
+ switch (regs->msr & SRR1_WAKEMASK) {
+ case SRR1_WAKEDEC:
+ set_dec(1);
++ break;
+ case SRR1_WAKEEE:
+ /*
+ * Handle these when interrupts get re-enabled and we take
+diff --git a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
+index 15396333a90bd..a4b020e4b6af0 100644
+--- a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
++++ b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
+@@ -214,6 +214,7 @@ void hlwd_pic_probe(void)
+ irq_set_chained_handler(cascade_virq,
+ hlwd_pic_irq_cascade);
+ hlwd_irq_host = host;
++ of_node_put(np);
+ break;
+ }
+ }
+diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c
+index f77a59b5c2e1a..df89d916236d9 100644
+--- a/arch/powerpc/platforms/powermac/low_i2c.c
++++ b/arch/powerpc/platforms/powermac/low_i2c.c
+@@ -582,6 +582,7 @@ static void __init kw_i2c_add(struct pmac_i2c_host_kw *host,
+ bus->close = kw_i2c_close;
+ bus->xfer = kw_i2c_xfer;
+ mutex_init(&bus->mutex);
++ lockdep_register_key(&bus->lock_key);
+ lockdep_set_class(&bus->mutex, &bus->lock_key);
+ if (controller == busnode)
+ bus->flags = pmac_i2c_multibus;
+@@ -810,6 +811,7 @@ static void __init pmu_i2c_probe(void)
+ bus->hostdata = bus + 1;
+ bus->xfer = pmu_i2c_xfer;
+ mutex_init(&bus->mutex);
++ lockdep_register_key(&bus->lock_key);
+ lockdep_set_class(&bus->mutex, &bus->lock_key);
+ bus->flags = pmac_i2c_multibus;
+ list_add(&bus->link, &pmac_i2c_busses);
+@@ -933,6 +935,7 @@ static void __init smu_i2c_probe(void)
+ bus->hostdata = bus + 1;
+ bus->xfer = smu_i2c_xfer;
+ mutex_init(&bus->mutex);
++ lockdep_register_key(&bus->lock_key);
+ lockdep_set_class(&bus->mutex, &bus->lock_key);
+ bus->flags = 0;
+ list_add(&bus->link, &pmac_i2c_busses);
+diff --git a/arch/powerpc/platforms/powernv/opal-lpc.c b/arch/powerpc/platforms/powernv/opal-lpc.c
+index 1e5d51db40f84..5390c888db162 100644
+--- a/arch/powerpc/platforms/powernv/opal-lpc.c
++++ b/arch/powerpc/platforms/powernv/opal-lpc.c
+@@ -396,6 +396,7 @@ void __init opal_lpc_init(void)
+ if (!of_get_property(np, "primary", NULL))
+ continue;
+ opal_lpc_chip_id = of_get_ibm_chip_id(np);
++ of_node_put(np);
+ break;
+ }
+ if (opal_lpc_chip_id < 0)
+diff --git a/arch/powerpc/platforms/powernv/opal-prd.c b/arch/powerpc/platforms/powernv/opal-prd.c
+index a191f4c60ce71..113bdb151f687 100644
+--- a/arch/powerpc/platforms/powernv/opal-prd.c
++++ b/arch/powerpc/platforms/powernv/opal-prd.c
+@@ -369,6 +369,12 @@ static struct notifier_block opal_prd_event_nb = {
+ .priority = 0,
+ };
+
++static struct notifier_block opal_prd_event_nb2 = {
++ .notifier_call = opal_prd_msg_notifier,
++ .next = NULL,
++ .priority = 0,
++};
++
+ static int opal_prd_probe(struct platform_device *pdev)
+ {
+ int rc;
+@@ -390,9 +396,10 @@ static int opal_prd_probe(struct platform_device *pdev)
+ return rc;
+ }
+
+- rc = opal_message_notifier_register(OPAL_MSG_PRD2, &opal_prd_event_nb);
++ rc = opal_message_notifier_register(OPAL_MSG_PRD2, &opal_prd_event_nb2);
+ if (rc) {
+ pr_err("Couldn't register PRD2 event notifier\n");
++ opal_message_notifier_unregister(OPAL_MSG_PRD, &opal_prd_event_nb);
+ return rc;
+ }
+
+@@ -401,6 +408,8 @@ static int opal_prd_probe(struct platform_device *pdev)
+ pr_err("failed to register miscdev\n");
+ opal_message_notifier_unregister(OPAL_MSG_PRD,
+ &opal_prd_event_nb);
++ opal_message_notifier_unregister(OPAL_MSG_PRD2,
++ &opal_prd_event_nb2);
+ return rc;
+ }
+
+@@ -411,6 +420,7 @@ static int opal_prd_remove(struct platform_device *pdev)
+ {
+ misc_deregister(&opal_prd_dev);
+ opal_message_notifier_unregister(OPAL_MSG_PRD, &opal_prd_event_nb);
++ opal_message_notifier_unregister(OPAL_MSG_PRD2, &opal_prd_event_nb2);
+ return 0;
+ }
+
+diff --git a/arch/powerpc/platforms/powernv/rng.c b/arch/powerpc/platforms/powernv/rng.c
+index 72c25295c1c2b..69c344c8884f3 100644
+--- a/arch/powerpc/platforms/powernv/rng.c
++++ b/arch/powerpc/platforms/powernv/rng.c
+@@ -43,7 +43,11 @@ static unsigned long rng_whiten(struct powernv_rng *rng, unsigned long val)
+ unsigned long parity;
+
+ /* Calculate the parity of the value */
+- asm ("popcntd %0,%1" : "=r" (parity) : "r" (val));
++ asm (".machine push; \
++ .machine power7; \
++ popcntd %0,%1; \
++ .machine pop;"
++ : "=r" (parity) : "r" (val));
+
+ /* xor our value with the previous mask */
+ val ^= rng->mask;
+diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
+index a52af8fbf5711..8322ca86d5acf 100644
+--- a/arch/powerpc/platforms/pseries/iommu.c
++++ b/arch/powerpc/platforms/pseries/iommu.c
+@@ -1092,15 +1092,6 @@ static phys_addr_t ddw_memory_hotplug_max(void)
+ phys_addr_t max_addr = memory_hotplug_max();
+ struct device_node *memory;
+
+- /*
+- * The "ibm,pmemory" can appear anywhere in the address space.
+- * Assuming it is still backed by page structs, set the upper limit
+- * for the huge DMA window as MAX_PHYSMEM_BITS.
+- */
+- if (of_find_node_by_type(NULL, "ibm,pmemory"))
+- return (sizeof(phys_addr_t) * 8 <= MAX_PHYSMEM_BITS) ?
+- (phys_addr_t) -1 : (1ULL << MAX_PHYSMEM_BITS);
+-
+ for_each_node_by_type(memory, "memory") {
+ unsigned long start, size;
+ int n_mem_addr_cells, n_mem_size_cells, len;
+@@ -1365,8 +1356,10 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
+ len = order_base_2(query.largest_available_block << page_shift);
+ win_name = DMA64_PROPNAME;
+ } else {
+- direct_mapping = true;
+- win_name = DIRECT64_PROPNAME;
++ direct_mapping = !default_win_removed ||
++ (len == MAX_PHYSMEM_BITS) ||
++ (!pmem_present && (len == max_ram_len));
++ win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
+ }
+
+ ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
+diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
+index e83e0891272d3..210a37a065fb7 100644
+--- a/arch/powerpc/platforms/pseries/mobility.c
++++ b/arch/powerpc/platforms/pseries/mobility.c
+@@ -63,6 +63,27 @@ static int mobility_rtas_call(int token, char *buf, s32 scope)
+
+ static int delete_dt_node(struct device_node *dn)
+ {
++ struct device_node *pdn;
++ bool is_platfac;
++
++ pdn = of_get_parent(dn);
++ is_platfac = of_node_is_type(dn, "ibm,platform-facilities") ||
++ of_node_is_type(pdn, "ibm,platform-facilities");
++ of_node_put(pdn);
++
++ /*
++ * The drivers that bind to nodes in the platform-facilities
++ * hierarchy don't support node removal, and the removal directive
++ * from firmware is always followed by an add of an equivalent
++ * node. The capability (e.g. RNG, encryption, compression)
++ * represented by the node is never interrupted by the migration.
++ * So ignore changes to this part of the tree.
++ */
++ if (is_platfac) {
++ pr_notice("ignoring remove operation for %pOFfp\n", dn);
++ return 0;
++ }
++
+ pr_debug("removing node %pOFfp\n", dn);
+ dlpar_detach_node(dn);
+ return 0;
+@@ -222,6 +243,19 @@ static int add_dt_node(struct device_node *parent_dn, __be32 drc_index)
+ if (!dn)
+ return -ENOENT;
+
++ /*
++ * Since delete_dt_node() ignores this node type, this is the
++ * necessary counterpart. We also know that a platform-facilities
++ * node returned from dlpar_configure_connector() has children
++ * attached, and dlpar_attach_node() only adds the parent, leaking
++ * the children. So ignore these on the add side for now.
++ */
++ if (of_node_is_type(dn, "ibm,platform-facilities")) {
++ pr_notice("ignoring add operation for %pOF\n", dn);
++ dlpar_free_cc_nodes(dn);
++ return 0;
++ }
++
+ rc = dlpar_attach_node(dn, parent_dn);
+ if (rc)
+ dlpar_free_cc_nodes(dn);
+diff --git a/arch/powerpc/platforms/pseries/pci_dlpar.c b/arch/powerpc/platforms/pseries/pci_dlpar.c
+index 90c9d3531694b..4ba8245681192 100644
+--- a/arch/powerpc/platforms/pseries/pci_dlpar.c
++++ b/arch/powerpc/platforms/pseries/pci_dlpar.c
+@@ -78,6 +78,9 @@ int remove_phb_dynamic(struct pci_controller *phb)
+
+ pseries_msi_free_domains(phb);
+
++ /* Keep a reference so phb isn't freed yet */
++ get_device(&host_bridge->dev);
++
+ /* Remove the PCI bus and unregister the bridge device from sysfs */
+ phb->bus = NULL;
+ pci_remove_bus(b);
+@@ -101,6 +104,7 @@ int remove_phb_dynamic(struct pci_controller *phb)
+ * the pcibios_free_controller_deferred() callback;
+ * see pseries_root_bridge_prepare().
+ */
++ put_device(&host_bridge->dev);
+
+ return 0;
+ }
+diff --git a/arch/powerpc/sysdev/dcr-low.S b/arch/powerpc/sysdev/dcr-low.S
+index efeeb1b885a17..329b9c4ae5429 100644
+--- a/arch/powerpc/sysdev/dcr-low.S
++++ b/arch/powerpc/sysdev/dcr-low.S
+@@ -11,7 +11,7 @@
+ #include <asm/export.h>
+
+ #define DCR_ACCESS_PROLOG(table) \
+- cmpli cr0,r3,1024; \
++ cmplwi cr0,r3,1024; \
+ rlwinm r3,r3,4,18,27; \
+ lis r5,table@h; \
+ ori r5,r5,table@l; \
+diff --git a/arch/powerpc/sysdev/fsl_gtm.c b/arch/powerpc/sysdev/fsl_gtm.c
+index 8963eaffb1b7b..39186ad6b3c3a 100644
+--- a/arch/powerpc/sysdev/fsl_gtm.c
++++ b/arch/powerpc/sysdev/fsl_gtm.c
+@@ -86,7 +86,7 @@ static LIST_HEAD(gtms);
+ */
+ struct gtm_timer *gtm_get_timer16(void)
+ {
+- struct gtm *gtm = NULL;
++ struct gtm *gtm;
+ int i;
+
+ list_for_each_entry(gtm, &gtms, list_node) {
+@@ -103,7 +103,7 @@ struct gtm_timer *gtm_get_timer16(void)
+ spin_unlock_irq(&gtm->lock);
+ }
+
+- if (gtm)
++ if (!list_empty(&gtms))
+ return ERR_PTR(-EBUSY);
+ return ERR_PTR(-ENODEV);
+ }
+diff --git a/arch/powerpc/sysdev/xive/Kconfig b/arch/powerpc/sysdev/xive/Kconfig
+index 97796c6b63f04..785c292d104b7 100644
+--- a/arch/powerpc/sysdev/xive/Kconfig
++++ b/arch/powerpc/sysdev/xive/Kconfig
+@@ -3,7 +3,6 @@ config PPC_XIVE
+ bool
+ select PPC_SMP_MUXED_IPI
+ select HARDIRQS_SW_RESEND
+- select IRQ_DOMAIN_NOMAP
+
+ config PPC_XIVE_NATIVE
+ bool
+diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c
+index c5d75c02ad8b5..7b69299c29123 100644
+--- a/arch/powerpc/sysdev/xive/common.c
++++ b/arch/powerpc/sysdev/xive/common.c
+@@ -1443,8 +1443,7 @@ static const struct irq_domain_ops xive_irq_domain_ops = {
+
+ static void __init xive_init_host(struct device_node *np)
+ {
+- xive_irq_domain = irq_domain_add_nomap(np, XIVE_MAX_IRQ,
+- &xive_irq_domain_ops, NULL);
++ xive_irq_domain = irq_domain_add_tree(np, &xive_irq_domain_ops, NULL);
+ if (WARN_ON(xive_irq_domain == NULL))
+ return;
+ irq_set_default_host(xive_irq_domain);
+diff --git a/arch/powerpc/sysdev/xive/spapr.c b/arch/powerpc/sysdev/xive/spapr.c
+index f143b6f111ac0..1179632560b8d 100644
+--- a/arch/powerpc/sysdev/xive/spapr.c
++++ b/arch/powerpc/sysdev/xive/spapr.c
+@@ -653,6 +653,9 @@ static int xive_spapr_debug_show(struct seq_file *m, void *private)
+ struct xive_irq_bitmap *xibm;
+ char *buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+
++ if (!buf)
++ return -ENOMEM;
++
+ list_for_each_entry(xibm, &xive_irq_bitmaps, list) {
+ memset(buf, 0, PAGE_SIZE);
+ bitmap_print_to_pagebuf(true, buf, xibm->bitmap, xibm->count);
+diff --git a/arch/powerpc/tools/relocs_check.sh b/arch/powerpc/tools/relocs_check.sh
+index 014e00e74d2b6..63792af004170 100755
+--- a/arch/powerpc/tools/relocs_check.sh
++++ b/arch/powerpc/tools/relocs_check.sh
+@@ -39,6 +39,7 @@ $objdump -R "$vmlinux" |
+ # R_PPC_NONE
+ grep -F -w -v 'R_PPC64_RELATIVE
+ R_PPC64_NONE
++R_PPC64_UADDR64
+ R_PPC_ADDR16_LO
+ R_PPC_ADDR16_HI
+ R_PPC_ADDR16_HA
+@@ -54,9 +55,3 @@ fi
+ num_bad=$(echo "$bad_relocs" | wc -l)
+ echo "WARNING: $num_bad bad relocations"
+ echo "$bad_relocs"
+-
+-# If we see this type of relocation it's an idication that
+-# we /may/ be using an old version of binutils.
+-if echo "$bad_relocs" | grep -q -F -w R_PPC64_UADDR64; then
+- echo "WARNING: You need at least binutils >= 2.19 to build a CONFIG_RELOCATABLE kernel"
+-fi
+diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
+index dd8241c009e53..8b28ff9d98d16 100644
+--- a/arch/powerpc/xmon/xmon.c
++++ b/arch/powerpc/xmon/xmon.c
+@@ -3264,8 +3264,7 @@ static void show_task(struct task_struct *volatile tsk)
+ * appropriate for calling from xmon. This could be moved
+ * to a common, generic, routine used by both.
+ */
+- state = (p_state == 0) ? 'R' :
+- (p_state < 0) ? 'U' :
++ state = (p_state == TASK_RUNNING) ? 'R' :
+ (p_state & TASK_UNINTERRUPTIBLE) ? 'D' :
+ (p_state & TASK_STOPPED) ? 'T' :
+ (p_state & TASK_TRACED) ? 'C' :
+diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
+index f076cee11af69..20bc4b8c13b92 100644
+--- a/arch/riscv/Kconfig
++++ b/arch/riscv/Kconfig
+@@ -158,10 +158,9 @@ config PA_BITS
+
+ config PAGE_OFFSET
+ hex
+- default 0xC0000000 if 32BIT && MAXPHYSMEM_1GB
++ default 0xC0000000 if 32BIT
+ default 0x80000000 if 64BIT && !MMU
+- default 0xffffffff80000000 if 64BIT && MAXPHYSMEM_2GB
+- default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB
++ default 0xffffffe000000000 if 64BIT
+
+ config KASAN_SHADOW_OFFSET
+ hex
+@@ -270,24 +269,6 @@ config MODULE_SECTIONS
+ bool
+ select HAVE_MOD_ARCH_SPECIFIC
+
+-choice
+- prompt "Maximum Physical Memory"
+- default MAXPHYSMEM_1GB if 32BIT
+- default MAXPHYSMEM_2GB if 64BIT && CMODEL_MEDLOW
+- default MAXPHYSMEM_128GB if 64BIT && CMODEL_MEDANY
+-
+- config MAXPHYSMEM_1GB
+- depends on 32BIT
+- bool "1GiB"
+- config MAXPHYSMEM_2GB
+- depends on 64BIT && CMODEL_MEDLOW
+- bool "2GiB"
+- config MAXPHYSMEM_128GB
+- depends on 64BIT && CMODEL_MEDANY
+- bool "128GiB"
+-endchoice
+-
+-
+ config SMP
+ bool "Symmetric Multi-Processing"
+ help
+diff --git a/arch/riscv/Kconfig.erratas b/arch/riscv/Kconfig.erratas
+index b44d6ecdb46e5..0aacd7052585b 100644
+--- a/arch/riscv/Kconfig.erratas
++++ b/arch/riscv/Kconfig.erratas
+@@ -2,6 +2,7 @@ menu "CPU errata selection"
+
+ config RISCV_ERRATA_ALTERNATIVE
+ bool "RISC-V alternative scheme"
++ depends on !XIP_KERNEL
+ default y
+ help
+ This Kconfig allows the kernel to automatically patch the
+diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs
+index 30676ebb16ebd..46a534f047931 100644
+--- a/arch/riscv/Kconfig.socs
++++ b/arch/riscv/Kconfig.socs
+@@ -14,8 +14,8 @@ config SOC_SIFIVE
+ select CLK_SIFIVE
+ select CLK_SIFIVE_PRCI
+ select SIFIVE_PLIC
+- select RISCV_ERRATA_ALTERNATIVE
+- select ERRATA_SIFIVE
++ select RISCV_ERRATA_ALTERNATIVE if !XIP_KERNEL
++ select ERRATA_SIFIVE if !XIP_KERNEL
+ help
+ This enables support for SiFive SoC platform hardware.
+
+diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
+index 0eb4568fbd290..e03f45f7711aa 100644
+--- a/arch/riscv/Makefile
++++ b/arch/riscv/Makefile
+@@ -52,6 +52,12 @@ riscv-march-$(CONFIG_ARCH_RV32I) := rv32ima
+ riscv-march-$(CONFIG_ARCH_RV64I) := rv64ima
+ riscv-march-$(CONFIG_FPU) := $(riscv-march-y)fd
+ riscv-march-$(CONFIG_RISCV_ISA_C) := $(riscv-march-y)c
++
++# Newer binutils versions default to ISA spec version 20191213 which moves some
++# instructions from the I extension to the Zicsr and Zifencei extensions.
++toolchain-need-zicsr-zifencei := $(call cc-option-yn, -march=$(riscv-march-y)_zicsr_zifencei)
++riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei
++
+ KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y))
+ KBUILD_AFLAGS += -march=$(riscv-march-y)
+
+@@ -108,11 +114,13 @@ PHONY += vdso_install
+ vdso_install:
+ $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso $@
+
++ifeq ($(KBUILD_EXTMOD),)
+ ifeq ($(CONFIG_MMU),y)
+ prepare: vdso_prepare
+ vdso_prepare: prepare0
+ $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso include/generated/vdso-offsets.h
+ endif
++endif
+
+ ifneq ($(CONFIG_XIP_KERNEL),y)
+ ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_CANAAN),yy)
+diff --git a/arch/riscv/boot/dts/canaan/k210.dtsi b/arch/riscv/boot/dts/canaan/k210.dtsi
+index 5e8ca81424821..780416d489aa7 100644
+--- a/arch/riscv/boot/dts/canaan/k210.dtsi
++++ b/arch/riscv/boot/dts/canaan/k210.dtsi
+@@ -113,7 +113,8 @@
+ compatible = "canaan,k210-plic", "sifive,plic-1.0.0";
+ reg = <0xC000000 0x4000000>;
+ interrupt-controller;
+- interrupts-extended = <&cpu0_intc 11 &cpu1_intc 11>;
++ interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>,
++ <&cpu1_intc 11>, <&cpu1_intc 9>;
+ riscv,ndev = <65>;
+ };
+
+diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts
+index 0bcaf35045e79..82e7f8069ae77 100644
+--- a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts
++++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts
+@@ -203,6 +203,8 @@
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <50000000>;
++ spi-tx-bus-width = <4>;
++ spi-rx-bus-width = <4>;
+ m25p,fast-read;
+ broken-flash-reset;
+ };
+diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts
+index ac8a03f5867ad..8d335233853a7 100644
+--- a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts
++++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts
+@@ -205,6 +205,8 @@
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <50000000>;
++ spi-tx-bus-width = <4>;
++ spi-rx-bus-width = <4>;
+ m25p,fast-read;
+ broken-flash-reset;
+ };
+diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts
+index 623998194bc18..6703cfc055887 100644
+--- a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts
++++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts
+@@ -213,6 +213,8 @@
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <50000000>;
++ spi-tx-bus-width = <4>;
++ spi-rx-bus-width = <4>;
+ m25p,fast-read;
+ broken-flash-reset;
+ };
+diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts
+index cf605ba0d67e4..ac0b56f7d2c9f 100644
+--- a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts
++++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts
+@@ -178,6 +178,8 @@
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ spi-max-frequency = <50000000>;
++ spi-tx-bus-width = <4>;
++ spi-rx-bus-width = <4>;
+ m25p,fast-read;
+ broken-flash-reset;
+ };
+diff --git a/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts
+index b254c60589a1c..cce5eca31f257 100644
+--- a/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts
++++ b/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts
+@@ -12,7 +12,7 @@
+ #address-cells = <2>;
+ #size-cells = <2>;
+ model = "Microchip PolarFire-SoC Icicle Kit";
+- compatible = "microchip,mpfs-icicle-kit";
++ compatible = "microchip,mpfs-icicle-kit", "microchip,mpfs";
+
+ aliases {
+ ethernet0 = &emac1;
+@@ -56,8 +56,17 @@
+ status = "okay";
+ };
+
+-&sdcard {
++&mmc {
+ status = "okay";
++
++ bus-width = <4>;
++ disable-wp;
++ cap-sd-highspeed;
++ card-detect-delay = <200>;
++ sd-uhs-sdr12;
++ sd-uhs-sdr25;
++ sd-uhs-sdr50;
++ sd-uhs-sdr104;
+ };
+
+ &emac0 {
+diff --git a/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi b/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi
+index 9d2fbbc1f7778..4ef4bcb748729 100644
+--- a/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi
++++ b/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi
+@@ -6,11 +6,8 @@
+ / {
+ #address-cells = <2>;
+ #size-cells = <2>;
+- model = "Microchip MPFS Icicle Kit";
+- compatible = "microchip,mpfs-icicle-kit";
+-
+- chosen {
+- };
++ model = "Microchip PolarFire SoC";
++ compatible = "microchip,mpfs";
+
+ cpus {
+ #address-cells = <1>;
+@@ -262,39 +259,14 @@
+ status = "disabled";
+ };
+
+- emmc: mmc@20008000 {
++ /* Common node entry for emmc/sd */
++ mmc: mmc@20008000 {
+ compatible = "cdns,sd4hc";
+ reg = <0x0 0x20008000 0x0 0x1000>;
+ interrupt-parent = <&plic>;
+ interrupts = <88 89>;
+ pinctrl-names = "default";
+ clocks = <&clkcfg 6>;
+- bus-width = <4>;
+- cap-mmc-highspeed;
+- mmc-ddr-3_3v;
+- max-frequency = <200000000>;
+- non-removable;
+- no-sd;
+- no-sdio;
+- voltage-ranges = <3300 3300>;
+- status = "disabled";
+- };
+-
+- sdcard: sdhc@20008000 {
+- compatible = "cdns,sd4hc";
+- reg = <0x0 0x20008000 0x0 0x1000>;
+- interrupt-parent = <&plic>;
+- interrupts = <88>;
+- pinctrl-names = "default";
+- clocks = <&clkcfg 6>;
+- bus-width = <4>;
+- disable-wp;
+- cap-sd-highspeed;
+- card-detect-delay = <200>;
+- sd-uhs-sdr12;
+- sd-uhs-sdr25;
+- sd-uhs-sdr50;
+- sd-uhs-sdr104;
+ max-frequency = <200000000>;
+ status = "disabled";
+ };
+diff --git a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
+index 7db8610534834..64c06c9b41dc8 100644
+--- a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
++++ b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
+@@ -166,7 +166,7 @@
+ clocks = <&prci PRCI_CLK_TLCLK>;
+ status = "disabled";
+ };
+- dma: dma@3000000 {
++ dma: dma-controller@3000000 {
+ compatible = "sifive,fu540-c000-pdma";
+ reg = <0x0 0x3000000 0x0 0x8000>;
+ interrupt-parent = <&plic0>;
+diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts
+index 60846e88ae4b1..22f971e971614 100644
+--- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts
++++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts
+@@ -80,6 +80,7 @@
+ spi-max-frequency = <20000000>;
+ voltage-ranges = <3300 3300>;
+ disable-wp;
++ gpios = <&gpio 11 GPIO_ACTIVE_LOW>;
+ };
+ };
+
+diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts
+index 2e4ea84f27e77..b40990210fb50 100644
+--- a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts
++++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts
+@@ -2,6 +2,7 @@
+ /* Copyright (c) 2020 SiFive, Inc */
+
+ #include "fu740-c000.dtsi"
++#include <dt-bindings/gpio/gpio.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ /* Clock frequency (in Hz) of the PCB crystal for rtcclk */
+@@ -228,6 +229,7 @@
+ spi-max-frequency = <20000000>;
+ voltage-ranges = <3300 3300>;
+ disable-wp;
++ gpios = <&gpio 15 GPIO_ACTIVE_LOW>;
+ };
+ };
+
+diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig
+index b16a2a12c82a8..3b9f83221f9c2 100644
+--- a/arch/riscv/configs/nommu_k210_defconfig
++++ b/arch/riscv/configs/nommu_k210_defconfig
+@@ -29,8 +29,6 @@ CONFIG_EMBEDDED=y
+ CONFIG_SLOB=y
+ # CONFIG_MMU is not set
+ CONFIG_SOC_CANAAN=y
+-CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic"
+-CONFIG_MAXPHYSMEM_2GB=y
+ CONFIG_SMP=y
+ CONFIG_NR_CPUS=2
+ CONFIG_CMDLINE="earlycon console=ttySIF0"
+diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig
+index 61f887f654199..15d1fd0a70184 100644
+--- a/arch/riscv/configs/nommu_k210_sdcard_defconfig
++++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig
+@@ -21,11 +21,9 @@ CONFIG_EMBEDDED=y
+ CONFIG_SLOB=y
+ # CONFIG_MMU is not set
+ CONFIG_SOC_CANAAN=y
+-CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic"
+-CONFIG_MAXPHYSMEM_2GB=y
+ CONFIG_SMP=y
+ CONFIG_NR_CPUS=2
+-CONFIG_CMDLINE="earlycon console=ttySIF0 rootdelay=2 root=/dev/mmcblk0p1 ro"
++CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro"
+ CONFIG_CMDLINE_FORCE=y
+ # CONFIG_SECCOMP is not set
+ # CONFIG_STACKPROTECTOR is not set
+diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig
+index e046a0babde43..f224be697785f 100644
+--- a/arch/riscv/configs/nommu_virt_defconfig
++++ b/arch/riscv/configs/nommu_virt_defconfig
+@@ -27,7 +27,6 @@ CONFIG_SLOB=y
+ # CONFIG_SLAB_MERGE_DEFAULT is not set
+ # CONFIG_MMU is not set
+ CONFIG_SOC_VIRT=y
+-CONFIG_MAXPHYSMEM_2GB=y
+ CONFIG_SMP=y
+ CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0"
+ CONFIG_CMDLINE_FORCE=y
+diff --git a/arch/riscv/include/asm/efi.h b/arch/riscv/include/asm/efi.h
+index 49b398fe99f1b..cc4f6787f9371 100644
+--- a/arch/riscv/include/asm/efi.h
++++ b/arch/riscv/include/asm/efi.h
+@@ -13,7 +13,6 @@
+
+ #ifdef CONFIG_EFI
+ extern void efi_init(void);
+-extern void efifb_setup_from_dmi(struct screen_info *si, const char *opt);
+ #else
+ #define efi_init()
+ #endif
+diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h
+index 4254ff2ff0494..1075beae1ac64 100644
+--- a/arch/riscv/include/asm/module.lds.h
++++ b/arch/riscv/include/asm/module.lds.h
+@@ -2,8 +2,8 @@
+ /* Copyright (C) 2017 Andes Technology Corporation */
+ #ifdef CONFIG_MODULE_SECTIONS
+ SECTIONS {
+- .plt (NOLOAD) : { BYTE(0) }
+- .got (NOLOAD) : { BYTE(0) }
+- .got.plt (NOLOAD) : { BYTE(0) }
++ .plt : { BYTE(0) }
++ .got : { BYTE(0) }
++ .got.plt : { BYTE(0) }
+ }
+ #endif
+diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
+index a7d2811f35365..62d0e6e61da83 100644
+--- a/arch/riscv/include/asm/smp.h
++++ b/arch/riscv/include/asm/smp.h
+@@ -43,7 +43,6 @@ void arch_send_call_function_ipi_mask(struct cpumask *mask);
+ void arch_send_call_function_single_ipi(int cpu);
+
+ int riscv_hartid_to_cpuid(int hartid);
+-void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
+
+ /* Set custom IPI operations */
+ void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops);
+@@ -85,13 +84,6 @@ static inline unsigned long cpuid_to_hartid_map(int cpu)
+ return boot_cpu_hartid;
+ }
+
+-static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in,
+- struct cpumask *out)
+-{
+- cpumask_clear(out);
+- cpumask_set_cpu(boot_cpu_hartid, out);
+-}
+-
+ static inline void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops)
+ {
+ }
+@@ -102,6 +94,8 @@ static inline void riscv_clear_ipi(void)
+
+ #endif /* CONFIG_SMP */
+
++void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
++
+ #if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP)
+ bool cpu_has_hotplug(unsigned int cpu);
+ #else
+diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
+index 60da0dcacf145..74d888c8d631a 100644
+--- a/arch/riscv/include/asm/thread_info.h
++++ b/arch/riscv/include/asm/thread_info.h
+@@ -11,11 +11,17 @@
+ #include <asm/page.h>
+ #include <linux/const.h>
+
++#ifdef CONFIG_KASAN
++#define KASAN_STACK_ORDER 1
++#else
++#define KASAN_STACK_ORDER 0
++#endif
++
+ /* thread information allocation */
+ #ifdef CONFIG_64BIT
+-#define THREAD_SIZE_ORDER (2)
++#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER)
+ #else
+-#define THREAD_SIZE_ORDER (1)
++#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER)
+ #endif
+ #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
+
+diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h
+index 507cae273bc62..d6a7428f6248d 100644
+--- a/arch/riscv/include/asm/timex.h
++++ b/arch/riscv/include/asm/timex.h
+@@ -41,7 +41,7 @@ static inline u32 get_cycles_hi(void)
+ static inline unsigned long random_get_entropy(void)
+ {
+ if (unlikely(clint_time_val == NULL))
+- return 0;
++ return random_get_entropy_fallback();
+ return get_cycles();
+ }
+ #define random_get_entropy() random_get_entropy()
+diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
+index 3397ddac1a30c..16308ef1e5787 100644
+--- a/arch/riscv/kernel/Makefile
++++ b/arch/riscv/kernel/Makefile
+@@ -50,6 +50,8 @@ obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o
+ obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
+ obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o
+
++obj-$(CONFIG_TRACE_IRQFLAGS) += trace_irq.o
++
+ obj-$(CONFIG_RISCV_BASE_PMU) += perf_event.o
+ obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o
+ obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o
+diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c
+index df84e0c13db18..66ddfba1cfbef 100644
+--- a/arch/riscv/kernel/cpu-hotplug.c
++++ b/arch/riscv/kernel/cpu-hotplug.c
+@@ -12,6 +12,7 @@
+ #include <linux/sched/hotplug.h>
+ #include <asm/irq.h>
+ #include <asm/cpu_ops.h>
++#include <asm/numa.h>
+ #include <asm/sbi.h>
+
+ void cpu_stop(void);
+@@ -46,6 +47,7 @@ int __cpu_disable(void)
+ return ret;
+
+ remove_cpu_topology(cpu);
++ numa_remove_cpu(cpu);
+ set_cpu_online(cpu, false);
+ irq_migrate_all_off_this_cpu();
+
+diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
+index 98f502654edd3..7e52ad5d61adb 100644
+--- a/arch/riscv/kernel/entry.S
++++ b/arch/riscv/kernel/entry.S
+@@ -108,7 +108,7 @@ _save_context:
+ .option pop
+
+ #ifdef CONFIG_TRACE_IRQFLAGS
+- call trace_hardirqs_off
++ call __trace_hardirqs_off
+ #endif
+
+ #ifdef CONFIG_CONTEXT_TRACKING
+@@ -144,7 +144,7 @@ skip_context_tracking:
+ li t0, EXC_BREAKPOINT
+ beq s4, t0, 1f
+ #ifdef CONFIG_TRACE_IRQFLAGS
+- call trace_hardirqs_on
++ call __trace_hardirqs_on
+ #endif
+ csrs CSR_STATUS, SR_IE
+
+@@ -235,7 +235,7 @@ ret_from_exception:
+ REG_L s0, PT_STATUS(sp)
+ csrc CSR_STATUS, SR_IE
+ #ifdef CONFIG_TRACE_IRQFLAGS
+- call trace_hardirqs_off
++ call __trace_hardirqs_off
+ #endif
+ #ifdef CONFIG_RISCV_M_MODE
+ /* the MPP value is too large to be used as an immediate arg for addi */
+@@ -271,10 +271,10 @@ restore_all:
+ REG_L s1, PT_STATUS(sp)
+ andi t0, s1, SR_PIE
+ beqz t0, 1f
+- call trace_hardirqs_on
++ call __trace_hardirqs_on
+ j 2f
+ 1:
+- call trace_hardirqs_off
++ call __trace_hardirqs_off
+ 2:
+ #endif
+ REG_L a0, PT_STATUS(sp)
+diff --git a/arch/riscv/kernel/kexec_relocate.S b/arch/riscv/kernel/kexec_relocate.S
+index a80b52a74f58c..059c5e216ae75 100644
+--- a/arch/riscv/kernel/kexec_relocate.S
++++ b/arch/riscv/kernel/kexec_relocate.S
+@@ -159,25 +159,15 @@ SYM_CODE_START(riscv_kexec_norelocate)
+ * s0: (const) Phys address to jump to
+ * s1: (const) Phys address of the FDT image
+ * s2: (const) The hartid of the current hart
+- * s3: (const) kernel_map.va_pa_offset, used when switching MMU off
+ */
+ mv s0, a1
+ mv s1, a2
+ mv s2, a3
+- mv s3, a4
+
+ /* Disable / cleanup interrupts */
+ csrw CSR_SIE, zero
+ csrw CSR_SIP, zero
+
+- /* Switch to physical addressing */
+- la s4, 1f
+- sub s4, s4, s3
+- csrw CSR_STVEC, s4
+- csrw CSR_SATP, zero
+-
+-.align 2
+-1:
+ /* Pass the arguments to the next kernel / Cleanup*/
+ mv a0, s2
+ mv a1, s1
+@@ -214,7 +204,15 @@ SYM_CODE_START(riscv_kexec_norelocate)
+ csrw CSR_SCAUSE, zero
+ csrw CSR_SSCRATCH, zero
+
+- jalr zero, a2, 0
++ /*
++ * Switch to physical addressing
++ * This will also trigger a jump to CSR_STVEC
++ * which in this case is the address of the new
++ * kernel.
++ */
++ csrw CSR_STVEC, a2
++ csrw CSR_SATP, zero
++
+ SYM_CODE_END(riscv_kexec_norelocate)
+
+ .section ".rodata"
+diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c
+index e6eca271a4d60..cbef0fc73afa8 100644
+--- a/arch/riscv/kernel/machine_kexec.c
++++ b/arch/riscv/kernel/machine_kexec.c
+@@ -169,7 +169,8 @@ machine_kexec(struct kimage *image)
+ struct kimage_arch *internal = &image->arch;
+ unsigned long jump_addr = (unsigned long) image->start;
+ unsigned long first_ind_entry = (unsigned long) &image->head;
+- unsigned long this_hart_id = raw_smp_processor_id();
++ unsigned long this_cpu_id = smp_processor_id();
++ unsigned long this_hart_id = cpuid_to_hartid_map(this_cpu_id);
+ unsigned long fdt_addr = internal->fdt_addr;
+ void *control_code_buffer = page_address(image->control_code_page);
+ riscv_kexec_method kexec_method = NULL;
+diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c
+index 68a9e3d1fe16a..4a48287513c37 100644
+--- a/arch/riscv/kernel/module.c
++++ b/arch/riscv/kernel/module.c
+@@ -13,6 +13,19 @@
+ #include <linux/pgtable.h>
+ #include <asm/sections.h>
+
++/*
++ * The auipc+jalr instruction pair can reach any PC-relative offset
++ * in the range [-2^31 - 2^11, 2^31 - 2^11)
++ */
++static bool riscv_insn_valid_32bit_offset(ptrdiff_t val)
++{
++#ifdef CONFIG_32BIT
++ return true;
++#else
++ return (-(1L << 31) - (1L << 11)) <= val && val < ((1L << 31) - (1L << 11));
++#endif
++}
++
+ static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v)
+ {
+ if (v != (u32)v) {
+@@ -95,7 +108,7 @@ static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location,
+ ptrdiff_t offset = (void *)v - (void *)location;
+ s32 hi20;
+
+- if (offset != (s32)offset) {
++ if (!riscv_insn_valid_32bit_offset(offset)) {
+ pr_err(
+ "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
+ me->name, (long long)v, location);
+@@ -197,10 +210,9 @@ static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+ {
+ ptrdiff_t offset = (void *)v - (void *)location;
+- s32 fill_v = offset;
+ u32 hi20, lo12;
+
+- if (offset != fill_v) {
++ if (!riscv_insn_valid_32bit_offset(offset)) {
+ /* Only emit the plt entry if offset over 32-bit range */
+ if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
+ offset = module_emit_plt_entry(me, v);
+@@ -224,10 +236,9 @@ static int apply_r_riscv_call_rela(struct module *me, u32 *location,
+ Elf_Addr v)
+ {
+ ptrdiff_t offset = (void *)v - (void *)location;
+- s32 fill_v = offset;
+ u32 hi20, lo12;
+
+- if (offset != fill_v) {
++ if (!riscv_insn_valid_32bit_offset(offset)) {
+ pr_err(
+ "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
+ me->name, (long long)v, location);
+diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
+index 0b552873a5778..765004b605132 100644
+--- a/arch/riscv/kernel/patch.c
++++ b/arch/riscv/kernel/patch.c
+@@ -104,7 +104,7 @@ static int patch_text_cb(void *data)
+ struct patch_insn *patch = data;
+ int ret = 0;
+
+- if (atomic_inc_return(&patch->cpu_count) == 1) {
++ if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
+ ret =
+ patch_text_nosync(patch->addr, &patch->insn,
+ GET_INSN_LENGTH(patch->insn));
+diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c
+index 0bb1854dce833..357f985041cb9 100644
+--- a/arch/riscv/kernel/perf_callchain.c
++++ b/arch/riscv/kernel/perf_callchain.c
+@@ -15,8 +15,8 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+ {
+ struct stackframe buftail;
+ unsigned long ra = 0;
+- unsigned long *user_frame_tail =
+- (unsigned long *)(fp - sizeof(struct stackframe));
++ unsigned long __user *user_frame_tail =
++ (unsigned long __user *)(fp - sizeof(struct stackframe));
+
+ /* Check accessibility of one struct frame_tail beyond */
+ if (!access_ok(user_frame_tail, sizeof(buftail)))
+@@ -56,10 +56,11 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ unsigned long fp = 0;
+
+ /* RISC-V does not support perf in guest mode. */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
++ if (guest_cbs && guest_cbs->is_in_guest())
+ return;
+
+ fp = regs->s0;
+@@ -72,14 +73,16 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+
+ static bool fill_callchain(void *entry, unsigned long pc)
+ {
+- return perf_callchain_store(entry, pc);
++ return perf_callchain_store(entry, pc) == 0;
+ }
+
+ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
+ /* RISC-V does not support perf in guest mode. */
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ pr_warn("RISC-V does not support perf in guest mode!");
+ return;
+ }
+diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
+index b9620e5f00baf..6c5caf5eb9061 100644
+--- a/arch/riscv/kernel/setup.c
++++ b/arch/riscv/kernel/setup.c
+@@ -59,6 +59,16 @@ atomic_t hart_lottery __section(".sdata")
+ unsigned long boot_cpu_hartid;
+ static DEFINE_PER_CPU(struct cpu, cpu_devices);
+
++void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out)
++{
++ int cpu;
++
++ cpumask_clear(out);
++ for_each_cpu(cpu, in)
++ cpumask_set_cpu(cpuid_to_hartid_map(cpu), out);
++}
++EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask);
++
+ /*
+ * Place kernel memory regions on the resource tree so that
+ * kexec-tools can retrieve them from /proc/iomem. While there
+diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
+index 921d9d7df4001..d0147294691d9 100644
+--- a/arch/riscv/kernel/smp.c
++++ b/arch/riscv/kernel/smp.c
+@@ -59,16 +59,6 @@ int riscv_hartid_to_cpuid(int hartid)
+ return -ENOENT;
+ }
+
+-void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out)
+-{
+- int cpu;
+-
+- cpumask_clear(out);
+- for_each_cpu(cpu, in)
+- cpumask_set_cpu(cpuid_to_hartid_map(cpu), out);
+-}
+-EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask);
+-
+ bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
+ {
+ return phys_id == cpuid_to_hartid_map(cpu);
+diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c
+index 315db3d0229bf..c2601150b91c8 100644
+--- a/arch/riscv/kernel/stacktrace.c
++++ b/arch/riscv/kernel/stacktrace.c
+@@ -22,15 +22,16 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs,
+ bool (*fn)(void *, unsigned long), void *arg)
+ {
+ unsigned long fp, sp, pc;
++ int level = 0;
+
+ if (regs) {
+ fp = frame_pointer(regs);
+ sp = user_stack_pointer(regs);
+ pc = instruction_pointer(regs);
+ } else if (task == NULL || task == current) {
+- fp = (unsigned long)__builtin_frame_address(1);
+- sp = (unsigned long)__builtin_frame_address(0);
+- pc = (unsigned long)__builtin_return_address(0);
++ fp = (unsigned long)__builtin_frame_address(0);
++ sp = sp_in_global;
++ pc = (unsigned long)walk_stackframe;
+ } else {
+ /* task blocked in __switch_to */
+ fp = task->thread.s[0];
+@@ -42,7 +43,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs,
+ unsigned long low, high;
+ struct stackframe *frame;
+
+- if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc)))
++ if (unlikely(!__kernel_text_address(pc) || (level++ >= 1 && !fn(arg, pc))))
+ break;
+
+ /* Validate frame pointer */
+diff --git a/arch/riscv/kernel/trace_irq.c b/arch/riscv/kernel/trace_irq.c
+new file mode 100644
+index 0000000000000..095ac976d7da1
+--- /dev/null
++++ b/arch/riscv/kernel/trace_irq.c
+@@ -0,0 +1,27 @@
++// SPDX-License-Identifier: GPL-2.0
++/*
++ * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com>
++ */
++
++#include <linux/irqflags.h>
++#include <linux/kprobes.h>
++#include "trace_irq.h"
++
++/*
++ * trace_hardirqs_on/off require the caller to setup frame pointer properly.
++ * Otherwise, CALLER_ADDR1 might trigger an pagging exception in kernel.
++ * Here we add one extra level so they can be safely called by low
++ * level entry code which $fp is used for other purpose.
++ */
++
++void __trace_hardirqs_on(void)
++{
++ trace_hardirqs_on();
++}
++NOKPROBE_SYMBOL(__trace_hardirqs_on);
++
++void __trace_hardirqs_off(void)
++{
++ trace_hardirqs_off();
++}
++NOKPROBE_SYMBOL(__trace_hardirqs_off);
+diff --git a/arch/riscv/kernel/trace_irq.h b/arch/riscv/kernel/trace_irq.h
+new file mode 100644
+index 0000000000000..99fe67377e5ed
+--- /dev/null
++++ b/arch/riscv/kernel/trace_irq.h
+@@ -0,0 +1,11 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (C) 2022 Changbin Du <changbin.du@gmail.com>
++ */
++#ifndef __TRACE_IRQ_H
++#define __TRACE_IRQ_H
++
++void __trace_hardirqs_on(void);
++void __trace_hardirqs_off(void);
++
++#endif /* __TRACE_IRQ_H */
+diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S
+index 07d1d2152ba5c..e0609e1f0864d 100644
+--- a/arch/riscv/lib/memmove.S
++++ b/arch/riscv/lib/memmove.S
+@@ -1,64 +1,316 @@
+-/* SPDX-License-Identifier: GPL-2.0 */
++/* SPDX-License-Identifier: GPL-2.0-only */
++/*
++ * Copyright (C) 2022 Michael T. Kloos <michael@michaelkloos.com>
++ */
+
+ #include <linux/linkage.h>
+ #include <asm/asm.h>
+
+-ENTRY(__memmove)
+-WEAK(memmove)
+- move t0, a0
+- move t1, a1
+-
+- beq a0, a1, exit_memcpy
+- beqz a2, exit_memcpy
+- srli t2, a2, 0x2
+-
+- slt t3, a0, a1
+- beqz t3, do_reverse
+-
+- andi a2, a2, 0x3
+- li t4, 1
+- beqz t2, byte_copy
+-
+-word_copy:
+- lw t3, 0(a1)
+- addi t2, t2, -1
+- addi a1, a1, 4
+- sw t3, 0(a0)
+- addi a0, a0, 4
+- bnez t2, word_copy
+- beqz a2, exit_memcpy
+- j byte_copy
+-
+-do_reverse:
+- add a0, a0, a2
+- add a1, a1, a2
+- andi a2, a2, 0x3
+- li t4, -1
+- beqz t2, reverse_byte_copy
+-
+-reverse_word_copy:
+- addi a1, a1, -4
+- addi t2, t2, -1
+- lw t3, 0(a1)
+- addi a0, a0, -4
+- sw t3, 0(a0)
+- bnez t2, reverse_word_copy
+- beqz a2, exit_memcpy
+-
+-reverse_byte_copy:
+- addi a0, a0, -1
+- addi a1, a1, -1
++SYM_FUNC_START(__memmove)
++SYM_FUNC_START_WEAK(memmove)
++ /*
++ * Returns
++ * a0 - dest
++ *
++ * Parameters
++ * a0 - Inclusive first byte of dest
++ * a1 - Inclusive first byte of src
++ * a2 - Length of copy n
++ *
++ * Because the return matches the parameter register a0,
++ * we will not clobber or modify that register.
++ *
++ * Note: This currently only works on little-endian.
++ * To port to big-endian, reverse the direction of shifts
++ * in the 2 misaligned fixup copy loops.
++ */
+
++ /* Return if nothing to do */
++ beq a0, a1, return_from_memmove
++ beqz a2, return_from_memmove
++
++ /*
++ * Register Uses
++ * Forward Copy: a1 - Index counter of src
++ * Reverse Copy: a4 - Index counter of src
++ * Forward Copy: t3 - Index counter of dest
++ * Reverse Copy: t4 - Index counter of dest
++ * Both Copy Modes: t5 - Inclusive first multibyte/aligned of dest
++ * Both Copy Modes: t6 - Non-Inclusive last multibyte/aligned of dest
++ * Both Copy Modes: t0 - Link / Temporary for load-store
++ * Both Copy Modes: t1 - Temporary for load-store
++ * Both Copy Modes: t2 - Temporary for load-store
++ * Both Copy Modes: a5 - dest to src alignment offset
++ * Both Copy Modes: a6 - Shift ammount
++ * Both Copy Modes: a7 - Inverse Shift ammount
++ * Both Copy Modes: a2 - Alternate breakpoint for unrolled loops
++ */
++
++ /*
++ * Solve for some register values now.
++ * Byte copy does not need t5 or t6.
++ */
++ mv t3, a0
++ add t4, a0, a2
++ add a4, a1, a2
++
++ /*
++ * Byte copy if copying less than (2 * SZREG) bytes. This can
++ * cause problems with the bulk copy implementation and is
++ * small enough not to bother.
++ */
++ andi t0, a2, -(2 * SZREG)
++ beqz t0, byte_copy
++
++ /*
++ * Now solve for t5 and t6.
++ */
++ andi t5, t3, -SZREG
++ andi t6, t4, -SZREG
++ /*
++ * If dest(Register t3) rounded down to the nearest naturally
++ * aligned SZREG address, does not equal dest, then add SZREG
++ * to find the low-bound of SZREG alignment in the dest memory
++ * region. Note that this could overshoot the dest memory
++ * region if n is less than SZREG. This is one reason why
++ * we always byte copy if n is less than SZREG.
++ * Otherwise, dest is already naturally aligned to SZREG.
++ */
++ beq t5, t3, 1f
++ addi t5, t5, SZREG
++ 1:
++
++ /*
++ * If the dest and src are co-aligned to SZREG, then there is
++ * no need for the full rigmarole of a full misaligned fixup copy.
++ * Instead, do a simpler co-aligned copy.
++ */
++ xor t0, a0, a1
++ andi t1, t0, (SZREG - 1)
++ beqz t1, coaligned_copy
++ /* Fall through to misaligned fixup copy */
++
++misaligned_fixup_copy:
++ bltu a1, a0, misaligned_fixup_copy_reverse
++
++misaligned_fixup_copy_forward:
++ jal t0, byte_copy_until_aligned_forward
++
++ andi a5, a1, (SZREG - 1) /* Find the alignment offset of src (a1) */
++ slli a6, a5, 3 /* Multiply by 8 to convert that to bits to shift */
++ sub a5, a1, t3 /* Find the difference between src and dest */
++ andi a1, a1, -SZREG /* Align the src pointer */
++ addi a2, t6, SZREG /* The other breakpoint for the unrolled loop*/
++
++ /*
++ * Compute The Inverse Shift
++ * a7 = XLEN - a6 = XLEN + -a6
++ * 2s complement negation to find the negative: -a6 = ~a6 + 1
++ * Add that to XLEN. XLEN = SZREG * 8.
++ */
++ not a7, a6
++ addi a7, a7, (SZREG * 8 + 1)
++
++ /*
++ * Fix Misalignment Copy Loop - Forward
++ * load_val0 = load_ptr[0];
++ * do {
++ * load_val1 = load_ptr[1];
++ * store_ptr += 2;
++ * store_ptr[0 - 2] = (load_val0 >> {a6}) | (load_val1 << {a7});
++ *
++ * if (store_ptr == {a2})
++ * break;
++ *
++ * load_val0 = load_ptr[2];
++ * load_ptr += 2;
++ * store_ptr[1 - 2] = (load_val1 >> {a6}) | (load_val0 << {a7});
++ *
++ * } while (store_ptr != store_ptr_end);
++ * store_ptr = store_ptr_end;
++ */
++
++ REG_L t0, (0 * SZREG)(a1)
++ 1:
++ REG_L t1, (1 * SZREG)(a1)
++ addi t3, t3, (2 * SZREG)
++ srl t0, t0, a6
++ sll t2, t1, a7
++ or t2, t0, t2
++ REG_S t2, ((0 * SZREG) - (2 * SZREG))(t3)
++
++ beq t3, a2, 2f
++
++ REG_L t0, (2 * SZREG)(a1)
++ addi a1, a1, (2 * SZREG)
++ srl t1, t1, a6
++ sll t2, t0, a7
++ or t2, t1, t2
++ REG_S t2, ((1 * SZREG) - (2 * SZREG))(t3)
++
++ bne t3, t6, 1b
++ 2:
++ mv t3, t6 /* Fix the dest pointer in case the loop was broken */
++
++ add a1, t3, a5 /* Restore the src pointer */
++ j byte_copy_forward /* Copy any remaining bytes */
++
++misaligned_fixup_copy_reverse:
++ jal t0, byte_copy_until_aligned_reverse
++
++ andi a5, a4, (SZREG - 1) /* Find the alignment offset of src (a4) */
++ slli a6, a5, 3 /* Multiply by 8 to convert that to bits to shift */
++ sub a5, a4, t4 /* Find the difference between src and dest */
++ andi a4, a4, -SZREG /* Align the src pointer */
++ addi a2, t5, -SZREG /* The other breakpoint for the unrolled loop*/
++
++ /*
++ * Compute The Inverse Shift
++ * a7 = XLEN - a6 = XLEN + -a6
++ * 2s complement negation to find the negative: -a6 = ~a6 + 1
++ * Add that to XLEN. XLEN = SZREG * 8.
++ */
++ not a7, a6
++ addi a7, a7, (SZREG * 8 + 1)
++
++ /*
++ * Fix Misalignment Copy Loop - Reverse
++ * load_val1 = load_ptr[0];
++ * do {
++ * load_val0 = load_ptr[-1];
++ * store_ptr -= 2;
++ * store_ptr[1] = (load_val0 >> {a6}) | (load_val1 << {a7});
++ *
++ * if (store_ptr == {a2})
++ * break;
++ *
++ * load_val1 = load_ptr[-2];
++ * load_ptr -= 2;
++ * store_ptr[0] = (load_val1 >> {a6}) | (load_val0 << {a7});
++ *
++ * } while (store_ptr != store_ptr_end);
++ * store_ptr = store_ptr_end;
++ */
++
++ REG_L t1, ( 0 * SZREG)(a4)
++ 1:
++ REG_L t0, (-1 * SZREG)(a4)
++ addi t4, t4, (-2 * SZREG)
++ sll t1, t1, a7
++ srl t2, t0, a6
++ or t2, t1, t2
++ REG_S t2, ( 1 * SZREG)(t4)
++
++ beq t4, a2, 2f
++
++ REG_L t1, (-2 * SZREG)(a4)
++ addi a4, a4, (-2 * SZREG)
++ sll t0, t0, a7
++ srl t2, t1, a6
++ or t2, t0, t2
++ REG_S t2, ( 0 * SZREG)(t4)
++
++ bne t4, t5, 1b
++ 2:
++ mv t4, t5 /* Fix the dest pointer in case the loop was broken */
++
++ add a4, t4, a5 /* Restore the src pointer */
++ j byte_copy_reverse /* Copy any remaining bytes */
++
++/*
++ * Simple copy loops for SZREG co-aligned memory locations.
++ * These also make calls to do byte copies for any unaligned
++ * data at their terminations.
++ */
++coaligned_copy:
++ bltu a1, a0, coaligned_copy_reverse
++
++coaligned_copy_forward:
++ jal t0, byte_copy_until_aligned_forward
++
++ 1:
++ REG_L t1, ( 0 * SZREG)(a1)
++ addi a1, a1, SZREG
++ addi t3, t3, SZREG
++ REG_S t1, (-1 * SZREG)(t3)
++ bne t3, t6, 1b
++
++ j byte_copy_forward /* Copy any remaining bytes */
++
++coaligned_copy_reverse:
++ jal t0, byte_copy_until_aligned_reverse
++
++ 1:
++ REG_L t1, (-1 * SZREG)(a4)
++ addi a4, a4, -SZREG
++ addi t4, t4, -SZREG
++ REG_S t1, ( 0 * SZREG)(t4)
++ bne t4, t5, 1b
++
++ j byte_copy_reverse /* Copy any remaining bytes */
++
++/*
++ * These are basically sub-functions within the function. They
++ * are used to byte copy until the dest pointer is in alignment.
++ * At which point, a bulk copy method can be used by the
++ * calling code. These work on the same registers as the bulk
++ * copy loops. Therefore, the register values can be picked
++ * up from where they were left and we avoid code duplication
++ * without any overhead except the call in and return jumps.
++ */
++byte_copy_until_aligned_forward:
++ beq t3, t5, 2f
++ 1:
++ lb t1, 0(a1)
++ addi a1, a1, 1
++ addi t3, t3, 1
++ sb t1, -1(t3)
++ bne t3, t5, 1b
++ 2:
++ jalr zero, 0x0(t0) /* Return to multibyte copy loop */
++
++byte_copy_until_aligned_reverse:
++ beq t4, t6, 2f
++ 1:
++ lb t1, -1(a4)
++ addi a4, a4, -1
++ addi t4, t4, -1
++ sb t1, 0(t4)
++ bne t4, t6, 1b
++ 2:
++ jalr zero, 0x0(t0) /* Return to multibyte copy loop */
++
++/*
++ * Simple byte copy loops.
++ * These will byte copy until they reach the end of data to copy.
++ * At that point, they will call to return from memmove.
++ */
+ byte_copy:
+- lb t3, 0(a1)
+- addi a2, a2, -1
+- sb t3, 0(a0)
+- add a1, a1, t4
+- add a0, a0, t4
+- bnez a2, byte_copy
+-
+-exit_memcpy:
+- move a0, t0
+- move a1, t1
+- ret
+-END(__memmove)
++ bltu a1, a0, byte_copy_reverse
++
++byte_copy_forward:
++ beq t3, t4, 2f
++ 1:
++ lb t1, 0(a1)
++ addi a1, a1, 1
++ addi t3, t3, 1
++ sb t1, -1(t3)
++ bne t3, t4, 1b
++ 2:
++ ret
++
++byte_copy_reverse:
++ beq t4, t3, 2f
++ 1:
++ lb t1, -1(a4)
++ addi a4, a4, -1
++ addi t4, t4, -1
++ sb t1, 0(t4)
++ bne t4, t3, 1b
++ 2:
++
++return_from_memmove:
++ ret
++
++SYM_FUNC_END(memmove)
++SYM_FUNC_END(__memmove)
+diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
+index 7ebaef10ea1b6..ac7a25298a04a 100644
+--- a/arch/riscv/mm/Makefile
++++ b/arch/riscv/mm/Makefile
+@@ -24,6 +24,9 @@ obj-$(CONFIG_KASAN) += kasan_init.o
+ ifdef CONFIG_KASAN
+ KASAN_SANITIZE_kasan_init.o := n
+ KASAN_SANITIZE_init.o := n
++ifdef CONFIG_DEBUG_VIRTUAL
++KASAN_SANITIZE_physaddr.o := n
++endif
+ endif
+
+ obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
+diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
+index c0cddf0fc22db..7f130ac3b9f9a 100644
+--- a/arch/riscv/mm/init.c
++++ b/arch/riscv/mm/init.c
+@@ -187,10 +187,10 @@ static void __init setup_bootmem(void)
+
+
+ phys_ram_end = memblock_end_of_DRAM();
+-#ifndef CONFIG_64BIT
+ #ifndef CONFIG_XIP_KERNEL
+ phys_ram_base = memblock_start_of_DRAM();
+ #endif
++#ifndef CONFIG_64BIT
+ /*
+ * memblock allocator is not aware of the fact that last 4K bytes of
+ * the addressable memory can not be mapped because of IS_ERR_VALUE
+@@ -218,8 +218,25 @@ static void __init setup_bootmem(void)
+ * early_init_fdt_reserve_self() since __pa() does
+ * not work for DTB pointers that are fixmap addresses
+ */
+- if (!IS_ENABLED(CONFIG_BUILTIN_DTB))
+- memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
++ if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) {
++ /*
++ * In case the DTB is not located in a memory region we won't
++ * be able to locate it later on via the linear mapping and
++ * get a segfault when accessing it via __va(dtb_early_pa).
++ * To avoid this situation copy DTB to a memory region.
++ * Note that memblock_phys_alloc will also reserve DTB region.
++ */
++ if (!memblock_is_memory(dtb_early_pa)) {
++ size_t fdt_size = fdt_totalsize(dtb_early_va);
++ phys_addr_t new_dtb_early_pa = memblock_phys_alloc(fdt_size, PAGE_SIZE);
++ void *new_dtb_early_va = early_memremap(new_dtb_early_pa, fdt_size);
++
++ memcpy(new_dtb_early_va, dtb_early_va, fdt_size);
++ early_memunmap(new_dtb_early_va, fdt_size);
++ _dtb_early_pa = new_dtb_early_pa;
++ } else
++ memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
++ }
+
+ early_init_fdt_scan_reserved_mem();
+ dma_contiguous_reserve(dma32_phys_limit);
+@@ -451,6 +468,7 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)
+ }
+
+ #ifdef CONFIG_XIP_KERNEL
++#define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base))
+ /* called from head.S with MMU off */
+ asmlinkage void __init __copy_data(void)
+ {
+@@ -813,13 +831,22 @@ static void __init reserve_crashkernel(void)
+ /*
+ * Current riscv boot protocol requires 2MB alignment for
+ * RV64 and 4MB alignment for RV32 (hugepage size)
++ *
++ * Try to alloc from 32bit addressible physical memory so that
++ * swiotlb can work on the crash kernel.
+ */
+ crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE,
+- search_start, search_end);
++ search_start,
++ min(search_end, (unsigned long) SZ_4G));
+ if (crash_base == 0) {
+- pr_warn("crashkernel: couldn't allocate %lldKB\n",
+- crash_size >> 10);
+- return;
++ /* Try again without restricting region to 32bit addressible memory */
++ crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE,
++ search_start, search_end);
++ if (crash_base == 0) {
++ pr_warn("crashkernel: couldn't allocate %lldKB\n",
++ crash_size >> 10);
++ return;
++ }
+ }
+
+ pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n",
+diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
+index 54294f83513d1..e26e367a3d9ef 100644
+--- a/arch/riscv/mm/kasan_init.c
++++ b/arch/riscv/mm/kasan_init.c
+@@ -22,8 +22,7 @@ asmlinkage void __init kasan_early_init(void)
+
+ for (i = 0; i < PTRS_PER_PTE; ++i)
+ set_pte(kasan_early_shadow_pte + i,
+- mk_pte(virt_to_page(kasan_early_shadow_page),
+- PAGE_KERNEL));
++ pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL));
+
+ for (i = 0; i < PTRS_PER_PMD; ++i)
+ set_pmd(kasan_early_shadow_pmd + i,
+diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
+index b86de61b8caa2..797041b5109a2 100644
+--- a/arch/s390/Kconfig
++++ b/arch/s390/Kconfig
+@@ -47,7 +47,7 @@ config ARCH_SUPPORTS_UPROBES
+ config KASAN_SHADOW_OFFSET
+ hex
+ depends on KASAN
+- default 0x18000000000000
++ default 0x1C000000000000
+
+ config S390
+ def_bool y
+diff --git a/arch/s390/Makefile b/arch/s390/Makefile
+index 450b351dfa8ef..c7b7a60f6405d 100644
+--- a/arch/s390/Makefile
++++ b/arch/s390/Makefile
+@@ -32,6 +32,16 @@ KBUILD_CFLAGS_DECOMPRESSOR += -fno-stack-protector
+ KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, address-of-packed-member)
+ KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g)
+ KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,))
++
++ifdef CONFIG_CC_IS_GCC
++ ifeq ($(call cc-ifversion, -ge, 1200, y), y)
++ ifeq ($(call cc-ifversion, -lt, 1300, y), y)
++ KBUILD_CFLAGS += $(call cc-disable-warning, array-bounds)
++ KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, array-bounds)
++ endif
++ endif
++endif
++
+ UTS_MACHINE := s390x
+ STACK_SIZE := $(if $(CONFIG_KASAN),65536,16384)
+ CHECKFLAGS += -D__s390__ -D__s390x__
+@@ -79,10 +89,12 @@ KBUILD_AFLAGS_DECOMPRESSOR += $(aflags-y)
+ KBUILD_CFLAGS_DECOMPRESSOR += $(cflags-y)
+
+ ifneq ($(call cc-option,-mstack-size=8192 -mstack-guard=128),)
+-cflags-$(CONFIG_CHECK_STACK) += -mstack-size=$(STACK_SIZE)
+-ifeq ($(call cc-option,-mstack-size=8192),)
+-cflags-$(CONFIG_CHECK_STACK) += -mstack-guard=$(CONFIG_STACK_GUARD)
+-endif
++ CC_FLAGS_CHECK_STACK := -mstack-size=$(STACK_SIZE)
++ ifeq ($(call cc-option,-mstack-size=8192),)
++ CC_FLAGS_CHECK_STACK += -mstack-guard=$(CONFIG_STACK_GUARD)
++ endif
++ export CC_FLAGS_CHECK_STACK
++ cflags-$(CONFIG_CHECK_STACK) += $(CC_FLAGS_CHECK_STACK)
+ endif
+
+ ifdef CONFIG_EXPOLINE
+diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
+index 6dc8d0a538640..b13352dd1e1cf 100644
+--- a/arch/s390/boot/startup.c
++++ b/arch/s390/boot/startup.c
+@@ -148,82 +148,56 @@ static void setup_ident_map_size(unsigned long max_physmem_end)
+
+ static void setup_kernel_memory_layout(void)
+ {
+- bool vmalloc_size_verified = false;
+- unsigned long vmemmap_off;
+- unsigned long vspace_left;
++ unsigned long vmemmap_start;
+ unsigned long rte_size;
+ unsigned long pages;
+- unsigned long vmax;
+
+ pages = ident_map_size / PAGE_SIZE;
+ /* vmemmap contains a multiple of PAGES_PER_SECTION struct pages */
+ vmemmap_size = SECTION_ALIGN_UP(pages) * sizeof(struct page);
+
+ /* choose kernel address space layout: 4 or 3 levels. */
+- vmemmap_off = round_up(ident_map_size, _REGION3_SIZE);
++ vmemmap_start = round_up(ident_map_size, _REGION3_SIZE);
+ if (IS_ENABLED(CONFIG_KASAN) ||
+ vmalloc_size > _REGION2_SIZE ||
+- vmemmap_off + vmemmap_size + vmalloc_size + MODULES_LEN > _REGION2_SIZE)
+- vmax = _REGION1_SIZE;
+- else
+- vmax = _REGION2_SIZE;
+-
+- /* keep vmemmap_off aligned to a top level region table entry */
+- rte_size = vmax == _REGION1_SIZE ? _REGION2_SIZE : _REGION3_SIZE;
+- MODULES_END = vmax;
+- if (is_prot_virt_host()) {
+- /*
+- * forcing modules and vmalloc area under the ultravisor
+- * secure storage limit, so that any vmalloc allocation
+- * we do could be used to back secure guest storage.
+- */
+- adjust_to_uv_max(&MODULES_END);
+- }
+-
+-#ifdef CONFIG_KASAN
+- if (MODULES_END < vmax) {
+- /* force vmalloc and modules below kasan shadow */
+- MODULES_END = min(MODULES_END, KASAN_SHADOW_START);
++ vmemmap_start + vmemmap_size + vmalloc_size + MODULES_LEN >
++ _REGION2_SIZE) {
++ MODULES_END = _REGION1_SIZE;
++ rte_size = _REGION2_SIZE;
+ } else {
+- /*
+- * leave vmalloc and modules above kasan shadow but make
+- * sure they don't overlap with it
+- */
+- vmalloc_size = min(vmalloc_size, vmax - KASAN_SHADOW_END - MODULES_LEN);
+- vmalloc_size_verified = true;
+- vspace_left = KASAN_SHADOW_START;
++ MODULES_END = _REGION2_SIZE;
++ rte_size = _REGION3_SIZE;
+ }
++ /*
++ * forcing modules and vmalloc area under the ultravisor
++ * secure storage limit, so that any vmalloc allocation
++ * we do could be used to back secure guest storage.
++ */
++ adjust_to_uv_max(&MODULES_END);
++#ifdef CONFIG_KASAN
++ /* force vmalloc and modules below kasan shadow */
++ MODULES_END = min(MODULES_END, KASAN_SHADOW_START);
+ #endif
+ MODULES_VADDR = MODULES_END - MODULES_LEN;
+ VMALLOC_END = MODULES_VADDR;
+
+- if (vmalloc_size_verified) {
+- VMALLOC_START = VMALLOC_END - vmalloc_size;
+- } else {
+- vmemmap_off = round_up(ident_map_size, rte_size);
+-
+- if (vmemmap_off + vmemmap_size > VMALLOC_END ||
+- vmalloc_size > VMALLOC_END - vmemmap_off - vmemmap_size) {
+- /*
+- * allow vmalloc area to occupy up to 1/2 of
+- * the rest virtual space left.
+- */
+- vmalloc_size = min(vmalloc_size, VMALLOC_END / 2);
+- }
+- VMALLOC_START = VMALLOC_END - vmalloc_size;
+- vspace_left = VMALLOC_START;
+- }
++ /* allow vmalloc area to occupy up to about 1/2 of the rest virtual space left */
++ vmalloc_size = min(vmalloc_size, round_down(VMALLOC_END / 2, _REGION3_SIZE));
++ VMALLOC_START = VMALLOC_END - vmalloc_size;
+
+- pages = vspace_left / (PAGE_SIZE + sizeof(struct page));
++ /* split remaining virtual space between 1:1 mapping & vmemmap array */
++ pages = VMALLOC_START / (PAGE_SIZE + sizeof(struct page));
+ pages = SECTION_ALIGN_UP(pages);
+- vmemmap_off = round_up(vspace_left - pages * sizeof(struct page), rte_size);
+- /* keep vmemmap left most starting from a fresh region table entry */
+- vmemmap_off = min(vmemmap_off, round_up(ident_map_size, rte_size));
+- /* take care that identity map is lower then vmemmap */
+- ident_map_size = min(ident_map_size, vmemmap_off);
++ /* keep vmemmap_start aligned to a top level region table entry */
++ vmemmap_start = round_down(VMALLOC_START - pages * sizeof(struct page), rte_size);
++ /* vmemmap_start is the future VMEM_MAX_PHYS, make sure it is within MAX_PHYSMEM */
++ vmemmap_start = min(vmemmap_start, 1UL << MAX_PHYSMEM_BITS);
++ /* make sure identity map doesn't overlay with vmemmap */
++ ident_map_size = min(ident_map_size, vmemmap_start);
+ vmemmap_size = SECTION_ALIGN_UP(ident_map_size / PAGE_SIZE) * sizeof(struct page);
+- VMALLOC_START = max(vmemmap_off + vmemmap_size, VMALLOC_START);
+- vmemmap = (struct page *)vmemmap_off;
++ /* make sure vmemmap doesn't overlay with vmalloc area */
++ VMALLOC_START = max(vmemmap_start + vmemmap_size, VMALLOC_START);
++ vmemmap = (struct page *)vmemmap_start;
+ }
+
+ /*
+diff --git a/arch/s390/hypfs/hypfs_vm.c b/arch/s390/hypfs/hypfs_vm.c
+index 33f973ff97442..e8f15dbb89d02 100644
+--- a/arch/s390/hypfs/hypfs_vm.c
++++ b/arch/s390/hypfs/hypfs_vm.c
+@@ -20,6 +20,7 @@
+
+ static char local_guest[] = " ";
+ static char all_guests[] = "* ";
++static char *all_groups = all_guests;
+ static char *guest_query;
+
+ struct diag2fc_data {
+@@ -62,10 +63,11 @@ static int diag2fc(int size, char* query, void *addr)
+
+ memcpy(parm_list.userid, query, NAME_LEN);
+ ASCEBC(parm_list.userid, NAME_LEN);
+- parm_list.addr = (unsigned long) addr ;
++ memcpy(parm_list.aci_grp, all_groups, NAME_LEN);
++ ASCEBC(parm_list.aci_grp, NAME_LEN);
++ parm_list.addr = (unsigned long)addr;
+ parm_list.size = size;
+ parm_list.fmt = 0x02;
+- memset(parm_list.aci_grp, 0x40, NAME_LEN);
+ rc = -1;
+
+ diag_stat_inc(DIAG_STAT_X2FC);
+diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h
+index 16dc57dd90b30..8511f0e59290f 100644
+--- a/arch/s390/include/asm/extable.h
++++ b/arch/s390/include/asm/extable.h
+@@ -69,8 +69,13 @@ static inline void swap_ex_entry_fixup(struct exception_table_entry *a,
+ {
+ a->fixup = b->fixup + delta;
+ b->fixup = tmp.fixup - delta;
+- a->handler = b->handler + delta;
+- b->handler = tmp.handler - delta;
++ a->handler = b->handler;
++ if (a->handler)
++ a->handler += delta;
++ b->handler = tmp.handler;
++ if (b->handler)
++ b->handler -= delta;
+ }
++#define swap_ex_entry_fixup swap_ex_entry_fixup
+
+ #endif
+diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h
+index ea398a05f6432..7f3c9ac34bd8d 100644
+--- a/arch/s390/include/asm/kexec.h
++++ b/arch/s390/include/asm/kexec.h
+@@ -74,6 +74,12 @@ void *kexec_file_add_components(struct kimage *image,
+ int arch_kexec_do_relocs(int r_type, void *loc, unsigned long val,
+ unsigned long addr);
+
++#define ARCH_HAS_KIMAGE_ARCH
++
++struct kimage_arch {
++ void *ipl_buf;
++};
++
+ extern const struct kexec_file_ops s390_kexec_image_ops;
+ extern const struct kexec_file_ops s390_kexec_elf_ops;
+
+diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h
+index e4dc64cc9c555..287bb88f76986 100644
+--- a/arch/s390/include/asm/pci_io.h
++++ b/arch/s390/include/asm/pci_io.h
+@@ -14,12 +14,13 @@
+
+ /* I/O Map */
+ #define ZPCI_IOMAP_SHIFT 48
+-#define ZPCI_IOMAP_ADDR_BASE 0x8000000000000000UL
++#define ZPCI_IOMAP_ADDR_SHIFT 62
++#define ZPCI_IOMAP_ADDR_BASE (1UL << ZPCI_IOMAP_ADDR_SHIFT)
+ #define ZPCI_IOMAP_ADDR_OFF_MASK ((1UL << ZPCI_IOMAP_SHIFT) - 1)
+ #define ZPCI_IOMAP_MAX_ENTRIES \
+- ((ULONG_MAX - ZPCI_IOMAP_ADDR_BASE + 1) / (1UL << ZPCI_IOMAP_SHIFT))
++ (1UL << (ZPCI_IOMAP_ADDR_SHIFT - ZPCI_IOMAP_SHIFT))
+ #define ZPCI_IOMAP_ADDR_IDX_MASK \
+- (~ZPCI_IOMAP_ADDR_OFF_MASK - ZPCI_IOMAP_ADDR_BASE)
++ ((ZPCI_IOMAP_ADDR_BASE - 1) & ~ZPCI_IOMAP_ADDR_OFF_MASK)
+
+ struct zpci_iomap_entry {
+ u32 fh;
+diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
+index 50d9b04ecbd14..bc50ee0e91ff1 100644
+--- a/arch/s390/include/asm/timex.h
++++ b/arch/s390/include/asm/timex.h
+@@ -201,6 +201,7 @@ static inline cycles_t get_cycles(void)
+ {
+ return (cycles_t) get_tod_clock() >> 2;
+ }
++#define get_cycles get_cycles
+
+ int get_phys_clock(unsigned long *clock);
+ void init_cpu_timer(void);
+diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c
+index d72a6df058d79..785d54c9350c4 100644
+--- a/arch/s390/kernel/crash_dump.c
++++ b/arch/s390/kernel/crash_dump.c
+@@ -191,8 +191,8 @@ static int copy_oldmem_user(void __user *dst, void *src, size_t count)
+ return rc;
+ } else {
+ /* Check for swapped kdump oldmem areas */
+- if (oldmem_data.start && from - oldmem_data.size < oldmem_data.size) {
+- from -= oldmem_data.size;
++ if (oldmem_data.start && from - oldmem_data.start < oldmem_data.size) {
++ from -= oldmem_data.start;
+ len = min(count, oldmem_data.size - from);
+ } else if (oldmem_data.start && from < oldmem_data.size) {
+ len = min(count, oldmem_data.size - from);
+diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
+index e2cc35775b996..5ad1dde23dc59 100644
+--- a/arch/s390/kernel/ipl.c
++++ b/arch/s390/kernel/ipl.c
+@@ -2156,7 +2156,7 @@ void *ipl_report_finish(struct ipl_report *report)
+
+ buf = vzalloc(report->size);
+ if (!buf)
+- return ERR_PTR(-ENOMEM);
++ goto out;
+ ptr = buf;
+
+ memcpy(ptr, report->ipib, report->ipib->hdr.len);
+@@ -2195,6 +2195,7 @@ void *ipl_report_finish(struct ipl_report *report)
+ }
+
+ BUG_ON(ptr > buf + report->size);
++out:
+ return buf;
+ }
+
+diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
+index 3a3145c4a3ba4..be5d432b902e0 100644
+--- a/arch/s390/kernel/irq.c
++++ b/arch/s390/kernel/irq.c
+@@ -138,7 +138,7 @@ void noinstr do_io_irq(struct pt_regs *regs)
+ struct pt_regs *old_regs = set_irq_regs(regs);
+ int from_idle;
+
+- irq_enter();
++ irq_enter_rcu();
+
+ if (user_mode(regs))
+ update_timer_sys();
+@@ -155,7 +155,8 @@ void noinstr do_io_irq(struct pt_regs *regs)
+ do_irq_async(regs, IO_INTERRUPT);
+ } while (MACHINE_IS_LPAR && irq_pending(regs));
+
+- irq_exit();
++ irq_exit_rcu();
++
+ set_irq_regs(old_regs);
+ irqentry_exit(regs, state);
+
+@@ -169,7 +170,7 @@ void noinstr do_ext_irq(struct pt_regs *regs)
+ struct pt_regs *old_regs = set_irq_regs(regs);
+ int from_idle;
+
+- irq_enter();
++ irq_enter_rcu();
+
+ if (user_mode(regs))
+ update_timer_sys();
+@@ -184,7 +185,7 @@ void noinstr do_ext_irq(struct pt_regs *regs)
+
+ do_irq_async(regs, EXT_INTERRUPT);
+
+- irq_exit();
++ irq_exit_rcu();
+ set_irq_regs(old_regs);
+ irqentry_exit(regs, state);
+
+diff --git a/arch/s390/kernel/machine_kexec_file.c b/arch/s390/kernel/machine_kexec_file.c
+index f9e4baa64b675..a81d6c43b9b61 100644
+--- a/arch/s390/kernel/machine_kexec_file.c
++++ b/arch/s390/kernel/machine_kexec_file.c
+@@ -12,6 +12,7 @@
+ #include <linux/kexec.h>
+ #include <linux/module_signature.h>
+ #include <linux/verification.h>
++#include <linux/vmalloc.h>
+ #include <asm/boot_data.h>
+ #include <asm/ipl.h>
+ #include <asm/setup.h>
+@@ -170,6 +171,7 @@ static int kexec_file_add_ipl_report(struct kimage *image,
+ struct kexec_buf buf;
+ unsigned long addr;
+ void *ptr, *end;
++ int ret;
+
+ buf.image = image;
+
+@@ -199,9 +201,13 @@ static int kexec_file_add_ipl_report(struct kimage *image,
+ ptr += len;
+ }
+
++ ret = -ENOMEM;
+ buf.buffer = ipl_report_finish(data->report);
++ if (!buf.buffer)
++ goto out;
+ buf.bufsz = data->report->size;
+ buf.memsz = buf.bufsz;
++ image->arch.ipl_buf = buf.buffer;
+
+ data->memsz += buf.memsz;
+
+@@ -209,7 +215,9 @@ static int kexec_file_add_ipl_report(struct kimage *image,
+ data->kernel_buf + offsetof(struct lowcore, ipl_parmblock_ptr);
+ *lc_ipl_parmblock_ptr = (__u32)buf.mem;
+
+- return kexec_add_buffer(&buf);
++ ret = kexec_add_buffer(&buf);
++out:
++ return ret;
+ }
+
+ void *kexec_file_add_components(struct kimage *image,
+@@ -269,6 +277,7 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
+ {
+ Elf_Rela *relas;
+ int i, r_type;
++ int ret;
+
+ relas = (void *)pi->ehdr + relsec->sh_offset;
+
+@@ -303,7 +312,15 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
+ addr = section->sh_addr + relas[i].r_offset;
+
+ r_type = ELF64_R_TYPE(relas[i].r_info);
+- arch_kexec_do_relocs(r_type, loc, val, addr);
++
++ if (r_type == R_390_PLT32DBL)
++ r_type = R_390_PC32DBL;
++
++ ret = arch_kexec_do_relocs(r_type, loc, val, addr);
++ if (ret) {
++ pr_err("Unknown rela relocation: %d\n", r_type);
++ return -ENOEXEC;
++ }
+ }
+ return 0;
+ }
+@@ -321,3 +338,11 @@ int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
+
+ return kexec_image_probe_default(image, buf, buf_len);
+ }
++
++int arch_kimage_file_post_load_cleanup(struct kimage *image)
++{
++ vfree(image->arch.ipl_buf);
++ image->arch.ipl_buf = NULL;
++
++ return kexec_image_post_load_cleanup_default(image);
++}
+diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c
+index b01ba460b7cad..b032e556eeb71 100644
+--- a/arch/s390/kernel/module.c
++++ b/arch/s390/kernel/module.c
+@@ -33,18 +33,19 @@
+ #define DEBUGP(fmt , ...)
+ #endif
+
+-#define PLT_ENTRY_SIZE 20
++#define PLT_ENTRY_SIZE 22
+
+ void *module_alloc(unsigned long size)
+ {
++ gfp_t gfp_mask = GFP_KERNEL;
+ void *p;
+
+ if (PAGE_ALIGN(size) > MODULES_LEN)
+ return NULL;
+ p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END,
+- GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
++ gfp_mask, PAGE_KERNEL_EXEC, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,
+ __builtin_return_address(0));
+- if (p && (kasan_module_alloc(p, size) < 0)) {
++ if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) {
+ vfree(p);
+ return NULL;
+ }
+@@ -340,27 +341,26 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab,
+ case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */
+ case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
+ if (info->plt_initialized == 0) {
+- unsigned int insn[5];
+- unsigned int *ip = me->core_layout.base +
+- me->arch.plt_offset +
+- info->plt_offset;
+-
+- insn[0] = 0x0d10e310; /* basr 1,0 */
+- insn[1] = 0x100a0004; /* lg 1,10(1) */
++ unsigned char insn[PLT_ENTRY_SIZE];
++ char *plt_base;
++ char *ip;
++
++ plt_base = me->core_layout.base + me->arch.plt_offset;
++ ip = plt_base + info->plt_offset;
++ *(int *)insn = 0x0d10e310; /* basr 1,0 */
++ *(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */
+ if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) {
+- unsigned int *ij;
+- ij = me->core_layout.base +
+- me->arch.plt_offset +
+- me->arch.plt_size - PLT_ENTRY_SIZE;
+- insn[2] = 0xa7f40000 + /* j __jump_r1 */
+- (unsigned int)(u16)
+- (((unsigned long) ij - 8 -
+- (unsigned long) ip) / 2);
++ char *jump_r1;
++
++ jump_r1 = plt_base + me->arch.plt_size -
++ PLT_ENTRY_SIZE;
++ /* brcl 0xf,__jump_r1 */
++ *(short *)&insn[8] = 0xc0f4;
++ *(int *)&insn[10] = (jump_r1 - (ip + 8)) / 2;
+ } else {
+- insn[2] = 0x07f10000; /* br %r1 */
++ *(int *)&insn[8] = 0x07f10000; /* br %r1 */
+ }
+- insn[3] = (unsigned int) (val >> 32);
+- insn[4] = (unsigned int) val;
++ *(long *)&insn[14] = val;
+
+ write(ip, insn, sizeof(insn));
+ info->plt_initialized = 1;
+diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
+index 20f8e1868853f..a50f2ff1b00e8 100644
+--- a/arch/s390/kernel/nmi.c
++++ b/arch/s390/kernel/nmi.c
+@@ -273,7 +273,14 @@ static int notrace s390_validate_registers(union mci mci, int umode)
+ /* Validate vector registers */
+ union ctlreg0 cr0;
+
+- if (!mci.vr) {
++ /*
++ * The vector validity must only be checked if not running a
++ * KVM guest. For KVM guests the machine check is forwarded by
++ * KVM and it is the responsibility of the guest to take
++ * appropriate actions. The host vector or FPU values have been
++ * saved by KVM and will be restored by KVM.
++ */
++ if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) {
+ /*
+ * Vector registers can't be restored. If the kernel
+ * currently uses vector registers the system is
+@@ -316,11 +323,21 @@ static int notrace s390_validate_registers(union mci mci, int umode)
+ if (cr2.gse) {
+ if (!mci.gs) {
+ /*
+- * Guarded storage register can't be restored and
+- * the current processes uses guarded storage.
+- * It has to be terminated.
++ * 2 cases:
++ * - machine check in kernel or userspace
++ * - machine check while running SIE (KVM guest)
++ * For kernel or userspace the userspace values of
++ * guarded storage control can not be recreated, the
++ * process must be terminated.
++ * For SIE the guest values of guarded storage can not
++ * be recreated. This is either due to a bug or due to
++ * GS being disabled in the guest. The guest will be
++ * notified by KVM code and the guests machine check
++ * handling must take care of this. The host values
++ * are saved by KVM and are not affected.
+ */
+- kill_task = 1;
++ if (!test_cpu_flag(CIF_MCCK_GUEST))
++ kill_task = 1;
+ } else {
+ load_gs_cb((struct gs_cb *)mcesa->guarded_storage_save_area);
+ }
+diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c
+index 4a99154fe6514..cceb8ec707e4b 100644
+--- a/arch/s390/kernel/perf_cpum_cf.c
++++ b/arch/s390/kernel/perf_cpum_cf.c
+@@ -687,8 +687,10 @@ static void cpumf_pmu_stop(struct perf_event *event, int flags)
+ false);
+ if (cfdiag_diffctr(cpuhw, event->hw.config_base))
+ cfdiag_push_sample(event, cpuhw);
+- } else
++ } else if (cpuhw->flags & PMU_F_RESERVED) {
++ /* Only update when PMU not hotplugged off */
+ hw_perf_event_update(event);
++ }
+ hwc->state |= PERF_HES_UPTODATE;
+ }
+ }
+diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
+index 67e5fff96ee06..ee67215a678a5 100644
+--- a/arch/s390/kernel/setup.c
++++ b/arch/s390/kernel/setup.c
+@@ -633,14 +633,6 @@ static struct notifier_block kdump_mem_nb = {
+
+ #endif
+
+-/*
+- * Make sure that the area above identity mapping is protected
+- */
+-static void __init reserve_above_ident_map(void)
+-{
+- memblock_reserve(ident_map_size, ULONG_MAX);
+-}
+-
+ /*
+ * Reserve memory for kdump kernel to be loaded with kexec
+ */
+@@ -824,9 +816,6 @@ static void __init setup_memory(void)
+ storage_key_init_range(start, end);
+
+ psw_set_key(PAGE_DEFAULT_KEY);
+-
+- /* Only cosmetics */
+- memblock_enforce_memory_limit(memblock_end_of_DRAM());
+ }
+
+ static void __init relocate_amode31_section(void)
+@@ -1005,11 +994,11 @@ void __init setup_arch(char **cmdline_p)
+ setup_control_program_code();
+
+ /* Do some memory reservations *before* memory is added to memblock */
+- reserve_above_ident_map();
+ reserve_kernel();
+ reserve_initrd();
+ reserve_certificate_list();
+ reserve_mem_detect_info();
++ memblock_set_current_limit(ident_map_size);
+ memblock_allow_resize();
+
+ /* Get information about *all* installed memory */
+diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
+index bcefc2173de45..4044826d72ae5 100644
+--- a/arch/s390/kernel/traps.c
++++ b/arch/s390/kernel/traps.c
+@@ -84,7 +84,7 @@ static void default_trap_handler(struct pt_regs *regs)
+ {
+ if (user_mode(regs)) {
+ report_user_fault(regs, SIGSEGV, 0);
+- do_exit(SIGSEGV);
++ force_exit_sig(SIGSEGV);
+ } else
+ die(regs, "Unknown program exception");
+ }
+@@ -142,10 +142,10 @@ static inline void do_fp_trap(struct pt_regs *regs, __u32 fpc)
+ do_trap(regs, SIGFPE, si_code, "floating point exception");
+ }
+
+-static void translation_exception(struct pt_regs *regs)
++static void translation_specification_exception(struct pt_regs *regs)
+ {
+ /* May never happen. */
+- panic("Translation exception");
++ panic("Translation-Specification Exception");
+ }
+
+ static void illegal_op(struct pt_regs *regs)
+@@ -374,7 +374,7 @@ static void (*pgm_check_table[128])(struct pt_regs *regs) = {
+ [0x0f] = hfp_divide_exception,
+ [0x10] = do_dat_exception,
+ [0x11] = do_dat_exception,
+- [0x12] = translation_exception,
++ [0x12] = translation_specification_exception,
+ [0x13] = special_op_exception,
+ [0x14] = default_trap_handler,
+ [0x15] = operand_exception,
+diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
+index 5a656c7b7a67a..f95ccbd396925 100644
+--- a/arch/s390/kernel/uv.c
++++ b/arch/s390/kernel/uv.c
+@@ -212,7 +212,7 @@ again:
+ uaddr = __gmap_translate(gmap, gaddr);
+ if (IS_ERR_VALUE(uaddr))
+ goto out;
+- vma = find_vma(gmap->mm, uaddr);
++ vma = vma_lookup(gmap->mm, uaddr);
+ if (!vma)
+ goto out;
+ /*
+diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile
+index 6568de2367010..0dea82b87e54b 100644
+--- a/arch/s390/kernel/vdso64/Makefile
++++ b/arch/s390/kernel/vdso64/Makefile
+@@ -8,8 +8,9 @@ ARCH_REL_TYPE_ABS += R_390_GOT|R_390_PLT
+ include $(srctree)/lib/vdso/Makefile
+ obj-vdso64 = vdso_user_wrapper.o note.o
+ obj-cvdso64 = vdso64_generic.o getcpu.o
+-CFLAGS_REMOVE_getcpu.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE)
+-CFLAGS_REMOVE_vdso64_generic.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE)
++VDSO_CFLAGS_REMOVE := -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) $(CC_FLAGS_CHECK_STACK)
++CFLAGS_REMOVE_getcpu.o = $(VDSO_CFLAGS_REMOVE)
++CFLAGS_REMOVE_vdso64_generic.o = $(VDSO_CFLAGS_REMOVE)
+
+ # Build rules
+
+diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
+index 2245f4b8d3629..8ce03a5ca8634 100644
+--- a/arch/s390/kvm/interrupt.c
++++ b/arch/s390/kvm/interrupt.c
+@@ -2115,6 +2115,13 @@ int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
+ return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
+ }
+
++int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
++{
++ struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
++
++ return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
++}
++
+ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
+diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
+index 1c97493d21e10..402597f9d0505 100644
+--- a/arch/s390/kvm/kvm-s390.c
++++ b/arch/s390/kvm/kvm-s390.c
+@@ -3447,7 +3447,7 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
+ {
+ /* do not poll with more than halt_poll_max_steal percent of steal time */
+ if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
+- halt_poll_max_steal) {
++ READ_ONCE(halt_poll_max_steal)) {
+ vcpu->stat.halt_no_poll_steal++;
+ return true;
+ }
+@@ -4642,10 +4642,15 @@ int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
+ }
+ }
+
+- /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
++ /*
++ * Set the VCPU to STOPPED and THEN clear the interrupt flag,
++ * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
++ * have been fully processed. This will ensure that the VCPU
++ * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
++ */
++ kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
+ kvm_s390_clear_stop_irq(vcpu);
+
+- kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
+ __disable_ibs_on_vcpu(vcpu);
+
+ for (i = 0; i < online_vcpus; i++) {
+@@ -4703,6 +4708,8 @@ static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
+ return -EINVAL;
+ if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
+ return -E2BIG;
++ if (!kvm_s390_pv_cpu_is_protected(vcpu))
++ return -EINVAL;
+
+ switch (mop->op) {
+ case KVM_S390_MEMOP_SIDA_READ:
+diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
+index 52bc8fbaa60ac..1539dd981104f 100644
+--- a/arch/s390/kvm/kvm-s390.h
++++ b/arch/s390/kvm/kvm-s390.h
+@@ -418,6 +418,7 @@ void kvm_s390_destroy_adapters(struct kvm *kvm);
+ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu);
+ extern struct kvm_device_ops kvm_flic_ops;
+ int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu);
++int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu);
+ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu);
+ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu,
+ void __user *buf, int len);
+diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
+index 53da4ceb16a3a..417154b314a64 100644
+--- a/arch/s390/kvm/priv.c
++++ b/arch/s390/kvm/priv.c
+@@ -397,6 +397,8 @@ static int handle_sske(struct kvm_vcpu *vcpu)
+ mmap_read_unlock(current->mm);
+ if (rc == -EFAULT)
+ return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
++ if (rc == -EAGAIN)
++ continue;
+ if (rc < 0)
+ return rc;
+ start += PAGE_SIZE;
+diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c
+index c8841f476e913..00d272d134c24 100644
+--- a/arch/s390/kvm/pv.c
++++ b/arch/s390/kvm/pv.c
+@@ -16,18 +16,17 @@
+
+ int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
+ {
+- int cc = 0;
++ int cc;
+
+- if (kvm_s390_pv_cpu_get_handle(vcpu)) {
+- cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
+- UVC_CMD_DESTROY_SEC_CPU, rc, rrc);
++ if (!kvm_s390_pv_cpu_get_handle(vcpu))
++ return 0;
++
++ cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), UVC_CMD_DESTROY_SEC_CPU, rc, rrc);
++
++ KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x",
++ vcpu->vcpu_id, *rc, *rrc);
++ WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc);
+
+- KVM_UV_EVENT(vcpu->kvm, 3,
+- "PROTVIRT DESTROY VCPU %d: rc %x rrc %x",
+- vcpu->vcpu_id, *rc, *rrc);
+- WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x",
+- *rc, *rrc);
+- }
+ /* Intended memory leak for something that should never happen. */
+ if (!cc)
+ free_pages(vcpu->arch.pv.stor_base,
+@@ -196,7 +195,7 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
+ uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base;
+ uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var;
+
+- cc = uv_call(0, (u64)&uvcb);
++ cc = uv_call_sched(0, (u64)&uvcb);
+ *rc = uvcb.header.rc;
+ *rrc = uvcb.header.rrc;
+ KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x",
+diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
+index 683036c1c92a8..3dc921e853b6e 100644
+--- a/arch/s390/kvm/sigp.c
++++ b/arch/s390/kvm/sigp.c
+@@ -288,6 +288,34 @@ static int handle_sigp_dst(struct kvm_vcpu *vcpu, u8 order_code,
+ if (!dst_vcpu)
+ return SIGP_CC_NOT_OPERATIONAL;
+
++ /*
++ * SIGP RESTART, SIGP STOP, and SIGP STOP AND STORE STATUS orders
++ * are processed asynchronously. Until the affected VCPU finishes
++ * its work and calls back into KVM to clear the (RESTART or STOP)
++ * interrupt, we need to return any new non-reset orders "busy".
++ *
++ * This is important because a single VCPU could issue:
++ * 1) SIGP STOP $DESTINATION
++ * 2) SIGP SENSE $DESTINATION
++ *
++ * If the SIGP SENSE would not be rejected as "busy", it could
++ * return an incorrect answer as to whether the VCPU is STOPPED
++ * or OPERATING.
++ */
++ if (order_code != SIGP_INITIAL_CPU_RESET &&
++ order_code != SIGP_CPU_RESET) {
++ /*
++ * Lockless check. Both SIGP STOP and SIGP (RE)START
++ * properly synchronize everything while processing
++ * their orders, while the guest cannot observe a
++ * difference when issuing other orders from two
++ * different VCPUs.
++ */
++ if (kvm_s390_is_stop_irq_pending(dst_vcpu) ||
++ kvm_s390_is_restart_irq_pending(dst_vcpu))
++ return SIGP_CC_BUSY;
++ }
++
+ switch (order_code) {
+ case SIGP_SENSE:
+ vcpu->stat.instruction_sigp_sense++;
+diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c
+index ecf327d743a03..c0635cf787e31 100644
+--- a/arch/s390/lib/test_unwind.c
++++ b/arch/s390/lib/test_unwind.c
+@@ -171,10 +171,11 @@ static noinline int unwindme_func4(struct unwindme *u)
+ }
+
+ /*
+- * trigger specification exception
++ * Trigger operation exception; use insn notation to bypass
++ * llvm's integrated assembler sanity checks.
+ */
+ asm volatile(
+- " mvcl %%r1,%%r1\n"
++ " .insn e,0x0000\n" /* illegal opcode */
+ "0: nopr %%r7\n"
+ EX_TABLE(0b, 0b)
+ :);
+diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
+index 4d3b33ce81c62..d63c0ccc5ccda 100644
+--- a/arch/s390/mm/gmap.c
++++ b/arch/s390/mm/gmap.c
+@@ -672,6 +672,7 @@ EXPORT_SYMBOL_GPL(gmap_fault);
+ */
+ void __gmap_zap(struct gmap *gmap, unsigned long gaddr)
+ {
++ struct vm_area_struct *vma;
+ unsigned long vmaddr;
+ spinlock_t *ptl;
+ pte_t *ptep;
+@@ -681,11 +682,17 @@ void __gmap_zap(struct gmap *gmap, unsigned long gaddr)
+ gaddr >> PMD_SHIFT);
+ if (vmaddr) {
+ vmaddr |= gaddr & ~PMD_MASK;
++
++ vma = vma_lookup(gmap->mm, vmaddr);
++ if (!vma || is_vm_hugetlb_page(vma))
++ return;
++
+ /* Get pointer to the page table entry */
+ ptep = get_locked_pte(gmap->mm, vmaddr, &ptl);
+- if (likely(ptep))
++ if (likely(ptep)) {
+ ptep_zap_unused(gmap->mm, vmaddr, ptep, 0);
+- pte_unmap_unlock(ptep, ptl);
++ pte_unmap_unlock(ptep, ptl);
++ }
+ }
+ }
+ EXPORT_SYMBOL_GPL(__gmap_zap);
+diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
+index 781965f7210eb..91e478e09b54b 100644
+--- a/arch/s390/mm/pgalloc.c
++++ b/arch/s390/mm/pgalloc.c
+@@ -244,13 +244,15 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
+ /* Free 2K page table fragment of a 4K page */
+ bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));
+ spin_lock_bh(&mm->context.lock);
+- mask = atomic_xor_bits(&page->_refcount, 1U << (bit + 24));
++ mask = atomic_xor_bits(&page->_refcount, 0x11U << (bit + 24));
+ mask >>= 24;
+ if (mask & 3)
+ list_add(&page->lru, &mm->context.pgtable_list);
+ else
+ list_del(&page->lru);
+ spin_unlock_bh(&mm->context.lock);
++ mask = atomic_xor_bits(&page->_refcount, 0x10U << (bit + 24));
++ mask >>= 24;
+ if (mask != 0)
+ return;
+ } else {
+diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
+index 034721a68d8fd..6ad634a27d5b9 100644
+--- a/arch/s390/mm/pgtable.c
++++ b/arch/s390/mm/pgtable.c
+@@ -429,22 +429,36 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
+ }
+
+ #ifdef CONFIG_PGSTE
+-static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr)
++static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp)
+ {
++ struct vm_area_struct *vma;
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pud_t *pud;
+- pmd_t *pmd;
++
++ /* We need a valid VMA, otherwise this is clearly a fault. */
++ vma = vma_lookup(mm, addr);
++ if (!vma)
++ return -EFAULT;
+
+ pgd = pgd_offset(mm, addr);
+- p4d = p4d_alloc(mm, pgd, addr);
+- if (!p4d)
+- return NULL;
+- pud = pud_alloc(mm, p4d, addr);
+- if (!pud)
+- return NULL;
+- pmd = pmd_alloc(mm, pud, addr);
+- return pmd;
++ if (!pgd_present(*pgd))
++ return -ENOENT;
++
++ p4d = p4d_offset(pgd, addr);
++ if (!p4d_present(*p4d))
++ return -ENOENT;
++
++ pud = pud_offset(p4d, addr);
++ if (!pud_present(*pud))
++ return -ENOENT;
++
++ /* Large PUDs are not supported yet. */
++ if (pud_large(*pud))
++ return -EFAULT;
++
++ *pmdp = pmd_offset(pud, addr);
++ return 0;
+ }
+ #endif
+
+@@ -778,8 +792,7 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp;
+ pte_t *ptep;
+
+- pmdp = pmd_alloc_map(mm, addr);
+- if (unlikely(!pmdp))
++ if (pmd_lookup(mm, addr, &pmdp))
+ return -EFAULT;
+
+ ptl = pmd_lock(mm, pmdp);
+@@ -881,8 +894,7 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
+ pte_t *ptep;
+ int cc = 0;
+
+- pmdp = pmd_alloc_map(mm, addr);
+- if (unlikely(!pmdp))
++ if (pmd_lookup(mm, addr, &pmdp))
+ return -EFAULT;
+
+ ptl = pmd_lock(mm, pmdp);
+@@ -935,15 +947,24 @@ int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp;
+ pte_t *ptep;
+
+- pmdp = pmd_alloc_map(mm, addr);
+- if (unlikely(!pmdp))
++ /*
++ * If we don't have a PTE table and if there is no huge page mapped,
++ * the storage key is 0.
++ */
++ *key = 0;
++
++ switch (pmd_lookup(mm, addr, &pmdp)) {
++ case -ENOENT:
++ return 0;
++ case 0:
++ break;
++ default:
+ return -EFAULT;
++ }
+
+ ptl = pmd_lock(mm, pmdp);
+ if (!pmd_present(*pmdp)) {
+- /* Not yet mapped memory has a zero key */
+ spin_unlock(ptl);
+- *key = 0;
+ return 0;
+ }
+
+@@ -988,6 +1009,7 @@ EXPORT_SYMBOL(get_guest_storage_key);
+ int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
+ unsigned long *oldpte, unsigned long *oldpgste)
+ {
++ struct vm_area_struct *vma;
+ unsigned long pgstev;
+ spinlock_t *ptl;
+ pgste_t pgste;
+@@ -997,6 +1019,10 @@ int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
+ WARN_ON_ONCE(orc > ESSA_MAX);
+ if (unlikely(orc > ESSA_MAX))
+ return -EINVAL;
++
++ vma = vma_lookup(mm, hva);
++ if (!vma || is_vm_hugetlb_page(vma))
++ return -EFAULT;
+ ptep = get_locked_pte(mm, hva, &ptl);
+ if (unlikely(!ptep))
+ return -EFAULT;
+@@ -1089,10 +1115,14 @@ EXPORT_SYMBOL(pgste_perform_essa);
+ int set_pgste_bits(struct mm_struct *mm, unsigned long hva,
+ unsigned long bits, unsigned long value)
+ {
++ struct vm_area_struct *vma;
+ spinlock_t *ptl;
+ pgste_t new;
+ pte_t *ptep;
+
++ vma = vma_lookup(mm, hva);
++ if (!vma || is_vm_hugetlb_page(vma))
++ return -EFAULT;
+ ptep = get_locked_pte(mm, hva, &ptl);
+ if (unlikely(!ptep))
+ return -EFAULT;
+@@ -1117,9 +1147,13 @@ EXPORT_SYMBOL(set_pgste_bits);
+ */
+ int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep)
+ {
++ struct vm_area_struct *vma;
+ spinlock_t *ptl;
+ pte_t *ptep;
+
++ vma = vma_lookup(mm, hva);
++ if (!vma || is_vm_hugetlb_page(vma))
++ return -EFAULT;
+ ptep = get_locked_pte(mm, hva, &ptl);
+ if (unlikely(!ptep))
+ return -EFAULT;
+diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
+index b833155ce8381..639924d983315 100644
+--- a/arch/s390/pci/pci.c
++++ b/arch/s390/pci/pci.c
+@@ -69,6 +69,7 @@ struct zpci_dev *get_zdev_by_fid(u32 fid)
+ list_for_each_entry(tmp, &zpci_list, entry) {
+ if (tmp->fid == fid) {
+ zdev = tmp;
++ zpci_zdev_get(zdev);
+ break;
+ }
+ }
+diff --git a/arch/s390/pci/pci_bus.h b/arch/s390/pci/pci_bus.h
+index e359d2686178b..ecef3a9e16c00 100644
+--- a/arch/s390/pci/pci_bus.h
++++ b/arch/s390/pci/pci_bus.h
+@@ -19,7 +19,8 @@ void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error);
+ void zpci_release_device(struct kref *kref);
+ static inline void zpci_zdev_put(struct zpci_dev *zdev)
+ {
+- kref_put(&zdev->kref, zpci_release_device);
++ if (zdev)
++ kref_put(&zdev->kref, zpci_release_device);
+ }
+
+ static inline void zpci_zdev_get(struct zpci_dev *zdev)
+diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
+index be077b39da336..5011d27461fd3 100644
+--- a/arch/s390/pci/pci_clp.c
++++ b/arch/s390/pci/pci_clp.c
+@@ -22,6 +22,8 @@
+ #include <asm/clp.h>
+ #include <uapi/asm/clp.h>
+
++#include "pci_bus.h"
++
+ bool zpci_unique_uid;
+
+ void update_uid_checking(bool new)
+@@ -403,8 +405,11 @@ static void __clp_add(struct clp_fh_list_entry *entry, void *data)
+ return;
+
+ zdev = get_zdev_by_fid(entry->fid);
+- if (!zdev)
+- zpci_create_device(entry->fid, entry->fh, entry->config_state);
++ if (zdev) {
++ zpci_zdev_put(zdev);
++ return;
++ }
++ zpci_create_device(entry->fid, entry->fh, entry->config_state);
+ }
+
+ int clp_scan_pci_devices(void)
+diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c
+index 5b8d647523f96..6d57625b8ed99 100644
+--- a/arch/s390/pci/pci_event.c
++++ b/arch/s390/pci/pci_event.c
+@@ -62,10 +62,12 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
+ pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid);
+
+ if (!pdev)
+- return;
++ goto no_pdev;
+
+ pdev->error_state = pci_channel_io_perm_failure;
+ pci_dev_put(pdev);
++no_pdev:
++ zpci_zdev_put(zdev);
+ }
+
+ void zpci_event_error(void *data)
+@@ -94,6 +96,7 @@ static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh)
+ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
+ {
+ struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);
++ bool existing_zdev = !!zdev;
+ enum zpci_state state;
+
+ zpci_err("avail CCDF:\n");
+@@ -156,6 +159,8 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
+ default:
+ break;
+ }
++ if (existing_zdev)
++ zpci_zdev_put(zdev);
+ }
+
+ void zpci_event_availability(void *data)
+diff --git a/arch/sh/Kconfig.debug b/arch/sh/Kconfig.debug
+index 958f790273ab9..10290e5c1f438 100644
+--- a/arch/sh/Kconfig.debug
++++ b/arch/sh/Kconfig.debug
+@@ -54,6 +54,7 @@ config DUMP_CODE
+
+ config DWARF_UNWINDER
+ bool "Enable the DWARF unwinder for stacktraces"
++ depends on DEBUG_KERNEL
+ select FRAME_POINTER
+ default n
+ help
+diff --git a/arch/sh/configs/titan_defconfig b/arch/sh/configs/titan_defconfig
+index ba887f1351be6..cd5c58916c65a 100644
+--- a/arch/sh/configs/titan_defconfig
++++ b/arch/sh/configs/titan_defconfig
+@@ -242,7 +242,6 @@ CONFIG_NFSD=y
+ CONFIG_NFSD_V3=y
+ CONFIG_SMB_FS=m
+ CONFIG_CIFS=m
+-CONFIG_CIFS_WEAK_PW_HASH=y
+ CONFIG_PARTITION_ADVANCED=y
+ CONFIG_NLS_CODEPAGE_437=m
+ CONFIG_NLS_ASCII=m
+diff --git a/arch/sh/include/asm/sfp-machine.h b/arch/sh/include/asm/sfp-machine.h
+index cbc7cf8c97ce6..2d2423478b71d 100644
+--- a/arch/sh/include/asm/sfp-machine.h
++++ b/arch/sh/include/asm/sfp-machine.h
+@@ -13,6 +13,14 @@
+ #ifndef _SFP_MACHINE_H
+ #define _SFP_MACHINE_H
+
++#ifdef __BIG_ENDIAN__
++#define __BYTE_ORDER __BIG_ENDIAN
++#define __LITTLE_ENDIAN 0
++#else
++#define __BYTE_ORDER __LITTLE_ENDIAN
++#define __BIG_ENDIAN 0
++#endif
++
+ #define _FP_W_TYPE_SIZE 32
+ #define _FP_W_TYPE unsigned long
+ #define _FP_WS_TYPE signed long
+diff --git a/arch/sh/kernel/cpu/fpu.c b/arch/sh/kernel/cpu/fpu.c
+index ae354a2931e7e..fd6db0ab19288 100644
+--- a/arch/sh/kernel/cpu/fpu.c
++++ b/arch/sh/kernel/cpu/fpu.c
+@@ -62,18 +62,20 @@ void fpu_state_restore(struct pt_regs *regs)
+ }
+
+ if (!tsk_used_math(tsk)) {
+- local_irq_enable();
++ int ret;
+ /*
+ * does a slab alloc which can sleep
+ */
+- if (init_fpu(tsk)) {
++ local_irq_enable();
++ ret = init_fpu(tsk);
++ local_irq_disable();
++ if (ret) {
+ /*
+ * ran out of memory!
+ */
+- do_group_exit(SIGKILL);
++ force_sig(SIGKILL);
+ return;
+ }
+- local_irq_disable();
+ }
+
+ grab_fpu(regs);
+diff --git a/arch/sh/kernel/cpu/sh4a/smp-shx3.c b/arch/sh/kernel/cpu/sh4a/smp-shx3.c
+index f8a2bec0f260b..1261dc7b84e8b 100644
+--- a/arch/sh/kernel/cpu/sh4a/smp-shx3.c
++++ b/arch/sh/kernel/cpu/sh4a/smp-shx3.c
+@@ -73,8 +73,9 @@ static void shx3_prepare_cpus(unsigned int max_cpus)
+ BUILD_BUG_ON(SMP_MSG_NR >= 8);
+
+ for (i = 0; i < SMP_MSG_NR; i++)
+- request_irq(104 + i, ipi_interrupt_handler,
+- IRQF_PERCPU, "IPI", (void *)(long)i);
++ if (request_irq(104 + i, ipi_interrupt_handler,
++ IRQF_PERCPU, "IPI", (void *)(long)i))
++ pr_err("Failed to request irq %d\n", i);
+
+ for (i = 0; i < max_cpus; i++)
+ set_cpu_present(i, true);
+diff --git a/arch/sh/math-emu/math.c b/arch/sh/math-emu/math.c
+index e8be0eca0444a..615ba932c398e 100644
+--- a/arch/sh/math-emu/math.c
++++ b/arch/sh/math-emu/math.c
+@@ -467,109 +467,6 @@ static int fpu_emulate(u16 code, struct sh_fpu_soft_struct *fregs, struct pt_reg
+ return id_sys(fregs, regs, code);
+ }
+
+-/**
+- * denormal_to_double - Given denormalized float number,
+- * store double float
+- *
+- * @fpu: Pointer to sh_fpu_soft structure
+- * @n: Index to FP register
+- */
+-static void denormal_to_double(struct sh_fpu_soft_struct *fpu, int n)
+-{
+- unsigned long du, dl;
+- unsigned long x = fpu->fpul;
+- int exp = 1023 - 126;
+-
+- if (x != 0 && (x & 0x7f800000) == 0) {
+- du = (x & 0x80000000);
+- while ((x & 0x00800000) == 0) {
+- x <<= 1;
+- exp--;
+- }
+- x &= 0x007fffff;
+- du |= (exp << 20) | (x >> 3);
+- dl = x << 29;
+-
+- fpu->fp_regs[n] = du;
+- fpu->fp_regs[n+1] = dl;
+- }
+-}
+-
+-/**
+- * ieee_fpe_handler - Handle denormalized number exception
+- *
+- * @regs: Pointer to register structure
+- *
+- * Returns 1 when it's handled (should not cause exception).
+- */
+-static int ieee_fpe_handler(struct pt_regs *regs)
+-{
+- unsigned short insn = *(unsigned short *)regs->pc;
+- unsigned short finsn;
+- unsigned long nextpc;
+- int nib[4] = {
+- (insn >> 12) & 0xf,
+- (insn >> 8) & 0xf,
+- (insn >> 4) & 0xf,
+- insn & 0xf};
+-
+- if (nib[0] == 0xb ||
+- (nib[0] == 0x4 && nib[2] == 0x0 && nib[3] == 0xb)) /* bsr & jsr */
+- regs->pr = regs->pc + 4;
+-
+- if (nib[0] == 0xa || nib[0] == 0xb) { /* bra & bsr */
+- nextpc = regs->pc + 4 + ((short) ((insn & 0xfff) << 4) >> 3);
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else if (nib[0] == 0x8 && nib[1] == 0xd) { /* bt/s */
+- if (regs->sr & 1)
+- nextpc = regs->pc + 4 + ((char) (insn & 0xff) << 1);
+- else
+- nextpc = regs->pc + 4;
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else if (nib[0] == 0x8 && nib[1] == 0xf) { /* bf/s */
+- if (regs->sr & 1)
+- nextpc = regs->pc + 4;
+- else
+- nextpc = regs->pc + 4 + ((char) (insn & 0xff) << 1);
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else if (nib[0] == 0x4 && nib[3] == 0xb &&
+- (nib[2] == 0x0 || nib[2] == 0x2)) { /* jmp & jsr */
+- nextpc = regs->regs[nib[1]];
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else if (nib[0] == 0x0 && nib[3] == 0x3 &&
+- (nib[2] == 0x0 || nib[2] == 0x2)) { /* braf & bsrf */
+- nextpc = regs->pc + 4 + regs->regs[nib[1]];
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else if (insn == 0x000b) { /* rts */
+- nextpc = regs->pr;
+- finsn = *(unsigned short *) (regs->pc + 2);
+- } else {
+- nextpc = regs->pc + 2;
+- finsn = insn;
+- }
+-
+- if ((finsn & 0xf1ff) == 0xf0ad) { /* fcnvsd */
+- struct task_struct *tsk = current;
+-
+- if ((tsk->thread.xstate->softfpu.fpscr & (1 << 17))) {
+- /* FPU error */
+- denormal_to_double (&tsk->thread.xstate->softfpu,
+- (finsn >> 8) & 0xf);
+- tsk->thread.xstate->softfpu.fpscr &=
+- ~(FPSCR_CAUSE_MASK | FPSCR_FLAG_MASK);
+- task_thread_info(tsk)->status |= TS_USEDFPU;
+- } else {
+- force_sig_fault(SIGFPE, FPE_FLTINV,
+- (void __user *)regs->pc);
+- }
+-
+- regs->pc = nextpc;
+- return 1;
+- }
+-
+- return 0;
+-}
+-
+ /**
+ * fpu_init - Initialize FPU registers
+ * @fpu: Pointer to software emulated FPU registers.
+diff --git a/arch/sparc/boot/Makefile b/arch/sparc/boot/Makefile
+index 849236d4eca48..45e5c76d449ea 100644
+--- a/arch/sparc/boot/Makefile
++++ b/arch/sparc/boot/Makefile
+@@ -22,7 +22,7 @@ ifeq ($(CONFIG_SPARC64),y)
+
+ # Actual linking
+
+-$(obj)/zImage: $(obj)/image
++$(obj)/zImage: $(obj)/image FORCE
+ $(call if_changed,gzip)
+ @echo ' kernel: $@ is ready'
+
+@@ -31,7 +31,7 @@ $(obj)/vmlinux.aout: vmlinux FORCE
+ @echo ' kernel: $@ is ready'
+ else
+
+-$(obj)/zImage: $(obj)/image
++$(obj)/zImage: $(obj)/image FORCE
+ $(call if_changed,strip)
+ @echo ' kernel: $@ is ready'
+
+@@ -44,7 +44,7 @@ OBJCOPYFLAGS_image.bin := -S -O binary -R .note -R .comment
+ $(obj)/image.bin: $(obj)/image FORCE
+ $(call if_changed,objcopy)
+
+-$(obj)/image.gz: $(obj)/image.bin
++$(obj)/image.gz: $(obj)/image.bin FORCE
+ $(call if_changed,gzip)
+
+ UIMAGE_LOADADDR = $(CONFIG_UBOOT_LOAD_ADDR)
+@@ -56,7 +56,7 @@ quiet_cmd_uimage.o = UIMAGE.O $@
+ -r -b binary $@ -o $@.o
+
+ targets += uImage
+-$(obj)/uImage: $(obj)/image.gz
++$(obj)/uImage: $(obj)/image.gz FORCE
+ $(call if_changed,uimage)
+ $(call if_changed,uimage.o)
+ @echo ' Image $@ is ready'
+diff --git a/arch/sparc/include/asm/timex_32.h b/arch/sparc/include/asm/timex_32.h
+index 542915b462097..f86326a6f89e0 100644
+--- a/arch/sparc/include/asm/timex_32.h
++++ b/arch/sparc/include/asm/timex_32.h
+@@ -9,8 +9,6 @@
+
+ #define CLOCK_TICK_RATE 1193180 /* Underlying HZ */
+
+-/* XXX Maybe do something better at some point... -DaveM */
+-typedef unsigned long cycles_t;
+-#define get_cycles() (0)
++#include <asm-generic/timex.h>
+
+ #endif
+diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c
+index 02f3ad55dfe31..74f80443b195f 100644
+--- a/arch/sparc/kernel/signal_32.c
++++ b/arch/sparc/kernel/signal_32.c
+@@ -65,7 +65,7 @@ struct rt_signal_frame {
+ */
+ static inline bool invalid_frame_pointer(void __user *fp, int fplen)
+ {
+- if ((((unsigned long) fp) & 15) || !__access_ok((unsigned long)fp, fplen))
++ if ((((unsigned long) fp) & 15) || !access_ok(fp, fplen))
+ return true;
+
+ return false;
+@@ -244,7 +244,7 @@ static int setup_frame(struct ksignal *ksig, struct pt_regs *regs,
+ get_sigframe(ksig, regs, sigframe_size);
+
+ if (invalid_frame_pointer(sf, sigframe_size)) {
+- do_exit(SIGILL);
++ force_exit_sig(SIGILL);
+ return -EINVAL;
+ }
+
+@@ -336,7 +336,7 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs,
+ sf = (struct rt_signal_frame __user *)
+ get_sigframe(ksig, regs, sigframe_size);
+ if (invalid_frame_pointer(sf, sigframe_size)) {
+- do_exit(SIGILL);
++ force_exit_sig(SIGILL);
+ return -EINVAL;
+ }
+
+diff --git a/arch/sparc/kernel/windows.c b/arch/sparc/kernel/windows.c
+index 69a6ba6e92937..8f20862ccc83e 100644
+--- a/arch/sparc/kernel/windows.c
++++ b/arch/sparc/kernel/windows.c
+@@ -121,8 +121,10 @@ void try_to_clear_window_buffer(struct pt_regs *regs, int who)
+
+ if ((sp & 7) ||
+ copy_to_user((char __user *) sp, &tp->reg_window[window],
+- sizeof(struct reg_window32)))
+- do_exit(SIGILL);
++ sizeof(struct reg_window32))) {
++ force_exit_sig(SIGILL);
++ return;
++ }
+ }
+ tp->w_saved = 0;
+ }
+diff --git a/arch/um/.gitignore b/arch/um/.gitignore
+index 6323e5571887e..d69ea5b562cee 100644
+--- a/arch/um/.gitignore
++++ b/arch/um/.gitignore
+@@ -2,3 +2,4 @@
+ kernel/config.c
+ kernel/config.tmp
+ kernel/vmlinux.lds
++kernel/capflags.c
+diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c
+index 6ead1e2404576..8ca67a6926830 100644
+--- a/arch/um/drivers/mconsole_kern.c
++++ b/arch/um/drivers/mconsole_kern.c
+@@ -224,7 +224,7 @@ void mconsole_go(struct mc_request *req)
+
+ void mconsole_stop(struct mc_request *req)
+ {
+- deactivate_fd(req->originating_fd, MCONSOLE_IRQ);
++ block_signals();
+ os_set_fd_block(req->originating_fd, 1);
+ mconsole_reply(req, "stopped", 0, 0);
+ for (;;) {
+@@ -247,6 +247,7 @@ void mconsole_stop(struct mc_request *req)
+ }
+ os_set_fd_block(req->originating_fd, 0);
+ mconsole_reply(req, "", 0, 0);
++ unblock_signals();
+ }
+
+ static DEFINE_SPINLOCK(mc_devices_lock);
+diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c
+index cd9dc0556e913..fefd343412c79 100644
+--- a/arch/um/drivers/ubd_kern.c
++++ b/arch/um/drivers/ubd_kern.c
+@@ -27,6 +27,7 @@
+ #include <linux/blk-mq.h>
+ #include <linux/ata.h>
+ #include <linux/hdreg.h>
++#include <linux/major.h>
+ #include <linux/cdrom.h>
+ #include <linux/proc_fs.h>
+ #include <linux/seq_file.h>
+diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c
+index c080666330234..0ab58016db22f 100644
+--- a/arch/um/drivers/virt-pci.c
++++ b/arch/um/drivers/virt-pci.c
+@@ -181,15 +181,15 @@ static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
+ /* buf->data is maximum size - we may only use parts of it */
+ struct um_pci_message_buffer *buf;
+ u8 *data;
+- unsigned long ret = ~0ULL;
++ unsigned long ret = ULONG_MAX;
+
+ if (!dev)
+- return ~0ULL;
++ return ULONG_MAX;
+
+ buf = get_cpu_var(um_pci_msg_bufs);
+ data = buf->data;
+
+- memset(data, 0xff, sizeof(data));
++ memset(buf->data, 0xff, sizeof(buf->data));
+
+ switch (size) {
+ case 1:
+@@ -304,7 +304,7 @@ static unsigned long um_pci_bar_read(void *priv, unsigned int offset,
+ /* buf->data is maximum size - we may only use parts of it */
+ struct um_pci_message_buffer *buf;
+ u8 *data;
+- unsigned long ret = ~0ULL;
++ unsigned long ret = ULONG_MAX;
+
+ buf = get_cpu_var(um_pci_msg_bufs);
+ data = buf->data;
+diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
+index d51e445df7976..7755cb4ff9fc6 100644
+--- a/arch/um/drivers/virtio_uml.c
++++ b/arch/um/drivers/virtio_uml.c
+@@ -1090,6 +1090,8 @@ static void virtio_uml_release_dev(struct device *d)
+ container_of(d, struct virtio_device, dev);
+ struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
+
++ time_travel_propagate_time();
++
+ /* might not have been opened due to not negotiating the feature */
+ if (vu_dev->req_fd >= 0) {
+ um_free_irq(vu_dev->irq, vu_dev);
+@@ -1136,6 +1138,8 @@ static int virtio_uml_probe(struct platform_device *pdev)
+ vu_dev->pdev = pdev;
+ vu_dev->req_fd = -1;
+
++ time_travel_propagate_time();
++
+ do {
+ rc = os_connect_socket(pdata->socket_path);
+ } while (rc == -EINTR);
+diff --git a/arch/um/include/asm/delay.h b/arch/um/include/asm/delay.h
+index 56fc2b8f2dd01..e79b2ab6f40c8 100644
+--- a/arch/um/include/asm/delay.h
++++ b/arch/um/include/asm/delay.h
+@@ -14,7 +14,7 @@ static inline void um_ndelay(unsigned long nsecs)
+ ndelay(nsecs);
+ }
+ #undef ndelay
+-#define ndelay um_ndelay
++#define ndelay(n) um_ndelay(n)
+
+ static inline void um_udelay(unsigned long usecs)
+ {
+@@ -26,5 +26,5 @@ static inline void um_udelay(unsigned long usecs)
+ udelay(usecs);
+ }
+ #undef udelay
+-#define udelay um_udelay
++#define udelay(n) um_udelay(n)
+ #endif /* __UM_DELAY_H */
+diff --git a/arch/um/include/asm/irqflags.h b/arch/um/include/asm/irqflags.h
+index dab5744e9253d..1e69ef5bc35e0 100644
+--- a/arch/um/include/asm/irqflags.h
++++ b/arch/um/include/asm/irqflags.h
+@@ -3,7 +3,7 @@
+ #define __UM_IRQFLAGS_H
+
+ extern int signals_enabled;
+-int set_signals(int enable);
++int um_set_signals(int enable);
+ void block_signals(void);
+ void unblock_signals(void);
+
+@@ -16,7 +16,7 @@ static inline unsigned long arch_local_save_flags(void)
+ #define arch_local_irq_restore arch_local_irq_restore
+ static inline void arch_local_irq_restore(unsigned long flags)
+ {
+- set_signals(flags);
++ um_set_signals(flags);
+ }
+
+ #define arch_local_irq_enable arch_local_irq_enable
+diff --git a/arch/um/include/asm/timex.h b/arch/um/include/asm/timex.h
+index e392a9a5bc9bd..9f27176adb26d 100644
+--- a/arch/um/include/asm/timex.h
++++ b/arch/um/include/asm/timex.h
+@@ -2,13 +2,8 @@
+ #ifndef __UM_TIMEX_H
+ #define __UM_TIMEX_H
+
+-typedef unsigned long cycles_t;
+-
+-static inline cycles_t get_cycles (void)
+-{
+- return 0;
+-}
+-
+ #define CLOCK_TICK_RATE (HZ)
+
++#include <asm-generic/timex.h>
++
+ #endif
+diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h
+index f512704a9ec7b..22b39de73c246 100644
+--- a/arch/um/include/asm/xor.h
++++ b/arch/um/include/asm/xor.h
+@@ -4,8 +4,10 @@
+
+ #ifdef CONFIG_64BIT
+ #undef CONFIG_X86_32
++#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_sse_pf64))
+ #else
+ #define CONFIG_X86_32 1
++#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_8regs))
+ #endif
+
+ #include <asm/cpufeature.h>
+@@ -16,7 +18,7 @@
+ #undef XOR_SELECT_TEMPLATE
+ /* pick an arbitrary one - measuring isn't possible with inf-cpu */
+ #define XOR_SELECT_TEMPLATE(x) \
+- (time_travel_mode == TT_MODE_INFCPU ? &xor_block_8regs : NULL)
++ (time_travel_mode == TT_MODE_INFCPU ? TT_CPU_INF_XOR_DEFAULT : x))
+ #endif
+
+ #endif
+diff --git a/arch/um/include/shared/longjmp.h b/arch/um/include/shared/longjmp.h
+index bdb2869b72b31..8863319039f3d 100644
+--- a/arch/um/include/shared/longjmp.h
++++ b/arch/um/include/shared/longjmp.h
+@@ -18,7 +18,7 @@ extern void longjmp(jmp_buf, int);
+ enable = *(volatile int *)&signals_enabled; \
+ n = setjmp(*buf); \
+ if(n != 0) \
+- set_signals_trace(enable); \
++ um_set_signals_trace(enable); \
+ n; })
+
+ #endif
+diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h
+index 96d400387c93e..03ffbdddcc480 100644
+--- a/arch/um/include/shared/os.h
++++ b/arch/um/include/shared/os.h
+@@ -238,8 +238,8 @@ extern void send_sigio_to_self(void);
+ extern int change_sig(int signal, int on);
+ extern void block_signals(void);
+ extern void unblock_signals(void);
+-extern int set_signals(int enable);
+-extern int set_signals_trace(int enable);
++extern int um_set_signals(int enable);
++extern int um_set_signals_trace(int enable);
+ extern int os_is_signal_stack(void);
+ extern void deliver_alarm(void);
+ extern void register_pm_wake_signal(void);
+diff --git a/arch/um/include/shared/registers.h b/arch/um/include/shared/registers.h
+index 0c50fa6e8a55b..fbb709a222839 100644
+--- a/arch/um/include/shared/registers.h
++++ b/arch/um/include/shared/registers.h
+@@ -16,8 +16,8 @@ extern int restore_fp_registers(int pid, unsigned long *fp_regs);
+ extern int save_fpx_registers(int pid, unsigned long *fp_regs);
+ extern int restore_fpx_registers(int pid, unsigned long *fp_regs);
+ extern int save_registers(int pid, struct uml_pt_regs *regs);
+-extern int restore_registers(int pid, struct uml_pt_regs *regs);
+-extern int init_registers(int pid);
++extern int restore_pid_registers(int pid, struct uml_pt_regs *regs);
++extern int init_pid_registers(int pid);
+ extern void get_safe_registers(unsigned long *regs, unsigned long *fp_regs);
+ extern unsigned long get_thread_reg(int reg, jmp_buf *buf);
+ extern int get_fp_registers(int pid, unsigned long *regs);
+diff --git a/arch/um/kernel/ksyms.c b/arch/um/kernel/ksyms.c
+index b1e5634398d09..3a85bde3e1734 100644
+--- a/arch/um/kernel/ksyms.c
++++ b/arch/um/kernel/ksyms.c
+@@ -6,7 +6,7 @@
+ #include <linux/module.h>
+ #include <os.h>
+
+-EXPORT_SYMBOL(set_signals);
++EXPORT_SYMBOL(um_set_signals);
+ EXPORT_SYMBOL(signals_enabled);
+
+ EXPORT_SYMBOL(os_stat_fd);
+diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
+index 3198c47673879..c32efb09db214 100644
+--- a/arch/um/kernel/trap.c
++++ b/arch/um/kernel/trap.c
+@@ -158,7 +158,7 @@ static void bad_segv(struct faultinfo fi, unsigned long ip)
+
+ void fatal_sigsegv(void)
+ {
+- force_sigsegv(SIGSEGV);
++ force_fatal_sig(SIGSEGV);
+ do_signal(&current->thread.regs);
+ /*
+ * This is to tell gcc that we're not returning - do_signal
+diff --git a/arch/um/os-Linux/registers.c b/arch/um/os-Linux/registers.c
+index 2d9270508e156..b123955be7acc 100644
+--- a/arch/um/os-Linux/registers.c
++++ b/arch/um/os-Linux/registers.c
+@@ -21,7 +21,7 @@ int save_registers(int pid, struct uml_pt_regs *regs)
+ return 0;
+ }
+
+-int restore_registers(int pid, struct uml_pt_regs *regs)
++int restore_pid_registers(int pid, struct uml_pt_regs *regs)
+ {
+ int err;
+
+@@ -36,7 +36,7 @@ int restore_registers(int pid, struct uml_pt_regs *regs)
+ static unsigned long exec_regs[MAX_REG_NR];
+ static unsigned long exec_fp_regs[FP_SIZE];
+
+-int init_registers(int pid)
++int init_pid_registers(int pid)
+ {
+ int err;
+
+diff --git a/arch/um/os-Linux/sigio.c b/arch/um/os-Linux/sigio.c
+index 6597ea1986ffa..9e71794839e87 100644
+--- a/arch/um/os-Linux/sigio.c
++++ b/arch/um/os-Linux/sigio.c
+@@ -132,7 +132,7 @@ static void update_thread(void)
+ int n;
+ char c;
+
+- flags = set_signals_trace(0);
++ flags = um_set_signals_trace(0);
+ CATCH_EINTR(n = write(sigio_private[0], &c, sizeof(c)));
+ if (n != sizeof(c)) {
+ printk(UM_KERN_ERR "update_thread : write failed, err = %d\n",
+@@ -147,7 +147,7 @@ static void update_thread(void)
+ goto fail;
+ }
+
+- set_signals_trace(flags);
++ um_set_signals_trace(flags);
+ return;
+ fail:
+ /* Critical section start */
+@@ -161,7 +161,7 @@ static void update_thread(void)
+ close(write_sigio_fds[0]);
+ close(write_sigio_fds[1]);
+ /* Critical section end */
+- set_signals_trace(flags);
++ um_set_signals_trace(flags);
+ }
+
+ int __add_sigio_fd(int fd)
+diff --git a/arch/um/os-Linux/signal.c b/arch/um/os-Linux/signal.c
+index 6cf098c23a394..24a403a70a020 100644
+--- a/arch/um/os-Linux/signal.c
++++ b/arch/um/os-Linux/signal.c
+@@ -94,7 +94,7 @@ void sig_handler(int sig, struct siginfo *si, mcontext_t *mc)
+
+ sig_handler_common(sig, si, mc);
+
+- set_signals_trace(enabled);
++ um_set_signals_trace(enabled);
+ }
+
+ static void timer_real_alarm_handler(mcontext_t *mc)
+@@ -126,7 +126,7 @@ void timer_alarm_handler(int sig, struct siginfo *unused_si, mcontext_t *mc)
+
+ signals_active &= ~SIGALRM_MASK;
+
+- set_signals_trace(enabled);
++ um_set_signals_trace(enabled);
+ }
+
+ void deliver_alarm(void) {
+@@ -348,7 +348,7 @@ void unblock_signals(void)
+ }
+ }
+
+-int set_signals(int enable)
++int um_set_signals(int enable)
+ {
+ int ret;
+ if (signals_enabled == enable)
+@@ -362,7 +362,7 @@ int set_signals(int enable)
+ return ret;
+ }
+
+-int set_signals_trace(int enable)
++int um_set_signals_trace(int enable)
+ {
+ int ret;
+ if (signals_enabled == enable)
+diff --git a/arch/um/os-Linux/start_up.c b/arch/um/os-Linux/start_up.c
+index 8a72c99994eb1..e3ee4db58b40d 100644
+--- a/arch/um/os-Linux/start_up.c
++++ b/arch/um/os-Linux/start_up.c
+@@ -368,7 +368,7 @@ void __init os_early_checks(void)
+ check_tmpexec();
+
+ pid = start_ptraced_child();
+- if (init_registers(pid))
++ if (init_pid_registers(pid))
+ fatal("Failed to initialize default registers");
+ stop_ptraced_child(pid, 1, 1);
+ }
+diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
+index d9830e7e1060f..d02b04d300966 100644
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@ -468,6 +468,18 @@ config RETPOLINE
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
++config CC_HAS_SLS
++ def_bool $(cc-option,-mharden-sls=all)
++
++config SLS
++ bool "Mitigate Straight-Line-Speculation"
++ depends on CC_HAS_SLS && X86_64
++ default n
++ help
++ Compile the kernel with straight-line-speculation options to guard
++ against straight line speculation. The kernel image might be slightly
++ larger.
++
+ config X86_CPU_RESCTRL
+ bool "x86 CPU resource control support"
+ depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
+@@ -1256,7 +1268,8 @@ config TOSHIBA
+
+ config I8K
+ tristate "Dell i8k legacy laptop support"
+- select HWMON
++ depends on HWMON
++ depends on PROC_FS
+ select SENSORS_DELL_SMM
+ help
+ This option enables legacy /proc/i8k userspace interface in hwmon
+@@ -1518,6 +1531,7 @@ config AMD_MEM_ENCRYPT
+ select ARCH_HAS_FORCE_DMA_UNENCRYPTED
+ select INSTRUCTION_DECODER
+ select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS
++ select ARCH_HAS_CC_PLATFORM
+ help
+ Say yes to enable support for the encryption of system memory.
+ This requires an AMD processor that supports Secure Memory
+@@ -1917,6 +1931,7 @@ config EFI
+ depends on ACPI
+ select UCS2_STRING
+ select EFI_RUNTIME_WRAPPERS
++ select ARCH_USE_MEMREMAP_PROT
+ help
+ This enables the kernel to use EFI runtime services that are
+ available (such as the EFI variable services).
+@@ -2795,6 +2810,11 @@ config IA32_AOUT
+ config X86_X32
+ bool "x32 ABI for 64-bit mode"
+ depends on X86_64
++ # llvm-objcopy does not convert x86_64 .note.gnu.property or
++ # compressed debug sections to x86_x32 properly:
++ # https://github.com/ClangBuiltLinux/linux/issues/514
++ # https://github.com/ClangBuiltLinux/linux/issues/1141
++ depends on $(success,$(OBJCOPY) --version | head -n1 | grep -qv llvm)
+ help
+ Include code to run binaries for the x32 native 32-bit ABI
+ for 64-bit processors. An x32 process gets access to the
+diff --git a/arch/x86/Makefile b/arch/x86/Makefile
+index 7488cfbbd2f60..300227818206b 100644
+--- a/arch/x86/Makefile
++++ b/arch/x86/Makefile
+@@ -179,6 +179,10 @@ ifdef CONFIG_RETPOLINE
+ endif
+ endif
+
++ifdef CONFIG_SLS
++ KBUILD_CFLAGS += -mharden-sls=all
++endif
++
+ KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
+
+ ifdef CONFIG_LTO_CLANG
+diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
+index 431bf7f846c3c..e118136460518 100644
+--- a/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/boot/compressed/Makefile
+@@ -28,7 +28,11 @@ KCOV_INSTRUMENT := n
+ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
+ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
+
+-KBUILD_CFLAGS := -m$(BITS) -O2
++# CLANG_FLAGS must come before any cc-disable-warning or cc-option calls in
++# case of cross compiling, as it has the '--target=' flag, which is needed to
++# avoid errors with '-march=i386', and future flags may depend on the target to
++# be valid.
++KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS)
+ KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
+ KBUILD_CFLAGS += -Wundef
+ KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
+@@ -47,7 +51,6 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS
+ # Disable relocation relaxation in case the link is not PIE.
+ KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no)
+ KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h
+-KBUILD_CFLAGS += $(CLANG_FLAGS)
+
+ # sev.c indirectly inludes inat-table.h which is generated during
+ # compilation and stored in $(objtree). Add the directory to the includes so
+diff --git a/arch/x86/boot/compressed/efi_thunk_64.S b/arch/x86/boot/compressed/efi_thunk_64.S
+index 8bb92e9f4e973..70052779b235c 100644
+--- a/arch/x86/boot/compressed/efi_thunk_64.S
++++ b/arch/x86/boot/compressed/efi_thunk_64.S
+@@ -93,7 +93,7 @@ SYM_FUNC_START(__efi64_thunk)
+
+ pop %rbx
+ pop %rbp
+- ret
++ RET
+ SYM_FUNC_END(__efi64_thunk)
+
+ .code32
+diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
+index 572c535cf45bc..fd9441f404570 100644
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -813,7 +813,7 @@ SYM_FUNC_START(efi32_pe_entry)
+ 2: popl %edi // restore callee-save registers
+ popl %ebx
+ leave
+- ret
++ RET
+ SYM_FUNC_END(efi32_pe_entry)
+
+ .section ".rodata"
+@@ -868,7 +868,7 @@ SYM_FUNC_START(startup32_set_idt_entry)
+
+ pop %ecx
+ pop %ebx
+- ret
++ RET
+ SYM_FUNC_END(startup32_set_idt_entry)
+ #endif
+
+@@ -884,7 +884,7 @@ SYM_FUNC_START(startup32_load_idt)
+ movl %eax, rva(boot32_idt_desc+2)(%ebp)
+ lidt rva(boot32_idt_desc)(%ebp)
+ #endif
+- ret
++ RET
+ SYM_FUNC_END(startup32_load_idt)
+
+ /*
+@@ -954,7 +954,7 @@ SYM_FUNC_START(startup32_check_sev_cbit)
+ popl %ebx
+ popl %eax
+ #endif
+- ret
++ RET
+ SYM_FUNC_END(startup32_check_sev_cbit)
+
+ /*
+diff --git a/arch/x86/boot/compressed/mem_encrypt.S b/arch/x86/boot/compressed/mem_encrypt.S
+index c1e81a848b2a5..a63424d13627b 100644
+--- a/arch/x86/boot/compressed/mem_encrypt.S
++++ b/arch/x86/boot/compressed/mem_encrypt.S
+@@ -58,7 +58,7 @@ SYM_FUNC_START(get_sev_encryption_bit)
+
+ #endif /* CONFIG_AMD_MEM_ENCRYPT */
+
+- ret
++ RET
+ SYM_FUNC_END(get_sev_encryption_bit)
+
+ /**
+@@ -92,7 +92,7 @@ SYM_CODE_START_LOCAL(sev_es_req_cpuid)
+ /* All good - return success */
+ xorl %eax, %eax
+ 1:
+- ret
++ RET
+ 2:
+ movl $-1, %eax
+ jmp 1b
+@@ -221,7 +221,7 @@ SYM_FUNC_START(set_sev_encryption_mask)
+ #endif
+
+ xor %rax, %rax
+- ret
++ RET
+ SYM_FUNC_END(set_sev_encryption_mask)
+
+ .data
+diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
+index e81885384f604..99398cbdae434 100644
+--- a/arch/x86/configs/i386_defconfig
++++ b/arch/x86/configs/i386_defconfig
+@@ -262,3 +262,4 @@ CONFIG_BLK_DEV_IO_TRACE=y
+ CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
+ CONFIG_EARLY_PRINTK_DBGP=y
+ CONFIG_DEBUG_BOOT_PARAMS=y
++CONFIG_KALLSYMS_ALL=y
+diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
+index e8a7a0af2bdaa..d7298b104a456 100644
+--- a/arch/x86/configs/x86_64_defconfig
++++ b/arch/x86/configs/x86_64_defconfig
+@@ -258,3 +258,4 @@ CONFIG_BLK_DEV_IO_TRACE=y
+ CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
+ CONFIG_EARLY_PRINTK_DBGP=y
+ CONFIG_DEBUG_BOOT_PARAMS=y
++CONFIG_KALLSYMS_ALL=y
+diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
+index f307c93fc90a7..c3af959648e62 100644
+--- a/arch/x86/crypto/Makefile
++++ b/arch/x86/crypto/Makefile
+@@ -62,7 +62,9 @@ obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
+ sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o
+
+ obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += blake2s-x86_64.o
+-blake2s-x86_64-y := blake2s-core.o blake2s-glue.o
++blake2s-x86_64-y := blake2s-shash.o
++obj-$(if $(CONFIG_CRYPTO_BLAKE2S_X86),y) += libblake2s-x86_64.o
++libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o
+
+ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
+ ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
+diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S
+index 51d46d93efbcc..b48ddebb47489 100644
+--- a/arch/x86/crypto/aegis128-aesni-asm.S
++++ b/arch/x86/crypto/aegis128-aesni-asm.S
+@@ -122,7 +122,7 @@ SYM_FUNC_START_LOCAL(__load_partial)
+ pxor T0, MSG
+
+ .Lld_partial_8:
+- ret
++ RET
+ SYM_FUNC_END(__load_partial)
+
+ /*
+@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(__store_partial)
+ mov %r10b, (%r9)
+
+ .Lst_partial_1:
+- ret
++ RET
+ SYM_FUNC_END(__store_partial)
+
+ /*
+@@ -225,7 +225,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_init)
+ movdqu STATE4, 0x40(STATEP)
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_init)
+
+ /*
+@@ -337,7 +337,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
+ movdqu STATE3, 0x30(STATEP)
+ movdqu STATE4, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lad_out_1:
+ movdqu STATE4, 0x00(STATEP)
+@@ -346,7 +346,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
+ movdqu STATE2, 0x30(STATEP)
+ movdqu STATE3, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lad_out_2:
+ movdqu STATE3, 0x00(STATEP)
+@@ -355,7 +355,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
+ movdqu STATE1, 0x30(STATEP)
+ movdqu STATE2, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lad_out_3:
+ movdqu STATE2, 0x00(STATEP)
+@@ -364,7 +364,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
+ movdqu STATE0, 0x30(STATEP)
+ movdqu STATE1, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lad_out_4:
+ movdqu STATE1, 0x00(STATEP)
+@@ -373,11 +373,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad)
+ movdqu STATE4, 0x30(STATEP)
+ movdqu STATE0, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lad_out:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_ad)
+
+ .macro encrypt_block a s0 s1 s2 s3 s4 i
+@@ -452,7 +452,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
+ movdqu STATE2, 0x30(STATEP)
+ movdqu STATE3, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lenc_out_1:
+ movdqu STATE3, 0x00(STATEP)
+@@ -461,7 +461,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
+ movdqu STATE1, 0x30(STATEP)
+ movdqu STATE2, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lenc_out_2:
+ movdqu STATE2, 0x00(STATEP)
+@@ -470,7 +470,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
+ movdqu STATE0, 0x30(STATEP)
+ movdqu STATE1, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lenc_out_3:
+ movdqu STATE1, 0x00(STATEP)
+@@ -479,7 +479,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
+ movdqu STATE4, 0x30(STATEP)
+ movdqu STATE0, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lenc_out_4:
+ movdqu STATE0, 0x00(STATEP)
+@@ -488,11 +488,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc)
+ movdqu STATE3, 0x30(STATEP)
+ movdqu STATE4, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Lenc_out:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_enc)
+
+ /*
+@@ -532,7 +532,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
+ movdqu STATE3, 0x40(STATEP)
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
+
+ .macro decrypt_block a s0 s1 s2 s3 s4 i
+@@ -606,7 +606,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
+ movdqu STATE2, 0x30(STATEP)
+ movdqu STATE3, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Ldec_out_1:
+ movdqu STATE3, 0x00(STATEP)
+@@ -615,7 +615,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
+ movdqu STATE1, 0x30(STATEP)
+ movdqu STATE2, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Ldec_out_2:
+ movdqu STATE2, 0x00(STATEP)
+@@ -624,7 +624,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
+ movdqu STATE0, 0x30(STATEP)
+ movdqu STATE1, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Ldec_out_3:
+ movdqu STATE1, 0x00(STATEP)
+@@ -633,7 +633,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
+ movdqu STATE4, 0x30(STATEP)
+ movdqu STATE0, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Ldec_out_4:
+ movdqu STATE0, 0x00(STATEP)
+@@ -642,11 +642,11 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec)
+ movdqu STATE3, 0x30(STATEP)
+ movdqu STATE4, 0x40(STATEP)
+ FRAME_END
+- ret
++ RET
+
+ .Ldec_out:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_dec)
+
+ /*
+@@ -696,7 +696,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
+ movdqu STATE3, 0x40(STATEP)
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
+
+ /*
+@@ -743,5 +743,5 @@ SYM_FUNC_START(crypto_aegis128_aesni_final)
+ movdqu MSG, (%rsi)
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(crypto_aegis128_aesni_final)
+diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
+index 3f0fc7dd87d77..c799838242a69 100644
+--- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
++++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
+@@ -525,7 +525,7 @@ ddq_add_8:
+ /* return updated IV */
+ vpshufb xbyteswap, xcounter, xcounter
+ vmovdqu xcounter, (p_iv)
+- ret
++ RET
+ .endm
+
+ /*
+diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
+index 4e3972570916e..363699dd72206 100644
+--- a/arch/x86/crypto/aesni-intel_asm.S
++++ b/arch/x86/crypto/aesni-intel_asm.S
+@@ -1594,7 +1594,7 @@ SYM_FUNC_START(aesni_gcm_dec)
+ GCM_ENC_DEC dec
+ GCM_COMPLETE arg10, arg11
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_dec)
+
+
+@@ -1683,7 +1683,7 @@ SYM_FUNC_START(aesni_gcm_enc)
+
+ GCM_COMPLETE arg10, arg11
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_enc)
+
+ /*****************************************************************************
+@@ -1701,7 +1701,7 @@ SYM_FUNC_START(aesni_gcm_init)
+ FUNC_SAVE
+ GCM_INIT %arg3, %arg4,%arg5, %arg6
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_init)
+
+ /*****************************************************************************
+@@ -1716,7 +1716,7 @@ SYM_FUNC_START(aesni_gcm_enc_update)
+ FUNC_SAVE
+ GCM_ENC_DEC enc
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_enc_update)
+
+ /*****************************************************************************
+@@ -1731,7 +1731,7 @@ SYM_FUNC_START(aesni_gcm_dec_update)
+ FUNC_SAVE
+ GCM_ENC_DEC dec
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_dec_update)
+
+ /*****************************************************************************
+@@ -1746,7 +1746,7 @@ SYM_FUNC_START(aesni_gcm_finalize)
+ FUNC_SAVE
+ GCM_COMPLETE %arg3 %arg4
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_finalize)
+
+ #endif
+@@ -1762,7 +1762,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_256a)
+ pxor %xmm1, %xmm0
+ movaps %xmm0, (TKEYP)
+ add $0x10, TKEYP
+- ret
++ RET
+ SYM_FUNC_END(_key_expansion_256a)
+ SYM_FUNC_END_ALIAS(_key_expansion_128)
+
+@@ -1787,7 +1787,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_192a)
+ shufps $0b01001110, %xmm2, %xmm1
+ movaps %xmm1, 0x10(TKEYP)
+ add $0x20, TKEYP
+- ret
++ RET
+ SYM_FUNC_END(_key_expansion_192a)
+
+ SYM_FUNC_START_LOCAL(_key_expansion_192b)
+@@ -1806,7 +1806,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_192b)
+
+ movaps %xmm0, (TKEYP)
+ add $0x10, TKEYP
+- ret
++ RET
+ SYM_FUNC_END(_key_expansion_192b)
+
+ SYM_FUNC_START_LOCAL(_key_expansion_256b)
+@@ -1818,7 +1818,7 @@ SYM_FUNC_START_LOCAL(_key_expansion_256b)
+ pxor %xmm1, %xmm2
+ movaps %xmm2, (TKEYP)
+ add $0x10, TKEYP
+- ret
++ RET
+ SYM_FUNC_END(_key_expansion_256b)
+
+ /*
+@@ -1933,7 +1933,7 @@ SYM_FUNC_START(aesni_set_key)
+ popl KEYP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_set_key)
+
+ /*
+@@ -1957,7 +1957,7 @@ SYM_FUNC_START(aesni_enc)
+ popl KEYP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_enc)
+
+ /*
+@@ -2014,7 +2014,7 @@ SYM_FUNC_START_LOCAL(_aesni_enc1)
+ aesenc KEY, STATE
+ movaps 0x70(TKEYP), KEY
+ aesenclast KEY, STATE
+- ret
++ RET
+ SYM_FUNC_END(_aesni_enc1)
+
+ /*
+@@ -2122,7 +2122,7 @@ SYM_FUNC_START_LOCAL(_aesni_enc4)
+ aesenclast KEY, STATE2
+ aesenclast KEY, STATE3
+ aesenclast KEY, STATE4
+- ret
++ RET
+ SYM_FUNC_END(_aesni_enc4)
+
+ /*
+@@ -2147,7 +2147,7 @@ SYM_FUNC_START(aesni_dec)
+ popl KEYP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_dec)
+
+ /*
+@@ -2204,7 +2204,7 @@ SYM_FUNC_START_LOCAL(_aesni_dec1)
+ aesdec KEY, STATE
+ movaps 0x70(TKEYP), KEY
+ aesdeclast KEY, STATE
+- ret
++ RET
+ SYM_FUNC_END(_aesni_dec1)
+
+ /*
+@@ -2312,7 +2312,7 @@ SYM_FUNC_START_LOCAL(_aesni_dec4)
+ aesdeclast KEY, STATE2
+ aesdeclast KEY, STATE3
+ aesdeclast KEY, STATE4
+- ret
++ RET
+ SYM_FUNC_END(_aesni_dec4)
+
+ /*
+@@ -2372,7 +2372,7 @@ SYM_FUNC_START(aesni_ecb_enc)
+ popl LEN
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_ecb_enc)
+
+ /*
+@@ -2433,7 +2433,7 @@ SYM_FUNC_START(aesni_ecb_dec)
+ popl LEN
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_ecb_dec)
+
+ /*
+@@ -2477,7 +2477,7 @@ SYM_FUNC_START(aesni_cbc_enc)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_cbc_enc)
+
+ /*
+@@ -2570,7 +2570,7 @@ SYM_FUNC_START(aesni_cbc_dec)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_cbc_dec)
+
+ /*
+@@ -2627,7 +2627,7 @@ SYM_FUNC_START(aesni_cts_cbc_enc)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_cts_cbc_enc)
+
+ /*
+@@ -2688,7 +2688,7 @@ SYM_FUNC_START(aesni_cts_cbc_dec)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_cts_cbc_dec)
+
+ .pushsection .rodata
+@@ -2725,7 +2725,7 @@ SYM_FUNC_START_LOCAL(_aesni_inc_init)
+ mov $1, TCTR_LOW
+ movq TCTR_LOW, INC
+ movq CTR, TCTR_LOW
+- ret
++ RET
+ SYM_FUNC_END(_aesni_inc_init)
+
+ /*
+@@ -2753,7 +2753,7 @@ SYM_FUNC_START_LOCAL(_aesni_inc)
+ .Linc_low:
+ movaps CTR, IV
+ pshufb BSWAP_MASK, IV
+- ret
++ RET
+ SYM_FUNC_END(_aesni_inc)
+
+ /*
+@@ -2816,7 +2816,7 @@ SYM_FUNC_START(aesni_ctr_enc)
+ movups IV, (IVP)
+ .Lctr_enc_just_ret:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(aesni_ctr_enc)
+
+ #endif
+@@ -2932,7 +2932,7 @@ SYM_FUNC_START(aesni_xts_encrypt)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+
+ .Lxts_enc_1x:
+ add $64, LEN
+@@ -3092,7 +3092,7 @@ SYM_FUNC_START(aesni_xts_decrypt)
+ popl IVP
+ #endif
+ FRAME_END
+- ret
++ RET
+
+ .Lxts_dec_1x:
+ add $64, LEN
+diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S
+index 98e3552b6e039..0852ab573fd30 100644
+--- a/arch/x86/crypto/aesni-intel_avx-x86_64.S
++++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S
+@@ -1767,7 +1767,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen2)
+ FUNC_SAVE
+ INIT GHASH_MUL_AVX, PRECOMPUTE_AVX
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_init_avx_gen2)
+
+ ###############################################################################
+@@ -1788,15 +1788,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen2)
+ # must be 192
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_enc_update:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_enc_update:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2)
+
+ ###############################################################################
+@@ -1817,15 +1817,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen2)
+ # must be 192
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_dec_update:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_dec_update:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2)
+
+ ###############################################################################
+@@ -1846,15 +1846,15 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen2)
+ # must be 192
+ GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_finalize:
+ GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_finalize:
+ GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_finalize_avx_gen2)
+
+ ###############################################################################
+@@ -2735,7 +2735,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen4)
+ FUNC_SAVE
+ INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_init_avx_gen4)
+
+ ###############################################################################
+@@ -2756,15 +2756,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen4)
+ # must be 192
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_enc_update4:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_enc_update4:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4)
+
+ ###############################################################################
+@@ -2785,15 +2785,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen4)
+ # must be 192
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_dec_update4:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_dec_update4:
+ GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4)
+
+ ###############################################################################
+@@ -2814,13 +2814,13 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen4)
+ # must be 192
+ GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ key_128_finalize4:
+ GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ key_256_finalize4:
+ GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4
+ FUNC_RESTORE
+- ret
++ RET
+ SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
+diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
+index 0fc961bef299c..41901ba9d3a2c 100644
+--- a/arch/x86/crypto/aesni-intel_glue.c
++++ b/arch/x86/crypto/aesni-intel_glue.c
+@@ -866,7 +866,7 @@ static int xts_crypt(struct skcipher_request *req, bool encrypt)
+ req = &subreq;
+
+ err = skcipher_walk_virt(&walk, req, false);
+- if (err)
++ if (!walk.nbytes)
+ return err;
+ } else {
+ tail = 0;
+@@ -1107,7 +1107,7 @@ static struct aead_alg aesni_aeads[] = { {
+ .cra_flags = CRYPTO_ALG_INTERNAL,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
+- .cra_alignmask = AESNI_ALIGN - 1,
++ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ }, {
+@@ -1124,7 +1124,7 @@ static struct aead_alg aesni_aeads[] = { {
+ .cra_flags = CRYPTO_ALG_INTERNAL,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
+- .cra_alignmask = AESNI_ALIGN - 1,
++ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ } };
+diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/crypto/blake2s-core.S
+index 2ca79974f8198..b50b35ff1fdba 100644
+--- a/arch/x86/crypto/blake2s-core.S
++++ b/arch/x86/crypto/blake2s-core.S
+@@ -171,7 +171,7 @@ SYM_FUNC_START(blake2s_compress_ssse3)
+ movdqu %xmm1,0x10(%rdi)
+ movdqu %xmm14,0x20(%rdi)
+ .Lendofloop:
+- ret
++ RET
+ SYM_FUNC_END(blake2s_compress_ssse3)
+
+ #ifdef CONFIG_AS_AVX512
+@@ -251,6 +251,6 @@ SYM_FUNC_START(blake2s_compress_avx512)
+ vmovdqu %xmm1,0x10(%rdi)
+ vmovdqu %xmm4,0x20(%rdi)
+ vzeroupper
+- retq
++ RET
+ SYM_FUNC_END(blake2s_compress_avx512)
+ #endif /* CONFIG_AS_AVX512 */
+diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c
+index a40365ab301ee..69853c13e8fb0 100644
+--- a/arch/x86/crypto/blake2s-glue.c
++++ b/arch/x86/crypto/blake2s-glue.c
+@@ -5,7 +5,6 @@
+
+ #include <crypto/internal/blake2s.h>
+ #include <crypto/internal/simd.h>
+-#include <crypto/internal/hash.h>
+
+ #include <linux/types.h>
+ #include <linux/jump_label.h>
+@@ -28,9 +27,8 @@ asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
+ static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
+ static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
+
+-void blake2s_compress_arch(struct blake2s_state *state,
+- const u8 *block, size_t nblocks,
+- const u32 inc)
++void blake2s_compress(struct blake2s_state *state, const u8 *block,
++ size_t nblocks, const u32 inc)
+ {
+ /* SIMD disables preemption, so relax after processing each page. */
+ BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
+@@ -56,49 +54,12 @@ void blake2s_compress_arch(struct blake2s_state *state,
+ block += blocks * BLAKE2S_BLOCK_SIZE;
+ } while (nblocks);
+ }
+-EXPORT_SYMBOL(blake2s_compress_arch);
+-
+-static int crypto_blake2s_update_x86(struct shash_desc *desc,
+- const u8 *in, unsigned int inlen)
+-{
+- return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch);
+-}
+-
+-static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out)
+-{
+- return crypto_blake2s_final(desc, out, blake2s_compress_arch);
+-}
+-
+-#define BLAKE2S_ALG(name, driver_name, digest_size) \
+- { \
+- .base.cra_name = name, \
+- .base.cra_driver_name = driver_name, \
+- .base.cra_priority = 200, \
+- .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
+- .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \
+- .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \
+- .base.cra_module = THIS_MODULE, \
+- .digestsize = digest_size, \
+- .setkey = crypto_blake2s_setkey, \
+- .init = crypto_blake2s_init, \
+- .update = crypto_blake2s_update_x86, \
+- .final = crypto_blake2s_final_x86, \
+- .descsize = sizeof(struct blake2s_state), \
+- }
+-
+-static struct shash_alg blake2s_algs[] = {
+- BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE),
+- BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE),
+-};
++EXPORT_SYMBOL(blake2s_compress);
+
+ static int __init blake2s_mod_init(void)
+ {
+- if (!boot_cpu_has(X86_FEATURE_SSSE3))
+- return 0;
+-
+- static_branch_enable(&blake2s_use_ssse3);
++ if (boot_cpu_has(X86_FEATURE_SSSE3))
++ static_branch_enable(&blake2s_use_ssse3);
+
+ if (IS_ENABLED(CONFIG_AS_AVX512) &&
+ boot_cpu_has(X86_FEATURE_AVX) &&
+@@ -109,26 +70,9 @@ static int __init blake2s_mod_init(void)
+ XFEATURE_MASK_AVX512, NULL))
+ static_branch_enable(&blake2s_use_avx512);
+
+- return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
+- crypto_register_shashes(blake2s_algs,
+- ARRAY_SIZE(blake2s_algs)) : 0;
+-}
+-
+-static void __exit blake2s_mod_exit(void)
+-{
+- if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3))
+- crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs));
++ return 0;
+ }
+
+ module_init(blake2s_mod_init);
+-module_exit(blake2s_mod_exit);
+
+-MODULE_ALIAS_CRYPTO("blake2s-128");
+-MODULE_ALIAS_CRYPTO("blake2s-128-x86");
+-MODULE_ALIAS_CRYPTO("blake2s-160");
+-MODULE_ALIAS_CRYPTO("blake2s-160-x86");
+-MODULE_ALIAS_CRYPTO("blake2s-224");
+-MODULE_ALIAS_CRYPTO("blake2s-224-x86");
+-MODULE_ALIAS_CRYPTO("blake2s-256");
+-MODULE_ALIAS_CRYPTO("blake2s-256-x86");
+ MODULE_LICENSE("GPL v2");
+diff --git a/arch/x86/crypto/blake2s-shash.c b/arch/x86/crypto/blake2s-shash.c
+new file mode 100644
+index 0000000000000..59ae28abe35cc
+--- /dev/null
++++ b/arch/x86/crypto/blake2s-shash.c
+@@ -0,0 +1,77 @@
++// SPDX-License-Identifier: GPL-2.0 OR MIT
++/*
++ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
++ */
++
++#include <crypto/internal/blake2s.h>
++#include <crypto/internal/simd.h>
++#include <crypto/internal/hash.h>
++
++#include <linux/types.h>
++#include <linux/kernel.h>
++#include <linux/module.h>
++#include <linux/sizes.h>
++
++#include <asm/cpufeature.h>
++#include <asm/processor.h>
++
++static int crypto_blake2s_update_x86(struct shash_desc *desc,
++ const u8 *in, unsigned int inlen)
++{
++ return crypto_blake2s_update(desc, in, inlen, false);
++}
++
++static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out)
++{
++ return crypto_blake2s_final(desc, out, false);
++}
++
++#define BLAKE2S_ALG(name, driver_name, digest_size) \
++ { \
++ .base.cra_name = name, \
++ .base.cra_driver_name = driver_name, \
++ .base.cra_priority = 200, \
++ .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
++ .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \
++ .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \
++ .base.cra_module = THIS_MODULE, \
++ .digestsize = digest_size, \
++ .setkey = crypto_blake2s_setkey, \
++ .init = crypto_blake2s_init, \
++ .update = crypto_blake2s_update_x86, \
++ .final = crypto_blake2s_final_x86, \
++ .descsize = sizeof(struct blake2s_state), \
++ }
++
++static struct shash_alg blake2s_algs[] = {
++ BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE),
++ BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE),
++};
++
++static int __init blake2s_mod_init(void)
++{
++ if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3))
++ return crypto_register_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs));
++ return 0;
++}
++
++static void __exit blake2s_mod_exit(void)
++{
++ if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3))
++ crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs));
++}
++
++module_init(blake2s_mod_init);
++module_exit(blake2s_mod_exit);
++
++MODULE_ALIAS_CRYPTO("blake2s-128");
++MODULE_ALIAS_CRYPTO("blake2s-128-x86");
++MODULE_ALIAS_CRYPTO("blake2s-160");
++MODULE_ALIAS_CRYPTO("blake2s-160-x86");
++MODULE_ALIAS_CRYPTO("blake2s-224");
++MODULE_ALIAS_CRYPTO("blake2s-224-x86");
++MODULE_ALIAS_CRYPTO("blake2s-256");
++MODULE_ALIAS_CRYPTO("blake2s-256-x86");
++MODULE_LICENSE("GPL v2");
+diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
+index 4222ac6d65848..802d715826891 100644
+--- a/arch/x86/crypto/blowfish-x86_64-asm_64.S
++++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S
+@@ -135,10 +135,10 @@ SYM_FUNC_START(__blowfish_enc_blk)
+ jnz .L__enc_xor;
+
+ write_block();
+- ret;
++ RET;
+ .L__enc_xor:
+ xor_block();
+- ret;
++ RET;
+ SYM_FUNC_END(__blowfish_enc_blk)
+
+ SYM_FUNC_START(blowfish_dec_blk)
+@@ -170,7 +170,7 @@ SYM_FUNC_START(blowfish_dec_blk)
+
+ movq %r11, %r12;
+
+- ret;
++ RET;
+ SYM_FUNC_END(blowfish_dec_blk)
+
+ /**********************************************************************
+@@ -322,14 +322,14 @@ SYM_FUNC_START(__blowfish_enc_blk_4way)
+
+ popq %rbx;
+ popq %r12;
+- ret;
++ RET;
+
+ .L__enc_xor4:
+ xor_block4();
+
+ popq %rbx;
+ popq %r12;
+- ret;
++ RET;
+ SYM_FUNC_END(__blowfish_enc_blk_4way)
+
+ SYM_FUNC_START(blowfish_dec_blk_4way)
+@@ -364,5 +364,5 @@ SYM_FUNC_START(blowfish_dec_blk_4way)
+ popq %rbx;
+ popq %r12;
+
+- ret;
++ RET;
+ SYM_FUNC_END(blowfish_dec_blk_4way)
+diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+index e2a0e0f4bf9d8..2e1658ddbe1a9 100644
+--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
++++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+@@ -192,7 +192,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c
+ roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+ %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
+ %rcx, (%r9));
+- ret;
++ RET;
+ SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
+
+ .align 8
+@@ -200,7 +200,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a
+ roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3,
+ %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
+ %rax, (%r9));
+- ret;
++ RET;
+ SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
+
+ /*
+@@ -778,7 +778,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk16)
+ %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
+
+ FRAME_END
+- ret;
++ RET;
+
+ .align 8
+ .Lenc_max32:
+@@ -865,7 +865,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk16)
+ %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
+
+ FRAME_END
+- ret;
++ RET;
+
+ .align 8
+ .Ldec_max32:
+@@ -906,7 +906,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way)
+ %xmm8, %rsi);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_ecb_enc_16way)
+
+ SYM_FUNC_START(camellia_ecb_dec_16way)
+@@ -936,7 +936,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way)
+ %xmm8, %rsi);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_ecb_dec_16way)
+
+ SYM_FUNC_START(camellia_cbc_dec_16way)
+@@ -987,5 +987,5 @@ SYM_FUNC_START(camellia_cbc_dec_16way)
+ %xmm8, %rsi);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_cbc_dec_16way)
+diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+index 706f70829a07e..0e4e9abbf4de3 100644
+--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
++++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+@@ -226,7 +226,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c
+ roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
+ %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
+ %rcx, (%r9));
+- ret;
++ RET;
+ SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
+
+ .align 8
+@@ -234,7 +234,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a
+ roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
+ %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
+ %rax, (%r9));
+- ret;
++ RET;
+ SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
+
+ /*
+@@ -814,7 +814,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk32)
+ %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
+
+ FRAME_END
+- ret;
++ RET;
+
+ .align 8
+ .Lenc_max32:
+@@ -901,7 +901,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk32)
+ %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
+
+ FRAME_END
+- ret;
++ RET;
+
+ .align 8
+ .Ldec_max32:
+@@ -946,7 +946,7 @@ SYM_FUNC_START(camellia_ecb_enc_32way)
+ vzeroupper;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_ecb_enc_32way)
+
+ SYM_FUNC_START(camellia_ecb_dec_32way)
+@@ -980,7 +980,7 @@ SYM_FUNC_START(camellia_ecb_dec_32way)
+ vzeroupper;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_ecb_dec_32way)
+
+ SYM_FUNC_START(camellia_cbc_dec_32way)
+@@ -1047,5 +1047,5 @@ SYM_FUNC_START(camellia_cbc_dec_32way)
+
+ addq $(16 * 32), %rsp;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_cbc_dec_32way)
+diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S
+index 1372e64088507..347c059f59403 100644
+--- a/arch/x86/crypto/camellia-x86_64-asm_64.S
++++ b/arch/x86/crypto/camellia-x86_64-asm_64.S
+@@ -213,13 +213,13 @@ SYM_FUNC_START(__camellia_enc_blk)
+ enc_outunpack(mov, RT1);
+
+ movq RR12, %r12;
+- ret;
++ RET;
+
+ .L__enc_xor:
+ enc_outunpack(xor, RT1);
+
+ movq RR12, %r12;
+- ret;
++ RET;
+ SYM_FUNC_END(__camellia_enc_blk)
+
+ SYM_FUNC_START(camellia_dec_blk)
+@@ -257,7 +257,7 @@ SYM_FUNC_START(camellia_dec_blk)
+ dec_outunpack();
+
+ movq RR12, %r12;
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_dec_blk)
+
+ /**********************************************************************
+@@ -448,14 +448,14 @@ SYM_FUNC_START(__camellia_enc_blk_2way)
+
+ movq RR12, %r12;
+ popq %rbx;
+- ret;
++ RET;
+
+ .L__enc2_xor:
+ enc_outunpack2(xor, RT2);
+
+ movq RR12, %r12;
+ popq %rbx;
+- ret;
++ RET;
+ SYM_FUNC_END(__camellia_enc_blk_2way)
+
+ SYM_FUNC_START(camellia_dec_blk_2way)
+@@ -495,5 +495,5 @@ SYM_FUNC_START(camellia_dec_blk_2way)
+
+ movq RR12, %r12;
+ movq RXOR, %rbx;
+- ret;
++ RET;
+ SYM_FUNC_END(camellia_dec_blk_2way)
+diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+index 8a6181b08b590..b258af420c92c 100644
+--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
++++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+@@ -279,7 +279,7 @@ SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
+ outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
+ outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__cast5_enc_blk16)
+
+ .align 16
+@@ -352,7 +352,7 @@ SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
+ outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
+ outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
+
+- ret;
++ RET;
+
+ .L__skip_dec:
+ vpsrldq $4, RKR, RKR;
+@@ -393,7 +393,7 @@ SYM_FUNC_START(cast5_ecb_enc_16way)
+
+ popq %r15;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast5_ecb_enc_16way)
+
+ SYM_FUNC_START(cast5_ecb_dec_16way)
+@@ -431,7 +431,7 @@ SYM_FUNC_START(cast5_ecb_dec_16way)
+
+ popq %r15;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast5_ecb_dec_16way)
+
+ SYM_FUNC_START(cast5_cbc_dec_16way)
+@@ -483,7 +483,7 @@ SYM_FUNC_START(cast5_cbc_dec_16way)
+ popq %r15;
+ popq %r12;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast5_cbc_dec_16way)
+
+ SYM_FUNC_START(cast5_ctr_16way)
+@@ -559,5 +559,5 @@ SYM_FUNC_START(cast5_ctr_16way)
+ popq %r15;
+ popq %r12;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast5_ctr_16way)
+diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+index fbddcecc3e3fc..82b716fd5dbac 100644
+--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
++++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+@@ -289,7 +289,7 @@ SYM_FUNC_START_LOCAL(__cast6_enc_blk8)
+ outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+ outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__cast6_enc_blk8)
+
+ .align 8
+@@ -336,7 +336,7 @@ SYM_FUNC_START_LOCAL(__cast6_dec_blk8)
+ outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+ outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__cast6_dec_blk8)
+
+ SYM_FUNC_START(cast6_ecb_enc_8way)
+@@ -359,7 +359,7 @@ SYM_FUNC_START(cast6_ecb_enc_8way)
+
+ popq %r15;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast6_ecb_enc_8way)
+
+ SYM_FUNC_START(cast6_ecb_dec_8way)
+@@ -382,7 +382,7 @@ SYM_FUNC_START(cast6_ecb_dec_8way)
+
+ popq %r15;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast6_ecb_dec_8way)
+
+ SYM_FUNC_START(cast6_cbc_dec_8way)
+@@ -408,5 +408,5 @@ SYM_FUNC_START(cast6_cbc_dec_8way)
+ popq %r15;
+ popq %r12;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(cast6_cbc_dec_8way)
+diff --git a/arch/x86/crypto/chacha-avx2-x86_64.S b/arch/x86/crypto/chacha-avx2-x86_64.S
+index ee9a40ab41093..f3d8fc0182493 100644
+--- a/arch/x86/crypto/chacha-avx2-x86_64.S
++++ b/arch/x86/crypto/chacha-avx2-x86_64.S
+@@ -193,7 +193,7 @@ SYM_FUNC_START(chacha_2block_xor_avx2)
+
+ .Ldone2:
+ vzeroupper
+- ret
++ RET
+
+ .Lxorpart2:
+ # xor remaining bytes from partial register into output
+@@ -498,7 +498,7 @@ SYM_FUNC_START(chacha_4block_xor_avx2)
+
+ .Ldone4:
+ vzeroupper
+- ret
++ RET
+
+ .Lxorpart4:
+ # xor remaining bytes from partial register into output
+@@ -992,7 +992,7 @@ SYM_FUNC_START(chacha_8block_xor_avx2)
+ .Ldone8:
+ vzeroupper
+ lea -8(%r10),%rsp
+- ret
++ RET
+
+ .Lxorpart8:
+ # xor remaining bytes from partial register into output
+diff --git a/arch/x86/crypto/chacha-avx512vl-x86_64.S b/arch/x86/crypto/chacha-avx512vl-x86_64.S
+index bb193fde123a0..259383e1ad440 100644
+--- a/arch/x86/crypto/chacha-avx512vl-x86_64.S
++++ b/arch/x86/crypto/chacha-avx512vl-x86_64.S
+@@ -166,13 +166,13 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl)
+
+ .Ldone2:
+ vzeroupper
+- ret
++ RET
+
+ .Lxorpart2:
+ # xor remaining bytes from partial register into output
+ mov %rcx,%rax
+ and $0xf,%rcx
+- jz .Ldone8
++ jz .Ldone2
+ mov %rax,%r9
+ and $~0xf,%r9
+
+@@ -432,13 +432,13 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl)
+
+ .Ldone4:
+ vzeroupper
+- ret
++ RET
+
+ .Lxorpart4:
+ # xor remaining bytes from partial register into output
+ mov %rcx,%rax
+ and $0xf,%rcx
+- jz .Ldone8
++ jz .Ldone4
+ mov %rax,%r9
+ and $~0xf,%r9
+
+@@ -812,7 +812,7 @@ SYM_FUNC_START(chacha_8block_xor_avx512vl)
+
+ .Ldone8:
+ vzeroupper
+- ret
++ RET
+
+ .Lxorpart8:
+ # xor remaining bytes from partial register into output
+diff --git a/arch/x86/crypto/chacha-ssse3-x86_64.S b/arch/x86/crypto/chacha-ssse3-x86_64.S
+index ca1788bfee162..7111949cd5b99 100644
+--- a/arch/x86/crypto/chacha-ssse3-x86_64.S
++++ b/arch/x86/crypto/chacha-ssse3-x86_64.S
+@@ -108,7 +108,7 @@ SYM_FUNC_START_LOCAL(chacha_permute)
+ sub $2,%r8d
+ jnz .Ldoubleround
+
+- ret
++ RET
+ SYM_FUNC_END(chacha_permute)
+
+ SYM_FUNC_START(chacha_block_xor_ssse3)
+@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_block_xor_ssse3)
+
+ .Ldone:
+ FRAME_END
+- ret
++ RET
+
+ .Lxorpart:
+ # xor remaining bytes from partial register into output
+@@ -217,7 +217,7 @@ SYM_FUNC_START(hchacha_block_ssse3)
+ movdqu %xmm3,0x10(%rsi)
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(hchacha_block_ssse3)
+
+ SYM_FUNC_START(chacha_4block_xor_ssse3)
+@@ -762,7 +762,7 @@ SYM_FUNC_START(chacha_4block_xor_ssse3)
+
+ .Ldone4:
+ lea -8(%r10),%rsp
+- ret
++ RET
+
+ .Lxorpart4:
+ # xor remaining bytes from partial register into output
+diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S
+index 6e7d4c4d32081..c392a6edbfff6 100644
+--- a/arch/x86/crypto/crc32-pclmul_asm.S
++++ b/arch/x86/crypto/crc32-pclmul_asm.S
+@@ -236,5 +236,5 @@ fold_64:
+ pxor %xmm2, %xmm1
+ pextrd $0x01, %xmm1, %eax
+
+- ret
++ RET
+ SYM_FUNC_END(crc32_pclmul_le_16)
+diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+index ac1f303eed0f4..80c0d22fc42c6 100644
+--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
++++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+@@ -306,7 +306,7 @@ do_return:
+ popq %rsi
+ popq %rdi
+ popq %rbx
+- ret
++ RET
+ SYM_FUNC_END(crc_pcl)
+
+ .section .rodata, "a", @progbits
+diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S
+index b2533d63030e5..721474abfb719 100644
+--- a/arch/x86/crypto/crct10dif-pcl-asm_64.S
++++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S
+@@ -257,7 +257,7 @@ SYM_FUNC_START(crc_t10dif_pcl)
+ # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0.
+
+ pextrw $0, %xmm0, %eax
+- ret
++ RET
+
+ .align 16
+ .Lless_than_256_bytes:
+diff --git a/arch/x86/crypto/des3_ede-asm_64.S b/arch/x86/crypto/des3_ede-asm_64.S
+index fac0fdc3f25da..f4c760f4cade6 100644
+--- a/arch/x86/crypto/des3_ede-asm_64.S
++++ b/arch/x86/crypto/des3_ede-asm_64.S
+@@ -243,7 +243,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk)
+ popq %r12;
+ popq %rbx;
+
+- ret;
++ RET;
+ SYM_FUNC_END(des3_ede_x86_64_crypt_blk)
+
+ /***********************************************************************
+@@ -528,7 +528,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk_3way)
+ popq %r12;
+ popq %rbx;
+
+- ret;
++ RET;
+ SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way)
+
+ .section .rodata, "a", @progbits
+diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
+index 99ac25e18e098..2bf8718999209 100644
+--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
++++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
+@@ -85,7 +85,7 @@ SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble)
+ psrlq $1, T2
+ pxor T2, T1
+ pxor T1, DATA
+- ret
++ RET
+ SYM_FUNC_END(__clmul_gf128mul_ble)
+
+ /* void clmul_ghash_mul(char *dst, const u128 *shash) */
+@@ -99,7 +99,7 @@ SYM_FUNC_START(clmul_ghash_mul)
+ pshufb BSWAP, DATA
+ movups DATA, (%rdi)
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(clmul_ghash_mul)
+
+ /*
+@@ -128,5 +128,5 @@ SYM_FUNC_START(clmul_ghash_update)
+ movups DATA, (%rdi)
+ .Lupdate_just_ret:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(clmul_ghash_update)
+diff --git a/arch/x86/crypto/nh-avx2-x86_64.S b/arch/x86/crypto/nh-avx2-x86_64.S
+index b22c7b9362726..6a0b15e7196a8 100644
+--- a/arch/x86/crypto/nh-avx2-x86_64.S
++++ b/arch/x86/crypto/nh-avx2-x86_64.S
+@@ -153,5 +153,5 @@ SYM_FUNC_START(nh_avx2)
+ vpaddq T1, T0, T0
+ vpaddq T4, T0, T0
+ vmovdqu T0, (HASH)
+- ret
++ RET
+ SYM_FUNC_END(nh_avx2)
+diff --git a/arch/x86/crypto/nh-sse2-x86_64.S b/arch/x86/crypto/nh-sse2-x86_64.S
+index d7ae22dd66839..34c567bbcb4fa 100644
+--- a/arch/x86/crypto/nh-sse2-x86_64.S
++++ b/arch/x86/crypto/nh-sse2-x86_64.S
+@@ -119,5 +119,5 @@ SYM_FUNC_START(nh_sse2)
+ paddq PASS2_SUMS, T1
+ movdqu T0, 0x00(HASH)
+ movdqu T1, 0x10(HASH)
+- ret
++ RET
+ SYM_FUNC_END(nh_sse2)
+diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl
+index 71fae5a09e56d..2077ce7a56479 100644
+--- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl
++++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl
+@@ -297,7 +297,7 @@ ___
+ $code.=<<___;
+ mov \$1,%eax
+ .Lno_key:
+- ret
++ RET
+ ___
+ &end_function("poly1305_init_x86_64");
+
+@@ -373,7 +373,7 @@ $code.=<<___;
+ .cfi_adjust_cfa_offset -48
+ .Lno_data:
+ .Lblocks_epilogue:
+- ret
++ RET
+ .cfi_endproc
+ ___
+ &end_function("poly1305_blocks_x86_64");
+@@ -399,7 +399,7 @@ $code.=<<___;
+ mov %rax,0($mac) # write result
+ mov %rcx,8($mac)
+
+- ret
++ RET
+ ___
+ &end_function("poly1305_emit_x86_64");
+ if ($avx) {
+@@ -429,7 +429,7 @@ ___
+ &poly1305_iteration();
+ $code.=<<___;
+ pop $ctx
+- ret
++ RET
+ .size __poly1305_block,.-__poly1305_block
+
+ .type __poly1305_init_avx,\@abi-omnipotent
+@@ -594,7 +594,7 @@ __poly1305_init_avx:
+
+ lea -48-64($ctx),$ctx # size [de-]optimization
+ pop %rbp
+- ret
++ RET
+ .size __poly1305_init_avx,.-__poly1305_init_avx
+ ___
+
+@@ -747,7 +747,7 @@ $code.=<<___;
+ .cfi_restore %rbp
+ .Lno_data_avx:
+ .Lblocks_avx_epilogue:
+- ret
++ RET
+ .cfi_endproc
+
+ .align 32
+@@ -1452,7 +1452,7 @@ $code.=<<___ if (!$win64);
+ ___
+ $code.=<<___;
+ vzeroupper
+- ret
++ RET
+ .cfi_endproc
+ ___
+ &end_function("poly1305_blocks_avx");
+@@ -1508,7 +1508,7 @@ $code.=<<___;
+ mov %rax,0($mac) # write result
+ mov %rcx,8($mac)
+
+- ret
++ RET
+ ___
+ &end_function("poly1305_emit_avx");
+
+@@ -1675,7 +1675,7 @@ $code.=<<___;
+ .cfi_restore %rbp
+ .Lno_data_avx2$suffix:
+ .Lblocks_avx2_epilogue$suffix:
+- ret
++ RET
+ .cfi_endproc
+
+ .align 32
+@@ -2201,7 +2201,7 @@ $code.=<<___ if (!$win64);
+ ___
+ $code.=<<___;
+ vzeroupper
+- ret
++ RET
+ .cfi_endproc
+ ___
+ if($avx > 2 && $avx512) {
+@@ -2792,7 +2792,7 @@ $code.=<<___ if (!$win64);
+ .cfi_def_cfa_register %rsp
+ ___
+ $code.=<<___;
+- ret
++ RET
+ .cfi_endproc
+ ___
+
+@@ -2893,7 +2893,7 @@ $code.=<<___ if ($flavour =~ /elf32/);
+ ___
+ $code.=<<___;
+ mov \$1,%eax
+- ret
++ RET
+ .size poly1305_init_base2_44,.-poly1305_init_base2_44
+ ___
+ {
+@@ -3010,7 +3010,7 @@ poly1305_blocks_vpmadd52:
+ jnz .Lblocks_vpmadd52_4x
+
+ .Lno_data_vpmadd52:
+- ret
++ RET
+ .size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52
+ ___
+ }
+@@ -3451,7 +3451,7 @@ poly1305_blocks_vpmadd52_4x:
+ vzeroall
+
+ .Lno_data_vpmadd52_4x:
+- ret
++ RET
+ .size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x
+ ___
+ }
+@@ -3824,7 +3824,7 @@ $code.=<<___;
+ vzeroall
+
+ .Lno_data_vpmadd52_8x:
+- ret
++ RET
+ .size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x
+ ___
+ }
+@@ -3861,7 +3861,7 @@ poly1305_emit_base2_44:
+ mov %rax,0($mac) # write result
+ mov %rcx,8($mac)
+
+- ret
++ RET
+ .size poly1305_emit_base2_44,.-poly1305_emit_base2_44
+ ___
+ } } }
+@@ -3916,7 +3916,7 @@ xor128_encrypt_n_pad:
+
+ .Ldone_enc:
+ mov $otp,%rax
+- ret
++ RET
+ .size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad
+
+ .globl xor128_decrypt_n_pad
+@@ -3967,7 +3967,7 @@ xor128_decrypt_n_pad:
+
+ .Ldone_dec:
+ mov $otp,%rax
+- ret
++ RET
+ .size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad
+ ___
+ }
+@@ -4109,7 +4109,7 @@ avx_handler:
+ pop %rbx
+ pop %rdi
+ pop %rsi
+- ret
++ RET
+ .size avx_handler,.-avx_handler
+
+ .section .pdata
+diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+index b7ee24df7fbae..82f2313f512b8 100644
+--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
++++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx)
+ write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_enc_blk8_avx)
+
+ .align 8
+@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx)
+ write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
+ write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_dec_blk8_avx)
+
+ SYM_FUNC_START(serpent_ecb_enc_8way_avx)
+@@ -673,7 +673,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx)
+ store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_ecb_enc_8way_avx)
+
+ SYM_FUNC_START(serpent_ecb_dec_8way_avx)
+@@ -691,7 +691,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx)
+ store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_ecb_dec_8way_avx)
+
+ SYM_FUNC_START(serpent_cbc_dec_8way_avx)
+@@ -709,5 +709,5 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx)
+ store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_cbc_dec_8way_avx)
+diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S
+index 9161b6e441f31..8ea34c9b93160 100644
+--- a/arch/x86/crypto/serpent-avx2-asm_64.S
++++ b/arch/x86/crypto/serpent-avx2-asm_64.S
+@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk16)
+ write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_enc_blk16)
+
+ .align 8
+@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk16)
+ write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
+ write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_dec_blk16)
+
+ SYM_FUNC_START(serpent_ecb_enc_16way)
+@@ -677,7 +677,7 @@ SYM_FUNC_START(serpent_ecb_enc_16way)
+ vzeroupper;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_ecb_enc_16way)
+
+ SYM_FUNC_START(serpent_ecb_dec_16way)
+@@ -699,7 +699,7 @@ SYM_FUNC_START(serpent_ecb_dec_16way)
+ vzeroupper;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_ecb_dec_16way)
+
+ SYM_FUNC_START(serpent_cbc_dec_16way)
+@@ -722,5 +722,5 @@ SYM_FUNC_START(serpent_cbc_dec_16way)
+ vzeroupper;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_cbc_dec_16way)
+diff --git a/arch/x86/crypto/serpent-sse2-i586-asm_32.S b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
+index 6379b99cb722e..8ccb03ad7cef5 100644
+--- a/arch/x86/crypto/serpent-sse2-i586-asm_32.S
++++ b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
+@@ -553,12 +553,12 @@ SYM_FUNC_START(__serpent_enc_blk_4way)
+
+ write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
+
+- ret;
++ RET;
+
+ .L__enc_xor4:
+ xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_enc_blk_4way)
+
+ SYM_FUNC_START(serpent_dec_blk_4way)
+@@ -612,5 +612,5 @@ SYM_FUNC_START(serpent_dec_blk_4way)
+ movl arg_dst(%esp), %eax;
+ write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA);
+
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_dec_blk_4way)
+diff --git a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
+index efb6dc17dc907..e0998a011d1dd 100644
+--- a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
++++ b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
+@@ -675,13 +675,13 @@ SYM_FUNC_START(__serpent_enc_blk_8way)
+ write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+
+ .L__enc_xor8:
+ xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__serpent_enc_blk_8way)
+
+ SYM_FUNC_START(serpent_dec_blk_8way)
+@@ -735,5 +735,5 @@ SYM_FUNC_START(serpent_dec_blk_8way)
+ write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
+ write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(serpent_dec_blk_8way)
+diff --git a/arch/x86/crypto/sha1_avx2_x86_64_asm.S b/arch/x86/crypto/sha1_avx2_x86_64_asm.S
+index 5eed620f46765..a96b2fd26dab4 100644
+--- a/arch/x86/crypto/sha1_avx2_x86_64_asm.S
++++ b/arch/x86/crypto/sha1_avx2_x86_64_asm.S
+@@ -674,7 +674,7 @@ _loop3:
+ pop %r12
+ pop %rbx
+
+- ret
++ RET
+
+ SYM_FUNC_END(\name)
+ .endm
+diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S
+index 5d8415f482bd7..2f94ec0e763bf 100644
+--- a/arch/x86/crypto/sha1_ni_asm.S
++++ b/arch/x86/crypto/sha1_ni_asm.S
+@@ -290,7 +290,7 @@ SYM_FUNC_START(sha1_ni_transform)
+ mov %rbp, %rsp
+ pop %rbp
+
+- ret
++ RET
+ SYM_FUNC_END(sha1_ni_transform)
+
+ .section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
+diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
+index d25668d2a1e92..263f916362e02 100644
+--- a/arch/x86/crypto/sha1_ssse3_asm.S
++++ b/arch/x86/crypto/sha1_ssse3_asm.S
+@@ -99,7 +99,7 @@
+ pop %rbp
+ pop %r12
+ pop %rbx
+- ret
++ RET
+
+ SYM_FUNC_END(\name)
+ .endm
+diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S
+index 4739cd31b9db1..3baa1ec390974 100644
+--- a/arch/x86/crypto/sha256-avx-asm.S
++++ b/arch/x86/crypto/sha256-avx-asm.S
+@@ -458,7 +458,7 @@ done_hash:
+ popq %r13
+ popq %r12
+ popq %rbx
+- ret
++ RET
+ SYM_FUNC_END(sha256_transform_avx)
+
+ .section .rodata.cst256.K256, "aM", @progbits, 256
+diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S
+index 4087f7432a7e8..9bcdbc47b8b4b 100644
+--- a/arch/x86/crypto/sha256-avx2-asm.S
++++ b/arch/x86/crypto/sha256-avx2-asm.S
+@@ -710,7 +710,7 @@ done_hash:
+ popq %r13
+ popq %r12
+ popq %rbx
+- ret
++ RET
+ SYM_FUNC_END(sha256_transform_rorx)
+
+ .section .rodata.cst512.K256, "aM", @progbits, 512
+diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S
+index ddfa863b4ee33..c4a5db612c327 100644
+--- a/arch/x86/crypto/sha256-ssse3-asm.S
++++ b/arch/x86/crypto/sha256-ssse3-asm.S
+@@ -472,7 +472,7 @@ done_hash:
+ popq %r12
+ popq %rbx
+
+- ret
++ RET
+ SYM_FUNC_END(sha256_transform_ssse3)
+
+ .section .rodata.cst256.K256, "aM", @progbits, 256
+diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/crypto/sha256_ni_asm.S
+index 7abade04a3a38..94d50dd27cb53 100644
+--- a/arch/x86/crypto/sha256_ni_asm.S
++++ b/arch/x86/crypto/sha256_ni_asm.S
+@@ -326,7 +326,7 @@ SYM_FUNC_START(sha256_ni_transform)
+
+ .Ldone_hash:
+
+- ret
++ RET
+ SYM_FUNC_END(sha256_ni_transform)
+
+ .section .rodata.cst256.K256, "aM", @progbits, 256
+diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S
+index 3d8f0fd4eea87..1fefe6dd3a9e2 100644
+--- a/arch/x86/crypto/sha512-avx-asm.S
++++ b/arch/x86/crypto/sha512-avx-asm.S
+@@ -361,7 +361,7 @@ updateblock:
+ pop %rbx
+
+ nowork:
+- ret
++ RET
+ SYM_FUNC_END(sha512_transform_avx)
+
+ ########################################################################
+diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S
+index 072cb0f0deae3..5cdaab7d69015 100644
+--- a/arch/x86/crypto/sha512-avx2-asm.S
++++ b/arch/x86/crypto/sha512-avx2-asm.S
+@@ -679,7 +679,7 @@ done_hash:
+ pop %r12
+ pop %rbx
+
+- ret
++ RET
+ SYM_FUNC_END(sha512_transform_rorx)
+
+ ########################################################################
+diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S
+index bd51c9070bedc..b84c22e06c5f7 100644
+--- a/arch/x86/crypto/sha512-ssse3-asm.S
++++ b/arch/x86/crypto/sha512-ssse3-asm.S
+@@ -363,7 +363,7 @@ updateblock:
+ pop %rbx
+
+ nowork:
+- ret
++ RET
+ SYM_FUNC_END(sha512_transform_ssse3)
+
+ ########################################################################
+diff --git a/arch/x86/crypto/sm4-aesni-avx-asm_64.S b/arch/x86/crypto/sm4-aesni-avx-asm_64.S
+index 1cc72b4804fab..4767ab61ff489 100644
+--- a/arch/x86/crypto/sm4-aesni-avx-asm_64.S
++++ b/arch/x86/crypto/sm4-aesni-avx-asm_64.S
+@@ -246,7 +246,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt4)
+ .Lblk4_store_output_done:
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx_crypt4)
+
+ .align 8
+@@ -356,7 +356,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk8)
+ vpshufb RTMP2, RB3, RB3;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(__sm4_crypt_blk8)
+
+ /*
+@@ -412,7 +412,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt8)
+ .Lblk8_store_output_done:
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx_crypt8)
+
+ /*
+@@ -487,7 +487,7 @@ SYM_FUNC_START(sm4_aesni_avx_ctr_enc_blk8)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8)
+
+ /*
+@@ -537,7 +537,7 @@ SYM_FUNC_START(sm4_aesni_avx_cbc_dec_blk8)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8)
+
+ /*
+@@ -590,5 +590,5 @@ SYM_FUNC_START(sm4_aesni_avx_cfb_dec_blk8)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8)
+diff --git a/arch/x86/crypto/sm4-aesni-avx2-asm_64.S b/arch/x86/crypto/sm4-aesni-avx2-asm_64.S
+index 9c5d3f3ad45a9..4732fe8bb65b6 100644
+--- a/arch/x86/crypto/sm4-aesni-avx2-asm_64.S
++++ b/arch/x86/crypto/sm4-aesni-avx2-asm_64.S
+@@ -268,7 +268,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk16)
+ vpshufb RTMP2, RB3, RB3;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(__sm4_crypt_blk16)
+
+ #define inc_le128(x, minus_one, tmp) \
+@@ -387,7 +387,7 @@ SYM_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16)
+
+ /*
+@@ -441,7 +441,7 @@ SYM_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16)
+
+ /*
+@@ -497,5 +497,5 @@ SYM_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16)
+
+ vzeroall;
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
+diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+index 37e63b3c664eb..31f9b2ec3857d 100644
+--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
++++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+@@ -267,7 +267,7 @@ SYM_FUNC_START_LOCAL(__twofish_enc_blk8)
+ outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
+ outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__twofish_enc_blk8)
+
+ .align 8
+@@ -307,7 +307,7 @@ SYM_FUNC_START_LOCAL(__twofish_dec_blk8)
+ outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
+ outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
+
+- ret;
++ RET;
+ SYM_FUNC_END(__twofish_dec_blk8)
+
+ SYM_FUNC_START(twofish_ecb_enc_8way)
+@@ -327,7 +327,7 @@ SYM_FUNC_START(twofish_ecb_enc_8way)
+ store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(twofish_ecb_enc_8way)
+
+ SYM_FUNC_START(twofish_ecb_dec_8way)
+@@ -347,7 +347,7 @@ SYM_FUNC_START(twofish_ecb_dec_8way)
+ store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(twofish_ecb_dec_8way)
+
+ SYM_FUNC_START(twofish_cbc_dec_8way)
+@@ -372,5 +372,5 @@ SYM_FUNC_START(twofish_cbc_dec_8way)
+ popq %r12;
+
+ FRAME_END
+- ret;
++ RET;
+ SYM_FUNC_END(twofish_cbc_dec_8way)
+diff --git a/arch/x86/crypto/twofish-i586-asm_32.S b/arch/x86/crypto/twofish-i586-asm_32.S
+index a6f09e4f2e463..3abcad6618840 100644
+--- a/arch/x86/crypto/twofish-i586-asm_32.S
++++ b/arch/x86/crypto/twofish-i586-asm_32.S
+@@ -260,7 +260,7 @@ SYM_FUNC_START(twofish_enc_blk)
+ pop %ebx
+ pop %ebp
+ mov $1, %eax
+- ret
++ RET
+ SYM_FUNC_END(twofish_enc_blk)
+
+ SYM_FUNC_START(twofish_dec_blk)
+@@ -317,5 +317,5 @@ SYM_FUNC_START(twofish_dec_blk)
+ pop %ebx
+ pop %ebp
+ mov $1, %eax
+- ret
++ RET
+ SYM_FUNC_END(twofish_dec_blk)
+diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+index bca4cea757ce2..d2288bf38a8a5 100644
+--- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
++++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+@@ -258,7 +258,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way)
+ popq %rbx;
+ popq %r12;
+ popq %r13;
+- ret;
++ RET;
+
+ .L__enc_xor3:
+ outunpack_enc3(xor);
+@@ -266,7 +266,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way)
+ popq %rbx;
+ popq %r12;
+ popq %r13;
+- ret;
++ RET;
+ SYM_FUNC_END(__twofish_enc_blk_3way)
+
+ SYM_FUNC_START(twofish_dec_blk_3way)
+@@ -301,5 +301,5 @@ SYM_FUNC_START(twofish_dec_blk_3way)
+ popq %rbx;
+ popq %r12;
+ popq %r13;
+- ret;
++ RET;
+ SYM_FUNC_END(twofish_dec_blk_3way)
+diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
+index d2e56232494a8..775af290cd196 100644
+--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
++++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
+@@ -252,7 +252,7 @@ SYM_FUNC_START(twofish_enc_blk)
+
+ popq R1
+ movl $1,%eax
+- ret
++ RET
+ SYM_FUNC_END(twofish_enc_blk)
+
+ SYM_FUNC_START(twofish_dec_blk)
+@@ -304,5 +304,5 @@ SYM_FUNC_START(twofish_dec_blk)
+
+ popq R1
+ movl $1,%eax
+- ret
++ RET
+ SYM_FUNC_END(twofish_dec_blk)
+diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
+index ccb9d32768f31..00413e37feee1 100644
+--- a/arch/x86/entry/entry_32.S
++++ b/arch/x86/entry/entry_32.S
+@@ -740,7 +740,7 @@ SYM_FUNC_START(schedule_tail_wrapper)
+ popl %eax
+
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(schedule_tail_wrapper)
+ .popsection
+
+diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
+index e38a4cf795d96..e23319ad3f42a 100644
+--- a/arch/x86/entry/entry_64.S
++++ b/arch/x86/entry/entry_64.S
+@@ -574,6 +574,10 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
+ ud2
+ 1:
+ #endif
++#ifdef CONFIG_XEN_PV
++ ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
++#endif
++
+ POP_REGS pop_rdi=0
+
+ /*
+@@ -734,7 +738,7 @@ SYM_FUNC_START(asm_load_gs_index)
+ 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
+ swapgs
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(asm_load_gs_index)
+ EXPORT_SYMBOL(asm_load_gs_index)
+
+@@ -885,11 +889,12 @@ SYM_CODE_START_LOCAL(paranoid_entry)
+ * is needed here.
+ */
+ SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
+- ret
++ RET
+
+ .Lparanoid_entry_checkgs:
+ /* EBX = 1 -> kernel GSBASE active, no restore required */
+ movl $1, %ebx
++
+ /*
+ * The kernel-enforced convention is a negative GSBASE indicates
+ * a kernel value. No SWAPGS needed on entry and exit.
+@@ -897,22 +902,15 @@ SYM_CODE_START_LOCAL(paranoid_entry)
+ movl $MSR_GS_BASE, %ecx
+ rdmsr
+ testl %edx, %edx
+- jns .Lparanoid_entry_swapgs
+- ret
++ js .Lparanoid_kernel_gsbase
+
+-.Lparanoid_entry_swapgs:
++ /* EBX = 0 -> SWAPGS required on exit */
++ xorl %ebx, %ebx
+ swapgs
++.Lparanoid_kernel_gsbase:
+
+- /*
+- * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
+- * unconditional CR3 write, even in the PTI case. So do an lfence
+- * to prevent GS speculation, regardless of whether PTI is enabled.
+- */
+ FENCE_SWAPGS_KERNEL_ENTRY
+-
+- /* EBX = 0 -> SWAPGS required on exit */
+- xorl %ebx, %ebx
+- ret
++ RET
+ SYM_CODE_END(paranoid_entry)
+
+ /*
+@@ -991,12 +989,7 @@ SYM_CODE_START_LOCAL(error_entry)
+ movq %rax, %rsp /* switch stack */
+ ENCODE_FRAME_POINTER
+ pushq %r12
+- ret
+-
+-.Lerror_entry_done_lfence:
+- FENCE_SWAPGS_KERNEL_ENTRY
+-.Lerror_entry_done:
+- ret
++ RET
+
+ /*
+ * There are two places in the kernel that can potentially fault with
+@@ -1020,8 +1013,14 @@ SYM_CODE_START_LOCAL(error_entry)
+ * .Lgs_change's error handler with kernel gsbase.
+ */
+ SWAPGS
+- FENCE_SWAPGS_USER_ENTRY
+- jmp .Lerror_entry_done
++
++ /*
++ * Issue an LFENCE to prevent GS speculation, regardless of whether it is a
++ * kernel or user gsbase.
++ */
++.Lerror_entry_done_lfence:
++ FENCE_SWAPGS_KERNEL_ENTRY
++ RET
+
+ .Lbstep_iret:
+ /* Fix truncated RIP */
+diff --git a/arch/x86/entry/thunk_32.S b/arch/x86/entry/thunk_32.S
+index f1f96d4d8cd60..7591bab060f70 100644
+--- a/arch/x86/entry/thunk_32.S
++++ b/arch/x86/entry/thunk_32.S
+@@ -24,7 +24,7 @@ SYM_CODE_START_NOALIGN(\name)
+ popl %edx
+ popl %ecx
+ popl %eax
+- ret
++ RET
+ _ASM_NOKPROBE(\name)
+ SYM_CODE_END(\name)
+ .endm
+diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S
+index 496b11ec469de..505b488fcc655 100644
+--- a/arch/x86/entry/thunk_64.S
++++ b/arch/x86/entry/thunk_64.S
+@@ -50,7 +50,7 @@ SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore)
+ popq %rsi
+ popq %rdi
+ popq %rbp
+- ret
++ RET
+ _ASM_NOKPROBE(__thunk_restore)
+ SYM_CODE_END(__thunk_restore)
+ #endif
+diff --git a/arch/x86/entry/vdso/vdso32/system_call.S b/arch/x86/entry/vdso/vdso32/system_call.S
+index 6ddd7a937b3e3..d33c6513fd2cb 100644
+--- a/arch/x86/entry/vdso/vdso32/system_call.S
++++ b/arch/x86/entry/vdso/vdso32/system_call.S
+@@ -78,7 +78,7 @@ SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL)
+ popl %ecx
+ CFI_RESTORE ecx
+ CFI_ADJUST_CFA_OFFSET -4
+- ret
++ RET
+ CFI_ENDPROC
+
+ .size __kernel_vsyscall,.-__kernel_vsyscall
+diff --git a/arch/x86/entry/vdso/vsgx.S b/arch/x86/entry/vdso/vsgx.S
+index 99dafac992e2c..d77d278ee9dd6 100644
+--- a/arch/x86/entry/vdso/vsgx.S
++++ b/arch/x86/entry/vdso/vsgx.S
+@@ -81,7 +81,7 @@ SYM_FUNC_START(__vdso_sgx_enter_enclave)
+ pop %rbx
+ leave
+ .cfi_def_cfa %rsp, 8
+- ret
++ RET
+
+ /* The out-of-line code runs with the pre-leave stack frame. */
+ .cfi_def_cfa %rbp, 16
+diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
+index 1b40b92970831..fd2ee9408e914 100644
+--- a/arch/x86/entry/vsyscall/vsyscall_64.c
++++ b/arch/x86/entry/vsyscall/vsyscall_64.c
+@@ -226,7 +226,8 @@ bool emulate_vsyscall(unsigned long error_code,
+ if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
+ warn_bad_vsyscall(KERN_DEBUG, regs,
+ "seccomp tried to change syscall nr or ip");
+- do_exit(SIGSYS);
++ force_exit_sig(SIGSYS);
++ return true;
+ }
+ regs->orig_ax = -1;
+ if (tmp)
+diff --git a/arch/x86/entry/vsyscall/vsyscall_emu_64.S b/arch/x86/entry/vsyscall/vsyscall_emu_64.S
+index 2e203f3a25a7b..15e35159ebb68 100644
+--- a/arch/x86/entry/vsyscall/vsyscall_emu_64.S
++++ b/arch/x86/entry/vsyscall/vsyscall_emu_64.S
+@@ -19,17 +19,17 @@ __vsyscall_page:
+
+ mov $__NR_gettimeofday, %rax
+ syscall
+- ret
++ RET
+
+ .balign 1024, 0xcc
+ mov $__NR_time, %rax
+ syscall
+- ret
++ RET
+
+ .balign 1024, 0xcc
+ mov $__NR_getcpu, %rax
+ syscall
+- ret
++ RET
+
+ .balign 4096, 0xcc
+
+diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
+index 6dfa8ddaa60f7..81d5e0a1f48cd 100644
+--- a/arch/x86/events/core.c
++++ b/arch/x86/events/core.c
+@@ -2762,10 +2762,11 @@ static bool perf_hw_regs(struct pt_regs *regs)
+ void
+ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct unwind_state state;
+ unsigned long addr;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* TODO: We don't support guest os callchain now */
+ return;
+ }
+@@ -2865,10 +2866,11 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent
+ void
+ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ struct stack_frame frame;
+ const struct stack_frame __user *fp;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
++ if (guest_cbs && guest_cbs->is_in_guest()) {
+ /* TODO: We don't support guest os callchain now */
+ return;
+ }
+@@ -2945,18 +2947,21 @@ static unsigned long code_segment_base(struct pt_regs *regs)
+
+ unsigned long perf_instruction_pointer(struct pt_regs *regs)
+ {
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+- return perf_guest_cbs->get_guest_ip();
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
++
++ if (guest_cbs && guest_cbs->is_in_guest())
++ return guest_cbs->get_guest_ip();
+
+ return regs->ip + code_segment_base(regs);
+ }
+
+ unsigned long perf_misc_flags(struct pt_regs *regs)
+ {
++ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+ int misc = 0;
+
+- if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+- if (perf_guest_cbs->is_user_mode())
++ if (guest_cbs && guest_cbs->is_in_guest()) {
++ if (guest_cbs->is_user_mode())
+ misc |= PERF_RECORD_MISC_GUEST_USER;
+ else
+ misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
+index 9a044438072ba..265cb203d9d5c 100644
+--- a/arch/x86/events/intel/core.c
++++ b/arch/x86/events/intel/core.c
+@@ -243,7 +243,8 @@ static struct extra_reg intel_skl_extra_regs[] __read_mostly = {
+
+ static struct event_constraint intel_icl_event_constraints[] = {
+ FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+- FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* INST_RETIRED.PREC_DIST */
++ FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* old INST_RETIRED.PREC_DIST */
++ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+@@ -280,7 +281,7 @@ static struct extra_reg intel_spr_extra_regs[] __read_mostly = {
+ INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
+ INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
+ INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
+- INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
++ INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff1f, FE),
+ INTEL_UEVENT_EXTRA_REG(0x40ad, MSR_PEBS_FRONTEND, 0x7, FE),
+ INTEL_UEVENT_EXTRA_REG(0x04c2, MSR_PEBS_FRONTEND, 0x8, FE),
+ EVENT_EXTRA_END
+@@ -288,7 +289,7 @@ static struct extra_reg intel_spr_extra_regs[] __read_mostly = {
+
+ static struct event_constraint intel_spr_event_constraints[] = {
+ FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+- FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* INST_RETIRED.PREC_DIST */
++ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+@@ -2787,6 +2788,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
+ {
+ struct perf_sample_data data;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
++ struct perf_guest_info_callbacks *guest_cbs;
+ int bit;
+ int handled = 0;
+ u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl);
+@@ -2853,9 +2855,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
+ */
+ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) {
+ handled++;
+- if (unlikely(perf_guest_cbs && perf_guest_cbs->is_in_guest() &&
+- perf_guest_cbs->handle_intel_pt_intr))
+- perf_guest_cbs->handle_intel_pt_intr();
++
++ guest_cbs = perf_get_guest_cbs();
++ if (unlikely(guest_cbs && guest_cbs->is_in_guest() &&
++ guest_cbs->handle_intel_pt_intr))
++ guest_cbs->handle_intel_pt_intr();
+ else
+ intel_pt_interrupt();
+ }
+@@ -2998,8 +3002,10 @@ intel_vlbr_constraints(struct perf_event *event)
+ {
+ struct event_constraint *c = &vlbr_constraint;
+
+- if (unlikely(constraint_match(c, event->hw.config)))
++ if (unlikely(constraint_match(c, event->hw.config))) {
++ event->hw.flags |= c->flags;
+ return c;
++ }
+
+ return NULL;
+ }
+@@ -4648,6 +4654,19 @@ static __initconst const struct x86_pmu intel_pmu = {
+ .lbr_read = intel_pmu_lbr_read_64,
+ .lbr_save = intel_pmu_lbr_save,
+ .lbr_restore = intel_pmu_lbr_restore,
++
++ /*
++ * SMM has access to all 4 rings and while traditionally SMM code only
++ * ran in CPL0, 2021-era firmware is starting to make use of CPL3 in SMM.
++ *
++ * Since the EVENTSEL.{USR,OS} CPL filtering makes no distinction
++ * between SMM or not, this results in what should be pure userspace
++ * counters including SMM data.
++ *
++ * This is a clear privilege issue, therefore globally disable
++ * counting SMM by default.
++ */
++ .attr_freeze_on_smi = 1,
+ };
+
+ static __init void intel_clovertown_quirk(void)
+@@ -5447,7 +5466,11 @@ static void intel_pmu_check_event_constraints(struct event_constraint *event_con
+ /* Disabled fixed counters which are not in CPUID */
+ c->idxmsk64 &= intel_ctrl;
+
+- if (c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES)
++ /*
++ * Don't extend the pseudo-encoding to the
++ * generic counters
++ */
++ if (!use_fixed_pseudo_encoding(c->code))
+ c->idxmsk64 |= (1ULL << num_counters) - 1;
+ }
+ c->idxmsk64 &=
+@@ -6181,6 +6204,19 @@ __init int intel_pmu_init(void)
+ pmu->num_counters = x86_pmu.num_counters;
+ pmu->num_counters_fixed = x86_pmu.num_counters_fixed;
+ }
++
++ /*
++ * Quirk: For some Alder Lake machine, when all E-cores are disabled in
++ * a BIOS, the leaf 0xA will enumerate all counters of P-cores. However,
++ * the X86_FEATURE_HYBRID_CPU is still set. The above codes will
++ * mistakenly add extra counters for P-cores. Correct the number of
++ * counters here.
++ */
++ if ((pmu->num_counters > 8) || (pmu->num_counters_fixed > 4)) {
++ pmu->num_counters = x86_pmu.num_counters;
++ pmu->num_counters_fixed = x86_pmu.num_counters_fixed;
++ }
++
+ pmu->max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, pmu->num_counters);
+ pmu->unconstrained = (struct event_constraint)
+ __EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1,
+diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
+index 8647713276a73..4dbb55a43dad2 100644
+--- a/arch/x86/events/intel/ds.c
++++ b/arch/x86/events/intel/ds.c
+@@ -923,7 +923,8 @@ struct event_constraint intel_skl_pebs_event_constraints[] = {
+ };
+
+ struct event_constraint intel_icl_pebs_event_constraints[] = {
+- INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
++ INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x100000000ULL), /* old INST_RETIRED.PREC_DIST */
++ INTEL_FLAGS_UEVENT_CONSTRAINT(0x0100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), /* SLOTS */
+
+ INTEL_PLD_CONSTRAINT(0x1cd, 0xff), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
+@@ -943,7 +944,7 @@ struct event_constraint intel_icl_pebs_event_constraints[] = {
+ };
+
+ struct event_constraint intel_spr_pebs_event_constraints[] = {
+- INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x100000000ULL),
++ INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
+
+ INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe),
+diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
+index 9e6d6eaeb4cb6..f455dd93f9219 100644
+--- a/arch/x86/events/intel/lbr.c
++++ b/arch/x86/events/intel/lbr.c
+@@ -1734,6 +1734,9 @@ static bool is_arch_lbr_xsave_available(void)
+ * Check the LBR state with the corresponding software structure.
+ * Disable LBR XSAVES support if the size doesn't match.
+ */
++ if (xfeature_size(XFEATURE_LBR) == 0)
++ return false;
++
+ if (WARN_ON(xfeature_size(XFEATURE_LBR) != get_lbr_state_size()))
+ return false;
+
+diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c
+index 7f406c14715fd..215aed65e9782 100644
+--- a/arch/x86/events/intel/pt.c
++++ b/arch/x86/events/intel/pt.c
+@@ -472,7 +472,7 @@ static u64 pt_config_filters(struct perf_event *event)
+ pt->filters.filter[range].msr_b = filter->msr_b;
+ }
+
+- rtit_ctl |= filter->config << pt_address_ranges[range].reg_off;
++ rtit_ctl |= (u64)filter->config << pt_address_ranges[range].reg_off;
+ }
+
+ return rtit_ctl;
+@@ -897,8 +897,9 @@ static void pt_handle_status(struct pt *pt)
+ * means we are already losing data; need to let the decoder
+ * know.
+ */
+- if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) ||
+- buf->output_off == pt_buffer_region_size(buf)) {
++ if (!buf->single &&
++ (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) ||
++ buf->output_off == pt_buffer_region_size(buf))) {
+ perf_aux_output_flag(&pt->handle,
+ PERF_AUX_FLAG_TRUNCATED);
+ advance++;
+diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h
+index 7280c8a3c8310..6d735611c281c 100644
+--- a/arch/x86/events/intel/uncore_discovery.h
++++ b/arch/x86/events/intel/uncore_discovery.h
+@@ -30,7 +30,7 @@
+
+
+ #define uncore_discovery_invalid_unit(unit) \
+- (!unit.table1 || !unit.ctl || !unit.table3 || \
++ (!unit.table1 || !unit.ctl || \
+ unit.table1 == -1ULL || unit.ctl == -1ULL || \
+ unit.table3 == -1ULL)
+
+diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c
+index 5ddc0f30db6fc..ed869443efb21 100644
+--- a/arch/x86/events/intel/uncore_snbep.c
++++ b/arch/x86/events/intel/uncore_snbep.c
+@@ -452,7 +452,7 @@
+ #define ICX_M3UPI_PCI_PMON_BOX_CTL 0xa0
+
+ /* ICX IMC */
+-#define ICX_NUMBER_IMC_CHN 2
++#define ICX_NUMBER_IMC_CHN 3
+ #define ICX_IMC_MEM_STRIDE 0x4
+
+ /* SPR */
+@@ -3608,6 +3608,9 @@ static int skx_cha_hw_config(struct intel_uncore_box *box, struct perf_event *ev
+ struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
+ struct extra_reg *er;
+ int idx = 0;
++ /* Any of the CHA events may be filtered by Thread/Core-ID.*/
++ if (event->hw.config & SNBEP_CBO_PMON_CTL_TID_EN)
++ idx = SKX_CHA_MSR_PMON_BOX_FILTER_TID;
+
+ for (er = skx_uncore_cha_extra_regs; er->msr; er++) {
+ if (er->event != (event->hw.config & er->config_mask))
+@@ -3675,6 +3678,7 @@ static struct event_constraint skx_uncore_iio_constraints[] = {
+ UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
+ UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
+ UNCORE_EVENT_CONSTRAINT(0xd4, 0xc),
++ UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
+ EVENT_CONSTRAINT_END
+ };
+
+@@ -4525,6 +4529,13 @@ static void snr_iio_cleanup_mapping(struct intel_uncore_type *type)
+ pmu_iio_cleanup_mapping(type, &snr_iio_mapping_group);
+ }
+
++static struct event_constraint snr_uncore_iio_constraints[] = {
++ UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
++ UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
++ UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
++ EVENT_CONSTRAINT_END
++};
++
+ static struct intel_uncore_type snr_uncore_iio = {
+ .name = "iio",
+ .num_counters = 4,
+@@ -4536,6 +4547,7 @@ static struct intel_uncore_type snr_uncore_iio = {
+ .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
+ .box_ctl = SNR_IIO_MSR_PMON_BOX_CTL,
+ .msr_offset = SNR_IIO_MSR_OFFSET,
++ .constraints = snr_uncore_iio_constraints,
+ .ops = &ivbep_uncore_msr_ops,
+ .format_group = &snr_uncore_iio_format_group,
+ .attr_update = snr_iio_attr_update,
+@@ -5076,8 +5088,10 @@ static struct event_constraint icx_uncore_iio_constraints[] = {
+ UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
+ UNCORE_EVENT_CONSTRAINT(0x03, 0x3),
+ UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
++ UNCORE_EVENT_CONSTRAINT(0x88, 0xc),
+ UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
+ UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
++ UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
+ EVENT_CONSTRAINT_END
+ };
+
+@@ -5463,12 +5477,12 @@ static struct intel_uncore_ops icx_uncore_mmio_ops = {
+ static struct intel_uncore_type icx_uncore_imc = {
+ .name = "imc",
+ .num_counters = 4,
+- .num_boxes = 8,
++ .num_boxes = 12,
+ .perf_ctr_bits = 48,
+ .fixed_ctr_bits = 48,
+ .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR,
+ .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL,
+- .event_descs = hswep_uncore_imc_events,
++ .event_descs = snr_uncore_imc_events,
+ .perf_ctr = SNR_IMC_MMIO_PMON_CTR0,
+ .event_ctl = SNR_IMC_MMIO_PMON_CTL0,
+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
+@@ -5647,6 +5661,7 @@ static struct intel_uncore_type spr_uncore_chabox = {
+ .event_mask = SPR_CHA_PMON_EVENT_MASK,
+ .event_mask_ext = SPR_RAW_EVENT_MASK_EXT,
+ .num_shared_regs = 1,
++ .constraints = skx_uncore_chabox_constraints,
+ .ops = &spr_uncore_chabox_ops,
+ .format_group = &spr_uncore_chabox_format_group,
+ .attr_update = uncore_alias_groups,
+@@ -5658,6 +5673,7 @@ static struct intel_uncore_type spr_uncore_iio = {
+ .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
+ .format_group = &snr_uncore_iio_format_group,
+ .attr_update = uncore_alias_groups,
++ .constraints = icx_uncore_iio_constraints,
+ };
+
+ static struct attribute *spr_uncore_raw_formats_attr[] = {
+@@ -5686,9 +5702,16 @@ static struct intel_uncore_type spr_uncore_irp = {
+
+ };
+
++static struct event_constraint spr_uncore_m2pcie_constraints[] = {
++ UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
++ UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
++ EVENT_CONSTRAINT_END
++};
++
+ static struct intel_uncore_type spr_uncore_m2pcie = {
+ SPR_UNCORE_COMMON_FORMAT(),
+ .name = "m2pcie",
++ .constraints = spr_uncore_m2pcie_constraints,
+ };
+
+ static struct intel_uncore_type spr_uncore_pcu = {
+@@ -5765,6 +5788,7 @@ static struct intel_uncore_type spr_uncore_upi = {
+ static struct intel_uncore_type spr_uncore_m3upi = {
+ SPR_UNCORE_PCI_COMMON_FORMAT(),
+ .name = "m3upi",
++ .constraints = icx_uncore_m3upi_constraints,
+ };
+
+ static struct intel_uncore_type spr_uncore_mdf = {
+diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c
+index 85feafacc445d..77e3a47af5ad5 100644
+--- a/arch/x86/events/rapl.c
++++ b/arch/x86/events/rapl.c
+@@ -536,11 +536,14 @@ static struct perf_msr intel_rapl_spr_msrs[] = {
+ * - perf_msr_probe(PERF_RAPL_MAX)
+ * - want to use same event codes across both architectures
+ */
+-static struct perf_msr amd_rapl_msrs[PERF_RAPL_MAX] = {
+- [PERF_RAPL_PKG] = { MSR_AMD_PKG_ENERGY_STATUS, &rapl_events_pkg_group, test_msr },
++static struct perf_msr amd_rapl_msrs[] = {
++ [PERF_RAPL_PP0] = { 0, &rapl_events_cores_group, 0, false, 0 },
++ [PERF_RAPL_PKG] = { MSR_AMD_PKG_ENERGY_STATUS, &rapl_events_pkg_group, test_msr, false, RAPL_MSR_MASK },
++ [PERF_RAPL_RAM] = { 0, &rapl_events_ram_group, 0, false, 0 },
++ [PERF_RAPL_PP1] = { 0, &rapl_events_gpu_group, 0, false, 0 },
++ [PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group, 0, false, 0 },
+ };
+
+-
+ static int rapl_cpu_offline(unsigned int cpu)
+ {
+ struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
+diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
+index 708a2712a516d..b6d48ca5b0f17 100644
+--- a/arch/x86/hyperv/hv_init.c
++++ b/arch/x86/hyperv/hv_init.c
+@@ -139,7 +139,6 @@ void set_hv_tscchange_cb(void (*cb)(void))
+ struct hv_reenlightenment_control re_ctrl = {
+ .vector = HYPERV_REENLIGHTENMENT_VECTOR,
+ .enabled = 1,
+- .target_vp = hv_vp_index[smp_processor_id()]
+ };
+ struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};
+
+@@ -148,13 +147,20 @@ void set_hv_tscchange_cb(void (*cb)(void))
+ return;
+ }
+
++ if (!hv_vp_index)
++ return;
++
+ hv_reenlightenment_cb = cb;
+
+ /* Make sure callback is registered before we write to MSRs */
+ wmb();
+
++ re_ctrl.target_vp = hv_vp_index[get_cpu()];
++
+ wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+ wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
++
++ put_cpu();
+ }
+ EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);
+
+@@ -342,20 +348,13 @@ static void __init hv_get_partition_id(void)
+ */
+ void __init hyperv_init(void)
+ {
+- u64 guest_id, required_msrs;
++ u64 guest_id;
+ union hv_x64_msr_hypercall_contents hypercall_msr;
+ int cpuhp;
+
+ if (x86_hyper_type != X86_HYPER_MS_HYPERV)
+ return;
+
+- /* Absolutely required MSRs */
+- required_msrs = HV_MSR_HYPERCALL_AVAILABLE |
+- HV_MSR_VP_INDEX_AVAILABLE;
+-
+- if ((ms_hyperv.features & required_msrs) != required_msrs)
+- return;
+-
+ if (hv_common_init())
+ return;
+
+diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
+index bd13736d0c054..0ad2378fe6ad7 100644
+--- a/arch/x86/hyperv/mmu.c
++++ b/arch/x86/hyperv/mmu.c
+@@ -68,15 +68,6 @@ static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
+
+ local_irq_save(flags);
+
+- /*
+- * Only check the mask _after_ interrupt has been disabled to avoid the
+- * mask changing under our feet.
+- */
+- if (cpumask_empty(cpus)) {
+- local_irq_restore(flags);
+- return;
+- }
+-
+ flush_pcpu = (struct hv_tlb_flush **)
+ this_cpu_ptr(hyperv_pcpu_input_arg);
+
+@@ -115,7 +106,9 @@ static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
+ * must. We will also check all VP numbers when walking the
+ * supplied CPU set to remain correct in all cases.
+ */
+- if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
++ cpu = cpumask_last(cpus);
++
++ if (cpu < nr_cpumask_bits && hv_cpu_number_to_vp_number(cpu) >= 64)
+ goto do_ex_hypercall;
+
+ for_each_cpu(cpu, cpus) {
+@@ -131,6 +124,12 @@ static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
+ __set_bit(vcpu, (unsigned long *)
+ &flush->processor_mask);
+ }
++
++ /* nothing to flush if 'processor_mask' ends up being empty */
++ if (!flush->processor_mask) {
++ local_irq_restore(flags);
++ return;
++ }
+ }
+
+ /*
+diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
+index 84b87538a15de..66570e95af398 100644
+--- a/arch/x86/include/asm/bug.h
++++ b/arch/x86/include/asm/bug.h
+@@ -22,7 +22,7 @@
+
+ #ifdef CONFIG_DEBUG_BUGVERBOSE
+
+-#define _BUG_FLAGS(ins, flags) \
++#define _BUG_FLAGS(ins, flags, extra) \
+ do { \
+ asm_inline volatile("1:\t" ins "\n" \
+ ".pushsection __bug_table,\"aw\"\n" \
+@@ -31,7 +31,8 @@ do { \
+ "\t.word %c1" "\t# bug_entry::line\n" \
+ "\t.word %c2" "\t# bug_entry::flags\n" \
+ "\t.org 2b+%c3\n" \
+- ".popsection" \
++ ".popsection\n" \
++ extra \
+ : : "i" (__FILE__), "i" (__LINE__), \
+ "i" (flags), \
+ "i" (sizeof(struct bug_entry))); \
+@@ -39,14 +40,15 @@ do { \
+
+ #else /* !CONFIG_DEBUG_BUGVERBOSE */
+
+-#define _BUG_FLAGS(ins, flags) \
++#define _BUG_FLAGS(ins, flags, extra) \
+ do { \
+ asm_inline volatile("1:\t" ins "\n" \
+ ".pushsection __bug_table,\"aw\"\n" \
+ "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \
+ "\t.word %c0" "\t# bug_entry::flags\n" \
+ "\t.org 2b+%c1\n" \
+- ".popsection" \
++ ".popsection\n" \
++ extra \
+ : : "i" (flags), \
+ "i" (sizeof(struct bug_entry))); \
+ } while (0)
+@@ -55,7 +57,7 @@ do { \
+
+ #else
+
+-#define _BUG_FLAGS(ins, flags) asm volatile(ins)
++#define _BUG_FLAGS(ins, flags, extra) asm volatile(ins)
+
+ #endif /* CONFIG_GENERIC_BUG */
+
+@@ -63,8 +65,8 @@ do { \
+ #define BUG() \
+ do { \
+ instrumentation_begin(); \
+- _BUG_FLAGS(ASM_UD2, 0); \
+- unreachable(); \
++ _BUG_FLAGS(ASM_UD2, 0, ""); \
++ __builtin_unreachable(); \
+ } while (0)
+
+ /*
+@@ -75,9 +77,9 @@ do { \
+ */
+ #define __WARN_FLAGS(flags) \
+ do { \
++ __auto_type __flags = BUGFLAG_WARNING|(flags); \
+ instrumentation_begin(); \
+- _BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
+- annotate_reachable(); \
++ _BUG_FLAGS(ASM_UD2, __flags, ASM_REACHABLE); \
+ instrumentation_end(); \
+ } while (0)
+
+diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
+index 7516e4199b3c6..20fd0acd7d800 100644
+--- a/arch/x86/include/asm/compat.h
++++ b/arch/x86/include/asm/compat.h
+@@ -28,15 +28,13 @@ typedef u16 compat_ipc_pid_t;
+ typedef __kernel_fsid_t compat_fsid_t;
+
+ struct compat_stat {
+- compat_dev_t st_dev;
+- u16 __pad1;
++ u32 st_dev;
+ compat_ino_t st_ino;
+ compat_mode_t st_mode;
+ compat_nlink_t st_nlink;
+ __compat_uid_t st_uid;
+ __compat_gid_t st_gid;
+- compat_dev_t st_rdev;
+- u16 __pad2;
++ u32 st_rdev;
+ u32 st_size;
+ u32 st_blksize;
+ u32 st_blocks;
+diff --git a/arch/x86/include/asm/cpu_entry_area.h b/arch/x86/include/asm/cpu_entry_area.h
+index 3d52b094850a9..dd5ea1bdf04c5 100644
+--- a/arch/x86/include/asm/cpu_entry_area.h
++++ b/arch/x86/include/asm/cpu_entry_area.h
+@@ -10,6 +10,12 @@
+
+ #ifdef CONFIG_X86_64
+
++#ifdef CONFIG_AMD_MEM_ENCRYPT
++#define VC_EXCEPTION_STKSZ EXCEPTION_STKSZ
++#else
++#define VC_EXCEPTION_STKSZ 0
++#endif
++
+ /* Macro to enforce the same ordering and stack sizes */
+ #define ESTACKS_MEMBERS(guardsize, optional_stack_size) \
+ char DF_stack_guard[guardsize]; \
+@@ -28,7 +34,7 @@
+
+ /* The exception stacks' physical storage. No guard pages required */
+ struct exception_stacks {
+- ESTACKS_MEMBERS(0, 0)
++ ESTACKS_MEMBERS(0, VC_EXCEPTION_STKSZ)
+ };
+
+ /* The effective cpu entry area mapping with guard pages. */
+diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
+index d0ce5cfd3ac14..4423759f619cc 100644
+--- a/arch/x86/include/asm/cpufeatures.h
++++ b/arch/x86/include/asm/cpufeatures.h
+@@ -204,7 +204,7 @@
+ /* FREE! ( 7*32+10) */
+ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */
+ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
+-#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */
++#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */
+ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */
+ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */
+ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */
+diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
+index 4d0b126835b8a..63158fd558567 100644
+--- a/arch/x86/include/asm/efi.h
++++ b/arch/x86/include/asm/efi.h
+@@ -197,8 +197,6 @@ static inline bool efi_runtime_supported(void)
+
+ extern void parse_efi_setup(u64 phys_addr, u32 data_len);
+
+-extern void efifb_setup_from_dmi(struct screen_info *si, const char *opt);
+-
+ extern void efi_thunk_runtime_setup(void);
+ efi_status_t efi_set_virtual_address_map(unsigned long memory_map_size,
+ unsigned long descriptor_size,
+diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h
+index 91d7182ad2d6e..4ec3613551e3b 100644
+--- a/arch/x86/include/asm/insn-eval.h
++++ b/arch/x86/include/asm/insn-eval.h
+@@ -21,6 +21,7 @@ int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs);
+ int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs);
+ unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx);
+ int insn_get_code_seg_params(struct pt_regs *regs);
++int insn_get_effective_ip(struct pt_regs *regs, unsigned long *ip);
+ int insn_fetch_from_user(struct pt_regs *regs,
+ unsigned char buf[MAX_INSN_SIZE]);
+ int insn_fetch_from_user_inatomic(struct pt_regs *regs,
+diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
+index 562854c608082..e087cd7837c31 100644
+--- a/arch/x86/include/asm/irq_stack.h
++++ b/arch/x86/include/asm/irq_stack.h
+@@ -77,11 +77,11 @@
+ * Function calls can clobber anything except the callee-saved
+ * registers. Tell the compiler.
+ */
+-#define call_on_irqstack(func, asm_call, argconstr...) \
++#define call_on_stack(stack, func, asm_call, argconstr...) \
+ { \
+ register void *tos asm("r11"); \
+ \
+- tos = ((void *)__this_cpu_read(hardirq_stack_ptr)); \
++ tos = ((void *)(stack)); \
+ \
+ asm_inline volatile( \
+ "movq %%rsp, (%[tos]) \n" \
+@@ -98,6 +98,26 @@
+ ); \
+ }
+
++#define ASM_CALL_ARG0 \
++ "call %P[__func] \n" \
++ ASM_REACHABLE
++
++#define ASM_CALL_ARG1 \
++ "movq %[arg1], %%rdi \n" \
++ ASM_CALL_ARG0
++
++#define ASM_CALL_ARG2 \
++ "movq %[arg2], %%rsi \n" \
++ ASM_CALL_ARG1
++
++#define ASM_CALL_ARG3 \
++ "movq %[arg3], %%rdx \n" \
++ ASM_CALL_ARG2
++
++#define call_on_irqstack(func, asm_call, argconstr...) \
++ call_on_stack(__this_cpu_read(hardirq_stack_ptr), \
++ func, asm_call, argconstr)
++
+ /* Macros to assert type correctness for run_*_on_irqstack macros */
+ #define assert_function_type(func, proto) \
+ static_assert(__builtin_types_compatible_p(typeof(&func), proto))
+@@ -147,8 +167,7 @@
+ */
+ #define ASM_CALL_SYSVEC \
+ "call irq_enter_rcu \n" \
+- "movq %[arg1], %%rdi \n" \
+- "call %P[__func] \n" \
++ ASM_CALL_ARG1 \
+ "call irq_exit_rcu \n"
+
+ #define SYSVEC_CONSTRAINTS , [arg1] "r" (regs)
+@@ -168,12 +187,10 @@
+ */
+ #define ASM_CALL_IRQ \
+ "call irq_enter_rcu \n" \
+- "movq %[arg1], %%rdi \n" \
+- "movl %[arg2], %%esi \n" \
+- "call %P[__func] \n" \
++ ASM_CALL_ARG2 \
+ "call irq_exit_rcu \n"
+
+-#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector)
++#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" ((unsigned long)vector)
+
+ #define run_irq_on_irqstack_cond(func, regs, vector) \
+ { \
+@@ -185,9 +202,6 @@
+ IRQ_CONSTRAINTS, regs, vector); \
+ }
+
+-#define ASM_CALL_SOFTIRQ \
+- "call %P[__func] \n"
+-
+ /*
+ * Macro to invoke __do_softirq on the irq stack. This is only called from
+ * task context when bottom halves are about to be reenabled and soft
+@@ -197,7 +211,7 @@
+ #define do_softirq_own_stack() \
+ { \
+ __this_cpu_write(hardirq_stack_inuse, true); \
+- call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
++ call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \
+ __this_cpu_write(hardirq_stack_inuse, false); \
+ }
+
+diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
+index cefe1d81e2e8b..9e50da3ed01a3 100644
+--- a/arch/x86/include/asm/kvm-x86-ops.h
++++ b/arch/x86/include/asm/kvm-x86-ops.h
+@@ -47,6 +47,7 @@ KVM_X86_OP(set_dr7)
+ KVM_X86_OP(cache_reg)
+ KVM_X86_OP(get_rflags)
+ KVM_X86_OP(set_rflags)
++KVM_X86_OP(get_if_flag)
+ KVM_X86_OP(tlb_flush_all)
+ KVM_X86_OP(tlb_flush_current)
+ KVM_X86_OP_NULL(tlb_remote_flush)
+diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
+index 13f64654dfff8..49d814b2a341a 100644
+--- a/arch/x86/include/asm/kvm_host.h
++++ b/arch/x86/include/asm/kvm_host.h
+@@ -98,7 +98,7 @@
+ KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+ #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26)
+ #define KVM_REQ_TLB_FLUSH_GUEST \
+- KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP)
++ KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+ #define KVM_REQ_APF_READY KVM_ARCH_REQ(28)
+ #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29)
+ #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
+@@ -364,6 +364,7 @@ union kvm_mmu_extended_role {
+ unsigned int cr4_smap:1;
+ unsigned int cr4_smep:1;
+ unsigned int cr4_la57:1;
++ unsigned int efer_lma:1;
+ };
+ };
+
+@@ -504,6 +505,7 @@ struct kvm_pmu {
+ u64 global_ctrl_mask;
+ u64 global_ovf_ctrl_mask;
+ u64 reserved_bits;
++ u64 raw_event_mask;
+ u8 version;
+ struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC];
+ struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED];
+@@ -751,7 +753,7 @@ struct kvm_vcpu_arch {
+ u8 preempted;
+ u64 msr_val;
+ u64 last_steal;
+- struct gfn_to_pfn_cache cache;
++ struct gfn_to_hva_cache cache;
+ } st;
+
+ u64 l1_tsc_offset;
+@@ -1340,6 +1342,7 @@ struct kvm_x86_ops {
+ void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
+ unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
+ void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
++ bool (*get_if_flag)(struct kvm_vcpu *vcpu);
+
+ void (*tlb_flush_all)(struct kvm_vcpu *vcpu);
+ void (*tlb_flush_current)(struct kvm_vcpu *vcpu);
+@@ -1485,6 +1488,7 @@ struct kvm_x86_ops {
+ };
+
+ struct kvm_x86_nested_ops {
++ void (*leave_nested)(struct kvm_vcpu *vcpu);
+ int (*check_events)(struct kvm_vcpu *vcpu);
+ bool (*hv_timer_pending)(struct kvm_vcpu *vcpu);
+ void (*triple_fault)(struct kvm_vcpu *vcpu);
+@@ -1507,6 +1511,7 @@ struct kvm_x86_init_ops {
+ int (*disabled_by_bios)(void);
+ int (*check_processor_compatibility)(void);
+ int (*hardware_setup)(void);
++ bool (*intel_pt_intr_in_guest)(void);
+
+ struct kvm_x86_ops *runtime_ops;
+ };
+@@ -1554,8 +1559,9 @@ static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
+ return -ENOTSUPP;
+ }
+
+-int kvm_mmu_module_init(void);
+-void kvm_mmu_module_exit(void);
++void kvm_mmu_x86_module_init(void);
++int kvm_mmu_vendor_module_init(void);
++void kvm_mmu_vendor_module_exit(void);
+
+ void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
+ int kvm_mmu_create(struct kvm_vcpu *vcpu);
+diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
+index 365111789cc68..030907922bd07 100644
+--- a/arch/x86/include/asm/linkage.h
++++ b/arch/x86/include/asm/linkage.h
+@@ -18,6 +18,20 @@
+ #define __ALIGN_STR __stringify(__ALIGN)
+ #endif
+
++#ifdef CONFIG_SLS
++#define RET ret; int3
++#else
++#define RET ret
++#endif
++
++#else /* __ASSEMBLY__ */
++
++#ifdef CONFIG_SLS
++#define ASM_RET "ret; int3\n\t"
++#else
++#define ASM_RET "ret\n\t"
++#endif
++
+ #endif /* __ASSEMBLY__ */
+
+ #endif /* _ASM_X86_LINKAGE_H */
+diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
+index 9c80c68d75b54..3fb9f5ebefa42 100644
+--- a/arch/x86/include/asm/mem_encrypt.h
++++ b/arch/x86/include/asm/mem_encrypt.h
+@@ -13,6 +13,7 @@
+ #ifndef __ASSEMBLY__
+
+ #include <linux/init.h>
++#include <linux/cc_platform.h>
+
+ #include <asm/bootparam.h>
+
+diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
+index ab45a220fac47..fcbfe94903bb8 100644
+--- a/arch/x86/include/asm/microcode.h
++++ b/arch/x86/include/asm/microcode.h
+@@ -132,10 +132,12 @@ extern void load_ucode_ap(void);
+ void reload_early_microcode(void);
+ extern bool get_builtin_firmware(struct cpio_data *cd, const char *name);
+ extern bool initrd_gone;
++void microcode_bsp_resume(void);
+ #else
+ static inline void __init load_ucode_bsp(void) { }
+ static inline void load_ucode_ap(void) { }
+ static inline void reload_early_microcode(void) { }
++static inline void microcode_bsp_resume(void) { }
+ static inline bool
+ get_builtin_firmware(struct cpio_data *cd, const char *name) { return false; }
+ #endif
+diff --git a/arch/x86/include/asm/msi.h b/arch/x86/include/asm/msi.h
+index b85147d75626e..d71c7e8b738d2 100644
+--- a/arch/x86/include/asm/msi.h
++++ b/arch/x86/include/asm/msi.h
+@@ -12,14 +12,17 @@ int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec,
+ /* Structs and defines for the X86 specific MSI message format */
+
+ typedef struct x86_msi_data {
+- u32 vector : 8,
+- delivery_mode : 3,
+- dest_mode_logical : 1,
+- reserved : 2,
+- active_low : 1,
+- is_level : 1;
+-
+- u32 dmar_subhandle;
++ union {
++ struct {
++ u32 vector : 8,
++ delivery_mode : 3,
++ dest_mode_logical : 1,
++ reserved : 2,
++ active_low : 1,
++ is_level : 1;
++ };
++ u32 dmar_subhandle;
++ };
+ } __attribute__ ((packed)) arch_msi_msg_data_t;
+ #define arch_msi_msg_data x86_msi_data
+
+diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
+index a7c413432b33d..66ae309840fef 100644
+--- a/arch/x86/include/asm/msr-index.h
++++ b/arch/x86/include/asm/msr-index.h
+@@ -128,9 +128,9 @@
+ #define TSX_CTRL_RTM_DISABLE BIT(0) /* Disable RTM feature */
+ #define TSX_CTRL_CPUID_CLEAR BIT(1) /* Disable TSX enumeration */
+
+-/* SRBDS support */
+ #define MSR_IA32_MCU_OPT_CTRL 0x00000123
+-#define RNGDS_MITG_DIS BIT(0)
++#define RNGDS_MITG_DIS BIT(0) /* SRBDS support */
++#define RTM_ALLOW BIT(1) /* TSX development mode */
+
+ #define MSR_IA32_SYSENTER_CS 0x00000174
+ #define MSR_IA32_SYSENTER_ESP 0x00000175
+diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
+index ec2d5c8c66947..b3dd514f80a0b 100644
+--- a/arch/x86/include/asm/nospec-branch.h
++++ b/arch/x86/include/asm/nospec-branch.h
+@@ -81,7 +81,7 @@
+ #ifdef CONFIG_RETPOLINE
+ ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
+ __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
+- __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD
++ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE
+ #else
+ jmp *%\reg
+ #endif
+@@ -91,7 +91,7 @@
+ #ifdef CONFIG_RETPOLINE
+ ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \
+ __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
+- __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD
++ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE
+ #else
+ call *%\reg
+ #endif
+@@ -133,7 +133,7 @@
+ "lfence;\n" \
+ ANNOTATE_RETPOLINE_SAFE \
+ "call *%[thunk_target]\n", \
+- X86_FEATURE_RETPOLINE_AMD)
++ X86_FEATURE_RETPOLINE_LFENCE)
+
+ # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+
+@@ -163,7 +163,7 @@
+ "lfence;\n" \
+ ANNOTATE_RETPOLINE_SAFE \
+ "call *%[thunk_target]\n", \
+- X86_FEATURE_RETPOLINE_AMD)
++ X86_FEATURE_RETPOLINE_LFENCE)
+
+ # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+ #endif
+@@ -175,9 +175,11 @@
+ /* The Spectre V2 mitigation variants */
+ enum spectre_v2_mitigation {
+ SPECTRE_V2_NONE,
+- SPECTRE_V2_RETPOLINE_GENERIC,
+- SPECTRE_V2_RETPOLINE_AMD,
+- SPECTRE_V2_IBRS_ENHANCED,
++ SPECTRE_V2_RETPOLINE,
++ SPECTRE_V2_LFENCE,
++ SPECTRE_V2_EIBRS,
++ SPECTRE_V2_EIBRS_RETPOLINE,
++ SPECTRE_V2_EIBRS_LFENCE,
+ };
+
+ /* The indirect branch speculation control variants */
+diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
+index a8d4ad8565681..e9e2c3ba59239 100644
+--- a/arch/x86/include/asm/page_64_types.h
++++ b/arch/x86/include/asm/page_64_types.h
+@@ -15,7 +15,7 @@
+ #define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER)
+ #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
+
+-#define EXCEPTION_STACK_ORDER (0 + KASAN_STACK_ORDER)
++#define EXCEPTION_STACK_ORDER (1 + KASAN_STACK_ORDER)
+ #define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+ #define IRQ_STACK_ORDER (2 + KASAN_STACK_ORDER)
+diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
+index da3a1ac82be58..4d8b2731f4f85 100644
+--- a/arch/x86/include/asm/paravirt.h
++++ b/arch/x86/include/asm/paravirt.h
+@@ -665,7 +665,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
+ "call " #func ";" \
+ PV_RESTORE_ALL_CALLER_REGS \
+ FRAME_END \
+- "ret;" \
++ ASM_RET \
+ ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \
+ ".popsection")
+
+diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
+index 8fc1b5003713f..a2b6626c681f5 100644
+--- a/arch/x86/include/asm/perf_event.h
++++ b/arch/x86/include/asm/perf_event.h
+@@ -241,6 +241,11 @@ struct x86_pmu_capability {
+ #define INTEL_PMC_IDX_FIXED_SLOTS (INTEL_PMC_IDX_FIXED + 3)
+ #define INTEL_PMC_MSK_FIXED_SLOTS (1ULL << INTEL_PMC_IDX_FIXED_SLOTS)
+
++static inline bool use_fixed_pseudo_encoding(u64 code)
++{
++ return !(code & 0xff);
++}
++
+ /*
+ * We model BTS tracing as another fixed-mode PMC.
+ *
+diff --git a/arch/x86/include/asm/pkru.h b/arch/x86/include/asm/pkru.h
+index ccc539faa5bbc..4d8b9448fe8d2 100644
+--- a/arch/x86/include/asm/pkru.h
++++ b/arch/x86/include/asm/pkru.h
+@@ -4,8 +4,8 @@
+
+ #include <asm/fpu/xstate.h>
+
+-#define PKRU_AD_BIT 0x1
+-#define PKRU_WD_BIT 0x2
++#define PKRU_AD_BIT 0x1u
++#define PKRU_WD_BIT 0x2u
+ #define PKRU_BITS_PER_PKEY 2
+
+ #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
+diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
+index 9ad2acaaae9b8..577f342dbfb27 100644
+--- a/arch/x86/include/asm/processor.h
++++ b/arch/x86/include/asm/processor.h
+@@ -518,6 +518,7 @@ struct thread_struct {
+ */
+ unsigned long iopl_emul;
+
++ unsigned int iopl_warn:1;
+ unsigned int sig_on_uaccess_err:1;
+
+ /*
+diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
+index 159622ee06748..1474cf96251dd 100644
+--- a/arch/x86/include/asm/qspinlock_paravirt.h
++++ b/arch/x86/include/asm/qspinlock_paravirt.h
+@@ -48,7 +48,7 @@ asm (".pushsection .text;"
+ "jne .slowpath;"
+ "pop %rdx;"
+ FRAME_END
+- "ret;"
++ ASM_RET
+ ".slowpath: "
+ "push %rsi;"
+ "movzbl %al,%esi;"
+@@ -56,7 +56,7 @@ asm (".pushsection .text;"
+ "pop %rsi;"
+ "pop %rdx;"
+ FRAME_END
+- "ret;"
++ ASM_RET
+ ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
+ ".popsection");
+
+diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h
+index 5db5d083c8732..331474b150f16 100644
+--- a/arch/x86/include/asm/realmode.h
++++ b/arch/x86/include/asm/realmode.h
+@@ -89,6 +89,7 @@ static inline void set_real_mode_mem(phys_addr_t mem)
+ }
+
+ void reserve_real_mode(void);
++void load_trampoline_pgtable(void);
+
+ #endif /* __ASSEMBLY__ */
+
+diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h
+index f248eb2ac2d4a..3881b5333eb81 100644
+--- a/arch/x86/include/asm/stacktrace.h
++++ b/arch/x86/include/asm/stacktrace.h
+@@ -38,6 +38,16 @@ int get_stack_info(unsigned long *stack, struct task_struct *task,
+ bool get_stack_info_noinstr(unsigned long *stack, struct task_struct *task,
+ struct stack_info *info);
+
++static __always_inline
++bool get_stack_guard_info(unsigned long *stack, struct stack_info *info)
++{
++ /* make sure it's not in the stack proper */
++ if (get_stack_info_noinstr(stack, current, info))
++ return false;
++ /* but if it is in the page below it, we hit a guard */
++ return get_stack_info_noinstr((void *)stack + PAGE_SIZE, current, info);
++}
++
+ const char *stack_type_name(enum stack_type type);
+
+ static inline bool on_stack(struct stack_info *info, void *addr, size_t len)
+diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h
+index cbb67b6030f97..3432345693927 100644
+--- a/arch/x86/include/asm/static_call.h
++++ b/arch/x86/include/asm/static_call.h
+@@ -35,7 +35,7 @@
+ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")
+
+ #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \
+- __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop")
++ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop")
+
+
+ #define ARCH_ADD_TRAMP_KEY(name) \
+diff --git a/arch/x86/include/asm/timex.h b/arch/x86/include/asm/timex.h
+index a4a8b1b16c0c1..956e4145311b1 100644
+--- a/arch/x86/include/asm/timex.h
++++ b/arch/x86/include/asm/timex.h
+@@ -5,6 +5,15 @@
+ #include <asm/processor.h>
+ #include <asm/tsc.h>
+
++static inline unsigned long random_get_entropy(void)
++{
++ if (!IS_ENABLED(CONFIG_X86_TSC) &&
++ !cpu_feature_enabled(X86_FEATURE_TSC))
++ return random_get_entropy_fallback();
++ return rdtsc();
++}
++#define random_get_entropy random_get_entropy
++
+ /* Assume we use the PIT time source for the clock tick */
+ #define CLOCK_TICK_RATE PIT_TICK_RATE
+
+diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
+index 9239399e54914..55160445ea78b 100644
+--- a/arch/x86/include/asm/topology.h
++++ b/arch/x86/include/asm/topology.h
+@@ -218,7 +218,7 @@ static inline void arch_set_max_freq_ratio(bool turbo_disabled)
+ }
+ #endif
+
+-#ifdef CONFIG_ACPI_CPPC_LIB
++#if defined(CONFIG_ACPI_CPPC_LIB) && defined(CONFIG_SMP)
+ void init_freq_invariance_cppc(void);
+ #define init_freq_invariance_cppc init_freq_invariance_cppc
+ #endif
+diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
+index 7f7200021bd13..6221be7cafc3b 100644
+--- a/arch/x86/include/asm/traps.h
++++ b/arch/x86/include/asm/traps.h
+@@ -40,9 +40,9 @@ void math_emulate(struct math_emu_info *);
+ bool fault_in_kernel_space(unsigned long address);
+
+ #ifdef CONFIG_VMAP_STACK
+-void __noreturn handle_stack_overflow(const char *message,
+- struct pt_regs *regs,
+- unsigned long fault_address);
++void __noreturn handle_stack_overflow(struct pt_regs *regs,
++ unsigned long fault_address,
++ struct stack_info *info);
+ #endif
+
+ #endif /* _ASM_X86_TRAPS_H */
+diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
+index 01a300a9700b9..fbdc3d9514943 100644
+--- a/arch/x86/include/asm/tsc.h
++++ b/arch/x86/include/asm/tsc.h
+@@ -20,13 +20,12 @@ extern void disable_TSC(void);
+
+ static inline cycles_t get_cycles(void)
+ {
+-#ifndef CONFIG_X86_TSC
+- if (!boot_cpu_has(X86_FEATURE_TSC))
++ if (!IS_ENABLED(CONFIG_X86_TSC) &&
++ !cpu_feature_enabled(X86_FEATURE_TSC))
+ return 0;
+-#endif
+-
+ return rdtsc();
+ }
++#define get_cycles get_cycles
+
+ extern struct system_counterval_t convert_art_to_tsc(u64 art);
+ extern struct system_counterval_t convert_art_ns_to_tsc(u64 art_ns);
+diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
+index 5c95d242f38d7..bb1430283c726 100644
+--- a/arch/x86/include/asm/uaccess.h
++++ b/arch/x86/include/asm/uaccess.h
+@@ -314,11 +314,12 @@ do { \
+ do { \
+ __chk_user_ptr(ptr); \
+ switch (size) { \
+- unsigned char x_u8__; \
+- case 1: \
++ case 1: { \
++ unsigned char x_u8__; \
+ __get_user_asm(x_u8__, ptr, "b", "=q", label); \
+ (x) = x_u8__; \
+ break; \
++ } \
+ case 2: \
+ __get_user_asm(x, ptr, "w", "=r", label); \
+ break; \
+diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h
+index ff4b52e37e60d..5adab895127e1 100644
+--- a/arch/x86/include/asm/xen/hypervisor.h
++++ b/arch/x86/include/asm/xen/hypervisor.h
+@@ -62,4 +62,9 @@ void xen_arch_register_cpu(int num);
+ void xen_arch_unregister_cpu(int num);
+ #endif
+
++#ifdef CONFIG_PVH
++void __init xen_pvh_init(struct boot_params *boot_params);
++void __init mem_map_via_hcall(struct boot_params *boot_params_p);
++#endif
++
+ #endif /* _ASM_X86_XEN_HYPERVISOR_H */
+diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
+index 8f4e8fa6ed759..2ff3e600f4269 100644
+--- a/arch/x86/kernel/Makefile
++++ b/arch/x86/kernel/Makefile
+@@ -21,6 +21,7 @@ CFLAGS_REMOVE_ftrace.o = -pg
+ CFLAGS_REMOVE_early_printk.o = -pg
+ CFLAGS_REMOVE_head64.o = -pg
+ CFLAGS_REMOVE_sev.o = -pg
++CFLAGS_REMOVE_cc_platform.o = -pg
+ endif
+
+ KASAN_SANITIZE_head$(BITS).o := n
+@@ -29,6 +30,7 @@ KASAN_SANITIZE_dumpstack_$(BITS).o := n
+ KASAN_SANITIZE_stacktrace.o := n
+ KASAN_SANITIZE_paravirt.o := n
+ KASAN_SANITIZE_sev.o := n
++KASAN_SANITIZE_cc_platform.o := n
+
+ # With some compiler versions the generated code results in boot hangs, caused
+ # by several compilation units. To be safe, disable all instrumentation.
+@@ -47,6 +49,7 @@ endif
+ KCOV_INSTRUMENT := n
+
+ CFLAGS_head$(BITS).o += -fno-stack-protector
++CFLAGS_cc_platform.o += -fno-stack-protector
+
+ CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace
+
+@@ -147,6 +150,9 @@ obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o
+ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
+
+ obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o
++
++obj-$(CONFIG_ARCH_HAS_CC_PLATFORM) += cc_platform.o
++
+ ###
+ # 64 bit specific files
+ ifeq ($(CONFIG_X86_64),y)
+diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
+index 14bcd59bcdee2..94ac7402c1ac2 100644
+--- a/arch/x86/kernel/acpi/boot.c
++++ b/arch/x86/kernel/acpi/boot.c
+@@ -1319,6 +1319,17 @@ static int __init disable_acpi_pci(const struct dmi_system_id *d)
+ return 0;
+ }
+
++static int __init disable_acpi_xsdt(const struct dmi_system_id *d)
++{
++ if (!acpi_force) {
++ pr_notice("%s detected: force use of acpi=rsdt\n", d->ident);
++ acpi_gbl_do_not_use_xsdt = TRUE;
++ } else {
++ pr_notice("Warning: DMI blacklist says broken, but acpi XSDT forced\n");
++ }
++ return 0;
++}
++
+ static int __init dmi_disable_acpi(const struct dmi_system_id *d)
+ {
+ if (!acpi_force) {
+@@ -1442,6 +1453,19 @@ static const struct dmi_system_id acpi_dmi_table[] __initconst = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 360"),
+ },
+ },
++ /*
++ * Boxes that need ACPI XSDT use disabled due to corrupted tables
++ */
++ {
++ .callback = disable_acpi_xsdt,
++ .ident = "Advantech DAC-BJ01",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "NEC"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "Bearlake CRB Board"),
++ DMI_MATCH(DMI_BIOS_VERSION, "V1.12"),
++ DMI_MATCH(DMI_BIOS_DATE, "02/01/2011"),
++ },
++ },
+ {}
+ };
+
+diff --git a/arch/x86/kernel/acpi/wakeup_32.S b/arch/x86/kernel/acpi/wakeup_32.S
+index daf88f8143c5f..cf69081073b54 100644
+--- a/arch/x86/kernel/acpi/wakeup_32.S
++++ b/arch/x86/kernel/acpi/wakeup_32.S
+@@ -60,7 +60,7 @@ save_registers:
+ popl saved_context_eflags
+
+ movl $ret_point, saved_eip
+- ret
++ RET
+
+
+ restore_registers:
+@@ -70,7 +70,7 @@ restore_registers:
+ movl saved_context_edi, %edi
+ pushl saved_context_eflags
+ popfl
+- ret
++ RET
+
+ SYM_CODE_START(do_suspend_lowlevel)
+ call save_processor_state
+@@ -86,7 +86,7 @@ SYM_CODE_START(do_suspend_lowlevel)
+ ret_point:
+ call restore_registers
+ call restore_processor_state
+- ret
++ RET
+ SYM_CODE_END(do_suspend_lowlevel)
+
+ .data
+diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
+index e9da3dc712541..ae0f718b8ebb5 100644
+--- a/arch/x86/kernel/alternative.c
++++ b/arch/x86/kernel/alternative.c
+@@ -537,7 +537,7 @@ asm (
+ " .type int3_magic, @function\n"
+ "int3_magic:\n"
+ " movl $1, (%" _ASM_ARG1 ")\n"
+-" ret\n"
++ ASM_RET
+ " .size int3_magic, .-int3_magic\n"
+ " .popsection\n"
+ );
+@@ -930,10 +930,13 @@ void text_poke_sync(void)
+ }
+
+ struct text_poke_loc {
+- s32 rel_addr; /* addr := _stext + rel_addr */
+- s32 rel32;
++ /* addr := _stext + rel_addr */
++ s32 rel_addr;
++ s32 disp;
++ u8 len;
+ u8 opcode;
+ const u8 text[POKE_MAX_OPCODE_SIZE];
++ /* see text_poke_bp_batch() */
+ u8 old;
+ };
+
+@@ -948,7 +951,8 @@ static struct bp_patching_desc *bp_desc;
+ static __always_inline
+ struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp)
+ {
+- struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */
++ /* rcu_dereference */
++ struct bp_patching_desc *desc = __READ_ONCE(*descp);
+
+ if (!desc || !arch_atomic_inc_not_zero(&desc->refs))
+ return NULL;
+@@ -982,7 +986,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
+ {
+ struct bp_patching_desc *desc;
+ struct text_poke_loc *tp;
+- int len, ret = 0;
++ int ret = 0;
+ void *ip;
+
+ if (user_mode(regs))
+@@ -1022,8 +1026,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
+ goto out_put;
+ }
+
+- len = text_opcode_size(tp->opcode);
+- ip += len;
++ ip += tp->len;
+
+ switch (tp->opcode) {
+ case INT3_INSN_OPCODE:
+@@ -1038,12 +1041,12 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
+ break;
+
+ case CALL_INSN_OPCODE:
+- int3_emulate_call(regs, (long)ip + tp->rel32);
++ int3_emulate_call(regs, (long)ip + tp->disp);
+ break;
+
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+- int3_emulate_jmp(regs, (long)ip + tp->rel32);
++ int3_emulate_jmp(regs, (long)ip + tp->disp);
+ break;
+
+ default:
+@@ -1118,7 +1121,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
+ */
+ for (do_sync = 0, i = 0; i < nr_entries; i++) {
+ u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, };
+- int len = text_opcode_size(tp[i].opcode);
++ int len = tp[i].len;
+
+ if (len - INT3_INSN_SIZE > 0) {
+ memcpy(old + INT3_INSN_SIZE,
+@@ -1195,20 +1198,36 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ const void *opcode, size_t len, const void *emulate)
+ {
+ struct insn insn;
+- int ret;
++ int ret, i;
+
+ memcpy((void *)tp->text, opcode, len);
+ if (!emulate)
+ emulate = opcode;
+
+ ret = insn_decode_kernel(&insn, emulate);
+-
+ BUG_ON(ret < 0);
+- BUG_ON(len != insn.length);
+
+ tp->rel_addr = addr - (void *)_stext;
++ tp->len = len;
+ tp->opcode = insn.opcode.bytes[0];
+
++ switch (tp->opcode) {
++ case RET_INSN_OPCODE:
++ case JMP32_INSN_OPCODE:
++ case JMP8_INSN_OPCODE:
++ /*
++ * Control flow instructions without implied execution of the
++ * next instruction can be padded with INT3.
++ */
++ for (i = insn.length; i < len; i++)
++ BUG_ON(tp->text[i] != INT3_INSN_OPCODE);
++ break;
++
++ default:
++ BUG_ON(len != insn.length);
++ };
++
++
+ switch (tp->opcode) {
+ case INT3_INSN_OPCODE:
+ case RET_INSN_OPCODE:
+@@ -1217,7 +1236,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ case CALL_INSN_OPCODE:
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+- tp->rel32 = insn.immediate.value;
++ tp->disp = insn.immediate.value;
+ break;
+
+ default: /* assume NOP */
+@@ -1225,13 +1244,13 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ case 2: /* NOP2 -- emulate as JMP8+0 */
+ BUG_ON(memcmp(emulate, x86_nops[len], len));
+ tp->opcode = JMP8_INSN_OPCODE;
+- tp->rel32 = 0;
++ tp->disp = 0;
+ break;
+
+ case 5: /* NOP5 -- emulate as JMP32+0 */
+ BUG_ON(memcmp(emulate, x86_nops[len], len));
+ tp->opcode = JMP32_INSN_OPCODE;
+- tp->rel32 = 0;
++ tp->disp = 0;
+ break;
+
+ default: /* unknown instruction */
+diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/kernel/cc_platform.c
+new file mode 100644
+index 0000000000000..03bb2f343ddb7
+--- /dev/null
++++ b/arch/x86/kernel/cc_platform.c
+@@ -0,0 +1,69 @@
++// SPDX-License-Identifier: GPL-2.0-only
++/*
++ * Confidential Computing Platform Capability checks
++ *
++ * Copyright (C) 2021 Advanced Micro Devices, Inc.
++ *
++ * Author: Tom Lendacky <thomas.lendacky@amd.com>
++ */
++
++#include <linux/export.h>
++#include <linux/cc_platform.h>
++#include <linux/mem_encrypt.h>
++
++#include <asm/processor.h>
++
++static bool __maybe_unused intel_cc_platform_has(enum cc_attr attr)
++{
++#ifdef CONFIG_INTEL_TDX_GUEST
++ return false;
++#else
++ return false;
++#endif
++}
++
++/*
++ * SME and SEV are very similar but they are not the same, so there are
++ * times that the kernel will need to distinguish between SME and SEV. The
++ * cc_platform_has() function is used for this. When a distinction isn't
++ * needed, the CC_ATTR_MEM_ENCRYPT attribute can be used.
++ *
++ * The trampoline code is a good example for this requirement. Before
++ * paging is activated, SME will access all memory as decrypted, but SEV
++ * will access all memory as encrypted. So, when APs are being brought
++ * up under SME the trampoline area cannot be encrypted, whereas under SEV
++ * the trampoline area must be encrypted.
++ */
++static bool amd_cc_platform_has(enum cc_attr attr)
++{
++#ifdef CONFIG_AMD_MEM_ENCRYPT
++ switch (attr) {
++ case CC_ATTR_MEM_ENCRYPT:
++ return sme_me_mask;
++
++ case CC_ATTR_HOST_MEM_ENCRYPT:
++ return sme_me_mask && !(sev_status & MSR_AMD64_SEV_ENABLED);
++
++ case CC_ATTR_GUEST_MEM_ENCRYPT:
++ return sev_status & MSR_AMD64_SEV_ENABLED;
++
++ case CC_ATTR_GUEST_STATE_ENCRYPT:
++ return sev_status & MSR_AMD64_SEV_ES_ENABLED;
++
++ default:
++ return false;
++ }
++#else
++ return false;
++#endif
++}
++
++
++bool cc_platform_has(enum cc_attr attr)
++{
++ if (sme_me_mask)
++ return amd_cc_platform_has(attr);
++
++ return false;
++}
++EXPORT_SYMBOL_GPL(cc_platform_has);
+diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
+index 2131af9f2fa23..4edb6f0f628c2 100644
+--- a/arch/x86/kernel/cpu/amd.c
++++ b/arch/x86/kernel/cpu/amd.c
+@@ -989,6 +989,8 @@ static void init_amd(struct cpuinfo_x86 *c)
+ if (cpu_has(c, X86_FEATURE_IRPERF) &&
+ !cpu_has_amd_erratum(c, amd_erratum_1054))
+ msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
++
++ check_null_seg_clears_base(c);
+ }
+
+ #ifdef CONFIG_X86_32
+diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
+index ecfca3bbcd968..13320a06e14bd 100644
+--- a/arch/x86/kernel/cpu/bugs.c
++++ b/arch/x86/kernel/cpu/bugs.c
+@@ -16,6 +16,7 @@
+ #include <linux/prctl.h>
+ #include <linux/sched/smt.h>
+ #include <linux/pgtable.h>
++#include <linux/bpf.h>
+
+ #include <asm/spec-ctrl.h>
+ #include <asm/cmdline.h>
+@@ -650,6 +651,32 @@ static inline const char *spectre_v2_module_string(void)
+ static inline const char *spectre_v2_module_string(void) { return ""; }
+ #endif
+
++#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
++#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
++#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
++
++#ifdef CONFIG_BPF_SYSCALL
++void unpriv_ebpf_notify(int new_state)
++{
++ if (new_state)
++ return;
++
++ /* Unprivileged eBPF is enabled */
++
++ switch (spectre_v2_enabled) {
++ case SPECTRE_V2_EIBRS:
++ pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
++ break;
++ case SPECTRE_V2_EIBRS_LFENCE:
++ if (sched_smt_active())
++ pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
++ break;
++ default:
++ break;
++ }
++}
++#endif
++
+ static inline bool match_option(const char *arg, int arglen, const char *opt)
+ {
+ int len = strlen(opt);
+@@ -664,7 +691,10 @@ enum spectre_v2_mitigation_cmd {
+ SPECTRE_V2_CMD_FORCE,
+ SPECTRE_V2_CMD_RETPOLINE,
+ SPECTRE_V2_CMD_RETPOLINE_GENERIC,
+- SPECTRE_V2_CMD_RETPOLINE_AMD,
++ SPECTRE_V2_CMD_RETPOLINE_LFENCE,
++ SPECTRE_V2_CMD_EIBRS,
++ SPECTRE_V2_CMD_EIBRS_RETPOLINE,
++ SPECTRE_V2_CMD_EIBRS_LFENCE,
+ };
+
+ enum spectre_v2_user_cmd {
+@@ -737,6 +767,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
+ return SPECTRE_V2_USER_CMD_AUTO;
+ }
+
++static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
++{
++ return (mode == SPECTRE_V2_EIBRS ||
++ mode == SPECTRE_V2_EIBRS_RETPOLINE ||
++ mode == SPECTRE_V2_EIBRS_LFENCE);
++}
++
+ static void __init
+ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
+ {
+@@ -804,7 +841,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
+ */
+ if (!boot_cpu_has(X86_FEATURE_STIBP) ||
+ !smt_possible ||
+- spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
++ spectre_v2_in_eibrs_mode(spectre_v2_enabled))
+ return;
+
+ /*
+@@ -824,9 +861,11 @@ set_mode:
+
+ static const char * const spectre_v2_strings[] = {
+ [SPECTRE_V2_NONE] = "Vulnerable",
+- [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
+- [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
+- [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
++ [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines",
++ [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE",
++ [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS",
++ [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
++ [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
+ };
+
+ static const struct {
+@@ -837,8 +876,12 @@ static const struct {
+ { "off", SPECTRE_V2_CMD_NONE, false },
+ { "on", SPECTRE_V2_CMD_FORCE, true },
+ { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
+- { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false },
++ { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
++ { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
+ { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
++ { "eibrs", SPECTRE_V2_CMD_EIBRS, false },
++ { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
++ { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
+ { "auto", SPECTRE_V2_CMD_AUTO, false },
+ };
+
+@@ -875,17 +918,30 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
+ }
+
+ if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
+- cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
+- cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
++ cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
++ cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
++ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
++ cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
+ !IS_ENABLED(CONFIG_RETPOLINE)) {
+- pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
++ pr_err("%s selected but not compiled in. Switching to AUTO select\n",
++ mitigation_options[i].option);
+ return SPECTRE_V2_CMD_AUTO;
+ }
+
+- if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
+- boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
+- boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
+- pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
++ if ((cmd == SPECTRE_V2_CMD_EIBRS ||
++ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
++ cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
++ !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
++ pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
++ mitigation_options[i].option);
++ return SPECTRE_V2_CMD_AUTO;
++ }
++
++ if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
++ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
++ !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
++ pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
++ mitigation_options[i].option);
+ return SPECTRE_V2_CMD_AUTO;
+ }
+
+@@ -894,6 +950,16 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
+ return cmd;
+ }
+
++static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
++{
++ if (!IS_ENABLED(CONFIG_RETPOLINE)) {
++ pr_err("Kernel not compiled with retpoline; no mitigation available!");
++ return SPECTRE_V2_NONE;
++ }
++
++ return SPECTRE_V2_RETPOLINE;
++}
++
+ static void __init spectre_v2_select_mitigation(void)
+ {
+ enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
+@@ -914,49 +980,64 @@ static void __init spectre_v2_select_mitigation(void)
+ case SPECTRE_V2_CMD_FORCE:
+ case SPECTRE_V2_CMD_AUTO:
+ if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
+- mode = SPECTRE_V2_IBRS_ENHANCED;
+- /* Force it so VMEXIT will restore correctly */
+- x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
+- wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
+- goto specv2_set_mode;
++ mode = SPECTRE_V2_EIBRS;
++ break;
+ }
+- if (IS_ENABLED(CONFIG_RETPOLINE))
+- goto retpoline_auto;
++
++ mode = spectre_v2_select_retpoline();
+ break;
+- case SPECTRE_V2_CMD_RETPOLINE_AMD:
+- if (IS_ENABLED(CONFIG_RETPOLINE))
+- goto retpoline_amd;
++
++ case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
++ pr_err(SPECTRE_V2_LFENCE_MSG);
++ mode = SPECTRE_V2_LFENCE;
+ break;
++
+ case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
+- if (IS_ENABLED(CONFIG_RETPOLINE))
+- goto retpoline_generic;
++ mode = SPECTRE_V2_RETPOLINE;
+ break;
++
+ case SPECTRE_V2_CMD_RETPOLINE:
+- if (IS_ENABLED(CONFIG_RETPOLINE))
+- goto retpoline_auto;
++ mode = spectre_v2_select_retpoline();
++ break;
++
++ case SPECTRE_V2_CMD_EIBRS:
++ mode = SPECTRE_V2_EIBRS;
++ break;
++
++ case SPECTRE_V2_CMD_EIBRS_LFENCE:
++ mode = SPECTRE_V2_EIBRS_LFENCE;
++ break;
++
++ case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
++ mode = SPECTRE_V2_EIBRS_RETPOLINE;
+ break;
+ }
+- pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
+- return;
+
+-retpoline_auto:
+- if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
+- boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
+- retpoline_amd:
+- if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
+- pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
+- goto retpoline_generic;
+- }
+- mode = SPECTRE_V2_RETPOLINE_AMD;
+- setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
+- setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+- } else {
+- retpoline_generic:
+- mode = SPECTRE_V2_RETPOLINE_GENERIC;
++ if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
++ pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
++
++ if (spectre_v2_in_eibrs_mode(mode)) {
++ /* Force it so VMEXIT will restore correctly */
++ x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
++ wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
++ }
++
++ switch (mode) {
++ case SPECTRE_V2_NONE:
++ case SPECTRE_V2_EIBRS:
++ break;
++
++ case SPECTRE_V2_LFENCE:
++ case SPECTRE_V2_EIBRS_LFENCE:
++ setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
++ fallthrough;
++
++ case SPECTRE_V2_RETPOLINE:
++ case SPECTRE_V2_EIBRS_RETPOLINE:
+ setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
++ break;
+ }
+
+-specv2_set_mode:
+ spectre_v2_enabled = mode;
+ pr_info("%s\n", spectre_v2_strings[mode]);
+
+@@ -982,7 +1063,7 @@ specv2_set_mode:
+ * the CPU supports Enhanced IBRS, kernel might un-intentionally not
+ * enable IBRS around firmware calls.
+ */
+- if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
++ if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
+ setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
+ pr_info("Enabling Restricted Speculation for firmware calls\n");
+ }
+@@ -1052,6 +1133,10 @@ void cpu_bugs_smt_update(void)
+ {
+ mutex_lock(&spec_ctrl_mutex);
+
++ if (sched_smt_active() && unprivileged_ebpf_enabled() &&
++ spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
++ pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
++
+ switch (spectre_v2_user_stibp) {
+ case SPECTRE_V2_USER_NONE:
+ break;
+@@ -1691,7 +1776,7 @@ static ssize_t tsx_async_abort_show_state(char *buf)
+
+ static char *stibp_state(void)
+ {
+- if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
++ if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
+ return "";
+
+ switch (spectre_v2_user_stibp) {
+@@ -1721,6 +1806,27 @@ static char *ibpb_state(void)
+ return "";
+ }
+
++static ssize_t spectre_v2_show_state(char *buf)
++{
++ if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
++ return sprintf(buf, "Vulnerable: LFENCE\n");
++
++ if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
++ return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
++
++ if (sched_smt_active() && unprivileged_ebpf_enabled() &&
++ spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
++ return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
++
++ return sprintf(buf, "%s%s%s%s%s%s\n",
++ spectre_v2_strings[spectre_v2_enabled],
++ ibpb_state(),
++ boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
++ stibp_state(),
++ boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
++ spectre_v2_module_string());
++}
++
+ static ssize_t srbds_show_state(char *buf)
+ {
+ return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
+@@ -1746,12 +1852,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
+ return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
+
+ case X86_BUG_SPECTRE_V2:
+- return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+- ibpb_state(),
+- boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
+- stibp_state(),
+- boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
+- spectre_v2_module_string());
++ return spectre_v2_show_state(buf);
+
+ case X86_BUG_SPEC_STORE_BYPASS:
+ return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
+diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
+index b3410f1ac2175..ec7d3ff5c6629 100644
+--- a/arch/x86/kernel/cpu/common.c
++++ b/arch/x86/kernel/cpu/common.c
+@@ -1396,9 +1396,8 @@ void __init early_cpu_init(void)
+ early_identify_cpu(&boot_cpu_data);
+ }
+
+-static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
++static bool detect_null_seg_behavior(void)
+ {
+-#ifdef CONFIG_X86_64
+ /*
+ * Empirically, writing zero to a segment selector on AMD does
+ * not clear the base, whereas writing zero to a segment
+@@ -1419,10 +1418,43 @@ static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
+ wrmsrl(MSR_FS_BASE, 1);
+ loadsegment(fs, 0);
+ rdmsrl(MSR_FS_BASE, tmp);
+- if (tmp != 0)
+- set_cpu_bug(c, X86_BUG_NULL_SEG);
+ wrmsrl(MSR_FS_BASE, old_base);
+-#endif
++ return tmp == 0;
++}
++
++void check_null_seg_clears_base(struct cpuinfo_x86 *c)
++{
++ /* BUG_NULL_SEG is only relevant with 64bit userspace */
++ if (!IS_ENABLED(CONFIG_X86_64))
++ return;
++
++ /* Zen3 CPUs advertise Null Selector Clears Base in CPUID. */
++ if (c->extended_cpuid_level >= 0x80000021 &&
++ cpuid_eax(0x80000021) & BIT(6))
++ return;
++
++ /*
++ * CPUID bit above wasn't set. If this kernel is still running
++ * as a HV guest, then the HV has decided not to advertize
++ * that CPUID bit for whatever reason. For example, one
++ * member of the migration pool might be vulnerable. Which
++ * means, the bug is present: set the BUG flag and return.
++ */
++ if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
++ set_cpu_bug(c, X86_BUG_NULL_SEG);
++ return;
++ }
++
++ /*
++ * Zen2 CPUs also have this behaviour, but no CPUID bit.
++ * 0x18 is the respective family for Hygon.
++ */
++ if ((c->x86 == 0x17 || c->x86 == 0x18) &&
++ detect_null_seg_behavior())
++ return;
++
++ /* All the remaining ones are affected */
++ set_cpu_bug(c, X86_BUG_NULL_SEG);
+ }
+
+ static void generic_identify(struct cpuinfo_x86 *c)
+@@ -1458,8 +1490,6 @@ static void generic_identify(struct cpuinfo_x86 *c)
+
+ get_model_name(c); /* Default name */
+
+- detect_null_seg_behavior(c);
+-
+ /*
+ * ESPFIX is a strange bug. All real CPUs have it. Paravirt
+ * systems that run Linux at CPL > 0 may or may not have the
+@@ -1684,6 +1714,8 @@ void identify_secondary_cpu(struct cpuinfo_x86 *c)
+ validate_apic_and_package_id(c);
+ x86_spec_ctrl_setup_ap();
+ update_srbds_msr();
++
++ tsx_ap_init();
+ }
+
+ static __init int setup_noclflush(char *arg)
+diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
+index 95521302630d4..2a8e584fc9913 100644
+--- a/arch/x86/kernel/cpu/cpu.h
++++ b/arch/x86/kernel/cpu/cpu.h
+@@ -55,11 +55,10 @@ enum tsx_ctrl_states {
+ extern __ro_after_init enum tsx_ctrl_states tsx_ctrl_state;
+
+ extern void __init tsx_init(void);
+-extern void tsx_enable(void);
+-extern void tsx_disable(void);
+-extern void tsx_clear_cpuid(void);
++void tsx_ap_init(void);
+ #else
+ static inline void tsx_init(void) { }
++static inline void tsx_ap_init(void) { }
+ #endif /* CONFIG_CPU_SUP_INTEL */
+
+ extern void get_cpu_cap(struct cpuinfo_x86 *c);
+@@ -75,6 +74,7 @@ extern int detect_extended_topology_early(struct cpuinfo_x86 *c);
+ extern int detect_extended_topology(struct cpuinfo_x86 *c);
+ extern int detect_ht_early(struct cpuinfo_x86 *c);
+ extern void detect_ht(struct cpuinfo_x86 *c);
++extern void check_null_seg_clears_base(struct cpuinfo_x86 *c);
+
+ unsigned int aperfmperf_get_khz(int cpu);
+
+diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
+index 6d50136f7ab98..3fcdda4c1e114 100644
+--- a/arch/x86/kernel/cpu/hygon.c
++++ b/arch/x86/kernel/cpu/hygon.c
+@@ -335,6 +335,8 @@ static void init_hygon(struct cpuinfo_x86 *c)
+ /* Hygon CPUs don't reset SS attributes on SYSRET, Xen does. */
+ if (!cpu_has(c, X86_FEATURE_XENPV))
+ set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
++
++ check_null_seg_clears_base(c);
+ }
+
+ static void cpu_detect_tlb_hygon(struct cpuinfo_x86 *c)
+diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
+index 8321c43554a1d..f7a5370a9b3b8 100644
+--- a/arch/x86/kernel/cpu/intel.c
++++ b/arch/x86/kernel/cpu/intel.c
+@@ -717,13 +717,6 @@ static void init_intel(struct cpuinfo_x86 *c)
+
+ init_intel_misc_features(c);
+
+- if (tsx_ctrl_state == TSX_CTRL_ENABLE)
+- tsx_enable();
+- else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
+- tsx_disable();
+- else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
+- tsx_clear_cpuid();
+-
+ split_lock_init();
+ bus_lock_init();
+
+diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c
+index 08831acc1d036..c0c57bd05f02d 100644
+--- a/arch/x86/kernel/cpu/mce/amd.c
++++ b/arch/x86/kernel/cpu/mce/amd.c
+@@ -400,7 +400,7 @@ static void threshold_restart_bank(void *_tr)
+ u32 hi, lo;
+
+ /* sysfs write might race against an offline operation */
+- if (this_cpu_read(threshold_banks))
++ if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off)
+ return;
+
+ rdmsr(tr->b->address, lo, hi);
+diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
+index 193204aee8801..e23e74e2f928d 100644
+--- a/arch/x86/kernel/cpu/mce/core.c
++++ b/arch/x86/kernel/cpu/mce/core.c
+@@ -295,11 +295,17 @@ static void wait_for_panic(void)
+ panic("Panicing machine check CPU died");
+ }
+
+-static void mce_panic(const char *msg, struct mce *final, char *exp)
++static noinstr void mce_panic(const char *msg, struct mce *final, char *exp)
+ {
+- int apei_err = 0;
+ struct llist_node *pending;
+ struct mce_evt_llist *l;
++ int apei_err = 0;
++
++ /*
++ * Allow instrumentation around external facilities usage. Not that it
++ * matters a whole lot since the machine is going to panic anyway.
++ */
++ instrumentation_begin();
+
+ if (!fake_panic) {
+ /*
+@@ -314,7 +320,7 @@ static void mce_panic(const char *msg, struct mce *final, char *exp)
+ } else {
+ /* Don't log too much for fake panic */
+ if (atomic_inc_return(&mce_fake_panicked) > 1)
+- return;
++ goto out;
+ }
+ pending = mce_gen_pool_prepare_records();
+ /* First print corrected ones that are still unlogged */
+@@ -352,6 +358,9 @@ static void mce_panic(const char *msg, struct mce *final, char *exp)
+ panic(msg);
+ } else
+ pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
++
++out:
++ instrumentation_end();
+ }
+
+ /* Support code for software error injection */
+@@ -682,7 +691,7 @@ static struct notifier_block mce_default_nb = {
+ /*
+ * Read ADDR and MISC registers.
+ */
+-static void mce_read_aux(struct mce *m, int i)
++static noinstr void mce_read_aux(struct mce *m, int i)
+ {
+ if (m->status & MCI_STATUS_MISCV)
+ m->misc = mce_rdmsrl(msr_ops.misc(i));
+@@ -1072,10 +1081,13 @@ static int mce_start(int *no_way_out)
+ * Synchronize between CPUs after main scanning loop.
+ * This invokes the bulk of the Monarch processing.
+ */
+-static int mce_end(int order)
++static noinstr int mce_end(int order)
+ {
+- int ret = -1;
+ u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
++ int ret = -1;
++
++ /* Allow instrumentation around external facilities. */
++ instrumentation_begin();
+
+ if (!timeout)
+ goto reset;
+@@ -1119,7 +1131,8 @@ static int mce_end(int order)
+ /*
+ * Don't reset anything. That's done by the Monarch.
+ */
+- return 0;
++ ret = 0;
++ goto out;
+ }
+
+ /*
+@@ -1135,6 +1148,10 @@ reset:
+ * Let others run again.
+ */
+ atomic_set(&mce_executing, 0);
++
++out:
++ instrumentation_end();
++
+ return ret;
+ }
+
+@@ -1454,6 +1471,14 @@ noinstr void do_machine_check(struct pt_regs *regs)
+ if (worst != MCE_AR_SEVERITY && !kill_current_task)
+ goto out;
+
++ /*
++ * Enable instrumentation around the external facilities like
++ * task_work_add() (via queue_task_work()), fixup_exception() etc.
++ * For now, that is. Fixing this properly would need a lot more involved
++ * reorganization.
++ */
++ instrumentation_begin();
++
+ /* Fault was in user mode and we need to take some action */
+ if ((m.cs & 3) == 3) {
+ /* If this triggers there is no way to recover. Die hard. */
+@@ -1479,6 +1504,9 @@ noinstr void do_machine_check(struct pt_regs *regs)
+ if (m.kflags & MCE_IN_KERNEL_COPYIN)
+ queue_task_work(&m, msg, kill_current_task);
+ }
++
++ instrumentation_end();
++
+ out:
+ mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
+ }
+diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c
+index 0bfc14041bbb4..b63b548497c14 100644
+--- a/arch/x86/kernel/cpu/mce/inject.c
++++ b/arch/x86/kernel/cpu/mce/inject.c
+@@ -350,7 +350,7 @@ static ssize_t flags_write(struct file *filp, const char __user *ubuf,
+ char buf[MAX_FLAG_OPT_SIZE], *__buf;
+ int err;
+
+- if (cnt > MAX_FLAG_OPT_SIZE)
++ if (!cnt || cnt > MAX_FLAG_OPT_SIZE)
+ return -EINVAL;
+
+ if (copy_from_user(&buf, ubuf, cnt))
+diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c
+index acfd5d9f93c68..baafbb37be678 100644
+--- a/arch/x86/kernel/cpu/mce/intel.c
++++ b/arch/x86/kernel/cpu/mce/intel.c
+@@ -486,6 +486,7 @@ static void intel_ppin_init(struct cpuinfo_x86 *c)
+ case INTEL_FAM6_BROADWELL_X:
+ case INTEL_FAM6_SKYLAKE_X:
+ case INTEL_FAM6_ICELAKE_X:
++ case INTEL_FAM6_ICELAKE_D:
+ case INTEL_FAM6_SAPPHIRERAPIDS_X:
+ case INTEL_FAM6_XEON_PHI_KNL:
+ case INTEL_FAM6_XEON_PHI_KNM:
+@@ -547,12 +548,13 @@ bool intel_filter_mce(struct mce *m)
+ {
+ struct cpuinfo_x86 *c = &boot_cpu_data;
+
+- /* MCE errata HSD131, HSM142, HSW131, BDM48, and HSM142 */
++ /* MCE errata HSD131, HSM142, HSW131, BDM48, HSM142 and SKX37 */
+ if ((c->x86 == 6) &&
+ ((c->x86_model == INTEL_FAM6_HASWELL) ||
+ (c->x86_model == INTEL_FAM6_HASWELL_L) ||
+ (c->x86_model == INTEL_FAM6_BROADWELL) ||
+- (c->x86_model == INTEL_FAM6_HASWELL_G)) &&
++ (c->x86_model == INTEL_FAM6_HASWELL_G) ||
++ (c->x86_model == INTEL_FAM6_SKYLAKE_X)) &&
+ (m->bank == 0) &&
+ ((m->status & 0xa0000000ffffffff) == 0x80000000000f0005))
+ return true;
+diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
+index efb69be41ab18..150ebfb8c12ed 100644
+--- a/arch/x86/kernel/cpu/microcode/core.c
++++ b/arch/x86/kernel/cpu/microcode/core.c
+@@ -775,9 +775,9 @@ static struct subsys_interface mc_cpu_interface = {
+ };
+
+ /**
+- * mc_bp_resume - Update boot CPU microcode during resume.
++ * microcode_bsp_resume - Update boot CPU microcode during resume.
+ */
+-static void mc_bp_resume(void)
++void microcode_bsp_resume(void)
+ {
+ int cpu = smp_processor_id();
+ struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+@@ -789,7 +789,7 @@ static void mc_bp_resume(void)
+ }
+
+ static struct syscore_ops mc_syscore_ops = {
+- .resume = mc_bp_resume,
++ .resume = microcode_bsp_resume,
+ };
+
+ static int mc_cpu_starting(unsigned int cpu)
+diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
+index e095c28d27ae8..ba0efc30fac52 100644
+--- a/arch/x86/kernel/cpu/mshyperv.c
++++ b/arch/x86/kernel/cpu/mshyperv.c
+@@ -79,7 +79,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0)
+ inc_irq_stat(hyperv_stimer0_count);
+ if (hv_stimer0_handler)
+ hv_stimer0_handler();
+- add_interrupt_randomness(HYPERV_STIMER0_VECTOR, 0);
++ add_interrupt_randomness(HYPERV_STIMER0_VECTOR);
+ ack_APIC_irq();
+
+ set_irq_regs(old_regs);
+@@ -163,12 +163,22 @@ static uint32_t __init ms_hyperv_platform(void)
+ cpuid(HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS,
+ &eax, &hyp_signature[0], &hyp_signature[1], &hyp_signature[2]);
+
+- if (eax >= HYPERV_CPUID_MIN &&
+- eax <= HYPERV_CPUID_MAX &&
+- !memcmp("Microsoft Hv", hyp_signature, 12))
+- return HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
++ if (eax < HYPERV_CPUID_MIN || eax > HYPERV_CPUID_MAX ||
++ memcmp("Microsoft Hv", hyp_signature, 12))
++ return 0;
+
+- return 0;
++ /* HYPERCALL and VP_INDEX MSRs are mandatory for all features. */
++ eax = cpuid_eax(HYPERV_CPUID_FEATURES);
++ if (!(eax & HV_MSR_HYPERCALL_AVAILABLE)) {
++ pr_warn("x86/hyperv: HYPERCALL MSR not available.\n");
++ return 0;
++ }
++ if (!(eax & HV_MSR_VP_INDEX_AVAILABLE)) {
++ pr_warn("x86/hyperv: VP_INDEX MSR not available.\n");
++ return 0;
++ }
++
++ return HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
+ }
+
+ static unsigned char hv_get_nmi_reason(void)
+diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c
+index 001808e3901cc..7c63a1911fae9 100644
+--- a/arch/x86/kernel/cpu/sgx/encl.c
++++ b/arch/x86/kernel/cpu/sgx/encl.c
+@@ -12,6 +12,30 @@
+ #include "encls.h"
+ #include "sgx.h"
+
++/*
++ * Calculate byte offset of a PCMD struct associated with an enclave page. PCMD's
++ * follow right after the EPC data in the backing storage. In addition to the
++ * visible enclave pages, there's one extra page slot for SECS, before PCMD
++ * structs.
++ */
++static inline pgoff_t sgx_encl_get_backing_page_pcmd_offset(struct sgx_encl *encl,
++ unsigned long page_index)
++{
++ pgoff_t epc_end_off = encl->size + sizeof(struct sgx_secs);
++
++ return epc_end_off + page_index * sizeof(struct sgx_pcmd);
++}
++
++/*
++ * Free a page from the backing storage in the given page index.
++ */
++static inline void sgx_encl_truncate_backing_page(struct sgx_encl *encl, unsigned long page_index)
++{
++ struct inode *inode = file_inode(encl->backing);
++
++ shmem_truncate_range(inode, PFN_PHYS(page_index), PFN_PHYS(page_index) + PAGE_SIZE - 1);
++}
++
+ /*
+ * ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC
+ * Pages" in the SDM.
+@@ -22,9 +46,11 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page,
+ {
+ unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK;
+ struct sgx_encl *encl = encl_page->encl;
++ pgoff_t page_index, page_pcmd_off;
+ struct sgx_pageinfo pginfo;
+ struct sgx_backing b;
+- pgoff_t page_index;
++ bool pcmd_page_empty;
++ u8 *pcmd_page;
+ int ret;
+
+ if (secs_page)
+@@ -32,14 +58,16 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page,
+ else
+ page_index = PFN_DOWN(encl->size);
+
++ page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index);
++
+ ret = sgx_encl_get_backing(encl, page_index, &b);
+ if (ret)
+ return ret;
+
+ pginfo.addr = encl_page->desc & PAGE_MASK;
+ pginfo.contents = (unsigned long)kmap_atomic(b.contents);
+- pginfo.metadata = (unsigned long)kmap_atomic(b.pcmd) +
+- b.pcmd_offset;
++ pcmd_page = kmap_atomic(b.pcmd);
++ pginfo.metadata = (unsigned long)pcmd_page + b.pcmd_offset;
+
+ if (secs_page)
+ pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page);
+@@ -55,11 +83,24 @@ static int __sgx_encl_eldu(struct sgx_encl_page *encl_page,
+ ret = -EFAULT;
+ }
+
+- kunmap_atomic((void *)(unsigned long)(pginfo.metadata - b.pcmd_offset));
++ memset(pcmd_page + b.pcmd_offset, 0, sizeof(struct sgx_pcmd));
++
++ /*
++ * The area for the PCMD in the page was zeroed above. Check if the
++ * whole page is now empty meaning that all PCMD's have been zeroed:
++ */
++ pcmd_page_empty = !memchr_inv(pcmd_page, 0, PAGE_SIZE);
++
++ kunmap_atomic(pcmd_page);
+ kunmap_atomic((void *)(unsigned long)pginfo.contents);
+
+ sgx_encl_put_backing(&b, false);
+
++ sgx_encl_truncate_backing_page(encl, page_index);
++
++ if (pcmd_page_empty)
++ sgx_encl_truncate_backing_page(encl, PFN_DOWN(page_pcmd_off));
++
+ return ret;
+ }
+
+@@ -410,6 +451,8 @@ void sgx_encl_release(struct kref *ref)
+ }
+
+ kfree(entry);
++ /* Invoke scheduler to prevent soft lockups. */
++ cond_resched();
+ }
+
+ xa_destroy(&encl->page_array);
+@@ -577,7 +620,7 @@ static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl,
+ int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
+ struct sgx_backing *backing)
+ {
+- pgoff_t pcmd_index = PFN_DOWN(encl->size) + 1 + (page_index >> 5);
++ pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index);
+ struct page *contents;
+ struct page *pcmd;
+
+@@ -585,7 +628,7 @@ int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
+ if (IS_ERR(contents))
+ return PTR_ERR(contents);
+
+- pcmd = sgx_encl_get_backing_page(encl, pcmd_index);
++ pcmd = sgx_encl_get_backing_page(encl, PFN_DOWN(page_pcmd_off));
+ if (IS_ERR(pcmd)) {
+ put_page(contents);
+ return PTR_ERR(pcmd);
+@@ -594,9 +637,7 @@ int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
+ backing->page_index = page_index;
+ backing->contents = contents;
+ backing->pcmd = pcmd;
+- backing->pcmd_offset =
+- (page_index & (PAGE_SIZE / sizeof(struct sgx_pcmd) - 1)) *
+- sizeof(struct sgx_pcmd);
++ backing->pcmd_offset = page_pcmd_off & (PAGE_SIZE - 1);
+
+ return 0;
+ }
+diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c
+index 63d3de02bbccb..8471a8b9b48e8 100644
+--- a/arch/x86/kernel/cpu/sgx/main.c
++++ b/arch/x86/kernel/cpu/sgx/main.c
+@@ -28,8 +28,7 @@ static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq);
+ static LIST_HEAD(sgx_active_page_list);
+ static DEFINE_SPINLOCK(sgx_reclaimer_lock);
+
+-/* The free page list lock protected variables prepend the lock. */
+-static unsigned long sgx_nr_free_pages;
++static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0);
+
+ /* Nodes with one or more EPC sections. */
+ static nodemask_t sgx_numa_mask;
+@@ -403,14 +402,15 @@ skip:
+
+ spin_lock(&node->lock);
+ list_add_tail(&epc_page->list, &node->free_page_list);
+- sgx_nr_free_pages++;
+ spin_unlock(&node->lock);
++ atomic_long_inc(&sgx_nr_free_pages);
+ }
+ }
+
+ static bool sgx_should_reclaim(unsigned long watermark)
+ {
+- return sgx_nr_free_pages < watermark && !list_empty(&sgx_active_page_list);
++ return atomic_long_read(&sgx_nr_free_pages) < watermark &&
++ !list_empty(&sgx_active_page_list);
+ }
+
+ static int ksgxd(void *p)
+@@ -471,9 +471,9 @@ static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid)
+
+ page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list);
+ list_del_init(&page->list);
+- sgx_nr_free_pages--;
+
+ spin_unlock(&node->lock);
++ atomic_long_dec(&sgx_nr_free_pages);
+
+ return page;
+ }
+@@ -625,9 +625,9 @@ void sgx_free_epc_page(struct sgx_epc_page *page)
+ spin_lock(&node->lock);
+
+ list_add_tail(&page->list, &node->free_page_list);
+- sgx_nr_free_pages++;
+
+ spin_unlock(&node->lock);
++ atomic_long_inc(&sgx_nr_free_pages);
+ }
+
+ static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size,
+diff --git a/arch/x86/kernel/cpu/tsx.c b/arch/x86/kernel/cpu/tsx.c
+index 9c7a5f0492929..ec7bbac3a9f29 100644
+--- a/arch/x86/kernel/cpu/tsx.c
++++ b/arch/x86/kernel/cpu/tsx.c
+@@ -19,7 +19,7 @@
+
+ enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;
+
+-void tsx_disable(void)
++static void tsx_disable(void)
+ {
+ u64 tsx;
+
+@@ -39,7 +39,7 @@ void tsx_disable(void)
+ wrmsrl(MSR_IA32_TSX_CTRL, tsx);
+ }
+
+-void tsx_enable(void)
++static void tsx_enable(void)
+ {
+ u64 tsx;
+
+@@ -58,7 +58,7 @@ void tsx_enable(void)
+ wrmsrl(MSR_IA32_TSX_CTRL, tsx);
+ }
+
+-static bool __init tsx_ctrl_is_supported(void)
++static bool tsx_ctrl_is_supported(void)
+ {
+ u64 ia32_cap = x86_read_arch_cap_msr();
+
+@@ -84,7 +84,45 @@ static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
+ return TSX_CTRL_ENABLE;
+ }
+
+-void tsx_clear_cpuid(void)
++/*
++ * Disabling TSX is not a trivial business.
++ *
++ * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
++ * which says that TSX is practically disabled (all transactions are
++ * aborted by default). When that bit is set, the kernel unconditionally
++ * disables TSX.
++ *
++ * In order to do that, however, it needs to dance a bit:
++ *
++ * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
++ * the MSR is present only when *two* CPUID bits are set:
++ *
++ * - X86_FEATURE_RTM_ALWAYS_ABORT
++ * - X86_FEATURE_TSX_FORCE_ABORT
++ *
++ * 2. The second method is for CPUs which do not have the above-mentioned
++ * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
++ * through that one. Those CPUs can also have the initially mentioned
++ * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
++ * applies: TSX gets disabled unconditionally.
++ *
++ * When either of the two methods are present, the kernel disables TSX and
++ * clears the respective RTM and HLE feature flags.
++ *
++ * An additional twist in the whole thing presents late microcode loading
++ * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
++ * bit to be set after the update.
++ *
++ * A subsequent hotplug operation on any logical CPU except the BSP will
++ * cause for the supported CPUID feature bits to get re-detected and, if
++ * RTM and HLE get cleared all of a sudden, but, userspace did consult
++ * them before the update, then funny explosions will happen. Long story
++ * short: the kernel doesn't modify CPUID feature bits after booting.
++ *
++ * That's why, this function's call in init_intel() doesn't clear the
++ * feature flags.
++ */
++static void tsx_clear_cpuid(void)
+ {
+ u64 msr;
+
+@@ -97,6 +135,39 @@ void tsx_clear_cpuid(void)
+ rdmsrl(MSR_TSX_FORCE_ABORT, msr);
+ msr |= MSR_TFA_TSX_CPUID_CLEAR;
+ wrmsrl(MSR_TSX_FORCE_ABORT, msr);
++ } else if (tsx_ctrl_is_supported()) {
++ rdmsrl(MSR_IA32_TSX_CTRL, msr);
++ msr |= TSX_CTRL_CPUID_CLEAR;
++ wrmsrl(MSR_IA32_TSX_CTRL, msr);
++ }
++}
++
++/*
++ * Disable TSX development mode
++ *
++ * When the microcode released in Feb 2022 is applied, TSX will be disabled by
++ * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
++ * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
++ * not recommended for production deployments. In particular, applying MD_CLEAR
++ * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
++ * execution attack may not be effective on these processors when Intel TSX is
++ * enabled with updated microcode.
++ */
++static void tsx_dev_mode_disable(void)
++{
++ u64 mcu_opt_ctrl;
++
++ /* Check if RTM_ALLOW exists */
++ if (!boot_cpu_has_bug(X86_BUG_TAA) || !tsx_ctrl_is_supported() ||
++ !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
++ return;
++
++ rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
++
++ if (mcu_opt_ctrl & RTM_ALLOW) {
++ mcu_opt_ctrl &= ~RTM_ALLOW;
++ wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
++ setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
+ }
+ }
+
+@@ -105,14 +176,14 @@ void __init tsx_init(void)
+ char arg[5] = {};
+ int ret;
+
++ tsx_dev_mode_disable();
++
+ /*
+- * Hardware will always abort a TSX transaction if both CPUID bits
+- * RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are set. In this case, it is
+- * better not to enumerate CPUID.RTM and CPUID.HLE bits. Clear them
+- * here.
++ * Hardware will always abort a TSX transaction when the CPUID bit
++ * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
++ * CPUID.RTM and CPUID.HLE bits. Clear them here.
+ */
+- if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
+- boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
++ if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
+ tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
+ tsx_clear_cpuid();
+ setup_clear_cpu_cap(X86_FEATURE_RTM);
+@@ -175,3 +246,16 @@ void __init tsx_init(void)
+ setup_force_cpu_cap(X86_FEATURE_HLE);
+ }
+ }
++
++void tsx_ap_init(void)
++{
++ tsx_dev_mode_disable();
++
++ if (tsx_ctrl_state == TSX_CTRL_ENABLE)
++ tsx_enable();
++ else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
++ tsx_disable();
++ else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
++ /* See comment over that function for more details. */
++ tsx_clear_cpuid();
++}
+diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
+index 5601b95944fae..6c5defd6569a3 100644
+--- a/arch/x86/kernel/dumpstack_64.c
++++ b/arch/x86/kernel/dumpstack_64.c
+@@ -32,9 +32,15 @@ const char *stack_type_name(enum stack_type type)
+ {
+ BUILD_BUG_ON(N_EXCEPTION_STACKS != 6);
+
++ if (type == STACK_TYPE_TASK)
++ return "TASK";
++
+ if (type == STACK_TYPE_IRQ)
+ return "IRQ";
+
++ if (type == STACK_TYPE_SOFTIRQ)
++ return "SOFTIRQ";
++
+ if (type == STACK_TYPE_ENTRY) {
+ /*
+ * On 64-bit, we have a generic entry stack that we
+diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
+index bc0657f0deedf..f267205f2d5a4 100644
+--- a/arch/x86/kernel/e820.c
++++ b/arch/x86/kernel/e820.c
+@@ -995,8 +995,10 @@ early_param("memmap", parse_memmap_opt);
+ */
+ void __init e820__reserve_setup_data(void)
+ {
++ struct setup_indirect *indirect;
+ struct setup_data *data;
+- u64 pa_data;
++ u64 pa_data, pa_next;
++ u32 len;
+
+ pa_data = boot_params.hdr.setup_data;
+ if (!pa_data)
+@@ -1004,6 +1006,14 @@ void __init e820__reserve_setup_data(void)
+
+ while (pa_data) {
+ data = early_memremap(pa_data, sizeof(*data));
++ if (!data) {
++ pr_warn("e820: failed to memremap setup_data entry\n");
++ return;
++ }
++
++ len = sizeof(*data);
++ pa_next = data->next;
++
+ e820__range_update(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
+
+ /*
+@@ -1015,18 +1025,27 @@ void __init e820__reserve_setup_data(void)
+ sizeof(*data) + data->len,
+ E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
+
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) {
+- e820__range_update(((struct setup_indirect *)data->data)->addr,
+- ((struct setup_indirect *)data->data)->len,
+- E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
+- e820__range_update_kexec(((struct setup_indirect *)data->data)->addr,
+- ((struct setup_indirect *)data->data)->len,
+- E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
++ if (data->type == SETUP_INDIRECT) {
++ len += data->len;
++ early_memunmap(data, sizeof(*data));
++ data = early_memremap(pa_data, len);
++ if (!data) {
++ pr_warn("e820: failed to memremap indirect setup_data\n");
++ return;
++ }
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT) {
++ e820__range_update(indirect->addr, indirect->len,
++ E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
++ e820__range_update_kexec(indirect->addr, indirect->len,
++ E820_TYPE_RAM, E820_TYPE_RESERVED_KERN);
++ }
+ }
+
+- pa_data = data->next;
+- early_memunmap(data, sizeof(*data));
++ pa_data = pa_next;
++ early_memunmap(data, len);
+ }
+
+ e820__update_table(e820_table);
+diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
+index 391a4e2b86049..8690fab95ae4b 100644
+--- a/arch/x86/kernel/early-quirks.c
++++ b/arch/x86/kernel/early-quirks.c
+@@ -515,6 +515,7 @@ static const struct intel_early_ops gen11_early_ops __initconst = {
+ .stolen_size = gen9_stolen_size,
+ };
+
++/* Intel integrated GPUs for which we need to reserve "stolen memory" */
+ static const struct pci_device_id intel_early_ids[] __initconst = {
+ INTEL_I830_IDS(&i830_early_ops),
+ INTEL_I845G_IDS(&i845_early_ops),
+@@ -591,6 +592,13 @@ static void __init intel_graphics_quirks(int num, int slot, int func)
+ u16 device;
+ int i;
+
++ /*
++ * Reserve "stolen memory" for an integrated GPU. If we've already
++ * found one, there's nothing to do for other (discrete) GPUs.
++ */
++ if (resource_size(&intel_graphics_stolen_res))
++ return;
++
+ device = read_pci_config_16(num, slot, func, PCI_DEVICE_ID);
+
+ for (i = 0; i < ARRAY_SIZE(intel_early_ids); i++) {
+@@ -703,7 +711,7 @@ static struct chipset early_qrk[] __initdata = {
+ { PCI_VENDOR_ID_INTEL, 0x3406, PCI_CLASS_BRIDGE_HOST,
+ PCI_BASE_CLASS_BRIDGE, 0, intel_remapping_check },
+ { PCI_VENDOR_ID_INTEL, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA, PCI_ANY_ID,
+- QFLAG_APPLY_ONCE, intel_graphics_quirks },
++ 0, intel_graphics_quirks },
+ /*
+ * HPET on the current version of the Baytrail platform has accuracy
+ * problems: it will halt in deep idle state - so we disable it.
+diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
+index 7ada7bd03a327..759e1cef5e695 100644
+--- a/arch/x86/kernel/fpu/core.c
++++ b/arch/x86/kernel/fpu/core.c
+@@ -25,17 +25,7 @@
+ */
+ union fpregs_state init_fpstate __ro_after_init;
+
+-/*
+- * Track whether the kernel is using the FPU state
+- * currently.
+- *
+- * This flag is used:
+- *
+- * - by IRQ context code to potentially use the FPU
+- * if it's unused.
+- *
+- * - to debug kernel_fpu_begin()/end() correctness
+- */
++/* Track in-kernel FPU usage */
+ static DEFINE_PER_CPU(bool, in_kernel_fpu);
+
+ /*
+@@ -43,42 +33,37 @@ static DEFINE_PER_CPU(bool, in_kernel_fpu);
+ */
+ DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
+
+-static bool kernel_fpu_disabled(void)
+-{
+- return this_cpu_read(in_kernel_fpu);
+-}
+-
+-static bool interrupted_kernel_fpu_idle(void)
+-{
+- return !kernel_fpu_disabled();
+-}
+-
+-/*
+- * Were we in user mode (or vm86 mode) when we were
+- * interrupted?
+- *
+- * Doing kernel_fpu_begin/end() is ok if we are running
+- * in an interrupt context from user mode - we'll just
+- * save the FPU state as required.
+- */
+-static bool interrupted_user_mode(void)
+-{
+- struct pt_regs *regs = get_irq_regs();
+- return regs && user_mode(regs);
+-}
+-
+ /*
+ * Can we use the FPU in kernel mode with the
+ * whole "kernel_fpu_begin/end()" sequence?
+- *
+- * It's always ok in process context (ie "not interrupt")
+- * but it is sometimes ok even from an irq.
+ */
+ bool irq_fpu_usable(void)
+ {
+- return !in_interrupt() ||
+- interrupted_user_mode() ||
+- interrupted_kernel_fpu_idle();
++ if (WARN_ON_ONCE(in_nmi()))
++ return false;
++
++ /* In kernel FPU usage already active? */
++ if (this_cpu_read(in_kernel_fpu))
++ return false;
++
++ /*
++ * When not in NMI or hard interrupt context, FPU can be used in:
++ *
++ * - Task context except from within fpregs_lock()'ed critical
++ * regions.
++ *
++ * - Soft interrupt processing context which cannot happen
++ * while in a fpregs_lock()'ed critical region.
++ */
++ if (!in_hardirq())
++ return true;
++
++ /*
++ * In hard interrupt context it's safe when soft interrupts
++ * are enabled, which means the interrupt did not hit in
++ * a fpregs_lock()'ed critical region.
++ */
++ return !softirq_count();
+ }
+ EXPORT_SYMBOL(irq_fpu_usable);
+
+diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c
+index 66ed317ebc0d3..125cbbe10fefa 100644
+--- a/arch/x86/kernel/fpu/regset.c
++++ b/arch/x86/kernel/fpu/regset.c
+@@ -87,11 +87,9 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
+ const void *kbuf, const void __user *ubuf)
+ {
+ struct fpu *fpu = &target->thread.fpu;
+- struct user32_fxsr_struct newstate;
++ struct fxregs_state newstate;
+ int ret;
+
+- BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state));
+-
+ if (!cpu_feature_enabled(X86_FEATURE_FXSR))
+ return -ENODEV;
+
+@@ -112,9 +110,10 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
+ /* Copy the state */
+ memcpy(&fpu->state.fxsave, &newstate, sizeof(newstate));
+
+- /* Clear xmm8..15 */
++ /* Clear xmm8..15 for 32-bit callers */
+ BUILD_BUG_ON(sizeof(fpu->state.fxsave.xmm_space) != 16 * 16);
+- memset(&fpu->state.fxsave.xmm_space[8], 0, 8 * 16);
++ if (in_ia32_syscall())
++ memset(&fpu->state.fxsave.xmm_space[8*4], 0, 8 * 16);
+
+ /* Mark FP and SSE as in use when XSAVE is enabled */
+ if (use_xsave())
+diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
+index 831b25c5e7058..7f71bd4dcd0d6 100644
+--- a/arch/x86/kernel/fpu/signal.c
++++ b/arch/x86/kernel/fpu/signal.c
+@@ -205,7 +205,7 @@ retry:
+ fpregs_unlock();
+
+ if (ret) {
+- if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size))
++ if (!fault_in_writeable(buf_fx, fpu_user_xstate_size))
+ goto retry;
+ return -EFAULT;
+ }
+@@ -278,10 +278,9 @@ retry:
+ if (ret != -EFAULT)
+ return -EINVAL;
+
+- ret = fault_in_pages_readable(buf, size);
+- if (!ret)
++ if (!fault_in_readable(buf, size))
+ goto retry;
+- return ret;
++ return -EFAULT;
+ }
+
+ /*
+diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
+index 1b3ce3b4a2a2f..847776cc1aa40 100644
+--- a/arch/x86/kernel/ftrace.c
++++ b/arch/x86/kernel/ftrace.c
+@@ -308,7 +308,7 @@ union ftrace_op_code_union {
+ } __attribute__((packed));
+ };
+
+-#define RET_SIZE 1
++#define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS)
+
+ static unsigned long
+ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
+diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
+index e405fe1a8bf41..a0ed0e4a2c0cd 100644
+--- a/arch/x86/kernel/ftrace_32.S
++++ b/arch/x86/kernel/ftrace_32.S
+@@ -19,7 +19,7 @@
+ #endif
+
+ SYM_FUNC_START(__fentry__)
+- ret
++ RET
+ SYM_FUNC_END(__fentry__)
+ EXPORT_SYMBOL(__fentry__)
+
+@@ -84,7 +84,7 @@ ftrace_graph_call:
+
+ /* This is weak to keep gas from relaxing the jumps */
+ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
+- ret
++ RET
+ SYM_CODE_END(ftrace_caller)
+
+ SYM_CODE_START(ftrace_regs_caller)
+@@ -177,7 +177,7 @@ SYM_CODE_START(ftrace_graph_caller)
+ popl %edx
+ popl %ecx
+ popl %eax
+- ret
++ RET
+ SYM_CODE_END(ftrace_graph_caller)
+
+ .globl return_to_handler
+diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
+index 7c273846c6877..d6af81d1b7883 100644
+--- a/arch/x86/kernel/ftrace_64.S
++++ b/arch/x86/kernel/ftrace_64.S
+@@ -132,7 +132,7 @@
+ #ifdef CONFIG_DYNAMIC_FTRACE
+
+ SYM_FUNC_START(__fentry__)
+- retq
++ RET
+ SYM_FUNC_END(__fentry__)
+ EXPORT_SYMBOL(__fentry__)
+
+@@ -181,11 +181,11 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
+
+ /*
+ * This is weak to keep gas from relaxing the jumps.
+- * It is also used to copy the retq for trampolines.
++ * It is also used to copy the RET for trampolines.
+ */
+ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
+ UNWIND_HINT_FUNC
+- retq
++ RET
+ SYM_FUNC_END(ftrace_epilogue)
+
+ SYM_FUNC_START(ftrace_regs_caller)
+@@ -299,7 +299,7 @@ fgraph_trace:
+ #endif
+
+ SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
+- retq
++ RET
+
+ trace:
+ /* save_mcount_regs fills in first two parameters */
+@@ -331,7 +331,7 @@ SYM_FUNC_START(ftrace_graph_caller)
+
+ restore_mcount_regs
+
+- retq
++ RET
+ SYM_FUNC_END(ftrace_graph_caller)
+
+ SYM_FUNC_START(return_to_handler)
+diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
+index d8c64dab0efe0..eb8656bac99b6 100644
+--- a/arch/x86/kernel/head_32.S
++++ b/arch/x86/kernel/head_32.S
+@@ -340,7 +340,7 @@ SYM_FUNC_END(startup_32_smp)
+ __INIT
+ setup_once:
+ andl $0,setup_once_ref /* Once is enough, thanks */
+- ret
++ RET
+
+ SYM_FUNC_START(early_idt_handler_array)
+ # 36(%esp) %eflags
+diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
+index 882213df37130..71f336425e58a 100644
+--- a/arch/x86/kernel/hpet.c
++++ b/arch/x86/kernel/hpet.c
+@@ -1435,8 +1435,12 @@ irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id)
+ hpet_rtc_timer_reinit();
+ memset(&curr_time, 0, sizeof(struct rtc_time));
+
+- if (hpet_rtc_flags & (RTC_UIE | RTC_AIE))
+- mc146818_get_time(&curr_time);
++ if (hpet_rtc_flags & (RTC_UIE | RTC_AIE)) {
++ if (unlikely(mc146818_get_time(&curr_time) < 0)) {
++ pr_err_ratelimited("unable to read current time from RTC\n");
++ return IRQ_HANDLED;
++ }
++ }
+
+ if (hpet_rtc_flags & RTC_UIE &&
+ curr_time.tm_sec != hpet_prev_update_sec) {
+diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
+index e28f6a5d14f1b..766ffe3ba3137 100644
+--- a/arch/x86/kernel/irq.c
++++ b/arch/x86/kernel/irq.c
+@@ -291,8 +291,10 @@ void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
+ {
+ if (handler)
+ kvm_posted_intr_wakeup_handler = handler;
+- else
++ else {
+ kvm_posted_intr_wakeup_handler = dummy_handler;
++ synchronize_rcu();
++ }
+ }
+ EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
+
+diff --git a/arch/x86/kernel/irqflags.S b/arch/x86/kernel/irqflags.S
+index 8ef35063964b1..b8db1022aa6ca 100644
+--- a/arch/x86/kernel/irqflags.S
++++ b/arch/x86/kernel/irqflags.S
+@@ -10,6 +10,6 @@
+ SYM_FUNC_START(native_save_fl)
+ pushf
+ pop %_ASM_AX
+- ret
++ RET
+ SYM_FUNC_END(native_save_fl)
+ EXPORT_SYMBOL(native_save_fl)
+diff --git a/arch/x86/kernel/kdebugfs.c b/arch/x86/kernel/kdebugfs.c
+index 64b6da95af984..e2e89bebcbc32 100644
+--- a/arch/x86/kernel/kdebugfs.c
++++ b/arch/x86/kernel/kdebugfs.c
+@@ -88,11 +88,13 @@ create_setup_data_node(struct dentry *parent, int no,
+
+ static int __init create_setup_data_nodes(struct dentry *parent)
+ {
++ struct setup_indirect *indirect;
+ struct setup_data_node *node;
+ struct setup_data *data;
+- int error;
++ u64 pa_data, pa_next;
+ struct dentry *d;
+- u64 pa_data;
++ int error;
++ u32 len;
+ int no = 0;
+
+ d = debugfs_create_dir("setup_data", parent);
+@@ -112,12 +114,29 @@ static int __init create_setup_data_nodes(struct dentry *parent)
+ error = -ENOMEM;
+ goto err_dir;
+ }
+-
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) {
+- node->paddr = ((struct setup_indirect *)data->data)->addr;
+- node->type = ((struct setup_indirect *)data->data)->type;
+- node->len = ((struct setup_indirect *)data->data)->len;
++ pa_next = data->next;
++
++ if (data->type == SETUP_INDIRECT) {
++ len = sizeof(*data) + data->len;
++ memunmap(data);
++ data = memremap(pa_data, len, MEMREMAP_WB);
++ if (!data) {
++ kfree(node);
++ error = -ENOMEM;
++ goto err_dir;
++ }
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT) {
++ node->paddr = indirect->addr;
++ node->type = indirect->type;
++ node->len = indirect->len;
++ } else {
++ node->paddr = pa_data;
++ node->type = data->type;
++ node->len = data->len;
++ }
+ } else {
+ node->paddr = pa_data;
+ node->type = data->type;
+@@ -125,7 +144,7 @@ static int __init create_setup_data_nodes(struct dentry *parent)
+ }
+
+ create_setup_data_node(d, no, node);
+- pa_data = data->next;
++ pa_data = pa_next;
+
+ memunmap(data);
+ no++;
+diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
+index b6e046e4b2895..f6727c67ae028 100644
+--- a/arch/x86/kernel/kprobes/core.c
++++ b/arch/x86/kernel/kprobes/core.c
+@@ -1044,7 +1044,7 @@ asm(
+ RESTORE_REGS_STRING
+ " popfl\n"
+ #endif
+- " ret\n"
++ ASM_RET
+ ".size kretprobe_trampoline, .-kretprobe_trampoline\n"
+ );
+ NOKPROBE_SYMBOL(kretprobe_trampoline);
+diff --git a/arch/x86/kernel/ksysfs.c b/arch/x86/kernel/ksysfs.c
+index d0a19121c6a4f..257892fcefa79 100644
+--- a/arch/x86/kernel/ksysfs.c
++++ b/arch/x86/kernel/ksysfs.c
+@@ -91,26 +91,41 @@ static int get_setup_data_paddr(int nr, u64 *paddr)
+
+ static int __init get_setup_data_size(int nr, size_t *size)
+ {
+- int i = 0;
++ u64 pa_data = boot_params.hdr.setup_data, pa_next;
++ struct setup_indirect *indirect;
+ struct setup_data *data;
+- u64 pa_data = boot_params.hdr.setup_data;
++ int i = 0;
++ u32 len;
+
+ while (pa_data) {
+ data = memremap(pa_data, sizeof(*data), MEMREMAP_WB);
+ if (!data)
+ return -ENOMEM;
++ pa_next = data->next;
++
+ if (nr == i) {
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT)
+- *size = ((struct setup_indirect *)data->data)->len;
+- else
++ if (data->type == SETUP_INDIRECT) {
++ len = sizeof(*data) + data->len;
++ memunmap(data);
++ data = memremap(pa_data, len, MEMREMAP_WB);
++ if (!data)
++ return -ENOMEM;
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT)
++ *size = indirect->len;
++ else
++ *size = data->len;
++ } else {
+ *size = data->len;
++ }
+
+ memunmap(data);
+ return 0;
+ }
+
+- pa_data = data->next;
++ pa_data = pa_next;
+ memunmap(data);
+ i++;
+ }
+@@ -120,9 +135,11 @@ static int __init get_setup_data_size(int nr, size_t *size)
+ static ssize_t type_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+ {
++ struct setup_indirect *indirect;
++ struct setup_data *data;
+ int nr, ret;
+ u64 paddr;
+- struct setup_data *data;
++ u32 len;
+
+ ret = kobj_to_setup_data_nr(kobj, &nr);
+ if (ret)
+@@ -135,10 +152,20 @@ static ssize_t type_show(struct kobject *kobj,
+ if (!data)
+ return -ENOMEM;
+
+- if (data->type == SETUP_INDIRECT)
+- ret = sprintf(buf, "0x%x\n", ((struct setup_indirect *)data->data)->type);
+- else
++ if (data->type == SETUP_INDIRECT) {
++ len = sizeof(*data) + data->len;
++ memunmap(data);
++ data = memremap(paddr, len, MEMREMAP_WB);
++ if (!data)
++ return -ENOMEM;
++
++ indirect = (struct setup_indirect *)data->data;
++
++ ret = sprintf(buf, "0x%x\n", indirect->type);
++ } else {
+ ret = sprintf(buf, "0x%x\n", data->type);
++ }
++
+ memunmap(data);
+ return ret;
+ }
+@@ -149,9 +176,10 @@ static ssize_t setup_data_data_read(struct file *fp,
+ char *buf,
+ loff_t off, size_t count)
+ {
++ struct setup_indirect *indirect;
++ struct setup_data *data;
+ int nr, ret = 0;
+ u64 paddr, len;
+- struct setup_data *data;
+ void *p;
+
+ ret = kobj_to_setup_data_nr(kobj, &nr);
+@@ -165,10 +193,27 @@ static ssize_t setup_data_data_read(struct file *fp,
+ if (!data)
+ return -ENOMEM;
+
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) {
+- paddr = ((struct setup_indirect *)data->data)->addr;
+- len = ((struct setup_indirect *)data->data)->len;
++ if (data->type == SETUP_INDIRECT) {
++ len = sizeof(*data) + data->len;
++ memunmap(data);
++ data = memremap(paddr, len, MEMREMAP_WB);
++ if (!data)
++ return -ENOMEM;
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT) {
++ paddr = indirect->addr;
++ len = indirect->len;
++ } else {
++ /*
++ * Even though this is technically undefined, return
++ * the data as though it is a normal setup_data struct.
++ * This will at least allow it to be inspected.
++ */
++ paddr += sizeof(*data);
++ len = data->len;
++ }
+ } else {
+ paddr += sizeof(*data);
+ len = data->len;
+diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
+index b656456c3a944..d36b58e705b6c 100644
+--- a/arch/x86/kernel/kvm.c
++++ b/arch/x86/kernel/kvm.c
+@@ -66,6 +66,7 @@ static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __align
+ DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible;
+ static int has_steal_clock = 0;
+
++static int has_guest_poll = 0;
+ /*
+ * No need for any "IO delay" on KVM
+ */
+@@ -457,19 +458,22 @@ static bool pv_tlb_flush_supported(void)
+ {
+ return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
+ !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
+- kvm_para_has_feature(KVM_FEATURE_STEAL_TIME));
++ kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
++ (num_possible_cpus() != 1));
+ }
+
+ static bool pv_ipi_supported(void)
+ {
+- return kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI);
++ return (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI) &&
++ (num_possible_cpus() != 1));
+ }
+
+ static bool pv_sched_yield_supported(void)
+ {
+ return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) &&
+ !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
+- kvm_para_has_feature(KVM_FEATURE_STEAL_TIME));
++ kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
++ (num_possible_cpus() != 1));
+ }
+
+ #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG)
+@@ -507,7 +511,7 @@ static void __send_ipi_mask(const struct cpumask *mask, int vector)
+ } else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) {
+ ipi_bitmap <<= min - apic_id;
+ min = apic_id;
+- } else if (apic_id < min + KVM_IPI_CLUSTER_SIZE) {
++ } else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) {
+ max = apic_id < max ? max : apic_id;
+ } else {
+ ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
+@@ -647,14 +651,26 @@ static int kvm_cpu_down_prepare(unsigned int cpu)
+
+ static int kvm_suspend(void)
+ {
++ u64 val = 0;
++
+ kvm_guest_cpu_offline(false);
+
++#ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
++ if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
++ rdmsrl(MSR_KVM_POLL_CONTROL, val);
++ has_guest_poll = !(val & 1);
++#endif
+ return 0;
+ }
+
+ static void kvm_resume(void)
+ {
+ kvm_cpu_online(raw_smp_processor_id());
++
++#ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
++ if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll)
++ wrmsrl(MSR_KVM_POLL_CONTROL, 0);
++#endif
+ }
+
+ static struct syscore_ops kvm_syscore_ops = {
+diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
+index 5e9a34b5bd741..32e546e416296 100644
+--- a/arch/x86/kernel/module.c
++++ b/arch/x86/kernel/module.c
+@@ -67,6 +67,7 @@ static unsigned long int get_module_load_offset(void)
+
+ void *module_alloc(unsigned long size)
+ {
++ gfp_t gfp_mask = GFP_KERNEL;
+ void *p;
+
+ if (PAGE_ALIGN(size) > MODULES_LEN)
+@@ -74,10 +75,10 @@ void *module_alloc(unsigned long size)
+
+ p = __vmalloc_node_range(size, MODULE_ALIGN,
+ MODULES_VADDR + get_module_load_offset(),
+- MODULES_END, GFP_KERNEL,
+- PAGE_KERNEL, 0, NUMA_NO_NODE,
++ MODULES_END, gfp_mask,
++ PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,
+ __builtin_return_address(0));
+- if (p && (kasan_module_alloc(p, size) < 0)) {
++ if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) {
+ vfree(p);
+ return NULL;
+ }
+@@ -269,6 +270,14 @@ int module_finalize(const Elf_Ehdr *hdr,
+ orc_ip = s;
+ }
+
++ /*
++ * See alternative_instructions() for the ordering rules between the
++ * various patching types.
++ */
++ if (para) {
++ void *pseg = (void *)para->sh_addr;
++ apply_paravirt(pseg, pseg + para->sh_size);
++ }
+ if (alt) {
+ /* patch .altinstructions */
+ void *aseg = (void *)alt->sh_addr;
+@@ -282,11 +291,6 @@ int module_finalize(const Elf_Ehdr *hdr,
+ tseg, tseg + text->sh_size);
+ }
+
+- if (para) {
+- void *pseg = (void *)para->sh_addr;
+- apply_paravirt(pseg, pseg + para->sh_size);
+- }
+-
+ /* make jump label nops */
+ jump_label_apply_nops(me);
+
+diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
+index 04cafc057bed4..f1cdb8891ad41 100644
+--- a/arch/x86/kernel/paravirt.c
++++ b/arch/x86/kernel/paravirt.c
+@@ -41,7 +41,7 @@ extern void _paravirt_nop(void);
+ asm (".pushsection .entry.text, \"ax\"\n"
+ ".global _paravirt_nop\n"
+ "_paravirt_nop:\n\t"
+- "ret\n\t"
++ ASM_RET
+ ".size _paravirt_nop, . - _paravirt_nop\n\t"
+ ".type _paravirt_nop, @function\n\t"
+ ".popsection");
+diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
+index 1d9463e3096b6..f2f733bcb2b95 100644
+--- a/arch/x86/kernel/process.c
++++ b/arch/x86/kernel/process.c
+@@ -132,6 +132,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg,
+ frame->ret_addr = (unsigned long) ret_from_fork;
+ p->thread.sp = (unsigned long) fork_frame;
+ p->thread.io_bitmap = NULL;
++ p->thread.iopl_warn = 0;
+ memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
+
+ #ifdef CONFIG_X86_64
+diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
+index 4c208ea3bd9f3..033d9c6a94689 100644
+--- a/arch/x86/kernel/ptrace.c
++++ b/arch/x86/kernel/ptrace.c
+@@ -1224,7 +1224,7 @@ static struct user_regset x86_64_regsets[] __ro_after_init = {
+ },
+ [REGSET_FP] = {
+ .core_note_type = NT_PRFPREG,
+- .n = sizeof(struct user_i387_struct) / sizeof(long),
++ .n = sizeof(struct fxregs_state) / sizeof(long),
+ .size = sizeof(long), .align = sizeof(long),
+ .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set
+ },
+@@ -1271,7 +1271,7 @@ static struct user_regset x86_32_regsets[] __ro_after_init = {
+ },
+ [REGSET_XFP] = {
+ .core_note_type = NT_PRXFPREG,
+- .n = sizeof(struct user32_fxsr_struct) / sizeof(u32),
++ .n = sizeof(struct fxregs_state) / sizeof(u32),
+ .size = sizeof(u32), .align = sizeof(u32),
+ .active = regset_xregset_fpregs_active, .regset_get = xfpregs_get, .set = xfpregs_set
+ },
+diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
+index 0a40df66a40de..fa700b46588e0 100644
+--- a/arch/x86/kernel/reboot.c
++++ b/arch/x86/kernel/reboot.c
+@@ -113,17 +113,9 @@ void __noreturn machine_real_restart(unsigned int type)
+ spin_unlock(&rtc_lock);
+
+ /*
+- * Switch back to the initial page table.
++ * Switch to the trampoline page table.
+ */
+-#ifdef CONFIG_X86_32
+- load_cr3(initial_page_table);
+-#else
+- write_cr3(real_mode_header->trampoline_pgd);
+-
+- /* Exiting long mode will fail if CR4.PCIDE is set. */
+- if (boot_cpu_has(X86_FEATURE_PCID))
+- cr4_clear_bits(X86_CR4_PCIDE);
+-#endif
++ load_trampoline_pgtable();
+
+ /* Jump to the identity-mapped low memory code */
+ #ifdef CONFIG_X86_32
+diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S
+index f469153eca8ab..fcc8a7699103a 100644
+--- a/arch/x86/kernel/relocate_kernel_32.S
++++ b/arch/x86/kernel/relocate_kernel_32.S
+@@ -91,7 +91,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
+ movl %edi, %eax
+ addl $(identity_mapped - relocate_kernel), %eax
+ pushl %eax
+- ret
++ RET
+ SYM_CODE_END(relocate_kernel)
+
+ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+@@ -159,7 +159,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+ xorl %edx, %edx
+ xorl %esi, %esi
+ xorl %ebp, %ebp
+- ret
++ RET
+ 1:
+ popl %edx
+ movl CP_PA_SWAP_PAGE(%edi), %esp
+@@ -190,7 +190,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+ movl %edi, %eax
+ addl $(virtual_mapped - relocate_kernel), %eax
+ pushl %eax
+- ret
++ RET
+ SYM_CODE_END(identity_mapped)
+
+ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
+@@ -208,7 +208,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
+ popl %edi
+ popl %esi
+ popl %ebx
+- ret
++ RET
+ SYM_CODE_END(virtual_mapped)
+
+ /* Do the copies */
+@@ -271,7 +271,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
+ popl %edi
+ popl %ebx
+ popl %ebp
+- ret
++ RET
+ SYM_CODE_END(swap_pages)
+
+ .globl kexec_control_code_size
+diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
+index c53271aebb64d..5019091af059e 100644
+--- a/arch/x86/kernel/relocate_kernel_64.S
++++ b/arch/x86/kernel/relocate_kernel_64.S
+@@ -104,7 +104,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
+ /* jump to identity mapped page */
+ addq $(identity_mapped - relocate_kernel), %r8
+ pushq %r8
+- ret
++ RET
+ SYM_CODE_END(relocate_kernel)
+
+ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+@@ -191,7 +191,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+ xorl %r14d, %r14d
+ xorl %r15d, %r15d
+
+- ret
++ RET
+
+ 1:
+ popq %rdx
+@@ -210,7 +210,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
+ call swap_pages
+ movq $virtual_mapped, %rax
+ pushq %rax
+- ret
++ RET
+ SYM_CODE_END(identity_mapped)
+
+ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
+@@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
+ popq %r12
+ popq %rbp
+ popq %rbx
+- ret
++ RET
+ SYM_CODE_END(virtual_mapped)
+
+ /* Do the copies */
+@@ -288,7 +288,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
+ lea PAGE_SIZE(%rax), %rsi
+ jmp 0b
+ 3:
+- ret
++ RET
+ SYM_CODE_END(swap_pages)
+
+ .globl kexec_control_code_size
+diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
+index 40ed44ead0631..8e56c4de00b9c 100644
+--- a/arch/x86/kernel/setup.c
++++ b/arch/x86/kernel/setup.c
+@@ -368,21 +368,41 @@ static void __init parse_setup_data(void)
+
+ static void __init memblock_x86_reserve_range_setup_data(void)
+ {
++ struct setup_indirect *indirect;
+ struct setup_data *data;
+- u64 pa_data;
++ u64 pa_data, pa_next;
++ u32 len;
+
+ pa_data = boot_params.hdr.setup_data;
+ while (pa_data) {
+ data = early_memremap(pa_data, sizeof(*data));
++ if (!data) {
++ pr_warn("setup: failed to memremap setup_data entry\n");
++ return;
++ }
++
++ len = sizeof(*data);
++ pa_next = data->next;
++
+ memblock_reserve(pa_data, sizeof(*data) + data->len);
+
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT)
+- memblock_reserve(((struct setup_indirect *)data->data)->addr,
+- ((struct setup_indirect *)data->data)->len);
++ if (data->type == SETUP_INDIRECT) {
++ len += data->len;
++ early_memunmap(data, sizeof(*data));
++ data = early_memremap(pa_data, len);
++ if (!data) {
++ pr_warn("setup: failed to memremap indirect setup_data\n");
++ return;
++ }
+
+- pa_data = data->next;
+- early_memunmap(data, sizeof(*data));
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT)
++ memblock_reserve(indirect->addr, indirect->len);
++ }
++
++ pa_data = pa_next;
++ early_memunmap(data, len);
+ }
+ }
+
+@@ -713,9 +733,6 @@ static void __init early_reserve_memory(void)
+
+ early_reserve_initrd();
+
+- if (efi_enabled(EFI_BOOT))
+- efi_memblock_x86_reserve_range();
+-
+ memblock_x86_reserve_range_setup_data();
+
+ reserve_ibft_region();
+@@ -890,6 +907,9 @@ void __init setup_arch(char **cmdline_p)
+
+ parse_early_param();
+
++ if (efi_enabled(EFI_BOOT))
++ efi_memblock_x86_reserve_range();
++
+ #ifdef CONFIG_MEMORY_HOTPLUG
+ /*
+ * Memory used by the kernel cannot be hot-removed because Linux
+diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c
+index a6895e440bc35..a0064cf77e562 100644
+--- a/arch/x86/kernel/sev.c
++++ b/arch/x86/kernel/sev.c
+@@ -46,16 +46,6 @@ static struct ghcb __initdata *boot_ghcb;
+ struct sev_es_runtime_data {
+ struct ghcb ghcb_page;
+
+- /* Physical storage for the per-CPU IST stack of the #VC handler */
+- char ist_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE);
+-
+- /*
+- * Physical storage for the per-CPU fall-back stack of the #VC handler.
+- * The fall-back stack is used when it is not safe to switch back to the
+- * interrupted stack in the #VC entry code.
+- */
+- char fallback_stack[EXCEPTION_STKSZ] __aligned(PAGE_SIZE);
+-
+ /*
+ * Reserve one page per CPU as backup storage for the unencrypted GHCB.
+ * It is needed when an NMI happens while the #VC handler uses the real
+@@ -99,27 +89,6 @@ DEFINE_STATIC_KEY_FALSE(sev_es_enable_key);
+ /* Needed in vc_early_forward_exception */
+ void do_early_exception(struct pt_regs *regs, int trapnr);
+
+-static void __init setup_vc_stacks(int cpu)
+-{
+- struct sev_es_runtime_data *data;
+- struct cpu_entry_area *cea;
+- unsigned long vaddr;
+- phys_addr_t pa;
+-
+- data = per_cpu(runtime_data, cpu);
+- cea = get_cpu_entry_area(cpu);
+-
+- /* Map #VC IST stack */
+- vaddr = CEA_ESTACK_BOT(&cea->estacks, VC);
+- pa = __pa(data->ist_stack);
+- cea_set_pte((void *)vaddr, pa, PAGE_KERNEL);
+-
+- /* Map VC fall-back stack */
+- vaddr = CEA_ESTACK_BOT(&cea->estacks, VC2);
+- pa = __pa(data->fallback_stack);
+- cea_set_pte((void *)vaddr, pa, PAGE_KERNEL);
+-}
+-
+ static __always_inline bool on_vc_stack(struct pt_regs *regs)
+ {
+ unsigned long sp = regs->sp;
+@@ -325,11 +294,6 @@ static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
+ char *dst, char *buf, size_t size)
+ {
+ unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
+- char __user *target = (char __user *)dst;
+- u64 d8;
+- u32 d4;
+- u16 d2;
+- u8 d1;
+
+ /*
+ * This function uses __put_user() independent of whether kernel or user
+@@ -351,26 +315,42 @@ static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
+ * instructions here would cause infinite nesting.
+ */
+ switch (size) {
+- case 1:
++ case 1: {
++ u8 d1;
++ u8 __user *target = (u8 __user *)dst;
++
+ memcpy(&d1, buf, 1);
+ if (__put_user(d1, target))
+ goto fault;
+ break;
+- case 2:
++ }
++ case 2: {
++ u16 d2;
++ u16 __user *target = (u16 __user *)dst;
++
+ memcpy(&d2, buf, 2);
+ if (__put_user(d2, target))
+ goto fault;
+ break;
+- case 4:
++ }
++ case 4: {
++ u32 d4;
++ u32 __user *target = (u32 __user *)dst;
++
+ memcpy(&d4, buf, 4);
+ if (__put_user(d4, target))
+ goto fault;
+ break;
+- case 8:
++ }
++ case 8: {
++ u64 d8;
++ u64 __user *target = (u64 __user *)dst;
++
+ memcpy(&d8, buf, 8);
+ if (__put_user(d8, target))
+ goto fault;
+ break;
++ }
+ default:
+ WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+ return ES_UNSUPPORTED;
+@@ -393,11 +373,6 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
+ char *src, char *buf, size_t size)
+ {
+ unsigned long error_code = X86_PF_PROT;
+- char __user *s = (char __user *)src;
+- u64 d8;
+- u32 d4;
+- u16 d2;
+- u8 d1;
+
+ /*
+ * This function uses __get_user() independent of whether kernel or user
+@@ -419,26 +394,41 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
+ * instructions here would cause infinite nesting.
+ */
+ switch (size) {
+- case 1:
++ case 1: {
++ u8 d1;
++ u8 __user *s = (u8 __user *)src;
++
+ if (__get_user(d1, s))
+ goto fault;
+ memcpy(buf, &d1, 1);
+ break;
+- case 2:
++ }
++ case 2: {
++ u16 d2;
++ u16 __user *s = (u16 __user *)src;
++
+ if (__get_user(d2, s))
+ goto fault;
+ memcpy(buf, &d2, 2);
+ break;
+- case 4:
++ }
++ case 4: {
++ u32 d4;
++ u32 __user *s = (u32 __user *)src;
++
+ if (__get_user(d4, s))
+ goto fault;
+ memcpy(buf, &d4, 4);
+ break;
+- case 8:
++ }
++ case 8: {
++ u64 d8;
++ u64 __user *s = (u64 __user *)src;
+ if (__get_user(d8, s))
+ goto fault;
+ memcpy(buf, &d8, 8);
+ break;
++ }
+ default:
+ WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+ return ES_UNSUPPORTED;
+@@ -787,7 +777,6 @@ void __init sev_es_init_vc_handling(void)
+ for_each_possible_cpu(cpu) {
+ alloc_runtime_data(cpu);
+ init_ghcb(cpu);
+- setup_vc_stacks(cpu);
+ }
+
+ sev_es_setup_play_dead();
+diff --git a/arch/x86/kernel/sev_verify_cbit.S b/arch/x86/kernel/sev_verify_cbit.S
+index ee04941a6546a..3355e27c69ebf 100644
+--- a/arch/x86/kernel/sev_verify_cbit.S
++++ b/arch/x86/kernel/sev_verify_cbit.S
+@@ -85,5 +85,5 @@ SYM_FUNC_START(sev_verify_cbit)
+ #endif
+ /* Return page-table pointer */
+ movq %rdi, %rax
+- ret
++ RET
+ SYM_FUNC_END(sev_verify_cbit)
+diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
+index ea028e736831a..3ec2cb881eef0 100644
+--- a/arch/x86/kernel/static_call.c
++++ b/arch/x86/kernel/static_call.c
+@@ -12,10 +12,11 @@ enum insn_type {
+ };
+
+ /*
+- * data16 data16 xorq %rax, %rax - a single 5 byte instruction that clears %rax
+- * The REX.W cancels the effect of any data16.
++ * cs cs cs xorl %eax, %eax - a single 5 byte instruction that clears %[er]ax
+ */
+-static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 };
++static const u8 xor5rax[] = { 0x2e, 0x2e, 0x2e, 0x31, 0xc0 };
++
++static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc };
+
+ static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
+ {
+@@ -42,8 +43,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void
+ break;
+
+ case RET:
+- code = text_gen_insn(RET_INSN_OPCODE, insn, func);
+- size = RET_INSN_SIZE;
++ code = &retinsn;
+ break;
+ }
+
+diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
+index a58800973aed3..928e1ac820e67 100644
+--- a/arch/x86/kernel/traps.c
++++ b/arch/x86/kernel/traps.c
+@@ -313,17 +313,19 @@ out:
+ }
+
+ #ifdef CONFIG_VMAP_STACK
+-__visible void __noreturn handle_stack_overflow(const char *message,
+- struct pt_regs *regs,
+- unsigned long fault_address)
++__visible void __noreturn handle_stack_overflow(struct pt_regs *regs,
++ unsigned long fault_address,
++ struct stack_info *info)
+ {
+- printk(KERN_EMERG "BUG: stack guard page was hit at %p (stack is %p..%p)\n",
+- (void *)fault_address, current->stack,
+- (char *)current->stack + THREAD_SIZE - 1);
+- die(message, regs, 0);
++ const char *name = stack_type_name(info->type);
++
++ printk(KERN_EMERG "BUG: %s stack guard page was hit at %p (stack is %p..%p)\n",
++ name, (void *)fault_address, info->begin, info->end);
++
++ die("stack guard page", regs, 0);
+
+ /* Be absolutely certain we don't return. */
+- panic("%s", message);
++ panic("%s stack guard hit", name);
+ }
+ #endif
+
+@@ -353,6 +355,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
+
+ #ifdef CONFIG_VMAP_STACK
+ unsigned long address = read_cr2();
++ struct stack_info info;
+ #endif
+
+ #ifdef CONFIG_X86_ESPFIX64
+@@ -455,10 +458,8 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
+ * stack even if the actual trigger for the double fault was
+ * something else.
+ */
+- if ((unsigned long)task_stack_page(tsk) - 1 - address < PAGE_SIZE) {
+- handle_stack_overflow("kernel stack overflow (double-fault)",
+- regs, address);
+- }
++ if (get_stack_guard_info((void *)address, &info))
++ handle_stack_overflow(regs, address, &info);
+ #endif
+
+ pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code);
+@@ -528,6 +529,36 @@ static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,
+
+ #define GPFSTR "general protection fault"
+
++static bool fixup_iopl_exception(struct pt_regs *regs)
++{
++ struct thread_struct *t = &current->thread;
++ unsigned char byte;
++ unsigned long ip;
++
++ if (!IS_ENABLED(CONFIG_X86_IOPL_IOPERM) || t->iopl_emul != 3)
++ return false;
++
++ if (insn_get_effective_ip(regs, &ip))
++ return false;
++
++ if (get_user(byte, (const char __user *)ip))
++ return false;
++
++ if (byte != 0xfa && byte != 0xfb)
++ return false;
++
++ if (!t->iopl_warn && printk_ratelimit()) {
++ pr_err("%s[%d] attempts to use CLI/STI, pretending it's a NOP, ip:%lx",
++ current->comm, task_pid_nr(current), ip);
++ print_vma_addr(KERN_CONT " in ", ip);
++ pr_cont("\n");
++ t->iopl_warn = 1;
++ }
++
++ regs->ip += 1;
++ return true;
++}
++
+ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
+ {
+ char desc[sizeof(GPFSTR) + 50 + 2*sizeof(unsigned long) + 1] = GPFSTR;
+@@ -553,6 +584,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
+ tsk = current;
+
+ if (user_mode(regs)) {
++ if (fixup_iopl_exception(regs))
++ goto exit;
++
+ tsk->thread.error_code = error_code;
+ tsk->thread.trap_nr = X86_TRAP_GP;
+
+@@ -625,6 +659,7 @@ static bool do_int3(struct pt_regs *regs)
+
+ return res == NOTIFY_STOP;
+ }
++NOKPROBE_SYMBOL(do_int3);
+
+ static void do_int3_user(struct pt_regs *regs)
+ {
+@@ -709,7 +744,7 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *r
+ stack = (unsigned long *)sp;
+
+ if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY ||
+- info.type >= STACK_TYPE_EXCEPTION_LAST)
++ info.type > STACK_TYPE_EXCEPTION_LAST)
+ sp = __this_cpu_ist_top_va(VC2);
+
+ sync:
+diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
+index 2e076a459a0c0..a698196377be9 100644
+--- a/arch/x86/kernel/tsc.c
++++ b/arch/x86/kernel/tsc.c
+@@ -1180,6 +1180,12 @@ void mark_tsc_unstable(char *reason)
+
+ EXPORT_SYMBOL_GPL(mark_tsc_unstable);
+
++static void __init tsc_disable_clocksource_watchdog(void)
++{
++ clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
++ clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
++}
++
+ static void __init check_system_tsc_reliable(void)
+ {
+ #if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC)
+@@ -1196,6 +1202,23 @@ static void __init check_system_tsc_reliable(void)
+ #endif
+ if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
+ tsc_clocksource_reliable = 1;
++
++ /*
++ * Disable the clocksource watchdog when the system has:
++ * - TSC running at constant frequency
++ * - TSC which does not stop in C-States
++ * - the TSC_ADJUST register which allows to detect even minimal
++ * modifications
++ * - not more than two sockets. As the number of sockets cannot be
++ * evaluated at the early boot stage where this has to be
++ * invoked, check the number of online memory nodes as a
++ * fallback solution which is an reasonable estimate.
++ */
++ if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
++ boot_cpu_has(X86_FEATURE_NONSTOP_TSC) &&
++ boot_cpu_has(X86_FEATURE_TSC_ADJUST) &&
++ nr_online_nodes <= 2)
++ tsc_disable_clocksource_watchdog();
+ }
+
+ /*
+@@ -1387,9 +1410,6 @@ static int __init init_tsc_clocksource(void)
+ if (tsc_unstable)
+ goto unreg;
+
+- if (tsc_clocksource_reliable || no_tsc_watchdog)
+- clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
+-
+ if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
+ clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
+
+@@ -1527,7 +1547,7 @@ void __init tsc_init(void)
+ }
+
+ if (tsc_clocksource_reliable || no_tsc_watchdog)
+- clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
++ tsc_disable_clocksource_watchdog();
+
+ clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
+ detect_art();
+diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c
+index 50a4515fe0ad1..9452dc9664b51 100644
+--- a/arch/x86/kernel/tsc_sync.c
++++ b/arch/x86/kernel/tsc_sync.c
+@@ -30,6 +30,7 @@ struct tsc_adjust {
+ };
+
+ static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);
++static struct timer_list tsc_sync_check_timer;
+
+ /*
+ * TSC's on different sockets may be reset asynchronously.
+@@ -77,6 +78,46 @@ void tsc_verify_tsc_adjust(bool resume)
+ }
+ }
+
++/*
++ * Normally the tsc_sync will be checked every time system enters idle
++ * state, but there is still caveat that a system won't enter idle,
++ * either because it's too busy or configured purposely to not enter
++ * idle.
++ *
++ * So setup a periodic timer (every 10 minutes) to make sure the check
++ * is always on.
++ */
++
++#define SYNC_CHECK_INTERVAL (HZ * 600)
++
++static void tsc_sync_check_timer_fn(struct timer_list *unused)
++{
++ int next_cpu;
++
++ tsc_verify_tsc_adjust(false);
++
++ /* Run the check for all onlined CPUs in turn */
++ next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
++ if (next_cpu >= nr_cpu_ids)
++ next_cpu = cpumask_first(cpu_online_mask);
++
++ tsc_sync_check_timer.expires += SYNC_CHECK_INTERVAL;
++ add_timer_on(&tsc_sync_check_timer, next_cpu);
++}
++
++static int __init start_sync_check_timer(void)
++{
++ if (!cpu_feature_enabled(X86_FEATURE_TSC_ADJUST) || tsc_clocksource_reliable)
++ return 0;
++
++ timer_setup(&tsc_sync_check_timer, tsc_sync_check_timer_fn, 0);
++ tsc_sync_check_timer.expires = jiffies + SYNC_CHECK_INTERVAL;
++ add_timer(&tsc_sync_check_timer);
++
++ return 0;
++}
++late_initcall(start_sync_check_timer);
++
+ static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,
+ unsigned int cpu, bool bootcpu)
+ {
+diff --git a/arch/x86/kernel/verify_cpu.S b/arch/x86/kernel/verify_cpu.S
+index 641f0fe1e5b4a..1258a5872d128 100644
+--- a/arch/x86/kernel/verify_cpu.S
++++ b/arch/x86/kernel/verify_cpu.S
+@@ -132,9 +132,9 @@ SYM_FUNC_START_LOCAL(verify_cpu)
+ .Lverify_cpu_no_longmode:
+ popf # Restore caller passed flags
+ movl $1,%eax
+- ret
++ RET
+ .Lverify_cpu_sse_ok:
+ popf # Restore caller passed flags
+ xorl %eax, %eax
+- ret
++ RET
+ SYM_FUNC_END(verify_cpu)
+diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
+index e5a7a10a0164d..17d58740891e2 100644
+--- a/arch/x86/kernel/vm86_32.c
++++ b/arch/x86/kernel/vm86_32.c
+@@ -142,6 +142,7 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval)
+
+ user_access_end();
+
++exit_vm86:
+ preempt_disable();
+ tsk->thread.sp0 = vm86->saved_sp0;
+ tsk->thread.sysenter_cs = __KERNEL_CS;
+@@ -161,7 +162,8 @@ Efault_end:
+ user_access_end();
+ Efault:
+ pr_alert("could not access userspace vm86 info\n");
+- do_exit(SIGSEGV);
++ force_exit_sig(SIGSEGV);
++ goto exit_vm86;
+ }
+
+ static int do_vm86_irq_handling(int subfunction, int irqnumber);
+diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
+index 751aa85a30012..b17c9b00669e5 100644
+--- a/arch/x86/kvm/cpuid.c
++++ b/arch/x86/kvm/cpuid.c
+@@ -232,6 +232,25 @@ u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
+ return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
+ }
+
++static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
++ int nent)
++{
++ int r;
++
++ r = kvm_check_cpuid(e2, nent);
++ if (r)
++ return r;
++
++ kvfree(vcpu->arch.cpuid_entries);
++ vcpu->arch.cpuid_entries = e2;
++ vcpu->arch.cpuid_nent = nent;
++
++ kvm_update_cpuid_runtime(vcpu);
++ kvm_vcpu_after_set_cpuid(vcpu);
++
++ return 0;
++}
++
+ /* when an old userspace process fills a new kernel module */
+ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
+ struct kvm_cpuid *cpuid,
+@@ -268,18 +287,9 @@ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
+ e2[i].padding[2] = 0;
+ }
+
+- r = kvm_check_cpuid(e2, cpuid->nent);
+- if (r) {
++ r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
++ if (r)
+ kvfree(e2);
+- goto out_free_cpuid;
+- }
+-
+- kvfree(vcpu->arch.cpuid_entries);
+- vcpu->arch.cpuid_entries = e2;
+- vcpu->arch.cpuid_nent = cpuid->nent;
+-
+- kvm_update_cpuid_runtime(vcpu);
+- kvm_vcpu_after_set_cpuid(vcpu);
+
+ out_free_cpuid:
+ kvfree(e);
+@@ -303,20 +313,11 @@ int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
+ return PTR_ERR(e2);
+ }
+
+- r = kvm_check_cpuid(e2, cpuid->nent);
+- if (r) {
++ r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
++ if (r)
+ kvfree(e2);
+- return r;
+- }
+-
+- kvfree(vcpu->arch.cpuid_entries);
+- vcpu->arch.cpuid_entries = e2;
+- vcpu->arch.cpuid_nent = cpuid->nent;
+
+- kvm_update_cpuid_runtime(vcpu);
+- kvm_vcpu_after_set_cpuid(vcpu);
+-
+- return 0;
++ return r;
+ }
+
+ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
+@@ -420,12 +421,13 @@ void kvm_set_cpu_caps(void)
+ );
+
+ kvm_cpu_cap_mask(CPUID_7_0_EBX,
+- F(FSGSBASE) | F(SGX) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) |
+- F(BMI2) | F(ERMS) | F(INVPCID) | F(RTM) | 0 /*MPX*/ | F(RDSEED) |
+- F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) |
+- F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) |
+- F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0 /*INTEL_PT*/
+- );
++ F(FSGSBASE) | F(SGX) | F(BMI1) | F(HLE) | F(AVX2) |
++ F(FDP_EXCPTN_ONLY) | F(SMEP) | F(BMI2) | F(ERMS) | F(INVPCID) |
++ F(RTM) | F(ZERO_FCS_FDS) | 0 /*MPX*/ | F(AVX512F) |
++ F(AVX512DQ) | F(RDSEED) | F(ADX) | F(SMAP) | F(AVX512IFMA) |
++ F(CLFLUSHOPT) | F(CLWB) | 0 /*INTEL_PT*/ | F(AVX512PF) |
++ F(AVX512ER) | F(AVX512CD) | F(SHA_NI) | F(AVX512BW) |
++ F(AVX512VL));
+
+ kvm_cpu_cap_mask(CPUID_7_ECX,
+ F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(RDPID) |
+@@ -723,6 +725,11 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
+ union cpuid10_eax eax;
+ union cpuid10_edx edx;
+
++ if (!static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
++ entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
++ break;
++ }
++
+ perf_get_x86_pmu_capability(&cap);
+
+ /*
+diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c
+index 54a83a7445384..f33c804a922ac 100644
+--- a/arch/x86/kvm/debugfs.c
++++ b/arch/x86/kvm/debugfs.c
+@@ -95,6 +95,9 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v)
+ unsigned int *log[KVM_NR_PAGE_SIZES], *cur;
+ int i, j, k, l, ret;
+
++ if (!kvm_memslots_have_rmaps(kvm))
++ return 0;
++
+ ret = -ENOMEM;
+ memset(log, 0, sizeof(log));
+ for (i = 0; i < KVM_NR_PAGE_SIZES; i++) {
+diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
+index 9a144ca8e1460..82eff14bd064d 100644
+--- a/arch/x86/kvm/emulate.c
++++ b/arch/x86/kvm/emulate.c
+@@ -315,7 +315,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
+ __FOP_FUNC(#name)
+
+ #define __FOP_RET(name) \
+- "ret \n\t" \
++ ASM_RET \
+ ".size " name ", .-" name "\n\t"
+
+ #define FOP_RET(name) \
+@@ -427,15 +427,30 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
+ FOP_END
+
+ /* Special case for SETcc - 1 instruction per cc */
++
++/*
++ * Depending on .config the SETcc functions look like:
++ *
++ * SETcc %al [3 bytes]
++ * RET [1 byte]
++ * INT3 [1 byte; CONFIG_SLS]
++ *
++ * Which gives possible sizes 4 or 5. When rounded up to the
++ * next power-of-two alignment they become 4 or 8.
++ */
++#define SETCC_LENGTH (4 + IS_ENABLED(CONFIG_SLS))
++#define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS))
++static_assert(SETCC_LENGTH <= SETCC_ALIGN);
++
+ #define FOP_SETCC(op) \
+- ".align 4 \n\t" \
++ ".align " __stringify(SETCC_ALIGN) " \n\t" \
+ ".type " #op ", @function \n\t" \
+ #op ": \n\t" \
+ #op " %al \n\t" \
+ __FOP_RET(#op)
+
+ asm(".pushsection .fixup, \"ax\"\n"
+- "kvm_fastop_exception: xor %esi, %esi; ret\n"
++ "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
+ ".popsection");
+
+ FOP_START(setcc)
+@@ -1053,7 +1068,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
+ static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
+ {
+ u8 rc;
+- void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
++ void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf);
+
+ flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
+ asm("push %[flags]; popf; " CALL_NOSPEC
+@@ -1614,11 +1629,6 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
+ goto exception;
+ }
+
+- if (!seg_desc.p) {
+- err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
+- goto exception;
+- }
+-
+ dpl = seg_desc.dpl;
+
+ switch (seg) {
+@@ -1658,6 +1668,10 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
+ case VCPU_SREG_TR:
+ if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
+ goto exception;
++ if (!seg_desc.p) {
++ err_vec = NP_VECTOR;
++ goto exception;
++ }
+ old_desc = seg_desc;
+ seg_desc.type |= 2; /* busy */
+ ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
+@@ -1682,6 +1696,11 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
+ break;
+ }
+
++ if (!seg_desc.p) {
++ err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
++ goto exception;
++ }
++
+ if (seg_desc.s) {
+ /* mark segment as accessed */
+ if (!(seg_desc.type & 1)) {
+@@ -3510,8 +3529,10 @@ static int em_rdpid(struct x86_emulate_ctxt *ctxt)
+ {
+ u64 tsc_aux = 0;
+
+- if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
++ if (!ctxt->ops->guest_has_rdpid(ctxt))
+ return emulate_ud(ctxt);
++
++ ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
+ ctxt->dst.val = tsc_aux;
+ return X86EMUL_CONTINUE;
+ }
+diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
+index d5124b520f761..762b43f0d9194 100644
+--- a/arch/x86/kvm/hyperv.c
++++ b/arch/x86/kvm/hyperv.c
+@@ -236,7 +236,7 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
+ struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
+ int ret;
+
+- if (!synic->active && !host)
++ if (!synic->active && (!host || data))
+ return 1;
+
+ trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host);
+@@ -282,6 +282,9 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
+ case HV_X64_MSR_EOM: {
+ int i;
+
++ if (!synic->active)
++ break;
++
+ for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
+ kvm_hv_notify_acked_sint(vcpu, i);
+ break;
+@@ -446,6 +449,9 @@ static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
+ struct kvm_lapic_irq irq;
+ int ret, vector;
+
++ if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
++ return -EINVAL;
++
+ if (sint >= ARRAY_SIZE(synic->sint))
+ return -EINVAL;
+
+@@ -658,7 +664,7 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
+
+- if (!synic->active && !host)
++ if (!synic->active && (!host || config))
+ return 1;
+
+ if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode &&
+@@ -687,7 +693,7 @@ static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
+
+- if (!synic->active && !host)
++ if (!synic->active && (!host || count))
+ return 1;
+
+ trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id,
+@@ -1749,7 +1755,7 @@ struct kvm_hv_hcall {
+ sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS];
+ };
+
+-static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex)
++static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
+ {
+ int i;
+ gpa_t gpa;
+@@ -1765,7 +1771,8 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool
+ int sparse_banks_len;
+ bool all_cpus;
+
+- if (!ex) {
++ if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST ||
++ hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) {
+ if (hc->fast) {
+ flush.address_space = hc->ingpa;
+ flush.flags = hc->outgpa;
+@@ -1819,7 +1826,8 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool
+
+ if (!all_cpus) {
+ if (hc->fast) {
+- if (sparse_banks_len > HV_HYPERCALL_MAX_XMM_REGISTERS - 1)
++ /* XMM0 is already consumed, each XMM holds two sparse banks. */
++ if (sparse_banks_len > 2 * (HV_HYPERCALL_MAX_XMM_REGISTERS - 1))
+ return HV_STATUS_INVALID_HYPERCALL_INPUT;
+ for (i = 0; i < sparse_banks_len; i += 2) {
+ sparse_banks[i] = sse128_lo(hc->xmm[i / 2 + 1]);
+@@ -1874,7 +1882,7 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector,
+ }
+ }
+
+-static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex)
++static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
+ {
+ struct kvm *kvm = vcpu->kvm;
+ struct hv_send_ipi_ex send_ipi_ex;
+@@ -1887,8 +1895,9 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool
+ int sparse_banks_len;
+ u32 vector;
+ bool all_cpus;
++ int i;
+
+- if (!ex) {
++ if (hc->code == HVCALL_SEND_IPI) {
+ if (!hc->fast) {
+ if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi,
+ sizeof(send_ipi))))
+@@ -1907,9 +1916,15 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool
+
+ trace_kvm_hv_send_ipi(vector, sparse_banks[0]);
+ } else {
+- if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex,
+- sizeof(send_ipi_ex))))
+- return HV_STATUS_INVALID_HYPERCALL_INPUT;
++ if (!hc->fast) {
++ if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex,
++ sizeof(send_ipi_ex))))
++ return HV_STATUS_INVALID_HYPERCALL_INPUT;
++ } else {
++ send_ipi_ex.vector = (u32)hc->ingpa;
++ send_ipi_ex.vp_set.format = hc->outgpa;
++ send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]);
++ }
+
+ trace_kvm_hv_send_ipi_ex(send_ipi_ex.vector,
+ send_ipi_ex.vp_set.format,
+@@ -1917,23 +1932,40 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool
+
+ vector = send_ipi_ex.vector;
+ valid_bank_mask = send_ipi_ex.vp_set.valid_bank_mask;
+- sparse_banks_len = bitmap_weight(&valid_bank_mask, 64) *
+- sizeof(sparse_banks[0]);
++ sparse_banks_len = bitmap_weight(&valid_bank_mask, 64);
+
+ all_cpus = send_ipi_ex.vp_set.format == HV_GENERIC_SET_ALL;
+
++ if (all_cpus)
++ goto check_and_send_ipi;
++
+ if (!sparse_banks_len)
+ goto ret_success;
+
+- if (!all_cpus &&
+- kvm_read_guest(kvm,
+- hc->ingpa + offsetof(struct hv_send_ipi_ex,
+- vp_set.bank_contents),
+- sparse_banks,
+- sparse_banks_len))
+- return HV_STATUS_INVALID_HYPERCALL_INPUT;
++ if (!hc->fast) {
++ if (kvm_read_guest(kvm,
++ hc->ingpa + offsetof(struct hv_send_ipi_ex,
++ vp_set.bank_contents),
++ sparse_banks,
++ sparse_banks_len * sizeof(sparse_banks[0])))
++ return HV_STATUS_INVALID_HYPERCALL_INPUT;
++ } else {
++ /*
++ * The lower half of XMM0 is already consumed, each XMM holds
++ * two sparse banks.
++ */
++ if (sparse_banks_len > (2 * HV_HYPERCALL_MAX_XMM_REGISTERS - 1))
++ return HV_STATUS_INVALID_HYPERCALL_INPUT;
++ for (i = 0; i < sparse_banks_len; i++) {
++ if (i % 2)
++ sparse_banks[i] = sse128_lo(hc->xmm[(i + 1) / 2]);
++ else
++ sparse_banks[i] = sse128_hi(hc->xmm[i / 2]);
++ }
++ }
+ }
+
++check_and_send_ipi:
+ if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
+ return HV_STATUS_INVALID_HYPERCALL_INPUT;
+
+@@ -2022,7 +2054,7 @@ static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
+ {
+ bool longmode;
+
+- longmode = is_64_bit_mode(vcpu);
++ longmode = is_64_bit_hypercall(vcpu);
+ if (longmode)
+ kvm_rax_write(vcpu, result);
+ else {
+@@ -2092,6 +2124,7 @@ static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc)
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
++ case HVCALL_SEND_IPI_EX:
+ return true;
+ }
+
+@@ -2171,7 +2204,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
+ }
+
+ #ifdef CONFIG_X86_64
+- if (is_64_bit_mode(vcpu)) {
++ if (is_64_bit_hypercall(vcpu)) {
+ hc.param = kvm_rcx_read(vcpu);
+ hc.ingpa = kvm_rdx_read(vcpu);
+ hc.outgpa = kvm_r8_read(vcpu);
+@@ -2243,46 +2276,28 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
+ kvm_hv_hypercall_complete_userspace;
+ return 0;
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
+- if (unlikely(!hc.rep_cnt || hc.rep_idx)) {
+- ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+- break;
+- }
+- ret = kvm_hv_flush_tlb(vcpu, &hc, false);
+- break;
+- case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
+- if (unlikely(hc.rep)) {
+- ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+- break;
+- }
+- ret = kvm_hv_flush_tlb(vcpu, &hc, false);
+- break;
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
+ if (unlikely(!hc.rep_cnt || hc.rep_idx)) {
+ ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+ break;
+ }
+- ret = kvm_hv_flush_tlb(vcpu, &hc, true);
++ ret = kvm_hv_flush_tlb(vcpu, &hc);
+ break;
++ case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
+ case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
+ if (unlikely(hc.rep)) {
+ ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+ break;
+ }
+- ret = kvm_hv_flush_tlb(vcpu, &hc, true);
++ ret = kvm_hv_flush_tlb(vcpu, &hc);
+ break;
+ case HVCALL_SEND_IPI:
+- if (unlikely(hc.rep)) {
+- ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+- break;
+- }
+- ret = kvm_hv_send_ipi(vcpu, &hc, false);
+- break;
+ case HVCALL_SEND_IPI_EX:
+- if (unlikely(hc.fast || hc.rep)) {
++ if (unlikely(hc.rep)) {
+ ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
+ break;
+ }
+- ret = kvm_hv_send_ipi(vcpu, &hc, true);
++ ret = kvm_hv_send_ipi(vcpu, &hc);
+ break;
+ case HVCALL_POST_DEBUG_DATA:
+ case HVCALL_RETRIEVE_DEBUG_DATA:
+diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c
+index 8c065da73f8e5..4e0f52660842b 100644
+--- a/arch/x86/kvm/ioapic.c
++++ b/arch/x86/kvm/ioapic.c
+@@ -96,7 +96,7 @@ static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic,
+ static void rtc_irq_eoi_tracking_reset(struct kvm_ioapic *ioapic)
+ {
+ ioapic->rtc_status.pending_eoi = 0;
+- bitmap_zero(ioapic->rtc_status.dest_map.map, KVM_MAX_VCPU_ID + 1);
++ bitmap_zero(ioapic->rtc_status.dest_map.map, KVM_MAX_VCPU_ID);
+ }
+
+ static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic);
+diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h
+index bbd4a5d18b5dc..f1b2b2a6ff4db 100644
+--- a/arch/x86/kvm/ioapic.h
++++ b/arch/x86/kvm/ioapic.h
+@@ -39,13 +39,13 @@ struct kvm_vcpu;
+
+ struct dest_map {
+ /* vcpu bitmap where IRQ has been sent */
+- DECLARE_BITMAP(map, KVM_MAX_VCPU_ID + 1);
++ DECLARE_BITMAP(map, KVM_MAX_VCPU_ID);
+
+ /*
+ * Vector sent to a given vcpu, only valid when
+ * the vcpu's bit in map is set
+ */
+- u8 vectors[KVM_MAX_VCPU_ID + 1];
++ u8 vectors[KVM_MAX_VCPU_ID];
+ };
+
+
+@@ -81,7 +81,6 @@ struct kvm_ioapic {
+ unsigned long irq_states[IOAPIC_NUM_PINS];
+ struct kvm_io_device dev;
+ struct kvm *kvm;
+- void (*ack_notifier)(void *opaque, int irq);
+ spinlock_t lock;
+ struct rtc_status rtc_status;
+ struct delayed_work eoi_inject;
+diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
+index 650642b18d151..c2d7cfe82d004 100644
+--- a/arch/x86/kvm/irq.h
++++ b/arch/x86/kvm/irq.h
+@@ -56,7 +56,6 @@ struct kvm_pic {
+ struct kvm_io_device dev_master;
+ struct kvm_io_device dev_slave;
+ struct kvm_io_device dev_elcr;
+- void (*ack_notifier)(void *opaque, int irq);
+ unsigned long irq_states[PIC_NUM_PINS];
+ };
+
+diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
+index 68b420289d7ed..fb09cd22cb7f5 100644
+--- a/arch/x86/kvm/kvm_emulate.h
++++ b/arch/x86/kvm/kvm_emulate.h
+@@ -226,6 +226,7 @@ struct x86_emulate_ops {
+ bool (*guest_has_long_mode)(struct x86_emulate_ctxt *ctxt);
+ bool (*guest_has_movbe)(struct x86_emulate_ctxt *ctxt);
+ bool (*guest_has_fxsr)(struct x86_emulate_ctxt *ctxt);
++ bool (*guest_has_rdpid)(struct x86_emulate_ctxt *ctxt);
+
+ void (*set_nmi_mask)(struct x86_emulate_ctxt *ctxt, bool masked);
+
+diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
+index d6ac32f3f650c..493d636e62316 100644
+--- a/arch/x86/kvm/lapic.c
++++ b/arch/x86/kvm/lapic.c
+@@ -113,7 +113,8 @@ static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
+
+ static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
+ {
+- return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
++ return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) &&
++ (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm));
+ }
+
+ bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
+@@ -676,38 +677,32 @@ static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
+ static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
+ {
+ u8 val;
+- if (pv_eoi_get_user(vcpu, &val) < 0) {
+- printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
+- (unsigned long long)vcpu->arch.pv_eoi.msr_val);
++ if (pv_eoi_get_user(vcpu, &val) < 0)
+ return false;
+- }
++
+ return val & KVM_PV_EOI_ENABLED;
+ }
+
+ static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
+ {
+- if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
+- printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
+- (unsigned long long)vcpu->arch.pv_eoi.msr_val);
++ if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0)
+ return;
+- }
++
+ __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
+ }
+
+ static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
+ {
+- if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
+- printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
+- (unsigned long long)vcpu->arch.pv_eoi.msr_val);
++ if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0)
+ return;
+- }
++
+ __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
+ }
+
+ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
+ {
+ int highest_irr;
+- if (apic->vcpu->arch.apicv_active)
++ if (kvm_x86_ops.sync_pir_to_irr)
+ highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
+ else
+ highest_irr = apic_find_highest_irr(apic);
+@@ -993,6 +988,10 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
+ *r = -1;
+
+ if (irq->shorthand == APIC_DEST_SELF) {
++ if (KVM_BUG_ON(!src, kvm)) {
++ *r = 0;
++ return true;
++ }
+ *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
+ return true;
+ }
+@@ -2127,10 +2126,9 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
+ break;
+
+ case APIC_SELF_IPI:
+- if (apic_x2apic_mode(apic)) {
+- kvm_lapic_reg_write(apic, APIC_ICR,
+- APIC_DEST_SELF | (val & APIC_VECTOR_MASK));
+- } else
++ if (apic_x2apic_mode(apic))
++ kvm_apic_send_ipi(apic, APIC_DEST_SELF | (val & APIC_VECTOR_MASK), 0);
++ else
+ ret = 1;
+ break;
+ default:
+@@ -2248,10 +2246,7 @@ void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
+
+ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
+ {
+- struct kvm_lapic *apic = vcpu->arch.apic;
+-
+- apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
+- | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
++ apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4);
+ }
+
+ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
+@@ -2629,7 +2624,7 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
+ kvm_apic_set_version(vcpu);
+
+ apic_update_ppr(apic);
+- hrtimer_cancel(&apic->lapic_timer.timer);
++ cancel_apic_timer(apic);
+ apic->lapic_timer.expired_tscdeadline = 0;
+ apic_update_lvtt(apic);
+ apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
+diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
+index e9688a9f7b579..7bb165c232334 100644
+--- a/arch/x86/kvm/mmu.h
++++ b/arch/x86/kvm/mmu.h
+@@ -49,6 +49,7 @@
+ X86_CR4_LA57)
+
+ #define KVM_MMU_CR0_ROLE_BITS (X86_CR0_PG | X86_CR0_WP)
++#define KVM_MMU_EFER_ROLE_BITS (EFER_LME | EFER_NX)
+
+ static __always_inline u64 rsvd_bits(int s, int e)
+ {
+diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
+index 0cc58901bf7a7..3417405077adf 100644
+--- a/arch/x86/kvm/mmu/mmu.c
++++ b/arch/x86/kvm/mmu/mmu.c
+@@ -1592,7 +1592,7 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
+ flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp);
+
+ if (is_tdp_mmu_enabled(kvm))
+- flush |= kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
++ flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
+
+ return flush;
+ }
+@@ -2188,10 +2188,10 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato
+ iterator->shadow_addr = root;
+ iterator->level = vcpu->arch.mmu->shadow_root_level;
+
+- if (iterator->level == PT64_ROOT_4LEVEL &&
++ if (iterator->level >= PT64_ROOT_4LEVEL &&
+ vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL &&
+ !vcpu->arch.mmu->direct_map)
+- --iterator->level;
++ iterator->level = PT32E_ROOT_LEVEL;
+
+ if (iterator->level == PT32E_ROOT_LEVEL) {
+ /*
+@@ -3314,6 +3314,8 @@ static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
+ return;
+
+ sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
++ if (WARN_ON(!sp))
++ return;
+
+ if (is_tdp_mmu_page(sp))
+ kvm_tdp_mmu_put_root(kvm, sp, false);
+@@ -3579,7 +3581,7 @@ set_root_pgd:
+ out_unlock:
+ write_unlock(&vcpu->kvm->mmu_lock);
+
+- return 0;
++ return r;
+ }
+
+ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
+@@ -3889,12 +3891,23 @@ static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
+ walk_shadow_page_lockless_end(vcpu);
+ }
+
++static u32 alloc_apf_token(struct kvm_vcpu *vcpu)
++{
++ /* make sure the token value is not 0 */
++ u32 id = vcpu->arch.apf.id;
++
++ if (id << 12 == 0)
++ vcpu->arch.apf.id = 1;
++
++ return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
++}
++
+ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
+ gfn_t gfn)
+ {
+ struct kvm_arch_async_pf arch;
+
+- arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
++ arch.token = alloc_apf_token(vcpu);
+ arch.gfn = gfn;
+ arch.direct_map = vcpu->arch.mmu->direct_map;
+ arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
+@@ -3956,6 +3969,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
+
+ *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL,
+ write, writable, hva);
++ return false;
+
+ out_retry:
+ *r = RET_PF_RETRY;
+@@ -4679,6 +4693,7 @@ static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu,
+ /* PKEY and LA57 are active iff long mode is active. */
+ ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
+ ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
++ ext.efer_lma = ____is_efer_lma(regs);
+ }
+
+ ext.valid = 1;
+@@ -4851,7 +4866,7 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
+ struct kvm_mmu *context = &vcpu->arch.guest_mmu;
+ struct kvm_mmu_role_regs regs = {
+ .cr0 = cr0,
+- .cr4 = cr4,
++ .cr4 = cr4 & ~X86_CR4_PKE,
+ .efer = efer,
+ };
+ union kvm_mmu_role new_role;
+@@ -4915,7 +4930,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
+ context->direct_map = false;
+
+ update_permission_bitmask(context, true);
+- update_pkru_bitmask(context);
++ context->pkru_mask = 0;
+ reset_rsvds_bits_mask_ept(vcpu, context, execonly);
+ reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
+ }
+@@ -5368,7 +5383,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+
+ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
+ {
+- kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE);
++ kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
+ ++vcpu->stat.invlpg;
+ }
+ EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
+@@ -5381,14 +5396,16 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
+ uint i;
+
+ if (pcid == kvm_get_active_pcid(vcpu)) {
+- mmu->invlpg(vcpu, gva, mmu->root_hpa);
++ if (mmu->invlpg)
++ mmu->invlpg(vcpu, gva, mmu->root_hpa);
+ tlb_flush = true;
+ }
+
+ for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
+ if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
+ pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
+- mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
++ if (mmu->invlpg)
++ mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
+ tlb_flush = true;
+ }
+ }
+@@ -5473,8 +5490,8 @@ slot_handle_level(struct kvm *kvm, const struct kvm_memory_slot *memslot,
+ }
+
+ static __always_inline bool
+-slot_handle_leaf(struct kvm *kvm, const struct kvm_memory_slot *memslot,
+- slot_level_handler fn, bool flush_on_yield)
++slot_handle_level_4k(struct kvm *kvm, const struct kvm_memory_slot *memslot,
++ slot_level_handler fn, bool flush_on_yield)
+ {
+ return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K,
+ PG_LEVEL_4K, flush_on_yield);
+@@ -5575,6 +5592,7 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm)
+ {
+ struct kvm_mmu_page *sp, *node;
+ int nr_zapped, batch = 0;
++ bool unstable;
+
+ restart:
+ list_for_each_entry_safe_reverse(sp, node,
+@@ -5606,11 +5624,12 @@ restart:
+ goto restart;
+ }
+
+- if (__kvm_mmu_prepare_zap_page(kvm, sp,
+- &kvm->arch.zapped_obsolete_pages, &nr_zapped)) {
+- batch += nr_zapped;
++ unstable = __kvm_mmu_prepare_zap_page(kvm, sp,
++ &kvm->arch.zapped_obsolete_pages, &nr_zapped);
++ batch += nr_zapped;
++
++ if (unstable)
+ goto restart;
+- }
+ }
+
+ /*
+@@ -5758,13 +5777,11 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
+ for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
+ flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start,
+ gfn_end, flush);
+- if (flush)
+- kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
+- gfn_end - gfn_start);
+ }
+
+ if (flush)
+- kvm_flush_remote_tlbs_with_address(kvm, gfn_start, gfn_end);
++ kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
++ gfn_end - gfn_start);
+
+ kvm_dec_notifier_count(kvm, gfn_start, gfn_end);
+
+@@ -5856,21 +5873,21 @@ restart:
+ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
+ const struct kvm_memory_slot *slot)
+ {
+- bool flush = false;
+-
+ if (kvm_memslots_have_rmaps(kvm)) {
+ write_lock(&kvm->mmu_lock);
+- flush = slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true);
+- if (flush)
++ /*
++ * Zap only 4k SPTEs since the legacy MMU only supports dirty
++ * logging at a 4k granularity and never creates collapsible
++ * 2m SPTEs during dirty logging.
++ */
++ if (slot_handle_level_4k(kvm, slot, kvm_mmu_zap_collapsible_spte, true))
+ kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
+ write_unlock(&kvm->mmu_lock);
+ }
+
+ if (is_tdp_mmu_enabled(kvm)) {
+ read_lock(&kvm->mmu_lock);
+- flush = kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot, flush);
+- if (flush)
+- kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
++ kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot);
+ read_unlock(&kvm->mmu_lock);
+ }
+ }
+@@ -5897,8 +5914,11 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
+
+ if (kvm_memslots_have_rmaps(kvm)) {
+ write_lock(&kvm->mmu_lock);
+- flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty,
+- false);
++ /*
++ * Clear dirty bits only on 4k SPTEs since the legacy MMU only
++ * support dirty logging at a 4k granularity.
++ */
++ flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
+ write_unlock(&kvm->mmu_lock);
+ }
+
+@@ -6091,12 +6111,24 @@ static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
+ return 0;
+ }
+
+-int kvm_mmu_module_init(void)
++/*
++ * nx_huge_pages needs to be resolved to true/false when kvm.ko is loaded, as
++ * its default value of -1 is technically undefined behavior for a boolean.
++ */
++void kvm_mmu_x86_module_init(void)
+ {
+- int ret = -ENOMEM;
+-
+ if (nx_huge_pages == -1)
+ __set_nx_huge_pages(get_nx_auto_mode());
++}
++
++/*
++ * The bulk of the MMU initialization is deferred until the vendor module is
++ * loaded as many of the masks/values may be modified by VMX or SVM, i.e. need
++ * to be reset when a potentially different vendor module is loaded.
++ */
++int kvm_mmu_vendor_module_init(void)
++{
++ int ret = -ENOMEM;
+
+ /*
+ * MMU roles use union aliasing which is, generally speaking, an
+@@ -6168,7 +6200,7 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
+ mmu_free_memory_caches(vcpu);
+ }
+
+-void kvm_mmu_module_exit(void)
++void kvm_mmu_vendor_module_exit(void)
+ {
+ mmu_destroy_caches();
+ percpu_counter_destroy(&kvm_total_used_mmu_pages);
+diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
+index 913d52a7923e6..a1811f51eda92 100644
+--- a/arch/x86/kvm/mmu/paging_tmpl.h
++++ b/arch/x86/kvm/mmu/paging_tmpl.h
+@@ -34,9 +34,8 @@
+ #define PT_HAVE_ACCESSED_DIRTY(mmu) true
+ #ifdef CONFIG_X86_64
+ #define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
+- #define CMPXCHG cmpxchg
++ #define CMPXCHG "cmpxchgq"
+ #else
+- #define CMPXCHG cmpxchg64
+ #define PT_MAX_FULL_LEVELS 2
+ #endif
+ #elif PTTYPE == 32
+@@ -52,7 +51,7 @@
+ #define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT
+ #define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT
+ #define PT_HAVE_ACCESSED_DIRTY(mmu) true
+- #define CMPXCHG cmpxchg
++ #define CMPXCHG "cmpxchgl"
+ #elif PTTYPE == PTTYPE_EPT
+ #define pt_element_t u64
+ #define guest_walker guest_walkerEPT
+@@ -65,7 +64,9 @@
+ #define PT_GUEST_DIRTY_SHIFT 9
+ #define PT_GUEST_ACCESSED_SHIFT 8
+ #define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad)
+- #define CMPXCHG cmpxchg64
++ #ifdef CONFIG_X86_64
++ #define CMPXCHG "cmpxchgq"
++ #endif
+ #define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
+ #else
+ #error Invalid PTTYPE value
+@@ -147,43 +148,39 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+ pt_element_t __user *ptep_user, unsigned index,
+ pt_element_t orig_pte, pt_element_t new_pte)
+ {
+- int npages;
+- pt_element_t ret;
+- pt_element_t *table;
+- struct page *page;
+-
+- npages = get_user_pages_fast((unsigned long)ptep_user, 1, FOLL_WRITE, &page);
+- if (likely(npages == 1)) {
+- table = kmap_atomic(page);
+- ret = CMPXCHG(&table[index], orig_pte, new_pte);
+- kunmap_atomic(table);
+-
+- kvm_release_page_dirty(page);
+- } else {
+- struct vm_area_struct *vma;
+- unsigned long vaddr = (unsigned long)ptep_user & PAGE_MASK;
+- unsigned long pfn;
+- unsigned long paddr;
+-
+- mmap_read_lock(current->mm);
+- vma = find_vma_intersection(current->mm, vaddr, vaddr + PAGE_SIZE);
+- if (!vma || !(vma->vm_flags & VM_PFNMAP)) {
+- mmap_read_unlock(current->mm);
+- return -EFAULT;
+- }
+- pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
+- paddr = pfn << PAGE_SHIFT;
+- table = memremap(paddr, PAGE_SIZE, MEMREMAP_WB);
+- if (!table) {
+- mmap_read_unlock(current->mm);
+- return -EFAULT;
+- }
+- ret = CMPXCHG(&table[index], orig_pte, new_pte);
+- memunmap(table);
+- mmap_read_unlock(current->mm);
+- }
++ int r = -EFAULT;
++
++ if (!user_access_begin(ptep_user, sizeof(pt_element_t)))
++ return -EFAULT;
++
++#ifdef CMPXCHG
++ asm volatile("1:" LOCK_PREFIX CMPXCHG " %[new], %[ptr]\n"
++ "mov $0, %[r]\n"
++ "setnz %b[r]\n"
++ "2:"
++ _ASM_EXTABLE_UA(1b, 2b)
++ : [ptr] "+m" (*ptep_user),
++ [old] "+a" (orig_pte),
++ [r] "+q" (r)
++ : [new] "r" (new_pte)
++ : "memory");
++#else
++ asm volatile("1:" LOCK_PREFIX "cmpxchg8b %[ptr]\n"
++ "movl $0, %[r]\n"
++ "jz 2f\n"
++ "incl %[r]\n"
++ "2:"
++ _ASM_EXTABLE_UA(1b, 2b)
++ : [ptr] "+m" (*ptep_user),
++ [old] "+A" (orig_pte),
++ [r] "+rm" (r)
++ : [new_lo] "b" ((u32)new_pte),
++ [new_hi] "c" ((u32)(new_pte >> 32))
++ : "memory");
++#endif
+
+- return (ret != orig_pte);
++ user_access_end();
++ return r;
+ }
+
+ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
+diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
+index eb7b227fc6cfe..31d6456d8ac33 100644
+--- a/arch/x86/kvm/mmu/spte.h
++++ b/arch/x86/kvm/mmu/spte.h
+@@ -310,12 +310,7 @@ static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
+ static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
+ u64 spte, int level)
+ {
+- /*
+- * Use a bitwise-OR instead of a logical-OR to aggregate the reserved
+- * bits and EPT's invalid memtype/XWR checks to avoid an extra Jcc
+- * (this is extremely unlikely to be short-circuited as true).
+- */
+- return __is_bad_mt_xwr(rsvd_check, spte) |
++ return __is_bad_mt_xwr(rsvd_check, spte) ||
+ __is_rsvd_bits_set(rsvd_check, spte, level);
+ }
+
+diff --git a/arch/x86/kvm/mmu/tdp_iter.c b/arch/x86/kvm/mmu/tdp_iter.c
+index b3ed302c1a359..caa96c270b954 100644
+--- a/arch/x86/kvm/mmu/tdp_iter.c
++++ b/arch/x86/kvm/mmu/tdp_iter.c
+@@ -26,6 +26,7 @@ static gfn_t round_gfn_for_level(gfn_t gfn, int level)
+ */
+ void tdp_iter_restart(struct tdp_iter *iter)
+ {
++ iter->yielded = false;
+ iter->yielded_gfn = iter->next_last_level_gfn;
+ iter->level = iter->root_level;
+
+@@ -160,6 +161,11 @@ static bool try_step_up(struct tdp_iter *iter)
+ */
+ void tdp_iter_next(struct tdp_iter *iter)
+ {
++ if (iter->yielded) {
++ tdp_iter_restart(iter);
++ return;
++ }
++
+ if (try_step_down(iter))
+ return;
+
+diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h
+index b1748b988d3ae..e19cabbcb65c8 100644
+--- a/arch/x86/kvm/mmu/tdp_iter.h
++++ b/arch/x86/kvm/mmu/tdp_iter.h
+@@ -45,6 +45,12 @@ struct tdp_iter {
+ * iterator walks off the end of the paging structure.
+ */
+ bool valid;
++ /*
++ * True if KVM dropped mmu_lock and yielded in the middle of a walk, in
++ * which case tdp_iter_next() needs to restart the walk at the root
++ * level instead of advancing to the next entry.
++ */
++ bool yielded;
+ };
+
+ /*
+diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
+index 64ccfc1fa5535..853780eb033b4 100644
+--- a/arch/x86/kvm/mmu/tdp_mmu.c
++++ b/arch/x86/kvm/mmu/tdp_mmu.c
+@@ -99,15 +99,18 @@ void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
+ }
+
+ /*
+- * Finds the next valid root after root (or the first valid root if root
+- * is NULL), takes a reference on it, and returns that next root. If root
+- * is not NULL, this thread should have already taken a reference on it, and
+- * that reference will be dropped. If no valid root is found, this
+- * function will return NULL.
++ * Returns the next root after @prev_root (or the first root if @prev_root is
++ * NULL). A reference to the returned root is acquired, and the reference to
++ * @prev_root is released (the caller obviously must hold a reference to
++ * @prev_root if it's non-NULL).
++ *
++ * If @only_valid is true, invalid roots are skipped.
++ *
++ * Returns NULL if the end of tdp_mmu_roots was reached.
+ */
+ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
+ struct kvm_mmu_page *prev_root,
+- bool shared)
++ bool shared, bool only_valid)
+ {
+ struct kvm_mmu_page *next_root;
+
+@@ -121,9 +124,14 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
+ next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots,
+ typeof(*next_root), link);
+
+- while (next_root && !kvm_tdp_mmu_get_root(kvm, next_root))
++ while (next_root) {
++ if ((!only_valid || !next_root->role.invalid) &&
++ kvm_tdp_mmu_get_root(kvm, next_root))
++ break;
++
+ next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,
+ &next_root->link, typeof(*next_root), link);
++ }
+
+ rcu_read_unlock();
+
+@@ -143,13 +151,19 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
+ * mode. In the unlikely event that this thread must free a root, the lock
+ * will be temporarily dropped and reacquired in write mode.
+ */
+-#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
+- for (_root = tdp_mmu_next_root(_kvm, NULL, _shared); \
+- _root; \
+- _root = tdp_mmu_next_root(_kvm, _root, _shared)) \
+- if (kvm_mmu_page_as_id(_root) != _as_id) { \
++#define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\
++ for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, _only_valid); \
++ _root; \
++ _root = tdp_mmu_next_root(_kvm, _root, _shared, _only_valid)) \
++ if (kvm_mmu_page_as_id(_root) != _as_id) { \
+ } else
+
++#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
++ __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)
++
++#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
++ __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, false)
++
+ #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \
+ list_for_each_entry_rcu(_root, &_kvm->arch.tdp_mmu_roots, link, \
+ lockdep_is_held_type(&kvm->mmu_lock, 0) || \
+@@ -199,7 +213,10 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu)
+
+ role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level);
+
+- /* Check for an existing root before allocating a new one. */
++ /*
++ * Check for an existing root before allocating a new one. Note, the
++ * role check prevents consuming an invalid root.
++ */
+ for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
+ if (root->role.word == role.word &&
+ kvm_tdp_mmu_get_root(kvm, root))
+@@ -316,9 +333,6 @@ static void handle_removed_tdp_mmu_page(struct kvm *kvm, tdp_ptep_t pt,
+ struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(pt));
+ int level = sp->role.level;
+ gfn_t base_gfn = sp->gfn;
+- u64 old_child_spte;
+- u64 *sptep;
+- gfn_t gfn;
+ int i;
+
+ trace_kvm_mmu_prepare_zap_page(sp);
+@@ -326,8 +340,9 @@ static void handle_removed_tdp_mmu_page(struct kvm *kvm, tdp_ptep_t pt,
+ tdp_mmu_unlink_page(kvm, sp, shared);
+
+ for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
+- sptep = rcu_dereference(pt) + i;
+- gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level);
++ u64 *sptep = rcu_dereference(pt) + i;
++ gfn_t gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level);
++ u64 old_child_spte;
+
+ if (shared) {
+ /*
+@@ -373,7 +388,7 @@ static void handle_removed_tdp_mmu_page(struct kvm *kvm, tdp_ptep_t pt,
+ shared);
+ }
+
+- kvm_flush_remote_tlbs_with_address(kvm, gfn,
++ kvm_flush_remote_tlbs_with_address(kvm, base_gfn,
+ KVM_PAGES_PER_HPAGE(level + 1));
+
+ call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);
+@@ -503,6 +518,8 @@ static inline bool tdp_mmu_set_spte_atomic_no_dirty_log(struct kvm *kvm,
+ struct tdp_iter *iter,
+ u64 new_spte)
+ {
++ WARN_ON_ONCE(iter->yielded);
++
+ lockdep_assert_held_read(&kvm->mmu_lock);
+
+ /*
+@@ -613,6 +630,8 @@ static inline void __tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
+ u64 new_spte, bool record_acc_track,
+ bool record_dirty_log)
+ {
++ WARN_ON_ONCE(iter->yielded);
++
+ lockdep_assert_held_write(&kvm->mmu_lock);
+
+ /*
+@@ -678,18 +697,19 @@ static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm,
+ * If this function should yield and flush is set, it will perform a remote
+ * TLB flush before yielding.
+ *
+- * If this function yields, it will also reset the tdp_iter's walk over the
+- * paging structure and the calling function should skip to the next
+- * iteration to allow the iterator to continue its traversal from the
+- * paging structure root.
++ * If this function yields, iter->yielded is set and the caller must skip to
++ * the next iteration, where tdp_iter_next() will reset the tdp_iter's walk
++ * over the paging structures to allow the iterator to continue its traversal
++ * from the paging structure root.
+ *
+- * Return true if this function yielded and the iterator's traversal was reset.
+- * Return false if a yield was not needed.
++ * Returns true if this function yielded.
+ */
+-static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm,
+- struct tdp_iter *iter, bool flush,
+- bool shared)
++static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm,
++ struct tdp_iter *iter,
++ bool flush, bool shared)
+ {
++ WARN_ON(iter->yielded);
++
+ /* Ensure forward progress has been made before yielding. */
+ if (iter->next_last_level_gfn == iter->yielded_gfn)
+ return false;
+@@ -709,12 +729,10 @@ static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm,
+
+ WARN_ON(iter->gfn > iter->next_last_level_gfn);
+
+- tdp_iter_restart(iter);
+-
+- return true;
++ iter->yielded = true;
+ }
+
+- return false;
++ return iter->yielded;
+ }
+
+ /*
+@@ -1083,8 +1101,8 @@ bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
+ struct kvm_mmu_page *root;
+
+ for_each_tdp_mmu_root(kvm, root, range->slot->as_id)
+- flush |= zap_gfn_range(kvm, root, range->start, range->end,
+- range->may_block, flush, false);
++ flush = zap_gfn_range(kvm, root, range->start, range->end,
++ range->may_block, flush, false);
+
+ return flush;
+ }
+@@ -1270,7 +1288,7 @@ bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
+
+ lockdep_assert_held_read(&kvm->mmu_lock);
+
+- for_each_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
++ for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
+ spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
+ slot->base_gfn + slot->npages, min_level);
+
+@@ -1298,6 +1316,9 @@ retry:
+ if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
+ continue;
+
++ if (!is_shadow_present_pte(iter.old_spte))
++ continue;
++
+ if (spte_ad_need_write_protect(iter.old_spte)) {
+ if (is_writable_pte(iter.old_spte))
+ new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
+@@ -1341,7 +1362,7 @@ bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
+
+ lockdep_assert_held_read(&kvm->mmu_lock);
+
+- for_each_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
++ for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
+ spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
+ slot->base_gfn + slot->npages);
+
+@@ -1415,10 +1436,9 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
+ * Clear leaf entries which could be replaced by large mappings, for
+ * GFNs within the slot.
+ */
+-static bool zap_collapsible_spte_range(struct kvm *kvm,
++static void zap_collapsible_spte_range(struct kvm *kvm,
+ struct kvm_mmu_page *root,
+- const struct kvm_memory_slot *slot,
+- bool flush)
++ const struct kvm_memory_slot *slot)
+ {
+ gfn_t start = slot->base_gfn;
+ gfn_t end = start + slot->npages;
+@@ -1429,10 +1449,8 @@ static bool zap_collapsible_spte_range(struct kvm *kvm,
+
+ tdp_root_for_each_pte(iter, root, start, end) {
+ retry:
+- if (tdp_mmu_iter_cond_resched(kvm, &iter, flush, true)) {
+- flush = false;
++ if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
+ continue;
+- }
+
+ if (!is_shadow_present_pte(iter.old_spte) ||
+ !is_last_spte(iter.old_spte, iter.level))
+@@ -1444,6 +1462,7 @@ retry:
+ pfn, PG_LEVEL_NUM))
+ continue;
+
++ /* Note, a successful atomic zap also does a remote TLB flush. */
+ if (!tdp_mmu_zap_spte_atomic(kvm, &iter)) {
+ /*
+ * The iter must explicitly re-read the SPTE because
+@@ -1452,30 +1471,24 @@ retry:
+ iter.old_spte = READ_ONCE(*rcu_dereference(iter.sptep));
+ goto retry;
+ }
+- flush = true;
+ }
+
+ rcu_read_unlock();
+-
+- return flush;
+ }
+
+ /*
+ * Clear non-leaf entries (and free associated page tables) which could
+ * be replaced by large mappings, for GFNs within the slot.
+ */
+-bool kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
+- const struct kvm_memory_slot *slot,
+- bool flush)
++void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
++ const struct kvm_memory_slot *slot)
+ {
+ struct kvm_mmu_page *root;
+
+ lockdep_assert_held_read(&kvm->mmu_lock);
+
+- for_each_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
+- flush = zap_collapsible_spte_range(kvm, root, slot, flush);
+-
+- return flush;
++ for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
++ zap_collapsible_spte_range(kvm, root, slot);
+ }
+
+ /*
+@@ -1500,12 +1513,12 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
+ !is_last_spte(iter.old_spte, iter.level))
+ continue;
+
+- if (!is_writable_pte(iter.old_spte))
+- break;
+-
+ new_spte = iter.old_spte &
+ ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask);
+
++ if (new_spte == iter.old_spte)
++ break;
++
+ tdp_mmu_set_spte(kvm, &iter, new_spte);
+ spte_set = true;
+ }
+diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h
+index 358f447d40120..39468b637d2e4 100644
+--- a/arch/x86/kvm/mmu/tdp_mmu.h
++++ b/arch/x86/kvm/mmu/tdp_mmu.h
+@@ -10,9 +10,6 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);
+ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm *kvm,
+ struct kvm_mmu_page *root)
+ {
+- if (root->role.invalid)
+- return false;
+-
+ return refcount_inc_not_zero(&root->tdp_mmu_root_count);
+ }
+
+@@ -66,9 +63,8 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
+ struct kvm_memory_slot *slot,
+ gfn_t gfn, unsigned long mask,
+ bool wrprot);
+-bool kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
+- const struct kvm_memory_slot *slot,
+- bool flush);
++void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
++ const struct kvm_memory_slot *slot);
+
+ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
+ struct kvm_memory_slot *slot, gfn_t gfn,
+diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
+index 0772bad9165c5..62333f9756a36 100644
+--- a/arch/x86/kvm/pmu.c
++++ b/arch/x86/kvm/pmu.c
+@@ -95,9 +95,8 @@ static void kvm_perf_overflow_intr(struct perf_event *perf_event,
+ }
+
+ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type,
+- unsigned config, bool exclude_user,
+- bool exclude_kernel, bool intr,
+- bool in_tx, bool in_tx_cp)
++ u64 config, bool exclude_user,
++ bool exclude_kernel, bool intr)
+ {
+ struct perf_event *event;
+ struct perf_event_attr attr = {
+@@ -113,16 +112,14 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type,
+
+ attr.sample_period = get_sample_period(pmc, pmc->counter);
+
+- if (in_tx)
+- attr.config |= HSW_IN_TX;
+- if (in_tx_cp) {
++ if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
++ guest_cpuid_is_intel(pmc->vcpu)) {
+ /*
+ * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
+ * period. Just clear the sample period so at least
+ * allocating the counter doesn't fail.
+ */
+ attr.sample_period = 0;
+- attr.config |= HSW_IN_TX_CHECKPOINTED;
+ }
+
+ event = perf_event_create_kernel_counter(&attr, -1, current,
+@@ -173,11 +170,12 @@ static bool pmc_resume_counter(struct kvm_pmc *pmc)
+
+ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
+ {
+- unsigned config, type = PERF_TYPE_RAW;
+- u8 event_select, unit_mask;
++ u64 config;
++ u32 type = PERF_TYPE_RAW;
+ struct kvm *kvm = pmc->vcpu->kvm;
+ struct kvm_pmu_event_filter *filter;
+ int i;
++ struct kvm_pmu *pmu = vcpu_to_pmu(pmc->vcpu);
+ bool allow_event = true;
+
+ if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
+@@ -206,23 +204,18 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
+ if (!allow_event)
+ return;
+
+- event_select = eventsel & ARCH_PERFMON_EVENTSEL_EVENT;
+- unit_mask = (eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
+-
+ if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE |
+ ARCH_PERFMON_EVENTSEL_INV |
+ ARCH_PERFMON_EVENTSEL_CMASK |
+ HSW_IN_TX |
+ HSW_IN_TX_CHECKPOINTED))) {
+- config = kvm_x86_ops.pmu_ops->find_arch_event(pmc_to_pmu(pmc),
+- event_select,
+- unit_mask);
++ config = kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc);
+ if (config != PERF_COUNT_HW_MAX)
+ type = PERF_TYPE_HARDWARE;
+ }
+
+ if (type == PERF_TYPE_RAW)
+- config = eventsel & X86_RAW_EVENT_MASK;
++ config = eventsel & pmu->raw_event_mask;
+
+ if (pmc->current_config == eventsel && pmc_resume_counter(pmc))
+ return;
+@@ -233,9 +226,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
+ pmc_reprogram_counter(pmc, type, config,
+ !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
+ !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
+- eventsel & ARCH_PERFMON_EVENTSEL_INT,
+- (eventsel & HSW_IN_TX),
+- (eventsel & HSW_IN_TX_CHECKPOINTED));
++ eventsel & ARCH_PERFMON_EVENTSEL_INT);
+ }
+ EXPORT_SYMBOL_GPL(reprogram_gp_counter);
+
+@@ -271,7 +262,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
+ kvm_x86_ops.pmu_ops->find_fixed_event(idx),
+ !(en_field & 0x2), /* exclude user */
+ !(en_field & 0x1), /* exclude kernel */
+- pmi, false, false);
++ pmi);
+ }
+ EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
+
+diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
+index 0e4f2b1fa9fbd..c206decb39fab 100644
+--- a/arch/x86/kvm/pmu.h
++++ b/arch/x86/kvm/pmu.h
+@@ -24,8 +24,7 @@ struct kvm_event_hw_type_mapping {
+ };
+
+ struct kvm_pmu_ops {
+- unsigned (*find_arch_event)(struct kvm_pmu *pmu, u8 event_select,
+- u8 unit_mask);
++ unsigned int (*pmc_perf_hw_id)(struct kvm_pmc *pmc);
+ unsigned (*find_fixed_event)(int idx);
+ bool (*pmc_is_enabled)(struct kvm_pmc *pmc);
+ struct kvm_pmc *(*pmc_idx_to_pmc)(struct kvm_pmu *pmu, int pmc_idx);
+@@ -142,6 +141,15 @@ static inline u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value)
+ return sample_period;
+ }
+
++static inline void pmc_update_sample_period(struct kvm_pmc *pmc)
++{
++ if (!pmc->perf_event || pmc->is_paused)
++ return;
++
++ perf_event_period(pmc->perf_event,
++ get_sample_period(pmc, pmc->counter));
++}
++
+ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel);
+ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int fixed_idx);
+ void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx);
+diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c
+index 8052d92069e01..3d3f8dfb80457 100644
+--- a/arch/x86/kvm/svm/avic.c
++++ b/arch/x86/kvm/svm/avic.c
+@@ -342,8 +342,6 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu)
+ avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh);
+ break;
+ case AVIC_IPI_FAILURE_INVALID_TARGET:
+- WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
+- index, vcpu->vcpu_id, icrh, icrl);
+ break;
+ case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
+ WARN_ONCE(1, "Invalid backing page\n");
+@@ -801,7 +799,7 @@ int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
+ {
+ struct kvm_kernel_irq_routing_entry *e;
+ struct kvm_irq_routing_table *irq_rt;
+- int idx, ret = -EINVAL;
++ int idx, ret = 0;
+
+ if (!kvm_arch_has_assigned_device(kvm) ||
+ !irq_remapping_cap(IRQ_POSTING_CAP))
+@@ -812,7 +810,13 @@ int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
+
+ idx = srcu_read_lock(&kvm->irq_srcu);
+ irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
+- WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
++
++ if (guest_irq >= irq_rt->nr_rt_entries ||
++ hlist_empty(&irq_rt->map[guest_irq])) {
++ pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
++ guest_irq, irq_rt->nr_rt_entries);
++ goto out;
++ }
+
+ hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
+ struct vcpu_data vcpu_info;
+@@ -943,15 +947,10 @@ out:
+ void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+ {
+ u64 entry;
+- /* ID = 0xff (broadcast), ID > 0xff (reserved) */
+ int h_physical_id = kvm_cpu_get_apicid(cpu);
+ struct vcpu_svm *svm = to_svm(vcpu);
+
+- /*
+- * Since the host physical APIC id is 8 bits,
+- * we can support host APIC ID upto 255.
+- */
+- if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
++ if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
+ return;
+
+ entry = READ_ONCE(*(svm->avic_physical_id_cache));
+@@ -988,16 +987,18 @@ void avic_vcpu_put(struct kvm_vcpu *vcpu)
+ static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
+ {
+ struct vcpu_svm *svm = to_svm(vcpu);
++ int cpu = get_cpu();
+
++ WARN_ON(cpu != vcpu->cpu);
+ svm->avic_is_running = is_run;
+
+- if (!kvm_vcpu_apicv_active(vcpu))
+- return;
+-
+- if (is_run)
+- avic_vcpu_load(vcpu, vcpu->cpu);
+- else
+- avic_vcpu_put(vcpu);
++ if (kvm_vcpu_apicv_active(vcpu)) {
++ if (is_run)
++ avic_vcpu_load(vcpu, cpu);
++ else
++ avic_vcpu_put(vcpu);
++ }
++ put_cpu();
+ }
+
+ void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
+diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
+index 510b833cbd399..556e7a3f35627 100644
+--- a/arch/x86/kvm/svm/nested.c
++++ b/arch/x86/kvm/svm/nested.c
+@@ -942,9 +942,9 @@ void svm_free_nested(struct vcpu_svm *svm)
+ /*
+ * Forcibly leave nested mode in order to be able to reset the VCPU later on.
+ */
+-void svm_leave_nested(struct vcpu_svm *svm)
++void svm_leave_nested(struct kvm_vcpu *vcpu)
+ {
+- struct kvm_vcpu *vcpu = &svm->vcpu;
++ struct vcpu_svm *svm = to_svm(vcpu);
+
+ if (is_guest_mode(vcpu)) {
+ svm->nested.nested_run_pending = 0;
+@@ -1313,7 +1313,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
+ return -EINVAL;
+
+ if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
+- svm_leave_nested(svm);
++ svm_leave_nested(vcpu);
+ svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
+ return 0;
+ }
+@@ -1357,18 +1357,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
+ !nested_vmcb_valid_sregs(vcpu, save))
+ goto out_free;
+
+- /*
+- * While the nested guest CR3 is already checked and set by
+- * KVM_SET_SREGS, it was set when nested state was yet loaded,
+- * thus MMU might not be initialized correctly.
+- * Set it again to fix this.
+- */
+-
+- ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
+- nested_npt_enabled(svm), false);
+- if (WARN_ON_ONCE(ret))
+- goto out_free;
+-
+
+ /*
+ * All checks done, we can enter guest mode. Userspace provides
+@@ -1378,7 +1366,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
+ */
+
+ if (is_guest_mode(vcpu))
+- svm_leave_nested(svm);
++ svm_leave_nested(vcpu);
+ else
+ svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;
+
+@@ -1394,6 +1382,20 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
+
+ svm_switch_vmcb(svm, &svm->nested.vmcb02);
+ nested_vmcb02_prepare_control(svm);
++
++ /*
++ * While the nested guest CR3 is already checked and set by
++ * KVM_SET_SREGS, it was set when nested state was yet loaded,
++ * thus MMU might not be initialized correctly.
++ * Set it again to fix this.
++ */
++
++ ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
++ nested_npt_enabled(svm), false);
++ if (WARN_ON_ONCE(ret))
++ goto out_free;
++
++
+ kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
+ ret = 0;
+ out_free:
+@@ -1432,6 +1434,7 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
+ }
+
+ struct kvm_x86_nested_ops svm_nested_ops = {
++ .leave_nested = svm_leave_nested,
+ .check_events = svm_check_nested_events,
+ .triple_fault = nested_svm_triple_fault,
+ .get_nested_state_pages = svm_get_nested_state_pages,
+diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c
+index fdf587f19c5fb..d35c94e13afb0 100644
+--- a/arch/x86/kvm/svm/pmu.c
++++ b/arch/x86/kvm/svm/pmu.c
+@@ -44,6 +44,22 @@ static struct kvm_event_hw_type_mapping amd_event_mapping[] = {
+ [7] = { 0xd1, 0x00, PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
+ };
+
++/* duplicated from amd_f17h_perfmon_event_map. */
++static struct kvm_event_hw_type_mapping amd_f17h_event_mapping[] = {
++ [0] = { 0x76, 0x00, PERF_COUNT_HW_CPU_CYCLES },
++ [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS },
++ [2] = { 0x60, 0xff, PERF_COUNT_HW_CACHE_REFERENCES },
++ [3] = { 0x64, 0x09, PERF_COUNT_HW_CACHE_MISSES },
++ [4] = { 0xc2, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
++ [5] = { 0xc3, 0x00, PERF_COUNT_HW_BRANCH_MISSES },
++ [6] = { 0x87, 0x02, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND },
++ [7] = { 0x87, 0x01, PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
++};
++
++/* amd_pmc_perf_hw_id depends on these being the same size */
++static_assert(ARRAY_SIZE(amd_event_mapping) ==
++ ARRAY_SIZE(amd_f17h_event_mapping));
++
+ static unsigned int get_msr_base(struct kvm_pmu *pmu, enum pmu_type type)
+ {
+ struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
+@@ -134,21 +150,27 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
+ return &pmu->gp_counters[msr_to_index(msr)];
+ }
+
+-static unsigned amd_find_arch_event(struct kvm_pmu *pmu,
+- u8 event_select,
+- u8 unit_mask)
++static unsigned int amd_pmc_perf_hw_id(struct kvm_pmc *pmc)
+ {
++ struct kvm_event_hw_type_mapping *event_mapping;
++ u8 event_select = pmc->eventsel & ARCH_PERFMON_EVENTSEL_EVENT;
++ u8 unit_mask = (pmc->eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
+ int i;
+
++ if (guest_cpuid_family(pmc->vcpu) >= 0x17)
++ event_mapping = amd_f17h_event_mapping;
++ else
++ event_mapping = amd_event_mapping;
++
+ for (i = 0; i < ARRAY_SIZE(amd_event_mapping); i++)
+- if (amd_event_mapping[i].eventsel == event_select
+- && amd_event_mapping[i].unit_mask == unit_mask)
++ if (event_mapping[i].eventsel == event_select
++ && event_mapping[i].unit_mask == unit_mask)
+ break;
+
+ if (i == ARRAY_SIZE(amd_event_mapping))
+ return PERF_COUNT_HW_MAX;
+
+- return amd_event_mapping[i].event_type;
++ return event_mapping[i].event_type;
+ }
+
+ /* return PERF_COUNT_HW_MAX as AMD doesn't have fixed events */
+@@ -256,17 +278,16 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+ pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_COUNTER);
+ if (pmc) {
+ pmc->counter += data - pmc_read_counter(pmc);
++ pmc_update_sample_period(pmc);
+ return 0;
+ }
+ /* MSR_EVNTSELn */
+ pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_EVNTSEL);
+ if (pmc) {
+- if (data == pmc->eventsel)
+- return 0;
+- if (!(data & pmu->reserved_bits)) {
++ data &= ~pmu->reserved_bits;
++ if (data != pmc->eventsel)
+ reprogram_gp_counter(pmc, data);
+- return 0;
+- }
++ return 0;
+ }
+
+ return 1;
+@@ -282,7 +303,8 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu)
+ pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS;
+
+ pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << 48) - 1;
+- pmu->reserved_bits = 0xffffffff00200000ull;
++ pmu->reserved_bits = 0xfffffff000280000ull;
++ pmu->raw_event_mask = AMD64_RAW_EVENT_MASK;
+ pmu->version = 1;
+ /* not applicable to AMD; but clean them to prevent any fall out */
+ pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
+@@ -320,7 +342,7 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu)
+ }
+
+ struct kvm_pmu_ops amd_pmu_ops = {
+- .find_arch_event = amd_find_arch_event,
++ .pmc_perf_hw_id = amd_pmc_perf_hw_id,
+ .find_fixed_event = amd_find_fixed_event,
+ .pmc_is_enabled = amd_pmc_is_enabled,
+ .pmc_idx_to_pmc = amd_pmc_idx_to_pmc,
+diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
+index 7e34d7163adab..c8c3212250618 100644
+--- a/arch/x86/kvm/svm/sev.c
++++ b/arch/x86/kvm/svm/sev.c
+@@ -1787,7 +1787,12 @@ int svm_vm_copy_asid_from(struct kvm *kvm, unsigned int source_fd)
+ mutex_unlock(&source_kvm->lock);
+ mutex_lock(&kvm->lock);
+
+- if (sev_guest(kvm)) {
++ /*
++ * Disallow out-of-band SEV/SEV-ES init if the target is already an
++ * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being
++ * created after SEV/SEV-ES initialization, e.g. to init intercepts.
++ */
++ if (sev_guest(kvm) || kvm->created_vcpus) {
+ ret = -EINVAL;
+ goto e_mirror_unlock;
+ }
+@@ -1800,6 +1805,7 @@ int svm_vm_copy_asid_from(struct kvm *kvm, unsigned int source_fd)
+ mirror_sev->fd = source_sev.fd;
+ mirror_sev->es_active = source_sev.es_active;
+ mirror_sev->handle = source_sev.handle;
++ INIT_LIST_HEAD(&mirror_sev->regions_list);
+ /*
+ * Do not copy ap_jump_table. Since the mirror does not share the same
+ * KVM contexts as the original, and they may have different
+@@ -1984,11 +1990,14 @@ static void sev_flush_guest_memory(struct vcpu_svm *svm, void *va,
+ unsigned long len)
+ {
+ /*
+- * If hardware enforced cache coherency for encrypted mappings of the
+- * same physical page is supported, nothing to do.
++ * If CPU enforced cache coherency for encrypted mappings of the
++ * same physical page is supported, use CLFLUSHOPT instead. NOTE: cache
++ * flush is still needed in order to work properly with DMA devices.
+ */
+- if (boot_cpu_has(X86_FEATURE_SME_COHERENT))
++ if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) {
++ clflush_cache_range(va, PAGE_SIZE);
+ return;
++ }
+
+ /*
+ * If the VM Page Flush MSR is supported, use it to flush the page
+@@ -2311,7 +2320,7 @@ void pre_sev_run(struct vcpu_svm *svm, int cpu)
+ }
+
+ #define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE)
+-static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
++static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+ {
+ struct vmcb_control_area *control = &svm->vmcb->control;
+ struct ghcb *ghcb = svm->ghcb;
+@@ -2322,14 +2331,14 @@ static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+ scratch_gpa_beg = ghcb_get_sw_scratch(ghcb);
+ if (!scratch_gpa_beg) {
+ pr_err("vmgexit: scratch gpa not provided\n");
+- return false;
++ return -EINVAL;
+ }
+
+ scratch_gpa_end = scratch_gpa_beg + len;
+ if (scratch_gpa_end < scratch_gpa_beg) {
+ pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n",
+ len, scratch_gpa_beg);
+- return false;
++ return -EINVAL;
+ }
+
+ if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) {
+@@ -2347,7 +2356,7 @@ static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+ scratch_gpa_end > ghcb_scratch_end) {
+ pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n",
+ scratch_gpa_beg, scratch_gpa_end);
+- return false;
++ return -EINVAL;
+ }
+
+ scratch_va = (void *)svm->ghcb;
+@@ -2360,18 +2369,18 @@ static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+ if (len > GHCB_SCRATCH_AREA_LIMIT) {
+ pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n",
+ len, GHCB_SCRATCH_AREA_LIMIT);
+- return false;
++ return -EINVAL;
+ }
+ scratch_va = kzalloc(len, GFP_KERNEL_ACCOUNT);
+ if (!scratch_va)
+- return false;
++ return -ENOMEM;
+
+ if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) {
+ /* Unable to copy scratch area from guest */
+ pr_err("vmgexit: kvm_read_guest for scratch area failed\n");
+
+ kfree(scratch_va);
+- return false;
++ return -EFAULT;
+ }
+
+ /*
+@@ -2387,7 +2396,7 @@ static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+ svm->ghcb_sa = scratch_va;
+ svm->ghcb_sa_len = len;
+
+- return true;
++ return 0;
+ }
+
+ static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask,
+@@ -2526,10 +2535,10 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
+ ghcb_set_sw_exit_info_1(ghcb, 0);
+ ghcb_set_sw_exit_info_2(ghcb, 0);
+
+- ret = -EINVAL;
+ switch (exit_code) {
+ case SVM_VMGEXIT_MMIO_READ:
+- if (!setup_vmgexit_scratch(svm, true, control->exit_info_2))
++ ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
++ if (ret)
+ break;
+
+ ret = kvm_sev_es_mmio_read(vcpu,
+@@ -2538,7 +2547,8 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
+ svm->ghcb_sa);
+ break;
+ case SVM_VMGEXIT_MMIO_WRITE:
+- if (!setup_vmgexit_scratch(svm, false, control->exit_info_2))
++ ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);
++ if (ret)
+ break;
+
+ ret = kvm_sev_es_mmio_write(vcpu,
+@@ -2581,6 +2591,7 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
+ vcpu_unimpl(vcpu,
+ "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n",
+ control->exit_info_1, control->exit_info_2);
++ ret = -EINVAL;
+ break;
+ default:
+ ret = svm_invoke_exit_handler(vcpu, exit_code);
+@@ -2593,6 +2604,7 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in)
+ {
+ int count;
+ int bytes;
++ int r;
+
+ if (svm->vmcb->control.exit_info_2 > INT_MAX)
+ return -EINVAL;
+@@ -2601,8 +2613,9 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in)
+ if (unlikely(check_mul_overflow(count, size, &bytes)))
+ return -EINVAL;
+
+- if (!setup_vmgexit_scratch(svm, in, bytes))
+- return -EINVAL;
++ r = setup_vmgexit_scratch(svm, in, bytes);
++ if (r)
++ return r;
+
+ return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->ghcb_sa, count, in);
+ }
+diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
+index 989685098b3ea..26f2da1590eda 100644
+--- a/arch/x86/kvm/svm/svm.c
++++ b/arch/x86/kvm/svm/svm.c
+@@ -281,7 +281,7 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
+
+ if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
+ if (!(efer & EFER_SVME)) {
+- svm_leave_nested(svm);
++ svm_leave_nested(vcpu);
+ svm_set_gif(svm, true);
+ /* #GP intercept is still needed for vmware backdoor */
+ if (!enable_vmware_backdoor)
+@@ -303,7 +303,11 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
+ return ret;
+ }
+
+- if (svm_gp_erratum_intercept)
++ /*
++ * Never intercept #GP for SEV guests, KVM can't
++ * decrypt guest memory to workaround the erratum.
++ */
++ if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
+ set_exception_intercept(svm, GP_VECTOR);
+ }
+ }
+@@ -1176,9 +1180,10 @@ static void init_vmcb(struct kvm_vcpu *vcpu)
+ * Guest access to VMware backdoor ports could legitimately
+ * trigger #GP because of TSS I/O permission bitmap.
+ * We intercept those #GP and allow access to them anyway
+- * as VMware does.
++ * as VMware does. Don't intercept #GP for SEV guests as KVM can't
++ * decrypt guest memory to decode the faulting instruction.
+ */
+- if (enable_vmware_backdoor)
++ if (enable_vmware_backdoor && !sev_guest(vcpu->kvm))
+ set_exception_intercept(svm, GP_VECTOR);
+
+ svm_set_intercept(svm, INTERCEPT_INTR);
+@@ -1517,6 +1522,15 @@ static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
+ to_svm(vcpu)->vmcb->save.rflags = rflags;
+ }
+
++static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
++{
++ struct vmcb *vmcb = to_svm(vcpu)->vmcb;
++
++ return sev_es_guest(vcpu->kvm)
++ ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
++ : kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
++}
++
+ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
+ {
+ switch (reg) {
+@@ -1713,6 +1727,7 @@ void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
+ {
+ struct vcpu_svm *svm = to_svm(vcpu);
+ u64 hcr0 = cr0;
++ bool old_paging = is_paging(vcpu);
+
+ #ifdef CONFIG_X86_64
+ if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) {
+@@ -1729,8 +1744,11 @@ void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
+ #endif
+ vcpu->arch.cr0 = cr0;
+
+- if (!npt_enabled)
++ if (!npt_enabled) {
+ hcr0 |= X86_CR0_PG | X86_CR0_WP;
++ if (old_paging != is_paging(vcpu))
++ svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
++ }
+
+ /*
+ * re-enable caching here because the QEMU bios
+@@ -1774,8 +1792,12 @@ void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
+ svm_flush_tlb(vcpu);
+
+ vcpu->arch.cr4 = cr4;
+- if (!npt_enabled)
++ if (!npt_enabled) {
+ cr4 |= X86_CR4_PAE;
++
++ if (!is_paging(vcpu))
++ cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
++ }
+ cr4 |= host_cr4_mce;
+ to_svm(vcpu)->vmcb->save.cr4 = cr4;
+ vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
+@@ -2224,10 +2246,6 @@ static int gp_interception(struct kvm_vcpu *vcpu)
+ if (error_code)
+ goto reinject;
+
+- /* All SVM instructions expect page aligned RAX */
+- if (svm->vmcb->save.rax & ~PAGE_MASK)
+- goto reinject;
+-
+ /* Decode the instruction for usage later */
+ if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
+ goto reinject;
+@@ -2245,8 +2263,13 @@ static int gp_interception(struct kvm_vcpu *vcpu)
+ if (!is_guest_mode(vcpu))
+ return kvm_emulate_instruction(vcpu,
+ EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
+- } else
++ } else {
++ /* All SVM instructions expect page aligned RAX */
++ if (svm->vmcb->save.rax & ~PAGE_MASK)
++ goto reinject;
++
+ return emulate_svm_instr(vcpu, opcode);
++ }
+
+ reinject:
+ kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
+@@ -3485,14 +3508,7 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
+ if (!gif_set(svm))
+ return true;
+
+- if (sev_es_guest(vcpu->kvm)) {
+- /*
+- * SEV-ES guests to not expose RFLAGS. Use the VMCB interrupt mask
+- * bit to determine the state of the IF flag.
+- */
+- if (!(vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK))
+- return true;
+- } else if (is_guest_mode(vcpu)) {
++ if (is_guest_mode(vcpu)) {
+ /* As long as interrupts are being delivered... */
+ if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
+ ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
+@@ -3503,7 +3519,7 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
+ if (nested_exit_on_intr(svm))
+ return false;
+ } else {
+- if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
++ if (!svm_get_if_flag(vcpu))
+ return true;
+ }
+
+@@ -4376,10 +4392,17 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
+ * Enter the nested guest now
+ */
+
++ vmcb_mark_all_dirty(svm->vmcb01.ptr);
++
+ vmcb12 = map.hva;
+ nested_load_control_from_vmcb12(svm, &vmcb12->control);
+ ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false);
+
++ if (ret)
++ goto unmap_save;
++
++ svm->nested.nested_run_pending = 1;
++
+ unmap_save:
+ kvm_vcpu_unmap(vcpu, &map_save, true);
+ unmap_map:
+@@ -4405,8 +4428,13 @@ static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int i
+ bool smep, smap, is_user;
+ unsigned long cr4;
+
++ /* Emulation is always possible when KVM has access to all guest state. */
++ if (!sev_guest(vcpu->kvm))
++ return true;
++
+ /*
+- * When the guest is an SEV-ES guest, emulation is not possible.
++ * Emulation is impossible for SEV-ES guests as KVM doesn't have access
++ * to guest register state.
+ */
+ if (sev_es_guest(vcpu->kvm))
+ return false;
+@@ -4454,23 +4482,27 @@ static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int i
+ if (likely(!insn || insn_len))
+ return true;
+
+- /*
+- * If RIP is invalid, go ahead with emulation which will cause an
+- * internal error exit.
+- */
+- if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT))
+- return true;
+-
+ cr4 = kvm_read_cr4(vcpu);
+ smep = cr4 & X86_CR4_SMEP;
+ smap = cr4 & X86_CR4_SMAP;
+ is_user = svm_get_cpl(vcpu) == 3;
+ if (smap && (!smep || is_user)) {
+- if (!sev_guest(vcpu->kvm))
+- return true;
+-
+ pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
+- kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
++
++ /*
++ * If the fault occurred in userspace, arbitrarily inject #GP
++ * to avoid killing the guest and to hopefully avoid confusing
++ * the guest kernel too much, e.g. injecting #PF would not be
++ * coherent with respect to the guest's page tables. Request
++ * triple fault if the fault occurred in the kernel as there's
++ * no fault that KVM can inject without confusing the guest.
++ * In practice, the triple fault is moot as no sane SEV kernel
++ * will execute from user memory while also running with SMAP=1.
++ */
++ if (is_user)
++ kvm_inject_gp(vcpu, 0);
++ else
++ kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
+ }
+
+ return false;
+@@ -4562,6 +4594,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
+ .cache_reg = svm_cache_reg,
+ .get_rflags = svm_get_rflags,
+ .set_rflags = svm_set_rflags,
++ .get_if_flag = svm_get_if_flag,
+
+ .tlb_flush_all = svm_flush_tlb,
+ .tlb_flush_current = svm_flush_tlb,
+@@ -4592,7 +4625,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
+ .load_eoi_exitmap = svm_load_eoi_exitmap,
+ .hwapic_irr_update = svm_hwapic_irr_update,
+ .hwapic_isr_update = svm_hwapic_isr_update,
+- .sync_pir_to_irr = kvm_lapic_find_highest_irr,
+ .apicv_post_state_restore = avic_post_state_restore,
+
+ .set_tss_addr = svm_set_tss_addr,
+diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
+index 5d30db599e10d..cf2d8365aeb4b 100644
+--- a/arch/x86/kvm/svm/svm.h
++++ b/arch/x86/kvm/svm/svm.h
+@@ -22,6 +22,8 @@
+ #include <asm/svm.h>
+ #include <asm/sev-common.h>
+
++#include "kvm_cache_regs.h"
++
+ #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
+
+ #define IOPM_SIZE PAGE_SIZE * 3
+@@ -461,7 +463,7 @@ static inline bool nested_exit_on_nmi(struct vcpu_svm *svm)
+
+ int enter_svm_guest_mode(struct kvm_vcpu *vcpu,
+ u64 vmcb_gpa, struct vmcb *vmcb12, bool from_vmrun);
+-void svm_leave_nested(struct vcpu_svm *svm);
++void svm_leave_nested(struct kvm_vcpu *vcpu);
+ void svm_free_nested(struct vcpu_svm *svm);
+ int svm_allocate_nested(struct vcpu_svm *svm);
+ int nested_svm_vmrun(struct kvm_vcpu *vcpu);
+@@ -497,7 +499,7 @@ extern struct kvm_x86_nested_ops svm_nested_ops;
+ #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31
+ #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31)
+
+-#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL)
++#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK GENMASK_ULL(11, 0)
+ #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12)
+ #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62)
+ #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63)
+diff --git a/arch/x86/kvm/svm/svm_onhyperv.c b/arch/x86/kvm/svm/svm_onhyperv.c
+index 98aa981c04ec5..8cdc62c74a964 100644
+--- a/arch/x86/kvm/svm/svm_onhyperv.c
++++ b/arch/x86/kvm/svm/svm_onhyperv.c
+@@ -4,7 +4,6 @@
+ */
+
+ #include <linux/kvm_host.h>
+-#include "kvm_cache_regs.h"
+
+ #include <asm/mshyperv.h>
+
+diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
+index 4fa17df123cd6..dfaeb47fcf2a7 100644
+--- a/arch/x86/kvm/svm/vmenter.S
++++ b/arch/x86/kvm/svm/vmenter.S
+@@ -148,7 +148,7 @@ SYM_FUNC_START(__svm_vcpu_run)
+ pop %edi
+ #endif
+ pop %_ASM_BP
+- ret
++ RET
+
+ 3: cmpb $0, kvm_rebooting
+ jne 2b
+@@ -202,7 +202,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
+ pop %edi
+ #endif
+ pop %_ASM_BP
+- ret
++ RET
+
+ 3: cmpb $0, kvm_rebooting
+ jne 2b
+diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c
+index ba6f99f584ac3..a7ed30d5647af 100644
+--- a/arch/x86/kvm/vmx/evmcs.c
++++ b/arch/x86/kvm/vmx/evmcs.c
+@@ -362,6 +362,7 @@ void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata)
+ case MSR_IA32_VMX_PROCBASED_CTLS2:
+ ctl_high &= ~EVMCS1_UNSUPPORTED_2NDEXEC;
+ break;
++ case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
+ case MSR_IA32_VMX_PINBASED_CTLS:
+ ctl_high &= ~EVMCS1_UNSUPPORTED_PINCTRL;
+ break;
+diff --git a/arch/x86/kvm/vmx/evmcs.h b/arch/x86/kvm/vmx/evmcs.h
+index 152ab0aa82cf6..b43976e4b9636 100644
+--- a/arch/x86/kvm/vmx/evmcs.h
++++ b/arch/x86/kvm/vmx/evmcs.h
+@@ -59,7 +59,9 @@ DECLARE_STATIC_KEY_FALSE(enable_evmcs);
+ SECONDARY_EXEC_SHADOW_VMCS | \
+ SECONDARY_EXEC_TSC_SCALING | \
+ SECONDARY_EXEC_PAUSE_LOOP_EXITING)
+-#define EVMCS1_UNSUPPORTED_VMEXIT_CTRL (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
++#define EVMCS1_UNSUPPORTED_VMEXIT_CTRL \
++ (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \
++ VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
+ #define EVMCS1_UNSUPPORTED_VMENTRY_CTRL (VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
+ #define EVMCS1_UNSUPPORTED_VMFUNC (VMX_VMFUNC_EPTP_SWITCHING)
+
+diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
+index eedcebf580041..1546a10ecb564 100644
+--- a/arch/x86/kvm/vmx/nested.c
++++ b/arch/x86/kvm/vmx/nested.c
+@@ -523,29 +523,6 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
+ return 0;
+ }
+
+-/*
+- * Check if MSR is intercepted for L01 MSR bitmap.
+- */
+-static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
+-{
+- unsigned long *msr_bitmap;
+- int f = sizeof(unsigned long);
+-
+- if (!cpu_has_vmx_msr_bitmap())
+- return true;
+-
+- msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
+-
+- if (msr <= 0x1fff) {
+- return !!test_bit(msr, msr_bitmap + 0x800 / f);
+- } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+- msr &= 0x1fff;
+- return !!test_bit(msr, msr_bitmap + 0xc00 / f);
+- }
+-
+- return true;
+-}
+-
+ /*
+ * If a msr is allowed by L0, we should check whether it is allowed by L1.
+ * The corresponding bit will be cleared unless both of L0 and L1 allow it.
+@@ -599,6 +576,34 @@ static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap)
+ }
+ }
+
++#define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \
++static inline \
++void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \
++ unsigned long *msr_bitmap_l1, \
++ unsigned long *msr_bitmap_l0, u32 msr) \
++{ \
++ if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \
++ vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \
++ vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \
++ else \
++ vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \
++}
++BUILD_NVMX_MSR_INTERCEPT_HELPER(read)
++BUILD_NVMX_MSR_INTERCEPT_HELPER(write)
++
++static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx,
++ unsigned long *msr_bitmap_l1,
++ unsigned long *msr_bitmap_l0,
++ u32 msr, int types)
++{
++ if (types & MSR_TYPE_R)
++ nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1,
++ msr_bitmap_l0, msr);
++ if (types & MSR_TYPE_W)
++ nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1,
++ msr_bitmap_l0, msr);
++}
++
+ /*
+ * Merge L0's and L1's MSR bitmap, return false to indicate that
+ * we do not use the hardware.
+@@ -606,10 +611,11 @@ static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap)
+ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
+ struct vmcs12 *vmcs12)
+ {
++ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ int msr;
+ unsigned long *msr_bitmap_l1;
+- unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
+- struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map;
++ unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap;
++ struct kvm_host_map *map = &vmx->nested.msr_bitmap_map;
+
+ /* Nothing to do if the MSR bitmap is not in use. */
+ if (!cpu_has_vmx_msr_bitmap() ||
+@@ -660,44 +666,27 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
+ }
+ }
+
+- /* KVM unconditionally exposes the FS/GS base MSRs to L1. */
++ /*
++ * Always check vmcs01's bitmap to honor userspace MSR filters and any
++ * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through.
++ */
+ #ifdef CONFIG_X86_64
+- nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
+- MSR_FS_BASE, MSR_TYPE_RW);
++ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
++ MSR_FS_BASE, MSR_TYPE_RW);
+
+- nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
+- MSR_GS_BASE, MSR_TYPE_RW);
++ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
++ MSR_GS_BASE, MSR_TYPE_RW);
+
+- nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
+- MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
++ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
++ MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
+ #endif
++ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
++ MSR_IA32_SPEC_CTRL, MSR_TYPE_RW);
+
+- /*
+- * Checking the L0->L1 bitmap is trying to verify two things:
+- *
+- * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
+- * ensures that we do not accidentally generate an L02 MSR bitmap
+- * from the L12 MSR bitmap that is too permissive.
+- * 2. That L1 or L2s have actually used the MSR. This avoids
+- * unnecessarily merging of the bitmap if the MSR is unused. This
+- * works properly because we only update the L01 MSR bitmap lazily.
+- * So even if L0 should pass L1 these MSRs, the L01 bitmap is only
+- * updated to reflect this when L1 (or its L2s) actually write to
+- * the MSR.
+- */
+- if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL))
+- nested_vmx_disable_intercept_for_msr(
+- msr_bitmap_l1, msr_bitmap_l0,
+- MSR_IA32_SPEC_CTRL,
+- MSR_TYPE_R | MSR_TYPE_W);
+-
+- if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD))
+- nested_vmx_disable_intercept_for_msr(
+- msr_bitmap_l1, msr_bitmap_l0,
+- MSR_IA32_PRED_CMD,
+- MSR_TYPE_W);
++ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
++ MSR_IA32_PRED_CMD, MSR_TYPE_W);
+
+- kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false);
++ kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false);
+
+ return true;
+ }
+@@ -1191,29 +1180,26 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu,
+ WARN_ON(!enable_vpid);
+
+ /*
+- * If VPID is enabled and used by vmc12, but L2 does not have a unique
+- * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate
+- * a VPID for L2, flush the current context as the effective ASID is
+- * common to both L1 and L2.
+- *
+- * Defer the flush so that it runs after vmcs02.EPTP has been set by
+- * KVM_REQ_LOAD_MMU_PGD (if nested EPT is enabled) and to avoid
+- * redundant flushes further down the nested pipeline.
+- *
+- * If a TLB flush isn't required due to any of the above, and vpid12 is
+- * changing then the new "virtual" VPID (vpid12) will reuse the same
+- * "real" VPID (vpid02), and so needs to be flushed. There's no direct
+- * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for
+- * all nested vCPUs. Remember, a flush on VM-Enter does not invalidate
+- * guest-physical mappings, so there is no need to sync the nEPT MMU.
++ * VPID is enabled and in use by vmcs12. If vpid12 is changing, then
++ * emulate a guest TLB flush as KVM does not track vpid12 history nor
++ * is the VPID incorporated into the MMU context. I.e. KVM must assume
++ * that the new vpid12 has never been used and thus represents a new
++ * guest ASID that cannot have entries in the TLB.
+ */
+- if (!nested_has_guest_tlb_tag(vcpu)) {
+- kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+- } else if (is_vmenter &&
+- vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
++ if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
+ vmx->nested.last_vpid = vmcs12->virtual_processor_id;
+- vpid_sync_context(nested_get_vpid02(vcpu));
++ kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
++ return;
+ }
++
++ /*
++ * If VPID is enabled, used by vmc12, and vpid12 is not changing but
++ * does not have a unique TLB tag (ASID), i.e. EPT is disabled and
++ * KVM was unable to allocate a VPID for L2, flush the current context
++ * as the effective ASID is common to both L1 and L2.
++ */
++ if (!nested_has_guest_tlb_tag(vcpu))
++ kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+ }
+
+ static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
+@@ -2623,8 +2609,10 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
+
+ if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
+ WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
+- vmcs12->guest_ia32_perf_global_ctrl)))
++ vmcs12->guest_ia32_perf_global_ctrl))) {
++ *entry_failure_code = ENTRY_FAIL_DEFAULT;
+ return -EINVAL;
++ }
+
+ kvm_rsp_write(vcpu, vmcs12->guest_rsp);
+ kvm_rip_write(vcpu, vmcs12->guest_rip);
+@@ -2865,6 +2853,17 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
+ return 0;
+ }
+
++static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,
++ struct vmcs12 *vmcs12)
++{
++#ifdef CONFIG_X86_64
++ if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) !=
++ !!(vcpu->arch.efer & EFER_LMA)))
++ return -EINVAL;
++#endif
++ return 0;
++}
++
+ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
+ struct vmcs12 *vmcs12)
+ {
+@@ -2889,18 +2888,16 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
+ return -EINVAL;
+
+ #ifdef CONFIG_X86_64
+- ia32e = !!(vcpu->arch.efer & EFER_LMA);
++ ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE);
+ #else
+ ia32e = false;
+ #endif
+
+ if (ia32e) {
+- if (CC(!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)) ||
+- CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
++ if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
+ return -EINVAL;
+ } else {
+- if (CC(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) ||
+- CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
++ if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
+ CC(vmcs12->host_cr4 & X86_CR4_PCIDE) ||
+ CC((vmcs12->host_rip) >> 32))
+ return -EINVAL;
+@@ -3360,8 +3357,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
+ };
+ u32 failed_index;
+
+- if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
+- kvm_vcpu_flush_tlb_current(vcpu);
++ kvm_service_local_tlb_flush_requests(vcpu);
+
+ evaluate_pending_interrupts = exec_controls_get(vmx) &
+ (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING);
+@@ -3570,6 +3566,9 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
+ if (nested_vmx_check_controls(vcpu, vmcs12))
+ return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
+
++ if (nested_vmx_check_address_space_size(vcpu, vmcs12))
++ return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
++
+ if (nested_vmx_check_host_state(vcpu, vmcs12))
+ return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
+
+@@ -4515,9 +4514,8 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
+ (void)nested_get_evmcs_page(vcpu);
+ }
+
+- /* Service the TLB flush request for L2 before switching to L1. */
+- if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
+- kvm_vcpu_flush_tlb_current(vcpu);
++ /* Service pending TLB flush requests for L2 before switching to L1. */
++ kvm_service_local_tlb_flush_requests(vcpu);
+
+ /*
+ * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
+@@ -4603,6 +4601,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
+ kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
+ }
+
++ if (vmx->nested.update_vmcs01_apicv_status) {
++ vmx->nested.update_vmcs01_apicv_status = false;
++ kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
++ }
++
+ if ((vm_exit_reason != -1) &&
+ (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)))
+ vmx->nested.need_vmcs12_to_shadow_sync = true;
+@@ -6750,6 +6753,7 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
+ }
+
+ struct kvm_x86_nested_ops vmx_nested_ops = {
++ .leave_nested = vmx_leave_nested,
+ .check_events = vmx_check_nested_events,
+ .hv_timer_pending = nested_vmx_preemption_timer_pending,
+ .triple_fault = nested_vmx_triple_fault,
+diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
+index 10cc4f65c4efd..e7275ce15a8b0 100644
+--- a/arch/x86/kvm/vmx/pmu_intel.c
++++ b/arch/x86/kvm/vmx/pmu_intel.c
+@@ -68,10 +68,11 @@ static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data)
+ reprogram_counter(pmu, bit);
+ }
+
+-static unsigned intel_find_arch_event(struct kvm_pmu *pmu,
+- u8 event_select,
+- u8 unit_mask)
++static unsigned int intel_pmc_perf_hw_id(struct kvm_pmc *pmc)
+ {
++ struct kvm_pmu *pmu = pmc_to_pmu(pmc);
++ u8 event_select = pmc->eventsel & ARCH_PERFMON_EVENTSEL_EVENT;
++ u8 unit_mask = (pmc->eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++)
+@@ -395,6 +396,7 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+ struct kvm_pmc *pmc;
+ u32 msr = msr_info->index;
+ u64 data = msr_info->data;
++ u64 reserved_bits;
+
+ switch (msr) {
+ case MSR_CORE_PERF_FIXED_CTR_CTRL:
+@@ -437,20 +439,20 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+ !(msr & MSR_PMC_FULL_WIDTH_BIT))
+ data = (s64)(s32)data;
+ pmc->counter += data - pmc_read_counter(pmc);
+- if (pmc->perf_event && !pmc->is_paused)
+- perf_event_period(pmc->perf_event,
+- get_sample_period(pmc, data));
++ pmc_update_sample_period(pmc);
+ return 0;
+ } else if ((pmc = get_fixed_pmc(pmu, msr))) {
+ pmc->counter += data - pmc_read_counter(pmc);
+- if (pmc->perf_event && !pmc->is_paused)
+- perf_event_period(pmc->perf_event,
+- get_sample_period(pmc, data));
++ pmc_update_sample_period(pmc);
+ return 0;
+ } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
+ if (data == pmc->eventsel)
+ return 0;
+- if (!(data & pmu->reserved_bits)) {
++ reserved_bits = pmu->reserved_bits;
++ if ((pmc->idx == 2) &&
++ (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED))
++ reserved_bits ^= HSW_IN_TX_CHECKPOINTED;
++ if (!(data & reserved_bits)) {
+ reprogram_gp_counter(pmc, data);
+ return 0;
+ }
+@@ -477,6 +479,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
+ pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
+ pmu->version = 0;
+ pmu->reserved_bits = 0xffffffff00200000ull;
++ pmu->raw_event_mask = X86_RAW_EVENT_MASK;
+
+ entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
+ if (!entry)
+@@ -523,8 +526,10 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
+ entry = kvm_find_cpuid_entry(vcpu, 7, 0);
+ if (entry &&
+ (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
+- (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM)))
+- pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED;
++ (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) {
++ pmu->reserved_bits ^= HSW_IN_TX;
++ pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
++ }
+
+ bitmap_set(pmu->all_valid_pmc_idx,
+ 0, pmu->nr_arch_gp_counters);
+@@ -706,7 +711,7 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
+ }
+
+ struct kvm_pmu_ops intel_pmu_ops = {
+- .find_arch_event = intel_find_arch_event,
++ .pmc_perf_hw_id = intel_pmc_perf_hw_id,
+ .find_fixed_event = intel_find_fixed_event,
+ .pmc_is_enabled = intel_pmc_is_enabled,
+ .pmc_idx_to_pmc = intel_pmc_idx_to_pmc,
+diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
+index 5f81ef092bd43..46fb83d6a286e 100644
+--- a/arch/x86/kvm/vmx/posted_intr.c
++++ b/arch/x86/kvm/vmx/posted_intr.c
+@@ -5,6 +5,7 @@
+ #include <asm/cpu.h>
+
+ #include "lapic.h"
++#include "irq.h"
+ #include "posted_intr.h"
+ #include "trace.h"
+ #include "vmx.h"
+@@ -14,7 +15,7 @@
+ * can find which vCPU should be waken up.
+ */
+ static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
+-static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
++static DEFINE_PER_CPU(raw_spinlock_t, blocked_vcpu_on_cpu_lock);
+
+ static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
+ {
+@@ -50,7 +51,7 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
+
+ /* The full case. */
+ do {
+- old.control = new.control = pi_desc->control;
++ old.control = new.control = READ_ONCE(pi_desc->control);
+
+ dest = cpu_physical_id(cpu);
+
+@@ -77,13 +78,18 @@ after_clear_sn:
+ pi_set_on(pi_desc);
+ }
+
++static bool vmx_can_use_vtd_pi(struct kvm *kvm)
++{
++ return irqchip_in_kernel(kvm) && enable_apicv &&
++ kvm_arch_has_assigned_device(kvm) &&
++ irq_remapping_cap(IRQ_POSTING_CAP);
++}
++
+ void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
+ {
+ struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
+
+- if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
+- !irq_remapping_cap(IRQ_POSTING_CAP) ||
+- !kvm_vcpu_apicv_active(vcpu))
++ if (!vmx_can_use_vtd_pi(vcpu->kvm))
+ return;
+
+ /* Set SN when the vCPU is preempted */
+@@ -98,7 +104,7 @@ static void __pi_post_block(struct kvm_vcpu *vcpu)
+ unsigned int dest;
+
+ do {
+- old.control = new.control = pi_desc->control;
++ old.control = new.control = READ_ONCE(pi_desc->control);
+ WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
+ "Wakeup handler not enabled while the VCPU is blocked\n");
+
+@@ -115,9 +121,9 @@ static void __pi_post_block(struct kvm_vcpu *vcpu)
+ new.control) != old.control);
+
+ if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
+- spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
++ raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
+ list_del(&vcpu->blocked_vcpu_list);
+- spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
++ raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
+ vcpu->pre_pcpu = -1;
+ }
+ }
+@@ -141,24 +147,23 @@ int pi_pre_block(struct kvm_vcpu *vcpu)
+ struct pi_desc old, new;
+ struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
+
+- if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
+- !irq_remapping_cap(IRQ_POSTING_CAP) ||
+- !kvm_vcpu_apicv_active(vcpu))
++ if (!vmx_can_use_vtd_pi(vcpu->kvm) ||
++ vmx_interrupt_blocked(vcpu))
+ return 0;
+
+ WARN_ON(irqs_disabled());
+ local_irq_disable();
+ if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
+ vcpu->pre_pcpu = vcpu->cpu;
+- spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
++ raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
+ list_add_tail(&vcpu->blocked_vcpu_list,
+ &per_cpu(blocked_vcpu_on_cpu,
+ vcpu->pre_pcpu));
+- spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
++ raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
+ }
+
+ do {
+- old.control = new.control = pi_desc->control;
++ old.control = new.control = READ_ONCE(pi_desc->control);
+
+ WARN((pi_desc->sn == 1),
+ "Warning: SN field of posted-interrupts "
+@@ -211,7 +216,7 @@ void pi_wakeup_handler(void)
+ struct kvm_vcpu *vcpu;
+ int cpu = smp_processor_id();
+
+- spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
++ raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
+ list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
+ blocked_vcpu_list) {
+ struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
+@@ -219,13 +224,13 @@ void pi_wakeup_handler(void)
+ if (pi_test_on(pi_desc) == 1)
+ kvm_vcpu_kick(vcpu);
+ }
+- spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
++ raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
+ }
+
+ void __init pi_init_cpu(int cpu)
+ {
+ INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
+- spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
++ raw_spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
+ }
+
+ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu)
+@@ -270,9 +275,7 @@ int pi_update_irte(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq,
+ struct vcpu_data vcpu_info;
+ int idx, ret = 0;
+
+- if (!kvm_arch_has_assigned_device(kvm) ||
+- !irq_remapping_cap(IRQ_POSTING_CAP) ||
+- !kvm_vcpu_apicv_active(kvm->vcpus[0]))
++ if (!vmx_can_use_vtd_pi(kvm))
+ return 0;
+
+ idx = srcu_read_lock(&kvm->irq_srcu);
+diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S
+index 3a6461694fc25..435c187927c48 100644
+--- a/arch/x86/kvm/vmx/vmenter.S
++++ b/arch/x86/kvm/vmx/vmenter.S
+@@ -49,14 +49,14 @@ SYM_FUNC_START_LOCAL(vmx_vmenter)
+ je 2f
+
+ 1: vmresume
+- ret
++ RET
+
+ 2: vmlaunch
+- ret
++ RET
+
+ 3: cmpb $0, kvm_rebooting
+ je 4f
+- ret
++ RET
+ 4: ud2
+
+ _ASM_EXTABLE(1b, 3b)
+@@ -89,7 +89,7 @@ SYM_FUNC_START(vmx_vmexit)
+ pop %_ASM_AX
+ .Lvmexit_skip_rsb:
+ #endif
+- ret
++ RET
+ SYM_FUNC_END(vmx_vmexit)
+
+ /**
+@@ -228,7 +228,7 @@ SYM_FUNC_START(__vmx_vcpu_run)
+ pop %edi
+ #endif
+ pop %_ASM_BP
+- ret
++ RET
+
+ /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
+ 2: mov $1, %eax
+@@ -293,7 +293,7 @@ SYM_FUNC_START(vmread_error_trampoline)
+ pop %_ASM_AX
+ pop %_ASM_BP
+
+- ret
++ RET
+ SYM_FUNC_END(vmread_error_trampoline)
+
+ SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
+@@ -326,5 +326,5 @@ SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
+ */
+ mov %_ASM_BP, %_ASM_SP
+ pop %_ASM_BP
+- ret
++ RET
+ SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
+diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
+index 7d595effb66f0..16a660a0ed5f7 100644
+--- a/arch/x86/kvm/vmx/vmx.c
++++ b/arch/x86/kvm/vmx/vmx.c
+@@ -769,24 +769,13 @@ void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu)
+ /*
+ * Check if MSR is intercepted for currently loaded MSR bitmap.
+ */
+-static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
++static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr)
+ {
+- unsigned long *msr_bitmap;
+- int f = sizeof(unsigned long);
+-
+- if (!cpu_has_vmx_msr_bitmap())
++ if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS))
+ return true;
+
+- msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
+-
+- if (msr <= 0x1fff) {
+- return !!test_bit(msr, msr_bitmap + 0x800 / f);
+- } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+- msr &= 0x1fff;
+- return !!test_bit(msr, msr_bitmap + 0xc00 / f);
+- }
+-
+- return true;
++ return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap,
++ MSR_IA32_SPEC_CTRL);
+ }
+
+ static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
+@@ -1370,6 +1359,11 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
+ vmx->emulation_required = vmx_emulation_required(vcpu);
+ }
+
++static bool vmx_get_if_flag(struct kvm_vcpu *vcpu)
++{
++ return vmx_get_rflags(vcpu) & X86_EFLAGS_IF;
++}
++
+ u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
+ {
+ u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
+@@ -2927,6 +2921,13 @@ static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
+ }
+ }
+
++static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu)
++{
++ if (is_guest_mode(vcpu))
++ return nested_get_vpid02(vcpu);
++ return to_vmx(vcpu)->vpid;
++}
++
+ static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_mmu *mmu = vcpu->arch.mmu;
+@@ -2939,31 +2940,29 @@ static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
+ if (enable_ept)
+ ept_sync_context(construct_eptp(vcpu, root_hpa,
+ mmu->shadow_root_level));
+- else if (!is_guest_mode(vcpu))
+- vpid_sync_context(to_vmx(vcpu)->vpid);
+ else
+- vpid_sync_context(nested_get_vpid02(vcpu));
++ vpid_sync_context(vmx_get_current_vpid(vcpu));
+ }
+
+ static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
+ {
+ /*
+- * vpid_sync_vcpu_addr() is a nop if vmx->vpid==0, see the comment in
++ * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
+ * vmx_flush_tlb_guest() for an explanation of why this is ok.
+ */
+- vpid_sync_vcpu_addr(to_vmx(vcpu)->vpid, addr);
++ vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr);
+ }
+
+ static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
+ {
+ /*
+- * vpid_sync_context() is a nop if vmx->vpid==0, e.g. if enable_vpid==0
+- * or a vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit
+- * are required to flush GVA->{G,H}PA mappings from the TLB if vpid is
++ * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a
++ * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are
++ * required to flush GVA->{G,H}PA mappings from the TLB if vpid is
+ * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed),
+ * i.e. no explicit INVVPID is necessary.
+ */
+- vpid_sync_context(to_vmx(vcpu)->vpid);
++ vpid_sync_context(vmx_get_current_vpid(vcpu));
+ }
+
+ void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu)
+@@ -3695,46 +3694,6 @@ void free_vpid(int vpid)
+ spin_unlock(&vmx_vpid_lock);
+ }
+
+-static void vmx_clear_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
+-{
+- int f = sizeof(unsigned long);
+-
+- if (msr <= 0x1fff)
+- __clear_bit(msr, msr_bitmap + 0x000 / f);
+- else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
+- __clear_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
+-}
+-
+-static void vmx_clear_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
+-{
+- int f = sizeof(unsigned long);
+-
+- if (msr <= 0x1fff)
+- __clear_bit(msr, msr_bitmap + 0x800 / f);
+- else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
+- __clear_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
+-}
+-
+-static void vmx_set_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
+-{
+- int f = sizeof(unsigned long);
+-
+- if (msr <= 0x1fff)
+- __set_bit(msr, msr_bitmap + 0x000 / f);
+- else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
+- __set_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
+-}
+-
+-static void vmx_set_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
+-{
+- int f = sizeof(unsigned long);
+-
+- if (msr <= 0x1fff)
+- __set_bit(msr, msr_bitmap + 0x800 / f);
+- else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
+- __set_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
+-}
+-
+ void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type)
+ {
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+@@ -4012,8 +3971,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
+ if (pi_test_and_set_on(&vmx->pi_desc))
+ return 0;
+
+- if (vcpu != kvm_get_running_vcpu() &&
+- !kvm_vcpu_trigger_posted_interrupt(vcpu, false))
++ if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
+ kvm_vcpu_kick(vcpu);
+
+ return 0;
+@@ -4140,6 +4098,11 @@ static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
+ {
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+
++ if (is_guest_mode(vcpu)) {
++ vmx->nested.update_vmcs01_apicv_status = true;
++ return;
++ }
++
+ pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
+ if (cpu_has_secondary_exec_ctrls()) {
+ if (kvm_vcpu_apicv_active(vcpu))
+@@ -4833,8 +4796,33 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
+ dr6 = vmx_get_exit_qual(vcpu);
+ if (!(vcpu->guest_debug &
+ (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
++ /*
++ * If the #DB was due to ICEBP, a.k.a. INT1, skip the
++ * instruction. ICEBP generates a trap-like #DB, but
++ * despite its interception control being tied to #DB,
++ * is an instruction intercept, i.e. the VM-Exit occurs
++ * on the ICEBP itself. Note, skipping ICEBP also
++ * clears STI and MOVSS blocking.
++ *
++ * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS
++ * if single-step is enabled in RFLAGS and STI or MOVSS
++ * blocking is active, as the CPU doesn't set the bit
++ * on VM-Exit due to #DB interception. VM-Entry has a
++ * consistency check that a single-step #DB is pending
++ * in this scenario as the previous instruction cannot
++ * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV
++ * don't modify RFLAGS), therefore the one instruction
++ * delay when activating single-step breakpoints must
++ * have already expired. Note, the CPU sets/clears BS
++ * as appropriate for all other VM-Exits types.
++ */
+ if (is_icebp(intr_info))
+ WARN_ON(!skip_emulated_instruction(vcpu));
++ else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
++ (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
++ (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)))
++ vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
++ vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS);
+
+ kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
+ return 1;
+@@ -5907,18 +5895,14 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
+ vmx_flush_pml_buffer(vcpu);
+
+ /*
+- * We should never reach this point with a pending nested VM-Enter, and
+- * more specifically emulation of L2 due to invalid guest state (see
+- * below) should never happen as that means we incorrectly allowed a
+- * nested VM-Enter with an invalid vmcs12.
++ * KVM should never reach this point with a pending nested VM-Enter.
++ * More specifically, short-circuiting VM-Entry to emulate L2 due to
++ * invalid guest state should never happen as that means KVM knowingly
++ * allowed a nested VM-Enter with an invalid vmcs12. More below.
+ */
+ if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm))
+ return -EIO;
+
+- /* If guest state is invalid, start emulating */
+- if (vmx->emulation_required)
+- return handle_invalid_guest_state(vcpu);
+-
+ if (is_guest_mode(vcpu)) {
+ /*
+ * PML is never enabled when running L2, bail immediately if a
+@@ -5940,10 +5924,30 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
+ */
+ nested_mark_vmcs12_pages_dirty(vcpu);
+
++ /*
++ * Synthesize a triple fault if L2 state is invalid. In normal
++ * operation, nested VM-Enter rejects any attempt to enter L2
++ * with invalid state. However, those checks are skipped if
++ * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If
++ * L2 state is invalid, it means either L1 modified SMRAM state
++ * or userspace provided bad state. Synthesize TRIPLE_FAULT as
++ * doing so is architecturally allowed in the RSM case, and is
++ * the least awful solution for the userspace case without
++ * risking false positives.
++ */
++ if (vmx->emulation_required) {
++ nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0);
++ return 1;
++ }
++
+ if (nested_vmx_reflect_vmexit(vcpu))
+ return 1;
+ }
+
++ /* If guest state is invalid, start emulating. L2 is handled above. */
++ if (vmx->emulation_required)
++ return handle_invalid_guest_state(vcpu);
++
+ if (exit_reason.failed_vmentry) {
+ dump_vmcs(vcpu);
+ vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
+@@ -6288,9 +6292,9 @@ static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
+ {
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ int max_irr;
+- bool max_irr_updated;
++ bool got_posted_interrupt;
+
+- if (KVM_BUG_ON(!vcpu->arch.apicv_active, vcpu->kvm))
++ if (KVM_BUG_ON(!enable_apicv, vcpu->kvm))
+ return -EIO;
+
+ if (pi_test_on(&vmx->pi_desc)) {
+@@ -6300,22 +6304,33 @@ static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
+ * But on x86 this is just a compiler barrier anyway.
+ */
+ smp_mb__after_atomic();
+- max_irr_updated =
++ got_posted_interrupt =
+ kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
+-
+- /*
+- * If we are running L2 and L1 has a new pending interrupt
+- * which can be injected, this may cause a vmexit or it may
+- * be injected into L2. Either way, this interrupt will be
+- * processed via KVM_REQ_EVENT, not RVI, because we do not use
+- * virtual interrupt delivery to inject L1 interrupts into L2.
+- */
+- if (is_guest_mode(vcpu) && max_irr_updated)
+- kvm_make_request(KVM_REQ_EVENT, vcpu);
+ } else {
+ max_irr = kvm_lapic_find_highest_irr(vcpu);
++ got_posted_interrupt = false;
+ }
+- vmx_hwapic_irr_update(vcpu, max_irr);
++
++ /*
++ * Newly recognized interrupts are injected via either virtual interrupt
++ * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is
++ * disabled in two cases:
++ *
++ * 1) If L2 is running and the vCPU has a new pending interrupt. If L1
++ * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a
++ * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected
++ * into L2, but KVM doesn't use virtual interrupt delivery to inject
++ * interrupts into L2, and so KVM_REQ_EVENT is again needed.
++ *
++ * 2) If APICv is disabled for this vCPU, assigned devices may still
++ * attempt to post interrupts. The posted interrupt vector will cause
++ * a VM-Exit and the subsequent entry will call sync_pir_to_irr.
++ */
++ if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu))
++ vmx_set_rvi(max_irr);
++ else if (got_posted_interrupt)
++ kvm_make_request(KVM_REQ_EVENT, vcpu);
++
+ return max_irr;
+ }
+
+@@ -6626,9 +6641,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
+ * consistency check VM-Exit due to invalid guest state and bail.
+ */
+ if (unlikely(vmx->emulation_required)) {
+-
+- /* We don't emulate invalid state of a nested guest */
+- vmx->fail = is_guest_mode(vcpu);
++ vmx->fail = 0;
+
+ vmx->exit_reason.full = EXIT_REASON_INVALID_STATE;
+ vmx->exit_reason.failed_vmentry = 1;
+@@ -6720,7 +6733,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
+ * If the L02 MSR bitmap does not intercept the MSR, then we need to
+ * save it.
+ */
+- if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
++ if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)))
+ vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
+
+ x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
+@@ -7524,6 +7537,7 @@ static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
+ if (ret)
+ return ret;
+
++ vmx->nested.nested_run_pending = 1;
+ vmx->nested.smm.guest_mode = false;
+ }
+ return 0;
+@@ -7551,6 +7565,8 @@ static void vmx_migrate_timers(struct kvm_vcpu *vcpu)
+
+ static void hardware_unsetup(void)
+ {
++ kvm_set_posted_intr_wakeup_handler(NULL);
++
+ if (nested)
+ nested_vmx_hardware_unsetup();
+
+@@ -7606,6 +7622,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
+ .cache_reg = vmx_cache_reg,
+ .get_rflags = vmx_get_rflags,
+ .set_rflags = vmx_set_rflags,
++ .get_if_flag = vmx_get_if_flag,
+
+ .tlb_flush_all = vmx_flush_tlb_all,
+ .tlb_flush_current = vmx_flush_tlb_current,
+@@ -7809,10 +7826,10 @@ static __init int hardware_setup(void)
+ ple_window_shrink = 0;
+ }
+
+- if (!cpu_has_vmx_apicv()) {
++ if (!cpu_has_vmx_apicv())
+ enable_apicv = 0;
++ if (!enable_apicv)
+ vmx_x86_ops.sync_pir_to_irr = NULL;
+- }
+
+ if (cpu_has_vmx_tsc_scaling()) {
+ kvm_has_tsc_control = true;
+@@ -7879,8 +7896,6 @@ static __init int hardware_setup(void)
+ vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
+ }
+
+- kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
+-
+ kvm_mce_cap_supported |= MCG_LMCE_P;
+
+ if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST)
+@@ -7904,6 +7919,9 @@ static __init int hardware_setup(void)
+ r = alloc_kvm_area();
+ if (r)
+ nested_vmx_hardware_unsetup();
++
++ kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
++
+ return r;
+ }
+
+@@ -7912,6 +7930,7 @@ static struct kvm_x86_init_ops vmx_init_ops __initdata = {
+ .disabled_by_bios = vmx_disabled_by_bios,
+ .check_processor_compatibility = vmx_check_processor_compat,
+ .hardware_setup = hardware_setup,
++ .intel_pt_intr_in_guest = vmx_pt_mode_is_host_guest,
+
+ .runtime_ops = &vmx_x86_ops,
+ };
+diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
+index 592217fd7d920..460c7bd8158cc 100644
+--- a/arch/x86/kvm/vmx/vmx.h
++++ b/arch/x86/kvm/vmx/vmx.h
+@@ -164,6 +164,7 @@ struct nested_vmx {
+ bool change_vmcs01_virtual_apic_mode;
+ bool reload_vmcs01_apic_access_page;
+ bool update_vmcs01_cpu_dirty_logging;
++ bool update_vmcs01_apicv_status;
+
+ /*
+ * Enlightened VMCS has been enabled. It does not mean that L1 has to
+@@ -400,6 +401,69 @@ static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr,
+
+ void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
+
++static inline bool vmx_test_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ return test_bit(msr, msr_bitmap + 0x000 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ return test_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
++ return true;
++}
++
++static inline bool vmx_test_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ return test_bit(msr, msr_bitmap + 0x800 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ return test_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
++ return true;
++}
++
++static inline void vmx_clear_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ __clear_bit(msr, msr_bitmap + 0x000 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ __clear_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
++}
++
++static inline void vmx_clear_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ __clear_bit(msr, msr_bitmap + 0x800 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ __clear_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
++}
++
++static inline void vmx_set_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ __set_bit(msr, msr_bitmap + 0x000 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ __set_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
++}
++
++static inline void vmx_set_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
++{
++ int f = sizeof(unsigned long);
++
++ if (msr <= 0x1fff)
++ __set_bit(msr, msr_bitmap + 0x800 / f);
++ else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
++ __set_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
++}
++
++
+ static inline u8 vmx_get_rvi(void)
+ {
+ return vmcs_read16(GUEST_INTR_STATUS) & 0xff;
+diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
+index bfe0de3008a60..75da9c0d5ae37 100644
+--- a/arch/x86/kvm/x86.c
++++ b/arch/x86/kvm/x86.c
+@@ -848,6 +848,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
+
+ memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
+ kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
++ kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
+ vcpu->arch.pdptrs_from_userspace = false;
+
+ out:
+@@ -1091,6 +1092,18 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
+ unsigned long roots_to_free = 0;
+ int i;
+
++ /*
++ * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
++ * this is reachable when running EPT=1 and unrestricted_guest=0, and
++ * also via the emulator. KVM's TDP page tables are not in the scope of
++ * the invalidation, but the guest's TLB entries need to be flushed as
++ * the CPU may have cached entries in its TLB for the target PCID.
++ */
++ if (unlikely(tdp_enabled)) {
++ kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
++ return;
++ }
++
+ /*
+ * If neither the current CR3 nor any of the prev_roots use the given
+ * PCID, then nothing needs to be done here because a resync will
+@@ -1311,7 +1324,7 @@ static const u32 msrs_to_save_all[] = {
+ MSR_IA32_UMWAIT_CONTROL,
+
+ MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
+- MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
++ MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
+ MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
+ MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
+ MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
+@@ -1592,8 +1605,7 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+ return r;
+ }
+
+- /* Update reserved bits */
+- if ((efer ^ old_efer) & EFER_NX)
++ if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
+ kvm_mmu_reset_context(vcpu);
+
+ return 0;
+@@ -3193,10 +3205,36 @@ static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
+ static_call(kvm_x86_tlb_flush_guest)(vcpu);
+ }
+
++
++static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
++{
++ ++vcpu->stat.tlb_flush;
++ static_call(kvm_x86_tlb_flush_current)(vcpu);
++}
++
++/*
++ * Service "local" TLB flush requests, which are specific to the current MMU
++ * context. In addition to the generic event handling in vcpu_enter_guest(),
++ * TLB flushes that are targeted at an MMU context also need to be serviced
++ * prior before nested VM-Enter/VM-Exit.
++ */
++void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
++{
++ if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
++ kvm_vcpu_flush_tlb_current(vcpu);
++
++ if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
++ kvm_vcpu_flush_tlb_guest(vcpu);
++}
++EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests);
++
+ static void record_steal_time(struct kvm_vcpu *vcpu)
+ {
+- struct kvm_host_map map;
+- struct kvm_steal_time *st;
++ struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
++ struct kvm_steal_time __user *st;
++ struct kvm_memslots *slots;
++ u64 steal;
++ u32 version;
+
+ if (kvm_xen_msr_enabled(vcpu->kvm)) {
+ kvm_xen_runstate_set_running(vcpu);
+@@ -3206,47 +3244,86 @@ static void record_steal_time(struct kvm_vcpu *vcpu)
+ if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
+ return;
+
+- /* -EAGAIN is returned in atomic context so we can just return. */
+- if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT,
+- &map, &vcpu->arch.st.cache, false))
++ if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
+ return;
+
+- st = map.hva +
+- offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
++ slots = kvm_memslots(vcpu->kvm);
++
++ if (unlikely(slots->generation != ghc->generation ||
++ kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
++ gfn_t gfn = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
++
++ /* We rely on the fact that it fits in a single page. */
++ BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
++
++ if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gfn, sizeof(*st)) ||
++ kvm_is_error_hva(ghc->hva) || !ghc->memslot)
++ return;
++ }
+
++ st = (struct kvm_steal_time __user *)ghc->hva;
+ /*
+ * Doing a TLB flush here, on the guest's behalf, can avoid
+ * expensive IPIs.
+ */
+ if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
+- u8 st_preempted = xchg(&st->preempted, 0);
++ u8 st_preempted = 0;
++ int err = -EFAULT;
++
++ if (!user_access_begin(st, sizeof(*st)))
++ return;
++
++ asm volatile("1: xchgb %0, %2\n"
++ "xor %1, %1\n"
++ "2:\n"
++ _ASM_EXTABLE_UA(1b, 2b)
++ : "+q" (st_preempted),
++ "+&r" (err),
++ "+m" (st->preempted));
++ if (err)
++ goto out;
++
++ user_access_end();
++
++ vcpu->arch.st.preempted = 0;
+
+ trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
+ st_preempted & KVM_VCPU_FLUSH_TLB);
+ if (st_preempted & KVM_VCPU_FLUSH_TLB)
+ kvm_vcpu_flush_tlb_guest(vcpu);
++
++ if (!user_access_begin(st, sizeof(*st)))
++ goto dirty;
+ } else {
+- st->preempted = 0;
+- }
++ if (!user_access_begin(st, sizeof(*st)))
++ return;
+
+- vcpu->arch.st.preempted = 0;
++ unsafe_put_user(0, &st->preempted, out);
++ vcpu->arch.st.preempted = 0;
++ }
+
+- if (st->version & 1)
+- st->version += 1; /* first time write, random junk */
++ unsafe_get_user(version, &st->version, out);
++ if (version & 1)
++ version += 1; /* first time write, random junk */
+
+- st->version += 1;
++ version += 1;
++ unsafe_put_user(version, &st->version, out);
+
+ smp_wmb();
+
+- st->steal += current->sched_info.run_delay -
++ unsafe_get_user(steal, &st->steal, out);
++ steal += current->sched_info.run_delay -
+ vcpu->arch.st.last_steal;
+ vcpu->arch.st.last_steal = current->sched_info.run_delay;
++ unsafe_put_user(steal, &st->steal, out);
+
+- smp_wmb();
+-
+- st->version += 1;
++ version += 1;
++ unsafe_put_user(version, &st->version, out);
+
+- kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);
++ out:
++ user_access_end();
++ dirty:
++ mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
+ }
+
+ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+@@ -3282,7 +3359,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+
+ if (!msr_info->host_initiated)
+ return 1;
+- if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM) && kvm_get_msr_feature(&msr_ent))
++ if (kvm_get_msr_feature(&msr_ent))
+ return 1;
+ if (data & ~msr_ent.data)
+ return 1;
+@@ -3376,6 +3453,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+ if (data & ~supported_xss)
+ return 1;
+ vcpu->arch.ia32_xss = data;
++ kvm_update_cpuid_runtime(vcpu);
+ break;
+ case MSR_SMI_COUNT:
+ if (!msr_info->host_initiated)
+@@ -4285,8 +4363,10 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+
+ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
+ {
+- struct kvm_host_map map;
+- struct kvm_steal_time *st;
++ struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
++ struct kvm_steal_time __user *st;
++ struct kvm_memslots *slots;
++ static const u8 preempted = KVM_VCPU_PREEMPTED;
+
+ if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
+ return;
+@@ -4294,16 +4374,23 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
+ if (vcpu->arch.st.preempted)
+ return;
+
+- if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
+- &vcpu->arch.st.cache, true))
++ /* This happens on process exit */
++ if (unlikely(current->mm != vcpu->kvm->mm))
+ return;
+
+- st = map.hva +
+- offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
++ slots = kvm_memslots(vcpu->kvm);
++
++ if (unlikely(slots->generation != ghc->generation ||
++ kvm_is_error_hva(ghc->hva) || !ghc->memslot))
++ return;
+
+- st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
++ st = (struct kvm_steal_time __user *)ghc->hva;
++ BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));
+
+- kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
++ if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))
++ vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
++
++ mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
+ }
+
+ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
+@@ -4331,8 +4418,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
+ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
+ struct kvm_lapic_state *s)
+ {
+- if (vcpu->arch.apicv_active)
+- static_call(kvm_x86_sync_pir_to_irr)(vcpu);
++ static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
+
+ return kvm_apic_get_state(vcpu, s);
+ }
+@@ -4642,8 +4728,10 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
+ vcpu->arch.apic->sipi_vector = events->sipi_vector;
+
+ if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
+- if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm)
++ if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
++ kvm_x86_ops.nested_ops->leave_nested(vcpu);
+ kvm_smm_changed(vcpu, events->smi.smm);
++ }
+
+ vcpu->arch.smi_pending = events->smi.pending;
+
+@@ -6948,7 +7036,13 @@ static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
+ unsigned short port, void *val, unsigned int count)
+ {
+ if (vcpu->arch.pio.count) {
+- /* Complete previous iteration. */
++ /*
++ * Complete a previous iteration that required userspace I/O.
++ * Note, @count isn't guaranteed to match pio.count as userspace
++ * can modify ECX before rerunning the vCPU. Ignore any such
++ * shenanigans as KVM doesn't support modifying the rep count,
++ * and the emulator ensures @count doesn't overflow the buffer.
++ */
+ } else {
+ int r = __emulator_pio_in(vcpu, size, port, count);
+ if (!r)
+@@ -6957,7 +7051,6 @@ static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
+ /* Results already available, fall through. */
+ }
+
+- WARN_ON(count != vcpu->arch.pio.count);
+ complete_emulator_pio_in(vcpu, val);
+ return 1;
+ }
+@@ -7300,6 +7393,11 @@ static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
+ return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
+ }
+
++static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)
++{
++ return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID);
++}
++
+ static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
+ {
+ return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);
+@@ -7382,6 +7480,7 @@ static const struct x86_emulate_ops emulate_ops = {
+ .guest_has_long_mode = emulator_guest_has_long_mode,
+ .guest_has_movbe = emulator_guest_has_movbe,
+ .guest_has_fxsr = emulator_guest_has_fxsr,
++ .guest_has_rdpid = emulator_guest_has_rdpid,
+ .set_nmi_mask = emulator_set_nmi_mask,
+ .get_hflags = emulator_get_hflags,
+ .exiting_smm = emulator_exiting_smm,
+@@ -7905,7 +8004,12 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
+ * updating interruptibility state and injecting single-step #DBs.
+ */
+ if (emulation_type & EMULTYPE_SKIP) {
+- kvm_rip_write(vcpu, ctxt->_eip);
++ if (ctxt->mode != X86EMUL_MODE_PROT64)
++ ctxt->eip = (u32)ctxt->_eip;
++ else
++ ctxt->eip = ctxt->_eip;
++
++ kvm_rip_write(vcpu, ctxt->eip);
+ if (ctxt->eflags & X86_EFLAGS_RF)
+ kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
+ return 1;
+@@ -7969,6 +8073,9 @@ restart:
+ writeback = false;
+ r = 0;
+ vcpu->arch.complete_userspace_io = complete_emulated_mmio;
++ } else if (vcpu->arch.complete_userspace_io) {
++ writeback = false;
++ r = 0;
+ } else if (r == EMULATION_RESTART)
+ goto restart;
+ else
+@@ -8340,7 +8447,7 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
+ .is_in_guest = kvm_is_in_guest,
+ .is_user_mode = kvm_is_user_mode,
+ .get_guest_ip = kvm_get_guest_ip,
+- .handle_intel_pt_intr = kvm_handle_intel_pt_intr,
++ .handle_intel_pt_intr = NULL,
+ };
+
+ #ifdef CONFIG_X86_64
+@@ -8455,14 +8562,12 @@ int kvm_arch_init(void *opaque)
+ }
+ kvm_nr_uret_msrs = 0;
+
+- r = kvm_mmu_module_init();
++ r = kvm_mmu_vendor_module_init();
+ if (r)
+ goto out_free_percpu;
+
+ kvm_timer_init();
+
+- perf_register_guest_info_callbacks(&kvm_guest_cbs);
+-
+ if (boot_cpu_has(X86_FEATURE_XSAVE)) {
+ host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+ supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
+@@ -8496,7 +8601,6 @@ void kvm_arch_exit(void)
+ clear_hv_tscchange_cb();
+ #endif
+ kvm_lapic_exit();
+- perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
+
+ if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+ cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
+@@ -8508,7 +8612,7 @@ void kvm_arch_exit(void)
+ cancel_work_sync(&pvclock_gtod_work);
+ #endif
+ kvm_x86_ops.hardware_enable = NULL;
+- kvm_mmu_module_exit();
++ kvm_mmu_vendor_module_exit();
+ free_percpu(user_return_msrs);
+ kmem_cache_destroy(x86_emulator_cache);
+ kmem_cache_destroy(x86_fpu_cache);
+@@ -8567,6 +8671,13 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
+ if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
+ return -KVM_EOPNOTSUPP;
+
++ /*
++ * When tsc is in permanent catchup mode guests won't be able to use
++ * pvclock_read_retry loop to get consistent view of pvclock
++ */
++ if (vcpu->arch.tsc_always_catchup)
++ return -KVM_EOPNOTSUPP;
++
+ if (!kvm_get_walltime_and_clockread(&ts, &cycle))
+ return -KVM_EOPNOTSUPP;
+
+@@ -8686,7 +8797,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
+
+ trace_kvm_hypercall(nr, a0, a1, a2, a3);
+
+- op_64_bit = is_64_bit_mode(vcpu);
++ op_64_bit = is_64_bit_hypercall(vcpu);
+ if (!op_64_bit) {
+ nr &= 0xFFFFFFFF;
+ a0 &= 0xFFFFFFFF;
+@@ -8790,14 +8901,7 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_run *kvm_run = vcpu->run;
+
+- /*
+- * if_flag is obsolete and useless, so do not bother
+- * setting it for SEV-ES guests. Userspace can just
+- * use kvm_run->ready_for_interrupt_injection.
+- */
+- kvm_run->if_flag = !vcpu->arch.guest_state_protected
+- && (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
+-
++ kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
+ kvm_run->cr8 = kvm_get_cr8(vcpu);
+ kvm_run->apic_base = kvm_get_apic_base(vcpu);
+
+@@ -9359,8 +9463,7 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
+ if (irqchip_split(vcpu->kvm))
+ kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
+ else {
+- if (vcpu->arch.apicv_active)
+- static_call(kvm_x86_sync_pir_to_irr)(vcpu);
++ static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
+ if (ioapic_in_kernel(vcpu->kvm))
+ kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
+ }
+@@ -9378,12 +9481,16 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
+ if (!kvm_apic_hw_enabled(vcpu->arch.apic))
+ return;
+
+- if (to_hv_vcpu(vcpu))
++ if (to_hv_vcpu(vcpu)) {
+ bitmap_or((ulong *)eoi_exit_bitmap,
+ vcpu->arch.ioapic_handled_vectors,
+ to_hv_synic(vcpu)->vec_bitmap, 256);
++ static_call(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
++ return;
++ }
+
+- static_call(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
++ static_call(kvm_x86_load_eoi_exitmap)(
++ vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
+ }
+
+ void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
+@@ -9475,10 +9582,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
+ /* Flushing all ASIDs flushes the current ASID... */
+ kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+ }
+- if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
+- kvm_vcpu_flush_tlb_current(vcpu);
+- if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
+- kvm_vcpu_flush_tlb_guest(vcpu);
++ kvm_service_local_tlb_flush_requests(vcpu);
+
+ if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
+ vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
+@@ -9629,10 +9733,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
+
+ /*
+ * This handles the case where a posted interrupt was
+- * notified with kvm_vcpu_kick.
++ * notified with kvm_vcpu_kick. Assigned devices can
++ * use the POSTED_INTR_VECTOR even if APICv is disabled,
++ * so do it even if APICv is disabled on this vCPU.
+ */
+- if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
+- static_call(kvm_x86_sync_pir_to_irr)(vcpu);
++ if (kvm_lapic_enabled(vcpu))
++ static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
+
+ if (kvm_vcpu_exit_request(vcpu)) {
+ vcpu->mode = OUTSIDE_GUEST_MODE;
+@@ -9668,8 +9774,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
+ if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
+ break;
+
+- if (vcpu->arch.apicv_active)
+- static_call(kvm_x86_sync_pir_to_irr)(vcpu);
++ if (kvm_lapic_enabled(vcpu))
++ static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
+
+ if (unlikely(kvm_vcpu_exit_request(vcpu))) {
+ exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
+@@ -10707,8 +10813,21 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
+ r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
+ if (r < 0)
+ goto fail_mmu_destroy;
+- if (kvm_apicv_activated(vcpu->kvm))
++
++ /*
++ * Defer evaluating inhibits until the vCPU is first run, as
++ * this vCPU will not get notified of any changes until this
++ * vCPU is visible to other vCPUs (marked online and added to
++ * the set of vCPUs). Opportunistically mark APICv active as
++ * VMX in particularly is highly unlikely to have inhibits.
++ * Ignore the current per-VM APICv state so that vCPU creation
++ * is guaranteed to run with a deterministic value, the request
++ * will ensure the vCPU gets the correct state before VM-Entry.
++ */
++ if (enable_apicv) {
+ vcpu->arch.apicv_active = true;
++ kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
++ }
+ } else
+ static_branch_inc(&kvm_has_noapic_vcpu);
+
+@@ -10817,11 +10936,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
+
+ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
+ {
+- struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;
+ int idx;
+
+- kvm_release_pfn(cache->pfn, cache->dirty, cache);
+-
+ kvmclock_reset(vcpu);
+
+ static_call(kvm_x86_vcpu_free)(vcpu);
+@@ -10908,7 +11024,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
+
+ vcpu->arch.msr_misc_features_enables = 0;
+
+- vcpu->arch.xcr0 = XFEATURE_MASK_FP;
++ __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
++ __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
+ }
+
+ memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
+@@ -10927,8 +11044,6 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
+ eax = 0x600;
+ kvm_rdx_write(vcpu, eax);
+
+- vcpu->arch.ia32_xss = 0;
+-
+ static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
+
+ kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
+@@ -11104,6 +11219,10 @@ int kvm_arch_hardware_setup(void *opaque)
+ memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
+ kvm_ops_static_call_update();
+
++ if (ops->intel_pt_intr_in_guest && ops->intel_pt_intr_in_guest())
++ kvm_guest_cbs.handle_intel_pt_intr = kvm_handle_intel_pt_intr;
++ perf_register_guest_info_callbacks(&kvm_guest_cbs);
++
+ if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
+ supported_xss = 0;
+
+@@ -11131,6 +11250,9 @@ int kvm_arch_hardware_setup(void *opaque)
+
+ void kvm_arch_hardware_unsetup(void)
+ {
++ perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
++ kvm_guest_cbs.handle_intel_pt_intr = NULL;
++
+ static_call(kvm_x86_hardware_unsetup)();
+ }
+
+@@ -12509,3 +12631,19 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit);
++
++static int __init kvm_x86_init(void)
++{
++ kvm_mmu_x86_module_init();
++ return 0;
++}
++module_init(kvm_x86_init);
++
++static void __exit kvm_x86_exit(void)
++{
++ /*
++ * If module_init() is implemented, module_exit() must also be
++ * implemented to allow module unload.
++ */
++}
++module_exit(kvm_x86_exit);
+diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
+index 7d66d63dc55a6..bc7ef6820ac8f 100644
+--- a/arch/x86/kvm/x86.h
++++ b/arch/x86/kvm/x86.h
+@@ -103,6 +103,7 @@ static inline unsigned int __shrink_ple_window(unsigned int val,
+
+ #define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL
+
++void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu);
+ int kvm_check_nested_events(struct kvm_vcpu *vcpu);
+
+ static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu)
+@@ -153,12 +154,24 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
+ {
+ int cs_db, cs_l;
+
++ WARN_ON_ONCE(vcpu->arch.guest_state_protected);
++
+ if (!is_long_mode(vcpu))
+ return false;
+ static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
+ return cs_l;
+ }
+
++static inline bool is_64_bit_hypercall(struct kvm_vcpu *vcpu)
++{
++ /*
++ * If running with protected guest state, the CS register is not
++ * accessible. The hypercall register values will have had to been
++ * provided in 64-bit mode, so assume the guest is in 64-bit.
++ */
++ return vcpu->arch.guest_state_protected || is_64_bit_mode(vcpu);
++}
++
+ static inline bool x86_exception_has_error_code(unsigned int vector)
+ {
+ static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) |
+@@ -173,12 +186,6 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
+ return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu;
+ }
+
+-static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
+-{
+- ++vcpu->stat.tlb_flush;
+- static_call(kvm_x86_tlb_flush_current)(vcpu);
+-}
+-
+ static inline int is_pae(struct kvm_vcpu *vcpu)
+ {
+ return kvm_read_cr4_bits(vcpu, X86_CR4_PAE);
+diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
+index 8f62baebd0286..ab9f88de6deb9 100644
+--- a/arch/x86/kvm/xen.c
++++ b/arch/x86/kvm/xen.c
+@@ -93,32 +93,57 @@ static void kvm_xen_update_runstate(struct kvm_vcpu *v, int state)
+ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state)
+ {
+ struct kvm_vcpu_xen *vx = &v->arch.xen;
++ struct gfn_to_hva_cache *ghc = &vx->runstate_cache;
++ struct kvm_memslots *slots = kvm_memslots(v->kvm);
++ bool atomic = (state == RUNSTATE_runnable);
+ uint64_t state_entry_time;
+- unsigned int offset;
++ int __user *user_state;
++ uint64_t __user *user_times;
+
+ kvm_xen_update_runstate(v, state);
+
+ if (!vx->runstate_set)
+ return;
+
+- BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c);
++ if (unlikely(slots->generation != ghc->generation || kvm_is_error_hva(ghc->hva)) &&
++ kvm_gfn_to_hva_cache_init(v->kvm, ghc, ghc->gpa, ghc->len))
++ return;
++
++ /* We made sure it fits in a single page */
++ BUG_ON(!ghc->memslot);
++
++ if (atomic)
++ pagefault_disable();
+
+- offset = offsetof(struct compat_vcpu_runstate_info, state_entry_time);
+-#ifdef CONFIG_X86_64
+ /*
+- * The only difference is alignment of uint64_t in 32-bit.
+- * So the first field 'state' is accessed directly using
+- * offsetof() (where its offset happens to be zero), while the
+- * remaining fields which are all uint64_t, start at 'offset'
+- * which we tweak here by adding 4.
++ * The only difference between 32-bit and 64-bit versions of the
++ * runstate struct us the alignment of uint64_t in 32-bit, which
++ * means that the 64-bit version has an additional 4 bytes of
++ * padding after the first field 'state'.
++ *
++ * So we use 'int __user *user_state' to point to the state field,
++ * and 'uint64_t __user *user_times' for runstate_entry_time. So
++ * the actual array of time[] in each state starts at user_times[1].
+ */
++ BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != 0);
++ BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state) != 0);
++ user_state = (int __user *)ghc->hva;
++
++ BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c);
++
++ user_times = (uint64_t __user *)(ghc->hva +
++ offsetof(struct compat_vcpu_runstate_info,
++ state_entry_time));
++#ifdef CONFIG_X86_64
+ BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
+ offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4);
+ BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) !=
+ offsetof(struct compat_vcpu_runstate_info, time) + 4);
+
+ if (v->kvm->arch.xen.long_mode)
+- offset = offsetof(struct vcpu_runstate_info, state_entry_time);
++ user_times = (uint64_t __user *)(ghc->hva +
++ offsetof(struct vcpu_runstate_info,
++ state_entry_time));
+ #endif
+ /*
+ * First write the updated state_entry_time at the appropriate
+@@ -132,10 +157,8 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state)
+ BUILD_BUG_ON(sizeof(((struct compat_vcpu_runstate_info *)0)->state_entry_time) !=
+ sizeof(state_entry_time));
+
+- if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
+- &state_entry_time, offset,
+- sizeof(state_entry_time)))
+- return;
++ if (__put_user(state_entry_time, user_times))
++ goto out;
+ smp_wmb();
+
+ /*
+@@ -149,11 +172,8 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state)
+ BUILD_BUG_ON(sizeof(((struct compat_vcpu_runstate_info *)0)->state) !=
+ sizeof(vx->current_runstate));
+
+- if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
+- &vx->current_runstate,
+- offsetof(struct vcpu_runstate_info, state),
+- sizeof(vx->current_runstate)))
+- return;
++ if (__put_user(vx->current_runstate, user_state))
++ goto out;
+
+ /*
+ * Write the actual runstate times immediately after the
+@@ -168,24 +188,23 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state)
+ BUILD_BUG_ON(sizeof(((struct vcpu_runstate_info *)0)->time) !=
+ sizeof(vx->runstate_times));
+
+- if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
+- &vx->runstate_times[0],
+- offset + sizeof(u64),
+- sizeof(vx->runstate_times)))
+- return;
+-
++ if (__copy_to_user(user_times + 1, vx->runstate_times, sizeof(vx->runstate_times)))
++ goto out;
+ smp_wmb();
+
+ /*
+ * Finally, clear the XEN_RUNSTATE_UPDATE bit in the guest's
+ * runstate_entry_time field.
+ */
+-
+ state_entry_time &= ~XEN_RUNSTATE_UPDATE;
+- if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
+- &state_entry_time, offset,
+- sizeof(state_entry_time)))
+- return;
++ __put_user(state_entry_time, user_times);
++ smp_wmb();
++
++ out:
++ mark_page_dirty_in_slot(v->kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
++
++ if (atomic)
++ pagefault_enable();
+ }
+
+ int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
+@@ -299,7 +318,7 @@ int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
+ break;
+
+ case KVM_XEN_ATTR_TYPE_SHARED_INFO:
+- data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_gfn);
++ data->u.shared_info.gfn = kvm->arch.xen.shinfo_gfn;
+ r = 0;
+ break;
+
+@@ -337,6 +356,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
+ break;
+ }
+
++ /* It must fit within a single page */
++ if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_info) > PAGE_SIZE) {
++ r = -EINVAL;
++ break;
++ }
++
+ r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
+ &vcpu->arch.xen.vcpu_info_cache,
+ data->u.gpa,
+@@ -354,6 +379,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
+ break;
+ }
+
++ /* It must fit within a single page */
++ if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct pvclock_vcpu_time_info) > PAGE_SIZE) {
++ r = -EINVAL;
++ break;
++ }
++
+ r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
+ &vcpu->arch.xen.vcpu_time_info_cache,
+ data->u.gpa,
+@@ -375,6 +406,12 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
+ break;
+ }
+
++ /* It must fit within a single page */
++ if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_runstate_info) > PAGE_SIZE) {
++ r = -EINVAL;
++ break;
++ }
++
+ r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
+ &vcpu->arch.xen.runstate_cache,
+ data->u.gpa,
+@@ -698,7 +735,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
+ kvm_hv_hypercall_enabled(vcpu))
+ return kvm_hv_hypercall(vcpu);
+
+- longmode = is_64_bit_mode(vcpu);
++ longmode = is_64_bit_hypercall(vcpu);
+ if (!longmode) {
+ params[0] = (u32)kvm_rbx_read(vcpu);
+ params[1] = (u32)kvm_rcx_read(vcpu);
+diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S
+index 16bc9130e7a5e..e768815e58ae4 100644
+--- a/arch/x86/lib/atomic64_386_32.S
++++ b/arch/x86/lib/atomic64_386_32.S
+@@ -9,81 +9,83 @@
+ #include <asm/alternative.h>
+
+ /* if you want SMP support, implement these with real spinlocks */
+-.macro LOCK reg
++.macro IRQ_SAVE reg
+ pushfl
+ cli
+ .endm
+
+-.macro UNLOCK reg
++.macro IRQ_RESTORE reg
+ popfl
+ .endm
+
+-#define BEGIN(op) \
++#define BEGIN_IRQ_SAVE(op) \
+ .macro endp; \
+ SYM_FUNC_END(atomic64_##op##_386); \
+ .purgem endp; \
+ .endm; \
+ SYM_FUNC_START(atomic64_##op##_386); \
+- LOCK v;
++ IRQ_SAVE v;
+
+ #define ENDP endp
+
+-#define RET \
+- UNLOCK v; \
+- ret
+-
+-#define RET_ENDP \
+- RET; \
+- ENDP
++#define RET_IRQ_RESTORE \
++ IRQ_RESTORE v; \
++ RET
+
+ #define v %ecx
+-BEGIN(read)
++BEGIN_IRQ_SAVE(read)
+ movl (v), %eax
+ movl 4(v), %edx
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(set)
++BEGIN_IRQ_SAVE(set)
+ movl %ebx, (v)
+ movl %ecx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(xchg)
++BEGIN_IRQ_SAVE(xchg)
+ movl (v), %eax
+ movl 4(v), %edx
+ movl %ebx, (v)
+ movl %ecx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %ecx
+-BEGIN(add)
++BEGIN_IRQ_SAVE(add)
+ addl %eax, (v)
+ adcl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %ecx
+-BEGIN(add_return)
++BEGIN_IRQ_SAVE(add_return)
+ addl (v), %eax
+ adcl 4(v), %edx
+ movl %eax, (v)
+ movl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %ecx
+-BEGIN(sub)
++BEGIN_IRQ_SAVE(sub)
+ subl %eax, (v)
+ sbbl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %ecx
+-BEGIN(sub_return)
++BEGIN_IRQ_SAVE(sub_return)
+ negl %edx
+ negl %eax
+ sbbl $0, %edx
+@@ -91,47 +93,52 @@ BEGIN(sub_return)
+ adcl 4(v), %edx
+ movl %eax, (v)
+ movl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(inc)
++BEGIN_IRQ_SAVE(inc)
+ addl $1, (v)
+ adcl $0, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(inc_return)
++BEGIN_IRQ_SAVE(inc_return)
+ movl (v), %eax
+ movl 4(v), %edx
+ addl $1, %eax
+ adcl $0, %edx
+ movl %eax, (v)
+ movl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(dec)
++BEGIN_IRQ_SAVE(dec)
+ subl $1, (v)
+ sbbl $0, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(dec_return)
++BEGIN_IRQ_SAVE(dec_return)
+ movl (v), %eax
+ movl 4(v), %edx
+ subl $1, %eax
+ sbbl $0, %edx
+ movl %eax, (v)
+ movl %edx, 4(v)
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(add_unless)
++BEGIN_IRQ_SAVE(add_unless)
+ addl %eax, %ecx
+ adcl %edx, %edi
+ addl (v), %eax
+@@ -143,7 +150,7 @@ BEGIN(add_unless)
+ movl %edx, 4(v)
+ movl $1, %eax
+ 2:
+- RET
++ RET_IRQ_RESTORE
+ 3:
+ cmpl %edx, %edi
+ jne 1b
+@@ -153,7 +160,7 @@ ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(inc_not_zero)
++BEGIN_IRQ_SAVE(inc_not_zero)
+ movl (v), %eax
+ movl 4(v), %edx
+ testl %eax, %eax
+@@ -165,7 +172,7 @@ BEGIN(inc_not_zero)
+ movl %edx, 4(v)
+ movl $1, %eax
+ 2:
+- RET
++ RET_IRQ_RESTORE
+ 3:
+ testl %edx, %edx
+ jne 1b
+@@ -174,7 +181,7 @@ ENDP
+ #undef v
+
+ #define v %esi
+-BEGIN(dec_if_positive)
++BEGIN_IRQ_SAVE(dec_if_positive)
+ movl (v), %eax
+ movl 4(v), %edx
+ subl $1, %eax
+@@ -183,5 +190,6 @@ BEGIN(dec_if_positive)
+ movl %eax, (v)
+ movl %edx, 4(v)
+ 1:
+-RET_ENDP
++ RET_IRQ_RESTORE
++ENDP
+ #undef v
+diff --git a/arch/x86/lib/atomic64_cx8_32.S b/arch/x86/lib/atomic64_cx8_32.S
+index ce6935690766f..90afb488b396a 100644
+--- a/arch/x86/lib/atomic64_cx8_32.S
++++ b/arch/x86/lib/atomic64_cx8_32.S
+@@ -18,7 +18,7 @@
+
+ SYM_FUNC_START(atomic64_read_cx8)
+ read64 %ecx
+- ret
++ RET
+ SYM_FUNC_END(atomic64_read_cx8)
+
+ SYM_FUNC_START(atomic64_set_cx8)
+@@ -28,7 +28,7 @@ SYM_FUNC_START(atomic64_set_cx8)
+ cmpxchg8b (%esi)
+ jne 1b
+
+- ret
++ RET
+ SYM_FUNC_END(atomic64_set_cx8)
+
+ SYM_FUNC_START(atomic64_xchg_cx8)
+@@ -37,7 +37,7 @@ SYM_FUNC_START(atomic64_xchg_cx8)
+ cmpxchg8b (%esi)
+ jne 1b
+
+- ret
++ RET
+ SYM_FUNC_END(atomic64_xchg_cx8)
+
+ .macro addsub_return func ins insc
+@@ -68,7 +68,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8)
+ popl %esi
+ popl %ebx
+ popl %ebp
+- ret
++ RET
+ SYM_FUNC_END(atomic64_\func\()_return_cx8)
+ .endm
+
+@@ -93,7 +93,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8)
+ movl %ebx, %eax
+ movl %ecx, %edx
+ popl %ebx
+- ret
++ RET
+ SYM_FUNC_END(atomic64_\func\()_return_cx8)
+ .endm
+
+@@ -118,7 +118,7 @@ SYM_FUNC_START(atomic64_dec_if_positive_cx8)
+ movl %ebx, %eax
+ movl %ecx, %edx
+ popl %ebx
+- ret
++ RET
+ SYM_FUNC_END(atomic64_dec_if_positive_cx8)
+
+ SYM_FUNC_START(atomic64_add_unless_cx8)
+@@ -149,7 +149,7 @@ SYM_FUNC_START(atomic64_add_unless_cx8)
+ addl $8, %esp
+ popl %ebx
+ popl %ebp
+- ret
++ RET
+ 4:
+ cmpl %edx, 4(%esp)
+ jne 2b
+@@ -176,5 +176,5 @@ SYM_FUNC_START(atomic64_inc_not_zero_cx8)
+ movl $1, %eax
+ 3:
+ popl %ebx
+- ret
++ RET
+ SYM_FUNC_END(atomic64_inc_not_zero_cx8)
+diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
+index 4304320e51f4d..929ad1747dea0 100644
+--- a/arch/x86/lib/checksum_32.S
++++ b/arch/x86/lib/checksum_32.S
+@@ -127,7 +127,7 @@ SYM_FUNC_START(csum_partial)
+ 8:
+ popl %ebx
+ popl %esi
+- ret
++ RET
+ SYM_FUNC_END(csum_partial)
+
+ #else
+@@ -245,7 +245,7 @@ SYM_FUNC_START(csum_partial)
+ 90:
+ popl %ebx
+ popl %esi
+- ret
++ RET
+ SYM_FUNC_END(csum_partial)
+
+ #endif
+@@ -371,7 +371,7 @@ EXC( movb %cl, (%edi) )
+ popl %esi
+ popl %edi
+ popl %ecx # equivalent to addl $4,%esp
+- ret
++ RET
+ SYM_FUNC_END(csum_partial_copy_generic)
+
+ #else
+@@ -447,7 +447,7 @@ EXC( movb %dl, (%edi) )
+ popl %esi
+ popl %edi
+ popl %ebx
+- ret
++ RET
+ SYM_FUNC_END(csum_partial_copy_generic)
+
+ #undef ROUND
+diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
+index c4c7dd115953c..fe59b8ac4fccd 100644
+--- a/arch/x86/lib/clear_page_64.S
++++ b/arch/x86/lib/clear_page_64.S
+@@ -17,7 +17,7 @@ SYM_FUNC_START(clear_page_rep)
+ movl $4096/8,%ecx
+ xorl %eax,%eax
+ rep stosq
+- ret
++ RET
+ SYM_FUNC_END(clear_page_rep)
+ EXPORT_SYMBOL_GPL(clear_page_rep)
+
+@@ -39,7 +39,7 @@ SYM_FUNC_START(clear_page_orig)
+ leaq 64(%rdi),%rdi
+ jnz .Lloop
+ nop
+- ret
++ RET
+ SYM_FUNC_END(clear_page_orig)
+ EXPORT_SYMBOL_GPL(clear_page_orig)
+
+@@ -47,6 +47,6 @@ SYM_FUNC_START(clear_page_erms)
+ movl $4096,%ecx
+ xorl %eax,%eax
+ rep stosb
+- ret
++ RET
+ SYM_FUNC_END(clear_page_erms)
+ EXPORT_SYMBOL_GPL(clear_page_erms)
+diff --git a/arch/x86/lib/cmpxchg16b_emu.S b/arch/x86/lib/cmpxchg16b_emu.S
+index 3542502faa3b7..33c70c0160ea0 100644
+--- a/arch/x86/lib/cmpxchg16b_emu.S
++++ b/arch/x86/lib/cmpxchg16b_emu.S
+@@ -37,11 +37,11 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
+
+ popfq
+ mov $1, %al
+- ret
++ RET
+
+ .Lnot_same:
+ popfq
+ xor %al,%al
+- ret
++ RET
+
+ SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
+diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
+index ca01ed6029f4f..6a912d58fecc3 100644
+--- a/arch/x86/lib/cmpxchg8b_emu.S
++++ b/arch/x86/lib/cmpxchg8b_emu.S
+@@ -32,7 +32,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
+ movl %ecx, 4(%esi)
+
+ popfl
+- ret
++ RET
+
+ .Lnot_same:
+ movl (%esi), %eax
+@@ -40,7 +40,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
+ movl 4(%esi), %edx
+
+ popfl
+- ret
++ RET
+
+ SYM_FUNC_END(cmpxchg8b_emu)
+ EXPORT_SYMBOL(cmpxchg8b_emu)
+diff --git a/arch/x86/lib/copy_mc_64.S b/arch/x86/lib/copy_mc_64.S
+index e5f77e2930349..2c623a2bbd26e 100644
+--- a/arch/x86/lib/copy_mc_64.S
++++ b/arch/x86/lib/copy_mc_64.S
+@@ -77,7 +77,7 @@ SYM_FUNC_START(copy_mc_fragile)
+ .L_done_memcpy_trap:
+ xorl %eax, %eax
+ .L_done:
+- ret
++ RET
+ SYM_FUNC_END(copy_mc_fragile)
+
+ .section .fixup, "ax"
+@@ -132,7 +132,7 @@ SYM_FUNC_START(copy_mc_enhanced_fast_string)
+ rep movsb
+ /* Copy successful. Return zero */
+ xorl %eax, %eax
+- ret
++ RET
+ SYM_FUNC_END(copy_mc_enhanced_fast_string)
+
+ .section .fixup, "ax"
+@@ -145,7 +145,7 @@ SYM_FUNC_END(copy_mc_enhanced_fast_string)
+ * user-copy routines.
+ */
+ movq %rcx, %rax
+- ret
++ RET
+
+ .previous
+
+diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S
+index db4b4f9197c7d..30ea644bf446d 100644
+--- a/arch/x86/lib/copy_page_64.S
++++ b/arch/x86/lib/copy_page_64.S
+@@ -17,7 +17,7 @@ SYM_FUNC_START(copy_page)
+ ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD
+ movl $4096/8, %ecx
+ rep movsq
+- ret
++ RET
+ SYM_FUNC_END(copy_page)
+ EXPORT_SYMBOL(copy_page)
+
+@@ -85,5 +85,5 @@ SYM_FUNC_START_LOCAL(copy_page_regs)
+ movq (%rsp), %rbx
+ movq 1*8(%rsp), %r12
+ addq $2*8, %rsp
+- ret
++ RET
+ SYM_FUNC_END(copy_page_regs)
+diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
+index 57b79c577496d..84cee84fc658a 100644
+--- a/arch/x86/lib/copy_user_64.S
++++ b/arch/x86/lib/copy_user_64.S
+@@ -105,7 +105,7 @@ SYM_FUNC_START(copy_user_generic_unrolled)
+ jnz 21b
+ 23: xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+
+ .section .fixup,"ax"
+ 30: shll $6,%ecx
+@@ -173,7 +173,7 @@ SYM_FUNC_START(copy_user_generic_string)
+ movsb
+ xorl %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+
+ .section .fixup,"ax"
+ 11: leal (%rdx,%rcx,8),%ecx
+@@ -207,7 +207,7 @@ SYM_FUNC_START(copy_user_enhanced_fast_string)
+ movsb
+ xorl %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+
+ .section .fixup,"ax"
+ 12: movl %ecx,%edx /* ecx is zerorest also */
+@@ -239,7 +239,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
+ 1: rep movsb
+ 2: mov %ecx,%eax
+ ASM_CLAC
+- ret
++ RET
+
+ /*
+ * Return zero to pretend that this copy succeeded. This
+@@ -250,7 +250,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
+ */
+ 3: xorl %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+
+ _ASM_EXTABLE_CPY(1b, 2b)
+ SYM_CODE_END(.Lcopy_user_handle_tail)
+@@ -361,7 +361,7 @@ SYM_FUNC_START(__copy_user_nocache)
+ xorl %eax,%eax
+ ASM_CLAC
+ sfence
+- ret
++ RET
+
+ .section .fixup,"ax"
+ .L_fixup_4x8b_copy:
+diff --git a/arch/x86/lib/csum-copy_64.S b/arch/x86/lib/csum-copy_64.S
+index 1fbd8ee9642d1..d9e16a2cf2856 100644
+--- a/arch/x86/lib/csum-copy_64.S
++++ b/arch/x86/lib/csum-copy_64.S
+@@ -201,7 +201,7 @@ SYM_FUNC_START(csum_partial_copy_generic)
+ movq 3*8(%rsp), %r13
+ movq 4*8(%rsp), %r15
+ addq $5*8, %rsp
+- ret
++ RET
+ .Lshort:
+ movl %ecx, %r10d
+ jmp .L1
+diff --git a/arch/x86/lib/error-inject.c b/arch/x86/lib/error-inject.c
+index be5b5fb1598bd..520897061ee09 100644
+--- a/arch/x86/lib/error-inject.c
++++ b/arch/x86/lib/error-inject.c
+@@ -1,5 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0
+
++#include <linux/linkage.h>
+ #include <linux/error-injection.h>
+ #include <linux/kprobes.h>
+
+@@ -10,7 +11,7 @@ asm(
+ ".type just_return_func, @function\n"
+ ".globl just_return_func\n"
+ "just_return_func:\n"
+- " ret\n"
++ ASM_RET
+ ".size just_return_func, .-just_return_func\n"
+ );
+
+diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
+index fa1bc2104b326..b70d98d79a9da 100644
+--- a/arch/x86/lib/getuser.S
++++ b/arch/x86/lib/getuser.S
+@@ -57,7 +57,7 @@ SYM_FUNC_START(__get_user_1)
+ 1: movzbl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_1)
+ EXPORT_SYMBOL(__get_user_1)
+
+@@ -71,7 +71,7 @@ SYM_FUNC_START(__get_user_2)
+ 2: movzwl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_2)
+ EXPORT_SYMBOL(__get_user_2)
+
+@@ -85,7 +85,7 @@ SYM_FUNC_START(__get_user_4)
+ 3: movl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_4)
+ EXPORT_SYMBOL(__get_user_4)
+
+@@ -100,7 +100,7 @@ SYM_FUNC_START(__get_user_8)
+ 4: movq (%_ASM_AX),%rdx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ #else
+ LOAD_TASK_SIZE_MINUS_N(7)
+ cmp %_ASM_DX,%_ASM_AX
+@@ -112,7 +112,7 @@ SYM_FUNC_START(__get_user_8)
+ 5: movl 4(%_ASM_AX),%ecx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ #endif
+ SYM_FUNC_END(__get_user_8)
+ EXPORT_SYMBOL(__get_user_8)
+@@ -124,7 +124,7 @@ SYM_FUNC_START(__get_user_nocheck_1)
+ 6: movzbl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_nocheck_1)
+ EXPORT_SYMBOL(__get_user_nocheck_1)
+
+@@ -134,7 +134,7 @@ SYM_FUNC_START(__get_user_nocheck_2)
+ 7: movzwl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_nocheck_2)
+ EXPORT_SYMBOL(__get_user_nocheck_2)
+
+@@ -144,7 +144,7 @@ SYM_FUNC_START(__get_user_nocheck_4)
+ 8: movl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_nocheck_4)
+ EXPORT_SYMBOL(__get_user_nocheck_4)
+
+@@ -159,7 +159,7 @@ SYM_FUNC_START(__get_user_nocheck_8)
+ #endif
+ xor %eax,%eax
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__get_user_nocheck_8)
+ EXPORT_SYMBOL(__get_user_nocheck_8)
+
+@@ -169,7 +169,7 @@ SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
+ bad_get_user:
+ xor %edx,%edx
+ mov $(-EFAULT),%_ASM_AX
+- ret
++ RET
+ SYM_CODE_END(.Lbad_get_user_clac)
+
+ #ifdef CONFIG_X86_32
+@@ -179,7 +179,7 @@ bad_get_user_8:
+ xor %edx,%edx
+ xor %ecx,%ecx
+ mov $(-EFAULT),%_ASM_AX
+- ret
++ RET
+ SYM_CODE_END(.Lbad_get_user_8_clac)
+ #endif
+
+diff --git a/arch/x86/lib/hweight.S b/arch/x86/lib/hweight.S
+index dbf8cc97b7f53..12c16c6aa44a3 100644
+--- a/arch/x86/lib/hweight.S
++++ b/arch/x86/lib/hweight.S
+@@ -32,7 +32,7 @@ SYM_FUNC_START(__sw_hweight32)
+ imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101
+ shrl $24, %eax # w = w_tmp >> 24
+ __ASM_SIZE(pop,) %__ASM_REG(dx)
+- ret
++ RET
+ SYM_FUNC_END(__sw_hweight32)
+ EXPORT_SYMBOL(__sw_hweight32)
+
+@@ -65,7 +65,7 @@ SYM_FUNC_START(__sw_hweight64)
+
+ popq %rdx
+ popq %rdi
+- ret
++ RET
+ #else /* CONFIG_X86_32 */
+ /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
+ pushl %ecx
+@@ -77,7 +77,7 @@ SYM_FUNC_START(__sw_hweight64)
+ addl %ecx, %eax # result
+
+ popl %ecx
+- ret
++ RET
+ #endif
+ SYM_FUNC_END(__sw_hweight64)
+ EXPORT_SYMBOL(__sw_hweight64)
+diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c
+index a1d24fdc07cf0..eb3ccffb9b9dc 100644
+--- a/arch/x86/lib/insn-eval.c
++++ b/arch/x86/lib/insn-eval.c
+@@ -1417,7 +1417,7 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs)
+ }
+ }
+
+-static int insn_get_effective_ip(struct pt_regs *regs, unsigned long *ip)
++int insn_get_effective_ip(struct pt_regs *regs, unsigned long *ip)
+ {
+ unsigned long seg_base = 0;
+
+diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c
+index c565def611e24..55e371cc69fd5 100644
+--- a/arch/x86/lib/insn.c
++++ b/arch/x86/lib/insn.c
+@@ -13,6 +13,7 @@
+ #endif
+ #include <asm/inat.h> /*__ignore_sync_check__ */
+ #include <asm/insn.h> /* __ignore_sync_check__ */
++#include <asm/unaligned.h> /* __ignore_sync_check__ */
+
+ #include <linux/errno.h>
+ #include <linux/kconfig.h>
+@@ -37,10 +38,10 @@
+ ((insn)->next_byte + sizeof(t) + n <= (insn)->end_kaddr)
+
+ #define __get_next(t, insn) \
+- ({ t r; memcpy(&r, insn->next_byte, sizeof(t)); insn->next_byte += sizeof(t); leXX_to_cpu(t, r); })
++ ({ t r = get_unaligned((t *)(insn)->next_byte); (insn)->next_byte += sizeof(t); leXX_to_cpu(t, r); })
+
+ #define __peek_nbyte_next(t, insn, n) \
+- ({ t r; memcpy(&r, (insn)->next_byte + n, sizeof(t)); leXX_to_cpu(t, r); })
++ ({ t r = get_unaligned((t *)(insn)->next_byte + n); leXX_to_cpu(t, r); })
+
+ #define get_next(t, insn) \
+ ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); })
+diff --git a/arch/x86/lib/iomap_copy_64.S b/arch/x86/lib/iomap_copy_64.S
+index cb5a1964506b1..a1f9416bf67a5 100644
+--- a/arch/x86/lib/iomap_copy_64.S
++++ b/arch/x86/lib/iomap_copy_64.S
+@@ -11,5 +11,5 @@
+ SYM_FUNC_START(__iowrite32_copy)
+ movl %edx,%ecx
+ rep movsd
+- ret
++ RET
+ SYM_FUNC_END(__iowrite32_copy)
+diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
+index 1cc9da6e29c79..59cf2343f3d90 100644
+--- a/arch/x86/lib/memcpy_64.S
++++ b/arch/x86/lib/memcpy_64.S
+@@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy)
+ rep movsq
+ movl %edx, %ecx
+ rep movsb
+- ret
++ RET
+ SYM_FUNC_END(memcpy)
+ SYM_FUNC_END_ALIAS(__memcpy)
+ EXPORT_SYMBOL(memcpy)
+@@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms)
+ movq %rdi, %rax
+ movq %rdx, %rcx
+ rep movsb
+- ret
++ RET
+ SYM_FUNC_END(memcpy_erms)
+
+ SYM_FUNC_START_LOCAL(memcpy_orig)
+@@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movq %r9, 1*8(%rdi)
+ movq %r10, -2*8(%rdi, %rdx)
+ movq %r11, -1*8(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_16bytes:
+ cmpl $8, %edx
+@@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movq -1*8(%rsi, %rdx), %r9
+ movq %r8, 0*8(%rdi)
+ movq %r9, -1*8(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_8bytes:
+ cmpl $4, %edx
+@@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movl -4(%rsi, %rdx), %r8d
+ movl %ecx, (%rdi)
+ movl %r8d, -4(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_3bytes:
+ subl $1, %edx
+@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movb %cl, (%rdi)
+
+ .Lend:
+- retq
++ RET
+ SYM_FUNC_END(memcpy_orig)
+
+ .popsection
+diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S
+index 64801010d312d..50ea390df7128 100644
+--- a/arch/x86/lib/memmove_64.S
++++ b/arch/x86/lib/memmove_64.S
+@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove)
+ /* FSRM implies ERMS => no length checks, do the copy directly */
+ .Lmemmove_begin_forward:
+ ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM
+- ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS
++ ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS
+
+ /*
+ * movsq instruction have many startup latency
+@@ -205,7 +205,7 @@ SYM_FUNC_START(__memmove)
+ movb (%rsi), %r11b
+ movb %r11b, (%rdi)
+ 13:
+- retq
++ RET
+ SYM_FUNC_END(__memmove)
+ SYM_FUNC_END_ALIAS(memmove)
+ EXPORT_SYMBOL(__memmove)
+diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
+index 9827ae267f96e..d624f2bc42f16 100644
+--- a/arch/x86/lib/memset_64.S
++++ b/arch/x86/lib/memset_64.S
+@@ -40,7 +40,7 @@ SYM_FUNC_START(__memset)
+ movl %edx,%ecx
+ rep stosb
+ movq %r9,%rax
+- ret
++ RET
+ SYM_FUNC_END(__memset)
+ SYM_FUNC_END_ALIAS(memset)
+ EXPORT_SYMBOL(memset)
+@@ -63,7 +63,7 @@ SYM_FUNC_START_LOCAL(memset_erms)
+ movq %rdx,%rcx
+ rep stosb
+ movq %r9,%rax
+- ret
++ RET
+ SYM_FUNC_END(memset_erms)
+
+ SYM_FUNC_START_LOCAL(memset_orig)
+@@ -125,7 +125,7 @@ SYM_FUNC_START_LOCAL(memset_orig)
+
+ .Lende:
+ movq %r10,%rax
+- ret
++ RET
+
+ .Lbad_alignment:
+ cmpq $7,%rdx
+diff --git a/arch/x86/lib/msr-reg.S b/arch/x86/lib/msr-reg.S
+index a2b9caa5274c8..ebd259f314963 100644
+--- a/arch/x86/lib/msr-reg.S
++++ b/arch/x86/lib/msr-reg.S
+@@ -35,7 +35,7 @@ SYM_FUNC_START(\op\()_safe_regs)
+ movl %edi, 28(%r10)
+ popq %r12
+ popq %rbx
+- ret
++ RET
+ 3:
+ movl $-EIO, %r11d
+ jmp 2b
+@@ -77,7 +77,7 @@ SYM_FUNC_START(\op\()_safe_regs)
+ popl %esi
+ popl %ebp
+ popl %ebx
+- ret
++ RET
+ 3:
+ movl $-EIO, 4(%esp)
+ jmp 2b
+diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S
+index 0ea344c5ea439..ecb2049c1273f 100644
+--- a/arch/x86/lib/putuser.S
++++ b/arch/x86/lib/putuser.S
+@@ -52,7 +52,7 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
+ 1: movb %al,(%_ASM_CX)
+ xor %ecx,%ecx
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__put_user_1)
+ EXPORT_SYMBOL(__put_user_1)
+ EXPORT_SYMBOL(__put_user_nocheck_1)
+@@ -66,7 +66,7 @@ SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
+ 2: movw %ax,(%_ASM_CX)
+ xor %ecx,%ecx
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__put_user_2)
+ EXPORT_SYMBOL(__put_user_2)
+ EXPORT_SYMBOL(__put_user_nocheck_2)
+@@ -80,7 +80,7 @@ SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
+ 3: movl %eax,(%_ASM_CX)
+ xor %ecx,%ecx
+ ASM_CLAC
+- ret
++ RET
+ SYM_FUNC_END(__put_user_4)
+ EXPORT_SYMBOL(__put_user_4)
+ EXPORT_SYMBOL(__put_user_nocheck_4)
+diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
+index ec9922cba30a4..9556ff5f4773f 100644
+--- a/arch/x86/lib/retpoline.S
++++ b/arch/x86/lib/retpoline.S
+@@ -23,7 +23,7 @@
+ .Ldo_rop_\@:
+ mov %\reg, (%_ASM_SP)
+ UNWIND_HINT_FUNC
+- ret
++ RET
+ .endm
+
+ .macro THUNK reg
+@@ -34,7 +34,7 @@ SYM_FUNC_START(__x86_indirect_thunk_\reg)
+
+ ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
+ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \
+- __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD
++ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE
+
+ SYM_FUNC_END(__x86_indirect_thunk_\reg)
+
+diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
+index 508c81e97ab10..f1c0befb62df5 100644
+--- a/arch/x86/lib/usercopy_64.c
++++ b/arch/x86/lib/usercopy_64.c
+@@ -121,7 +121,7 @@ void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
+
+ /* cache copy and flush to align dest */
+ if (!IS_ALIGNED(dest, 8)) {
+- unsigned len = min_t(unsigned, size, ALIGN(dest, 8) - dest);
++ size_t len = min_t(size_t, size, ALIGN(dest, 8) - dest);
+
+ memcpy((void *) dest, (void *) source, len);
+ clean_cache_range((void *) dest, len);
+diff --git a/arch/x86/math-emu/div_Xsig.S b/arch/x86/math-emu/div_Xsig.S
+index 951da2ad54bbf..8c270ab415bee 100644
+--- a/arch/x86/math-emu/div_Xsig.S
++++ b/arch/x86/math-emu/div_Xsig.S
+@@ -341,7 +341,7 @@ L_exit:
+ popl %esi
+
+ leave
+- ret
++ RET
+
+
+ #ifdef PARANOID
+diff --git a/arch/x86/math-emu/div_small.S b/arch/x86/math-emu/div_small.S
+index d047d1816abe9..637439bfefa47 100644
+--- a/arch/x86/math-emu/div_small.S
++++ b/arch/x86/math-emu/div_small.S
+@@ -44,5 +44,5 @@ SYM_FUNC_START(FPU_div_small)
+ popl %esi
+
+ leave
+- ret
++ RET
+ SYM_FUNC_END(FPU_div_small)
+diff --git a/arch/x86/math-emu/mul_Xsig.S b/arch/x86/math-emu/mul_Xsig.S
+index 4afc7b1fa6e95..54a031b661421 100644
+--- a/arch/x86/math-emu/mul_Xsig.S
++++ b/arch/x86/math-emu/mul_Xsig.S
+@@ -62,7 +62,7 @@ SYM_FUNC_START(mul32_Xsig)
+
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(mul32_Xsig)
+
+
+@@ -115,7 +115,7 @@ SYM_FUNC_START(mul64_Xsig)
+
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(mul64_Xsig)
+
+
+@@ -175,5 +175,5 @@ SYM_FUNC_START(mul_Xsig_Xsig)
+
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(mul_Xsig_Xsig)
+diff --git a/arch/x86/math-emu/polynom_Xsig.S b/arch/x86/math-emu/polynom_Xsig.S
+index 702315eecb860..35fd723fc0df8 100644
+--- a/arch/x86/math-emu/polynom_Xsig.S
++++ b/arch/x86/math-emu/polynom_Xsig.S
+@@ -133,5 +133,5 @@ L_accum_done:
+ popl %edi
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(polynomial_Xsig)
+diff --git a/arch/x86/math-emu/reg_norm.S b/arch/x86/math-emu/reg_norm.S
+index cad1d60b1e844..594936eeed67a 100644
+--- a/arch/x86/math-emu/reg_norm.S
++++ b/arch/x86/math-emu/reg_norm.S
+@@ -72,7 +72,7 @@ L_exit_valid:
+ L_exit:
+ popl %ebx
+ leave
+- ret
++ RET
+
+
+ L_zero:
+@@ -138,7 +138,7 @@ L_exit_nuo_valid:
+
+ popl %ebx
+ leave
+- ret
++ RET
+
+ L_exit_nuo_zero:
+ movl TAG_Zero,%eax
+@@ -146,5 +146,5 @@ L_exit_nuo_zero:
+
+ popl %ebx
+ leave
+- ret
++ RET
+ SYM_FUNC_END(FPU_normalize_nuo)
+diff --git a/arch/x86/math-emu/reg_round.S b/arch/x86/math-emu/reg_round.S
+index 4a9fc3cc5a4d4..0bb2a092161af 100644
+--- a/arch/x86/math-emu/reg_round.S
++++ b/arch/x86/math-emu/reg_round.S
+@@ -437,7 +437,7 @@ fpu_Arith_exit:
+ popl %edi
+ popl %esi
+ leave
+- ret
++ RET
+
+
+ /*
+diff --git a/arch/x86/math-emu/reg_u_add.S b/arch/x86/math-emu/reg_u_add.S
+index 9c9e2c810afe8..07247287a3af7 100644
+--- a/arch/x86/math-emu/reg_u_add.S
++++ b/arch/x86/math-emu/reg_u_add.S
+@@ -164,6 +164,6 @@ L_exit:
+ popl %edi
+ popl %esi
+ leave
+- ret
++ RET
+ #endif /* PARANOID */
+ SYM_FUNC_END(FPU_u_add)
+diff --git a/arch/x86/math-emu/reg_u_div.S b/arch/x86/math-emu/reg_u_div.S
+index e2fb5c2644c55..b5a41e2fc484c 100644
+--- a/arch/x86/math-emu/reg_u_div.S
++++ b/arch/x86/math-emu/reg_u_div.S
+@@ -468,7 +468,7 @@ L_exit:
+ popl %esi
+
+ leave
+- ret
++ RET
+ #endif /* PARANOID */
+
+ SYM_FUNC_END(FPU_u_div)
+diff --git a/arch/x86/math-emu/reg_u_mul.S b/arch/x86/math-emu/reg_u_mul.S
+index 0c779c87ac5b3..e2588b24b8c2c 100644
+--- a/arch/x86/math-emu/reg_u_mul.S
++++ b/arch/x86/math-emu/reg_u_mul.S
+@@ -144,7 +144,7 @@ L_exit:
+ popl %edi
+ popl %esi
+ leave
+- ret
++ RET
+ #endif /* PARANOID */
+
+ SYM_FUNC_END(FPU_u_mul)
+diff --git a/arch/x86/math-emu/reg_u_sub.S b/arch/x86/math-emu/reg_u_sub.S
+index e9bb7c248649f..4c900c29e4ff2 100644
+--- a/arch/x86/math-emu/reg_u_sub.S
++++ b/arch/x86/math-emu/reg_u_sub.S
+@@ -270,5 +270,5 @@ L_exit:
+ popl %edi
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(FPU_u_sub)
+diff --git a/arch/x86/math-emu/round_Xsig.S b/arch/x86/math-emu/round_Xsig.S
+index d9d7de8dbd7b6..126c40473badb 100644
+--- a/arch/x86/math-emu/round_Xsig.S
++++ b/arch/x86/math-emu/round_Xsig.S
+@@ -78,7 +78,7 @@ L_exit:
+ popl %esi
+ popl %ebx
+ leave
+- ret
++ RET
+ SYM_FUNC_END(round_Xsig)
+
+
+@@ -138,5 +138,5 @@ L_n_exit:
+ popl %esi
+ popl %ebx
+ leave
+- ret
++ RET
+ SYM_FUNC_END(norm_Xsig)
+diff --git a/arch/x86/math-emu/shr_Xsig.S b/arch/x86/math-emu/shr_Xsig.S
+index 726af985f7582..f726bf6f6396e 100644
+--- a/arch/x86/math-emu/shr_Xsig.S
++++ b/arch/x86/math-emu/shr_Xsig.S
+@@ -45,7 +45,7 @@ SYM_FUNC_START(shr_Xsig)
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_31:
+ cmpl $64,%ecx
+@@ -61,7 +61,7 @@ L_more_than_31:
+ movl $0,8(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_63:
+ cmpl $96,%ecx
+@@ -76,7 +76,7 @@ L_more_than_63:
+ movl %edx,8(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_95:
+ xorl %eax,%eax
+@@ -85,5 +85,5 @@ L_more_than_95:
+ movl %eax,8(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(shr_Xsig)
+diff --git a/arch/x86/math-emu/wm_shrx.S b/arch/x86/math-emu/wm_shrx.S
+index 4fc89174caf0c..f608a28a4c43a 100644
+--- a/arch/x86/math-emu/wm_shrx.S
++++ b/arch/x86/math-emu/wm_shrx.S
+@@ -55,7 +55,7 @@ SYM_FUNC_START(FPU_shrx)
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_31:
+ cmpl $64,%ecx
+@@ -70,7 +70,7 @@ L_more_than_31:
+ movl $0,4(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_63:
+ cmpl $96,%ecx
+@@ -84,7 +84,7 @@ L_more_than_63:
+ movl %edx,4(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+
+ L_more_than_95:
+ xorl %eax,%eax
+@@ -92,7 +92,7 @@ L_more_than_95:
+ movl %eax,4(%esi)
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(FPU_shrx)
+
+
+@@ -146,7 +146,7 @@ SYM_FUNC_START(FPU_shrxs)
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+
+ /* Shift by [0..31] bits */
+ Ls_less_than_32:
+@@ -163,7 +163,7 @@ Ls_less_than_32:
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+
+ /* Shift by [64..95] bits */
+ Ls_more_than_63:
+@@ -189,7 +189,7 @@ Ls_more_than_63:
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+
+ Ls_more_than_95:
+ /* Shift by [96..inf) bits */
+@@ -203,5 +203,5 @@ Ls_more_than_95:
+ popl %ebx
+ popl %esi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(FPU_shrxs)
+diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
+index f5e1e60c9095f..6c2f1b76a0b61 100644
+--- a/arch/x86/mm/cpu_entry_area.c
++++ b/arch/x86/mm/cpu_entry_area.c
+@@ -110,6 +110,13 @@ static void __init percpu_setup_exception_stacks(unsigned int cpu)
+ cea_map_stack(NMI);
+ cea_map_stack(DB);
+ cea_map_stack(MCE);
++
++ if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
++ if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) {
++ cea_map_stack(VC);
++ cea_map_stack(VC2);
++ }
++ }
+ }
+ #else
+ static inline void percpu_setup_exception_stacks(unsigned int cpu)
+diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
+index 84a2c8c4af735..4bfed53e210ec 100644
+--- a/arch/x86/mm/fault.c
++++ b/arch/x86/mm/fault.c
+@@ -32,6 +32,7 @@
+ #include <asm/pgtable_areas.h> /* VMALLOC_START, ... */
+ #include <asm/kvm_para.h> /* kvm_handle_async_pf */
+ #include <asm/vdso.h> /* fixup_vdso_exception() */
++#include <asm/irq_stack.h>
+
+ #define CREATE_TRACE_POINTS
+ #include <asm/trace/exceptions.h>
+@@ -631,6 +632,9 @@ static noinline void
+ page_fault_oops(struct pt_regs *regs, unsigned long error_code,
+ unsigned long address)
+ {
++#ifdef CONFIG_VMAP_STACK
++ struct stack_info info;
++#endif
+ unsigned long flags;
+ int sig;
+
+@@ -649,9 +653,7 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code,
+ * that we're in vmalloc space to avoid this.
+ */
+ if (is_vmalloc_addr((void *)address) &&
+- (((unsigned long)current->stack - 1 - address < PAGE_SIZE) ||
+- address - ((unsigned long)current->stack + THREAD_SIZE) < PAGE_SIZE)) {
+- unsigned long stack = __this_cpu_ist_top_va(DF) - sizeof(void *);
++ get_stack_guard_info((void *)address, &info)) {
+ /*
+ * We're likely to be running with very little stack space
+ * left. It's plausible that we'd hit this condition but
+@@ -662,13 +664,11 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code,
+ * and then double-fault, though, because we're likely to
+ * break the console driver and lose most of the stack dump.
+ */
+- asm volatile ("movq %[stack], %%rsp\n\t"
+- "call handle_stack_overflow\n\t"
+- "1: jmp 1b"
+- : ASM_CALL_CONSTRAINT
+- : "D" ("kernel stack overflow (page fault)"),
+- "S" (regs), "d" (address),
+- [stack] "rm" (stack));
++ call_on_stack(__this_cpu_ist_top_va(DF) - sizeof(void*),
++ handle_stack_overflow,
++ ASM_CALL_ARG3,
++ , [arg1] "r" (regs), [arg2] "r" (address), [arg3] "r" (&info));
++
+ unreachable();
+ }
+ #endif
+diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
+index 36098226a9573..b01f5d2caad04 100644
+--- a/arch/x86/mm/init_64.c
++++ b/arch/x86/mm/init_64.c
+@@ -902,6 +902,8 @@ static void __meminit vmemmap_use_sub_pmd(unsigned long start, unsigned long end
+
+ static void __meminit vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end)
+ {
++ const unsigned long page = ALIGN_DOWN(start, PMD_SIZE);
++
+ vmemmap_flush_unused_pmd();
+
+ /*
+@@ -914,8 +916,7 @@ static void __meminit vmemmap_use_new_sub_pmd(unsigned long start, unsigned long
+ * Mark with PAGE_UNUSED the unused parts of the new memmap range
+ */
+ if (!IS_ALIGNED(start, PMD_SIZE))
+- memset((void *)start, PAGE_UNUSED,
+- start - ALIGN_DOWN(start, PMD_SIZE));
++ memset((void *)page, PAGE_UNUSED, start - page);
+
+ /*
+ * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of
+diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
+index 60ade7dd71bd9..7ce9b8dd87577 100644
+--- a/arch/x86/mm/ioremap.c
++++ b/arch/x86/mm/ioremap.c
+@@ -614,6 +614,7 @@ static bool memremap_is_efi_data(resource_size_t phys_addr,
+ static bool memremap_is_setup_data(resource_size_t phys_addr,
+ unsigned long size)
+ {
++ struct setup_indirect *indirect;
+ struct setup_data *data;
+ u64 paddr, paddr_next;
+
+@@ -626,6 +627,10 @@ static bool memremap_is_setup_data(resource_size_t phys_addr,
+
+ data = memremap(paddr, sizeof(*data),
+ MEMREMAP_WB | MEMREMAP_DEC);
++ if (!data) {
++ pr_warn("failed to memremap setup_data entry\n");
++ return false;
++ }
+
+ paddr_next = data->next;
+ len = data->len;
+@@ -635,10 +640,21 @@ static bool memremap_is_setup_data(resource_size_t phys_addr,
+ return true;
+ }
+
+- if (data->type == SETUP_INDIRECT &&
+- ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) {
+- paddr = ((struct setup_indirect *)data->data)->addr;
+- len = ((struct setup_indirect *)data->data)->len;
++ if (data->type == SETUP_INDIRECT) {
++ memunmap(data);
++ data = memremap(paddr, sizeof(*data) + len,
++ MEMREMAP_WB | MEMREMAP_DEC);
++ if (!data) {
++ pr_warn("failed to memremap indirect setup_data\n");
++ return false;
++ }
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT) {
++ paddr = indirect->addr;
++ len = indirect->len;
++ }
+ }
+
+ memunmap(data);
+@@ -659,22 +675,51 @@ static bool memremap_is_setup_data(resource_size_t phys_addr,
+ static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
+ unsigned long size)
+ {
++ struct setup_indirect *indirect;
+ struct setup_data *data;
+ u64 paddr, paddr_next;
+
+ paddr = boot_params.hdr.setup_data;
+ while (paddr) {
+- unsigned int len;
++ unsigned int len, size;
+
+ if (phys_addr == paddr)
+ return true;
+
+ data = early_memremap_decrypted(paddr, sizeof(*data));
++ if (!data) {
++ pr_warn("failed to early memremap setup_data entry\n");
++ return false;
++ }
++
++ size = sizeof(*data);
+
+ paddr_next = data->next;
+ len = data->len;
+
+- early_memunmap(data, sizeof(*data));
++ if ((phys_addr > paddr) && (phys_addr < (paddr + len))) {
++ early_memunmap(data, sizeof(*data));
++ return true;
++ }
++
++ if (data->type == SETUP_INDIRECT) {
++ size += len;
++ early_memunmap(data, sizeof(*data));
++ data = early_memremap_decrypted(paddr, size);
++ if (!data) {
++ pr_warn("failed to early memremap indirect setup_data\n");
++ return false;
++ }
++
++ indirect = (struct setup_indirect *)data->data;
++
++ if (indirect->type != SETUP_INDIRECT) {
++ paddr = indirect->addr;
++ len = indirect->len;
++ }
++ }
++
++ early_memunmap(data, size);
+
+ if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
+ return true;
+diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
+index ff08dc4636347..e29b1418d00c7 100644
+--- a/arch/x86/mm/mem_encrypt.c
++++ b/arch/x86/mm/mem_encrypt.c
+@@ -20,6 +20,7 @@
+ #include <linux/bitops.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/virtio_config.h>
++#include <linux/cc_platform.h>
+
+ #include <asm/tlbflush.h>
+ #include <asm/fixmap.h>
+diff --git a/arch/x86/mm/mem_encrypt_boot.S b/arch/x86/mm/mem_encrypt_boot.S
+index 17d292b7072f1..3d1dba05fce4a 100644
+--- a/arch/x86/mm/mem_encrypt_boot.S
++++ b/arch/x86/mm/mem_encrypt_boot.S
+@@ -65,7 +65,7 @@ SYM_FUNC_START(sme_encrypt_execute)
+ movq %rbp, %rsp /* Restore original stack pointer */
+ pop %rbp
+
+- ret
++ RET
+ SYM_FUNC_END(sme_encrypt_execute)
+
+ SYM_FUNC_START(__enc_copy)
+@@ -151,6 +151,6 @@ SYM_FUNC_START(__enc_copy)
+ pop %r12
+ pop %r15
+
+- ret
++ RET
+ .L__enc_copy_end:
+ SYM_FUNC_END(__enc_copy)
+diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
+index 470b202084306..700ce8fdea87c 100644
+--- a/arch/x86/mm/mem_encrypt_identity.c
++++ b/arch/x86/mm/mem_encrypt_identity.c
+@@ -27,6 +27,15 @@
+ #undef CONFIG_PARAVIRT_XXL
+ #undef CONFIG_PARAVIRT_SPINLOCKS
+
++/*
++ * This code runs before CPU feature bits are set. By default, the
++ * pgtable_l5_enabled() function uses bit X86_FEATURE_LA57 to determine if
++ * 5-level paging is active, so that won't work here. USE_EARLY_PGTABLE_L5
++ * is provided to handle this situation and, instead, use a variable that
++ * has been set by the early boot code.
++ */
++#define USE_EARLY_PGTABLE_L5
++
+ #include <linux/kernel.h>
+ #include <linux/mm.h>
+ #include <linux/mem_encrypt.h>
+diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
+index 59ba2968af1b3..511172d70825c 100644
+--- a/arch/x86/mm/tlb.c
++++ b/arch/x86/mm/tlb.c
+@@ -854,13 +854,11 @@ done:
+ nr_invalidate);
+ }
+
+-static bool tlb_is_not_lazy(int cpu)
++static bool tlb_is_not_lazy(int cpu, void *data)
+ {
+ return !per_cpu(cpu_tlbstate_shared.is_lazy, cpu);
+ }
+
+-static DEFINE_PER_CPU(cpumask_t, flush_tlb_mask);
+-
+ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared);
+ EXPORT_PER_CPU_SYMBOL(cpu_tlbstate_shared);
+
+@@ -889,36 +887,11 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask,
+ * up on the new contents of what used to be page tables, while
+ * doing a speculative memory access.
+ */
+- if (info->freed_tables) {
++ if (info->freed_tables)
+ on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true);
+- } else {
+- /*
+- * Although we could have used on_each_cpu_cond_mask(),
+- * open-coding it has performance advantages, as it eliminates
+- * the need for indirect calls or retpolines. In addition, it
+- * allows to use a designated cpumask for evaluating the
+- * condition, instead of allocating one.
+- *
+- * This code works under the assumption that there are no nested
+- * TLB flushes, an assumption that is already made in
+- * flush_tlb_mm_range().
+- *
+- * cond_cpumask is logically a stack-local variable, but it is
+- * more efficient to have it off the stack and not to allocate
+- * it on demand. Preemption is disabled and this code is
+- * non-reentrant.
+- */
+- struct cpumask *cond_cpumask = this_cpu_ptr(&flush_tlb_mask);
+- int cpu;
+-
+- cpumask_clear(cond_cpumask);
+-
+- for_each_cpu(cpu, cpumask) {
+- if (tlb_is_not_lazy(cpu))
+- __cpumask_set_cpu(cpu, cond_cpumask);
+- }
+- on_each_cpu_mask(cond_cpumask, flush_tlb_func, (void *)info, true);
+- }
++ else
++ on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func,
++ (void *)info, 1, cpumask);
+ }
+
+ void flush_tlb_multi(const struct cpumask *cpumask,
+diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
+index 9ea57389c554b..d99434dc215c2 100644
+--- a/arch/x86/net/bpf_jit_comp.c
++++ b/arch/x86/net/bpf_jit_comp.c
+@@ -721,6 +721,20 @@ static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)
+ *pprog = prog;
+ }
+
++/*
++ * Similar version of maybe_emit_mod() for a single register
++ */
++static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)
++{
++ u8 *prog = *pprog;
++
++ if (is64)
++ EMIT1(add_1mod(0x48, reg));
++ else if (is_ereg(reg))
++ EMIT1(add_1mod(0x40, reg));
++ *pprog = prog;
++}
++
+ /* LDX: dst_reg = *(u8*)(src_reg + off) */
+ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
+ {
+@@ -951,10 +965,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+ /* neg dst */
+ case BPF_ALU | BPF_NEG:
+ case BPF_ALU64 | BPF_NEG:
+- if (BPF_CLASS(insn->code) == BPF_ALU64)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_ALU64);
+ EMIT2(0xF7, add_1reg(0xD8, dst_reg));
+ break;
+
+@@ -968,10 +980,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+ case BPF_ALU64 | BPF_AND | BPF_K:
+ case BPF_ALU64 | BPF_OR | BPF_K:
+ case BPF_ALU64 | BPF_XOR | BPF_K:
+- if (BPF_CLASS(insn->code) == BPF_ALU64)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_ALU64);
+
+ /*
+ * b3 holds 'normal' opcode, b2 short form only valid
+@@ -1112,10 +1122,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+ case BPF_ALU64 | BPF_LSH | BPF_K:
+ case BPF_ALU64 | BPF_RSH | BPF_K:
+ case BPF_ALU64 | BPF_ARSH | BPF_K:
+- if (BPF_CLASS(insn->code) == BPF_ALU64)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_ALU64);
+
+ b3 = simple_alu_opcodes[BPF_OP(insn->code)];
+ if (imm32 == 1)
+@@ -1146,10 +1154,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+ }
+
+ /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
+- if (BPF_CLASS(insn->code) == BPF_ALU64)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_ALU64);
+
+ b3 = simple_alu_opcodes[BPF_OP(insn->code)];
+ EMIT2(0xD3, add_1reg(b3, dst_reg));
+@@ -1274,19 +1280,54 @@ st: if (is_imm8(insn->off))
+ case BPF_LDX | BPF_MEM | BPF_DW:
+ case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
+- /* test src_reg, src_reg */
+- maybe_emit_mod(&prog, src_reg, src_reg, true); /* always 1 byte */
+- EMIT2(0x85, add_2reg(0xC0, src_reg, src_reg));
+- /* jne start_of_ldx */
+- EMIT2(X86_JNE, 0);
++ /* Though the verifier prevents negative insn->off in BPF_PROBE_MEM
++ * add abs(insn->off) to the limit to make sure that negative
++ * offset won't be an issue.
++ * insn->off is s16, so it won't affect valid pointers.
++ */
++ u64 limit = TASK_SIZE_MAX + PAGE_SIZE + abs(insn->off);
++ u8 *end_of_jmp1, *end_of_jmp2;
++
++ /* Conservatively check that src_reg + insn->off is a kernel address:
++ * 1. src_reg + insn->off >= limit
++ * 2. src_reg + insn->off doesn't become small positive.
++ * Cannot do src_reg + insn->off >= limit in one branch,
++ * since it needs two spare registers, but JIT has only one.
++ */
++
++ /* movabsq r11, limit */
++ EMIT2(add_1mod(0x48, AUX_REG), add_1reg(0xB8, AUX_REG));
++ EMIT((u32)limit, 4);
++ EMIT(limit >> 32, 4);
++ /* cmp src_reg, r11 */
++ maybe_emit_mod(&prog, src_reg, AUX_REG, true);
++ EMIT2(0x39, add_2reg(0xC0, src_reg, AUX_REG));
++ /* if unsigned '<' goto end_of_jmp2 */
++ EMIT2(X86_JB, 0);
++ end_of_jmp1 = prog;
++
++ /* mov r11, src_reg */
++ emit_mov_reg(&prog, true, AUX_REG, src_reg);
++ /* add r11, insn->off */
++ maybe_emit_1mod(&prog, AUX_REG, true);
++ EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off);
++ /* jmp if not carry to start_of_ldx
++ * Otherwise ERR_PTR(-EINVAL) + 128 will be the user addr
++ * that has to be rejected.
++ */
++ EMIT2(0x73 /* JNC */, 0);
++ end_of_jmp2 = prog;
++
+ /* xor dst_reg, dst_reg */
+ emit_mov_imm32(&prog, false, dst_reg, 0);
+ /* jmp byte_after_ldx */
+ EMIT2(0xEB, 0);
+
+- /* populate jmp_offset for JNE above */
+- temp[4] = prog - temp - 5 /* sizeof(test + jne) */;
++ /* populate jmp_offset for JB above to jump to xor dst_reg */
++ end_of_jmp1[-1] = end_of_jmp2 - end_of_jmp1;
++ /* populate jmp_offset for JNC above to jump to start_of_ldx */
+ start_of_ldx = prog;
++ end_of_jmp2[-1] = start_of_ldx - end_of_jmp2;
+ }
+ emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
+@@ -1332,7 +1373,7 @@ st: if (is_imm8(insn->off))
+ * End result: x86 insn "mov rbx, qword ptr [rax+0x14]"
+ * of 4 bytes will be ignored and rbx will be zero inited.
+ */
+- ex->fixup = (prog - temp) | (reg2pt_regs[dst_reg] << 8);
++ ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8);
+ }
+ break;
+
+@@ -1459,10 +1500,8 @@ st: if (is_imm8(insn->off))
+ case BPF_JMP | BPF_JSET | BPF_K:
+ case BPF_JMP32 | BPF_JSET | BPF_K:
+ /* test dst_reg, imm32 */
+- if (BPF_CLASS(insn->code) == BPF_JMP)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_JMP);
+ EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
+ goto emit_cond_jmp;
+
+@@ -1495,10 +1534,8 @@ st: if (is_imm8(insn->off))
+ }
+
+ /* cmp dst_reg, imm8/32 */
+- if (BPF_CLASS(insn->code) == BPF_JMP)
+- EMIT1(add_1mod(0x48, dst_reg));
+- else if (is_ereg(dst_reg))
+- EMIT1(add_1mod(0x40, dst_reg));
++ maybe_emit_1mod(&prog, dst_reg,
++ BPF_CLASS(insn->code) == BPF_JMP);
+
+ if (is_imm8(imm32))
+ EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
+diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c
+index 2edd86649468f..615a76d700194 100644
+--- a/arch/x86/pci/fixup.c
++++ b/arch/x86/pci/fixup.c
+@@ -353,8 +353,8 @@ static void pci_fixup_video(struct pci_dev *pdev)
+ }
+ }
+ }
+-DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID,
+- PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_video);
++DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_ANY_ID, PCI_ANY_ID,
++ PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_video);
+
+
+ static const struct dmi_system_id msi_k8t_dmi_table[] = {
+diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
+index 5debe4ac6f819..f153e9ab8c966 100644
+--- a/arch/x86/pci/xen.c
++++ b/arch/x86/pci/xen.c
+@@ -472,7 +472,6 @@ static __init void xen_setup_pci_msi(void)
+ xen_msi_ops.setup_msi_irqs = xen_setup_msi_irqs;
+ }
+ xen_msi_ops.teardown_msi_irqs = xen_pv_teardown_msi_irqs;
+- pci_msi_ignore_mask = 1;
+ } else if (xen_hvm_domain()) {
+ xen_msi_ops.setup_msi_irqs = xen_hvm_setup_msi_irqs;
+ xen_msi_ops.teardown_msi_irqs = xen_teardown_msi_irqs;
+@@ -486,6 +485,11 @@ static __init void xen_setup_pci_msi(void)
+ * in allocating the native domain and never use it.
+ */
+ x86_init.irqs.create_pci_msi_domain = xen_create_pci_msi_domain;
++ /*
++ * With XEN PIRQ/Eventchannels in use PCI/MSI[-X] masking is solely
++ * controlled by the hypervisor.
++ */
++ pci_msi_ignore_mask = 1;
+ }
+
+ #else /* CONFIG_PCI_MSI */
+diff --git a/arch/x86/platform/efi/efi_stub_32.S b/arch/x86/platform/efi/efi_stub_32.S
+index 09ec84f6ef517..f3cfdb1c9a359 100644
+--- a/arch/x86/platform/efi/efi_stub_32.S
++++ b/arch/x86/platform/efi/efi_stub_32.S
+@@ -56,5 +56,5 @@ SYM_FUNC_START(efi_call_svam)
+
+ movl 16(%esp), %ebx
+ leave
+- ret
++ RET
+ SYM_FUNC_END(efi_call_svam)
+diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S
+index 90380a17ab238..2206b8bc47b8a 100644
+--- a/arch/x86/platform/efi/efi_stub_64.S
++++ b/arch/x86/platform/efi/efi_stub_64.S
+@@ -23,5 +23,5 @@ SYM_FUNC_START(__efi_call)
+ mov %rsi, %rcx
+ CALL_NOSPEC rdi
+ leave
+- ret
++ RET
+ SYM_FUNC_END(__efi_call)
+diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S
+index fd3dd1708eba5..f2a8eec69f8f6 100644
+--- a/arch/x86/platform/efi/efi_thunk_64.S
++++ b/arch/x86/platform/efi/efi_thunk_64.S
+@@ -63,7 +63,7 @@ SYM_CODE_START(__efi64_thunk)
+ 1: movq 24(%rsp), %rsp
+ pop %rbx
+ pop %rbp
+- retq
++ RET
+
+ .code32
+ 2: pushl $__KERNEL_CS
+diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c
+index b15ebfe40a73e..b0b848d6933af 100644
+--- a/arch/x86/platform/efi/quirks.c
++++ b/arch/x86/platform/efi/quirks.c
+@@ -277,7 +277,8 @@ void __init efi_arch_mem_reserve(phys_addr_t addr, u64 size)
+ return;
+ }
+
+- new = early_memremap(data.phys_map, data.size);
++ new = early_memremap_prot(data.phys_map, data.size,
++ pgprot_val(pgprot_encrypted(FIXMAP_PAGE_NORMAL)));
+ if (!new) {
+ pr_err("Failed to map new boot services memmap\n");
+ return;
+diff --git a/arch/x86/platform/olpc/xo1-wakeup.S b/arch/x86/platform/olpc/xo1-wakeup.S
+index 75f4faff84682..3a5abffe5660d 100644
+--- a/arch/x86/platform/olpc/xo1-wakeup.S
++++ b/arch/x86/platform/olpc/xo1-wakeup.S
+@@ -77,7 +77,7 @@ save_registers:
+ pushfl
+ popl saved_context_eflags
+
+- ret
++ RET
+
+ restore_registers:
+ movl saved_context_ebp, %ebp
+@@ -88,7 +88,7 @@ restore_registers:
+ pushl saved_context_eflags
+ popfl
+
+- ret
++ RET
+
+ SYM_CODE_START(do_olpc_suspend_lowlevel)
+ call save_processor_state
+@@ -109,7 +109,7 @@ ret_point:
+
+ call restore_registers
+ call restore_processor_state
+- ret
++ RET
+ SYM_CODE_END(do_olpc_suspend_lowlevel)
+
+ .data
+diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
+index 6665f88020983..732cb075d7072 100644
+--- a/arch/x86/power/cpu.c
++++ b/arch/x86/power/cpu.c
+@@ -25,6 +25,7 @@
+ #include <asm/cpu.h>
+ #include <asm/mmu_context.h>
+ #include <asm/cpu_device_id.h>
++#include <asm/microcode.h>
+
+ #ifdef CONFIG_X86_32
+ __visible unsigned long saved_context_ebx;
+@@ -40,7 +41,8 @@ static void msr_save_context(struct saved_context *ctxt)
+ struct saved_msr *end = msr + ctxt->saved_msrs.num;
+
+ while (msr < end) {
+- msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
++ if (msr->valid)
++ rdmsrl(msr->info.msr_no, msr->info.reg.q);
+ msr++;
+ }
+ }
+@@ -261,11 +263,18 @@ static void notrace __restore_processor_state(struct saved_context *ctxt)
+ x86_platform.restore_sched_clock_state();
+ mtrr_bp_restore();
+ perf_restore_debug_store();
+- msr_restore_context(ctxt);
+
+ c = &cpu_data(smp_processor_id());
+ if (cpu_has(c, X86_FEATURE_MSR_IA32_FEAT_CTL))
+ init_ia32_feat_ctl(c);
++
++ microcode_bsp_resume();
++
++ /*
++ * This needs to happen after the microcode has been updated upon resume
++ * because some of the MSRs are "emulated" in microcode.
++ */
++ msr_restore_context(ctxt);
+ }
+
+ /* Needed by apm.c */
+@@ -424,8 +433,10 @@ static int msr_build_context(const u32 *msr_id, const int num)
+ }
+
+ for (i = saved_msrs->num, j = 0; i < total_num; i++, j++) {
++ u64 dummy;
++
+ msr_array[i].info.msr_no = msr_id[j];
+- msr_array[i].valid = false;
++ msr_array[i].valid = !rdmsrl_safe(msr_id[j], &dummy);
+ msr_array[i].info.reg.q = 0;
+ }
+ saved_msrs->num = total_num;
+@@ -500,10 +511,24 @@ static int pm_cpu_check(const struct x86_cpu_id *c)
+ return ret;
+ }
+
++static void pm_save_spec_msr(void)
++{
++ u32 spec_msr_id[] = {
++ MSR_IA32_SPEC_CTRL,
++ MSR_IA32_TSX_CTRL,
++ MSR_TSX_FORCE_ABORT,
++ MSR_IA32_MCU_OPT_CTRL,
++ MSR_AMD64_LS_CFG,
++ };
++
++ msr_build_context(spec_msr_id, ARRAY_SIZE(spec_msr_id));
++}
++
+ static int pm_check_save_msr(void)
+ {
+ dmi_check_system(msr_save_dmi_table);
+ pm_cpu_check(msr_save_cpu_table);
++ pm_save_spec_msr();
+
+ return 0;
+ }
+diff --git a/arch/x86/power/hibernate_asm_32.S b/arch/x86/power/hibernate_asm_32.S
+index 8786653ad3c06..5606a15cf9a17 100644
+--- a/arch/x86/power/hibernate_asm_32.S
++++ b/arch/x86/power/hibernate_asm_32.S
+@@ -32,7 +32,7 @@ SYM_FUNC_START(swsusp_arch_suspend)
+ FRAME_BEGIN
+ call swsusp_save
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(swsusp_arch_suspend)
+
+ SYM_CODE_START(restore_image)
+@@ -108,5 +108,5 @@ SYM_FUNC_START(restore_registers)
+ /* tell the hibernation core that we've just restored the memory */
+ movl %eax, in_suspend
+
+- ret
++ RET
+ SYM_FUNC_END(restore_registers)
+diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S
+index d9bed596d849c..0a0539e1cc814 100644
+--- a/arch/x86/power/hibernate_asm_64.S
++++ b/arch/x86/power/hibernate_asm_64.S
+@@ -66,7 +66,7 @@ SYM_FUNC_START(restore_registers)
+ /* tell the hibernation core that we've just restored the memory */
+ movq %rax, in_suspend(%rip)
+
+- ret
++ RET
+ SYM_FUNC_END(restore_registers)
+
+ SYM_FUNC_START(swsusp_arch_suspend)
+@@ -96,7 +96,7 @@ SYM_FUNC_START(swsusp_arch_suspend)
+ FRAME_BEGIN
+ call swsusp_save
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(swsusp_arch_suspend)
+
+ SYM_FUNC_START(restore_image)
+diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
+index 31b5856010cba..1d20ed4b28729 100644
+--- a/arch/x86/realmode/init.c
++++ b/arch/x86/realmode/init.c
+@@ -17,6 +17,32 @@ u32 *trampoline_cr4_features;
+ /* Hold the pgd entry used on booting additional CPUs */
+ pgd_t trampoline_pgd_entry;
+
++void load_trampoline_pgtable(void)
++{
++#ifdef CONFIG_X86_32
++ load_cr3(initial_page_table);
++#else
++ /*
++ * This function is called before exiting to real-mode and that will
++ * fail with CR4.PCIDE still set.
++ */
++ if (boot_cpu_has(X86_FEATURE_PCID))
++ cr4_clear_bits(X86_CR4_PCIDE);
++
++ write_cr3(real_mode_header->trampoline_pgd);
++#endif
++
++ /*
++ * The CR3 write above will not flush global TLB entries.
++ * Stale, global entries from previous page tables may still be
++ * present. Flush those stale entries.
++ *
++ * This ensures that memory accessed while running with
++ * trampoline_pgd is *actually* mapped into trampoline_pgd.
++ */
++ __flush_tlb_all();
++}
++
+ void __init reserve_real_mode(void)
+ {
+ phys_addr_t mem;
+@@ -72,6 +98,7 @@ static void __init setup_real_mode(void)
+ #ifdef CONFIG_X86_64
+ u64 *trampoline_pgd;
+ u64 efer;
++ int i;
+ #endif
+
+ base = (unsigned char *)real_mode_header;
+@@ -128,8 +155,17 @@ static void __init setup_real_mode(void)
+ trampoline_header->flags = 0;
+
+ trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
++
++ /* Map the real mode stub as virtual == physical */
+ trampoline_pgd[0] = trampoline_pgd_entry.pgd;
+- trampoline_pgd[511] = init_top_pgt[511].pgd;
++
++ /*
++ * Include the entirety of the kernel mapping into the trampoline
++ * PGD. This way, all mappings present in the normal kernel page
++ * tables are usable while running on trampoline_pgd.
++ */
++ for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++)
++ trampoline_pgd[i] = init_top_pgt[i].pgd;
+ #endif
+
+ sme_sev_setup_real_mode(trampoline_header);
+diff --git a/arch/x86/um/checksum_32.S b/arch/x86/um/checksum_32.S
+index 13f118dec74f8..aed782ab77213 100644
+--- a/arch/x86/um/checksum_32.S
++++ b/arch/x86/um/checksum_32.S
+@@ -110,7 +110,7 @@ csum_partial:
+ 7:
+ popl %ebx
+ popl %esi
+- ret
++ RET
+
+ #else
+
+@@ -208,7 +208,7 @@ csum_partial:
+ 80:
+ popl %ebx
+ popl %esi
+- ret
++ RET
+
+ #endif
+ EXPORT_SYMBOL(csum_partial)
+diff --git a/arch/x86/um/setjmp_32.S b/arch/x86/um/setjmp_32.S
+index 62eaf8c80e041..2d991ddbcca57 100644
+--- a/arch/x86/um/setjmp_32.S
++++ b/arch/x86/um/setjmp_32.S
+@@ -34,7 +34,7 @@ kernel_setjmp:
+ movl %esi,12(%edx)
+ movl %edi,16(%edx)
+ movl %ecx,20(%edx) # Return address
+- ret
++ RET
+
+ .size kernel_setjmp,.-kernel_setjmp
+
+diff --git a/arch/x86/um/setjmp_64.S b/arch/x86/um/setjmp_64.S
+index 1b5d40d4ff46d..b46acb6a8ebd8 100644
+--- a/arch/x86/um/setjmp_64.S
++++ b/arch/x86/um/setjmp_64.S
+@@ -33,7 +33,7 @@ kernel_setjmp:
+ movq %r14,40(%rdi)
+ movq %r15,48(%rdi)
+ movq %rsi,56(%rdi) # Return address
+- ret
++ RET
+
+ .size kernel_setjmp,.-kernel_setjmp
+
+diff --git a/arch/x86/um/shared/sysdep/syscalls_64.h b/arch/x86/um/shared/sysdep/syscalls_64.h
+index 8a7d5e1da98e5..1e6875b4ffd83 100644
+--- a/arch/x86/um/shared/sysdep/syscalls_64.h
++++ b/arch/x86/um/shared/sysdep/syscalls_64.h
+@@ -10,13 +10,12 @@
+ #include <linux/msg.h>
+ #include <linux/shm.h>
+
+-typedef long syscall_handler_t(void);
++typedef long syscall_handler_t(long, long, long, long, long, long);
+
+ extern syscall_handler_t *sys_call_table[];
+
+ #define EXECUTE_SYSCALL(syscall, regs) \
+- (((long (*)(long, long, long, long, long, long)) \
+- (*sys_call_table[syscall]))(UPT_SYSCALL_ARG1(&regs->regs), \
++ (((*sys_call_table[syscall]))(UPT_SYSCALL_ARG1(&regs->regs), \
+ UPT_SYSCALL_ARG2(&regs->regs), \
+ UPT_SYSCALL_ARG3(&regs->regs), \
+ UPT_SYSCALL_ARG4(&regs->regs), \
+diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c
+index 58f51667e2e4b..8249685b40960 100644
+--- a/arch/x86/um/syscalls_64.c
++++ b/arch/x86/um/syscalls_64.c
+@@ -11,6 +11,7 @@
+ #include <linux/uaccess.h>
+ #include <asm/prctl.h> /* XXX This should get the constants from libc */
+ #include <os.h>
++#include <registers.h>
+
+ long arch_prctl(struct task_struct *task, int option,
+ unsigned long __user *arg2)
+@@ -35,7 +36,7 @@ long arch_prctl(struct task_struct *task, int option,
+ switch (option) {
+ case ARCH_SET_FS:
+ case ARCH_SET_GS:
+- ret = restore_registers(pid, &current->thread.regs.regs);
++ ret = restore_pid_registers(pid, &current->thread.regs.regs);
+ if (ret)
+ return ret;
+ break;
+diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
+index a7b7d674f5005..133ef31639df1 100644
+--- a/arch/x86/xen/enlighten_pv.c
++++ b/arch/x86/xen/enlighten_pv.c
+@@ -1364,10 +1364,6 @@ asmlinkage __visible void __init xen_start_kernel(void)
+
+ xen_acpi_sleep_register();
+
+- /* Avoid searching for BIOS MP tables */
+- x86_init.mpparse.find_smp_config = x86_init_noop;
+- x86_init.mpparse.get_smp_config = x86_init_uint_noop;
+-
+ xen_boot_params_init_edd();
+
+ #ifdef CONFIG_ACPI
+diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
+index e13b0b49fcdfc..d7249f4c90f1b 100644
+--- a/arch/x86/xen/pmu.c
++++ b/arch/x86/xen/pmu.c
+@@ -512,10 +512,7 @@ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
+ return ret;
+ }
+
+-bool is_xen_pmu(int cpu)
+-{
+- return (get_xenpmu_data() != NULL);
+-}
++bool is_xen_pmu;
+
+ void xen_pmu_init(int cpu)
+ {
+@@ -526,7 +523,7 @@ void xen_pmu_init(int cpu)
+
+ BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
+
+- if (xen_hvm_domain())
++ if (xen_hvm_domain() || (cpu != 0 && !is_xen_pmu))
+ return;
+
+ xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
+@@ -547,7 +544,8 @@ void xen_pmu_init(int cpu)
+ per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
+ per_cpu(xenpmu_shared, cpu).flags = 0;
+
+- if (cpu == 0) {
++ if (!is_xen_pmu) {
++ is_xen_pmu = true;
+ perf_register_guest_info_callbacks(&xen_guest_cbs);
+ xen_pmu_arch_init();
+ }
+diff --git a/arch/x86/xen/pmu.h b/arch/x86/xen/pmu.h
+index 0e83a160589bc..65c58894fc79f 100644
+--- a/arch/x86/xen/pmu.h
++++ b/arch/x86/xen/pmu.h
+@@ -4,6 +4,8 @@
+
+ #include <xen/interface/xenpmu.h>
+
++extern bool is_xen_pmu;
++
+ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id);
+ #ifdef CONFIG_XEN_HAVE_VPMU
+ void xen_pmu_init(int cpu);
+@@ -12,7 +14,6 @@ void xen_pmu_finish(int cpu);
+ static inline void xen_pmu_init(int cpu) {}
+ static inline void xen_pmu_finish(int cpu) {}
+ #endif
+-bool is_xen_pmu(int cpu);
+ bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err);
+ bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err);
+ int pmu_apic_update(uint32_t reg);
+diff --git a/arch/x86/xen/smp_hvm.c b/arch/x86/xen/smp_hvm.c
+index 6ff3c887e0b99..b70afdff419ca 100644
+--- a/arch/x86/xen/smp_hvm.c
++++ b/arch/x86/xen/smp_hvm.c
+@@ -19,6 +19,12 @@ static void __init xen_hvm_smp_prepare_boot_cpu(void)
+ */
+ xen_vcpu_setup(0);
+
++ /*
++ * Called again in case the kernel boots on vcpu >= MAX_VIRT_CPUS.
++ * Refer to comments in xen_hvm_init_time_ops().
++ */
++ xen_hvm_init_time_ops();
++
+ /*
+ * The alternative logic (which patches the unlock/lock) runs before
+ * the smp bootup up code is activated. Hence we need to set this up
+diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
+index 7ed56c6075b0c..cd5539fc5eb45 100644
+--- a/arch/x86/xen/smp_pv.c
++++ b/arch/x86/xen/smp_pv.c
+@@ -129,7 +129,7 @@ int xen_smp_intr_init_pv(unsigned int cpu)
+ per_cpu(xen_irq_work, cpu).irq = rc;
+ per_cpu(xen_irq_work, cpu).name = callfunc_name;
+
+- if (is_xen_pmu(cpu)) {
++ if (is_xen_pmu) {
+ pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
+ rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
+ xen_pmu_irq_handler,
+@@ -148,28 +148,12 @@ int xen_smp_intr_init_pv(unsigned int cpu)
+ return rc;
+ }
+
+-static void __init xen_fill_possible_map(void)
+-{
+- int i, rc;
+-
+- if (xen_initial_domain())
+- return;
+-
+- for (i = 0; i < nr_cpu_ids; i++) {
+- rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
+- if (rc >= 0) {
+- num_processors++;
+- set_cpu_possible(i, true);
+- }
+- }
+-}
+-
+-static void __init xen_filter_cpu_maps(void)
++static void __init _get_smp_config(unsigned int early)
+ {
+ int i, rc;
+ unsigned int subtract = 0;
+
+- if (!xen_initial_domain())
++ if (early)
+ return;
+
+ num_processors = 0;
+@@ -210,7 +194,6 @@ static void __init xen_pv_smp_prepare_boot_cpu(void)
+ * sure the old memory can be recycled. */
+ make_lowmem_page_readwrite(xen_initial_gdt);
+
+- xen_filter_cpu_maps();
+ xen_setup_vcpu_info_placement();
+
+ /*
+@@ -486,5 +469,8 @@ static const struct smp_ops xen_smp_ops __initconst = {
+ void __init xen_smp_init(void)
+ {
+ smp_ops = xen_smp_ops;
+- xen_fill_possible_map();
++
++ /* Avoid searching for BIOS MP tables */
++ x86_init.mpparse.find_smp_config = x86_init_noop;
++ x86_init.mpparse.get_smp_config = _get_smp_config;
+ }
+diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
+index d9c945ee11008..9ef0a5cca96ee 100644
+--- a/arch/x86/xen/time.c
++++ b/arch/x86/xen/time.c
+@@ -558,6 +558,11 @@ static void xen_hvm_setup_cpu_clockevents(void)
+
+ void __init xen_hvm_init_time_ops(void)
+ {
++ static bool hvm_time_initialized;
++
++ if (hvm_time_initialized)
++ return;
++
+ /*
+ * vector callback is needed otherwise we cannot receive interrupts
+ * on cpu > 0 and at this point we don't know how many cpus are
+@@ -567,7 +572,22 @@ void __init xen_hvm_init_time_ops(void)
+ return;
+
+ if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
+- pr_info("Xen doesn't support pvclock on HVM, disable pv timer");
++ pr_info_once("Xen doesn't support pvclock on HVM, disable pv timer");
++ return;
++ }
++
++ /*
++ * Only MAX_VIRT_CPUS 'vcpu_info' are embedded inside 'shared_info'.
++ * The __this_cpu_read(xen_vcpu) is still NULL when Xen HVM guest
++ * boots on vcpu >= MAX_VIRT_CPUS (e.g., kexec), To access
++ * __this_cpu_read(xen_vcpu) via xen_clocksource_read() will panic.
++ *
++ * The xen_hvm_init_time_ops() should be called again later after
++ * __this_cpu_read(xen_vcpu) is available.
++ */
++ if (!__this_cpu_read(xen_vcpu)) {
++ pr_info("Delay xen_init_time_common() as kernel is running on vcpu=%d\n",
++ xen_vcpu_nr(0));
+ return;
+ }
+
+@@ -577,6 +597,8 @@ void __init xen_hvm_init_time_ops(void)
+ x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
+
+ x86_platform.set_wallclock = xen_set_wallclock;
++
++ hvm_time_initialized = true;
+ }
+ #endif
+
+diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S
+index 1e626444712be..962d30ea01a22 100644
+--- a/arch/x86/xen/xen-asm.S
++++ b/arch/x86/xen/xen-asm.S
+@@ -20,6 +20,7 @@
+
+ #include <linux/init.h>
+ #include <linux/linkage.h>
++#include <../entry/calling.h>
+
+ /*
+ * Enable events. This clears the event mask and tests the pending
+@@ -44,7 +45,7 @@ SYM_FUNC_START(xen_irq_enable_direct)
+ call check_events
+ 1:
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(xen_irq_enable_direct)
+
+
+@@ -54,7 +55,7 @@ SYM_FUNC_END(xen_irq_enable_direct)
+ */
+ SYM_FUNC_START(xen_irq_disable_direct)
+ movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+- ret
++ RET
+ SYM_FUNC_END(xen_irq_disable_direct)
+
+ /*
+@@ -70,7 +71,7 @@ SYM_FUNC_START(xen_save_fl_direct)
+ testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+ setz %ah
+ addb %ah, %ah
+- ret
++ RET
+ SYM_FUNC_END(xen_save_fl_direct)
+
+ /*
+@@ -99,7 +100,7 @@ SYM_FUNC_START(check_events)
+ pop %rcx
+ pop %rax
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(check_events)
+
+ SYM_FUNC_START(xen_read_cr2)
+@@ -107,14 +108,14 @@ SYM_FUNC_START(xen_read_cr2)
+ _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
+ _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(xen_read_cr2);
+
+ SYM_FUNC_START(xen_read_cr2_direct)
+ FRAME_BEGIN
+ _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
+ FRAME_END
+- ret
++ RET
+ SYM_FUNC_END(xen_read_cr2_direct);
+
+ .macro xen_pv_trap name
+@@ -191,6 +192,25 @@ SYM_CODE_START(xen_iret)
+ jmp hypercall_iret
+ SYM_CODE_END(xen_iret)
+
++/*
++ * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
++ * also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
++ * in XEN pv would cause %rsp to move up to the top of the kernel stack and
++ * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
++ * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
++ * frame at the same address is useless.
++ */
++SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
++ UNWIND_HINT_REGS
++ POP_REGS
++
++ /* stackleak_erase() can work safely on the kernel stack. */
++ STACKLEAK_ERASE_NOCLOBBER
++
++ addq $8, %rsp /* skip regs->orig_ax */
++ jmp xen_iret
++SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
++
+ /*
+ * Xen handles syscall callbacks much like ordinary exceptions, which
+ * means we have:
+diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
+index cb6538ae2fe07..565062932ef11 100644
+--- a/arch/x86/xen/xen-head.S
++++ b/arch/x86/xen/xen-head.S
+@@ -70,7 +70,7 @@ SYM_CODE_START(hypercall_page)
+ .rept (PAGE_SIZE / 32)
+ UNWIND_HINT_FUNC
+ .skip 31, 0x90
+- ret
++ RET
+ .endr
+
+ #define HYPERCALL(n) \
+diff --git a/arch/xtensa/boot/dts/xtfpga-flash-128m.dtsi b/arch/xtensa/boot/dts/xtfpga-flash-128m.dtsi
+index 9bf8bad1dd18a..c33932568aa73 100644
+--- a/arch/xtensa/boot/dts/xtfpga-flash-128m.dtsi
++++ b/arch/xtensa/boot/dts/xtfpga-flash-128m.dtsi
+@@ -8,19 +8,19 @@
+ reg = <0x00000000 0x08000000>;
+ bank-width = <2>;
+ device-width = <2>;
+- partition@0x0 {
++ partition@0 {
+ label = "data";
+ reg = <0x00000000 0x06000000>;
+ };
+- partition@0x6000000 {
++ partition@6000000 {
+ label = "boot loader area";
+ reg = <0x06000000 0x00800000>;
+ };
+- partition@0x6800000 {
++ partition@6800000 {
+ label = "kernel image";
+ reg = <0x06800000 0x017e0000>;
+ };
+- partition@0x7fe0000 {
++ partition@7fe0000 {
+ label = "boot environment";
+ reg = <0x07fe0000 0x00020000>;
+ };
+diff --git a/arch/xtensa/boot/dts/xtfpga-flash-16m.dtsi b/arch/xtensa/boot/dts/xtfpga-flash-16m.dtsi
+index 40c2f81f7cb66..7bde2ab2d6fb5 100644
+--- a/arch/xtensa/boot/dts/xtfpga-flash-16m.dtsi
++++ b/arch/xtensa/boot/dts/xtfpga-flash-16m.dtsi
+@@ -8,19 +8,19 @@
+ reg = <0x08000000 0x01000000>;
+ bank-width = <2>;
+ device-width = <2>;
+- partition@0x0 {
++ partition@0 {
+ label = "boot loader area";
+ reg = <0x00000000 0x00400000>;
+ };
+- partition@0x400000 {
++ partition@400000 {
+ label = "kernel image";
+ reg = <0x00400000 0x00600000>;
+ };
+- partition@0xa00000 {
++ partition@a00000 {
+ label = "data";
+ reg = <0x00a00000 0x005e0000>;
+ };
+- partition@0xfe0000 {
++ partition@fe0000 {
+ label = "boot environment";
+ reg = <0x00fe0000 0x00020000>;
+ };
+diff --git a/arch/xtensa/boot/dts/xtfpga-flash-4m.dtsi b/arch/xtensa/boot/dts/xtfpga-flash-4m.dtsi
+index fb8d3a9f33c23..0655b868749a4 100644
+--- a/arch/xtensa/boot/dts/xtfpga-flash-4m.dtsi
++++ b/arch/xtensa/boot/dts/xtfpga-flash-4m.dtsi
+@@ -8,11 +8,11 @@
+ reg = <0x08000000 0x00400000>;
+ bank-width = <2>;
+ device-width = <2>;
+- partition@0x0 {
++ partition@0 {
+ label = "boot loader area";
+ reg = <0x00000000 0x003f0000>;
+ };
+- partition@0x3f0000 {
++ partition@3f0000 {
+ label = "boot environment";
+ reg = <0x003f0000 0x00010000>;
+ };
+diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
+index bd5aeb7955675..a63eca1266577 100644
+--- a/arch/xtensa/include/asm/pgtable.h
++++ b/arch/xtensa/include/asm/pgtable.h
+@@ -411,6 +411,10 @@ extern void update_mmu_cache(struct vm_area_struct * vma,
+
+ typedef pte_t *pte_addr_t;
+
++void update_mmu_tlb(struct vm_area_struct *vma,
++ unsigned long address, pte_t *ptep);
++#define __HAVE_ARCH_UPDATE_MMU_TLB
++
+ #endif /* !defined (__ASSEMBLY__) */
+
+ #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h
+index 7f63aca6a0d34..9dd4efe1bf0bd 100644
+--- a/arch/xtensa/include/asm/processor.h
++++ b/arch/xtensa/include/asm/processor.h
+@@ -226,8 +226,8 @@ extern unsigned long get_wchan(struct task_struct *p);
+
+ #define xtensa_set_sr(x, sr) \
+ ({ \
+- unsigned int v = (unsigned int)(x); \
+- __asm__ __volatile__ ("wsr %0, "__stringify(sr) :: "a"(v)); \
++ __asm__ __volatile__ ("wsr %0, "__stringify(sr) :: \
++ "a"((unsigned int)(x))); \
+ })
+
+ #define xtensa_get_sr(sr) \
+diff --git a/arch/xtensa/include/asm/timex.h b/arch/xtensa/include/asm/timex.h
+index 233ec75e60c69..3f2462f2d0270 100644
+--- a/arch/xtensa/include/asm/timex.h
++++ b/arch/xtensa/include/asm/timex.h
+@@ -29,10 +29,6 @@
+
+ extern unsigned long ccount_freq;
+
+-typedef unsigned long long cycles_t;
+-
+-#define get_cycles() (0)
+-
+ void local_timer_setup(unsigned cpu);
+
+ /*
+@@ -59,4 +55,6 @@ static inline void set_linux_timer (unsigned long ccompare)
+ xtensa_set_sr(ccompare, SREG_CCOMPARE + LINUX_TIMER);
+ }
+
++#include <asm-generic/timex.h>
++
+ #endif /* _XTENSA_TIMEX_H */
+diff --git a/arch/xtensa/kernel/coprocessor.S b/arch/xtensa/kernel/coprocessor.S
+index 45cc0ae0af6f9..c7b9f12896f20 100644
+--- a/arch/xtensa/kernel/coprocessor.S
++++ b/arch/xtensa/kernel/coprocessor.S
+@@ -29,7 +29,7 @@
+ .if XTENSA_HAVE_COPROCESSOR(x); \
+ .align 4; \
+ .Lsave_cp_regs_cp##x: \
+- xchal_cp##x##_store a2 a4 a5 a6 a7; \
++ xchal_cp##x##_store a2 a3 a4 a5 a6; \
+ jx a0; \
+ .endif
+
+@@ -46,7 +46,7 @@
+ .if XTENSA_HAVE_COPROCESSOR(x); \
+ .align 4; \
+ .Lload_cp_regs_cp##x: \
+- xchal_cp##x##_load a2 a4 a5 a6 a7; \
++ xchal_cp##x##_load a2 a3 a4 a5 a6; \
+ jx a0; \
+ .endif
+
+diff --git a/arch/xtensa/kernel/jump_label.c b/arch/xtensa/kernel/jump_label.c
+index 61cf6497a646b..ad1841cecdfb7 100644
+--- a/arch/xtensa/kernel/jump_label.c
++++ b/arch/xtensa/kernel/jump_label.c
+@@ -40,7 +40,7 @@ static int patch_text_stop_machine(void *data)
+ {
+ struct patch *patch = data;
+
+- if (atomic_inc_return(&patch->cpu_count) == 1) {
++ if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
+ local_patch_text(patch->addr, patch->data, patch->sz);
+ atomic_inc(&patch->cpu_count);
+ } else {
+@@ -61,7 +61,7 @@ static void patch_text(unsigned long addr, const void *data, size_t sz)
+ .data = data,
+ };
+ stop_machine_cpuslocked(patch_text_stop_machine,
+- &patch, NULL);
++ &patch, cpu_online_mask);
+ } else {
+ unsigned long flags;
+
+diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c
+index f436cf2efd8b7..27a477dae2322 100644
+--- a/arch/xtensa/mm/tlb.c
++++ b/arch/xtensa/mm/tlb.c
+@@ -162,6 +162,12 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
+ }
+ }
+
++void update_mmu_tlb(struct vm_area_struct *vma,
++ unsigned long address, pte_t *ptep)
++{
++ local_flush_tlb_page(vma, address);
++}
++
+ #ifdef CONFIG_DEBUG_TLB_SANITY
+
+ static unsigned get_pte_for_vaddr(unsigned vaddr)
+diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c
+index 81d7c7e8f7e96..10b79d3c74e07 100644
+--- a/arch/xtensa/platforms/iss/console.c
++++ b/arch/xtensa/platforms/iss/console.c
+@@ -36,24 +36,19 @@ static void rs_poll(struct timer_list *);
+ static struct tty_driver *serial_driver;
+ static struct tty_port serial_port;
+ static DEFINE_TIMER(serial_timer, rs_poll);
+-static DEFINE_SPINLOCK(timer_lock);
+
+ static int rs_open(struct tty_struct *tty, struct file * filp)
+ {
+- spin_lock_bh(&timer_lock);
+ if (tty->count == 1)
+ mod_timer(&serial_timer, jiffies + SERIAL_TIMER_VALUE);
+- spin_unlock_bh(&timer_lock);
+
+ return 0;
+ }
+
+ static void rs_close(struct tty_struct *tty, struct file * filp)
+ {
+- spin_lock_bh(&timer_lock);
+ if (tty->count == 1)
+ del_timer_sync(&serial_timer);
+- spin_unlock_bh(&timer_lock);
+ }
+
+
+@@ -73,8 +68,6 @@ static void rs_poll(struct timer_list *unused)
+ int rd = 1;
+ unsigned char c;
+
+- spin_lock(&timer_lock);
+-
+ while (simc_poll(0)) {
+ rd = simc_read(0, &c, 1);
+ if (rd <= 0)
+@@ -87,7 +80,6 @@ static void rs_poll(struct timer_list *unused)
+ tty_flip_buffer_push(port);
+ if (rd)
+ mod_timer(&serial_timer, jiffies + SERIAL_TIMER_VALUE);
+- spin_unlock(&timer_lock);
+ }
+
+
+diff --git a/block/bdev.c b/block/bdev.c
+index 485a258b0ab37..18abafb135e0b 100644
+--- a/block/bdev.c
++++ b/block/bdev.c
+@@ -184,14 +184,13 @@ int sb_min_blocksize(struct super_block *sb, int size)
+
+ EXPORT_SYMBOL(sb_min_blocksize);
+
+-int __sync_blockdev(struct block_device *bdev, int wait)
++int sync_blockdev_nowait(struct block_device *bdev)
+ {
+ if (!bdev)
+ return 0;
+- if (!wait)
+- return filemap_flush(bdev->bd_inode->i_mapping);
+- return filemap_write_and_wait(bdev->bd_inode->i_mapping);
++ return filemap_flush(bdev->bd_inode->i_mapping);
+ }
++EXPORT_SYMBOL_GPL(sync_blockdev_nowait);
+
+ /*
+ * Write out and wait upon all the dirty data associated with a block
+@@ -199,7 +198,9 @@ int __sync_blockdev(struct block_device *bdev, int wait)
+ */
+ int sync_blockdev(struct block_device *bdev)
+ {
+- return __sync_blockdev(bdev, 1);
++ if (!bdev)
++ return 0;
++ return filemap_write_and_wait(bdev->bd_inode->i_mapping);
+ }
+ EXPORT_SYMBOL(sync_blockdev);
+
+@@ -1016,7 +1017,7 @@ int __invalidate_device(struct block_device *bdev, bool kill_dirty)
+ }
+ EXPORT_SYMBOL(__invalidate_device);
+
+-void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
++void sync_bdevs(bool wait)
+ {
+ struct inode *inode, *old_inode = NULL;
+
+@@ -1047,8 +1048,19 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
+ bdev = I_BDEV(inode);
+
+ mutex_lock(&bdev->bd_disk->open_mutex);
+- if (bdev->bd_openers)
+- func(bdev, arg);
++ if (!bdev->bd_openers) {
++ ; /* skip */
++ } else if (wait) {
++ /*
++ * We keep the error status of individual mapping so
++ * that applications can catch the writeback error using
++ * fsync(2). See filemap_fdatawait_keep_errors() for
++ * details.
++ */
++ filemap_fdatawait_keep_errors(inode->i_mapping);
++ } else {
++ filemap_fdatawrite(inode->i_mapping);
++ }
+ mutex_unlock(&bdev->bd_disk->open_mutex);
+
+ spin_lock(&blockdev_superblock->s_inode_list_lock);
+diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
+index 85b8e1c3a762d..e37af3f8a733d 100644
+--- a/block/bfq-cgroup.c
++++ b/block/bfq-cgroup.c
+@@ -644,6 +644,12 @@ void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ {
+ struct bfq_entity *entity = &bfqq->entity;
+
++ /*
++ * oom_bfqq is not allowed to move, oom_bfqq will hold ref to root_group
++ * until elevator exit.
++ */
++ if (bfqq == &bfqd->oom_bfqq)
++ return;
+ /*
+ * Get extra reference to prevent bfqq from being freed in
+ * next possible expire or deactivate.
+diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
+index 480e1a1348596..63d2d66dece56 100644
+--- a/block/bfq-iosched.c
++++ b/block/bfq-iosched.c
+@@ -2662,6 +2662,15 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
+ * are likely to increase the throughput.
+ */
+ bfqq->new_bfqq = new_bfqq;
++ /*
++ * The above assignment schedules the following redirections:
++ * each time some I/O for bfqq arrives, the process that
++ * generated that I/O is disassociated from bfqq and
++ * associated with new_bfqq. Here we increases new_bfqq->ref
++ * in advance, adding the number of processes that are
++ * expected to be associated with new_bfqq as they happen to
++ * issue I/O.
++ */
+ new_bfqq->ref += process_refs;
+ return new_bfqq;
+ }
+@@ -2724,6 +2733,10 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ {
+ struct bfq_queue *in_service_bfqq, *new_bfqq;
+
++ /* if a merge has already been setup, then proceed with that first */
++ if (bfqq->new_bfqq)
++ return bfqq->new_bfqq;
++
+ /*
+ * Check delayed stable merge for rotational or non-queueing
+ * devs. For this branch to be executed, bfqq must not be
+@@ -2825,9 +2838,6 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+ if (bfq_too_late_for_merging(bfqq))
+ return NULL;
+
+- if (bfqq->new_bfqq)
+- return bfqq->new_bfqq;
+-
+ if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq))
+ return NULL;
+
+@@ -5061,7 +5071,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
+ struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
+ struct request *rq;
+ struct bfq_queue *in_serv_queue;
+- bool waiting_rq, idle_timer_disabled;
++ bool waiting_rq, idle_timer_disabled = false;
+
+ spin_lock_irq(&bfqd->lock);
+
+@@ -5069,14 +5079,15 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
+ waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue);
+
+ rq = __bfq_dispatch_request(hctx);
+-
+- idle_timer_disabled =
+- waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
++ if (in_serv_queue == bfqd->in_service_queue) {
++ idle_timer_disabled =
++ waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
++ }
+
+ spin_unlock_irq(&bfqd->lock);
+-
+- bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue,
+- idle_timer_disabled);
++ bfq_update_dispatch_stats(hctx->queue, rq,
++ idle_timer_disabled ? in_serv_queue : NULL,
++ idle_timer_disabled);
+
+ return rq;
+ }
+@@ -5991,48 +6002,7 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
+
+ spin_lock_irq(&bfqd->lock);
+ bfqq = bfq_init_rq(rq);
+-
+- /*
+- * Reqs with at_head or passthrough flags set are to be put
+- * directly into dispatch list. Additional case for putting rq
+- * directly into the dispatch queue: the only active
+- * bfq_queues are bfqq and either its waker bfq_queue or one
+- * of its woken bfq_queues. The rationale behind this
+- * additional condition is as follows:
+- * - consider a bfq_queue, say Q1, detected as a waker of
+- * another bfq_queue, say Q2
+- * - by definition of a waker, Q1 blocks the I/O of Q2, i.e.,
+- * some I/O of Q1 needs to be completed for new I/O of Q2
+- * to arrive. A notable example of waker is journald
+- * - so, Q1 and Q2 are in any respect the queues of two
+- * cooperating processes (or of two cooperating sets of
+- * processes): the goal of Q1's I/O is doing what needs to
+- * be done so that new Q2's I/O can finally be
+- * issued. Therefore, if the service of Q1's I/O is delayed,
+- * then Q2's I/O is delayed too. Conversely, if Q2's I/O is
+- * delayed, the goal of Q1's I/O is hindered.
+- * - as a consequence, if some I/O of Q1/Q2 arrives while
+- * Q2/Q1 is the only queue in service, there is absolutely
+- * no point in delaying the service of such an I/O. The
+- * only possible result is a throughput loss
+- * - so, when the above condition holds, the best option is to
+- * have the new I/O dispatched as soon as possible
+- * - the most effective and efficient way to attain the above
+- * goal is to put the new I/O directly in the dispatch
+- * list
+- * - as an additional restriction, Q1 and Q2 must be the only
+- * busy queues for this commit to put the I/O of Q2/Q1 in
+- * the dispatch list. This is necessary, because, if also
+- * other queues are waiting for service, then putting new
+- * I/O directly in the dispatch list may evidently cause a
+- * violation of service guarantees for the other queues
+- */
+- if (!bfqq ||
+- (bfqq != bfqd->in_service_queue &&
+- bfqd->in_service_queue != NULL &&
+- bfq_tot_busy_queues(bfqd) == 1 + bfq_bfqq_busy(bfqq) &&
+- (bfqq->waker_bfqq == bfqd->in_service_queue ||
+- bfqd->in_service_queue->waker_bfqq == bfqq)) || at_head) {
++ if (!bfqq || at_head) {
+ if (at_head)
+ list_add(&rq->queuelist, &bfqd->dispatch);
+ else
+@@ -6059,7 +6029,6 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
+ * merge).
+ */
+ cmd_flags = rq->cmd_flags;
+-
+ spin_unlock_irq(&bfqd->lock);
+
+ bfq_update_insert_stats(q, bfqq, idle_timer_disabled,
+@@ -6920,6 +6889,8 @@ static void bfq_exit_queue(struct elevator_queue *e)
+ spin_unlock_irq(&bfqd->lock);
+ #endif
+
++ wbt_enable_default(bfqd->queue);
++
+ kfree(bfqd);
+ }
+
+diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
+index b74cc0da118ec..709b901de3ca9 100644
+--- a/block/bfq-wf2q.c
++++ b/block/bfq-wf2q.c
+@@ -519,7 +519,7 @@ unsigned short bfq_ioprio_to_weight(int ioprio)
+ static unsigned short bfq_weight_to_ioprio(int weight)
+ {
+ return max_t(int, 0,
+- IOPRIO_NR_LEVELS * BFQ_WEIGHT_CONVERSION_COEFF - weight);
++ IOPRIO_NR_LEVELS - weight / BFQ_WEIGHT_CONVERSION_COEFF);
+ }
+
+ static void bfq_get_entity(struct bfq_entity *entity)
+diff --git a/block/bio-integrity.c b/block/bio-integrity.c
+index 6b47cddbbca17..4a7c33ed9a661 100644
+--- a/block/bio-integrity.c
++++ b/block/bio-integrity.c
+@@ -373,7 +373,7 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
+ unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
+
+- bip->bip_iter.bi_sector += bytes_done >> 9;
++ bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9);
+ bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
+ }
+
+diff --git a/block/bio.c b/block/bio.c
+index a6fb6a0b42955..8906c9856a7de 100644
+--- a/block/bio.c
++++ b/block/bio.c
+@@ -567,7 +567,8 @@ void bio_truncate(struct bio *bio, unsigned new_size)
+ offset = new_size - done;
+ else
+ offset = 0;
+- zero_user(bv.bv_page, offset, bv.bv_len - offset);
++ zero_user(bv.bv_page, bv.bv_offset + offset,
++ bv.bv_len - offset);
+ truncated = true;
+ }
+ done += bv.bv_len;
+@@ -1551,7 +1552,7 @@ EXPORT_SYMBOL(bio_split);
+ void bio_trim(struct bio *bio, sector_t offset, sector_t size)
+ {
+ if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS ||
+- offset + size > bio->bi_iter.bi_size))
++ offset + size > bio_sectors(bio)))
+ return;
+
+ size <<= 9;
+diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
+index 9a1c5839dd469..07a2524e6efdc 100644
+--- a/block/blk-cgroup.c
++++ b/block/blk-cgroup.c
+@@ -633,6 +633,14 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
+
+ q = bdev->bd_disk->queue;
+
++ /*
++ * blkcg_deactivate_policy() requires queue to be frozen, we can grab
++ * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
++ */
++ ret = blk_queue_enter(q, 0);
++ if (ret)
++ goto fail;
++
+ rcu_read_lock();
+ spin_lock_irq(&q->queue_lock);
+
+@@ -667,13 +675,13 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
+ new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
+ if (unlikely(!new_blkg)) {
+ ret = -ENOMEM;
+- goto fail;
++ goto fail_exit_queue;
+ }
+
+ if (radix_tree_preload(GFP_KERNEL)) {
+ blkg_free(new_blkg);
+ ret = -ENOMEM;
+- goto fail;
++ goto fail_exit_queue;
+ }
+
+ rcu_read_lock();
+@@ -702,6 +710,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
+ goto success;
+ }
+ success:
++ blk_queue_exit(q);
+ ctx->bdev = bdev;
+ ctx->blkg = blkg;
+ ctx->body = input;
+@@ -712,6 +721,8 @@ fail_preloaded:
+ fail_unlock:
+ spin_unlock_irq(&q->queue_lock);
+ rcu_read_unlock();
++fail_exit_queue:
++ blk_queue_exit(q);
+ fail:
+ blkdev_put_no_open(bdev);
+ /*
+@@ -844,11 +855,11 @@ static void blkcg_fill_root_iostats(void)
+ blk_queue_root_blkg(bdev->bd_disk->queue);
+ struct blkg_iostat tmp;
+ int cpu;
++ unsigned long flags;
+
+ memset(&tmp, 0, sizeof(tmp));
+ for_each_possible_cpu(cpu) {
+ struct disk_stats *cpu_dkstats;
+- unsigned long flags;
+
+ cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
+ tmp.ios[BLKG_IOSTAT_READ] +=
+@@ -864,11 +875,11 @@ static void blkcg_fill_root_iostats(void)
+ cpu_dkstats->sectors[STAT_WRITE] << 9;
+ tmp.bytes[BLKG_IOSTAT_DISCARD] +=
+ cpu_dkstats->sectors[STAT_DISCARD] << 9;
+-
+- flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+- blkg_iostat_set(&blkg->iostat.cur, &tmp);
+- u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
+ }
++
++ flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
++ blkg_iostat_set(&blkg->iostat.cur, &tmp);
++ u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
+ }
+ }
+
+diff --git a/block/blk-core.c b/block/blk-core.c
+index 4d8f5fe915887..5009b9f1c3c9d 100644
+--- a/block/blk-core.c
++++ b/block/blk-core.c
+@@ -49,6 +49,7 @@
+ #include "blk-mq.h"
+ #include "blk-mq-sched.h"
+ #include "blk-pm.h"
++#include "blk-rq-qos.h"
+
+ struct dentry *blk_debugfs_root;
+
+@@ -350,13 +351,6 @@ void blk_queue_start_drain(struct request_queue *q)
+ wake_up_all(&q->mq_freeze_wq);
+ }
+
+-void blk_set_queue_dying(struct request_queue *q)
+-{
+- blk_queue_flag_set(QUEUE_FLAG_DYING, q);
+- blk_queue_start_drain(q);
+-}
+-EXPORT_SYMBOL_GPL(blk_set_queue_dying);
+-
+ /**
+ * blk_cleanup_queue - shutdown a request queue
+ * @q: request queue to shutdown
+@@ -374,7 +368,8 @@ void blk_cleanup_queue(struct request_queue *q)
+ WARN_ON_ONCE(blk_queue_registered(q));
+
+ /* mark @q DYING, no new request or merges will be allowed afterwards */
+- blk_set_queue_dying(q);
++ blk_queue_flag_set(QUEUE_FLAG_DYING, q);
++ blk_queue_start_drain(q);
+
+ blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
+ blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
+@@ -386,11 +381,16 @@ void blk_cleanup_queue(struct request_queue *q)
+ */
+ blk_freeze_queue(q);
+
++ /* cleanup rq qos structures for queue without disk */
++ rq_qos_exit(q);
++
+ blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
+
+ blk_sync_queue(q);
+- if (queue_is_mq(q))
++ if (queue_is_mq(q)) {
++ blk_mq_cancel_work_sync(q);
+ blk_mq_exit_queue(q);
++ }
+
+ /*
+ * In theory, request pool of sched_tags belongs to request queue.
+@@ -887,10 +887,8 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio)
+ if (unlikely(!current->io_context))
+ create_task_io_context(current, GFP_ATOMIC, q->node);
+
+- if (blk_throtl_bio(bio)) {
+- blkcg_bio_issue_init(bio);
++ if (blk_throtl_bio(bio))
+ return false;
+- }
+
+ blk_cgroup_bio_start(bio);
+ blkcg_bio_issue_init(bio);
+@@ -1293,20 +1291,32 @@ void blk_account_io_start(struct request *rq)
+ }
+
+ static unsigned long __part_start_io_acct(struct block_device *part,
+- unsigned int sectors, unsigned int op)
++ unsigned int sectors, unsigned int op,
++ unsigned long start_time)
+ {
+ const int sgrp = op_stat_group(op);
+- unsigned long now = READ_ONCE(jiffies);
+
+ part_stat_lock();
+- update_io_ticks(part, now, false);
++ update_io_ticks(part, start_time, false);
+ part_stat_inc(part, ios[sgrp]);
+ part_stat_add(part, sectors[sgrp], sectors);
+ part_stat_local_inc(part, in_flight[op_is_write(op)]);
+ part_stat_unlock();
+
+- return now;
++ return start_time;
++}
++
++/**
++ * bio_start_io_acct_time - start I/O accounting for bio based drivers
++ * @bio: bio to start account for
++ * @start_time: start time that should be passed back to bio_end_io_acct().
++ */
++void bio_start_io_acct_time(struct bio *bio, unsigned long start_time)
++{
++ __part_start_io_acct(bio->bi_bdev, bio_sectors(bio),
++ bio_op(bio), start_time);
+ }
++EXPORT_SYMBOL_GPL(bio_start_io_acct_time);
+
+ /**
+ * bio_start_io_acct - start I/O accounting for bio based drivers
+@@ -1316,14 +1326,15 @@ static unsigned long __part_start_io_acct(struct block_device *part,
+ */
+ unsigned long bio_start_io_acct(struct bio *bio)
+ {
+- return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio));
++ return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio),
++ bio_op(bio), jiffies);
+ }
+ EXPORT_SYMBOL_GPL(bio_start_io_acct);
+
+ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
+ unsigned int op)
+ {
+- return __part_start_io_acct(disk->part0, sectors, op);
++ return __part_start_io_acct(disk->part0, sectors, op, jiffies);
+ }
+ EXPORT_SYMBOL(disk_start_io_acct);
+
+diff --git a/block/blk-flush.c b/block/blk-flush.c
+index 4201728bf3a5a..94a86acbb7f67 100644
+--- a/block/blk-flush.c
++++ b/block/blk-flush.c
+@@ -235,8 +235,10 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error)
+ * avoiding use-after-free.
+ */
+ WRITE_ONCE(flush_rq->state, MQ_RQ_IDLE);
+- if (fq->rq_status != BLK_STS_OK)
++ if (fq->rq_status != BLK_STS_OK) {
+ error = fq->rq_status;
++ fq->rq_status = BLK_STS_OK;
++ }
+
+ if (!q->elevator) {
+ flush_rq->tag = BLK_MQ_NO_TAG;
+diff --git a/block/blk-iocost.c b/block/blk-iocost.c
+index b3880e4ba22a1..10851493940c3 100644
+--- a/block/blk-iocost.c
++++ b/block/blk-iocost.c
+@@ -2311,11 +2311,28 @@ static void ioc_timer_fn(struct timer_list *timer)
+ hwm = current_hweight_max(iocg);
+ new_hwi = hweight_after_donation(iocg, old_hwi, hwm,
+ usage, &now);
+- if (new_hwi < hwm) {
++ /*
++ * Donation calculation assumes hweight_after_donation
++ * to be positive, a condition that a donor w/ hwa < 2
++ * can't meet. Don't bother with donation if hwa is
++ * below 2. It's not gonna make a meaningful difference
++ * anyway.
++ */
++ if (new_hwi < hwm && hwa >= 2) {
+ iocg->hweight_donating = hwa;
+ iocg->hweight_after_donation = new_hwi;
+ list_add(&iocg->surplus_list, &surpluses);
+- } else {
++ } else if (!iocg->abs_vdebt) {
++ /*
++ * @iocg doesn't have enough to donate. Reset
++ * its inuse to active.
++ *
++ * Don't reset debtors as their inuse's are
++ * owned by debt handling. This shouldn't affect
++ * donation calculuation in any meaningful way
++ * as @iocg doesn't have a meaningful amount of
++ * share anyway.
++ */
+ TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
+ iocg->inuse, iocg->active,
+ iocg->hweight_inuse, new_hwi);
+diff --git a/block/blk-map.c b/block/blk-map.c
+index 4526adde01564..c7f71d83eff18 100644
+--- a/block/blk-map.c
++++ b/block/blk-map.c
+@@ -446,7 +446,7 @@ static struct bio *bio_copy_kern(struct request_queue *q, void *data,
+ if (bytes > len)
+ bytes = len;
+
+- page = alloc_page(GFP_NOIO | gfp_mask);
++ page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask);
+ if (!page)
+ goto cleanup;
+
+diff --git a/block/blk-merge.c b/block/blk-merge.c
+index 7a5c81c02c800..bbe66a9010bf1 100644
+--- a/block/blk-merge.c
++++ b/block/blk-merge.c
+@@ -7,6 +7,7 @@
+ #include <linux/bio.h>
+ #include <linux/blkdev.h>
+ #include <linux/scatterlist.h>
++#include <linux/blk-cgroup.h>
+
+ #include <trace/events/block.h>
+
+@@ -561,6 +562,9 @@ static inline unsigned int blk_rq_get_max_segments(struct request *rq)
+ static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
+ unsigned int nr_phys_segs)
+ {
++ if (!blk_cgroup_mergeable(req, bio))
++ goto no_merge;
++
+ if (blk_integrity_merge_bio(req->q, req, bio) == false)
+ goto no_merge;
+
+@@ -657,6 +661,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
+ if (total_phys_segments > blk_rq_get_max_segments(req))
+ return 0;
+
++ if (!blk_cgroup_mergeable(req, next->bio))
++ return 0;
++
+ if (blk_integrity_merge_rq(q, req, next) == false)
+ return 0;
+
+@@ -863,6 +870,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
+ if (rq->rq_disk != bio->bi_bdev->bd_disk)
+ return false;
+
++ /* don't merge across cgroup boundaries */
++ if (!blk_cgroup_mergeable(rq, bio))
++ return false;
++
+ /* only merge integrity protected bio into ditto rq */
+ if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
+ return false;
+diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
+index 0f006cabfd914..35770e33c817a 100644
+--- a/block/blk-mq-sched.c
++++ b/block/blk-mq-sched.c
+@@ -208,11 +208,18 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
+
+ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
+ {
++ unsigned long end = jiffies + HZ;
+ int ret;
+
+ do {
+ ret = __blk_mq_do_dispatch_sched(hctx);
+- } while (ret == 1);
++ if (ret != 1)
++ break;
++ if (need_resched() || time_is_before_jiffies(end)) {
++ blk_mq_delay_run_hw_queue(hctx, 0);
++ break;
++ }
++ } while (1);
+
+ return ret;
+ }
+diff --git a/block/blk-mq.c b/block/blk-mq.c
+index 652a31fc3bb38..82de39926a9f6 100644
+--- a/block/blk-mq.c
++++ b/block/blk-mq.c
+@@ -763,7 +763,6 @@ void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
+ /* this request will be re-inserted to io scheduler queue */
+ blk_mq_sched_requeue_request(rq);
+
+- BUG_ON(!list_empty(&rq->queuelist));
+ blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
+ }
+ EXPORT_SYMBOL(blk_mq_requeue_request);
+@@ -2148,14 +2147,14 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
+ }
+
+ /*
+- * Allow 4x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
++ * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
+ * queues. This is important for md arrays to benefit from merging
+ * requests.
+ */
+ static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
+ {
+ if (plug->multiple_queues)
+- return BLK_MAX_REQUEST_COUNT * 4;
++ return BLK_MAX_REQUEST_COUNT * 2;
+ return BLK_MAX_REQUEST_COUNT;
+ }
+
+@@ -4019,6 +4018,19 @@ unsigned int blk_mq_rq_cpu(struct request *rq)
+ }
+ EXPORT_SYMBOL(blk_mq_rq_cpu);
+
++void blk_mq_cancel_work_sync(struct request_queue *q)
++{
++ if (queue_is_mq(q)) {
++ struct blk_mq_hw_ctx *hctx;
++ int i;
++
++ cancel_delayed_work_sync(&q->requeue_work);
++
++ queue_for_each_hw_ctx(q, hctx, i)
++ cancel_delayed_work_sync(&hctx->run_work);
++ }
++}
++
+ static int __init blk_mq_init(void)
+ {
+ int i;
+diff --git a/block/blk-mq.h b/block/blk-mq.h
+index d08779f77a265..7cdca23b6263d 100644
+--- a/block/blk-mq.h
++++ b/block/blk-mq.h
+@@ -129,6 +129,8 @@ extern int blk_mq_sysfs_register(struct request_queue *q);
+ extern void blk_mq_sysfs_unregister(struct request_queue *q);
+ extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
+
++void blk_mq_cancel_work_sync(struct request_queue *q);
++
+ void blk_mq_release(struct request_queue *q);
+
+ static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
+diff --git a/block/blk-pm.c b/block/blk-pm.c
+index 17bd020268d42..2dad62cc15727 100644
+--- a/block/blk-pm.c
++++ b/block/blk-pm.c
+@@ -163,27 +163,19 @@ EXPORT_SYMBOL(blk_pre_runtime_resume);
+ /**
+ * blk_post_runtime_resume - Post runtime resume processing
+ * @q: the queue of the device
+- * @err: return value of the device's runtime_resume function
+ *
+ * Description:
+- * Update the queue's runtime status according to the return value of the
+- * device's runtime_resume function. If the resume was successful, call
+- * blk_set_runtime_active() to do the real work of restarting the queue.
++ * For historical reasons, this routine merely calls blk_set_runtime_active()
++ * to do the real work of restarting the queue. It does this regardless of
++ * whether the device's runtime-resume succeeded; even if it failed the
++ * driver or error handler will need to communicate with the device.
+ *
+ * This function should be called near the end of the device's
+ * runtime_resume callback.
+ */
+-void blk_post_runtime_resume(struct request_queue *q, int err)
++void blk_post_runtime_resume(struct request_queue *q)
+ {
+- if (!q->dev)
+- return;
+- if (!err) {
+- blk_set_runtime_active(q);
+- } else {
+- spin_lock_irq(&q->queue_lock);
+- q->rpm_status = RPM_SUSPENDED;
+- spin_unlock_irq(&q->queue_lock);
+- }
++ blk_set_runtime_active(q);
+ }
+ EXPORT_SYMBOL(blk_post_runtime_resume);
+
+@@ -201,7 +193,7 @@ EXPORT_SYMBOL(blk_post_runtime_resume);
+ * runtime PM status and re-enable peeking requests from the queue. It
+ * should be called before first request is added to the queue.
+ *
+- * This function is also called by blk_post_runtime_resume() for successful
++ * This function is also called by blk_post_runtime_resume() for
+ * runtime resumes. It does everything necessary to restart the queue.
+ */
+ void blk_set_runtime_active(struct request_queue *q)
+diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
+index 614d9d47de36b..00021f0123701 100644
+--- a/block/blk-sysfs.c
++++ b/block/blk-sysfs.c
+@@ -805,16 +805,6 @@ static void blk_release_queue(struct kobject *kobj)
+
+ blk_free_queue_stats(q->stats);
+
+- if (queue_is_mq(q)) {
+- struct blk_mq_hw_ctx *hctx;
+- int i;
+-
+- cancel_delayed_work_sync(&q->requeue_work);
+-
+- queue_for_each_hw_ctx(q, hctx, i)
+- cancel_delayed_work_sync(&hctx->run_work);
+- }
+-
+ blk_exit_queue(q);
+
+ blk_queue_free_zone_bitmaps(q);
+@@ -964,15 +954,17 @@ void blk_unregister_queue(struct gendisk *disk)
+ */
+ if (queue_is_mq(q))
+ blk_mq_unregister_dev(disk_to_dev(disk), q);
+-
+- kobject_uevent(&q->kobj, KOBJ_REMOVE);
+- kobject_del(&q->kobj);
+ blk_trace_remove_sysfs(disk_to_dev(disk));
+
+ mutex_lock(&q->sysfs_lock);
+ if (q->elevator)
+ elv_unregister_queue(q);
+ mutex_unlock(&q->sysfs_lock);
++
++ /* Now that we've deleted all child objects, we can delete the queue. */
++ kobject_uevent(&q->kobj, KOBJ_REMOVE);
++ kobject_del(&q->kobj);
++
+ mutex_unlock(&q->sysfs_dir_lock);
+
+ kobject_put(&disk_to_dev(disk)->kobj);
+diff --git a/block/blk-wbt.c b/block/blk-wbt.c
+index 874c1c37bf0c6..0c119be0e8133 100644
+--- a/block/blk-wbt.c
++++ b/block/blk-wbt.c
+@@ -357,6 +357,9 @@ static void wb_timer_fn(struct blk_stat_callback *cb)
+ unsigned int inflight = wbt_inflight(rwb);
+ int status;
+
++ if (!rwb->rqos.q->disk)
++ return;
++
+ status = latency_exceeded(rwb, cb->stat);
+
+ trace_wbt_timer(rwb->rqos.q->disk->bdi, status, rqd->scale_step,
+diff --git a/block/blk-zoned.c b/block/blk-zoned.c
+index 1d0c76c18fc52..774ecc598bee2 100644
+--- a/block/blk-zoned.c
++++ b/block/blk-zoned.c
+@@ -429,9 +429,10 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
+ op = REQ_OP_ZONE_RESET;
+
+ /* Invalidate the page cache, including dirty pages. */
++ filemap_invalidate_lock(bdev->bd_inode->i_mapping);
+ ret = blkdev_truncate_zone_range(bdev, mode, &zrange);
+ if (ret)
+- return ret;
++ goto fail;
+ break;
+ case BLKOPENZONE:
+ op = REQ_OP_ZONE_OPEN;
+@@ -449,15 +450,9 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
+ ret = blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors,
+ GFP_KERNEL);
+
+- /*
+- * Invalidate the page cache again for zone reset: writes can only be
+- * direct for zoned devices so concurrent writes would not add any page
+- * to the page cache after/during reset. The page cache may be filled
+- * again due to concurrent reads though and dropping the pages for
+- * these is fine.
+- */
+- if (!ret && cmd == BLKRESETZONE)
+- ret = blkdev_truncate_zone_range(bdev, mode, &zrange);
++fail:
++ if (cmd == BLKRESETZONE)
++ filemap_invalidate_unlock(bdev->bd_inode->i_mapping);
+
+ return ret;
+ }
+diff --git a/block/blk.h b/block/blk.h
+index 6c3c00a8fe19d..aab72194d2266 100644
+--- a/block/blk.h
++++ b/block/blk.h
+@@ -184,6 +184,12 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
+ void blk_account_io_start(struct request *req);
+ void blk_account_io_done(struct request *req, u64 now);
+
++/*
++ * Plug flush limits
++ */
++#define BLK_MAX_REQUEST_COUNT 32
++#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
++
+ /*
+ * Internal elevator interface
+ */
+diff --git a/block/elevator.c b/block/elevator.c
+index ff45d8388f487..1b5e57f6115f3 100644
+--- a/block/elevator.c
++++ b/block/elevator.c
+@@ -523,8 +523,6 @@ void elv_unregister_queue(struct request_queue *q)
+ kobject_del(&e->kobj);
+
+ e->registered = 0;
+- /* Re-enable throttling in case elevator disabled it */
+- wbt_enable_default(q);
+ }
+ }
+
+@@ -694,12 +692,18 @@ void elevator_init_mq(struct request_queue *q)
+ if (!e)
+ return;
+
++ /*
++ * We are called before adding disk, when there isn't any FS I/O,
++ * so freezing queue plus canceling dispatch work is enough to
++ * drain any dispatch activities originated from passthrough
++ * requests, then no need to quiesce queue which may add long boot
++ * latency, especially when lots of disks are involved.
++ */
+ blk_mq_freeze_queue(q);
+- blk_mq_quiesce_queue(q);
++ blk_mq_cancel_work_sync(q);
+
+ err = blk_mq_init_sched(q, e);
+
+- blk_mq_unquiesce_queue(q);
+ blk_mq_unfreeze_queue(q);
+
+ if (err) {
+diff --git a/block/genhd.c b/block/genhd.c
+index ab12ae6e636e8..74e19d67ceab5 100644
+--- a/block/genhd.c
++++ b/block/genhd.c
+@@ -19,6 +19,7 @@
+ #include <linux/seq_file.h>
+ #include <linux/slab.h>
+ #include <linux/kmod.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/idr.h>
+ #include <linux/log2.h>
+@@ -323,7 +324,7 @@ int blk_alloc_ext_minor(void)
+ {
+ int idx;
+
+- idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT, GFP_KERNEL);
++ idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT - 1, GFP_KERNEL);
+ if (idx == -ENOSPC)
+ return -EBUSY;
+ return idx;
+@@ -420,6 +421,8 @@ int device_add_disk(struct device *parent, struct gendisk *disk,
+ DISK_MAX_PARTS);
+ disk->minors = DISK_MAX_PARTS;
+ }
++ if (disk->first_minor + disk->minors > MINORMASK + 1)
++ return -EINVAL;
+ } else {
+ if (WARN_ON(disk->minors))
+ return -EINVAL;
+@@ -432,10 +435,6 @@ int device_add_disk(struct device *parent, struct gendisk *disk,
+ disk->flags |= GENHD_FL_EXT_DEVT;
+ }
+
+- ret = disk_alloc_events(disk);
+- if (ret)
+- goto out_free_ext_minor;
+-
+ /* delay uevents, until we scanned partition table */
+ dev_set_uevent_suppress(ddev, 1);
+
+@@ -446,7 +445,12 @@ int device_add_disk(struct device *parent, struct gendisk *disk,
+ ddev->devt = MKDEV(disk->major, disk->first_minor);
+ ret = device_add(ddev);
+ if (ret)
+- goto out_disk_release_events;
++ goto out_free_ext_minor;
++
++ ret = disk_alloc_events(disk);
++ if (ret)
++ goto out_device_del;
++
+ if (!sysfs_deprecated) {
+ ret = sysfs_create_link(block_depr, &ddev->kobj,
+ kobject_name(&ddev->kobj));
+@@ -467,11 +471,15 @@ int device_add_disk(struct device *parent, struct gendisk *disk,
+
+ disk->part0->bd_holder_dir =
+ kobject_create_and_add("holders", &ddev->kobj);
+- if (!disk->part0->bd_holder_dir)
++ if (!disk->part0->bd_holder_dir) {
++ ret = -ENOMEM;
+ goto out_del_integrity;
++ }
+ disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);
+- if (!disk->slave_dir)
++ if (!disk->slave_dir) {
++ ret = -ENOMEM;
+ goto out_put_holder_dir;
++ }
+
+ ret = bd_register_pending_holders(disk);
+ if (ret < 0)
+@@ -530,8 +538,6 @@ out_del_block_link:
+ sysfs_remove_link(block_depr, dev_name(ddev));
+ out_device_del:
+ device_del(ddev);
+-out_disk_release_events:
+- disk_release_events(disk);
+ out_free_ext_minor:
+ if (disk->major == BLOCK_EXT_MAJOR)
+ blk_free_ext_minor(disk->first_minor);
+@@ -539,6 +545,20 @@ out_free_ext_minor:
+ }
+ EXPORT_SYMBOL(device_add_disk);
+
++/**
++ * blk_mark_disk_dead - mark a disk as dead
++ * @disk: disk to mark as dead
++ *
++ * Mark as disk as dead (e.g. surprise removed) and don't accept any new I/O
++ * to this disk.
++ */
++void blk_mark_disk_dead(struct gendisk *disk)
++{
++ set_bit(GD_DEAD, &disk->state);
++ blk_queue_start_drain(disk->queue);
++}
++EXPORT_SYMBOL_GPL(blk_mark_disk_dead);
++
+ /**
+ * del_gendisk - remove the gendisk
+ * @disk: the struct gendisk to remove
+@@ -1082,6 +1102,8 @@ static void disk_release(struct device *dev)
+ might_sleep();
+ WARN_ON_ONCE(disk_live(disk));
+
++ blk_mq_cancel_work_sync(disk->queue);
++
+ disk_release_events(disk);
+ kfree(disk->random);
+ xa_destroy(&disk->part_tbl);
+diff --git a/block/holder.c b/block/holder.c
+index 9dc084182337f..27cddce1b4461 100644
+--- a/block/holder.c
++++ b/block/holder.c
+@@ -1,5 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+ #include <linux/genhd.h>
++#include <linux/slab.h>
+
+ struct bd_holder_disk {
+ struct list_head list;
+diff --git a/block/ioctl.c b/block/ioctl.c
+index eb0491e90b9a0..cd506a9029630 100644
+--- a/block/ioctl.c
++++ b/block/ioctl.c
+@@ -113,6 +113,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode,
+ uint64_t range[2];
+ uint64_t start, len;
+ struct request_queue *q = bdev_get_queue(bdev);
++ struct inode *inode = bdev->bd_inode;
+ int err;
+
+ if (!(mode & FMODE_WRITE))
+@@ -135,12 +136,17 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode,
+ if (start + len > i_size_read(bdev->bd_inode))
+ return -EINVAL;
+
++ filemap_invalidate_lock(inode->i_mapping);
+ err = truncate_bdev_range(bdev, mode, start, start + len - 1);
+ if (err)
+- return err;
++ goto fail;
+
+- return blkdev_issue_discard(bdev, start >> 9, len >> 9,
+- GFP_KERNEL, flags);
++ err = blkdev_issue_discard(bdev, start >> 9, len >> 9,
++ GFP_KERNEL, flags);
++
++fail:
++ filemap_invalidate_unlock(inode->i_mapping);
++ return err;
+ }
+
+ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode,
+@@ -148,6 +154,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode,
+ {
+ uint64_t range[2];
+ uint64_t start, end, len;
++ struct inode *inode = bdev->bd_inode;
+ int err;
+
+ if (!(mode & FMODE_WRITE))
+@@ -170,12 +177,17 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode,
+ return -EINVAL;
+
+ /* Invalidate the page cache, including dirty pages */
++ filemap_invalidate_lock(inode->i_mapping);
+ err = truncate_bdev_range(bdev, mode, start, end);
+ if (err)
+- return err;
++ goto fail;
++
++ err = blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL,
++ BLKDEV_ZERO_NOUNMAP);
+
+- return blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL,
+- BLKDEV_ZERO_NOUNMAP);
++fail:
++ filemap_invalidate_unlock(inode->i_mapping);
++ return err;
+ }
+
+ static int put_ushort(unsigned short __user *argp, unsigned short val)
+@@ -633,7 +645,7 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg)
+ (bdev->bd_disk->bdi->ra_pages * PAGE_SIZE) / 512);
+ case BLKGETSIZE:
+ size = i_size_read(bdev->bd_inode);
+- if ((size >> 9) > ~0UL)
++ if ((size >> 9) > ~(compat_ulong_t)0)
+ return -EFBIG;
+ return compat_put_ulong(argp, size >> 9);
+
+diff --git a/block/ioprio.c b/block/ioprio.c
+index 0e4ff245f2bf2..6f01d35a5145a 100644
+--- a/block/ioprio.c
++++ b/block/ioprio.c
+@@ -69,7 +69,14 @@ int ioprio_check_cap(int ioprio)
+
+ switch (class) {
+ case IOPRIO_CLASS_RT:
+- if (!capable(CAP_SYS_NICE) && !capable(CAP_SYS_ADMIN))
++ /*
++ * Originally this only checked for CAP_SYS_ADMIN,
++ * which was implicitly allowed for pid 0 by security
++ * modules such as SELinux. Make sure we check
++ * CAP_SYS_ADMIN first to avoid a denial/avc for
++ * possibly missing CAP_SYS_NICE permission.
++ */
++ if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_NICE))
+ return -EPERM;
+ fallthrough;
+ /* rt has prio field too */
+@@ -213,6 +220,7 @@ SYSCALL_DEFINE2(ioprio_get, int, which, int, who)
+ pgrp = task_pgrp(current);
+ else
+ pgrp = find_vpid(who);
++ read_lock(&tasklist_lock);
+ do_each_pid_thread(pgrp, PIDTYPE_PGID, p) {
+ tmpio = get_task_ioprio(p);
+ if (tmpio < 0)
+@@ -222,6 +230,8 @@ SYSCALL_DEFINE2(ioprio_get, int, which, int, who)
+ else
+ ret = ioprio_best(ret, tmpio);
+ } while_each_pid_thread(pgrp, PIDTYPE_PGID, p);
++ read_unlock(&tasklist_lock);
++
+ break;
+ case IOPRIO_WHO_USER:
+ uid = make_kuid(current_user_ns(), who);
+diff --git a/block/mq-deadline.c b/block/mq-deadline.c
+index 7f3c3932b723e..cd2342d297048 100644
+--- a/block/mq-deadline.c
++++ b/block/mq-deadline.c
+@@ -811,7 +811,7 @@ SHOW_JIFFIES(deadline_read_expire_show, dd->fifo_expire[DD_READ]);
+ SHOW_JIFFIES(deadline_write_expire_show, dd->fifo_expire[DD_WRITE]);
+ SHOW_INT(deadline_writes_starved_show, dd->writes_starved);
+ SHOW_INT(deadline_front_merges_show, dd->front_merges);
+-SHOW_INT(deadline_async_depth_show, dd->front_merges);
++SHOW_INT(deadline_async_depth_show, dd->async_depth);
+ SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch);
+ #undef SHOW_INT
+ #undef SHOW_JIFFIES
+@@ -840,7 +840,7 @@ STORE_JIFFIES(deadline_read_expire_store, &dd->fifo_expire[DD_READ], 0, INT_MAX)
+ STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MAX);
+ STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX);
+ STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1);
+-STORE_INT(deadline_async_depth_store, &dd->front_merges, 1, INT_MAX);
++STORE_INT(deadline_async_depth_store, &dd->async_depth, 1, INT_MAX);
+ STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX);
+ #undef STORE_FUNCTION
+ #undef STORE_INT
+diff --git a/block/partitions/core.c b/block/partitions/core.c
+index 7bea19dd9458f..b9e9af84f5188 100644
+--- a/block/partitions/core.c
++++ b/block/partitions/core.c
+@@ -5,6 +5,7 @@
+ * Copyright (C) 2020 Christoph Hellwig
+ */
+ #include <linux/fs.h>
++#include <linux/major.h>
+ #include <linux/slab.h>
+ #include <linux/ctype.h>
+ #include <linux/genhd.h>
+diff --git a/crypto/Kconfig b/crypto/Kconfig
+index 536df4b6b825c..55718de561375 100644
+--- a/crypto/Kconfig
++++ b/crypto/Kconfig
+@@ -233,12 +233,12 @@ config CRYPTO_DH
+
+ config CRYPTO_ECC
+ tristate
++ select CRYPTO_RNG_DEFAULT
+
+ config CRYPTO_ECDH
+ tristate "ECDH algorithm"
+ select CRYPTO_ECC
+ select CRYPTO_KPP
+- select CRYPTO_RNG_DEFAULT
+ help
+ Generic implementation of the ECDH algorithm
+
+@@ -1919,9 +1919,10 @@ config CRYPTO_STATS
+ config CRYPTO_HASH_INFO
+ bool
+
+-source "lib/crypto/Kconfig"
+ source "drivers/crypto/Kconfig"
+ source "crypto/asymmetric_keys/Kconfig"
+ source "certs/Kconfig"
+
+ endif # if CRYPTO
++
++source "lib/crypto/Kconfig"
+diff --git a/crypto/algapi.c b/crypto/algapi.c
+index 43f999dba4dc0..f3d95af3e4284 100644
+--- a/crypto/algapi.c
++++ b/crypto/algapi.c
+@@ -1277,3 +1277,4 @@ module_exit(crypto_algapi_exit);
+
+ MODULE_LICENSE("GPL");
+ MODULE_DESCRIPTION("Cryptographic algorithms API");
++MODULE_SOFTDEP("pre: cryptomgr");
+diff --git a/crypto/api.c b/crypto/api.c
+index c4eda56cff891..5ffcd3ab4a753 100644
+--- a/crypto/api.c
++++ b/crypto/api.c
+@@ -603,4 +603,3 @@ EXPORT_SYMBOL_GPL(crypto_req_done);
+
+ MODULE_DESCRIPTION("Cryptographic core API");
+ MODULE_LICENSE("GPL");
+-MODULE_SOFTDEP("pre: cryptomgr");
+diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c
+index 0b4d07aa88111..f94a1d1ad3a6c 100644
+--- a/crypto/asymmetric_keys/pkcs7_verify.c
++++ b/crypto/asymmetric_keys/pkcs7_verify.c
+@@ -174,12 +174,6 @@ static int pkcs7_find_key(struct pkcs7_message *pkcs7,
+ pr_devel("Sig %u: Found cert serial match X.509[%u]\n",
+ sinfo->index, certix);
+
+- if (strcmp(x509->pub->pkey_algo, sinfo->sig->pkey_algo) != 0) {
+- pr_warn("Sig %u: X.509 algo and PKCS#7 sig algo don't match\n",
+- sinfo->index);
+- continue;
+- }
+-
+ sinfo->signer = x509;
+ return 0;
+ }
+diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c
+index 4fefb219bfdc8..7c9e6be35c30c 100644
+--- a/crypto/asymmetric_keys/public_key.c
++++ b/crypto/asymmetric_keys/public_key.c
+@@ -60,39 +60,83 @@ static void public_key_destroy(void *payload0, void *payload3)
+ }
+
+ /*
+- * Determine the crypto algorithm name.
++ * Given a public_key, and an encoding and hash_algo to be used for signing
++ * and/or verification with that key, determine the name of the corresponding
++ * akcipher algorithm. Also check that encoding and hash_algo are allowed.
+ */
+-static
+-int software_key_determine_akcipher(const char *encoding,
+- const char *hash_algo,
+- const struct public_key *pkey,
+- char alg_name[CRYPTO_MAX_ALG_NAME])
++static int
++software_key_determine_akcipher(const struct public_key *pkey,
++ const char *encoding, const char *hash_algo,
++ char alg_name[CRYPTO_MAX_ALG_NAME])
+ {
+ int n;
+
+- if (strcmp(encoding, "pkcs1") == 0) {
+- /* The data wangled by the RSA algorithm is typically padded
+- * and encoded in some manner, such as EMSA-PKCS1-1_5 [RFC3447
+- * sec 8.2].
++ if (!encoding)
++ return -EINVAL;
++
++ if (strcmp(pkey->pkey_algo, "rsa") == 0) {
++ /*
++ * RSA signatures usually use EMSA-PKCS1-1_5 [RFC3447 sec 8.2].
++ */
++ if (strcmp(encoding, "pkcs1") == 0) {
++ if (!hash_algo)
++ n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME,
++ "pkcs1pad(%s)",
++ pkey->pkey_algo);
++ else
++ n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME,
++ "pkcs1pad(%s,%s)",
++ pkey->pkey_algo, hash_algo);
++ return n >= CRYPTO_MAX_ALG_NAME ? -EINVAL : 0;
++ }
++ if (strcmp(encoding, "raw") != 0)
++ return -EINVAL;
++ /*
++ * Raw RSA cannot differentiate between different hash
++ * algorithms.
++ */
++ if (hash_algo)
++ return -EINVAL;
++ } else if (strncmp(pkey->pkey_algo, "ecdsa", 5) == 0) {
++ if (strcmp(encoding, "x962") != 0)
++ return -EINVAL;
++ /*
++ * ECDSA signatures are taken over a raw hash, so they don't
++ * differentiate between different hash algorithms. That means
++ * that the verifier should hard-code a specific hash algorithm.
++ * Unfortunately, in practice ECDSA is used with multiple SHAs,
++ * so we have to allow all of them and not just one.
+ */
+ if (!hash_algo)
+- n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME,
+- "pkcs1pad(%s)",
+- pkey->pkey_algo);
+- else
+- n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME,
+- "pkcs1pad(%s,%s)",
+- pkey->pkey_algo, hash_algo);
+- return n >= CRYPTO_MAX_ALG_NAME ? -EINVAL : 0;
+- }
+-
+- if (strcmp(encoding, "raw") == 0 ||
+- strcmp(encoding, "x962") == 0) {
+- strcpy(alg_name, pkey->pkey_algo);
+- return 0;
++ return -EINVAL;
++ if (strcmp(hash_algo, "sha1") != 0 &&
++ strcmp(hash_algo, "sha224") != 0 &&
++ strcmp(hash_algo, "sha256") != 0 &&
++ strcmp(hash_algo, "sha384") != 0 &&
++ strcmp(hash_algo, "sha512") != 0)
++ return -EINVAL;
++ } else if (strcmp(pkey->pkey_algo, "sm2") == 0) {
++ if (strcmp(encoding, "raw") != 0)
++ return -EINVAL;
++ if (!hash_algo)
++ return -EINVAL;
++ if (strcmp(hash_algo, "sm3") != 0)
++ return -EINVAL;
++ } else if (strcmp(pkey->pkey_algo, "ecrdsa") == 0) {
++ if (strcmp(encoding, "raw") != 0)
++ return -EINVAL;
++ if (!hash_algo)
++ return -EINVAL;
++ if (strcmp(hash_algo, "streebog256") != 0 &&
++ strcmp(hash_algo, "streebog512") != 0)
++ return -EINVAL;
++ } else {
++ /* Unknown public key algorithm */
++ return -ENOPKG;
+ }
+-
+- return -ENOPKG;
++ if (strscpy(alg_name, pkey->pkey_algo, CRYPTO_MAX_ALG_NAME) < 0)
++ return -EINVAL;
++ return 0;
+ }
+
+ static u8 *pkey_pack_u32(u8 *dst, u32 val)
+@@ -113,9 +157,8 @@ static int software_key_query(const struct kernel_pkey_params *params,
+ u8 *key, *ptr;
+ int ret, len;
+
+- ret = software_key_determine_akcipher(params->encoding,
+- params->hash_algo,
+- pkey, alg_name);
++ ret = software_key_determine_akcipher(pkey, params->encoding,
++ params->hash_algo, alg_name);
+ if (ret < 0)
+ return ret;
+
+@@ -179,9 +222,8 @@ static int software_key_eds_op(struct kernel_pkey_params *params,
+
+ pr_devel("==>%s()\n", __func__);
+
+- ret = software_key_determine_akcipher(params->encoding,
+- params->hash_algo,
+- pkey, alg_name);
++ ret = software_key_determine_akcipher(pkey, params->encoding,
++ params->hash_algo, alg_name);
+ if (ret < 0)
+ return ret;
+
+@@ -325,9 +367,23 @@ int public_key_verify_signature(const struct public_key *pkey,
+ BUG_ON(!sig);
+ BUG_ON(!sig->s);
+
+- ret = software_key_determine_akcipher(sig->encoding,
+- sig->hash_algo,
+- pkey, alg_name);
++ /*
++ * If the signature specifies a public key algorithm, it *must* match
++ * the key's actual public key algorithm.
++ *
++ * Small exception: ECDSA signatures don't specify the curve, but ECDSA
++ * keys do. So the strings can mismatch slightly in that case:
++ * "ecdsa-nist-*" for the key, but "ecdsa" for the signature.
++ */
++ if (sig->pkey_algo) {
++ if (strcmp(pkey->pkey_algo, sig->pkey_algo) != 0 &&
++ (strncmp(pkey->pkey_algo, "ecdsa-", 6) != 0 ||
++ strcmp(sig->pkey_algo, "ecdsa") != 0))
++ return -EKEYREJECTED;
++ }
++
++ ret = software_key_determine_akcipher(pkey, sig->encoding,
++ sig->hash_algo, alg_name);
+ if (ret < 0)
+ return ret;
+
+diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c
+index 3d45161b271a4..7fd56df8b9194 100644
+--- a/crypto/asymmetric_keys/x509_public_key.c
++++ b/crypto/asymmetric_keys/x509_public_key.c
+@@ -128,12 +128,6 @@ int x509_check_for_self_signed(struct x509_certificate *cert)
+ goto out;
+ }
+
+- ret = -EKEYREJECTED;
+- if (strcmp(cert->pub->pkey_algo, cert->sig->pkey_algo) != 0 &&
+- (strncmp(cert->pub->pkey_algo, "ecdsa-", 6) != 0 ||
+- strcmp(cert->sig->pkey_algo, "ecdsa") != 0))
+- goto out;
+-
+ ret = public_key_verify_signature(cert->pub, cert->sig);
+ if (ret < 0) {
+ if (ret == -ENOPKG) {
+diff --git a/crypto/authenc.c b/crypto/authenc.c
+index 670bf1a01d00e..17f674a7cdff5 100644
+--- a/crypto/authenc.c
++++ b/crypto/authenc.c
+@@ -253,7 +253,7 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req,
+ dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen);
+
+ skcipher_request_set_tfm(skreq, ctx->enc);
+- skcipher_request_set_callback(skreq, aead_request_flags(req),
++ skcipher_request_set_callback(skreq, flags,
+ req->base.complete, req->base.data);
+ skcipher_request_set_crypt(skreq, src, dst,
+ req->cryptlen - authsize, req->iv);
+diff --git a/crypto/blake2s_generic.c b/crypto/blake2s_generic.c
+index 72fe480f9bd67..5f96a21f87883 100644
+--- a/crypto/blake2s_generic.c
++++ b/crypto/blake2s_generic.c
+@@ -15,12 +15,12 @@
+ static int crypto_blake2s_update_generic(struct shash_desc *desc,
+ const u8 *in, unsigned int inlen)
+ {
+- return crypto_blake2s_update(desc, in, inlen, blake2s_compress_generic);
++ return crypto_blake2s_update(desc, in, inlen, true);
+ }
+
+ static int crypto_blake2s_final_generic(struct shash_desc *desc, u8 *out)
+ {
+- return crypto_blake2s_final(desc, out, blake2s_compress_generic);
++ return crypto_blake2s_final(desc, out, true);
+ }
+
+ #define BLAKE2S_ALG(name, driver_name, digest_size) \
+diff --git a/crypto/drbg.c b/crypto/drbg.c
+index ea85d4a0fe9e9..03c9ef768c227 100644
+--- a/crypto/drbg.c
++++ b/crypto/drbg.c
+@@ -1491,12 +1491,13 @@ static int drbg_generate_long(struct drbg_state *drbg,
+ return 0;
+ }
+
+-static void drbg_schedule_async_seed(struct random_ready_callback *rdy)
++static int drbg_schedule_async_seed(struct notifier_block *nb, unsigned long action, void *data)
+ {
+- struct drbg_state *drbg = container_of(rdy, struct drbg_state,
++ struct drbg_state *drbg = container_of(nb, struct drbg_state,
+ random_ready);
+
+ schedule_work(&drbg->seed_work);
++ return 0;
+ }
+
+ static int drbg_prepare_hrng(struct drbg_state *drbg)
+@@ -1511,10 +1512,8 @@ static int drbg_prepare_hrng(struct drbg_state *drbg)
+
+ INIT_WORK(&drbg->seed_work, drbg_async_seed);
+
+- drbg->random_ready.owner = THIS_MODULE;
+- drbg->random_ready.func = drbg_schedule_async_seed;
+-
+- err = add_random_ready_callback(&drbg->random_ready);
++ drbg->random_ready.notifier_call = drbg_schedule_async_seed;
++ err = register_random_ready_notifier(&drbg->random_ready);
+
+ switch (err) {
+ case 0:
+@@ -1525,7 +1524,7 @@ static int drbg_prepare_hrng(struct drbg_state *drbg)
+ fallthrough;
+
+ default:
+- drbg->random_ready.func = NULL;
++ drbg->random_ready.notifier_call = NULL;
+ return err;
+ }
+
+@@ -1629,8 +1628,8 @@ free_everything:
+ */
+ static int drbg_uninstantiate(struct drbg_state *drbg)
+ {
+- if (drbg->random_ready.func) {
+- del_random_ready_callback(&drbg->random_ready);
++ if (drbg->random_ready.notifier_call) {
++ unregister_random_ready_notifier(&drbg->random_ready);
+ cancel_work_sync(&drbg->seed_work);
+ }
+
+diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
+index a11b3208760f3..f6d3a84e3c214 100644
+--- a/crypto/jitterentropy.c
++++ b/crypto/jitterentropy.c
+@@ -265,7 +265,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
+ {
+ __u64 delta2 = jent_delta(ec->last_delta, current_delta);
+ __u64 delta3 = jent_delta(ec->last_delta2, delta2);
+- unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK;
+
+ ec->last_delta = current_delta;
+ ec->last_delta2 = delta2;
+@@ -274,7 +273,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
+ * Insert the result of the comparison of two back-to-back time
+ * deltas.
+ */
+- jent_apt_insert(ec, delta_masked);
++ jent_apt_insert(ec, current_delta);
+
+ if (!current_delta || !delta2 || !delta3) {
+ /* RCT with a stuck bit */
+diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
+index d569c7ed6c800..9d10b846ccf73 100644
+--- a/crypto/pcrypt.c
++++ b/crypto/pcrypt.c
+@@ -78,12 +78,14 @@ static void pcrypt_aead_enc(struct padata_priv *padata)
+ {
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_request *req = pcrypt_request_ctx(preq);
++ int ret;
+
+- padata->info = crypto_aead_encrypt(req);
++ ret = crypto_aead_encrypt(req);
+
+- if (padata->info == -EINPROGRESS)
++ if (ret == -EINPROGRESS)
+ return;
+
++ padata->info = ret;
+ padata_do_serial(padata);
+ }
+
+@@ -123,12 +125,14 @@ static void pcrypt_aead_dec(struct padata_priv *padata)
+ {
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_request *req = pcrypt_request_ctx(preq);
++ int ret;
+
+- padata->info = crypto_aead_decrypt(req);
++ ret = crypto_aead_decrypt(req);
+
+- if (padata->info == -EINPROGRESS)
++ if (ret == -EINPROGRESS)
+ return;
+
++ padata->info = ret;
+ padata_do_serial(padata);
+ }
+
+diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c
+index 8ac3e73e8ea65..9d804831c8b3f 100644
+--- a/crypto/rsa-pkcs1pad.c
++++ b/crypto/rsa-pkcs1pad.c
+@@ -476,6 +476,8 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err)
+ pos++;
+
+ if (digest_info) {
++ if (digest_info->size > dst_len - pos)
++ goto done;
+ if (crypto_memneq(out_buf + pos, digest_info->data,
+ digest_info->size))
+ goto done;
+@@ -495,7 +497,7 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err)
+ sg_nents_for_len(req->src,
+ req->src_len + req->dst_len),
+ req_ctx->out_buf + ctx->key_size,
+- req->dst_len, ctx->key_size);
++ req->dst_len, req->src_len);
+ /* Do the actual verification step. */
+ if (memcmp(req_ctx->out_buf + ctx->key_size, out_buf + pos,
+ req->dst_len) != 0)
+@@ -538,7 +540,7 @@ static int pkcs1pad_verify(struct akcipher_request *req)
+
+ if (WARN_ON(req->dst) ||
+ WARN_ON(!req->dst_len) ||
+- !ctx->key_size || req->src_len < ctx->key_size)
++ !ctx->key_size || req->src_len != ctx->key_size)
+ return -EINVAL;
+
+ req_ctx->out_buf = kmalloc(ctx->key_size + req->dst_len, GFP_KERNEL);
+@@ -621,6 +623,11 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb)
+
+ rsa_alg = crypto_spawn_akcipher_alg(&ctx->spawn);
+
++ if (strcmp(rsa_alg->base.cra_name, "rsa") != 0) {
++ err = -EINVAL;
++ goto err_free_inst;
++ }
++
+ err = -ENAMETOOLONG;
+ hash_name = crypto_attr_alg_name(tb[2]);
+ if (IS_ERR(hash_name)) {
+diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
+index 82b0400985a51..00149657a4bc1 100644
+--- a/crypto/tcrypt.c
++++ b/crypto/tcrypt.c
+@@ -1333,7 +1333,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
+
+ if (bs > XBUFSIZE * PAGE_SIZE) {
+ pr_err("template (%u) too big for buffer (%lu)\n",
+- *b_size, XBUFSIZE * PAGE_SIZE);
++ bs, XBUFSIZE * PAGE_SIZE);
+ goto out;
+ }
+
+@@ -1386,8 +1386,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
+ memset(cur->xbuf[p], 0xff, k);
+
+ skcipher_request_set_crypt(cur->req, cur->sg,
+- cur->sg, *b_size,
+- iv);
++ cur->sg, bs, iv);
+ }
+
+ if (secs) {
+diff --git a/crypto/xts.c b/crypto/xts.c
+index 6c12f30dbdd6d..63c85b9e64e08 100644
+--- a/crypto/xts.c
++++ b/crypto/xts.c
+@@ -466,3 +466,4 @@ MODULE_LICENSE("GPL");
+ MODULE_DESCRIPTION("XTS block cipher mode");
+ MODULE_ALIAS_CRYPTO("xts");
+ MODULE_IMPORT_NS(CRYPTO_INTERNAL);
++MODULE_SOFTDEP("pre: ecb");
+diff --git a/drivers/Makefile b/drivers/Makefile
+index be5d40ae14882..a110338c860c7 100644
+--- a/drivers/Makefile
++++ b/drivers/Makefile
+@@ -41,8 +41,7 @@ obj-$(CONFIG_DMADEVICES) += dma/
+ # SOC specific infrastructure drivers.
+ obj-y += soc/
+
+-obj-$(CONFIG_VIRTIO) += virtio/
+-obj-$(CONFIG_VIRTIO_PCI_LIB) += virtio/
++obj-y += virtio/
+ obj-$(CONFIG_VDPA) += vdpa/
+ obj-$(CONFIG_XEN) += xen/
+
+diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c
+index 580ec796816bc..78ca4987e619e 100644
+--- a/drivers/accessibility/speakup/speakup_dectlk.c
++++ b/drivers/accessibility/speakup/speakup_dectlk.c
+@@ -44,6 +44,7 @@ static struct var_t vars[] = {
+ { CAPS_START, .u.s = {"[:dv ap 160] " } },
+ { CAPS_STOP, .u.s = {"[:dv ap 100 ] " } },
+ { RATE, .u.n = {"[:ra %d] ", 180, 75, 650, 0, 0, NULL } },
++ { PITCH, .u.n = {"[:dv ap %d] ", 122, 50, 350, 0, 0, NULL } },
+ { INFLECTION, .u.n = {"[:dv pr %d] ", 100, 0, 10000, 0, 0, NULL } },
+ { VOL, .u.n = {"[:dv g5 %d] ", 86, 60, 86, 0, 0, NULL } },
+ { PUNCT, .u.n = {"[:pu %c] ", 0, 0, 2, 0, 0, "nsa" } },
+diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c
+index b0cb662233f1a..81aff651a0d49 100644
+--- a/drivers/acpi/ac.c
++++ b/drivers/acpi/ac.c
+@@ -61,6 +61,7 @@ static SIMPLE_DEV_PM_OPS(acpi_ac_pm, NULL, acpi_ac_resume);
+
+ static int ac_sleep_before_get_state_ms;
+ static int ac_check_pmic = 1;
++static int ac_only;
+
+ static struct acpi_driver acpi_ac_driver = {
+ .name = "ac",
+@@ -93,6 +94,11 @@ static int acpi_ac_get_state(struct acpi_ac *ac)
+ if (!ac)
+ return -EINVAL;
+
++ if (ac_only) {
++ ac->state = 1;
++ return 0;
++ }
++
+ status = acpi_evaluate_integer(ac->device->handle, "_PSR", NULL,
+ &ac->state);
+ if (ACPI_FAILURE(status)) {
+@@ -200,6 +206,12 @@ static int __init ac_do_not_check_pmic_quirk(const struct dmi_system_id *d)
+ return 0;
+ }
+
++static int __init ac_only_quirk(const struct dmi_system_id *d)
++{
++ ac_only = 1;
++ return 0;
++}
++
+ /* Please keep this list alphabetically sorted */
+ static const struct dmi_system_id ac_dmi_table[] __initconst = {
+ {
+@@ -209,6 +221,13 @@ static const struct dmi_system_id ac_dmi_table[] __initconst = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "EF20EA"),
+ },
+ },
++ {
++ /* Kodlix GK45 returning incorrect state */
++ .callback = ac_only_quirk,
++ .matches = {
++ DMI_MATCH(DMI_PRODUCT_NAME, "GK45"),
++ },
++ },
+ {
+ /* Lenovo Ideapad Miix 320, AXP288 PMIC, separate fuel-gauge */
+ .callback = ac_do_not_check_pmic_quirk,
+diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
+index d41b810e367c4..4366d36ef1198 100644
+--- a/drivers/acpi/acpica/acglobal.h
++++ b/drivers/acpi/acpica/acglobal.h
+@@ -226,6 +226,8 @@ extern struct acpi_bit_register_info
+ acpi_gbl_bit_register_info[ACPI_NUM_BITREG];
+ ACPI_GLOBAL(u8, acpi_gbl_sleep_type_a);
+ ACPI_GLOBAL(u8, acpi_gbl_sleep_type_b);
++ACPI_GLOBAL(u8, acpi_gbl_sleep_type_a_s0);
++ACPI_GLOBAL(u8, acpi_gbl_sleep_type_b_s0);
+
+ /*****************************************************************************
+ *
+diff --git a/drivers/acpi/acpica/exfield.c b/drivers/acpi/acpica/exfield.c
+index 06f3c9df1e22d..8618500f23b39 100644
+--- a/drivers/acpi/acpica/exfield.c
++++ b/drivers/acpi/acpica/exfield.c
+@@ -330,12 +330,7 @@ acpi_ex_write_data_to_field(union acpi_operand_object *source_desc,
+ obj_desc->field.base_byte_offset,
+ source_desc->buffer.pointer, data_length);
+
+- if ((obj_desc->field.region_obj->region.address ==
+- PCC_MASTER_SUBSPACE
+- && MASTER_SUBSPACE_COMMAND(obj_desc->field.
+- base_byte_offset))
+- || GENERIC_SUBSPACE_COMMAND(obj_desc->field.
+- base_byte_offset)) {
++ if (MASTER_SUBSPACE_COMMAND(obj_desc->field.base_byte_offset)) {
+
+ /* Perform the write */
+
+diff --git a/drivers/acpi/acpica/exoparg1.c b/drivers/acpi/acpica/exoparg1.c
+index b639e930d6429..44b7c350ed5ca 100644
+--- a/drivers/acpi/acpica/exoparg1.c
++++ b/drivers/acpi/acpica/exoparg1.c
+@@ -1007,7 +1007,8 @@ acpi_status acpi_ex_opcode_1A_0T_1R(struct acpi_walk_state *walk_state)
+ (walk_state, return_desc,
+ &temp_desc);
+ if (ACPI_FAILURE(status)) {
+- goto cleanup;
++ return_ACPI_STATUS
++ (status);
+ }
+
+ return_desc = temp_desc;
+diff --git a/drivers/acpi/acpica/hwesleep.c b/drivers/acpi/acpica/hwesleep.c
+index 803402aefaeb6..7ee2939c08cd4 100644
+--- a/drivers/acpi/acpica/hwesleep.c
++++ b/drivers/acpi/acpica/hwesleep.c
+@@ -104,7 +104,9 @@ acpi_status acpi_hw_extended_sleep(u8 sleep_state)
+
+ /* Flush caches, as per ACPI specification */
+
+- ACPI_FLUSH_CPU_CACHE();
++ if (sleep_state < ACPI_STATE_S4) {
++ ACPI_FLUSH_CPU_CACHE();
++ }
+
+ status = acpi_os_enter_sleep(sleep_state, sleep_control, 0);
+ if (status == AE_CTRL_TERMINATE) {
+@@ -147,17 +149,13 @@ acpi_status acpi_hw_extended_sleep(u8 sleep_state)
+
+ acpi_status acpi_hw_extended_wake_prep(u8 sleep_state)
+ {
+- acpi_status status;
+ u8 sleep_type_value;
+
+ ACPI_FUNCTION_TRACE(hw_extended_wake_prep);
+
+- status = acpi_get_sleep_type_data(ACPI_STATE_S0,
+- &acpi_gbl_sleep_type_a,
+- &acpi_gbl_sleep_type_b);
+- if (ACPI_SUCCESS(status)) {
++ if (acpi_gbl_sleep_type_a_s0 != ACPI_SLEEP_TYPE_INVALID) {
+ sleep_type_value =
+- ((acpi_gbl_sleep_type_a << ACPI_X_SLEEP_TYPE_POSITION) &
++ ((acpi_gbl_sleep_type_a_s0 << ACPI_X_SLEEP_TYPE_POSITION) &
+ ACPI_X_SLEEP_TYPE_MASK);
+
+ (void)acpi_write((u64)(sleep_type_value | ACPI_X_SLEEP_ENABLE),
+diff --git a/drivers/acpi/acpica/hwsleep.c b/drivers/acpi/acpica/hwsleep.c
+index 14baa13bf8482..5efa3d8e483e0 100644
+--- a/drivers/acpi/acpica/hwsleep.c
++++ b/drivers/acpi/acpica/hwsleep.c
+@@ -110,7 +110,9 @@ acpi_status acpi_hw_legacy_sleep(u8 sleep_state)
+
+ /* Flush caches, as per ACPI specification */
+
+- ACPI_FLUSH_CPU_CACHE();
++ if (sleep_state < ACPI_STATE_S4) {
++ ACPI_FLUSH_CPU_CACHE();
++ }
+
+ status = acpi_os_enter_sleep(sleep_state, pm1a_control, pm1b_control);
+ if (status == AE_CTRL_TERMINATE) {
+@@ -179,7 +181,7 @@ acpi_status acpi_hw_legacy_sleep(u8 sleep_state)
+
+ acpi_status acpi_hw_legacy_wake_prep(u8 sleep_state)
+ {
+- acpi_status status;
++ acpi_status status = AE_OK;
+ struct acpi_bit_register_info *sleep_type_reg_info;
+ struct acpi_bit_register_info *sleep_enable_reg_info;
+ u32 pm1a_control;
+@@ -192,10 +194,7 @@ acpi_status acpi_hw_legacy_wake_prep(u8 sleep_state)
+ * This is unclear from the ACPI Spec, but it is required
+ * by some machines.
+ */
+- status = acpi_get_sleep_type_data(ACPI_STATE_S0,
+- &acpi_gbl_sleep_type_a,
+- &acpi_gbl_sleep_type_b);
+- if (ACPI_SUCCESS(status)) {
++ if (acpi_gbl_sleep_type_a_s0 != ACPI_SLEEP_TYPE_INVALID) {
+ sleep_type_reg_info =
+ acpi_hw_get_bit_register_info(ACPI_BITREG_SLEEP_TYPE);
+ sleep_enable_reg_info =
+@@ -216,9 +215,9 @@ acpi_status acpi_hw_legacy_wake_prep(u8 sleep_state)
+
+ /* Insert the SLP_TYP bits */
+
+- pm1a_control |= (acpi_gbl_sleep_type_a <<
++ pm1a_control |= (acpi_gbl_sleep_type_a_s0 <<
+ sleep_type_reg_info->bit_position);
+- pm1b_control |= (acpi_gbl_sleep_type_b <<
++ pm1b_control |= (acpi_gbl_sleep_type_b_s0 <<
+ sleep_type_reg_info->bit_position);
+
+ /* Write the control registers and ignore any errors */
+diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c
+index 89b12afed564e..ba77598ee43e8 100644
+--- a/drivers/acpi/acpica/hwxfsleep.c
++++ b/drivers/acpi/acpica/hwxfsleep.c
+@@ -162,8 +162,6 @@ acpi_status acpi_enter_sleep_state_s4bios(void)
+ return_ACPI_STATUS(status);
+ }
+
+- ACPI_FLUSH_CPU_CACHE();
+-
+ status = acpi_hw_write_port(acpi_gbl_FADT.smi_command,
+ (u32)acpi_gbl_FADT.s4_bios_request, 8);
+ if (ACPI_FAILURE(status)) {
+@@ -217,6 +215,13 @@ acpi_status acpi_enter_sleep_state_prep(u8 sleep_state)
+ return_ACPI_STATUS(status);
+ }
+
++ status = acpi_get_sleep_type_data(ACPI_STATE_S0,
++ &acpi_gbl_sleep_type_a_s0,
++ &acpi_gbl_sleep_type_b_s0);
++ if (ACPI_FAILURE(status)) {
++ acpi_gbl_sleep_type_a_s0 = ACPI_SLEEP_TYPE_INVALID;
++ }
++
+ /* Execute the _PTS method (Prepare To Sleep) */
+
+ arg_list.count = 1;
+diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c
+index 915c2433463d7..e7c30ce06e189 100644
+--- a/drivers/acpi/acpica/nswalk.c
++++ b/drivers/acpi/acpica/nswalk.c
+@@ -169,6 +169,9 @@ acpi_ns_walk_namespace(acpi_object_type type,
+
+ if (start_node == ACPI_ROOT_OBJECT) {
+ start_node = acpi_gbl_root_node;
++ if (!start_node) {
++ return_ACPI_STATUS(AE_NO_NAMESPACE);
++ }
+ }
+
+ /* Null child means "get first node" */
+diff --git a/drivers/acpi/acpica/utdelete.c b/drivers/acpi/acpica/utdelete.c
+index e5ba9795ec696..8d7736d2d2699 100644
+--- a/drivers/acpi/acpica/utdelete.c
++++ b/drivers/acpi/acpica/utdelete.c
+@@ -422,6 +422,7 @@ acpi_ut_update_ref_count(union acpi_operand_object *object, u32 action)
+ ACPI_WARNING((AE_INFO,
+ "Obj %p, Reference Count is already zero, cannot decrement\n",
+ object));
++ return;
+ }
+
+ ACPI_DEBUG_PRINT_RAW((ACPI_DB_ALLOCATIONS,
+diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c
+index 19e50fcbf4d6f..598fd19b65fa4 100644
+--- a/drivers/acpi/apei/bert.c
++++ b/drivers/acpi/apei/bert.c
+@@ -29,6 +29,7 @@
+
+ #undef pr_fmt
+ #define pr_fmt(fmt) "BERT: " fmt
++#define ACPI_BERT_PRINT_MAX_LEN 1024
+
+ static int bert_disable;
+
+@@ -58,8 +59,11 @@ static void __init bert_print_all(struct acpi_bert_region *region,
+ }
+
+ pr_info_once("Error records from previous boot:\n");
+-
+- cper_estatus_print(KERN_INFO HW_ERR, estatus);
++ if (region_len < ACPI_BERT_PRINT_MAX_LEN)
++ cper_estatus_print(KERN_INFO HW_ERR, estatus);
++ else
++ pr_info_once("Max print length exceeded, table data is available at:\n"
++ "/sys/firmware/acpi/tables/data/BERT");
+
+ /*
+ * Because the boot error source is "one-time polled" type,
+@@ -77,7 +81,7 @@ static int __init setup_bert_disable(char *str)
+ {
+ bert_disable = 1;
+
+- return 0;
++ return 1;
+ }
+ __setup("bert_disable", setup_bert_disable);
+
+diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c
+index 242f3c2d55330..698d67cee0527 100644
+--- a/drivers/acpi/apei/erst.c
++++ b/drivers/acpi/apei/erst.c
+@@ -891,7 +891,7 @@ EXPORT_SYMBOL_GPL(erst_clear);
+ static int __init setup_erst_disable(char *str)
+ {
+ erst_disable = 1;
+- return 0;
++ return 1;
+ }
+
+ __setup("erst_disable", setup_erst_disable);
+diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c
+index 277f00b288d14..317bba602ad54 100644
+--- a/drivers/acpi/apei/hest.c
++++ b/drivers/acpi/apei/hest.c
+@@ -223,7 +223,7 @@ err:
+ static int __init setup_hest_disable(char *str)
+ {
+ hest_disable = HEST_DISABLED;
+- return 0;
++ return 1;
+ }
+
+ __setup("hest_disable", setup_hest_disable);
+diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
+index 3b23fb775ac45..f2f8f05662deb 100644
+--- a/drivers/acpi/arm64/iort.c
++++ b/drivers/acpi/arm64/iort.c
+@@ -1361,9 +1361,17 @@ static void __init arm_smmu_v3_pmcg_init_resources(struct resource *res,
+ res[0].start = pmcg->page0_base_address;
+ res[0].end = pmcg->page0_base_address + SZ_4K - 1;
+ res[0].flags = IORESOURCE_MEM;
+- res[1].start = pmcg->page1_base_address;
+- res[1].end = pmcg->page1_base_address + SZ_4K - 1;
+- res[1].flags = IORESOURCE_MEM;
++ /*
++ * The initial version in DEN0049C lacked a way to describe register
++ * page 1, which makes it broken for most PMCG implementations; in
++ * that case, just let the driver fail gracefully if it expects to
++ * find a second memory resource.
++ */
++ if (node->revision > 0) {
++ res[1].start = pmcg->page1_base_address;
++ res[1].end = pmcg->page1_base_address + SZ_4K - 1;
++ res[1].flags = IORESOURCE_MEM;
++ }
+
+ if (pmcg->overflow_gsiv)
+ acpi_iort_register_irq(pmcg->overflow_gsiv, "overflow",
+diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
+index dae91f906cea9..56db7b4da5140 100644
+--- a/drivers/acpi/battery.c
++++ b/drivers/acpi/battery.c
+@@ -53,12 +53,17 @@ static int battery_bix_broken_package;
+ static int battery_notification_delay_ms;
+ static int battery_ac_is_broken;
+ static int battery_check_pmic = 1;
++static int battery_quirk_notcharging;
+ static unsigned int cache_time = 1000;
+ module_param(cache_time, uint, 0644);
+ MODULE_PARM_DESC(cache_time, "cache time in milliseconds");
+
+ static const struct acpi_device_id battery_device_ids[] = {
+ {"PNP0C0A", 0},
++
++ /* Microsoft Surface Go 3 */
++ {"MSHW0146", 0},
++
+ {"", 0},
+ };
+
+@@ -169,7 +174,7 @@ static int acpi_battery_is_charged(struct acpi_battery *battery)
+ return 1;
+
+ /* fallback to using design values for broken batteries */
+- if (battery->design_capacity == battery->capacity_now)
++ if (battery->design_capacity <= battery->capacity_now)
+ return 1;
+
+ /* we don't do any sort of metric based on percentages */
+@@ -217,6 +222,8 @@ static int acpi_battery_get_property(struct power_supply *psy,
+ val->intval = POWER_SUPPLY_STATUS_CHARGING;
+ else if (acpi_battery_is_charged(battery))
+ val->intval = POWER_SUPPLY_STATUS_FULL;
++ else if (battery_quirk_notcharging)
++ val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING;
+ else
+ val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
+ break;
+@@ -1111,6 +1118,12 @@ battery_do_not_check_pmic_quirk(const struct dmi_system_id *d)
+ return 0;
+ }
+
++static int __init battery_quirk_not_charging(const struct dmi_system_id *d)
++{
++ battery_quirk_notcharging = 1;
++ return 0;
++}
++
+ static const struct dmi_system_id bat_dmi_table[] __initconst = {
+ {
+ /* NEC LZ750/LS */
+@@ -1155,6 +1168,27 @@ static const struct dmi_system_id bat_dmi_table[] __initconst = {
+ DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
+ },
+ },
++ {
++ /*
++ * On Lenovo ThinkPads the BIOS specification defines
++ * a state when the bits for charging and discharging
++ * are both set to 0. That state is "Not Charging".
++ */
++ .callback = battery_quirk_not_charging,
++ .ident = "Lenovo ThinkPad",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
++ DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad"),
++ },
++ },
++ {
++ /* Microsoft Surface Go 3 */
++ .callback = battery_notification_delay_quirk,
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "Surface Go 3"),
++ },
++ },
+ {},
+ };
+
+diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
+index fa923a9292244..3500744e6862e 100644
+--- a/drivers/acpi/bus.c
++++ b/drivers/acpi/bus.c
+@@ -98,8 +98,8 @@ int acpi_bus_get_status(struct acpi_device *device)
+ acpi_status status;
+ unsigned long long sta;
+
+- if (acpi_device_always_present(device)) {
+- acpi_set_device_status(device, ACPI_STA_DEFAULT);
++ if (acpi_device_override_status(device, &sta)) {
++ acpi_set_device_status(device, sta);
+ return 0;
+ }
+
+@@ -332,21 +332,32 @@ static void acpi_bus_osc_negotiate_platform_control(void)
+ if (ACPI_FAILURE(acpi_run_osc(handle, &context)))
+ return;
+
+- kfree(context.ret.pointer);
++ capbuf_ret = context.ret.pointer;
++ if (context.ret.length <= OSC_SUPPORT_DWORD) {
++ kfree(context.ret.pointer);
++ return;
++ }
+
+- /* Now run _OSC again with query flag clear */
++ /*
++ * Now run _OSC again with query flag clear and with the caps
++ * supported by both the OS and the platform.
++ */
+ capbuf[OSC_QUERY_DWORD] = 0;
++ capbuf[OSC_SUPPORT_DWORD] = capbuf_ret[OSC_SUPPORT_DWORD];
++ kfree(context.ret.pointer);
+
+ if (ACPI_FAILURE(acpi_run_osc(handle, &context)))
+ return;
+
+ capbuf_ret = context.ret.pointer;
+- osc_sb_apei_support_acked =
+- capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT;
+- osc_pc_lpi_support_confirmed =
+- capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT;
+- osc_sb_native_usb4_support_confirmed =
+- capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_NATIVE_USB4_SUPPORT;
++ if (context.ret.length > OSC_SUPPORT_DWORD) {
++ osc_sb_apei_support_acked =
++ capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT;
++ osc_pc_lpi_support_confirmed =
++ capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT;
++ osc_sb_native_usb4_support_confirmed =
++ capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_NATIVE_USB4_SUPPORT;
++ }
+
+ kfree(context.ret.pointer);
+ }
+diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
+index bd482108310cf..71b87c16f92da 100644
+--- a/drivers/acpi/cppc_acpi.c
++++ b/drivers/acpi/cppc_acpi.c
+@@ -411,7 +411,7 @@ bool acpi_cpc_valid(void)
+ struct cpc_desc *cpc_ptr;
+ int cpu;
+
+- for_each_possible_cpu(cpu) {
++ for_each_present_cpu(cpu) {
+ cpc_ptr = per_cpu(cpc_desc_ptr, cpu);
+ if (!cpc_ptr)
+ return false;
+@@ -703,6 +703,11 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
+ cpc_obj = &out_obj->package.elements[0];
+ if (cpc_obj->type == ACPI_TYPE_INTEGER) {
+ num_ent = cpc_obj->integer.value;
++ if (num_ent <= 1) {
++ pr_debug("Unexpected _CPC NumEntries value (%d) for CPU:%d\n",
++ num_ent, pr->id);
++ goto out_free;
++ }
+ } else {
+ pr_debug("Unexpected entry type(%d) for NumEntries\n",
+ cpc_obj->type);
+@@ -1011,7 +1016,14 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val)
+ static int cppc_get_perf(int cpunum, enum cppc_regs reg_idx, u64 *perf)
+ {
+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
+- struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
++ struct cpc_register_resource *reg;
++
++ if (!cpc_desc) {
++ pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
++ return -ENODEV;
++ }
++
++ reg = &cpc_desc->cpc_regs[reg_idx];
+
+ if (CPC_IN_PCC(reg)) {
+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
+diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
+index e629e891d1bb3..9b859ff976e89 100644
+--- a/drivers/acpi/ec.c
++++ b/drivers/acpi/ec.c
+@@ -166,6 +166,7 @@ struct acpi_ec_query {
+ struct transaction transaction;
+ struct work_struct work;
+ struct acpi_ec_query_handler *handler;
++ struct acpi_ec *ec;
+ };
+
+ static int acpi_ec_query(struct acpi_ec *ec, u8 *data);
+@@ -452,6 +453,7 @@ static void acpi_ec_submit_query(struct acpi_ec *ec)
+ ec_dbg_evt("Command(%s) submitted/blocked",
+ acpi_ec_cmd_string(ACPI_EC_COMMAND_QUERY));
+ ec->nr_pending_queries++;
++ ec->events_in_progress++;
+ queue_work(ec_wq, &ec->work);
+ }
+ }
+@@ -518,7 +520,7 @@ static void acpi_ec_enable_event(struct acpi_ec *ec)
+ #ifdef CONFIG_PM_SLEEP
+ static void __acpi_ec_flush_work(void)
+ {
+- drain_workqueue(ec_wq); /* flush ec->work */
++ flush_workqueue(ec_wq); /* flush ec->work */
+ flush_workqueue(ec_query_wq); /* flush queries */
+ }
+
+@@ -1103,7 +1105,7 @@ void acpi_ec_remove_query_handler(struct acpi_ec *ec, u8 query_bit)
+ }
+ EXPORT_SYMBOL_GPL(acpi_ec_remove_query_handler);
+
+-static struct acpi_ec_query *acpi_ec_create_query(u8 *pval)
++static struct acpi_ec_query *acpi_ec_create_query(struct acpi_ec *ec, u8 *pval)
+ {
+ struct acpi_ec_query *q;
+ struct transaction *t;
+@@ -1111,11 +1113,13 @@ static struct acpi_ec_query *acpi_ec_create_query(u8 *pval)
+ q = kzalloc(sizeof (struct acpi_ec_query), GFP_KERNEL);
+ if (!q)
+ return NULL;
++
+ INIT_WORK(&q->work, acpi_ec_event_processor);
+ t = &q->transaction;
+ t->command = ACPI_EC_COMMAND_QUERY;
+ t->rdata = pval;
+ t->rlen = 1;
++ q->ec = ec;
+ return q;
+ }
+
+@@ -1132,13 +1136,21 @@ static void acpi_ec_event_processor(struct work_struct *work)
+ {
+ struct acpi_ec_query *q = container_of(work, struct acpi_ec_query, work);
+ struct acpi_ec_query_handler *handler = q->handler;
++ struct acpi_ec *ec = q->ec;
+
+ ec_dbg_evt("Query(0x%02x) started", handler->query_bit);
++
+ if (handler->func)
+ handler->func(handler->data);
+ else if (handler->handle)
+ acpi_evaluate_object(handler->handle, NULL, NULL, NULL);
++
+ ec_dbg_evt("Query(0x%02x) stopped", handler->query_bit);
++
++ spin_lock_irq(&ec->lock);
++ ec->queries_in_progress--;
++ spin_unlock_irq(&ec->lock);
++
+ acpi_ec_delete_query(q);
+ }
+
+@@ -1148,7 +1160,7 @@ static int acpi_ec_query(struct acpi_ec *ec, u8 *data)
+ int result;
+ struct acpi_ec_query *q;
+
+- q = acpi_ec_create_query(&value);
++ q = acpi_ec_create_query(ec, &value);
+ if (!q)
+ return -ENOMEM;
+
+@@ -1170,19 +1182,20 @@ static int acpi_ec_query(struct acpi_ec *ec, u8 *data)
+ }
+
+ /*
+- * It is reported that _Qxx are evaluated in a parallel way on
+- * Windows:
++ * It is reported that _Qxx are evaluated in a parallel way on Windows:
+ * https://bugzilla.kernel.org/show_bug.cgi?id=94411
+ *
+- * Put this log entry before schedule_work() in order to make
+- * it appearing before any other log entries occurred during the
+- * work queue execution.
++ * Put this log entry before queue_work() to make it appear in the log
++ * before any other messages emitted during workqueue handling.
+ */
+ ec_dbg_evt("Query(0x%02x) scheduled", value);
+- if (!queue_work(ec_query_wq, &q->work)) {
+- ec_dbg_evt("Query(0x%02x) overlapped", value);
+- result = -EBUSY;
+- }
++
++ spin_lock_irq(&ec->lock);
++
++ ec->queries_in_progress++;
++ queue_work(ec_query_wq, &q->work);
++
++ spin_unlock_irq(&ec->lock);
+
+ err_exit:
+ if (result)
+@@ -1240,6 +1253,10 @@ static void acpi_ec_event_handler(struct work_struct *work)
+ ec_dbg_evt("Event stopped");
+
+ acpi_ec_check_event(ec);
++
++ spin_lock_irqsave(&ec->lock, flags);
++ ec->events_in_progress--;
++ spin_unlock_irqrestore(&ec->lock, flags);
+ }
+
+ static void acpi_ec_handle_interrupt(struct acpi_ec *ec)
+@@ -2021,6 +2038,7 @@ void acpi_ec_set_gpe_wake_mask(u8 action)
+
+ bool acpi_ec_dispatch_gpe(void)
+ {
++ bool work_in_progress;
+ u32 ret;
+
+ if (!first_ec)
+@@ -2041,8 +2059,19 @@ bool acpi_ec_dispatch_gpe(void)
+ if (ret == ACPI_INTERRUPT_HANDLED)
+ pm_pr_dbg("ACPI EC GPE dispatched\n");
+
+- /* Flush the event and query workqueues. */
+- acpi_ec_flush_work();
++ /* Drain EC work. */
++ do {
++ acpi_ec_flush_work();
++
++ pm_pr_dbg("ACPI EC work flushed\n");
++
++ spin_lock_irq(&first_ec->lock);
++
++ work_in_progress = first_ec->events_in_progress +
++ first_ec->queries_in_progress > 0;
++
++ spin_unlock_irq(&first_ec->lock);
++ } while (work_in_progress && !pm_wakeup_pending());
+
+ return false;
+ }
+diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
+index d91b560e88674..54b2be94d23dc 100644
+--- a/drivers/acpi/internal.h
++++ b/drivers/acpi/internal.h
+@@ -183,6 +183,8 @@ struct acpi_ec {
+ struct work_struct work;
+ unsigned long timestamp;
+ unsigned long nr_pending_queries;
++ unsigned int events_in_progress;
++ unsigned int queries_in_progress;
+ bool busy_polling;
+ unsigned int polling_guard;
+ };
+diff --git a/drivers/acpi/pmic/intel_pmic.c b/drivers/acpi/pmic/intel_pmic.c
+index a371f273f99dd..9cde299eba880 100644
+--- a/drivers/acpi/pmic/intel_pmic.c
++++ b/drivers/acpi/pmic/intel_pmic.c
+@@ -211,31 +211,36 @@ static acpi_status intel_pmic_regs_handler(u32 function,
+ void *handler_context, void *region_context)
+ {
+ struct intel_pmic_opregion *opregion = region_context;
+- int result = 0;
++ int result = -EINVAL;
++
++ if (function == ACPI_WRITE) {
++ switch (address) {
++ case 0:
++ return AE_OK;
++ case 1:
++ opregion->ctx.addr |= (*value64 & 0xff) << 8;
++ return AE_OK;
++ case 2:
++ opregion->ctx.addr |= *value64 & 0xff;
++ return AE_OK;
++ case 3:
++ opregion->ctx.val = *value64 & 0xff;
++ return AE_OK;
++ case 4:
++ if (*value64) {
++ result = regmap_write(opregion->regmap, opregion->ctx.addr,
++ opregion->ctx.val);
++ } else {
++ result = regmap_read(opregion->regmap, opregion->ctx.addr,
++ &opregion->ctx.val);
++ }
++ opregion->ctx.addr = 0;
++ }
++ }
+
+- switch (address) {
+- case 0:
+- return AE_OK;
+- case 1:
+- opregion->ctx.addr |= (*value64 & 0xff) << 8;
+- return AE_OK;
+- case 2:
+- opregion->ctx.addr |= *value64 & 0xff;
++ if (function == ACPI_READ && address == 3) {
++ *value64 = opregion->ctx.val;
+ return AE_OK;
+- case 3:
+- opregion->ctx.val = *value64 & 0xff;
+- return AE_OK;
+- case 4:
+- if (*value64) {
+- result = regmap_write(opregion->regmap, opregion->ctx.addr,
+- opregion->ctx.val);
+- } else {
+- result = regmap_read(opregion->regmap, opregion->ctx.addr,
+- &opregion->ctx.val);
+- if (result == 0)
+- *value64 = opregion->ctx.val;
+- }
+- memset(&opregion->ctx, 0x00, sizeof(opregion->ctx));
+ }
+
+ if (result < 0) {
+diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
+index f0ed4414edb1f..c95eedd58f5bf 100644
+--- a/drivers/acpi/power.c
++++ b/drivers/acpi/power.c
+@@ -52,7 +52,6 @@ struct acpi_power_resource {
+ u32 order;
+ unsigned int ref_count;
+ u8 state;
+- bool wakeup_enabled;
+ struct mutex resource_lock;
+ struct list_head dependents;
+ };
+@@ -615,20 +614,19 @@ int acpi_power_wakeup_list_init(struct list_head *list, int *system_level_p)
+
+ list_for_each_entry(entry, list, node) {
+ struct acpi_power_resource *resource = entry->resource;
+- int result;
+ u8 state;
+
+ mutex_lock(&resource->resource_lock);
+
+- result = acpi_power_get_state(resource, &state);
+- if (result) {
+- mutex_unlock(&resource->resource_lock);
+- return result;
+- }
+- if (state == ACPI_POWER_RESOURCE_STATE_ON) {
+- resource->ref_count++;
+- resource->wakeup_enabled = true;
+- }
++ /*
++ * Make sure that the power resource state and its reference
++ * counter value are consistent with each other.
++ */
++ if (!resource->ref_count &&
++ !acpi_power_get_state(resource, &state) &&
++ state == ACPI_POWER_RESOURCE_STATE_ON)
++ __acpi_power_off(resource);
++
+ if (system_level > resource->system_level)
+ system_level = resource->system_level;
+
+@@ -711,7 +709,6 @@ int acpi_device_sleep_wake(struct acpi_device *dev,
+ */
+ int acpi_enable_wakeup_device_power(struct acpi_device *dev, int sleep_state)
+ {
+- struct acpi_power_resource_entry *entry;
+ int err = 0;
+
+ if (!dev || !dev->wakeup.flags.valid)
+@@ -722,26 +719,13 @@ int acpi_enable_wakeup_device_power(struct acpi_device *dev, int sleep_state)
+ if (dev->wakeup.prepare_count++)
+ goto out;
+
+- list_for_each_entry(entry, &dev->wakeup.resources, node) {
+- struct acpi_power_resource *resource = entry->resource;
+-
+- mutex_lock(&resource->resource_lock);
+-
+- if (!resource->wakeup_enabled) {
+- err = acpi_power_on_unlocked(resource);
+- if (!err)
+- resource->wakeup_enabled = true;
+- }
+-
+- mutex_unlock(&resource->resource_lock);
+-
+- if (err) {
+- dev_err(&dev->dev,
+- "Cannot turn wakeup power resources on\n");
+- dev->wakeup.flags.valid = 0;
+- goto out;
+- }
++ err = acpi_power_on_list(&dev->wakeup.resources);
++ if (err) {
++ dev_err(&dev->dev, "Cannot turn on wakeup power resources\n");
++ dev->wakeup.flags.valid = 0;
++ goto out;
+ }
++
+ /*
+ * Passing 3 as the third argument below means the device may be
+ * put into arbitrary power state afterward.
+@@ -771,39 +755,31 @@ int acpi_disable_wakeup_device_power(struct acpi_device *dev)
+
+ mutex_lock(&acpi_device_lock);
+
+- if (--dev->wakeup.prepare_count > 0)
++ /* Do nothing if wakeup power has not been enabled for this device. */
++ if (dev->wakeup.prepare_count <= 0)
+ goto out;
+
+- /*
+- * Executing the code below even if prepare_count is already zero when
+- * the function is called may be useful, for example for initialisation.
+- */
+- if (dev->wakeup.prepare_count < 0)
+- dev->wakeup.prepare_count = 0;
++ if (--dev->wakeup.prepare_count > 0)
++ goto out;
+
+ err = acpi_device_sleep_wake(dev, 0, 0, 0);
+ if (err)
+ goto out;
+
++ /*
++ * All of the power resources in the list need to be turned off even if
++ * there are errors.
++ */
+ list_for_each_entry(entry, &dev->wakeup.resources, node) {
+- struct acpi_power_resource *resource = entry->resource;
+-
+- mutex_lock(&resource->resource_lock);
+-
+- if (resource->wakeup_enabled) {
+- err = acpi_power_off_unlocked(resource);
+- if (!err)
+- resource->wakeup_enabled = false;
+- }
+-
+- mutex_unlock(&resource->resource_lock);
++ int ret;
+
+- if (err) {
+- dev_err(&dev->dev,
+- "Cannot turn wakeup power resources off\n");
+- dev->wakeup.flags.valid = 0;
+- break;
+- }
++ ret = acpi_power_off(entry->resource);
++ if (ret && !err)
++ err = ret;
++ }
++ if (err) {
++ dev_err(&dev->dev, "Cannot turn off wakeup power resources\n");
++ dev->wakeup.flags.valid = 0;
+ }
+
+ out:
+diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
+index f37fba9e5ba0b..aedcb92491f2e 100644
+--- a/drivers/acpi/processor_idle.c
++++ b/drivers/acpi/processor_idle.c
+@@ -789,9 +789,11 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
+ state->enter = acpi_idle_enter;
+
+ state->flags = 0;
+- if (cx->type == ACPI_STATE_C1 || cx->type == ACPI_STATE_C2) {
++ if (cx->type == ACPI_STATE_C1 || cx->type == ACPI_STATE_C2 ||
++ cx->type == ACPI_STATE_C3) {
+ state->enter_dead = acpi_idle_play_dead;
+- drv->safe_state_index = count;
++ if (cx->type != ACPI_STATE_C3)
++ drv->safe_state_index = count;
+ }
+ /*
+ * Halt-induced C1 is not good for ->enter_s2idle, because it
+@@ -1075,6 +1077,11 @@ static int flatten_lpi_states(struct acpi_processor *pr,
+ return 0;
+ }
+
++int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
++{
++ return -EOPNOTSUPP;
++}
++
+ static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
+ {
+ int ret, i;
+@@ -1083,6 +1090,11 @@ static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
+ struct acpi_device *d = NULL;
+ struct acpi_lpi_states_array info[2], *tmp, *prev, *curr;
+
++ /* make sure our architecture has support */
++ ret = acpi_processor_ffh_lpi_probe(pr->id);
++ if (ret == -EOPNOTSUPP)
++ return ret;
++
+ if (!osc_pc_lpi_support_confirmed)
+ return -EOPNOTSUPP;
+
+@@ -1134,11 +1146,6 @@ static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
+ return 0;
+ }
+
+-int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
+-{
+- return -ENODEV;
+-}
+-
+ int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi)
+ {
+ return -ENODEV;
+diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
+index e312ebaed8db4..11e4c930a9813 100644
+--- a/drivers/acpi/property.c
++++ b/drivers/acpi/property.c
+@@ -685,7 +685,7 @@ int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
+ */
+ if (obj->type == ACPI_TYPE_LOCAL_REFERENCE) {
+ if (index)
+- return -EINVAL;
++ return -ENOENT;
+
+ ret = acpi_bus_get_device(obj->reference.handle, &device);
+ if (ret)
+@@ -1090,15 +1090,10 @@ struct fwnode_handle *acpi_node_get_parent(const struct fwnode_handle *fwnode)
+ /* All data nodes have parent pointer so just return that */
+ return to_acpi_data_node(fwnode)->parent;
+ } else if (is_acpi_device_node(fwnode)) {
+- acpi_handle handle, parent_handle;
+-
+- handle = to_acpi_device_node(fwnode)->handle;
+- if (ACPI_SUCCESS(acpi_get_parent(handle, &parent_handle))) {
+- struct acpi_device *adev;
++ struct device *dev = to_acpi_device_node(fwnode)->dev.parent;
+
+- if (!acpi_bus_get_device(parent_handle, &adev))
+- return acpi_fwnode_handle(adev);
+- }
++ if (dev)
++ return acpi_fwnode_handle(to_acpi_device(dev));
+ }
+
+ return NULL;
+diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c
+index ee78a210c6068..3c25ce8c95ba1 100644
+--- a/drivers/acpi/resource.c
++++ b/drivers/acpi/resource.c
+@@ -16,6 +16,7 @@
+ #include <linux/ioport.h>
+ #include <linux/slab.h>
+ #include <linux/irq.h>
++#include <linux/dmi.h>
+
+ #ifdef CONFIG_X86
+ #define valid_IRQ(i) (((i) != 0) && ((i) != 2))
+@@ -380,9 +381,58 @@ unsigned int acpi_dev_get_irq_type(int triggering, int polarity)
+ }
+ EXPORT_SYMBOL_GPL(acpi_dev_get_irq_type);
+
++static const struct dmi_system_id medion_laptop[] = {
++ {
++ .ident = "MEDION P15651",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "MEDION"),
++ DMI_MATCH(DMI_BOARD_NAME, "M15T"),
++ },
++ },
++ {
++ .ident = "MEDION S17405",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "MEDION"),
++ DMI_MATCH(DMI_BOARD_NAME, "M17T"),
++ },
++ },
++ { }
++};
++
++struct irq_override_cmp {
++ const struct dmi_system_id *system;
++ unsigned char irq;
++ unsigned char triggering;
++ unsigned char polarity;
++ unsigned char shareable;
++};
++
++static const struct irq_override_cmp skip_override_table[] = {
++ { medion_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 },
++};
++
++static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity,
++ u8 shareable)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(skip_override_table); i++) {
++ const struct irq_override_cmp *entry = &skip_override_table[i];
++
++ if (dmi_check_system(entry->system) &&
++ entry->irq == gsi &&
++ entry->triggering == triggering &&
++ entry->polarity == polarity &&
++ entry->shareable == shareable)
++ return false;
++ }
++
++ return true;
++}
++
+ static void acpi_dev_get_irqresource(struct resource *res, u32 gsi,
+ u8 triggering, u8 polarity, u8 shareable,
+- bool legacy)
++ bool check_override)
+ {
+ int irq, p, t;
+
+@@ -401,7 +451,9 @@ static void acpi_dev_get_irqresource(struct resource *res, u32 gsi,
+ * using extended IRQ descriptors we take the IRQ configuration
+ * from _CRS directly.
+ */
+- if (legacy && !acpi_get_override_irq(gsi, &t, &p)) {
++ if (check_override &&
++ acpi_dev_irq_override(gsi, triggering, polarity, shareable) &&
++ !acpi_get_override_irq(gsi, &t, &p)) {
+ u8 trig = t ? ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE;
+ u8 pol = p ? ACPI_ACTIVE_LOW : ACPI_ACTIVE_HIGH;
+
+diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
+index 5b54c80b9d32a..6e9cd41c5f9b1 100644
+--- a/drivers/acpi/scan.c
++++ b/drivers/acpi/scan.c
+@@ -1690,6 +1690,7 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device)
+ {
+ struct list_head resource_list;
+ bool is_serial_bus_slave = false;
++ static const struct acpi_device_id ignore_serial_bus_ids[] = {
+ /*
+ * These devices have multiple I2cSerialBus resources and an i2c-client
+ * must be instantiated for each, each with its own i2c_device_id.
+@@ -1698,11 +1699,18 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device)
+ * drivers/platform/x86/i2c-multi-instantiate.c driver, which knows
+ * which i2c_device_id to use for each resource.
+ */
+- static const struct acpi_device_id i2c_multi_instantiate_ids[] = {
+ {"BSG1160", },
+ {"BSG2150", },
+ {"INT33FE", },
+ {"INT3515", },
++ /*
++ * HIDs of device with an UartSerialBusV2 resource for which userspace
++ * expects a regular tty cdev to be created (instead of the in kernel
++ * serdev) and which have a kernel driver which expects a platform_dev
++ * such as the rfkill-gpio driver.
++ */
++ {"BCM4752", },
++ {"LNV4752", },
+ {}
+ };
+
+@@ -1716,8 +1724,7 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device)
+ fwnode_property_present(&device->fwnode, "baud")))
+ return true;
+
+- /* Instantiate a pdev for the i2c-multi-instantiate drv to bind to */
+- if (!acpi_match_device_ids(device, i2c_multi_instantiate_ids))
++ if (!acpi_match_device_ids(device, ignore_serial_bus_ids))
+ return false;
+
+ INIT_LIST_HEAD(&resource_list);
+diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
+index 3023224515abe..7ae09e4b45927 100644
+--- a/drivers/acpi/sleep.c
++++ b/drivers/acpi/sleep.c
+@@ -767,6 +767,7 @@ bool acpi_s2idle_wake(void)
+ return true;
+ }
+
++ pm_wakeup_clear(acpi_sci_irq);
+ rearm_wake_irq(acpi_sci_irq);
+ }
+
+diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
+index 00c0ebaab29f7..6e23b76aef5dc 100644
+--- a/drivers/acpi/sysfs.c
++++ b/drivers/acpi/sysfs.c
+@@ -415,19 +415,30 @@ static ssize_t acpi_data_show(struct file *filp, struct kobject *kobj,
+ loff_t offset, size_t count)
+ {
+ struct acpi_data_attr *data_attr;
+- void *base;
+- ssize_t rc;
++ void __iomem *base;
++ ssize_t size;
+
+ data_attr = container_of(bin_attr, struct acpi_data_attr, attr);
++ size = data_attr->attr.size;
++
++ if (offset < 0)
++ return -EINVAL;
++
++ if (offset >= size)
++ return 0;
+
+- base = acpi_os_map_memory(data_attr->addr, data_attr->attr.size);
++ if (count > size - offset)
++ count = size - offset;
++
++ base = acpi_os_map_iomem(data_attr->addr, size);
+ if (!base)
+ return -ENOMEM;
+- rc = memory_read_from_buffer(buf, count, &offset, base,
+- data_attr->attr.size);
+- acpi_os_unmap_memory(base, data_attr->attr.size);
+
+- return rc;
++ memcpy_fromio(buf, base + offset, count);
++
++ acpi_os_unmap_iomem(base, size);
++
++ return count;
+ }
+
+ static int acpi_bert_data_init(void *th, struct acpi_data_attr *data_attr)
+diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
+index 33474fd969913..7b9793cb55c50 100644
+--- a/drivers/acpi/video_detect.c
++++ b/drivers/acpi/video_detect.c
+@@ -409,6 +409,81 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "GA503"),
+ },
+ },
++ /*
++ * Clevo NL5xRU and NL5xNU/TUXEDO Aura 15 Gen1 and Gen2 have both a
++ * working native and video interface. However the default detection
++ * mechanism first registers the video interface before unregistering
++ * it again and switching to the native interface during boot. This
++ * results in a dangling SBIOS request for backlight change for some
++ * reason, causing the backlight to switch to ~2% once per boot on the
++ * first power cord connect or disconnect event. Setting the native
++ * interface explicitly circumvents this buggy behaviour, by avoiding
++ * the unregistering process.
++ */
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xRU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xRU"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xRU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "SchenkerTechnologiesGmbH"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xRU"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xRU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Notebook"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xRU"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xRU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
++ DMI_MATCH(DMI_BOARD_NAME, "AURA1501"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xRU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
++ DMI_MATCH(DMI_BOARD_NAME, "EDUBOOK1502"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xNU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xNU"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xNU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "SchenkerTechnologiesGmbH"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xNU"),
++ },
++ },
++ {
++ .callback = video_detect_force_native,
++ .ident = "Clevo NL5xNU",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Notebook"),
++ DMI_MATCH(DMI_BOARD_NAME, "NL5xNU"),
++ },
++ },
+
+ /*
+ * Desktops which falsely report a backlight and which our heuristics
+diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c
+index 1c48358b43ba3..e0185e841b2a3 100644
+--- a/drivers/acpi/x86/s2idle.c
++++ b/drivers/acpi/x86/s2idle.c
+@@ -424,15 +424,11 @@ static int lps0_device_attach(struct acpi_device *adev,
+ mem_sleep_current = PM_SUSPEND_TO_IDLE;
+
+ /*
+- * Some Intel based LPS0 systems, like ASUS Zenbook UX430UNR/i7-8550U don't
+- * use intel-hid or intel-vbtn but require the EC GPE to be enabled while
+- * suspended for certain wakeup devices to work, so mark it as wakeup-capable.
+- *
+- * Only enable on !AMD as enabling this universally causes problems for a number
+- * of AMD based systems.
++ * Some LPS0 systems, like ASUS Zenbook UX430UNR/i7-8550U, require the
++ * EC GPE to be enabled while suspended for certain wakeup devices to
++ * work, so mark it as wakeup-capable.
+ */
+- if (!acpi_s2idle_vendor_amd())
+- acpi_ec_mark_gpe_for_wake();
++ acpi_ec_mark_gpe_for_wake();
+
+ return 0;
+ }
+diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c
+index f22f23933063b..b3fb428461c6f 100644
+--- a/drivers/acpi/x86/utils.c
++++ b/drivers/acpi/x86/utils.c
+@@ -22,58 +22,71 @@
+ * Some BIOS-es (temporarily) hide specific APCI devices to work around Windows
+ * driver bugs. We use DMI matching to match known cases of this.
+ *
+- * We work around this by always reporting ACPI_STA_DEFAULT for these
+- * devices. Note this MUST only be done for devices where this is safe.
++ * Likewise sometimes some not-actually present devices are sometimes
++ * reported as present, which may cause issues.
+ *
+- * This forcing of devices to be present is limited to specific CPU (SoC)
+- * models both to avoid potentially causing trouble on other models and
+- * because some HIDs are re-used on different SoCs for completely
+- * different devices.
++ * We work around this by using the below quirk list to override the status
++ * reported by the _STA method with a fixed value (ACPI_STA_DEFAULT or 0).
++ * Note this MUST only be done for devices where this is safe.
++ *
++ * This status overriding is limited to specific CPU (SoC) models both to
++ * avoid potentially causing trouble on other models and because some HIDs
++ * are re-used on different SoCs for completely different devices.
+ */
+-struct always_present_id {
++struct override_status_id {
+ struct acpi_device_id hid[2];
+ struct x86_cpu_id cpu_ids[2];
+ struct dmi_system_id dmi_ids[2]; /* Optional */
+ const char *uid;
++ const char *path;
++ unsigned long long status;
+ };
+
+-#define X86_MATCH(model) X86_MATCH_INTEL_FAM6_MODEL(model, NULL)
+-
+-#define ENTRY(hid, uid, cpu_models, dmi...) { \
++#define ENTRY(status, hid, uid, path, cpu_model, dmi...) { \
+ { { hid, }, {} }, \
+- { cpu_models, {} }, \
++ { X86_MATCH_INTEL_FAM6_MODEL(cpu_model, NULL), {} }, \
+ { { .matches = dmi }, {} }, \
+ uid, \
++ path, \
++ status, \
+ }
+
+-static const struct always_present_id always_present_ids[] = {
++#define PRESENT_ENTRY_HID(hid, uid, cpu_model, dmi...) \
++ ENTRY(ACPI_STA_DEFAULT, hid, uid, NULL, cpu_model, dmi)
++
++#define NOT_PRESENT_ENTRY_HID(hid, uid, cpu_model, dmi...) \
++ ENTRY(0, hid, uid, NULL, cpu_model, dmi)
++
++#define PRESENT_ENTRY_PATH(path, cpu_model, dmi...) \
++ ENTRY(ACPI_STA_DEFAULT, "", NULL, path, cpu_model, dmi)
++
++#define NOT_PRESENT_ENTRY_PATH(path, cpu_model, dmi...) \
++ ENTRY(0, "", NULL, path, cpu_model, dmi)
++
++static const struct override_status_id override_status_ids[] = {
+ /*
+ * Bay / Cherry Trail PWM directly poked by GPU driver in win10,
+ * but Linux uses a separate PWM driver, harmless if not used.
+ */
+- ENTRY("80860F09", "1", X86_MATCH(ATOM_SILVERMONT), {}),
+- ENTRY("80862288", "1", X86_MATCH(ATOM_AIRMONT), {}),
++ PRESENT_ENTRY_HID("80860F09", "1", ATOM_SILVERMONT, {}),
++ PRESENT_ENTRY_HID("80862288", "1", ATOM_AIRMONT, {}),
+
+- /* Lenovo Yoga Book uses PWM2 for keyboard backlight control */
+- ENTRY("80862289", "2", X86_MATCH(ATOM_AIRMONT), {
+- DMI_MATCH(DMI_PRODUCT_NAME, "Lenovo YB1-X9"),
+- }),
+ /*
+ * The INT0002 device is necessary to clear wakeup interrupt sources
+ * on Cherry Trail devices, without it we get nobody cared IRQ msgs.
+ */
+- ENTRY("INT0002", "1", X86_MATCH(ATOM_AIRMONT), {}),
++ PRESENT_ENTRY_HID("INT0002", "1", ATOM_AIRMONT, {}),
+ /*
+ * On the Dell Venue 11 Pro 7130 and 7139, the DSDT hides
+ * the touchscreen ACPI device until a certain time
+ * after _SB.PCI0.GFX0.LCD.LCD1._ON gets called has passed
+ * *and* _STA has been called at least 3 times since.
+ */
+- ENTRY("SYNA7500", "1", X86_MATCH(HASWELL_L), {
++ PRESENT_ENTRY_HID("SYNA7500", "1", HASWELL_L, {
+ DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7130"),
+ }),
+- ENTRY("SYNA7500", "1", X86_MATCH(HASWELL_L), {
++ PRESENT_ENTRY_HID("SYNA7500", "1", HASWELL_L, {
+ DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7139"),
+ }),
+@@ -81,54 +94,83 @@ static const struct always_present_id always_present_ids[] = {
+ /*
+ * The GPD win BIOS dated 20170221 has disabled the accelerometer, the
+ * drivers sometimes cause crashes under Windows and this is how the
+- * manufacturer has solved this :| Note that the the DMI data is less
+- * generic then it seems, a board_vendor of "AMI Corporation" is quite
+- * rare and a board_name of "Default String" also is rare.
++ * manufacturer has solved this :| The DMI match may not seem unique,
++ * but it is. In the 67000+ DMI decode dumps from linux-hardware.org
++ * only 116 have board_vendor set to "AMI Corporation" and of those 116
++ * only the GPD win and pocket entries' board_name is "Default string".
+ *
+ * Unfortunately the GPD pocket also uses these strings and its BIOS
+ * was copy-pasted from the GPD win, so it has a disabled KIOX000A
+ * node which we should not enable, thus we also check the BIOS date.
+ */
+- ENTRY("KIOX000A", "1", X86_MATCH(ATOM_AIRMONT), {
++ PRESENT_ENTRY_HID("KIOX000A", "1", ATOM_AIRMONT, {
+ DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+ DMI_MATCH(DMI_BOARD_NAME, "Default string"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Default string"),
+ DMI_MATCH(DMI_BIOS_DATE, "02/21/2017")
+ }),
+- ENTRY("KIOX000A", "1", X86_MATCH(ATOM_AIRMONT), {
++ PRESENT_ENTRY_HID("KIOX000A", "1", ATOM_AIRMONT, {
+ DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+ DMI_MATCH(DMI_BOARD_NAME, "Default string"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Default string"),
+ DMI_MATCH(DMI_BIOS_DATE, "03/20/2017")
+ }),
+- ENTRY("KIOX000A", "1", X86_MATCH(ATOM_AIRMONT), {
++ PRESENT_ENTRY_HID("KIOX000A", "1", ATOM_AIRMONT, {
+ DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+ DMI_MATCH(DMI_BOARD_NAME, "Default string"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Default string"),
+ DMI_MATCH(DMI_BIOS_DATE, "05/25/2017")
+ }),
++
++ /*
++ * The GPD win/pocket have a PCI wifi card, but its DSDT has the SDIO
++ * mmc controller enabled and that has a child-device which _PS3
++ * method sets a GPIO causing the PCI wifi card to turn off.
++ * See above remark about uniqueness of the DMI match.
++ */
++ NOT_PRESENT_ENTRY_PATH("\\_SB_.PCI0.SDHB.BRC1", ATOM_AIRMONT, {
++ DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
++ DMI_EXACT_MATCH(DMI_BOARD_NAME, "Default string"),
++ DMI_EXACT_MATCH(DMI_BOARD_SERIAL, "Default string"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Default string"),
++ }),
+ };
+
+-bool acpi_device_always_present(struct acpi_device *adev)
++bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *status)
+ {
+ bool ret = false;
+ unsigned int i;
+
+- for (i = 0; i < ARRAY_SIZE(always_present_ids); i++) {
+- if (acpi_match_device_ids(adev, always_present_ids[i].hid))
++ for (i = 0; i < ARRAY_SIZE(override_status_ids); i++) {
++ if (!x86_match_cpu(override_status_ids[i].cpu_ids))
+ continue;
+
+- if (!adev->pnp.unique_id ||
+- strcmp(adev->pnp.unique_id, always_present_ids[i].uid))
++ if (override_status_ids[i].dmi_ids[0].matches[0].slot &&
++ !dmi_check_system(override_status_ids[i].dmi_ids))
+ continue;
+
+- if (!x86_match_cpu(always_present_ids[i].cpu_ids))
+- continue;
++ if (override_status_ids[i].path) {
++ struct acpi_buffer path = { ACPI_ALLOCATE_BUFFER, NULL };
++ bool match;
+
+- if (always_present_ids[i].dmi_ids[0].matches[0].slot &&
+- !dmi_check_system(always_present_ids[i].dmi_ids))
+- continue;
++ if (acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &path))
++ continue;
++
++ match = strcmp((char *)path.pointer, override_status_ids[i].path) == 0;
++ kfree(path.pointer);
++
++ if (!match)
++ continue;
++ } else {
++ if (acpi_match_device_ids(adev, override_status_ids[i].hid))
++ continue;
++
++ if (!adev->pnp.unique_id ||
++ strcmp(adev->pnp.unique_id, override_status_ids[i].uid))
++ continue;
++ }
+
++ *status = override_status_ids[i].status;
+ ret = true;
+ break;
+ }
+diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c
+index 962041148482c..6c0f7f4f7d1de 100644
+--- a/drivers/amba/bus.c
++++ b/drivers/amba/bus.c
+@@ -377,9 +377,6 @@ static int amba_device_try_add(struct amba_device *dev, struct resource *parent)
+ void __iomem *tmp;
+ int i, ret;
+
+- WARN_ON(dev->irq[0] == (unsigned int)-1);
+- WARN_ON(dev->irq[1] == (unsigned int)-1);
+-
+ ret = request_resource(parent, &dev->res);
+ if (ret)
+ goto err_out;
+diff --git a/drivers/android/binder.c b/drivers/android/binder.c
+index 9edacc8b97688..99ae919255f4d 100644
+--- a/drivers/android/binder.c
++++ b/drivers/android/binder.c
+@@ -1608,15 +1608,21 @@ static void binder_cleanup_transaction(struct binder_transaction *t,
+ /**
+ * binder_get_object() - gets object and checks for valid metadata
+ * @proc: binder_proc owning the buffer
++ * @u: sender's user pointer to base of buffer
+ * @buffer: binder_buffer that we're parsing.
+ * @offset: offset in the @buffer at which to validate an object.
+ * @object: struct binder_object to read into
+ *
+- * Return: If there's a valid metadata object at @offset in @buffer, the
++ * Copy the binder object at the given offset into @object. If @u is
++ * provided then the copy is from the sender's buffer. If not, then
++ * it is copied from the target's @buffer.
++ *
++ * Return: If there's a valid metadata object at @offset, the
+ * size of that object. Otherwise, it returns zero. The object
+ * is read into the struct binder_object pointed to by @object.
+ */
+ static size_t binder_get_object(struct binder_proc *proc,
++ const void __user *u,
+ struct binder_buffer *buffer,
+ unsigned long offset,
+ struct binder_object *object)
+@@ -1626,10 +1632,16 @@ static size_t binder_get_object(struct binder_proc *proc,
+ size_t object_size = 0;
+
+ read_size = min_t(size_t, sizeof(*object), buffer->data_size - offset);
+- if (offset > buffer->data_size || read_size < sizeof(*hdr) ||
+- binder_alloc_copy_from_buffer(&proc->alloc, object, buffer,
+- offset, read_size))
++ if (offset > buffer->data_size || read_size < sizeof(*hdr))
+ return 0;
++ if (u) {
++ if (copy_from_user(object, u + offset, read_size))
++ return 0;
++ } else {
++ if (binder_alloc_copy_from_buffer(&proc->alloc, object, buffer,
++ offset, read_size))
++ return 0;
++ }
+
+ /* Ok, now see if we read a complete object. */
+ hdr = &object->hdr;
+@@ -1702,7 +1714,7 @@ static struct binder_buffer_object *binder_validate_ptr(
+ b, buffer_offset,
+ sizeof(object_offset)))
+ return NULL;
+- object_size = binder_get_object(proc, b, object_offset, object);
++ object_size = binder_get_object(proc, NULL, b, object_offset, object);
+ if (!object_size || object->hdr.type != BINDER_TYPE_PTR)
+ return NULL;
+ if (object_offsetp)
+@@ -1767,7 +1779,8 @@ static bool binder_validate_fixup(struct binder_proc *proc,
+ unsigned long buffer_offset;
+ struct binder_object last_object;
+ struct binder_buffer_object *last_bbo;
+- size_t object_size = binder_get_object(proc, b, last_obj_offset,
++ size_t object_size = binder_get_object(proc, NULL, b,
++ last_obj_offset,
+ &last_object);
+ if (object_size != sizeof(*last_bbo))
+ return false;
+@@ -1870,7 +1883,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
+ binder_dec_node(buffer->target_node, 1, 0);
+
+ off_start_offset = ALIGN(buffer->data_size, sizeof(void *));
+- off_end_offset = is_failure ? failed_at :
++ off_end_offset = is_failure && failed_at ? failed_at :
+ off_start_offset + buffer->offsets_size;
+ for (buffer_offset = off_start_offset; buffer_offset < off_end_offset;
+ buffer_offset += sizeof(binder_size_t)) {
+@@ -1882,7 +1895,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
+ if (!binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
+ buffer, buffer_offset,
+ sizeof(object_offset)))
+- object_size = binder_get_object(proc, buffer,
++ object_size = binder_get_object(proc, NULL, buffer,
+ object_offset, &object);
+ if (object_size == 0) {
+ pr_err("transaction release %d bad object at offset %lld, size %zd\n",
+@@ -1956,9 +1969,8 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
+ binder_size_t fd_buf_size;
+ binder_size_t num_valid;
+
+- if (proc->tsk != current->group_leader) {
++ if (is_failure) {
+ /*
+- * Nothing to do if running in sender context
+ * The fd fixups have not been applied so no
+ * fds need to be closed.
+ */
+@@ -2056,7 +2068,7 @@ static int binder_translate_binder(struct flat_binder_object *fp,
+ ret = -EINVAL;
+ goto done;
+ }
+- if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) {
++ if (security_binder_transfer_binder(proc->cred, target_proc->cred)) {
+ ret = -EPERM;
+ goto done;
+ }
+@@ -2102,7 +2114,7 @@ static int binder_translate_handle(struct flat_binder_object *fp,
+ proc->pid, thread->pid, fp->handle);
+ return -EINVAL;
+ }
+- if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) {
++ if (security_binder_transfer_binder(proc->cred, target_proc->cred)) {
+ ret = -EPERM;
+ goto done;
+ }
+@@ -2190,7 +2202,7 @@ static int binder_translate_fd(u32 fd, binder_size_t fd_offset,
+ ret = -EBADF;
+ goto err_fget;
+ }
+- ret = security_binder_transfer_file(proc->tsk, target_proc->tsk, file);
++ ret = security_binder_transfer_file(proc->cred, target_proc->cred, file);
+ if (ret < 0) {
+ ret = -EPERM;
+ goto err_security;
+@@ -2270,8 +2282,8 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda,
+ if (!ret)
+ ret = binder_translate_fd(fd, offset, t, thread,
+ in_reply_to);
+- if (ret < 0)
+- return ret;
++ if (ret)
++ return ret > 0 ? -EINVAL : ret;
+ }
+ return 0;
+ }
+@@ -2456,6 +2468,7 @@ static void binder_transaction(struct binder_proc *proc,
+ binder_size_t off_start_offset, off_end_offset;
+ binder_size_t off_min;
+ binder_size_t sg_buf_offset, sg_buf_end_offset;
++ binder_size_t user_offset = 0;
+ struct binder_proc *target_proc = NULL;
+ struct binder_thread *target_thread = NULL;
+ struct binder_node *target_node = NULL;
+@@ -2470,6 +2483,8 @@ static void binder_transaction(struct binder_proc *proc,
+ int t_debug_id = atomic_inc_return(&binder_last_id);
+ char *secctx = NULL;
+ u32 secctx_sz = 0;
++ const void __user *user_buffer = (const void __user *)
++ (uintptr_t)tr->data.ptr.buffer;
+
+ e = binder_transaction_log_add(&binder_transaction_log);
+ e->debug_id = t_debug_id;
+@@ -2595,8 +2610,8 @@ static void binder_transaction(struct binder_proc *proc,
+ return_error_line = __LINE__;
+ goto err_invalid_target_handle;
+ }
+- if (security_binder_transaction(proc->tsk,
+- target_proc->tsk) < 0) {
++ if (security_binder_transaction(proc->cred,
++ target_proc->cred) < 0) {
+ return_error = BR_FAILED_REPLY;
+ return_error_param = -EPERM;
+ return_error_line = __LINE__;
+@@ -2722,16 +2737,7 @@ static void binder_transaction(struct binder_proc *proc,
+ u32 secid;
+ size_t added_size;
+
+- /*
+- * Arguably this should be the task's subjective LSM secid but
+- * we can't reliably access the subjective creds of a task
+- * other than our own so we must use the objective creds, which
+- * are safe to access. The downside is that if a task is
+- * temporarily overriding it's creds it will not be reflected
+- * here; however, it isn't clear that binder would handle that
+- * case well anyway.
+- */
+- security_task_getsecid_obj(proc->tsk, &secid);
++ security_cred_getsecid(proc->cred, &secid);
+ ret = security_secid_to_secctx(secid, &secctx, &secctx_sz);
+ if (ret) {
+ return_error = BR_FAILED_REPLY;
+@@ -2790,19 +2796,6 @@ static void binder_transaction(struct binder_proc *proc,
+ t->buffer->clear_on_free = !!(t->flags & TF_CLEAR_BUF);
+ trace_binder_transaction_alloc_buf(t->buffer);
+
+- if (binder_alloc_copy_user_to_buffer(
+- &target_proc->alloc,
+- t->buffer, 0,
+- (const void __user *)
+- (uintptr_t)tr->data.ptr.buffer,
+- tr->data_size)) {
+- binder_user_error("%d:%d got transaction with invalid data ptr\n",
+- proc->pid, thread->pid);
+- return_error = BR_FAILED_REPLY;
+- return_error_param = -EFAULT;
+- return_error_line = __LINE__;
+- goto err_copy_data_failed;
+- }
+ if (binder_alloc_copy_user_to_buffer(
+ &target_proc->alloc,
+ t->buffer,
+@@ -2847,6 +2840,7 @@ static void binder_transaction(struct binder_proc *proc,
+ size_t object_size;
+ struct binder_object object;
+ binder_size_t object_offset;
++ binder_size_t copy_size;
+
+ if (binder_alloc_copy_from_buffer(&target_proc->alloc,
+ &object_offset,
+@@ -2858,8 +2852,27 @@ static void binder_transaction(struct binder_proc *proc,
+ return_error_line = __LINE__;
+ goto err_bad_offset;
+ }
+- object_size = binder_get_object(target_proc, t->buffer,
+- object_offset, &object);
++
++ /*
++ * Copy the source user buffer up to the next object
++ * that will be processed.
++ */
++ copy_size = object_offset - user_offset;
++ if (copy_size && (user_offset > object_offset ||
++ binder_alloc_copy_user_to_buffer(
++ &target_proc->alloc,
++ t->buffer, user_offset,
++ user_buffer + user_offset,
++ copy_size))) {
++ binder_user_error("%d:%d got transaction with invalid data ptr\n",
++ proc->pid, thread->pid);
++ return_error = BR_FAILED_REPLY;
++ return_error_param = -EFAULT;
++ return_error_line = __LINE__;
++ goto err_copy_data_failed;
++ }
++ object_size = binder_get_object(target_proc, user_buffer,
++ t->buffer, object_offset, &object);
+ if (object_size == 0 || object_offset < off_min) {
+ binder_user_error("%d:%d got transaction with invalid offset (%lld, min %lld max %lld) or object.\n",
+ proc->pid, thread->pid,
+@@ -2871,6 +2884,11 @@ static void binder_transaction(struct binder_proc *proc,
+ return_error_line = __LINE__;
+ goto err_bad_offset;
+ }
++ /*
++ * Set offset to the next buffer fragment to be
++ * copied
++ */
++ user_offset = object_offset + object_size;
+
+ hdr = &object.hdr;
+ off_min = object_offset + object_size;
+@@ -2966,9 +2984,14 @@ static void binder_transaction(struct binder_proc *proc,
+ }
+ ret = binder_translate_fd_array(fda, parent, t, thread,
+ in_reply_to);
+- if (ret < 0) {
++ if (!ret)
++ ret = binder_alloc_copy_to_buffer(&target_proc->alloc,
++ t->buffer,
++ object_offset,
++ fda, sizeof(*fda));
++ if (ret) {
+ return_error = BR_FAILED_REPLY;
+- return_error_param = ret;
++ return_error_param = ret > 0 ? -EINVAL : ret;
+ return_error_line = __LINE__;
+ goto err_translate_failed;
+ }
+@@ -3038,6 +3061,19 @@ static void binder_transaction(struct binder_proc *proc,
+ goto err_bad_object_type;
+ }
+ }
++ /* Done processing objects, copy the rest of the buffer */
++ if (binder_alloc_copy_user_to_buffer(
++ &target_proc->alloc,
++ t->buffer, user_offset,
++ user_buffer + user_offset,
++ tr->data_size - user_offset)) {
++ binder_user_error("%d:%d got transaction with invalid data ptr\n",
++ proc->pid, thread->pid);
++ return_error = BR_FAILED_REPLY;
++ return_error_param = -EFAULT;
++ return_error_line = __LINE__;
++ goto err_copy_data_failed;
++ }
+ if (t->buffer->oneway_spam_suspect)
+ tcomplete->type = BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT;
+ else
+@@ -3185,6 +3221,7 @@ err_invalid_target_handle:
+ * binder_free_buf() - free the specified buffer
+ * @proc: binder proc that owns buffer
+ * @buffer: buffer to be freed
++ * @is_failure: failed to send transaction
+ *
+ * If buffer for an async transaction, enqueue the next async
+ * transaction from the node.
+@@ -3194,7 +3231,7 @@ err_invalid_target_handle:
+ static void
+ binder_free_buf(struct binder_proc *proc,
+ struct binder_thread *thread,
+- struct binder_buffer *buffer)
++ struct binder_buffer *buffer, bool is_failure)
+ {
+ binder_inner_proc_lock(proc);
+ if (buffer->transaction) {
+@@ -3222,7 +3259,7 @@ binder_free_buf(struct binder_proc *proc,
+ binder_node_inner_unlock(buf_node);
+ }
+ trace_binder_transaction_buffer_release(buffer);
+- binder_transaction_buffer_release(proc, thread, buffer, 0, false);
++ binder_transaction_buffer_release(proc, thread, buffer, 0, is_failure);
+ binder_alloc_free_buf(&proc->alloc, buffer);
+ }
+
+@@ -3424,7 +3461,7 @@ static int binder_thread_write(struct binder_proc *proc,
+ proc->pid, thread->pid, (u64)data_ptr,
+ buffer->debug_id,
+ buffer->transaction ? "active" : "finished");
+- binder_free_buf(proc, thread, buffer);
++ binder_free_buf(proc, thread, buffer, false);
+ break;
+ }
+
+@@ -4117,7 +4154,7 @@ retry:
+ buffer->transaction = NULL;
+ binder_cleanup_transaction(t, "fd fixups failed",
+ BR_FAILED_REPLY);
+- binder_free_buf(proc, thread, buffer);
++ binder_free_buf(proc, thread, buffer, true);
+ binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
+ "%d:%d %stransaction %d fd fixups failed %d/%d, line %d\n",
+ proc->pid, thread->pid,
+@@ -4353,6 +4390,7 @@ static void binder_free_proc(struct binder_proc *proc)
+ }
+ binder_alloc_deferred_release(&proc->alloc);
+ put_task_struct(proc->tsk);
++ put_cred(proc->cred);
+ binder_stats_deleted(BINDER_STAT_PROC);
+ kfree(proc);
+ }
+@@ -4430,23 +4468,20 @@ static int binder_thread_release(struct binder_proc *proc,
+ __release(&t->lock);
+
+ /*
+- * If this thread used poll, make sure we remove the waitqueue
+- * from any epoll data structures holding it with POLLFREE.
+- * waitqueue_active() is safe to use here because we're holding
+- * the inner lock.
++ * If this thread used poll, make sure we remove the waitqueue from any
++ * poll data structures holding it.
+ */
+- if ((thread->looper & BINDER_LOOPER_STATE_POLL) &&
+- waitqueue_active(&thread->wait)) {
+- wake_up_poll(&thread->wait, EPOLLHUP | POLLFREE);
+- }
++ if (thread->looper & BINDER_LOOPER_STATE_POLL)
++ wake_up_pollfree(&thread->wait);
+
+ binder_inner_proc_unlock(thread->proc);
+
+ /*
+- * This is needed to avoid races between wake_up_poll() above and
+- * and ep_remove_waitqueue() called for other reasons (eg the epoll file
+- * descriptor being closed); ep_remove_waitqueue() holds an RCU read
+- * lock, so we can be sure it's done after calling synchronize_rcu().
++ * This is needed to avoid races between wake_up_pollfree() above and
++ * someone else removing the last entry from the queue for other reasons
++ * (e.g. ep_remove_wait_queue() being called due to an epoll file
++ * descriptor being closed). Such other users hold an RCU read lock, so
++ * we can be sure they're done after we call synchronize_rcu().
+ */
+ if (thread->looper & BINDER_LOOPER_STATE_POLL)
+ synchronize_rcu();
+@@ -4564,7 +4599,7 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp,
+ ret = -EBUSY;
+ goto out;
+ }
+- ret = security_binder_set_context_mgr(proc->tsk);
++ ret = security_binder_set_context_mgr(proc->cred);
+ if (ret < 0)
+ goto out;
+ if (uid_valid(context->binder_context_mgr_uid)) {
+@@ -5055,6 +5090,7 @@ static int binder_open(struct inode *nodp, struct file *filp)
+ spin_lock_init(&proc->outer_lock);
+ get_task_struct(current->group_leader);
+ proc->tsk = current->group_leader;
++ proc->cred = get_cred(filp->f_cred);
+ INIT_LIST_HEAD(&proc->todo);
+ init_waitqueue_head(&proc->freeze_wait);
+ proc->default_priority = task_nice(current);
+diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
+index 340515f54498c..47bc74a8c7b6f 100644
+--- a/drivers/android/binder_alloc.c
++++ b/drivers/android/binder_alloc.c
+@@ -671,7 +671,7 @@ static void binder_free_buf_locked(struct binder_alloc *alloc,
+ BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size);
+
+ if (buffer->async_transaction) {
+- alloc->free_async_space += size + sizeof(struct binder_buffer);
++ alloc->free_async_space += buffer_size + sizeof(struct binder_buffer);
+
+ binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
+ "%d: binder_free_buf size %zd async free %zd\n",
+diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h
+index 402c4d4362a83..d6b6b8cb73465 100644
+--- a/drivers/android/binder_internal.h
++++ b/drivers/android/binder_internal.h
+@@ -364,6 +364,9 @@ struct binder_ref {
+ * (invariant after initialized)
+ * @tsk task_struct for group_leader of process
+ * (invariant after initialized)
++ * @cred struct cred associated with the `struct file`
++ * in binder_open()
++ * (invariant after initialized)
+ * @deferred_work_node: element for binder_deferred_list
+ * (protected by binder_deferred_lock)
+ * @deferred_work: bitmap of deferred work to perform
+@@ -426,6 +429,7 @@ struct binder_proc {
+ struct list_head waiting_threads;
+ int pid;
+ struct task_struct *tsk;
++ const struct cred *cred;
+ struct hlist_node deferred_work_node;
+ int deferred_work;
+ int outstanding_txns;
+diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
+index 186cbf90c8ead..812731e80f8e0 100644
+--- a/drivers/ata/ahci.c
++++ b/drivers/ata/ahci.c
+@@ -442,6 +442,7 @@ static const struct pci_device_id ahci_pci_tbl[] = {
+ /* AMD */
+ { PCI_VDEVICE(AMD, 0x7800), board_ahci }, /* AMD Hudson-2 */
+ { PCI_VDEVICE(AMD, 0x7900), board_ahci }, /* AMD CZ */
++ { PCI_VDEVICE(AMD, 0x7901), board_ahci_mobile }, /* AMD Green Sardine */
+ /* AMD is using RAID class only for ahci controllers */
+ { PCI_VENDOR_ID_AMD, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
+ PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci },
+diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
+index 5b3fa2cbe7223..395772fa39432 100644
+--- a/drivers/ata/libahci.c
++++ b/drivers/ata/libahci.c
+@@ -2305,6 +2305,18 @@ int ahci_port_resume(struct ata_port *ap)
+ EXPORT_SYMBOL_GPL(ahci_port_resume);
+
+ #ifdef CONFIG_PM
++static void ahci_handle_s2idle(struct ata_port *ap)
++{
++ void __iomem *port_mmio = ahci_port_base(ap);
++ u32 devslp;
++
++ if (pm_suspend_via_firmware())
++ return;
++ devslp = readl(port_mmio + PORT_DEVSLP);
++ if ((devslp & PORT_DEVSLP_ADSE))
++ ata_msleep(ap, devslp_idle_timeout);
++}
++
+ static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
+ {
+ const char *emsg = NULL;
+@@ -2318,6 +2330,9 @@ static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
+ ata_port_freeze(ap);
+ }
+
++ if (acpi_storage_d3(ap->host->dev))
++ ahci_handle_s2idle(ap);
++
+ ahci_rpm_put_port(ap);
+ return rc;
+ }
+diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
+index eed65311b5d1d..a0343b7c9addf 100644
+--- a/drivers/ata/libata-core.c
++++ b/drivers/ata/libata-core.c
+@@ -2007,7 +2007,7 @@ unsigned int ata_read_log_page(struct ata_device *dev, u8 log,
+
+ retry:
+ ata_tf_init(dev, &tf);
+- if (dev->dma_mode && ata_id_has_read_log_dma_ext(dev->id) &&
++ if (ata_dma_enabled(dev) && ata_id_has_read_log_dma_ext(dev->id) &&
+ !(dev->horkage & ATA_HORKAGE_NO_DMA_LOG)) {
+ tf.command = ATA_CMD_READ_LOG_DMA_EXT;
+ tf.protocol = ATA_PROT_DMA;
+@@ -2031,8 +2031,9 @@ retry:
+ dev->horkage |= ATA_HORKAGE_NO_DMA_LOG;
+ goto retry;
+ }
+- ata_dev_err(dev, "Read log page 0x%02x failed, Emask 0x%x\n",
+- (unsigned int)page, err_mask);
++ ata_dev_err(dev,
++ "Read log 0x%02x page 0x%02x failed, Emask 0x%x\n",
++ (unsigned int)log, (unsigned int)page, err_mask);
+ }
+
+ return err_mask;
+@@ -2166,6 +2167,9 @@ static void ata_dev_config_ncq_prio(struct ata_device *dev)
+ struct ata_port *ap = dev->link->ap;
+ unsigned int err_mask;
+
++ if (!ata_identify_page_supported(dev, ATA_LOG_SATA_SETTINGS))
++ return;
++
+ err_mask = ata_read_log_page(dev,
+ ATA_LOG_IDENTIFY_DEVICE,
+ ATA_LOG_SATA_SETTINGS,
+@@ -2442,7 +2446,8 @@ static void ata_dev_config_devslp(struct ata_device *dev)
+ * Check device sleep capability. Get DevSlp timing variables
+ * from SATA Settings page of Identify Device Data Log.
+ */
+- if (!ata_id_has_devslp(dev->id))
++ if (!ata_id_has_devslp(dev->id) ||
++ !ata_identify_page_supported(dev, ATA_LOG_SATA_SETTINGS))
+ return;
+
+ err_mask = ata_read_log_page(dev,
+@@ -3851,6 +3856,8 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
+ { "VRFDFC22048UCHC-TE*", NULL, ATA_HORKAGE_NODMA },
+ /* Odd clown on sil3726/4726 PMPs */
+ { "Config Disk", NULL, ATA_HORKAGE_DISABLE },
++ /* Similar story with ASMedia 1092 */
++ { "ASMT109x- Config", NULL, ATA_HORKAGE_DISABLE },
+
+ /* Weird ATAPI devices */
+ { "TORiSAN DVD-ROM DRD-N216", NULL, ATA_HORKAGE_MAX_SEC_128 },
+@@ -3992,6 +3999,9 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
+ ATA_HORKAGE_ZERO_AFTER_TRIM, },
+ { "Crucial_CT*MX100*", "MU01", ATA_HORKAGE_NO_NCQ_TRIM |
+ ATA_HORKAGE_ZERO_AFTER_TRIM, },
++ { "Samsung SSD 840 EVO*", NULL, ATA_HORKAGE_NO_NCQ_TRIM |
++ ATA_HORKAGE_NO_DMA_LOG |
++ ATA_HORKAGE_ZERO_AFTER_TRIM, },
+ { "Samsung SSD 840*", NULL, ATA_HORKAGE_NO_NCQ_TRIM |
+ ATA_HORKAGE_ZERO_AFTER_TRIM, },
+ { "Samsung SSD 850*", NULL, ATA_HORKAGE_NO_NCQ_TRIM |
+@@ -4007,6 +4017,7 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
+
+ /* devices that don't properly handle TRIM commands */
+ { "SuperSSpeed S238*", NULL, ATA_HORKAGE_NOTRIM, },
++ { "M88V29*", NULL, ATA_HORKAGE_NOTRIM, },
+
+ /*
+ * As defined, the DRAT (Deterministic Read After Trim) and RZAT
+diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
+index bf9c4b6c5c3d4..1d4a6f1e88cd1 100644
+--- a/drivers/ata/libata-eh.c
++++ b/drivers/ata/libata-eh.c
+@@ -93,6 +93,12 @@ static const unsigned long ata_eh_identify_timeouts[] = {
+ ULONG_MAX,
+ };
+
++static const unsigned long ata_eh_revalidate_timeouts[] = {
++ 15000, /* Some drives are slow to read log pages when waking-up */
++ 15000, /* combined time till here is enough even for media access */
++ ULONG_MAX,
++};
++
+ static const unsigned long ata_eh_flush_timeouts[] = {
+ 15000, /* be generous with flush */
+ 15000, /* ditto */
+@@ -129,6 +135,8 @@ static const struct ata_eh_cmd_timeout_ent
+ ata_eh_cmd_timeout_table[ATA_EH_CMD_TIMEOUT_TABLE_SIZE] = {
+ { .commands = CMDS(ATA_CMD_ID_ATA, ATA_CMD_ID_ATAPI),
+ .timeouts = ata_eh_identify_timeouts, },
++ { .commands = CMDS(ATA_CMD_READ_LOG_EXT, ATA_CMD_READ_LOG_DMA_EXT),
++ .timeouts = ata_eh_revalidate_timeouts, },
+ { .commands = CMDS(ATA_CMD_READ_NATIVE_MAX, ATA_CMD_READ_NATIVE_MAX_EXT),
+ .timeouts = ata_eh_other_timeouts, },
+ { .commands = CMDS(ATA_CMD_SET_MAX, ATA_CMD_SET_MAX_EXT),
+diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
+index 1fb4611f7eeb9..10303611d17b9 100644
+--- a/drivers/ata/libata-scsi.c
++++ b/drivers/ata/libata-scsi.c
+@@ -2826,8 +2826,19 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
+ goto invalid_fld;
+ }
+
+- if (ata_is_ncq(tf->protocol) && (cdb[2 + cdb_offset] & 0x3) == 0)
+- tf->protocol = ATA_PROT_NCQ_NODATA;
++ if ((cdb[2 + cdb_offset] & 0x3) == 0) {
++ /*
++ * When T_LENGTH is zero (No data is transferred), dir should
++ * be DMA_NONE.
++ */
++ if (scmd->sc_data_direction != DMA_NONE) {
++ fp = 2 + cdb_offset;
++ goto invalid_fld;
++ }
++
++ if (ata_is_ncq(tf->protocol))
++ tf->protocol = ATA_PROT_NCQ_NODATA;
++ }
+
+ /* enable LBA */
+ tf->flags |= ATA_TFLAG_LBA;
+diff --git a/drivers/ata/pata_hpt37x.c b/drivers/ata/pata_hpt37x.c
+index f242157bc81bb..9d371859e81ed 100644
+--- a/drivers/ata/pata_hpt37x.c
++++ b/drivers/ata/pata_hpt37x.c
+@@ -919,6 +919,20 @@ static int hpt37x_init_one(struct pci_dev *dev, const struct pci_device_id *id)
+ irqmask &= ~0x10;
+ pci_write_config_byte(dev, 0x5a, irqmask);
+
++ /*
++ * HPT371 chips physically have only one channel, the secondary one,
++ * but the primary channel registers do exist! Go figure...
++ * So, we manually disable the non-existing channel here
++ * (if the BIOS hasn't done this already).
++ */
++ if (dev->device == PCI_DEVICE_ID_TTI_HPT371) {
++ u8 mcr1;
++
++ pci_read_config_byte(dev, 0x50, &mcr1);
++ mcr1 &= ~0x04;
++ pci_write_config_byte(dev, 0x50, mcr1);
++ }
++
+ /*
+ * default to pci clock. make sure MA15/16 are set to output
+ * to prevent drives having problems with 40-pin cables. Needed
+@@ -950,14 +964,14 @@ static int hpt37x_init_one(struct pci_dev *dev, const struct pci_device_id *id)
+
+ if ((freq >> 12) != 0xABCDE) {
+ int i;
+- u8 sr;
++ u16 sr;
+ u32 total = 0;
+
+ pr_warn("BIOS has not set timing clocks\n");
+
+ /* This is the process the HPT371 BIOS is reported to use */
+ for (i = 0; i < 128; i++) {
+- pci_read_config_byte(dev, 0x78, &sr);
++ pci_read_config_word(dev, 0x78, &sr);
+ total += sr & 0x1FF;
+ udelay(15);
+ }
+diff --git a/drivers/ata/pata_marvell.c b/drivers/ata/pata_marvell.c
+index 361597d14c569..d45a75bfc0169 100644
+--- a/drivers/ata/pata_marvell.c
++++ b/drivers/ata/pata_marvell.c
+@@ -83,6 +83,8 @@ static int marvell_cable_detect(struct ata_port *ap)
+ switch(ap->port_no)
+ {
+ case 0:
++ if (!ap->ioaddr.bmdma_addr)
++ return ATA_CBL_PATA_UNK;
+ if (ioread8(ap->ioaddr.bmdma_addr + 1) & 1)
+ return ATA_CBL_PATA40;
+ return ATA_CBL_PATA80;
+diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c
+index 338c2e50f7591..29e2b0dfba309 100644
+--- a/drivers/ata/sata_dwc_460ex.c
++++ b/drivers/ata/sata_dwc_460ex.c
+@@ -145,7 +145,11 @@ struct sata_dwc_device {
+ #endif
+ };
+
+-#define SATA_DWC_QCMD_MAX 32
++/*
++ * Allow one extra special slot for commands and DMA management
++ * to account for libata internal commands.
++ */
++#define SATA_DWC_QCMD_MAX (ATA_MAX_QUEUE + 1)
+
+ struct sata_dwc_device_port {
+ struct sata_dwc_device *hsdev;
+diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c
+index e5838b23c9e0a..3b31a4f596d86 100644
+--- a/drivers/ata/sata_fsl.c
++++ b/drivers/ata/sata_fsl.c
+@@ -1394,6 +1394,14 @@ static int sata_fsl_init_controller(struct ata_host *host)
+ return 0;
+ }
+
++static void sata_fsl_host_stop(struct ata_host *host)
++{
++ struct sata_fsl_host_priv *host_priv = host->private_data;
++
++ iounmap(host_priv->hcr_base);
++ kfree(host_priv);
++}
++
+ /*
+ * scsi mid-layer and libata interface structures
+ */
+@@ -1426,6 +1434,8 @@ static struct ata_port_operations sata_fsl_ops = {
+ .port_start = sata_fsl_port_start,
+ .port_stop = sata_fsl_port_stop,
+
++ .host_stop = sata_fsl_host_stop,
++
+ .pmp_attach = sata_fsl_pmp_attach,
+ .pmp_detach = sata_fsl_pmp_detach,
+ };
+@@ -1480,9 +1490,9 @@ static int sata_fsl_probe(struct platform_device *ofdev)
+ host_priv->ssr_base = ssr_base;
+ host_priv->csr_base = csr_base;
+
+- irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);
+- if (!irq) {
+- dev_err(&ofdev->dev, "invalid irq from platform\n");
++ irq = platform_get_irq(ofdev, 0);
++ if (irq < 0) {
++ retval = irq;
+ goto error_exit_with_cleanup;
+ }
+ host_priv->irq = irq;
+@@ -1557,10 +1567,6 @@ static int sata_fsl_remove(struct platform_device *ofdev)
+
+ ata_host_detach(host);
+
+- irq_dispose_mapping(host_priv->irq);
+- iounmap(host_priv->hcr_base);
+- kfree(host_priv);
+-
+ return 0;
+ }
+
+diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
+index 422753d52244b..a31ffe16e626f 100644
+--- a/drivers/atm/eni.c
++++ b/drivers/atm/eni.c
+@@ -1112,6 +1112,8 @@ DPRINTK("iovcnt = %d\n",skb_shinfo(skb)->nr_frags);
+ skb_data3 = skb->data[3];
+ paddr = dma_map_single(&eni_dev->pci_dev->dev,skb->data,skb->len,
+ DMA_TO_DEVICE);
++ if (dma_mapping_error(&eni_dev->pci_dev->dev, paddr))
++ return enq_next;
+ ENI_PRV_PADDR(skb) = paddr;
+ /* prepare DMA queue entries */
+ j = 0;
+diff --git a/drivers/atm/firestream.c b/drivers/atm/firestream.c
+index 3bc3c314a467b..4f67404fe64c7 100644
+--- a/drivers/atm/firestream.c
++++ b/drivers/atm/firestream.c
+@@ -1676,6 +1676,8 @@ static int fs_init(struct fs_dev *dev)
+ dev->hw_base = pci_resource_start(pci_dev, 0);
+
+ dev->base = ioremap(dev->hw_base, 0x1000);
++ if (!dev->base)
++ return 1;
+
+ reset_chip (dev);
+
+diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c
+index 304accde365c8..6c010d4efa4ae 100644
+--- a/drivers/auxdisplay/charlcd.c
++++ b/drivers/auxdisplay/charlcd.c
+@@ -578,6 +578,9 @@ static int charlcd_init(struct charlcd *lcd)
+ * Since charlcd_init_display() needs to write data, we have to
+ * enable mark the LCD initialized just before.
+ */
++ if (WARN_ON(!lcd->ops->init_display))
++ return -EINVAL;
++
+ ret = lcd->ops->init_display(lcd);
+ if (ret)
+ return ret;
+diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c
+index 1e69cc6d21a0d..ed58083499907 100644
+--- a/drivers/auxdisplay/ht16k33.c
++++ b/drivers/auxdisplay/ht16k33.c
+@@ -219,6 +219,15 @@ static const struct backlight_ops ht16k33_bl_ops = {
+ .check_fb = ht16k33_bl_check_fb,
+ };
+
++/*
++ * Blank events will be passed to the actual device handling the backlight when
++ * we return zero here.
++ */
++static int ht16k33_blank(int blank, struct fb_info *info)
++{
++ return 0;
++}
++
+ static int ht16k33_mmap(struct fb_info *info, struct vm_area_struct *vma)
+ {
+ struct ht16k33_priv *priv = info->par;
+@@ -231,6 +240,7 @@ static const struct fb_ops ht16k33_fb_ops = {
+ .owner = THIS_MODULE,
+ .fb_read = fb_sys_read,
+ .fb_write = fb_sys_write,
++ .fb_blank = ht16k33_blank,
+ .fb_fillrect = sys_fillrect,
+ .fb_copyarea = sys_copyarea,
+ .fb_imageblit = sys_imageblit,
+@@ -413,6 +423,33 @@ static int ht16k33_probe(struct i2c_client *client,
+ if (err)
+ return err;
+
++ /* Backlight */
++ memset(&bl_props, 0, sizeof(struct backlight_properties));
++ bl_props.type = BACKLIGHT_RAW;
++ bl_props.max_brightness = MAX_BRIGHTNESS;
++
++ bl = devm_backlight_device_register(&client->dev, DRIVER_NAME"-bl",
++ &client->dev, priv,
++ &ht16k33_bl_ops, &bl_props);
++ if (IS_ERR(bl)) {
++ dev_err(&client->dev, "failed to register backlight\n");
++ return PTR_ERR(bl);
++ }
++
++ err = of_property_read_u32(node, "default-brightness-level",
++ &dft_brightness);
++ if (err) {
++ dft_brightness = MAX_BRIGHTNESS;
++ } else if (dft_brightness > MAX_BRIGHTNESS) {
++ dev_warn(&client->dev,
++ "invalid default brightness level: %u, using %u\n",
++ dft_brightness, MAX_BRIGHTNESS);
++ dft_brightness = MAX_BRIGHTNESS;
++ }
++
++ bl->props.brightness = dft_brightness;
++ ht16k33_bl_update_status(bl);
++
+ /* Framebuffer (2 bytes per column) */
+ BUILD_BUG_ON(PAGE_SIZE < HT16K33_FB_SIZE);
+ fbdev->buffer = (unsigned char *) get_zeroed_page(GFP_KERNEL);
+@@ -445,6 +482,7 @@ static int ht16k33_probe(struct i2c_client *client,
+ fbdev->info->screen_size = HT16K33_FB_SIZE;
+ fbdev->info->fix = ht16k33_fb_fix;
+ fbdev->info->var = ht16k33_fb_var;
++ fbdev->info->bl_dev = bl;
+ fbdev->info->pseudo_palette = NULL;
+ fbdev->info->flags = FBINFO_FLAG_DEFAULT;
+ fbdev->info->par = priv;
+@@ -460,34 +498,6 @@ static int ht16k33_probe(struct i2c_client *client,
+ goto err_fbdev_unregister;
+ }
+
+- /* Backlight */
+- memset(&bl_props, 0, sizeof(struct backlight_properties));
+- bl_props.type = BACKLIGHT_RAW;
+- bl_props.max_brightness = MAX_BRIGHTNESS;
+-
+- bl = devm_backlight_device_register(&client->dev, DRIVER_NAME"-bl",
+- &client->dev, priv,
+- &ht16k33_bl_ops, &bl_props);
+- if (IS_ERR(bl)) {
+- dev_err(&client->dev, "failed to register backlight\n");
+- err = PTR_ERR(bl);
+- goto err_fbdev_unregister;
+- }
+-
+- err = of_property_read_u32(node, "default-brightness-level",
+- &dft_brightness);
+- if (err) {
+- dft_brightness = MAX_BRIGHTNESS;
+- } else if (dft_brightness > MAX_BRIGHTNESS) {
+- dev_warn(&client->dev,
+- "invalid default brightness level: %u, using %u\n",
+- dft_brightness, MAX_BRIGHTNESS);
+- dft_brightness = MAX_BRIGHTNESS;
+- }
+-
+- bl->props.brightness = dft_brightness;
+- ht16k33_bl_update_status(bl);
+-
+ ht16k33_fb_queue(priv);
+ return 0;
+
+diff --git a/drivers/auxdisplay/img-ascii-lcd.c b/drivers/auxdisplay/img-ascii-lcd.c
+index 1cce409ce5cac..e33ce0151cdfd 100644
+--- a/drivers/auxdisplay/img-ascii-lcd.c
++++ b/drivers/auxdisplay/img-ascii-lcd.c
+@@ -280,6 +280,16 @@ static int img_ascii_lcd_display(struct img_ascii_lcd_ctx *ctx,
+ if (msg[count - 1] == '\n')
+ count--;
+
++ if (!count) {
++ /* clear the LCD */
++ devm_kfree(&ctx->pdev->dev, ctx->message);
++ ctx->message = NULL;
++ ctx->message_len = 0;
++ memset(ctx->curr, ' ', ctx->cfg->num_chars);
++ ctx->cfg->update(ctx);
++ return 0;
++ }
++
+ new_msg = devm_kmalloc(&ctx->pdev->dev, count + 1, GFP_KERNEL);
+ if (!new_msg)
+ return -ENOMEM;
+diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c
+index 38ba08628ccb3..2578b2d454397 100644
+--- a/drivers/auxdisplay/lcd2s.c
++++ b/drivers/auxdisplay/lcd2s.c
+@@ -238,7 +238,7 @@ static int lcd2s_redefine_char(struct charlcd *lcd, char *esc)
+ if (buf[1] > 7)
+ return 1;
+
+- i = 0;
++ i = 2;
+ shift = 0;
+ value = 0;
+ while (*esc && i < LCD2S_CHARACTER_SIZE + 2) {
+@@ -298,6 +298,10 @@ static int lcd2s_i2c_probe(struct i2c_client *i2c,
+ I2C_FUNC_SMBUS_WRITE_BLOCK_DATA))
+ return -EIO;
+
++ lcd2s = devm_kzalloc(&i2c->dev, sizeof(*lcd2s), GFP_KERNEL);
++ if (!lcd2s)
++ return -ENOMEM;
++
+ /* Test, if the display is responding */
+ err = lcd2s_i2c_smbus_write_byte(i2c, LCD2S_CMD_DISPLAY_OFF);
+ if (err < 0)
+@@ -307,12 +311,6 @@ static int lcd2s_i2c_probe(struct i2c_client *i2c,
+ if (!lcd)
+ return -ENOMEM;
+
+- lcd2s = kzalloc(sizeof(struct lcd2s_data), GFP_KERNEL);
+- if (!lcd2s) {
+- err = -ENOMEM;
+- goto fail1;
+- }
+-
+ lcd->drvdata = lcd2s;
+ lcd2s->i2c = i2c;
+ lcd2s->charlcd = lcd;
+@@ -321,26 +319,24 @@ static int lcd2s_i2c_probe(struct i2c_client *i2c,
+ err = device_property_read_u32(&i2c->dev, "display-height-chars",
+ &lcd->height);
+ if (err)
+- goto fail2;
++ goto fail1;
+
+ err = device_property_read_u32(&i2c->dev, "display-width-chars",
+ &lcd->width);
+ if (err)
+- goto fail2;
++ goto fail1;
+
+ lcd->ops = &lcd2s_ops;
+
+ err = charlcd_register(lcd2s->charlcd);
+ if (err)
+- goto fail2;
++ goto fail1;
+
+ i2c_set_clientdata(i2c, lcd2s);
+ return 0;
+
+-fail2:
+- kfree(lcd2s);
+ fail1:
+- kfree(lcd);
++ charlcd_free(lcd2s->charlcd);
+ return err;
+ }
+
+@@ -349,7 +345,7 @@ static int lcd2s_i2c_remove(struct i2c_client *i2c)
+ struct lcd2s_data *lcd2s = i2c_get_clientdata(i2c);
+
+ charlcd_unregister(lcd2s->charlcd);
+- kfree(lcd2s->charlcd);
++ charlcd_free(lcd2s->charlcd);
+ return 0;
+ }
+
+diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
+index 43407665918f3..ef4fc89f085d8 100644
+--- a/drivers/base/arch_topology.c
++++ b/drivers/base/arch_topology.c
+@@ -609,7 +609,7 @@ void update_siblings_masks(unsigned int cpuid)
+ for_each_online_cpu(cpu) {
+ cpu_topo = &cpu_topology[cpu];
+
+- if (cpuid_topo->llc_id == cpu_topo->llc_id) {
++ if (cpu_topo->llc_id != -1 && cpuid_topo->llc_id == cpu_topo->llc_id) {
+ cpumask_set_cpu(cpu, &cpuid_topo->llc_sibling);
+ cpumask_set_cpu(cpuid, &cpu_topo->llc_sibling);
+ }
+diff --git a/drivers/base/component.c b/drivers/base/component.c
+index 5e79299f6c3ff..870485cbbb87c 100644
+--- a/drivers/base/component.c
++++ b/drivers/base/component.c
+@@ -246,7 +246,7 @@ static int try_to_bring_up_master(struct master *master,
+ return 0;
+ }
+
+- if (!devres_open_group(master->parent, NULL, GFP_KERNEL))
++ if (!devres_open_group(master->parent, master, GFP_KERNEL))
+ return -ENOMEM;
+
+ /* Found all components */
+@@ -258,6 +258,7 @@ static int try_to_bring_up_master(struct master *master,
+ return ret;
+ }
+
++ devres_close_group(master->parent, NULL);
+ master->bound = true;
+ return 1;
+ }
+@@ -282,7 +283,7 @@ static void take_down_master(struct master *master)
+ {
+ if (master->bound) {
+ master->ops->unbind(master->parent);
+- devres_release_group(master->parent, NULL);
++ devres_release_group(master->parent, master);
+ master->bound = false;
+ }
+ }
+diff --git a/drivers/base/core.c b/drivers/base/core.c
+index 249da496581a0..8e73a34e10055 100644
+--- a/drivers/base/core.c
++++ b/drivers/base/core.c
+@@ -485,8 +485,7 @@ static void device_link_release_fn(struct work_struct *work)
+ /* Ensure that all references to the link object have been dropped. */
+ device_link_synchronize_removal();
+
+- while (refcount_dec_not_one(&link->rpm_active))
+- pm_runtime_put(link->supplier);
++ pm_runtime_release_supplier(link, true);
+
+ put_device(link->consumer);
+ put_device(link->supplier);
+@@ -821,9 +820,7 @@ struct device_link *device_link_add(struct device *consumer,
+ dev_bus_name(supplier), dev_name(supplier),
+ dev_bus_name(consumer), dev_name(consumer));
+ if (device_register(&link->link_dev)) {
+- put_device(consumer);
+- put_device(supplier);
+- kfree(link);
++ put_device(&link->link_dev);
+ link = NULL;
+ goto out;
+ }
+diff --git a/drivers/base/dd.c b/drivers/base/dd.c
+index 68ea1f949daa9..95ae347df1373 100644
+--- a/drivers/base/dd.c
++++ b/drivers/base/dd.c
+@@ -296,6 +296,7 @@ int driver_deferred_probe_check_state(struct device *dev)
+
+ return -EPROBE_DEFER;
+ }
++EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state);
+
+ static void deferred_probe_timeout_work_func(struct work_struct *work)
+ {
+@@ -629,6 +630,9 @@ re_probe:
+ drv->remove(dev);
+
+ devres_release_all(dev);
++ arch_teardown_dma_ops(dev);
++ kfree(dev->dma_range_map);
++ dev->dma_range_map = NULL;
+ driver_sysfs_remove(dev);
+ dev->driver = NULL;
+ dev_set_drvdata(dev, NULL);
+@@ -806,7 +810,7 @@ static int __init save_async_options(char *buf)
+ pr_warn("Too long list of driver names for 'driver_async_probe'!\n");
+
+ strlcpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN);
+- return 0;
++ return 1;
+ }
+ __setup("driver_async_probe=", save_async_options);
+
+@@ -1208,6 +1212,8 @@ static void __device_release_driver(struct device *dev, struct device *parent)
+
+ devres_release_all(dev);
+ arch_teardown_dma_ops(dev);
++ kfree(dev->dma_range_map);
++ dev->dma_range_map = NULL;
+ dev->driver = NULL;
+ dev_set_drvdata(dev, NULL);
+ if (dev->pm_domain && dev->pm_domain->dismiss)
+diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
+index 8be352ab4ddbf..fa13ad49d2116 100644
+--- a/drivers/base/devtmpfs.c
++++ b/drivers/base/devtmpfs.c
+@@ -59,8 +59,15 @@ static struct dentry *public_dev_mount(struct file_system_type *fs_type, int fla
+ const char *dev_name, void *data)
+ {
+ struct super_block *s = mnt->mnt_sb;
++ int err;
++
+ atomic_inc(&s->s_active);
+ down_write(&s->s_umount);
++ err = reconfigure_single(s, flags, data);
++ if (err < 0) {
++ deactivate_locked_super(s);
++ return ERR_PTR(err);
++ }
+ return dget(s->s_root);
+ }
+
+diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
+index bdbedc6660a87..04ede46f75123 100644
+--- a/drivers/base/firmware_loader/main.c
++++ b/drivers/base/firmware_loader/main.c
+@@ -100,12 +100,15 @@ static struct firmware_cache fw_cache;
+ extern struct builtin_fw __start_builtin_fw[];
+ extern struct builtin_fw __end_builtin_fw[];
+
+-static void fw_copy_to_prealloc_buf(struct firmware *fw,
++static bool fw_copy_to_prealloc_buf(struct firmware *fw,
+ void *buf, size_t size)
+ {
+- if (!buf || size < fw->size)
+- return;
++ if (!buf)
++ return true;
++ if (size < fw->size)
++ return false;
+ memcpy(buf, fw->data, fw->size);
++ return true;
+ }
+
+ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name,
+@@ -117,9 +120,7 @@ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name,
+ if (strcmp(name, b_fw->name) == 0) {
+ fw->size = b_fw->size;
+ fw->data = b_fw->data;
+- fw_copy_to_prealloc_buf(fw, buf, size);
+-
+- return true;
++ return fw_copy_to_prealloc_buf(fw, buf, size);
+ }
+ }
+
+@@ -794,6 +795,8 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
+ size_t offset, u32 opt_flags)
+ {
+ struct firmware *fw = NULL;
++ struct cred *kern_cred = NULL;
++ const struct cred *old_cred;
+ bool nondirect = false;
+ int ret;
+
+@@ -810,6 +813,18 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
+ if (ret <= 0) /* error or already assigned */
+ goto out;
+
++ /*
++ * We are about to try to access the firmware file. Because we may have been
++ * called by a driver when serving an unrelated request from userland, we use
++ * the kernel credentials to read the file.
++ */
++ kern_cred = prepare_kernel_cred(NULL);
++ if (!kern_cred) {
++ ret = -ENOMEM;
++ goto out;
++ }
++ old_cred = override_creds(kern_cred);
++
+ ret = fw_get_filesystem_firmware(device, fw->priv, "", NULL);
+
+ /* Only full reads can support decompression, platform, and sysfs. */
+@@ -835,6 +850,9 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
+ } else
+ ret = assign_fw(fw, device);
+
++ revert_creds(old_cred);
++ put_cred(kern_cred);
++
+ out:
+ if (ret < 0) {
+ fw_abort_batch_reqs(fw);
+diff --git a/drivers/base/memory.c b/drivers/base/memory.c
+index 365cd4a7f2397..60c38f9cf1a75 100644
+--- a/drivers/base/memory.c
++++ b/drivers/base/memory.c
+@@ -663,14 +663,16 @@ static int init_memory_block(unsigned long block_id, unsigned long state,
+ mem->nr_vmemmap_pages = nr_vmemmap_pages;
+ INIT_LIST_HEAD(&mem->group_next);
+
++ ret = register_memory(mem);
++ if (ret)
++ return ret;
++
+ if (group) {
+ mem->group = group;
+ list_add(&mem->group_next, &group->memory_blocks);
+ }
+
+- ret = register_memory(mem);
+-
+- return ret;
++ return 0;
+ }
+
+ static int add_memory_block(unsigned long base_section_nr)
+diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
+index 5db704f02e712..7e8039d1884cc 100644
+--- a/drivers/base/power/domain.c
++++ b/drivers/base/power/domain.c
+@@ -2058,9 +2058,9 @@ static int genpd_remove(struct generic_pm_domain *genpd)
+ kfree(link);
+ }
+
+- genpd_debug_remove(genpd);
+ list_del(&genpd->gpd_list_node);
+ genpd_unlock(genpd);
++ genpd_debug_remove(genpd);
+ cancel_work_sync(&genpd->power_off_work);
+ if (genpd_is_cpu_domain(genpd))
+ free_cpumask_var(genpd->cpus);
+diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
+index cbea78e79f3df..8c4819fe73d4c 100644
+--- a/drivers/base/power/main.c
++++ b/drivers/base/power/main.c
+@@ -711,6 +711,7 @@ static void dpm_noirq_resume_devices(pm_message_t state)
+ dev = to_device(dpm_noirq_list.next);
+ get_device(dev);
+ list_move_tail(&dev->power.entry, &dpm_late_early_list);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ if (!is_async(dev)) {
+@@ -725,8 +726,9 @@ static void dpm_noirq_resume_devices(pm_message_t state)
+ }
+ }
+
+- mutex_lock(&dpm_list_mtx);
+ put_device(dev);
++
++ mutex_lock(&dpm_list_mtx);
+ }
+ mutex_unlock(&dpm_list_mtx);
+ async_synchronize_full();
+@@ -852,6 +854,7 @@ void dpm_resume_early(pm_message_t state)
+ dev = to_device(dpm_late_early_list.next);
+ get_device(dev);
+ list_move_tail(&dev->power.entry, &dpm_suspended_list);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ if (!is_async(dev)) {
+@@ -865,8 +868,10 @@ void dpm_resume_early(pm_message_t state)
+ pm_dev_err(dev, state, " early", error);
+ }
+ }
+- mutex_lock(&dpm_list_mtx);
++
+ put_device(dev);
++
++ mutex_lock(&dpm_list_mtx);
+ }
+ mutex_unlock(&dpm_list_mtx);
+ async_synchronize_full();
+@@ -1029,7 +1034,12 @@ void dpm_resume(pm_message_t state)
+ }
+ if (!list_empty(&dev->power.entry))
+ list_move_tail(&dev->power.entry, &dpm_prepared_list);
++
++ mutex_unlock(&dpm_list_mtx);
++
+ put_device(dev);
++
++ mutex_lock(&dpm_list_mtx);
+ }
+ mutex_unlock(&dpm_list_mtx);
+ async_synchronize_full();
+@@ -1051,7 +1061,7 @@ static void device_complete(struct device *dev, pm_message_t state)
+ const char *info = NULL;
+
+ if (dev->power.syscore)
+- return;
++ goto out;
+
+ device_lock(dev);
+
+@@ -1081,6 +1091,7 @@ static void device_complete(struct device *dev, pm_message_t state)
+
+ device_unlock(dev);
+
++out:
+ pm_runtime_put(dev);
+ }
+
+@@ -1106,14 +1117,16 @@ void dpm_complete(pm_message_t state)
+ get_device(dev);
+ dev->power.is_prepared = false;
+ list_move(&dev->power.entry, &list);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ trace_device_pm_callback_start(dev, "", state.event);
+ device_complete(dev, state);
+ trace_device_pm_callback_end(dev, 0);
+
+- mutex_lock(&dpm_list_mtx);
+ put_device(dev);
++
++ mutex_lock(&dpm_list_mtx);
+ }
+ list_splice(&list, &dpm_list);
+ mutex_unlock(&dpm_list_mtx);
+@@ -1298,17 +1311,21 @@ static int dpm_noirq_suspend_devices(pm_message_t state)
+ error = device_suspend_noirq(dev);
+
+ mutex_lock(&dpm_list_mtx);
++
+ if (error) {
+ pm_dev_err(dev, state, " noirq", error);
+ dpm_save_failed_dev(dev_name(dev));
+- put_device(dev);
+- break;
+- }
+- if (!list_empty(&dev->power.entry))
++ } else if (!list_empty(&dev->power.entry)) {
+ list_move(&dev->power.entry, &dpm_noirq_list);
++ }
++
++ mutex_unlock(&dpm_list_mtx);
++
+ put_device(dev);
+
+- if (async_error)
++ mutex_lock(&dpm_list_mtx);
++
++ if (error || async_error)
+ break;
+ }
+ mutex_unlock(&dpm_list_mtx);
+@@ -1475,23 +1492,28 @@ int dpm_suspend_late(pm_message_t state)
+ struct device *dev = to_device(dpm_suspended_list.prev);
+
+ get_device(dev);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ error = device_suspend_late(dev);
+
+ mutex_lock(&dpm_list_mtx);
++
+ if (!list_empty(&dev->power.entry))
+ list_move(&dev->power.entry, &dpm_late_early_list);
+
+ if (error) {
+ pm_dev_err(dev, state, " late", error);
+ dpm_save_failed_dev(dev_name(dev));
+- put_device(dev);
+- break;
+ }
++
++ mutex_unlock(&dpm_list_mtx);
++
+ put_device(dev);
+
+- if (async_error)
++ mutex_lock(&dpm_list_mtx);
++
++ if (error || async_error)
+ break;
+ }
+ mutex_unlock(&dpm_list_mtx);
+@@ -1751,21 +1773,27 @@ int dpm_suspend(pm_message_t state)
+ struct device *dev = to_device(dpm_prepared_list.prev);
+
+ get_device(dev);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ error = device_suspend(dev);
+
+ mutex_lock(&dpm_list_mtx);
++
+ if (error) {
+ pm_dev_err(dev, state, "", error);
+ dpm_save_failed_dev(dev_name(dev));
+- put_device(dev);
+- break;
+- }
+- if (!list_empty(&dev->power.entry))
++ } else if (!list_empty(&dev->power.entry)) {
+ list_move(&dev->power.entry, &dpm_suspended_list);
++ }
++
++ mutex_unlock(&dpm_list_mtx);
++
+ put_device(dev);
+- if (async_error)
++
++ mutex_lock(&dpm_list_mtx);
++
++ if (error || async_error)
+ break;
+ }
+ mutex_unlock(&dpm_list_mtx);
+@@ -1794,9 +1822,6 @@ static int device_prepare(struct device *dev, pm_message_t state)
+ int (*callback)(struct device *) = NULL;
+ int ret = 0;
+
+- if (dev->power.syscore)
+- return 0;
+-
+ /*
+ * If a device's parent goes into runtime suspend at the wrong time,
+ * it won't be possible to resume the device. To prevent this we
+@@ -1805,6 +1830,9 @@ static int device_prepare(struct device *dev, pm_message_t state)
+ */
+ pm_runtime_get_noresume(dev);
+
++ if (dev->power.syscore)
++ return 0;
++
+ device_lock(dev);
+
+ dev->power.wakeup_path = false;
+@@ -1878,10 +1906,11 @@ int dpm_prepare(pm_message_t state)
+ device_block_probing();
+
+ mutex_lock(&dpm_list_mtx);
+- while (!list_empty(&dpm_list)) {
++ while (!list_empty(&dpm_list) && !error) {
+ struct device *dev = to_device(dpm_list.next);
+
+ get_device(dev);
++
+ mutex_unlock(&dpm_list_mtx);
+
+ trace_device_pm_callback_start(dev, "", state.event);
+@@ -1889,21 +1918,23 @@ int dpm_prepare(pm_message_t state)
+ trace_device_pm_callback_end(dev, error);
+
+ mutex_lock(&dpm_list_mtx);
+- if (error) {
+- if (error == -EAGAIN) {
+- put_device(dev);
+- error = 0;
+- continue;
+- }
++
++ if (!error) {
++ dev->power.is_prepared = true;
++ if (!list_empty(&dev->power.entry))
++ list_move_tail(&dev->power.entry, &dpm_prepared_list);
++ } else if (error == -EAGAIN) {
++ error = 0;
++ } else {
+ dev_info(dev, "not prepared for power transition: code %d\n",
+ error);
+- put_device(dev);
+- break;
+ }
+- dev->power.is_prepared = true;
+- if (!list_empty(&dev->power.entry))
+- list_move_tail(&dev->power.entry, &dpm_prepared_list);
++
++ mutex_unlock(&dpm_list_mtx);
++
+ put_device(dev);
++
++ mutex_lock(&dpm_list_mtx);
+ }
+ mutex_unlock(&dpm_list_mtx);
+ trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
+@@ -1991,7 +2022,9 @@ static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
+
+ void device_pm_check_callbacks(struct device *dev)
+ {
+- spin_lock_irq(&dev->power.lock);
++ unsigned long flags;
++
++ spin_lock_irqsave(&dev->power.lock, flags);
+ dev->power.no_pm_callbacks =
+ (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
+ !dev->bus->suspend && !dev->bus->resume)) &&
+@@ -2000,7 +2033,7 @@ void device_pm_check_callbacks(struct device *dev)
+ (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
+ (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
+ !dev->driver->suspend && !dev->driver->resume));
+- spin_unlock_irq(&dev->power.lock);
++ spin_unlock_irqrestore(&dev->power.lock, flags);
+ }
+
+ bool dev_pm_skip_suspend(struct device *dev)
+diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
+index ec94049442b99..44ae3909e64bb 100644
+--- a/drivers/base/power/runtime.c
++++ b/drivers/base/power/runtime.c
+@@ -305,19 +305,40 @@ static int rpm_get_suppliers(struct device *dev)
+ return 0;
+ }
+
++/**
++ * pm_runtime_release_supplier - Drop references to device link's supplier.
++ * @link: Target device link.
++ * @check_idle: Whether or not to check if the supplier device is idle.
++ *
++ * Drop all runtime PM references associated with @link to its supplier device
++ * and if @check_idle is set, check if that device is idle (and so it can be
++ * suspended).
++ */
++void pm_runtime_release_supplier(struct device_link *link, bool check_idle)
++{
++ struct device *supplier = link->supplier;
++
++ /*
++ * The additional power.usage_count check is a safety net in case
++ * the rpm_active refcount becomes saturated, in which case
++ * refcount_dec_not_one() would return true forever, but it is not
++ * strictly necessary.
++ */
++ while (refcount_dec_not_one(&link->rpm_active) &&
++ atomic_read(&supplier->power.usage_count) > 0)
++ pm_runtime_put_noidle(supplier);
++
++ if (check_idle)
++ pm_request_idle(supplier);
++}
++
+ static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
+ {
+ struct device_link *link;
+
+ list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
+- device_links_read_lock_held()) {
+-
+- while (refcount_dec_not_one(&link->rpm_active))
+- pm_runtime_put_noidle(link->supplier);
+-
+- if (try_to_suspend)
+- pm_request_idle(link->supplier);
+- }
++ device_links_read_lock_held())
++ pm_runtime_release_supplier(link, try_to_suspend);
+ }
+
+ static void rpm_put_suppliers(struct device *dev)
+@@ -1770,9 +1791,7 @@ void pm_runtime_drop_link(struct device_link *link)
+ return;
+
+ pm_runtime_drop_link_count(link->consumer);
+-
+- while (refcount_dec_not_one(&link->rpm_active))
+- pm_runtime_put(link->supplier);
++ pm_runtime_release_supplier(link, true);
+ }
+
+ static bool pm_runtime_need_not_resume(struct device *dev)
+diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c
+index 94665037f4a35..72b7a92337b18 100644
+--- a/drivers/base/power/trace.c
++++ b/drivers/base/power/trace.c
+@@ -120,7 +120,11 @@ static unsigned int read_magic_time(void)
+ struct rtc_time time;
+ unsigned int val;
+
+- mc146818_get_time(&time);
++ if (mc146818_get_time(&time) < 0) {
++ pr_err("Unable to read current time from RTC\n");
++ return 0;
++ }
++
+ pr_info("RTC time: %ptRt, date: %ptRd\n", &time, &time);
+ val = time.tm_year; /* 100 years */
+ if (val > 100)
+diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
+index 99bda0da23a82..8666590201c9a 100644
+--- a/drivers/base/power/wakeup.c
++++ b/drivers/base/power/wakeup.c
+@@ -34,7 +34,8 @@ suspend_state_t pm_suspend_target_state;
+ bool events_check_enabled __read_mostly;
+
+ /* First wakeup IRQ seen by the kernel in the last cycle. */
+-unsigned int pm_wakeup_irq __read_mostly;
++static unsigned int wakeup_irq[2] __read_mostly;
++static DEFINE_RAW_SPINLOCK(wakeup_irq_lock);
+
+ /* If greater than 0 and the system is suspending, terminate the suspend. */
+ static atomic_t pm_abort_suspend __read_mostly;
+@@ -942,19 +943,45 @@ void pm_system_cancel_wakeup(void)
+ atomic_dec_if_positive(&pm_abort_suspend);
+ }
+
+-void pm_wakeup_clear(bool reset)
++void pm_wakeup_clear(unsigned int irq_number)
+ {
+- pm_wakeup_irq = 0;
+- if (reset)
++ raw_spin_lock_irq(&wakeup_irq_lock);
++
++ if (irq_number && wakeup_irq[0] == irq_number)
++ wakeup_irq[0] = wakeup_irq[1];
++ else
++ wakeup_irq[0] = 0;
++
++ wakeup_irq[1] = 0;
++
++ raw_spin_unlock_irq(&wakeup_irq_lock);
++
++ if (!irq_number)
+ atomic_set(&pm_abort_suspend, 0);
+ }
+
+ void pm_system_irq_wakeup(unsigned int irq_number)
+ {
+- if (pm_wakeup_irq == 0) {
+- pm_wakeup_irq = irq_number;
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&wakeup_irq_lock, flags);
++
++ if (wakeup_irq[0] == 0)
++ wakeup_irq[0] = irq_number;
++ else if (wakeup_irq[1] == 0)
++ wakeup_irq[1] = irq_number;
++ else
++ irq_number = 0;
++
++ raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags);
++
++ if (irq_number)
+ pm_system_wakeup();
+- }
++}
++
++unsigned int pm_wakeup_irq(void)
++{
++ return wakeup_irq[0];
+ }
+
+ /**
+diff --git a/drivers/base/property.c b/drivers/base/property.c
+index 453918eb7390c..4c77837769c6e 100644
+--- a/drivers/base/property.c
++++ b/drivers/base/property.c
+@@ -1269,8 +1269,10 @@ fwnode_graph_devcon_match(struct fwnode_handle *fwnode, const char *con_id,
+
+ fwnode_graph_for_each_endpoint(fwnode, ep) {
+ node = fwnode_graph_get_remote_port_parent(ep);
+- if (!fwnode_device_is_available(node))
++ if (!fwnode_device_is_available(node)) {
++ fwnode_handle_put(node);
+ continue;
++ }
+
+ ret = match(node, con_id, data);
+ fwnode_handle_put(node);
+diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c
+index d2656581a6085..4a446259a184e 100644
+--- a/drivers/base/regmap/regmap-irq.c
++++ b/drivers/base/regmap/regmap-irq.c
+@@ -189,11 +189,9 @@ static void regmap_irq_sync_unlock(struct irq_data *data)
+ ret = regmap_write(map, reg, d->mask_buf[i]);
+ if (d->chip->clear_ack) {
+ if (d->chip->ack_invert && !ret)
+- ret = regmap_write(map, reg,
+- d->mask_buf[i]);
++ ret = regmap_write(map, reg, UINT_MAX);
+ else if (!ret)
+- ret = regmap_write(map, reg,
+- ~d->mask_buf[i]);
++ ret = regmap_write(map, reg, 0);
+ }
+ if (ret != 0)
+ dev_err(d->map->dev, "Failed to ack 0x%x: %d\n",
+@@ -556,11 +554,9 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
+ data->status_buf[i]);
+ if (chip->clear_ack) {
+ if (chip->ack_invert && !ret)
+- ret = regmap_write(map, reg,
+- data->status_buf[i]);
++ ret = regmap_write(map, reg, UINT_MAX);
+ else if (!ret)
+- ret = regmap_write(map, reg,
+- ~data->status_buf[i]);
++ ret = regmap_write(map, reg, 0);
+ }
+ if (ret != 0)
+ dev_err(map->dev, "Failed to ack 0x%x: %d\n",
+@@ -817,13 +813,9 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode,
+ d->status_buf[i] & d->mask_buf[i]);
+ if (chip->clear_ack) {
+ if (chip->ack_invert && !ret)
+- ret = regmap_write(map, reg,
+- (d->status_buf[i] &
+- d->mask_buf[i]));
++ ret = regmap_write(map, reg, UINT_MAX);
+ else if (!ret)
+- ret = regmap_write(map, reg,
+- ~(d->status_buf[i] &
+- d->mask_buf[i]));
++ ret = regmap_write(map, reg, 0);
+ }
+ if (ret != 0) {
+ dev_err(map->dev, "Failed to ack 0x%x: %d\n",
+diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
+index 21a0c2562ec06..f7811641ed5ae 100644
+--- a/drivers/base/regmap/regmap.c
++++ b/drivers/base/regmap/regmap.c
+@@ -647,6 +647,7 @@ int regmap_attach_dev(struct device *dev, struct regmap *map,
+ if (ret)
+ return ret;
+
++ regmap_debugfs_exit(map);
+ regmap_debugfs_init(map);
+
+ /* Add a devres resource for dev_get_regmap() */
+diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c
+index c46f6a8e14d23..3ba1232ce8451 100644
+--- a/drivers/base/swnode.c
++++ b/drivers/base/swnode.c
+@@ -535,7 +535,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode,
+ return -ENOENT;
+
+ if (nargs_prop) {
+- error = property_entry_read_int_array(swnode->node->properties,
++ error = property_entry_read_int_array(ref->node->properties,
+ nargs_prop, sizeof(u32),
+ &nargs_prop_val, 1);
+ if (error)
+diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
+index ab3e37aa1830c..f93cb989241cc 100644
+--- a/drivers/block/Kconfig
++++ b/drivers/block/Kconfig
+@@ -33,6 +33,22 @@ config BLK_DEV_FD
+ To compile this driver as a module, choose M here: the
+ module will be called floppy.
+
++config BLK_DEV_FD_RAWCMD
++ bool "Support for raw floppy disk commands (DEPRECATED)"
++ depends on BLK_DEV_FD
++ help
++ If you want to use actual physical floppies and expect to do
++ special low-level hardware accesses to them (access and use
++ non-standard formats, for example), then enable this.
++
++ Note that the code enabled by this option is rarely used and
++ might be unstable or insecure, and distros should not enable it.
++
++ Note: FDRAWCMD is deprecated and will be removed from the kernel
++ in the near future.
++
++ If unsure, say N.
++
+ config AMIGA_FLOPPY
+ tristate "Amiga floppy support"
+ depends on AMIGA
+diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c
+index 8b1714021498c..1ed557cb5ed23 100644
+--- a/drivers/block/amiflop.c
++++ b/drivers/block/amiflop.c
+@@ -61,6 +61,7 @@
+ #include <linux/hdreg.h>
+ #include <linux/delay.h>
+ #include <linux/init.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/fs.h>
+ #include <linux/blk-mq.h>
+diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c
+index a093644ac39fb..82faaa4581579 100644
+--- a/drivers/block/ataflop.c
++++ b/drivers/block/ataflop.c
+@@ -68,6 +68,7 @@
+ #include <linux/delay.h>
+ #include <linux/init.h>
+ #include <linux/blk-mq.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/completion.h>
+ #include <linux/wait.h>
+@@ -298,6 +299,7 @@ static struct atari_floppy_struct {
+ disk change detection) */
+ int flags; /* flags */
+ struct gendisk *disk[NUM_DISK_MINORS];
++ bool registered[NUM_DISK_MINORS];
+ int ref;
+ int type;
+ struct blk_mq_tag_set tag_set;
+@@ -456,10 +458,20 @@ static DEFINE_TIMER(fd_timer, check_change);
+
+ static void fd_end_request_cur(blk_status_t err)
+ {
++ DPRINT(("fd_end_request_cur(), bytes %d of %d\n",
++ blk_rq_cur_bytes(fd_request),
++ blk_rq_bytes(fd_request)));
++
+ if (!blk_update_request(fd_request, err,
+ blk_rq_cur_bytes(fd_request))) {
++ DPRINT(("calling __blk_mq_end_request()\n"));
+ __blk_mq_end_request(fd_request, err);
+ fd_request = NULL;
++ } else {
++ /* requeue rest of request */
++ DPRINT(("calling blk_mq_requeue_request()\n"));
++ blk_mq_requeue_request(fd_request, true);
++ fd_request = NULL;
+ }
+ }
+
+@@ -653,9 +665,6 @@ static inline void copy_buffer(void *from, void *to)
+ *p2++ = *p1++;
+ }
+
+-
+-
+-
+ /* General Interrupt Handling */
+
+ static void (*FloppyIRQHandler)( int status ) = NULL;
+@@ -700,12 +709,21 @@ static void fd_error( void )
+ if (fd_request->error_count >= MAX_ERRORS) {
+ printk(KERN_ERR "fd%d: too many errors.\n", SelectedDrive );
+ fd_end_request_cur(BLK_STS_IOERR);
++ finish_fdc();
++ return;
+ }
+ else if (fd_request->error_count == RECALIBRATE_ERRORS) {
+ printk(KERN_WARNING "fd%d: recalibrating\n", SelectedDrive );
+ if (SelectedDrive != -1)
+ SUD.track = -1;
+ }
++ /* need to re-run request to recalibrate */
++ atari_disable_irq( IRQ_MFP_FDC );
++
++ setup_req_params( SelectedDrive );
++ do_fd_action( SelectedDrive );
++
++ atari_enable_irq( IRQ_MFP_FDC );
+ }
+
+
+@@ -732,8 +750,10 @@ static int do_format(int drive, int type, struct atari_format_descr *desc)
+ if (type) {
+ type--;
+ if (type >= NUM_DISK_MINORS ||
+- minor2disktype[type].drive_types > DriveType)
++ minor2disktype[type].drive_types > DriveType) {
++ finish_fdc();
+ return -EINVAL;
++ }
+ }
+
+ q = unit[drive].disk[type]->queue;
+@@ -751,6 +771,7 @@ static int do_format(int drive, int type, struct atari_format_descr *desc)
+ }
+
+ if (!UDT || desc->track >= UDT->blocks/UDT->spt/2 || desc->head >= 2) {
++ finish_fdc();
+ ret = -EINVAL;
+ goto out;
+ }
+@@ -791,6 +812,7 @@ static int do_format(int drive, int type, struct atari_format_descr *desc)
+
+ wait_for_completion(&format_wait);
+
++ finish_fdc();
+ ret = FormatError ? -EIO : 0;
+ out:
+ blk_mq_unquiesce_queue(q);
+@@ -825,6 +847,7 @@ static void do_fd_action( int drive )
+ else {
+ /* all sectors finished */
+ fd_end_request_cur(BLK_STS_OK);
++ finish_fdc();
+ return;
+ }
+ }
+@@ -1229,6 +1252,7 @@ static void fd_rwsec_done1(int status)
+ else {
+ /* all sectors finished */
+ fd_end_request_cur(BLK_STS_OK);
++ finish_fdc();
+ }
+ return;
+
+@@ -1350,7 +1374,7 @@ static void fd_times_out(struct timer_list *unused)
+
+ static void finish_fdc( void )
+ {
+- if (!NeedSeek) {
++ if (!NeedSeek || !stdma_is_locked_by(floppy_irq)) {
+ finish_fdc_done( 0 );
+ }
+ else {
+@@ -1385,7 +1409,8 @@ static void finish_fdc_done( int dummy )
+ start_motor_off_timer();
+
+ local_irq_save(flags);
+- stdma_release();
++ if (stdma_is_locked_by(floppy_irq))
++ stdma_release();
+ local_irq_restore(flags);
+
+ DPRINT(("finish_fdc() finished\n"));
+@@ -1475,15 +1500,6 @@ static void setup_req_params( int drive )
+ ReqTrack, ReqSector, (unsigned long)ReqData ));
+ }
+
+-static void ataflop_commit_rqs(struct blk_mq_hw_ctx *hctx)
+-{
+- spin_lock_irq(&ataflop_lock);
+- atari_disable_irq(IRQ_MFP_FDC);
+- finish_fdc();
+- atari_enable_irq(IRQ_MFP_FDC);
+- spin_unlock_irq(&ataflop_lock);
+-}
+-
+ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ const struct blk_mq_queue_data *bd)
+ {
+@@ -1491,6 +1507,10 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ int drive = floppy - unit;
+ int type = floppy->type;
+
++ DPRINT(("Queue request: drive %d type %d sectors %d of %d last %d\n",
++ drive, type, blk_rq_cur_sectors(bd->rq),
++ blk_rq_sectors(bd->rq), bd->last));
++
+ spin_lock_irq(&ataflop_lock);
+ if (fd_request) {
+ spin_unlock_irq(&ataflop_lock);
+@@ -1511,6 +1531,7 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ /* drive not connected */
+ printk(KERN_ERR "Unknown Device: fd%d\n", drive );
+ fd_end_request_cur(BLK_STS_IOERR);
++ stdma_release();
+ goto out;
+ }
+
+@@ -1527,11 +1548,13 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ if (--type >= NUM_DISK_MINORS) {
+ printk(KERN_WARNING "fd%d: invalid disk format", drive );
+ fd_end_request_cur(BLK_STS_IOERR);
++ stdma_release();
+ goto out;
+ }
+ if (minor2disktype[type].drive_types > DriveType) {
+ printk(KERN_WARNING "fd%d: unsupported disk format", drive );
+ fd_end_request_cur(BLK_STS_IOERR);
++ stdma_release();
+ goto out;
+ }
+ type = minor2disktype[type].index;
+@@ -1550,8 +1573,6 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ setup_req_params( drive );
+ do_fd_action( drive );
+
+- if (bd->last)
+- finish_fdc();
+ atari_enable_irq( IRQ_MFP_FDC );
+
+ out:
+@@ -1634,6 +1655,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode,
+ /* what if type > 0 here? Overwrite specified entry ? */
+ if (type) {
+ /* refuse to re-set a predefined type for now */
++ finish_fdc();
+ return -EINVAL;
+ }
+
+@@ -1701,8 +1723,10 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode,
+
+ /* sanity check */
+ if (setprm.track != dtp->blocks/dtp->spt/2 ||
+- setprm.head != 2)
++ setprm.head != 2) {
++ finish_fdc();
+ return -EINVAL;
++ }
+
+ UDT = dtp;
+ set_capacity(disk, UDT->blocks);
+@@ -1962,7 +1986,6 @@ static const struct block_device_operations floppy_fops = {
+
+ static const struct blk_mq_ops ataflop_mq_ops = {
+ .queue_rq = ataflop_queue_rq,
+- .commit_rqs = ataflop_commit_rqs,
+ };
+
+ static int ataflop_alloc_disk(unsigned int drive, unsigned int type)
+@@ -1986,8 +2009,6 @@ static int ataflop_alloc_disk(unsigned int drive, unsigned int type)
+ return 0;
+ }
+
+-static DEFINE_MUTEX(ataflop_probe_lock);
+-
+ static void ataflop_probe(dev_t dev)
+ {
+ int drive = MINOR(dev) & 3;
+@@ -1998,12 +2019,46 @@ static void ataflop_probe(dev_t dev)
+
+ if (drive >= FD_MAX_UNITS || type >= NUM_DISK_MINORS)
+ return;
+- mutex_lock(&ataflop_probe_lock);
+ if (!unit[drive].disk[type]) {
+- if (ataflop_alloc_disk(drive, type) == 0)
++ if (ataflop_alloc_disk(drive, type) == 0) {
+ add_disk(unit[drive].disk[type]);
++ unit[drive].registered[type] = true;
++ }
++ }
++}
++
++static void atari_floppy_cleanup(void)
++{
++ int i;
++ int type;
++
++ for (i = 0; i < FD_MAX_UNITS; i++) {
++ for (type = 0; type < NUM_DISK_MINORS; type++) {
++ if (!unit[i].disk[type])
++ continue;
++ del_gendisk(unit[i].disk[type]);
++ blk_cleanup_queue(unit[i].disk[type]->queue);
++ put_disk(unit[i].disk[type]);
++ }
++ blk_mq_free_tag_set(&unit[i].tag_set);
++ }
++
++ del_timer_sync(&fd_timer);
++ atari_stram_free(DMABuffer);
++}
++
++static void atari_cleanup_floppy_disk(struct atari_floppy_struct *fs)
++{
++ int type;
++
++ for (type = 0; type < NUM_DISK_MINORS; type++) {
++ if (!fs->disk[type])
++ continue;
++ if (fs->registered[type])
++ del_gendisk(fs->disk[type]);
++ blk_cleanup_disk(fs->disk[type]);
+ }
+- mutex_unlock(&ataflop_probe_lock);
++ blk_mq_free_tag_set(&fs->tag_set);
+ }
+
+ static int __init atari_floppy_init (void)
+@@ -2015,11 +2070,6 @@ static int __init atari_floppy_init (void)
+ /* Amiga, Mac, ... don't have Atari-compatible floppy :-) */
+ return -ENODEV;
+
+- mutex_lock(&ataflop_probe_lock);
+- ret = __register_blkdev(FLOPPY_MAJOR, "fd", ataflop_probe);
+- if (ret)
+- goto out_unlock;
+-
+ for (i = 0; i < FD_MAX_UNITS; i++) {
+ memset(&unit[i].tag_set, 0, sizeof(unit[i].tag_set));
+ unit[i].tag_set.ops = &ataflop_mq_ops;
+@@ -2065,6 +2115,7 @@ static int __init atari_floppy_init (void)
+ unit[i].track = -1;
+ unit[i].flags = 0;
+ add_disk(unit[i].disk[0]);
++ unit[i].registered[0] = true;
+ }
+
+ printk(KERN_INFO "Atari floppy driver: max. %cD, %strack buffering\n",
+@@ -2072,18 +2123,17 @@ static int __init atari_floppy_init (void)
+ UseTrackbuffer ? "" : "no ");
+ config_types();
+
+- return 0;
++ ret = __register_blkdev(FLOPPY_MAJOR, "fd", ataflop_probe);
++ if (ret) {
++ printk(KERN_ERR "atari_floppy_init: cannot register block device\n");
++ atari_floppy_cleanup();
++ }
++ return ret;
+
+ err:
+- while (--i >= 0) {
+- blk_cleanup_queue(unit[i].disk[0]->queue);
+- put_disk(unit[i].disk[0]);
+- blk_mq_free_tag_set(&unit[i].tag_set);
+- }
++ while (--i >= 0)
++ atari_cleanup_floppy_disk(&unit[i]);
+
+- unregister_blkdev(FLOPPY_MAJOR, "fd");
+-out_unlock:
+- mutex_unlock(&ataflop_probe_lock);
+ return ret;
+ }
+
+@@ -2128,22 +2178,8 @@ __setup("floppy=", atari_floppy_setup);
+
+ static void __exit atari_floppy_exit(void)
+ {
+- int i, type;
+-
+- for (i = 0; i < FD_MAX_UNITS; i++) {
+- for (type = 0; type < NUM_DISK_MINORS; type++) {
+- if (!unit[i].disk[type])
+- continue;
+- del_gendisk(unit[i].disk[type]);
+- blk_cleanup_queue(unit[i].disk[type]->queue);
+- put_disk(unit[i].disk[type]);
+- }
+- blk_mq_free_tag_set(&unit[i].tag_set);
+- }
+ unregister_blkdev(FLOPPY_MAJOR, "fd");
+-
+- del_timer_sync(&fd_timer);
+- atari_stram_free( DMABuffer );
++ atari_floppy_cleanup();
+ }
+
+ module_init(atari_floppy_init)
+diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
+index 5d9181382ce19..0a5766a2f1618 100644
+--- a/drivers/block/drbd/drbd_int.h
++++ b/drivers/block/drbd/drbd_int.h
+@@ -1642,22 +1642,22 @@ struct sib_info {
+ };
+ void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib);
+
+-extern void notify_resource_state(struct sk_buff *,
++extern int notify_resource_state(struct sk_buff *,
+ unsigned int,
+ struct drbd_resource *,
+ struct resource_info *,
+ enum drbd_notification_type);
+-extern void notify_device_state(struct sk_buff *,
++extern int notify_device_state(struct sk_buff *,
+ unsigned int,
+ struct drbd_device *,
+ struct device_info *,
+ enum drbd_notification_type);
+-extern void notify_connection_state(struct sk_buff *,
++extern int notify_connection_state(struct sk_buff *,
+ unsigned int,
+ struct drbd_connection *,
+ struct connection_info *,
+ enum drbd_notification_type);
+-extern void notify_peer_device_state(struct sk_buff *,
++extern int notify_peer_device_state(struct sk_buff *,
+ unsigned int,
+ struct drbd_peer_device *,
+ struct peer_device_info *,
+diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
+index 55234a558e98b..6db0333b5b7a1 100644
+--- a/drivers/block/drbd/drbd_main.c
++++ b/drivers/block/drbd/drbd_main.c
+@@ -171,7 +171,7 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
+ unsigned int set_size)
+ {
+ struct drbd_request *r;
+- struct drbd_request *req = NULL;
++ struct drbd_request *req = NULL, *tmp = NULL;
+ int expect_epoch = 0;
+ int expect_size = 0;
+
+@@ -225,8 +225,11 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
+ * to catch requests being barrier-acked "unexpectedly".
+ * It usually should find the same req again, or some READ preceding it. */
+ list_for_each_entry(req, &connection->transfer_log, tl_requests)
+- if (req->epoch == expect_epoch)
++ if (req->epoch == expect_epoch) {
++ tmp = req;
+ break;
++ }
++ req = list_prepare_entry(tmp, &connection->transfer_log, tl_requests);
+ list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
+ if (req->epoch != expect_epoch)
+ break;
+@@ -2737,6 +2740,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
+ sprintf(disk->disk_name, "drbd%d", minor);
+ disk->private_data = device;
+
++ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, disk->queue);
+ blk_queue_write_cache(disk->queue, true, true);
+ /* Setting the max_hw_sectors to an odd value of 8kibyte here
+ This triggers a max_bio_size message upon first attach or connect */
+diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
+index 44ccf8b4f4b29..69184cf17b6ad 100644
+--- a/drivers/block/drbd/drbd_nl.c
++++ b/drivers/block/drbd/drbd_nl.c
+@@ -4617,7 +4617,7 @@ static int nla_put_notification_header(struct sk_buff *msg,
+ return drbd_notification_header_to_skb(msg, &nh, true);
+ }
+
+-void notify_resource_state(struct sk_buff *skb,
++int notify_resource_state(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_resource *resource,
+ struct resource_info *resource_info,
+@@ -4659,16 +4659,17 @@ void notify_resource_state(struct sk_buff *skb,
+ if (err && err != -ESRCH)
+ goto failed;
+ }
+- return;
++ return 0;
+
+ nla_put_failure:
+ nlmsg_free(skb);
+ failed:
+ drbd_err(resource, "Error %d while broadcasting event. Event seq:%u\n",
+ err, seq);
++ return err;
+ }
+
+-void notify_device_state(struct sk_buff *skb,
++int notify_device_state(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_device *device,
+ struct device_info *device_info,
+@@ -4708,16 +4709,17 @@ void notify_device_state(struct sk_buff *skb,
+ if (err && err != -ESRCH)
+ goto failed;
+ }
+- return;
++ return 0;
+
+ nla_put_failure:
+ nlmsg_free(skb);
+ failed:
+ drbd_err(device, "Error %d while broadcasting event. Event seq:%u\n",
+ err, seq);
++ return err;
+ }
+
+-void notify_connection_state(struct sk_buff *skb,
++int notify_connection_state(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_connection *connection,
+ struct connection_info *connection_info,
+@@ -4757,16 +4759,17 @@ void notify_connection_state(struct sk_buff *skb,
+ if (err && err != -ESRCH)
+ goto failed;
+ }
+- return;
++ return 0;
+
+ nla_put_failure:
+ nlmsg_free(skb);
+ failed:
+ drbd_err(connection, "Error %d while broadcasting event. Event seq:%u\n",
+ err, seq);
++ return err;
+ }
+
+-void notify_peer_device_state(struct sk_buff *skb,
++int notify_peer_device_state(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_peer_device *peer_device,
+ struct peer_device_info *peer_device_info,
+@@ -4807,13 +4810,14 @@ void notify_peer_device_state(struct sk_buff *skb,
+ if (err && err != -ESRCH)
+ goto failed;
+ }
+- return;
++ return 0;
+
+ nla_put_failure:
+ nlmsg_free(skb);
+ failed:
+ drbd_err(peer_device, "Error %d while broadcasting event. Event seq:%u\n",
+ err, seq);
++ return err;
+ }
+
+ void notify_helper(enum drbd_notification_type type,
+@@ -4864,7 +4868,7 @@ fail:
+ err, seq);
+ }
+
+-static void notify_initial_state_done(struct sk_buff *skb, unsigned int seq)
++static int notify_initial_state_done(struct sk_buff *skb, unsigned int seq)
+ {
+ struct drbd_genlmsghdr *dh;
+ int err;
+@@ -4878,11 +4882,12 @@ static void notify_initial_state_done(struct sk_buff *skb, unsigned int seq)
+ if (nla_put_notification_header(skb, NOTIFY_EXISTS))
+ goto nla_put_failure;
+ genlmsg_end(skb, dh);
+- return;
++ return 0;
+
+ nla_put_failure:
+ nlmsg_free(skb);
+ pr_err("Error %d sending event. Event seq:%u\n", err, seq);
++ return err;
+ }
+
+ static void free_state_changes(struct list_head *list)
+@@ -4909,6 +4914,7 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+ unsigned int seq = cb->args[2];
+ unsigned int n;
+ enum drbd_notification_type flags = 0;
++ int err = 0;
+
+ /* There is no need for taking notification_mutex here: it doesn't
+ matter if the initial state events mix with later state chage
+@@ -4917,32 +4923,32 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+
+ cb->args[5]--;
+ if (cb->args[5] == 1) {
+- notify_initial_state_done(skb, seq);
++ err = notify_initial_state_done(skb, seq);
+ goto out;
+ }
+ n = cb->args[4]++;
+ if (cb->args[4] < cb->args[3])
+ flags |= NOTIFY_CONTINUES;
+ if (n < 1) {
+- notify_resource_state_change(skb, seq, state_change->resource,
++ err = notify_resource_state_change(skb, seq, state_change->resource,
+ NOTIFY_EXISTS | flags);
+ goto next;
+ }
+ n--;
+ if (n < state_change->n_connections) {
+- notify_connection_state_change(skb, seq, &state_change->connections[n],
++ err = notify_connection_state_change(skb, seq, &state_change->connections[n],
+ NOTIFY_EXISTS | flags);
+ goto next;
+ }
+ n -= state_change->n_connections;
+ if (n < state_change->n_devices) {
+- notify_device_state_change(skb, seq, &state_change->devices[n],
++ err = notify_device_state_change(skb, seq, &state_change->devices[n],
+ NOTIFY_EXISTS | flags);
+ goto next;
+ }
+ n -= state_change->n_devices;
+ if (n < state_change->n_devices * state_change->n_connections) {
+- notify_peer_device_state_change(skb, seq, &state_change->peer_devices[n],
++ err = notify_peer_device_state_change(skb, seq, &state_change->peer_devices[n],
+ NOTIFY_EXISTS | flags);
+ goto next;
+ }
+@@ -4957,7 +4963,10 @@ next:
+ cb->args[4] = 0;
+ }
+ out:
+- return skb->len;
++ if (err)
++ return err;
++ else
++ return skb->len;
+ }
+
+ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
+index 5ca233644d705..47e0d105b462e 100644
+--- a/drivers/block/drbd/drbd_req.c
++++ b/drivers/block/drbd/drbd_req.c
+@@ -180,7 +180,8 @@ void start_new_tl_epoch(struct drbd_connection *connection)
+ void complete_master_bio(struct drbd_device *device,
+ struct bio_and_error *m)
+ {
+- m->bio->bi_status = errno_to_blk_status(m->error);
++ if (unlikely(m->error))
++ m->bio->bi_status = errno_to_blk_status(m->error);
+ bio_endio(m->bio);
+ dec_ap_bio(device);
+ }
+diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
+index b8a27818ab3f8..4ee11aef6672b 100644
+--- a/drivers/block/drbd/drbd_state.c
++++ b/drivers/block/drbd/drbd_state.c
+@@ -1537,7 +1537,7 @@ int drbd_bitmap_io_from_worker(struct drbd_device *device,
+ return rv;
+ }
+
+-void notify_resource_state_change(struct sk_buff *skb,
++int notify_resource_state_change(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_resource_state_change *resource_state_change,
+ enum drbd_notification_type type)
+@@ -1550,10 +1550,10 @@ void notify_resource_state_change(struct sk_buff *skb,
+ .res_susp_fen = resource_state_change->susp_fen[NEW],
+ };
+
+- notify_resource_state(skb, seq, resource, &resource_info, type);
++ return notify_resource_state(skb, seq, resource, &resource_info, type);
+ }
+
+-void notify_connection_state_change(struct sk_buff *skb,
++int notify_connection_state_change(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_connection_state_change *connection_state_change,
+ enum drbd_notification_type type)
+@@ -1564,10 +1564,10 @@ void notify_connection_state_change(struct sk_buff *skb,
+ .conn_role = connection_state_change->peer_role[NEW],
+ };
+
+- notify_connection_state(skb, seq, connection, &connection_info, type);
++ return notify_connection_state(skb, seq, connection, &connection_info, type);
+ }
+
+-void notify_device_state_change(struct sk_buff *skb,
++int notify_device_state_change(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_device_state_change *device_state_change,
+ enum drbd_notification_type type)
+@@ -1577,10 +1577,10 @@ void notify_device_state_change(struct sk_buff *skb,
+ .dev_disk_state = device_state_change->disk_state[NEW],
+ };
+
+- notify_device_state(skb, seq, device, &device_info, type);
++ return notify_device_state(skb, seq, device, &device_info, type);
+ }
+
+-void notify_peer_device_state_change(struct sk_buff *skb,
++int notify_peer_device_state_change(struct sk_buff *skb,
+ unsigned int seq,
+ struct drbd_peer_device_state_change *p,
+ enum drbd_notification_type type)
+@@ -1594,7 +1594,7 @@ void notify_peer_device_state_change(struct sk_buff *skb,
+ .peer_resync_susp_dependency = p->resync_susp_dependency[NEW],
+ };
+
+- notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type);
++ return notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type);
+ }
+
+ static void broadcast_state_change(struct drbd_state_change *state_change)
+@@ -1602,7 +1602,7 @@ static void broadcast_state_change(struct drbd_state_change *state_change)
+ struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+ bool resource_state_has_changed;
+ unsigned int n_device, n_connection, n_peer_device, n_peer_devices;
+- void (*last_func)(struct sk_buff *, unsigned int, void *,
++ int (*last_func)(struct sk_buff *, unsigned int, void *,
+ enum drbd_notification_type) = NULL;
+ void *last_arg = NULL;
+
+diff --git a/drivers/block/drbd/drbd_state_change.h b/drivers/block/drbd/drbd_state_change.h
+index ba80f612d6abb..d5b0479bc9a66 100644
+--- a/drivers/block/drbd/drbd_state_change.h
++++ b/drivers/block/drbd/drbd_state_change.h
+@@ -44,19 +44,19 @@ extern struct drbd_state_change *remember_old_state(struct drbd_resource *, gfp_
+ extern void copy_old_to_new_state_change(struct drbd_state_change *);
+ extern void forget_state_change(struct drbd_state_change *);
+
+-extern void notify_resource_state_change(struct sk_buff *,
++extern int notify_resource_state_change(struct sk_buff *,
+ unsigned int,
+ struct drbd_resource_state_change *,
+ enum drbd_notification_type type);
+-extern void notify_connection_state_change(struct sk_buff *,
++extern int notify_connection_state_change(struct sk_buff *,
+ unsigned int,
+ struct drbd_connection_state_change *,
+ enum drbd_notification_type type);
+-extern void notify_device_state_change(struct sk_buff *,
++extern int notify_device_state_change(struct sk_buff *,
+ unsigned int,
+ struct drbd_device_state_change *,
+ enum drbd_notification_type type);
+-extern void notify_peer_device_state_change(struct sk_buff *,
++extern int notify_peer_device_state_change(struct sk_buff *,
+ unsigned int,
+ struct drbd_peer_device_state_change *,
+ enum drbd_notification_type type);
+diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
+index fef79ea52e3ed..db0b3e8982fe5 100644
+--- a/drivers/block/floppy.c
++++ b/drivers/block/floppy.c
+@@ -184,6 +184,7 @@ static int print_unex = 1;
+ #include <linux/ioport.h>
+ #include <linux/interrupt.h>
+ #include <linux/init.h>
++#include <linux/major.h>
+ #include <linux/platform_device.h>
+ #include <linux/mod_devicetable.h>
+ #include <linux/mutex.h>
+@@ -508,8 +509,8 @@ static unsigned long fdc_busy;
+ static DECLARE_WAIT_QUEUE_HEAD(fdc_wait);
+ static DECLARE_WAIT_QUEUE_HEAD(command_done);
+
+-/* Errors during formatting are counted here. */
+-static int format_errors;
++/* errors encountered on the current (or last) request */
++static int floppy_errors;
+
+ /* Format request descriptor. */
+ static struct format_descr format_req;
+@@ -529,7 +530,6 @@ static struct format_descr format_req;
+ static char *floppy_track_buffer;
+ static int max_buffer_sectors;
+
+-static int *errors;
+ typedef void (*done_f)(int);
+ static const struct cont_t {
+ void (*interrupt)(void);
+@@ -1014,7 +1014,7 @@ static DECLARE_DELAYED_WORK(fd_timer, fd_timer_workfn);
+ static void cancel_activity(void)
+ {
+ do_floppy = NULL;
+- cancel_delayed_work_sync(&fd_timer);
++ cancel_delayed_work(&fd_timer);
+ cancel_work_sync(&floppy_work);
+ }
+
+@@ -1454,7 +1454,7 @@ static int interpret_errors(void)
+ if (drive_params[current_drive].flags & FTD_MSG)
+ DPRINT("Over/Underrun - retrying\n");
+ bad = 0;
+- } else if (*errors >= drive_params[current_drive].max_errors.reporting) {
++ } else if (floppy_errors >= drive_params[current_drive].max_errors.reporting) {
+ print_errors();
+ }
+ if (reply_buffer[ST2] & ST2_WC || reply_buffer[ST2] & ST2_BC)
+@@ -2094,7 +2094,7 @@ static void bad_flp_intr(void)
+ if (!next_valid_format(current_drive))
+ return;
+ }
+- err_count = ++(*errors);
++ err_count = ++floppy_errors;
+ INFBOUND(write_errors[current_drive].badness, err_count);
+ if (err_count > drive_params[current_drive].max_errors.abort)
+ cont->done(0);
+@@ -2240,9 +2240,8 @@ static int do_format(int drive, struct format_descr *tmp_format_req)
+ return -EINVAL;
+ }
+ format_req = *tmp_format_req;
+- format_errors = 0;
+ cont = &format_cont;
+- errors = &format_errors;
++ floppy_errors = 0;
+ ret = wait_til_done(redo_format, true);
+ if (ret == -EINTR)
+ return -EINTR;
+@@ -2760,10 +2759,11 @@ static int set_next_request(void)
+ current_req = list_first_entry_or_null(&floppy_reqs, struct request,
+ queuelist);
+ if (current_req) {
+- current_req->error_count = 0;
++ floppy_errors = 0;
+ list_del_init(&current_req->queuelist);
++ return 1;
+ }
+- return current_req != NULL;
++ return 0;
+ }
+
+ /* Starts or continues processing request. Will automatically unlock the
+@@ -2822,7 +2822,6 @@ do_request:
+ _floppy = floppy_type + drive_params[current_drive].autodetect[drive_state[current_drive].probed_format];
+ } else
+ probing = 0;
+- errors = &(current_req->error_count);
+ tmp = make_raw_rw_request();
+ if (tmp < 2) {
+ request_done(tmp);
+@@ -2983,6 +2982,8 @@ static const char *drive_name(int type, int drive)
+ return "(null)";
+ }
+
++#ifdef CONFIG_BLK_DEV_FD_RAWCMD
++
+ /* raw commands */
+ static void raw_cmd_done(int flag)
+ {
+@@ -3080,6 +3081,8 @@ static void raw_cmd_free(struct floppy_raw_cmd **ptr)
+ }
+ }
+
++#define MAX_LEN (1UL << MAX_ORDER << PAGE_SHIFT)
++
+ static int raw_cmd_copyin(int cmd, void __user *param,
+ struct floppy_raw_cmd **rcmd)
+ {
+@@ -3107,7 +3110,7 @@ loop:
+ ptr->resultcode = 0;
+
+ if (ptr->flags & (FD_RAW_READ | FD_RAW_WRITE)) {
+- if (ptr->length <= 0)
++ if (ptr->length <= 0 || ptr->length >= MAX_LEN)
+ return -EINVAL;
+ ptr->kernel_data = (char *)fd_dma_mem_alloc(ptr->length);
+ fallback_on_nodma_alloc(&ptr->kernel_data, ptr->length);
+@@ -3180,6 +3183,35 @@ static int raw_cmd_ioctl(int cmd, void __user *param)
+ return ret;
+ }
+
++static int floppy_raw_cmd_ioctl(int type, int drive, int cmd,
++ void __user *param)
++{
++ int ret;
++
++ pr_warn_once("Note: FDRAWCMD is deprecated and will be removed from the kernel in the near future.\n");
++
++ if (type)
++ return -EINVAL;
++ if (lock_fdc(drive))
++ return -EINTR;
++ set_floppy(drive);
++ ret = raw_cmd_ioctl(cmd, param);
++ if (ret == -EINTR)
++ return -EINTR;
++ process_fd_request();
++ return ret;
++}
++
++#else /* CONFIG_BLK_DEV_FD_RAWCMD */
++
++static int floppy_raw_cmd_ioctl(int type, int drive, int cmd,
++ void __user *param)
++{
++ return -EOPNOTSUPP;
++}
++
++#endif
++
+ static int invalidate_drive(struct block_device *bdev)
+ {
+ /* invalidate the buffer track to force a reread */
+@@ -3368,7 +3400,6 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int
+ {
+ int drive = (long)bdev->bd_disk->private_data;
+ int type = ITYPE(drive_state[drive].fd_device);
+- int i;
+ int ret;
+ int size;
+ union inparam {
+@@ -3519,16 +3550,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int
+ outparam = &write_errors[drive];
+ break;
+ case FDRAWCMD:
+- if (type)
+- return -EINVAL;
+- if (lock_fdc(drive))
+- return -EINTR;
+- set_floppy(drive);
+- i = raw_cmd_ioctl(cmd, (void __user *)param);
+- if (i == -EINTR)
+- return -EINTR;
+- process_fd_request();
+- return i;
++ return floppy_raw_cmd_ioctl(type, drive, cmd, (void __user *)param);
+ case FDTWADDLE:
+ if (lock_fdc(drive))
+ return -EINTR;
+@@ -4478,6 +4500,7 @@ static const struct blk_mq_ops floppy_mq_ops = {
+ };
+
+ static struct platform_device floppy_device[N_DRIVE];
++static bool registered[N_DRIVE];
+
+ static bool floppy_available(int drive)
+ {
+@@ -4693,6 +4716,8 @@ static int __init do_floppy_init(void)
+ if (err)
+ goto out_remove_drives;
+
++ registered[drive] = true;
++
+ device_add_disk(&floppy_device[drive].dev, disks[drive][0],
+ NULL);
+ }
+@@ -4703,7 +4728,8 @@ out_remove_drives:
+ while (drive--) {
+ if (floppy_available(drive)) {
+ del_gendisk(disks[drive][0]);
+- platform_device_unregister(&floppy_device[drive]);
++ if (registered[drive])
++ platform_device_unregister(&floppy_device[drive]);
+ }
+ }
+ out_release_dma:
+@@ -4946,7 +4972,8 @@ static void __exit floppy_module_exit(void)
+ if (disks[drive][i])
+ del_gendisk(disks[drive][i]);
+ }
+- platform_device_unregister(&floppy_device[drive]);
++ if (registered[drive])
++ platform_device_unregister(&floppy_device[drive]);
+ }
+ for (i = 0; i < ARRAY_SIZE(floppy_type); i++) {
+ if (disks[drive][i])
+diff --git a/drivers/block/loop.c b/drivers/block/loop.c
+index 7bf4686af774e..8cba10aafadb3 100644
+--- a/drivers/block/loop.c
++++ b/drivers/block/loop.c
+@@ -79,6 +79,7 @@
+ #include <linux/ioprio.h>
+ #include <linux/blk-cgroup.h>
+ #include <linux/sched/mm.h>
++#include <linux/statfs.h>
+
+ #include "loop.h"
+
+@@ -272,19 +273,6 @@ static void __loop_update_dio(struct loop_device *lo, bool dio)
+ blk_mq_unfreeze_queue(lo->lo_queue);
+ }
+
+-/**
+- * loop_validate_block_size() - validates the passed in block size
+- * @bsize: size to validate
+- */
+-static int
+-loop_validate_block_size(unsigned short bsize)
+-{
+- if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
+- return -EINVAL;
+-
+- return 0;
+-}
+-
+ /**
+ * loop_set_size() - sets device size and notifies userspace
+ * @lo: struct loop_device to set the size for
+@@ -856,33 +844,33 @@ static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
+
+ static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
+ {
+- return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
++ return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset);
+ }
+
+ static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
+ {
+- return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
++ return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
+ }
+
+ static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
+ {
+ int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
+
+- return sprintf(buf, "%s\n", autoclear ? "1" : "0");
++ return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0");
+ }
+
+ static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
+ {
+ int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
+
+- return sprintf(buf, "%s\n", partscan ? "1" : "0");
++ return sysfs_emit(buf, "%s\n", partscan ? "1" : "0");
+ }
+
+ static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
+ {
+ int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
+
+- return sprintf(buf, "%s\n", dio ? "1" : "0");
++ return sysfs_emit(buf, "%s\n", dio ? "1" : "0");
+ }
+
+ LOOP_ATTR_RO(backing_file);
+@@ -952,8 +940,13 @@ static void loop_config_discard(struct loop_device *lo)
+ granularity = 0;
+
+ } else {
++ struct kstatfs sbuf;
++
+ max_discard_sectors = UINT_MAX >> 9;
+- granularity = inode->i_sb->s_blocksize;
++ if (!vfs_statfs(&file->f_path, &sbuf))
++ granularity = sbuf.f_bsize;
++ else
++ max_discard_sectors = 0;
+ }
+
+ if (max_discard_sectors) {
+@@ -1236,7 +1229,7 @@ static int loop_configure(struct loop_device *lo, fmode_t mode,
+ }
+
+ if (config->block_size) {
+- error = loop_validate_block_size(config->block_size);
++ error = blk_validate_block_size(config->block_size);
+ if (error)
+ goto out_unlock;
+ }
+@@ -1759,7 +1752,7 @@ static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
+ if (lo->lo_state != Lo_bound)
+ return -ENXIO;
+
+- err = loop_validate_block_size(arg);
++ err = blk_validate_block_size(arg);
+ if (err)
+ return err;
+
+@@ -2442,7 +2435,7 @@ static int loop_control_remove(int idx)
+ int ret;
+
+ if (idx < 0) {
+- pr_warn("deleting an unspecified loop device is not supported.\n");
++ pr_warn_once("deleting an unspecified loop device is not supported.\n");
+ return -EINVAL;
+ }
+
+diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
+index 901855717cb53..ba61e72741eab 100644
+--- a/drivers/block/mtip32xx/mtip32xx.c
++++ b/drivers/block/mtip32xx/mtip32xx.c
+@@ -4112,7 +4112,7 @@ static void mtip_pci_remove(struct pci_dev *pdev)
+ "Completion workers still active!\n");
+ }
+
+- blk_set_queue_dying(dd->queue);
++ blk_mark_disk_dead(dd->disk);
+ set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
+
+ /* Clean up the block layer. */
+diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c
+index 26798da661bd4..bcaabf038947c 100644
+--- a/drivers/block/n64cart.c
++++ b/drivers/block/n64cart.c
+@@ -88,7 +88,7 @@ static blk_qc_t n64cart_submit_bio(struct bio *bio)
+ {
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+- struct device *dev = bio->bi_disk->private_data;
++ struct device *dev = bio->bi_bdev->bd_disk->private_data;
+ u32 pos = bio->bi_iter.bi_sector << SECTOR_SHIFT;
+
+ bio_for_each_segment(bvec, bio, iter) {
+diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
+index 1183f7872b713..582b23befb5c4 100644
+--- a/drivers/block/nbd.c
++++ b/drivers/block/nbd.c
+@@ -122,10 +122,10 @@ struct nbd_device {
+ struct work_struct remove_work;
+
+ struct list_head list;
+- struct task_struct *task_recv;
+ struct task_struct *task_setup;
+
+ unsigned long flags;
++ pid_t pid; /* pid of nbd-client, if attached */
+
+ char *backend;
+ };
+@@ -217,7 +217,7 @@ static ssize_t pid_show(struct device *dev,
+ struct gendisk *disk = dev_to_disk(dev);
+ struct nbd_device *nbd = (struct nbd_device *)disk->private_data;
+
+- return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv));
++ return sprintf(buf, "%d\n", nbd->pid);
+ }
+
+ static const struct device_attribute pid_attr = {
+@@ -254,7 +254,7 @@ static void nbd_dev_remove(struct nbd_device *nbd)
+ mutex_lock(&nbd_index_mutex);
+ idr_remove(&nbd_index_idr, nbd->index);
+ mutex_unlock(&nbd_index_mutex);
+-
++ destroy_workqueue(nbd->recv_workq);
+ kfree(nbd);
+ }
+
+@@ -329,7 +329,7 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize,
+ nbd->config->bytesize = bytesize;
+ nbd->config->blksize_bits = __ffs(blksize);
+
+- if (!nbd->task_recv)
++ if (!nbd->pid)
+ return 0;
+
+ if (nbd->config->flags & NBD_FLAG_SEND_TRIM) {
+@@ -1241,7 +1241,7 @@ static void nbd_config_put(struct nbd_device *nbd)
+ if (test_and_clear_bit(NBD_RT_HAS_PID_FILE,
+ &config->runtime_flags))
+ device_remove_file(disk_to_dev(nbd->disk), &pid_attr);
+- nbd->task_recv = NULL;
++ nbd->pid = 0;
+ if (test_and_clear_bit(NBD_RT_HAS_BACKEND_FILE,
+ &config->runtime_flags)) {
+ device_remove_file(disk_to_dev(nbd->disk), &backend_attr);
+@@ -1260,10 +1260,6 @@ static void nbd_config_put(struct nbd_device *nbd)
+ kfree(nbd->config);
+ nbd->config = NULL;
+
+- if (nbd->recv_workq)
+- destroy_workqueue(nbd->recv_workq);
+- nbd->recv_workq = NULL;
+-
+ nbd->tag_set.timeout = 0;
+ nbd->disk->queue->limits.discard_granularity = 0;
+ nbd->disk->queue->limits.discard_alignment = 0;
+@@ -1282,7 +1278,7 @@ static int nbd_start_device(struct nbd_device *nbd)
+ int num_connections = config->num_connections;
+ int error = 0, i;
+
+- if (nbd->task_recv)
++ if (nbd->pid)
+ return -EBUSY;
+ if (!config->socks)
+ return -EINVAL;
+@@ -1292,16 +1288,8 @@ static int nbd_start_device(struct nbd_device *nbd)
+ return -EINVAL;
+ }
+
+- nbd->recv_workq = alloc_workqueue("knbd%d-recv",
+- WQ_MEM_RECLAIM | WQ_HIGHPRI |
+- WQ_UNBOUND, 0, nbd->index);
+- if (!nbd->recv_workq) {
+- dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n");
+- return -ENOMEM;
+- }
+-
+ blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections);
+- nbd->task_recv = current;
++ nbd->pid = task_pid_nr(current);
+
+ nbd_parse_flags(nbd);
+
+@@ -1557,8 +1545,8 @@ static int nbd_dbg_tasks_show(struct seq_file *s, void *unused)
+ {
+ struct nbd_device *nbd = s->private;
+
+- if (nbd->task_recv)
+- seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv));
++ if (nbd->pid)
++ seq_printf(s, "recv: %d\n", nbd->pid);
+
+ return 0;
+ }
+@@ -1725,6 +1713,15 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs)
+ }
+ nbd->disk = disk;
+
++ nbd->recv_workq = alloc_workqueue("nbd%d-recv",
++ WQ_MEM_RECLAIM | WQ_HIGHPRI |
++ WQ_UNBOUND, 0, nbd->index);
++ if (!nbd->recv_workq) {
++ dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n");
++ err = -ENOMEM;
++ goto out_err_disk;
++ }
++
+ /*
+ * Tell the block layer that we are not a rotational device
+ */
+@@ -1749,20 +1746,22 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs)
+ disk->major = NBD_MAJOR;
+
+ /* Too big first_minor can cause duplicate creation of
+- * sysfs files/links, since first_minor will be truncated to
+- * byte in __device_add_disk().
++ * sysfs files/links, since index << part_shift might overflow, or
++ * MKDEV() expect that the max bits of first_minor is 20.
+ */
+ disk->first_minor = index << part_shift;
+- if (disk->first_minor > 0xff) {
++ if (disk->first_minor < index || disk->first_minor > MINORMASK) {
+ err = -EINVAL;
+- goto out_free_idr;
++ goto out_free_work;
+ }
+
+ disk->minors = 1 << part_shift;
+ disk->fops = &nbd_fops;
+ disk->private_data = nbd;
+ sprintf(disk->disk_name, "nbd%d", index);
+- add_disk(disk);
++ err = add_disk(disk);
++ if (err)
++ goto out_free_work;
+
+ /*
+ * Now publish the device.
+@@ -1771,6 +1770,10 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs)
+ nbd_total_devices++;
+ return nbd;
+
++out_free_work:
++ destroy_workqueue(nbd->recv_workq);
++out_err_disk:
++ blk_cleanup_disk(disk);
+ out_free_idr:
+ mutex_lock(&nbd_index_mutex);
+ idr_remove(&nbd_index_idr, index);
+@@ -2024,13 +2027,10 @@ static void nbd_disconnect_and_put(struct nbd_device *nbd)
+ nbd_disconnect(nbd);
+ sock_shutdown(nbd);
+ /*
+- * Make sure recv thread has finished, so it does not drop the last
+- * config ref and try to destroy the workqueue from inside the work
+- * queue. And this also ensure that we can safely call nbd_clear_que()
++ * Make sure recv thread has finished, we can safely call nbd_clear_que()
+ * to cancel the inflight I/Os.
+ */
+- if (nbd->recv_workq)
+- flush_workqueue(nbd->recv_workq);
++ flush_workqueue(nbd->recv_workq);
+ nbd_clear_que(nbd);
+ nbd->task_setup = NULL;
+ mutex_unlock(&nbd->config_lock);
+@@ -2135,7 +2135,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
+ mutex_lock(&nbd->config_lock);
+ config = nbd->config;
+ if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) ||
+- !nbd->task_recv) {
++ !nbd->pid) {
+ dev_err(nbd_to_dev(nbd),
+ "not configured, cannot reconfigure\n");
+ ret = -EINVAL;
+diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
+index e65c9d706f6fb..c4a52f33604dc 100644
+--- a/drivers/block/rbd.c
++++ b/drivers/block/rbd.c
+@@ -7182,7 +7182,7 @@ static ssize_t do_rbd_remove(struct bus_type *bus,
+ * IO to complete/fail.
+ */
+ blk_mq_freeze_queue(rbd_dev->disk->queue);
+- blk_set_queue_dying(rbd_dev->disk->queue);
++ blk_mark_disk_dead(rbd_dev->disk);
+ }
+
+ del_gendisk(rbd_dev->disk);
+diff --git a/drivers/block/swim.c b/drivers/block/swim.c
+index 7ccc8d2a41bc6..3911d0833e1b9 100644
+--- a/drivers/block/swim.c
++++ b/drivers/block/swim.c
+@@ -16,6 +16,7 @@
+ #include <linux/fd.h>
+ #include <linux/slab.h>
+ #include <linux/blk-mq.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/hdreg.h>
+ #include <linux/kernel.h>
+diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
+index 303caf2d17d0c..c0b8a26892a51 100644
+--- a/drivers/block/virtio_blk.c
++++ b/drivers/block/virtio_blk.c
+@@ -815,9 +815,17 @@ static int virtblk_probe(struct virtio_device *vdev)
+ err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE,
+ struct virtio_blk_config, blk_size,
+ &blk_size);
+- if (!err)
++ if (!err) {
++ err = blk_validate_block_size(blk_size);
++ if (err) {
++ dev_err(&vdev->dev,
++ "virtio_blk: invalid block size: 0x%x\n",
++ blk_size);
++ goto out_cleanup_disk;
++ }
++
+ blk_queue_logical_block_size(q, blk_size);
+- else
++ } else
+ blk_size = queue_logical_block_size(q);
+
+ /* Use topology information if available */
+@@ -859,9 +867,15 @@ static int virtblk_probe(struct virtio_device *vdev)
+
+ virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
+ &v);
++
++ /*
++ * max_discard_seg == 0 is out of spec but we always
++ * handled it.
++ */
++ if (!v)
++ v = sg_elems - 2;
+ blk_queue_max_discard_segments(q,
+- min_not_zero(v,
+- MAX_DISCARD_SEGMENTS));
++ min(v, MAX_DISCARD_SEGMENTS));
+
+ blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
+ }
+diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
+index 72902104f1112..390817cf12212 100644
+--- a/drivers/block/xen-blkfront.c
++++ b/drivers/block/xen-blkfront.c
+@@ -42,6 +42,7 @@
+ #include <linux/cdrom.h>
+ #include <linux/module.h>
+ #include <linux/slab.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/scatterlist.h>
+ #include <linux/bitmap.h>
+@@ -1290,7 +1291,8 @@ free_shadow:
+ rinfo->ring_ref[i] = GRANT_INVALID_REF;
+ }
+ }
+- free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE));
++ free_pages_exact(rinfo->ring.sring,
++ info->nr_ring_pages * XEN_PAGE_SIZE);
+ rinfo->ring.sring = NULL;
+
+ if (rinfo->irq)
+@@ -1374,9 +1376,15 @@ static int blkif_get_final_status(enum blk_req_status s1,
+ return BLKIF_RSP_OKAY;
+ }
+
+-static bool blkif_completion(unsigned long *id,
+- struct blkfront_ring_info *rinfo,
+- struct blkif_response *bret)
++/*
++ * Return values:
++ * 1 response processed.
++ * 0 missing further responses.
++ * -1 error while processing.
++ */
++static int blkif_completion(unsigned long *id,
++ struct blkfront_ring_info *rinfo,
++ struct blkif_response *bret)
+ {
+ int i = 0;
+ struct scatterlist *sg;
+@@ -1399,7 +1407,7 @@ static bool blkif_completion(unsigned long *id,
+
+ /* Wait the second response if not yet here. */
+ if (s2->status < REQ_DONE)
+- return false;
++ return 0;
+
+ bret->status = blkif_get_final_status(s->status,
+ s2->status);
+@@ -1450,42 +1458,43 @@ static bool blkif_completion(unsigned long *id,
+ }
+ /* Add the persistent grant into the list of free grants */
+ for (i = 0; i < num_grant; i++) {
+- if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
++ if (!gnttab_try_end_foreign_access(s->grants_used[i]->gref)) {
+ /*
+ * If the grant is still mapped by the backend (the
+ * backend has chosen to make this grant persistent)
+ * we add it at the head of the list, so it will be
+ * reused first.
+ */
+- if (!info->feature_persistent)
+- pr_alert_ratelimited("backed has not unmapped grant: %u\n",
+- s->grants_used[i]->gref);
++ if (!info->feature_persistent) {
++ pr_alert("backed has not unmapped grant: %u\n",
++ s->grants_used[i]->gref);
++ return -1;
++ }
+ list_add(&s->grants_used[i]->node, &rinfo->grants);
+ rinfo->persistent_gnts_c++;
+ } else {
+ /*
+- * If the grant is not mapped by the backend we end the
+- * foreign access and add it to the tail of the list,
+- * so it will not be picked again unless we run out of
+- * persistent grants.
++ * If the grant is not mapped by the backend we add it
++ * to the tail of the list, so it will not be picked
++ * again unless we run out of persistent grants.
+ */
+- gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
+ s->grants_used[i]->gref = GRANT_INVALID_REF;
+ list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
+ }
+ }
+ if (s->req.operation == BLKIF_OP_INDIRECT) {
+ for (i = 0; i < INDIRECT_GREFS(num_grant); i++) {
+- if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
+- if (!info->feature_persistent)
+- pr_alert_ratelimited("backed has not unmapped grant: %u\n",
+- s->indirect_grants[i]->gref);
++ if (!gnttab_try_end_foreign_access(s->indirect_grants[i]->gref)) {
++ if (!info->feature_persistent) {
++ pr_alert("backed has not unmapped grant: %u\n",
++ s->indirect_grants[i]->gref);
++ return -1;
++ }
+ list_add(&s->indirect_grants[i]->node, &rinfo->grants);
+ rinfo->persistent_gnts_c++;
+ } else {
+ struct page *indirect_page;
+
+- gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
+ /*
+ * Add the used indirect page back to the list of
+ * available pages for indirect grefs.
+@@ -1500,7 +1509,7 @@ static bool blkif_completion(unsigned long *id,
+ }
+ }
+
+- return true;
++ return 1;
+ }
+
+ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+@@ -1511,9 +1520,12 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+ unsigned long flags;
+ struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
+ struct blkfront_info *info = rinfo->dev_info;
++ unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
+
+- if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
++ if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) {
++ xen_irq_lateeoi(irq, XEN_EOI_FLAG_SPURIOUS);
+ return IRQ_HANDLED;
++ }
+
+ spin_lock_irqsave(&rinfo->ring_lock, flags);
+ again:
+@@ -1529,6 +1541,8 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+ unsigned long id;
+ unsigned int op;
+
++ eoiflag = 0;
++
+ RING_COPY_RESPONSE(&rinfo->ring, i, &bret);
+ id = bret.id;
+
+@@ -1561,12 +1575,17 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+ }
+
+ if (bret.operation != BLKIF_OP_DISCARD) {
++ int ret;
++
+ /*
+ * We may need to wait for an extra response if the
+ * I/O request is split in 2
+ */
+- if (!blkif_completion(&id, rinfo, &bret))
++ ret = blkif_completion(&id, rinfo, &bret);
++ if (!ret)
+ continue;
++ if (unlikely(ret < 0))
++ goto err;
+ }
+
+ if (add_id_to_freelist(rinfo, id)) {
+@@ -1645,6 +1664,8 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+
+ spin_unlock_irqrestore(&rinfo->ring_lock, flags);
+
++ xen_irq_lateeoi(irq, eoiflag);
++
+ return IRQ_HANDLED;
+
+ err:
+@@ -1652,6 +1673,8 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
+
+ spin_unlock_irqrestore(&rinfo->ring_lock, flags);
+
++ /* No EOI in order to avoid further interrupts. */
++
+ pr_alert("%s disabled for further use\n", info->gd->disk_name);
+ return IRQ_HANDLED;
+ }
+@@ -1669,8 +1692,7 @@ static int setup_blkring(struct xenbus_device *dev,
+ for (i = 0; i < info->nr_ring_pages; i++)
+ rinfo->ring_ref[i] = GRANT_INVALID_REF;
+
+- sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
+- get_order(ring_size));
++ sring = alloc_pages_exact(ring_size, GFP_NOIO);
+ if (!sring) {
+ xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
+ return -ENOMEM;
+@@ -1680,7 +1702,7 @@ static int setup_blkring(struct xenbus_device *dev,
+
+ err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
+ if (err < 0) {
+- free_pages((unsigned long)sring, get_order(ring_size));
++ free_pages_exact(sring, ring_size);
+ rinfo->ring.sring = NULL;
+ goto fail;
+ }
+@@ -1691,8 +1713,8 @@ static int setup_blkring(struct xenbus_device *dev,
+ if (err)
+ goto fail;
+
+- err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
+- "blkif", rinfo);
++ err = bind_evtchn_to_irqhandler_lateeoi(rinfo->evtchn, blkif_interrupt,
++ 0, "blkif", rinfo);
+ if (err <= 0) {
+ xenbus_dev_fatal(dev, err,
+ "bind_evtchn_to_irqhandler failed");
+@@ -2119,7 +2141,7 @@ static void blkfront_closing(struct blkfront_info *info)
+
+ /* No more blkif_request(). */
+ blk_mq_stop_hw_queues(info->rq);
+- blk_set_queue_dying(info->rq);
++ blk_mark_disk_dead(info->gd);
+ set_capacity(info->gd, 0);
+
+ for_each_rinfo(info, rinfo, i) {
+@@ -2520,11 +2542,10 @@ static void purge_persistent_grants(struct blkfront_info *info)
+ list_for_each_entry_safe(gnt_list_entry, tmp, &rinfo->grants,
+ node) {
+ if (gnt_list_entry->gref == GRANT_INVALID_REF ||
+- gnttab_query_foreign_access(gnt_list_entry->gref))
++ !gnttab_try_end_foreign_access(gnt_list_entry->gref))
+ continue;
+
+ list_del(&gnt_list_entry->node);
+- gnttab_end_foreign_access(gnt_list_entry->gref, 0, 0UL);
+ rinfo->persistent_gnts_c--;
+ gnt_list_entry->gref = GRANT_INVALID_REF;
+ list_add_tail(&gnt_list_entry->node, &rinfo->grants);
+diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
+index fcaf2750f68f7..6383c81ac5b37 100644
+--- a/drivers/block/zram/zram_drv.c
++++ b/drivers/block/zram/zram_drv.c
+@@ -910,7 +910,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf,
+ zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.',
+ zram_test_flag(zram, index, ZRAM_IDLE) ? 'i' : '.');
+
+- if (count < copied) {
++ if (count <= copied) {
+ zram_slot_unlock(zram, index);
+ break;
+ }
+diff --git a/drivers/bluetooth/bfusb.c b/drivers/bluetooth/bfusb.c
+index 5a321b4076aab..cab93935cc7f1 100644
+--- a/drivers/bluetooth/bfusb.c
++++ b/drivers/bluetooth/bfusb.c
+@@ -628,6 +628,9 @@ static int bfusb_probe(struct usb_interface *intf, const struct usb_device_id *i
+ data->bulk_out_ep = bulk_out_ep->desc.bEndpointAddress;
+ data->bulk_pkt_size = le16_to_cpu(bulk_out_ep->desc.wMaxPacketSize);
+
++ if (!data->bulk_pkt_size)
++ goto done;
++
+ rwlock_init(&data->lock);
+
+ data->reassembly = NULL;
+diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
+index e4182acee488c..d9ceca7a7935c 100644
+--- a/drivers/bluetooth/btbcm.c
++++ b/drivers/bluetooth/btbcm.c
+@@ -8,6 +8,7 @@
+
+ #include <linux/module.h>
+ #include <linux/firmware.h>
++#include <linux/dmi.h>
+ #include <asm/unaligned.h>
+
+ #include <net/bluetooth/bluetooth.h>
+@@ -343,6 +344,52 @@ static struct sk_buff *btbcm_read_usb_product(struct hci_dev *hdev)
+ return skb;
+ }
+
++static const struct dmi_system_id disable_broken_read_transmit_power[] = {
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,1"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,2"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,4"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir8,1"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir8,2"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "iMac20,1"),
++ },
++ },
++ {
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "iMac20,2"),
++ },
++ },
++ { }
++};
++
+ static int btbcm_read_info(struct hci_dev *hdev)
+ {
+ struct sk_buff *skb;
+@@ -363,6 +410,10 @@ static int btbcm_read_info(struct hci_dev *hdev)
+ bt_dev_info(hdev, "BCM: features 0x%2.2x", skb->data[1]);
+ kfree_skb(skb);
+
++ /* Read DMI and disable broken Read LE Min/Max Tx Power */
++ if (dmi_first_match(disable_broken_read_transmit_power))
++ set_bit(HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER, &hdev->quirks);
++
+ return 0;
+ }
+
+diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
+index f1705b46fc889..d122cc9739175 100644
+--- a/drivers/bluetooth/btintel.c
++++ b/drivers/bluetooth/btintel.c
+@@ -2193,8 +2193,15 @@ static int btintel_setup_combined(struct hci_dev *hdev)
+ * As a workaround, send HCI Reset command first which will reset the
+ * number of completed commands and allow normal command processing
+ * from now on.
++ *
++ * Regarding the INTEL_BROKEN_SHUTDOWN_LED flag, these devices maybe
++ * in the SW_RFKILL ON state as a workaround of fixing LED issue during
++ * the shutdown() procedure, and once the device is in SW_RFKILL ON
++ * state, the only way to exit out of it is sending the HCI_Reset
++ * command.
+ */
+- if (btintel_test_flag(hdev, INTEL_BROKEN_INITIAL_NCMD)) {
++ if (btintel_test_flag(hdev, INTEL_BROKEN_INITIAL_NCMD) ||
++ btintel_test_flag(hdev, INTEL_BROKEN_SHUTDOWN_LED)) {
+ skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL,
+ HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+@@ -2256,19 +2263,18 @@ static int btintel_setup_combined(struct hci_dev *hdev)
+
+ /* Apply the device specific HCI quirks
+ *
+- * WBS for SdP - SdP and Stp have a same hw_varaint but
+- * different fw_variant
++ * WBS for SdP - For the Legacy ROM products, only SdP
++ * supports the WBS. But the version information is not
++ * enough to use here because the StP2 and SdP have same
++ * hw_variant and fw_variant. So, this flag is set by
++ * the transport driver (btusb) based on the HW info
++ * (idProduct)
+ */
+- if (ver.hw_variant == 0x08 && ver.fw_variant == 0x22)
++ if (!btintel_test_flag(hdev,
++ INTEL_ROM_LEGACY_NO_WBS_SUPPORT))
+ set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED,
+ &hdev->quirks);
+
+- /* These devices have an issue with LED which doesn't
+- * go off immediately during shutdown. Set the flag
+- * here to send the LED OFF command during shutdown.
+- */
+- btintel_set_flag(hdev, INTEL_BROKEN_LED);
+-
+ err = btintel_legacy_rom_setup(hdev, &ver);
+ break;
+ case 0x0b: /* SfP */
+@@ -2329,10 +2335,14 @@ static int btintel_setup_combined(struct hci_dev *hdev)
+ case 0x12: /* ThP */
+ case 0x13: /* HrP */
+ case 0x14: /* CcP */
+- /* Some legacy bootloader devices from JfP supports both old
+- * and TLV based HCI_Intel_Read_Version command. But we don't
+- * want to use the TLV based setup routines for those legacy
+- * bootloader device.
++ /* Some legacy bootloader devices starting from JfP,
++ * the operational firmware supports both old and TLV based
++ * HCI_Intel_Read_Version command based on the command
++ * parameter.
++ *
++ * For upgrading firmware case, the TLV based version cannot
++ * be used because the firmware filename for legacy bootloader
++ * is based on the old format.
+ *
+ * Also, it is not easy to convert TLV based version from the
+ * legacy version format.
+@@ -2344,6 +2354,20 @@ static int btintel_setup_combined(struct hci_dev *hdev)
+ err = btintel_read_version(hdev, &ver);
+ if (err)
+ return err;
++
++ /* Apply the device specific HCI quirks
++ *
++ * All Legacy bootloader devices support WBS
++ */
++ set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks);
++
++ /* Valid LE States quirk for JfP/ThP familiy */
++ if (ver.hw_variant == 0x11 || ver.hw_variant == 0x12)
++ set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
++
++ /* Setup MSFT Extension support */
++ btintel_set_msft_opcode(hdev, ver.hw_variant);
++
+ err = btintel_bootloader_setup(hdev, &ver);
+ break;
+ case 0x17:
+@@ -2399,9 +2423,10 @@ static int btintel_shutdown_combined(struct hci_dev *hdev)
+
+ /* Some platforms have an issue with BT LED when the interface is
+ * down or BT radio is turned off, which takes 5 seconds to BT LED
+- * goes off. This command turns off the BT LED immediately.
++ * goes off. As a workaround, sends HCI_Intel_SW_RFKILL to put the
++ * device in the RFKILL ON state which turns off the BT LED immediately.
+ */
+- if (btintel_test_flag(hdev, INTEL_BROKEN_LED)) {
++ if (btintel_test_flag(hdev, INTEL_BROKEN_SHUTDOWN_LED)) {
+ skb = __hci_cmd_sync(hdev, 0xfc3f, 0, NULL, HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+ ret = PTR_ERR(skb);
+diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
+index aa64072bbe68d..2b85ebf633211 100644
+--- a/drivers/bluetooth/btintel.h
++++ b/drivers/bluetooth/btintel.h
+@@ -145,8 +145,9 @@ enum {
+ INTEL_FIRMWARE_FAILED,
+ INTEL_BOOTING,
+ INTEL_BROKEN_INITIAL_NCMD,
+- INTEL_BROKEN_LED,
++ INTEL_BROKEN_SHUTDOWN_LED,
+ INTEL_ROM_LEGACY,
++ INTEL_ROM_LEGACY_NO_WBS_SUPPORT,
+
+ __INTEL_NUM_FLAGS,
+ };
+diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c
+index 9872ef18f9fea..ff1f5dfbb6dbf 100644
+--- a/drivers/bluetooth/btmtksdio.c
++++ b/drivers/bluetooth/btmtksdio.c
+@@ -981,6 +981,8 @@ static int btmtksdio_probe(struct sdio_func *func,
+ hdev->manufacturer = 70;
+ set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks);
+
++ sdio_set_drvdata(func, bdev);
++
+ err = hci_register_dev(hdev);
+ if (err < 0) {
+ dev_err(&func->dev, "Can't register HCI device\n");
+@@ -988,8 +990,6 @@ static int btmtksdio_probe(struct sdio_func *func,
+ return err;
+ }
+
+- sdio_set_drvdata(func, bdev);
+-
+ /* pm_runtime_enable would be done after the firmware is being
+ * downloaded because the core layer probably already enables
+ * runtime PM for this func such as the case host->caps &
+@@ -1042,6 +1042,8 @@ static int btmtksdio_runtime_suspend(struct device *dev)
+ if (!bdev)
+ return 0;
+
++ sdio_set_host_pm_flags(func, MMC_PM_KEEP_POWER);
++
+ sdio_claim_host(bdev->func);
+
+ sdio_writel(bdev->func, C_FW_OWN_REQ_SET, MTK_REG_CHLPCR, &err);
+diff --git a/drivers/bluetooth/btmtkuart.c b/drivers/bluetooth/btmtkuart.c
+index e9d91d7c0db48..9ba22b13b4fa0 100644
+--- a/drivers/bluetooth/btmtkuart.c
++++ b/drivers/bluetooth/btmtkuart.c
+@@ -158,8 +158,10 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev,
+ int err;
+
+ hlen = sizeof(*hdr) + wmt_params->dlen;
+- if (hlen > 255)
+- return -EINVAL;
++ if (hlen > 255) {
++ err = -EINVAL;
++ goto err_free_skb;
++ }
+
+ hdr = (struct mtk_wmt_hdr *)&wc;
+ hdr->dir = 1;
+@@ -173,7 +175,7 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev,
+ err = __hci_cmd_send(hdev, 0xfc6f, hlen, &wc);
+ if (err < 0) {
+ clear_bit(BTMTKUART_TX_WAIT_VND_EVT, &bdev->tx_state);
+- return err;
++ goto err_free_skb;
+ }
+
+ /* The vendor specific WMT commands are all answered by a vendor
+@@ -190,13 +192,14 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev,
+ if (err == -EINTR) {
+ bt_dev_err(hdev, "Execution of wmt command interrupted");
+ clear_bit(BTMTKUART_TX_WAIT_VND_EVT, &bdev->tx_state);
+- return err;
++ goto err_free_skb;
+ }
+
+ if (err) {
+ bt_dev_err(hdev, "Execution of wmt command timed out");
+ clear_bit(BTMTKUART_TX_WAIT_VND_EVT, &bdev->tx_state);
+- return -ETIMEDOUT;
++ err = -ETIMEDOUT;
++ goto err_free_skb;
+ }
+
+ /* Parse and handle the return WMT event */
+diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
+index 60d2fce59a71d..a68edbc7be0ff 100644
+--- a/drivers/bluetooth/btusb.c
++++ b/drivers/bluetooth/btusb.c
+@@ -59,7 +59,9 @@ static struct usb_driver btusb_driver;
+ #define BTUSB_WIDEBAND_SPEECH 0x400000
+ #define BTUSB_VALID_LE_STATES 0x800000
+ #define BTUSB_QCA_WCN6855 0x1000000
++#define BTUSB_INTEL_BROKEN_SHUTDOWN_LED 0x2000000
+ #define BTUSB_INTEL_BROKEN_INITIAL_NCMD 0x4000000
++#define BTUSB_INTEL_NO_WBS_SUPPORT 0x8000000
+
+ static const struct usb_device_id btusb_table[] = {
+ /* Generic Bluetooth USB device */
+@@ -295,6 +297,24 @@ static const struct usb_device_id blacklist_table[] = {
+ { USB_DEVICE(0x0cf3, 0xe600), .driver_info = BTUSB_QCA_WCN6855 |
+ BTUSB_WIDEBAND_SPEECH |
+ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x0489, 0xe0cc), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x0489, 0xe0d6), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x0489, 0xe0e3), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x10ab, 0x9309), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x10ab, 0x9409), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x0489, 0xe0d0), .driver_info = BTUSB_QCA_WCN6855 |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
+
+ /* Broadcom BCM2035 */
+ { USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
+@@ -365,10 +385,15 @@ static const struct usb_device_id blacklist_table[] = {
+ { USB_DEVICE(0x8087, 0x0033), .driver_info = BTUSB_INTEL_COMBINED },
+ { USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR },
+ { USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL_COMBINED |
+- BTUSB_INTEL_BROKEN_INITIAL_NCMD },
+- { USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL_COMBINED },
++ BTUSB_INTEL_NO_WBS_SUPPORT |
++ BTUSB_INTEL_BROKEN_INITIAL_NCMD |
++ BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
++ { USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL_COMBINED |
++ BTUSB_INTEL_NO_WBS_SUPPORT |
++ BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
+ { USB_DEVICE(0x8087, 0x0a2b), .driver_info = BTUSB_INTEL_COMBINED },
+- { USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL_COMBINED },
++ { USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL_COMBINED |
++ BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
+ { USB_DEVICE(0x8087, 0x0aaa), .driver_info = BTUSB_INTEL_COMBINED },
+
+ /* Other Intel Bluetooth devices */
+@@ -382,8 +407,18 @@ static const struct usb_device_id blacklist_table[] = {
+ BTUSB_WIDEBAND_SPEECH },
+
+ /* Realtek 8852AE Bluetooth devices */
++ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
+ { USB_DEVICE(0x0bda, 0xc852), .driver_info = BTUSB_REALTEK |
+ BTUSB_WIDEBAND_SPEECH },
++ { USB_DEVICE(0x0bda, 0x385a), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
++ { USB_DEVICE(0x0bda, 0x4852), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
++ { USB_DEVICE(0x04c5, 0x165c), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
++ { USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
+
+ /* Realtek Bluetooth devices */
+ { USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
+@@ -410,10 +445,21 @@ static const struct usb_device_id blacklist_table[] = {
+ { USB_DEVICE(0x13d3, 0x3563), .driver_info = BTUSB_MEDIATEK |
+ BTUSB_WIDEBAND_SPEECH |
+ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x13d3, 0x3564), .driver_info = BTUSB_MEDIATEK |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
+ { USB_DEVICE(0x0489, 0xe0cd), .driver_info = BTUSB_MEDIATEK |
+ BTUSB_WIDEBAND_SPEECH |
+ BTUSB_VALID_LE_STATES },
+
++ /* MediaTek MT7922A Bluetooth devices */
++ { USB_DEVICE(0x0489, 0xe0d8), .driver_info = BTUSB_MEDIATEK |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++ { USB_DEVICE(0x0489, 0xe0d9), .driver_info = BTUSB_MEDIATEK |
++ BTUSB_WIDEBAND_SPEECH |
++ BTUSB_VALID_LE_STATES },
++
+ /* Additional Realtek 8723AE Bluetooth devices */
+ { USB_DEVICE(0x0930, 0x021d), .driver_info = BTUSB_REALTEK },
+ { USB_DEVICE(0x13d3, 0x3394), .driver_info = BTUSB_REALTEK },
+@@ -433,9 +479,15 @@ static const struct usb_device_id blacklist_table[] = {
+ { USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK },
+ { USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
+
++ /* Additional Realtek 8761B Bluetooth devices */
++ { USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
++
+ /* Additional Realtek 8761BU Bluetooth devices */
+ { USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
+ BTUSB_WIDEBAND_SPEECH },
++ { USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK |
++ BTUSB_WIDEBAND_SPEECH },
+
+ /* Additional Realtek 8821AE Bluetooth devices */
+ { USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
+@@ -451,10 +503,6 @@ static const struct usb_device_id blacklist_table[] = {
+ /* Additional Realtek 8822CE Bluetooth devices */
+ { USB_DEVICE(0x04ca, 0x4005), .driver_info = BTUSB_REALTEK |
+ BTUSB_WIDEBAND_SPEECH },
+- /* Bluetooth component of Realtek 8852AE device */
+- { USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK |
+- BTUSB_WIDEBAND_SPEECH },
+-
+ { USB_DEVICE(0x04c5, 0x161f), .driver_info = BTUSB_REALTEK |
+ BTUSB_WIDEBAND_SPEECH },
+ { USB_DEVICE(0x0b05, 0x18ef), .driver_info = BTUSB_REALTEK |
+@@ -2217,6 +2265,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb)
+ skb = bt_skb_alloc(HCI_WMT_MAX_EVENT_SIZE, GFP_ATOMIC);
+ if (!skb) {
+ hdev->stat.err_rx++;
++ kfree(urb->setup_packet);
+ return;
+ }
+
+@@ -2237,6 +2286,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb)
+ data->evt_skb = skb_clone(skb, GFP_ATOMIC);
+ if (!data->evt_skb) {
+ kfree_skb(skb);
++ kfree(urb->setup_packet);
+ return;
+ }
+ }
+@@ -2245,6 +2295,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb)
+ if (err < 0) {
+ kfree_skb(data->evt_skb);
+ data->evt_skb = NULL;
++ kfree(urb->setup_packet);
+ return;
+ }
+
+@@ -2255,6 +2306,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb)
+ wake_up_bit(&data->flags,
+ BTUSB_TX_WAIT_VND_EVT);
+ }
++ kfree(urb->setup_packet);
+ return;
+ } else if (urb->status == -ENOENT) {
+ /* Avoid suspend failed when usb_kill_urb */
+@@ -2275,6 +2327,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb)
+ usb_anchor_urb(urb, &data->ctrl_anchor);
+ err = usb_submit_urb(urb, GFP_ATOMIC);
+ if (err < 0) {
++ kfree(urb->setup_packet);
+ /* -EPERM: urb is being killed;
+ * -ENODEV: device got disconnected
+ */
+@@ -2515,6 +2568,7 @@ static int btusb_mtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwnam
+ } else {
+ bt_dev_err(hdev, "Failed wmt patch dwnld status (%d)",
+ status);
++ err = -EIO;
+ goto err_release_fw;
+ }
+ }
+@@ -2804,11 +2858,16 @@ static int btusb_mtk_setup(struct hci_dev *hdev)
+ case 0x7668:
+ fwname = FIRMWARE_MT7668;
+ break;
++ case 0x7922:
+ case 0x7961:
+ snprintf(fw_bin_name, sizeof(fw_bin_name),
+ "mediatek/BT_RAM_CODE_MT%04x_1_%x_hdr.bin",
+ dev_id & 0xffff, (fw_version & 0xff) + 1);
+ err = btusb_mtk_setup_firmware_79xx(hdev, fw_bin_name);
++ if (err < 0) {
++ bt_dev_err(hdev, "Failed to set up firmware (%d)", err);
++ return err;
++ }
+
+ /* It's Device EndPoint Reset Option Register */
+ btusb_mtk_uhw_reg_write(data, MTK_EP_RST_OPT, MTK_EP_RST_IN_OUT_OPT);
+@@ -2828,6 +2887,7 @@ static int btusb_mtk_setup(struct hci_dev *hdev)
+ }
+
+ hci_set_msft_opcode(hdev, 0xFD30);
++ hci_set_aosp_capable(hdev);
+ goto done;
+ default:
+ bt_dev_err(hdev, "Unsupported hardware variant (%08x)",
+@@ -3806,8 +3866,14 @@ static int btusb_probe(struct usb_interface *intf,
+ hdev->send = btusb_send_frame_intel;
+ hdev->cmd_timeout = btusb_intel_cmd_timeout;
+
++ if (id->driver_info & BTUSB_INTEL_NO_WBS_SUPPORT)
++ btintel_set_flag(hdev, INTEL_ROM_LEGACY_NO_WBS_SUPPORT);
++
+ if (id->driver_info & BTUSB_INTEL_BROKEN_INITIAL_NCMD)
+ btintel_set_flag(hdev, INTEL_BROKEN_INITIAL_NCMD);
++
++ if (id->driver_info & BTUSB_INTEL_BROKEN_SHUTDOWN_LED)
++ btintel_set_flag(hdev, INTEL_BROKEN_SHUTDOWN_LED);
+ }
+
+ if (id->driver_info & BTUSB_MARVELL)
+diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
+index ef54afa293574..7abf99f0ee399 100644
+--- a/drivers/bluetooth/hci_bcm.c
++++ b/drivers/bluetooth/hci_bcm.c
+@@ -1188,7 +1188,12 @@ static int bcm_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ dev->dev = &pdev->dev;
+- dev->irq = platform_get_irq(pdev, 0);
++
++ ret = platform_get_irq(pdev, 0);
++ if (ret < 0)
++ return ret;
++
++ dev->irq = ret;
+
+ /* Initialize routing field to an unused value */
+ dev->pcm_int_params[0] = 0xff;
+diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c
+index 0c0dedece59c5..e0ea9d25bb393 100644
+--- a/drivers/bluetooth/hci_h5.c
++++ b/drivers/bluetooth/hci_h5.c
+@@ -587,9 +587,11 @@ static int h5_recv(struct hci_uart *hu, const void *data, int count)
+ count -= processed;
+ }
+
+- pm_runtime_get(&hu->serdev->dev);
+- pm_runtime_mark_last_busy(&hu->serdev->dev);
+- pm_runtime_put_autosuspend(&hu->serdev->dev);
++ if (hu->serdev) {
++ pm_runtime_get(&hu->serdev->dev);
++ pm_runtime_mark_last_busy(&hu->serdev->dev);
++ pm_runtime_put_autosuspend(&hu->serdev->dev);
++ }
+
+ return 0;
+ }
+@@ -627,9 +629,11 @@ static int h5_enqueue(struct hci_uart *hu, struct sk_buff *skb)
+ break;
+ }
+
+- pm_runtime_get_sync(&hu->serdev->dev);
+- pm_runtime_mark_last_busy(&hu->serdev->dev);
+- pm_runtime_put_autosuspend(&hu->serdev->dev);
++ if (hu->serdev) {
++ pm_runtime_get_sync(&hu->serdev->dev);
++ pm_runtime_mark_last_busy(&hu->serdev->dev);
++ pm_runtime_put_autosuspend(&hu->serdev->dev);
++ }
+
+ return 0;
+ }
+@@ -846,6 +850,8 @@ static int h5_serdev_probe(struct serdev_device *serdev)
+ h5->vnd = data->vnd;
+ }
+
++ if (data->driver_info & H5_INFO_WAKEUP_DISABLE)
++ set_bit(H5_WAKEUP_DISABLE, &h5->flags);
+
+ h5->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_LOW);
+ if (IS_ERR(h5->enable_gpio))
+@@ -860,9 +866,6 @@ static int h5_serdev_probe(struct serdev_device *serdev)
+ if (err)
+ return err;
+
+- if (data->driver_info & H5_INFO_WAKEUP_DISABLE)
+- set_bit(H5_WAKEUP_DISABLE, &h5->flags);
+-
+ return 0;
+ }
+
+@@ -962,11 +965,13 @@ static void h5_btrtl_open(struct h5 *h5)
+ serdev_device_set_parity(h5->hu->serdev, SERDEV_PARITY_EVEN);
+ serdev_device_set_baudrate(h5->hu->serdev, 115200);
+
+- pm_runtime_set_active(&h5->hu->serdev->dev);
+- pm_runtime_use_autosuspend(&h5->hu->serdev->dev);
+- pm_runtime_set_autosuspend_delay(&h5->hu->serdev->dev,
+- SUSPEND_TIMEOUT_MS);
+- pm_runtime_enable(&h5->hu->serdev->dev);
++ if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags)) {
++ pm_runtime_set_active(&h5->hu->serdev->dev);
++ pm_runtime_use_autosuspend(&h5->hu->serdev->dev);
++ pm_runtime_set_autosuspend_delay(&h5->hu->serdev->dev,
++ SUSPEND_TIMEOUT_MS);
++ pm_runtime_enable(&h5->hu->serdev->dev);
++ }
+
+ /* The controller needs up to 500ms to wakeup */
+ gpiod_set_value_cansleep(h5->enable_gpio, 1);
+@@ -976,7 +981,8 @@ static void h5_btrtl_open(struct h5 *h5)
+
+ static void h5_btrtl_close(struct h5 *h5)
+ {
+- pm_runtime_disable(&h5->hu->serdev->dev);
++ if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags))
++ pm_runtime_disable(&h5->hu->serdev->dev);
+
+ gpiod_set_value_cansleep(h5->device_wake_gpio, 0);
+ gpiod_set_value_cansleep(h5->enable_gpio, 0);
+diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
+index 53deea2eb7b4d..8eb7fddfb9300 100644
+--- a/drivers/bluetooth/hci_qca.c
++++ b/drivers/bluetooth/hci_qca.c
+@@ -1927,6 +1927,9 @@ static int qca_power_off(struct hci_dev *hdev)
+ hu->hdev->hw_error = NULL;
+ hu->hdev->cmd_timeout = NULL;
+
++ del_timer_sync(&qca->wake_retrans_timer);
++ del_timer_sync(&qca->tx_idle_timer);
++
+ /* Stop sending shutdown command if soc crashes. */
+ if (soc_type != QCA_ROME
+ && qca->memdump_state == QCA_MEMDUMP_IDLE) {
+@@ -2055,14 +2058,14 @@ static int qca_serdev_probe(struct serdev_device *serdev)
+
+ qcadev->bt_en = devm_gpiod_get_optional(&serdev->dev, "enable",
+ GPIOD_OUT_LOW);
+- if (!qcadev->bt_en && data->soc_type == QCA_WCN6750) {
++ if (IS_ERR_OR_NULL(qcadev->bt_en) && data->soc_type == QCA_WCN6750) {
+ dev_err(&serdev->dev, "failed to acquire BT_EN gpio\n");
+ power_ctrl_enabled = false;
+ }
+
+ qcadev->sw_ctrl = devm_gpiod_get_optional(&serdev->dev, "swctrl",
+ GPIOD_IN);
+- if (!qcadev->sw_ctrl && data->soc_type == QCA_WCN6750)
++ if (IS_ERR_OR_NULL(qcadev->sw_ctrl) && data->soc_type == QCA_WCN6750)
+ dev_warn(&serdev->dev, "failed to acquire SW_CTRL gpio\n");
+
+ qcadev->susclk = devm_clk_get_optional(&serdev->dev, NULL);
+@@ -2084,7 +2087,7 @@ static int qca_serdev_probe(struct serdev_device *serdev)
+
+ qcadev->bt_en = devm_gpiod_get_optional(&serdev->dev, "enable",
+ GPIOD_OUT_LOW);
+- if (!qcadev->bt_en) {
++ if (IS_ERR_OR_NULL(qcadev->bt_en)) {
+ dev_warn(&serdev->dev, "failed to acquire enable gpio\n");
+ power_ctrl_enabled = false;
+ }
+diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
+index 3b00d82d36cf7..4cda890ce6470 100644
+--- a/drivers/bluetooth/hci_serdev.c
++++ b/drivers/bluetooth/hci_serdev.c
+@@ -305,6 +305,8 @@ int hci_uart_register_device(struct hci_uart *hu,
+ if (err)
+ return err;
+
++ percpu_init_rwsem(&hu->proto_lock);
++
+ err = p->open(hu);
+ if (err)
+ goto err_open;
+@@ -327,7 +329,6 @@ int hci_uart_register_device(struct hci_uart *hu,
+
+ INIT_WORK(&hu->init_ready, hci_uart_init_work);
+ INIT_WORK(&hu->write_work, hci_uart_write_work);
+- percpu_init_rwsem(&hu->proto_lock);
+
+ /* Only when vendor specific setup callback is provided, consider
+ * the manufacturer information valid. This avoids filling in the
+diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c
+index 8ab26dec5f6e8..8469f9876dd26 100644
+--- a/drivers/bluetooth/hci_vhci.c
++++ b/drivers/bluetooth/hci_vhci.c
+@@ -121,6 +121,8 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode)
+ if (opcode & 0x80)
+ set_bit(HCI_QUIRK_RAW_DEVICE, &hdev->quirks);
+
++ set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
++
+ if (hci_register_dev(hdev) < 0) {
+ BT_ERR("Can't register HCI device");
+ hci_free_dev(hdev);
+diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c
+index 57908ce4fae85..076e4942a3f0e 100644
+--- a/drivers/bluetooth/virtio_bt.c
++++ b/drivers/bluetooth/virtio_bt.c
+@@ -202,6 +202,9 @@ static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb)
+ hci_skb_pkt_type(skb) = pkt_type;
+ hci_recv_frame(vbt->hdev, skb);
+ break;
++ default:
++ kfree_skb(skb);
++ break;
+ }
+ }
+
+diff --git a/drivers/bus/mhi/core/debugfs.c b/drivers/bus/mhi/core/debugfs.c
+index 858d7516410bb..d818586c229d2 100644
+--- a/drivers/bus/mhi/core/debugfs.c
++++ b/drivers/bus/mhi/core/debugfs.c
+@@ -60,16 +60,16 @@ static int mhi_debugfs_events_show(struct seq_file *m, void *d)
+ }
+
+ seq_printf(m, "Index: %d intmod count: %lu time: %lu",
+- i, (er_ctxt->intmod & EV_CTX_INTMODC_MASK) >>
++ i, (le32_to_cpu(er_ctxt->intmod) & EV_CTX_INTMODC_MASK) >>
+ EV_CTX_INTMODC_SHIFT,
+- (er_ctxt->intmod & EV_CTX_INTMODT_MASK) >>
++ (le32_to_cpu(er_ctxt->intmod) & EV_CTX_INTMODT_MASK) >>
+ EV_CTX_INTMODT_SHIFT);
+
+- seq_printf(m, " base: 0x%0llx len: 0x%llx", er_ctxt->rbase,
+- er_ctxt->rlen);
++ seq_printf(m, " base: 0x%0llx len: 0x%llx", le64_to_cpu(er_ctxt->rbase),
++ le64_to_cpu(er_ctxt->rlen));
+
+- seq_printf(m, " rp: 0x%llx wp: 0x%llx", er_ctxt->rp,
+- er_ctxt->wp);
++ seq_printf(m, " rp: 0x%llx wp: 0x%llx", le64_to_cpu(er_ctxt->rp),
++ le64_to_cpu(er_ctxt->wp));
+
+ seq_printf(m, " local rp: 0x%pK db: 0x%pad\n", ring->rp,
+ &mhi_event->db_cfg.db_val);
+@@ -106,18 +106,18 @@ static int mhi_debugfs_channels_show(struct seq_file *m, void *d)
+
+ seq_printf(m,
+ "%s(%u) state: 0x%lx brstmode: 0x%lx pollcfg: 0x%lx",
+- mhi_chan->name, mhi_chan->chan, (chan_ctxt->chcfg &
++ mhi_chan->name, mhi_chan->chan, (le32_to_cpu(chan_ctxt->chcfg) &
+ CHAN_CTX_CHSTATE_MASK) >> CHAN_CTX_CHSTATE_SHIFT,
+- (chan_ctxt->chcfg & CHAN_CTX_BRSTMODE_MASK) >>
+- CHAN_CTX_BRSTMODE_SHIFT, (chan_ctxt->chcfg &
++ (le32_to_cpu(chan_ctxt->chcfg) & CHAN_CTX_BRSTMODE_MASK) >>
++ CHAN_CTX_BRSTMODE_SHIFT, (le32_to_cpu(chan_ctxt->chcfg) &
+ CHAN_CTX_POLLCFG_MASK) >> CHAN_CTX_POLLCFG_SHIFT);
+
+- seq_printf(m, " type: 0x%x event ring: %u", chan_ctxt->chtype,
+- chan_ctxt->erindex);
++ seq_printf(m, " type: 0x%x event ring: %u", le32_to_cpu(chan_ctxt->chtype),
++ le32_to_cpu(chan_ctxt->erindex));
+
+ seq_printf(m, " base: 0x%llx len: 0x%llx rp: 0x%llx wp: 0x%llx",
+- chan_ctxt->rbase, chan_ctxt->rlen, chan_ctxt->rp,
+- chan_ctxt->wp);
++ le64_to_cpu(chan_ctxt->rbase), le64_to_cpu(chan_ctxt->rlen),
++ le64_to_cpu(chan_ctxt->rp), le64_to_cpu(chan_ctxt->wp));
+
+ seq_printf(m, " local rp: 0x%pK local wp: 0x%pK db: 0x%pad\n",
+ ring->rp, ring->wp,
+diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c
+index 5aaca6d0f52b2..4183945fc2c41 100644
+--- a/drivers/bus/mhi/core/init.c
++++ b/drivers/bus/mhi/core/init.c
+@@ -290,17 +290,17 @@ int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
+ if (mhi_chan->offload_ch)
+ continue;
+
+- tmp = chan_ctxt->chcfg;
++ tmp = le32_to_cpu(chan_ctxt->chcfg);
+ tmp &= ~CHAN_CTX_CHSTATE_MASK;
+ tmp |= (MHI_CH_STATE_DISABLED << CHAN_CTX_CHSTATE_SHIFT);
+ tmp &= ~CHAN_CTX_BRSTMODE_MASK;
+ tmp |= (mhi_chan->db_cfg.brstmode << CHAN_CTX_BRSTMODE_SHIFT);
+ tmp &= ~CHAN_CTX_POLLCFG_MASK;
+ tmp |= (mhi_chan->db_cfg.pollcfg << CHAN_CTX_POLLCFG_SHIFT);
+- chan_ctxt->chcfg = tmp;
++ chan_ctxt->chcfg = cpu_to_le32(tmp);
+
+- chan_ctxt->chtype = mhi_chan->type;
+- chan_ctxt->erindex = mhi_chan->er_index;
++ chan_ctxt->chtype = cpu_to_le32(mhi_chan->type);
++ chan_ctxt->erindex = cpu_to_le32(mhi_chan->er_index);
+
+ mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
+ mhi_chan->tre_ring.db_addr = (void __iomem *)&chan_ctxt->wp;
+@@ -325,14 +325,14 @@ int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
+ if (mhi_event->offload_ev)
+ continue;
+
+- tmp = er_ctxt->intmod;
++ tmp = le32_to_cpu(er_ctxt->intmod);
+ tmp &= ~EV_CTX_INTMODC_MASK;
+ tmp &= ~EV_CTX_INTMODT_MASK;
+ tmp |= (mhi_event->intmod << EV_CTX_INTMODT_SHIFT);
+- er_ctxt->intmod = tmp;
++ er_ctxt->intmod = cpu_to_le32(tmp);
+
+- er_ctxt->ertype = MHI_ER_TYPE_VALID;
+- er_ctxt->msivec = mhi_event->irq;
++ er_ctxt->ertype = cpu_to_le32(MHI_ER_TYPE_VALID);
++ er_ctxt->msivec = cpu_to_le32(mhi_event->irq);
+ mhi_event->db_cfg.db_mode = true;
+
+ ring->el_size = sizeof(struct mhi_tre);
+@@ -346,9 +346,9 @@ int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
+ * ring is empty
+ */
+ ring->rp = ring->wp = ring->base;
+- er_ctxt->rbase = ring->iommu_base;
++ er_ctxt->rbase = cpu_to_le64(ring->iommu_base);
+ er_ctxt->rp = er_ctxt->wp = er_ctxt->rbase;
+- er_ctxt->rlen = ring->len;
++ er_ctxt->rlen = cpu_to_le64(ring->len);
+ ring->ctxt_wp = &er_ctxt->wp;
+ }
+
+@@ -375,9 +375,9 @@ int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
+ goto error_alloc_cmd;
+
+ ring->rp = ring->wp = ring->base;
+- cmd_ctxt->rbase = ring->iommu_base;
++ cmd_ctxt->rbase = cpu_to_le64(ring->iommu_base);
+ cmd_ctxt->rp = cmd_ctxt->wp = cmd_ctxt->rbase;
+- cmd_ctxt->rlen = ring->len;
++ cmd_ctxt->rlen = cpu_to_le64(ring->len);
+ ring->ctxt_wp = &cmd_ctxt->wp;
+ }
+
+@@ -578,10 +578,10 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ chan_ctxt->rp = 0;
+ chan_ctxt->wp = 0;
+
+- tmp = chan_ctxt->chcfg;
++ tmp = le32_to_cpu(chan_ctxt->chcfg);
+ tmp &= ~CHAN_CTX_CHSTATE_MASK;
+ tmp |= (MHI_CH_STATE_DISABLED << CHAN_CTX_CHSTATE_SHIFT);
+- chan_ctxt->chcfg = tmp;
++ chan_ctxt->chcfg = cpu_to_le32(tmp);
+
+ /* Update to all cores */
+ smp_wmb();
+@@ -615,14 +615,14 @@ int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ return -ENOMEM;
+ }
+
+- tmp = chan_ctxt->chcfg;
++ tmp = le32_to_cpu(chan_ctxt->chcfg);
+ tmp &= ~CHAN_CTX_CHSTATE_MASK;
+ tmp |= (MHI_CH_STATE_ENABLED << CHAN_CTX_CHSTATE_SHIFT);
+- chan_ctxt->chcfg = tmp;
++ chan_ctxt->chcfg = cpu_to_le32(tmp);
+
+- chan_ctxt->rbase = tre_ring->iommu_base;
++ chan_ctxt->rbase = cpu_to_le64(tre_ring->iommu_base);
+ chan_ctxt->rp = chan_ctxt->wp = chan_ctxt->rbase;
+- chan_ctxt->rlen = tre_ring->len;
++ chan_ctxt->rlen = cpu_to_le64(tre_ring->len);
+ tre_ring->ctxt_wp = &chan_ctxt->wp;
+
+ tre_ring->rp = tre_ring->wp = tre_ring->base;
+@@ -788,6 +788,7 @@ static int parse_ch_cfg(struct mhi_controller *mhi_cntrl,
+ mhi_chan->offload_ch = ch_cfg->offload_channel;
+ mhi_chan->db_cfg.reset_req = ch_cfg->doorbell_mode_switch;
+ mhi_chan->pre_alloc = ch_cfg->auto_queue;
++ mhi_chan->wake_capable = ch_cfg->wake_capable;
+
+ /*
+ * If MHI host allocates buffers, then the channel direction
+diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h
+index 3a732afaf73ed..c02c4d48b744c 100644
+--- a/drivers/bus/mhi/core/internal.h
++++ b/drivers/bus/mhi/core/internal.h
+@@ -209,14 +209,14 @@ extern struct bus_type mhi_bus_type;
+ #define EV_CTX_INTMODT_MASK GENMASK(31, 16)
+ #define EV_CTX_INTMODT_SHIFT 16
+ struct mhi_event_ctxt {
+- __u32 intmod;
+- __u32 ertype;
+- __u32 msivec;
+-
+- __u64 rbase __packed __aligned(4);
+- __u64 rlen __packed __aligned(4);
+- __u64 rp __packed __aligned(4);
+- __u64 wp __packed __aligned(4);
++ __le32 intmod;
++ __le32 ertype;
++ __le32 msivec;
++
++ __le64 rbase __packed __aligned(4);
++ __le64 rlen __packed __aligned(4);
++ __le64 rp __packed __aligned(4);
++ __le64 wp __packed __aligned(4);
+ };
+
+ #define CHAN_CTX_CHSTATE_MASK GENMASK(7, 0)
+@@ -227,25 +227,25 @@ struct mhi_event_ctxt {
+ #define CHAN_CTX_POLLCFG_SHIFT 10
+ #define CHAN_CTX_RESERVED_MASK GENMASK(31, 16)
+ struct mhi_chan_ctxt {
+- __u32 chcfg;
+- __u32 chtype;
+- __u32 erindex;
+-
+- __u64 rbase __packed __aligned(4);
+- __u64 rlen __packed __aligned(4);
+- __u64 rp __packed __aligned(4);
+- __u64 wp __packed __aligned(4);
++ __le32 chcfg;
++ __le32 chtype;
++ __le32 erindex;
++
++ __le64 rbase __packed __aligned(4);
++ __le64 rlen __packed __aligned(4);
++ __le64 rp __packed __aligned(4);
++ __le64 wp __packed __aligned(4);
+ };
+
+ struct mhi_cmd_ctxt {
+- __u32 reserved0;
+- __u32 reserved1;
+- __u32 reserved2;
+-
+- __u64 rbase __packed __aligned(4);
+- __u64 rlen __packed __aligned(4);
+- __u64 rp __packed __aligned(4);
+- __u64 wp __packed __aligned(4);
++ __le32 reserved0;
++ __le32 reserved1;
++ __le32 reserved2;
++
++ __le64 rbase __packed __aligned(4);
++ __le64 rlen __packed __aligned(4);
++ __le64 rp __packed __aligned(4);
++ __le64 wp __packed __aligned(4);
+ };
+
+ struct mhi_ctxt {
+@@ -258,8 +258,8 @@ struct mhi_ctxt {
+ };
+
+ struct mhi_tre {
+- u64 ptr;
+- u32 dword[2];
++ __le64 ptr;
++ __le32 dword[2];
+ };
+
+ struct bhi_vec_entry {
+@@ -277,57 +277,58 @@ enum mhi_cmd_type {
+ /* No operation command */
+ #define MHI_TRE_CMD_NOOP_PTR (0)
+ #define MHI_TRE_CMD_NOOP_DWORD0 (0)
+-#define MHI_TRE_CMD_NOOP_DWORD1 (MHI_CMD_NOP << 16)
++#define MHI_TRE_CMD_NOOP_DWORD1 (cpu_to_le32(MHI_CMD_NOP << 16))
+
+ /* Channel reset command */
+ #define MHI_TRE_CMD_RESET_PTR (0)
+ #define MHI_TRE_CMD_RESET_DWORD0 (0)
+-#define MHI_TRE_CMD_RESET_DWORD1(chid) ((chid << 24) | \
+- (MHI_CMD_RESET_CHAN << 16))
++#define MHI_TRE_CMD_RESET_DWORD1(chid) (cpu_to_le32((chid << 24) | \
++ (MHI_CMD_RESET_CHAN << 16)))
+
+ /* Channel stop command */
+ #define MHI_TRE_CMD_STOP_PTR (0)
+ #define MHI_TRE_CMD_STOP_DWORD0 (0)
+-#define MHI_TRE_CMD_STOP_DWORD1(chid) ((chid << 24) | \
+- (MHI_CMD_STOP_CHAN << 16))
++#define MHI_TRE_CMD_STOP_DWORD1(chid) (cpu_to_le32((chid << 24) | \
++ (MHI_CMD_STOP_CHAN << 16)))
+
+ /* Channel start command */
+ #define MHI_TRE_CMD_START_PTR (0)
+ #define MHI_TRE_CMD_START_DWORD0 (0)
+-#define MHI_TRE_CMD_START_DWORD1(chid) ((chid << 24) | \
+- (MHI_CMD_START_CHAN << 16))
++#define MHI_TRE_CMD_START_DWORD1(chid) (cpu_to_le32((chid << 24) | \
++ (MHI_CMD_START_CHAN << 16)))
+
+-#define MHI_TRE_GET_CMD_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF)
+-#define MHI_TRE_GET_CMD_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF)
++#define MHI_TRE_GET_DWORD(tre, word) (le32_to_cpu((tre)->dword[(word)]))
++#define MHI_TRE_GET_CMD_CHID(tre) ((MHI_TRE_GET_DWORD(tre, 1) >> 24) & 0xFF)
++#define MHI_TRE_GET_CMD_TYPE(tre) ((MHI_TRE_GET_DWORD(tre, 1) >> 16) & 0xFF)
+
+ /* Event descriptor macros */
+-#define MHI_TRE_EV_PTR(ptr) (ptr)
+-#define MHI_TRE_EV_DWORD0(code, len) ((code << 24) | len)
+-#define MHI_TRE_EV_DWORD1(chid, type) ((chid << 24) | (type << 16))
+-#define MHI_TRE_GET_EV_PTR(tre) ((tre)->ptr)
+-#define MHI_TRE_GET_EV_CODE(tre) (((tre)->dword[0] >> 24) & 0xFF)
+-#define MHI_TRE_GET_EV_LEN(tre) ((tre)->dword[0] & 0xFFFF)
+-#define MHI_TRE_GET_EV_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF)
+-#define MHI_TRE_GET_EV_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF)
+-#define MHI_TRE_GET_EV_STATE(tre) (((tre)->dword[0] >> 24) & 0xFF)
+-#define MHI_TRE_GET_EV_EXECENV(tre) (((tre)->dword[0] >> 24) & 0xFF)
+-#define MHI_TRE_GET_EV_SEQ(tre) ((tre)->dword[0])
+-#define MHI_TRE_GET_EV_TIME(tre) ((tre)->ptr)
+-#define MHI_TRE_GET_EV_COOKIE(tre) lower_32_bits((tre)->ptr)
+-#define MHI_TRE_GET_EV_VEID(tre) (((tre)->dword[0] >> 16) & 0xFF)
+-#define MHI_TRE_GET_EV_LINKSPEED(tre) (((tre)->dword[1] >> 24) & 0xFF)
+-#define MHI_TRE_GET_EV_LINKWIDTH(tre) ((tre)->dword[0] & 0xFF)
++#define MHI_TRE_EV_PTR(ptr) (cpu_to_le64(ptr))
++#define MHI_TRE_EV_DWORD0(code, len) (cpu_to_le32((code << 24) | len))
++#define MHI_TRE_EV_DWORD1(chid, type) (cpu_to_le32((chid << 24) | (type << 16)))
++#define MHI_TRE_GET_EV_PTR(tre) (le64_to_cpu((tre)->ptr))
++#define MHI_TRE_GET_EV_CODE(tre) ((MHI_TRE_GET_DWORD(tre, 0) >> 24) & 0xFF)
++#define MHI_TRE_GET_EV_LEN(tre) (MHI_TRE_GET_DWORD(tre, 0) & 0xFFFF)
++#define MHI_TRE_GET_EV_CHID(tre) ((MHI_TRE_GET_DWORD(tre, 1) >> 24) & 0xFF)
++#define MHI_TRE_GET_EV_TYPE(tre) ((MHI_TRE_GET_DWORD(tre, 1) >> 16) & 0xFF)
++#define MHI_TRE_GET_EV_STATE(tre) ((MHI_TRE_GET_DWORD(tre, 0) >> 24) & 0xFF)
++#define MHI_TRE_GET_EV_EXECENV(tre) ((MHI_TRE_GET_DWORD(tre, 0) >> 24) & 0xFF)
++#define MHI_TRE_GET_EV_SEQ(tre) MHI_TRE_GET_DWORD(tre, 0)
++#define MHI_TRE_GET_EV_TIME(tre) (MHI_TRE_GET_EV_PTR(tre))
++#define MHI_TRE_GET_EV_COOKIE(tre) lower_32_bits(MHI_TRE_GET_EV_PTR(tre))
++#define MHI_TRE_GET_EV_VEID(tre) ((MHI_TRE_GET_DWORD(tre, 0) >> 16) & 0xFF)
++#define MHI_TRE_GET_EV_LINKSPEED(tre) ((MHI_TRE_GET_DWORD(tre, 1) >> 24) & 0xFF)
++#define MHI_TRE_GET_EV_LINKWIDTH(tre) (MHI_TRE_GET_DWORD(tre, 0) & 0xFF)
+
+ /* Transfer descriptor macros */
+-#define MHI_TRE_DATA_PTR(ptr) (ptr)
+-#define MHI_TRE_DATA_DWORD0(len) (len & MHI_MAX_MTU)
+-#define MHI_TRE_DATA_DWORD1(bei, ieot, ieob, chain) ((2 << 16) | (bei << 10) \
+- | (ieot << 9) | (ieob << 8) | chain)
++#define MHI_TRE_DATA_PTR(ptr) (cpu_to_le64(ptr))
++#define MHI_TRE_DATA_DWORD0(len) (cpu_to_le32(len & MHI_MAX_MTU))
++#define MHI_TRE_DATA_DWORD1(bei, ieot, ieob, chain) (cpu_to_le32((2 << 16) | (bei << 10) \
++ | (ieot << 9) | (ieob << 8) | chain))
+
+ /* RSC transfer descriptor macros */
+-#define MHI_RSCTRE_DATA_PTR(ptr, len) (((u64)len << 48) | ptr)
+-#define MHI_RSCTRE_DATA_DWORD0(cookie) (cookie)
+-#define MHI_RSCTRE_DATA_DWORD1 (MHI_PKT_TYPE_COALESCING << 16)
++#define MHI_RSCTRE_DATA_PTR(ptr, len) (cpu_to_le64(((u64)len << 48) | ptr))
++#define MHI_RSCTRE_DATA_DWORD0(cookie) (cpu_to_le32(cookie))
++#define MHI_RSCTRE_DATA_DWORD1 (cpu_to_le32(MHI_PKT_TYPE_COALESCING << 16))
+
+ enum mhi_pkt_type {
+ MHI_PKT_TYPE_INVALID = 0x0,
+@@ -499,7 +500,7 @@ struct state_transition {
+ struct mhi_ring {
+ dma_addr_t dma_handle;
+ dma_addr_t iommu_base;
+- u64 *ctxt_wp; /* point to ctxt wp */
++ __le64 *ctxt_wp; /* point to ctxt wp */
+ void *pre_aligned;
+ void *base;
+ void *rp;
+diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c
+index b15c5bc37dd4f..9a94b8d66f575 100644
+--- a/drivers/bus/mhi/core/main.c
++++ b/drivers/bus/mhi/core/main.c
+@@ -114,7 +114,7 @@ void mhi_ring_er_db(struct mhi_event *mhi_event)
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg,
+- ring->db_addr, *ring->ctxt_wp);
++ ring->db_addr, le64_to_cpu(*ring->ctxt_wp));
+ }
+
+ void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd)
+@@ -123,7 +123,7 @@ void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd)
+ struct mhi_ring *ring = &mhi_cmd->ring;
+
+ db = ring->iommu_base + (ring->wp - ring->base);
+- *ring->ctxt_wp = db;
++ *ring->ctxt_wp = cpu_to_le64(db);
+ mhi_write_db(mhi_cntrl, ring->db_addr, db);
+ }
+
+@@ -140,7 +140,7 @@ void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl,
+ * before letting h/w know there is new element to fetch.
+ */
+ dma_wmb();
+- *ring->ctxt_wp = db;
++ *ring->ctxt_wp = cpu_to_le64(db);
+
+ mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg,
+ ring->db_addr, db);
+@@ -432,7 +432,7 @@ irqreturn_t mhi_irq_handler(int irq_number, void *dev)
+ struct mhi_event_ctxt *er_ctxt =
+ &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
+ struct mhi_ring *ev_ring = &mhi_event->ring;
+- dma_addr_t ptr = er_ctxt->rp;
++ dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
+ void *dev_rp;
+
+ if (!is_valid_ring_ptr(ev_ring, ptr)) {
+@@ -537,14 +537,14 @@ static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl,
+
+ /* Update the WP */
+ ring->wp += ring->el_size;
+- ctxt_wp = *ring->ctxt_wp + ring->el_size;
++ ctxt_wp = le64_to_cpu(*ring->ctxt_wp) + ring->el_size;
+
+ if (ring->wp >= (ring->base + ring->len)) {
+ ring->wp = ring->base;
+ ctxt_wp = ring->iommu_base;
+ }
+
+- *ring->ctxt_wp = ctxt_wp;
++ *ring->ctxt_wp = cpu_to_le64(ctxt_wp);
+
+ /* Update the RP */
+ ring->rp += ring->el_size;
+@@ -801,7 +801,7 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+ u32 chan;
+ int count = 0;
+- dma_addr_t ptr = er_ctxt->rp;
++ dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
+
+ /*
+ * This is a quick check to avoid unnecessary event processing
+@@ -940,7 +940,7 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
+ mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
+ local_rp = ev_ring->rp;
+
+- ptr = er_ctxt->rp;
++ ptr = le64_to_cpu(er_ctxt->rp);
+ if (!is_valid_ring_ptr(ev_ring, ptr)) {
+ dev_err(&mhi_cntrl->mhi_dev->dev,
+ "Event ring rp points outside of the event ring\n");
+@@ -970,7 +970,7 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
+ int count = 0;
+ u32 chan;
+ struct mhi_chan *mhi_chan;
+- dma_addr_t ptr = er_ctxt->rp;
++ dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
+
+ if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state)))
+ return -EIO;
+@@ -1011,7 +1011,7 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
+ mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
+ local_rp = ev_ring->rp;
+
+- ptr = er_ctxt->rp;
++ ptr = le64_to_cpu(er_ctxt->rp);
+ if (!is_valid_ring_ptr(ev_ring, ptr)) {
+ dev_err(&mhi_cntrl->mhi_dev->dev,
+ "Event ring rp points outside of the event ring\n");
+@@ -1529,7 +1529,7 @@ static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl,
+ /* mark all stale events related to channel as STALE event */
+ spin_lock_irqsave(&mhi_event->lock, flags);
+
+- ptr = er_ctxt->rp;
++ ptr = le64_to_cpu(er_ctxt->rp);
+ if (!is_valid_ring_ptr(ev_ring, ptr)) {
+ dev_err(&mhi_cntrl->mhi_dev->dev,
+ "Event ring rp points outside of the event ring\n");
+diff --git a/drivers/bus/mhi/core/pm.c b/drivers/bus/mhi/core/pm.c
+index fb99e3727155b..1020268a075a5 100644
+--- a/drivers/bus/mhi/core/pm.c
++++ b/drivers/bus/mhi/core/pm.c
+@@ -218,7 +218,7 @@ int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl)
+ continue;
+
+ ring->wp = ring->base + ring->len - ring->el_size;
+- *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
++ *ring->ctxt_wp = cpu_to_le64(ring->iommu_base + ring->len - ring->el_size);
+ /* Update all cores */
+ smp_wmb();
+
+@@ -420,7 +420,7 @@ static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl)
+ continue;
+
+ ring->wp = ring->base + ring->len - ring->el_size;
+- *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
++ *ring->ctxt_wp = cpu_to_le64(ring->iommu_base + ring->len - ring->el_size);
+ /* Update to all cores */
+ smp_wmb();
+
+@@ -881,7 +881,7 @@ int mhi_pm_suspend(struct mhi_controller *mhi_cntrl)
+ }
+ EXPORT_SYMBOL_GPL(mhi_pm_suspend);
+
+-int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
++static int __mhi_pm_resume(struct mhi_controller *mhi_cntrl, bool force)
+ {
+ struct mhi_chan *itr, *tmp;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+@@ -898,8 +898,12 @@ int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+- if (mhi_get_mhi_state(mhi_cntrl) != MHI_STATE_M3)
+- return -EINVAL;
++ if (mhi_get_mhi_state(mhi_cntrl) != MHI_STATE_M3) {
++ dev_warn(dev, "Resuming from non M3 state (%s)\n",
++ TO_MHI_STATE_STR(mhi_get_mhi_state(mhi_cntrl)));
++ if (!force)
++ return -EINVAL;
++ }
+
+ /* Notify clients about exiting LPM */
+ list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+@@ -940,8 +944,19 @@ int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
+
+ return 0;
+ }
++
++int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
++{
++ return __mhi_pm_resume(mhi_cntrl, false);
++}
+ EXPORT_SYMBOL_GPL(mhi_pm_resume);
+
++int mhi_pm_resume_force(struct mhi_controller *mhi_cntrl)
++{
++ return __mhi_pm_resume(mhi_cntrl, true);
++}
++EXPORT_SYMBOL_GPL(mhi_pm_resume_force);
++
+ int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl)
+ {
+ int ret;
+@@ -1038,7 +1053,7 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+ enum mhi_ee_type current_ee;
+ enum dev_st_transition next_state;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+- u32 val;
++ u32 interval_us = 25000; /* poll register field every 25 milliseconds */
+ int ret;
+
+ dev_info(dev, "Requested to power ON\n");
+@@ -1055,10 +1070,6 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+ mutex_lock(&mhi_cntrl->pm_mutex);
+ mhi_cntrl->pm_state = MHI_PM_DISABLE;
+
+- ret = mhi_init_irq_setup(mhi_cntrl);
+- if (ret)
+- goto error_setup_irq;
+-
+ /* Setup BHI INTVEC */
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+@@ -1072,7 +1083,7 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+ dev_err(dev, "%s is not a valid EE for power on\n",
+ TO_MHI_EXEC_STR(current_ee));
+ ret = -EIO;
+- goto error_async_power_up;
++ goto error_exit;
+ }
+
+ state = mhi_get_mhi_state(mhi_cntrl);
+@@ -1081,20 +1092,12 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+
+ if (state == MHI_STATE_SYS_ERR) {
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
+- ret = wait_event_timeout(mhi_cntrl->state_event,
+- MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
+- mhi_read_reg_field(mhi_cntrl,
+- mhi_cntrl->regs,
+- MHICTRL,
+- MHICTRL_RESET_MASK,
+- MHICTRL_RESET_SHIFT,
+- &val) ||
+- !val,
+- msecs_to_jiffies(mhi_cntrl->timeout_ms));
+- if (!ret) {
+- ret = -EIO;
++ ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
++ MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 0,
++ interval_us);
++ if (ret) {
+ dev_info(dev, "Failed to reset MHI due to syserr state\n");
+- goto error_async_power_up;
++ goto error_exit;
+ }
+
+ /*
+@@ -1104,6 +1107,10 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+ }
+
++ ret = mhi_init_irq_setup(mhi_cntrl);
++ if (ret)
++ goto error_exit;
++
+ /* Transition to next state */
+ next_state = MHI_IN_PBL(current_ee) ?
+ DEV_ST_TRANSITION_PBL : DEV_ST_TRANSITION_READY;
+@@ -1116,10 +1123,7 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+
+ return 0;
+
+-error_async_power_up:
+- mhi_deinit_free_irq(mhi_cntrl);
+-
+-error_setup_irq:
++error_exit:
+ mhi_cntrl->pm_state = MHI_PM_DISABLE;
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c
+index 59a4896a80309..0982642a7907e 100644
+--- a/drivers/bus/mhi/pci_generic.c
++++ b/drivers/bus/mhi/pci_generic.c
+@@ -20,7 +20,7 @@
+
+ #define MHI_PCI_DEFAULT_BAR_NUM 0
+
+-#define MHI_POST_RESET_DELAY_MS 500
++#define MHI_POST_RESET_DELAY_MS 2000
+
+ #define HEALTH_CHECK_PERIOD (HZ * 2)
+
+@@ -327,6 +327,7 @@ static const struct mhi_pci_dev_info mhi_quectel_em1xx_info = {
+ .config = &modem_quectel_em1xx_config,
+ .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+ .dma_data_width = 32,
++ .mru_default = 32768,
+ .sideband_wake = true,
+ };
+
+@@ -366,6 +367,7 @@ static const struct mhi_pci_dev_info mhi_foxconn_sdx55_info = {
+ .config = &modem_foxconn_sdx55_config,
+ .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+ .dma_data_width = 32,
++ .mru_default = 32768,
+ .sideband_wake = false,
+ };
+
+@@ -401,6 +403,7 @@ static const struct mhi_pci_dev_info mhi_mv31_info = {
+ .config = &modem_mv31_config,
+ .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+ .dma_data_width = 32,
++ .mru_default = 32768,
+ };
+
+ static const struct pci_device_id mhi_pci_id_table[] = {
+@@ -1017,8 +1020,9 @@ static int __maybe_unused mhi_pci_freeze(struct device *dev)
+ * the intermediate restore kernel reinitializes MHI device with new
+ * context.
+ */
++ flush_work(&mhi_pdev->recovery_work);
+ if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
+- mhi_power_down(mhi_cntrl, false);
++ mhi_power_down(mhi_cntrl, true);
+ mhi_unprepare_after_power_down(mhi_cntrl);
+ }
+
+@@ -1042,6 +1046,7 @@ static const struct dev_pm_ops mhi_pci_pm_ops = {
+ .resume = mhi_pci_resume,
+ .freeze = mhi_pci_freeze,
+ .thaw = mhi_pci_restore,
++ .poweroff = mhi_pci_freeze,
+ .restore = mhi_pci_restore,
+ #endif
+ };
+diff --git a/drivers/bus/mips_cdmm.c b/drivers/bus/mips_cdmm.c
+index 626dedd110cbc..fca0d0669aa97 100644
+--- a/drivers/bus/mips_cdmm.c
++++ b/drivers/bus/mips_cdmm.c
+@@ -351,6 +351,7 @@ phys_addr_t __weak mips_cdmm_phys_base(void)
+ np = of_find_compatible_node(NULL, NULL, "mti,mips-cdmm");
+ if (np) {
+ err = of_address_to_resource(np, 0, &res);
++ of_node_put(np);
+ if (!err)
+ return res.start;
+ }
+diff --git a/drivers/bus/sunxi-rsb.c b/drivers/bus/sunxi-rsb.c
+index 6f225dddc74f4..60b082fe2ed02 100644
+--- a/drivers/bus/sunxi-rsb.c
++++ b/drivers/bus/sunxi-rsb.c
+@@ -227,6 +227,8 @@ static struct sunxi_rsb_device *sunxi_rsb_device_create(struct sunxi_rsb *rsb,
+
+ dev_dbg(&rdev->dev, "device %s registered\n", dev_name(&rdev->dev));
+
++ return rdev;
++
+ err_device_add:
+ put_device(&rdev->dev);
+
+@@ -687,11 +689,11 @@ err_clk_disable:
+
+ static void sunxi_rsb_hw_exit(struct sunxi_rsb *rsb)
+ {
+- /* Keep the clock and PM reference counts consistent. */
+- if (pm_runtime_status_suspended(rsb->dev))
+- pm_runtime_resume(rsb->dev);
+ reset_control_assert(rsb->rstc);
+- clk_disable_unprepare(rsb->clk);
++
++ /* Keep the clock and PM reference counts consistent. */
++ if (!pm_runtime_status_suspended(rsb->dev))
++ clk_disable_unprepare(rsb->clk);
+ }
+
+ static int __maybe_unused sunxi_rsb_runtime_suspend(struct device *dev)
+diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
+index 6a8b7fb5be58d..00d46f3ae22fb 100644
+--- a/drivers/bus/ti-sysc.c
++++ b/drivers/bus/ti-sysc.c
+@@ -6,6 +6,7 @@
+ #include <linux/io.h>
+ #include <linux/clk.h>
+ #include <linux/clkdev.h>
++#include <linux/cpu_pm.h>
+ #include <linux/delay.h>
+ #include <linux/list.h>
+ #include <linux/module.h>
+@@ -17,6 +18,7 @@
+ #include <linux/of_platform.h>
+ #include <linux/slab.h>
+ #include <linux/sys_soc.h>
++#include <linux/timekeeping.h>
+ #include <linux/iopoll.h>
+
+ #include <linux/platform_data/ti-sysc.h>
+@@ -51,11 +53,18 @@ struct sysc_address {
+ struct list_head node;
+ };
+
++struct sysc_module {
++ struct sysc *ddata;
++ struct list_head node;
++};
++
+ struct sysc_soc_info {
+ unsigned long general_purpose:1;
+ enum sysc_soc soc;
+- struct mutex list_lock; /* disabled modules list lock */
++ struct mutex list_lock; /* disabled and restored modules list lock */
+ struct list_head disabled_modules;
++ struct list_head restored_modules;
++ struct notifier_block nb;
+ };
+
+ enum sysc_clocks {
+@@ -223,37 +232,77 @@ static u32 sysc_read_sysstatus(struct sysc *ddata)
+ return sysc_read(ddata, offset);
+ }
+
+-/* Poll on reset status */
+-static int sysc_wait_softreset(struct sysc *ddata)
++static int sysc_poll_reset_sysstatus(struct sysc *ddata)
+ {
+- u32 sysc_mask, syss_done, rstval;
+- int syss_offset, error = 0;
+-
+- if (ddata->cap->regbits->srst_shift < 0)
+- return 0;
+-
+- syss_offset = ddata->offsets[SYSC_SYSSTATUS];
+- sysc_mask = BIT(ddata->cap->regbits->srst_shift);
++ int error, retries;
++ u32 syss_done, rstval;
+
+ if (ddata->cfg.quirks & SYSS_QUIRK_RESETDONE_INVERTED)
+ syss_done = 0;
+ else
+ syss_done = ddata->cfg.syss_mask;
+
+- if (syss_offset >= 0) {
++ if (likely(!timekeeping_suspended)) {
+ error = readx_poll_timeout_atomic(sysc_read_sysstatus, ddata,
+ rstval, (rstval & ddata->cfg.syss_mask) ==
+ syss_done, 100, MAX_MODULE_SOFTRESET_WAIT);
++ } else {
++ retries = MAX_MODULE_SOFTRESET_WAIT;
++ while (retries--) {
++ rstval = sysc_read_sysstatus(ddata);
++ if ((rstval & ddata->cfg.syss_mask) == syss_done)
++ return 0;
++ udelay(2); /* Account for udelay flakeyness */
++ }
++ error = -ETIMEDOUT;
++ }
+
+- } else if (ddata->cfg.quirks & SYSC_QUIRK_RESET_STATUS) {
++ return error;
++}
++
++static int sysc_poll_reset_sysconfig(struct sysc *ddata)
++{
++ int error, retries;
++ u32 sysc_mask, rstval;
++
++ sysc_mask = BIT(ddata->cap->regbits->srst_shift);
++
++ if (likely(!timekeeping_suspended)) {
+ error = readx_poll_timeout_atomic(sysc_read_sysconfig, ddata,
+ rstval, !(rstval & sysc_mask),
+ 100, MAX_MODULE_SOFTRESET_WAIT);
++ } else {
++ retries = MAX_MODULE_SOFTRESET_WAIT;
++ while (retries--) {
++ rstval = sysc_read_sysconfig(ddata);
++ if (!(rstval & sysc_mask))
++ return 0;
++ udelay(2); /* Account for udelay flakeyness */
++ }
++ error = -ETIMEDOUT;
+ }
+
+ return error;
+ }
+
++/* Poll on reset status */
++static int sysc_wait_softreset(struct sysc *ddata)
++{
++ int syss_offset, error = 0;
++
++ if (ddata->cap->regbits->srst_shift < 0)
++ return 0;
++
++ syss_offset = ddata->offsets[SYSC_SYSSTATUS];
++
++ if (syss_offset >= 0)
++ error = sysc_poll_reset_sysstatus(ddata);
++ else if (ddata->cfg.quirks & SYSC_QUIRK_RESET_STATUS)
++ error = sysc_poll_reset_sysconfig(ddata);
++
++ return error;
++}
++
+ static int sysc_add_named_clock_from_child(struct sysc *ddata,
+ const char *name,
+ const char *optfck_name)
+@@ -1518,7 +1567,7 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
+ 0xffffffff, SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_SWSUP_MSTANDBY),
+ SYSC_QUIRK("usb_otg_hs", 0, 0, 0x10, -ENODEV, 0x4ea2080d, 0xffffffff,
+ SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_SWSUP_MSTANDBY |
+- SYSC_QUIRK_REINIT_ON_RESUME),
++ SYSC_QUIRK_REINIT_ON_CTX_LOST),
+ SYSC_QUIRK("wdt", 0, 0, 0x10, 0x14, 0x502a0500, 0xfffff0f0,
+ SYSC_MODULE_QUIRK_WDT),
+ /* PRUSS on am3, am4 and am5 */
+@@ -2401,6 +2450,78 @@ static struct dev_pm_domain sysc_child_pm_domain = {
+ }
+ };
+
++/* Caller needs to take list_lock if ever used outside of cpu_pm */
++static void sysc_reinit_modules(struct sysc_soc_info *soc)
++{
++ struct sysc_module *module;
++ struct list_head *pos;
++ struct sysc *ddata;
++
++ list_for_each(pos, &sysc_soc->restored_modules) {
++ module = list_entry(pos, struct sysc_module, node);
++ ddata = module->ddata;
++ sysc_reinit_module(ddata, ddata->enabled);
++ }
++}
++
++/**
++ * sysc_context_notifier - optionally reset and restore module after idle
++ * @nb: notifier block
++ * @cmd: unused
++ * @v: unused
++ *
++ * Some interconnect target modules need to be restored, or reset and restored
++ * on CPU_PM CPU_PM_CLUSTER_EXIT notifier. This is needed at least for am335x
++ * OTG and GPMC target modules even if the modules are unused.
++ */
++static int sysc_context_notifier(struct notifier_block *nb, unsigned long cmd,
++ void *v)
++{
++ struct sysc_soc_info *soc;
++
++ soc = container_of(nb, struct sysc_soc_info, nb);
++
++ switch (cmd) {
++ case CPU_CLUSTER_PM_ENTER:
++ break;
++ case CPU_CLUSTER_PM_ENTER_FAILED: /* No need to restore context */
++ break;
++ case CPU_CLUSTER_PM_EXIT:
++ sysc_reinit_modules(soc);
++ break;
++ }
++
++ return NOTIFY_OK;
++}
++
++/**
++ * sysc_add_restored - optionally add reset and restore quirk hanlling
++ * @ddata: device data
++ */
++static void sysc_add_restored(struct sysc *ddata)
++{
++ struct sysc_module *restored_module;
++
++ restored_module = kzalloc(sizeof(*restored_module), GFP_KERNEL);
++ if (!restored_module)
++ return;
++
++ restored_module->ddata = ddata;
++
++ mutex_lock(&sysc_soc->list_lock);
++
++ list_add(&restored_module->node, &sysc_soc->restored_modules);
++
++ if (sysc_soc->nb.notifier_call)
++ goto out_unlock;
++
++ sysc_soc->nb.notifier_call = sysc_context_notifier;
++ cpu_pm_register_notifier(&sysc_soc->nb);
++
++out_unlock:
++ mutex_unlock(&sysc_soc->list_lock);
++}
++
+ /**
+ * sysc_legacy_idle_quirk - handle children in omap_device compatible way
+ * @ddata: device driver data
+@@ -2900,12 +3021,14 @@ static int sysc_add_disabled(unsigned long base)
+ }
+
+ /*
+- * One time init to detect the booted SoC and disable unavailable features.
++ * One time init to detect the booted SoC, disable unavailable features
++ * and initialize list for optional cpu_pm notifier.
++ *
+ * Note that we initialize static data shared across all ti-sysc instances
+ * so ddata is only used for SoC type. This can be called from module_init
+ * once we no longer need to rely on platform data.
+ */
+-static int sysc_init_soc(struct sysc *ddata)
++static int sysc_init_static_data(struct sysc *ddata)
+ {
+ const struct soc_device_attribute *match;
+ struct ti_sysc_platform_data *pdata;
+@@ -2921,6 +3044,7 @@ static int sysc_init_soc(struct sysc *ddata)
+
+ mutex_init(&sysc_soc->list_lock);
+ INIT_LIST_HEAD(&sysc_soc->disabled_modules);
++ INIT_LIST_HEAD(&sysc_soc->restored_modules);
+ sysc_soc->general_purpose = true;
+
+ pdata = dev_get_platdata(ddata->dev);
+@@ -2985,15 +3109,24 @@ static int sysc_init_soc(struct sysc *ddata)
+ return 0;
+ }
+
+-static void sysc_cleanup_soc(void)
++static void sysc_cleanup_static_data(void)
+ {
++ struct sysc_module *restored_module;
+ struct sysc_address *disabled_module;
+ struct list_head *pos, *tmp;
+
+ if (!sysc_soc)
+ return;
+
++ if (sysc_soc->nb.notifier_call)
++ cpu_pm_unregister_notifier(&sysc_soc->nb);
++
+ mutex_lock(&sysc_soc->list_lock);
++ list_for_each_safe(pos, tmp, &sysc_soc->restored_modules) {
++ restored_module = list_entry(pos, struct sysc_module, node);
++ list_del(pos);
++ kfree(restored_module);
++ }
+ list_for_each_safe(pos, tmp, &sysc_soc->disabled_modules) {
+ disabled_module = list_entry(pos, struct sysc_address, node);
+ list_del(pos);
+@@ -3029,13 +3162,27 @@ static int sysc_check_disabled_devices(struct sysc *ddata)
+ */
+ static int sysc_check_active_timer(struct sysc *ddata)
+ {
++ int error;
++
+ if (ddata->cap->type != TI_SYSC_OMAP2_TIMER &&
+ ddata->cap->type != TI_SYSC_OMAP4_TIMER)
+ return 0;
+
++ /*
++ * Quirk for omap3 beagleboard revision A to B4 to use gpt12.
++ * Revision C and later are fixed with commit 23885389dbbb ("ARM:
++ * dts: Fix timer regression for beagleboard revision c"). This all
++ * can be dropped if we stop supporting old beagleboard revisions
++ * A to B4 at some point.
++ */
++ if (sysc_soc->soc == SOC_3430)
++ error = -ENXIO;
++ else
++ error = -EBUSY;
++
+ if ((ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT) &&
+ (ddata->cfg.quirks & SYSC_QUIRK_NO_IDLE))
+- return -ENXIO;
++ return error;
+
+ return 0;
+ }
+@@ -3061,7 +3208,7 @@ static int sysc_probe(struct platform_device *pdev)
+ ddata->dev = &pdev->dev;
+ platform_set_drvdata(pdev, ddata);
+
+- error = sysc_init_soc(ddata);
++ error = sysc_init_static_data(ddata);
+ if (error)
+ return error;
+
+@@ -3159,6 +3306,9 @@ static int sysc_probe(struct platform_device *pdev)
+ pm_runtime_put(&pdev->dev);
+ }
+
++ if (ddata->cfg.quirks & SYSC_QUIRK_REINIT_ON_CTX_LOST)
++ sysc_add_restored(ddata);
++
+ return 0;
+
+ err:
+@@ -3240,7 +3390,7 @@ static void __exit sysc_exit(void)
+ {
+ bus_unregister_notifier(&platform_bus_type, &sysc_nb);
+ platform_driver_unregister(&sysc_driver);
+- sysc_cleanup_soc();
++ sysc_cleanup_static_data();
+ }
+ module_exit(sysc_exit);
+
+diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
+index 740811893c570..55f48375e3fe5 100644
+--- a/drivers/char/Kconfig
++++ b/drivers/char/Kconfig
+@@ -449,6 +449,7 @@ config RANDOM_TRUST_BOOTLOADER
+ device randomness. Say Y here to assume the entropy provided by the
+ booloader is trustworthy so it will be added to the kernel's entropy
+ pool. Otherwise, say N here so it will be regarded as device input that
+- only mixes the entropy pool.
++ only mixes the entropy pool. This can also be configured at boot with
++ "random.trust_bootloader=on/off".
+
+ endmenu
+diff --git a/drivers/char/agp/parisc-agp.c b/drivers/char/agp/parisc-agp.c
+index ed3c4c42fc23b..d68d05d5d3838 100644
+--- a/drivers/char/agp/parisc-agp.c
++++ b/drivers/char/agp/parisc-agp.c
+@@ -281,7 +281,7 @@ agp_ioc_init(void __iomem *ioc_regs)
+ return 0;
+ }
+
+-static int
++static int __init
+ lba_find_capability(int cap)
+ {
+ struct _parisc_agp_info *info = &parisc_agp_info;
+@@ -366,7 +366,7 @@ fail:
+ return error;
+ }
+
+-static int
++static int __init
+ find_quicksilver(struct device *dev, void *data)
+ {
+ struct parisc_device **lba = data;
+@@ -378,7 +378,7 @@ find_quicksilver(struct device *dev, void *data)
+ return 0;
+ }
+
+-static int
++static int __init
+ parisc_agp_init(void)
+ {
+ extern struct sba_device *sba_list;
+diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
+index 239eca4d68055..650c7d9180802 100644
+--- a/drivers/char/hw_random/Kconfig
++++ b/drivers/char/hw_random/Kconfig
+@@ -414,7 +414,7 @@ config HW_RANDOM_MESON
+
+ config HW_RANDOM_CAVIUM
+ tristate "Cavium ThunderX Random Number Generator support"
+- depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
++ depends on HW_RANDOM && PCI && ARCH_THUNDER
+ default HW_RANDOM
+ help
+ This driver provides kernel-side support for the Random Number
+diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c
+index ecb71c4317a50..8cf0ef501341e 100644
+--- a/drivers/char/hw_random/atmel-rng.c
++++ b/drivers/char/hw_random/atmel-rng.c
+@@ -114,6 +114,7 @@ static int atmel_trng_probe(struct platform_device *pdev)
+
+ err_register:
+ clk_disable_unprepare(trng->clk);
++ atmel_trng_disable(trng);
+ return ret;
+ }
+
+diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c
+index 3de4a6a443ef9..6f66919652bf5 100644
+--- a/drivers/char/hw_random/cavium-rng-vf.c
++++ b/drivers/char/hw_random/cavium-rng-vf.c
+@@ -1,10 +1,7 @@
++// SPDX-License-Identifier: GPL-2.0
+ /*
+- * Hardware Random Number Generator support for Cavium, Inc.
+- * Thunder processor family.
+- *
+- * This file is subject to the terms and conditions of the GNU General Public
+- * License. See the file "COPYING" in the main directory of this archive
+- * for more details.
++ * Hardware Random Number Generator support.
++ * Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
+ *
+ * Copyright (C) 2016 Cavium, Inc.
+ */
+@@ -15,16 +12,146 @@
+ #include <linux/pci.h>
+ #include <linux/pci_ids.h>
+
++#include <asm/arch_timer.h>
++
++/* PCI device IDs */
++#define PCI_DEVID_CAVIUM_RNG_PF 0xA018
++#define PCI_DEVID_CAVIUM_RNG_VF 0xA033
++
++#define HEALTH_STATUS_REG 0x38
++
++/* RST device info */
++#define PCI_DEVICE_ID_RST_OTX2 0xA085
++#define RST_BOOT_REG 0x1600ULL
++#define CLOCK_BASE_RATE 50000000ULL
++#define MSEC_TO_NSEC(x) (x * 1000000)
++
+ struct cavium_rng {
+ struct hwrng ops;
+ void __iomem *result;
++ void __iomem *pf_regbase;
++ struct pci_dev *pdev;
++ u64 clock_rate;
++ u64 prev_error;
++ u64 prev_time;
+ };
+
++static inline bool is_octeontx(struct pci_dev *pdev)
++{
++ if (midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_83XX,
++ MIDR_CPU_VAR_REV(0, 0),
++ MIDR_CPU_VAR_REV(3, 0)) ||
++ midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_81XX,
++ MIDR_CPU_VAR_REV(0, 0),
++ MIDR_CPU_VAR_REV(3, 0)) ||
++ midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX,
++ MIDR_CPU_VAR_REV(0, 0),
++ MIDR_CPU_VAR_REV(3, 0)))
++ return true;
++
++ return false;
++}
++
++static u64 rng_get_coprocessor_clkrate(void)
++{
++ u64 ret = CLOCK_BASE_RATE * 16; /* Assume 800Mhz as default */
++ struct pci_dev *pdev;
++ void __iomem *base;
++
++ pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
++ PCI_DEVICE_ID_RST_OTX2, NULL);
++ if (!pdev)
++ goto error;
++
++ base = pci_ioremap_bar(pdev, 0);
++ if (!base)
++ goto error_put_pdev;
++
++ /* RST: PNR_MUL * 50Mhz gives clockrate */
++ ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT_REG) >> 33) & 0x3F);
++
++ iounmap(base);
++
++error_put_pdev:
++ pci_dev_put(pdev);
++
++error:
++ return ret;
++}
++
++static int check_rng_health(struct cavium_rng *rng)
++{
++ u64 cur_err, cur_time;
++ u64 status, cycles;
++ u64 time_elapsed;
++
++
++ /* Skip checking health for OcteonTx */
++ if (!rng->pf_regbase)
++ return 0;
++
++ status = readq(rng->pf_regbase + HEALTH_STATUS_REG);
++ if (status & BIT_ULL(0)) {
++ dev_err(&rng->pdev->dev, "HWRNG: Startup health test failed\n");
++ return -EIO;
++ }
++
++ cycles = status >> 1;
++ if (!cycles)
++ return 0;
++
++ cur_time = arch_timer_read_counter();
++
++ /* RNM_HEALTH_STATUS[CYCLES_SINCE_HEALTH_FAILURE]
++ * Number of coprocessor cycles times 2 since the last failure.
++ * This field doesn't get cleared/updated until another failure.
++ */
++ cycles = cycles / 2;
++ cur_err = (cycles * 1000000000) / rng->clock_rate; /* In nanosec */
++
++ /* Ignore errors that happenned a long time ago, these
++ * are most likely false positive errors.
++ */
++ if (cur_err > MSEC_TO_NSEC(10)) {
++ rng->prev_error = 0;
++ rng->prev_time = 0;
++ return 0;
++ }
++
++ if (rng->prev_error) {
++ /* Calculate time elapsed since last error
++ * '1' tick of CNTVCT is 10ns, since it runs at 100Mhz.
++ */
++ time_elapsed = (cur_time - rng->prev_time) * 10;
++ time_elapsed += rng->prev_error;
++
++ /* Check if current error is a new one or the old one itself.
++ * If error is a new one then consider there is a persistent
++ * issue with entropy, declare hardware failure.
++ */
++ if (cur_err < time_elapsed) {
++ dev_err(&rng->pdev->dev, "HWRNG failure detected\n");
++ rng->prev_error = cur_err;
++ rng->prev_time = cur_time;
++ return -EIO;
++ }
++ }
++
++ rng->prev_error = cur_err;
++ rng->prev_time = cur_time;
++ return 0;
++}
++
+ /* Read data from the RNG unit */
+ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
+ {
+ struct cavium_rng *p = container_of(rng, struct cavium_rng, ops);
+ unsigned int size = max;
++ int err = 0;
++
++ err = check_rng_health(p);
++ if (err)
++ return err;
+
+ while (size >= 8) {
+ *((u64 *)dat) = readq(p->result);
+@@ -39,6 +166,39 @@ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
+ return max;
+ }
+
++static int cavium_map_pf_regs(struct cavium_rng *rng)
++{
++ struct pci_dev *pdev;
++
++ /* Health status is not supported on 83xx, skip mapping PF CSRs */
++ if (is_octeontx(rng->pdev)) {
++ rng->pf_regbase = NULL;
++ return 0;
++ }
++
++ pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
++ PCI_DEVID_CAVIUM_RNG_PF, NULL);
++ if (!pdev) {
++ dev_err(&pdev->dev, "Cannot find RNG PF device\n");
++ return -EIO;
++ }
++
++ rng->pf_regbase = ioremap(pci_resource_start(pdev, 0),
++ pci_resource_len(pdev, 0));
++ if (!rng->pf_regbase) {
++ dev_err(&pdev->dev, "Failed to map PF CSR region\n");
++ pci_dev_put(pdev);
++ return -ENOMEM;
++ }
++
++ pci_dev_put(pdev);
++
++ /* Get co-processor clock rate */
++ rng->clock_rate = rng_get_coprocessor_clkrate();
++
++ return 0;
++}
++
+ /* Map Cavium RNG to an HWRNG object */
+ static int cavium_rng_probe_vf(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+@@ -50,6 +210,8 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
+ if (!rng)
+ return -ENOMEM;
+
++ rng->pdev = pdev;
++
+ /* Map the RNG result */
+ rng->result = pcim_iomap(pdev, 0, 0);
+ if (!rng->result) {
+@@ -67,6 +229,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
+
+ pci_set_drvdata(pdev, rng);
+
++ /* Health status is available only at PF, hence map PF registers. */
++ ret = cavium_map_pf_regs(rng);
++ if (ret)
++ return ret;
++
+ ret = devm_hwrng_register(&pdev->dev, &rng->ops);
+ if (ret) {
+ dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
+@@ -76,10 +243,18 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
+ return 0;
+ }
+
++/* Remove the VF */
++static void cavium_rng_remove_vf(struct pci_dev *pdev)
++{
++ struct cavium_rng *rng;
++
++ rng = pci_get_drvdata(pdev);
++ iounmap(rng->pf_regbase);
++}
+
+ static const struct pci_device_id cavium_rng_vf_id_table[] = {
+- { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
+- {0,},
++ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CAVIUM_RNG_VF) },
++ { 0, }
+ };
+ MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table);
+
+@@ -87,8 +262,9 @@ static struct pci_driver cavium_rng_vf_driver = {
+ .name = "cavium_rng_vf",
+ .id_table = cavium_rng_vf_id_table,
+ .probe = cavium_rng_probe_vf,
++ .remove = cavium_rng_remove_vf,
+ };
+ module_pci_driver(cavium_rng_vf_driver);
+
+ MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
+-MODULE_LICENSE("GPL");
++MODULE_LICENSE("GPL v2");
+diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c
+index 63d6e68c24d2f..b96579222408b 100644
+--- a/drivers/char/hw_random/cavium-rng.c
++++ b/drivers/char/hw_random/cavium-rng.c
+@@ -1,10 +1,7 @@
++// SPDX-License-Identifier: GPL-2.0
+ /*
+- * Hardware Random Number Generator support for Cavium Inc.
+- * Thunder processor family.
+- *
+- * This file is subject to the terms and conditions of the GNU General Public
+- * License. See the file "COPYING" in the main directory of this archive
+- * for more details.
++ * Hardware Random Number Generator support.
++ * Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
+ *
+ * Copyright (C) 2016 Cavium, Inc.
+ */
+@@ -91,4 +88,4 @@ static struct pci_driver cavium_rng_pf_driver = {
+
+ module_pci_driver(cavium_rng_pf_driver);
+ MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
+-MODULE_LICENSE("GPL");
++MODULE_LICENSE("GPL v2");
+diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
+index a3db27916256d..cfb085de876b7 100644
+--- a/drivers/char/hw_random/core.c
++++ b/drivers/char/hw_random/core.c
+@@ -15,6 +15,7 @@
+ #include <linux/err.h>
+ #include <linux/fs.h>
+ #include <linux/hw_random.h>
++#include <linux/random.h>
+ #include <linux/kernel.h>
+ #include <linux/kthread.h>
+ #include <linux/sched/signal.h>
+diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c
+index 8ad7b515a51b8..6c00ea0085553 100644
+--- a/drivers/char/hw_random/mtk-rng.c
++++ b/drivers/char/hw_random/mtk-rng.c
+@@ -166,8 +166,13 @@ static int mtk_rng_runtime_resume(struct device *dev)
+ return mtk_rng_init(&priv->rng);
+ }
+
+-static UNIVERSAL_DEV_PM_OPS(mtk_rng_pm_ops, mtk_rng_runtime_suspend,
+- mtk_rng_runtime_resume, NULL);
++static const struct dev_pm_ops mtk_rng_pm_ops = {
++ SET_RUNTIME_PM_OPS(mtk_rng_runtime_suspend,
++ mtk_rng_runtime_resume, NULL)
++ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
++ pm_runtime_force_resume)
++};
++
+ #define MTK_RNG_PM_OPS (&mtk_rng_pm_ops)
+ #else /* CONFIG_PM */
+ #define MTK_RNG_PM_OPS NULL
+diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c
+index 67947a19aa225..e8f9621e79541 100644
+--- a/drivers/char/hw_random/nomadik-rng.c
++++ b/drivers/char/hw_random/nomadik-rng.c
+@@ -65,14 +65,14 @@ static int nmk_rng_probe(struct amba_device *dev, const struct amba_id *id)
+ out_release:
+ amba_release_regions(dev);
+ out_clk:
+- clk_disable(rng_clk);
++ clk_disable_unprepare(rng_clk);
+ return ret;
+ }
+
+ static void nmk_rng_remove(struct amba_device *dev)
+ {
+ amba_release_regions(dev);
+- clk_disable(rng_clk);
++ clk_disable_unprepare(rng_clk);
+ }
+
+ static const struct amba_id nmk_rng_ids[] = {
+diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
+index e96cb5c4f97a3..fe91090e04a46 100644
+--- a/drivers/char/ipmi/ipmi_msghandler.c
++++ b/drivers/char/ipmi/ipmi_msghandler.c
+@@ -191,6 +191,8 @@ struct ipmi_user {
+ struct work_struct remove_work;
+ };
+
++static struct workqueue_struct *remove_work_wq;
++
+ static struct ipmi_user *acquire_ipmi_user(struct ipmi_user *user, int *index)
+ __acquires(user->release_barrier)
+ {
+@@ -1261,7 +1263,7 @@ static void free_user(struct kref *ref)
+ struct ipmi_user *user = container_of(ref, struct ipmi_user, refcount);
+
+ /* SRCU cleanup must happen in task context. */
+- schedule_work(&user->remove_work);
++ queue_work(remove_work_wq, &user->remove_work);
+ }
+
+ static void _ipmi_destroy_user(struct ipmi_user *user)
+@@ -2930,7 +2932,7 @@ cleanup_bmc_device(struct kref *ref)
+ * with removing the device attributes while reading a device
+ * attribute.
+ */
+- schedule_work(&bmc->remove_work);
++ queue_work(remove_work_wq, &bmc->remove_work);
+ }
+
+ /*
+@@ -4789,7 +4791,9 @@ static atomic_t recv_msg_inuse_count = ATOMIC_INIT(0);
+ static void free_smi_msg(struct ipmi_smi_msg *msg)
+ {
+ atomic_dec(&smi_msg_inuse_count);
+- kfree(msg);
++ /* Try to keep as much stuff out of the panic path as possible. */
++ if (!oops_in_progress)
++ kfree(msg);
+ }
+
+ struct ipmi_smi_msg *ipmi_alloc_smi_msg(void)
+@@ -4808,7 +4812,9 @@ EXPORT_SYMBOL(ipmi_alloc_smi_msg);
+ static void free_recv_msg(struct ipmi_recv_msg *msg)
+ {
+ atomic_dec(&recv_msg_inuse_count);
+- kfree(msg);
++ /* Try to keep as much stuff out of the panic path as possible. */
++ if (!oops_in_progress)
++ kfree(msg);
+ }
+
+ static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void)
+@@ -4826,7 +4832,7 @@ static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void)
+
+ void ipmi_free_recv_msg(struct ipmi_recv_msg *msg)
+ {
+- if (msg->user)
++ if (msg->user && !oops_in_progress)
+ kref_put(&msg->user->refcount, free_user);
+ msg->done(msg);
+ }
+@@ -5142,7 +5148,16 @@ static int ipmi_init_msghandler(void)
+ if (initialized)
+ goto out;
+
+- init_srcu_struct(&ipmi_interfaces_srcu);
++ rv = init_srcu_struct(&ipmi_interfaces_srcu);
++ if (rv)
++ goto out;
++
++ remove_work_wq = create_singlethread_workqueue("ipmi-msghandler-remove-wq");
++ if (!remove_work_wq) {
++ pr_err("unable to create ipmi-msghandler-remove-wq workqueue");
++ rv = -ENOMEM;
++ goto out_wq;
++ }
+
+ timer_setup(&ipmi_timer, ipmi_timeout, 0);
+ mod_timer(&ipmi_timer, jiffies + IPMI_TIMEOUT_JIFFIES);
+@@ -5151,6 +5166,9 @@ static int ipmi_init_msghandler(void)
+
+ initialized = true;
+
++out_wq:
++ if (rv)
++ cleanup_srcu_struct(&ipmi_interfaces_srcu);
+ out:
+ mutex_unlock(&ipmi_interfaces_mutex);
+ return rv;
+@@ -5174,6 +5192,8 @@ static void __exit cleanup_ipmi(void)
+ int count;
+
+ if (initialized) {
++ destroy_workqueue(remove_work_wq);
++
+ atomic_notifier_chain_unregister(&panic_notifier_list,
+ &panic_block);
+
+diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
+index 20d5af92966d4..8d7a8898e80b0 100644
+--- a/drivers/char/ipmi/ipmi_ssif.c
++++ b/drivers/char/ipmi/ipmi_ssif.c
+@@ -1659,6 +1659,9 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
+ }
+ }
+
++ ssif_info->client = client;
++ i2c_set_clientdata(client, ssif_info);
++
+ rv = ssif_check_and_remove(client, ssif_info);
+ /* If rv is 0 and addr source is not SI_ACPI, continue probing */
+ if (!rv && ssif_info->addr_source == SI_ACPI) {
+@@ -1679,9 +1682,6 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
+ ipmi_addr_src_to_str(ssif_info->addr_source),
+ client->addr, client->adapter->name, slave_addr);
+
+- ssif_info->client = client;
+- i2c_set_clientdata(client, ssif_info);
+-
+ /* Now check for system interface capabilities */
+ msg[0] = IPMI_NETFN_APP_REQUEST << 2;
+ msg[1] = IPMI_GET_SYSTEM_INTERFACE_CAPABILITIES_CMD;
+@@ -1881,6 +1881,7 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
+
+ dev_err(&ssif_info->client->dev,
+ "Unable to start IPMI SSIF: %d\n", rv);
++ i2c_set_clientdata(client, NULL);
+ kfree(ssif_info);
+ }
+ kfree(resp);
+diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c
+index e4ff3b50de7f3..883b4a3410122 100644
+--- a/drivers/char/ipmi/ipmi_watchdog.c
++++ b/drivers/char/ipmi/ipmi_watchdog.c
+@@ -342,13 +342,17 @@ static atomic_t msg_tofree = ATOMIC_INIT(0);
+ static DECLARE_COMPLETION(msg_wait);
+ static void msg_free_smi(struct ipmi_smi_msg *msg)
+ {
+- if (atomic_dec_and_test(&msg_tofree))
+- complete(&msg_wait);
++ if (atomic_dec_and_test(&msg_tofree)) {
++ if (!oops_in_progress)
++ complete(&msg_wait);
++ }
+ }
+ static void msg_free_recv(struct ipmi_recv_msg *msg)
+ {
+- if (atomic_dec_and_test(&msg_tofree))
+- complete(&msg_wait);
++ if (atomic_dec_and_test(&msg_tofree)) {
++ if (!oops_in_progress)
++ complete(&msg_wait);
++ }
+ }
+ static struct ipmi_smi_msg smi_msg = {
+ .done = msg_free_smi
+@@ -434,8 +438,10 @@ static int _ipmi_set_timeout(int do_heartbeat)
+ rv = __ipmi_set_timeout(&smi_msg,
+ &recv_msg,
+ &send_heartbeat_now);
+- if (rv)
++ if (rv) {
++ atomic_set(&msg_tofree, 0);
+ return rv;
++ }
+
+ wait_for_completion(&msg_wait);
+
+@@ -497,7 +503,7 @@ static void panic_halt_ipmi_heartbeat(void)
+ msg.cmd = IPMI_WDOG_RESET_TIMER;
+ msg.data = NULL;
+ msg.data_len = 0;
+- atomic_inc(&panic_done_count);
++ atomic_add(2, &panic_done_count);
+ rv = ipmi_request_supply_msgs(watchdog_user,
+ (struct ipmi_addr *) &addr,
+ 0,
+@@ -507,7 +513,7 @@ static void panic_halt_ipmi_heartbeat(void)
+ &panic_halt_heartbeat_recv_msg,
+ 1);
+ if (rv)
+- atomic_dec(&panic_done_count);
++ atomic_sub(2, &panic_done_count);
+ }
+
+ static struct ipmi_smi_msg panic_halt_smi_msg = {
+@@ -531,12 +537,12 @@ static void panic_halt_ipmi_set_timeout(void)
+ /* Wait for the messages to be free. */
+ while (atomic_read(&panic_done_count) != 0)
+ ipmi_poll_interface(watchdog_user);
+- atomic_inc(&panic_done_count);
++ atomic_add(2, &panic_done_count);
+ rv = __ipmi_set_timeout(&panic_halt_smi_msg,
+ &panic_halt_recv_msg,
+ &send_heartbeat_now);
+ if (rv) {
+- atomic_dec(&panic_done_count);
++ atomic_sub(2, &panic_done_count);
+ pr_warn("Unable to extend the watchdog timeout\n");
+ } else {
+ if (send_heartbeat_now)
+@@ -580,6 +586,7 @@ restart:
+ &recv_msg,
+ 1);
+ if (rv) {
++ atomic_set(&msg_tofree, 0);
+ pr_warn("heartbeat send failure: %d\n", rv);
+ return rv;
+ }
+diff --git a/drivers/char/ipmi/kcs_bmc_serio.c b/drivers/char/ipmi/kcs_bmc_serio.c
+index 7948cabde50b4..7e2067628a6ce 100644
+--- a/drivers/char/ipmi/kcs_bmc_serio.c
++++ b/drivers/char/ipmi/kcs_bmc_serio.c
+@@ -73,10 +73,12 @@ static int kcs_bmc_serio_add_device(struct kcs_bmc_device *kcs_bmc)
+ struct serio *port;
+
+ priv = devm_kzalloc(kcs_bmc->dev, sizeof(*priv), GFP_KERNEL);
++ if (!priv)
++ return -ENOMEM;
+
+ /* Use kzalloc() as the allocation is cleaned up with kfree() via serio_unregister_port() */
+ port = kzalloc(sizeof(*port), GFP_KERNEL);
+- if (!(priv && port))
++ if (!port)
+ return -ENOMEM;
+
+ port->id.type = SERIO_8042;
+diff --git a/drivers/char/mwave/3780i.h b/drivers/char/mwave/3780i.h
+index 9ccb6b270b071..95164246afd1a 100644
+--- a/drivers/char/mwave/3780i.h
++++ b/drivers/char/mwave/3780i.h
+@@ -68,7 +68,7 @@ typedef struct {
+ unsigned char ClockControl:1; /* RW: Clock control: 0=normal, 1=stop 3780i clocks */
+ unsigned char SoftReset:1; /* RW: Soft reset 0=normal, 1=soft reset active */
+ unsigned char ConfigMode:1; /* RW: Configuration mode, 0=normal, 1=config mode */
+- unsigned char Reserved:5; /* 0: Reserved */
++ unsigned short Reserved:13; /* 0: Reserved */
+ } DSP_ISA_SLAVE_CONTROL;
+
+
+diff --git a/drivers/char/random.c b/drivers/char/random.c
+index 605969ed0f965..ca17a658c2147 100644
+--- a/drivers/char/random.c
++++ b/drivers/char/random.c
+@@ -1,310 +1,26 @@
++// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+ /*
+- * random.c -- A strong random number generator
+- *
+- * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All
+- * Rights Reserved.
+- *
++ * Copyright (C) 2017-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005
+- *
+- * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All
+- * rights reserved.
+- *
+- * Redistribution and use in source and binary forms, with or without
+- * modification, are permitted provided that the following conditions
+- * are met:
+- * 1. Redistributions of source code must retain the above copyright
+- * notice, and the entire permission notice in its entirety,
+- * including the disclaimer of warranties.
+- * 2. Redistributions in binary form must reproduce the above copyright
+- * notice, this list of conditions and the following disclaimer in the
+- * documentation and/or other materials provided with the distribution.
+- * 3. The name of the author may not be used to endorse or promote
+- * products derived from this software without specific prior
+- * written permission.
+- *
+- * ALTERNATIVELY, this product may be distributed under the terms of
+- * the GNU General Public License, in which case the provisions of the GPL are
+- * required INSTEAD OF the above restrictions. (This clause is
+- * necessary due to a potential bad interaction between the GPL and
+- * the restrictions contained in a BSD-style copyright.)
+- *
+- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
+- * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
+- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
+- * DAMAGE.
+- */
+-
+-/*
+- * (now, with legal B.S. out of the way.....)
+- *
+- * This routine gathers environmental noise from device drivers, etc.,
+- * and returns good random numbers, suitable for cryptographic use.
+- * Besides the obvious cryptographic uses, these numbers are also good
+- * for seeding TCP sequence numbers, and other places where it is
+- * desirable to have numbers which are not only random, but hard to
+- * predict by an attacker.
+- *
+- * Theory of operation
+- * ===================
+- *
+- * Computers are very predictable devices. Hence it is extremely hard
+- * to produce truly random numbers on a computer --- as opposed to
+- * pseudo-random numbers, which can easily generated by using a
+- * algorithm. Unfortunately, it is very easy for attackers to guess
+- * the sequence of pseudo-random number generators, and for some
+- * applications this is not acceptable. So instead, we must try to
+- * gather "environmental noise" from the computer's environment, which
+- * must be hard for outside attackers to observe, and use that to
+- * generate random numbers. In a Unix environment, this is best done
+- * from inside the kernel.
+- *
+- * Sources of randomness from the environment include inter-keyboard
+- * timings, inter-interrupt timings from some interrupts, and other
+- * events which are both (a) non-deterministic and (b) hard for an
+- * outside observer to measure. Randomness from these sources are
+- * added to an "entropy pool", which is mixed using a CRC-like function.
+- * This is not cryptographically strong, but it is adequate assuming
+- * the randomness is not chosen maliciously, and it is fast enough that
+- * the overhead of doing it on every interrupt is very reasonable.
+- * As random bytes are mixed into the entropy pool, the routines keep
+- * an *estimate* of how many bits of randomness have been stored into
+- * the random number generator's internal state.
+- *
+- * When random bytes are desired, they are obtained by taking the SHA
+- * hash of the contents of the "entropy pool". The SHA hash avoids
+- * exposing the internal state of the entropy pool. It is believed to
+- * be computationally infeasible to derive any useful information
+- * about the input of SHA from its output. Even if it is possible to
+- * analyze SHA in some clever way, as long as the amount of data
+- * returned from the generator is less than the inherent entropy in
+- * the pool, the output data is totally unpredictable. For this
+- * reason, the routine decreases its internal estimate of how many
+- * bits of "true randomness" are contained in the entropy pool as it
+- * outputs random numbers.
+- *
+- * If this estimate goes to zero, the routine can still generate
+- * random numbers; however, an attacker may (at least in theory) be
+- * able to infer the future output of the generator from prior
+- * outputs. This requires successful cryptanalysis of SHA, which is
+- * not believed to be feasible, but there is a remote possibility.
+- * Nonetheless, these numbers should be useful for the vast majority
+- * of purposes.
+- *
+- * Exported interfaces ---- output
+- * ===============================
+- *
+- * There are four exported interfaces; two for use within the kernel,
+- * and two or use from userspace.
+- *
+- * Exported interfaces ---- userspace output
+- * -----------------------------------------
+- *
+- * The userspace interfaces are two character devices /dev/random and
+- * /dev/urandom. /dev/random is suitable for use when very high
+- * quality randomness is desired (for example, for key generation or
+- * one-time pads), as it will only return a maximum of the number of
+- * bits of randomness (as estimated by the random number generator)
+- * contained in the entropy pool.
+- *
+- * The /dev/urandom device does not have this limit, and will return
+- * as many bytes as are requested. As more and more random bytes are
+- * requested without giving time for the entropy pool to recharge,
+- * this will result in random numbers that are merely cryptographically
+- * strong. For many applications, however, this is acceptable.
+- *
+- * Exported interfaces ---- kernel output
+- * --------------------------------------
+- *
+- * The primary kernel interface is
+- *
+- * void get_random_bytes(void *buf, int nbytes);
+- *
+- * This interface will return the requested number of random bytes,
+- * and place it in the requested buffer. This is equivalent to a
+- * read from /dev/urandom.
+- *
+- * For less critical applications, there are the functions:
+- *
+- * u32 get_random_u32()
+- * u64 get_random_u64()
+- * unsigned int get_random_int()
+- * unsigned long get_random_long()
+- *
+- * These are produced by a cryptographic RNG seeded from get_random_bytes,
+- * and so do not deplete the entropy pool as much. These are recommended
+- * for most in-kernel operations *if the result is going to be stored in
+- * the kernel*.
+- *
+- * Specifically, the get_random_int() family do not attempt to do
+- * "anti-backtracking". If you capture the state of the kernel (e.g.
+- * by snapshotting the VM), you can figure out previous get_random_int()
+- * return values. But if the value is stored in the kernel anyway,
+- * this is not a problem.
+- *
+- * It *is* safe to expose get_random_int() output to attackers (e.g. as
+- * network cookies); given outputs 1..n, it's not feasible to predict
+- * outputs 0 or n+1. The only concern is an attacker who breaks into
+- * the kernel later; the get_random_int() engine is not reseeded as
+- * often as the get_random_bytes() one.
+- *
+- * get_random_bytes() is needed for keys that need to stay secret after
+- * they are erased from the kernel. For example, any key that will
+- * be wrapped and stored encrypted. And session encryption keys: we'd
+- * like to know that after the session is closed and the keys erased,
+- * the plaintext is unrecoverable to someone who recorded the ciphertext.
+- *
+- * But for network ports/cookies, stack canaries, PRNG seeds, address
+- * space layout randomization, session *authentication* keys, or other
+- * applications where the sensitive data is stored in the kernel in
+- * plaintext for as long as it's sensitive, the get_random_int() family
+- * is just fine.
+- *
+- * Consider ASLR. We want to keep the address space secret from an
+- * outside attacker while the process is running, but once the address
+- * space is torn down, it's of no use to an attacker any more. And it's
+- * stored in kernel data structures as long as it's alive, so worrying
+- * about an attacker's ability to extrapolate it from the get_random_int()
+- * CRNG is silly.
+- *
+- * Even some cryptographic keys are safe to generate with get_random_int().
+- * In particular, keys for SipHash are generally fine. Here, knowledge
+- * of the key authorizes you to do something to a kernel object (inject
+- * packets to a network connection, or flood a hash table), and the
+- * key is stored with the object being protected. Once it goes away,
+- * we no longer care if anyone knows the key.
+- *
+- * prandom_u32()
+- * -------------
+- *
+- * For even weaker applications, see the pseudorandom generator
+- * prandom_u32(), prandom_max(), and prandom_bytes(). If the random
+- * numbers aren't security-critical at all, these are *far* cheaper.
+- * Useful for self-tests, random error simulation, randomized backoffs,
+- * and any other application where you trust that nobody is trying to
+- * maliciously mess with you by guessing the "random" numbers.
+- *
+- * Exported interfaces ---- input
+- * ==============================
+- *
+- * The current exported interfaces for gathering environmental noise
+- * from the devices are:
+- *
+- * void add_device_randomness(const void *buf, unsigned int size);
+- * void add_input_randomness(unsigned int type, unsigned int code,
+- * unsigned int value);
+- * void add_interrupt_randomness(int irq, int irq_flags);
+- * void add_disk_randomness(struct gendisk *disk);
+- *
+- * add_device_randomness() is for adding data to the random pool that
+- * is likely to differ between two devices (or possibly even per boot).
+- * This would be things like MAC addresses or serial numbers, or the
+- * read-out of the RTC. This does *not* add any actual entropy to the
+- * pool, but it initializes the pool to different values for devices
+- * that might otherwise be identical and have very little entropy
+- * available to them (particularly common in the embedded world).
+- *
+- * add_input_randomness() uses the input layer interrupt timing, as well as
+- * the event type information from the hardware.
+- *
+- * add_interrupt_randomness() uses the interrupt timing as random
+- * inputs to the entropy pool. Using the cycle counters and the irq source
+- * as inputs, it feeds the randomness roughly once a second.
+- *
+- * add_disk_randomness() uses what amounts to the seek time of block
+- * layer request events, on a per-disk_devt basis, as input to the
+- * entropy pool. Note that high-speed solid state drives with very low
+- * seek times do not make for good sources of entropy, as their seek
+- * times are usually fairly consistent.
+- *
+- * All of these routines try to estimate how many bits of randomness a
+- * particular randomness source. They do this by keeping track of the
+- * first and second order deltas of the event timings.
+- *
+- * Ensuring unpredictability at system startup
+- * ============================================
+- *
+- * When any operating system starts up, it will go through a sequence
+- * of actions that are fairly predictable by an adversary, especially
+- * if the start-up does not involve interaction with a human operator.
+- * This reduces the actual number of bits of unpredictability in the
+- * entropy pool below the value in entropy_count. In order to
+- * counteract this effect, it helps to carry information in the
+- * entropy pool across shut-downs and start-ups. To do this, put the
+- * following lines an appropriate script which is run during the boot
+- * sequence:
+- *
+- * echo "Initializing random number generator..."
+- * random_seed=/var/run/random-seed
+- * # Carry a random seed from start-up to start-up
+- * # Load and then save the whole entropy pool
+- * if [ -f $random_seed ]; then
+- * cat $random_seed >/dev/urandom
+- * else
+- * touch $random_seed
+- * fi
+- * chmod 600 $random_seed
+- * dd if=/dev/urandom of=$random_seed count=1 bs=512
+- *
+- * and the following lines in an appropriate script which is run as
+- * the system is shutdown:
+- *
+- * # Carry a random seed from shut-down to start-up
+- * # Save the whole entropy pool
+- * echo "Saving random seed..."
+- * random_seed=/var/run/random-seed
+- * touch $random_seed
+- * chmod 600 $random_seed
+- * dd if=/dev/urandom of=$random_seed count=1 bs=512
+- *
+- * For example, on most modern systems using the System V init
+- * scripts, such code fragments would be found in
+- * /etc/rc.d/init.d/random. On older Linux systems, the correct script
+- * location might be in /etc/rcb.d/rc.local or /etc/rc.d/rc.0.
+- *
+- * Effectively, these commands cause the contents of the entropy pool
+- * to be saved at shut-down time and reloaded into the entropy pool at
+- * start-up. (The 'dd' in the addition to the bootup script is to
+- * make sure that /etc/random-seed is different for every start-up,
+- * even if the system crashes without executing rc.0.) Even with
+- * complete knowledge of the start-up activities, predicting the state
+- * of the entropy pool requires knowledge of the previous history of
+- * the system.
+- *
+- * Configuring the /dev/random driver under Linux
+- * ==============================================
+- *
+- * The /dev/random driver under Linux uses minor numbers 8 and 9 of
+- * the /dev/mem major number (#1). So if your system does not have
+- * /dev/random and /dev/urandom created already, they can be created
+- * by using the commands:
+- *
+- * mknod /dev/random c 1 8
+- * mknod /dev/urandom c 1 9
+- *
+- * Acknowledgements:
+- * =================
+- *
+- * Ideas for constructing this random number generator were derived
+- * from Pretty Good Privacy's random number generator, and from private
+- * discussions with Phil Karn. Colin Plumb provided a faster random
+- * number generator, which speed up the mixing function of the entropy
+- * pool, taken from PGPfone. Dale Worley has also contributed many
+- * useful ideas and suggestions to improve this driver.
+- *
+- * Any flaws in the design are solely my responsibility, and should
+- * not be attributed to the Phil, Colin, or any of authors of PGP.
+- *
+- * Further background information on this topic may be obtained from
+- * RFC 1750, "Randomness Recommendations for Security", by Donald
+- * Eastlake, Steve Crocker, and Jeff Schiller.
++ * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All rights reserved.
++ *
++ * This driver produces cryptographically secure pseudorandom data. It is divided
++ * into roughly six sections, each with a section header:
++ *
++ * - Initialization and readiness waiting.
++ * - Fast key erasure RNG, the "crng".
++ * - Entropy accumulation and extraction routines.
++ * - Entropy collection routines.
++ * - Userspace reader/writer interfaces.
++ * - Sysctl interface.
++ *
++ * The high level overview is that there is one input pool, into which
++ * various pieces of data are hashed. Prior to initialization, some of that
++ * data is then "credited" as having a certain number of bits of entropy.
++ * When enough bits of entropy are available, the hash is finalized and
++ * handed as a key to a stream cipher that expands it indefinitely for
++ * various consumers. This key is periodically refreshed as the various
++ * entropy collectors, described below, add data to the input pool.
+ */
+
+ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+@@ -327,7 +43,6 @@
+ #include <linux/spinlock.h>
+ #include <linux/kthread.h>
+ #include <linux/percpu.h>
+-#include <linux/fips.h>
+ #include <linux/ptrace.h>
+ #include <linux/workqueue.h>
+ #include <linux/irq.h>
+@@ -335,774 +50,809 @@
+ #include <linux/syscalls.h>
+ #include <linux/completion.h>
+ #include <linux/uuid.h>
++#include <linux/uaccess.h>
++#include <linux/siphash.h>
++#include <linux/uio.h>
+ #include <crypto/chacha.h>
+-#include <crypto/sha1.h>
+-
++#include <crypto/blake2s.h>
+ #include <asm/processor.h>
+-#include <linux/uaccess.h>
+ #include <asm/irq.h>
+ #include <asm/irq_regs.h>
+ #include <asm/io.h>
+
+-#define CREATE_TRACE_POINTS
+-#include <trace/events/random.h>
+-
+-/* #define ADD_INTERRUPT_BENCH */
++/*********************************************************************
++ *
++ * Initialization and readiness waiting.
++ *
++ * Much of the RNG infrastructure is devoted to various dependencies
++ * being able to wait until the RNG has collected enough entropy and
++ * is ready for safe consumption.
++ *
++ *********************************************************************/
+
+ /*
+- * Configuration information
++ * crng_init is protected by base_crng->lock, and only increases
++ * its value (from empty->early->ready).
+ */
+-#define INPUT_POOL_SHIFT 12
+-#define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5))
+-#define OUTPUT_POOL_SHIFT 10
+-#define OUTPUT_POOL_WORDS (1 << (OUTPUT_POOL_SHIFT-5))
+-#define EXTRACT_SIZE 10
+-
++static enum {
++ CRNG_EMPTY = 0, /* Little to no entropy collected */
++ CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
++ CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
++} crng_init __read_mostly = CRNG_EMPTY;
++static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
++#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
++/* Various types of waiters for crng_init->CRNG_READY transition. */
++static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
++static struct fasync_struct *fasync;
++static DEFINE_SPINLOCK(random_ready_chain_lock);
++static RAW_NOTIFIER_HEAD(random_ready_chain);
+
+-#define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long))
++/* Control how we warn userspace. */
++static struct ratelimit_state urandom_warning =
++ RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
++static int ratelimit_disable __read_mostly =
++ IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM);
++module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
++MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
+
+ /*
+- * To allow fractional bits to be tracked, the entropy_count field is
+- * denominated in units of 1/8th bits.
++ * Returns whether or not the input pool has been seeded and thus guaranteed
++ * to supply cryptographically secure random numbers. This applies to: the
++ * /dev/urandom device, the get_random_bytes function, and the get_random_{u32,
++ * ,u64,int,long} family of functions.
+ *
+- * 2*(ENTROPY_SHIFT + poolbitshift) must <= 31, or the multiply in
+- * credit_entropy_bits() needs to be 64 bits wide.
++ * Returns: true if the input pool has been seeded.
++ * false if the input pool has not been seeded.
+ */
+-#define ENTROPY_SHIFT 3
+-#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
++bool rng_is_initialized(void)
++{
++ return crng_ready();
++}
++EXPORT_SYMBOL(rng_is_initialized);
+
+-/*
+- * If the entropy count falls under this number of bits, then we
+- * should wake up processes which are selecting or polling on write
+- * access to /dev/random.
+- */
+-static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
++static void __cold crng_set_ready(struct work_struct *work)
++{
++ static_branch_enable(&crng_is_ready);
++}
++
++/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
++static void try_to_generate_entropy(void);
+
+ /*
+- * Originally, we used a primitive polynomial of degree .poolwords
+- * over GF(2). The taps for various sizes are defined below. They
+- * were chosen to be evenly spaced except for the last tap, which is 1
+- * to get the twisting happening as fast as possible.
+- *
+- * For the purposes of better mixing, we use the CRC-32 polynomial as
+- * well to make a (modified) twisted Generalized Feedback Shift
+- * Register. (See M. Matsumoto & Y. Kurita, 1992. Twisted GFSR
+- * generators. ACM Transactions on Modeling and Computer Simulation
+- * 2(3):179-194. Also see M. Matsumoto & Y. Kurita, 1994. Twisted
+- * GFSR generators II. ACM Transactions on Modeling and Computer
+- * Simulation 4:254-266)
+- *
+- * Thanks to Colin Plumb for suggesting this.
+- *
+- * The mixing operation is much less sensitive than the output hash,
+- * where we use SHA-1. All that we want of mixing operation is that
+- * it be a good non-cryptographic hash; i.e. it not produce collisions
+- * when fed "random" data of the sort we expect to see. As long as
+- * the pool state differs for different inputs, we have preserved the
+- * input entropy and done a good job. The fact that an intelligent
+- * attacker can construct inputs that will produce controlled
+- * alterations to the pool's state is not important because we don't
+- * consider such inputs to contribute any randomness. The only
+- * property we need with respect to them is that the attacker can't
+- * increase his/her knowledge of the pool's state. Since all
+- * additions are reversible (knowing the final state and the input,
+- * you can reconstruct the initial state), if an attacker has any
+- * uncertainty about the initial state, he/she can only shuffle that
+- * uncertainty about, but never cause any collisions (which would
+- * decrease the uncertainty).
++ * Wait for the input pool to be seeded and thus guaranteed to supply
++ * cryptographically secure random numbers. This applies to: the /dev/urandom
++ * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
++ * family of functions. Using any of these functions without first calling
++ * this function forfeits the guarantee of security.
+ *
+- * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and
+- * Videau in their paper, "The Linux Pseudorandom Number Generator
+- * Revisited" (see: http://eprint.iacr.org/2012/251.pdf). In their
+- * paper, they point out that we are not using a true Twisted GFSR,
+- * since Matsumoto & Kurita used a trinomial feedback polynomial (that
+- * is, with only three taps, instead of the six that we are using).
+- * As a result, the resulting polynomial is neither primitive nor
+- * irreducible, and hence does not have a maximal period over
+- * GF(2**32). They suggest a slight change to the generator
+- * polynomial which improves the resulting TGFSR polynomial to be
+- * irreducible, which we have made here.
++ * Returns: 0 if the input pool has been seeded.
++ * -ERESTARTSYS if the function was interrupted by a signal.
+ */
+-static const struct poolinfo {
+- int poolbitshift, poolwords, poolbytes, poolfracbits;
+-#define S(x) ilog2(x)+5, (x), (x)*4, (x) << (ENTROPY_SHIFT+5)
+- int tap1, tap2, tap3, tap4, tap5;
+-} poolinfo_table[] = {
+- /* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */
+- /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */
+- { S(128), 104, 76, 51, 25, 1 },
+-};
++int wait_for_random_bytes(void)
++{
++ while (!crng_ready()) {
++ int ret;
++
++ try_to_generate_entropy();
++ ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
++ if (ret)
++ return ret > 0 ? 0 : ret;
++ }
++ return 0;
++}
++EXPORT_SYMBOL(wait_for_random_bytes);
+
+ /*
+- * Static global variables
++ * Add a callback function that will be invoked when the input
++ * pool is initialised.
++ *
++ * returns: 0 if callback is successfully added
++ * -EALREADY if pool is already initialised (callback not called)
+ */
+-static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
+-static struct fasync_struct *fasync;
+-
+-static DEFINE_SPINLOCK(random_ready_list_lock);
+-static LIST_HEAD(random_ready_list);
++int __cold register_random_ready_notifier(struct notifier_block *nb)
++{
++ unsigned long flags;
++ int ret = -EALREADY;
+
+-struct crng_state {
+- __u32 state[16];
+- unsigned long init_time;
+- spinlock_t lock;
+-};
++ if (crng_ready())
++ return ret;
+
+-static struct crng_state primary_crng = {
+- .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
+-};
++ spin_lock_irqsave(&random_ready_chain_lock, flags);
++ if (!crng_ready())
++ ret = raw_notifier_chain_register(&random_ready_chain, nb);
++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
++ return ret;
++}
++EXPORT_SYMBOL(register_random_ready_notifier);
+
+ /*
+- * crng_init = 0 --> Uninitialized
+- * 1 --> Initialized
+- * 2 --> Initialized from input_pool
+- *
+- * crng_init is protected by primary_crng->lock, and only increases
+- * its value (from 0->1->2).
++ * Delete a previously registered readiness callback function.
+ */
+-static int crng_init = 0;
+-#define crng_ready() (likely(crng_init > 1))
+-static int crng_init_cnt = 0;
+-static unsigned long crng_global_init_time = 0;
+-#define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE)
+-static void _extract_crng(struct crng_state *crng, __u8 out[CHACHA_BLOCK_SIZE]);
+-static void _crng_backtrack_protect(struct crng_state *crng,
+- __u8 tmp[CHACHA_BLOCK_SIZE], int used);
+-static void process_random_ready_list(void);
+-static void _get_random_bytes(void *buf, int nbytes);
+-
+-static struct ratelimit_state unseeded_warning =
+- RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
+-static struct ratelimit_state urandom_warning =
+- RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
++int __cold unregister_random_ready_notifier(struct notifier_block *nb)
++{
++ unsigned long flags;
++ int ret;
+
+-static int ratelimit_disable __read_mostly;
++ spin_lock_irqsave(&random_ready_chain_lock, flags);
++ ret = raw_notifier_chain_unregister(&random_ready_chain, nb);
++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
++ return ret;
++}
++EXPORT_SYMBOL(unregister_random_ready_notifier);
+
+-module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
+-MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
++static void __cold process_random_ready_list(void)
++{
++ unsigned long flags;
+
+-/**********************************************************************
++ spin_lock_irqsave(&random_ready_chain_lock, flags);
++ raw_notifier_call_chain(&random_ready_chain, 0, NULL);
++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
++}
++
++#define warn_unseeded_randomness() \
++ if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \
++ printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \
++ __func__, (void *)_RET_IP_, crng_init)
++
++
++/*********************************************************************
+ *
+- * OS independent entropy store. Here are the functions which handle
+- * storing entropy in an entropy pool.
++ * Fast key erasure RNG, the "crng".
+ *
+- **********************************************************************/
++ * These functions expand entropy from the entropy extractor into
++ * long streams for external consumption using the "fast key erasure"
++ * RNG described at <https://blog.cr.yp.to/20170723-random.html>.
++ *
++ * There are a few exported interfaces for use by other drivers:
++ *
++ * void get_random_bytes(void *buf, size_t len)
++ * u32 get_random_u32()
++ * u64 get_random_u64()
++ * unsigned int get_random_int()
++ * unsigned long get_random_long()
++ *
++ * These interfaces will return the requested number of random bytes
++ * into the given buffer or as a return value. This is equivalent to
++ * a read from /dev/urandom. The u32, u64, int, and long family of
++ * functions may be higher performance for one-off random integers,
++ * because they do a bit of buffering and do not invoke reseeding
++ * until the buffer is emptied.
++ *
++ *********************************************************************/
+
+-struct entropy_store;
+-struct entropy_store {
+- /* read-only data: */
+- const struct poolinfo *poolinfo;
+- __u32 *pool;
+- const char *name;
++enum {
++ CRNG_RESEED_START_INTERVAL = HZ,
++ CRNG_RESEED_INTERVAL = 60 * HZ
++};
+
+- /* read-write data: */
++static struct {
++ u8 key[CHACHA_KEY_SIZE] __aligned(__alignof__(long));
++ unsigned long birth;
++ unsigned long generation;
+ spinlock_t lock;
+- unsigned short add_ptr;
+- unsigned short input_rotate;
+- int entropy_count;
+- unsigned int last_data_init:1;
+- __u8 last_data[EXTRACT_SIZE];
++} base_crng = {
++ .lock = __SPIN_LOCK_UNLOCKED(base_crng.lock)
+ };
+
+-static ssize_t extract_entropy(struct entropy_store *r, void *buf,
+- size_t nbytes, int min, int rsvd);
+-static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
+- size_t nbytes, int fips);
+-
+-static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
+-static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
+-
+-static struct entropy_store input_pool = {
+- .poolinfo = &poolinfo_table[0],
+- .name = "input",
+- .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
+- .pool = input_pool_data
++struct crng {
++ u8 key[CHACHA_KEY_SIZE];
++ unsigned long generation;
++ local_lock_t lock;
+ };
+
+-static __u32 const twist_table[8] = {
+- 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
+- 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
+-
+-/*
+- * This function adds bytes into the entropy "pool". It does not
+- * update the entropy estimate. The caller should call
+- * credit_entropy_bits if this is appropriate.
+- *
+- * The pool is stirred with a primitive polynomial of the appropriate
+- * degree, and then twisted. We twist by three bits at a time because
+- * it's cheap to do so and helps slightly in the expected case where
+- * the entropy is concentrated in the low-order bits.
+- */
+-static void _mix_pool_bytes(struct entropy_store *r, const void *in,
+- int nbytes)
+-{
+- unsigned long i, tap1, tap2, tap3, tap4, tap5;
+- int input_rotate;
+- int wordmask = r->poolinfo->poolwords - 1;
+- const char *bytes = in;
+- __u32 w;
+-
+- tap1 = r->poolinfo->tap1;
+- tap2 = r->poolinfo->tap2;
+- tap3 = r->poolinfo->tap3;
+- tap4 = r->poolinfo->tap4;
+- tap5 = r->poolinfo->tap5;
+-
+- input_rotate = r->input_rotate;
+- i = r->add_ptr;
+-
+- /* mix one byte at a time to simplify size handling and churn faster */
+- while (nbytes--) {
+- w = rol32(*bytes++, input_rotate);
+- i = (i - 1) & wordmask;
+-
+- /* XOR in the various taps */
+- w ^= r->pool[i];
+- w ^= r->pool[(i + tap1) & wordmask];
+- w ^= r->pool[(i + tap2) & wordmask];
+- w ^= r->pool[(i + tap3) & wordmask];
+- w ^= r->pool[(i + tap4) & wordmask];
+- w ^= r->pool[(i + tap5) & wordmask];
+-
+- /* Mix the result back in with a twist */
+- r->pool[i] = (w >> 3) ^ twist_table[w & 7];
+-
+- /*
+- * Normally, we add 7 bits of rotation to the pool.
+- * At the beginning of the pool, add an extra 7 bits
+- * rotation, so that successive passes spread the
+- * input bits across the pool evenly.
+- */
+- input_rotate = (input_rotate + (i ? 7 : 14)) & 31;
+- }
+-
+- r->input_rotate = input_rotate;
+- r->add_ptr = i;
+-}
++static DEFINE_PER_CPU(struct crng, crngs) = {
++ .generation = ULONG_MAX,
++ .lock = INIT_LOCAL_LOCK(crngs.lock),
++};
+
+-static void __mix_pool_bytes(struct entropy_store *r, const void *in,
+- int nbytes)
+-{
+- trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_);
+- _mix_pool_bytes(r, in, nbytes);
+-}
++/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
++static void extract_entropy(void *buf, size_t len);
+
+-static void mix_pool_bytes(struct entropy_store *r, const void *in,
+- int nbytes)
++/* This extracts a new crng key from the input pool. */
++static void crng_reseed(void)
+ {
+ unsigned long flags;
++ unsigned long next_gen;
++ u8 key[CHACHA_KEY_SIZE];
+
+- trace_mix_pool_bytes(r->name, nbytes, _RET_IP_);
+- spin_lock_irqsave(&r->lock, flags);
+- _mix_pool_bytes(r, in, nbytes);
+- spin_unlock_irqrestore(&r->lock, flags);
+-}
++ extract_entropy(key, sizeof(key));
+
+-struct fast_pool {
+- __u32 pool[4];
+- unsigned long last;
+- unsigned short reg_idx;
+- unsigned char count;
+-};
++ /*
++ * We copy the new key into the base_crng, overwriting the old one,
++ * and update the generation counter. We avoid hitting ULONG_MAX,
++ * because the per-cpu crngs are initialized to ULONG_MAX, so this
++ * forces new CPUs that come online to always initialize.
++ */
++ spin_lock_irqsave(&base_crng.lock, flags);
++ memcpy(base_crng.key, key, sizeof(base_crng.key));
++ next_gen = base_crng.generation + 1;
++ if (next_gen == ULONG_MAX)
++ ++next_gen;
++ WRITE_ONCE(base_crng.generation, next_gen);
++ WRITE_ONCE(base_crng.birth, jiffies);
++ if (!static_branch_likely(&crng_is_ready))
++ crng_init = CRNG_READY;
++ spin_unlock_irqrestore(&base_crng.lock, flags);
++ memzero_explicit(key, sizeof(key));
++}
+
+ /*
+- * This is a fast mixing routine used by the interrupt randomness
+- * collector. It's hardcoded for an 128 bit pool and assumes that any
+- * locks that might be needed are taken by the caller.
++ * This generates a ChaCha block using the provided key, and then
++ * immediately overwites that key with half the block. It returns
++ * the resultant ChaCha state to the user, along with the second
++ * half of the block containing 32 bytes of random data that may
++ * be used; random_data_len may not be greater than 32.
++ *
++ * The returned ChaCha state contains within it a copy of the old
++ * key value, at index 4, so the state should always be zeroed out
++ * immediately after using in order to maintain forward secrecy.
++ * If the state cannot be erased in a timely manner, then it is
++ * safer to set the random_data parameter to &chacha_state[4] so
++ * that this function overwrites it before returning.
+ */
+-static void fast_mix(struct fast_pool *f)
++static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE],
++ u32 chacha_state[CHACHA_STATE_WORDS],
++ u8 *random_data, size_t random_data_len)
+ {
+- __u32 a = f->pool[0], b = f->pool[1];
+- __u32 c = f->pool[2], d = f->pool[3];
+-
+- a += b; c += d;
+- b = rol32(b, 6); d = rol32(d, 27);
+- d ^= a; b ^= c;
++ u8 first_block[CHACHA_BLOCK_SIZE];
+
+- a += b; c += d;
+- b = rol32(b, 16); d = rol32(d, 14);
+- d ^= a; b ^= c;
++ BUG_ON(random_data_len > 32);
+
+- a += b; c += d;
+- b = rol32(b, 6); d = rol32(d, 27);
+- d ^= a; b ^= c;
++ chacha_init_consts(chacha_state);
++ memcpy(&chacha_state[4], key, CHACHA_KEY_SIZE);
++ memset(&chacha_state[12], 0, sizeof(u32) * 4);
++ chacha20_block(chacha_state, first_block);
+
+- a += b; c += d;
+- b = rol32(b, 16); d = rol32(d, 14);
+- d ^= a; b ^= c;
+-
+- f->pool[0] = a; f->pool[1] = b;
+- f->pool[2] = c; f->pool[3] = d;
+- f->count++;
++ memcpy(key, first_block, CHACHA_KEY_SIZE);
++ memcpy(random_data, first_block + CHACHA_KEY_SIZE, random_data_len);
++ memzero_explicit(first_block, sizeof(first_block));
+ }
+
+-static void process_random_ready_list(void)
++/*
++ * Return whether the crng seed is considered to be sufficiently old
++ * that a reseeding is needed. This happens if the last reseeding
++ * was CRNG_RESEED_INTERVAL ago, or during early boot, at an interval
++ * proportional to the uptime.
++ */
++static bool crng_has_old_seed(void)
+ {
+- unsigned long flags;
+- struct random_ready_callback *rdy, *tmp;
+-
+- spin_lock_irqsave(&random_ready_list_lock, flags);
+- list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) {
+- struct module *owner = rdy->owner;
+-
+- list_del_init(&rdy->list);
+- rdy->func(rdy);
+- module_put(owner);
++ static bool early_boot = true;
++ unsigned long interval = CRNG_RESEED_INTERVAL;
++
++ if (unlikely(READ_ONCE(early_boot))) {
++ time64_t uptime = ktime_get_seconds();
++ if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
++ WRITE_ONCE(early_boot, false);
++ else
++ interval = max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
++ (unsigned int)uptime / 2 * HZ);
+ }
+- spin_unlock_irqrestore(&random_ready_list_lock, flags);
++ return time_is_before_jiffies(READ_ONCE(base_crng.birth) + interval);
+ }
+
+ /*
+- * Credit (or debit) the entropy store with n bits of entropy.
+- * Use credit_entropy_bits_safe() if the value comes from userspace
+- * or otherwise should be checked for extreme values.
++ * This function returns a ChaCha state that you may use for generating
++ * random data. It also returns up to 32 bytes on its own of random data
++ * that may be used; random_data_len may not be greater than 32.
+ */
+-static void credit_entropy_bits(struct entropy_store *r, int nbits)
++static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS],
++ u8 *random_data, size_t random_data_len)
+ {
+- int entropy_count, orig;
+- const int pool_size = r->poolinfo->poolfracbits;
+- int nfrac = nbits << ENTROPY_SHIFT;
++ unsigned long flags;
++ struct crng *crng;
+
+- if (!nbits)
+- return;
++ BUG_ON(random_data_len > 32);
+
+-retry:
+- entropy_count = orig = READ_ONCE(r->entropy_count);
+- if (nfrac < 0) {
+- /* Debit */
+- entropy_count += nfrac;
+- } else {
+- /*
+- * Credit: we have to account for the possibility of
+- * overwriting already present entropy. Even in the
+- * ideal case of pure Shannon entropy, new contributions
+- * approach the full value asymptotically:
+- *
+- * entropy <- entropy + (pool_size - entropy) *
+- * (1 - exp(-add_entropy/pool_size))
+- *
+- * For add_entropy <= pool_size/2 then
+- * (1 - exp(-add_entropy/pool_size)) >=
+- * (add_entropy/pool_size)*0.7869...
+- * so we can approximate the exponential with
+- * 3/4*add_entropy/pool_size and still be on the
+- * safe side by adding at most pool_size/2 at a time.
+- *
+- * The use of pool_size-2 in the while statement is to
+- * prevent rounding artifacts from making the loop
+- * arbitrarily long; this limits the loop to log2(pool_size)*2
+- * turns no matter how large nbits is.
+- */
+- int pnfrac = nfrac;
+- const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2;
+- /* The +2 corresponds to the /4 in the denominator */
+-
+- do {
+- unsigned int anfrac = min(pnfrac, pool_size/2);
+- unsigned int add =
+- ((pool_size - entropy_count)*anfrac*3) >> s;
+-
+- entropy_count += add;
+- pnfrac -= anfrac;
+- } while (unlikely(entropy_count < pool_size-2 && pnfrac));
++ /*
++ * For the fast path, we check whether we're ready, unlocked first, and
++ * then re-check once locked later. In the case where we're really not
++ * ready, we do fast key erasure with the base_crng directly, extracting
++ * when crng_init is CRNG_EMPTY.
++ */
++ if (!crng_ready()) {
++ bool ready;
++
++ spin_lock_irqsave(&base_crng.lock, flags);
++ ready = crng_ready();
++ if (!ready) {
++ if (crng_init == CRNG_EMPTY)
++ extract_entropy(base_crng.key, sizeof(base_crng.key));
++ crng_fast_key_erasure(base_crng.key, chacha_state,
++ random_data, random_data_len);
++ }
++ spin_unlock_irqrestore(&base_crng.lock, flags);
++ if (!ready)
++ return;
+ }
+
+- if (WARN_ON(entropy_count < 0)) {
+- pr_warn("negative entropy/overflow: pool %s count %d\n",
+- r->name, entropy_count);
+- entropy_count = 0;
+- } else if (entropy_count > pool_size)
+- entropy_count = pool_size;
+- if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
+- goto retry;
+-
+- trace_credit_entropy_bits(r->name, nbits,
+- entropy_count >> ENTROPY_SHIFT, _RET_IP_);
++ /*
++ * If the base_crng is old enough, we reseed, which in turn bumps the
++ * generation counter that we check below.
++ */
++ if (unlikely(crng_has_old_seed()))
++ crng_reseed();
+
+- if (r == &input_pool) {
+- int entropy_bits = entropy_count >> ENTROPY_SHIFT;
++ local_lock_irqsave(&crngs.lock, flags);
++ crng = raw_cpu_ptr(&crngs);
+
+- if (crng_init < 2 && entropy_bits >= 128)
+- crng_reseed(&primary_crng, r);
++ /*
++ * If our per-cpu crng is older than the base_crng, then it means
++ * somebody reseeded the base_crng. In that case, we do fast key
++ * erasure on the base_crng, and use its output as the new key
++ * for our per-cpu crng. This brings us up to date with base_crng.
++ */
++ if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {
++ spin_lock(&base_crng.lock);
++ crng_fast_key_erasure(base_crng.key, chacha_state,
++ crng->key, sizeof(crng->key));
++ crng->generation = base_crng.generation;
++ spin_unlock(&base_crng.lock);
+ }
++
++ /*
++ * Finally, when we've made it this far, our per-cpu crng has an up
++ * to date key, and we can do fast key erasure with it to produce
++ * some random data and a ChaCha state for the caller. All other
++ * branches of this function are "unlikely", so most of the time we
++ * should wind up here immediately.
++ */
++ crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len);
++ local_unlock_irqrestore(&crngs.lock, flags);
+ }
+
+-static int credit_entropy_bits_safe(struct entropy_store *r, int nbits)
++static void _get_random_bytes(void *buf, size_t len)
+ {
+- const int nbits_max = r->poolinfo->poolwords * 32;
+-
+- if (nbits < 0)
+- return -EINVAL;
++ u32 chacha_state[CHACHA_STATE_WORDS];
++ u8 tmp[CHACHA_BLOCK_SIZE];
++ size_t first_block_len;
+
+- /* Cap the value to avoid overflows */
+- nbits = min(nbits, nbits_max);
++ if (!len)
++ return;
+
+- credit_entropy_bits(r, nbits);
+- return 0;
+-}
++ first_block_len = min_t(size_t, 32, len);
++ crng_make_state(chacha_state, buf, first_block_len);
++ len -= first_block_len;
++ buf += first_block_len;
+
+-/*********************************************************************
+- *
+- * CRNG using CHACHA20
+- *
+- *********************************************************************/
++ while (len) {
++ if (len < CHACHA_BLOCK_SIZE) {
++ chacha20_block(chacha_state, tmp);
++ memcpy(buf, tmp, len);
++ memzero_explicit(tmp, sizeof(tmp));
++ break;
++ }
+
+-#define CRNG_RESEED_INTERVAL (300*HZ)
++ chacha20_block(chacha_state, buf);
++ if (unlikely(chacha_state[12] == 0))
++ ++chacha_state[13];
++ len -= CHACHA_BLOCK_SIZE;
++ buf += CHACHA_BLOCK_SIZE;
++ }
+
+-static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
++ memzero_explicit(chacha_state, sizeof(chacha_state));
++}
+
+-#ifdef CONFIG_NUMA
+ /*
+- * Hack to deal with crazy userspace progams when they are all trying
+- * to access /dev/urandom in parallel. The programs are almost
+- * certainly doing something terribly wrong, but we'll work around
+- * their brain damage.
++ * This function is the exported kernel interface. It returns some
++ * number of good random numbers, suitable for key generation, seeding
++ * TCP sequence numbers, etc. It does not rely on the hardware random
++ * number generator. For random bytes direct from the hardware RNG
++ * (when available), use get_random_bytes_arch(). In order to ensure
++ * that the randomness provided by this function is okay, the function
++ * wait_for_random_bytes() should be called and return 0 at least once
++ * at any point prior.
+ */
+-static struct crng_state **crng_node_pool __read_mostly;
+-#endif
+-
+-static void invalidate_batched_entropy(void);
+-static void numa_crng_init(void);
+-
+-static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
+-static int __init parse_trust_cpu(char *arg)
++void get_random_bytes(void *buf, size_t len)
+ {
+- return kstrtobool(arg, &trust_cpu);
++ warn_unseeded_randomness();
++ _get_random_bytes(buf, len);
+ }
+-early_param("random.trust_cpu", parse_trust_cpu);
++EXPORT_SYMBOL(get_random_bytes);
+
+-static bool crng_init_try_arch(struct crng_state *crng)
++static ssize_t get_random_bytes_user(struct iov_iter *iter)
+ {
+- int i;
+- bool arch_init = true;
+- unsigned long rv;
+-
+- for (i = 4; i < 16; i++) {
+- if (!arch_get_random_seed_long(&rv) &&
+- !arch_get_random_long(&rv)) {
+- rv = random_get_entropy();
+- arch_init = false;
+- }
+- crng->state[i] ^= rv;
++ u32 chacha_state[CHACHA_STATE_WORDS];
++ u8 block[CHACHA_BLOCK_SIZE];
++ size_t ret = 0, copied;
++
++ if (unlikely(!iov_iter_count(iter)))
++ return 0;
++
++ /*
++ * Immediately overwrite the ChaCha key at index 4 with random
++ * bytes, in case userspace causes copy_to_user() below to sleep
++ * forever, so that we still retain forward secrecy in that case.
++ */
++ crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA_KEY_SIZE);
++ /*
++ * However, if we're doing a read of len <= 32, we don't need to
++ * use chacha_state after, so we can simply return those bytes to
++ * the user directly.
++ */
++ if (iov_iter_count(iter) <= CHACHA_KEY_SIZE) {
++ ret = copy_to_iter(&chacha_state[4], CHACHA_KEY_SIZE, iter);
++ goto out_zero_chacha;
+ }
+
+- return arch_init;
+-}
++ for (;;) {
++ chacha20_block(chacha_state, block);
++ if (unlikely(chacha_state[12] == 0))
++ ++chacha_state[13];
+
+-static bool __init crng_init_try_arch_early(struct crng_state *crng)
+-{
+- int i;
+- bool arch_init = true;
+- unsigned long rv;
+-
+- for (i = 4; i < 16; i++) {
+- if (!arch_get_random_seed_long_early(&rv) &&
+- !arch_get_random_long_early(&rv)) {
+- rv = random_get_entropy();
+- arch_init = false;
++ copied = copy_to_iter(block, sizeof(block), iter);
++ ret += copied;
++ if (!iov_iter_count(iter) || copied != sizeof(block))
++ break;
++
++ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
++ if (ret % PAGE_SIZE == 0) {
++ if (signal_pending(current))
++ break;
++ cond_resched();
+ }
+- crng->state[i] ^= rv;
+ }
+
+- return arch_init;
++ memzero_explicit(block, sizeof(block));
++out_zero_chacha:
++ memzero_explicit(chacha_state, sizeof(chacha_state));
++ return ret ? ret : -EFAULT;
+ }
+
+-static void __maybe_unused crng_initialize_secondary(struct crng_state *crng)
+-{
+- chacha_init_consts(crng->state);
+- _get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
+- crng_init_try_arch(crng);
+- crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
+-}
++/*
++ * Batched entropy returns random integers. The quality of the random
++ * number is good as /dev/urandom. In order to ensure that the randomness
++ * provided by this function is okay, the function wait_for_random_bytes()
++ * should be called and return 0 at least once at any point prior.
++ */
+
+-static void __init crng_initialize_primary(struct crng_state *crng)
++#define DEFINE_BATCHED_ENTROPY(type) \
++struct batch_ ##type { \
++ /* \
++ * We make this 1.5x a ChaCha block, so that we get the \
++ * remaining 32 bytes from fast key erasure, plus one full \
++ * block from the detached ChaCha state. We can increase \
++ * the size of this later if needed so long as we keep the \
++ * formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE. \
++ */ \
++ type entropy[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
++ local_lock_t lock; \
++ unsigned long generation; \
++ unsigned int position; \
++}; \
++ \
++static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
++ .lock = INIT_LOCAL_LOCK(batched_entropy_ ##type.lock), \
++ .position = UINT_MAX \
++}; \
++ \
++type get_random_ ##type(void) \
++{ \
++ type ret; \
++ unsigned long flags; \
++ struct batch_ ##type *batch; \
++ unsigned long next_gen; \
++ \
++ warn_unseeded_randomness(); \
++ \
++ if (!crng_ready()) { \
++ _get_random_bytes(&ret, sizeof(ret)); \
++ return ret; \
++ } \
++ \
++ local_lock_irqsave(&batched_entropy_ ##type.lock, flags); \
++ batch = raw_cpu_ptr(&batched_entropy_##type); \
++ \
++ next_gen = READ_ONCE(base_crng.generation); \
++ if (batch->position >= ARRAY_SIZE(batch->entropy) || \
++ next_gen != batch->generation) { \
++ _get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
++ batch->position = 0; \
++ batch->generation = next_gen; \
++ } \
++ \
++ ret = batch->entropy[batch->position]; \
++ batch->entropy[batch->position] = 0; \
++ ++batch->position; \
++ local_unlock_irqrestore(&batched_entropy_ ##type.lock, flags); \
++ return ret; \
++} \
++EXPORT_SYMBOL(get_random_ ##type);
++
++DEFINE_BATCHED_ENTROPY(u64)
++DEFINE_BATCHED_ENTROPY(u32)
++
++#ifdef CONFIG_SMP
++/*
++ * This function is called when the CPU is coming up, with entry
++ * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
++ */
++int __cold random_prepare_cpu(unsigned int cpu)
+ {
+- chacha_init_consts(crng->state);
+- _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0);
+- if (crng_init_try_arch_early(crng) && trust_cpu) {
+- invalidate_batched_entropy();
+- numa_crng_init();
+- crng_init = 2;
+- pr_notice("crng done (trusting CPU's manufacturer)\n");
+- }
+- crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
++ /*
++ * When the cpu comes back online, immediately invalidate both
++ * the per-cpu crng and all batches, so that we serve fresh
++ * randomness.
++ */
++ per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;
++ per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;
++ per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;
++ return 0;
+ }
++#endif
+
+-#ifdef CONFIG_NUMA
+-static void do_numa_crng_init(struct work_struct *work)
++/*
++ * This function will use the architecture-specific hardware random
++ * number generator if it is available. It is not recommended for
++ * use. Use get_random_bytes() instead. It returns the number of
++ * bytes filled in.
++ */
++size_t __must_check get_random_bytes_arch(void *buf, size_t len)
+ {
+- int i;
+- struct crng_state *crng;
+- struct crng_state **pool;
+-
+- pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL);
+- for_each_online_node(i) {
+- crng = kmalloc_node(sizeof(struct crng_state),
+- GFP_KERNEL | __GFP_NOFAIL, i);
+- spin_lock_init(&crng->lock);
+- crng_initialize_secondary(crng);
+- pool[i] = crng;
+- }
+- mb();
+- if (cmpxchg(&crng_node_pool, NULL, pool)) {
+- for_each_node(i)
+- kfree(pool[i]);
+- kfree(pool);
++ size_t left = len;
++ u8 *p = buf;
++
++ while (left) {
++ unsigned long v;
++ size_t block_len = min_t(size_t, left, sizeof(unsigned long));
++
++ if (!arch_get_random_long(&v))
++ break;
++
++ memcpy(p, &v, block_len);
++ p += block_len;
++ left -= block_len;
+ }
++
++ return len - left;
+ }
++EXPORT_SYMBOL(get_random_bytes_arch);
++
++
++/**********************************************************************
++ *
++ * Entropy accumulation and extraction routines.
++ *
++ * Callers may add entropy via:
++ *
++ * static void mix_pool_bytes(const void *buf, size_t len)
++ *
++ * After which, if added entropy should be credited:
++ *
++ * static void credit_init_bits(size_t bits)
++ *
++ * Finally, extract entropy via:
++ *
++ * static void extract_entropy(void *buf, size_t len)
++ *
++ **********************************************************************/
+
+-static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init);
++enum {
++ POOL_BITS = BLAKE2S_HASH_SIZE * 8,
++ POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
++ POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
++};
+
+-static void numa_crng_init(void)
++static struct {
++ struct blake2s_state hash;
++ spinlock_t lock;
++ unsigned int init_bits;
++} input_pool = {
++ .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
++ BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
++ BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 },
++ .hash.outlen = BLAKE2S_HASH_SIZE,
++ .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
++};
++
++static void _mix_pool_bytes(const void *buf, size_t len)
+ {
+- schedule_work(&numa_crng_init_work);
++ blake2s_update(&input_pool.hash, buf, len);
+ }
+-#else
+-static void numa_crng_init(void) {}
+-#endif
+
+ /*
+- * crng_fast_load() can be called by code in the interrupt service
+- * path. So we can't afford to dilly-dally.
++ * This function adds bytes into the input pool. It does not
++ * update the initialization bit counter; the caller should call
++ * credit_init_bits if this is appropriate.
+ */
+-static int crng_fast_load(const char *cp, size_t len)
++static void mix_pool_bytes(const void *buf, size_t len)
+ {
+ unsigned long flags;
+- char *p;
+
+- if (!spin_trylock_irqsave(&primary_crng.lock, flags))
+- return 0;
+- if (crng_init != 0) {
+- spin_unlock_irqrestore(&primary_crng.lock, flags);
+- return 0;
+- }
+- p = (unsigned char *) &primary_crng.state[4];
+- while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
+- p[crng_init_cnt % CHACHA_KEY_SIZE] ^= *cp;
+- cp++; crng_init_cnt++; len--;
+- }
+- spin_unlock_irqrestore(&primary_crng.lock, flags);
+- if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
+- invalidate_batched_entropy();
+- crng_init = 1;
+- pr_notice("fast init done\n");
+- }
+- return 1;
++ spin_lock_irqsave(&input_pool.lock, flags);
++ _mix_pool_bytes(buf, len);
++ spin_unlock_irqrestore(&input_pool.lock, flags);
+ }
+
+ /*
+- * crng_slow_load() is called by add_device_randomness, which has two
+- * attributes. (1) We can't trust the buffer passed to it is
+- * guaranteed to be unpredictable (so it might not have any entropy at
+- * all), and (2) it doesn't have the performance constraints of
+- * crng_fast_load().
+- *
+- * So we do something more comprehensive which is guaranteed to touch
+- * all of the primary_crng's state, and which uses a LFSR with a
+- * period of 255 as part of the mixing algorithm. Finally, we do
+- * *not* advance crng_init_cnt since buffer we may get may be something
+- * like a fixed DMI table (for example), which might very well be
+- * unique to the machine, but is otherwise unvarying.
++ * This is an HKDF-like construction for using the hashed collected entropy
++ * as a PRF key, that's then expanded block-by-block.
+ */
+-static int crng_slow_load(const char *cp, size_t len)
++static void extract_entropy(void *buf, size_t len)
+ {
+- unsigned long flags;
+- static unsigned char lfsr = 1;
+- unsigned char tmp;
+- unsigned i, max = CHACHA_KEY_SIZE;
+- const char * src_buf = cp;
+- char * dest_buf = (char *) &primary_crng.state[4];
+-
+- if (!spin_trylock_irqsave(&primary_crng.lock, flags))
+- return 0;
+- if (crng_init != 0) {
+- spin_unlock_irqrestore(&primary_crng.lock, flags);
+- return 0;
++ unsigned long flags;
++ u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
++ struct {
++ unsigned long rdseed[32 / sizeof(long)];
++ size_t counter;
++ } block;
++ size_t i;
++
++ for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) {
++ if (!arch_get_random_seed_long(&block.rdseed[i]) &&
++ !arch_get_random_long(&block.rdseed[i]))
++ block.rdseed[i] = random_get_entropy();
+ }
+- if (len > max)
+- max = len;
+-
+- for (i = 0; i < max ; i++) {
+- tmp = lfsr;
+- lfsr >>= 1;
+- if (tmp & 1)
+- lfsr ^= 0xE1;
+- tmp = dest_buf[i % CHACHA_KEY_SIZE];
+- dest_buf[i % CHACHA_KEY_SIZE] ^= src_buf[i % len] ^ lfsr;
+- lfsr += (tmp << 3) | (tmp >> 5);
++
++ spin_lock_irqsave(&input_pool.lock, flags);
++
++ /* seed = HASHPRF(last_key, entropy_input) */
++ blake2s_final(&input_pool.hash, seed);
++
++ /* next_key = HASHPRF(seed, RDSEED || 0) */
++ block.counter = 0;
++ blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed));
++ blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key));
++
++ spin_unlock_irqrestore(&input_pool.lock, flags);
++ memzero_explicit(next_key, sizeof(next_key));
++
++ while (len) {
++ i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
++ /* output = HASHPRF(seed, RDSEED || ++counter) */
++ ++block.counter;
++ blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed));
++ len -= i;
++ buf += i;
+ }
+- spin_unlock_irqrestore(&primary_crng.lock, flags);
+- return 1;
++
++ memzero_explicit(seed, sizeof(seed));
++ memzero_explicit(&block, sizeof(block));
+ }
+
+-static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
++#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
++
++static void __cold _credit_init_bits(size_t bits)
+ {
+- unsigned long flags;
+- int i, num;
+- union {
+- __u8 block[CHACHA_BLOCK_SIZE];
+- __u32 key[8];
+- } buf;
+-
+- if (r) {
+- num = extract_entropy(r, &buf, 32, 16, 0);
+- if (num == 0)
+- return;
+- } else {
+- _extract_crng(&primary_crng, buf.block);
+- _crng_backtrack_protect(&primary_crng, buf.block,
+- CHACHA_KEY_SIZE);
+- }
+- spin_lock_irqsave(&crng->lock, flags);
+- for (i = 0; i < 8; i++) {
+- unsigned long rv;
+- if (!arch_get_random_seed_long(&rv) &&
+- !arch_get_random_long(&rv))
+- rv = random_get_entropy();
+- crng->state[i+4] ^= buf.key[i] ^ rv;
+- }
+- memzero_explicit(&buf, sizeof(buf));
+- crng->init_time = jiffies;
+- spin_unlock_irqrestore(&crng->lock, flags);
+- if (crng == &primary_crng && crng_init < 2) {
+- invalidate_batched_entropy();
+- numa_crng_init();
+- crng_init = 2;
++ static struct execute_work set_ready;
++ unsigned int new, orig, add;
++ unsigned long flags;
++
++ if (!bits)
++ return;
++
++ add = min_t(size_t, bits, POOL_BITS);
++
++ do {
++ orig = READ_ONCE(input_pool.init_bits);
++ new = min_t(unsigned int, POOL_BITS, orig + add);
++ } while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
++
++ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
++ crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
++ execute_in_process_context(crng_set_ready, &set_ready);
+ process_random_ready_list();
+ wake_up_interruptible(&crng_init_wait);
+ kill_fasync(&fasync, SIGIO, POLL_IN);
+ pr_notice("crng init done\n");
+- if (unseeded_warning.missed) {
+- pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
+- unseeded_warning.missed);
+- unseeded_warning.missed = 0;
+- }
+- if (urandom_warning.missed) {
++ if (urandom_warning.missed)
+ pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
+ urandom_warning.missed);
+- urandom_warning.missed = 0;
++ } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
++ spin_lock_irqsave(&base_crng.lock, flags);
++ /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
++ if (crng_init == CRNG_EMPTY) {
++ extract_entropy(base_crng.key, sizeof(base_crng.key));
++ crng_init = CRNG_EARLY;
+ }
++ spin_unlock_irqrestore(&base_crng.lock, flags);
+ }
+ }
+
+-static void _extract_crng(struct crng_state *crng,
+- __u8 out[CHACHA_BLOCK_SIZE])
++
++/**********************************************************************
++ *
++ * Entropy collection routines.
++ *
++ * The following exported functions are used for pushing entropy into
++ * the above entropy accumulation routines:
++ *
++ * void add_device_randomness(const void *buf, size_t len);
++ * void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
++ * void add_bootloader_randomness(const void *buf, size_t len);
++ * void add_interrupt_randomness(int irq);
++ * void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
++ * void add_disk_randomness(struct gendisk *disk);
++ *
++ * add_device_randomness() adds data to the input pool that
++ * is likely to differ between two devices (or possibly even per boot).
++ * This would be things like MAC addresses or serial numbers, or the
++ * read-out of the RTC. This does *not* credit any actual entropy to
++ * the pool, but it initializes the pool to different values for devices
++ * that might otherwise be identical and have very little entropy
++ * available to them (particularly common in the embedded world).
++ *
++ * add_hwgenerator_randomness() is for true hardware RNGs, and will credit
++ * entropy as specified by the caller. If the entropy pool is full it will
++ * block until more entropy is needed.
++ *
++ * add_bootloader_randomness() is called by bootloader drivers, such as EFI
++ * and device tree, and credits its input depending on whether or not the
++ * configuration option CONFIG_RANDOM_TRUST_BOOTLOADER is set.
++ *
++ * add_interrupt_randomness() uses the interrupt timing as random
++ * inputs to the entropy pool. Using the cycle counters and the irq source
++ * as inputs, it feeds the input pool roughly once a second or after 64
++ * interrupts, crediting 1 bit of entropy for whichever comes first.
++ *
++ * add_input_randomness() uses the input layer interrupt timing, as well
++ * as the event type information from the hardware.
++ *
++ * add_disk_randomness() uses what amounts to the seek time of block
++ * layer request events, on a per-disk_devt basis, as input to the
++ * entropy pool. Note that high-speed solid state drives with very low
++ * seek times do not make for good sources of entropy, as their seek
++ * times are usually fairly consistent.
++ *
++ * The last two routines try to estimate how many bits of entropy
++ * to credit. They do this by keeping track of the first and second
++ * order deltas of the event timings.
++ *
++ **********************************************************************/
++
++static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
++static bool trust_bootloader __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER);
++static int __init parse_trust_cpu(char *arg)
+ {
+- unsigned long v, flags;
+-
+- if (crng_ready() &&
+- (time_after(crng_global_init_time, crng->init_time) ||
+- time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL)))
+- crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL);
+- spin_lock_irqsave(&crng->lock, flags);
+- if (arch_get_random_long(&v))
+- crng->state[14] ^= v;
+- chacha20_block(&crng->state[0], out);
+- if (crng->state[12] == 0)
+- crng->state[13]++;
+- spin_unlock_irqrestore(&crng->lock, flags);
++ return kstrtobool(arg, &trust_cpu);
+ }
+-
+-static void extract_crng(__u8 out[CHACHA_BLOCK_SIZE])
++static int __init parse_trust_bootloader(char *arg)
+ {
+- struct crng_state *crng = NULL;
+-
+-#ifdef CONFIG_NUMA
+- if (crng_node_pool)
+- crng = crng_node_pool[numa_node_id()];
+- if (crng == NULL)
+-#endif
+- crng = &primary_crng;
+- _extract_crng(crng, out);
++ return kstrtobool(arg, &trust_bootloader);
+ }
++early_param("random.trust_cpu", parse_trust_cpu);
++early_param("random.trust_bootloader", parse_trust_bootloader);
+
+ /*
+- * Use the leftover bytes from the CRNG block output (if there is
+- * enough) to mutate the CRNG key to provide backtracking protection.
++ * The first collection of entropy occurs at system boot while interrupts
++ * are still turned off. Here we push in latent entropy, RDSEED, a timestamp,
++ * utsname(), and the command line. Depending on the above configuration knob,
++ * RDSEED may be considered sufficient for initialization. Note that much
++ * earlier setup may already have pushed entropy into the input pool by the
++ * time we get here.
+ */
+-static void _crng_backtrack_protect(struct crng_state *crng,
+- __u8 tmp[CHACHA_BLOCK_SIZE], int used)
+-{
+- unsigned long flags;
+- __u32 *s, *d;
+- int i;
+-
+- used = round_up(used, sizeof(__u32));
+- if (used + CHACHA_KEY_SIZE > CHACHA_BLOCK_SIZE) {
+- extract_crng(tmp);
+- used = 0;
+- }
+- spin_lock_irqsave(&crng->lock, flags);
+- s = (__u32 *) &tmp[used];
+- d = &crng->state[4];
+- for (i=0; i < 8; i++)
+- *d++ ^= *s++;
+- spin_unlock_irqrestore(&crng->lock, flags);
+-}
+-
+-static void crng_backtrack_protect(__u8 tmp[CHACHA_BLOCK_SIZE], int used)
++int __init random_init(const char *command_line)
+ {
+- struct crng_state *crng = NULL;
++ ktime_t now = ktime_get_real();
++ unsigned int i, arch_bytes;
++ unsigned long entropy;
+
+-#ifdef CONFIG_NUMA
+- if (crng_node_pool)
+- crng = crng_node_pool[numa_node_id()];
+- if (crng == NULL)
++#if defined(LATENT_ENTROPY_PLUGIN)
++ static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
++ _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
+ #endif
+- crng = &primary_crng;
+- _crng_backtrack_protect(crng, tmp, used);
+-}
+
+-static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
+-{
+- ssize_t ret = 0, i = CHACHA_BLOCK_SIZE;
+- __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
+- int large_request = (nbytes > 256);
+-
+- while (nbytes) {
+- if (large_request && need_resched()) {
+- if (signal_pending(current)) {
+- if (ret == 0)
+- ret = -ERESTARTSYS;
+- break;
+- }
+- schedule();
++ for (i = 0, arch_bytes = BLAKE2S_BLOCK_SIZE;
++ i < BLAKE2S_BLOCK_SIZE; i += sizeof(entropy)) {
++ if (!arch_get_random_seed_long_early(&entropy) &&
++ !arch_get_random_long_early(&entropy)) {
++ entropy = random_get_entropy();
++ arch_bytes -= sizeof(entropy);
+ }
+-
+- extract_crng(tmp);
+- i = min_t(int, nbytes, CHACHA_BLOCK_SIZE);
+- if (copy_to_user(buf, tmp, i)) {
+- ret = -EFAULT;
+- break;
+- }
+-
+- nbytes -= i;
+- buf += i;
+- ret += i;
++ _mix_pool_bytes(&entropy, sizeof(entropy));
+ }
+- crng_backtrack_protect(tmp, i);
++ _mix_pool_bytes(&now, sizeof(now));
++ _mix_pool_bytes(utsname(), sizeof(*(utsname())));
++ _mix_pool_bytes(command_line, strlen(command_line));
++ add_latent_entropy();
+
+- /* Wipe data just written to memory */
+- memzero_explicit(tmp, sizeof(tmp));
++ if (crng_ready())
++ crng_reseed();
++ else if (trust_cpu)
++ credit_init_bits(arch_bytes * 8);
+
+- return ret;
++ return 0;
+ }
+
+-
+-/*********************************************************************
+- *
+- * Entropy input management
+- *
+- *********************************************************************/
+-
+-/* There is one of these per entropy source */
+-struct timer_rand_state {
+- cycles_t last_time;
+- long last_delta, last_delta2;
+-};
+-
+-#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
+-
+ /*
+ * Add device- or boot-specific data to the input pool to help
+ * initialize it.
+@@ -1111,681 +861,271 @@ struct timer_rand_state {
+ * the entropy pool having similar initial state across largely
+ * identical devices.
+ */
+-void add_device_randomness(const void *buf, unsigned int size)
++void add_device_randomness(const void *buf, size_t len)
+ {
+- unsigned long time = random_get_entropy() ^ jiffies;
++ unsigned long entropy = random_get_entropy();
+ unsigned long flags;
+
+- if (!crng_ready() && size)
+- crng_slow_load(buf, size);
+-
+- trace_add_device_randomness(size, _RET_IP_);
+ spin_lock_irqsave(&input_pool.lock, flags);
+- _mix_pool_bytes(&input_pool, buf, size);
+- _mix_pool_bytes(&input_pool, &time, sizeof(time));
++ _mix_pool_bytes(&entropy, sizeof(entropy));
++ _mix_pool_bytes(buf, len);
+ spin_unlock_irqrestore(&input_pool.lock, flags);
+ }
+ EXPORT_SYMBOL(add_device_randomness);
+
+-static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE;
+-
+ /*
+- * This function adds entropy to the entropy "pool" by using timing
+- * delays. It uses the timer_rand_state structure to make an estimate
+- * of how many bits of entropy this call has added to the pool.
+- *
+- * The number "num" is also added to the pool - it should somehow describe
+- * the type of event which just happened. This is currently 0-255 for
+- * keyboard scan codes, and 256 upwards for interrupts.
+- *
++ * Interface for in-kernel drivers of true hardware RNGs.
++ * Those devices may produce endless random bits and will be throttled
++ * when our pool is full.
+ */
+-static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
++void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy)
+ {
+- struct entropy_store *r;
+- struct {
+- long jiffies;
+- unsigned cycles;
+- unsigned num;
+- } sample;
+- long delta, delta2, delta3;
+-
+- sample.jiffies = jiffies;
+- sample.cycles = random_get_entropy();
+- sample.num = num;
+- r = &input_pool;
+- mix_pool_bytes(r, &sample, sizeof(sample));
++ mix_pool_bytes(buf, len);
++ credit_init_bits(entropy);
+
+ /*
+- * Calculate number of bits of randomness we probably added.
+- * We take into account the first, second and third-order deltas
+- * in order to make our estimate.
++ * Throttle writing to once every CRNG_RESEED_INTERVAL, unless
++ * we're not yet initialized.
+ */
+- delta = sample.jiffies - READ_ONCE(state->last_time);
+- WRITE_ONCE(state->last_time, sample.jiffies);
+-
+- delta2 = delta - READ_ONCE(state->last_delta);
+- WRITE_ONCE(state->last_delta, delta);
+-
+- delta3 = delta2 - READ_ONCE(state->last_delta2);
+- WRITE_ONCE(state->last_delta2, delta2);
+-
+- if (delta < 0)
+- delta = -delta;
+- if (delta2 < 0)
+- delta2 = -delta2;
+- if (delta3 < 0)
+- delta3 = -delta3;
+- if (delta > delta2)
+- delta = delta2;
+- if (delta > delta3)
+- delta = delta3;
+-
+- /*
+- * delta is now minimum absolute delta.
+- * Round down by 1 bit on general principles,
+- * and limit entropy estimate to 12 bits.
+- */
+- credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
++ if (!kthread_should_stop() && crng_ready())
++ schedule_timeout_interruptible(CRNG_RESEED_INTERVAL);
+ }
++EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
+
+-void add_input_randomness(unsigned int type, unsigned int code,
+- unsigned int value)
++/*
++ * Handle random seed passed by bootloader, and credit it if
++ * CONFIG_RANDOM_TRUST_BOOTLOADER is set.
++ */
++void __cold add_bootloader_randomness(const void *buf, size_t len)
+ {
+- static unsigned char last_value;
+-
+- /* ignore autorepeat and the like */
+- if (value == last_value)
+- return;
+-
+- last_value = value;
+- add_timer_randomness(&input_timer_state,
+- (type << 4) ^ code ^ (code >> 4) ^ value);
+- trace_add_input_randomness(ENTROPY_BITS(&input_pool));
++ mix_pool_bytes(buf, len);
++ if (trust_bootloader)
++ credit_init_bits(len * 8);
+ }
+-EXPORT_SYMBOL_GPL(add_input_randomness);
+-
+-static DEFINE_PER_CPU(struct fast_pool, irq_randomness);
+-
+-#ifdef ADD_INTERRUPT_BENCH
+-static unsigned long avg_cycles, avg_deviation;
++EXPORT_SYMBOL_GPL(add_bootloader_randomness);
+
+-#define AVG_SHIFT 8 /* Exponential average factor k=1/256 */
+-#define FIXED_1_2 (1 << (AVG_SHIFT-1))
++struct fast_pool {
++ struct work_struct mix;
++ unsigned long pool[4];
++ unsigned long last;
++ unsigned int count;
++};
+
+-static void add_interrupt_bench(cycles_t start)
+-{
+- long delta = random_get_entropy() - start;
+-
+- /* Use a weighted moving average */
+- delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT);
+- avg_cycles += delta;
+- /* And average deviation */
+- delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT);
+- avg_deviation += delta;
+-}
++static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
++#ifdef CONFIG_64BIT
++#define FASTMIX_PERM SIPHASH_PERMUTATION
++ .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 }
+ #else
+-#define add_interrupt_bench(x)
++#define FASTMIX_PERM HSIPHASH_PERMUTATION
++ .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 }
+ #endif
+-
+-static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs)
+-{
+- __u32 *ptr = (__u32 *) regs;
+- unsigned int idx;
+-
+- if (regs == NULL)
+- return 0;
+- idx = READ_ONCE(f->reg_idx);
+- if (idx >= sizeof(struct pt_regs) / sizeof(__u32))
+- idx = 0;
+- ptr += idx++;
+- WRITE_ONCE(f->reg_idx, idx);
+- return *ptr;
+-}
+-
+-void add_interrupt_randomness(int irq, int irq_flags)
+-{
+- struct entropy_store *r;
+- struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
+- struct pt_regs *regs = get_irq_regs();
+- unsigned long now = jiffies;
+- cycles_t cycles = random_get_entropy();
+- __u32 c_high, j_high;
+- __u64 ip;
+-
+- if (cycles == 0)
+- cycles = get_reg(fast_pool, regs);
+- c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
+- j_high = (sizeof(now) > 4) ? now >> 32 : 0;
+- fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
+- fast_pool->pool[1] ^= now ^ c_high;
+- ip = regs ? instruction_pointer(regs) : _RET_IP_;
+- fast_pool->pool[2] ^= ip;
+- fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
+- get_reg(fast_pool, regs);
+-
+- fast_mix(fast_pool);
+- add_interrupt_bench(cycles);
+-
+- if (unlikely(crng_init == 0)) {
+- if ((fast_pool->count >= 64) &&
+- crng_fast_load((char *) fast_pool->pool,
+- sizeof(fast_pool->pool))) {
+- fast_pool->count = 0;
+- fast_pool->last = now;
+- }
+- return;
+- }
+-
+- if ((fast_pool->count < 64) &&
+- !time_after(now, fast_pool->last + HZ))
+- return;
+-
+- r = &input_pool;
+- if (!spin_trylock(&r->lock))
+- return;
+-
+- fast_pool->last = now;
+- __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
+- spin_unlock(&r->lock);
+-
+- fast_pool->count = 0;
+-
+- /* award one bit for the contents of the fast pool */
+- credit_entropy_bits(r, 1);
+-}
+-EXPORT_SYMBOL_GPL(add_interrupt_randomness);
+-
+-#ifdef CONFIG_BLOCK
+-void add_disk_randomness(struct gendisk *disk)
+-{
+- if (!disk || !disk->random)
+- return;
+- /* first major is 1, so we get >= 0x200 here */
+- add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
+- trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool));
+-}
+-EXPORT_SYMBOL_GPL(add_disk_randomness);
+-#endif
+-
+-/*********************************************************************
+- *
+- * Entropy extraction routines
+- *
+- *********************************************************************/
++};
+
+ /*
+- * This function decides how many bytes to actually take from the
+- * given pool, and also debits the entropy count accordingly.
++ * This is [Half]SipHash-1-x, starting from an empty key. Because
++ * the key is fixed, it assumes that its inputs are non-malicious,
++ * and therefore this has no security on its own. s represents the
++ * four-word SipHash state, while v represents a two-word input.
+ */
+-static size_t account(struct entropy_store *r, size_t nbytes, int min,
+- int reserved)
++static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
+ {
+- int entropy_count, orig, have_bytes;
+- size_t ibytes, nfrac;
+-
+- BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
+-
+- /* Can we pull enough? */
+-retry:
+- entropy_count = orig = READ_ONCE(r->entropy_count);
+- ibytes = nbytes;
+- /* never pull more than available */
+- have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
+-
+- if ((have_bytes -= reserved) < 0)
+- have_bytes = 0;
+- ibytes = min_t(size_t, ibytes, have_bytes);
+- if (ibytes < min)
+- ibytes = 0;
+-
+- if (WARN_ON(entropy_count < 0)) {
+- pr_warn("negative entropy count: pool %s count %d\n",
+- r->name, entropy_count);
+- entropy_count = 0;
+- }
+- nfrac = ibytes << (ENTROPY_SHIFT + 3);
+- if ((size_t) entropy_count > nfrac)
+- entropy_count -= nfrac;
+- else
+- entropy_count = 0;
+-
+- if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
+- goto retry;
+-
+- trace_debit_entropy(r->name, 8 * ibytes);
+- if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) {
+- wake_up_interruptible(&random_write_wait);
+- kill_fasync(&fasync, SIGIO, POLL_OUT);
+- }
+-
+- return ibytes;
++ s[3] ^= v1;
++ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
++ s[0] ^= v1;
++ s[3] ^= v2;
++ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
++ s[0] ^= v2;
+ }
+
++#ifdef CONFIG_SMP
+ /*
+- * This function does the actual extraction for extract_entropy.
+- *
+- * Note: we assume that .poolwords is a multiple of 16 words.
++ * This function is called when the CPU has just come online, with
++ * entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
+ */
+-static void extract_buf(struct entropy_store *r, __u8 *out)
++int __cold random_online_cpu(unsigned int cpu)
+ {
+- int i;
+- union {
+- __u32 w[5];
+- unsigned long l[LONGS(20)];
+- } hash;
+- __u32 workspace[SHA1_WORKSPACE_WORDS];
+- unsigned long flags;
+-
+ /*
+- * If we have an architectural hardware random number
+- * generator, use it for SHA's initial vector
++ * During CPU shutdown and before CPU onlining, add_interrupt_
++ * randomness() may schedule mix_interrupt_randomness(), and
++ * set the MIX_INFLIGHT flag. However, because the worker can
++ * be scheduled on a different CPU during this period, that
++ * flag will never be cleared. For that reason, we zero out
++ * the flag here, which runs just after workqueues are onlined
++ * for the CPU again. This also has the effect of setting the
++ * irq randomness count to zero so that new accumulated irqs
++ * are fresh.
+ */
+- sha1_init(hash.w);
+- for (i = 0; i < LONGS(20); i++) {
+- unsigned long v;
+- if (!arch_get_random_long(&v))
+- break;
+- hash.l[i] = v;
+- }
+-
+- /* Generate a hash across the pool, 16 words (512 bits) at a time */
+- spin_lock_irqsave(&r->lock, flags);
+- for (i = 0; i < r->poolinfo->poolwords; i += 16)
+- sha1_transform(hash.w, (__u8 *)(r->pool + i), workspace);
++ per_cpu_ptr(&irq_randomness, cpu)->count = 0;
++ return 0;
++}
++#endif
+
++static void mix_interrupt_randomness(struct work_struct *work)
++{
++ struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
+ /*
+- * We mix the hash back into the pool to prevent backtracking
+- * attacks (where the attacker knows the state of the pool
+- * plus the current outputs, and attempts to find previous
+- * ouputs), unless the hash function can be inverted. By
+- * mixing at least a SHA1 worth of hash data back, we make
+- * brute-forcing the feedback as hard as brute-forcing the
+- * hash.
++ * The size of the copied stack pool is explicitly 2 longs so that we
++ * only ever ingest half of the siphash output each time, retaining
++ * the other half as the next "key" that carries over. The entropy is
++ * supposed to be sufficiently dispersed between bits so on average
++ * we don't wind up "losing" some.
+ */
+- __mix_pool_bytes(r, hash.w, sizeof(hash.w));
+- spin_unlock_irqrestore(&r->lock, flags);
++ unsigned long pool[2];
++ unsigned int count;
+
+- memzero_explicit(workspace, sizeof(workspace));
++ /* Check to see if we're running on the wrong CPU due to hotplug. */
++ local_irq_disable();
++ if (fast_pool != this_cpu_ptr(&irq_randomness)) {
++ local_irq_enable();
++ return;
++ }
+
+ /*
+- * In case the hash function has some recognizable output
+- * pattern, we fold it in half. Thus, we always feed back
+- * twice as much data as we output.
++ * Copy the pool to the stack so that the mixer always has a
++ * consistent view, before we reenable irqs again.
+ */
+- hash.w[0] ^= hash.w[3];
+- hash.w[1] ^= hash.w[4];
+- hash.w[2] ^= rol32(hash.w[2], 16);
+-
+- memcpy(out, &hash, EXTRACT_SIZE);
+- memzero_explicit(&hash, sizeof(hash));
+-}
+-
+-static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
+- size_t nbytes, int fips)
+-{
+- ssize_t ret = 0, i;
+- __u8 tmp[EXTRACT_SIZE];
+- unsigned long flags;
+-
+- while (nbytes) {
+- extract_buf(r, tmp);
+-
+- if (fips) {
+- spin_lock_irqsave(&r->lock, flags);
+- if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
+- panic("Hardware RNG duplicated output!\n");
+- memcpy(r->last_data, tmp, EXTRACT_SIZE);
+- spin_unlock_irqrestore(&r->lock, flags);
+- }
+- i = min_t(int, nbytes, EXTRACT_SIZE);
+- memcpy(buf, tmp, i);
+- nbytes -= i;
+- buf += i;
+- ret += i;
+- }
++ memcpy(pool, fast_pool->pool, sizeof(pool));
++ count = fast_pool->count;
++ fast_pool->count = 0;
++ fast_pool->last = jiffies;
++ local_irq_enable();
+
+- /* Wipe data just returned from memory */
+- memzero_explicit(tmp, sizeof(tmp));
++ mix_pool_bytes(pool, sizeof(pool));
++ credit_init_bits(max(1u, (count & U16_MAX) / 64));
+
+- return ret;
++ memzero_explicit(pool, sizeof(pool));
+ }
+
+-/*
+- * This function extracts randomness from the "entropy pool", and
+- * returns it in a buffer.
+- *
+- * The min parameter specifies the minimum amount we can pull before
+- * failing to avoid races that defeat catastrophic reseeding while the
+- * reserved parameter indicates how much entropy we must leave in the
+- * pool after each pull to avoid starving other readers.
+- */
+-static ssize_t extract_entropy(struct entropy_store *r, void *buf,
+- size_t nbytes, int min, int reserved)
++void add_interrupt_randomness(int irq)
+ {
+- __u8 tmp[EXTRACT_SIZE];
+- unsigned long flags;
++ enum { MIX_INFLIGHT = 1U << 31 };
++ unsigned long entropy = random_get_entropy();
++ struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
++ struct pt_regs *regs = get_irq_regs();
++ unsigned int new_count;
+
+- /* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */
+- if (fips_enabled) {
+- spin_lock_irqsave(&r->lock, flags);
+- if (!r->last_data_init) {
+- r->last_data_init = 1;
+- spin_unlock_irqrestore(&r->lock, flags);
+- trace_extract_entropy(r->name, EXTRACT_SIZE,
+- ENTROPY_BITS(r), _RET_IP_);
+- extract_buf(r, tmp);
+- spin_lock_irqsave(&r->lock, flags);
+- memcpy(r->last_data, tmp, EXTRACT_SIZE);
+- }
+- spin_unlock_irqrestore(&r->lock, flags);
+- }
+-
+- trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
+- nbytes = account(r, nbytes, min, reserved);
+-
+- return _extract_entropy(r, buf, nbytes, fips_enabled);
+-}
++ fast_mix(fast_pool->pool, entropy,
++ (regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(irq));
++ new_count = ++fast_pool->count;
+
+-#define warn_unseeded_randomness(previous) \
+- _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))
+-
+-static void _warn_unseeded_randomness(const char *func_name, void *caller,
+- void **previous)
+-{
+-#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
+- const bool print_once = false;
+-#else
+- static bool print_once __read_mostly;
+-#endif
+-
+- if (print_once ||
+- crng_ready() ||
+- (previous && (caller == READ_ONCE(*previous))))
++ if (new_count & MIX_INFLIGHT)
+ return;
+- WRITE_ONCE(*previous, caller);
+-#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
+- print_once = true;
+-#endif
+- if (__ratelimit(&unseeded_warning))
+- printk_deferred(KERN_NOTICE "random: %s called from %pS "
+- "with crng_init=%d\n", func_name, caller,
+- crng_init);
+-}
+
+-/*
+- * This function is the exported kernel interface. It returns some
+- * number of good random numbers, suitable for key generation, seeding
+- * TCP sequence numbers, etc. It does not rely on the hardware random
+- * number generator. For random bytes direct from the hardware RNG
+- * (when available), use get_random_bytes_arch(). In order to ensure
+- * that the randomness provided by this function is okay, the function
+- * wait_for_random_bytes() should be called and return 0 at least once
+- * at any point prior.
+- */
+-static void _get_random_bytes(void *buf, int nbytes)
+-{
+- __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
+-
+- trace_get_random_bytes(nbytes, _RET_IP_);
+-
+- while (nbytes >= CHACHA_BLOCK_SIZE) {
+- extract_crng(buf);
+- buf += CHACHA_BLOCK_SIZE;
+- nbytes -= CHACHA_BLOCK_SIZE;
+- }
+-
+- if (nbytes > 0) {
+- extract_crng(tmp);
+- memcpy(buf, tmp, nbytes);
+- crng_backtrack_protect(tmp, nbytes);
+- } else
+- crng_backtrack_protect(tmp, CHACHA_BLOCK_SIZE);
+- memzero_explicit(tmp, sizeof(tmp));
+-}
+-
+-void get_random_bytes(void *buf, int nbytes)
+-{
+- static void *previous;
+-
+- warn_unseeded_randomness(&previous);
+- _get_random_bytes(buf, nbytes);
+-}
+-EXPORT_SYMBOL(get_random_bytes);
+-
+-
+-/*
+- * Each time the timer fires, we expect that we got an unpredictable
+- * jump in the cycle counter. Even if the timer is running on another
+- * CPU, the timer activity will be touching the stack of the CPU that is
+- * generating entropy..
+- *
+- * Note that we don't re-arm the timer in the timer itself - we are
+- * happy to be scheduled away, since that just makes the load more
+- * complex, but we do not want the timer to keep ticking unless the
+- * entropy loop is running.
+- *
+- * So the re-arming always happens in the entropy loop itself.
+- */
+-static void entropy_timer(struct timer_list *t)
+-{
+- credit_entropy_bits(&input_pool, 1);
+-}
+-
+-/*
+- * If we have an actual cycle counter, see if we can
+- * generate enough entropy with timing noise
+- */
+-static void try_to_generate_entropy(void)
+-{
+- struct {
+- unsigned long now;
+- struct timer_list timer;
+- } stack;
+-
+- stack.now = random_get_entropy();
+-
+- /* Slow counter - or none. Don't even bother */
+- if (stack.now == random_get_entropy())
++ if (new_count < 64 && !time_is_before_jiffies(fast_pool->last + HZ))
+ return;
+
+- timer_setup_on_stack(&stack.timer, entropy_timer, 0);
+- while (!crng_ready()) {
+- if (!timer_pending(&stack.timer))
+- mod_timer(&stack.timer, jiffies+1);
+- mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
+- schedule();
+- stack.now = random_get_entropy();
+- }
+-
+- del_timer_sync(&stack.timer);
+- destroy_timer_on_stack(&stack.timer);
+- mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
++ if (unlikely(!fast_pool->mix.func))
++ INIT_WORK(&fast_pool->mix, mix_interrupt_randomness);
++ fast_pool->count |= MIX_INFLIGHT;
++ queue_work_on(raw_smp_processor_id(), system_highpri_wq, &fast_pool->mix);
+ }
++EXPORT_SYMBOL_GPL(add_interrupt_randomness);
+
+-/*
+- * Wait for the urandom pool to be seeded and thus guaranteed to supply
+- * cryptographically secure random numbers. This applies to: the /dev/urandom
+- * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
+- * family of functions. Using any of these functions without first calling
+- * this function forfeits the guarantee of security.
+- *
+- * Returns: 0 if the urandom pool has been seeded.
+- * -ERESTARTSYS if the function was interrupted by a signal.
+- */
+-int wait_for_random_bytes(void)
+-{
+- if (likely(crng_ready()))
+- return 0;
+-
+- do {
+- int ret;
+- ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
+- if (ret)
+- return ret > 0 ? 0 : ret;
+-
+- try_to_generate_entropy();
+- } while (!crng_ready());
+-
+- return 0;
+-}
+-EXPORT_SYMBOL(wait_for_random_bytes);
+-
+-/*
+- * Returns whether or not the urandom pool has been seeded and thus guaranteed
+- * to supply cryptographically secure random numbers. This applies to: the
+- * /dev/urandom device, the get_random_bytes function, and the get_random_{u32,
+- * ,u64,int,long} family of functions.
+- *
+- * Returns: true if the urandom pool has been seeded.
+- * false if the urandom pool has not been seeded.
+- */
+-bool rng_is_initialized(void)
+-{
+- return crng_ready();
+-}
+-EXPORT_SYMBOL(rng_is_initialized);
++/* There is one of these per entropy source */
++struct timer_rand_state {
++ unsigned long last_time;
++ long last_delta, last_delta2;
++};
+
+ /*
+- * Add a callback function that will be invoked when the nonblocking
+- * pool is initialised.
+- *
+- * returns: 0 if callback is successfully added
+- * -EALREADY if pool is already initialised (callback not called)
+- * -ENOENT if module for callback is not alive
++ * This function adds entropy to the entropy "pool" by using timing
++ * delays. It uses the timer_rand_state structure to make an estimate
++ * of how many bits of entropy this call has added to the pool. The
++ * value "num" is also added to the pool; it should somehow describe
++ * the type of event that just happened.
+ */
+-int add_random_ready_callback(struct random_ready_callback *rdy)
++static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
+ {
+- struct module *owner;
+- unsigned long flags;
+- int err = -EALREADY;
+-
+- if (crng_ready())
+- return err;
++ unsigned long entropy = random_get_entropy(), now = jiffies, flags;
++ long delta, delta2, delta3;
++ unsigned int bits;
+
+- owner = rdy->owner;
+- if (!try_module_get(owner))
+- return -ENOENT;
++ /*
++ * If we're in a hard IRQ, add_interrupt_randomness() will be called
++ * sometime after, so mix into the fast pool.
++ */
++ if (in_hardirq()) {
++ fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);
++ } else {
++ spin_lock_irqsave(&input_pool.lock, flags);
++ _mix_pool_bytes(&entropy, sizeof(entropy));
++ _mix_pool_bytes(&num, sizeof(num));
++ spin_unlock_irqrestore(&input_pool.lock, flags);
++ }
+
+- spin_lock_irqsave(&random_ready_list_lock, flags);
+ if (crng_ready())
+- goto out;
+-
+- owner = NULL;
+-
+- list_add(&rdy->list, &random_ready_list);
+- err = 0;
+-
+-out:
+- spin_unlock_irqrestore(&random_ready_list_lock, flags);
+-
+- module_put(owner);
+-
+- return err;
+-}
+-EXPORT_SYMBOL(add_random_ready_callback);
+-
+-/*
+- * Delete a previously registered readiness callback function.
+- */
+-void del_random_ready_callback(struct random_ready_callback *rdy)
+-{
+- unsigned long flags;
+- struct module *owner = NULL;
+-
+- spin_lock_irqsave(&random_ready_list_lock, flags);
+- if (!list_empty(&rdy->list)) {
+- list_del_init(&rdy->list);
+- owner = rdy->owner;
+- }
+- spin_unlock_irqrestore(&random_ready_list_lock, flags);
++ return;
+
+- module_put(owner);
+-}
+-EXPORT_SYMBOL(del_random_ready_callback);
++ /*
++ * Calculate number of bits of randomness we probably added.
++ * We take into account the first, second and third-order deltas
++ * in order to make our estimate.
++ */
++ delta = now - READ_ONCE(state->last_time);
++ WRITE_ONCE(state->last_time, now);
+
+-/*
+- * This function will use the architecture-specific hardware random
+- * number generator if it is available. The arch-specific hw RNG will
+- * almost certainly be faster than what we can do in software, but it
+- * is impossible to verify that it is implemented securely (as
+- * opposed, to, say, the AES encryption of a sequence number using a
+- * key known by the NSA). So it's useful if we need the speed, but
+- * only if we're willing to trust the hardware manufacturer not to
+- * have put in a back door.
+- *
+- * Return number of bytes filled in.
+- */
+-int __must_check get_random_bytes_arch(void *buf, int nbytes)
+-{
+- int left = nbytes;
+- char *p = buf;
++ delta2 = delta - READ_ONCE(state->last_delta);
++ WRITE_ONCE(state->last_delta, delta);
+
+- trace_get_random_bytes_arch(left, _RET_IP_);
+- while (left) {
+- unsigned long v;
+- int chunk = min_t(int, left, sizeof(unsigned long));
++ delta3 = delta2 - READ_ONCE(state->last_delta2);
++ WRITE_ONCE(state->last_delta2, delta2);
+
+- if (!arch_get_random_long(&v))
+- break;
++ if (delta < 0)
++ delta = -delta;
++ if (delta2 < 0)
++ delta2 = -delta2;
++ if (delta3 < 0)
++ delta3 = -delta3;
++ if (delta > delta2)
++ delta = delta2;
++ if (delta > delta3)
++ delta = delta3;
+
+- memcpy(p, &v, chunk);
+- p += chunk;
+- left -= chunk;
+- }
++ /*
++ * delta is now minimum absolute delta. Round down by 1 bit
++ * on general principles, and limit entropy estimate to 11 bits.
++ */
++ bits = min(fls(delta >> 1), 11);
+
+- return nbytes - left;
++ /*
++ * As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
++ * will run after this, which uses a different crediting scheme of 1 bit
++ * per every 64 interrupts. In order to let that function do accounting
++ * close to the one in this function, we credit a full 64/64 bit per bit,
++ * and then subtract one to account for the extra one added.
++ */
++ if (in_hardirq())
++ this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
++ else
++ _credit_init_bits(bits);
+ }
+-EXPORT_SYMBOL(get_random_bytes_arch);
+
+-/*
+- * init_std_data - initialize pool with system data
+- *
+- * @r: pool to initialize
+- *
+- * This function clears the pool's entropy count and mixes some system
+- * data into the pool to prepare it for use. The pool is not cleared
+- * as that can only decrease the entropy in the pool.
+- */
+-static void __init init_std_data(struct entropy_store *r)
++void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
+ {
+- int i;
+- ktime_t now = ktime_get_real();
+- unsigned long rv;
+-
+- mix_pool_bytes(r, &now, sizeof(now));
+- for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) {
+- if (!arch_get_random_seed_long(&rv) &&
+- !arch_get_random_long(&rv))
+- rv = random_get_entropy();
+- mix_pool_bytes(r, &rv, sizeof(rv));
+- }
+- mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
++ static unsigned char last_value;
++ static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
++
++ /* Ignore autorepeat and the like. */
++ if (value == last_value)
++ return;
++
++ last_value = value;
++ add_timer_randomness(&input_timer_state,
++ (type << 4) ^ code ^ (code >> 4) ^ value);
+ }
++EXPORT_SYMBOL_GPL(add_input_randomness);
+
+-/*
+- * Note that setup_arch() may call add_device_randomness()
+- * long before we get here. This allows seeding of the pools
+- * with some platform dependent data very early in the boot
+- * process. But it limits our options here. We must use
+- * statically allocated structures that already have all
+- * initializations complete at compile time. We should also
+- * take care not to overwrite the precious per platform data
+- * we were given.
+- */
+-int __init rand_initialize(void)
++#ifdef CONFIG_BLOCK
++void add_disk_randomness(struct gendisk *disk)
+ {
+- init_std_data(&input_pool);
+- crng_initialize_primary(&primary_crng);
+- crng_global_init_time = jiffies;
+- if (ratelimit_disable) {
+- urandom_warning.interval = 0;
+- unseeded_warning.interval = 0;
+- }
+- return 0;
++ if (!disk || !disk->random)
++ return;
++ /* First major is 1, so we get >= 0x200 here. */
++ add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
+ }
++EXPORT_SYMBOL_GPL(add_disk_randomness);
+
+-#ifdef CONFIG_BLOCK
+-void rand_initialize_disk(struct gendisk *disk)
++void __cold rand_initialize_disk(struct gendisk *disk)
+ {
+ struct timer_rand_state *state;
+
+@@ -1801,116 +1141,189 @@ void rand_initialize_disk(struct gendisk *disk)
+ }
+ #endif
+
+-static ssize_t
+-urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes,
+- loff_t *ppos)
++/*
++ * Each time the timer fires, we expect that we got an unpredictable
++ * jump in the cycle counter. Even if the timer is running on another
++ * CPU, the timer activity will be touching the stack of the CPU that is
++ * generating entropy..
++ *
++ * Note that we don't re-arm the timer in the timer itself - we are
++ * happy to be scheduled away, since that just makes the load more
++ * complex, but we do not want the timer to keep ticking unless the
++ * entropy loop is running.
++ *
++ * So the re-arming always happens in the entropy loop itself.
++ */
++static void __cold entropy_timer(struct timer_list *t)
+ {
+- int ret;
+-
+- nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3));
+- ret = extract_crng_user(buf, nbytes);
+- trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool));
+- return ret;
++ credit_init_bits(1);
+ }
+
+-static ssize_t
+-urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
++/*
++ * If we have an actual cycle counter, see if we can
++ * generate enough entropy with timing noise
++ */
++static void __cold try_to_generate_entropy(void)
+ {
+- unsigned long flags;
+- static int maxwarn = 10;
++ struct {
++ unsigned long entropy;
++ struct timer_list timer;
++ } stack;
++
++ stack.entropy = random_get_entropy();
++
++ /* Slow counter - or none. Don't even bother */
++ if (stack.entropy == random_get_entropy())
++ return;
+
+- if (!crng_ready() && maxwarn > 0) {
+- maxwarn--;
+- if (__ratelimit(&urandom_warning))
+- pr_notice("%s: uninitialized urandom read (%zd bytes read)\n",
+- current->comm, nbytes);
+- spin_lock_irqsave(&primary_crng.lock, flags);
+- crng_init_cnt = 0;
+- spin_unlock_irqrestore(&primary_crng.lock, flags);
++ timer_setup_on_stack(&stack.timer, entropy_timer, 0);
++ while (!crng_ready() && !signal_pending(current)) {
++ if (!timer_pending(&stack.timer))
++ mod_timer(&stack.timer, jiffies + 1);
++ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
++ schedule();
++ stack.entropy = random_get_entropy();
+ }
+
+- return urandom_read_nowarn(file, buf, nbytes, ppos);
++ del_timer_sync(&stack.timer);
++ destroy_timer_on_stack(&stack.timer);
++ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
+ }
+
+-static ssize_t
+-random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
++
++/**********************************************************************
++ *
++ * Userspace reader/writer interfaces.
++ *
++ * getrandom(2) is the primary modern interface into the RNG and should
++ * be used in preference to anything else.
++ *
++ * Reading from /dev/random has the same functionality as calling
++ * getrandom(2) with flags=0. In earlier versions, however, it had
++ * vastly different semantics and should therefore be avoided, to
++ * prevent backwards compatibility issues.
++ *
++ * Reading from /dev/urandom has the same functionality as calling
++ * getrandom(2) with flags=GRND_INSECURE. Because it does not block
++ * waiting for the RNG to be ready, it should not be used.
++ *
++ * Writing to either /dev/random or /dev/urandom adds entropy to
++ * the input pool but does not credit it.
++ *
++ * Polling on /dev/random indicates when the RNG is initialized, on
++ * the read side, and when it wants new entropy, on the write side.
++ *
++ * Both /dev/random and /dev/urandom have the same set of ioctls for
++ * adding entropy, getting the entropy count, zeroing the count, and
++ * reseeding the crng.
++ *
++ **********************************************************************/
++
++SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
+ {
++ struct iov_iter iter;
++ struct iovec iov;
+ int ret;
+
+- ret = wait_for_random_bytes();
+- if (ret != 0)
++ if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
++ return -EINVAL;
++
++ /*
++ * Requesting insecure and blocking randomness at the same time makes
++ * no sense.
++ */
++ if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
++ return -EINVAL;
++
++ if (!crng_ready() && !(flags & GRND_INSECURE)) {
++ if (flags & GRND_NONBLOCK)
++ return -EAGAIN;
++ ret = wait_for_random_bytes();
++ if (unlikely(ret))
++ return ret;
++ }
++
++ ret = import_single_range(READ, ubuf, len, &iov, &iter);
++ if (unlikely(ret))
+ return ret;
+- return urandom_read_nowarn(file, buf, nbytes, ppos);
++ return get_random_bytes_user(&iter);
+ }
+
+-static __poll_t
+-random_poll(struct file *file, poll_table * wait)
++static __poll_t random_poll(struct file *file, poll_table *wait)
+ {
+- __poll_t mask;
+-
+ poll_wait(file, &crng_init_wait, wait);
+- poll_wait(file, &random_write_wait, wait);
+- mask = 0;
+- if (crng_ready())
+- mask |= EPOLLIN | EPOLLRDNORM;
+- if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
+- mask |= EPOLLOUT | EPOLLWRNORM;
+- return mask;
++ return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM;
+ }
+
+-static int
+-write_pool(struct entropy_store *r, const char __user *buffer, size_t count)
++static ssize_t write_pool_user(struct iov_iter *iter)
+ {
+- size_t bytes;
+- __u32 t, buf[16];
+- const char __user *p = buffer;
++ u8 block[BLAKE2S_BLOCK_SIZE];
++ ssize_t ret = 0;
++ size_t copied;
+
+- while (count > 0) {
+- int b, i = 0;
++ if (unlikely(!iov_iter_count(iter)))
++ return 0;
+
+- bytes = min(count, sizeof(buf));
+- if (copy_from_user(&buf, p, bytes))
+- return -EFAULT;
++ for (;;) {
++ copied = copy_from_iter(block, sizeof(block), iter);
++ ret += copied;
++ mix_pool_bytes(block, copied);
++ if (!iov_iter_count(iter) || copied != sizeof(block))
++ break;
+
+- for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) {
+- if (!arch_get_random_int(&t))
++ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
++ if (ret % PAGE_SIZE == 0) {
++ if (signal_pending(current))
+ break;
+- buf[i] ^= t;
++ cond_resched();
+ }
++ }
++
++ memzero_explicit(block, sizeof(block));
++ return ret ? ret : -EFAULT;
++}
++
++static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
++{
++ return write_pool_user(iter);
++}
+
+- count -= bytes;
+- p += bytes;
++static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
++{
++ static int maxwarn = 10;
+
+- mix_pool_bytes(r, buf, bytes);
+- cond_resched();
++ if (!crng_ready()) {
++ if (!ratelimit_disable && maxwarn <= 0)
++ ++urandom_warning.missed;
++ else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
++ --maxwarn;
++ pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
++ current->comm, iov_iter_count(iter));
++ }
+ }
+
+- return 0;
++ return get_random_bytes_user(iter);
+ }
+
+-static ssize_t random_write(struct file *file, const char __user *buffer,
+- size_t count, loff_t *ppos)
++static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
+ {
+- size_t ret;
++ int ret;
+
+- ret = write_pool(&input_pool, buffer, count);
+- if (ret)
++ ret = wait_for_random_bytes();
++ if (ret != 0)
+ return ret;
+-
+- return (ssize_t)count;
++ return get_random_bytes_user(iter);
+ }
+
+ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+ {
+- int size, ent_count;
+ int __user *p = (int __user *)arg;
+- int retval;
++ int ent_count;
+
+ switch (cmd) {
+ case RNDGETENTCNT:
+- /* inherently racy, no point locking */
+- ent_count = ENTROPY_BITS(&input_pool);
+- if (put_user(ent_count, p))
++ /* Inherently racy, no point locking. */
++ if (put_user(input_pool.init_bits, p))
+ return -EFAULT;
+ return 0;
+ case RNDADDTOENTCNT:
+@@ -1918,38 +1331,48 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+ return -EPERM;
+ if (get_user(ent_count, p))
+ return -EFAULT;
+- return credit_entropy_bits_safe(&input_pool, ent_count);
+- case RNDADDENTROPY:
++ if (ent_count < 0)
++ return -EINVAL;
++ credit_init_bits(ent_count);
++ return 0;
++ case RNDADDENTROPY: {
++ struct iov_iter iter;
++ struct iovec iov;
++ ssize_t ret;
++ int len;
++
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ if (get_user(ent_count, p++))
+ return -EFAULT;
+ if (ent_count < 0)
+ return -EINVAL;
+- if (get_user(size, p++))
++ if (get_user(len, p++))
++ return -EFAULT;
++ ret = import_single_range(WRITE, p, len, &iov, &iter);
++ if (unlikely(ret))
++ return ret;
++ ret = write_pool_user(&iter);
++ if (unlikely(ret < 0))
++ return ret;
++ /* Since we're crediting, enforce that it was all written into the pool. */
++ if (unlikely(ret != len))
+ return -EFAULT;
+- retval = write_pool(&input_pool, (const char __user *)p,
+- size);
+- if (retval < 0)
+- return retval;
+- return credit_entropy_bits_safe(&input_pool, ent_count);
++ credit_init_bits(ent_count);
++ return 0;
++ }
+ case RNDZAPENTCNT:
+ case RNDCLEARPOOL:
+- /*
+- * Clear the entropy pool counters. We no longer clear
+- * the entropy pool, as that's silly.
+- */
++ /* No longer has any effect. */
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+- input_pool.entropy_count = 0;
+ return 0;
+ case RNDRESEEDCRNG:
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+- if (crng_init < 2)
++ if (!crng_ready())
+ return -ENODATA;
+- crng_reseed(&primary_crng, &input_pool);
+- crng_global_init_time = jiffies - 1;
++ crng_reseed();
+ return 0;
+ default:
+ return -EINVAL;
+@@ -1962,55 +1385,56 @@ static int random_fasync(int fd, struct file *filp, int on)
+ }
+
+ const struct file_operations random_fops = {
+- .read = random_read,
+- .write = random_write,
+- .poll = random_poll,
++ .read_iter = random_read_iter,
++ .write_iter = random_write_iter,
++ .poll = random_poll,
+ .unlocked_ioctl = random_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .fasync = random_fasync,
+ .llseek = noop_llseek,
++ .splice_read = generic_file_splice_read,
++ .splice_write = iter_file_splice_write,
+ };
+
+ const struct file_operations urandom_fops = {
+- .read = urandom_read,
+- .write = random_write,
++ .read_iter = urandom_read_iter,
++ .write_iter = random_write_iter,
+ .unlocked_ioctl = random_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .fasync = random_fasync,
+ .llseek = noop_llseek,
++ .splice_read = generic_file_splice_read,
++ .splice_write = iter_file_splice_write,
+ };
+
+-SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
+- unsigned int, flags)
+-{
+- int ret;
+-
+- if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE))
+- return -EINVAL;
+-
+- /*
+- * Requesting insecure and blocking randomness at the same time makes
+- * no sense.
+- */
+- if ((flags & (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM))
+- return -EINVAL;
+-
+- if (count > INT_MAX)
+- count = INT_MAX;
+-
+- if (!(flags & GRND_INSECURE) && !crng_ready()) {
+- if (flags & GRND_NONBLOCK)
+- return -EAGAIN;
+- ret = wait_for_random_bytes();
+- if (unlikely(ret))
+- return ret;
+- }
+- return urandom_read_nowarn(NULL, buf, count, NULL);
+-}
+
+ /********************************************************************
+ *
+- * Sysctl interface
++ * Sysctl interface.
++ *
++ * These are partly unused legacy knobs with dummy values to not break
++ * userspace and partly still useful things. They are usually accessible
++ * in /proc/sys/kernel/random/ and are as follows:
++ *
++ * - boot_id - a UUID representing the current boot.
++ *
++ * - uuid - a random UUID, different each time the file is read.
++ *
++ * - poolsize - the number of bits of entropy that the input pool can
++ * hold, tied to the POOL_BITS constant.
++ *
++ * - entropy_avail - the number of bits of entropy currently in the
++ * input pool. Always <= poolsize.
++ *
++ * - write_wakeup_threshold - the amount of entropy in the input pool
++ * below which write polls to /dev/random will unblock, requesting
++ * more entropy, tied to the POOL_READY_BITS constant. It is writable
++ * to avoid breaking old userspaces, but writing to it does not
++ * change any behavior of the RNG.
++ *
++ * - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
++ * It is writable to avoid breaking old userspaces, but writing
++ * to it does not change any behavior of the RNG.
+ *
+ ********************************************************************/
+
+@@ -2018,25 +1442,28 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
+
+ #include <linux/sysctl.h>
+
+-static int min_write_thresh;
+-static int max_write_thresh = INPUT_POOL_WORDS * 32;
+-static int random_min_urandom_seed = 60;
+-static char sysctl_bootid[16];
++static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
++static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
++static int sysctl_poolsize = POOL_BITS;
++static u8 sysctl_bootid[UUID_SIZE];
+
+ /*
+ * This function is used to return both the bootid UUID, and random
+- * UUID. The difference is in whether table->data is NULL; if it is,
++ * UUID. The difference is in whether table->data is NULL; if it is,
+ * then a new UUID is generated and returned to the user.
+- *
+- * If the user accesses this via the proc interface, the UUID will be
+- * returned as an ASCII string in the standard UUID format; if via the
+- * sysctl system call, as 16 bytes of binary data.
+ */
+-static int proc_do_uuid(struct ctl_table *table, int write,
+- void *buffer, size_t *lenp, loff_t *ppos)
++static int proc_do_uuid(struct ctl_table *table, int write, void *buf,
++ size_t *lenp, loff_t *ppos)
+ {
+- struct ctl_table fake_table;
+- unsigned char buf[64], tmp_uuid[16], *uuid;
++ u8 tmp_uuid[UUID_SIZE], *uuid;
++ char uuid_string[UUID_STRING_LEN + 1];
++ struct ctl_table fake_table = {
++ .data = uuid_string,
++ .maxlen = UUID_STRING_LEN
++ };
++
++ if (write)
++ return -EPERM;
+
+ uuid = table->data;
+ if (!uuid) {
+@@ -2051,32 +1478,17 @@ static int proc_do_uuid(struct ctl_table *table, int write,
+ spin_unlock(&bootid_spinlock);
+ }
+
+- sprintf(buf, "%pU", uuid);
+-
+- fake_table.data = buf;
+- fake_table.maxlen = sizeof(buf);
+-
+- return proc_dostring(&fake_table, write, buffer, lenp, ppos);
++ snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid);
++ return proc_dostring(&fake_table, 0, buf, lenp, ppos);
+ }
+
+-/*
+- * Return entropy available scaled to integral bits
+- */
+-static int proc_do_entropy(struct ctl_table *table, int write,
+- void *buffer, size_t *lenp, loff_t *ppos)
++/* The same as proc_dointvec, but writes don't change anything. */
++static int proc_do_rointvec(struct ctl_table *table, int write, void *buf,
++ size_t *lenp, loff_t *ppos)
+ {
+- struct ctl_table fake_table;
+- int entropy_count;
+-
+- entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
+-
+- fake_table.data = &entropy_count;
+- fake_table.maxlen = sizeof(entropy_count);
+-
+- return proc_dointvec(&fake_table, write, buffer, lenp, ppos);
++ return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
+ }
+
+-static int sysctl_poolsize = INPUT_POOL_WORDS * 32;
+ extern struct ctl_table random_table[];
+ struct ctl_table random_table[] = {
+ {
+@@ -2088,218 +1500,36 @@ struct ctl_table random_table[] = {
+ },
+ {
+ .procname = "entropy_avail",
++ .data = &input_pool.init_bits,
+ .maxlen = sizeof(int),
+ .mode = 0444,
+- .proc_handler = proc_do_entropy,
+- .data = &input_pool.entropy_count,
++ .proc_handler = proc_dointvec,
+ },
+ {
+ .procname = "write_wakeup_threshold",
+- .data = &random_write_wakeup_bits,
++ .data = &sysctl_random_write_wakeup_bits,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+- .proc_handler = proc_dointvec_minmax,
+- .extra1 = &min_write_thresh,
+- .extra2 = &max_write_thresh,
++ .proc_handler = proc_do_rointvec,
+ },
+ {
+ .procname = "urandom_min_reseed_secs",
+- .data = &random_min_urandom_seed,
++ .data = &sysctl_random_min_urandom_seed,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+- .proc_handler = proc_dointvec,
++ .proc_handler = proc_do_rointvec,
+ },
+ {
+ .procname = "boot_id",
+ .data = &sysctl_bootid,
+- .maxlen = 16,
+ .mode = 0444,
+ .proc_handler = proc_do_uuid,
+ },
+ {
+ .procname = "uuid",
+- .maxlen = 16,
+ .mode = 0444,
+ .proc_handler = proc_do_uuid,
+ },
+-#ifdef ADD_INTERRUPT_BENCH
+- {
+- .procname = "add_interrupt_avg_cycles",
+- .data = &avg_cycles,
+- .maxlen = sizeof(avg_cycles),
+- .mode = 0444,
+- .proc_handler = proc_doulongvec_minmax,
+- },
+- {
+- .procname = "add_interrupt_avg_deviation",
+- .data = &avg_deviation,
+- .maxlen = sizeof(avg_deviation),
+- .mode = 0444,
+- .proc_handler = proc_doulongvec_minmax,
+- },
+-#endif
+ { }
+ };
+-#endif /* CONFIG_SYSCTL */
+-
+-struct batched_entropy {
+- union {
+- u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)];
+- u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)];
+- };
+- unsigned int position;
+- spinlock_t batch_lock;
+-};
+-
+-/*
+- * Get a random word for internal kernel use only. The quality of the random
+- * number is good as /dev/urandom, but there is no backtrack protection, with
+- * the goal of being quite fast and not depleting entropy. In order to ensure
+- * that the randomness provided by this function is okay, the function
+- * wait_for_random_bytes() should be called and return 0 at least once at any
+- * point prior.
+- */
+-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
+- .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
+-};
+-
+-u64 get_random_u64(void)
+-{
+- u64 ret;
+- unsigned long flags;
+- struct batched_entropy *batch;
+- static void *previous;
+-
+- warn_unseeded_randomness(&previous);
+-
+- batch = raw_cpu_ptr(&batched_entropy_u64);
+- spin_lock_irqsave(&batch->batch_lock, flags);
+- if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
+- extract_crng((u8 *)batch->entropy_u64);
+- batch->position = 0;
+- }
+- ret = batch->entropy_u64[batch->position++];
+- spin_unlock_irqrestore(&batch->batch_lock, flags);
+- return ret;
+-}
+-EXPORT_SYMBOL(get_random_u64);
+-
+-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
+- .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
+-};
+-u32 get_random_u32(void)
+-{
+- u32 ret;
+- unsigned long flags;
+- struct batched_entropy *batch;
+- static void *previous;
+-
+- warn_unseeded_randomness(&previous);
+-
+- batch = raw_cpu_ptr(&batched_entropy_u32);
+- spin_lock_irqsave(&batch->batch_lock, flags);
+- if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
+- extract_crng((u8 *)batch->entropy_u32);
+- batch->position = 0;
+- }
+- ret = batch->entropy_u32[batch->position++];
+- spin_unlock_irqrestore(&batch->batch_lock, flags);
+- return ret;
+-}
+-EXPORT_SYMBOL(get_random_u32);
+-
+-/* It's important to invalidate all potential batched entropy that might
+- * be stored before the crng is initialized, which we can do lazily by
+- * simply resetting the counter to zero so that it's re-extracted on the
+- * next usage. */
+-static void invalidate_batched_entropy(void)
+-{
+- int cpu;
+- unsigned long flags;
+-
+- for_each_possible_cpu (cpu) {
+- struct batched_entropy *batched_entropy;
+-
+- batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
+- spin_lock_irqsave(&batched_entropy->batch_lock, flags);
+- batched_entropy->position = 0;
+- spin_unlock(&batched_entropy->batch_lock);
+-
+- batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
+- spin_lock(&batched_entropy->batch_lock);
+- batched_entropy->position = 0;
+- spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
+- }
+-}
+-
+-/**
+- * randomize_page - Generate a random, page aligned address
+- * @start: The smallest acceptable address the caller will take.
+- * @range: The size of the area, starting at @start, within which the
+- * random address must fall.
+- *
+- * If @start + @range would overflow, @range is capped.
+- *
+- * NOTE: Historical use of randomize_range, which this replaces, presumed that
+- * @start was already page aligned. We now align it regardless.
+- *
+- * Return: A page aligned address within [start, start + range). On error,
+- * @start is returned.
+- */
+-unsigned long
+-randomize_page(unsigned long start, unsigned long range)
+-{
+- if (!PAGE_ALIGNED(start)) {
+- range -= PAGE_ALIGN(start) - start;
+- start = PAGE_ALIGN(start);
+- }
+-
+- if (start > ULONG_MAX - range)
+- range = ULONG_MAX - start;
+-
+- range >>= PAGE_SHIFT;
+-
+- if (range == 0)
+- return start;
+-
+- return start + (get_random_long() % range << PAGE_SHIFT);
+-}
+-
+-/* Interface for in-kernel drivers of true hardware RNGs.
+- * Those devices may produce endless random bits and will be throttled
+- * when our pool is full.
+- */
+-void add_hwgenerator_randomness(const char *buffer, size_t count,
+- size_t entropy)
+-{
+- struct entropy_store *poolp = &input_pool;
+-
+- if (unlikely(crng_init == 0)) {
+- crng_fast_load(buffer, count);
+- return;
+- }
+-
+- /* Suspend writing if we're above the trickle threshold.
+- * We'll be woken up again once below random_write_wakeup_thresh,
+- * or when the calling thread is about to terminate.
+- */
+- wait_event_interruptible(random_write_wait, kthread_should_stop() ||
+- ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
+- mix_pool_bytes(poolp, buffer, count);
+- credit_entropy_bits(poolp, entropy);
+-}
+-EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
+-
+-/* Handle random seed passed by bootloader.
+- * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
+- * it would be regarded as device data.
+- * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
+- */
+-void add_bootloader_randomness(const void *buf, unsigned int size)
+-{
+- if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER))
+- add_hwgenerator_randomness(buf, size, size * 8);
+- else
+- add_device_randomness(buf, size);
+-}
+-EXPORT_SYMBOL_GPL(add_bootloader_randomness);
++#endif /* CONFIG_SYSCTL */
+diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
+index ddaeceb7e1091..65d800ecc9964 100644
+--- a/drivers/char/tpm/tpm-chip.c
++++ b/drivers/char/tpm/tpm-chip.c
+@@ -274,14 +274,6 @@ static void tpm_dev_release(struct device *dev)
+ kfree(chip);
+ }
+
+-static void tpm_devs_release(struct device *dev)
+-{
+- struct tpm_chip *chip = container_of(dev, struct tpm_chip, devs);
+-
+- /* release the master device reference */
+- put_device(&chip->dev);
+-}
+-
+ /**
+ * tpm_class_shutdown() - prepare the TPM device for loss of power.
+ * @dev: device to which the chip is associated.
+@@ -344,7 +336,6 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
+ chip->dev_num = rc;
+
+ device_initialize(&chip->dev);
+- device_initialize(&chip->devs);
+
+ chip->dev.class = tpm_class;
+ chip->dev.class->shutdown_pre = tpm_class_shutdown;
+@@ -352,29 +343,12 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
+ chip->dev.parent = pdev;
+ chip->dev.groups = chip->groups;
+
+- chip->devs.parent = pdev;
+- chip->devs.class = tpmrm_class;
+- chip->devs.release = tpm_devs_release;
+- /* get extra reference on main device to hold on
+- * behalf of devs. This holds the chip structure
+- * while cdevs is in use. The corresponding put
+- * is in the tpm_devs_release (TPM2 only)
+- */
+- if (chip->flags & TPM_CHIP_FLAG_TPM2)
+- get_device(&chip->dev);
+-
+ if (chip->dev_num == 0)
+ chip->dev.devt = MKDEV(MISC_MAJOR, TPM_MINOR);
+ else
+ chip->dev.devt = MKDEV(MAJOR(tpm_devt), chip->dev_num);
+
+- chip->devs.devt =
+- MKDEV(MAJOR(tpm_devt), chip->dev_num + TPM_NUM_DEVICES);
+-
+ rc = dev_set_name(&chip->dev, "tpm%d", chip->dev_num);
+- if (rc)
+- goto out;
+- rc = dev_set_name(&chip->devs, "tpmrm%d", chip->dev_num);
+ if (rc)
+ goto out;
+
+@@ -382,9 +356,7 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
+ chip->flags |= TPM_CHIP_FLAG_VIRTUAL;
+
+ cdev_init(&chip->cdev, &tpm_fops);
+- cdev_init(&chip->cdevs, &tpmrm_fops);
+ chip->cdev.owner = THIS_MODULE;
+- chip->cdevs.owner = THIS_MODULE;
+
+ rc = tpm2_init_space(&chip->work_space, TPM2_SPACE_BUFFER_SIZE);
+ if (rc) {
+@@ -396,7 +368,6 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
+ return chip;
+
+ out:
+- put_device(&chip->devs);
+ put_device(&chip->dev);
+ return ERR_PTR(rc);
+ }
+@@ -445,14 +416,9 @@ static int tpm_add_char_device(struct tpm_chip *chip)
+ }
+
+ if (chip->flags & TPM_CHIP_FLAG_TPM2) {
+- rc = cdev_device_add(&chip->cdevs, &chip->devs);
+- if (rc) {
+- dev_err(&chip->devs,
+- "unable to cdev_device_add() %s, major %d, minor %d, err=%d\n",
+- dev_name(&chip->devs), MAJOR(chip->devs.devt),
+- MINOR(chip->devs.devt), rc);
+- return rc;
+- }
++ rc = tpm_devs_add(chip);
++ if (rc)
++ goto err_del_cdev;
+ }
+
+ /* Make the chip available. */
+@@ -460,6 +426,10 @@ static int tpm_add_char_device(struct tpm_chip *chip)
+ idr_replace(&dev_nums_idr, chip, chip->dev_num);
+ mutex_unlock(&idr_lock);
+
++ return 0;
++
++err_del_cdev:
++ cdev_device_del(&chip->cdev, &chip->dev);
+ return rc;
+ }
+
+@@ -474,13 +444,21 @@ static void tpm_del_char_device(struct tpm_chip *chip)
+
+ /* Make the driver uncallable. */
+ down_write(&chip->ops_sem);
+- if (chip->flags & TPM_CHIP_FLAG_TPM2) {
+- if (!tpm_chip_start(chip)) {
+- tpm2_shutdown(chip, TPM2_SU_CLEAR);
+- tpm_chip_stop(chip);
++
++ /*
++ * Check if chip->ops is still valid: In case that the controller
++ * drivers shutdown handler unregisters the controller in its
++ * shutdown handler we are called twice and chip->ops to NULL.
++ */
++ if (chip->ops) {
++ if (chip->flags & TPM_CHIP_FLAG_TPM2) {
++ if (!tpm_chip_start(chip)) {
++ tpm2_shutdown(chip, TPM2_SU_CLEAR);
++ tpm_chip_stop(chip);
++ }
+ }
++ chip->ops = NULL;
+ }
+- chip->ops = NULL;
+ up_write(&chip->ops_sem);
+ }
+
+@@ -641,7 +619,7 @@ void tpm_chip_unregister(struct tpm_chip *chip)
+ hwrng_unregister(&chip->hwrng);
+ tpm_bios_log_teardown(chip);
+ if (chip->flags & TPM_CHIP_FLAG_TPM2)
+- cdev_device_del(&chip->cdevs, &chip->devs);
++ tpm_devs_remove(chip);
+ tpm_del_char_device(chip);
+ }
+ EXPORT_SYMBOL_GPL(tpm_chip_unregister);
+diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c
+index c08cbb306636b..dc4c0a0a51290 100644
+--- a/drivers/char/tpm/tpm-dev-common.c
++++ b/drivers/char/tpm/tpm-dev-common.c
+@@ -69,7 +69,13 @@ static void tpm_dev_async_work(struct work_struct *work)
+ ret = tpm_dev_transmit(priv->chip, priv->space, priv->data_buffer,
+ sizeof(priv->data_buffer));
+ tpm_put_ops(priv->chip);
+- if (ret > 0) {
++
++ /*
++ * If ret is > 0 then tpm_dev_transmit returned the size of the
++ * response. If ret is < 0 then tpm_dev_transmit failed and
++ * returned an error code.
++ */
++ if (ret != 0) {
+ priv->response_length = ret;
+ mod_timer(&priv->user_read_timer, jiffies + (120 * HZ));
+ }
+diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
+index 283f78211c3a7..2163c6ee0d364 100644
+--- a/drivers/char/tpm/tpm.h
++++ b/drivers/char/tpm/tpm.h
+@@ -234,6 +234,8 @@ int tpm2_prepare_space(struct tpm_chip *chip, struct tpm_space *space, u8 *cmd,
+ size_t cmdsiz);
+ int tpm2_commit_space(struct tpm_chip *chip, struct tpm_space *space, void *buf,
+ size_t *bufsiz);
++int tpm_devs_add(struct tpm_chip *chip);
++void tpm_devs_remove(struct tpm_chip *chip);
+
+ void tpm_bios_log_setup(struct tpm_chip *chip);
+ void tpm_bios_log_teardown(struct tpm_chip *chip);
+diff --git a/drivers/char/tpm/tpm2-space.c b/drivers/char/tpm/tpm2-space.c
+index 784b8b3cb903f..ffb35f0154c16 100644
+--- a/drivers/char/tpm/tpm2-space.c
++++ b/drivers/char/tpm/tpm2-space.c
+@@ -58,12 +58,12 @@ int tpm2_init_space(struct tpm_space *space, unsigned int buf_size)
+
+ void tpm2_del_space(struct tpm_chip *chip, struct tpm_space *space)
+ {
+- mutex_lock(&chip->tpm_mutex);
+- if (!tpm_chip_start(chip)) {
++
++ if (tpm_try_get_ops(chip) == 0) {
+ tpm2_flush_sessions(chip, space);
+- tpm_chip_stop(chip);
++ tpm_put_ops(chip);
+ }
+- mutex_unlock(&chip->tpm_mutex);
++
+ kfree(space->context_buf);
+ kfree(space->session_buf);
+ }
+@@ -455,6 +455,9 @@ static int tpm2_map_response_body(struct tpm_chip *chip, u32 cc, u8 *rsp,
+ if (be32_to_cpu(data->capability) != TPM2_CAP_HANDLES)
+ return 0;
+
++ if (be32_to_cpu(data->count) > (UINT_MAX - TPM_HEADER_SIZE - 9) / 4)
++ return -EFAULT;
++
+ if (len != TPM_HEADER_SIZE + 9 + 4 * be32_to_cpu(data->count))
+ return -EFAULT;
+
+@@ -571,3 +574,68 @@ out:
+ dev_err(&chip->dev, "%s: error %d\n", __func__, rc);
+ return rc;
+ }
++
++/*
++ * Put the reference to the main device.
++ */
++static void tpm_devs_release(struct device *dev)
++{
++ struct tpm_chip *chip = container_of(dev, struct tpm_chip, devs);
++
++ /* release the master device reference */
++ put_device(&chip->dev);
++}
++
++/*
++ * Remove the device file for exposed TPM spaces and release the device
++ * reference. This may also release the reference to the master device.
++ */
++void tpm_devs_remove(struct tpm_chip *chip)
++{
++ cdev_device_del(&chip->cdevs, &chip->devs);
++ put_device(&chip->devs);
++}
++
++/*
++ * Add a device file to expose TPM spaces. Also take a reference to the
++ * main device.
++ */
++int tpm_devs_add(struct tpm_chip *chip)
++{
++ int rc;
++
++ device_initialize(&chip->devs);
++ chip->devs.parent = chip->dev.parent;
++ chip->devs.class = tpmrm_class;
++
++ /*
++ * Get extra reference on main device to hold on behalf of devs.
++ * This holds the chip structure while cdevs is in use. The
++ * corresponding put is in the tpm_devs_release.
++ */
++ get_device(&chip->dev);
++ chip->devs.release = tpm_devs_release;
++ chip->devs.devt = MKDEV(MAJOR(tpm_devt), chip->dev_num + TPM_NUM_DEVICES);
++ cdev_init(&chip->cdevs, &tpmrm_fops);
++ chip->cdevs.owner = THIS_MODULE;
++
++ rc = dev_set_name(&chip->devs, "tpmrm%d", chip->dev_num);
++ if (rc)
++ goto err_put_devs;
++
++ rc = cdev_device_add(&chip->cdevs, &chip->devs);
++ if (rc) {
++ dev_err(&chip->devs,
++ "unable to cdev_device_add() %s, major %d, minor %d, err=%d\n",
++ dev_name(&chip->devs), MAJOR(chip->devs.devt),
++ MINOR(chip->devs.devt), rc);
++ goto err_put_devs;
++ }
++
++ return 0;
++
++err_put_devs:
++ put_device(&chip->devs);
++
++ return rc;
++}
+diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
+index 69579efb247b3..dc56b976d8162 100644
+--- a/drivers/char/tpm/tpm_tis_core.c
++++ b/drivers/char/tpm/tpm_tis_core.c
+@@ -48,6 +48,7 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
+ unsigned long timeout, wait_queue_head_t *queue,
+ bool check_cancel)
+ {
++ struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
+ unsigned long stop;
+ long rc;
+ u8 status;
+@@ -80,8 +81,8 @@ again:
+ }
+ } else {
+ do {
+- usleep_range(TPM_TIMEOUT_USECS_MIN,
+- TPM_TIMEOUT_USECS_MAX);
++ usleep_range(priv->timeout_min,
++ priv->timeout_max);
+ status = chip->ops->status(chip);
+ if ((status & mask) == mask)
+ return 0;
+@@ -945,9 +946,24 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
+ chip->timeout_b = msecs_to_jiffies(TIS_TIMEOUT_B_MAX);
+ chip->timeout_c = msecs_to_jiffies(TIS_TIMEOUT_C_MAX);
+ chip->timeout_d = msecs_to_jiffies(TIS_TIMEOUT_D_MAX);
++ priv->timeout_min = TPM_TIMEOUT_USECS_MIN;
++ priv->timeout_max = TPM_TIMEOUT_USECS_MAX;
+ priv->phy_ops = phy_ops;
++
+ dev_set_drvdata(&chip->dev, priv);
+
++ rc = tpm_tis_read32(priv, TPM_DID_VID(0), &vendor);
++ if (rc < 0)
++ return rc;
++
++ priv->manufacturer_id = vendor;
++
++ if (priv->manufacturer_id == TPM_VID_ATML &&
++ !(chip->flags & TPM_CHIP_FLAG_TPM2)) {
++ priv->timeout_min = TIS_TIMEOUT_MIN_ATML;
++ priv->timeout_max = TIS_TIMEOUT_MAX_ATML;
++ }
++
+ if (is_bsw()) {
+ priv->ilb_base_addr = ioremap(INTEL_LEGACY_BLK_BASE_ADDR,
+ ILB_REMAP_SIZE);
+@@ -978,7 +994,15 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
+ intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT |
+ TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT;
+ intmask &= ~TPM_GLOBAL_INT_ENABLE;
++
++ rc = request_locality(chip, 0);
++ if (rc < 0) {
++ rc = -ENODEV;
++ goto out_err;
++ }
++
+ tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask);
++ release_locality(chip, 0);
+
+ rc = tpm_chip_start(chip);
+ if (rc)
+@@ -988,12 +1012,6 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
+ if (rc)
+ goto out_err;
+
+- rc = tpm_tis_read32(priv, TPM_DID_VID(0), &vendor);
+- if (rc < 0)
+- goto out_err;
+-
+- priv->manufacturer_id = vendor;
+-
+ rc = tpm_tis_read8(priv, TPM_RID(0), &rid);
+ if (rc < 0)
+ goto out_err;
+diff --git a/drivers/char/tpm/tpm_tis_core.h b/drivers/char/tpm/tpm_tis_core.h
+index b2a3c6c72882d..3be24f221e32a 100644
+--- a/drivers/char/tpm/tpm_tis_core.h
++++ b/drivers/char/tpm/tpm_tis_core.h
+@@ -54,6 +54,8 @@ enum tis_defaults {
+ TIS_MEM_LEN = 0x5000,
+ TIS_SHORT_TIMEOUT = 750, /* ms */
+ TIS_LONG_TIMEOUT = 2000, /* 2 sec */
++ TIS_TIMEOUT_MIN_ATML = 14700, /* usecs */
++ TIS_TIMEOUT_MAX_ATML = 15000, /* usecs */
+ };
+
+ /* Some timeout values are needed before it is known whether the chip is
+@@ -98,6 +100,8 @@ struct tpm_tis_data {
+ wait_queue_head_t read_queue;
+ const struct tpm_tis_phy_ops *phy_ops;
+ unsigned short rng_quality;
++ unsigned int timeout_min; /* usecs */
++ unsigned int timeout_max; /* usecs */
+ };
+
+ struct tpm_tis_phy_ops {
+diff --git a/drivers/char/tpm/tpm_tis_spi_main.c b/drivers/char/tpm/tpm_tis_spi_main.c
+index 54584b4b00d19..aaa59a00eeaef 100644
+--- a/drivers/char/tpm/tpm_tis_spi_main.c
++++ b/drivers/char/tpm/tpm_tis_spi_main.c
+@@ -267,6 +267,7 @@ static const struct spi_device_id tpm_tis_spi_id[] = {
+ { "st33htpm-spi", (unsigned long)tpm_tis_spi_probe },
+ { "slb9670", (unsigned long)tpm_tis_spi_probe },
+ { "tpm_tis_spi", (unsigned long)tpm_tis_spi_probe },
++ { "tpm_tis-spi", (unsigned long)tpm_tis_spi_probe },
+ { "cr50", (unsigned long)cr50_spi_probe },
+ {}
+ };
+diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
+index 7eaf303a7a86f..77bc993d75130 100644
+--- a/drivers/char/virtio_console.c
++++ b/drivers/char/virtio_console.c
+@@ -1956,6 +1956,13 @@ static void virtcons_remove(struct virtio_device *vdev)
+ list_del(&portdev->list);
+ spin_unlock_irq(&pdrvdata_lock);
+
++ /* Device is going away, exit any polling for buffers */
++ virtio_break_device(vdev);
++ if (use_multiport(portdev))
++ flush_work(&portdev->control_work);
++ else
++ flush_work(&portdev->config_work);
++
+ /* Disable interrupts for vqs */
+ vdev->config->reset(vdev);
+ /* Finish up work that's lined up */
+@@ -2229,7 +2236,7 @@ static struct virtio_driver virtio_rproc_serial = {
+ .remove = virtcons_remove,
+ };
+
+-static int __init init(void)
++static int __init virtio_console_init(void)
+ {
+ int err;
+
+@@ -2264,7 +2271,7 @@ free:
+ return err;
+ }
+
+-static void __exit fini(void)
++static void __exit virtio_console_fini(void)
+ {
+ reclaim_dma_bufs();
+
+@@ -2274,8 +2281,8 @@ static void __exit fini(void)
+ class_destroy(pdrvdata.class);
+ debugfs_remove_recursive(pdrvdata.debugfs_dir);
+ }
+-module_init(init);
+-module_exit(fini);
++module_init(virtio_console_init);
++module_exit(virtio_console_fini);
+
+ MODULE_DESCRIPTION("Virtio console driver");
+ MODULE_LICENSE("GPL");
+diff --git a/drivers/char/xillybus/xillyusb.c b/drivers/char/xillybus/xillyusb.c
+index e7f88f35c7028..dc3551796e5ed 100644
+--- a/drivers/char/xillybus/xillyusb.c
++++ b/drivers/char/xillybus/xillyusb.c
+@@ -1912,6 +1912,7 @@ static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)
+
+ dealloc:
+ endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */
++ xdev->msg_ep = NULL;
+ return -ENOMEM;
+ }
+
+diff --git a/drivers/clk/actions/owl-s700.c b/drivers/clk/actions/owl-s700.c
+index a2f34d13fb543..6ea7da1d6d755 100644
+--- a/drivers/clk/actions/owl-s700.c
++++ b/drivers/clk/actions/owl-s700.c
+@@ -162,6 +162,7 @@ static struct clk_div_table hdmia_div_table[] = {
+
+ static struct clk_div_table rmii_div_table[] = {
+ {0, 4}, {1, 10},
++ {0, 0}
+ };
+
+ /* divider clocks */
+diff --git a/drivers/clk/actions/owl-s900.c b/drivers/clk/actions/owl-s900.c
+index 790890978424a..5144ada2c7e1a 100644
+--- a/drivers/clk/actions/owl-s900.c
++++ b/drivers/clk/actions/owl-s900.c
+@@ -140,7 +140,7 @@ static struct clk_div_table rmii_ref_div_table[] = {
+
+ static struct clk_div_table usb3_mac_div_table[] = {
+ { 1, 2 }, { 2, 3 }, { 3, 4 },
+- { 0, 8 },
++ { 0, 0 }
+ };
+
+ static struct clk_div_table i2s_div_table[] = {
+diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c
+index b656d25a97678..fe772baeb15ff 100644
+--- a/drivers/clk/at91/clk-generated.c
++++ b/drivers/clk/at91/clk-generated.c
+@@ -106,6 +106,10 @@ static void clk_generated_best_diff(struct clk_rate_request *req,
+ tmp_rate = parent_rate;
+ else
+ tmp_rate = parent_rate / div;
++
++ if (tmp_rate < req->min_rate || tmp_rate > req->max_rate)
++ return;
++
+ tmp_diff = abs(req->rate - tmp_rate);
+
+ if (*best_diff < 0 || *best_diff >= tmp_diff) {
+diff --git a/drivers/clk/at91/clk-master.c b/drivers/clk/at91/clk-master.c
+index a80427980bf73..04d0dd8385945 100644
+--- a/drivers/clk/at91/clk-master.c
++++ b/drivers/clk/at91/clk-master.c
+@@ -280,7 +280,7 @@ static int clk_master_pres_set_rate(struct clk_hw *hw, unsigned long rate,
+
+ else if (pres == 3)
+ pres = MASTER_PRES_MAX;
+- else
++ else if (pres)
+ pres = ffs(pres) - 1;
+
+ spin_lock_irqsave(master->lock, flags);
+@@ -309,7 +309,7 @@ static unsigned long clk_master_pres_recalc_rate(struct clk_hw *hw,
+ spin_unlock_irqrestore(master->lock, flags);
+
+ pres = (val >> master->layout->pres_shift) & MASTER_PRES_MASK;
+- if (pres == 3 && characteristics->have_div3_pres)
++ if (pres == MASTER_PRES_MAX && characteristics->have_div3_pres)
+ pres = 3;
+ else
+ pres = (1 << pres);
+@@ -610,7 +610,7 @@ static int clk_sama7g5_master_set_rate(struct clk_hw *hw, unsigned long rate,
+
+ if (div == 3)
+ div = MASTER_PRES_MAX;
+- else
++ else if (div)
+ div = ffs(div) - 1;
+
+ spin_lock_irqsave(master->lock, flags);
+diff --git a/drivers/clk/at91/clk-sam9x60-pll.c b/drivers/clk/at91/clk-sam9x60-pll.c
+index 34e3ab13741ac..1f52409475e9c 100644
+--- a/drivers/clk/at91/clk-sam9x60-pll.c
++++ b/drivers/clk/at91/clk-sam9x60-pll.c
+@@ -71,8 +71,8 @@ static unsigned long sam9x60_frac_pll_recalc_rate(struct clk_hw *hw,
+ struct sam9x60_pll_core *core = to_sam9x60_pll_core(hw);
+ struct sam9x60_frac *frac = to_sam9x60_frac(core);
+
+- return (parent_rate * (frac->mul + 1) +
+- ((u64)parent_rate * frac->frac >> 22));
++ return parent_rate * (frac->mul + 1) +
++ DIV_ROUND_CLOSEST_ULL((u64)parent_rate * frac->frac, (1 << 22));
+ }
+
+ static int sam9x60_frac_pll_prepare(struct clk_hw *hw)
+diff --git a/drivers/clk/at91/pmc.c b/drivers/clk/at91/pmc.c
+index 20ee9dccee787..b40035b011d0a 100644
+--- a/drivers/clk/at91/pmc.c
++++ b/drivers/clk/at91/pmc.c
+@@ -267,6 +267,11 @@ static int __init pmc_register_ops(void)
+ if (!np)
+ return -ENODEV;
+
++ if (!of_device_is_available(np)) {
++ of_node_put(np);
++ return -ENODEV;
++ }
++
+ pmcreg = device_node_to_regmap(np);
+ of_node_put(np);
+ if (IS_ERR(pmcreg))
+diff --git a/drivers/clk/at91/sama7g5.c b/drivers/clk/at91/sama7g5.c
+index cf8c079aa086a..9eed97a299d0f 100644
+--- a/drivers/clk/at91/sama7g5.c
++++ b/drivers/clk/at91/sama7g5.c
+@@ -687,16 +687,16 @@ static const struct {
+ { .n = "pdmc0_gclk",
+ .id = 68,
+ .r = { .max = 50000000 },
+- .pp = { "syspll_divpmcck", "baudpll_divpmcck", },
+- .pp_mux_table = { 5, 8, },
++ .pp = { "syspll_divpmcck", "audiopll_divpmcck", },
++ .pp_mux_table = { 5, 9, },
+ .pp_count = 2,
+ .pp_chg_id = INT_MIN, },
+
+ { .n = "pdmc1_gclk",
+ .id = 69,
+ .r = { .max = 50000000, },
+- .pp = { "syspll_divpmcck", "baudpll_divpmcck", },
+- .pp_mux_table = { 5, 8, },
++ .pp = { "syspll_divpmcck", "audiopll_divpmcck", },
++ .pp_mux_table = { 5, 9, },
+ .pp_count = 2,
+ .pp_chg_id = INT_MIN, },
+
+@@ -982,16 +982,7 @@ static void __init sama7g5_pmc_setup(struct device_node *np)
+ }
+
+ parent_names[0] = "cpupll_divpmcck";
+- hw = at91_clk_register_master_pres(regmap, "cpuck", 1, parent_names,
+- &mck0_layout, &mck0_characteristics,
+- &pmc_mck0_lock,
+- CLK_SET_RATE_PARENT, 0);
+- if (IS_ERR(hw))
+- goto err_free;
+-
+- sama7g5_pmc->chws[PMC_CPU] = hw;
+-
+- hw = at91_clk_register_master_div(regmap, "mck0", "cpuck",
++ hw = at91_clk_register_master_div(regmap, "mck0", "cpupll_divpmcck",
+ &mck0_layout, &mck0_characteristics,
+ &pmc_mck0_lock, 0);
+ if (IS_ERR(hw))
+diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
+index a254512965eb8..3667b4d731e71 100644
+--- a/drivers/clk/bcm/clk-bcm2835.c
++++ b/drivers/clk/bcm/clk-bcm2835.c
+@@ -932,8 +932,7 @@ static int bcm2835_clock_is_on(struct clk_hw *hw)
+
+ static u32 bcm2835_clock_choose_div(struct clk_hw *hw,
+ unsigned long rate,
+- unsigned long parent_rate,
+- bool round_up)
++ unsigned long parent_rate)
+ {
+ struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+ const struct bcm2835_clock_data *data = clock->data;
+@@ -945,10 +944,6 @@ static u32 bcm2835_clock_choose_div(struct clk_hw *hw,
+
+ rem = do_div(temp, rate);
+ div = temp;
+-
+- /* Round up and mask off the unused bits */
+- if (round_up && ((div & unused_frac_mask) != 0 || rem != 0))
+- div += unused_frac_mask + 1;
+ div &= ~unused_frac_mask;
+
+ /* different clamping limits apply for a mash clock */
+@@ -1079,7 +1074,7 @@ static int bcm2835_clock_set_rate(struct clk_hw *hw,
+ struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+ struct bcm2835_cprman *cprman = clock->cprman;
+ const struct bcm2835_clock_data *data = clock->data;
+- u32 div = bcm2835_clock_choose_div(hw, rate, parent_rate, false);
++ u32 div = bcm2835_clock_choose_div(hw, rate, parent_rate);
+ u32 ctl;
+
+ spin_lock(&cprman->regs_lock);
+@@ -1130,7 +1125,7 @@ static unsigned long bcm2835_clock_choose_div_and_prate(struct clk_hw *hw,
+
+ if (!(BIT(parent_idx) & data->set_rate_parent)) {
+ *prate = clk_hw_get_rate(parent);
+- *div = bcm2835_clock_choose_div(hw, rate, *prate, true);
++ *div = bcm2835_clock_choose_div(hw, rate, *prate);
+
+ *avgrate = bcm2835_clock_rate_from_divisor(clock, *prate, *div);
+
+@@ -1216,7 +1211,7 @@ static int bcm2835_clock_determine_rate(struct clk_hw *hw,
+ rate = bcm2835_clock_choose_div_and_prate(hw, i, req->rate,
+ &div, &prate,
+ &avgrate);
+- if (rate > best_rate && rate <= req->rate) {
++ if (abs(req->rate - rate) < abs(req->rate - best_rate)) {
+ best_parent = parent;
+ best_prate = prate;
+ best_rate = rate;
+diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c
+index bc3be5f3eae15..24dab2312bc6f 100644
+--- a/drivers/clk/clk-ast2600.c
++++ b/drivers/clk/clk-ast2600.c
+@@ -51,6 +51,8 @@ static DEFINE_SPINLOCK(aspeed_g6_clk_lock);
+ static struct clk_hw_onecell_data *aspeed_g6_clk_data;
+
+ static void __iomem *scu_g6_base;
++/* AST2600 revision: A0, A1, A2, etc */
++static u8 soc_rev;
+
+ /*
+ * Clocks marked with CLK_IS_CRITICAL:
+@@ -191,9 +193,8 @@ static struct clk_hw *ast2600_calc_pll(const char *name, u32 val)
+ static struct clk_hw *ast2600_calc_apll(const char *name, u32 val)
+ {
+ unsigned int mult, div;
+- u32 chip_id = readl(scu_g6_base + ASPEED_G6_SILICON_REV);
+
+- if (((chip_id & CHIP_REVISION_ID) >> 16) >= 2) {
++ if (soc_rev >= 2) {
+ if (val & BIT(24)) {
+ /* Pass through mode */
+ mult = div = 1;
+@@ -707,7 +708,7 @@ static const u32 ast2600_a1_axi_ahb200_tbl[] = {
+ static void __init aspeed_g6_cc(struct regmap *map)
+ {
+ struct clk_hw *hw;
+- u32 val, div, divbits, chip_id, axi_div, ahb_div;
++ u32 val, div, divbits, axi_div, ahb_div;
+
+ clk_hw_register_fixed_rate(NULL, "clkin", NULL, 0, 25000000);
+
+@@ -738,8 +739,7 @@ static void __init aspeed_g6_cc(struct regmap *map)
+ axi_div = 2;
+
+ divbits = (val >> 11) & 0x3;
+- regmap_read(map, ASPEED_G6_SILICON_REV, &chip_id);
+- if (chip_id & BIT(16)) {
++ if (soc_rev >= 1) {
+ if (!divbits) {
+ ahb_div = ast2600_a1_axi_ahb200_tbl[(val >> 8) & 0x3];
+ if (val & BIT(16))
+@@ -784,6 +784,8 @@ static void __init aspeed_g6_cc_init(struct device_node *np)
+ if (!scu_g6_base)
+ return;
+
++ soc_rev = (readl(scu_g6_base + ASPEED_G6_SILICON_REV) & CHIP_REVISION_ID) >> 16;
++
+ aspeed_g6_clk_data = kzalloc(struct_size(aspeed_g6_clk_data, hws,
+ ASPEED_G6_NUM_CLKS), GFP_KERNEL);
+ if (!aspeed_g6_clk_data)
+diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c
+index e6d6599d310a1..fad78a22218e8 100644
+--- a/drivers/clk/clk-bm1880.c
++++ b/drivers/clk/clk-bm1880.c
+@@ -522,14 +522,6 @@ static struct clk_hw *bm1880_clk_register_pll(struct bm1880_pll_hw_clock *pll_cl
+ return hw;
+ }
+
+-static void bm1880_clk_unregister_pll(struct clk_hw *hw)
+-{
+- struct bm1880_pll_hw_clock *pll_hw = to_bm1880_pll_clk(hw);
+-
+- clk_hw_unregister(hw);
+- kfree(pll_hw);
+-}
+-
+ static int bm1880_clk_register_plls(struct bm1880_pll_hw_clock *clks,
+ int num_clks,
+ struct bm1880_clock_data *data)
+@@ -555,7 +547,7 @@ static int bm1880_clk_register_plls(struct bm1880_pll_hw_clock *clks,
+
+ err_clk:
+ while (i--)
+- bm1880_clk_unregister_pll(data->hw_data.hws[clks[i].pll.id]);
++ clk_hw_unregister(data->hw_data.hws[clks[i].pll.id]);
+
+ return PTR_ERR(hw);
+ }
+@@ -695,14 +687,6 @@ static struct clk_hw *bm1880_clk_register_div(struct bm1880_div_hw_clock *div_cl
+ return hw;
+ }
+
+-static void bm1880_clk_unregister_div(struct clk_hw *hw)
+-{
+- struct bm1880_div_hw_clock *div_hw = to_bm1880_div_clk(hw);
+-
+- clk_hw_unregister(hw);
+- kfree(div_hw);
+-}
+-
+ static int bm1880_clk_register_divs(struct bm1880_div_hw_clock *clks,
+ int num_clks,
+ struct bm1880_clock_data *data)
+@@ -729,7 +713,7 @@ static int bm1880_clk_register_divs(struct bm1880_div_hw_clock *clks,
+
+ err_clk:
+ while (i--)
+- bm1880_clk_unregister_div(data->hw_data.hws[clks[i].div.id]);
++ clk_hw_unregister(data->hw_data.hws[clks[i].div.id]);
+
+ return PTR_ERR(hw);
+ }
+diff --git a/drivers/clk/clk-clps711x.c b/drivers/clk/clk-clps711x.c
+index a2c6486ef1708..f8417ee2961aa 100644
+--- a/drivers/clk/clk-clps711x.c
++++ b/drivers/clk/clk-clps711x.c
+@@ -28,11 +28,13 @@ static const struct clk_div_table spi_div_table[] = {
+ { .val = 1, .div = 8, },
+ { .val = 2, .div = 2, },
+ { .val = 3, .div = 1, },
++ { /* sentinel */ }
+ };
+
+ static const struct clk_div_table timer_div_table[] = {
+ { .val = 0, .div = 256, },
+ { .val = 1, .div = 1, },
++ { /* sentinel */ }
+ };
+
+ struct clps711x_clk {
+diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c
+index 57ae183982d8c..4de098b6b0d4e 100644
+--- a/drivers/clk/clk-si5341.c
++++ b/drivers/clk/clk-si5341.c
+@@ -798,6 +798,15 @@ static unsigned long si5341_output_clk_recalc_rate(struct clk_hw *hw,
+ u32 r_divider;
+ u8 r[3];
+
++ err = regmap_read(output->data->regmap,
++ SI5341_OUT_CONFIG(output), &val);
++ if (err < 0)
++ return err;
++
++ /* If SI5341_OUT_CFG_RDIV_FORCE2 is set, r_divider is 2 */
++ if (val & SI5341_OUT_CFG_RDIV_FORCE2)
++ return parent_rate / 2;
++
+ err = regmap_bulk_read(output->data->regmap,
+ SI5341_OUT_R_REG(output), r, 3);
+ if (err < 0)
+@@ -814,13 +823,6 @@ static unsigned long si5341_output_clk_recalc_rate(struct clk_hw *hw,
+ r_divider += 1;
+ r_divider <<= 1;
+
+- err = regmap_read(output->data->regmap,
+- SI5341_OUT_CONFIG(output), &val);
+- if (err < 0)
+- return err;
+-
+- if (val & SI5341_OUT_CFG_RDIV_FORCE2)
+- r_divider = 2;
+
+ return parent_rate / r_divider;
+ }
+@@ -1740,7 +1742,7 @@ static int si5341_probe(struct i2c_client *client,
+ clk_prepare(data->clk[i].hw.clk);
+ }
+
+- err = of_clk_add_hw_provider(client->dev.of_node, of_clk_si5341_get,
++ err = devm_of_clk_add_hw_provider(&client->dev, of_clk_si5341_get,
+ data);
+ if (err) {
+ dev_err(&client->dev, "unable to add clk provider\n");
+diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
+index af46176ad0539..473dfe632cc57 100644
+--- a/drivers/clk/clk-stm32f4.c
++++ b/drivers/clk/clk-stm32f4.c
+@@ -129,7 +129,6 @@ static const struct stm32f4_gate_data stm32f429_gates[] __initconst = {
+ { STM32F4_RCC_APB2ENR, 20, "spi5", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" },
+- { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" },
+ };
+
+ static const struct stm32f4_gate_data stm32f469_gates[] __initconst = {
+@@ -211,7 +210,6 @@ static const struct stm32f4_gate_data stm32f469_gates[] __initconst = {
+ { STM32F4_RCC_APB2ENR, 20, "spi5", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" },
+- { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" },
+ };
+
+ static const struct stm32f4_gate_data stm32f746_gates[] __initconst = {
+@@ -286,7 +284,6 @@ static const struct stm32f4_gate_data stm32f746_gates[] __initconst = {
+ { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 23, "sai2", "apb2_div" },
+- { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" },
+ };
+
+ static const struct stm32f4_gate_data stm32f769_gates[] __initconst = {
+@@ -364,7 +361,6 @@ static const struct stm32f4_gate_data stm32f769_gates[] __initconst = {
+ { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 23, "sai2", "apb2_div" },
+- { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" },
+ { STM32F4_RCC_APB2ENR, 30, "mdio", "apb2_div" },
+ };
+
+diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
+index 65508eb89ec99..d6dc58bd07b33 100644
+--- a/drivers/clk/clk.c
++++ b/drivers/clk/clk.c
+@@ -631,6 +631,24 @@ static void clk_core_get_boundaries(struct clk_core *core,
+ *max_rate = min(*max_rate, clk_user->max_rate);
+ }
+
++static bool clk_core_check_boundaries(struct clk_core *core,
++ unsigned long min_rate,
++ unsigned long max_rate)
++{
++ struct clk *user;
++
++ lockdep_assert_held(&prepare_lock);
++
++ if (min_rate > core->max_rate || max_rate < core->min_rate)
++ return false;
++
++ hlist_for_each_entry(user, &core->clks, clks_node)
++ if (min_rate > user->max_rate || max_rate < user->min_rate)
++ return false;
++
++ return true;
++}
++
+ void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate,
+ unsigned long max_rate)
+ {
+@@ -2347,6 +2365,11 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
+ clk->min_rate = min;
+ clk->max_rate = max;
+
++ if (!clk_core_check_boundaries(clk->core, min, max)) {
++ ret = -EINVAL;
++ goto out;
++ }
++
+ rate = clk_core_get_rate_nolock(clk->core);
+ if (rate < min || rate > max) {
+ /*
+@@ -2375,6 +2398,7 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
+ }
+ }
+
++out:
+ if (clk->exclusive_count)
+ clk_core_rate_protect(clk->core);
+
+@@ -3340,6 +3364,24 @@ static int __init clk_debug_init(void)
+ {
+ struct clk_core *core;
+
++#ifdef CLOCK_ALLOW_WRITE_DEBUGFS
++ pr_warn("\n");
++ pr_warn("********************************************************************\n");
++ pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
++ pr_warn("** **\n");
++ pr_warn("** WRITEABLE clk DebugFS SUPPORT HAS BEEN ENABLED IN THIS KERNEL **\n");
++ pr_warn("** **\n");
++ pr_warn("** This means that this kernel is built to expose clk operations **\n");
++ pr_warn("** such as parent or rate setting, enabling, disabling, etc. **\n");
++ pr_warn("** to userspace, which may compromise security on your system. **\n");
++ pr_warn("** **\n");
++ pr_warn("** If you see this message and you are not debugging the **\n");
++ pr_warn("** kernel, report this immediately to your vendor! **\n");
++ pr_warn("** **\n");
++ pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
++ pr_warn("********************************************************************\n");
++#endif
++
+ rootdir = debugfs_create_dir("clk", NULL);
+
+ debugfs_create_file("clk_summary", 0444, rootdir, &all_lists,
+@@ -3392,6 +3434,19 @@ static void clk_core_reparent_orphans_nolock(void)
+ __clk_set_parent_after(orphan, parent, NULL);
+ __clk_recalc_accuracies(orphan);
+ __clk_recalc_rates(orphan, 0);
++
++ /*
++ * __clk_init_parent() will set the initial req_rate to
++ * 0 if the clock doesn't have clk_ops::recalc_rate and
++ * is an orphan when it's registered.
++ *
++ * 'req_rate' is used by clk_set_rate_range() and
++ * clk_put() to trigger a clk_set_rate() call whenever
++ * the boundaries are modified. Let's make sure
++ * 'req_rate' is set to something non-zero so that
++ * clk_set_rate_range() doesn't drop the frequency.
++ */
++ orphan->req_rate = orphan->rate;
+ }
+ }
+ }
+@@ -3415,6 +3470,14 @@ static int __clk_core_init(struct clk_core *core)
+
+ clk_prepare_lock();
+
++ /*
++ * Set hw->core after grabbing the prepare_lock to synchronize with
++ * callers of clk_core_fill_parent_index() where we treat hw->core
++ * being NULL as the clk not being registered yet. This is crucial so
++ * that clks aren't parented until their parent is fully registered.
++ */
++ core->hw->core = core;
++
+ ret = clk_pm_runtime_get(core);
+ if (ret)
+ goto unlock;
+@@ -3579,8 +3642,10 @@ static int __clk_core_init(struct clk_core *core)
+ out:
+ clk_pm_runtime_put(core);
+ unlock:
+- if (ret)
++ if (ret) {
+ hlist_del_init(&core->child_node);
++ core->hw->core = NULL;
++ }
+
+ clk_prepare_unlock();
+
+@@ -3702,8 +3767,9 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
+ struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id)
+ {
+ struct device *dev = hw->core->dev;
++ const char *name = dev ? dev_name(dev) : NULL;
+
+- return clk_hw_create_clk(dev, hw, dev_name(dev), con_id);
++ return clk_hw_create_clk(dev, hw, name, con_id);
+ }
+ EXPORT_SYMBOL(clk_hw_get_clk);
+
+@@ -3844,7 +3910,6 @@ __clk_register(struct device *dev, struct device_node *np, struct clk_hw *hw)
+ core->num_parents = init->num_parents;
+ core->min_rate = 0;
+ core->max_rate = ULONG_MAX;
+- hw->core = core;
+
+ ret = clk_core_populate_parent_map(core, init);
+ if (ret)
+@@ -3862,7 +3927,7 @@ __clk_register(struct device *dev, struct device_node *np, struct clk_hw *hw)
+ goto fail_create_clk;
+ }
+
+- clk_core_link_consumer(hw->core, hw->clk);
++ clk_core_link_consumer(core, hw->clk);
+
+ ret = __clk_core_init(core);
+ if (!ret)
+diff --git a/drivers/clk/hisilicon/clk-hi3559a.c b/drivers/clk/hisilicon/clk-hi3559a.c
+index 56012a3d02192..9ea1a80acbe8b 100644
+--- a/drivers/clk/hisilicon/clk-hi3559a.c
++++ b/drivers/clk/hisilicon/clk-hi3559a.c
+@@ -611,8 +611,8 @@ static struct hisi_mux_clock hi3559av100_shub_mux_clks[] = {
+
+
+ /* shub div clk */
+-static struct clk_div_table shub_spi_clk_table[] = {{0, 8}, {1, 4}, {2, 2}};
+-static struct clk_div_table shub_uart_div_clk_table[] = {{1, 8}, {2, 4}};
++static struct clk_div_table shub_spi_clk_table[] = {{0, 8}, {1, 4}, {2, 2}, {/*sentinel*/}};
++static struct clk_div_table shub_uart_div_clk_table[] = {{1, 8}, {2, 4}, {/*sentinel*/}};
+
+ static struct hisi_divider_clock hi3559av100_shub_div_clks[] = {
+ { HI3559AV100_SHUB_SPI_SOURCE_CLK, "clk_spi_clk", "shub_clk", 0, 0x20, 24, 2,
+diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
+index 5dbb6a9377324..206e4c43f68f8 100644
+--- a/drivers/clk/imx/clk-imx6ul.c
++++ b/drivers/clk/imx/clk-imx6ul.c
+@@ -161,7 +161,6 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
+ hws[IMX6UL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+- hws[IMX6UL_CLK_CSI_SEL] = imx_clk_hw_mux_flags("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels), CLK_SET_RATE_PARENT);
+
+ /* Do not bypass PLLs initially */
+ clk_set_parent(hws[IMX6UL_PLL1_BYPASS]->clk, hws[IMX6UL_CLK_PLL1]->clk);
+@@ -270,6 +269,7 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
+ hws[IMX6UL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6UL_CLK_LCDIF_PRE_SEL] = imx_clk_hw_mux_flags("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_CLK_LCDIF_SEL] = imx_clk_hw_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
++ hws[IMX6UL_CLK_CSI_SEL] = imx_clk_hw_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels));
+
+ hws[IMX6UL_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
+ hws[IMX6UL_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
+diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
+index c4e0f1c07192f..3f6fd7ef2a68f 100644
+--- a/drivers/clk/imx/clk-imx7d.c
++++ b/drivers/clk/imx/clk-imx7d.c
+@@ -849,7 +849,6 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
+ hws[IMX7D_WDOG4_ROOT_CLK] = imx_clk_hw_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0);
+ hws[IMX7D_KPP_ROOT_CLK] = imx_clk_hw_gate4("kpp_root_clk", "ipg_root_clk", base + 0x4aa0, 0);
+ hws[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_hw_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0);
+- hws[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_hw_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0);
+ hws[IMX7D_WRCLK_ROOT_CLK] = imx_clk_hw_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0);
+ hws[IMX7D_USB_CTRL_CLK] = imx_clk_hw_gate4("usb_ctrl_clk", "ahb_root_clk", base + 0x4680, 0);
+ hws[IMX7D_USB_PHY1_CLK] = imx_clk_hw_gate4("usb_phy1_clk", "pll_usb1_main_clk", base + 0x46a0, 0);
+diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c
+index c55577604e16a..021355a247081 100644
+--- a/drivers/clk/imx/clk-imx8mn.c
++++ b/drivers/clk/imx/clk-imx8mn.c
+@@ -277,9 +277,9 @@ static const char * const imx8mn_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audi
+
+ static const char * const imx8mn_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", };
+
+-static const char * const imx8mn_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m",
+- "sys_pll1_200m", "audio_pll2_out", "vpu_pll",
+- "sys_pll1_80m", };
++static const char * const imx8mn_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "dummy",
++ "sys_pll1_200m", "audio_pll2_out", "sys_pll2_500m",
++ "dummy", "sys_pll1_80m", };
+ static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m",
+ "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out",
+ "video_pll1_out", "osc_32k", };
+diff --git a/drivers/clk/imx/clk-imx8qxp-lpcg.c b/drivers/clk/imx/clk-imx8qxp-lpcg.c
+index d3e905cf867d7..5e31a6a24b3a3 100644
+--- a/drivers/clk/imx/clk-imx8qxp-lpcg.c
++++ b/drivers/clk/imx/clk-imx8qxp-lpcg.c
+@@ -248,7 +248,7 @@ static int imx_lpcg_parse_clks_from_dt(struct platform_device *pdev,
+
+ for (i = 0; i < count; i++) {
+ idx = bit_offset[i] / 4;
+- if (idx > IMX_LPCG_MAX_CLKS) {
++ if (idx >= IMX_LPCG_MAX_CLKS) {
+ dev_warn(&pdev->dev, "invalid bit offset of clock %d\n",
+ i);
+ ret = -EINVAL;
+@@ -370,7 +370,7 @@ static struct platform_driver imx8qxp_lpcg_clk_driver = {
+ .probe = imx8qxp_lpcg_clk_probe,
+ };
+
+-builtin_platform_driver(imx8qxp_lpcg_clk_driver);
++module_platform_driver(imx8qxp_lpcg_clk_driver);
+
+ MODULE_AUTHOR("Aisheng Dong <aisheng.dong@nxp.com>");
+ MODULE_DESCRIPTION("NXP i.MX8QXP LPCG clock driver");
+diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c
+index c53a688d8ccca..40a2efb1329be 100644
+--- a/drivers/clk/imx/clk-imx8qxp.c
++++ b/drivers/clk/imx/clk-imx8qxp.c
+@@ -308,7 +308,7 @@ static struct platform_driver imx8qxp_clk_driver = {
+ },
+ .probe = imx8qxp_clk_probe,
+ };
+-builtin_platform_driver(imx8qxp_clk_driver);
++module_platform_driver(imx8qxp_clk_driver);
+
+ MODULE_AUTHOR("Aisheng Dong <aisheng.dong@nxp.com>");
+ MODULE_DESCRIPTION("NXP i.MX8QXP clock driver");
+diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c
+index 266c7595d3302..af31633a8862e 100644
+--- a/drivers/clk/ingenic/cgu.c
++++ b/drivers/clk/ingenic/cgu.c
+@@ -453,15 +453,15 @@ ingenic_clk_calc_div(struct clk_hw *hw,
+ }
+
+ /* Impose hardware constraints */
+- div = min_t(unsigned, div, 1 << clk_info->div.bits);
+- div = max_t(unsigned, div, 1);
++ div = clamp_t(unsigned int, div, clk_info->div.div,
++ clk_info->div.div << clk_info->div.bits);
+
+ /*
+ * If the divider value itself must be divided before being written to
+ * the divider register, we must ensure we don't have any bits set that
+ * would be lost as a result of doing so.
+ */
+- div /= clk_info->div.div;
++ div = DIV_ROUND_UP(div, clk_info->div.div);
+ div *= clk_info->div.div;
+
+ return div;
+diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c
+index 5154b0cf8ad6c..66ff141da0a42 100644
+--- a/drivers/clk/ingenic/jz4725b-cgu.c
++++ b/drivers/clk/ingenic/jz4725b-cgu.c
+@@ -139,11 +139,10 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
+ },
+
+ [JZ4725B_CLK_I2S] = {
+- "i2s", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
++ "i2s", CGU_CLK_MUX | CGU_CLK_DIV,
+ .parents = { JZ4725B_CLK_EXT, JZ4725B_CLK_PLL_HALF, -1, -1 },
+ .mux = { CGU_REG_CPCCR, 31, 1 },
+ .div = { CGU_REG_I2SCDR, 0, 1, 9, -1, -1, -1 },
+- .gate = { CGU_REG_CLKGR, 6 },
+ },
+
+ [JZ4725B_CLK_SPI] = {
+diff --git a/drivers/clk/loongson1/clk-loongson1c.c b/drivers/clk/loongson1/clk-loongson1c.c
+index 703f87622cf5f..1ebf740380efb 100644
+--- a/drivers/clk/loongson1/clk-loongson1c.c
++++ b/drivers/clk/loongson1/clk-loongson1c.c
+@@ -37,6 +37,7 @@ static const struct clk_div_table ahb_div_table[] = {
+ [1] = { .val = 1, .div = 4 },
+ [2] = { .val = 2, .div = 3 },
+ [3] = { .val = 3, .div = 3 },
++ [4] = { /* sentinel */ }
+ };
+
+ void __init ls1x_clk_init(void)
+diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
+index d6eed760327d0..608e0e8ca49a8 100644
+--- a/drivers/clk/meson/gxbb.c
++++ b/drivers/clk/meson/gxbb.c
+@@ -713,6 +713,35 @@ static struct clk_regmap gxbb_mpll_prediv = {
+ };
+
+ static struct clk_regmap gxbb_mpll0_div = {
++ .data = &(struct meson_clk_mpll_data){
++ .sdm = {
++ .reg_off = HHI_MPLL_CNTL7,
++ .shift = 0,
++ .width = 14,
++ },
++ .sdm_en = {
++ .reg_off = HHI_MPLL_CNTL,
++ .shift = 25,
++ .width = 1,
++ },
++ .n2 = {
++ .reg_off = HHI_MPLL_CNTL7,
++ .shift = 16,
++ .width = 9,
++ },
++ .lock = &meson_clk_lock,
++ },
++ .hw.init = &(struct clk_init_data){
++ .name = "mpll0_div",
++ .ops = &meson_clk_mpll_ops,
++ .parent_hws = (const struct clk_hw *[]) {
++ &gxbb_mpll_prediv.hw
++ },
++ .num_parents = 1,
++ },
++};
++
++static struct clk_regmap gxl_mpll0_div = {
+ .data = &(struct meson_clk_mpll_data){
+ .sdm = {
+ .reg_off = HHI_MPLL_CNTL7,
+@@ -749,7 +778,16 @@ static struct clk_regmap gxbb_mpll0 = {
+ .hw.init = &(struct clk_init_data){
+ .name = "mpll0",
+ .ops = &clk_regmap_gate_ops,
+- .parent_hws = (const struct clk_hw *[]) { &gxbb_mpll0_div.hw },
++ .parent_data = &(const struct clk_parent_data) {
++ /*
++ * Note:
++ * GXL and GXBB have different SDM_EN registers. We
++ * fallback to the global naming string mechanism so
++ * mpll0_div picks up the appropriate one.
++ */
++ .name = "mpll0_div",
++ .index = -1,
++ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+@@ -3044,7 +3082,7 @@ static struct clk_hw_onecell_data gxl_hw_onecell_data = {
+ [CLKID_VAPB_1] = &gxbb_vapb_1.hw,
+ [CLKID_VAPB_SEL] = &gxbb_vapb_sel.hw,
+ [CLKID_VAPB] = &gxbb_vapb.hw,
+- [CLKID_MPLL0_DIV] = &gxbb_mpll0_div.hw,
++ [CLKID_MPLL0_DIV] = &gxl_mpll0_div.hw,
+ [CLKID_MPLL1_DIV] = &gxbb_mpll1_div.hw,
+ [CLKID_MPLL2_DIV] = &gxbb_mpll2_div.hw,
+ [CLKID_MPLL_PREDIV] = &gxbb_mpll_prediv.hw,
+@@ -3439,7 +3477,7 @@ static struct clk_regmap *const gxl_clk_regmaps[] = {
+ &gxbb_mpll0,
+ &gxbb_mpll1,
+ &gxbb_mpll2,
+- &gxbb_mpll0_div,
++ &gxl_mpll0_div,
+ &gxbb_mpll1_div,
+ &gxbb_mpll2_div,
+ &gxbb_cts_amclk_div,
+diff --git a/drivers/clk/mvebu/ap-cpu-clk.c b/drivers/clk/mvebu/ap-cpu-clk.c
+index 08ba59ec3fb17..71bdd7c3ff034 100644
+--- a/drivers/clk/mvebu/ap-cpu-clk.c
++++ b/drivers/clk/mvebu/ap-cpu-clk.c
+@@ -256,12 +256,15 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
+ int cpu, err;
+
+ err = of_property_read_u32(dn, "reg", &cpu);
+- if (WARN_ON(err))
++ if (WARN_ON(err)) {
++ of_node_put(dn);
+ return err;
++ }
+
+ /* If cpu2 or cpu3 is enabled */
+ if (cpu & APN806_CLUSTER_NUM_MASK) {
+ nclusters = 2;
++ of_node_put(dn);
+ break;
+ }
+ }
+@@ -288,8 +291,10 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
+ int cpu, err;
+
+ err = of_property_read_u32(dn, "reg", &cpu);
+- if (WARN_ON(err))
++ if (WARN_ON(err)) {
++ of_node_put(dn);
+ return err;
++ }
+
+ cluster_index = cpu & APN806_CLUSTER_NUM_MASK;
+ cluster_index >>= APN806_CLUSTER_NUM_OFFSET;
+@@ -301,6 +306,7 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
+ parent = of_clk_get(np, cluster_index);
+ if (IS_ERR(parent)) {
+ dev_err(dev, "Could not get the clock parent\n");
++ of_node_put(dn);
+ return -EINVAL;
+ }
+ parent_name = __clk_get_name(parent);
+@@ -319,8 +325,10 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
+ init.parent_names = &parent_name;
+
+ ret = devm_clk_hw_register(dev, &ap_cpu_clk[cluster_index].hw);
+- if (ret)
++ if (ret) {
++ of_node_put(dn);
+ return ret;
++ }
+ ap_cpu_data->hws[cluster_index] = &ap_cpu_clk[cluster_index].hw;
+ }
+
+diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
+index eaedcceb766f9..8f65b9bdafce4 100644
+--- a/drivers/clk/qcom/clk-alpha-pll.c
++++ b/drivers/clk/qcom/clk-alpha-pll.c
+@@ -1429,6 +1429,15 @@ EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_fabia_ops);
+ void clk_trion_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
+ const struct alpha_pll_config *config)
+ {
++ /*
++ * If the bootloader left the PLL enabled it's likely that there are
++ * RCGs that will lock up if we disable the PLL below.
++ */
++ if (trion_pll_is_enabled(pll, regmap)) {
++ pr_debug("Trion PLL is already enabled, skipping configuration\n");
++ return;
++ }
++
+ clk_alpha_pll_write_config(regmap, PLL_L_VAL(pll), config->l);
+ regmap_write(regmap, PLL_CAL_L_VAL(pll), TRION_PLL_CAL_VAL);
+ clk_alpha_pll_write_config(regmap, PLL_ALPHA_VAL(pll), config->alpha);
+diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c
+index e1b1b426fae4b..f675fd969c4de 100644
+--- a/drivers/clk/qcom/clk-rcg2.c
++++ b/drivers/clk/qcom/clk-rcg2.c
+@@ -264,7 +264,7 @@ static int clk_rcg2_determine_floor_rate(struct clk_hw *hw,
+
+ static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
+ {
+- u32 cfg, mask;
++ u32 cfg, mask, d_val, not2d_val, n_minus_m;
+ struct clk_hw *hw = &rcg->clkr.hw;
+ int ret, index = qcom_find_src_index(hw, rcg->parent_map, f->src);
+
+@@ -283,8 +283,17 @@ static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
+ if (ret)
+ return ret;
+
++ /* Calculate 2d value */
++ d_val = f->n;
++
++ n_minus_m = f->n - f->m;
++ n_minus_m *= 2;
++
++ d_val = clamp_t(u32, d_val, f->m, n_minus_m);
++ not2d_val = ~d_val & mask;
++
+ ret = regmap_update_bits(rcg->clkr.regmap,
+- RCG_D_OFFSET(rcg), mask, ~f->n);
++ RCG_D_OFFSET(rcg), mask, not2d_val);
+ if (ret)
+ return ret;
+ }
+@@ -720,6 +729,7 @@ static const struct frac_entry frac_table_pixel[] = {
+ { 2, 9 },
+ { 4, 9 },
+ { 1, 1 },
++ { 2, 3 },
+ { }
+ };
+
+diff --git a/drivers/clk/qcom/clk-regmap-mux.c b/drivers/clk/qcom/clk-regmap-mux.c
+index b2d00b4519634..45d9cca28064f 100644
+--- a/drivers/clk/qcom/clk-regmap-mux.c
++++ b/drivers/clk/qcom/clk-regmap-mux.c
+@@ -28,7 +28,7 @@ static u8 mux_get_parent(struct clk_hw *hw)
+ val &= mask;
+
+ if (mux->parent_map)
+- return qcom_find_src_index(hw, mux->parent_map, val);
++ return qcom_find_cfg_index(hw, mux->parent_map, val);
+
+ return val;
+ }
+diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c
+index 60d2a78d13950..2af04fc4abfa9 100644
+--- a/drivers/clk/qcom/common.c
++++ b/drivers/clk/qcom/common.c
+@@ -69,6 +69,18 @@ int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map, u8 src)
+ }
+ EXPORT_SYMBOL_GPL(qcom_find_src_index);
+
++int qcom_find_cfg_index(struct clk_hw *hw, const struct parent_map *map, u8 cfg)
++{
++ int i, num_parents = clk_hw_get_num_parents(hw);
++
++ for (i = 0; i < num_parents; i++)
++ if (cfg == map[i].cfg)
++ return i;
++
++ return -ENOENT;
++}
++EXPORT_SYMBOL_GPL(qcom_find_cfg_index);
++
+ struct regmap *
+ qcom_cc_map(struct platform_device *pdev, const struct qcom_cc_desc *desc)
+ {
+diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h
+index bb39a7e106d8a..9c8f7b798d9fc 100644
+--- a/drivers/clk/qcom/common.h
++++ b/drivers/clk/qcom/common.h
+@@ -49,6 +49,8 @@ extern void
+ qcom_pll_set_fsm_mode(struct regmap *m, u32 reg, u8 bias_count, u8 lock_count);
+ extern int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map,
+ u8 src);
++extern int qcom_find_cfg_index(struct clk_hw *hw, const struct parent_map *map,
++ u8 cfg);
+
+ extern int qcom_cc_register_board_clk(struct device *dev, const char *path,
+ const char *name, unsigned long rate);
+diff --git a/drivers/clk/qcom/dispcc-sc7180.c b/drivers/clk/qcom/dispcc-sc7180.c
+index 538e4963c9152..5d2ae297e7413 100644
+--- a/drivers/clk/qcom/dispcc-sc7180.c
++++ b/drivers/clk/qcom/dispcc-sc7180.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+ /*
+- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
++ * Copyright (c) 2019, 2022, The Linux Foundation. All rights reserved.
+ */
+
+ #include <linux/clk-provider.h>
+@@ -625,6 +625,9 @@ static struct clk_branch disp_cc_mdss_vsync_clk = {
+
+ static struct gdsc mdss_gdsc = {
+ .gdscr = 0x3000,
++ .en_rest_wait_val = 0x2,
++ .en_few_wait_val = 0x2,
++ .clk_dis_wait_val = 0xf,
+ .pd = {
+ .name = "mdss_gdsc",
+ },
+diff --git a/drivers/clk/qcom/dispcc-sc7280.c b/drivers/clk/qcom/dispcc-sc7280.c
+index 4ef4ae231794b..ad596d567f6ab 100644
+--- a/drivers/clk/qcom/dispcc-sc7280.c
++++ b/drivers/clk/qcom/dispcc-sc7280.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+ /*
+- * Copyright (c) 2021, The Linux Foundation. All rights reserved.
++ * Copyright (c) 2021-2022, The Linux Foundation. All rights reserved.
+ */
+
+ #include <linux/clk-provider.h>
+@@ -787,6 +787,9 @@ static struct clk_branch disp_cc_sleep_clk = {
+
+ static struct gdsc disp_cc_mdss_core_gdsc = {
+ .gdscr = 0x1004,
++ .en_rest_wait_val = 0x2,
++ .en_few_wait_val = 0x2,
++ .clk_dis_wait_val = 0xf,
+ .pd = {
+ .name = "disp_cc_mdss_core_gdsc",
+ },
+diff --git a/drivers/clk/qcom/dispcc-sm8250.c b/drivers/clk/qcom/dispcc-sm8250.c
+index bf9ffe1a1cf47..73c5feea9818b 100644
+--- a/drivers/clk/qcom/dispcc-sm8250.c
++++ b/drivers/clk/qcom/dispcc-sm8250.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0
+ /*
+- * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
++ * Copyright (c) 2018-2020, 2022, The Linux Foundation. All rights reserved.
+ */
+
+ #include <linux/clk-provider.h>
+@@ -1125,6 +1125,9 @@ static struct clk_branch disp_cc_mdss_vsync_clk = {
+
+ static struct gdsc mdss_gdsc = {
+ .gdscr = 0x3000,
++ .en_rest_wait_val = 0x2,
++ .en_few_wait_val = 0x2,
++ .clk_dis_wait_val = 0xf,
+ .pd = {
+ .name = "mdss_gdsc",
+ },
+diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c
+index 108fe27bee10f..541016db3c4bb 100644
+--- a/drivers/clk/qcom/gcc-ipq8074.c
++++ b/drivers/clk/qcom/gcc-ipq8074.c
+@@ -60,11 +60,6 @@ static const struct parent_map gcc_xo_gpll0_gpll0_out_main_div2_map[] = {
+ { P_GPLL0_DIV2, 4 },
+ };
+
+-static const char * const gcc_xo_gpll0[] = {
+- "xo",
+- "gpll0",
+-};
+-
+ static const struct parent_map gcc_xo_gpll0_map[] = {
+ { P_XO, 0 },
+ { P_GPLL0, 1 },
+@@ -956,6 +951,11 @@ static struct clk_rcg2 blsp1_uart6_apps_clk_src = {
+ },
+ };
+
++static const struct clk_parent_data gcc_xo_gpll0[] = {
++ { .fw_name = "xo" },
++ { .hw = &gpll0.clkr.hw },
++};
++
+ static const struct freq_tbl ftbl_pcie_axi_clk_src[] = {
+ F(19200000, P_XO, 1, 0, 0),
+ F(200000000, P_GPLL0, 4, 0, 0),
+@@ -969,7 +969,7 @@ static struct clk_rcg2 pcie0_axi_clk_src = {
+ .parent_map = gcc_xo_gpll0_map,
+ .clkr.hw.init = &(struct clk_init_data){
+ .name = "pcie0_axi_clk_src",
+- .parent_names = gcc_xo_gpll0,
++ .parent_data = gcc_xo_gpll0,
+ .num_parents = 2,
+ .ops = &clk_rcg2_ops,
+ },
+@@ -1016,7 +1016,7 @@ static struct clk_rcg2 pcie1_axi_clk_src = {
+ .parent_map = gcc_xo_gpll0_map,
+ .clkr.hw.init = &(struct clk_init_data){
+ .name = "pcie1_axi_clk_src",
+- .parent_names = gcc_xo_gpll0,
++ .parent_data = gcc_xo_gpll0,
+ .num_parents = 2,
+ .ops = &clk_rcg2_ops,
+ },
+@@ -1074,7 +1074,7 @@ static struct clk_rcg2 sdcc1_apps_clk_src = {
+ .name = "sdcc1_apps_clk_src",
+ .parent_names = gcc_xo_gpll0_gpll2_gpll0_out_main_div2,
+ .num_parents = 4,
+- .ops = &clk_rcg2_ops,
++ .ops = &clk_rcg2_floor_ops,
+ },
+ };
+
+@@ -1330,7 +1330,7 @@ static struct clk_rcg2 nss_ce_clk_src = {
+ .parent_map = gcc_xo_gpll0_map,
+ .clkr.hw.init = &(struct clk_init_data){
+ .name = "nss_ce_clk_src",
+- .parent_names = gcc_xo_gpll0,
++ .parent_data = gcc_xo_gpll0,
+ .num_parents = 2,
+ .ops = &clk_rcg2_ops,
+ },
+@@ -4329,8 +4329,7 @@ static struct clk_rcg2 pcie0_rchng_clk_src = {
+ .parent_map = gcc_xo_gpll0_map,
+ .clkr.hw.init = &(struct clk_init_data){
+ .name = "pcie0_rchng_clk_src",
+- .parent_hws = (const struct clk_hw *[]) {
+- &gpll0.clkr.hw },
++ .parent_data = gcc_xo_gpll0,
+ .num_parents = 2,
+ .ops = &clk_rcg2_ops,
+ },
+diff --git a/drivers/clk/qcom/gcc-msm8994.c b/drivers/clk/qcom/gcc-msm8994.c
+index 144d2ba7a9bef..463a444c8a7e4 100644
+--- a/drivers/clk/qcom/gcc-msm8994.c
++++ b/drivers/clk/qcom/gcc-msm8994.c
+@@ -108,6 +108,7 @@ static struct clk_alpha_pll gpll4_early = {
+
+ static struct clk_alpha_pll_postdiv gpll4 = {
+ .offset = 0x1dc0,
++ .width = 4,
+ .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+ .clkr.hw.init = &(struct clk_init_data)
+ {
+diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
+index 3c3a7ff045621..9b1674b28d45d 100644
+--- a/drivers/clk/qcom/gcc-msm8996.c
++++ b/drivers/clk/qcom/gcc-msm8996.c
+@@ -2937,20 +2937,6 @@ static struct clk_branch gcc_smmu_aggre0_ahb_clk = {
+ },
+ };
+
+-static struct clk_branch gcc_aggre1_pnoc_ahb_clk = {
+- .halt_reg = 0x82014,
+- .clkr = {
+- .enable_reg = 0x82014,
+- .enable_mask = BIT(0),
+- .hw.init = &(struct clk_init_data){
+- .name = "gcc_aggre1_pnoc_ahb_clk",
+- .parent_names = (const char *[]){ "periph_noc_clk_src" },
+- .num_parents = 1,
+- .ops = &clk_branch2_ops,
+- },
+- },
+-};
+-
+ static struct clk_branch gcc_aggre2_ufs_axi_clk = {
+ .halt_reg = 0x83014,
+ .clkr = {
+@@ -3474,7 +3460,6 @@ static struct clk_regmap *gcc_msm8996_clocks[] = {
+ [GCC_AGGRE0_CNOC_AHB_CLK] = &gcc_aggre0_cnoc_ahb_clk.clkr,
+ [GCC_SMMU_AGGRE0_AXI_CLK] = &gcc_smmu_aggre0_axi_clk.clkr,
+ [GCC_SMMU_AGGRE0_AHB_CLK] = &gcc_smmu_aggre0_ahb_clk.clkr,
+- [GCC_AGGRE1_PNOC_AHB_CLK] = &gcc_aggre1_pnoc_ahb_clk.clkr,
+ [GCC_AGGRE2_UFS_AXI_CLK] = &gcc_aggre2_ufs_axi_clk.clkr,
+ [GCC_AGGRE2_USB3_AXI_CLK] = &gcc_aggre2_usb3_axi_clk.clkr,
+ [GCC_QSPI_AHB_CLK] = &gcc_qspi_ahb_clk.clkr,
+diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c
+index 6cefcdc869905..ce7c5ba2b9b7a 100644
+--- a/drivers/clk/qcom/gcc-sc7280.c
++++ b/drivers/clk/qcom/gcc-sc7280.c
+@@ -2998,7 +2998,7 @@ static struct clk_branch gcc_cfg_noc_lpass_clk = {
+ .enable_mask = BIT(0),
+ .hw.init = &(struct clk_init_data){
+ .name = "gcc_cfg_noc_lpass_clk",
+- .ops = &clk_branch2_ops,
++ .ops = &clk_branch2_aon_ops,
+ },
+ },
+ };
+diff --git a/drivers/clk/qcom/gcc-sm6125.c b/drivers/clk/qcom/gcc-sm6125.c
+index 543cfab7561f9..431b55bb0d2f7 100644
+--- a/drivers/clk/qcom/gcc-sm6125.c
++++ b/drivers/clk/qcom/gcc-sm6125.c
+@@ -1121,7 +1121,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
+ .name = "gcc_sdcc1_apps_clk_src",
+ .parent_data = gcc_parent_data_1,
+ .num_parents = ARRAY_SIZE(gcc_parent_data_1),
+- .ops = &clk_rcg2_ops,
++ .ops = &clk_rcg2_floor_ops,
+ },
+ };
+
+@@ -1143,7 +1143,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
+ .name = "gcc_sdcc1_ice_core_clk_src",
+ .parent_data = gcc_parent_data_0,
+ .num_parents = ARRAY_SIZE(gcc_parent_data_0),
+- .ops = &clk_rcg2_floor_ops,
++ .ops = &clk_rcg2_ops,
+ },
+ };
+
+diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
+index 4ece326ea233e..cf23cfd7e4674 100644
+--- a/drivers/clk/qcom/gdsc.c
++++ b/drivers/clk/qcom/gdsc.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+ /*
+- * Copyright (c) 2015, 2017-2018, The Linux Foundation. All rights reserved.
++ * Copyright (c) 2015, 2017-2018, 2022, The Linux Foundation. All rights reserved.
+ */
+
+ #include <linux/bitops.h>
+@@ -34,9 +34,14 @@
+ #define CFG_GDSCR_OFFSET 0x4
+
+ /* Wait 2^n CXO cycles between all states. Here, n=2 (4 cycles). */
+-#define EN_REST_WAIT_VAL (0x2 << 20)
+-#define EN_FEW_WAIT_VAL (0x8 << 16)
+-#define CLK_DIS_WAIT_VAL (0x2 << 12)
++#define EN_REST_WAIT_VAL 0x2
++#define EN_FEW_WAIT_VAL 0x8
++#define CLK_DIS_WAIT_VAL 0x2
++
++/* Transition delay shifts */
++#define EN_REST_WAIT_SHIFT 20
++#define EN_FEW_WAIT_SHIFT 16
++#define CLK_DIS_WAIT_SHIFT 12
+
+ #define RETAIN_MEM BIT(14)
+ #define RETAIN_PERIPH BIT(13)
+@@ -341,7 +346,18 @@ static int gdsc_init(struct gdsc *sc)
+ */
+ mask = HW_CONTROL_MASK | SW_OVERRIDE_MASK |
+ EN_REST_WAIT_MASK | EN_FEW_WAIT_MASK | CLK_DIS_WAIT_MASK;
+- val = EN_REST_WAIT_VAL | EN_FEW_WAIT_VAL | CLK_DIS_WAIT_VAL;
++
++ if (!sc->en_rest_wait_val)
++ sc->en_rest_wait_val = EN_REST_WAIT_VAL;
++ if (!sc->en_few_wait_val)
++ sc->en_few_wait_val = EN_FEW_WAIT_VAL;
++ if (!sc->clk_dis_wait_val)
++ sc->clk_dis_wait_val = CLK_DIS_WAIT_VAL;
++
++ val = sc->en_rest_wait_val << EN_REST_WAIT_SHIFT |
++ sc->en_few_wait_val << EN_FEW_WAIT_SHIFT |
++ sc->clk_dis_wait_val << CLK_DIS_WAIT_SHIFT;
++
+ ret = regmap_update_bits(sc->regmap, sc->gdscr, mask, val);
+ if (ret)
+ return ret;
+diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h
+index 5bb396b344d16..762f1b5e1ec51 100644
+--- a/drivers/clk/qcom/gdsc.h
++++ b/drivers/clk/qcom/gdsc.h
+@@ -1,6 +1,6 @@
+ /* SPDX-License-Identifier: GPL-2.0-only */
+ /*
+- * Copyright (c) 2015, 2017-2018, The Linux Foundation. All rights reserved.
++ * Copyright (c) 2015, 2017-2018, 2022, The Linux Foundation. All rights reserved.
+ */
+
+ #ifndef __QCOM_GDSC_H__
+@@ -22,6 +22,9 @@ struct reset_controller_dev;
+ * @cxcs: offsets of branch registers to toggle mem/periph bits in
+ * @cxc_count: number of @cxcs
+ * @pwrsts: Possible powerdomain power states
++ * @en_rest_wait_val: transition delay value for receiving enr ack signal
++ * @en_few_wait_val: transition delay value for receiving enf ack signal
++ * @clk_dis_wait_val: transition delay value for halting clock
+ * @resets: ids of resets associated with this gdsc
+ * @reset_count: number of @resets
+ * @rcdev: reset controller
+@@ -35,6 +38,9 @@ struct gdsc {
+ unsigned int clamp_io_ctrl;
+ unsigned int *cxcs;
+ unsigned int cxc_count;
++ unsigned int en_rest_wait_val;
++ unsigned int en_few_wait_val;
++ unsigned int clk_dis_wait_val;
+ const u8 pwrsts;
+ /* Powerdomain allowable state bitfields */
+ #define PWRSTS_OFF BIT(0)
+diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c
+index 761922ea5db76..1c92e73cd2b8c 100644
+--- a/drivers/clk/renesas/rzg2l-cpg.c
++++ b/drivers/clk/renesas/rzg2l-cpg.c
+@@ -638,10 +638,16 @@ static void rzg2l_cpg_detach_dev(struct generic_pm_domain *unused, struct device
+ pm_clk_destroy(dev);
+ }
+
++static void rzg2l_cpg_genpd_remove(void *data)
++{
++ pm_genpd_remove(data);
++}
++
+ static int __init rzg2l_cpg_add_clk_domain(struct device *dev)
+ {
+ struct device_node *np = dev->of_node;
+ struct generic_pm_domain *genpd;
++ int ret;
+
+ genpd = devm_kzalloc(dev, sizeof(*genpd), GFP_KERNEL);
+ if (!genpd)
+@@ -652,10 +658,15 @@ static int __init rzg2l_cpg_add_clk_domain(struct device *dev)
+ GENPD_FLAG_ACTIVE_WAKEUP;
+ genpd->attach_dev = rzg2l_cpg_attach_dev;
+ genpd->detach_dev = rzg2l_cpg_detach_dev;
+- pm_genpd_init(genpd, &pm_domain_always_on_gov, false);
++ ret = pm_genpd_init(genpd, &pm_domain_always_on_gov, false);
++ if (ret)
++ return ret;
+
+- of_genpd_add_provider_simple(np, genpd);
+- return 0;
++ ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, genpd);
++ if (ret)
++ return ret;
++
++ return of_genpd_add_provider_simple(np, genpd);
+ }
+
+ static int __init rzg2l_cpg_probe(struct platform_device *pdev)
+diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c
+index 75ca855e720df..6e5440841d1ee 100644
+--- a/drivers/clk/rockchip/clk-rk3568.c
++++ b/drivers/clk/rockchip/clk-rk3568.c
+@@ -1038,13 +1038,13 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = {
+ RK3568_CLKGATE_CON(20), 8, GFLAGS),
+ GATE(HCLK_VOP, "hclk_vop", "hclk_vo", 0,
+ RK3568_CLKGATE_CON(20), 9, GFLAGS),
+- COMPOSITE(DCLK_VOP0, "dclk_vop0", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT,
++ COMPOSITE(DCLK_VOP0, "dclk_vop0", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_NO_REPARENT,
+ RK3568_CLKSEL_CON(39), 10, 2, MFLAGS, 0, 8, DFLAGS,
+ RK3568_CLKGATE_CON(20), 10, GFLAGS),
+- COMPOSITE(DCLK_VOP1, "dclk_vop1", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT,
++ COMPOSITE(DCLK_VOP1, "dclk_vop1", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_NO_REPARENT,
+ RK3568_CLKSEL_CON(40), 10, 2, MFLAGS, 0, 8, DFLAGS,
+ RK3568_CLKGATE_CON(20), 11, GFLAGS),
+- COMPOSITE(DCLK_VOP2, "dclk_vop2", hpll_vpll_gpll_cpll_p, 0,
++ COMPOSITE(DCLK_VOP2, "dclk_vop2", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_NO_REPARENT,
+ RK3568_CLKSEL_CON(41), 10, 2, MFLAGS, 0, 8, DFLAGS,
+ RK3568_CLKGATE_CON(20), 12, GFLAGS),
+ GATE(CLK_VOP_PWM, "clk_vop_pwm", "xin24m", 0,
+diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
+index b7be7e11b0dfe..bb8a844309bf5 100644
+--- a/drivers/clk/rockchip/clk.c
++++ b/drivers/clk/rockchip/clk.c
+@@ -180,6 +180,7 @@ static void rockchip_fractional_approximation(struct clk_hw *hw,
+ unsigned long rate, unsigned long *parent_rate,
+ unsigned long *m, unsigned long *n)
+ {
++ struct clk_fractional_divider *fd = to_clk_fd(hw);
+ unsigned long p_rate, p_parent_rate;
+ struct clk_hw *p_parent;
+
+@@ -190,6 +191,8 @@ static void rockchip_fractional_approximation(struct clk_hw *hw,
+ *parent_rate = p_parent_rate;
+ }
+
++ fd->flags |= CLK_FRAC_DIVIDER_POWER_OF_TWO_PS;
++
+ clk_fractional_divider_general_approximation(hw, rate, parent_rate, m, n);
+ }
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
+index f32366d9336e7..bd9a8782fec3d 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
++++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
+@@ -1464,7 +1464,7 @@ static void __init sun4i_ccu_init(struct device_node *node,
+ val &= ~GENMASK(7, 6);
+ writel(val | (2 << 6), reg + SUN4I_AHB_REG);
+
+- sunxi_ccu_probe(node, reg, desc);
++ of_sunxi_ccu_probe(node, reg, desc);
+ }
+
+ static void __init sun4i_a10_ccu_setup(struct device_node *node)
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c
+index a56142b909938..6f2a589705561 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c
+@@ -196,7 +196,7 @@ static int sun50i_a100_r_ccu_probe(struct platform_device *pdev)
+ if (IS_ERR(reg))
+ return PTR_ERR(reg);
+
+- return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a100_r_ccu_desc);
++ return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun50i_a100_r_ccu_desc);
+ }
+
+ static const struct of_device_id sun50i_a100_r_ccu_ids[] = {
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c
+index 81b48c73d389f..913bb08e6dee8 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c
+@@ -1247,7 +1247,7 @@ static int sun50i_a100_ccu_probe(struct platform_device *pdev)
+ writel(val, reg + sun50i_a100_usb2_clk_regs[i]);
+ }
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a100_ccu_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun50i_a100_ccu_desc);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
+index 149cfde817cba..54f25c624f020 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
+@@ -955,7 +955,7 @@ static int sun50i_a64_ccu_probe(struct platform_device *pdev)
+
+ writel(0x515, reg + SUN50I_A64_PLL_MIPI_REG);
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a64_ccu_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun50i_a64_ccu_desc);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
+index f8909a7ed5539..f30d7eb5424d8 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
+@@ -232,7 +232,7 @@ static void __init sunxi_r_ccu_init(struct device_node *node,
+ return;
+ }
+
+- sunxi_ccu_probe(node, reg, desc);
++ of_sunxi_ccu_probe(node, reg, desc);
+ }
+
+ static void __init sun50i_h6_r_ccu_setup(struct device_node *node)
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
+index bff446b782907..c0800da2fa3d7 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
+@@ -1240,7 +1240,7 @@ static int sun50i_h6_ccu_probe(struct platform_device *pdev)
+ val |= BIT(24);
+ writel(val, reg + SUN50I_H6_HDMI_CEC_CLK_REG);
+
+- return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_h6_ccu_desc);
++ return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun50i_h6_ccu_desc);
+ }
+
+ static const struct of_device_id sun50i_h6_ccu_ids[] = {
+diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c
+index 225307305880e..22eb18079a154 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c
++++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c
+@@ -1141,9 +1141,7 @@ static void __init sun50i_h616_ccu_setup(struct device_node *node)
+ val |= BIT(24);
+ writel(val, reg + SUN50I_H616_HDMI_CEC_CLK_REG);
+
+- i = sunxi_ccu_probe(node, reg, &sun50i_h616_ccu_desc);
+- if (i)
+- pr_err("%pOF: probing clocks fails: %d\n", node, i);
++ of_sunxi_ccu_probe(node, reg, &sun50i_h616_ccu_desc);
+ }
+
+ CLK_OF_DECLARE(sun50i_h616_ccu, "allwinner,sun50i-h616-ccu",
+diff --git a/drivers/clk/sunxi-ng/ccu-sun5i.c b/drivers/clk/sunxi-ng/ccu-sun5i.c
+index b78e9b507c1c6..1f4bc0e773a7e 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun5i.c
++++ b/drivers/clk/sunxi-ng/ccu-sun5i.c
+@@ -1012,7 +1012,7 @@ static void __init sun5i_ccu_init(struct device_node *node,
+ val &= ~GENMASK(7, 6);
+ writel(val | (2 << 6), reg + SUN5I_AHB_REG);
+
+- sunxi_ccu_probe(node, reg, desc);
++ of_sunxi_ccu_probe(node, reg, desc);
+ }
+
+ static void __init sun5i_a10s_ccu_setup(struct device_node *node)
+diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
+index 9b40d53266a3f..3df5c0b415804 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
++++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
+@@ -1257,7 +1257,7 @@ static void __init sun6i_a31_ccu_setup(struct device_node *node)
+ val |= 0x3 << 12;
+ writel(val, reg + SUN6I_A31_AHB1_REG);
+
+- sunxi_ccu_probe(node, reg, &sun6i_a31_ccu_desc);
++ of_sunxi_ccu_probe(node, reg, &sun6i_a31_ccu_desc);
+
+ ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk,
+ &sun6i_a31_cpu_nb);
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
+index 103aa504f6c8a..577bb235d6584 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
+@@ -745,7 +745,7 @@ static void __init sun8i_a23_ccu_setup(struct device_node *node)
+ val &= ~BIT(16);
+ writel(val, reg + SUN8I_A23_PLL_MIPI_REG);
+
+- sunxi_ccu_probe(node, reg, &sun8i_a23_ccu_desc);
++ of_sunxi_ccu_probe(node, reg, &sun8i_a23_ccu_desc);
+ }
+ CLK_OF_DECLARE(sun8i_a23_ccu, "allwinner,sun8i-a23-ccu",
+ sun8i_a23_ccu_setup);
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
+index 91838cd110377..8f65cd03f5acc 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
+@@ -805,7 +805,7 @@ static void __init sun8i_a33_ccu_setup(struct device_node *node)
+ val &= ~BIT(16);
+ writel(val, reg + SUN8I_A33_PLL_MIPI_REG);
+
+- sunxi_ccu_probe(node, reg, &sun8i_a33_ccu_desc);
++ of_sunxi_ccu_probe(node, reg, &sun8i_a33_ccu_desc);
+
+ /* Gate then ungate PLL CPU after any rate changes */
+ ccu_pll_notifier_register(&sun8i_a33_pll_cpu_nb);
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
+index 2b434521c5ccf..c2ddcd2ddab4e 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
+@@ -906,7 +906,7 @@ static int sun8i_a83t_ccu_probe(struct platform_device *pdev)
+ sun8i_a83t_cpu_pll_fixup(reg + SUN8I_A83T_PLL_C0CPUX_REG);
+ sun8i_a83t_cpu_pll_fixup(reg + SUN8I_A83T_PLL_C1CPUX_REG);
+
+- return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun8i_a83t_ccu_desc);
++ return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun8i_a83t_ccu_desc);
+ }
+
+ static const struct of_device_id sun8i_a83t_ccu_ids[] = {
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
+index 524f33275bc73..4b94b6041b271 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
+@@ -342,7 +342,7 @@ static int sunxi_de2_clk_probe(struct platform_device *pdev)
+ goto err_disable_mod_clk;
+ }
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg, ccu_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, ccu_desc);
+ if (ret)
+ goto err_assert_reset;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+index 7e629a4493afd..d2fc2903787d8 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+@@ -1154,7 +1154,7 @@ static void __init sunxi_h3_h5_ccu_init(struct device_node *node,
+ val &= ~GENMASK(19, 16);
+ writel(val | (0 << 16), reg + SUN8I_H3_PLL_AUDIO_REG);
+
+- sunxi_ccu_probe(node, reg, desc);
++ of_sunxi_ccu_probe(node, reg, desc);
+
+ /* Gate then ungate PLL CPU after any rate changes */
+ ccu_pll_notifier_register(&sun8i_h3_pll_cpu_nb);
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
+index 4c8c491b87c27..9e754d1f754a1 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
+@@ -265,7 +265,7 @@ static void __init sunxi_r_ccu_init(struct device_node *node,
+ return;
+ }
+
+- sunxi_ccu_probe(node, reg, desc);
++ of_sunxi_ccu_probe(node, reg, desc);
+ }
+
+ static void __init sun8i_a83t_r_ccu_setup(struct device_node *node)
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
+index 84153418453f4..002e0c3a04dbe 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
+@@ -1346,7 +1346,7 @@ static int sun8i_r40_ccu_probe(struct platform_device *pdev)
+ if (IS_ERR(regmap))
+ return PTR_ERR(regmap);
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun8i_r40_ccu_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun8i_r40_ccu_desc);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
+index f49724a22540e..ce150f83ab54e 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
++++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
+@@ -822,7 +822,7 @@ static void __init sun8i_v3_v3s_ccu_init(struct device_node *node,
+ val &= ~GENMASK(19, 16);
+ writel(val, reg + SUN8I_V3S_PLL_AUDIO_REG);
+
+- sunxi_ccu_probe(node, reg, ccu_desc);
++ of_sunxi_ccu_probe(node, reg, ccu_desc);
+ }
+
+ static void __init sun8i_v3s_ccu_setup(struct device_node *node)
+diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c
+index 6616e8114f623..261e64416f26a 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c
++++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c
+@@ -246,8 +246,7 @@ static int sun9i_a80_de_clk_probe(struct platform_device *pdev)
+ goto err_disable_clk;
+ }
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg,
+- &sun9i_a80_de_clk_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun9i_a80_de_clk_desc);
+ if (ret)
+ goto err_assert_reset;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
+index 4b4a507d04edf..596243b3e0fa3 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
++++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
+@@ -117,8 +117,7 @@ static int sun9i_a80_usb_clk_probe(struct platform_device *pdev)
+ return ret;
+ }
+
+- ret = sunxi_ccu_probe(pdev->dev.of_node, reg,
+- &sun9i_a80_usb_clk_desc);
++ ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun9i_a80_usb_clk_desc);
+ if (ret)
+ goto err_disable_clk;
+
+diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c
+index ef29582676f6e..97aaed0e68500 100644
+--- a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c
++++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c
+@@ -1231,7 +1231,7 @@ static int sun9i_a80_ccu_probe(struct platform_device *pdev)
+ sun9i_a80_cpu_pll_fixup(reg + SUN9I_A80_PLL_C0CPUX_REG);
+ sun9i_a80_cpu_pll_fixup(reg + SUN9I_A80_PLL_C1CPUX_REG);
+
+- return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun9i_a80_ccu_desc);
++ return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun9i_a80_ccu_desc);
+ }
+
+ static const struct of_device_id sun9i_a80_ccu_ids[] = {
+diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
+index 7ecc3a5a5b5e1..61ad7ee91c114 100644
+--- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
++++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
+@@ -538,7 +538,7 @@ static void __init suniv_f1c100s_ccu_setup(struct device_node *node)
+ val &= ~GENMASK(19, 16);
+ writel(val | (3 << 16), reg + SUNIV_PLL_AUDIO_REG);
+
+- sunxi_ccu_probe(node, reg, &suniv_ccu_desc);
++ of_sunxi_ccu_probe(node, reg, &suniv_ccu_desc);
+
+ /* Gate then ungate PLL CPU after any rate changes */
+ ccu_pll_notifier_register(&suniv_pll_cpu_nb);
+diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c
+index 2e20e650b6c01..88cb569e58358 100644
+--- a/drivers/clk/sunxi-ng/ccu_common.c
++++ b/drivers/clk/sunxi-ng/ccu_common.c
+@@ -7,6 +7,7 @@
+
+ #include <linux/clk.h>
+ #include <linux/clk-provider.h>
++#include <linux/device.h>
+ #include <linux/iopoll.h>
+ #include <linux/slab.h>
+
+@@ -14,6 +15,11 @@
+ #include "ccu_gate.h"
+ #include "ccu_reset.h"
+
++struct sunxi_ccu {
++ const struct sunxi_ccu_desc *desc;
++ struct ccu_reset reset;
++};
++
+ static DEFINE_SPINLOCK(ccu_lock);
+
+ void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock)
+@@ -79,12 +85,15 @@ int ccu_pll_notifier_register(struct ccu_pll_nb *pll_nb)
+ &pll_nb->clk_nb);
+ }
+
+-int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
+- const struct sunxi_ccu_desc *desc)
++static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev,
++ struct device_node *node, void __iomem *reg,
++ const struct sunxi_ccu_desc *desc)
+ {
+ struct ccu_reset *reset;
+ int i, ret;
+
++ ccu->desc = desc;
++
+ for (i = 0; i < desc->num_ccu_clks; i++) {
+ struct ccu_common *cclk = desc->ccu_clks[i];
+
+@@ -103,7 +112,10 @@ int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
+ continue;
+
+ name = hw->init->name;
+- ret = of_clk_hw_register(node, hw);
++ if (dev)
++ ret = clk_hw_register(dev, hw);
++ else
++ ret = of_clk_hw_register(node, hw);
+ if (ret) {
+ pr_err("Couldn't register clock %d - %s\n", i, name);
+ goto err_clk_unreg;
+@@ -115,15 +127,10 @@ int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
+ if (ret)
+ goto err_clk_unreg;
+
+- reset = kzalloc(sizeof(*reset), GFP_KERNEL);
+- if (!reset) {
+- ret = -ENOMEM;
+- goto err_alloc_reset;
+- }
+-
++ reset = &ccu->reset;
+ reset->rcdev.of_node = node;
+ reset->rcdev.ops = &ccu_reset_ops;
+- reset->rcdev.owner = THIS_MODULE;
++ reset->rcdev.owner = dev ? dev->driver->owner : THIS_MODULE;
+ reset->rcdev.nr_resets = desc->num_resets;
+ reset->base = reg;
+ reset->lock = &ccu_lock;
+@@ -131,13 +138,11 @@ int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
+
+ ret = reset_controller_register(&reset->rcdev);
+ if (ret)
+- goto err_of_clk_unreg;
++ goto err_del_provider;
+
+ return 0;
+
+-err_of_clk_unreg:
+- kfree(reset);
+-err_alloc_reset:
++err_del_provider:
+ of_clk_del_provider(node);
+ err_clk_unreg:
+ while (--i >= 0) {
+@@ -149,3 +154,59 @@ err_clk_unreg:
+ }
+ return ret;
+ }
++
++static void devm_sunxi_ccu_release(struct device *dev, void *res)
++{
++ struct sunxi_ccu *ccu = res;
++ const struct sunxi_ccu_desc *desc = ccu->desc;
++ int i;
++
++ reset_controller_unregister(&ccu->reset.rcdev);
++ of_clk_del_provider(dev->of_node);
++
++ for (i = 0; i < desc->hw_clks->num; i++) {
++ struct clk_hw *hw = desc->hw_clks->hws[i];
++
++ if (!hw)
++ continue;
++ clk_hw_unregister(hw);
++ }
++}
++
++int devm_sunxi_ccu_probe(struct device *dev, void __iomem *reg,
++ const struct sunxi_ccu_desc *desc)
++{
++ struct sunxi_ccu *ccu;
++ int ret;
++
++ ccu = devres_alloc(devm_sunxi_ccu_release, sizeof(*ccu), GFP_KERNEL);
++ if (!ccu)
++ return -ENOMEM;
++
++ ret = sunxi_ccu_probe(ccu, dev, dev->of_node, reg, desc);
++ if (ret) {
++ devres_free(ccu);
++ return ret;
++ }
++
++ devres_add(dev, ccu);
++
++ return 0;
++}
++
++void of_sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
++ const struct sunxi_ccu_desc *desc)
++{
++ struct sunxi_ccu *ccu;
++ int ret;
++
++ ccu = kzalloc(sizeof(*ccu), GFP_KERNEL);
++ if (!ccu)
++ return;
++
++ ret = sunxi_ccu_probe(ccu, NULL, node, reg, desc);
++ if (ret) {
++ pr_err("%pOF: probing clocks failed: %d\n", node, ret);
++ kfree(ccu);
++ }
++}
+diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h
+index 04e7a12200a21..98a1834b58bb4 100644
+--- a/drivers/clk/sunxi-ng/ccu_common.h
++++ b/drivers/clk/sunxi-ng/ccu_common.h
+@@ -63,7 +63,9 @@ struct ccu_pll_nb {
+
+ int ccu_pll_notifier_register(struct ccu_pll_nb *pll_nb);
+
+-int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
+- const struct sunxi_ccu_desc *desc);
++int devm_sunxi_ccu_probe(struct device *dev, void __iomem *reg,
++ const struct sunxi_ccu_desc *desc);
++void of_sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
++ const struct sunxi_ccu_desc *desc);
+
+ #endif /* _COMMON_H_ */
+diff --git a/drivers/clk/sunxi/clk-sun9i-mmc.c b/drivers/clk/sunxi/clk-sun9i-mmc.c
+index 542b31d6e96dd..636bcf2439ef2 100644
+--- a/drivers/clk/sunxi/clk-sun9i-mmc.c
++++ b/drivers/clk/sunxi/clk-sun9i-mmc.c
+@@ -109,6 +109,8 @@ static int sun9i_a80_mmc_config_clk_probe(struct platform_device *pdev)
+ spin_lock_init(&data->lock);
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++ if (!r)
++ return -EINVAL;
+ /* one clock/reset pair per word */
+ count = DIV_ROUND_UP((resource_size(r)), SUN9I_MMC_WIDTH);
+ data->membase = devm_ioremap_resource(&pdev->dev, r);
+diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c
+index 74c1d894cca86..219c80653dbdb 100644
+--- a/drivers/clk/tegra/clk-tegra124-emc.c
++++ b/drivers/clk/tegra/clk-tegra124-emc.c
+@@ -198,6 +198,7 @@ static struct tegra_emc *emc_ensure_emc_driver(struct tegra_clk_emc *tegra)
+
+ tegra->emc = platform_get_drvdata(pdev);
+ if (!tegra->emc) {
++ put_device(&pdev->dev);
+ pr_err("%s: cannot find EMC driver\n", __func__);
+ return NULL;
+ }
+diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c
+index 3da33c786d77c..29eafab4353ef 100644
+--- a/drivers/clk/ti/clk.c
++++ b/drivers/clk/ti/clk.c
+@@ -131,7 +131,7 @@ int ti_clk_setup_ll_ops(struct ti_clk_ll_ops *ops)
+ void __init ti_dt_clocks_register(struct ti_dt_clk oclks[])
+ {
+ struct ti_dt_clk *c;
+- struct device_node *node, *parent;
++ struct device_node *node, *parent, *child;
+ struct clk *clk;
+ struct of_phandle_args clkspec;
+ char buf[64];
+@@ -171,10 +171,13 @@ void __init ti_dt_clocks_register(struct ti_dt_clk oclks[])
+ node = of_find_node_by_name(NULL, buf);
+ if (num_args && compat_mode) {
+ parent = node;
+- node = of_get_child_by_name(parent, "clock");
+- if (!node)
+- node = of_get_child_by_name(parent, "clk");
+- of_node_put(parent);
++ child = of_get_child_by_name(parent, "clock");
++ if (!child)
++ child = of_get_child_by_name(parent, "clk");
++ if (child) {
++ of_node_put(parent);
++ node = child;
++ }
+ }
+
+ clkspec.np = node;
+diff --git a/drivers/clk/uniphier/clk-uniphier-fixed-rate.c b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
+index 5319cd3804801..3bc55ab75314b 100644
+--- a/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
++++ b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
+@@ -24,6 +24,7 @@ struct clk_hw *uniphier_clk_register_fixed_rate(struct device *dev,
+
+ init.name = name;
+ init.ops = &clk_fixed_rate_ops;
++ init.flags = 0;
+ init.parent_names = NULL;
+ init.num_parents = 0;
+
+diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
+index 0f5e3983951a8..08f8cb944a2ac 100644
+--- a/drivers/clocksource/Kconfig
++++ b/drivers/clocksource/Kconfig
+@@ -24,6 +24,7 @@ config I8253_LOCK
+
+ config OMAP_DM_TIMER
+ bool
++ select TIMER_OF
+
+ config CLKBLD_I8253
+ def_bool y if CLKSRC_I8253 || CLKEVT_I8253 || I8253_LOCK
+diff --git a/drivers/clocksource/acpi_pm.c b/drivers/clocksource/acpi_pm.c
+index eb596ff9e7bb3..279ddff81ab49 100644
+--- a/drivers/clocksource/acpi_pm.c
++++ b/drivers/clocksource/acpi_pm.c
+@@ -229,8 +229,10 @@ static int __init parse_pmtmr(char *arg)
+ int ret;
+
+ ret = kstrtouint(arg, 16, &base);
+- if (ret)
+- return ret;
++ if (ret) {
++ pr_warn("PMTMR: invalid 'pmtmr=' value: '%s'\n", arg);
++ return 1;
++ }
+
+ pr_info("PMTMR IOPort override: 0x%04x -> 0x%04x\n", pmtmr_ioport,
+ base);
+diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c
+index 3819ef5b70989..3245eb0c602d2 100644
+--- a/drivers/clocksource/dw_apb_timer_of.c
++++ b/drivers/clocksource/dw_apb_timer_of.c
+@@ -47,7 +47,7 @@ static int __init timer_get_base_and_rate(struct device_node *np,
+ pr_warn("pclk for %pOFn is present, but could not be activated\n",
+ np);
+
+- if (!of_property_read_u32(np, "clock-freq", rate) &&
++ if (!of_property_read_u32(np, "clock-freq", rate) ||
+ !of_property_read_u32(np, "clock-frequency", rate))
+ return 0;
+
+diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c
+index 5e3e96d3d1b98..cc2a961ddd3be 100644
+--- a/drivers/clocksource/exynos_mct.c
++++ b/drivers/clocksource/exynos_mct.c
+@@ -504,11 +504,14 @@ static int exynos4_mct_dying_cpu(unsigned int cpu)
+ return 0;
+ }
+
+-static int __init exynos4_timer_resources(struct device_node *np, void __iomem *base)
++static int __init exynos4_timer_resources(struct device_node *np)
+ {
+- int err, cpu;
+ struct clk *mct_clk, *tick_clk;
+
++ reg_base = of_iomap(np, 0);
++ if (!reg_base)
++ panic("%s: unable to ioremap mct address space\n", __func__);
++
+ tick_clk = of_clk_get_by_name(np, "fin_pll");
+ if (IS_ERR(tick_clk))
+ panic("%s: unable to determine tick clock rate\n", __func__);
+@@ -519,9 +522,32 @@ static int __init exynos4_timer_resources(struct device_node *np, void __iomem *
+ panic("%s: unable to retrieve mct clock instance\n", __func__);
+ clk_prepare_enable(mct_clk);
+
+- reg_base = base;
+- if (!reg_base)
+- panic("%s: unable to ioremap mct address space\n", __func__);
++ return 0;
++}
++
++static int __init exynos4_timer_interrupts(struct device_node *np,
++ unsigned int int_type)
++{
++ int nr_irqs, i, err, cpu;
++
++ mct_int_type = int_type;
++
++ /* This driver uses only one global timer interrupt */
++ mct_irqs[MCT_G0_IRQ] = irq_of_parse_and_map(np, MCT_G0_IRQ);
++
++ /*
++ * Find out the number of local irqs specified. The local
++ * timer irqs are specified after the four global timer
++ * irqs are specified.
++ */
++ nr_irqs = of_irq_count(np);
++ if (nr_irqs > ARRAY_SIZE(mct_irqs)) {
++ pr_err("exynos-mct: too many (%d) interrupts configured in DT\n",
++ nr_irqs);
++ nr_irqs = ARRAY_SIZE(mct_irqs);
++ }
++ for (i = MCT_L0_IRQ; i < nr_irqs; i++)
++ mct_irqs[i] = irq_of_parse_and_map(np, i);
+
+ if (mct_int_type == MCT_INT_PPI) {
+
+@@ -532,11 +558,14 @@ static int __init exynos4_timer_resources(struct device_node *np, void __iomem *
+ mct_irqs[MCT_L0_IRQ], err);
+ } else {
+ for_each_possible_cpu(cpu) {
+- int mct_irq = mct_irqs[MCT_L0_IRQ + cpu];
++ int mct_irq;
+ struct mct_clock_event_device *pcpu_mevt =
+ per_cpu_ptr(&percpu_mct_tick, cpu);
+
+ pcpu_mevt->evt.irq = -1;
++ if (MCT_L0_IRQ + cpu >= ARRAY_SIZE(mct_irqs))
++ break;
++ mct_irq = mct_irqs[MCT_L0_IRQ + cpu];
+
+ irq_set_status_flags(mct_irq, IRQ_NOAUTOEN);
+ if (request_irq(mct_irq,
+@@ -581,24 +610,13 @@ out_irq:
+
+ static int __init mct_init_dt(struct device_node *np, unsigned int int_type)
+ {
+- u32 nr_irqs, i;
+ int ret;
+
+- mct_int_type = int_type;
+-
+- /* This driver uses only one global timer interrupt */
+- mct_irqs[MCT_G0_IRQ] = irq_of_parse_and_map(np, MCT_G0_IRQ);
+-
+- /*
+- * Find out the number of local irqs specified. The local
+- * timer irqs are specified after the four global timer
+- * irqs are specified.
+- */
+- nr_irqs = of_irq_count(np);
+- for (i = MCT_L0_IRQ; i < nr_irqs; i++)
+- mct_irqs[i] = irq_of_parse_and_map(np, i);
++ ret = exynos4_timer_resources(np);
++ if (ret)
++ return ret;
+
+- ret = exynos4_timer_resources(np, of_iomap(np, 0));
++ ret = exynos4_timer_interrupts(np, int_type);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/clocksource/timer-microchip-pit64b.c b/drivers/clocksource/timer-microchip-pit64b.c
+index cfa4ec7ef3968..790d2c9b42a70 100644
+--- a/drivers/clocksource/timer-microchip-pit64b.c
++++ b/drivers/clocksource/timer-microchip-pit64b.c
+@@ -165,7 +165,7 @@ static u64 mchp_pit64b_clksrc_read(struct clocksource *cs)
+ return mchp_pit64b_cnt_read(mchp_pit64b_cs_base);
+ }
+
+-static u64 mchp_pit64b_sched_read_clk(void)
++static u64 notrace mchp_pit64b_sched_read_clk(void)
+ {
+ return mchp_pit64b_cnt_read(mchp_pit64b_cs_base);
+ }
+diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
+index 529cc6a51cdb3..c3f54d9912be7 100644
+--- a/drivers/clocksource/timer-of.c
++++ b/drivers/clocksource/timer-of.c
+@@ -157,9 +157,9 @@ static __init int timer_of_base_init(struct device_node *np,
+ of_base->base = of_base->name ?
+ of_io_request_and_map(np, of_base->index, of_base->name) :
+ of_iomap(np, of_base->index);
+- if (IS_ERR(of_base->base)) {
+- pr_err("Failed to iomap (%s)\n", of_base->name);
+- return PTR_ERR(of_base->base);
++ if (IS_ERR_OR_NULL(of_base->base)) {
++ pr_err("Failed to iomap (%s:%s)\n", np->name, of_base->name);
++ return of_base->base ? PTR_ERR(of_base->base) : -ENOMEM;
+ }
+
+ return 0;
+diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c
+index b6f97960d8ee0..2737407ff0698 100644
+--- a/drivers/clocksource/timer-ti-dm-systimer.c
++++ b/drivers/clocksource/timer-ti-dm-systimer.c
+@@ -241,8 +241,7 @@ static void __init dmtimer_systimer_assign_alwon(void)
+ bool quirk_unreliable_oscillator = false;
+
+ /* Quirk unreliable 32 KiHz oscillator with incomplete dts */
+- if (of_machine_is_compatible("ti,omap3-beagle") ||
+- of_machine_is_compatible("timll,omap3-devkit8000")) {
++ if (of_machine_is_compatible("ti,omap3-beagle-ab4")) {
+ quirk_unreliable_oscillator = true;
+ counter_32k = -ENODEV;
+ }
+@@ -695,9 +694,9 @@ static int __init dmtimer_percpu_quirk_init(struct device_node *np, u32 pa)
+ return 0;
+ }
+
+- if (pa == 0x48034000) /* dra7 dmtimer3 */
++ if (pa == 0x4882c000) /* dra7 dmtimer15 */
+ return dmtimer_percpu_timer_init(np, 0);
+- else if (pa == 0x48036000) /* dra7 dmtimer4 */
++ else if (pa == 0x4882e000) /* dra7 dmtimer16 */
+ return dmtimer_percpu_timer_init(np, 1);
+
+ return 0;
+diff --git a/drivers/comedi/drivers/dt9812.c b/drivers/comedi/drivers/dt9812.c
+index 634f57730c1e0..704b04d2980d3 100644
+--- a/drivers/comedi/drivers/dt9812.c
++++ b/drivers/comedi/drivers/dt9812.c
+@@ -32,6 +32,7 @@
+ #include <linux/kernel.h>
+ #include <linux/module.h>
+ #include <linux/errno.h>
++#include <linux/slab.h>
+ #include <linux/uaccess.h>
+
+ #include "../comedi_usb.h"
+@@ -237,22 +238,42 @@ static int dt9812_read_info(struct comedi_device *dev,
+ {
+ struct usb_device *usb = comedi_to_usb_dev(dev);
+ struct dt9812_private *devpriv = dev->private;
+- struct dt9812_usb_cmd cmd;
++ struct dt9812_usb_cmd *cmd;
++ size_t tbuf_size;
+ int count, ret;
++ void *tbuf;
+
+- cmd.cmd = cpu_to_le32(DT9812_R_FLASH_DATA);
+- cmd.u.flash_data_info.address =
++ tbuf_size = max(sizeof(*cmd), buf_size);
++
++ tbuf = kzalloc(tbuf_size, GFP_KERNEL);
++ if (!tbuf)
++ return -ENOMEM;
++
++ cmd = tbuf;
++
++ cmd->cmd = cpu_to_le32(DT9812_R_FLASH_DATA);
++ cmd->u.flash_data_info.address =
+ cpu_to_le16(DT9812_DIAGS_BOARD_INFO_ADDR + offset);
+- cmd.u.flash_data_info.numbytes = cpu_to_le16(buf_size);
++ cmd->u.flash_data_info.numbytes = cpu_to_le16(buf_size);
+
+ /* DT9812 only responds to 32 byte writes!! */
+ ret = usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
+- &cmd, 32, &count, DT9812_USB_TIMEOUT);
++ cmd, sizeof(*cmd), &count, DT9812_USB_TIMEOUT);
+ if (ret)
+- return ret;
++ goto out;
++
++ ret = usb_bulk_msg(usb, usb_rcvbulkpipe(usb, devpriv->cmd_rd.addr),
++ tbuf, buf_size, &count, DT9812_USB_TIMEOUT);
++ if (!ret) {
++ if (count == buf_size)
++ memcpy(buf, tbuf, buf_size);
++ else
++ ret = -EREMOTEIO;
++ }
++out:
++ kfree(tbuf);
+
+- return usb_bulk_msg(usb, usb_rcvbulkpipe(usb, devpriv->cmd_rd.addr),
+- buf, buf_size, &count, DT9812_USB_TIMEOUT);
++ return ret;
+ }
+
+ static int dt9812_read_multiple_registers(struct comedi_device *dev,
+@@ -261,22 +282,42 @@ static int dt9812_read_multiple_registers(struct comedi_device *dev,
+ {
+ struct usb_device *usb = comedi_to_usb_dev(dev);
+ struct dt9812_private *devpriv = dev->private;
+- struct dt9812_usb_cmd cmd;
++ struct dt9812_usb_cmd *cmd;
+ int i, count, ret;
++ size_t buf_size;
++ void *buf;
+
+- cmd.cmd = cpu_to_le32(DT9812_R_MULTI_BYTE_REG);
+- cmd.u.read_multi_info.count = reg_count;
++ buf_size = max_t(size_t, sizeof(*cmd), reg_count);
++
++ buf = kzalloc(buf_size, GFP_KERNEL);
++ if (!buf)
++ return -ENOMEM;
++
++ cmd = buf;
++
++ cmd->cmd = cpu_to_le32(DT9812_R_MULTI_BYTE_REG);
++ cmd->u.read_multi_info.count = reg_count;
+ for (i = 0; i < reg_count; i++)
+- cmd.u.read_multi_info.address[i] = address[i];
++ cmd->u.read_multi_info.address[i] = address[i];
+
+ /* DT9812 only responds to 32 byte writes!! */
+ ret = usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
+- &cmd, 32, &count, DT9812_USB_TIMEOUT);
++ cmd, sizeof(*cmd), &count, DT9812_USB_TIMEOUT);
+ if (ret)
+- return ret;
++ goto out;
++
++ ret = usb_bulk_msg(usb, usb_rcvbulkpipe(usb, devpriv->cmd_rd.addr),
++ buf, reg_count, &count, DT9812_USB_TIMEOUT);
++ if (!ret) {
++ if (count == reg_count)
++ memcpy(value, buf, reg_count);
++ else
++ ret = -EREMOTEIO;
++ }
++out:
++ kfree(buf);
+
+- return usb_bulk_msg(usb, usb_rcvbulkpipe(usb, devpriv->cmd_rd.addr),
+- value, reg_count, &count, DT9812_USB_TIMEOUT);
++ return ret;
+ }
+
+ static int dt9812_write_multiple_registers(struct comedi_device *dev,
+@@ -285,19 +326,27 @@ static int dt9812_write_multiple_registers(struct comedi_device *dev,
+ {
+ struct usb_device *usb = comedi_to_usb_dev(dev);
+ struct dt9812_private *devpriv = dev->private;
+- struct dt9812_usb_cmd cmd;
++ struct dt9812_usb_cmd *cmd;
+ int i, count;
++ int ret;
++
++ cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
++ if (!cmd)
++ return -ENOMEM;
+
+- cmd.cmd = cpu_to_le32(DT9812_W_MULTI_BYTE_REG);
+- cmd.u.read_multi_info.count = reg_count;
++ cmd->cmd = cpu_to_le32(DT9812_W_MULTI_BYTE_REG);
++ cmd->u.read_multi_info.count = reg_count;
+ for (i = 0; i < reg_count; i++) {
+- cmd.u.write_multi_info.write[i].address = address[i];
+- cmd.u.write_multi_info.write[i].value = value[i];
++ cmd->u.write_multi_info.write[i].address = address[i];
++ cmd->u.write_multi_info.write[i].value = value[i];
+ }
+
+ /* DT9812 only responds to 32 byte writes!! */
+- return usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
+- &cmd, 32, &count, DT9812_USB_TIMEOUT);
++ ret = usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
++ cmd, sizeof(*cmd), &count, DT9812_USB_TIMEOUT);
++ kfree(cmd);
++
++ return ret;
+ }
+
+ static int dt9812_rmw_multiple_registers(struct comedi_device *dev,
+@@ -306,17 +355,25 @@ static int dt9812_rmw_multiple_registers(struct comedi_device *dev,
+ {
+ struct usb_device *usb = comedi_to_usb_dev(dev);
+ struct dt9812_private *devpriv = dev->private;
+- struct dt9812_usb_cmd cmd;
++ struct dt9812_usb_cmd *cmd;
+ int i, count;
++ int ret;
++
++ cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
++ if (!cmd)
++ return -ENOMEM;
+
+- cmd.cmd = cpu_to_le32(DT9812_RMW_MULTI_BYTE_REG);
+- cmd.u.rmw_multi_info.count = reg_count;
++ cmd->cmd = cpu_to_le32(DT9812_RMW_MULTI_BYTE_REG);
++ cmd->u.rmw_multi_info.count = reg_count;
+ for (i = 0; i < reg_count; i++)
+- cmd.u.rmw_multi_info.rmw[i] = rmw[i];
++ cmd->u.rmw_multi_info.rmw[i] = rmw[i];
+
+ /* DT9812 only responds to 32 byte writes!! */
+- return usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
+- &cmd, 32, &count, DT9812_USB_TIMEOUT);
++ ret = usb_bulk_msg(usb, usb_sndbulkpipe(usb, devpriv->cmd_wr.addr),
++ cmd, sizeof(*cmd), &count, DT9812_USB_TIMEOUT);
++ kfree(cmd);
++
++ return ret;
+ }
+
+ static int dt9812_digital_in(struct comedi_device *dev, u8 *bits)
+diff --git a/drivers/comedi/drivers/ni_usb6501.c b/drivers/comedi/drivers/ni_usb6501.c
+index 5b6d9d783b2f7..c42987b74b1dc 100644
+--- a/drivers/comedi/drivers/ni_usb6501.c
++++ b/drivers/comedi/drivers/ni_usb6501.c
+@@ -144,6 +144,10 @@ static const u8 READ_COUNTER_RESPONSE[] = {0x00, 0x01, 0x00, 0x10,
+ 0x00, 0x00, 0x00, 0x02,
+ 0x00, 0x00, 0x00, 0x00};
+
++/* Largest supported packets */
++static const size_t TX_MAX_SIZE = sizeof(SET_PORT_DIR_REQUEST);
++static const size_t RX_MAX_SIZE = sizeof(READ_PORT_RESPONSE);
++
+ enum commands {
+ READ_PORT,
+ WRITE_PORT,
+@@ -501,6 +505,12 @@ static int ni6501_find_endpoints(struct comedi_device *dev)
+ if (!devpriv->ep_rx || !devpriv->ep_tx)
+ return -ENODEV;
+
++ if (usb_endpoint_maxp(devpriv->ep_rx) < RX_MAX_SIZE)
++ return -ENODEV;
++
++ if (usb_endpoint_maxp(devpriv->ep_tx) < TX_MAX_SIZE)
++ return -ENODEV;
++
+ return 0;
+ }
+
+diff --git a/drivers/comedi/drivers/vmk80xx.c b/drivers/comedi/drivers/vmk80xx.c
+index 9f920819cd742..4b00a9ea611ab 100644
+--- a/drivers/comedi/drivers/vmk80xx.c
++++ b/drivers/comedi/drivers/vmk80xx.c
+@@ -90,6 +90,9 @@ enum {
+ #define IC3_VERSION BIT(0)
+ #define IC6_VERSION BIT(1)
+
++#define MIN_BUF_SIZE 64
++#define PACKET_TIMEOUT 10000 /* ms */
++
+ enum vmk80xx_model {
+ VMK8055_MODEL,
+ VMK8061_MODEL
+@@ -157,22 +160,21 @@ static void vmk80xx_do_bulk_msg(struct comedi_device *dev)
+ __u8 rx_addr;
+ unsigned int tx_pipe;
+ unsigned int rx_pipe;
+- size_t size;
++ size_t tx_size;
++ size_t rx_size;
+
+ tx_addr = devpriv->ep_tx->bEndpointAddress;
+ rx_addr = devpriv->ep_rx->bEndpointAddress;
+ tx_pipe = usb_sndbulkpipe(usb, tx_addr);
+ rx_pipe = usb_rcvbulkpipe(usb, rx_addr);
++ tx_size = usb_endpoint_maxp(devpriv->ep_tx);
++ rx_size = usb_endpoint_maxp(devpriv->ep_rx);
+
+- /*
+- * The max packet size attributes of the K8061
+- * input/output endpoints are identical
+- */
+- size = usb_endpoint_maxp(devpriv->ep_tx);
++ usb_bulk_msg(usb, tx_pipe, devpriv->usb_tx_buf, tx_size, NULL,
++ PACKET_TIMEOUT);
+
+- usb_bulk_msg(usb, tx_pipe, devpriv->usb_tx_buf,
+- size, NULL, devpriv->ep_tx->bInterval);
+- usb_bulk_msg(usb, rx_pipe, devpriv->usb_rx_buf, size, NULL, HZ * 10);
++ usb_bulk_msg(usb, rx_pipe, devpriv->usb_rx_buf, rx_size, NULL,
++ PACKET_TIMEOUT);
+ }
+
+ static int vmk80xx_read_packet(struct comedi_device *dev)
+@@ -191,7 +193,7 @@ static int vmk80xx_read_packet(struct comedi_device *dev)
+ pipe = usb_rcvintpipe(usb, ep->bEndpointAddress);
+ return usb_interrupt_msg(usb, pipe, devpriv->usb_rx_buf,
+ usb_endpoint_maxp(ep), NULL,
+- HZ * 10);
++ PACKET_TIMEOUT);
+ }
+
+ static int vmk80xx_write_packet(struct comedi_device *dev, int cmd)
+@@ -212,7 +214,7 @@ static int vmk80xx_write_packet(struct comedi_device *dev, int cmd)
+ pipe = usb_sndintpipe(usb, ep->bEndpointAddress);
+ return usb_interrupt_msg(usb, pipe, devpriv->usb_tx_buf,
+ usb_endpoint_maxp(ep), NULL,
+- HZ * 10);
++ PACKET_TIMEOUT);
+ }
+
+ static int vmk80xx_reset_device(struct comedi_device *dev)
+@@ -678,12 +680,12 @@ static int vmk80xx_alloc_usb_buffers(struct comedi_device *dev)
+ struct vmk80xx_private *devpriv = dev->private;
+ size_t size;
+
+- size = usb_endpoint_maxp(devpriv->ep_rx);
++ size = max(usb_endpoint_maxp(devpriv->ep_rx), MIN_BUF_SIZE);
+ devpriv->usb_rx_buf = kzalloc(size, GFP_KERNEL);
+ if (!devpriv->usb_rx_buf)
+ return -ENOMEM;
+
+- size = usb_endpoint_maxp(devpriv->ep_tx);
++ size = max(usb_endpoint_maxp(devpriv->ep_rx), MIN_BUF_SIZE);
+ devpriv->usb_tx_buf = kzalloc(size, GFP_KERNEL);
+ if (!devpriv->usb_tx_buf)
+ return -ENOMEM;
+diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
+index d4c27022b9c9b..e0ff09d66c96b 100644
+--- a/drivers/cpufreq/cppc_cpufreq.c
++++ b/drivers/cpufreq/cppc_cpufreq.c
+@@ -303,52 +303,48 @@ static u64 cppc_get_dmi_max_khz(void)
+
+ /*
+ * If CPPC lowest_freq and nominal_freq registers are exposed then we can
+- * use them to convert perf to freq and vice versa
+- *
+- * If the perf/freq point lies between Nominal and Lowest, we can treat
+- * (Low perf, Low freq) and (Nom Perf, Nom freq) as 2D co-ordinates of a line
+- * and extrapolate the rest
+- * For perf/freq > Nominal, we use the ratio perf:freq at Nominal for conversion
++ * use them to convert perf to freq and vice versa. The conversion is
++ * extrapolated as an affine function passing by the 2 points:
++ * - (Low perf, Low freq)
++ * - (Nominal perf, Nominal perf)
+ */
+ static unsigned int cppc_cpufreq_perf_to_khz(struct cppc_cpudata *cpu_data,
+ unsigned int perf)
+ {
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
++ s64 retval, offset = 0;
+ static u64 max_khz;
+ u64 mul, div;
+
+ if (caps->lowest_freq && caps->nominal_freq) {
+- if (perf >= caps->nominal_perf) {
+- mul = caps->nominal_freq;
+- div = caps->nominal_perf;
+- } else {
+- mul = caps->nominal_freq - caps->lowest_freq;
+- div = caps->nominal_perf - caps->lowest_perf;
+- }
++ mul = caps->nominal_freq - caps->lowest_freq;
++ div = caps->nominal_perf - caps->lowest_perf;
++ offset = caps->nominal_freq - div64_u64(caps->nominal_perf * mul, div);
+ } else {
+ if (!max_khz)
+ max_khz = cppc_get_dmi_max_khz();
+ mul = max_khz;
+ div = caps->highest_perf;
+ }
+- return (u64)perf * mul / div;
++
++ retval = offset + div64_u64(perf * mul, div);
++ if (retval >= 0)
++ return retval;
++ return 0;
+ }
+
+ static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu_data,
+ unsigned int freq)
+ {
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
++ s64 retval, offset = 0;
+ static u64 max_khz;
+ u64 mul, div;
+
+ if (caps->lowest_freq && caps->nominal_freq) {
+- if (freq >= caps->nominal_freq) {
+- mul = caps->nominal_perf;
+- div = caps->nominal_freq;
+- } else {
+- mul = caps->lowest_perf;
+- div = caps->lowest_freq;
+- }
++ mul = caps->nominal_perf - caps->lowest_perf;
++ div = caps->nominal_freq - caps->lowest_freq;
++ offset = caps->nominal_perf - div64_u64(caps->nominal_freq * mul, div);
+ } else {
+ if (!max_khz)
+ max_khz = cppc_get_dmi_max_khz();
+@@ -356,7 +352,10 @@ static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu_data,
+ div = max_khz;
+ }
+
+- return (u64)freq * mul / div;
++ retval = offset + div64_u64(freq * mul, div);
++ if (retval >= 0)
++ return retval;
++ return 0;
+ }
+
+ static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
+diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
+index 5782b15a8caad..eeac6d8092298 100644
+--- a/drivers/cpufreq/cpufreq.c
++++ b/drivers/cpufreq/cpufreq.c
+@@ -1004,10 +1004,9 @@ static struct kobj_type ktype_cpufreq = {
+ .release = cpufreq_sysfs_release,
+ };
+
+-static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
++static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
++ struct device *dev)
+ {
+- struct device *dev = get_cpu_device(cpu);
+-
+ if (unlikely(!dev))
+ return;
+
+@@ -1391,7 +1390,7 @@ static int cpufreq_online(unsigned int cpu)
+ if (new_policy) {
+ for_each_cpu(j, policy->related_cpus) {
+ per_cpu(cpufreq_cpu_data, j) = policy;
+- add_cpu_dev_symlink(policy, j);
++ add_cpu_dev_symlink(policy, j, get_cpu_device(j));
+ }
+
+ policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
+@@ -1403,7 +1402,7 @@ static int cpufreq_online(unsigned int cpu)
+
+ ret = freq_qos_add_request(&policy->constraints,
+ policy->min_freq_req, FREQ_QOS_MIN,
+- policy->min);
++ FREQ_QOS_MIN_DEFAULT_VALUE);
+ if (ret < 0) {
+ /*
+ * So we don't call freq_qos_remove_request() for an
+@@ -1423,7 +1422,7 @@ static int cpufreq_online(unsigned int cpu)
+
+ ret = freq_qos_add_request(&policy->constraints,
+ policy->max_freq_req, FREQ_QOS_MAX,
+- policy->max);
++ FREQ_QOS_MAX_DEFAULT_VALUE);
+ if (ret < 0) {
+ policy->max_freq_req = NULL;
+ goto out_destroy_policy;
+@@ -1565,7 +1564,7 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
+ /* Create sysfs link on CPU registration */
+ policy = per_cpu(cpufreq_cpu_data, cpu);
+ if (policy)
+- add_cpu_dev_symlink(policy, cpu);
++ add_cpu_dev_symlink(policy, cpu, dev);
+
+ return 0;
+ }
+@@ -2523,8 +2522,15 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
+ if (ret)
+ return ret;
+
++ /*
++ * Resolve policy min/max to available frequencies. It ensures
++ * no frequency resolution will neither overshoot the requested maximum
++ * nor undershoot the requested minimum.
++ */
+ policy->min = new_data.min;
+ policy->max = new_data.max;
++ policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
++ policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
+ trace_cpu_frequency_limits(policy);
+
+ policy->cached_target_freq = UINT_MAX;
+diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
+index 8c176b7dae415..8a2c6b58b6524 100644
+--- a/drivers/cpufreq/intel_pstate.c
++++ b/drivers/cpufreq/intel_pstate.c
+@@ -335,6 +335,8 @@ static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
+
+ static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
+
++#define CPPC_MAX_PERF U8_MAX
++
+ static void intel_pstate_set_itmt_prio(int cpu)
+ {
+ struct cppc_perf_caps cppc_perf;
+@@ -345,6 +347,14 @@ static void intel_pstate_set_itmt_prio(int cpu)
+ if (ret)
+ return;
+
++ /*
++ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff.
++ * In this case we can't use CPPC.highest_perf to enable ITMT.
++ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide.
++ */
++ if (cppc_perf.highest_perf == CPPC_MAX_PERF)
++ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));
++
+ /*
+ * The priorities can be set regardless of whether or not
+ * sched_set_itmt_support(true) has been called and it is valid to
+@@ -537,7 +547,8 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)
+ * scaling factor is too high, so recompute it to make the HWP_CAP
+ * highest performance correspond to the maximum turbo frequency.
+ */
+- if (turbo_freq < cpu->pstate.turbo_pstate * scaling) {
++ cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling;
++ if (turbo_freq < cpu->pstate.turbo_freq) {
+ cpu->pstate.turbo_freq = turbo_freq;
+ scaling = DIV_ROUND_UP(turbo_freq, cpu->pstate.turbo_pstate);
+ cpu->pstate.scaling = scaling;
+@@ -998,9 +1009,22 @@ static void intel_pstate_hwp_offline(struct cpudata *cpu)
+ */
+ value &= ~GENMASK_ULL(31, 24);
+ value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
+- WRITE_ONCE(cpu->hwp_req_cached, value);
++ /*
++ * However, make sure that EPP will be set to "performance" when
++ * the CPU is brought back online again and the "performance"
++ * scaling algorithm is still in effect.
++ */
++ cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
+ }
+
++ /*
++ * Clear the desired perf field in the cached HWP request value to
++ * prevent nonzero desired values from being leaked into the active
++ * mode.
++ */
++ value &= ~HWP_DESIRED_PERF(~0L);
++ WRITE_ONCE(cpu->hwp_req_cached, value);
++
+ value &= ~GENMASK_ULL(31, 0);
+ min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
+
+@@ -2241,6 +2265,7 @@ static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
+ X86_MATCH(BROADWELL_D, core_funcs),
+ X86_MATCH(BROADWELL_X, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
++ X86_MATCH(ICELAKE_X, core_funcs),
+ {}
+ };
+
+@@ -2902,6 +2927,27 @@ static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+ return intel_pstate_cpu_exit(policy);
+ }
+
++static int intel_cpufreq_suspend(struct cpufreq_policy *policy)
++{
++ intel_pstate_suspend(policy);
++
++ if (hwp_active) {
++ struct cpudata *cpu = all_cpu_data[policy->cpu];
++ u64 value = READ_ONCE(cpu->hwp_req_cached);
++
++ /*
++ * Clear the desired perf field in MSR_HWP_REQUEST in case
++ * intel_cpufreq_adjust_perf() is in use and the last value
++ * written by it may not be suitable.
++ */
++ value &= ~HWP_DESIRED_PERF(~0L);
++ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
++ WRITE_ONCE(cpu->hwp_req_cached, value);
++ }
++
++ return 0;
++}
++
+ static struct cpufreq_driver intel_cpufreq = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = intel_cpufreq_verify_policy,
+@@ -2911,7 +2957,7 @@ static struct cpufreq_driver intel_cpufreq = {
+ .exit = intel_cpufreq_cpu_exit,
+ .offline = intel_cpufreq_cpu_offline,
+ .online = intel_pstate_cpu_online,
+- .suspend = intel_pstate_suspend,
++ .suspend = intel_cpufreq_suspend,
+ .resume = intel_pstate_resume,
+ .update_limits = intel_pstate_update_limits,
+ .name = "intel_cpufreq",
+diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
+index a2be0df7e1747..bb2f59fd0de43 100644
+--- a/drivers/cpufreq/qcom-cpufreq-hw.c
++++ b/drivers/cpufreq/qcom-cpufreq-hw.c
+@@ -24,12 +24,16 @@
+ #define CLK_HW_DIV 2
+ #define LUT_TURBO_IND 1
+
++#define GT_IRQ_STATUS BIT(2)
++
+ #define HZ_PER_KHZ 1000
+
+ struct qcom_cpufreq_soc_data {
+ u32 reg_enable;
++ u32 reg_domain_state;
+ u32 reg_freq_lut;
+ u32 reg_volt_lut;
++ u32 reg_intr_clr;
+ u32 reg_current_vote;
+ u32 reg_perf_state;
+ u8 lut_row_size;
+@@ -266,28 +270,31 @@ static void qcom_get_related_cpus(int index, struct cpumask *m)
+ }
+ }
+
+-static unsigned int qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data)
++static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data)
+ {
+- unsigned int val = readl_relaxed(data->base + data->soc_data->reg_current_vote);
++ unsigned int lval;
++
++ if (data->soc_data->reg_current_vote)
++ lval = readl_relaxed(data->base + data->soc_data->reg_current_vote) & 0x3ff;
++ else
++ lval = readl_relaxed(data->base + data->soc_data->reg_domain_state) & 0xff;
+
+- return (val & 0x3FF) * 19200;
++ return lval * xo_rate;
+ }
+
+ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data)
+ {
+ unsigned long max_capacity, capacity, freq_hz, throttled_freq;
+ struct cpufreq_policy *policy = data->policy;
+- int cpu = cpumask_first(policy->cpus);
++ int cpu = cpumask_first(policy->related_cpus);
+ struct device *dev = get_cpu_device(cpu);
+ struct dev_pm_opp *opp;
+- unsigned int freq;
+
+ /*
+ * Get the h/w throttled frequency, normalize it using the
+ * registered opp table and use it to calculate thermal pressure.
+ */
+- freq = qcom_lmh_get_throttle_freq(data);
+- freq_hz = freq * HZ_PER_KHZ;
++ freq_hz = qcom_lmh_get_throttle_freq(data);
+
+ opp = dev_pm_opp_find_freq_floor(dev, &freq_hz);
+ if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)
+@@ -304,7 +311,8 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data)
+ if (capacity > max_capacity)
+ capacity = max_capacity;
+
+- arch_set_thermal_pressure(policy->cpus, max_capacity - capacity);
++ arch_set_thermal_pressure(policy->related_cpus,
++ max_capacity - capacity);
+
+ /*
+ * In the unlikely case policy is unregistered do not enable
+@@ -342,9 +350,13 @@ static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data)
+
+ /* Disable interrupt and enable polling */
+ disable_irq_nosync(c_data->throttle_irq);
+- qcom_lmh_dcvs_notify(c_data);
++ schedule_delayed_work(&c_data->throttle_work, 0);
+
+- return 0;
++ if (c_data->soc_data->reg_intr_clr)
++ writel_relaxed(GT_IRQ_STATUS,
++ c_data->base + c_data->soc_data->reg_intr_clr);
++
++ return IRQ_HANDLED;
+ }
+
+ static const struct qcom_cpufreq_soc_data qcom_soc_data = {
+@@ -358,8 +370,10 @@ static const struct qcom_cpufreq_soc_data qcom_soc_data = {
+
+ static const struct qcom_cpufreq_soc_data epss_soc_data = {
+ .reg_enable = 0x0,
++ .reg_domain_state = 0x20,
+ .reg_freq_lut = 0x100,
+ .reg_volt_lut = 0x200,
++ .reg_intr_clr = 0x308,
+ .reg_perf_state = 0x320,
+ .lut_row_size = 4,
+ };
+diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c
+index d1744b5d96190..6dfa86971a757 100644
+--- a/drivers/cpufreq/qcom-cpufreq-nvmem.c
++++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c
+@@ -130,7 +130,7 @@ static void get_krait_bin_format_b(struct device *cpu_dev,
+ }
+
+ /* Check PVS_BLOW_STATUS */
+- pte_efuse = *(((u32 *)buf) + 4);
++ pte_efuse = *(((u32 *)buf) + 1);
+ pte_efuse &= BIT(21);
+ if (pte_efuse) {
+ dev_dbg(cpu_dev, "PVS bin: %d\n", *pvs);
+diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
+index 2deed8d8773fa..75e1bf3a08f7c 100644
+--- a/drivers/cpufreq/sun50i-cpufreq-nvmem.c
++++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
+@@ -98,8 +98,10 @@ static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ ret = sun50i_cpufreq_get_efuse(&speed);
+- if (ret)
++ if (ret) {
++ kfree(opp_tables);
+ return ret;
++ }
+
+ snprintf(name, MAX_NAME_LEN, "speed%d", speed);
+
+diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c
+index 53ec9585ccd44..469e18547d06c 100644
+--- a/drivers/cpuidle/sysfs.c
++++ b/drivers/cpuidle/sysfs.c
+@@ -488,6 +488,7 @@ static int cpuidle_add_state_sysfs(struct cpuidle_device *device)
+ &kdev->kobj, "state%d", i);
+ if (ret) {
+ kobject_put(&kobj->kobj);
++ kfree(kobj);
+ goto error_state;
+ }
+ cpuidle_add_s2idle_attr_group(kobj);
+@@ -619,6 +620,7 @@ static int cpuidle_add_driver_sysfs(struct cpuidle_device *dev)
+ &kdev->kobj, "driver");
+ if (ret) {
+ kobject_put(&kdrv->kobj);
++ kfree(kdrv);
+ return ret;
+ }
+
+@@ -705,7 +707,6 @@ int cpuidle_add_sysfs(struct cpuidle_device *dev)
+ if (!kdev)
+ return -ENOMEM;
+ kdev->dev = dev;
+- dev->kobj_dev = kdev;
+
+ init_completion(&kdev->kobj_unregister);
+
+@@ -713,9 +714,11 @@ int cpuidle_add_sysfs(struct cpuidle_device *dev)
+ "cpuidle");
+ if (error) {
+ kobject_put(&kdev->kobj);
++ kfree(kdev);
+ return error;
+ }
+
++ dev->kobj_dev = kdev;
+ kobject_uevent(&kdev->kobj, KOBJ_ADD);
+
+ return 0;
+diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c
+index 54ae8d16e4931..35e3cadccac2b 100644
+--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c
++++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c
+@@ -11,6 +11,7 @@
+ * You could find a link for the datasheet in Documentation/arm/sunxi.rst
+ */
+
++#include <linux/bottom_half.h>
+ #include <linux/crypto.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/io.h>
+@@ -283,7 +284,9 @@ static int sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq)
+
+ flow = rctx->flow;
+ err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm));
++ local_bh_disable();
+ crypto_finalize_skcipher_request(engine, breq, err);
++ local_bh_enable();
+ return 0;
+ }
+
+diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c
+index 88194718a806c..859b7522faaac 100644
+--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c
++++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c
+@@ -9,6 +9,7 @@
+ *
+ * You could find the datasheet in Documentation/arm/sunxi.rst
+ */
++#include <linux/bottom_half.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/pm_runtime.h>
+ #include <linux/scatterlist.h>
+@@ -414,6 +415,8 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq)
+ theend:
+ kfree(buf);
+ kfree(result);
++ local_bh_disable();
+ crypto_finalize_hash_request(engine, breq, err);
++ local_bh_enable();
+ return 0;
+ }
+diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
+index 9ef1c85c4aaa5..554e400d41cad 100644
+--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
++++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c
+@@ -11,6 +11,7 @@
+ * You could find a link for the datasheet in Documentation/arm/sunxi.rst
+ */
+
++#include <linux/bottom_half.h>
+ #include <linux/crypto.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/io.h>
+@@ -274,7 +275,9 @@ static int sun8i_ss_handle_cipher_request(struct crypto_engine *engine, void *ar
+ struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
+
+ err = sun8i_ss_cipher(breq);
++ local_bh_disable();
+ crypto_finalize_skcipher_request(engine, breq, err);
++ local_bh_enable();
+
+ return 0;
+ }
+diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
+index 80e89066dbd1a..319fe3279a716 100644
+--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
++++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
+@@ -30,6 +30,8 @@
+ static const struct ss_variant ss_a80_variant = {
+ .alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES,
+ },
++ .alg_hash = { SS_ID_NOTSUPP, SS_ID_NOTSUPP, SS_ID_NOTSUPP, SS_ID_NOTSUPP,
++ },
+ .op_mode = { SS_OP_ECB, SS_OP_CBC,
+ },
+ .ss_clks = {
+diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
+index 3c073eb3db038..1a71ed49d2333 100644
+--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
++++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
+@@ -9,6 +9,7 @@
+ *
+ * You could find the datasheet in Documentation/arm/sunxi.rst
+ */
++#include <linux/bottom_half.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/pm_runtime.h>
+ #include <linux/scatterlist.h>
+@@ -442,6 +443,8 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq)
+ theend:
+ kfree(pad);
+ kfree(result);
++ local_bh_disable();
+ crypto_finalize_hash_request(engine, breq, err);
++ local_bh_enable();
+ return 0;
+ }
+diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c
+index c6865cbd334b2..e79514fce731f 100644
+--- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c
++++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c
+@@ -265,7 +265,9 @@ static int meson_handle_cipher_request(struct crypto_engine *engine,
+ struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
+
+ err = meson_cipher(breq);
++ local_bh_disable();
+ crypto_finalize_skcipher_request(engine, breq, err);
++ local_bh_enable();
+
+ return 0;
+ }
+diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
+index 9391ccc03382d..fe05584031914 100644
+--- a/drivers/crypto/atmel-aes.c
++++ b/drivers/crypto/atmel-aes.c
+@@ -960,6 +960,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
+ ctx = crypto_tfm_ctx(areq->tfm);
+
+ dd->areq = areq;
++ dd->ctx = ctx;
+ start_async = (areq != new_areq);
+ dd->is_async = start_async;
+
+@@ -1274,7 +1275,6 @@ static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
+ ctx->base.dd = dd;
+- ctx->base.dd->ctx = &ctx->base;
+ ctx->base.start = atmel_aes_start;
+
+ return 0;
+@@ -1291,7 +1291,6 @@ static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
+ ctx->base.dd = dd;
+- ctx->base.dd->ctx = &ctx->base;
+ ctx->base.start = atmel_aes_ctr_start;
+
+ return 0;
+@@ -1783,7 +1782,6 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm)
+
+ crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
+ ctx->base.dd = dd;
+- ctx->base.dd->ctx = &ctx->base;
+ ctx->base.start = atmel_aes_gcm_start;
+
+ return 0;
+@@ -1927,7 +1925,6 @@ static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) +
+ crypto_skcipher_reqsize(ctx->fallback_tfm));
+ ctx->base.dd = dd;
+- ctx->base.dd->ctx = &ctx->base;
+ ctx->base.start = atmel_aes_xts_start;
+
+ return 0;
+@@ -2154,7 +2151,6 @@ static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
+ crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
+ auth_reqsize));
+ ctx->base.dd = dd;
+- ctx->base.dd->ctx = &ctx->base;
+ ctx->base.start = atmel_aes_authenc_start;
+
+ return 0;
+diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
+index 8697ae53b0633..d3d8bb0a69900 100644
+--- a/drivers/crypto/caam/caamalg.c
++++ b/drivers/crypto/caam/caamalg.c
+@@ -1533,6 +1533,9 @@ static int aead_do_one_req(struct crypto_engine *engine, void *areq)
+
+ ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
+
++ if (ret == -ENOSPC && engine->retry_support)
++ return ret;
++
+ if (ret != -EINPROGRESS) {
+ aead_unmap(ctx->jrdev, rctx->edesc, req);
+ kfree(rctx->edesc);
+@@ -1762,6 +1765,9 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
+
+ ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
+
++ if (ret == -ENOSPC && engine->retry_support)
++ return ret;
++
+ if (ret != -EINPROGRESS) {
+ skcipher_unmap(ctx->jrdev, rctx->edesc, req);
+ kfree(rctx->edesc);
+diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
+index 8b8ed77d8715d..6753f0e6e55d1 100644
+--- a/drivers/crypto/caam/caamalg_qi2.c
++++ b/drivers/crypto/caam/caamalg_qi2.c
+@@ -5470,7 +5470,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
+ dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
+ dpaa2_fd_set_flc(&fd, req->flc_dma);
+
+- ppriv = this_cpu_ptr(priv->ppriv);
++ ppriv = raw_cpu_ptr(priv->ppriv);
+ for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
+ err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
+ &fd);
+diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
+index e8a6d8bc43b5d..36ef738e4a181 100644
+--- a/drivers/crypto/caam/caamhash.c
++++ b/drivers/crypto/caam/caamhash.c
+@@ -765,6 +765,9 @@ static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
+
+ ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
+
++ if (ret == -ENOSPC && engine->retry_support)
++ return ret;
++
+ if (ret != -EINPROGRESS) {
+ ahash_unmap(jrdev, state->edesc, req, 0);
+ kfree(state->edesc);
+diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
+index e313233ec6de7..8867275767101 100644
+--- a/drivers/crypto/caam/caampkc.c
++++ b/drivers/crypto/caam/caampkc.c
+@@ -380,6 +380,9 @@ static int akcipher_do_one_req(struct crypto_engine *engine, void *areq)
+
+ ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req);
+
++ if (ret == -ENOSPC && engine->retry_support)
++ return ret;
++
+ if (ret != -EINPROGRESS) {
+ rsa_pub_unmap(jrdev, req_ctx->edesc, req);
+ rsa_io_unmap(jrdev, req_ctx->edesc, req);
+@@ -1153,16 +1156,27 @@ static struct caam_akcipher_alg caam_rsa = {
+ int caam_pkc_init(struct device *ctrldev)
+ {
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
+- u32 pk_inst;
++ u32 pk_inst, pkha;
+ int err;
+ init_done = false;
+
+ /* Determine public key hardware accelerator presence. */
+- if (priv->era < 10)
++ if (priv->era < 10) {
+ pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
+ CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
+- else
+- pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK;
++ } else {
++ pkha = rd_reg32(&priv->ctrl->vreg.pkha);
++ pk_inst = pkha & CHA_VER_NUM_MASK;
++
++ /*
++ * Newer CAAMs support partially disabled functionality. If this is the
++ * case, the number is non-zero, but this bit is set to indicate that
++ * no encryption or decryption is supported. Only signing and verifying
++ * is supported.
++ */
++ if (pkha & CHA_VER_MISC_PKHA_NO_CRYPT)
++ pk_inst = 0;
++ }
+
+ /* Do not register algorithms if PKHA is not present. */
+ if (!pk_inst)
+diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
+index af61f3a2c0d46..3738625c02509 100644
+--- a/drivers/crypto/caam/regs.h
++++ b/drivers/crypto/caam/regs.h
+@@ -322,6 +322,9 @@ struct version_regs {
+ /* CHA Miscellaneous Information - AESA_MISC specific */
+ #define CHA_VER_MISC_AES_GCM BIT(1 + CHA_VER_MISC_SHIFT)
+
++/* CHA Miscellaneous Information - PKHA_MISC specific */
++#define CHA_VER_MISC_PKHA_NO_CRYPT BIT(7 + CHA_VER_MISC_SHIFT)
++
+ /*
+ * caam_perfmon - Performance Monitor/Secure Memory Status/
+ * CAAM Global Status/Component Version IDs
+diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
+index d718db224be42..7d4b4ad1db1f3 100644
+--- a/drivers/crypto/ccp/ccp-dmaengine.c
++++ b/drivers/crypto/ccp/ccp-dmaengine.c
+@@ -632,6 +632,20 @@ static int ccp_terminate_all(struct dma_chan *dma_chan)
+ return 0;
+ }
+
++static void ccp_dma_release(struct ccp_device *ccp)
++{
++ struct ccp_dma_chan *chan;
++ struct dma_chan *dma_chan;
++ unsigned int i;
++
++ for (i = 0; i < ccp->cmd_q_count; i++) {
++ chan = ccp->ccp_dma_chan + i;
++ dma_chan = &chan->dma_chan;
++ tasklet_kill(&chan->cleanup_tasklet);
++ list_del_rcu(&dma_chan->device_node);
++ }
++}
++
+ int ccp_dmaengine_register(struct ccp_device *ccp)
+ {
+ struct ccp_dma_chan *chan;
+@@ -736,6 +750,7 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
+ return 0;
+
+ err_reg:
++ ccp_dma_release(ccp);
+ kmem_cache_destroy(ccp->dma_desc_cache);
+
+ err_cache:
+@@ -752,6 +767,7 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp)
+ return;
+
+ dma_async_device_unregister(dma_dev);
++ ccp_dma_release(ccp);
+
+ kmem_cache_destroy(ccp->dma_desc_cache);
+ kmem_cache_destroy(ccp->dma_cmd_cache);
+diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
+index 2ecb0e1f65d8d..5040726fc119c 100644
+--- a/drivers/crypto/ccp/sev-dev.c
++++ b/drivers/crypto/ccp/sev-dev.c
+@@ -241,7 +241,7 @@ static int __sev_platform_init_locked(int *error)
+ struct psp_device *psp = psp_master;
+ struct sev_data_init data;
+ struct sev_device *sev;
+- int rc = 0;
++ int psp_ret = -1, rc = 0;
+
+ if (!psp || !psp->sev_data)
+ return -ENODEV;
+@@ -266,7 +266,21 @@ static int __sev_platform_init_locked(int *error)
+ data.tmr_len = SEV_ES_TMR_SIZE;
+ }
+
+- rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
++ rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret);
++ if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
++ /*
++ * Initialization command returned an integrity check failure
++ * status code, meaning that firmware load and validation of SEV
++ * related persistent data has failed. Retrying the
++ * initialization function should succeed by replacing the state
++ * with a reset state.
++ */
++ dev_dbg(sev->dev, "SEV: retrying INIT command");
++ rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret);
++ }
++ if (error)
++ *error = psp_ret;
++
+ if (rc)
+ return rc;
+
+@@ -1091,18 +1105,6 @@ void sev_pci_init(void)
+
+ /* Initialize the platform */
+ rc = sev_platform_init(&error);
+- if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) {
+- /*
+- * INIT command returned an integrity check failure
+- * status code, meaning that firmware load and
+- * validation of SEV related persistent data has
+- * failed and persistent state has been erased.
+- * Retrying INIT command here should succeed.
+- */
+- dev_dbg(sev->dev, "SEV: retrying INIT command");
+- rc = sev_platform_init(&error);
+- }
+-
+ if (rc) {
+ dev_err(sev->dev, "SEV: failed to INIT error %#x\n", error);
+ return;
+diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c
+index a5e041d9d2cf1..11e0278c8631d 100644
+--- a/drivers/crypto/ccree/cc_buffer_mgr.c
++++ b/drivers/crypto/ccree/cc_buffer_mgr.c
+@@ -258,6 +258,13 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg,
+ {
+ int ret = 0;
+
++ if (!nbytes) {
++ *mapped_nents = 0;
++ *lbytes = 0;
++ *nents = 0;
++ return 0;
++ }
++
+ *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes);
+ if (*nents > max_sg_nents) {
+ *nents = 0;
+diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c
+index 78833491f534d..309da6334a0a0 100644
+--- a/drivers/crypto/ccree/cc_cipher.c
++++ b/drivers/crypto/ccree/cc_cipher.c
+@@ -257,8 +257,8 @@ static void cc_cipher_exit(struct crypto_tfm *tfm)
+ &ctx_p->user.key_dma_addr);
+
+ /* Free key buffer in context */
+- kfree_sensitive(ctx_p->user.key);
+ dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key);
++ kfree_sensitive(ctx_p->user.key);
+ }
+
+ struct tdes_keys {
+diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
+index e599ac6dc162a..790fa9058a36d 100644
+--- a/drivers/crypto/ccree/cc_driver.c
++++ b/drivers/crypto/ccree/cc_driver.c
+@@ -103,7 +103,8 @@ MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);
+ static void init_cc_cache_params(struct cc_drvdata *drvdata)
+ {
+ struct device *dev = drvdata_to_dev(drvdata);
+- u32 cache_params, ace_const, val, mask;
++ u32 cache_params, ace_const, val;
++ u64 mask;
+
+ /* compute CC_AXIM_CACHE_PARAMS */
+ cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
+diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c
+index c1c2b1d866639..f2be0a7d7f7ac 100644
+--- a/drivers/crypto/gemini/sl3516-ce-cipher.c
++++ b/drivers/crypto/gemini/sl3516-ce-cipher.c
+@@ -264,7 +264,9 @@ static int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *a
+ struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
+
+ err = sl3516_ce_cipher(breq);
++ local_bh_disable();
+ crypto_finalize_skcipher_request(engine, breq, err);
++ local_bh_enable();
+
+ return 0;
+ }
+diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+index a032c192ef1d6..7ba7641723a0b 100644
+--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
++++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+@@ -1865,7 +1865,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
+ */
+ if (memcmp(ptr, p, ctx->key_sz) == 0) {
+ dev_err(dev, "gx is p!\n");
+- return -EINVAL;
++ goto err;
+ } else if (memcmp(ptr, p, ctx->key_sz) > 0) {
+ hpre_curve25519_src_modulo_p(ptr);
+ }
+diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
+index 369562d34d66a..b616d2d8e7736 100644
+--- a/drivers/crypto/hisilicon/qm.c
++++ b/drivers/crypto/hisilicon/qm.c
+@@ -4107,7 +4107,7 @@ static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)
+ static int qm_vf_read_qos(struct hisi_qm *qm)
+ {
+ int cnt = 0;
+- int ret;
++ int ret = -EINVAL;
+
+ /* reset mailbox qos val */
+ qm->mb_qos = 0;
+@@ -5986,7 +5986,7 @@ int hisi_qm_resume(struct device *dev)
+ if (ret)
+ pci_err(pdev, "failed to start qm(%d)\n", ret);
+
+- return 0;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(hisi_qm_resume);
+
+diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c
+index 6a45bd23b3635..090920ed50c8f 100644
+--- a/drivers/crypto/hisilicon/sec2/sec_crypto.c
++++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c
+@@ -2284,9 +2284,10 @@ static int sec_aead_soft_crypto(struct sec_ctx *ctx,
+ struct aead_request *aead_req,
+ bool encrypt)
+ {
+- struct aead_request *subreq = aead_request_ctx(aead_req);
+ struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
+ struct device *dev = ctx->dev;
++ struct aead_request *subreq;
++ int ret;
+
+ /* Kunpeng920 aead mode not support input 0 size */
+ if (!a_ctx->fallback_aead_tfm) {
+@@ -2294,6 +2295,10 @@ static int sec_aead_soft_crypto(struct sec_ctx *ctx,
+ return -EINVAL;
+ }
+
++ subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);
++ if (!subreq)
++ return -ENOMEM;
++
+ aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
+ aead_request_set_callback(subreq, aead_req->base.flags,
+ aead_req->base.complete, aead_req->base.data);
+@@ -2301,8 +2306,13 @@ static int sec_aead_soft_crypto(struct sec_ctx *ctx,
+ aead_req->cryptlen, aead_req->iv);
+ aead_request_set_ad(subreq, aead_req->assoclen);
+
+- return encrypt ? crypto_aead_encrypt(subreq) :
+- crypto_aead_decrypt(subreq);
++ if (encrypt)
++ ret = crypto_aead_encrypt(subreq);
++ else
++ ret = crypto_aead_decrypt(subreq);
++ aead_request_free(subreq);
++
++ return ret;
+ }
+
+ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
+diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c
+index 90551bf38b523..03d239cfdf8c6 100644
+--- a/drivers/crypto/hisilicon/sec2/sec_main.c
++++ b/drivers/crypto/hisilicon/sec2/sec_main.c
+@@ -443,9 +443,11 @@ static int sec_engine_init(struct hisi_qm *qm)
+
+ writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG);
+
+- /* Enable sm4 extra mode, as ctr/ecb */
+- writel_relaxed(SEC_BD_ERR_CHK_EN0,
+- qm->io_base + SEC_BD_ERR_CHK_EN_REG0);
++ /* HW V2 enable sm4 extra mode, as ctr/ecb */
++ if (qm->ver < QM_HW_V3)
++ writel_relaxed(SEC_BD_ERR_CHK_EN0,
++ qm->io_base + SEC_BD_ERR_CHK_EN_REG0);
++
+ /* Enable sm4 xts mode multiple iv */
+ writel_relaxed(SEC_BD_ERR_CHK_EN1,
+ qm->io_base + SEC_BD_ERR_CHK_EN_REG1);
+diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
+index 146a55ac4b9b0..be1ad55a208f6 100644
+--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
++++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
+@@ -494,12 +494,11 @@ static ssize_t kvf_limits_store(struct device *dev,
+ {
+ struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
+ int lfs_num;
++ int ret;
+
+- if (kstrtoint(buf, 0, &lfs_num)) {
+- dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
+- lfs_num, num_online_cpus());
+- return -EINVAL;
+- }
++ ret = kstrtoint(buf, 0, &lfs_num);
++ if (ret)
++ return ret;
+ if (lfs_num < 1 || lfs_num > num_online_cpus()) {
+ dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
+ lfs_num, num_online_cpus());
+diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
+index dff34b3ec09e1..7c1b92aaab398 100644
+--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
++++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
+@@ -29,7 +29,8 @@ static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev,
+ bool found = false;
+ int i;
+
+- if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
++ if (eng_grp->g->engs_num < 0 ||
++ eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
+ dev_err(dev, "unsupported number of engines %d on octeontx2\n",
+ eng_grp->g->engs_num);
+ return bmap;
+diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+index a72723455df72..570074e23b60e 100644
+--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
++++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+@@ -1274,6 +1274,7 @@ static int aead_do_fallback(struct aead_request *req, bool is_enc)
+ req->base.complete, req->base.data);
+ aead_request_set_crypt(&rctx->fbk_req, req->src,
+ req->dst, req->cryptlen, req->iv);
++ aead_request_set_ad(&rctx->fbk_req, req->assoclen);
+ ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
+ crypto_aead_decrypt(&rctx->fbk_req);
+ } else {
+@@ -1633,16 +1634,13 @@ static inline int cpt_register_algs(void)
+ {
+ int i, err = 0;
+
+- if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
+- for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
+- otx2_cpt_skciphers[i].base.cra_flags &=
+- ~CRYPTO_ALG_DEAD;
++ for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
++ otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
+
+- err = crypto_register_skciphers(otx2_cpt_skciphers,
+- ARRAY_SIZE(otx2_cpt_skciphers));
+- if (err)
+- return err;
+- }
++ err = crypto_register_skciphers(otx2_cpt_skciphers,
++ ARRAY_SIZE(otx2_cpt_skciphers));
++ if (err)
++ return err;
+
+ for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
+ otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
+diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
+index d19e5ffb5104b..d6f9e2fe863d7 100644
+--- a/drivers/crypto/mxs-dcp.c
++++ b/drivers/crypto/mxs-dcp.c
+@@ -331,7 +331,7 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
+ memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128);
+ }
+
+- for_each_sg(req->src, src, sg_nents(src), i) {
++ for_each_sg(req->src, src, sg_nents(req->src), i) {
+ src_buf = sg_virt(src);
+ len = sg_dma_len(src);
+ tlen += len;
+diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
+index 9b968ac4ee7b6..a196bb8b17010 100644
+--- a/drivers/crypto/omap-aes.c
++++ b/drivers/crypto/omap-aes.c
+@@ -1302,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev)
+
+ static int omap_aes_resume(struct device *dev)
+ {
+- pm_runtime_resume_and_get(dev);
++ pm_runtime_get_sync(dev);
+ return 0;
+ }
+ #endif
+diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
+index 33d8e50dcbdac..88c0ded411f15 100644
+--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
++++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
+@@ -1,5 +1,6 @@
+ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
+ /* Copyright(c) 2020 Intel Corporation */
++#include <linux/iopoll.h>
+ #include <adf_accel_devices.h>
+ #include <adf_common_drv.h>
+ #include <adf_pf2vf_msg.h>
+@@ -161,6 +162,35 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev)
+ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0);
+ }
+
++static int adf_init_device(struct adf_accel_dev *accel_dev)
++{
++ void __iomem *addr;
++ u32 status;
++ u32 csr;
++ int ret;
++
++ addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
++
++ /* Temporarily mask PM interrupt */
++ csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2);
++ csr |= ADF_4XXX_PM_SOU;
++ ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr);
++
++ /* Set DRV_ACTIVE bit to power up the device */
++ ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE);
++
++ /* Poll status register to make sure the device is powered up */
++ ret = read_poll_timeout(ADF_CSR_RD, status,
++ status & ADF_4XXX_PM_INIT_STATE,
++ ADF_4XXX_PM_POLL_DELAY_US,
++ ADF_4XXX_PM_POLL_TIMEOUT_US, true, addr,
++ ADF_4XXX_PM_STATUS);
++ if (ret)
++ dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
++
++ return ret;
++}
++
+ static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
+ {
+ return 0;
+@@ -215,6 +245,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
+ hw_data->exit_arb = adf_exit_arb;
+ hw_data->get_arb_mapping = adf_get_arbiter_mapping;
+ hw_data->enable_ints = adf_enable_ints;
++ hw_data->init_device = adf_init_device;
+ hw_data->reset_device = adf_reset_flr;
+ hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
+ hw_data->uof_get_num_objs = uof_get_num_objs;
+diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
+index 4fe2a776293c2..924bac6feb372 100644
+--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
++++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
+@@ -62,6 +62,16 @@
+ #define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578)
+ #define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970)
+
++/* Power management */
++#define ADF_4XXX_PM_POLL_DELAY_US 20
++#define ADF_4XXX_PM_POLL_TIMEOUT_US USEC_PER_SEC
++#define ADF_4XXX_PM_STATUS (0x50A00C)
++#define ADF_4XXX_PM_INTERRUPT (0x50A028)
++#define ADF_4XXX_PM_DRV_ACTIVE BIT(20)
++#define ADF_4XXX_PM_INIT_STATE BIT(21)
++/* Power management source in ERRSOU2 and ERRMSK2 */
++#define ADF_4XXX_PM_SOU BIT(18)
++
+ /* Firmware Binaries */
+ #define ADF_4XXX_FW "qat_4xxx.bin"
+ #define ADF_4XXX_MMP "qat_4xxx_mmp.bin"
+diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c
+index 359fb7989dfbf..8fd44703115f2 100644
+--- a/drivers/crypto/qat/qat_4xxx/adf_drv.c
++++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c
+@@ -52,6 +52,13 @@ static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
+ if (ret)
+ goto err;
+
++ /* Temporarily set the number of crypto instances to zero to avoid
++ * registering the crypto algorithms.
++ * This will be removed when the algorithms will support the
++ * CRYPTO_TFM_REQ_MAY_BACKLOG flag
++ */
++ instances = 0;
++
+ for (i = 0; i < instances; i++) {
+ val = i;
+ bank = i * 2;
+diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h
+index 38c0af6d4e43e..580566cfcb04c 100644
+--- a/drivers/crypto/qat/qat_common/adf_accel_devices.h
++++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h
+@@ -166,6 +166,7 @@ struct adf_hw_device_data {
+ int (*init_arb)(struct adf_accel_dev *accel_dev);
+ void (*exit_arb)(struct adf_accel_dev *accel_dev);
+ const u32 *(*get_arb_mapping)(void);
++ int (*init_device)(struct adf_accel_dev *accel_dev);
+ void (*disable_iov)(struct adf_accel_dev *accel_dev);
+ void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,
+ bool enable);
+diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c
+index 60bc7b991d351..e3749e5817d94 100644
+--- a/drivers/crypto/qat/qat_common/adf_init.c
++++ b/drivers/crypto/qat/qat_common/adf_init.c
+@@ -79,6 +79,11 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
+ return -EFAULT;
+ }
+
++ if (hw_data->init_device && hw_data->init_device(accel_dev)) {
++ dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n");
++ return -EFAULT;
++ }
++
+ if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
+ dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
+ return -EFAULT;
+diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
+index 976b9ab7617cd..7ec81989beb03 100644
+--- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
++++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
+@@ -117,37 +117,19 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
+
+ mutex_lock(lock);
+
+- /* Check if PF2VF CSR is in use by remote function */
++ /* Check if the PFVF CSR is in use by remote function */
+ val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
+ if ((val & remote_in_use_mask) == remote_in_use_pattern) {
+ dev_dbg(&GET_DEV(accel_dev),
+- "PF2VF CSR in use by remote function\n");
++ "PFVF CSR in use by remote function\n");
+ ret = -EBUSY;
+ goto out;
+ }
+
+- /* Attempt to get ownership of PF2VF CSR */
+ msg &= ~local_in_use_mask;
+ msg |= local_in_use_pattern;
+- ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg);
+
+- /* Wait in case remote func also attempting to get ownership */
+- msleep(ADF_IOV_MSG_COLLISION_DETECT_DELAY);
+-
+- val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
+- if ((val & local_in_use_mask) != local_in_use_pattern) {
+- dev_dbg(&GET_DEV(accel_dev),
+- "PF2VF CSR in use by remote - collision detected\n");
+- ret = -EBUSY;
+- goto out;
+- }
+-
+- /*
+- * This function now owns the PV2VF CSR. The IN_USE_BY pattern must
+- * remain in the PF2VF CSR for all writes including ACK from remote
+- * until this local function relinquishes the CSR. Send the message
+- * by interrupting the remote.
+- */
++ /* Attempt to get ownership of the PFVF CSR */
+ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit);
+
+ /* Wait for confirmation from remote func it received the message */
+@@ -162,7 +144,14 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
+ ret = -EIO;
+ }
+
+- /* Finished with PF2VF CSR; relinquish it and leave msg in CSR */
++ if (val != msg) {
++ dev_dbg(&GET_DEV(accel_dev),
++ "Collision - PFVF CSR overwritten by remote function\n");
++ ret = -EIO;
++ goto out;
++ }
++
++ /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */
+ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask);
+ out:
+ mutex_unlock(lock);
+@@ -170,12 +159,13 @@ out:
+ }
+
+ /**
+- * adf_iov_putmsg() - send PF2VF message
++ * adf_iov_putmsg() - send PFVF message
+ * @accel_dev: Pointer to acceleration device.
+ * @msg: Message to send
+- * @vf_nr: VF number to which the message will be sent
++ * @vf_nr: VF number to which the message will be sent if on PF, ignored
++ * otherwise
+ *
+- * Function sends a message from the PF to a VF
++ * Function sends a message through the PFVF channel
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+@@ -204,6 +194,11 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info)
+
+ /* Read message from the VF */
+ msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr));
++ if (!(msg & ADF_VF2PF_INT)) {
++ dev_info(&GET_DEV(accel_dev),
++ "Spurious VF2PF interrupt, msg %X. Ignored\n", msg);
++ goto out;
++ }
+
+ /* To ACK, clear the VF2PFINT bit */
+ msg &= ~ADF_VF2PF_INT;
+@@ -287,6 +282,7 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info)
+ if (resp && adf_iov_putmsg(accel_dev, resp, vf_nr))
+ dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n");
+
++out:
+ /* re-enable interrupt on PF from this VF */
+ adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr));
+
+diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c
+index 7828a6573f3e2..2e300c255ab94 100644
+--- a/drivers/crypto/qat/qat_common/adf_vf_isr.c
++++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c
+@@ -101,6 +101,11 @@ static void adf_pf2vf_bh_handler(void *data)
+
+ /* Read the message from PF */
+ msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0));
++ if (!(msg & ADF_PF2VF_INT)) {
++ dev_info(&GET_DEV(accel_dev),
++ "Spurious PF2VF interrupt, msg %X. Ignored\n", msg);
++ goto out;
++ }
+
+ if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM))
+ /* Ignore legacy non-system (non-kernel) PF2VF messages */
+@@ -149,6 +154,7 @@ static void adf_pf2vf_bh_handler(void *data)
+ msg &= ~ADF_PF2VF_INT;
+ ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
+
++out:
+ /* Re-enable PF2VF interrupts */
+ adf_enable_pf2vf_interrupts(accel_dev);
+ return;
+diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
+index ece6776fbd53d..3efbb38836010 100644
+--- a/drivers/crypto/qat/qat_common/qat_crypto.c
++++ b/drivers/crypto/qat/qat_common/qat_crypto.c
+@@ -136,6 +136,13 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev)
+ if (ret)
+ goto err;
+
++ /* Temporarily set the number of crypto instances to zero to avoid
++ * registering the crypto algorithms.
++ * This will be removed when the algorithms will support the
++ * CRYPTO_TFM_REQ_MAY_BACKLOG flag
++ */
++ instances = 0;
++
+ for (i = 0; i < instances; i++) {
+ val = i;
+ snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
+diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
+index 290e2446a2f35..97a530171f07a 100644
+--- a/drivers/crypto/qce/aead.c
++++ b/drivers/crypto/qce/aead.c
+@@ -802,8 +802,8 @@ static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_devi
+
+ ret = crypto_register_aead(alg);
+ if (ret) {
+- kfree(tmpl);
+ dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);
++ kfree(tmpl);
+ return ret;
+ }
+
+diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
+index 8e6fcf2c21cc0..59159f5e64e52 100644
+--- a/drivers/crypto/qce/sha.c
++++ b/drivers/crypto/qce/sha.c
+@@ -498,8 +498,8 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def,
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+- kfree(tmpl);
+ dev_err(qce->dev, "%s registration failed\n", base->cra_name);
++ kfree(tmpl);
+ return ret;
+ }
+
+diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
+index 8ff10928f581d..3d27cd5210ef5 100644
+--- a/drivers/crypto/qce/skcipher.c
++++ b/drivers/crypto/qce/skcipher.c
+@@ -484,8 +484,8 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def,
+
+ ret = crypto_register_skcipher(alg);
+ if (ret) {
+- kfree(tmpl);
+ dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);
++ kfree(tmpl);
+ return ret;
+ }
+
+diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c
+index 99ba8d51d1020..031b5f701a0a3 100644
+--- a/drivers/crypto/qcom-rng.c
++++ b/drivers/crypto/qcom-rng.c
+@@ -8,6 +8,7 @@
+ #include <linux/clk.h>
+ #include <linux/crypto.h>
+ #include <linux/io.h>
++#include <linux/iopoll.h>
+ #include <linux/module.h>
+ #include <linux/of.h>
+ #include <linux/platform_device.h>
+@@ -43,16 +44,19 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max)
+ {
+ unsigned int currsize = 0;
+ u32 val;
++ int ret;
+
+ /* read random data from hardware */
+ do {
+- val = readl_relaxed(rng->base + PRNG_STATUS);
+- if (!(val & PRNG_STATUS_DATA_AVAIL))
+- break;
++ ret = readl_poll_timeout(rng->base + PRNG_STATUS, val,
++ val & PRNG_STATUS_DATA_AVAIL,
++ 200, 10000);
++ if (ret)
++ return ret;
+
+ val = readl_relaxed(rng->base + PRNG_DATA_OUT);
+ if (!val)
+- break;
++ return -EINVAL;
+
+ if ((max - currsize) >= WORD_SZ) {
+ memcpy(data, &val, WORD_SZ);
+@@ -65,7 +69,7 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max)
+ }
+ } while (currsize < max);
+
+- return currsize;
++ return 0;
+ }
+
+ static int qcom_rng_generate(struct crypto_rng *tfm,
+@@ -87,7 +91,7 @@ static int qcom_rng_generate(struct crypto_rng *tfm,
+ mutex_unlock(&rng->lock);
+ clk_disable_unprepare(rng->clk);
+
+- return 0;
++ return ret;
+ }
+
+ static int qcom_rng_seed(struct crypto_rng *tfm, const u8 *seed,
+diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c
+index 1cece1a7d3f00..5bbf0d2722e11 100644
+--- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c
++++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c
+@@ -506,7 +506,6 @@ struct rk_crypto_tmp rk_ecb_des3_ede_alg = {
+ .exit = rk_ablk_exit_tfm,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+- .ivsize = DES_BLOCK_SIZE,
+ .setkey = rk_tdes_setkey,
+ .encrypt = rk_des3_ede_ecb_encrypt,
+ .decrypt = rk_des3_ede_ecb_decrypt,
+diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
+index 55aa3a71169b0..7717e9e5977bb 100644
+--- a/drivers/crypto/s5p-sss.c
++++ b/drivers/crypto/s5p-sss.c
+@@ -2171,6 +2171,8 @@ static int s5p_aes_probe(struct platform_device *pdev)
+
+ variant = find_s5p_sss_version(pdev);
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++ if (!res)
++ return -EINVAL;
+
+ /*
+ * Note: HASH and PRNG uses the same registers in secss, avoid
+diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c
+index 75867c0b00172..90a920e7f6642 100644
+--- a/drivers/crypto/stm32/stm32-crc32.c
++++ b/drivers/crypto/stm32/stm32-crc32.c
+@@ -279,7 +279,7 @@ static struct shash_alg algs[] = {
+ .digestsize = CHKSUM_DIGEST_SIZE,
+ .base = {
+ .cra_name = "crc32",
+- .cra_driver_name = DRIVER_NAME,
++ .cra_driver_name = "stm32-crc32-crc32",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
+ .cra_blocksize = CHKSUM_BLOCK_SIZE,
+@@ -301,7 +301,7 @@ static struct shash_alg algs[] = {
+ .digestsize = CHKSUM_DIGEST_SIZE,
+ .base = {
+ .cra_name = "crc32c",
+- .cra_driver_name = DRIVER_NAME,
++ .cra_driver_name = "stm32-crc32-crc32c",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
+ .cra_blocksize = CHKSUM_BLOCK_SIZE,
+@@ -384,8 +384,10 @@ static int stm32_crc_remove(struct platform_device *pdev)
+ struct stm32_crc *crc = platform_get_drvdata(pdev);
+ int ret = pm_runtime_get_sync(crc->dev);
+
+- if (ret < 0)
++ if (ret < 0) {
++ pm_runtime_put_noidle(crc->dev);
+ return ret;
++ }
+
+ spin_lock(&crc_list.lock);
+ list_del(&crc->list);
+diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c
+index 7389a0536ff02..81eb136b6c11d 100644
+--- a/drivers/crypto/stm32/stm32-cryp.c
++++ b/drivers/crypto/stm32/stm32-cryp.c
+@@ -37,7 +37,6 @@
+ /* Mode mask = bits [15..0] */
+ #define FLG_MODE_MASK GENMASK(15, 0)
+ /* Bit [31..16] status */
+-#define FLG_CCM_PADDED_WA BIT(16)
+
+ /* Registers */
+ #define CRYP_CR 0x00000000
+@@ -105,8 +104,6 @@
+ /* Misc */
+ #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
+ #define GCM_CTR_INIT 2
+-#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset)
+-#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset)
+ #define CRYP_AUTOSUSPEND_DELAY 50
+
+ struct stm32_cryp_caps {
+@@ -144,26 +141,16 @@ struct stm32_cryp {
+ size_t authsize;
+ size_t hw_blocksize;
+
+- size_t total_in;
+- size_t total_in_save;
+- size_t total_out;
+- size_t total_out_save;
++ size_t payload_in;
++ size_t header_in;
++ size_t payload_out;
+
+- struct scatterlist *in_sg;
+ struct scatterlist *out_sg;
+- struct scatterlist *out_sg_save;
+-
+- struct scatterlist in_sgl;
+- struct scatterlist out_sgl;
+- bool sgs_copied;
+-
+- int in_sg_len;
+- int out_sg_len;
+
+ struct scatter_walk in_walk;
+ struct scatter_walk out_walk;
+
+- u32 last_ctr[4];
++ __be32 last_ctr[4];
+ u32 gcm_ctr;
+ };
+
+@@ -262,6 +249,7 @@ static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
+ }
+
+ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
++static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
+
+ static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
+ {
+@@ -283,103 +271,6 @@ static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
+ return cryp;
+ }
+
+-static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
+- size_t align)
+-{
+- int len = 0;
+-
+- if (!total)
+- return 0;
+-
+- if (!IS_ALIGNED(total, align))
+- return -EINVAL;
+-
+- while (sg) {
+- if (!IS_ALIGNED(sg->offset, sizeof(u32)))
+- return -EINVAL;
+-
+- if (!IS_ALIGNED(sg->length, align))
+- return -EINVAL;
+-
+- len += sg->length;
+- sg = sg_next(sg);
+- }
+-
+- if (len != total)
+- return -EINVAL;
+-
+- return 0;
+-}
+-
+-static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
+-{
+- int ret;
+-
+- ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
+- cryp->hw_blocksize);
+- if (ret)
+- return ret;
+-
+- ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
+- cryp->hw_blocksize);
+-
+- return ret;
+-}
+-
+-static void sg_copy_buf(void *buf, struct scatterlist *sg,
+- unsigned int start, unsigned int nbytes, int out)
+-{
+- struct scatter_walk walk;
+-
+- if (!nbytes)
+- return;
+-
+- scatterwalk_start(&walk, sg);
+- scatterwalk_advance(&walk, start);
+- scatterwalk_copychunks(buf, &walk, nbytes, out);
+- scatterwalk_done(&walk, out, 0);
+-}
+-
+-static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
+-{
+- void *buf_in, *buf_out;
+- int pages, total_in, total_out;
+-
+- if (!stm32_cryp_check_io_aligned(cryp)) {
+- cryp->sgs_copied = 0;
+- return 0;
+- }
+-
+- total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
+- pages = total_in ? get_order(total_in) : 1;
+- buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
+-
+- total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
+- pages = total_out ? get_order(total_out) : 1;
+- buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
+-
+- if (!buf_in || !buf_out) {
+- dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
+- cryp->sgs_copied = 0;
+- return -EFAULT;
+- }
+-
+- sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
+-
+- sg_init_one(&cryp->in_sgl, buf_in, total_in);
+- cryp->in_sg = &cryp->in_sgl;
+- cryp->in_sg_len = 1;
+-
+- sg_init_one(&cryp->out_sgl, buf_out, total_out);
+- cryp->out_sg_save = cryp->out_sg;
+- cryp->out_sg = &cryp->out_sgl;
+- cryp->out_sg_len = 1;
+-
+- cryp->sgs_copied = 1;
+-
+- return 0;
+-}
+-
+ static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)
+ {
+ if (!iv)
+@@ -481,16 +372,99 @@ static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
+
+ /* Wait for end of processing */
+ ret = stm32_cryp_wait_enable(cryp);
+- if (ret)
++ if (ret) {
+ dev_err(cryp->dev, "Timeout (gcm init)\n");
++ return ret;
++ }
+
+- return ret;
++ /* Prepare next phase */
++ if (cryp->areq->assoclen) {
++ cfg |= CR_PH_HEADER;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++ } else if (stm32_cryp_get_input_text_len(cryp)) {
++ cfg |= CR_PH_PAYLOAD;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++ }
++
++ return 0;
++}
++
++static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp)
++{
++ u32 cfg;
++ int err;
++
++ /* Check if whole header written */
++ if (!cryp->header_in) {
++ /* Wait for completion */
++ err = stm32_cryp_wait_busy(cryp);
++ if (err) {
++ dev_err(cryp->dev, "Timeout (gcm/ccm header)\n");
++ stm32_cryp_write(cryp, CRYP_IMSCR, 0);
++ stm32_cryp_finish_req(cryp, err);
++ return;
++ }
++
++ if (stm32_cryp_get_input_text_len(cryp)) {
++ /* Phase 3 : payload */
++ cfg = stm32_cryp_read(cryp, CRYP_CR);
++ cfg &= ~CR_CRYPEN;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++
++ cfg &= ~CR_PH_MASK;
++ cfg |= CR_PH_PAYLOAD | CR_CRYPEN;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++ } else {
++ /*
++ * Phase 4 : tag.
++ * Nothing to read, nothing to write, caller have to
++ * end request
++ */
++ }
++ }
++}
++
++static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp)
++{
++ unsigned int i;
++ size_t written;
++ size_t len;
++ u32 alen = cryp->areq->assoclen;
++ u32 block[AES_BLOCK_32] = {0};
++ u8 *b8 = (u8 *)block;
++
++ if (alen <= 65280) {
++ /* Write first u32 of B1 */
++ b8[0] = (alen >> 8) & 0xFF;
++ b8[1] = alen & 0xFF;
++ len = 2;
++ } else {
++ /* Build the two first u32 of B1 */
++ b8[0] = 0xFF;
++ b8[1] = 0xFE;
++ b8[2] = (alen & 0xFF000000) >> 24;
++ b8[3] = (alen & 0x00FF0000) >> 16;
++ b8[4] = (alen & 0x0000FF00) >> 8;
++ b8[5] = alen & 0x000000FF;
++ len = 6;
++ }
++
++ written = min_t(size_t, AES_BLOCK_SIZE - len, alen);
++
++ scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0);
++ for (i = 0; i < AES_BLOCK_32; i++)
++ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
++
++ cryp->header_in -= written;
++
++ stm32_crypt_gcmccm_end_header(cryp);
+ }
+
+ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
+ {
+ int ret;
+- u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
++ u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32];
++ u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32;
+ __be32 *bd;
+ u32 *d;
+ unsigned int i, textlen;
+@@ -531,10 +505,24 @@ static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
+
+ /* Wait for end of processing */
+ ret = stm32_cryp_wait_enable(cryp);
+- if (ret)
++ if (ret) {
+ dev_err(cryp->dev, "Timeout (ccm init)\n");
++ return ret;
++ }
+
+- return ret;
++ /* Prepare next phase */
++ if (cryp->areq->assoclen) {
++ cfg |= CR_PH_HEADER | CR_CRYPEN;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++
++ /* Write first (special) block (may move to next phase [payload]) */
++ stm32_cryp_write_ccm_first_header(cryp);
++ } else if (stm32_cryp_get_input_text_len(cryp)) {
++ cfg |= CR_PH_PAYLOAD;
++ stm32_cryp_write(cryp, CRYP_CR, cfg);
++ }
++
++ return 0;
+ }
+
+ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
+@@ -542,7 +530,7 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
+ int ret;
+ u32 cfg, hw_mode;
+
+- pm_runtime_resume_and_get(cryp->dev);
++ pm_runtime_get_sync(cryp->dev);
+
+ /* Disable interrupt */
+ stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+@@ -605,16 +593,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
+ if (ret)
+ return ret;
+
+- /* Phase 2 : header (authenticated data) */
+- if (cryp->areq->assoclen) {
+- cfg |= CR_PH_HEADER;
+- } else if (stm32_cryp_get_input_text_len(cryp)) {
+- cfg |= CR_PH_PAYLOAD;
+- stm32_cryp_write(cryp, CRYP_CR, cfg);
+- } else {
+- cfg |= CR_PH_INIT;
+- }
+-
+ break;
+
+ case CR_DES_CBC:
+@@ -633,8 +611,6 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
+
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+- cryp->flags &= ~FLG_CCM_PADDED_WA;
+-
+ return 0;
+ }
+
+@@ -644,28 +620,9 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
+ /* Phase 4 : output tag */
+ err = stm32_cryp_read_auth_tag(cryp);
+
+- if (!err && (!(is_gcm(cryp) || is_ccm(cryp))))
++ if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp))))
+ stm32_cryp_get_iv(cryp);
+
+- if (cryp->sgs_copied) {
+- void *buf_in, *buf_out;
+- int pages, len;
+-
+- buf_in = sg_virt(&cryp->in_sgl);
+- buf_out = sg_virt(&cryp->out_sgl);
+-
+- sg_copy_buf(buf_out, cryp->out_sg_save, 0,
+- cryp->total_out_save, 1);
+-
+- len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
+- pages = len ? get_order(len) : 1;
+- free_pages((unsigned long)buf_in, pages);
+-
+- len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
+- pages = len ? get_order(len) : 1;
+- free_pages((unsigned long)buf_out, pages);
+- }
+-
+ pm_runtime_mark_last_busy(cryp->dev);
+ pm_runtime_put_autosuspend(cryp->dev);
+
+@@ -674,8 +631,6 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
+ else
+ crypto_finalize_skcipher_request(cryp->engine, cryp->req,
+ err);
+-
+- memset(cryp->ctx->key, 0, cryp->ctx->keylen);
+ }
+
+ static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
+@@ -801,7 +756,20 @@ static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+ {
+- return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
++ switch (authsize) {
++ case 4:
++ case 8:
++ case 12:
++ case 13:
++ case 14:
++ case 15:
++ case 16:
++ break;
++ default:
++ return -EINVAL;
++ }
++
++ return 0;
+ }
+
+ static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
+@@ -825,31 +793,61 @@ static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
+
+ static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % AES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % AES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
+ }
+
+ static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % AES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % AES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
+ }
+
+ static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
+ }
+
+@@ -863,53 +861,122 @@ static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
+ return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
+ }
+
++static inline int crypto_ccm_check_iv(const u8 *iv)
++{
++ /* 2 <= L <= 8, so 1 <= L' <= 7. */
++ if (iv[0] < 1 || iv[0] > 7)
++ return -EINVAL;
++
++ return 0;
++}
++
+ static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
+ {
++ int err;
++
++ err = crypto_ccm_check_iv(req->iv);
++ if (err)
++ return err;
++
+ return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
+ {
++ int err;
++
++ err = crypto_ccm_check_iv(req->iv);
++ if (err)
++ return err;
++
+ return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
+ }
+
+ static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
+ }
+
+ static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
+ }
+
+ static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
+ }
+
+ static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
+ }
+
+ static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)
+ {
++ if (req->cryptlen % DES_BLOCK_SIZE)
++ return -EINVAL;
++
++ if (req->cryptlen == 0)
++ return 0;
++
+ return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
+ }
+
+@@ -919,6 +986,7 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req,
+ struct stm32_cryp_ctx *ctx;
+ struct stm32_cryp *cryp;
+ struct stm32_cryp_reqctx *rctx;
++ struct scatterlist *in_sg;
+ int ret;
+
+ if (!req && !areq)
+@@ -944,76 +1012,55 @@ static int stm32_cryp_prepare_req(struct skcipher_request *req,
+ if (req) {
+ cryp->req = req;
+ cryp->areq = NULL;
+- cryp->total_in = req->cryptlen;
+- cryp->total_out = cryp->total_in;
++ cryp->header_in = 0;
++ cryp->payload_in = req->cryptlen;
++ cryp->payload_out = req->cryptlen;
++ cryp->authsize = 0;
+ } else {
+ /*
+ * Length of input and output data:
+ * Encryption case:
+- * INPUT = AssocData || PlainText
++ * INPUT = AssocData || PlainText
+ * <- assoclen -> <- cryptlen ->
+- * <------- total_in ----------->
+ *
+- * OUTPUT = AssocData || CipherText || AuthTag
+- * <- assoclen -> <- cryptlen -> <- authsize ->
+- * <---------------- total_out ----------------->
++ * OUTPUT = AssocData || CipherText || AuthTag
++ * <- assoclen -> <-- cryptlen --> <- authsize ->
+ *
+ * Decryption case:
+- * INPUT = AssocData || CipherText || AuthTag
+- * <- assoclen -> <--------- cryptlen --------->
+- * <- authsize ->
+- * <---------------- total_in ------------------>
++ * INPUT = AssocData || CipherTex || AuthTag
++ * <- assoclen ---> <---------- cryptlen ---------->
+ *
+- * OUTPUT = AssocData || PlainText
+- * <- assoclen -> <- crypten - authsize ->
+- * <---------- total_out ----------------->
++ * OUTPUT = AssocData || PlainText
++ * <- assoclen -> <- cryptlen - authsize ->
+ */
+ cryp->areq = areq;
+ cryp->req = NULL;
+ cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
+- cryp->total_in = areq->assoclen + areq->cryptlen;
+- if (is_encrypt(cryp))
+- /* Append auth tag to output */
+- cryp->total_out = cryp->total_in + cryp->authsize;
+- else
+- /* No auth tag in output */
+- cryp->total_out = cryp->total_in - cryp->authsize;
++ if (is_encrypt(cryp)) {
++ cryp->payload_in = areq->cryptlen;
++ cryp->header_in = areq->assoclen;
++ cryp->payload_out = areq->cryptlen;
++ } else {
++ cryp->payload_in = areq->cryptlen - cryp->authsize;
++ cryp->header_in = areq->assoclen;
++ cryp->payload_out = cryp->payload_in;
++ }
+ }
+
+- cryp->total_in_save = cryp->total_in;
+- cryp->total_out_save = cryp->total_out;
++ in_sg = req ? req->src : areq->src;
++ scatterwalk_start(&cryp->in_walk, in_sg);
+
+- cryp->in_sg = req ? req->src : areq->src;
+ cryp->out_sg = req ? req->dst : areq->dst;
+- cryp->out_sg_save = cryp->out_sg;
+-
+- cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
+- if (cryp->in_sg_len < 0) {
+- dev_err(cryp->dev, "Cannot get in_sg_len\n");
+- ret = cryp->in_sg_len;
+- return ret;
+- }
+-
+- cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
+- if (cryp->out_sg_len < 0) {
+- dev_err(cryp->dev, "Cannot get out_sg_len\n");
+- ret = cryp->out_sg_len;
+- return ret;
+- }
+-
+- ret = stm32_cryp_copy_sgs(cryp);
+- if (ret)
+- return ret;
+-
+- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
+ scatterwalk_start(&cryp->out_walk, cryp->out_sg);
+
+ if (is_gcm(cryp) || is_ccm(cryp)) {
+ /* In output, jump after assoc data */
+- scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
+- cryp->total_out -= cryp->areq->assoclen;
++ scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2);
+ }
+
++ if (is_ctr(cryp))
++ memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr));
++
+ ret = stm32_cryp_hw_init(cryp);
+ return ret;
+ }
+@@ -1061,8 +1108,7 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
+ if (!cryp)
+ return -ENODEV;
+
+- if (unlikely(!cryp->areq->assoclen &&
+- !stm32_cryp_get_input_text_len(cryp))) {
++ if (unlikely(!cryp->payload_in && !cryp->header_in)) {
+ /* No input data to process: get tag and finish */
+ stm32_cryp_finish_req(cryp, 0);
+ return 0;
+@@ -1071,43 +1117,10 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
+ return stm32_cryp_cpu_start(cryp);
+ }
+
+-static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
+- unsigned int n)
+-{
+- scatterwalk_advance(&cryp->out_walk, n);
+-
+- if (unlikely(cryp->out_sg->length == _walked_out)) {
+- cryp->out_sg = sg_next(cryp->out_sg);
+- if (cryp->out_sg) {
+- scatterwalk_start(&cryp->out_walk, cryp->out_sg);
+- return (sg_virt(cryp->out_sg) + _walked_out);
+- }
+- }
+-
+- return (u32 *)((u8 *)dst + n);
+-}
+-
+-static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
+- unsigned int n)
+-{
+- scatterwalk_advance(&cryp->in_walk, n);
+-
+- if (unlikely(cryp->in_sg->length == _walked_in)) {
+- cryp->in_sg = sg_next(cryp->in_sg);
+- if (cryp->in_sg) {
+- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
+- return (sg_virt(cryp->in_sg) + _walked_in);
+- }
+- }
+-
+- return (u32 *)((u8 *)src + n);
+-}
+-
+ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
+ {
+- u32 cfg, size_bit, *dst, d32;
+- u8 *d8;
+- unsigned int i, j;
++ u32 cfg, size_bit;
++ unsigned int i;
+ int ret = 0;
+
+ /* Update Config */
+@@ -1130,7 +1143,7 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
+ stm32_cryp_write(cryp, CRYP_DIN, size_bit);
+
+ size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
+- cryp->areq->cryptlen - AES_BLOCK_SIZE;
++ cryp->areq->cryptlen - cryp->authsize;
+ size_bit *= 8;
+ if (cryp->caps->swap_final)
+ size_bit = (__force u32)cpu_to_be32(size_bit);
+@@ -1139,11 +1152,9 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
+ stm32_cryp_write(cryp, CRYP_DIN, size_bit);
+ } else {
+ /* CCM: write CTR0 */
+- u8 iv[AES_BLOCK_SIZE];
+- u32 *iv32 = (u32 *)iv;
+- __be32 *biv;
+-
+- biv = (void *)iv;
++ u32 iv32[AES_BLOCK_32];
++ u8 *iv = (u8 *)iv32;
++ __be32 *biv = (__be32 *)iv32;
+
+ memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
+ memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
+@@ -1165,39 +1176,18 @@ static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
+ }
+
+ if (is_encrypt(cryp)) {
++ u32 out_tag[AES_BLOCK_32];
++
+ /* Get and write tag */
+- dst = sg_virt(cryp->out_sg) + _walked_out;
++ for (i = 0; i < AES_BLOCK_32; i++)
++ out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
+
+- for (i = 0; i < AES_BLOCK_32; i++) {
+- if (cryp->total_out >= sizeof(u32)) {
+- /* Read a full u32 */
+- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
+-
+- dst = stm32_cryp_next_out(cryp, dst,
+- sizeof(u32));
+- cryp->total_out -= sizeof(u32);
+- } else if (!cryp->total_out) {
+- /* Empty fifo out (data from input padding) */
+- stm32_cryp_read(cryp, CRYP_DOUT);
+- } else {
+- /* Read less than an u32 */
+- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
+- d8 = (u8 *)&d32;
+-
+- for (j = 0; j < cryp->total_out; j++) {
+- *((u8 *)dst) = *(d8++);
+- dst = stm32_cryp_next_out(cryp, dst, 1);
+- }
+- cryp->total_out = 0;
+- }
+- }
++ scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1);
+ } else {
+ /* Get and check tag */
+ u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
+
+- scatterwalk_map_and_copy(in_tag, cryp->in_sg,
+- cryp->total_in_save - cryp->authsize,
+- cryp->authsize, 0);
++ scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0);
+
+ for (i = 0; i < AES_BLOCK_32; i++)
+ out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
+@@ -1217,115 +1207,59 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
+ {
+ u32 cr;
+
+- if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
+- cryp->last_ctr[3] = 0;
+- cryp->last_ctr[2]++;
+- if (!cryp->last_ctr[2]) {
+- cryp->last_ctr[1]++;
+- if (!cryp->last_ctr[1])
+- cryp->last_ctr[0]++;
+- }
++ if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) {
++ /*
++ * In this case, we need to increment manually the ctr counter,
++ * as HW doesn't handle the U32 carry.
++ */
++ crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr));
+
+ cr = stm32_cryp_read(cryp, CRYP_CR);
+ stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
+
+- stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr);
++ stm32_cryp_hw_write_iv(cryp, cryp->last_ctr);
+
+ stm32_cryp_write(cryp, CRYP_CR, cr);
+ }
+
+- cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
+- cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
+- cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
+- cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
++ /* The IV registers are BE */
++ cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
++ cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
++ cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
++ cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
+ }
+
+-static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
++static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
+ {
+- unsigned int i, j;
+- u32 d32, *dst;
+- u8 *d8;
+- size_t tag_size;
+-
+- /* Do no read tag now (if any) */
+- if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
+- tag_size = cryp->authsize;
+- else
+- tag_size = 0;
+-
+- dst = sg_virt(cryp->out_sg) + _walked_out;
++ unsigned int i;
++ u32 block[AES_BLOCK_32];
+
+- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
+- if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
+- /* Read a full u32 */
+- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
++ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
++ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
+
+- dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
+- cryp->total_out -= sizeof(u32);
+- } else if (cryp->total_out == tag_size) {
+- /* Empty fifo out (data from input padding) */
+- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
+- } else {
+- /* Read less than an u32 */
+- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
+- d8 = (u8 *)&d32;
+-
+- for (j = 0; j < cryp->total_out - tag_size; j++) {
+- *((u8 *)dst) = *(d8++);
+- dst = stm32_cryp_next_out(cryp, dst, 1);
+- }
+- cryp->total_out = tag_size;
+- }
+- }
+-
+- return !(cryp->total_out - tag_size) || !cryp->total_in;
++ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_out), 1);
++ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_out);
+ }
+
+ static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
+ {
+- unsigned int i, j;
+- u32 *src;
+- u8 d8[4];
+- size_t tag_size;
+-
+- /* Do no write tag (if any) */
+- if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
+- tag_size = cryp->authsize;
+- else
+- tag_size = 0;
+-
+- src = sg_virt(cryp->in_sg) + _walked_in;
++ unsigned int i;
++ u32 block[AES_BLOCK_32] = {0};
+
+- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
+- if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
+- /* Write a full u32 */
+- stm32_cryp_write(cryp, CRYP_DIN, *src);
++ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_in), 0);
++ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
++ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
+
+- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
+- cryp->total_in -= sizeof(u32);
+- } else if (cryp->total_in == tag_size) {
+- /* Write padding data */
+- stm32_cryp_write(cryp, CRYP_DIN, 0);
+- } else {
+- /* Write less than an u32 */
+- memset(d8, 0, sizeof(u32));
+- for (j = 0; j < cryp->total_in - tag_size; j++) {
+- d8[j] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+- }
+-
+- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+- cryp->total_in = tag_size;
+- }
+- }
++ cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in);
+ }
+
+ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
+ {
+ int err;
+- u32 cfg, tmp[AES_BLOCK_32];
+- size_t total_in_ori = cryp->total_in;
+- struct scatterlist *out_sg_ori = cryp->out_sg;
++ u32 cfg, block[AES_BLOCK_32] = {0};
+ unsigned int i;
+
+ /* 'Special workaround' procedure described in the datasheet */
+@@ -1350,18 +1284,25 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
+
+ /* b) pad and write the last block */
+ stm32_cryp_irq_write_block(cryp);
+- cryp->total_in = total_in_ori;
++ /* wait end of process */
+ err = stm32_cryp_wait_output(cryp);
+ if (err) {
+- dev_err(cryp->dev, "Timeout (write gcm header)\n");
++ dev_err(cryp->dev, "Timeout (write gcm last data)\n");
+ return stm32_cryp_finish_req(cryp, err);
+ }
+
+ /* c) get and store encrypted data */
+- stm32_cryp_irq_read_data(cryp);
+- scatterwalk_map_and_copy(tmp, out_sg_ori,
+- cryp->total_in_save - total_in_ori,
+- total_in_ori, 0);
++ /*
++ * Same code as stm32_cryp_irq_read_data(), but we want to store
++ * block value
++ */
++ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
++ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
++
++ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_out), 1);
++ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_out);
+
+ /* d) change mode back to AES GCM */
+ cfg &= ~CR_ALGO_MASK;
+@@ -1374,19 +1315,13 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+ /* f) write padded data */
+- for (i = 0; i < AES_BLOCK_32; i++) {
+- if (cryp->total_in)
+- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
+- else
+- stm32_cryp_write(cryp, CRYP_DIN, 0);
+-
+- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
+- }
++ for (i = 0; i < AES_BLOCK_32; i++)
++ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
+
+ /* g) Empty fifo out */
+ err = stm32_cryp_wait_output(cryp);
+ if (err) {
+- dev_err(cryp->dev, "Timeout (write gcm header)\n");
++ dev_err(cryp->dev, "Timeout (write gcm padded data)\n");
+ return stm32_cryp_finish_req(cryp, err);
+ }
+
+@@ -1399,16 +1334,14 @@ static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
+
+ static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
+ {
+- u32 cfg, payload_bytes;
++ u32 cfg;
+
+ /* disable ip, set NPBLB and reneable ip */
+ cfg = stm32_cryp_read(cryp, CRYP_CR);
+ cfg &= ~CR_CRYPEN;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+- payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
+- cryp->total_in;
+- cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
++ cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT;
+ cfg |= CR_CRYPEN;
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+ }
+@@ -1417,13 +1350,11 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
+ {
+ int err = 0;
+ u32 cfg, iv1tmp;
+- u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
+- size_t last_total_out, total_in_ori = cryp->total_in;
+- struct scatterlist *out_sg_ori = cryp->out_sg;
++ u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32];
++ u32 block[AES_BLOCK_32] = {0};
+ unsigned int i;
+
+ /* 'Special workaround' procedure described in the datasheet */
+- cryp->flags |= FLG_CCM_PADDED_WA;
+
+ /* a) disable ip */
+ stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+@@ -1453,7 +1384,7 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
+
+ /* b) pad and write the last block */
+ stm32_cryp_irq_write_block(cryp);
+- cryp->total_in = total_in_ori;
++ /* wait end of process */
+ err = stm32_cryp_wait_output(cryp);
+ if (err) {
+ dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
+@@ -1461,13 +1392,16 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
+ }
+
+ /* c) get and store decrypted data */
+- last_total_out = cryp->total_out;
+- stm32_cryp_irq_read_data(cryp);
++ /*
++ * Same code as stm32_cryp_irq_read_data(), but we want to store
++ * block value
++ */
++ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
++ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
+
+- memset(tmp, 0, sizeof(tmp));
+- scatterwalk_map_and_copy(tmp, out_sg_ori,
+- cryp->total_out_save - last_total_out,
+- last_total_out, 0);
++ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
++ cryp->payload_out), 1);
++ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out);
+
+ /* d) Load again CRYP_CSGCMCCMxR */
+ for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
+@@ -1484,10 +1418,10 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
+ stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+ /* g) XOR and write padded data */
+- for (i = 0; i < ARRAY_SIZE(tmp); i++) {
+- tmp[i] ^= cstmp1[i];
+- tmp[i] ^= cstmp2[i];
+- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
++ for (i = 0; i < ARRAY_SIZE(block); i++) {
++ block[i] ^= cstmp1[i];
++ block[i] ^= cstmp2[i];
++ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
+ }
+
+ /* h) wait for completion */
+@@ -1501,30 +1435,34 @@ static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
+
+ static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
+ {
+- if (unlikely(!cryp->total_in)) {
++ if (unlikely(!cryp->payload_in)) {
+ dev_warn(cryp->dev, "No more data to process\n");
+ return;
+ }
+
+- if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
++ if (unlikely(cryp->payload_in < AES_BLOCK_SIZE &&
+ (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
+ is_encrypt(cryp))) {
+ /* Padding for AES GCM encryption */
+- if (cryp->caps->padding_wa)
++ if (cryp->caps->padding_wa) {
+ /* Special case 1 */
+- return stm32_cryp_irq_write_gcm_padded_data(cryp);
++ stm32_cryp_irq_write_gcm_padded_data(cryp);
++ return;
++ }
+
+ /* Setting padding bytes (NBBLB) */
+ stm32_cryp_irq_set_npblb(cryp);
+ }
+
+- if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
++ if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) &&
+ (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
+ is_decrypt(cryp))) {
+ /* Padding for AES CCM decryption */
+- if (cryp->caps->padding_wa)
++ if (cryp->caps->padding_wa) {
+ /* Special case 2 */
+- return stm32_cryp_irq_write_ccm_padded_data(cryp);
++ stm32_cryp_irq_write_ccm_padded_data(cryp);
++ return;
++ }
+
+ /* Setting padding bytes (NBBLB) */
+ stm32_cryp_irq_set_npblb(cryp);
+@@ -1536,192 +1474,60 @@ static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
+ stm32_cryp_irq_write_block(cryp);
+ }
+
+-static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
++static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp)
+ {
+- int err;
+- unsigned int i, j;
+- u32 cfg, *src;
+-
+- src = sg_virt(cryp->in_sg) + _walked_in;
+-
+- for (i = 0; i < AES_BLOCK_32; i++) {
+- stm32_cryp_write(cryp, CRYP_DIN, *src);
+-
+- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
+- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
+-
+- /* Check if whole header written */
+- if ((cryp->total_in_save - cryp->total_in) ==
+- cryp->areq->assoclen) {
+- /* Write padding if needed */
+- for (j = i + 1; j < AES_BLOCK_32; j++)
+- stm32_cryp_write(cryp, CRYP_DIN, 0);
+-
+- /* Wait for completion */
+- err = stm32_cryp_wait_busy(cryp);
+- if (err) {
+- dev_err(cryp->dev, "Timeout (gcm header)\n");
+- return stm32_cryp_finish_req(cryp, err);
+- }
+-
+- if (stm32_cryp_get_input_text_len(cryp)) {
+- /* Phase 3 : payload */
+- cfg = stm32_cryp_read(cryp, CRYP_CR);
+- cfg &= ~CR_CRYPEN;
+- stm32_cryp_write(cryp, CRYP_CR, cfg);
+-
+- cfg &= ~CR_PH_MASK;
+- cfg |= CR_PH_PAYLOAD;
+- cfg |= CR_CRYPEN;
+- stm32_cryp_write(cryp, CRYP_CR, cfg);
+- } else {
+- /* Phase 4 : tag */
+- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+- stm32_cryp_finish_req(cryp, 0);
+- }
+-
+- break;
+- }
+-
+- if (!cryp->total_in)
+- break;
+- }
+-}
++ unsigned int i;
++ u32 block[AES_BLOCK_32] = {0};
++ size_t written;
+
+-static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
+-{
+- int err;
+- unsigned int i = 0, j, k;
+- u32 alen, cfg, *src;
+- u8 d8[4];
+-
+- src = sg_virt(cryp->in_sg) + _walked_in;
+- alen = cryp->areq->assoclen;
+-
+- if (!_walked_in) {
+- if (cryp->areq->assoclen <= 65280) {
+- /* Write first u32 of B1 */
+- d8[0] = (alen >> 8) & 0xFF;
+- d8[1] = alen & 0xFF;
+- d8[2] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+- d8[3] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+-
+- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+- i++;
+-
+- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
+- } else {
+- /* Build the two first u32 of B1 */
+- d8[0] = 0xFF;
+- d8[1] = 0xFE;
+- d8[2] = alen & 0xFF000000;
+- d8[3] = alen & 0x00FF0000;
+-
+- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+- i++;
+-
+- d8[0] = alen & 0x0000FF00;
+- d8[1] = alen & 0x000000FF;
+- d8[2] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+- d8[3] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+-
+- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+- i++;
+-
+- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
+- }
+- }
++ written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in);
+
+- /* Write next u32 */
+- for (; i < AES_BLOCK_32; i++) {
+- /* Build an u32 */
+- memset(d8, 0, sizeof(u32));
+- for (k = 0; k < sizeof(u32); k++) {
+- d8[k] = *((u8 *)src);
+- src = stm32_cryp_next_in(cryp, src, 1);
+-
+- cryp->total_in -= min_t(size_t, 1, cryp->total_in);
+- if ((cryp->total_in_save - cryp->total_in) == alen)
+- break;
+- }
++ scatterwalk_copychunks(block, &cryp->in_walk, written, 0);
++ for (i = 0; i < AES_BLOCK_32; i++)
++ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
+
+- stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+-
+- if ((cryp->total_in_save - cryp->total_in) == alen) {
+- /* Write padding if needed */
+- for (j = i + 1; j < AES_BLOCK_32; j++)
+- stm32_cryp_write(cryp, CRYP_DIN, 0);
+-
+- /* Wait for completion */
+- err = stm32_cryp_wait_busy(cryp);
+- if (err) {
+- dev_err(cryp->dev, "Timeout (ccm header)\n");
+- return stm32_cryp_finish_req(cryp, err);
+- }
+-
+- if (stm32_cryp_get_input_text_len(cryp)) {
+- /* Phase 3 : payload */
+- cfg = stm32_cryp_read(cryp, CRYP_CR);
+- cfg &= ~CR_CRYPEN;
+- stm32_cryp_write(cryp, CRYP_CR, cfg);
+-
+- cfg &= ~CR_PH_MASK;
+- cfg |= CR_PH_PAYLOAD;
+- cfg |= CR_CRYPEN;
+- stm32_cryp_write(cryp, CRYP_CR, cfg);
+- } else {
+- /* Phase 4 : tag */
+- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+- stm32_cryp_finish_req(cryp, 0);
+- }
++ cryp->header_in -= written;
+
+- break;
+- }
+- }
++ stm32_crypt_gcmccm_end_header(cryp);
+ }
+
+ static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
+ {
+ struct stm32_cryp *cryp = arg;
+ u32 ph;
++ u32 it_mask = stm32_cryp_read(cryp, CRYP_IMSCR);
+
+ if (cryp->irq_status & MISR_OUT)
+ /* Output FIFO IRQ: read data */
+- if (unlikely(stm32_cryp_irq_read_data(cryp))) {
+- /* All bytes processed, finish */
+- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+- stm32_cryp_finish_req(cryp, 0);
+- return IRQ_HANDLED;
+- }
++ stm32_cryp_irq_read_data(cryp);
+
+ if (cryp->irq_status & MISR_IN) {
+- if (is_gcm(cryp)) {
++ if (is_gcm(cryp) || is_ccm(cryp)) {
+ ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
+ if (unlikely(ph == CR_PH_HEADER))
+ /* Write Header */
+- stm32_cryp_irq_write_gcm_header(cryp);
+- else
+- /* Input FIFO IRQ: write data */
+- stm32_cryp_irq_write_data(cryp);
+- cryp->gcm_ctr++;
+- } else if (is_ccm(cryp)) {
+- ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
+- if (unlikely(ph == CR_PH_HEADER))
+- /* Write Header */
+- stm32_cryp_irq_write_ccm_header(cryp);
++ stm32_cryp_irq_write_gcmccm_header(cryp);
+ else
+ /* Input FIFO IRQ: write data */
+ stm32_cryp_irq_write_data(cryp);
++ if (is_gcm(cryp))
++ cryp->gcm_ctr++;
+ } else {
+ /* Input FIFO IRQ: write data */
+ stm32_cryp_irq_write_data(cryp);
+ }
+ }
+
++ /* Mask useless interrupts */
++ if (!cryp->payload_in && !cryp->header_in)
++ it_mask &= ~IMSCR_IN;
++ if (!cryp->payload_out)
++ it_mask &= ~IMSCR_OUT;
++ stm32_cryp_write(cryp, CRYP_IMSCR, it_mask);
++
++ if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out)
++ stm32_cryp_finish_req(cryp, 0);
++
+ return IRQ_HANDLED;
+ }
+
+@@ -1742,7 +1548,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1759,7 +1565,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1777,7 +1583,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1795,7 +1601,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1812,7 +1618,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1830,7 +1636,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1847,7 +1653,7 @@ static struct skcipher_alg crypto_algs[] = {
+ .base.cra_flags = CRYPTO_ALG_ASYNC,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .base.cra_alignmask = 0xf,
++ .base.cra_alignmask = 0,
+ .base.cra_module = THIS_MODULE,
+
+ .init = stm32_cryp_init_tfm,
+@@ -1877,7 +1683,7 @@ static struct aead_alg aead_algs[] = {
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .cra_alignmask = 0xf,
++ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ },
+@@ -1897,7 +1703,7 @@ static struct aead_alg aead_algs[] = {
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
+- .cra_alignmask = 0xf,
++ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ },
+@@ -2025,8 +1831,6 @@ err_engine1:
+ list_del(&cryp->list);
+ spin_unlock(&cryp_list.lock);
+
+- pm_runtime_disable(dev);
+- pm_runtime_put_noidle(dev);
+ pm_runtime_disable(dev);
+ pm_runtime_put_noidle(dev);
+
+diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
+index 389de9e3302d5..d33006d43f761 100644
+--- a/drivers/crypto/stm32/stm32-hash.c
++++ b/drivers/crypto/stm32/stm32-hash.c
+@@ -813,7 +813,7 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err)
+ static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
+ struct stm32_hash_request_ctx *rctx)
+ {
+- pm_runtime_resume_and_get(hdev->dev);
++ pm_runtime_get_sync(hdev->dev);
+
+ if (!(HASH_FLAGS_INIT & hdev->flags)) {
+ stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
+@@ -962,7 +962,7 @@ static int stm32_hash_export(struct ahash_request *req, void *out)
+ u32 *preg;
+ unsigned int i;
+
+- pm_runtime_resume_and_get(hdev->dev);
++ pm_runtime_get_sync(hdev->dev);
+
+ while ((stm32_hash_read(hdev, HASH_SR) & HASH_SR_BUSY))
+ cpu_relax();
+@@ -1000,7 +1000,7 @@ static int stm32_hash_import(struct ahash_request *req, const void *in)
+
+ preg = rctx->hw_context;
+
+- pm_runtime_resume_and_get(hdev->dev);
++ pm_runtime_get_sync(hdev->dev);
+
+ stm32_hash_write(hdev, HASH_IMR, *preg++);
+ stm32_hash_write(hdev, HASH_STR, *preg++);
+diff --git a/drivers/crypto/vmx/Kconfig b/drivers/crypto/vmx/Kconfig
+index c85fab7ef0bdd..b2c28b87f14b3 100644
+--- a/drivers/crypto/vmx/Kconfig
++++ b/drivers/crypto/vmx/Kconfig
+@@ -2,7 +2,11 @@
+ config CRYPTO_DEV_VMX_ENCRYPT
+ tristate "Encryption acceleration support on P8 CPU"
+ depends on CRYPTO_DEV_VMX
++ select CRYPTO_AES
++ select CRYPTO_CBC
++ select CRYPTO_CTR
+ select CRYPTO_GHASH
++ select CRYPTO_XTS
+ default m
+ help
+ Support for VMX cryptographic acceleration instructions on Power8 CPU.
+diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
+index 41de4a136ecd7..2e7027a3fef3b 100644
+--- a/drivers/cxl/core/regs.c
++++ b/drivers/cxl/core/regs.c
+@@ -35,7 +35,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
+ struct cxl_component_reg_map *map)
+ {
+ int cap, cap_count;
+- u64 cap_array;
++ u32 cap_array;
+
+ *map = (struct cxl_component_reg_map) { 0 };
+
+@@ -45,11 +45,11 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
+ */
+ base += CXL_CM_OFFSET;
+
+- cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET);
++ cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET);
+
+ if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
+ dev_err(dev,
+- "Couldn't locate the CXL.cache and CXL.mem capability array header./n");
++ "Couldn't locate the CXL.cache and CXL.mem capability array header.\n");
+ return;
+ }
+
+diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
+index 8e45aa07d662f..5444b5a7fd3c4 100644
+--- a/drivers/cxl/pci.c
++++ b/drivers/cxl/pci.c
+@@ -972,7 +972,7 @@ static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm,
+ if (pci_resource_len(pdev, bar) < offset) {
+ dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar,
+ &pdev->resource[bar], (unsigned long long)offset);
+- return IOMEM_ERR_PTR(-ENXIO);
++ return NULL;
+ }
+
+ addr = pci_iomap(pdev, bar, 0);
+diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
+index 9652c3ee41e7f..2bb2f9a0499f7 100644
+--- a/drivers/cxl/pmem.c
++++ b/drivers/cxl/pmem.c
+@@ -149,14 +149,24 @@ static void cxl_nvb_update_state(struct work_struct *work)
+ put_device(&cxl_nvb->dev);
+ }
+
++static void cxl_nvdimm_bridge_state_work(struct cxl_nvdimm_bridge *cxl_nvb)
++{
++ /*
++ * Take a reference that the workqueue will drop if new work
++ * gets queued.
++ */
++ get_device(&cxl_nvb->dev);
++ if (!queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
++ put_device(&cxl_nvb->dev);
++}
++
+ static void cxl_nvdimm_bridge_remove(struct device *dev)
+ {
+ struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
+
+ if (cxl_nvb->state == CXL_NVB_ONLINE)
+ cxl_nvb->state = CXL_NVB_OFFLINE;
+- if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
+- get_device(&cxl_nvb->dev);
++ cxl_nvdimm_bridge_state_work(cxl_nvb);
+ }
+
+ static int cxl_nvdimm_bridge_probe(struct device *dev)
+@@ -177,8 +187,7 @@ static int cxl_nvdimm_bridge_probe(struct device *dev)
+ }
+
+ cxl_nvb->state = CXL_NVB_ONLINE;
+- if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
+- get_device(&cxl_nvb->dev);
++ cxl_nvdimm_bridge_state_work(cxl_nvb);
+
+ return 0;
+ }
+diff --git a/drivers/dax/super.c b/drivers/dax/super.c
+index fc89e91beea7c..7610e4a9ac4e2 100644
+--- a/drivers/dax/super.c
++++ b/drivers/dax/super.c
+@@ -678,6 +678,7 @@ static int dax_fs_init(void)
+ static void dax_fs_exit(void)
+ {
+ kern_unmount(dax_mnt);
++ rcu_barrier();
+ kmem_cache_destroy(dax_cache);
+ }
+
+diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
+index 63d32261b63ff..f9217e300eea9 100644
+--- a/drivers/dma-buf/dma-buf.c
++++ b/drivers/dma-buf/dma-buf.c
+@@ -74,7 +74,7 @@ static void dma_buf_release(struct dentry *dentry)
+ * If you hit this BUG() it means someone dropped their ref to the
+ * dma-buf while still having pending operation to the buffer.
+ */
+- BUG_ON(dmabuf->cb_shared.active || dmabuf->cb_excl.active);
++ BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);
+
+ dma_buf_stats_teardown(dmabuf);
+ dmabuf->ops->release(dmabuf);
+@@ -82,6 +82,7 @@ static void dma_buf_release(struct dentry *dentry)
+ if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])
+ dma_resv_fini(dmabuf->resv);
+
++ WARN_ON(!list_empty(&dmabuf->attachments));
+ module_put(dmabuf->owner);
+ kfree(dmabuf->name);
+ kfree(dmabuf);
+@@ -205,16 +206,55 @@ static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
+ wake_up_locked_poll(dcb->poll, dcb->active);
+ dcb->active = 0;
+ spin_unlock_irqrestore(&dcb->poll->lock, flags);
++ dma_fence_put(fence);
++}
++
++static bool dma_buf_poll_shared(struct dma_resv *resv,
++ struct dma_buf_poll_cb_t *dcb)
++{
++ struct dma_resv_list *fobj = dma_resv_shared_list(resv);
++ struct dma_fence *fence;
++ int i, r;
++
++ if (!fobj)
++ return false;
++
++ for (i = 0; i < fobj->shared_count; ++i) {
++ fence = rcu_dereference_protected(fobj->shared[i],
++ dma_resv_held(resv));
++ dma_fence_get(fence);
++ r = dma_fence_add_callback(fence, &dcb->cb, dma_buf_poll_cb);
++ if (!r)
++ return true;
++ dma_fence_put(fence);
++ }
++
++ return false;
++}
++
++static bool dma_buf_poll_excl(struct dma_resv *resv,
++ struct dma_buf_poll_cb_t *dcb)
++{
++ struct dma_fence *fence = dma_resv_excl_fence(resv);
++ int r;
++
++ if (!fence)
++ return false;
++
++ dma_fence_get(fence);
++ r = dma_fence_add_callback(fence, &dcb->cb, dma_buf_poll_cb);
++ if (!r)
++ return true;
++ dma_fence_put(fence);
++
++ return false;
+ }
+
+ static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
+ {
+ struct dma_buf *dmabuf;
+ struct dma_resv *resv;
+- struct dma_resv_list *fobj;
+- struct dma_fence *fence_excl;
+ __poll_t events;
+- unsigned shared_count, seq;
+
+ dmabuf = file->private_data;
+ if (!dmabuf || !dmabuf->resv)
+@@ -228,101 +268,50 @@ static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
+ if (!events)
+ return 0;
+
+-retry:
+- seq = read_seqcount_begin(&resv->seq);
+- rcu_read_lock();
+-
+- fobj = rcu_dereference(resv->fence);
+- if (fobj)
+- shared_count = fobj->shared_count;
+- else
+- shared_count = 0;
+- fence_excl = dma_resv_excl_fence(resv);
+- if (read_seqcount_retry(&resv->seq, seq)) {
+- rcu_read_unlock();
+- goto retry;
+- }
+-
+- if (fence_excl && (!(events & EPOLLOUT) || shared_count == 0)) {
+- struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;
+- __poll_t pevents = EPOLLIN;
++ dma_resv_lock(resv, NULL);
+
+- if (shared_count == 0)
+- pevents |= EPOLLOUT;
++ if (events & EPOLLOUT) {
++ struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_out;
+
++ /* Check that callback isn't busy */
+ spin_lock_irq(&dmabuf->poll.lock);
+- if (dcb->active) {
+- dcb->active |= pevents;
+- events &= ~pevents;
+- } else
+- dcb->active = pevents;
++ if (dcb->active)
++ events &= ~EPOLLOUT;
++ else
++ dcb->active = EPOLLOUT;
+ spin_unlock_irq(&dmabuf->poll.lock);
+
+- if (events & pevents) {
+- if (!dma_fence_get_rcu(fence_excl)) {
+- /* force a recheck */
+- events &= ~pevents;
+- dma_buf_poll_cb(NULL, &dcb->cb);
+- } else if (!dma_fence_add_callback(fence_excl, &dcb->cb,
+- dma_buf_poll_cb)) {
+- events &= ~pevents;
+- dma_fence_put(fence_excl);
+- } else {
+- /*
+- * No callback queued, wake up any additional
+- * waiters.
+- */
+- dma_fence_put(fence_excl);
++ if (events & EPOLLOUT) {
++ if (!dma_buf_poll_shared(resv, dcb) &&
++ !dma_buf_poll_excl(resv, dcb))
++ /* No callback queued, wake up any other waiters */
+ dma_buf_poll_cb(NULL, &dcb->cb);
+- }
++ else
++ events &= ~EPOLLOUT;
+ }
+ }
+
+- if ((events & EPOLLOUT) && shared_count > 0) {
+- struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared;
+- int i;
++ if (events & EPOLLIN) {
++ struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_in;
+
+- /* Only queue a new callback if no event has fired yet */
++ /* Check that callback isn't busy */
+ spin_lock_irq(&dmabuf->poll.lock);
+ if (dcb->active)
+- events &= ~EPOLLOUT;
++ events &= ~EPOLLIN;
+ else
+- dcb->active = EPOLLOUT;
++ dcb->active = EPOLLIN;
+ spin_unlock_irq(&dmabuf->poll.lock);
+
+- if (!(events & EPOLLOUT))
+- goto out;
+-
+- for (i = 0; i < shared_count; ++i) {
+- struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
+-
+- if (!dma_fence_get_rcu(fence)) {
+- /*
+- * fence refcount dropped to zero, this means
+- * that fobj has been freed
+- *
+- * call dma_buf_poll_cb and force a recheck!
+- */
+- events &= ~EPOLLOUT;
++ if (events & EPOLLIN) {
++ if (!dma_buf_poll_excl(resv, dcb))
++ /* No callback queued, wake up any other waiters */
+ dma_buf_poll_cb(NULL, &dcb->cb);
+- break;
+- }
+- if (!dma_fence_add_callback(fence, &dcb->cb,
+- dma_buf_poll_cb)) {
+- dma_fence_put(fence);
+- events &= ~EPOLLOUT;
+- break;
+- }
+- dma_fence_put(fence);
++ else
++ events &= ~EPOLLIN;
+ }
+-
+- /* No callback queued, wake up any additional waiters. */
+- if (i == shared_count)
+- dma_buf_poll_cb(NULL, &dcb->cb);
+ }
+
+-out:
+- rcu_read_unlock();
++ dma_resv_unlock(resv);
+ return events;
+ }
+
+@@ -447,6 +436,7 @@ static inline int is_dma_buf_file(struct file *file)
+
+ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
+ {
++ static atomic64_t dmabuf_inode = ATOMIC64_INIT(0);
+ struct file *file;
+ struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
+
+@@ -456,6 +446,13 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
+ inode->i_size = dmabuf->size;
+ inode_set_bytes(inode, dmabuf->size);
+
++ /*
++ * The ->i_ino acquired from get_next_ino() is not unique thus
++ * not suitable for using it as dentry name by dmabuf stats.
++ * Override ->i_ino with the unique and dmabuffs specific
++ * value.
++ */
++ inode->i_ino = atomic64_add_return(1, &dmabuf_inode);
+ file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
+ flags, &dma_buf_fops);
+ if (IS_ERR(file))
+@@ -565,8 +562,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
+ dmabuf->owner = exp_info->owner;
+ spin_lock_init(&dmabuf->name_lock);
+ init_waitqueue_head(&dmabuf->poll);
+- dmabuf->cb_excl.poll = dmabuf->cb_shared.poll = &dmabuf->poll;
+- dmabuf->cb_excl.active = dmabuf->cb_shared.active = 0;
++ dmabuf->cb_in.poll = dmabuf->cb_out.poll = &dmabuf->poll;
++ dmabuf->cb_in.active = dmabuf->cb_out.active = 0;
+
+ if (!resv) {
+ resv = (struct dma_resv *)&dmabuf[1];
+@@ -583,10 +580,6 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
+ file->f_mode |= FMODE_LSEEK;
+ dmabuf->file = file;
+
+- ret = dma_buf_stats_setup(dmabuf);
+- if (ret)
+- goto err_sysfs;
+-
+ mutex_init(&dmabuf->lock);
+ INIT_LIST_HEAD(&dmabuf->attachments);
+
+@@ -594,6 +587,10 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
+ list_add(&dmabuf->list_node, &db_list.head);
+ mutex_unlock(&db_list.lock);
+
++ ret = dma_buf_stats_setup(dmabuf);
++ if (ret)
++ goto err_sysfs;
++
+ return dmabuf;
+
+ err_sysfs:
+diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c
+index d3fbd950be944..3e07f961e2f3d 100644
+--- a/drivers/dma-buf/dma-fence-array.c
++++ b/drivers/dma-buf/dma-fence-array.c
+@@ -104,7 +104,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence)
+ {
+ struct dma_fence_array *array = to_dma_fence_array(fence);
+
+- return atomic_read(&array->num_pending) <= 0;
++ if (atomic_read(&array->num_pending) > 0)
++ return false;
++
++ dma_fence_array_clear_pending_error(array);
++ return true;
+ }
+
+ static void dma_fence_array_release(struct dma_fence *fence)
+diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
+index 56bf5ad01ad54..8f5848aa144fe 100644
+--- a/drivers/dma-buf/dma-heap.c
++++ b/drivers/dma-buf/dma-heap.c
+@@ -14,6 +14,7 @@
+ #include <linux/xarray.h>
+ #include <linux/list.h>
+ #include <linux/slab.h>
++#include <linux/nospec.h>
+ #include <linux/uaccess.h>
+ #include <linux/syscalls.h>
+ #include <linux/dma-heap.h>
+@@ -135,6 +136,7 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd,
+ if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds))
+ return -EINVAL;
+
++ nr = array_index_nospec(nr, ARRAY_SIZE(dma_heap_ioctl_cmds));
+ /* Get the kernel ioctl cmd that matches */
+ kcmd = dma_heap_ioctl_cmds[nr];
+
+diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c
+index 0c05b79870f96..83f02bd51dda6 100644
+--- a/drivers/dma-buf/heaps/cma_heap.c
++++ b/drivers/dma-buf/heaps/cma_heap.c
+@@ -124,10 +124,11 @@ static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+ struct cma_heap_buffer *buffer = dmabuf->priv;
+ struct dma_heap_attachment *a;
+
++ mutex_lock(&buffer->lock);
++
+ if (buffer->vmap_cnt)
+ invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+- mutex_lock(&buffer->lock);
+ list_for_each_entry(a, &buffer->attachments, list) {
+ if (!a->mapped)
+ continue;
+@@ -144,10 +145,11 @@ static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+ struct cma_heap_buffer *buffer = dmabuf->priv;
+ struct dma_heap_attachment *a;
+
++ mutex_lock(&buffer->lock);
++
+ if (buffer->vmap_cnt)
+ flush_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+- mutex_lock(&buffer->lock);
+ list_for_each_entry(a, &buffer->attachments, list) {
+ if (!a->mapped)
+ continue;
+diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c
+index 23a7e74ef9666..8660508f3684f 100644
+--- a/drivers/dma-buf/heaps/system_heap.c
++++ b/drivers/dma-buf/heaps/system_heap.c
+@@ -289,7 +289,7 @@ static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
+ int i;
+
+ table = &buffer->sg_table;
+- for_each_sg(table->sgl, sg, table->nents, i) {
++ for_each_sgtable_sg(table, sg, i) {
+ struct page *page = sg_page(sg);
+
+ __free_pages(page, compound_order(page));
+diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
+index c57a609db75be..e7330684d3b82 100644
+--- a/drivers/dma-buf/udmabuf.c
++++ b/drivers/dma-buf/udmabuf.c
+@@ -190,6 +190,10 @@ static long udmabuf_create(struct miscdevice *device,
+ if (ubuf->pagecount > pglimit)
+ goto err;
+ }
++
++ if (!ubuf->pagecount)
++ goto err;
++
+ ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages),
+ GFP_KERNEL);
+ if (!ubuf->pages) {
+diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
+index ab78e0f6afd70..177a537971a1b 100644
+--- a/drivers/dma/at_xdmac.c
++++ b/drivers/dma/at_xdmac.c
+@@ -99,6 +99,7 @@
+ #define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */
+ #define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */
+ #define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */
++#define AT_XDMAC_CNDC_NDVIEW_MASK GENMASK(28, 27)
+ #define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */
+ #define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */
+ #define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */
+@@ -155,7 +156,7 @@
+ #define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */
+ #define AT_XDMAC_CC_WRIP_DONE (0x0 << 23)
+ #define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23)
+-#define AT_XDMAC_CC_PERID(i) (0x7f & (i) << 24) /* Channel Peripheral Identifier */
++#define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */
+ #define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */
+ #define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
+ #define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
+@@ -252,15 +253,15 @@ struct at_xdmac {
+
+ /* Linked List Descriptor */
+ struct at_xdmac_lld {
+- dma_addr_t mbr_nda; /* Next Descriptor Member */
+- u32 mbr_ubc; /* Microblock Control Member */
+- dma_addr_t mbr_sa; /* Source Address Member */
+- dma_addr_t mbr_da; /* Destination Address Member */
+- u32 mbr_cfg; /* Configuration Register */
+- u32 mbr_bc; /* Block Control Register */
+- u32 mbr_ds; /* Data Stride Register */
+- u32 mbr_sus; /* Source Microblock Stride Register */
+- u32 mbr_dus; /* Destination Microblock Stride Register */
++ u32 mbr_nda; /* Next Descriptor Member */
++ u32 mbr_ubc; /* Microblock Control Member */
++ u32 mbr_sa; /* Source Address Member */
++ u32 mbr_da; /* Destination Address Member */
++ u32 mbr_cfg; /* Configuration Register */
++ u32 mbr_bc; /* Block Control Register */
++ u32 mbr_ds; /* Data Stride Register */
++ u32 mbr_sus; /* Source Microblock Stride Register */
++ u32 mbr_dus; /* Destination Microblock Stride Register */
+ };
+
+ /* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */
+@@ -385,9 +386,6 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
+
+ dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
+
+- if (at_xdmac_chan_is_enabled(atchan))
+- return;
+-
+ /* Set transfer as active to not try to start it again. */
+ first->active_xfer = true;
+
+@@ -405,7 +403,8 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
+ */
+ if (at_xdmac_chan_is_cyclic(atchan))
+ reg = AT_XDMAC_CNDC_NDVIEW_NDV1;
+- else if (first->lld.mbr_ubc & AT_XDMAC_MBR_UBC_NDV3)
++ else if ((first->lld.mbr_ubc &
++ AT_XDMAC_CNDC_NDVIEW_MASK) == AT_XDMAC_MBR_UBC_NDV3)
+ reg = AT_XDMAC_CNDC_NDVIEW_NDV3;
+ else
+ reg = AT_XDMAC_CNDC_NDVIEW_NDV2;
+@@ -476,13 +475,12 @@ static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx)
+ spin_lock_irqsave(&atchan->lock, irqflags);
+ cookie = dma_cookie_assign(tx);
+
++ list_add_tail(&desc->xfer_node, &atchan->xfers_list);
++ spin_unlock_irqrestore(&atchan->lock, irqflags);
++
+ dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
+ __func__, atchan, desc);
+- list_add_tail(&desc->xfer_node, &atchan->xfers_list);
+- if (list_is_singular(&atchan->xfers_list))
+- at_xdmac_start_xfer(atchan, desc);
+
+- spin_unlock_irqrestore(&atchan->lock, irqflags);
+ return cookie;
+ }
+
+@@ -1452,7 +1450,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
+ {
+ struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
+ struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
+- struct at_xdmac_desc *desc, *_desc;
++ struct at_xdmac_desc *desc, *_desc, *iter;
+ struct list_head *descs_list;
+ enum dma_status ret;
+ int residue, retry;
+@@ -1567,11 +1565,13 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
+ * microblock.
+ */
+ descs_list = &desc->descs_list;
+- list_for_each_entry_safe(desc, _desc, descs_list, desc_node) {
+- dwidth = at_xdmac_get_dwidth(desc->lld.mbr_cfg);
+- residue -= (desc->lld.mbr_ubc & 0xffffff) << dwidth;
+- if ((desc->lld.mbr_nda & 0xfffffffc) == cur_nda)
++ list_for_each_entry_safe(iter, _desc, descs_list, desc_node) {
++ dwidth = at_xdmac_get_dwidth(iter->lld.mbr_cfg);
++ residue -= (iter->lld.mbr_ubc & 0xffffff) << dwidth;
++ if ((iter->lld.mbr_nda & 0xfffffffc) == cur_nda) {
++ desc = iter;
+ break;
++ }
+ }
+ residue += cur_ubc << dwidth;
+
+@@ -1623,14 +1623,17 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
+ struct at_xdmac_desc *desc;
+ struct dma_async_tx_descriptor *txd;
+
+- if (!list_empty(&atchan->xfers_list)) {
+- desc = list_first_entry(&atchan->xfers_list,
+- struct at_xdmac_desc, xfer_node);
+- txd = &desc->tx_dma_desc;
+-
+- if (txd->flags & DMA_PREP_INTERRUPT)
+- dmaengine_desc_get_callback_invoke(txd, NULL);
++ spin_lock_irq(&atchan->lock);
++ if (list_empty(&atchan->xfers_list)) {
++ spin_unlock_irq(&atchan->lock);
++ return;
+ }
++ desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,
++ xfer_node);
++ spin_unlock_irq(&atchan->lock);
++ txd = &desc->tx_dma_desc;
++ if (txd->flags & DMA_PREP_INTERRUPT)
++ dmaengine_desc_get_callback_invoke(txd, NULL);
+ }
+
+ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
+@@ -1784,11 +1787,9 @@ static void at_xdmac_issue_pending(struct dma_chan *chan)
+
+ dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);
+
+- if (!at_xdmac_chan_is_cyclic(atchan)) {
+- spin_lock_irqsave(&atchan->lock, flags);
+- at_xdmac_advance_work(atchan);
+- spin_unlock_irqrestore(&atchan->lock, flags);
+- }
++ spin_lock_irqsave(&atchan->lock, flags);
++ at_xdmac_advance_work(atchan);
++ spin_unlock_irqrestore(&atchan->lock, flags);
+
+ return;
+ }
+@@ -1926,6 +1927,30 @@ static void at_xdmac_free_chan_resources(struct dma_chan *chan)
+ return;
+ }
+
++static void at_xdmac_axi_config(struct platform_device *pdev)
++{
++ struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
++ bool dev_m2m = false;
++ u32 dma_requests;
++
++ if (!atxdmac->layout->axi_config)
++ return; /* Not supported */
++
++ if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
++ &dma_requests)) {
++ dev_info(&pdev->dev, "controller in mem2mem mode.\n");
++ dev_m2m = true;
++ }
++
++ if (dev_m2m) {
++ at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
++ at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
++ } else {
++ at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
++ at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
++ }
++}
++
+ #ifdef CONFIG_PM
+ static int atmel_xdmac_prepare(struct device *dev)
+ {
+@@ -1975,6 +2000,7 @@ static int atmel_xdmac_resume(struct device *dev)
+ struct at_xdmac *atxdmac = dev_get_drvdata(dev);
+ struct at_xdmac_chan *atchan;
+ struct dma_chan *chan, *_chan;
++ struct platform_device *pdev = container_of(dev, struct platform_device, dev);
+ int i;
+ int ret;
+
+@@ -1982,6 +2008,8 @@ static int atmel_xdmac_resume(struct device *dev)
+ if (ret)
+ return ret;
+
++ at_xdmac_axi_config(pdev);
++
+ /* Clear pending interrupts. */
+ for (i = 0; i < atxdmac->dma.chancnt; i++) {
+ atchan = &atxdmac->chan[i];
+@@ -2007,30 +2035,6 @@ static int atmel_xdmac_resume(struct device *dev)
+ }
+ #endif /* CONFIG_PM_SLEEP */
+
+-static void at_xdmac_axi_config(struct platform_device *pdev)
+-{
+- struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
+- bool dev_m2m = false;
+- u32 dma_requests;
+-
+- if (!atxdmac->layout->axi_config)
+- return; /* Not supported */
+-
+- if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
+- &dma_requests)) {
+- dev_info(&pdev->dev, "controller in mem2mem mode.\n");
+- dev_m2m = true;
+- }
+-
+- if (dev_m2m) {
+- at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
+- at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
+- } else {
+- at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
+- at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
+- }
+-}
+-
+ static int at_xdmac_probe(struct platform_device *pdev)
+ {
+ struct at_xdmac *atxdmac;
+diff --git a/drivers/dma/bestcomm/ata.c b/drivers/dma/bestcomm/ata.c
+index 2fd87f83cf90b..e169f18da551f 100644
+--- a/drivers/dma/bestcomm/ata.c
++++ b/drivers/dma/bestcomm/ata.c
+@@ -133,7 +133,7 @@ void bcom_ata_reset_bd(struct bcom_task *tsk)
+ struct bcom_ata_var *var;
+
+ /* Reset all BD */
+- memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
++ memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
+
+ tsk->index = 0;
+ tsk->outdex = 0;
+diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c
+index d91cbbe7a48fb..8c42e5ca00a99 100644
+--- a/drivers/dma/bestcomm/bestcomm.c
++++ b/drivers/dma/bestcomm/bestcomm.c
+@@ -95,7 +95,7 @@ bcom_task_alloc(int bd_count, int bd_size, int priv_size)
+ tsk->bd = bcom_sram_alloc(bd_count * bd_size, 4, &tsk->bd_pa);
+ if (!tsk->bd)
+ goto error;
+- memset(tsk->bd, 0x00, bd_count * bd_size);
++ memset_io(tsk->bd, 0x00, bd_count * bd_size);
+
+ tsk->num_bd = bd_count;
+ tsk->bd_size = bd_size;
+@@ -186,16 +186,16 @@ bcom_load_image(int task, u32 *task_image)
+ inc = bcom_task_inc(task);
+
+ /* Clear & copy */
+- memset(var, 0x00, BCOM_VAR_SIZE);
+- memset(inc, 0x00, BCOM_INC_SIZE);
++ memset_io(var, 0x00, BCOM_VAR_SIZE);
++ memset_io(inc, 0x00, BCOM_INC_SIZE);
+
+ desc_src = (u32 *)(hdr + 1);
+ var_src = desc_src + hdr->desc_size;
+ inc_src = var_src + hdr->var_size;
+
+- memcpy(desc, desc_src, hdr->desc_size * sizeof(u32));
+- memcpy(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32));
+- memcpy(inc, inc_src, hdr->inc_size * sizeof(u32));
++ memcpy_toio(desc, desc_src, hdr->desc_size * sizeof(u32));
++ memcpy_toio(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32));
++ memcpy_toio(inc, inc_src, hdr->inc_size * sizeof(u32));
+
+ return 0;
+ }
+@@ -302,13 +302,13 @@ static int bcom_engine_init(void)
+ return -ENOMEM;
+ }
+
+- memset(bcom_eng->tdt, 0x00, tdt_size);
+- memset(bcom_eng->ctx, 0x00, ctx_size);
+- memset(bcom_eng->var, 0x00, var_size);
+- memset(bcom_eng->fdt, 0x00, fdt_size);
++ memset_io(bcom_eng->tdt, 0x00, tdt_size);
++ memset_io(bcom_eng->ctx, 0x00, ctx_size);
++ memset_io(bcom_eng->var, 0x00, var_size);
++ memset_io(bcom_eng->fdt, 0x00, fdt_size);
+
+ /* Copy the FDT for the EU#3 */
+- memcpy(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops));
++ memcpy_toio(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops));
+
+ /* Initialize Task base structure */
+ for (task=0; task<BCOM_MAX_TASKS; task++)
+diff --git a/drivers/dma/bestcomm/fec.c b/drivers/dma/bestcomm/fec.c
+index 7f1fb1c999e43..d203618ac11fe 100644
+--- a/drivers/dma/bestcomm/fec.c
++++ b/drivers/dma/bestcomm/fec.c
+@@ -140,7 +140,7 @@ bcom_fec_rx_reset(struct bcom_task *tsk)
+ tsk->index = 0;
+ tsk->outdex = 0;
+
+- memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
++ memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
+
+ /* Configure some stuff */
+ bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_RX_BD_PRAGMA);
+@@ -241,7 +241,7 @@ bcom_fec_tx_reset(struct bcom_task *tsk)
+ tsk->index = 0;
+ tsk->outdex = 0;
+
+- memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
++ memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
+
+ /* Configure some stuff */
+ bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_TX_BD_PRAGMA);
+diff --git a/drivers/dma/bestcomm/gen_bd.c b/drivers/dma/bestcomm/gen_bd.c
+index 906ddba6a6f5d..8a24a5cbc2633 100644
+--- a/drivers/dma/bestcomm/gen_bd.c
++++ b/drivers/dma/bestcomm/gen_bd.c
+@@ -142,7 +142,7 @@ bcom_gen_bd_rx_reset(struct bcom_task *tsk)
+ tsk->index = 0;
+ tsk->outdex = 0;
+
+- memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
++ memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
+
+ /* Configure some stuff */
+ bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_RX_BD_PRAGMA);
+@@ -226,7 +226,7 @@ bcom_gen_bd_tx_reset(struct bcom_task *tsk)
+ tsk->index = 0;
+ tsk->outdex = 0;
+
+- memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
++ memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
+
+ /* Configure some stuff */
+ bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_TX_BD_PRAGMA);
+diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h
+index 1bfbd64b13717..53f16d3f00294 100644
+--- a/drivers/dma/dmaengine.h
++++ b/drivers/dma/dmaengine.h
+@@ -176,7 +176,7 @@ dmaengine_desc_get_callback_invoke(struct dma_async_tx_descriptor *tx,
+ static inline bool
+ dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb)
+ {
+- return (cb->callback) ? true : false;
++ return cb->callback || cb->callback_result;
+ }
+
+ struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
+diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
+index 329fc2e57b703..b5b8f8181e776 100644
+--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
++++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
+@@ -415,8 +415,11 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
+ (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
+ /* Linked list */
+ #ifdef CONFIG_64BIT
+- SET_CH_64(dw, chan->dir, chan->id, llp.reg,
+- chunk->ll_region.paddr);
++ /* llp is not aligned on 64bit -> keep 32bit accesses */
++ SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
++ lower_32_bits(chunk->ll_region.paddr));
++ SET_CH_32(dw, chan->dir, chan->id, llp.msb,
++ upper_32_bits(chunk->ll_region.paddr));
+ #else /* CONFIG_64BIT */
+ SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
+ lower_32_bits(chunk->ll_region.paddr));
+diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c
+index c855a0e4f9ff4..f680e9b40bf77 100644
+--- a/drivers/dma/hisi_dma.c
++++ b/drivers/dma/hisi_dma.c
+@@ -30,7 +30,7 @@
+ #define HISI_DMA_MODE 0x217c
+ #define HISI_DMA_OFFSET 0x100
+
+-#define HISI_DMA_MSI_NUM 30
++#define HISI_DMA_MSI_NUM 32
+ #define HISI_DMA_CHAN_NUM 30
+ #define HISI_DMA_Q_DEPTH_VAL 1024
+
+diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
+index 83a5ff2ecf2a0..e622245c93804 100644
+--- a/drivers/dma/idxd/device.c
++++ b/drivers/dma/idxd/device.c
+@@ -394,8 +394,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
+ lockdep_assert_held(&wq->wq_lock);
+ memset(wq->wqcfg, 0, idxd->wqcfg_size);
+ wq->type = IDXD_WQT_NONE;
+- wq->size = 0;
+- wq->group = NULL;
+ wq->threshold = 0;
+ wq->priority = 0;
+ wq->ats_dis = 0;
+@@ -404,6 +402,14 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
+ memset(wq->name, 0, WQ_NAME_SIZE);
+ }
+
++static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)
++{
++ lockdep_assert_held(&wq->wq_lock);
++
++ wq->size = 0;
++ wq->group = NULL;
++}
++
+ static void idxd_wq_ref_release(struct percpu_ref *ref)
+ {
+ struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
+@@ -427,7 +433,6 @@ void idxd_wq_quiesce(struct idxd_wq *wq)
+ {
+ percpu_ref_kill(&wq->wq_active);
+ wait_for_completion(&wq->wq_dead);
+- percpu_ref_exit(&wq->wq_active);
+ }
+
+ /* Device control bits */
+@@ -584,6 +589,8 @@ void idxd_device_reset(struct idxd_device *idxd)
+ spin_lock(&idxd->dev_lock);
+ idxd_device_clear_state(idxd);
+ idxd->state = IDXD_DEV_DISABLED;
++ idxd_unmask_error_interrupts(idxd);
++ idxd_msix_perm_setup(idxd);
+ spin_unlock(&idxd->dev_lock);
+ }
+
+@@ -692,11 +699,16 @@ static void idxd_groups_clear_state(struct idxd_device *idxd)
+ memset(&group->grpcfg, 0, sizeof(group->grpcfg));
+ group->num_engines = 0;
+ group->num_wqs = 0;
+- group->use_token_limit = false;
+- group->tokens_allowed = 0;
+- group->tokens_reserved = 0;
+- group->tc_a = -1;
+- group->tc_b = -1;
++ group->use_rdbuf_limit = false;
++ group->rdbufs_allowed = 0;
++ group->rdbufs_reserved = 0;
++ if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) {
++ group->tc_a = 1;
++ group->tc_b = 1;
++ } else {
++ group->tc_a = -1;
++ group->tc_b = -1;
++ }
+ }
+ }
+
+@@ -712,11 +724,15 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
+ idxd_wq_disable_cleanup(wq);
+ wq->state = IDXD_WQ_DISABLED;
+ }
++ idxd_wq_device_reset_cleanup(wq);
+ }
+ }
+
+ void idxd_device_clear_state(struct idxd_device *idxd)
+ {
++ if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
++ return;
++
+ idxd_groups_clear_state(idxd);
+ idxd_engines_clear_state(idxd);
+ idxd_device_wqs_clear_state(idxd);
+@@ -791,10 +807,10 @@ static int idxd_groups_config_write(struct idxd_device *idxd)
+ int i;
+ struct device *dev = &idxd->pdev->dev;
+
+- /* Setup bandwidth token limit */
+- if (idxd->token_limit) {
++ /* Setup bandwidth rdbuf limit */
++ if (idxd->hw.gen_cap.config_en && idxd->rdbuf_limit) {
+ reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
+- reg.token_limit = idxd->token_limit;
++ reg.rdbuf_limit = idxd->rdbuf_limit;
+ iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
+ }
+
+@@ -935,13 +951,12 @@ static void idxd_group_flags_setup(struct idxd_device *idxd)
+ group->tc_b = group->grpcfg.flags.tc_b = 1;
+ else
+ group->grpcfg.flags.tc_b = group->tc_b;
+- group->grpcfg.flags.use_token_limit = group->use_token_limit;
+- group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
+- if (group->tokens_allowed)
+- group->grpcfg.flags.tokens_allowed =
+- group->tokens_allowed;
++ group->grpcfg.flags.use_rdbuf_limit = group->use_rdbuf_limit;
++ group->grpcfg.flags.rdbufs_reserved = group->rdbufs_reserved;
++ if (group->rdbufs_allowed)
++ group->grpcfg.flags.rdbufs_allowed = group->rdbufs_allowed;
+ else
+- group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
++ group->grpcfg.flags.rdbufs_allowed = idxd->max_rdbufs;
+ }
+ }
+
+@@ -1136,7 +1151,7 @@ int idxd_device_load_config(struct idxd_device *idxd)
+ int i, rc;
+
+ reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
+- idxd->token_limit = reg.token_limit;
++ idxd->rdbuf_limit = reg.rdbuf_limit;
+
+ for (i = 0; i < idxd->max_groups; i++) {
+ struct idxd_group *group = idxd->groups[i];
+diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
+index e0f056c1d1f56..c39e9483206ad 100644
+--- a/drivers/dma/idxd/dma.c
++++ b/drivers/dma/idxd/dma.c
+@@ -311,6 +311,7 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
+
+ err_dma:
+ idxd_wq_quiesce(wq);
++ percpu_ref_exit(&wq->wq_active);
+ err_ref:
+ idxd_wq_free_resources(wq);
+ err_res_alloc:
+@@ -328,9 +329,9 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
+ mutex_lock(&wq->wq_lock);
+ idxd_wq_quiesce(wq);
+ idxd_unregister_dma_channel(wq);
+- __drv_disable_wq(wq);
+ idxd_wq_free_resources(wq);
+- wq->type = IDXD_WQT_NONE;
++ __drv_disable_wq(wq);
++ percpu_ref_exit(&wq->wq_active);
+ mutex_unlock(&wq->wq_lock);
+ }
+
+diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
+index bfcb03329f778..833af18a99ee5 100644
+--- a/drivers/dma/idxd/idxd.h
++++ b/drivers/dma/idxd/idxd.h
+@@ -84,9 +84,9 @@ struct idxd_group {
+ int id;
+ int num_engines;
+ int num_wqs;
+- bool use_token_limit;
+- u8 tokens_allowed;
+- u8 tokens_reserved;
++ bool use_rdbuf_limit;
++ u8 rdbufs_allowed;
++ u8 rdbufs_reserved;
+ int tc_a;
+ int tc_b;
+ };
+@@ -278,11 +278,11 @@ struct idxd_device {
+ u32 max_batch_size;
+ int max_groups;
+ int max_engines;
+- int max_tokens;
++ int max_rdbufs;
+ int max_wqs;
+ int max_wq_size;
+- int token_limit;
+- int nr_tokens; /* non-reserved tokens */
++ int rdbuf_limit;
++ int nr_rdbufs; /* non-reserved read buffers */
+ unsigned int wqcfg_size;
+
+ union sw_err_reg sw_err;
+diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
+index eb09bc591c316..6263d9825250b 100644
+--- a/drivers/dma/idxd/init.c
++++ b/drivers/dma/idxd/init.c
+@@ -464,9 +464,9 @@ static void idxd_read_caps(struct idxd_device *idxd)
+ dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
+ idxd->max_groups = idxd->hw.group_cap.num_groups;
+ dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
+- idxd->max_tokens = idxd->hw.group_cap.total_tokens;
+- dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
+- idxd->nr_tokens = idxd->max_tokens;
++ idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs;
++ dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs);
++ idxd->nr_rdbufs = idxd->max_rdbufs;
+
+ /* read engine capabilities */
+ idxd->hw.engine_cap.bits =
+@@ -797,11 +797,19 @@ static void idxd_remove(struct pci_dev *pdev)
+ int msixcnt = pci_msix_vec_count(pdev);
+ int i;
+
+- dev_dbg(&pdev->dev, "%s called\n", __func__);
++ idxd_unregister_devices(idxd);
++ /*
++ * When ->release() is called for the idxd->conf_dev, it frees all the memory related
++ * to the idxd context. The driver still needs those bits in order to do the rest of
++ * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref
++ * on the device here to hold off the freeing while allowing the idxd sub-driver
++ * to unbind.
++ */
++ get_device(idxd_confdev(idxd));
++ device_unregister(idxd_confdev(idxd));
+ idxd_shutdown(pdev);
+ if (device_pasid_enabled(idxd))
+ idxd_disable_system_pasid(idxd);
+- idxd_unregister_devices(idxd);
+
+ for (i = 0; i < msixcnt; i++) {
+ irq_entry = &idxd->irq_entries[i];
+@@ -815,7 +823,7 @@ static void idxd_remove(struct pci_dev *pdev)
+ pci_disable_device(pdev);
+ destroy_workqueue(idxd->wq);
+ perfmon_pmu_remove(idxd);
+- device_unregister(idxd_confdev(idxd));
++ put_device(idxd_confdev(idxd));
+ }
+
+ static struct pci_driver idxd_pci_driver = {
+diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c
+index ca88fa7a328e7..6d6af0dc3c0ec 100644
+--- a/drivers/dma/idxd/irq.c
++++ b/drivers/dma/idxd/irq.c
+@@ -63,6 +63,9 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
+ int i;
+ bool err = false;
+
++ if (cause & IDXD_INTC_HALT_STATE)
++ goto halt;
++
+ if (cause & IDXD_INTC_ERR) {
+ spin_lock(&idxd->dev_lock);
+ for (i = 0; i < 4; i++)
+@@ -121,6 +124,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
+ if (!err)
+ return 0;
+
++halt:
+ gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
+ if (gensts.state == IDXD_DEVICE_STATE_HALT) {
+ idxd->state = IDXD_DEV_HALTED;
+@@ -133,9 +137,10 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
+ INIT_WORK(&idxd->work, idxd_device_reinit);
+ queue_work(idxd->wq, &idxd->work);
+ } else {
+- spin_lock(&idxd->dev_lock);
++ idxd->state = IDXD_DEV_HALTED;
+ idxd_wqs_quiesce(idxd);
+ idxd_wqs_unmap_portal(idxd);
++ spin_lock(&idxd->dev_lock);
+ idxd_device_clear_state(idxd);
+ dev_err(&idxd->pdev->dev,
+ "idxd halted, need %s.\n",
+diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h
+index ffc7550a77eeb..c0961c1ac161d 100644
+--- a/drivers/dma/idxd/registers.h
++++ b/drivers/dma/idxd/registers.h
+@@ -65,9 +65,9 @@ union wq_cap_reg {
+ union group_cap_reg {
+ struct {
+ u64 num_groups:8;
+- u64 total_tokens:8;
+- u64 token_en:1;
+- u64 token_limit:1;
++ u64 total_rdbufs:8; /* formerly total_tokens */
++ u64 rdbuf_ctrl:1; /* formerly token_en */
++ u64 rdbuf_limit:1; /* formerly token_limit */
+ u64 rsvd:46;
+ };
+ u64 bits;
+@@ -111,7 +111,7 @@ union offsets_reg {
+ #define IDXD_GENCFG_OFFSET 0x80
+ union gencfg_reg {
+ struct {
+- u32 token_limit:8;
++ u32 rdbuf_limit:8;
+ u32 rsvd:4;
+ u32 user_int_en:1;
+ u32 rsvd2:19;
+@@ -158,6 +158,7 @@ enum idxd_device_reset_type {
+ #define IDXD_INTC_CMD 0x02
+ #define IDXD_INTC_OCCUPY 0x04
+ #define IDXD_INTC_PERFMON_OVFL 0x08
++#define IDXD_INTC_HALT_STATE 0x10
+
+ #define IDXD_CMD_OFFSET 0xa0
+ union idxd_command_reg {
+@@ -287,10 +288,10 @@ union group_flags {
+ u32 tc_a:3;
+ u32 tc_b:3;
+ u32 rsvd:1;
+- u32 use_token_limit:1;
+- u32 tokens_reserved:8;
++ u32 use_rdbuf_limit:1;
++ u32 rdbufs_reserved:8;
+ u32 rsvd2:4;
+- u32 tokens_allowed:8;
++ u32 rdbufs_allowed:8;
+ u32 rsvd3:4;
+ };
+ u32 bits;
+diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c
+index de76fb4abac24..83452fbbb168b 100644
+--- a/drivers/dma/idxd/submit.c
++++ b/drivers/dma/idxd/submit.c
+@@ -106,6 +106,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
+ {
+ struct idxd_desc *d, *t, *found = NULL;
+ struct llist_node *head;
++ LIST_HEAD(flist);
+
+ desc->completion->status = IDXD_COMP_DESC_ABORT;
+ /*
+@@ -120,7 +121,11 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
+ found = desc;
+ continue;
+ }
+- list_add_tail(&desc->list, &ie->work_list);
++
++ if (d->completion->status)
++ list_add_tail(&d->list, &flist);
++ else
++ list_add_tail(&d->list, &ie->work_list);
+ }
+ }
+
+@@ -130,6 +135,17 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
+
+ if (found)
+ complete_desc(found, IDXD_COMPLETE_ABORT);
++
++ /*
++ * complete_desc() will return desc to allocator and the desc can be
++ * acquired by a different process and the desc->list can be modified.
++ * Delete desc from list so the list trasversing does not get corrupted
++ * by the other process.
++ */
++ list_for_each_entry_safe(d, t, &flist, list) {
++ list_del_init(&d->list);
++ complete_desc(d, IDXD_COMPLETE_NORMAL);
++ }
+ }
+
+ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
+diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
+index a9025be940db2..33d94c67fedb9 100644
+--- a/drivers/dma/idxd/sysfs.c
++++ b/drivers/dma/idxd/sysfs.c
+@@ -99,17 +99,17 @@ struct device_type idxd_engine_device_type = {
+
+ /* Group attributes */
+
+-static void idxd_set_free_tokens(struct idxd_device *idxd)
++static void idxd_set_free_rdbufs(struct idxd_device *idxd)
+ {
+- int i, tokens;
++ int i, rdbufs;
+
+- for (i = 0, tokens = 0; i < idxd->max_groups; i++) {
++ for (i = 0, rdbufs = 0; i < idxd->max_groups; i++) {
+ struct idxd_group *g = idxd->groups[i];
+
+- tokens += g->tokens_reserved;
++ rdbufs += g->rdbufs_reserved;
+ }
+
+- idxd->nr_tokens = idxd->max_tokens - tokens;
++ idxd->nr_rdbufs = idxd->max_rdbufs - rdbufs;
+ }
+
+ static ssize_t group_tokens_reserved_show(struct device *dev,
+@@ -118,7 +118,7 @@ static ssize_t group_tokens_reserved_show(struct device *dev,
+ {
+ struct idxd_group *group = confdev_to_group(dev);
+
+- return sysfs_emit(buf, "%u\n", group->tokens_reserved);
++ return sysfs_emit(buf, "%u\n", group->rdbufs_reserved);
+ }
+
+ static ssize_t group_tokens_reserved_store(struct device *dev,
+@@ -143,14 +143,14 @@ static ssize_t group_tokens_reserved_store(struct device *dev,
+ if (idxd->state == IDXD_DEV_ENABLED)
+ return -EPERM;
+
+- if (val > idxd->max_tokens)
++ if (val > idxd->max_rdbufs)
+ return -EINVAL;
+
+- if (val > idxd->nr_tokens + group->tokens_reserved)
++ if (val > idxd->nr_rdbufs + group->rdbufs_reserved)
+ return -EINVAL;
+
+- group->tokens_reserved = val;
+- idxd_set_free_tokens(idxd);
++ group->rdbufs_reserved = val;
++ idxd_set_free_rdbufs(idxd);
+ return count;
+ }
+
+@@ -164,7 +164,7 @@ static ssize_t group_tokens_allowed_show(struct device *dev,
+ {
+ struct idxd_group *group = confdev_to_group(dev);
+
+- return sysfs_emit(buf, "%u\n", group->tokens_allowed);
++ return sysfs_emit(buf, "%u\n", group->rdbufs_allowed);
+ }
+
+ static ssize_t group_tokens_allowed_store(struct device *dev,
+@@ -190,10 +190,10 @@ static ssize_t group_tokens_allowed_store(struct device *dev,
+ return -EPERM;
+
+ if (val < 4 * group->num_engines ||
+- val > group->tokens_reserved + idxd->nr_tokens)
++ val > group->rdbufs_reserved + idxd->nr_rdbufs)
+ return -EINVAL;
+
+- group->tokens_allowed = val;
++ group->rdbufs_allowed = val;
+ return count;
+ }
+
+@@ -207,7 +207,7 @@ static ssize_t group_use_token_limit_show(struct device *dev,
+ {
+ struct idxd_group *group = confdev_to_group(dev);
+
+- return sysfs_emit(buf, "%u\n", group->use_token_limit);
++ return sysfs_emit(buf, "%u\n", group->use_rdbuf_limit);
+ }
+
+ static ssize_t group_use_token_limit_store(struct device *dev,
+@@ -232,10 +232,10 @@ static ssize_t group_use_token_limit_store(struct device *dev,
+ if (idxd->state == IDXD_DEV_ENABLED)
+ return -EPERM;
+
+- if (idxd->token_limit == 0)
++ if (idxd->rdbuf_limit == 0)
+ return -EPERM;
+
+- group->use_token_limit = !!val;
++ group->use_rdbuf_limit = !!val;
+ return count;
+ }
+
+@@ -842,6 +842,9 @@ static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attr
+ u64 xfer_size;
+ int rc;
+
++ if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
++ return -EPERM;
++
+ if (wq->state != IDXD_WQ_DISABLED)
+ return -EPERM;
+
+@@ -876,6 +879,9 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu
+ u64 batch_size;
+ int rc;
+
++ if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
++ return -EPERM;
++
+ if (wq->state != IDXD_WQ_DISABLED)
+ return -EPERM;
+
+@@ -1161,7 +1167,7 @@ static ssize_t max_tokens_show(struct device *dev,
+ {
+ struct idxd_device *idxd = confdev_to_idxd(dev);
+
+- return sysfs_emit(buf, "%u\n", idxd->max_tokens);
++ return sysfs_emit(buf, "%u\n", idxd->max_rdbufs);
+ }
+ static DEVICE_ATTR_RO(max_tokens);
+
+@@ -1170,7 +1176,7 @@ static ssize_t token_limit_show(struct device *dev,
+ {
+ struct idxd_device *idxd = confdev_to_idxd(dev);
+
+- return sysfs_emit(buf, "%u\n", idxd->token_limit);
++ return sysfs_emit(buf, "%u\n", idxd->rdbuf_limit);
+ }
+
+ static ssize_t token_limit_store(struct device *dev,
+@@ -1191,13 +1197,13 @@ static ssize_t token_limit_store(struct device *dev,
+ if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
+ return -EPERM;
+
+- if (!idxd->hw.group_cap.token_limit)
++ if (!idxd->hw.group_cap.rdbuf_limit)
+ return -EPERM;
+
+- if (val > idxd->hw.group_cap.total_tokens)
++ if (val > idxd->hw.group_cap.total_rdbufs)
+ return -EINVAL;
+
+- idxd->token_limit = val;
++ idxd->rdbuf_limit = val;
+ return count;
+ }
+ static DEVICE_ATTR_RW(token_limit);
+diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
+index cacc725ca5459..2300d965a3f44 100644
+--- a/drivers/dma/imx-sdma.c
++++ b/drivers/dma/imx-sdma.c
+@@ -198,12 +198,12 @@ struct sdma_script_start_addrs {
+ s32 per_2_firi_addr;
+ s32 mcu_2_firi_addr;
+ s32 uart_2_per_addr;
+- s32 uart_2_mcu_ram_addr;
++ s32 uart_2_mcu_addr;
+ s32 per_2_app_addr;
+ s32 mcu_2_app_addr;
+ s32 per_2_per_addr;
+ s32 uartsh_2_per_addr;
+- s32 uartsh_2_mcu_ram_addr;
++ s32 uartsh_2_mcu_addr;
+ s32 per_2_shp_addr;
+ s32 mcu_2_shp_addr;
+ s32 ata_2_mcu_addr;
+@@ -232,8 +232,8 @@ struct sdma_script_start_addrs {
+ s32 mcu_2_ecspi_addr;
+ s32 mcu_2_sai_addr;
+ s32 sai_2_mcu_addr;
+- s32 uart_2_mcu_addr;
+- s32 uartsh_2_mcu_addr;
++ s32 uart_2_mcu_rom_addr;
++ s32 uartsh_2_mcu_rom_addr;
+ /* End of v3 array */
+ s32 mcu_2_zqspi_addr;
+ /* End of v4 array */
+@@ -1780,17 +1780,17 @@ static void sdma_add_scripts(struct sdma_engine *sdma,
+ saddr_arr[i] = addr_arr[i];
+
+ /*
+- * get uart_2_mcu_addr/uartsh_2_mcu_addr rom script specially because
+- * they are now replaced by uart_2_mcu_ram_addr/uartsh_2_mcu_ram_addr
+- * to be compatible with legacy freescale/nxp sdma firmware, and they
+- * are located in the bottom part of sdma_script_start_addrs which are
+- * beyond the SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1.
++ * For compatibility with NXP internal legacy kernel before 4.19 which
++ * is based on uart ram script and mainline kernel based on uart rom
++ * script, both uart ram/rom scripts are present in newer sdma
++ * firmware. Use the rom versions if they are present (V3 or newer).
+ */
+- if (addr->uart_2_mcu_addr)
+- sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_addr;
+- if (addr->uartsh_2_mcu_addr)
+- sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_addr;
+-
++ if (sdma->script_number >= SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3) {
++ if (addr->uart_2_mcu_rom_addr)
++ sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_rom_addr;
++ if (addr->uartsh_2_mcu_rom_addr)
++ sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_rom_addr;
++ }
+ }
+
+ static void sdma_load_firmware(const struct firmware *fw, void *context)
+@@ -1869,7 +1869,7 @@ static int sdma_event_remap(struct sdma_engine *sdma)
+ u32 reg, val, shift, num_map, i;
+ int ret = 0;
+
+- if (IS_ERR(np) || IS_ERR(gpr_np))
++ if (IS_ERR(np) || !gpr_np)
+ goto out;
+
+ event_remap = of_find_property(np, propname, NULL);
+@@ -1917,7 +1917,7 @@ static int sdma_event_remap(struct sdma_engine *sdma)
+ }
+
+ out:
+- if (!IS_ERR(gpr_np))
++ if (gpr_np)
+ of_node_put(gpr_np);
+
+ return ret;
+diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c
+index 375e7e647df6b..a1517ef1f4a01 100644
+--- a/drivers/dma/mediatek/mtk-uart-apdma.c
++++ b/drivers/dma/mediatek/mtk-uart-apdma.c
+@@ -274,7 +274,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan)
+ unsigned int status;
+ int ret;
+
+- ret = pm_runtime_get_sync(mtkd->ddev.dev);
++ ret = pm_runtime_resume_and_get(mtkd->ddev.dev);
+ if (ret < 0) {
+ pm_runtime_put_noidle(chan->device->dev);
+ return ret;
+@@ -288,18 +288,21 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan)
+ ret = readx_poll_timeout(readl, c->base + VFF_EN,
+ status, !status, 10, 100);
+ if (ret)
+- return ret;
++ goto err_pm;
+
+ ret = request_irq(c->irq, mtk_uart_apdma_irq_handler,
+ IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan);
+ if (ret < 0) {
+ dev_err(chan->device->dev, "Can't request dma IRQ\n");
+- return -EINVAL;
++ ret = -EINVAL;
++ goto err_pm;
+ }
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B);
+
++err_pm:
++ pm_runtime_put_noidle(mtkd->ddev.dev);
+ return ret;
+ }
+
+diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
+index 89f1814ff27a0..26d11885c50ec 100644
+--- a/drivers/dma/mmp_pdma.c
++++ b/drivers/dma/mmp_pdma.c
+@@ -727,12 +727,6 @@ static int mmp_pdma_config_write(struct dma_chan *dchan,
+
+ chan->dir = direction;
+ chan->dev_addr = addr;
+- /* FIXME: drivers should be ported over to use the filter
+- * function. Once that's done, the following two lines can
+- * be removed.
+- */
+- if (cfg->slave_id)
+- chan->drcmr = cfg->slave_id;
+
+ return 0;
+ }
+diff --git a/drivers/dma/ptdma/ptdma-dev.c b/drivers/dma/ptdma/ptdma-dev.c
+index 8a6bf291a73fe..daafea5bc35d9 100644
+--- a/drivers/dma/ptdma/ptdma-dev.c
++++ b/drivers/dma/ptdma/ptdma-dev.c
+@@ -207,7 +207,7 @@ int pt_core_init(struct pt_device *pt)
+ if (!cmd_q->qbase) {
+ dev_err(dev, "unable to allocate command queue\n");
+ ret = -ENOMEM;
+- goto e_dma_alloc;
++ goto e_destroy_pool;
+ }
+
+ cmd_q->qidx = 0;
+@@ -229,8 +229,10 @@ int pt_core_init(struct pt_device *pt)
+
+ /* Request an irq */
+ ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
+- if (ret)
+- goto e_pool;
++ if (ret) {
++ dev_err(dev, "unable to allocate an IRQ\n");
++ goto e_free_dma;
++ }
+
+ /* Update the device registers with queue information. */
+ cmd_q->qcontrol &= ~CMD_Q_SIZE;
+@@ -250,21 +252,20 @@ int pt_core_init(struct pt_device *pt)
+ /* Register the DMA engine support */
+ ret = pt_dmaengine_register(pt);
+ if (ret)
+- goto e_dmaengine;
++ goto e_free_irq;
+
+ /* Set up debugfs entries */
+ ptdma_debugfs_setup(pt);
+
+ return 0;
+
+-e_dmaengine:
++e_free_irq:
+ free_irq(pt->pt_irq, pt);
+
+-e_dma_alloc:
++e_free_dma:
+ dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
+
+-e_pool:
+- dev_err(dev, "unable to allocate an IRQ\n");
++e_destroy_pool:
+ dma_pool_destroy(pt->cmd_q.dma_pool);
+
+ return ret;
+diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
+index 4a2a796e348c1..aa6e552249ab9 100644
+--- a/drivers/dma/pxa_dma.c
++++ b/drivers/dma/pxa_dma.c
+@@ -910,13 +910,6 @@ static void pxad_get_config(struct pxad_chan *chan,
+ *dcmd |= PXA_DCMD_BURST16;
+ else if (maxburst == 32)
+ *dcmd |= PXA_DCMD_BURST32;
+-
+- /* FIXME: drivers should be ported over to use the filter
+- * function. Once that's done, the following two lines can
+- * be removed.
+- */
+- if (chan->cfg.slave_id)
+- chan->drcmr = chan->cfg.slave_id;
+ }
+
+ static struct dma_async_tx_descriptor *
+diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
+index 6885b3dcd7a97..f4c46b3b6d9d7 100644
+--- a/drivers/dma/sh/rcar-dmac.c
++++ b/drivers/dma/sh/rcar-dmac.c
+@@ -1868,8 +1868,13 @@ static int rcar_dmac_probe(struct platform_device *pdev)
+
+ dmac->dev = &pdev->dev;
+ platform_set_drvdata(pdev, dmac);
+- dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK);
+- dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40));
++ ret = dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK);
++ if (ret)
++ return ret;
++
++ ret = dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40));
++ if (ret)
++ return ret;
+
+ ret = rcar_dmac_parse_of(&pdev->dev, dmac);
+ if (ret < 0)
+diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c
+index 962b6e05287b5..d95c421877fb7 100644
+--- a/drivers/dma/st_fdma.c
++++ b/drivers/dma/st_fdma.c
+@@ -874,4 +874,4 @@ MODULE_LICENSE("GPL v2");
+ MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver");
+ MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>");
+ MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>");
+-MODULE_ALIAS("platform: " DRIVER_NAME);
++MODULE_ALIAS("platform:" DRIVER_NAME);
+diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
+index 9063c727962ed..7dfc743ac4338 100644
+--- a/drivers/dma/stm32-dma.c
++++ b/drivers/dma/stm32-dma.c
+@@ -270,7 +270,6 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
+ u32 threshold)
+ {
+ enum dma_slave_buswidth max_width;
+- u64 addr = buf_addr;
+
+ if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL)
+ max_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+@@ -281,7 +280,7 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
+ max_width > DMA_SLAVE_BUSWIDTH_1_BYTE)
+ max_width = max_width >> 1;
+
+- if (do_div(addr, max_width))
++ if (buf_addr & (max_width - 1))
+ max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+
+ return max_width;
+@@ -753,8 +752,14 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
+ if (src_bus_width < 0)
+ return src_bus_width;
+
+- /* Set memory burst size */
+- src_maxburst = STM32_DMA_MAX_BURST;
++ /*
++ * Set memory burst size - burst not possible if address is not aligned on
++ * the address boundary equal to the size of the transfer
++ */
++ if (buf_addr & (buf_len - 1))
++ src_maxburst = 1;
++ else
++ src_maxburst = STM32_DMA_MAX_BURST;
+ src_best_burst = stm32_dma_get_best_burst(buf_len,
+ src_maxburst,
+ fifoth,
+@@ -803,8 +808,14 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
+ if (dst_bus_width < 0)
+ return dst_bus_width;
+
+- /* Set memory burst size */
+- dst_maxburst = STM32_DMA_MAX_BURST;
++ /*
++ * Set memory burst size - burst not possible if address is not aligned on
++ * the address boundary equal to the size of the transfer
++ */
++ if (buf_addr & (buf_len - 1))
++ dst_maxburst = 1;
++ else
++ dst_maxburst = STM32_DMA_MAX_BURST;
+ dst_best_burst = stm32_dma_get_best_burst(buf_len,
+ dst_maxburst,
+ fifoth,
+diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c
+index a42164389ebc2..d5d55732adba1 100644
+--- a/drivers/dma/stm32-dmamux.c
++++ b/drivers/dma/stm32-dmamux.c
+@@ -292,10 +292,12 @@ static int stm32_dmamux_probe(struct platform_device *pdev)
+ ret = of_dma_router_register(node, stm32_dmamux_route_allocate,
+ &stm32_dmamux->dmarouter);
+ if (ret)
+- goto err_clk;
++ goto pm_disable;
+
+ return 0;
+
++pm_disable:
++ pm_runtime_disable(&pdev->dev);
+ err_clk:
+ clk_disable_unprepare(stm32_dmamux->clk);
+
+diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c
+index 18cbd1e43c2e8..f17a9ffcd00da 100644
+--- a/drivers/dma/stm32-mdma.c
++++ b/drivers/dma/stm32-mdma.c
+@@ -184,7 +184,7 @@
+ #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))
+ #define STM32_MDMA_CTBR_DBUS BIT(17)
+ #define STM32_MDMA_CTBR_SBUS BIT(16)
+-#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(7, 0)
++#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0)
+ #define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \
+ STM32_MDMA_CTBR_TSEL_MASK)
+
+diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
+index b1115a6d1935c..d1dff3a29db59 100644
+--- a/drivers/dma/tegra210-adma.c
++++ b/drivers/dma/tegra210-adma.c
+@@ -867,7 +867,7 @@ static int tegra_adma_probe(struct platform_device *pdev)
+
+ pm_runtime_enable(&pdev->dev);
+
+- ret = pm_runtime_get_sync(&pdev->dev);
++ ret = pm_runtime_resume_and_get(&pdev->dev);
+ if (ret < 0)
+ goto rpm_disable;
+
+diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
+index a35858610780c..041d8e32d6300 100644
+--- a/drivers/dma/ti/k3-udma.c
++++ b/drivers/dma/ti/k3-udma.c
+@@ -1348,6 +1348,7 @@ static int bcdma_get_bchan(struct udma_chan *uc)
+ {
+ struct udma_dev *ud = uc->ud;
+ enum udma_tp_level tpl;
++ int ret;
+
+ if (uc->bchan) {
+ dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
+@@ -1365,8 +1366,11 @@ static int bcdma_get_bchan(struct udma_chan *uc)
+ tpl = ud->bchan_tpl.levels - 1;
+
+ uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
+- if (IS_ERR(uc->bchan))
+- return PTR_ERR(uc->bchan);
++ if (IS_ERR(uc->bchan)) {
++ ret = PTR_ERR(uc->bchan);
++ uc->bchan = NULL;
++ return ret;
++ }
+
+ uc->tchan = uc->bchan;
+
+@@ -1376,6 +1380,7 @@ static int bcdma_get_bchan(struct udma_chan *uc)
+ static int udma_get_tchan(struct udma_chan *uc)
+ {
+ struct udma_dev *ud = uc->ud;
++ int ret;
+
+ if (uc->tchan) {
+ dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
+@@ -1390,8 +1395,11 @@ static int udma_get_tchan(struct udma_chan *uc)
+ */
+ uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
+ uc->config.mapped_channel_id);
+- if (IS_ERR(uc->tchan))
+- return PTR_ERR(uc->tchan);
++ if (IS_ERR(uc->tchan)) {
++ ret = PTR_ERR(uc->tchan);
++ uc->tchan = NULL;
++ return ret;
++ }
+
+ if (ud->tflow_cnt) {
+ int tflow_id;
+@@ -1421,6 +1429,7 @@ static int udma_get_tchan(struct udma_chan *uc)
+ static int udma_get_rchan(struct udma_chan *uc)
+ {
+ struct udma_dev *ud = uc->ud;
++ int ret;
+
+ if (uc->rchan) {
+ dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
+@@ -1435,8 +1444,13 @@ static int udma_get_rchan(struct udma_chan *uc)
+ */
+ uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
+ uc->config.mapped_channel_id);
++ if (IS_ERR(uc->rchan)) {
++ ret = PTR_ERR(uc->rchan);
++ uc->rchan = NULL;
++ return ret;
++ }
+
+- return PTR_ERR_OR_ZERO(uc->rchan);
++ return 0;
+ }
+
+ static int udma_get_chan_pair(struct udma_chan *uc)
+@@ -1490,6 +1504,7 @@ static int udma_get_chan_pair(struct udma_chan *uc)
+ static int udma_get_rflow(struct udma_chan *uc, int flow_id)
+ {
+ struct udma_dev *ud = uc->ud;
++ int ret;
+
+ if (!uc->rchan) {
+ dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
+@@ -1503,8 +1518,13 @@ static int udma_get_rflow(struct udma_chan *uc, int flow_id)
+ }
+
+ uc->rflow = __udma_get_rflow(ud, flow_id);
++ if (IS_ERR(uc->rflow)) {
++ ret = PTR_ERR(uc->rflow);
++ uc->rflow = NULL;
++ return ret;
++ }
+
+- return PTR_ERR_OR_ZERO(uc->rflow);
++ return 0;
+ }
+
+ static void bcdma_put_bchan(struct udma_chan *uc)
+diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c
+index d6b8a202474f4..290836b7e1be2 100644
+--- a/drivers/dma/uniphier-xdmac.c
++++ b/drivers/dma/uniphier-xdmac.c
+@@ -131,8 +131,9 @@ uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc)
+ static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc,
+ struct uniphier_xdmac_desc *xd)
+ {
+- u32 src_mode, src_addr, src_width;
+- u32 dst_mode, dst_addr, dst_width;
++ u32 src_mode, src_width;
++ u32 dst_mode, dst_width;
++ dma_addr_t src_addr, dst_addr;
+ u32 val, its, tnum;
+ enum dma_slave_buswidth buswidth;
+
+diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c
+index b280a53e8570a..ce5c66e6897d2 100644
+--- a/drivers/dma/xilinx/xilinx_dpdma.c
++++ b/drivers/dma/xilinx/xilinx_dpdma.c
+@@ -271,9 +271,6 @@ struct xilinx_dpdma_device {
+ /* -----------------------------------------------------------------------------
+ * DebugFS
+ */
+-
+-#ifdef CONFIG_DEBUG_FS
+-
+ #define XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE 32
+ #define XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR "65535"
+
+@@ -299,7 +296,7 @@ struct xilinx_dpdma_debugfs_request {
+
+ static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan)
+ {
+- if (chan->id == dpdma_debugfs.chan_id)
++ if (IS_ENABLED(CONFIG_DEBUG_FS) && chan->id == dpdma_debugfs.chan_id)
+ dpdma_debugfs.xilinx_dpdma_irq_done_count++;
+ }
+
+@@ -462,16 +459,6 @@ static void xilinx_dpdma_debugfs_init(struct xilinx_dpdma_device *xdev)
+ dev_err(xdev->dev, "Failed to create debugfs testcase file\n");
+ }
+
+-#else
+-static void xilinx_dpdma_debugfs_init(struct xilinx_dpdma_device *xdev)
+-{
+-}
+-
+-static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan)
+-{
+-}
+-#endif /* CONFIG_DEBUG_FS */
+-
+ /* -----------------------------------------------------------------------------
+ * I/O Accessors
+ */
+diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
+index 3a6d2416cb0f6..5dd29789f97d3 100644
+--- a/drivers/edac/altera_edac.c
++++ b/drivers/edac/altera_edac.c
+@@ -350,7 +350,7 @@ static int altr_sdram_probe(struct platform_device *pdev)
+ if (irq < 0) {
+ edac_printk(KERN_ERR, EDAC_MC,
+ "No irq %d in DT\n", irq);
+- return -ENODEV;
++ return irq;
+ }
+
+ /* Arria10 has a 2nd IRQ */
+diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
+index 99b06a3e8fb12..4fce75013674f 100644
+--- a/drivers/edac/amd64_edac.c
++++ b/drivers/edac/amd64_edac.c
+@@ -1065,12 +1065,14 @@ static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
+ #define CS_ODD_PRIMARY BIT(1)
+ #define CS_EVEN_SECONDARY BIT(2)
+ #define CS_ODD_SECONDARY BIT(3)
++#define CS_3R_INTERLEAVE BIT(4)
+
+ #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
+ #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
+
+ static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
+ {
++ u8 base, count = 0;
+ int cs_mode = 0;
+
+ if (csrow_enabled(2 * dimm, ctrl, pvt))
+@@ -1083,6 +1085,20 @@ static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
+ if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
+ cs_mode |= CS_ODD_SECONDARY;
+
++ /*
++ * 3 Rank inteleaving support.
++ * There should be only three bases enabled and their two masks should
++ * be equal.
++ */
++ for_each_chip_select(base, ctrl, pvt)
++ count += csrow_enabled(base, ctrl, pvt);
++
++ if (count == 3 &&
++ pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) {
++ edac_dbg(1, "3R interleaving in use.\n");
++ cs_mode |= CS_3R_INTERLEAVE;
++ }
++
+ return cs_mode;
+ }
+
+@@ -1891,10 +1907,14 @@ static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
+ *
+ * The MSB is the number of bits in the full mask because BIT[0] is
+ * always 0.
++ *
++ * In the special 3 Rank interleaving case, a single bit is flipped
++ * without swapping with the most significant bit. This can be handled
++ * by keeping the MSB where it is and ignoring the single zero bit.
+ */
+ msb = fls(addr_mask_orig) - 1;
+ weight = hweight_long(addr_mask_orig);
+- num_zero_bits = msb - weight;
++ num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
+
+ /* Take the number of zero bits off from the top of the mask. */
+ addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
+diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
+index 2c5975674723a..a859ddd9d4a13 100644
+--- a/drivers/edac/edac_mc.c
++++ b/drivers/edac/edac_mc.c
+@@ -215,7 +215,7 @@ void *edac_align_ptr(void **p, unsigned int size, int n_elems)
+ else
+ return (char *)ptr;
+
+- r = (unsigned long)p % align;
++ r = (unsigned long)ptr % align;
+
+ if (r == 0)
+ return (char *)ptr;
+diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
+index 83345bfac246f..6cf50ee0b77c5 100644
+--- a/drivers/edac/i10nm_base.c
++++ b/drivers/edac/i10nm_base.c
+@@ -358,6 +358,9 @@ static int i10nm_get_hbm_munits(void)
+
+ mbase = ioremap(base + off, I10NM_HBM_IMC_MMIO_SIZE);
+ if (!mbase) {
++ pci_dev_put(d->imc[lmc].mdev);
++ d->imc[lmc].mdev = NULL;
++
+ i10nm_printk(KERN_ERR, "Failed to ioremap for hbm mc 0x%llx\n",
+ base + off);
+ return -ENOMEM;
+@@ -368,6 +371,12 @@ static int i10nm_get_hbm_munits(void)
+
+ mcmtr = I10NM_GET_MCMTR(&d->imc[lmc], 0);
+ if (!I10NM_IS_HBM_IMC(mcmtr)) {
++ iounmap(d->imc[lmc].mbase);
++ d->imc[lmc].mbase = NULL;
++ d->imc[lmc].hbm_mc = false;
++ pci_dev_put(d->imc[lmc].mdev);
++ d->imc[lmc].mdev = NULL;
++
+ i10nm_printk(KERN_ERR, "This isn't an hbm mc!\n");
+ return -ENODEV;
+ }
+diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
+index 4c626fcd4dcbb..1522d4aa2ca62 100644
+--- a/drivers/edac/sb_edac.c
++++ b/drivers/edac/sb_edac.c
+@@ -1052,7 +1052,7 @@ static u64 haswell_get_tohm(struct sbridge_pvt *pvt)
+ pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, &reg);
+ rc = ((reg << 6) | rc) << 26;
+
+- return rc | 0x1ffffff;
++ return rc | 0x3ffffff;
+ }
+
+ static u64 knl_get_tolm(struct sbridge_pvt *pvt)
+diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c
+index 7d08627e738b3..8557781bb8dce 100644
+--- a/drivers/edac/synopsys_edac.c
++++ b/drivers/edac/synopsys_edac.c
+@@ -163,6 +163,11 @@
+ #define ECC_STAT_CECNT_SHIFT 8
+ #define ECC_STAT_BITNUM_MASK 0x7F
+
++/* ECC error count register definitions */
++#define ECC_ERRCNT_UECNT_MASK 0xFFFF0000
++#define ECC_ERRCNT_UECNT_SHIFT 16
++#define ECC_ERRCNT_CECNT_MASK 0xFFFF
++
+ /* DDR QOS Interrupt register definitions */
+ #define DDR_QOS_IRQ_STAT_OFST 0x20200
+ #define DDR_QOSUE_MASK 0x4
+@@ -418,15 +423,16 @@ static int zynqmp_get_error_info(struct synps_edac_priv *priv)
+ base = priv->baseaddr;
+ p = &priv->stat;
+
++ regval = readl(base + ECC_ERRCNT_OFST);
++ p->ce_cnt = regval & ECC_ERRCNT_CECNT_MASK;
++ p->ue_cnt = (regval & ECC_ERRCNT_UECNT_MASK) >> ECC_ERRCNT_UECNT_SHIFT;
++ if (!p->ce_cnt)
++ goto ue_err;
++
+ regval = readl(base + ECC_STAT_OFST);
+ if (!regval)
+ return 1;
+
+- p->ce_cnt = (regval & ECC_STAT_CECNT_MASK) >> ECC_STAT_CECNT_SHIFT;
+- p->ue_cnt = (regval & ECC_STAT_UECNT_MASK) >> ECC_STAT_UECNT_SHIFT;
+- if (!p->ce_cnt)
+- goto ue_err;
+-
+ p->ceinfo.bitpos = (regval & ECC_STAT_BITNUM_MASK);
+
+ regval = readl(base + ECC_CEADDR0_OFST);
+@@ -1352,8 +1358,7 @@ static int mc_probe(struct platform_device *pdev)
+ }
+ }
+
+- if (of_device_is_compatible(pdev->dev.of_node,
+- "xlnx,zynqmp-ddrc-2.40a"))
++ if (priv->p_data->quirks & DDR_ECC_INTR_SUPPORT)
+ setup_address_map(priv);
+ #endif
+
+diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c
+index 2ccd1db5e98ff..7197f9fa02457 100644
+--- a/drivers/edac/xgene_edac.c
++++ b/drivers/edac/xgene_edac.c
+@@ -1919,7 +1919,7 @@ static int xgene_edac_probe(struct platform_device *pdev)
+ irq = platform_get_irq_optional(pdev, i);
+ if (irq < 0) {
+ dev_err(&pdev->dev, "No IRQ resource\n");
+- rc = -EINVAL;
++ rc = irq;
+ goto out_err;
+ }
+ rc = devm_request_irq(&pdev->dev, irq,
+diff --git a/drivers/firewire/core-card.c b/drivers/firewire/core-card.c
+index 54be88167c60b..f3b3953cac834 100644
+--- a/drivers/firewire/core-card.c
++++ b/drivers/firewire/core-card.c
+@@ -668,6 +668,7 @@ EXPORT_SYMBOL_GPL(fw_card_release);
+ void fw_core_remove_card(struct fw_card *card)
+ {
+ struct fw_card_driver dummy_driver = dummy_driver_template;
++ unsigned long flags;
+
+ card->driver->update_phy_reg(card, 4,
+ PHY_LINK_ACTIVE | PHY_CONTENDER, 0);
+@@ -682,7 +683,9 @@ void fw_core_remove_card(struct fw_card *card)
+ dummy_driver.stop_iso = card->driver->stop_iso;
+ card->driver = &dummy_driver;
+
++ spin_lock_irqsave(&card->lock, flags);
+ fw_destroy_nodes(card);
++ spin_unlock_irqrestore(&card->lock, flags);
+
+ /* Wait for all users, especially device workqueue jobs, to finish. */
+ fw_card_put(card);
+diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
+index fb6c651214f32..b0cc3f1e9bb00 100644
+--- a/drivers/firewire/core-cdev.c
++++ b/drivers/firewire/core-cdev.c
+@@ -1480,6 +1480,7 @@ static void outbound_phy_packet_callback(struct fw_packet *packet,
+ {
+ struct outbound_phy_packet_event *e =
+ container_of(packet, struct outbound_phy_packet_event, p);
++ struct client *e_client;
+
+ switch (status) {
+ /* expected: */
+@@ -1496,9 +1497,10 @@ static void outbound_phy_packet_callback(struct fw_packet *packet,
+ }
+ e->phy_packet.data[0] = packet->timestamp;
+
++ e_client = e->client;
+ queue_event(e->client, &e->event, &e->phy_packet,
+ sizeof(e->phy_packet) + e->phy_packet.length, NULL, 0);
+- client_put(e->client);
++ client_put(e_client);
+ }
+
+ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
+diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c
+index b63d55f5ebd33..f40c815343812 100644
+--- a/drivers/firewire/core-topology.c
++++ b/drivers/firewire/core-topology.c
+@@ -375,16 +375,13 @@ static void report_found_node(struct fw_card *card,
+ card->bm_retries = 0;
+ }
+
++/* Must be called with card->lock held */
+ void fw_destroy_nodes(struct fw_card *card)
+ {
+- unsigned long flags;
+-
+- spin_lock_irqsave(&card->lock, flags);
+ card->color++;
+ if (card->local_node != NULL)
+ for_each_fw_node(card, card->local_node, report_lost_node);
+ card->local_node = NULL;
+- spin_unlock_irqrestore(&card->lock, flags);
+ }
+
+ static void move_tree(struct fw_node *node0, struct fw_node *node1, int port)
+@@ -510,6 +507,8 @@ void fw_core_handle_bus_reset(struct fw_card *card, int node_id, int generation,
+ struct fw_node *local_node;
+ unsigned long flags;
+
++ spin_lock_irqsave(&card->lock, flags);
++
+ /*
+ * If the selfID buffer is not the immediate successor of the
+ * previously processed one, we cannot reliably compare the
+@@ -521,8 +520,6 @@ void fw_core_handle_bus_reset(struct fw_card *card, int node_id, int generation,
+ card->bm_retries = 0;
+ }
+
+- spin_lock_irqsave(&card->lock, flags);
+-
+ card->broadcast_channel_allocated = card->broadcast_channel_auto_allocated;
+ card->node_id = node_id;
+ /*
+diff --git a/drivers/firewire/core-transaction.c b/drivers/firewire/core-transaction.c
+index ac487c96bb717..6c20815cc8d16 100644
+--- a/drivers/firewire/core-transaction.c
++++ b/drivers/firewire/core-transaction.c
+@@ -73,24 +73,25 @@ static int try_cancel_split_timeout(struct fw_transaction *t)
+ static int close_transaction(struct fw_transaction *transaction,
+ struct fw_card *card, int rcode)
+ {
+- struct fw_transaction *t;
++ struct fw_transaction *t = NULL, *iter;
+ unsigned long flags;
+
+ spin_lock_irqsave(&card->lock, flags);
+- list_for_each_entry(t, &card->transaction_list, link) {
+- if (t == transaction) {
+- if (!try_cancel_split_timeout(t)) {
++ list_for_each_entry(iter, &card->transaction_list, link) {
++ if (iter == transaction) {
++ if (!try_cancel_split_timeout(iter)) {
+ spin_unlock_irqrestore(&card->lock, flags);
+ goto timed_out;
+ }
+- list_del_init(&t->link);
+- card->tlabel_mask &= ~(1ULL << t->tlabel);
++ list_del_init(&iter->link);
++ card->tlabel_mask &= ~(1ULL << iter->tlabel);
++ t = iter;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&card->lock, flags);
+
+- if (&t->link != &card->transaction_list) {
++ if (t) {
+ t->callback(card, rcode, NULL, 0, t->callback_data);
+ return 0;
+ }
+@@ -935,7 +936,7 @@ EXPORT_SYMBOL(fw_core_handle_request);
+
+ void fw_core_handle_response(struct fw_card *card, struct fw_packet *p)
+ {
+- struct fw_transaction *t;
++ struct fw_transaction *t = NULL, *iter;
+ unsigned long flags;
+ u32 *data;
+ size_t data_length;
+@@ -947,20 +948,21 @@ void fw_core_handle_response(struct fw_card *card, struct fw_packet *p)
+ rcode = HEADER_GET_RCODE(p->header[1]);
+
+ spin_lock_irqsave(&card->lock, flags);
+- list_for_each_entry(t, &card->transaction_list, link) {
+- if (t->node_id == source && t->tlabel == tlabel) {
+- if (!try_cancel_split_timeout(t)) {
++ list_for_each_entry(iter, &card->transaction_list, link) {
++ if (iter->node_id == source && iter->tlabel == tlabel) {
++ if (!try_cancel_split_timeout(iter)) {
+ spin_unlock_irqrestore(&card->lock, flags);
+ goto timed_out;
+ }
+- list_del_init(&t->link);
+- card->tlabel_mask &= ~(1ULL << t->tlabel);
++ list_del_init(&iter->link);
++ card->tlabel_mask &= ~(1ULL << iter->tlabel);
++ t = iter;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&card->lock, flags);
+
+- if (&t->link == &card->transaction_list) {
++ if (!t) {
+ timed_out:
+ fw_notice(card, "unsolicited response (source %x, tlabel %x)\n",
+ source, tlabel);
+diff --git a/drivers/firewire/sbp2.c b/drivers/firewire/sbp2.c
+index 4d5054211550b..2ceed9287435f 100644
+--- a/drivers/firewire/sbp2.c
++++ b/drivers/firewire/sbp2.c
+@@ -408,7 +408,7 @@ static void sbp2_status_write(struct fw_card *card, struct fw_request *request,
+ void *payload, size_t length, void *callback_data)
+ {
+ struct sbp2_logical_unit *lu = callback_data;
+- struct sbp2_orb *orb;
++ struct sbp2_orb *orb = NULL, *iter;
+ struct sbp2_status status;
+ unsigned long flags;
+
+@@ -433,17 +433,18 @@ static void sbp2_status_write(struct fw_card *card, struct fw_request *request,
+
+ /* Lookup the orb corresponding to this status write. */
+ spin_lock_irqsave(&lu->tgt->lock, flags);
+- list_for_each_entry(orb, &lu->orb_list, link) {
++ list_for_each_entry(iter, &lu->orb_list, link) {
+ if (STATUS_GET_ORB_HIGH(status) == 0 &&
+- STATUS_GET_ORB_LOW(status) == orb->request_bus) {
+- orb->rcode = RCODE_COMPLETE;
+- list_del(&orb->link);
++ STATUS_GET_ORB_LOW(status) == iter->request_bus) {
++ iter->rcode = RCODE_COMPLETE;
++ list_del(&iter->link);
++ orb = iter;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&lu->tgt->lock, flags);
+
+- if (&orb->link != &lu->orb_list) {
++ if (orb) {
+ orb->callback(orb, &status);
+ kref_put(&orb->kref, free_orb); /* orb callback reference */
+ } else {
+diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
+index de416f9e79213..f5219334fd3a5 100644
+--- a/drivers/firmware/arm_scmi/base.c
++++ b/drivers/firmware/arm_scmi/base.c
+@@ -34,6 +34,12 @@ struct scmi_msg_resp_base_attributes {
+ __le16 reserved;
+ };
+
++struct scmi_msg_resp_base_discover_agent {
++ __le32 agent_id;
++ u8 name[SCMI_MAX_STR_SIZE];
++};
++
++
+ struct scmi_msg_base_error_notify {
+ __le32 event_control;
+ #define BASE_TP_NOTIFY_ALL BIT(0)
+@@ -225,18 +231,21 @@ static int scmi_base_discover_agent_get(const struct scmi_protocol_handle *ph,
+ int id, char *name)
+ {
+ int ret;
++ struct scmi_msg_resp_base_discover_agent *agent_info;
+ struct scmi_xfer *t;
+
+ ret = ph->xops->xfer_get_init(ph, BASE_DISCOVER_AGENT,
+- sizeof(__le32), SCMI_MAX_STR_SIZE, &t);
++ sizeof(__le32), sizeof(*agent_info), &t);
+ if (ret)
+ return ret;
+
+ put_unaligned_le32(id, t->tx.buf);
+
+ ret = ph->xops->do_xfer(ph, t);
+- if (!ret)
+- strlcpy(name, t->rx.buf, SCMI_MAX_STR_SIZE);
++ if (!ret) {
++ agent_info = t->rx.buf;
++ strlcpy(name, agent_info->name, SCMI_MAX_STR_SIZE);
++ }
+
+ ph->xops->xfer_put(ph, t);
+
+diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
+index 35b56c8ba0c0e..492f3a9197ec2 100644
+--- a/drivers/firmware/arm_scmi/clock.c
++++ b/drivers/firmware/arm_scmi/clock.c
+@@ -204,7 +204,8 @@ scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
+
+ if (rate_discrete && rate) {
+ clk->list.num_rates = tot_rate_cnt;
+- sort(rate, tot_rate_cnt, sizeof(*rate), rate_cmp_func, NULL);
++ sort(clk->list.rates, tot_rate_cnt, sizeof(*rate),
++ rate_cmp_func, NULL);
+ }
+
+ clk->rate_discrete = rate_discrete;
+diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
+index b406b3f78f467..e815b8f987393 100644
+--- a/drivers/firmware/arm_scmi/driver.c
++++ b/drivers/firmware/arm_scmi/driver.c
+@@ -652,7 +652,8 @@ static void scmi_handle_response(struct scmi_chan_info *cinfo,
+
+ xfer = scmi_xfer_command_acquire(cinfo, msg_hdr);
+ if (IS_ERR(xfer)) {
+- scmi_clear_channel(info, cinfo);
++ if (MSG_XTRACT_TYPE(msg_hdr) == MSG_TYPE_DELAYED_RESP)
++ scmi_clear_channel(info, cinfo);
+ return;
+ }
+
+@@ -2112,7 +2113,7 @@ static void __exit scmi_driver_exit(void)
+ }
+ module_exit(scmi_driver_exit);
+
+-MODULE_ALIAS("platform: arm-scmi");
++MODULE_ALIAS("platform:arm-scmi");
+ MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");
+ MODULE_DESCRIPTION("ARM SCMI protocol driver");
+ MODULE_LICENSE("GPL v2");
+diff --git a/drivers/firmware/arm_scmi/scmi_pm_domain.c b/drivers/firmware/arm_scmi/scmi_pm_domain.c
+index 4371fdcd5a73f..581d34c957695 100644
+--- a/drivers/firmware/arm_scmi/scmi_pm_domain.c
++++ b/drivers/firmware/arm_scmi/scmi_pm_domain.c
+@@ -138,9 +138,7 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev)
+ scmi_pd_data->domains = domains;
+ scmi_pd_data->num_domains = num_domains;
+
+- of_genpd_add_provider_onecell(np, scmi_pd_data);
+-
+- return 0;
++ return of_genpd_add_provider_onecell(np, scmi_pd_data);
+ }
+
+ static const struct scmi_device_id scmi_id_table[] = {
+diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
+index 308471586381f..cdbb287bd8bcd 100644
+--- a/drivers/firmware/arm_scmi/sensors.c
++++ b/drivers/firmware/arm_scmi/sensors.c
+@@ -637,7 +637,7 @@ static int scmi_sensor_config_get(const struct scmi_protocol_handle *ph,
+ if (ret)
+ return ret;
+
+- put_unaligned_le32(cpu_to_le32(sensor_id), t->tx.buf);
++ put_unaligned_le32(sensor_id, t->tx.buf);
+ ret = ph->xops->do_xfer(ph, t);
+ if (!ret) {
+ struct sensors_info *si = ph->get_priv(ph);
+diff --git a/drivers/firmware/arm_scmi/virtio.c b/drivers/firmware/arm_scmi/virtio.c
+index 11e8efb713751..87039c5c03fdb 100644
+--- a/drivers/firmware/arm_scmi/virtio.c
++++ b/drivers/firmware/arm_scmi/virtio.c
+@@ -82,7 +82,8 @@ static bool scmi_vio_have_vq_rx(struct virtio_device *vdev)
+ }
+
+ static int scmi_vio_feed_vq_rx(struct scmi_vio_channel *vioch,
+- struct scmi_vio_msg *msg)
++ struct scmi_vio_msg *msg,
++ struct device *dev)
+ {
+ struct scatterlist sg_in;
+ int rc;
+@@ -94,8 +95,7 @@ static int scmi_vio_feed_vq_rx(struct scmi_vio_channel *vioch,
+
+ rc = virtqueue_add_inbuf(vioch->vqueue, &sg_in, 1, msg, GFP_ATOMIC);
+ if (rc)
+- dev_err_once(vioch->cinfo->dev,
+- "failed to add to virtqueue (%d)\n", rc);
++ dev_err_once(dev, "failed to add to virtqueue (%d)\n", rc);
+ else
+ virtqueue_kick(vioch->vqueue);
+
+@@ -108,7 +108,7 @@ static void scmi_finalize_message(struct scmi_vio_channel *vioch,
+ struct scmi_vio_msg *msg)
+ {
+ if (vioch->is_rx) {
+- scmi_vio_feed_vq_rx(vioch, msg);
++ scmi_vio_feed_vq_rx(vioch, msg, vioch->cinfo->dev);
+ } else {
+ /* Here IRQs are assumed to be already disabled by the caller */
+ spin_lock(&vioch->lock);
+@@ -269,7 +269,7 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
+ list_add_tail(&msg->list, &vioch->free_list);
+ spin_unlock_irqrestore(&vioch->lock, flags);
+ } else {
+- scmi_vio_feed_vq_rx(vioch, msg);
++ scmi_vio_feed_vq_rx(vioch, msg, cinfo->dev);
+ }
+ }
+
+diff --git a/drivers/firmware/arm_scmi/voltage.c b/drivers/firmware/arm_scmi/voltage.c
+index a5048956a0be9..ac08e819088bb 100644
+--- a/drivers/firmware/arm_scmi/voltage.c
++++ b/drivers/firmware/arm_scmi/voltage.c
+@@ -156,7 +156,7 @@ static int scmi_voltage_descriptors_get(const struct scmi_protocol_handle *ph,
+ int cnt;
+
+ cmd->domain_id = cpu_to_le32(v->id);
+- cmd->level_index = desc_index;
++ cmd->level_index = cpu_to_le32(desc_index);
+ ret = ph->xops->do_xfer(ph, tl);
+ if (ret)
+ break;
+diff --git a/drivers/firmware/efi/apple-properties.c b/drivers/firmware/efi/apple-properties.c
+index 4c3201e290e29..ea84108035eb0 100644
+--- a/drivers/firmware/efi/apple-properties.c
++++ b/drivers/firmware/efi/apple-properties.c
+@@ -24,7 +24,7 @@ static bool dump_properties __initdata;
+ static int __init dump_properties_enable(char *arg)
+ {
+ dump_properties = true;
+- return 0;
++ return 1;
+ }
+
+ __setup("dump_apple_properties", dump_properties_enable);
+diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c
+index b19ce1a83f91a..b2c829e95bd14 100644
+--- a/drivers/firmware/efi/efi-init.c
++++ b/drivers/firmware/efi/efi-init.c
+@@ -235,6 +235,11 @@ void __init efi_init(void)
+ }
+
+ reserve_regions();
++ /*
++ * For memblock manipulation, the cap should come after the memblock_add().
++ * And now, memblock is fully populated, it is time to do capping.
++ */
++ early_init_dt_check_for_usable_mem_range();
+ efi_esrt_init();
+ efi_mokvar_table_init();
+
+diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c
+index 0ef086e43090b..7e771c56c13c6 100644
+--- a/drivers/firmware/efi/efi-pstore.c
++++ b/drivers/firmware/efi/efi-pstore.c
+@@ -266,7 +266,7 @@ static int efi_pstore_write(struct pstore_record *record)
+ efi_name[i] = name[i];
+
+ ret = efivar_entry_set_safe(efi_name, vendor, PSTORE_EFI_ATTRIBUTES,
+- preemptible(), record->size, record->psi->buf);
++ false, record->size, record->psi->buf);
+
+ if (record->reason == KMSG_DUMP_OOPS && try_module_get(THIS_MODULE))
+ if (!schedule_work(&efivar_work))
+diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
+index 847f33ffc4aed..e3df82d5d37a8 100644
+--- a/drivers/firmware/efi/efi.c
++++ b/drivers/firmware/efi/efi.c
+@@ -209,7 +209,7 @@ static int __init efivar_ssdt_setup(char *str)
+ memcpy(efivar_ssdt, str, strlen(str));
+ else
+ pr_warn("efivar_ssdt: name too long: %s\n", str);
+- return 0;
++ return 1;
+ }
+ __setup("efivar_ssdt=", efivar_ssdt_setup);
+
+@@ -719,6 +719,13 @@ void __init efi_systab_report_header(const efi_table_hdr_t *systab_hdr,
+ systab_hdr->revision >> 16,
+ systab_hdr->revision & 0xffff,
+ vendor);
++
++ if (IS_ENABLED(CONFIG_X86_64) &&
++ systab_hdr->revision > EFI_1_10_SYSTEM_TABLE_REVISION &&
++ !strcmp(vendor, "Apple")) {
++ pr_info("Apple Mac detected, using EFI v1.10 runtime services only\n");
++ efi.runtime_version = EFI_1_10_SYSTEM_TABLE_REVISION;
++ }
+ }
+
+ static __initdata char memory_type_name[][13] = {
+diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c
+index 2363fee9211c9..9cc556013d085 100644
+--- a/drivers/firmware/efi/libstub/arm64-stub.c
++++ b/drivers/firmware/efi/libstub/arm64-stub.c
+@@ -119,9 +119,9 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
+ if (image->image_base != _text)
+ efi_err("FIRMWARE BUG: efi_loaded_image_t::image_base has bogus value\n");
+
+- if (!IS_ALIGNED((u64)_text, EFI_KIMG_ALIGN))
+- efi_err("FIRMWARE BUG: kernel image not aligned on %ldk boundary\n",
+- EFI_KIMG_ALIGN >> 10);
++ if (!IS_ALIGNED((u64)_text, SEGMENT_ALIGN))
++ efi_err("FIRMWARE BUG: kernel image not aligned on %dk boundary\n",
++ SEGMENT_ALIGN >> 10);
+
+ kernel_size = _edata - _text;
+ kernel_memsize = kernel_size + (_end - _edata);
+diff --git a/drivers/firmware/efi/libstub/riscv-stub.c b/drivers/firmware/efi/libstub/riscv-stub.c
+index 380e4e2513994..9c460843442f5 100644
+--- a/drivers/firmware/efi/libstub/riscv-stub.c
++++ b/drivers/firmware/efi/libstub/riscv-stub.c
+@@ -25,7 +25,7 @@ typedef void __noreturn (*jump_kernel_func)(unsigned int, unsigned long);
+
+ static u32 hartid;
+
+-static u32 get_boot_hartid_from_fdt(void)
++static int get_boot_hartid_from_fdt(void)
+ {
+ const void *fdt;
+ int chosen_node, len;
+@@ -33,23 +33,26 @@ static u32 get_boot_hartid_from_fdt(void)
+
+ fdt = get_efi_config_table(DEVICE_TREE_GUID);
+ if (!fdt)
+- return U32_MAX;
++ return -EINVAL;
+
+ chosen_node = fdt_path_offset(fdt, "/chosen");
+ if (chosen_node < 0)
+- return U32_MAX;
++ return -EINVAL;
+
+ prop = fdt_getprop((void *)fdt, chosen_node, "boot-hartid", &len);
+ if (!prop || len != sizeof(u32))
+- return U32_MAX;
++ return -EINVAL;
+
+- return fdt32_to_cpu(*prop);
++ hartid = fdt32_to_cpu(*prop);
++ return 0;
+ }
+
+ efi_status_t check_platform_features(void)
+ {
+- hartid = get_boot_hartid_from_fdt();
+- if (hartid == U32_MAX) {
++ int ret;
++
++ ret = get_boot_hartid_from_fdt();
++ if (ret) {
+ efi_err("/chosen/boot-hartid missing or invalid!\n");
+ return EFI_UNSUPPORTED;
+ }
+diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c
+index abdc8a6a39631..cae590bd08f27 100644
+--- a/drivers/firmware/efi/vars.c
++++ b/drivers/firmware/efi/vars.c
+@@ -742,6 +742,7 @@ int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes,
+ {
+ const struct efivar_operations *ops;
+ efi_status_t status;
++ unsigned long varsize;
+
+ if (!__efivars)
+ return -EINVAL;
+@@ -764,15 +765,17 @@ int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes,
+ return efivar_entry_set_nonblocking(name, vendor, attributes,
+ size, data);
+
++ varsize = size + ucs2_strsize(name, 1024);
+ if (!block) {
+ if (down_trylock(&efivars_lock))
+ return -EBUSY;
++ status = check_var_size_nonblocking(attributes, varsize);
+ } else {
+ if (down_interruptible(&efivars_lock))
+ return -EINTR;
++ status = check_var_size(attributes, varsize);
+ }
+
+- status = check_var_size(attributes, size + ucs2_strsize(name, 1024));
+ if (status != EFI_SUCCESS) {
+ up(&efivars_lock);
+ return -ENOSPC;
+diff --git a/drivers/firmware/google/Kconfig b/drivers/firmware/google/Kconfig
+index 97968aece54f8..983e07dc022ed 100644
+--- a/drivers/firmware/google/Kconfig
++++ b/drivers/firmware/google/Kconfig
+@@ -3,9 +3,9 @@ menuconfig GOOGLE_FIRMWARE
+ bool "Google Firmware Drivers"
+ default n
+ help
+- These firmware drivers are used by Google's servers. They are
+- only useful if you are working directly on one of their
+- proprietary servers. If in doubt, say "N".
++ These firmware drivers are used by Google servers,
++ Chromebooks and other devices using coreboot firmware.
++ If in doubt, say "N".
+
+ if GOOGLE_FIRMWARE
+
+@@ -21,7 +21,7 @@ config GOOGLE_SMI
+
+ config GOOGLE_COREBOOT_TABLE
+ tristate "Coreboot Table Access"
+- depends on ACPI || OF
++ depends on HAS_IOMEM && (ACPI || OF)
+ help
+ This option enables the coreboot_table module, which provides other
+ firmware modules access to the coreboot table. The coreboot table
+diff --git a/drivers/firmware/psci/psci_checker.c b/drivers/firmware/psci/psci_checker.c
+index 9a369a2eda71d..116eb465cdb42 100644
+--- a/drivers/firmware/psci/psci_checker.c
++++ b/drivers/firmware/psci/psci_checker.c
+@@ -155,7 +155,7 @@ static int alloc_init_cpu_groups(cpumask_var_t **pcpu_groups)
+ if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
+ return -ENOMEM;
+
+- cpu_groups = kcalloc(nb_available_cpus, sizeof(cpu_groups),
++ cpu_groups = kcalloc(nb_available_cpus, sizeof(*cpu_groups),
+ GFP_KERNEL);
+ if (!cpu_groups) {
+ free_cpumask_var(tmp);
+diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
+index 2ee97bab74409..2b5214d5c0daf 100644
+--- a/drivers/firmware/qcom_scm.c
++++ b/drivers/firmware/qcom_scm.c
+@@ -252,7 +252,7 @@ static bool __qcom_scm_is_call_available(struct device *dev, u32 svc_id,
+ break;
+ default:
+ pr_err("Unknown SMC convention being used\n");
+- return -EINVAL;
++ return false;
+ }
+
+ ret = qcom_scm_call(dev, &desc, &res);
+@@ -749,12 +749,6 @@ int qcom_scm_iommu_secure_ptbl_init(u64 addr, u32 size, u32 spare)
+ };
+ int ret;
+
+- desc.args[0] = addr;
+- desc.args[1] = size;
+- desc.args[2] = spare;
+- desc.arginfo = QCOM_SCM_ARGS(3, QCOM_SCM_RW, QCOM_SCM_VAL,
+- QCOM_SCM_VAL);
+-
+ ret = qcom_scm_call(__scm->dev, &desc, NULL);
+
+ /* the pg table has been initialized already, ignore the error */
+diff --git a/drivers/firmware/qemu_fw_cfg.c b/drivers/firmware/qemu_fw_cfg.c
+index 172c751a4f6c2..f08e056ed0ae4 100644
+--- a/drivers/firmware/qemu_fw_cfg.c
++++ b/drivers/firmware/qemu_fw_cfg.c
+@@ -388,9 +388,7 @@ static void fw_cfg_sysfs_cache_cleanup(void)
+ struct fw_cfg_sysfs_entry *entry, *next;
+
+ list_for_each_entry_safe(entry, next, &fw_cfg_entry_cache, list) {
+- /* will end up invoking fw_cfg_sysfs_cache_delist()
+- * via each object's release() method (i.e. destructor)
+- */
++ fw_cfg_sysfs_cache_delist(entry);
+ kobject_put(&entry->kobj);
+ }
+ }
+@@ -448,7 +446,6 @@ static void fw_cfg_sysfs_release_entry(struct kobject *kobj)
+ {
+ struct fw_cfg_sysfs_entry *entry = to_entry(kobj);
+
+- fw_cfg_sysfs_cache_delist(entry);
+ kfree(entry);
+ }
+
+@@ -601,20 +598,18 @@ static int fw_cfg_register_file(const struct fw_cfg_file *f)
+ /* set file entry information */
+ entry->size = be32_to_cpu(f->size);
+ entry->select = be16_to_cpu(f->select);
+- memcpy(entry->name, f->name, FW_CFG_MAX_FILE_PATH);
++ strscpy(entry->name, f->name, FW_CFG_MAX_FILE_PATH);
+
+ /* register entry under "/sys/firmware/qemu_fw_cfg/by_key/" */
+ err = kobject_init_and_add(&entry->kobj, &fw_cfg_sysfs_entry_ktype,
+ fw_cfg_sel_ko, "%d", entry->select);
+- if (err) {
+- kobject_put(&entry->kobj);
+- return err;
+- }
++ if (err)
++ goto err_put_entry;
+
+ /* add raw binary content access */
+ err = sysfs_create_bin_file(&entry->kobj, &fw_cfg_sysfs_attr_raw);
+ if (err)
+- goto err_add_raw;
++ goto err_del_entry;
+
+ /* try adding "/sys/firmware/qemu_fw_cfg/by_name/" symlink */
+ fw_cfg_build_symlink(fw_cfg_fname_kset, &entry->kobj, entry->name);
+@@ -623,9 +618,10 @@ static int fw_cfg_register_file(const struct fw_cfg_file *f)
+ fw_cfg_sysfs_cache_enlist(entry);
+ return 0;
+
+-err_add_raw:
++err_del_entry:
+ kobject_del(&entry->kobj);
+- kfree(entry);
++err_put_entry:
++ kobject_put(&entry->kobj);
+ return err;
+ }
+
+diff --git a/drivers/firmware/scpi_pm_domain.c b/drivers/firmware/scpi_pm_domain.c
+index 51201600d789b..800673910b511 100644
+--- a/drivers/firmware/scpi_pm_domain.c
++++ b/drivers/firmware/scpi_pm_domain.c
+@@ -16,7 +16,6 @@ struct scpi_pm_domain {
+ struct generic_pm_domain genpd;
+ struct scpi_ops *ops;
+ u32 domain;
+- char name[30];
+ };
+
+ /*
+@@ -110,8 +109,13 @@ static int scpi_pm_domain_probe(struct platform_device *pdev)
+
+ scpi_pd->domain = i;
+ scpi_pd->ops = scpi_ops;
+- sprintf(scpi_pd->name, "%pOFn.%d", np, i);
+- scpi_pd->genpd.name = scpi_pd->name;
++ scpi_pd->genpd.name = devm_kasprintf(dev, GFP_KERNEL,
++ "%pOFn.%d", np, i);
++ if (!scpi_pd->genpd.name) {
++ dev_err(dev, "Failed to allocate genpd name:%pOFn.%d\n",
++ np, i);
++ continue;
++ }
+ scpi_pd->genpd.power_off = scpi_pd_power_off;
+ scpi_pd->genpd.power_on = scpi_pd_power_on;
+
+diff --git a/drivers/firmware/smccc/soc_id.c b/drivers/firmware/smccc/soc_id.c
+index 581aa5e9b0778..dd7c3d5e8b0bb 100644
+--- a/drivers/firmware/smccc/soc_id.c
++++ b/drivers/firmware/smccc/soc_id.c
+@@ -50,7 +50,7 @@ static int __init smccc_soc_init(void)
+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+ ARM_SMCCC_ARCH_SOC_ID, &res);
+
+- if (res.a0 == SMCCC_RET_NOT_SUPPORTED) {
++ if ((int)res.a0 == SMCCC_RET_NOT_SUPPORTED) {
+ pr_info("ARCH_SOC_ID not implemented, skipping ....\n");
+ return 0;
+ }
+diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
+index 2a7687911c097..53c7e3f8cfde2 100644
+--- a/drivers/firmware/stratix10-svc.c
++++ b/drivers/firmware/stratix10-svc.c
+@@ -477,7 +477,7 @@ static int svc_normal_to_secure_thread(void *data)
+ case INTEL_SIP_SMC_RSU_ERROR:
+ pr_err("%s: STATUS_ERROR\n", __func__);
+ cbdata->status = BIT(SVC_STATUS_ERROR);
+- cbdata->kaddr1 = NULL;
++ cbdata->kaddr1 = &res.a1;
+ cbdata->kaddr2 = NULL;
+ cbdata->kaddr3 = NULL;
+ pdata->chan->scl->receive_cb(pdata->chan->scl, cbdata);
+diff --git a/drivers/firmware/sysfb_simplefb.c b/drivers/firmware/sysfb_simplefb.c
+index b86761904949c..757cc8b9f3de9 100644
+--- a/drivers/firmware/sysfb_simplefb.c
++++ b/drivers/firmware/sysfb_simplefb.c
+@@ -114,11 +114,20 @@ __init int sysfb_create_simplefb(const struct screen_info *si,
+
+ ret = platform_device_add_resources(pd, &res, 1);
+ if (ret)
+- return ret;
++ goto err_put_device;
+
+ ret = platform_device_add_data(pd, mode, sizeof(*mode));
+ if (ret)
+- return ret;
++ goto err_put_device;
+
+- return platform_device_add(pd);
++ ret = platform_device_add(pd);
++ if (ret)
++ goto err_put_device;
++
++ return 0;
++
++err_put_device:
++ platform_device_put(pd);
++
++ return ret;
+ }
+diff --git a/drivers/fsi/fsi-master-aspeed.c b/drivers/fsi/fsi-master-aspeed.c
+index 8606e55c1721c..0bed2fab80558 100644
+--- a/drivers/fsi/fsi-master-aspeed.c
++++ b/drivers/fsi/fsi-master-aspeed.c
+@@ -542,25 +542,28 @@ static int fsi_master_aspeed_probe(struct platform_device *pdev)
+ return rc;
+ }
+
+- aspeed = devm_kzalloc(&pdev->dev, sizeof(*aspeed), GFP_KERNEL);
++ aspeed = kzalloc(sizeof(*aspeed), GFP_KERNEL);
+ if (!aspeed)
+ return -ENOMEM;
+
+ aspeed->dev = &pdev->dev;
+
+ aspeed->base = devm_platform_ioremap_resource(pdev, 0);
+- if (IS_ERR(aspeed->base))
+- return PTR_ERR(aspeed->base);
++ if (IS_ERR(aspeed->base)) {
++ rc = PTR_ERR(aspeed->base);
++ goto err_free_aspeed;
++ }
+
+ aspeed->clk = devm_clk_get(aspeed->dev, NULL);
+ if (IS_ERR(aspeed->clk)) {
+ dev_err(aspeed->dev, "couldn't get clock\n");
+- return PTR_ERR(aspeed->clk);
++ rc = PTR_ERR(aspeed->clk);
++ goto err_free_aspeed;
+ }
+ rc = clk_prepare_enable(aspeed->clk);
+ if (rc) {
+ dev_err(aspeed->dev, "couldn't enable clock\n");
+- return rc;
++ goto err_free_aspeed;
+ }
+
+ rc = setup_cfam_reset(aspeed);
+@@ -595,7 +598,7 @@ static int fsi_master_aspeed_probe(struct platform_device *pdev)
+ rc = opb_readl(aspeed, ctrl_base + FSI_MVER, &raw);
+ if (rc) {
+ dev_err(&pdev->dev, "failed to read hub version\n");
+- return rc;
++ goto err_release;
+ }
+
+ reg = be32_to_cpu(raw);
+@@ -634,6 +637,8 @@ static int fsi_master_aspeed_probe(struct platform_device *pdev)
+
+ err_release:
+ clk_disable_unprepare(aspeed->clk);
++err_free_aspeed:
++ kfree(aspeed);
+ return rc;
+ }
+
+diff --git a/drivers/fsi/fsi-scom.c b/drivers/fsi/fsi-scom.c
+index da1486bb6a144..bcb756dc98663 100644
+--- a/drivers/fsi/fsi-scom.c
++++ b/drivers/fsi/fsi-scom.c
+@@ -145,7 +145,7 @@ static int put_indirect_scom_form0(struct scom_device *scom, uint64_t value,
+ uint64_t addr, uint32_t *status)
+ {
+ uint64_t ind_data, ind_addr;
+- int rc, retries, err = 0;
++ int rc, err;
+
+ if (value & ~XSCOM_DATA_IND_DATA)
+ return -EINVAL;
+@@ -156,19 +156,14 @@ static int put_indirect_scom_form0(struct scom_device *scom, uint64_t value,
+ if (rc || (*status & SCOM_STATUS_ANY_ERR))
+ return rc;
+
+- for (retries = 0; retries < SCOM_MAX_IND_RETRIES; retries++) {
+- rc = __get_scom(scom, &ind_data, addr, status);
+- if (rc || (*status & SCOM_STATUS_ANY_ERR))
+- return rc;
++ rc = __get_scom(scom, &ind_data, addr, status);
++ if (rc || (*status & SCOM_STATUS_ANY_ERR))
++ return rc;
+
+- err = (ind_data & XSCOM_DATA_IND_ERR_MASK) >> XSCOM_DATA_IND_ERR_SHIFT;
+- *status = err << SCOM_STATUS_PIB_RESP_SHIFT;
+- if ((ind_data & XSCOM_DATA_IND_COMPLETE) || (err != SCOM_PIB_BLOCKED))
+- return 0;
++ err = (ind_data & XSCOM_DATA_IND_ERR_MASK) >> XSCOM_DATA_IND_ERR_SHIFT;
++ *status = err << SCOM_STATUS_PIB_RESP_SHIFT;
+
+- msleep(1);
+- }
+- return rc;
++ return 0;
+ }
+
+ static int put_indirect_scom_form1(struct scom_device *scom, uint64_t value,
+@@ -188,7 +183,7 @@ static int get_indirect_scom_form0(struct scom_device *scom, uint64_t *value,
+ uint64_t addr, uint32_t *status)
+ {
+ uint64_t ind_data, ind_addr;
+- int rc, retries, err = 0;
++ int rc, err;
+
+ ind_addr = addr & XSCOM_ADDR_DIRECT_PART;
+ ind_data = (addr & XSCOM_ADDR_INDIRECT_PART) | XSCOM_DATA_IND_READ;
+@@ -196,21 +191,15 @@ static int get_indirect_scom_form0(struct scom_device *scom, uint64_t *value,
+ if (rc || (*status & SCOM_STATUS_ANY_ERR))
+ return rc;
+
+- for (retries = 0; retries < SCOM_MAX_IND_RETRIES; retries++) {
+- rc = __get_scom(scom, &ind_data, addr, status);
+- if (rc || (*status & SCOM_STATUS_ANY_ERR))
+- return rc;
+-
+- err = (ind_data & XSCOM_DATA_IND_ERR_MASK) >> XSCOM_DATA_IND_ERR_SHIFT;
+- *status = err << SCOM_STATUS_PIB_RESP_SHIFT;
+- *value = ind_data & XSCOM_DATA_IND_DATA;
++ rc = __get_scom(scom, &ind_data, addr, status);
++ if (rc || (*status & SCOM_STATUS_ANY_ERR))
++ return rc;
+
+- if ((ind_data & XSCOM_DATA_IND_COMPLETE) || (err != SCOM_PIB_BLOCKED))
+- return 0;
++ err = (ind_data & XSCOM_DATA_IND_ERR_MASK) >> XSCOM_DATA_IND_ERR_SHIFT;
++ *status = err << SCOM_STATUS_PIB_RESP_SHIFT;
++ *value = ind_data & XSCOM_DATA_IND_DATA;
+
+- msleep(1);
+- }
+- return rc;
++ return 0;
+ }
+
+ static int raw_put_scom(struct scom_device *scom, uint64_t value,
+@@ -289,7 +278,7 @@ static int put_scom(struct scom_device *scom, uint64_t value,
+ int rc;
+
+ rc = raw_put_scom(scom, value, addr, &status);
+- if (rc == -ENODEV)
++ if (rc)
+ return rc;
+
+ rc = handle_fsi2pib_status(scom, status);
+@@ -308,7 +297,7 @@ static int get_scom(struct scom_device *scom, uint64_t *value,
+ int rc;
+
+ rc = raw_get_scom(scom, value, addr, &status);
+- if (rc == -ENODEV)
++ if (rc)
+ return rc;
+
+ rc = handle_fsi2pib_status(scom, status);
+diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
+index fae5141251e5d..947474f6abb45 100644
+--- a/drivers/gpio/Kconfig
++++ b/drivers/gpio/Kconfig
+@@ -523,6 +523,7 @@ config GPIO_REG
+ config GPIO_ROCKCHIP
+ tristate "Rockchip GPIO support"
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
++ select GENERIC_IRQ_CHIP
+ select GPIOLIB_IRQCHIP
+ default ARCH_ROCKCHIP
+ help
+diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c
+index 34e35b64dcdc0..23047dc84ef1b 100644
+--- a/drivers/gpio/gpio-aggregator.c
++++ b/drivers/gpio/gpio-aggregator.c
+@@ -273,7 +273,8 @@ static int gpio_fwd_get(struct gpio_chip *chip, unsigned int offset)
+ {
+ struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+- return gpiod_get_value(fwd->descs[offset]);
++ return chip->can_sleep ? gpiod_get_value_cansleep(fwd->descs[offset])
++ : gpiod_get_value(fwd->descs[offset]);
+ }
+
+ static int gpio_fwd_get_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
+@@ -292,7 +293,10 @@ static int gpio_fwd_get_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
+ for_each_set_bit(i, mask, fwd->chip.ngpio)
+ descs[j++] = fwd->descs[i];
+
+- error = gpiod_get_array_value(j, descs, NULL, values);
++ if (fwd->chip.can_sleep)
++ error = gpiod_get_array_value_cansleep(j, descs, NULL, values);
++ else
++ error = gpiod_get_array_value(j, descs, NULL, values);
+ if (error)
+ return error;
+
+@@ -327,7 +331,10 @@ static void gpio_fwd_set(struct gpio_chip *chip, unsigned int offset, int value)
+ {
+ struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+- gpiod_set_value(fwd->descs[offset], value);
++ if (chip->can_sleep)
++ gpiod_set_value_cansleep(fwd->descs[offset], value);
++ else
++ gpiod_set_value(fwd->descs[offset], value);
+ }
+
+ static void gpio_fwd_set_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
+@@ -346,7 +353,10 @@ static void gpio_fwd_set_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
+ descs[j++] = fwd->descs[i];
+ }
+
+- gpiod_set_array_value(j, descs, NULL, values);
++ if (fwd->chip.can_sleep)
++ gpiod_set_array_value_cansleep(j, descs, NULL, values);
++ else
++ gpiod_set_array_value(j, descs, NULL, values);
+ }
+
+ static void gpio_fwd_set_multiple_locked(struct gpio_chip *chip,
+diff --git a/drivers/gpio/gpio-aspeed-sgpio.c b/drivers/gpio/gpio-aspeed-sgpio.c
+index 3d6ef37a7702a..454cefbeecf0e 100644
+--- a/drivers/gpio/gpio-aspeed-sgpio.c
++++ b/drivers/gpio/gpio-aspeed-sgpio.c
+@@ -31,7 +31,7 @@ struct aspeed_sgpio {
+ struct gpio_chip chip;
+ struct irq_chip intc;
+ struct clk *pclk;
+- spinlock_t lock;
++ raw_spinlock_t lock;
+ void __iomem *base;
+ int irq;
+ };
+@@ -173,12 +173,12 @@ static int aspeed_sgpio_get(struct gpio_chip *gc, unsigned int offset)
+ enum aspeed_sgpio_reg reg;
+ int rc = 0;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ reg = aspeed_sgpio_is_input(offset) ? reg_val : reg_rdata;
+ rc = !!(ioread32(bank_reg(gpio, bank, reg)) & GPIO_BIT(offset));
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return rc;
+ }
+@@ -215,11 +215,11 @@ static void aspeed_sgpio_set(struct gpio_chip *gc, unsigned int offset, int val)
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ unsigned long flags;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ sgpio_set_value(gc, offset, val);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static int aspeed_sgpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+@@ -236,9 +236,9 @@ static int aspeed_sgpio_dir_out(struct gpio_chip *gc, unsigned int offset, int v
+ /* No special action is required for setting the direction; we'll
+ * error-out in sgpio_set_value if this isn't an output GPIO */
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ rc = sgpio_set_value(gc, offset, val);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return rc;
+ }
+@@ -277,11 +277,11 @@ static void aspeed_sgpio_irq_ack(struct irq_data *d)
+
+ status_addr = bank_reg(gpio, bank, reg_irq_status);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ iowrite32(bit, status_addr);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
+@@ -296,7 +296,7 @@ static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
+ irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
+ addr = bank_reg(gpio, bank, reg_irq_enable);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ reg = ioread32(addr);
+ if (set)
+@@ -306,7 +306,7 @@ static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
+
+ iowrite32(reg, addr);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static void aspeed_sgpio_irq_mask(struct irq_data *d)
+@@ -355,7 +355,7 @@ static int aspeed_sgpio_set_type(struct irq_data *d, unsigned int type)
+ return -EINVAL;
+ }
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ addr = bank_reg(gpio, bank, reg_irq_type0);
+ reg = ioread32(addr);
+@@ -372,7 +372,7 @@ static int aspeed_sgpio_set_type(struct irq_data *d, unsigned int type)
+ reg = (reg & ~bit) | type2;
+ iowrite32(reg, addr);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ irq_set_handler_locked(d, handler);
+
+@@ -395,7 +395,7 @@ static void aspeed_sgpio_irq_handler(struct irq_desc *desc)
+ reg = ioread32(bank_reg(data, bank, reg_irq_status));
+
+ for_each_set_bit(p, &reg, 32)
+- generic_handle_domain_irq(gc->irq.domain, i * 32 + p * 2);
++ generic_handle_domain_irq(gc->irq.domain, (i * 32 + p) * 2);
+ }
+
+ chained_irq_exit(ic, desc);
+@@ -467,7 +467,7 @@ static int aspeed_sgpio_reset_tolerance(struct gpio_chip *chip,
+
+ reg = bank_reg(gpio, to_bank(offset), reg_tolerance);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ val = readl(reg);
+
+@@ -478,7 +478,7 @@ static int aspeed_sgpio_reset_tolerance(struct gpio_chip *chip,
+
+ writel(val, reg);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+ }
+@@ -575,7 +575,7 @@ static int __init aspeed_sgpio_probe(struct platform_device *pdev)
+ iowrite32(FIELD_PREP(ASPEED_SGPIO_CLK_DIV_MASK, sgpio_clk_div) | gpio_cnt_regval |
+ ASPEED_SGPIO_ENABLE, gpio->base + ASPEED_SGPIO_CTRL);
+
+- spin_lock_init(&gpio->lock);
++ raw_spin_lock_init(&gpio->lock);
+
+ gpio->chip.parent = &pdev->dev;
+ gpio->chip.ngpio = nr_gpios * 2;
+diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
+index 3c8f20c57695f..318a7d95a1a8b 100644
+--- a/drivers/gpio/gpio-aspeed.c
++++ b/drivers/gpio/gpio-aspeed.c
+@@ -53,7 +53,7 @@ struct aspeed_gpio_config {
+ struct aspeed_gpio {
+ struct gpio_chip chip;
+ struct irq_chip irqc;
+- spinlock_t lock;
++ raw_spinlock_t lock;
+ void __iomem *base;
+ int irq;
+ const struct aspeed_gpio_config *config;
+@@ -413,14 +413,14 @@ static void aspeed_gpio_set(struct gpio_chip *gc, unsigned int offset,
+ unsigned long flags;
+ bool copro;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ copro = aspeed_gpio_copro_request(gpio, offset);
+
+ __aspeed_gpio_set(gc, offset, val);
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static int aspeed_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+@@ -435,7 +435,7 @@ static int aspeed_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+ if (!have_input(gpio, offset))
+ return -ENOTSUPP;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ reg = ioread32(addr);
+ reg &= ~GPIO_BIT(offset);
+@@ -445,7 +445,7 @@ static int aspeed_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+ }
+@@ -463,7 +463,7 @@ static int aspeed_gpio_dir_out(struct gpio_chip *gc,
+ if (!have_output(gpio, offset))
+ return -ENOTSUPP;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ reg = ioread32(addr);
+ reg |= GPIO_BIT(offset);
+@@ -474,7 +474,7 @@ static int aspeed_gpio_dir_out(struct gpio_chip *gc,
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+ }
+@@ -492,11 +492,11 @@ static int aspeed_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+ if (!have_output(gpio, offset))
+ return GPIO_LINE_DIRECTION_IN;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ val = ioread32(bank_reg(gpio, bank, reg_dir)) & GPIO_BIT(offset);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return val ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+ }
+@@ -539,14 +539,14 @@ static void aspeed_gpio_irq_ack(struct irq_data *d)
+
+ status_addr = bank_reg(gpio, bank, reg_irq_status);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ copro = aspeed_gpio_copro_request(gpio, offset);
+
+ iowrite32(bit, status_addr);
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
+@@ -565,7 +565,7 @@ static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
+
+ addr = bank_reg(gpio, bank, reg_irq_enable);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ copro = aspeed_gpio_copro_request(gpio, offset);
+
+ reg = ioread32(addr);
+@@ -577,7 +577,7 @@ static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ }
+
+ static void aspeed_gpio_irq_mask(struct irq_data *d)
+@@ -629,7 +629,7 @@ static int aspeed_gpio_set_type(struct irq_data *d, unsigned int type)
+ return -EINVAL;
+ }
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ copro = aspeed_gpio_copro_request(gpio, offset);
+
+ addr = bank_reg(gpio, bank, reg_irq_type0);
+@@ -649,7 +649,7 @@ static int aspeed_gpio_set_type(struct irq_data *d, unsigned int type)
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ irq_set_handler_locked(d, handler);
+
+@@ -716,7 +716,7 @@ static int aspeed_gpio_reset_tolerance(struct gpio_chip *chip,
+
+ treg = bank_reg(gpio, to_bank(offset), reg_tolerance);
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+ copro = aspeed_gpio_copro_request(gpio, offset);
+
+ val = readl(treg);
+@@ -730,7 +730,7 @@ static int aspeed_gpio_reset_tolerance(struct gpio_chip *chip,
+
+ if (copro)
+ aspeed_gpio_copro_release(gpio, offset);
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+ }
+@@ -856,7 +856,7 @@ static int enable_debounce(struct gpio_chip *chip, unsigned int offset,
+ return rc;
+ }
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ if (timer_allocation_registered(gpio, offset)) {
+ rc = unregister_allocated_timer(gpio, offset);
+@@ -916,7 +916,7 @@ static int enable_debounce(struct gpio_chip *chip, unsigned int offset,
+ configure_timer(gpio, offset, i);
+
+ out:
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return rc;
+ }
+@@ -927,13 +927,13 @@ static int disable_debounce(struct gpio_chip *chip, unsigned int offset)
+ unsigned long flags;
+ int rc;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ rc = unregister_allocated_timer(gpio, offset);
+ if (!rc)
+ configure_timer(gpio, offset, 0);
+
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return rc;
+ }
+@@ -1015,7 +1015,7 @@ int aspeed_gpio_copro_grab_gpio(struct gpio_desc *desc,
+ return -EINVAL;
+ bindex = offset >> 3;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ /* Sanity check, this shouldn't happen */
+ if (gpio->cf_copro_bankmap[bindex] == 0xff) {
+@@ -1036,7 +1036,7 @@ int aspeed_gpio_copro_grab_gpio(struct gpio_desc *desc,
+ if (bit)
+ *bit = GPIO_OFFSET(offset);
+ bail:
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ return rc;
+ }
+ EXPORT_SYMBOL_GPL(aspeed_gpio_copro_grab_gpio);
+@@ -1060,7 +1060,7 @@ int aspeed_gpio_copro_release_gpio(struct gpio_desc *desc)
+ return -EINVAL;
+ bindex = offset >> 3;
+
+- spin_lock_irqsave(&gpio->lock, flags);
++ raw_spin_lock_irqsave(&gpio->lock, flags);
+
+ /* Sanity check, this shouldn't happen */
+ if (gpio->cf_copro_bankmap[bindex] == 0) {
+@@ -1074,7 +1074,7 @@ int aspeed_gpio_copro_release_gpio(struct gpio_desc *desc)
+ aspeed_gpio_change_cmd_source(gpio, bank, bindex,
+ GPIO_CMDSRC_ARM);
+ bail:
+- spin_unlock_irqrestore(&gpio->lock, flags);
++ raw_spin_unlock_irqrestore(&gpio->lock, flags);
+ return rc;
+ }
+ EXPORT_SYMBOL_GPL(aspeed_gpio_copro_release_gpio);
+@@ -1148,7 +1148,7 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
+ if (IS_ERR(gpio->base))
+ return PTR_ERR(gpio->base);
+
+- spin_lock_init(&gpio->lock);
++ raw_spin_lock_init(&gpio->lock);
+
+ gpio_id = of_match_node(aspeed_gpio_of_table, pdev->dev.of_node);
+ if (!gpio_id)
+diff --git a/drivers/gpio/gpio-dln2.c b/drivers/gpio/gpio-dln2.c
+index 026903e3ef543..08b9e2cf4f2d6 100644
+--- a/drivers/gpio/gpio-dln2.c
++++ b/drivers/gpio/gpio-dln2.c
+@@ -46,6 +46,7 @@
+ struct dln2_gpio {
+ struct platform_device *pdev;
+ struct gpio_chip gpio;
++ struct irq_chip irqchip;
+
+ /*
+ * Cache pin direction to save us one transfer, since the hardware has
+@@ -383,15 +384,6 @@ static void dln2_irq_bus_unlock(struct irq_data *irqd)
+ mutex_unlock(&dln2->irq_lock);
+ }
+
+-static struct irq_chip dln2_gpio_irqchip = {
+- .name = "dln2-irq",
+- .irq_mask = dln2_irq_mask,
+- .irq_unmask = dln2_irq_unmask,
+- .irq_set_type = dln2_irq_set_type,
+- .irq_bus_lock = dln2_irq_bus_lock,
+- .irq_bus_sync_unlock = dln2_irq_bus_unlock,
+-};
+-
+ static void dln2_gpio_event(struct platform_device *pdev, u16 echo,
+ const void *data, int len)
+ {
+@@ -473,8 +465,15 @@ static int dln2_gpio_probe(struct platform_device *pdev)
+ dln2->gpio.direction_output = dln2_gpio_direction_output;
+ dln2->gpio.set_config = dln2_gpio_set_config;
+
++ dln2->irqchip.name = "dln2-irq",
++ dln2->irqchip.irq_mask = dln2_irq_mask,
++ dln2->irqchip.irq_unmask = dln2_irq_unmask,
++ dln2->irqchip.irq_set_type = dln2_irq_set_type,
++ dln2->irqchip.irq_bus_lock = dln2_irq_bus_lock,
++ dln2->irqchip.irq_bus_sync_unlock = dln2_irq_bus_unlock,
++
+ girq = &dln2->gpio.irq;
+- girq->chip = &dln2_gpio_irqchip;
++ girq->chip = &dln2->irqchip;
+ /* The event comes from the outside so no parent handler */
+ girq->parent_handler = NULL;
+ girq->num_parents = 0;
+diff --git a/drivers/gpio/gpio-idt3243x.c b/drivers/gpio/gpio-idt3243x.c
+index 50003ad2e5898..52b8b72ded77f 100644
+--- a/drivers/gpio/gpio-idt3243x.c
++++ b/drivers/gpio/gpio-idt3243x.c
+@@ -132,7 +132,7 @@ static int idt_gpio_probe(struct platform_device *pdev)
+ struct device *dev = &pdev->dev;
+ struct gpio_irq_chip *girq;
+ struct idt_gpio_ctrl *ctrl;
+- unsigned int parent_irq;
++ int parent_irq;
+ int ngpios;
+ int ret;
+
+@@ -164,8 +164,8 @@ static int idt_gpio_probe(struct platform_device *pdev)
+ return PTR_ERR(ctrl->pic);
+
+ parent_irq = platform_get_irq(pdev, 0);
+- if (!parent_irq)
+- return -EINVAL;
++ if (parent_irq < 0)
++ return parent_irq;
+
+ girq = &ctrl->gc.irq;
+ girq->chip = &idt_gpio_irqchip;
+diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c
+index 70d6ae20b1da5..a964e25ea6206 100644
+--- a/drivers/gpio/gpio-mpc8xxx.c
++++ b/drivers/gpio/gpio-mpc8xxx.c
+@@ -47,7 +47,7 @@ struct mpc8xxx_gpio_chip {
+ unsigned offset, int value);
+
+ struct irq_domain *irq;
+- unsigned int irqn;
++ int irqn;
+ };
+
+ /*
+@@ -388,8 +388,8 @@ static int mpc8xxx_probe(struct platform_device *pdev)
+ }
+
+ mpc8xxx_gc->irqn = platform_get_irq(pdev, 0);
+- if (!mpc8xxx_gc->irqn)
+- return 0;
++ if (mpc8xxx_gc->irqn < 0)
++ return mpc8xxx_gc->irqn;
+
+ mpc8xxx_gc->irq = irq_domain_create_linear(fwnode,
+ MPC8XXX_GPIO_PINS,
+diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
+index 8f429d9f36616..1448dc874dfc3 100644
+--- a/drivers/gpio/gpio-mvebu.c
++++ b/drivers/gpio/gpio-mvebu.c
+@@ -707,6 +707,9 @@ static int mvebu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+ unsigned long flags;
+ unsigned int on, off;
+
++ if (state->polarity != PWM_POLARITY_NORMAL)
++ return -EINVAL;
++
+ val = (unsigned long long) mvpwm->clk_rate * state->duty_cycle;
+ do_div(val, NSEC_PER_SEC);
+ if (val > UINT_MAX + 1ULL)
+@@ -871,13 +874,6 @@ static int mvebu_pwm_probe(struct platform_device *pdev,
+ mvpwm->chip.dev = dev;
+ mvpwm->chip.ops = &mvebu_pwm_ops;
+ mvpwm->chip.npwm = mvchip->chip.ngpio;
+- /*
+- * There may already be some PWM allocated, so we can't force
+- * mvpwm->chip.base to a fixed point like mvchip->chip.base.
+- * So, we let pwmchip_add() do the numbering and take the next free
+- * region.
+- */
+- mvpwm->chip.base = -1;
+
+ spin_lock_init(&mvpwm->lock);
+
+diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
+index d2fe76f3f34fd..8726921a11294 100644
+--- a/drivers/gpio/gpio-pca953x.c
++++ b/drivers/gpio/gpio-pca953x.c
+@@ -762,11 +762,11 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pendin
+ bitmap_xor(cur_stat, new_stat, old_stat, gc->ngpio);
+ bitmap_and(trigger, cur_stat, chip->irq_mask, gc->ngpio);
+
++ bitmap_copy(chip->irq_stat, new_stat, gc->ngpio);
++
+ if (bitmap_empty(trigger, gc->ngpio))
+ return false;
+
+- bitmap_copy(chip->irq_stat, new_stat, gc->ngpio);
+-
+ bitmap_and(cur_stat, chip->irq_trig_fall, old_stat, gc->ngpio);
+ bitmap_and(old_stat, chip->irq_trig_raise, new_stat, gc->ngpio);
+ bitmap_or(new_stat, old_stat, cur_stat, gc->ngpio);
+diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c
+index eeeb39bc171dc..bd75401b549d1 100644
+--- a/drivers/gpio/gpio-realtek-otto.c
++++ b/drivers/gpio/gpio-realtek-otto.c
+@@ -205,7 +205,7 @@ static void realtek_gpio_irq_handler(struct irq_desc *desc)
+ status = realtek_gpio_read_isr(ctrl, lines_done / 8);
+ port_pin_count = min(gc->ngpio - lines_done, 8U);
+ for_each_set_bit(offset, &status, port_pin_count)
+- generic_handle_domain_irq(gc->irq.domain, offset);
++ generic_handle_domain_irq(gc->irq.domain, offset + lines_done);
+ }
+
+ chained_irq_exit(irq_chip, desc);
+diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c
+index ce63cbd14d69a..24155c038f6d0 100644
+--- a/drivers/gpio/gpio-rockchip.c
++++ b/drivers/gpio/gpio-rockchip.c
+@@ -410,10 +410,8 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type)
+ level = rockchip_gpio_readl(bank, bank->gpio_regs->int_type);
+ polarity = rockchip_gpio_readl(bank, bank->gpio_regs->int_polarity);
+
+- switch (type) {
+- case IRQ_TYPE_EDGE_BOTH:
++ if (type == IRQ_TYPE_EDGE_BOTH) {
+ if (bank->gpio_type == GPIO_TYPE_V2) {
+- bank->toggle_edge_mode &= ~mask;
+ rockchip_gpio_writel_bit(bank, d->hwirq, 1,
+ bank->gpio_regs->int_bothedge);
+ goto out;
+@@ -431,30 +429,34 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type)
+ else
+ polarity |= mask;
+ }
+- break;
+- case IRQ_TYPE_EDGE_RISING:
+- bank->toggle_edge_mode &= ~mask;
+- level |= mask;
+- polarity |= mask;
+- break;
+- case IRQ_TYPE_EDGE_FALLING:
+- bank->toggle_edge_mode &= ~mask;
+- level |= mask;
+- polarity &= ~mask;
+- break;
+- case IRQ_TYPE_LEVEL_HIGH:
+- bank->toggle_edge_mode &= ~mask;
+- level &= ~mask;
+- polarity |= mask;
+- break;
+- case IRQ_TYPE_LEVEL_LOW:
+- bank->toggle_edge_mode &= ~mask;
+- level &= ~mask;
+- polarity &= ~mask;
+- break;
+- default:
+- ret = -EINVAL;
+- goto out;
++ } else {
++ if (bank->gpio_type == GPIO_TYPE_V2) {
++ rockchip_gpio_writel_bit(bank, d->hwirq, 0,
++ bank->gpio_regs->int_bothedge);
++ } else {
++ bank->toggle_edge_mode &= ~mask;
++ }
++ switch (type) {
++ case IRQ_TYPE_EDGE_RISING:
++ level |= mask;
++ polarity |= mask;
++ break;
++ case IRQ_TYPE_EDGE_FALLING:
++ level |= mask;
++ polarity &= ~mask;
++ break;
++ case IRQ_TYPE_LEVEL_HIGH:
++ level &= ~mask;
++ polarity |= mask;
++ break;
++ case IRQ_TYPE_LEVEL_LOW:
++ level &= ~mask;
++ polarity &= ~mask;
++ break;
++ default:
++ ret = -EINVAL;
++ goto out;
++ }
+ }
+
+ rockchip_gpio_writel(bank, level, bank->gpio_regs->int_type);
+diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c
+index 403f9e833d6a3..7d82388b4ab7c 100644
+--- a/drivers/gpio/gpio-sifive.c
++++ b/drivers/gpio/gpio-sifive.c
+@@ -223,7 +223,7 @@ static int sifive_gpio_probe(struct platform_device *pdev)
+ NULL,
+ chip->base + SIFIVE_GPIO_OUTPUT_EN,
+ chip->base + SIFIVE_GPIO_INPUT_EN,
+- 0);
++ BGPIOF_READ_OUTPUT_REG_SET);
+ if (ret) {
+ dev_err(dev, "unable to init generic GPIO\n");
+ return ret;
+diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c
+index c99858f40a27e..00762de3d4096 100644
+--- a/drivers/gpio/gpio-tegra186.c
++++ b/drivers/gpio/gpio-tegra186.c
+@@ -337,9 +337,12 @@ static int tegra186_gpio_of_xlate(struct gpio_chip *chip,
+ return offset + pin;
+ }
+
++#define to_tegra_gpio(x) container_of((x), struct tegra_gpio, gpio)
++
+ static void tegra186_irq_ack(struct irq_data *data)
+ {
+- struct tegra_gpio *gpio = irq_data_get_irq_chip_data(data);
++ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
++ struct tegra_gpio *gpio = to_tegra_gpio(gc);
+ void __iomem *base;
+
+ base = tegra186_gpio_get_base(gpio, data->hwirq);
+@@ -351,7 +354,8 @@ static void tegra186_irq_ack(struct irq_data *data)
+
+ static void tegra186_irq_mask(struct irq_data *data)
+ {
+- struct tegra_gpio *gpio = irq_data_get_irq_chip_data(data);
++ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
++ struct tegra_gpio *gpio = to_tegra_gpio(gc);
+ void __iomem *base;
+ u32 value;
+
+@@ -366,7 +370,8 @@ static void tegra186_irq_mask(struct irq_data *data)
+
+ static void tegra186_irq_unmask(struct irq_data *data)
+ {
+- struct tegra_gpio *gpio = irq_data_get_irq_chip_data(data);
++ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
++ struct tegra_gpio *gpio = to_tegra_gpio(gc);
+ void __iomem *base;
+ u32 value;
+
+@@ -381,7 +386,8 @@ static void tegra186_irq_unmask(struct irq_data *data)
+
+ static int tegra186_irq_set_type(struct irq_data *data, unsigned int type)
+ {
+- struct tegra_gpio *gpio = irq_data_get_irq_chip_data(data);
++ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
++ struct tegra_gpio *gpio = to_tegra_gpio(gc);
+ void __iomem *base;
+ u32 value;
+
+diff --git a/drivers/gpio/gpio-ts4900.c b/drivers/gpio/gpio-ts4900.c
+index d885032cf814d..d918d2df4de2c 100644
+--- a/drivers/gpio/gpio-ts4900.c
++++ b/drivers/gpio/gpio-ts4900.c
+@@ -1,7 +1,7 @@
+ /*
+ * Digital I/O driver for Technologic Systems I2C FPGA Core
+ *
+- * Copyright (C) 2015 Technologic Systems
++ * Copyright (C) 2015, 2018 Technologic Systems
+ * Copyright (C) 2016 Savoir-Faire Linux
+ *
+ * This program is free software; you can redistribute it and/or
+@@ -55,19 +55,33 @@ static int ts4900_gpio_direction_input(struct gpio_chip *chip,
+ {
+ struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+
+- /*
+- * This will clear the output enable bit, the other bits are
+- * dontcare when this is cleared
++ /* Only clear the OE bit here, requires a RMW. Prevents potential issue
++ * with OE and data getting to the physical pin at different times.
+ */
+- return regmap_write(priv->regmap, offset, 0);
++ return regmap_update_bits(priv->regmap, offset, TS4900_GPIO_OE, 0);
+ }
+
+ static int ts4900_gpio_direction_output(struct gpio_chip *chip,
+ unsigned int offset, int value)
+ {
+ struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
++ unsigned int reg;
+ int ret;
+
++ /* If changing from an input to an output, we need to first set the
++ * proper data bit to what is requested and then set OE bit. This
++ * prevents a glitch that can occur on the IO line
++ */
++ regmap_read(priv->regmap, offset, &reg);
++ if (!(reg & TS4900_GPIO_OE)) {
++ if (value)
++ reg = TS4900_GPIO_OUT;
++ else
++ reg &= ~TS4900_GPIO_OUT;
++
++ regmap_write(priv->regmap, offset, reg);
++ }
++
+ if (value)
+ ret = regmap_write(priv->regmap, offset, TS4900_GPIO_OE |
+ TS4900_GPIO_OUT);
+diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
+index e0f2b67558e74..47e191e11c696 100644
+--- a/drivers/gpio/gpio-vf610.c
++++ b/drivers/gpio/gpio-vf610.c
+@@ -125,9 +125,13 @@ static int vf610_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
+ {
+ struct vf610_gpio_port *port = gpiochip_get_data(chip);
+ unsigned long mask = BIT(gpio);
++ u32 val;
+
+- if (port->sdata && port->sdata->have_paddr)
+- vf610_gpio_writel(mask, port->gpio_base + GPIO_PDDR);
++ if (port->sdata && port->sdata->have_paddr) {
++ val = vf610_gpio_readl(port->gpio_base + GPIO_PDDR);
++ val |= mask;
++ vf610_gpio_writel(val, port->gpio_base + GPIO_PDDR);
++ }
+
+ vf610_gpio_set(chip, gpio, value);
+
+diff --git a/drivers/gpio/gpio-virtio.c b/drivers/gpio/gpio-virtio.c
+index d24f1c9264bc9..dd3b23c9580b1 100644
+--- a/drivers/gpio/gpio-virtio.c
++++ b/drivers/gpio/gpio-virtio.c
+@@ -81,11 +81,7 @@ static int _virtio_gpio_req(struct virtio_gpio *vgpio, u16 type, u16 gpio,
+ virtqueue_kick(vgpio->request_vq);
+ mutex_unlock(&vgpio->lock);
+
+- if (!wait_for_completion_timeout(&line->completion, HZ)) {
+- dev_err(dev, "GPIO operation timed out\n");
+- ret = -ETIMEDOUT;
+- goto out;
+- }
++ wait_for_completion(&line->completion);
+
+ if (unlikely(res->status != VIRTIO_GPIO_STATUS_OK)) {
+ dev_err(dev, "GPIO request failed: %d\n", gpio);
+diff --git a/drivers/gpio/gpio-visconti.c b/drivers/gpio/gpio-visconti.c
+index 47455810bdb91..e6534ea1eaa7a 100644
+--- a/drivers/gpio/gpio-visconti.c
++++ b/drivers/gpio/gpio-visconti.c
+@@ -130,7 +130,6 @@ static int visconti_gpio_probe(struct platform_device *pdev)
+ struct gpio_irq_chip *girq;
+ struct irq_domain *parent;
+ struct device_node *irq_parent;
+- struct fwnode_handle *fwnode;
+ int ret;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+@@ -150,14 +149,12 @@ static int visconti_gpio_probe(struct platform_device *pdev)
+ }
+
+ parent = irq_find_host(irq_parent);
++ of_node_put(irq_parent);
+ if (!parent) {
+ dev_err(dev, "No IRQ parent domain\n");
+ return -ENODEV;
+ }
+
+- fwnode = of_node_to_fwnode(irq_parent);
+- of_node_put(irq_parent);
+-
+ ret = bgpio_init(&priv->gpio_chip, dev, 4,
+ priv->base + GPIO_IDATA,
+ priv->base + GPIO_OSET,
+@@ -180,7 +177,7 @@ static int visconti_gpio_probe(struct platform_device *pdev)
+
+ girq = &priv->gpio_chip.irq;
+ girq->chip = irq_chip;
+- girq->fwnode = fwnode;
++ girq->fwnode = of_node_to_fwnode(dev->of_node);
+ girq->parent_domain = parent;
+ girq->child_to_parent_hwirq = visconti_gpio_child_to_parent_hwirq;
+ girq->populate_parent_alloc_arg = visconti_gpio_populate_parent_fwspec;
+diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
+index 47712b6903b51..53be0bdf2bc38 100644
+--- a/drivers/gpio/gpiolib-acpi.c
++++ b/drivers/gpio/gpiolib-acpi.c
+@@ -311,7 +311,8 @@ static struct gpio_desc *acpi_request_own_gpiod(struct gpio_chip *chip,
+ if (IS_ERR(desc))
+ return desc;
+
+- ret = gpio_set_debounce_timeout(desc, agpio->debounce_timeout);
++ /* ACPI uses hundredths of milliseconds units */
++ ret = gpio_set_debounce_timeout(desc, agpio->debounce_timeout * 10);
+ if (ret)
+ dev_warn(chip->parent,
+ "Failed to set debounce-timeout for pin 0x%04X, err %d\n",
+@@ -391,8 +392,8 @@ static acpi_status acpi_gpiochip_alloc_event(struct acpi_resource *ares,
+ pin = agpio->pin_table[0];
+
+ if (pin <= 255) {
+- char ev_name[5];
+- sprintf(ev_name, "_%c%02hhX",
++ char ev_name[8];
++ sprintf(ev_name, "_%c%02X",
+ agpio->triggering == ACPI_EDGE_SENSITIVE ? 'E' : 'L',
+ pin);
+ if (ACPI_SUCCESS(acpi_get_handle(handle, ev_name, &evt_handle)))
+@@ -1052,17 +1053,25 @@ int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, int ind
+ if (ret < 0)
+ return ret;
+
+- ret = gpio_set_debounce_timeout(desc, info.debounce);
++ /* ACPI uses hundredths of milliseconds units */
++ ret = gpio_set_debounce_timeout(desc, info.debounce * 10);
+ if (ret)
+ return ret;
+
+ irq_flags = acpi_dev_get_irq_type(info.triggering,
+ info.polarity);
+
+- /* Set type if specified and different than the current one */
+- if (irq_flags != IRQ_TYPE_NONE &&
+- irq_flags != irq_get_trigger_type(irq))
+- irq_set_irq_type(irq, irq_flags);
++ /*
++ * If the IRQ is not already in use then set type
++ * if specified and different than the current one.
++ */
++ if (can_request_irq(irq, irq_flags)) {
++ if (irq_flags != IRQ_TYPE_NONE &&
++ irq_flags != irq_get_trigger_type(irq))
++ irq_set_irq_type(irq, irq_flags);
++ } else {
++ dev_dbg(&adev->dev, "IRQ %d already in use\n", irq);
++ }
+
+ return irq;
+ }
+diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c
+index c7b5446d01fd2..ffa0256cad5a0 100644
+--- a/drivers/gpio/gpiolib-cdev.c
++++ b/drivers/gpio/gpiolib-cdev.c
+@@ -330,7 +330,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
+ goto out_free_lh;
+ }
+
+- ret = gpiod_request(desc, lh->label);
++ ret = gpiod_request_user(desc, lh->label);
+ if (ret)
+ goto out_free_lh;
+ lh->descs[i] = desc;
+@@ -1378,7 +1378,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
+ goto out_free_linereq;
+ }
+
+- ret = gpiod_request(desc, lr->label);
++ ret = gpiod_request_user(desc, lr->label);
+ if (ret)
+ goto out_free_linereq;
+
+@@ -1764,7 +1764,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
+ }
+ }
+
+- ret = gpiod_request(desc, le->label);
++ ret = gpiod_request_user(desc, le->label);
+ if (ret)
+ goto out_free_le;
+ le->desc = desc;
+diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
+index 0ad288ab6262d..a5b34c2487679 100644
+--- a/drivers/gpio/gpiolib-of.c
++++ b/drivers/gpio/gpiolib-of.c
+@@ -912,7 +912,7 @@ static void of_gpiochip_init_valid_mask(struct gpio_chip *chip)
+ i, &start);
+ of_property_read_u32_index(np, "gpio-reserved-ranges",
+ i + 1, &count);
+- if (start >= chip->ngpio || start + count >= chip->ngpio)
++ if (start >= chip->ngpio || start + count > chip->ngpio)
+ continue;
+
+ bitmap_clear(chip->valid_mask, start, count);
+diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
+index 4098bc7f88b7e..44c1ad51b3fe9 100644
+--- a/drivers/gpio/gpiolib-sysfs.c
++++ b/drivers/gpio/gpiolib-sysfs.c
+@@ -475,12 +475,9 @@ static ssize_t export_store(struct class *class,
+ * they may be undone on its behalf too.
+ */
+
+- status = gpiod_request(desc, "sysfs");
+- if (status) {
+- if (status == -EPROBE_DEFER)
+- status = -ENODEV;
++ status = gpiod_request_user(desc, "sysfs");
++ if (status)
+ goto done;
+- }
+
+ status = gpiod_set_transitory(desc, false);
+ if (!status) {
+diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
+index d1b9b721218f2..320baed949ee8 100644
+--- a/drivers/gpio/gpiolib.c
++++ b/drivers/gpio/gpiolib.c
+@@ -1368,6 +1368,16 @@ static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset)
+ {
+ struct irq_domain *domain = gc->irq.domain;
+
++#ifdef CONFIG_GPIOLIB_IRQCHIP
++ /*
++ * Avoid race condition with other code, which tries to lookup
++ * an IRQ before the irqchip has been properly registered,
++ * i.e. while gpiochip is still being brought up.
++ */
++ if (!gc->irq.initialized)
++ return -EPROBE_DEFER;
++#endif
++
+ if (!gpiochip_irqchip_irq_valid(gc, offset))
+ return -ENXIO;
+
+@@ -1550,6 +1560,15 @@ static int gpiochip_add_irqchip(struct gpio_chip *gc,
+
+ gpiochip_set_irq_hooks(gc);
+
++ /*
++ * Using barrier() here to prevent compiler from reordering
++ * gc->irq.initialized before initialization of above
++ * GPIO chip irq members.
++ */
++ barrier();
++
++ gc->irq.initialized = true;
++
+ acpi_gpiochip_request_interrupts(gc);
+
+ return 0;
+@@ -2186,6 +2205,16 @@ static int gpio_set_bias(struct gpio_desc *desc)
+ return gpio_set_config_with_argument_optional(desc, bias, arg);
+ }
+
++/**
++ * gpio_set_debounce_timeout() - Set debounce timeout
++ * @desc: GPIO descriptor to set the debounce timeout
++ * @debounce: Debounce timeout in microseconds
++ *
++ * The function calls the certain GPIO driver to set debounce timeout
++ * in the hardware.
++ *
++ * Returns 0 on success, or negative error code otherwise.
++ */
+ int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce)
+ {
+ return gpio_set_config_with_argument_optional(desc,
+@@ -3106,6 +3135,16 @@ int gpiod_to_irq(const struct gpio_desc *desc)
+
+ return retirq;
+ }
++#ifdef CONFIG_GPIOLIB_IRQCHIP
++ if (gc->irq.chip) {
++ /*
++ * Avoid race condition with other code, which tries to lookup
++ * an IRQ before the irqchip has been properly registered,
++ * i.e. while gpiochip is still being brought up.
++ */
++ return -EPROBE_DEFER;
++ }
++#endif
+ return -ENXIO;
+ }
+ EXPORT_SYMBOL_GPL(gpiod_to_irq);
+diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
+index 30bc3f80f83e6..c31f4626915de 100644
+--- a/drivers/gpio/gpiolib.h
++++ b/drivers/gpio/gpiolib.h
+@@ -135,6 +135,18 @@ struct gpio_desc {
+
+ int gpiod_request(struct gpio_desc *desc, const char *label);
+ void gpiod_free(struct gpio_desc *desc);
++
++static inline int gpiod_request_user(struct gpio_desc *desc, const char *label)
++{
++ int ret;
++
++ ret = gpiod_request(desc, label);
++ if (ret == -EPROBE_DEFER)
++ ret = -ENODEV;
++
++ return ret;
++}
++
+ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
+ unsigned long lflags, enum gpiod_flags dflags);
+ int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce);
+diff --git a/drivers/gpu/drm/amd/amdgpu/ObjectID.h b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
+index 5b393622f5920..a0f0a17e224fe 100644
+--- a/drivers/gpu/drm/amd/amdgpu/ObjectID.h
++++ b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
+@@ -119,6 +119,7 @@
+ #define CONNECTOR_OBJECT_ID_eDP 0x14
+ #define CONNECTOR_OBJECT_ID_MXM 0x15
+ #define CONNECTOR_OBJECT_ID_LVDS_eDP 0x16
++#define CONNECTOR_OBJECT_ID_USBC 0x17
+
+ /* deleted */
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+index 269437b013280..2eebefd26fa82 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+@@ -1069,6 +1069,7 @@ struct amdgpu_device {
+ bool runpm;
+ bool in_runpm;
+ bool has_pr3;
++ bool is_fw_fb;
+
+ bool pm_sysfs_en;
+ bool ucode_sysfs_en;
+@@ -1078,8 +1079,6 @@ struct amdgpu_device {
+ char product_name[32];
+ char serial[20];
+
+- struct amdgpu_autodump autodump;
+-
+ atomic_t throttling_logging_enabled;
+ struct ratelimit_state throttling_logging_rs;
+ uint32_t ras_hw_enabled;
+@@ -1398,12 +1397,10 @@ int amdgpu_acpi_smart_shift_update(struct drm_device *dev, enum amdgpu_ss ss_sta
+ int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
+
+ void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps);
+-bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev);
+ void amdgpu_acpi_detect(void);
+ #else
+ static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
+ static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
+-static inline bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { return false; }
+ static inline void amdgpu_acpi_detect(void) { }
+ static inline bool amdgpu_acpi_is_power_shift_control_supported(void) { return false; }
+ static inline int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
+@@ -1412,6 +1409,16 @@ static inline int amdgpu_acpi_smart_shift_update(struct drm_device *dev,
+ enum amdgpu_ss ss_state) { return 0; }
+ #endif
+
++#if defined(CONFIG_ACPI) && defined(CONFIG_SUSPEND)
++bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev);
++bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev);
++bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev);
++#else
++static inline bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { return false; }
++static inline bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev) { return false; }
++static inline bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev) { return false; }
++#endif
++
+ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+ uint64_t addr, struct amdgpu_bo **bo,
+ struct amdgpu_bo_va_mapping **mapping);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+index 4811b0faafd9a..98ac53ee6bb55 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+@@ -1031,6 +1031,34 @@ void amdgpu_acpi_detect(void)
+ }
+ }
+
++#if IS_ENABLED(CONFIG_SUSPEND)
++/**
++ * amdgpu_acpi_is_s3_active
++ *
++ * @adev: amdgpu_device_pointer
++ *
++ * returns true if supported, false if not.
++ */
++bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev)
++{
++ return !(adev->flags & AMD_IS_APU) ||
++ (pm_suspend_target_state == PM_SUSPEND_MEM);
++}
++
++/**
++ * amdgpu_acpi_should_gpu_reset
++ *
++ * @adev: amdgpu_device_pointer
++ *
++ * returns true if should reset GPU, false if not
++ */
++bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev)
++{
++ if (adev->flags & AMD_IS_APU)
++ return false;
++ return pm_suspend_target_state != PM_SUSPEND_TO_IDLE;
++}
++
+ /**
+ * amdgpu_acpi_is_s0ix_active
+ *
+@@ -1040,11 +1068,24 @@ void amdgpu_acpi_detect(void)
+ */
+ bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev)
+ {
+-#if IS_ENABLED(CONFIG_AMD_PMC) && IS_ENABLED(CONFIG_SUSPEND)
+- if (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0) {
+- if (adev->flags & AMD_IS_APU)
+- return pm_suspend_target_state == PM_SUSPEND_TO_IDLE;
++ if (!(adev->flags & AMD_IS_APU) ||
++ (pm_suspend_target_state != PM_SUSPEND_TO_IDLE))
++ return false;
++
++ if (!(acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0)) {
++ dev_warn_once(adev->dev,
++ "Power consumption will be higher as BIOS has not been configured for suspend-to-idle.\n"
++ "To use suspend-to-idle change the sleep mode in BIOS setup.\n");
++ return false;
+ }
+-#endif
++
++#if !IS_ENABLED(CONFIG_AMD_PMC)
++ dev_warn_once(adev->dev,
++ "Power consumption will be higher as the kernel has not been compiled with CONFIG_AMD_PMC.\n");
+ return false;
++#else
++ return true;
++#endif /* CONFIG_AMD_PMC */
+ }
++
++#endif /* CONFIG_SUSPEND */
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+index 054c1a224defb..ab36cce59d2e4 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+@@ -1393,7 +1393,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
+ struct sg_table *sg = NULL;
+ uint64_t user_addr = 0;
+ struct amdgpu_bo *bo;
+- struct drm_gem_object *gobj;
++ struct drm_gem_object *gobj = NULL;
+ u32 domain, alloc_domain;
+ u64 alloc_flags;
+ int ret;
+@@ -1503,14 +1503,16 @@ allocate_init_user_pages_failed:
+ remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+ drm_vma_node_revoke(&gobj->vma_node, drm_priv);
+ err_node_allow:
+- amdgpu_bo_unref(&bo);
+ /* Don't unreserve system mem limit twice */
+ goto err_reserve_limit;
+ err_bo_create:
+ unreserve_mem_limit(adev, size, alloc_domain, !!sg);
+ err_reserve_limit:
+ mutex_destroy(&(*mem)->lock);
+- kfree(*mem);
++ if (gobj)
++ drm_gem_object_put(gobj);
++ else
++ kfree(*mem);
+ err:
+ if (sg) {
+ sg_free_table(sg);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+index 15c45b2a39835..714178f1b6c6e 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+@@ -61,7 +61,7 @@ static void amdgpu_bo_list_free(struct kref *ref)
+
+ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
+ struct drm_amdgpu_bo_list_entry *info,
+- unsigned num_entries, struct amdgpu_bo_list **result)
++ size_t num_entries, struct amdgpu_bo_list **result)
+ {
+ unsigned last_entry = 0, first_userptr = num_entries;
+ struct amdgpu_bo_list_entry *array;
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
+index c905a4cfc173d..044b41f0bfd9c 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
+@@ -61,7 +61,7 @@ int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
+ int amdgpu_bo_list_create(struct amdgpu_device *adev,
+ struct drm_file *filp,
+ struct drm_amdgpu_bo_list_entry *info,
+- unsigned num_entries,
++ size_t num_entries,
+ struct amdgpu_bo_list **list);
+
+ static inline struct amdgpu_bo_list_entry *
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+index b9c11c2b2885a..a09876bb7ec8b 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+@@ -175,7 +175,7 @@ int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector)
+
+ /* Check if bpc is within clock limit. Try to degrade gracefully otherwise */
+ if ((bpc == 12) && (mode_clock * 3/2 > max_tmds_clock)) {
+- if ((connector->display_info.edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30) &&
++ if ((connector->display_info.edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30) &&
+ (mode_clock * 5/4 <= max_tmds_clock))
+ bpc = 10;
+ else
+@@ -387,6 +387,9 @@ amdgpu_connector_lcd_native_mode(struct drm_encoder *encoder)
+ native_mode->vdisplay != 0 &&
+ native_mode->clock != 0) {
+ mode = drm_mode_duplicate(dev, native_mode);
++ if (!mode)
++ return NULL;
++
+ mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+ drm_mode_set_name(mode);
+
+@@ -401,6 +404,9 @@ amdgpu_connector_lcd_native_mode(struct drm_encoder *encoder)
+ * simpler.
+ */
+ mode = drm_cvt_mode(dev, native_mode->hdisplay, native_mode->vdisplay, 60, true, false, false);
++ if (!mode)
++ return NULL;
++
+ mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+ DRM_DEBUG_KMS("Adding cvt approximation of native panel mode %s\n", mode->name);
+ }
+@@ -827,6 +833,7 @@ static int amdgpu_connector_vga_get_modes(struct drm_connector *connector)
+
+ amdgpu_connector_get_edid(connector);
+ ret = amdgpu_connector_ddc_get_modes(connector);
++ amdgpu_get_native_mode(connector);
+
+ return ret;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+index 913f9eaa9cd65..aa823f1541992 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+@@ -1508,6 +1508,7 @@ int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
+ return 0;
+
+ default:
++ dma_fence_put(fence);
+ return -EINVAL;
+ }
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+index 463b9c0283f7e..348629ea0e153 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+@@ -27,7 +27,6 @@
+ #include <linux/pci.h>
+ #include <linux/uaccess.h>
+ #include <linux/pm_runtime.h>
+-#include <linux/poll.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_pm.h"
+@@ -37,85 +36,7 @@
+ #include "amdgpu_securedisplay.h"
+ #include "amdgpu_fw_attestation.h"
+
+-int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
+-{
+ #if defined(CONFIG_DEBUG_FS)
+- unsigned long timeout = 600 * HZ;
+- int ret;
+-
+- wake_up_interruptible(&adev->autodump.gpu_hang);
+-
+- ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
+- if (ret == 0) {
+- pr_err("autodump: timeout, move on to gpu recovery\n");
+- return -ETIMEDOUT;
+- }
+-#endif
+- return 0;
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
+-{
+- struct amdgpu_device *adev = inode->i_private;
+- int ret;
+-
+- file->private_data = adev;
+-
+- ret = down_read_killable(&adev->reset_sem);
+- if (ret)
+- return ret;
+-
+- if (adev->autodump.dumping.done) {
+- reinit_completion(&adev->autodump.dumping);
+- ret = 0;
+- } else {
+- ret = -EBUSY;
+- }
+-
+- up_read(&adev->reset_sem);
+-
+- return ret;
+-}
+-
+-static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
+-{
+- struct amdgpu_device *adev = file->private_data;
+-
+- complete_all(&adev->autodump.dumping);
+- return 0;
+-}
+-
+-static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
+-{
+- struct amdgpu_device *adev = file->private_data;
+-
+- poll_wait(file, &adev->autodump.gpu_hang, poll_table);
+-
+- if (amdgpu_in_reset(adev))
+- return POLLIN | POLLRDNORM | POLLWRNORM;
+-
+- return 0;
+-}
+-
+-static const struct file_operations autodump_debug_fops = {
+- .owner = THIS_MODULE,
+- .open = amdgpu_debugfs_autodump_open,
+- .poll = amdgpu_debugfs_autodump_poll,
+- .release = amdgpu_debugfs_autodump_release,
+-};
+-
+-static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
+-{
+- init_completion(&adev->autodump.dumping);
+- complete_all(&adev->autodump.dumping);
+- init_waitqueue_head(&adev->autodump.gpu_hang);
+-
+- debugfs_create_file("amdgpu_autodump", 0600,
+- adev_to_drm(adev)->primary->debugfs_root,
+- adev, &autodump_debug_fops);
+-}
+
+ /**
+ * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
+@@ -1255,7 +1176,7 @@ static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
+ return r;
+ }
+
+- *val = amdgpu_bo_evict_vram(adev);
++ *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
+
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
+@@ -1268,17 +1189,15 @@ static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)data;
+ struct drm_device *dev = adev_to_drm(adev);
+- struct ttm_resource_manager *man;
+ int r;
+
+ r = pm_runtime_get_sync(dev->dev);
+ if (r < 0) {
+- pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
++ pm_runtime_put_autosuspend(dev->dev);
+ return r;
+ }
+
+- man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
+- *val = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
++ *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
+
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
+@@ -1588,7 +1507,6 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
+ }
+
+ amdgpu_ras_debugfs_create_all(adev);
+- amdgpu_debugfs_autodump_init(adev);
+ amdgpu_rap_debugfs_init(adev);
+ amdgpu_securedisplay_debugfs_init(adev);
+ amdgpu_fw_attestation_debugfs_init(adev);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+index 141a8474e24f2..8b641f40fdf66 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+@@ -26,10 +26,6 @@
+ /*
+ * Debugfs
+ */
+-struct amdgpu_autodump {
+- struct completion dumping;
+- struct wait_queue_head gpu_hang;
+-};
+
+ int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
+ int amdgpu_debugfs_init(struct amdgpu_device *adev);
+@@ -37,4 +33,3 @@ void amdgpu_debugfs_fini(struct amdgpu_device *adev);
+ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
+ void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
+ void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
+-int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+index af9bdf16eefd4..a926b5ebbfdf5 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+@@ -30,6 +30,7 @@
+ #include <linux/module.h>
+ #include <linux/console.h>
+ #include <linux/slab.h>
++#include <linux/pci.h>
+
+ #include <drm/drm_atomic_helper.h>
+ #include <drm/drm_probe_helper.h>
+@@ -2069,6 +2070,8 @@ out:
+ */
+ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
+ {
++ struct drm_device *dev = adev_to_drm(adev);
++ struct pci_dev *parent;
+ int i, r;
+
+ amdgpu_device_enable_virtual_display(adev);
+@@ -2168,6 +2171,18 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
+ return -EINVAL;
+ }
+
++ if (amdgpu_has_atpx() &&
++ (amdgpu_is_atpx_hybrid() ||
++ amdgpu_has_atpx_dgpu_power_cntl()) &&
++ ((adev->flags & AMD_IS_APU) == 0) &&
++ !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
++ adev->flags |= AMD_IS_PX;
++
++ if (!(adev->flags & AMD_IS_APU)) {
++ parent = pci_upstream_bridge(adev->pdev);
++ adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
++ }
++
+ amdgpu_amdkfd_device_probe(adev);
+
+ adev->pm.pp_feature = amdgpu_pp_feature_mask;
+@@ -2394,6 +2409,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
+ if (r)
+ goto init_failed;
+
++ r = amdgpu_amdkfd_resume_iommu(adev);
++ if (r)
++ goto init_failed;
++
+ r = amdgpu_device_ip_hw_init_phase1(adev);
+ if (r)
+ goto init_failed;
+@@ -2432,10 +2451,6 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
+ if (!adev->gmc.xgmi.pending_reset)
+ amdgpu_amdkfd_device_init(adev);
+
+- r = amdgpu_amdkfd_resume_iommu(adev);
+- if (r)
+- goto init_failed;
+-
+ amdgpu_fru_get_product_info(adev);
+
+ init_failed:
+@@ -2745,6 +2760,11 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
+ adev->ip_blocks[i].status.hw = false;
+ }
+
++ if (amdgpu_sriov_vf(adev)) {
++ if (amdgpu_virt_release_full_gpu(adev, false))
++ DRM_ERROR("failed to release exclusive mode on fini\n");
++ }
++
+ return 0;
+ }
+
+@@ -2805,10 +2825,6 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
+
+ amdgpu_ras_fini(adev);
+
+- if (amdgpu_sriov_vf(adev))
+- if (amdgpu_virt_release_full_gpu(adev, false))
+- DRM_ERROR("failed to release exclusive mode on fini\n");
+-
+ return 0;
+ }
+
+@@ -3531,6 +3547,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
+ adev->rmmio_size = pci_resource_len(adev->pdev, 2);
+ }
+
++ for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
++ atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
++
+ adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
+ if (adev->rmmio == NULL) {
+ return -ENOMEM;
+@@ -3850,7 +3869,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
+ /* disable all interrupts */
+ amdgpu_irq_disable_all(adev);
+ if (adev->mode_info.mode_config_initialized){
+- if (!amdgpu_device_has_dc_support(adev))
++ if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
+ drm_helper_force_disable_all(adev_to_drm(adev));
+ else
+ drm_atomic_helper_shutdown(adev_to_drm(adev));
+@@ -3876,8 +3895,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
+
+ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
+ {
+- amdgpu_device_ip_fini(adev);
+ amdgpu_fence_driver_sw_fini(adev);
++ amdgpu_device_ip_fini(adev);
+ release_firmware(adev->firmware.gpu_info_fw);
+ adev->firmware.gpu_info_fw = NULL;
+ adev->accel_working = false;
+@@ -3909,6 +3928,25 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
+
+ }
+
++/**
++ * amdgpu_device_evict_resources - evict device resources
++ * @adev: amdgpu device object
++ *
++ * Evicts all ttm device resources(vram BOs, gart table) from the lru list
++ * of the vram memory type. Mainly used for evicting device resources
++ * at suspend time.
++ *
++ */
++static void amdgpu_device_evict_resources(struct amdgpu_device *adev)
++{
++ /* No need to evict vram on APUs for suspend to ram or s2idle */
++ if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
++ return;
++
++ if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM))
++ DRM_WARN("evicting device resources failed\n");
++
++}
+
+ /*
+ * Suspend & resume.
+@@ -3949,17 +3987,16 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
+ if (!adev->in_s0ix)
+ amdgpu_amdkfd_suspend(adev, adev->in_runpm);
+
+- /* evict vram memory */
+- amdgpu_bo_evict_vram(adev);
++ /* First evict vram memory */
++ amdgpu_device_evict_resources(adev);
+
+ amdgpu_fence_driver_hw_fini(adev);
+
+ amdgpu_device_ip_suspend_phase2(adev);
+- /* evict remaining vram memory
+- * This second call to evict vram is to evict the gart page table
+- * using the CPU.
++ /* This second call to evict device resources is to evict
++ * the gart page table using the CPU.
+ */
+- amdgpu_bo_evict_vram(adev);
++ amdgpu_device_evict_resources(adev);
+
+ return 0;
+ }
+@@ -4466,10 +4503,6 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
+ if (reset_context->reset_req_dev == adev)
+ job = reset_context->job;
+
+- /* no need to dump if device is not in good state during probe period */
+- if (!adev->gmc.xgmi.pending_reset)
+- amdgpu_debugfs_wait_dump(adev);
+-
+ if (amdgpu_sriov_vf(adev)) {
+ /* stop the data exchange thread */
+ amdgpu_virt_fini_data_exchange(adev);
+@@ -5130,7 +5163,7 @@ skip_hw_reset:
+ drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
+ }
+
+- if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
++ if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
+ drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
+ }
+
+@@ -5610,7 +5643,7 @@ void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring)
+ {
+ #ifdef CONFIG_X86_64
+- if (adev->flags & AMD_IS_APU)
++ if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
+ return;
+ #endif
+ if (adev->gmc.xgmi.connected_to_cpu)
+@@ -5626,7 +5659,7 @@ void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring)
+ {
+ #ifdef CONFIG_X86_64
+- if (adev->flags & AMD_IS_APU)
++ if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
+ return;
+ #endif
+ if (adev->gmc.xgmi.connected_to_cpu)
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+index ada7bc19118ac..a919f5daacd91 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+@@ -415,10 +415,15 @@ void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev)
+ }
+ }
+
++union gc_info {
++ struct gc_info_v1_0 v1;
++ struct gc_info_v2_0 v2;
++};
++
+ int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
+ {
+ struct binary_header *bhdr;
+- struct gc_info_v1_0 *gc_info;
++ union gc_info *gc_info;
+
+ if (!adev->mman.discovery_bin) {
+ DRM_ERROR("ip discovery uninitialized\n");
+@@ -426,27 +431,54 @@ int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
+ }
+
+ bhdr = (struct binary_header *)adev->mman.discovery_bin;
+- gc_info = (struct gc_info_v1_0 *)(adev->mman.discovery_bin +
++ gc_info = (union gc_info *)(adev->mman.discovery_bin +
+ le16_to_cpu(bhdr->table_list[GC].offset));
+-
+- adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->gc_num_se);
+- adev->gfx.config.max_cu_per_sh = 2 * (le32_to_cpu(gc_info->gc_num_wgp0_per_sa) +
+- le32_to_cpu(gc_info->gc_num_wgp1_per_sa));
+- adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->gc_num_sa_per_se);
+- adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->gc_num_rb_per_se);
+- adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->gc_num_gl2c);
+- adev->gfx.config.max_gprs = le32_to_cpu(gc_info->gc_num_gprs);
+- adev->gfx.config.max_gs_threads = le32_to_cpu(gc_info->gc_num_max_gs_thds);
+- adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gc_info->gc_gs_table_depth);
+- adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gc_info->gc_gsprim_buff_depth);
+- adev->gfx.config.double_offchip_lds_buf = le32_to_cpu(gc_info->gc_double_offchip_lds_buffer);
+- adev->gfx.cu_info.wave_front_size = le32_to_cpu(gc_info->gc_wave_size);
+- adev->gfx.cu_info.max_waves_per_simd = le32_to_cpu(gc_info->gc_max_waves_per_simd);
+- adev->gfx.cu_info.max_scratch_slots_per_cu = le32_to_cpu(gc_info->gc_max_scratch_slots_per_cu);
+- adev->gfx.cu_info.lds_size = le32_to_cpu(gc_info->gc_lds_size);
+- adev->gfx.config.num_sc_per_sh = le32_to_cpu(gc_info->gc_num_sc_per_se) /
+- le32_to_cpu(gc_info->gc_num_sa_per_se);
+- adev->gfx.config.num_packer_per_sc = le32_to_cpu(gc_info->gc_num_packer_per_sc);
+-
++ switch (gc_info->v1.header.version_major) {
++ case 1:
++ adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->v1.gc_num_se);
++ adev->gfx.config.max_cu_per_sh = 2 * (le32_to_cpu(gc_info->v1.gc_num_wgp0_per_sa) +
++ le32_to_cpu(gc_info->v1.gc_num_wgp1_per_sa));
++ adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->v1.gc_num_sa_per_se);
++ adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->v1.gc_num_rb_per_se);
++ adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->v1.gc_num_gl2c);
++ adev->gfx.config.max_gprs = le32_to_cpu(gc_info->v1.gc_num_gprs);
++ adev->gfx.config.max_gs_threads = le32_to_cpu(gc_info->v1.gc_num_max_gs_thds);
++ adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gc_info->v1.gc_gs_table_depth);
++ adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gc_info->v1.gc_gsprim_buff_depth);
++ adev->gfx.config.double_offchip_lds_buf = le32_to_cpu(gc_info->v1.gc_double_offchip_lds_buffer);
++ adev->gfx.cu_info.wave_front_size = le32_to_cpu(gc_info->v1.gc_wave_size);
++ adev->gfx.cu_info.max_waves_per_simd = le32_to_cpu(gc_info->v1.gc_max_waves_per_simd);
++ adev->gfx.cu_info.max_scratch_slots_per_cu = le32_to_cpu(gc_info->v1.gc_max_scratch_slots_per_cu);
++ adev->gfx.cu_info.lds_size = le32_to_cpu(gc_info->v1.gc_lds_size);
++ adev->gfx.config.num_sc_per_sh = le32_to_cpu(gc_info->v1.gc_num_sc_per_se) /
++ le32_to_cpu(gc_info->v1.gc_num_sa_per_se);
++ adev->gfx.config.num_packer_per_sc = le32_to_cpu(gc_info->v1.gc_num_packer_per_sc);
++ break;
++ case 2:
++ adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->v2.gc_num_se);
++ adev->gfx.config.max_cu_per_sh = le32_to_cpu(gc_info->v2.gc_num_cu_per_sh);
++ adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->v2.gc_num_sh_per_se);
++ adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->v2.gc_num_rb_per_se);
++ adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->v2.gc_num_tccs);
++ adev->gfx.config.max_gprs = le32_to_cpu(gc_info->v2.gc_num_gprs);
++ adev->gfx.config.max_gs_threads = le32_to_cpu(gc_info->v2.gc_num_max_gs_thds);
++ adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gc_info->v2.gc_gs_table_depth);
++ adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gc_info->v2.gc_gsprim_buff_depth);
++ adev->gfx.config.double_offchip_lds_buf = le32_to_cpu(gc_info->v2.gc_double_offchip_lds_buffer);
++ adev->gfx.cu_info.wave_front_size = le32_to_cpu(gc_info->v2.gc_wave_size);
++ adev->gfx.cu_info.max_waves_per_simd = le32_to_cpu(gc_info->v2.gc_max_waves_per_simd);
++ adev->gfx.cu_info.max_scratch_slots_per_cu = le32_to_cpu(gc_info->v2.gc_max_scratch_slots_per_cu);
++ adev->gfx.cu_info.lds_size = le32_to_cpu(gc_info->v2.gc_lds_size);
++ adev->gfx.config.num_sc_per_sh = le32_to_cpu(gc_info->v2.gc_num_sc_per_se) /
++ le32_to_cpu(gc_info->v2.gc_num_sh_per_se);
++ adev->gfx.config.num_packer_per_sc = le32_to_cpu(gc_info->v2.gc_num_packer_per_sc);
++ break;
++ default:
++ dev_err(adev->dev,
++ "Unhandled GC info table %d.%d\n",
++ gc_info->v1.header.version_major,
++ gc_info->v1.header.version_minor);
++ return -EINVAL;
++ }
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+index dc50c05f23fc2..5c08047adb594 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+@@ -1145,7 +1145,7 @@ int amdgpu_display_framebuffer_init(struct drm_device *dev,
+ if (ret)
+ return ret;
+
+- if (!dev->mode_config.allow_fb_modifiers) {
++ if (!dev->mode_config.allow_fb_modifiers && !adev->enable_virtual_display) {
+ drm_WARN_ONCE(dev, adev->family >= AMDGPU_FAMILY_AI,
+ "GFX9+ requires FB check based on format modifier\n");
+ ret = check_tiling_flags_gfx6(rfb);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+index ae6ab93c868b8..7444484a12bf8 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+@@ -384,7 +384,7 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
+ struct amdgpu_vm_bo_base *bo_base;
+ int r;
+
+- if (bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
++ if (!bo->tbo.resource || bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
+ return;
+
+ r = ttm_bo_validate(&bo->tbo, &placement, &ctx);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+index f18240f873878..129661f728bf2 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+@@ -38,6 +38,7 @@
+ #include <drm/drm_probe_helper.h>
+ #include <linux/mmu_notifier.h>
+ #include <linux/suspend.h>
++#include <linux/fb.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_irq.h"
+@@ -679,7 +680,7 @@ MODULE_PARM_DESC(sched_policy,
+ * Maximum number of processes that HWS can schedule concurrently. The maximum is the
+ * number of VMIDs assigned to the HWS, which is also the default.
+ */
+-int hws_max_conc_proc = 8;
++int hws_max_conc_proc = -1;
+ module_param(hws_max_conc_proc, int, 0444);
+ MODULE_PARM_DESC(hws_max_conc_proc,
+ "Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))");
+@@ -890,6 +891,717 @@ MODULE_PARM_DESC(smu_pptable_id,
+ "specify pptable id to be used (-1 = auto(default) value, 0 = use pptable from vbios, > 0 = soft pptable id)");
+ module_param_named(smu_pptable_id, amdgpu_smu_pptable_id, int, 0444);
+
++/* These devices are not supported by amdgpu.
++ * They are supported by the mach64, r128, radeon drivers
++ */
++static const u16 amdgpu_unsupported_pciidlist[] = {
++ /* mach64 */
++ 0x4354,
++ 0x4358,
++ 0x4554,
++ 0x4742,
++ 0x4744,
++ 0x4749,
++ 0x474C,
++ 0x474D,
++ 0x474E,
++ 0x474F,
++ 0x4750,
++ 0x4751,
++ 0x4752,
++ 0x4753,
++ 0x4754,
++ 0x4755,
++ 0x4756,
++ 0x4757,
++ 0x4758,
++ 0x4759,
++ 0x475A,
++ 0x4C42,
++ 0x4C44,
++ 0x4C47,
++ 0x4C49,
++ 0x4C4D,
++ 0x4C4E,
++ 0x4C50,
++ 0x4C51,
++ 0x4C52,
++ 0x4C53,
++ 0x5654,
++ 0x5655,
++ 0x5656,
++ /* r128 */
++ 0x4c45,
++ 0x4c46,
++ 0x4d46,
++ 0x4d4c,
++ 0x5041,
++ 0x5042,
++ 0x5043,
++ 0x5044,
++ 0x5045,
++ 0x5046,
++ 0x5047,
++ 0x5048,
++ 0x5049,
++ 0x504A,
++ 0x504B,
++ 0x504C,
++ 0x504D,
++ 0x504E,
++ 0x504F,
++ 0x5050,
++ 0x5051,
++ 0x5052,
++ 0x5053,
++ 0x5054,
++ 0x5055,
++ 0x5056,
++ 0x5057,
++ 0x5058,
++ 0x5245,
++ 0x5246,
++ 0x5247,
++ 0x524b,
++ 0x524c,
++ 0x534d,
++ 0x5446,
++ 0x544C,
++ 0x5452,
++ /* radeon */
++ 0x3150,
++ 0x3151,
++ 0x3152,
++ 0x3154,
++ 0x3155,
++ 0x3E50,
++ 0x3E54,
++ 0x4136,
++ 0x4137,
++ 0x4144,
++ 0x4145,
++ 0x4146,
++ 0x4147,
++ 0x4148,
++ 0x4149,
++ 0x414A,
++ 0x414B,
++ 0x4150,
++ 0x4151,
++ 0x4152,
++ 0x4153,
++ 0x4154,
++ 0x4155,
++ 0x4156,
++ 0x4237,
++ 0x4242,
++ 0x4336,
++ 0x4337,
++ 0x4437,
++ 0x4966,
++ 0x4967,
++ 0x4A48,
++ 0x4A49,
++ 0x4A4A,
++ 0x4A4B,
++ 0x4A4C,
++ 0x4A4D,
++ 0x4A4E,
++ 0x4A4F,
++ 0x4A50,
++ 0x4A54,
++ 0x4B48,
++ 0x4B49,
++ 0x4B4A,
++ 0x4B4B,
++ 0x4B4C,
++ 0x4C57,
++ 0x4C58,
++ 0x4C59,
++ 0x4C5A,
++ 0x4C64,
++ 0x4C66,
++ 0x4C67,
++ 0x4E44,
++ 0x4E45,
++ 0x4E46,
++ 0x4E47,
++ 0x4E48,
++ 0x4E49,
++ 0x4E4A,
++ 0x4E4B,
++ 0x4E50,
++ 0x4E51,
++ 0x4E52,
++ 0x4E53,
++ 0x4E54,
++ 0x4E56,
++ 0x5144,
++ 0x5145,
++ 0x5146,
++ 0x5147,
++ 0x5148,
++ 0x514C,
++ 0x514D,
++ 0x5157,
++ 0x5158,
++ 0x5159,
++ 0x515A,
++ 0x515E,
++ 0x5460,
++ 0x5462,
++ 0x5464,
++ 0x5548,
++ 0x5549,
++ 0x554A,
++ 0x554B,
++ 0x554C,
++ 0x554D,
++ 0x554E,
++ 0x554F,
++ 0x5550,
++ 0x5551,
++ 0x5552,
++ 0x5554,
++ 0x564A,
++ 0x564B,
++ 0x564F,
++ 0x5652,
++ 0x5653,
++ 0x5657,
++ 0x5834,
++ 0x5835,
++ 0x5954,
++ 0x5955,
++ 0x5974,
++ 0x5975,
++ 0x5960,
++ 0x5961,
++ 0x5962,
++ 0x5964,
++ 0x5965,
++ 0x5969,
++ 0x5a41,
++ 0x5a42,
++ 0x5a61,
++ 0x5a62,
++ 0x5b60,
++ 0x5b62,
++ 0x5b63,
++ 0x5b64,
++ 0x5b65,
++ 0x5c61,
++ 0x5c63,
++ 0x5d48,
++ 0x5d49,
++ 0x5d4a,
++ 0x5d4c,
++ 0x5d4d,
++ 0x5d4e,
++ 0x5d4f,
++ 0x5d50,
++ 0x5d52,
++ 0x5d57,
++ 0x5e48,
++ 0x5e4a,
++ 0x5e4b,
++ 0x5e4c,
++ 0x5e4d,
++ 0x5e4f,
++ 0x6700,
++ 0x6701,
++ 0x6702,
++ 0x6703,
++ 0x6704,
++ 0x6705,
++ 0x6706,
++ 0x6707,
++ 0x6708,
++ 0x6709,
++ 0x6718,
++ 0x6719,
++ 0x671c,
++ 0x671d,
++ 0x671f,
++ 0x6720,
++ 0x6721,
++ 0x6722,
++ 0x6723,
++ 0x6724,
++ 0x6725,
++ 0x6726,
++ 0x6727,
++ 0x6728,
++ 0x6729,
++ 0x6738,
++ 0x6739,
++ 0x673e,
++ 0x6740,
++ 0x6741,
++ 0x6742,
++ 0x6743,
++ 0x6744,
++ 0x6745,
++ 0x6746,
++ 0x6747,
++ 0x6748,
++ 0x6749,
++ 0x674A,
++ 0x6750,
++ 0x6751,
++ 0x6758,
++ 0x6759,
++ 0x675B,
++ 0x675D,
++ 0x675F,
++ 0x6760,
++ 0x6761,
++ 0x6762,
++ 0x6763,
++ 0x6764,
++ 0x6765,
++ 0x6766,
++ 0x6767,
++ 0x6768,
++ 0x6770,
++ 0x6771,
++ 0x6772,
++ 0x6778,
++ 0x6779,
++ 0x677B,
++ 0x6840,
++ 0x6841,
++ 0x6842,
++ 0x6843,
++ 0x6849,
++ 0x684C,
++ 0x6850,
++ 0x6858,
++ 0x6859,
++ 0x6880,
++ 0x6888,
++ 0x6889,
++ 0x688A,
++ 0x688C,
++ 0x688D,
++ 0x6898,
++ 0x6899,
++ 0x689b,
++ 0x689c,
++ 0x689d,
++ 0x689e,
++ 0x68a0,
++ 0x68a1,
++ 0x68a8,
++ 0x68a9,
++ 0x68b0,
++ 0x68b8,
++ 0x68b9,
++ 0x68ba,
++ 0x68be,
++ 0x68bf,
++ 0x68c0,
++ 0x68c1,
++ 0x68c7,
++ 0x68c8,
++ 0x68c9,
++ 0x68d8,
++ 0x68d9,
++ 0x68da,
++ 0x68de,
++ 0x68e0,
++ 0x68e1,
++ 0x68e4,
++ 0x68e5,
++ 0x68e8,
++ 0x68e9,
++ 0x68f1,
++ 0x68f2,
++ 0x68f8,
++ 0x68f9,
++ 0x68fa,
++ 0x68fe,
++ 0x7100,
++ 0x7101,
++ 0x7102,
++ 0x7103,
++ 0x7104,
++ 0x7105,
++ 0x7106,
++ 0x7108,
++ 0x7109,
++ 0x710A,
++ 0x710B,
++ 0x710C,
++ 0x710E,
++ 0x710F,
++ 0x7140,
++ 0x7141,
++ 0x7142,
++ 0x7143,
++ 0x7144,
++ 0x7145,
++ 0x7146,
++ 0x7147,
++ 0x7149,
++ 0x714A,
++ 0x714B,
++ 0x714C,
++ 0x714D,
++ 0x714E,
++ 0x714F,
++ 0x7151,
++ 0x7152,
++ 0x7153,
++ 0x715E,
++ 0x715F,
++ 0x7180,
++ 0x7181,
++ 0x7183,
++ 0x7186,
++ 0x7187,
++ 0x7188,
++ 0x718A,
++ 0x718B,
++ 0x718C,
++ 0x718D,
++ 0x718F,
++ 0x7193,
++ 0x7196,
++ 0x719B,
++ 0x719F,
++ 0x71C0,
++ 0x71C1,
++ 0x71C2,
++ 0x71C3,
++ 0x71C4,
++ 0x71C5,
++ 0x71C6,
++ 0x71C7,
++ 0x71CD,
++ 0x71CE,
++ 0x71D2,
++ 0x71D4,
++ 0x71D5,
++ 0x71D6,
++ 0x71DA,
++ 0x71DE,
++ 0x7200,
++ 0x7210,
++ 0x7211,
++ 0x7240,
++ 0x7243,
++ 0x7244,
++ 0x7245,
++ 0x7246,
++ 0x7247,
++ 0x7248,
++ 0x7249,
++ 0x724A,
++ 0x724B,
++ 0x724C,
++ 0x724D,
++ 0x724E,
++ 0x724F,
++ 0x7280,
++ 0x7281,
++ 0x7283,
++ 0x7284,
++ 0x7287,
++ 0x7288,
++ 0x7289,
++ 0x728B,
++ 0x728C,
++ 0x7290,
++ 0x7291,
++ 0x7293,
++ 0x7297,
++ 0x7834,
++ 0x7835,
++ 0x791e,
++ 0x791f,
++ 0x793f,
++ 0x7941,
++ 0x7942,
++ 0x796c,
++ 0x796d,
++ 0x796e,
++ 0x796f,
++ 0x9400,
++ 0x9401,
++ 0x9402,
++ 0x9403,
++ 0x9405,
++ 0x940A,
++ 0x940B,
++ 0x940F,
++ 0x94A0,
++ 0x94A1,
++ 0x94A3,
++ 0x94B1,
++ 0x94B3,
++ 0x94B4,
++ 0x94B5,
++ 0x94B9,
++ 0x9440,
++ 0x9441,
++ 0x9442,
++ 0x9443,
++ 0x9444,
++ 0x9446,
++ 0x944A,
++ 0x944B,
++ 0x944C,
++ 0x944E,
++ 0x9450,
++ 0x9452,
++ 0x9456,
++ 0x945A,
++ 0x945B,
++ 0x945E,
++ 0x9460,
++ 0x9462,
++ 0x946A,
++ 0x946B,
++ 0x947A,
++ 0x947B,
++ 0x9480,
++ 0x9487,
++ 0x9488,
++ 0x9489,
++ 0x948A,
++ 0x948F,
++ 0x9490,
++ 0x9491,
++ 0x9495,
++ 0x9498,
++ 0x949C,
++ 0x949E,
++ 0x949F,
++ 0x94C0,
++ 0x94C1,
++ 0x94C3,
++ 0x94C4,
++ 0x94C5,
++ 0x94C6,
++ 0x94C7,
++ 0x94C8,
++ 0x94C9,
++ 0x94CB,
++ 0x94CC,
++ 0x94CD,
++ 0x9500,
++ 0x9501,
++ 0x9504,
++ 0x9505,
++ 0x9506,
++ 0x9507,
++ 0x9508,
++ 0x9509,
++ 0x950F,
++ 0x9511,
++ 0x9515,
++ 0x9517,
++ 0x9519,
++ 0x9540,
++ 0x9541,
++ 0x9542,
++ 0x954E,
++ 0x954F,
++ 0x9552,
++ 0x9553,
++ 0x9555,
++ 0x9557,
++ 0x955f,
++ 0x9580,
++ 0x9581,
++ 0x9583,
++ 0x9586,
++ 0x9587,
++ 0x9588,
++ 0x9589,
++ 0x958A,
++ 0x958B,
++ 0x958C,
++ 0x958D,
++ 0x958E,
++ 0x958F,
++ 0x9590,
++ 0x9591,
++ 0x9593,
++ 0x9595,
++ 0x9596,
++ 0x9597,
++ 0x9598,
++ 0x9599,
++ 0x959B,
++ 0x95C0,
++ 0x95C2,
++ 0x95C4,
++ 0x95C5,
++ 0x95C6,
++ 0x95C7,
++ 0x95C9,
++ 0x95CC,
++ 0x95CD,
++ 0x95CE,
++ 0x95CF,
++ 0x9610,
++ 0x9611,
++ 0x9612,
++ 0x9613,
++ 0x9614,
++ 0x9615,
++ 0x9616,
++ 0x9640,
++ 0x9641,
++ 0x9642,
++ 0x9643,
++ 0x9644,
++ 0x9645,
++ 0x9647,
++ 0x9648,
++ 0x9649,
++ 0x964a,
++ 0x964b,
++ 0x964c,
++ 0x964e,
++ 0x964f,
++ 0x9710,
++ 0x9711,
++ 0x9712,
++ 0x9713,
++ 0x9714,
++ 0x9715,
++ 0x9802,
++ 0x9803,
++ 0x9804,
++ 0x9805,
++ 0x9806,
++ 0x9807,
++ 0x9808,
++ 0x9809,
++ 0x980A,
++ 0x9900,
++ 0x9901,
++ 0x9903,
++ 0x9904,
++ 0x9905,
++ 0x9906,
++ 0x9907,
++ 0x9908,
++ 0x9909,
++ 0x990A,
++ 0x990B,
++ 0x990C,
++ 0x990D,
++ 0x990E,
++ 0x990F,
++ 0x9910,
++ 0x9913,
++ 0x9917,
++ 0x9918,
++ 0x9919,
++ 0x9990,
++ 0x9991,
++ 0x9992,
++ 0x9993,
++ 0x9994,
++ 0x9995,
++ 0x9996,
++ 0x9997,
++ 0x9998,
++ 0x9999,
++ 0x999A,
++ 0x999B,
++ 0x999C,
++ 0x999D,
++ 0x99A0,
++ 0x99A2,
++ 0x99A4,
++ /* radeon secondary ids */
++ 0x3171,
++ 0x3e70,
++ 0x4164,
++ 0x4165,
++ 0x4166,
++ 0x4168,
++ 0x4170,
++ 0x4171,
++ 0x4172,
++ 0x4173,
++ 0x496e,
++ 0x4a69,
++ 0x4a6a,
++ 0x4a6b,
++ 0x4a70,
++ 0x4a74,
++ 0x4b69,
++ 0x4b6b,
++ 0x4b6c,
++ 0x4c6e,
++ 0x4e64,
++ 0x4e65,
++ 0x4e66,
++ 0x4e67,
++ 0x4e68,
++ 0x4e69,
++ 0x4e6a,
++ 0x4e71,
++ 0x4f73,
++ 0x5569,
++ 0x556b,
++ 0x556d,
++ 0x556f,
++ 0x5571,
++ 0x5854,
++ 0x5874,
++ 0x5940,
++ 0x5941,
++ 0x5b72,
++ 0x5b73,
++ 0x5b74,
++ 0x5b75,
++ 0x5d44,
++ 0x5d45,
++ 0x5d6d,
++ 0x5d6f,
++ 0x5d72,
++ 0x5d77,
++ 0x5e6b,
++ 0x5e6d,
++ 0x7120,
++ 0x7124,
++ 0x7129,
++ 0x712e,
++ 0x712f,
++ 0x7162,
++ 0x7163,
++ 0x7166,
++ 0x7167,
++ 0x7172,
++ 0x7173,
++ 0x71a0,
++ 0x71a1,
++ 0x71a3,
++ 0x71a7,
++ 0x71bb,
++ 0x71e0,
++ 0x71e1,
++ 0x71e2,
++ 0x71e6,
++ 0x71e7,
++ 0x71f2,
++ 0x7269,
++ 0x726b,
++ 0x726e,
++ 0x72a0,
++ 0x72a8,
++ 0x72b1,
++ 0x72b3,
++ 0x793f,
++};
++
+ static const struct pci_device_id pciidlist[] = {
+ #ifdef CONFIG_DRM_AMDGPU_SI
+ {0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+@@ -1246,14 +1958,45 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
+
+ static const struct drm_driver amdgpu_kms_driver;
+
++static bool amdgpu_is_fw_framebuffer(resource_size_t base,
++ resource_size_t size)
++{
++ bool found = false;
++#if IS_REACHABLE(CONFIG_FB)
++ struct apertures_struct *a;
++
++ a = alloc_apertures(1);
++ if (!a)
++ return false;
++
++ a->ranges[0].base = base;
++ a->ranges[0].size = size;
++
++ found = is_firmware_framebuffer(a);
++ kfree(a);
++#endif
++ return found;
++}
++
+ static int amdgpu_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *ent)
+ {
+ struct drm_device *ddev;
+ struct amdgpu_device *adev;
+ unsigned long flags = ent->driver_data;
+- int ret, retry = 0;
++ int ret, retry = 0, i;
+ bool supports_atomic = false;
++ bool is_fw_fb;
++ resource_size_t base, size;
++
++ if (amdgpu_aspm == -1 && !pcie_aspm_enabled(pdev))
++ amdgpu_aspm = 0;
++
++ /* skip devices which are owned by radeon */
++ for (i = 0; i < ARRAY_SIZE(amdgpu_unsupported_pciidlist); i++) {
++ if (amdgpu_unsupported_pciidlist[i] == pdev->device)
++ return -ENODEV;
++ }
+
+ if (amdgpu_virtual_display ||
+ amdgpu_device_asic_has_dc_support(flags & AMD_ASIC_MASK))
+@@ -1310,6 +2053,10 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
+ }
+ #endif
+
++ base = pci_resource_start(pdev, 0);
++ size = pci_resource_len(pdev, 0);
++ is_fw_fb = amdgpu_is_fw_framebuffer(base, size);
++
+ /* Get rid of things like offb */
+ ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &amdgpu_kms_driver);
+ if (ret)
+@@ -1322,6 +2069,7 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
+ adev->dev = &pdev->dev;
+ adev->pdev = pdev;
+ ddev = adev_to_drm(adev);
++ adev->is_fw_fb = is_fw_fb;
+
+ if (!supports_atomic)
+ ddev->driver_features &= ~DRIVER_ATOMIC;
+@@ -1471,13 +2219,20 @@ static void amdgpu_drv_delayed_reset_work_handler(struct work_struct *work)
+ static int amdgpu_pmops_prepare(struct device *dev)
+ {
+ struct drm_device *drm_dev = dev_get_drvdata(dev);
++ struct amdgpu_device *adev = drm_to_adev(drm_dev);
+
+ /* Return a positive number here so
+ * DPM_FLAG_SMART_SUSPEND works properly
+ */
+ if (amdgpu_device_supports_boco(drm_dev))
+- return pm_runtime_suspended(dev) &&
+- pm_suspend_via_firmware();
++ return pm_runtime_suspended(dev);
++
++ /* if we will not support s3 or s2i for the device
++ * then skip suspend
++ */
++ if (!amdgpu_acpi_is_s0ix_active(adev) &&
++ !amdgpu_acpi_is_s3_active(adev))
++ return 1;
+
+ return 0;
+ }
+@@ -1491,15 +2246,23 @@ static int amdgpu_pmops_suspend(struct device *dev)
+ {
+ struct drm_device *drm_dev = dev_get_drvdata(dev);
+ struct amdgpu_device *adev = drm_to_adev(drm_dev);
+- int r;
+
+ if (amdgpu_acpi_is_s0ix_active(adev))
+ adev->in_s0ix = true;
+- adev->in_s3 = true;
+- r = amdgpu_device_suspend(drm_dev, true);
+- adev->in_s3 = false;
++ else
++ adev->in_s3 = true;
++ return amdgpu_device_suspend(drm_dev, true);
++}
+
+- return r;
++static int amdgpu_pmops_suspend_noirq(struct device *dev)
++{
++ struct drm_device *drm_dev = dev_get_drvdata(dev);
++ struct amdgpu_device *adev = drm_to_adev(drm_dev);
++
++ if (amdgpu_acpi_should_gpu_reset(adev))
++ return amdgpu_asic_reset(adev);
++
++ return 0;
+ }
+
+ static int amdgpu_pmops_resume(struct device *dev)
+@@ -1511,6 +2274,8 @@ static int amdgpu_pmops_resume(struct device *dev)
+ r = amdgpu_device_resume(drm_dev, true);
+ if (amdgpu_acpi_is_s0ix_active(adev))
+ adev->in_s0ix = false;
++ else
++ adev->in_s3 = false;
+ return r;
+ }
+
+@@ -1575,12 +2340,27 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
+ if (amdgpu_device_supports_px(drm_dev))
+ drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+
++ /*
++ * By setting mp1_state as PP_MP1_STATE_UNLOAD, MP1 will do some
++ * proper cleanups and put itself into a state ready for PNP. That
++ * can address some random resuming failure observed on BOCO capable
++ * platforms.
++ * TODO: this may be also needed for PX capable platform.
++ */
++ if (amdgpu_device_supports_boco(drm_dev))
++ adev->mp1_state = PP_MP1_STATE_UNLOAD;
++
+ ret = amdgpu_device_suspend(drm_dev, false);
+ if (ret) {
+ adev->in_runpm = false;
++ if (amdgpu_device_supports_boco(drm_dev))
++ adev->mp1_state = PP_MP1_STATE_NONE;
+ return ret;
+ }
+
++ if (amdgpu_device_supports_boco(drm_dev))
++ adev->mp1_state = PP_MP1_STATE_NONE;
++
+ if (amdgpu_device_supports_px(drm_dev)) {
+ /* Only need to handle PCI state in the driver for ATPX
+ * PCI core handles it for _PR3.
+@@ -1719,6 +2499,7 @@ static const struct dev_pm_ops amdgpu_pm_ops = {
+ .prepare = amdgpu_pmops_prepare,
+ .complete = amdgpu_pmops_complete,
+ .suspend = amdgpu_pmops_suspend,
++ .suspend_noirq = amdgpu_pmops_suspend_noirq,
+ .resume = amdgpu_pmops_resume,
+ .freeze = amdgpu_pmops_freeze,
+ .thaw = amdgpu_pmops_thaw,
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+index 8d682befe0d68..14499f0de32dc 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+@@ -552,9 +552,6 @@ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
+ if (!ring || !ring->fence_drv.initialized)
+ continue;
+
+- if (!ring->no_scheduler)
+- drm_sched_stop(&ring->sched, NULL);
+-
+ /* You can't wait for HW to signal if it's gone */
+ if (!drm_dev_is_unplugged(&adev->ddev))
+ r = amdgpu_fence_wait_empty(ring);
+@@ -614,11 +611,6 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
+ if (!ring || !ring->fence_drv.initialized)
+ continue;
+
+- if (!ring->no_scheduler) {
+- drm_sched_resubmit_jobs(&ring->sched);
+- drm_sched_start(&ring->sched, true);
+- }
+-
+ /* enable the interrupt */
+ if (ring->fence_drv.irq_src)
+ amdgpu_irq_get(adev, ring->fence_drv.irq_src,
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+index d6aa032890ee8..a1e63ba4c54a5 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+@@ -61,7 +61,7 @@ static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
+ }
+
+ ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
+- TTM_BO_VM_NUM_PREFAULT, 1);
++ TTM_BO_VM_NUM_PREFAULT);
+
+ drm_dev_exit(idx);
+ } else {
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+index 1916ec84dd71f..e7845df6cad22 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+@@ -266,7 +266,7 @@ static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
+ * adev->gfx.mec.num_pipe_per_mec
+ * adev->gfx.mec.num_queue_per_pipe;
+
+- while (queue_bit-- >= 0) {
++ while (--queue_bit >= 0) {
+ if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap))
+ continue;
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+index f3d62e196901a..0c7963dfacad1 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+@@ -223,7 +223,7 @@ int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
+ */
+ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
+ {
+- unsigned int count = AMDGPU_IH_MAX_NUM_IVS;
++ unsigned int count;
+ u32 wptr;
+
+ if (!ih->enabled || adev->shutdown)
+@@ -232,6 +232,7 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
+ wptr = amdgpu_ih_get_wptr(adev, ih);
+
+ restart_ih:
++ count = AMDGPU_IH_MAX_NUM_IVS;
+ DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, ih->rptr, wptr);
+
+ /* Order reading of wptr vs. reading of IH ring data */
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+index cc2e0c9cfe0a1..4f3c62adccbde 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+@@ -333,7 +333,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
+ if (!amdgpu_device_has_dc_support(adev)) {
+ if (!adev->enable_virtual_display)
+ /* Disable vblank IRQs aggressively for power-saving */
+- /* XXX: can this be enabled for DC? */
+ adev_to_drm(adev)->vblank_disable_immediate = true;
+
+ r = drm_vblank_init(adev_to_drm(adev), adev->mode_info.num_crtc);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+index 7e45640fbee02..6744427577b36 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+@@ -152,21 +152,10 @@ static void amdgpu_get_audio_func(struct amdgpu_device *adev)
+ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags)
+ {
+ struct drm_device *dev;
+- struct pci_dev *parent;
+ int r, acpi_status;
+
+ dev = adev_to_drm(adev);
+
+- if (amdgpu_has_atpx() &&
+- (amdgpu_is_atpx_hybrid() ||
+- amdgpu_has_atpx_dgpu_power_cntl()) &&
+- ((flags & AMD_IS_APU) == 0) &&
+- !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
+- flags |= AMD_IS_PX;
+-
+- parent = pci_upstream_bridge(adev->pdev);
+- adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
+-
+ /* amdgpu_device_init should report only fatal error
+ * like memory allocation failure or iomapping failure,
+ * or memory manager initialization failure, it must
+@@ -206,6 +195,12 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags)
+ adev->runpm = true;
+ break;
+ }
++ /* XXX: disable runtime pm if we are the primary adapter
++ * to avoid displays being re-enabled after DPMS.
++ * This needs to be sorted out and fixed properly.
++ */
++ if (adev->is_fw_fb)
++ adev->runpm = false;
+ if (adev->runpm)
+ dev_info(adev->dev, "Using BACO for runtime pm\n");
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+index 01a78c7865367..41d0b3a7b65d8 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+@@ -1038,29 +1038,6 @@ void amdgpu_bo_unpin(struct amdgpu_bo *bo)
+ }
+ }
+
+-/**
+- * amdgpu_bo_evict_vram - evict VRAM buffers
+- * @adev: amdgpu device object
+- *
+- * Evicts all VRAM buffers on the lru list of the memory type.
+- * Mainly used for evicting vram at suspend time.
+- *
+- * Returns:
+- * 0 for success or a negative error code on failure.
+- */
+-int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
+-{
+- struct ttm_resource_manager *man;
+-
+- if (adev->in_s3 && (adev->flags & AMD_IS_APU)) {
+- /* No need to evict vram on APUs for suspend to ram */
+- return 0;
+- }
+-
+- man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
+- return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
+-}
+-
+ static const char *amdgpu_vram_names[] = {
+ "UNKNOWN",
+ "GDDR1",
+@@ -1343,7 +1320,8 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
+ !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE))
+ return;
+
+- dma_resv_lock(bo->base.resv, NULL);
++ if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))
++ return;
+
+ r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence);
+ if (!WARN_ON(r)) {
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+index 9d6c001c15f89..d8ef8a53a562d 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+@@ -304,7 +304,6 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain);
+ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
+ u64 min_offset, u64 max_offset);
+ void amdgpu_bo_unpin(struct amdgpu_bo *bo);
+-int amdgpu_bo_evict_vram(struct amdgpu_device *adev);
+ int amdgpu_bo_init(struct amdgpu_device *adev);
+ void amdgpu_bo_fini(struct amdgpu_device *adev);
+ int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+index 9b41cb8c3de54..86e2090bbd6e0 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+@@ -2207,12 +2207,16 @@ static int psp_hw_start(struct psp_context *psp)
+ return ret;
+ }
+
++ if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
++ goto skip_pin_bo;
++
+ ret = psp_tmr_init(psp);
+ if (ret) {
+ DRM_ERROR("PSP tmr init failed!\n");
+ return ret;
+ }
+
++skip_pin_bo:
+ /*
+ * For ASICs with DF Cstate management centralized
+ * to PMFW, TMR setup should be performed after PMFW
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+index 94126dc396888..51c76d6322c94 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+@@ -1892,7 +1892,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
+ unsigned i;
+ int r;
+
+- if (direct_submit && !ring->sched.ready) {
++ if (!direct_submit && !ring->sched.ready) {
+ DRM_ERROR("Trying to move memory with ring turned off.\n");
+ return -EINVAL;
+ }
+@@ -2036,6 +2036,36 @@ error_free:
+ return r;
+ }
+
++/**
++ * amdgpu_ttm_evict_resources - evict memory buffers
++ * @adev: amdgpu device object
++ * @mem_type: evicted BO's memory type
++ *
++ * Evicts all @mem_type buffers on the lru list of the memory type.
++ *
++ * Returns:
++ * 0 for success or a negative error code on failure.
++ */
++int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
++{
++ struct ttm_resource_manager *man;
++
++ switch (mem_type) {
++ case TTM_PL_VRAM:
++ case TTM_PL_TT:
++ case AMDGPU_PL_GWS:
++ case AMDGPU_PL_GDS:
++ case AMDGPU_PL_OA:
++ man = ttm_manager_type(&adev->mman.bdev, mem_type);
++ break;
++ default:
++ DRM_ERROR("Trying to evict invalid memory type\n");
++ return -EINVAL;
++ }
++
++ return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
++}
++
+ #if defined(CONFIG_DEBUG_FS)
+
+ static int amdgpu_mm_vram_table_show(struct seq_file *m, void *unused)
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+index 3205fd5200601..639c7b41e30b9 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+@@ -190,6 +190,7 @@ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
+ uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem);
+ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
+ struct ttm_resource *mem);
++int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type);
+
+ void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+index ca058fbcccd43..16787c675f35e 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+@@ -24,6 +24,7 @@
+ #include <linux/module.h>
+
+ #include <drm/drm_drv.h>
++#include <xen/xen.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_ras.h"
+@@ -694,7 +695,8 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev)
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+
+ if (!reg) {
+- if (is_virtual_machine()) /* passthrough mode exclus sriov mod */
++ /* passthrough mode exclus sriov mod */
++ if (is_virtual_machine() && !xen_initial_domain())
+ adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
+ }
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+index ce982afeff913..7d58bf410be05 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+@@ -142,15 +142,16 @@ static void amdgpu_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
+ static void amdgpu_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
+ struct drm_atomic_state *state)
+ {
++ unsigned long flags;
+ if (crtc->state->event) {
+- spin_lock(&crtc->dev->event_lock);
++ spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+ if (drm_crtc_vblank_get(crtc) != 0)
+ drm_crtc_send_vblank_event(crtc, crtc->state->event);
+ else
+ drm_crtc_arm_vblank_event(crtc, crtc->state->event);
+
+- spin_unlock(&crtc->dev->event_lock);
++ spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+ crtc->state->event = NULL;
+ }
+@@ -504,8 +505,8 @@ static int amdgpu_vkms_sw_fini(void *handle)
+ int i = 0;
+
+ for (i = 0; i < adev->mode_info.num_crtc; i++)
+- if (adev->mode_info.crtcs[i])
+- hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
++ if (adev->amdgpu_vkms_output[i].vblank_hrtimer.function)
++ hrtimer_cancel(&adev->amdgpu_vkms_output[i].vblank_hrtimer);
+
+ kfree(adev->mode_info.bios_hardcoded_edid);
+ kfree(adev->amdgpu_vkms_output);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+index 6b15cad78de9d..fd37bb39774c8 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+@@ -768,11 +768,17 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+ * Check if all VM PDs/PTs are ready for updates
+ *
+ * Returns:
+- * True if eviction list is empty.
++ * True if VM is not evicting.
+ */
+ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
+ {
+- return list_empty(&vm->evicted);
++ bool ret;
++
++ amdgpu_vm_eviction_lock(vm);
++ ret = !vm->evicting;
++ amdgpu_vm_eviction_unlock(vm);
++
++ return ret && list_empty(&vm->evicted);
+ }
+
+ /**
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+index 978ac927ac11d..a799e0b1ff736 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+@@ -386,6 +386,7 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
+ "%s", "xgmi_hive_info");
+ if (ret) {
+ dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
++ kobject_put(&hive->kobj);
+ kfree(hive);
+ hive = NULL;
+ goto pro_end;
+diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
+index 54f28c075f214..f10ce740a29cc 100644
+--- a/drivers/gpu/drm/amd/amdgpu/cik.c
++++ b/drivers/gpu/drm/amd/amdgpu/cik.c
+@@ -1428,6 +1428,10 @@ static int cik_asic_reset(struct amdgpu_device *adev)
+ {
+ int r;
+
++ /* APUs don't have full asic reset */
++ if (adev->flags & AMD_IS_APU)
++ return 0;
++
+ if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
+ dev_info(adev->dev, "BACO reset\n");
+ r = amdgpu_dpm_baco_reset(adev);
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+index 16dbe593cba2e..970d59a21005a 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+@@ -7729,8 +7729,19 @@ static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+ switch (adev->asic_type) {
+ case CHIP_VANGOGH:
+ case CHIP_YELLOW_CARP:
+- clock = (uint64_t)RREG32_SOC15(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh) |
+- ((uint64_t)RREG32_SOC15(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh) << 32ULL);
++ preempt_disable();
++ clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh);
++ clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh);
++ hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Vangogh);
++ /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
++ * roughly every 42 seconds.
++ */
++ if (hi_check != clock_hi) {
++ clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Vangogh);
++ clock_hi = hi_check;
++ }
++ preempt_enable();
++ clock = clock_lo | (clock_hi << 32ULL);
+ break;
+ default:
+ preempt_disable();
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+index 025184a556ee6..db27fcf87cd04 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+@@ -140,6 +140,11 @@ MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
+ #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
+ #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
+
++#define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
++#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
++#define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
++#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
++
+ enum ta_ras_gfx_subblock {
+ /*CPC*/
+ TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
+@@ -1267,6 +1272,8 @@ static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
+ { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
+ /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
+ { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
++ /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
++ { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
+ { 0, 0, 0, 0, 0 },
+ };
+
+@@ -3056,8 +3063,8 @@ static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
+ AMD_PG_SUPPORT_CP |
+ AMD_PG_SUPPORT_GDS |
+ AMD_PG_SUPPORT_RLC_SMU_HS)) {
+- WREG32(mmRLC_JUMP_TABLE_RESTORE,
+- adev->gfx.rlc.cp_table_gpu_addr >> 8);
++ WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
++ adev->gfx.rlc.cp_table_gpu_addr >> 8);
+ gfx_v9_0_init_gfx_power_gating(adev);
+ }
+ }
+@@ -4228,19 +4235,38 @@ failed_kiq_read:
+
+ static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+ {
+- uint64_t clock;
++ uint64_t clock, clock_lo, clock_hi, hi_check;
+
+- amdgpu_gfx_off_ctrl(adev, false);
+- mutex_lock(&adev->gfx.gpu_clock_mutex);
+- if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
+- clock = gfx_v9_0_kiq_read_clock(adev);
+- } else {
+- WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+- clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
+- ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
++ switch (adev->asic_type) {
++ case CHIP_RENOIR:
++ preempt_disable();
++ clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
++ clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
++ hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
++ /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
++ * roughly every 42 seconds.
++ */
++ if (hi_check != clock_hi) {
++ clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
++ clock_hi = hi_check;
++ }
++ preempt_enable();
++ clock = clock_lo | (clock_hi << 32ULL);
++ break;
++ default:
++ amdgpu_gfx_off_ctrl(adev, false);
++ mutex_lock(&adev->gfx.gpu_clock_mutex);
++ if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
++ clock = gfx_v9_0_kiq_read_clock(adev);
++ } else {
++ WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
++ clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
++ ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
++ }
++ mutex_unlock(&adev->gfx.gpu_clock_mutex);
++ amdgpu_gfx_off_ctrl(adev, true);
++ break;
+ }
+- mutex_unlock(&adev->gfx.gpu_clock_mutex);
+- amdgpu_gfx_off_ctrl(adev, true);
+ return clock;
+ }
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+index bda1542ef1ddf..f51fd0688eca7 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+@@ -162,7 +162,6 @@ static void gfxhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC);/* XXX for emulation. */
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
+index 14c1c1a297dd3..6e0ace2fbfab1 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
+@@ -196,7 +196,6 @@ static void gfxhub_v2_0_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
+index 1a374ec0514a5..9328991e8807f 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
+@@ -197,7 +197,6 @@ static void gfxhub_v2_1_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+index e47104a1f5596..93a4da4284ede 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+@@ -788,7 +788,7 @@ static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
+ adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
+
+ #ifdef CONFIG_X86_64
+- if (adev->flags & AMD_IS_APU) {
++ if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
+ adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
+ adev->gmc.aper_size = adev->gmc.real_vram_size;
+ }
+@@ -1021,10 +1021,14 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)
+ return -EINVAL;
+ }
+
++ if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
++ goto skip_pin_bo;
++
+ r = amdgpu_gart_table_vram_pin(adev);
+ if (r)
+ return r;
+
++skip_pin_bo:
+ r = adev->gfxhub.funcs->gart_enable(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+index 0e81e03e9b498..0fe714f54cca9 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+@@ -841,12 +841,12 @@ static int gmc_v6_0_sw_init(void *handle)
+
+ adev->gmc.mc_mask = 0xffffffffffULL;
+
+- r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
++ r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));
+ if (r) {
+ dev_warn(adev->dev, "No suitable DMA available.\n");
+ return r;
+ }
+- adev->need_swiotlb = drm_need_swiotlb(44);
++ adev->need_swiotlb = drm_need_swiotlb(40);
+
+ r = gmc_v6_0_init_microcode(adev);
+ if (r) {
+diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+index 0a50fdaced7e5..63c47f61d0dfd 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+@@ -381,8 +381,9 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
+ adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
+
+ #ifdef CONFIG_X86_64
+- if (adev->flags & AMD_IS_APU &&
+- adev->gmc.real_vram_size > adev->gmc.aper_size) {
++ if ((adev->flags & AMD_IS_APU) &&
++ adev->gmc.real_vram_size > adev->gmc.aper_size &&
++ !amdgpu_passthrough(adev)) {
+ adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
+ adev->gmc.aper_size = adev->gmc.real_vram_size;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+index 492ebed2915be..bef9610084f10 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+@@ -515,10 +515,10 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
+ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
+ {
+ int r;
++ u32 tmp;
+
+ adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev);
+ if (!adev->gmc.vram_width) {
+- u32 tmp;
+ int chansize, numchan;
+
+ /* Get VRAM informations */
+@@ -562,8 +562,15 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
+ adev->gmc.vram_width = numchan * chansize;
+ }
+ /* size in MB on si */
+- adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+- adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
++ tmp = RREG32(mmCONFIG_MEMSIZE);
++ /* some boards may have garbage in the upper 16 bits */
++ if (tmp & 0xffff0000) {
++ DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
++ if (tmp & 0xffff)
++ tmp &= 0xffff;
++ }
++ adev->gmc.mc_vram_size = tmp * 1024ULL * 1024ULL;
++ adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
+
+ if (!(adev->flags & AMD_IS_APU)) {
+ r = amdgpu_device_resize_fb_bar(adev);
+@@ -574,7 +581,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
+ adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
+
+ #ifdef CONFIG_X86_64
+- if (adev->flags & AMD_IS_APU) {
++ if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
+ adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
+ adev->gmc.aper_size = adev->gmc.real_vram_size;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+index 5551359d5dfdc..0e731016921be 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+@@ -72,6 +72,9 @@
+ #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
+ #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
+
++#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea
++#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2
++
+
+ static const char *gfxhub_client_ids[] = {
+ "CB",
+@@ -1103,6 +1106,8 @@ static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
+ u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
+ unsigned size;
+
++ /* TODO move to DC so GMC doesn't need to hard-code DCN registers */
++
+ if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+ size = AMDGPU_VBIOS_VGA_ALLOCATION;
+ } else {
+@@ -1110,7 +1115,6 @@ static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
+
+ switch (adev->asic_type) {
+ case CHIP_RAVEN:
+- case CHIP_RENOIR:
+ viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
+ size = (REG_GET_FIELD(viewport,
+ HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
+@@ -1118,6 +1122,14 @@ static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
+ HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
+ 4);
+ break;
++ case CHIP_RENOIR:
++ viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
++ size = (REG_GET_FIELD(viewport,
++ HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
++ REG_GET_FIELD(viewport,
++ HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
++ 4);
++ break;
+ case CHIP_VEGA10:
+ case CHIP_VEGA12:
+ case CHIP_VEGA20:
+@@ -1375,7 +1387,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
+ */
+
+ /* check whether both host-gpu and gpu-gpu xgmi links exist */
+- if ((adev->flags & AMD_IS_APU) ||
++ if (((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
+ (adev->gmc.xgmi.supported &&
+ adev->gmc.xgmi.connected_to_cpu)) {
+ adev->gmc.aper_base =
+@@ -1640,7 +1652,7 @@ static int gmc_v9_0_sw_fini(void *handle)
+ amdgpu_gem_force_release(adev);
+ amdgpu_vm_manager_fini(adev);
+ amdgpu_gart_table_vram_free(adev);
+- amdgpu_bo_unref(&adev->gmc.pdb0_bo);
++ amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
+ amdgpu_bo_fini(adev);
+
+ return 0;
+@@ -1708,10 +1720,14 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
+ return -EINVAL;
+ }
+
++ if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
++ goto skip_pin_bo;
++
+ r = amdgpu_gart_table_vram_pin(adev);
+ if (r)
+ return r;
+
++skip_pin_bo:
+ r = adev->gfxhub.funcs->gart_enable(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+index a99953833820e..b3bede1dc41da 100644
+--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+@@ -145,7 +145,6 @@ static void mmhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC);/* XXX for emulation. */
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
+diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c
+index f80a14a1b82dc..f5f7181f9af5f 100644
+--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c
++++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c
+@@ -165,7 +165,6 @@ static void mmhub_v1_7_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC);/* XXX for emulation. */
+ tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
+diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
+index 7ded6b2f058ef..2e58ed2caa485 100644
+--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
+@@ -269,7 +269,6 @@ static void mmhub_v2_0_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
+index 88e457a150e02..c63b6b9349350 100644
+--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
++++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
+@@ -194,7 +194,6 @@ static void mmhub_v2_3_init_tlb_regs(struct amdgpu_device *adev)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c
+index c4ef822bbe8c5..ff49eeaf78824 100644
+--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c
++++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c
+@@ -189,8 +189,6 @@ static void mmhub_v9_4_init_tlb_regs(struct amdgpu_device *adev, int hubid)
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+- tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
+- ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC);/* XXX for emulation. */
+ tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+index 8931000dcd418..e37948c157692 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+@@ -2062,6 +2062,10 @@ static int sdma_v4_0_suspend(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ /* SMU saves SDMA state for us */
++ if (adev->in_s0ix)
++ return 0;
++
+ return sdma_v4_0_hw_fini(adev);
+ }
+
+@@ -2069,6 +2073,10 @@ static int sdma_v4_0_resume(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ /* SMU restores SDMA state for us */
++ if (adev->in_s0ix)
++ return 0;
++
+ return sdma_v4_0_hw_init(adev);
+ }
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
+index 0fc97c364fd76..6439d5c3d8d8b 100644
+--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
++++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
+@@ -607,8 +607,8 @@ soc15_asic_reset_method(struct amdgpu_device *adev)
+ static int soc15_asic_reset(struct amdgpu_device *adev)
+ {
+ /* original raven doesn't have full asic reset */
+- if ((adev->apu_flags & AMD_APU_IS_RAVEN) &&
+- !(adev->apu_flags & AMD_APU_IS_RAVEN2))
++ if ((adev->apu_flags & AMD_APU_IS_RAVEN) ||
++ (adev->apu_flags & AMD_APU_IS_RAVEN2))
+ return 0;
+
+ switch (soc15_asic_reset_method(adev)) {
+@@ -1273,8 +1273,11 @@ static int soc15_common_early_init(void *handle)
+ AMD_CG_SUPPORT_SDMA_LS |
+ AMD_CG_SUPPORT_VCN_MGCG;
+
++ /*
++ * MMHUB PG needs to be disabled for Picasso for
++ * stability reasons.
++ */
+ adev->pg_flags = AMD_PG_SUPPORT_SDMA |
+- AMD_PG_SUPPORT_MMHUB |
+ AMD_PG_SUPPORT_VCN;
+ } else {
+ adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
+diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
+index 7232241e3bfb2..0fef925b66024 100644
+--- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
++++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
+@@ -698,6 +698,19 @@ static int uvd_v3_1_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->uvd.idle_work);
++
++ if (RREG32(mmUVD_STATUS) != 0)
++ uvd_v3_1_stop(adev);
++
++ return 0;
++}
++
++static int uvd_v3_1_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -722,17 +735,6 @@ static int uvd_v3_1_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- if (RREG32(mmUVD_STATUS) != 0)
+- uvd_v3_1_stop(adev);
+-
+- return 0;
+-}
+-
+-static int uvd_v3_1_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = uvd_v3_1_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+index 52d6de969f462..c108b83817951 100644
+--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
++++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+@@ -212,6 +212,19 @@ static int uvd_v4_2_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->uvd.idle_work);
++
++ if (RREG32(mmUVD_STATUS) != 0)
++ uvd_v4_2_stop(adev);
++
++ return 0;
++}
++
++static int uvd_v4_2_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -236,17 +249,6 @@ static int uvd_v4_2_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- if (RREG32(mmUVD_STATUS) != 0)
+- uvd_v4_2_stop(adev);
+-
+- return 0;
+-}
+-
+-static int uvd_v4_2_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = uvd_v4_2_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+index db6d06758e4d4..563493d1f8306 100644
+--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+@@ -210,6 +210,19 @@ static int uvd_v5_0_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->uvd.idle_work);
++
++ if (RREG32(mmUVD_STATUS) != 0)
++ uvd_v5_0_stop(adev);
++
++ return 0;
++}
++
++static int uvd_v5_0_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -234,17 +247,6 @@ static int uvd_v5_0_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- if (RREG32(mmUVD_STATUS) != 0)
+- uvd_v5_0_stop(adev);
+-
+- return 0;
+-}
+-
+-static int uvd_v5_0_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = uvd_v5_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+index bc571833632ea..72f8762907681 100644
+--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+@@ -543,6 +543,19 @@ static int uvd_v6_0_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->uvd.idle_work);
++
++ if (RREG32(mmUVD_STATUS) != 0)
++ uvd_v6_0_stop(adev);
++
++ return 0;
++}
++
++static int uvd_v6_0_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -567,17 +580,6 @@ static int uvd_v6_0_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- if (RREG32(mmUVD_STATUS) != 0)
+- uvd_v6_0_stop(adev);
+-
+- return 0;
+-}
+-
+-static int uvd_v6_0_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = uvd_v6_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+index b6e82d75561f6..1fd9ca21a091b 100644
+--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+@@ -606,6 +606,23 @@ static int uvd_v7_0_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->uvd.idle_work);
++
++ if (!amdgpu_sriov_vf(adev))
++ uvd_v7_0_stop(adev);
++ else {
++ /* full access mode, so don't touch any UVD register */
++ DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
++ }
++
++ return 0;
++}
++
++static int uvd_v7_0_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -630,21 +647,6 @@ static int uvd_v7_0_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- if (!amdgpu_sriov_vf(adev))
+- uvd_v7_0_stop(adev);
+- else {
+- /* full access mode, so don't touch any UVD register */
+- DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
+- }
+-
+- return 0;
+-}
+-
+-static int uvd_v7_0_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = uvd_v7_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+index b70c17f0c52e8..98952fd387e73 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+@@ -479,6 +479,17 @@ static int vce_v2_0_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->vce.idle_work);
++
++ return 0;
++}
++
++static int vce_v2_0_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -502,14 +513,6 @@ static int vce_v2_0_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- return 0;
+-}
+-
+-static int vce_v2_0_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = vce_v2_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+index 9de66893ccd6d..8fb5df7181e09 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+@@ -490,6 +490,21 @@ static int vce_v3_0_hw_fini(void *handle)
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
++ cancel_delayed_work_sync(&adev->vce.idle_work);
++
++ r = vce_v3_0_wait_for_idle(handle);
++ if (r)
++ return r;
++
++ vce_v3_0_stop(adev);
++ return vce_v3_0_set_clockgating_state(adev, AMD_CG_STATE_GATE);
++}
++
++static int vce_v3_0_suspend(void *handle)
++{
++ int r;
++ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++
+ /*
+ * Proper cleanups before halting the HW engine:
+ * - cancel the delayed idle work
+@@ -513,19 +528,6 @@ static int vce_v3_0_hw_fini(void *handle)
+ AMD_CG_STATE_GATE);
+ }
+
+- r = vce_v3_0_wait_for_idle(handle);
+- if (r)
+- return r;
+-
+- vce_v3_0_stop(adev);
+- return vce_v3_0_set_clockgating_state(adev, AMD_CG_STATE_GATE);
+-}
+-
+-static int vce_v3_0_suspend(void *handle)
+-{
+- int r;
+- struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+-
+ r = vce_v3_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+index fec902b800c28..70b8c88d30513 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+@@ -542,29 +542,8 @@ static int vce_v4_0_hw_fini(void *handle)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+- /*
+- * Proper cleanups before halting the HW engine:
+- * - cancel the delayed idle work
+- * - enable powergating
+- * - enable clockgating
+- * - disable dpm
+- *
+- * TODO: to align with the VCN implementation, move the
+- * jobs for clockgating/powergating/dpm setting to
+- * ->set_powergating_state().
+- */
+ cancel_delayed_work_sync(&adev->vce.idle_work);
+
+- if (adev->pm.dpm_enabled) {
+- amdgpu_dpm_enable_vce(adev, false);
+- } else {
+- amdgpu_asic_set_vce_clocks(adev, 0, 0);
+- amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+- AMD_PG_STATE_GATE);
+- amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+- AMD_CG_STATE_GATE);
+- }
+-
+ if (!amdgpu_sriov_vf(adev)) {
+ /* vce_v4_0_wait_for_idle(handle); */
+ vce_v4_0_stop(adev);
+@@ -594,6 +573,29 @@ static int vce_v4_0_suspend(void *handle)
+ drm_dev_exit(idx);
+ }
+
++ /*
++ * Proper cleanups before halting the HW engine:
++ * - cancel the delayed idle work
++ * - enable powergating
++ * - enable clockgating
++ * - disable dpm
++ *
++ * TODO: to align with the VCN implementation, move the
++ * jobs for clockgating/powergating/dpm setting to
++ * ->set_powergating_state().
++ */
++ cancel_delayed_work_sync(&adev->vce.idle_work);
++
++ if (adev->pm.dpm_enabled) {
++ amdgpu_dpm_enable_vce(adev, false);
++ } else {
++ amdgpu_asic_set_vce_clocks(adev, 0, 0);
++ amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
++ AMD_PG_STATE_GATE);
++ amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
++ AMD_CG_STATE_GATE);
++ }
++
+ r = vce_v4_0_hw_fini(adev);
+ if (r)
+ return r;
+diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+index 121ee9f2b8d16..462008d506904 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+@@ -253,6 +253,13 @@ static int vcn_v1_0_suspend(void *handle)
+ {
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
++ bool idle_work_unexecuted;
++
++ idle_work_unexecuted = cancel_delayed_work_sync(&adev->vcn.idle_work);
++ if (idle_work_unexecuted) {
++ if (adev->pm.dpm_enabled)
++ amdgpu_dpm_enable_uvd(adev, false);
++ }
+
+ r = vcn_v1_0_hw_fini(adev);
+ if (r)
+diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+index f4686e918e0d1..c405075a572c1 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+@@ -22,6 +22,7 @@
+ */
+
+ #include <linux/firmware.h>
++#include <drm/drm_drv.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_vcn.h"
+@@ -192,11 +193,14 @@ static int vcn_v2_0_sw_init(void *handle)
+ */
+ static int vcn_v2_0_sw_fini(void *handle)
+ {
+- int r;
++ int r, idx;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared_cpu_addr;
+
+- fw_shared->present_flag_0 = 0;
++ if (drm_dev_enter(&adev->ddev, &idx)) {
++ fw_shared->present_flag_0 = 0;
++ drm_dev_exit(idx);
++ }
+
+ amdgpu_virt_free_mm_table(adev);
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+index e0c0c3734432e..a0956d8623770 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
++++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+@@ -22,6 +22,7 @@
+ */
+
+ #include <linux/firmware.h>
++#include <drm/drm_drv.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_vcn.h"
+@@ -233,17 +234,21 @@ static int vcn_v2_5_sw_init(void *handle)
+ */
+ static int vcn_v2_5_sw_fini(void *handle)
+ {
+- int i, r;
++ int i, r, idx;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ volatile struct amdgpu_fw_shared *fw_shared;
+
+- for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+- if (adev->vcn.harvest_config & (1 << i))
+- continue;
+- fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
+- fw_shared->present_flag_0 = 0;
++ if (drm_dev_enter(&adev->ddev, &idx)) {
++ for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
++ if (adev->vcn.harvest_config & (1 << i))
++ continue;
++ fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
++ fw_shared->present_flag_0 = 0;
++ }
++ drm_dev_exit(idx);
+ }
+
++
+ if (amdgpu_sriov_vf(adev))
+ amdgpu_virt_free_mm_table(adev);
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+index 3d18aab88b4e2..6e56bef4fdf81 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+@@ -601,8 +601,8 @@ static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx
+ AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
+
+ /* VCN global tiling registers */
+- WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
+- UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
++ WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
++ UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
+ }
+
+ static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
+@@ -1508,8 +1508,11 @@ static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
+
+ static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
+ {
++ struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
+ uint32_t tmp;
+
++ vcn_v3_0_pause_dpg_mode(adev, 0, &state);
++
+ /* Wait for power status to be 1 */
+ SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
+index fe9a7cc8d9eb0..6645ebbd2696c 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vi.c
++++ b/drivers/gpu/drm/amd/amdgpu/vi.c
+@@ -956,6 +956,10 @@ static int vi_asic_reset(struct amdgpu_device *adev)
+ {
+ int r;
+
++ /* APUs don't have full asic reset */
++ if (adev->flags & AMD_IS_APU)
++ return 0;
++
+ if (vi_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
+ dev_info(adev->dev, "BACO reset\n");
+ r = amdgpu_dpm_baco_reset(adev);
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+index 86afd37b098d6..6688129df240e 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+@@ -1807,13 +1807,9 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
+ if (!args->start_addr || !args->size)
+ return -EINVAL;
+
+- mutex_lock(&p->mutex);
+-
+ r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
+ args->attrs);
+
+- mutex_unlock(&p->mutex);
+-
+ return r;
+ }
+ #else
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+index cfedfb1e8596c..e574aa32a111d 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+@@ -1060,6 +1060,9 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
+ return -ENODEV;
+ /* same everything but the other direction */
+ props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
++ if (!props2)
++ return -ENOMEM;
++
+ props2->node_from = id_to;
+ props2->node_to = id_from;
+ props2->kobj = NULL;
+@@ -1560,7 +1563,7 @@ int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
+ /* Fetch the CRAT table from ACPI */
+ status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
+ if (status == AE_NOT_FOUND) {
+- pr_warn("CRAT table not found\n");
++ pr_info("CRAT table not found\n");
+ return -ENODATA;
+ } else if (ACPI_FAILURE(status)) {
+ const char *err = acpi_format_exception(status);
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+index 4a416231b24c8..660eb7097cfc0 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+@@ -834,15 +834,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ }
+
+ /* Verify module parameters regarding mapped process number*/
+- if ((hws_max_conc_proc < 0)
+- || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
+- dev_err(kfd_device,
+- "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
+- hws_max_conc_proc, kfd->vm_info.vmid_num_kfd,
+- kfd->vm_info.vmid_num_kfd);
++ if (hws_max_conc_proc >= 0)
++ kfd->max_proc_per_quantum = min((u32)hws_max_conc_proc, kfd->vm_info.vmid_num_kfd);
++ else
+ kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
+- } else
+- kfd->max_proc_per_quantum = hws_max_conc_proc;
+
+ /* calculate max size of mqds needed for queues */
+ size = max_num_of_queues_per_device *
+@@ -916,6 +911,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ kfd_double_confirm_iommu_support(kfd);
+
+ if (kfd_iommu_device_init(kfd)) {
++ kfd->use_iommu_v2 = false;
+ dev_err(kfd_device, "Error initializing iommuv2\n");
+ goto device_iommu_error;
+ }
+@@ -924,6 +920,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+
+ svm_migrate_init((struct amdgpu_device *)kfd->kgd);
+
++ if(kgd2kfd_resume_iommu(kfd))
++ goto device_iommu_error;
++
+ if (kfd_resume(kfd))
+ goto kfd_resume_error;
+
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+index f8fce9d05f50c..442857f3bde77 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+@@ -138,19 +138,33 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,
+ }
+
+ static void increment_queue_count(struct device_queue_manager *dqm,
+- enum kfd_queue_type type)
++ struct qcm_process_device *qpd,
++ struct queue *q)
+ {
+ dqm->active_queue_count++;
+- if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
++ q->properties.type == KFD_QUEUE_TYPE_DIQ)
+ dqm->active_cp_queue_count++;
++
++ if (q->properties.is_gws) {
++ dqm->gws_queue_count++;
++ qpd->mapped_gws_queue = true;
++ }
+ }
+
+ static void decrement_queue_count(struct device_queue_manager *dqm,
+- enum kfd_queue_type type)
++ struct qcm_process_device *qpd,
++ struct queue *q)
+ {
+ dqm->active_queue_count--;
+- if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
++ q->properties.type == KFD_QUEUE_TYPE_DIQ)
+ dqm->active_cp_queue_count--;
++
++ if (q->properties.is_gws) {
++ dqm->gws_queue_count--;
++ qpd->mapped_gws_queue = false;
++ }
+ }
+
+ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
+@@ -390,7 +404,7 @@ add_queue_to_list:
+ list_add(&q->list, &qpd->queues_list);
+ qpd->queue_count++;
+ if (q->properties.is_active)
+- increment_queue_count(dqm, q->properties.type);
++ increment_queue_count(dqm, qpd, q);
+
+ /*
+ * Unconditionally increment this counter, regardless of the queue's
+@@ -515,13 +529,8 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
+ deallocate_vmid(dqm, qpd, q);
+ }
+ qpd->queue_count--;
+- if (q->properties.is_active) {
+- decrement_queue_count(dqm, q->properties.type);
+- if (q->properties.is_gws) {
+- dqm->gws_queue_count--;
+- qpd->mapped_gws_queue = false;
+- }
+- }
++ if (q->properties.is_active)
++ decrement_queue_count(dqm, qpd, q);
+
+ return retval;
+ }
+@@ -613,12 +622,11 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
+ * dqm->active_queue_count to determine whether a new runlist must be
+ * uploaded.
+ */
+- if (q->properties.is_active && !prev_active)
+- increment_queue_count(dqm, q->properties.type);
+- else if (!q->properties.is_active && prev_active)
+- decrement_queue_count(dqm, q->properties.type);
+-
+- if (q->gws && !q->properties.is_gws) {
++ if (q->properties.is_active && !prev_active) {
++ increment_queue_count(dqm, &pdd->qpd, q);
++ } else if (!q->properties.is_active && prev_active) {
++ decrement_queue_count(dqm, &pdd->qpd, q);
++ } else if (q->gws && !q->properties.is_gws) {
+ if (q->properties.is_active) {
+ dqm->gws_queue_count++;
+ pdd->qpd.mapped_gws_queue = true;
+@@ -680,11 +688,7 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
+ q->properties.is_active = false;
+- decrement_queue_count(dqm, q->properties.type);
+- if (q->properties.is_gws) {
+- dqm->gws_queue_count--;
+- qpd->mapped_gws_queue = false;
+- }
++ decrement_queue_count(dqm, qpd, q);
+
+ if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
+ continue;
+@@ -730,7 +734,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
+ continue;
+
+ q->properties.is_active = false;
+- decrement_queue_count(dqm, q->properties.type);
++ decrement_queue_count(dqm, qpd, q);
+ }
+ pdd->last_evict_timestamp = get_jiffies_64();
+ retval = execute_queues_cpsch(dqm,
+@@ -801,11 +805,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
+ q->properties.is_active = true;
+- increment_queue_count(dqm, q->properties.type);
+- if (q->properties.is_gws) {
+- dqm->gws_queue_count++;
+- qpd->mapped_gws_queue = true;
+- }
++ increment_queue_count(dqm, qpd, q);
+
+ if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
+ continue;
+@@ -863,7 +863,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
+ continue;
+
+ q->properties.is_active = true;
+- increment_queue_count(dqm, q->properties.type);
++ increment_queue_count(dqm, &pdd->qpd, q);
+ }
+ retval = execute_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+@@ -1225,6 +1225,11 @@ static int stop_cpsch(struct device_queue_manager *dqm)
+ bool hanging;
+
+ dqm_lock(dqm);
++ if (!dqm->sched_running) {
++ dqm_unlock(dqm);
++ return 0;
++ }
++
+ if (!dqm->is_hws_hang)
+ unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+ hanging = dqm->is_hws_hang || dqm->is_resetting;
+@@ -1260,7 +1265,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
+ dqm->total_queue_count);
+
+ list_add(&kq->list, &qpd->priv_queue_list);
+- increment_queue_count(dqm, kq->queue->properties.type);
++ increment_queue_count(dqm, qpd, kq->queue);
+ qpd->is_debug = true;
+ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+ dqm_unlock(dqm);
+@@ -1274,7 +1279,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
+ {
+ dqm_lock(dqm);
+ list_del(&kq->list);
+- decrement_queue_count(dqm, kq->queue->properties.type);
++ decrement_queue_count(dqm, qpd, kq->queue);
+ qpd->is_debug = false;
+ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+ /*
+@@ -1341,7 +1346,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
+ qpd->queue_count++;
+
+ if (q->properties.is_active) {
+- increment_queue_count(dqm, q->properties.type);
++ increment_queue_count(dqm, qpd, q);
+
+ execute_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+@@ -1543,15 +1548,11 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
+ list_del(&q->list);
+ qpd->queue_count--;
+ if (q->properties.is_active) {
+- decrement_queue_count(dqm, q->properties.type);
++ decrement_queue_count(dqm, qpd, q);
+ retval = execute_queues_cpsch(dqm,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+ if (retval == -ETIME)
+ qpd->reset_wavefronts = true;
+- if (q->properties.is_gws) {
+- dqm->gws_queue_count--;
+- qpd->mapped_gws_queue = false;
+- }
+ }
+
+ /*
+@@ -1742,7 +1743,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
+ /* Clean all kernel queues */
+ list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
+ list_del(&kq->list);
+- decrement_queue_count(dqm, kq->queue->properties.type);
++ decrement_queue_count(dqm, qpd, kq->queue);
+ qpd->is_debug = false;
+ dqm->total_queue_count--;
+ filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
+@@ -1755,13 +1756,8 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
+ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ deallocate_sdma_queue(dqm, q);
+
+- if (q->properties.is_active) {
+- decrement_queue_count(dqm, q->properties.type);
+- if (q->properties.is_gws) {
+- dqm->gws_queue_count--;
+- qpd->mapped_gws_queue = false;
+- }
+- }
++ if (q->properties.is_active)
++ decrement_queue_count(dqm, qpd, q);
+
+ dqm->total_queue_count--;
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+index 3eea4edee355d..b8bdd796cd911 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+@@ -531,6 +531,8 @@ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
+ event_waiters = kmalloc_array(num_events,
+ sizeof(struct kfd_event_waiter),
+ GFP_KERNEL);
++ if (!event_waiters)
++ return NULL;
+
+ for (i = 0; (event_waiters) && (i < num_events) ; i++) {
+ init_wait(&event_waiters[i].wait);
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
+index ed4bc5f844ce7..766b3660c8c86 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
+@@ -270,15 +270,6 @@ int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
+ return ret;
+ }
+
+- ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client,
+- O_RDWR);
+- if (ret < 0) {
+- kfifo_free(&client->fifo);
+- kfree(client);
+- return ret;
+- }
+- *fd = ret;
+-
+ init_waitqueue_head(&client->wait_queue);
+ spin_lock_init(&client->lock);
+ client->events = 0;
+@@ -288,5 +279,20 @@ int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
+ list_add_rcu(&client->list, &dev->smi_clients);
+ spin_unlock(&dev->smi_lock);
+
++ ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client,
++ O_RDWR);
++ if (ret < 0) {
++ spin_lock(&dev->smi_lock);
++ list_del_rcu(&client->list);
++ spin_unlock(&dev->smi_lock);
++
++ synchronize_rcu();
++
++ kfifo_free(&client->fifo);
++ kfree(client);
++ return ret;
++ }
++ *fd = ret;
++
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+index 9d0f65a90002d..830809b694dd9 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+@@ -936,7 +936,7 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
+ }
+
+ static int
+-svm_range_split_tail(struct svm_range *prange, struct svm_range *new,
++svm_range_split_tail(struct svm_range *prange,
+ uint64_t new_last, struct list_head *insert_list)
+ {
+ struct svm_range *tail;
+@@ -948,7 +948,7 @@ svm_range_split_tail(struct svm_range *prange, struct svm_range *new,
+ }
+
+ static int
+-svm_range_split_head(struct svm_range *prange, struct svm_range *new,
++svm_range_split_head(struct svm_range *prange,
+ uint64_t new_start, struct list_head *insert_list)
+ {
+ struct svm_range *head;
+@@ -1307,7 +1307,7 @@ struct svm_validate_context {
+ struct svm_range *prange;
+ bool intr;
+ unsigned long bitmap[MAX_GPU_INSTANCE];
+- struct ttm_validate_buffer tv[MAX_GPU_INSTANCE+1];
++ struct ttm_validate_buffer tv[MAX_GPU_INSTANCE];
+ struct list_head validate_list;
+ struct ww_acquire_ctx ticket;
+ };
+@@ -1334,11 +1334,6 @@ static int svm_range_reserve_bos(struct svm_validate_context *ctx)
+ ctx->tv[gpuidx].num_shared = 4;
+ list_add(&ctx->tv[gpuidx].head, &ctx->validate_list);
+ }
+- if (ctx->prange->svm_bo && ctx->prange->ttm_res) {
+- ctx->tv[MAX_GPU_INSTANCE].bo = &ctx->prange->svm_bo->bo->tbo;
+- ctx->tv[MAX_GPU_INSTANCE].num_shared = 1;
+- list_add(&ctx->tv[MAX_GPU_INSTANCE].head, &ctx->validate_list);
+- }
+
+ r = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->validate_list,
+ ctx->intr, NULL);
+@@ -1570,7 +1565,6 @@ retry_flush_work:
+ static void svm_range_restore_work(struct work_struct *work)
+ {
+ struct delayed_work *dwork = to_delayed_work(work);
+- struct amdkfd_process_info *process_info;
+ struct svm_range_list *svms;
+ struct svm_range *prange;
+ struct kfd_process *p;
+@@ -1590,12 +1584,10 @@ static void svm_range_restore_work(struct work_struct *work)
+ * the lifetime of this thread, kfd_process and mm will be valid.
+ */
+ p = container_of(svms, struct kfd_process, svms);
+- process_info = p->kgd_process_info;
+ mm = p->mm;
+ if (!mm)
+ return;
+
+- mutex_lock(&process_info->lock);
+ svm_range_list_lock_and_flush_work(svms, mm);
+ mutex_lock(&svms->lock);
+
+@@ -1648,7 +1640,6 @@ static void svm_range_restore_work(struct work_struct *work)
+ out_reschedule:
+ mutex_unlock(&svms->lock);
+ mmap_write_unlock(mm);
+- mutex_unlock(&process_info->lock);
+
+ /* If validation failed, reschedule another attempt */
+ if (evicted_ranges) {
+@@ -1764,49 +1755,54 @@ static struct svm_range *svm_range_clone(struct svm_range *old)
+ }
+
+ /**
+- * svm_range_handle_overlap - split overlap ranges
+- * @svms: svm range list header
+- * @new: range added with this attributes
+- * @start: range added start address, in pages
+- * @last: range last address, in pages
+- * @update_list: output, the ranges attributes are updated. For set_attr, this
+- * will do validation and map to GPUs. For unmap, this will be
+- * removed and unmap from GPUs
+- * @insert_list: output, the ranges will be inserted into svms, attributes are
+- * not changes. For set_attr, this will add into svms.
+- * @remove_list:output, the ranges will be removed from svms
+- * @left: the remaining range after overlap, For set_attr, this will be added
+- * as new range.
++ * svm_range_add - add svm range and handle overlap
++ * @p: the range add to this process svms
++ * @start: page size aligned
++ * @size: page size aligned
++ * @nattr: number of attributes
++ * @attrs: array of attributes
++ * @update_list: output, the ranges need validate and update GPU mapping
++ * @insert_list: output, the ranges need insert to svms
++ * @remove_list: output, the ranges are replaced and need remove from svms
+ *
+- * Total have 5 overlap cases.
++ * Check if the virtual address range has overlap with any existing ranges,
++ * split partly overlapping ranges and add new ranges in the gaps. All changes
++ * should be applied to the range_list and interval tree transactionally. If
++ * any range split or allocation fails, the entire update fails. Therefore any
++ * existing overlapping svm_ranges are cloned and the original svm_ranges left
++ * unchanged.
+ *
+- * This function handles overlap of an address interval with existing
+- * struct svm_ranges for applying new attributes. This may require
+- * splitting existing struct svm_ranges. All changes should be applied to
+- * the range_list and interval tree transactionally. If any split operation
+- * fails, the entire update fails. Therefore the existing overlapping
+- * svm_ranges are cloned and the original svm_ranges left unchanged. If the
+- * transaction succeeds, the modified clones are added and the originals
+- * freed. Otherwise the clones are removed and the old svm_ranges remain.
++ * If the transaction succeeds, the caller can update and insert clones and
++ * new ranges, then free the originals.
+ *
+- * Context: The caller must hold svms->lock
++ * Otherwise the caller can free the clones and new ranges, while the old
++ * svm_ranges remain unchanged.
++ *
++ * Context: Process context, caller must hold svms->lock
++ *
++ * Return:
++ * 0 - OK, otherwise error code
+ */
+ static int
+-svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new,
+- unsigned long start, unsigned long last,
+- struct list_head *update_list,
+- struct list_head *insert_list,
+- struct list_head *remove_list,
+- unsigned long *left)
++svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
++ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
++ struct list_head *update_list, struct list_head *insert_list,
++ struct list_head *remove_list)
+ {
++ unsigned long last = start + size - 1UL;
++ struct svm_range_list *svms = &p->svms;
+ struct interval_tree_node *node;
++ struct svm_range new = {0};
+ struct svm_range *prange;
+ struct svm_range *tmp;
+ int r = 0;
+
++ pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last);
++
+ INIT_LIST_HEAD(update_list);
+ INIT_LIST_HEAD(insert_list);
+ INIT_LIST_HEAD(remove_list);
++ svm_range_apply_attrs(p, &new, nattr, attrs);
+
+ node = interval_tree_iter_first(&svms->objects, start, last);
+ while (node) {
+@@ -1834,14 +1830,14 @@ svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new,
+
+ if (node->start < start) {
+ pr_debug("change old range start\n");
+- r = svm_range_split_head(prange, new, start,
++ r = svm_range_split_head(prange, start,
+ insert_list);
+ if (r)
+ goto out;
+ }
+ if (node->last > last) {
+ pr_debug("change old range last\n");
+- r = svm_range_split_tail(prange, new, last,
++ r = svm_range_split_tail(prange, last,
+ insert_list);
+ if (r)
+ goto out;
+@@ -1853,7 +1849,7 @@ svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new,
+ prange = old;
+ }
+
+- if (!svm_range_is_same_attrs(prange, new))
++ if (!svm_range_is_same_attrs(prange, &new))
+ list_add(&prange->update_list, update_list);
+
+ /* insert a new node if needed */
+@@ -1873,8 +1869,16 @@ svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new,
+ start = next_start;
+ }
+
+- if (left && start <= last)
+- *left = last - start + 1;
++ /* add a final range at the end if needed */
++ if (start <= last) {
++ prange = svm_range_new(svms, start, last);
++ if (!prange) {
++ r = -ENOMEM;
++ goto out;
++ }
++ list_add(&prange->insert_list, insert_list);
++ list_add(&prange->update_list, update_list);
++ }
+
+ out:
+ if (r)
+@@ -2702,59 +2706,6 @@ svm_range_is_valid(struct mm_struct *mm, uint64_t start, uint64_t size)
+ return true;
+ }
+
+-/**
+- * svm_range_add - add svm range and handle overlap
+- * @p: the range add to this process svms
+- * @start: page size aligned
+- * @size: page size aligned
+- * @nattr: number of attributes
+- * @attrs: array of attributes
+- * @update_list: output, the ranges need validate and update GPU mapping
+- * @insert_list: output, the ranges need insert to svms
+- * @remove_list: output, the ranges are replaced and need remove from svms
+- *
+- * Check if the virtual address range has overlap with the registered ranges,
+- * split the overlapped range, copy and adjust pages address and vram nodes in
+- * old and new ranges.
+- *
+- * Context: Process context, caller must hold svms->lock
+- *
+- * Return:
+- * 0 - OK, otherwise error code
+- */
+-static int
+-svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
+- uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
+- struct list_head *update_list, struct list_head *insert_list,
+- struct list_head *remove_list)
+-{
+- uint64_t last = start + size - 1UL;
+- struct svm_range_list *svms;
+- struct svm_range new = {0};
+- struct svm_range *prange;
+- unsigned long left = 0;
+- int r = 0;
+-
+- pr_debug("svms 0x%p [0x%llx 0x%llx]\n", &p->svms, start, last);
+-
+- svm_range_apply_attrs(p, &new, nattr, attrs);
+-
+- svms = &p->svms;
+-
+- r = svm_range_handle_overlap(svms, &new, start, last, update_list,
+- insert_list, remove_list, &left);
+- if (r)
+- return r;
+-
+- if (left) {
+- prange = svm_range_new(svms, last - left + 1, last);
+- list_add(&prange->insert_list, insert_list);
+- list_add(&prange->update_list, update_list);
+- }
+-
+- return 0;
+-}
+-
+ /**
+ * svm_range_best_prefetch_location - decide the best prefetch location
+ * @prange: svm range structure
+@@ -2979,7 +2930,6 @@ static int
+ svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size,
+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
+ {
+- struct amdkfd_process_info *process_info = p->kgd_process_info;
+ struct mm_struct *mm = current->mm;
+ struct list_head update_list;
+ struct list_head insert_list;
+@@ -2998,8 +2948,6 @@ svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size,
+
+ svms = &p->svms;
+
+- mutex_lock(&process_info->lock);
+-
+ svm_range_list_lock_and_flush_work(svms, mm);
+
+ if (!svm_range_is_valid(mm, start, size)) {
+@@ -3075,8 +3023,6 @@ out_unlock_range:
+ mutex_unlock(&svms->lock);
+ mmap_read_unlock(mm);
+ out:
+- mutex_unlock(&process_info->lock);
+-
+ pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid,
+ &p->svms, start, start + size - 1, r);
+
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+index 1ea31dcc7a8b0..ec75613618b19 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+@@ -1141,8 +1141,15 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
+ case CHIP_RAVEN:
+ case CHIP_RENOIR:
+ init_data.flags.gpu_vm_support = true;
+- if (ASICREV_IS_GREEN_SARDINE(adev->external_rev_id))
++ switch (adev->dm.dmcub_fw_version) {
++ case 0: /* development */
++ case 0x1: /* linux-firmware.git hash 6d9f399 */
++ case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */
++ init_data.flags.disable_dmcu = false;
++ break;
++ default:
+ init_data.flags.disable_dmcu = true;
++ }
+ break;
+ case CHIP_VANGOGH:
+ case CHIP_YELLOW_CARP:
+@@ -2206,6 +2213,9 @@ static int dm_resume(void *handle)
+ if (amdgpu_in_reset(adev)) {
+ dc_state = dm->cached_dc_state;
+
++ if (dc_enable_dmub_notifications(adev->dm.dc))
++ amdgpu_dm_outbox_init(adev);
++
+ r = dm_dmub_hw_init(adev);
+ if (r)
+ DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r);
+@@ -2217,8 +2227,8 @@ static int dm_resume(void *handle)
+
+ for (i = 0; i < dc_state->stream_count; i++) {
+ dc_state->streams[i]->mode_changed = true;
+- for (j = 0; j < dc_state->stream_status->plane_count; j++) {
+- dc_state->stream_status->plane_states[j]->update_flags.raw
++ for (j = 0; j < dc_state->stream_status[i].plane_count; j++) {
++ dc_state->stream_status[i].plane_states[j]->update_flags.raw
+ = 0xffffffff;
+ }
+ }
+@@ -2253,6 +2263,10 @@ static int dm_resume(void *handle)
+ /* TODO: Remove dc_state->dccg, use dc->dccg directly. */
+ dc_resource_state_construct(dm->dc, dm_state->context);
+
++ /* Re-enable outbox interrupts for DPIA. */
++ if (dc_enable_dmub_notifications(adev->dm.dc))
++ amdgpu_dm_outbox_init(adev);
++
+ /* Before powering on DC we need to re-initialize DMUB. */
+ r = dm_dmub_hw_init(adev);
+ if (r)
+@@ -2282,7 +2296,8 @@ static int dm_resume(void *handle)
+ * this is the case when traversing through already created
+ * MST connectors, should be skipped
+ */
+- if (aconnector->mst_port)
++ if (aconnector->dc_link &&
++ aconnector->dc_link->type == dc_connection_mst_branch)
+ continue;
+
+ mutex_lock(&aconnector->hpd_lock);
+@@ -3213,7 +3228,7 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev)
+
+ /* Use GRPH_PFLIP interrupt */
+ for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT;
+- i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + adev->mode_info.num_crtc - 1;
++ i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1;
+ i++) {
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq);
+ if (r) {
+@@ -3508,7 +3523,7 @@ static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *cap
+ max - min);
+ }
+
+-static int amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
++static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
+ int bl_idx,
+ u32 user_brightness)
+ {
+@@ -3536,7 +3551,8 @@ static int amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
+ DRM_DEBUG("DM: Failed to update backlight on eDP[%d]\n", bl_idx);
+ }
+
+- return rc ? 0 : 1;
++ if (rc)
++ dm->actual_brightness[bl_idx] = user_brightness;
+ }
+
+ static int amdgpu_dm_backlight_update_status(struct backlight_device *bd)
+@@ -3798,6 +3814,9 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
+ }
+ #endif
+
++ /* Disable vblank IRQs aggressively for power-saving. */
++ adev_to_drm(adev)->vblank_disable_immediate = true;
++
+ /* loops over all connectors on the board */
+ for (i = 0; i < link_cnt; i++) {
+ struct dc_link *link = NULL;
+@@ -3839,8 +3858,17 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
+ } else if (dc_link_detect(link, DETECT_REASON_BOOT)) {
+ amdgpu_dm_update_connector_after_detect(aconnector);
+ register_backlight_device(dm, link);
++
++ if (dm->num_of_edps)
++ update_connector_ext_caps(aconnector);
+ if (amdgpu_dc_feature_mask & DC_PSR_MASK)
+ amdgpu_dm_set_psr_caps(link);
++
++ /* TODO: Fix vblank control helpers to delay PSR entry to allow this when
++ * PSR is also supported.
++ */
++ if (link->psr_settings.psr_feature_enabled)
++ adev_to_drm(adev)->vblank_disable_immediate = false;
+ }
+
+
+@@ -5587,6 +5615,7 @@ static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
+ struct dsc_dec_dpcd_caps *dsc_caps)
+ {
+ stream->timing.flags.DSC = 0;
++ dsc_caps->is_dsc_supported = false;
+
+ if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
+ dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
+@@ -7527,6 +7556,9 @@ static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder,
+ mode = amdgpu_dm_create_common_mode(encoder,
+ common_modes[i].name, common_modes[i].w,
+ common_modes[i].h);
++ if (!mode)
++ continue;
++
+ drm_mode_probed_add(connector, mode);
+ amdgpu_dm_connector->num_modes++;
+ }
+@@ -9286,7 +9318,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
+ /* restore the backlight level */
+ for (i = 0; i < dm->num_of_edps; i++) {
+ if (dm->backlight_dev[i] &&
+- (amdgpu_dm_backlight_get_level(dm, i) != dm->brightness[i]))
++ (dm->actual_brightness[i] != dm->brightness[i]))
+ amdgpu_dm_backlight_set_level(dm, i, dm->brightness[i]);
+ }
+ #endif
+@@ -10196,10 +10228,13 @@ static int dm_check_crtc_cursor(struct drm_atomic_state *state,
+ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc)
+ {
+ struct drm_connector *connector;
+- struct drm_connector_state *conn_state;
++ struct drm_connector_state *conn_state, *old_conn_state;
+ struct amdgpu_dm_connector *aconnector = NULL;
+ int i;
+- for_each_new_connector_in_state(state, connector, conn_state, i) {
++ for_each_oldnew_connector_in_state(state, connector, old_conn_state, conn_state, i) {
++ if (!conn_state->crtc)
++ conn_state = old_conn_state;
++
+ if (conn_state->crtc != crtc)
+ continue;
+
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+index d1d353a7c77d3..46d6e65f6bd48 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+@@ -446,6 +446,12 @@ struct amdgpu_display_manager {
+ * cached backlight values.
+ */
+ u32 brightness[AMDGPU_DM_MAX_NUM_EDP];
++ /**
++ * @actual_brightness:
++ *
++ * last successfully applied backlight values.
++ */
++ u32 actual_brightness[AMDGPU_DM_MAX_NUM_EDP];
+ };
+
+ enum dsc_clock_force_state {
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+index cce062adc4391..8a441a22c46ec 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+@@ -314,6 +314,14 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
+ ret = -EINVAL;
+ goto cleanup;
+ }
++
++ if ((aconn->base.connector_type != DRM_MODE_CONNECTOR_DisplayPort) &&
++ (aconn->base.connector_type != DRM_MODE_CONNECTOR_eDP)) {
++ DRM_DEBUG_DRIVER("No DP connector available for CRC source\n");
++ ret = -EINVAL;
++ goto cleanup;
++ }
++
+ }
+
+ #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+index 8080bba5b7a76..5c9f5214bc4e9 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+@@ -229,8 +229,10 @@ static ssize_t dp_link_settings_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -247,6 +249,7 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf,
+ {
+ struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+ struct dc_link *link = connector->dc_link;
++ struct dc *dc = (struct dc *)link->dc;
+ struct dc_link_settings prefer_link_settings;
+ char *wr_buf = NULL;
+ const uint32_t wr_buf_size = 40;
+@@ -313,7 +316,7 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf,
+ prefer_link_settings.lane_count = param[0];
+ prefer_link_settings.link_rate = param[1];
+
+- dp_retrain_link_dp_test(link, &prefer_link_settings, false);
++ dc_link_set_preferred_training_settings(dc, &prefer_link_settings, NULL, link, true);
+
+ kfree(wr_buf);
+ return size;
+@@ -387,8 +390,10 @@ static ssize_t dp_phy_settings_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user((*(rd_buf + result)), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -1315,8 +1320,10 @@ static ssize_t dp_dsc_clock_en_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -1332,8 +1339,10 @@ static ssize_t dp_dsc_clock_en_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -1502,8 +1511,10 @@ static ssize_t dp_dsc_slice_width_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -1519,8 +1530,10 @@ static ssize_t dp_dsc_slice_width_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -1687,8 +1700,10 @@ static ssize_t dp_dsc_slice_height_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -1704,8 +1719,10 @@ static ssize_t dp_dsc_slice_height_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -1868,8 +1885,10 @@ static ssize_t dp_dsc_bits_per_pixel_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -1885,8 +1904,10 @@ static ssize_t dp_dsc_bits_per_pixel_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -2044,8 +2065,10 @@ static ssize_t dp_dsc_pic_width_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -2061,8 +2084,10 @@ static ssize_t dp_dsc_pic_width_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -2101,8 +2126,10 @@ static ssize_t dp_dsc_pic_height_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -2118,8 +2145,10 @@ static ssize_t dp_dsc_pic_height_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -2173,8 +2202,10 @@ static ssize_t dp_dsc_chunk_size_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -2190,8 +2221,10 @@ static ssize_t dp_dsc_chunk_size_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -2245,8 +2278,10 @@ static ssize_t dp_dsc_slice_bpg_offset_read(struct file *f, char __user *buf,
+ break;
+ }
+
+- if (!pipe_ctx)
++ if (!pipe_ctx) {
++ kfree(rd_buf);
+ return -ENXIO;
++ }
+
+ dsc = pipe_ctx->stream_res.dsc;
+ if (dsc)
+@@ -2262,8 +2297,10 @@ static ssize_t dp_dsc_slice_bpg_offset_read(struct file *f, char __user *buf,
+ break;
+
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+
+ buf += 1;
+ size -= 1;
+@@ -2907,10 +2944,13 @@ static int crc_win_update_set(void *data, u64 val)
+ struct amdgpu_device *adev = drm_to_adev(new_crtc->dev);
+ struct crc_rd_work *crc_rd_wrk = adev->dm.crc_rd_wrk;
+
++ if (!crc_rd_wrk)
++ return 0;
++
+ if (val) {
+ spin_lock_irq(&adev_to_drm(adev)->event_lock);
+ spin_lock_irq(&crc_rd_wrk->crc_rd_work_lock);
+- if (crc_rd_wrk && crc_rd_wrk->crtc) {
++ if (crc_rd_wrk->crtc) {
+ old_crtc = crc_rd_wrk->crtc;
+ old_acrtc = to_amdgpu_crtc(old_crtc);
+ }
+@@ -3250,8 +3290,10 @@ static ssize_t dcc_en_bits_read(
+ dc->hwss.get_dcc_en_bits(dc, dcc_en_bits);
+
+ rd_buf = kcalloc(rd_buf_size, sizeof(char), GFP_KERNEL);
+- if (!rd_buf)
++ if (!rd_buf) {
++ kfree(dcc_en_bits);
+ return -ENOMEM;
++ }
+
+ for (i = 0; i < num_pipes; i++)
+ offset += snprintf(rd_buf + offset, rd_buf_size - offset,
+@@ -3264,8 +3306,10 @@ static ssize_t dcc_en_bits_read(
+ if (*pos >= rd_buf_size)
+ break;
+ r = put_user(*(rd_buf + result), buf);
+- if (r)
++ if (r) {
++ kfree(rd_buf);
+ return r; /* r = -EFAULT */
++ }
+ buf += 1;
+ size -= 1;
+ *pos += 1;
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+index 6fee12c91ef59..d793eec69d61e 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+@@ -40,6 +40,39 @@
+
+ #include "dm_helpers.h"
+
++struct monitor_patch_info {
++ unsigned int manufacturer_id;
++ unsigned int product_id;
++ void (*patch_func)(struct dc_edid_caps *edid_caps, unsigned int param);
++ unsigned int patch_param;
++};
++static void set_max_dsc_bpp_limit(struct dc_edid_caps *edid_caps, unsigned int param);
++
++static const struct monitor_patch_info monitor_patch_table[] = {
++{0x6D1E, 0x5BBF, set_max_dsc_bpp_limit, 15},
++{0x6D1E, 0x5B9A, set_max_dsc_bpp_limit, 15},
++};
++
++static void set_max_dsc_bpp_limit(struct dc_edid_caps *edid_caps, unsigned int param)
++{
++ if (edid_caps)
++ edid_caps->panel_patch.max_dsc_target_bpp_limit = param;
++}
++
++static int amdgpu_dm_patch_edid_caps(struct dc_edid_caps *edid_caps)
++{
++ int i, ret = 0;
++
++ for (i = 0; i < ARRAY_SIZE(monitor_patch_table); i++)
++ if ((edid_caps->manufacturer_id == monitor_patch_table[i].manufacturer_id)
++ && (edid_caps->product_id == monitor_patch_table[i].product_id)) {
++ monitor_patch_table[i].patch_func(edid_caps, monitor_patch_table[i].patch_param);
++ ret++;
++ }
++
++ return ret;
++}
++
+ /* dm_helpers_parse_edid_caps
+ *
+ * Parse edid caps
+@@ -125,6 +158,8 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
+ kfree(sads);
+ kfree(sadb);
+
++ amdgpu_dm_patch_edid_caps(edid_caps);
++
+ return result;
+ }
+
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+index 7af0d58c231b6..74885ff77f96a 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+@@ -36,6 +36,8 @@
+ #include "dm_helpers.h"
+
+ #include "dc_link_ddc.h"
++#include "ddc_service_types.h"
++#include "dpcd_defs.h"
+
+ #include "i2caux_interface.h"
+ #include "dmub_cmd.h"
+@@ -155,6 +157,16 @@ static const struct drm_connector_funcs dm_dp_mst_connector_funcs = {
+ };
+
+ #if defined(CONFIG_DRM_AMD_DC_DCN)
++static bool needs_dsc_aux_workaround(struct dc_link *link)
++{
++ if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_90CC24 &&
++ (link->dpcd_caps.dpcd_rev.raw == DPCD_REV_14 || link->dpcd_caps.dpcd_rev.raw == DPCD_REV_12) &&
++ link->dpcd_caps.sink_count.bits.SINK_COUNT >= 2)
++ return true;
++
++ return false;
++}
++
+ static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnector)
+ {
+ struct dc_sink *dc_sink = aconnector->dc_sink;
+@@ -164,7 +176,7 @@ static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnecto
+ u8 *dsc_branch_dec_caps = NULL;
+
+ aconnector->dsc_aux = drm_dp_mst_dsc_aux_for_port(port);
+-#if defined(CONFIG_HP_HOOK_WORKAROUND)
++
+ /*
+ * drm_dp_mst_dsc_aux_for_port() will return NULL for certain configs
+ * because it only check the dsc/fec caps of the "port variable" and not the dock
+@@ -174,10 +186,10 @@ static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnecto
+ * Workaround: explicitly check the use case above and use the mst dock's aux as dsc_aux
+ *
+ */
+-
+- if (!aconnector->dsc_aux && !port->parent->port_parent)
++ if (!aconnector->dsc_aux && !port->parent->port_parent &&
++ needs_dsc_aux_workaround(aconnector->dc_link))
+ aconnector->dsc_aux = &aconnector->mst_port->dm_dp_aux.aux;
+-#endif
++
+ if (!aconnector->dsc_aux)
+ return false;
+
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+index 70a554f1e725a..7072fb2ec07fa 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+@@ -74,10 +74,8 @@ bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream)
+
+ link = stream->link;
+
+- psr_config.psr_version = link->dpcd_caps.psr_caps.psr_version;
+-
+- if (psr_config.psr_version > 0) {
+- psr_config.psr_exit_link_training_required = 0x1;
++ if (link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED) {
++ psr_config.psr_version = link->psr_settings.psr_version;
+ psr_config.psr_frame_capture_indication_req = 0;
+ psr_config.psr_rfb_setup_time = 0x37;
+ psr_config.psr_sdp_transmit_line_num_deadline = 0x20;
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+index bb31541f80723..6420527fe476c 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+@@ -306,8 +306,7 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
+ case FAMILY_NV:
+ if (ASICREV_IS_SIENNA_CICHLID_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
+ dcn3_clk_mgr_destroy(clk_mgr);
+- }
+- if (ASICREV_IS_DIMGREY_CAVEFISH_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
++ } else if (ASICREV_IS_DIMGREY_CAVEFISH_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
+ dcn3_clk_mgr_destroy(clk_mgr);
+ }
+ if (ASICREV_IS_BEIGE_GOBY_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+index 1861a147a7fa1..5c5cbeb59c4d9 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+@@ -437,8 +437,10 @@ static void dcn3_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base)
+ clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1;
+
+ /* Refresh bounding box */
++ DC_FP_START();
+ clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box(
+ clk_mgr->base.ctx->dc, clk_mgr_base->bw_params);
++ DC_FP_END();
+ }
+
+ static bool dcn3_is_smu_present(struct clk_mgr *clk_mgr_base)
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
+index 7046da14bb2a5..329ce4e84b83c 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
+@@ -582,32 +582,32 @@ static struct wm_table lpddr5_wm_table = {
+ .wm_inst = WM_A,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.65333,
+- .sr_exit_time_us = 5.32,
+- .sr_enter_plus_exit_time_us = 6.38,
++ .sr_exit_time_us = 13.5,
++ .sr_enter_plus_exit_time_us = 16.5,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_B,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.65333,
+- .sr_exit_time_us = 9.82,
+- .sr_enter_plus_exit_time_us = 11.196,
++ .sr_exit_time_us = 13.5,
++ .sr_enter_plus_exit_time_us = 16.5,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_C,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.65333,
+- .sr_exit_time_us = 9.89,
+- .sr_enter_plus_exit_time_us = 11.24,
++ .sr_exit_time_us = 13.5,
++ .sr_enter_plus_exit_time_us = 16.5,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_D,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.65333,
+- .sr_exit_time_us = 9.748,
+- .sr_enter_plus_exit_time_us = 11.102,
++ .sr_exit_time_us = 13.5,
++ .sr_enter_plus_exit_time_us = 16.5,
+ .valid = true,
+ },
+ }
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+index 377c4e53a2b37..8f6e6496ea787 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+@@ -157,6 +157,7 @@ static void dcn31_update_clocks(struct clk_mgr *clk_mgr_base,
+ union display_idle_optimization_u idle_info = { 0 };
+ idle_info.idle_info.df_request_disabled = 1;
+ idle_info.idle_info.phy_ref_clk_off = 1;
++ idle_info.idle_info.s0i2_rdy = 1;
+ dcn31_smu_set_display_idle_optimization(clk_mgr, idle_info.data);
+ /* update power state */
+ clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_LOW_POWER;
+@@ -323,38 +324,38 @@ static struct clk_bw_params dcn31_bw_params = {
+
+ };
+
+-static struct wm_table ddr4_wm_table = {
++static struct wm_table ddr5_wm_table = {
+ .entries = {
+ {
+ .wm_inst = WM_A,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.72,
+- .sr_exit_time_us = 6.09,
+- .sr_enter_plus_exit_time_us = 7.14,
++ .sr_exit_time_us = 9,
++ .sr_enter_plus_exit_time_us = 11,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_B,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.72,
+- .sr_exit_time_us = 10.12,
+- .sr_enter_plus_exit_time_us = 11.48,
++ .sr_exit_time_us = 9,
++ .sr_enter_plus_exit_time_us = 11,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_C,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.72,
+- .sr_exit_time_us = 10.12,
+- .sr_enter_plus_exit_time_us = 11.48,
++ .sr_exit_time_us = 9,
++ .sr_enter_plus_exit_time_us = 11,
+ .valid = true,
+ },
+ {
+ .wm_inst = WM_D,
+ .wm_type = WM_TYPE_PSTATE_CHG,
+ .pstate_latency_us = 11.72,
+- .sr_exit_time_us = 10.12,
+- .sr_enter_plus_exit_time_us = 11.48,
++ .sr_exit_time_us = 9,
++ .sr_enter_plus_exit_time_us = 11,
+ .valid = true,
+ },
+ }
+@@ -682,7 +683,7 @@ void dcn31_clk_mgr_construct(
+ if (ctx->dc_bios->integrated_info->memory_type == LpDdr5MemType) {
+ dcn31_bw_params.wm_table = lpddr5_wm_table;
+ } else {
+- dcn31_bw_params.wm_table = ddr4_wm_table;
++ dcn31_bw_params.wm_table = ddr5_wm_table;
+ }
+ /* Saved clocks configured at boot for debug purposes */
+ dcn31_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, &clk_mgr->base.base, &log_info);
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
+index 8c2b77eb94593..21d2cbc3cbb20 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
+@@ -119,6 +119,16 @@ int dcn31_smu_send_msg_with_param(
+
+ result = dcn31_smu_wait_for_response(clk_mgr, 10, 200000);
+
++ if (result == VBIOSSMC_Result_Failed) {
++ if (msg_id == VBIOSSMC_MSG_TransferTableDram2Smu &&
++ param == TABLE_WATERMARKS)
++ DC_LOG_WARNING("Watermarks table not configured properly by SMU");
++ else
++ ASSERT(0);
++ REG_WRITE(MP1_SMN_C2PMSG_91, VBIOSSMC_Result_OK);
++ return -1;
++ }
++
+ if (IS_SMU_TIMEOUT(result)) {
+ ASSERT(0);
+ dm_helpers_smu_timeout(CTX, msg_id, param, 10 * 200000);
+diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
+index c798c65d42765..b37c4d2e7a1e0 100644
+--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
++++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
+@@ -891,10 +891,13 @@ static bool dc_construct(struct dc *dc,
+ goto fail;
+ #ifdef CONFIG_DRM_AMD_DC_DCN
+ dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present;
+-#endif
+
+- if (dc->res_pool->funcs->update_bw_bounding_box)
++ if (dc->res_pool->funcs->update_bw_bounding_box) {
++ DC_FP_START();
+ dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params);
++ DC_FP_END();
++ }
++#endif
+
+ /* Creation of current_state must occur after dc->dml
+ * is initialized in dc_create_resource_pool because
+@@ -1118,6 +1121,8 @@ struct dc *dc_create(const struct dc_init_data *init_params)
+
+ dc->caps.max_dp_protocol_version = DP_VERSION_1_4;
+
++ dc->caps.max_otg_num = dc->res_pool->res_cap->num_timing_generator;
++
+ if (dc->res_pool->dmcu != NULL)
+ dc->versions.dmcu_version = dc->res_pool->dmcu->dmcu_version;
+ }
+@@ -2703,7 +2708,8 @@ static void commit_planes_for_stream(struct dc *dc,
+ #endif
+
+ if ((update_type != UPDATE_TYPE_FAST) && stream->update_flags.bits.dsc_changed)
+- if (top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
++ if (top_pipe_to_program &&
++ top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
+ if (should_use_dmub_lock(stream->link)) {
+ union dmub_hw_lock_flags hw_locks = { 0 };
+ struct dmub_hw_lock_inst_flags inst_flags = { 0 };
+diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+index 1e44b13c1c7de..3c4205248efc2 100644
+--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
++++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+@@ -1696,6 +1696,8 @@ static void enable_stream_features(struct pipe_ctx *pipe_ctx)
+ union down_spread_ctrl old_downspread;
+ union down_spread_ctrl new_downspread;
+
++ memset(&old_downspread, 0, sizeof(old_downspread));
++
+ core_link_read_dpcd(link, DP_DOWNSPREAD_CTRL,
+ &old_downspread.raw, sizeof(old_downspread));
+
+diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+index 6d655e158267a..05f81d44aa6c2 100644
+--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
++++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+@@ -3118,7 +3118,7 @@ static void dp_test_get_audio_test_data(struct dc_link *link, bool disable_video
+ &dpcd_pattern_type.value,
+ sizeof(dpcd_pattern_type));
+
+- channel_count = dpcd_test_mode.bits.channel_count + 1;
++ channel_count = min(dpcd_test_mode.bits.channel_count + 1, AUDIO_CHANNELS_COUNT);
+
+ // read pattern periods for requested channels when sawTooth pattern is requested
+ if (dpcd_pattern_type.value == AUDIO_TEST_PATTERN_SAWTOOTH ||
+@@ -3650,7 +3650,9 @@ bool dp_retrieve_lttpr_cap(struct dc_link *link)
+ lttpr_dpcd_data,
+ sizeof(lttpr_dpcd_data));
+ if (status != DC_OK) {
+- dm_error("%s: Read LTTPR caps data failed.\n", __func__);
++#if defined(CONFIG_DRM_AMD_DC_DCN)
++ DC_LOG_DP2("%s: Read LTTPR caps data failed.\n", __func__);
++#endif
+ return false;
+ }
+
+@@ -3913,6 +3915,26 @@ static bool retrieve_link_cap(struct dc_link *link)
+ dp_hw_fw_revision.ieee_fw_rev,
+ sizeof(dp_hw_fw_revision.ieee_fw_rev));
+
++ /* Quirk for Apple MBP 2018 15" Retina panels: wrong DP_MAX_LINK_RATE */
++ {
++ uint8_t str_mbp_2018[] = { 101, 68, 21, 103, 98, 97 };
++ uint8_t fwrev_mbp_2018[] = { 7, 4 };
++ uint8_t fwrev_mbp_2018_vega[] = { 8, 4 };
++
++ /* We also check for the firmware revision as 16,1 models have an
++ * identical device id and are incorrectly quirked otherwise.
++ */
++ if ((link->dpcd_caps.sink_dev_id == 0x0010fa) &&
++ !memcmp(link->dpcd_caps.sink_dev_id_str, str_mbp_2018,
++ sizeof(str_mbp_2018)) &&
++ (!memcmp(link->dpcd_caps.sink_fw_revision, fwrev_mbp_2018,
++ sizeof(fwrev_mbp_2018)) ||
++ !memcmp(link->dpcd_caps.sink_fw_revision, fwrev_mbp_2018_vega,
++ sizeof(fwrev_mbp_2018_vega)))) {
++ link->reported_link_cap.link_rate = LINK_RATE_RBR2;
++ }
++ }
++
+ memset(&link->dpcd_caps.dsc_caps, '\0',
+ sizeof(link->dpcd_caps.dsc_caps));
+ memset(&link->dpcd_caps.fec_cap, '\0', sizeof(link->dpcd_caps.fec_cap));
+@@ -4690,7 +4712,7 @@ enum dc_status dp_set_fec_ready(struct dc_link *link, bool ready)
+ link_enc->funcs->fec_set_ready(link_enc, true);
+ link->fec_state = dc_link_fec_ready;
+ } else {
+- link_enc->funcs->fec_set_ready(link->link_enc, false);
++ link_enc->funcs->fec_set_ready(link_enc, false);
+ link->fec_state = dc_link_fec_not_ready;
+ dm_error("dpcd write failed to set fec_ready");
+ }
+diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+index a60396d5be445..82f1f27baaf38 100644
+--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+@@ -1599,6 +1599,9 @@ static bool are_stream_backends_same(
+ if (is_timing_changed(stream_a, stream_b))
+ return false;
+
++ if (stream_a->signal != stream_b->signal)
++ return false;
++
+ if (stream_a->dpms_off != stream_b->dpms_off)
+ return false;
+
+@@ -1623,6 +1626,10 @@ bool dc_is_stream_unchanged(
+ if (old_stream->ignore_msa_timing_param != stream->ignore_msa_timing_param)
+ return false;
+
++ /*compare audio info*/
++ if (memcmp(&old_stream->audio_info, &stream->audio_info, sizeof(stream->audio_info)) != 0)
++ return false;
++
+ return true;
+ }
+
+@@ -1795,9 +1802,6 @@ enum dc_status dc_remove_stream_from_ctx(
+ dc->res_pool,
+ del_pipe->stream_res.stream_enc,
+ false);
+- /* Release link encoder from stream in new dc_state. */
+- if (dc->res_pool->funcs->link_enc_unassign)
+- dc->res_pool->funcs->link_enc_unassign(new_ctx, del_pipe->stream);
+
+ if (del_pipe->stream_res.audio)
+ update_audio_usage(
+diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
+index 3ab52d9a82cf6..e0f58fab5e8ed 100644
+--- a/drivers/gpu/drm/amd/display/dc/dc.h
++++ b/drivers/gpu/drm/amd/display/dc/dc.h
+@@ -185,6 +185,7 @@ struct dc_caps {
+ struct dc_color_caps color;
+ bool vbios_lttpr_aware;
+ bool vbios_lttpr_enable;
++ uint32_t max_otg_num;
+ };
+
+ struct dc_bug_wa {
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+index f4f423d0b8c3f..80595d7f060c3 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+@@ -940,6 +940,7 @@ static const struct hubbub_funcs hubbub1_funcs = {
+ .program_watermarks = hubbub1_program_watermarks,
+ .is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
+ .allow_self_refresh_control = hubbub1_allow_self_refresh_control,
++ .verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high,
+ };
+
+ void hubbub1_construct(struct hubbub *hubbub,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+index df8a7718a85fc..93f31e4aeecb3 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+@@ -1052,9 +1052,13 @@ static bool dcn10_hw_wa_force_recovery(struct dc *dc)
+
+ void dcn10_verify_allow_pstate_change_high(struct dc *dc)
+ {
++ struct hubbub *hubbub = dc->res_pool->hubbub;
+ static bool should_log_hw_state; /* prevent hw state log by default */
+
+- if (!hubbub1_verify_allow_pstate_change_high(dc->res_pool->hubbub)) {
++ if (!hubbub->funcs->verify_allow_pstate_change_high)
++ return;
++
++ if (!hubbub->funcs->verify_allow_pstate_change_high(hubbub)) {
+ int i = 0;
+
+ if (should_log_hw_state)
+@@ -1063,8 +1067,8 @@ void dcn10_verify_allow_pstate_change_high(struct dc *dc)
+ TRACE_DC_PIPE_STATE(pipe_ctx, i, MAX_PIPES);
+ BREAK_TO_DEBUGGER();
+ if (dcn10_hw_wa_force_recovery(dc)) {
+- /*check again*/
+- if (!hubbub1_verify_allow_pstate_change_high(dc->res_pool->hubbub))
++ /*check again*/
++ if (!hubbub->funcs->verify_allow_pstate_change_high(hubbub))
+ BREAK_TO_DEBUGGER();
+ }
+ }
+@@ -1435,6 +1439,9 @@ void dcn10_init_hw(struct dc *dc)
+ }
+ }
+
++ if (hws->funcs.enable_power_gating_plane)
++ hws->funcs.enable_power_gating_plane(dc->hwseq, true);
++
+ /* If taking control over from VBIOS, we may want to optimize our first
+ * mode set, so we need to skip powering down pipes until we know which
+ * pipes we want to use.
+@@ -1487,8 +1494,6 @@ void dcn10_init_hw(struct dc *dc)
+
+ REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+ }
+- if (hws->funcs.enable_power_gating_plane)
+- hws->funcs.enable_power_gating_plane(dc->hwseq, true);
+
+ if (dc->clk_mgr->funcs->notify_wm_ranges)
+ dc->clk_mgr->funcs->notify_wm_ranges(dc->clk_mgr);
+@@ -1522,7 +1527,7 @@ void dcn10_power_down_on_boot(struct dc *dc)
+ for (i = 0; i < dc->link_count; i++) {
+ struct dc_link *link = dc->links[i];
+
+- if (link->link_enc->funcs->is_dig_enabled &&
++ if (link->link_enc && link->link_enc->funcs->is_dig_enabled &&
+ link->link_enc->funcs->is_dig_enabled(link->link_enc) &&
+ dc->hwss.power_down) {
+ dc->hwss.power_down(dc);
+@@ -2455,14 +2460,18 @@ void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+ struct mpc *mpc = dc->res_pool->mpc;
+ struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
+
+- if (per_pixel_alpha)
+- blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
+- else
+- blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
+-
+ blnd_cfg.overlap_only = false;
+ blnd_cfg.global_gain = 0xff;
+
++ if (per_pixel_alpha && pipe_ctx->plane_state->global_alpha) {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN;
++ blnd_cfg.global_gain = pipe_ctx->plane_state->global_alpha_value;
++ } else if (per_pixel_alpha) {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
++ } else {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
++ }
++
+ if (pipe_ctx->plane_state->global_alpha)
+ blnd_cfg.global_alpha = pipe_ctx->plane_state->global_alpha_value;
+ else
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
+index 34001a30d449a..10e613ec7d24f 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
+@@ -78,6 +78,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
+ .get_clock = dcn10_get_clock,
+ .get_vupdate_offset_from_vsync = dcn10_get_vupdate_offset_from_vsync,
+ .calc_vupdate_position = dcn10_calc_vupdate_position,
++ .power_down = dce110_power_down,
+ .set_backlight_level = dce110_set_backlight_level,
+ .set_abm_immediate_disable = dce110_set_abm_immediate_disable,
+ .set_pipe = dce110_set_pipe,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
+index a47ba1d45be92..9f8d7f92300bb 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
+@@ -2297,14 +2297,18 @@ void dcn20_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+ struct mpc *mpc = dc->res_pool->mpc;
+ struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
+
+- if (per_pixel_alpha)
+- blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
+- else
+- blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
+-
+ blnd_cfg.overlap_only = false;
+ blnd_cfg.global_gain = 0xff;
+
++ if (per_pixel_alpha && pipe_ctx->plane_state->global_alpha) {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN;
++ blnd_cfg.global_gain = pipe_ctx->plane_state->global_alpha_value;
++ } else if (per_pixel_alpha) {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
++ } else {
++ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
++ }
++
+ if (pipe_ctx->plane_state->global_alpha)
+ blnd_cfg.global_alpha = pipe_ctx->plane_state->global_alpha_value;
+ else
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
+index e3e01b17c164e..ede11eb120d4f 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
+@@ -1854,7 +1854,9 @@ static void swizzle_to_dml_params(
+ case DC_SW_VAR_D_X:
+ *sw_mode = dm_sw_var_d_x;
+ break;
+-
++ case DC_SW_VAR_R_X:
++ *sw_mode = dm_sw_var_r_x;
++ break;
+ default:
+ ASSERT(0); /* Not supported */
+ break;
+@@ -3152,7 +3154,7 @@ void dcn20_calculate_dlg_params(
+
+ context->bw_ctx.dml.funcs.rq_dlg_get_rq_reg(&context->bw_ctx.dml,
+ &context->res_ctx.pipe_ctx[i].rq_regs,
+- pipes[pipe_idx].pipe);
++ &pipes[pipe_idx].pipe);
+ pipe_idx++;
+ }
+ }
+@@ -3668,16 +3670,22 @@ static bool init_soc_bounding_box(struct dc *dc,
+ clock_limits_available = (status == PP_SMU_RESULT_OK);
+ }
+
+- if (clock_limits_available && uclk_states_available && num_states)
++ if (clock_limits_available && uclk_states_available && num_states) {
++ DC_FP_START();
+ dcn20_update_bounding_box(dc, loaded_bb, &max_clocks, uclk_states, num_states);
+- else if (clock_limits_available)
++ DC_FP_END();
++ } else if (clock_limits_available) {
++ DC_FP_START();
+ dcn20_cap_soc_clocks(loaded_bb, max_clocks);
++ DC_FP_END();
++ }
+ }
+
+ loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
+ loaded_ip->max_num_dpp = pool->base.pipe_count;
++ DC_FP_START();
+ dcn20_patch_bounding_box(dc, loaded_bb);
+-
++ DC_FP_END();
+ return true;
+ }
+
+@@ -3697,8 +3705,6 @@ static bool dcn20_resource_construct(
+ enum dml_project dml_project_version =
+ get_dml_project_version(ctx->asic_id.hw_internal_rev);
+
+- DC_FP_START();
+-
+ ctx->dc_bios->regs = &bios_regs;
+ pool->base.funcs = &dcn20_res_pool_funcs;
+
+@@ -4047,12 +4053,10 @@ static bool dcn20_resource_construct(
+ pool->base.oem_device = NULL;
+ }
+
+- DC_FP_END();
+ return true;
+
+ create_fail:
+
+- DC_FP_END();
+ dcn20_resource_destruct(pool);
+
+ return false;
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
+index fbbdf99761838..5b8274b8c3845 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
+@@ -1428,6 +1428,7 @@ static struct clock_source *dcn21_clock_source_create(
+ return &clk_src->base;
+ }
+
++ kfree(clk_src);
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
+index f4414de96acc5..152c9c5733f1c 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
+@@ -448,6 +448,7 @@ static const struct hubbub_funcs hubbub30_funcs = {
+ .program_watermarks = hubbub3_program_watermarks,
+ .allow_self_refresh_control = hubbub1_allow_self_refresh_control,
+ .is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
++ .verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high,
+ .force_wm_propagate_to_pipes = hubbub3_force_wm_propagate_to_pipes,
+ .force_pstate_change_control = hubbub3_force_pstate_change_control,
+ .init_watermarks = hubbub3_init_watermarks,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
+index fafed1e4a998d..f834573758113 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
+@@ -570,6 +570,9 @@ void dcn30_init_hw(struct dc *dc)
+ }
+ }
+
++ if (hws->funcs.enable_power_gating_plane)
++ hws->funcs.enable_power_gating_plane(dc->hwseq, true);
++
+ /* If taking control over from VBIOS, we may want to optimize our first
+ * mode set, so we need to skip powering down pipes until we know which
+ * pipes we want to use.
+@@ -647,8 +650,6 @@ void dcn30_init_hw(struct dc *dc)
+
+ REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+ }
+- if (hws->funcs.enable_power_gating_plane)
+- hws->funcs.enable_power_gating_plane(dc->hwseq, true);
+
+ if (!dcb->funcs->is_accelerated_mode(dcb) && dc->res_pool->hubbub->funcs->init_watermarks)
+ dc->res_pool->hubbub->funcs->init_watermarks(dc->res_pool->hubbub);
+@@ -1002,7 +1003,8 @@ void dcn30_set_disp_pattern_generator(const struct dc *dc,
+ /* turning off DPG */
+ pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, false);
+ for (mpcc_pipe = pipe_ctx->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe)
+- mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, false);
++ if (mpcc_pipe->plane_res.hubp)
++ mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, false);
+
+ stream_res->opp->funcs->opp_set_disp_pattern_generator(stream_res->opp, test_pattern, color_space,
+ color_depth, solid_color, width, height, offset);
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
+index a0de309475a97..0294d0cc47595 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
+@@ -840,7 +840,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .timing_trace = false,
+ .clock_trace = true,
+ .disable_pplib_clock_request = true,
+- .pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
++ .pipe_split_policy = MPC_SPLIT_DYNAMIC,
+ .force_single_disp_pipe_split = false,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+@@ -1879,7 +1879,6 @@ static noinline bool dcn30_internal_validate_bw(
+ dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
+ pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+
+- DC_FP_START();
+ if (!pipe_cnt) {
+ out = true;
+ goto validate_out;
+@@ -2103,7 +2102,6 @@ validate_fail:
+ out = false;
+
+ validate_out:
+- DC_FP_END();
+ return out;
+ }
+
+@@ -2304,7 +2302,9 @@ bool dcn30_validate_bandwidth(struct dc *dc,
+
+ BW_VAL_TRACE_COUNT();
+
++ DC_FP_START();
+ out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
++ DC_FP_END();
+
+ if (pipe_cnt == 0)
+ goto validate_out;
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
+index 1e3bd2e9cdcc4..a046664e20316 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
+@@ -60,6 +60,7 @@ static const struct hubbub_funcs hubbub301_funcs = {
+ .program_watermarks = hubbub3_program_watermarks,
+ .allow_self_refresh_control = hubbub1_allow_self_refresh_control,
+ .is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
++ .verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high,
+ .force_wm_propagate_to_pipes = hubbub3_force_wm_propagate_to_pipes,
+ .force_pstate_change_control = hubbub3_force_pstate_change_control,
+ .hubbub_read_state = hubbub2_read_state,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
+index 912285fdce18e..dea358b01791c 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
+@@ -863,7 +863,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .disable_clock_gate = true,
+ .disable_pplib_clock_request = true,
+ .disable_pplib_wm_range = true,
+- .pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
++ .pipe_split_policy = MPC_SPLIT_AVOID,
+ .force_single_disp_pipe_split = false,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+@@ -1622,12 +1622,106 @@ static void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *b
+ dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30);
+ }
+
++static void calculate_wm_set_for_vlevel(
++ int vlevel,
++ struct wm_range_table_entry *table_entry,
++ struct dcn_watermarks *wm_set,
++ struct display_mode_lib *dml,
++ display_e2e_pipe_params_st *pipes,
++ int pipe_cnt)
++{
++ double dram_clock_change_latency_cached = dml->soc.dram_clock_change_latency_us;
++
++ ASSERT(vlevel < dml->soc.num_states);
++ /* only pipe 0 is read for voltage and dcf/soc clocks */
++ pipes[0].clks_cfg.voltage = vlevel;
++ pipes[0].clks_cfg.dcfclk_mhz = dml->soc.clock_limits[vlevel].dcfclk_mhz;
++ pipes[0].clks_cfg.socclk_mhz = dml->soc.clock_limits[vlevel].socclk_mhz;
++
++ dml->soc.dram_clock_change_latency_us = table_entry->pstate_latency_us;
++ dml->soc.sr_exit_time_us = table_entry->sr_exit_time_us;
++ dml->soc.sr_enter_plus_exit_time_us = table_entry->sr_enter_plus_exit_time_us;
++
++ wm_set->urgent_ns = get_wm_urgent(dml, pipes, pipe_cnt) * 1000;
++ wm_set->cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000;
++ wm_set->cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000;
++ wm_set->cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000;
++ wm_set->pte_meta_urgent_ns = get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000;
++ wm_set->frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000;
++ wm_set->frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000;
++ wm_set->urgent_latency_ns = get_urgent_latency(dml, pipes, pipe_cnt) * 1000;
++ dml->soc.dram_clock_change_latency_us = dram_clock_change_latency_cached;
++
++}
++
++static void dcn301_calculate_wm_and_dlg(
++ struct dc *dc, struct dc_state *context,
++ display_e2e_pipe_params_st *pipes,
++ int pipe_cnt,
++ int vlevel_req)
++{
++ int i, pipe_idx;
++ int vlevel, vlevel_max;
++ struct wm_range_table_entry *table_entry;
++ struct clk_bw_params *bw_params = dc->clk_mgr->bw_params;
++
++ ASSERT(bw_params);
++
++ vlevel_max = bw_params->clk_table.num_entries - 1;
++
++ /* WM Set D */
++ table_entry = &bw_params->wm_table.entries[WM_D];
++ if (table_entry->wm_type == WM_TYPE_RETRAINING)
++ vlevel = 0;
++ else
++ vlevel = vlevel_max;
++ calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.d,
++ &context->bw_ctx.dml, pipes, pipe_cnt);
++ /* WM Set C */
++ table_entry = &bw_params->wm_table.entries[WM_C];
++ vlevel = min(max(vlevel_req, 2), vlevel_max);
++ calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.c,
++ &context->bw_ctx.dml, pipes, pipe_cnt);
++ /* WM Set B */
++ table_entry = &bw_params->wm_table.entries[WM_B];
++ vlevel = min(max(vlevel_req, 1), vlevel_max);
++ calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.b,
++ &context->bw_ctx.dml, pipes, pipe_cnt);
++
++ /* WM Set A */
++ table_entry = &bw_params->wm_table.entries[WM_A];
++ vlevel = min(vlevel_req, vlevel_max);
++ calculate_wm_set_for_vlevel(vlevel, table_entry, &context->bw_ctx.bw.dcn.watermarks.a,
++ &context->bw_ctx.dml, pipes, pipe_cnt);
++
++ for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
++ if (!context->res_ctx.pipe_ctx[i].stream)
++ continue;
++
++ pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt);
++ pipes[pipe_idx].clks_cfg.dppclk_mhz = get_dppclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
++
++ if (dc->config.forced_clocks) {
++ pipes[pipe_idx].clks_cfg.dispclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dispclk_mhz;
++ pipes[pipe_idx].clks_cfg.dppclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dppclk_mhz;
++ }
++ if (dc->debug.min_disp_clk_khz > pipes[pipe_idx].clks_cfg.dispclk_mhz * 1000)
++ pipes[pipe_idx].clks_cfg.dispclk_mhz = dc->debug.min_disp_clk_khz / 1000.0;
++ if (dc->debug.min_dpp_clk_khz > pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
++ pipes[pipe_idx].clks_cfg.dppclk_mhz = dc->debug.min_dpp_clk_khz / 1000.0;
++
++ pipe_idx++;
++ }
++
++ dcn20_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel);
++}
++
+ static struct resource_funcs dcn301_res_pool_funcs = {
+ .destroy = dcn301_destroy_resource_pool,
+ .link_enc_create = dcn301_link_encoder_create,
+ .panel_cntl_create = dcn301_panel_cntl_create,
+ .validate_bandwidth = dcn30_validate_bandwidth,
+- .calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
++ .calculate_wm_and_dlg = dcn301_calculate_wm_and_dlg,
+ .update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+ .populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
+ .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
+index 7d3ff5d444023..2292bb82026e2 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
+@@ -211,7 +211,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .timing_trace = false,
+ .clock_trace = true,
+ .disable_pplib_clock_request = true,
+- .pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
++ .pipe_split_policy = MPC_SPLIT_DYNAMIC,
+ .force_single_disp_pipe_split = false,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
+index dd38796ba30ad..589ddab61c2a9 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
+@@ -193,7 +193,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .timing_trace = false,
+ .clock_trace = true,
+ .disable_pplib_clock_request = true,
+- .pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
++ .pipe_split_policy = MPC_SPLIT_DYNAMIC,
+ .force_single_disp_pipe_split = false,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
+index 90c73a1cb9861..208d2dc8b1d1a 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
+@@ -24,6 +24,7 @@
+ */
+
+
++#include <linux/delay.h>
+ #include "dcn30/dcn30_hubbub.h"
+ #include "dcn31_hubbub.h"
+ #include "dm_services.h"
+@@ -138,8 +139,11 @@ static uint32_t convert_and_clamp(
+ ret_val = wm_ns * refclk_mhz;
+ ret_val /= 1000;
+
+- if (ret_val > clamp_value)
++ if (ret_val > clamp_value) {
++ /* clamping WMs is abnormal, unexpected and may lead to underflow*/
++ ASSERT(0);
+ ret_val = clamp_value;
++ }
+
+ return ret_val;
+ }
+@@ -159,7 +163,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->a.urgent_ns > hubbub2->watermarks.a.urgent_ns) {
+ hubbub2->watermarks.a.urgent_ns = watermarks->a.urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, 0,
+ DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
+
+@@ -193,7 +197,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->a.urgent_latency_ns > hubbub2->watermarks.a.urgent_latency_ns) {
+ hubbub2->watermarks.a.urgent_latency_ns = watermarks->a.urgent_latency_ns;
+ prog_wm_value = convert_and_clamp(watermarks->a.urgent_latency_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, 0,
+ DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, prog_wm_value);
+ } else if (watermarks->a.urgent_latency_ns < hubbub2->watermarks.a.urgent_latency_ns)
+@@ -203,7 +207,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->b.urgent_ns > hubbub2->watermarks.b.urgent_ns) {
+ hubbub2->watermarks.b.urgent_ns = watermarks->b.urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->b.urgent_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, 0,
+ DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
+
+@@ -237,7 +241,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->b.urgent_latency_ns > hubbub2->watermarks.b.urgent_latency_ns) {
+ hubbub2->watermarks.b.urgent_latency_ns = watermarks->b.urgent_latency_ns;
+ prog_wm_value = convert_and_clamp(watermarks->b.urgent_latency_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, 0,
+ DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, prog_wm_value);
+ } else if (watermarks->b.urgent_latency_ns < hubbub2->watermarks.b.urgent_latency_ns)
+@@ -247,7 +251,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->c.urgent_ns > hubbub2->watermarks.c.urgent_ns) {
+ hubbub2->watermarks.c.urgent_ns = watermarks->c.urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->c.urgent_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, 0,
+ DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
+
+@@ -281,7 +285,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->c.urgent_latency_ns > hubbub2->watermarks.c.urgent_latency_ns) {
+ hubbub2->watermarks.c.urgent_latency_ns = watermarks->c.urgent_latency_ns;
+ prog_wm_value = convert_and_clamp(watermarks->c.urgent_latency_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, 0,
+ DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, prog_wm_value);
+ } else if (watermarks->c.urgent_latency_ns < hubbub2->watermarks.c.urgent_latency_ns)
+@@ -291,7 +295,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->d.urgent_ns > hubbub2->watermarks.d.urgent_ns) {
+ hubbub2->watermarks.d.urgent_ns = watermarks->d.urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->d.urgent_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, 0,
+ DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
+
+@@ -325,7 +329,7 @@ static bool hubbub31_program_urgent_watermarks(
+ if (safe_to_lower || watermarks->d.urgent_latency_ns > hubbub2->watermarks.d.urgent_latency_ns) {
+ hubbub2->watermarks.d.urgent_latency_ns = watermarks->d.urgent_latency_ns;
+ prog_wm_value = convert_and_clamp(watermarks->d.urgent_latency_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0x3fff);
+ REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, 0,
+ DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, prog_wm_value);
+ } else if (watermarks->d.urgent_latency_ns < hubbub2->watermarks.d.urgent_latency_ns)
+@@ -351,7 +355,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
+@@ -367,7 +371,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->a.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
+@@ -383,7 +387,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_Z8_A calculated =%d\n"
+@@ -399,7 +403,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->a.cstate_pstate.cstate_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_Z8_A calculated =%d\n"
+@@ -416,7 +420,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_B calculated =%d\n"
+@@ -432,7 +436,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->b.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
+@@ -448,7 +452,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_Z8_B calculated =%d\n"
+@@ -464,7 +468,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->b.cstate_pstate.cstate_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_Z8_B calculated =%d\n"
+@@ -481,7 +485,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_C calculated =%d\n"
+@@ -497,7 +501,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->c.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
+@@ -513,7 +517,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_Z8_C calculated =%d\n"
+@@ -529,7 +533,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->c.cstate_pstate.cstate_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_Z8_C calculated =%d\n"
+@@ -546,7 +550,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_D calculated =%d\n"
+@@ -562,7 +566,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->d.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_exit_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
+@@ -578,7 +582,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_Z8_D calculated =%d\n"
+@@ -594,7 +598,7 @@ static bool hubbub31_program_stutter_watermarks(
+ watermarks->d.cstate_pstate.cstate_exit_z8_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_exit_z8_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_Z8_D calculated =%d\n"
+@@ -625,7 +629,7 @@ static bool hubbub31_program_pstate_watermarks(
+ watermarks->a.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.pstate_change_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
+@@ -642,7 +646,7 @@ static bool hubbub31_program_pstate_watermarks(
+ watermarks->b.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.pstate_change_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n"
+@@ -659,7 +663,7 @@ static bool hubbub31_program_pstate_watermarks(
+ watermarks->c.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.pstate_change_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n"
+@@ -676,7 +680,7 @@ static bool hubbub31_program_pstate_watermarks(
+ watermarks->d.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.pstate_change_ns,
+- refclk_mhz, 0x1fffff);
++ refclk_mhz, 0xffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
+@@ -946,6 +950,65 @@ static void hubbub31_get_dchub_ref_freq(struct hubbub *hubbub,
+ }
+ }
+
++static bool hubbub31_verify_allow_pstate_change_high(struct hubbub *hubbub)
++{
++ struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
++
++ /*
++ * Pstate latency is ~20us so if we wait over 40us and pstate allow
++ * still not asserted, we are probably stuck and going to hang
++ */
++ const unsigned int pstate_wait_timeout_us = 100;
++ const unsigned int pstate_wait_expected_timeout_us = 40;
++
++ static unsigned int max_sampled_pstate_wait_us; /* data collection */
++ static bool forced_pstate_allow; /* help with revert wa */
++
++ unsigned int debug_data = 0;
++ unsigned int i;
++
++ if (forced_pstate_allow) {
++ /* we hacked to force pstate allow to prevent hang last time
++ * we verify_allow_pstate_change_high. so disable force
++ * here so we can check status
++ */
++ REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
++ DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 0,
++ DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 0);
++ forced_pstate_allow = false;
++ }
++
++ REG_WRITE(DCHUBBUB_TEST_DEBUG_INDEX, hubbub2->debug_test_index_pstate);
++
++ for (i = 0; i < pstate_wait_timeout_us; i++) {
++ debug_data = REG_READ(DCHUBBUB_TEST_DEBUG_DATA);
++
++ /* Debug bit is specific to ASIC. */
++ if (debug_data & (1 << 26)) {
++ if (i > pstate_wait_expected_timeout_us)
++ DC_LOG_WARNING("pstate took longer than expected ~%dus\n", i);
++ return true;
++ }
++ if (max_sampled_pstate_wait_us < i)
++ max_sampled_pstate_wait_us = i;
++
++ udelay(1);
++ }
++
++ /* force pstate allow to prevent system hang
++ * and break to debugger to investigate
++ */
++ REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
++ DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 1,
++ DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 1);
++ forced_pstate_allow = true;
++
++ DC_LOG_WARNING("pstate TEST_DEBUG_DATA: 0x%X\n",
++ debug_data);
++
++ return false;
++}
++
+ static const struct hubbub_funcs hubbub31_funcs = {
+ .update_dchub = hubbub2_update_dchub,
+ .init_dchub_sys_ctx = hubbub31_init_dchub_sys_ctx,
+@@ -958,6 +1021,7 @@ static const struct hubbub_funcs hubbub31_funcs = {
+ .program_watermarks = hubbub31_program_watermarks,
+ .allow_self_refresh_control = hubbub1_allow_self_refresh_control,
+ .is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
++ .verify_allow_pstate_change_high = hubbub31_verify_allow_pstate_change_high,
+ .program_det_size = dcn31_program_det_size,
+ .program_compbuf_size = dcn31_program_compbuf_size,
+ .init_crb = dcn31_init_crb,
+@@ -979,5 +1043,7 @@ void hubbub31_construct(struct dcn20_hubbub *hubbub31,
+ hubbub31->detile_buf_size = det_size_kb * 1024;
+ hubbub31->pixel_chunk_size = pixel_chunk_size_kb * 1024;
+ hubbub31->crb_size_segs = config_return_buffer_size_kb / DCN31_CRB_SEGMENT_SIZE_KB;
++
++ hubbub31->debug_test_index_pstate = 0x6;
+ }
+
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
+index 3afa1159a5f7d..b72d080b302a1 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
+@@ -204,6 +204,9 @@ void dcn31_init_hw(struct dc *dc)
+ }
+ }
+
++ if (hws->funcs.enable_power_gating_plane)
++ hws->funcs.enable_power_gating_plane(dc->hwseq, true);
++
+ /* If taking control over from VBIOS, we may want to optimize our first
+ * mode set, so we need to skip powering down pipes until we know which
+ * pipes we want to use.
+@@ -287,8 +290,6 @@ void dcn31_init_hw(struct dc *dc)
+
+ REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+ }
+- if (hws->funcs.enable_power_gating_plane)
+- hws->funcs.enable_power_gating_plane(dc->hwseq, true);
+
+ if (!dcb->funcs->is_accelerated_mode(dcb) && dc->res_pool->hubbub->funcs->init_watermarks)
+ dc->res_pool->hubbub->funcs->init_watermarks(dc->res_pool->hubbub);
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
+index 40011cd3c8ef0..4e9fe090b770a 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
+@@ -100,6 +100,8 @@ static const struct hw_sequencer_funcs dcn31_funcs = {
+ .z10_save_init = dcn31_z10_save_init,
+ .is_abm_supported = dcn31_is_abm_supported,
+ .set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
++ .optimize_pwr_state = dcn21_optimize_pwr_state,
++ .exit_optimized_pwr_state = dcn21_exit_optimized_pwr_state,
+ .update_visual_confirm_color = dcn20_update_visual_confirm_color,
+ };
+
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
+index 79e92ecca96c1..a5ef9d5e7685f 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
+@@ -352,6 +352,14 @@ static const struct dce110_clk_src_regs clk_src_regs[] = {
+ clk_src_regs(3, D),
+ clk_src_regs(4, E)
+ };
++/*pll_id being rempped in dmub, in driver it is logical instance*/
++static const struct dce110_clk_src_regs clk_src_regs_b0[] = {
++ clk_src_regs(0, A),
++ clk_src_regs(1, B),
++ clk_src_regs(2, F),
++ clk_src_regs(3, G),
++ clk_src_regs(4, E)
++};
+
+ static const struct dce110_clk_src_shift cs_shift = {
+ CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+@@ -462,7 +470,8 @@ static const struct dcn30_afmt_mask afmt_mask = {
+ SE_DCN3_REG_LIST(id)\
+ }
+
+-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
++/* Some encoders won't be initialized here - but they're logical, not physical. */
++static const struct dcn10_stream_enc_registers stream_enc_regs[ENGINE_ID_COUNT] = {
+ stream_enc_regs(0),
+ stream_enc_regs(1),
+ stream_enc_regs(2),
+@@ -923,7 +932,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .timing_trace = false,
+ .clock_trace = true,
+ .disable_pplib_clock_request = false,
+- .pipe_split_policy = MPC_SPLIT_AVOID,
++ .pipe_split_policy = MPC_SPLIT_DYNAMIC,
+ .force_single_disp_pipe_split = false,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+@@ -931,7 +940,7 @@ static const struct dc_debug_options debug_defaults_drv = {
+ .max_downscale_src_width = 4096,/*upto true 4K*/
+ .disable_pplib_wm_range = false,
+ .scl_reset_length10 = true,
+- .sanity_checks = false,
++ .sanity_checks = true,
+ .underflow_assert_delay_us = 0xFFFFFFFF,
+ .dwb_fi_phase = -1, // -1 = disable,
+ .dmub_command_table = true,
+@@ -2019,14 +2028,27 @@ static bool dcn31_resource_construct(
+ dcn30_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL1,
+ &clk_src_regs[1], false);
+- pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
++ /*move phypllx_pixclk_resync to dmub next*/
++ if (dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0) {
++ pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
++ dcn30_clock_source_create(ctx, ctx->dc_bios,
++ CLOCK_SOURCE_COMBO_PHY_PLL2,
++ &clk_src_regs_b0[2], false);
++ pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
++ dcn30_clock_source_create(ctx, ctx->dc_bios,
++ CLOCK_SOURCE_COMBO_PHY_PLL3,
++ &clk_src_regs_b0[3], false);
++ } else {
++ pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+ dcn30_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL2,
+ &clk_src_regs[2], false);
+- pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
++ pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+ dcn30_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL3,
+ &clk_src_regs[3], false);
++ }
++
+ pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
+ dcn30_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL4,
+diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h
+index 93571c9769967..cc4bed675588c 100644
+--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h
++++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h
+@@ -39,4 +39,35 @@ struct resource_pool *dcn31_create_resource_pool(
+ const struct dc_init_data *init_data,
+ struct dc *dc);
+
++/*temp: B0 specific before switch to dcn313 headers*/
++#ifndef regPHYPLLF_PIXCLK_RESYNC_CNTL
++#define regPHYPLLF_PIXCLK_RESYNC_CNTL 0x007e
++#define regPHYPLLF_PIXCLK_RESYNC_CNTL_BASE_IDX 1
++#define regPHYPLLG_PIXCLK_RESYNC_CNTL 0x005f
++#define regPHYPLLG_PIXCLK_RESYNC_CNTL_BASE_IDX 1
++
++//PHYPLLF_PIXCLK_RESYNC_CNTL
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE__SHIFT 0x8
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE_MASK 0x00000100L
++#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
++
++//PHYPLLG_PIXCLK_RESYNC_CNTL
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE__SHIFT 0x8
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE_MASK 0x00000100L
++#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
++#endif
+ #endif /* _DCN31_RESOURCE_H_ */
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
+index 56055df2e8d2e..9009b92490f34 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
++++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
+@@ -70,6 +70,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_mode_vba_30.o := $(dml_ccflags) $(fram
+ CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_ccflags)
+ CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_mode_vba_31.o := $(dml_ccflags) $(frame_warn_flag)
+ CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_rq_dlg_calc_31.o := $(dml_ccflags)
++CFLAGS_$(AMDDALPATH)/dc/dml/dsc/rc_calc_fpu.o := $(dml_ccflags)
+ CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_ccflags)
+ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_rcflags)
+ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn2x/dcn2x.o := $(dml_rcflags)
+@@ -84,6 +85,7 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_rcfla
+ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn31/display_mode_vba_31.o := $(dml_rcflags)
+ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn31/display_rq_dlg_calc_31.o := $(dml_rcflags)
+ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_rcflags)
++CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dsc/rc_calc_fpu.o := $(dml_rcflags)
+ endif
+ CFLAGS_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_ccflags)
+ CFLAGS_$(AMDDALPATH)/dc/dml/display_rq_dlg_helpers.o := $(dml_ccflags)
+@@ -99,6 +101,7 @@ DML += dcn20/display_rq_dlg_calc_20v2.o dcn20/display_mode_vba_20v2.o
+ DML += dcn21/display_rq_dlg_calc_21.o dcn21/display_mode_vba_21.o
+ DML += dcn30/display_mode_vba_30.o dcn30/display_rq_dlg_calc_30.o
+ DML += dcn31/display_mode_vba_31.o dcn31/display_rq_dlg_calc_31.o
++DML += dsc/rc_calc_fpu.o
+ endif
+
+ AMD_DAL_DML = $(addprefix $(AMDDALPATH)/dc/dml/,$(DML))
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
+index 2091dd8c252da..8c168f348a27f 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
+@@ -768,12 +768,12 @@ static void dml20_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+
+ void dml20_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ display_rq_params_st rq_param = {0};
+
+ memset(rq_regs, 0, sizeof(*rq_regs));
+- dml20_rq_dlg_get_rq_params(mode_lib, &rq_param, pipe_param.src);
++ dml20_rq_dlg_get_rq_params(mode_lib, &rq_param, pipe_param->src);
+ extract_rq_regs(mode_lib, rq_regs, rq_param);
+
+ print__rq_regs_st(mode_lib, *rq_regs);
+@@ -1549,7 +1549,7 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
+ void dml20_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h
+index d0b90947f5409..8b23867e97c18 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h
+@@ -43,7 +43,7 @@ struct display_mode_lib;
+ void dml20_rq_dlg_get_rq_reg(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+
+
+ // Function: dml_rq_dlg_get_dlg_reg
+@@ -61,7 +61,7 @@ void dml20_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
+index 1a0c14e465faa..26ececfd40cdc 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
+@@ -768,12 +768,12 @@ static void dml20v2_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+
+ void dml20v2_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ display_rq_params_st rq_param = {0};
+
+ memset(rq_regs, 0, sizeof(*rq_regs));
+- dml20v2_rq_dlg_get_rq_params(mode_lib, &rq_param, pipe_param.src);
++ dml20v2_rq_dlg_get_rq_params(mode_lib, &rq_param, pipe_param->src);
+ extract_rq_regs(mode_lib, rq_regs, rq_param);
+
+ print__rq_regs_st(mode_lib, *rq_regs);
+@@ -1550,7 +1550,7 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
+ void dml20v2_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.h b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.h
+index 27cf8bed9376f..2b4e46ea1c3df 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.h
+@@ -43,7 +43,7 @@ struct display_mode_lib;
+ void dml20v2_rq_dlg_get_rq_reg(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+
+
+ // Function: dml_rq_dlg_get_dlg_reg
+@@ -61,7 +61,7 @@ void dml20v2_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
+index 287e31052b307..736978c4d40a1 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
+@@ -694,7 +694,7 @@ static void get_surf_rq_param(
+ display_data_rq_sizing_params_st *rq_sizing_param,
+ display_data_rq_dlg_params_st *rq_dlg_param,
+ display_data_rq_misc_params_st *rq_misc_param,
+- const display_pipe_params_st pipe_param,
++ const display_pipe_params_st *pipe_param,
+ bool is_chroma)
+ {
+ bool mode_422 = false;
+@@ -706,30 +706,30 @@ static void get_surf_rq_param(
+
+ // FIXME check if ppe apply for both luma and chroma in 422 case
+ if (is_chroma) {
+- vp_width = pipe_param.src.viewport_width_c / ppe;
+- vp_height = pipe_param.src.viewport_height_c;
+- data_pitch = pipe_param.src.data_pitch_c;
+- meta_pitch = pipe_param.src.meta_pitch_c;
++ vp_width = pipe_param->src.viewport_width_c / ppe;
++ vp_height = pipe_param->src.viewport_height_c;
++ data_pitch = pipe_param->src.data_pitch_c;
++ meta_pitch = pipe_param->src.meta_pitch_c;
+ } else {
+- vp_width = pipe_param.src.viewport_width / ppe;
+- vp_height = pipe_param.src.viewport_height;
+- data_pitch = pipe_param.src.data_pitch;
+- meta_pitch = pipe_param.src.meta_pitch;
++ vp_width = pipe_param->src.viewport_width / ppe;
++ vp_height = pipe_param->src.viewport_height;
++ data_pitch = pipe_param->src.data_pitch;
++ meta_pitch = pipe_param->src.meta_pitch;
+ }
+
+- if (pipe_param.dest.odm_combine) {
++ if (pipe_param->dest.odm_combine) {
+ unsigned int access_dir;
+ unsigned int full_src_vp_width;
+ unsigned int hactive_half;
+ unsigned int src_hactive_half;
+- access_dir = (pipe_param.src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
+- hactive_half = pipe_param.dest.hactive / 2;
++ access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
++ hactive_half = pipe_param->dest.hactive / 2;
+ if (is_chroma) {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio_c * pipe_param.dest.full_recout_width;
+- src_hactive_half = pipe_param.scale_ratio_depth.hscl_ratio_c * hactive_half;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
++ src_hactive_half = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_half;
+ } else {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio * pipe_param.dest.full_recout_width;
+- src_hactive_half = pipe_param.scale_ratio_depth.hscl_ratio * hactive_half;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
++ src_hactive_half = pipe_param->scale_ratio_depth.hscl_ratio * hactive_half;
+ }
+
+ if (access_dir == 0) {
+@@ -754,7 +754,7 @@ static void get_surf_rq_param(
+ rq_sizing_param->meta_chunk_bytes = 2048;
+ rq_sizing_param->min_meta_chunk_bytes = 256;
+
+- if (pipe_param.src.hostvm)
++ if (pipe_param->src.hostvm)
+ rq_sizing_param->mpte_group_bytes = 512;
+ else
+ rq_sizing_param->mpte_group_bytes = 2048;
+@@ -768,23 +768,23 @@ static void get_surf_rq_param(
+ vp_height,
+ data_pitch,
+ meta_pitch,
+- pipe_param.src.source_format,
+- pipe_param.src.sw_mode,
+- pipe_param.src.macro_tile_size,
+- pipe_param.src.source_scan,
+- pipe_param.src.hostvm,
++ pipe_param->src.source_format,
++ pipe_param->src.sw_mode,
++ pipe_param->src.macro_tile_size,
++ pipe_param->src.source_scan,
++ pipe_param->src.hostvm,
+ is_chroma);
+ }
+
+ static void dml_rq_dlg_get_rq_params(
+ struct display_mode_lib *mode_lib,
+ display_rq_params_st *rq_param,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ // get param for luma surface
+- rq_param->yuv420 = pipe_param.src.source_format == dm_420_8
+- || pipe_param.src.source_format == dm_420_10;
+- rq_param->yuv420_10bpc = pipe_param.src.source_format == dm_420_10;
++ rq_param->yuv420 = pipe_param->src.source_format == dm_420_8
++ || pipe_param->src.source_format == dm_420_10;
++ rq_param->yuv420_10bpc = pipe_param->src.source_format == dm_420_10;
+
+ get_surf_rq_param(
+ mode_lib,
+@@ -794,7 +794,7 @@ static void dml_rq_dlg_get_rq_params(
+ pipe_param,
+ 0);
+
+- if (is_dual_plane((enum source_format_class) (pipe_param.src.source_format))) {
++ if (is_dual_plane((enum source_format_class) (pipe_param->src.source_format))) {
+ // get param for chroma surface
+ get_surf_rq_param(
+ mode_lib,
+@@ -806,14 +806,14 @@ static void dml_rq_dlg_get_rq_params(
+ }
+
+ // calculate how to split the det buffer space between luma and chroma
+- handle_det_buf_split(mode_lib, rq_param, pipe_param.src);
++ handle_det_buf_split(mode_lib, rq_param, pipe_param->src);
+ print__rq_params_st(mode_lib, *rq_param);
+ }
+
+ void dml21_rq_dlg_get_rq_reg(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ display_rq_params_st rq_param = {0};
+
+@@ -1658,7 +1658,7 @@ void dml21_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+@@ -1696,7 +1696,7 @@ void dml21_rq_dlg_get_dlg_reg(
+ // system parameter calculation done
+
+ dml_print("DML_DLG: Calculation for pipe[%d] start\n\n", pipe_idx);
+- dml_rq_dlg_get_rq_params(mode_lib, &rq_param, e2e_pipe_param[pipe_idx].pipe);
++ dml_rq_dlg_get_rq_params(mode_lib, &rq_param, &e2e_pipe_param[pipe_idx].pipe);
+ dml_rq_dlg_get_dlg_params(
+ mode_lib,
+ e2e_pipe_param,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.h b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.h
+index e8f7785e3fc63..af6ad0ca9cf8a 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.h
+@@ -44,7 +44,7 @@ struct display_mode_lib;
+ void dml21_rq_dlg_get_rq_reg(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+
+ // Function: dml_rq_dlg_get_dlg_reg
+ // Calculate and return DLG and TTU register struct given the system setting
+@@ -61,7 +61,7 @@ void dml21_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
+index 0d934fae1c3a6..2120e0941a095 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
+@@ -747,7 +747,7 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
+ display_data_rq_sizing_params_st *rq_sizing_param,
+ display_data_rq_dlg_params_st *rq_dlg_param,
+ display_data_rq_misc_params_st *rq_misc_param,
+- const display_pipe_params_st pipe_param,
++ const display_pipe_params_st *pipe_param,
+ bool is_chroma,
+ bool is_alpha)
+ {
+@@ -761,32 +761,32 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
+
+ // FIXME check if ppe apply for both luma and chroma in 422 case
+ if (is_chroma | is_alpha) {
+- vp_width = pipe_param.src.viewport_width_c / ppe;
+- vp_height = pipe_param.src.viewport_height_c;
+- data_pitch = pipe_param.src.data_pitch_c;
+- meta_pitch = pipe_param.src.meta_pitch_c;
+- surface_height = pipe_param.src.surface_height_y / 2.0;
++ vp_width = pipe_param->src.viewport_width_c / ppe;
++ vp_height = pipe_param->src.viewport_height_c;
++ data_pitch = pipe_param->src.data_pitch_c;
++ meta_pitch = pipe_param->src.meta_pitch_c;
++ surface_height = pipe_param->src.surface_height_y / 2.0;
+ } else {
+- vp_width = pipe_param.src.viewport_width / ppe;
+- vp_height = pipe_param.src.viewport_height;
+- data_pitch = pipe_param.src.data_pitch;
+- meta_pitch = pipe_param.src.meta_pitch;
+- surface_height = pipe_param.src.surface_height_y;
++ vp_width = pipe_param->src.viewport_width / ppe;
++ vp_height = pipe_param->src.viewport_height;
++ data_pitch = pipe_param->src.data_pitch;
++ meta_pitch = pipe_param->src.meta_pitch;
++ surface_height = pipe_param->src.surface_height_y;
+ }
+
+- if (pipe_param.dest.odm_combine) {
++ if (pipe_param->dest.odm_combine) {
+ unsigned int access_dir = 0;
+ unsigned int full_src_vp_width = 0;
+ unsigned int hactive_odm = 0;
+ unsigned int src_hactive_odm = 0;
+- access_dir = (pipe_param.src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
+- hactive_odm = pipe_param.dest.hactive / ((unsigned int)pipe_param.dest.odm_combine*2);
++ access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
++ hactive_odm = pipe_param->dest.hactive / ((unsigned int) pipe_param->dest.odm_combine*2);
+ if (is_chroma) {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio_c * pipe_param.dest.full_recout_width;
+- src_hactive_odm = pipe_param.scale_ratio_depth.hscl_ratio_c * hactive_odm;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
++ src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm;
+ } else {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio * pipe_param.dest.full_recout_width;
+- src_hactive_odm = pipe_param.scale_ratio_depth.hscl_ratio * hactive_odm;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
++ src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm;
+ }
+
+ if (access_dir == 0) {
+@@ -815,7 +815,7 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
+ rq_sizing_param->meta_chunk_bytes = 2048;
+ rq_sizing_param->min_meta_chunk_bytes = 256;
+
+- if (pipe_param.src.hostvm)
++ if (pipe_param->src.hostvm)
+ rq_sizing_param->mpte_group_bytes = 512;
+ else
+ rq_sizing_param->mpte_group_bytes = 2048;
+@@ -828,28 +828,28 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
+ vp_height,
+ data_pitch,
+ meta_pitch,
+- pipe_param.src.source_format,
+- pipe_param.src.sw_mode,
+- pipe_param.src.macro_tile_size,
+- pipe_param.src.source_scan,
+- pipe_param.src.hostvm,
++ pipe_param->src.source_format,
++ pipe_param->src.sw_mode,
++ pipe_param->src.macro_tile_size,
++ pipe_param->src.source_scan,
++ pipe_param->src.hostvm,
+ is_chroma,
+ surface_height);
+ }
+
+ static void dml_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+ display_rq_params_st *rq_param,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ // get param for luma surface
+- rq_param->yuv420 = pipe_param.src.source_format == dm_420_8
+- || pipe_param.src.source_format == dm_420_10
+- || pipe_param.src.source_format == dm_rgbe_alpha
+- || pipe_param.src.source_format == dm_420_12;
++ rq_param->yuv420 = pipe_param->src.source_format == dm_420_8
++ || pipe_param->src.source_format == dm_420_10
++ || pipe_param->src.source_format == dm_rgbe_alpha
++ || pipe_param->src.source_format == dm_420_12;
+
+- rq_param->yuv420_10bpc = pipe_param.src.source_format == dm_420_10;
++ rq_param->yuv420_10bpc = pipe_param->src.source_format == dm_420_10;
+
+- rq_param->rgbe_alpha = (pipe_param.src.source_format == dm_rgbe_alpha)?1:0;
++ rq_param->rgbe_alpha = (pipe_param->src.source_format == dm_rgbe_alpha)?1:0;
+
+ get_surf_rq_param(mode_lib,
+ &(rq_param->sizing.rq_l),
+@@ -859,7 +859,7 @@ static void dml_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+ 0,
+ 0);
+
+- if (is_dual_plane((enum source_format_class)(pipe_param.src.source_format))) {
++ if (is_dual_plane((enum source_format_class)(pipe_param->src.source_format))) {
+ // get param for chroma surface
+ get_surf_rq_param(mode_lib,
+ &(rq_param->sizing.rq_c),
+@@ -871,13 +871,13 @@ static void dml_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+ }
+
+ // calculate how to split the det buffer space between luma and chroma
+- handle_det_buf_split(mode_lib, rq_param, pipe_param.src);
++ handle_det_buf_split(mode_lib, rq_param, pipe_param->src);
+ print__rq_params_st(mode_lib, *rq_param);
+ }
+
+ void dml30_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param)
++ const display_pipe_params_st *pipe_param)
+ {
+ display_rq_params_st rq_param = { 0 };
+
+@@ -1831,7 +1831,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
+ void dml30_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+@@ -1866,7 +1866,7 @@ void dml30_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ // system parameter calculation done
+
+ dml_print("DML_DLG: Calculation for pipe[%d] start\n\n", pipe_idx);
+- dml_rq_dlg_get_rq_params(mode_lib, &rq_param, e2e_pipe_param[pipe_idx].pipe);
++ dml_rq_dlg_get_rq_params(mode_lib, &rq_param, &e2e_pipe_param[pipe_idx].pipe);
+ dml_rq_dlg_get_dlg_params(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.h b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.h
+index c04965cceff35..625e41f8d5751 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.h
+@@ -41,7 +41,7 @@ struct display_mode_lib;
+ // See also: <display_rq_regs_st>
+ void dml30_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+
+ // Function: dml_rq_dlg_get_dlg_reg
+ // Calculate and return DLG and TTU register struct given the system setting
+@@ -57,7 +57,7 @@ void dml30_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ void dml30_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
+index c23905bc733ae..57bd4e3f8a823 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
+@@ -738,7 +738,7 @@ static void get_surf_rq_param(
+ display_data_rq_sizing_params_st *rq_sizing_param,
+ display_data_rq_dlg_params_st *rq_dlg_param,
+ display_data_rq_misc_params_st *rq_misc_param,
+- const display_pipe_params_st pipe_param,
++ const display_pipe_params_st *pipe_param,
+ bool is_chroma,
+ bool is_alpha)
+ {
+@@ -752,33 +752,33 @@ static void get_surf_rq_param(
+
+ // FIXME check if ppe apply for both luma and chroma in 422 case
+ if (is_chroma | is_alpha) {
+- vp_width = pipe_param.src.viewport_width_c / ppe;
+- vp_height = pipe_param.src.viewport_height_c;
+- data_pitch = pipe_param.src.data_pitch_c;
+- meta_pitch = pipe_param.src.meta_pitch_c;
+- surface_height = pipe_param.src.surface_height_y / 2.0;
++ vp_width = pipe_param->src.viewport_width_c / ppe;
++ vp_height = pipe_param->src.viewport_height_c;
++ data_pitch = pipe_param->src.data_pitch_c;
++ meta_pitch = pipe_param->src.meta_pitch_c;
++ surface_height = pipe_param->src.surface_height_y / 2.0;
+ } else {
+- vp_width = pipe_param.src.viewport_width / ppe;
+- vp_height = pipe_param.src.viewport_height;
+- data_pitch = pipe_param.src.data_pitch;
+- meta_pitch = pipe_param.src.meta_pitch;
+- surface_height = pipe_param.src.surface_height_y;
++ vp_width = pipe_param->src.viewport_width / ppe;
++ vp_height = pipe_param->src.viewport_height;
++ data_pitch = pipe_param->src.data_pitch;
++ meta_pitch = pipe_param->src.meta_pitch;
++ surface_height = pipe_param->src.surface_height_y;
+ }
+
+- if (pipe_param.dest.odm_combine) {
++ if (pipe_param->dest.odm_combine) {
+ unsigned int access_dir;
+ unsigned int full_src_vp_width;
+ unsigned int hactive_odm;
+ unsigned int src_hactive_odm;
+
+- access_dir = (pipe_param.src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
+- hactive_odm = pipe_param.dest.hactive / ((unsigned int) pipe_param.dest.odm_combine * 2);
++ access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
++ hactive_odm = pipe_param->dest.hactive / ((unsigned int) pipe_param->dest.odm_combine * 2);
+ if (is_chroma) {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio_c * pipe_param.dest.full_recout_width;
+- src_hactive_odm = pipe_param.scale_ratio_depth.hscl_ratio_c * hactive_odm;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
++ src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm;
+ } else {
+- full_src_vp_width = pipe_param.scale_ratio_depth.hscl_ratio * pipe_param.dest.full_recout_width;
+- src_hactive_odm = pipe_param.scale_ratio_depth.hscl_ratio * hactive_odm;
++ full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
++ src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm;
+ }
+
+ if (access_dir == 0) {
+@@ -808,7 +808,7 @@ static void get_surf_rq_param(
+ rq_sizing_param->meta_chunk_bytes = 2048;
+ rq_sizing_param->min_meta_chunk_bytes = 256;
+
+- if (pipe_param.src.hostvm)
++ if (pipe_param->src.hostvm)
+ rq_sizing_param->mpte_group_bytes = 512;
+ else
+ rq_sizing_param->mpte_group_bytes = 2048;
+@@ -822,38 +822,38 @@ static void get_surf_rq_param(
+ vp_height,
+ data_pitch,
+ meta_pitch,
+- pipe_param.src.source_format,
+- pipe_param.src.sw_mode,
+- pipe_param.src.macro_tile_size,
+- pipe_param.src.source_scan,
+- pipe_param.src.hostvm,
++ pipe_param->src.source_format,
++ pipe_param->src.sw_mode,
++ pipe_param->src.macro_tile_size,
++ pipe_param->src.source_scan,
++ pipe_param->src.hostvm,
+ is_chroma,
+ surface_height);
+ }
+
+-static void dml_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib, display_rq_params_st *rq_param, const display_pipe_params_st pipe_param)
++static void dml_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib, display_rq_params_st *rq_param, const display_pipe_params_st *pipe_param)
+ {
+ // get param for luma surface
+- rq_param->yuv420 = pipe_param.src.source_format == dm_420_8 || pipe_param.src.source_format == dm_420_10 || pipe_param.src.source_format == dm_rgbe_alpha
+- || pipe_param.src.source_format == dm_420_12;
++ rq_param->yuv420 = pipe_param->src.source_format == dm_420_8 || pipe_param->src.source_format == dm_420_10 || pipe_param->src.source_format == dm_rgbe_alpha
++ || pipe_param->src.source_format == dm_420_12;
+
+- rq_param->yuv420_10bpc = pipe_param.src.source_format == dm_420_10;
++ rq_param->yuv420_10bpc = pipe_param->src.source_format == dm_420_10;
+
+- rq_param->rgbe_alpha = (pipe_param.src.source_format == dm_rgbe_alpha) ? 1 : 0;
++ rq_param->rgbe_alpha = (pipe_param->src.source_format == dm_rgbe_alpha) ? 1 : 0;
+
+ get_surf_rq_param(mode_lib, &(rq_param->sizing.rq_l), &(rq_param->dlg.rq_l), &(rq_param->misc.rq_l), pipe_param, 0, 0);
+
+- if (is_dual_plane((enum source_format_class) (pipe_param.src.source_format))) {
++ if (is_dual_plane((enum source_format_class) (pipe_param->src.source_format))) {
+ // get param for chroma surface
+ get_surf_rq_param(mode_lib, &(rq_param->sizing.rq_c), &(rq_param->dlg.rq_c), &(rq_param->misc.rq_c), pipe_param, 1, rq_param->rgbe_alpha);
+ }
+
+ // calculate how to split the det buffer space between luma and chroma
+- handle_det_buf_split(mode_lib, rq_param, pipe_param.src);
++ handle_det_buf_split(mode_lib, rq_param, pipe_param->src);
+ print__rq_params_st(mode_lib, *rq_param);
+ }
+
+-void dml31_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib, display_rq_regs_st *rq_regs, const display_pipe_params_st pipe_param)
++void dml31_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib, display_rq_regs_st *rq_regs, const display_pipe_params_st *pipe_param)
+ {
+ display_rq_params_st rq_param = {0};
+
+@@ -1677,7 +1677,7 @@ void dml31_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+@@ -1704,7 +1704,7 @@ void dml31_rq_dlg_get_dlg_reg(
+ // system parameter calculation done
+
+ dml_print("DML_DLG: Calculation for pipe[%d] start\n\n", pipe_idx);
+- dml_rq_dlg_get_rq_params(mode_lib, &rq_param, e2e_pipe_param[pipe_idx].pipe);
++ dml_rq_dlg_get_rq_params(mode_lib, &rq_param, &e2e_pipe_param[pipe_idx].pipe);
+ dml_rq_dlg_get_dlg_params(
+ mode_lib,
+ e2e_pipe_param,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.h b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.h
+index adf8518f761f9..8ee991351699d 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.h
+@@ -41,7 +41,7 @@ struct display_mode_lib;
+ // See also: <display_rq_regs_st>
+ void dml31_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+
+ // Function: dml_rq_dlg_get_dlg_reg
+ // Calculate and return DLG and TTU register struct given the system setting
+@@ -57,7 +57,7 @@ void dml31_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ void dml31_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
+index 1051ca1a23b8a..edb9f7567d6d9 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
+@@ -80,11 +80,11 @@ enum dm_swizzle_mode {
+ dm_sw_SPARE_13 = 24,
+ dm_sw_64kb_s_x = 25,
+ dm_sw_64kb_d_x = 26,
+- dm_sw_SPARE_14 = 27,
++ dm_sw_64kb_r_x = 27,
+ dm_sw_SPARE_15 = 28,
+ dm_sw_var_s_x = 29,
+ dm_sw_var_d_x = 30,
+- dm_sw_64kb_r_x,
++ dm_sw_var_r_x = 31,
+ dm_sw_gfx7_2d_thin_l_vp,
+ dm_sw_gfx7_2d_thin_gl,
+ };
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
+index d42a0aeca6be2..72b1957022aa2 100644
+--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
++++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
+@@ -49,7 +49,7 @@ struct dml_funcs {
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+- display_e2e_pipe_params_st *e2e_pipe_param,
++ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+@@ -60,7 +60,7 @@ struct dml_funcs {
+ void (*rq_dlg_get_rq_reg)(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+- const display_pipe_params_st pipe_param);
++ const display_pipe_params_st *pipe_param);
+ void (*recalculate)(struct display_mode_lib *mode_lib);
+ void (*validate)(struct display_mode_lib *mode_lib);
+ };
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h
+new file mode 100644
+index 0000000000000..e5fac9f4181d8
+--- /dev/null
++++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h
+@@ -0,0 +1,704 @@
++
++/*
++ * Copyright 2017 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ * Authors: AMD
++ *
++ */
++
++
++const qp_table qp_table_422_10bpc_min = {
++ { 6, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
++ { 6.5, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
++ { 7, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 11, 15} },
++ { 7.5, { 0, 2, 4, 6, 6, 6, 6, 7, 7, 7, 8, 9, 9, 11, 15} },
++ { 8, { 0, 2, 3, 5, 5, 6, 6, 7, 7, 7, 8, 8, 9, 11, 14} },
++ { 8.5, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 14} },
++ { 9, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
++ { 9.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
++ { 10, { 0, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
++ {10.5, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
++ { 11, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
++ {11.5, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 10, 11} },
++ { 12, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 10} },
++ {12.5, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
++ { 13, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 6, 8, 8, 9} },
++ {13.5, { 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 6, 7, 8, 9} },
++ { 14, { 0, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 6, 7, 7, 8} },
++ {14.5, { 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 8} },
++ { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 6, 8} },
++ {15.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
++ { 16, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
++ {16.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
++ { 17, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 6} },
++ {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
++ { 18, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 5} },
++ {18.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
++ { 19, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 4} },
++ {19.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} },
++ { 20, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 3} }
++};
++
++
++const qp_table qp_table_444_8bpc_max = {
++ { 6, { 4, 6, 8, 8, 9, 9, 9, 10, 11, 12, 12, 12, 12, 13, 15} },
++ { 6.5, { 4, 6, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 12, 13, 15} },
++ { 7, { 4, 5, 7, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 13, 14} },
++ { 7.5, { 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
++ { 8, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
++ { 8.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
++ { 9, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
++ { 9.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
++ { 10, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
++ {10.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 10, 11, 12} },
++ { 11, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11} },
++ {11.5, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
++ { 12, { 2, 3, 4, 5, 6, 6, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
++ {12.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
++ { 13, { 1, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 10} },
++ {13.5, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
++ { 14, { 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
++ {14.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9} },
++ { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
++ {15.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
++ { 16, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
++ {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
++ { 17, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
++ {17.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
++ { 18, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
++ {18.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
++ { 19, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
++ {19.5, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
++ { 20, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
++ {20.5, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
++ { 21, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
++ {21.5, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
++ { 22, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
++ {22.5, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
++ { 23, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
++ {23.5, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
++ { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4} }
++};
++
++
++const qp_table qp_table_420_12bpc_max = {
++ { 4, {11, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 21, 22} },
++ { 4.5, {10, 11, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
++ { 5, { 9, 11, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 19, 20, 21} },
++ { 5.5, { 8, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
++ { 6, { 6, 9, 11, 12, 13, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
++ { 6.5, { 6, 8, 10, 11, 11, 13, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
++ { 7, { 5, 7, 9, 10, 10, 12, 13, 14, 14, 15, 16, 16, 17, 17, 18} },
++ { 7.5, { 5, 7, 8, 9, 9, 11, 12, 13, 14, 14, 15, 15, 16, 16, 17} },
++ { 8, { 4, 6, 7, 8, 8, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
++ { 8.5, { 3, 6, 6, 7, 7, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
++ { 9, { 3, 5, 6, 7, 7, 10, 11, 12, 12, 13, 13, 14, 14, 14, 15} },
++ { 9.5, { 2, 5, 6, 6, 7, 9, 10, 11, 12, 12, 13, 13, 13, 14, 15} },
++ { 10, { 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 13, 13, 15} },
++ {10.5, { 2, 3, 5, 5, 6, 7, 8, 9, 11, 11, 12, 12, 12, 12, 14} },
++ { 11, { 1, 3, 4, 5, 6, 6, 7, 9, 10, 11, 11, 11, 12, 12, 13} },
++ {11.5, { 1, 2, 3, 4, 5, 6, 6, 8, 9, 10, 10, 11, 11, 11, 13} },
++ { 12, { 1, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10, 10, 12} },
++ {12.5, { 1, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
++ { 13, { 1, 1, 1, 2, 4, 4, 6, 6, 7, 8, 8, 9, 9, 9, 11} },
++ {13.5, { 1, 1, 1, 2, 3, 4, 5, 5, 6, 7, 8, 8, 8, 9, 11} },
++ { 14, { 1, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
++ {14.5, { 0, 1, 1, 1, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
++ { 15, { 0, 1, 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 6, 7, 9} },
++ {15.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
++ { 16, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 7} },
++ {16.5, { 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 7} },
++ { 17, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
++ {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
++ { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} }
++};
++
++
++const qp_table qp_table_444_10bpc_min = {
++ { 6, { 0, 4, 7, 7, 9, 9, 9, 9, 9, 10, 10, 10, 10, 12, 18} },
++ { 6.5, { 0, 4, 6, 7, 8, 8, 9, 9, 9, 9, 10, 10, 10, 12, 18} },
++ { 7, { 0, 4, 6, 6, 8, 8, 8, 8, 8, 9, 9, 10, 10, 12, 17} },
++ { 7.5, { 0, 4, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 10, 12, 17} },
++ { 8, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
++ { 8.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
++ { 9, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
++ { 9.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
++ { 10, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
++ {10.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
++ { 11, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
++ {11.5, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
++ { 12, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
++ {12.5, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 8, 9, 9, 11, 14} },
++ { 13, { 0, 2, 4, 4, 5, 6, 7, 7, 7, 7, 8, 9, 9, 11, 13} },
++ {13.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 13} },
++ { 14, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 9, 9, 11, 13} },
++ {14.5, { 0, 2, 3, 4, 5, 5, 6, 6, 6, 7, 7, 8, 9, 11, 12} },
++ { 15, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
++ {15.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
++ { 16, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
++ {16.5, { 0, 1, 2, 3, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
++ { 17, { 0, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 9, 11} },
++ {17.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 11} },
++ { 18, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
++ {18.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
++ { 19, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
++ {19.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
++ { 20, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 9} },
++ {20.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 9} },
++ { 21, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 7, 9} },
++ {21.5, { 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8} },
++ { 22, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
++ {22.5, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
++ { 23, { 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
++ {23.5, { 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
++ { 24, { 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 5, 7} },
++ {24.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
++ { 25, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
++ {25.5, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
++ { 26, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 5} },
++ {26.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
++ { 27, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
++ {27.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
++ { 28, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} },
++ {28.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 4} },
++ { 29, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3} },
++ {29.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3} },
++ { 30, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
++};
++
++
++const qp_table qp_table_420_8bpc_max = {
++ { 4, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 13, 14} },
++ { 4.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
++ { 5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 12, 13} },
++ { 5.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
++ { 6, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
++ { 6.5, { 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
++ { 7, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 9, 10} },
++ { 7.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9} },
++ { 8, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
++ { 8.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
++ { 9, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7} },
++ { 9.5, { 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
++ { 10, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6} },
++ {10.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
++ { 11, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5} },
++ {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
++ { 12, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} }
++};
++
++
++const qp_table qp_table_444_8bpc_min = {
++ { 6, { 0, 1, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 9, 14} },
++ { 6.5, { 0, 1, 2, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 9, 14} },
++ { 7, { 0, 0, 2, 2, 4, 4, 4, 4, 4, 5, 5, 6, 6, 9, 13} },
++ { 7.5, { 0, 0, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 9, 13} },
++ { 8, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
++ { 8.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
++ { 9, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
++ { 9.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
++ { 10, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
++ {10.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
++ { 11, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
++ {11.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
++ { 12, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
++ {12.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
++ { 13, { 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 4, 5, 5, 7, 9} },
++ {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
++ { 14, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
++ {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
++ { 15, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
++ {15.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
++ { 16, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
++ {16.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
++ { 17, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
++ {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
++ { 18, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
++ {18.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
++ { 19, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
++ {19.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
++ { 20, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
++ {20.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
++ { 21, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
++ {21.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
++ { 22, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
++ {22.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
++ { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
++ {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
++ { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3} }
++};
++
++
++const qp_table qp_table_444_12bpc_min = {
++ { 6, { 0, 5, 11, 11, 13, 13, 13, 13, 13, 14, 14, 14, 14, 17, 22} },
++ { 6.5, { 0, 5, 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 17, 22} },
++ { 7, { 0, 5, 10, 10, 12, 12, 12, 12, 12, 13, 13, 14, 14, 17, 21} },
++ { 7.5, { 0, 5, 9, 10, 11, 12, 12, 12, 12, 12, 13, 13, 14, 17, 21} },
++ { 8, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
++ { 8.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
++ { 9, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
++ { 9.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
++ { 10, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
++ {10.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
++ { 11, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ {11.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ { 12, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ {12.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ { 13, { 0, 4, 7, 8, 9, 11, 11, 11, 11, 11, 13, 13, 13, 15, 17} },
++ {13.5, { 0, 3, 6, 7, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 17} },
++ { 14, { 0, 3, 5, 6, 9, 9, 9, 10, 11, 11, 12, 13, 13, 15, 17} },
++ {14.5, { 0, 2, 5, 6, 8, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ { 15, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ {15.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ { 16, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
++ {16.5, { 0, 2, 3, 5, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
++ { 17, { 0, 2, 3, 5, 5, 6, 9, 9, 10, 10, 11, 11, 12, 13, 15} },
++ {17.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 15} },
++ { 18, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
++ {18.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
++ { 19, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ {19.5, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ { 20, { 0, 1, 2, 3, 4, 5, 7, 8, 8, 8, 9, 10, 10, 11, 13} },
++ {20.5, { 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 13} },
++ { 21, { 0, 1, 2, 3, 4, 5, 5, 7, 7, 8, 9, 10, 10, 11, 13} },
++ {21.5, { 0, 1, 2, 3, 3, 4, 5, 7, 7, 8, 9, 10, 10, 11, 12} },
++ { 22, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 12} },
++ {22.5, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 11} },
++ { 23, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
++ {23.5, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
++ { 24, { 0, 0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 8, 8, 9, 11} },
++ {24.5, { 0, 0, 1, 2, 3, 4, 4, 6, 6, 7, 8, 8, 8, 9, 11} },
++ { 25, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 8, 8, 8, 10} },
++ {25.5, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
++ { 26, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 7, 7, 9} },
++ {26.5, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 7, 9} },
++ { 27, { 0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
++ {27.5, { 0, 0, 1, 1, 2, 2, 4, 4, 4, 5, 6, 7, 7, 7, 9} },
++ { 28, { 0, 0, 0, 1, 1, 2, 3, 4, 4, 4, 6, 6, 6, 7, 9} },
++ {28.5, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
++ { 29, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 8} },
++ {29.5, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7} },
++ { 30, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 5, 5, 5, 5, 7} },
++ {30.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
++ { 31, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
++ {31.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
++ { 32, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
++ {32.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 6} },
++ { 33, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
++ {33.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
++ { 34, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 5} },
++ {34.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 5} },
++ { 35, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
++ {35.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4} },
++ { 36, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
++};
++
++
++const qp_table qp_table_420_12bpc_min = {
++ { 4, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 21} },
++ { 4.5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
++ { 5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
++ { 5.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
++ { 6, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ { 6.5, { 0, 4, 6, 8, 9, 10, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ { 7, { 0, 3, 5, 7, 9, 10, 10, 11, 11, 11, 13, 13, 13, 15, 17} },
++ { 7.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 16} },
++ { 8, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ { 8.5, { 0, 2, 4, 6, 6, 9, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
++ { 9, { 0, 2, 4, 6, 6, 9, 9, 10, 10, 11, 11, 12, 13, 13, 14} },
++ { 9.5, { 0, 2, 4, 5, 6, 8, 8, 9, 10, 10, 11, 12, 12, 13, 14} },
++ { 10, { 0, 2, 3, 5, 6, 7, 8, 8, 9, 10, 10, 12, 12, 12, 14} },
++ {10.5, { 0, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 11, 11, 13} },
++ { 11, { 0, 2, 3, 4, 5, 5, 6, 8, 8, 9, 9, 10, 11, 11, 12} },
++ {11.5, { 0, 1, 2, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 10, 12} },
++ { 12, { 0, 0, 2, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
++ {12.5, { 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 8, 8, 9, 10} },
++ { 13, { 0, 0, 0, 1, 3, 3, 5, 5, 6, 7, 7, 8, 8, 8, 10} },
++ {13.5, { 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 7, 8, 10} },
++ { 14, { 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7, 7, 9} },
++ {14.5, { 0, 0, 0, 0, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
++ { 15, { 0, 0, 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 8} },
++ {15.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
++ { 16, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 6} },
++ {16.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
++ { 17, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
++ {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
++ { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} }
++};
++
++
++const qp_table qp_table_422_12bpc_min = {
++ { 6, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
++ { 6.5, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
++ { 7, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
++ { 7.5, { 0, 4, 8, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
++ { 8, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
++ { 8.5, { 0, 3, 6, 8, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 18} },
++ { 9, { 0, 3, 5, 8, 9, 10, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
++ { 9.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
++ { 10, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ {10.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
++ { 11, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
++ {11.5, { 0, 2, 4, 6, 7, 7, 9, 9, 10, 11, 11, 12, 12, 14, 15} },
++ { 12, { 0, 2, 4, 6, 6, 6, 8, 8, 9, 9, 11, 11, 12, 13, 14} },
++ {12.5, { 0, 1, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 13, 14} },
++ { 13, { 0, 1, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 11, 12, 13} },
++ {13.5, { 0, 1, 3, 3, 4, 5, 7, 7, 8, 8, 10, 10, 10, 12, 13} },
++ { 14, { 0, 0, 2, 3, 4, 5, 6, 6, 7, 7, 9, 10, 10, 11, 12} },
++ {14.5, { 0, 0, 1, 3, 4, 4, 6, 6, 6, 7, 9, 9, 9, 11, 12} },
++ { 15, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 6, 8, 9, 9, 10, 12} },
++ {15.5, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 10, 11} },
++ { 16, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 9, 11} },
++ {16.5, { 0, 0, 0, 2, 2, 3, 4, 5, 5, 5, 6, 7, 7, 9, 10} },
++ { 17, { 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 6, 6, 6, 8, 10} },
++ {17.5, { 0, 0, 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 8, 9} },
++ { 18, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 9} },
++ {18.5, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 3, 5, 5, 5, 7, 9} },
++ { 19, { 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6, 8} },
++ {19.5, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 6, 8} },
++ { 20, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
++ {20.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 7} },
++ { 21, { 0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
++ {21.5, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
++ { 22, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 6} },
++ {22.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} },
++ { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 5} },
++ {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} },
++ { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} }
++};
++
++
++const qp_table qp_table_422_12bpc_max = {
++ { 6, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
++ { 6.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
++ { 7, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
++ { 7.5, { 9, 10, 12, 14, 15, 15, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
++ { 8, { 6, 9, 10, 12, 14, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
++ { 8.5, { 6, 8, 9, 11, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
++ { 9, { 5, 7, 8, 10, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
++ { 9.5, { 5, 7, 7, 9, 10, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
++ { 10, { 4, 6, 6, 8, 9, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
++ {10.5, { 4, 6, 6, 8, 9, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
++ { 11, { 4, 5, 6, 8, 9, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
++ {11.5, { 3, 5, 6, 8, 9, 9, 11, 11, 12, 13, 13, 14, 14, 15, 16} },
++ { 12, { 3, 5, 6, 8, 8, 8, 10, 10, 11, 11, 13, 13, 14, 14, 15} },
++ {12.5, { 3, 4, 6, 7, 8, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15} },
++ { 13, { 2, 4, 5, 6, 7, 7, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
++ {13.5, { 2, 4, 5, 5, 6, 7, 9, 9, 10, 10, 12, 12, 12, 13, 14} },
++ { 14, { 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 11, 12, 12, 12, 13} },
++ {14.5, { 2, 3, 3, 5, 6, 6, 8, 8, 8, 9, 11, 11, 11, 12, 13} },
++ { 15, { 2, 3, 3, 5, 5, 6, 7, 8, 8, 8, 10, 11, 11, 11, 13} },
++ {15.5, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 11, 12} },
++ { 16, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 10, 12} },
++ {16.5, { 1, 2, 2, 4, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 11} },
++ { 17, { 1, 1, 2, 3, 4, 4, 6, 6, 6, 7, 8, 8, 8, 9, 11} },
++ {17.5, { 1, 1, 2, 3, 4, 4, 5, 6, 6, 6, 7, 8, 8, 9, 10} },
++ { 18, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 7, 7, 8, 8, 10} },
++ {18.5, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 5, 7, 7, 7, 8, 10} },
++ { 19, { 1, 1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 7, 9} },
++ {19.5, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 7, 9} },
++ { 20, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 6, 8} },
++ {20.5, { 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
++ { 21, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
++ {21.5, { 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
++ { 22, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 7} },
++ {22.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
++ { 23, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
++ {23.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
++ { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} }
++};
++
++
++const qp_table qp_table_444_12bpc_max = {
++ { 6, {12, 14, 16, 16, 17, 17, 17, 18, 19, 20, 20, 20, 20, 21, 23} },
++ { 6.5, {12, 14, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 20, 21, 23} },
++ { 7, {12, 13, 15, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 21, 22} },
++ { 7.5, {12, 13, 14, 15, 15, 16, 16, 17, 18, 18, 19, 19, 20, 21, 22} },
++ { 8, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
++ { 8.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
++ { 9, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
++ { 9.5, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
++ { 10, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
++ {10.5, {10, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 18, 19, 20} },
++ { 11, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 18, 18, 19} },
++ {11.5, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
++ { 12, { 6, 9, 12, 13, 14, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
++ {12.5, { 6, 9, 12, 13, 14, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
++ { 13, { 5, 9, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, 16, 17, 18} },
++ {13.5, { 5, 8, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
++ { 14, { 5, 8, 10, 11, 12, 12, 12, 13, 14, 14, 15, 16, 16, 16, 18} },
++ {14.5, { 4, 7, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 17} },
++ { 15, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
++ {15.5, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
++ { 16, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
++ {16.5, { 4, 5, 7, 8, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
++ { 17, { 4, 5, 7, 8, 8, 9, 11, 11, 12, 12, 12, 13, 13, 14, 16} },
++ {17.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 16} },
++ { 18, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
++ {18.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
++ { 19, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
++ {19.5, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
++ { 20, { 2, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11, 11, 12, 14} },
++ {20.5, { 2, 3, 5, 5, 7, 8, 8, 8, 9, 10, 10, 11, 11, 12, 14} },
++ { 21, { 2, 3, 5, 5, 7, 7, 7, 8, 8, 9, 10, 11, 11, 12, 14} },
++ {21.5, { 2, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 12, 13} },
++ { 22, { 2, 2, 4, 5, 6, 6, 7, 7, 8, 9, 10, 10, 10, 11, 13} },
++ {22.5, { 2, 2, 4, 5, 5, 6, 7, 7, 8, 9, 10, 10, 10, 11, 12} },
++ { 23, { 2, 2, 4, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
++ {23.5, { 2, 2, 3, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
++ { 24, { 2, 2, 3, 4, 4, 5, 7, 7, 7, 8, 9, 9, 9, 10, 12} },
++ {24.5, { 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 12} },
++ { 25, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 9, 9, 11} },
++ {25.5, { 1, 1, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 11} },
++ { 26, { 1, 1, 3, 3, 3, 4, 5, 6, 6, 7, 8, 8, 8, 8, 10} },
++ {26.5, { 1, 1, 2, 3, 3, 4, 5, 6, 6, 6, 8, 8, 8, 8, 10} },
++ { 27, { 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 10} },
++ {27.5, { 1, 1, 2, 2, 3, 3, 5, 5, 5, 6, 7, 8, 8, 8, 10} },
++ { 28, { 0, 1, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 8, 10} },
++ {28.5, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
++ { 29, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 9} },
++ {29.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8} },
++ { 30, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 6, 6, 6, 6, 8} },
++ {30.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
++ { 31, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
++ {31.5, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
++ { 32, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 7} },
++ {32.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 7} },
++ { 33, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
++ {33.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
++ { 34, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
++ {34.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 6} },
++ { 35, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
++ {35.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 5} },
++ { 36, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} }
++};
++
++
++const qp_table qp_table_420_8bpc_min = {
++ { 4, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 9, 13} },
++ { 4.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
++ { 5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
++ { 5.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
++ { 6, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
++ { 6.5, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
++ { 7, { 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
++ { 7.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
++ { 8, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
++ { 8.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
++ { 9, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6} },
++ { 9.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
++ { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5} },
++ {10.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
++ { 11, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
++ {11.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
++ { 12, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 3} }
++};
++
++
++const qp_table qp_table_422_8bpc_min = {
++ { 6, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
++ { 6.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
++ { 7, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
++ { 7.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
++ { 8, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
++ { 8.5, { 0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 7, 10} },
++ { 9, { 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
++ { 9.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 9} },
++ { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
++ {10.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
++ { 11, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
++ {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
++ { 12, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6} },
++ {12.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
++ { 13, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5} },
++ {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 5} },
++ { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4} },
++ {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} },
++ { 15, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 4} },
++ {15.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
++ { 16, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} }
++};
++
++
++const qp_table qp_table_422_10bpc_max = {
++ { 6, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
++ { 6.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
++ { 7, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
++ { 7.5, { 5, 6, 8, 10, 11, 11, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
++ { 8, { 4, 6, 7, 9, 10, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
++ { 8.5, { 4, 5, 6, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
++ { 9, { 3, 4, 5, 7, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
++ { 9.5, { 3, 4, 4, 6, 6, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
++ { 10, { 2, 3, 3, 5, 5, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ {10.5, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ { 11, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
++ {11.5, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 9, 9, 10, 10, 11, 12} },
++ { 12, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 8, 9, 9, 10, 10, 11} },
++ {12.5, { 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
++ { 13, { 1, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10} },
++ {13.5, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 8, 9, 10} },
++ { 14, { 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 8, 9} },
++ {14.5, { 1, 2, 2, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9} },
++ { 15, { 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 9} },
++ {15.5, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
++ { 16, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 8} },
++ {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 6, 7} },
++ { 17, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 7} },
++ {17.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
++ { 18, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 6} },
++ {18.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 6} },
++ { 19, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 5} },
++ {19.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 5} },
++ { 20, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 4} }
++};
++
++
++const qp_table qp_table_420_10bpc_max = {
++ { 4, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 17, 18} },
++ { 4.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
++ { 5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 16, 17} },
++ { 5.5, { 6, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
++ { 6, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
++ { 6.5, { 4, 5, 7, 8, 8, 9, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
++ { 7, { 3, 4, 6, 7, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
++ { 7.5, { 3, 4, 5, 6, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13} },
++ { 8, { 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ { 8.5, { 1, 3, 3, 4, 4, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
++ { 9, { 1, 3, 3, 4, 4, 6, 7, 8, 8, 9, 9, 10, 10, 10, 11} },
++ { 9.5, { 1, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
++ { 10, { 1, 2, 3, 3, 4, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
++ {10.5, { 1, 1, 3, 3, 3, 4, 5, 5, 7, 7, 8, 8, 8, 8, 10} },
++ { 11, { 0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 7, 7, 8, 8, 9} },
++ {11.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 7, 9} },
++ { 12, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 8} },
++ {12.5, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
++ { 13, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 7} },
++ {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
++ { 14, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
++ {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 5} },
++ { 15, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} }
++};
++
++
++const qp_table qp_table_420_10bpc_min = {
++ { 4, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 13, 17} },
++ { 4.5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
++ { 5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
++ { 5.5, { 0, 3, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 15} },
++ { 6, { 0, 2, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
++ { 6.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 14} },
++ { 7, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 11, 13} },
++ { 7.5, { 0, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 9, 11, 12} },
++ { 8, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
++ { 8.5, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
++ { 9, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 7, 8, 9, 9, 10} },
++ { 9.5, { 0, 2, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10} },
++ { 10, { 0, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 8, 8, 8, 10} },
++ {10.5, { 0, 0, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
++ { 11, { 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8} },
++ {11.5, { 0, 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 6, 6, 6, 8} },
++ { 12, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 5, 7} },
++ {12.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
++ { 13, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
++ {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
++ { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
++ {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
++ { 15, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} }
++};
++
++
++const qp_table qp_table_444_10bpc_max = {
++ { 6, { 8, 10, 12, 12, 13, 13, 13, 14, 15, 16, 16, 16, 16, 17, 19} },
++ { 6.5, { 8, 10, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 16, 17, 19} },
++ { 7, { 8, 9, 11, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 17, 18} },
++ { 7.5, { 8, 9, 10, 11, 11, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
++ { 8, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
++ { 8.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
++ { 9, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
++ { 9.5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
++ { 10, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
++ {10.5, { 6, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 14, 15, 16} },
++ { 11, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 14, 14, 15} },
++ {11.5, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
++ { 12, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
++ {12.5, { 4, 6, 8, 9, 10, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
++ { 13, { 3, 6, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, 13, 14} },
++ {13.5, { 3, 5, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
++ { 14, { 3, 5, 6, 7, 8, 8, 8, 9, 10, 10, 11, 12, 12, 12, 14} },
++ {14.5, { 2, 4, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 13} },
++ { 15, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ {15.5, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
++ { 16, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
++ {16.5, { 2, 3, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
++ { 17, { 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10, 12} },
++ {17.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 12} },
++ { 18, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
++ {18.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
++ { 19, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
++ {19.5, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
++ { 20, { 1, 2, 3, 3, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 10} },
++ {20.5, { 1, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 10} },
++ { 21, { 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7, 7, 8, 10} },
++ {21.5, { 1, 2, 3, 3, 3, 3, 4, 5, 5, 5, 6, 7, 7, 8, 9} },
++ { 22, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 9} },
++ {22.5, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
++ { 23, { 1, 1, 2, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
++ {23.5, { 1, 1, 1, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
++ { 24, { 1, 1, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 6, 8} },
++ {24.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 8} },
++ { 25, { 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
++ {25.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
++ { 26, { 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 4, 6} },
++ {26.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 6} },
++ { 27, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
++ {27.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
++ { 28, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
++ {28.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
++ { 29, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4} },
++ {29.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
++ { 30, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} }
++};
++
++
++const qp_table qp_table_422_8bpc_max = {
++ { 6, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
++ { 6.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
++ { 7, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
++ { 7.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
++ { 8, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
++ { 8.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
++ { 9, { 1, 2, 3, 4, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
++ { 9.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 10} },
++ { 10, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
++ {10.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
++ { 11, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
++ {11.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8} },
++ { 12, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7} },
++ {12.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7} },
++ { 13, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6} },
++ {13.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
++ { 14, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5} },
++ {14.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5} },
++ { 15, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 5} },
++ {15.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
++ { 16, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} }
++};
++
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
+new file mode 100644
+index 0000000000000..3ee858f311d12
+--- /dev/null
++++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
+@@ -0,0 +1,291 @@
++/*
++ * Copyright 2021 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ * Authors: AMD
++ *
++ */
++
++#include "rc_calc_fpu.h"
++
++#include "qp_tables.h"
++#include "amdgpu_dm/dc_fpu.h"
++
++#define table_hash(mode, bpc, max_min) ((mode << 16) | (bpc << 8) | max_min)
++
++#define MODE_SELECT(val444, val422, val420) \
++ (cm == CM_444 || cm == CM_RGB) ? (val444) : (cm == CM_422 ? (val422) : (val420))
++
++
++#define TABLE_CASE(mode, bpc, max) case (table_hash(mode, BPC_##bpc, max)): \
++ table = qp_table_##mode##_##bpc##bpc_##max; \
++ table_size = sizeof(qp_table_##mode##_##bpc##bpc_##max)/sizeof(*qp_table_##mode##_##bpc##bpc_##max); \
++ break
++
++static int median3(int a, int b, int c)
++{
++ if (a > b)
++ swap(a, b);
++ if (b > c)
++ swap(b, c);
++ if (a > b)
++ swap(b, c);
++
++ return b;
++}
++
++static double dsc_roundf(double num)
++{
++ if (num < 0.0)
++ num = num - 0.5;
++ else
++ num = num + 0.5;
++
++ return (int)(num);
++}
++
++static double dsc_ceil(double num)
++{
++ double retval = (int)num;
++
++ if (retval != num && num > 0)
++ retval = num + 1;
++
++ return (int)retval;
++}
++
++static void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc,
++ enum max_min max_min, float bpp)
++{
++ int mode = MODE_SELECT(444, 422, 420);
++ int sel = table_hash(mode, bpc, max_min);
++ int table_size = 0;
++ int index;
++ const struct qp_entry *table = 0L;
++
++ // alias enum
++ enum { min = DAL_MM_MIN, max = DAL_MM_MAX };
++ switch (sel) {
++ TABLE_CASE(444, 8, max);
++ TABLE_CASE(444, 8, min);
++ TABLE_CASE(444, 10, max);
++ TABLE_CASE(444, 10, min);
++ TABLE_CASE(444, 12, max);
++ TABLE_CASE(444, 12, min);
++ TABLE_CASE(422, 8, max);
++ TABLE_CASE(422, 8, min);
++ TABLE_CASE(422, 10, max);
++ TABLE_CASE(422, 10, min);
++ TABLE_CASE(422, 12, max);
++ TABLE_CASE(422, 12, min);
++ TABLE_CASE(420, 8, max);
++ TABLE_CASE(420, 8, min);
++ TABLE_CASE(420, 10, max);
++ TABLE_CASE(420, 10, min);
++ TABLE_CASE(420, 12, max);
++ TABLE_CASE(420, 12, min);
++ }
++
++ if (table == 0)
++ return;
++
++ index = (bpp - table[0].bpp) * 2;
++
++ /* requested size is bigger than the table */
++ if (index >= table_size) {
++ dm_error("ERROR: Requested rc_calc to find a bpp entry that exceeds the table size\n");
++ return;
++ }
++
++ memcpy(qps, table[index].qps, sizeof(qp_set));
++}
++
++static void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
++{
++ int *p = ofs;
++
++ if (mode == CM_444 || mode == CM_RGB) {
++ *p++ = (bpp <= 6) ? (0) : ((((bpp >= 8) && (bpp <= 12))) ? (2) : ((bpp >= 15) ? (10) : ((((bpp > 6) && (bpp < 8))) ? (0 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (2 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
++ *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (8) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
++ *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (6) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
++ *p++ = (bpp <= 6) ? (-4) : ((((bpp >= 8) && (bpp <= 12))) ? (-2) : ((bpp >= 15) ? (4) : ((((bpp > 6) && (bpp < 8))) ? (-4 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-2 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
++ *p++ = (bpp <= 6) ? (-6) : ((((bpp >= 8) && (bpp <= 12))) ? (-4) : ((bpp >= 15) ? (2) : ((((bpp > 6) && (bpp < 8))) ? (-6 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-4 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
++ *p++ = (bpp <= 12) ? (-6) : ((bpp >= 15) ? (0) : (-6 + dsc_roundf((bpp - 12) * (6 / 3.0))));
++ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-2) : (-8 + dsc_roundf((bpp - 12) * (6 / 3.0))));
++ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-4) : (-8 + dsc_roundf((bpp - 12) * (4 / 3.0))));
++ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0))));
++ *p++ = (bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0))));
++ *p++ = -10;
++ *p++ = (bpp <= 6) ? (-12) : ((bpp >= 8) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2 / 2.0))));
++ *p++ = -12;
++ *p++ = -12;
++ *p++ = -12;
++ } else if (mode == CM_422) {
++ *p++ = (bpp <= 8) ? (2) : ((bpp >= 10) ? (10) : (2 + dsc_roundf((bpp - 8) * (8 / 2.0))));
++ *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (8) : (0 + dsc_roundf((bpp - 8) * (8 / 2.0))));
++ *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (6) : (0 + dsc_roundf((bpp - 8) * (6 / 2.0))));
++ *p++ = (bpp <= 8) ? (-2) : ((bpp >= 10) ? (4) : (-2 + dsc_roundf((bpp - 8) * (6 / 2.0))));
++ *p++ = (bpp <= 8) ? (-4) : ((bpp >= 10) ? (2) : (-4 + dsc_roundf((bpp - 8) * (6 / 2.0))));
++ *p++ = (bpp <= 8) ? (-6) : ((bpp >= 10) ? (0) : (-6 + dsc_roundf((bpp - 8) * (6 / 2.0))));
++ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-2) : (-8 + dsc_roundf((bpp - 8) * (6 / 2.0))));
++ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-4) : (-8 + dsc_roundf((bpp - 8) * (4 / 2.0))));
++ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-6) : (-8 + dsc_roundf((bpp - 8) * (2 / 2.0))));
++ *p++ = (bpp <= 8) ? (-10) : ((bpp >= 10) ? (-8) : (-10 + dsc_roundf((bpp - 8) * (2 / 2.0))));
++ *p++ = -10;
++ *p++ = (bpp <= 6) ? (-12) : ((bpp >= 7) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2.0 / 1))));
++ *p++ = -12;
++ *p++ = -12;
++ *p++ = -12;
++ } else {
++ *p++ = (bpp <= 6) ? (2) : ((bpp >= 8) ? (10) : (2 + dsc_roundf((bpp - 6) * (8 / 2.0))));
++ *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (8) : (0 + dsc_roundf((bpp - 6) * (8 / 2.0))));
++ *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (6) : (0 + dsc_roundf((bpp - 6) * (6 / 2.0))));
++ *p++ = (bpp <= 6) ? (-2) : ((bpp >= 8) ? (4) : (-2 + dsc_roundf((bpp - 6) * (6 / 2.0))));
++ *p++ = (bpp <= 6) ? (-4) : ((bpp >= 8) ? (2) : (-4 + dsc_roundf((bpp - 6) * (6 / 2.0))));
++ *p++ = (bpp <= 6) ? (-6) : ((bpp >= 8) ? (0) : (-6 + dsc_roundf((bpp - 6) * (6 / 2.0))));
++ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-2) : (-8 + dsc_roundf((bpp - 6) * (6 / 2.0))));
++ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-4) : (-8 + dsc_roundf((bpp - 6) * (4 / 2.0))));
++ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-6) : (-8 + dsc_roundf((bpp - 6) * (2 / 2.0))));
++ *p++ = (bpp <= 6) ? (-10) : ((bpp >= 8) ? (-8) : (-10 + dsc_roundf((bpp - 6) * (2 / 2.0))));
++ *p++ = -10;
++ *p++ = (bpp <= 4) ? (-12) : ((bpp >= 5) ? (-10) : (-12 + dsc_roundf((bpp - 4) * (2 / 1.0))));
++ *p++ = -12;
++ *p++ = -12;
++ *p++ = -12;
++ }
++}
++
++void _do_calc_rc_params(struct rc_params *rc,
++ enum colour_mode cm,
++ enum bits_per_comp bpc,
++ u16 drm_bpp,
++ bool is_navite_422_or_420,
++ int slice_width,
++ int slice_height,
++ int minor_version)
++{
++ float bpp;
++ float bpp_group;
++ float initial_xmit_delay_factor;
++ int padding_pixels;
++ int i;
++
++ dc_assert_fp_enabled();
++
++ bpp = ((float)drm_bpp / 16.0);
++ /* in native_422 or native_420 modes, the bits_per_pixel is double the
++ * target bpp (the latter is what calc_rc_params expects)
++ */
++ if (is_navite_422_or_420)
++ bpp /= 2.0;
++
++ rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
++ rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
++
++ bpp_group = MODE_SELECT(bpp, bpp * 2.0, bpp * 2.0);
++
++ switch (cm) {
++ case CM_420:
++ rc->initial_fullness_offset = (bpp >= 6) ? (2048) : ((bpp <= 4) ? (6144) : ((((bpp > 4) && (bpp <= 5))) ? (6144 - dsc_roundf((bpp - 4) * (512))) : (5632 - dsc_roundf((bpp - 5) * (3584)))));
++ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 3) - (3 * bpp_group)));
++ rc->second_line_bpg_offset = median3(0, 12, (int)((3 * bpc * 3) - (3 * bpp_group)));
++ break;
++ case CM_422:
++ rc->initial_fullness_offset = (bpp >= 8) ? (2048) : ((bpp <= 7) ? (5632) : (5632 - dsc_roundf((bpp - 7) * (3584))));
++ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 4) - (3 * bpp_group)));
++ rc->second_line_bpg_offset = 0;
++ break;
++ case CM_444:
++ case CM_RGB:
++ rc->initial_fullness_offset = (bpp >= 12) ? (2048) : ((bpp <= 8) ? (6144) : ((((bpp > 8) && (bpp <= 10))) ? (6144 - dsc_roundf((bpp - 8) * (512 / 2))) : (5632 - dsc_roundf((bpp - 10) * (3584 / 2)))));
++ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)(((3 * bpc + (cm == CM_444 ? 0 : 2)) * 3) - (3 * bpp_group)));
++ rc->second_line_bpg_offset = 0;
++ break;
++ }
++
++ initial_xmit_delay_factor = (cm == CM_444 || cm == CM_RGB) ? 1.0 : 2.0;
++ rc->initial_xmit_delay = dsc_roundf(8192.0/2.0/bpp/initial_xmit_delay_factor);
++
++ if (cm == CM_422 || cm == CM_420)
++ slice_width /= 2;
++
++ padding_pixels = ((slice_width % 3) != 0) ? (3 - (slice_width % 3)) * (rc->initial_xmit_delay / slice_width) : 0;
++ if (3 * bpp_group >= (((rc->initial_xmit_delay + 2) / 3) * (3 + (cm == CM_422)))) {
++ if ((rc->initial_xmit_delay + padding_pixels) % 3 == 1)
++ rc->initial_xmit_delay++;
++ }
++
++ rc->flatness_min_qp = ((bpc == BPC_8) ? (3) : ((bpc == BPC_10) ? (7) : (11))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
++ rc->flatness_max_qp = ((bpc == BPC_8) ? (12) : ((bpc == BPC_10) ? (16) : (20))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
++ rc->flatness_det_thresh = 2 << (bpc - 8);
++
++ get_qp_set(rc->qp_min, cm, bpc, DAL_MM_MIN, bpp);
++ get_qp_set(rc->qp_max, cm, bpc, DAL_MM_MAX, bpp);
++ if (cm == CM_444 && minor_version == 1) {
++ for (i = 0; i < QP_SET_SIZE; ++i) {
++ rc->qp_min[i] = rc->qp_min[i] > 0 ? rc->qp_min[i] - 1 : 0;
++ rc->qp_max[i] = rc->qp_max[i] > 0 ? rc->qp_max[i] - 1 : 0;
++ }
++ }
++ get_ofs_set(rc->ofs, cm, bpp);
++
++ /* fixed parameters */
++ rc->rc_model_size = 8192;
++ rc->rc_edge_factor = 6;
++ rc->rc_tgt_offset_hi = 3;
++ rc->rc_tgt_offset_lo = 3;
++
++ rc->rc_buf_thresh[0] = 896;
++ rc->rc_buf_thresh[1] = 1792;
++ rc->rc_buf_thresh[2] = 2688;
++ rc->rc_buf_thresh[3] = 3584;
++ rc->rc_buf_thresh[4] = 4480;
++ rc->rc_buf_thresh[5] = 5376;
++ rc->rc_buf_thresh[6] = 6272;
++ rc->rc_buf_thresh[7] = 6720;
++ rc->rc_buf_thresh[8] = 7168;
++ rc->rc_buf_thresh[9] = 7616;
++ rc->rc_buf_thresh[10] = 7744;
++ rc->rc_buf_thresh[11] = 7872;
++ rc->rc_buf_thresh[12] = 8000;
++ rc->rc_buf_thresh[13] = 8064;
++}
++
++u32 _do_bytes_per_pixel_calc(int slice_width,
++ u16 drm_bpp,
++ bool is_navite_422_or_420)
++{
++ float bpp;
++ u32 bytes_per_pixel;
++ double d_bytes_per_pixel;
++
++ dc_assert_fp_enabled();
++
++ bpp = ((float)drm_bpp / 16.0);
++ d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width;
++ // TODO: Make sure the formula for calculating this is precise (ceiling
++ // vs. floor, and at what point they should be applied)
++ if (is_navite_422_or_420)
++ d_bytes_per_pixel /= 2;
++
++ bytes_per_pixel = (u32)dsc_ceil(d_bytes_per_pixel * 0x10000000);
++
++ return bytes_per_pixel;
++}
+diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.h
+new file mode 100644
+index 0000000000000..b93b95409fbe2
+--- /dev/null
++++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.h
+@@ -0,0 +1,94 @@
++/*
++ * Copyright 2021 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ * Authors: AMD
++ *
++ */
++
++#ifndef __RC_CALC_FPU_H__
++#define __RC_CALC_FPU_H__
++
++#include "os_types.h"
++#include <drm/drm_dsc.h>
++
++#define QP_SET_SIZE 15
++
++typedef int qp_set[QP_SET_SIZE];
++
++struct rc_params {
++ int rc_quant_incr_limit0;
++ int rc_quant_incr_limit1;
++ int initial_fullness_offset;
++ int initial_xmit_delay;
++ int first_line_bpg_offset;
++ int second_line_bpg_offset;
++ int flatness_min_qp;
++ int flatness_max_qp;
++ int flatness_det_thresh;
++ qp_set qp_min;
++ qp_set qp_max;
++ qp_set ofs;
++ int rc_model_size;
++ int rc_edge_factor;
++ int rc_tgt_offset_hi;
++ int rc_tgt_offset_lo;
++ int rc_buf_thresh[QP_SET_SIZE - 1];
++};
++
++enum colour_mode {
++ CM_RGB, /* 444 RGB */
++ CM_444, /* 444 YUV or simple 422 */
++ CM_422, /* native 422 */
++ CM_420 /* native 420 */
++};
++
++enum bits_per_comp {
++ BPC_8 = 8,
++ BPC_10 = 10,
++ BPC_12 = 12
++};
++
++enum max_min {
++ DAL_MM_MIN = 0,
++ DAL_MM_MAX = 1
++};
++
++struct qp_entry {
++ float bpp;
++ const qp_set qps;
++};
++
++typedef struct qp_entry qp_table[];
++
++u32 _do_bytes_per_pixel_calc(int slice_width,
++ u16 drm_bpp,
++ bool is_navite_422_or_420);
++
++void _do_calc_rc_params(struct rc_params *rc,
++ enum colour_mode cm,
++ enum bits_per_comp bpc,
++ u16 drm_bpp,
++ bool is_navite_422_or_420,
++ int slice_width,
++ int slice_height,
++ int minor_version);
++
++#endif
+diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
+index 8d31eb75c6a6e..a2537229ee88b 100644
+--- a/drivers/gpu/drm/amd/display/dc/dsc/Makefile
++++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
+@@ -1,35 +1,6 @@
+ # SPDX-License-Identifier: MIT
+ #
+ # Makefile for the 'dsc' sub-component of DAL.
+-
+-ifdef CONFIG_X86
+-dsc_ccflags := -mhard-float -msse
+-endif
+-
+-ifdef CONFIG_PPC64
+-dsc_ccflags := -mhard-float -maltivec
+-endif
+-
+-ifdef CONFIG_CC_IS_GCC
+-ifeq ($(call cc-ifversion, -lt, 0701, y), y)
+-IS_OLD_GCC = 1
+-endif
+-endif
+-
+-ifdef CONFIG_X86
+-ifdef IS_OLD_GCC
+-# Stack alignment mismatch, proceed with caution.
+-# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
+-# (8B stack alignment).
+-dsc_ccflags += -mpreferred-stack-boundary=4
+-else
+-dsc_ccflags += -msse2
+-endif
+-endif
+-
+-CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_ccflags)
+-CFLAGS_REMOVE_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_rcflags)
+-
+ DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o
+
+ AMD_DAL_DSC = $(addprefix $(AMDDALPATH)/dc/dsc/,$(DSC))
+diff --git a/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h b/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h
+deleted file mode 100644
+index e5fac9f4181d8..0000000000000
+--- a/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h
++++ /dev/null
+@@ -1,704 +0,0 @@
+-
+-/*
+- * Copyright 2017 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- * Authors: AMD
+- *
+- */
+-
+-
+-const qp_table qp_table_422_10bpc_min = {
+- { 6, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
+- { 6.5, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
+- { 7, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 11, 15} },
+- { 7.5, { 0, 2, 4, 6, 6, 6, 6, 7, 7, 7, 8, 9, 9, 11, 15} },
+- { 8, { 0, 2, 3, 5, 5, 6, 6, 7, 7, 7, 8, 8, 9, 11, 14} },
+- { 8.5, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 14} },
+- { 9, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
+- { 9.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
+- { 10, { 0, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+- {10.5, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+- { 11, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
+- {11.5, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 10, 11} },
+- { 12, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 10} },
+- {12.5, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+- { 13, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 6, 8, 8, 9} },
+- {13.5, { 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 6, 7, 8, 9} },
+- { 14, { 0, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 6, 7, 7, 8} },
+- {14.5, { 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 8} },
+- { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 6, 8} },
+- {15.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+- { 16, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
+- {16.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
+- { 17, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 6} },
+- {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+- { 18, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 5} },
+- {18.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+- { 19, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 4} },
+- {19.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} },
+- { 20, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 3} }
+-};
+-
+-
+-const qp_table qp_table_444_8bpc_max = {
+- { 6, { 4, 6, 8, 8, 9, 9, 9, 10, 11, 12, 12, 12, 12, 13, 15} },
+- { 6.5, { 4, 6, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 12, 13, 15} },
+- { 7, { 4, 5, 7, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 13, 14} },
+- { 7.5, { 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+- { 8, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+- { 8.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+- { 9, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
+- { 9.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
+- { 10, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+- {10.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 10, 11, 12} },
+- { 11, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11} },
+- {11.5, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+- { 12, { 2, 3, 4, 5, 6, 6, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+- {12.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+- { 13, { 1, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 10} },
+- {13.5, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
+- { 14, { 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+- {14.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9} },
+- { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+- {15.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+- { 16, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
+- {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
+- { 17, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
+- {17.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
+- { 18, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
+- {18.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
+- { 19, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
+- {19.5, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
+- { 20, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
+- {20.5, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
+- { 21, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+- {21.5, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+- { 22, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
+- {22.5, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
+- { 23, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+- {23.5, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+- { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_420_12bpc_max = {
+- { 4, {11, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 21, 22} },
+- { 4.5, {10, 11, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+- { 5, { 9, 11, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 19, 20, 21} },
+- { 5.5, { 8, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
+- { 6, { 6, 9, 11, 12, 13, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+- { 6.5, { 6, 8, 10, 11, 11, 13, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+- { 7, { 5, 7, 9, 10, 10, 12, 13, 14, 14, 15, 16, 16, 17, 17, 18} },
+- { 7.5, { 5, 7, 8, 9, 9, 11, 12, 13, 14, 14, 15, 15, 16, 16, 17} },
+- { 8, { 4, 6, 7, 8, 8, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+- { 8.5, { 3, 6, 6, 7, 7, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+- { 9, { 3, 5, 6, 7, 7, 10, 11, 12, 12, 13, 13, 14, 14, 14, 15} },
+- { 9.5, { 2, 5, 6, 6, 7, 9, 10, 11, 12, 12, 13, 13, 13, 14, 15} },
+- { 10, { 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 13, 13, 15} },
+- {10.5, { 2, 3, 5, 5, 6, 7, 8, 9, 11, 11, 12, 12, 12, 12, 14} },
+- { 11, { 1, 3, 4, 5, 6, 6, 7, 9, 10, 11, 11, 11, 12, 12, 13} },
+- {11.5, { 1, 2, 3, 4, 5, 6, 6, 8, 9, 10, 10, 11, 11, 11, 13} },
+- { 12, { 1, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10, 10, 12} },
+- {12.5, { 1, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
+- { 13, { 1, 1, 1, 2, 4, 4, 6, 6, 7, 8, 8, 9, 9, 9, 11} },
+- {13.5, { 1, 1, 1, 2, 3, 4, 5, 5, 6, 7, 8, 8, 8, 9, 11} },
+- { 14, { 1, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+- {14.5, { 0, 1, 1, 1, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+- { 15, { 0, 1, 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 6, 7, 9} },
+- {15.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
+- { 16, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 7} },
+- {16.5, { 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 7} },
+- { 17, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+- {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+- { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} }
+-};
+-
+-
+-const qp_table qp_table_444_10bpc_min = {
+- { 6, { 0, 4, 7, 7, 9, 9, 9, 9, 9, 10, 10, 10, 10, 12, 18} },
+- { 6.5, { 0, 4, 6, 7, 8, 8, 9, 9, 9, 9, 10, 10, 10, 12, 18} },
+- { 7, { 0, 4, 6, 6, 8, 8, 8, 8, 8, 9, 9, 10, 10, 12, 17} },
+- { 7.5, { 0, 4, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 10, 12, 17} },
+- { 8, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
+- { 8.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
+- { 9, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+- { 9.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+- { 10, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
+- {10.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
+- { 11, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+- {11.5, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+- { 12, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+- {12.5, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 8, 9, 9, 11, 14} },
+- { 13, { 0, 2, 4, 4, 5, 6, 7, 7, 7, 7, 8, 9, 9, 11, 13} },
+- {13.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 13} },
+- { 14, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 9, 9, 11, 13} },
+- {14.5, { 0, 2, 3, 4, 5, 5, 6, 6, 6, 7, 7, 8, 9, 11, 12} },
+- { 15, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
+- {15.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
+- { 16, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
+- {16.5, { 0, 1, 2, 3, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
+- { 17, { 0, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 9, 11} },
+- {17.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 11} },
+- { 18, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+- {18.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+- { 19, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
+- {19.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
+- { 20, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 9} },
+- {20.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 9} },
+- { 21, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 7, 9} },
+- {21.5, { 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8} },
+- { 22, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
+- {22.5, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+- { 23, { 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
+- {23.5, { 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
+- { 24, { 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 5, 7} },
+- {24.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+- { 25, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+- {25.5, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+- { 26, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 5} },
+- {26.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
+- { 27, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+- {27.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+- { 28, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} },
+- {28.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 4} },
+- { 29, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3} },
+- {29.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3} },
+- { 30, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
+-};
+-
+-
+-const qp_table qp_table_420_8bpc_max = {
+- { 4, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 13, 14} },
+- { 4.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+- { 5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 12, 13} },
+- { 5.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
+- { 6, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+- { 6.5, { 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+- { 7, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 9, 10} },
+- { 7.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9} },
+- { 8, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+- { 8.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
+- { 9, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7} },
+- { 9.5, { 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+- { 10, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6} },
+- {10.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+- { 11, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5} },
+- {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+- { 12, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} }
+-};
+-
+-
+-const qp_table qp_table_444_8bpc_min = {
+- { 6, { 0, 1, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 9, 14} },
+- { 6.5, { 0, 1, 2, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 9, 14} },
+- { 7, { 0, 0, 2, 2, 4, 4, 4, 4, 4, 5, 5, 6, 6, 9, 13} },
+- { 7.5, { 0, 0, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 9, 13} },
+- { 8, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
+- { 8.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
+- { 9, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
+- { 9.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
+- { 10, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+- {10.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+- { 11, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+- {11.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+- { 12, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+- {12.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
+- { 13, { 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 4, 5, 5, 7, 9} },
+- {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+- { 14, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+- {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
+- { 15, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+- {15.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+- { 16, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+- {16.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+- { 17, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+- {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+- { 18, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+- {18.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+- { 19, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
+- {19.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
+- { 20, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+- {20.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+- { 21, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+- {21.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+- { 22, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
+- {22.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
+- { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
+- {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
+- { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3} }
+-};
+-
+-
+-const qp_table qp_table_444_12bpc_min = {
+- { 6, { 0, 5, 11, 11, 13, 13, 13, 13, 13, 14, 14, 14, 14, 17, 22} },
+- { 6.5, { 0, 5, 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 17, 22} },
+- { 7, { 0, 5, 10, 10, 12, 12, 12, 12, 12, 13, 13, 14, 14, 17, 21} },
+- { 7.5, { 0, 5, 9, 10, 11, 12, 12, 12, 12, 12, 13, 13, 14, 17, 21} },
+- { 8, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
+- { 8.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
+- { 9, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+- { 9.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+- { 10, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+- {10.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+- { 11, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- {11.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- { 12, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- {12.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- { 13, { 0, 4, 7, 8, 9, 11, 11, 11, 11, 11, 13, 13, 13, 15, 17} },
+- {13.5, { 0, 3, 6, 7, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 17} },
+- { 14, { 0, 3, 5, 6, 9, 9, 9, 10, 11, 11, 12, 13, 13, 15, 17} },
+- {14.5, { 0, 2, 5, 6, 8, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- { 15, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- {15.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- { 16, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
+- {16.5, { 0, 2, 3, 5, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
+- { 17, { 0, 2, 3, 5, 5, 6, 9, 9, 10, 10, 11, 11, 12, 13, 15} },
+- {17.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 15} },
+- { 18, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+- {18.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+- { 19, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- {19.5, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- { 20, { 0, 1, 2, 3, 4, 5, 7, 8, 8, 8, 9, 10, 10, 11, 13} },
+- {20.5, { 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 13} },
+- { 21, { 0, 1, 2, 3, 4, 5, 5, 7, 7, 8, 9, 10, 10, 11, 13} },
+- {21.5, { 0, 1, 2, 3, 3, 4, 5, 7, 7, 8, 9, 10, 10, 11, 12} },
+- { 22, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 12} },
+- {22.5, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 11} },
+- { 23, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
+- {23.5, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
+- { 24, { 0, 0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 8, 8, 9, 11} },
+- {24.5, { 0, 0, 1, 2, 3, 4, 4, 6, 6, 7, 8, 8, 8, 9, 11} },
+- { 25, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 8, 8, 8, 10} },
+- {25.5, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+- { 26, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 7, 7, 9} },
+- {26.5, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 7, 9} },
+- { 27, { 0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+- {27.5, { 0, 0, 1, 1, 2, 2, 4, 4, 4, 5, 6, 7, 7, 7, 9} },
+- { 28, { 0, 0, 0, 1, 1, 2, 3, 4, 4, 4, 6, 6, 6, 7, 9} },
+- {28.5, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
+- { 29, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 8} },
+- {29.5, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7} },
+- { 30, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 5, 5, 5, 5, 7} },
+- {30.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
+- { 31, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
+- {31.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+- { 32, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
+- {32.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 6} },
+- { 33, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+- {33.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+- { 34, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 5} },
+- {34.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 5} },
+- { 35, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
+- {35.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4} },
+- { 36, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
+-};
+-
+-
+-const qp_table qp_table_420_12bpc_min = {
+- { 4, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 21} },
+- { 4.5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+- { 5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+- { 5.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+- { 6, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- { 6.5, { 0, 4, 6, 8, 9, 10, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- { 7, { 0, 3, 5, 7, 9, 10, 10, 11, 11, 11, 13, 13, 13, 15, 17} },
+- { 7.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 16} },
+- { 8, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- { 8.5, { 0, 2, 4, 6, 6, 9, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
+- { 9, { 0, 2, 4, 6, 6, 9, 9, 10, 10, 11, 11, 12, 13, 13, 14} },
+- { 9.5, { 0, 2, 4, 5, 6, 8, 8, 9, 10, 10, 11, 12, 12, 13, 14} },
+- { 10, { 0, 2, 3, 5, 6, 7, 8, 8, 9, 10, 10, 12, 12, 12, 14} },
+- {10.5, { 0, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 11, 11, 13} },
+- { 11, { 0, 2, 3, 4, 5, 5, 6, 8, 8, 9, 9, 10, 11, 11, 12} },
+- {11.5, { 0, 1, 2, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 10, 12} },
+- { 12, { 0, 0, 2, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
+- {12.5, { 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 8, 8, 9, 10} },
+- { 13, { 0, 0, 0, 1, 3, 3, 5, 5, 6, 7, 7, 8, 8, 8, 10} },
+- {13.5, { 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 7, 8, 10} },
+- { 14, { 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7, 7, 9} },
+- {14.5, { 0, 0, 0, 0, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
+- { 15, { 0, 0, 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 8} },
+- {15.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+- { 16, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 6} },
+- {16.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+- { 17, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+- {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+- { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_422_12bpc_min = {
+- { 6, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
+- { 6.5, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
+- { 7, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+- { 7.5, { 0, 4, 8, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+- { 8, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+- { 8.5, { 0, 3, 6, 8, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 18} },
+- { 9, { 0, 3, 5, 8, 9, 10, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
+- { 9.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
+- { 10, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- {10.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+- { 11, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
+- {11.5, { 0, 2, 4, 6, 7, 7, 9, 9, 10, 11, 11, 12, 12, 14, 15} },
+- { 12, { 0, 2, 4, 6, 6, 6, 8, 8, 9, 9, 11, 11, 12, 13, 14} },
+- {12.5, { 0, 1, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 13, 14} },
+- { 13, { 0, 1, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 11, 12, 13} },
+- {13.5, { 0, 1, 3, 3, 4, 5, 7, 7, 8, 8, 10, 10, 10, 12, 13} },
+- { 14, { 0, 0, 2, 3, 4, 5, 6, 6, 7, 7, 9, 10, 10, 11, 12} },
+- {14.5, { 0, 0, 1, 3, 4, 4, 6, 6, 6, 7, 9, 9, 9, 11, 12} },
+- { 15, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 6, 8, 9, 9, 10, 12} },
+- {15.5, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 10, 11} },
+- { 16, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 9, 11} },
+- {16.5, { 0, 0, 0, 2, 2, 3, 4, 5, 5, 5, 6, 7, 7, 9, 10} },
+- { 17, { 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 6, 6, 6, 8, 10} },
+- {17.5, { 0, 0, 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 8, 9} },
+- { 18, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 9} },
+- {18.5, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 3, 5, 5, 5, 7, 9} },
+- { 19, { 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6, 8} },
+- {19.5, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 6, 8} },
+- { 20, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+- {20.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 7} },
+- { 21, { 0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+- {21.5, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+- { 22, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 6} },
+- {22.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} },
+- { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 5} },
+- {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} },
+- { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_422_12bpc_max = {
+- { 6, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+- { 6.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+- { 7, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
+- { 7.5, { 9, 10, 12, 14, 15, 15, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
+- { 8, { 6, 9, 10, 12, 14, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+- { 8.5, { 6, 8, 9, 11, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+- { 9, { 5, 7, 8, 10, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
+- { 9.5, { 5, 7, 7, 9, 10, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
+- { 10, { 4, 6, 6, 8, 9, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+- {10.5, { 4, 6, 6, 8, 9, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+- { 11, { 4, 5, 6, 8, 9, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+- {11.5, { 3, 5, 6, 8, 9, 9, 11, 11, 12, 13, 13, 14, 14, 15, 16} },
+- { 12, { 3, 5, 6, 8, 8, 8, 10, 10, 11, 11, 13, 13, 14, 14, 15} },
+- {12.5, { 3, 4, 6, 7, 8, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15} },
+- { 13, { 2, 4, 5, 6, 7, 7, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
+- {13.5, { 2, 4, 5, 5, 6, 7, 9, 9, 10, 10, 12, 12, 12, 13, 14} },
+- { 14, { 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 11, 12, 12, 12, 13} },
+- {14.5, { 2, 3, 3, 5, 6, 6, 8, 8, 8, 9, 11, 11, 11, 12, 13} },
+- { 15, { 2, 3, 3, 5, 5, 6, 7, 8, 8, 8, 10, 11, 11, 11, 13} },
+- {15.5, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 11, 12} },
+- { 16, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 10, 12} },
+- {16.5, { 1, 2, 2, 4, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 11} },
+- { 17, { 1, 1, 2, 3, 4, 4, 6, 6, 6, 7, 8, 8, 8, 9, 11} },
+- {17.5, { 1, 1, 2, 3, 4, 4, 5, 6, 6, 6, 7, 8, 8, 9, 10} },
+- { 18, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 7, 7, 8, 8, 10} },
+- {18.5, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 5, 7, 7, 7, 8, 10} },
+- { 19, { 1, 1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 7, 9} },
+- {19.5, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 7, 9} },
+- { 20, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 6, 8} },
+- {20.5, { 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
+- { 21, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
+- {21.5, { 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+- { 22, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 7} },
+- {22.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+- { 23, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+- {23.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+- { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} }
+-};
+-
+-
+-const qp_table qp_table_444_12bpc_max = {
+- { 6, {12, 14, 16, 16, 17, 17, 17, 18, 19, 20, 20, 20, 20, 21, 23} },
+- { 6.5, {12, 14, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 20, 21, 23} },
+- { 7, {12, 13, 15, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 21, 22} },
+- { 7.5, {12, 13, 14, 15, 15, 16, 16, 17, 18, 18, 19, 19, 20, 21, 22} },
+- { 8, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+- { 8.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+- { 9, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
+- { 9.5, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
+- { 10, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
+- {10.5, {10, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 18, 19, 20} },
+- { 11, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 18, 18, 19} },
+- {11.5, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+- { 12, { 6, 9, 12, 13, 14, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+- {12.5, { 6, 9, 12, 13, 14, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+- { 13, { 5, 9, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, 16, 17, 18} },
+- {13.5, { 5, 8, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
+- { 14, { 5, 8, 10, 11, 12, 12, 12, 13, 14, 14, 15, 16, 16, 16, 18} },
+- {14.5, { 4, 7, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 17} },
+- { 15, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+- {15.5, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+- { 16, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
+- {16.5, { 4, 5, 7, 8, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
+- { 17, { 4, 5, 7, 8, 8, 9, 11, 11, 12, 12, 12, 13, 13, 14, 16} },
+- {17.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 16} },
+- { 18, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
+- {18.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
+- { 19, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
+- {19.5, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
+- { 20, { 2, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11, 11, 12, 14} },
+- {20.5, { 2, 3, 5, 5, 7, 8, 8, 8, 9, 10, 10, 11, 11, 12, 14} },
+- { 21, { 2, 3, 5, 5, 7, 7, 7, 8, 8, 9, 10, 11, 11, 12, 14} },
+- {21.5, { 2, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 12, 13} },
+- { 22, { 2, 2, 4, 5, 6, 6, 7, 7, 8, 9, 10, 10, 10, 11, 13} },
+- {22.5, { 2, 2, 4, 5, 5, 6, 7, 7, 8, 9, 10, 10, 10, 11, 12} },
+- { 23, { 2, 2, 4, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
+- {23.5, { 2, 2, 3, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
+- { 24, { 2, 2, 3, 4, 4, 5, 7, 7, 7, 8, 9, 9, 9, 10, 12} },
+- {24.5, { 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 12} },
+- { 25, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 9, 9, 11} },
+- {25.5, { 1, 1, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 11} },
+- { 26, { 1, 1, 3, 3, 3, 4, 5, 6, 6, 7, 8, 8, 8, 8, 10} },
+- {26.5, { 1, 1, 2, 3, 3, 4, 5, 6, 6, 6, 8, 8, 8, 8, 10} },
+- { 27, { 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 10} },
+- {27.5, { 1, 1, 2, 2, 3, 3, 5, 5, 5, 6, 7, 8, 8, 8, 10} },
+- { 28, { 0, 1, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 8, 10} },
+- {28.5, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+- { 29, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 9} },
+- {29.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8} },
+- { 30, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 6, 6, 6, 6, 8} },
+- {30.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
+- { 31, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
+- {31.5, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
+- { 32, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 7} },
+- {32.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 7} },
+- { 33, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+- {33.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+- { 34, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
+- {34.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 6} },
+- { 35, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
+- {35.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 5} },
+- { 36, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_420_8bpc_min = {
+- { 4, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 9, 13} },
+- { 4.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+- { 5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+- { 5.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+- { 6, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+- { 6.5, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
+- { 7, { 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+- { 7.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
+- { 8, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+- { 8.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+- { 9, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6} },
+- { 9.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+- { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5} },
+- {10.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
+- { 11, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
+- {11.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+- { 12, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 3} }
+-};
+-
+-
+-const qp_table qp_table_422_8bpc_min = {
+- { 6, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+- { 6.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+- { 7, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+- { 7.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+- { 8, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+- { 8.5, { 0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 7, 10} },
+- { 9, { 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+- { 9.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 9} },
+- { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+- {10.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+- { 11, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+- {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+- { 12, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6} },
+- {12.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+- { 13, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5} },
+- {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 5} },
+- { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4} },
+- {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} },
+- { 15, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 4} },
+- {15.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
+- { 16, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} }
+-};
+-
+-
+-const qp_table qp_table_422_10bpc_max = {
+- { 6, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+- { 6.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+- { 7, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+- { 7.5, { 5, 6, 8, 10, 11, 11, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
+- { 8, { 4, 6, 7, 9, 10, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+- { 8.5, { 4, 5, 6, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+- { 9, { 3, 4, 5, 7, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
+- { 9.5, { 3, 4, 4, 6, 6, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+- { 10, { 2, 3, 3, 5, 5, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- {10.5, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- { 11, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+- {11.5, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 9, 9, 10, 10, 11, 12} },
+- { 12, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 8, 9, 9, 10, 10, 11} },
+- {12.5, { 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+- { 13, { 1, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10} },
+- {13.5, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 8, 9, 10} },
+- { 14, { 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 8, 9} },
+- {14.5, { 1, 2, 2, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9} },
+- { 15, { 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 9} },
+- {15.5, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
+- { 16, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 8} },
+- {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 6, 7} },
+- { 17, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 7} },
+- {17.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
+- { 18, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 6} },
+- {18.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 6} },
+- { 19, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 5} },
+- {19.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 5} },
+- { 20, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_420_10bpc_max = {
+- { 4, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 17, 18} },
+- { 4.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+- { 5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 16, 17} },
+- { 5.5, { 6, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
+- { 6, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+- { 6.5, { 4, 5, 7, 8, 8, 9, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+- { 7, { 3, 4, 6, 7, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
+- { 7.5, { 3, 4, 5, 6, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13} },
+- { 8, { 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- { 8.5, { 1, 3, 3, 4, 4, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+- { 9, { 1, 3, 3, 4, 4, 6, 7, 8, 8, 9, 9, 10, 10, 10, 11} },
+- { 9.5, { 1, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
+- { 10, { 1, 2, 3, 3, 4, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
+- {10.5, { 1, 1, 3, 3, 3, 4, 5, 5, 7, 7, 8, 8, 8, 8, 10} },
+- { 11, { 0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 7, 7, 8, 8, 9} },
+- {11.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 7, 9} },
+- { 12, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 8} },
+- {12.5, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+- { 13, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 7} },
+- {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+- { 14, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+- {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 5} },
+- { 15, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} }
+-};
+-
+-
+-const qp_table qp_table_420_10bpc_min = {
+- { 4, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 13, 17} },
+- { 4.5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+- { 5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+- { 5.5, { 0, 3, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 15} },
+- { 6, { 0, 2, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+- { 6.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 14} },
+- { 7, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 11, 13} },
+- { 7.5, { 0, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 9, 11, 12} },
+- { 8, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+- { 8.5, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
+- { 9, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 7, 8, 9, 9, 10} },
+- { 9.5, { 0, 2, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10} },
+- { 10, { 0, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 8, 8, 8, 10} },
+- {10.5, { 0, 0, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+- { 11, { 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8} },
+- {11.5, { 0, 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 6, 6, 6, 8} },
+- { 12, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 5, 7} },
+- {12.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
+- { 13, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+- {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
+- { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+- {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+- { 15, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_444_10bpc_max = {
+- { 6, { 8, 10, 12, 12, 13, 13, 13, 14, 15, 16, 16, 16, 16, 17, 19} },
+- { 6.5, { 8, 10, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 16, 17, 19} },
+- { 7, { 8, 9, 11, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 17, 18} },
+- { 7.5, { 8, 9, 10, 11, 11, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
+- { 8, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+- { 8.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+- { 9, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
+- { 9.5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
+- { 10, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+- {10.5, { 6, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 14, 15, 16} },
+- { 11, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 14, 14, 15} },
+- {11.5, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+- { 12, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+- {12.5, { 4, 6, 8, 9, 10, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+- { 13, { 3, 6, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, 13, 14} },
+- {13.5, { 3, 5, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
+- { 14, { 3, 5, 6, 7, 8, 8, 8, 9, 10, 10, 11, 12, 12, 12, 14} },
+- {14.5, { 2, 4, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 13} },
+- { 15, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- {15.5, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+- { 16, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
+- {16.5, { 2, 3, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
+- { 17, { 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10, 12} },
+- {17.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 12} },
+- { 18, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
+- {18.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
+- { 19, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
+- {19.5, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
+- { 20, { 1, 2, 3, 3, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 10} },
+- {20.5, { 1, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 10} },
+- { 21, { 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7, 7, 8, 10} },
+- {21.5, { 1, 2, 3, 3, 3, 3, 4, 5, 5, 5, 6, 7, 7, 8, 9} },
+- { 22, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 9} },
+- {22.5, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
+- { 23, { 1, 1, 2, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
+- {23.5, { 1, 1, 1, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
+- { 24, { 1, 1, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 6, 8} },
+- {24.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 8} },
+- { 25, { 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
+- {25.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
+- { 26, { 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 4, 6} },
+- {26.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 6} },
+- { 27, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+- {27.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+- { 28, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
+- {28.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+- { 29, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4} },
+- {29.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
+- { 30, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} }
+-};
+-
+-
+-const qp_table qp_table_422_8bpc_max = {
+- { 6, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+- { 6.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+- { 7, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+- { 7.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
+- { 8, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+- { 8.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+- { 9, { 1, 2, 3, 4, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
+- { 9.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 10} },
+- { 10, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+- {10.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+- { 11, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
+- {11.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8} },
+- { 12, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7} },
+- {12.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7} },
+- { 13, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6} },
+- {13.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
+- { 14, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5} },
+- {14.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5} },
+- { 15, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 5} },
+- {15.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
+- { 16, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} }
+-};
+-
+diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
+index 7b294f637881a..b19d3aeb5962c 100644
+--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
++++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
+@@ -23,266 +23,7 @@
+ * Authors: AMD
+ *
+ */
+-#include <drm/drm_dsc.h>
+-
+-#include "os_types.h"
+ #include "rc_calc.h"
+-#include "qp_tables.h"
+-
+-#define table_hash(mode, bpc, max_min) ((mode << 16) | (bpc << 8) | max_min)
+-
+-#define MODE_SELECT(val444, val422, val420) \
+- (cm == CM_444 || cm == CM_RGB) ? (val444) : (cm == CM_422 ? (val422) : (val420))
+-
+-
+-#define TABLE_CASE(mode, bpc, max) case (table_hash(mode, BPC_##bpc, max)): \
+- table = qp_table_##mode##_##bpc##bpc_##max; \
+- table_size = sizeof(qp_table_##mode##_##bpc##bpc_##max)/sizeof(*qp_table_##mode##_##bpc##bpc_##max); \
+- break
+-
+-
+-static void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc,
+- enum max_min max_min, float bpp)
+-{
+- int mode = MODE_SELECT(444, 422, 420);
+- int sel = table_hash(mode, bpc, max_min);
+- int table_size = 0;
+- int index;
+- const struct qp_entry *table = 0L;
+-
+- // alias enum
+- enum { min = DAL_MM_MIN, max = DAL_MM_MAX };
+- switch (sel) {
+- TABLE_CASE(444, 8, max);
+- TABLE_CASE(444, 8, min);
+- TABLE_CASE(444, 10, max);
+- TABLE_CASE(444, 10, min);
+- TABLE_CASE(444, 12, max);
+- TABLE_CASE(444, 12, min);
+- TABLE_CASE(422, 8, max);
+- TABLE_CASE(422, 8, min);
+- TABLE_CASE(422, 10, max);
+- TABLE_CASE(422, 10, min);
+- TABLE_CASE(422, 12, max);
+- TABLE_CASE(422, 12, min);
+- TABLE_CASE(420, 8, max);
+- TABLE_CASE(420, 8, min);
+- TABLE_CASE(420, 10, max);
+- TABLE_CASE(420, 10, min);
+- TABLE_CASE(420, 12, max);
+- TABLE_CASE(420, 12, min);
+- }
+-
+- if (table == 0)
+- return;
+-
+- index = (bpp - table[0].bpp) * 2;
+-
+- /* requested size is bigger than the table */
+- if (index >= table_size) {
+- dm_error("ERROR: Requested rc_calc to find a bpp entry that exceeds the table size\n");
+- return;
+- }
+-
+- memcpy(qps, table[index].qps, sizeof(qp_set));
+-}
+-
+-static double dsc_roundf(double num)
+-{
+- if (num < 0.0)
+- num = num - 0.5;
+- else
+- num = num + 0.5;
+-
+- return (int)(num);
+-}
+-
+-static double dsc_ceil(double num)
+-{
+- double retval = (int)num;
+-
+- if (retval != num && num > 0)
+- retval = num + 1;
+-
+- return (int)retval;
+-}
+-
+-static void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
+-{
+- int *p = ofs;
+-
+- if (mode == CM_444 || mode == CM_RGB) {
+- *p++ = (bpp <= 6) ? (0) : ((((bpp >= 8) && (bpp <= 12))) ? (2) : ((bpp >= 15) ? (10) : ((((bpp > 6) && (bpp < 8))) ? (0 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (2 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
+- *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (8) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
+- *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (6) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+- *p++ = (bpp <= 6) ? (-4) : ((((bpp >= 8) && (bpp <= 12))) ? (-2) : ((bpp >= 15) ? (4) : ((((bpp > 6) && (bpp < 8))) ? (-4 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-2 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+- *p++ = (bpp <= 6) ? (-6) : ((((bpp >= 8) && (bpp <= 12))) ? (-4) : ((bpp >= 15) ? (2) : ((((bpp > 6) && (bpp < 8))) ? (-6 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-4 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+- *p++ = (bpp <= 12) ? (-6) : ((bpp >= 15) ? (0) : (-6 + dsc_roundf((bpp - 12) * (6 / 3.0))));
+- *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-2) : (-8 + dsc_roundf((bpp - 12) * (6 / 3.0))));
+- *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-4) : (-8 + dsc_roundf((bpp - 12) * (4 / 3.0))));
+- *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0))));
+- *p++ = (bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0))));
+- *p++ = -10;
+- *p++ = (bpp <= 6) ? (-12) : ((bpp >= 8) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+- *p++ = -12;
+- *p++ = -12;
+- *p++ = -12;
+- } else if (mode == CM_422) {
+- *p++ = (bpp <= 8) ? (2) : ((bpp >= 10) ? (10) : (2 + dsc_roundf((bpp - 8) * (8 / 2.0))));
+- *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (8) : (0 + dsc_roundf((bpp - 8) * (8 / 2.0))));
+- *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (6) : (0 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+- *p++ = (bpp <= 8) ? (-2) : ((bpp >= 10) ? (4) : (-2 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+- *p++ = (bpp <= 8) ? (-4) : ((bpp >= 10) ? (2) : (-4 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+- *p++ = (bpp <= 8) ? (-6) : ((bpp >= 10) ? (0) : (-6 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+- *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-2) : (-8 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+- *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-4) : (-8 + dsc_roundf((bpp - 8) * (4 / 2.0))));
+- *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-6) : (-8 + dsc_roundf((bpp - 8) * (2 / 2.0))));
+- *p++ = (bpp <= 8) ? (-10) : ((bpp >= 10) ? (-8) : (-10 + dsc_roundf((bpp - 8) * (2 / 2.0))));
+- *p++ = -10;
+- *p++ = (bpp <= 6) ? (-12) : ((bpp >= 7) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2.0 / 1))));
+- *p++ = -12;
+- *p++ = -12;
+- *p++ = -12;
+- } else {
+- *p++ = (bpp <= 6) ? (2) : ((bpp >= 8) ? (10) : (2 + dsc_roundf((bpp - 6) * (8 / 2.0))));
+- *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (8) : (0 + dsc_roundf((bpp - 6) * (8 / 2.0))));
+- *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (6) : (0 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+- *p++ = (bpp <= 6) ? (-2) : ((bpp >= 8) ? (4) : (-2 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+- *p++ = (bpp <= 6) ? (-4) : ((bpp >= 8) ? (2) : (-4 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+- *p++ = (bpp <= 6) ? (-6) : ((bpp >= 8) ? (0) : (-6 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+- *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-2) : (-8 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+- *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-4) : (-8 + dsc_roundf((bpp - 6) * (4 / 2.0))));
+- *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-6) : (-8 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+- *p++ = (bpp <= 6) ? (-10) : ((bpp >= 8) ? (-8) : (-10 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+- *p++ = -10;
+- *p++ = (bpp <= 4) ? (-12) : ((bpp >= 5) ? (-10) : (-12 + dsc_roundf((bpp - 4) * (2 / 1.0))));
+- *p++ = -12;
+- *p++ = -12;
+- *p++ = -12;
+- }
+-}
+-
+-static int median3(int a, int b, int c)
+-{
+- if (a > b)
+- swap(a, b);
+- if (b > c)
+- swap(b, c);
+- if (a > b)
+- swap(b, c);
+-
+- return b;
+-}
+-
+-static void _do_calc_rc_params(struct rc_params *rc, enum colour_mode cm,
+- enum bits_per_comp bpc, u16 drm_bpp,
+- bool is_navite_422_or_420,
+- int slice_width, int slice_height,
+- int minor_version)
+-{
+- float bpp;
+- float bpp_group;
+- float initial_xmit_delay_factor;
+- int padding_pixels;
+- int i;
+-
+- bpp = ((float)drm_bpp / 16.0);
+- /* in native_422 or native_420 modes, the bits_per_pixel is double the
+- * target bpp (the latter is what calc_rc_params expects)
+- */
+- if (is_navite_422_or_420)
+- bpp /= 2.0;
+-
+- rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+- rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+-
+- bpp_group = MODE_SELECT(bpp, bpp * 2.0, bpp * 2.0);
+-
+- switch (cm) {
+- case CM_420:
+- rc->initial_fullness_offset = (bpp >= 6) ? (2048) : ((bpp <= 4) ? (6144) : ((((bpp > 4) && (bpp <= 5))) ? (6144 - dsc_roundf((bpp - 4) * (512))) : (5632 - dsc_roundf((bpp - 5) * (3584)))));
+- rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 3) - (3 * bpp_group)));
+- rc->second_line_bpg_offset = median3(0, 12, (int)((3 * bpc * 3) - (3 * bpp_group)));
+- break;
+- case CM_422:
+- rc->initial_fullness_offset = (bpp >= 8) ? (2048) : ((bpp <= 7) ? (5632) : (5632 - dsc_roundf((bpp - 7) * (3584))));
+- rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 4) - (3 * bpp_group)));
+- rc->second_line_bpg_offset = 0;
+- break;
+- case CM_444:
+- case CM_RGB:
+- rc->initial_fullness_offset = (bpp >= 12) ? (2048) : ((bpp <= 8) ? (6144) : ((((bpp > 8) && (bpp <= 10))) ? (6144 - dsc_roundf((bpp - 8) * (512 / 2))) : (5632 - dsc_roundf((bpp - 10) * (3584 / 2)))));
+- rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)(((3 * bpc + (cm == CM_444 ? 0 : 2)) * 3) - (3 * bpp_group)));
+- rc->second_line_bpg_offset = 0;
+- break;
+- }
+-
+- initial_xmit_delay_factor = (cm == CM_444 || cm == CM_RGB) ? 1.0 : 2.0;
+- rc->initial_xmit_delay = dsc_roundf(8192.0/2.0/bpp/initial_xmit_delay_factor);
+-
+- if (cm == CM_422 || cm == CM_420)
+- slice_width /= 2;
+-
+- padding_pixels = ((slice_width % 3) != 0) ? (3 - (slice_width % 3)) * (rc->initial_xmit_delay / slice_width) : 0;
+- if (3 * bpp_group >= (((rc->initial_xmit_delay + 2) / 3) * (3 + (cm == CM_422)))) {
+- if ((rc->initial_xmit_delay + padding_pixels) % 3 == 1)
+- rc->initial_xmit_delay++;
+- }
+-
+- rc->flatness_min_qp = ((bpc == BPC_8) ? (3) : ((bpc == BPC_10) ? (7) : (11))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+- rc->flatness_max_qp = ((bpc == BPC_8) ? (12) : ((bpc == BPC_10) ? (16) : (20))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+- rc->flatness_det_thresh = 2 << (bpc - 8);
+-
+- get_qp_set(rc->qp_min, cm, bpc, DAL_MM_MIN, bpp);
+- get_qp_set(rc->qp_max, cm, bpc, DAL_MM_MAX, bpp);
+- if (cm == CM_444 && minor_version == 1) {
+- for (i = 0; i < QP_SET_SIZE; ++i) {
+- rc->qp_min[i] = rc->qp_min[i] > 0 ? rc->qp_min[i] - 1 : 0;
+- rc->qp_max[i] = rc->qp_max[i] > 0 ? rc->qp_max[i] - 1 : 0;
+- }
+- }
+- get_ofs_set(rc->ofs, cm, bpp);
+-
+- /* fixed parameters */
+- rc->rc_model_size = 8192;
+- rc->rc_edge_factor = 6;
+- rc->rc_tgt_offset_hi = 3;
+- rc->rc_tgt_offset_lo = 3;
+-
+- rc->rc_buf_thresh[0] = 896;
+- rc->rc_buf_thresh[1] = 1792;
+- rc->rc_buf_thresh[2] = 2688;
+- rc->rc_buf_thresh[3] = 3584;
+- rc->rc_buf_thresh[4] = 4480;
+- rc->rc_buf_thresh[5] = 5376;
+- rc->rc_buf_thresh[6] = 6272;
+- rc->rc_buf_thresh[7] = 6720;
+- rc->rc_buf_thresh[8] = 7168;
+- rc->rc_buf_thresh[9] = 7616;
+- rc->rc_buf_thresh[10] = 7744;
+- rc->rc_buf_thresh[11] = 7872;
+- rc->rc_buf_thresh[12] = 8000;
+- rc->rc_buf_thresh[13] = 8064;
+-}
+-
+-static u32 _do_bytes_per_pixel_calc(int slice_width, u16 drm_bpp,
+- bool is_navite_422_or_420)
+-{
+- float bpp;
+- u32 bytes_per_pixel;
+- double d_bytes_per_pixel;
+-
+- bpp = ((float)drm_bpp / 16.0);
+- d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width;
+- // TODO: Make sure the formula for calculating this is precise (ceiling
+- // vs. floor, and at what point they should be applied)
+- if (is_navite_422_or_420)
+- d_bytes_per_pixel /= 2;
+-
+- bytes_per_pixel = (u32)dsc_ceil(d_bytes_per_pixel * 0x10000000);
+-
+- return bytes_per_pixel;
+-}
+
+ /**
+ * calc_rc_params - reads the user's cmdline mode
+diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
+index 262f06afcbf95..c2340e001b578 100644
+--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
++++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
+@@ -27,55 +27,7 @@
+ #ifndef __RC_CALC_H__
+ #define __RC_CALC_H__
+
+-
+-#define QP_SET_SIZE 15
+-
+-typedef int qp_set[QP_SET_SIZE];
+-
+-struct rc_params {
+- int rc_quant_incr_limit0;
+- int rc_quant_incr_limit1;
+- int initial_fullness_offset;
+- int initial_xmit_delay;
+- int first_line_bpg_offset;
+- int second_line_bpg_offset;
+- int flatness_min_qp;
+- int flatness_max_qp;
+- int flatness_det_thresh;
+- qp_set qp_min;
+- qp_set qp_max;
+- qp_set ofs;
+- int rc_model_size;
+- int rc_edge_factor;
+- int rc_tgt_offset_hi;
+- int rc_tgt_offset_lo;
+- int rc_buf_thresh[QP_SET_SIZE - 1];
+-};
+-
+-enum colour_mode {
+- CM_RGB, /* 444 RGB */
+- CM_444, /* 444 YUV or simple 422 */
+- CM_422, /* native 422 */
+- CM_420 /* native 420 */
+-};
+-
+-enum bits_per_comp {
+- BPC_8 = 8,
+- BPC_10 = 10,
+- BPC_12 = 12
+-};
+-
+-enum max_min {
+- DAL_MM_MIN = 0,
+- DAL_MM_MAX = 1
+-};
+-
+-struct qp_entry {
+- float bpp;
+- const qp_set qps;
+-};
+-
+-typedef struct qp_entry qp_table[];
++#include "dml/dsc/rc_calc_fpu.h"
+
+ void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps);
+ u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps);
+diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
+index ef830aded5b1c..1e19dd674e5a2 100644
+--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
++++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
+@@ -22,7 +22,6 @@
+ * Authors: AMD
+ *
+ */
+-#include "os_types.h"
+ #include <drm/drm_dsc.h>
+ #include "dscc_types.h"
+ #include "rc_calc.h"
+diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+index 713f5558f5e17..9195dec294c2d 100644
+--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
++++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+@@ -154,6 +154,8 @@ struct hubbub_funcs {
+ bool (*is_allow_self_refresh_enabled)(struct hubbub *hubbub);
+ void (*allow_self_refresh_control)(struct hubbub *hubbub, bool allow);
+
++ bool (*verify_allow_pstate_change_high)(struct hubbub *hubbub);
++
+ void (*apply_DEDCN21_147_wa)(struct hubbub *hubbub);
+
+ void (*force_wm_propagate_to_pipes)(struct hubbub *hubbub);
+diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c
+index ed54e1c819bed..a728087b3f3d6 100644
+--- a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c
++++ b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c
+@@ -266,14 +266,6 @@ static const struct irq_source_info_funcs vline0_irq_info_funcs = {
+ .funcs = &pflip_irq_info_funcs\
+ }
+
+-#define vupdate_int_entry(reg_num)\
+- [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\
+- IRQ_REG_ENTRY(OTG, reg_num,\
+- OTG_GLOBAL_SYNC_STATUS, VUPDATE_INT_EN,\
+- OTG_GLOBAL_SYNC_STATUS, VUPDATE_EVENT_CLEAR),\
+- .funcs = &vblank_irq_info_funcs\
+- }
+-
+ /* vupdate_no_lock_int_entry maps to DC_IRQ_SOURCE_VUPDATEx, to match semantic
+ * of DCE's DC_IRQ_SOURCE_VUPDATEx.
+ */
+@@ -402,12 +394,6 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = {
+ dc_underflow_int_entry(6),
+ [DC_IRQ_SOURCE_DMCU_SCP] = dummy_irq_entry(),
+ [DC_IRQ_SOURCE_VBIOS_SW] = dummy_irq_entry(),
+- vupdate_int_entry(0),
+- vupdate_int_entry(1),
+- vupdate_int_entry(2),
+- vupdate_int_entry(3),
+- vupdate_int_entry(4),
+- vupdate_int_entry(5),
+ vupdate_no_lock_int_entry(0),
+ vupdate_no_lock_int_entry(1),
+ vupdate_no_lock_int_entry(2),
+diff --git a/drivers/gpu/drm/amd/display/include/logger_types.h b/drivers/gpu/drm/amd/display/include/logger_types.h
+index 571fcf23cea92..a3a9ea077f505 100644
+--- a/drivers/gpu/drm/amd/display/include/logger_types.h
++++ b/drivers/gpu/drm/amd/display/include/logger_types.h
+@@ -72,6 +72,9 @@
+ #define DC_LOG_DSC(...) DRM_DEBUG_KMS(__VA_ARGS__)
+ #define DC_LOG_SMU(...) pr_debug("[SMU_MSG]:"__VA_ARGS__)
+ #define DC_LOG_DWB(...) DRM_DEBUG_KMS(__VA_ARGS__)
++#if defined(CONFIG_DRM_AMD_DC_DCN)
++#define DC_LOG_DP2(...) DRM_DEBUG_KMS(__VA_ARGS__)
++#endif
+
+ struct dal_logger;
+
+diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+index 57f198de5e2cb..4e075b01d48bb 100644
+--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
++++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+@@ -100,7 +100,8 @@ enum vsc_packet_revision {
+ //PB7 = MD0
+ #define MASK_VTEM_MD0__VRR_EN 0x01
+ #define MASK_VTEM_MD0__M_CONST 0x02
+-#define MASK_VTEM_MD0__RESERVED2 0x0C
++#define MASK_VTEM_MD0__QMS_EN 0x04
++#define MASK_VTEM_MD0__RESERVED2 0x08
+ #define MASK_VTEM_MD0__FVA_FACTOR_M1 0xF0
+
+ //MD1
+@@ -109,7 +110,7 @@ enum vsc_packet_revision {
+ //MD2
+ #define MASK_VTEM_MD2__BASE_REFRESH_RATE_98 0x03
+ #define MASK_VTEM_MD2__RB 0x04
+-#define MASK_VTEM_MD2__RESERVED3 0xF8
++#define MASK_VTEM_MD2__NEXT_TFR 0xF8
+
+ //MD3
+ #define MASK_VTEM_MD3__BASE_REFRESH_RATE_07 0xFF
+diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
+index 257f280d3d53f..bd077ea224a40 100644
+--- a/drivers/gpu/drm/amd/include/amd_shared.h
++++ b/drivers/gpu/drm/amd/include/amd_shared.h
+@@ -98,7 +98,8 @@ enum amd_ip_block_type {
+ AMD_IP_BLOCK_TYPE_ACP,
+ AMD_IP_BLOCK_TYPE_VCN,
+ AMD_IP_BLOCK_TYPE_MES,
+- AMD_IP_BLOCK_TYPE_JPEG
++ AMD_IP_BLOCK_TYPE_JPEG,
++ AMD_IP_BLOCK_TYPE_NUM,
+ };
+
+ enum amd_clockgating_state {
+diff --git a/drivers/gpu/drm/amd/include/discovery.h b/drivers/gpu/drm/amd/include/discovery.h
+index 7ec4331e67f26..a486769b66c6a 100644
+--- a/drivers/gpu/drm/amd/include/discovery.h
++++ b/drivers/gpu/drm/amd/include/discovery.h
+@@ -143,6 +143,55 @@ struct gc_info_v1_0 {
+ uint32_t gc_num_gl2a;
+ };
+
++struct gc_info_v1_1 {
++ struct gpu_info_header header;
++
++ uint32_t gc_num_se;
++ uint32_t gc_num_wgp0_per_sa;
++ uint32_t gc_num_wgp1_per_sa;
++ uint32_t gc_num_rb_per_se;
++ uint32_t gc_num_gl2c;
++ uint32_t gc_num_gprs;
++ uint32_t gc_num_max_gs_thds;
++ uint32_t gc_gs_table_depth;
++ uint32_t gc_gsprim_buff_depth;
++ uint32_t gc_parameter_cache_depth;
++ uint32_t gc_double_offchip_lds_buffer;
++ uint32_t gc_wave_size;
++ uint32_t gc_max_waves_per_simd;
++ uint32_t gc_max_scratch_slots_per_cu;
++ uint32_t gc_lds_size;
++ uint32_t gc_num_sc_per_se;
++ uint32_t gc_num_sa_per_se;
++ uint32_t gc_num_packer_per_sc;
++ uint32_t gc_num_gl2a;
++ uint32_t gc_num_tcp_per_sa;
++ uint32_t gc_num_sdp_interface;
++ uint32_t gc_num_tcps;
++};
++
++struct gc_info_v2_0 {
++ struct gpu_info_header header;
++
++ uint32_t gc_num_se;
++ uint32_t gc_num_cu_per_sh;
++ uint32_t gc_num_sh_per_se;
++ uint32_t gc_num_rb_per_se;
++ uint32_t gc_num_tccs;
++ uint32_t gc_num_gprs;
++ uint32_t gc_num_max_gs_thds;
++ uint32_t gc_gs_table_depth;
++ uint32_t gc_gsprim_buff_depth;
++ uint32_t gc_parameter_cache_depth;
++ uint32_t gc_double_offchip_lds_buffer;
++ uint32_t gc_wave_size;
++ uint32_t gc_max_waves_per_simd;
++ uint32_t gc_max_scratch_slots_per_cu;
++ uint32_t gc_lds_size;
++ uint32_t gc_num_sc_per_se;
++ uint32_t gc_num_packer_per_sc;
++};
++
+ typedef struct harvest_info_header {
+ uint32_t signature; /* Table Signature */
+ uint32_t version; /* Table Version */
+diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+index 03581d5b18360..a68496b3f9296 100644
+--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
++++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+@@ -927,6 +927,13 @@ int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, uint32_t block
+ {
+ int ret = 0;
+ const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
++ enum ip_power_state pwr_state = gate ? POWER_STATE_OFF : POWER_STATE_ON;
++
++ if (atomic_read(&adev->pm.pwr_state[block_type]) == pwr_state) {
++ dev_dbg(adev->dev, "IP block%d already in the target %s state!",
++ block_type, gate ? "gate" : "ungate");
++ return 0;
++ }
+
+ switch (block_type) {
+ case AMD_IP_BLOCK_TYPE_UVD:
+@@ -979,6 +986,9 @@ int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, uint32_t block
+ break;
+ }
+
++ if (!ret)
++ atomic_set(&adev->pm.pwr_state[block_type], pwr_state);
++
+ return ret;
+ }
+
+@@ -1035,6 +1045,17 @@ bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
+
+ if (!pp_funcs || !pp_funcs->get_asic_baco_capability)
+ return false;
++ /* Don't use baco for reset in S3.
++ * This is a workaround for some platforms
++ * where entering BACO during suspend
++ * seems to cause reboots or hangs.
++ * This might be related to the fact that BACO controls
++ * power to the whole GPU including devices like audio and USB.
++ * Powering down/up everything may adversely affect these other
++ * devices. Needs more investigation.
++ */
++ if (adev->in_s3)
++ return false;
+
+ if (pp_funcs->get_asic_baco_capability(pp_handle, &baco_cap))
+ return false;
+diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+index 249cb0aeb5ae4..6aaf1230655f4 100644
+--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
++++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+@@ -2117,6 +2117,12 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
+ }
+ }
+
++ /* setting should not be allowed from VF if not in one VF mode */
++ if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev)) {
++ dev_attr->attr.mode &= ~S_IWUGO;
++ dev_attr->store = NULL;
++ }
++
+ #undef DEVICE_ATTR_IS
+
+ return 0;
+@@ -3439,8 +3445,7 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
+ attr == &sensor_dev_attr_power2_cap_min.dev_attr.attr ||
+ attr == &sensor_dev_attr_power2_cap.dev_attr.attr ||
+ attr == &sensor_dev_attr_power2_cap_default.dev_attr.attr ||
+- attr == &sensor_dev_attr_power2_label.dev_attr.attr ||
+- attr == &sensor_dev_attr_power1_label.dev_attr.attr))
++ attr == &sensor_dev_attr_power2_label.dev_attr.attr))
+ return 0;
+
+ return effective_mode;
+diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+index 98f1b3d8c1d59..16e3f72d31b9f 100644
+--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
++++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+@@ -417,6 +417,12 @@ struct amdgpu_dpm {
+ enum amd_dpm_forced_level forced_level;
+ };
+
++enum ip_power_state {
++ POWER_STATE_UNKNOWN,
++ POWER_STATE_ON,
++ POWER_STATE_OFF,
++};
++
+ struct amdgpu_pm {
+ struct mutex mutex;
+ u32 current_sclk;
+@@ -452,6 +458,8 @@ struct amdgpu_pm {
+ struct i2c_adapter smu_i2c;
+ struct mutex smu_i2c_mutex;
+ struct list_head pm_attr_list;
++
++ atomic_t pwr_state[AMD_IP_BLOCK_TYPE_NUM];
+ };
+
+ #define R600_SSTU_DFLT 0
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
+index 1de3ae77e03ed..cf74621f94a75 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
+@@ -773,13 +773,13 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetHardMinFclkByFreq,
+ hwmgr->display_config->num_display > 3 ?
+- data->clock_vol_info.vdd_dep_on_fclk->entries[0].clk :
++ (data->clock_vol_info.vdd_dep_on_fclk->entries[0].clk / 100) :
+ min_mclk,
+ NULL);
+
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetHardMinSocclkByFreq,
+- data->clock_vol_info.vdd_dep_on_socclk->entries[0].clk,
++ data->clock_vol_info.vdd_dep_on_socclk->entries[0].clk / 100,
+ NULL);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetHardMinVcn,
+@@ -792,11 +792,11 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
+ NULL);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetSoftMaxFclkByFreq,
+- data->clock_vol_info.vdd_dep_on_fclk->entries[index_fclk].clk,
++ data->clock_vol_info.vdd_dep_on_fclk->entries[index_fclk].clk / 100,
+ NULL);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetSoftMaxSocclkByFreq,
+- data->clock_vol_info.vdd_dep_on_socclk->entries[index_socclk].clk,
++ data->clock_vol_info.vdd_dep_on_socclk->entries[index_socclk].clk / 100,
+ NULL);
+ smum_send_msg_to_smc_with_parameter(hwmgr,
+ PPSMC_MSG_SetSoftMaxVcn,
+@@ -1036,13 +1036,13 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ else
+ i = 1;
+
+- size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
++ size += sprintf(buf + size, "0: %uMhz %s\n",
+ data->gfx_min_freq_limit/100,
+ i == 0 ? "*" : "");
+- size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
++ size += sprintf(buf + size, "1: %uMhz %s\n",
+ i == 1 ? now : SMU10_UMD_PSTATE_GFXCLK,
+ i == 1 ? "*" : "");
+- size += sysfs_emit_at(buf, size, "2: %uMhz %s\n",
++ size += sprintf(buf + size, "2: %uMhz %s\n",
+ data->gfx_max_freq_limit/100,
+ i == 2 ? "*" : "");
+ break;
+@@ -1050,7 +1050,7 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetFclkFrequency, &now);
+
+ for (i = 0; i < mclk_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i,
+ mclk_table->entries[i].clk / 100,
+ ((mclk_table->entries[i].clk / 100)
+@@ -1065,10 +1065,10 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ if (ret)
+ return ret;
+
+- size = sysfs_emit(buf, "%s:\n", "OD_SCLK");
+- size += sysfs_emit_at(buf, size, "0: %10uMhz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_SCLK");
++ size += sprintf(buf + size, "0: %10uMhz\n",
+ (data->gfx_actual_soft_min_freq > 0) ? data->gfx_actual_soft_min_freq : min_freq);
+- size += sysfs_emit_at(buf, size, "1: %10uMhz\n",
++ size += sprintf(buf + size, "1: %10uMhz\n",
+ (data->gfx_actual_soft_max_freq > 0) ? data->gfx_actual_soft_max_freq : max_freq);
+ }
+ break;
+@@ -1081,8 +1081,8 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ if (ret)
+ return ret;
+
+- size = sysfs_emit(buf, "%s:\n", "OD_RANGE");
+- size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_RANGE");
++ size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
+ min_freq, max_freq);
+ }
+ break;
+@@ -1456,6 +1456,8 @@ static int smu10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
+ if (!buf)
+ return -EINVAL;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ size += sysfs_emit_at(buf, size, "%s %16s %s %s %s %s\n",title[0],
+ title[1], title[2], title[3], title[4], title[5]);
+
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+index e7803ce8f67aa..611969bf45207 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+@@ -4926,7 +4926,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
+ now = i;
+
+ for (i = 0; i < sclk_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, sclk_table->dpm_levels[i].value / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -4941,7 +4941,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
+ now = i;
+
+ for (i = 0; i < mclk_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, mclk_table->dpm_levels[i].value / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -4955,7 +4955,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
+ now = i;
+
+ for (i = 0; i < pcie_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %s %s\n", i,
++ size += sprintf(buf + size, "%d: %s %s\n", i,
+ (pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x8" :
+ (pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" :
+ (pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "",
+@@ -4963,32 +4963,32 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
+ break;
+ case OD_SCLK:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_SCLK");
++ size += sprintf(buf + size, "%s:\n", "OD_SCLK");
+ for (i = 0; i < odn_sclk_table->num_of_pl; i++)
+- size += sysfs_emit_at(buf, size, "%d: %10uMHz %10umV\n",
++ size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
+ i, odn_sclk_table->entries[i].clock/100,
+ odn_sclk_table->entries[i].vddc);
+ }
+ break;
+ case OD_MCLK:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_MCLK");
++ size += sprintf(buf + size, "%s:\n", "OD_MCLK");
+ for (i = 0; i < odn_mclk_table->num_of_pl; i++)
+- size += sysfs_emit_at(buf, size, "%d: %10uMHz %10umV\n",
++ size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
+ i, odn_mclk_table->entries[i].clock/100,
+ odn_mclk_table->entries[i].vddc);
+ }
+ break;
+ case OD_RANGE:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_RANGE");
+- size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_RANGE");
++ size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
+ data->golden_dpm_table.sclk_table.dpm_levels[0].value/100,
+ hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
+- size += sysfs_emit_at(buf, size, "MCLK: %7uMHz %10uMHz\n",
++ size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
+ data->golden_dpm_table.mclk_table.dpm_levels[0].value/100,
+ hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
+- size += sysfs_emit_at(buf, size, "VDDC: %7umV %11umV\n",
++ size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
+ data->odn_dpm_table.min_vddc,
+ data->odn_dpm_table.max_vddc);
+ }
+@@ -5518,6 +5518,8 @@ static int smu7_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
+ if (!buf)
+ return -EINVAL;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ size += sysfs_emit_at(buf, size, "%s %16s %16s %16s %16s %16s %16s %16s\n",
+ title[0], title[1], title[2], title[3],
+ title[4], title[5], title[6], title[7]);
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
+index b94a77e4e7147..03bf8f0692228 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
+@@ -1559,7 +1559,7 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr,
+ CURR_SCLK_INDEX);
+
+ for (i = 0; i < sclk_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, sclk_table->entries[i].clk / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -1571,7 +1571,7 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr,
+ CURR_MCLK_INDEX);
+
+ for (i = SMU8_NUM_NBPMEMORYCLOCK; i > 0; i--)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ SMU8_NUM_NBPMEMORYCLOCK-i, data->sys_info.nbp_memory_clock[i-1] / 100,
+ (SMU8_NUM_NBPMEMORYCLOCK-i == now) ? "*" : "");
+ break;
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h
+index ad33983a8064e..2a75da1e9f035 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.h
+@@ -109,6 +109,19 @@ int phm_irq_process(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry);
+
++/*
++ * Helper function to make sysfs_emit_at() happy. Align buf to
++ * the current page boundary and record the offset.
++ */
++static inline void phm_get_sysfs_buf(char **buf, int *offset)
++{
++ if (!*buf || !offset)
++ return;
++
++ *offset = offset_in_page(*buf);
++ *buf -= *offset;
++}
++
+ int smu9_register_irq_handlers(struct pp_hwmgr *hwmgr);
+
+ void *smu_atom_get_data_table(void *dev, uint32_t table, uint16_t *size,
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
+index c152a61ddd2c9..e6336654c5655 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
+@@ -4548,6 +4548,8 @@ static int vega10_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf)
+ int ret = 0;
+ int size = 0;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ ret = vega10_get_enabled_smc_features(hwmgr, &features_enabled);
+ PP_ASSERT_WITH_CODE(!ret,
+ "[EnableAllSmuFeatures] Failed to get enabled smc features!",
+@@ -4650,7 +4652,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ else
+ count = sclk_table->count;
+ for (i = 0; i < count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, sclk_table->dpm_levels[i].value / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -4661,7 +4663,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentUclkIndex, &now);
+
+ for (i = 0; i < mclk_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, mclk_table->dpm_levels[i].value / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -4672,7 +4674,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentSocclkIndex, &now);
+
+ for (i = 0; i < soc_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, soc_table->dpm_levels[i].value / 100,
+ (i == now) ? "*" : "");
+ break;
+@@ -4684,7 +4686,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ PPSMC_MSG_GetClockFreqMHz, CLK_DCEFCLK, &now);
+
+ for (i = 0; i < dcef_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, dcef_table->dpm_levels[i].value / 100,
+ (dcef_table->dpm_levels[i].value / 100 == now) ?
+ "*" : "");
+@@ -4698,7 +4700,7 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+ gen_speed = pptable->PcieGenSpeed[i];
+ lane_width = pptable->PcieLaneCount[i];
+
+- size += sysfs_emit_at(buf, size, "%d: %s %s %s\n", i,
++ size += sprintf(buf + size, "%d: %s %s %s\n", i,
+ (gen_speed == 0) ? "2.5GT/s," :
+ (gen_speed == 1) ? "5.0GT/s," :
+ (gen_speed == 2) ? "8.0GT/s," :
+@@ -4717,34 +4719,34 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
+
+ case OD_SCLK:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_SCLK");
++ size += sprintf(buf + size, "%s:\n", "OD_SCLK");
+ podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_sclk;
+ for (i = 0; i < podn_vdd_dep->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %10uMhz %10umV\n",
++ size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
+ i, podn_vdd_dep->entries[i].clk / 100,
+ podn_vdd_dep->entries[i].vddc);
+ }
+ break;
+ case OD_MCLK:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_MCLK");
++ size += sprintf(buf + size, "%s:\n", "OD_MCLK");
+ podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_mclk;
+ for (i = 0; i < podn_vdd_dep->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %10uMhz %10umV\n",
++ size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
+ i, podn_vdd_dep->entries[i].clk/100,
+ podn_vdd_dep->entries[i].vddc);
+ }
+ break;
+ case OD_RANGE:
+ if (hwmgr->od_enabled) {
+- size = sysfs_emit(buf, "%s:\n", "OD_RANGE");
+- size += sysfs_emit_at(buf, size, "SCLK: %7uMHz %10uMHz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_RANGE");
++ size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
+ data->golden_dpm_table.gfx_table.dpm_levels[0].value/100,
+ hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
+- size += sysfs_emit_at(buf, size, "MCLK: %7uMHz %10uMHz\n",
++ size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
+ data->golden_dpm_table.mem_table.dpm_levels[0].value/100,
+ hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
+- size += sysfs_emit_at(buf, size, "VDDC: %7umV %11umV\n",
++ size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
+ data->odn_dpm_table.min_vddc,
+ data->odn_dpm_table.max_vddc);
+ }
+@@ -5112,6 +5114,8 @@ static int vega10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
+ if (!buf)
+ return -EINVAL;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ size += sysfs_emit_at(buf, size, "%s %16s %s %s %s %s\n",title[0],
+ title[1], title[2], title[3], title[4], title[5]);
+
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
+index 8558718e15a8f..a2f4d6773d458 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
+@@ -2141,6 +2141,8 @@ static int vega12_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf)
+ int ret = 0;
+ int size = 0;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ ret = vega12_get_enabled_smc_features(hwmgr, &features_enabled);
+ PP_ASSERT_WITH_CODE(!ret,
+ "[EnableAllSmuFeatures] Failed to get enabled smc features!",
+@@ -2256,7 +2258,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
+ "Attempt to get gfx clk levels Failed!",
+ return -1);
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
+ break;
+@@ -2272,7 +2274,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
+ "Attempt to get memory clk levels Failed!",
+ return -1);
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
+ break;
+@@ -2290,7 +2292,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
+ "Attempt to get soc clk levels Failed!",
+ return -1);
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : "");
+ break;
+@@ -2308,7 +2310,7 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
+ "Attempt to get dcef clk levels Failed!",
+ return -1);
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : "");
+ break;
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
+index 0cf39c1244b1c..85d55ab4e369f 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
+@@ -3238,6 +3238,8 @@ static int vega20_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf)
+ int ret = 0;
+ int size = 0;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ ret = vega20_get_enabled_smc_features(hwmgr, &features_enabled);
+ PP_ASSERT_WITH_CODE(!ret,
+ "[EnableAllSmuFeatures] Failed to get enabled smc features!",
+@@ -3372,13 +3374,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ return ret);
+
+ if (vega20_get_sclks(hwmgr, &clocks)) {
+- size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n",
++ size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
+ now / 100);
+ break;
+ }
+
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+ break;
+@@ -3390,13 +3392,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ return ret);
+
+ if (vega20_get_memclocks(hwmgr, &clocks)) {
+- size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n",
++ size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
+ now / 100);
+ break;
+ }
+
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+ break;
+@@ -3408,13 +3410,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ return ret);
+
+ if (vega20_get_socclocks(hwmgr, &clocks)) {
+- size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n",
++ size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
+ now / 100);
+ break;
+ }
+
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+ break;
+@@ -3426,7 +3428,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ return ret);
+
+ for (i = 0; i < fclk_dpm_table->count; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, fclk_dpm_table->dpm_levels[i].value,
+ fclk_dpm_table->dpm_levels[i].value == (now / 100) ? "*" : "");
+ break;
+@@ -3438,13 +3440,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ return ret);
+
+ if (vega20_get_dcefclocks(hwmgr, &clocks)) {
+- size += sysfs_emit_at(buf, size, "0: %uMhz * (DPM disabled)\n",
++ size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
+ now / 100);
+ break;
+ }
+
+ for (i = 0; i < clocks.num_levels; i++)
+- size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
++ size += sprintf(buf + size, "%d: %uMhz %s\n",
+ i, clocks.data[i].clocks_in_khz / 1000,
+ (clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+ break;
+@@ -3458,7 +3460,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ gen_speed = pptable->PcieGenSpeed[i];
+ lane_width = pptable->PcieLaneCount[i];
+
+- size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
++ size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i,
+ (gen_speed == 0) ? "2.5GT/s," :
+ (gen_speed == 1) ? "5.0GT/s," :
+ (gen_speed == 2) ? "8.0GT/s," :
+@@ -3479,18 +3481,18 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ case OD_SCLK:
+ if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) {
+- size = sysfs_emit(buf, "%s:\n", "OD_SCLK");
+- size += sysfs_emit_at(buf, size, "0: %10uMhz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_SCLK");
++ size += sprintf(buf + size, "0: %10uMhz\n",
+ od_table->GfxclkFmin);
+- size += sysfs_emit_at(buf, size, "1: %10uMhz\n",
++ size += sprintf(buf + size, "1: %10uMhz\n",
+ od_table->GfxclkFmax);
+ }
+ break;
+
+ case OD_MCLK:
+ if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) {
+- size = sysfs_emit(buf, "%s:\n", "OD_MCLK");
+- size += sysfs_emit_at(buf, size, "1: %10uMhz\n",
++ size += sprintf(buf + size, "%s:\n", "OD_MCLK");
++ size += sprintf(buf + size, "1: %10uMhz\n",
+ od_table->UclkFmax);
+ }
+
+@@ -3503,14 +3505,14 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
+- size = sysfs_emit(buf, "%s:\n", "OD_VDDC_CURVE");
+- size += sysfs_emit_at(buf, size, "0: %10uMhz %10dmV\n",
++ size += sprintf(buf + size, "%s:\n", "OD_VDDC_CURVE");
++ size += sprintf(buf + size, "0: %10uMhz %10dmV\n",
+ od_table->GfxclkFreq1,
+ od_table->GfxclkVolt1 / VOLTAGE_SCALE);
+- size += sysfs_emit_at(buf, size, "1: %10uMhz %10dmV\n",
++ size += sprintf(buf + size, "1: %10uMhz %10dmV\n",
+ od_table->GfxclkFreq2,
+ od_table->GfxclkVolt2 / VOLTAGE_SCALE);
+- size += sysfs_emit_at(buf, size, "2: %10uMhz %10dmV\n",
++ size += sprintf(buf + size, "2: %10uMhz %10dmV\n",
+ od_table->GfxclkFreq3,
+ od_table->GfxclkVolt3 / VOLTAGE_SCALE);
+ }
+@@ -3518,17 +3520,17 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ break;
+
+ case OD_RANGE:
+- size = sysfs_emit(buf, "%s:\n", "OD_RANGE");
++ size += sprintf(buf + size, "%s:\n", "OD_RANGE");
+
+ if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) {
+- size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n",
++ size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n",
+ od8_settings[OD8_SETTING_GFXCLK_FMIN].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_FMAX].max_value);
+ }
+
+ if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) {
+- size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n",
++ size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n",
+ od8_settings[OD8_SETTING_UCLK_FMAX].min_value,
+ od8_settings[OD8_SETTING_UCLK_FMAX].max_value);
+ }
+@@ -3539,22 +3541,22 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
++ size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
+ od8_settings[OD8_SETTING_GFXCLK_FREQ1].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_FREQ1].max_value);
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
++ size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].max_value);
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
++ size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
+ od8_settings[OD8_SETTING_GFXCLK_FREQ2].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_FREQ2].max_value);
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
++ size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].max_value);
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
++ size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
+ od8_settings[OD8_SETTING_GFXCLK_FREQ3].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_FREQ3].max_value);
+- size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
++ size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].min_value,
+ od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].max_value);
+ }
+@@ -4003,6 +4005,8 @@ static int vega20_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
+ if (!buf)
+ return -EINVAL;
+
++ phm_get_sysfs_buf(&buf, &size);
++
+ size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s\n",
+ title[0], title[1], title[2], title[3], title[4], title[5],
+ title[6], title[7], title[8], title[9], title[10]);
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+index 04863a7971155..952a8aa69b9ee 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+@@ -138,7 +138,7 @@ int smu_get_dpm_freq_range(struct smu_context *smu,
+ uint32_t *min,
+ uint32_t *max)
+ {
+- int ret = 0;
++ int ret = -ENOTSUPP;
+
+ if (!min && !max)
+ return -EINVAL;
+@@ -1536,9 +1536,7 @@ static int smu_suspend(void *handle)
+
+ smu->watermarks_bitmap &= ~(WATERMARKS_LOADED);
+
+- /* skip CGPG when in S0ix */
+- if (smu->is_apu && !adev->in_s0ix)
+- smu_set_gfx_cgpg(&adev->smu, false);
++ smu_set_gfx_cgpg(&adev->smu, false);
+
+ return 0;
+ }
+@@ -1569,8 +1567,7 @@ static int smu_resume(void *handle)
+ return ret;
+ }
+
+- if (smu->is_apu)
+- smu_set_gfx_cgpg(&adev->smu, true);
++ smu_set_gfx_cgpg(&adev->smu, true);
+
+ smu->disable_uclk_switch = 0;
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+index b1ad451af06bd..dfba0bc732073 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+@@ -1265,7 +1265,7 @@ static int navi10_print_clk_levels(struct smu_context *smu,
+ enum smu_clk_type clk_type, char *buf)
+ {
+ uint16_t *curve_settings;
+- int i, size = 0, ret = 0;
++ int i, levels, size = 0, ret = 0;
+ uint32_t cur_value = 0, value = 0, count = 0;
+ uint32_t freq_values[3] = {0};
+ uint32_t mark_index = 0;
+@@ -1319,14 +1319,17 @@ static int navi10_print_clk_levels(struct smu_context *smu,
+ freq_values[1] = cur_value;
+ mark_index = cur_value == freq_values[0] ? 0 :
+ cur_value == freq_values[2] ? 2 : 1;
+- if (mark_index != 1)
+- freq_values[1] = (freq_values[0] + freq_values[2]) / 2;
+
+- for (i = 0; i < 3; i++) {
++ levels = 3;
++ if (mark_index != 1) {
++ levels = 2;
++ freq_values[1] = freq_values[2];
++ }
++
++ for (i = 0; i < levels; i++) {
+ size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i],
+ i == mark_index ? "*" : "");
+ }
+-
+ }
+ break;
+ case SMU_PCIE:
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+index ca57221e39629..574a9d7f7a5e7 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+@@ -418,6 +418,36 @@ static int sienna_cichlid_store_powerplay_table(struct smu_context *smu)
+ return 0;
+ }
+
++static int sienna_cichlid_patch_pptable_quirk(struct smu_context *smu)
++{
++ struct amdgpu_device *adev = smu->adev;
++ uint32_t *board_reserved;
++ uint16_t *freq_table_gfx;
++ uint32_t i;
++
++ /* Fix some OEM SKU specific stability issues */
++ GET_PPTABLE_MEMBER(BoardReserved, &board_reserved);
++ if ((adev->pdev->device == 0x73DF) &&
++ (adev->pdev->revision == 0XC3) &&
++ (adev->pdev->subsystem_device == 0x16C2) &&
++ (adev->pdev->subsystem_vendor == 0x1043))
++ board_reserved[0] = 1387;
++
++ GET_PPTABLE_MEMBER(FreqTableGfx, &freq_table_gfx);
++ if ((adev->pdev->device == 0x73DF) &&
++ (adev->pdev->revision == 0XC3) &&
++ ((adev->pdev->subsystem_device == 0x16C2) ||
++ (adev->pdev->subsystem_device == 0x133C)) &&
++ (adev->pdev->subsystem_vendor == 0x1043)) {
++ for (i = 0; i < NUM_GFXCLK_DPM_LEVELS; i++) {
++ if (freq_table_gfx[i] > 2500)
++ freq_table_gfx[i] = 2500;
++ }
++ }
++
++ return 0;
++}
++
+ static int sienna_cichlid_setup_pptable(struct smu_context *smu)
+ {
+ int ret = 0;
+@@ -438,7 +468,7 @@ static int sienna_cichlid_setup_pptable(struct smu_context *smu)
+ if (ret)
+ return ret;
+
+- return ret;
++ return sienna_cichlid_patch_pptable_quirk(smu);
+ }
+
+ static int sienna_cichlid_tables_init(struct smu_context *smu)
+@@ -1278,21 +1308,37 @@ static int sienna_cichlid_populate_umd_state_clk(struct smu_context *smu)
+ &dpm_context->dpm_tables.soc_table;
+ struct smu_umd_pstate_table *pstate_table =
+ &smu->pstate_table;
++ struct amdgpu_device *adev = smu->adev;
+
+ pstate_table->gfxclk_pstate.min = gfx_table->min;
+ pstate_table->gfxclk_pstate.peak = gfx_table->max;
+- if (gfx_table->max >= SIENNA_CICHLID_UMD_PSTATE_PROFILING_GFXCLK)
+- pstate_table->gfxclk_pstate.standard = SIENNA_CICHLID_UMD_PSTATE_PROFILING_GFXCLK;
+
+ pstate_table->uclk_pstate.min = mem_table->min;
+ pstate_table->uclk_pstate.peak = mem_table->max;
+- if (mem_table->max >= SIENNA_CICHLID_UMD_PSTATE_PROFILING_MEMCLK)
+- pstate_table->uclk_pstate.standard = SIENNA_CICHLID_UMD_PSTATE_PROFILING_MEMCLK;
+
+ pstate_table->socclk_pstate.min = soc_table->min;
+ pstate_table->socclk_pstate.peak = soc_table->max;
+- if (soc_table->max >= SIENNA_CICHLID_UMD_PSTATE_PROFILING_SOCCLK)
++
++ switch (adev->asic_type) {
++ case CHIP_SIENNA_CICHLID:
++ case CHIP_NAVY_FLOUNDER:
++ pstate_table->gfxclk_pstate.standard = SIENNA_CICHLID_UMD_PSTATE_PROFILING_GFXCLK;
++ pstate_table->uclk_pstate.standard = SIENNA_CICHLID_UMD_PSTATE_PROFILING_MEMCLK;
+ pstate_table->socclk_pstate.standard = SIENNA_CICHLID_UMD_PSTATE_PROFILING_SOCCLK;
++ break;
++ case CHIP_DIMGREY_CAVEFISH:
++ pstate_table->gfxclk_pstate.standard = DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_GFXCLK;
++ pstate_table->uclk_pstate.standard = DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_MEMCLK;
++ pstate_table->socclk_pstate.standard = DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_SOCCLK;
++ break;
++ case CHIP_BEIGE_GOBY:
++ pstate_table->gfxclk_pstate.standard = BEIGE_GOBY_UMD_PSTATE_PROFILING_GFXCLK;
++ pstate_table->uclk_pstate.standard = BEIGE_GOBY_UMD_PSTATE_PROFILING_MEMCLK;
++ pstate_table->socclk_pstate.standard = BEIGE_GOBY_UMD_PSTATE_PROFILING_SOCCLK;
++ break;
++ default:
++ break;
++ }
+
+ return 0;
+ }
+@@ -3728,14 +3774,14 @@ static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu,
+
+ static int sienna_cichlid_enable_mgpu_fan_boost(struct smu_context *smu)
+ {
+- struct smu_table_context *table_context = &smu->smu_table;
+- PPTable_t *smc_pptable = table_context->driver_pptable;
++ uint16_t *mgpu_fan_boost_limit_rpm;
+
++ GET_PPTABLE_MEMBER(MGpuFanBoostLimitRpm, &mgpu_fan_boost_limit_rpm);
+ /*
+ * Skip the MGpuFanBoost setting for those ASICs
+ * which do not support it
+ */
+- if (!smc_pptable->MGpuFanBoostLimitRpm)
++ if (*mgpu_fan_boost_limit_rpm == 0)
+ return 0;
+
+ return smu_cmn_send_smc_msg_with_param(smu,
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h
+index 38cd0ece24f6b..42f705c7a36f8 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.h
+@@ -33,6 +33,14 @@ typedef enum {
+ #define SIENNA_CICHLID_UMD_PSTATE_PROFILING_SOCCLK 960
+ #define SIENNA_CICHLID_UMD_PSTATE_PROFILING_MEMCLK 1000
+
++#define DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_GFXCLK 1950
++#define DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_SOCCLK 960
++#define DIMGREY_CAVEFISH_UMD_PSTATE_PROFILING_MEMCLK 676
++
++#define BEIGE_GOBY_UMD_PSTATE_PROFILING_GFXCLK 2200
++#define BEIGE_GOBY_UMD_PSTATE_PROFILING_SOCCLK 960
++#define BEIGE_GOBY_UMD_PSTATE_PROFILING_MEMCLK 1000
++
+ extern void sienna_cichlid_set_ppt_funcs(struct smu_context *smu);
+
+ #endif
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+index f6ef0ce6e9e2c..a9dceef4a7011 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+@@ -1386,52 +1386,38 @@ static int vangogh_set_performance_level(struct smu_context *smu,
+ uint32_t soc_mask, mclk_mask, fclk_mask;
+ uint32_t vclk_mask = 0, dclk_mask = 0;
+
++ smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
++ smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
++
+ switch (level) {
+ case AMD_DPM_FORCED_LEVEL_HIGH:
+- smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
++ smu->gfx_actual_hard_min_freq = smu->gfx_default_soft_max_freq;
+ smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+
+ ret = vangogh_force_dpm_limit_value(smu, true);
++ if (ret)
++ return ret;
+ break;
+ case AMD_DPM_FORCED_LEVEL_LOW:
+ smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+- smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+-
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
++ smu->gfx_actual_soft_max_freq = smu->gfx_default_hard_min_freq;
+
+ ret = vangogh_force_dpm_limit_value(smu, false);
++ if (ret)
++ return ret;
+ break;
+ case AMD_DPM_FORCED_LEVEL_AUTO:
+ smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+ smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+-
+ ret = vangogh_unforce_dpm_levels(smu);
+- break;
+- case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+- smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+- smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+-
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+-
+- ret = smu_cmn_send_smc_msg_with_param(smu,
+- SMU_MSG_SetHardMinGfxClk,
+- VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL);
+- if (ret)
+- return ret;
+-
+- ret = smu_cmn_send_smc_msg_with_param(smu,
+- SMU_MSG_SetSoftMaxGfxClk,
+- VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL);
+ if (ret)
+ return ret;
++ break;
++ case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
++ smu->gfx_actual_hard_min_freq = VANGOGH_UMD_PSTATE_STANDARD_GFXCLK;
++ smu->gfx_actual_soft_max_freq = VANGOGH_UMD_PSTATE_STANDARD_GFXCLK;
+
+ ret = vangogh_get_profiling_clk_mask(smu, level,
+ &vclk_mask,
+@@ -1446,32 +1432,15 @@ static int vangogh_set_performance_level(struct smu_context *smu,
+ vangogh_force_clk_levels(smu, SMU_SOCCLK, 1 << soc_mask);
+ vangogh_force_clk_levels(smu, SMU_VCLK, 1 << vclk_mask);
+ vangogh_force_clk_levels(smu, SMU_DCLK, 1 << dclk_mask);
+-
+ break;
+ case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+ smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+- smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+-
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+-
+- ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinVcn,
+- VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL);
+- if (ret)
+- return ret;
+-
+- ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxVcn,
+- VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL);
+- if (ret)
+- return ret;
++ smu->gfx_actual_soft_max_freq = smu->gfx_default_hard_min_freq;
+ break;
+ case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+ smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+ smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+-
+ ret = vangogh_get_profiling_clk_mask(smu, level,
+ NULL,
+ NULL,
+@@ -1484,29 +1453,29 @@ static int vangogh_set_performance_level(struct smu_context *smu,
+ vangogh_force_clk_levels(smu, SMU_FCLK, 1 << fclk_mask);
+ break;
+ case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+- smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+- smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
+-
+- smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq;
+- smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq;
+-
+- ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk,
+- VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL);
+- if (ret)
+- return ret;
++ smu->gfx_actual_hard_min_freq = VANGOGH_UMD_PSTATE_PEAK_GFXCLK;
++ smu->gfx_actual_soft_max_freq = VANGOGH_UMD_PSTATE_PEAK_GFXCLK;
+
+- ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk,
+- VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL);
++ ret = vangogh_set_peak_clock_by_device(smu);
+ if (ret)
+ return ret;
+-
+- ret = vangogh_set_peak_clock_by_device(smu);
+ break;
+ case AMD_DPM_FORCED_LEVEL_MANUAL:
+ case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
+ default:
+- break;
++ return 0;
+ }
++
++ ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk,
++ smu->gfx_actual_hard_min_freq, NULL);
++ if (ret)
++ return ret;
++
++ ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk,
++ smu->gfx_actual_soft_max_freq, NULL);
++ if (ret)
++ return ret;
++
+ return ret;
+ }
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c
+index d60b8c5e87157..9c91e79c955fb 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c
+@@ -120,7 +120,8 @@ int smu_v12_0_powergate_sdma(struct smu_context *smu, bool gate)
+
+ int smu_v12_0_set_gfx_cgpg(struct smu_context *smu, bool enable)
+ {
+- if (!(smu->adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
++ /* Until now the SMU12 only implemented for Renoir series so here neen't do APU check. */
++ if (!(smu->adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) || smu->adev->in_s0ix)
+ return 0;
+
+ return smu_cmn_send_smc_msg_with_param(smu,
+@@ -191,6 +192,9 @@ int smu_v12_0_fini_smc_tables(struct smu_context *smu)
+ kfree(smu_table->watermarks_table);
+ smu_table->watermarks_table = NULL;
+
++ kfree(smu_table->gpu_metrics_table);
++ smu_table->gpu_metrics_table = NULL;
++
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+index 5019903db492a..c9cfeb094750d 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+@@ -1619,7 +1619,7 @@ static int aldebaran_allow_xgmi_power_down(struct smu_context *smu, bool en)
+ {
+ return smu_cmn_send_smc_msg_with_param(smu,
+ SMU_MSG_GmiPwrDnControl,
+- en ? 1 : 0,
++ en ? 0 : 1,
+ NULL);
+ }
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+index a0e50f23b1dd7..8d4aa16b2ae7c 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+@@ -197,6 +197,7 @@ int smu_v13_0_check_fw_status(struct smu_context *smu)
+
+ int smu_v13_0_check_fw_version(struct smu_context *smu)
+ {
++ struct amdgpu_device *adev = smu->adev;
+ uint32_t if_version = 0xff, smu_version = 0xff;
+ uint16_t smu_major;
+ uint8_t smu_minor, smu_debug;
+@@ -209,6 +210,8 @@ int smu_v13_0_check_fw_version(struct smu_context *smu)
+ smu_major = (smu_version >> 16) & 0xffff;
+ smu_minor = (smu_version >> 8) & 0xff;
+ smu_debug = (smu_version >> 0) & 0xff;
++ if (smu->is_apu)
++ adev->pm.fw_version = smu_version;
+
+ switch (smu->adev->asic_type) {
+ case CHIP_ALDEBARAN:
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
+index a403657151ba1..0e1a843608e43 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
+@@ -291,14 +291,9 @@ static int yellow_carp_post_smu_init(struct smu_context *smu)
+
+ static int yellow_carp_mode_reset(struct smu_context *smu, int type)
+ {
+- int ret = 0, index = 0;
+-
+- index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_MSG,
+- SMU_MSG_GfxDeviceDriverReset);
+- if (index < 0)
+- return index == -EACCES ? 0 : index;
++ int ret = 0;
+
+- ret = smu_cmn_send_smc_msg_with_param(smu, (uint16_t)index, type, NULL);
++ ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_GfxDeviceDriverReset, type, NULL);
+ if (ret)
+ dev_err(smu->adev->dev, "Failed to mode reset!\n");
+
+diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
+index 21909642ee4ca..8fbb25913327c 100644
+--- a/drivers/gpu/drm/armada/armada_gem.c
++++ b/drivers/gpu/drm/armada/armada_gem.c
+@@ -336,7 +336,7 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
+ struct drm_armada_gem_pwrite *args = data;
+ struct armada_gem_object *dobj;
+ char __user *ptr;
+- int ret;
++ int ret = 0;
+
+ DRM_DEBUG_DRIVER("handle %u off %u size %u ptr 0x%llx\n",
+ args->handle, args->offset, args->size, args->ptr);
+@@ -349,9 +349,8 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
+ if (!access_ok(ptr, args->size))
+ return -EFAULT;
+
+- ret = fault_in_pages_readable(ptr, args->size);
+- if (ret)
+- return ret;
++ if (fault_in_readable(ptr, args->size))
++ return -EFAULT;
+
+ dobj = armada_gem_object_lookup(file, args->handle);
+ if (dobj == NULL)
+diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
+index b53fee6f1c170..65f172807a0d5 100644
+--- a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
++++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
+@@ -291,7 +291,7 @@ vga_pw_show(struct device *dev, struct device_attribute *attr, char *buf)
+ if (rc)
+ return rc;
+
+- return sprintf(buf, "%u\n", reg & 1);
++ return sprintf(buf, "%u\n", reg);
+ }
+ static DEVICE_ATTR_RO(vga_pw);
+
+diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
+index 1e30eaeb0e1b3..d5c98f79d58d3 100644
+--- a/drivers/gpu/drm/ast/ast_mode.c
++++ b/drivers/gpu/drm/ast/ast_mode.c
+@@ -1121,7 +1121,10 @@ static void ast_crtc_reset(struct drm_crtc *crtc)
+ if (crtc->state)
+ crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+
+- __drm_atomic_helper_crtc_reset(crtc, &ast_state->base);
++ if (ast_state)
++ __drm_atomic_helper_crtc_reset(crtc, &ast_state->base);
++ else
++ __drm_atomic_helper_crtc_reset(crtc, NULL);
+ }
+
+ static struct drm_crtc_state *
+diff --git a/drivers/gpu/drm/ast/ast_tables.h b/drivers/gpu/drm/ast/ast_tables.h
+index d9eb353a4bf09..dbe1cc620f6e6 100644
+--- a/drivers/gpu/drm/ast/ast_tables.h
++++ b/drivers/gpu/drm/ast/ast_tables.h
+@@ -282,8 +282,6 @@ static const struct ast_vbios_enhtable res_1360x768[] = {
+ };
+
+ static const struct ast_vbios_enhtable res_1600x900[] = {
+- {1800, 1600, 24, 80, 1000, 900, 1, 3, VCLK108, /* 60Hz */
+- (SyncPP | Charx8Dot | LineCompareOff | WideScreenMode | NewModeInfo), 60, 3, 0x3A },
+ {1760, 1600, 48, 32, 926, 900, 3, 5, VCLK97_75, /* 60Hz CVT RB */
+ (SyncNP | Charx8Dot | LineCompareOff | WideScreenMode | NewModeInfo |
+ AST2500PreCatchCRT), 60, 1, 0x3A },
+diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
+index 431b6e12a81fe..68ec45abc1fbf 100644
+--- a/drivers/gpu/drm/bridge/Kconfig
++++ b/drivers/gpu/drm/bridge/Kconfig
+@@ -8,7 +8,6 @@ config DRM_BRIDGE
+ config DRM_PANEL_BRIDGE
+ def_bool y
+ depends on DRM_BRIDGE
+- depends on DRM_KMS_HELPER
+ select DRM_PANEL
+ help
+ DRM bridge wrapper of DRM panels
+@@ -30,6 +29,7 @@ config DRM_CDNS_DSI
+ config DRM_CHIPONE_ICN6211
+ tristate "Chipone ICN6211 MIPI-DSI/RGB Converter bridge"
+ depends on OF
++ select DRM_KMS_HELPER
+ select DRM_MIPI_DSI
+ select DRM_PANEL_BRIDGE
+ help
+diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
+index 05e3abb5a0c9a..1b00dfda6e0d9 100644
+--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
++++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
+@@ -169,6 +169,7 @@
+ #define ADV7511_PACKET_ENABLE_SPARE2 BIT(1)
+ #define ADV7511_PACKET_ENABLE_SPARE1 BIT(0)
+
++#define ADV7535_REG_POWER2_HPD_OVERRIDE BIT(6)
+ #define ADV7511_REG_POWER2_HPD_SRC_MASK 0xc0
+ #define ADV7511_REG_POWER2_HPD_SRC_BOTH 0x00
+ #define ADV7511_REG_POWER2_HPD_SRC_HPD 0x40
+diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+index 76555ae64e9ce..c02f3ec60b04c 100644
+--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
++++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+@@ -351,11 +351,17 @@ static void __adv7511_power_on(struct adv7511 *adv7511)
+ * from standby or are enabled. When the HPD goes low the adv7511 is
+ * reset and the outputs are disabled which might cause the monitor to
+ * go to standby again. To avoid this we ignore the HPD pin for the
+- * first few seconds after enabling the output.
++ * first few seconds after enabling the output. On the other hand
++ * adv7535 require to enable HPD Override bit for proper HPD.
+ */
+- regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
+- ADV7511_REG_POWER2_HPD_SRC_MASK,
+- ADV7511_REG_POWER2_HPD_SRC_NONE);
++ if (adv7511->type == ADV7535)
++ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
++ ADV7535_REG_POWER2_HPD_OVERRIDE,
++ ADV7535_REG_POWER2_HPD_OVERRIDE);
++ else
++ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
++ ADV7511_REG_POWER2_HPD_SRC_MASK,
++ ADV7511_REG_POWER2_HPD_SRC_NONE);
+ }
+
+ static void adv7511_power_on(struct adv7511 *adv7511)
+@@ -375,6 +381,10 @@ static void adv7511_power_on(struct adv7511 *adv7511)
+ static void __adv7511_power_off(struct adv7511 *adv7511)
+ {
+ /* TODO: setup additional power down modes */
++ if (adv7511->type == ADV7535)
++ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
++ ADV7535_REG_POWER2_HPD_OVERRIDE, 0);
++
+ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER,
+ ADV7511_POWER_POWER_DOWN,
+ ADV7511_POWER_POWER_DOWN);
+@@ -672,9 +682,14 @@ adv7511_detect(struct adv7511 *adv7511, struct drm_connector *connector)
+ status = connector_status_disconnected;
+ } else {
+ /* Renable HPD sensing */
+- regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
+- ADV7511_REG_POWER2_HPD_SRC_MASK,
+- ADV7511_REG_POWER2_HPD_SRC_BOTH);
++ if (adv7511->type == ADV7535)
++ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
++ ADV7535_REG_POWER2_HPD_OVERRIDE,
++ ADV7535_REG_POWER2_HPD_OVERRIDE);
++ else
++ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER2,
++ ADV7511_REG_POWER2_HPD_SRC_MASK,
++ ADV7511_REG_POWER2_HPD_SRC_BOTH);
+ }
+
+ adv7511->status = status;
+diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+index cab6c8b92efd4..6a4f20fccf841 100644
+--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
++++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+@@ -998,11 +998,21 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp,
+ if (!blocking)
+ return 0;
+
++ /*
++ * db[1]!=0: entering PSR, wait for fully active remote frame buffer.
++ * db[1]==0: exiting PSR, wait for either
++ * (a) ACTIVE_RESYNC - the sink "must display the
++ * incoming active frames from the Source device with no visible
++ * glitches and/or artifacts", even though timings may still be
++ * re-synchronizing; or
++ * (b) INACTIVE - the transition is fully complete.
++ */
+ ret = readx_poll_timeout(analogix_dp_get_psr_status, dp, psr_status,
+ psr_status >= 0 &&
+ ((vsc->db[1] && psr_status == DP_PSR_SINK_ACTIVE_RFB) ||
+- (!vsc->db[1] && psr_status == DP_PSR_SINK_INACTIVE)), 1500,
+- DP_TIMEOUT_PSR_LOOP_MS * 1000);
++ (!vsc->db[1] && (psr_status == DP_PSR_SINK_ACTIVE_RESYNC ||
++ psr_status == DP_PSR_SINK_INACTIVE))),
++ 1500, DP_TIMEOUT_PSR_LOOP_MS * 1000);
+ if (ret) {
+ dev_warn(dp->dev, "Failed to apply PSR %d\n", ret);
+ return ret;
+diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
+index 14d73fb1dd15b..392a9c56e9a00 100644
+--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
++++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
+@@ -720,7 +720,7 @@ static int edid_read(struct anx7625_data *ctx,
+ ret = sp_tx_aux_rd(ctx, 0xf1);
+
+ if (ret) {
+- sp_tx_rst_aux(ctx);
++ ret = sp_tx_rst_aux(ctx);
+ DRM_DEV_DEBUG_DRIVER(dev, "edid read fail, reset!\n");
+ } else {
+ ret = anx7625_reg_block_read(ctx, ctx->i2c.rx_p0_client,
+@@ -735,7 +735,7 @@ static int edid_read(struct anx7625_data *ctx,
+ if (cnt > EDID_TRY_CNT)
+ return -EIO;
+
+- return 0;
++ return ret;
+ }
+
+ static int segments_edid_read(struct anx7625_data *ctx,
+@@ -785,13 +785,14 @@ static int segments_edid_read(struct anx7625_data *ctx,
+ if (cnt > EDID_TRY_CNT)
+ return -EIO;
+
+- return 0;
++ return ret;
+ }
+
+ static int sp_tx_edid_read(struct anx7625_data *ctx,
+ u8 *pedid_blocks_buf)
+ {
+- u8 offset, edid_pos;
++ u8 offset;
++ int edid_pos;
+ int count, blocks_num;
+ u8 pblock_buf[MAX_DPCD_BUFFER_SIZE];
+ u8 i, j;
+@@ -887,7 +888,11 @@ static int sp_tx_edid_read(struct anx7625_data *ctx,
+ }
+
+ /* Reset aux channel */
+- sp_tx_rst_aux(ctx);
++ ret = sp_tx_rst_aux(ctx);
++ if (ret < 0) {
++ DRM_DEV_ERROR(dev, "Failed to reset aux channel!\n");
++ return ret;
++ }
+
+ return (blocks_num + 1);
+ }
+diff --git a/drivers/gpu/drm/bridge/cdns-dsi.c b/drivers/gpu/drm/bridge/cdns-dsi.c
+index e6e331071a00d..dd57b104aec3a 100644
+--- a/drivers/gpu/drm/bridge/cdns-dsi.c
++++ b/drivers/gpu/drm/bridge/cdns-dsi.c
+@@ -1286,6 +1286,7 @@ static const struct of_device_id cdns_dsi_of_match[] = {
+ { .compatible = "cdns,dsi" },
+ { },
+ };
++MODULE_DEVICE_TABLE(of, cdns_dsi_of_match);
+
+ static struct platform_driver cdns_dsi_platform_driver = {
+ .probe = cdns_dsi_drm_probe,
+diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c
+index 05eb759da6fc6..847a0dce7f1d3 100644
+--- a/drivers/gpu/drm/bridge/display-connector.c
++++ b/drivers/gpu/drm/bridge/display-connector.c
+@@ -107,7 +107,7 @@ static int display_connector_probe(struct platform_device *pdev)
+ {
+ struct display_connector *conn;
+ unsigned int type;
+- const char *label;
++ const char *label = NULL;
+ int ret;
+
+ conn = devm_kzalloc(&pdev->dev, sizeof(*conn), GFP_KERNEL);
+diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
+index 2f2a09adb4bc8..06b59b422c696 100644
+--- a/drivers/gpu/drm/bridge/ite-it66121.c
++++ b/drivers/gpu/drm/bridge/ite-it66121.c
+@@ -889,7 +889,7 @@ unlock:
+ static int it66121_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+ {
+- u32 vendor_ids[2], device_ids[2], revision_id;
++ u32 revision_id, vendor_ids[2] = { 0 }, device_ids[2] = { 0 };
+ struct device_node *ep;
+ int ret;
+ struct it66121_ctx *ctx;
+@@ -918,11 +918,26 @@ static int it66121_probe(struct i2c_client *client,
+ return -EINVAL;
+
+ ep = of_graph_get_remote_node(dev->of_node, 1, -1);
+- if (!ep)
+- return -EPROBE_DEFER;
++ if (!ep) {
++ dev_err(ctx->dev, "The endpoint is unconnected\n");
++ return -EINVAL;
++ }
++
++ if (!of_device_is_available(ep)) {
++ of_node_put(ep);
++ dev_err(ctx->dev, "The remote device is disabled\n");
++ return -ENODEV;
++ }
+
+ ctx->next_bridge = of_drm_find_bridge(ep);
+ of_node_put(ep);
++ if (!ctx->next_bridge) {
++ dev_dbg(ctx->dev, "Next bridge not found, deferring probe\n");
++ return -EPROBE_DEFER;
++ }
++
++ if (!ctx->next_bridge)
++ return -EPROBE_DEFER;
+
+ i2c_set_clientdata(client, ctx);
+ mutex_init(&ctx->lock);
+diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+index 3cac16db970f0..010657ea7af78 100644
+--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
++++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+@@ -167,9 +167,10 @@ static void lt9611uxc_hpd_work(struct work_struct *work)
+ struct lt9611uxc *lt9611uxc = container_of(work, struct lt9611uxc, work);
+ bool connected;
+
+- if (lt9611uxc->connector.dev)
+- drm_kms_helper_hotplug_event(lt9611uxc->connector.dev);
+- else {
++ if (lt9611uxc->connector.dev) {
++ if (lt9611uxc->connector.dev->mode_config.funcs)
++ drm_kms_helper_hotplug_event(lt9611uxc->connector.dev);
++ } else {
+
+ mutex_lock(&lt9611uxc->ocm_lock);
+ connected = lt9611uxc->hdmi_connected;
+@@ -339,6 +340,8 @@ static int lt9611uxc_connector_init(struct drm_bridge *bridge, struct lt9611uxc
+ return -ENODEV;
+ }
+
++ lt9611uxc->connector.polled = DRM_CONNECTOR_POLL_HPD;
++
+ drm_connector_helper_add(&lt9611uxc->connector,
+ &lt9611uxc_bridge_connector_helper_funcs);
+ ret = drm_connector_init(bridge->dev, &lt9611uxc->connector,
+diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+index d2808c4a6fb1c..cce98bf2a4e73 100644
+--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
++++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+@@ -306,19 +306,10 @@ out:
+ mutex_unlock(&ge_b850v3_lvds_dev_mutex);
+ }
+
+-static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c,
+- const struct i2c_device_id *id)
++static int ge_b850v3_register(void)
+ {
++ struct i2c_client *stdp4028_i2c = ge_b850v3_lvds_ptr->stdp4028_i2c;
+ struct device *dev = &stdp4028_i2c->dev;
+- int ret;
+-
+- ret = ge_b850v3_lvds_init(dev);
+-
+- if (ret)
+- return ret;
+-
+- ge_b850v3_lvds_ptr->stdp4028_i2c = stdp4028_i2c;
+- i2c_set_clientdata(stdp4028_i2c, ge_b850v3_lvds_ptr);
+
+ /* drm bridge initialization */
+ ge_b850v3_lvds_ptr->bridge.funcs = &ge_b850v3_lvds_funcs;
+@@ -343,6 +334,27 @@ static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c,
+ "ge-b850v3-lvds-dp", ge_b850v3_lvds_ptr);
+ }
+
++static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c,
++ const struct i2c_device_id *id)
++{
++ struct device *dev = &stdp4028_i2c->dev;
++ int ret;
++
++ ret = ge_b850v3_lvds_init(dev);
++
++ if (ret)
++ return ret;
++
++ ge_b850v3_lvds_ptr->stdp4028_i2c = stdp4028_i2c;
++ i2c_set_clientdata(stdp4028_i2c, ge_b850v3_lvds_ptr);
++
++ /* Only register after both bridges are probed */
++ if (!ge_b850v3_lvds_ptr->stdp2690_i2c)
++ return 0;
++
++ return ge_b850v3_register();
++}
++
+ static int stdp4028_ge_b850v3_fw_remove(struct i2c_client *stdp4028_i2c)
+ {
+ ge_b850v3_lvds_remove();
+@@ -386,7 +398,11 @@ static int stdp2690_ge_b850v3_fw_probe(struct i2c_client *stdp2690_i2c,
+ ge_b850v3_lvds_ptr->stdp2690_i2c = stdp2690_i2c;
+ i2c_set_clientdata(stdp2690_i2c, ge_b850v3_lvds_ptr);
+
+- return 0;
++ /* Only register after both bridges are probed */
++ if (!ge_b850v3_lvds_ptr->stdp4028_i2c)
++ return 0;
++
++ return ge_b850v3_register();
+ }
+
+ static int stdp2690_ge_b850v3_fw_remove(struct i2c_client *stdp2690_i2c)
+diff --git a/drivers/gpu/drm/bridge/nwl-dsi.c b/drivers/gpu/drm/bridge/nwl-dsi.c
+index ed8ac5059cd26..691039aba87f4 100644
+--- a/drivers/gpu/drm/bridge/nwl-dsi.c
++++ b/drivers/gpu/drm/bridge/nwl-dsi.c
+@@ -7,6 +7,7 @@
+ */
+
+ #include <linux/bitfield.h>
++#include <linux/bits.h>
+ #include <linux/clk.h>
+ #include <linux/irq.h>
+ #include <linux/math64.h>
+@@ -196,12 +197,9 @@ static u32 ps2bc(struct nwl_dsi *dsi, unsigned long long ps)
+ /*
+ * ui2bc - UI time periods to byte clock cycles
+ */
+-static u32 ui2bc(struct nwl_dsi *dsi, unsigned long long ui)
++static u32 ui2bc(unsigned int ui)
+ {
+- u32 bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
+-
+- return DIV64_U64_ROUND_UP(ui * dsi->lanes,
+- dsi->mode.clock * 1000 * bpp);
++ return DIV_ROUND_UP(ui, BITS_PER_BYTE);
+ }
+
+ /*
+@@ -232,12 +230,12 @@ static int nwl_dsi_config_host(struct nwl_dsi *dsi)
+ }
+
+ /* values in byte clock cycles */
+- cycles = ui2bc(dsi, cfg->clk_pre);
++ cycles = ui2bc(cfg->clk_pre);
+ DRM_DEV_DEBUG_DRIVER(dsi->dev, "cfg_t_pre: 0x%x\n", cycles);
+ nwl_dsi_write(dsi, NWL_DSI_CFG_T_PRE, cycles);
+ cycles = ps2bc(dsi, cfg->lpx + cfg->clk_prepare + cfg->clk_zero);
+ DRM_DEV_DEBUG_DRIVER(dsi->dev, "cfg_tx_gap (pre): 0x%x\n", cycles);
+- cycles += ui2bc(dsi, cfg->clk_pre);
++ cycles += ui2bc(cfg->clk_pre);
+ DRM_DEV_DEBUG_DRIVER(dsi->dev, "cfg_t_post: 0x%x\n", cycles);
+ nwl_dsi_write(dsi, NWL_DSI_CFG_T_POST, cycles);
+ cycles = ps2bc(dsi, cfg->hs_exit);
+@@ -863,18 +861,19 @@ nwl_dsi_bridge_mode_set(struct drm_bridge *bridge,
+ memcpy(&dsi->mode, adjusted_mode, sizeof(dsi->mode));
+ drm_mode_debug_printmodeline(adjusted_mode);
+
+- pm_runtime_get_sync(dev);
++ if (pm_runtime_resume_and_get(dev) < 0)
++ return;
+
+ if (clk_prepare_enable(dsi->lcdif_clk) < 0)
+- return;
++ goto runtime_put;
+ if (clk_prepare_enable(dsi->core_clk) < 0)
+- return;
++ goto runtime_put;
+
+ /* Step 1 from DSI reset-out instructions */
+ ret = reset_control_deassert(dsi->rst_pclk);
+ if (ret < 0) {
+ DRM_DEV_ERROR(dev, "Failed to deassert PCLK: %d\n", ret);
+- return;
++ goto runtime_put;
+ }
+
+ /* Step 2 from DSI reset-out instructions */
+@@ -884,13 +883,18 @@ nwl_dsi_bridge_mode_set(struct drm_bridge *bridge,
+ ret = reset_control_deassert(dsi->rst_esc);
+ if (ret < 0) {
+ DRM_DEV_ERROR(dev, "Failed to deassert ESC: %d\n", ret);
+- return;
++ goto runtime_put;
+ }
+ ret = reset_control_deassert(dsi->rst_byte);
+ if (ret < 0) {
+ DRM_DEV_ERROR(dev, "Failed to deassert BYTE: %d\n", ret);
+- return;
++ goto runtime_put;
+ }
++
++ return;
++
++runtime_put:
++ pm_runtime_put_sync(dev);
+ }
+
+ static void
+@@ -939,6 +943,40 @@ static void nwl_dsi_bridge_detach(struct drm_bridge *bridge)
+ drm_of_panel_bridge_remove(dsi->dev->of_node, 1, 0);
+ }
+
++static u32 *nwl_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
++ struct drm_bridge_state *bridge_state,
++ struct drm_crtc_state *crtc_state,
++ struct drm_connector_state *conn_state,
++ u32 output_fmt,
++ unsigned int *num_input_fmts)
++{
++ u32 *input_fmts, input_fmt;
++
++ *num_input_fmts = 0;
++
++ switch (output_fmt) {
++ /* If MEDIA_BUS_FMT_FIXED is tested, return default bus format */
++ case MEDIA_BUS_FMT_FIXED:
++ input_fmt = MEDIA_BUS_FMT_RGB888_1X24;
++ break;
++ case MEDIA_BUS_FMT_RGB888_1X24:
++ case MEDIA_BUS_FMT_RGB666_1X18:
++ case MEDIA_BUS_FMT_RGB565_1X16:
++ input_fmt = output_fmt;
++ break;
++ default:
++ return NULL;
++ }
++
++ input_fmts = kcalloc(1, sizeof(*input_fmts), GFP_KERNEL);
++ if (!input_fmts)
++ return NULL;
++ input_fmts[0] = input_fmt;
++ *num_input_fmts = 1;
++
++ return input_fmts;
++}
++
+ static const struct drm_bridge_funcs nwl_dsi_bridge_funcs = {
+ .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+ .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+@@ -946,6 +984,7 @@ static const struct drm_bridge_funcs nwl_dsi_bridge_funcs = {
+ .atomic_check = nwl_dsi_bridge_atomic_check,
+ .atomic_enable = nwl_dsi_bridge_atomic_enable,
+ .atomic_disable = nwl_dsi_bridge_atomic_disable,
++ .atomic_get_input_bus_fmts = nwl_bridge_atomic_get_input_bus_fmts,
+ .mode_set = nwl_dsi_bridge_mode_set,
+ .mode_valid = nwl_dsi_bridge_mode_valid,
+ .attach = nwl_dsi_bridge_attach,
+@@ -1171,6 +1210,7 @@ static int nwl_dsi_probe(struct platform_device *pdev)
+
+ ret = nwl_dsi_select_input(dsi);
+ if (ret < 0) {
++ pm_runtime_disable(dev);
+ mipi_dsi_host_unregister(&dsi->dsi_host);
+ return ret;
+ }
+diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
+index 843265d7f1b12..ec7745c31da07 100644
+--- a/drivers/gpu/drm/bridge/sil-sii8620.c
++++ b/drivers/gpu/drm/bridge/sil-sii8620.c
+@@ -2120,7 +2120,7 @@ static void sii8620_init_rcp_input_dev(struct sii8620 *ctx)
+ if (ret) {
+ dev_err(ctx->dev, "Failed to register RC device\n");
+ ctx->error = ret;
+- rc_free_device(ctx->rc_dev);
++ rc_free_device(rc_dev);
+ return;
+ }
+ ctx->rc_dev = rc_dev;
+diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
+index d0db1acf11d73..7d2ed0ed2fe26 100644
+--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
+@@ -320,13 +320,17 @@ static int dw_hdmi_open(struct snd_pcm_substream *substream)
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct snd_dw_hdmi *dw = substream->private_data;
+ void __iomem *base = dw->data.base;
++ u8 *eld;
+ int ret;
+
+ runtime->hw = dw_hdmi_hw;
+
+- ret = snd_pcm_hw_constraint_eld(runtime, dw->data.eld);
+- if (ret < 0)
+- return ret;
++ eld = dw->data.get_eld(dw->data.hdmi);
++ if (eld) {
++ ret = snd_pcm_hw_constraint_eld(runtime, eld);
++ if (ret < 0)
++ return ret;
++ }
+
+ ret = snd_pcm_limit_hw_rates(runtime);
+ if (ret < 0)
+diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-audio.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-audio.h
+index cb07dc0da5a70..f72d27208ebef 100644
+--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-audio.h
++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-audio.h
+@@ -9,15 +9,15 @@ struct dw_hdmi_audio_data {
+ void __iomem *base;
+ int irq;
+ struct dw_hdmi *hdmi;
+- u8 *eld;
++ u8 *(*get_eld)(struct dw_hdmi *hdmi);
+ };
+
+ struct dw_hdmi_i2s_audio_data {
+ struct dw_hdmi *hdmi;
+- u8 *eld;
+
+ void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+ u8 (*read)(struct dw_hdmi *hdmi, int offset);
++ u8 *(*get_eld)(struct dw_hdmi *hdmi);
+ };
+
+ #endif
+diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
+index feb04f127b550..f50b47ac11a82 100644
+--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
+@@ -135,8 +135,15 @@ static int dw_hdmi_i2s_get_eld(struct device *dev, void *data, uint8_t *buf,
+ size_t len)
+ {
+ struct dw_hdmi_i2s_audio_data *audio = data;
++ u8 *eld;
++
++ eld = audio->get_eld(audio->hdmi);
++ if (eld)
++ memcpy(buf, eld, min_t(size_t, MAX_ELD_BYTES, len));
++ else
++ /* Pass en empty ELD if connector not available */
++ memset(buf, 0, len);
+
+- memcpy(buf, audio->eld, min_t(size_t, MAX_ELD_BYTES, len));
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+index f08d0fded61f7..25d58dcfc87e1 100644
+--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
++++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+@@ -757,6 +757,14 @@ static void hdmi_enable_audio_clk(struct dw_hdmi *hdmi, bool enable)
+ hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+ }
+
++static u8 *hdmi_audio_get_eld(struct dw_hdmi *hdmi)
++{
++ if (!hdmi->curr_conn)
++ return NULL;
++
++ return hdmi->curr_conn->eld;
++}
++
+ static void dw_hdmi_ahb_audio_enable(struct dw_hdmi *hdmi)
+ {
+ hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n);
+@@ -2543,8 +2551,9 @@ static u32 *dw_hdmi_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
+ if (!output_fmts)
+ return NULL;
+
+- /* If dw-hdmi is the only bridge, avoid negociating with ourselves */
+- if (list_is_singular(&bridge->encoder->bridge_chain)) {
++ /* If dw-hdmi is the first or only bridge, avoid negociating with ourselves */
++ if (list_is_singular(&bridge->encoder->bridge_chain) ||
++ list_is_first(&bridge->chain_node, &bridge->encoder->bridge_chain)) {
+ *num_output_fmts = 1;
+ output_fmts[0] = MEDIA_BUS_FMT_FIXED;
+
+@@ -3431,7 +3440,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
+ audio.base = hdmi->regs;
+ audio.irq = irq;
+ audio.hdmi = hdmi;
+- audio.eld = hdmi->connector.eld;
++ audio.get_eld = hdmi_audio_get_eld;
+ hdmi->enable_audio = dw_hdmi_ahb_audio_enable;
+ hdmi->disable_audio = dw_hdmi_ahb_audio_disable;
+
+@@ -3444,7 +3453,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
+ struct dw_hdmi_i2s_audio_data audio;
+
+ audio.hdmi = hdmi;
+- audio.eld = hdmi->connector.eld;
++ audio.get_eld = hdmi_audio_get_eld;
+ audio.write = hdmi_writeb;
+ audio.read = hdmi_readb;
+ hdmi->enable_audio = dw_hdmi_i2s_audio_enable;
+diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+index e44e18a0112af..56c3fd08c6a0b 100644
+--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
++++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+@@ -1199,6 +1199,7 @@ __dw_mipi_dsi_probe(struct platform_device *pdev,
+ ret = mipi_dsi_host_register(&dsi->dsi_host);
+ if (ret) {
+ dev_err(dev, "Failed to register MIPI host: %d\n", ret);
++ pm_runtime_disable(dev);
+ dw_mipi_dsi_debugfs_remove(dsi);
+ return ERR_PTR(ret);
+ }
+diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+index 41d48a393e7f5..45a5f1e48f0ef 100644
+--- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c
++++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+@@ -188,6 +188,7 @@ static const struct regmap_config ti_sn65dsi86_regmap_config = {
+ .val_bits = 8,
+ .volatile_table = &ti_sn_bridge_volatile_table,
+ .cache_type = REGCACHE_NONE,
++ .max_register = 0xFF,
+ };
+
+ static void ti_sn65dsi86_write_u16(struct ti_sn65dsi86 *pdata,
+@@ -1472,6 +1473,7 @@ static inline void ti_sn_gpio_unregister(void) {}
+
+ static void ti_sn65dsi86_runtime_disable(void *data)
+ {
++ pm_runtime_dont_use_autosuspend(data);
+ pm_runtime_disable(data);
+ }
+
+@@ -1531,11 +1533,11 @@ static int ti_sn65dsi86_probe(struct i2c_client *client,
+ "failed to get reference clock\n");
+
+ pm_runtime_enable(dev);
++ pm_runtime_set_autosuspend_delay(pdata->dev, 500);
++ pm_runtime_use_autosuspend(pdata->dev);
+ ret = devm_add_action_or_reset(dev, ti_sn65dsi86_runtime_disable, dev);
+ if (ret)
+ return ret;
+- pm_runtime_set_autosuspend_delay(pdata->dev, 500);
+- pm_runtime_use_autosuspend(pdata->dev);
+
+ ti_sn65dsi86_debugfs_init(pdata);
+
+diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
+index ff1416cd609a5..a1e4c7905ebbe 100644
+--- a/drivers/gpu/drm/drm_atomic.c
++++ b/drivers/gpu/drm/drm_atomic.c
+@@ -1310,8 +1310,10 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
+
+ DRM_DEBUG_ATOMIC("checking %p\n", state);
+
+- for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
+- requested_crtc |= drm_crtc_mask(crtc);
++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
++ if (new_crtc_state->enable)
++ requested_crtc |= drm_crtc_mask(crtc);
++ }
+
+ for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
+ ret = drm_atomic_plane_check(old_plane_state, new_plane_state);
+@@ -1360,8 +1362,10 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
+ }
+ }
+
+- for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
+- affected_crtc |= drm_crtc_mask(crtc);
++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
++ if (new_crtc_state->enable)
++ affected_crtc |= drm_crtc_mask(crtc);
++ }
+
+ /*
+ * For commits that allow modesets drivers can add other CRTCs to the
+diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
+index 2c0c6ec928200..ff2bc9a118011 100644
+--- a/drivers/gpu/drm/drm_atomic_helper.c
++++ b/drivers/gpu/drm/drm_atomic_helper.c
+@@ -1001,7 +1001,7 @@ crtc_needs_disable(struct drm_crtc_state *old_state,
+ * it's in self refresh mode and needs to be fully disabled.
+ */
+ return old_state->active ||
+- (old_state->self_refresh_active && !new_state->enable) ||
++ (old_state->self_refresh_active && !new_state->active) ||
+ new_state->self_refresh_active;
+ }
+
+diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
+index 909f318331816..f195c70131373 100644
+--- a/drivers/gpu/drm/drm_atomic_uapi.c
++++ b/drivers/gpu/drm/drm_atomic_uapi.c
+@@ -76,15 +76,17 @@ int drm_atomic_set_mode_for_crtc(struct drm_crtc_state *state,
+ state->mode_blob = NULL;
+
+ if (mode) {
++ struct drm_property_blob *blob;
++
+ drm_mode_convert_to_umode(&umode, mode);
+- state->mode_blob =
+- drm_property_create_blob(state->crtc->dev,
+- sizeof(umode),
+- &umode);
+- if (IS_ERR(state->mode_blob))
+- return PTR_ERR(state->mode_blob);
++ blob = drm_property_create_blob(crtc->dev,
++ sizeof(umode), &umode);
++ if (IS_ERR(blob))
++ return PTR_ERR(blob);
+
+ drm_mode_copy(&state->mode, mode);
++
++ state->mode_blob = blob;
+ state->enable = true;
+ drm_dbg_atomic(crtc->dev,
+ "Set [MODE:%s] for [CRTC:%d:%s] state %p\n",
+diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
+index 2ba257b1ae208..e9b7926d9b663 100644
+--- a/drivers/gpu/drm/drm_connector.c
++++ b/drivers/gpu/drm/drm_connector.c
+@@ -2233,6 +2233,9 @@ EXPORT_SYMBOL(drm_connector_atomic_hdr_metadata_equal);
+ void drm_connector_set_vrr_capable_property(
+ struct drm_connector *connector, bool capable)
+ {
++ if (!connector->vrr_capable_property)
++ return;
++
+ drm_object_property_set_value(&connector->base,
+ connector->vrr_capable_property,
+ capable);
+diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
+index 6d0f2c447f3b9..7bb24523a7493 100644
+--- a/drivers/gpu/drm/drm_dp_helper.c
++++ b/drivers/gpu/drm/drm_dp_helper.c
+@@ -3214,27 +3214,13 @@ int drm_edp_backlight_enable(struct drm_dp_aux *aux, const struct drm_edp_backli
+ const u16 level)
+ {
+ int ret;
+- u8 dpcd_buf, new_dpcd_buf;
++ u8 dpcd_buf = DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD;
+
+- ret = drm_dp_dpcd_readb(aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf);
+- if (ret != 1) {
+- drm_dbg_kms(aux->drm_dev,
+- "%s: Failed to read backlight mode: %d\n", aux->name, ret);
+- return ret < 0 ? ret : -EIO;
+- }
+-
+- new_dpcd_buf = dpcd_buf;
+-
+- if ((dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) != DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD) {
+- new_dpcd_buf &= ~DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
+- new_dpcd_buf |= DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD;
+-
+- if (bl->pwmgen_bit_count) {
+- ret = drm_dp_dpcd_writeb(aux, DP_EDP_PWMGEN_BIT_COUNT, bl->pwmgen_bit_count);
+- if (ret != 1)
+- drm_dbg_kms(aux->drm_dev, "%s: Failed to write aux pwmgen bit count: %d\n",
+- aux->name, ret);
+- }
++ if (bl->pwmgen_bit_count) {
++ ret = drm_dp_dpcd_writeb(aux, DP_EDP_PWMGEN_BIT_COUNT, bl->pwmgen_bit_count);
++ if (ret != 1)
++ drm_dbg_kms(aux->drm_dev, "%s: Failed to write aux pwmgen bit count: %d\n",
++ aux->name, ret);
+ }
+
+ if (bl->pwm_freq_pre_divider) {
+@@ -3244,16 +3230,14 @@ int drm_edp_backlight_enable(struct drm_dp_aux *aux, const struct drm_edp_backli
+ "%s: Failed to write aux backlight frequency: %d\n",
+ aux->name, ret);
+ else
+- new_dpcd_buf |= DP_EDP_BACKLIGHT_FREQ_AUX_SET_ENABLE;
++ dpcd_buf |= DP_EDP_BACKLIGHT_FREQ_AUX_SET_ENABLE;
+ }
+
+- if (new_dpcd_buf != dpcd_buf) {
+- ret = drm_dp_dpcd_writeb(aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, new_dpcd_buf);
+- if (ret != 1) {
+- drm_dbg_kms(aux->drm_dev, "%s: Failed to write aux backlight mode: %d\n",
+- aux->name, ret);
+- return ret < 0 ? ret : -EIO;
+- }
++ ret = drm_dp_dpcd_writeb(aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, dpcd_buf);
++ if (ret != 1) {
++ drm_dbg_kms(aux->drm_dev, "%s: Failed to write aux backlight mode: %d\n",
++ aux->name, ret);
++ return ret < 0 ? ret : -EIO;
+ }
+
+ ret = drm_edp_backlight_set_level(aux, bl, level);
+diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
+index 86d13d6bc4631..b3e3babe18c0e 100644
+--- a/drivers/gpu/drm/drm_dp_mst_topology.c
++++ b/drivers/gpu/drm/drm_dp_mst_topology.c
+@@ -4834,6 +4834,7 @@ static void fetch_monitor_name(struct drm_dp_mst_topology_mgr *mgr,
+
+ mst_edid = drm_dp_mst_get_edid(port->connector, mgr, port);
+ drm_edid_get_monitor_name(mst_edid, name, namelen);
++ kfree(mst_edid);
+ }
+
+ /**
+diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
+index 7a5097467ba5c..b3a1636d1b984 100644
+--- a/drivers/gpu/drm/drm_drv.c
++++ b/drivers/gpu/drm/drm_drv.c
+@@ -581,6 +581,7 @@ static int drm_dev_init(struct drm_device *dev,
+ const struct drm_driver *driver,
+ struct device *parent)
+ {
++ struct inode *inode;
+ int ret;
+
+ if (!drm_core_init_complete) {
+@@ -617,13 +618,15 @@ static int drm_dev_init(struct drm_device *dev,
+ if (ret)
+ return ret;
+
+- dev->anon_inode = drm_fs_inode_new();
+- if (IS_ERR(dev->anon_inode)) {
+- ret = PTR_ERR(dev->anon_inode);
++ inode = drm_fs_inode_new();
++ if (IS_ERR(inode)) {
++ ret = PTR_ERR(inode);
+ DRM_ERROR("Cannot allocate anonymous inode: %d\n", ret);
+ goto err;
+ }
+
++ dev->anon_inode = inode;
++
+ if (drm_core_check_feature(dev, DRIVER_RENDER)) {
+ ret = drm_minor_alloc(dev, DRM_MINOR_RENDER);
+ if (ret)
+diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
+index ea9a79bc95839..ee6f44f9a81c3 100644
+--- a/drivers/gpu/drm/drm_edid.c
++++ b/drivers/gpu/drm/drm_edid.c
+@@ -4776,7 +4776,8 @@ bool drm_detect_monitor_audio(struct edid *edid)
+ if (!edid_ext)
+ goto end;
+
+- has_audio = ((edid_ext[3] & EDID_BASIC_AUDIO) != 0);
++ has_audio = (edid_ext[0] == CEA_EXT &&
++ (edid_ext[3] & EDID_BASIC_AUDIO) != 0);
+
+ if (has_audio) {
+ DRM_DEBUG_KMS("Monitor has basic audio support\n");
+@@ -5003,21 +5004,21 @@ static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector,
+
+ if (hdmi[6] & DRM_EDID_HDMI_DC_30) {
+ dc_bpc = 10;
+- info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_30;
++ info->edid_hdmi_rgb444_dc_modes |= DRM_EDID_HDMI_DC_30;
+ DRM_DEBUG("%s: HDMI sink does deep color 30.\n",
+ connector->name);
+ }
+
+ if (hdmi[6] & DRM_EDID_HDMI_DC_36) {
+ dc_bpc = 12;
+- info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_36;
++ info->edid_hdmi_rgb444_dc_modes |= DRM_EDID_HDMI_DC_36;
+ DRM_DEBUG("%s: HDMI sink does deep color 36.\n",
+ connector->name);
+ }
+
+ if (hdmi[6] & DRM_EDID_HDMI_DC_48) {
+ dc_bpc = 16;
+- info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_48;
++ info->edid_hdmi_rgb444_dc_modes |= DRM_EDID_HDMI_DC_48;
+ DRM_DEBUG("%s: HDMI sink does deep color 48.\n",
+ connector->name);
+ }
+@@ -5032,16 +5033,9 @@ static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector,
+ connector->name, dc_bpc);
+ info->bpc = dc_bpc;
+
+- /*
+- * Deep color support mandates RGB444 support for all video
+- * modes and forbids YCRCB422 support for all video modes per
+- * HDMI 1.3 spec.
+- */
+- info->color_formats = DRM_COLOR_FORMAT_RGB444;
+-
+ /* YCRCB444 is optional according to spec. */
+ if (hdmi[6] & DRM_EDID_HDMI_DC_Y444) {
+- info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
++ info->edid_hdmi_ycbcr444_dc_modes = info->edid_hdmi_rgb444_dc_modes;
+ DRM_DEBUG("%s: HDMI sink does YCRCB444 in deep color.\n",
+ connector->name);
+ }
+@@ -5205,6 +5199,7 @@ u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edi
+ if (!(edid->input & DRM_EDID_INPUT_DIGITAL))
+ return quirks;
+
++ info->color_formats |= DRM_COLOR_FORMAT_RGB444;
+ drm_parse_cea_ext(connector, edid);
+
+ /*
+@@ -5253,7 +5248,6 @@ u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edi
+ DRM_DEBUG("%s: Assigning EDID-1.4 digital sink color depth as %d bpc.\n",
+ connector->name, info->bpc);
+
+- info->color_formats |= DRM_COLOR_FORMAT_RGB444;
+ if (edid->features & DRM_EDID_FEATURE_RGB_YCRCB444)
+ info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
+ if (edid->features & DRM_EDID_FEATURE_RGB_YCRCB422)
+diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
+index 8e7a124d6c5a3..ed589e7182bb4 100644
+--- a/drivers/gpu/drm/drm_fb_helper.c
++++ b/drivers/gpu/drm/drm_fb_helper.c
+@@ -1743,7 +1743,13 @@ void drm_fb_helper_fill_info(struct fb_info *info,
+ sizes->fb_width, sizes->fb_height);
+
+ info->par = fb_helper;
+- snprintf(info->fix.id, sizeof(info->fix.id), "%s",
++ /*
++ * The DRM drivers fbdev emulation device name can be confusing if the
++ * driver name also has a "drm" suffix on it. Leading to names such as
++ * "simpledrmdrmfb" in /proc/fb. Unfortunately, it's an uAPI and can't
++ * be changed due user-space tools (e.g: pm-utils) matching against it.
++ */
++ snprintf(info->fix.id, sizeof(info->fix.id), "%sdrmfb",
+ fb_helper->dev->driver->name);
+
+ }
+@@ -2340,6 +2346,7 @@ static int drm_fb_helper_generic_probe(struct drm_fb_helper *fb_helper,
+ fbi->fbops = &drm_fbdev_fb_ops;
+ fbi->screen_size = fb->height * fb->pitches[0];
+ fbi->fix.smem_len = fbi->screen_size;
++ fbi->flags = FBINFO_DEFAULT;
+
+ drm_fb_helper_fill_info(fbi, fb_helper, sizes);
+
+@@ -2347,19 +2354,21 @@ static int drm_fb_helper_generic_probe(struct drm_fb_helper *fb_helper,
+ fbi->screen_buffer = vzalloc(fbi->screen_size);
+ if (!fbi->screen_buffer)
+ return -ENOMEM;
++ fbi->flags |= FBINFO_VIRTFB | FBINFO_READS_FAST;
+
+ fbi->fbdefio = &drm_fbdev_defio;
+-
+ fb_deferred_io_init(fbi);
+ } else {
+ /* buffer is mapped for HW framebuffer */
+ ret = drm_client_buffer_vmap(fb_helper->buffer, &map);
+ if (ret)
+ return ret;
+- if (map.is_iomem)
++ if (map.is_iomem) {
+ fbi->screen_base = map.vaddr_iomem;
+- else
++ } else {
+ fbi->screen_buffer = map.vaddr;
++ fbi->flags |= FBINFO_VIRTFB;
++ }
+
+ /*
+ * Shamelessly leak the physical address to user-space. As
+diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
+index d53388199f34c..6533efa840204 100644
+--- a/drivers/gpu/drm/drm_gem_cma_helper.c
++++ b/drivers/gpu/drm/drm_gem_cma_helper.c
+@@ -210,8 +210,13 @@ void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
+ dma_buf_vunmap(gem_obj->import_attach->dmabuf, &map);
+ drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
+ } else if (cma_obj->vaddr) {
+- dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
+- cma_obj->vaddr, cma_obj->paddr);
++ if (cma_obj->map_noncoherent)
++ dma_free_noncoherent(gem_obj->dev->dev, cma_obj->base.size,
++ cma_obj->vaddr, cma_obj->paddr,
++ DMA_TO_DEVICE);
++ else
++ dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
++ cma_obj->vaddr, cma_obj->paddr);
+ }
+
+ drm_gem_object_release(gem_obj);
+@@ -510,6 +515,7 @@ int drm_gem_cma_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+ */
+ vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node);
+ vma->vm_flags &= ~VM_PFNMAP;
++ vma->vm_flags |= VM_DONTEXPAND;
+
+ cma_obj = to_drm_gem_cma_obj(obj);
+
+diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c
+index e1b2ce4921ae7..f5ab891731d0b 100644
+--- a/drivers/gpu/drm/drm_panel_orientation_quirks.c
++++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c
+@@ -109,6 +109,18 @@ static const struct drm_dmi_panel_orientation_data lcd1200x1920_rightside_up = {
+ .orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
+ };
+
++static const struct drm_dmi_panel_orientation_data lcd1280x1920_rightside_up = {
++ .width = 1280,
++ .height = 1920,
++ .orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
++};
++
++static const struct drm_dmi_panel_orientation_data lcd1600x2560_leftside_up = {
++ .width = 1600,
++ .height = 2560,
++ .orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP,
++};
++
+ static const struct dmi_system_id orientation_data[] = {
+ { /* Acer One 10 (S1003) */
+ .matches = {
+@@ -154,6 +166,12 @@ static const struct dmi_system_id orientation_data[] = {
+ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "MicroPC"),
+ },
+ .driver_data = (void *)&lcd720x1280_rightside_up,
++ }, { /* GPD Win Max */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "GPD"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "G1619-01"),
++ },
++ .driver_data = (void *)&lcd800x1280_rightside_up,
+ }, { /*
+ * GPD Pocket, note that the the DMI data is less generic then
+ * it seems, devices with a board-vendor of "AMI Corporation"
+@@ -205,6 +223,13 @@ static const struct dmi_system_id orientation_data[] = {
+ DMI_EXACT_MATCH(DMI_BOARD_NAME, "TW891"),
+ },
+ .driver_data = (void *)&itworks_tw891,
++ }, { /* KD Kurio Smart C15200 2-in-1 */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "KD Interactive"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Kurio Smart"),
++ DMI_EXACT_MATCH(DMI_BOARD_NAME, "KDM960BCP"),
++ },
++ .driver_data = (void *)&lcd800x1280_rightside_up,
+ }, { /*
+ * Lenovo Ideapad Miix 310 laptop, only some production batches
+ * have a portrait screen, the resolution checks makes the quirk
+@@ -223,12 +248,23 @@ static const struct dmi_system_id orientation_data[] = {
+ DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
+ },
+ .driver_data = (void *)&lcd800x1280_rightside_up,
+- }, { /* Lenovo Ideapad D330 */
++ }, { /* Lenovo Ideapad D330-10IGM (HD) */
+ .matches = {
+ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+- DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "81H3"),
+ DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad D330-10IGM"),
+ },
++ .driver_data = (void *)&lcd800x1280_rightside_up,
++ }, { /* Lenovo Ideapad D330-10IGM (FHD) */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad D330-10IGM"),
++ },
++ .driver_data = (void *)&lcd1200x1920_rightside_up,
++ }, { /* Lenovo Yoga Book X90F / X91F / X91L */
++ .matches = {
++ /* Non exact match to match all versions */
++ DMI_MATCH(DMI_PRODUCT_NAME, "Lenovo YB1-X9"),
++ },
+ .driver_data = (void *)&lcd1200x1920_rightside_up,
+ }, { /* OneGX1 Pro */
+ .matches = {
+@@ -237,6 +273,25 @@ static const struct dmi_system_id orientation_data[] = {
+ DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Default string"),
+ },
+ .driver_data = (void *)&onegx1_pro,
++ }, { /* OneXPlayer */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ONE-NETBOOK TECHNOLOGY CO., LTD."),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "ONE XPLAYER"),
++ },
++ .driver_data = (void *)&lcd1600x2560_leftside_up,
++ }, { /* Samsung GalaxyBook 10.6 */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Galaxy Book 10.6"),
++ },
++ .driver_data = (void *)&lcd1280x1920_rightside_up,
++ }, { /* Valve Steam Deck */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Valve"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Jupiter"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "1"),
++ },
++ .driver_data = (void *)&lcd800x1280_rightside_up,
+ }, { /* VIOS LTH17 */
+ .matches = {
+ DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"),
+diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
+index 5b2d0ca03705c..838b32b70bce6 100644
+--- a/drivers/gpu/drm/drm_plane_helper.c
++++ b/drivers/gpu/drm/drm_plane_helper.c
+@@ -123,7 +123,6 @@ static int drm_plane_helper_check_update(struct drm_plane *plane,
+ .crtc_w = drm_rect_width(dst),
+ .crtc_h = drm_rect_height(dst),
+ .rotation = rotation,
+- .visible = *visible,
+ };
+ struct drm_crtc_state crtc_state = {
+ .crtc = crtc,
+diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
+index deb23dbec8b52..d6c7f4f9a7a29 100644
+--- a/drivers/gpu/drm/drm_prime.c
++++ b/drivers/gpu/drm/drm_prime.c
+@@ -719,11 +719,13 @@ int drm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+ if (obj->funcs && obj->funcs->mmap) {
+ vma->vm_ops = obj->funcs->vm_ops;
+
++ drm_gem_object_get(obj);
+ ret = obj->funcs->mmap(obj, vma);
+- if (ret)
++ if (ret) {
++ drm_gem_object_put(obj);
+ return ret;
++ }
+ vma->vm_private_data = obj;
+- drm_gem_object_get(obj);
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
+index c9a9d74f338c1..7e48dcd1bee4d 100644
+--- a/drivers/gpu/drm/drm_syncobj.c
++++ b/drivers/gpu/drm/drm_syncobj.c
+@@ -404,8 +404,17 @@ int drm_syncobj_find_fence(struct drm_file *file_private,
+
+ if (*fence) {
+ ret = dma_fence_chain_find_seqno(fence, point);
+- if (!ret)
++ if (!ret) {
++ /* If the requested seqno is already signaled
++ * drm_syncobj_find_fence may return a NULL
++ * fence. To make sure the recipient gets
++ * signalled, use a new fence instead.
++ */
++ if (!*fence)
++ *fence = dma_fence_get_stub();
++
+ goto out;
++ }
+ dma_fence_put(*fence);
+ } else {
+ ret = -EINVAL;
+@@ -844,12 +853,57 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
+ &args->handle);
+ }
+
++
++/*
++ * Try to flatten a dma_fence_chain into a dma_fence_array so that it can be
++ * added as timeline fence to a chain again.
++ */
++static int drm_syncobj_flatten_chain(struct dma_fence **f)
++{
++ struct dma_fence_chain *chain = to_dma_fence_chain(*f);
++ struct dma_fence *tmp, **fences;
++ struct dma_fence_array *array;
++ unsigned int count;
++
++ if (!chain)
++ return 0;
++
++ count = 0;
++ dma_fence_chain_for_each(tmp, &chain->base)
++ ++count;
++
++ fences = kmalloc_array(count, sizeof(*fences), GFP_KERNEL);
++ if (!fences)
++ return -ENOMEM;
++
++ count = 0;
++ dma_fence_chain_for_each(tmp, &chain->base)
++ fences[count++] = dma_fence_get(tmp);
++
++ array = dma_fence_array_create(count, fences,
++ dma_fence_context_alloc(1),
++ 1, false);
++ if (!array)
++ goto free_fences;
++
++ dma_fence_put(*f);
++ *f = &array->base;
++ return 0;
++
++free_fences:
++ while (count--)
++ dma_fence_put(fences[count]);
++
++ kfree(fences);
++ return -ENOMEM;
++}
++
+ static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
+ struct drm_syncobj_transfer *args)
+ {
+ struct drm_syncobj *timeline_syncobj = NULL;
+- struct dma_fence *fence;
+ struct dma_fence_chain *chain;
++ struct dma_fence *fence;
+ int ret;
+
+ timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
+@@ -860,16 +914,22 @@ static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
+ args->src_point, args->flags,
+ &fence);
+ if (ret)
+- goto err;
++ goto err_put_timeline;
++
++ ret = drm_syncobj_flatten_chain(&fence);
++ if (ret)
++ goto err_free_fence;
++
+ chain = dma_fence_chain_alloc();
+ if (!chain) {
+ ret = -ENOMEM;
+- goto err1;
++ goto err_free_fence;
+ }
++
+ drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
+-err1:
++err_free_fence:
+ dma_fence_put(fence);
+-err:
++err_put_timeline:
+ drm_syncobj_put(timeline_syncobj);
+
+ return ret;
+diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+index 486259e154aff..90488ab8c6d8e 100644
+--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
++++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+@@ -469,6 +469,12 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
+ return -EINVAL;
+ }
+
++ if (args->stream_size > SZ_128K || args->nr_relocs > SZ_128K ||
++ args->nr_bos > SZ_128K || args->nr_pmrs > 128) {
++ DRM_ERROR("submit arguments out of size limits\n");
++ return -EINVAL;
++ }
++
+ /*
+ * Copy the command submission and bo array to kernel space in
+ * one go, and do this outside of any locks.
+diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+index 1c75c8ed5bcea..85eddd492774d 100644
+--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
++++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+@@ -130,6 +130,7 @@ struct etnaviv_gpu {
+
+ /* hang detection */
+ u32 hangcheck_dma_addr;
++ u32 hangcheck_fence;
+
+ void __iomem *mmio;
+ int irq;
+diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+index feb6da1b6cebc..bbf391f48f949 100644
+--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
++++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+@@ -107,8 +107,10 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
+ */
+ dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
+ change = dma_addr - gpu->hangcheck_dma_addr;
+- if (change < 0 || change > 16) {
++ if (gpu->completed_fence != gpu->hangcheck_fence ||
++ change < 0 || change > 16) {
+ gpu->hangcheck_dma_addr = dma_addr;
++ gpu->hangcheck_fence = gpu->completed_fence;
+ goto out_no_timeout;
+ }
+
+diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+index cd818a6291835..00e53de4812bb 100644
+--- a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
++++ b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+@@ -225,12 +225,29 @@ static int hyperv_vmbus_remove(struct hv_device *hdev)
+ {
+ struct drm_device *dev = hv_get_drvdata(hdev);
+ struct hyperv_drm_device *hv = to_hv(dev);
++ struct pci_dev *pdev;
+
+ drm_dev_unplug(dev);
+ drm_atomic_helper_shutdown(dev);
+ vmbus_close(hdev->channel);
+ hv_set_drvdata(hdev, NULL);
+- vmbus_free_mmio(hv->mem->start, hv->fb_size);
++
++ /*
++ * Free allocated MMIO memory only on Gen2 VMs.
++ * On Gen1 VMs, release the PCI device
++ */
++ if (efi_enabled(EFI_BOOT)) {
++ vmbus_free_mmio(hv->mem->start, hv->fb_size);
++ } else {
++ pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT,
++ PCI_DEVICE_ID_HYPERV_VIDEO, NULL);
++ if (!pdev) {
++ drm_err(dev, "Unable to find PCI Hyper-V video\n");
++ return -ENODEV;
++ }
++ pci_release_region(pdev, 0);
++ pci_dev_put(pdev);
++ }
+
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
+index f960f5d7664e6..fe6b34774483f 100644
+--- a/drivers/gpu/drm/i915/Kconfig
++++ b/drivers/gpu/drm/i915/Kconfig
+@@ -101,6 +101,7 @@ config DRM_I915_USERPTR
+ config DRM_I915_GVT
+ bool "Enable Intel GVT-g graphics virtualization host support"
+ depends on DRM_I915
++ depends on X86
+ depends on 64BIT
+ default n
+ help
+diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
+index 335ba9f43d8f7..26cf754229451 100644
+--- a/drivers/gpu/drm/i915/Makefile
++++ b/drivers/gpu/drm/i915/Makefile
+@@ -211,6 +211,8 @@ i915-y += \
+ display/intel_dpio_phy.o \
+ display/intel_dpll.o \
+ display/intel_dpll_mgr.o \
++ display/intel_dpt.o \
++ display/intel_drrs.o \
+ display/intel_dsb.o \
+ display/intel_fb.o \
+ display/intel_fbc.o \
+diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
+index be352e9f0afc1..63baaf6988ade 100644
+--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
++++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
+@@ -584,6 +584,7 @@ void g4x_hdmi_init(struct drm_i915_private *dev_priv,
+ else
+ intel_encoder->enable = g4x_enable_hdmi;
+ }
++ intel_encoder->shutdown = intel_hdmi_encoder_shutdown;
+
+ intel_encoder->type = INTEL_OUTPUT_HDMI;
+ intel_encoder->power_domain = intel_port_to_power_domain(port);
+diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
+index a3eae3f3eadce..638a00b2dc2d2 100644
+--- a/drivers/gpu/drm/i915/display/icl_dsi.c
++++ b/drivers/gpu/drm/i915/display/icl_dsi.c
+@@ -711,10 +711,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder,
+ intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val);
+
+ for_each_dsi_phy(phy, intel_dsi->phys) {
+- if (DISPLAY_VER(dev_priv) >= 12)
+- val |= ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy);
+- else
+- val &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy);
++ val &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy);
+ }
+ intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val);
+
+@@ -1150,8 +1147,6 @@ static void
+ gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder,
+ const struct intel_crtc_state *crtc_state)
+ {
+- struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+-
+ /* step 4a: power up all lanes of the DDI used by DSI */
+ gen11_dsi_power_up_lanes(encoder);
+
+@@ -1177,8 +1172,7 @@ gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder,
+ gen11_dsi_configure_transcoder(encoder, crtc_state);
+
+ /* Step 4l: Gate DDI clocks */
+- if (DISPLAY_VER(dev_priv) == 11)
+- gen11_dsi_gate_clocks(encoder);
++ gen11_dsi_gate_clocks(encoder);
+ }
+
+ static void gen11_dsi_powerup_panel(struct intel_encoder *encoder)
+diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
+index fd71346aac7bc..32d5a556b7eac 100644
+--- a/drivers/gpu/drm/i915/display/intel_bios.c
++++ b/drivers/gpu/drm/i915/display/intel_bios.c
+@@ -1692,6 +1692,39 @@ static u8 map_ddc_pin(struct drm_i915_private *i915, u8 vbt_pin)
+ return 0;
+ }
+
++static u8 dvo_port_type(u8 dvo_port)
++{
++ switch (dvo_port) {
++ case DVO_PORT_HDMIA:
++ case DVO_PORT_HDMIB:
++ case DVO_PORT_HDMIC:
++ case DVO_PORT_HDMID:
++ case DVO_PORT_HDMIE:
++ case DVO_PORT_HDMIF:
++ case DVO_PORT_HDMIG:
++ case DVO_PORT_HDMIH:
++ case DVO_PORT_HDMII:
++ return DVO_PORT_HDMIA;
++ case DVO_PORT_DPA:
++ case DVO_PORT_DPB:
++ case DVO_PORT_DPC:
++ case DVO_PORT_DPD:
++ case DVO_PORT_DPE:
++ case DVO_PORT_DPF:
++ case DVO_PORT_DPG:
++ case DVO_PORT_DPH:
++ case DVO_PORT_DPI:
++ return DVO_PORT_DPA;
++ case DVO_PORT_MIPIA:
++ case DVO_PORT_MIPIB:
++ case DVO_PORT_MIPIC:
++ case DVO_PORT_MIPID:
++ return DVO_PORT_MIPIA;
++ default:
++ return dvo_port;
++ }
++}
++
+ static enum port __dvo_port_to_port(int n_ports, int n_dvo,
+ const int port_mapping[][3], u8 dvo_port)
+ {
+@@ -2622,35 +2655,17 @@ bool intel_bios_is_port_edp(struct drm_i915_private *i915, enum port port)
+ return false;
+ }
+
+-static bool child_dev_is_dp_dual_mode(const struct child_device_config *child,
+- enum port port)
++static bool child_dev_is_dp_dual_mode(const struct child_device_config *child)
+ {
+- static const struct {
+- u16 dp, hdmi;
+- } port_mapping[] = {
+- /*
+- * Buggy VBTs may declare DP ports as having
+- * HDMI type dvo_port :( So let's check both.
+- */
+- [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, },
+- [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, },
+- [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, },
+- [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, },
+- [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, },
+- };
+-
+- if (port == PORT_A || port >= ARRAY_SIZE(port_mapping))
+- return false;
+-
+ if ((child->device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) !=
+ (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS))
+ return false;
+
+- if (child->dvo_port == port_mapping[port].dp)
++ if (dvo_port_type(child->dvo_port) == DVO_PORT_DPA)
+ return true;
+
+ /* Only accept a HDMI dvo_port as DP++ if it has an AUX channel */
+- if (child->dvo_port == port_mapping[port].hdmi &&
++ if (dvo_port_type(child->dvo_port) == DVO_PORT_HDMIA &&
+ child->aux_channel != 0)
+ return true;
+
+@@ -2660,10 +2675,36 @@ static bool child_dev_is_dp_dual_mode(const struct child_device_config *child,
+ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *i915,
+ enum port port)
+ {
++ static const struct {
++ u16 dp, hdmi;
++ } port_mapping[] = {
++ /*
++ * Buggy VBTs may declare DP ports as having
++ * HDMI type dvo_port :( So let's check both.
++ */
++ [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, },
++ [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, },
++ [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, },
++ [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, },
++ [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, },
++ };
+ const struct intel_bios_encoder_data *devdata;
+
++ if (HAS_DDI(i915)) {
++ const struct intel_bios_encoder_data *devdata;
++
++ devdata = intel_bios_encoder_data_lookup(i915, port);
++
++ return devdata && child_dev_is_dp_dual_mode(&devdata->child);
++ }
++
++ if (port == PORT_A || port >= ARRAY_SIZE(port_mapping))
++ return false;
++
+ list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+- if (child_dev_is_dp_dual_mode(&devdata->child, port))
++ if ((devdata->child.dvo_port == port_mapping[port].dp ||
++ devdata->child.dvo_port == port_mapping[port].hdmi) &&
++ child_dev_is_dp_dual_mode(&devdata->child))
+ return true;
+ }
+
+diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
+index 4b94256d73197..ea48620f76d9c 100644
+--- a/drivers/gpu/drm/i915/display/intel_bw.c
++++ b/drivers/gpu/drm/i915/display/intel_bw.c
+@@ -681,6 +681,7 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
+ unsigned int max_bw_point = 0, max_bw = 0;
+ unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points;
+ unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points;
++ bool changed = false;
+ u32 mask = 0;
+
+ /* FIXME earlier gens need some checks too */
+@@ -724,6 +725,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
+ new_bw_state->data_rate[crtc->pipe] = new_data_rate;
+ new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
+
++ changed = true;
++
+ drm_dbg_kms(&dev_priv->drm,
+ "pipe %c data rate %u num active planes %u\n",
+ pipe_name(crtc->pipe),
+@@ -731,7 +734,19 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
+ new_bw_state->num_active_planes[crtc->pipe]);
+ }
+
+- if (!new_bw_state)
++ old_bw_state = intel_atomic_get_old_bw_state(state);
++ new_bw_state = intel_atomic_get_new_bw_state(state);
++
++ if (new_bw_state &&
++ intel_can_enable_sagv(dev_priv, old_bw_state) !=
++ intel_can_enable_sagv(dev_priv, new_bw_state))
++ changed = true;
++
++ /*
++ * If none of our inputs (data rates, number of active
++ * planes, SAGV yes/no) changed then nothing to do here.
++ */
++ if (!changed)
+ return 0;
+
+ ret = intel_atomic_lock_global_state(&new_bw_state->base);
+@@ -804,7 +819,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
+ * cause.
+ */
+ if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
+- allowed_points = BIT(max_bw_point);
++ allowed_points &= ADLS_PSF_PT_MASK;
++ allowed_points |= BIT(max_bw_point);
+ drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
+ max_bw_point);
+ }
+@@ -814,7 +830,6 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
+ */
+ new_bw_state->qgv_points_mask = ~allowed_points & mask;
+
+- old_bw_state = intel_atomic_get_old_bw_state(state);
+ /*
+ * If the actual mask had changed we need to make sure that
+ * the commits are serialized(in case this is a nomodeset, nonblocking)
+diff --git a/drivers/gpu/drm/i915/display/intel_bw.h b/drivers/gpu/drm/i915/display/intel_bw.h
+index 46c6eecbd9175..0ceaed1c96562 100644
+--- a/drivers/gpu/drm/i915/display/intel_bw.h
++++ b/drivers/gpu/drm/i915/display/intel_bw.h
+@@ -30,19 +30,19 @@ struct intel_bw_state {
+ */
+ u8 pipe_sagv_reject;
+
++ /* bitmask of active pipes */
++ u8 active_pipes;
++
+ /*
+ * Current QGV points mask, which restricts
+ * some particular SAGV states, not to confuse
+ * with pipe_sagv_mask.
+ */
+- u8 qgv_points_mask;
++ u16 qgv_points_mask;
+
+ unsigned int data_rate[I915_MAX_PIPES];
+ u8 num_active_planes[I915_MAX_PIPES];
+
+- /* bitmask of active pipes */
+- u8 active_pipes;
+-
+ int min_cdclk;
+ };
+
+diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
+index bd184325d0c75..f61901e26409e 100644
+--- a/drivers/gpu/drm/i915/display/intel_ddi.c
++++ b/drivers/gpu/drm/i915/display/intel_ddi.c
+@@ -40,6 +40,7 @@
+ #include "intel_dp_link_training.h"
+ #include "intel_dp_mst.h"
+ #include "intel_dpio_phy.h"
++#include "intel_drrs.h"
+ #include "intel_dsi.h"
+ #include "intel_fdi.h"
+ #include "intel_fifo_underrun.h"
+@@ -4432,6 +4433,7 @@ static void intel_ddi_encoder_shutdown(struct intel_encoder *encoder)
+ enum phy phy = intel_port_to_phy(i915, encoder->port);
+
+ intel_dp_encoder_shutdown(encoder);
++ intel_hdmi_encoder_shutdown(encoder);
+
+ if (!intel_phy_is_tc(i915, phy))
+ return;
+diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
+index ba2c08f1a797c..876620455ed31 100644
+--- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
++++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
+@@ -476,14 +476,14 @@ static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi = {
+ static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_dp[] = {
+ /* NT mV Trans mV db */
+ { .icl = { 0xA, 0x33, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */
+- { .icl = { 0xA, 0x47, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */
+- { .icl = { 0xC, 0x64, 0x34, 0x00, 0x0B } }, /* 350 700 6.0 */
+- { .icl = { 0x6, 0x7F, 0x30, 0x00, 0x0F } }, /* 350 900 8.2 */
++ { .icl = { 0xA, 0x47, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */
++ { .icl = { 0xC, 0x64, 0x33, 0x00, 0x0C } }, /* 350 700 6.0 */
++ { .icl = { 0x6, 0x7F, 0x2F, 0x00, 0x10 } }, /* 350 900 8.2 */
+ { .icl = { 0xA, 0x46, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */
+- { .icl = { 0xC, 0x64, 0x38, 0x00, 0x07 } }, /* 500 700 2.9 */
++ { .icl = { 0xC, 0x64, 0x37, 0x00, 0x08 } }, /* 500 700 2.9 */
+ { .icl = { 0x6, 0x7F, 0x32, 0x00, 0x0D } }, /* 500 900 5.1 */
+ { .icl = { 0xC, 0x61, 0x3F, 0x00, 0x00 } }, /* 650 700 0.6 */
+- { .icl = { 0x6, 0x7F, 0x38, 0x00, 0x07 } }, /* 600 900 3.5 */
++ { .icl = { 0x6, 0x7F, 0x37, 0x00, 0x08 } }, /* 600 900 3.5 */
+ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */
+ };
+
+diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
+index 17f44ffea5866..c9b051ab18e02 100644
+--- a/drivers/gpu/drm/i915/display/intel_display.c
++++ b/drivers/gpu/drm/i915/display/intel_display.c
+@@ -84,6 +84,7 @@
+ #include "intel_display_types.h"
+ #include "intel_dmc.h"
+ #include "intel_dp_link_training.h"
++#include "intel_dpt.h"
+ #include "intel_fbc.h"
+ #include "intel_fdi.h"
+ #include "intel_fbdev.h"
+@@ -126,182 +127,6 @@ static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
+ static void intel_modeset_setup_hw_state(struct drm_device *dev,
+ struct drm_modeset_acquire_ctx *ctx);
+
+-struct i915_dpt {
+- struct i915_address_space vm;
+-
+- struct drm_i915_gem_object *obj;
+- struct i915_vma *vma;
+- void __iomem *iomem;
+-};
+-
+-#define i915_is_dpt(vm) ((vm)->is_dpt)
+-
+-static inline struct i915_dpt *
+-i915_vm_to_dpt(struct i915_address_space *vm)
+-{
+- BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
+- GEM_BUG_ON(!i915_is_dpt(vm));
+- return container_of(vm, struct i915_dpt, vm);
+-}
+-
+-#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
+-
+-static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
+-{
+- writeq(pte, addr);
+-}
+-
+-static void dpt_insert_page(struct i915_address_space *vm,
+- dma_addr_t addr,
+- u64 offset,
+- enum i915_cache_level level,
+- u32 flags)
+-{
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+- gen8_pte_t __iomem *base = dpt->iomem;
+-
+- gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
+- vm->pte_encode(addr, level, flags));
+-}
+-
+-static void dpt_insert_entries(struct i915_address_space *vm,
+- struct i915_vma *vma,
+- enum i915_cache_level level,
+- u32 flags)
+-{
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+- gen8_pte_t __iomem *base = dpt->iomem;
+- const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags);
+- struct sgt_iter sgt_iter;
+- dma_addr_t addr;
+- int i;
+-
+- /*
+- * Note that we ignore PTE_READ_ONLY here. The caller must be careful
+- * not to allow the user to override access to a read only page.
+- */
+-
+- i = vma->node.start / I915_GTT_PAGE_SIZE;
+- for_each_sgt_daddr(addr, sgt_iter, vma->pages)
+- gen8_set_pte(&base[i++], pte_encode | addr);
+-}
+-
+-static void dpt_clear_range(struct i915_address_space *vm,
+- u64 start, u64 length)
+-{
+-}
+-
+-static void dpt_bind_vma(struct i915_address_space *vm,
+- struct i915_vm_pt_stash *stash,
+- struct i915_vma *vma,
+- enum i915_cache_level cache_level,
+- u32 flags)
+-{
+- struct drm_i915_gem_object *obj = vma->obj;
+- u32 pte_flags;
+-
+- /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
+- pte_flags = 0;
+- if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj))
+- pte_flags |= PTE_READ_ONLY;
+- if (i915_gem_object_is_lmem(obj))
+- pte_flags |= PTE_LM;
+-
+- vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
+-
+- vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
+-
+- /*
+- * Without aliasing PPGTT there's no difference between
+- * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
+- * upgrade to both bound if we bind either to avoid double-binding.
+- */
+- atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
+-}
+-
+-static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma)
+-{
+- vm->clear_range(vm, vma->node.start, vma->size);
+-}
+-
+-static void dpt_cleanup(struct i915_address_space *vm)
+-{
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+-
+- i915_gem_object_put(dpt->obj);
+-}
+-
+-static struct i915_address_space *
+-intel_dpt_create(struct intel_framebuffer *fb)
+-{
+- struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
+- struct drm_i915_private *i915 = to_i915(obj->dev);
+- struct drm_i915_gem_object *dpt_obj;
+- struct i915_address_space *vm;
+- struct i915_dpt *dpt;
+- size_t size;
+- int ret;
+-
+- if (intel_fb_needs_pot_stride_remap(fb))
+- size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
+- else
+- size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
+-
+- size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
+-
+- if (HAS_LMEM(i915))
+- dpt_obj = i915_gem_object_create_lmem(i915, size, 0);
+- else
+- dpt_obj = i915_gem_object_create_stolen(i915, size);
+- if (IS_ERR(dpt_obj))
+- return ERR_CAST(dpt_obj);
+-
+- ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
+- if (ret) {
+- i915_gem_object_put(dpt_obj);
+- return ERR_PTR(ret);
+- }
+-
+- dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
+- if (!dpt) {
+- i915_gem_object_put(dpt_obj);
+- return ERR_PTR(-ENOMEM);
+- }
+-
+- vm = &dpt->vm;
+-
+- vm->gt = &i915->gt;
+- vm->i915 = i915;
+- vm->dma = i915->drm.dev;
+- vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
+- vm->is_dpt = true;
+-
+- i915_address_space_init(vm, VM_CLASS_DPT);
+-
+- vm->insert_page = dpt_insert_page;
+- vm->clear_range = dpt_clear_range;
+- vm->insert_entries = dpt_insert_entries;
+- vm->cleanup = dpt_cleanup;
+-
+- vm->vma_ops.bind_vma = dpt_bind_vma;
+- vm->vma_ops.unbind_vma = dpt_unbind_vma;
+- vm->vma_ops.set_pages = ggtt_set_pages;
+- vm->vma_ops.clear_pages = clear_pages;
+-
+- vm->pte_encode = gen8_ggtt_pte_encode;
+-
+- dpt->obj = dpt_obj;
+-
+- return &dpt->vm;
+-}
+-
+-static void intel_dpt_destroy(struct i915_address_space *vm)
+-{
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+-
+- i915_vm_close(&dpt->vm);
+-}
+-
+ /* returns HPLL frequency in kHz */
+ int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
+ {
+@@ -1833,8 +1658,8 @@ static void fixup_plane_bitmasks(struct intel_crtc_state *crtc_state)
+ }
+ }
+
+-static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
+- struct intel_plane *plane)
++void intel_plane_disable_noatomic(struct intel_crtc *crtc,
++ struct intel_plane *plane)
+ {
+ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+ struct intel_crtc_state *crtc_state =
+@@ -1879,49 +1704,6 @@ static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
+ intel_wait_for_vblank(dev_priv, crtc->pipe);
+ }
+
+-static struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
+-{
+- struct drm_i915_private *i915 = vm->i915;
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+- intel_wakeref_t wakeref;
+- struct i915_vma *vma;
+- void __iomem *iomem;
+-
+- wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+- atomic_inc(&i915->gpu_error.pending_fb_pin);
+-
+- vma = i915_gem_object_ggtt_pin(dpt->obj, NULL, 0, 4096,
+- HAS_LMEM(i915) ? 0 : PIN_MAPPABLE);
+- if (IS_ERR(vma))
+- goto err;
+-
+- iomem = i915_vma_pin_iomap(vma);
+- i915_vma_unpin(vma);
+- if (IS_ERR(iomem)) {
+- vma = iomem;
+- goto err;
+- }
+-
+- dpt->vma = vma;
+- dpt->iomem = iomem;
+-
+- i915_vma_get(vma);
+-
+-err:
+- atomic_dec(&i915->gpu_error.pending_fb_pin);
+- intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+-
+- return vma;
+-}
+-
+-static void intel_dpt_unpin(struct i915_address_space *vm)
+-{
+- struct i915_dpt *dpt = i915_vm_to_dpt(vm);
+-
+- i915_vma_unpin_iomap(dpt->vma);
+- i915_vma_put(dpt->vma);
+-}
+-
+ static bool
+ intel_reuse_initial_plane_obj(struct drm_i915_private *i915,
+ const struct intel_initial_plane_config *plane_config,
+@@ -13435,6 +13217,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
+ vlv_wm_sanitize(dev_priv);
+ } else if (DISPLAY_VER(dev_priv) >= 9) {
+ skl_wm_get_hw_state(dev_priv);
++ skl_wm_sanitize(dev_priv);
+ } else if (HAS_PCH_SPLIT(dev_priv)) {
+ ilk_wm_get_hw_state(dev_priv);
+ }
+diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
+index 284936f0ddab2..6a7a91b380805 100644
+--- a/drivers/gpu/drm/i915/display/intel_display.h
++++ b/drivers/gpu/drm/i915/display/intel_display.h
+@@ -629,6 +629,8 @@ void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state);
+ struct intel_encoder *
+ intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
+ const struct intel_crtc_state *crtc_state);
++void intel_plane_disable_noatomic(struct intel_crtc *crtc,
++ struct intel_plane *plane);
+
+ unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
+ int color_plane);
+diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+index 8fdacb252bb19..b136a0fc0963b 100644
+--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
++++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+@@ -13,6 +13,7 @@
+ #include "intel_display_types.h"
+ #include "intel_dmc.h"
+ #include "intel_dp.h"
++#include "intel_drrs.h"
+ #include "intel_fbc.h"
+ #include "intel_hdcp.h"
+ #include "intel_hdmi.h"
+diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
+index 6beeeeba1bed2..b56850d964919 100644
+--- a/drivers/gpu/drm/i915/display/intel_display_types.h
++++ b/drivers/gpu/drm/i915/display/intel_display_types.h
+@@ -1639,6 +1639,9 @@ struct intel_dp {
+ struct intel_dp_pcon_frl frl;
+
+ struct intel_psr psr;
++
++ /* When we last wrote the OUI for eDP */
++ unsigned long last_oui_write;
+ };
+
+ enum lspcon_vendor {
+diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
+index b3c8e1c450efb..0e04d4dd1c132 100644
+--- a/drivers/gpu/drm/i915/display/intel_dmc.c
++++ b/drivers/gpu/drm/i915/display/intel_dmc.c
+@@ -375,6 +375,44 @@ static void dmc_set_fw_offset(struct intel_dmc *dmc,
+ }
+ }
+
++static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
++ const u32 *mmioaddr, u32 mmio_count,
++ int header_ver, u8 dmc_id)
++{
++ struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
++ u32 start_range, end_range;
++ int i;
++
++ if (dmc_id >= DMC_FW_MAX) {
++ drm_warn(&i915->drm, "Unsupported firmware id %u\n", dmc_id);
++ return false;
++ }
++
++ if (header_ver == 1) {
++ start_range = DMC_MMIO_START_RANGE;
++ end_range = DMC_MMIO_END_RANGE;
++ } else if (dmc_id == DMC_FW_MAIN) {
++ start_range = TGL_MAIN_MMIO_START;
++ end_range = TGL_MAIN_MMIO_END;
++ } else if (DISPLAY_VER(i915) >= 13) {
++ start_range = ADLP_PIPE_MMIO_START;
++ end_range = ADLP_PIPE_MMIO_END;
++ } else if (DISPLAY_VER(i915) >= 12) {
++ start_range = TGL_PIPE_MMIO_START(dmc_id);
++ end_range = TGL_PIPE_MMIO_END(dmc_id);
++ } else {
++ drm_warn(&i915->drm, "Unknown mmio range for sanity check");
++ return false;
++ }
++
++ for (i = 0; i < mmio_count; i++) {
++ if (mmioaddr[i] < start_range || mmioaddr[i] > end_range)
++ return false;
++ }
++
++ return true;
++}
++
+ static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
+ const struct intel_dmc_header_base *dmc_header,
+ size_t rem_size, u8 dmc_id)
+@@ -444,6 +482,12 @@ static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
+ return 0;
+ }
+
++ if (!dmc_mmio_addr_sanity_check(dmc, mmioaddr, mmio_count,
++ dmc_header->header_ver, dmc_id)) {
++ drm_err(&i915->drm, "DMC firmware has Wrong MMIO Addresses\n");
++ return 0;
++ }
++
+ for (i = 0; i < mmio_count; i++) {
+ dmc_info->mmioaddr[i] = _MMIO(mmioaddr[i]);
+ dmc_info->mmiodata[i] = mmiodata[i];
+@@ -606,7 +650,7 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv,
+ continue;
+
+ offset = readcount + dmc->dmc_info[id].dmc_offset * 4;
+- if (fw->size - offset < 0) {
++ if (offset > fw->size) {
+ drm_err(&dev_priv->drm, "Reading beyond the fw_size\n");
+ continue;
+ }
+diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
+index 5cf152be44877..631cf7d4323c8 100644
+--- a/drivers/gpu/drm/i915/display/intel_dp.c
++++ b/drivers/gpu/drm/i915/display/intel_dp.c
+@@ -29,6 +29,7 @@
+ #include <linux/i2c.h>
+ #include <linux/notifier.h>
+ #include <linux/slab.h>
++#include <linux/timekeeping.h>
+ #include <linux/types.h>
+
+ #include <asm/byteorder.h>
+@@ -55,6 +56,7 @@
+ #include "intel_dp_mst.h"
+ #include "intel_dpio_phy.h"
+ #include "intel_dpll.h"
++#include "intel_drrs.h"
+ #include "intel_fifo_underrun.h"
+ #include "intel_hdcp.h"
+ #include "intel_hdmi.h"
+@@ -111,6 +113,12 @@ bool intel_dp_is_edp(struct intel_dp *intel_dp)
+ static void intel_dp_unset_edid(struct intel_dp *intel_dp);
+ static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc);
+
++static void intel_dp_set_default_sink_rates(struct intel_dp *intel_dp)
++{
++ intel_dp->sink_rates[0] = 162000;
++ intel_dp->num_sink_rates = 1;
++}
++
+ /* update sink rates from dpcd */
+ static void intel_dp_set_sink_rates(struct intel_dp *intel_dp)
+ {
+@@ -1603,46 +1611,6 @@ intel_dp_compute_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp,
+ intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA);
+ }
+
+-static void
+-intel_dp_drrs_compute_config(struct intel_dp *intel_dp,
+- struct intel_crtc_state *pipe_config,
+- int output_bpp, bool constant_n)
+-{
+- struct intel_connector *intel_connector = intel_dp->attached_connector;
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+- int pixel_clock;
+-
+- if (pipe_config->vrr.enable)
+- return;
+-
+- /*
+- * DRRS and PSR can't be enable together, so giving preference to PSR
+- * as it allows more power-savings by complete shutting down display,
+- * so to guarantee this, intel_dp_drrs_compute_config() must be called
+- * after intel_psr_compute_config().
+- */
+- if (pipe_config->has_psr)
+- return;
+-
+- if (!intel_connector->panel.downclock_mode ||
+- dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT)
+- return;
+-
+- pipe_config->has_drrs = true;
+-
+- pixel_clock = intel_connector->panel.downclock_mode->clock;
+- if (pipe_config->splitter.enable)
+- pixel_clock /= pipe_config->splitter.link_count;
+-
+- intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock,
+- pipe_config->port_clock, &pipe_config->dp_m2_n2,
+- constant_n, pipe_config->fec_enable);
+-
+- /* FIXME: abstract this better */
+- if (pipe_config->splitter.enable)
+- pipe_config->dp_m2_n2.gmch_m *= pipe_config->splitter.link_count;
+-}
+-
+ int
+ intel_dp_compute_config(struct intel_encoder *encoder,
+ struct intel_crtc_state *pipe_config,
+@@ -1767,6 +1735,12 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
+ intel_dp->lane_count = lane_count;
+ }
+
++static void intel_dp_reset_max_link_params(struct intel_dp *intel_dp)
++{
++ intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
++ intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp);
++}
++
+ /* Enable backlight PWM and backlight PP control. */
+ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
+ const struct drm_connector_state *conn_state)
+@@ -1852,6 +1826,16 @@ intel_edp_init_source_oui(struct intel_dp *intel_dp, bool careful)
+
+ if (drm_dp_dpcd_write(&intel_dp->aux, DP_SOURCE_OUI, oui, sizeof(oui)) < 0)
+ drm_err(&i915->drm, "Failed to write source OUI\n");
++
++ intel_dp->last_oui_write = jiffies;
++}
++
++void intel_dp_wait_source_oui(struct intel_dp *intel_dp)
++{
++ struct drm_i915_private *i915 = dp_to_i915(intel_dp);
++
++ drm_dbg_kms(&i915->drm, "Performing OUI wait\n");
++ wait_remaining_ms_from_jiffies(intel_dp->last_oui_write, 30);
+ }
+
+ /* If the device supports it, try to set the power state appropriately */
+@@ -1926,8 +1910,7 @@ void intel_dp_sync_state(struct intel_encoder *encoder,
+ if (intel_dp->dpcd[DP_DPCD_REV] == 0)
+ intel_dp_get_dpcd(intel_dp);
+
+- intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
+- intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp);
++ intel_dp_reset_max_link_params(intel_dp);
+ }
+
+ bool intel_dp_initial_fastset_check(struct intel_encoder *encoder,
+@@ -2462,6 +2445,9 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
+ */
+ intel_psr_init_dpcd(intel_dp);
+
++ /* Clear the default sink rates */
++ intel_dp->num_sink_rates = 0;
++
+ /* Read the eDP 1.4+ supported link rates. */
+ if (intel_dp->edp_dpcd[0] >= DP_EDP_14) {
+ __le16 sink_rates[DP_MAX_SUPPORTED_RATES];
+@@ -2497,6 +2483,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
+ intel_dp_set_sink_rates(intel_dp);
+
+ intel_dp_set_common_rates(intel_dp);
++ intel_dp_reset_max_link_params(intel_dp);
+
+ /* Read the eDP DSC DPCD registers */
+ if (DISPLAY_VER(dev_priv) >= 10)
+@@ -4240,12 +4227,7 @@ intel_dp_detect(struct drm_connector *connector,
+ * supports link training fallback params.
+ */
+ if (intel_dp->reset_link_params || intel_dp->is_mst) {
+- /* Initial max link lane count */
+- intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
+-
+- /* Initial max link rate */
+- intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp);
+-
++ intel_dp_reset_max_link_params(intel_dp);
+ intel_dp->reset_link_params = false;
+ }
+
+@@ -4617,7 +4599,7 @@ intel_dp_hpd_pulse(struct intel_digital_port *dig_port, bool long_hpd)
+ struct intel_dp *intel_dp = &dig_port->dp;
+
+ if (dig_port->base.type == INTEL_OUTPUT_EDP &&
+- (long_hpd || !intel_pps_have_power(intel_dp))) {
++ (long_hpd || !intel_pps_have_panel_power_or_vdd(intel_dp))) {
+ /*
+ * vdd off can generate a long/short pulse on eDP which
+ * would require vdd on to handle it, and thus we
+@@ -4716,432 +4698,6 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect
+ drm_connector_attach_vrr_capable_property(connector);
+ }
+
+-/**
+- * intel_dp_set_drrs_state - program registers for RR switch to take effect
+- * @dev_priv: i915 device
+- * @crtc_state: a pointer to the active intel_crtc_state
+- * @refresh_rate: RR to be programmed
+- *
+- * This function gets called when refresh rate (RR) has to be changed from
+- * one frequency to another. Switches can be between high and low RR
+- * supported by the panel or to any other RR based on media playback (in
+- * this case, RR value needs to be passed from user space).
+- *
+- * The caller of this function needs to take a lock on dev_priv->drrs.
+- */
+-static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
+- const struct intel_crtc_state *crtc_state,
+- int refresh_rate)
+-{
+- struct intel_dp *intel_dp = dev_priv->drrs.dp;
+- struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+- enum drrs_refresh_rate_type index = DRRS_HIGH_RR;
+-
+- if (refresh_rate <= 0) {
+- drm_dbg_kms(&dev_priv->drm,
+- "Refresh rate should be positive non-zero.\n");
+- return;
+- }
+-
+- if (intel_dp == NULL) {
+- drm_dbg_kms(&dev_priv->drm, "DRRS not supported.\n");
+- return;
+- }
+-
+- if (!crtc) {
+- drm_dbg_kms(&dev_priv->drm,
+- "DRRS: intel_crtc not initialized\n");
+- return;
+- }
+-
+- if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) {
+- drm_dbg_kms(&dev_priv->drm, "Only Seamless DRRS supported.\n");
+- return;
+- }
+-
+- if (drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode) ==
+- refresh_rate)
+- index = DRRS_LOW_RR;
+-
+- if (index == dev_priv->drrs.refresh_rate_type) {
+- drm_dbg_kms(&dev_priv->drm,
+- "DRRS requested for previously set RR...ignoring\n");
+- return;
+- }
+-
+- if (!crtc_state->hw.active) {
+- drm_dbg_kms(&dev_priv->drm,
+- "eDP encoder disabled. CRTC not Active\n");
+- return;
+- }
+-
+- if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) {
+- switch (index) {
+- case DRRS_HIGH_RR:
+- intel_dp_set_m_n(crtc_state, M1_N1);
+- break;
+- case DRRS_LOW_RR:
+- intel_dp_set_m_n(crtc_state, M2_N2);
+- break;
+- case DRRS_MAX_RR:
+- default:
+- drm_err(&dev_priv->drm,
+- "Unsupported refreshrate type\n");
+- }
+- } else if (DISPLAY_VER(dev_priv) > 6) {
+- i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder);
+- u32 val;
+-
+- val = intel_de_read(dev_priv, reg);
+- if (index > DRRS_HIGH_RR) {
+- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+- val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV;
+- else
+- val |= PIPECONF_EDP_RR_MODE_SWITCH;
+- } else {
+- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+- val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV;
+- else
+- val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
+- }
+- intel_de_write(dev_priv, reg, val);
+- }
+-
+- dev_priv->drrs.refresh_rate_type = index;
+-
+- drm_dbg_kms(&dev_priv->drm, "eDP Refresh Rate set to : %dHz\n",
+- refresh_rate);
+-}
+-
+-static void
+-intel_edp_drrs_enable_locked(struct intel_dp *intel_dp)
+-{
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+-
+- dev_priv->drrs.busy_frontbuffer_bits = 0;
+- dev_priv->drrs.dp = intel_dp;
+-}
+-
+-/**
+- * intel_edp_drrs_enable - init drrs struct if supported
+- * @intel_dp: DP struct
+- * @crtc_state: A pointer to the active crtc state.
+- *
+- * Initializes frontbuffer_bits and drrs.dp
+- */
+-void intel_edp_drrs_enable(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state)
+-{
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+-
+- if (!crtc_state->has_drrs)
+- return;
+-
+- drm_dbg_kms(&dev_priv->drm, "Enabling DRRS\n");
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+-
+- if (dev_priv->drrs.dp) {
+- drm_warn(&dev_priv->drm, "DRRS already enabled\n");
+- goto unlock;
+- }
+-
+- intel_edp_drrs_enable_locked(intel_dp);
+-
+-unlock:
+- mutex_unlock(&dev_priv->drrs.mutex);
+-}
+-
+-static void
+-intel_edp_drrs_disable_locked(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state)
+-{
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+-
+- if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) {
+- int refresh;
+-
+- refresh = drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode);
+- intel_dp_set_drrs_state(dev_priv, crtc_state, refresh);
+- }
+-
+- dev_priv->drrs.dp = NULL;
+-}
+-
+-/**
+- * intel_edp_drrs_disable - Disable DRRS
+- * @intel_dp: DP struct
+- * @old_crtc_state: Pointer to old crtc_state.
+- *
+- */
+-void intel_edp_drrs_disable(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *old_crtc_state)
+-{
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+-
+- if (!old_crtc_state->has_drrs)
+- return;
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+- if (!dev_priv->drrs.dp) {
+- mutex_unlock(&dev_priv->drrs.mutex);
+- return;
+- }
+-
+- intel_edp_drrs_disable_locked(intel_dp, old_crtc_state);
+- mutex_unlock(&dev_priv->drrs.mutex);
+-
+- cancel_delayed_work_sync(&dev_priv->drrs.work);
+-}
+-
+-/**
+- * intel_edp_drrs_update - Update DRRS state
+- * @intel_dp: Intel DP
+- * @crtc_state: new CRTC state
+- *
+- * This function will update DRRS states, disabling or enabling DRRS when
+- * executing fastsets. For full modeset, intel_edp_drrs_disable() and
+- * intel_edp_drrs_enable() should be called instead.
+- */
+-void
+-intel_edp_drrs_update(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state)
+-{
+- struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+-
+- if (dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT)
+- return;
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+-
+- /* New state matches current one? */
+- if (crtc_state->has_drrs == !!dev_priv->drrs.dp)
+- goto unlock;
+-
+- if (crtc_state->has_drrs)
+- intel_edp_drrs_enable_locked(intel_dp);
+- else
+- intel_edp_drrs_disable_locked(intel_dp, crtc_state);
+-
+-unlock:
+- mutex_unlock(&dev_priv->drrs.mutex);
+-}
+-
+-static void intel_edp_drrs_downclock_work(struct work_struct *work)
+-{
+- struct drm_i915_private *dev_priv =
+- container_of(work, typeof(*dev_priv), drrs.work.work);
+- struct intel_dp *intel_dp;
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+-
+- intel_dp = dev_priv->drrs.dp;
+-
+- if (!intel_dp)
+- goto unlock;
+-
+- /*
+- * The delayed work can race with an invalidate hence we need to
+- * recheck.
+- */
+-
+- if (dev_priv->drrs.busy_frontbuffer_bits)
+- goto unlock;
+-
+- if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) {
+- struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
+-
+- intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+- drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode));
+- }
+-
+-unlock:
+- mutex_unlock(&dev_priv->drrs.mutex);
+-}
+-
+-/**
+- * intel_edp_drrs_invalidate - Disable Idleness DRRS
+- * @dev_priv: i915 device
+- * @frontbuffer_bits: frontbuffer plane tracking bits
+- *
+- * This function gets called everytime rendering on the given planes start.
+- * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR).
+- *
+- * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
+- */
+-void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
+- unsigned int frontbuffer_bits)
+-{
+- struct intel_dp *intel_dp;
+- struct drm_crtc *crtc;
+- enum pipe pipe;
+-
+- if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED)
+- return;
+-
+- cancel_delayed_work(&dev_priv->drrs.work);
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+-
+- intel_dp = dev_priv->drrs.dp;
+- if (!intel_dp) {
+- mutex_unlock(&dev_priv->drrs.mutex);
+- return;
+- }
+-
+- crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
+- pipe = to_intel_crtc(crtc)->pipe;
+-
+- frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe);
+- dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits;
+-
+- /* invalidate means busy screen hence upclock */
+- if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
+- intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+- drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode));
+-
+- mutex_unlock(&dev_priv->drrs.mutex);
+-}
+-
+-/**
+- * intel_edp_drrs_flush - Restart Idleness DRRS
+- * @dev_priv: i915 device
+- * @frontbuffer_bits: frontbuffer plane tracking bits
+- *
+- * This function gets called every time rendering on the given planes has
+- * completed or flip on a crtc is completed. So DRRS should be upclocked
+- * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again,
+- * if no other planes are dirty.
+- *
+- * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
+- */
+-void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
+- unsigned int frontbuffer_bits)
+-{
+- struct intel_dp *intel_dp;
+- struct drm_crtc *crtc;
+- enum pipe pipe;
+-
+- if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED)
+- return;
+-
+- cancel_delayed_work(&dev_priv->drrs.work);
+-
+- mutex_lock(&dev_priv->drrs.mutex);
+-
+- intel_dp = dev_priv->drrs.dp;
+- if (!intel_dp) {
+- mutex_unlock(&dev_priv->drrs.mutex);
+- return;
+- }
+-
+- crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
+- pipe = to_intel_crtc(crtc)->pipe;
+-
+- frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe);
+- dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits;
+-
+- /* flush means busy screen hence upclock */
+- if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
+- intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+- drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode));
+-
+- /*
+- * flush also means no more activity hence schedule downclock, if all
+- * other fbs are quiescent too
+- */
+- if (!dev_priv->drrs.busy_frontbuffer_bits)
+- schedule_delayed_work(&dev_priv->drrs.work,
+- msecs_to_jiffies(1000));
+- mutex_unlock(&dev_priv->drrs.mutex);
+-}
+-
+-/**
+- * DOC: Display Refresh Rate Switching (DRRS)
+- *
+- * Display Refresh Rate Switching (DRRS) is a power conservation feature
+- * which enables swtching between low and high refresh rates,
+- * dynamically, based on the usage scenario. This feature is applicable
+- * for internal panels.
+- *
+- * Indication that the panel supports DRRS is given by the panel EDID, which
+- * would list multiple refresh rates for one resolution.
+- *
+- * DRRS is of 2 types - static and seamless.
+- * Static DRRS involves changing refresh rate (RR) by doing a full modeset
+- * (may appear as a blink on screen) and is used in dock-undock scenario.
+- * Seamless DRRS involves changing RR without any visual effect to the user
+- * and can be used during normal system usage. This is done by programming
+- * certain registers.
+- *
+- * Support for static/seamless DRRS may be indicated in the VBT based on
+- * inputs from the panel spec.
+- *
+- * DRRS saves power by switching to low RR based on usage scenarios.
+- *
+- * The implementation is based on frontbuffer tracking implementation. When
+- * there is a disturbance on the screen triggered by user activity or a periodic
+- * system activity, DRRS is disabled (RR is changed to high RR). When there is
+- * no movement on screen, after a timeout of 1 second, a switch to low RR is
+- * made.
+- *
+- * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate()
+- * and intel_edp_drrs_flush() are called.
+- *
+- * DRRS can be further extended to support other internal panels and also
+- * the scenario of video playback wherein RR is set based on the rate
+- * requested by userspace.
+- */
+-
+-/**
+- * intel_dp_drrs_init - Init basic DRRS work and mutex.
+- * @connector: eDP connector
+- * @fixed_mode: preferred mode of panel
+- *
+- * This function is called only once at driver load to initialize basic
+- * DRRS stuff.
+- *
+- * Returns:
+- * Downclock mode if panel supports it, else return NULL.
+- * DRRS support is determined by the presence of downclock mode (apart
+- * from VBT setting).
+- */
+-static struct drm_display_mode *
+-intel_dp_drrs_init(struct intel_connector *connector,
+- struct drm_display_mode *fixed_mode)
+-{
+- struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+- struct drm_display_mode *downclock_mode = NULL;
+-
+- INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
+- mutex_init(&dev_priv->drrs.mutex);
+-
+- if (DISPLAY_VER(dev_priv) <= 6) {
+- drm_dbg_kms(&dev_priv->drm,
+- "DRRS supported for Gen7 and above\n");
+- return NULL;
+- }
+-
+- if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) {
+- drm_dbg_kms(&dev_priv->drm, "VBT doesn't support DRRS\n");
+- return NULL;
+- }
+-
+- downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode);
+- if (!downclock_mode) {
+- drm_dbg_kms(&dev_priv->drm,
+- "Downclock mode is not found. DRRS not supported\n");
+- return NULL;
+- }
+-
+- dev_priv->drrs.type = dev_priv->vbt.drrs_type;
+-
+- dev_priv->drrs.refresh_rate_type = DRRS_HIGH_RR;
+- drm_dbg_kms(&dev_priv->drm,
+- "seamless DRRS supported for eDP panel.\n");
+- return downclock_mode;
+-}
+-
+ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
+ struct intel_connector *intel_connector)
+ {
+@@ -5296,6 +4852,9 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
+ return false;
+
+ intel_dp_set_source_rates(intel_dp);
++ intel_dp_set_default_sink_rates(intel_dp);
++ intel_dp_set_common_rates(intel_dp);
++ intel_dp_reset_max_link_params(intel_dp);
+
+ intel_dp->reset_link_params = true;
+ intel_dp->pps.pps_pipe = INVALID_PIPE;
+diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
+index 680631b5b4378..3dd6ebc2f6b14 100644
+--- a/drivers/gpu/drm/i915/display/intel_dp.h
++++ b/drivers/gpu/drm/i915/display/intel_dp.h
+@@ -70,17 +70,6 @@ int intel_dp_max_link_rate(struct intel_dp *intel_dp);
+ int intel_dp_max_lane_count(struct intel_dp *intel_dp);
+ int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
+
+-void intel_edp_drrs_enable(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state);
+-void intel_edp_drrs_disable(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state);
+-void intel_edp_drrs_update(struct intel_dp *intel_dp,
+- const struct intel_crtc_state *crtc_state);
+-void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
+- unsigned int frontbuffer_bits);
+-void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
+- unsigned int frontbuffer_bits);
+-
+ void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
+ u8 *link_bw, u8 *rate_select);
+ bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp);
+@@ -129,4 +118,6 @@ void intel_dp_pcon_dsc_configure(struct intel_dp *intel_dp,
+ const struct intel_crtc_state *crtc_state);
+ void intel_dp_phy_test(struct intel_encoder *encoder);
+
++void intel_dp_wait_source_oui(struct intel_dp *intel_dp);
++
+ #endif /* __INTEL_DP_H__ */
+diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+index 6ac568617ef37..e7b90863aa43d 100644
+--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
++++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+@@ -35,6 +35,7 @@
+ */
+
+ #include "intel_display_types.h"
++#include "intel_dp.h"
+ #include "intel_dp_aux_backlight.h"
+ #include "intel_panel.h"
+
+@@ -95,6 +96,14 @@
+
+ #define INTEL_EDP_BRIGHTNESS_OPTIMIZATION_1 0x359
+
++enum intel_dp_aux_backlight_modparam {
++ INTEL_DP_AUX_BACKLIGHT_AUTO = -1,
++ INTEL_DP_AUX_BACKLIGHT_OFF = 0,
++ INTEL_DP_AUX_BACKLIGHT_ON = 1,
++ INTEL_DP_AUX_BACKLIGHT_FORCE_VESA = 2,
++ INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL = 3,
++};
++
+ /* Intel EDP backlight callbacks */
+ static bool
+ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
+@@ -106,6 +115,8 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
+ int ret;
+ u8 tcon_cap[4];
+
++ intel_dp_wait_source_oui(intel_dp);
++
+ ret = drm_dp_dpcd_read(aux, INTEL_EDP_HDR_TCON_CAP0, tcon_cap, sizeof(tcon_cap));
+ if (ret != sizeof(tcon_cap))
+ return false;
+@@ -122,6 +133,24 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
+ return false;
+ }
+
++ /*
++ * If we don't have HDR static metadata there is no way to
++ * runtime detect used range for nits based control. For now
++ * do not use Intel proprietary eDP backlight control if we
++ * don't have this data in panel EDID. In case we find panel
++ * which supports only nits based control, but doesn't provide
++ * HDR static metadata we need to start maintaining table of
++ * ranges for such panels.
++ */
++ if (i915->params.enable_dpcd_backlight != INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL &&
++ !(connector->base.hdr_sink_metadata.hdmi_type1.metadata_type &
++ BIT(HDMI_STATIC_METADATA_TYPE1))) {
++ drm_info(&i915->drm,
++ "Panel is missing HDR static metadata. Possible support for Intel HDR backlight interface is not used. If your backlight controls don't work try booting with i915.enable_dpcd_backlight=%d. needs this, please file a _new_ bug report on drm/i915, see " FDO_BUG_URL " for details.\n",
++ INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL);
++ return false;
++ }
++
+ panel->backlight.edp.intel.sdr_uses_aux =
+ tcon_cap[2] & INTEL_EDP_SDR_TCON_BRIGHTNESS_AUX_CAP;
+
+@@ -204,6 +233,8 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state,
+ int ret;
+ u8 old_ctrl, ctrl;
+
++ intel_dp_wait_source_oui(intel_dp);
++
+ ret = drm_dp_dpcd_readb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &old_ctrl);
+ if (ret != 1) {
+ drm_err(&i915->drm, "Failed to read current backlight control mode: %d\n", ret);
+@@ -368,14 +399,6 @@ static const struct intel_panel_bl_funcs intel_dp_vesa_bl_funcs = {
+ .get = intel_dp_aux_vesa_get_backlight,
+ };
+
+-enum intel_dp_aux_backlight_modparam {
+- INTEL_DP_AUX_BACKLIGHT_AUTO = -1,
+- INTEL_DP_AUX_BACKLIGHT_OFF = 0,
+- INTEL_DP_AUX_BACKLIGHT_ON = 1,
+- INTEL_DP_AUX_BACKLIGHT_FORCE_VESA = 2,
+- INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL = 3,
+-};
+-
+ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
+ {
+ struct drm_device *dev = connector->base.dev;
+diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c
+new file mode 100644
+index 0000000000000..22acd945a9e47
+--- /dev/null
++++ b/drivers/gpu/drm/i915/display/intel_dpt.c
+@@ -0,0 +1,229 @@
++// SPDX-License-Identifier: MIT
++/*
++ * Copyright © 2021 Intel Corporation
++ */
++
++#include "i915_drv.h"
++#include "intel_display_types.h"
++#include "intel_dpt.h"
++#include "intel_fb.h"
++#include "gt/gen8_ppgtt.h"
++
++struct i915_dpt {
++ struct i915_address_space vm;
++
++ struct drm_i915_gem_object *obj;
++ struct i915_vma *vma;
++ void __iomem *iomem;
++};
++
++#define i915_is_dpt(vm) ((vm)->is_dpt)
++
++static inline struct i915_dpt *
++i915_vm_to_dpt(struct i915_address_space *vm)
++{
++ BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
++ GEM_BUG_ON(!i915_is_dpt(vm));
++ return container_of(vm, struct i915_dpt, vm);
++}
++
++#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
++
++static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
++{
++ writeq(pte, addr);
++}
++
++static void dpt_insert_page(struct i915_address_space *vm,
++ dma_addr_t addr,
++ u64 offset,
++ enum i915_cache_level level,
++ u32 flags)
++{
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++ gen8_pte_t __iomem *base = dpt->iomem;
++
++ gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
++ vm->pte_encode(addr, level, flags));
++}
++
++static void dpt_insert_entries(struct i915_address_space *vm,
++ struct i915_vma *vma,
++ enum i915_cache_level level,
++ u32 flags)
++{
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++ gen8_pte_t __iomem *base = dpt->iomem;
++ const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags);
++ struct sgt_iter sgt_iter;
++ dma_addr_t addr;
++ int i;
++
++ /*
++ * Note that we ignore PTE_READ_ONLY here. The caller must be careful
++ * not to allow the user to override access to a read only page.
++ */
++
++ i = vma->node.start / I915_GTT_PAGE_SIZE;
++ for_each_sgt_daddr(addr, sgt_iter, vma->pages)
++ gen8_set_pte(&base[i++], pte_encode | addr);
++}
++
++static void dpt_clear_range(struct i915_address_space *vm,
++ u64 start, u64 length)
++{
++}
++
++static void dpt_bind_vma(struct i915_address_space *vm,
++ struct i915_vm_pt_stash *stash,
++ struct i915_vma *vma,
++ enum i915_cache_level cache_level,
++ u32 flags)
++{
++ struct drm_i915_gem_object *obj = vma->obj;
++ u32 pte_flags;
++
++ /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
++ pte_flags = 0;
++ if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj))
++ pte_flags |= PTE_READ_ONLY;
++ if (i915_gem_object_is_lmem(obj))
++ pte_flags |= PTE_LM;
++
++ vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
++
++ vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
++
++ /*
++ * Without aliasing PPGTT there's no difference between
++ * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
++ * upgrade to both bound if we bind either to avoid double-binding.
++ */
++ atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
++}
++
++static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma)
++{
++ vm->clear_range(vm, vma->node.start, vma->size);
++}
++
++static void dpt_cleanup(struct i915_address_space *vm)
++{
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++
++ i915_gem_object_put(dpt->obj);
++}
++
++struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
++{
++ struct drm_i915_private *i915 = vm->i915;
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++ intel_wakeref_t wakeref;
++ struct i915_vma *vma;
++ void __iomem *iomem;
++
++ wakeref = intel_runtime_pm_get(&i915->runtime_pm);
++ atomic_inc(&i915->gpu_error.pending_fb_pin);
++
++ vma = i915_gem_object_ggtt_pin(dpt->obj, NULL, 0, 4096,
++ HAS_LMEM(i915) ? 0 : PIN_MAPPABLE);
++ if (IS_ERR(vma))
++ goto err;
++
++ iomem = i915_vma_pin_iomap(vma);
++ i915_vma_unpin(vma);
++ if (IS_ERR(iomem)) {
++ vma = iomem;
++ goto err;
++ }
++
++ dpt->vma = vma;
++ dpt->iomem = iomem;
++
++ i915_vma_get(vma);
++
++err:
++ atomic_dec(&i915->gpu_error.pending_fb_pin);
++ intel_runtime_pm_put(&i915->runtime_pm, wakeref);
++
++ return vma;
++}
++
++void intel_dpt_unpin(struct i915_address_space *vm)
++{
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++
++ i915_vma_unpin_iomap(dpt->vma);
++ i915_vma_put(dpt->vma);
++}
++
++struct i915_address_space *
++intel_dpt_create(struct intel_framebuffer *fb)
++{
++ struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
++ struct drm_i915_private *i915 = to_i915(obj->dev);
++ struct drm_i915_gem_object *dpt_obj;
++ struct i915_address_space *vm;
++ struct i915_dpt *dpt;
++ size_t size;
++ int ret;
++
++ if (intel_fb_needs_pot_stride_remap(fb))
++ size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
++ else
++ size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
++
++ size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
++
++ if (HAS_LMEM(i915))
++ dpt_obj = i915_gem_object_create_lmem(i915, size, 0);
++ else
++ dpt_obj = i915_gem_object_create_stolen(i915, size);
++ if (IS_ERR(dpt_obj))
++ return ERR_CAST(dpt_obj);
++
++ ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
++ if (ret) {
++ i915_gem_object_put(dpt_obj);
++ return ERR_PTR(ret);
++ }
++
++ dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
++ if (!dpt) {
++ i915_gem_object_put(dpt_obj);
++ return ERR_PTR(-ENOMEM);
++ }
++
++ vm = &dpt->vm;
++
++ vm->gt = &i915->gt;
++ vm->i915 = i915;
++ vm->dma = i915->drm.dev;
++ vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
++ vm->is_dpt = true;
++
++ i915_address_space_init(vm, VM_CLASS_DPT);
++
++ vm->insert_page = dpt_insert_page;
++ vm->clear_range = dpt_clear_range;
++ vm->insert_entries = dpt_insert_entries;
++ vm->cleanup = dpt_cleanup;
++
++ vm->vma_ops.bind_vma = dpt_bind_vma;
++ vm->vma_ops.unbind_vma = dpt_unbind_vma;
++ vm->vma_ops.set_pages = ggtt_set_pages;
++ vm->vma_ops.clear_pages = clear_pages;
++
++ vm->pte_encode = gen8_ggtt_pte_encode;
++
++ dpt->obj = dpt_obj;
++
++ return &dpt->vm;
++}
++
++void intel_dpt_destroy(struct i915_address_space *vm)
++{
++ struct i915_dpt *dpt = i915_vm_to_dpt(vm);
++
++ i915_vm_close(&dpt->vm);
++}
+diff --git a/drivers/gpu/drm/i915/display/intel_dpt.h b/drivers/gpu/drm/i915/display/intel_dpt.h
+new file mode 100644
+index 0000000000000..45142b8f849f6
+--- /dev/null
++++ b/drivers/gpu/drm/i915/display/intel_dpt.h
+@@ -0,0 +1,19 @@
++/* SPDX-License-Identifier: MIT */
++/*
++ * Copyright © 2021 Intel Corporation
++ */
++
++#ifndef __INTEL_DPT_H__
++#define __INTEL_DPT_H__
++
++struct i915_address_space;
++struct i915_vma;
++struct intel_framebuffer;
++
++void intel_dpt_destroy(struct i915_address_space *vm);
++struct i915_vma *intel_dpt_pin(struct i915_address_space *vm);
++void intel_dpt_unpin(struct i915_address_space *vm);
++struct i915_address_space *
++intel_dpt_create(struct intel_framebuffer *fb);
++
++#endif /* __INTEL_DPT_H__ */
+diff --git a/drivers/gpu/drm/i915/display/intel_drrs.c b/drivers/gpu/drm/i915/display/intel_drrs.c
+new file mode 100644
+index 0000000000000..3c7d6bf579484
+--- /dev/null
++++ b/drivers/gpu/drm/i915/display/intel_drrs.c
+@@ -0,0 +1,485 @@
++// SPDX-License-Identifier: MIT
++/*
++ * Copyright © 2021 Intel Corporation
++ */
++
++#include "i915_drv.h"
++#include "intel_atomic.h"
++#include "intel_de.h"
++#include "intel_display_types.h"
++#include "intel_drrs.h"
++#include "intel_panel.h"
++
++/**
++ * DOC: Display Refresh Rate Switching (DRRS)
++ *
++ * Display Refresh Rate Switching (DRRS) is a power conservation feature
++ * which enables swtching between low and high refresh rates,
++ * dynamically, based on the usage scenario. This feature is applicable
++ * for internal panels.
++ *
++ * Indication that the panel supports DRRS is given by the panel EDID, which
++ * would list multiple refresh rates for one resolution.
++ *
++ * DRRS is of 2 types - static and seamless.
++ * Static DRRS involves changing refresh rate (RR) by doing a full modeset
++ * (may appear as a blink on screen) and is used in dock-undock scenario.
++ * Seamless DRRS involves changing RR without any visual effect to the user
++ * and can be used during normal system usage. This is done by programming
++ * certain registers.
++ *
++ * Support for static/seamless DRRS may be indicated in the VBT based on
++ * inputs from the panel spec.
++ *
++ * DRRS saves power by switching to low RR based on usage scenarios.
++ *
++ * The implementation is based on frontbuffer tracking implementation. When
++ * there is a disturbance on the screen triggered by user activity or a periodic
++ * system activity, DRRS is disabled (RR is changed to high RR). When there is
++ * no movement on screen, after a timeout of 1 second, a switch to low RR is
++ * made.
++ *
++ * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate()
++ * and intel_edp_drrs_flush() are called.
++ *
++ * DRRS can be further extended to support other internal panels and also
++ * the scenario of video playback wherein RR is set based on the rate
++ * requested by userspace.
++ */
++
++void
++intel_dp_drrs_compute_config(struct intel_dp *intel_dp,
++ struct intel_crtc_state *pipe_config,
++ int output_bpp, bool constant_n)
++{
++ struct intel_connector *intel_connector = intel_dp->attached_connector;
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++ int pixel_clock;
++
++ if (pipe_config->vrr.enable)
++ return;
++
++ /*
++ * DRRS and PSR can't be enable together, so giving preference to PSR
++ * as it allows more power-savings by complete shutting down display,
++ * so to guarantee this, intel_dp_drrs_compute_config() must be called
++ * after intel_psr_compute_config().
++ */
++ if (pipe_config->has_psr)
++ return;
++
++ if (!intel_connector->panel.downclock_mode ||
++ dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT)
++ return;
++
++ pipe_config->has_drrs = true;
++
++ pixel_clock = intel_connector->panel.downclock_mode->clock;
++ if (pipe_config->splitter.enable)
++ pixel_clock /= pipe_config->splitter.link_count;
++
++ intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock,
++ pipe_config->port_clock, &pipe_config->dp_m2_n2,
++ constant_n, pipe_config->fec_enable);
++
++ /* FIXME: abstract this better */
++ if (pipe_config->splitter.enable)
++ pipe_config->dp_m2_n2.gmch_m *= pipe_config->splitter.link_count;
++}
++
++/**
++ * intel_dp_set_drrs_state - program registers for RR switch to take effect
++ * @dev_priv: i915 device
++ * @crtc_state: a pointer to the active intel_crtc_state
++ * @refresh_rate: RR to be programmed
++ *
++ * This function gets called when refresh rate (RR) has to be changed from
++ * one frequency to another. Switches can be between high and low RR
++ * supported by the panel or to any other RR based on media playback (in
++ * this case, RR value needs to be passed from user space).
++ *
++ * The caller of this function needs to take a lock on dev_priv->drrs.
++ */
++static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
++ const struct intel_crtc_state *crtc_state,
++ int refresh_rate)
++{
++ struct intel_dp *intel_dp = dev_priv->drrs.dp;
++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
++ enum drrs_refresh_rate_type index = DRRS_HIGH_RR;
++
++ if (refresh_rate <= 0) {
++ drm_dbg_kms(&dev_priv->drm,
++ "Refresh rate should be positive non-zero.\n");
++ return;
++ }
++
++ if (intel_dp == NULL) {
++ drm_dbg_kms(&dev_priv->drm, "DRRS not supported.\n");
++ return;
++ }
++
++ if (!crtc) {
++ drm_dbg_kms(&dev_priv->drm,
++ "DRRS: intel_crtc not initialized\n");
++ return;
++ }
++
++ if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) {
++ drm_dbg_kms(&dev_priv->drm, "Only Seamless DRRS supported.\n");
++ return;
++ }
++
++ if (drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode) ==
++ refresh_rate)
++ index = DRRS_LOW_RR;
++
++ if (index == dev_priv->drrs.refresh_rate_type) {
++ drm_dbg_kms(&dev_priv->drm,
++ "DRRS requested for previously set RR...ignoring\n");
++ return;
++ }
++
++ if (!crtc_state->hw.active) {
++ drm_dbg_kms(&dev_priv->drm,
++ "eDP encoder disabled. CRTC not Active\n");
++ return;
++ }
++
++ if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) {
++ switch (index) {
++ case DRRS_HIGH_RR:
++ intel_dp_set_m_n(crtc_state, M1_N1);
++ break;
++ case DRRS_LOW_RR:
++ intel_dp_set_m_n(crtc_state, M2_N2);
++ break;
++ case DRRS_MAX_RR:
++ default:
++ drm_err(&dev_priv->drm,
++ "Unsupported refreshrate type\n");
++ }
++ } else if (DISPLAY_VER(dev_priv) > 6) {
++ i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder);
++ u32 val;
++
++ val = intel_de_read(dev_priv, reg);
++ if (index > DRRS_HIGH_RR) {
++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
++ val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV;
++ else
++ val |= PIPECONF_EDP_RR_MODE_SWITCH;
++ } else {
++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
++ val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV;
++ else
++ val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
++ }
++ intel_de_write(dev_priv, reg, val);
++ }
++
++ dev_priv->drrs.refresh_rate_type = index;
++
++ drm_dbg_kms(&dev_priv->drm, "eDP Refresh Rate set to : %dHz\n",
++ refresh_rate);
++}
++
++static void
++intel_edp_drrs_enable_locked(struct intel_dp *intel_dp)
++{
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++
++ dev_priv->drrs.busy_frontbuffer_bits = 0;
++ dev_priv->drrs.dp = intel_dp;
++}
++
++/**
++ * intel_edp_drrs_enable - init drrs struct if supported
++ * @intel_dp: DP struct
++ * @crtc_state: A pointer to the active crtc state.
++ *
++ * Initializes frontbuffer_bits and drrs.dp
++ */
++void intel_edp_drrs_enable(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state)
++{
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++
++ if (!crtc_state->has_drrs)
++ return;
++
++ drm_dbg_kms(&dev_priv->drm, "Enabling DRRS\n");
++
++ mutex_lock(&dev_priv->drrs.mutex);
++
++ if (dev_priv->drrs.dp) {
++ drm_warn(&dev_priv->drm, "DRRS already enabled\n");
++ goto unlock;
++ }
++
++ intel_edp_drrs_enable_locked(intel_dp);
++
++unlock:
++ mutex_unlock(&dev_priv->drrs.mutex);
++}
++
++static void
++intel_edp_drrs_disable_locked(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state)
++{
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++
++ if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) {
++ int refresh;
++
++ refresh = drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode);
++ intel_dp_set_drrs_state(dev_priv, crtc_state, refresh);
++ }
++
++ dev_priv->drrs.dp = NULL;
++}
++
++/**
++ * intel_edp_drrs_disable - Disable DRRS
++ * @intel_dp: DP struct
++ * @old_crtc_state: Pointer to old crtc_state.
++ *
++ */
++void intel_edp_drrs_disable(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *old_crtc_state)
++{
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++
++ if (!old_crtc_state->has_drrs)
++ return;
++
++ mutex_lock(&dev_priv->drrs.mutex);
++ if (!dev_priv->drrs.dp) {
++ mutex_unlock(&dev_priv->drrs.mutex);
++ return;
++ }
++
++ intel_edp_drrs_disable_locked(intel_dp, old_crtc_state);
++ mutex_unlock(&dev_priv->drrs.mutex);
++
++ cancel_delayed_work_sync(&dev_priv->drrs.work);
++}
++
++/**
++ * intel_edp_drrs_update - Update DRRS state
++ * @intel_dp: Intel DP
++ * @crtc_state: new CRTC state
++ *
++ * This function will update DRRS states, disabling or enabling DRRS when
++ * executing fastsets. For full modeset, intel_edp_drrs_disable() and
++ * intel_edp_drrs_enable() should be called instead.
++ */
++void
++intel_edp_drrs_update(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state)
++{
++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
++
++ if (dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT)
++ return;
++
++ mutex_lock(&dev_priv->drrs.mutex);
++
++ /* New state matches current one? */
++ if (crtc_state->has_drrs == !!dev_priv->drrs.dp)
++ goto unlock;
++
++ if (crtc_state->has_drrs)
++ intel_edp_drrs_enable_locked(intel_dp);
++ else
++ intel_edp_drrs_disable_locked(intel_dp, crtc_state);
++
++unlock:
++ mutex_unlock(&dev_priv->drrs.mutex);
++}
++
++static void intel_edp_drrs_downclock_work(struct work_struct *work)
++{
++ struct drm_i915_private *dev_priv =
++ container_of(work, typeof(*dev_priv), drrs.work.work);
++ struct intel_dp *intel_dp;
++
++ mutex_lock(&dev_priv->drrs.mutex);
++
++ intel_dp = dev_priv->drrs.dp;
++
++ if (!intel_dp)
++ goto unlock;
++
++ /*
++ * The delayed work can race with an invalidate hence we need to
++ * recheck.
++ */
++
++ if (dev_priv->drrs.busy_frontbuffer_bits)
++ goto unlock;
++
++ if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) {
++ struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
++
++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
++ drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode));
++ }
++
++unlock:
++ mutex_unlock(&dev_priv->drrs.mutex);
++}
++
++/**
++ * intel_edp_drrs_invalidate - Disable Idleness DRRS
++ * @dev_priv: i915 device
++ * @frontbuffer_bits: frontbuffer plane tracking bits
++ *
++ * This function gets called everytime rendering on the given planes start.
++ * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR).
++ *
++ * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
++ */
++void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
++ unsigned int frontbuffer_bits)
++{
++ struct intel_dp *intel_dp;
++ struct drm_crtc *crtc;
++ enum pipe pipe;
++
++ if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED)
++ return;
++
++ cancel_delayed_work(&dev_priv->drrs.work);
++
++ mutex_lock(&dev_priv->drrs.mutex);
++
++ intel_dp = dev_priv->drrs.dp;
++ if (!intel_dp) {
++ mutex_unlock(&dev_priv->drrs.mutex);
++ return;
++ }
++
++ crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
++ pipe = to_intel_crtc(crtc)->pipe;
++
++ frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe);
++ dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits;
++
++ /* invalidate means busy screen hence upclock */
++ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
++ drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode));
++
++ mutex_unlock(&dev_priv->drrs.mutex);
++}
++
++/**
++ * intel_edp_drrs_flush - Restart Idleness DRRS
++ * @dev_priv: i915 device
++ * @frontbuffer_bits: frontbuffer plane tracking bits
++ *
++ * This function gets called every time rendering on the given planes has
++ * completed or flip on a crtc is completed. So DRRS should be upclocked
++ * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again,
++ * if no other planes are dirty.
++ *
++ * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
++ */
++void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
++ unsigned int frontbuffer_bits)
++{
++ struct intel_dp *intel_dp;
++ struct drm_crtc *crtc;
++ enum pipe pipe;
++
++ if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED)
++ return;
++
++ cancel_delayed_work(&dev_priv->drrs.work);
++
++ mutex_lock(&dev_priv->drrs.mutex);
++
++ intel_dp = dev_priv->drrs.dp;
++ if (!intel_dp) {
++ mutex_unlock(&dev_priv->drrs.mutex);
++ return;
++ }
++
++ crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
++ pipe = to_intel_crtc(crtc)->pipe;
++
++ frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe);
++ dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits;
++
++ /* flush means busy screen hence upclock */
++ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
++ drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode));
++
++ /*
++ * flush also means no more activity hence schedule downclock, if all
++ * other fbs are quiescent too
++ */
++ if (!dev_priv->drrs.busy_frontbuffer_bits)
++ schedule_delayed_work(&dev_priv->drrs.work,
++ msecs_to_jiffies(1000));
++ mutex_unlock(&dev_priv->drrs.mutex);
++}
++
++/**
++ * intel_dp_drrs_init - Init basic DRRS work and mutex.
++ * @connector: eDP connector
++ * @fixed_mode: preferred mode of panel
++ *
++ * This function is called only once at driver load to initialize basic
++ * DRRS stuff.
++ *
++ * Returns:
++ * Downclock mode if panel supports it, else return NULL.
++ * DRRS support is determined by the presence of downclock mode (apart
++ * from VBT setting).
++ */
++struct drm_display_mode *
++intel_dp_drrs_init(struct intel_connector *connector,
++ struct drm_display_mode *fixed_mode)
++{
++ struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
++ struct intel_encoder *encoder = connector->encoder;
++ struct drm_display_mode *downclock_mode = NULL;
++
++ INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
++ mutex_init(&dev_priv->drrs.mutex);
++
++ if (DISPLAY_VER(dev_priv) <= 6) {
++ drm_dbg_kms(&dev_priv->drm,
++ "DRRS supported for Gen7 and above\n");
++ return NULL;
++ }
++
++ if ((DISPLAY_VER(dev_priv) < 8 && !HAS_GMCH(dev_priv)) &&
++ encoder->port != PORT_A) {
++ drm_dbg_kms(&dev_priv->drm,
++ "DRRS only supported on eDP port A\n");
++ return NULL;
++ }
++
++ if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) {
++ drm_dbg_kms(&dev_priv->drm, "VBT doesn't support DRRS\n");
++ return NULL;
++ }
++
++ downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode);
++ if (!downclock_mode) {
++ drm_dbg_kms(&dev_priv->drm,
++ "Downclock mode is not found. DRRS not supported\n");
++ return NULL;
++ }
++
++ dev_priv->drrs.type = dev_priv->vbt.drrs_type;
++
++ dev_priv->drrs.refresh_rate_type = DRRS_HIGH_RR;
++ drm_dbg_kms(&dev_priv->drm,
++ "seamless DRRS supported for eDP panel.\n");
++ return downclock_mode;
++}
+diff --git a/drivers/gpu/drm/i915/display/intel_drrs.h b/drivers/gpu/drm/i915/display/intel_drrs.h
+new file mode 100644
+index 0000000000000..ffa175b4cf4f4
+--- /dev/null
++++ b/drivers/gpu/drm/i915/display/intel_drrs.h
+@@ -0,0 +1,32 @@
++/* SPDX-License-Identifier: MIT */
++/*
++ * Copyright © 2021 Intel Corporation
++ */
++
++#ifndef __INTEL_DRRS_H__
++#define __INTEL_DRRS_H__
++
++#include <linux/types.h>
++
++struct drm_i915_private;
++struct intel_crtc_state;
++struct intel_connector;
++struct intel_dp;
++
++void intel_edp_drrs_enable(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state);
++void intel_edp_drrs_disable(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state);
++void intel_edp_drrs_update(struct intel_dp *intel_dp,
++ const struct intel_crtc_state *crtc_state);
++void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
++ unsigned int frontbuffer_bits);
++void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
++ unsigned int frontbuffer_bits);
++void intel_dp_drrs_compute_config(struct intel_dp *intel_dp,
++ struct intel_crtc_state *pipe_config,
++ int output_bpp, bool constant_n);
++struct drm_display_mode *intel_dp_drrs_init(struct intel_connector *connector,
++ struct drm_display_mode *fixed_mode);
++
++#endif /* __INTEL_DRRS_H__ */
+diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
+index c60a81a81c09c..c6413c5409420 100644
+--- a/drivers/gpu/drm/i915/display/intel_fb.c
++++ b/drivers/gpu/drm/i915/display/intel_fb.c
+@@ -172,8 +172,9 @@ static void intel_fb_plane_dims(const struct intel_framebuffer *fb, int color_pl
+
+ intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, &fb->base, main_plane);
+ intel_fb_plane_get_subsampling(&hsub, &vsub, &fb->base, color_plane);
+- *w = fb->base.width / main_hsub / hsub;
+- *h = fb->base.height / main_vsub / vsub;
++
++ *w = DIV_ROUND_UP(fb->base.width, main_hsub * hsub);
++ *h = DIV_ROUND_UP(fb->base.height, main_vsub * vsub);
+ }
+
+ static u32 intel_adjust_tile_offset(int *x, int *y,
+diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+index 8e75debcce1a9..e4834d84ce5e3 100644
+--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
++++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+@@ -62,6 +62,7 @@
+ #include "intel_display_types.h"
+ #include "intel_fbc.h"
+ #include "intel_frontbuffer.h"
++#include "intel_drrs.h"
+ #include "intel_psr.h"
+
+ /**
+diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
+index b04685bb6439c..926ddc6599f5e 100644
+--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
++++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
+@@ -53,21 +53,20 @@
+ #include "intel_panel.h"
+ #include "intel_snps_phy.h"
+
+-static struct drm_device *intel_hdmi_to_dev(struct intel_hdmi *intel_hdmi)
++static struct drm_i915_private *intel_hdmi_to_i915(struct intel_hdmi *intel_hdmi)
+ {
+- return hdmi_to_dig_port(intel_hdmi)->base.base.dev;
++ return to_i915(hdmi_to_dig_port(intel_hdmi)->base.base.dev);
+ }
+
+ static void
+ assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi)
+ {
+- struct drm_device *dev = intel_hdmi_to_dev(intel_hdmi);
+- struct drm_i915_private *dev_priv = to_i915(dev);
++ struct drm_i915_private *dev_priv = intel_hdmi_to_i915(intel_hdmi);
+ u32 enabled_bits;
+
+ enabled_bits = HAS_DDI(dev_priv) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
+
+- drm_WARN(dev,
++ drm_WARN(&dev_priv->drm,
+ intel_de_read(dev_priv, intel_hdmi->hdmi_reg) & enabled_bits,
+ "HDMI port enabled, expecting disabled\n");
+ }
+@@ -1246,13 +1245,14 @@ static void hsw_set_infoframes(struct intel_encoder *encoder,
+
+ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable)
+ {
+- struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi));
+- struct i2c_adapter *adapter =
+- intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus);
++ struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi);
++ struct i2c_adapter *adapter;
+
+ if (hdmi->dp_dual_mode.type < DRM_DP_DUAL_MODE_TYPE2_DVI)
+ return;
+
++ adapter = intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus);
++
+ drm_dbg_kms(&dev_priv->drm, "%s DP dual mode adaptor TMDS output\n",
+ enable ? "Enabling" : "Disabling");
+
+@@ -1830,7 +1830,8 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
+ int clock, bool respect_downstream_limits,
+ bool has_hdmi_sink)
+ {
+- struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi));
++ struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi);
++ enum phy phy = intel_port_to_phy(dev_priv, hdmi_to_dig_port(hdmi)->base.port);
+
+ if (clock < 25000)
+ return MODE_CLOCK_LOW;
+@@ -1851,6 +1852,14 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
+ if (IS_CHERRYVIEW(dev_priv) && clock > 216000 && clock < 240000)
+ return MODE_CLOCK_RANGE;
+
++ /* ICL+ combo PHY PLL can't generate 500-533.2 MHz */
++ if (intel_phy_is_combo(dev_priv, phy) && clock > 500000 && clock < 533200)
++ return MODE_CLOCK_RANGE;
++
++ /* ICL+ TC PHY PLL can't generate 500-532.8 MHz */
++ if (intel_phy_is_tc(dev_priv, phy) && clock > 500000 && clock < 532800)
++ return MODE_CLOCK_RANGE;
++
+ /*
+ * SNPS PHYs' MPLLB table-based programming can only handle a fixed
+ * set of link rates.
+@@ -1892,7 +1901,7 @@ static bool intel_hdmi_bpc_possible(struct drm_connector *connector,
+ if (ycbcr420_output)
+ return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_36;
+ else
+- return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_36;
++ return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_36;
+ case 10:
+ if (DISPLAY_VER(i915) < 11)
+ return false;
+@@ -1903,7 +1912,7 @@ static bool intel_hdmi_bpc_possible(struct drm_connector *connector,
+ if (ycbcr420_output)
+ return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_30;
+ else
+- return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30;
++ return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30;
+ case 8:
+ return true;
+ default:
+@@ -1946,8 +1955,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
+ struct drm_display_mode *mode)
+ {
+ struct intel_hdmi *hdmi = intel_attached_hdmi(to_intel_connector(connector));
+- struct drm_device *dev = intel_hdmi_to_dev(hdmi);
+- struct drm_i915_private *dev_priv = to_i915(dev);
++ struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi);
+ enum drm_mode_status status;
+ int clock = mode->clock;
+ int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+@@ -2260,6 +2268,17 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
+ return 0;
+ }
+
++void intel_hdmi_encoder_shutdown(struct intel_encoder *encoder)
++{
++ struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
++
++ /*
++ * Give a hand to buggy BIOSen which forget to turn
++ * the TMDS output buffers back on after a reboot.
++ */
++ intel_dp_dual_mode_set_tmds_output(intel_hdmi, true);
++}
++
+ static void
+ intel_hdmi_unset_edid(struct drm_connector *connector)
+ {
+diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h
+index b43a180d007e0..2bf440eb400ab 100644
+--- a/drivers/gpu/drm/i915/display/intel_hdmi.h
++++ b/drivers/gpu/drm/i915/display/intel_hdmi.h
+@@ -28,6 +28,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *dig_port,
+ int intel_hdmi_compute_config(struct intel_encoder *encoder,
+ struct intel_crtc_state *pipe_config,
+ struct drm_connector_state *conn_state);
++void intel_hdmi_encoder_shutdown(struct intel_encoder *encoder);
+ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
+ struct drm_connector *connector,
+ bool high_tmds_clock_ratio,
+diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
+index 3855fba709807..f7f49b69830fa 100644
+--- a/drivers/gpu/drm/i915/display/intel_opregion.c
++++ b/drivers/gpu/drm/i915/display/intel_opregion.c
+@@ -361,6 +361,21 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
+ port++;
+ }
+
++ /*
++ * The port numbering and mapping here is bizarre. The now-obsolete
++ * swsci spec supports ports numbered [0..4]. Port E is handled as a
++ * special case, but port F and beyond are not. The functionality is
++ * supposed to be obsolete for new platforms. Just bail out if the port
++ * number is out of bounds after mapping.
++ */
++ if (port > 4) {
++ drm_dbg_kms(&dev_priv->drm,
++ "[ENCODER:%d:%s] port %c (index %u) out of bounds for display power state notification\n",
++ intel_encoder->base.base.id, intel_encoder->base.name,
++ port_name(intel_encoder->port), port);
++ return -EINVAL;
++ }
++
+ if (!enable)
+ parm |= 4 << 8;
+
+diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
+index 7e3f5c6ca4846..dfa5f18171e3b 100644
+--- a/drivers/gpu/drm/i915/display/intel_overlay.c
++++ b/drivers/gpu/drm/i915/display/intel_overlay.c
+@@ -959,6 +959,9 @@ static int check_overlay_dst(struct intel_overlay *overlay,
+ const struct intel_crtc_state *pipe_config =
+ overlay->crtc->config;
+
++ if (rec->dst_height == 0 || rec->dst_width == 0)
++ return -EINVAL;
++
+ if (rec->dst_x < pipe_config->pipe_src_w &&
+ rec->dst_x + rec->dst_width <= pipe_config->pipe_src_w &&
+ rec->dst_y < pipe_config->pipe_src_h &&
+diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
+index a36ec4a818ff5..466bf6820641d 100644
+--- a/drivers/gpu/drm/i915/display/intel_pps.c
++++ b/drivers/gpu/drm/i915/display/intel_pps.c
+@@ -1074,14 +1074,14 @@ static void intel_pps_vdd_sanitize(struct intel_dp *intel_dp)
+ edp_panel_vdd_schedule_off(intel_dp);
+ }
+
+-bool intel_pps_have_power(struct intel_dp *intel_dp)
++bool intel_pps_have_panel_power_or_vdd(struct intel_dp *intel_dp)
+ {
+ intel_wakeref_t wakeref;
+ bool have_power = false;
+
+ with_intel_pps_lock(intel_dp, wakeref) {
+- have_power = edp_have_panel_power(intel_dp) &&
+- edp_have_panel_vdd(intel_dp);
++ have_power = edp_have_panel_power(intel_dp) ||
++ edp_have_panel_vdd(intel_dp);
+ }
+
+ return have_power;
+diff --git a/drivers/gpu/drm/i915/display/intel_pps.h b/drivers/gpu/drm/i915/display/intel_pps.h
+index fbbcca782e7b6..9fe7be4fe867e 100644
+--- a/drivers/gpu/drm/i915/display/intel_pps.h
++++ b/drivers/gpu/drm/i915/display/intel_pps.h
+@@ -36,7 +36,7 @@ void intel_pps_vdd_on(struct intel_dp *intel_dp);
+ void intel_pps_on(struct intel_dp *intel_dp);
+ void intel_pps_off(struct intel_dp *intel_dp);
+ void intel_pps_vdd_off_sync(struct intel_dp *intel_dp);
+-bool intel_pps_have_power(struct intel_dp *intel_dp);
++bool intel_pps_have_panel_power_or_vdd(struct intel_dp *intel_dp);
+ void intel_pps_wait_power_cycle(struct intel_dp *intel_dp);
+
+ void intel_pps_init(struct intel_dp *intel_dp);
+diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
+index 1b0daf649e823..a3d0c57ec0f0b 100644
+--- a/drivers/gpu/drm/i915/display/intel_psr.c
++++ b/drivers/gpu/drm/i915/display/intel_psr.c
+@@ -936,6 +936,20 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
+ return false;
+ }
+
++ /* Wa_16011303918:adl-p */
++ if (crtc_state->vrr.enable &&
++ IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
++ drm_dbg_kms(&dev_priv->drm,
++ "PSR2 not enabled, not compatible with HW stepping + VRR\n");
++ return false;
++ }
++
++ if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
++ drm_dbg_kms(&dev_priv->drm,
++ "PSR2 not enabled, PSR2 SDP indication do not fit in hblank\n");
++ return false;
++ }
++
+ if (HAS_PSR2_SEL_FETCH(dev_priv)) {
+ if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&
+ !HAS_PSR_HW_TRACKING(dev_priv)) {
+@@ -949,12 +963,12 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
+ if (!crtc_state->enable_psr2_sel_fetch &&
+ IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) {
+ drm_dbg_kms(&dev_priv->drm, "PSR2 HW tracking is not supported this Display stepping\n");
+- return false;
++ goto unsupported;
+ }
+
+ if (!psr2_granularity_check(intel_dp, crtc_state)) {
+ drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, SU granularity not compatible\n");
+- return false;
++ goto unsupported;
+ }
+
+ if (!crtc_state->enable_psr2_sel_fetch &&
+@@ -963,25 +977,15 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
+ "PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
+ crtc_hdisplay, crtc_vdisplay,
+ psr_max_h, psr_max_v);
+- return false;
+- }
+-
+- if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
+- drm_dbg_kms(&dev_priv->drm,
+- "PSR2 not enabled, PSR2 SDP indication do not fit in hblank\n");
+- return false;
+- }
+-
+- /* Wa_16011303918:adl-p */
+- if (crtc_state->vrr.enable &&
+- IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
+- drm_dbg_kms(&dev_priv->drm,
+- "PSR2 not enabled, not compatible with HW stepping + VRR\n");
+- return false;
++ goto unsupported;
+ }
+
+ tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);
+ return true;
++
++unsupported:
++ crtc_state->enable_psr2_sel_fetch = false;
++ return false;
+ }
+
+ void intel_psr_compute_config(struct intel_dp *intel_dp,
+diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c
+index 18b52b64af955..536b319ffe5ba 100644
+--- a/drivers/gpu/drm/i915/display/intel_snps_phy.c
++++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c
+@@ -32,7 +32,7 @@ void intel_snps_phy_wait_for_calibration(struct drm_i915_private *dev_priv)
+ if (intel_de_wait_for_clear(dev_priv, ICL_PHY_MISC(phy),
+ DG2_PHY_DP_TX_ACK_MASK, 25))
+ DRM_ERROR("SNPS PHY %c failed to calibrate after 25ms.\n",
+- phy);
++ phy_name(phy));
+ }
+ }
+
+diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
+index 3ffece568ed98..0e885440be242 100644
+--- a/drivers/gpu/drm/i915/display/intel_tc.c
++++ b/drivers/gpu/drm/i915/display/intel_tc.c
+@@ -291,10 +291,11 @@ static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port)
+ static bool adl_tc_phy_status_complete(struct intel_digital_port *dig_port)
+ {
+ struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
++ enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port);
+ struct intel_uncore *uncore = &i915->uncore;
+ u32 val;
+
+- val = intel_uncore_read(uncore, TCSS_DDI_STATUS(dig_port->tc_phy_fia_idx));
++ val = intel_uncore_read(uncore, TCSS_DDI_STATUS(tc_port));
+ if (val == 0xffffffff) {
+ drm_dbg_kms(&i915->drm,
+ "Port %s: PHY in TCCOLD, assuming not complete\n",
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+index 5130e8ed95647..28e07040cf47a 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
++++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+@@ -66,7 +66,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
+ * mmap ioctl is disallowed for all discrete platforms,
+ * and for all platforms with GRAPHICS_VER > 12.
+ */
+- if (IS_DGFX(i915) || GRAPHICS_VER(i915) > 12)
++ if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0))
+ return -EOPNOTSUPP;
+
+ if (args->flags & ~(I915_MMAP_WC))
+@@ -438,7 +438,7 @@ vm_access(struct vm_area_struct *area, unsigned long addr,
+ return -EACCES;
+
+ addr -= area->vm_start;
+- if (addr >= obj->base.size)
++ if (range_overflows_t(u64, addr, len, obj->base.size))
+ return -EINVAL;
+
+ i915_gem_ww_ctx_init(&ww, true);
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+index 2471f36aaff38..3012cbe5b0b7c 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
++++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+@@ -298,6 +298,7 @@ struct drm_i915_gem_object {
+ I915_BO_ALLOC_USER)
+ #define I915_BO_READONLY BIT(4)
+ #define I915_TILING_QUIRK_BIT 5 /* unknown swizzling; do not release! */
++#define I915_BO_WAS_BOUND_BIT 6
+
+ /**
+ * @mem_flags - Mutable placement-related flags
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+index 8eb1c3a6fc9cd..9053cea3395a6 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
++++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+@@ -10,6 +10,8 @@
+ #include "i915_gem_lmem.h"
+ #include "i915_gem_mman.h"
+
++#include "gt/intel_gt.h"
++
+ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+ struct sg_table *pages,
+ unsigned int sg_page_sizes)
+@@ -160,7 +162,6 @@ retry:
+ /* Immediately discard the backing storage */
+ void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
+ {
+- drm_gem_free_mmap_offset(&obj->base);
+ if (obj->ops->truncate)
+ obj->ops->truncate(obj);
+ }
+@@ -218,6 +219,14 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
+ __i915_gem_object_reset_page_iter(obj);
+ obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
+
++ if (test_and_clear_bit(I915_BO_WAS_BOUND_BIT, &obj->flags)) {
++ struct drm_i915_private *i915 = to_i915(obj->base.dev);
++ intel_wakeref_t wakeref;
++
++ with_intel_runtime_pm_if_active(&i915->runtime_pm, wakeref)
++ intel_gt_invalidate_tlbs(&i915->gt);
++ }
++
+ return pages;
+ }
+
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+index 6ea13159bffcc..4b823fbfe76a1 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
++++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+@@ -759,11 +759,9 @@ static void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj)
+ if (obj->mm.madv != I915_MADV_WILLNEED) {
+ bo->priority = I915_TTM_PRIO_PURGE;
+ } else if (!i915_gem_object_has_pages(obj)) {
+- if (bo->priority < I915_TTM_PRIO_HAS_PAGES)
+- bo->priority = I915_TTM_PRIO_HAS_PAGES;
++ bo->priority = I915_TTM_PRIO_NO_PAGES;
+ } else {
+- if (bo->priority > I915_TTM_PRIO_NO_PAGES)
+- bo->priority = I915_TTM_PRIO_NO_PAGES;
++ bo->priority = I915_TTM_PRIO_HAS_PAGES;
+ }
+
+ ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);
+diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+index de5f9c86b9a44..cafb0608ffb46 100644
+--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
++++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+@@ -2140,10 +2140,6 @@ static void __execlists_unhold(struct i915_request *rq)
+ if (p->flags & I915_DEPENDENCY_WEAK)
+ continue;
+
+- /* Propagate any change in error status */
+- if (rq->fence.error)
+- i915_request_set_error_once(w, rq->fence.error);
+-
+ if (w->engine != rq->engine)
+ continue;
+
+diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
+index 62d40c9866427..e1e1d17d49fdd 100644
+--- a/drivers/gpu/drm/i915/gt/intel_gt.c
++++ b/drivers/gpu/drm/i915/gt/intel_gt.c
+@@ -29,6 +29,8 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
+
+ spin_lock_init(&gt->irq_lock);
+
++ mutex_init(&gt->tlb_invalidate_lock);
++
+ INIT_LIST_HEAD(&gt->closed_vma);
+ spin_lock_init(&gt->closed_lock);
+
+@@ -895,3 +897,103 @@ void intel_gt_info_print(const struct intel_gt_info *info,
+
+ intel_sseu_dump(&info->sseu, p);
+ }
++
++struct reg_and_bit {
++ i915_reg_t reg;
++ u32 bit;
++};
++
++static struct reg_and_bit
++get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
++ const i915_reg_t *regs, const unsigned int num)
++{
++ const unsigned int class = engine->class;
++ struct reg_and_bit rb = { };
++
++ if (drm_WARN_ON_ONCE(&engine->i915->drm,
++ class >= num || !regs[class].reg))
++ return rb;
++
++ rb.reg = regs[class];
++ if (gen8 && class == VIDEO_DECODE_CLASS)
++ rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
++ else
++ rb.bit = engine->instance;
++
++ rb.bit = BIT(rb.bit);
++
++ return rb;
++}
++
++void intel_gt_invalidate_tlbs(struct intel_gt *gt)
++{
++ static const i915_reg_t gen8_regs[] = {
++ [RENDER_CLASS] = GEN8_RTCR,
++ [VIDEO_DECODE_CLASS] = GEN8_M1TCR, /* , GEN8_M2TCR */
++ [VIDEO_ENHANCEMENT_CLASS] = GEN8_VTCR,
++ [COPY_ENGINE_CLASS] = GEN8_BTCR,
++ };
++ static const i915_reg_t gen12_regs[] = {
++ [RENDER_CLASS] = GEN12_GFX_TLB_INV_CR,
++ [VIDEO_DECODE_CLASS] = GEN12_VD_TLB_INV_CR,
++ [VIDEO_ENHANCEMENT_CLASS] = GEN12_VE_TLB_INV_CR,
++ [COPY_ENGINE_CLASS] = GEN12_BLT_TLB_INV_CR,
++ };
++ struct drm_i915_private *i915 = gt->i915;
++ struct intel_uncore *uncore = gt->uncore;
++ struct intel_engine_cs *engine;
++ enum intel_engine_id id;
++ const i915_reg_t *regs;
++ unsigned int num = 0;
++
++ if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
++ return;
++
++ if (GRAPHICS_VER(i915) == 12) {
++ regs = gen12_regs;
++ num = ARRAY_SIZE(gen12_regs);
++ } else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
++ regs = gen8_regs;
++ num = ARRAY_SIZE(gen8_regs);
++ } else if (GRAPHICS_VER(i915) < 8) {
++ return;
++ }
++
++ if (drm_WARN_ONCE(&i915->drm, !num,
++ "Platform does not implement TLB invalidation!"))
++ return;
++
++ GEM_TRACE("\n");
++
++ assert_rpm_wakelock_held(&i915->runtime_pm);
++
++ mutex_lock(&gt->tlb_invalidate_lock);
++ intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
++
++ for_each_engine(engine, gt, id) {
++ /*
++ * HW architecture suggest typical invalidation time at 40us,
++ * with pessimistic cases up to 100us and a recommendation to
++ * cap at 1ms. We go a bit higher just in case.
++ */
++ const unsigned int timeout_us = 100;
++ const unsigned int timeout_ms = 4;
++ struct reg_and_bit rb;
++
++ rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
++ if (!i915_mmio_reg_offset(rb.reg))
++ continue;
++
++ intel_uncore_write_fw(uncore, rb.reg, rb.bit);
++ if (__intel_wait_for_register_fw(uncore,
++ rb.reg, rb.bit, 0,
++ timeout_us, timeout_ms,
++ NULL))
++ drm_err_ratelimited(&gt->i915->drm,
++ "%s TLB invalidation did not complete in %ums!\n",
++ engine->name, timeout_ms);
++ }
++
++ intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
++ mutex_unlock(&gt->tlb_invalidate_lock);
++}
+diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h
+index 74e771871a9bd..c0169d6017c2d 100644
+--- a/drivers/gpu/drm/i915/gt/intel_gt.h
++++ b/drivers/gpu/drm/i915/gt/intel_gt.h
+@@ -90,4 +90,6 @@ void intel_gt_info_print(const struct intel_gt_info *info,
+
+ void intel_gt_watchdog_work(struct work_struct *work);
+
++void intel_gt_invalidate_tlbs(struct intel_gt *gt);
++
+ #endif /* __INTEL_GT_H__ */
+diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
+index a81e21bf1bd1a..9fbcbcc6c35db 100644
+--- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
++++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
+@@ -72,6 +72,8 @@ struct intel_gt {
+
+ struct intel_uc uc;
+
++ struct mutex tlb_invalidate_lock;
++
+ struct intel_gt_timelines {
+ spinlock_t lock; /* protects active_list */
+ struct list_head active_list;
+diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
+index aae609d7d85dd..6b5ab19a2ada9 100644
+--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
++++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
+@@ -621,13 +621,6 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
+ FF_MODE2_GS_TIMER_MASK,
+ FF_MODE2_GS_TIMER_224,
+ 0, false);
+-
+- /*
+- * Wa_14012131227:dg1
+- * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
+- */
+- wa_masked_en(wal, GEN7_COMMON_SLICE_CHICKEN1,
+- GEN9_RHWO_OPTIMIZATION_DISABLE);
+ }
+
+ static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
+diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+index 65a3e7fdb2b2c..95ff630157b9c 100644
+--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+@@ -133,7 +133,7 @@ static int guc_action_slpc_unset_param(struct intel_guc *guc, u8 id)
+ {
+ u32 request[] = {
+ GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
+- SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 2),
++ SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1),
+ id,
+ };
+
+diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+index 87d8dc8f51b96..93c9de8f43e8e 100644
+--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
++++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+@@ -148,11 +148,12 @@ static inline void clr_context_registered(struct intel_context *ce)
+ #define SCHED_STATE_BLOCKED_SHIFT 4
+ #define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT)
+ #define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT)
++
+ static inline void init_sched_state(struct intel_context *ce)
+ {
+ /* Only should be called from guc_lrc_desc_pin() */
+ atomic_set(&ce->guc_sched_state_no_lock, 0);
+- ce->guc_state.sched_state = 0;
++ ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK;
+ }
+
+ static inline bool
+@@ -352,20 +353,29 @@ static inline void set_lrc_desc_registered(struct intel_guc *guc, u32 id,
+ xa_unlock_irqrestore(&guc->context_lookup, flags);
+ }
+
++static void decr_outstanding_submission_g2h(struct intel_guc *guc)
++{
++ if (atomic_dec_and_test(&guc->outstanding_submission_g2h))
++ wake_up_all(&guc->ct.wq);
++}
++
+ static int guc_submission_send_busy_loop(struct intel_guc *guc,
+ const u32 *action,
+ u32 len,
+ u32 g2h_len_dw,
+ bool loop)
+ {
+- int err;
+-
+- err = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
++ /*
++ * We always loop when a send requires a reply (i.e. g2h_len_dw > 0),
++ * so we don't handle the case where we don't get a reply because we
++ * aborted the send due to the channel being busy.
++ */
++ GEM_BUG_ON(g2h_len_dw && !loop);
+
+- if (!err && g2h_len_dw)
++ if (g2h_len_dw)
+ atomic_inc(&guc->outstanding_submission_g2h);
+
+- return err;
++ return intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
+ }
+
+ int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
+@@ -616,7 +626,7 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
+ init_sched_state(ce);
+
+ if (pending_enable || destroyed || deregister) {
+- atomic_dec(&guc->outstanding_submission_g2h);
++ decr_outstanding_submission_g2h(guc);
+ if (deregister)
+ guc_signal_context_fence(ce);
+ if (destroyed) {
+@@ -635,7 +645,7 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
+ intel_engine_signal_breadcrumbs(ce->engine);
+ }
+ intel_context_sched_disable_unpin(ce);
+- atomic_dec(&guc->outstanding_submission_g2h);
++ decr_outstanding_submission_g2h(guc);
+ spin_lock_irqsave(&ce->guc_state.lock, flags);
+ guc_blocked_fence_complete(ce);
+ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+@@ -797,15 +807,13 @@ __unwind_incomplete_requests(struct intel_context *ce)
+
+ spin_lock_irqsave(&sched_engine->lock, flags);
+ spin_lock(&ce->guc_active.lock);
+- list_for_each_entry_safe(rq, rn,
+- &ce->guc_active.requests,
+- sched.link) {
++ list_for_each_entry_safe_reverse(rq, rn,
++ &ce->guc_active.requests,
++ sched.link) {
+ if (i915_request_completed(rq))
+ continue;
+
+ list_del_init(&rq->sched.link);
+- spin_unlock(&ce->guc_active.lock);
+-
+ __i915_request_unsubmit(rq);
+
+ /* Push the request back into the queue for later resubmission. */
+@@ -816,10 +824,8 @@ __unwind_incomplete_requests(struct intel_context *ce)
+ }
+ GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine));
+
+- list_add_tail(&rq->sched.link, pl);
++ list_add(&rq->sched.link, pl);
+ set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+-
+- spin_lock(&ce->guc_active.lock);
+ }
+ spin_unlock(&ce->guc_active.lock);
+ spin_unlock_irqrestore(&sched_engine->lock, flags);
+@@ -828,17 +834,33 @@ __unwind_incomplete_requests(struct intel_context *ce)
+ static void __guc_reset_context(struct intel_context *ce, bool stalled)
+ {
+ struct i915_request *rq;
++ unsigned long flags;
+ u32 head;
++ bool skip = false;
+
+ intel_context_get(ce);
+
+ /*
+- * GuC will implicitly mark the context as non-schedulable
+- * when it sends the reset notification. Make sure our state
+- * reflects this change. The context will be marked enabled
+- * on resubmission.
++ * GuC will implicitly mark the context as non-schedulable when it sends
++ * the reset notification. Make sure our state reflects this change. The
++ * context will be marked enabled on resubmission.
++ *
++ * XXX: If the context is reset as a result of the request cancellation
++ * this G2H is received after the schedule disable complete G2H which is
++ * wrong as this creates a race between the request cancellation code
++ * re-submitting the context and this G2H handler. This is a bug in the
++ * GuC but can be worked around in the meantime but converting this to a
++ * NOP if a pending enable is in flight as this indicates that a request
++ * cancellation has occurred.
+ */
+- clr_context_enabled(ce);
++ spin_lock_irqsave(&ce->guc_state.lock, flags);
++ if (likely(!context_pending_enable(ce)))
++ clr_context_enabled(ce);
++ else
++ skip = true;
++ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
++ if (unlikely(skip))
++ goto out_put;
+
+ rq = intel_context_find_active_request(ce);
+ if (!rq) {
+@@ -857,6 +879,7 @@ static void __guc_reset_context(struct intel_context *ce, bool stalled)
+ out_replay:
+ guc_reset_state(ce, head, stalled);
+ __unwind_incomplete_requests(ce);
++out_put:
+ intel_context_put(ce);
+ }
+
+@@ -1233,8 +1256,7 @@ static int register_context(struct intel_context *ce, bool loop)
+ }
+
+ static int __guc_action_deregister_context(struct intel_guc *guc,
+- u32 guc_id,
+- bool loop)
++ u32 guc_id)
+ {
+ u32 action[] = {
+ INTEL_GUC_ACTION_DEREGISTER_CONTEXT,
+@@ -1243,16 +1265,16 @@ static int __guc_action_deregister_context(struct intel_guc *guc,
+
+ return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
+ G2H_LEN_DW_DEREGISTER_CONTEXT,
+- loop);
++ true);
+ }
+
+-static int deregister_context(struct intel_context *ce, u32 guc_id, bool loop)
++static int deregister_context(struct intel_context *ce, u32 guc_id)
+ {
+ struct intel_guc *guc = ce_to_guc(ce);
+
+ trace_intel_context_deregister(ce);
+
+- return __guc_action_deregister_context(guc, guc_id, loop);
++ return __guc_action_deregister_context(guc, guc_id);
+ }
+
+ static intel_engine_mask_t adjust_engine_mask(u8 class, intel_engine_mask_t mask)
+@@ -1340,26 +1362,23 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop)
+ * registering this context.
+ */
+ if (context_registered) {
++ bool disabled;
++ unsigned long flags;
++
+ trace_intel_context_steal_guc_id(ce);
+- if (!loop) {
++ GEM_BUG_ON(!loop);
++
++ /* Seal race with Reset */
++ spin_lock_irqsave(&ce->guc_state.lock, flags);
++ disabled = submission_disabled(guc);
++ if (likely(!disabled)) {
+ set_context_wait_for_deregister_to_register(ce);
+ intel_context_get(ce);
+- } else {
+- bool disabled;
+- unsigned long flags;
+-
+- /* Seal race with Reset */
+- spin_lock_irqsave(&ce->guc_state.lock, flags);
+- disabled = submission_disabled(guc);
+- if (likely(!disabled)) {
+- set_context_wait_for_deregister_to_register(ce);
+- intel_context_get(ce);
+- }
+- spin_unlock_irqrestore(&ce->guc_state.lock, flags);
+- if (unlikely(disabled)) {
+- reset_lrc_desc(guc, desc_idx);
+- return 0; /* Will get registered later */
+- }
++ }
++ spin_unlock_irqrestore(&ce->guc_state.lock, flags);
++ if (unlikely(disabled)) {
++ reset_lrc_desc(guc, desc_idx);
++ return 0; /* Will get registered later */
+ }
+
+ /*
+@@ -1367,13 +1386,9 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop)
+ * context whose guc_id was stolen.
+ */
+ with_intel_runtime_pm(runtime_pm, wakeref)
+- ret = deregister_context(ce, ce->guc_id, loop);
+- if (unlikely(ret == -EBUSY)) {
+- clr_context_wait_for_deregister_to_register(ce);
+- intel_context_put(ce);
+- } else if (unlikely(ret == -ENODEV)) {
++ ret = deregister_context(ce, ce->guc_id);
++ if (unlikely(ret == -ENODEV))
+ ret = 0; /* Will get registered later */
+- }
+ } else {
+ with_intel_runtime_pm(runtime_pm, wakeref)
+ ret = register_context(ce, loop);
+@@ -1548,6 +1563,23 @@ static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
+ return &ce->guc_blocked;
+ }
+
++#define SCHED_STATE_MULTI_BLOCKED_MASK \
++ (SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED)
++#define SCHED_STATE_NO_UNBLOCK \
++ (SCHED_STATE_MULTI_BLOCKED_MASK | \
++ SCHED_STATE_PENDING_DISABLE | \
++ SCHED_STATE_BANNED)
++
++static bool context_cant_unblock(struct intel_context *ce)
++{
++ lockdep_assert_held(&ce->guc_state.lock);
++
++ return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) ||
++ context_guc_id_invalid(ce) ||
++ !lrc_desc_registered(ce_to_guc(ce), ce->guc_id) ||
++ !intel_context_is_pinned(ce);
++}
++
+ static void guc_context_unblock(struct intel_context *ce)
+ {
+ struct intel_guc *guc = ce_to_guc(ce);
+@@ -1562,9 +1594,7 @@ static void guc_context_unblock(struct intel_context *ce)
+ spin_lock_irqsave(&ce->guc_state.lock, flags);
+
+ if (unlikely(submission_disabled(guc) ||
+- !intel_context_is_pinned(ce) ||
+- context_pending_disable(ce) ||
+- context_blocked(ce) > 1)) {
++ context_cant_unblock(ce))) {
+ enable = false;
+ } else {
+ enable = true;
+@@ -1601,6 +1631,13 @@ static void guc_context_cancel_request(struct intel_context *ce,
+ guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),
+ true);
+ }
++
++ /*
++ * XXX: Racey if context is reset, see comment in
++ * __guc_reset_context().
++ */
++ flush_work(&ce_to_guc(ce)->ct.requests.worker);
++
+ guc_context_unblock(ce);
+ }
+ }
+@@ -1730,7 +1767,7 @@ static inline void guc_lrc_desc_unpin(struct intel_context *ce)
+ GEM_BUG_ON(context_enabled(ce));
+
+ clr_context_registered(ce);
+- deregister_context(ce, ce->guc_id, true);
++ deregister_context(ce, ce->guc_id);
+ }
+
+ static void __guc_context_destroy(struct intel_context *ce)
+@@ -2583,12 +2620,6 @@ g2h_context_lookup(struct intel_guc *guc, u32 desc_idx)
+ return ce;
+ }
+
+-static void decr_outstanding_submission_g2h(struct intel_guc *guc)
+-{
+- if (atomic_dec_and_test(&guc->outstanding_submission_g2h))
+- wake_up_all(&guc->ct.wq);
+-}
+-
+ int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
+ const u32 *msg,
+ u32 len)
+@@ -2721,7 +2752,12 @@ static void guc_handle_context_reset(struct intel_guc *guc,
+ {
+ trace_intel_context_reset(ce);
+
+- if (likely(!intel_context_is_banned(ce))) {
++ /*
++ * XXX: Racey if request cancellation has occurred, see comment in
++ * __guc_reset_context().
++ */
++ if (likely(!intel_context_is_banned(ce) &&
++ !context_blocked(ce))) {
+ capture_error_state(guc, ce);
+ guc_context_replay(ce);
+ }
+diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
+index 1bbd09ad52873..1ad7259fb1f0c 100644
+--- a/drivers/gpu/drm/i915/i915_pci.c
++++ b/drivers/gpu/drm/i915/i915_pci.c
+@@ -865,7 +865,7 @@ static const struct intel_device_info jsl_info = {
+ }, \
+ TGL_CURSOR_OFFSETS, \
+ .has_global_mocs = 1, \
+- .display.has_dsb = 1
++ .display.has_dsb = 0 /* FIXME: LUT load is broken with DSB */
+
+ static const struct intel_device_info tgl_info = {
+ GEN12_FEATURES,
+diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
+index 9023d4ecf3b37..3c70aa5229e5a 100644
+--- a/drivers/gpu/drm/i915/i915_reg.h
++++ b/drivers/gpu/drm/i915/i915_reg.h
+@@ -2669,6 +2669,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
+ #define GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING (1 << 28)
+ #define GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT (1 << 24)
+
++#define GEN8_RTCR _MMIO(0x4260)
++#define GEN8_M1TCR _MMIO(0x4264)
++#define GEN8_M2TCR _MMIO(0x4268)
++#define GEN8_BTCR _MMIO(0x426c)
++#define GEN8_VTCR _MMIO(0x4270)
++
+ #if 0
+ #define PRB0_TAIL _MMIO(0x2030)
+ #define PRB0_HEAD _MMIO(0x2034)
+@@ -2763,6 +2769,11 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
+ #define FAULT_VA_HIGH_BITS (0xf << 0)
+ #define FAULT_GTT_SEL (1 << 4)
+
++#define GEN12_GFX_TLB_INV_CR _MMIO(0xced8)
++#define GEN12_VD_TLB_INV_CR _MMIO(0xcedc)
++#define GEN12_VE_TLB_INV_CR _MMIO(0xcee0)
++#define GEN12_BLT_TLB_INV_CR _MMIO(0xcee4)
++
+ #define GEN12_AUX_ERR_DBG _MMIO(0x43f4)
+
+ #define FPGA_DBG _MMIO(0x42300)
+@@ -7447,7 +7458,7 @@ enum {
+ #define _SEL_FETCH_PLANE_BASE_6_A 0x70940
+ #define _SEL_FETCH_PLANE_BASE_7_A 0x70960
+ #define _SEL_FETCH_PLANE_BASE_CUR_A 0x70880
+-#define _SEL_FETCH_PLANE_BASE_1_B 0x70990
++#define _SEL_FETCH_PLANE_BASE_1_B 0x71890
+
+ #define _SEL_FETCH_PLANE_BASE_A(plane) _PICK(plane, \
+ _SEL_FETCH_PLANE_BASE_1_A, \
+@@ -7807,6 +7818,22 @@ enum {
+ /* MMIO address range for DMC program (0x80000 - 0x82FFF) */
+ #define DMC_MMIO_START_RANGE 0x80000
+ #define DMC_MMIO_END_RANGE 0x8FFFF
++#define DMC_V1_MMIO_START_RANGE 0x80000
++#define TGL_MAIN_MMIO_START 0x8F000
++#define TGL_MAIN_MMIO_END 0x8FFFF
++#define _TGL_PIPEA_MMIO_START 0x92000
++#define _TGL_PIPEA_MMIO_END 0x93FFF
++#define _TGL_PIPEB_MMIO_START 0x96000
++#define _TGL_PIPEB_MMIO_END 0x97FFF
++#define ADLP_PIPE_MMIO_START 0x5F000
++#define ADLP_PIPE_MMIO_END 0x5FFFF
++
++#define TGL_PIPE_MMIO_START(dmc_id) _PICK_EVEN(((dmc_id) - 1), _TGL_PIPEA_MMIO_START,\
++ _TGL_PIPEB_MMIO_START)
++
++#define TGL_PIPE_MMIO_END(dmc_id) _PICK_EVEN(((dmc_id) - 1), _TGL_PIPEA_MMIO_END,\
++ _TGL_PIPEB_MMIO_END)
++
+ #define SKL_DMC_DC3_DC5_COUNT _MMIO(0x80030)
+ #define SKL_DMC_DC5_DC6_COUNT _MMIO(0x8002C)
+ #define BXT_DMC_DC3_DC5_COUNT _MMIO(0x80038)
+diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
+index 4b7fc4647e460..dfd20060812bc 100644
+--- a/drivers/gpu/drm/i915/i915_vma.c
++++ b/drivers/gpu/drm/i915/i915_vma.c
+@@ -434,6 +434,9 @@ int i915_vma_bind(struct i915_vma *vma,
+ vma->ops->bind_vma(vma->vm, NULL, vma, cache_level, bind_flags);
+ }
+
++ if (vma->obj)
++ set_bit(I915_BO_WAS_BOUND_BIT, &vma->obj->flags);
++
+ atomic_or(bind_flags, &vma->flags);
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c
+index d1d4b97b86f59..287f5a3d0b354 100644
+--- a/drivers/gpu/drm/i915/intel_pch.c
++++ b/drivers/gpu/drm/i915/intel_pch.c
+@@ -108,6 +108,7 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id)
+ /* Comet Lake V PCH is based on KBP, which is SPT compatible */
+ return PCH_SPT;
+ case INTEL_PCH_ICP_DEVICE_ID_TYPE:
++ case INTEL_PCH_ICP2_DEVICE_ID_TYPE:
+ drm_dbg_kms(&dev_priv->drm, "Found Ice Lake PCH\n");
+ drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
+ return PCH_ICP;
+@@ -123,7 +124,6 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id)
+ !IS_GEN9_BC(dev_priv));
+ return PCH_TGP;
+ case INTEL_PCH_JSP_DEVICE_ID_TYPE:
+- case INTEL_PCH_JSP2_DEVICE_ID_TYPE:
+ drm_dbg_kms(&dev_priv->drm, "Found Jasper Lake PCH\n");
+ drm_WARN_ON(&dev_priv->drm, !IS_JSL_EHL(dev_priv));
+ return PCH_JSP;
+diff --git a/drivers/gpu/drm/i915/intel_pch.h b/drivers/gpu/drm/i915/intel_pch.h
+index 7c0d83d292dcc..994c56fcb1991 100644
+--- a/drivers/gpu/drm/i915/intel_pch.h
++++ b/drivers/gpu/drm/i915/intel_pch.h
+@@ -50,11 +50,11 @@ enum intel_pch {
+ #define INTEL_PCH_CMP2_DEVICE_ID_TYPE 0x0680
+ #define INTEL_PCH_CMP_V_DEVICE_ID_TYPE 0xA380
+ #define INTEL_PCH_ICP_DEVICE_ID_TYPE 0x3480
++#define INTEL_PCH_ICP2_DEVICE_ID_TYPE 0x3880
+ #define INTEL_PCH_MCC_DEVICE_ID_TYPE 0x4B00
+ #define INTEL_PCH_TGP_DEVICE_ID_TYPE 0xA080
+ #define INTEL_PCH_TGP2_DEVICE_ID_TYPE 0x4380
+ #define INTEL_PCH_JSP_DEVICE_ID_TYPE 0x4D80
+-#define INTEL_PCH_JSP2_DEVICE_ID_TYPE 0x3880
+ #define INTEL_PCH_ADP_DEVICE_ID_TYPE 0x7A80
+ #define INTEL_PCH_ADP2_DEVICE_ID_TYPE 0x5180
+ #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100
+diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
+index a725792d5248b..57c1dda76b94a 100644
+--- a/drivers/gpu/drm/i915/intel_pm.c
++++ b/drivers/gpu/drm/i915/intel_pm.c
+@@ -3063,9 +3063,9 @@ static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
+ * The BIOS provided WM memory latency values are often
+ * inadequate for high resolution displays. Adjust them.
+ */
+- changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
+- ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
+- ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
++ changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
++ changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
++ changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
+
+ if (!changed)
+ return;
+@@ -3713,8 +3713,7 @@ skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
+ MISSING_CASE(DISPLAY_VER(dev_priv));
+ }
+
+- /* Default to an unusable block time */
+- dev_priv->sagv_block_time_us = -1;
++ dev_priv->sagv_block_time_us = 0;
+ }
+
+ /*
+@@ -4020,6 +4019,17 @@ static int intel_compute_sagv_mask(struct intel_atomic_state *state)
+ return ret;
+ }
+
++ if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
++ intel_can_enable_sagv(dev_priv, old_bw_state)) {
++ ret = intel_atomic_serialize_global_state(&new_bw_state->base);
++ if (ret)
++ return ret;
++ } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
++ ret = intel_atomic_lock_global_state(&new_bw_state->base);
++ if (ret)
++ return ret;
++ }
++
+ for_each_new_intel_crtc_in_state(state, crtc,
+ new_crtc_state, i) {
+ struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
+@@ -4035,17 +4045,6 @@ static int intel_compute_sagv_mask(struct intel_atomic_state *state)
+ intel_can_enable_sagv(dev_priv, new_bw_state);
+ }
+
+- if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
+- intel_can_enable_sagv(dev_priv, old_bw_state)) {
+- ret = intel_atomic_serialize_global_state(&new_bw_state->base);
+- if (ret)
+- return ret;
+- } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
+- ret = intel_atomic_lock_global_state(&new_bw_state->base);
+- if (ret)
+- return ret;
+- }
+-
+ return 0;
+ }
+
+@@ -4708,6 +4707,10 @@ static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
+ };
+
+ static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
++ /*
++ * Keep the join_mbus cases first so check_mbus_joined()
++ * will prefer them over the !join_mbus cases.
++ */
+ {
+ .active_pipes = BIT(PIPE_A),
+ .dbuf_mask = {
+@@ -4722,6 +4725,20 @@ static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
+ },
+ .join_mbus = true,
+ },
++ {
++ .active_pipes = BIT(PIPE_A),
++ .dbuf_mask = {
++ [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
++ },
++ .join_mbus = false,
++ },
++ {
++ .active_pipes = BIT(PIPE_B),
++ .dbuf_mask = {
++ [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
++ },
++ .join_mbus = false,
++ },
+ {
+ .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
+ .dbuf_mask = {
+@@ -4826,7 +4843,7 @@ static bool check_mbus_joined(u8 active_pipes,
+ {
+ int i;
+
+- for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
++ for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
+ if (dbuf_slices[i].active_pipes == active_pipes)
+ return dbuf_slices[i].join_mbus;
+ }
+@@ -4838,13 +4855,14 @@ static bool adlp_check_mbus_joined(u8 active_pipes)
+ return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
+ }
+
+-static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
++static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
+ const struct dbuf_slice_conf_entry *dbuf_slices)
+ {
+ int i;
+
+- for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
+- if (dbuf_slices[i].active_pipes == active_pipes)
++ for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
++ if (dbuf_slices[i].active_pipes == active_pipes &&
++ dbuf_slices[i].join_mbus == join_mbus)
+ return dbuf_slices[i].dbuf_mask[pipe];
+ }
+ return 0;
+@@ -4855,7 +4873,7 @@ static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
+ * returns correspondent DBuf slice mask as stated in BSpec for particular
+ * platform.
+ */
+-static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
++static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+ {
+ /*
+ * FIXME: For ICL this is still a bit unclear as prev BSpec revision
+@@ -4869,37 +4887,41 @@ static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
+ * still here - we will need it once those additional constraints
+ * pop up.
+ */
+- return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
++ return compute_dbuf_slices(pipe, active_pipes, join_mbus,
++ icl_allowed_dbufs);
+ }
+
+-static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
++static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+ {
+- return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
++ return compute_dbuf_slices(pipe, active_pipes, join_mbus,
++ tgl_allowed_dbufs);
+ }
+
+-static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
++static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+ {
+- return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
++ return compute_dbuf_slices(pipe, active_pipes, join_mbus,
++ adlp_allowed_dbufs);
+ }
+
+-static u32 dg2_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
++static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+ {
+- return compute_dbuf_slices(pipe, active_pipes, dg2_allowed_dbufs);
++ return compute_dbuf_slices(pipe, active_pipes, join_mbus,
++ dg2_allowed_dbufs);
+ }
+
+-static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
++static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
+ {
+ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+ enum pipe pipe = crtc->pipe;
+
+ if (IS_DG2(dev_priv))
+- return dg2_compute_dbuf_slices(pipe, active_pipes);
++ return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+ else if (IS_ALDERLAKE_P(dev_priv))
+- return adlp_compute_dbuf_slices(pipe, active_pipes);
++ return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+ else if (DISPLAY_VER(dev_priv) == 12)
+- return tgl_compute_dbuf_slices(pipe, active_pipes);
++ return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+ else if (DISPLAY_VER(dev_priv) == 11)
+- return icl_compute_dbuf_slices(pipe, active_pipes);
++ return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+ /*
+ * For anything else just return one slice yet.
+ * Should be extended for other platforms.
+@@ -5612,7 +5634,7 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
+ result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
+ result->enable = true;
+
+- if (DISPLAY_VER(dev_priv) < 12)
++ if (DISPLAY_VER(dev_priv) < 12 && dev_priv->sagv_block_time_us)
+ result->can_sagv = latency >= dev_priv->sagv_block_time_us;
+ }
+
+@@ -5643,7 +5665,10 @@ static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
+ struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
+ struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
+ struct skl_wm_level *levels = plane_wm->wm;
+- unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
++ unsigned int latency = 0;
++
++ if (dev_priv->sagv_block_time_us)
++ latency = dev_priv->sagv_block_time_us + dev_priv->wm.skl_latency[0];
+
+ skl_compute_plane_wm(crtc_state, 0, latency,
+ wm_params, &levels[0],
+@@ -6110,11 +6135,16 @@ skl_compute_ddb(struct intel_atomic_state *state)
+ return ret;
+ }
+
++ if (IS_ALDERLAKE_P(dev_priv))
++ new_dbuf_state->joined_mbus =
++ adlp_check_mbus_joined(new_dbuf_state->active_pipes);
++
+ for_each_intel_crtc(&dev_priv->drm, crtc) {
+ enum pipe pipe = crtc->pipe;
+
+ new_dbuf_state->slices[pipe] =
+- skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
++ skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
++ new_dbuf_state->joined_mbus);
+
+ if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
+ continue;
+@@ -6126,9 +6156,6 @@ skl_compute_ddb(struct intel_atomic_state *state)
+
+ new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
+
+- if (IS_ALDERLAKE_P(dev_priv))
+- new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);
+-
+ if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
+ old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
+ ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
+@@ -6609,6 +6636,7 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
+ enum pipe pipe = crtc->pipe;
+ unsigned int mbus_offset;
+ enum plane_id plane_id;
++ u8 slices;
+
+ skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
+ crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
+@@ -6628,19 +6656,22 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
+ skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
+ }
+
+- dbuf_state->slices[pipe] =
+- skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
+-
+ dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
+
+ /*
+ * Used for checking overlaps, so we need absolute
+ * offsets instead of MBUS relative offsets.
+ */
+- mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
++ slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
++ dbuf_state->joined_mbus);
++ mbus_offset = mbus_ddb_offset(dev_priv, slices);
+ crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
+ crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
+
++ /* The slices actually used by the planes on the pipe */
++ dbuf_state->slices[pipe] =
++ skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
++
+ drm_dbg_kms(&dev_priv->drm,
+ "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
+ crtc->base.base.id, crtc->base.name,
+@@ -6652,6 +6683,74 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
+ dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
+ }
+
++static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
++{
++ const struct intel_dbuf_state *dbuf_state =
++ to_intel_dbuf_state(i915->dbuf.obj.state);
++ struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
++ struct intel_crtc *crtc;
++
++ for_each_intel_crtc(&i915->drm, crtc) {
++ const struct intel_crtc_state *crtc_state =
++ to_intel_crtc_state(crtc->base.state);
++
++ entries[crtc->pipe] = crtc_state->wm.skl.ddb;
++ }
++
++ for_each_intel_crtc(&i915->drm, crtc) {
++ const struct intel_crtc_state *crtc_state =
++ to_intel_crtc_state(crtc->base.state);
++ u8 slices;
++
++ slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
++ dbuf_state->joined_mbus);
++ if (dbuf_state->slices[crtc->pipe] & ~slices)
++ return true;
++
++ if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
++ I915_MAX_PIPES, crtc->pipe))
++ return true;
++ }
++
++ return false;
++}
++
++void skl_wm_sanitize(struct drm_i915_private *i915)
++{
++ struct intel_crtc *crtc;
++
++ /*
++ * On TGL/RKL (at least) the BIOS likes to assign the planes
++ * to the wrong DBUF slices. This will cause an infinite loop
++ * in skl_commit_modeset_enables() as it can't find a way to
++ * transition between the old bogus DBUF layout to the new
++ * proper DBUF layout without DBUF allocation overlaps between
++ * the planes (which cannot be allowed or else the hardware
++ * may hang). If we detect a bogus DBUF layout just turn off
++ * all the planes so that skl_commit_modeset_enables() can
++ * simply ignore them.
++ */
++ if (!skl_dbuf_is_misconfigured(i915))
++ return;
++
++ drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
++
++ for_each_intel_crtc(&i915->drm, crtc) {
++ struct intel_plane *plane = to_intel_plane(crtc->base.primary);
++ const struct intel_plane_state *plane_state =
++ to_intel_plane_state(plane->base.state);
++ struct intel_crtc_state *crtc_state =
++ to_intel_crtc_state(crtc->base.state);
++
++ if (plane_state->uapi.visible)
++ intel_plane_disable_noatomic(crtc, plane);
++
++ drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
++
++ memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
++ }
++}
++
+ static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
+ {
+ struct drm_device *dev = crtc->base.dev;
+diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
+index 91f23b7f0af2e..79d89fe22d8c8 100644
+--- a/drivers/gpu/drm/i915/intel_pm.h
++++ b/drivers/gpu/drm/i915/intel_pm.h
+@@ -48,6 +48,7 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
+ struct skl_pipe_wm *out);
+ void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
+ void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
++void skl_wm_sanitize(struct drm_i915_private *dev_priv);
+ bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
+ const struct intel_bw_state *bw_state);
+ void intel_sagv_pre_plane_update(struct intel_atomic_state *state);
+diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
+index 6b38bc2811c1b..de8d0558389c4 100644
+--- a/drivers/gpu/drm/i915/intel_uncore.c
++++ b/drivers/gpu/drm/i915/intel_uncore.c
+@@ -718,7 +718,8 @@ void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
+ }
+
+ static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
+- enum forcewake_domains fw_domains)
++ enum forcewake_domains fw_domains,
++ bool delayed)
+ {
+ struct intel_uncore_forcewake_domain *domain;
+ unsigned int tmp;
+@@ -733,7 +734,11 @@ static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
+ continue;
+ }
+
+- uncore->funcs.force_wake_put(uncore, domain->mask);
++ if (delayed &&
++ !(domain->uncore->fw_domains_timer & domain->mask))
++ fw_domain_arm_timer(domain);
++ else
++ uncore->funcs.force_wake_put(uncore, domain->mask);
+ }
+ }
+
+@@ -754,7 +759,20 @@ void intel_uncore_forcewake_put(struct intel_uncore *uncore,
+ return;
+
+ spin_lock_irqsave(&uncore->lock, irqflags);
+- __intel_uncore_forcewake_put(uncore, fw_domains);
++ __intel_uncore_forcewake_put(uncore, fw_domains, false);
++ spin_unlock_irqrestore(&uncore->lock, irqflags);
++}
++
++void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
++ enum forcewake_domains fw_domains)
++{
++ unsigned long irqflags;
++
++ if (!uncore->funcs.force_wake_put)
++ return;
++
++ spin_lock_irqsave(&uncore->lock, irqflags);
++ __intel_uncore_forcewake_put(uncore, fw_domains, true);
+ spin_unlock_irqrestore(&uncore->lock, irqflags);
+ }
+
+@@ -796,7 +814,7 @@ void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
+ if (!uncore->funcs.force_wake_put)
+ return;
+
+- __intel_uncore_forcewake_put(uncore, fw_domains);
++ __intel_uncore_forcewake_put(uncore, fw_domains, false);
+ }
+
+ void assert_forcewakes_inactive(struct intel_uncore *uncore)
+diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
+index 3c0b0a8b5250d..4c63209dcf530 100644
+--- a/drivers/gpu/drm/i915/intel_uncore.h
++++ b/drivers/gpu/drm/i915/intel_uncore.h
+@@ -229,6 +229,8 @@ void intel_uncore_forcewake_get(struct intel_uncore *uncore,
+ enum forcewake_domains domains);
+ void intel_uncore_forcewake_put(struct intel_uncore *uncore,
+ enum forcewake_domains domains);
++void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
++ enum forcewake_domains domains);
+ void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
+ enum forcewake_domains fw_domains);
+
+diff --git a/drivers/gpu/drm/imx/dw_hdmi-imx.c b/drivers/gpu/drm/imx/dw_hdmi-imx.c
+index 87428fb23d9ff..a2277a0d6d06f 100644
+--- a/drivers/gpu/drm/imx/dw_hdmi-imx.c
++++ b/drivers/gpu/drm/imx/dw_hdmi-imx.c
+@@ -222,6 +222,7 @@ static int dw_hdmi_imx_probe(struct platform_device *pdev)
+ struct device_node *np = pdev->dev.of_node;
+ const struct of_device_id *match = of_match_node(dw_hdmi_imx_dt_ids, np);
+ struct imx_hdmi *hdmi;
++ int ret;
+
+ hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
+ if (!hdmi)
+@@ -243,10 +244,15 @@ static int dw_hdmi_imx_probe(struct platform_device *pdev)
+ hdmi->bridge = of_drm_find_bridge(np);
+ if (!hdmi->bridge) {
+ dev_err(hdmi->dev, "Unable to find bridge\n");
++ dw_hdmi_remove(hdmi->hdmi);
+ return -ENODEV;
+ }
+
+- return component_add(&pdev->dev, &dw_hdmi_imx_ops);
++ ret = component_add(&pdev->dev, &dw_hdmi_imx_ops);
++ if (ret)
++ dw_hdmi_remove(hdmi->hdmi);
++
++ return ret;
+ }
+
+ static int dw_hdmi_imx_remove(struct platform_device *pdev)
+diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
+index 9558e9e1b431b..cb685fe2039b4 100644
+--- a/drivers/gpu/drm/imx/imx-drm-core.c
++++ b/drivers/gpu/drm/imx/imx-drm-core.c
+@@ -81,7 +81,6 @@ static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
+ struct drm_plane_state *old_plane_state, *new_plane_state;
+ bool plane_disabling = false;
+ int i;
+- bool fence_cookie = dma_fence_begin_signalling();
+
+ drm_atomic_helper_commit_modeset_disables(dev, state);
+
+@@ -112,7 +111,6 @@ static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
+ }
+
+ drm_atomic_helper_commit_hw_done(state);
+- dma_fence_end_signalling(fence_cookie);
+ }
+
+ static const struct drm_mode_config_helper_funcs imx_drm_mode_config_helpers = {
+diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
+index e5078d03020d9..fb0e951248f68 100644
+--- a/drivers/gpu/drm/imx/imx-ldb.c
++++ b/drivers/gpu/drm/imx/imx-ldb.c
+@@ -572,6 +572,8 @@ static int imx_ldb_panel_ddc(struct device *dev,
+ edidp = of_get_property(child, "edid", &edid_len);
+ if (edidp) {
+ channel->edid = kmemdup(edidp, edid_len, GFP_KERNEL);
++ if (!channel->edid)
++ return -ENOMEM;
+ } else if (!channel->panel) {
+ /* fallback to display-timings node */
+ ret = of_get_drm_display_mode(child,
+diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
+index a8aba0141ce71..63ba2ad846791 100644
+--- a/drivers/gpu/drm/imx/parallel-display.c
++++ b/drivers/gpu/drm/imx/parallel-display.c
+@@ -75,8 +75,10 @@ static int imx_pd_connector_get_modes(struct drm_connector *connector)
+ ret = of_get_drm_display_mode(np, &imxpd->mode,
+ &imxpd->bus_flags,
+ OF_USE_NATIVE_MODE);
+- if (ret)
++ if (ret) {
++ drm_mode_destroy(connector->dev, mode);
+ return ret;
++ }
+
+ drm_mode_copy(mode, &imxpd->mode);
+ mode->type |= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+@@ -217,14 +219,6 @@ static int imx_pd_bridge_atomic_check(struct drm_bridge *bridge,
+ if (!imx_pd_format_supported(bus_fmt))
+ return -EINVAL;
+
+- if (bus_flags &
+- ~(DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_DE_HIGH |
+- DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE |
+- DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)) {
+- dev_warn(imxpd->dev, "invalid bus_flags (%x)\n", bus_flags);
+- return -EINVAL;
+- }
+-
+ bridge_state->output_bus_cfg.flags = bus_flags;
+ bridge_state->input_bus_cfg.flags = bus_flags;
+ imx_crtc_state->bus_flags = bus_flags;
+diff --git a/drivers/gpu/drm/kmb/kmb_plane.c b/drivers/gpu/drm/kmb/kmb_plane.c
+index 00404ba4126dd..2735b8eb35376 100644
+--- a/drivers/gpu/drm/kmb/kmb_plane.c
++++ b/drivers/gpu/drm/kmb/kmb_plane.c
+@@ -158,12 +158,6 @@ static void kmb_plane_atomic_disable(struct drm_plane *plane,
+ case LAYER_1:
+ kmb->plane_status[plane_id].ctrl = LCD_CTRL_VL2_ENABLE;
+ break;
+- case LAYER_2:
+- kmb->plane_status[plane_id].ctrl = LCD_CTRL_GL1_ENABLE;
+- break;
+- case LAYER_3:
+- kmb->plane_status[plane_id].ctrl = LCD_CTRL_GL2_ENABLE;
+- break;
+ }
+
+ kmb->plane_status[plane_id].disable = true;
+diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c
+index 65fdca366e41f..36c9905894278 100644
+--- a/drivers/gpu/drm/lima/lima_device.c
++++ b/drivers/gpu/drm/lima/lima_device.c
+@@ -357,6 +357,7 @@ int lima_device_init(struct lima_device *ldev)
+ int err, i;
+
+ dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
++ dma_set_max_seg_size(ldev->dev, UINT_MAX);
+
+ err = lima_clk_init(ldev);
+ if (err)
+diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
+index 93b40c245f007..5d90d2eb00193 100644
+--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
++++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
+@@ -11,6 +11,7 @@
+ #include <linux/of_platform.h>
+ #include <linux/phy/phy.h>
+ #include <linux/platform_device.h>
++#include <linux/reset.h>
+
+ #include <video/mipi_display.h>
+ #include <video/videomode.h>
+@@ -980,8 +981,10 @@ static int mtk_dsi_bind(struct device *dev, struct device *master, void *data)
+ struct mtk_dsi *dsi = dev_get_drvdata(dev);
+
+ ret = mtk_dsi_encoder_init(drm, dsi);
++ if (ret)
++ return ret;
+
+- return ret;
++ return device_reset_optional(dev);
+ }
+
+ static void mtk_dsi_unbind(struct device *dev, struct device *master,
+diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
+index 5838c44cbf6f0..3196189429bcf 100644
+--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
++++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
+@@ -1224,12 +1224,14 @@ static int mtk_hdmi_bridge_mode_valid(struct drm_bridge *bridge,
+ return MODE_BAD;
+ }
+
+- if (hdmi->conf->cea_modes_only && !drm_match_cea_mode(mode))
+- return MODE_BAD;
++ if (hdmi->conf) {
++ if (hdmi->conf->cea_modes_only && !drm_match_cea_mode(mode))
++ return MODE_BAD;
+
+- if (hdmi->conf->max_mode_clock &&
+- mode->clock > hdmi->conf->max_mode_clock)
+- return MODE_CLOCK_HIGH;
++ if (hdmi->conf->max_mode_clock &&
++ mode->clock > hdmi->conf->max_mode_clock)
++ return MODE_CLOCK_HIGH;
++ }
+
+ if (mode->clock < 27000)
+ return MODE_CLOCK_LOW;
+diff --git a/drivers/gpu/drm/meson/Makefile b/drivers/gpu/drm/meson/Makefile
+index 28a519cdf66b8..523fce45f16ba 100644
+--- a/drivers/gpu/drm/meson/Makefile
++++ b/drivers/gpu/drm/meson/Makefile
+@@ -2,6 +2,7 @@
+ meson-drm-y := meson_drv.o meson_plane.o meson_crtc.o meson_venc_cvbs.o
+ meson-drm-y += meson_viu.o meson_vpp.o meson_venc.o meson_vclk.o meson_overlay.o
+ meson-drm-y += meson_rdma.o meson_osd_afbcd.o
++meson-drm-y += meson_encoder_hdmi.o
+
+ obj-$(CONFIG_DRM_MESON) += meson-drm.o
+ obj-$(CONFIG_DRM_MESON_DW_HDMI) += meson_dw_hdmi.o
+diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
+index bc0d60df04ae4..c98525d60df5b 100644
+--- a/drivers/gpu/drm/meson/meson_drv.c
++++ b/drivers/gpu/drm/meson/meson_drv.c
+@@ -32,6 +32,7 @@
+ #include "meson_osd_afbcd.h"
+ #include "meson_registers.h"
+ #include "meson_venc_cvbs.h"
++#include "meson_encoder_hdmi.h"
+ #include "meson_viu.h"
+ #include "meson_vpp.h"
+ #include "meson_rdma.h"
+@@ -206,8 +207,7 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
+ priv->compat = match->compat;
+ priv->afbcd.ops = match->afbcd_ops;
+
+- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vpu");
+- regs = devm_ioremap_resource(dev, res);
++ regs = devm_platform_ioremap_resource_byname(pdev, "vpu");
+ if (IS_ERR(regs)) {
+ ret = PTR_ERR(regs);
+ goto free_drm;
+@@ -302,38 +302,42 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
+ if (priv->afbcd.ops) {
+ ret = priv->afbcd.ops->init(priv);
+ if (ret)
+- return ret;
++ goto free_drm;
+ }
+
+ /* Encoder Initialization */
+
+ ret = meson_venc_cvbs_create(priv);
+ if (ret)
+- goto free_drm;
++ goto exit_afbcd;
+
+ if (has_components) {
+ ret = component_bind_all(drm->dev, drm);
+ if (ret) {
+ dev_err(drm->dev, "Couldn't bind all components\n");
+- goto free_drm;
++ goto exit_afbcd;
+ }
+ }
+
++ ret = meson_encoder_hdmi_init(priv);
++ if (ret)
++ goto exit_afbcd;
++
+ ret = meson_plane_create(priv);
+ if (ret)
+- goto free_drm;
++ goto exit_afbcd;
+
+ ret = meson_overlay_create(priv);
+ if (ret)
+- goto free_drm;
++ goto exit_afbcd;
+
+ ret = meson_crtc_create(priv);
+ if (ret)
+- goto free_drm;
++ goto exit_afbcd;
+
+ ret = request_irq(priv->vsync_irq, meson_irq, 0, drm->driver->name, drm);
+ if (ret)
+- goto free_drm;
++ goto exit_afbcd;
+
+ drm_mode_config_reset(drm);
+
+@@ -351,6 +355,9 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
+
+ uninstall_irq:
+ free_irq(priv->vsync_irq, drm);
++exit_afbcd:
++ if (priv->afbcd.ops)
++ priv->afbcd.ops->exit(priv);
+ free_drm:
+ drm_dev_put(drm);
+
+@@ -381,10 +388,8 @@ static void meson_drv_unbind(struct device *dev)
+ free_irq(priv->vsync_irq, drm);
+ drm_dev_put(drm);
+
+- if (priv->afbcd.ops) {
+- priv->afbcd.ops->reset(priv);
+- meson_rdma_free(priv);
+- }
++ if (priv->afbcd.ops)
++ priv->afbcd.ops->exit(priv);
+ }
+
+ static const struct component_master_ops meson_drv_master_ops = {
+diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c
+index 2ed87cfdd7353..fb540a503efed 100644
+--- a/drivers/gpu/drm/meson/meson_dw_hdmi.c
++++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c
+@@ -22,14 +22,11 @@
+ #include <drm/drm_probe_helper.h>
+ #include <drm/drm_print.h>
+
+-#include <linux/media-bus-format.h>
+ #include <linux/videodev2.h>
+
+ #include "meson_drv.h"
+ #include "meson_dw_hdmi.h"
+ #include "meson_registers.h"
+-#include "meson_vclk.h"
+-#include "meson_venc.h"
+
+ #define DRIVER_NAME "meson-dw-hdmi"
+ #define DRIVER_DESC "Amlogic Meson HDMI-TX DRM driver"
+@@ -135,8 +132,6 @@ struct meson_dw_hdmi_data {
+ };
+
+ struct meson_dw_hdmi {
+- struct drm_encoder encoder;
+- struct drm_bridge bridge;
+ struct dw_hdmi_plat_data dw_plat_data;
+ struct meson_drm *priv;
+ struct device *dev;
+@@ -148,12 +143,8 @@ struct meson_dw_hdmi {
+ struct regulator *hdmi_supply;
+ u32 irq_stat;
+ struct dw_hdmi *hdmi;
+- unsigned long output_bus_fmt;
++ struct drm_bridge *bridge;
+ };
+-#define encoder_to_meson_dw_hdmi(x) \
+- container_of(x, struct meson_dw_hdmi, encoder)
+-#define bridge_to_meson_dw_hdmi(x) \
+- container_of(x, struct meson_dw_hdmi, bridge)
+
+ static inline int dw_hdmi_is_compatible(struct meson_dw_hdmi *dw_hdmi,
+ const char *compat)
+@@ -295,14 +286,14 @@ static inline void dw_hdmi_dwc_write_bits(struct meson_dw_hdmi *dw_hdmi,
+
+ /* Setup PHY bandwidth modes */
+ static void meson_hdmi_phy_setup_mode(struct meson_dw_hdmi *dw_hdmi,
+- const struct drm_display_mode *mode)
++ const struct drm_display_mode *mode,
++ bool mode_is_420)
+ {
+ struct meson_drm *priv = dw_hdmi->priv;
+ unsigned int pixel_clock = mode->clock;
+
+ /* For 420, pixel clock is half unlike venc clock */
+- if (dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
+- pixel_clock /= 2;
++ if (mode_is_420) pixel_clock /= 2;
+
+ if (dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxl-dw-hdmi") ||
+ dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxm-dw-hdmi")) {
+@@ -374,68 +365,25 @@ static inline void meson_dw_hdmi_phy_reset(struct meson_dw_hdmi *dw_hdmi)
+ mdelay(2);
+ }
+
+-static void dw_hdmi_set_vclk(struct meson_dw_hdmi *dw_hdmi,
+- const struct drm_display_mode *mode)
+-{
+- struct meson_drm *priv = dw_hdmi->priv;
+- int vic = drm_match_cea_mode(mode);
+- unsigned int phy_freq;
+- unsigned int vclk_freq;
+- unsigned int venc_freq;
+- unsigned int hdmi_freq;
+-
+- vclk_freq = mode->clock;
+-
+- /* For 420, pixel clock is half unlike venc clock */
+- if (dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
+- vclk_freq /= 2;
+-
+- /* TMDS clock is pixel_clock * 10 */
+- phy_freq = vclk_freq * 10;
+-
+- if (!vic) {
+- meson_vclk_setup(priv, MESON_VCLK_TARGET_DMT, phy_freq,
+- vclk_freq, vclk_freq, vclk_freq, false);
+- return;
+- }
+-
+- /* 480i/576i needs global pixel doubling */
+- if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+- vclk_freq *= 2;
+-
+- venc_freq = vclk_freq;
+- hdmi_freq = vclk_freq;
+-
+- /* VENC double pixels for 1080i, 720p and YUV420 modes */
+- if (meson_venc_hdmi_venc_repeat(vic) ||
+- dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
+- venc_freq *= 2;
+-
+- vclk_freq = max(venc_freq, hdmi_freq);
+-
+- if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+- venc_freq /= 2;
+-
+- DRM_DEBUG_DRIVER("vclk:%d phy=%d venc=%d hdmi=%d enci=%d\n",
+- phy_freq, vclk_freq, venc_freq, hdmi_freq,
+- priv->venc.hdmi_use_enci);
+-
+- meson_vclk_setup(priv, MESON_VCLK_TARGET_HDMI, phy_freq, vclk_freq,
+- venc_freq, hdmi_freq, priv->venc.hdmi_use_enci);
+-}
+-
+ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
+ const struct drm_display_info *display,
+ const struct drm_display_mode *mode)
+ {
+ struct meson_dw_hdmi *dw_hdmi = (struct meson_dw_hdmi *)data;
++ bool is_hdmi2_sink = display->hdmi.scdc.supported;
+ struct meson_drm *priv = dw_hdmi->priv;
+ unsigned int wr_clk =
+ readl_relaxed(priv->io_base + _REG(VPU_HDMI_SETTING));
++ bool mode_is_420 = false;
+
+ DRM_DEBUG_DRIVER("\"%s\" div%d\n", mode->name,
+ mode->clock > 340000 ? 40 : 10);
+
++ if (drm_mode_is_420_only(display, mode) ||
++ (!is_hdmi2_sink &&
++ drm_mode_is_420_also(display, mode)))
++ mode_is_420 = true;
++
+ /* Enable clocks */
+ regmap_update_bits(priv->hhi, HHI_HDMI_CLK_CNTL, 0xffff, 0x100);
+
+@@ -457,8 +405,7 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
+ dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12));
+
+ /* TMDS pattern setup */
+- if (mode->clock > 340000 &&
+- dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_YUV8_1X24) {
++ if (mode->clock > 340000 && !mode_is_420) {
+ dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01,
+ 0);
+ dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_23,
+@@ -476,7 +423,7 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
+ dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_CNTL, 0x2);
+
+ /* Setup PHY parameters */
+- meson_hdmi_phy_setup_mode(dw_hdmi, mode);
++ meson_hdmi_phy_setup_mode(dw_hdmi, mode, mode_is_420);
+
+ /* Setup PHY */
+ regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1,
+@@ -622,214 +569,15 @@ static irqreturn_t dw_hdmi_top_thread_irq(int irq, void *dev_id)
+ dw_hdmi_setup_rx_sense(dw_hdmi->hdmi, hpd_connected,
+ hpd_connected);
+
+- drm_helper_hpd_irq_event(dw_hdmi->encoder.dev);
++ drm_helper_hpd_irq_event(dw_hdmi->bridge->dev);
++ drm_bridge_hpd_notify(dw_hdmi->bridge,
++ hpd_connected ? connector_status_connected
++ : connector_status_disconnected);
+ }
+
+ return IRQ_HANDLED;
+ }
+
+-static enum drm_mode_status
+-dw_hdmi_mode_valid(struct dw_hdmi *hdmi, void *data,
+- const struct drm_display_info *display_info,
+- const struct drm_display_mode *mode)
+-{
+- struct meson_dw_hdmi *dw_hdmi = data;
+- struct meson_drm *priv = dw_hdmi->priv;
+- bool is_hdmi2_sink = display_info->hdmi.scdc.supported;
+- unsigned int phy_freq;
+- unsigned int vclk_freq;
+- unsigned int venc_freq;
+- unsigned int hdmi_freq;
+- int vic = drm_match_cea_mode(mode);
+- enum drm_mode_status status;
+-
+- DRM_DEBUG_DRIVER("Modeline " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
+-
+- /* If sink does not support 540MHz, reject the non-420 HDMI2 modes */
+- if (display_info->max_tmds_clock &&
+- mode->clock > display_info->max_tmds_clock &&
+- !drm_mode_is_420_only(display_info, mode) &&
+- !drm_mode_is_420_also(display_info, mode))
+- return MODE_BAD;
+-
+- /* Check against non-VIC supported modes */
+- if (!vic) {
+- status = meson_venc_hdmi_supported_mode(mode);
+- if (status != MODE_OK)
+- return status;
+-
+- return meson_vclk_dmt_supported_freq(priv, mode->clock);
+- /* Check against supported VIC modes */
+- } else if (!meson_venc_hdmi_supported_vic(vic))
+- return MODE_BAD;
+-
+- vclk_freq = mode->clock;
+-
+- /* For 420, pixel clock is half unlike venc clock */
+- if (drm_mode_is_420_only(display_info, mode) ||
+- (!is_hdmi2_sink &&
+- drm_mode_is_420_also(display_info, mode)))
+- vclk_freq /= 2;
+-
+- /* TMDS clock is pixel_clock * 10 */
+- phy_freq = vclk_freq * 10;
+-
+- /* 480i/576i needs global pixel doubling */
+- if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+- vclk_freq *= 2;
+-
+- venc_freq = vclk_freq;
+- hdmi_freq = vclk_freq;
+-
+- /* VENC double pixels for 1080i, 720p and YUV420 modes */
+- if (meson_venc_hdmi_venc_repeat(vic) ||
+- drm_mode_is_420_only(display_info, mode) ||
+- (!is_hdmi2_sink &&
+- drm_mode_is_420_also(display_info, mode)))
+- venc_freq *= 2;
+-
+- vclk_freq = max(venc_freq, hdmi_freq);
+-
+- if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+- venc_freq /= 2;
+-
+- dev_dbg(dw_hdmi->dev, "%s: vclk:%d phy=%d venc=%d hdmi=%d\n",
+- __func__, phy_freq, vclk_freq, venc_freq, hdmi_freq);
+-
+- return meson_vclk_vic_supported_freq(priv, phy_freq, vclk_freq);
+-}
+-
+-/* Encoder */
+-
+-static const u32 meson_dw_hdmi_out_bus_fmts[] = {
+- MEDIA_BUS_FMT_YUV8_1X24,
+- MEDIA_BUS_FMT_UYYVYY8_0_5X24,
+-};
+-
+-static void meson_venc_hdmi_encoder_destroy(struct drm_encoder *encoder)
+-{
+- drm_encoder_cleanup(encoder);
+-}
+-
+-static const struct drm_encoder_funcs meson_venc_hdmi_encoder_funcs = {
+- .destroy = meson_venc_hdmi_encoder_destroy,
+-};
+-
+-static u32 *
+-meson_venc_hdmi_encoder_get_inp_bus_fmts(struct drm_bridge *bridge,
+- struct drm_bridge_state *bridge_state,
+- struct drm_crtc_state *crtc_state,
+- struct drm_connector_state *conn_state,
+- u32 output_fmt,
+- unsigned int *num_input_fmts)
+-{
+- u32 *input_fmts = NULL;
+- int i;
+-
+- *num_input_fmts = 0;
+-
+- for (i = 0 ; i < ARRAY_SIZE(meson_dw_hdmi_out_bus_fmts) ; ++i) {
+- if (output_fmt == meson_dw_hdmi_out_bus_fmts[i]) {
+- *num_input_fmts = 1;
+- input_fmts = kcalloc(*num_input_fmts,
+- sizeof(*input_fmts),
+- GFP_KERNEL);
+- if (!input_fmts)
+- return NULL;
+-
+- input_fmts[0] = output_fmt;
+-
+- break;
+- }
+- }
+-
+- return input_fmts;
+-}
+-
+-static int meson_venc_hdmi_encoder_atomic_check(struct drm_bridge *bridge,
+- struct drm_bridge_state *bridge_state,
+- struct drm_crtc_state *crtc_state,
+- struct drm_connector_state *conn_state)
+-{
+- struct meson_dw_hdmi *dw_hdmi = bridge_to_meson_dw_hdmi(bridge);
+-
+- dw_hdmi->output_bus_fmt = bridge_state->output_bus_cfg.format;
+-
+- DRM_DEBUG_DRIVER("output_bus_fmt %lx\n", dw_hdmi->output_bus_fmt);
+-
+- return 0;
+-}
+-
+-static void meson_venc_hdmi_encoder_disable(struct drm_bridge *bridge)
+-{
+- struct meson_dw_hdmi *dw_hdmi = bridge_to_meson_dw_hdmi(bridge);
+- struct meson_drm *priv = dw_hdmi->priv;
+-
+- DRM_DEBUG_DRIVER("\n");
+-
+- writel_bits_relaxed(0x3, 0,
+- priv->io_base + _REG(VPU_HDMI_SETTING));
+-
+- writel_relaxed(0, priv->io_base + _REG(ENCI_VIDEO_EN));
+- writel_relaxed(0, priv->io_base + _REG(ENCP_VIDEO_EN));
+-}
+-
+-static void meson_venc_hdmi_encoder_enable(struct drm_bridge *bridge)
+-{
+- struct meson_dw_hdmi *dw_hdmi = bridge_to_meson_dw_hdmi(bridge);
+- struct meson_drm *priv = dw_hdmi->priv;
+-
+- DRM_DEBUG_DRIVER("%s\n", priv->venc.hdmi_use_enci ? "VENCI" : "VENCP");
+-
+- if (priv->venc.hdmi_use_enci)
+- writel_relaxed(1, priv->io_base + _REG(ENCI_VIDEO_EN));
+- else
+- writel_relaxed(1, priv->io_base + _REG(ENCP_VIDEO_EN));
+-}
+-
+-static void meson_venc_hdmi_encoder_mode_set(struct drm_bridge *bridge,
+- const struct drm_display_mode *mode,
+- const struct drm_display_mode *adjusted_mode)
+-{
+- struct meson_dw_hdmi *dw_hdmi = bridge_to_meson_dw_hdmi(bridge);
+- struct meson_drm *priv = dw_hdmi->priv;
+- int vic = drm_match_cea_mode(mode);
+- unsigned int ycrcb_map = VPU_HDMI_OUTPUT_CBYCR;
+- bool yuv420_mode = false;
+-
+- DRM_DEBUG_DRIVER("\"%s\" vic %d\n", mode->name, vic);
+-
+- if (dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24) {
+- ycrcb_map = VPU_HDMI_OUTPUT_CRYCB;
+- yuv420_mode = true;
+- }
+-
+- /* VENC + VENC-DVI Mode setup */
+- meson_venc_hdmi_mode_set(priv, vic, ycrcb_map, yuv420_mode, mode);
+-
+- /* VCLK Set clock */
+- dw_hdmi_set_vclk(dw_hdmi, mode);
+-
+- if (dw_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
+- /* Setup YUV420 to HDMI-TX, no 10bit diphering */
+- writel_relaxed(2 | (2 << 2),
+- priv->io_base + _REG(VPU_HDMI_FMT_CTRL));
+- else
+- /* Setup YUV444 to HDMI-TX, no 10bit diphering */
+- writel_relaxed(0, priv->io_base + _REG(VPU_HDMI_FMT_CTRL));
+-}
+-
+-static const struct drm_bridge_funcs meson_venc_hdmi_encoder_bridge_funcs = {
+- .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+- .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+- .atomic_get_input_bus_fmts = meson_venc_hdmi_encoder_get_inp_bus_fmts,
+- .atomic_reset = drm_atomic_helper_bridge_reset,
+- .atomic_check = meson_venc_hdmi_encoder_atomic_check,
+- .enable = meson_venc_hdmi_encoder_enable,
+- .disable = meson_venc_hdmi_encoder_disable,
+- .mode_set = meson_venc_hdmi_encoder_mode_set,
+-};
+-
+ /* DW HDMI Regmap */
+
+ static int meson_dw_hdmi_reg_read(void *context, unsigned int reg,
+@@ -876,28 +624,6 @@ static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = {
+ .dwc_write = dw_hdmi_g12a_dwc_write,
+ };
+
+-static bool meson_hdmi_connector_is_available(struct device *dev)
+-{
+- struct device_node *ep, *remote;
+-
+- /* HDMI Connector is on the second port, first endpoint */
+- ep = of_graph_get_endpoint_by_regs(dev->of_node, 1, 0);
+- if (!ep)
+- return false;
+-
+- /* If the endpoint node exists, consider it enabled */
+- remote = of_graph_get_remote_port(ep);
+- if (remote) {
+- of_node_put(ep);
+- return true;
+- }
+-
+- of_node_put(ep);
+- of_node_put(remote);
+-
+- return false;
+-}
+-
+ static void meson_dw_hdmi_init(struct meson_dw_hdmi *meson_dw_hdmi)
+ {
+ struct meson_drm *priv = meson_dw_hdmi->priv;
+@@ -976,19 +702,11 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
+ struct drm_device *drm = data;
+ struct meson_drm *priv = drm->dev_private;
+ struct dw_hdmi_plat_data *dw_plat_data;
+- struct drm_bridge *next_bridge;
+- struct drm_encoder *encoder;
+- struct resource *res;
+ int irq;
+ int ret;
+
+ DRM_DEBUG_DRIVER("\n");
+
+- if (!meson_hdmi_connector_is_available(dev)) {
+- dev_info(drm->dev, "HDMI Output connector not available\n");
+- return -ENODEV;
+- }
+-
+ match = of_device_get_match_data(&pdev->dev);
+ if (!match) {
+ dev_err(&pdev->dev, "failed to get match data\n");
+@@ -1004,7 +722,6 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
+ meson_dw_hdmi->dev = dev;
+ meson_dw_hdmi->data = match;
+ dw_plat_data = &meson_dw_hdmi->dw_plat_data;
+- encoder = &meson_dw_hdmi->encoder;
+
+ meson_dw_hdmi->hdmi_supply = devm_regulator_get_optional(dev, "hdmi");
+ if (IS_ERR(meson_dw_hdmi->hdmi_supply)) {
+@@ -1042,8 +759,7 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
+ return PTR_ERR(meson_dw_hdmi->hdmitx_phy);
+ }
+
+- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+- meson_dw_hdmi->hdmitx = devm_ioremap_resource(dev, res);
++ meson_dw_hdmi->hdmitx = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(meson_dw_hdmi->hdmitx))
+ return PTR_ERR(meson_dw_hdmi->hdmitx);
+
+@@ -1076,28 +792,11 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
+ return ret;
+ }
+
+- /* Encoder */
+-
+- ret = drm_encoder_init(drm, encoder, &meson_venc_hdmi_encoder_funcs,
+- DRM_MODE_ENCODER_TMDS, "meson_hdmi");
+- if (ret) {
+- dev_err(priv->dev, "Failed to init HDMI encoder\n");
+- return ret;
+- }
+-
+- meson_dw_hdmi->bridge.funcs = &meson_venc_hdmi_encoder_bridge_funcs;
+- drm_bridge_attach(encoder, &meson_dw_hdmi->bridge, NULL, 0);
+-
+- encoder->possible_crtcs = BIT(0);
+-
+ meson_dw_hdmi_init(meson_dw_hdmi);
+
+- DRM_DEBUG_DRIVER("encoder initialized\n");
+-
+ /* Bridge / Connector */
+
+ dw_plat_data->priv_data = meson_dw_hdmi;
+- dw_plat_data->mode_valid = dw_hdmi_mode_valid;
+ dw_plat_data->phy_ops = &meson_dw_hdmi_phy_ops;
+ dw_plat_data->phy_name = "meson_dw_hdmi_phy";
+ dw_plat_data->phy_data = meson_dw_hdmi;
+@@ -1112,15 +811,11 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
+
+ platform_set_drvdata(pdev, meson_dw_hdmi);
+
+- meson_dw_hdmi->hdmi = dw_hdmi_probe(pdev,
+- &meson_dw_hdmi->dw_plat_data);
++ meson_dw_hdmi->hdmi = dw_hdmi_probe(pdev, &meson_dw_hdmi->dw_plat_data);
+ if (IS_ERR(meson_dw_hdmi->hdmi))
+ return PTR_ERR(meson_dw_hdmi->hdmi);
+
+- next_bridge = of_drm_find_bridge(pdev->dev.of_node);
+- if (next_bridge)
+- drm_bridge_attach(encoder, next_bridge,
+- &meson_dw_hdmi->bridge, 0);
++ meson_dw_hdmi->bridge = of_drm_find_bridge(pdev->dev.of_node);
+
+ DRM_DEBUG_DRIVER("HDMI controller initialized\n");
+
+diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+new file mode 100644
+index 0000000000000..db332fa4cd548
+--- /dev/null
++++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+@@ -0,0 +1,370 @@
++// SPDX-License-Identifier: GPL-2.0-or-later
++/*
++ * Copyright (C) 2016 BayLibre, SAS
++ * Author: Neil Armstrong <narmstrong@baylibre.com>
++ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
++ */
++
++#include <linux/clk.h>
++#include <linux/component.h>
++#include <linux/kernel.h>
++#include <linux/module.h>
++#include <linux/of_device.h>
++#include <linux/of_graph.h>
++#include <linux/regulator/consumer.h>
++#include <linux/reset.h>
++
++#include <drm/drm_atomic_helper.h>
++#include <drm/drm_bridge.h>
++#include <drm/drm_device.h>
++#include <drm/drm_edid.h>
++#include <drm/drm_probe_helper.h>
++#include <drm/drm_simple_kms_helper.h>
++
++#include <linux/media-bus-format.h>
++#include <linux/videodev2.h>
++
++#include "meson_drv.h"
++#include "meson_registers.h"
++#include "meson_vclk.h"
++#include "meson_venc.h"
++#include "meson_encoder_hdmi.h"
++
++struct meson_encoder_hdmi {
++ struct drm_encoder encoder;
++ struct drm_bridge bridge;
++ struct drm_bridge *next_bridge;
++ struct meson_drm *priv;
++ unsigned long output_bus_fmt;
++};
++
++#define bridge_to_meson_encoder_hdmi(x) \
++ container_of(x, struct meson_encoder_hdmi, bridge)
++
++static int meson_encoder_hdmi_attach(struct drm_bridge *bridge,
++ enum drm_bridge_attach_flags flags)
++{
++ struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
++
++ return drm_bridge_attach(bridge->encoder, encoder_hdmi->next_bridge,
++ &encoder_hdmi->bridge, flags);
++}
++
++static void meson_encoder_hdmi_set_vclk(struct meson_encoder_hdmi *encoder_hdmi,
++ const struct drm_display_mode *mode)
++{
++ struct meson_drm *priv = encoder_hdmi->priv;
++ int vic = drm_match_cea_mode(mode);
++ unsigned int phy_freq;
++ unsigned int vclk_freq;
++ unsigned int venc_freq;
++ unsigned int hdmi_freq;
++
++ vclk_freq = mode->clock;
++
++ /* For 420, pixel clock is half unlike venc clock */
++ if (encoder_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
++ vclk_freq /= 2;
++
++ /* TMDS clock is pixel_clock * 10 */
++ phy_freq = vclk_freq * 10;
++
++ if (!vic) {
++ meson_vclk_setup(priv, MESON_VCLK_TARGET_DMT, phy_freq,
++ vclk_freq, vclk_freq, vclk_freq, false);
++ return;
++ }
++
++ /* 480i/576i needs global pixel doubling */
++ if (mode->flags & DRM_MODE_FLAG_DBLCLK)
++ vclk_freq *= 2;
++
++ venc_freq = vclk_freq;
++ hdmi_freq = vclk_freq;
++
++ /* VENC double pixels for 1080i, 720p and YUV420 modes */
++ if (meson_venc_hdmi_venc_repeat(vic) ||
++ encoder_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
++ venc_freq *= 2;
++
++ vclk_freq = max(venc_freq, hdmi_freq);
++
++ if (mode->flags & DRM_MODE_FLAG_DBLCLK)
++ venc_freq /= 2;
++
++ dev_dbg(priv->dev, "vclk:%d phy=%d venc=%d hdmi=%d enci=%d\n",
++ phy_freq, vclk_freq, venc_freq, hdmi_freq,
++ priv->venc.hdmi_use_enci);
++
++ meson_vclk_setup(priv, MESON_VCLK_TARGET_HDMI, phy_freq, vclk_freq,
++ venc_freq, hdmi_freq, priv->venc.hdmi_use_enci);
++}
++
++static enum drm_mode_status meson_encoder_hdmi_mode_valid(struct drm_bridge *bridge,
++ const struct drm_display_info *display_info,
++ const struct drm_display_mode *mode)
++{
++ struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
++ struct meson_drm *priv = encoder_hdmi->priv;
++ bool is_hdmi2_sink = display_info->hdmi.scdc.supported;
++ unsigned int phy_freq;
++ unsigned int vclk_freq;
++ unsigned int venc_freq;
++ unsigned int hdmi_freq;
++ int vic = drm_match_cea_mode(mode);
++ enum drm_mode_status status;
++
++ dev_dbg(priv->dev, "Modeline " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
++
++ /* If sink does not support 540MHz, reject the non-420 HDMI2 modes */
++ if (display_info->max_tmds_clock &&
++ mode->clock > display_info->max_tmds_clock &&
++ !drm_mode_is_420_only(display_info, mode) &&
++ !drm_mode_is_420_also(display_info, mode))
++ return MODE_BAD;
++
++ /* Check against non-VIC supported modes */
++ if (!vic) {
++ status = meson_venc_hdmi_supported_mode(mode);
++ if (status != MODE_OK)
++ return status;
++
++ return meson_vclk_dmt_supported_freq(priv, mode->clock);
++ /* Check against supported VIC modes */
++ } else if (!meson_venc_hdmi_supported_vic(vic))
++ return MODE_BAD;
++
++ vclk_freq = mode->clock;
++
++ /* For 420, pixel clock is half unlike venc clock */
++ if (drm_mode_is_420_only(display_info, mode) ||
++ (!is_hdmi2_sink &&
++ drm_mode_is_420_also(display_info, mode)))
++ vclk_freq /= 2;
++
++ /* TMDS clock is pixel_clock * 10 */
++ phy_freq = vclk_freq * 10;
++
++ /* 480i/576i needs global pixel doubling */
++ if (mode->flags & DRM_MODE_FLAG_DBLCLK)
++ vclk_freq *= 2;
++
++ venc_freq = vclk_freq;
++ hdmi_freq = vclk_freq;
++
++ /* VENC double pixels for 1080i, 720p and YUV420 modes */
++ if (meson_venc_hdmi_venc_repeat(vic) ||
++ drm_mode_is_420_only(display_info, mode) ||
++ (!is_hdmi2_sink &&
++ drm_mode_is_420_also(display_info, mode)))
++ venc_freq *= 2;
++
++ vclk_freq = max(venc_freq, hdmi_freq);
++
++ if (mode->flags & DRM_MODE_FLAG_DBLCLK)
++ venc_freq /= 2;
++
++ dev_dbg(priv->dev, "%s: vclk:%d phy=%d venc=%d hdmi=%d\n",
++ __func__, phy_freq, vclk_freq, venc_freq, hdmi_freq);
++
++ return meson_vclk_vic_supported_freq(priv, phy_freq, vclk_freq);
++}
++
++static void meson_encoder_hdmi_atomic_enable(struct drm_bridge *bridge,
++ struct drm_bridge_state *bridge_state)
++{
++ struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
++ struct drm_atomic_state *state = bridge_state->base.state;
++ unsigned int ycrcb_map = VPU_HDMI_OUTPUT_CBYCR;
++ struct meson_drm *priv = encoder_hdmi->priv;
++ struct drm_connector_state *conn_state;
++ const struct drm_display_mode *mode;
++ struct drm_crtc_state *crtc_state;
++ struct drm_connector *connector;
++ bool yuv420_mode = false;
++ int vic;
++
++ connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
++ if (WARN_ON(!connector))
++ return;
++
++ conn_state = drm_atomic_get_new_connector_state(state, connector);
++ if (WARN_ON(!conn_state))
++ return;
++
++ crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc);
++ if (WARN_ON(!crtc_state))
++ return;
++
++ mode = &crtc_state->adjusted_mode;
++
++ vic = drm_match_cea_mode(mode);
++
++ dev_dbg(priv->dev, "\"%s\" vic %d\n", mode->name, vic);
++
++ if (encoder_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24) {
++ ycrcb_map = VPU_HDMI_OUTPUT_CRYCB;
++ yuv420_mode = true;
++ }
++
++ /* VENC + VENC-DVI Mode setup */
++ meson_venc_hdmi_mode_set(priv, vic, ycrcb_map, yuv420_mode, mode);
++
++ /* VCLK Set clock */
++ meson_encoder_hdmi_set_vclk(encoder_hdmi, mode);
++
++ if (encoder_hdmi->output_bus_fmt == MEDIA_BUS_FMT_UYYVYY8_0_5X24)
++ /* Setup YUV420 to HDMI-TX, no 10bit diphering */
++ writel_relaxed(2 | (2 << 2),
++ priv->io_base + _REG(VPU_HDMI_FMT_CTRL));
++ else
++ /* Setup YUV444 to HDMI-TX, no 10bit diphering */
++ writel_relaxed(0, priv->io_base + _REG(VPU_HDMI_FMT_CTRL));
++
++ dev_dbg(priv->dev, "%s\n", priv->venc.hdmi_use_enci ? "VENCI" : "VENCP");
++
++ if (priv->venc.hdmi_use_enci)
++ writel_relaxed(1, priv->io_base + _REG(ENCI_VIDEO_EN));
++ else
++ writel_relaxed(1, priv->io_base + _REG(ENCP_VIDEO_EN));
++}
++
++static void meson_encoder_hdmi_atomic_disable(struct drm_bridge *bridge,
++ struct drm_bridge_state *bridge_state)
++{
++ struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
++ struct meson_drm *priv = encoder_hdmi->priv;
++
++ writel_bits_relaxed(0x3, 0,
++ priv->io_base + _REG(VPU_HDMI_SETTING));
++
++ writel_relaxed(0, priv->io_base + _REG(ENCI_VIDEO_EN));
++ writel_relaxed(0, priv->io_base + _REG(ENCP_VIDEO_EN));
++}
++
++static const u32 meson_encoder_hdmi_out_bus_fmts[] = {
++ MEDIA_BUS_FMT_YUV8_1X24,
++ MEDIA_BUS_FMT_UYYVYY8_0_5X24,
++};
++
++static u32 *
++meson_encoder_hdmi_get_inp_bus_fmts(struct drm_bridge *bridge,
++ struct drm_bridge_state *bridge_state,
++ struct drm_crtc_state *crtc_state,
++ struct drm_connector_state *conn_state,
++ u32 output_fmt,
++ unsigned int *num_input_fmts)
++{
++ u32 *input_fmts = NULL;
++ int i;
++
++ *num_input_fmts = 0;
++
++ for (i = 0 ; i < ARRAY_SIZE(meson_encoder_hdmi_out_bus_fmts) ; ++i) {
++ if (output_fmt == meson_encoder_hdmi_out_bus_fmts[i]) {
++ *num_input_fmts = 1;
++ input_fmts = kcalloc(*num_input_fmts,
++ sizeof(*input_fmts),
++ GFP_KERNEL);
++ if (!input_fmts)
++ return NULL;
++
++ input_fmts[0] = output_fmt;
++
++ break;
++ }
++ }
++
++ return input_fmts;
++}
++
++static int meson_encoder_hdmi_atomic_check(struct drm_bridge *bridge,
++ struct drm_bridge_state *bridge_state,
++ struct drm_crtc_state *crtc_state,
++ struct drm_connector_state *conn_state)
++{
++ struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
++ struct drm_connector_state *old_conn_state =
++ drm_atomic_get_old_connector_state(conn_state->state, conn_state->connector);
++ struct meson_drm *priv = encoder_hdmi->priv;
++
++ encoder_hdmi->output_bus_fmt = bridge_state->output_bus_cfg.format;
++
++ dev_dbg(priv->dev, "output_bus_fmt %lx\n", encoder_hdmi->output_bus_fmt);
++
++ if (!drm_connector_atomic_hdr_metadata_equal(old_conn_state, conn_state))
++ crtc_state->mode_changed = true;
++
++ return 0;
++}
++
++static const struct drm_bridge_funcs meson_encoder_hdmi_bridge_funcs = {
++ .attach = meson_encoder_hdmi_attach,
++ .mode_valid = meson_encoder_hdmi_mode_valid,
++ .atomic_enable = meson_encoder_hdmi_atomic_enable,
++ .atomic_disable = meson_encoder_hdmi_atomic_disable,
++ .atomic_get_input_bus_fmts = meson_encoder_hdmi_get_inp_bus_fmts,
++ .atomic_check = meson_encoder_hdmi_atomic_check,
++ .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
++ .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
++ .atomic_reset = drm_atomic_helper_bridge_reset,
++};
++
++int meson_encoder_hdmi_init(struct meson_drm *priv)
++{
++ struct meson_encoder_hdmi *meson_encoder_hdmi;
++ struct device_node *remote;
++ int ret;
++
++ meson_encoder_hdmi = devm_kzalloc(priv->dev, sizeof(*meson_encoder_hdmi), GFP_KERNEL);
++ if (!meson_encoder_hdmi)
++ return -ENOMEM;
++
++ /* HDMI Transceiver Bridge */
++ remote = of_graph_get_remote_node(priv->dev->of_node, 1, 0);
++ if (!remote) {
++ dev_err(priv->dev, "HDMI transceiver device is disabled");
++ return 0;
++ }
++
++ meson_encoder_hdmi->next_bridge = of_drm_find_bridge(remote);
++ if (!meson_encoder_hdmi->next_bridge) {
++ dev_err(priv->dev, "Failed to find HDMI transceiver bridge\n");
++ return -EPROBE_DEFER;
++ }
++
++ /* HDMI Encoder Bridge */
++ meson_encoder_hdmi->bridge.funcs = &meson_encoder_hdmi_bridge_funcs;
++ meson_encoder_hdmi->bridge.of_node = priv->dev->of_node;
++ meson_encoder_hdmi->bridge.type = DRM_MODE_CONNECTOR_HDMIA;
++
++ drm_bridge_add(&meson_encoder_hdmi->bridge);
++
++ meson_encoder_hdmi->priv = priv;
++
++ /* Encoder */
++ ret = drm_simple_encoder_init(priv->drm, &meson_encoder_hdmi->encoder,
++ DRM_MODE_ENCODER_TMDS);
++ if (ret) {
++ dev_err(priv->dev, "Failed to init HDMI encoder: %d\n", ret);
++ return ret;
++ }
++
++ meson_encoder_hdmi->encoder.possible_crtcs = BIT(0);
++
++ /* Attach HDMI Encoder Bridge to Encoder */
++ ret = drm_bridge_attach(&meson_encoder_hdmi->encoder, &meson_encoder_hdmi->bridge, NULL, 0);
++ if (ret) {
++ dev_err(priv->dev, "Failed to attach bridge: %d\n", ret);
++ return ret;
++ }
++
++ /*
++ * We should have now in place:
++ * encoder->[hdmi encoder bridge]->[dw-hdmi bridge]->[dw-hdmi connector]
++ */
++
++ dev_dbg(priv->dev, "HDMI encoder initialized\n");
++
++ return 0;
++}
+diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.h b/drivers/gpu/drm/meson/meson_encoder_hdmi.h
+new file mode 100644
+index 0000000000000..ed19494f09563
+--- /dev/null
++++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.h
+@@ -0,0 +1,12 @@
++/* SPDX-License-Identifier: GPL-2.0-or-later */
++/*
++ * Copyright (C) 2021 BayLibre, SAS
++ * Author: Neil Armstrong <narmstrong@baylibre.com>
++ */
++
++#ifndef __MESON_ENCODER_HDMI_H
++#define __MESON_ENCODER_HDMI_H
++
++int meson_encoder_hdmi_init(struct meson_drm *priv);
++
++#endif /* __MESON_ENCODER_HDMI_H */
+diff --git a/drivers/gpu/drm/meson/meson_osd_afbcd.c b/drivers/gpu/drm/meson/meson_osd_afbcd.c
+index ffc6b584dbf85..0cdbe899402f8 100644
+--- a/drivers/gpu/drm/meson/meson_osd_afbcd.c
++++ b/drivers/gpu/drm/meson/meson_osd_afbcd.c
+@@ -79,11 +79,6 @@ static bool meson_gxm_afbcd_supported_fmt(u64 modifier, uint32_t format)
+ return meson_gxm_afbcd_pixel_fmt(modifier, format) >= 0;
+ }
+
+-static int meson_gxm_afbcd_init(struct meson_drm *priv)
+-{
+- return 0;
+-}
+-
+ static int meson_gxm_afbcd_reset(struct meson_drm *priv)
+ {
+ writel_relaxed(VIU_SW_RESET_OSD1_AFBCD,
+@@ -93,6 +88,16 @@ static int meson_gxm_afbcd_reset(struct meson_drm *priv)
+ return 0;
+ }
+
++static int meson_gxm_afbcd_init(struct meson_drm *priv)
++{
++ return 0;
++}
++
++static void meson_gxm_afbcd_exit(struct meson_drm *priv)
++{
++ meson_gxm_afbcd_reset(priv);
++}
++
+ static int meson_gxm_afbcd_enable(struct meson_drm *priv)
+ {
+ writel_relaxed(FIELD_PREP(OSD1_AFBCD_ID_FIFO_THRD, 0x40) |
+@@ -172,6 +177,7 @@ static int meson_gxm_afbcd_setup(struct meson_drm *priv)
+
+ struct meson_afbcd_ops meson_afbcd_gxm_ops = {
+ .init = meson_gxm_afbcd_init,
++ .exit = meson_gxm_afbcd_exit,
+ .reset = meson_gxm_afbcd_reset,
+ .enable = meson_gxm_afbcd_enable,
+ .disable = meson_gxm_afbcd_disable,
+@@ -269,6 +275,18 @@ static bool meson_g12a_afbcd_supported_fmt(u64 modifier, uint32_t format)
+ return meson_g12a_afbcd_pixel_fmt(modifier, format) >= 0;
+ }
+
++static int meson_g12a_afbcd_reset(struct meson_drm *priv)
++{
++ meson_rdma_reset(priv);
++
++ meson_rdma_writel_sync(priv, VIU_SW_RESET_G12A_AFBC_ARB |
++ VIU_SW_RESET_G12A_OSD1_AFBCD,
++ VIU_SW_RESET);
++ meson_rdma_writel_sync(priv, 0, VIU_SW_RESET);
++
++ return 0;
++}
++
+ static int meson_g12a_afbcd_init(struct meson_drm *priv)
+ {
+ int ret;
+@@ -286,16 +304,10 @@ static int meson_g12a_afbcd_init(struct meson_drm *priv)
+ return 0;
+ }
+
+-static int meson_g12a_afbcd_reset(struct meson_drm *priv)
++static void meson_g12a_afbcd_exit(struct meson_drm *priv)
+ {
+- meson_rdma_reset(priv);
+-
+- meson_rdma_writel_sync(priv, VIU_SW_RESET_G12A_AFBC_ARB |
+- VIU_SW_RESET_G12A_OSD1_AFBCD,
+- VIU_SW_RESET);
+- meson_rdma_writel_sync(priv, 0, VIU_SW_RESET);
+-
+- return 0;
++ meson_g12a_afbcd_reset(priv);
++ meson_rdma_free(priv);
+ }
+
+ static int meson_g12a_afbcd_enable(struct meson_drm *priv)
+@@ -380,6 +392,7 @@ static int meson_g12a_afbcd_setup(struct meson_drm *priv)
+
+ struct meson_afbcd_ops meson_afbcd_g12a_ops = {
+ .init = meson_g12a_afbcd_init,
++ .exit = meson_g12a_afbcd_exit,
+ .reset = meson_g12a_afbcd_reset,
+ .enable = meson_g12a_afbcd_enable,
+ .disable = meson_g12a_afbcd_disable,
+diff --git a/drivers/gpu/drm/meson/meson_osd_afbcd.h b/drivers/gpu/drm/meson/meson_osd_afbcd.h
+index 5e5523304f42f..e77ddeb6416f3 100644
+--- a/drivers/gpu/drm/meson/meson_osd_afbcd.h
++++ b/drivers/gpu/drm/meson/meson_osd_afbcd.h
+@@ -14,6 +14,7 @@
+
+ struct meson_afbcd_ops {
+ int (*init)(struct meson_drm *priv);
++ void (*exit)(struct meson_drm *priv);
+ int (*reset)(struct meson_drm *priv);
+ int (*enable)(struct meson_drm *priv);
+ int (*disable)(struct meson_drm *priv);
+diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
+index fd98e8bbc5500..2c7271f545dcc 100644
+--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
++++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
+@@ -529,7 +529,10 @@ static void mgag200_set_format_regs(struct mga_device *mdev,
+ WREG_GFX(3, 0x00);
+ WREG_GFX(4, 0x00);
+ WREG_GFX(5, 0x40);
+- WREG_GFX(6, 0x05);
++ /* GCTL6 should be 0x05, but we configure memmapsl to 0xb8000 (text mode),
++ * so that it doesn't hang when running kexec/kdump on G200_SE rev42.
++ */
++ WREG_GFX(6, 0x0d);
+ WREG_GFX(7, 0x0f);
+ WREG_GFX(8, 0x0f);
+
+diff --git a/drivers/gpu/drm/mgag200/mgag200_pll.c b/drivers/gpu/drm/mgag200/mgag200_pll.c
+index e9ae22b4f8138..52be08b744ade 100644
+--- a/drivers/gpu/drm/mgag200/mgag200_pll.c
++++ b/drivers/gpu/drm/mgag200/mgag200_pll.c
+@@ -404,9 +404,9 @@ mgag200_pixpll_update_g200wb(struct mgag200_pll *pixpll, const struct mgag200_pl
+ udelay(50);
+
+ /* program pixel pll register */
+- WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
+- WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
+- WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
++ WREG_DAC(MGA1064_WB_PIX_PLLC_N, xpixpllcn);
++ WREG_DAC(MGA1064_WB_PIX_PLLC_M, xpixpllcm);
++ WREG_DAC(MGA1064_WB_PIX_PLLC_P, xpixpllcp);
+
+ udelay(50);
+
+diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
+index 3ddf739a6f9b8..c49b239231190 100644
+--- a/drivers/gpu/drm/msm/Kconfig
++++ b/drivers/gpu/drm/msm/Kconfig
+@@ -63,6 +63,7 @@ config DRM_MSM_HDMI_HDCP
+ config DRM_MSM_DP
+ bool "Enable DisplayPort support in MSM DRM driver"
+ depends on DRM_MSM
++ select RATIONAL
+ default y
+ help
+ Compile in support for DP driver in MSM DRM driver. DP external
+diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+index 8b73f70766a47..4347a104755a9 100644
+--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
++++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+@@ -516,11 +516,11 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+ struct platform_device *pdev = to_platform_device(gmu->dev);
+ void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
+- void __iomem *seqptr;
++ void __iomem *seqptr = NULL;
+ uint32_t pdc_address_offset;
+ bool pdc_in_aop = false;
+
+- if (!pdcptr)
++ if (IS_ERR(pdcptr))
+ goto err;
+
+ if (adreno_is_a650(adreno_gpu) || adreno_is_a660_family(adreno_gpu))
+@@ -532,7 +532,7 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
+
+ if (!pdc_in_aop) {
+ seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq");
+- if (!seqptr)
++ if (IS_ERR(seqptr))
+ goto err;
+ }
+
+diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+index 267a880811d65..9b41e2f82fc23 100644
+--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
++++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+@@ -658,19 +658,23 @@ static void a6xx_set_cp_protect(struct msm_gpu *gpu)
+ {
+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+ const u32 *regs = a6xx_protect;
+- unsigned i, count = ARRAY_SIZE(a6xx_protect), count_max = 32;
+-
+- BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
+- BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
++ unsigned i, count, count_max;
+
+ if (adreno_is_a650(adreno_gpu)) {
+ regs = a650_protect;
+ count = ARRAY_SIZE(a650_protect);
+ count_max = 48;
++ BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
+ } else if (adreno_is_a660_family(adreno_gpu)) {
+ regs = a660_protect;
+ count = ARRAY_SIZE(a660_protect);
+ count_max = 48;
++ BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48);
++ } else {
++ regs = a6xx_protect;
++ count = ARRAY_SIZE(a6xx_protect);
++ count_max = 32;
++ BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
+ }
+
+ /*
+@@ -1424,17 +1428,24 @@ static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
+ {
+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+ struct msm_gpu *gpu = &adreno_gpu->base;
+- u32 gpu_scid, cntl1_regval = 0;
++ u32 cntl1_regval = 0;
+
+ if (IS_ERR(a6xx_gpu->llc_mmio))
+ return;
+
+ if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
+- gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
++ u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
+
+ gpu_scid &= 0x1f;
+ cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
+ (gpu_scid << 15) | (gpu_scid << 20);
++
++ /* On A660, the SCID programming for UCHE traffic is done in
++ * A6XX_GBIF_SCACHE_CNTL0[14:10]
++ */
++ if (adreno_is_a660_family(adreno_gpu))
++ gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
++ (1 << 8), (gpu_scid << 10) | (1 << 8));
+ }
+
+ /*
+@@ -1471,13 +1482,6 @@ static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
+ }
+
+ gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
+-
+- /* On A660, the SCID programming for UCHE traffic is done in
+- * A6XX_GBIF_SCACHE_CNTL0[14:10]
+- */
+- if (adreno_is_a660_family(adreno_gpu))
+- gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
+- (1 << 8), (gpu_scid << 10) | (1 << 8));
+ }
+
+ static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
+@@ -1557,6 +1561,8 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
+ for (i = 0; i < gpu->nr_rings; i++)
+ a6xx_gpu->shadow[i] = 0;
+
++ gpu->suspend_count++;
++
+ return 0;
+ }
+
+@@ -1705,7 +1711,7 @@ a6xx_create_private_address_space(struct msm_gpu *gpu)
+ return ERR_CAST(mmu);
+
+ return msm_gem_address_space_create(mmu,
+- "gpu", 0x100000000ULL, 0x1ffffffffULL);
++ "gpu", 0x100000000ULL, SZ_4G);
+ }
+
+ static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
+diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
+index e8f65cd8eca6e..bfac7e47cb396 100644
+--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
++++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
+@@ -777,12 +777,12 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu,
+ struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+ a6xx_state->gmu_registers = state_kcalloc(a6xx_state,
+- 2, sizeof(*a6xx_state->gmu_registers));
++ 3, sizeof(*a6xx_state->gmu_registers));
+
+ if (!a6xx_state->gmu_registers)
+ return;
+
+- a6xx_state->nr_gmu_registers = 2;
++ a6xx_state->nr_gmu_registers = 3;
+
+ /* Get the CX GMU registers from AHB */
+ _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[0],
+diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+index 0e9d3fa1544be..6bde3e234ec81 100644
+--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
++++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+@@ -1107,7 +1107,7 @@ static void _dpu_encoder_virt_enable_helper(struct drm_encoder *drm_enc)
+ }
+
+
+- if (dpu_enc->disp_info.intf_type == DRM_MODE_CONNECTOR_DisplayPort &&
++ if (dpu_enc->disp_info.intf_type == DRM_MODE_ENCODER_TMDS &&
+ dpu_enc->cur_master->hw_mdptop &&
+ dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select)
+ dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select(
+diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c
+index a98e964c3b6fa..355894a3b48c3 100644
+--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c
++++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c
+@@ -26,9 +26,16 @@ static void dpu_setup_dspp_pcc(struct dpu_hw_dspp *ctx,
+ struct dpu_hw_pcc_cfg *cfg)
+ {
+
+- u32 base = ctx->cap->sblk->pcc.base;
++ u32 base;
+
+- if (!ctx || !base) {
++ if (!ctx) {
++ DRM_ERROR("invalid ctx %pK\n", ctx);
++ return;
++ }
++
++ base = ctx->cap->sblk->pcc.base;
++
++ if (!base) {
+ DRM_ERROR("invalid ctx %pK pcc base 0x%x\n", ctx, base);
+ return;
+ }
+diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
+index 69eed79324865..f9460672176aa 100644
+--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
++++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
+@@ -138,11 +138,13 @@ static int _sspp_subblk_offset(struct dpu_hw_pipe *ctx,
+ u32 *idx)
+ {
+ int rc = 0;
+- const struct dpu_sspp_sub_blks *sblk = ctx->cap->sblk;
++ const struct dpu_sspp_sub_blks *sblk;
+
+- if (!ctx)
++ if (!ctx || !ctx->cap || !ctx->cap->sblk)
+ return -EINVAL;
+
++ sblk = ctx->cap->sblk;
++
+ switch (s_id) {
+ case DPU_SSPP_SRC:
+ *idx = sblk->src_blk.base;
+@@ -419,7 +421,7 @@ static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_pipe *ctx,
+
+ (void)pe;
+ if (_sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED3, &idx) || !sspp
+- || !scaler3_cfg || !ctx || !ctx->cap || !ctx->cap->sblk)
++ || !scaler3_cfg)
+ return;
+
+ dpu_hw_setup_scaler3(&ctx->hw, scaler3_cfg, idx,
+diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+index ae48f41821cfe..93d916858d5ad 100644
+--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
++++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+@@ -73,8 +73,8 @@ static int _dpu_danger_signal_status(struct seq_file *s,
+ &status);
+ } else {
+ seq_puts(s, "\nSafe signal status:\n");
+- if (kms->hw_mdp->ops.get_danger_status)
+- kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
++ if (kms->hw_mdp->ops.get_safe_status)
++ kms->hw_mdp->ops.get_safe_status(kms->hw_mdp,
+ &status);
+ }
+ pm_runtime_put_sync(&kms->pdev->dev);
+@@ -908,6 +908,10 @@ static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms)
+ return 0;
+
+ mmu = msm_iommu_new(dpu_kms->dev->dev, domain);
++ if (IS_ERR(mmu)) {
++ iommu_domain_free(domain);
++ return PTR_ERR(mmu);
++ }
+ aspace = msm_gem_address_space_create(mmu, "dpu1",
+ 0x1000, 0x100000000 - 0x1000);
+
+diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
+index f9c83d6e427ad..24fbaf562d418 100644
+--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
++++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
+@@ -35,6 +35,14 @@ int dpu_rm_destroy(struct dpu_rm *rm)
+ {
+ int i;
+
++ for (i = 0; i < ARRAY_SIZE(rm->dspp_blks); i++) {
++ struct dpu_hw_dspp *hw;
++
++ if (rm->dspp_blks[i]) {
++ hw = to_dpu_hw_dspp(rm->dspp_blks[i]);
++ dpu_hw_dspp_destroy(hw);
++ }
++ }
+ for (i = 0; i < ARRAY_SIZE(rm->pingpong_blks); i++) {
+ struct dpu_hw_pingpong *hw;
+
+diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+index c6b69afcbac89..50e854207c70a 100644
+--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
++++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+@@ -90,7 +90,10 @@ static void mdp5_plane_reset(struct drm_plane *plane)
+ __drm_atomic_helper_plane_destroy_state(plane->state);
+
+ kfree(to_mdp5_plane_state(plane->state));
++ plane->state = NULL;
+ mdp5_state = kzalloc(sizeof(*mdp5_state), GFP_KERNEL);
++ if (!mdp5_state)
++ return;
+
+ if (plane->type == DRM_PLANE_TYPE_PRIMARY)
+ mdp5_state->base.zpos = STAGE_BASE;
+diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
+index cabe15190ec18..369e57f73a470 100644
+--- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
++++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
+@@ -169,6 +169,8 @@ void msm_disp_snapshot_add_block(struct msm_disp_state *disp_state, u32 len,
+ va_list va;
+
+ new_blk = kzalloc(sizeof(struct msm_disp_state_block), GFP_KERNEL);
++ if (!new_blk)
++ return;
+
+ va_start(va, fmt);
+
+diff --git a/drivers/gpu/drm/msm/dp/dp_aux.c b/drivers/gpu/drm/msm/dp/dp_aux.c
+index eb40d8413bca9..6d36f63c33388 100644
+--- a/drivers/gpu/drm/msm/dp/dp_aux.c
++++ b/drivers/gpu/drm/msm/dp/dp_aux.c
+@@ -33,6 +33,7 @@ struct dp_aux_private {
+ bool read;
+ bool no_send_addr;
+ bool no_send_stop;
++ bool initted;
+ u32 offset;
+ u32 segment;
+
+@@ -331,6 +332,10 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux,
+ }
+
+ mutex_lock(&aux->mutex);
++ if (!aux->initted) {
++ ret = -EIO;
++ goto exit;
++ }
+
+ dp_aux_update_offset_and_segment(aux, msg);
+ dp_aux_transfer_helper(aux, msg, true);
+@@ -380,6 +385,8 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux,
+ }
+
+ aux->cmd_busy = false;
++
++exit:
+ mutex_unlock(&aux->mutex);
+
+ return ret;
+@@ -431,8 +438,13 @@ void dp_aux_init(struct drm_dp_aux *dp_aux)
+
+ aux = container_of(dp_aux, struct dp_aux_private, dp_aux);
+
++ mutex_lock(&aux->mutex);
++
+ dp_catalog_aux_enable(aux->catalog, true);
+ aux->retry_cnt = 0;
++ aux->initted = true;
++
++ mutex_unlock(&aux->mutex);
+ }
+
+ void dp_aux_deinit(struct drm_dp_aux *dp_aux)
+@@ -441,7 +453,12 @@ void dp_aux_deinit(struct drm_dp_aux *dp_aux)
+
+ aux = container_of(dp_aux, struct dp_aux_private, dp_aux);
+
++ mutex_lock(&aux->mutex);
++
++ aux->initted = false;
+ dp_catalog_aux_enable(aux->catalog, false);
++
++ mutex_unlock(&aux->mutex);
+ }
+
+ int dp_aux_register(struct drm_dp_aux *dp_aux)
+diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c
+index 62e75dc8afc63..4af281d97493f 100644
+--- a/drivers/gpu/drm/msm/dp/dp_ctrl.c
++++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c
+@@ -1744,6 +1744,9 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl)
+ /* end with failure */
+ break; /* lane == 1 already */
+ }
++
++ /* stop link training before start re training */
++ dp_ctrl_clear_training_pattern(ctrl);
+ }
+ }
+
+diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c
+index a0392e4d8134c..d5198b435638c 100644
+--- a/drivers/gpu/drm/msm/dp/dp_display.c
++++ b/drivers/gpu/drm/msm/dp/dp_display.c
+@@ -1442,6 +1442,7 @@ int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev,
+ struct drm_encoder *encoder)
+ {
+ struct msm_drm_private *priv;
++ struct dp_display_private *dp_priv;
+ int ret;
+
+ if (WARN_ON(!encoder) || WARN_ON(!dp_display) || WARN_ON(!dev))
+@@ -1450,6 +1451,8 @@ int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev,
+ priv = dev->dev_private;
+ dp_display->drm_dev = dev;
+
++ dp_priv = container_of(dp_display, struct dp_display_private, dp_display);
++
+ ret = dp_display_request_irq(dp_display);
+ if (ret) {
+ DRM_ERROR("request_irq failed, ret=%d\n", ret);
+@@ -1467,6 +1470,8 @@ int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev,
+ return ret;
+ }
+
++ dp_priv->panel->connector = dp_display->connector;
++
+ priv->connectors[priv->num_connectors++] = dp_display->connector;
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c
+index 2181b60e1d1d8..62b742e701d2c 100644
+--- a/drivers/gpu/drm/msm/dp/dp_panel.c
++++ b/drivers/gpu/drm/msm/dp/dp_panel.c
+@@ -206,12 +206,6 @@ int dp_panel_read_sink_caps(struct dp_panel *dp_panel,
+ rc = -ETIMEDOUT;
+ goto end;
+ }
+-
+- /* fail safe edid */
+- mutex_lock(&connector->dev->mode_config.mutex);
+- if (drm_add_modes_noedid(connector, 640, 480))
+- drm_set_preferred_mode(connector, 640, 480);
+- mutex_unlock(&connector->dev->mode_config.mutex);
+ }
+
+ if (panel->aux_cfg_update_done) {
+diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
+index 75ae3008b68f4..122fadcf7cc1e 100644
+--- a/drivers/gpu/drm/msm/dsi/dsi.c
++++ b/drivers/gpu/drm/msm/dsi/dsi.c
+@@ -40,7 +40,12 @@ static int dsi_get_phy(struct msm_dsi *msm_dsi)
+
+ of_node_put(phy_node);
+
+- if (!phy_pdev || !msm_dsi->phy) {
++ if (!phy_pdev) {
++ DRM_DEV_ERROR(&pdev->dev, "%s: phy driver is not ready\n", __func__);
++ return -EPROBE_DEFER;
++ }
++ if (!msm_dsi->phy) {
++ put_device(&phy_pdev->dev);
+ DRM_DEV_ERROR(&pdev->dev, "%s: phy driver is not ready\n", __func__);
+ return -EPROBE_DEFER;
+ }
+@@ -215,9 +220,13 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
+ goto fail;
+ }
+
+- if (!msm_dsi_manager_validate_current_config(msm_dsi->id)) {
+- ret = -EINVAL;
+- goto fail;
++ if (msm_dsi_is_bonded_dsi(msm_dsi) &&
++ !msm_dsi_is_master_dsi(msm_dsi)) {
++ /*
++ * Do not return an eror here,
++ * Just skip creating encoder/connector for the slave-DSI.
++ */
++ return 0;
+ }
+
+ msm_dsi->encoder = encoder;
+diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
+index b50db91cb8a7e..a63666e59d19e 100644
+--- a/drivers/gpu/drm/msm/dsi/dsi.h
++++ b/drivers/gpu/drm/msm/dsi/dsi.h
+@@ -82,7 +82,6 @@ int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg);
+ bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len);
+ int msm_dsi_manager_register(struct msm_dsi *msm_dsi);
+ void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi);
+-bool msm_dsi_manager_validate_current_config(u8 id);
+ void msm_dsi_manager_tpg_enable(void);
+
+ /* msm dsi */
+@@ -107,6 +106,8 @@ void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host,
+ u32 dma_base, u32 len);
+ int msm_dsi_host_enable(struct mipi_dsi_host *host);
+ int msm_dsi_host_disable(struct mipi_dsi_host *host);
++void msm_dsi_host_enable_irq(struct mipi_dsi_host *host);
++void msm_dsi_host_disable_irq(struct mipi_dsi_host *host);
+ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
+ struct msm_dsi_phy_shared_timings *phy_shared_timings,
+ bool is_bonded_dsi, struct msm_dsi_phy *phy);
+diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
+index c86b5090fae60..eea679a52e863 100644
+--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
++++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
+@@ -115,16 +115,16 @@ struct msm_dsi_host {
+ struct clk *pixel_clk_src;
+ struct clk *byte_intf_clk;
+
+- u32 byte_clk_rate;
+- u32 pixel_clk_rate;
+- u32 esc_clk_rate;
++ unsigned long byte_clk_rate;
++ unsigned long pixel_clk_rate;
++ unsigned long esc_clk_rate;
+
+ /* DSI v2 specific clocks */
+ struct clk *src_clk;
+ struct clk *esc_clk_src;
+ struct clk *dsi_clk_src;
+
+- u32 src_clk_rate;
++ unsigned long src_clk_rate;
+
+ struct gpio_desc *disp_en_gpio;
+ struct gpio_desc *te_gpio;
+@@ -498,10 +498,10 @@ int msm_dsi_runtime_resume(struct device *dev)
+
+ int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host)
+ {
+- u32 byte_intf_rate;
++ unsigned long byte_intf_rate;
+ int ret;
+
+- DBG("Set clk rates: pclk=%d, byteclk=%d",
++ DBG("Set clk rates: pclk=%d, byteclk=%lu",
+ msm_host->mode->clock, msm_host->byte_clk_rate);
+
+ ret = dev_pm_opp_set_rate(&msm_host->pdev->dev,
+@@ -583,7 +583,7 @@ int dsi_link_clk_set_rate_v2(struct msm_dsi_host *msm_host)
+ {
+ int ret;
+
+- DBG("Set clk rates: pclk=%d, byteclk=%d, esc_clk=%d, dsi_src_clk=%d",
++ DBG("Set clk rates: pclk=%d, byteclk=%lu, esc_clk=%lu, dsi_src_clk=%lu",
+ msm_host->mode->clock, msm_host->byte_clk_rate,
+ msm_host->esc_clk_rate, msm_host->src_clk_rate);
+
+@@ -673,10 +673,10 @@ void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host)
+ clk_disable_unprepare(msm_host->byte_clk);
+ }
+
+-static u32 dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
++static unsigned long dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
+ {
+ struct drm_display_mode *mode = msm_host->mode;
+- u32 pclk_rate;
++ unsigned long pclk_rate;
+
+ pclk_rate = mode->clock * 1000;
+
+@@ -696,7 +696,7 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
+ {
+ u8 lanes = msm_host->lanes;
+ u32 bpp = dsi_get_bpp(msm_host->format);
+- u32 pclk_rate = dsi_get_pclk_rate(msm_host, is_bonded_dsi);
++ unsigned long pclk_rate = dsi_get_pclk_rate(msm_host, is_bonded_dsi);
+ u64 pclk_bpp = (u64)pclk_rate * bpp;
+
+ if (lanes == 0) {
+@@ -713,7 +713,7 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
+ msm_host->pixel_clk_rate = pclk_rate;
+ msm_host->byte_clk_rate = pclk_bpp;
+
+- DBG("pclk=%d, bclk=%d", msm_host->pixel_clk_rate,
++ DBG("pclk=%lu, bclk=%lu", msm_host->pixel_clk_rate,
+ msm_host->byte_clk_rate);
+
+ }
+@@ -772,7 +772,7 @@ int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
+
+ msm_host->esc_clk_rate = msm_host->byte_clk_rate / esc_div;
+
+- DBG("esc=%d, src=%d", msm_host->esc_clk_rate,
++ DBG("esc=%lu, src=%lu", msm_host->esc_clk_rate,
+ msm_host->src_clk_rate);
+
+ return 0;
+@@ -1696,6 +1696,8 @@ static int dsi_host_parse_lane_data(struct msm_dsi_host *msm_host,
+ if (!prop) {
+ DRM_DEV_DEBUG(dev,
+ "failed to find data lane mapping, using default\n");
++ /* Set the number of date lanes to 4 by default. */
++ msm_host->num_data_lanes = 4;
+ return 0;
+ }
+
+@@ -1898,6 +1900,23 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
+ return ret;
+ }
+
++ msm_host->irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
++ if (msm_host->irq < 0) {
++ ret = msm_host->irq;
++ dev_err(&pdev->dev, "failed to get irq: %d\n", ret);
++ return ret;
++ }
++
++ /* do not autoenable, will be enabled later */
++ ret = devm_request_irq(&pdev->dev, msm_host->irq, dsi_host_irq,
++ IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN,
++ "dsi_isr", msm_host);
++ if (ret < 0) {
++ dev_err(&pdev->dev, "failed to request IRQ%u: %d\n",
++ msm_host->irq, ret);
++ return ret;
++ }
++
+ init_completion(&msm_host->dma_comp);
+ init_completion(&msm_host->video_comp);
+ mutex_init(&msm_host->dev_mutex);
+@@ -1941,25 +1960,8 @@ int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
+ {
+ struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+ const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+- struct platform_device *pdev = msm_host->pdev;
+ int ret;
+
+- msm_host->irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
+- if (msm_host->irq < 0) {
+- ret = msm_host->irq;
+- DRM_DEV_ERROR(dev->dev, "failed to get irq: %d\n", ret);
+- return ret;
+- }
+-
+- ret = devm_request_irq(&pdev->dev, msm_host->irq,
+- dsi_host_irq, IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+- "dsi_isr", msm_host);
+- if (ret < 0) {
+- DRM_DEV_ERROR(&pdev->dev, "failed to request IRQ%u: %d\n",
+- msm_host->irq, ret);
+- return ret;
+- }
+-
+ msm_host->dev = dev;
+ ret = cfg_hnd->ops->tx_buf_alloc(msm_host, SZ_4K);
+ if (ret) {
+@@ -2315,6 +2317,20 @@ void msm_dsi_host_get_phy_clk_req(struct mipi_dsi_host *host,
+ clk_req->escclk_rate = msm_host->esc_clk_rate;
+ }
+
++void msm_dsi_host_enable_irq(struct mipi_dsi_host *host)
++{
++ struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
++
++ enable_irq(msm_host->irq);
++}
++
++void msm_dsi_host_disable_irq(struct mipi_dsi_host *host)
++{
++ struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
++
++ disable_irq(msm_host->irq);
++}
++
+ int msm_dsi_host_enable(struct mipi_dsi_host *host)
+ {
+ struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
+index c41d39f5b7cf4..6e43672f58078 100644
+--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
++++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
+@@ -377,6 +377,14 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
+ }
+ }
+
++ /*
++ * Enable before preparing the panel, disable after unpreparing, so
++ * that the panel can communicate over the DSI link.
++ */
++ msm_dsi_host_enable_irq(host);
++ if (is_bonded_dsi && msm_dsi1)
++ msm_dsi_host_enable_irq(msm_dsi1->host);
++
+ /* Always call panel functions once, because even for dual panels,
+ * there is only one drm_panel instance.
+ */
+@@ -411,6 +419,10 @@ host_en_fail:
+ if (panel)
+ drm_panel_unprepare(panel);
+ panel_prep_fail:
++ msm_dsi_host_disable_irq(host);
++ if (is_bonded_dsi && msm_dsi1)
++ msm_dsi_host_disable_irq(msm_dsi1->host);
++
+ if (is_bonded_dsi && msm_dsi1)
+ msm_dsi_host_power_off(msm_dsi1->host);
+ host1_on_fail:
+@@ -523,6 +535,10 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge)
+ id, ret);
+ }
+
++ msm_dsi_host_disable_irq(host);
++ if (is_bonded_dsi && msm_dsi1)
++ msm_dsi_host_disable_irq(msm_dsi1->host);
++
+ /* Save PHY status if it is a clock source */
+ msm_dsi_phy_pll_save_state(msm_dsi->phy);
+
+@@ -627,27 +643,10 @@ struct drm_connector *msm_dsi_manager_connector_init(u8 id)
+ return connector;
+
+ fail:
+- connector->funcs->destroy(msm_dsi->connector);
++ connector->funcs->destroy(connector);
+ return ERR_PTR(ret);
+ }
+
+-bool msm_dsi_manager_validate_current_config(u8 id)
+-{
+- bool is_bonded_dsi = IS_BONDED_DSI();
+-
+- /*
+- * For bonded DSI, we only have one drm panel. For this
+- * use case, we register only one bridge/connector.
+- * Skip bridge/connector initialisation if it is
+- * slave-DSI for bonded DSI configuration.
+- */
+- if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) {
+- DBG("Skip bridge registration for slave DSI->id: %d\n", id);
+- return false;
+- }
+- return true;
+-}
+-
+ /* initialize bridge */
+ struct drm_bridge *msm_dsi_manager_bridge_init(u8 id)
+ {
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+index 8c65ef6968caf..a878b8b079c64 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+@@ -806,12 +806,14 @@ int msm_dsi_phy_enable(struct msm_dsi_phy *phy,
+ struct msm_dsi_phy_clk_request *clk_req,
+ struct msm_dsi_phy_shared_timings *shared_timings)
+ {
+- struct device *dev = &phy->pdev->dev;
++ struct device *dev;
+ int ret;
+
+ if (!phy || !phy->cfg->ops.enable)
+ return -EINVAL;
+
++ dev = &phy->pdev->dev;
++
+ ret = dsi_phy_enable_resource(phy);
+ if (ret) {
+ DRM_DEV_ERROR(dev, "%s: resource enable failed, %d\n",
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
+index d8128f50b0dd5..0b782cc18b3f4 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
+@@ -562,7 +562,9 @@ static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm, struct clk_hw **prov
+ char clk_name[32], parent[32], vco_name[32];
+ char parent2[32], parent3[32], parent4[32];
+ struct clk_init_data vco_init = {
+- .parent_names = (const char *[]){ "xo" },
++ .parent_data = &(const struct clk_parent_data) {
++ .fw_name = "ref",
++ },
+ .num_parents = 1,
+ .name = vco_name,
+ .flags = CLK_IGNORE_UNUSED,
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
+index 5b4e991f220d6..1c1e9861b93f7 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
+@@ -804,7 +804,9 @@ static int pll_14nm_register(struct dsi_pll_14nm *pll_14nm, struct clk_hw **prov
+ {
+ char clk_name[32], parent[32], vco_name[32];
+ struct clk_init_data vco_init = {
+- .parent_names = (const char *[]){ "xo" },
++ .parent_data = &(const struct clk_parent_data) {
++ .fw_name = "ref",
++ },
+ .num_parents = 1,
+ .name = vco_name,
+ .flags = CLK_IGNORE_UNUSED,
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
+index 2da673a2add69..48eab80b548e1 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
+@@ -521,7 +521,9 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
+ {
+ char clk_name[32], parent1[32], parent2[32], vco_name[32];
+ struct clk_init_data vco_init = {
+- .parent_names = (const char *[]){ "xo" },
++ .parent_data = &(const struct clk_parent_data) {
++ .fw_name = "ref", .name = "xo",
++ },
+ .num_parents = 1,
+ .name = vco_name,
+ .flags = CLK_IGNORE_UNUSED,
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
+index 71ed4aa0dc67e..fc56cdcc9ad64 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
+@@ -385,7 +385,9 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
+ {
+ char *clk_name, *parent_name, *vco_name;
+ struct clk_init_data vco_init = {
+- .parent_names = (const char *[]){ "pxo" },
++ .parent_data = &(const struct clk_parent_data) {
++ .fw_name = "ref",
++ },
+ .num_parents = 1,
+ .flags = CLK_IGNORE_UNUSED,
+ .ops = &clk_ops_dsi_pll_28nm_vco,
+diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
+index cb297b08458e4..8cc1ef8199ac9 100644
+--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
++++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
+@@ -590,7 +590,9 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide
+ char clk_name[32], parent[32], vco_name[32];
+ char parent2[32], parent3[32], parent4[32];
+ struct clk_init_data vco_init = {
+- .parent_names = (const char *[]){ "bi_tcxo" },
++ .parent_data = &(const struct clk_parent_data) {
++ .fw_name = "ref",
++ },
+ .num_parents = 1,
+ .name = vco_name,
+ .flags = CLK_IGNORE_UNUSED,
+@@ -864,20 +866,26 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
+ /* Alter PHY configurations if data rate less than 1.5GHZ*/
+ less_than_1500_mhz = (clk_req->bitclk_rate <= 1500000000);
+
+- /* For C-PHY, no low power settings for lower clk rate */
+- if (phy->cphy_mode)
+- less_than_1500_mhz = false;
+-
+ if (phy->cfg->quirks & DSI_PHY_7NM_QUIRK_V4_1) {
+ vreg_ctrl_0 = less_than_1500_mhz ? 0x53 : 0x52;
+- glbl_rescode_top_ctrl = less_than_1500_mhz ? 0x3d : 0x00;
+- glbl_rescode_bot_ctrl = less_than_1500_mhz ? 0x39 : 0x3c;
++ if (phy->cphy_mode) {
++ glbl_rescode_top_ctrl = 0x00;
++ glbl_rescode_bot_ctrl = 0x3c;
++ } else {
++ glbl_rescode_top_ctrl = less_than_1500_mhz ? 0x3d : 0x00;
++ glbl_rescode_bot_ctrl = less_than_1500_mhz ? 0x39 : 0x3c;
++ }
+ glbl_str_swi_cal_sel_ctrl = 0x00;
+ glbl_hstx_str_ctrl_0 = 0x88;
+ } else {
+ vreg_ctrl_0 = less_than_1500_mhz ? 0x5B : 0x59;
+- glbl_str_swi_cal_sel_ctrl = less_than_1500_mhz ? 0x03 : 0x00;
+- glbl_hstx_str_ctrl_0 = less_than_1500_mhz ? 0x66 : 0x88;
++ if (phy->cphy_mode) {
++ glbl_str_swi_cal_sel_ctrl = 0x03;
++ glbl_hstx_str_ctrl_0 = 0x66;
++ } else {
++ glbl_str_swi_cal_sel_ctrl = less_than_1500_mhz ? 0x03 : 0x00;
++ glbl_hstx_str_ctrl_0 = less_than_1500_mhz ? 0x66 : 0x88;
++ }
+ glbl_rescode_top_ctrl = 0x03;
+ glbl_rescode_bot_ctrl = 0x3c;
+ }
+diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
+index 737453b6e5966..94f948ef279d1 100644
+--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
++++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
+@@ -97,10 +97,15 @@ static int msm_hdmi_get_phy(struct hdmi *hdmi)
+
+ of_node_put(phy_node);
+
+- if (!phy_pdev || !hdmi->phy) {
++ if (!phy_pdev) {
+ DRM_DEV_ERROR(&pdev->dev, "phy driver is not ready\n");
+ return -EPROBE_DEFER;
+ }
++ if (!hdmi->phy) {
++ DRM_DEV_ERROR(&pdev->dev, "phy driver is not ready\n");
++ put_device(&phy_pdev->dev);
++ return -EPROBE_DEFER;
++ }
+
+ hdmi->phy_dev = get_device(&phy_pdev->dev);
+
+diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
+index 09d2d279c30ae..dee13fedee3b5 100644
+--- a/drivers/gpu/drm/msm/msm_debugfs.c
++++ b/drivers/gpu/drm/msm/msm_debugfs.c
+@@ -77,6 +77,7 @@ static int msm_gpu_open(struct inode *inode, struct file *file)
+ goto free_priv;
+
+ pm_runtime_get_sync(&gpu->pdev->dev);
++ msm_gpu_hw_init(gpu);
+ show_priv->state = gpu->funcs->gpu_state_get(gpu);
+ pm_runtime_put_sync(&gpu->pdev->dev);
+
+diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
+index d4e09703a87db..bbf999c665174 100644
+--- a/drivers/gpu/drm/msm/msm_drv.c
++++ b/drivers/gpu/drm/msm/msm_drv.c
+@@ -437,7 +437,7 @@ static int msm_init_vram(struct drm_device *dev)
+ of_node_put(node);
+ if (ret)
+ return ret;
+- size = r.end - r.start;
++ size = r.end - r.start + 1;
+ DRM_INFO("using VRAM carveout: %lx@%pa\n", size, &r.start);
+
+ /* if we have no IOMMU, then we need to use carveout allocator.
+@@ -938,29 +938,18 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
+ return ret;
+ }
+
+-static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
+- struct drm_file *file)
++static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id,
++ ktime_t timeout)
+ {
+- struct msm_drm_private *priv = dev->dev_private;
+- struct drm_msm_wait_fence *args = data;
+- ktime_t timeout = to_ktime(args->timeout);
+- struct msm_gpu_submitqueue *queue;
+- struct msm_gpu *gpu = priv->gpu;
+ struct dma_fence *fence;
+ int ret;
+
+- if (args->pad) {
+- DRM_ERROR("invalid pad: %08x\n", args->pad);
++ if (fence_id > queue->last_fence) {
++ DRM_ERROR_RATELIMITED("waiting on invalid fence: %u (of %u)\n",
++ fence_id, queue->last_fence);
+ return -EINVAL;
+ }
+
+- if (!gpu)
+- return 0;
+-
+- queue = msm_submitqueue_get(file->driver_priv, args->queueid);
+- if (!queue)
+- return -ENOENT;
+-
+ /*
+ * Map submitqueue scoped "seqno" (which is actually an idr key)
+ * back to underlying dma-fence
+@@ -972,7 +961,7 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
+ ret = mutex_lock_interruptible(&queue->lock);
+ if (ret)
+ return ret;
+- fence = idr_find(&queue->fence_idr, args->fence);
++ fence = idr_find(&queue->fence_idr, fence_id);
+ if (fence)
+ fence = dma_fence_get_rcu(fence);
+ mutex_unlock(&queue->lock);
+@@ -988,6 +977,32 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
+ }
+
+ dma_fence_put(fence);
++
++ return ret;
++}
++
++static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
++ struct drm_file *file)
++{
++ struct msm_drm_private *priv = dev->dev_private;
++ struct drm_msm_wait_fence *args = data;
++ struct msm_gpu_submitqueue *queue;
++ int ret;
++
++ if (args->pad) {
++ DRM_ERROR("invalid pad: %08x\n", args->pad);
++ return -EINVAL;
++ }
++
++ if (!priv->gpu)
++ return 0;
++
++ queue = msm_submitqueue_get(file->driver_priv, args->queueid);
++ if (!queue)
++ return -ENOENT;
++
++ ret = wait_fence(queue, args->fence, to_ktime(args->timeout));
++
+ msm_submitqueue_put(queue);
+
+ return ret;
+diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
+index 22308a1b66fc3..d280dd64744de 100644
+--- a/drivers/gpu/drm/msm/msm_gem.c
++++ b/drivers/gpu/drm/msm/msm_gem.c
+@@ -937,6 +937,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m,
+ get_pid_task(aspace->pid, PIDTYPE_PID);
+ if (task) {
+ comm = kstrdup(task->comm, GFP_KERNEL);
++ put_task_struct(task);
+ } else {
+ comm = NULL;
+ }
+@@ -1055,8 +1056,7 @@ static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct
+ {
+ struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+- vma->vm_flags &= ~VM_PFNMAP;
+- vma->vm_flags |= VM_MIXEDMAP | VM_DONTEXPAND;
++ vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP;
+ vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags));
+
+ return 0;
+@@ -1132,6 +1132,7 @@ static int msm_gem_new_impl(struct drm_device *dev,
+ msm_obj->flags = flags;
+ msm_obj->madv = MSM_MADV_WILLNEED;
+
++ INIT_LIST_HEAD(&msm_obj->node);
+ INIT_LIST_HEAD(&msm_obj->vmas);
+
+ *obj = &msm_obj->base;
+@@ -1166,7 +1167,7 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32
+
+ ret = msm_gem_new_impl(dev, size, flags, &obj);
+ if (ret)
+- goto fail;
++ return ERR_PTR(ret);
+
+ msm_obj = to_msm_bo(obj);
+
+@@ -1250,7 +1251,7 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
+
+ ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
+ if (ret)
+- goto fail;
++ return ERR_PTR(ret);
+
+ drm_gem_private_object_init(dev, obj, size);
+
+diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
+index 151d19e4453cd..7fb7ff043bcd7 100644
+--- a/drivers/gpu/drm/msm/msm_gem_submit.c
++++ b/drivers/gpu/drm/msm/msm_gem_submit.c
+@@ -780,6 +780,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
+ args->nr_cmds);
+ if (IS_ERR(submit)) {
+ ret = PTR_ERR(submit);
++ submit = NULL;
+ goto out_unlock;
+ }
+
+@@ -886,7 +887,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
+ * to the underlying fence.
+ */
+ submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
+- submit->user_fence, 0, INT_MAX, GFP_KERNEL);
++ submit->user_fence, 1, INT_MAX, GFP_KERNEL);
+ if (submit->fence_id < 0) {
+ ret = submit->fence_id = 0;
+ submit->fence_id = 0;
+@@ -911,6 +912,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
+ drm_sched_entity_push_job(&submit->base, queue->entity);
+
+ args->fence = submit->fence_id;
++ queue->last_fence = submit->fence_id;
+
+ msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
+ msm_process_post_deps(post_deps, args->nr_out_syncobjs,
+diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
+index 8a3a592da3a4d..2c46cd968ac4c 100644
+--- a/drivers/gpu/drm/msm/msm_gpu.c
++++ b/drivers/gpu/drm/msm/msm_gpu.c
+@@ -296,7 +296,7 @@ static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,
+ state->bos = kcalloc(nr,
+ sizeof(struct msm_gpu_state_bo), GFP_KERNEL);
+
+- for (i = 0; i < submit->nr_bos; i++) {
++ for (i = 0; state->bos && i < submit->nr_bos; i++) {
+ if (should_dump(submit, i)) {
+ msm_gpu_crashstate_get_bo(state, submit->bos[i].obj,
+ submit->bos[i].iova, submit->bos[i].flags);
+diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
+index ee25d556c8a10..2e2424066e701 100644
+--- a/drivers/gpu/drm/msm/msm_gpu.h
++++ b/drivers/gpu/drm/msm/msm_gpu.h
+@@ -352,6 +352,8 @@ static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio,
+ * @ring_nr: the ringbuffer used by this submitqueue, which is determined
+ * by the submitqueue's priority
+ * @faults: the number of GPU hangs associated with this submitqueue
++ * @last_fence: the sequence number of the last allocated fence (for error
++ * checking)
+ * @ctx: the per-drm_file context associated with the submitqueue (ie.
+ * which set of pgtables do submits jobs associated with the
+ * submitqueue use)
+@@ -367,6 +369,7 @@ struct msm_gpu_submitqueue {
+ u32 flags;
+ u32 ring_nr;
+ int faults;
++ uint32_t last_fence;
+ struct msm_file_private *ctx;
+ struct list_head node;
+ struct idr fence_idr;
+diff --git a/drivers/gpu/drm/msm/msm_gpu_devfreq.c b/drivers/gpu/drm/msm/msm_gpu_devfreq.c
+index 20006d060b5b5..4ac2a4eb984d8 100644
+--- a/drivers/gpu/drm/msm/msm_gpu_devfreq.c
++++ b/drivers/gpu/drm/msm/msm_gpu_devfreq.c
+@@ -20,6 +20,10 @@ static int msm_devfreq_target(struct device *dev, unsigned long *freq,
+ struct msm_gpu *gpu = dev_to_gpu(dev);
+ struct dev_pm_opp *opp;
+
++ /*
++ * Note that devfreq_recommended_opp() can modify the freq
++ * to something that actually is in the opp table:
++ */
+ opp = devfreq_recommended_opp(dev, freq, flags);
+
+ /*
+@@ -28,6 +32,7 @@ static int msm_devfreq_target(struct device *dev, unsigned long *freq,
+ */
+ if (gpu->devfreq.idle_freq) {
+ gpu->devfreq.idle_freq = *freq;
++ dev_pm_opp_put(opp);
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c
+index b8621c6e05546..7cb158bcbcf67 100644
+--- a/drivers/gpu/drm/msm/msm_submitqueue.c
++++ b/drivers/gpu/drm/msm/msm_submitqueue.c
+@@ -101,6 +101,7 @@ get_sched_entity(struct msm_file_private *ctx, struct msm_ringbuffer *ring,
+
+ ret = drm_sched_entity_init(entity, sched_prio, &sched, 1, NULL);
+ if (ret) {
++ mutex_unlock(&entity_lock);
+ kfree(entity);
+ return ERR_PTR(ret);
+ }
+diff --git a/drivers/gpu/drm/mxsfb/mxsfb_kms.c b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
+index 89dd618d78f31..988bc4fbd78df 100644
+--- a/drivers/gpu/drm/mxsfb/mxsfb_kms.c
++++ b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
+@@ -361,7 +361,17 @@ static void mxsfb_crtc_atomic_enable(struct drm_crtc *crtc,
+ bridge_state =
+ drm_atomic_get_new_bridge_state(state,
+ mxsfb->bridge);
+- bus_format = bridge_state->input_bus_cfg.format;
++ if (!bridge_state)
++ bus_format = MEDIA_BUS_FMT_FIXED;
++ else
++ bus_format = bridge_state->input_bus_cfg.format;
++
++ if (bus_format == MEDIA_BUS_FMT_FIXED) {
++ dev_warn_once(drm->dev,
++ "Bridge does not provide bus format, assuming MEDIA_BUS_FMT_RGB888_1X24.\n"
++ "Please fix bridge driver by handling atomic_get_input_bus_fmts.\n");
++ bus_format = MEDIA_BUS_FMT_RGB888_1X24;
++ }
+ }
+
+ /* If there is no bridge, use bus format from connector */
+diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
+index 7739f46470d3e..99fee4d8cd318 100644
+--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
++++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
+@@ -205,7 +205,7 @@ nv04_display_destroy(struct drm_device *dev)
+ nvif_notify_dtor(&disp->flip);
+
+ nouveau_display(dev)->priv = NULL;
+- kfree(disp);
++ vfree(disp);
+
+ nvif_object_unmap(&drm->client.device.object);
+ }
+@@ -223,7 +223,7 @@ nv04_display_create(struct drm_device *dev)
+ struct nv04_display *disp;
+ int i, ret;
+
+- disp = kzalloc(sizeof(*disp), GFP_KERNEL);
++ disp = vzalloc(sizeof(*disp));
+ if (!disp)
+ return -ENOMEM;
+
+diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
+index 1cbd71abc80aa..aa8ed08fe9a7c 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
++++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
+@@ -46,8 +46,9 @@ static bool
+ nouveau_get_backlight_name(char backlight_name[BL_NAME_SIZE],
+ struct nouveau_backlight *bl)
+ {
+- const int nb = ida_simple_get(&bl_ida, 0, 0, GFP_KERNEL);
+- if (nb < 0 || nb >= 100)
++ const int nb = ida_alloc_max(&bl_ida, 99, GFP_KERNEL);
++
++ if (nb < 0)
+ return false;
+ if (nb > 0)
+ snprintf(backlight_name, BL_NAME_SIZE, "nv_backlight%d", nb);
+@@ -101,7 +102,6 @@ nv40_backlight_init(struct nouveau_encoder *encoder,
+ if (!(nvif_rd32(device, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
+ return -ENODEV;
+
+- props->type = BACKLIGHT_RAW;
+ props->max_brightness = 31;
+ *ops = &nv40_bl_ops;
+ return 0;
+@@ -294,7 +294,8 @@ nv50_backlight_init(struct nouveau_backlight *bl,
+ struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+ struct nvif_object *device = &drm->client.device.object;
+
+- if (!nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(ffs(nv_encoder->dcb->or) - 1)))
++ if (!nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(ffs(nv_encoder->dcb->or) - 1)) ||
++ nv_conn->base.status != connector_status_connected)
+ return -ENODEV;
+
+ if (nv_conn->type == DCB_CONNECTOR_eDP) {
+@@ -339,7 +340,6 @@ nv50_backlight_init(struct nouveau_backlight *bl,
+ else
+ *ops = &nva3_bl_ops;
+
+- props->type = BACKLIGHT_RAW;
+ props->max_brightness = 100;
+
+ return 0;
+@@ -407,11 +407,12 @@ nouveau_backlight_init(struct drm_connector *connector)
+ goto fail_alloc;
+ }
+
++ props.type = BACKLIGHT_RAW;
+ bl->dev = backlight_device_register(backlight_name, connector->kdev,
+ nv_encoder, ops, &props);
+ if (IS_ERR(bl->dev)) {
+ if (bl->id >= 0)
+- ida_simple_remove(&bl_ida, bl->id);
++ ida_free(&bl_ida, bl->id);
+ ret = PTR_ERR(bl->dev);
+ goto fail_alloc;
+ }
+@@ -439,7 +440,7 @@ nouveau_backlight_fini(struct drm_connector *connector)
+ return;
+
+ if (bl->id >= 0)
+- ida_simple_remove(&bl_ida, bl->id);
++ ida_free(&bl_ida, bl->id);
+
+ backlight_device_unregister(bl->dev);
+ nv_conn->backlight = NULL;
+diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
+index 6109cd9e33991..e7efd9ede8e4b 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
++++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
+@@ -562,6 +562,7 @@ nouveau_drm_device_init(struct drm_device *dev)
+ nvkm_dbgopt(nouveau_debug, "DRM");
+
+ INIT_LIST_HEAD(&drm->clients);
++ mutex_init(&drm->clients_lock);
+ spin_lock_init(&drm->tile.lock);
+
+ /* workaround an odd issue on nvc1 by disabling the device's
+@@ -632,6 +633,7 @@ fail_alloc:
+ static void
+ nouveau_drm_device_fini(struct drm_device *dev)
+ {
++ struct nouveau_cli *cli, *temp_cli;
+ struct nouveau_drm *drm = nouveau_drm(dev);
+
+ if (nouveau_pmops_runtime()) {
+@@ -656,9 +658,28 @@ nouveau_drm_device_fini(struct drm_device *dev)
+ nouveau_ttm_fini(drm);
+ nouveau_vga_fini(drm);
+
++ /*
++ * There may be existing clients from as-yet unclosed files. For now,
++ * clean them up here rather than deferring until the file is closed,
++ * but this likely not correct if we want to support hot-unplugging
++ * properly.
++ */
++ mutex_lock(&drm->clients_lock);
++ list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) {
++ list_del(&cli->head);
++ mutex_lock(&cli->mutex);
++ if (cli->abi16)
++ nouveau_abi16_fini(cli->abi16);
++ mutex_unlock(&cli->mutex);
++ nouveau_cli_fini(cli);
++ kfree(cli);
++ }
++ mutex_unlock(&drm->clients_lock);
++
+ nouveau_cli_fini(&drm->client);
+ nouveau_cli_fini(&drm->master);
+ nvif_parent_dtor(&drm->parent);
++ mutex_destroy(&drm->clients_lock);
+ kfree(drm);
+ }
+
+@@ -796,7 +817,7 @@ nouveau_drm_device_remove(struct drm_device *dev)
+ struct nvkm_client *client;
+ struct nvkm_device *device;
+
+- drm_dev_unregister(dev);
++ drm_dev_unplug(dev);
+
+ client = nvxx_client(&drm->client.base);
+ device = nvkm_device_find(client->device);
+@@ -1090,9 +1111,9 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
+
+ fpriv->driver_priv = cli;
+
+- mutex_lock(&drm->client.mutex);
++ mutex_lock(&drm->clients_lock);
+ list_add(&cli->head, &drm->clients);
+- mutex_unlock(&drm->client.mutex);
++ mutex_unlock(&drm->clients_lock);
+
+ done:
+ if (ret && cli) {
+@@ -1110,6 +1131,16 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv)
+ {
+ struct nouveau_cli *cli = nouveau_cli(fpriv);
+ struct nouveau_drm *drm = nouveau_drm(dev);
++ int dev_index;
++
++ /*
++ * The device is gone, and as it currently stands all clients are
++ * cleaned up in the removal codepath. In the future this may change
++ * so that we can support hot-unplugging, but for now we immediately
++ * return to avoid a double-free situation.
++ */
++ if (!drm_dev_enter(dev, &dev_index))
++ return;
+
+ pm_runtime_get_sync(dev->dev);
+
+@@ -1118,14 +1149,15 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv)
+ nouveau_abi16_fini(cli->abi16);
+ mutex_unlock(&cli->mutex);
+
+- mutex_lock(&drm->client.mutex);
++ mutex_lock(&drm->clients_lock);
+ list_del(&cli->head);
+- mutex_unlock(&drm->client.mutex);
++ mutex_unlock(&drm->clients_lock);
+
+ nouveau_cli_fini(cli);
+ kfree(cli);
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
++ drm_dev_exit(dev_index);
+ }
+
+ static const struct drm_ioctl_desc
+diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
+index ba65f136cf481..b2a970aa9bf4b 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
++++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
+@@ -139,6 +139,11 @@ struct nouveau_drm {
+
+ struct list_head clients;
+
++ /**
++ * @clients_lock: Protects access to the @clients list of &struct nouveau_cli.
++ */
++ struct mutex clients_lock;
++
+ u8 old_pm_cap;
+
+ struct {
+diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
+index 05d0b3eb36904..0ae416aa76dcb 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
++++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
+@@ -353,15 +353,22 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
+
+ if (ret)
+ return ret;
+- }
+
+- fobj = dma_resv_shared_list(resv);
+- fence = dma_resv_excl_fence(resv);
++ fobj = NULL;
++ } else {
++ fobj = dma_resv_shared_list(resv);
++ }
+
+- if (fence) {
++ /* Waiting for the exclusive fence first causes performance regressions
++ * under some circumstances. So manually wait for the shared ones first.
++ */
++ for (i = 0; i < (fobj ? fobj->shared_count : 0) && !ret; ++i) {
+ struct nouveau_channel *prev = NULL;
+ bool must_wait = true;
+
++ fence = rcu_dereference_protected(fobj->shared[i],
++ dma_resv_held(resv));
++
+ f = nouveau_local_fence(fence, chan->drm);
+ if (f) {
+ rcu_read_lock();
+@@ -373,20 +380,13 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
+
+ if (must_wait)
+ ret = dma_fence_wait(fence, intr);
+-
+- return ret;
+ }
+
+- if (!exclusive || !fobj)
+- return ret;
+-
+- for (i = 0; i < fobj->shared_count && !ret; ++i) {
++ fence = dma_resv_excl_fence(resv);
++ if (fence) {
+ struct nouveau_channel *prev = NULL;
+ bool must_wait = true;
+
+- fence = rcu_dereference_protected(fobj->shared[i],
+- dma_resv_held(resv));
+-
+ f = nouveau_local_fence(fence, chan->drm);
+ if (f) {
+ rcu_read_lock();
+@@ -398,6 +398,8 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
+
+ if (must_wait)
+ ret = dma_fence_wait(fence, intr);
++
++ return ret;
+ }
+
+ return ret;
+diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
+index 8c2ecc2827232..c89d5964148fd 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
++++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
+@@ -56,7 +56,7 @@ static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf)
+
+ nouveau_bo_del_io_reserve_lru(bo);
+ prot = vm_get_page_prot(vma->vm_flags);
+- ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1);
++ ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
+ nouveau_bo_add_io_reserve_lru(bo);
+ if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
+ return ret;
+diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c
+index b0c3422cb01fa..9985bfde015a6 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_svm.c
++++ b/drivers/gpu/drm/nouveau/nouveau_svm.c
+@@ -162,10 +162,14 @@ nouveau_svmm_bind(struct drm_device *dev, void *data,
+ */
+
+ mm = get_task_mm(current);
++ if (!mm) {
++ return -EINVAL;
++ }
+ mmap_read_lock(mm);
+
+ if (!cli->svm.svmm) {
+ mmap_read_unlock(mm);
++ mmput(mm);
+ return -EINVAL;
+ }
+
+diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c
+index 704df0f2d1f16..09a112af2f893 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c
++++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c
+@@ -78,6 +78,6 @@ int
+ gt215_ce_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
+ struct nvkm_engine **pengine)
+ {
+- return nvkm_falcon_new_(&gt215_ce, device, type, inst,
++ return nvkm_falcon_new_(&gt215_ce, device, type, -1,
+ (device->chipset != 0xaf), 0x104000, pengine);
+ }
+diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+index ca75c5f6ecaf8..88d262ba648cf 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
++++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+@@ -2626,6 +2626,27 @@ nv174_chipset = {
+ .fifo = { 0x00000001, ga102_fifo_new },
+ };
+
++static const struct nvkm_device_chip
++nv176_chipset = {
++ .name = "GA106",
++ .bar = { 0x00000001, tu102_bar_new },
++ .bios = { 0x00000001, nvkm_bios_new },
++ .devinit = { 0x00000001, ga100_devinit_new },
++ .fb = { 0x00000001, ga102_fb_new },
++ .gpio = { 0x00000001, ga102_gpio_new },
++ .i2c = { 0x00000001, gm200_i2c_new },
++ .imem = { 0x00000001, nv50_instmem_new },
++ .mc = { 0x00000001, ga100_mc_new },
++ .mmu = { 0x00000001, tu102_mmu_new },
++ .pci = { 0x00000001, gp100_pci_new },
++ .privring = { 0x00000001, gm200_privring_new },
++ .timer = { 0x00000001, gk20a_timer_new },
++ .top = { 0x00000001, ga100_top_new },
++ .disp = { 0x00000001, ga102_disp_new },
++ .dma = { 0x00000001, gv100_dma_new },
++ .fifo = { 0x00000001, ga102_fifo_new },
++};
++
+ static const struct nvkm_device_chip
+ nv177_chipset = {
+ .name = "GA107",
+@@ -3072,6 +3093,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
+ case 0x168: device->chip = &nv168_chipset; break;
+ case 0x172: device->chip = &nv172_chipset; break;
+ case 0x174: device->chip = &nv174_chipset; break;
++ case 0x176: device->chip = &nv176_chipset; break;
+ case 0x177: device->chip = &nv177_chipset; break;
+ default:
+ if (nvkm_boolopt(device->cfgopt, "NvEnableUnsupportedChipsets", false)) {
+@@ -3147,8 +3169,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
+ WARN_ON(device->chip->ptr.inst & ~((1 << ARRAY_SIZE(device->ptr)) - 1)); \
+ for (j = 0; device->chip->ptr.inst && j < ARRAY_SIZE(device->ptr); j++) { \
+ if ((device->chip->ptr.inst & BIT(j)) && (subdev_mask & BIT_ULL(type))) { \
+- int inst = (device->chip->ptr.inst == 1) ? -1 : (j); \
+- ret = device->chip->ptr.ctor(device, (type), inst, &device->ptr[j]); \
++ ret = device->chip->ptr.ctor(device, (type), (j), &device->ptr[j]); \
+ subdev = nvkm_device_subdev(device, (type), (j)); \
+ if (ret) { \
+ nvkm_subdev_del(&subdev); \
+diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+index d0d52c1d4aee0..950a3de3e1166 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
++++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+@@ -123,7 +123,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
+
+ mutex_init(&tdev->iommu.mutex);
+
+- if (iommu_present(&platform_bus_type)) {
++ if (device_iommu_mapped(dev)) {
+ tdev->iommu.domain = iommu_domain_alloc(&platform_bus_type);
+ if (!tdev->iommu.domain)
+ goto error;
+diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
+index 6e3c450eaacef..3ff49344abc77 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
++++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
+@@ -62,7 +62,6 @@ gv100_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet,
+ nvkm_wr32(device, 0x6f0108 + hdmi, vendor_infoframe.header);
+ nvkm_wr32(device, 0x6f010c + hdmi, vendor_infoframe.subpack0_low);
+ nvkm_wr32(device, 0x6f0110 + hdmi, vendor_infoframe.subpack0_high);
+- nvkm_wr32(device, 0x6f0110 + hdmi, 0x00000000);
+ nvkm_wr32(device, 0x6f0114 + hdmi, 0x00000000);
+ nvkm_wr32(device, 0x6f0118 + hdmi, 0x00000000);
+ nvkm_wr32(device, 0x6f011c + hdmi, 0x00000000);
+diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
+index 262641a014b06..c91130a6be2a1 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
++++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
+@@ -117,8 +117,12 @@ nvkm_falcon_disable(struct nvkm_falcon *falcon)
+ int
+ nvkm_falcon_reset(struct nvkm_falcon *falcon)
+ {
+- nvkm_falcon_disable(falcon);
+- return nvkm_falcon_enable(falcon);
++ if (!falcon->func->reset) {
++ nvkm_falcon_disable(falcon);
++ return nvkm_falcon_enable(falcon);
++ }
++
++ return falcon->func->reset(falcon);
+ }
+
+ int
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c
+index cdb1ead26d84f..82b4c8e1457c2 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c
+@@ -207,11 +207,13 @@ int
+ gm200_acr_wpr_parse(struct nvkm_acr *acr)
+ {
+ const struct wpr_header *hdr = (void *)acr->wpr_fw->data;
++ struct nvkm_acr_lsfw *lsfw;
+
+ while (hdr->falcon_id != WPR_HEADER_V0_FALCON_ID_INVALID) {
+ wpr_header_dump(&acr->subdev, hdr);
+- if (!nvkm_acr_lsfw_add(NULL, acr, NULL, (hdr++)->falcon_id))
+- return -ENOMEM;
++ lsfw = nvkm_acr_lsfw_add(NULL, acr, NULL, (hdr++)->falcon_id);
++ if (IS_ERR(lsfw))
++ return PTR_ERR(lsfw);
+ }
+
+ return 0;
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c
+index fb9132a39bb1a..fd97a935a380e 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c
+@@ -161,11 +161,13 @@ int
+ gp102_acr_wpr_parse(struct nvkm_acr *acr)
+ {
+ const struct wpr_header_v1 *hdr = (void *)acr->wpr_fw->data;
++ struct nvkm_acr_lsfw *lsfw;
+
+ while (hdr->falcon_id != WPR_HEADER_V1_FALCON_ID_INVALID) {
+ wpr_header_v1_dump(&acr->subdev, hdr);
+- if (!nvkm_acr_lsfw_add(NULL, acr, NULL, (hdr++)->falcon_id))
+- return -ENOMEM;
++ lsfw = nvkm_acr_lsfw_add(NULL, acr, NULL, (hdr++)->falcon_id);
++ if (IS_ERR(lsfw))
++ return PTR_ERR(lsfw);
+ }
+
+ return 0;
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c
+index 667fa016496ee..a6ea89a5d51ab 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c
+@@ -142,11 +142,12 @@ nvkm_acr_hsfw_load_bl(struct nvkm_acr *acr, const char *name, int ver,
+
+ hsfw->imem_size = desc->code_size;
+ hsfw->imem_tag = desc->start_tag;
+- hsfw->imem = kmalloc(desc->code_size, GFP_KERNEL);
+- memcpy(hsfw->imem, data + desc->code_off, desc->code_size);
+-
++ hsfw->imem = kmemdup(data + desc->code_off, desc->code_size, GFP_KERNEL);
+ nvkm_firmware_put(fw);
+- return 0;
++ if (!hsfw->imem)
++ return -ENOMEM;
++ else
++ return 0;
+ }
+
+ int
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
+index d0f52d59fc2f9..64e423dddd9e7 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
+@@ -38,7 +38,7 @@ nvbios_addr(struct nvkm_bios *bios, u32 *addr, u8 size)
+ *addr += bios->imaged_addr;
+ }
+
+- if (unlikely(*addr + size >= bios->size)) {
++ if (unlikely(*addr + size > bios->size)) {
+ nvkm_error(&bios->subdev, "OOB %d %08x %08x\n", size, p, *addr);
+ return false;
+ }
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
+index 24382875fb4f3..455e95a89259f 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
+@@ -94,20 +94,13 @@ nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend)
+ return 0;
+ }
+
+-static int
++static void
+ nvkm_pmu_reset(struct nvkm_pmu *pmu)
+ {
+ struct nvkm_device *device = pmu->subdev.device;
+
+ if (!pmu->func->enabled(pmu))
+- return 0;
+-
+- /* Inhibit interrupts, and wait for idle. */
+- nvkm_wr32(device, 0x10a014, 0x0000ffff);
+- nvkm_msec(device, 2000,
+- if (!nvkm_rd32(device, 0x10a04c))
+- break;
+- );
++ return;
+
+ /* Reset. */
+ if (pmu->func->reset)
+@@ -118,25 +111,37 @@ nvkm_pmu_reset(struct nvkm_pmu *pmu)
+ if (!(nvkm_rd32(device, 0x10a10c) & 0x00000006))
+ break;
+ );
+-
+- return 0;
+ }
+
+ static int
+ nvkm_pmu_preinit(struct nvkm_subdev *subdev)
+ {
+ struct nvkm_pmu *pmu = nvkm_pmu(subdev);
+- return nvkm_pmu_reset(pmu);
++ nvkm_pmu_reset(pmu);
++ return 0;
+ }
+
+ static int
+ nvkm_pmu_init(struct nvkm_subdev *subdev)
+ {
+ struct nvkm_pmu *pmu = nvkm_pmu(subdev);
+- int ret = nvkm_pmu_reset(pmu);
+- if (ret == 0 && pmu->func->init)
+- ret = pmu->func->init(pmu);
+- return ret;
++ struct nvkm_device *device = pmu->subdev.device;
++
++ if (!pmu->func->init)
++ return 0;
++
++ if (pmu->func->enabled(pmu)) {
++ /* Inhibit interrupts, and wait for idle. */
++ nvkm_wr32(device, 0x10a014, 0x0000ffff);
++ nvkm_msec(device, 2000,
++ if (!nvkm_rd32(device, 0x10a04c))
++ break;
++ );
++
++ nvkm_pmu_reset(pmu);
++ }
++
++ return pmu->func->init(pmu);
+ }
+
+ static void *
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c
+index 5968c7696596c..40439e329aa9f 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c
+@@ -23,9 +23,38 @@
+ */
+ #include "priv.h"
+
++static int
++gm200_pmu_flcn_reset(struct nvkm_falcon *falcon)
++{
++ struct nvkm_pmu *pmu = container_of(falcon, typeof(*pmu), falcon);
++
++ nvkm_falcon_wr32(falcon, 0x014, 0x0000ffff);
++ pmu->func->reset(pmu);
++ return nvkm_falcon_enable(falcon);
++}
++
++const struct nvkm_falcon_func
++gm200_pmu_flcn = {
++ .debug = 0xc08,
++ .fbif = 0xe00,
++ .load_imem = nvkm_falcon_v1_load_imem,
++ .load_dmem = nvkm_falcon_v1_load_dmem,
++ .read_dmem = nvkm_falcon_v1_read_dmem,
++ .bind_context = nvkm_falcon_v1_bind_context,
++ .wait_for_halt = nvkm_falcon_v1_wait_for_halt,
++ .clear_interrupt = nvkm_falcon_v1_clear_interrupt,
++ .set_start_addr = nvkm_falcon_v1_set_start_addr,
++ .start = nvkm_falcon_v1_start,
++ .enable = nvkm_falcon_v1_enable,
++ .disable = nvkm_falcon_v1_disable,
++ .reset = gm200_pmu_flcn_reset,
++ .cmdq = { 0x4a0, 0x4b0, 4 },
++ .msgq = { 0x4c8, 0x4cc, 0 },
++};
++
+ static const struct nvkm_pmu_func
+ gm200_pmu = {
+- .flcn = &gt215_pmu_flcn,
++ .flcn = &gm200_pmu_flcn,
+ .enabled = gf100_pmu_enabled,
+ .reset = gf100_pmu_reset,
+ };
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
+index 148706977eec7..612310d5d4812 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
+@@ -211,11 +211,12 @@ gm20b_pmu_recv(struct nvkm_pmu *pmu)
+
+ static const struct nvkm_pmu_func
+ gm20b_pmu = {
+- .flcn = &gt215_pmu_flcn,
++ .flcn = &gm200_pmu_flcn,
+ .enabled = gf100_pmu_enabled,
+ .intr = gt215_pmu_intr,
+ .recv = gm20b_pmu_recv,
+ .initmsg = gm20b_pmu_initmsg,
++ .reset = gf100_pmu_reset,
+ };
+
+ #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
+index 00da1b873ce81..1a6f9c3af5ecd 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
+@@ -23,7 +23,7 @@
+ */
+ #include "priv.h"
+
+-static void
++void
+ gp102_pmu_reset(struct nvkm_pmu *pmu)
+ {
+ struct nvkm_device *device = pmu->subdev.device;
+@@ -39,7 +39,7 @@ gp102_pmu_enabled(struct nvkm_pmu *pmu)
+
+ static const struct nvkm_pmu_func
+ gp102_pmu = {
+- .flcn = &gt215_pmu_flcn,
++ .flcn = &gm200_pmu_flcn,
+ .enabled = gp102_pmu_enabled,
+ .reset = gp102_pmu_reset,
+ };
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c
+index 461f722656e24..94cfb1791af6e 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c
+@@ -78,11 +78,12 @@ gp10b_pmu_acr = {
+
+ static const struct nvkm_pmu_func
+ gp10b_pmu = {
+- .flcn = &gt215_pmu_flcn,
++ .flcn = &gm200_pmu_flcn,
+ .enabled = gf100_pmu_enabled,
+ .intr = gt215_pmu_intr,
+ .recv = gm20b_pmu_recv,
+ .initmsg = gm20b_pmu_initmsg,
++ .reset = gp102_pmu_reset,
+ };
+
+ #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
+index e7860d1773539..21abf31f44420 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
+@@ -41,9 +41,12 @@ int gt215_pmu_send(struct nvkm_pmu *, u32[2], u32, u32, u32, u32);
+
+ bool gf100_pmu_enabled(struct nvkm_pmu *);
+ void gf100_pmu_reset(struct nvkm_pmu *);
++void gp102_pmu_reset(struct nvkm_pmu *pmu);
+
+ void gk110_pmu_pgob(struct nvkm_pmu *, bool);
+
++extern const struct nvkm_falcon_func gm200_pmu_flcn;
++
+ void gm20b_pmu_acr_bld_patch(struct nvkm_acr *, u32, s64);
+ void gm20b_pmu_acr_bld_write(struct nvkm_acr *, u32, struct nvkm_acr_lsfw *);
+ int gm20b_pmu_acr_boot(struct nvkm_falcon *);
+diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
+index 418638e6e3b0a..af1402d83d51c 100644
+--- a/drivers/gpu/drm/panel/Kconfig
++++ b/drivers/gpu/drm/panel/Kconfig
+@@ -83,6 +83,8 @@ config DRM_PANEL_SIMPLE
+ depends on PM
+ select VIDEOMODE_HELPERS
+ select DRM_DP_AUX_BUS
++ select DRM_DP_HELPER
++ select DRM_KMS_HELPER
+ help
+ DRM panel driver for dumb panels that need at most a regulator and
+ a GPIO to be powered up. Optionally a backlight can be attached so
+diff --git a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
+index 581661b506f81..f9c1f7bc8218c 100644
+--- a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
++++ b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
+@@ -227,7 +227,13 @@ static int feiyang_dsi_probe(struct mipi_dsi_device *dsi)
+ dsi->format = MIPI_DSI_FMT_RGB888;
+ dsi->lanes = 4;
+
+- return mipi_dsi_attach(dsi);
++ ret = mipi_dsi_attach(dsi);
++ if (ret < 0) {
++ drm_panel_remove(&ctx->panel);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int feiyang_dsi_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
+index 2c3378a259b1e..e1542451ef9d0 100644
+--- a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
++++ b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
+@@ -612,8 +612,10 @@ static int ili9341_dbi_probe(struct spi_device *spi, struct gpio_desc *dc,
+ int ret;
+
+ vcc = devm_regulator_get_optional(dev, "vcc");
+- if (IS_ERR(vcc))
++ if (IS_ERR(vcc)) {
+ dev_err(dev, "get optional vcc failed\n");
++ vcc = NULL;
++ }
+
+ dbidev = devm_drm_dev_alloc(dev, &ili9341_dbi_driver,
+ struct mipi_dbi_dev, drm);
+diff --git a/drivers/gpu/drm/panel/panel-innolux-p079zca.c b/drivers/gpu/drm/panel/panel-innolux-p079zca.c
+index aea3162253914..f194b62e290ca 100644
+--- a/drivers/gpu/drm/panel/panel-innolux-p079zca.c
++++ b/drivers/gpu/drm/panel/panel-innolux-p079zca.c
+@@ -484,6 +484,7 @@ static void innolux_panel_del(struct innolux_panel *innolux)
+ static int innolux_panel_probe(struct mipi_dsi_device *dsi)
+ {
+ const struct panel_desc *desc;
++ struct innolux_panel *innolux;
+ int err;
+
+ desc = of_device_get_match_data(&dsi->dev);
+@@ -495,7 +496,14 @@ static int innolux_panel_probe(struct mipi_dsi_device *dsi)
+ if (err < 0)
+ return err;
+
+- return mipi_dsi_attach(dsi);
++ err = mipi_dsi_attach(dsi);
++ if (err < 0) {
++ innolux = mipi_dsi_get_drvdata(dsi);
++ innolux_panel_del(innolux);
++ return err;
++ }
++
++ return 0;
+ }
+
+ static int innolux_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
+index 733010b5e4f53..3c86ad262d5e0 100644
+--- a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
++++ b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
+@@ -473,7 +473,13 @@ static int jdi_panel_probe(struct mipi_dsi_device *dsi)
+ if (ret < 0)
+ return ret;
+
+- return mipi_dsi_attach(dsi);
++ ret = mipi_dsi_attach(dsi);
++ if (ret < 0) {
++ jdi_panel_del(jdi);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int jdi_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c b/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
+index 86e4213e8bb13..daccb1fd5fdad 100644
+--- a/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
++++ b/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
+@@ -406,7 +406,13 @@ static int kingdisplay_panel_probe(struct mipi_dsi_device *dsi)
+ if (err < 0)
+ return err;
+
+- return mipi_dsi_attach(dsi);
++ err = mipi_dsi_attach(dsi);
++ if (err < 0) {
++ kingdisplay_panel_del(kingdisplay);
++ return err;
++ }
++
++ return 0;
+ }
+
+ static int kingdisplay_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+index 533cd3934b8b7..839b263fb3c0f 100644
+--- a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
++++ b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+@@ -656,7 +656,13 @@ static int nt36672a_panel_probe(struct mipi_dsi_device *dsi)
+ if (err < 0)
+ return err;
+
+- return mipi_dsi_attach(dsi);
++ err = mipi_dsi_attach(dsi);
++ if (err < 0) {
++ drm_panel_remove(&pinfo->base);
++ return err;
++ }
++
++ return 0;
+ }
+
+ static int nt36672a_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
+index 3c20beeb17819..3991f5d950af4 100644
+--- a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
++++ b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
+@@ -241,7 +241,13 @@ static int wuxga_nt_panel_probe(struct mipi_dsi_device *dsi)
+ if (ret < 0)
+ return ret;
+
+- return mipi_dsi_attach(dsi);
++ ret = mipi_dsi_attach(dsi);
++ if (ret < 0) {
++ wuxga_nt_panel_del(wuxga_nt);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int wuxga_nt_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
+index 46029c5610c80..145047e193946 100644
+--- a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
++++ b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
+@@ -229,7 +229,7 @@ static void rpi_touchscreen_i2c_write(struct rpi_touchscreen *ts,
+
+ ret = i2c_smbus_write_byte_data(ts->i2c, reg, val);
+ if (ret)
+- dev_err(&ts->dsi->dev, "I2C write failed: %d\n", ret);
++ dev_err(&ts->i2c->dev, "I2C write failed: %d\n", ret);
+ }
+
+ static int rpi_touchscreen_write(struct rpi_touchscreen *ts, u16 reg, u32 val)
+@@ -265,7 +265,7 @@ static int rpi_touchscreen_noop(struct drm_panel *panel)
+ return 0;
+ }
+
+-static int rpi_touchscreen_enable(struct drm_panel *panel)
++static int rpi_touchscreen_prepare(struct drm_panel *panel)
+ {
+ struct rpi_touchscreen *ts = panel_to_ts(panel);
+ int i;
+@@ -295,6 +295,13 @@ static int rpi_touchscreen_enable(struct drm_panel *panel)
+ rpi_touchscreen_write(ts, DSI_STARTDSI, 0x01);
+ msleep(100);
+
++ return 0;
++}
++
++static int rpi_touchscreen_enable(struct drm_panel *panel)
++{
++ struct rpi_touchscreen *ts = panel_to_ts(panel);
++
+ /* Turn on the backlight. */
+ rpi_touchscreen_i2c_write(ts, REG_PWM, 255);
+
+@@ -349,7 +356,7 @@ static int rpi_touchscreen_get_modes(struct drm_panel *panel,
+ static const struct drm_panel_funcs rpi_touchscreen_funcs = {
+ .disable = rpi_touchscreen_disable,
+ .unprepare = rpi_touchscreen_noop,
+- .prepare = rpi_touchscreen_noop,
++ .prepare = rpi_touchscreen_prepare,
+ .enable = rpi_touchscreen_enable,
+ .get_modes = rpi_touchscreen_get_modes,
+ };
+diff --git a/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
+index a3782830ae3c4..1fb579a574d9f 100644
+--- a/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
++++ b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
+@@ -199,7 +199,13 @@ static int rb070d30_panel_dsi_probe(struct mipi_dsi_device *dsi)
+ dsi->format = MIPI_DSI_FMT_RGB888;
+ dsi->lanes = 4;
+
+- return mipi_dsi_attach(dsi);
++ ret = mipi_dsi_attach(dsi);
++ if (ret < 0) {
++ drm_panel_remove(&ctx->panel);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int rb070d30_panel_dsi_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
+index ea63799ff2a1e..29fde3823212b 100644
+--- a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
++++ b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
+@@ -247,6 +247,7 @@ static int s6e88a0_ams452ef01_probe(struct mipi_dsi_device *dsi)
+ ret = mipi_dsi_attach(dsi);
+ if (ret < 0) {
+ dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
++ drm_panel_remove(&ctx->panel);
+ return ret;
+ }
+
+diff --git a/drivers/gpu/drm/panel/panel-samsung-sofef00.c b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
+index 8cb1853574bb8..6d107e14fcc55 100644
+--- a/drivers/gpu/drm/panel/panel-samsung-sofef00.c
++++ b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
+@@ -302,6 +302,7 @@ static int sofef00_panel_probe(struct mipi_dsi_device *dsi)
+ ret = mipi_dsi_attach(dsi);
+ if (ret < 0) {
+ dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
++ drm_panel_remove(&ctx->panel);
+ return ret;
+ }
+
+diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
+index b937e24dac8e0..25829a0a8e801 100644
+--- a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
++++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
+@@ -296,7 +296,13 @@ static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
+ if (ret < 0)
+ return ret;
+
+- return mipi_dsi_attach(dsi);
++ ret = mipi_dsi_attach(dsi);
++ if (ret < 0) {
++ sharp_nt_panel_del(sharp_nt);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
+diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
+index 9b6c4e6c38a1b..f9242c19b4582 100644
+--- a/drivers/gpu/drm/panel/panel-simple.c
++++ b/drivers/gpu/drm/panel/panel-simple.c
+@@ -721,6 +721,7 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc,
+ err = panel_dpi_probe(dev, panel);
+ if (err)
+ goto free_ddc;
++ desc = panel->desc;
+ } else {
+ if (!of_get_display_timing(dev->of_node, "panel-timing", &dt))
+ panel_simple_parse_panel_timing_node(dev, panel, &dt);
+@@ -2509,7 +2510,7 @@ static const struct display_timing innolux_g070y2_l01_timing = {
+ static const struct panel_desc innolux_g070y2_l01 = {
+ .timings = &innolux_g070y2_l01_timing,
+ .num_timings = 1,
+- .bpc = 6,
++ .bpc = 8,
+ .size = {
+ .width = 152,
+ .height = 91,
+diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
+index bbe628b306ee3..f8355de6e335d 100644
+--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
++++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
+@@ -360,8 +360,11 @@ int panfrost_gpu_init(struct panfrost_device *pfdev)
+
+ panfrost_gpu_init_features(pfdev);
+
+- dma_set_mask_and_coherent(pfdev->dev,
++ err = dma_set_mask_and_coherent(pfdev->dev,
+ DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features)));
++ if (err)
++ return err;
++
+ dma_set_max_seg_size(pfdev->dev, UINT_MAX);
+
+ irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu");
+diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
+index 0fce73b9a6469..70bd84b7ef2b0 100644
+--- a/drivers/gpu/drm/radeon/atombios_encoders.c
++++ b/drivers/gpu/drm/radeon/atombios_encoders.c
+@@ -198,7 +198,8 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
+ * so don't register a backlight device
+ */
+ if ((rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) &&
+- (rdev->pdev->device == 0x6741))
++ (rdev->pdev->device == 0x6741) &&
++ !dmi_match(DMI_PRODUCT_NAME, "iMac12,1"))
+ return;
+
+ if (!radeon_encoder->enc_priv)
+diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
+index 607ad5620bd99..1546abcadacf4 100644
+--- a/drivers/gpu/drm/radeon/radeon_connectors.c
++++ b/drivers/gpu/drm/radeon/radeon_connectors.c
+@@ -204,7 +204,7 @@ int radeon_get_monitor_bpc(struct drm_connector *connector)
+
+ /* Check if bpc is within clock limit. Try to degrade gracefully otherwise */
+ if ((bpc == 12) && (mode_clock * 3/2 > max_tmds_clock)) {
+- if ((connector->display_info.edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30) &&
++ if ((connector->display_info.edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30) &&
+ (mode_clock * 5/4 <= max_tmds_clock))
+ bpc = 10;
+ else
+diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
+index 458f92a708879..a36a4f2c76b09 100644
+--- a/drivers/gpu/drm/radeon/radeon_gem.c
++++ b/drivers/gpu/drm/radeon/radeon_gem.c
+@@ -61,7 +61,7 @@ static vm_fault_t radeon_gem_fault(struct vm_fault *vmf)
+ goto unlock_resv;
+
+ ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
+- TTM_BO_VM_NUM_PREFAULT, 1);
++ TTM_BO_VM_NUM_PREFAULT);
+ if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
+ goto unlock_mclk;
+
+diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
+index 482fb0ae6cb5d..0e14907f2043e 100644
+--- a/drivers/gpu/drm/radeon/radeon_kms.c
++++ b/drivers/gpu/drm/radeon/radeon_kms.c
+@@ -648,6 +648,8 @@ void radeon_driver_lastclose_kms(struct drm_device *dev)
+ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
+ {
+ struct radeon_device *rdev = dev->dev_private;
++ struct radeon_fpriv *fpriv;
++ struct radeon_vm *vm;
+ int r;
+
+ file_priv->driver_priv = NULL;
+@@ -660,48 +662,52 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
+
+ /* new gpu have virtual address space support */
+ if (rdev->family >= CHIP_CAYMAN) {
+- struct radeon_fpriv *fpriv;
+- struct radeon_vm *vm;
+
+ fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
+ if (unlikely(!fpriv)) {
+ r = -ENOMEM;
+- goto out_suspend;
++ goto err_suspend;
+ }
+
+ if (rdev->accel_working) {
+ vm = &fpriv->vm;
+ r = radeon_vm_init(rdev, vm);
+- if (r) {
+- kfree(fpriv);
+- goto out_suspend;
+- }
++ if (r)
++ goto err_fpriv;
+
+ r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
+- if (r) {
+- radeon_vm_fini(rdev, vm);
+- kfree(fpriv);
+- goto out_suspend;
+- }
++ if (r)
++ goto err_vm_fini;
+
+ /* map the ib pool buffer read only into
+ * virtual address space */
+ vm->ib_bo_va = radeon_vm_bo_add(rdev, vm,
+ rdev->ring_tmp_bo.bo);
++ if (!vm->ib_bo_va) {
++ r = -ENOMEM;
++ goto err_vm_fini;
++ }
++
+ r = radeon_vm_bo_set_addr(rdev, vm->ib_bo_va,
+ RADEON_VA_IB_OFFSET,
+ RADEON_VM_PAGE_READABLE |
+ RADEON_VM_PAGE_SNOOPED);
+- if (r) {
+- radeon_vm_fini(rdev, vm);
+- kfree(fpriv);
+- goto out_suspend;
+- }
++ if (r)
++ goto err_vm_fini;
+ }
+ file_priv->driver_priv = fpriv;
+ }
+
+-out_suspend:
++ pm_runtime_mark_last_busy(dev->dev);
++ pm_runtime_put_autosuspend(dev->dev);
++ return 0;
++
++err_vm_fini:
++ radeon_vm_fini(rdev, vm);
++err_fpriv:
++ kfree(fpriv);
++
++err_suspend:
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
+ return r;
+diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+index ea7e39d035457..ee7e375ee6724 100644
+--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
++++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+@@ -215,6 +215,7 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
+ const struct drm_display_mode *mode = &rcrtc->crtc.state->adjusted_mode;
+ struct rcar_du_device *rcdu = rcrtc->dev;
+ unsigned long mode_clock = mode->clock * 1000;
++ unsigned int hdse_offset;
+ u32 dsmr;
+ u32 escr;
+
+@@ -298,10 +299,15 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
+ | DSMR_DIPM_DISP | DSMR_CSPM;
+ rcar_du_crtc_write(rcrtc, DSMR, dsmr);
+
++ hdse_offset = 19;
++ if (rcrtc->group->cmms_mask & BIT(rcrtc->index % 2))
++ hdse_offset += 25;
++
+ /* Display timings */
+- rcar_du_crtc_write(rcrtc, HDSR, mode->htotal - mode->hsync_start - 19);
++ rcar_du_crtc_write(rcrtc, HDSR, mode->htotal - mode->hsync_start -
++ hdse_offset);
+ rcar_du_crtc_write(rcrtc, HDER, mode->htotal - mode->hsync_start +
+- mode->hdisplay - 19);
++ mode->hdisplay - hdse_offset);
+ rcar_du_crtc_write(rcrtc, HSWR, mode->hsync_end -
+ mode->hsync_start - 1);
+ rcar_du_crtc_write(rcrtc, HCR, mode->htotal - 1);
+@@ -836,6 +842,7 @@ rcar_du_crtc_mode_valid(struct drm_crtc *crtc,
+ struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+ struct rcar_du_device *rcdu = rcrtc->dev;
+ bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
++ unsigned int min_sync_porch;
+ unsigned int vbp;
+
+ if (interlaced && !rcar_du_has(rcdu, RCAR_DU_FEATURE_INTERLACED))
+@@ -843,9 +850,14 @@ rcar_du_crtc_mode_valid(struct drm_crtc *crtc,
+
+ /*
+ * The hardware requires a minimum combined horizontal sync and back
+- * porch of 20 pixels and a minimum vertical back porch of 3 lines.
++ * porch of 20 pixels (when CMM isn't used) or 45 pixels (when CMM is
++ * used), and a minimum vertical back porch of 3 lines.
+ */
+- if (mode->htotal - mode->hsync_start < 20)
++ min_sync_porch = 20;
++ if (rcrtc->group->cmms_mask & BIT(rcrtc->index % 2))
++ min_sync_porch += 25;
++
++ if (mode->htotal - mode->hsync_start < min_sync_porch)
+ return MODE_HBLANK_NARROW;
+
+ vbp = (mode->vtotal - mode->vsync_end) / (interlaced ? 2 : 1);
+diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+index a2262bee5aa47..59c3d8ef6bf9a 100644
+--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
++++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+@@ -268,6 +268,8 @@ struct dw_mipi_dsi_rockchip {
+ struct dw_mipi_dsi *dmd;
+ const struct rockchip_dw_dsi_chip_data *cdata;
+ struct dw_mipi_dsi_plat_data pdata;
++
++ bool dsi_bound;
+ };
+
+ struct dphy_pll_parameter_map {
+@@ -773,10 +775,6 @@ static void dw_mipi_dsi_encoder_enable(struct drm_encoder *encoder)
+ if (mux < 0)
+ return;
+
+- pm_runtime_get_sync(dsi->dev);
+- if (dsi->slave)
+- pm_runtime_get_sync(dsi->slave->dev);
+-
+ /*
+ * For the RK3399, the clk of grf must be enabled before writing grf
+ * register. And for RK3288 or other soc, this grf_clk must be NULL,
+@@ -795,20 +793,10 @@ static void dw_mipi_dsi_encoder_enable(struct drm_encoder *encoder)
+ clk_disable_unprepare(dsi->grf_clk);
+ }
+
+-static void dw_mipi_dsi_encoder_disable(struct drm_encoder *encoder)
+-{
+- struct dw_mipi_dsi_rockchip *dsi = to_dsi(encoder);
+-
+- if (dsi->slave)
+- pm_runtime_put(dsi->slave->dev);
+- pm_runtime_put(dsi->dev);
+-}
+-
+ static const struct drm_encoder_helper_funcs
+ dw_mipi_dsi_encoder_helper_funcs = {
+ .atomic_check = dw_mipi_dsi_encoder_atomic_check,
+ .enable = dw_mipi_dsi_encoder_enable,
+- .disable = dw_mipi_dsi_encoder_disable,
+ };
+
+ static int rockchip_dsi_drm_create_encoder(struct dw_mipi_dsi_rockchip *dsi,
+@@ -938,10 +926,14 @@ static int dw_mipi_dsi_rockchip_bind(struct device *dev,
+ put_device(second);
+ }
+
++ pm_runtime_get_sync(dsi->dev);
++ if (dsi->slave)
++ pm_runtime_get_sync(dsi->slave->dev);
++
+ ret = clk_prepare_enable(dsi->pllref_clk);
+ if (ret) {
+ DRM_DEV_ERROR(dev, "Failed to enable pllref_clk: %d\n", ret);
+- return ret;
++ goto out_pm_runtime;
+ }
+
+ /*
+@@ -953,7 +945,7 @@ static int dw_mipi_dsi_rockchip_bind(struct device *dev,
+ ret = clk_prepare_enable(dsi->grf_clk);
+ if (ret) {
+ DRM_DEV_ERROR(dsi->dev, "Failed to enable grf_clk: %d\n", ret);
+- return ret;
++ goto out_pll_clk;
+ }
+
+ dw_mipi_dsi_rockchip_config(dsi);
+@@ -965,16 +957,27 @@ static int dw_mipi_dsi_rockchip_bind(struct device *dev,
+ ret = rockchip_dsi_drm_create_encoder(dsi, drm_dev);
+ if (ret) {
+ DRM_DEV_ERROR(dev, "Failed to create drm encoder\n");
+- return ret;
++ goto out_pll_clk;
+ }
+
+ ret = dw_mipi_dsi_bind(dsi->dmd, &dsi->encoder);
+ if (ret) {
+ DRM_DEV_ERROR(dev, "Failed to bind: %d\n", ret);
+- return ret;
++ goto out_pll_clk;
+ }
+
++ dsi->dsi_bound = true;
++
+ return 0;
++
++out_pll_clk:
++ clk_disable_unprepare(dsi->pllref_clk);
++out_pm_runtime:
++ pm_runtime_put(dsi->dev);
++ if (dsi->slave)
++ pm_runtime_put(dsi->slave->dev);
++
++ return ret;
+ }
+
+ static void dw_mipi_dsi_rockchip_unbind(struct device *dev,
+@@ -986,9 +989,15 @@ static void dw_mipi_dsi_rockchip_unbind(struct device *dev,
+ if (dsi->is_slave)
+ return;
+
++ dsi->dsi_bound = false;
++
+ dw_mipi_dsi_unbind(dsi->dmd);
+
+ clk_disable_unprepare(dsi->pllref_clk);
++
++ pm_runtime_put(dsi->dev);
++ if (dsi->slave)
++ pm_runtime_put(dsi->slave->dev);
+ }
+
+ static const struct component_ops dw_mipi_dsi_rockchip_ops = {
+@@ -1276,6 +1285,36 @@ static const struct phy_ops dw_mipi_dsi_dphy_ops = {
+ .exit = dw_mipi_dsi_dphy_exit,
+ };
+
++static int __maybe_unused dw_mipi_dsi_rockchip_resume(struct device *dev)
++{
++ struct dw_mipi_dsi_rockchip *dsi = dev_get_drvdata(dev);
++ int ret;
++
++ /*
++ * Re-configure DSI state, if we were previously initialized. We need
++ * to do this before rockchip_drm_drv tries to re-enable() any panels.
++ */
++ if (dsi->dsi_bound) {
++ ret = clk_prepare_enable(dsi->grf_clk);
++ if (ret) {
++ DRM_DEV_ERROR(dsi->dev, "Failed to enable grf_clk: %d\n", ret);
++ return ret;
++ }
++
++ dw_mipi_dsi_rockchip_config(dsi);
++ if (dsi->slave)
++ dw_mipi_dsi_rockchip_config(dsi->slave);
++
++ clk_disable_unprepare(dsi->grf_clk);
++ }
++
++ return 0;
++}
++
++static const struct dev_pm_ops dw_mipi_dsi_rockchip_pm_ops = {
++ SET_LATE_SYSTEM_SLEEP_PM_OPS(NULL, dw_mipi_dsi_rockchip_resume)
++};
++
+ static int dw_mipi_dsi_rockchip_probe(struct platform_device *pdev)
+ {
+ struct device *dev = &pdev->dev;
+@@ -1397,14 +1436,10 @@ static int dw_mipi_dsi_rockchip_probe(struct platform_device *pdev)
+ if (ret != -EPROBE_DEFER)
+ DRM_DEV_ERROR(dev,
+ "Failed to probe dw_mipi_dsi: %d\n", ret);
+- goto err_clkdisable;
++ return ret;
+ }
+
+ return 0;
+-
+-err_clkdisable:
+- clk_disable_unprepare(dsi->pllref_clk);
+- return ret;
+ }
+
+ static int dw_mipi_dsi_rockchip_remove(struct platform_device *pdev)
+@@ -1593,6 +1628,7 @@ struct platform_driver dw_mipi_dsi_rockchip_driver = {
+ .remove = dw_mipi_dsi_rockchip_remove,
+ .driver = {
+ .of_match_table = dw_mipi_dsi_rockchip_dt_ids,
++ .pm = &dw_mipi_dsi_rockchip_pm_ops,
+ .name = "dw-mipi-dsi-rockchip",
+ },
+ };
+diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+index 830bdd5e9b7ce..8677c82716784 100644
+--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
++++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+@@ -529,13 +529,6 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
+ return ret;
+ }
+
+- ret = clk_prepare_enable(hdmi->vpll_clk);
+- if (ret) {
+- DRM_DEV_ERROR(hdmi->dev, "Failed to enable HDMI vpll: %d\n",
+- ret);
+- return ret;
+- }
+-
+ hdmi->phy = devm_phy_optional_get(dev, "hdmi");
+ if (IS_ERR(hdmi->phy)) {
+ ret = PTR_ERR(hdmi->phy);
+@@ -544,6 +537,13 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
+ return ret;
+ }
+
++ ret = clk_prepare_enable(hdmi->vpll_clk);
++ if (ret) {
++ DRM_DEV_ERROR(hdmi->dev, "Failed to enable HDMI vpll: %d\n",
++ ret);
++ return ret;
++ }
++
+ drm_encoder_helper_add(encoder, &dw_hdmi_rockchip_encoder_helper_funcs);
+ drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+
+diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+index ca7cc82125cbc..8c873fcd0e99f 100644
+--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
++++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+@@ -902,6 +902,7 @@ static const struct vop_win_phy rk3399_win01_data = {
+ .enable = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 0),
+ .format = VOP_REG(RK3288_WIN0_CTRL0, 0x7, 1),
+ .rb_swap = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 12),
++ .x_mir_en = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 21),
+ .y_mir_en = VOP_REG(RK3288_WIN0_CTRL0, 0x1, 22),
+ .act_info = VOP_REG(RK3288_WIN0_ACT_INFO, 0x1fff1fff, 0),
+ .dsp_info = VOP_REG(RK3288_WIN0_DSP_INFO, 0x0fff0fff, 0),
+@@ -912,6 +913,7 @@ static const struct vop_win_phy rk3399_win01_data = {
+ .uv_vir = VOP_REG(RK3288_WIN0_VIR, 0x3fff, 16),
+ .src_alpha_ctl = VOP_REG(RK3288_WIN0_SRC_ALPHA_CTRL, 0xff, 0),
+ .dst_alpha_ctl = VOP_REG(RK3288_WIN0_DST_ALPHA_CTRL, 0xff, 0),
++ .channel = VOP_REG(RK3288_WIN0_CTRL2, 0xff, 0),
+ };
+
+ /*
+@@ -922,11 +924,11 @@ static const struct vop_win_phy rk3399_win01_data = {
+ static const struct vop_win_data rk3399_vop_win_data[] = {
+ { .base = 0x00, .phy = &rk3399_win01_data,
+ .type = DRM_PLANE_TYPE_PRIMARY },
+- { .base = 0x40, .phy = &rk3288_win01_data,
++ { .base = 0x40, .phy = &rk3368_win01_data,
+ .type = DRM_PLANE_TYPE_OVERLAY },
+- { .base = 0x00, .phy = &rk3288_win23_data,
++ { .base = 0x00, .phy = &rk3368_win23_data,
+ .type = DRM_PLANE_TYPE_OVERLAY },
+- { .base = 0x50, .phy = &rk3288_win23_data,
++ { .base = 0x50, .phy = &rk3368_win23_data,
+ .type = DRM_PLANE_TYPE_CURSOR },
+ };
+
+diff --git a/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c b/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c
+index 6b4759ed6bfd4..c491429f1a029 100644
+--- a/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c
++++ b/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c
+@@ -131,8 +131,10 @@ sideband_msg_req_encode_decode(struct drm_dp_sideband_msg_req_body *in)
+ return false;
+
+ txmsg = kzalloc(sizeof(*txmsg), GFP_KERNEL);
+- if (!txmsg)
++ if (!txmsg) {
++ kfree(out);
+ return false;
++ }
+
+ drm_dp_encode_sideband_req(in, txmsg);
+ ret = drm_dp_decode_sideband_req(txmsg, out);
+diff --git a/drivers/gpu/drm/sun4i/Kconfig b/drivers/gpu/drm/sun4i/Kconfig
+index 5755f0432e774..8c796de53222c 100644
+--- a/drivers/gpu/drm/sun4i/Kconfig
++++ b/drivers/gpu/drm/sun4i/Kconfig
+@@ -46,6 +46,7 @@ config DRM_SUN6I_DSI
+ default MACH_SUN8I
+ select CRC_CCITT
+ select DRM_MIPI_DSI
++ select RESET_CONTROLLER
+ select PHY_SUN6I_MIPI_DPHY
+ help
+ Choose this option if you want have an Allwinner SoC with
+diff --git a/drivers/gpu/drm/sun4i/sun4i_frontend.c b/drivers/gpu/drm/sun4i/sun4i_frontend.c
+index edb60ae0a9b75..faecc2935039b 100644
+--- a/drivers/gpu/drm/sun4i/sun4i_frontend.c
++++ b/drivers/gpu/drm/sun4i/sun4i_frontend.c
+@@ -222,13 +222,11 @@ void sun4i_frontend_update_buffer(struct sun4i_frontend *frontend,
+
+ /* Set the physical address of the buffer in memory */
+ paddr = drm_fb_cma_get_gem_addr(fb, state, 0);
+- paddr -= PHYS_OFFSET;
+ DRM_DEBUG_DRIVER("Setting buffer #0 address to %pad\n", &paddr);
+ regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR0_REG, paddr);
+
+ if (fb->format->num_planes > 1) {
+ paddr = drm_fb_cma_get_gem_addr(fb, state, swap ? 2 : 1);
+- paddr -= PHYS_OFFSET;
+ DRM_DEBUG_DRIVER("Setting buffer #1 address to %pad\n", &paddr);
+ regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR1_REG,
+ paddr);
+@@ -236,7 +234,6 @@ void sun4i_frontend_update_buffer(struct sun4i_frontend *frontend,
+
+ if (fb->format->num_planes > 2) {
+ paddr = drm_fb_cma_get_gem_addr(fb, state, swap ? 1 : 2);
+- paddr -= PHYS_OFFSET;
+ DRM_DEBUG_DRIVER("Setting buffer #2 address to %pad\n", &paddr);
+ regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR2_REG,
+ paddr);
+diff --git a/drivers/gpu/drm/sun4i/sun8i_csc.h b/drivers/gpu/drm/sun4i/sun8i_csc.h
+index a55a38ad849c1..022cafa6c06cb 100644
+--- a/drivers/gpu/drm/sun4i/sun8i_csc.h
++++ b/drivers/gpu/drm/sun4i/sun8i_csc.h
+@@ -16,8 +16,8 @@ struct sun8i_mixer;
+ #define CCSC10_OFFSET 0xA0000
+ #define CCSC11_OFFSET 0xF0000
+
+-#define SUN8I_CSC_CTRL(base) (base + 0x0)
+-#define SUN8I_CSC_COEFF(base, i) (base + 0x10 + 4 * i)
++#define SUN8I_CSC_CTRL(base) ((base) + 0x0)
++#define SUN8I_CSC_COEFF(base, i) ((base) + 0x10 + 4 * (i))
+
+ #define SUN8I_CSC_CTRL_EN BIT(0)
+
+diff --git a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c
+index b64d93da651d2..5e2b0175df36f 100644
+--- a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c
++++ b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c
+@@ -658,8 +658,10 @@ int sun8i_hdmi_phy_get(struct sun8i_dw_hdmi *hdmi, struct device_node *node)
+ return -EPROBE_DEFER;
+
+ phy = platform_get_drvdata(pdev);
+- if (!phy)
++ if (!phy) {
++ put_device(&pdev->dev);
+ return -EPROBE_DEFER;
++ }
+
+ hdmi->phy = phy;
+
+diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.h b/drivers/gpu/drm/sun4i/sun8i_mixer.h
+index 145833a9d82d4..5b3fbee186713 100644
+--- a/drivers/gpu/drm/sun4i/sun8i_mixer.h
++++ b/drivers/gpu/drm/sun4i/sun8i_mixer.h
+@@ -111,10 +111,10 @@
+ /* format 13 is semi-planar YUV411 VUVU */
+ #define SUN8I_MIXER_FBFMT_YUV411 14
+ /* format 15 doesn't exist */
+-/* format 16 is P010 YVU */
+-#define SUN8I_MIXER_FBFMT_P010_YUV 17
+-/* format 18 is P210 YVU */
+-#define SUN8I_MIXER_FBFMT_P210_YUV 19
++#define SUN8I_MIXER_FBFMT_P010_YUV 16
++/* format 17 is P010 YVU */
++#define SUN8I_MIXER_FBFMT_P210_YUV 18
++/* format 19 is P210 YVU */
+ /* format 20 is packed YVU444 10-bit */
+ /* format 21 is packed YUV444 10-bit */
+
+diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
+index 8d37d6b00562a..611cd8dad46ed 100644
+--- a/drivers/gpu/drm/tegra/drm.c
++++ b/drivers/gpu/drm/tegra/drm.c
+@@ -21,6 +21,10 @@
+ #include <drm/drm_prime.h>
+ #include <drm/drm_vblank.h>
+
++#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
++#include <asm/dma-iommu.h>
++#endif
++
+ #include "dc.h"
+ #include "drm.h"
+ #include "gem.h"
+@@ -936,6 +940,17 @@ int host1x_client_iommu_attach(struct host1x_client *client)
+ struct iommu_group *group = NULL;
+ int err;
+
++#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
++ if (client->dev->archdata.mapping) {
++ struct dma_iommu_mapping *mapping =
++ to_dma_iommu_mapping(client->dev);
++ arm_iommu_detach_device(client->dev);
++ arm_iommu_release_mapping(mapping);
++
++ domain = iommu_get_domain_for_dev(client->dev);
++ }
++#endif
++
+ /*
+ * If the host1x client is already attached to an IOMMU domain that is
+ * not the shared IOMMU domain, don't try to attach it to a different
+diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
+index f46d377f0c304..de1333dc0d867 100644
+--- a/drivers/gpu/drm/tegra/dsi.c
++++ b/drivers/gpu/drm/tegra/dsi.c
+@@ -1538,8 +1538,10 @@ static int tegra_dsi_ganged_probe(struct tegra_dsi *dsi)
+ dsi->slave = platform_get_drvdata(gangster);
+ of_node_put(np);
+
+- if (!dsi->slave)
++ if (!dsi->slave) {
++ put_device(&gangster->dev);
+ return -EPROBE_DEFER;
++ }
+
+ dsi->slave->master = dsi;
+ }
+diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c
+index de288cba39055..ba3722f1b8651 100644
+--- a/drivers/gpu/drm/tegra/gr2d.c
++++ b/drivers/gpu/drm/tegra/gr2d.c
+@@ -4,9 +4,11 @@
+ */
+
+ #include <linux/clk.h>
++#include <linux/delay.h>
+ #include <linux/iommu.h>
+ #include <linux/module.h>
+ #include <linux/of_device.h>
++#include <linux/reset.h>
+
+ #include "drm.h"
+ #include "gem.h"
+@@ -19,6 +21,7 @@ struct gr2d_soc {
+ struct gr2d {
+ struct tegra_drm_client client;
+ struct host1x_channel *channel;
++ struct reset_control *rst;
+ struct clk *clk;
+
+ const struct gr2d_soc *soc;
+@@ -208,6 +211,12 @@ static int gr2d_probe(struct platform_device *pdev)
+ if (!syncpts)
+ return -ENOMEM;
+
++ gr2d->rst = devm_reset_control_get(dev, NULL);
++ if (IS_ERR(gr2d->rst)) {
++ dev_err(dev, "cannot get reset\n");
++ return PTR_ERR(gr2d->rst);
++ }
++
+ gr2d->clk = devm_clk_get(dev, NULL);
+ if (IS_ERR(gr2d->clk)) {
+ dev_err(dev, "cannot get clock\n");
+@@ -220,6 +229,14 @@ static int gr2d_probe(struct platform_device *pdev)
+ return err;
+ }
+
++ usleep_range(2000, 4000);
++
++ err = reset_control_deassert(gr2d->rst);
++ if (err < 0) {
++ dev_err(dev, "failed to deassert reset: %d\n", err);
++ goto disable_clk;
++ }
++
+ INIT_LIST_HEAD(&gr2d->client.base.list);
+ gr2d->client.base.ops = &gr2d_client_ops;
+ gr2d->client.base.dev = dev;
+@@ -234,8 +251,7 @@ static int gr2d_probe(struct platform_device *pdev)
+ err = host1x_client_register(&gr2d->client.base);
+ if (err < 0) {
+ dev_err(dev, "failed to register host1x client: %d\n", err);
+- clk_disable_unprepare(gr2d->clk);
+- return err;
++ goto assert_rst;
+ }
+
+ /* initialize address register map */
+@@ -245,6 +261,13 @@ static int gr2d_probe(struct platform_device *pdev)
+ platform_set_drvdata(pdev, gr2d);
+
+ return 0;
++
++assert_rst:
++ (void)reset_control_assert(gr2d->rst);
++disable_clk:
++ clk_disable_unprepare(gr2d->clk);
++
++ return err;
+ }
+
+ static int gr2d_remove(struct platform_device *pdev)
+@@ -259,6 +282,12 @@ static int gr2d_remove(struct platform_device *pdev)
+ return err;
+ }
+
++ err = reset_control_assert(gr2d->rst);
++ if (err < 0)
++ dev_err(&pdev->dev, "failed to assert reset: %d\n", err);
++
++ usleep_range(2000, 4000);
++
+ clk_disable_unprepare(gr2d->clk);
+
+ return 0;
+diff --git a/drivers/gpu/drm/tegra/submit.c b/drivers/gpu/drm/tegra/submit.c
+index 776f825df52fa..aba9d0c9d9031 100644
+--- a/drivers/gpu/drm/tegra/submit.c
++++ b/drivers/gpu/drm/tegra/submit.c
+@@ -475,8 +475,10 @@ static void release_job(struct host1x_job *job)
+ kfree(job_data->used_mappings);
+ kfree(job_data);
+
+- if (pm_runtime_enabled(client->base.dev))
++ if (pm_runtime_enabled(client->base.dev)) {
++ pm_runtime_mark_last_busy(client->base.dev);
+ pm_runtime_put_autosuspend(client->base.dev);
++ }
+ }
+
+ int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
+diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
+index c02010ff2b7f2..da4af53719917 100644
+--- a/drivers/gpu/drm/tegra/vic.c
++++ b/drivers/gpu/drm/tegra/vic.c
+@@ -5,6 +5,7 @@
+
+ #include <linux/clk.h>
+ #include <linux/delay.h>
++#include <linux/dma-mapping.h>
+ #include <linux/host1x.h>
+ #include <linux/iommu.h>
+ #include <linux/module.h>
+@@ -232,10 +233,8 @@ static int vic_load_firmware(struct vic *vic)
+
+ if (!client->group) {
+ virt = dma_alloc_coherent(vic->dev, size, &iova, GFP_KERNEL);
+-
+- err = dma_mapping_error(vic->dev, iova);
+- if (err < 0)
+- return err;
++ if (!virt)
++ return -ENOMEM;
+ } else {
+ virt = tegra_drm_alloc(tegra, size, &iova);
+ }
+diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
+index 481b48bde0473..3e3f9ba1e8858 100644
+--- a/drivers/gpu/drm/tiny/simpledrm.c
++++ b/drivers/gpu/drm/tiny/simpledrm.c
+@@ -458,7 +458,7 @@ static struct drm_display_mode simpledrm_mode(unsigned int width,
+ {
+ struct drm_display_mode mode = { SIMPLEDRM_MODE(width, height) };
+
+- mode.clock = 60 /* Hz */ * mode.hdisplay * mode.vdisplay;
++ mode.clock = mode.hdisplay * mode.vdisplay * 60 / 1000 /* kHz */;
+ drm_mode_set_name(&mode);
+
+ return mode;
+@@ -779,6 +779,9 @@ static int simpledrm_device_init_modeset(struct simpledrm_device *sdev)
+ if (ret)
+ return ret;
+ drm_connector_helper_add(connector, &simpledrm_connector_helper_funcs);
++ drm_connector_set_panel_orientation_with_quirk(connector,
++ DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
++ mode->hdisplay, mode->vdisplay);
+
+ formats = simpledrm_device_formats(sdev, &nformats);
+
+diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
+index bb9e02c31946e..900edaf5d68ee 100644
+--- a/drivers/gpu/drm/ttm/ttm_bo.c
++++ b/drivers/gpu/drm/ttm/ttm_bo.c
+@@ -724,6 +724,8 @@ int ttm_mem_evict_first(struct ttm_device *bdev,
+ ret = ttm_bo_evict(bo, ctx);
+ if (locked)
+ ttm_bo_unreserve(bo);
++ else
++ ttm_bo_move_to_lru_tail_unlocked(bo);
+
+ ttm_bo_put(bo);
+ return ret;
+diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
+index f56be5bc0861e..4a655ab23c89d 100644
+--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
++++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
+@@ -171,89 +171,6 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
+ }
+ EXPORT_SYMBOL(ttm_bo_vm_reserve);
+
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+-/**
+- * ttm_bo_vm_insert_huge - Insert a pfn for PUD or PMD faults
+- * @vmf: Fault data
+- * @bo: The buffer object
+- * @page_offset: Page offset from bo start
+- * @fault_page_size: The size of the fault in pages.
+- * @pgprot: The page protections.
+- * Does additional checking whether it's possible to insert a PUD or PMD
+- * pfn and performs the insertion.
+- *
+- * Return: VM_FAULT_NOPAGE on successful insertion, VM_FAULT_FALLBACK if
+- * a huge fault was not possible, or on insertion error.
+- */
+-static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
+- struct ttm_buffer_object *bo,
+- pgoff_t page_offset,
+- pgoff_t fault_page_size,
+- pgprot_t pgprot)
+-{
+- pgoff_t i;
+- vm_fault_t ret;
+- unsigned long pfn;
+- pfn_t pfnt;
+- struct ttm_tt *ttm = bo->ttm;
+- bool write = vmf->flags & FAULT_FLAG_WRITE;
+-
+- /* Fault should not cross bo boundary. */
+- page_offset &= ~(fault_page_size - 1);
+- if (page_offset + fault_page_size > bo->resource->num_pages)
+- goto out_fallback;
+-
+- if (bo->resource->bus.is_iomem)
+- pfn = ttm_bo_io_mem_pfn(bo, page_offset);
+- else
+- pfn = page_to_pfn(ttm->pages[page_offset]);
+-
+- /* pfn must be fault_page_size aligned. */
+- if ((pfn & (fault_page_size - 1)) != 0)
+- goto out_fallback;
+-
+- /* Check that memory is contiguous. */
+- if (!bo->resource->bus.is_iomem) {
+- for (i = 1; i < fault_page_size; ++i) {
+- if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i)
+- goto out_fallback;
+- }
+- } else if (bo->bdev->funcs->io_mem_pfn) {
+- for (i = 1; i < fault_page_size; ++i) {
+- if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i)
+- goto out_fallback;
+- }
+- }
+-
+- pfnt = __pfn_to_pfn_t(pfn, PFN_DEV);
+- if (fault_page_size == (HPAGE_PMD_SIZE >> PAGE_SHIFT))
+- ret = vmf_insert_pfn_pmd_prot(vmf, pfnt, pgprot, write);
+-#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+- else if (fault_page_size == (HPAGE_PUD_SIZE >> PAGE_SHIFT))
+- ret = vmf_insert_pfn_pud_prot(vmf, pfnt, pgprot, write);
+-#endif
+- else
+- WARN_ON_ONCE(ret = VM_FAULT_FALLBACK);
+-
+- if (ret != VM_FAULT_NOPAGE)
+- goto out_fallback;
+-
+- return VM_FAULT_NOPAGE;
+-out_fallback:
+- count_vm_event(THP_FAULT_FALLBACK);
+- return VM_FAULT_FALLBACK;
+-}
+-#else
+-static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
+- struct ttm_buffer_object *bo,
+- pgoff_t page_offset,
+- pgoff_t fault_page_size,
+- pgprot_t pgprot)
+-{
+- return VM_FAULT_FALLBACK;
+-}
+-#endif
+-
+ /**
+ * ttm_bo_vm_fault_reserved - TTM fault helper
+ * @vmf: The struct vm_fault given as argument to the fault callback
+@@ -261,7 +178,6 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
+ * @num_prefault: Maximum number of prefault pages. The caller may want to
+ * specify this based on madvice settings and the size of the GPU object
+ * backed by the memory.
+- * @fault_page_size: The size of the fault in pages.
+ *
+ * This function inserts one or more page table entries pointing to the
+ * memory backing the buffer object, and then returns a return code
+@@ -275,8 +191,7 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
+ */
+ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
+ pgprot_t prot,
+- pgoff_t num_prefault,
+- pgoff_t fault_page_size)
++ pgoff_t num_prefault)
+ {
+ struct vm_area_struct *vma = vmf->vma;
+ struct ttm_buffer_object *bo = vma->vm_private_data;
+@@ -327,11 +242,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
+ prot = pgprot_decrypted(prot);
+ }
+
+- /* We don't prefault on huge faults. Yet. */
+- if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && fault_page_size != 1)
+- return ttm_bo_vm_insert_huge(vmf, bo, page_offset,
+- fault_page_size, prot);
+-
+ /*
+ * Speculatively prefault a number of pages. Only error on
+ * first page.
+@@ -429,7 +339,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
+
+ prot = vma->vm_page_prot;
+ if (drm_dev_enter(ddev, &idx)) {
+- ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1);
++ ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
+ drm_dev_exit(idx);
+ } else {
+ ret = ttm_bo_vm_dummy_page(vmf, prot);
+@@ -519,11 +429,6 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
+
+ switch (bo->resource->mem_type) {
+ case TTM_PL_SYSTEM:
+- if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
+- ret = ttm_tt_swapin(bo->ttm);
+- if (unlikely(ret != 0))
+- return ret;
+- }
+ fallthrough;
+ case TTM_PL_TT:
+ ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);
+diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
+index 3750fd2161317..930574ad2bca9 100644
+--- a/drivers/gpu/drm/udl/udl_connector.c
++++ b/drivers/gpu/drm/udl/udl_connector.c
+@@ -30,7 +30,7 @@ static int udl_get_edid_block(void *data, u8 *buf, unsigned int block,
+ int bval = (i + block * EDID_LENGTH) << 8;
+ ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ 0x02, (0x80 | (0x02 << 5)), bval,
+- 0xA1, read_buff, 2, HZ);
++ 0xA1, read_buff, 2, 1000);
+ if (ret < 1) {
+ DRM_ERROR("Read EDID byte %d failed err %x\n", i, ret);
+ kfree(read_buff);
+diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
+index 9403c3b36aca6..6407a006d6ec4 100644
+--- a/drivers/gpu/drm/v3d/v3d_drv.c
++++ b/drivers/gpu/drm/v3d/v3d_drv.c
+@@ -221,6 +221,7 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
+ int ret;
+ u32 mmu_debug;
+ u32 ident1;
++ u64 mask;
+
+
+ v3d = devm_drm_dev_alloc(dev, &v3d_drm_driver, struct v3d_dev, drm);
+@@ -240,8 +241,11 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
+ return ret;
+
+ mmu_debug = V3D_READ(V3D_MMU_DEBUG_INFO);
+- dma_set_mask_and_coherent(dev,
+- DMA_BIT_MASK(30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_PA_WIDTH)));
++ mask = DMA_BIT_MASK(30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_PA_WIDTH));
++ ret = dma_set_mask_and_coherent(dev, mask);
++ if (ret)
++ return ret;
++
+ v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH);
+
+ ident1 = V3D_READ(V3D_HUB_IDENT1);
+diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
+index 5689da118197e..805d6f6cba0e2 100644
+--- a/drivers/gpu/drm/v3d/v3d_gem.c
++++ b/drivers/gpu/drm/v3d/v3d_gem.c
+@@ -197,8 +197,8 @@ v3d_clean_caches(struct v3d_dev *v3d)
+
+ V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, V3D_L2TCACTL_TMUWCF);
+ if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
+- V3D_L2TCACTL_L2TFLS), 100)) {
+- DRM_ERROR("Timeout waiting for L1T write combiner flush\n");
++ V3D_L2TCACTL_TMUWCF), 100)) {
++ DRM_ERROR("Timeout waiting for TMU write combiner flush\n");
+ }
+
+ mutex_lock(&v3d->cache_clean_lock);
+@@ -625,7 +625,7 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
+
+ if (!render->base.perfmon) {
+ ret = -ENOENT;
+- goto fail;
++ goto fail_perfmon;
+ }
+ }
+
+@@ -678,6 +678,7 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
+
+ fail_unreserve:
+ mutex_unlock(&v3d->sched_lock);
++fail_perfmon:
+ drm_gem_unlock_reservations(last_job->bo,
+ last_job->bo_count, &acquire_ctx);
+ fail:
+@@ -854,7 +855,7 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
+ args->perfmon_id);
+ if (!job->base.perfmon) {
+ ret = -ENOENT;
+- goto fail;
++ goto fail_perfmon;
+ }
+ }
+
+@@ -886,6 +887,7 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
+
+ fail_unreserve:
+ mutex_unlock(&v3d->sched_lock);
++fail_perfmon:
+ drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
+ &acquire_ctx);
+ fail:
+diff --git a/drivers/gpu/drm/vboxvideo/vbox_main.c b/drivers/gpu/drm/vboxvideo/vbox_main.c
+index f28779715ccda..c9e8b3a63c621 100644
+--- a/drivers/gpu/drm/vboxvideo/vbox_main.c
++++ b/drivers/gpu/drm/vboxvideo/vbox_main.c
+@@ -127,8 +127,8 @@ int vbox_hw_init(struct vbox_private *vbox)
+ /* Create guest-heap mem-pool use 2^4 = 16 byte chunks */
+ vbox->guest_pool = devm_gen_pool_create(vbox->ddev.dev, 4, -1,
+ "vboxvideo-accel");
+- if (!vbox->guest_pool)
+- return -ENOMEM;
++ if (IS_ERR(vbox->guest_pool))
++ return PTR_ERR(vbox->guest_pool);
+
+ ret = gen_pool_add_virt(vbox->guest_pool,
+ (unsigned long)vbox->guest_heap,
+diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
+index fddaeb0b09c11..f642bd6e71ff4 100644
+--- a/drivers/gpu/drm/vc4/vc4_bo.c
++++ b/drivers/gpu/drm/vc4/vc4_bo.c
+@@ -391,7 +391,7 @@ struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
+
+ bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+ if (!bo)
+- return ERR_PTR(-ENOMEM);
++ return NULL;
+
+ bo->madv = VC4_MADV_WILLNEED;
+ refcount_set(&bo->usecnt, 0);
+diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
+index 18f5009ce90e3..3e61184e194c9 100644
+--- a/drivers/gpu/drm/vc4/vc4_crtc.c
++++ b/drivers/gpu/drm/vc4/vc4_crtc.c
+@@ -32,6 +32,7 @@
+ #include <linux/clk.h>
+ #include <linux/component.h>
+ #include <linux/of_device.h>
++#include <linux/pm_runtime.h>
+
+ #include <drm/drm_atomic.h>
+ #include <drm/drm_atomic_helper.h>
+@@ -42,6 +43,7 @@
+ #include <drm/drm_vblank.h>
+
+ #include "vc4_drv.h"
++#include "vc4_hdmi.h"
+ #include "vc4_regs.h"
+
+ #define HVS_FIFO_LATENCY_PIX 6
+@@ -496,8 +498,10 @@ int vc4_crtc_disable_at_boot(struct drm_crtc *crtc)
+ enum vc4_encoder_type encoder_type;
+ const struct vc4_pv_data *pv_data;
+ struct drm_encoder *encoder;
++ struct vc4_hdmi *vc4_hdmi;
+ unsigned encoder_sel;
+ int channel;
++ int ret;
+
+ if (!(of_device_is_compatible(vc4_crtc->pdev->dev.of_node,
+ "brcm,bcm2711-pixelvalve2") ||
+@@ -525,7 +529,22 @@ int vc4_crtc_disable_at_boot(struct drm_crtc *crtc)
+ if (WARN_ON(!encoder))
+ return 0;
+
+- return vc4_crtc_disable(crtc, encoder, NULL, channel);
++ vc4_hdmi = encoder_to_vc4_hdmi(encoder);
++ ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
++ if (ret)
++ return ret;
++
++ ret = vc4_crtc_disable(crtc, encoder, NULL, channel);
++ if (ret)
++ return ret;
++
++ /*
++ * post_crtc_powerdown will have called pm_runtime_put, so we
++ * don't need it here otherwise we'll get the reference counting
++ * wrong.
++ */
++
++ return 0;
+ }
+
+ static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
+@@ -691,14 +710,14 @@ static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
+ struct drm_crtc *crtc = &vc4_crtc->base;
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+- struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
+- u32 chan = vc4_state->assigned_channel;
++ u32 chan = vc4_crtc->current_hvs_channel;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->event_lock, flags);
++ spin_lock(&vc4_crtc->irq_lock);
+ if (vc4_crtc->event &&
+- (vc4_state->mm.start == HVS_READ(SCALER_DISPLACTX(chan)) ||
+- vc4_state->feed_txp)) {
++ (vc4_crtc->current_dlist == HVS_READ(SCALER_DISPLACTX(chan)) ||
++ vc4_crtc->feeds_txp)) {
+ drm_crtc_send_vblank_event(crtc, vc4_crtc->event);
+ vc4_crtc->event = NULL;
+ drm_crtc_vblank_put(crtc);
+@@ -711,6 +730,7 @@ static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
+ */
+ vc4_hvs_unmask_underrun(dev, chan);
+ }
++ spin_unlock(&vc4_crtc->irq_lock);
+ spin_unlock_irqrestore(&dev->event_lock, flags);
+ }
+
+@@ -876,7 +896,6 @@ struct drm_crtc_state *vc4_crtc_duplicate_state(struct drm_crtc *crtc)
+ return NULL;
+
+ old_vc4_state = to_vc4_crtc_state(crtc->state);
+- vc4_state->feed_txp = old_vc4_state->feed_txp;
+ vc4_state->margins = old_vc4_state->margins;
+ vc4_state->assigned_channel = old_vc4_state->assigned_channel;
+
+@@ -937,6 +956,7 @@ static const struct drm_crtc_funcs vc4_crtc_funcs = {
+ static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
+ .mode_valid = vc4_crtc_mode_valid,
+ .atomic_check = vc4_crtc_atomic_check,
++ .atomic_begin = vc4_hvs_atomic_begin,
+ .atomic_flush = vc4_hvs_atomic_flush,
+ .atomic_enable = vc4_crtc_atomic_enable,
+ .atomic_disable = vc4_crtc_atomic_disable,
+@@ -1111,6 +1131,7 @@ int vc4_crtc_init(struct drm_device *drm, struct vc4_crtc *vc4_crtc,
+ return PTR_ERR(primary_plane);
+ }
+
++ spin_lock_init(&vc4_crtc->irq_lock);
+ drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
+ crtc_funcs, NULL);
+ drm_crtc_helper_add(crtc, crtc_helper_funcs);
+diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
+index ef73e0aaf7261..4b550ebd9572d 100644
+--- a/drivers/gpu/drm/vc4/vc4_drv.h
++++ b/drivers/gpu/drm/vc4/vc4_drv.h
+@@ -495,6 +495,33 @@ struct vc4_crtc {
+ struct drm_pending_vblank_event *event;
+
+ struct debugfs_regset32 regset;
++
++ /**
++ * @feeds_txp: True if the CRTC feeds our writeback controller.
++ */
++ bool feeds_txp;
++
++ /**
++ * @irq_lock: Spinlock protecting the resources shared between
++ * the atomic code and our vblank handler.
++ */
++ spinlock_t irq_lock;
++
++ /**
++ * @current_dlist: Start offset of the display list currently
++ * set in the HVS for that CRTC. Protected by @irq_lock, and
++ * copied in vc4_hvs_update_dlist() for the CRTC interrupt
++ * handler to have access to that value.
++ */
++ unsigned int current_dlist;
++
++ /**
++ * @current_hvs_channel: HVS channel currently assigned to the
++ * CRTC. Protected by @irq_lock, and copied in
++ * vc4_hvs_atomic_begin() for the CRTC interrupt handler to have
++ * access to that value.
++ */
++ unsigned int current_hvs_channel;
+ };
+
+ static inline struct vc4_crtc *
+@@ -521,7 +548,6 @@ struct vc4_crtc_state {
+ struct drm_crtc_state base;
+ /* Dlist area for this CRTC configuration. */
+ struct drm_mm_node mm;
+- bool feed_txp;
+ bool txp_armed;
+ unsigned int assigned_channel;
+
+@@ -908,6 +934,7 @@ extern struct platform_driver vc4_hvs_driver;
+ void vc4_hvs_stop_channel(struct drm_device *dev, unsigned int output);
+ int vc4_hvs_get_fifo_from_output(struct drm_device *dev, unsigned int output);
+ int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state);
++void vc4_hvs_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_state *state);
+ void vc4_hvs_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state);
+ void vc4_hvs_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state);
+ void vc4_hvs_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state);
+diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
+index a185027911ce5..ca85063166609 100644
+--- a/drivers/gpu/drm/vc4/vc4_dsi.c
++++ b/drivers/gpu/drm/vc4/vc4_dsi.c
+@@ -846,7 +846,7 @@ static void vc4_dsi_encoder_enable(struct drm_encoder *encoder)
+ unsigned long phy_clock;
+ int ret;
+
+- ret = pm_runtime_get_sync(dev);
++ ret = pm_runtime_resume_and_get(dev);
+ if (ret) {
+ DRM_ERROR("Failed to runtime PM enable on DSI%d\n", dsi->variant->port);
+ return;
+@@ -1262,7 +1262,6 @@ static int vc4_dsi_host_attach(struct mipi_dsi_host *host,
+ struct mipi_dsi_device *device)
+ {
+ struct vc4_dsi *dsi = host_to_dsi(host);
+- int ret;
+
+ dsi->lanes = device->lanes;
+ dsi->channel = device->channel;
+@@ -1297,18 +1296,15 @@ static int vc4_dsi_host_attach(struct mipi_dsi_host *host,
+ return 0;
+ }
+
+- ret = component_add(&dsi->pdev->dev, &vc4_dsi_ops);
+- if (ret) {
+- mipi_dsi_host_unregister(&dsi->dsi_host);
+- return ret;
+- }
+-
+- return 0;
++ return component_add(&dsi->pdev->dev, &vc4_dsi_ops);
+ }
+
+ static int vc4_dsi_host_detach(struct mipi_dsi_host *host,
+ struct mipi_dsi_device *device)
+ {
++ struct vc4_dsi *dsi = host_to_dsi(host);
++
++ component_del(&dsi->pdev->dev, &vc4_dsi_ops);
+ return 0;
+ }
+
+@@ -1706,9 +1702,7 @@ static int vc4_dsi_dev_remove(struct platform_device *pdev)
+ struct device *dev = &pdev->dev;
+ struct vc4_dsi *dsi = dev_get_drvdata(dev);
+
+- component_del(&pdev->dev, &vc4_dsi_ops);
+ mipi_dsi_host_unregister(&dsi->dsi_host);
+-
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
+index ed8a4b7f8b6e2..ef7bea7c43a0f 100644
+--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
++++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
+@@ -38,6 +38,7 @@
+ #include <drm/drm_scdc_helper.h>
+ #include <linux/clk.h>
+ #include <linux/component.h>
++#include <linux/gpio/consumer.h>
+ #include <linux/i2c.h>
+ #include <linux/of_address.h>
+ #include <linux/of_gpio.h>
+@@ -94,6 +95,7 @@
+ # define VC4_HD_M_SW_RST BIT(2)
+ # define VC4_HD_M_ENABLE BIT(0)
+
++#define HSM_MIN_CLOCK_FREQ 120000000
+ #define CEC_CLOCK_FREQ 40000
+
+ #define HDMI_14_MAX_TMDS_CLK (340 * 1000 * 1000)
+@@ -161,12 +163,16 @@ static void vc4_hdmi_cec_update_clk_div(struct vc4_hdmi *vc4_hdmi)
+ static void vc4_hdmi_cec_update_clk_div(struct vc4_hdmi *vc4_hdmi) {}
+ #endif
+
++static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder);
++
+ static enum drm_connector_status
+ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
+ {
+ struct vc4_hdmi *vc4_hdmi = connector_to_vc4_hdmi(connector);
+ bool connected = false;
+
++ WARN_ON(pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev));
++
+ if (vc4_hdmi->hpd_gpio &&
+ gpiod_get_value_cansleep(vc4_hdmi->hpd_gpio)) {
+ connected = true;
+@@ -187,10 +193,13 @@ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
+ }
+ }
+
++ vc4_hdmi_enable_scrambling(&vc4_hdmi->encoder.base.base);
++ pm_runtime_put(&vc4_hdmi->pdev->dev);
+ return connector_status_connected;
+ }
+
+ cec_phys_addr_invalidate(vc4_hdmi->cec_adap);
++ pm_runtime_put(&vc4_hdmi->pdev->dev);
+ return connector_status_disconnected;
+ }
+
+@@ -627,7 +636,6 @@ static void vc4_hdmi_encoder_post_crtc_powerdown(struct drm_encoder *encoder,
+ vc4_hdmi->variant->phy_disable(vc4_hdmi);
+
+ clk_disable_unprepare(vc4_hdmi->pixel_bvb_clock);
+- clk_disable_unprepare(vc4_hdmi->hsm_clock);
+ clk_disable_unprepare(vc4_hdmi->pixel_clock);
+
+ ret = pm_runtime_put(&vc4_hdmi->pdev->dev);
+@@ -893,28 +901,10 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
+ conn_state_to_vc4_hdmi_conn_state(conn_state);
+ struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode;
+ struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+- unsigned long bvb_rate, pixel_rate, hsm_rate;
++ unsigned long pixel_rate = vc4_conn_state->pixel_rate;
++ unsigned long bvb_rate, hsm_rate;
+ int ret;
+
+- ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
+- if (ret < 0) {
+- DRM_ERROR("Failed to retain power domain: %d\n", ret);
+- return;
+- }
+-
+- pixel_rate = vc4_conn_state->pixel_rate;
+- ret = clk_set_rate(vc4_hdmi->pixel_clock, pixel_rate);
+- if (ret) {
+- DRM_ERROR("Failed to set pixel clock rate: %d\n", ret);
+- return;
+- }
+-
+- ret = clk_prepare_enable(vc4_hdmi->pixel_clock);
+- if (ret) {
+- DRM_ERROR("Failed to turn on pixel clock: %d\n", ret);
+- return;
+- }
+-
+ /*
+ * As stated in RPi's vc4 firmware "HDMI state machine (HSM) clock must
+ * be faster than pixel clock, infinitesimally faster, tested in
+@@ -938,13 +928,25 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
+ return;
+ }
+
+- ret = clk_prepare_enable(vc4_hdmi->hsm_clock);
+- if (ret) {
+- DRM_ERROR("Failed to turn on HSM clock: %d\n", ret);
+- clk_disable_unprepare(vc4_hdmi->pixel_clock);
++ ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
++ if (ret < 0) {
++ DRM_ERROR("Failed to retain power domain: %d\n", ret);
+ return;
+ }
+
++ ret = clk_set_rate(vc4_hdmi->pixel_clock, pixel_rate);
++ if (ret) {
++ DRM_ERROR("Failed to set pixel clock rate: %d\n", ret);
++ goto err_put_runtime_pm;
++ }
++
++ ret = clk_prepare_enable(vc4_hdmi->pixel_clock);
++ if (ret) {
++ DRM_ERROR("Failed to turn on pixel clock: %d\n", ret);
++ goto err_put_runtime_pm;
++ }
++
++
+ vc4_hdmi_cec_update_clk_div(vc4_hdmi);
+
+ if (pixel_rate > 297000000)
+@@ -957,17 +959,13 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
+ ret = clk_set_min_rate(vc4_hdmi->pixel_bvb_clock, bvb_rate);
+ if (ret) {
+ DRM_ERROR("Failed to set pixel bvb clock rate: %d\n", ret);
+- clk_disable_unprepare(vc4_hdmi->hsm_clock);
+- clk_disable_unprepare(vc4_hdmi->pixel_clock);
+- return;
++ goto err_disable_pixel_clock;
+ }
+
+ ret = clk_prepare_enable(vc4_hdmi->pixel_bvb_clock);
+ if (ret) {
+ DRM_ERROR("Failed to turn on pixel bvb clock: %d\n", ret);
+- clk_disable_unprepare(vc4_hdmi->hsm_clock);
+- clk_disable_unprepare(vc4_hdmi->pixel_clock);
+- return;
++ goto err_disable_pixel_clock;
+ }
+
+ if (vc4_hdmi->variant->phy_init)
+@@ -980,6 +978,15 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
+
+ if (vc4_hdmi->variant->set_timings)
+ vc4_hdmi->variant->set_timings(vc4_hdmi, conn_state, mode);
++
++ return;
++
++err_disable_pixel_clock:
++ clk_disable_unprepare(vc4_hdmi->pixel_clock);
++err_put_runtime_pm:
++ pm_runtime_put(&vc4_hdmi->pdev->dev);
++
++ return;
+ }
+
+ static void vc4_hdmi_encoder_pre_crtc_enable(struct drm_encoder *encoder,
+@@ -1084,6 +1091,7 @@ static int vc4_hdmi_encoder_atomic_check(struct drm_encoder *encoder,
+ unsigned long long tmds_rate;
+
+ if (vc4_hdmi->variant->unsupported_odd_h_timings &&
++ !(mode->flags & DRM_MODE_FLAG_DBLCLK) &&
+ ((mode->hdisplay % 2) || (mode->hsync_start % 2) ||
+ (mode->hsync_end % 2) || (mode->htotal % 2)))
+ return -EINVAL;
+@@ -1131,6 +1139,7 @@ vc4_hdmi_encoder_mode_valid(struct drm_encoder *encoder,
+ struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+
+ if (vc4_hdmi->variant->unsupported_odd_h_timings &&
++ !(mode->flags & DRM_MODE_FLAG_DBLCLK) &&
+ ((mode->hdisplay % 2) || (mode->hsync_start % 2) ||
+ (mode->hsync_end % 2) || (mode->htotal % 2)))
+ return MODE_H_ILLEGAL;
+@@ -1514,6 +1523,7 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
+ dev_err(dev, "Couldn't register the HDMI codec: %ld\n", PTR_ERR(codec_pdev));
+ return PTR_ERR(codec_pdev);
+ }
++ vc4_hdmi->audio.codec_pdev = codec_pdev;
+
+ dai_link->cpus = &vc4_hdmi->audio.cpu;
+ dai_link->codecs = &vc4_hdmi->audio.codec;
+@@ -1553,6 +1563,12 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
+
+ }
+
++static void vc4_hdmi_audio_exit(struct vc4_hdmi *vc4_hdmi)
++{
++ platform_device_unregister(vc4_hdmi->audio.codec_pdev);
++ vc4_hdmi->audio.codec_pdev = NULL;
++}
++
+ static irqreturn_t vc4_hdmi_hpd_irq_thread(int irq, void *priv)
+ {
+ struct vc4_hdmi *vc4_hdmi = priv;
+@@ -2098,6 +2114,27 @@ static int vc5_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
+ return 0;
+ }
+
++static int __maybe_unused vc4_hdmi_runtime_suspend(struct device *dev)
++{
++ struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
++
++ clk_disable_unprepare(vc4_hdmi->hsm_clock);
++
++ return 0;
++}
++
++static int vc4_hdmi_runtime_resume(struct device *dev)
++{
++ struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
++ int ret;
++
++ ret = clk_prepare_enable(vc4_hdmi->hsm_clock);
++ if (ret)
++ return ret;
++
++ return 0;
++}
++
+ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
+ {
+ const struct vc4_hdmi_variant *variant = of_device_get_match_data(dev);
+@@ -2161,6 +2198,31 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
+ vc4_hdmi->disable_4kp60 = true;
+ }
+
++ /*
++ * If we boot without any cable connected to the HDMI connector,
++ * the firmware will skip the HSM initialization and leave it
++ * with a rate of 0, resulting in a bus lockup when we're
++ * accessing the registers even if it's enabled.
++ *
++ * Let's put a sensible default at runtime_resume so that we
++ * don't end up in this situation.
++ */
++ ret = clk_set_min_rate(vc4_hdmi->hsm_clock, HSM_MIN_CLOCK_FREQ);
++ if (ret)
++ goto err_put_ddc;
++
++ /*
++ * We need to have the device powered up at this point to call
++ * our reset hook and for the CEC init.
++ */
++ ret = vc4_hdmi_runtime_resume(dev);
++ if (ret)
++ goto err_put_ddc;
++
++ pm_runtime_get_noresume(dev);
++ pm_runtime_set_active(dev);
++ pm_runtime_enable(dev);
++
+ if (vc4_hdmi->variant->reset)
+ vc4_hdmi->variant->reset(vc4_hdmi);
+
+@@ -2172,8 +2234,6 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
+ clk_prepare_enable(vc4_hdmi->pixel_bvb_clock);
+ }
+
+- pm_runtime_enable(dev);
+-
+ drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+ drm_encoder_helper_add(encoder, &vc4_hdmi_encoder_helper_funcs);
+
+@@ -2197,6 +2257,8 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
+ vc4_hdmi_debugfs_regs,
+ vc4_hdmi);
+
++ pm_runtime_put_sync(dev);
++
+ return 0;
+
+ err_free_cec:
+@@ -2207,6 +2269,7 @@ err_destroy_conn:
+ vc4_hdmi_connector_destroy(&vc4_hdmi->connector);
+ err_destroy_encoder:
+ drm_encoder_cleanup(encoder);
++ pm_runtime_put_sync(dev);
+ pm_runtime_disable(dev);
+ err_put_ddc:
+ put_device(&vc4_hdmi->ddc->dev);
+@@ -2243,6 +2306,7 @@ static void vc4_hdmi_unbind(struct device *dev, struct device *master,
+ kfree(vc4_hdmi->hdmi_regset.regs);
+ kfree(vc4_hdmi->hd_regset.regs);
+
++ vc4_hdmi_audio_exit(vc4_hdmi);
+ vc4_hdmi_cec_exit(vc4_hdmi);
+ vc4_hdmi_hotplug_exit(vc4_hdmi);
+ vc4_hdmi_connector_destroy(&vc4_hdmi->connector);
+@@ -2352,11 +2416,18 @@ static const struct of_device_id vc4_hdmi_dt_match[] = {
+ {}
+ };
+
++static const struct dev_pm_ops vc4_hdmi_pm_ops = {
++ SET_RUNTIME_PM_OPS(vc4_hdmi_runtime_suspend,
++ vc4_hdmi_runtime_resume,
++ NULL)
++};
++
+ struct platform_driver vc4_hdmi_driver = {
+ .probe = vc4_hdmi_dev_probe,
+ .remove = vc4_hdmi_dev_remove,
+ .driver = {
+ .name = "vc4_hdmi",
+ .of_match_table = vc4_hdmi_dt_match,
++ .pm = &vc4_hdmi_pm_ops,
+ },
+ };
+diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.h b/drivers/gpu/drm/vc4/vc4_hdmi.h
+index 33e9f665ab8e4..c0492da736833 100644
+--- a/drivers/gpu/drm/vc4/vc4_hdmi.h
++++ b/drivers/gpu/drm/vc4/vc4_hdmi.h
+@@ -113,6 +113,7 @@ struct vc4_hdmi_audio {
+ struct snd_soc_dai_link_component platform;
+ struct snd_dmaengine_dai_dma_data dma_data;
+ struct hdmi_audio_infoframe infoframe;
++ struct platform_device *codec_pdev;
+ bool streaming;
+ };
+
+diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
+index c239045e05d6f..604933e20e6a2 100644
+--- a/drivers/gpu/drm/vc4/vc4_hvs.c
++++ b/drivers/gpu/drm/vc4/vc4_hvs.c
+@@ -365,17 +365,16 @@ static void vc4_hvs_update_dlist(struct drm_crtc *crtc)
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
++ unsigned long flags;
+
+ if (crtc->state->event) {
+- unsigned long flags;
+-
+ crtc->state->event->pipe = drm_crtc_index(crtc);
+
+ WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+ spin_lock_irqsave(&dev->event_lock, flags);
+
+- if (!vc4_state->feed_txp || vc4_state->txp_armed) {
++ if (!vc4_crtc->feeds_txp || vc4_state->txp_armed) {
+ vc4_crtc->event = crtc->state->event;
+ crtc->state->event = NULL;
+ }
+@@ -388,6 +387,22 @@ static void vc4_hvs_update_dlist(struct drm_crtc *crtc)
+ HVS_WRITE(SCALER_DISPLISTX(vc4_state->assigned_channel),
+ vc4_state->mm.start);
+ }
++
++ spin_lock_irqsave(&vc4_crtc->irq_lock, flags);
++ vc4_crtc->current_dlist = vc4_state->mm.start;
++ spin_unlock_irqrestore(&vc4_crtc->irq_lock, flags);
++}
++
++void vc4_hvs_atomic_begin(struct drm_crtc *crtc,
++ struct drm_atomic_state *state)
++{
++ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
++ struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
++ unsigned long flags;
++
++ spin_lock_irqsave(&vc4_crtc->irq_lock, flags);
++ vc4_crtc->current_hvs_channel = vc4_state->assigned_channel;
++ spin_unlock_irqrestore(&vc4_crtc->irq_lock, flags);
+ }
+
+ void vc4_hvs_atomic_enable(struct drm_crtc *crtc,
+@@ -395,10 +410,9 @@ void vc4_hvs_atomic_enable(struct drm_crtc *crtc,
+ {
+ struct drm_device *dev = crtc->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+- struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+- struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(new_crtc_state);
+ struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+- bool oneshot = vc4_state->feed_txp;
++ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
++ bool oneshot = vc4_crtc->feeds_txp;
+
+ vc4_hvs_update_dlist(crtc);
+ vc4_hvs_init_channel(vc4, crtc, mode, oneshot);
+diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
+index f0b3e4cf5bceb..6030d4a821555 100644
+--- a/drivers/gpu/drm/vc4/vc4_kms.c
++++ b/drivers/gpu/drm/vc4/vc4_kms.c
+@@ -233,6 +233,7 @@ static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
+ unsigned int i;
+
+ for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
++ struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+ struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
+ u32 dispctrl;
+ u32 dsp3_mux;
+@@ -253,7 +254,7 @@ static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
+ * TXP IP, and we need to disable the FIFO2 -> pixelvalve1
+ * route.
+ */
+- if (vc4_state->feed_txp)
++ if (vc4_crtc->feeds_txp)
+ dsp3_mux = VC4_SET_FIELD(3, SCALER_DISPCTRL_DSP3_MUX);
+ else
+ dsp3_mux = VC4_SET_FIELD(2, SCALER_DISPCTRL_DSP3_MUX);
+@@ -337,10 +338,10 @@ static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
+ struct drm_device *dev = state->dev;
+ struct vc4_dev *vc4 = to_vc4_dev(dev);
+ struct vc4_hvs *hvs = vc4->hvs;
+- struct drm_crtc_state *old_crtc_state;
+ struct drm_crtc_state *new_crtc_state;
+ struct drm_crtc *crtc;
+ struct vc4_hvs_state *old_hvs_state;
++ unsigned int channel;
+ int i;
+
+ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
+@@ -353,30 +354,32 @@ static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
+ vc4_hvs_mask_underrun(dev, vc4_crtc_state->assigned_channel);
+ }
+
+- if (vc4->hvs->hvs5)
+- clk_set_min_rate(hvs->core_clk, 500000000);
+-
+ old_hvs_state = vc4_hvs_get_old_global_state(state);
+- if (!old_hvs_state)
++ if (IS_ERR(old_hvs_state))
+ return;
+
+- for_each_old_crtc_in_state(state, crtc, old_crtc_state, i) {
+- struct vc4_crtc_state *vc4_crtc_state =
+- to_vc4_crtc_state(old_crtc_state);
+- unsigned int channel = vc4_crtc_state->assigned_channel;
++ for (channel = 0; channel < HVS_NUM_CHANNELS; channel++) {
++ struct drm_crtc_commit *commit;
+ int ret;
+
+- if (channel == VC4_HVS_CHANNEL_DISABLED)
++ if (!old_hvs_state->fifo_state[channel].in_use)
+ continue;
+
+- if (!old_hvs_state->fifo_state[channel].in_use)
++ commit = old_hvs_state->fifo_state[channel].pending_commit;
++ if (!commit)
+ continue;
+
+- ret = drm_crtc_commit_wait(old_hvs_state->fifo_state[channel].pending_commit);
++ ret = drm_crtc_commit_wait(commit);
+ if (ret)
+ drm_err(dev, "Timed out waiting for commit\n");
++
++ drm_crtc_commit_put(commit);
++ old_hvs_state->fifo_state[channel].pending_commit = NULL;
+ }
+
++ if (vc4->hvs->hvs5)
++ clk_set_min_rate(hvs->core_clk, 500000000);
++
+ drm_atomic_helper_commit_modeset_disables(dev, state);
+
+ vc4_ctm_commit(vc4, state);
+@@ -410,8 +413,8 @@ static int vc4_atomic_commit_setup(struct drm_atomic_state *state)
+ unsigned int i;
+
+ hvs_state = vc4_hvs_get_new_global_state(state);
+- if (!hvs_state)
+- return -EINVAL;
++ if (WARN_ON(IS_ERR(hvs_state)))
++ return PTR_ERR(hvs_state);
+
+ for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
+ struct vc4_crtc_state *vc4_crtc_state =
+@@ -668,12 +671,6 @@ vc4_hvs_channels_duplicate_state(struct drm_private_obj *obj)
+
+ for (i = 0; i < HVS_NUM_CHANNELS; i++) {
+ state->fifo_state[i].in_use = old_state->fifo_state[i].in_use;
+-
+- if (!old_state->fifo_state[i].pending_commit)
+- continue;
+-
+- state->fifo_state[i].pending_commit =
+- drm_crtc_commit_get(old_state->fifo_state[i].pending_commit);
+ }
+
+ return &state->base;
+@@ -762,8 +759,8 @@ static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
+ unsigned int i;
+
+ hvs_new_state = vc4_hvs_get_global_state(state);
+- if (!hvs_new_state)
+- return -EINVAL;
++ if (IS_ERR(hvs_new_state))
++ return PTR_ERR(hvs_new_state);
+
+ for (i = 0; i < ARRAY_SIZE(hvs_new_state->fifo_state); i++)
+ if (!hvs_new_state->fifo_state[i].in_use)
+diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c
+index 2fc7f4b5fa098..9809ca3e29451 100644
+--- a/drivers/gpu/drm/vc4/vc4_txp.c
++++ b/drivers/gpu/drm/vc4/vc4_txp.c
+@@ -391,7 +391,6 @@ static int vc4_txp_atomic_check(struct drm_crtc *crtc,
+ {
+ struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
+ crtc);
+- struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
+ int ret;
+
+ ret = vc4_hvs_atomic_check(crtc, state);
+@@ -399,7 +398,6 @@ static int vc4_txp_atomic_check(struct drm_crtc *crtc,
+ return ret;
+
+ crtc_state->no_vblank = true;
+- vc4_state->feed_txp = true;
+
+ return 0;
+ }
+@@ -437,6 +435,7 @@ static void vc4_txp_atomic_disable(struct drm_crtc *crtc,
+
+ static const struct drm_crtc_helper_funcs vc4_txp_crtc_helper_funcs = {
+ .atomic_check = vc4_txp_atomic_check,
++ .atomic_begin = vc4_hvs_atomic_begin,
+ .atomic_flush = vc4_hvs_atomic_flush,
+ .atomic_enable = vc4_txp_atomic_enable,
+ .atomic_disable = vc4_txp_atomic_disable,
+@@ -482,6 +481,7 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
+
+ vc4_crtc->pdev = pdev;
+ vc4_crtc->data = &vc4_txp_crtc_data;
++ vc4_crtc->feeds_txp = true;
+
+ txp->pdev = pdev;
+
+diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
+index 2de61b63ef91d..48d3c9955f0dd 100644
+--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
++++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
+@@ -248,6 +248,9 @@ void virtio_gpu_array_put_free(struct virtio_gpu_object_array *objs)
+ {
+ u32 i;
+
++ if (!objs)
++ return;
++
+ for (i = 0; i < objs->nents; i++)
+ drm_gem_object_put(objs->objs[i]);
+ virtio_gpu_array_free(objs);
+diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
+index 2e71e91278b45..93a41d018dca6 100644
+--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
++++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
+@@ -91,9 +91,7 @@ virtio_gpu_get_vbuf(struct virtio_gpu_device *vgdev,
+ {
+ struct virtio_gpu_vbuffer *vbuf;
+
+- vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL);
+- if (!vbuf)
+- return ERR_PTR(-ENOMEM);
++ vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL | __GFP_NOFAIL);
+
+ BUG_ON(size > MAX_INLINE_CMD_SIZE ||
+ size < sizeof(struct virtio_gpu_ctrl_hdr));
+@@ -147,10 +145,6 @@ static void *virtio_gpu_alloc_cmd_resp(struct virtio_gpu_device *vgdev,
+
+ vbuf = virtio_gpu_get_vbuf(vgdev, cmd_size,
+ resp_size, resp_buf, cb);
+- if (IS_ERR(vbuf)) {
+- *vbuffer_p = NULL;
+- return ERR_CAST(vbuf);
+- }
+ *vbuffer_p = vbuf;
+ return (struct virtio_gpu_command *)vbuf->buf;
+ }
+diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile
+index bc323f7d40321..18edc7ca5b454 100644
+--- a/drivers/gpu/drm/vmwgfx/Makefile
++++ b/drivers/gpu/drm/vmwgfx/Makefile
+@@ -9,9 +9,8 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
+ vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \
+ vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \
+ vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \
+- vmwgfx_devcaps.o ttm_object.o ttm_memory.o
++ vmwgfx_devcaps.o ttm_object.o ttm_memory.o vmwgfx_system_manager.o
+
+ vmwgfx-$(CONFIG_DRM_FBDEV_EMULATION) += vmwgfx_fb.o
+-vmwgfx-$(CONFIG_TRANSPARENT_HUGEPAGE) += vmwgfx_thp.o
+
+ obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o
+diff --git a/drivers/gpu/drm/vmwgfx/ttm_memory.c b/drivers/gpu/drm/vmwgfx/ttm_memory.c
+index edd17c30d5a51..2ced4c06ca451 100644
+--- a/drivers/gpu/drm/vmwgfx/ttm_memory.c
++++ b/drivers/gpu/drm/vmwgfx/ttm_memory.c
+@@ -34,7 +34,6 @@
+ #include <linux/mm.h>
+ #include <linux/module.h>
+ #include <linux/slab.h>
+-#include <linux/swap.h>
+
+ #include <drm/drm_device.h>
+ #include <drm/drm_file.h>
+@@ -173,69 +172,7 @@ static struct kobj_type ttm_mem_zone_kobj_type = {
+ .sysfs_ops = &ttm_mem_zone_ops,
+ .default_attrs = ttm_mem_zone_attrs,
+ };
+-
+-static struct attribute ttm_mem_global_lower_mem_limit = {
+- .name = "lower_mem_limit",
+- .mode = S_IRUGO | S_IWUSR
+-};
+-
+-static ssize_t ttm_mem_global_show(struct kobject *kobj,
+- struct attribute *attr,
+- char *buffer)
+-{
+- struct ttm_mem_global *glob =
+- container_of(kobj, struct ttm_mem_global, kobj);
+- uint64_t val = 0;
+-
+- spin_lock(&glob->lock);
+- val = glob->lower_mem_limit;
+- spin_unlock(&glob->lock);
+- /* convert from number of pages to KB */
+- val <<= (PAGE_SHIFT - 10);
+- return snprintf(buffer, PAGE_SIZE, "%llu\n",
+- (unsigned long long) val);
+-}
+-
+-static ssize_t ttm_mem_global_store(struct kobject *kobj,
+- struct attribute *attr,
+- const char *buffer,
+- size_t size)
+-{
+- int chars;
+- uint64_t val64;
+- unsigned long val;
+- struct ttm_mem_global *glob =
+- container_of(kobj, struct ttm_mem_global, kobj);
+-
+- chars = sscanf(buffer, "%lu", &val);
+- if (chars == 0)
+- return size;
+-
+- val64 = val;
+- /* convert from KB to number of pages */
+- val64 >>= (PAGE_SHIFT - 10);
+-
+- spin_lock(&glob->lock);
+- glob->lower_mem_limit = val64;
+- spin_unlock(&glob->lock);
+-
+- return size;
+-}
+-
+-static struct attribute *ttm_mem_global_attrs[] = {
+- &ttm_mem_global_lower_mem_limit,
+- NULL
+-};
+-
+-static const struct sysfs_ops ttm_mem_global_ops = {
+- .show = &ttm_mem_global_show,
+- .store = &ttm_mem_global_store,
+-};
+-
+-static struct kobj_type ttm_mem_glob_kobj_type = {
+- .sysfs_ops = &ttm_mem_global_ops,
+- .default_attrs = ttm_mem_global_attrs,
+-};
++static struct kobj_type ttm_mem_glob_kobj_type = {0};
+
+ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
+ bool from_wq, uint64_t extra)
+@@ -435,11 +372,6 @@ int ttm_mem_global_init(struct ttm_mem_global *glob, struct device *dev)
+
+ si_meminfo(&si);
+
+- spin_lock(&glob->lock);
+- /* set it as 0 by default to keep original behavior of OOM */
+- glob->lower_mem_limit = 0;
+- spin_unlock(&glob->lock);
+-
+ ret = ttm_mem_init_kernel_zone(glob, &si);
+ if (unlikely(ret != 0))
+ goto out_no_zone;
+@@ -527,35 +459,6 @@ void ttm_mem_global_free(struct ttm_mem_global *glob,
+ }
+ EXPORT_SYMBOL(ttm_mem_global_free);
+
+-/*
+- * check if the available mem is under lower memory limit
+- *
+- * a. if no swap disk at all or free swap space is under swap_mem_limit
+- * but available system mem is bigger than sys_mem_limit, allow TTM
+- * allocation;
+- *
+- * b. if the available system mem is less than sys_mem_limit but free
+- * swap disk is bigger than swap_mem_limit, allow TTM allocation.
+- */
+-bool
+-ttm_check_under_lowerlimit(struct ttm_mem_global *glob,
+- uint64_t num_pages,
+- struct ttm_operation_ctx *ctx)
+-{
+- int64_t available;
+-
+- /* We allow over commit during suspend */
+- if (ctx->force_alloc)
+- return false;
+-
+- available = get_nr_swap_pages() + si_mem_available();
+- available -= num_pages;
+- if (available < glob->lower_mem_limit)
+- return true;
+-
+- return false;
+-}
+-
+ static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
+ struct ttm_mem_zone *single_zone,
+ uint64_t amount, bool reserve)
+diff --git a/drivers/gpu/drm/vmwgfx/ttm_memory.h b/drivers/gpu/drm/vmwgfx/ttm_memory.h
+index c50dba7744854..7b0d617ebcb1e 100644
+--- a/drivers/gpu/drm/vmwgfx/ttm_memory.h
++++ b/drivers/gpu/drm/vmwgfx/ttm_memory.h
+@@ -50,8 +50,6 @@
+ * @work: The workqueue callback for the shrink queue.
+ * @lock: Lock to protect the @shrink - and the memory accounting members,
+ * that is, essentially the whole structure with some exceptions.
+- * @lower_mem_limit: include lower limit of swap space and lower limit of
+- * system memory.
+ * @zones: Array of pointers to accounting zones.
+ * @num_zones: Number of populated entries in the @zones array.
+ * @zone_kernel: Pointer to the kernel zone.
+@@ -69,7 +67,6 @@ extern struct ttm_mem_global {
+ struct workqueue_struct *swap_queue;
+ struct work_struct work;
+ spinlock_t lock;
+- uint64_t lower_mem_limit;
+ struct ttm_mem_zone *zones[TTM_MEM_MAX_ZONES];
+ unsigned int num_zones;
+ struct ttm_mem_zone *zone_kernel;
+@@ -91,6 +88,5 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
+ void ttm_mem_global_free_page(struct ttm_mem_global *glob,
+ struct page *page, uint64_t size);
+ size_t ttm_round_pot(size_t size);
+-bool ttm_check_under_lowerlimit(struct ttm_mem_global *glob, uint64_t num_pages,
+- struct ttm_operation_ctx *ctx);
++
+ #endif
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
+index 67db472d3493c..162dfeb1cc5ad 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
+@@ -145,6 +145,13 @@ struct vmw_fifo_state *vmw_fifo_create(struct vmw_private *dev_priv)
+ (unsigned int) max,
+ (unsigned int) min,
+ (unsigned int) fifo->capabilities);
++
++ if (unlikely(min >= max)) {
++ drm_warn(&dev_priv->drm,
++ "FIFO memory is not usable. Driver failed to initialize.");
++ return ERR_PTR(-ENXIO);
++ }
++
+ return fifo;
+ }
+
+@@ -521,7 +528,7 @@ int vmw_cmd_send_fence(struct vmw_private *dev_priv, uint32_t *seqno)
+ *seqno = atomic_add_return(1, &dev_priv->marker_seq);
+ } while (*seqno == 0);
+
+- if (!(vmw_fifo_caps(dev_priv) & SVGA_FIFO_CAP_FENCE)) {
++ if (!vmw_has_fences(dev_priv)) {
+
+ /*
+ * Don't request hardware to send a fence. The
+@@ -668,11 +675,14 @@ int vmw_cmd_emit_dummy_query(struct vmw_private *dev_priv,
+ */
+ bool vmw_cmd_supported(struct vmw_private *vmw)
+ {
+- if ((vmw->capabilities & (SVGA_CAP_COMMAND_BUFFERS |
+- SVGA_CAP_CMD_BUFFERS_2)) != 0)
+- return true;
++ bool has_cmdbufs =
++ (vmw->capabilities & (SVGA_CAP_COMMAND_BUFFERS |
++ SVGA_CAP_CMD_BUFFERS_2)) != 0;
++ if (vmw_is_svga_v3(vmw))
++ return (has_cmdbufs &&
++ (vmw->capabilities & SVGA_CAP_GBOBJECTS) != 0);
+ /*
+ * We have FIFO cmd's
+ */
+- return vmw->fifo_mem != NULL;
++ return has_cmdbufs || vmw->fifo_mem != NULL;
+ }
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+index ab9a1750e1dff..8449d09c06f7a 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+@@ -707,23 +707,15 @@ static int vmw_dma_masks(struct vmw_private *dev_priv)
+ static int vmw_vram_manager_init(struct vmw_private *dev_priv)
+ {
+ int ret;
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+- ret = vmw_thp_init(dev_priv);
+-#else
+ ret = ttm_range_man_init(&dev_priv->bdev, TTM_PL_VRAM, false,
+ dev_priv->vram_size >> PAGE_SHIFT);
+-#endif
+ ttm_resource_manager_set_used(ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM), false);
+ return ret;
+ }
+
+ static void vmw_vram_manager_fini(struct vmw_private *dev_priv)
+ {
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+- vmw_thp_fini(dev_priv);
+-#else
+ ttm_range_man_fini(&dev_priv->bdev, TTM_PL_VRAM);
+-#endif
+ }
+
+ static int vmw_setup_pci_resources(struct vmw_private *dev,
+@@ -1071,6 +1063,12 @@ static int vmw_driver_load(struct vmw_private *dev_priv, u32 pci_id)
+ "3D will be disabled.\n");
+ dev_priv->has_mob = false;
+ }
++ if (vmw_sys_man_init(dev_priv) != 0) {
++ drm_info(&dev_priv->drm,
++ "No MOB page table memory available. "
++ "3D will be disabled.\n");
++ dev_priv->has_mob = false;
++ }
+ }
+
+ if (dev_priv->has_mob && (dev_priv->capabilities & SVGA_CAP_DX)) {
+@@ -1121,8 +1119,10 @@ out_no_fifo:
+ vmw_overlay_close(dev_priv);
+ vmw_kms_close(dev_priv);
+ out_no_kms:
+- if (dev_priv->has_mob)
++ if (dev_priv->has_mob) {
+ vmw_gmrid_man_fini(dev_priv, VMW_PL_MOB);
++ vmw_sys_man_fini(dev_priv);
++ }
+ if (dev_priv->has_gmr)
+ vmw_gmrid_man_fini(dev_priv, VMW_PL_GMR);
+ vmw_devcaps_destroy(dev_priv);
+@@ -1172,8 +1172,10 @@ static void vmw_driver_unload(struct drm_device *dev)
+ vmw_gmrid_man_fini(dev_priv, VMW_PL_GMR);
+
+ vmw_release_device_early(dev_priv);
+- if (dev_priv->has_mob)
++ if (dev_priv->has_mob) {
+ vmw_gmrid_man_fini(dev_priv, VMW_PL_MOB);
++ vmw_sys_man_fini(dev_priv);
++ }
+ vmw_devcaps_destroy(dev_priv);
+ vmw_vram_manager_fini(dev_priv);
+ ttm_device_fini(&dev_priv->bdev);
+@@ -1617,34 +1619,40 @@ static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver);
+ if (ret)
+- return ret;
++ goto out_error;
+
+ ret = pcim_enable_device(pdev);
+ if (ret)
+- return ret;
++ goto out_error;
+
+ vmw = devm_drm_dev_alloc(&pdev->dev, &driver,
+ struct vmw_private, drm);
+- if (IS_ERR(vmw))
+- return PTR_ERR(vmw);
++ if (IS_ERR(vmw)) {
++ ret = PTR_ERR(vmw);
++ goto out_error;
++ }
+
+ pci_set_drvdata(pdev, &vmw->drm);
+
+ ret = ttm_mem_global_init(&ttm_mem_glob, &pdev->dev);
+ if (ret)
+- return ret;
++ goto out_error;
+
+ ret = vmw_driver_load(vmw, ent->device);
+ if (ret)
+- return ret;
++ goto out_release;
+
+ ret = drm_dev_register(&vmw->drm, 0);
+- if (ret) {
+- vmw_driver_unload(&vmw->drm);
+- return ret;
+- }
++ if (ret)
++ goto out_unload;
+
+ return 0;
++out_unload:
++ vmw_driver_unload(&vmw->drm);
++out_release:
++ ttm_mem_global_release(&ttm_mem_glob);
++out_error:
++ return ret;
+ }
+
+ static int __init vmwgfx_init(void)
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+index a833751099b55..288e883177bea 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+@@ -59,11 +59,8 @@
+ #define VMWGFX_DRIVER_MINOR 19
+ #define VMWGFX_DRIVER_PATCHLEVEL 0
+ #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
+-#define VMWGFX_MAX_RELOCATIONS 2048
+-#define VMWGFX_MAX_VALIDATIONS 2048
+ #define VMWGFX_MAX_DISPLAYS 16
+ #define VMWGFX_CMD_BOUNCE_INIT_SIZE 32768
+-#define VMWGFX_ENABLE_SCREEN_TARGET_OTABLE 1
+
+ #define VMWGFX_PCI_ID_SVGA2 0x0405
+ #define VMWGFX_PCI_ID_SVGA3 0x0406
+@@ -82,8 +79,9 @@
+ VMWGFX_NUM_GB_SURFACE +\
+ VMWGFX_NUM_GB_SCREEN_TARGET)
+
+-#define VMW_PL_GMR (TTM_PL_PRIV + 0)
+-#define VMW_PL_MOB (TTM_PL_PRIV + 1)
++#define VMW_PL_GMR (TTM_PL_PRIV + 0)
++#define VMW_PL_MOB (TTM_PL_PRIV + 1)
++#define VMW_PL_SYSTEM (TTM_PL_PRIV + 2)
+
+ #define VMW_RES_CONTEXT ttm_driver_type0
+ #define VMW_RES_SURFACE ttm_driver_type1
+@@ -1039,7 +1037,6 @@ extern struct ttm_placement vmw_vram_placement;
+ extern struct ttm_placement vmw_vram_sys_placement;
+ extern struct ttm_placement vmw_vram_gmr_placement;
+ extern struct ttm_placement vmw_sys_placement;
+-extern struct ttm_placement vmw_evictable_placement;
+ extern struct ttm_placement vmw_srf_placement;
+ extern struct ttm_placement vmw_mob_placement;
+ extern struct ttm_placement vmw_nonfixed_placement;
+@@ -1115,15 +1112,14 @@ extern int vmw_execbuf_fence_commands(struct drm_file *file_priv,
+ struct vmw_private *dev_priv,
+ struct vmw_fence_obj **p_fence,
+ uint32_t *p_handle);
+-extern void vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
++extern int vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
+ struct vmw_fpriv *vmw_fp,
+ int ret,
+ struct drm_vmw_fence_rep __user
+ *user_fence_rep,
+ struct vmw_fence_obj *fence,
+ uint32_t fence_handle,
+- int32_t out_fence_fd,
+- struct sync_file *sync_file);
++ int32_t out_fence_fd);
+ bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd);
+
+ /**
+@@ -1251,6 +1247,12 @@ int vmw_overlay_num_free_overlays(struct vmw_private *dev_priv);
+ int vmw_gmrid_man_init(struct vmw_private *dev_priv, int type);
+ void vmw_gmrid_man_fini(struct vmw_private *dev_priv, int type);
+
++/**
++ * System memory manager
++ */
++int vmw_sys_man_init(struct vmw_private *dev_priv);
++void vmw_sys_man_fini(struct vmw_private *dev_priv);
++
+ /**
+ * Prime - vmwgfx_prime.c
+ */
+@@ -1550,16 +1552,7 @@ void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
+ pgoff_t start, pgoff_t end);
+ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);
+ vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+-vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
+- enum page_entry_size pe_size);
+-#endif
+
+-/* Transparent hugepage support - vmwgfx_thp.c */
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+-extern int vmw_thp_init(struct vmw_private *dev_priv);
+-void vmw_thp_fini(struct vmw_private *dev_priv);
+-#endif
+
+ /**
+ * VMW_DEBUG_KMS - Debug output for kernel mode-setting
+@@ -1684,4 +1677,12 @@ static inline void vmw_irq_status_write(struct vmw_private *vmw,
+ outl(status, vmw->io_start + SVGA_IRQSTATUS_PORT);
+ }
+
++static inline bool vmw_has_fences(struct vmw_private *vmw)
++{
++ if ((vmw->capabilities & (SVGA_CAP_COMMAND_BUFFERS |
++ SVGA_CAP_CMD_BUFFERS_2)) != 0)
++ return true;
++ return (vmw_fifo_caps(vmw) & SVGA_FIFO_CAP_FENCE) != 0;
++}
++
+ #endif
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+index 5f2ffa9de5c8f..9144e8f88c812 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+@@ -3823,17 +3823,17 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,
+ * Also if copying fails, user-space will be unable to signal the fence object
+ * so we wait for it immediately, and then unreference the user-space reference.
+ */
+-void
++int
+ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
+ struct vmw_fpriv *vmw_fp, int ret,
+ struct drm_vmw_fence_rep __user *user_fence_rep,
+ struct vmw_fence_obj *fence, uint32_t fence_handle,
+- int32_t out_fence_fd, struct sync_file *sync_file)
++ int32_t out_fence_fd)
+ {
+ struct drm_vmw_fence_rep fence_rep;
+
+ if (user_fence_rep == NULL)
+- return;
++ return 0;
+
+ memset(&fence_rep, 0, sizeof(fence_rep));
+
+@@ -3861,20 +3861,14 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
+ * handle.
+ */
+ if (unlikely(ret != 0) && (fence_rep.error == 0)) {
+- if (sync_file)
+- fput(sync_file->file);
+-
+- if (fence_rep.fd != -1) {
+- put_unused_fd(fence_rep.fd);
+- fence_rep.fd = -1;
+- }
+-
+ ttm_ref_object_base_unref(vmw_fp->tfile, fence_handle,
+ TTM_REF_USAGE);
+ VMW_DEBUG_USER("Fence copy error. Syncing.\n");
+ (void) vmw_fence_obj_wait(fence, false, false,
+ VMW_FENCE_WAIT_TIMEOUT);
+ }
++
++ return ret ? -EFAULT : 0;
+ }
+
+ /**
+@@ -4212,16 +4206,23 @@ int vmw_execbuf_process(struct drm_file *file_priv,
+
+ (void) vmw_fence_obj_wait(fence, false, false,
+ VMW_FENCE_WAIT_TIMEOUT);
++ }
++ }
++
++ ret = vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv), ret,
++ user_fence_rep, fence, handle, out_fence_fd);
++
++ if (sync_file) {
++ if (ret) {
++ /* usercopy of fence failed, put the file object */
++ fput(sync_file->file);
++ put_unused_fd(out_fence_fd);
+ } else {
+ /* Link the fence with the FD created earlier */
+ fd_install(out_fence_fd, sync_file->file);
+ }
+ }
+
+- vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv), ret,
+- user_fence_rep, fence, handle, out_fence_fd,
+- sync_file);
+-
+ /* Don't unreference when handing fence out */
+ if (unlikely(out_fence != NULL)) {
+ *out_fence = fence;
+@@ -4239,7 +4240,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
+ */
+ vmw_validation_unref_lists(&val_ctx);
+
+- return 0;
++ return ret;
+
+ out_unlock_binding:
+ mutex_unlock(&dev_priv->binding_mutex);
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+index d18c6a56e3dca..f18ed03a8b2d6 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+@@ -490,7 +490,7 @@ static int vmw_fb_kms_detach(struct vmw_fb_par *par,
+
+ static int vmw_fb_kms_framebuffer(struct fb_info *info)
+ {
+- struct drm_mode_fb_cmd2 mode_cmd;
++ struct drm_mode_fb_cmd2 mode_cmd = {0};
+ struct vmw_fb_par *par = info->par;
+ struct fb_var_screeninfo *var = &info->var;
+ struct drm_framebuffer *cur_fb;
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+index 9fe12329a4d58..b32ddbb992de2 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+@@ -85,6 +85,22 @@ fman_from_fence(struct vmw_fence_obj *fence)
+ return container_of(fence->base.lock, struct vmw_fence_manager, lock);
+ }
+
++static u32 vmw_fence_goal_read(struct vmw_private *vmw)
++{
++ if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
++ return vmw_read(vmw, SVGA_REG_FENCE_GOAL);
++ else
++ return vmw_fifo_mem_read(vmw, SVGA_FIFO_FENCE_GOAL);
++}
++
++static void vmw_fence_goal_write(struct vmw_private *vmw, u32 value)
++{
++ if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
++ vmw_write(vmw, SVGA_REG_FENCE_GOAL, value);
++ else
++ vmw_fifo_mem_write(vmw, SVGA_FIFO_FENCE_GOAL, value);
++}
++
+ /*
+ * Note on fencing subsystem usage of irqs:
+ * Typically the vmw_fences_update function is called
+@@ -400,7 +416,7 @@ static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
+ if (likely(!fman->seqno_valid))
+ return false;
+
+- goal_seqno = vmw_fifo_mem_read(fman->dev_priv, SVGA_FIFO_FENCE_GOAL);
++ goal_seqno = vmw_fence_goal_read(fman->dev_priv);
+ if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
+ return false;
+
+@@ -408,9 +424,8 @@ static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
+ list_for_each_entry(fence, &fman->fence_list, head) {
+ if (!list_empty(&fence->seq_passed_actions)) {
+ fman->seqno_valid = true;
+- vmw_fifo_mem_write(fman->dev_priv,
+- SVGA_FIFO_FENCE_GOAL,
+- fence->base.seqno);
++ vmw_fence_goal_write(fman->dev_priv,
++ fence->base.seqno);
+ break;
+ }
+ }
+@@ -442,13 +457,12 @@ static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
+ if (dma_fence_is_signaled_locked(&fence->base))
+ return false;
+
+- goal_seqno = vmw_fifo_mem_read(fman->dev_priv, SVGA_FIFO_FENCE_GOAL);
++ goal_seqno = vmw_fence_goal_read(fman->dev_priv);
+ if (likely(fman->seqno_valid &&
+ goal_seqno - fence->base.seqno < VMW_FENCE_WRAP))
+ return false;
+
+- vmw_fifo_mem_write(fman->dev_priv, SVGA_FIFO_FENCE_GOAL,
+- fence->base.seqno);
++ vmw_fence_goal_write(fman->dev_priv, fence->base.seqno);
+ fman->seqno_valid = true;
+
+ return true;
+@@ -1159,7 +1173,7 @@ int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
+ }
+
+ vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
+- handle, -1, NULL);
++ handle, -1);
+ vmw_fence_obj_unreference(&fence);
+ return 0;
+ out_no_create:
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+index c5191de365ca1..fe4732bf2c9d2 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+@@ -32,6 +32,14 @@
+
+ #define VMW_FENCE_WRAP (1 << 24)
+
++static u32 vmw_irqflag_fence_goal(struct vmw_private *vmw)
++{
++ if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
++ return SVGA_IRQFLAG_REG_FENCE_GOAL;
++ else
++ return SVGA_IRQFLAG_FENCE_GOAL;
++}
++
+ /**
+ * vmw_thread_fn - Deferred (process context) irq handler
+ *
+@@ -96,7 +104,7 @@ static irqreturn_t vmw_irq_handler(int irq, void *arg)
+ wake_up_all(&dev_priv->fifo_queue);
+
+ if ((masked_status & (SVGA_IRQFLAG_ANY_FENCE |
+- SVGA_IRQFLAG_FENCE_GOAL)) &&
++ vmw_irqflag_fence_goal(dev_priv))) &&
+ !test_and_set_bit(VMW_IRQTHREAD_FENCE, dev_priv->irqthread_pending))
+ ret = IRQ_WAKE_THREAD;
+
+@@ -137,8 +145,7 @@ bool vmw_seqno_passed(struct vmw_private *dev_priv,
+ if (likely(dev_priv->last_read_seqno - seqno < VMW_FENCE_WRAP))
+ return true;
+
+- if (!(vmw_fifo_caps(dev_priv) & SVGA_FIFO_CAP_FENCE) &&
+- vmw_fifo_idle(dev_priv, seqno))
++ if (!vmw_has_fences(dev_priv) && vmw_fifo_idle(dev_priv, seqno))
+ return true;
+
+ /**
+@@ -160,6 +167,7 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
+ unsigned long timeout)
+ {
+ struct vmw_fifo_state *fifo_state = dev_priv->fifo;
++ bool fifo_down = false;
+
+ uint32_t count = 0;
+ uint32_t signal_seq;
+@@ -176,12 +184,14 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
+ */
+
+ if (fifo_idle) {
+- down_read(&fifo_state->rwsem);
+ if (dev_priv->cman) {
+ ret = vmw_cmdbuf_idle(dev_priv->cman, interruptible,
+ 10*HZ);
+ if (ret)
+ goto out_err;
++ } else if (fifo_state) {
++ down_read(&fifo_state->rwsem);
++ fifo_down = true;
+ }
+ }
+
+@@ -218,12 +228,12 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
+ }
+ }
+ finish_wait(&dev_priv->fence_queue, &__wait);
+- if (ret == 0 && fifo_idle)
++ if (ret == 0 && fifo_idle && fifo_state)
+ vmw_fence_write(dev_priv, signal_seq);
+
+ wake_up_all(&dev_priv->fence_queue);
+ out_err:
+- if (fifo_idle)
++ if (fifo_down)
+ up_read(&fifo_state->rwsem);
+
+ return ret;
+@@ -266,13 +276,13 @@ void vmw_seqno_waiter_remove(struct vmw_private *dev_priv)
+
+ void vmw_goal_waiter_add(struct vmw_private *dev_priv)
+ {
+- vmw_generic_waiter_add(dev_priv, SVGA_IRQFLAG_FENCE_GOAL,
++ vmw_generic_waiter_add(dev_priv, vmw_irqflag_fence_goal(dev_priv),
+ &dev_priv->goal_queue_waiters);
+ }
+
+ void vmw_goal_waiter_remove(struct vmw_private *dev_priv)
+ {
+- vmw_generic_waiter_remove(dev_priv, SVGA_IRQFLAG_FENCE_GOAL,
++ vmw_generic_waiter_remove(dev_priv, vmw_irqflag_fence_goal(dev_priv),
+ &dev_priv->goal_queue_waiters);
+ }
+
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+index 74fa419092138..50c64e7813be1 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+@@ -1336,7 +1336,6 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
+ ret = vmw_kms_new_framebuffer_surface(dev_priv, surface, &vfb,
+ mode_cmd,
+ is_bo_proxy);
+-
+ /*
+ * vmw_create_bo_proxy() adds a reference that is no longer
+ * needed
+@@ -1398,13 +1397,16 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
+ ret = vmw_user_lookup_handle(dev_priv, tfile,
+ mode_cmd->handles[0],
+ &surface, &bo);
+- if (ret)
++ if (ret) {
++ DRM_ERROR("Invalid buffer object handle %u (0x%x).\n",
++ mode_cmd->handles[0], mode_cmd->handles[0]);
+ goto err_out;
++ }
+
+
+ if (!bo &&
+ !vmw_kms_srf_ok(dev_priv, mode_cmd->width, mode_cmd->height)) {
+- DRM_ERROR("Surface size cannot exceed %dx%d",
++ DRM_ERROR("Surface size cannot exceed %dx%d\n",
+ dev_priv->texture_max_width,
+ dev_priv->texture_max_height);
+ goto err_out;
+@@ -2516,7 +2518,7 @@ void vmw_kms_helper_validation_finish(struct vmw_private *dev_priv,
+ if (file_priv)
+ vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv),
+ ret, user_fence_rep, fence,
+- handle, -1, NULL);
++ handle, -1);
+ if (out_fence)
+ *out_fence = fence;
+ else
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+index f9394207dd3cc..632e587519722 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+@@ -1,7 +1,7 @@
+ // SPDX-License-Identifier: GPL-2.0 OR MIT
+ /**************************************************************************
+ *
+- * Copyright 2012-2015 VMware, Inc., Palo Alto, CA., USA
++ * Copyright 2012-2021 VMware, Inc., Palo Alto, CA., USA
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+@@ -29,12 +29,6 @@
+
+ #include "vmwgfx_drv.h"
+
+-/*
+- * If we set up the screen target otable, screen objects stop working.
+- */
+-
+-#define VMW_OTABLE_SETUP_SUB ((VMWGFX_ENABLE_SCREEN_TARGET_OTABLE ? 0 : 1))
+-
+ #ifdef CONFIG_64BIT
+ #define VMW_PPN_SIZE 8
+ #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PT64_0
+@@ -75,7 +69,7 @@ static const struct vmw_otable pre_dx_tables[] = {
+ {VMWGFX_NUM_GB_CONTEXT * sizeof(SVGAOTableContextEntry), NULL, true},
+ {VMWGFX_NUM_GB_SHADER * sizeof(SVGAOTableShaderEntry), NULL, true},
+ {VMWGFX_NUM_GB_SCREEN_TARGET * sizeof(SVGAOTableScreenTargetEntry),
+- NULL, VMWGFX_ENABLE_SCREEN_TARGET_OTABLE}
++ NULL, true}
+ };
+
+ static const struct vmw_otable dx_tables[] = {
+@@ -84,7 +78,7 @@ static const struct vmw_otable dx_tables[] = {
+ {VMWGFX_NUM_GB_CONTEXT * sizeof(SVGAOTableContextEntry), NULL, true},
+ {VMWGFX_NUM_GB_SHADER * sizeof(SVGAOTableShaderEntry), NULL, true},
+ {VMWGFX_NUM_GB_SCREEN_TARGET * sizeof(SVGAOTableScreenTargetEntry),
+- NULL, VMWGFX_ENABLE_SCREEN_TARGET_OTABLE},
++ NULL, true},
+ {VMWGFX_NUM_DXCONTEXT * sizeof(SVGAOTableDXContextEntry), NULL, true},
+ };
+
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
+index e5a9a5cbd01a7..922317d1acc8a 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
+@@ -477,7 +477,7 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
+ else
+ prot = vm_get_page_prot(vma->vm_flags);
+
+- ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault, 1);
++ ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
+ if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
+ return ret;
+
+@@ -486,73 +486,3 @@ out_unlock:
+
+ return ret;
+ }
+-
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+-vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
+- enum page_entry_size pe_size)
+-{
+- struct vm_area_struct *vma = vmf->vma;
+- struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
+- vma->vm_private_data;
+- struct vmw_buffer_object *vbo =
+- container_of(bo, struct vmw_buffer_object, base);
+- pgprot_t prot;
+- vm_fault_t ret;
+- pgoff_t fault_page_size;
+- bool write = vmf->flags & FAULT_FLAG_WRITE;
+-
+- switch (pe_size) {
+- case PE_SIZE_PMD:
+- fault_page_size = HPAGE_PMD_SIZE >> PAGE_SHIFT;
+- break;
+-#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+- case PE_SIZE_PUD:
+- fault_page_size = HPAGE_PUD_SIZE >> PAGE_SHIFT;
+- break;
+-#endif
+- default:
+- WARN_ON_ONCE(1);
+- return VM_FAULT_FALLBACK;
+- }
+-
+- /* Always do write dirty-tracking and COW on PTE level. */
+- if (write && (READ_ONCE(vbo->dirty) || is_cow_mapping(vma->vm_flags)))
+- return VM_FAULT_FALLBACK;
+-
+- ret = ttm_bo_vm_reserve(bo, vmf);
+- if (ret)
+- return ret;
+-
+- if (vbo->dirty) {
+- pgoff_t allowed_prefault;
+- unsigned long page_offset;
+-
+- page_offset = vmf->pgoff -
+- drm_vma_node_start(&bo->base.vma_node);
+- if (page_offset >= bo->resource->num_pages ||
+- vmw_resources_clean(vbo, page_offset,
+- page_offset + PAGE_SIZE,
+- &allowed_prefault)) {
+- ret = VM_FAULT_SIGBUS;
+- goto out_unlock;
+- }
+-
+- /*
+- * Write protect, so we get a new fault on write, and can
+- * split.
+- */
+- prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
+- } else {
+- prot = vm_get_page_prot(vma->vm_flags);
+- }
+-
+- ret = ttm_bo_vm_fault_reserved(vmf, prot, 1, fault_page_size);
+- if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
+- return ret;
+-
+-out_unlock:
+- dma_resv_unlock(bo->base.resv);
+-
+- return ret;
+-}
+-#endif
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+index d85310b2608dd..f5e90d0e2d0f8 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+@@ -1872,8 +1872,8 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
+ int i, ret;
+
+
+- /* Do nothing if Screen Target support is turned off */
+- if (!VMWGFX_ENABLE_SCREEN_TARGET_OTABLE || !dev_priv->has_mob)
++ /* Do nothing if there's no support for MOBs */
++ if (!dev_priv->has_mob)
+ return -ENOSYS;
+
+ if (!(dev_priv->capabilities & SVGA_CAP_GBOBJECTS))
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_system_manager.c b/drivers/gpu/drm/vmwgfx/vmwgfx_system_manager.c
+new file mode 100644
+index 0000000000000..b0005b03a6174
+--- /dev/null
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_system_manager.c
+@@ -0,0 +1,90 @@
++/* SPDX-License-Identifier: GPL-2.0 OR MIT */
++/*
++ * Copyright 2021 VMware, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person
++ * obtaining a copy of this software and associated documentation
++ * files (the "Software"), to deal in the Software without
++ * restriction, including without limitation the rights to use, copy,
++ * modify, merge, publish, distribute, sublicense, and/or sell copies
++ * of the Software, and to permit persons to whom the Software is
++ * furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be
++ * included in all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
++ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
++ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
++ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
++ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
++ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#include "vmwgfx_drv.h"
++
++#include <drm/ttm/ttm_bo_driver.h>
++#include <drm/ttm/ttm_device.h>
++#include <drm/ttm/ttm_placement.h>
++#include <drm/ttm/ttm_resource.h>
++#include <linux/slab.h>
++
++
++static int vmw_sys_man_alloc(struct ttm_resource_manager *man,
++ struct ttm_buffer_object *bo,
++ const struct ttm_place *place,
++ struct ttm_resource **res)
++{
++ *res = kzalloc(sizeof(**res), GFP_KERNEL);
++ if (!*res)
++ return -ENOMEM;
++
++ ttm_resource_init(bo, place, *res);
++ return 0;
++}
++
++static void vmw_sys_man_free(struct ttm_resource_manager *man,
++ struct ttm_resource *res)
++{
++ kfree(res);
++}
++
++static const struct ttm_resource_manager_func vmw_sys_manager_func = {
++ .alloc = vmw_sys_man_alloc,
++ .free = vmw_sys_man_free,
++};
++
++int vmw_sys_man_init(struct vmw_private *dev_priv)
++{
++ struct ttm_device *bdev = &dev_priv->bdev;
++ struct ttm_resource_manager *man =
++ kzalloc(sizeof(*man), GFP_KERNEL);
++
++ if (!man)
++ return -ENOMEM;
++
++ man->use_tt = true;
++ man->func = &vmw_sys_manager_func;
++
++ ttm_resource_manager_init(man, 0);
++ ttm_set_driver_manager(bdev, VMW_PL_SYSTEM, man);
++ ttm_resource_manager_set_used(man, true);
++ return 0;
++}
++
++void vmw_sys_man_fini(struct vmw_private *dev_priv)
++{
++ struct ttm_resource_manager *man = ttm_manager_type(&dev_priv->bdev,
++ VMW_PL_SYSTEM);
++
++ ttm_resource_manager_evict_all(&dev_priv->bdev, man);
++
++ ttm_resource_manager_set_used(man, false);
++ ttm_resource_manager_cleanup(man);
++
++ ttm_set_driver_manager(&dev_priv->bdev, VMW_PL_SYSTEM, NULL);
++ kfree(man);
++}
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_thp.c b/drivers/gpu/drm/vmwgfx/vmwgfx_thp.c
+deleted file mode 100644
+index 2a3d3468e4e0a..0000000000000
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_thp.c
++++ /dev/null
+@@ -1,184 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0 OR MIT
+-/*
+- * Huge page-table-entry support for IO memory.
+- *
+- * Copyright (C) 2007-2019 Vmware, Inc. All rights reservedd.
+- */
+-#include "vmwgfx_drv.h"
+-#include <drm/ttm/ttm_bo_driver.h>
+-#include <drm/ttm/ttm_placement.h>
+-#include <drm/ttm/ttm_range_manager.h>
+-
+-/**
+- * struct vmw_thp_manager - Range manager implementing huge page alignment
+- *
+- * @manager: TTM resource manager.
+- * @mm: The underlying range manager. Protected by @lock.
+- * @lock: Manager lock.
+- */
+-struct vmw_thp_manager {
+- struct ttm_resource_manager manager;
+- struct drm_mm mm;
+- spinlock_t lock;
+-};
+-
+-static struct vmw_thp_manager *to_thp_manager(struct ttm_resource_manager *man)
+-{
+- return container_of(man, struct vmw_thp_manager, manager);
+-}
+-
+-static const struct ttm_resource_manager_func vmw_thp_func;
+-
+-static int vmw_thp_insert_aligned(struct ttm_buffer_object *bo,
+- struct drm_mm *mm, struct drm_mm_node *node,
+- unsigned long align_pages,
+- const struct ttm_place *place,
+- struct ttm_resource *mem,
+- unsigned long lpfn,
+- enum drm_mm_insert_mode mode)
+-{
+- if (align_pages >= bo->page_alignment &&
+- (!bo->page_alignment || align_pages % bo->page_alignment == 0)) {
+- return drm_mm_insert_node_in_range(mm, node,
+- mem->num_pages,
+- align_pages, 0,
+- place->fpfn, lpfn, mode);
+- }
+-
+- return -ENOSPC;
+-}
+-
+-static int vmw_thp_get_node(struct ttm_resource_manager *man,
+- struct ttm_buffer_object *bo,
+- const struct ttm_place *place,
+- struct ttm_resource **res)
+-{
+- struct vmw_thp_manager *rman = to_thp_manager(man);
+- struct drm_mm *mm = &rman->mm;
+- struct ttm_range_mgr_node *node;
+- unsigned long align_pages;
+- unsigned long lpfn;
+- enum drm_mm_insert_mode mode = DRM_MM_INSERT_BEST;
+- int ret;
+-
+- node = kzalloc(struct_size(node, mm_nodes, 1), GFP_KERNEL);
+- if (!node)
+- return -ENOMEM;
+-
+- ttm_resource_init(bo, place, &node->base);
+-
+- lpfn = place->lpfn;
+- if (!lpfn)
+- lpfn = man->size;
+-
+- mode = DRM_MM_INSERT_BEST;
+- if (place->flags & TTM_PL_FLAG_TOPDOWN)
+- mode = DRM_MM_INSERT_HIGH;
+-
+- spin_lock(&rman->lock);
+- if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) {
+- align_pages = (HPAGE_PUD_SIZE >> PAGE_SHIFT);
+- if (node->base.num_pages >= align_pages) {
+- ret = vmw_thp_insert_aligned(bo, mm, &node->mm_nodes[0],
+- align_pages, place,
+- &node->base, lpfn, mode);
+- if (!ret)
+- goto found_unlock;
+- }
+- }
+-
+- align_pages = (HPAGE_PMD_SIZE >> PAGE_SHIFT);
+- if (node->base.num_pages >= align_pages) {
+- ret = vmw_thp_insert_aligned(bo, mm, &node->mm_nodes[0],
+- align_pages, place, &node->base,
+- lpfn, mode);
+- if (!ret)
+- goto found_unlock;
+- }
+-
+- ret = drm_mm_insert_node_in_range(mm, &node->mm_nodes[0],
+- node->base.num_pages,
+- bo->page_alignment, 0,
+- place->fpfn, lpfn, mode);
+-found_unlock:
+- spin_unlock(&rman->lock);
+-
+- if (unlikely(ret)) {
+- kfree(node);
+- } else {
+- node->base.start = node->mm_nodes[0].start;
+- *res = &node->base;
+- }
+-
+- return ret;
+-}
+-
+-static void vmw_thp_put_node(struct ttm_resource_manager *man,
+- struct ttm_resource *res)
+-{
+- struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res);
+- struct vmw_thp_manager *rman = to_thp_manager(man);
+-
+- spin_lock(&rman->lock);
+- drm_mm_remove_node(&node->mm_nodes[0]);
+- spin_unlock(&rman->lock);
+-
+- kfree(node);
+-}
+-
+-int vmw_thp_init(struct vmw_private *dev_priv)
+-{
+- struct vmw_thp_manager *rman;
+-
+- rman = kzalloc(sizeof(*rman), GFP_KERNEL);
+- if (!rman)
+- return -ENOMEM;
+-
+- ttm_resource_manager_init(&rman->manager,
+- dev_priv->vram_size >> PAGE_SHIFT);
+-
+- rman->manager.func = &vmw_thp_func;
+- drm_mm_init(&rman->mm, 0, rman->manager.size);
+- spin_lock_init(&rman->lock);
+-
+- ttm_set_driver_manager(&dev_priv->bdev, TTM_PL_VRAM, &rman->manager);
+- ttm_resource_manager_set_used(&rman->manager, true);
+- return 0;
+-}
+-
+-void vmw_thp_fini(struct vmw_private *dev_priv)
+-{
+- struct ttm_resource_manager *man = ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM);
+- struct vmw_thp_manager *rman = to_thp_manager(man);
+- struct drm_mm *mm = &rman->mm;
+- int ret;
+-
+- ttm_resource_manager_set_used(man, false);
+-
+- ret = ttm_resource_manager_evict_all(&dev_priv->bdev, man);
+- if (ret)
+- return;
+- spin_lock(&rman->lock);
+- drm_mm_clean(mm);
+- drm_mm_takedown(mm);
+- spin_unlock(&rman->lock);
+- ttm_resource_manager_cleanup(man);
+- ttm_set_driver_manager(&dev_priv->bdev, TTM_PL_VRAM, NULL);
+- kfree(rman);
+-}
+-
+-static void vmw_thp_debug(struct ttm_resource_manager *man,
+- struct drm_printer *printer)
+-{
+- struct vmw_thp_manager *rman = to_thp_manager(man);
+-
+- spin_lock(&rman->lock);
+- drm_mm_print(&rman->mm, printer);
+- spin_unlock(&rman->lock);
+-}
+-
+-static const struct ttm_resource_manager_func vmw_thp_func = {
+- .alloc = vmw_thp_get_node,
+- .free = vmw_thp_put_node,
+- .debug = vmw_thp_debug
+-};
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
+index 8b8991e3ed2d0..450bb1e9626f7 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
+@@ -92,6 +92,13 @@ static const struct ttm_place gmr_vram_placement_flags[] = {
+ }
+ };
+
++static const struct ttm_place vmw_sys_placement_flags = {
++ .fpfn = 0,
++ .lpfn = 0,
++ .mem_type = VMW_PL_SYSTEM,
++ .flags = 0
++};
++
+ struct ttm_placement vmw_vram_gmr_placement = {
+ .num_placement = 2,
+ .placement = vram_gmr_placement_flags,
+@@ -113,28 +120,11 @@ struct ttm_placement vmw_sys_placement = {
+ .busy_placement = &sys_placement_flags
+ };
+
+-static const struct ttm_place evictable_placement_flags[] = {
+- {
+- .fpfn = 0,
+- .lpfn = 0,
+- .mem_type = TTM_PL_SYSTEM,
+- .flags = 0
+- }, {
+- .fpfn = 0,
+- .lpfn = 0,
+- .mem_type = TTM_PL_VRAM,
+- .flags = 0
+- }, {
+- .fpfn = 0,
+- .lpfn = 0,
+- .mem_type = VMW_PL_GMR,
+- .flags = 0
+- }, {
+- .fpfn = 0,
+- .lpfn = 0,
+- .mem_type = VMW_PL_MOB,
+- .flags = 0
+- }
++struct ttm_placement vmw_pt_sys_placement = {
++ .num_placement = 1,
++ .placement = &vmw_sys_placement_flags,
++ .num_busy_placement = 1,
++ .busy_placement = &vmw_sys_placement_flags
+ };
+
+ static const struct ttm_place nonfixed_placement_flags[] = {
+@@ -156,13 +146,6 @@ static const struct ttm_place nonfixed_placement_flags[] = {
+ }
+ };
+
+-struct ttm_placement vmw_evictable_placement = {
+- .num_placement = 4,
+- .placement = evictable_placement_flags,
+- .num_busy_placement = 1,
+- .busy_placement = &sys_placement_flags
+-};
+-
+ struct ttm_placement vmw_srf_placement = {
+ .num_placement = 1,
+ .num_busy_placement = 2,
+@@ -484,6 +467,9 @@ static int vmw_ttm_bind(struct ttm_device *bdev,
+ &vmw_be->vsgt, ttm->num_pages,
+ vmw_be->gmr_id);
+ break;
++ case VMW_PL_SYSTEM:
++ /* Nothing to be done for a system bind */
++ break;
+ default:
+ BUG();
+ }
+@@ -507,6 +493,8 @@ static void vmw_ttm_unbind(struct ttm_device *bdev,
+ case VMW_PL_MOB:
+ vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
+ break;
++ case VMW_PL_SYSTEM:
++ break;
+ default:
+ BUG();
+ }
+@@ -628,6 +616,7 @@ static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *
+
+ switch (mem->mem_type) {
+ case TTM_PL_SYSTEM:
++ case VMW_PL_SYSTEM:
+ case VMW_PL_GMR:
+ case VMW_PL_MOB:
+ return 0;
+@@ -674,6 +663,11 @@ static void vmw_swap_notify(struct ttm_buffer_object *bo)
+ (void) ttm_bo_wait(bo, false, false);
+ }
+
++static bool vmw_memtype_is_system(uint32_t mem_type)
++{
++ return mem_type == TTM_PL_SYSTEM || mem_type == VMW_PL_SYSTEM;
++}
++
+ static int vmw_move(struct ttm_buffer_object *bo,
+ bool evict,
+ struct ttm_operation_ctx *ctx,
+@@ -684,7 +678,7 @@ static int vmw_move(struct ttm_buffer_object *bo,
+ struct ttm_resource_manager *new_man = ttm_manager_type(bo->bdev, new_mem->mem_type);
+ int ret;
+
+- if (new_man->use_tt && new_mem->mem_type != TTM_PL_SYSTEM) {
++ if (new_man->use_tt && !vmw_memtype_is_system(new_mem->mem_type)) {
+ ret = vmw_ttm_bind(bo->bdev, bo->ttm, new_mem);
+ if (ret)
+ return ret;
+@@ -693,7 +687,7 @@ static int vmw_move(struct ttm_buffer_object *bo,
+ vmw_move_notify(bo, bo->resource, new_mem);
+
+ if (old_man->use_tt && new_man->use_tt) {
+- if (bo->resource->mem_type == TTM_PL_SYSTEM) {
++ if (vmw_memtype_is_system(bo->resource->mem_type)) {
+ ttm_bo_move_null(bo, new_mem);
+ return 0;
+ }
+@@ -740,7 +734,7 @@ int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
+ int ret;
+
+ ret = vmw_bo_create_kernel(dev_priv, bo_size,
+- &vmw_sys_placement,
++ &vmw_pt_sys_placement,
+ &bo);
+ if (unlikely(ret != 0))
+ return ret;
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+index e6b1f98ec99f0..0a4c340252ec4 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+@@ -61,9 +61,6 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma)
+ .fault = vmw_bo_vm_fault,
+ .open = ttm_bo_vm_open,
+ .close = ttm_bo_vm_close,
+-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+- .huge_fault = vmw_bo_vm_huge_fault,
+-#endif
+ };
+ struct drm_file *file_priv = filp->private_data;
+ struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
+diff --git a/drivers/gpu/host1x/Kconfig b/drivers/gpu/host1x/Kconfig
+index 6dab94adf25e5..6815b4db17c1b 100644
+--- a/drivers/gpu/host1x/Kconfig
++++ b/drivers/gpu/host1x/Kconfig
+@@ -2,6 +2,7 @@
+ config TEGRA_HOST1X
+ tristate "NVIDIA Tegra host1x driver"
+ depends on ARCH_TEGRA || (ARM && COMPILE_TEST)
++ select DMA_SHARED_BUFFER
+ select IOMMU_IOVA
+ help
+ Driver for the NVIDIA Tegra host1x hardware.
+diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
+index fbb6447b8659e..fc9f54282f7d6 100644
+--- a/drivers/gpu/host1x/dev.c
++++ b/drivers/gpu/host1x/dev.c
+@@ -18,6 +18,10 @@
+ #include <trace/events/host1x.h>
+ #undef CREATE_TRACE_POINTS
+
++#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
++#include <asm/dma-iommu.h>
++#endif
++
+ #include "bus.h"
+ #include "channel.h"
+ #include "debug.h"
+@@ -238,6 +242,17 @@ static struct iommu_domain *host1x_iommu_attach(struct host1x *host)
+ struct iommu_domain *domain = iommu_get_domain_for_dev(host->dev);
+ int err;
+
++#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
++ if (host->dev->archdata.mapping) {
++ struct dma_iommu_mapping *mapping =
++ to_dma_iommu_mapping(host->dev);
++ arm_iommu_detach_device(host->dev);
++ arm_iommu_release_mapping(mapping);
++
++ domain = iommu_get_domain_for_dev(host->dev);
++ }
++#endif
++
+ /*
+ * We may not always want to enable IOMMU support (for example if the
+ * host1x firewall is already enabled and we don't support addressing
+@@ -511,6 +526,7 @@ static int host1x_remove(struct platform_device *pdev)
+ host1x_syncpt_deinit(host);
+ reset_control_assert(host->rst);
+ clk_disable_unprepare(host->clk);
++ host1x_channel_list_free(&host->channel_list);
+ host1x_iommu_exit(host);
+
+ return 0;
+diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
+index d198a10848c6b..a89a408182e60 100644
+--- a/drivers/gpu/host1x/syncpt.c
++++ b/drivers/gpu/host1x/syncpt.c
+@@ -225,27 +225,12 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
+ void *ref;
+ struct host1x_waitlist *waiter;
+ int err = 0, check_count = 0;
+- u32 val;
+
+ if (value)
+- *value = 0;
+-
+- /* first check cache */
+- if (host1x_syncpt_is_expired(sp, thresh)) {
+- if (value)
+- *value = host1x_syncpt_load(sp);
++ *value = host1x_syncpt_load(sp);
+
++ if (host1x_syncpt_is_expired(sp, thresh))
+ return 0;
+- }
+-
+- /* try to read from register */
+- val = host1x_hw_syncpt_load(sp->host, sp);
+- if (host1x_syncpt_is_expired(sp, thresh)) {
+- if (value)
+- *value = val;
+-
+- goto done;
+- }
+
+ if (!timeout) {
+ err = -EAGAIN;
+diff --git a/drivers/gpu/ipu-v3/ipu-di.c b/drivers/gpu/ipu-v3/ipu-di.c
+index 666223c6bec4d..0a34e0ab4fe60 100644
+--- a/drivers/gpu/ipu-v3/ipu-di.c
++++ b/drivers/gpu/ipu-v3/ipu-di.c
+@@ -447,8 +447,9 @@ static void ipu_di_config_clock(struct ipu_di *di,
+
+ error = rate / (sig->mode.pixelclock / 1000);
+
+- dev_dbg(di->ipu->dev, " IPU clock can give %lu with divider %u, error %d.%u%%\n",
+- rate, div, (signed)(error - 1000) / 10, error % 10);
++ dev_dbg(di->ipu->dev, " IPU clock can give %lu with divider %u, error %c%d.%d%%\n",
++ rate, div, error < 1000 ? '-' : '+',
++ abs(error - 1000) / 10, abs(error - 1000) % 10);
+
+ /* Allow a 1% error */
+ if (error < 1010 && error >= 990) {
+diff --git a/drivers/greybus/svc.c b/drivers/greybus/svc.c
+index ce7740ef449ba..51d0875a34800 100644
+--- a/drivers/greybus/svc.c
++++ b/drivers/greybus/svc.c
+@@ -866,8 +866,14 @@ static int gb_svc_hello(struct gb_operation *op)
+
+ gb_svc_debugfs_init(svc);
+
+- return gb_svc_queue_deferred_request(op);
++ ret = gb_svc_queue_deferred_request(op);
++ if (ret)
++ goto err_remove_debugfs;
++
++ return 0;
+
++err_remove_debugfs:
++ gb_svc_debugfs_exit(svc);
+ err_unregister_device:
+ gb_svc_watchdog_destroy(svc);
+ device_del(&svc->dev);
+diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
+index 3c33bf572d6d3..9235ab7161e3a 100644
+--- a/drivers/hid/Kconfig
++++ b/drivers/hid/Kconfig
+@@ -207,14 +207,14 @@ config HID_CHERRY
+
+ config HID_CHICONY
+ tristate "Chicony devices"
+- depends on HID
++ depends on USB_HID
+ default !EXPERT
+ help
+ Support for Chicony Tactical pad and special keys on Chicony keyboards.
+
+ config HID_CORSAIR
+ tristate "Corsair devices"
+- depends on HID && USB && LEDS_CLASS
++ depends on USB_HID && LEDS_CLASS
+ help
+ Support for Corsair devices that are not fully compliant with the
+ HID standard.
+@@ -245,7 +245,7 @@ config HID_MACALLY
+
+ config HID_PRODIKEYS
+ tristate "Prodikeys PC-MIDI Keyboard support"
+- depends on HID && SND
++ depends on USB_HID && SND
+ select SND_RAWMIDI
+ help
+ Support for Prodikeys PC-MIDI Keyboard device support.
+@@ -553,7 +553,7 @@ config HID_LENOVO
+
+ config HID_LOGITECH
+ tristate "Logitech devices"
+- depends on HID
++ depends on USB_HID
+ depends on LEDS_CLASS
+ default !EXPERT
+ help
+@@ -919,7 +919,7 @@ config HID_SAITEK
+
+ config HID_SAMSUNG
+ tristate "Samsung InfraRed remote control or keyboards"
+- depends on HID
++ depends on USB_HID
+ help
+ Support for Samsung InfraRed remote control or keyboards.
+
+diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_client.c b/drivers/hid/amd-sfh-hid/amd_sfh_client.c
+index 840fd075c56f1..6284db50ec9bf 100644
+--- a/drivers/hid/amd-sfh-hid/amd_sfh_client.c
++++ b/drivers/hid/amd-sfh-hid/amd_sfh_client.c
+@@ -226,6 +226,17 @@ int amd_sfh_hid_client_init(struct amd_mp2_dev *privdata)
+ dev_dbg(dev, "sid 0x%x status 0x%x\n",
+ cl_data->sensor_idx[i], cl_data->sensor_sts[i]);
+ }
++ if (privdata->mp2_ops->discovery_status &&
++ privdata->mp2_ops->discovery_status(privdata) == 0) {
++ amd_sfh_hid_client_deinit(privdata);
++ for (i = 0; i < cl_data->num_hid_devices; i++) {
++ devm_kfree(dev, cl_data->feature_report[i]);
++ devm_kfree(dev, in_data->input_report[i]);
++ devm_kfree(dev, cl_data->report_descr[i]);
++ }
++ dev_warn(dev, "Failed to discover, sensors not enabled\n");
++ return -EOPNOTSUPP;
++ }
+ schedule_delayed_work(&cl_data->work_buffer, msecs_to_jiffies(AMD_SFH_IDLE_LOOP));
+ return 0;
+
+diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
+index 05c007b213f24..ae8f1f2536e94 100644
+--- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
++++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
+@@ -36,11 +36,11 @@ static int amd_sfh_wait_response_v2(struct amd_mp2_dev *mp2, u8 sid, u32 sensor_
+ {
+ union cmd_response cmd_resp;
+
+- /* Get response with status within a max of 800 ms timeout */
++ /* Get response with status within a max of 1600 ms timeout */
+ if (!readl_poll_timeout(mp2->mmio + AMD_P2C_MSG(0), cmd_resp.resp,
+ (cmd_resp.response_v2.response == sensor_sts &&
+ cmd_resp.response_v2.status == 0 && (sid == 0xff ||
+- cmd_resp.response_v2.sensor_id == sid)), 500, 800000))
++ cmd_resp.response_v2.sensor_id == sid)), 500, 1600000))
+ return cmd_resp.response_v2.response;
+
+ return SENSOR_DISABLED;
+@@ -88,6 +88,50 @@ static void amd_stop_all_sensor_v2(struct amd_mp2_dev *privdata)
+ writel(cmd_base.ul, privdata->mmio + AMD_C2P_MSG0);
+ }
+
++static void amd_sfh_clear_intr_v2(struct amd_mp2_dev *privdata)
++{
++ if (readl(privdata->mmio + AMD_P2C_MSG(4))) {
++ writel(0, privdata->mmio + AMD_P2C_MSG(4));
++ writel(0xf, privdata->mmio + AMD_P2C_MSG(5));
++ }
++}
++
++static void amd_sfh_clear_intr(struct amd_mp2_dev *privdata)
++{
++ if (privdata->mp2_ops->clear_intr)
++ privdata->mp2_ops->clear_intr(privdata);
++}
++
++static irqreturn_t amd_sfh_irq_handler(int irq, void *data)
++{
++ amd_sfh_clear_intr(data);
++
++ return IRQ_HANDLED;
++}
++
++static int amd_sfh_irq_init_v2(struct amd_mp2_dev *privdata)
++{
++ int rc;
++
++ pci_intx(privdata->pdev, true);
++
++ rc = devm_request_irq(&privdata->pdev->dev, privdata->pdev->irq,
++ amd_sfh_irq_handler, 0, DRIVER_NAME, privdata);
++ if (rc) {
++ dev_err(&privdata->pdev->dev, "failed to request irq %d err=%d\n",
++ privdata->pdev->irq, rc);
++ return rc;
++ }
++
++ return 0;
++}
++
++static int amd_sfh_dis_sts_v2(struct amd_mp2_dev *privdata)
++{
++ return (readl(privdata->mmio + AMD_P2C_MSG(1)) &
++ SENSOR_DISCOVERY_STATUS_MASK) >> SENSOR_DISCOVERY_STATUS_SHIFT;
++}
++
+ void amd_start_sensor(struct amd_mp2_dev *privdata, struct amd_mp2_sensor_info info)
+ {
+ union sfh_cmd_param cmd_param;
+@@ -192,6 +236,8 @@ static void amd_mp2_pci_remove(void *privdata)
+ struct amd_mp2_dev *mp2 = privdata;
+ amd_sfh_hid_client_deinit(privdata);
+ mp2->mp2_ops->stop_all(mp2);
++ pci_intx(mp2->pdev, false);
++ amd_sfh_clear_intr(mp2);
+ }
+
+ static const struct amd_mp2_ops amd_sfh_ops_v2 = {
+@@ -199,6 +245,9 @@ static const struct amd_mp2_ops amd_sfh_ops_v2 = {
+ .stop = amd_stop_sensor_v2,
+ .stop_all = amd_stop_all_sensor_v2,
+ .response = amd_sfh_wait_response_v2,
++ .clear_intr = amd_sfh_clear_intr_v2,
++ .init_intr = amd_sfh_irq_init_v2,
++ .discovery_status = amd_sfh_dis_sts_v2,
+ };
+
+ static const struct amd_mp2_ops amd_sfh_ops = {
+@@ -224,6 +273,14 @@ static void mp2_select_ops(struct amd_mp2_dev *privdata)
+ }
+ }
+
++static int amd_sfh_irq_init(struct amd_mp2_dev *privdata)
++{
++ if (privdata->mp2_ops->init_intr)
++ return privdata->mp2_ops->init_intr(privdata);
++
++ return 0;
++}
++
+ static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ {
+ struct amd_mp2_dev *privdata;
+@@ -257,9 +314,20 @@ static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i
+
+ mp2_select_ops(privdata);
+
++ rc = amd_sfh_irq_init(privdata);
++ if (rc) {
++ dev_err(&pdev->dev, "amd_sfh_irq_init failed\n");
++ return rc;
++ }
++
+ rc = amd_sfh_hid_client_init(privdata);
+- if (rc)
++ if (rc) {
++ amd_sfh_clear_intr(privdata);
++ dev_err(&pdev->dev, "amd_sfh_hid_client_init failed\n");
+ return rc;
++ }
++
++ amd_sfh_clear_intr(privdata);
+
+ return devm_add_action_or_reset(&pdev->dev, amd_mp2_pci_remove, privdata);
+ }
+@@ -287,6 +355,9 @@ static int __maybe_unused amd_mp2_pci_resume(struct device *dev)
+ }
+ }
+
++ schedule_delayed_work(&cl_data->work_buffer, msecs_to_jiffies(AMD_SFH_IDLE_LOOP));
++ amd_sfh_clear_intr(mp2);
++
+ return 0;
+ }
+
+@@ -310,6 +381,9 @@ static int __maybe_unused amd_mp2_pci_suspend(struct device *dev)
+ }
+ }
+
++ cancel_delayed_work_sync(&cl_data->work_buffer);
++ amd_sfh_clear_intr(mp2);
++
+ return 0;
+ }
+
+diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
+index 1ff6f83cb6fd1..2d3203d3daeb3 100644
+--- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
++++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
+@@ -38,6 +38,9 @@
+
+ #define AMD_SFH_IDLE_LOOP 200
+
++#define SENSOR_DISCOVERY_STATUS_MASK GENMASK(5, 3)
++#define SENSOR_DISCOVERY_STATUS_SHIFT 3
++
+ /* SFH Command register */
+ union sfh_cmd_base {
+ u32 ul;
+@@ -48,7 +51,7 @@ union sfh_cmd_base {
+ } s;
+ struct {
+ u32 cmd_id : 4;
+- u32 intr_enable : 1;
++ u32 intr_disable : 1;
+ u32 rsvd1 : 3;
+ u32 length : 7;
+ u32 mem_type : 1;
+@@ -140,5 +143,8 @@ struct amd_mp2_ops {
+ void (*stop)(struct amd_mp2_dev *privdata, u16 sensor_idx);
+ void (*stop_all)(struct amd_mp2_dev *privdata);
+ int (*response)(struct amd_mp2_dev *mp2, u8 sid, u32 sensor_sts);
++ void (*clear_intr)(struct amd_mp2_dev *privdata);
++ int (*init_intr)(struct amd_mp2_dev *privdata);
++ int (*discovery_status)(struct amd_mp2_dev *privdata);
+ };
+ #endif
+diff --git a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
+index 0c36972193821..07eb3281b88db 100644
+--- a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
++++ b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
+@@ -26,6 +26,7 @@
+ #define HID_USAGE_SENSOR_STATE_READY_ENUM 0x02
+ #define HID_USAGE_SENSOR_STATE_INITIALIZING_ENUM 0x05
+ #define HID_USAGE_SENSOR_EVENT_DATA_UPDATED_ENUM 0x04
++#define ILLUMINANCE_MASK GENMASK(14, 0)
+
+ int get_report_descriptor(int sensor_idx, u8 *rep_desc)
+ {
+@@ -245,7 +246,8 @@ u8 get_input_report(u8 current_index, int sensor_idx, int report_id, struct amd_
+ get_common_inputs(&als_input.common_property, report_id);
+ /* For ALS ,V2 Platforms uses C2P_MSG5 register instead of DRAM access method */
+ if (supported_input == V2_STATUS)
+- als_input.illuminance_value = (int)readl(privdata->mmio + AMD_C2P_MSG(5));
++ als_input.illuminance_value =
++ readl(privdata->mmio + AMD_C2P_MSG(5)) & ILLUMINANCE_MASK;
+ else
+ als_input.illuminance_value =
+ (int)sensor_virt_addr[0] / AMD_SFH_FW_MULTIPLIER;
+diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c
+index 6ccfa0cb997ab..b683c0e8557d4 100644
+--- a/drivers/hid/hid-apple.c
++++ b/drivers/hid/hid-apple.c
+@@ -429,7 +429,7 @@ static int apple_input_configured(struct hid_device *hdev,
+
+ if ((asc->quirks & APPLE_HAS_FN) && !asc->fn_found) {
+ hid_info(hdev, "Fn key not found (Apple Wireless Keyboard clone?), disabling Fn key handling\n");
+- asc->quirks = 0;
++ asc->quirks &= ~APPLE_HAS_FN;
+ }
+
+ return 0;
+diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
+index f3ecddc519ee8..08c9a9a60ae47 100644
+--- a/drivers/hid/hid-asus.c
++++ b/drivers/hid/hid-asus.c
+@@ -1028,8 +1028,7 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ if (drvdata->quirks & QUIRK_IS_MULTITOUCH)
+ drvdata->tp = &asus_i2c_tp;
+
+- if ((drvdata->quirks & QUIRK_T100_KEYBOARD) &&
+- hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
++ if ((drvdata->quirks & QUIRK_T100_KEYBOARD) && hid_is_usb(hdev)) {
+ struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+
+ if (intf->altsetting->desc.bInterfaceNumber == T100_TPAD_INTF) {
+@@ -1057,8 +1056,7 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ drvdata->tp = &asus_t100chi_tp;
+ }
+
+- if ((drvdata->quirks & QUIRK_MEDION_E1239T) &&
+- hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
++ if ((drvdata->quirks & QUIRK_MEDION_E1239T) && hid_is_usb(hdev)) {
+ struct usb_host_interface *alt =
+ to_usb_interface(hdev->dev.parent)->altsetting;
+
+diff --git a/drivers/hid/hid-bigbenff.c b/drivers/hid/hid-bigbenff.c
+index db6da21ade063..74ad8bf98bfd5 100644
+--- a/drivers/hid/hid-bigbenff.c
++++ b/drivers/hid/hid-bigbenff.c
+@@ -191,7 +191,7 @@ static void bigben_worker(struct work_struct *work)
+ struct bigben_device, worker);
+ struct hid_field *report_field = bigben->report->field[0];
+
+- if (bigben->removed)
++ if (bigben->removed || !report_field)
+ return;
+
+ if (bigben->work_led) {
+diff --git a/drivers/hid/hid-chicony.c b/drivers/hid/hid-chicony.c
+index ca556d39da2ae..f04d2aa23efe4 100644
+--- a/drivers/hid/hid-chicony.c
++++ b/drivers/hid/hid-chicony.c
+@@ -114,6 +114,9 @@ static int ch_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+ int ret;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
+ ret = hid_parse(hdev);
+ if (ret) {
+diff --git a/drivers/hid/hid-corsair.c b/drivers/hid/hid-corsair.c
+index 902a60e249ed2..8c895c820b672 100644
+--- a/drivers/hid/hid-corsair.c
++++ b/drivers/hid/hid-corsair.c
+@@ -553,7 +553,12 @@ static int corsair_probe(struct hid_device *dev, const struct hid_device_id *id)
+ int ret;
+ unsigned long quirks = id->driver_data;
+ struct corsair_drvdata *drvdata;
+- struct usb_interface *usbif = to_usb_interface(dev->dev.parent);
++ struct usb_interface *usbif;
++
++ if (!hid_is_usb(dev))
++ return -EINVAL;
++
++ usbif = to_usb_interface(dev->dev.parent);
+
+ drvdata = devm_kzalloc(&dev->dev, sizeof(struct corsair_drvdata),
+ GFP_KERNEL);
+diff --git a/drivers/hid/hid-debug.c b/drivers/hid/hid-debug.c
+index fa57d05badf70..f48d3534e0200 100644
+--- a/drivers/hid/hid-debug.c
++++ b/drivers/hid/hid-debug.c
+@@ -825,7 +825,9 @@ static const char *keys[KEY_MAX + 1] = {
+ [KEY_F22] = "F22", [KEY_F23] = "F23",
+ [KEY_F24] = "F24", [KEY_PLAYCD] = "PlayCD",
+ [KEY_PAUSECD] = "PauseCD", [KEY_PROG3] = "Prog3",
+- [KEY_PROG4] = "Prog4", [KEY_SUSPEND] = "Suspend",
++ [KEY_PROG4] = "Prog4",
++ [KEY_ALL_APPLICATIONS] = "AllApplications",
++ [KEY_SUSPEND] = "Suspend",
+ [KEY_CLOSE] = "Close", [KEY_PLAY] = "Play",
+ [KEY_FASTFORWARD] = "FastForward", [KEY_BASSBOOST] = "BassBoost",
+ [KEY_PRINT] = "Print", [KEY_HP] = "HP",
+@@ -934,6 +936,7 @@ static const char *keys[KEY_MAX + 1] = {
+ [KEY_ASSISTANT] = "Assistant",
+ [KEY_KBD_LAYOUT_NEXT] = "KbdLayoutNext",
+ [KEY_EMOJI_PICKER] = "EmojiPicker",
++ [KEY_DICTATE] = "Dictate",
+ [KEY_BRIGHTNESS_MIN] = "BrightnessMin",
+ [KEY_BRIGHTNESS_MAX] = "BrightnessMax",
+ [KEY_BRIGHTNESS_AUTO] = "BrightnessAuto",
+diff --git a/drivers/hid/hid-elan.c b/drivers/hid/hid-elan.c
+index 021049805bb71..3091355d48df6 100644
+--- a/drivers/hid/hid-elan.c
++++ b/drivers/hid/hid-elan.c
+@@ -50,7 +50,7 @@ struct elan_drvdata {
+
+ static int is_not_elan_touchpad(struct hid_device *hdev)
+ {
+- if (hdev->bus == BUS_USB) {
++ if (hid_is_usb(hdev)) {
+ struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+
+ return (intf->altsetting->desc.bInterfaceNumber !=
+diff --git a/drivers/hid/hid-elo.c b/drivers/hid/hid-elo.c
+index 383dfda8c12fc..2876cb6a7dcab 100644
+--- a/drivers/hid/hid-elo.c
++++ b/drivers/hid/hid-elo.c
+@@ -228,15 +228,16 @@ static int elo_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+ struct elo_priv *priv;
+ int ret;
+- struct usb_device *udev;
++
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ INIT_DELAYED_WORK(&priv->work, elo_work);
+- udev = interface_to_usbdev(to_usb_interface(hdev->dev.parent));
+- priv->usbdev = usb_get_dev(udev);
++ priv->usbdev = interface_to_usbdev(to_usb_interface(hdev->dev.parent));
+
+ hid_set_drvdata(hdev, priv);
+
+@@ -267,8 +268,6 @@ static void elo_remove(struct hid_device *hdev)
+ {
+ struct elo_priv *priv = hid_get_drvdata(hdev);
+
+- usb_put_dev(priv->usbdev);
+-
+ hid_hw_stop(hdev);
+ cancel_delayed_work_sync(&priv->work);
+ kfree(priv);
+diff --git a/drivers/hid/hid-ft260.c b/drivers/hid/hid-ft260.c
+index 4ef1c3b8094ea..183eeb3863b38 100644
+--- a/drivers/hid/hid-ft260.c
++++ b/drivers/hid/hid-ft260.c
+@@ -915,6 +915,9 @@ static int ft260_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ struct ft260_get_chip_version_report version;
+ int ret;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ dev = devm_kzalloc(&hdev->dev, sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return -ENOMEM;
+diff --git a/drivers/hid/hid-google-hammer.c b/drivers/hid/hid-google-hammer.c
+index 8123b871a3ebf..0403beb3104b9 100644
+--- a/drivers/hid/hid-google-hammer.c
++++ b/drivers/hid/hid-google-hammer.c
+@@ -585,6 +585,8 @@ static void hammer_remove(struct hid_device *hdev)
+ static const struct hid_device_id hammer_devices[] = {
+ { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
+ USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_DON) },
++ { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
++ USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_EEL) },
+ { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
+ USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_HAMMER) },
+ { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
+diff --git a/drivers/hid/hid-holtek-kbd.c b/drivers/hid/hid-holtek-kbd.c
+index 0a38e8e9bc783..403506b9697e7 100644
+--- a/drivers/hid/hid-holtek-kbd.c
++++ b/drivers/hid/hid-holtek-kbd.c
+@@ -140,12 +140,17 @@ static int holtek_kbd_input_event(struct input_dev *dev, unsigned int type,
+ static int holtek_kbd_probe(struct hid_device *hdev,
+ const struct hid_device_id *id)
+ {
+- struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+- int ret = hid_parse(hdev);
++ struct usb_interface *intf;
++ int ret;
++
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
+
++ ret = hid_parse(hdev);
+ if (!ret)
+ ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+
++ intf = to_usb_interface(hdev->dev.parent);
+ if (!ret && intf->cur_altsetting->desc.bInterfaceNumber == 1) {
+ struct hid_input *hidinput;
+ list_for_each_entry(hidinput, &hdev->inputs, list) {
+diff --git a/drivers/hid/hid-holtek-mouse.c b/drivers/hid/hid-holtek-mouse.c
+index 195b735b001d0..7c907939bfae1 100644
+--- a/drivers/hid/hid-holtek-mouse.c
++++ b/drivers/hid/hid-holtek-mouse.c
+@@ -62,6 +62,29 @@ static __u8 *holtek_mouse_report_fixup(struct hid_device *hdev, __u8 *rdesc,
+ return rdesc;
+ }
+
++static int holtek_mouse_probe(struct hid_device *hdev,
++ const struct hid_device_id *id)
++{
++ int ret;
++
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
++ ret = hid_parse(hdev);
++ if (ret) {
++ hid_err(hdev, "hid parse failed: %d\n", ret);
++ return ret;
++ }
++
++ ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
++ if (ret) {
++ hid_err(hdev, "hw start failed: %d\n", ret);
++ return ret;
++ }
++
++ return 0;
++}
++
+ static const struct hid_device_id holtek_mouse_devices[] = {
+ { HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT,
+ USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A067) },
+@@ -83,6 +106,7 @@ static struct hid_driver holtek_mouse_driver = {
+ .name = "holtek_mouse",
+ .id_table = holtek_mouse_devices,
+ .report_fixup = holtek_mouse_report_fixup,
++ .probe = holtek_mouse_probe,
+ };
+
+ module_hid_driver(holtek_mouse_driver);
+diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
+index 29564b370341e..645a5f566d233 100644
+--- a/drivers/hid/hid-ids.h
++++ b/drivers/hid/hid-ids.h
+@@ -394,6 +394,8 @@
+ #define USB_DEVICE_ID_HP_X2 0x074d
+ #define USB_DEVICE_ID_HP_X2_10_COVER 0x0755
+ #define I2C_DEVICE_ID_HP_SPECTRE_X360_15 0x2817
++#define I2C_DEVICE_ID_HP_ENVY_X360_15T_DR100 0x29CF
++#define USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN 0x2544
+ #define USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN 0x2706
+ #define I2C_DEVICE_ID_SURFACE_GO_TOUCHSCREEN 0x261A
+
+@@ -496,6 +498,7 @@
+ #define USB_DEVICE_ID_GOOGLE_MAGNEMITE 0x503d
+ #define USB_DEVICE_ID_GOOGLE_MOONBALL 0x5044
+ #define USB_DEVICE_ID_GOOGLE_DON 0x5050
++#define USB_DEVICE_ID_GOOGLE_EEL 0x5057
+
+ #define USB_VENDOR_ID_GOTOP 0x08f2
+ #define USB_DEVICE_ID_SUPER_Q2 0x007f
+@@ -881,6 +884,7 @@
+ #define USB_DEVICE_ID_MS_TOUCH_COVER_2 0x07a7
+ #define USB_DEVICE_ID_MS_TYPE_COVER_2 0x07a9
+ #define USB_DEVICE_ID_MS_POWER_COVER 0x07da
++#define USB_DEVICE_ID_MS_SURFACE3_COVER 0x07de
+ #define USB_DEVICE_ID_MS_XBOX_ONE_S_CONTROLLER 0x02fd
+ #define USB_DEVICE_ID_MS_PIXART_MOUSE 0x00cb
+ #define USB_DEVICE_ID_8BITDO_SN30_PRO_PLUS 0x02e0
+@@ -1276,6 +1280,9 @@
+ #define USB_DEVICE_ID_WEIDA_8752 0xC300
+ #define USB_DEVICE_ID_WEIDA_8755 0xC301
+
++#define USB_VENDOR_ID_WINBOND 0x0416
++#define USB_DEVICE_ID_TSTP_MTOUCH 0xc168
++
+ #define USB_VENDOR_ID_WISEGROUP 0x0925
+ #define USB_DEVICE_ID_SMARTJOY_PLUS 0x0005
+ #define USB_DEVICE_ID_SUPER_JOY_BOX_3 0x8888
+@@ -1346,6 +1353,7 @@
+ #define USB_VENDOR_ID_UGTIZER 0x2179
+ #define USB_DEVICE_ID_UGTIZER_TABLET_GP0610 0x0053
+ #define USB_DEVICE_ID_UGTIZER_TABLET_GT5040 0x0077
++#define USB_DEVICE_ID_UGTIZER_TABLET_WP5540 0x0004
+
+ #define USB_VENDOR_ID_VIEWSONIC 0x0543
+ #define USB_DEVICE_ID_VIEWSONIC_PD1011 0xe621
+diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
+index 4b5ebeacd2836..3172987a9ebd3 100644
+--- a/drivers/hid/hid-input.c
++++ b/drivers/hid/hid-input.c
+@@ -160,6 +160,7 @@ static int hidinput_setkeycode(struct input_dev *dev,
+ if (usage) {
+ *old_keycode = usage->type == EV_KEY ?
+ usage->code : KEY_RESERVED;
++ usage->type = EV_KEY;
+ usage->code = ke->keycode;
+
+ clear_bit(*old_keycode, dev->keybit);
+@@ -324,6 +325,10 @@ static const struct hid_device_id hid_battery_quirks[] = {
+ HID_BATTERY_QUIRK_IGNORE },
+ { HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN),
+ HID_BATTERY_QUIRK_IGNORE },
++ { HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN),
++ HID_BATTERY_QUIRK_IGNORE },
++ { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_ENVY_X360_15T_DR100),
++ HID_BATTERY_QUIRK_IGNORE },
+ { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_15),
+ HID_BATTERY_QUIRK_IGNORE },
+ { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_SURFACE_GO_TOUCHSCREEN),
+@@ -650,10 +655,9 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
+ code += KEY_MACRO1;
+ else
+ code += BTN_TRIGGER_HAPPY - 0x1e;
+- } else {
+- goto ignore;
++ break;
+ }
+- break;
++ fallthrough;
+ default:
+ switch (field->physical) {
+ case HID_GD_MOUSE:
+@@ -987,6 +991,7 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
+ case 0x0cd: map_key_clear(KEY_PLAYPAUSE); break;
+ case 0x0cf: map_key_clear(KEY_VOICECOMMAND); break;
+
++ case 0x0d8: map_key_clear(KEY_DICTATE); break;
+ case 0x0d9: map_key_clear(KEY_EMOJI_PICKER); break;
+
+ case 0x0e0: map_abs_clear(ABS_VOLUME); break;
+@@ -1078,6 +1083,8 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
+
+ case 0x29d: map_key_clear(KEY_KBD_LAYOUT_NEXT); break;
+
++ case 0x2a2: map_key_clear(KEY_ALL_APPLICATIONS); break;
++
+ case 0x2c7: map_key_clear(KEY_KBDINPUTASSIST_PREV); break;
+ case 0x2c8: map_key_clear(KEY_KBDINPUTASSIST_NEXT); break;
+ case 0x2c9: map_key_clear(KEY_KBDINPUTASSIST_PREVGROUP); break;
+@@ -1328,6 +1335,12 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct
+
+ input = field->hidinput->input;
+
++ if (usage->type == EV_ABS &&
++ (((*quirks & HID_QUIRK_X_INVERT) && usage->code == ABS_X) ||
++ ((*quirks & HID_QUIRK_Y_INVERT) && usage->code == ABS_Y))) {
++ value = field->logical_maximum - value;
++ }
++
+ if (usage->hat_min < usage->hat_max || usage->hat_dir) {
+ int hat_dir = usage->hat_dir;
+ if (!hat_dir)
+diff --git a/drivers/hid/hid-lg.c b/drivers/hid/hid-lg.c
+index d40af911df635..fb3f7258009c2 100644
+--- a/drivers/hid/hid-lg.c
++++ b/drivers/hid/hid-lg.c
+@@ -749,12 +749,18 @@ static int lg_raw_event(struct hid_device *hdev, struct hid_report *report,
+
+ static int lg_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+- struct usb_interface *iface = to_usb_interface(hdev->dev.parent);
+- __u8 iface_num = iface->cur_altsetting->desc.bInterfaceNumber;
++ struct usb_interface *iface;
++ __u8 iface_num;
+ unsigned int connect_mask = HID_CONNECT_DEFAULT;
+ struct lg_drv_data *drv_data;
+ int ret;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
++ iface = to_usb_interface(hdev->dev.parent);
++ iface_num = iface->cur_altsetting->desc.bInterfaceNumber;
++
+ /* G29 only work with the 1st interface */
+ if ((hdev->product == USB_DEVICE_ID_LOGITECH_G29_WHEEL) &&
+ (iface_num != 0)) {
+diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c
+index a0017b010c342..c358778e070bc 100644
+--- a/drivers/hid/hid-logitech-dj.c
++++ b/drivers/hid/hid-logitech-dj.c
+@@ -1068,6 +1068,7 @@ static void logi_hidpp_recv_queue_notif(struct hid_device *hdev,
+ workitem.reports_supported |= STD_KEYBOARD;
+ break;
+ case 0x0f:
++ case 0x11:
+ device_type = "eQUAD Lightspeed 1.2";
+ logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
+ workitem.reports_supported |= STD_KEYBOARD;
+@@ -1777,7 +1778,7 @@ static int logi_dj_probe(struct hid_device *hdev,
+ case recvr_type_bluetooth: no_dj_interfaces = 2; break;
+ case recvr_type_dinovo: no_dj_interfaces = 2; break;
+ }
+- if (hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
++ if (hid_is_usb(hdev)) {
+ intf = to_usb_interface(hdev->dev.parent);
+ if (intf && intf->altsetting->desc.bInterfaceNumber >=
+ no_dj_interfaces) {
+diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c
+index 686788ebf3e1e..b8b08f0a8c541 100644
+--- a/drivers/hid/hid-magicmouse.c
++++ b/drivers/hid/hid-magicmouse.c
+@@ -57,6 +57,8 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie
+ #define MOUSE_REPORT_ID 0x29
+ #define MOUSE2_REPORT_ID 0x12
+ #define DOUBLE_REPORT_ID 0xf7
++#define USB_BATTERY_TIMEOUT_MS 60000
++
+ /* These definitions are not precise, but they're close enough. (Bits
+ * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem
+ * to be some kind of bit mask -- 0x20 may be a near-field reading,
+@@ -140,6 +142,7 @@ struct magicmouse_sc {
+
+ struct hid_device *hdev;
+ struct delayed_work work;
++ struct timer_list battery_timer;
+ };
+
+ static int magicmouse_firm_touch(struct magicmouse_sc *msc)
+@@ -256,8 +259,11 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda
+ unsigned long now = jiffies;
+ int step_x = msc->touches[id].scroll_x - x;
+ int step_y = msc->touches[id].scroll_y - y;
+- int step_hr = ((64 - (int)scroll_speed) * msc->scroll_accel) /
+- SCROLL_HR_STEPS;
++ int step_hr =
++ max_t(int,
++ ((64 - (int)scroll_speed) * msc->scroll_accel) /
++ SCROLL_HR_STEPS,
++ 1);
+ int step_x_hr = msc->touches[id].scroll_x_hr - x;
+ int step_y_hr = msc->touches[id].scroll_y_hr - y;
+
+@@ -735,6 +741,44 @@ static void magicmouse_enable_mt_work(struct work_struct *work)
+ hid_err(msc->hdev, "unable to request touch data (%d)\n", ret);
+ }
+
++static int magicmouse_fetch_battery(struct hid_device *hdev)
++{
++#ifdef CONFIG_HID_BATTERY_STRENGTH
++ struct hid_report_enum *report_enum;
++ struct hid_report *report;
++
++ if (!hdev->battery || hdev->vendor != USB_VENDOR_ID_APPLE ||
++ (hdev->product != USB_DEVICE_ID_APPLE_MAGICMOUSE2 &&
++ hdev->product != USB_DEVICE_ID_APPLE_MAGICTRACKPAD2))
++ return -1;
++
++ report_enum = &hdev->report_enum[hdev->battery_report_type];
++ report = report_enum->report_id_hash[hdev->battery_report_id];
++
++ if (!report || report->maxfield < 1)
++ return -1;
++
++ if (hdev->battery_capacity == hdev->battery_max)
++ return -1;
++
++ hid_hw_request(hdev, report, HID_REQ_GET_REPORT);
++ return 0;
++#else
++ return -1;
++#endif
++}
++
++static void magicmouse_battery_timer_tick(struct timer_list *t)
++{
++ struct magicmouse_sc *msc = from_timer(msc, t, battery_timer);
++ struct hid_device *hdev = msc->hdev;
++
++ if (magicmouse_fetch_battery(hdev) == 0) {
++ mod_timer(&msc->battery_timer,
++ jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS));
++ }
++}
++
+ static int magicmouse_probe(struct hid_device *hdev,
+ const struct hid_device_id *id)
+ {
+@@ -742,11 +786,6 @@ static int magicmouse_probe(struct hid_device *hdev,
+ struct hid_report *report;
+ int ret;
+
+- if (id->vendor == USB_VENDOR_ID_APPLE &&
+- id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 &&
+- hdev->type != HID_TYPE_USBMOUSE)
+- return -ENODEV;
+-
+ msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL);
+ if (msc == NULL) {
+ hid_err(hdev, "can't alloc magicmouse descriptor\n");
+@@ -772,6 +811,16 @@ static int magicmouse_probe(struct hid_device *hdev,
+ return ret;
+ }
+
++ timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0);
++ mod_timer(&msc->battery_timer,
++ jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS));
++ magicmouse_fetch_battery(hdev);
++
++ if (id->vendor == USB_VENDOR_ID_APPLE &&
++ (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 ||
++ (id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 && hdev->type != HID_TYPE_USBMOUSE)))
++ return 0;
++
+ if (!msc->input) {
+ hid_err(hdev, "magicmouse input not registered\n");
+ ret = -ENOMEM;
+@@ -824,6 +873,7 @@ static int magicmouse_probe(struct hid_device *hdev,
+
+ return 0;
+ err_stop_hw:
++ del_timer_sync(&msc->battery_timer);
+ hid_hw_stop(hdev);
+ return ret;
+ }
+@@ -832,17 +882,52 @@ static void magicmouse_remove(struct hid_device *hdev)
+ {
+ struct magicmouse_sc *msc = hid_get_drvdata(hdev);
+
+- if (msc)
++ if (msc) {
+ cancel_delayed_work_sync(&msc->work);
++ del_timer_sync(&msc->battery_timer);
++ }
+
+ hid_hw_stop(hdev);
+ }
+
++static __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc,
++ unsigned int *rsize)
++{
++ /*
++ * Change the usage from:
++ * 0x06, 0x00, 0xff, // Usage Page (Vendor Defined Page 1) 0
++ * 0x09, 0x0b, // Usage (Vendor Usage 0x0b) 3
++ * To:
++ * 0x05, 0x01, // Usage Page (Generic Desktop) 0
++ * 0x09, 0x02, // Usage (Mouse) 2
++ */
++ if (hdev->vendor == USB_VENDOR_ID_APPLE &&
++ (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 ||
++ hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2) &&
++ *rsize == 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) {
++ hid_info(hdev,
++ "fixing up magicmouse battery report descriptor\n");
++ *rsize = *rsize - 1;
++ rdesc = kmemdup(rdesc + 1, *rsize, GFP_KERNEL);
++ if (!rdesc)
++ return NULL;
++
++ rdesc[0] = 0x05;
++ rdesc[1] = 0x01;
++ rdesc[2] = 0x09;
++ rdesc[3] = 0x02;
++ }
++
++ return rdesc;
++}
++
+ static const struct hid_device_id magic_mice[] = {
+ { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
+ USB_DEVICE_ID_APPLE_MAGICMOUSE), .driver_data = 0 },
+ { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE,
+ USB_DEVICE_ID_APPLE_MAGICMOUSE2), .driver_data = 0 },
++ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE,
++ USB_DEVICE_ID_APPLE_MAGICMOUSE2), .driver_data = 0 },
+ { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
+ USB_DEVICE_ID_APPLE_MAGICTRACKPAD), .driver_data = 0 },
+ { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE,
+@@ -858,6 +943,7 @@ static struct hid_driver magicmouse_driver = {
+ .id_table = magic_mice,
+ .probe = magicmouse_probe,
+ .remove = magicmouse_remove,
++ .report_fixup = magicmouse_report_fixup,
+ .raw_event = magicmouse_raw_event,
+ .event = magicmouse_event,
+ .input_mapping = magicmouse_input_mapping,
+diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
+index 3ea7cb1cda84c..e1afddb7b33d8 100644
+--- a/drivers/hid/hid-multitouch.c
++++ b/drivers/hid/hid-multitouch.c
+@@ -193,6 +193,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app);
+ /* reserved 0x0014 */
+ #define MT_CLS_WIN_8_FORCE_MULTI_INPUT 0x0015
+ #define MT_CLS_WIN_8_DISABLE_WAKEUP 0x0016
++#define MT_CLS_WIN_8_NO_STICKY_FINGERS 0x0017
+
+ /* vendor specific classes */
+ #define MT_CLS_3M 0x0101
+@@ -294,6 +295,13 @@ static const struct mt_class mt_classes[] = {
+ MT_QUIRK_WIN8_PTP_BUTTONS |
+ MT_QUIRK_DISABLE_WAKEUP,
+ .export_all_inputs = true },
++ { .name = MT_CLS_WIN_8_NO_STICKY_FINGERS,
++ .quirks = MT_QUIRK_ALWAYS_VALID |
++ MT_QUIRK_IGNORE_DUPLICATES |
++ MT_QUIRK_HOVERING |
++ MT_QUIRK_CONTACT_CNT_ACCURATE |
++ MT_QUIRK_WIN8_PTP_BUTTONS,
++ .export_all_inputs = true },
+
+ /*
+ * vendor specific classes
+@@ -2120,6 +2128,11 @@ static const struct hid_device_id mt_devices[] = {
+ MT_USB_DEVICE(USB_VENDOR_ID_VTL,
+ USB_DEVICE_ID_VTL_MULTITOUCH_FF3F) },
+
++ /* Winbond Electronics Corp. */
++ { .driver_data = MT_CLS_WIN_8_NO_STICKY_FINGERS,
++ HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH_WIN_8,
++ USB_VENDOR_ID_WINBOND, USB_DEVICE_ID_TSTP_MTOUCH) },
++
+ /* Wistron panels */
+ { .driver_data = MT_CLS_NSMU,
+ MT_USB_DEVICE(USB_VENDOR_ID_WISTRON,
+diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c
+index 2666af02d5c1a..e4e9471d0f1e9 100644
+--- a/drivers/hid/hid-prodikeys.c
++++ b/drivers/hid/hid-prodikeys.c
+@@ -798,12 +798,18 @@ static int pk_raw_event(struct hid_device *hdev, struct hid_report *report,
+ static int pk_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+ int ret;
+- struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+- unsigned short ifnum = intf->cur_altsetting->desc.bInterfaceNumber;
++ struct usb_interface *intf;
++ unsigned short ifnum;
+ unsigned long quirks = id->driver_data;
+ struct pk_device *pk;
+ struct pcmidi_snd *pm = NULL;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
++ intf = to_usb_interface(hdev->dev.parent);
++ ifnum = intf->cur_altsetting->desc.bInterfaceNumber;
++
+ pk = kzalloc(sizeof(*pk), GFP_KERNEL);
+ if (pk == NULL) {
+ hid_err(hdev, "can't alloc descriptor\n");
+diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
+index 2e104682c22b9..544d1197aca48 100644
+--- a/drivers/hid/hid-quirks.c
++++ b/drivers/hid/hid-quirks.c
+@@ -124,6 +124,7 @@ static const struct hid_device_id hid_quirks[] = {
+ { HID_USB_DEVICE(USB_VENDOR_ID_MCS, USB_DEVICE_ID_MCS_GAMEPADBLOCK), HID_QUIRK_MULTI_INPUT },
+ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PIXART_MOUSE), HID_QUIRK_ALWAYS_POLL },
+ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER), HID_QUIRK_NO_INIT_REPORTS },
++ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE3_COVER), HID_QUIRK_NO_INIT_REPORTS },
+ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE_PRO_2), HID_QUIRK_NO_INIT_REPORTS },
+ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TOUCH_COVER_2), HID_QUIRK_NO_INIT_REPORTS },
+ { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_2), HID_QUIRK_NO_INIT_REPORTS },
+@@ -186,6 +187,7 @@ static const struct hid_device_id hid_quirks[] = {
+ { HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_TURBOX_KEYBOARD), HID_QUIRK_NOGET },
+ { HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_KNA5), HID_QUIRK_MULTI_INPUT },
+ { HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWA60), HID_QUIRK_MULTI_INPUT },
++ { HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_WP5540), HID_QUIRK_MULTI_INPUT },
+ { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH), HID_QUIRK_MULTI_INPUT },
+ { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH), HID_QUIRK_MULTI_INPUT },
+ { HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET), HID_QUIRK_MULTI_INPUT },
+diff --git a/drivers/hid/hid-roccat-arvo.c b/drivers/hid/hid-roccat-arvo.c
+index 4556d2a50f754..d94ee0539421e 100644
+--- a/drivers/hid/hid-roccat-arvo.c
++++ b/drivers/hid/hid-roccat-arvo.c
+@@ -344,6 +344,9 @@ static int arvo_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-isku.c b/drivers/hid/hid-roccat-isku.c
+index ce5f22519956a..e95d59cd8d075 100644
+--- a/drivers/hid/hid-roccat-isku.c
++++ b/drivers/hid/hid-roccat-isku.c
+@@ -324,6 +324,9 @@ static int isku_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c
+index 1ca64481145ee..e8522eacf7973 100644
+--- a/drivers/hid/hid-roccat-kone.c
++++ b/drivers/hid/hid-roccat-kone.c
+@@ -749,6 +749,9 @@ static int kone_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-koneplus.c b/drivers/hid/hid-roccat-koneplus.c
+index 0316edf8c5bb4..1896c69ea512f 100644
+--- a/drivers/hid/hid-roccat-koneplus.c
++++ b/drivers/hid/hid-roccat-koneplus.c
+@@ -431,6 +431,9 @@ static int koneplus_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-konepure.c b/drivers/hid/hid-roccat-konepure.c
+index 5248b3c7cf785..cf8eeb33a1257 100644
+--- a/drivers/hid/hid-roccat-konepure.c
++++ b/drivers/hid/hid-roccat-konepure.c
+@@ -133,6 +133,9 @@ static int konepure_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-kovaplus.c b/drivers/hid/hid-roccat-kovaplus.c
+index 9600128815705..6fb9b9563769d 100644
+--- a/drivers/hid/hid-roccat-kovaplus.c
++++ b/drivers/hid/hid-roccat-kovaplus.c
+@@ -501,6 +501,9 @@ static int kovaplus_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-lua.c b/drivers/hid/hid-roccat-lua.c
+index 4a88a76d5c622..d5ddf0d68346b 100644
+--- a/drivers/hid/hid-roccat-lua.c
++++ b/drivers/hid/hid-roccat-lua.c
+@@ -160,6 +160,9 @@ static int lua_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-pyra.c b/drivers/hid/hid-roccat-pyra.c
+index 989927defe8db..4fcc8e7d276f2 100644
+--- a/drivers/hid/hid-roccat-pyra.c
++++ b/drivers/hid/hid-roccat-pyra.c
+@@ -449,6 +449,9 @@ static int pyra_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-ryos.c b/drivers/hid/hid-roccat-ryos.c
+index 3956a6c9c5217..5bf1971a2b14d 100644
+--- a/drivers/hid/hid-roccat-ryos.c
++++ b/drivers/hid/hid-roccat-ryos.c
+@@ -141,6 +141,9 @@ static int ryos_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-roccat-savu.c b/drivers/hid/hid-roccat-savu.c
+index 818701f7a0281..a784bb4ee6512 100644
+--- a/drivers/hid/hid-roccat-savu.c
++++ b/drivers/hid/hid-roccat-savu.c
+@@ -113,6 +113,9 @@ static int savu_probe(struct hid_device *hdev,
+ {
+ int retval;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ retval = hid_parse(hdev);
+ if (retval) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-samsung.c b/drivers/hid/hid-samsung.c
+index 2e1c31156eca0..cf5992e970940 100644
+--- a/drivers/hid/hid-samsung.c
++++ b/drivers/hid/hid-samsung.c
+@@ -152,6 +152,9 @@ static int samsung_probe(struct hid_device *hdev,
+ int ret;
+ unsigned int cmask = HID_CONNECT_DEFAULT;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ ret = hid_parse(hdev);
+ if (ret) {
+ hid_err(hdev, "parse failed\n");
+diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
+index d1b107d547f54..60ec2b29d54de 100644
+--- a/drivers/hid/hid-sony.c
++++ b/drivers/hid/hid-sony.c
+@@ -3000,7 +3000,6 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ sc->quirks = quirks;
+ hid_set_drvdata(hdev, sc);
+ sc->hdev = hdev;
+- usbdev = to_usb_device(sc->hdev->dev.parent->parent);
+
+ ret = hid_parse(hdev);
+ if (ret) {
+@@ -3038,14 +3037,23 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ */
+ if (!(hdev->claimed & HID_CLAIMED_INPUT)) {
+ hid_err(hdev, "failed to claim input\n");
+- hid_hw_stop(hdev);
+- return -ENODEV;
++ ret = -ENODEV;
++ goto err;
+ }
+
+ if (sc->quirks & (GHL_GUITAR_PS3WIIU | GHL_GUITAR_PS4)) {
++ if (!hid_is_usb(hdev)) {
++ ret = -EINVAL;
++ goto err;
++ }
++
++ usbdev = to_usb_device(sc->hdev->dev.parent->parent);
++
+ sc->ghl_urb = usb_alloc_urb(0, GFP_ATOMIC);
+- if (!sc->ghl_urb)
+- return -ENOMEM;
++ if (!sc->ghl_urb) {
++ ret = -ENOMEM;
++ goto err;
++ }
+
+ if (sc->quirks & GHL_GUITAR_PS3WIIU)
+ ret = ghl_init_urb(sc, usbdev, ghl_ps3wiiu_magic_data,
+@@ -3055,7 +3063,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ ARRAY_SIZE(ghl_ps4_magic_data));
+ if (ret) {
+ hid_err(hdev, "error preparing URB\n");
+- return ret;
++ goto err;
+ }
+
+ timer_setup(&sc->ghl_poke_timer, ghl_magic_poke, 0);
+@@ -3064,6 +3072,10 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ }
+
+ return ret;
++
++err:
++ hid_hw_stop(hdev);
++ return ret;
+ }
+
+ static void sony_remove(struct hid_device *hdev)
+diff --git a/drivers/hid/hid-thrustmaster.c b/drivers/hid/hid-thrustmaster.c
+index d44550aa88057..a28c3e5756506 100644
+--- a/drivers/hid/hid-thrustmaster.c
++++ b/drivers/hid/hid-thrustmaster.c
+@@ -64,7 +64,9 @@ struct tm_wheel_info {
+ */
+ static const struct tm_wheel_info tm_wheels_infos[] = {
+ {0x0306, 0x0006, "Thrustmaster T150RS"},
++ {0x0200, 0x0005, "Thrustmaster T300RS (Missing Attachment)"},
+ {0x0206, 0x0005, "Thrustmaster T300RS"},
++ {0x0209, 0x0005, "Thrustmaster T300RS (Open Wheel Attachment)"},
+ {0x0204, 0x0005, "Thrustmaster T300 Ferrari Alcantara Edition"},
+ {0x0002, 0x0002, "Thrustmaster T500RS"}
+ //{0x0407, 0x0001, "Thrustmaster TMX"}
+@@ -158,6 +160,12 @@ static void thrustmaster_interrupts(struct hid_device *hdev)
+ return;
+ }
+
++ if (usbif->cur_altsetting->desc.bNumEndpoints < 2) {
++ kfree(send_buf);
++ hid_err(hdev, "Wrong number of endpoints?\n");
++ return;
++ }
++
+ ep = &usbif->cur_altsetting->endpoint[1];
+ b_ep = ep->desc.bEndpointAddress;
+
+@@ -274,6 +282,9 @@ static int thrustmaster_probe(struct hid_device *hdev, const struct hid_device_i
+ int ret = 0;
+ struct tm_wheel *tm_wheel = 0;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ ret = hid_parse(hdev);
+ if (ret) {
+ hid_err(hdev, "parse failed with error %d\n", ret);
+diff --git a/drivers/hid/hid-u2fzero.c b/drivers/hid/hid-u2fzero.c
+index d70cd3d7f583b..ac3fd870673d2 100644
+--- a/drivers/hid/hid-u2fzero.c
++++ b/drivers/hid/hid-u2fzero.c
+@@ -132,7 +132,7 @@ static int u2fzero_recv(struct u2fzero_device *dev,
+
+ ret = (wait_for_completion_timeout(
+ &ctx.done, msecs_to_jiffies(USB_CTRL_SET_TIMEOUT)));
+- if (ret < 0) {
++ if (ret == 0) {
+ usb_kill_urb(dev->urb);
+ hid_err(hdev, "urb submission timed out");
+ } else {
+@@ -191,6 +191,8 @@ static int u2fzero_rng_read(struct hwrng *rng, void *data,
+ struct u2f_hid_msg resp;
+ int ret;
+ size_t actual_length;
++ /* valid packets must have a correct header */
++ int min_length = offsetof(struct u2f_hid_msg, init.data);
+
+ if (!dev->present) {
+ hid_dbg(dev->hdev, "device not present");
+@@ -200,12 +202,12 @@ static int u2fzero_rng_read(struct hwrng *rng, void *data,
+ ret = u2fzero_recv(dev, &req, &resp);
+
+ /* ignore errors or packets without data */
+- if (ret < offsetof(struct u2f_hid_msg, init.data))
++ if (ret < min_length)
+ return 0;
+
+ /* only take the minimum amount of data it is safe to take */
+- actual_length = min3((size_t)ret - offsetof(struct u2f_hid_msg,
+- init.data), U2F_HID_MSG_LEN(resp), max);
++ actual_length = min3((size_t)ret - min_length,
++ U2F_HID_MSG_LEN(resp), max);
+
+ memcpy(data, resp.init.data, actual_length);
+
+@@ -288,7 +290,7 @@ static int u2fzero_probe(struct hid_device *hdev,
+ unsigned int minor;
+ int ret;
+
+- if (!hid_is_using_ll_driver(hdev, &usb_hid_driver))
++ if (!hid_is_usb(hdev))
+ return -EINVAL;
+
+ dev = devm_kzalloc(&hdev->dev, sizeof(*dev), GFP_KERNEL);
+diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c
+index 6a9865dd703c0..d8ab0139e5cda 100644
+--- a/drivers/hid/hid-uclogic-core.c
++++ b/drivers/hid/hid-uclogic-core.c
+@@ -164,6 +164,9 @@ static int uclogic_probe(struct hid_device *hdev,
+ struct uclogic_drvdata *drvdata = NULL;
+ bool params_initialized = false;
+
++ if (!hid_is_usb(hdev))
++ return -EINVAL;
++
+ /*
+ * libinput requires the pad interface to be on a different node
+ * than the pen, so use QUIRK_MULTI_INPUT for all tablets.
+diff --git a/drivers/hid/hid-uclogic-params.c b/drivers/hid/hid-uclogic-params.c
+index 3d67b748a3b95..3e70f969fb849 100644
+--- a/drivers/hid/hid-uclogic-params.c
++++ b/drivers/hid/hid-uclogic-params.c
+@@ -66,7 +66,7 @@ static int uclogic_params_get_str_desc(__u8 **pbuf, struct hid_device *hdev,
+ __u8 idx, size_t len)
+ {
+ int rc;
+- struct usb_device *udev = hid_to_usb_dev(hdev);
++ struct usb_device *udev;
+ __u8 *buf = NULL;
+
+ /* Check arguments */
+@@ -75,6 +75,8 @@ static int uclogic_params_get_str_desc(__u8 **pbuf, struct hid_device *hdev,
+ goto cleanup;
+ }
+
++ udev = hid_to_usb_dev(hdev);
++
+ buf = kmalloc(len, GFP_KERNEL);
+ if (buf == NULL) {
+ rc = -ENOMEM;
+@@ -450,7 +452,7 @@ static int uclogic_params_frame_init_v1_buttonpad(
+ {
+ int rc;
+ bool found = false;
+- struct usb_device *usb_dev = hid_to_usb_dev(hdev);
++ struct usb_device *usb_dev;
+ char *str_buf = NULL;
+ const size_t str_len = 16;
+
+@@ -460,6 +462,8 @@ static int uclogic_params_frame_init_v1_buttonpad(
+ goto cleanup;
+ }
+
++ usb_dev = hid_to_usb_dev(hdev);
++
+ /*
+ * Enable generic button mode
+ */
+@@ -707,9 +711,9 @@ static int uclogic_params_huion_init(struct uclogic_params *params,
+ struct hid_device *hdev)
+ {
+ int rc;
+- struct usb_device *udev = hid_to_usb_dev(hdev);
+- struct usb_interface *iface = to_usb_interface(hdev->dev.parent);
+- __u8 bInterfaceNumber = iface->cur_altsetting->desc.bInterfaceNumber;
++ struct usb_device *udev;
++ struct usb_interface *iface;
++ __u8 bInterfaceNumber;
+ bool found;
+ /* The resulting parameters (noop) */
+ struct uclogic_params p = {0, };
+@@ -723,6 +727,10 @@ static int uclogic_params_huion_init(struct uclogic_params *params,
+ goto cleanup;
+ }
+
++ udev = hid_to_usb_dev(hdev);
++ iface = to_usb_interface(hdev->dev.parent);
++ bInterfaceNumber = iface->cur_altsetting->desc.bInterfaceNumber;
++
+ /* If it's not a pen interface */
+ if (bInterfaceNumber != 0) {
+ /* TODO: Consider marking the interface invalid */
+@@ -834,21 +842,25 @@ int uclogic_params_init(struct uclogic_params *params,
+ struct hid_device *hdev)
+ {
+ int rc;
+- struct usb_device *udev = hid_to_usb_dev(hdev);
+- __u8 bNumInterfaces = udev->config->desc.bNumInterfaces;
+- struct usb_interface *iface = to_usb_interface(hdev->dev.parent);
+- __u8 bInterfaceNumber = iface->cur_altsetting->desc.bInterfaceNumber;
++ struct usb_device *udev;
++ __u8 bNumInterfaces;
++ struct usb_interface *iface;
++ __u8 bInterfaceNumber;
+ bool found;
+ /* The resulting parameters (noop) */
+ struct uclogic_params p = {0, };
+
+ /* Check arguments */
+- if (params == NULL || hdev == NULL ||
+- !hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
++ if (params == NULL || hdev == NULL || !hid_is_usb(hdev)) {
+ rc = -EINVAL;
+ goto cleanup;
+ }
+
++ udev = hid_to_usb_dev(hdev);
++ bNumInterfaces = udev->config->desc.bNumInterfaces;
++ iface = to_usb_interface(hdev->dev.parent);
++ bInterfaceNumber = iface->cur_altsetting->desc.bInterfaceNumber;
++
+ /*
+ * Set replacement report descriptor if the original matches the
+ * specified size. Otherwise keep interface unchanged.
+diff --git a/drivers/hid/hid-vivaldi.c b/drivers/hid/hid-vivaldi.c
+index cd7ada48b1d9f..d57ec17670379 100644
+--- a/drivers/hid/hid-vivaldi.c
++++ b/drivers/hid/hid-vivaldi.c
+@@ -57,6 +57,9 @@ static int vivaldi_probe(struct hid_device *hdev,
+ int ret;
+
+ drvdata = devm_kzalloc(&hdev->dev, sizeof(*drvdata), GFP_KERNEL);
++ if (!drvdata)
++ return -ENOMEM;
++
+ hid_set_drvdata(hdev, drvdata);
+
+ ret = hid_parse(hdev);
+@@ -71,10 +74,11 @@ static void vivaldi_feature_mapping(struct hid_device *hdev,
+ struct hid_usage *usage)
+ {
+ struct vivaldi_data *drvdata = hid_get_drvdata(hdev);
++ struct hid_report *report = field->report;
+ int fn_key;
+ int ret;
+ u32 report_len;
+- u8 *buf;
++ u8 *report_data, *buf;
+
+ if (field->logical != HID_USAGE_FN_ROW_PHYSMAP ||
+ (usage->hid & HID_USAGE_PAGE) != HID_UP_ORDINAL)
+@@ -86,12 +90,24 @@ static void vivaldi_feature_mapping(struct hid_device *hdev,
+ if (fn_key > drvdata->max_function_row_key)
+ drvdata->max_function_row_key = fn_key;
+
+- buf = hid_alloc_report_buf(field->report, GFP_KERNEL);
+- if (!buf)
++ report_data = buf = hid_alloc_report_buf(report, GFP_KERNEL);
++ if (!report_data)
+ return;
+
+- report_len = hid_report_len(field->report);
+- ret = hid_hw_raw_request(hdev, field->report->id, buf,
++ report_len = hid_report_len(report);
++ if (!report->id) {
++ /*
++ * hid_hw_raw_request() will stuff report ID (which will be 0)
++ * into the first byte of the buffer even for unnumbered
++ * reports, so we need to account for this to avoid getting
++ * -EOVERFLOW in return.
++ * Note that hid_alloc_report_buf() adds 7 bytes to the size
++ * so we can safely say that we have space for an extra byte.
++ */
++ report_len++;
++ }
++
++ ret = hid_hw_raw_request(hdev, report->id, report_data,
+ report_len, HID_FEATURE_REPORT,
+ HID_REQ_GET_REPORT);
+ if (ret < 0) {
+@@ -100,7 +116,16 @@ static void vivaldi_feature_mapping(struct hid_device *hdev,
+ goto out;
+ }
+
+- ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf,
++ if (!report->id) {
++ /*
++ * Undo the damage from hid_hw_raw_request() for unnumbered
++ * reports.
++ */
++ report_data++;
++ report_len--;
++ }
++
++ ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, report_data,
+ report_len, 0);
+ if (ret) {
+ dev_warn(&hdev->dev, "failed to report feature %d\n",
+@@ -118,7 +143,7 @@ out:
+ static int vivaldi_input_configured(struct hid_device *hdev,
+ struct hid_input *hidinput)
+ {
+- return sysfs_create_group(&hdev->dev.kobj, &input_attribute_group);
++ return devm_device_add_group(&hdev->dev, &input_attribute_group);
+ }
+
+ static const struct hid_device_id vivaldi_table[] = {
+diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c
+index a6f0257a26de3..b96ae15e0ad91 100644
+--- a/drivers/hid/i2c-hid/i2c-hid-acpi.c
++++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c
+@@ -111,7 +111,7 @@ static int i2c_hid_acpi_probe(struct i2c_client *client)
+ }
+
+ return i2c_hid_core_probe(client, &ihid_acpi->ops,
+- hid_descriptor_address);
++ hid_descriptor_address, 0);
+ }
+
+ static const struct acpi_device_id i2c_hid_acpi_match[] = {
+diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c
+index 517141138b007..65c1f20ec420a 100644
+--- a/drivers/hid/i2c-hid/i2c-hid-core.c
++++ b/drivers/hid/i2c-hid/i2c-hid-core.c
+@@ -615,6 +615,17 @@ static int i2c_hid_get_raw_report(struct hid_device *hid,
+ if (report_type == HID_OUTPUT_REPORT)
+ return -EINVAL;
+
++ /*
++ * In case of unnumbered reports the response from the device will
++ * not have the report ID that the upper layers expect, so we need
++ * to stash it the buffer ourselves and adjust the data size.
++ */
++ if (!report_number) {
++ buf[0] = 0;
++ buf++;
++ count--;
++ }
++
+ /* +2 bytes to include the size of the reply in the query buffer */
+ ask_count = min(count + 2, (size_t)ihid->bufsize);
+
+@@ -636,6 +647,9 @@ static int i2c_hid_get_raw_report(struct hid_device *hid,
+ count = min(count, ret_count - 2);
+ memcpy(buf, ihid->rawbuf + 2, count);
+
++ if (!report_number)
++ count++;
++
+ return count;
+ }
+
+@@ -652,17 +666,19 @@ static int i2c_hid_output_raw_report(struct hid_device *hid, __u8 *buf,
+
+ mutex_lock(&ihid->reset_lock);
+
+- if (report_id) {
+- buf++;
+- count--;
+- }
+-
++ /*
++ * Note that both numbered and unnumbered reports passed here
++ * are supposed to have report ID stored in the 1st byte of the
++ * buffer, so we strip it off unconditionally before passing payload
++ * to i2c_hid_set_or_send_report which takes care of encoding
++ * everything properly.
++ */
+ ret = i2c_hid_set_or_send_report(client,
+ report_type == HID_FEATURE_REPORT ? 0x03 : 0x02,
+- report_id, buf, count, use_data);
++ report_id, buf + 1, count - 1, use_data);
+
+- if (report_id && ret >= 0)
+- ret++; /* add report_id to the number of transfered bytes */
++ if (ret >= 0)
++ ret++; /* add report_id to the number of transferred bytes */
+
+ mutex_unlock(&ihid->reset_lock);
+
+@@ -912,7 +928,7 @@ static void i2c_hid_core_shutdown_tail(struct i2c_hid *ihid)
+ }
+
+ int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops,
+- u16 hid_descriptor_address)
++ u16 hid_descriptor_address, u32 quirks)
+ {
+ int ret;
+ struct i2c_hid *ihid;
+@@ -1009,6 +1025,8 @@ int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops,
+ goto err_mem_free;
+ }
+
++ hid->quirks |= quirks;
++
+ return 0;
+
+ err_mem_free:
+diff --git a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c
+index 52674149a2750..ec6c73f75ffe0 100644
+--- a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c
++++ b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c
+@@ -27,7 +27,6 @@ struct i2c_hid_of_goodix {
+
+ struct regulator *vdd;
+ struct notifier_block nb;
+- struct mutex regulator_mutex;
+ struct gpio_desc *reset_gpio;
+ const struct goodix_i2c_hid_timing_data *timings;
+ };
+@@ -67,8 +66,6 @@ static int ihid_goodix_vdd_notify(struct notifier_block *nb,
+ container_of(nb, struct i2c_hid_of_goodix, nb);
+ int ret = NOTIFY_OK;
+
+- mutex_lock(&ihid_goodix->regulator_mutex);
+-
+ switch (event) {
+ case REGULATOR_EVENT_PRE_DISABLE:
+ gpiod_set_value_cansleep(ihid_goodix->reset_gpio, 1);
+@@ -87,8 +84,6 @@ static int ihid_goodix_vdd_notify(struct notifier_block *nb,
+ break;
+ }
+
+- mutex_unlock(&ihid_goodix->regulator_mutex);
+-
+ return ret;
+ }
+
+@@ -102,8 +97,6 @@ static int i2c_hid_of_goodix_probe(struct i2c_client *client,
+ if (!ihid_goodix)
+ return -ENOMEM;
+
+- mutex_init(&ihid_goodix->regulator_mutex);
+-
+ ihid_goodix->ops.power_up = goodix_i2c_hid_power_up;
+ ihid_goodix->ops.power_down = goodix_i2c_hid_power_down;
+
+@@ -130,27 +123,30 @@ static int i2c_hid_of_goodix_probe(struct i2c_client *client,
+ * long. Holding the controller in reset apparently draws extra
+ * power.
+ */
+- mutex_lock(&ihid_goodix->regulator_mutex);
+ ihid_goodix->nb.notifier_call = ihid_goodix_vdd_notify;
+ ret = devm_regulator_register_notifier(ihid_goodix->vdd, &ihid_goodix->nb);
+- if (ret) {
+- mutex_unlock(&ihid_goodix->regulator_mutex);
++ if (ret)
+ return dev_err_probe(&client->dev, ret,
+ "regulator notifier request failed\n");
+- }
+
+ /*
+ * If someone else is holding the regulator on (or the regulator is
+ * an always-on one) we might never be told to deassert reset. Do it
+- * now. Here we'll assume that someone else might have _just
+- * barely_ turned the regulator on so we'll do the full
+- * "post_power_delay" just in case.
++ * now... and temporarily bump the regulator reference count just to
++ * make sure it is impossible for this to race with our own notifier!
++ * We also assume that someone else might have _just barely_ turned
++ * the regulator on so we'll do the full "post_power_delay" just in
++ * case.
+ */
+- if (ihid_goodix->reset_gpio && regulator_is_enabled(ihid_goodix->vdd))
++ if (ihid_goodix->reset_gpio && regulator_is_enabled(ihid_goodix->vdd)) {
++ ret = regulator_enable(ihid_goodix->vdd);
++ if (ret)
++ return ret;
+ goodix_i2c_hid_deassert_reset(ihid_goodix, true);
+- mutex_unlock(&ihid_goodix->regulator_mutex);
++ regulator_disable(ihid_goodix->vdd);
++ }
+
+- return i2c_hid_core_probe(client, &ihid_goodix->ops, 0x0001);
++ return i2c_hid_core_probe(client, &ihid_goodix->ops, 0x0001, 0);
+ }
+
+ static const struct goodix_i2c_hid_timing_data goodix_gt7375p_timing_data = {
+diff --git a/drivers/hid/i2c-hid/i2c-hid-of.c b/drivers/hid/i2c-hid/i2c-hid-of.c
+index 4bf7cea926379..97a27a803f58d 100644
+--- a/drivers/hid/i2c-hid/i2c-hid-of.c
++++ b/drivers/hid/i2c-hid/i2c-hid-of.c
+@@ -21,6 +21,7 @@
+
+ #include <linux/delay.h>
+ #include <linux/device.h>
++#include <linux/hid.h>
+ #include <linux/i2c.h>
+ #include <linux/kernel.h>
+ #include <linux/module.h>
+@@ -71,6 +72,7 @@ static int i2c_hid_of_probe(struct i2c_client *client,
+ struct device *dev = &client->dev;
+ struct i2c_hid_of *ihid_of;
+ u16 hid_descriptor_address;
++ u32 quirks = 0;
+ int ret;
+ u32 val;
+
+@@ -105,8 +107,14 @@ static int i2c_hid_of_probe(struct i2c_client *client,
+ if (ret)
+ return ret;
+
++ if (device_property_read_bool(dev, "touchscreen-inverted-x"))
++ quirks |= HID_QUIRK_X_INVERT;
++
++ if (device_property_read_bool(dev, "touchscreen-inverted-y"))
++ quirks |= HID_QUIRK_Y_INVERT;
++
+ return i2c_hid_core_probe(client, &ihid_of->ops,
+- hid_descriptor_address);
++ hid_descriptor_address, quirks);
+ }
+
+ static const struct of_device_id i2c_hid_of_match[] = {
+diff --git a/drivers/hid/i2c-hid/i2c-hid.h b/drivers/hid/i2c-hid/i2c-hid.h
+index 05a7827d211af..236cc062d5ef8 100644
+--- a/drivers/hid/i2c-hid/i2c-hid.h
++++ b/drivers/hid/i2c-hid/i2c-hid.h
+@@ -32,7 +32,7 @@ struct i2chid_ops {
+ };
+
+ int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops,
+- u16 hid_descriptor_address);
++ u16 hid_descriptor_address, u32 quirks);
+ int i2c_hid_core_remove(struct i2c_client *client);
+
+ void i2c_hid_core_shutdown(struct i2c_client *client);
+diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
+index 1c5039081db27..8e9d9450cb835 100644
+--- a/drivers/hid/intel-ish-hid/ipc/pci-ish.c
++++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
+@@ -266,7 +266,8 @@ static void __maybe_unused ish_resume_handler(struct work_struct *work)
+
+ if (ish_should_leave_d0i3(pdev) && !dev->suspend_flag
+ && IPC_IS_ISH_ILUP(fwsts)) {
+- disable_irq_wake(pdev->irq);
++ if (device_may_wakeup(&pdev->dev))
++ disable_irq_wake(pdev->irq);
+
+ ish_set_host_ready(dev);
+
+@@ -337,7 +338,8 @@ static int __maybe_unused ish_suspend(struct device *device)
+ */
+ pci_save_state(pdev);
+
+- enable_irq_wake(pdev->irq);
++ if (device_may_wakeup(&pdev->dev))
++ enable_irq_wake(pdev->irq);
+ }
+ } else {
+ /*
+diff --git a/drivers/hid/intel-ish-hid/ishtp-fw-loader.c b/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
+index 1b486f2627477..6b511fadf7ad2 100644
+--- a/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
++++ b/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
+@@ -657,21 +657,12 @@ static int ish_fw_xfer_direct_dma(struct ishtp_cl_data *client_data,
+ */
+ payload_max_size &= ~(L1_CACHE_BYTES - 1);
+
+- dma_buf = kmalloc(payload_max_size, GFP_KERNEL | GFP_DMA32);
++ dma_buf = dma_alloc_coherent(devc, payload_max_size, &dma_buf_phy, GFP_KERNEL);
+ if (!dma_buf) {
+ client_data->flag_retry = true;
+ return -ENOMEM;
+ }
+
+- dma_buf_phy = dma_map_single(devc, dma_buf, payload_max_size,
+- DMA_TO_DEVICE);
+- if (dma_mapping_error(devc, dma_buf_phy)) {
+- dev_err(cl_data_to_dev(client_data), "DMA map failed\n");
+- client_data->flag_retry = true;
+- rv = -ENOMEM;
+- goto end_err_dma_buf_release;
+- }
+-
+ ldr_xfer_dma_frag.fragment.hdr.command = LOADER_CMD_XFER_FRAGMENT;
+ ldr_xfer_dma_frag.fragment.xfer_mode = LOADER_XFER_MODE_DIRECT_DMA;
+ ldr_xfer_dma_frag.ddr_phys_addr = (u64)dma_buf_phy;
+@@ -691,14 +682,7 @@ static int ish_fw_xfer_direct_dma(struct ishtp_cl_data *client_data,
+ ldr_xfer_dma_frag.fragment.size = fragment_size;
+ memcpy(dma_buf, &fw->data[fragment_offset], fragment_size);
+
+- dma_sync_single_for_device(devc, dma_buf_phy,
+- payload_max_size,
+- DMA_TO_DEVICE);
+-
+- /*
+- * Flush cache here because the dma_sync_single_for_device()
+- * does not do for x86.
+- */
++ /* Flush cache to be sure the data is in main memory. */
+ clflush_cache_range(dma_buf, payload_max_size);
+
+ dev_dbg(cl_data_to_dev(client_data),
+@@ -721,15 +705,8 @@ static int ish_fw_xfer_direct_dma(struct ishtp_cl_data *client_data,
+ fragment_offset += fragment_size;
+ }
+
+- dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
+- kfree(dma_buf);
+- return 0;
+-
+ end_err_resp_buf_release:
+- /* Free ISH buffer if not done already, in error case */
+- dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
+-end_err_dma_buf_release:
+- kfree(dma_buf);
++ dma_free_coherent(devc, payload_max_size, dma_buf, dma_buf_phy);
+ return rv;
+ }
+
+diff --git a/drivers/hid/surface-hid/surface_hid.c b/drivers/hid/surface-hid/surface_hid.c
+index a3a70e4f3f6c9..d4aa8c81903ae 100644
+--- a/drivers/hid/surface-hid/surface_hid.c
++++ b/drivers/hid/surface-hid/surface_hid.c
+@@ -209,7 +209,7 @@ static int surface_hid_probe(struct ssam_device *sdev)
+
+ shid->notif.base.priority = 1;
+ shid->notif.base.fn = ssam_hid_event_fn;
+- shid->notif.event.reg = SSAM_EVENT_REGISTRY_REG;
++ shid->notif.event.reg = SSAM_EVENT_REGISTRY_REG(sdev->uid.target);
+ shid->notif.event.id.target_category = sdev->uid.category;
+ shid->notif.event.id.instance = sdev->uid.instance;
+ shid->notif.event.mask = SSAM_EVENT_MASK_STRICT;
+@@ -230,7 +230,7 @@ static void surface_hid_remove(struct ssam_device *sdev)
+ }
+
+ static const struct ssam_device_id surface_hid_match[] = {
+- { SSAM_SDEV(HID, 0x02, SSAM_ANY_IID, 0x00) },
++ { SSAM_SDEV(HID, SSAM_ANY_TID, SSAM_ANY_IID, 0x00) },
+ { },
+ };
+ MODULE_DEVICE_TABLE(ssam, surface_hid_match);
+diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c
+index 8fe3efcb83271..fc06d8bb42e0f 100644
+--- a/drivers/hid/uhid.c
++++ b/drivers/hid/uhid.c
+@@ -28,11 +28,22 @@
+
+ struct uhid_device {
+ struct mutex devlock;
++
++ /* This flag tracks whether the HID device is usable for commands from
++ * userspace. The flag is already set before hid_add_device(), which
++ * runs in workqueue context, to allow hid_add_device() to communicate
++ * with userspace.
++ * However, if hid_add_device() fails, the flag is cleared without
++ * holding devlock.
++ * We guarantee that if @running changes from true to false while you're
++ * holding @devlock, it's still fine to access @hid.
++ */
+ bool running;
+
+ __u8 *rd_data;
+ uint rd_size;
+
++ /* When this is NULL, userspace may use UHID_CREATE/UHID_CREATE2. */
+ struct hid_device *hid;
+ struct uhid_event input_buf;
+
+@@ -63,9 +74,18 @@ static void uhid_device_add_worker(struct work_struct *work)
+ if (ret) {
+ hid_err(uhid->hid, "Cannot register HID device: error %d\n", ret);
+
+- hid_destroy_device(uhid->hid);
+- uhid->hid = NULL;
++ /* We used to call hid_destroy_device() here, but that's really
++ * messy to get right because we have to coordinate with
++ * concurrent writes from userspace that might be in the middle
++ * of using uhid->hid.
++ * Just leave uhid->hid as-is for now, and clean it up when
++ * userspace tries to close or reinitialize the uhid instance.
++ *
++ * However, we do have to clear the ->running flag and do a
++ * wakeup to make sure userspace knows that the device is gone.
++ */
+ uhid->running = false;
++ wake_up_interruptible(&uhid->report_wait);
+ }
+ }
+
+@@ -474,7 +494,7 @@ static int uhid_dev_create2(struct uhid_device *uhid,
+ void *rd_data;
+ int ret;
+
+- if (uhid->running)
++ if (uhid->hid)
+ return -EALREADY;
+
+ rd_size = ev->u.create2.rd_size;
+@@ -556,7 +576,7 @@ static int uhid_dev_create(struct uhid_device *uhid,
+
+ static int uhid_dev_destroy(struct uhid_device *uhid)
+ {
+- if (!uhid->running)
++ if (!uhid->hid)
+ return -EINVAL;
+
+ uhid->running = false;
+@@ -565,6 +585,7 @@ static int uhid_dev_destroy(struct uhid_device *uhid)
+ cancel_work_sync(&uhid->worker);
+
+ hid_destroy_device(uhid->hid);
++ uhid->hid = NULL;
+ kfree(uhid->rd_data);
+
+ return 0;
+diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
+index 93f49b766376e..b1bbf297f66b0 100644
+--- a/drivers/hid/wacom_sys.c
++++ b/drivers/hid/wacom_sys.c
+@@ -726,7 +726,7 @@ static void wacom_retrieve_hid_descriptor(struct hid_device *hdev,
+ * Skip the query for this type and modify defaults based on
+ * interface number.
+ */
+- if (features->type == WIRELESS) {
++ if (features->type == WIRELESS && intf) {
+ if (intf->cur_altsetting->desc.bInterfaceNumber == 0)
+ features->device_type = WACOM_DEVICETYPE_WL_MONITOR;
+ else
+@@ -2217,7 +2217,7 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix)
+ if ((features->type == HID_GENERIC) && !strcmp("Wacom HID", features->name)) {
+ char *product_name = wacom->hdev->name;
+
+- if (hid_is_using_ll_driver(wacom->hdev, &usb_hid_driver)) {
++ if (hid_is_usb(wacom->hdev)) {
+ struct usb_interface *intf = to_usb_interface(wacom->hdev->dev.parent);
+ struct usb_device *dev = interface_to_usbdev(intf);
+ product_name = dev->product;
+@@ -2454,6 +2454,9 @@ static void wacom_wireless_work(struct work_struct *work)
+
+ wacom_destroy_battery(wacom);
+
++ if (!usbdev)
++ return;
++
+ /* Stylus interface */
+ hdev1 = usb_get_intfdata(usbdev->config->interface[1]);
+ wacom1 = hid_get_drvdata(hdev1);
+@@ -2733,8 +2736,6 @@ static void wacom_mode_change_work(struct work_struct *work)
+ static int wacom_probe(struct hid_device *hdev,
+ const struct hid_device_id *id)
+ {
+- struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+- struct usb_device *dev = interface_to_usbdev(intf);
+ struct wacom *wacom;
+ struct wacom_wac *wacom_wac;
+ struct wacom_features *features;
+@@ -2769,8 +2770,14 @@ static int wacom_probe(struct hid_device *hdev,
+ wacom_wac->hid_data.inputmode = -1;
+ wacom_wac->mode_report = -1;
+
+- wacom->usbdev = dev;
+- wacom->intf = intf;
++ if (hid_is_usb(hdev)) {
++ struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
++ struct usb_device *dev = interface_to_usbdev(intf);
++
++ wacom->usbdev = dev;
++ wacom->intf = intf;
++ }
++
+ mutex_init(&wacom->lock);
+ INIT_DELAYED_WORK(&wacom->init_work, wacom_init_work);
+ INIT_WORK(&wacom->wireless_work, wacom_wireless_work);
+diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
+index 33a6908995b1b..a7176fc0635dd 100644
+--- a/drivers/hid/wacom_wac.c
++++ b/drivers/hid/wacom_wac.c
+@@ -2588,6 +2588,24 @@ static void wacom_wac_finger_slot(struct wacom_wac *wacom_wac,
+ }
+ }
+
++static bool wacom_wac_slot_is_active(struct input_dev *dev, int key)
++{
++ struct input_mt *mt = dev->mt;
++ struct input_mt_slot *s;
++
++ if (!mt)
++ return false;
++
++ for (s = mt->slots; s != mt->slots + mt->num_slots; s++) {
++ if (s->key == key &&
++ input_mt_get_value(s, ABS_MT_TRACKING_ID) >= 0) {
++ return true;
++ }
++ }
++
++ return false;
++}
++
+ static void wacom_wac_finger_event(struct hid_device *hdev,
+ struct hid_field *field, struct hid_usage *usage, __s32 value)
+ {
+@@ -2603,6 +2621,9 @@ static void wacom_wac_finger_event(struct hid_device *hdev,
+ return;
+
+ switch (equivalent_usage) {
++ case HID_DG_CONFIDENCE:
++ wacom_wac->hid_data.confidence = value;
++ break;
+ case HID_GD_X:
+ wacom_wac->hid_data.x = value;
+ break;
+@@ -2635,8 +2656,14 @@ static void wacom_wac_finger_event(struct hid_device *hdev,
+ }
+
+ if (usage->usage_index + 1 == field->report_count) {
+- if (equivalent_usage == wacom_wac->hid_data.last_slot_field)
+- wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input);
++ if (equivalent_usage == wacom_wac->hid_data.last_slot_field) {
++ bool touch_removed = wacom_wac_slot_is_active(wacom_wac->touch_input,
++ wacom_wac->hid_data.id) && !wacom_wac->hid_data.tipswitch;
++
++ if (wacom_wac->hid_data.confidence || touch_removed) {
++ wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input);
++ }
++ }
+ }
+ }
+
+@@ -2653,6 +2680,12 @@ static void wacom_wac_finger_pre_report(struct hid_device *hdev,
+
+ wacom_wac->is_invalid_bt_frame = false;
+
++ hid_data->confidence = true;
++
++ hid_data->cc_report = 0;
++ hid_data->cc_index = -1;
++ hid_data->cc_value_index = -1;
++
+ for (i = 0; i < report->maxfield; i++) {
+ struct hid_field *field = report->field[i];
+ int j;
+@@ -2686,11 +2719,14 @@ static void wacom_wac_finger_pre_report(struct hid_device *hdev,
+ hid_data->cc_index >= 0) {
+ struct hid_field *field = report->field[hid_data->cc_index];
+ int value = field->value[hid_data->cc_value_index];
+- if (value)
++ if (value) {
+ hid_data->num_expected = value;
++ hid_data->num_received = 0;
++ }
+ }
+ else {
+ hid_data->num_expected = wacom_wac->features.touch_max;
++ hid_data->num_received = 0;
+ }
+ }
+
+@@ -2718,6 +2754,7 @@ static void wacom_wac_finger_report(struct hid_device *hdev,
+
+ input_sync(input);
+ wacom_wac->hid_data.num_received = 0;
++ wacom_wac->hid_data.num_expected = 0;
+
+ /* keep touch state for pen event */
+ wacom_wac->shared->touch_down = wacom_wac_finger_count_touches(wacom_wac);
+diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
+index 8b2d4e5b2303c..466b62cc16dc1 100644
+--- a/drivers/hid/wacom_wac.h
++++ b/drivers/hid/wacom_wac.h
+@@ -301,6 +301,7 @@ struct hid_data {
+ bool barrelswitch;
+ bool barrelswitch2;
+ bool serialhi;
++ bool confidence;
+ int x;
+ int y;
+ int pressure;
+diff --git a/drivers/hsi/hsi_core.c b/drivers/hsi/hsi_core.c
+index ec90713564e32..884066109699c 100644
+--- a/drivers/hsi/hsi_core.c
++++ b/drivers/hsi/hsi_core.c
+@@ -102,6 +102,7 @@ struct hsi_client *hsi_new_client(struct hsi_port *port,
+ if (device_register(&cl->device) < 0) {
+ pr_err("hsi: failed to register client: %s\n", info->name);
+ put_device(&cl->device);
++ goto err;
+ }
+
+ return cl;
+diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
+index d1123ceb38f3f..9a074cbdef78c 100644
+--- a/drivers/hv/Kconfig
++++ b/drivers/hv/Kconfig
+@@ -18,6 +18,7 @@ config HYPERV_TIMER
+ config HYPERV_UTILS
+ tristate "Microsoft Hyper-V Utilities driver"
+ depends on HYPERV && CONNECTOR && NLS
++ depends on PTP_1588_CLOCK_OPTIONAL
+ help
+ Select this option to enable the Hyper-V Utilities.
+
+diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
+index 142308526ec6a..ce76fc3827998 100644
+--- a/drivers/hv/channel_mgmt.c
++++ b/drivers/hv/channel_mgmt.c
+@@ -380,7 +380,7 @@ void vmbus_channel_map_relid(struct vmbus_channel *channel)
+ * execute:
+ *
+ * (a) In the "normal (i.e., not resuming from hibernation)" path,
+- * the full barrier in smp_store_mb() guarantees that the store
++ * the full barrier in virt_store_mb() guarantees that the store
+ * is propagated to all CPUs before the add_channel_work work
+ * is queued. In turn, add_channel_work is queued before the
+ * channel's ring buffer is allocated/initialized and the
+@@ -392,14 +392,14 @@ void vmbus_channel_map_relid(struct vmbus_channel *channel)
+ * recv_int_page before retrieving the channel pointer from the
+ * array of channels.
+ *
+- * (b) In the "resuming from hibernation" path, the smp_store_mb()
++ * (b) In the "resuming from hibernation" path, the virt_store_mb()
+ * guarantees that the store is propagated to all CPUs before
+ * the VMBus connection is marked as ready for the resume event
+ * (cf. check_ready_for_resume_event()). The interrupt handler
+ * of the VMBus driver and vmbus_chan_sched() can not run before
+ * vmbus_bus_resume() has completed execution (cf. resume_noirq).
+ */
+- smp_store_mb(
++ virt_store_mb(
+ vmbus_connection.channels[channel->offermsg.child_relid],
+ channel);
+ }
+diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
+index 7f11ea07d698f..3cf334c46c312 100644
+--- a/drivers/hv/hv_balloon.c
++++ b/drivers/hv/hv_balloon.c
+@@ -480,7 +480,7 @@ module_param(pressure_report_delay, uint, (S_IRUGO | S_IWUSR));
+ MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure");
+ static atomic_t trans_id = ATOMIC_INIT(0);
+
+-static int dm_ring_size = 20 * 1024;
++static int dm_ring_size = VMBUS_RING_SIZE(16 * 1024);
+
+ /*
+ * Driver specific state.
+@@ -1563,7 +1563,7 @@ static void balloon_onchannelcallback(void *context)
+ break;
+
+ default:
+- pr_warn("Unhandled message: type: %d\n", dm_hdr->type);
++ pr_warn_ratelimited("Unhandled message: type: %d\n", dm_hdr->type);
+
+ }
+ }
+@@ -1653,6 +1653,38 @@ static void disable_page_reporting(void)
+ }
+ }
+
++static int ballooning_enabled(void)
++{
++ /*
++ * Disable ballooning if the page size is not 4k (HV_HYP_PAGE_SIZE),
++ * since currently it's unclear to us whether an unballoon request can
++ * make sure all page ranges are guest page size aligned.
++ */
++ if (PAGE_SIZE != HV_HYP_PAGE_SIZE) {
++ pr_info("Ballooning disabled because page size is not 4096 bytes\n");
++ return 0;
++ }
++
++ return 1;
++}
++
++static int hot_add_enabled(void)
++{
++ /*
++ * Disable hot add on ARM64, because we currently rely on
++ * memory_add_physaddr_to_nid() to get a node id of a hot add range,
++ * however ARM64's memory_add_physaddr_to_nid() always return 0 and
++ * DM_MEM_HOT_ADD_REQUEST doesn't have the NUMA node information for
++ * add_memory().
++ */
++ if (IS_ENABLED(CONFIG_ARM64)) {
++ pr_info("Memory hot add disabled on ARM64\n");
++ return 0;
++ }
++
++ return 1;
++}
++
+ static int balloon_connect_vsp(struct hv_device *dev)
+ {
+ struct dm_version_request version_req;
+@@ -1660,6 +1692,13 @@ static int balloon_connect_vsp(struct hv_device *dev)
+ unsigned long t;
+ int ret;
+
++ /*
++ * max_pkt_size should be large enough for one vmbus packet header plus
++ * our receive buffer size. Hyper-V sends messages up to
++ * HV_HYP_PAGE_SIZE bytes long on balloon channel.
++ */
++ dev->channel->max_pkt_size = HV_HYP_PAGE_SIZE * 2;
++
+ ret = vmbus_open(dev->channel, dm_ring_size, dm_ring_size, NULL, 0,
+ balloon_onchannelcallback, dev);
+ if (ret)
+@@ -1717,8 +1756,8 @@ static int balloon_connect_vsp(struct hv_device *dev)
+ * currently still requires the bits to be set, so we have to add code
+ * to fail the host's hot-add and balloon up/down requests, if any.
+ */
+- cap_msg.caps.cap_bits.balloon = 1;
+- cap_msg.caps.cap_bits.hot_add = 1;
++ cap_msg.caps.cap_bits.balloon = ballooning_enabled();
++ cap_msg.caps.cap_bits.hot_add = hot_add_enabled();
+
+ /*
+ * Specify our alignment requirements as it relates
+diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
+index 314015d9e912d..f4091143213b0 100644
+--- a/drivers/hv/ring_buffer.c
++++ b/drivers/hv/ring_buffer.c
+@@ -408,7 +408,16 @@ int hv_ringbuffer_read(struct vmbus_channel *channel,
+ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi)
+ {
+ u32 priv_read_loc = rbi->priv_read_index;
+- u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index);
++ u32 write_loc;
++
++ /*
++ * The Hyper-V host writes the packet data, then uses
++ * store_release() to update the write_index. Use load_acquire()
++ * here to prevent loads of the packet data from being re-ordered
++ * before the read of the write_index and potentially getting
++ * stale data.
++ */
++ write_loc = virt_load_acquire(&rbi->ring_buffer->write_index);
+
+ if (write_loc >= priv_read_loc)
+ return write_loc - priv_read_loc;
+diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
+index 392c1ac4f8193..50d9113f54025 100644
+--- a/drivers/hv/vmbus_drv.c
++++ b/drivers/hv/vmbus_drv.c
+@@ -76,8 +76,8 @@ static int hyperv_panic_event(struct notifier_block *nb, unsigned long val,
+
+ /*
+ * Hyper-V should be notified only once about a panic. If we will be
+- * doing hyperv_report_panic_msg() later with kmsg data, don't do
+- * the notification here.
++ * doing hv_kmsg_dump() with kmsg data later, don't do the notification
++ * here.
+ */
+ if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE
+ && hyperv_report_reg()) {
+@@ -99,8 +99,8 @@ static int hyperv_die_event(struct notifier_block *nb, unsigned long val,
+
+ /*
+ * Hyper-V should be notified only once about a panic. If we will be
+- * doing hyperv_report_panic_msg() later with kmsg data, don't do
+- * the notification here.
++ * doing hv_kmsg_dump() with kmsg data later, don't do the notification
++ * here.
+ */
+ if (hyperv_report_reg())
+ hyperv_report_panic(regs, val, true);
+@@ -1381,7 +1381,7 @@ static void vmbus_isr(void)
+ tasklet_schedule(&hv_cpu->msg_dpc);
+ }
+
+- add_interrupt_randomness(vmbus_interrupt, 0);
++ add_interrupt_randomness(vmbus_interrupt);
+ }
+
+ static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id)
+@@ -1545,14 +1545,20 @@ static int vmbus_bus_init(void)
+ if (ret)
+ goto err_connect;
+
++ if (hv_is_isolation_supported())
++ sysctl_record_panic_msg = 0;
++
+ /*
+ * Only register if the crash MSRs are available
+ */
+ if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) {
+ u64 hyperv_crash_ctl;
+ /*
+- * Sysctl registration is not fatal, since by default
+- * reporting is enabled.
++ * Panic message recording (sysctl_record_panic_msg)
++ * is enabled by default in non-isolated guests and
++ * disabled by default in isolated guests; the panic
++ * message recording won't be available in isolated
++ * guests should the following registration fail.
+ */
+ hv_ctl_table_hdr = register_sysctl_table(hv_root_table);
+ if (!hv_ctl_table_hdr)
+@@ -2027,8 +2033,10 @@ int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel)
+ kobj->kset = dev->channels_kset;
+ ret = kobject_init_and_add(kobj, &vmbus_chan_ktype, NULL,
+ "%u", relid);
+- if (ret)
++ if (ret) {
++ kobject_put(kobj);
+ return ret;
++ }
+
+ ret = sysfs_create_group(kobj, &vmbus_chan_group);
+
+@@ -2037,6 +2045,7 @@ int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel)
+ * The calling functions' error handling paths will cleanup the
+ * empty channel directory.
+ */
++ kobject_put(kobj);
+ dev_err(device, "Unable to set up channel sysfs files\n");
+ return ret;
+ }
+@@ -2773,10 +2782,15 @@ static void __exit vmbus_exit(void)
+ if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) {
+ kmsg_dump_unregister(&hv_kmsg_dumper);
+ unregister_die_notifier(&hyperv_die_block);
+- atomic_notifier_chain_unregister(&panic_notifier_list,
+- &hyperv_panic_block);
+ }
+
++ /*
++ * The panic notifier is always registered, hence we should
++ * also unconditionally unregister it here as well.
++ */
++ atomic_notifier_chain_unregister(&panic_notifier_list,
++ &hyperv_panic_block);
++
+ free_page((unsigned long)hv_panic_page);
+ unregister_sysctl_table(hv_ctl_table_hdr);
+ hv_ctl_table_hdr = NULL;
+diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
+index c4578e8f34bb5..51f1caa10d113 100644
+--- a/drivers/hwmon/Kconfig
++++ b/drivers/hwmon/Kconfig
+@@ -944,7 +944,7 @@ config SENSORS_LTC4261
+
+ config SENSORS_LTQ_CPUTEMP
+ bool "Lantiq cpu temperature sensor driver"
+- depends on LANTIQ
++ depends on SOC_XWAY
+ help
+ If you say yes here you get support for the temperature
+ sensor inside your CPU.
+@@ -1317,7 +1317,7 @@ config SENSORS_LM90
+ Maxim MAX6646, MAX6647, MAX6648, MAX6649, MAX6654, MAX6657, MAX6658,
+ MAX6659, MAX6680, MAX6681, MAX6692, MAX6695, MAX6696,
+ ON Semiconductor NCT1008, Winbond/Nuvoton W83L771W/G/AWG/ASG,
+- Philips SA56004, GMT G781, and Texas Instruments TMP451
++ Philips SA56004, GMT G781, Texas Instruments TMP451 and TMP461
+ sensor chips.
+
+ This driver can also be built as a module. If so, the module
+diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
+index d519aca4a9d64..c67cd037a93fd 100644
+--- a/drivers/hwmon/adt7470.c
++++ b/drivers/hwmon/adt7470.c
+@@ -19,6 +19,7 @@
+ #include <linux/log2.h>
+ #include <linux/kthread.h>
+ #include <linux/regmap.h>
++#include <linux/sched.h>
+ #include <linux/slab.h>
+ #include <linux/util_macros.h>
+
+@@ -294,11 +295,10 @@ static int adt7470_update_thread(void *p)
+ adt7470_read_temperatures(data);
+ mutex_unlock(&data->lock);
+
+- set_current_state(TASK_INTERRUPTIBLE);
+ if (kthread_should_stop())
+ break;
+
+- schedule_timeout(msecs_to_jiffies(data->auto_update_interval));
++ schedule_timeout_interruptible(msecs_to_jiffies(data->auto_update_interval));
+ }
+
+ return 0;
+@@ -662,6 +662,9 @@ static int adt7470_fan_write(struct device *dev, u32 attr, int channel, long val
+ struct adt7470_data *data = dev_get_drvdata(dev);
+ int err;
+
++ if (val <= 0)
++ return -EINVAL;
++
+ val = FAN_RPM_TO_PERIOD(val);
+ val = clamp_val(val, 1, 65534);
+
+diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
+index 731d5117f9f10..14389fd7afb89 100644
+--- a/drivers/hwmon/corsair-psu.c
++++ b/drivers/hwmon/corsair-psu.c
+@@ -729,7 +729,7 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id
+ corsairpsu_check_cmd_support(priv);
+
+ priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "corsairpsu", priv,
+- &corsairpsu_chip_info, 0);
++ &corsairpsu_chip_info, NULL);
+
+ if (IS_ERR(priv->hwmon_dev)) {
+ ret = PTR_ERR(priv->hwmon_dev);
+diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
+index 774c1b0715d91..9cb1c3588038f 100644
+--- a/drivers/hwmon/dell-smm-hwmon.c
++++ b/drivers/hwmon/dell-smm-hwmon.c
+@@ -326,7 +326,7 @@ static int i8k_enable_fan_auto_mode(const struct dell_smm_data *data, bool enabl
+ }
+
+ /*
+- * Set the fan speed (off, low, high). Returns the new fan status.
++ * Set the fan speed (off, low, high, ...).
+ */
+ static int i8k_set_fan(const struct dell_smm_data *data, int fan, int speed)
+ {
+@@ -338,7 +338,7 @@ static int i8k_set_fan(const struct dell_smm_data *data, int fan, int speed)
+ speed = (speed < 0) ? 0 : ((speed > data->i8k_fan_max) ? data->i8k_fan_max : speed);
+ regs.ebx = (fan & 0xff) | (speed << 8);
+
+- return i8k_smm(&regs) ? : i8k_get_fan_status(data, fan);
++ return i8k_smm(&regs);
+ }
+
+ static int __init i8k_get_temp_type(int sensor)
+@@ -452,7 +452,7 @@ static int
+ i8k_ioctl_unlocked(struct file *fp, struct dell_smm_data *data, unsigned int cmd, unsigned long arg)
+ {
+ int val = 0;
+- int speed;
++ int speed, err;
+ unsigned char buff[16];
+ int __user *argp = (int __user *)arg;
+
+@@ -513,7 +513,11 @@ i8k_ioctl_unlocked(struct file *fp, struct dell_smm_data *data, unsigned int cmd
+ if (copy_from_user(&speed, argp + 1, sizeof(int)))
+ return -EFAULT;
+
+- val = i8k_set_fan(data, val, speed);
++ err = i8k_set_fan(data, val, speed);
++ if (err < 0)
++ return err;
++
++ val = i8k_get_fan_status(data, val);
+ break;
+
+ default:
+@@ -623,10 +627,9 @@ static void __init i8k_init_procfs(struct device *dev)
+ {
+ struct dell_smm_data *data = dev_get_drvdata(dev);
+
+- /* Register the proc entry */
+- proc_create_data("i8k", 0, NULL, &i8k_proc_ops, data);
+-
+- devm_add_action_or_reset(dev, i8k_exit_procfs, NULL);
++ /* Only register exit function if creation was successful */
++ if (proc_create_data("i8k", 0, NULL, &i8k_proc_ops, data))
++ devm_add_action_or_reset(dev, i8k_exit_procfs, NULL);
+ }
+
+ #else
+diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c
+index 4dec793fd07d5..94b35723ee7ad 100644
+--- a/drivers/hwmon/f71882fg.c
++++ b/drivers/hwmon/f71882fg.c
+@@ -1577,8 +1577,9 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *devattr,
+ temp *= 125;
+ if (sign)
+ temp -= 128000;
+- } else
+- temp = data->temp[nr] * 1000;
++ } else {
++ temp = ((s8)data->temp[nr]) * 1000;
++ }
+
+ return sprintf(buf, "%d\n", temp);
+ }
+diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
+index 8d3b1dae31df1..3ae961986fc31 100644
+--- a/drivers/hwmon/hwmon.c
++++ b/drivers/hwmon/hwmon.c
+@@ -214,12 +214,14 @@ static int hwmon_thermal_add_sensor(struct device *dev, int index)
+
+ tzd = devm_thermal_zone_of_sensor_register(dev, index, tdata,
+ &hwmon_thermal_ops);
+- /*
+- * If CONFIG_THERMAL_OF is disabled, this returns -ENODEV,
+- * so ignore that error but forward any other error.
+- */
+- if (IS_ERR(tzd) && (PTR_ERR(tzd) != -ENODEV))
+- return PTR_ERR(tzd);
++ if (IS_ERR(tzd)) {
++ if (PTR_ERR(tzd) != -ENODEV)
++ return PTR_ERR(tzd);
++ dev_info(dev, "temp%d_input not attached to any thermal zone\n",
++ index + 1);
++ devm_kfree(dev, tdata);
++ return 0;
++ }
+
+ err = devm_add_action(dev, hwmon_thermal_remove_sensor, &tdata->node);
+ if (err)
+@@ -796,8 +798,10 @@ __hwmon_device_register(struct device *dev, const char *name, void *drvdata,
+ dev_set_drvdata(hdev, drvdata);
+ dev_set_name(hdev, HWMON_ID_FORMAT, id);
+ err = device_register(hdev);
+- if (err)
+- goto free_hwmon;
++ if (err) {
++ put_device(hdev);
++ goto ida_remove;
++ }
+
+ INIT_LIST_HEAD(&hwdev->tzdata);
+
+diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
+index 567b7c521f388..1c9493c708132 100644
+--- a/drivers/hwmon/lm90.c
++++ b/drivers/hwmon/lm90.c
+@@ -35,13 +35,14 @@
+ * explicitly as max6659, or if its address is not 0x4c.
+ * These chips lack the remote temperature offset feature.
+ *
+- * This driver also supports the MAX6654 chip made by Maxim. This chip can
+- * be at 9 different addresses, similar to MAX6680/MAX6681. The MAX6654 is
+- * otherwise similar to MAX6657/MAX6658/MAX6659. Extended range is available
+- * by setting the configuration register accordingly, and is done during
+- * initialization. Extended precision is only available at conversion rates
+- * of 1 Hz and slower. Note that extended precision is not enabled by
+- * default, as this driver initializes all chips to 2 Hz by design.
++ * This driver also supports the MAX6654 chip made by Maxim. This chip can be
++ * at 9 different addresses, similar to MAX6680/MAX6681. The MAX6654 is similar
++ * to MAX6657/MAX6658/MAX6659, but does not support critical temperature
++ * limits. Extended range is available by setting the configuration register
++ * accordingly, and is done during initialization. Extended precision is only
++ * available at conversion rates of 1 Hz and slower. Note that extended
++ * precision is not enabled by default, as this driver initializes all chips
++ * to 2 Hz by design.
+ *
+ * This driver also supports the MAX6646, MAX6647, MAX6648, MAX6649 and
+ * MAX6692 chips made by Maxim. These are again similar to the LM86,
+@@ -69,10 +70,10 @@
+ * This driver also supports the G781 from GMT. This device is compatible
+ * with the ADM1032.
+ *
+- * This driver also supports TMP451 from Texas Instruments. This device is
+- * supported in both compatibility and extended mode. It's mostly compatible
+- * with ADT7461 except for local temperature low byte register and max
+- * conversion rate.
++ * This driver also supports TMP451 and TMP461 from Texas Instruments.
++ * Those devices are supported in both compatibility and extended mode.
++ * They are mostly compatible with ADT7461 except for local temperature
++ * low byte register and max conversion rate.
+ *
+ * Since the LM90 was the first chipset supported by this driver, most
+ * comments will refer to this chipset, but are actually general and
+@@ -112,7 +113,7 @@ static const unsigned short normal_i2c[] = {
+ 0x4d, 0x4e, 0x4f, I2C_CLIENT_END };
+
+ enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680,
+- max6646, w83l771, max6696, sa56004, g781, tmp451, max6654 };
++ max6646, w83l771, max6696, sa56004, g781, tmp451, tmp461, max6654 };
+
+ /*
+ * The LM90 registers
+@@ -168,8 +169,12 @@ enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680,
+
+ #define LM90_MAX_CONVRATE_MS 16000 /* Maximum conversion rate in ms */
+
+-/* TMP451 registers */
++/* TMP451/TMP461 registers */
+ #define TMP451_REG_R_LOCAL_TEMPL 0x15
++#define TMP451_REG_CONALERT 0x22
++
++#define TMP461_REG_CHEN 0x16
++#define TMP461_REG_DFC 0x24
+
+ /*
+ * Device flags
+@@ -182,7 +187,10 @@ enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680,
+ #define LM90_HAVE_EMERGENCY_ALARM (1 << 5)/* emergency alarm */
+ #define LM90_HAVE_TEMP3 (1 << 6) /* 3rd temperature sensor */
+ #define LM90_HAVE_BROKEN_ALERT (1 << 7) /* Broken alert */
+-#define LM90_PAUSE_FOR_CONFIG (1 << 8) /* Pause conversion for config */
++#define LM90_HAVE_EXTENDED_TEMP (1 << 8) /* extended temperature support*/
++#define LM90_PAUSE_FOR_CONFIG (1 << 9) /* Pause conversion for config */
++#define LM90_HAVE_CRIT (1 << 10)/* Chip supports CRIT/OVERT register */
++#define LM90_HAVE_CRIT_ALRM_SWP (1 << 11)/* critical alarm bits swapped */
+
+ /* LM90 status */
+ #define LM90_STATUS_LTHRM (1 << 0) /* local THERM limit tripped */
+@@ -192,6 +200,7 @@ enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680,
+ #define LM90_STATUS_RHIGH (1 << 4) /* remote high temp limit tripped */
+ #define LM90_STATUS_LLOW (1 << 5) /* local low temp limit tripped */
+ #define LM90_STATUS_LHIGH (1 << 6) /* local high temp limit tripped */
++#define LM90_STATUS_BUSY (1 << 7) /* conversion is ongoing */
+
+ #define MAX6696_STATUS2_R2THRM (1 << 1) /* remote2 THERM limit tripped */
+ #define MAX6696_STATUS2_R2OPEN (1 << 2) /* remote2 is an open circuit */
+@@ -229,6 +238,7 @@ static const struct i2c_device_id lm90_id[] = {
+ { "w83l771", w83l771 },
+ { "sa56004", sa56004 },
+ { "tmp451", tmp451 },
++ { "tmp461", tmp461 },
+ { }
+ };
+ MODULE_DEVICE_TABLE(i2c, lm90_id);
+@@ -326,6 +336,10 @@ static const struct of_device_id __maybe_unused lm90_of_match[] = {
+ .compatible = "ti,tmp451",
+ .data = (void *)tmp451
+ },
++ {
++ .compatible = "ti,tmp461",
++ .data = (void *)tmp461
++ },
+ { },
+ };
+ MODULE_DEVICE_TABLE(of, lm90_of_match);
+@@ -344,85 +358,99 @@ struct lm90_params {
+ static const struct lm90_params lm90_params[] = {
+ [adm1032] = {
+ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
+- | LM90_HAVE_BROKEN_ALERT,
++ | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 10,
+ },
+ [adt7461] = {
+ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
+- | LM90_HAVE_BROKEN_ALERT,
++ | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP
++ | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 10,
+ },
+ [g781] = {
+ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
+- | LM90_HAVE_BROKEN_ALERT,
++ | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+- .max_convrate = 8,
++ .max_convrate = 7,
+ },
+ [lm86] = {
+- .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
++ | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7b,
+ .max_convrate = 9,
+ },
+ [lm90] = {
+- .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
++ | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7b,
+ .max_convrate = 9,
+ },
+ [lm99] = {
+- .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
++ | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7b,
+ .max_convrate = 9,
+ },
+ [max6646] = {
++ .flags = LM90_HAVE_CRIT | LM90_HAVE_BROKEN_ALERT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 6,
+ .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL,
+ },
+ [max6654] = {
++ .flags = LM90_HAVE_BROKEN_ALERT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 7,
+ .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL,
+ },
+ [max6657] = {
+- .flags = LM90_PAUSE_FOR_CONFIG,
++ .flags = LM90_PAUSE_FOR_CONFIG | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 8,
+ .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL,
+ },
+ [max6659] = {
+- .flags = LM90_HAVE_EMERGENCY,
++ .flags = LM90_HAVE_EMERGENCY | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 8,
+ .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL,
+ },
+ [max6680] = {
+- .flags = LM90_HAVE_OFFSET,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_CRIT
++ | LM90_HAVE_CRIT_ALRM_SWP | LM90_HAVE_BROKEN_ALERT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 7,
+ },
+ [max6696] = {
+ .flags = LM90_HAVE_EMERGENCY
+- | LM90_HAVE_EMERGENCY_ALARM | LM90_HAVE_TEMP3,
++ | LM90_HAVE_EMERGENCY_ALARM | LM90_HAVE_TEMP3 | LM90_HAVE_CRIT,
+ .alert_alarms = 0x1c7c,
+ .max_convrate = 6,
+ .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL,
+ },
+ [w83l771] = {
+- .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 8,
+ },
+ [sa56004] = {
+- .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT,
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7b,
+ .max_convrate = 9,
+ .reg_local_ext = SA56004_REG_R_LOCAL_TEMPL,
+ },
+ [tmp451] = {
+ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
+- | LM90_HAVE_BROKEN_ALERT,
++ | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT,
++ .alert_alarms = 0x7c,
++ .max_convrate = 9,
++ .reg_local_ext = TMP451_REG_R_LOCAL_TEMPL,
++ },
++ [tmp461] = {
++ .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT
++ | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT,
+ .alert_alarms = 0x7c,
+ .max_convrate = 9,
+ .reg_local_ext = TMP451_REG_R_LOCAL_TEMPL,
+@@ -651,20 +679,22 @@ static int lm90_update_limits(struct device *dev)
+ struct i2c_client *client = data->client;
+ int val;
+
+- val = lm90_read_reg(client, LM90_REG_R_LOCAL_CRIT);
+- if (val < 0)
+- return val;
+- data->temp8[LOCAL_CRIT] = val;
++ if (data->flags & LM90_HAVE_CRIT) {
++ val = lm90_read_reg(client, LM90_REG_R_LOCAL_CRIT);
++ if (val < 0)
++ return val;
++ data->temp8[LOCAL_CRIT] = val;
+
+- val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT);
+- if (val < 0)
+- return val;
+- data->temp8[REMOTE_CRIT] = val;
++ val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT);
++ if (val < 0)
++ return val;
++ data->temp8[REMOTE_CRIT] = val;
+
+- val = lm90_read_reg(client, LM90_REG_R_TCRIT_HYST);
+- if (val < 0)
+- return val;
+- data->temp_hyst = val;
++ val = lm90_read_reg(client, LM90_REG_R_TCRIT_HYST);
++ if (val < 0)
++ return val;
++ data->temp_hyst = val;
++ }
+
+ val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH);
+ if (val < 0)
+@@ -792,7 +822,7 @@ static int lm90_update_device(struct device *dev)
+ val = lm90_read_reg(client, LM90_REG_R_STATUS);
+ if (val < 0)
+ return val;
+- data->alarms = val; /* lower 8 bit of alarms */
++ data->alarms = val & ~LM90_STATUS_BUSY;
+
+ if (data->kind == max6696) {
+ val = lm90_select_remote_channel(data, 1);
+@@ -819,7 +849,7 @@ static int lm90_update_device(struct device *dev)
+ * Re-enable ALERT# output if it was originally enabled and
+ * relevant alarms are all clear
+ */
+- if (!(data->config_orig & 0x80) &&
++ if ((client->irq || !(data->config_orig & 0x80)) &&
+ !(data->alarms & data->alert_alarms)) {
+ if (data->config & 0x80) {
+ dev_dbg(&client->dev, "Re-enabling ALERT#\n");
+@@ -998,7 +1028,7 @@ static int lm90_get_temp11(struct lm90_data *data, int index)
+ s16 temp11 = data->temp11[index];
+ int temp;
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ temp = temp_from_u16_adt7461(data, temp11);
+ else if (data->kind == max6646)
+ temp = temp_from_u16(temp11);
+@@ -1035,7 +1065,7 @@ static int lm90_set_temp11(struct lm90_data *data, int index, long val)
+ val -= 16000;
+ }
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ data->temp11[index] = temp_to_u16_adt7461(data, val);
+ else if (data->kind == max6646)
+ data->temp11[index] = temp_to_u8(val) << 8;
+@@ -1062,7 +1092,7 @@ static int lm90_get_temp8(struct lm90_data *data, int index)
+ s8 temp8 = data->temp8[index];
+ int temp;
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ temp = temp_from_u8_adt7461(data, temp8);
+ else if (data->kind == max6646)
+ temp = temp_from_u8(temp8);
+@@ -1098,7 +1128,7 @@ static int lm90_set_temp8(struct lm90_data *data, int index, long val)
+ val -= 16000;
+ }
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ data->temp8[index] = temp_to_u8_adt7461(data, val);
+ else if (data->kind == max6646)
+ data->temp8[index] = temp_to_u8(val);
+@@ -1116,7 +1146,7 @@ static int lm90_get_temphyst(struct lm90_data *data, int index)
+ {
+ int temp;
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ temp = temp_from_u8_adt7461(data, data->temp8[index]);
+ else if (data->kind == max6646)
+ temp = temp_from_u8(data->temp8[index]);
+@@ -1136,15 +1166,15 @@ static int lm90_set_temphyst(struct lm90_data *data, long val)
+ int temp;
+ int err;
+
+- if (data->kind == adt7461 || data->kind == tmp451)
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP)
+ temp = temp_from_u8_adt7461(data, data->temp8[LOCAL_CRIT]);
+ else if (data->kind == max6646)
+ temp = temp_from_u8(data->temp8[LOCAL_CRIT]);
+ else
+ temp = temp_from_s8(data->temp8[LOCAL_CRIT]);
+
+- /* prevent integer underflow */
+- val = max(val, -128000l);
++ /* prevent integer overflow/underflow */
++ val = clamp_val(val, -128000l, 255000l);
+
+ data->temp_hyst = hyst_to_reg(temp - val);
+ err = i2c_smbus_write_byte_data(client, LM90_REG_W_TCRIT_HYST,
+@@ -1175,6 +1205,7 @@ static const u8 lm90_temp_emerg_index[3] = {
+ static const u8 lm90_min_alarm_bits[3] = { 5, 3, 11 };
+ static const u8 lm90_max_alarm_bits[3] = { 6, 4, 12 };
+ static const u8 lm90_crit_alarm_bits[3] = { 0, 1, 9 };
++static const u8 lm90_crit_alarm_bits_swapped[3] = { 1, 0, 9 };
+ static const u8 lm90_emergency_alarm_bits[3] = { 15, 13, 14 };
+ static const u8 lm90_fault_bits[3] = { 0, 2, 10 };
+
+@@ -1200,7 +1231,10 @@ static int lm90_temp_read(struct device *dev, u32 attr, int channel, long *val)
+ *val = (data->alarms >> lm90_max_alarm_bits[channel]) & 1;
+ break;
+ case hwmon_temp_crit_alarm:
+- *val = (data->alarms >> lm90_crit_alarm_bits[channel]) & 1;
++ if (data->flags & LM90_HAVE_CRIT_ALRM_SWP)
++ *val = (data->alarms >> lm90_crit_alarm_bits_swapped[channel]) & 1;
++ else
++ *val = (data->alarms >> lm90_crit_alarm_bits[channel]) & 1;
+ break;
+ case hwmon_temp_emergency_alarm:
+ *val = (data->alarms >> lm90_emergency_alarm_bits[channel]) & 1;
+@@ -1448,12 +1482,11 @@ static int lm90_detect(struct i2c_client *client,
+ if (man_id < 0 || chip_id < 0 || config1 < 0 || convrate < 0)
+ return -ENODEV;
+
+- if (man_id == 0x01 || man_id == 0x5C || man_id == 0x41) {
++ if (man_id == 0x01 || man_id == 0x5C || man_id == 0xA1) {
+ config2 = i2c_smbus_read_byte_data(client, LM90_REG_R_CONFIG2);
+ if (config2 < 0)
+ return -ENODEV;
+- } else
+- config2 = 0; /* Make compiler happy */
++ }
+
+ if ((address == 0x4C || address == 0x4D)
+ && man_id == 0x01) { /* National Semiconductor */
+@@ -1627,18 +1660,26 @@ static int lm90_detect(struct i2c_client *client,
+ && convrate <= 0x08)
+ name = "g781";
+ } else
+- if (address == 0x4C
+- && man_id == 0x55) { /* Texas Instruments */
+- int local_ext;
++ if (man_id == 0x55 && chip_id == 0x00 &&
++ (config1 & 0x1B) == 0x00 && convrate <= 0x09) {
++ int local_ext, conalert, chen, dfc;
+
+ local_ext = i2c_smbus_read_byte_data(client,
+ TMP451_REG_R_LOCAL_TEMPL);
+-
+- if (chip_id == 0x00 /* TMP451 */
+- && (config1 & 0x1B) == 0x00
+- && convrate <= 0x09
+- && (local_ext & 0x0F) == 0x00)
+- name = "tmp451";
++ conalert = i2c_smbus_read_byte_data(client,
++ TMP451_REG_CONALERT);
++ chen = i2c_smbus_read_byte_data(client, TMP461_REG_CHEN);
++ dfc = i2c_smbus_read_byte_data(client, TMP461_REG_DFC);
++
++ if ((local_ext & 0x0F) == 0x00 &&
++ (conalert & 0xf1) == 0x01 &&
++ (chen & 0xfc) == 0x00 &&
++ (dfc & 0xfc) == 0x00) {
++ if (address == 0x4c && !(chen & 0x03))
++ name = "tmp451";
++ else if (address >= 0x48 && address <= 0x4f)
++ name = "tmp461";
++ }
+ }
+
+ if (!name) { /* identification failed */
+@@ -1685,7 +1726,7 @@ static int lm90_init_client(struct i2c_client *client, struct lm90_data *data)
+ lm90_set_convrate(client, data, 500); /* 500ms; 2Hz conversion rate */
+
+ /* Check Temperature Range Select */
+- if (data->kind == adt7461 || data->kind == tmp451) {
++ if (data->flags & LM90_HAVE_EXTENDED_TEMP) {
+ if (config & 0x04)
+ data->flags |= LM90_FLAG_ADT7461_EXT;
+ }
+@@ -1767,22 +1808,22 @@ static bool lm90_is_tripped(struct i2c_client *client, u16 *status)
+
+ if (st & LM90_STATUS_LLOW)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_min, 0);
++ hwmon_temp_min_alarm, 0);
+ if (st & LM90_STATUS_RLOW)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_min, 1);
++ hwmon_temp_min_alarm, 1);
+ if (st2 & MAX6696_STATUS2_R2LOW)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_min, 2);
++ hwmon_temp_min_alarm, 2);
+ if (st & LM90_STATUS_LHIGH)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_max, 0);
++ hwmon_temp_max_alarm, 0);
+ if (st & LM90_STATUS_RHIGH)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_max, 1);
++ hwmon_temp_max_alarm, 1);
+ if (st2 & MAX6696_STATUS2_R2HIGH)
+ hwmon_notify_event(data->hwmon_dev, hwmon_temp,
+- hwmon_temp_max, 2);
++ hwmon_temp_max_alarm, 2);
+
+ return true;
+ }
+@@ -1878,11 +1919,14 @@ static int lm90_probe(struct i2c_client *client)
+ info->config = data->channel_config;
+
+ data->channel_config[0] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX |
+- HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM |
+- HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM;
++ HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM;
+ data->channel_config[1] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX |
+- HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM |
+- HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT;
++ HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_FAULT;
++
++ if (data->flags & LM90_HAVE_CRIT) {
++ data->channel_config[0] |= HWMON_T_CRIT | HWMON_T_CRIT_ALARM | HWMON_T_CRIT_HYST;
++ data->channel_config[1] |= HWMON_T_CRIT | HWMON_T_CRIT_ALARM | HWMON_T_CRIT_HYST;
++ }
+
+ if (data->flags & LM90_HAVE_OFFSET)
+ data->channel_config[1] |= HWMON_T_OFFSET;
+diff --git a/drivers/hwmon/mr75203.c b/drivers/hwmon/mr75203.c
+index 868243dba1ee0..1ba1e31459690 100644
+--- a/drivers/hwmon/mr75203.c
++++ b/drivers/hwmon/mr75203.c
+@@ -93,7 +93,7 @@
+ #define VM_CH_REQ BIT(21)
+
+ #define IP_TMR 0x05
+-#define POWER_DELAY_CYCLE_256 0x80
++#define POWER_DELAY_CYCLE_256 0x100
+ #define POWER_DELAY_CYCLE_64 0x40
+
+ #define PVT_POLL_DELAY_US 20
+diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
+index d209e0afc2caa..66d3e88b54172 100644
+--- a/drivers/hwmon/pmbus/lm25066.c
++++ b/drivers/hwmon/pmbus/lm25066.c
+@@ -51,26 +51,31 @@ struct __coeff {
+ #define PSC_CURRENT_IN_L (PSC_NUM_CLASSES)
+ #define PSC_POWER_L (PSC_NUM_CLASSES + 1)
+
+-static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
++static struct __coeff lm25066_coeff[][PSC_NUM_CLASSES + 2] = {
+ [lm25056] = {
+ [PSC_VOLTAGE_IN] = {
+ .m = 16296,
++ .b = 1343,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN] = {
+ .m = 13797,
++ .b = -1833,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN_L] = {
+ .m = 6726,
++ .b = -537,
+ .R = -2,
+ },
+ [PSC_POWER] = {
+ .m = 5501,
++ .b = -2908,
+ .R = -3,
+ },
+ [PSC_POWER_L] = {
+ .m = 26882,
++ .b = -5646,
+ .R = -4,
+ },
+ [PSC_TEMPERATURE] = {
+@@ -82,26 +87,32 @@ static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
+ [lm25066] = {
+ [PSC_VOLTAGE_IN] = {
+ .m = 22070,
++ .b = -1800,
+ .R = -2,
+ },
+ [PSC_VOLTAGE_OUT] = {
+ .m = 22070,
++ .b = -1800,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN] = {
+ .m = 13661,
++ .b = -5200,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN_L] = {
+ .m = 6852,
++ .b = -3100,
+ .R = -2,
+ },
+ [PSC_POWER] = {
+ .m = 736,
++ .b = -3300,
+ .R = -2,
+ },
+ [PSC_POWER_L] = {
+ .m = 369,
++ .b = -1900,
+ .R = -2,
+ },
+ [PSC_TEMPERATURE] = {
+@@ -111,26 +122,32 @@ static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
+ [lm5064] = {
+ [PSC_VOLTAGE_IN] = {
+ .m = 4611,
++ .b = -642,
+ .R = -2,
+ },
+ [PSC_VOLTAGE_OUT] = {
+ .m = 4621,
++ .b = 423,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN] = {
+ .m = 10742,
++ .b = 1552,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN_L] = {
+ .m = 5456,
++ .b = 2118,
+ .R = -2,
+ },
+ [PSC_POWER] = {
+ .m = 1204,
++ .b = 8524,
+ .R = -3,
+ },
+ [PSC_POWER_L] = {
+ .m = 612,
++ .b = 11202,
+ .R = -3,
+ },
+ [PSC_TEMPERATURE] = {
+@@ -140,26 +157,32 @@ static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
+ [lm5066] = {
+ [PSC_VOLTAGE_IN] = {
+ .m = 4587,
++ .b = -1200,
+ .R = -2,
+ },
+ [PSC_VOLTAGE_OUT] = {
+ .m = 4587,
++ .b = -2400,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN] = {
+ .m = 10753,
++ .b = -1200,
+ .R = -2,
+ },
+ [PSC_CURRENT_IN_L] = {
+ .m = 5405,
++ .b = -600,
+ .R = -2,
+ },
+ [PSC_POWER] = {
+ .m = 1204,
++ .b = -6000,
+ .R = -3,
+ },
+ [PSC_POWER_L] = {
+ .m = 605,
++ .b = -8000,
+ .R = -3,
+ },
+ [PSC_TEMPERATURE] = {
+diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
+index e0aa8aa46d8c4..ef3a8ecde4dfc 100644
+--- a/drivers/hwmon/pmbus/pmbus.h
++++ b/drivers/hwmon/pmbus/pmbus.h
+@@ -319,6 +319,7 @@ enum pmbus_fan_mode { percent = 0, rpm };
+ /*
+ * STATUS_VOUT, STATUS_INPUT
+ */
++#define PB_VOLTAGE_VIN_OFF BIT(3)
+ #define PB_VOLTAGE_UV_FAULT BIT(4)
+ #define PB_VOLTAGE_UV_WARNING BIT(5)
+ #define PB_VOLTAGE_OV_WARNING BIT(6)
+diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
+index 776ee2237be20..5f8f824d997f8 100644
+--- a/drivers/hwmon/pmbus/pmbus_core.c
++++ b/drivers/hwmon/pmbus/pmbus_core.c
+@@ -911,6 +911,11 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b,
+ pmbus_update_sensor_data(client, s2);
+
+ regval = status & mask;
++ if (regval) {
++ ret = pmbus_write_byte_data(client, page, reg, regval);
++ if (ret)
++ goto unlock;
++ }
+ if (s1 && s2) {
+ s64 v1, v2;
+
+@@ -1368,7 +1373,7 @@ static const struct pmbus_limit_attr vin_limit_attrs[] = {
+ .reg = PMBUS_VIN_UV_FAULT_LIMIT,
+ .attr = "lcrit",
+ .alarm = "lcrit_alarm",
+- .sbit = PB_VOLTAGE_UV_FAULT,
++ .sbit = PB_VOLTAGE_UV_FAULT | PB_VOLTAGE_VIN_OFF,
+ }, {
+ .reg = PMBUS_VIN_OV_WARN_LIMIT,
+ .attr = "max",
+@@ -2321,6 +2326,9 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data,
+ data->has_status_word = true;
+ }
+
++ /* Make sure PEC is disabled, will be enabled later if needed */
++ client->flags &= ~I2C_CLIENT_PEC;
++
+ /* Enable PEC if the controller and bus supports it */
+ if (!(data->flags & PMBUS_NO_CAPABILITY)) {
+ ret = i2c_smbus_read_byte_data(client, PMBUS_CAPABILITY);
+@@ -2386,10 +2394,14 @@ static int pmbus_regulator_is_enabled(struct regulator_dev *rdev)
+ {
+ struct device *dev = rdev_get_dev(rdev);
+ struct i2c_client *client = to_i2c_client(dev->parent);
++ struct pmbus_data *data = i2c_get_clientdata(client);
+ u8 page = rdev_get_id(rdev);
+ int ret;
+
++ mutex_lock(&data->update_lock);
+ ret = pmbus_read_byte_data(client, page, PMBUS_OPERATION);
++ mutex_unlock(&data->update_lock);
++
+ if (ret < 0)
+ return ret;
+
+@@ -2400,11 +2412,17 @@ static int _pmbus_regulator_on_off(struct regulator_dev *rdev, bool enable)
+ {
+ struct device *dev = rdev_get_dev(rdev);
+ struct i2c_client *client = to_i2c_client(dev->parent);
++ struct pmbus_data *data = i2c_get_clientdata(client);
+ u8 page = rdev_get_id(rdev);
++ int ret;
+
+- return pmbus_update_byte_data(client, page, PMBUS_OPERATION,
+- PB_OPERATION_CONTROL_ON,
+- enable ? PB_OPERATION_CONTROL_ON : 0);
++ mutex_lock(&data->update_lock);
++ ret = pmbus_update_byte_data(client, page, PMBUS_OPERATION,
++ PB_OPERATION_CONTROL_ON,
++ enable ? PB_OPERATION_CONTROL_ON : 0);
++ mutex_unlock(&data->update_lock);
++
++ return ret;
+ }
+
+ static int pmbus_regulator_enable(struct regulator_dev *rdev)
+diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c
+index 17518b4cab1b0..f12b9a28a232d 100644
+--- a/drivers/hwmon/pwm-fan.c
++++ b/drivers/hwmon/pwm-fan.c
+@@ -336,8 +336,6 @@ static int pwm_fan_probe(struct platform_device *pdev)
+ return ret;
+ }
+
+- ctx->pwm_value = MAX_PWM;
+-
+ pwm_init_state(ctx->pwm, &ctx->pwm_state);
+
+ /*
+diff --git a/drivers/hwmon/sch56xx-common.c b/drivers/hwmon/sch56xx-common.c
+index 40cdadad35e52..f85eede6d7663 100644
+--- a/drivers/hwmon/sch56xx-common.c
++++ b/drivers/hwmon/sch56xx-common.c
+@@ -422,7 +422,7 @@ void sch56xx_watchdog_register(struct device *parent, u16 addr, u32 revision,
+ data->wddev.max_timeout = 255 * 60;
+ watchdog_set_nowayout(&data->wddev, nowayout);
+ if (output_enable & SCH56XX_WDOG_OUTPUT_ENABLE)
+- set_bit(WDOG_ACTIVE, &data->wddev.status);
++ set_bit(WDOG_HW_RUNNING, &data->wddev.status);
+
+ /* Since the watchdog uses a downcounter there is no register to read
+ the BIOS set timeout from (if any was set at all) ->
+diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c
+index 9dc210b55e69b..48466b0a4bb05 100644
+--- a/drivers/hwmon/tmp401.c
++++ b/drivers/hwmon/tmp401.c
+@@ -730,10 +730,21 @@ static int tmp401_probe(struct i2c_client *client)
+ return 0;
+ }
+
++static const struct of_device_id __maybe_unused tmp4xx_of_match[] = {
++ { .compatible = "ti,tmp401", },
++ { .compatible = "ti,tmp411", },
++ { .compatible = "ti,tmp431", },
++ { .compatible = "ti,tmp432", },
++ { .compatible = "ti,tmp435", },
++ { },
++};
++MODULE_DEVICE_TABLE(of, tmp4xx_of_match);
++
+ static struct i2c_driver tmp401_driver = {
+ .class = I2C_CLASS_HWMON,
+ .driver = {
+ .name = "tmp401",
++ .of_match_table = of_match_ptr(tmp4xx_of_match),
+ },
+ .probe_new = tmp401_probe,
+ .id_table = tmp401_id,
+diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c
+index e2a3620cbf489..8988b2ed2ea6f 100644
+--- a/drivers/hwtracing/coresight/coresight-cti-core.c
++++ b/drivers/hwtracing/coresight/coresight-cti-core.c
+@@ -175,7 +175,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata)
+ coresight_disclaim_device_unlocked(csdev);
+ CS_LOCK(drvdata->base);
+ spin_unlock(&drvdata->spinlock);
+- pm_runtime_put(dev);
++ pm_runtime_put(dev->parent);
+ return 0;
+
+ /* not disabled this call */
+diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+index a0640fa5c55bd..57e94424a8d65 100644
+--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
++++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+@@ -367,8 +367,12 @@ static ssize_t mode_store(struct device *dev,
+ mode = ETM_MODE_QELEM(config->mode);
+ /* start by clearing QE bits */
+ config->cfg &= ~(BIT(13) | BIT(14));
+- /* if supported, Q elements with instruction counts are enabled */
+- if ((mode & BIT(0)) && (drvdata->q_support & BIT(0)))
++ /*
++ * if supported, Q elements with instruction counts are enabled.
++ * Always set the low bit for any requested mode. Valid combos are
++ * 0b00, 0b01 and 0b11.
++ */
++ if (mode && drvdata->q_support)
+ config->cfg |= BIT(13);
+ /*
+ * if supported, Q elements with and without instruction
+diff --git a/drivers/hwtracing/coresight/coresight-syscfg.c b/drivers/hwtracing/coresight/coresight-syscfg.c
+index 43054568430f2..c30989e0675f5 100644
+--- a/drivers/hwtracing/coresight/coresight-syscfg.c
++++ b/drivers/hwtracing/coresight/coresight-syscfg.c
+@@ -791,7 +791,7 @@ static int cscfg_create_device(void)
+
+ err = device_register(dev);
+ if (err)
+- cscfg_dev_release(dev);
++ put_device(dev);
+
+ create_dev_exit_unlock:
+ mutex_unlock(&cscfg_mutex);
+diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c
+index 1768684968797..7dddb85b90591 100644
+--- a/drivers/hwtracing/coresight/coresight-trbe.c
++++ b/drivers/hwtracing/coresight/coresight-trbe.c
+@@ -366,7 +366,7 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
+
+ static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
+ {
+- struct trbe_buf *buf = perf_get_aux(handle);
++ struct trbe_buf *buf = etm_perf_sink_config(handle);
+ u64 limit = __trbe_normal_offset(handle);
+ u64 head = PERF_IDX2OFF(handle->head, buf);
+
+@@ -869,6 +869,10 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp
+ if (WARN_ON(trbe_csdev))
+ return;
+
++ /* If the TRBE was not probed on the CPU, we shouldn't be here */
++ if (WARN_ON(!cpudata->drvdata))
++ return;
++
+ dev = &cpudata->drvdata->pdev->dev;
+ desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
+ if (!desc.name)
+@@ -950,7 +954,9 @@ static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
+ return -ENOMEM;
+
+ for_each_cpu(cpu, &drvdata->supported_cpus) {
+- smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1);
++ /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
++ if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
++ continue;
+ if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
+ arm_trbe_register_coresight_cpu(drvdata, cpu);
+ if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
+diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
+index e17790fe35a74..fea403431f228 100644
+--- a/drivers/i2c/busses/Kconfig
++++ b/drivers/i2c/busses/Kconfig
+@@ -488,7 +488,7 @@ config I2C_BRCMSTB
+
+ config I2C_CADENCE
+ tristate "Cadence I2C Controller"
+- depends on ARCH_ZYNQ || ARM64 || XTENSA
++ depends on ARCH_ZYNQ || ARM64 || XTENSA || COMPILE_TEST
+ help
+ Say yes here to select Cadence I2C Host Controller. This controller is
+ e.g. used by Xilinx Zynq.
+@@ -677,7 +677,7 @@ config I2C_IMG
+
+ config I2C_IMX
+ tristate "IMX I2C interface"
+- depends on ARCH_MXC || ARCH_LAYERSCAPE || COLDFIRE
++ depends on ARCH_MXC || ARCH_LAYERSCAPE || COLDFIRE || COMPILE_TEST
+ select I2C_SLAVE
+ help
+ Say Y here if you want to use the IIC bus controller on
+@@ -921,7 +921,7 @@ config I2C_QCOM_GENI
+
+ config I2C_QUP
+ tristate "Qualcomm QUP based I2C controller"
+- depends on ARCH_QCOM
++ depends on ARCH_QCOM || COMPILE_TEST
+ help
+ If you say yes to this option, support will be included for the
+ built-in I2C interface on the Qualcomm SoCs.
+diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c
+index 37443edbf7546..f72c6576d8a36 100644
+--- a/drivers/i2c/busses/i2c-bcm2835.c
++++ b/drivers/i2c/busses/i2c-bcm2835.c
+@@ -23,6 +23,11 @@
+ #define BCM2835_I2C_FIFO 0x10
+ #define BCM2835_I2C_DIV 0x14
+ #define BCM2835_I2C_DEL 0x18
++/*
++ * 16-bit field for the number of SCL cycles to wait after rising SCL
++ * before deciding the slave is not responding. 0 disables the
++ * timeout detection.
++ */
+ #define BCM2835_I2C_CLKT 0x1c
+
+ #define BCM2835_I2C_C_READ BIT(0)
+@@ -402,7 +407,7 @@ static const struct i2c_adapter_quirks bcm2835_i2c_quirks = {
+ static int bcm2835_i2c_probe(struct platform_device *pdev)
+ {
+ struct bcm2835_i2c_dev *i2c_dev;
+- struct resource *mem, *irq;
++ struct resource *mem;
+ int ret;
+ struct i2c_adapter *adap;
+ struct clk *mclk;
+@@ -449,21 +454,20 @@ static int bcm2835_i2c_probe(struct platform_device *pdev)
+ ret = clk_prepare_enable(i2c_dev->bus_clk);
+ if (ret) {
+ dev_err(&pdev->dev, "Couldn't prepare clock");
+- return ret;
++ goto err_put_exclusive_rate;
+ }
+
+- irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+- if (!irq) {
+- dev_err(&pdev->dev, "No IRQ resource\n");
+- return -ENODEV;
++ i2c_dev->irq = platform_get_irq(pdev, 0);
++ if (i2c_dev->irq < 0) {
++ ret = i2c_dev->irq;
++ goto err_disable_unprepare_clk;
+ }
+- i2c_dev->irq = irq->start;
+
+ ret = request_irq(i2c_dev->irq, bcm2835_i2c_isr, IRQF_SHARED,
+ dev_name(&pdev->dev), i2c_dev);
+ if (ret) {
+ dev_err(&pdev->dev, "Could not request IRQ\n");
+- return -ENODEV;
++ goto err_disable_unprepare_clk;
+ }
+
+ adap = &i2c_dev->adapter;
+@@ -477,11 +481,26 @@ static int bcm2835_i2c_probe(struct platform_device *pdev)
+ adap->dev.of_node = pdev->dev.of_node;
+ adap->quirks = of_device_get_match_data(&pdev->dev);
+
++ /*
++ * Disable the hardware clock stretching timeout. SMBUS
++ * specifies a limit for how long the device can stretch the
++ * clock, but core I2C doesn't.
++ */
++ bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_CLKT, 0);
+ bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, 0);
+
+ ret = i2c_add_adapter(adap);
+ if (ret)
+- free_irq(i2c_dev->irq, i2c_dev);
++ goto err_free_irq;
++
++ return 0;
++
++err_free_irq:
++ free_irq(i2c_dev->irq, i2c_dev);
++err_disable_unprepare_clk:
++ clk_disable_unprepare(i2c_dev->bus_clk);
++err_put_exclusive_rate:
++ clk_rate_exclusive_put(i2c_dev->bus_clk);
+
+ return ret;
+ }
+diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c
+index 490ee3962645d..b00f35c0b0662 100644
+--- a/drivers/i2c/busses/i2c-brcmstb.c
++++ b/drivers/i2c/busses/i2c-brcmstb.c
+@@ -673,7 +673,7 @@ static int brcmstb_i2c_probe(struct platform_device *pdev)
+
+ /* set the data in/out register size for compatible SoCs */
+ if (of_device_is_compatible(dev->device->of_node,
+- "brcmstb,brcmper-i2c"))
++ "brcm,brcmper-i2c"))
+ dev->data_regsz = sizeof(u8);
+ else
+ dev->data_regsz = sizeof(u32);
+diff --git a/drivers/i2c/busses/i2c-cbus-gpio.c b/drivers/i2c/busses/i2c-cbus-gpio.c
+index 72df563477b1c..f8639a4457d23 100644
+--- a/drivers/i2c/busses/i2c-cbus-gpio.c
++++ b/drivers/i2c/busses/i2c-cbus-gpio.c
+@@ -195,8 +195,9 @@ static u32 cbus_i2c_func(struct i2c_adapter *adapter)
+ }
+
+ static const struct i2c_algorithm cbus_i2c_algo = {
+- .smbus_xfer = cbus_i2c_smbus_xfer,
+- .functionality = cbus_i2c_func,
++ .smbus_xfer = cbus_i2c_smbus_xfer,
++ .smbus_xfer_atomic = cbus_i2c_smbus_xfer,
++ .functionality = cbus_i2c_func,
+ };
+
+ static int cbus_i2c_remove(struct platform_device *pdev)
+diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c
+index 0f409a4c2da0d..5b45941bcbddc 100644
+--- a/drivers/i2c/busses/i2c-designware-pcidrv.c
++++ b/drivers/i2c/busses/i2c-designware-pcidrv.c
+@@ -39,10 +39,10 @@ enum dw_pci_ctl_id_t {
+ };
+
+ struct dw_scl_sda_cfg {
+- u32 ss_hcnt;
+- u32 fs_hcnt;
+- u32 ss_lcnt;
+- u32 fs_lcnt;
++ u16 ss_hcnt;
++ u16 fs_hcnt;
++ u16 ss_lcnt;
++ u16 fs_lcnt;
+ u32 sda_hold;
+ };
+
+diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
+index 89ae78ef1a1cc..98e39a17fb830 100644
+--- a/drivers/i2c/busses/i2c-i801.c
++++ b/drivers/i2c/busses/i2c-i801.c
+@@ -763,6 +763,11 @@ static int i801_block_transaction(struct i801_priv *priv, union i2c_smbus_data *
+ int result = 0;
+ unsigned char hostc;
+
++ if (read_write == I2C_SMBUS_READ && command == I2C_SMBUS_BLOCK_DATA)
++ data->block[0] = I2C_SMBUS_BLOCK_MAX;
++ else if (data->block[0] < 1 || data->block[0] > I2C_SMBUS_BLOCK_MAX)
++ return -EPROTO;
++
+ if (command == I2C_SMBUS_I2C_BLOCK_DATA) {
+ if (read_write == I2C_SMBUS_WRITE) {
+ /* set I2C_EN bit in configuration register */
+@@ -776,16 +781,6 @@ static int i801_block_transaction(struct i801_priv *priv, union i2c_smbus_data *
+ }
+ }
+
+- if (read_write == I2C_SMBUS_WRITE
+- || command == I2C_SMBUS_I2C_BLOCK_DATA) {
+- if (data->block[0] < 1)
+- data->block[0] = 1;
+- if (data->block[0] > I2C_SMBUS_BLOCK_MAX)
+- data->block[0] = I2C_SMBUS_BLOCK_MAX;
+- } else {
+- data->block[0] = 32; /* max for SMBus block reads */
+- }
+-
+ /* Experience has shown that the block buffer can only be used for
+ SMBus (not I2C) block transactions, even though the datasheet
+ doesn't mention this limitation. */
+@@ -1493,7 +1488,6 @@ static struct platform_device *
+ i801_add_tco_spt(struct i801_priv *priv, struct pci_dev *pci_dev,
+ struct resource *tco_res)
+ {
+- static DEFINE_MUTEX(p2sb_mutex);
+ struct resource *res;
+ unsigned int devfn;
+ u64 base64_addr;
+@@ -1506,7 +1500,7 @@ i801_add_tco_spt(struct i801_priv *priv, struct pci_dev *pci_dev,
+ * enumerated by the PCI subsystem, so we need to unhide/hide it
+ * to lookup the P2SB BAR.
+ */
+- mutex_lock(&p2sb_mutex);
++ pci_lock_rescan_remove();
+
+ devfn = PCI_DEVFN(PCI_SLOT(pci_dev->devfn), 1);
+
+@@ -1524,7 +1518,7 @@ i801_add_tco_spt(struct i801_priv *priv, struct pci_dev *pci_dev,
+ /* Hide the P2SB device, if it was hidden before */
+ if (hidden)
+ pci_bus_write_config_byte(pci_dev->bus, devfn, 0xe1, hidden);
+- mutex_unlock(&p2sb_mutex);
++ pci_unlock_rescan_remove();
+
+ res = &tco_res[1];
+ if (pci_dev->device == PCI_DEVICE_ID_INTEL_DNV_SMBUS)
+diff --git a/drivers/i2c/busses/i2c-meson.c b/drivers/i2c/busses/i2c-meson.c
+index ef73a42577cc7..07eb819072c4f 100644
+--- a/drivers/i2c/busses/i2c-meson.c
++++ b/drivers/i2c/busses/i2c-meson.c
+@@ -465,18 +465,18 @@ static int meson_i2c_probe(struct platform_device *pdev)
+ */
+ meson_i2c_set_mask(i2c, REG_CTRL, REG_CTRL_START, 0);
+
+- ret = i2c_add_adapter(&i2c->adap);
+- if (ret < 0) {
+- clk_disable_unprepare(i2c->clk);
+- return ret;
+- }
+-
+ /* Disable filtering */
+ meson_i2c_set_mask(i2c, REG_SLAVE_ADDR,
+ REG_SLV_SDA_FILTER | REG_SLV_SCL_FILTER, 0);
+
+ meson_i2c_set_clk_div(i2c, timings.bus_freq_hz);
+
++ ret = i2c_add_adapter(&i2c->adap);
++ if (ret < 0) {
++ clk_disable_unprepare(i2c->clk);
++ return ret;
++ }
++
+ return 0;
+ }
+
+diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c
+index a6ea1eb1394e1..6c698c10d3cdb 100644
+--- a/drivers/i2c/busses/i2c-mpc.c
++++ b/drivers/i2c/busses/i2c-mpc.c
+@@ -119,23 +119,30 @@ static inline void writeccr(struct mpc_i2c *i2c, u32 x)
+ /* Sometimes 9th clock pulse isn't generated, and slave doesn't release
+ * the bus, because it wants to send ACK.
+ * Following sequence of enabling/disabling and sending start/stop generates
+- * the 9 pulses, so it's all OK.
++ * the 9 pulses, each with a START then ending with STOP, so it's all OK.
+ */
+ static void mpc_i2c_fixup(struct mpc_i2c *i2c)
+ {
+ int k;
+- u32 delay_val = 1000000 / i2c->real_clk + 1;
+-
+- if (delay_val < 2)
+- delay_val = 2;
++ unsigned long flags;
+
+ for (k = 9; k; k--) {
+ writeccr(i2c, 0);
+- writeccr(i2c, CCR_MSTA | CCR_MTX | CCR_MEN);
++ writeb(0, i2c->base + MPC_I2C_SR); /* clear any status bits */
++ writeccr(i2c, CCR_MEN | CCR_MSTA); /* START */
++ readb(i2c->base + MPC_I2C_DR); /* init xfer */
++ udelay(15); /* let it hit the bus */
++ local_irq_save(flags); /* should not be delayed further */
++ writeccr(i2c, CCR_MEN | CCR_MSTA | CCR_RSTA); /* delay SDA */
+ readb(i2c->base + MPC_I2C_DR);
+- writeccr(i2c, CCR_MEN);
+- udelay(delay_val << 1);
++ if (k != 1)
++ udelay(5);
++ local_irq_restore(flags);
+ }
++ writeccr(i2c, CCR_MEN); /* Initiate STOP */
++ readb(i2c->base + MPC_I2C_DR);
++ udelay(15); /* Let STOP propagate */
++ writeccr(i2c, 0);
+ }
+
+ static int i2c_mpc_wait_sr(struct mpc_i2c *i2c, int mask)
+@@ -492,7 +499,7 @@ static void mpc_i2c_finish(struct mpc_i2c *i2c, int rc)
+
+ static void mpc_i2c_do_action(struct mpc_i2c *i2c)
+ {
+- struct i2c_msg *msg = &i2c->msgs[i2c->curr_msg];
++ struct i2c_msg *msg = NULL;
+ int dir = 0;
+ int recv_len = 0;
+ u8 byte;
+@@ -501,10 +508,13 @@ static void mpc_i2c_do_action(struct mpc_i2c *i2c)
+
+ i2c->cntl_bits &= ~(CCR_RSTA | CCR_MTX | CCR_TXAK);
+
+- if (msg->flags & I2C_M_RD)
+- dir = 1;
+- if (msg->flags & I2C_M_RECV_LEN)
+- recv_len = 1;
++ if (i2c->action != MPC_I2C_ACTION_STOP) {
++ msg = &i2c->msgs[i2c->curr_msg];
++ if (msg->flags & I2C_M_RD)
++ dir = 1;
++ if (msg->flags & I2C_M_RECV_LEN)
++ recv_len = 1;
++ }
+
+ switch (i2c->action) {
+ case MPC_I2C_ACTION_RESTART:
+@@ -581,7 +591,7 @@ static void mpc_i2c_do_action(struct mpc_i2c *i2c)
+ break;
+ }
+
+- if (msg->len == i2c->byte_posn) {
++ if (msg && msg->len == i2c->byte_posn) {
+ i2c->curr_msg++;
+ i2c->byte_posn = 0;
+
+@@ -636,7 +646,7 @@ static irqreturn_t mpc_i2c_isr(int irq, void *dev_id)
+ status = readb(i2c->base + MPC_I2C_SR);
+ if (status & CSR_MIF) {
+ /* Wait up to 100us for transfer to properly complete */
+- readb_poll_timeout(i2c->base + MPC_I2C_SR, status, !(status & CSR_MCF), 0, 100);
++ readb_poll_timeout_atomic(i2c->base + MPC_I2C_SR, status, status & CSR_MCF, 0, 100);
+ writeb(0, i2c->base + MPC_I2C_SR);
+ mpc_i2c_do_intr(i2c, status);
+ return IRQ_HANDLED;
+diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
+index 7d4b3eb7077ad..72acda59eb399 100644
+--- a/drivers/i2c/busses/i2c-mt65xx.c
++++ b/drivers/i2c/busses/i2c-mt65xx.c
+@@ -195,7 +195,7 @@ static const u16 mt_i2c_regs_v2[] = {
+ [OFFSET_CLOCK_DIV] = 0x48,
+ [OFFSET_SOFTRESET] = 0x50,
+ [OFFSET_SCL_MIS_COMP_POINT] = 0x90,
+- [OFFSET_DEBUGSTAT] = 0xe0,
++ [OFFSET_DEBUGSTAT] = 0xe4,
+ [OFFSET_DEBUGCTRL] = 0xe8,
+ [OFFSET_FIFO_STAT] = 0xf4,
+ [OFFSET_FIFO_THRESH] = 0xf8,
+diff --git a/drivers/i2c/busses/i2c-mt7621.c b/drivers/i2c/busses/i2c-mt7621.c
+index 45fe4a7fe0c03..901f0fb04fee4 100644
+--- a/drivers/i2c/busses/i2c-mt7621.c
++++ b/drivers/i2c/busses/i2c-mt7621.c
+@@ -304,7 +304,8 @@ static int mtk_i2c_probe(struct platform_device *pdev)
+
+ if (i2c->bus_freq == 0) {
+ dev_warn(i2c->dev, "clock-frequency 0 not supported\n");
+- return -EINVAL;
++ ret = -EINVAL;
++ goto err_disable_clk;
+ }
+
+ adap = &i2c->adap;
+@@ -322,10 +323,15 @@ static int mtk_i2c_probe(struct platform_device *pdev)
+
+ ret = i2c_add_adapter(adap);
+ if (ret < 0)
+- return ret;
++ goto err_disable_clk;
+
+ dev_info(&pdev->dev, "clock %u kHz\n", i2c->bus_freq / 1000);
+
++ return 0;
++
++err_disable_clk:
++ clk_disable_unprepare(i2c->clk);
++
+ return ret;
+ }
+
+diff --git a/drivers/i2c/busses/i2c-pasemi.c b/drivers/i2c/busses/i2c-pasemi.c
+index 20f2772c0e79b..2c909522f0f38 100644
+--- a/drivers/i2c/busses/i2c-pasemi.c
++++ b/drivers/i2c/busses/i2c-pasemi.c
+@@ -137,6 +137,12 @@ static int pasemi_i2c_xfer_msg(struct i2c_adapter *adapter,
+
+ TXFIFO_WR(smbus, msg->buf[msg->len-1] |
+ (stop ? MTXFIFO_STOP : 0));
++
++ if (stop) {
++ err = pasemi_smb_waitready(smbus);
++ if (err)
++ goto reset_out;
++ }
+ }
+
+ return 0;
+diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
+index 8c1b31ed0c429..ac8e7d60672a1 100644
+--- a/drivers/i2c/busses/i2c-piix4.c
++++ b/drivers/i2c/busses/i2c-piix4.c
+@@ -77,6 +77,7 @@
+
+ /* SB800 constants */
+ #define SB800_PIIX4_SMB_IDX 0xcd6
++#define SB800_PIIX4_SMB_MAP_SIZE 2
+
+ #define KERNCZ_IMC_IDX 0x3e
+ #define KERNCZ_IMC_DATA 0x3f
+@@ -97,6 +98,9 @@
+ #define SB800_PIIX4_PORT_IDX_MASK_KERNCZ 0x18
+ #define SB800_PIIX4_PORT_IDX_SHIFT_KERNCZ 3
+
++#define SB800_PIIX4_FCH_PM_ADDR 0xFED80300
++#define SB800_PIIX4_FCH_PM_SIZE 8
++
+ /* insmod parameters */
+
+ /* If force is set to anything different from 0, we forcibly enable the
+@@ -155,6 +159,12 @@ static const char *piix4_main_port_names_sb800[PIIX4_MAX_ADAPTERS] = {
+ };
+ static const char *piix4_aux_port_name_sb800 = " port 1";
+
++struct sb800_mmio_cfg {
++ void __iomem *addr;
++ struct resource *res;
++ bool use_mmio;
++};
++
+ struct i2c_piix4_adapdata {
+ unsigned short smba;
+
+@@ -162,8 +172,75 @@ struct i2c_piix4_adapdata {
+ bool sb800_main;
+ bool notify_imc;
+ u8 port; /* Port number, shifted */
++ struct sb800_mmio_cfg mmio_cfg;
+ };
+
++static int piix4_sb800_region_request(struct device *dev,
++ struct sb800_mmio_cfg *mmio_cfg)
++{
++ if (mmio_cfg->use_mmio) {
++ struct resource *res;
++ void __iomem *addr;
++
++ res = request_mem_region_muxed(SB800_PIIX4_FCH_PM_ADDR,
++ SB800_PIIX4_FCH_PM_SIZE,
++ "sb800_piix4_smb");
++ if (!res) {
++ dev_err(dev,
++ "SMBus base address memory region 0x%x already in use.\n",
++ SB800_PIIX4_FCH_PM_ADDR);
++ return -EBUSY;
++ }
++
++ addr = ioremap(SB800_PIIX4_FCH_PM_ADDR,
++ SB800_PIIX4_FCH_PM_SIZE);
++ if (!addr) {
++ release_resource(res);
++ dev_err(dev, "SMBus base address mapping failed.\n");
++ return -ENOMEM;
++ }
++
++ mmio_cfg->res = res;
++ mmio_cfg->addr = addr;
++
++ return 0;
++ }
++
++ if (!request_muxed_region(SB800_PIIX4_SMB_IDX, SB800_PIIX4_SMB_MAP_SIZE,
++ "sb800_piix4_smb")) {
++ dev_err(dev,
++ "SMBus base address index region 0x%x already in use.\n",
++ SB800_PIIX4_SMB_IDX);
++ return -EBUSY;
++ }
++
++ return 0;
++}
++
++static void piix4_sb800_region_release(struct device *dev,
++ struct sb800_mmio_cfg *mmio_cfg)
++{
++ if (mmio_cfg->use_mmio) {
++ iounmap(mmio_cfg->addr);
++ release_resource(mmio_cfg->res);
++ return;
++ }
++
++ release_region(SB800_PIIX4_SMB_IDX, SB800_PIIX4_SMB_MAP_SIZE);
++}
++
++static bool piix4_sb800_use_mmio(struct pci_dev *PIIX4_dev)
++{
++ /*
++ * cd6h/cd7h port I/O accesses can be disabled on AMD processors
++ * w/ SMBus PCI revision ID 0x51 or greater. MMIO is supported on
++ * the same processors and is the recommended access method.
++ */
++ return (PIIX4_dev->vendor == PCI_VENDOR_ID_AMD &&
++ PIIX4_dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS &&
++ PIIX4_dev->revision >= 0x51);
++}
++
+ static int piix4_setup(struct pci_dev *PIIX4_dev,
+ const struct pci_device_id *id)
+ {
+@@ -263,12 +340,61 @@ static int piix4_setup(struct pci_dev *PIIX4_dev,
+ return piix4_smba;
+ }
+
++static int piix4_setup_sb800_smba(struct pci_dev *PIIX4_dev,
++ u8 smb_en,
++ u8 aux,
++ u8 *smb_en_status,
++ unsigned short *piix4_smba)
++{
++ struct sb800_mmio_cfg mmio_cfg;
++ u8 smba_en_lo;
++ u8 smba_en_hi;
++ int retval;
++
++ mmio_cfg.use_mmio = piix4_sb800_use_mmio(PIIX4_dev);
++ retval = piix4_sb800_region_request(&PIIX4_dev->dev, &mmio_cfg);
++ if (retval)
++ return retval;
++
++ if (mmio_cfg.use_mmio) {
++ smba_en_lo = ioread8(mmio_cfg.addr);
++ smba_en_hi = ioread8(mmio_cfg.addr + 1);
++ } else {
++ outb_p(smb_en, SB800_PIIX4_SMB_IDX);
++ smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
++ outb_p(smb_en + 1, SB800_PIIX4_SMB_IDX);
++ smba_en_hi = inb_p(SB800_PIIX4_SMB_IDX + 1);
++ }
++
++ piix4_sb800_region_release(&PIIX4_dev->dev, &mmio_cfg);
++
++ if (!smb_en) {
++ *smb_en_status = smba_en_lo & 0x10;
++ *piix4_smba = smba_en_hi << 8;
++ if (aux)
++ *piix4_smba |= 0x20;
++ } else {
++ *smb_en_status = smba_en_lo & 0x01;
++ *piix4_smba = ((smba_en_hi << 8) | smba_en_lo) & 0xffe0;
++ }
++
++ if (!*smb_en_status) {
++ dev_err(&PIIX4_dev->dev,
++ "SMBus Host Controller not enabled!\n");
++ return -ENODEV;
++ }
++
++ return 0;
++}
++
+ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
+ const struct pci_device_id *id, u8 aux)
+ {
+ unsigned short piix4_smba;
+- u8 smba_en_lo, smba_en_hi, smb_en, smb_en_status, port_sel;
++ u8 smb_en, smb_en_status, port_sel;
+ u8 i2ccfg, i2ccfg_offset = 0x10;
++ struct sb800_mmio_cfg mmio_cfg;
++ int retval;
+
+ /* SB800 and later SMBus does not support forcing address */
+ if (force || force_addr) {
+@@ -290,35 +416,11 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
+ else
+ smb_en = (aux) ? 0x28 : 0x2c;
+
+- if (!request_muxed_region(SB800_PIIX4_SMB_IDX, 2, "sb800_piix4_smb")) {
+- dev_err(&PIIX4_dev->dev,
+- "SMB base address index region 0x%x already in use.\n",
+- SB800_PIIX4_SMB_IDX);
+- return -EBUSY;
+- }
+-
+- outb_p(smb_en, SB800_PIIX4_SMB_IDX);
+- smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
+- outb_p(smb_en + 1, SB800_PIIX4_SMB_IDX);
+- smba_en_hi = inb_p(SB800_PIIX4_SMB_IDX + 1);
++ retval = piix4_setup_sb800_smba(PIIX4_dev, smb_en, aux, &smb_en_status,
++ &piix4_smba);
+
+- release_region(SB800_PIIX4_SMB_IDX, 2);
+-
+- if (!smb_en) {
+- smb_en_status = smba_en_lo & 0x10;
+- piix4_smba = smba_en_hi << 8;
+- if (aux)
+- piix4_smba |= 0x20;
+- } else {
+- smb_en_status = smba_en_lo & 0x01;
+- piix4_smba = ((smba_en_hi << 8) | smba_en_lo) & 0xffe0;
+- }
+-
+- if (!smb_en_status) {
+- dev_err(&PIIX4_dev->dev,
+- "SMBus Host Controller not enabled!\n");
+- return -ENODEV;
+- }
++ if (retval)
++ return retval;
+
+ if (acpi_check_region(piix4_smba, SMBIOSIZE, piix4_driver.name))
+ return -ENODEV;
+@@ -371,10 +473,11 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
+ piix4_port_shift_sb800 = SB800_PIIX4_PORT_IDX_SHIFT;
+ }
+ } else {
+- if (!request_muxed_region(SB800_PIIX4_SMB_IDX, 2,
+- "sb800_piix4_smb")) {
++ mmio_cfg.use_mmio = piix4_sb800_use_mmio(PIIX4_dev);
++ retval = piix4_sb800_region_request(&PIIX4_dev->dev, &mmio_cfg);
++ if (retval) {
+ release_region(piix4_smba, SMBIOSIZE);
+- return -EBUSY;
++ return retval;
+ }
+
+ outb_p(SB800_PIIX4_PORT_IDX_SEL, SB800_PIIX4_SMB_IDX);
+@@ -384,7 +487,7 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
+ SB800_PIIX4_PORT_IDX;
+ piix4_port_mask_sb800 = SB800_PIIX4_PORT_IDX_MASK;
+ piix4_port_shift_sb800 = SB800_PIIX4_PORT_IDX_SHIFT;
+- release_region(SB800_PIIX4_SMB_IDX, 2);
++ piix4_sb800_region_release(&PIIX4_dev->dev, &mmio_cfg);
+ }
+
+ dev_info(&PIIX4_dev->dev,
+@@ -662,6 +765,29 @@ static void piix4_imc_wakeup(void)
+ release_region(KERNCZ_IMC_IDX, 2);
+ }
+
++static int piix4_sb800_port_sel(u8 port, struct sb800_mmio_cfg *mmio_cfg)
++{
++ u8 smba_en_lo, val;
++
++ if (mmio_cfg->use_mmio) {
++ smba_en_lo = ioread8(mmio_cfg->addr + piix4_port_sel_sb800);
++ val = (smba_en_lo & ~piix4_port_mask_sb800) | port;
++ if (smba_en_lo != val)
++ iowrite8(val, mmio_cfg->addr + piix4_port_sel_sb800);
++
++ return (smba_en_lo & piix4_port_mask_sb800);
++ }
++
++ outb_p(piix4_port_sel_sb800, SB800_PIIX4_SMB_IDX);
++ smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
++
++ val = (smba_en_lo & ~piix4_port_mask_sb800) | port;
++ if (smba_en_lo != val)
++ outb_p(val, SB800_PIIX4_SMB_IDX + 1);
++
++ return (smba_en_lo & piix4_port_mask_sb800);
++}
++
+ /*
+ * Handles access to multiple SMBus ports on the SB800.
+ * The port is selected by bits 2:1 of the smb_en register (0x2c).
+@@ -678,12 +804,12 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr,
+ unsigned short piix4_smba = adapdata->smba;
+ int retries = MAX_TIMEOUT;
+ int smbslvcnt;
+- u8 smba_en_lo;
+- u8 port;
++ u8 prev_port;
+ int retval;
+
+- if (!request_muxed_region(SB800_PIIX4_SMB_IDX, 2, "sb800_piix4_smb"))
+- return -EBUSY;
++ retval = piix4_sb800_region_request(&adap->dev, &adapdata->mmio_cfg);
++ if (retval)
++ return retval;
+
+ /* Request the SMBUS semaphore, avoid conflicts with the IMC */
+ smbslvcnt = inb_p(SMBSLVCNT);
+@@ -738,18 +864,12 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr,
+ }
+ }
+
+- outb_p(piix4_port_sel_sb800, SB800_PIIX4_SMB_IDX);
+- smba_en_lo = inb_p(SB800_PIIX4_SMB_IDX + 1);
+-
+- port = adapdata->port;
+- if ((smba_en_lo & piix4_port_mask_sb800) != port)
+- outb_p((smba_en_lo & ~piix4_port_mask_sb800) | port,
+- SB800_PIIX4_SMB_IDX + 1);
++ prev_port = piix4_sb800_port_sel(adapdata->port, &adapdata->mmio_cfg);
+
+ retval = piix4_access(adap, addr, flags, read_write,
+ command, size, data);
+
+- outb_p(smba_en_lo, SB800_PIIX4_SMB_IDX + 1);
++ piix4_sb800_port_sel(prev_port, &adapdata->mmio_cfg);
+
+ /* Release the semaphore */
+ outb_p(smbslvcnt | 0x20, SMBSLVCNT);
+@@ -758,7 +878,7 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr,
+ piix4_imc_wakeup();
+
+ release:
+- release_region(SB800_PIIX4_SMB_IDX, 2);
++ piix4_sb800_region_release(&adap->dev, &adapdata->mmio_cfg);
+ return retval;
+ }
+
+@@ -836,6 +956,7 @@ static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
+ return -ENOMEM;
+ }
+
++ adapdata->mmio_cfg.use_mmio = piix4_sb800_use_mmio(dev);
+ adapdata->smba = smba;
+ adapdata->sb800_main = sb800_main;
+ adapdata->port = port << piix4_port_shift_sb800;
+diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c
+index c1de8eb66169f..cf54f1cb4c57a 100644
+--- a/drivers/i2c/busses/i2c-qcom-cci.c
++++ b/drivers/i2c/busses/i2c-qcom-cci.c
+@@ -558,7 +558,7 @@ static int cci_probe(struct platform_device *pdev)
+ cci->master[idx].adap.quirks = &cci->data->quirks;
+ cci->master[idx].adap.algo = &cci_algo;
+ cci->master[idx].adap.dev.parent = dev;
+- cci->master[idx].adap.dev.of_node = child;
++ cci->master[idx].adap.dev.of_node = of_node_get(child);
+ cci->master[idx].master = idx;
+ cci->master[idx].cci = cci;
+
+@@ -643,8 +643,10 @@ static int cci_probe(struct platform_device *pdev)
+ continue;
+
+ ret = i2c_add_adapter(&cci->master[i].adap);
+- if (ret < 0)
++ if (ret < 0) {
++ of_node_put(cci->master[i].adap.dev.of_node);
+ goto error_i2c;
++ }
+ }
+
+ pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC);
+@@ -655,9 +657,11 @@ static int cci_probe(struct platform_device *pdev)
+ return 0;
+
+ error_i2c:
+- for (; i >= 0; i--) {
+- if (cci->master[i].cci)
++ for (--i ; i >= 0; i--) {
++ if (cci->master[i].cci) {
+ i2c_del_adapter(&cci->master[i].adap);
++ of_node_put(cci->master[i].adap.dev.of_node);
++ }
+ }
+ error:
+ disable_irq(cci->irq);
+@@ -673,8 +677,10 @@ static int cci_remove(struct platform_device *pdev)
+ int i;
+
+ for (i = 0; i < cci->data->num_masters; i++) {
+- if (cci->master[i].cci)
++ if (cci->master[i].cci) {
+ i2c_del_adapter(&cci->master[i].adap);
++ of_node_put(cci->master[i].adap.dev.of_node);
++ }
+ cci_halt(cci, i);
+ }
+
+diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
+index 819ab4ee517e1..02ddb237f69af 100644
+--- a/drivers/i2c/busses/i2c-rk3x.c
++++ b/drivers/i2c/busses/i2c-rk3x.c
+@@ -423,8 +423,8 @@ static void rk3x_i2c_handle_read(struct rk3x_i2c *i2c, unsigned int ipd)
+ if (!(ipd & REG_INT_MBRF))
+ return;
+
+- /* ack interrupt */
+- i2c_writel(i2c, REG_INT_MBRF, REG_IPD);
++ /* ack interrupt (read also produces a spurious START flag, clear it too) */
++ i2c_writel(i2c, REG_INT_MBRF | REG_INT_START, REG_IPD);
+
+ /* Can only handle a maximum of 32 bytes at a time */
+ if (len > 32)
+diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
+index b9b19a2a2ffa0..50d5ae81d2271 100644
+--- a/drivers/i2c/busses/i2c-stm32f7.c
++++ b/drivers/i2c/busses/i2c-stm32f7.c
+@@ -1493,6 +1493,7 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
+ {
+ struct stm32f7_i2c_dev *i2c_dev = data;
+ struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
++ struct stm32_i2c_dma *dma = i2c_dev->dma;
+ void __iomem *base = i2c_dev->base;
+ u32 status, mask;
+ int ret = IRQ_HANDLED;
+@@ -1518,6 +1519,10 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
+ dev_dbg(i2c_dev->dev, "<%s>: Receive NACK (addr %x)\n",
+ __func__, f7_msg->addr);
+ writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR);
++ if (i2c_dev->use_dma) {
++ stm32f7_i2c_disable_dma_req(i2c_dev);
++ dmaengine_terminate_all(dma->chan_using);
++ }
+ f7_msg->result = -ENXIO;
+ }
+
+@@ -1533,7 +1538,7 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
+ /* Clear STOP flag */
+ writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR);
+
+- if (i2c_dev->use_dma) {
++ if (i2c_dev->use_dma && !f7_msg->result) {
+ ret = IRQ_WAKE_THREAD;
+ } else {
+ i2c_dev->master_mode = false;
+@@ -1546,7 +1551,7 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
+ if (f7_msg->stop) {
+ mask = STM32F7_I2C_CR2_STOP;
+ stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask);
+- } else if (i2c_dev->use_dma) {
++ } else if (i2c_dev->use_dma && !f7_msg->result) {
+ ret = IRQ_WAKE_THREAD;
+ } else if (f7_msg->smbus) {
+ stm32f7_i2c_smbus_rep_start(i2c_dev);
+@@ -1696,12 +1701,23 @@ static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap,
+ time_left = wait_for_completion_timeout(&i2c_dev->complete,
+ i2c_dev->adap.timeout);
+ ret = f7_msg->result;
++ if (ret) {
++ /*
++ * It is possible that some unsent data have already been
++ * written into TXDR. To avoid sending old data in a
++ * further transfer, flush TXDR in case of any error
++ */
++ writel_relaxed(STM32F7_I2C_ISR_TXE,
++ i2c_dev->base + STM32F7_I2C_ISR);
++ goto pm_free;
++ }
+
+ if (!time_left) {
+ dev_dbg(i2c_dev->dev, "Access to slave 0x%x timed out\n",
+ i2c_dev->msg->addr);
+ if (i2c_dev->use_dma)
+ dmaengine_terminate_all(dma->chan_using);
++ stm32f7_i2c_wait_free_bus(i2c_dev);
+ ret = -ETIMEDOUT;
+ }
+
+@@ -1744,13 +1760,22 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
+ timeout = wait_for_completion_timeout(&i2c_dev->complete,
+ i2c_dev->adap.timeout);
+ ret = f7_msg->result;
+- if (ret)
++ if (ret) {
++ /*
++ * It is possible that some unsent data have already been
++ * written into TXDR. To avoid sending old data in a
++ * further transfer, flush TXDR in case of any error
++ */
++ writel_relaxed(STM32F7_I2C_ISR_TXE,
++ i2c_dev->base + STM32F7_I2C_ISR);
+ goto pm_free;
++ }
+
+ if (!timeout) {
+ dev_dbg(dev, "Access to slave 0x%x timed out\n", f7_msg->addr);
+ if (i2c_dev->use_dma)
+ dmaengine_terminate_all(dma->chan_using);
++ stm32f7_i2c_wait_free_bus(i2c_dev);
+ ret = -ETIMEDOUT;
+ goto pm_free;
+ }
+diff --git a/drivers/i2c/busses/i2c-virtio.c b/drivers/i2c/busses/i2c-virtio.c
+index f10a603b13fb0..5cb21d7da05b6 100644
+--- a/drivers/i2c/busses/i2c-virtio.c
++++ b/drivers/i2c/busses/i2c-virtio.c
+@@ -22,24 +22,24 @@
+ /**
+ * struct virtio_i2c - virtio I2C data
+ * @vdev: virtio device for this controller
+- * @completion: completion of virtio I2C message
+ * @adap: I2C adapter for this controller
+ * @vq: the virtio virtqueue for communication
+ */
+ struct virtio_i2c {
+ struct virtio_device *vdev;
+- struct completion completion;
+ struct i2c_adapter adap;
+ struct virtqueue *vq;
+ };
+
+ /**
+ * struct virtio_i2c_req - the virtio I2C request structure
++ * @completion: completion of virtio I2C message
+ * @out_hdr: the OUT header of the virtio I2C message
+ * @buf: the buffer into which data is read, or from which it's written
+ * @in_hdr: the IN header of the virtio I2C message
+ */
+ struct virtio_i2c_req {
++ struct completion completion;
+ struct virtio_i2c_out_hdr out_hdr ____cacheline_aligned;
+ uint8_t *buf ____cacheline_aligned;
+ struct virtio_i2c_in_hdr in_hdr ____cacheline_aligned;
+@@ -47,9 +47,11 @@ struct virtio_i2c_req {
+
+ static void virtio_i2c_msg_done(struct virtqueue *vq)
+ {
+- struct virtio_i2c *vi = vq->vdev->priv;
++ struct virtio_i2c_req *req;
++ unsigned int len;
+
+- complete(&vi->completion);
++ while ((req = virtqueue_get_buf(vq, &len)))
++ complete(&req->completion);
+ }
+
+ static int virtio_i2c_prepare_reqs(struct virtqueue *vq,
+@@ -62,6 +64,8 @@ static int virtio_i2c_prepare_reqs(struct virtqueue *vq,
+ for (i = 0; i < num; i++) {
+ int outcnt = 0, incnt = 0;
+
++ init_completion(&reqs[i].completion);
++
+ /*
+ * We don't support 0 length messages and so filter out
+ * 0 length transfers by using i2c_adapter_quirks.
+@@ -106,24 +110,17 @@ static int virtio_i2c_prepare_reqs(struct virtqueue *vq,
+
+ static int virtio_i2c_complete_reqs(struct virtqueue *vq,
+ struct virtio_i2c_req *reqs,
+- struct i2c_msg *msgs, int num,
+- bool timedout)
++ struct i2c_msg *msgs, int num)
+ {
+- struct virtio_i2c_req *req;
+- bool failed = timedout;
+- unsigned int len;
++ bool failed = false;
+ int i, j = 0;
+
+ for (i = 0; i < num; i++) {
+- /* Detach the ith request from the vq */
+- req = virtqueue_get_buf(vq, &len);
++ struct virtio_i2c_req *req = &reqs[i];
+
+- /*
+- * Condition req == &reqs[i] should always meet since we have
+- * total num requests in the vq. reqs[i] can never be NULL here.
+- */
+- if (!failed && (WARN_ON(req != &reqs[i]) ||
+- req->in_hdr.status != VIRTIO_I2C_MSG_OK))
++ wait_for_completion(&req->completion);
++
++ if (!failed && req->in_hdr.status != VIRTIO_I2C_MSG_OK)
+ failed = true;
+
+ i2c_put_dma_safe_msg_buf(reqs[i].buf, &msgs[i], !failed);
+@@ -132,7 +129,7 @@ static int virtio_i2c_complete_reqs(struct virtqueue *vq,
+ j++;
+ }
+
+- return timedout ? -ETIMEDOUT : j;
++ return j;
+ }
+
+ static int virtio_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+@@ -141,7 +138,6 @@ static int virtio_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+ struct virtio_i2c *vi = i2c_get_adapdata(adap);
+ struct virtqueue *vq = vi->vq;
+ struct virtio_i2c_req *reqs;
+- unsigned long time_left;
+ int count;
+
+ reqs = kcalloc(num, sizeof(*reqs), GFP_KERNEL);
+@@ -160,15 +156,9 @@ static int virtio_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+ * remote here to clear the virtqueue, so we can try another set of
+ * messages later on.
+ */
+-
+- reinit_completion(&vi->completion);
+ virtqueue_kick(vq);
+
+- time_left = wait_for_completion_timeout(&vi->completion, adap->timeout);
+- if (!time_left)
+- dev_err(&adap->dev, "virtio i2c backend timeout.\n");
+-
+- count = virtio_i2c_complete_reqs(vq, reqs, msgs, count, !time_left);
++ count = virtio_i2c_complete_reqs(vq, reqs, msgs, count);
+
+ err_free:
+ kfree(reqs);
+@@ -215,8 +205,6 @@ static int virtio_i2c_probe(struct virtio_device *vdev)
+ vdev->priv = vi;
+ vi->vdev = vdev;
+
+- init_completion(&vi->completion);
+-
+ ret = virtio_i2c_setup_vqs(vi);
+ if (ret)
+ return ret;
+diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
+index bb93db98404ef..612343771ce25 100644
+--- a/drivers/i2c/busses/i2c-xiic.c
++++ b/drivers/i2c/busses/i2c-xiic.c
+@@ -756,7 +756,6 @@ static const struct i2c_adapter_quirks xiic_quirks = {
+
+ static const struct i2c_adapter xiic_adapter = {
+ .owner = THIS_MODULE,
+- .name = DRIVER_NAME,
+ .class = I2C_CLASS_DEPRECATED,
+ .algo = &xiic_algorithm,
+ .quirks = &xiic_quirks,
+@@ -793,6 +792,8 @@ static int xiic_i2c_probe(struct platform_device *pdev)
+ i2c_set_adapdata(&i2c->adap, i2c);
+ i2c->adap.dev.parent = &pdev->dev;
+ i2c->adap.dev.of_node = pdev->dev.of_node;
++ snprintf(i2c->adap.name, sizeof(i2c->adap.name),
++ DRIVER_NAME " %s", pdev->name);
+
+ mutex_init(&i2c->lock);
+ init_waitqueue_head(&i2c->wait);
+diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c
+index 126d1393e548b..9ce20652d4942 100644
+--- a/drivers/i2c/busses/i2c-xlr.c
++++ b/drivers/i2c/busses/i2c-xlr.c
+@@ -431,11 +431,15 @@ static int xlr_i2c_probe(struct platform_device *pdev)
+ i2c_set_adapdata(&priv->adap, priv);
+ ret = i2c_add_numbered_adapter(&priv->adap);
+ if (ret < 0)
+- return ret;
++ goto err_unprepare_clk;
+
+ platform_set_drvdata(pdev, priv);
+ dev_info(&priv->adap.dev, "Added I2C Bus.\n");
+ return 0;
++
++err_unprepare_clk:
++ clk_unprepare(clk);
++ return ret;
+ }
+
+ static int xlr_i2c_remove(struct platform_device *pdev)
+diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
+index 54964fbe3f033..cfbef70e8ba70 100644
+--- a/drivers/i2c/i2c-core-base.c
++++ b/drivers/i2c/i2c-core-base.c
+@@ -466,14 +466,12 @@ static int i2c_smbus_host_notify_to_irq(const struct i2c_client *client)
+ static int i2c_device_probe(struct device *dev)
+ {
+ struct i2c_client *client = i2c_verify_client(dev);
+- struct i2c_adapter *adap;
+ struct i2c_driver *driver;
+ int status;
+
+ if (!client)
+ return 0;
+
+- adap = client->adapter;
+ client->irq = client->init_irq;
+
+ if (!client->irq) {
+@@ -539,14 +537,6 @@ static int i2c_device_probe(struct device *dev)
+
+ dev_dbg(dev, "probe\n");
+
+- if (adap->bus_regulator) {
+- status = regulator_enable(adap->bus_regulator);
+- if (status < 0) {
+- dev_err(&adap->dev, "Failed to enable bus regulator\n");
+- goto err_clear_wakeup_irq;
+- }
+- }
+-
+ status = of_clk_set_defaults(dev->of_node, false);
+ if (status < 0)
+ goto err_clear_wakeup_irq;
+@@ -604,10 +594,8 @@ put_sync_adapter:
+ static void i2c_device_remove(struct device *dev)
+ {
+ struct i2c_client *client = to_i2c_client(dev);
+- struct i2c_adapter *adap;
+ struct i2c_driver *driver;
+
+- adap = client->adapter;
+ driver = to_i2c_driver(dev->driver);
+ if (driver->remove) {
+ int status;
+@@ -622,8 +610,6 @@ static void i2c_device_remove(struct device *dev)
+ devres_release_group(&client->dev, client->devres_group_id);
+
+ dev_pm_domain_detach(&client->dev, true);
+- if (!pm_runtime_status_suspended(&client->dev) && adap->bus_regulator)
+- regulator_disable(adap->bus_regulator);
+
+ dev_pm_clear_wake_irq(&client->dev);
+ device_init_wakeup(&client->dev, false);
+@@ -633,86 +619,6 @@ static void i2c_device_remove(struct device *dev)
+ pm_runtime_put(&client->adapter->dev);
+ }
+
+-#ifdef CONFIG_PM_SLEEP
+-static int i2c_resume_early(struct device *dev)
+-{
+- struct i2c_client *client = i2c_verify_client(dev);
+- int err;
+-
+- if (!client)
+- return 0;
+-
+- if (pm_runtime_status_suspended(&client->dev) &&
+- client->adapter->bus_regulator) {
+- err = regulator_enable(client->adapter->bus_regulator);
+- if (err)
+- return err;
+- }
+-
+- return pm_generic_resume_early(&client->dev);
+-}
+-
+-static int i2c_suspend_late(struct device *dev)
+-{
+- struct i2c_client *client = i2c_verify_client(dev);
+- int err;
+-
+- if (!client)
+- return 0;
+-
+- err = pm_generic_suspend_late(&client->dev);
+- if (err)
+- return err;
+-
+- if (!pm_runtime_status_suspended(&client->dev) &&
+- client->adapter->bus_regulator)
+- return regulator_disable(client->adapter->bus_regulator);
+-
+- return 0;
+-}
+-#endif
+-
+-#ifdef CONFIG_PM
+-static int i2c_runtime_resume(struct device *dev)
+-{
+- struct i2c_client *client = i2c_verify_client(dev);
+- int err;
+-
+- if (!client)
+- return 0;
+-
+- if (client->adapter->bus_regulator) {
+- err = regulator_enable(client->adapter->bus_regulator);
+- if (err)
+- return err;
+- }
+-
+- return pm_generic_runtime_resume(&client->dev);
+-}
+-
+-static int i2c_runtime_suspend(struct device *dev)
+-{
+- struct i2c_client *client = i2c_verify_client(dev);
+- int err;
+-
+- if (!client)
+- return 0;
+-
+- err = pm_generic_runtime_suspend(&client->dev);
+- if (err)
+- return err;
+-
+- if (client->adapter->bus_regulator)
+- return regulator_disable(client->adapter->bus_regulator);
+- return 0;
+-}
+-#endif
+-
+-static const struct dev_pm_ops i2c_device_pm = {
+- SET_LATE_SYSTEM_SLEEP_PM_OPS(i2c_suspend_late, i2c_resume_early)
+- SET_RUNTIME_PM_OPS(i2c_runtime_suspend, i2c_runtime_resume, NULL)
+-};
+-
+ static void i2c_device_shutdown(struct device *dev)
+ {
+ struct i2c_client *client = i2c_verify_client(dev);
+@@ -772,7 +678,6 @@ struct bus_type i2c_bus_type = {
+ .probe = i2c_device_probe,
+ .remove = i2c_device_remove,
+ .shutdown = i2c_device_shutdown,
+- .pm = &i2c_device_pm,
+ };
+ EXPORT_SYMBOL_GPL(i2c_bus_type);
+
+diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c
+index bce0e8bb78520..6fd2b6718b086 100644
+--- a/drivers/i2c/i2c-dev.c
++++ b/drivers/i2c/i2c-dev.c
+@@ -535,6 +535,9 @@ static long compat_i2cdev_ioctl(struct file *file, unsigned int cmd, unsigned lo
+ sizeof(rdwr_arg)))
+ return -EFAULT;
+
++ if (!rdwr_arg.msgs || rdwr_arg.nmsgs == 0)
++ return -EINVAL;
++
+ if (rdwr_arg.nmsgs > I2C_RDWR_IOCTL_MAX_MSGS)
+ return -EINVAL;
+
+@@ -665,16 +668,21 @@ static int i2cdev_attach_adapter(struct device *dev, void *dummy)
+ i2c_dev->dev.class = i2c_dev_class;
+ i2c_dev->dev.parent = &adap->dev;
+ i2c_dev->dev.release = i2cdev_dev_release;
+- dev_set_name(&i2c_dev->dev, "i2c-%d", adap->nr);
++
++ res = dev_set_name(&i2c_dev->dev, "i2c-%d", adap->nr);
++ if (res)
++ goto err_put_i2c_dev;
+
+ res = cdev_device_add(&i2c_dev->cdev, &i2c_dev->dev);
+- if (res) {
+- put_i2c_dev(i2c_dev, false);
+- return res;
+- }
++ if (res)
++ goto err_put_i2c_dev;
+
+ pr_debug("adapter [%s] registered as minor %d\n", adap->name, adap->nr);
+ return 0;
++
++err_put_i2c_dev:
++ put_i2c_dev(i2c_dev, false);
++ return res;
+ }
+
+ static int i2cdev_detach_adapter(struct device *dev, void *dummy)
+diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
+index 5365199a31f41..f7a7405d4350a 100644
+--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c
++++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c
+@@ -261,7 +261,7 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
+
+ err = device_create_file(&pdev->dev, &dev_attr_available_masters);
+ if (err)
+- goto err_rollback;
++ goto err_rollback_activation;
+
+ err = device_create_file(&pdev->dev, &dev_attr_current_master);
+ if (err)
+@@ -271,8 +271,9 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
+
+ err_rollback_available:
+ device_remove_file(&pdev->dev, &dev_attr_available_masters);
+-err_rollback:
++err_rollback_activation:
+ i2c_demux_deactivate_master(priv);
++err_rollback:
+ for (j = 0; j < i; j++) {
+ of_node_put(priv->chan[j].parent_np);
+ of_changeset_destroy(&priv->chan[j].chgset);
+diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c
+index c3b4c677b4429..dfe18dcd008d4 100644
+--- a/drivers/i3c/master.c
++++ b/drivers/i3c/master.c
+@@ -343,7 +343,8 @@ struct bus_type i3c_bus_type = {
+ static enum i3c_addr_slot_status
+ i3c_bus_get_addr_slot_status(struct i3c_bus *bus, u16 addr)
+ {
+- int status, bitpos = addr * 2;
++ unsigned long status;
++ int bitpos = addr * 2;
+
+ if (addr > I2C_MAX_ADDR)
+ return I3C_ADDR_SLOT_RSVD;
+diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c
+index 03a368da51b95..51a8608203de7 100644
+--- a/drivers/i3c/master/dw-i3c-master.c
++++ b/drivers/i3c/master/dw-i3c-master.c
+@@ -793,6 +793,10 @@ static int dw_i3c_master_daa(struct i3c_master_controller *m)
+ return -ENOMEM;
+
+ pos = dw_i3c_master_get_free_pos(master);
++ if (pos < 0) {
++ dw_i3c_master_free_xfer(xfer);
++ return pos;
++ }
+ cmd = &xfer->cmds[0];
+ cmd->cmd_hi = 0x1;
+ cmd->cmd_lo = COMMAND_PORT_DEV_COUNT(master->maxdevs - pos) |
+diff --git a/drivers/i3c/master/mipi-i3c-hci/dat_v1.c b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
+index 783e551a2c85a..97bb49ff5b53b 100644
+--- a/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
++++ b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
+@@ -160,9 +160,7 @@ static int hci_dat_v1_get_index(struct i3c_hci *hci, u8 dev_addr)
+ unsigned int dat_idx;
+ u32 dat_w0;
+
+- for (dat_idx = find_first_bit(hci->DAT_data, hci->DAT_entries);
+- dat_idx < hci->DAT_entries;
+- dat_idx = find_next_bit(hci->DAT_data, hci->DAT_entries, dat_idx)) {
++ for_each_set_bit(dat_idx, hci->DAT_data, hci->DAT_entries) {
+ dat_w0 = dat_w0_read(dat_idx);
+ if (FIELD_GET(DAT_0_DYNAMIC_ADDRESS, dat_w0) == dev_addr)
+ return dat_idx;
+diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c
+index e8693a42ad464..3af763b4a9737 100644
+--- a/drivers/iio/accel/bmc150-accel-core.c
++++ b/drivers/iio/accel/bmc150-accel-core.c
+@@ -1782,11 +1782,14 @@ int bmc150_accel_core_probe(struct device *dev, struct regmap *regmap, int irq,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(dev, "Unable to register iio device\n");
+- goto err_trigger_unregister;
++ goto err_pm_cleanup;
+ }
+
+ return 0;
+
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(dev);
++ pm_runtime_disable(dev);
+ err_trigger_unregister:
+ bmc150_accel_unregister_triggers(data, BMC150_ACCEL_TRIGGERS - 1);
+ err_buffer_cleanup:
+diff --git a/drivers/iio/accel/fxls8962af-core.c b/drivers/iio/accel/fxls8962af-core.c
+index f41db9e0249a7..a2d29cabb3892 100644
+--- a/drivers/iio/accel/fxls8962af-core.c
++++ b/drivers/iio/accel/fxls8962af-core.c
+@@ -154,12 +154,20 @@ struct fxls8962af_data {
+ u8 watermark;
+ };
+
+-const struct regmap_config fxls8962af_regmap_conf = {
++const struct regmap_config fxls8962af_i2c_regmap_conf = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .max_register = FXLS8962AF_MAX_REG,
+ };
+-EXPORT_SYMBOL_GPL(fxls8962af_regmap_conf);
++EXPORT_SYMBOL_GPL(fxls8962af_i2c_regmap_conf);
++
++const struct regmap_config fxls8962af_spi_regmap_conf = {
++ .reg_bits = 8,
++ .pad_bits = 8,
++ .val_bits = 8,
++ .max_register = FXLS8962AF_MAX_REG,
++};
++EXPORT_SYMBOL_GPL(fxls8962af_spi_regmap_conf);
+
+ enum {
+ fxls8962af_idx_x,
+diff --git a/drivers/iio/accel/fxls8962af-i2c.c b/drivers/iio/accel/fxls8962af-i2c.c
+index cfb004b204559..6bde9891effbf 100644
+--- a/drivers/iio/accel/fxls8962af-i2c.c
++++ b/drivers/iio/accel/fxls8962af-i2c.c
+@@ -18,7 +18,7 @@ static int fxls8962af_probe(struct i2c_client *client)
+ {
+ struct regmap *regmap;
+
+- regmap = devm_regmap_init_i2c(client, &fxls8962af_regmap_conf);
++ regmap = devm_regmap_init_i2c(client, &fxls8962af_i2c_regmap_conf);
+ if (IS_ERR(regmap)) {
+ dev_err(&client->dev, "Failed to initialize i2c regmap\n");
+ return PTR_ERR(regmap);
+diff --git a/drivers/iio/accel/fxls8962af-spi.c b/drivers/iio/accel/fxls8962af-spi.c
+index 57108d3d480b6..6f4dff3238d3c 100644
+--- a/drivers/iio/accel/fxls8962af-spi.c
++++ b/drivers/iio/accel/fxls8962af-spi.c
+@@ -18,7 +18,7 @@ static int fxls8962af_probe(struct spi_device *spi)
+ {
+ struct regmap *regmap;
+
+- regmap = devm_regmap_init_spi(spi, &fxls8962af_regmap_conf);
++ regmap = devm_regmap_init_spi(spi, &fxls8962af_spi_regmap_conf);
+ if (IS_ERR(regmap)) {
+ dev_err(&spi->dev, "Failed to initialize spi regmap\n");
+ return PTR_ERR(regmap);
+diff --git a/drivers/iio/accel/fxls8962af.h b/drivers/iio/accel/fxls8962af.h
+index b67572c3ef069..9cbe98c3ba9a2 100644
+--- a/drivers/iio/accel/fxls8962af.h
++++ b/drivers/iio/accel/fxls8962af.h
+@@ -17,6 +17,7 @@ int fxls8962af_core_probe(struct device *dev, struct regmap *regmap, int irq);
+ int fxls8962af_core_remove(struct device *dev);
+
+ extern const struct dev_pm_ops fxls8962af_pm_ops;
+-extern const struct regmap_config fxls8962af_regmap_conf;
++extern const struct regmap_config fxls8962af_i2c_regmap_conf;
++extern const struct regmap_config fxls8962af_spi_regmap_conf;
+
+ #endif /* _FXLS8962AF_H_ */
+diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c
+index a51fdd3c9b5b5..ba6c8ca488b1a 100644
+--- a/drivers/iio/accel/kxcjk-1013.c
++++ b/drivers/iio/accel/kxcjk-1013.c
+@@ -1589,14 +1589,16 @@ static int kxcjk1013_probe(struct i2c_client *client,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(&client->dev, "unable to register iio device\n");
+- goto err_buffer_cleanup;
++ goto err_pm_cleanup;
+ }
+
+ return 0;
+
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(&client->dev);
++ pm_runtime_disable(&client->dev);
+ err_buffer_cleanup:
+- if (data->dready_trig)
+- iio_triggered_buffer_cleanup(indio_dev);
++ iio_triggered_buffer_cleanup(indio_dev);
+ err_trigger_unregister:
+ if (data->dready_trig)
+ iio_trigger_unregister(data->dready_trig);
+@@ -1618,8 +1620,8 @@ static int kxcjk1013_remove(struct i2c_client *client)
+ pm_runtime_disable(&client->dev);
+ pm_runtime_set_suspended(&client->dev);
+
++ iio_triggered_buffer_cleanup(indio_dev);
+ if (data->dready_trig) {
+- iio_triggered_buffer_cleanup(indio_dev);
+ iio_trigger_unregister(data->dready_trig);
+ iio_trigger_unregister(data->motion_trig);
+ }
+diff --git a/drivers/iio/accel/kxsd9.c b/drivers/iio/accel/kxsd9.c
+index bf7ed9e7d00f4..e56ecc075014e 100644
+--- a/drivers/iio/accel/kxsd9.c
++++ b/drivers/iio/accel/kxsd9.c
+@@ -224,14 +224,14 @@ static irqreturn_t kxsd9_trigger_handler(int irq, void *p)
+ hw_values.chan,
+ sizeof(hw_values.chan));
+ if (ret) {
+- dev_err(st->dev,
+- "error reading data\n");
+- return ret;
++ dev_err(st->dev, "error reading data: %d\n", ret);
++ goto out;
+ }
+
+ iio_push_to_buffers_with_timestamp(indio_dev,
+ &hw_values,
+ iio_get_time_ns(indio_dev));
++out:
+ iio_trigger_notify_done(indio_dev->trig);
+
+ return IRQ_HANDLED;
+diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
+index 715b8138fb715..21a99467f3646 100644
+--- a/drivers/iio/accel/mma8452.c
++++ b/drivers/iio/accel/mma8452.c
+@@ -176,6 +176,7 @@ static const struct mma8452_event_regs trans_ev_regs = {
+ * @enabled_events: event flags enabled and handled by this driver
+ */
+ struct mma_chip_info {
++ const char *name;
+ u8 chip_id;
+ const struct iio_chan_spec *channels;
+ int num_channels;
+@@ -1301,6 +1302,7 @@ enum {
+
+ static const struct mma_chip_info mma_chip_info_table[] = {
+ [mma8451] = {
++ .name = "mma8451",
+ .chip_id = MMA8451_DEVICE_ID,
+ .channels = mma8451_channels,
+ .num_channels = ARRAY_SIZE(mma8451_channels),
+@@ -1325,6 +1327,7 @@ static const struct mma_chip_info mma_chip_info_table[] = {
+ MMA8452_INT_FF_MT,
+ },
+ [mma8452] = {
++ .name = "mma8452",
+ .chip_id = MMA8452_DEVICE_ID,
+ .channels = mma8452_channels,
+ .num_channels = ARRAY_SIZE(mma8452_channels),
+@@ -1341,6 +1344,7 @@ static const struct mma_chip_info mma_chip_info_table[] = {
+ MMA8452_INT_FF_MT,
+ },
+ [mma8453] = {
++ .name = "mma8453",
+ .chip_id = MMA8453_DEVICE_ID,
+ .channels = mma8453_channels,
+ .num_channels = ARRAY_SIZE(mma8453_channels),
+@@ -1357,6 +1361,7 @@ static const struct mma_chip_info mma_chip_info_table[] = {
+ MMA8452_INT_FF_MT,
+ },
+ [mma8652] = {
++ .name = "mma8652",
+ .chip_id = MMA8652_DEVICE_ID,
+ .channels = mma8652_channels,
+ .num_channels = ARRAY_SIZE(mma8652_channels),
+@@ -1366,6 +1371,7 @@ static const struct mma_chip_info mma_chip_info_table[] = {
+ .enabled_events = MMA8452_INT_FF_MT,
+ },
+ [mma8653] = {
++ .name = "mma8653",
+ .chip_id = MMA8653_DEVICE_ID,
+ .channels = mma8653_channels,
+ .num_channels = ARRAY_SIZE(mma8653_channels),
+@@ -1380,6 +1386,7 @@ static const struct mma_chip_info mma_chip_info_table[] = {
+ .enabled_events = MMA8452_INT_FF_MT,
+ },
+ [fxls8471] = {
++ .name = "fxls8471",
+ .chip_id = FXLS8471_DEVICE_ID,
+ .channels = mma8451_channels,
+ .num_channels = ARRAY_SIZE(mma8451_channels),
+@@ -1470,7 +1477,7 @@ static int mma8452_trigger_setup(struct iio_dev *indio_dev)
+ if (ret)
+ return ret;
+
+- indio_dev->trig = trig;
++ indio_dev->trig = iio_trigger_get(trig);
+
+ return 0;
+ }
+@@ -1522,13 +1529,6 @@ static int mma8452_probe(struct i2c_client *client,
+ struct mma8452_data *data;
+ struct iio_dev *indio_dev;
+ int ret;
+- const struct of_device_id *match;
+-
+- match = of_match_device(mma8452_dt_ids, &client->dev);
+- if (!match) {
+- dev_err(&client->dev, "unknown device model\n");
+- return -ENODEV;
+- }
+
+ indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+ if (!indio_dev)
+@@ -1537,7 +1537,14 @@ static int mma8452_probe(struct i2c_client *client,
+ data = iio_priv(indio_dev);
+ data->client = client;
+ mutex_init(&data->lock);
+- data->chip_info = match->data;
++
++ data->chip_info = device_get_match_data(&client->dev);
++ if (!data->chip_info && id) {
++ data->chip_info = &mma_chip_info_table[id->driver_data];
++ } else {
++ dev_err(&client->dev, "unknown device model\n");
++ return -ENODEV;
++ }
+
+ data->vdd_reg = devm_regulator_get(&client->dev, "vdd");
+ if (IS_ERR(data->vdd_reg))
+@@ -1581,11 +1588,11 @@ static int mma8452_probe(struct i2c_client *client,
+ }
+
+ dev_info(&client->dev, "registering %s accelerometer; ID 0x%x\n",
+- match->compatible, data->chip_info->chip_id);
++ data->chip_info->name, data->chip_info->chip_id);
+
+ i2c_set_clientdata(client, indio_dev);
+ indio_dev->info = &mma8452_info;
+- indio_dev->name = id->name;
++ indio_dev->name = data->chip_info->name;
+ indio_dev->modes = INDIO_DIRECT_MODE;
+ indio_dev->channels = data->chip_info->channels;
+ indio_dev->num_channels = data->chip_info->num_channels;
+@@ -1810,7 +1817,7 @@ MODULE_DEVICE_TABLE(i2c, mma8452_id);
+ static struct i2c_driver mma8452_driver = {
+ .driver = {
+ .name = "mma8452",
+- .of_match_table = of_match_ptr(mma8452_dt_ids),
++ .of_match_table = mma8452_dt_ids,
+ .pm = &mma8452_pm_ops,
+ },
+ .probe = mma8452_probe,
+diff --git a/drivers/iio/accel/mma9551.c b/drivers/iio/accel/mma9551.c
+index 4c359fb054801..c53a3398b14c4 100644
+--- a/drivers/iio/accel/mma9551.c
++++ b/drivers/iio/accel/mma9551.c
+@@ -495,11 +495,14 @@ static int mma9551_probe(struct i2c_client *client,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(&client->dev, "unable to register iio device\n");
+- goto out_poweroff;
++ goto err_pm_cleanup;
+ }
+
+ return 0;
+
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(&client->dev);
++ pm_runtime_disable(&client->dev);
+ out_poweroff:
+ mma9551_set_device_state(client, false);
+
+diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c
+index ba3ecb3b57dcd..1599b75724d4f 100644
+--- a/drivers/iio/accel/mma9553.c
++++ b/drivers/iio/accel/mma9553.c
+@@ -1134,12 +1134,15 @@ static int mma9553_probe(struct i2c_client *client,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(&client->dev, "unable to register iio device\n");
+- goto out_poweroff;
++ goto err_pm_cleanup;
+ }
+
+ dev_dbg(&indio_dev->dev, "Registered device %s\n", name);
+ return 0;
+
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(&client->dev);
++ pm_runtime_disable(&client->dev);
+ out_poweroff:
+ mma9551_set_device_state(client, false);
+ return ret;
+diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
+index f711756e41e3d..cba57459e90ab 100644
+--- a/drivers/iio/accel/st_accel_i2c.c
++++ b/drivers/iio/accel/st_accel_i2c.c
+@@ -193,10 +193,10 @@ static int st_accel_i2c_remove(struct i2c_client *client)
+ {
+ struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_accel_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c
+index bb45d9ff95b85..5167fae1ee8ec 100644
+--- a/drivers/iio/accel/st_accel_spi.c
++++ b/drivers/iio/accel/st_accel_spi.c
+@@ -143,10 +143,10 @@ static int st_accel_spi_remove(struct spi_device *spi)
+ {
+ struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_accel_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/adc/ad7124.c b/drivers/iio/adc/ad7124.c
+index e45c600fccc0b..18c154afbd7ac 100644
+--- a/drivers/iio/adc/ad7124.c
++++ b/drivers/iio/adc/ad7124.c
+@@ -76,7 +76,7 @@
+ #define AD7124_CONFIG_REF_SEL(x) FIELD_PREP(AD7124_CONFIG_REF_SEL_MSK, x)
+ #define AD7124_CONFIG_PGA_MSK GENMASK(2, 0)
+ #define AD7124_CONFIG_PGA(x) FIELD_PREP(AD7124_CONFIG_PGA_MSK, x)
+-#define AD7124_CONFIG_IN_BUFF_MSK GENMASK(7, 6)
++#define AD7124_CONFIG_IN_BUFF_MSK GENMASK(6, 5)
+ #define AD7124_CONFIG_IN_BUFF(x) FIELD_PREP(AD7124_CONFIG_IN_BUFF_MSK, x)
+
+ /* AD7124_FILTER_X */
+diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c
+index 2c5c8a3672b2d..aa42ba759fa1a 100644
+--- a/drivers/iio/adc/ad7768-1.c
++++ b/drivers/iio/adc/ad7768-1.c
+@@ -480,8 +480,8 @@ static irqreturn_t ad7768_trigger_handler(int irq, void *p)
+ iio_push_to_buffers_with_timestamp(indio_dev, &st->data.scan,
+ iio_get_time_ns(indio_dev));
+
+- iio_trigger_notify_done(indio_dev->trig);
+ err_unlock:
++ iio_trigger_notify_done(indio_dev->trig);
+ mutex_unlock(&st->lock);
+
+ return IRQ_HANDLED;
+diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
+index ea5ca163d8796..c4de706012e51 100644
+--- a/drivers/iio/adc/at91-sama5d2_adc.c
++++ b/drivers/iio/adc/at91-sama5d2_adc.c
+@@ -1377,7 +1377,8 @@ static int at91_adc_read_info_raw(struct iio_dev *indio_dev,
+ *val = st->conversion_value;
+ ret = at91_adc_adjust_val_osr(st, val);
+ if (chan->scan_type.sign == 's')
+- *val = sign_extend32(*val, 11);
++ *val = sign_extend32(*val,
++ chan->scan_type.realbits - 1);
+ st->conversion_done = false;
+ }
+
+diff --git a/drivers/iio/adc/axp20x_adc.c b/drivers/iio/adc/axp20x_adc.c
+index 3e0c0233b4315..df99f1365c398 100644
+--- a/drivers/iio/adc/axp20x_adc.c
++++ b/drivers/iio/adc/axp20x_adc.c
+@@ -251,19 +251,8 @@ static int axp22x_adc_raw(struct iio_dev *indio_dev,
+ struct iio_chan_spec const *chan, int *val)
+ {
+ struct axp20x_adc_iio *info = iio_priv(indio_dev);
+- int size;
+
+- /*
+- * N.B.: Unlike the Chinese datasheets tell, the charging current is
+- * stored on 12 bits, not 13 bits. Only discharging current is on 13
+- * bits.
+- */
+- if (chan->type == IIO_CURRENT && chan->channel == AXP22X_BATT_DISCHRG_I)
+- size = 13;
+- else
+- size = 12;
+-
+- *val = axp20x_read_variable_width(info->regmap, chan->address, size);
++ *val = axp20x_read_variable_width(info->regmap, chan->address, 12);
+ if (*val < 0)
+ return *val;
+
+@@ -386,9 +375,8 @@ static int axp22x_adc_scale(struct iio_chan_spec const *chan, int *val,
+ return IIO_VAL_INT_PLUS_MICRO;
+
+ case IIO_CURRENT:
+- *val = 0;
+- *val2 = 500000;
+- return IIO_VAL_INT_PLUS_MICRO;
++ *val = 1;
++ return IIO_VAL_INT;
+
+ case IIO_TEMP:
+ *val = 100;
+diff --git a/drivers/iio/adc/dln2-adc.c b/drivers/iio/adc/dln2-adc.c
+index 16407664182ce..97d162a3cba4e 100644
+--- a/drivers/iio/adc/dln2-adc.c
++++ b/drivers/iio/adc/dln2-adc.c
+@@ -248,7 +248,6 @@ static int dln2_adc_set_chan_period(struct dln2_adc *dln2,
+ static int dln2_adc_read(struct dln2_adc *dln2, unsigned int channel)
+ {
+ int ret, i;
+- struct iio_dev *indio_dev = platform_get_drvdata(dln2->pdev);
+ u16 conflict;
+ __le16 value;
+ int olen = sizeof(value);
+@@ -257,13 +256,9 @@ static int dln2_adc_read(struct dln2_adc *dln2, unsigned int channel)
+ .chan = channel,
+ };
+
+- ret = iio_device_claim_direct_mode(indio_dev);
+- if (ret < 0)
+- return ret;
+-
+ ret = dln2_adc_set_chan_enabled(dln2, channel, true);
+ if (ret < 0)
+- goto release_direct;
++ return ret;
+
+ ret = dln2_adc_set_port_enabled(dln2, true, &conflict);
+ if (ret < 0) {
+@@ -300,8 +295,6 @@ disable_port:
+ dln2_adc_set_port_enabled(dln2, false, NULL);
+ disable_chan:
+ dln2_adc_set_chan_enabled(dln2, channel, false);
+-release_direct:
+- iio_device_release_direct_mode(indio_dev);
+
+ return ret;
+ }
+@@ -337,10 +330,16 @@ static int dln2_adc_read_raw(struct iio_dev *indio_dev,
+
+ switch (mask) {
+ case IIO_CHAN_INFO_RAW:
++ ret = iio_device_claim_direct_mode(indio_dev);
++ if (ret < 0)
++ return ret;
++
+ mutex_lock(&dln2->mutex);
+ ret = dln2_adc_read(dln2, chan->channel);
+ mutex_unlock(&dln2->mutex);
+
++ iio_device_release_direct_mode(indio_dev);
++
+ if (ret < 0)
+ return ret;
+
+@@ -656,7 +655,11 @@ static int dln2_adc_probe(struct platform_device *pdev)
+ return -ENOMEM;
+ }
+ iio_trigger_set_drvdata(dln2->trig, dln2);
+- devm_iio_trigger_register(dev, dln2->trig);
++ ret = devm_iio_trigger_register(dev, dln2->trig);
++ if (ret) {
++ dev_err(dev, "failed to register trigger: %d\n", ret);
++ return ret;
++ }
+ iio_trigger_set_immutable(indio_dev, dln2->trig);
+
+ ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL,
+diff --git a/drivers/iio/adc/men_z188_adc.c b/drivers/iio/adc/men_z188_adc.c
+index 42ea8bc7e7805..adc5ceaef8c93 100644
+--- a/drivers/iio/adc/men_z188_adc.c
++++ b/drivers/iio/adc/men_z188_adc.c
+@@ -103,6 +103,7 @@ static int men_z188_probe(struct mcb_device *dev,
+ struct z188_adc *adc;
+ struct iio_dev *indio_dev;
+ struct resource *mem;
++ int ret;
+
+ indio_dev = devm_iio_device_alloc(&dev->dev, sizeof(struct z188_adc));
+ if (!indio_dev)
+@@ -128,8 +129,14 @@ static int men_z188_probe(struct mcb_device *dev,
+ adc->mem = mem;
+ mcb_set_drvdata(dev, indio_dev);
+
+- return iio_device_register(indio_dev);
++ ret = iio_device_register(indio_dev);
++ if (ret)
++ goto err_unmap;
++
++ return 0;
+
++err_unmap:
++ iounmap(adc->base);
+ err:
+ mcb_release_mem(mem);
+ return -ENXIO;
+diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
+index 5088de835bb15..e3e75413b49e7 100644
+--- a/drivers/iio/adc/stm32-adc.c
++++ b/drivers/iio/adc/stm32-adc.c
+@@ -975,6 +975,7 @@ static void stm32h7_adc_unprepare(struct iio_dev *indio_dev)
+ {
+ struct stm32_adc *adc = iio_priv(indio_dev);
+
++ stm32_adc_writel(adc, STM32H7_ADC_PCSEL, 0);
+ stm32h7_adc_disable(indio_dev);
+ stm32h7_adc_enter_pwr_down(adc);
+ }
+diff --git a/drivers/iio/adc/ti-adc081c.c b/drivers/iio/adc/ti-adc081c.c
+index 16fc608db36a5..bd48b073e7200 100644
+--- a/drivers/iio/adc/ti-adc081c.c
++++ b/drivers/iio/adc/ti-adc081c.c
+@@ -19,6 +19,7 @@
+ #include <linux/i2c.h>
+ #include <linux/module.h>
+ #include <linux/mod_devicetable.h>
++#include <linux/property.h>
+
+ #include <linux/iio/iio.h>
+ #include <linux/iio/buffer.h>
+@@ -156,13 +157,16 @@ static int adc081c_probe(struct i2c_client *client,
+ {
+ struct iio_dev *iio;
+ struct adc081c *adc;
+- struct adcxx1c_model *model;
++ const struct adcxx1c_model *model;
+ int err;
+
+ if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA))
+ return -EOPNOTSUPP;
+
+- model = &adcxx1c_models[id->driver_data];
++ if (dev_fwnode(&client->dev))
++ model = device_get_match_data(&client->dev);
++ else
++ model = &adcxx1c_models[id->driver_data];
+
+ iio = devm_iio_device_alloc(&client->dev, sizeof(*adc));
+ if (!iio)
+@@ -210,10 +214,17 @@ static const struct i2c_device_id adc081c_id[] = {
+ };
+ MODULE_DEVICE_TABLE(i2c, adc081c_id);
+
++static const struct acpi_device_id adc081c_acpi_match[] = {
++ /* Used on some AAEON boards */
++ { "ADC081C", (kernel_ulong_t)&adcxx1c_models[ADC081C] },
++ { }
++};
++MODULE_DEVICE_TABLE(acpi, adc081c_acpi_match);
++
+ static const struct of_device_id adc081c_of_match[] = {
+- { .compatible = "ti,adc081c" },
+- { .compatible = "ti,adc101c" },
+- { .compatible = "ti,adc121c" },
++ { .compatible = "ti,adc081c", .data = &adcxx1c_models[ADC081C] },
++ { .compatible = "ti,adc101c", .data = &adcxx1c_models[ADC101C] },
++ { .compatible = "ti,adc121c", .data = &adcxx1c_models[ADC121C] },
+ { }
+ };
+ MODULE_DEVICE_TABLE(of, adc081c_of_match);
+@@ -222,6 +233,7 @@ static struct i2c_driver adc081c_driver = {
+ .driver = {
+ .name = "adc081c",
+ .of_match_table = adc081c_of_match,
++ .acpi_match_table = adc081c_acpi_match,
+ },
+ .probe = adc081c_probe,
+ .id_table = adc081c_id,
+diff --git a/drivers/iio/adc/ti-tsc2046.c b/drivers/iio/adc/ti-tsc2046.c
+index 170950d5dd499..e8fc4d01f30b6 100644
+--- a/drivers/iio/adc/ti-tsc2046.c
++++ b/drivers/iio/adc/ti-tsc2046.c
+@@ -388,7 +388,7 @@ static int tsc2046_adc_update_scan_mode(struct iio_dev *indio_dev,
+ mutex_lock(&priv->slock);
+
+ size = 0;
+- for_each_set_bit(ch_idx, active_scan_mask, indio_dev->num_channels) {
++ for_each_set_bit(ch_idx, active_scan_mask, ARRAY_SIZE(priv->l)) {
+ size += tsc2046_adc_group_set_layout(priv, group, ch_idx);
+ tsc2046_adc_group_set_cmd(priv, group, ch_idx);
+ group++;
+@@ -398,7 +398,7 @@ static int tsc2046_adc_update_scan_mode(struct iio_dev *indio_dev,
+ priv->xfer.len = size;
+ priv->time_per_scan_us = size * 8 * priv->time_per_bit_ns / NSEC_PER_USEC;
+
+- if (priv->scan_interval_us > priv->time_per_scan_us)
++ if (priv->scan_interval_us < priv->time_per_scan_us)
+ dev_warn(&priv->spi->dev, "The scan interval (%d) is less then calculated scan time (%d)\n",
+ priv->scan_interval_us, priv->time_per_scan_us);
+
+@@ -548,7 +548,7 @@ static int tsc2046_adc_setup_spi_msg(struct tsc2046_adc_priv *priv)
+ * enabled.
+ */
+ size = 0;
+- for (ch_idx = 0; ch_idx < priv->dcfg->num_channels; ch_idx++)
++ for (ch_idx = 0; ch_idx < ARRAY_SIZE(priv->l); ch_idx++)
+ size += tsc2046_adc_group_set_layout(priv, ch_idx, ch_idx);
+
+ priv->tx = devm_kzalloc(&priv->spi->dev, size, GFP_KERNEL);
+diff --git a/drivers/iio/adc/twl6030-gpadc.c b/drivers/iio/adc/twl6030-gpadc.c
+index c6416ad795ca4..256177b15c511 100644
+--- a/drivers/iio/adc/twl6030-gpadc.c
++++ b/drivers/iio/adc/twl6030-gpadc.c
+@@ -911,6 +911,8 @@ static int twl6030_gpadc_probe(struct platform_device *pdev)
+ ret = devm_request_threaded_irq(dev, irq, NULL,
+ twl6030_gpadc_irq_handler,
+ IRQF_ONESHOT, "twl6030_gpadc", indio_dev);
++ if (ret)
++ return ret;
+
+ ret = twl6030_gpadc_enable_irq(TWL6030_GPADC_RT_SW1_EOC_MASK);
+ if (ret < 0) {
+diff --git a/drivers/iio/afe/iio-rescale.c b/drivers/iio/afe/iio-rescale.c
+index 774eb3044edd8..271d73e420c42 100644
+--- a/drivers/iio/afe/iio-rescale.c
++++ b/drivers/iio/afe/iio-rescale.c
+@@ -39,7 +39,7 @@ static int rescale_read_raw(struct iio_dev *indio_dev,
+ int *val, int *val2, long mask)
+ {
+ struct rescale *rescale = iio_priv(indio_dev);
+- unsigned long long tmp;
++ s64 tmp;
+ int ret;
+
+ switch (mask) {
+@@ -77,10 +77,10 @@ static int rescale_read_raw(struct iio_dev *indio_dev,
+ *val2 = rescale->denominator;
+ return IIO_VAL_FRACTIONAL;
+ case IIO_VAL_FRACTIONAL_LOG2:
+- tmp = *val * 1000000000LL;
+- do_div(tmp, rescale->denominator);
++ tmp = (s64)*val * 1000000000LL;
++ tmp = div_s64(tmp, rescale->denominator);
+ tmp *= rescale->numerator;
+- do_div(tmp, 1000000000LL);
++ tmp = div_s64(tmp, 1000000000LL);
+ *val = tmp;
+ return ret;
+ default:
+diff --git a/drivers/iio/dac/ad5446.c b/drivers/iio/dac/ad5446.c
+index 488ec69967d67..cafb8c7790154 100644
+--- a/drivers/iio/dac/ad5446.c
++++ b/drivers/iio/dac/ad5446.c
+@@ -178,7 +178,7 @@ static int ad5446_read_raw(struct iio_dev *indio_dev,
+
+ switch (m) {
+ case IIO_CHAN_INFO_RAW:
+- *val = st->cached_val;
++ *val = st->cached_val >> chan->scan_type.shift;
+ return IIO_VAL_INT;
+ case IIO_CHAN_INFO_SCALE:
+ *val = st->vref_mv;
+@@ -531,8 +531,15 @@ static int ad5622_write(struct ad5446_state *st, unsigned val)
+ {
+ struct i2c_client *client = to_i2c_client(st->dev);
+ __be16 data = cpu_to_be16(val);
++ int ret;
++
++ ret = i2c_master_send(client, (char *)&data, sizeof(data));
++ if (ret < 0)
++ return ret;
++ if (ret != sizeof(data))
++ return -EIO;
+
+- return i2c_master_send(client, (char *)&data, sizeof(data));
++ return 0;
+ }
+
+ /*
+diff --git a/drivers/iio/dac/ad5592r-base.c b/drivers/iio/dac/ad5592r-base.c
+index 0405e92b9e8c3..987264410278c 100644
+--- a/drivers/iio/dac/ad5592r-base.c
++++ b/drivers/iio/dac/ad5592r-base.c
+@@ -523,7 +523,7 @@ static int ad5592r_alloc_channels(struct iio_dev *iio_dev)
+ if (!ret)
+ st->channel_modes[reg] = tmp;
+
+- fwnode_property_read_u32(child, "adi,off-state", &tmp);
++ ret = fwnode_property_read_u32(child, "adi,off-state", &tmp);
+ if (!ret)
+ st->channel_offstate[reg] = tmp;
+ }
+diff --git a/drivers/iio/dac/ad5766.c b/drivers/iio/dac/ad5766.c
+index 3104ec32dfaca..dafda84fdea35 100644
+--- a/drivers/iio/dac/ad5766.c
++++ b/drivers/iio/dac/ad5766.c
+@@ -503,13 +503,13 @@ static int ad5766_get_output_range(struct ad5766_state *st)
+ int i, ret, min, max, tmp[2];
+
+ ret = device_property_read_u32_array(&st->spi->dev,
+- "output-range-voltage",
++ "output-range-microvolts",
+ tmp, 2);
+ if (ret)
+ return ret;
+
+- min = tmp[0] / 1000;
+- max = tmp[1] / 1000;
++ min = tmp[0] / 1000000;
++ max = tmp[1] / 1000000;
+ for (i = 0; i < ARRAY_SIZE(ad5766_span_tbl); i++) {
+ if (ad5766_span_tbl[i].min != min ||
+ ad5766_span_tbl[i].max != max)
+diff --git a/drivers/iio/dac/ad5770r.c b/drivers/iio/dac/ad5770r.c
+index 8107f7bbbe3c5..7e2fd32e993a6 100644
+--- a/drivers/iio/dac/ad5770r.c
++++ b/drivers/iio/dac/ad5770r.c
+@@ -522,7 +522,7 @@ static int ad5770r_channel_config(struct ad5770r_state *st)
+ return -EINVAL;
+
+ device_for_each_child_node(&st->spi->dev, child) {
+- ret = fwnode_property_read_u32(child, "num", &num);
++ ret = fwnode_property_read_u32(child, "reg", &num);
+ if (ret)
+ goto err_child_out;
+ if (num >= AD5770R_MAX_CHANNELS) {
+diff --git a/drivers/iio/gyro/adxrs290.c b/drivers/iio/gyro/adxrs290.c
+index 3e0734ddafe36..600e9725da788 100644
+--- a/drivers/iio/gyro/adxrs290.c
++++ b/drivers/iio/gyro/adxrs290.c
+@@ -7,6 +7,7 @@
+ */
+
+ #include <linux/bitfield.h>
++#include <linux/bitops.h>
+ #include <linux/delay.h>
+ #include <linux/device.h>
+ #include <linux/kernel.h>
+@@ -124,7 +125,7 @@ static int adxrs290_get_rate_data(struct iio_dev *indio_dev, const u8 cmd, int *
+ goto err_unlock;
+ }
+
+- *val = temp;
++ *val = sign_extend32(temp, 15);
+
+ err_unlock:
+ mutex_unlock(&st->lock);
+@@ -146,7 +147,7 @@ static int adxrs290_get_temp_data(struct iio_dev *indio_dev, int *val)
+ }
+
+ /* extract lower 12 bits temperature reading */
+- *val = temp & 0x0FFF;
++ *val = sign_extend32(temp, 11);
+
+ err_unlock:
+ mutex_unlock(&st->lock);
+diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c
+index 17b939a367ad0..81a6d09788bd7 100644
+--- a/drivers/iio/gyro/bmg160_core.c
++++ b/drivers/iio/gyro/bmg160_core.c
+@@ -1188,11 +1188,14 @@ int bmg160_core_probe(struct device *dev, struct regmap *regmap, int irq,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(dev, "unable to register iio device\n");
+- goto err_buffer_cleanup;
++ goto err_pm_cleanup;
+ }
+
+ return 0;
+
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(dev);
++ pm_runtime_disable(dev);
+ err_buffer_cleanup:
+ iio_triggered_buffer_cleanup(indio_dev);
+ err_trigger_unregister:
+diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c
+index 04dd6a7969ea7..4cfa0d4395605 100644
+--- a/drivers/iio/gyro/itg3200_buffer.c
++++ b/drivers/iio/gyro/itg3200_buffer.c
+@@ -61,9 +61,9 @@ static irqreturn_t itg3200_trigger_handler(int irq, void *p)
+
+ iio_push_to_buffers_with_timestamp(indio_dev, &scan, pf->timestamp);
+
++error_ret:
+ iio_trigger_notify_done(indio_dev->trig);
+
+-error_ret:
+ return IRQ_HANDLED;
+ }
+
+diff --git a/drivers/iio/gyro/st_gyro_i2c.c b/drivers/iio/gyro/st_gyro_i2c.c
+index 3ef86e16ee656..a8164fe48b857 100644
+--- a/drivers/iio/gyro/st_gyro_i2c.c
++++ b/drivers/iio/gyro/st_gyro_i2c.c
+@@ -106,10 +106,10 @@ static int st_gyro_i2c_remove(struct i2c_client *client)
+ {
+ struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_gyro_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/gyro/st_gyro_spi.c b/drivers/iio/gyro/st_gyro_spi.c
+index 41d835493347c..9d8916871b4bf 100644
+--- a/drivers/iio/gyro/st_gyro_spi.c
++++ b/drivers/iio/gyro/st_gyro_spi.c
+@@ -110,10 +110,10 @@ static int st_gyro_spi_remove(struct spi_device *spi)
+ {
+ struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_gyro_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/imu/adis.c b/drivers/iio/imu/adis.c
+index b9a06ca29beec..d4e692b187cda 100644
+--- a/drivers/iio/imu/adis.c
++++ b/drivers/iio/imu/adis.c
+@@ -430,6 +430,8 @@ int __adis_initial_startup(struct adis *adis)
+ if (ret)
+ return ret;
+
++ adis_enable_irq(adis, false);
++
+ if (!adis->data->prod_id_reg)
+ return 0;
+
+@@ -526,7 +528,7 @@ int adis_init(struct adis *adis, struct iio_dev *indio_dev,
+ adis->current_page = 0;
+ }
+
+- return adis_enable_irq(adis, false);
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(adis_init);
+
+diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
+index ed129321a14da..f9b4540db1f43 100644
+--- a/drivers/iio/imu/adis16480.c
++++ b/drivers/iio/imu/adis16480.c
+@@ -1403,6 +1403,7 @@ static int adis16480_probe(struct spi_device *spi)
+ {
+ const struct spi_device_id *id = spi_get_device_id(spi);
+ const struct adis_data *adis16480_data;
++ irq_handler_t trigger_handler = NULL;
+ struct iio_dev *indio_dev;
+ struct adis16480 *st;
+ int ret;
+@@ -1474,8 +1475,12 @@ static int adis16480_probe(struct spi_device *spi)
+ st->clk_freq = st->chip_info->int_clk;
+ }
+
++ /* Only use our trigger handler if burst mode is supported */
++ if (adis16480_data->burst_len)
++ trigger_handler = adis16480_trigger_handler;
++
+ ret = devm_adis_setup_buffer_and_trigger(&st->adis, indio_dev,
+- adis16480_trigger_handler);
++ trigger_handler);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c
+index 824b5124a5f55..01336105792ee 100644
+--- a/drivers/iio/imu/bmi160/bmi160_core.c
++++ b/drivers/iio/imu/bmi160/bmi160_core.c
+@@ -730,7 +730,7 @@ static int bmi160_chip_init(struct bmi160_data *data, bool use_spi)
+
+ ret = regmap_write(data->regmap, BMI160_REG_CMD, BMI160_CMD_SOFTRESET);
+ if (ret)
+- return ret;
++ goto disable_regulator;
+
+ usleep_range(BMI160_SOFTRESET_USLEEP, BMI160_SOFTRESET_USLEEP + 1);
+
+@@ -741,29 +741,37 @@ static int bmi160_chip_init(struct bmi160_data *data, bool use_spi)
+ if (use_spi) {
+ ret = regmap_read(data->regmap, BMI160_REG_DUMMY, &val);
+ if (ret)
+- return ret;
++ goto disable_regulator;
+ }
+
+ ret = regmap_read(data->regmap, BMI160_REG_CHIP_ID, &val);
+ if (ret) {
+ dev_err(dev, "Error reading chip id\n");
+- return ret;
++ goto disable_regulator;
+ }
+ if (val != BMI160_CHIP_ID_VAL) {
+ dev_err(dev, "Wrong chip id, got %x expected %x\n",
+ val, BMI160_CHIP_ID_VAL);
+- return -ENODEV;
++ ret = -ENODEV;
++ goto disable_regulator;
+ }
+
+ ret = bmi160_set_mode(data, BMI160_ACCEL, true);
+ if (ret)
+- return ret;
++ goto disable_regulator;
+
+ ret = bmi160_set_mode(data, BMI160_GYRO, true);
+ if (ret)
+- return ret;
++ goto disable_accel;
+
+ return 0;
++
++disable_accel:
++ bmi160_set_mode(data, BMI160_ACCEL, false);
++
++disable_regulator:
++ regulator_bulk_disable(ARRAY_SIZE(data->supplies), data->supplies);
++ return ret;
+ }
+
+ static int bmi160_data_rdy_trigger_set_state(struct iio_trigger *trig,
+diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
+index 85b1934cec60e..53891010a91de 100644
+--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
++++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
+@@ -18,12 +18,15 @@ static int inv_icm42600_i2c_bus_setup(struct inv_icm42600_state *st)
+ unsigned int mask, val;
+ int ret;
+
+- /* setup interface registers */
+- ret = regmap_update_bits(st->map, INV_ICM42600_REG_INTF_CONFIG6,
+- INV_ICM42600_INTF_CONFIG6_MASK,
+- INV_ICM42600_INTF_CONFIG6_I3C_EN);
+- if (ret)
+- return ret;
++ /*
++ * setup interface registers
++ * This register write to REG_INTF_CONFIG6 enables a spike filter that
++ * is impacting the line and can prevent the I2C ACK to be seen by the
++ * controller. So we don't test the return value.
++ */
++ regmap_update_bits(st->map, INV_ICM42600_REG_INTF_CONFIG6,
++ INV_ICM42600_INTF_CONFIG6_MASK,
++ INV_ICM42600_INTF_CONFIG6_I3C_EN);
+
+ ret = regmap_update_bits(st->map, INV_ICM42600_REG_INTF_CONFIG4,
+ INV_ICM42600_INTF_CONFIG4_I3C_BUS_ONLY, 0);
+diff --git a/drivers/iio/imu/kmx61.c b/drivers/iio/imu/kmx61.c
+index 1dabfd615dabf..f89724481df93 100644
+--- a/drivers/iio/imu/kmx61.c
++++ b/drivers/iio/imu/kmx61.c
+@@ -1385,7 +1385,7 @@ static int kmx61_probe(struct i2c_client *client,
+ ret = iio_device_register(data->acc_indio_dev);
+ if (ret < 0) {
+ dev_err(&client->dev, "Failed to register acc iio device\n");
+- goto err_buffer_cleanup_mag;
++ goto err_pm_cleanup;
+ }
+
+ ret = iio_device_register(data->mag_indio_dev);
+@@ -1398,6 +1398,9 @@ static int kmx61_probe(struct i2c_client *client,
+
+ err_iio_unregister_acc:
+ iio_device_unregister(data->acc_indio_dev);
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(&client->dev);
++ pm_runtime_disable(&client->dev);
+ err_buffer_cleanup_mag:
+ if (client->irq > 0)
+ iio_triggered_buffer_cleanup(data->mag_indio_dev);
+diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+index db45f1fc0b817..a778aceba3b10 100644
+--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
++++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+@@ -1279,6 +1279,8 @@ st_lsm6dsx_set_odr(struct st_lsm6dsx_sensor *sensor, u32 req_odr)
+ int err;
+
+ switch (sensor->id) {
++ case ST_LSM6DSX_ID_GYRO:
++ break;
+ case ST_LSM6DSX_ID_EXT0:
+ case ST_LSM6DSX_ID_EXT1:
+ case ST_LSM6DSX_ID_EXT2:
+@@ -1304,8 +1306,8 @@ st_lsm6dsx_set_odr(struct st_lsm6dsx_sensor *sensor, u32 req_odr)
+ }
+ break;
+ }
+- default:
+- break;
++ default: /* should never occur */
++ return -EINVAL;
+ }
+
+ if (req_odr > 0) {
+@@ -1370,8 +1372,12 @@ static int st_lsm6dsx_read_oneshot(struct st_lsm6dsx_sensor *sensor,
+ if (err < 0)
+ return err;
+
++ /*
++ * we need to wait for sensor settling time before
++ * reading data in order to avoid corrupted samples
++ */
+ delay = 1000000000 / sensor->odr;
+- usleep_range(delay, 2 * delay);
++ usleep_range(3 * delay, 4 * delay);
+
+ err = st_lsm6dsx_read_locked(hw, addr, &data, sizeof(data));
+ if (err < 0)
+diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
+index a95cc2da56be6..c81dbd2f09727 100644
+--- a/drivers/iio/industrialio-buffer.c
++++ b/drivers/iio/industrialio-buffer.c
+@@ -1312,6 +1312,11 @@ static struct attribute *iio_buffer_wrap_attr(struct iio_buffer *buffer,
+ iio_attr->buffer = buffer;
+ memcpy(&iio_attr->dev_attr, dattr, sizeof(iio_attr->dev_attr));
+ iio_attr->dev_attr.attr.name = kstrdup_const(attr->name, GFP_KERNEL);
++ if (!iio_attr->dev_attr.attr.name) {
++ kfree(iio_attr);
++ return NULL;
++ }
++
+ sysfs_attr_init(&iio_attr->dev_attr.attr);
+
+ list_add(&iio_attr->l, &buffer->buffer_attr_list);
+@@ -1362,10 +1367,10 @@ static int iio_buffer_register_legacy_sysfs_groups(struct iio_dev *indio_dev,
+
+ return 0;
+
+-error_free_buffer_attrs:
+- kfree(iio_dev_opaque->legacy_buffer_group.attrs);
+ error_free_scan_el_attrs:
+ kfree(iio_dev_opaque->legacy_scan_el_group.attrs);
++error_free_buffer_attrs:
++ kfree(iio_dev_opaque->legacy_buffer_group.attrs);
+
+ return ret;
+ }
+@@ -1441,9 +1446,17 @@ static long iio_device_buffer_getfd(struct iio_dev *indio_dev, unsigned long arg
+ }
+
+ if (copy_to_user(ival, &fd, sizeof(fd))) {
+- put_unused_fd(fd);
+- ret = -EFAULT;
+- goto error_free_ib;
++ /*
++ * "Leak" the fd, as there's not much we can do about this
++ * anyway. 'fd' might have been closed already, as
++ * anon_inode_getfd() called fd_install() on it, which made
++ * it reachable by userland.
++ *
++ * Instead of allowing a malicious user to play tricks with
++ * us, rely on the process exit path to do any necessary
++ * cleanup, as in releasing the file, if still needed.
++ */
++ return -EFAULT;
+ }
+
+ return 0;
+@@ -1531,6 +1544,7 @@ static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,
+ sizeof(struct attribute *) * buffer_attrcount);
+
+ buffer_attrcount += ARRAY_SIZE(iio_buffer_attrs);
++ buffer->buffer_group.attrs = attr;
+
+ for (i = 0; i < buffer_attrcount; i++) {
+ struct attribute *wrapped;
+@@ -1538,7 +1552,7 @@ static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,
+ wrapped = iio_buffer_wrap_attr(buffer, attr[i]);
+ if (!wrapped) {
+ ret = -ENOMEM;
+- goto error_free_scan_mask;
++ goto error_free_buffer_attrs;
+ }
+ attr[i] = wrapped;
+ }
+@@ -1553,8 +1567,6 @@ static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,
+ goto error_free_buffer_attrs;
+ }
+
+- buffer->buffer_group.attrs = attr;
+-
+ ret = iio_device_register_sysfs_group(indio_dev, &buffer->buffer_group);
+ if (ret)
+ goto error_free_buffer_attr_group_name;
+@@ -1583,8 +1595,12 @@ error_cleanup_dynamic:
+ return ret;
+ }
+
+-static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer)
++static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer,
++ struct iio_dev *indio_dev,
++ int index)
+ {
++ if (index == 0)
++ iio_buffer_unregister_legacy_sysfs_groups(indio_dev);
+ bitmap_free(buffer->scan_mask);
+ kfree(buffer->buffer_group.name);
+ kfree(buffer->buffer_group.attrs);
+@@ -1616,7 +1632,7 @@ int iio_buffers_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
+ buffer = iio_dev_opaque->attached_buffers[i];
+ ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, i);
+ if (ret) {
+- unwind_idx = i;
++ unwind_idx = i - 1;
+ goto error_unwind_sysfs_and_mask;
+ }
+ }
+@@ -1638,7 +1654,7 @@ int iio_buffers_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
+ error_unwind_sysfs_and_mask:
+ for (; unwind_idx >= 0; unwind_idx--) {
+ buffer = iio_dev_opaque->attached_buffers[unwind_idx];
+- __iio_buffer_free_sysfs_and_mask(buffer);
++ __iio_buffer_free_sysfs_and_mask(buffer, indio_dev, unwind_idx);
+ }
+ return ret;
+ }
+@@ -1655,11 +1671,9 @@ void iio_buffers_free_sysfs_and_mask(struct iio_dev *indio_dev)
+ iio_device_ioctl_handler_unregister(iio_dev_opaque->buffer_ioctl_handler);
+ kfree(iio_dev_opaque->buffer_ioctl_handler);
+
+- iio_buffer_unregister_legacy_sysfs_groups(indio_dev);
+-
+ for (i = iio_dev_opaque->attached_buffers_cnt - 1; i >= 0; i--) {
+ buffer = iio_dev_opaque->attached_buffers[i];
+- __iio_buffer_free_sysfs_and_mask(buffer);
++ __iio_buffer_free_sysfs_and_mask(buffer, indio_dev, i);
+ }
+ }
+
+diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
+index 2dbb37e09b8cf..3e1e86d987cc2 100644
+--- a/drivers/iio/industrialio-core.c
++++ b/drivers/iio/industrialio-core.c
+@@ -1600,6 +1600,7 @@ static void iio_device_unregister_sysfs(struct iio_dev *indio_dev)
+ kfree(iio_dev_opaque->chan_attr_group.attrs);
+ iio_dev_opaque->chan_attr_group.attrs = NULL;
+ kfree(iio_dev_opaque->groups);
++ iio_dev_opaque->groups = NULL;
+ }
+
+ static void iio_dev_release(struct device *device)
+@@ -1664,7 +1665,13 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv)
+ kfree(iio_dev_opaque);
+ return NULL;
+ }
+- dev_set_name(&indio_dev->dev, "iio:device%d", iio_dev_opaque->id);
++
++ if (dev_set_name(&indio_dev->dev, "iio:device%d", iio_dev_opaque->id)) {
++ ida_simple_remove(&iio_ida, iio_dev_opaque->id);
++ kfree(iio_dev_opaque);
++ return NULL;
++ }
++
+ INIT_LIST_HEAD(&iio_dev_opaque->buffer_list);
+ INIT_LIST_HEAD(&iio_dev_opaque->ioctl_handlers);
+
+diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
+index b23caa2f2aa1f..f504ed351b3e2 100644
+--- a/drivers/iio/industrialio-trigger.c
++++ b/drivers/iio/industrialio-trigger.c
+@@ -162,6 +162,39 @@ static struct iio_trigger *iio_trigger_acquire_by_name(const char *name)
+ return trig;
+ }
+
++static void iio_reenable_work_fn(struct work_struct *work)
++{
++ struct iio_trigger *trig = container_of(work, struct iio_trigger,
++ reenable_work);
++
++ /*
++ * This 'might' occur after the trigger state is set to disabled -
++ * in that case the driver should skip reenabling.
++ */
++ trig->ops->reenable(trig);
++}
++
++/*
++ * In general, reenable callbacks may need to sleep and this path is
++ * not performance sensitive, so just queue up a work item
++ * to reneable the trigger for us.
++ *
++ * Races that can cause this.
++ * 1) A handler occurs entirely in interrupt context so the counter
++ * the final decrement is still in this interrupt.
++ * 2) The trigger has been removed, but one last interrupt gets through.
++ *
++ * For (1) we must call reenable, but not in atomic context.
++ * For (2) it should be safe to call reenanble, if drivers never blindly
++ * reenable after state is off.
++ */
++static void iio_trigger_notify_done_atomic(struct iio_trigger *trig)
++{
++ if (atomic_dec_and_test(&trig->use_count) && trig->ops &&
++ trig->ops->reenable)
++ schedule_work(&trig->reenable_work);
++}
++
+ void iio_trigger_poll(struct iio_trigger *trig)
+ {
+ int i;
+@@ -173,7 +206,7 @@ void iio_trigger_poll(struct iio_trigger *trig)
+ if (trig->subirqs[i].enabled)
+ generic_handle_irq(trig->subirq_base + i);
+ else
+- iio_trigger_notify_done(trig);
++ iio_trigger_notify_done_atomic(trig);
+ }
+ }
+ }
+@@ -535,6 +568,7 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent,
+ trig->dev.type = &iio_trig_type;
+ trig->dev.bus = &iio_bus_type;
+ device_initialize(&trig->dev);
++ INIT_WORK(&trig->reenable_work, iio_reenable_work_fn);
+
+ mutex_init(&trig->pool_lock);
+ trig->subirq_base = irq_alloc_descs(-1, 0,
+@@ -556,7 +590,6 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent,
+ irq_modify_status(trig->subirq_base + i,
+ IRQ_NOREQUEST | IRQ_NOAUTOEN, IRQ_NOPROBE);
+ }
+- get_device(&trig->dev);
+
+ return trig;
+
+diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
+index 391a3380a1d10..b5966365d769c 100644
+--- a/drivers/iio/inkern.c
++++ b/drivers/iio/inkern.c
+@@ -578,28 +578,50 @@ EXPORT_SYMBOL_GPL(iio_read_channel_average_raw);
+ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan,
+ int raw, int *processed, unsigned int scale)
+ {
+- int scale_type, scale_val, scale_val2, offset;
++ int scale_type, scale_val, scale_val2;
++ int offset_type, offset_val, offset_val2;
+ s64 raw64 = raw;
+- int ret;
+
+- ret = iio_channel_read(chan, &offset, NULL, IIO_CHAN_INFO_OFFSET);
+- if (ret >= 0)
+- raw64 += offset;
++ offset_type = iio_channel_read(chan, &offset_val, &offset_val2,
++ IIO_CHAN_INFO_OFFSET);
++ if (offset_type >= 0) {
++ switch (offset_type) {
++ case IIO_VAL_INT:
++ break;
++ case IIO_VAL_INT_PLUS_MICRO:
++ case IIO_VAL_INT_PLUS_NANO:
++ /*
++ * Both IIO_VAL_INT_PLUS_MICRO and IIO_VAL_INT_PLUS_NANO
++ * implicitely truncate the offset to it's integer form.
++ */
++ break;
++ case IIO_VAL_FRACTIONAL:
++ offset_val /= offset_val2;
++ break;
++ case IIO_VAL_FRACTIONAL_LOG2:
++ offset_val >>= offset_val2;
++ break;
++ default:
++ return -EINVAL;
++ }
++
++ raw64 += offset_val;
++ }
+
+ scale_type = iio_channel_read(chan, &scale_val, &scale_val2,
+ IIO_CHAN_INFO_SCALE);
+ if (scale_type < 0) {
+ /*
+- * Just pass raw values as processed if no scaling is
+- * available.
++ * If no channel scaling is available apply consumer scale to
++ * raw value and return.
+ */
+- *processed = raw;
++ *processed = raw * scale;
+ return 0;
+ }
+
+ switch (scale_type) {
+ case IIO_VAL_INT:
+- *processed = raw64 * scale_val;
++ *processed = raw64 * scale_val * scale;
+ break;
+ case IIO_VAL_INT_PLUS_MICRO:
+ if (scale_val2 < 0)
+diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
+index 1830221da48d2..f0bd0ad34f222 100644
+--- a/drivers/iio/light/ltr501.c
++++ b/drivers/iio/light/ltr501.c
+@@ -1273,7 +1273,7 @@ static irqreturn_t ltr501_trigger_handler(int irq, void *p)
+ ret = regmap_bulk_read(data->regmap, LTR501_ALS_DATA1,
+ als_buf, sizeof(als_buf));
+ if (ret < 0)
+- return ret;
++ goto done;
+ if (test_bit(0, indio_dev->active_scan_mask))
+ scan.channels[j++] = le16_to_cpu(als_buf[1]);
+ if (test_bit(1, indio_dev->active_scan_mask))
+diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c
+index 07e91846307c7..fc63856ed54de 100644
+--- a/drivers/iio/light/stk3310.c
++++ b/drivers/iio/light/stk3310.c
+@@ -546,9 +546,8 @@ static irqreturn_t stk3310_irq_event_handler(int irq, void *private)
+ mutex_lock(&data->lock);
+ ret = regmap_field_read(data->reg_flag_nf, &dir);
+ if (ret < 0) {
+- dev_err(&data->client->dev, "register read failed\n");
+- mutex_unlock(&data->lock);
+- return ret;
++ dev_err(&data->client->dev, "register read failed: %d\n", ret);
++ goto out;
+ }
+ event = IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 1,
+ IIO_EV_TYPE_THRESH,
+@@ -560,6 +559,7 @@ static irqreturn_t stk3310_irq_event_handler(int irq, void *private)
+ ret = regmap_field_write(data->reg_flag_psint, 0);
+ if (ret < 0)
+ dev_err(&data->client->dev, "failed to reset interrupts\n");
++out:
+ mutex_unlock(&data->lock);
+
+ return IRQ_HANDLED;
+diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
+index 42b8a2680e3aa..1509fd0cbb50f 100644
+--- a/drivers/iio/magnetometer/ak8975.c
++++ b/drivers/iio/magnetometer/ak8975.c
+@@ -389,6 +389,7 @@ static int ak8975_power_on(const struct ak8975_data *data)
+ if (ret) {
+ dev_warn(&data->client->dev,
+ "Failed to enable specified Vid supply\n");
++ regulator_disable(data->vdd);
+ return ret;
+ }
+
+diff --git a/drivers/iio/magnetometer/bmc150_magn.c b/drivers/iio/magnetometer/bmc150_magn.c
+index f96f531753495..3d4d21f979fab 100644
+--- a/drivers/iio/magnetometer/bmc150_magn.c
++++ b/drivers/iio/magnetometer/bmc150_magn.c
+@@ -962,13 +962,14 @@ int bmc150_magn_probe(struct device *dev, struct regmap *regmap,
+ ret = iio_device_register(indio_dev);
+ if (ret < 0) {
+ dev_err(dev, "unable to register iio device\n");
+- goto err_disable_runtime_pm;
++ goto err_pm_cleanup;
+ }
+
+ dev_dbg(dev, "Registered device %s\n", name);
+ return 0;
+
+-err_disable_runtime_pm:
++err_pm_cleanup:
++ pm_runtime_dont_use_autosuspend(dev);
+ pm_runtime_disable(dev);
+ err_buffer_cleanup:
+ iio_triggered_buffer_cleanup(indio_dev);
+diff --git a/drivers/iio/magnetometer/st_magn_i2c.c b/drivers/iio/magnetometer/st_magn_i2c.c
+index 2dfe4ee99591b..fa78f0a3b53ea 100644
+--- a/drivers/iio/magnetometer/st_magn_i2c.c
++++ b/drivers/iio/magnetometer/st_magn_i2c.c
+@@ -102,10 +102,10 @@ static int st_magn_i2c_remove(struct i2c_client *client)
+ {
+ struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_magn_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/magnetometer/st_magn_spi.c b/drivers/iio/magnetometer/st_magn_spi.c
+index fba9787963952..ff43cbf61b056 100644
+--- a/drivers/iio/magnetometer/st_magn_spi.c
++++ b/drivers/iio/magnetometer/st_magn_spi.c
+@@ -96,10 +96,10 @@ static int st_magn_spi_remove(struct spi_device *spi)
+ {
+ struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_magn_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/pressure/st_pressure_i2c.c b/drivers/iio/pressure/st_pressure_i2c.c
+index 52fa98f24478d..6215de677017e 100644
+--- a/drivers/iio/pressure/st_pressure_i2c.c
++++ b/drivers/iio/pressure/st_pressure_i2c.c
+@@ -119,10 +119,10 @@ static int st_press_i2c_remove(struct i2c_client *client)
+ {
+ struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_press_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/iio/pressure/st_pressure_spi.c b/drivers/iio/pressure/st_pressure_spi.c
+index ee393df54cee8..e220cf0b125f1 100644
+--- a/drivers/iio/pressure/st_pressure_spi.c
++++ b/drivers/iio/pressure/st_pressure_spi.c
+@@ -102,10 +102,10 @@ static int st_press_spi_remove(struct spi_device *spi)
+ {
+ struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+- st_sensors_power_disable(indio_dev);
+-
+ st_press_common_remove(indio_dev);
+
++ st_sensors_power_disable(indio_dev);
++
+ return 0;
+ }
+
+@@ -117,6 +117,10 @@ static const struct spi_device_id st_press_id_table[] = {
+ { LPS33HW_PRESS_DEV_NAME },
+ { LPS35HW_PRESS_DEV_NAME },
+ { LPS22HH_PRESS_DEV_NAME },
++ { "lps001wp-press" },
++ { "lps25h-press", },
++ { "lps331ap-press" },
++ { "lps22hb-press" },
+ {},
+ };
+ MODULE_DEVICE_TABLE(spi, st_press_id_table);
+diff --git a/drivers/iio/trigger/stm32-timer-trigger.c b/drivers/iio/trigger/stm32-timer-trigger.c
+index 33083877cd19d..4353b749ecef2 100644
+--- a/drivers/iio/trigger/stm32-timer-trigger.c
++++ b/drivers/iio/trigger/stm32-timer-trigger.c
+@@ -912,6 +912,6 @@ static struct platform_driver stm32_timer_trigger_driver = {
+ };
+ module_platform_driver(stm32_timer_trigger_driver);
+
+-MODULE_ALIAS("platform: stm32-timer-trigger");
++MODULE_ALIAS("platform:stm32-timer-trigger");
+ MODULE_DESCRIPTION("STMicroelectronics STM32 Timer Trigger driver");
+ MODULE_LICENSE("GPL v2");
+diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
+index c903b74f46a46..1c107d6d03b99 100644
+--- a/drivers/infiniband/core/cm.c
++++ b/drivers/infiniband/core/cm.c
+@@ -2824,6 +2824,7 @@ static int cm_dreq_handler(struct cm_work *work)
+ switch (cm_id_priv->id.state) {
+ case IB_CM_REP_SENT:
+ case IB_CM_DREQ_SENT:
++ case IB_CM_MRA_REP_RCVD:
+ ib_cancel_mad(cm_id_priv->msg);
+ break;
+ case IB_CM_ESTABLISHED:
+@@ -2831,8 +2832,6 @@ static int cm_dreq_handler(struct cm_work *work)
+ cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
+ ib_cancel_mad(cm_id_priv->msg);
+ break;
+- case IB_CM_MRA_REP_RCVD:
+- break;
+ case IB_CM_TIMEWAIT:
+ atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
+ [CM_DREQ_COUNTER]);
+@@ -3322,7 +3321,7 @@ static int cm_lap_handler(struct cm_work *work)
+ ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
+ if (ret) {
+ rdma_destroy_ah_attr(&ah_attr);
+- return -EINVAL;
++ goto deref;
+ }
+
+ spin_lock_irq(&cm_id_priv->lock);
+diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
+index 704ce595542c5..a814dabcdff43 100644
+--- a/drivers/infiniband/core/cma.c
++++ b/drivers/infiniband/core/cma.c
+@@ -67,8 +67,8 @@ static const char * const cma_events[] = {
+ [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit",
+ };
+
+-static void cma_set_mgid(struct rdma_id_private *id_priv, struct sockaddr *addr,
+- union ib_gid *mgid);
++static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
++ enum ib_gid_type gid_type);
+
+ const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
+ {
+@@ -766,6 +766,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
+ unsigned int p;
+ u16 pkey, index;
+ enum ib_port_state port_state;
++ int ret;
+ int i;
+
+ cma_dev = NULL;
+@@ -784,9 +785,14 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
+
+ if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
+ continue;
+- for (i = 0; !rdma_query_gid(cur_dev->device,
+- p, i, &gid);
+- i++) {
++
++ for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len;
++ ++i) {
++ ret = rdma_query_gid(cur_dev->device, p, i,
++ &gid);
++ if (ret)
++ continue;
++
+ if (!memcmp(&gid, dgid, sizeof(gid))) {
+ cma_dev = cur_dev;
+ sgid = gid;
+@@ -1838,17 +1844,19 @@ static void destroy_mc(struct rdma_id_private *id_priv,
+ if (dev_addr->bound_dev_if)
+ ndev = dev_get_by_index(dev_addr->net,
+ dev_addr->bound_dev_if);
+- if (ndev) {
++ if (ndev && !send_only) {
++ enum ib_gid_type gid_type;
+ union ib_gid mgid;
+
+- cma_set_mgid(id_priv, (struct sockaddr *)&mc->addr,
+- &mgid);
+-
+- if (!send_only)
+- cma_igmp_send(ndev, &mgid, false);
+-
+- dev_put(ndev);
++ gid_type = id_priv->cma_dev->default_gid_type
++ [id_priv->id.port_num -
++ rdma_start_port(
++ id_priv->cma_dev->device)];
++ cma_iboe_set_mgid((struct sockaddr *)&mc->addr, &mgid,
++ gid_type);
++ cma_igmp_send(ndev, &mgid, false);
+ }
++ dev_put(ndev);
+
+ cancel_work_sync(&mc->iboe_join.work);
+ }
+@@ -2632,7 +2640,7 @@ int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
+ {
+ struct rdma_id_private *id_priv;
+
+- if (id->qp_type != IB_QPT_RC)
++ if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI)
+ return -EINVAL;
+
+ id_priv = container_of(id, struct rdma_id_private, id);
+@@ -3360,22 +3368,30 @@ err:
+ static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ const struct sockaddr *dst_addr)
+ {
+- if (!src_addr || !src_addr->sa_family) {
+- src_addr = (struct sockaddr *) &id->route.addr.src_addr;
+- src_addr->sa_family = dst_addr->sa_family;
+- if (IS_ENABLED(CONFIG_IPV6) &&
+- dst_addr->sa_family == AF_INET6) {
+- struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr;
+- struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr;
+- src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
+- if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
+- id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id;
+- } else if (dst_addr->sa_family == AF_IB) {
+- ((struct sockaddr_ib *) src_addr)->sib_pkey =
+- ((struct sockaddr_ib *) dst_addr)->sib_pkey;
+- }
++ struct sockaddr_storage zero_sock = {};
++
++ if (src_addr && src_addr->sa_family)
++ return rdma_bind_addr(id, src_addr);
++
++ /*
++ * When the src_addr is not specified, automatically supply an any addr
++ */
++ zero_sock.ss_family = dst_addr->sa_family;
++ if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
++ struct sockaddr_in6 *src_addr6 =
++ (struct sockaddr_in6 *)&zero_sock;
++ struct sockaddr_in6 *dst_addr6 =
++ (struct sockaddr_in6 *)dst_addr;
++
++ src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
++ if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
++ id->route.addr.dev_addr.bound_dev_if =
++ dst_addr6->sin6_scope_id;
++ } else if (dst_addr->sa_family == AF_IB) {
++ ((struct sockaddr_ib *)&zero_sock)->sib_pkey =
++ ((struct sockaddr_ib *)dst_addr)->sib_pkey;
+ }
+- return rdma_bind_addr(id, src_addr);
++ return rdma_bind_addr(id, (struct sockaddr *)&zero_sock);
+ }
+
+ /*
+@@ -4031,8 +4047,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
+
+ memset(&req, 0, sizeof req);
+ offset = cma_user_data_offset(id_priv);
+- req.private_data_len = offset + conn_param->private_data_len;
+- if (req.private_data_len < conn_param->private_data_len)
++ if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
+ return -EINVAL;
+
+ if (req.private_data_len) {
+@@ -4091,8 +4106,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv,
+
+ memset(&req, 0, sizeof req);
+ offset = cma_user_data_offset(id_priv);
+- req.private_data_len = offset + conn_param->private_data_len;
+- if (req.private_data_len < conn_param->private_data_len)
++ if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
+ return -EINVAL;
+
+ if (req.private_data_len) {
+diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
+index f4814bb7f082f..6ab46648af909 100644
+--- a/drivers/infiniband/core/device.c
++++ b/drivers/infiniband/core/device.c
+@@ -2461,7 +2461,8 @@ int ib_find_gid(struct ib_device *device, union ib_gid *gid,
+ ++i) {
+ ret = rdma_query_gid(device, port, i, &tmp_gid);
+ if (ret)
+- return ret;
++ continue;
++
+ if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
+ *port_num = port;
+ if (index)
+diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
+index 6146c3c1cbe5c..8d709986b88c7 100644
+--- a/drivers/infiniband/core/sysfs.c
++++ b/drivers/infiniband/core/sysfs.c
+@@ -757,7 +757,7 @@ static void ib_port_release(struct kobject *kobj)
+ if (port->hw_stats_data)
+ kfree(port->hw_stats_data->stats);
+ kfree(port->hw_stats_data);
+- kfree(port);
++ kvfree(port);
+ }
+
+ static void ib_port_gid_attr_release(struct kobject *kobj)
+@@ -1189,7 +1189,7 @@ static struct ib_port *setup_port(struct ib_core_device *coredev, int port_num,
+ struct ib_port *p;
+ int ret;
+
+- p = kzalloc(struct_size(p, attrs_list,
++ p = kvzalloc(struct_size(p, attrs_list,
+ attr->gid_tbl_len + attr->pkey_tbl_len),
+ GFP_KERNEL);
+ if (!p)
+diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
+index 2b72c4fa95506..9d6ac9dff39a2 100644
+--- a/drivers/infiniband/core/ucma.c
++++ b/drivers/infiniband/core/ucma.c
+@@ -95,6 +95,7 @@ struct ucma_context {
+ u64 uid;
+
+ struct list_head list;
++ struct list_head mc_list;
+ struct work_struct close_work;
+ };
+
+@@ -105,6 +106,7 @@ struct ucma_multicast {
+
+ u64 uid;
+ u8 join_state;
++ struct list_head list;
+ struct sockaddr_storage addr;
+ };
+
+@@ -198,6 +200,7 @@ static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)
+
+ INIT_WORK(&ctx->close_work, ucma_close_id);
+ init_completion(&ctx->comp);
++ INIT_LIST_HEAD(&ctx->mc_list);
+ /* So list_del() will work if we don't do ucma_finish_ctx() */
+ INIT_LIST_HEAD(&ctx->list);
+ ctx->file = file;
+@@ -484,19 +487,19 @@ err1:
+
+ static void ucma_cleanup_multicast(struct ucma_context *ctx)
+ {
+- struct ucma_multicast *mc;
+- unsigned long index;
++ struct ucma_multicast *mc, *tmp;
+
+- xa_for_each(&multicast_table, index, mc) {
+- if (mc->ctx != ctx)
+- continue;
++ xa_lock(&multicast_table);
++ list_for_each_entry_safe(mc, tmp, &ctx->mc_list, list) {
++ list_del(&mc->list);
+ /*
+ * At this point mc->ctx->ref is 0 so the mc cannot leave the
+ * lock on the reader and this is enough serialization
+ */
+- xa_erase(&multicast_table, index);
++ __xa_erase(&multicast_table, mc->id);
+ kfree(mc);
+ }
++ xa_unlock(&multicast_table);
+ }
+
+ static void ucma_cleanup_mc_events(struct ucma_multicast *mc)
+@@ -1469,12 +1472,16 @@ static ssize_t ucma_process_join(struct ucma_file *file,
+ mc->uid = cmd->uid;
+ memcpy(&mc->addr, addr, cmd->addr_size);
+
+- if (xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b,
++ xa_lock(&multicast_table);
++ if (__xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b,
+ GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto err_free_mc;
+ }
+
++ list_add_tail(&mc->list, &ctx->mc_list);
++ xa_unlock(&multicast_table);
++
+ mutex_lock(&ctx->mutex);
+ ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr,
+ join_state, mc);
+@@ -1500,8 +1507,11 @@ err_leave_multicast:
+ mutex_unlock(&ctx->mutex);
+ ucma_cleanup_mc_events(mc);
+ err_xa_erase:
+- xa_erase(&multicast_table, mc->id);
++ xa_lock(&multicast_table);
++ list_del(&mc->list);
++ __xa_erase(&multicast_table, mc->id);
+ err_free_mc:
++ xa_unlock(&multicast_table);
+ kfree(mc);
+ err_put_ctx:
+ ucma_put_ctx(ctx);
+@@ -1569,15 +1579,17 @@ static ssize_t ucma_leave_multicast(struct ucma_file *file,
+ mc = ERR_PTR(-EINVAL);
+ else if (!refcount_inc_not_zero(&mc->ctx->ref))
+ mc = ERR_PTR(-ENXIO);
+- else
+- __xa_erase(&multicast_table, mc->id);
+- xa_unlock(&multicast_table);
+
+ if (IS_ERR(mc)) {
++ xa_unlock(&multicast_table);
+ ret = PTR_ERR(mc);
+ goto out;
+ }
+
++ list_del(&mc->list);
++ __xa_erase(&multicast_table, mc->id);
++ xa_unlock(&multicast_table);
++
+ mutex_lock(&mc->ctx->mutex);
+ rdma_leave_multicast(mc->ctx->cm_id, (struct sockaddr *) &mc->addr);
+ mutex_unlock(&mc->ctx->mutex);
+diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
+index 740e6b2efe0e7..d1345d76d9b12 100644
+--- a/drivers/infiniband/core/uverbs_cmd.c
++++ b/drivers/infiniband/core/uverbs_cmd.c
+@@ -837,11 +837,8 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
+ new_mr->device = new_pd->device;
+ new_mr->pd = new_pd;
+ new_mr->type = IB_MR_TYPE_USER;
+- new_mr->dm = NULL;
+- new_mr->sig_attrs = NULL;
+ new_mr->uobject = uobj;
+ atomic_inc(&new_pd->usecnt);
+- new_mr->iova = cmd.hca_va;
+ new_uobj->object = new_mr;
+
+ rdma_restrack_new(&new_mr->res, RDMA_RESTRACK_MR);
+diff --git a/drivers/infiniband/core/uverbs_marshall.c b/drivers/infiniband/core/uverbs_marshall.c
+index b8d715c68ca44..11a0806469162 100644
+--- a/drivers/infiniband/core/uverbs_marshall.c
++++ b/drivers/infiniband/core/uverbs_marshall.c
+@@ -66,7 +66,7 @@ void ib_copy_ah_attr_to_user(struct ib_device *device,
+ struct rdma_ah_attr *src = ah_attr;
+ struct rdma_ah_attr conv_ah;
+
+- memset(&dst->grh.reserved, 0, sizeof(dst->grh.reserved));
++ memset(&dst->grh, 0, sizeof(dst->grh));
+
+ if ((ah_attr->type == RDMA_AH_ATTR_TYPE_OPA) &&
+ (rdma_ah_get_dlid(ah_attr) > be16_to_cpu(IB_LID_PERMISSIVE)) &&
+diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
+index 2f2c7646fce17..a02916a3a79ce 100644
+--- a/drivers/infiniband/core/uverbs_uapi.c
++++ b/drivers/infiniband/core/uverbs_uapi.c
+@@ -447,6 +447,9 @@ static int uapi_finalize(struct uverbs_api *uapi)
+ uapi->num_write_ex = max_write_ex + 1;
+ data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
+ sizeof(*uapi->write_methods), GFP_KERNEL);
++ if (!data)
++ return -ENOMEM;
++
+ for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
+ data[i] = &uapi->notsupp_method;
+ uapi->write_methods = data;
+diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
+index 89a2b21976d63..59e20936b8007 100644
+--- a/drivers/infiniband/core/verbs.c
++++ b/drivers/infiniband/core/verbs.c
+@@ -1232,6 +1232,9 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
+ INIT_LIST_HEAD(&qp->rdma_mrs);
+ INIT_LIST_HEAD(&qp->sig_mrs);
+
++ qp->send_cq = attr->send_cq;
++ qp->recv_cq = attr->recv_cq;
++
+ rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
+ WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
+ rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
+@@ -2150,6 +2153,7 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ return mr;
+
+ mr->device = pd->device;
++ mr->type = IB_MR_TYPE_USER;
+ mr->pd = pd;
+ mr->dm = NULL;
+ atomic_inc(&pd->usecnt);
+diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+index 408dfbcc47b5e..b7ec3a3926785 100644
+--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
++++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+@@ -3354,8 +3354,11 @@ static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
+ struct ib_wc *wc,
+ struct bnxt_qplib_cqe *cqe)
+ {
++ struct bnxt_re_dev *rdev;
++ u16 vlan_id = 0;
+ u8 nw_type;
+
++ rdev = qp->rdev;
+ wc->opcode = IB_WC_RECV;
+ wc->status = __rc_to_ib_wc_status(cqe->status);
+
+@@ -3367,9 +3370,12 @@ static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
+ memcpy(wc->smac, cqe->smac, ETH_ALEN);
+ wc->wc_flags |= IB_WC_WITH_SMAC;
+ if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
+- wc->vlan_id = (cqe->cfa_meta & 0xFFF);
+- if (wc->vlan_id < 0x1000)
+- wc->wc_flags |= IB_WC_WITH_VLAN;
++ vlan_id = (cqe->cfa_meta & 0xFFF);
++ }
++ /* Mark only if vlan_id is non zero */
++ if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
++ wc->vlan_id = vlan_id;
++ wc->wc_flags |= IB_WC_WITH_VLAN;
+ }
+ nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
+ CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
+diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+index d4d4959c2434c..bd153aa7e9ab3 100644
+--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
++++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+@@ -707,12 +707,13 @@ int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+ int rc = 0;
+
+ RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
+- req.srq_cid = cpu_to_le32(srq->id);
+
+ /* Configure the request */
+ sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
+ if (!sbuf)
+ return -ENOMEM;
++ req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
++ req.srq_cid = cpu_to_le32(srq->id);
+ sb = sbuf->sb;
+ rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
+ (void *)sbuf, 0);
+diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+index 5d384def5e5fe..d2d39126f1852 100644
+--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
++++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+@@ -618,8 +618,6 @@ int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res,
+ if (!cmdq->cmdq_bitmap)
+ goto fail;
+
+- cmdq->bmap_size = bmap_size;
+-
+ /* Allocate one extra to hold the QP1 entries */
+ rcfw->qp_tbl_size = qp_tbl_sz + 1;
+ rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),
+@@ -667,8 +665,8 @@ void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
+ iounmap(cmdq->cmdq_mbox.reg.bar_reg);
+ iounmap(creq->creq_db.reg.bar_reg);
+
+- indx = find_first_bit(cmdq->cmdq_bitmap, cmdq->bmap_size);
+- if (indx != cmdq->bmap_size)
++ indx = find_first_bit(cmdq->cmdq_bitmap, rcfw->cmdq_depth);
++ if (indx != rcfw->cmdq_depth)
+ dev_err(&rcfw->pdev->dev,
+ "disabling RCFW with pending cmd-bit %lx\n", indx);
+
+diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+index 9474c00465821..0c6d0b70ce890 100644
+--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
++++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+@@ -152,7 +152,6 @@ struct bnxt_qplib_cmdq_ctx {
+ wait_queue_head_t waitq;
+ unsigned long flags;
+ unsigned long *cmdq_bitmap;
+- u32 bmap_size;
+ u32 seq_num;
+ };
+
+diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
+index d20b4ef2c853d..ffbd9a89981e7 100644
+--- a/drivers/infiniband/hw/cxgb4/qp.c
++++ b/drivers/infiniband/hw/cxgb4/qp.c
+@@ -2460,6 +2460,7 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+ memset(attr, 0, sizeof(*attr));
+ memset(init_attr, 0, sizeof(*init_attr));
+ attr->qp_state = to_ib_qp_state(qhp->attr.state);
++ attr->cur_qp_state = to_ib_qp_state(qhp->attr.state);
+ init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
+ init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
+ init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
+diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
+index 37273dc0c03ca..b0d587254fe66 100644
+--- a/drivers/infiniband/hw/hfi1/chip.c
++++ b/drivers/infiniband/hw/hfi1/chip.c
+@@ -8414,6 +8414,8 @@ static void receive_interrupt_common(struct hfi1_ctxtdata *rcd)
+ */
+ static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
+ {
++ if (!rcd->rcvhdrq)
++ return;
+ clear_recv_intr(rcd);
+ if (check_packet_present(rcd))
+ force_recv_intr(rcd);
+diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
+index de411884386bf..385e6cff0d279 100644
+--- a/drivers/infiniband/hw/hfi1/driver.c
++++ b/drivers/infiniband/hw/hfi1/driver.c
+@@ -1011,6 +1011,8 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
+ struct hfi1_packet packet;
+ int skip_pkt = 0;
+
++ if (!rcd->rcvhdrq)
++ return RCV_PKT_OK;
+ /* Control context will always use the slow path interrupt handler */
+ needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
+
+diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
+index e3679d076eaaf..ec2a45c5cf575 100644
+--- a/drivers/infiniband/hw/hfi1/init.c
++++ b/drivers/infiniband/hw/hfi1/init.c
+@@ -112,7 +112,6 @@ static int hfi1_create_kctxt(struct hfi1_devdata *dd,
+ rcd->fast_handler = get_dma_rtail_setting(rcd) ?
+ handle_receive_interrupt_dma_rtail :
+ handle_receive_interrupt_nodma_rtail;
+- rcd->slow_handler = handle_receive_interrupt;
+
+ hfi1_set_seq_cnt(rcd, 1);
+
+@@ -333,6 +332,8 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
+ rcd->numa_id = numa;
+ rcd->rcv_array_groups = dd->rcv_entries.ngroups;
+ rcd->rhf_rcv_function_map = normal_rhf_rcv_functions;
++ rcd->slow_handler = handle_receive_interrupt;
++ rcd->do_interrupt = rcd->slow_handler;
+ rcd->msix_intr = CCE_NUM_MSIX_VECTORS;
+
+ mutex_init(&rcd->exp_mutex);
+@@ -873,18 +874,6 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
+ if (ret)
+ goto done;
+
+- /* allocate dummy tail memory for all receive contexts */
+- dd->rcvhdrtail_dummy_kvaddr = dma_alloc_coherent(&dd->pcidev->dev,
+- sizeof(u64),
+- &dd->rcvhdrtail_dummy_dma,
+- GFP_KERNEL);
+-
+- if (!dd->rcvhdrtail_dummy_kvaddr) {
+- dd_dev_err(dd, "cannot allocate dummy tail memory\n");
+- ret = -ENOMEM;
+- goto done;
+- }
+-
+ /* dd->rcd can be NULL if early initialization failed */
+ for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) {
+ /*
+@@ -897,8 +886,6 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
+ if (!rcd)
+ continue;
+
+- rcd->do_interrupt = &handle_receive_interrupt;
+-
+ lastfail = hfi1_create_rcvhdrq(dd, rcd);
+ if (!lastfail)
+ lastfail = hfi1_setup_eagerbufs(rcd);
+@@ -1119,7 +1106,7 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
+ rcd->egrbufs.rcvtids = NULL;
+
+ for (e = 0; e < rcd->egrbufs.alloced; e++) {
+- if (rcd->egrbufs.buffers[e].dma)
++ if (rcd->egrbufs.buffers[e].addr)
+ dma_free_coherent(&dd->pcidev->dev,
+ rcd->egrbufs.buffers[e].len,
+ rcd->egrbufs.buffers[e].addr,
+@@ -1200,6 +1187,11 @@ void hfi1_free_devdata(struct hfi1_devdata *dd)
+ dd->tx_opstats = NULL;
+ kfree(dd->comp_vect);
+ dd->comp_vect = NULL;
++ if (dd->rcvhdrtail_dummy_kvaddr)
++ dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
++ (void *)dd->rcvhdrtail_dummy_kvaddr,
++ dd->rcvhdrtail_dummy_dma);
++ dd->rcvhdrtail_dummy_kvaddr = NULL;
+ sdma_clean(dd, dd->num_sdma);
+ rvt_dealloc_device(&dd->verbs_dev.rdi);
+ }
+@@ -1297,6 +1289,15 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
+ goto bail;
+ }
+
++ /* allocate dummy tail memory for all receive contexts */
++ dd->rcvhdrtail_dummy_kvaddr =
++ dma_alloc_coherent(&dd->pcidev->dev, sizeof(u64),
++ &dd->rcvhdrtail_dummy_dma, GFP_KERNEL);
++ if (!dd->rcvhdrtail_dummy_kvaddr) {
++ ret = -ENOMEM;
++ goto bail;
++ }
++
+ atomic_set(&dd->ipoib_rsm_usr_num, 0);
+ return dd;
+
+@@ -1504,13 +1505,6 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
+
+ free_credit_return(dd);
+
+- if (dd->rcvhdrtail_dummy_kvaddr) {
+- dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
+- (void *)dd->rcvhdrtail_dummy_kvaddr,
+- dd->rcvhdrtail_dummy_dma);
+- dd->rcvhdrtail_dummy_kvaddr = NULL;
+- }
+-
+ /*
+ * Free any resources still in use (usually just kernel contexts)
+ * at unload; we do for ctxtcnt, because that's what we allocate.
+diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c
+index e594a961f513e..3e475814b6fa6 100644
+--- a/drivers/infiniband/hw/hfi1/ipoib_main.c
++++ b/drivers/infiniband/hw/hfi1/ipoib_main.c
+@@ -22,26 +22,35 @@ static int hfi1_ipoib_dev_init(struct net_device *dev)
+ int ret;
+
+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
++ if (!dev->tstats)
++ return -ENOMEM;
+
+ ret = priv->netdev_ops->ndo_init(dev);
+ if (ret)
+- return ret;
++ goto out_ret;
+
+ ret = hfi1_netdev_add_data(priv->dd,
+ qpn_from_mac(priv->netdev->dev_addr),
+ dev);
+ if (ret < 0) {
+ priv->netdev_ops->ndo_uninit(dev);
+- return ret;
++ goto out_ret;
+ }
+
+ return 0;
++out_ret:
++ free_percpu(dev->tstats);
++ dev->tstats = NULL;
++ return ret;
+ }
+
+ static void hfi1_ipoib_dev_uninit(struct net_device *dev)
+ {
+ struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+
++ free_percpu(dev->tstats);
++ dev->tstats = NULL;
++
+ hfi1_netdev_remove_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr));
+
+ priv->netdev_ops->ndo_uninit(dev);
+@@ -166,12 +175,7 @@ static void hfi1_ipoib_netdev_dtor(struct net_device *dev)
+ hfi1_ipoib_rxq_deinit(priv->netdev);
+
+ free_percpu(dev->tstats);
+-}
+-
+-static void hfi1_ipoib_free_rdma_netdev(struct net_device *dev)
+-{
+- hfi1_ipoib_netdev_dtor(dev);
+- free_netdev(dev);
++ dev->tstats = NULL;
+ }
+
+ static void hfi1_ipoib_set_id(struct net_device *dev, int id)
+@@ -211,24 +215,23 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device,
+ priv->port_num = port_num;
+ priv->netdev_ops = netdev->netdev_ops;
+
+- netdev->netdev_ops = &hfi1_ipoib_netdev_ops;
+-
+ ib_query_pkey(device, port_num, priv->pkey_index, &priv->pkey);
+
+ rc = hfi1_ipoib_txreq_init(priv);
+ if (rc) {
+ dd_dev_err(dd, "IPoIB netdev TX init - failed(%d)\n", rc);
+- hfi1_ipoib_free_rdma_netdev(netdev);
+ return rc;
+ }
+
+ rc = hfi1_ipoib_rxq_init(netdev);
+ if (rc) {
+ dd_dev_err(dd, "IPoIB netdev RX init - failed(%d)\n", rc);
+- hfi1_ipoib_free_rdma_netdev(netdev);
++ hfi1_ipoib_txreq_deinit(priv);
+ return rc;
+ }
+
++ netdev->netdev_ops = &hfi1_ipoib_netdev_ops;
++
+ netdev->priv_destructor = hfi1_ipoib_netdev_dtor;
+ netdev->needs_free_netdev = true;
+
+diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
+index 876cc78a22cca..7333646021bb8 100644
+--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
++++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
+@@ -80,6 +80,9 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
+ unsigned long flags;
+ struct list_head del_list;
+
++ /* Prevent freeing of mm until we are completely finished. */
++ mmgrab(handler->mn.mm);
++
+ /* Unregister first so we don't get any more notifications. */
+ mmu_notifier_unregister(&handler->mn, handler->mn.mm);
+
+@@ -102,6 +105,9 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
+
+ do_remove(handler, &del_list);
+
++ /* Now the mm may be freed. */
++ mmdrop(handler->mn.mm);
++
+ kfree(handler);
+ }
+
+diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
+index 2b6c24b7b5865..f07d328689d3d 100644
+--- a/drivers/infiniband/hw/hfi1/sdma.c
++++ b/drivers/infiniband/hw/hfi1/sdma.c
+@@ -838,8 +838,8 @@ struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
+ if (current->nr_cpus_allowed != 1)
+ goto out;
+
+- cpu_id = smp_processor_id();
+ rcu_read_lock();
++ cpu_id = smp_processor_id();
+ rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
+ sdma_rht_params);
+
+diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
+index 26bea51869bf0..ef8e0bdacb516 100644
+--- a/drivers/infiniband/hw/hfi1/verbs.c
++++ b/drivers/infiniband/hw/hfi1/verbs.c
+@@ -1397,8 +1397,7 @@ static int query_port(struct rvt_dev_info *rdi, u32 port_num,
+ 4096 : hfi1_max_mtu), IB_MTU_4096);
+ props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu :
+ mtu_to_enum(ppd->ibmtu, IB_MTU_4096);
+- props->phys_mtu = HFI1_CAP_IS_KSET(AIP) ? hfi1_max_mtu :
+- ib_mtu_enum_to_int(props->max_mtu);
++ props->phys_mtu = hfi1_max_mtu;
+
+ return 0;
+ }
+diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+index d5f3faa1627a4..96fe73ba689c1 100644
+--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
++++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+@@ -33,6 +33,7 @@
+ #include <linux/acpi.h>
+ #include <linux/etherdevice.h>
+ #include <linux/interrupt.h>
++#include <linux/iopoll.h>
+ #include <linux/kernel.h>
+ #include <linux/types.h>
+ #include <net/addrconf.h>
+@@ -1050,9 +1051,14 @@ static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
+ unsigned long instance_stage,
+ unsigned long reset_stage)
+ {
++#define HW_RESET_TIMEOUT_US 1000000
++#define HW_RESET_SLEEP_US 1000
++
+ struct hns_roce_v2_priv *priv = hr_dev->priv;
+ struct hnae3_handle *handle = priv->handle;
+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
++ unsigned long val;
++ int ret;
+
+ /* When hardware reset is detected, we should stop sending mailbox&cmq&
+ * doorbell to hardware. If now in .init_instance() function, we should
+@@ -1064,7 +1070,11 @@ static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
+ * again.
+ */
+ hr_dev->dis_db = true;
+- if (!ops->get_hw_reset_stat(handle))
++
++ ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,
++ val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,
++ HW_RESET_TIMEOUT_US, false, handle);
++ if (!ret)
+ hr_dev->is_reset = true;
+
+ if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
+@@ -1594,11 +1604,17 @@ static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
+ {
+ struct hns_roce_cmq_desc desc;
+ struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
++ u32 clock_cycles_of_1us;
+
+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
+ false);
+
+- hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, 0x3e8);
++ if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
++ clock_cycles_of_1us = HNS_ROCE_1NS_CFG;
++ else
++ clock_cycles_of_1us = HNS_ROCE_1US_CFG;
++
++ hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us);
+ hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);
+
+ return hns_roce_cmq_send(hr_dev, &desc, 1);
+@@ -3328,7 +3344,7 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
+ memset(cq_context, 0, sizeof(*cq_context));
+
+ hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);
+- hr_reg_write(cq_context, CQC_ARM_ST, REG_NXT_CEQE);
++ hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED);
+ hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth));
+ hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector);
+ hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn);
+@@ -4399,8 +4415,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
+ mtu = ib_mtu_enum_to_int(ib_mtu);
+ if (WARN_ON(mtu <= 0))
+ return -EINVAL;
+-#define MAX_LP_MSG_LEN 65536
+- /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */
++#define MAX_LP_MSG_LEN 16384
++ /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */
+ lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / mtu);
+ if (WARN_ON(lp_pktn_ini >= 0xF))
+ return -EINVAL;
+@@ -4802,6 +4818,30 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
+ return ret;
+ }
+
++static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout)
++{
++#define QP_ACK_TIMEOUT_MAX_HIP08 20
++#define QP_ACK_TIMEOUT_OFFSET 10
++#define QP_ACK_TIMEOUT_MAX 31
++
++ if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
++ if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) {
++ ibdev_warn(&hr_dev->ib_dev,
++ "Local ACK timeout shall be 0 to 20.\n");
++ return false;
++ }
++ *timeout += QP_ACK_TIMEOUT_OFFSET;
++ } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) {
++ if (*timeout > QP_ACK_TIMEOUT_MAX) {
++ ibdev_warn(&hr_dev->ib_dev,
++ "Local ACK timeout shall be 0 to 31.\n");
++ return false;
++ }
++ }
++
++ return true;
++}
++
+ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
+ const struct ib_qp_attr *attr,
+ int attr_mask,
+@@ -4811,6 +4851,7 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+ int ret = 0;
++ u8 timeout;
+
+ if (attr_mask & IB_QP_AV) {
+ ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
+@@ -4820,12 +4861,10 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
+ }
+
+ if (attr_mask & IB_QP_TIMEOUT) {
+- if (attr->timeout < 31) {
+- hr_reg_write(context, QPC_AT, attr->timeout);
++ timeout = attr->timeout;
++ if (check_qp_timeout_cfg_range(hr_dev, &timeout)) {
++ hr_reg_write(context, QPC_AT, timeout);
+ hr_reg_clear(qpc_mask, QPC_AT);
+- } else {
+- ibdev_warn(&hr_dev->ib_dev,
+- "Local ACK timeout shall be 0 to 30.\n");
+ }
+ }
+
+@@ -4882,7 +4921,9 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
+ set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
+
+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+- hr_reg_write(context, QPC_MIN_RNR_TIME, attr->min_rnr_timer);
++ hr_reg_write(context, QPC_MIN_RNR_TIME,
++ hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?
++ HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer);
+ hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);
+ }
+
+@@ -5499,6 +5540,16 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
+
+ hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);
+ hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);
++
++ if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
++ if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
++ dev_info(hr_dev->dev,
++ "cq_period(%u) reached the upper limit, adjusted to 65.\n",
++ cq_period);
++ cq_period = HNS_ROCE_MAX_CQ_PERIOD;
++ }
++ cq_period *= HNS_ROCE_CLOCK_ADJUST;
++ }
+ hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);
+ hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);
+
+@@ -5894,6 +5945,15 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
+ hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);
+ hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt);
+
++ if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
++ if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
++ dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n",
++ eq->eq_period);
++ eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD;
++ }
++ eq->eq_period *= HNS_ROCE_CLOCK_ADJUST;
++ }
++
+ hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period);
+ hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);
+ hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3);
+@@ -6397,10 +6457,8 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
+ if (!hr_dev)
+ return 0;
+
+- hr_dev->is_reset = true;
+ hr_dev->active = false;
+ hr_dev->dis_db = true;
+-
+ hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
+
+ return 0;
+diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+index 4d904d5e82be4..35c61da7ba156 100644
+--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
++++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+@@ -1444,6 +1444,14 @@ struct hns_roce_dip {
+ struct list_head node; /* all dips are on a list */
+ };
+
++/* only for RNR timeout issue of HIP08 */
++#define HNS_ROCE_CLOCK_ADJUST 1000
++#define HNS_ROCE_MAX_CQ_PERIOD 65
++#define HNS_ROCE_MAX_EQ_PERIOD 65
++#define HNS_ROCE_RNR_TIMER_10NS 1
++#define HNS_ROCE_1US_CFG 999
++#define HNS_ROCE_1NS_CFG 0
++
+ #define HNS_ROCE_AEQ_DEFAULT_BURST_NUM 0x0
+ #define HNS_ROCE_AEQ_DEFAULT_INTERVAL 0x0
+ #define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x0
+diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
+index 5d39bd08582af..1f2209de88122 100644
+--- a/drivers/infiniband/hw/hns/hns_roce_main.c
++++ b/drivers/infiniband/hw/hns/hns_roce_main.c
+@@ -269,6 +269,9 @@ static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,
+ static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index,
+ u16 *pkey)
+ {
++ if (index > 0)
++ return -EINVAL;
++
+ *pkey = PKEY_ID;
+
+ return 0;
+@@ -349,7 +352,7 @@ static int hns_roce_mmap(struct ib_ucontext *context,
+ return rdma_user_mmap_io(context, vma,
+ to_hr_ucontext(context)->uar.pfn,
+ PAGE_SIZE,
+- pgprot_noncached(vma->vm_page_prot),
++ pgprot_device(vma->vm_page_prot),
+ NULL);
+
+ /* vm_pgoff: 1 -- TPTR */
+diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
+index 6eee9deadd122..e64ef6903fb4f 100644
+--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
++++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
+@@ -259,7 +259,7 @@ static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
+
+ static void free_srq_wrid(struct hns_roce_srq *srq)
+ {
+- kfree(srq->wrid);
++ kvfree(srq->wrid);
+ srq->wrid = NULL;
+ }
+
+diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c
+index 6dea0a49d1718..632f65e53b63f 100644
+--- a/drivers/infiniband/hw/irdma/cm.c
++++ b/drivers/infiniband/hw/irdma/cm.c
+@@ -2305,10 +2305,8 @@ err:
+ return NULL;
+ }
+
+-static void irdma_cm_node_free_cb(struct rcu_head *rcu_head)
++static void irdma_destroy_connection(struct irdma_cm_node *cm_node)
+ {
+- struct irdma_cm_node *cm_node =
+- container_of(rcu_head, struct irdma_cm_node, rcu_head);
+ struct irdma_cm_core *cm_core = cm_node->cm_core;
+ struct irdma_qp *iwqp;
+ struct irdma_cm_info nfo;
+@@ -2356,7 +2354,6 @@ static void irdma_cm_node_free_cb(struct rcu_head *rcu_head)
+ }
+
+ cm_core->cm_free_ah(cm_node);
+- kfree(cm_node);
+ }
+
+ /**
+@@ -2384,8 +2381,9 @@ void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)
+
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+- /* wait for all list walkers to exit their grace period */
+- call_rcu(&cm_node->rcu_head, irdma_cm_node_free_cb);
++ irdma_destroy_connection(cm_node);
++
++ kfree_rcu(cm_node, rcu_head);
+ }
+
+ /**
+@@ -3244,15 +3242,10 @@ enum irdma_status_code irdma_setup_cm_core(struct irdma_device *iwdev,
+ */
+ void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)
+ {
+- unsigned long flags;
+-
+ if (!cm_core)
+ return;
+
+- spin_lock_irqsave(&cm_core->ht_lock, flags);
+- if (timer_pending(&cm_core->tcp_timer))
+- del_timer_sync(&cm_core->tcp_timer);
+- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
++ del_timer_sync(&cm_core->tcp_timer);
+
+ destroy_workqueue(cm_core->event_wq);
+ cm_core->dev->ws_reset(&cm_core->iwdev->vsi);
+@@ -3465,12 +3458,6 @@ static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
+ }
+
+ cm_id = iwqp->cm_id;
+- /* make sure we havent already closed this connection */
+- if (!cm_id) {
+- spin_unlock_irqrestore(&iwqp->lock, flags);
+- return;
+- }
+-
+ original_hw_tcp_state = iwqp->hw_tcp_state;
+ original_ibqp_state = iwqp->ibqp_state;
+ last_ae = iwqp->last_aeq;
+@@ -3492,11 +3479,11 @@ static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
+ disconn_status = -ECONNRESET;
+ }
+
+- if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
+- original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
+- last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
+- last_ae == IRDMA_AE_BAD_CLOSE ||
+- last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset)) {
++ if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
++ original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
++ last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
++ last_ae == IRDMA_AE_BAD_CLOSE ||
++ last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {
+ issue_close = 1;
+ iwqp->cm_id = NULL;
+ qp->term_flags = 0;
+diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
+index f1e5515256e0a..1ac7067e21be1 100644
+--- a/drivers/infiniband/hw/irdma/ctrl.c
++++ b/drivers/infiniband/hw/irdma/ctrl.c
+@@ -431,7 +431,7 @@ enum irdma_status_code irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_c
+
+ cqp = qp->dev->cqp;
+ if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
+- qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))
++ qp->qp_uk.qp_id >= (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt))
+ return IRDMA_ERR_INVALID_QP_ID;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+@@ -2551,10 +2551,10 @@ static enum irdma_status_code irdma_sc_cq_create(struct irdma_sc_cq *cq,
+ enum irdma_status_code ret_code = 0;
+
+ cqp = cq->dev->cqp;
+- if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))
++ if (cq->cq_uk.cq_id >= (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt))
+ return IRDMA_ERR_INVALID_CQ_ID;
+
+- if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
++ if (cq->ceq_id >= (cq->dev->hmc_fpm_misc.max_ceqs))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+
+ ceq = cq->dev->ceq[cq->ceq_id];
+@@ -3656,7 +3656,7 @@ enum irdma_status_code irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
+ info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+- if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
++ if (info->ceq_id >= (info->dev->hmc_fpm_misc.max_ceqs))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+@@ -4205,7 +4205,7 @@ enum irdma_status_code irdma_sc_ccq_init(struct irdma_sc_cq *cq,
+ info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+- if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
++ if (info->ceq_id >= (info->dev->hmc_fpm_misc.max_ceqs ))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
+index 7de525a5ccf8c..4f763e552eae8 100644
+--- a/drivers/infiniband/hw/irdma/hw.c
++++ b/drivers/infiniband/hw/irdma/hw.c
+@@ -60,6 +60,8 @@ static void irdma_iwarp_ce_handler(struct irdma_sc_cq *iwcq)
+ {
+ struct irdma_cq *cq = iwcq->back_cq;
+
++ if (!cq->user_mode)
++ cq->armed = false;
+ if (cq->ibcq.comp_handler)
+ cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+ }
+@@ -146,6 +148,7 @@ static void irdma_set_flush_fields(struct irdma_sc_qp *qp,
+ qp->flush_code = FLUSH_PROT_ERR;
+ break;
+ case IRDMA_AE_AMP_BAD_QP:
++ case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
+ qp->flush_code = FLUSH_LOC_QP_OP_ERR;
+ break;
+ case IRDMA_AE_AMP_BAD_STAG_KEY:
+@@ -156,7 +159,6 @@ static void irdma_set_flush_fields(struct irdma_sc_qp *qp,
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ case IRDMA_AE_IB_INVALID_REQUEST:
+ case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
+- case IRDMA_AE_IB_REMOTE_OP_ERROR:
+ qp->flush_code = FLUSH_REM_ACCESS_ERR;
+ qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+@@ -184,6 +186,9 @@ static void irdma_set_flush_fields(struct irdma_sc_qp *qp,
+ case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
+ qp->flush_code = FLUSH_MW_BIND_ERR;
+ break;
++ case IRDMA_AE_IB_REMOTE_OP_ERROR:
++ qp->flush_code = FLUSH_REM_OP_ERR;
++ break;
+ default:
+ qp->flush_code = FLUSH_FATAL_ERR;
+ break;
+@@ -1603,7 +1608,7 @@ static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf)
+ info.fpm_commit_buf = mem.va;
+
+ info.bar0 = rf->hw.hw_addr;
+- info.hmc_fn_id = PCI_FUNC(rf->pcidev->devfn);
++ info.hmc_fn_id = rf->pf_id;
+ info.hw = &rf->hw;
+ status = irdma_sc_dev_init(rf->rdma_ver, &rf->sc_dev, &info);
+ if (status)
+diff --git a/drivers/infiniband/hw/irdma/i40iw_if.c b/drivers/infiniband/hw/irdma/i40iw_if.c
+index d219f64b2c3d5..a6f758b61b0c4 100644
+--- a/drivers/infiniband/hw/irdma/i40iw_if.c
++++ b/drivers/infiniband/hw/irdma/i40iw_if.c
+@@ -77,6 +77,7 @@ static void i40iw_fill_device_info(struct irdma_device *iwdev, struct i40e_info
+ rf->rdma_ver = IRDMA_GEN_1;
+ rf->gen_ops.request_reset = i40iw_request_reset;
+ rf->pcidev = cdev_info->pcidev;
++ rf->pf_id = cdev_info->fid;
+ rf->hw.hw_addr = cdev_info->hw_addr;
+ rf->cdev = cdev_info;
+ rf->msix_count = cdev_info->msix_count;
+diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
+index 51a41359e0b41..c556a36e76703 100644
+--- a/drivers/infiniband/hw/irdma/main.c
++++ b/drivers/infiniband/hw/irdma/main.c
+@@ -226,6 +226,7 @@ static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf
+ rf->hw.hw_addr = pf->hw.hw_addr;
+ rf->pcidev = pf->pdev;
+ rf->msix_count = pf->num_rdma_msix;
++ rf->pf_id = pf->hw.pf_id;
+ rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector];
+ rf->default_vsi.vsi_idx = vsi->vsi_num;
+ rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;
+diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
+index b678fe712447e..454b4b370386c 100644
+--- a/drivers/infiniband/hw/irdma/main.h
++++ b/drivers/infiniband/hw/irdma/main.h
+@@ -257,6 +257,7 @@ struct irdma_pci_f {
+ u8 *mem_rsrc;
+ u8 rdma_ver;
+ u8 rst_to;
++ u8 pf_id;
+ enum irdma_protocol_used protocol_used;
+ u32 sd_type;
+ u32 msix_count;
+@@ -541,6 +542,7 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
+ void (*callback_fcn)(struct irdma_cqp_request *cqp_request),
+ void *cb_param);
+ void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
++bool irdma_cq_empty(struct irdma_cq *iwcq);
+ int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr);
+ int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
+diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c
+index aeeb1c310965d..fed49da770f3b 100644
+--- a/drivers/infiniband/hw/irdma/pble.c
++++ b/drivers/infiniband/hw/irdma/pble.c
+@@ -25,8 +25,7 @@ void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
+ list_del(&chunk->list);
+ if (chunk->type == PBLE_SD_PAGED)
+ irdma_pble_free_paged_mem(chunk);
+- if (chunk->bitmapbuf)
+- kfree(chunk->bitmapmem.va);
++ bitmap_free(chunk->bitmapbuf);
+ kfree(chunk->chunkmem.va);
+ }
+ }
+@@ -283,7 +282,6 @@ add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
+ "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",
+ pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
+ pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3);
+- list_add(&chunk->list, &pble_rsrc->pinfo.clist);
+ sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?
+ sd_entry->u.pd_table.pd_page_addr.pa :
+ sd_entry->u.bp.addr.pa;
+@@ -295,12 +293,12 @@ add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
+ goto error;
+ }
+
++ list_add(&chunk->list, &pble_rsrc->pinfo.clist);
+ sd_entry->valid = true;
+ return 0;
+
+ error:
+- if (chunk->bitmapbuf)
+- kfree(chunk->bitmapmem.va);
++ bitmap_free(chunk->bitmapbuf);
+ kfree(chunk->chunkmem.va);
+
+ return ret_code;
+diff --git a/drivers/infiniband/hw/irdma/pble.h b/drivers/infiniband/hw/irdma/pble.h
+index e1b3b8118a2ca..aa20827dcc9de 100644
+--- a/drivers/infiniband/hw/irdma/pble.h
++++ b/drivers/infiniband/hw/irdma/pble.h
+@@ -78,7 +78,6 @@ struct irdma_chunk {
+ u32 pg_cnt;
+ enum irdma_alloc_type type;
+ struct irdma_sc_dev *dev;
+- struct irdma_virt_mem bitmapmem;
+ struct irdma_virt_mem chunkmem;
+ };
+
+diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
+index ac91ea5296db9..85d4212f59dbc 100644
+--- a/drivers/infiniband/hw/irdma/utils.c
++++ b/drivers/infiniband/hw/irdma/utils.c
+@@ -150,31 +150,35 @@ int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr)
+ {
+ struct in_ifaddr *ifa = ptr;
+- struct net_device *netdev = ifa->ifa_dev->dev;
++ struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
+ struct irdma_device *iwdev;
+ struct ib_device *ibdev;
+ u32 local_ipaddr;
+
+- ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
++ real_dev = rdma_vlan_dev_real_dev(netdev);
++ if (!real_dev)
++ real_dev = netdev;
++
++ ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
+ if (!ibdev)
+ return NOTIFY_DONE;
+
+ iwdev = to_iwdev(ibdev);
+ local_ipaddr = ntohl(ifa->ifa_address);
+ ibdev_dbg(&iwdev->ibdev,
+- "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", netdev,
+- event, &local_ipaddr, netdev->dev_addr);
++ "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
++ event, &local_ipaddr, real_dev->dev_addr);
+ switch (event) {
+ case NETDEV_DOWN:
+- irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
++ irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
+ &local_ipaddr, true, IRDMA_ARP_DELETE);
+- irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false);
++ irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
+ irdma_gid_change_event(&iwdev->ibdev);
+ break;
+ case NETDEV_UP:
+ case NETDEV_CHANGEADDR:
+- irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr);
+- irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true);
++ irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
++ irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
+ irdma_gid_change_event(&iwdev->ibdev);
+ break;
+ default:
+@@ -196,32 +200,36 @@ int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr)
+ {
+ struct inet6_ifaddr *ifa = ptr;
+- struct net_device *netdev = ifa->idev->dev;
++ struct net_device *real_dev, *netdev = ifa->idev->dev;
+ struct irdma_device *iwdev;
+ struct ib_device *ibdev;
+ u32 local_ipaddr6[4];
+
+- ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
++ real_dev = rdma_vlan_dev_real_dev(netdev);
++ if (!real_dev)
++ real_dev = netdev;
++
++ ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
+ if (!ibdev)
+ return NOTIFY_DONE;
+
+ iwdev = to_iwdev(ibdev);
+ irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
+ ibdev_dbg(&iwdev->ibdev,
+- "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", netdev,
+- event, local_ipaddr6, netdev->dev_addr);
++ "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
++ event, local_ipaddr6, real_dev->dev_addr);
+ switch (event) {
+ case NETDEV_DOWN:
+- irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
++ irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
+ local_ipaddr6, false, IRDMA_ARP_DELETE);
+- irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false);
++ irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
+ irdma_gid_change_event(&iwdev->ibdev);
+ break;
+ case NETDEV_UP:
+ case NETDEV_CHANGEADDR:
+ irdma_add_arp(iwdev->rf, local_ipaddr6, false,
+- netdev->dev_addr);
+- irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true);
++ real_dev->dev_addr);
++ irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
+ irdma_gid_change_event(&iwdev->ibdev);
+ break;
+ default:
+@@ -243,21 +251,23 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr)
+ {
+ struct neighbour *neigh = ptr;
++ struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
+ struct irdma_device *iwdev;
+ struct ib_device *ibdev;
+ __be32 *p;
+ u32 local_ipaddr[4] = {};
+ bool ipv4 = true;
+
+- ibdev = ib_device_get_by_netdev((struct net_device *)neigh->dev,
+- RDMA_DRIVER_IRDMA);
+- if (!ibdev)
+- return NOTIFY_DONE;
+-
+- iwdev = to_iwdev(ibdev);
+-
+ switch (event) {
+ case NETEVENT_NEIGH_UPDATE:
++ real_dev = rdma_vlan_dev_real_dev(netdev);
++ if (!real_dev)
++ real_dev = netdev;
++ ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
++ if (!ibdev)
++ return NOTIFY_DONE;
++
++ iwdev = to_iwdev(ibdev);
+ p = (__be32 *)neigh->primary_key;
+ if (neigh->tbl->family == AF_INET6) {
+ ipv4 = false;
+@@ -278,13 +288,12 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event,
+ irdma_manage_arp_cache(iwdev->rf, neigh->ha,
+ local_ipaddr, ipv4,
+ IRDMA_ARP_DELETE);
++ ib_device_put(ibdev);
+ break;
+ default:
+ break;
+ }
+
+- ib_device_put(ibdev);
+-
+ return NOTIFY_DONE;
+ }
+
+@@ -2284,15 +2293,10 @@ enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
+
+ sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
+
+- pchunk->bitmapmem.size = sizeofbitmap >> 3;
+- pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL);
+-
+- if (!pchunk->bitmapmem.va)
++ pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
++ if (!pchunk->bitmapbuf)
+ return IRDMA_ERR_NO_MEMORY;
+
+- pchunk->bitmapbuf = pchunk->bitmapmem.va;
+- bitmap_zero(pchunk->bitmapbuf, sizeofbitmap);
+-
+ pchunk->sizeofbitmap = sizeofbitmap;
+ /* each pble is 8 bytes hence shift by 3 */
+ pprm->total_pble_alloc += pchunk->size >> 3;
+@@ -2536,3 +2540,18 @@ void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
+ ibevent.element.qp = &iwqp->ibqp;
+ iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
+ }
++
++bool irdma_cq_empty(struct irdma_cq *iwcq)
++{
++ struct irdma_cq_uk *ukcq;
++ u64 qword3;
++ __le64 *cqe;
++ u8 polarity;
++
++ ukcq = &iwcq->sc_cq.cq_uk;
++ cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
++ get_64bit_val(cqe, 24, &qword3);
++ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
++
++ return polarity != ukcq->polarity;
++}
+diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
+index 102dc9342f2a2..8a3ac4257e867 100644
+--- a/drivers/infiniband/hw/irdma/verbs.c
++++ b/drivers/infiniband/hw/irdma/verbs.c
+@@ -1617,13 +1617,13 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+
+ if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {
+ if (dont_wait) {
+- if (iwqp->cm_id && iwqp->hw_tcp_state) {
++ if (iwqp->hw_tcp_state) {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
+ iwqp->last_aeq = IRDMA_AE_RESET_SENT;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+- irdma_cm_disconn(iwqp);
+ }
++ irdma_cm_disconn(iwqp);
+ } else {
+ int close_timer_started;
+
+@@ -2506,7 +2506,7 @@ static int irdma_dealloc_mw(struct ib_mw *ibmw)
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.dealloc_stag.info;
+ memset(info, 0, sizeof(*info));
+- info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
++ info->pd_id = iwpd->sc_pd.pd_id;
+ info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
+ info->mr = false;
+ cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
+@@ -3018,7 +3018,7 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.dealloc_stag.info;
+ memset(info, 0, sizeof(*info));
+- info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
++ info->pd_id = iwpd->sc_pd.pd_id;
+ info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
+ info->mr = true;
+ if (iwpbl->pbl_allocated)
+@@ -3604,18 +3604,31 @@ static int irdma_req_notify_cq(struct ib_cq *ibcq,
+ struct irdma_cq *iwcq;
+ struct irdma_cq_uk *ukcq;
+ unsigned long flags;
+- enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT;
++ enum irdma_cmpl_notify cq_notify;
++ bool promo_event = false;
++ int ret = 0;
+
++ cq_notify = notify_flags == IB_CQ_SOLICITED ?
++ IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT;
+ iwcq = to_iwcq(ibcq);
+ ukcq = &iwcq->sc_cq.cq_uk;
+- if (notify_flags == IB_CQ_SOLICITED)
+- cq_notify = IRDMA_CQ_COMPL_SOLICITED;
+
+ spin_lock_irqsave(&iwcq->lock, flags);
+- irdma_uk_cq_request_notification(ukcq, cq_notify);
++ /* Only promote to arm the CQ for any event if the last arm event was solicited. */
++ if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED)
++ promo_event = true;
++
++ if (!iwcq->armed || promo_event) {
++ iwcq->armed = true;
++ iwcq->last_notify = cq_notify;
++ irdma_uk_cq_request_notification(ukcq, cq_notify);
++ }
++
++ if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && !irdma_cq_empty(iwcq))
++ ret = 1;
+ spin_unlock_irqrestore(&iwcq->lock, flags);
+
+- return 0;
++ return ret;
+ }
+
+ static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num,
+diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
+index 5c244cd321a3a..d0fdef8d09ead 100644
+--- a/drivers/infiniband/hw/irdma/verbs.h
++++ b/drivers/infiniband/hw/irdma/verbs.h
+@@ -110,6 +110,8 @@ struct irdma_cq {
+ u16 cq_size;
+ u16 cq_num;
+ bool user_mode;
++ bool armed;
++ enum irdma_cmpl_notify last_notify;
+ u32 polled_cmpls;
+ u32 cq_mem_size;
+ struct irdma_dma_mem kmem;
+diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
+index f367f4a4abffc..53d83212cda81 100644
+--- a/drivers/infiniband/hw/mlx4/main.c
++++ b/drivers/infiniband/hw/mlx4/main.c
+@@ -2217,6 +2217,11 @@ static const struct ib_device_ops mlx4_ib_hw_stats_ops = {
+ .get_hw_stats = mlx4_ib_get_hw_stats,
+ };
+
++static const struct ib_device_ops mlx4_ib_hw_stats_ops1 = {
++ .alloc_hw_device_stats = mlx4_ib_alloc_hw_device_stats,
++ .get_hw_stats = mlx4_ib_get_hw_stats,
++};
++
+ static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)
+ {
+ struct mlx4_ib_diag_counters *diag = ibdev->diag_counters;
+@@ -2229,9 +2234,16 @@ static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)
+ return 0;
+
+ for (i = 0; i < MLX4_DIAG_COUNTERS_TYPES; i++) {
+- /* i == 1 means we are building port counters */
+- if (i && !per_port)
+- continue;
++ /*
++ * i == 1 means we are building port counters, set a different
++ * stats ops without port stats callback.
++ */
++ if (i && !per_port) {
++ ib_set_device_ops(&ibdev->ib_dev,
++ &mlx4_ib_hw_stats_ops1);
++
++ return 0;
++ }
+
+ ret = __mlx4_ib_alloc_diag_counters(ibdev, &diag[i].name,
+ &diag[i].offset,
+@@ -3237,7 +3249,7 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
+ case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:
+ ew = kmalloc(sizeof *ew, GFP_ATOMIC);
+ if (!ew)
+- break;
++ return;
+
+ INIT_WORK(&ew->work, handle_port_mgmt_change_event);
+ memcpy(&ew->ib_eqe, eqe, sizeof *eqe);
+diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
+index 8662f462e2a5f..3a1a4ac9dd33d 100644
+--- a/drivers/infiniband/hw/mlx4/qp.c
++++ b/drivers/infiniband/hw/mlx4/qp.c
+@@ -1099,8 +1099,10 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,
+ if (dev->steering_support ==
+ MLX4_STEERING_MODE_DEVICE_MANAGED)
+ qp->flags |= MLX4_IB_QP_NETIF;
+- else
++ else {
++ err = -EINVAL;
+ goto err;
++ }
+ }
+
+ err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
+diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
+index e95967aefe788..21beded40066d 100644
+--- a/drivers/infiniband/hw/mlx5/devx.c
++++ b/drivers/infiniband/hw/mlx5/devx.c
+@@ -1891,8 +1891,10 @@ subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
+ key_level2,
+ obj_event,
+ GFP_KERNEL);
+- if (err)
++ if (err) {
++ kfree(obj_event);
+ return err;
++ }
+ INIT_LIST_HEAD(&obj_event->obj_sub_list);
+ }
+
+diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
+index 22e2f4d79743d..cf203f879d340 100644
+--- a/drivers/infiniband/hw/mlx5/mr.c
++++ b/drivers/infiniband/hw/mlx5/mr.c
+@@ -536,8 +536,10 @@ static void __cache_work_func(struct mlx5_cache_ent *ent)
+ spin_lock_irq(&ent->lock);
+ if (ent->disabled)
+ goto out;
+- if (need_delay)
++ if (need_delay) {
+ queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
++ goto out;
++ }
+ remove_cache_mr_locked(ent);
+ queue_adjust_cache_locked(ent);
+ }
+@@ -580,6 +582,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
+ ent = &cache->ent[entry];
+ spin_lock_irq(&ent->lock);
+ if (list_empty(&ent->head)) {
++ queue_adjust_cache_locked(ent);
++ ent->miss++;
+ spin_unlock_irq(&ent->lock);
+ mr = create_cache_mr(ent);
+ if (IS_ERR(mr))
+@@ -631,6 +635,7 @@ static void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
+ {
+ struct mlx5_cache_ent *ent = mr->cache_ent;
+
++ WRITE_ONCE(dev->cache.last_add, jiffies);
+ spin_lock_irq(&ent->lock);
+ list_add_tail(&mr->list, &ent->head);
+ ent->available_mrs++;
+diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
+index dcb3653db72d7..f652d083ff20f 100644
+--- a/drivers/infiniband/hw/qedr/verbs.c
++++ b/drivers/infiniband/hw/qedr/verbs.c
+@@ -1941,6 +1941,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
+ /* db offset was calculated in copy_qp_uresp, now set in the user q */
+ if (qedr_qp_has_sq(qp)) {
+ qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
++ qp->sq.max_wr = attrs->cap.max_send_wr;
+ rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
+ &qp->usq.db_rec_data->db_data,
+ DB_REC_WIDTH_32B,
+@@ -1951,6 +1952,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
+
+ if (qedr_qp_has_rq(qp)) {
+ qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
++ qp->rq.max_wr = attrs->cap.max_recv_wr;
+ rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
+ &qp->urq.db_rec_data->db_data,
+ DB_REC_WIDTH_32B,
+@@ -2744,15 +2746,18 @@ int qedr_query_qp(struct ib_qp *ibqp,
+ int rc = 0;
+
+ memset(&params, 0, sizeof(params));
+-
+- rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
+- if (rc)
+- goto err;
+-
+ memset(qp_attr, 0, sizeof(*qp_attr));
+ memset(qp_init_attr, 0, sizeof(*qp_init_attr));
+
+- qp_attr->qp_state = qedr_get_ibqp_state(params.state);
++ if (qp->qp_type != IB_QPT_GSI) {
++ rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
++ if (rc)
++ goto err;
++ qp_attr->qp_state = qedr_get_ibqp_state(params.state);
++ } else {
++ qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
++ }
++
+ qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
+ qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
+ qp_attr->path_mig_state = IB_MIG_MIGRATED;
+diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c
+index 0a3b28142c05b..41c272980f91c 100644
+--- a/drivers/infiniband/hw/qib/qib_sysfs.c
++++ b/drivers/infiniband/hw/qib/qib_sysfs.c
+@@ -541,7 +541,7 @@ static struct attribute *port_diagc_attributes[] = {
+ };
+
+ static const struct attribute_group port_diagc_group = {
+- .name = "linkcontrol",
++ .name = "diag_counters",
+ .attrs = port_diagc_attributes,
+ };
+
+diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c
+index ac11943a5ddb0..bf2f30d67949d 100644
+--- a/drivers/infiniband/hw/qib/qib_user_sdma.c
++++ b/drivers/infiniband/hw/qib/qib_user_sdma.c
+@@ -941,7 +941,7 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
+ &addrlimit) ||
+ addrlimit > type_max(typeof(pkt->addrlimit))) {
+ ret = -EINVAL;
+- goto free_pbc;
++ goto free_pkt;
+ }
+ pkt->addrlimit = addrlimit;
+
+diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
+index 3305f2744bfaa..8ef112f883a77 100644
+--- a/drivers/infiniband/sw/rdmavt/qp.c
++++ b/drivers/infiniband/sw/rdmavt/qp.c
+@@ -3073,6 +3073,8 @@ do_write:
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
+ goto inv_err;
++ if (unlikely(wqe->atomic_wr.remote_addr & (sizeof(u64) - 1)))
++ goto inv_err;
+ if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
+ wqe->atomic_wr.remote_addr,
+ wqe->atomic_wr.rkey,
+@@ -3188,7 +3190,11 @@ serr_no_r_lock:
+ spin_lock_irqsave(&sqp->s_lock, flags);
+ rvt_send_complete(sqp, wqe, send_status);
+ if (sqp->ibqp.qp_type == IB_QPT_RC) {
+- int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
++ int lastwqe;
++
++ spin_lock(&sqp->r_lock);
++ lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
++ spin_unlock(&sqp->r_lock);
+
+ sqp->s_flags &= ~RVT_S_BUSY;
+ spin_unlock_irqrestore(&sqp->s_lock, flags);
+diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
+index f0c954575bdec..4fd73b51fabf2 100644
+--- a/drivers/infiniband/sw/rxe/rxe_loc.h
++++ b/drivers/infiniband/sw/rxe/rxe_loc.h
+@@ -86,6 +86,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
+ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
+ int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
+ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
++int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
+ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+ void rxe_mr_cleanup(struct rxe_pool_entry *arg);
+
+diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
+index 5890a82462161..bedcf15aaea75 100644
+--- a/drivers/infiniband/sw/rxe/rxe_mr.c
++++ b/drivers/infiniband/sw/rxe/rxe_mr.c
+@@ -48,8 +48,14 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
+ u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);
+ u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
+
+- mr->ibmr.lkey = lkey;
+- mr->ibmr.rkey = rkey;
++ /* set ibmr->l/rkey and also copy into private l/rkey
++ * for user MRs these will always be the same
++ * for cases where caller 'owns' the key portion
++ * they may be different until REG_MR WQE is executed.
++ */
++ mr->lkey = mr->ibmr.lkey = lkey;
++ mr->rkey = mr->ibmr.rkey = rkey;
++
+ mr->state = RXE_MR_STATE_INVALID;
+ mr->type = RXE_MR_TYPE_NONE;
+ mr->map_shift = ilog2(RXE_BUF_PER_MAP);
+@@ -191,10 +197,8 @@ int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr)
+ {
+ int err;
+
+- rxe_mr_init(0, mr);
+-
+- /* In fastreg, we also set the rkey */
+- mr->ibmr.rkey = mr->ibmr.lkey;
++ /* always allow remote access for FMRs */
++ rxe_mr_init(IB_ACCESS_REMOTE, mr);
+
+ err = rxe_mr_alloc(mr, max_pages);
+ if (err)
+@@ -507,8 +511,8 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
+ if (!mr)
+ return NULL;
+
+- if (unlikely((type == RXE_LOOKUP_LOCAL && mr_lkey(mr) != key) ||
+- (type == RXE_LOOKUP_REMOTE && mr_rkey(mr) != key) ||
++ if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
++ (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
+ mr_pd(mr) != pd || (access && !(access & mr->access)) ||
+ mr->state != RXE_MR_STATE_VALID)) {
+ rxe_drop_ref(mr);
+@@ -531,9 +535,9 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+ goto err;
+ }
+
+- if (rkey != mr->ibmr.rkey) {
+- pr_err("%s: rkey (%#x) doesn't match mr->ibmr.rkey (%#x)\n",
+- __func__, rkey, mr->ibmr.rkey);
++ if (rkey != mr->rkey) {
++ pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
++ __func__, rkey, mr->rkey);
+ ret = -EINVAL;
+ goto err_drop_ref;
+ }
+@@ -554,6 +558,49 @@ err:
+ return ret;
+ }
+
++/* user can (re)register fast MR by executing a REG_MR WQE.
++ * user is expected to hold a reference on the ib mr until the
++ * WQE completes.
++ * Once a fast MR is created this is the only way to change the
++ * private keys. It is the responsibility of the user to maintain
++ * the ib mr keys in sync with rxe mr keys.
++ */
++int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
++{
++ struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);
++ u32 key = wqe->wr.wr.reg.key;
++ u32 access = wqe->wr.wr.reg.access;
++
++ /* user can only register MR in free state */
++ if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
++ pr_warn("%s: mr->lkey = 0x%x not free\n",
++ __func__, mr->lkey);
++ return -EINVAL;
++ }
++
++ /* user can only register mr with qp in same protection domain */
++ if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
++ pr_warn("%s: qp->pd and mr->pd don't match\n",
++ __func__);
++ return -EINVAL;
++ }
++
++ /* user is only allowed to change key portion of l/rkey */
++ if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
++ pr_warn("%s: key = 0x%x has wrong index mr->lkey = 0x%x\n",
++ __func__, key, mr->lkey);
++ return -EINVAL;
++ }
++
++ mr->access = access;
++ mr->lkey = key;
++ mr->rkey = (access & IB_ACCESS_REMOTE) ? key : 0;
++ mr->iova = wqe->wr.wr.reg.mr->iova;
++ mr->state = RXE_MR_STATE_VALID;
++
++ return 0;
++}
++
+ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+ {
+ struct rxe_mr *mr = to_rmr(ibmr);
+diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c
+index 5ba77df7598ed..a5e2ea7d80f02 100644
+--- a/drivers/infiniband/sw/rxe/rxe_mw.c
++++ b/drivers/infiniband/sw/rxe/rxe_mw.c
+@@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
+ }
+
+ rxe_add_index(mw);
+- ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
++ mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
+ mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
+ RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
+ spin_lock_init(&mw->lock);
+@@ -71,6 +71,8 @@ int rxe_dealloc_mw(struct ib_mw *ibmw)
+ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+ struct rxe_mw *mw, struct rxe_mr *mr)
+ {
++ u32 key = wqe->wr.wr.mw.rkey & 0xff;
++
+ if (mw->ibmw.type == IB_MW_TYPE_1) {
+ if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
+ pr_err_once(
+@@ -108,7 +110,7 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+ }
+ }
+
+- if (unlikely((wqe->wr.wr.mw.rkey & 0xff) == (mw->ibmw.rkey & 0xff))) {
++ if (unlikely(key == (mw->rkey & 0xff))) {
+ pr_err_once("attempt to bind MW with same key\n");
+ return -EINVAL;
+ }
+@@ -161,13 +163,9 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+ static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+ struct rxe_mw *mw, struct rxe_mr *mr)
+ {
+- u32 rkey;
+- u32 new_rkey;
+-
+- rkey = mw->ibmw.rkey;
+- new_rkey = (rkey & 0xffffff00) | (wqe->wr.wr.mw.rkey & 0x000000ff);
++ u32 key = wqe->wr.wr.mw.rkey & 0xff;
+
+- mw->ibmw.rkey = new_rkey;
++ mw->rkey = (mw->rkey & ~0xff) | key;
+ mw->access = wqe->wr.wr.mw.access;
+ mw->state = RXE_MW_STATE_VALID;
+ mw->addr = wqe->wr.wr.mw.addr;
+@@ -197,29 +195,29 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
+ struct rxe_mw *mw;
+ struct rxe_mr *mr;
+ struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
++ u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
++ u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
+ unsigned long flags;
+
+- mw = rxe_pool_get_index(&rxe->mw_pool,
+- wqe->wr.wr.mw.mw_rkey >> 8);
++ mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
+ if (unlikely(!mw)) {
+ ret = -EINVAL;
+ goto err;
+ }
+
+- if (unlikely(mw->ibmw.rkey != wqe->wr.wr.mw.mw_rkey)) {
++ if (unlikely(mw->rkey != mw_rkey)) {
+ ret = -EINVAL;
+ goto err_drop_mw;
+ }
+
+ if (likely(wqe->wr.wr.mw.length)) {
+- mr = rxe_pool_get_index(&rxe->mr_pool,
+- wqe->wr.wr.mw.mr_lkey >> 8);
++ mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);
+ if (unlikely(!mr)) {
+ ret = -EINVAL;
+ goto err_drop_mw;
+ }
+
+- if (unlikely(mr->ibmr.lkey != wqe->wr.wr.mw.mr_lkey)) {
++ if (unlikely(mr->lkey != mr_lkey)) {
+ ret = -EINVAL;
+ goto err_drop_mr;
+ }
+@@ -292,7 +290,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
+ goto err;
+ }
+
+- if (rkey != mw->ibmw.rkey) {
++ if (rkey != mw->rkey) {
+ ret = -EINVAL;
+ goto err_drop_ref;
+ }
+@@ -323,7 +321,7 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
+ if (!mw)
+ return NULL;
+
+- if (unlikely((rxe_mw_rkey(mw) != rkey) || rxe_mw_pd(mw) != pd ||
++ if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
+ (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
+ (mw->length == 0) ||
+ (access && !(access & mw->access)) ||
+diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.c b/drivers/infiniband/sw/rxe/rxe_opcode.c
+index 3ef5a10a6efd8..47ebaac8f4754 100644
+--- a/drivers/infiniband/sw/rxe/rxe_opcode.c
++++ b/drivers/infiniband/sw/rxe/rxe_opcode.c
+@@ -117,7 +117,7 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = {
+ }
+ },
+ [IB_OPCODE_RC_SEND_MIDDLE] = {
+- .name = "IB_OPCODE_RC_SEND_MIDDLE]",
++ .name = "IB_OPCODE_RC_SEND_MIDDLE",
+ .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK
+ | RXE_MIDDLE_MASK,
+ .length = RXE_BTH_BYTES,
+diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
+index 742e6ec93686c..b5a70cbe94aac 100644
+--- a/drivers/infiniband/sw/rxe/rxe_param.h
++++ b/drivers/infiniband/sw/rxe/rxe_param.h
+@@ -113,7 +113,7 @@ enum rxe_device_param {
+ /* default/initial rxe port parameters */
+ enum rxe_port_param {
+ RXE_PORT_GID_TBL_LEN = 1024,
+- RXE_PORT_PORT_CAP_FLAGS = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP,
++ RXE_PORT_PORT_CAP_FLAGS = IB_PORT_CM_SUP,
+ RXE_PORT_MAX_MSG_SZ = 0x800000,
+ RXE_PORT_BAD_PKEY_CNTR = 0,
+ RXE_PORT_QKEY_VIOL_CNTR = 0,
+diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
+index 1ab6af7ddb254..ed326d82725cd 100644
+--- a/drivers/infiniband/sw/rxe/rxe_qp.c
++++ b/drivers/infiniband/sw/rxe/rxe_qp.c
+@@ -367,6 +367,7 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd,
+
+ err2:
+ rxe_queue_cleanup(qp->sq.queue);
++ qp->sq.queue = NULL;
+ err1:
+ qp->pd = NULL;
+ qp->rcq = NULL;
+diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
+index 3894197a82f62..fc996fd31e589 100644
+--- a/drivers/infiniband/sw/rxe/rxe_req.c
++++ b/drivers/infiniband/sw/rxe/rxe_req.c
+@@ -572,7 +572,6 @@ static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
+ {
+ u8 opcode = wqe->wr.opcode;
+- struct rxe_mr *mr;
+ u32 rkey;
+ int ret;
+
+@@ -590,14 +589,11 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
+ }
+ break;
+ case IB_WR_REG_MR:
+- mr = to_rmr(wqe->wr.wr.reg.mr);
+- rxe_add_ref(mr);
+- mr->state = RXE_MR_STATE_VALID;
+- mr->access = wqe->wr.wr.reg.access;
+- mr->ibmr.lkey = wqe->wr.wr.reg.key;
+- mr->ibmr.rkey = wqe->wr.wr.reg.key;
+- mr->iova = wqe->wr.wr.reg.mr->iova;
+- rxe_drop_ref(mr);
++ ret = rxe_reg_fast_mr(qp, wqe);
++ if (unlikely(ret)) {
++ wqe->status = IB_WC_LOC_QP_OP_ERR;
++ return ret;
++ }
+ break;
+ case IB_WR_BIND_MW:
+ ret = rxe_bind_mw(qp, wqe);
+diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
+index 5501227ddc650..8ed172ab0beb4 100644
+--- a/drivers/infiniband/sw/rxe/rxe_resp.c
++++ b/drivers/infiniband/sw/rxe/rxe_resp.c
+@@ -830,6 +830,10 @@ static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
+ return RESPST_ERR_INVALIDATE_RKEY;
+ }
+
++ if (pkt->mask & RXE_END_MASK)
++ /* We successfully processed this new request. */
++ qp->resp.msn++;
++
+ /* next expected psn, read handles this separately */
+ qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
+ qp->resp.ack_psn = qp->resp.psn;
+@@ -837,11 +841,9 @@ static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
+ qp->resp.opcode = pkt->opcode;
+ qp->resp.status = IB_WC_SUCCESS;
+
+- if (pkt->mask & RXE_COMP_MASK) {
+- /* We successfully processed this new request. */
+- qp->resp.msn++;
++ if (pkt->mask & RXE_COMP_MASK)
+ return RESPST_COMPLETE;
+- } else if (qp_type(qp) == IB_QPT_RC)
++ else if (qp_type(qp) == IB_QPT_RC)
+ return RESPST_ACKNOWLEDGE;
+ else
+ return RESPST_CLEANUP;
+diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
+index ac2a2148027f4..d90b1d77de347 100644
+--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
++++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
+@@ -313,6 +313,8 @@ struct rxe_mr {
+
+ struct ib_umem *umem;
+
++ u32 lkey;
++ u32 rkey;
+ enum rxe_mr_state state;
+ enum rxe_mr_type type;
+ u64 va;
+@@ -350,6 +352,7 @@ struct rxe_mw {
+ enum rxe_mw_state state;
+ struct rxe_qp *qp; /* Type 2 only */
+ struct rxe_mr *mr;
++ u32 rkey;
+ int access;
+ u64 addr;
+ u64 length;
+@@ -474,26 +477,11 @@ static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
+ return to_rpd(mr->ibmr.pd);
+ }
+
+-static inline u32 mr_lkey(struct rxe_mr *mr)
+-{
+- return mr->ibmr.lkey;
+-}
+-
+-static inline u32 mr_rkey(struct rxe_mr *mr)
+-{
+- return mr->ibmr.rkey;
+-}
+-
+ static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
+ {
+ return to_rpd(mw->ibmw.pd);
+ }
+
+-static inline u32 rxe_mw_rkey(struct rxe_mw *mw)
+-{
+- return mw->ibmw.rkey;
+-}
+-
+ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
+
+ void rxe_mc_cleanup(struct rxe_pool_entry *arg);
+diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h
+index 368959ae9a8cc..df03d84c6868a 100644
+--- a/drivers/infiniband/sw/siw/siw.h
++++ b/drivers/infiniband/sw/siw/siw.h
+@@ -644,14 +644,9 @@ static inline struct siw_sqe *orq_get_current(struct siw_qp *qp)
+ return &qp->orq[qp->orq_get % qp->attrs.orq_size];
+ }
+
+-static inline struct siw_sqe *orq_get_tail(struct siw_qp *qp)
+-{
+- return &qp->orq[qp->orq_put % qp->attrs.orq_size];
+-}
+-
+ static inline struct siw_sqe *orq_get_free(struct siw_qp *qp)
+ {
+- struct siw_sqe *orq_e = orq_get_tail(qp);
++ struct siw_sqe *orq_e = &qp->orq[qp->orq_put % qp->attrs.orq_size];
+
+ if (READ_ONCE(orq_e->flags) == 0)
+ return orq_e;
+diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c
+index 7a5ed86ffc9f9..18a64ccbb0e58 100644
+--- a/drivers/infiniband/sw/siw/siw_cm.c
++++ b/drivers/infiniband/sw/siw/siw_cm.c
+@@ -968,14 +968,15 @@ static void siw_accept_newconn(struct siw_cep *cep)
+
+ siw_cep_set_inuse(new_cep);
+ rv = siw_proc_mpareq(new_cep);
+- siw_cep_set_free(new_cep);
+-
+ if (rv != -EAGAIN) {
+ siw_cep_put(cep);
+ new_cep->listen_cep = NULL;
+- if (rv)
++ if (rv) {
++ siw_cep_set_free(new_cep);
+ goto error;
++ }
+ }
++ siw_cep_set_free(new_cep);
+ }
+ return;
+
+diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c
+index 60116f20653c7..875ea6f1b04a2 100644
+--- a/drivers/infiniband/sw/siw/siw_qp_rx.c
++++ b/drivers/infiniband/sw/siw/siw_qp_rx.c
+@@ -1153,11 +1153,12 @@ static int siw_check_tx_fence(struct siw_qp *qp)
+
+ spin_lock_irqsave(&qp->orq_lock, flags);
+
+- rreq = orq_get_current(qp);
+-
+ /* free current orq entry */
++ rreq = orq_get_current(qp);
+ WRITE_ONCE(rreq->flags, 0);
+
++ qp->orq_get++;
++
+ if (qp->tx_ctx.orq_fence) {
+ if (unlikely(tx_waiting->wr_status != SIW_WR_QUEUED)) {
+ pr_warn("siw: [QP %u]: fence resume: bad status %d\n",
+@@ -1165,10 +1166,12 @@ static int siw_check_tx_fence(struct siw_qp *qp)
+ rv = -EPROTO;
+ goto out;
+ }
+- /* resume SQ processing */
++ /* resume SQ processing, if possible */
+ if (tx_waiting->sqe.opcode == SIW_OP_READ ||
+ tx_waiting->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
+- rreq = orq_get_tail(qp);
++
++ /* SQ processing was stopped because of a full ORQ */
++ rreq = orq_get_free(qp);
+ if (unlikely(!rreq)) {
+ pr_warn("siw: [QP %u]: no ORQE\n", qp_id(qp));
+ rv = -EPROTO;
+@@ -1181,15 +1184,14 @@ static int siw_check_tx_fence(struct siw_qp *qp)
+ resume_tx = 1;
+
+ } else if (siw_orq_empty(qp)) {
++ /*
++ * SQ processing was stopped by fenced work request.
++ * Resume since all previous Read's are now completed.
++ */
+ qp->tx_ctx.orq_fence = 0;
+ resume_tx = 1;
+- } else {
+- pr_warn("siw: [QP %u]: fence resume: orq idx: %d:%d\n",
+- qp_id(qp), qp->orq_get, qp->orq_put);
+- rv = -EPROTO;
+ }
+ }
+- qp->orq_get++;
+ out:
+ spin_unlock_irqrestore(&qp->orq_lock, flags);
+
+diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
+index 1b36350601faa..aa3f60d54a70f 100644
+--- a/drivers/infiniband/sw/siw/siw_verbs.c
++++ b/drivers/infiniband/sw/siw/siw_verbs.c
+@@ -311,7 +311,8 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+
+ if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
+ siw_dbg(base_dev, "too many QP's\n");
+- return -ENOMEM;
++ rv = -ENOMEM;
++ goto err_atomic;
+ }
+ if (attrs->qp_type != IB_QPT_RC) {
+ siw_dbg(base_dev, "only RC QP's supported\n");
+diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+index bc8824b4ee0d4..a23438bacf12c 100644
+--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
++++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+@@ -867,7 +867,7 @@ static struct rtrs_clt_sess *get_next_path_min_latency(struct path_it *it)
+ struct rtrs_clt_sess *min_path = NULL;
+ struct rtrs_clt *clt = it->clt;
+ struct rtrs_clt_sess *sess;
+- ktime_t min_latency = INT_MAX;
++ ktime_t min_latency = KTIME_MAX;
+ ktime_t latency;
+
+ list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) {
+@@ -2664,6 +2664,8 @@ static void rtrs_clt_dev_release(struct device *dev)
+ {
+ struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
+
++ mutex_destroy(&clt->paths_ev_mutex);
++ mutex_destroy(&clt->paths_mutex);
+ kfree(clt);
+ }
+
+@@ -2693,6 +2695,8 @@ static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,
+ return ERR_PTR(-ENOMEM);
+ }
+
++ clt->dev.class = rtrs_clt_dev_class;
++ clt->dev.release = rtrs_clt_dev_release;
+ uuid_gen(&clt->paths_uuid);
+ INIT_LIST_HEAD_RCU(&clt->paths_list);
+ clt->paths_num = paths_num;
+@@ -2709,53 +2713,51 @@ static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,
+ init_waitqueue_head(&clt->permits_wait);
+ mutex_init(&clt->paths_ev_mutex);
+ mutex_init(&clt->paths_mutex);
++ device_initialize(&clt->dev);
+
+- clt->dev.class = rtrs_clt_dev_class;
+- clt->dev.release = rtrs_clt_dev_release;
+ err = dev_set_name(&clt->dev, "%s", sessname);
+ if (err)
+- goto err;
++ goto err_put;
++
+ /*
+ * Suppress user space notification until
+ * sysfs files are created
+ */
+ dev_set_uevent_suppress(&clt->dev, true);
+- err = device_register(&clt->dev);
+- if (err) {
+- put_device(&clt->dev);
+- goto err;
+- }
++ err = device_add(&clt->dev);
++ if (err)
++ goto err_put;
+
+ clt->kobj_paths = kobject_create_and_add("paths", &clt->dev.kobj);
+ if (!clt->kobj_paths) {
+ err = -ENOMEM;
+- goto err_dev;
++ goto err_del;
+ }
+ err = rtrs_clt_create_sysfs_root_files(clt);
+ if (err) {
+ kobject_del(clt->kobj_paths);
+ kobject_put(clt->kobj_paths);
+- goto err_dev;
++ goto err_del;
+ }
+ dev_set_uevent_suppress(&clt->dev, false);
+ kobject_uevent(&clt->dev.kobj, KOBJ_ADD);
+
+ return clt;
+-err_dev:
+- device_unregister(&clt->dev);
+-err:
++err_del:
++ device_del(&clt->dev);
++err_put:
+ free_percpu(clt->pcpu_path);
+- kfree(clt);
++ put_device(&clt->dev);
+ return ERR_PTR(err);
+ }
+
+ static void free_clt(struct rtrs_clt *clt)
+ {
+- free_permits(clt);
+ free_percpu(clt->pcpu_path);
+- mutex_destroy(&clt->paths_ev_mutex);
+- mutex_destroy(&clt->paths_mutex);
+- /* release callback will free clt in last put */
++
++ /*
++ * release callback will free clt and destroy mutexes in last put
++ */
+ device_unregister(&clt->dev);
+ }
+
+@@ -2866,6 +2868,7 @@ void rtrs_clt_close(struct rtrs_clt *clt)
+ rtrs_clt_destroy_sess_files(sess, NULL);
+ kobject_put(&sess->kobj);
+ }
++ free_permits(clt);
+ free_clt(clt);
+ }
+ EXPORT_SYMBOL(rtrs_clt_close);
+diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
+index 71eda91e810cf..5d416ec228717 100644
+--- a/drivers/infiniband/ulp/srp/ib_srp.c
++++ b/drivers/infiniband/ulp/srp/ib_srp.c
+@@ -4038,9 +4038,11 @@ static void srp_remove_one(struct ib_device *device, void *client_data)
+ spin_unlock(&host->target_lock);
+
+ /*
+- * Wait for tl_err and target port removal tasks.
++ * srp_queue_remove_work() queues a call to
++ * srp_remove_target(). The latter function cancels
++ * target->tl_err_work so waiting for the remove works to
++ * finish is sufficient.
+ */
+- flush_workqueue(system_long_wq);
+ flush_workqueue(srp_remove_wq);
+
+ kfree(host);
+diff --git a/drivers/input/input.c b/drivers/input/input.c
+index ccaeb24263854..ba246fabc6c17 100644
+--- a/drivers/input/input.c
++++ b/drivers/input/input.c
+@@ -47,6 +47,17 @@ static DEFINE_MUTEX(input_mutex);
+
+ static const struct input_value input_value_sync = { EV_SYN, SYN_REPORT, 1 };
+
++static const unsigned int input_max_code[EV_CNT] = {
++ [EV_KEY] = KEY_MAX,
++ [EV_REL] = REL_MAX,
++ [EV_ABS] = ABS_MAX,
++ [EV_MSC] = MSC_MAX,
++ [EV_SW] = SW_MAX,
++ [EV_LED] = LED_MAX,
++ [EV_SND] = SND_MAX,
++ [EV_FF] = FF_MAX,
++};
++
+ static inline int is_event_supported(unsigned int code,
+ unsigned long *bm, unsigned int max)
+ {
+@@ -2074,6 +2085,14 @@ EXPORT_SYMBOL(input_get_timestamp);
+ */
+ void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code)
+ {
++ if (type < EV_CNT && input_max_code[type] &&
++ code > input_max_code[type]) {
++ pr_err("%s: invalid code %u for type %u\n", __func__, code,
++ type);
++ dump_stack();
++ return;
++ }
++
+ switch (type) {
+ case EV_KEY:
+ __set_bit(code, dev->keybit);
+diff --git a/drivers/input/joystick/iforce/iforce-usb.c b/drivers/input/joystick/iforce/iforce-usb.c
+index 6c554c11a7ac3..ea58805c480fa 100644
+--- a/drivers/input/joystick/iforce/iforce-usb.c
++++ b/drivers/input/joystick/iforce/iforce-usb.c
+@@ -92,7 +92,7 @@ static int iforce_usb_get_id(struct iforce *iforce, u8 id,
+ id,
+ USB_TYPE_VENDOR | USB_DIR_IN |
+ USB_RECIP_INTERFACE,
+- 0, 0, buf, IFORCE_MAX_LENGTH, HZ);
++ 0, 0, buf, IFORCE_MAX_LENGTH, 1000);
+ if (status < 0) {
+ dev_err(&iforce_usb->intf->dev,
+ "usb_submit_urb failed: %d\n", status);
+diff --git a/drivers/input/joystick/spaceball.c b/drivers/input/joystick/spaceball.c
+index 429411c6c0a8e..a85a4f33aea8c 100644
+--- a/drivers/input/joystick/spaceball.c
++++ b/drivers/input/joystick/spaceball.c
+@@ -19,6 +19,7 @@
+ #include <linux/module.h>
+ #include <linux/input.h>
+ #include <linux/serio.h>
++#include <asm/unaligned.h>
+
+ #define DRIVER_DESC "SpaceTec SpaceBall 2003/3003/4000 FLX driver"
+
+@@ -75,9 +76,15 @@ static void spaceball_process_packet(struct spaceball* spaceball)
+
+ case 'D': /* Ball data */
+ if (spaceball->idx != 15) return;
+- for (i = 0; i < 6; i++)
++ /*
++ * Skip first three bytes; read six axes worth of data.
++ * Axis values are signed 16-bit big-endian.
++ */
++ data += 3;
++ for (i = 0; i < ARRAY_SIZE(spaceball_axes); i++) {
+ input_report_abs(dev, spaceball_axes[i],
+- (__s16)((data[2 * i + 3] << 8) | data[2 * i + 2]));
++ (__s16)get_unaligned_be16(&data[i * 2]));
++ }
+ break;
+
+ case 'K': /* Button data */
+diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig
+index e75650e98c9ef..e402915cc0c00 100644
+--- a/drivers/input/keyboard/Kconfig
++++ b/drivers/input/keyboard/Kconfig
+@@ -556,7 +556,7 @@ config KEYBOARD_PMIC8XXX
+
+ config KEYBOARD_SAMSUNG
+ tristate "Samsung keypad support"
+- depends on HAVE_CLK
++ depends on HAS_IOMEM && HAVE_CLK
+ select INPUT_MATRIXKMAP
+ help
+ Say Y here if you want to use the keypad on your Samsung mobile
+diff --git a/drivers/input/keyboard/omap4-keypad.c b/drivers/input/keyboard/omap4-keypad.c
+index 43375b38ee592..8a7ce41b8c56e 100644
+--- a/drivers/input/keyboard/omap4-keypad.c
++++ b/drivers/input/keyboard/omap4-keypad.c
+@@ -393,7 +393,7 @@ static int omap4_keypad_probe(struct platform_device *pdev)
+ * revision register.
+ */
+ error = pm_runtime_get_sync(dev);
+- if (error) {
++ if (error < 0) {
+ dev_err(dev, "pm_runtime_get_sync() failed\n");
+ pm_runtime_put_noidle(dev);
+ return error;
+diff --git a/drivers/input/misc/ariel-pwrbutton.c b/drivers/input/misc/ariel-pwrbutton.c
+index 17bbaac8b80c8..cdc80715b5fd6 100644
+--- a/drivers/input/misc/ariel-pwrbutton.c
++++ b/drivers/input/misc/ariel-pwrbutton.c
+@@ -149,12 +149,19 @@ static const struct of_device_id ariel_pwrbutton_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, ariel_pwrbutton_of_match);
+
++static const struct spi_device_id ariel_pwrbutton_spi_ids[] = {
++ { .name = "wyse-ariel-ec-input" },
++ { }
++};
++MODULE_DEVICE_TABLE(spi, ariel_pwrbutton_spi_ids);
++
+ static struct spi_driver ariel_pwrbutton_driver = {
+ .driver = {
+ .name = "dell-wyse-ariel-ec-input",
+ .of_match_table = ariel_pwrbutton_of_match,
+ },
+ .probe = ariel_pwrbutton_probe,
++ .id_table = ariel_pwrbutton_spi_ids,
+ };
+ module_spi_driver(ariel_pwrbutton_driver);
+
+diff --git a/drivers/input/misc/iqs626a.c b/drivers/input/misc/iqs626a.c
+index d57e996732cf4..23b5dd9552dcc 100644
+--- a/drivers/input/misc/iqs626a.c
++++ b/drivers/input/misc/iqs626a.c
+@@ -456,9 +456,10 @@ struct iqs626_private {
+ unsigned int suspend_mode;
+ };
+
+-static int iqs626_parse_events(struct iqs626_private *iqs626,
+- const struct fwnode_handle *ch_node,
+- enum iqs626_ch_id ch_id)
++static noinline_for_stack int
++iqs626_parse_events(struct iqs626_private *iqs626,
++ const struct fwnode_handle *ch_node,
++ enum iqs626_ch_id ch_id)
+ {
+ struct iqs626_sys_reg *sys_reg = &iqs626->sys_reg;
+ struct i2c_client *client = iqs626->client;
+@@ -604,9 +605,10 @@ static int iqs626_parse_events(struct iqs626_private *iqs626,
+ return 0;
+ }
+
+-static int iqs626_parse_ati_target(struct iqs626_private *iqs626,
+- const struct fwnode_handle *ch_node,
+- enum iqs626_ch_id ch_id)
++static noinline_for_stack int
++iqs626_parse_ati_target(struct iqs626_private *iqs626,
++ const struct fwnode_handle *ch_node,
++ enum iqs626_ch_id ch_id)
+ {
+ struct iqs626_sys_reg *sys_reg = &iqs626->sys_reg;
+ struct i2c_client *client = iqs626->client;
+@@ -885,9 +887,10 @@ static int iqs626_parse_trackpad(struct iqs626_private *iqs626,
+ return 0;
+ }
+
+-static int iqs626_parse_channel(struct iqs626_private *iqs626,
+- const struct fwnode_handle *ch_node,
+- enum iqs626_ch_id ch_id)
++static noinline_for_stack int
++iqs626_parse_channel(struct iqs626_private *iqs626,
++ const struct fwnode_handle *ch_node,
++ enum iqs626_ch_id ch_id)
+ {
+ struct iqs626_sys_reg *sys_reg = &iqs626->sys_reg;
+ struct i2c_client *client = iqs626->client;
+diff --git a/drivers/input/mouse/appletouch.c b/drivers/input/mouse/appletouch.c
+index bfa26651c0be7..627048bc6a12e 100644
+--- a/drivers/input/mouse/appletouch.c
++++ b/drivers/input/mouse/appletouch.c
+@@ -916,6 +916,8 @@ static int atp_probe(struct usb_interface *iface,
+ set_bit(BTN_TOOL_TRIPLETAP, input_dev->keybit);
+ set_bit(BTN_LEFT, input_dev->keybit);
+
++ INIT_WORK(&dev->work, atp_reinit);
++
+ error = input_register_device(dev->input);
+ if (error)
+ goto err_free_buffer;
+@@ -923,8 +925,6 @@ static int atp_probe(struct usb_interface *iface,
+ /* save our data pointer in this interface device */
+ usb_set_intfdata(iface, dev);
+
+- INIT_WORK(&dev->work, atp_reinit);
+-
+ return 0;
+
+ err_free_buffer:
+diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
+index 47af62c122672..e1758d5ffe421 100644
+--- a/drivers/input/mouse/elan_i2c_core.c
++++ b/drivers/input/mouse/elan_i2c_core.c
+@@ -186,55 +186,21 @@ static int elan_get_fwinfo(u16 ic_type, u8 iap_version, u16 *validpage_count,
+ return 0;
+ }
+
+-static int elan_enable_power(struct elan_tp_data *data)
++static int elan_set_power(struct elan_tp_data *data, bool on)
+ {
+ int repeat = ETP_RETRY_COUNT;
+ int error;
+
+- error = regulator_enable(data->vcc);
+- if (error) {
+- dev_err(&data->client->dev,
+- "failed to enable regulator: %d\n", error);
+- return error;
+- }
+-
+ do {
+- error = data->ops->power_control(data->client, true);
++ error = data->ops->power_control(data->client, on);
+ if (error >= 0)
+ return 0;
+
+ msleep(30);
+ } while (--repeat > 0);
+
+- dev_err(&data->client->dev, "failed to enable power: %d\n", error);
+- return error;
+-}
+-
+-static int elan_disable_power(struct elan_tp_data *data)
+-{
+- int repeat = ETP_RETRY_COUNT;
+- int error;
+-
+- do {
+- error = data->ops->power_control(data->client, false);
+- if (!error) {
+- error = regulator_disable(data->vcc);
+- if (error) {
+- dev_err(&data->client->dev,
+- "failed to disable regulator: %d\n",
+- error);
+- /* Attempt to power the chip back up */
+- data->ops->power_control(data->client, true);
+- break;
+- }
+-
+- return 0;
+- }
+-
+- msleep(30);
+- } while (--repeat > 0);
+-
+- dev_err(&data->client->dev, "failed to disable power: %d\n", error);
++ dev_err(&data->client->dev, "failed to set power %s: %d\n",
++ on ? "on" : "off", error);
+ return error;
+ }
+
+@@ -1399,9 +1365,19 @@ static int __maybe_unused elan_suspend(struct device *dev)
+ /* Enable wake from IRQ */
+ data->irq_wake = (enable_irq_wake(client->irq) == 0);
+ } else {
+- ret = elan_disable_power(data);
++ ret = elan_set_power(data, false);
++ if (ret)
++ goto err;
++
++ ret = regulator_disable(data->vcc);
++ if (ret) {
++ dev_err(dev, "error %d disabling regulator\n", ret);
++ /* Attempt to power the chip back up */
++ elan_set_power(data, true);
++ }
+ }
+
++err:
+ mutex_unlock(&data->sysfs_mutex);
+ return ret;
+ }
+@@ -1412,12 +1388,18 @@ static int __maybe_unused elan_resume(struct device *dev)
+ struct elan_tp_data *data = i2c_get_clientdata(client);
+ int error;
+
+- if (device_may_wakeup(dev) && data->irq_wake) {
++ if (!device_may_wakeup(dev)) {
++ error = regulator_enable(data->vcc);
++ if (error) {
++ dev_err(dev, "error %d enabling regulator\n", error);
++ goto err;
++ }
++ } else if (data->irq_wake) {
+ disable_irq_wake(client->irq);
+ data->irq_wake = false;
+ }
+
+- error = elan_enable_power(data);
++ error = elan_set_power(data, true);
+ if (error) {
+ dev_err(dev, "power up when resuming failed: %d\n", error);
+ goto err;
+diff --git a/drivers/input/mouse/elantech.c b/drivers/input/mouse/elantech.c
+index 2d0bc029619ff..ece97f8c6a3e3 100644
+--- a/drivers/input/mouse/elantech.c
++++ b/drivers/input/mouse/elantech.c
+@@ -517,6 +517,19 @@ static void elantech_report_trackpoint(struct psmouse *psmouse,
+ case 0x16008020U:
+ case 0x26800010U:
+ case 0x36808000U:
++
++ /*
++ * This firmware misreport coordinates for trackpoint
++ * occasionally. Discard packets outside of [-127, 127] range
++ * to prevent cursor jumps.
++ */
++ if (packet[4] == 0x80 || packet[5] == 0x80 ||
++ packet[1] >> 7 == packet[4] >> 7 ||
++ packet[2] >> 7 == packet[5] >> 7) {
++ elantech_debug("discarding packet [%6ph]\n", packet);
++ break;
++
++ }
+ x = packet[4] - (int)((packet[1]^0x80) << 1);
+ y = (int)((packet[2]^0x80) << 1) - packet[5];
+
+@@ -1575,7 +1588,13 @@ static const struct dmi_system_id no_hw_res_dmi_table[] = {
+ */
+ static int elantech_change_report_id(struct psmouse *psmouse)
+ {
+- unsigned char param[2] = { 0x10, 0x03 };
++ /*
++ * NOTE: the code is expecting to receive param[] as an array of 3
++ * items (see __ps2_command()), even if in this case only 2 are
++ * actually needed. Make sure the array size is 3 to avoid potential
++ * stack out-of-bound accesses.
++ */
++ unsigned char param[3] = { 0x10, 0x03 };
+
+ if (elantech_write_reg_params(psmouse, 0x7, param) ||
+ elantech_read_reg_params(psmouse, 0x7, param) ||
+diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h
+index a5a0035536462..148a7c5fd0e22 100644
+--- a/drivers/input/serio/i8042-x86ia64io.h
++++ b/drivers/input/serio/i8042-x86ia64io.h
+@@ -272,6 +272,13 @@ static const struct dmi_system_id __initconst i8042_dmi_nomux_table[] = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "LifeBook S6230"),
+ },
+ },
++ {
++ /* Fujitsu Lifebook T725 laptop */
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "LIFEBOOK T725"),
++ },
++ },
+ {
+ /* Fujitsu Lifebook U745 */
+ .matches = {
+@@ -840,6 +847,13 @@ static const struct dmi_system_id __initconst i8042_dmi_notimeout_table[] = {
+ DMI_MATCH(DMI_PRODUCT_NAME, "LIFEBOOK AH544"),
+ },
+ },
++ {
++ /* Fujitsu Lifebook T725 laptop */
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "LIFEBOOK T725"),
++ },
++ },
+ {
+ /* Fujitsu U574 laptop */
+ /* https://bugzilla.kernel.org/show_bug.cgi?id=69731 */
+@@ -981,6 +995,24 @@ static const struct dmi_system_id __initconst i8042_dmi_kbdreset_table[] = {
+ { }
+ };
+
++static const struct dmi_system_id i8042_dmi_probe_defer_table[] __initconst = {
++ {
++ /* ASUS ZenBook UX425UA */
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX425UA"),
++ },
++ },
++ {
++ /* ASUS ZenBook UM325UA */
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
++ DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX325UA_UM325UA"),
++ },
++ },
++ { }
++};
++
+ #endif /* CONFIG_X86 */
+
+ #ifdef CONFIG_PNP
+@@ -1301,6 +1333,9 @@ static int __init i8042_platform_init(void)
+ if (dmi_check_system(i8042_dmi_kbdreset_table))
+ i8042_kbdreset = true;
+
++ if (dmi_check_system(i8042_dmi_probe_defer_table))
++ i8042_probe_defer = true;
++
+ /*
+ * A20 was already enabled during early kernel init. But some buggy
+ * BIOSes (in MSI Laptops) require A20 to be enabled using 8042 to
+diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c
+index 0b9f1d0a8f8b0..3fc0a89cc785c 100644
+--- a/drivers/input/serio/i8042.c
++++ b/drivers/input/serio/i8042.c
+@@ -45,6 +45,10 @@ static bool i8042_unlock;
+ module_param_named(unlock, i8042_unlock, bool, 0);
+ MODULE_PARM_DESC(unlock, "Ignore keyboard lock.");
+
++static bool i8042_probe_defer;
++module_param_named(probe_defer, i8042_probe_defer, bool, 0);
++MODULE_PARM_DESC(probe_defer, "Allow deferred probing.");
++
+ enum i8042_controller_reset_mode {
+ I8042_RESET_NEVER,
+ I8042_RESET_ALWAYS,
+@@ -711,7 +715,7 @@ static int i8042_set_mux_mode(bool multiplex, unsigned char *mux_version)
+ * LCS/Telegraphics.
+ */
+
+-static int __init i8042_check_mux(void)
++static int i8042_check_mux(void)
+ {
+ unsigned char mux_version;
+
+@@ -740,10 +744,10 @@ static int __init i8042_check_mux(void)
+ /*
+ * The following is used to test AUX IRQ delivery.
+ */
+-static struct completion i8042_aux_irq_delivered __initdata;
+-static bool i8042_irq_being_tested __initdata;
++static struct completion i8042_aux_irq_delivered;
++static bool i8042_irq_being_tested;
+
+-static irqreturn_t __init i8042_aux_test_irq(int irq, void *dev_id)
++static irqreturn_t i8042_aux_test_irq(int irq, void *dev_id)
+ {
+ unsigned long flags;
+ unsigned char str, data;
+@@ -770,7 +774,7 @@ static irqreturn_t __init i8042_aux_test_irq(int irq, void *dev_id)
+ * verifies success by readinng CTR. Used when testing for presence of AUX
+ * port.
+ */
+-static int __init i8042_toggle_aux(bool on)
++static int i8042_toggle_aux(bool on)
+ {
+ unsigned char param;
+ int i;
+@@ -798,7 +802,7 @@ static int __init i8042_toggle_aux(bool on)
+ * the presence of an AUX interface.
+ */
+
+-static int __init i8042_check_aux(void)
++static int i8042_check_aux(void)
+ {
+ int retval = -1;
+ bool irq_registered = false;
+@@ -1005,7 +1009,7 @@ static int i8042_controller_init(void)
+
+ if (i8042_command(&ctr[n++ % 2], I8042_CMD_CTL_RCTR)) {
+ pr_err("Can't read CTR while initializing i8042\n");
+- return -EIO;
++ return i8042_probe_defer ? -EPROBE_DEFER : -EIO;
+ }
+
+ } while (n < 2 || ctr[0] != ctr[1]);
+@@ -1320,7 +1324,7 @@ static void i8042_shutdown(struct platform_device *dev)
+ i8042_controller_reset(false);
+ }
+
+-static int __init i8042_create_kbd_port(void)
++static int i8042_create_kbd_port(void)
+ {
+ struct serio *serio;
+ struct i8042_port *port = &i8042_ports[I8042_KBD_PORT_NO];
+@@ -1349,7 +1353,7 @@ static int __init i8042_create_kbd_port(void)
+ return 0;
+ }
+
+-static int __init i8042_create_aux_port(int idx)
++static int i8042_create_aux_port(int idx)
+ {
+ struct serio *serio;
+ int port_no = idx < 0 ? I8042_AUX_PORT_NO : I8042_MUX_PORT_NO + idx;
+@@ -1386,13 +1390,13 @@ static int __init i8042_create_aux_port(int idx)
+ return 0;
+ }
+
+-static void __init i8042_free_kbd_port(void)
++static void i8042_free_kbd_port(void)
+ {
+ kfree(i8042_ports[I8042_KBD_PORT_NO].serio);
+ i8042_ports[I8042_KBD_PORT_NO].serio = NULL;
+ }
+
+-static void __init i8042_free_aux_ports(void)
++static void i8042_free_aux_ports(void)
+ {
+ int i;
+
+@@ -1402,7 +1406,7 @@ static void __init i8042_free_aux_ports(void)
+ }
+ }
+
+-static void __init i8042_register_ports(void)
++static void i8042_register_ports(void)
+ {
+ int i;
+
+@@ -1443,7 +1447,7 @@ static void i8042_free_irqs(void)
+ i8042_aux_irq_registered = i8042_kbd_irq_registered = false;
+ }
+
+-static int __init i8042_setup_aux(void)
++static int i8042_setup_aux(void)
+ {
+ int (*aux_enable)(void);
+ int error;
+@@ -1485,7 +1489,7 @@ static int __init i8042_setup_aux(void)
+ return error;
+ }
+
+-static int __init i8042_setup_kbd(void)
++static int i8042_setup_kbd(void)
+ {
+ int error;
+
+@@ -1535,7 +1539,7 @@ static int i8042_kbd_bind_notifier(struct notifier_block *nb,
+ return 0;
+ }
+
+-static int __init i8042_probe(struct platform_device *dev)
++static int i8042_probe(struct platform_device *dev)
+ {
+ int error;
+
+@@ -1600,6 +1604,7 @@ static struct platform_driver i8042_driver = {
+ .pm = &i8042_pm_ops,
+ #endif
+ },
++ .probe = i8042_probe,
+ .remove = i8042_remove,
+ .shutdown = i8042_shutdown,
+ };
+@@ -1610,7 +1615,6 @@ static struct notifier_block i8042_kbd_bind_notifier_block = {
+
+ static int __init i8042_init(void)
+ {
+- struct platform_device *pdev;
+ int err;
+
+ dbg_init();
+@@ -1626,17 +1630,29 @@ static int __init i8042_init(void)
+ /* Set this before creating the dev to allow i8042_command to work right away */
+ i8042_present = true;
+
+- pdev = platform_create_bundle(&i8042_driver, i8042_probe, NULL, 0, NULL, 0);
+- if (IS_ERR(pdev)) {
+- err = PTR_ERR(pdev);
++ err = platform_driver_register(&i8042_driver);
++ if (err)
+ goto err_platform_exit;
++
++ i8042_platform_device = platform_device_alloc("i8042", -1);
++ if (!i8042_platform_device) {
++ err = -ENOMEM;
++ goto err_unregister_driver;
+ }
+
++ err = platform_device_add(i8042_platform_device);
++ if (err)
++ goto err_free_device;
++
+ bus_register_notifier(&serio_bus, &i8042_kbd_bind_notifier_block);
+ panic_blink = i8042_panic_blink;
+
+ return 0;
+
++err_free_device:
++ platform_device_put(i8042_platform_device);
++err_unregister_driver:
++ platform_driver_unregister(&i8042_driver);
+ err_platform_exit:
+ i8042_platform_exit();
+ return err;
+diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c
+index fcb1b646436a5..1581f6ef09279 100644
+--- a/drivers/input/tablet/aiptek.c
++++ b/drivers/input/tablet/aiptek.c
+@@ -1787,15 +1787,13 @@ aiptek_probe(struct usb_interface *intf, const struct usb_device_id *id)
+ input_set_abs_params(inputdev, ABS_TILT_Y, AIPTEK_TILT_MIN, AIPTEK_TILT_MAX, 0, 0);
+ input_set_abs_params(inputdev, ABS_WHEEL, AIPTEK_WHEEL_MIN, AIPTEK_WHEEL_MAX - 1, 0, 0);
+
+- /* Verify that a device really has an endpoint */
+- if (intf->cur_altsetting->desc.bNumEndpoints < 1) {
++ err = usb_find_common_endpoints(intf->cur_altsetting,
++ NULL, NULL, &endpoint, NULL);
++ if (err) {
+ dev_err(&intf->dev,
+- "interface has %d endpoints, but must have minimum 1\n",
+- intf->cur_altsetting->desc.bNumEndpoints);
+- err = -EINVAL;
++ "interface has no int in endpoints, but must have minimum 1\n");
+ goto fail3;
+ }
+- endpoint = &intf->cur_altsetting->endpoint[0].desc;
+
+ /* Go set up our URB, which is called when the tablet receives
+ * input.
+diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
+index 05de92c0293bc..eb66cd2689b7c 100644
+--- a/drivers/input/touchscreen/atmel_mxt_ts.c
++++ b/drivers/input/touchscreen/atmel_mxt_ts.c
+@@ -1882,7 +1882,7 @@ static int mxt_read_info_block(struct mxt_data *data)
+ if (error) {
+ dev_err(&client->dev, "Error %d parsing object table\n", error);
+ mxt_free_object_table(data);
+- goto err_free_mem;
++ return error;
+ }
+
+ data->object_table = (struct mxt_object *)(id_buf + MXT_OBJECT_START);
+diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c
+index 68f542bb809f4..b9e2219efbb8f 100644
+--- a/drivers/input/touchscreen/elants_i2c.c
++++ b/drivers/input/touchscreen/elants_i2c.c
+@@ -117,6 +117,19 @@
+ #define ELAN_POWERON_DELAY_USEC 500
+ #define ELAN_RESET_DELAY_MSEC 20
+
++/* FW boot code version */
++#define BC_VER_H_BYTE_FOR_EKTH3900x1_I2C 0x72
++#define BC_VER_H_BYTE_FOR_EKTH3900x2_I2C 0x82
++#define BC_VER_H_BYTE_FOR_EKTH3900x3_I2C 0x92
++#define BC_VER_H_BYTE_FOR_EKTH5312x1_I2C 0x6D
++#define BC_VER_H_BYTE_FOR_EKTH5312x2_I2C 0x6E
++#define BC_VER_H_BYTE_FOR_EKTH5312cx1_I2C 0x77
++#define BC_VER_H_BYTE_FOR_EKTH5312cx2_I2C 0x78
++#define BC_VER_H_BYTE_FOR_EKTH5312x1_I2C_USB 0x67
++#define BC_VER_H_BYTE_FOR_EKTH5312x2_I2C_USB 0x68
++#define BC_VER_H_BYTE_FOR_EKTH5312cx1_I2C_USB 0x74
++#define BC_VER_H_BYTE_FOR_EKTH5312cx2_I2C_USB 0x75
++
+ enum elants_chip_id {
+ EKTH3500,
+ EKTF3624,
+@@ -736,6 +749,37 @@ static int elants_i2c_validate_remark_id(struct elants_data *ts,
+ return 0;
+ }
+
++static bool elants_i2c_should_check_remark_id(struct elants_data *ts)
++{
++ struct i2c_client *client = ts->client;
++ const u8 bootcode_version = ts->iap_version;
++ bool check;
++
++ /* I2C eKTH3900 and eKTH5312 are NOT support Remark ID */
++ if ((bootcode_version == BC_VER_H_BYTE_FOR_EKTH3900x1_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH3900x2_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH3900x3_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312x1_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312x2_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312cx1_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312cx2_I2C) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312x1_I2C_USB) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312x2_I2C_USB) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312cx1_I2C_USB) ||
++ (bootcode_version == BC_VER_H_BYTE_FOR_EKTH5312cx2_I2C_USB)) {
++ dev_dbg(&client->dev,
++ "eKTH3900/eKTH5312(0x%02x) are not support remark id\n",
++ bootcode_version);
++ check = false;
++ } else if (bootcode_version >= 0x60) {
++ check = true;
++ } else {
++ check = false;
++ }
++
++ return check;
++}
++
+ static int elants_i2c_do_update_firmware(struct i2c_client *client,
+ const struct firmware *fw,
+ bool force)
+@@ -749,7 +793,7 @@ static int elants_i2c_do_update_firmware(struct i2c_client *client,
+ u16 send_id;
+ int page, n_fw_pages;
+ int error;
+- bool check_remark_id = ts->iap_version >= 0x60;
++ bool check_remark_id = elants_i2c_should_check_remark_id(ts);
+
+ /* Recovery mode detection! */
+ if (force) {
+diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c
+index 4f53d3c57e698..5051a1766aac6 100644
+--- a/drivers/input/touchscreen/goodix.c
++++ b/drivers/input/touchscreen/goodix.c
+@@ -162,6 +162,7 @@ static const struct goodix_chip_id goodix_chip_ids[] = {
+ { .id = "911", .data = &gt911_chip_data },
+ { .id = "9271", .data = &gt911_chip_data },
+ { .id = "9110", .data = &gt911_chip_data },
++ { .id = "9111", .data = &gt911_chip_data },
+ { .id = "927", .data = &gt911_chip_data },
+ { .id = "928", .data = &gt911_chip_data },
+
+diff --git a/drivers/input/touchscreen/ili210x.c b/drivers/input/touchscreen/ili210x.c
+index 30576a5f2f045..f437eefec94ad 100644
+--- a/drivers/input/touchscreen/ili210x.c
++++ b/drivers/input/touchscreen/ili210x.c
+@@ -420,9 +420,9 @@ static int ili210x_i2c_probe(struct i2c_client *client,
+ if (error)
+ return error;
+
+- usleep_range(50, 100);
++ usleep_range(12000, 15000);
+ gpiod_set_value_cansleep(reset_gpio, 0);
+- msleep(100);
++ msleep(160);
+ }
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+diff --git a/drivers/input/touchscreen/st1232.c b/drivers/input/touchscreen/st1232.c
+index 6abae665ca71d..9d1dea6996a22 100644
+--- a/drivers/input/touchscreen/st1232.c
++++ b/drivers/input/touchscreen/st1232.c
+@@ -92,7 +92,7 @@ static int st1232_ts_wait_ready(struct st1232_ts_data *ts)
+ unsigned int retries;
+ int error;
+
+- for (retries = 10; retries; retries--) {
++ for (retries = 100; retries; retries--) {
+ error = st1232_ts_read_data(ts, REG_STATUS, 1);
+ if (!error) {
+ switch (ts->read_buf[0]) {
+diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
+index bc11203c9cf78..72e0b767e1ba4 100644
+--- a/drivers/input/touchscreen/stmfts.c
++++ b/drivers/input/touchscreen/stmfts.c
+@@ -339,11 +339,11 @@ static int stmfts_input_open(struct input_dev *dev)
+
+ err = pm_runtime_get_sync(&sdata->client->dev);
+ if (err < 0)
+- return err;
++ goto out;
+
+ err = i2c_smbus_write_byte(sdata->client, STMFTS_MS_MT_SENSE_ON);
+ if (err)
+- return err;
++ goto out;
+
+ mutex_lock(&sdata->mutex);
+ sdata->running = true;
+@@ -366,7 +366,9 @@ static int stmfts_input_open(struct input_dev *dev)
+ "failed to enable touchkey\n");
+ }
+
+- return 0;
++out:
++ pm_runtime_put_noidle(&sdata->client->dev);
++ return err;
+ }
+
+ static void stmfts_input_close(struct input_dev *dev)
+diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c
+index 83e685557a197..cfc943423241f 100644
+--- a/drivers/input/touchscreen/ti_am335x_tsc.c
++++ b/drivers/input/touchscreen/ti_am335x_tsc.c
+@@ -131,7 +131,8 @@ static void titsc_step_config(struct titsc *ts_dev)
+ u32 stepenable;
+
+ config = STEPCONFIG_MODE_HWSYNC |
+- STEPCONFIG_AVG_16 | ts_dev->bit_xp;
++ STEPCONFIG_AVG_16 | ts_dev->bit_xp |
++ STEPCONFIG_INM_ADCREFM;
+ switch (ts_dev->wires) {
+ case 4:
+ config |= STEPCONFIG_INP(ts_dev->inp_yp) | ts_dev->bit_xn;
+@@ -195,7 +196,10 @@ static void titsc_step_config(struct titsc *ts_dev)
+ STEPCONFIG_OPENDLY);
+
+ end_step++;
+- config |= STEPCONFIG_INP(ts_dev->inp_yn);
++ config = STEPCONFIG_MODE_HWSYNC |
++ STEPCONFIG_AVG_16 | ts_dev->bit_yp |
++ ts_dev->bit_xn | STEPCONFIG_INM_ADCREFM |
++ STEPCONFIG_INP(ts_dev->inp_yn);
+ titsc_writel(ts_dev, REG_STEPCONFIG(end_step), config);
+ titsc_writel(ts_dev, REG_STEPDELAY(end_step),
+ STEPCONFIG_OPENDLY);
+diff --git a/drivers/input/touchscreen/zinitix.c b/drivers/input/touchscreen/zinitix.c
+index b8d901099378d..400957f4c8c9c 100644
+--- a/drivers/input/touchscreen/zinitix.c
++++ b/drivers/input/touchscreen/zinitix.c
+@@ -135,7 +135,7 @@ struct point_coord {
+
+ struct touch_event {
+ __le16 status;
+- u8 finger_cnt;
++ u8 finger_mask;
+ u8 time_stamp;
+ struct point_coord point_coord[MAX_SUPPORTED_FINGER_NUM];
+ };
+@@ -311,11 +311,32 @@ static int zinitix_send_power_on_sequence(struct bt541_ts_data *bt541)
+ static void zinitix_report_finger(struct bt541_ts_data *bt541, int slot,
+ const struct point_coord *p)
+ {
++ u16 x, y;
++
++ if (unlikely(!(p->sub_status &
++ (SUB_BIT_UP | SUB_BIT_DOWN | SUB_BIT_MOVE)))) {
++ dev_dbg(&bt541->client->dev, "unknown finger event %#02x\n",
++ p->sub_status);
++ return;
++ }
++
++ x = le16_to_cpu(p->x);
++ y = le16_to_cpu(p->y);
++
+ input_mt_slot(bt541->input_dev, slot);
+- input_mt_report_slot_state(bt541->input_dev, MT_TOOL_FINGER, true);
+- touchscreen_report_pos(bt541->input_dev, &bt541->prop,
+- le16_to_cpu(p->x), le16_to_cpu(p->y), true);
+- input_report_abs(bt541->input_dev, ABS_MT_TOUCH_MAJOR, p->width);
++ if (input_mt_report_slot_state(bt541->input_dev, MT_TOOL_FINGER,
++ !(p->sub_status & SUB_BIT_UP))) {
++ touchscreen_report_pos(bt541->input_dev,
++ &bt541->prop, x, y, true);
++ input_report_abs(bt541->input_dev,
++ ABS_MT_TOUCH_MAJOR, p->width);
++ dev_dbg(&bt541->client->dev, "finger %d %s (%u, %u)\n",
++ slot, p->sub_status & SUB_BIT_DOWN ? "down" : "move",
++ x, y);
++ } else {
++ dev_dbg(&bt541->client->dev, "finger %d up (%u, %u)\n",
++ slot, x, y);
++ }
+ }
+
+ static irqreturn_t zinitix_ts_irq_handler(int irq, void *bt541_handler)
+@@ -323,6 +344,7 @@ static irqreturn_t zinitix_ts_irq_handler(int irq, void *bt541_handler)
+ struct bt541_ts_data *bt541 = bt541_handler;
+ struct i2c_client *client = bt541->client;
+ struct touch_event touch_event;
++ unsigned long finger_mask;
+ int error;
+ int i;
+
+@@ -335,10 +357,14 @@ static irqreturn_t zinitix_ts_irq_handler(int irq, void *bt541_handler)
+ goto out;
+ }
+
+- for (i = 0; i < MAX_SUPPORTED_FINGER_NUM; i++)
+- if (touch_event.point_coord[i].sub_status & SUB_BIT_EXIST)
+- zinitix_report_finger(bt541, i,
+- &touch_event.point_coord[i]);
++ finger_mask = touch_event.finger_mask;
++ for_each_set_bit(i, &finger_mask, MAX_SUPPORTED_FINGER_NUM) {
++ const struct point_coord *p = &touch_event.point_coord[i];
++
++ /* Only process contacts that are actually reported */
++ if (p->sub_status & SUB_BIT_EXIST)
++ zinitix_report_finger(bt541, i, p);
++ }
+
+ input_mt_sync_frame(bt541->input_dev);
+ input_sync(bt541->input_dev);
+@@ -488,6 +514,15 @@ static int zinitix_ts_probe(struct i2c_client *client)
+ return error;
+ }
+
++ error = devm_request_threaded_irq(&client->dev, client->irq,
++ NULL, zinitix_ts_irq_handler,
++ IRQF_ONESHOT | IRQF_NO_AUTOEN,
++ client->name, bt541);
++ if (error) {
++ dev_err(&client->dev, "Failed to request IRQ: %d\n", error);
++ return error;
++ }
++
+ error = zinitix_init_input_dev(bt541);
+ if (error) {
+ dev_err(&client->dev,
+@@ -513,15 +548,6 @@ static int zinitix_ts_probe(struct i2c_client *client)
+ return -EINVAL;
+ }
+
+- error = devm_request_threaded_irq(&client->dev, client->irq,
+- NULL, zinitix_ts_irq_handler,
+- IRQF_ONESHOT | IRQF_NO_AUTOEN,
+- client->name, bt541);
+- if (error) {
+- dev_err(&client->dev, "Failed to request IRQ: %d\n", error);
+- return error;
+- }
+-
+ return 0;
+ }
+
+diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
+index 9050ca1f4285c..808f6e7a80482 100644
+--- a/drivers/interconnect/core.c
++++ b/drivers/interconnect/core.c
+@@ -1087,9 +1087,15 @@ static int of_count_icc_providers(struct device_node *np)
+ {
+ struct device_node *child;
+ int count = 0;
++ const struct of_device_id __maybe_unused ignore_list[] = {
++ { .compatible = "qcom,sc7180-ipa-virt" },
++ { .compatible = "qcom,sdx55-ipa-virt" },
++ {}
++ };
+
+ for_each_available_child_of_node(np, child) {
+- if (of_property_read_bool(child, "#interconnect-cells"))
++ if (of_property_read_bool(child, "#interconnect-cells") &&
++ likely(!of_match_node(ignore_list, child)))
+ count++;
+ count += of_count_icc_providers(child);
+ }
+diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c
+index 54de49ca7808a..ddf1805ded0c0 100644
+--- a/drivers/interconnect/qcom/icc-rpm.c
++++ b/drivers/interconnect/qcom/icc-rpm.c
+@@ -68,6 +68,7 @@ static int qcom_icc_set(struct icc_node *src, struct icc_node *dst)
+ rate = max(sum_bw, max_peak_bw);
+
+ do_div(rate, qn->buswidth);
++ rate = min_t(u64, rate, LONG_MAX);
+
+ if (qn->rate == rate)
+ return 0;
+diff --git a/drivers/interconnect/qcom/sdx55.c b/drivers/interconnect/qcom/sdx55.c
+index 03d604f84cc57..e3ac25a997b71 100644
+--- a/drivers/interconnect/qcom/sdx55.c
++++ b/drivers/interconnect/qcom/sdx55.c
+@@ -18,7 +18,6 @@
+ #include "icc-rpmh.h"
+ #include "sdx55.h"
+
+-DEFINE_QNODE(ipa_core_master, SDX55_MASTER_IPA_CORE, 1, 8, SDX55_SLAVE_IPA_CORE);
+ DEFINE_QNODE(llcc_mc, SDX55_MASTER_LLCC, 4, 4, SDX55_SLAVE_EBI_CH0);
+ DEFINE_QNODE(acm_tcu, SDX55_MASTER_TCU_0, 1, 8, SDX55_SLAVE_LLCC, SDX55_SLAVE_MEM_NOC_SNOC, SDX55_SLAVE_MEM_NOC_PCIE_SNOC);
+ DEFINE_QNODE(qnm_snoc_gc, SDX55_MASTER_SNOC_GC_MEM_NOC, 1, 8, SDX55_SLAVE_LLCC);
+@@ -40,7 +39,6 @@ DEFINE_QNODE(xm_pcie, SDX55_MASTER_PCIE, 1, 8, SDX55_SLAVE_ANOC_SNOC);
+ DEFINE_QNODE(xm_qdss_etr, SDX55_MASTER_QDSS_ETR, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_AOSS, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG);
+ DEFINE_QNODE(xm_sdc1, SDX55_MASTER_SDCC_1, 1, 8, SDX55_SLAVE_AOSS, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP, SDX55_SLAVE_AUDIO);
+ DEFINE_QNODE(xm_usb3, SDX55_MASTER_USB3, 1, 8, SDX55_SLAVE_ANOC_SNOC);
+-DEFINE_QNODE(ipa_core_slave, SDX55_SLAVE_IPA_CORE, 1, 8);
+ DEFINE_QNODE(ebi, SDX55_SLAVE_EBI_CH0, 1, 4);
+ DEFINE_QNODE(qns_llcc, SDX55_SLAVE_LLCC, 1, 16, SDX55_SLAVE_EBI_CH0);
+ DEFINE_QNODE(qns_memnoc_snoc, SDX55_SLAVE_MEM_NOC_SNOC, 1, 8, SDX55_MASTER_MEM_NOC_SNOC);
+@@ -82,7 +80,6 @@ DEFINE_QNODE(xs_sys_tcu_cfg, SDX55_SLAVE_TCU, 1, 8);
+ DEFINE_QBCM(bcm_mc0, "MC0", true, &ebi);
+ DEFINE_QBCM(bcm_sh0, "SH0", true, &qns_llcc);
+ DEFINE_QBCM(bcm_ce0, "CE0", false, &qxm_crypto);
+-DEFINE_QBCM(bcm_ip0, "IP0", false, &ipa_core_slave);
+ DEFINE_QBCM(bcm_pn0, "PN0", false, &qhm_snoc_cfg);
+ DEFINE_QBCM(bcm_sh3, "SH3", false, &xm_apps_rdwr);
+ DEFINE_QBCM(bcm_sh4, "SH4", false, &qns_memnoc_snoc, &qns_sys_pcie);
+@@ -219,22 +216,6 @@ static const struct qcom_icc_desc sdx55_system_noc = {
+ .num_bcms = ARRAY_SIZE(system_noc_bcms),
+ };
+
+-static struct qcom_icc_bcm *ipa_virt_bcms[] = {
+- &bcm_ip0,
+-};
+-
+-static struct qcom_icc_node *ipa_virt_nodes[] = {
+- [MASTER_IPA_CORE] = &ipa_core_master,
+- [SLAVE_IPA_CORE] = &ipa_core_slave,
+-};
+-
+-static const struct qcom_icc_desc sdx55_ipa_virt = {
+- .nodes = ipa_virt_nodes,
+- .num_nodes = ARRAY_SIZE(ipa_virt_nodes),
+- .bcms = ipa_virt_bcms,
+- .num_bcms = ARRAY_SIZE(ipa_virt_bcms),
+-};
+-
+ static const struct of_device_id qnoc_of_match[] = {
+ { .compatible = "qcom,sdx55-mc-virt",
+ .data = &sdx55_mc_virt},
+@@ -242,8 +223,6 @@ static const struct of_device_id qnoc_of_match[] = {
+ .data = &sdx55_mem_noc},
+ { .compatible = "qcom,sdx55-system-noc",
+ .data = &sdx55_system_noc},
+- { .compatible = "qcom,sdx55-ipa-virt",
+- .data = &sdx55_ipa_virt},
+ { }
+ };
+ MODULE_DEVICE_TABLE(of, qnoc_of_match);
+diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
+index 416815a525d67..bb95edf74415b 100644
+--- a/drivers/iommu/amd/amd_iommu.h
++++ b/drivers/iommu/amd/amd_iommu.h
+@@ -14,6 +14,7 @@
+ extern irqreturn_t amd_iommu_int_thread(int irq, void *data);
+ extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
+ extern void amd_iommu_apply_erratum_63(u16 devid);
++extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu);
+ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
+ extern int amd_iommu_init_devices(void);
+ extern void amd_iommu_uninit_devices(void);
+diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
+index 8dbe61e2b3c15..b6e0bf186cf54 100644
+--- a/drivers/iommu/amd/amd_iommu_types.h
++++ b/drivers/iommu/amd/amd_iommu_types.h
+@@ -110,6 +110,7 @@
+ #define PASID_MASK 0x0000ffff
+
+ /* MMIO status bits */
++#define MMIO_STATUS_EVT_OVERFLOW_INT_MASK (1 << 0)
+ #define MMIO_STATUS_EVT_INT_MASK (1 << 1)
+ #define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2)
+ #define MMIO_STATUS_PPR_INT_MASK (1 << 6)
+@@ -643,8 +644,6 @@ struct amd_iommu {
+ /* DebugFS Info */
+ struct dentry *debugfs;
+ #endif
+- /* IRQ notifier for IntCapXT interrupt */
+- struct irq_affinity_notify intcapxt_notify;
+ };
+
+ static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
+diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
+index 2a822b229bd05..204ed33c56dce 100644
+--- a/drivers/iommu/amd/init.c
++++ b/drivers/iommu/amd/init.c
+@@ -21,6 +21,7 @@
+ #include <linux/export.h>
+ #include <linux/kmemleak.h>
+ #include <linux/mem_encrypt.h>
++#include <linux/iopoll.h>
+ #include <asm/pci-direct.h>
+ #include <asm/iommu.h>
+ #include <asm/apic.h>
+@@ -654,6 +655,16 @@ static int __init alloc_command_buffer(struct amd_iommu *iommu)
+ return iommu->cmd_buf ? 0 : -ENOMEM;
+ }
+
++/*
++ * This function restarts event logging in case the IOMMU experienced
++ * an event log buffer overflow.
++ */
++void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
++{
++ iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
++ iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
++}
++
+ /*
+ * This function resets the command buffer if the IOMMU stopped fetching
+ * commands from it.
+@@ -804,16 +815,27 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
+ {
+ #ifdef CONFIG_IRQ_REMAP
+ u32 status, i;
++ u64 entry;
+
+ if (!iommu->ga_log)
+ return -EINVAL;
+
+- status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+-
+ /* Check if already running */
+- if (status & (MMIO_STATUS_GALOG_RUN_MASK))
++ status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
++ if (WARN_ON(status & (MMIO_STATUS_GALOG_RUN_MASK)))
+ return 0;
+
++ entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
++ memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
++ &entry, sizeof(entry));
++ entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
++ (BIT_ULL(52)-1)) & ~7ULL;
++ memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
++ &entry, sizeof(entry));
++ writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
++ writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
++
++
+ iommu_feature_enable(iommu, CONTROL_GAINT_EN);
+ iommu_feature_enable(iommu, CONTROL_GALOG_EN);
+
+@@ -821,9 +843,10 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
+ status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+ if (status & (MMIO_STATUS_GALOG_RUN_MASK))
+ break;
++ udelay(10);
+ }
+
+- if (i >= LOOP_TIMEOUT)
++ if (WARN_ON(i >= LOOP_TIMEOUT))
+ return -EINVAL;
+ #endif /* CONFIG_IRQ_REMAP */
+ return 0;
+@@ -832,8 +855,6 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
+ static int iommu_init_ga_log(struct amd_iommu *iommu)
+ {
+ #ifdef CONFIG_IRQ_REMAP
+- u64 entry;
+-
+ if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+ return 0;
+
+@@ -847,16 +868,6 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
+ if (!iommu->ga_log_tail)
+ goto err_out;
+
+- entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
+- memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
+- &entry, sizeof(entry));
+- entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
+- (BIT_ULL(52)-1)) & ~7ULL;
+- memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
+- &entry, sizeof(entry));
+- writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
+- writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
+-
+ return 0;
+ err_out:
+ free_ga_log(iommu);
+@@ -2013,48 +2024,18 @@ union intcapxt {
+ };
+ } __attribute__ ((packed));
+
+-/*
+- * There isn't really any need to mask/unmask at the irqchip level because
+- * the 64-bit INTCAPXT registers can be updated atomically without tearing
+- * when the affinity is being updated.
+- */
+-static void intcapxt_unmask_irq(struct irq_data *data)
+-{
+-}
+-
+-static void intcapxt_mask_irq(struct irq_data *data)
+-{
+-}
+
+ static struct irq_chip intcapxt_controller;
+
+ static int intcapxt_irqdomain_activate(struct irq_domain *domain,
+ struct irq_data *irqd, bool reserve)
+ {
+- struct amd_iommu *iommu = irqd->chip_data;
+- struct irq_cfg *cfg = irqd_cfg(irqd);
+- union intcapxt xt;
+-
+- xt.capxt = 0ULL;
+- xt.dest_mode_logical = apic->dest_mode_logical;
+- xt.vector = cfg->vector;
+- xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
+- xt.destid_24_31 = cfg->dest_apicid >> 24;
+-
+- /**
+- * Current IOMMU implemtation uses the same IRQ for all
+- * 3 IOMMU interrupts.
+- */
+- writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
+- writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
+- writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
+ return 0;
+ }
+
+ static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
+ struct irq_data *irqd)
+ {
+- intcapxt_mask_irq(irqd);
+ }
+
+
+@@ -2088,6 +2069,38 @@ static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq
+ irq_domain_free_irqs_top(domain, virq, nr_irqs);
+ }
+
++
++static void intcapxt_unmask_irq(struct irq_data *irqd)
++{
++ struct amd_iommu *iommu = irqd->chip_data;
++ struct irq_cfg *cfg = irqd_cfg(irqd);
++ union intcapxt xt;
++
++ xt.capxt = 0ULL;
++ xt.dest_mode_logical = apic->dest_mode_logical;
++ xt.vector = cfg->vector;
++ xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
++ xt.destid_24_31 = cfg->dest_apicid >> 24;
++
++ /**
++ * Current IOMMU implementation uses the same IRQ for all
++ * 3 IOMMU interrupts.
++ */
++ writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
++ writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
++ writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
++}
++
++static void intcapxt_mask_irq(struct irq_data *irqd)
++{
++ struct amd_iommu *iommu = irqd->chip_data;
++
++ writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
++ writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
++ writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
++}
++
++
+ static int intcapxt_set_affinity(struct irq_data *irqd,
+ const struct cpumask *mask, bool force)
+ {
+@@ -2097,8 +2110,12 @@ static int intcapxt_set_affinity(struct irq_data *irqd,
+ ret = parent->chip->irq_set_affinity(parent, mask, force);
+ if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
+ return ret;
++ return 0;
++}
+
+- return intcapxt_irqdomain_activate(irqd->domain, irqd, false);
++static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
++{
++ return on ? -EOPNOTSUPP : 0;
+ }
+
+ static struct irq_chip intcapxt_controller = {
+@@ -2108,7 +2125,8 @@ static struct irq_chip intcapxt_controller = {
+ .irq_ack = irq_chip_ack_parent,
+ .irq_retrigger = irq_chip_retrigger_hierarchy,
+ .irq_set_affinity = intcapxt_set_affinity,
+- .flags = IRQCHIP_SKIP_SET_WAKE,
++ .irq_set_wake = intcapxt_set_wake,
++ .flags = IRQCHIP_MASK_ON_SUSPEND,
+ };
+
+ static const struct irq_domain_ops intcapxt_domain_ops = {
+@@ -2170,7 +2188,6 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu)
+ return ret;
+ }
+
+- iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
+ return 0;
+ }
+
+@@ -2193,6 +2210,10 @@ static int iommu_init_irq(struct amd_iommu *iommu)
+
+ iommu->int_enabled = true;
+ enable_faults:
++
++ if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
++ iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
++
+ iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
+
+ if (iommu->ppr_log != NULL)
+diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
+index 182c93a43efd8..1eddf557636d7 100644
+--- a/drivers/iommu/amd/io_pgtable.c
++++ b/drivers/iommu/amd/io_pgtable.c
+@@ -519,12 +519,6 @@ static void v1_free_pgtable(struct io_pgtable *iop)
+
+ dom = container_of(pgtable, struct protection_domain, iop);
+
+- /* Update data structure */
+- amd_iommu_domain_clr_pt_root(dom);
+-
+- /* Make changes visible to IOMMUs */
+- amd_iommu_domain_update(dom);
+-
+ /* Page-table is not visible to IOMMU anymore, so free it */
+ BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
+ pgtable->mode > PAGE_MODE_6_LEVEL);
+@@ -532,6 +526,12 @@ static void v1_free_pgtable(struct io_pgtable *iop)
+ root = (unsigned long)pgtable->root;
+ freelist = free_sub_pt(root, pgtable->mode, freelist);
+
++ /* Update data structure */
++ amd_iommu_domain_clr_pt_root(dom);
++
++ /* Make changes visible to IOMMUs */
++ amd_iommu_domain_update(dom);
++
+ free_page_list(freelist);
+ }
+
+diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
+index 1722bb161841f..f46eb73970216 100644
+--- a/drivers/iommu/amd/iommu.c
++++ b/drivers/iommu/amd/iommu.c
+@@ -742,7 +742,8 @@ amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }
+ #endif /* !CONFIG_IRQ_REMAP */
+
+ #define AMD_IOMMU_INT_MASK \
+- (MMIO_STATUS_EVT_INT_MASK | \
++ (MMIO_STATUS_EVT_OVERFLOW_INT_MASK | \
++ MMIO_STATUS_EVT_INT_MASK | \
+ MMIO_STATUS_PPR_INT_MASK | \
+ MMIO_STATUS_GALOG_INT_MASK)
+
+@@ -752,7 +753,7 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data)
+ u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+
+ while (status & AMD_IOMMU_INT_MASK) {
+- /* Enable EVT and PPR and GA interrupts again */
++ /* Enable interrupt sources again */
+ writel(AMD_IOMMU_INT_MASK,
+ iommu->mmio_base + MMIO_STATUS_OFFSET);
+
+@@ -773,6 +774,11 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data)
+ }
+ #endif
+
++ if (status & MMIO_STATUS_EVT_OVERFLOW_INT_MASK) {
++ pr_info_ratelimited("IOMMU event log overflow\n");
++ amd_iommu_restart_event_logging(iommu);
++ }
++
+ /*
+ * Hardware bug: ERBT1312
+ * When re-enabling interrupt (by writing 1
+diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c
+index a9e568276c99f..a45c5536d2506 100644
+--- a/drivers/iommu/amd/iommu_v2.c
++++ b/drivers/iommu/amd/iommu_v2.c
+@@ -928,10 +928,8 @@ static int __init amd_iommu_v2_init(void)
+ {
+ int ret;
+
+- pr_info("AMD IOMMUv2 driver by Joerg Roedel <jroedel@suse.de>\n");
+-
+ if (!amd_iommu_v2_supported()) {
+- pr_info("AMD IOMMUv2 functionality not available on this system\n");
++ pr_info("AMD IOMMUv2 functionality not available on this system - This is not a bug.\n");
+ /*
+ * Load anyway to provide the symbols to other modules
+ * which may use AMD IOMMUv2 optionally.
+@@ -946,6 +944,8 @@ static int __init amd_iommu_v2_init(void)
+
+ amd_iommu_register_ppr_notifier(&ppr_nb);
+
++ pr_info("AMD IOMMUv2 loaded and initialized\n");
++
+ return 0;
+
+ out:
+diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c
+index fdfa39ec2a4d4..baba4571c8152 100644
+--- a/drivers/iommu/apple-dart.c
++++ b/drivers/iommu/apple-dart.c
+@@ -70,6 +70,8 @@
+ #define DART_ERROR_ADDR_HI 0x54
+ #define DART_ERROR_ADDR_LO 0x50
+
++#define DART_STREAMS_ENABLE 0xfc
++
+ #define DART_TCR(sid) (0x100 + 4 * (sid))
+ #define DART_TCR_TRANSLATE_ENABLE BIT(7)
+ #define DART_TCR_BYPASS0_ENABLE BIT(8)
+@@ -301,6 +303,9 @@ static int apple_dart_hw_reset(struct apple_dart *dart)
+ apple_dart_hw_disable_dma(&stream_map);
+ apple_dart_hw_clear_all_ttbrs(&stream_map);
+
++ /* enable all streams globally since TCR is used to control isolation */
++ writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE);
++
+ /* clear any pending errors before the interrupt is unmasked */
+ writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);
+
+@@ -752,6 +757,7 @@ static const struct iommu_ops apple_dart_iommu_ops = {
+ .of_xlate = apple_dart_of_xlate,
+ .def_domain_type = apple_dart_def_domain_type,
+ .pgsize_bitmap = -1UL, /* Restricted during dart probe */
++ .owner = THIS_MODULE,
+ };
+
+ static irqreturn_t apple_dart_irq(int irq, void *dev)
+@@ -827,16 +833,15 @@ static int apple_dart_probe(struct platform_device *pdev)
+ dart->dev = dev;
+ spin_lock_init(&dart->lock);
+
+- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++ dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
++ if (IS_ERR(dart->regs))
++ return PTR_ERR(dart->regs);
++
+ if (resource_size(res) < 0x4000) {
+ dev_err(dev, "MMIO region too small (%pr)\n", res);
+ return -EINVAL;
+ }
+
+- dart->regs = devm_ioremap_resource(dev, res);
+- if (IS_ERR(dart->regs))
+- return PTR_ERR(dart->regs);
+-
+ dart->irq = platform_get_irq(pdev, 0);
+ if (dart->irq < 0)
+ return -ENODEV;
+diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+index ee66d1f4cb81e..f763c1430d152 100644
+--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
++++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+@@ -183,7 +183,14 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
+ {
+ struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
+ struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+- size_t size = end - start + 1;
++ size_t size;
++
++ /*
++ * The mm_types defines vm_end as the first byte after the end address,
++ * different from IOMMU subsystem using the last address of an address
++ * range. So do a simple translation here by calculating size correctly.
++ */
++ size = end - start;
+
+ if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
+ arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
+diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+index a388e318f86e0..430315135cff7 100644
+--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
++++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+@@ -1552,6 +1552,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
+ dev_info(smmu->dev, "\t0x%016llx\n",
+ (unsigned long long)evt[i]);
+
++ cond_resched();
+ }
+
+ /*
+diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c b/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c
+index 01e9b50b10a18..87bf522b9d2ee 100644
+--- a/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c
++++ b/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c
+@@ -258,6 +258,34 @@ static void nvidia_smmu_probe_finalize(struct arm_smmu_device *smmu, struct devi
+ dev_name(dev), err);
+ }
+
++static int nvidia_smmu_init_context(struct arm_smmu_domain *smmu_domain,
++ struct io_pgtable_cfg *pgtbl_cfg,
++ struct device *dev)
++{
++ struct arm_smmu_device *smmu = smmu_domain->smmu;
++ const struct device_node *np = smmu->dev->of_node;
++
++ /*
++ * Tegra194 and Tegra234 SoCs have the erratum that causes walk cache
++ * entries to not be invalidated correctly. The problem is that the walk
++ * cache index generated for IOVA is not same across translation and
++ * invalidation requests. This is leading to page faults when PMD entry
++ * is released during unmap and populated with new PTE table during
++ * subsequent map request. Disabling large page mappings avoids the
++ * release of PMD entry and avoid translations seeing stale PMD entry in
++ * walk cache.
++ * Fix this by limiting the page mappings to PAGE_SIZE on Tegra194 and
++ * Tegra234.
++ */
++ if (of_device_is_compatible(np, "nvidia,tegra234-smmu") ||
++ of_device_is_compatible(np, "nvidia,tegra194-smmu")) {
++ smmu->pgsize_bitmap = PAGE_SIZE;
++ pgtbl_cfg->pgsize_bitmap = smmu->pgsize_bitmap;
++ }
++
++ return 0;
++}
++
+ static const struct arm_smmu_impl nvidia_smmu_impl = {
+ .read_reg = nvidia_smmu_read_reg,
+ .write_reg = nvidia_smmu_write_reg,
+@@ -268,10 +296,12 @@ static const struct arm_smmu_impl nvidia_smmu_impl = {
+ .global_fault = nvidia_smmu_global_fault,
+ .context_fault = nvidia_smmu_context_fault,
+ .probe_finalize = nvidia_smmu_probe_finalize,
++ .init_context = nvidia_smmu_init_context,
+ };
+
+ static const struct arm_smmu_impl nvidia_smmu_single_impl = {
+ .probe_finalize = nvidia_smmu_probe_finalize,
++ .init_context = nvidia_smmu_init_context,
+ };
+
+ struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu)
+diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
+index 55690af1b25d0..c998960495b4e 100644
+--- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
++++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
+@@ -51,7 +51,7 @@ static void qcom_adreno_smmu_get_fault_info(const void *cookie,
+ info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
+ info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
+ info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
+- info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
++ info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
+ info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
+ }
+
+diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
+index 896bea04c347e..276b6fcc80ad9 100644
+--- a/drivers/iommu/dma-iommu.c
++++ b/drivers/iommu/dma-iommu.c
+@@ -317,6 +317,11 @@ static bool dev_is_untrusted(struct device *dev)
+ return dev_is_pci(dev) && to_pci_dev(dev)->untrusted;
+ }
+
++static bool dev_use_swiotlb(struct device *dev)
++{
++ return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev);
++}
++
+ /* sysfs updates are serialised by the mutex of the group owning @domain */
+ int iommu_dma_init_fq(struct iommu_domain *domain)
+ {
+@@ -510,23 +515,6 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
+ iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
+ }
+
+-static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr,
+- size_t size, enum dma_data_direction dir,
+- unsigned long attrs)
+-{
+- struct iommu_domain *domain = iommu_get_dma_domain(dev);
+- phys_addr_t phys;
+-
+- phys = iommu_iova_to_phys(domain, dma_addr);
+- if (WARN_ON(!phys))
+- return;
+-
+- __iommu_dma_unmap(dev, dma_addr, size);
+-
+- if (unlikely(is_swiotlb_buffer(dev, phys)))
+- swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
+-}
+-
+ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
+ size_t size, int prot, u64 dma_mask)
+ {
+@@ -553,52 +541,6 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
+ return iova + iova_off;
+ }
+
+-static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
+- size_t org_size, dma_addr_t dma_mask, bool coherent,
+- enum dma_data_direction dir, unsigned long attrs)
+-{
+- int prot = dma_info_to_prot(dir, coherent, attrs);
+- struct iommu_domain *domain = iommu_get_dma_domain(dev);
+- struct iommu_dma_cookie *cookie = domain->iova_cookie;
+- struct iova_domain *iovad = &cookie->iovad;
+- size_t aligned_size = org_size;
+- void *padding_start;
+- size_t padding_size;
+- dma_addr_t iova;
+-
+- /*
+- * If both the physical buffer start address and size are
+- * page aligned, we don't need to use a bounce page.
+- */
+- if (IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev) &&
+- iova_offset(iovad, phys | org_size)) {
+- aligned_size = iova_align(iovad, org_size);
+- phys = swiotlb_tbl_map_single(dev, phys, org_size,
+- aligned_size, dir, attrs);
+-
+- if (phys == DMA_MAPPING_ERROR)
+- return DMA_MAPPING_ERROR;
+-
+- /* Cleanup the padding area. */
+- padding_start = phys_to_virt(phys);
+- padding_size = aligned_size;
+-
+- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+- (dir == DMA_TO_DEVICE ||
+- dir == DMA_BIDIRECTIONAL)) {
+- padding_start += org_size;
+- padding_size -= org_size;
+- }
+-
+- memset(padding_start, 0, padding_size);
+- }
+-
+- iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask);
+- if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
+- swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs);
+- return iova;
+-}
+-
+ static void __iommu_dma_free_pages(struct page **pages, int count)
+ {
+ while (count--)
+@@ -794,7 +736,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev,
+ {
+ phys_addr_t phys;
+
+- if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev))
++ if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
+ return;
+
+ phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
+@@ -810,7 +752,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev,
+ {
+ phys_addr_t phys;
+
+- if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev))
++ if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
+ return;
+
+ phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
+@@ -828,17 +770,13 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev,
+ struct scatterlist *sg;
+ int i;
+
+- if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev))
+- return;
+-
+- for_each_sg(sgl, sg, nelems, i) {
+- if (!dev_is_dma_coherent(dev))
++ if (dev_use_swiotlb(dev))
++ for_each_sg(sgl, sg, nelems, i)
++ iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
++ sg->length, dir);
++ else if (!dev_is_dma_coherent(dev))
++ for_each_sg(sgl, sg, nelems, i)
+ arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
+-
+- if (is_swiotlb_buffer(dev, sg_phys(sg)))
+- swiotlb_sync_single_for_cpu(dev, sg_phys(sg),
+- sg->length, dir);
+- }
+ }
+
+ static void iommu_dma_sync_sg_for_device(struct device *dev,
+@@ -848,17 +786,14 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,
+ struct scatterlist *sg;
+ int i;
+
+- if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev))
+- return;
+-
+- for_each_sg(sgl, sg, nelems, i) {
+- if (is_swiotlb_buffer(dev, sg_phys(sg)))
+- swiotlb_sync_single_for_device(dev, sg_phys(sg),
+- sg->length, dir);
+-
+- if (!dev_is_dma_coherent(dev))
++ if (dev_use_swiotlb(dev))
++ for_each_sg(sgl, sg, nelems, i)
++ iommu_dma_sync_single_for_device(dev,
++ sg_dma_address(sg),
++ sg->length, dir);
++ else if (!dev_is_dma_coherent(dev))
++ for_each_sg(sgl, sg, nelems, i)
+ arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
+- }
+ }
+
+ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
+@@ -867,22 +802,66 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
+ {
+ phys_addr_t phys = page_to_phys(page) + offset;
+ bool coherent = dev_is_dma_coherent(dev);
+- dma_addr_t dma_handle;
++ int prot = dma_info_to_prot(dir, coherent, attrs);
++ struct iommu_domain *domain = iommu_get_dma_domain(dev);
++ struct iommu_dma_cookie *cookie = domain->iova_cookie;
++ struct iova_domain *iovad = &cookie->iovad;
++ dma_addr_t iova, dma_mask = dma_get_mask(dev);
++
++ /*
++ * If both the physical buffer start address and size are
++ * page aligned, we don't need to use a bounce page.
++ */
++ if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) {
++ void *padding_start;
++ size_t padding_size, aligned_size;
++
++ aligned_size = iova_align(iovad, size);
++ phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size,
++ iova_mask(iovad), dir, attrs);
++
++ if (phys == DMA_MAPPING_ERROR)
++ return DMA_MAPPING_ERROR;
+
+- dma_handle = __iommu_dma_map_swiotlb(dev, phys, size, dma_get_mask(dev),
+- coherent, dir, attrs);
+- if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+- dma_handle != DMA_MAPPING_ERROR)
++ /* Cleanup the padding area. */
++ padding_start = phys_to_virt(phys);
++ padding_size = aligned_size;
++
++ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
++ (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) {
++ padding_start += size;
++ padding_size -= size;
++ }
++
++ memset(padding_start, 0, padding_size);
++ }
++
++ if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+ arch_sync_dma_for_device(phys, size, dir);
+- return dma_handle;
++
++ iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
++ if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
++ swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
++ return iova;
+ }
+
+ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
+ size_t size, enum dma_data_direction dir, unsigned long attrs)
+ {
+- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+- iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir);
+- __iommu_dma_unmap_swiotlb(dev, dma_handle, size, dir, attrs);
++ struct iommu_domain *domain = iommu_get_dma_domain(dev);
++ phys_addr_t phys;
++
++ phys = iommu_iova_to_phys(domain, dma_handle);
++ if (WARN_ON(!phys))
++ return;
++
++ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev))
++ arch_sync_dma_for_cpu(phys, size, dir);
++
++ __iommu_dma_unmap(dev, dma_handle, size);
++
++ if (unlikely(is_swiotlb_buffer(dev, phys)))
++ swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
+ }
+
+ /*
+@@ -967,7 +946,7 @@ static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *s
+ int i;
+
+ for_each_sg(sg, s, nents, i)
+- __iommu_dma_unmap_swiotlb(dev, sg_dma_address(s),
++ iommu_dma_unmap_page(dev, sg_dma_address(s),
+ sg_dma_len(s), dir, attrs);
+ }
+
+@@ -978,9 +957,8 @@ static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg,
+ int i;
+
+ for_each_sg(sg, s, nents, i) {
+- sg_dma_address(s) = __iommu_dma_map_swiotlb(dev, sg_phys(s),
+- s->length, dma_get_mask(dev),
+- dev_is_dma_coherent(dev), dir, attrs);
++ sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s),
++ s->offset, s->length, dir, attrs);
+ if (sg_dma_address(s) == DMA_MAPPING_ERROR)
+ goto out_unmap;
+ sg_dma_len(s) = s->length;
+@@ -1016,15 +994,16 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
+
+ if (static_branch_unlikely(&iommu_deferred_attach_enabled)) {
+ ret = iommu_deferred_attach(dev, domain);
+- goto out;
++ if (ret)
++ goto out;
+ }
+
++ if (dev_use_swiotlb(dev))
++ return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs);
++
+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+ iommu_dma_sync_sg_for_device(dev, sg, nents, dir);
+
+- if (dev_is_untrusted(dev))
+- return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs);
+-
+ /*
+ * Work out how much IOVA space we need, and align the segments to
+ * IOVA granules for the IOMMU driver to handle. With some clever
+@@ -1097,14 +1076,14 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
+ struct scatterlist *tmp;
+ int i;
+
+- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+- iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
+-
+- if (dev_is_untrusted(dev)) {
++ if (dev_use_swiotlb(dev)) {
+ iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs);
+ return;
+ }
+
++ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
++ iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
++
+ /*
+ * The scatterlist segments are mapped into a single
+ * contiguous IOVA allocation, so this is incredibly easy.
+diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
+index d75f59ae28e6e..91a5c75966f32 100644
+--- a/drivers/iommu/intel/iommu.c
++++ b/drivers/iommu/intel/iommu.c
+@@ -412,6 +412,7 @@ static int __init intel_iommu_setup(char *str)
+ {
+ if (!str)
+ return -EINVAL;
++
+ while (*str) {
+ if (!strncmp(str, "on", 2)) {
+ dmar_disabled = 0;
+@@ -441,13 +442,16 @@ static int __init intel_iommu_setup(char *str)
+ } else if (!strncmp(str, "tboot_noforce", 13)) {
+ pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
+ intel_iommu_tboot_noforce = 1;
++ } else {
++ pr_notice("Unknown option - '%s'\n", str);
+ }
+
+ str += strcspn(str, ",");
+ while (*str == ',')
+ str++;
+ }
+- return 0;
++
++ return 1;
+ }
+ __setup("intel_iommu=", intel_iommu_setup);
+
+@@ -1222,13 +1226,11 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
+ pte = &pte[pfn_level_offset(pfn, level)];
+
+ do {
+- unsigned long level_pfn;
++ unsigned long level_pfn = pfn & level_mask(level);
+
+ if (!dma_pte_present(pte))
+ goto next;
+
+- level_pfn = pfn & level_mask(level);
+-
+ /* If range covers entire pagetable, free it */
+ if (start_pfn <= level_pfn &&
+ last_pfn >= level_pfn + level_size(level) - 1) {
+@@ -1249,7 +1251,7 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
+ freelist);
+ }
+ next:
+- pfn += level_size(level);
++ pfn = level_pfn + level_size(level);
+ } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
+
+ if (first_pte)
+@@ -1635,7 +1637,8 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
+ unsigned long pfn, unsigned int pages,
+ int ih, int map)
+ {
+- unsigned int mask = ilog2(__roundup_pow_of_two(pages));
++ unsigned int aligned_pages = __roundup_pow_of_two(pages);
++ unsigned int mask = ilog2(aligned_pages);
+ uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
+ u16 did = domain->iommu_did[iommu->seq_id];
+
+@@ -1647,10 +1650,30 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
+ if (domain_use_first_level(domain)) {
+ domain_flush_piotlb(iommu, domain, addr, pages, ih);
+ } else {
++ unsigned long bitmask = aligned_pages - 1;
++
++ /*
++ * PSI masks the low order bits of the base address. If the
++ * address isn't aligned to the mask, then compute a mask value
++ * needed to ensure the target range is flushed.
++ */
++ if (unlikely(bitmask & pfn)) {
++ unsigned long end_pfn = pfn + pages - 1, shared_bits;
++
++ /*
++ * Since end_pfn <= pfn + bitmask, the only way bits
++ * higher than bitmask can differ in pfn and end_pfn is
++ * by carrying. This means after masking out bitmask,
++ * high bits starting with the first set bit in
++ * shared_bits are all equal in both pfn and end_pfn.
++ */
++ shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
++ mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;
++ }
++
+ /*
+ * Fallback to domain selective flush if no PSI support or
+- * the size is too big. PSI requires page size to be 2 ^ x,
+- * and the base address is naturally aligned to the size.
++ * the size is too big.
+ */
+ if (!cap_pgsel_inv(iommu->cap) ||
+ mask > cap_max_amask_val(iommu->cap))
+@@ -2649,7 +2672,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
+ spin_unlock_irqrestore(&device_domain_lock, flags);
+
+ /* PASID table is mandatory for a PCI device in scalable mode. */
+- if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
++ if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
+ ret = intel_pasid_alloc_table(dev);
+ if (ret) {
+ dev_err(dev, "PASID table allocation failed\n");
+diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c
+index f912fe45bea2c..a673195978843 100644
+--- a/drivers/iommu/intel/irq_remapping.c
++++ b/drivers/iommu/intel/irq_remapping.c
+@@ -569,9 +569,8 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
+ fn, &intel_ir_domain_ops,
+ iommu);
+ if (!iommu->ir_domain) {
+- irq_domain_free_fwnode(fn);
+ pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);
+- goto out_free_bitmap;
++ goto out_free_fwnode;
+ }
+ iommu->ir_msi_domain =
+ arch_create_remap_msi_irq_domain(iommu->ir_domain,
+@@ -595,7 +594,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
+
+ if (dmar_enable_qi(iommu)) {
+ pr_err("Failed to enable queued invalidation\n");
+- goto out_free_bitmap;
++ goto out_free_ir_domain;
+ }
+ }
+
+@@ -619,6 +618,14 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
+
+ return 0;
+
++out_free_ir_domain:
++ if (iommu->ir_msi_domain)
++ irq_domain_remove(iommu->ir_msi_domain);
++ iommu->ir_msi_domain = NULL;
++ irq_domain_remove(iommu->ir_domain);
++ iommu->ir_domain = NULL;
++out_free_fwnode:
++ irq_domain_free_fwnode(fn);
+ out_free_bitmap:
+ bitmap_free(bitmap);
+ out_free_pages:
+diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
+index 0c228787704f3..3a9468b1d2c3c 100644
+--- a/drivers/iommu/intel/svm.c
++++ b/drivers/iommu/intel/svm.c
+@@ -978,6 +978,10 @@ bad_req:
+ goto bad_req;
+ }
+
++ /* Drop Stop Marker message. No need for a response. */
++ if (unlikely(req->lpig && !req->rd_req && !req->wr_req))
++ goto prq_advance;
++
+ if (!svm || svm->pasid != req->pasid) {
+ /*
+ * It can't go away, because the driver is not permitted
+diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
+index bfb6acb651e5f..be066c1503d37 100644
+--- a/drivers/iommu/io-pgtable-arm-v7s.c
++++ b/drivers/iommu/io-pgtable-arm-v7s.c
+@@ -246,13 +246,17 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
+ __GFP_ZERO | ARM_V7S_TABLE_GFP_DMA, get_order(size));
+ else if (lvl == 2)
+ table = kmem_cache_zalloc(data->l2_tables, gfp);
++
++ if (!table)
++ return NULL;
++
+ phys = virt_to_phys(table);
+ if (phys != (arm_v7s_iopte)phys) {
+ /* Doesn't fit in PTE */
+ dev_err(dev, "Page table does not fit in PTE: %pa", &phys);
+ goto out_free;
+ }
+- if (table && !cfg->coherent_walk) {
++ if (!cfg->coherent_walk) {
+ dma = dma_map_single(dev, table, size, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, dma))
+ goto out_free;
+diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
+index dd9e47189d0d9..94ff319ae8acc 100644
+--- a/drivers/iommu/io-pgtable-arm.c
++++ b/drivers/iommu/io-pgtable-arm.c
+@@ -315,11 +315,12 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
+ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
+ arm_lpae_iopte *ptep,
+ arm_lpae_iopte curr,
+- struct io_pgtable_cfg *cfg)
++ struct arm_lpae_io_pgtable *data)
+ {
+ arm_lpae_iopte old, new;
++ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+
+- new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE;
++ new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
+ new |= ARM_LPAE_PTE_NSTABLE;
+
+@@ -380,7 +381,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
+ if (!cptep)
+ return -ENOMEM;
+
+- pte = arm_lpae_install_table(cptep, ptep, 0, cfg);
++ pte = arm_lpae_install_table(cptep, ptep, 0, data);
+ if (pte)
+ __arm_lpae_free_pages(cptep, tblsz, cfg);
+ } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
+@@ -592,7 +593,7 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
+ __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
+ }
+
+- pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg);
++ pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
+ if (pte != blk_pte) {
+ __arm_lpae_free_pages(tablep, tablesz, cfg);
+ /*
+diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
+index 3303d707bab4b..7f409e9eea4b7 100644
+--- a/drivers/iommu/iommu.c
++++ b/drivers/iommu/iommu.c
+@@ -206,9 +206,14 @@ static struct dev_iommu *dev_iommu_get(struct device *dev)
+
+ static void dev_iommu_free(struct device *dev)
+ {
+- iommu_fwspec_free(dev);
+- kfree(dev->iommu);
++ struct dev_iommu *param = dev->iommu;
++
+ dev->iommu = NULL;
++ if (param->fwspec) {
++ fwnode_handle_put(param->fwspec->iommu_fwnode);
++ kfree(param->fwspec);
++ }
++ kfree(param);
+ }
+
+ static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
+@@ -287,11 +292,11 @@ int iommu_probe_device(struct device *dev)
+ */
+ mutex_lock(&group->mutex);
+ iommu_alloc_default_domain(group, dev);
+- mutex_unlock(&group->mutex);
+
+ if (group->default_domain) {
+ ret = __iommu_attach_device(group->default_domain, dev);
+ if (ret) {
++ mutex_unlock(&group->mutex);
+ iommu_group_put(group);
+ goto err_release;
+ }
+@@ -299,6 +304,7 @@ int iommu_probe_device(struct device *dev)
+
+ iommu_create_device_direct_mappings(group, dev);
+
++ mutex_unlock(&group->mutex);
+ iommu_group_put(group);
+
+ if (ops->probe_finalize)
+diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
+index 9e8bc802ac053..cae5a73ff518c 100644
+--- a/drivers/iommu/iova.c
++++ b/drivers/iommu/iova.c
+@@ -83,8 +83,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad)
+ if (!has_iova_flush_queue(iovad))
+ return;
+
+- if (timer_pending(&iovad->fq_timer))
+- del_timer(&iovad->fq_timer);
++ del_timer_sync(&iovad->fq_timer);
+
+ fq_destroy_all_entries(iovad);
+
+@@ -155,10 +154,11 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
+ cached_iova = to_iova(iovad->cached32_node);
+ if (free == cached_iova ||
+ (free->pfn_hi < iovad->dma_32bit_pfn &&
+- free->pfn_lo >= cached_iova->pfn_lo)) {
++ free->pfn_lo >= cached_iova->pfn_lo))
+ iovad->cached32_node = rb_next(&free->node);
++
++ if (free->pfn_lo < iovad->dma_32bit_pfn)
+ iovad->max32_alloc_size = iovad->dma_32bit_pfn;
+- }
+
+ cached_iova = to_iova(iovad->cached_node);
+ if (free->pfn_lo >= cached_iova->pfn_lo)
+diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
+index d38ff29a76e8f..96708cd2757f7 100644
+--- a/drivers/iommu/ipmmu-vmsa.c
++++ b/drivers/iommu/ipmmu-vmsa.c
+@@ -982,7 +982,9 @@ static int ipmmu_probe(struct platform_device *pdev)
+ bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
+ mmu->features = of_device_get_match_data(&pdev->dev);
+ memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
+- dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
++ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
++ if (ret)
++ return ret;
+
+ /* Map I/O memory and request IRQ. */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
+index d837adfd1da55..5971a11686662 100644
+--- a/drivers/iommu/mtk_iommu.c
++++ b/drivers/iommu/mtk_iommu.c
+@@ -550,7 +550,9 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
+ phys_addr_t pa;
+
+ pa = dom->iop->iova_to_phys(dom->iop, iova);
+- if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
++ if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) &&
++ dom->data->enable_4GB &&
++ pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
+ pa &= ~BIT_ULL(32);
+
+ return pa;
+@@ -560,22 +562,52 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
+ {
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct mtk_iommu_data *data;
++ struct device_link *link;
++ struct device *larbdev;
++ unsigned int larbid, larbidx, i;
+
+ if (!fwspec || fwspec->ops != &mtk_iommu_ops)
+ return ERR_PTR(-ENODEV); /* Not a iommu client device */
+
+ data = dev_iommu_priv_get(dev);
+
++ /*
++ * Link the consumer device with the smi-larb device(supplier).
++ * The device that connects with each a larb is a independent HW.
++ * All the ports in each a device should be in the same larbs.
++ */
++ larbid = MTK_M4U_TO_LARB(fwspec->ids[0]);
++ for (i = 1; i < fwspec->num_ids; i++) {
++ larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]);
++ if (larbid != larbidx) {
++ dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n",
++ larbid, larbidx);
++ return ERR_PTR(-EINVAL);
++ }
++ }
++ larbdev = data->larb_imu[larbid].dev;
++ link = device_link_add(dev, larbdev,
++ DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS);
++ if (!link)
++ dev_err(dev, "Unable to link %s\n", dev_name(larbdev));
+ return &data->iommu;
+ }
+
+ static void mtk_iommu_release_device(struct device *dev)
+ {
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
++ struct mtk_iommu_data *data;
++ struct device *larbdev;
++ unsigned int larbid;
+
+ if (!fwspec || fwspec->ops != &mtk_iommu_ops)
+ return;
+
++ data = dev_iommu_priv_get(dev);
++ larbid = MTK_M4U_TO_LARB(fwspec->ids[0]);
++ larbdev = data->larb_imu[larbid].dev;
++ device_link_remove(dev, larbdev);
++
+ iommu_fwspec_free(dev);
+ }
+
+@@ -846,7 +878,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
+ plarbdev = of_find_device_by_node(larbnode);
+ if (!plarbdev) {
+ of_node_put(larbnode);
+- return -EPROBE_DEFER;
++ return -ENODEV;
+ }
+ data->larb_imu[id].dev = &plarbdev->dev;
+
+diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
+index be22fcf988cee..bc7ee90b9373d 100644
+--- a/drivers/iommu/mtk_iommu_v1.c
++++ b/drivers/iommu/mtk_iommu_v1.c
+@@ -423,7 +423,18 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ struct of_phandle_args iommu_spec;
+ struct mtk_iommu_data *data;
+- int err, idx = 0;
++ int err, idx = 0, larbid, larbidx;
++ struct device_link *link;
++ struct device *larbdev;
++
++ /*
++ * In the deferred case, free the existed fwspec.
++ * Always initialize the fwspec internally.
++ */
++ if (fwspec) {
++ iommu_fwspec_free(dev);
++ fwspec = dev_iommu_fwspec_get(dev);
++ }
+
+ while (!of_parse_phandle_with_args(dev->of_node, "iommus",
+ "#iommu-cells",
+@@ -444,6 +455,23 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
+
+ data = dev_iommu_priv_get(dev);
+
++ /* Link the consumer device with the smi-larb device(supplier) */
++ larbid = mt2701_m4u_to_larb(fwspec->ids[0]);
++ for (idx = 1; idx < fwspec->num_ids; idx++) {
++ larbidx = mt2701_m4u_to_larb(fwspec->ids[idx]);
++ if (larbid != larbidx) {
++ dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n",
++ larbid, larbidx);
++ return ERR_PTR(-EINVAL);
++ }
++ }
++
++ larbdev = data->larb_imu[larbid].dev;
++ link = device_link_add(dev, larbdev,
++ DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS);
++ if (!link)
++ dev_err(dev, "Unable to link %s\n", dev_name(larbdev));
++
+ return &data->iommu;
+ }
+
+@@ -464,10 +492,18 @@ static void mtk_iommu_probe_finalize(struct device *dev)
+ static void mtk_iommu_release_device(struct device *dev)
+ {
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
++ struct mtk_iommu_data *data;
++ struct device *larbdev;
++ unsigned int larbid;
+
+ if (!fwspec || fwspec->ops != &mtk_iommu_ops)
+ return;
+
++ data = dev_iommu_priv_get(dev);
++ larbid = mt2701_m4u_to_larb(fwspec->ids[0]);
++ larbdev = data->larb_imu[larbid].dev;
++ device_link_remove(dev, larbdev);
++
+ iommu_fwspec_free(dev);
+ }
+
+@@ -595,7 +631,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
+ plarbdev = of_find_device_by_node(larbnode);
+ if (!plarbdev) {
+ of_node_put(larbnode);
+- return -EPROBE_DEFER;
++ return -ENODEV;
+ }
+ data->larb_imu[i].dev = &plarbdev->dev;
+
+diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
+index 91749654fd490..be60f6f3a265d 100644
+--- a/drivers/iommu/omap-iommu.c
++++ b/drivers/iommu/omap-iommu.c
+@@ -1661,7 +1661,7 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev)
+ num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus",
+ sizeof(phandle));
+ if (num_iommus < 0)
+- return 0;
++ return ERR_PTR(-ENODEV);
+
+ arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL);
+ if (!arch_data)
+diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
+index 5cb260820eda6..7f23ad61c094f 100644
+--- a/drivers/iommu/rockchip-iommu.c
++++ b/drivers/iommu/rockchip-iommu.c
+@@ -200,8 +200,8 @@ static inline phys_addr_t rk_dte_pt_address(u32 dte)
+ #define DTE_HI_MASK2 GENMASK(7, 4)
+ #define DTE_HI_SHIFT1 24 /* shift bit 8 to bit 32 */
+ #define DTE_HI_SHIFT2 32 /* shift bit 4 to bit 36 */
+-#define PAGE_DESC_HI_MASK1 GENMASK_ULL(39, 36)
+-#define PAGE_DESC_HI_MASK2 GENMASK_ULL(35, 32)
++#define PAGE_DESC_HI_MASK1 GENMASK_ULL(35, 32)
++#define PAGE_DESC_HI_MASK2 GENMASK_ULL(39, 36)
+
+ static inline phys_addr_t rk_dte_pt_address_v2(u32 dte)
+ {
+diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
+index 0a281833f6117..abbdaeb4bf8f1 100644
+--- a/drivers/iommu/tegra-smmu.c
++++ b/drivers/iommu/tegra-smmu.c
+@@ -808,8 +808,10 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
+ return NULL;
+
+ mc = platform_get_drvdata(pdev);
+- if (!mc)
++ if (!mc) {
++ put_device(&pdev->dev);
+ return NULL;
++ }
+
+ return mc->smmu;
+ }
+diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c
+index 53e0fb0562c11..c91ddb0cf1448 100644
+--- a/drivers/irqchip/irq-armada-370-xp.c
++++ b/drivers/irqchip/irq-armada-370-xp.c
+@@ -232,16 +232,12 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq,
+ int hwirq, i;
+
+ mutex_lock(&msi_used_lock);
++ hwirq = bitmap_find_free_region(msi_used, PCI_MSI_DOORBELL_NR,
++ order_base_2(nr_irqs));
++ mutex_unlock(&msi_used_lock);
+
+- hwirq = bitmap_find_next_zero_area(msi_used, PCI_MSI_DOORBELL_NR,
+- 0, nr_irqs, 0);
+- if (hwirq >= PCI_MSI_DOORBELL_NR) {
+- mutex_unlock(&msi_used_lock);
++ if (hwirq < 0)
+ return -ENOSPC;
+- }
+-
+- bitmap_set(msi_used, hwirq, nr_irqs);
+- mutex_unlock(&msi_used_lock);
+
+ for (i = 0; i < nr_irqs; i++) {
+ irq_domain_set_info(domain, virq + i, hwirq + i,
+@@ -250,7 +246,7 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq,
+ NULL, NULL);
+ }
+
+- return hwirq;
++ return 0;
+ }
+
+ static void armada_370_xp_msi_free(struct irq_domain *domain,
+@@ -259,7 +255,7 @@ static void armada_370_xp_msi_free(struct irq_domain *domain,
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+
+ mutex_lock(&msi_used_lock);
+- bitmap_clear(msi_used, d->hwirq, nr_irqs);
++ bitmap_release_region(msi_used, d->hwirq, order_base_2(nr_irqs));
+ mutex_unlock(&msi_used_lock);
+ }
+
+diff --git a/drivers/irqchip/irq-aspeed-scu-ic.c b/drivers/irqchip/irq-aspeed-scu-ic.c
+index f3c6855a4cefb..18b77c3e6db4b 100644
+--- a/drivers/irqchip/irq-aspeed-scu-ic.c
++++ b/drivers/irqchip/irq-aspeed-scu-ic.c
+@@ -76,8 +76,8 @@ static void aspeed_scu_ic_irq_handler(struct irq_desc *desc)
+ generic_handle_domain_irq(scu_ic->irq_domain,
+ bit - scu_ic->irq_shift);
+
+- regmap_update_bits(scu_ic->scu, scu_ic->reg, mask,
+- BIT(bit + ASPEED_SCU_IC_STATUS_SHIFT));
++ regmap_write_bits(scu_ic->scu, scu_ic->reg, mask,
++ BIT(bit + ASPEED_SCU_IC_STATUS_SHIFT));
+ }
+
+ chained_irq_exit(chip, desc);
+diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c
+index e3483789f4df3..1bd0621c4ce2a 100644
+--- a/drivers/irqchip/irq-bcm6345-l1.c
++++ b/drivers/irqchip/irq-bcm6345-l1.c
+@@ -140,7 +140,7 @@ static void bcm6345_l1_irq_handle(struct irq_desc *desc)
+ for_each_set_bit(hwirq, &pending, IRQS_PER_WORD) {
+ irq = irq_linear_revmap(intc->domain, base + hwirq);
+ if (irq)
+- do_IRQ(irq);
++ generic_handle_irq(irq);
+ else
+ spurious_interrupt();
+ }
+diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
+index eb0882d153666..fc1bfffc468f3 100644
+--- a/drivers/irqchip/irq-gic-v3-its.c
++++ b/drivers/irqchip/irq-gic-v3-its.c
+@@ -742,7 +742,7 @@ static struct its_collection *its_build_invall_cmd(struct its_node *its,
+
+ its_fixup_cmd(cmd);
+
+- return NULL;
++ return desc->its_invall_cmd.col;
+ }
+
+ static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
+@@ -3007,18 +3007,12 @@ static int __init allocate_lpi_tables(void)
+ return 0;
+ }
+
+-static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
++static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
+ {
+ u32 count = 1000000; /* 1s! */
+ bool clean;
+ u64 val;
+
+- val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
+- val &= ~GICR_VPENDBASER_Valid;
+- val &= ~clr;
+- val |= set;
+- gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
+-
+ do {
+ val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
+ clean = !(val & GICR_VPENDBASER_Dirty);
+@@ -3029,10 +3023,26 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
+ }
+ } while (!clean && count);
+
+- if (unlikely(val & GICR_VPENDBASER_Dirty)) {
++ if (unlikely(!clean))
+ pr_err_ratelimited("ITS virtual pending table not cleaning\n");
++
++ return val;
++}
++
++static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
++{
++ u64 val;
++
++ /* Make sure we wait until the RD is done with the initial scan */
++ val = read_vpend_dirty_clear(vlpi_base);
++ val &= ~GICR_VPENDBASER_Valid;
++ val &= ~clr;
++ val |= set;
++ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
++
++ val = read_vpend_dirty_clear(vlpi_base);
++ if (unlikely(val & GICR_VPENDBASER_Dirty))
+ val |= GICR_VPENDBASER_PendingLast;
+- }
+
+ return val;
+ }
+diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
+index fd4e9a37fea67..1269284461dae 100644
+--- a/drivers/irqchip/irq-gic-v3.c
++++ b/drivers/irqchip/irq-gic-v3.c
+@@ -206,11 +206,11 @@ static inline void __iomem *gic_dist_base(struct irq_data *d)
+ }
+ }
+
+-static void gic_do_wait_for_rwp(void __iomem *base)
++static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
+ {
+ u32 count = 1000000; /* 1s! */
+
+- while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
++ while (readl_relaxed(base + GICD_CTLR) & bit) {
+ count--;
+ if (!count) {
+ pr_err_ratelimited("RWP timeout, gone fishing\n");
+@@ -224,13 +224,13 @@ static void gic_do_wait_for_rwp(void __iomem *base)
+ /* Wait for completion of a distributor change */
+ static void gic_dist_wait_for_rwp(void)
+ {
+- gic_do_wait_for_rwp(gic_data.dist_base);
++ gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
+ }
+
+ /* Wait for completion of a redistributor change */
+ static void gic_redist_wait_for_rwp(void)
+ {
+- gic_do_wait_for_rwp(gic_data_rdist_rd_base());
++ gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
+ }
+
+ #ifdef CONFIG_ARM64
+@@ -920,6 +920,22 @@ static int __gic_update_rdist_properties(struct redist_region *region,
+ {
+ u64 typer = gic_read_typer(ptr + GICR_TYPER);
+
++ /* Boot-time cleanip */
++ if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
++ u64 val;
++
++ /* Deactivate any present vPE */
++ val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
++ if (val & GICR_VPENDBASER_Valid)
++ gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
++ ptr + SZ_128K + GICR_VPENDBASER);
++
++ /* Mark the VPE table as invalid */
++ val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
++ val &= ~GICR_VPROPBASER_4_1_VALID;
++ gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
++ }
++
+ gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
+
+ /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */
+@@ -1450,6 +1466,12 @@ static int gic_irq_domain_translate(struct irq_domain *d,
+ if(fwspec->param_count != 2)
+ return -EINVAL;
+
++ if (fwspec->param[0] < 16) {
++ pr_err(FW_BUG "Illegal GSI%d translation request\n",
++ fwspec->param[0]);
++ return -EINVAL;
++ }
++
+ *hwirq = fwspec->param[0];
+ *type = fwspec->param[1];
+
+diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
+index 5f22c9d65e578..99077f30f6999 100644
+--- a/drivers/irqchip/irq-gic.c
++++ b/drivers/irqchip/irq-gic.c
+@@ -1085,6 +1085,12 @@ static int gic_irq_domain_translate(struct irq_domain *d,
+ if(fwspec->param_count != 2)
+ return -EINVAL;
+
++ if (fwspec->param[0] < 16) {
++ pr_err(FW_BUG "Illegal GSI%d translation request\n",
++ fwspec->param[0]);
++ return -EINVAL;
++ }
++
+ *hwirq = fwspec->param[0];
+ *type = fwspec->param[1];
+
+diff --git a/drivers/irqchip/irq-nvic.c b/drivers/irqchip/irq-nvic.c
+index b31c4cff4d3a5..47b3b165479e8 100644
+--- a/drivers/irqchip/irq-nvic.c
++++ b/drivers/irqchip/irq-nvic.c
+@@ -26,7 +26,7 @@
+
+ #define NVIC_ISER 0x000
+ #define NVIC_ICER 0x080
+-#define NVIC_IPR 0x300
++#define NVIC_IPR 0x400
+
+ #define NVIC_MAX_BANKS 16
+ /*
+@@ -92,6 +92,7 @@ static int __init nvic_of_init(struct device_node *node,
+
+ if (!nvic_irq_domain) {
+ pr_warn("Failed to allocate irq domain\n");
++ iounmap(nvic_base);
+ return -ENOMEM;
+ }
+
+@@ -101,6 +102,7 @@ static int __init nvic_of_init(struct device_node *node,
+ if (ret) {
+ pr_warn("Failed to allocate irq chips\n");
+ irq_domain_remove(nvic_irq_domain);
++ iounmap(nvic_base);
+ return ret;
+ }
+
+diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c
+index fd9f275592d29..50a56820c99bc 100644
+--- a/drivers/irqchip/irq-realtek-rtl.c
++++ b/drivers/irqchip/irq-realtek-rtl.c
+@@ -62,7 +62,7 @@ static struct irq_chip realtek_ictl_irq = {
+
+ static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
+ {
+- irq_set_chip_and_handler(hw, &realtek_ictl_irq, handle_level_irq);
++ irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq);
+
+ return 0;
+ }
+@@ -76,16 +76,20 @@ static void realtek_irq_dispatch(struct irq_desc *desc)
+ {
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct irq_domain *domain;
+- unsigned int pending;
++ unsigned long pending;
++ unsigned int soc_int;
+
+ chained_irq_enter(chip, desc);
+ pending = readl(REG(RTL_ICTL_GIMR)) & readl(REG(RTL_ICTL_GISR));
++
+ if (unlikely(!pending)) {
+ spurious_interrupt();
+ goto out;
+ }
++
+ domain = irq_desc_get_handler_data(desc);
+- generic_handle_domain_irq(domain, __ffs(pending));
++ for_each_set_bit(soc_int, &pending, 32)
++ generic_handle_domain_irq(domain, soc_int);
+
+ out:
+ chained_irq_exit(chip, desc);
+@@ -95,7 +99,8 @@ out:
+ * SoC interrupts are cascaded to MIPS CPU interrupts according to the
+ * interrupt-map in the device tree. Each SoC interrupt gets 4 bits for
+ * the CPU interrupt in an Interrupt Routing Register. Max 32 SoC interrupts
+- * thus go into 4 IRRs.
++ * thus go into 4 IRRs. A routing value of '0' means the interrupt is left
++ * disconnected. Routing values {1..15} connect to output lines {0..14}.
+ */
+ static int __init map_interrupts(struct device_node *node, struct irq_domain *domain)
+ {
+@@ -134,7 +139,7 @@ static int __init map_interrupts(struct device_node *node, struct irq_domain *do
+ of_node_put(cpu_ictl);
+
+ cpu_int = be32_to_cpup(imap + 2);
+- if (cpu_int > 7)
++ if (cpu_int > 7 || cpu_int < 2)
+ return -EINVAL;
+
+ if (!(mips_irqs_set & BIT(cpu_int))) {
+@@ -143,7 +148,8 @@ static int __init map_interrupts(struct device_node *node, struct irq_domain *do
+ mips_irqs_set |= BIT(cpu_int);
+ }
+
+- regs[(soc_int * 4) / 32] |= cpu_int << (soc_int * 4) % 32;
++ /* Use routing values (1..6) for CPU interrupts (2..7) */
++ regs[(soc_int * 4) / 32] |= (cpu_int - 1) << (soc_int * 4) % 32;
+ imap += 3;
+ }
+
+diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
+index cf74cfa820453..09cc98266d30f 100644
+--- a/drivers/irqchip/irq-sifive-plic.c
++++ b/drivers/irqchip/irq-sifive-plic.c
+@@ -163,7 +163,13 @@ static void plic_irq_eoi(struct irq_data *d)
+ {
+ struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
+
+- writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
++ if (irqd_irq_masked(d)) {
++ plic_irq_unmask(d);
++ writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
++ plic_irq_mask(d);
++ } else {
++ writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
++ }
+ }
+
+ static struct irq_chip plic_chip = {
+@@ -392,3 +398,4 @@ out_free_priv:
+
+ IRQCHIP_DECLARE(sifive_plic, "sifive,plic-1.0.0", plic_init);
+ IRQCHIP_DECLARE(riscv_plic0, "riscv,plic0", plic_init); /* for legacy systems */
++IRQCHIP_DECLARE(thead_c900_plic, "thead,c900-plic", plic_init); /* for firmware driver */
+diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c
+index 173e6520e06ec..c0b457f26ec41 100644
+--- a/drivers/irqchip/qcom-pdc.c
++++ b/drivers/irqchip/qcom-pdc.c
+@@ -56,17 +56,18 @@ static u32 pdc_reg_read(int reg, u32 i)
+ static void pdc_enable_intr(struct irq_data *d, bool on)
+ {
+ int pin_out = d->hwirq;
++ unsigned long flags;
+ u32 index, mask;
+ u32 enable;
+
+ index = pin_out / 32;
+ mask = pin_out % 32;
+
+- raw_spin_lock(&pdc_lock);
++ raw_spin_lock_irqsave(&pdc_lock, flags);
+ enable = pdc_reg_read(IRQ_ENABLE_BANK, index);
+ enable = on ? ENABLE_INTR(enable, mask) : CLEAR_INTR(enable, mask);
+ pdc_reg_write(IRQ_ENABLE_BANK, index, enable);
+- raw_spin_unlock(&pdc_lock);
++ raw_spin_unlock_irqrestore(&pdc_lock, flags);
+ }
+
+ static void qcom_pdc_gic_disable(struct irq_data *d)
+diff --git a/drivers/isdn/hardware/mISDN/hfcpci.c b/drivers/isdn/hardware/mISDN/hfcpci.c
+index bd087cca1c1d2..af17459c1a5c0 100644
+--- a/drivers/isdn/hardware/mISDN/hfcpci.c
++++ b/drivers/isdn/hardware/mISDN/hfcpci.c
+@@ -2005,7 +2005,11 @@ setup_hw(struct hfc_pci *hc)
+ }
+ /* Allocate memory for FIFOS */
+ /* the memory needs to be on a 32k boundary within the first 4G */
+- dma_set_mask(&hc->pdev->dev, 0xFFFF8000);
++ if (dma_set_mask(&hc->pdev->dev, 0xFFFF8000)) {
++ printk(KERN_WARNING
++ "HFC-PCI: No usable DMA configuration!\n");
++ return -EIO;
++ }
+ buffer = dma_alloc_coherent(&hc->pdev->dev, 0x8000, &hc->hw.dmahandle,
+ GFP_KERNEL);
+ /* We silently assume the address is okay if nonzero */
+diff --git a/drivers/isdn/mISDN/core.c b/drivers/isdn/mISDN/core.c
+index 55891e4204460..a41b4b2645941 100644
+--- a/drivers/isdn/mISDN/core.c
++++ b/drivers/isdn/mISDN/core.c
+@@ -381,7 +381,7 @@ mISDNInit(void)
+ err = mISDN_inittimer(&debug);
+ if (err)
+ goto error2;
+- err = l1_init(&debug);
++ err = Isdnl1_Init(&debug);
+ if (err)
+ goto error3;
+ err = Isdnl2_Init(&debug);
+@@ -395,7 +395,7 @@ mISDNInit(void)
+ error5:
+ Isdnl2_cleanup();
+ error4:
+- l1_cleanup();
++ Isdnl1_cleanup();
+ error3:
+ mISDN_timer_cleanup();
+ error2:
+@@ -408,7 +408,7 @@ static void mISDN_cleanup(void)
+ {
+ misdn_sock_cleanup();
+ Isdnl2_cleanup();
+- l1_cleanup();
++ Isdnl1_cleanup();
+ mISDN_timer_cleanup();
+ class_unregister(&mISDN_class);
+
+diff --git a/drivers/isdn/mISDN/core.h b/drivers/isdn/mISDN/core.h
+index 23b44d3033279..42599f49c189d 100644
+--- a/drivers/isdn/mISDN/core.h
++++ b/drivers/isdn/mISDN/core.h
+@@ -60,8 +60,8 @@ struct Bprotocol *get_Bprotocol4id(u_int);
+ extern int mISDN_inittimer(u_int *);
+ extern void mISDN_timer_cleanup(void);
+
+-extern int l1_init(u_int *);
+-extern void l1_cleanup(void);
++extern int Isdnl1_Init(u_int *);
++extern void Isdnl1_cleanup(void);
+ extern int Isdnl2_Init(u_int *);
+ extern void Isdnl2_cleanup(void);
+
+diff --git a/drivers/isdn/mISDN/dsp_pipeline.c b/drivers/isdn/mISDN/dsp_pipeline.c
+index e11ca6bbc7f41..c3b2c99b5cd5c 100644
+--- a/drivers/isdn/mISDN/dsp_pipeline.c
++++ b/drivers/isdn/mISDN/dsp_pipeline.c
+@@ -192,7 +192,7 @@ void dsp_pipeline_destroy(struct dsp_pipeline *pipeline)
+ int dsp_pipeline_build(struct dsp_pipeline *pipeline, const char *cfg)
+ {
+ int found = 0;
+- char *dup, *tok, *name, *args;
++ char *dup, *next, *tok, *name, *args;
+ struct dsp_element_entry *entry, *n;
+ struct dsp_pipeline_entry *pipeline_entry;
+ struct mISDN_dsp_element *elem;
+@@ -203,10 +203,10 @@ int dsp_pipeline_build(struct dsp_pipeline *pipeline, const char *cfg)
+ if (!list_empty(&pipeline->list))
+ _dsp_pipeline_destroy(pipeline);
+
+- dup = kstrdup(cfg, GFP_ATOMIC);
++ dup = next = kstrdup(cfg, GFP_ATOMIC);
+ if (!dup)
+ return 0;
+- while ((tok = strsep(&dup, "|"))) {
++ while ((tok = strsep(&next, "|"))) {
+ if (!strlen(tok))
+ continue;
+ name = strsep(&tok, "(");
+diff --git a/drivers/isdn/mISDN/layer1.c b/drivers/isdn/mISDN/layer1.c
+index 98a3bc6c17009..7b31c25a550e3 100644
+--- a/drivers/isdn/mISDN/layer1.c
++++ b/drivers/isdn/mISDN/layer1.c
+@@ -398,7 +398,7 @@ create_l1(struct dchannel *dch, dchannel_l1callback *dcb) {
+ EXPORT_SYMBOL(create_l1);
+
+ int
+-l1_init(u_int *deb)
++Isdnl1_Init(u_int *deb)
+ {
+ debug = deb;
+ l1fsm_s.state_count = L1S_STATE_COUNT;
+@@ -409,7 +409,7 @@ l1_init(u_int *deb)
+ }
+
+ void
+-l1_cleanup(void)
++Isdnl1_cleanup(void)
+ {
+ mISDN_FsmFree(&l1fsm_s);
+ }
+diff --git a/drivers/leds/leds-lp55xx-common.c b/drivers/leds/leds-lp55xx-common.c
+index d1657c46ee2f8..9fdfc1b9a1a0c 100644
+--- a/drivers/leds/leds-lp55xx-common.c
++++ b/drivers/leds/leds-lp55xx-common.c
+@@ -439,6 +439,8 @@ int lp55xx_init_device(struct lp55xx_chip *chip)
+ return -EINVAL;
+
+ if (pdata->enable_gpiod) {
++ gpiod_direction_output(pdata->enable_gpiod, 0);
++
+ gpiod_set_consumer_name(pdata->enable_gpiod, "LP55xx enable");
+ gpiod_set_value(pdata->enable_gpiod, 0);
+ usleep_range(1000, 2000); /* Keep enable down at least 1ms */
+@@ -694,7 +696,7 @@ struct lp55xx_platform_data *lp55xx_of_populate_pdata(struct device *dev,
+ of_property_read_u8(np, "clock-mode", &pdata->clock_mode);
+
+ pdata->enable_gpiod = devm_gpiod_get_optional(dev, "enable",
+- GPIOD_OUT_LOW);
++ GPIOD_ASIS);
+ if (IS_ERR(pdata->enable_gpiod))
+ return ERR_CAST(pdata->enable_gpiod);
+
+diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c
+index 0ce75c6b36b65..850d4004c50e0 100644
+--- a/drivers/mailbox/imx-mailbox.c
++++ b/drivers/mailbox/imx-mailbox.c
+@@ -13,6 +13,7 @@
+ #include <linux/module.h>
+ #include <linux/of_device.h>
+ #include <linux/pm_runtime.h>
++#include <linux/suspend.h>
+ #include <linux/slab.h>
+
+ #define IMX_MU_CHANS 16
+@@ -67,6 +68,7 @@ struct imx_mu_priv {
+ const struct imx_mu_dcfg *dcfg;
+ struct clk *clk;
+ int irq;
++ bool suspend;
+
+ u32 xcr[4];
+
+@@ -307,6 +309,9 @@ static irqreturn_t imx_mu_isr(int irq, void *p)
+ return IRQ_NONE;
+ }
+
++ if (priv->suspend)
++ pm_system_wakeup();
++
+ return IRQ_HANDLED;
+ }
+
+@@ -652,6 +657,8 @@ static int __maybe_unused imx_mu_suspend_noirq(struct device *dev)
+ priv->xcr[i] = imx_mu_read(priv, priv->dcfg->xCR[i]);
+ }
+
++ priv->suspend = true;
++
+ return 0;
+ }
+
+@@ -668,11 +675,13 @@ static int __maybe_unused imx_mu_resume_noirq(struct device *dev)
+ * send failed, may lead to system freeze. This issue
+ * is observed by testing freeze mode suspend.
+ */
+- if (!imx_mu_read(priv, priv->dcfg->xCR[0]) && !priv->clk) {
++ if (!priv->clk && !imx_mu_read(priv, priv->dcfg->xCR[0])) {
+ for (i = 0; i < IMX_MU_xCR_MAX; i++)
+ imx_mu_write(priv, priv->xcr[i], priv->dcfg->xCR[i]);
+ }
+
++ priv->suspend = false;
++
+ return 0;
+ }
+
+diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c
+index 0d6e2231a2c75..4e34854d12389 100644
+--- a/drivers/mailbox/mailbox-mpfs.c
++++ b/drivers/mailbox/mailbox-mpfs.c
+@@ -232,7 +232,7 @@ static int mpfs_mbox_probe(struct platform_device *pdev)
+ }
+
+ static const struct of_device_id mpfs_mbox_of_match[] = {
+- {.compatible = "microchip,polarfire-soc-mailbox", },
++ {.compatible = "microchip,mpfs-mailbox", },
+ {},
+ };
+ MODULE_DEVICE_TABLE(of, mpfs_mbox_of_match);
+diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c
+index 64175a893312e..3583c2aad0edc 100644
+--- a/drivers/mailbox/mtk-cmdq-mailbox.c
++++ b/drivers/mailbox/mtk-cmdq-mailbox.c
+@@ -195,7 +195,6 @@ static void cmdq_task_exec_done(struct cmdq_task *task, int sta)
+ struct cmdq_task_cb *cb = &task->pkt->async_cb;
+ struct cmdq_cb_data data;
+
+- WARN_ON(cb->cb == (cmdq_async_flush_cb)NULL);
+ data.sta = sta;
+ data.data = cb->data;
+ data.pkt = task->pkt;
+@@ -532,7 +531,8 @@ static int cmdq_probe(struct platform_device *pdev)
+ struct device_node *phandle = dev->of_node;
+ struct device_node *node;
+ int alias_id = 0;
+- char clk_name[4] = "gce";
++ static const char * const clk_name = "gce";
++ static const char * const clk_names[] = { "gce0", "gce1" };
+
+ cmdq = devm_kzalloc(dev, sizeof(*cmdq), GFP_KERNEL);
+ if (!cmdq)
+@@ -570,12 +570,9 @@ static int cmdq_probe(struct platform_device *pdev)
+
+ if (cmdq->gce_num > 1) {
+ for_each_child_of_node(phandle->parent, node) {
+- char clk_id[8];
+-
+ alias_id = of_alias_get_id(node, clk_name);
+- if (alias_id < cmdq->gce_num) {
+- snprintf(clk_id, sizeof(clk_id), "%s%d", clk_name, alias_id);
+- cmdq->clocks[alias_id].id = clk_id;
++ if (alias_id >= 0 && alias_id < cmdq->gce_num) {
++ cmdq->clocks[alias_id].id = clk_names[alias_id];
+ cmdq->clocks[alias_id].clk = of_clk_get(node, 0);
+ if (IS_ERR(cmdq->clocks[alias_id].clk)) {
+ dev_err(dev, "failed to get gce clk: %d\n", alias_id);
+@@ -663,7 +660,7 @@ static const struct gce_plat gce_plat_v5 = {
+ .thread_nr = 24,
+ .shift = 3,
+ .control_by_sw = true,
+- .gce_num = 2
++ .gce_num = 1
+ };
+
+ static const struct gce_plat gce_plat_v6 = {
+diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c
+index acd0675da681e..78f7265039c66 100644
+--- a/drivers/mailbox/tegra-hsp.c
++++ b/drivers/mailbox/tegra-hsp.c
+@@ -412,6 +412,11 @@ static int tegra_hsp_mailbox_flush(struct mbox_chan *chan,
+ value = tegra_hsp_channel_readl(ch, HSP_SM_SHRD_MBOX);
+ if ((value & HSP_SM_SHRD_MBOX_FULL) == 0) {
+ mbox_chan_txdone(chan, 0);
++
++ /* Wait until channel is empty */
++ if (chan->active_req != NULL)
++ continue;
++
+ return 0;
+ }
+
+diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
+index 0595559de174a..8eecc9df319b5 100644
+--- a/drivers/md/bcache/btree.c
++++ b/drivers/md/bcache/btree.c
+@@ -378,7 +378,7 @@ static void do_btree_node_write(struct btree *b)
+ struct bvec_iter_all iter_all;
+
+ bio_for_each_segment_all(bv, b->bio, iter_all) {
+- memcpy(bvec_virt(bv), addr, PAGE_SIZE);
++ memcpy(page_address(bv->bv_page), addr, PAGE_SIZE);
+ addr += PAGE_SIZE;
+ }
+
+@@ -2060,9 +2060,11 @@ int bch_btree_check(struct cache_set *c)
+ }
+ }
+
++ /*
++ * Must wait for all threads to stop.
++ */
+ wait_event_interruptible(check_state->wait,
+- atomic_read(&check_state->started) == 0 ||
+- test_bit(CACHE_SET_IO_DISABLE, &c->flags));
++ atomic_read(&check_state->started) == 0);
+
+ for (i = 0; i < check_state->total_threads; i++) {
+ if (check_state->infos[i].result) {
+diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
+index f2874c77ff797..3f72c0f40613d 100644
+--- a/drivers/md/bcache/super.c
++++ b/drivers/md/bcache/super.c
+@@ -885,9 +885,9 @@ static void bcache_device_free(struct bcache_device *d)
+ bcache_device_detach(d);
+
+ if (disk) {
+- blk_cleanup_disk(disk);
+ ida_simple_remove(&bcache_device_idx,
+ first_minor_to_idx(disk->first_minor));
++ blk_cleanup_disk(disk);
+ }
+
+ bioset_exit(&d->bio_split);
+diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
+index 8120da278161e..1c03a3b72f8aa 100644
+--- a/drivers/md/bcache/writeback.c
++++ b/drivers/md/bcache/writeback.c
+@@ -998,9 +998,11 @@ void bch_sectors_dirty_init(struct bcache_device *d)
+ }
+ }
+
++ /*
++ * Must wait for all threads to stop.
++ */
+ wait_event_interruptible(state->wait,
+- atomic_read(&state->started) == 0 ||
+- test_bit(CACHE_SET_IO_DISABLE, &c->flags));
++ atomic_read(&state->started) == 0);
+
+ out:
+ kfree(state);
+diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
+index 55dccdfbcb22e..5a7d270b32c01 100644
+--- a/drivers/md/dm-core.h
++++ b/drivers/md/dm-core.h
+@@ -65,6 +65,8 @@ struct mapped_device {
+ struct gendisk *disk;
+ struct dax_device *dax_dev;
+
++ unsigned long __percpu *pending_io;
++
+ /*
+ * A list of ios that arrived while we were suspended.
+ */
+diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
+index 916b7da16de25..154139bf7d22b 100644
+--- a/drivers/md/dm-crypt.c
++++ b/drivers/md/dm-crypt.c
+@@ -2579,7 +2579,7 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string
+
+ static int get_key_size(char **key_string)
+ {
+- return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1;
++ return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
+ }
+
+ #endif /* CONFIG_KEYS */
+diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
+index dc03b70f6e65c..e2a51c184a254 100644
+--- a/drivers/md/dm-integrity.c
++++ b/drivers/md/dm-integrity.c
+@@ -2459,9 +2459,11 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
+ dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
+ sec &= ~(sector_t)(ic->sectors_per_block - 1);
+ }
++ if (unlikely(sec >= ic->provided_data_sectors)) {
++ journal_entry_set_unused(je);
++ continue;
++ }
+ }
+- if (unlikely(sec >= ic->provided_data_sectors))
+- continue;
+ get_area_and_offset(ic, sec, &area, &offset);
+ restore_last_bytes(ic, access_journal_data(ic, i, j), je);
+ for (k = j + 1; k < ic->journal_section_entries; k++) {
+@@ -4381,6 +4383,7 @@ try_smaller_buffer:
+ }
+
+ if (ic->internal_hash) {
++ size_t recalc_tags_size;
+ ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1);
+ if (!ic->recalc_wq ) {
+ ti->error = "Cannot allocate workqueue";
+@@ -4394,8 +4397,10 @@ try_smaller_buffer:
+ r = -ENOMEM;
+ goto bad;
+ }
+- ic->recalc_tags = kvmalloc_array(RECALC_SECTORS >> ic->sb->log2_sectors_per_block,
+- ic->tag_size, GFP_KERNEL);
++ recalc_tags_size = (RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size;
++ if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size)
++ recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size;
++ ic->recalc_tags = kvmalloc(recalc_tags_size, GFP_KERNEL);
+ if (!ic->recalc_tags) {
+ ti->error = "Cannot allocate tags for recalculating";
+ r = -ENOMEM;
+diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
+index 21fe8652b095b..901abd6dea419 100644
+--- a/drivers/md/dm-ioctl.c
++++ b/drivers/md/dm-ioctl.c
+@@ -18,6 +18,7 @@
+ #include <linux/dm-ioctl.h>
+ #include <linux/hdreg.h>
+ #include <linux/compat.h>
++#include <linux/nospec.h>
+
+ #include <linux/uaccess.h>
+ #include <linux/ima.h>
+@@ -1788,6 +1789,7 @@ static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
+ if (unlikely(cmd >= ARRAY_SIZE(_ioctls)))
+ return NULL;
+
++ cmd = array_index_nospec(cmd, ARRAY_SIZE(_ioctls));
+ *ioctl_flags = _ioctls[cmd].flags;
+ return _ioctls[cmd].fn;
+ }
+diff --git a/drivers/md/dm-ps-historical-service-time.c b/drivers/md/dm-ps-historical-service-time.c
+index 1856a1b125cc1..82f2a06153dc0 100644
+--- a/drivers/md/dm-ps-historical-service-time.c
++++ b/drivers/md/dm-ps-historical-service-time.c
+@@ -432,7 +432,7 @@ static struct dm_path *hst_select_path(struct path_selector *ps,
+ {
+ struct selector *s = ps->context;
+ struct path_info *pi = NULL, *best = NULL;
+- u64 time_now = sched_clock();
++ u64 time_now = ktime_get_ns();
+ struct dm_path *ret = NULL;
+ unsigned long flags;
+
+@@ -473,7 +473,7 @@ static int hst_start_io(struct path_selector *ps, struct dm_path *path,
+
+ static u64 path_service_time(struct path_info *pi, u64 start_time)
+ {
+- u64 sched_now = ktime_get_ns();
++ u64 now = ktime_get_ns();
+
+ /* if a previous disk request has finished after this IO was
+ * sent to the hardware, pretend the submission happened
+@@ -482,11 +482,11 @@ static u64 path_service_time(struct path_info *pi, u64 start_time)
+ if (time_after64(pi->last_finish, start_time))
+ start_time = pi->last_finish;
+
+- pi->last_finish = sched_now;
+- if (time_before64(sched_now, start_time))
++ pi->last_finish = now;
++ if (time_before64(now, start_time))
+ return 0;
+
+- return sched_now - start_time;
++ return now - start_time;
+ }
+
+ static int hst_end_io(struct path_selector *ps, struct dm_path *path,
+diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
+index a896dea9750e4..53a9b16c7b2e6 100644
+--- a/drivers/md/dm-rq.c
++++ b/drivers/md/dm-rq.c
+@@ -500,8 +500,13 @@ static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
+
+ if (unlikely(!ti)) {
+ int srcu_idx;
+- struct dm_table *map = dm_get_live_table(md, &srcu_idx);
++ struct dm_table *map;
+
++ map = dm_get_live_table(md, &srcu_idx);
++ if (unlikely(!map)) {
++ dm_put_live_table(md, srcu_idx);
++ return BLK_STS_RESOURCE;
++ }
+ ti = dm_table_find_target(map, 0);
+ dm_put_live_table(md, srcu_idx);
+ }
+diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c
+index 35d368c418d03..0e039a8c0bf2e 100644
+--- a/drivers/md/dm-stats.c
++++ b/drivers/md/dm-stats.c
+@@ -195,6 +195,7 @@ void dm_stats_init(struct dm_stats *stats)
+
+ mutex_init(&stats->mutex);
+ INIT_LIST_HEAD(&stats->list);
++ stats->precise_timestamps = false;
+ stats->last = alloc_percpu(struct dm_stats_last_position);
+ for_each_possible_cpu(cpu) {
+ last = per_cpu_ptr(stats->last, cpu);
+@@ -231,6 +232,22 @@ void dm_stats_cleanup(struct dm_stats *stats)
+ mutex_destroy(&stats->mutex);
+ }
+
++static void dm_stats_recalc_precise_timestamps(struct dm_stats *stats)
++{
++ struct list_head *l;
++ struct dm_stat *tmp_s;
++ bool precise_timestamps = false;
++
++ list_for_each(l, &stats->list) {
++ tmp_s = container_of(l, struct dm_stat, list_entry);
++ if (tmp_s->stat_flags & STAT_PRECISE_TIMESTAMPS) {
++ precise_timestamps = true;
++ break;
++ }
++ }
++ stats->precise_timestamps = precise_timestamps;
++}
++
+ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
+ sector_t step, unsigned stat_flags,
+ unsigned n_histogram_entries,
+@@ -376,6 +393,9 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
+ }
+ ret_id = s->id;
+ list_add_tail_rcu(&s->list_entry, l);
++
++ dm_stats_recalc_precise_timestamps(stats);
++
+ mutex_unlock(&stats->mutex);
+
+ resume_callback(md);
+@@ -418,6 +438,9 @@ static int dm_stats_delete(struct dm_stats *stats, int id)
+ }
+
+ list_del_rcu(&s->list_entry);
++
++ dm_stats_recalc_precise_timestamps(stats);
++
+ mutex_unlock(&stats->mutex);
+
+ /*
+@@ -621,13 +644,14 @@ static void __dm_stat_bio(struct dm_stat *s, int bi_rw,
+
+ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw,
+ sector_t bi_sector, unsigned bi_sectors, bool end,
+- unsigned long duration_jiffies,
++ unsigned long start_time,
+ struct dm_stats_aux *stats_aux)
+ {
+ struct dm_stat *s;
+ sector_t end_sector;
+ struct dm_stats_last_position *last;
+ bool got_precise_time;
++ unsigned long duration_jiffies = 0;
+
+ if (unlikely(!bi_sectors))
+ return;
+@@ -647,16 +671,16 @@ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw,
+ ));
+ WRITE_ONCE(last->last_sector, end_sector);
+ WRITE_ONCE(last->last_rw, bi_rw);
+- }
++ } else
++ duration_jiffies = jiffies - start_time;
+
+ rcu_read_lock();
+
+ got_precise_time = false;
+ list_for_each_entry_rcu(s, &stats->list, list_entry) {
+ if (s->stat_flags & STAT_PRECISE_TIMESTAMPS && !got_precise_time) {
+- if (!end)
+- stats_aux->duration_ns = ktime_to_ns(ktime_get());
+- else
++ /* start (!end) duration_ns is set by DM core's alloc_io() */
++ if (end)
+ stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns;
+ got_precise_time = true;
+ }
+diff --git a/drivers/md/dm-stats.h b/drivers/md/dm-stats.h
+index 2ddfae678f320..09c81a1ec057d 100644
+--- a/drivers/md/dm-stats.h
++++ b/drivers/md/dm-stats.h
+@@ -13,8 +13,7 @@ struct dm_stats {
+ struct mutex mutex;
+ struct list_head list; /* list of struct dm_stat */
+ struct dm_stats_last_position __percpu *last;
+- sector_t last_sector;
+- unsigned last_rw;
++ bool precise_timestamps;
+ };
+
+ struct dm_stats_aux {
+@@ -32,7 +31,7 @@ int dm_stats_message(struct mapped_device *md, unsigned argc, char **argv,
+
+ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw,
+ sector_t bi_sector, unsigned bi_sectors, bool end,
+- unsigned long duration_jiffies,
++ unsigned long start_time,
+ struct dm_stats_aux *aux);
+
+ static inline bool dm_stats_used(struct dm_stats *st)
+@@ -40,4 +39,10 @@ static inline bool dm_stats_used(struct dm_stats *st)
+ return !list_empty(&st->list);
+ }
+
++static inline void dm_stats_record_start(struct dm_stats *stats, struct dm_stats_aux *aux)
++{
++ if (unlikely(stats->precise_timestamps))
++ aux->duration_ns = ktime_to_ns(ktime_get());
++}
++
+ #endif
+diff --git a/drivers/md/dm.c b/drivers/md/dm.c
+index 76d9da49fda75..36449422e7e07 100644
+--- a/drivers/md/dm.c
++++ b/drivers/md/dm.c
+@@ -484,33 +484,48 @@ u64 dm_start_time_ns_from_clone(struct bio *bio)
+ }
+ EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
+
+-static void start_io_acct(struct dm_io *io)
++static bool bio_is_flush_with_data(struct bio *bio)
+ {
+- struct mapped_device *md = io->md;
+- struct bio *bio = io->orig_bio;
+-
+- io->start_time = bio_start_io_acct(bio);
+- if (unlikely(dm_stats_used(&md->stats)))
+- dm_stats_account_io(&md->stats, bio_data_dir(bio),
+- bio->bi_iter.bi_sector, bio_sectors(bio),
+- false, 0, &io->stats_aux);
++ return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
+ }
+
+-static void end_io_acct(struct mapped_device *md, struct bio *bio,
+- unsigned long start_time, struct dm_stats_aux *stats_aux)
++static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio,
++ unsigned long start_time, struct dm_stats_aux *stats_aux)
+ {
+- unsigned long duration = jiffies - start_time;
++ bool is_flush_with_data;
++ unsigned int bi_size;
+
+- bio_end_io_acct(bio, start_time);
++ /* If REQ_PREFLUSH set save any payload but do not account it */
++ is_flush_with_data = bio_is_flush_with_data(bio);
++ if (is_flush_with_data) {
++ bi_size = bio->bi_iter.bi_size;
++ bio->bi_iter.bi_size = 0;
++ }
++
++ if (!end)
++ bio_start_io_acct_time(bio, start_time);
++ else
++ bio_end_io_acct(bio, start_time);
+
+ if (unlikely(dm_stats_used(&md->stats)))
+ dm_stats_account_io(&md->stats, bio_data_dir(bio),
+ bio->bi_iter.bi_sector, bio_sectors(bio),
+- true, duration, stats_aux);
++ end, start_time, stats_aux);
++
++ /* Restore bio's payload so it does get accounted upon requeue */
++ if (is_flush_with_data)
++ bio->bi_iter.bi_size = bi_size;
++}
+
+- /* nudge anyone waiting on suspend queue */
+- if (unlikely(wq_has_sleeper(&md->wait)))
+- wake_up(&md->wait);
++static void start_io_acct(struct dm_io *io)
++{
++ dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux);
++}
++
++static void end_io_acct(struct mapped_device *md, struct bio *bio,
++ unsigned long start_time, struct dm_stats_aux *stats_aux)
++{
++ dm_io_acct(true, md, bio, start_time, stats_aux);
+ }
+
+ static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
+@@ -531,11 +546,14 @@ static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
+ io->magic = DM_IO_MAGIC;
+ io->status = 0;
+ atomic_set(&io->io_count, 1);
++ this_cpu_inc(*md->pending_io);
+ io->orig_bio = bio;
+ io->md = md;
+ spin_lock_init(&io->endio_lock);
+
+- start_io_acct(io);
++ io->start_time = jiffies;
++
++ dm_stats_record_start(&md->stats, &io->stats_aux);
+
+ return io;
+ }
+@@ -826,11 +844,17 @@ void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
+ stats_aux = io->stats_aux;
+ free_io(md, io);
+ end_io_acct(md, bio, start_time, &stats_aux);
++ smp_wmb();
++ this_cpu_dec(*md->pending_io);
++
++ /* nudge anyone waiting on suspend queue */
++ if (unlikely(wq_has_sleeper(&md->wait)))
++ wake_up(&md->wait);
+
+ if (io_error == BLK_STS_DM_REQUEUE)
+ return;
+
+- if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
++ if (bio_is_flush_with_data(bio)) {
+ /*
+ * Preflush done for flush with data, reissue
+ * without REQ_PREFLUSH.
+@@ -1514,9 +1538,6 @@ static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
+ ci->sector = bio->bi_iter.bi_sector;
+ }
+
+-#define __dm_part_stat_sub(part, field, subnd) \
+- (part_stat_get(part, field) -= (subnd))
+-
+ /*
+ * Entry point to split a bio into clones and submit them to the targets.
+ */
+@@ -1553,23 +1574,12 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md,
+ GFP_NOIO, &md->queue->bio_split);
+ ci.io->orig_bio = b;
+
+- /*
+- * Adjust IO stats for each split, otherwise upon queue
+- * reentry there will be redundant IO accounting.
+- * NOTE: this is a stop-gap fix, a proper fix involves
+- * significant refactoring of DM core's bio splitting
+- * (by eliminating DM's splitting and just using bio_split)
+- */
+- part_stat_lock();
+- __dm_part_stat_sub(dm_disk(md)->part0,
+- sectors[op_stat_group(bio_op(bio))], ci.sector_count);
+- part_stat_unlock();
+-
+ bio_chain(b, bio);
+ trace_block_split(b, bio->bi_iter.bi_sector);
+ ret = submit_bio_noacct(bio);
+ }
+ }
++ start_io_acct(ci.io);
+
+ /* drop the extra reference count */
+ dm_io_dec_pending(ci.io, errno_to_blk_status(error));
+@@ -1584,15 +1594,10 @@ static blk_qc_t dm_submit_bio(struct bio *bio)
+ struct dm_table *map;
+
+ map = dm_get_live_table(md, &srcu_idx);
+- if (unlikely(!map)) {
+- DMERR_LIMIT("%s: mapping table unavailable, erroring io",
+- dm_device_name(md));
+- bio_io_error(bio);
+- goto out;
+- }
+
+- /* If suspended, queue this IO for later */
+- if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
++ /* If suspended, or map not yet available, queue this IO for later */
++ if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) ||
++ unlikely(!map)) {
+ if (bio->bi_opf & REQ_NOWAIT)
+ bio_wouldblock_error(bio);
+ else if (bio->bi_opf & REQ_RAHEAD)
+@@ -1696,6 +1701,7 @@ static void cleanup_mapped_device(struct mapped_device *md)
+ md->dax_dev = NULL;
+ }
+
++ dm_cleanup_zoned_dev(md);
+ if (md->disk) {
+ spin_lock(&_minor_lock);
+ md->disk->private_data = NULL;
+@@ -1708,6 +1714,11 @@ static void cleanup_mapped_device(struct mapped_device *md)
+ blk_cleanup_disk(md->disk);
+ }
+
++ if (md->pending_io) {
++ free_percpu(md->pending_io);
++ md->pending_io = NULL;
++ }
++
+ cleanup_srcu_struct(&md->io_barrier);
+
+ mutex_destroy(&md->suspend_lock);
+@@ -1716,7 +1727,6 @@ static void cleanup_mapped_device(struct mapped_device *md)
+ mutex_destroy(&md->swap_bios_lock);
+
+ dm_mq_cleanup_mapped_device(md);
+- dm_cleanup_zoned_dev(md);
+ }
+
+ /*
+@@ -1794,8 +1804,10 @@ static struct mapped_device *alloc_dev(int minor)
+ if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
+ md->dax_dev = alloc_dax(md, md->disk->disk_name,
+ &dm_dax_ops, 0);
+- if (IS_ERR(md->dax_dev))
++ if (IS_ERR(md->dax_dev)) {
++ md->dax_dev = NULL;
+ goto bad;
++ }
+ }
+
+ format_dev_t(md->name, MKDEV(_major, minor));
+@@ -1804,6 +1816,10 @@ static struct mapped_device *alloc_dev(int minor)
+ if (!md->wq)
+ goto bad;
+
++ md->pending_io = alloc_percpu(unsigned long);
++ if (!md->pending_io)
++ goto bad;
++
+ dm_stats_init(&md->stats);
+
+ /* Populate the mapping, nobody knows we exist yet */
+@@ -2168,7 +2184,7 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
+ set_bit(DMF_FREEING, &md->flags);
+ spin_unlock(&_minor_lock);
+
+- blk_set_queue_dying(md->queue);
++ blk_mark_disk_dead(md->disk);
+
+ /*
+ * Take suspend_lock so that presuspend and postsuspend methods
+@@ -2219,16 +2235,13 @@ void dm_put(struct mapped_device *md)
+ }
+ EXPORT_SYMBOL_GPL(dm_put);
+
+-static bool md_in_flight_bios(struct mapped_device *md)
++static bool dm_in_flight_bios(struct mapped_device *md)
+ {
+ int cpu;
+- struct block_device *part = dm_disk(md)->part0;
+- long sum = 0;
++ unsigned long sum = 0;
+
+- for_each_possible_cpu(cpu) {
+- sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
+- sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
+- }
++ for_each_possible_cpu(cpu)
++ sum += *per_cpu_ptr(md->pending_io, cpu);
+
+ return sum != 0;
+ }
+@@ -2241,7 +2254,7 @@ static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int ta
+ while (true) {
+ prepare_to_wait(&md->wait, &wait, task_state);
+
+- if (!md_in_flight_bios(md))
++ if (!dm_in_flight_bios(md))
+ break;
+
+ if (signal_pending_state(task_state, current)) {
+@@ -2253,6 +2266,8 @@ static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int ta
+ }
+ finish_wait(&md->wait, &wait);
+
++ smp_rmb();
++
+ return r;
+ }
+
+diff --git a/drivers/md/md.c b/drivers/md/md.c
+index 6c0c3d0d905aa..5ce2648cbe5b5 100644
+--- a/drivers/md/md.c
++++ b/drivers/md/md.c
+@@ -51,6 +51,7 @@
+ #include <linux/hdreg.h>
+ #include <linux/proc_fs.h>
+ #include <linux/random.h>
++#include <linux/major.h>
+ #include <linux/module.h>
+ #include <linux/reboot.h>
+ #include <linux/file.h>
+@@ -2193,6 +2194,7 @@ super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
+
+ if (!num_sectors || num_sectors > max_sectors)
+ num_sectors = max_sectors;
++ rdev->sb_start = sb_start;
+ }
+ sb = page_address(rdev->sb_page);
+ sb->data_size = cpu_to_le64(num_sectors);
+@@ -2976,7 +2978,11 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
+ * -write_error - clears WriteErrorSeen
+ * {,-}failfast - set/clear FailFast
+ */
++
++ struct mddev *mddev = rdev->mddev;
+ int err = -EINVAL;
++ bool need_update_sb = false;
++
+ if (cmd_match(buf, "faulty") && rdev->mddev->pers) {
+ md_error(rdev->mddev, rdev);
+ if (test_bit(Faulty, &rdev->flags))
+@@ -2991,7 +2997,6 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
+ if (rdev->raid_disk >= 0)
+ err = -EBUSY;
+ else {
+- struct mddev *mddev = rdev->mddev;
+ err = 0;
+ if (mddev_is_clustered(mddev))
+ err = md_cluster_ops->remove_disk(mddev, rdev);
+@@ -3008,10 +3013,12 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
+ } else if (cmd_match(buf, "writemostly")) {
+ set_bit(WriteMostly, &rdev->flags);
+ mddev_create_serial_pool(rdev->mddev, rdev, false);
++ need_update_sb = true;
+ err = 0;
+ } else if (cmd_match(buf, "-writemostly")) {
+ mddev_destroy_serial_pool(rdev->mddev, rdev, false);
+ clear_bit(WriteMostly, &rdev->flags);
++ need_update_sb = true;
+ err = 0;
+ } else if (cmd_match(buf, "blocked")) {
+ set_bit(Blocked, &rdev->flags);
+@@ -3037,9 +3044,11 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
+ err = 0;
+ } else if (cmd_match(buf, "failfast")) {
+ set_bit(FailFast, &rdev->flags);
++ need_update_sb = true;
+ err = 0;
+ } else if (cmd_match(buf, "-failfast")) {
+ clear_bit(FailFast, &rdev->flags);
++ need_update_sb = true;
+ err = 0;
+ } else if (cmd_match(buf, "-insync") && rdev->raid_disk >= 0 &&
+ !test_bit(Journal, &rdev->flags)) {
+@@ -3118,6 +3127,8 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
+ clear_bit(ExternalBbl, &rdev->flags);
+ err = 0;
+ }
++ if (need_update_sb)
++ md_update_sb(mddev, 1);
+ if (!err)
+ sysfs_notify_dirent_safe(rdev->sysfs_state);
+ return err ? err : len;
+@@ -5862,13 +5873,6 @@ int md_run(struct mddev *mddev)
+ if (err)
+ goto exit_bio_set;
+ }
+- if (mddev->level != 1 && mddev->level != 10 &&
+- !bioset_initialized(&mddev->io_acct_set)) {
+- err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
+- offsetof(struct md_io_acct, bio_clone), 0);
+- if (err)
+- goto exit_sync_set;
+- }
+
+ spin_lock(&pers_lock);
+ pers = find_pers(mddev->level, mddev->clevel);
+@@ -6045,9 +6049,6 @@ bitmap_abort:
+ module_put(pers->owner);
+ md_bitmap_destroy(mddev);
+ abort:
+- if (mddev->level != 1 && mddev->level != 10)
+- bioset_exit(&mddev->io_acct_set);
+-exit_sync_set:
+ bioset_exit(&mddev->sync_set);
+ exit_bio_set:
+ bioset_exit(&mddev->bio_set);
+@@ -8580,6 +8581,23 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
+ }
+ EXPORT_SYMBOL_GPL(md_submit_discard_bio);
+
++int acct_bioset_init(struct mddev *mddev)
++{
++ int err = 0;
++
++ if (!bioset_initialized(&mddev->io_acct_set))
++ err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
++ offsetof(struct md_io_acct, bio_clone), 0);
++ return err;
++}
++EXPORT_SYMBOL_GPL(acct_bioset_init);
++
++void acct_bioset_exit(struct mddev *mddev)
++{
++ bioset_exit(&mddev->io_acct_set);
++}
++EXPORT_SYMBOL_GPL(acct_bioset_exit);
++
+ static void md_end_io_acct(struct bio *bio)
+ {
+ struct md_io_acct *md_io_acct = bio->bi_private;
+diff --git a/drivers/md/md.h b/drivers/md/md.h
+index 4c96c36bd01a1..62852d7011457 100644
+--- a/drivers/md/md.h
++++ b/drivers/md/md.h
+@@ -721,6 +721,8 @@ extern void md_error(struct mddev *mddev, struct md_rdev *rdev);
+ extern void md_finish_reshape(struct mddev *mddev);
+ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
+ struct bio *bio, sector_t start, sector_t size);
++int acct_bioset_init(struct mddev *mddev);
++void acct_bioset_exit(struct mddev *mddev);
+ void md_account_bio(struct mddev *mddev, struct bio **bio);
+
+ extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio);
+diff --git a/drivers/md/persistent-data/dm-btree-remove.c b/drivers/md/persistent-data/dm-btree-remove.c
+index 70532335c7c7e..cb670f16e98e9 100644
+--- a/drivers/md/persistent-data/dm-btree-remove.c
++++ b/drivers/md/persistent-data/dm-btree-remove.c
+@@ -423,9 +423,9 @@ static int rebalance_children(struct shadow_spine *s,
+
+ memcpy(n, dm_block_data(child),
+ dm_bm_block_size(dm_tm_get_bm(info->tm)));
+- dm_tm_unlock(info->tm, child);
+
+ dm_tm_dec(info->tm, dm_block_location(child));
++ dm_tm_unlock(info->tm, child);
+ return 0;
+ }
+
+diff --git a/drivers/md/persistent-data/dm-btree.c b/drivers/md/persistent-data/dm-btree.c
+index 0703ca7a7d9a4..5ce64e93aae74 100644
+--- a/drivers/md/persistent-data/dm-btree.c
++++ b/drivers/md/persistent-data/dm-btree.c
+@@ -81,14 +81,16 @@ void inc_children(struct dm_transaction_manager *tm, struct btree_node *n,
+ }
+
+ static int insert_at(size_t value_size, struct btree_node *node, unsigned index,
+- uint64_t key, void *value)
+- __dm_written_to_disk(value)
++ uint64_t key, void *value)
++ __dm_written_to_disk(value)
+ {
+ uint32_t nr_entries = le32_to_cpu(node->header.nr_entries);
++ uint32_t max_entries = le32_to_cpu(node->header.max_entries);
+ __le64 key_le = cpu_to_le64(key);
+
+ if (index > nr_entries ||
+- index >= le32_to_cpu(node->header.max_entries)) {
++ index >= max_entries ||
++ nr_entries >= max_entries) {
+ DMERR("too many entries in btree node for insert");
+ __dm_unbless_for_disk(value);
+ return -ENOMEM;
+diff --git a/drivers/md/persistent-data/dm-space-map-common.c b/drivers/md/persistent-data/dm-space-map-common.c
+index 4a6a2a9b4eb49..bfbfa750e0160 100644
+--- a/drivers/md/persistent-data/dm-space-map-common.c
++++ b/drivers/md/persistent-data/dm-space-map-common.c
+@@ -283,6 +283,11 @@ int sm_ll_lookup_bitmap(struct ll_disk *ll, dm_block_t b, uint32_t *result)
+ struct disk_index_entry ie_disk;
+ struct dm_block *blk;
+
++ if (b >= ll->nr_blocks) {
++ DMERR_LIMIT("metadata block out of bounds");
++ return -EINVAL;
++ }
++
+ b = do_div(index, ll->entries_per_block);
+ r = ll->load_ie(ll, index, &ie_disk);
+ if (r < 0)
+diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
+index 62c8b6adac70e..b59a77b31b90d 100644
+--- a/drivers/md/raid0.c
++++ b/drivers/md/raid0.c
+@@ -356,7 +356,21 @@ static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks
+ return array_sectors;
+ }
+
+-static void raid0_free(struct mddev *mddev, void *priv);
++static void free_conf(struct mddev *mddev, struct r0conf *conf)
++{
++ kfree(conf->strip_zone);
++ kfree(conf->devlist);
++ kfree(conf);
++ mddev->private = NULL;
++}
++
++static void raid0_free(struct mddev *mddev, void *priv)
++{
++ struct r0conf *conf = priv;
++
++ free_conf(mddev, conf);
++ acct_bioset_exit(mddev);
++}
+
+ static int raid0_run(struct mddev *mddev)
+ {
+@@ -370,11 +384,16 @@ static int raid0_run(struct mddev *mddev)
+ if (md_check_no_bitmap(mddev))
+ return -EINVAL;
+
++ if (acct_bioset_init(mddev)) {
++ pr_err("md/raid0:%s: alloc acct bioset failed.\n", mdname(mddev));
++ return -ENOMEM;
++ }
++
+ /* if private is not null, we are here after takeover */
+ if (mddev->private == NULL) {
+ ret = create_strip_zones(mddev, &conf);
+ if (ret < 0)
+- return ret;
++ goto exit_acct_set;
+ mddev->private = conf;
+ }
+ conf = mddev->private;
+@@ -413,17 +432,16 @@ static int raid0_run(struct mddev *mddev)
+ dump_zones(mddev);
+
+ ret = md_integrity_register(mddev);
++ if (ret)
++ goto free;
+
+ return ret;
+-}
+
+-static void raid0_free(struct mddev *mddev, void *priv)
+-{
+- struct r0conf *conf = priv;
+-
+- kfree(conf->strip_zone);
+- kfree(conf->devlist);
+- kfree(conf);
++free:
++ free_conf(mddev, conf);
++exit_acct_set:
++ acct_bioset_exit(mddev);
++ return ret;
+ }
+
+ static void raid0_handle_discard(struct mddev *mddev, struct bio *bio)
+diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
+index 19598bd38939d..9fa4794936426 100644
+--- a/drivers/md/raid1.c
++++ b/drivers/md/raid1.c
+@@ -1502,6 +1502,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
+ * allocate memory, or a reader on WriteMostly
+ * is waiting for behind writes to flush */
+ if (bitmap &&
++ test_bit(WriteMostly, &rdev->flags) &&
+ (atomic_read(&bitmap->behind_writes)
+ < mddev->bitmap_info.max_write_behind) &&
+ !waitqueue_active(&bitmap->behind_wait)) {
+diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
+index 02ed53b20654c..b9d062f0a02b2 100644
+--- a/drivers/md/raid5.c
++++ b/drivers/md/raid5.c
+@@ -7446,12 +7446,19 @@ static int raid5_run(struct mddev *mddev)
+ struct md_rdev *rdev;
+ struct md_rdev *journal_dev = NULL;
+ sector_t reshape_offset = 0;
+- int i;
++ int i, ret = 0;
+ long long min_offset_diff = 0;
+ int first = 1;
+
+- if (mddev_init_writes_pending(mddev) < 0)
++ if (acct_bioset_init(mddev)) {
++ pr_err("md/raid456:%s: alloc acct bioset failed.\n", mdname(mddev));
+ return -ENOMEM;
++ }
++
++ if (mddev_init_writes_pending(mddev) < 0) {
++ ret = -ENOMEM;
++ goto exit_acct_set;
++ }
+
+ if (mddev->recovery_cp != MaxSector)
+ pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
+@@ -7482,7 +7489,8 @@ static int raid5_run(struct mddev *mddev)
+ (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
+ pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+
+ if (mddev->reshape_position != MaxSector) {
+@@ -7507,13 +7515,15 @@ static int raid5_run(struct mddev *mddev)
+ if (journal_dev) {
+ pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+
+ if (mddev->new_level != mddev->level) {
+ pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+ old_disks = mddev->raid_disks - mddev->delta_disks;
+ /* reshape_position must be on a new-stripe boundary, and one
+@@ -7529,7 +7539,8 @@ static int raid5_run(struct mddev *mddev)
+ if (sector_div(here_new, chunk_sectors * new_data_disks)) {
+ pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+ reshape_offset = here_new * chunk_sectors;
+ /* here_new is the stripe we will write to */
+@@ -7551,7 +7562,8 @@ static int raid5_run(struct mddev *mddev)
+ else if (mddev->ro == 0) {
+ pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+ } else if (mddev->reshape_backwards
+ ? (here_new * chunk_sectors + min_offset_diff <=
+@@ -7561,7 +7573,8 @@ static int raid5_run(struct mddev *mddev)
+ /* Reading from the same stripe as writing to - bad */
+ pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
+ mdname(mddev));
+- return -EINVAL;
++ ret = -EINVAL;
++ goto exit_acct_set;
+ }
+ pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
+ /* OK, we should be able to continue; */
+@@ -7585,8 +7598,10 @@ static int raid5_run(struct mddev *mddev)
+ else
+ conf = mddev->private;
+
+- if (IS_ERR(conf))
+- return PTR_ERR(conf);
++ if (IS_ERR(conf)) {
++ ret = PTR_ERR(conf);
++ goto exit_acct_set;
++ }
+
+ if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
+ if (!journal_dev) {
+@@ -7786,7 +7801,10 @@ abort:
+ free_conf(conf);
+ mddev->private = NULL;
+ pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
+- return -EIO;
++ ret = -EIO;
++exit_acct_set:
++ acct_bioset_exit(mddev);
++ return ret;
+ }
+
+ static void raid5_free(struct mddev *mddev, void *priv)
+@@ -7794,6 +7812,7 @@ static void raid5_free(struct mddev *mddev, void *priv)
+ struct r5conf *conf = priv;
+
+ free_conf(conf);
++ acct_bioset_exit(mddev);
+ mddev->to_remove = &raid5_attrs_group;
+ }
+
+diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
+index b07812657cee6..f3f24c63536b6 100644
+--- a/drivers/media/Kconfig
++++ b/drivers/media/Kconfig
+@@ -141,10 +141,10 @@ config MEDIA_TEST_SUPPORT
+ prompt "Test drivers" if MEDIA_SUPPORT_FILTER
+ default y if !MEDIA_SUPPORT_FILTER
+ help
+- Those drivers should not be used on production Kernels, but
+- can be useful on debug ones. It enables several dummy drivers
+- that simulate a real hardware. Very useful to test userspace
+- applications and to validate if the subsystem core is doesn't
++ These drivers should not be used on production kernels, but
++ can be useful on debug ones. This option enables several dummy drivers
++ that simulate real hardware. Very useful to test userspace
++ applications and to validate if the subsystem core doesn't
+ have regressions.
+
+ Say Y if you want to use some virtual test driver.
+diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c
+index 79fa36de8a04a..1f599e300e42e 100644
+--- a/drivers/media/cec/core/cec-adap.c
++++ b/drivers/media/cec/core/cec-adap.c
+@@ -161,10 +161,10 @@ static void cec_queue_event(struct cec_adapter *adap,
+ u64 ts = ktime_get_ns();
+ struct cec_fh *fh;
+
+- mutex_lock(&adap->devnode.lock);
++ mutex_lock(&adap->devnode.lock_fhs);
+ list_for_each_entry(fh, &adap->devnode.fhs, list)
+ cec_queue_event_fh(fh, ev, ts);
+- mutex_unlock(&adap->devnode.lock);
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+
+ /* Notify userspace that the CEC pin changed state at the given time. */
+@@ -178,11 +178,12 @@ void cec_queue_pin_cec_event(struct cec_adapter *adap, bool is_high,
+ };
+ struct cec_fh *fh;
+
+- mutex_lock(&adap->devnode.lock);
+- list_for_each_entry(fh, &adap->devnode.fhs, list)
++ mutex_lock(&adap->devnode.lock_fhs);
++ list_for_each_entry(fh, &adap->devnode.fhs, list) {
+ if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
+ cec_queue_event_fh(fh, &ev, ktime_to_ns(ts));
+- mutex_unlock(&adap->devnode.lock);
++ }
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+ EXPORT_SYMBOL_GPL(cec_queue_pin_cec_event);
+
+@@ -195,10 +196,10 @@ void cec_queue_pin_hpd_event(struct cec_adapter *adap, bool is_high, ktime_t ts)
+ };
+ struct cec_fh *fh;
+
+- mutex_lock(&adap->devnode.lock);
++ mutex_lock(&adap->devnode.lock_fhs);
+ list_for_each_entry(fh, &adap->devnode.fhs, list)
+ cec_queue_event_fh(fh, &ev, ktime_to_ns(ts));
+- mutex_unlock(&adap->devnode.lock);
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+ EXPORT_SYMBOL_GPL(cec_queue_pin_hpd_event);
+
+@@ -211,10 +212,10 @@ void cec_queue_pin_5v_event(struct cec_adapter *adap, bool is_high, ktime_t ts)
+ };
+ struct cec_fh *fh;
+
+- mutex_lock(&adap->devnode.lock);
++ mutex_lock(&adap->devnode.lock_fhs);
+ list_for_each_entry(fh, &adap->devnode.fhs, list)
+ cec_queue_event_fh(fh, &ev, ktime_to_ns(ts));
+- mutex_unlock(&adap->devnode.lock);
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+ EXPORT_SYMBOL_GPL(cec_queue_pin_5v_event);
+
+@@ -286,12 +287,12 @@ static void cec_queue_msg_monitor(struct cec_adapter *adap,
+ u32 monitor_mode = valid_la ? CEC_MODE_MONITOR :
+ CEC_MODE_MONITOR_ALL;
+
+- mutex_lock(&adap->devnode.lock);
++ mutex_lock(&adap->devnode.lock_fhs);
+ list_for_each_entry(fh, &adap->devnode.fhs, list) {
+ if (fh->mode_follower >= monitor_mode)
+ cec_queue_msg_fh(fh, msg);
+ }
+- mutex_unlock(&adap->devnode.lock);
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+
+ /*
+@@ -302,12 +303,12 @@ static void cec_queue_msg_followers(struct cec_adapter *adap,
+ {
+ struct cec_fh *fh;
+
+- mutex_lock(&adap->devnode.lock);
++ mutex_lock(&adap->devnode.lock_fhs);
+ list_for_each_entry(fh, &adap->devnode.fhs, list) {
+ if (fh->mode_follower == CEC_MODE_FOLLOWER)
+ cec_queue_msg_fh(fh, msg);
+ }
+- mutex_unlock(&adap->devnode.lock);
++ mutex_unlock(&adap->devnode.lock_fhs);
+ }
+
+ /* Notify userspace of an adapter state change. */
+@@ -1199,6 +1200,7 @@ void cec_received_msg_ts(struct cec_adapter *adap,
+ if (abort)
+ dst->rx_status |= CEC_RX_STATUS_FEATURE_ABORT;
+ msg->flags = dst->flags;
++ msg->sequence = dst->sequence;
+ /* Remove it from the wait_queue */
+ list_del_init(&data->list);
+
+@@ -1572,6 +1574,7 @@ void __cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block)
+ /* Disabling monitor all mode should always succeed */
+ if (adap->monitor_all_cnt)
+ WARN_ON(call_op(adap, adap_monitor_all_enable, false));
++ /* serialize adap_enable */
+ mutex_lock(&adap->devnode.lock);
+ if (adap->needs_hpd || list_empty(&adap->devnode.fhs)) {
+ WARN_ON(adap->ops->adap_enable(adap, false));
+@@ -1583,14 +1586,16 @@ void __cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block)
+ return;
+ }
+
++ /* serialize adap_enable */
+ mutex_lock(&adap->devnode.lock);
+ adap->last_initiator = 0xff;
+ adap->transmit_in_progress = false;
+
+- if ((adap->needs_hpd || list_empty(&adap->devnode.fhs)) &&
+- adap->ops->adap_enable(adap, true)) {
+- mutex_unlock(&adap->devnode.lock);
+- return;
++ if (adap->needs_hpd || list_empty(&adap->devnode.fhs)) {
++ if (adap->ops->adap_enable(adap, true)) {
++ mutex_unlock(&adap->devnode.lock);
++ return;
++ }
+ }
+
+ if (adap->monitor_all_cnt &&
+diff --git a/drivers/media/cec/core/cec-api.c b/drivers/media/cec/core/cec-api.c
+index 769e6b4cddce3..52c30e4e20055 100644
+--- a/drivers/media/cec/core/cec-api.c
++++ b/drivers/media/cec/core/cec-api.c
+@@ -586,6 +586,7 @@ static int cec_open(struct inode *inode, struct file *filp)
+ return err;
+ }
+
++ /* serialize adap_enable */
+ mutex_lock(&devnode->lock);
+ if (list_empty(&devnode->fhs) &&
+ !adap->needs_hpd &&
+@@ -624,7 +625,9 @@ static int cec_open(struct inode *inode, struct file *filp)
+ }
+ #endif
+
++ mutex_lock(&devnode->lock_fhs);
+ list_add(&fh->list, &devnode->fhs);
++ mutex_unlock(&devnode->lock_fhs);
+ mutex_unlock(&devnode->lock);
+
+ return 0;
+@@ -653,8 +656,11 @@ static int cec_release(struct inode *inode, struct file *filp)
+ cec_monitor_all_cnt_dec(adap);
+ mutex_unlock(&adap->lock);
+
++ /* serialize adap_enable */
+ mutex_lock(&devnode->lock);
++ mutex_lock(&devnode->lock_fhs);
+ list_del(&fh->list);
++ mutex_unlock(&devnode->lock_fhs);
+ if (cec_is_registered(adap) && list_empty(&devnode->fhs) &&
+ !adap->needs_hpd && adap->phys_addr == CEC_PHYS_ADDR_INVALID) {
+ WARN_ON(adap->ops->adap_enable(adap, false));
+diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c
+index 551689d371a71..ec67065d52021 100644
+--- a/drivers/media/cec/core/cec-core.c
++++ b/drivers/media/cec/core/cec-core.c
+@@ -169,8 +169,10 @@ static void cec_devnode_unregister(struct cec_adapter *adap)
+ devnode->registered = false;
+ devnode->unregistered = true;
+
++ mutex_lock(&devnode->lock_fhs);
+ list_for_each_entry(fh, &devnode->fhs, list)
+ wake_up_interruptible(&fh->wait);
++ mutex_unlock(&devnode->lock_fhs);
+
+ mutex_unlock(&devnode->lock);
+
+@@ -272,6 +274,7 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
+
+ /* adap->devnode initialization */
+ INIT_LIST_HEAD(&adap->devnode.fhs);
++ mutex_init(&adap->devnode.lock_fhs);
+ mutex_init(&adap->devnode.lock);
+
+ adap->kthread = kthread_run(cec_thread_func, adap, "cec-%s", name);
+diff --git a/drivers/media/cec/core/cec-pin.c b/drivers/media/cec/core/cec-pin.c
+index 8c613aa649c6f..0eb90cc0ffb0f 100644
+--- a/drivers/media/cec/core/cec-pin.c
++++ b/drivers/media/cec/core/cec-pin.c
+@@ -1033,6 +1033,7 @@ static int cec_pin_thread_func(void *_adap)
+ {
+ struct cec_adapter *adap = _adap;
+ struct cec_pin *pin = adap->pin;
++ bool irq_enabled = false;
+
+ for (;;) {
+ wait_event_interruptible(pin->kthread_waitq,
+@@ -1060,6 +1061,7 @@ static int cec_pin_thread_func(void *_adap)
+ ns_to_ktime(pin->work_rx_msg.rx_ts));
+ msg->len = 0;
+ }
++
+ if (pin->work_tx_status) {
+ unsigned int tx_status = pin->work_tx_status;
+
+@@ -1083,27 +1085,39 @@ static int cec_pin_thread_func(void *_adap)
+ switch (atomic_xchg(&pin->work_irq_change,
+ CEC_PIN_IRQ_UNCHANGED)) {
+ case CEC_PIN_IRQ_DISABLE:
+- pin->ops->disable_irq(adap);
++ if (irq_enabled) {
++ pin->ops->disable_irq(adap);
++ irq_enabled = false;
++ }
+ cec_pin_high(pin);
+ cec_pin_to_idle(pin);
+ hrtimer_start(&pin->timer, ns_to_ktime(0),
+ HRTIMER_MODE_REL);
+ break;
+ case CEC_PIN_IRQ_ENABLE:
++ if (irq_enabled)
++ break;
+ pin->enable_irq_failed = !pin->ops->enable_irq(adap);
+ if (pin->enable_irq_failed) {
+ cec_pin_to_idle(pin);
+ hrtimer_start(&pin->timer, ns_to_ktime(0),
+ HRTIMER_MODE_REL);
++ } else {
++ irq_enabled = true;
+ }
+ break;
+ default:
+ break;
+ }
+-
+ if (kthread_should_stop())
+ break;
+ }
++ if (pin->ops->disable_irq && irq_enabled)
++ pin->ops->disable_irq(adap);
++ hrtimer_cancel(&pin->timer);
++ cec_pin_read(pin);
++ cec_pin_to_idle(pin);
++ pin->state = CEC_ST_OFF;
+ return 0;
+ }
+
+@@ -1130,13 +1144,7 @@ static int cec_pin_adap_enable(struct cec_adapter *adap, bool enable)
+ hrtimer_start(&pin->timer, ns_to_ktime(0),
+ HRTIMER_MODE_REL);
+ } else {
+- if (pin->ops->disable_irq)
+- pin->ops->disable_irq(adap);
+- hrtimer_cancel(&pin->timer);
+ kthread_stop(pin->kthread);
+- cec_pin_read(pin);
+- cec_pin_to_idle(pin);
+- pin->state = CEC_ST_OFF;
+ }
+ return 0;
+ }
+@@ -1157,11 +1165,8 @@ void cec_pin_start_timer(struct cec_pin *pin)
+ if (pin->state != CEC_ST_RX_IRQ)
+ return;
+
+- atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_UNCHANGED);
+- pin->ops->disable_irq(pin->adap);
+- cec_pin_high(pin);
+- cec_pin_to_idle(pin);
+- hrtimer_start(&pin->timer, ns_to_ktime(0), HRTIMER_MODE_REL);
++ atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_DISABLE);
++ wake_up_interruptible(&pin->kthread_waitq);
+ }
+
+ static int cec_pin_adap_transmit(struct cec_adapter *adap, u8 attempts,
+diff --git a/drivers/media/common/saa7146/saa7146_fops.c b/drivers/media/common/saa7146/saa7146_fops.c
+index baf5772c52a96..be32159777142 100644
+--- a/drivers/media/common/saa7146/saa7146_fops.c
++++ b/drivers/media/common/saa7146/saa7146_fops.c
+@@ -521,7 +521,7 @@ int saa7146_vv_init(struct saa7146_dev* dev, struct saa7146_ext_vv *ext_vv)
+ ERR("out of memory. aborting.\n");
+ kfree(vv);
+ v4l2_ctrl_handler_free(hdl);
+- return -1;
++ return -ENOMEM;
+ }
+
+ saa7146_video_uops.init(dev,vv);
+diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
+index 508ac295eb06e..033b0c83272fe 100644
+--- a/drivers/media/common/videobuf2/videobuf2-core.c
++++ b/drivers/media/common/videobuf2/videobuf2-core.c
+@@ -68,13 +68,13 @@ module_param(debug, int, 0644);
+ err; \
+ })
+
+-#define call_ptr_memop(vb, op, args...) \
++#define call_ptr_memop(op, vb, args...) \
+ ({ \
+ struct vb2_queue *_q = (vb)->vb2_queue; \
+ void *ptr; \
+ \
+ log_memop(vb, op); \
+- ptr = _q->mem_ops->op ? _q->mem_ops->op(args) : NULL; \
++ ptr = _q->mem_ops->op ? _q->mem_ops->op(vb, args) : NULL; \
+ if (!IS_ERR_OR_NULL(ptr)) \
+ (vb)->cnt_mem_ ## op++; \
+ ptr; \
+@@ -144,9 +144,9 @@ module_param(debug, int, 0644);
+ ((vb)->vb2_queue->mem_ops->op ? \
+ (vb)->vb2_queue->mem_ops->op(args) : 0)
+
+-#define call_ptr_memop(vb, op, args...) \
++#define call_ptr_memop(op, vb, args...) \
+ ((vb)->vb2_queue->mem_ops->op ? \
+- (vb)->vb2_queue->mem_ops->op(args) : NULL)
++ (vb)->vb2_queue->mem_ops->op(vb, args) : NULL)
+
+ #define call_void_memop(vb, op, args...) \
+ do { \
+@@ -230,9 +230,10 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
+ if (size < vb->planes[plane].length)
+ goto free;
+
+- mem_priv = call_ptr_memop(vb, alloc,
+- q->alloc_devs[plane] ? : q->dev,
+- q->dma_attrs, size, q->dma_dir, q->gfp_flags);
++ mem_priv = call_ptr_memop(alloc,
++ vb,
++ q->alloc_devs[plane] ? : q->dev,
++ size);
+ if (IS_ERR_OR_NULL(mem_priv)) {
+ if (mem_priv)
+ ret = PTR_ERR(mem_priv);
+@@ -975,7 +976,7 @@ void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
+ if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
+ return NULL;
+
+- return call_ptr_memop(vb, vaddr, vb->planes[plane_no].mem_priv);
++ return call_ptr_memop(vaddr, vb, vb->planes[plane_no].mem_priv);
+
+ }
+ EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
+@@ -985,7 +986,7 @@ void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
+ if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
+ return NULL;
+
+- return call_ptr_memop(vb, cookie, vb->planes[plane_no].mem_priv);
++ return call_ptr_memop(cookie, vb, vb->planes[plane_no].mem_priv);
+ }
+ EXPORT_SYMBOL_GPL(vb2_plane_cookie);
+
+@@ -1125,10 +1126,11 @@ static int __prepare_userptr(struct vb2_buffer *vb)
+ vb->planes[plane].data_offset = 0;
+
+ /* Acquire each plane's memory */
+- mem_priv = call_ptr_memop(vb, get_userptr,
+- q->alloc_devs[plane] ? : q->dev,
+- planes[plane].m.userptr,
+- planes[plane].length, q->dma_dir);
++ mem_priv = call_ptr_memop(get_userptr,
++ vb,
++ q->alloc_devs[plane] ? : q->dev,
++ planes[plane].m.userptr,
++ planes[plane].length);
+ if (IS_ERR(mem_priv)) {
+ dprintk(q, 1, "failed acquiring userspace memory for plane %d\n",
+ plane);
+@@ -1249,9 +1251,11 @@ static int __prepare_dmabuf(struct vb2_buffer *vb)
+ vb->planes[plane].data_offset = 0;
+
+ /* Acquire each plane's memory */
+- mem_priv = call_ptr_memop(vb, attach_dmabuf,
+- q->alloc_devs[plane] ? : q->dev,
+- dbuf, planes[plane].length, q->dma_dir);
++ mem_priv = call_ptr_memop(attach_dmabuf,
++ vb,
++ q->alloc_devs[plane] ? : q->dev,
++ dbuf,
++ planes[plane].length);
+ if (IS_ERR(mem_priv)) {
+ dprintk(q, 1, "failed to attach dmabuf\n");
+ ret = PTR_ERR(mem_priv);
+@@ -2187,8 +2191,10 @@ int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
+
+ vb_plane = &vb->planes[plane];
+
+- dbuf = call_ptr_memop(vb, get_dmabuf, vb_plane->mem_priv,
+- flags & O_ACCMODE);
++ dbuf = call_ptr_memop(get_dmabuf,
++ vb,
++ vb_plane->mem_priv,
++ flags & O_ACCMODE);
+ if (IS_ERR_OR_NULL(dbuf)) {
+ dprintk(q, 1, "failed to export buffer %d, plane %d\n",
+ index, plane);
+diff --git a/drivers/media/common/videobuf2/videobuf2-dma-contig.c b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
+index a7f61ba854405..f8c65b0401054 100644
+--- a/drivers/media/common/videobuf2/videobuf2-dma-contig.c
++++ b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
+@@ -40,6 +40,8 @@ struct vb2_dc_buf {
+
+ /* DMABUF related */
+ struct dma_buf_attachment *db_attach;
++
++ struct vb2_buffer *vb;
+ };
+
+ /*********************************************/
+@@ -66,14 +68,14 @@ static unsigned long vb2_dc_get_contiguous_size(struct sg_table *sgt)
+ /* callbacks for all buffers */
+ /*********************************************/
+
+-static void *vb2_dc_cookie(void *buf_priv)
++static void *vb2_dc_cookie(struct vb2_buffer *vb, void *buf_priv)
+ {
+ struct vb2_dc_buf *buf = buf_priv;
+
+ return &buf->dma_addr;
+ }
+
+-static void *vb2_dc_vaddr(void *buf_priv)
++static void *vb2_dc_vaddr(struct vb2_buffer *vb, void *buf_priv)
+ {
+ struct vb2_dc_buf *buf = buf_priv;
+ struct dma_buf_map map;
+@@ -137,9 +139,9 @@ static void vb2_dc_put(void *buf_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_dc_alloc(struct device *dev, unsigned long attrs,
+- unsigned long size, enum dma_data_direction dma_dir,
+- gfp_t gfp_flags)
++static void *vb2_dc_alloc(struct vb2_buffer *vb,
++ struct device *dev,
++ unsigned long size)
+ {
+ struct vb2_dc_buf *buf;
+
+@@ -150,11 +152,12 @@ static void *vb2_dc_alloc(struct device *dev, unsigned long attrs,
+ if (!buf)
+ return ERR_PTR(-ENOMEM);
+
+- buf->attrs = attrs;
++ buf->attrs = vb->vb2_queue->dma_attrs;
+ buf->cookie = dma_alloc_attrs(dev, size, &buf->dma_addr,
+- GFP_KERNEL | gfp_flags, buf->attrs);
++ GFP_KERNEL | vb->vb2_queue->gfp_flags,
++ buf->attrs);
+ if (!buf->cookie) {
+- dev_err(dev, "dma_alloc_coherent of size %ld failed\n", size);
++ dev_err(dev, "dma_alloc_coherent of size %lu failed\n", size);
+ kfree(buf);
+ return ERR_PTR(-ENOMEM);
+ }
+@@ -165,11 +168,12 @@ static void *vb2_dc_alloc(struct device *dev, unsigned long attrs,
+ /* Prevent the device from being released while the buffer is used */
+ buf->dev = get_device(dev);
+ buf->size = size;
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+
+ buf->handler.refcount = &buf->refcount;
+ buf->handler.put = vb2_dc_put;
+ buf->handler.arg = buf;
++ buf->vb = vb;
+
+ refcount_set(&buf->refcount, 1);
+
+@@ -200,9 +204,9 @@ static int vb2_dc_mmap(void *buf_priv, struct vm_area_struct *vma)
+
+ vma->vm_ops->open(vma);
+
+- pr_debug("%s: mapped dma addr 0x%08lx at 0x%08lx, size %ld\n",
+- __func__, (unsigned long)buf->dma_addr, vma->vm_start,
+- buf->size);
++ pr_debug("%s: mapped dma addr 0x%08lx at 0x%08lx, size %lu\n",
++ __func__, (unsigned long)buf->dma_addr, vma->vm_start,
++ buf->size);
+
+ return 0;
+ }
+@@ -397,7 +401,9 @@ static struct sg_table *vb2_dc_get_base_sgt(struct vb2_dc_buf *buf)
+ return sgt;
+ }
+
+-static struct dma_buf *vb2_dc_get_dmabuf(void *buf_priv, unsigned long flags)
++static struct dma_buf *vb2_dc_get_dmabuf(struct vb2_buffer *vb,
++ void *buf_priv,
++ unsigned long flags)
+ {
+ struct vb2_dc_buf *buf = buf_priv;
+ struct dma_buf *dbuf;
+@@ -459,8 +465,8 @@ static void vb2_dc_put_userptr(void *buf_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr,
+- unsigned long size, enum dma_data_direction dma_dir)
++static void *vb2_dc_get_userptr(struct vb2_buffer *vb, struct device *dev,
++ unsigned long vaddr, unsigned long size)
+ {
+ struct vb2_dc_buf *buf;
+ struct frame_vector *vec;
+@@ -490,7 +496,8 @@ static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr,
+ return ERR_PTR(-ENOMEM);
+
+ buf->dev = dev;
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
++ buf->vb = vb;
+
+ offset = lower_32_bits(offset_in_page(vaddr));
+ vec = vb2_create_framevec(vaddr, size);
+@@ -660,8 +667,8 @@ static void vb2_dc_detach_dmabuf(void *mem_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_dc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+- unsigned long size, enum dma_data_direction dma_dir)
++static void *vb2_dc_attach_dmabuf(struct vb2_buffer *vb, struct device *dev,
++ struct dma_buf *dbuf, unsigned long size)
+ {
+ struct vb2_dc_buf *buf;
+ struct dma_buf_attachment *dba;
+@@ -677,6 +684,8 @@ static void *vb2_dc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+ return ERR_PTR(-ENOMEM);
+
+ buf->dev = dev;
++ buf->vb = vb;
++
+ /* create attachment for the dmabuf with the user device */
+ dba = dma_buf_attach(dbuf, buf->dev);
+ if (IS_ERR(dba)) {
+@@ -685,7 +694,7 @@ static void *vb2_dc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+ return dba;
+ }
+
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->size = size;
+ buf->db_attach = dba;
+
+diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
+index c5b06a5095661..0d6389dd9b0c6 100644
+--- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c
++++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
+@@ -51,6 +51,8 @@ struct vb2_dma_sg_buf {
+ struct vb2_vmarea_handler handler;
+
+ struct dma_buf_attachment *db_attach;
++
++ struct vb2_buffer *vb;
+ };
+
+ static void vb2_dma_sg_put(void *buf_priv);
+@@ -96,9 +98,8 @@ static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,
+ return 0;
+ }
+
+-static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs,
+- unsigned long size, enum dma_data_direction dma_dir,
+- gfp_t gfp_flags)
++static void *vb2_dma_sg_alloc(struct vb2_buffer *vb, struct device *dev,
++ unsigned long size)
+ {
+ struct vb2_dma_sg_buf *buf;
+ struct sg_table *sgt;
+@@ -113,7 +114,7 @@ static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs,
+ return ERR_PTR(-ENOMEM);
+
+ buf->vaddr = NULL;
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->offset = 0;
+ buf->size = size;
+ /* size is already page aligned */
+@@ -130,7 +131,7 @@ static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs,
+ if (!buf->pages)
+ goto fail_pages_array_alloc;
+
+- ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);
++ ret = vb2_dma_sg_alloc_compacted(buf, vb->vb2_queue->gfp_flags);
+ if (ret)
+ goto fail_pages_alloc;
+
+@@ -154,6 +155,7 @@ static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs,
+ buf->handler.refcount = &buf->refcount;
+ buf->handler.put = vb2_dma_sg_put;
+ buf->handler.arg = buf;
++ buf->vb = vb;
+
+ refcount_set(&buf->refcount, 1);
+
+@@ -213,9 +215,8 @@ static void vb2_dma_sg_finish(void *buf_priv)
+ dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
+ }
+
+-static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr,
+- unsigned long size,
+- enum dma_data_direction dma_dir)
++static void *vb2_dma_sg_get_userptr(struct vb2_buffer *vb, struct device *dev,
++ unsigned long vaddr, unsigned long size)
+ {
+ struct vb2_dma_sg_buf *buf;
+ struct sg_table *sgt;
+@@ -230,10 +231,11 @@ static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr,
+
+ buf->vaddr = NULL;
+ buf->dev = dev;
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->offset = vaddr & ~PAGE_MASK;
+ buf->size = size;
+ buf->dma_sgt = &buf->sg_table;
++ buf->vb = vb;
+ vec = vb2_create_framevec(vaddr, size);
+ if (IS_ERR(vec))
+ goto userptr_fail_pfnvec;
+@@ -292,7 +294,7 @@ static void vb2_dma_sg_put_userptr(void *buf_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_dma_sg_vaddr(void *buf_priv)
++static void *vb2_dma_sg_vaddr(struct vb2_buffer *vb, void *buf_priv)
+ {
+ struct vb2_dma_sg_buf *buf = buf_priv;
+ struct dma_buf_map map;
+@@ -511,7 +513,9 @@ static const struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
+ .release = vb2_dma_sg_dmabuf_ops_release,
+ };
+
+-static struct dma_buf *vb2_dma_sg_get_dmabuf(void *buf_priv, unsigned long flags)
++static struct dma_buf *vb2_dma_sg_get_dmabuf(struct vb2_buffer *vb,
++ void *buf_priv,
++ unsigned long flags)
+ {
+ struct vb2_dma_sg_buf *buf = buf_priv;
+ struct dma_buf *dbuf;
+@@ -605,8 +609,8 @@ static void vb2_dma_sg_detach_dmabuf(void *mem_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_dma_sg_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+- unsigned long size, enum dma_data_direction dma_dir)
++static void *vb2_dma_sg_attach_dmabuf(struct vb2_buffer *vb, struct device *dev,
++ struct dma_buf *dbuf, unsigned long size)
+ {
+ struct vb2_dma_sg_buf *buf;
+ struct dma_buf_attachment *dba;
+@@ -630,14 +634,15 @@ static void *vb2_dma_sg_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+ return dba;
+ }
+
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->size = size;
+ buf->db_attach = dba;
++ buf->vb = vb;
+
+ return buf;
+ }
+
+-static void *vb2_dma_sg_cookie(void *buf_priv)
++static void *vb2_dma_sg_cookie(struct vb2_buffer *vb, void *buf_priv)
+ {
+ struct vb2_dma_sg_buf *buf = buf_priv;
+
+diff --git a/drivers/media/common/videobuf2/videobuf2-vmalloc.c b/drivers/media/common/videobuf2/videobuf2-vmalloc.c
+index 83f95258ec8c6..ef36abd912dcc 100644
+--- a/drivers/media/common/videobuf2/videobuf2-vmalloc.c
++++ b/drivers/media/common/videobuf2/videobuf2-vmalloc.c
+@@ -34,13 +34,12 @@ struct vb2_vmalloc_buf {
+
+ static void vb2_vmalloc_put(void *buf_priv);
+
+-static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs,
+- unsigned long size, enum dma_data_direction dma_dir,
+- gfp_t gfp_flags)
++static void *vb2_vmalloc_alloc(struct vb2_buffer *vb, struct device *dev,
++ unsigned long size)
+ {
+ struct vb2_vmalloc_buf *buf;
+
+- buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags);
++ buf = kzalloc(sizeof(*buf), GFP_KERNEL | vb->vb2_queue->gfp_flags);
+ if (!buf)
+ return ERR_PTR(-ENOMEM);
+
+@@ -52,7 +51,7 @@ static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs,
+ return ERR_PTR(-ENOMEM);
+ }
+
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->handler.refcount = &buf->refcount;
+ buf->handler.put = vb2_vmalloc_put;
+ buf->handler.arg = buf;
+@@ -71,9 +70,8 @@ static void vb2_vmalloc_put(void *buf_priv)
+ }
+ }
+
+-static void *vb2_vmalloc_get_userptr(struct device *dev, unsigned long vaddr,
+- unsigned long size,
+- enum dma_data_direction dma_dir)
++static void *vb2_vmalloc_get_userptr(struct vb2_buffer *vb, struct device *dev,
++ unsigned long vaddr, unsigned long size)
+ {
+ struct vb2_vmalloc_buf *buf;
+ struct frame_vector *vec;
+@@ -84,7 +82,7 @@ static void *vb2_vmalloc_get_userptr(struct device *dev, unsigned long vaddr,
+ if (!buf)
+ return ERR_PTR(-ENOMEM);
+
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ offset = vaddr & ~PAGE_MASK;
+ buf->size = size;
+ vec = vb2_create_framevec(vaddr, size);
+@@ -147,7 +145,7 @@ static void vb2_vmalloc_put_userptr(void *buf_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_vmalloc_vaddr(void *buf_priv)
++static void *vb2_vmalloc_vaddr(struct vb2_buffer *vb, void *buf_priv)
+ {
+ struct vb2_vmalloc_buf *buf = buf_priv;
+
+@@ -339,7 +337,9 @@ static const struct dma_buf_ops vb2_vmalloc_dmabuf_ops = {
+ .release = vb2_vmalloc_dmabuf_ops_release,
+ };
+
+-static struct dma_buf *vb2_vmalloc_get_dmabuf(void *buf_priv, unsigned long flags)
++static struct dma_buf *vb2_vmalloc_get_dmabuf(struct vb2_buffer *vb,
++ void *buf_priv,
++ unsigned long flags)
+ {
+ struct vb2_vmalloc_buf *buf = buf_priv;
+ struct dma_buf *dbuf;
+@@ -403,8 +403,10 @@ static void vb2_vmalloc_detach_dmabuf(void *mem_priv)
+ kfree(buf);
+ }
+
+-static void *vb2_vmalloc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+- unsigned long size, enum dma_data_direction dma_dir)
++static void *vb2_vmalloc_attach_dmabuf(struct vb2_buffer *vb,
++ struct device *dev,
++ struct dma_buf *dbuf,
++ unsigned long size)
+ {
+ struct vb2_vmalloc_buf *buf;
+
+@@ -416,7 +418,7 @@ static void *vb2_vmalloc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
+ return ERR_PTR(-ENOMEM);
+
+ buf->dbuf = dbuf;
+- buf->dma_dir = dma_dir;
++ buf->dma_dir = vb->vb2_queue->dma_dir;
+ buf->size = size;
+
+ return buf;
+diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c
+index 5d5a48475a54f..01f288fa37e0e 100644
+--- a/drivers/media/dvb-core/dmxdev.c
++++ b/drivers/media/dvb-core/dmxdev.c
+@@ -1413,7 +1413,7 @@ static const struct dvb_device dvbdev_dvr = {
+ };
+ int dvb_dmxdev_init(struct dmxdev *dmxdev, struct dvb_adapter *dvb_adapter)
+ {
+- int i;
++ int i, ret;
+
+ if (dmxdev->demux->open(dmxdev->demux) < 0)
+ return -EUSERS;
+@@ -1432,14 +1432,26 @@ int dvb_dmxdev_init(struct dmxdev *dmxdev, struct dvb_adapter *dvb_adapter)
+ DMXDEV_STATE_FREE);
+ }
+
+- dvb_register_device(dvb_adapter, &dmxdev->dvbdev, &dvbdev_demux, dmxdev,
++ ret = dvb_register_device(dvb_adapter, &dmxdev->dvbdev, &dvbdev_demux, dmxdev,
+ DVB_DEVICE_DEMUX, dmxdev->filternum);
+- dvb_register_device(dvb_adapter, &dmxdev->dvr_dvbdev, &dvbdev_dvr,
++ if (ret < 0)
++ goto err_register_dvbdev;
++
++ ret = dvb_register_device(dvb_adapter, &dmxdev->dvr_dvbdev, &dvbdev_dvr,
+ dmxdev, DVB_DEVICE_DVR, dmxdev->filternum);
++ if (ret < 0)
++ goto err_register_dvr_dvbdev;
+
+ dvb_ringbuffer_init(&dmxdev->dvr_buffer, NULL, 8192);
+
+ return 0;
++
++err_register_dvr_dvbdev:
++ dvb_unregister_device(dmxdev->dvbdev);
++err_register_dvbdev:
++ vfree(dmxdev->filter);
++ dmxdev->filter = NULL;
++ return ret;
+ }
+
+ EXPORT_SYMBOL(dvb_dmxdev_init);
+diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c
+index bb02354a48b81..d67f2dd997d06 100644
+--- a/drivers/media/dvb-frontends/dib8000.c
++++ b/drivers/media/dvb-frontends/dib8000.c
+@@ -4473,8 +4473,10 @@ static struct dvb_frontend *dib8000_init(struct i2c_adapter *i2c_adap, u8 i2c_ad
+
+ state->timf_default = cfg->pll->timf;
+
+- if (dib8000_identify(&state->i2c) == 0)
++ if (dib8000_identify(&state->i2c) == 0) {
++ kfree(fe);
+ goto error;
++ }
+
+ dibx000_init_i2c_master(&state->i2c_master, DIB8000, state->i2c.adap, state->i2c.addr);
+
+diff --git a/drivers/media/dvb-frontends/mn88443x.c b/drivers/media/dvb-frontends/mn88443x.c
+index e4528784f8477..fff212c0bf3b5 100644
+--- a/drivers/media/dvb-frontends/mn88443x.c
++++ b/drivers/media/dvb-frontends/mn88443x.c
+@@ -204,11 +204,18 @@ struct mn88443x_priv {
+ struct regmap *regmap_t;
+ };
+
+-static void mn88443x_cmn_power_on(struct mn88443x_priv *chip)
++static int mn88443x_cmn_power_on(struct mn88443x_priv *chip)
+ {
++ struct device *dev = &chip->client_s->dev;
+ struct regmap *r_t = chip->regmap_t;
++ int ret;
+
+- clk_prepare_enable(chip->mclk);
++ ret = clk_prepare_enable(chip->mclk);
++ if (ret) {
++ dev_err(dev, "Failed to prepare and enable mclk: %d\n",
++ ret);
++ return ret;
++ }
+
+ gpiod_set_value_cansleep(chip->reset_gpio, 1);
+ usleep_range(100, 1000);
+@@ -222,6 +229,8 @@ static void mn88443x_cmn_power_on(struct mn88443x_priv *chip)
+ } else {
+ regmap_write(r_t, HIZSET3, 0x8f);
+ }
++
++ return 0;
+ }
+
+ static void mn88443x_cmn_power_off(struct mn88443x_priv *chip)
+@@ -738,7 +747,10 @@ static int mn88443x_probe(struct i2c_client *client,
+ chip->fe.demodulator_priv = chip;
+ i2c_set_clientdata(client, chip);
+
+- mn88443x_cmn_power_on(chip);
++ ret = mn88443x_cmn_power_on(chip);
++ if (ret)
++ goto err_i2c_t;
++
+ mn88443x_s_sleep(chip);
+ mn88443x_t_sleep(chip);
+
+diff --git a/drivers/media/firewire/firedtv-avc.c b/drivers/media/firewire/firedtv-avc.c
+index 2bf9467b917d1..71991f8638e6b 100644
+--- a/drivers/media/firewire/firedtv-avc.c
++++ b/drivers/media/firewire/firedtv-avc.c
+@@ -1165,7 +1165,11 @@ int avc_ca_pmt(struct firedtv *fdtv, char *msg, int length)
+ read_pos += program_info_length;
+ write_pos += program_info_length;
+ }
+- while (read_pos < length) {
++ while (read_pos + 4 < length) {
++ if (write_pos + 4 >= sizeof(c->operand) - 4) {
++ ret = -EINVAL;
++ goto out;
++ }
+ c->operand[write_pos++] = msg[read_pos++];
+ c->operand[write_pos++] = msg[read_pos++];
+ c->operand[write_pos++] = msg[read_pos++];
+@@ -1177,13 +1181,17 @@ int avc_ca_pmt(struct firedtv *fdtv, char *msg, int length)
+ c->operand[write_pos++] = es_info_length >> 8;
+ c->operand[write_pos++] = es_info_length & 0xff;
+ if (es_info_length > 0) {
++ if (read_pos >= length) {
++ ret = -EINVAL;
++ goto out;
++ }
+ pmt_cmd_id = msg[read_pos++];
+ if (pmt_cmd_id != 1 && pmt_cmd_id != 4)
+ dev_err(fdtv->device, "invalid pmt_cmd_id %d at stream level\n",
+ pmt_cmd_id);
+
+- if (es_info_length > sizeof(c->operand) - 4 -
+- write_pos) {
++ if (es_info_length > sizeof(c->operand) - 4 - write_pos ||
++ es_info_length > length - read_pos) {
+ ret = -EINVAL;
+ goto out;
+ }
+diff --git a/drivers/media/firewire/firedtv-ci.c b/drivers/media/firewire/firedtv-ci.c
+index 9363d005e2b61..e0d57e09dab0c 100644
+--- a/drivers/media/firewire/firedtv-ci.c
++++ b/drivers/media/firewire/firedtv-ci.c
+@@ -134,6 +134,8 @@ static int fdtv_ca_pmt(struct firedtv *fdtv, void *arg)
+ } else {
+ data_length = msg->msg[3];
+ }
++ if (data_length > sizeof(msg->msg) - data_pos)
++ return -EINVAL;
+
+ return avc_ca_pmt(fdtv, &msg->msg[data_pos], data_length);
+ }
+diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
+index 08feb3e8c1bf6..6157e73eef24e 100644
+--- a/drivers/media/i2c/Kconfig
++++ b/drivers/media/i2c/Kconfig
+@@ -597,6 +597,7 @@ config VIDEO_AK881X
+ config VIDEO_THS8200
+ tristate "Texas Instruments THS8200 video encoder"
+ depends on VIDEO_V4L2 && I2C
++ select V4L2_ASYNC
+ help
+ Support for the Texas Instruments THS8200 video encoder.
+
+diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c
+index 41f4e749a859c..2217004264e4b 100644
+--- a/drivers/media/i2c/adv7511-v4l2.c
++++ b/drivers/media/i2c/adv7511-v4l2.c
+@@ -544,7 +544,7 @@ static void log_infoframe(struct v4l2_subdev *sd, const struct adv7511_cfg_read_
+ buffer[3] = 0;
+ buffer[3] = hdmi_infoframe_checksum(buffer, len + 4);
+
+- if (hdmi_infoframe_unpack(&frame, buffer, sizeof(buffer)) < 0) {
++ if (hdmi_infoframe_unpack(&frame, buffer, len + 4) < 0) {
+ v4l2_err(sd, "%s: unpack of %s infoframe failed\n", __func__, cri->desc);
+ return;
+ }
+diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
+index 122e1fdccd962..d688ffff7a074 100644
+--- a/drivers/media/i2c/adv7604.c
++++ b/drivers/media/i2c/adv7604.c
+@@ -2484,7 +2484,7 @@ static int adv76xx_read_infoframe(struct v4l2_subdev *sd, int index,
+ buffer[i + 3] = infoframe_read(sd,
+ adv76xx_cri[index].payload_addr + i);
+
+- if (hdmi_infoframe_unpack(frame, buffer, sizeof(buffer)) < 0) {
++ if (hdmi_infoframe_unpack(frame, buffer, len + 3) < 0) {
+ v4l2_err(sd, "%s: unpack of %s infoframe failed\n", __func__,
+ adv76xx_cri[index].desc);
+ return -ENOENT;
+diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c
+index 7f8acbdf0db4a..8ab4c63839b49 100644
+--- a/drivers/media/i2c/adv7842.c
++++ b/drivers/media/i2c/adv7842.c
+@@ -2593,7 +2593,7 @@ static void log_infoframe(struct v4l2_subdev *sd, const struct adv7842_cfg_read_
+ for (i = 0; i < len; i++)
+ buffer[i + 3] = infoframe_read(sd, cri->payload_addr + i);
+
+- if (hdmi_infoframe_unpack(&frame, buffer, sizeof(buffer)) < 0) {
++ if (hdmi_infoframe_unpack(&frame, buffer, len + 3) < 0) {
+ v4l2_err(sd, "%s: unpack of %s infoframe failed\n", __func__, cri->desc);
+ return;
+ }
+diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
+index 81cdf37216ca7..c249507aa2dbc 100644
+--- a/drivers/media/i2c/imx258.c
++++ b/drivers/media/i2c/imx258.c
+@@ -1260,18 +1260,18 @@ static int imx258_probe(struct i2c_client *client)
+ return -ENOMEM;
+
+ imx258->clk = devm_clk_get_optional(&client->dev, NULL);
++ if (IS_ERR(imx258->clk))
++ return dev_err_probe(&client->dev, PTR_ERR(imx258->clk),
++ "error getting clock\n");
+ if (!imx258->clk) {
+ dev_dbg(&client->dev,
+ "no clock provided, using clock-frequency property\n");
+
+ device_property_read_u32(&client->dev, "clock-frequency", &val);
+- if (val != IMX258_INPUT_CLOCK_FREQ)
+- return -EINVAL;
+- } else if (IS_ERR(imx258->clk)) {
+- return dev_err_probe(&client->dev, PTR_ERR(imx258->clk),
+- "error getting clock\n");
++ } else {
++ val = clk_get_rate(imx258->clk);
+ }
+- if (clk_get_rate(imx258->clk) != IMX258_INPUT_CLOCK_FREQ) {
++ if (val != IMX258_INPUT_CLOCK_FREQ) {
+ dev_err(&client->dev, "input clock frequency not supported\n");
+ return -EINVAL;
+ }
+diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
+index 0dce92872176d..4d9b64c61f603 100644
+--- a/drivers/media/i2c/imx274.c
++++ b/drivers/media/i2c/imx274.c
+@@ -1367,6 +1367,10 @@ static int imx274_s_frame_interval(struct v4l2_subdev *sd,
+ int min, max, def;
+ int ret;
+
++ ret = pm_runtime_resume_and_get(&imx274->client->dev);
++ if (ret < 0)
++ return ret;
++
+ mutex_lock(&imx274->lock);
+ ret = imx274_set_frame_interval(imx274, fi->interval);
+
+@@ -1398,6 +1402,7 @@ static int imx274_s_frame_interval(struct v4l2_subdev *sd,
+
+ unlock:
+ mutex_unlock(&imx274->lock);
++ pm_runtime_put(&imx274->client->dev);
+
+ return ret;
+ }
+diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
+index 92376592455ee..56674173524fd 100644
+--- a/drivers/media/i2c/ir-kbd-i2c.c
++++ b/drivers/media/i2c/ir-kbd-i2c.c
+@@ -791,6 +791,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
+ rc_proto = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+ RC_PROTO_BIT_RC6_6A_32;
+ ir_codes = RC_MAP_HAUPPAUGE;
++ ir->polling_interval = 125;
+ probe_tx = true;
+ break;
+ }
+diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c
+index 6eb88ef997836..3ae1b28c8351b 100644
+--- a/drivers/media/i2c/mt9p031.c
++++ b/drivers/media/i2c/mt9p031.c
+@@ -78,7 +78,9 @@
+ #define MT9P031_PIXEL_CLOCK_INVERT (1 << 15)
+ #define MT9P031_PIXEL_CLOCK_SHIFT(n) ((n) << 8)
+ #define MT9P031_PIXEL_CLOCK_DIVIDE(n) ((n) << 0)
+-#define MT9P031_FRAME_RESTART 0x0b
++#define MT9P031_RESTART 0x0b
++#define MT9P031_FRAME_PAUSE_RESTART (1 << 1)
++#define MT9P031_FRAME_RESTART (1 << 0)
+ #define MT9P031_SHUTTER_DELAY 0x0c
+ #define MT9P031_RST 0x0d
+ #define MT9P031_RST_ENABLE 1
+@@ -444,9 +446,23 @@ static int mt9p031_set_params(struct mt9p031 *mt9p031)
+ static int mt9p031_s_stream(struct v4l2_subdev *subdev, int enable)
+ {
+ struct mt9p031 *mt9p031 = to_mt9p031(subdev);
++ struct i2c_client *client = v4l2_get_subdevdata(subdev);
++ int val;
+ int ret;
+
+ if (!enable) {
++ /* enable pause restart */
++ val = MT9P031_FRAME_PAUSE_RESTART;
++ ret = mt9p031_write(client, MT9P031_RESTART, val);
++ if (ret < 0)
++ return ret;
++
++ /* enable restart + keep pause restart set */
++ val |= MT9P031_FRAME_RESTART;
++ ret = mt9p031_write(client, MT9P031_RESTART, val);
++ if (ret < 0)
++ return ret;
++
+ /* Stop sensor readout */
+ ret = mt9p031_set_output_control(mt9p031,
+ MT9P031_OUTPUT_CONTROL_CEN, 0);
+@@ -466,6 +482,16 @@ static int mt9p031_s_stream(struct v4l2_subdev *subdev, int enable)
+ if (ret < 0)
+ return ret;
+
++ /*
++ * - clear pause restart
++ * - don't clear restart as clearing restart manually can cause
++ * undefined behavior
++ */
++ val = MT9P031_FRAME_RESTART;
++ ret = mt9p031_write(client, MT9P031_RESTART, val);
++ if (ret < 0)
++ return ret;
++
+ return mt9p031_pll_enable(mt9p031);
+ }
+
+diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
+index ddbd71394db33..db5a19babe67d 100644
+--- a/drivers/media/i2c/ov5640.c
++++ b/drivers/media/i2c/ov5640.c
+@@ -2293,7 +2293,6 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd,
+ struct ov5640_dev *sensor = to_ov5640_dev(sd);
+ const struct ov5640_mode_info *new_mode;
+ struct v4l2_mbus_framefmt *mbus_fmt = &format->format;
+- struct v4l2_mbus_framefmt *fmt;
+ int ret;
+
+ if (format->pad != 0)
+@@ -2311,12 +2310,10 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd,
+ if (ret)
+ goto out;
+
+- if (format->which == V4L2_SUBDEV_FORMAT_TRY)
+- fmt = v4l2_subdev_get_try_format(sd, sd_state, 0);
+- else
+- fmt = &sensor->fmt;
+-
+- *fmt = *mbus_fmt;
++ if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
++ *v4l2_subdev_get_try_format(sd, sd_state, 0) = *mbus_fmt;
++ goto out;
++ }
+
+ if (new_mode != sensor->current_mode) {
+ sensor->current_mode = new_mode;
+@@ -2325,6 +2322,9 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd,
+ if (mbus_fmt->code != sensor->fmt.code)
+ sensor->pending_fmt_change = true;
+
++ /* update format even if code is unchanged, resolution might change */
++ sensor->fmt = *mbus_fmt;
++
+ __v4l2_ctrl_s_ctrl_int64(sensor->ctrls.pixel_rate,
+ ov5640_calc_pixel_rate(sensor));
+ out:
+diff --git a/drivers/media/i2c/ov5648.c b/drivers/media/i2c/ov5648.c
+index 947d437ed0efe..ef8b52dc9401d 100644
+--- a/drivers/media/i2c/ov5648.c
++++ b/drivers/media/i2c/ov5648.c
+@@ -639,7 +639,7 @@ struct ov5648_ctrls {
+ struct v4l2_ctrl *pixel_rate;
+
+ struct v4l2_ctrl_handler handler;
+-} __packed;
++};
+
+ struct ov5648_sensor {
+ struct device *dev;
+@@ -1778,8 +1778,14 @@ static int ov5648_state_configure(struct ov5648_sensor *sensor,
+
+ static int ov5648_state_init(struct ov5648_sensor *sensor)
+ {
+- return ov5648_state_configure(sensor, &ov5648_modes[0],
+- ov5648_mbus_codes[0]);
++ int ret;
++
++ mutex_lock(&sensor->mutex);
++ ret = ov5648_state_configure(sensor, &ov5648_modes[0],
++ ov5648_mbus_codes[0]);
++ mutex_unlock(&sensor->mutex);
++
++ return ret;
+ }
+
+ /* Sensor Base */
+diff --git a/drivers/media/i2c/ov6650.c b/drivers/media/i2c/ov6650.c
+index f67412150b16b..eb59dc8bb5929 100644
+--- a/drivers/media/i2c/ov6650.c
++++ b/drivers/media/i2c/ov6650.c
+@@ -472,9 +472,16 @@ static int ov6650_get_selection(struct v4l2_subdev *sd,
+ {
+ struct i2c_client *client = v4l2_get_subdevdata(sd);
+ struct ov6650 *priv = to_ov6650(client);
++ struct v4l2_rect *rect;
+
+- if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+- return -EINVAL;
++ if (sel->which == V4L2_SUBDEV_FORMAT_TRY) {
++ /* pre-select try crop rectangle */
++ rect = &sd_state->pads->try_crop;
++
++ } else {
++ /* pre-select active crop rectangle */
++ rect = &priv->rect;
++ }
+
+ switch (sel->target) {
+ case V4L2_SEL_TGT_CROP_BOUNDS:
+@@ -483,14 +490,33 @@ static int ov6650_get_selection(struct v4l2_subdev *sd,
+ sel->r.width = W_CIF;
+ sel->r.height = H_CIF;
+ return 0;
++
+ case V4L2_SEL_TGT_CROP:
+- sel->r = priv->rect;
++ /* use selected crop rectangle */
++ sel->r = *rect;
+ return 0;
++
+ default:
+ return -EINVAL;
+ }
+ }
+
++static bool is_unscaled_ok(int width, int height, struct v4l2_rect *rect)
++{
++ return width > rect->width >> 1 || height > rect->height >> 1;
++}
++
++static void ov6650_bind_align_crop_rectangle(struct v4l2_rect *rect)
++{
++ v4l_bound_align_image(&rect->width, 2, W_CIF, 1,
++ &rect->height, 2, H_CIF, 1, 0);
++ v4l_bound_align_image(&rect->left, DEF_HSTRT << 1,
++ (DEF_HSTRT << 1) + W_CIF - (__s32)rect->width, 1,
++ &rect->top, DEF_VSTRT << 1,
++ (DEF_VSTRT << 1) + H_CIF - (__s32)rect->height,
++ 1, 0);
++}
++
+ static int ov6650_set_selection(struct v4l2_subdev *sd,
+ struct v4l2_subdev_state *sd_state,
+ struct v4l2_subdev_selection *sel)
+@@ -499,18 +525,30 @@ static int ov6650_set_selection(struct v4l2_subdev *sd,
+ struct ov6650 *priv = to_ov6650(client);
+ int ret;
+
+- if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+- sel->target != V4L2_SEL_TGT_CROP)
++ if (sel->target != V4L2_SEL_TGT_CROP)
+ return -EINVAL;
+
+- v4l_bound_align_image(&sel->r.width, 2, W_CIF, 1,
+- &sel->r.height, 2, H_CIF, 1, 0);
+- v4l_bound_align_image(&sel->r.left, DEF_HSTRT << 1,
+- (DEF_HSTRT << 1) + W_CIF - (__s32)sel->r.width, 1,
+- &sel->r.top, DEF_VSTRT << 1,
+- (DEF_VSTRT << 1) + H_CIF - (__s32)sel->r.height,
+- 1, 0);
++ ov6650_bind_align_crop_rectangle(&sel->r);
++
++ if (sel->which == V4L2_SUBDEV_FORMAT_TRY) {
++ struct v4l2_rect *crop = &sd_state->pads->try_crop;
++ struct v4l2_mbus_framefmt *mf = &sd_state->pads->try_fmt;
++ /* detect current pad config scaling factor */
++ bool half_scale = !is_unscaled_ok(mf->width, mf->height, crop);
++
++ /* store new crop rectangle */
++ *crop = sel->r;
+
++ /* adjust frame size */
++ mf->width = crop->width >> half_scale;
++ mf->height = crop->height >> half_scale;
++
++ return 0;
++ }
++
++ /* V4L2_SUBDEV_FORMAT_ACTIVE */
++
++ /* apply new crop rectangle */
+ ret = ov6650_reg_write(client, REG_HSTRT, sel->r.left >> 1);
+ if (!ret) {
+ priv->rect.width += priv->rect.left - sel->r.left;
+@@ -562,30 +600,13 @@ static int ov6650_get_fmt(struct v4l2_subdev *sd,
+ return 0;
+ }
+
+-static bool is_unscaled_ok(int width, int height, struct v4l2_rect *rect)
+-{
+- return width > rect->width >> 1 || height > rect->height >> 1;
+-}
+-
+ #define to_clkrc(div) ((div) - 1)
+
+ /* set the format we will capture in */
+-static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
++static int ov6650_s_fmt(struct v4l2_subdev *sd, u32 code, bool half_scale)
+ {
+ struct i2c_client *client = v4l2_get_subdevdata(sd);
+ struct ov6650 *priv = to_ov6650(client);
+- bool half_scale = !is_unscaled_ok(mf->width, mf->height, &priv->rect);
+- struct v4l2_subdev_selection sel = {
+- .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+- .target = V4L2_SEL_TGT_CROP,
+- .r.left = priv->rect.left + (priv->rect.width >> 1) -
+- (mf->width >> (1 - half_scale)),
+- .r.top = priv->rect.top + (priv->rect.height >> 1) -
+- (mf->height >> (1 - half_scale)),
+- .r.width = mf->width << half_scale,
+- .r.height = mf->height << half_scale,
+- };
+- u32 code = mf->code;
+ u8 coma_set = 0, coma_mask = 0, coml_set, coml_mask;
+ int ret;
+
+@@ -653,9 +674,7 @@ static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
+ coma_mask |= COMA_QCIF;
+ }
+
+- ret = ov6650_set_selection(sd, NULL, &sel);
+- if (!ret)
+- ret = ov6650_reg_rmw(client, REG_COMA, coma_set, coma_mask);
++ ret = ov6650_reg_rmw(client, REG_COMA, coma_set, coma_mask);
+ if (!ret) {
+ priv->half_scale = half_scale;
+
+@@ -674,14 +693,12 @@ static int ov6650_set_fmt(struct v4l2_subdev *sd,
+ struct v4l2_mbus_framefmt *mf = &format->format;
+ struct i2c_client *client = v4l2_get_subdevdata(sd);
+ struct ov6650 *priv = to_ov6650(client);
++ struct v4l2_rect *crop;
++ bool half_scale;
+
+ if (format->pad)
+ return -EINVAL;
+
+- if (is_unscaled_ok(mf->width, mf->height, &priv->rect))
+- v4l_bound_align_image(&mf->width, 2, W_CIF, 1,
+- &mf->height, 2, H_CIF, 1, 0);
+-
+ switch (mf->code) {
+ case MEDIA_BUS_FMT_Y10_1X10:
+ mf->code = MEDIA_BUS_FMT_Y8_1X8;
+@@ -699,10 +716,17 @@ static int ov6650_set_fmt(struct v4l2_subdev *sd,
+ break;
+ }
+
++ if (format->which == V4L2_SUBDEV_FORMAT_TRY)
++ crop = &sd_state->pads->try_crop;
++ else
++ crop = &priv->rect;
++
++ half_scale = !is_unscaled_ok(mf->width, mf->height, crop);
++
+ if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
+- /* store media bus format code and frame size in pad config */
+- sd_state->pads->try_fmt.width = mf->width;
+- sd_state->pads->try_fmt.height = mf->height;
++ /* store new mbus frame format code and size in pad config */
++ sd_state->pads->try_fmt.width = crop->width >> half_scale;
++ sd_state->pads->try_fmt.height = crop->height >> half_scale;
+ sd_state->pads->try_fmt.code = mf->code;
+
+ /* return default mbus frame format updated with pad config */
+@@ -712,9 +736,11 @@ static int ov6650_set_fmt(struct v4l2_subdev *sd,
+ mf->code = sd_state->pads->try_fmt.code;
+
+ } else {
+- /* apply new media bus format code and frame size */
+- int ret = ov6650_s_fmt(sd, mf);
++ int ret = 0;
+
++ /* apply new media bus frame format and scaling if changed */
++ if (mf->code != priv->code || half_scale != priv->half_scale)
++ ret = ov6650_s_fmt(sd, mf->code, half_scale);
+ if (ret)
+ return ret;
+
+@@ -890,9 +916,8 @@ static int ov6650_video_probe(struct v4l2_subdev *sd)
+ if (!ret)
+ ret = ov6650_prog_dflt(client, xclk->clkrc);
+ if (!ret) {
+- struct v4l2_mbus_framefmt mf = ov6650_def_fmt;
+-
+- ret = ov6650_s_fmt(sd, &mf);
++ /* driver default frame format, no scaling */
++ ret = ov6650_s_fmt(sd, ov6650_def_fmt.code, false);
+ }
+ if (!ret)
+ ret = v4l2_ctrl_handler_setup(&priv->hdl);
+diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c
+index ce50f3ea87b8e..92f6c3a940cfb 100644
+--- a/drivers/media/i2c/ov8865.c
++++ b/drivers/media/i2c/ov8865.c
+@@ -2330,27 +2330,27 @@ static int ov8865_sensor_power(struct ov8865_sensor *sensor, bool on)
+ if (ret) {
+ dev_err(sensor->dev,
+ "failed to enable DOVDD regulator\n");
+- goto disable;
++ return ret;
+ }
+
+ ret = regulator_enable(sensor->avdd);
+ if (ret) {
+ dev_err(sensor->dev,
+ "failed to enable AVDD regulator\n");
+- goto disable;
++ goto disable_dovdd;
+ }
+
+ ret = regulator_enable(sensor->dvdd);
+ if (ret) {
+ dev_err(sensor->dev,
+ "failed to enable DVDD regulator\n");
+- goto disable;
++ goto disable_avdd;
+ }
+
+ ret = clk_prepare_enable(sensor->extclk);
+ if (ret) {
+ dev_err(sensor->dev, "failed to enable EXTCLK clock\n");
+- goto disable;
++ goto disable_dvdd;
+ }
+
+ gpiod_set_value_cansleep(sensor->reset, 0);
+@@ -2359,14 +2359,16 @@ static int ov8865_sensor_power(struct ov8865_sensor *sensor, bool on)
+ /* Time to enter streaming mode according to power timings. */
+ usleep_range(10000, 12000);
+ } else {
+-disable:
+ gpiod_set_value_cansleep(sensor->powerdown, 1);
+ gpiod_set_value_cansleep(sensor->reset, 1);
+
+ clk_disable_unprepare(sensor->extclk);
+
++disable_dvdd:
+ regulator_disable(sensor->dvdd);
++disable_avdd:
+ regulator_disable(sensor->avdd);
++disable_dovdd:
+ regulator_disable(sensor->dovdd);
+ }
+
+@@ -2891,14 +2893,16 @@ static int ov8865_probe(struct i2c_client *client)
+ if (ret)
+ goto error_mutex;
+
++ mutex_lock(&sensor->mutex);
+ ret = ov8865_state_init(sensor);
++ mutex_unlock(&sensor->mutex);
+ if (ret)
+ goto error_ctrls;
+
+ /* Runtime PM */
+
+- pm_runtime_enable(sensor->dev);
+ pm_runtime_set_suspended(sensor->dev);
++ pm_runtime_enable(sensor->dev);
+
+ /* V4L2 subdev register */
+
+diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c
+index 6070aaf0b32ea..4dafa9f1cf522 100644
+--- a/drivers/media/i2c/tda1997x.c
++++ b/drivers/media/i2c/tda1997x.c
+@@ -1248,13 +1248,13 @@ tda1997x_parse_infoframe(struct tda1997x_state *state, u16 addr)
+ {
+ struct v4l2_subdev *sd = &state->sd;
+ union hdmi_infoframe frame;
+- u8 buffer[40];
++ u8 buffer[40] = { 0 };
+ u8 reg;
+ int len, err;
+
+ /* read data */
+ len = io_readn(sd, addr, sizeof(buffer), buffer);
+- err = hdmi_infoframe_unpack(&frame, buffer, sizeof(buffer));
++ err = hdmi_infoframe_unpack(&frame, buffer, len);
+ if (err) {
+ v4l_err(state->client,
+ "failed parsing %d byte infoframe: 0x%04x/0x%02x\n",
+@@ -1928,13 +1928,13 @@ static int tda1997x_log_infoframe(struct v4l2_subdev *sd, int addr)
+ {
+ struct tda1997x_state *state = to_state(sd);
+ union hdmi_infoframe frame;
+- u8 buffer[40];
++ u8 buffer[40] = { 0 };
+ int len, err;
+
+ /* read data */
+ len = io_readn(sd, addr, sizeof(buffer), buffer);
+ v4l2_dbg(1, debug, sd, "infoframe: addr=%d len=%d\n", addr, len);
+- err = hdmi_infoframe_unpack(&frame, buffer, sizeof(buffer));
++ err = hdmi_infoframe_unpack(&frame, buffer, len);
+ if (err) {
+ v4l_err(state->client,
+ "failed parsing %d byte infoframe: 0x%04x/0x%02x\n",
+diff --git a/drivers/media/pci/b2c2/flexcop-pci.c b/drivers/media/pci/b2c2/flexcop-pci.c
+index 6a4c7cb0ad0f9..486c8ec0fa60d 100644
+--- a/drivers/media/pci/b2c2/flexcop-pci.c
++++ b/drivers/media/pci/b2c2/flexcop-pci.c
+@@ -185,6 +185,8 @@ static irqreturn_t flexcop_pci_isr(int irq, void *dev_id)
+ dma_addr_t cur_addr =
+ fc->read_ibi_reg(fc,dma1_008).dma_0x8.dma_cur_addr << 2;
+ u32 cur_pos = cur_addr - fc_pci->dma[0].dma_addr0;
++ if (cur_pos > fc_pci->dma[0].size * 2)
++ goto error;
+
+ deb_irq("%u irq: %08x cur_addr: %llx: cur_pos: %08x, last_cur_pos: %08x ",
+ jiffies_to_usecs(jiffies - fc_pci->last_irq),
+@@ -225,6 +227,7 @@ static irqreturn_t flexcop_pci_isr(int irq, void *dev_id)
+ ret = IRQ_NONE;
+ }
+
++error:
+ spin_unlock_irqrestore(&fc_pci->irq_lock, flags);
+ return ret;
+ }
+diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
+index 0e9df8b35ac66..661ebfa7bf3f5 100644
+--- a/drivers/media/pci/bt8xx/bttv-driver.c
++++ b/drivers/media/pci/bt8xx/bttv-driver.c
+@@ -3890,7 +3890,7 @@ static int bttv_register_video(struct bttv *btv)
+
+ /* video */
+ vdev_init(btv, &btv->video_dev, &bttv_video_template, "video");
+- btv->video_dev.device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_TUNER |
++ btv->video_dev.device_caps = V4L2_CAP_VIDEO_CAPTURE |
+ V4L2_CAP_READWRITE | V4L2_CAP_STREAMING;
+ if (btv->tuner_type != TUNER_ABSENT)
+ btv->video_dev.device_caps |= V4L2_CAP_TUNER;
+@@ -3911,7 +3911,7 @@ static int bttv_register_video(struct bttv *btv)
+ /* vbi */
+ vdev_init(btv, &btv->vbi_dev, &bttv_video_template, "vbi");
+ btv->vbi_dev.device_caps = V4L2_CAP_VBI_CAPTURE | V4L2_CAP_READWRITE |
+- V4L2_CAP_STREAMING | V4L2_CAP_TUNER;
++ V4L2_CAP_STREAMING;
+ if (btv->tuner_type != TUNER_ABSENT)
+ btv->vbi_dev.device_caps |= V4L2_CAP_TUNER;
+
+diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c
+index ab14d35214aa8..25dc8d4dc5b73 100644
+--- a/drivers/media/pci/cx23885/cx23885-alsa.c
++++ b/drivers/media/pci/cx23885/cx23885-alsa.c
+@@ -550,7 +550,7 @@ struct cx23885_audio_dev *cx23885_audio_register(struct cx23885_dev *dev)
+ SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1,
+ THIS_MODULE, sizeof(struct cx23885_audio_dev), &card);
+ if (err < 0)
+- goto error;
++ goto error_msg;
+
+ chip = (struct cx23885_audio_dev *) card->private_data;
+ chip->dev = dev;
+@@ -576,6 +576,7 @@ struct cx23885_audio_dev *cx23885_audio_register(struct cx23885_dev *dev)
+
+ error:
+ snd_card_free(card);
++error_msg:
+ pr_err("%s(): Failed to register analog audio adapter\n",
+ __func__);
+
+diff --git a/drivers/media/pci/cx88/cx88-mpeg.c b/drivers/media/pci/cx88/cx88-mpeg.c
+index 680e1e3fe89b7..2c1d5137ac470 100644
+--- a/drivers/media/pci/cx88/cx88-mpeg.c
++++ b/drivers/media/pci/cx88/cx88-mpeg.c
+@@ -162,6 +162,9 @@ int cx8802_start_dma(struct cx8802_dev *dev,
+ cx_write(MO_TS_GPCNTRL, GP_COUNT_CONTROL_RESET);
+ q->count = 0;
+
++ /* clear interrupt status register */
++ cx_write(MO_TS_INTSTAT, 0x1f1111);
++
+ /* enable irqs */
+ dprintk(1, "setting the interrupt mask\n");
+ cx_set(MO_PCI_INTMSK, core->pci_irqmask | PCI_INT_TSINT);
+diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h
+index 4cf92dee65271..ce3a7ca51736e 100644
+--- a/drivers/media/pci/ivtv/ivtv-driver.h
++++ b/drivers/media/pci/ivtv/ivtv-driver.h
+@@ -330,7 +330,6 @@ struct ivtv_stream {
+ struct ivtv *itv; /* for ease of use */
+ const char *name; /* name of the stream */
+ int type; /* stream type */
+- u32 caps; /* V4L2 capabilities */
+
+ struct v4l2_fh *fh; /* pointer to the streaming filehandle */
+ spinlock_t qlock; /* locks access to the queues */
+diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
+index da19b2e95e6cf..6d2d3348e3215 100644
+--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
++++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
+@@ -438,7 +438,7 @@ static int ivtv_g_fmt_vid_out_overlay(struct file *file, void *fh, struct v4l2_f
+ struct ivtv_stream *s = &itv->streams[fh2id(fh)->type];
+ struct v4l2_window *winfmt = &fmt->fmt.win;
+
+- if (!(s->caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
++ if (!(s->vdev.device_caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
+ return -EINVAL;
+ if (!itv->osd_video_pbase)
+ return -EINVAL;
+@@ -549,7 +549,7 @@ static int ivtv_try_fmt_vid_out_overlay(struct file *file, void *fh, struct v4l2
+ u32 chromakey = fmt->fmt.win.chromakey;
+ u8 global_alpha = fmt->fmt.win.global_alpha;
+
+- if (!(s->caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
++ if (!(s->vdev.device_caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
+ return -EINVAL;
+ if (!itv->osd_video_pbase)
+ return -EINVAL;
+@@ -1383,7 +1383,7 @@ static int ivtv_g_fbuf(struct file *file, void *fh, struct v4l2_framebuffer *fb)
+ 0,
+ };
+
+- if (!(s->caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
++ if (!(s->vdev.device_caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
+ return -ENOTTY;
+ if (!itv->osd_video_pbase)
+ return -ENOTTY;
+@@ -1450,7 +1450,7 @@ static int ivtv_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuffe
+ struct ivtv_stream *s = &itv->streams[fh2id(fh)->type];
+ struct yuv_playback_info *yi = &itv->yuv_info;
+
+- if (!(s->caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
++ if (!(s->vdev.device_caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
+ return -ENOTTY;
+ if (!itv->osd_video_pbase)
+ return -ENOTTY;
+@@ -1470,7 +1470,7 @@ static int ivtv_overlay(struct file *file, void *fh, unsigned int on)
+ struct ivtv *itv = id->itv;
+ struct ivtv_stream *s = &itv->streams[fh2id(fh)->type];
+
+- if (!(s->caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
++ if (!(s->vdev.device_caps & V4L2_CAP_VIDEO_OUTPUT_OVERLAY))
+ return -ENOTTY;
+ if (!itv->osd_video_pbase)
+ return -ENOTTY;
+diff --git a/drivers/media/pci/ivtv/ivtv-streams.c b/drivers/media/pci/ivtv/ivtv-streams.c
+index f04ee84bab5fd..f9de5d1605fe3 100644
+--- a/drivers/media/pci/ivtv/ivtv-streams.c
++++ b/drivers/media/pci/ivtv/ivtv-streams.c
+@@ -176,7 +176,7 @@ static void ivtv_stream_init(struct ivtv *itv, int type)
+ s->itv = itv;
+ s->type = type;
+ s->name = ivtv_stream_info[type].name;
+- s->caps = ivtv_stream_info[type].v4l2_caps;
++ s->vdev.device_caps = ivtv_stream_info[type].v4l2_caps;
+
+ if (ivtv_stream_info[type].pio)
+ s->dma = PCI_DMA_NONE;
+@@ -299,12 +299,9 @@ static int ivtv_reg_dev(struct ivtv *itv, int type)
+ if (s_mpg->vdev.v4l2_dev)
+ num = s_mpg->vdev.num + ivtv_stream_info[type].num_offset;
+ }
+- s->vdev.device_caps = s->caps;
+- if (itv->osd_video_pbase) {
+- itv->streams[IVTV_DEC_STREAM_TYPE_YUV].vdev.device_caps |=
+- V4L2_CAP_VIDEO_OUTPUT_OVERLAY;
+- itv->streams[IVTV_DEC_STREAM_TYPE_MPG].vdev.device_caps |=
+- V4L2_CAP_VIDEO_OUTPUT_OVERLAY;
++ if (itv->osd_video_pbase && (type == IVTV_DEC_STREAM_TYPE_YUV ||
++ type == IVTV_DEC_STREAM_TYPE_MPG)) {
++ s->vdev.device_caps |= V4L2_CAP_VIDEO_OUTPUT_OVERLAY;
+ itv->v4l2_cap |= V4L2_CAP_VIDEO_OUTPUT_OVERLAY;
+ }
+ video_set_drvdata(&s->vdev, s);
+diff --git a/drivers/media/pci/ivtv/ivtvfb.c b/drivers/media/pci/ivtv/ivtvfb.c
+index e2d56dca5be40..5ad03b2a50bdb 100644
+--- a/drivers/media/pci/ivtv/ivtvfb.c
++++ b/drivers/media/pci/ivtv/ivtvfb.c
+@@ -36,7 +36,7 @@
+ #include <linux/fb.h>
+ #include <linux/ivtvfb.h>
+
+-#ifdef CONFIG_X86_64
++#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
+ #include <asm/memtype.h>
+ #endif
+
+@@ -1157,7 +1157,7 @@ static int ivtvfb_init_card(struct ivtv *itv)
+ {
+ int rc;
+
+-#ifdef CONFIG_X86_64
++#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
+ if (pat_enabled()) {
+ if (ivtvfb_force_pat) {
+ pr_info("PAT is enabled. Write-combined framebuffer caching will be disabled.\n");
+diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+index 6f3125c2d0976..77bae14685513 100644
+--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
++++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+@@ -258,19 +258,24 @@ static irqreturn_t netup_unidvb_isr(int irq, void *dev_id)
+ if ((reg40 & AVL_IRQ_ASSERTED) != 0) {
+ /* IRQ is being signaled */
+ reg_isr = readw(ndev->bmmio0 + REG_ISR);
+- if (reg_isr & NETUP_UNIDVB_IRQ_I2C0) {
+- iret = netup_i2c_interrupt(&ndev->i2c[0]);
+- } else if (reg_isr & NETUP_UNIDVB_IRQ_I2C1) {
+- iret = netup_i2c_interrupt(&ndev->i2c[1]);
+- } else if (reg_isr & NETUP_UNIDVB_IRQ_SPI) {
++ if (reg_isr & NETUP_UNIDVB_IRQ_SPI)
+ iret = netup_spi_interrupt(ndev->spi);
+- } else if (reg_isr & NETUP_UNIDVB_IRQ_DMA1) {
+- iret = netup_dma_interrupt(&ndev->dma[0]);
+- } else if (reg_isr & NETUP_UNIDVB_IRQ_DMA2) {
+- iret = netup_dma_interrupt(&ndev->dma[1]);
+- } else if (reg_isr & NETUP_UNIDVB_IRQ_CI) {
+- iret = netup_ci_interrupt(ndev);
++ else if (!ndev->old_fw) {
++ if (reg_isr & NETUP_UNIDVB_IRQ_I2C0) {
++ iret = netup_i2c_interrupt(&ndev->i2c[0]);
++ } else if (reg_isr & NETUP_UNIDVB_IRQ_I2C1) {
++ iret = netup_i2c_interrupt(&ndev->i2c[1]);
++ } else if (reg_isr & NETUP_UNIDVB_IRQ_DMA1) {
++ iret = netup_dma_interrupt(&ndev->dma[0]);
++ } else if (reg_isr & NETUP_UNIDVB_IRQ_DMA2) {
++ iret = netup_dma_interrupt(&ndev->dma[1]);
++ } else if (reg_isr & NETUP_UNIDVB_IRQ_CI) {
++ iret = netup_ci_interrupt(ndev);
++ } else {
++ goto err;
++ }
+ } else {
++err:
+ dev_err(&pci_dev->dev,
+ "%s(): unknown interrupt 0x%x\n",
+ __func__, reg_isr);
+diff --git a/drivers/media/pci/saa7134/saa7134-alsa.c b/drivers/media/pci/saa7134/saa7134-alsa.c
+index fb24d2ed3621b..d3cde05a6ebab 100644
+--- a/drivers/media/pci/saa7134/saa7134-alsa.c
++++ b/drivers/media/pci/saa7134/saa7134-alsa.c
+@@ -1214,7 +1214,7 @@ static int alsa_device_exit(struct saa7134_dev *dev)
+
+ static int saa7134_alsa_init(void)
+ {
+- struct saa7134_dev *dev = NULL;
++ struct saa7134_dev *dev;
+
+ saa7134_dmasound_init = alsa_device_init;
+ saa7134_dmasound_exit = alsa_device_exit;
+@@ -1229,7 +1229,7 @@ static int saa7134_alsa_init(void)
+ alsa_device_init(dev);
+ }
+
+- if (dev == NULL)
++ if (list_empty(&saa7134_devlist))
+ pr_info("saa7134 ALSA: no saa7134 cards found\n");
+
+ return 0;
+diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
+index 2214c74bbbf15..3947701cd6c7e 100644
+--- a/drivers/media/pci/saa7146/hexium_gemini.c
++++ b/drivers/media/pci/saa7146/hexium_gemini.c
+@@ -284,7 +284,12 @@ static int hexium_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_d
+ hexium_set_input(hexium, 0);
+ hexium->cur_input = 0;
+
+- saa7146_vv_init(dev, &vv_data);
++ ret = saa7146_vv_init(dev, &vv_data);
++ if (ret) {
++ i2c_del_adapter(&hexium->i2c_adapter);
++ kfree(hexium);
++ return ret;
++ }
+
+ vv_data.vid_ops.vidioc_enum_input = vidioc_enum_input;
+ vv_data.vid_ops.vidioc_g_input = vidioc_g_input;
+diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
+index 39d14c179d229..2eb4bee16b71f 100644
+--- a/drivers/media/pci/saa7146/hexium_orion.c
++++ b/drivers/media/pci/saa7146/hexium_orion.c
+@@ -355,10 +355,16 @@ static struct saa7146_ext_vv vv_data;
+ static int hexium_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_data *info)
+ {
+ struct hexium *hexium = (struct hexium *) dev->ext_priv;
++ int ret;
+
+ DEB_EE("\n");
+
+- saa7146_vv_init(dev, &vv_data);
++ ret = saa7146_vv_init(dev, &vv_data);
++ if (ret) {
++ pr_err("Error in saa7146_vv_init()\n");
++ return ret;
++ }
++
+ vv_data.vid_ops.vidioc_enum_input = vidioc_enum_input;
+ vv_data.vid_ops.vidioc_g_input = vidioc_g_input;
+ vv_data.vid_ops.vidioc_s_input = vidioc_s_input;
+diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
+index 73fc901ecf3db..bf0b9b0914cd5 100644
+--- a/drivers/media/pci/saa7146/mxb.c
++++ b/drivers/media/pci/saa7146/mxb.c
+@@ -683,10 +683,16 @@ static struct saa7146_ext_vv vv_data;
+ static int mxb_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_data *info)
+ {
+ struct mxb *mxb;
++ int ret;
+
+ DEB_EE("dev:%p\n", dev);
+
+- saa7146_vv_init(dev, &vv_data);
++ ret = saa7146_vv_init(dev, &vv_data);
++ if (ret) {
++ ERR("Error in saa7146_vv_init()");
++ return ret;
++ }
++
+ if (mxb_probe(dev)) {
+ saa7146_vv_release(dev);
+ return -1;
+diff --git a/drivers/media/platform/allegro-dvt/allegro-core.c b/drivers/media/platform/allegro-dvt/allegro-core.c
+index 887b492e4ad1c..14a119b43bca0 100644
+--- a/drivers/media/platform/allegro-dvt/allegro-core.c
++++ b/drivers/media/platform/allegro-dvt/allegro-core.c
+@@ -2185,6 +2185,15 @@ static irqreturn_t allegro_irq_thread(int irq, void *data)
+ {
+ struct allegro_dev *dev = data;
+
++ /*
++ * The firmware is initialized after the mailbox is setup. We further
++ * check the AL5_ITC_CPU_IRQ_STA register, if the firmware actually
++ * triggered the interrupt. Although this should not happen, make sure
++ * that we ignore interrupts, if the mailbox is not initialized.
++ */
++ if (!dev->mbox_status)
++ return IRQ_NONE;
++
+ allegro_mbox_notify(dev->mbox_status);
+
+ return IRQ_HANDLED;
+diff --git a/drivers/media/platform/aspeed-video.c b/drivers/media/platform/aspeed-video.c
+index 7bb6babdcade0..757a58829a512 100644
+--- a/drivers/media/platform/aspeed-video.c
++++ b/drivers/media/platform/aspeed-video.c
+@@ -151,7 +151,7 @@
+ #define VE_SRC_TB_EDGE_DET_BOT GENMASK(28, VE_SRC_TB_EDGE_DET_BOT_SHF)
+
+ #define VE_MODE_DETECT_STATUS 0x098
+-#define VE_MODE_DETECT_H_PIXELS GENMASK(11, 0)
++#define VE_MODE_DETECT_H_PERIOD GENMASK(11, 0)
+ #define VE_MODE_DETECT_V_LINES_SHF 16
+ #define VE_MODE_DETECT_V_LINES GENMASK(27, VE_MODE_DETECT_V_LINES_SHF)
+ #define VE_MODE_DETECT_STATUS_VSYNC BIT(28)
+@@ -162,6 +162,8 @@
+ #define VE_SYNC_STATUS_VSYNC_SHF 16
+ #define VE_SYNC_STATUS_VSYNC GENMASK(27, VE_SYNC_STATUS_VSYNC_SHF)
+
++#define VE_H_TOTAL_PIXELS 0x0A0
++
+ #define VE_INTERRUPT_CTRL 0x304
+ #define VE_INTERRUPT_STATUS 0x308
+ #define VE_INTERRUPT_MODE_DETECT_WD BIT(0)
+@@ -500,6 +502,10 @@ static void aspeed_video_enable_mode_detect(struct aspeed_video *video)
+ aspeed_video_update(video, VE_INTERRUPT_CTRL, 0,
+ VE_INTERRUPT_MODE_DETECT);
+
++ /* Disable mode detect in order to re-trigger */
++ aspeed_video_update(video, VE_SEQ_CTRL,
++ VE_SEQ_CTRL_TRIG_MODE_DET, 0);
++
+ /* Trigger mode detect */
+ aspeed_video_update(video, VE_SEQ_CTRL, 0, VE_SEQ_CTRL_TRIG_MODE_DET);
+ }
+@@ -552,6 +558,8 @@ static void aspeed_video_irq_res_change(struct aspeed_video *video, ulong delay)
+ set_bit(VIDEO_RES_CHANGE, &video->flags);
+ clear_bit(VIDEO_FRAME_INPRG, &video->flags);
+
++ video->v4l2_input_status = V4L2_IN_ST_NO_SIGNAL;
++
+ aspeed_video_off(video);
+ aspeed_video_bufs_done(video, VB2_BUF_STATE_ERROR);
+
+@@ -759,6 +767,7 @@ static void aspeed_video_get_resolution(struct aspeed_video *video)
+ u32 src_lr_edge;
+ u32 src_tb_edge;
+ u32 sync;
++ u32 htotal;
+ struct v4l2_bt_timings *det = &video->detected_timings;
+
+ det->width = MIN_WIDTH;
+@@ -786,10 +795,6 @@ static void aspeed_video_get_resolution(struct aspeed_video *video)
+ return;
+ }
+
+- /* Disable mode detect in order to re-trigger */
+- aspeed_video_update(video, VE_SEQ_CTRL,
+- VE_SEQ_CTRL_TRIG_MODE_DET, 0);
+-
+ aspeed_video_check_and_set_polarity(video);
+
+ aspeed_video_enable_mode_detect(video);
+@@ -807,6 +812,7 @@ static void aspeed_video_get_resolution(struct aspeed_video *video)
+ src_tb_edge = aspeed_video_read(video, VE_SRC_TB_EDGE_DET);
+ mds = aspeed_video_read(video, VE_MODE_DETECT_STATUS);
+ sync = aspeed_video_read(video, VE_SYNC_STATUS);
++ htotal = aspeed_video_read(video, VE_H_TOTAL_PIXELS);
+
+ video->frame_bottom = (src_tb_edge & VE_SRC_TB_EDGE_DET_BOT) >>
+ VE_SRC_TB_EDGE_DET_BOT_SHF;
+@@ -823,8 +829,7 @@ static void aspeed_video_get_resolution(struct aspeed_video *video)
+ VE_SRC_LR_EDGE_DET_RT_SHF;
+ video->frame_left = src_lr_edge & VE_SRC_LR_EDGE_DET_LEFT;
+ det->hfrontporch = video->frame_left;
+- det->hbackporch = (mds & VE_MODE_DETECT_H_PIXELS) -
+- video->frame_right;
++ det->hbackporch = htotal - video->frame_right;
+ det->hsync = sync & VE_SYNC_STATUS_HSYNC;
+ if (video->frame_left > video->frame_right)
+ continue;
+@@ -1337,7 +1342,6 @@ static void aspeed_video_resolution_work(struct work_struct *work)
+ struct delayed_work *dwork = to_delayed_work(work);
+ struct aspeed_video *video = container_of(dwork, struct aspeed_video,
+ res_work);
+- u32 input_status = video->v4l2_input_status;
+
+ aspeed_video_on(video);
+
+@@ -1350,8 +1354,7 @@ static void aspeed_video_resolution_work(struct work_struct *work)
+ aspeed_video_get_resolution(video);
+
+ if (video->detected_timings.width != video->active_timings.width ||
+- video->detected_timings.height != video->active_timings.height ||
+- input_status != video->v4l2_input_status) {
++ video->detected_timings.height != video->active_timings.height) {
+ static const struct v4l2_event ev = {
+ .type = V4L2_EVENT_SOURCE_CHANGE,
+ .u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION,
+diff --git a/drivers/media/platform/atmel/atmel-isc-base.c b/drivers/media/platform/atmel/atmel-isc-base.c
+index 136ab7cf36edc..f768be3c40595 100644
+--- a/drivers/media/platform/atmel/atmel-isc-base.c
++++ b/drivers/media/platform/atmel/atmel-isc-base.c
+@@ -123,11 +123,9 @@ static int isc_clk_prepare(struct clk_hw *hw)
+ struct isc_clk *isc_clk = to_isc_clk(hw);
+ int ret;
+
+- if (isc_clk->id == ISC_ISPCK) {
+- ret = pm_runtime_resume_and_get(isc_clk->dev);
+- if (ret < 0)
+- return ret;
+- }
++ ret = pm_runtime_resume_and_get(isc_clk->dev);
++ if (ret < 0)
++ return ret;
+
+ return isc_wait_clk_stable(hw);
+ }
+@@ -138,8 +136,7 @@ static void isc_clk_unprepare(struct clk_hw *hw)
+
+ isc_wait_clk_stable(hw);
+
+- if (isc_clk->id == ISC_ISPCK)
+- pm_runtime_put_sync(isc_clk->dev);
++ pm_runtime_put_sync(isc_clk->dev);
+ }
+
+ static int isc_clk_enable(struct clk_hw *hw)
+@@ -186,16 +183,13 @@ static int isc_clk_is_enabled(struct clk_hw *hw)
+ u32 status;
+ int ret;
+
+- if (isc_clk->id == ISC_ISPCK) {
+- ret = pm_runtime_resume_and_get(isc_clk->dev);
+- if (ret < 0)
+- return 0;
+- }
++ ret = pm_runtime_resume_and_get(isc_clk->dev);
++ if (ret < 0)
++ return 0;
+
+ regmap_read(isc_clk->regmap, ISC_CLKSR, &status);
+
+- if (isc_clk->id == ISC_ISPCK)
+- pm_runtime_put_sync(isc_clk->dev);
++ pm_runtime_put_sync(isc_clk->dev);
+
+ return status & ISC_CLK(isc_clk->id) ? 1 : 0;
+ }
+@@ -325,6 +319,9 @@ static int isc_clk_register(struct isc_device *isc, unsigned int id)
+ const char *parent_names[3];
+ int num_parents;
+
++ if (id == ISC_ISPCK && !isc->ispck_required)
++ return 0;
++
+ num_parents = of_clk_get_parent_count(np);
+ if (num_parents < 1 || num_parents > 3)
+ return -EINVAL;
+@@ -1372,14 +1369,12 @@ static int isc_enum_framesizes(struct file *file, void *fh,
+ struct v4l2_frmsizeenum *fsize)
+ {
+ struct isc_device *isc = video_drvdata(file);
+- struct v4l2_subdev_frame_size_enum fse = {
+- .code = isc->config.sd_format->mbus_code,
+- .index = fsize->index,
+- .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+- };
+ int ret = -EINVAL;
+ int i;
+
++ if (fsize->index)
++ return -EINVAL;
++
+ for (i = 0; i < isc->num_user_formats; i++)
+ if (isc->user_formats[i]->fourcc == fsize->pixel_format)
+ ret = 0;
+@@ -1391,14 +1386,14 @@ static int isc_enum_framesizes(struct file *file, void *fh,
+ if (ret)
+ return ret;
+
+- ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
+- NULL, &fse);
+- if (ret)
+- return ret;
++ fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
+
+- fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+- fsize->discrete.width = fse.max_width;
+- fsize->discrete.height = fse.max_height;
++ fsize->stepwise.min_width = 16;
++ fsize->stepwise.max_width = isc->max_width;
++ fsize->stepwise.min_height = 16;
++ fsize->stepwise.max_height = isc->max_height;
++ fsize->stepwise.step_width = 1;
++ fsize->stepwise.step_height = 1;
+
+ return 0;
+ }
+diff --git a/drivers/media/platform/atmel/atmel-isc.h b/drivers/media/platform/atmel/atmel-isc.h
+index 19cc60dfcbe0f..2bfcb135ef13b 100644
+--- a/drivers/media/platform/atmel/atmel-isc.h
++++ b/drivers/media/platform/atmel/atmel-isc.h
+@@ -178,6 +178,7 @@ struct isc_reg_offsets {
+ * @hclock: Hclock clock input (refer datasheet)
+ * @ispck: iscpck clock (refer datasheet)
+ * @isc_clks: ISC clocks
++ * @ispck_required: ISC requires ISP Clock initialization
+ * @dcfg: DMA master configuration, architecture dependent
+ *
+ * @dev: Registered device driver
+@@ -252,6 +253,7 @@ struct isc_device {
+ struct clk *hclock;
+ struct clk *ispck;
+ struct isc_clk isc_clks[2];
++ bool ispck_required;
+ u32 dcfg;
+
+ struct device *dev;
+diff --git a/drivers/media/platform/atmel/atmel-sama5d2-isc.c b/drivers/media/platform/atmel/atmel-sama5d2-isc.c
+index b66f1d174e9d7..e29a9193bac81 100644
+--- a/drivers/media/platform/atmel/atmel-sama5d2-isc.c
++++ b/drivers/media/platform/atmel/atmel-sama5d2-isc.c
+@@ -454,6 +454,9 @@ static int atmel_isc_probe(struct platform_device *pdev)
+ /* sama5d2-isc - 8 bits per beat */
+ isc->dcfg = ISC_DCFG_YMBSIZE_BEATS8 | ISC_DCFG_CMBSIZE_BEATS8;
+
++ /* sama5d2-isc : ISPCK is required and mandatory */
++ isc->ispck_required = true;
++
+ ret = isc_pipeline_init(isc);
+ if (ret)
+ return ret;
+@@ -476,22 +479,6 @@ static int atmel_isc_probe(struct platform_device *pdev)
+ dev_err(dev, "failed to init isc clock: %d\n", ret);
+ goto unprepare_hclk;
+ }
+-
+- isc->ispck = isc->isc_clks[ISC_ISPCK].clk;
+-
+- ret = clk_prepare_enable(isc->ispck);
+- if (ret) {
+- dev_err(dev, "failed to enable ispck: %d\n", ret);
+- goto unprepare_hclk;
+- }
+-
+- /* ispck should be greater or equal to hclock */
+- ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock));
+- if (ret) {
+- dev_err(dev, "failed to set ispck rate: %d\n", ret);
+- goto unprepare_clk;
+- }
+-
+ ret = v4l2_device_register(dev, &isc->v4l2_dev);
+ if (ret) {
+ dev_err(dev, "unable to register v4l2 device.\n");
+@@ -545,19 +532,35 @@ static int atmel_isc_probe(struct platform_device *pdev)
+ pm_runtime_enable(dev);
+ pm_request_idle(dev);
+
++ isc->ispck = isc->isc_clks[ISC_ISPCK].clk;
++
++ ret = clk_prepare_enable(isc->ispck);
++ if (ret) {
++ dev_err(dev, "failed to enable ispck: %d\n", ret);
++ goto cleanup_subdev;
++ }
++
++ /* ispck should be greater or equal to hclock */
++ ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock));
++ if (ret) {
++ dev_err(dev, "failed to set ispck rate: %d\n", ret);
++ goto unprepare_clk;
++ }
++
+ regmap_read(isc->regmap, ISC_VERSION + isc->offsets.version, &ver);
+ dev_info(dev, "Microchip ISC version %x\n", ver);
+
+ return 0;
+
++unprepare_clk:
++ clk_disable_unprepare(isc->ispck);
++
+ cleanup_subdev:
+ isc_subdev_cleanup(isc);
+
+ unregister_v4l2_device:
+ v4l2_device_unregister(&isc->v4l2_dev);
+
+-unprepare_clk:
+- clk_disable_unprepare(isc->ispck);
+ unprepare_hclk:
+ clk_disable_unprepare(isc->hclock);
+
+diff --git a/drivers/media/platform/atmel/atmel-sama7g5-isc.c b/drivers/media/platform/atmel/atmel-sama7g5-isc.c
+index f2785131ff569..6a5d3f7ce75e5 100644
+--- a/drivers/media/platform/atmel/atmel-sama7g5-isc.c
++++ b/drivers/media/platform/atmel/atmel-sama7g5-isc.c
+@@ -447,6 +447,9 @@ static int microchip_xisc_probe(struct platform_device *pdev)
+ /* sama7g5-isc RAM access port is full AXI4 - 32 bits per beat */
+ isc->dcfg = ISC_DCFG_YMBSIZE_BEATS32 | ISC_DCFG_CMBSIZE_BEATS32;
+
++ /* sama7g5-isc : ISPCK does not exist, ISC is clocked by MCK */
++ isc->ispck_required = false;
++
+ ret = isc_pipeline_init(isc);
+ if (ret)
+ return ret;
+@@ -470,25 +473,10 @@ static int microchip_xisc_probe(struct platform_device *pdev)
+ goto unprepare_hclk;
+ }
+
+- isc->ispck = isc->isc_clks[ISC_ISPCK].clk;
+-
+- ret = clk_prepare_enable(isc->ispck);
+- if (ret) {
+- dev_err(dev, "failed to enable ispck: %d\n", ret);
+- goto unprepare_hclk;
+- }
+-
+- /* ispck should be greater or equal to hclock */
+- ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock));
+- if (ret) {
+- dev_err(dev, "failed to set ispck rate: %d\n", ret);
+- goto unprepare_clk;
+- }
+-
+ ret = v4l2_device_register(dev, &isc->v4l2_dev);
+ if (ret) {
+ dev_err(dev, "unable to register v4l2 device.\n");
+- goto unprepare_clk;
++ goto unprepare_hclk;
+ }
+
+ ret = xisc_parse_dt(dev, isc);
+@@ -549,8 +537,6 @@ cleanup_subdev:
+ unregister_v4l2_device:
+ v4l2_device_unregister(&isc->v4l2_dev);
+
+-unprepare_clk:
+- clk_disable_unprepare(isc->ispck);
+ unprepare_hclk:
+ clk_disable_unprepare(isc->hclock);
+
+@@ -569,7 +555,6 @@ static int microchip_xisc_remove(struct platform_device *pdev)
+
+ v4l2_device_unregister(&isc->v4l2_dev);
+
+- clk_disable_unprepare(isc->ispck);
+ clk_disable_unprepare(isc->hclock);
+
+ isc_clk_cleanup(isc);
+@@ -581,7 +566,6 @@ static int __maybe_unused xisc_runtime_suspend(struct device *dev)
+ {
+ struct isc_device *isc = dev_get_drvdata(dev);
+
+- clk_disable_unprepare(isc->ispck);
+ clk_disable_unprepare(isc->hclock);
+
+ return 0;
+@@ -596,10 +580,6 @@ static int __maybe_unused xisc_runtime_resume(struct device *dev)
+ if (ret)
+ return ret;
+
+- ret = clk_prepare_enable(isc->ispck);
+- if (ret)
+- clk_disable_unprepare(isc->hclock);
+-
+ return ret;
+ }
+
+diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
+index 0e312b0842d7f..4a553f42ff0a3 100644
+--- a/drivers/media/platform/coda/coda-common.c
++++ b/drivers/media/platform/coda/coda-common.c
+@@ -408,6 +408,7 @@ static struct vdoa_data *coda_get_vdoa_data(void)
+ if (!vdoa_data)
+ vdoa_data = ERR_PTR(-EPROBE_DEFER);
+
++ put_device(&vdoa_pdev->dev);
+ out:
+ of_node_put(vdoa_node);
+
+@@ -1537,11 +1538,13 @@ static void coda_pic_run_work(struct work_struct *work)
+
+ if (!wait_for_completion_timeout(&ctx->completion,
+ msecs_to_jiffies(1000))) {
+- dev_err(dev->dev, "CODA PIC_RUN timeout\n");
++ if (ctx->use_bit) {
++ dev_err(dev->dev, "CODA PIC_RUN timeout\n");
+
+- ctx->hold = true;
++ ctx->hold = true;
+
+- coda_hw_reset(ctx);
++ coda_hw_reset(ctx);
++ }
+
+ if (ctx->ops->run_timeout)
+ ctx->ops->run_timeout(ctx);
+diff --git a/drivers/media/platform/coda/coda-jpeg.c b/drivers/media/platform/coda/coda-jpeg.c
+index b11cfbe166dd3..a72f4655e5ad5 100644
+--- a/drivers/media/platform/coda/coda-jpeg.c
++++ b/drivers/media/platform/coda/coda-jpeg.c
+@@ -1127,7 +1127,8 @@ static int coda9_jpeg_prepare_encode(struct coda_ctx *ctx)
+ coda_write(dev, 0, CODA9_REG_JPEG_GBU_BT_PTR);
+ coda_write(dev, 0, CODA9_REG_JPEG_GBU_WD_PTR);
+ coda_write(dev, 0, CODA9_REG_JPEG_GBU_BBSR);
+- coda_write(dev, 0, CODA9_REG_JPEG_BBC_STRM_CTRL);
++ coda_write(dev, BIT(31) | ((end_addr - start_addr - header_len) / 256),
++ CODA9_REG_JPEG_BBC_STRM_CTRL);
+ coda_write(dev, 0, CODA9_REG_JPEG_GBU_CTRL);
+ coda_write(dev, 0, CODA9_REG_JPEG_GBU_FF_RPTR);
+ coda_write(dev, 127, CODA9_REG_JPEG_GBU_BBER);
+@@ -1257,6 +1258,23 @@ static void coda9_jpeg_finish_encode(struct coda_ctx *ctx)
+ coda_hw_reset(ctx);
+ }
+
++static void coda9_jpeg_encode_timeout(struct coda_ctx *ctx)
++{
++ struct coda_dev *dev = ctx->dev;
++ u32 end_addr, wr_ptr;
++
++ /* Handle missing BBC overflow interrupt via timeout */
++ end_addr = coda_read(dev, CODA9_REG_JPEG_BBC_END_ADDR);
++ wr_ptr = coda_read(dev, CODA9_REG_JPEG_BBC_WR_PTR);
++ if (wr_ptr >= end_addr - 256) {
++ v4l2_err(&dev->v4l2_dev, "JPEG too large for capture buffer\n");
++ coda9_jpeg_finish_encode(ctx);
++ return;
++ }
++
++ coda_hw_reset(ctx);
++}
++
+ static void coda9_jpeg_release(struct coda_ctx *ctx)
+ {
+ int i;
+@@ -1276,6 +1294,7 @@ const struct coda_context_ops coda9_jpeg_encode_ops = {
+ .start_streaming = coda9_jpeg_start_encoding,
+ .prepare_run = coda9_jpeg_prepare_encode,
+ .finish_run = coda9_jpeg_finish_encode,
++ .run_timeout = coda9_jpeg_encode_timeout,
+ .release = coda9_jpeg_release,
+ };
+
+diff --git a/drivers/media/platform/coda/imx-vdoa.c b/drivers/media/platform/coda/imx-vdoa.c
+index 8bc0d83718193..dd6e2e320264e 100644
+--- a/drivers/media/platform/coda/imx-vdoa.c
++++ b/drivers/media/platform/coda/imx-vdoa.c
+@@ -287,7 +287,11 @@ static int vdoa_probe(struct platform_device *pdev)
+ struct resource *res;
+ int ret;
+
+- dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
++ ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
++ if (ret) {
++ dev_err(&pdev->dev, "DMA enable failed\n");
++ return ret;
++ }
+
+ vdoa = devm_kzalloc(&pdev->dev, sizeof(*vdoa), GFP_KERNEL);
+ if (!vdoa)
+diff --git a/drivers/media/platform/davinci/vpif.c b/drivers/media/platform/davinci/vpif.c
+index f1ce10828b8e5..5658c7f148d79 100644
+--- a/drivers/media/platform/davinci/vpif.c
++++ b/drivers/media/platform/davinci/vpif.c
+@@ -428,6 +428,7 @@ static int vpif_probe(struct platform_device *pdev)
+ static struct resource *res, *res_irq;
+ struct platform_device *pdev_capture, *pdev_display;
+ struct device_node *endpoint = NULL;
++ int ret;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ vpif_base = devm_ioremap_resource(&pdev->dev, res);
+@@ -457,8 +458,8 @@ static int vpif_probe(struct platform_device *pdev)
+ res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!res_irq) {
+ dev_warn(&pdev->dev, "Missing IRQ resource.\n");
+- pm_runtime_put(&pdev->dev);
+- return -EINVAL;
++ ret = -EINVAL;
++ goto err_put_rpm;
+ }
+
+ pdev_capture = devm_kzalloc(&pdev->dev, sizeof(*pdev_capture),
+@@ -492,10 +493,17 @@ static int vpif_probe(struct platform_device *pdev)
+ }
+
+ return 0;
++
++err_put_rpm:
++ pm_runtime_put(&pdev->dev);
++ pm_runtime_disable(&pdev->dev);
++
++ return ret;
+ }
+
+ static int vpif_remove(struct platform_device *pdev)
+ {
++ pm_runtime_put(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
+ return 0;
+ }
+diff --git a/drivers/media/platform/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/imx-jpeg/mxc-jpeg.c
+index 755138063ee61..37905547466bd 100644
+--- a/drivers/media/platform/imx-jpeg/mxc-jpeg.c
++++ b/drivers/media/platform/imx-jpeg/mxc-jpeg.c
+@@ -575,6 +575,10 @@ static irqreturn_t mxc_jpeg_dec_irq(int irq, void *priv)
+
+ dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+ src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
++ if (!dst_buf || !src_buf) {
++ dev_err(dev, "No source or destination buffer.\n");
++ goto job_unlock;
++ }
+ jpeg_src_buf = vb2_to_mxc_buf(&src_buf->vb2_buf);
+
+ if (dec_ret & SLOT_STATUS_ENC_CONFIG_ERR) {
+@@ -921,8 +925,13 @@ static void mxc_jpeg_device_run(void *priv)
+ v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, true);
+
+ jpeg_src_buf = vb2_to_mxc_buf(&src_buf->vb2_buf);
++ if (q_data_cap->fmt->colplanes != dst_buf->vb2_buf.num_planes) {
++ dev_err(dev, "Capture format %s has %d planes, but capture buffer has %d planes\n",
++ q_data_cap->fmt->name, q_data_cap->fmt->colplanes,
++ dst_buf->vb2_buf.num_planes);
++ jpeg_src_buf->jpeg_parse_error = true;
++ }
+ if (jpeg_src_buf->jpeg_parse_error) {
+- jpeg->slot_data[ctx->slot].used = false;
+ v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+ v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
+@@ -2088,6 +2097,8 @@ err_m2m:
+ v4l2_device_unregister(&jpeg->v4l2_dev);
+
+ err_register:
++ mxc_jpeg_detach_pm_domains(jpeg);
++
+ err_irq:
+ return ret;
+ }
+diff --git a/drivers/media/platform/imx-pxp.c b/drivers/media/platform/imx-pxp.c
+index 4321edc0c23d9..8e9c6fee75a48 100644
+--- a/drivers/media/platform/imx-pxp.c
++++ b/drivers/media/platform/imx-pxp.c
+@@ -1661,6 +1661,8 @@ static int pxp_probe(struct platform_device *pdev)
+ if (irq < 0)
+ return irq;
+
++ spin_lock_init(&dev->irqlock);
++
+ ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, pxp_irq_handler,
+ IRQF_ONESHOT, dev_name(&pdev->dev), dev);
+ if (ret < 0) {
+@@ -1678,8 +1680,6 @@ static int pxp_probe(struct platform_device *pdev)
+ goto err_clk;
+ }
+
+- spin_lock_init(&dev->irqlock);
+-
+ ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
+ if (ret)
+ goto err_clk;
+diff --git a/drivers/media/platform/meson/ge2d/ge2d.c b/drivers/media/platform/meson/ge2d/ge2d.c
+index a1393fefa8aea..a373dea9866bb 100644
+--- a/drivers/media/platform/meson/ge2d/ge2d.c
++++ b/drivers/media/platform/meson/ge2d/ge2d.c
+@@ -215,35 +215,35 @@ static void ge2d_hw_start(struct meson_ge2d *ge2d)
+
+ regmap_write(ge2d->map, GE2D_SRC1_CLIPY_START_END,
+ FIELD_PREP(GE2D_START, ctx->in.crop.top) |
+- FIELD_PREP(GE2D_END, ctx->in.crop.top + ctx->in.crop.height));
++ FIELD_PREP(GE2D_END, ctx->in.crop.top + ctx->in.crop.height - 1));
+ regmap_write(ge2d->map, GE2D_SRC1_CLIPX_START_END,
+ FIELD_PREP(GE2D_START, ctx->in.crop.left) |
+- FIELD_PREP(GE2D_END, ctx->in.crop.left + ctx->in.crop.width));
++ FIELD_PREP(GE2D_END, ctx->in.crop.left + ctx->in.crop.width - 1));
+ regmap_write(ge2d->map, GE2D_SRC2_CLIPY_START_END,
+ FIELD_PREP(GE2D_START, ctx->out.crop.top) |
+- FIELD_PREP(GE2D_END, ctx->out.crop.top + ctx->out.crop.height));
++ FIELD_PREP(GE2D_END, ctx->out.crop.top + ctx->out.crop.height - 1));
+ regmap_write(ge2d->map, GE2D_SRC2_CLIPX_START_END,
+ FIELD_PREP(GE2D_START, ctx->out.crop.left) |
+- FIELD_PREP(GE2D_END, ctx->out.crop.left + ctx->out.crop.width));
++ FIELD_PREP(GE2D_END, ctx->out.crop.left + ctx->out.crop.width - 1));
+ regmap_write(ge2d->map, GE2D_DST_CLIPY_START_END,
+ FIELD_PREP(GE2D_START, ctx->out.crop.top) |
+- FIELD_PREP(GE2D_END, ctx->out.crop.top + ctx->out.crop.height));
++ FIELD_PREP(GE2D_END, ctx->out.crop.top + ctx->out.crop.height - 1));
+ regmap_write(ge2d->map, GE2D_DST_CLIPX_START_END,
+ FIELD_PREP(GE2D_START, ctx->out.crop.left) |
+- FIELD_PREP(GE2D_END, ctx->out.crop.left + ctx->out.crop.width));
++ FIELD_PREP(GE2D_END, ctx->out.crop.left + ctx->out.crop.width - 1));
+
+ regmap_write(ge2d->map, GE2D_SRC1_Y_START_END,
+- FIELD_PREP(GE2D_END, ctx->in.pix_fmt.height));
++ FIELD_PREP(GE2D_END, ctx->in.pix_fmt.height - 1));
+ regmap_write(ge2d->map, GE2D_SRC1_X_START_END,
+- FIELD_PREP(GE2D_END, ctx->in.pix_fmt.width));
++ FIELD_PREP(GE2D_END, ctx->in.pix_fmt.width - 1));
+ regmap_write(ge2d->map, GE2D_SRC2_Y_START_END,
+- FIELD_PREP(GE2D_END, ctx->out.pix_fmt.height));
++ FIELD_PREP(GE2D_END, ctx->out.pix_fmt.height - 1));
+ regmap_write(ge2d->map, GE2D_SRC2_X_START_END,
+- FIELD_PREP(GE2D_END, ctx->out.pix_fmt.width));
++ FIELD_PREP(GE2D_END, ctx->out.pix_fmt.width - 1));
+ regmap_write(ge2d->map, GE2D_DST_Y_START_END,
+- FIELD_PREP(GE2D_END, ctx->out.pix_fmt.height));
++ FIELD_PREP(GE2D_END, ctx->out.pix_fmt.height - 1));
+ regmap_write(ge2d->map, GE2D_DST_X_START_END,
+- FIELD_PREP(GE2D_END, ctx->out.pix_fmt.width));
++ FIELD_PREP(GE2D_END, ctx->out.pix_fmt.width - 1));
+
+ /* Color, no blend, use source color */
+ reg = GE2D_ALU_DO_COLOR_OPERATION_LOGIC(LOGIC_OPERATION_COPY,
+@@ -779,11 +779,7 @@ static int ge2d_s_ctrl(struct v4l2_ctrl *ctrl)
+ * If the rotation parameter changes the OUTPUT frames
+ * parameters, take them in account
+ */
+- if (fmt.width != ctx->out.pix_fmt.width ||
+- fmt.height != ctx->out.pix_fmt.width ||
+- fmt.bytesperline > ctx->out.pix_fmt.bytesperline ||
+- fmt.sizeimage > ctx->out.pix_fmt.sizeimage)
+- ctx->out.pix_fmt = fmt;
++ ctx->out.pix_fmt = fmt;
+
+ break;
+ }
+diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
+index 416f356af363d..d97a6765693f1 100644
+--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
++++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
+@@ -793,7 +793,7 @@ static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count)
+ {
+ struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(q);
+ struct venc_enc_param param;
+- int ret;
++ int ret, pm_ret;
+ int i;
+
+ /* Once state turn into MTK_STATE_ABORT, we need stop_streaming
+@@ -845,9 +845,9 @@ static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count)
+ return 0;
+
+ err_set_param:
+- ret = pm_runtime_put(&ctx->dev->plat_dev->dev);
+- if (ret < 0)
+- mtk_v4l2_err("pm_runtime_put fail %d", ret);
++ pm_ret = pm_runtime_put(&ctx->dev->plat_dev->dev);
++ if (pm_ret < 0)
++ mtk_v4l2_err("pm_runtime_put fail %d", pm_ret);
+
+ err_start_stream:
+ for (i = 0; i < q->num_buffers; ++i) {
+diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
+index 45d1870c83dd7..4ced20ca647b1 100644
+--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
++++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
+@@ -218,11 +218,11 @@ static int fops_vcodec_release(struct file *file)
+ mtk_v4l2_debug(1, "[%d] encoder", ctx->id);
+ mutex_lock(&dev->dev_mutex);
+
++ v4l2_m2m_ctx_release(ctx->m2m_ctx);
+ mtk_vcodec_enc_release(ctx);
+ v4l2_fh_del(&ctx->fh);
+ v4l2_fh_exit(&ctx->fh);
+ v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
+- v4l2_m2m_ctx_release(ctx->m2m_ctx);
+
+ list_del_init(&ctx->list);
+ kfree(ctx);
+diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_fw_vpu.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_fw_vpu.c
+index cd27f637dbe7c..cfc7ebed8fb7a 100644
+--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_fw_vpu.c
++++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_fw_vpu.c
+@@ -102,6 +102,8 @@ struct mtk_vcodec_fw *mtk_vcodec_fw_vpu_init(struct mtk_vcodec_dev *dev,
+ vpu_wdt_reg_handler(fw_pdev, mtk_vcodec_vpu_reset_handler, dev, rst_id);
+
+ fw = devm_kzalloc(&dev->plat_dev->dev, sizeof(*fw), GFP_KERNEL);
++ if (!fw)
++ return ERR_PTR(-ENOMEM);
+ fw->type = VPU;
+ fw->ops = &mtk_vcodec_vpu_msg;
+ fw->pdev = fw_pdev;
+diff --git a/drivers/media/platform/mtk-vpu/mtk_vpu.c b/drivers/media/platform/mtk-vpu/mtk_vpu.c
+index ec290dde59cfd..7f1647da0ade0 100644
+--- a/drivers/media/platform/mtk-vpu/mtk_vpu.c
++++ b/drivers/media/platform/mtk-vpu/mtk_vpu.c
+@@ -848,7 +848,8 @@ static int mtk_vpu_probe(struct platform_device *pdev)
+ vpu->wdt.wq = create_singlethread_workqueue("vpu_wdt");
+ if (!vpu->wdt.wq) {
+ dev_err(dev, "initialize wdt workqueue failed\n");
+- return -ENOMEM;
++ ret = -ENOMEM;
++ goto clk_unprepare;
+ }
+ INIT_WORK(&vpu->wdt.ws, vpu_wdt_reset_func);
+ mutex_init(&vpu->vpu_mutex);
+@@ -942,6 +943,8 @@ disable_vpu_clk:
+ vpu_clock_disable(vpu);
+ workqueue_destroy:
+ destroy_workqueue(vpu->wdt.wq);
++clk_unprepare:
++ clk_unprepare(vpu->clk);
+
+ return ret;
+ }
+diff --git a/drivers/media/platform/qcom/camss/camss-csid-170.c b/drivers/media/platform/qcom/camss/camss-csid-170.c
+index ac22ff29d2a9f..82f59933ad7b3 100644
+--- a/drivers/media/platform/qcom/camss/camss-csid-170.c
++++ b/drivers/media/platform/qcom/camss/camss-csid-170.c
+@@ -105,7 +105,8 @@
+ #define CSID_RDI_CTRL(rdi) ((IS_LITE ? 0x208 : 0x308)\
+ + 0x100 * (rdi))
+ #define RDI_CTRL_HALT_CMD 0
+-#define ALT_CMD_RESUME_AT_FRAME_BOUNDARY 1
++#define HALT_CMD_HALT_AT_FRAME_BOUNDARY 0
++#define HALT_CMD_RESUME_AT_FRAME_BOUNDARY 1
+ #define RDI_CTRL_HALT_MODE 2
+
+ #define CSID_RDI_FRM_DROP_PATTERN(rdi) ((IS_LITE ? 0x20C : 0x30C)\
+@@ -366,7 +367,7 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
+ val |= input_format->width & 0x1fff << TPG_DT_n_CFG_0_FRAME_WIDTH;
+ writel_relaxed(val, csid->base + CSID_TPG_DT_n_CFG_0(0));
+
+- val = DATA_TYPE_RAW_10BIT << TPG_DT_n_CFG_1_DATA_TYPE;
++ val = format->data_type << TPG_DT_n_CFG_1_DATA_TYPE;
+ writel_relaxed(val, csid->base + CSID_TPG_DT_n_CFG_1(0));
+
+ val = tg->mode << TPG_DT_n_CFG_2_PAYLOAD_MODE;
+@@ -382,8 +383,9 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
+ val = 1 << RDI_CFG0_BYTE_CNTR_EN;
+ val |= 1 << RDI_CFG0_FORMAT_MEASURE_EN;
+ val |= 1 << RDI_CFG0_TIMESTAMP_EN;
++ /* note: for non-RDI path, this should be format->decode_format */
+ val |= DECODE_FORMAT_PAYLOAD_ONLY << RDI_CFG0_DECODE_FORMAT;
+- val |= DATA_TYPE_RAW_10BIT << RDI_CFG0_DATA_TYPE;
++ val |= format->data_type << RDI_CFG0_DATA_TYPE;
+ val |= vc << RDI_CFG0_VIRTUAL_CHANNEL;
+ val |= dt_id << RDI_CFG0_DT_ID;
+ writel_relaxed(val, csid->base + CSID_RDI_CFG0(0));
+@@ -443,13 +445,10 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
+ val |= 1 << CSI2_RX_CFG1_MISR_EN;
+ writel_relaxed(val, csid->base + CSID_CSI2_RX_CFG1); // csi2_vc_mode_shift_val ?
+
+- /* error irqs start at BIT(11) */
+- writel_relaxed(~0u, csid->base + CSID_CSI2_RX_IRQ_MASK);
+-
+- /* RDI irq */
+- writel_relaxed(~0u, csid->base + CSID_TOP_IRQ_MASK);
+-
+- val = 1 << RDI_CTRL_HALT_CMD;
++ if (enable)
++ val = HALT_CMD_RESUME_AT_FRAME_BOUNDARY << RDI_CTRL_HALT_CMD;
++ else
++ val = HALT_CMD_HALT_AT_FRAME_BOUNDARY << RDI_CTRL_HALT_CMD;
+ writel_relaxed(val, csid->base + CSID_RDI_CTRL(0));
+ }
+
+diff --git a/drivers/media/platform/qcom/camss/camss-vfe-170.c b/drivers/media/platform/qcom/camss/camss-vfe-170.c
+index 8594d275b41d1..02cb8005504a2 100644
+--- a/drivers/media/platform/qcom/camss/camss-vfe-170.c
++++ b/drivers/media/platform/qcom/camss/camss-vfe-170.c
+@@ -399,17 +399,7 @@ static irqreturn_t vfe_isr(int irq, void *dev)
+ */
+ static int vfe_halt(struct vfe_device *vfe)
+ {
+- unsigned long time;
+-
+- reinit_completion(&vfe->halt_complete);
+-
+- time = wait_for_completion_timeout(&vfe->halt_complete,
+- msecs_to_jiffies(VFE_HALT_TIMEOUT_MS));
+- if (!time) {
+- dev_err(vfe->camss->dev, "VFE halt timeout\n");
+- return -EIO;
+- }
+-
++ /* rely on vfe_disable_output() to stop the VFE */
+ return 0;
+ }
+
+diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c
+index 91b15842c5558..1f0181b6353c9 100644
+--- a/drivers/media/platform/qcom/venus/core.c
++++ b/drivers/media/platform/qcom/venus/core.c
+@@ -349,11 +349,11 @@ static int venus_probe(struct platform_device *pdev)
+
+ ret = venus_firmware_init(core);
+ if (ret)
+- goto err_runtime_disable;
++ goto err_of_depopulate;
+
+ ret = venus_boot(core);
+ if (ret)
+- goto err_runtime_disable;
++ goto err_firmware_deinit;
+
+ ret = hfi_core_resume(core, true);
+ if (ret)
+@@ -385,6 +385,10 @@ err_dev_unregister:
+ v4l2_device_unregister(&core->v4l2_dev);
+ err_venus_shutdown:
+ venus_shutdown(core);
++err_firmware_deinit:
++ venus_firmware_deinit(core);
++err_of_depopulate:
++ of_platform_depopulate(dev);
+ err_runtime_disable:
+ pm_runtime_put_noidle(dev);
+ pm_runtime_set_suspended(dev);
+@@ -472,7 +476,8 @@ static __maybe_unused int venus_runtime_suspend(struct device *dev)
+ err_video_path:
+ icc_set_bw(core->cpucfg_path, kbps_to_icc(1000), 0);
+ err_cpucfg_path:
+- pm_ops->core_power(core, POWER_ON);
++ if (pm_ops->core_power)
++ pm_ops->core_power(core, POWER_ON);
+
+ return ret;
+ }
+diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.c b/drivers/media/platform/qcom/venus/hfi_cmds.c
+index 60f4b8e4b8d02..1bf5db7673ebf 100644
+--- a/drivers/media/platform/qcom/venus/hfi_cmds.c
++++ b/drivers/media/platform/qcom/venus/hfi_cmds.c
+@@ -1054,6 +1054,8 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
+ pkt->shdr.hdr.size += sizeof(u32) + sizeof(*info);
+ break;
+ }
++ case HFI_PROPERTY_PARAM_VENC_HDR10_PQ_SEI:
++ return -ENOTSUPP;
+
+ /* FOLLOWING PROPERTIES ARE NOT IMPLEMENTED IN CORE YET */
+ case HFI_PROPERTY_CONFIG_BUFFER_REQUIREMENTS:
+diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c
+index 3e2345eb47f7c..a591dd315ebcc 100644
+--- a/drivers/media/platform/qcom/venus/pm_helpers.c
++++ b/drivers/media/platform/qcom/venus/pm_helpers.c
+@@ -163,14 +163,12 @@ static u32 load_per_type(struct venus_core *core, u32 session_type)
+ struct venus_inst *inst = NULL;
+ u32 mbs_per_sec = 0;
+
+- mutex_lock(&core->lock);
+ list_for_each_entry(inst, &core->instances, list) {
+ if (inst->session_type != session_type)
+ continue;
+
+ mbs_per_sec += load_per_instance(inst);
+ }
+- mutex_unlock(&core->lock);
+
+ return mbs_per_sec;
+ }
+@@ -219,14 +217,12 @@ static int load_scale_bw(struct venus_core *core)
+ struct venus_inst *inst = NULL;
+ u32 mbs_per_sec, avg, peak, total_avg = 0, total_peak = 0;
+
+- mutex_lock(&core->lock);
+ list_for_each_entry(inst, &core->instances, list) {
+ mbs_per_sec = load_per_instance(inst);
+ mbs_to_bw(inst, mbs_per_sec, &avg, &peak);
+ total_avg += avg;
+ total_peak += peak;
+ }
+- mutex_unlock(&core->lock);
+
+ /*
+ * keep minimum bandwidth vote for "video-mem" path,
+@@ -253,8 +249,9 @@ static int load_scale_v1(struct venus_inst *inst)
+ struct device *dev = core->dev;
+ u32 mbs_per_sec;
+ unsigned int i;
+- int ret;
++ int ret = 0;
+
++ mutex_lock(&core->lock);
+ mbs_per_sec = load_per_type(core, VIDC_SESSION_TYPE_ENC) +
+ load_per_type(core, VIDC_SESSION_TYPE_DEC);
+
+@@ -279,17 +276,19 @@ set_freq:
+ if (ret) {
+ dev_err(dev, "failed to set clock rate %lu (%d)\n",
+ freq, ret);
+- return ret;
++ goto exit;
+ }
+
+ ret = load_scale_bw(core);
+ if (ret) {
+ dev_err(dev, "failed to set bandwidth (%d)\n",
+ ret);
+- return ret;
++ goto exit;
+ }
+
+- return 0;
++exit:
++ mutex_unlock(&core->lock);
++ return ret;
+ }
+
+ static int core_get_v1(struct venus_core *core)
+@@ -587,8 +586,8 @@ min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load, bool lo
+ if (inst->session_type == VIDC_SESSION_TYPE_DEC)
+ vpp_freq = inst_pos->clk_data.vpp_freq;
+ else if (inst->session_type == VIDC_SESSION_TYPE_ENC)
+- vpp_freq = low_power ? inst_pos->clk_data.vpp_freq :
+- inst_pos->clk_data.low_power_freq;
++ vpp_freq = low_power ? inst_pos->clk_data.low_power_freq :
++ inst_pos->clk_data.vpp_freq;
+ else
+ continue;
+
+@@ -1085,12 +1084,16 @@ static unsigned long calculate_inst_freq(struct venus_inst *inst,
+ if (inst->state != INST_START)
+ return 0;
+
+- if (inst->session_type == VIDC_SESSION_TYPE_ENC)
++ if (inst->session_type == VIDC_SESSION_TYPE_ENC) {
+ vpp_freq_per_mb = inst->flags & VENUS_LOW_POWER ?
+ inst->clk_data.low_power_freq :
+ inst->clk_data.vpp_freq;
+
+- vpp_freq = mbs_per_sec * vpp_freq_per_mb;
++ vpp_freq = mbs_per_sec * vpp_freq_per_mb;
++ } else {
++ vpp_freq = mbs_per_sec * inst->clk_data.vpp_freq;
++ }
++
+ /* 21 / 20 is overhead factor */
+ vpp_freq += vpp_freq / 20;
+ vsp_freq = mbs_per_sec * inst->clk_data.vsp_freq;
+@@ -1112,13 +1115,13 @@ static int load_scale_v4(struct venus_inst *inst)
+ struct device *dev = core->dev;
+ unsigned long freq = 0, freq_core1 = 0, freq_core2 = 0;
+ unsigned long filled_len = 0;
+- int i, ret;
++ int i, ret = 0;
+
+ for (i = 0; i < inst->num_input_bufs; i++)
+ filled_len = max(filled_len, inst->payloads[i]);
+
+ if (inst->session_type == VIDC_SESSION_TYPE_DEC && !filled_len)
+- return 0;
++ return ret;
+
+ freq = calculate_inst_freq(inst, filled_len);
+ inst->clk_data.freq = freq;
+@@ -1134,7 +1137,6 @@ static int load_scale_v4(struct venus_inst *inst)
+ freq_core2 += inst->clk_data.freq;
+ }
+ }
+- mutex_unlock(&core->lock);
+
+ freq = max(freq_core1, freq_core2);
+
+@@ -1158,17 +1160,19 @@ set_freq:
+ if (ret) {
+ dev_err(dev, "failed to set clock rate %lu (%d)\n",
+ freq, ret);
+- return ret;
++ goto exit;
+ }
+
+ ret = load_scale_bw(core);
+ if (ret) {
+ dev_err(dev, "failed to set bandwidth (%d)\n",
+ ret);
+- return ret;
++ goto exit;
+ }
+
+- return 0;
++exit:
++ mutex_unlock(&core->lock);
++ return ret;
+ }
+
+ static const struct venus_pm_ops pm_ops_v4 = {
+diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c
+index bc1c42dd53c04..c4e0fe14c058d 100644
+--- a/drivers/media/platform/qcom/venus/venc.c
++++ b/drivers/media/platform/qcom/venus/venc.c
+@@ -604,8 +604,8 @@ static int venc_set_properties(struct venus_inst *inst)
+
+ ptype = HFI_PROPERTY_PARAM_VENC_H264_TRANSFORM_8X8;
+ h264_transform.enable_type = 0;
+- if (ctr->profile.h264 == HFI_H264_PROFILE_HIGH ||
+- ctr->profile.h264 == HFI_H264_PROFILE_CONSTRAINED_HIGH)
++ if (ctr->profile.h264 == V4L2_MPEG_VIDEO_H264_PROFILE_HIGH ||
++ ctr->profile.h264 == V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH)
+ h264_transform.enable_type = ctr->h264_8x8_transform;
+
+ ret = hfi_session_set_property(inst, ptype, &h264_transform);
+diff --git a/drivers/media/platform/qcom/venus/venc_ctrls.c b/drivers/media/platform/qcom/venus/venc_ctrls.c
+index 1ada42df314dc..ea5805e71c143 100644
+--- a/drivers/media/platform/qcom/venus/venc_ctrls.c
++++ b/drivers/media/platform/qcom/venus/venc_ctrls.c
+@@ -320,8 +320,8 @@ static int venc_op_s_ctrl(struct v4l2_ctrl *ctrl)
+ ctr->intra_refresh_period = ctrl->val;
+ break;
+ case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM:
+- if (ctr->profile.h264 != HFI_H264_PROFILE_HIGH &&
+- ctr->profile.h264 != HFI_H264_PROFILE_CONSTRAINED_HIGH)
++ if (ctr->profile.h264 != V4L2_MPEG_VIDEO_H264_PROFILE_HIGH &&
++ ctr->profile.h264 != V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH)
+ return -EINVAL;
+
+ /*
+@@ -457,7 +457,7 @@ int venc_ctrl_init(struct venus_inst *inst)
+ V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP, 1, 51, 1, 1);
+
+ v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops,
+- V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM, 0, 1, 1, 0);
++ V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM, 0, 1, 1, 1);
+
+ v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP, 1, 51, 1, 1);
+diff --git a/drivers/media/platform/rcar-vin/rcar-csi2.c b/drivers/media/platform/rcar-vin/rcar-csi2.c
+index e28eff0396888..0c5e2f7e04beb 100644
+--- a/drivers/media/platform/rcar-vin/rcar-csi2.c
++++ b/drivers/media/platform/rcar-vin/rcar-csi2.c
+@@ -445,16 +445,23 @@ static int rcsi2_wait_phy_start(struct rcar_csi2 *priv,
+ static int rcsi2_set_phypll(struct rcar_csi2 *priv, unsigned int mbps)
+ {
+ const struct rcsi2_mbps_reg *hsfreq;
++ const struct rcsi2_mbps_reg *hsfreq_prev = NULL;
+
+- for (hsfreq = priv->info->hsfreqrange; hsfreq->mbps != 0; hsfreq++)
++ for (hsfreq = priv->info->hsfreqrange; hsfreq->mbps != 0; hsfreq++) {
+ if (hsfreq->mbps >= mbps)
+ break;
++ hsfreq_prev = hsfreq;
++ }
+
+ if (!hsfreq->mbps) {
+ dev_err(priv->dev, "Unsupported PHY speed (%u Mbps)", mbps);
+ return -ERANGE;
+ }
+
++ if (hsfreq_prev &&
++ ((mbps - hsfreq_prev->mbps) <= (hsfreq->mbps - mbps)))
++ hsfreq = hsfreq_prev;
++
+ rcsi2_write(priv, PHYPLL_REG, PHYPLL_HSFREQRANGE(hsfreq->reg));
+
+ return 0;
+@@ -553,6 +560,8 @@ static int rcsi2_start_receiver(struct rcar_csi2 *priv)
+
+ /* Code is validated in set_fmt. */
+ format = rcsi2_code_to_fmt(priv->mf.code);
++ if (!format)
++ return -EINVAL;
+
+ /*
+ * Enable all supported CSI-2 channels with virtual channel and
+@@ -980,10 +989,17 @@ static int rcsi2_phtw_write_mbps(struct rcar_csi2 *priv, unsigned int mbps,
+ const struct rcsi2_mbps_reg *values, u16 code)
+ {
+ const struct rcsi2_mbps_reg *value;
++ const struct rcsi2_mbps_reg *prev_value = NULL;
+
+- for (value = values; value->mbps; value++)
++ for (value = values; value->mbps; value++) {
+ if (value->mbps >= mbps)
+ break;
++ prev_value = value;
++ }
++
++ if (prev_value &&
++ ((mbps - prev_value->mbps) <= (value->mbps - mbps)))
++ value = prev_value;
+
+ if (!value->mbps) {
+ dev_err(priv->dev, "Unsupported PHY speed (%u Mbps)", mbps);
+diff --git a/drivers/media/platform/rcar-vin/rcar-dma.c b/drivers/media/platform/rcar-vin/rcar-dma.c
+index f5f722ab1d4e8..520d044bfb8d5 100644
+--- a/drivers/media/platform/rcar-vin/rcar-dma.c
++++ b/drivers/media/platform/rcar-vin/rcar-dma.c
+@@ -904,7 +904,8 @@ static void rvin_fill_hw_slot(struct rvin_dev *vin, int slot)
+ vin->format.sizeimage / 2;
+ break;
+ }
+- } else if (vin->state != RUNNING || list_empty(&vin->buf_list)) {
++ } else if ((vin->state != STOPPED && vin->state != RUNNING) ||
++ list_empty(&vin->buf_list)) {
+ vin->buf_hw[slot].buffer = NULL;
+ vin->buf_hw[slot].type = FULL;
+ phys_addr = vin->scratch_phys;
+diff --git a/drivers/media/platform/rcar-vin/rcar-v4l2.c b/drivers/media/platform/rcar-vin/rcar-v4l2.c
+index 0d141155f0e3e..eb8c79bac540f 100644
+--- a/drivers/media/platform/rcar-vin/rcar-v4l2.c
++++ b/drivers/media/platform/rcar-vin/rcar-v4l2.c
+@@ -175,20 +175,27 @@ static void rvin_format_align(struct rvin_dev *vin, struct v4l2_pix_format *pix)
+ break;
+ }
+
+- /* HW limit width to a multiple of 32 (2^5) for NV12/16 else 2 (2^1) */
++ /* Hardware limits width alignment based on format. */
+ switch (pix->pixelformat) {
++ /* Multiple of 32 (2^5) for NV12/16. */
+ case V4L2_PIX_FMT_NV12:
+ case V4L2_PIX_FMT_NV16:
+ walign = 5;
+ break;
+- default:
++ /* Multiple of 2 (2^1) for YUV. */
++ case V4L2_PIX_FMT_YUYV:
++ case V4L2_PIX_FMT_UYVY:
+ walign = 1;
+ break;
++ /* No multiple for RGB. */
++ default:
++ walign = 0;
++ break;
+ }
+
+ /* Limit to VIN capabilities */
+- v4l_bound_align_image(&pix->width, 2, vin->info->max_width, walign,
+- &pix->height, 4, vin->info->max_height, 2, 0);
++ v4l_bound_align_image(&pix->width, 5, vin->info->max_width, walign,
++ &pix->height, 2, vin->info->max_height, 0, 0);
+
+ pix->bytesperline = rvin_format_bytesperline(vin, pix);
+ pix->sizeimage = rvin_format_sizeimage(pix);
+diff --git a/drivers/media/platform/rockchip/rga/rga.c b/drivers/media/platform/rockchip/rga/rga.c
+index 6759091b15e09..d99ea8973b678 100644
+--- a/drivers/media/platform/rockchip/rga/rga.c
++++ b/drivers/media/platform/rockchip/rga/rga.c
+@@ -895,7 +895,7 @@ static int rga_probe(struct platform_device *pdev)
+ }
+ rga->dst_mmu_pages =
+ (unsigned int *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 3);
+- if (rga->dst_mmu_pages) {
++ if (!rga->dst_mmu_pages) {
+ ret = -ENOMEM;
+ goto free_src_pages;
+ }
+diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
+index 7474150b94ed3..560f928c37520 100644
+--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
++++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
+@@ -426,7 +426,7 @@ static void rkisp1_debug_init(struct rkisp1_device *rkisp1)
+ {
+ struct rkisp1_debug *debug = &rkisp1->debug;
+
+- debug->debugfs_dir = debugfs_create_dir(RKISP1_DRIVER_NAME, NULL);
++ debug->debugfs_dir = debugfs_create_dir(dev_name(rkisp1->dev), NULL);
+ debugfs_create_ulong("data_loss", 0444, debug->debugfs_dir,
+ &debug->data_loss);
+ debugfs_create_ulong("outform_size_err", 0444, debug->debugfs_dir,
+diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c
+index eba2b9f040df0..f336a95432732 100644
+--- a/drivers/media/platform/s5p-mfc/s5p_mfc.c
++++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c
+@@ -1283,11 +1283,15 @@ static int s5p_mfc_probe(struct platform_device *pdev)
+ spin_lock_init(&dev->condlock);
+ dev->plat_dev = pdev;
+ if (!dev->plat_dev) {
+- dev_err(&pdev->dev, "No platform data specified\n");
++ mfc_err("No platform data specified\n");
+ return -ENODEV;
+ }
+
+ dev->variant = of_device_get_match_data(&pdev->dev);
++ if (!dev->variant) {
++ dev_err(&pdev->dev, "Failed to get device MFC hardware variant information\n");
++ return -ENOENT;
++ }
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
+diff --git a/drivers/media/platform/stm32/stm32-dcmi.c b/drivers/media/platform/stm32/stm32-dcmi.c
+index d914ccef98317..6110718645a4f 100644
+--- a/drivers/media/platform/stm32/stm32-dcmi.c
++++ b/drivers/media/platform/stm32/stm32-dcmi.c
+@@ -128,6 +128,7 @@ struct stm32_dcmi {
+ int sequence;
+ struct list_head buffers;
+ struct dcmi_buf *active;
++ int irq;
+
+ struct v4l2_device v4l2_dev;
+ struct video_device *vdev;
+@@ -1759,6 +1760,14 @@ static int dcmi_graph_notify_complete(struct v4l2_async_notifier *notifier)
+ return ret;
+ }
+
++ ret = devm_request_threaded_irq(dcmi->dev, dcmi->irq, dcmi_irq_callback,
++ dcmi_irq_thread, IRQF_ONESHOT,
++ dev_name(dcmi->dev), dcmi);
++ if (ret) {
++ dev_err(dcmi->dev, "Unable to request irq %d\n", dcmi->irq);
++ return ret;
++ }
++
+ return 0;
+ }
+
+@@ -1914,6 +1923,8 @@ static int dcmi_probe(struct platform_device *pdev)
+ if (irq <= 0)
+ return irq ? irq : -ENXIO;
+
++ dcmi->irq = irq;
++
+ dcmi->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!dcmi->res) {
+ dev_err(&pdev->dev, "Could not get resource\n");
+@@ -1926,14 +1937,6 @@ static int dcmi_probe(struct platform_device *pdev)
+ return PTR_ERR(dcmi->regs);
+ }
+
+- ret = devm_request_threaded_irq(&pdev->dev, irq, dcmi_irq_callback,
+- dcmi_irq_thread, IRQF_ONESHOT,
+- dev_name(&pdev->dev), dcmi);
+- if (ret) {
+- dev_err(&pdev->dev, "Unable to request irq %d\n", irq);
+- return ret;
+- }
+-
+ mclk = devm_clk_get(&pdev->dev, "mclk");
+ if (IS_ERR(mclk)) {
+ if (PTR_ERR(mclk) != -EPROBE_DEFER)
+diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
+index 07b2161392d21..5ba3e29f794fd 100644
+--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
++++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
+@@ -467,7 +467,7 @@ static const struct v4l2_ioctl_ops sun6i_video_ioctl_ops = {
+ static int sun6i_video_open(struct file *file)
+ {
+ struct sun6i_video *video = video_drvdata(file);
+- int ret;
++ int ret = 0;
+
+ if (mutex_lock_interruptible(&video->lock))
+ return -ERESTARTSYS;
+@@ -481,10 +481,8 @@ static int sun6i_video_open(struct file *file)
+ goto fh_release;
+
+ /* check if already powered */
+- if (!v4l2_fh_is_singular_file(file)) {
+- ret = -EBUSY;
++ if (!v4l2_fh_is_singular_file(file))
+ goto unlock;
+- }
+
+ ret = sun6i_csi_set_power(video->csi, true);
+ if (ret < 0)
+diff --git a/drivers/media/platform/ti-vpe/cal-video.c b/drivers/media/platform/ti-vpe/cal-video.c
+index 7799da1cc261b..3e936a2ca36c6 100644
+--- a/drivers/media/platform/ti-vpe/cal-video.c
++++ b/drivers/media/platform/ti-vpe/cal-video.c
+@@ -823,6 +823,9 @@ static int cal_ctx_v4l2_init_formats(struct cal_ctx *ctx)
+ /* Enumerate sub device formats and enable all matching local formats */
+ ctx->active_fmt = devm_kcalloc(ctx->cal->dev, cal_num_formats,
+ sizeof(*ctx->active_fmt), GFP_KERNEL);
++ if (!ctx->active_fmt)
++ return -ENOMEM;
++
+ ctx->num_active_fmt = 0;
+
+ for (j = 0, i = 0; ; ++j) {
+diff --git a/drivers/media/radio/radio-wl1273.c b/drivers/media/radio/radio-wl1273.c
+index 1123768731676..484046471c03f 100644
+--- a/drivers/media/radio/radio-wl1273.c
++++ b/drivers/media/radio/radio-wl1273.c
+@@ -1279,7 +1279,7 @@ static int wl1273_fm_vidioc_querycap(struct file *file, void *priv,
+
+ strscpy(capability->driver, WL1273_FM_DRIVER_NAME,
+ sizeof(capability->driver));
+- strscpy(capability->card, "Texas Instruments Wl1273 FM Radio",
++ strscpy(capability->card, "TI Wl1273 FM Radio",
+ sizeof(capability->card));
+ strscpy(capability->bus_info, radio->bus_type,
+ sizeof(capability->bus_info));
+diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c
+index f491420d7b538..76d39e2e87706 100644
+--- a/drivers/media/radio/si470x/radio-si470x-i2c.c
++++ b/drivers/media/radio/si470x/radio-si470x-i2c.c
+@@ -11,7 +11,7 @@
+
+ /* driver definitions */
+ #define DRIVER_AUTHOR "Joonyoung Shim <jy0922.shim@samsung.com>";
+-#define DRIVER_CARD "Silicon Labs Si470x FM Radio Receiver"
++#define DRIVER_CARD "Silicon Labs Si470x FM Radio"
+ #define DRIVER_DESC "I2C radio driver for Si470x FM Radio Receivers"
+ #define DRIVER_VERSION "1.0.2"
+
+@@ -368,7 +368,7 @@ static int si470x_i2c_probe(struct i2c_client *client)
+ if (radio->hdl.error) {
+ retval = radio->hdl.error;
+ dev_err(&client->dev, "couldn't register control\n");
+- goto err_dev;
++ goto err_all;
+ }
+
+ /* video device initialization */
+@@ -463,7 +463,6 @@ static int si470x_i2c_probe(struct i2c_client *client)
+ return 0;
+ err_all:
+ v4l2_ctrl_handler_free(&radio->hdl);
+-err_dev:
+ v4l2_device_unregister(&radio->v4l2_dev);
+ err_initial:
+ return retval;
+diff --git a/drivers/media/radio/si470x/radio-si470x-usb.c b/drivers/media/radio/si470x/radio-si470x-usb.c
+index fedff68d8c496..3f8634a465730 100644
+--- a/drivers/media/radio/si470x/radio-si470x-usb.c
++++ b/drivers/media/radio/si470x/radio-si470x-usb.c
+@@ -16,7 +16,7 @@
+
+ /* driver definitions */
+ #define DRIVER_AUTHOR "Tobias Lorenz <tobias.lorenz@gmx.net>"
+-#define DRIVER_CARD "Silicon Labs Si470x FM Radio Receiver"
++#define DRIVER_CARD "Silicon Labs Si470x FM Radio"
+ #define DRIVER_DESC "USB radio driver for Si470x FM Radio Receivers"
+ #define DRIVER_VERSION "1.0.10"
+
+diff --git a/drivers/media/rc/gpio-ir-tx.c b/drivers/media/rc/gpio-ir-tx.c
+index c6cd2e6d8e654..a50701cfbbd7b 100644
+--- a/drivers/media/rc/gpio-ir-tx.c
++++ b/drivers/media/rc/gpio-ir-tx.c
+@@ -48,11 +48,29 @@ static int gpio_ir_tx_set_carrier(struct rc_dev *dev, u32 carrier)
+ return 0;
+ }
+
++static void delay_until(ktime_t until)
++{
++ /*
++ * delta should never exceed 0.5 seconds (IR_MAX_DURATION) and on
++ * m68k ndelay(s64) does not compile; so use s32 rather than s64.
++ */
++ s32 delta;
++
++ while (true) {
++ delta = ktime_us_delta(until, ktime_get());
++ if (delta <= 0)
++ return;
++
++ /* udelay more than 1ms may not work */
++ delta = min(delta, 1000);
++ udelay(delta);
++ }
++}
++
+ static void gpio_ir_tx_unmodulated(struct gpio_ir *gpio_ir, uint *txbuf,
+ uint count)
+ {
+ ktime_t edge;
+- s32 delta;
+ int i;
+
+ local_irq_disable();
+@@ -63,9 +81,7 @@ static void gpio_ir_tx_unmodulated(struct gpio_ir *gpio_ir, uint *txbuf,
+ gpiod_set_value(gpio_ir->gpio, !(i % 2));
+
+ edge = ktime_add_us(edge, txbuf[i]);
+- delta = ktime_us_delta(edge, ktime_get());
+- if (delta > 0)
+- udelay(delta);
++ delay_until(edge);
+ }
+
+ gpiod_set_value(gpio_ir->gpio, 0);
+@@ -97,9 +113,7 @@ static void gpio_ir_tx_modulated(struct gpio_ir *gpio_ir, uint *txbuf,
+ if (i % 2) {
+ // space
+ edge = ktime_add_us(edge, txbuf[i]);
+- delta = ktime_us_delta(edge, ktime_get());
+- if (delta > 0)
+- udelay(delta);
++ delay_until(edge);
+ } else {
+ // pulse
+ ktime_t last = ktime_add_us(edge, txbuf[i]);
+diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c
+index effaa5751d6c9..3e9988ee785f0 100644
+--- a/drivers/media/rc/igorplugusb.c
++++ b/drivers/media/rc/igorplugusb.c
+@@ -64,9 +64,11 @@ static void igorplugusb_irdata(struct igorplugusb *ir, unsigned len)
+ if (start >= len) {
+ dev_err(ir->dev, "receive overflow invalid: %u", overflow);
+ } else {
+- if (overflow > 0)
++ if (overflow > 0) {
+ dev_warn(ir->dev, "receive overflow, at least %u lost",
+ overflow);
++ ir_raw_event_reset(ir->rc);
++ }
+
+ do {
+ rawir.duration = ir->buf_in[i] * 85;
+diff --git a/drivers/media/rc/ir_toy.c b/drivers/media/rc/ir_toy.c
+index 48d52baec1a1c..7f394277478b3 100644
+--- a/drivers/media/rc/ir_toy.c
++++ b/drivers/media/rc/ir_toy.c
+@@ -310,7 +310,7 @@ static int irtoy_tx(struct rc_dev *rc, uint *txbuf, uint count)
+ buf[i] = cpu_to_be16(v);
+ }
+
+- buf[count] = 0xffff;
++ buf[count] = cpu_to_be16(0xffff);
+
+ irtoy->tx_buf = buf;
+ irtoy->tx_len = size;
+@@ -429,7 +429,7 @@ static int irtoy_probe(struct usb_interface *intf,
+ err = usb_submit_urb(irtoy->urb_in, GFP_KERNEL);
+ if (err != 0) {
+ dev_err(irtoy->dev, "fail to submit in urb: %d\n", err);
+- return err;
++ goto free_rcdev;
+ }
+
+ err = irtoy_setup(irtoy);
+diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
+index 5bc23e8c6d91d..4f77d4ebacdc5 100644
+--- a/drivers/media/rc/ite-cir.c
++++ b/drivers/media/rc/ite-cir.c
+@@ -242,7 +242,7 @@ static irqreturn_t ite_cir_isr(int irq, void *data)
+ }
+
+ /* check for the receive interrupt */
+- if (iflags & ITE_IRQ_RX_FIFO) {
++ if (iflags & (ITE_IRQ_RX_FIFO | ITE_IRQ_RX_FIFO_OVERRUN)) {
+ /* read the FIFO bytes */
+ rx_bytes = dev->params->get_rx_bytes(dev, rx_buf,
+ ITE_RX_FIFO_LEN);
+diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
+index e03dd1f0144f0..5f296f985b07a 100644
+--- a/drivers/media/rc/mceusb.c
++++ b/drivers/media/rc/mceusb.c
+@@ -1386,6 +1386,7 @@ static void mceusb_dev_recv(struct urb *urb)
+ case -ECONNRESET:
+ case -ENOENT:
+ case -EILSEQ:
++ case -EPROTO:
+ case -ESHUTDOWN:
+ usb_unlink_urb(urb);
+ return;
+@@ -1429,7 +1430,7 @@ static void mceusb_gen1_init(struct mceusb_dev *ir)
+ */
+ ret = usb_control_msg(ir->usbdev, usb_rcvctrlpipe(ir->usbdev, 0),
+ USB_REQ_SET_ADDRESS, USB_TYPE_VENDOR, 0, 0,
+- data, USB_CTRL_MSG_SZ, HZ * 3);
++ data, USB_CTRL_MSG_SZ, 3000);
+ dev_dbg(dev, "set address - ret = %d", ret);
+ dev_dbg(dev, "set address - data[0] = %d, data[1] = %d",
+ data[0], data[1]);
+@@ -1437,20 +1438,20 @@ static void mceusb_gen1_init(struct mceusb_dev *ir)
+ /* set feature: bit rate 38400 bps */
+ ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0),
+ USB_REQ_SET_FEATURE, USB_TYPE_VENDOR,
+- 0xc04e, 0x0000, NULL, 0, HZ * 3);
++ 0xc04e, 0x0000, NULL, 0, 3000);
+
+ dev_dbg(dev, "set feature - ret = %d", ret);
+
+ /* bRequest 4: set char length to 8 bits */
+ ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0),
+ 4, USB_TYPE_VENDOR,
+- 0x0808, 0x0000, NULL, 0, HZ * 3);
++ 0x0808, 0x0000, NULL, 0, 3000);
+ dev_dbg(dev, "set char length - retB = %d", ret);
+
+ /* bRequest 2: set handshaking to use DTR/DSR */
+ ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0),
+ 2, USB_TYPE_VENDOR,
+- 0x0000, 0x0100, NULL, 0, HZ * 3);
++ 0x0000, 0x0100, NULL, 0, 3000);
+ dev_dbg(dev, "set handshake - retC = %d", ret);
+
+ /* device resume */
+diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c
+index ac85464864b9d..cb22316b3f002 100644
+--- a/drivers/media/rc/redrat3.c
++++ b/drivers/media/rc/redrat3.c
+@@ -404,7 +404,7 @@ static int redrat3_send_cmd(int cmd, struct redrat3_dev *rr3)
+ udev = rr3->udev;
+ res = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), cmd,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- 0x0000, 0x0000, data, sizeof(u8), HZ * 10);
++ 0x0000, 0x0000, data, sizeof(u8), 10000);
+
+ if (res < 0) {
+ dev_err(rr3->dev, "%s: Error sending rr3 cmd res %d, data %d",
+@@ -480,7 +480,7 @@ static u32 redrat3_get_timeout(struct redrat3_dev *rr3)
+ pipe = usb_rcvctrlpipe(rr3->udev, 0);
+ ret = usb_control_msg(rr3->udev, pipe, RR3_GET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- RR3_IR_IO_SIG_TIMEOUT, 0, tmp, len, HZ * 5);
++ RR3_IR_IO_SIG_TIMEOUT, 0, tmp, len, 5000);
+ if (ret != len)
+ dev_warn(rr3->dev, "Failed to read timeout from hardware\n");
+ else {
+@@ -510,7 +510,7 @@ static int redrat3_set_timeout(struct rc_dev *rc_dev, unsigned int timeoutus)
+ ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), RR3_SET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+ RR3_IR_IO_SIG_TIMEOUT, 0, timeout, sizeof(*timeout),
+- HZ * 25);
++ 25000);
+ dev_dbg(dev, "set ir parm timeout %d ret 0x%02x\n",
+ be32_to_cpu(*timeout), ret);
+
+@@ -542,32 +542,32 @@ static void redrat3_reset(struct redrat3_dev *rr3)
+ *val = 0x01;
+ rc = usb_control_msg(udev, rxpipe, RR3_RESET,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- RR3_CPUCS_REG_ADDR, 0, val, len, HZ * 25);
++ RR3_CPUCS_REG_ADDR, 0, val, len, 25000);
+ dev_dbg(dev, "reset returned 0x%02x\n", rc);
+
+ *val = length_fuzz;
+ rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- RR3_IR_IO_LENGTH_FUZZ, 0, val, len, HZ * 25);
++ RR3_IR_IO_LENGTH_FUZZ, 0, val, len, 25000);
+ dev_dbg(dev, "set ir parm len fuzz %d rc 0x%02x\n", *val, rc);
+
+ *val = (65536 - (minimum_pause * 2000)) / 256;
+ rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- RR3_IR_IO_MIN_PAUSE, 0, val, len, HZ * 25);
++ RR3_IR_IO_MIN_PAUSE, 0, val, len, 25000);
+ dev_dbg(dev, "set ir parm min pause %d rc 0x%02x\n", *val, rc);
+
+ *val = periods_measure_carrier;
+ rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- RR3_IR_IO_PERIODS_MF, 0, val, len, HZ * 25);
++ RR3_IR_IO_PERIODS_MF, 0, val, len, 25000);
+ dev_dbg(dev, "set ir parm periods measure carrier %d rc 0x%02x", *val,
+ rc);
+
+ *val = RR3_DRIVER_MAXLENS;
+ rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- RR3_IR_IO_MAX_LENGTHS, 0, val, len, HZ * 25);
++ RR3_IR_IO_MAX_LENGTHS, 0, val, len, 25000);
+ dev_dbg(dev, "set ir parm max lens %d rc 0x%02x\n", *val, rc);
+
+ kfree(val);
+@@ -585,7 +585,7 @@ static void redrat3_get_firmware_rev(struct redrat3_dev *rr3)
+ rc = usb_control_msg(rr3->udev, usb_rcvctrlpipe(rr3->udev, 0),
+ RR3_FW_VERSION,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- 0, 0, buffer, RR3_FW_VERSION_LEN, HZ * 5);
++ 0, 0, buffer, RR3_FW_VERSION_LEN, 5000);
+
+ if (rc >= 0)
+ dev_info(rr3->dev, "Firmware rev: %s", buffer);
+@@ -825,14 +825,14 @@ static int redrat3_transmit_ir(struct rc_dev *rcdev, unsigned *txbuf,
+
+ pipe = usb_sndbulkpipe(rr3->udev, rr3->ep_out->bEndpointAddress);
+ ret = usb_bulk_msg(rr3->udev, pipe, irdata,
+- sendbuf_len, &ret_len, 10 * HZ);
++ sendbuf_len, &ret_len, 10000);
+ dev_dbg(dev, "sent %d bytes, (ret %d)\n", ret_len, ret);
+
+ /* now tell the hardware to transmit what we sent it */
+ pipe = usb_rcvctrlpipe(rr3->udev, 0);
+ ret = usb_control_msg(rr3->udev, pipe, RR3_TX_SEND_SIGNAL,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- 0, 0, irdata, 2, HZ * 10);
++ 0, 0, irdata, 2, 10000);
+
+ if (ret < 0)
+ dev_err(dev, "Error: control msg send failed, rc %d\n", ret);
+diff --git a/drivers/media/spi/cxd2880-spi.c b/drivers/media/spi/cxd2880-spi.c
+index b91a1e845b972..506f52c1af101 100644
+--- a/drivers/media/spi/cxd2880-spi.c
++++ b/drivers/media/spi/cxd2880-spi.c
+@@ -618,7 +618,7 @@ fail_frontend:
+ fail_attach:
+ dvb_unregister_adapter(&dvb_spi->adapter);
+ fail_adapter:
+- if (!dvb_spi->vcc_supply)
++ if (dvb_spi->vcc_supply)
+ regulator_disable(dvb_spi->vcc_supply);
+ fail_regulator:
+ kfree(dvb_spi);
+diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+index 75617709c8ce2..82620613d56b8 100644
+--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
++++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+@@ -564,6 +564,10 @@ static int vidtv_bridge_remove(struct platform_device *pdev)
+
+ static void vidtv_bridge_dev_release(struct device *dev)
+ {
++ struct vidtv_dvb *dvb;
++
++ dvb = dev_get_drvdata(dev);
++ kfree(dvb);
+ }
+
+ static struct platform_device vidtv_bridge_dev = {
+diff --git a/drivers/media/test-drivers/vidtv/vidtv_s302m.c b/drivers/media/test-drivers/vidtv/vidtv_s302m.c
+index d79b65854627c..4676083cee3b8 100644
+--- a/drivers/media/test-drivers/vidtv/vidtv_s302m.c
++++ b/drivers/media/test-drivers/vidtv/vidtv_s302m.c
+@@ -455,6 +455,9 @@ struct vidtv_encoder
+ e->name = kstrdup(args.name, GFP_KERNEL);
+
+ e->encoder_buf = vzalloc(VIDTV_S302M_BUF_SZ);
++ if (!e->encoder_buf)
++ goto out_kfree_e;
++
+ e->encoder_buf_sz = VIDTV_S302M_BUF_SZ;
+ e->encoder_buf_offset = 0;
+
+@@ -467,10 +470,8 @@ struct vidtv_encoder
+ e->is_video_encoder = false;
+
+ ctx = kzalloc(priv_sz, GFP_KERNEL);
+- if (!ctx) {
+- kfree(e);
+- return NULL;
+- }
++ if (!ctx)
++ goto out_kfree_buf;
+
+ e->ctx = ctx;
+ ctx->last_duration = 0;
+@@ -498,6 +499,14 @@ struct vidtv_encoder
+ e->next = NULL;
+
+ return e;
++
++out_kfree_buf:
++ kfree(e->encoder_buf);
++
++out_kfree_e:
++ kfree(e->name);
++ kfree(e);
++ return NULL;
+ }
+
+ void vidtv_s302m_encoder_destroy(struct vidtv_encoder *e)
+diff --git a/drivers/media/tuners/msi001.c b/drivers/media/tuners/msi001.c
+index 78e6fd600d8ef..44247049a3190 100644
+--- a/drivers/media/tuners/msi001.c
++++ b/drivers/media/tuners/msi001.c
+@@ -442,6 +442,13 @@ static int msi001_probe(struct spi_device *spi)
+ V4L2_CID_RF_TUNER_BANDWIDTH_AUTO, 0, 1, 1, 1);
+ dev->bandwidth = v4l2_ctrl_new_std(&dev->hdl, &msi001_ctrl_ops,
+ V4L2_CID_RF_TUNER_BANDWIDTH, 200000, 8000000, 1, 200000);
++ if (dev->hdl.error) {
++ ret = dev->hdl.error;
++ dev_err(&spi->dev, "Could not initialize controls\n");
++ /* control init failed, free handler */
++ goto err_ctrl_handler_free;
++ }
++
+ v4l2_ctrl_auto_cluster(2, &dev->bandwidth_auto, 0, false);
+ dev->lna_gain = v4l2_ctrl_new_std(&dev->hdl, &msi001_ctrl_ops,
+ V4L2_CID_RF_TUNER_LNA_GAIN, 0, 1, 1, 1);
+diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c
+index fefb2625f6558..75ddf7ed1faff 100644
+--- a/drivers/media/tuners/si2157.c
++++ b/drivers/media/tuners/si2157.c
+@@ -90,7 +90,7 @@ static int si2157_init(struct dvb_frontend *fe)
+ dev_dbg(&client->dev, "\n");
+
+ /* Try to get Xtal trim property, to verify tuner still running */
+- memcpy(cmd.args, "\x15\x00\x04\x02", 4);
++ memcpy(cmd.args, "\x15\x00\x02\x04", 4);
+ cmd.wlen = 4;
+ cmd.rlen = 4;
+ ret = si2157_cmd_execute(client, &cmd);
+diff --git a/drivers/media/usb/b2c2/flexcop-usb.c b/drivers/media/usb/b2c2/flexcop-usb.c
+index 5d38171b7638c..bfeb92d93de39 100644
+--- a/drivers/media/usb/b2c2/flexcop-usb.c
++++ b/drivers/media/usb/b2c2/flexcop-usb.c
+@@ -87,7 +87,7 @@ static int flexcop_usb_readwrite_dw(struct flexcop_device *fc, u16 wRegOffsPCI,
+ 0,
+ fc_usb->data,
+ sizeof(u32),
+- B2C2_WAIT_FOR_OPERATION_RDW * HZ);
++ B2C2_WAIT_FOR_OPERATION_RDW);
+
+ if (ret != sizeof(u32)) {
+ err("error while %s dword from %d (%d).", read ? "reading" :
+@@ -155,7 +155,7 @@ static int flexcop_usb_v8_memory_req(struct flexcop_usb *fc_usb,
+ wIndex,
+ fc_usb->data,
+ buflen,
+- nWaitTime * HZ);
++ nWaitTime);
+ if (ret != buflen)
+ ret = -EIO;
+
+@@ -248,13 +248,13 @@ static int flexcop_usb_i2c_req(struct flexcop_i2c_adapter *i2c,
+ /* DKT 020208 - add this to support special case of DiSEqC */
+ case USB_FUNC_I2C_CHECKWRITE:
+ pipe = B2C2_USB_CTRL_PIPE_OUT;
+- nWaitTime = 2;
++ nWaitTime = 2000;
+ request_type |= USB_DIR_OUT;
+ break;
+ case USB_FUNC_I2C_READ:
+ case USB_FUNC_I2C_REPEATREAD:
+ pipe = B2C2_USB_CTRL_PIPE_IN;
+- nWaitTime = 2;
++ nWaitTime = 2000;
+ request_type |= USB_DIR_IN;
+ break;
+ default:
+@@ -281,7 +281,7 @@ static int flexcop_usb_i2c_req(struct flexcop_i2c_adapter *i2c,
+ wIndex,
+ fc_usb->data,
+ buflen,
+- nWaitTime * HZ);
++ nWaitTime);
+
+ if (ret != buflen)
+ ret = -EIO;
+diff --git a/drivers/media/usb/b2c2/flexcop-usb.h b/drivers/media/usb/b2c2/flexcop-usb.h
+index 2f230bf72252b..c7cca1a5ee59d 100644
+--- a/drivers/media/usb/b2c2/flexcop-usb.h
++++ b/drivers/media/usb/b2c2/flexcop-usb.h
+@@ -91,13 +91,13 @@ typedef enum {
+ UTILITY_SRAM_TESTVERIFY = 0x16,
+ } flexcop_usb_utility_function_t;
+
+-#define B2C2_WAIT_FOR_OPERATION_RW (1*HZ)
+-#define B2C2_WAIT_FOR_OPERATION_RDW (3*HZ)
+-#define B2C2_WAIT_FOR_OPERATION_WDW (1*HZ)
++#define B2C2_WAIT_FOR_OPERATION_RW 1000
++#define B2C2_WAIT_FOR_OPERATION_RDW 3000
++#define B2C2_WAIT_FOR_OPERATION_WDW 1000
+
+-#define B2C2_WAIT_FOR_OPERATION_V8READ (3*HZ)
+-#define B2C2_WAIT_FOR_OPERATION_V8WRITE (3*HZ)
+-#define B2C2_WAIT_FOR_OPERATION_V8FLASH (3*HZ)
++#define B2C2_WAIT_FOR_OPERATION_V8READ 3000
++#define B2C2_WAIT_FOR_OPERATION_V8WRITE 3000
++#define B2C2_WAIT_FOR_OPERATION_V8FLASH 3000
+
+ typedef enum {
+ V8_MEMORY_PAGE_DVB_CI = 0x20,
+diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c
+index 76aac06f9fb8e..cba03b2864738 100644
+--- a/drivers/media/usb/cpia2/cpia2_usb.c
++++ b/drivers/media/usb/cpia2/cpia2_usb.c
+@@ -550,7 +550,7 @@ static int write_packet(struct usb_device *udev,
+ 0, /* index */
+ buf, /* buffer */
+ size,
+- HZ);
++ 1000);
+
+ kfree(buf);
+ return ret;
+@@ -582,7 +582,7 @@ static int read_packet(struct usb_device *udev,
+ 0, /* index */
+ buf, /* buffer */
+ size,
+- HZ);
++ 1000);
+
+ if (ret >= 0)
+ memcpy(registers, buf, size);
+diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf.c b/drivers/media/usb/dvb-usb-v2/mxl111sf.c
+index 7865fa0a82957..cd5861a30b6f8 100644
+--- a/drivers/media/usb/dvb-usb-v2/mxl111sf.c
++++ b/drivers/media/usb/dvb-usb-v2/mxl111sf.c
+@@ -931,8 +931,6 @@ static int mxl111sf_init(struct dvb_usb_device *d)
+ .len = sizeof(eeprom), .buf = eeprom },
+ };
+
+- mutex_init(&state->msg_lock);
+-
+ ret = get_chip_info(state);
+ if (mxl_fail(ret))
+ pr_err("failed to get chip info during probe");
+@@ -1074,6 +1072,14 @@ static int mxl111sf_get_stream_config_dvbt(struct dvb_frontend *fe,
+ return 0;
+ }
+
++static int mxl111sf_probe(struct dvb_usb_device *dev)
++{
++ struct mxl111sf_state *state = d_to_priv(dev);
++
++ mutex_init(&state->msg_lock);
++ return 0;
++}
++
+ static struct dvb_usb_device_properties mxl111sf_props_dvbt = {
+ .driver_name = KBUILD_MODNAME,
+ .owner = THIS_MODULE,
+@@ -1083,6 +1089,7 @@ static struct dvb_usb_device_properties mxl111sf_props_dvbt = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_dvbt,
+ .tuner_attach = mxl111sf_attach_tuner,
+@@ -1124,6 +1131,7 @@ static struct dvb_usb_device_properties mxl111sf_props_atsc = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_atsc,
+ .tuner_attach = mxl111sf_attach_tuner,
+@@ -1165,6 +1173,7 @@ static struct dvb_usb_device_properties mxl111sf_props_mh = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_mh,
+ .tuner_attach = mxl111sf_attach_tuner,
+@@ -1233,6 +1242,7 @@ static struct dvb_usb_device_properties mxl111sf_props_atsc_mh = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_atsc_mh,
+ .tuner_attach = mxl111sf_attach_tuner,
+@@ -1311,6 +1321,7 @@ static struct dvb_usb_device_properties mxl111sf_props_mercury = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_mercury,
+ .tuner_attach = mxl111sf_attach_tuner,
+@@ -1381,6 +1392,7 @@ static struct dvb_usb_device_properties mxl111sf_props_mercury_mh = {
+ .generic_bulk_ctrl_endpoint = 0x02,
+ .generic_bulk_ctrl_endpoint_response = 0x81,
+
++ .probe = mxl111sf_probe,
+ .i2c_algo = &mxl111sf_i2c_algo,
+ .frontend_attach = mxl111sf_frontend_attach_mercury_mh,
+ .tuner_attach = mxl111sf_attach_tuner,
+diff --git a/drivers/media/usb/dvb-usb/az6027.c b/drivers/media/usb/dvb-usb/az6027.c
+index 1c39b61cde29b..86788771175b7 100644
+--- a/drivers/media/usb/dvb-usb/az6027.c
++++ b/drivers/media/usb/dvb-usb/az6027.c
+@@ -391,6 +391,7 @@ static struct rc_map_table rc_map_az6027_table[] = {
+ /* remote control stuff (does not work with my box) */
+ static int az6027_rc_query(struct dvb_usb_device *d, u32 *event, int *state)
+ {
++ *state = REMOTE_NO_KEY_PRESSED;
+ return 0;
+ }
+
+diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c
+index 70219b3e85666..7ea8f68b0f458 100644
+--- a/drivers/media/usb/dvb-usb/dib0700_core.c
++++ b/drivers/media/usb/dvb-usb/dib0700_core.c
+@@ -618,8 +618,6 @@ int dib0700_streaming_ctrl(struct dvb_usb_adapter *adap, int onoff)
+ deb_info("the endpoint number (%i) is not correct, use the adapter id instead", adap->fe_adap[0].stream.props.endpoint);
+ if (onoff)
+ st->channel_state |= 1 << (adap->id);
+- else
+- st->channel_state |= 1 << ~(adap->id);
+ } else {
+ if (onoff)
+ st->channel_state |= 1 << (adap->fe_adap[0].stream.props.endpoint-2);
+diff --git a/drivers/media/usb/dvb-usb/dibusb-common.c b/drivers/media/usb/dvb-usb/dibusb-common.c
+index 02b51d1a1b67c..aff60c10cb0b2 100644
+--- a/drivers/media/usb/dvb-usb/dibusb-common.c
++++ b/drivers/media/usb/dvb-usb/dibusb-common.c
+@@ -223,7 +223,7 @@ int dibusb_read_eeprom_byte(struct dvb_usb_device *d, u8 offs, u8 *val)
+ u8 *buf;
+ int rc;
+
+- buf = kmalloc(2, GFP_KERNEL);
++ buf = kzalloc(2, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+diff --git a/drivers/media/usb/dvb-usb/dw2102.c b/drivers/media/usb/dvb-usb/dw2102.c
+index f0e686b05dc63..ca75ebdc10b37 100644
+--- a/drivers/media/usb/dvb-usb/dw2102.c
++++ b/drivers/media/usb/dvb-usb/dw2102.c
+@@ -2150,46 +2150,153 @@ static struct dvb_usb_device_properties s6x0_properties = {
+ }
+ };
+
+-static const struct dvb_usb_device_description d1100 = {
+- "Prof 1100 USB ",
+- {&dw2102_table[PROF_1100], NULL},
+- {NULL},
+-};
++static struct dvb_usb_device_properties p1100_properties = {
++ .caps = DVB_USB_IS_AN_I2C_ADAPTER,
++ .usb_ctrl = DEVICE_SPECIFIC,
++ .size_of_priv = sizeof(struct dw2102_state),
++ .firmware = P1100_FIRMWARE,
++ .no_reconnect = 1,
+
+-static const struct dvb_usb_device_description d660 = {
+- "TeVii S660 USB",
+- {&dw2102_table[TEVII_S660], NULL},
+- {NULL},
+-};
++ .i2c_algo = &s6x0_i2c_algo,
++ .rc.core = {
++ .rc_interval = 150,
++ .rc_codes = RC_MAP_TBS_NEC,
++ .module_name = "dw2102",
++ .allowed_protos = RC_PROTO_BIT_NEC,
++ .rc_query = prof_rc_query,
++ },
+
+-static const struct dvb_usb_device_description d480_1 = {
+- "TeVii S480.1 USB",
+- {&dw2102_table[TEVII_S480_1], NULL},
+- {NULL},
++ .generic_bulk_ctrl_endpoint = 0x81,
++ .num_adapters = 1,
++ .download_firmware = dw2102_load_firmware,
++ .read_mac_address = s6x0_read_mac_address,
++ .adapter = {
++ {
++ .num_frontends = 1,
++ .fe = {{
++ .frontend_attach = stv0288_frontend_attach,
++ .stream = {
++ .type = USB_BULK,
++ .count = 8,
++ .endpoint = 0x82,
++ .u = {
++ .bulk = {
++ .buffersize = 4096,
++ }
++ }
++ },
++ } },
++ }
++ },
++ .num_device_descs = 1,
++ .devices = {
++ {"Prof 1100 USB ",
++ {&dw2102_table[PROF_1100], NULL},
++ {NULL},
++ },
++ }
+ };
+
+-static const struct dvb_usb_device_description d480_2 = {
+- "TeVii S480.2 USB",
+- {&dw2102_table[TEVII_S480_2], NULL},
+- {NULL},
+-};
++static struct dvb_usb_device_properties s660_properties = {
++ .caps = DVB_USB_IS_AN_I2C_ADAPTER,
++ .usb_ctrl = DEVICE_SPECIFIC,
++ .size_of_priv = sizeof(struct dw2102_state),
++ .firmware = S660_FIRMWARE,
++ .no_reconnect = 1,
+
+-static const struct dvb_usb_device_description d7500 = {
+- "Prof 7500 USB DVB-S2",
+- {&dw2102_table[PROF_7500], NULL},
+- {NULL},
+-};
++ .i2c_algo = &s6x0_i2c_algo,
++ .rc.core = {
++ .rc_interval = 150,
++ .rc_codes = RC_MAP_TEVII_NEC,
++ .module_name = "dw2102",
++ .allowed_protos = RC_PROTO_BIT_NEC,
++ .rc_query = dw2102_rc_query,
++ },
+
+-static const struct dvb_usb_device_description d421 = {
+- "TeVii S421 PCI",
+- {&dw2102_table[TEVII_S421], NULL},
+- {NULL},
++ .generic_bulk_ctrl_endpoint = 0x81,
++ .num_adapters = 1,
++ .download_firmware = dw2102_load_firmware,
++ .read_mac_address = s6x0_read_mac_address,
++ .adapter = {
++ {
++ .num_frontends = 1,
++ .fe = {{
++ .frontend_attach = ds3000_frontend_attach,
++ .stream = {
++ .type = USB_BULK,
++ .count = 8,
++ .endpoint = 0x82,
++ .u = {
++ .bulk = {
++ .buffersize = 4096,
++ }
++ }
++ },
++ } },
++ }
++ },
++ .num_device_descs = 3,
++ .devices = {
++ {"TeVii S660 USB",
++ {&dw2102_table[TEVII_S660], NULL},
++ {NULL},
++ },
++ {"TeVii S480.1 USB",
++ {&dw2102_table[TEVII_S480_1], NULL},
++ {NULL},
++ },
++ {"TeVii S480.2 USB",
++ {&dw2102_table[TEVII_S480_2], NULL},
++ {NULL},
++ },
++ }
+ };
+
+-static const struct dvb_usb_device_description d632 = {
+- "TeVii S632 USB",
+- {&dw2102_table[TEVII_S632], NULL},
+- {NULL},
++static struct dvb_usb_device_properties p7500_properties = {
++ .caps = DVB_USB_IS_AN_I2C_ADAPTER,
++ .usb_ctrl = DEVICE_SPECIFIC,
++ .size_of_priv = sizeof(struct dw2102_state),
++ .firmware = P7500_FIRMWARE,
++ .no_reconnect = 1,
++
++ .i2c_algo = &s6x0_i2c_algo,
++ .rc.core = {
++ .rc_interval = 150,
++ .rc_codes = RC_MAP_TBS_NEC,
++ .module_name = "dw2102",
++ .allowed_protos = RC_PROTO_BIT_NEC,
++ .rc_query = prof_rc_query,
++ },
++
++ .generic_bulk_ctrl_endpoint = 0x81,
++ .num_adapters = 1,
++ .download_firmware = dw2102_load_firmware,
++ .read_mac_address = s6x0_read_mac_address,
++ .adapter = {
++ {
++ .num_frontends = 1,
++ .fe = {{
++ .frontend_attach = prof_7500_frontend_attach,
++ .stream = {
++ .type = USB_BULK,
++ .count = 8,
++ .endpoint = 0x82,
++ .u = {
++ .bulk = {
++ .buffersize = 4096,
++ }
++ }
++ },
++ } },
++ }
++ },
++ .num_device_descs = 1,
++ .devices = {
++ {"Prof 7500 USB DVB-S2",
++ {&dw2102_table[PROF_7500], NULL},
++ {NULL},
++ },
++ }
+ };
+
+ static struct dvb_usb_device_properties su3000_properties = {
+@@ -2273,6 +2380,59 @@ static struct dvb_usb_device_properties su3000_properties = {
+ }
+ };
+
++static struct dvb_usb_device_properties s421_properties = {
++ .caps = DVB_USB_IS_AN_I2C_ADAPTER,
++ .usb_ctrl = DEVICE_SPECIFIC,
++ .size_of_priv = sizeof(struct dw2102_state),
++ .power_ctrl = su3000_power_ctrl,
++ .num_adapters = 1,
++ .identify_state = su3000_identify_state,
++ .i2c_algo = &su3000_i2c_algo,
++
++ .rc.core = {
++ .rc_interval = 150,
++ .rc_codes = RC_MAP_SU3000,
++ .module_name = "dw2102",
++ .allowed_protos = RC_PROTO_BIT_RC5,
++ .rc_query = su3000_rc_query,
++ },
++
++ .read_mac_address = su3000_read_mac_address,
++
++ .generic_bulk_ctrl_endpoint = 0x01,
++
++ .adapter = {
++ {
++ .num_frontends = 1,
++ .fe = {{
++ .streaming_ctrl = su3000_streaming_ctrl,
++ .frontend_attach = m88rs2000_frontend_attach,
++ .stream = {
++ .type = USB_BULK,
++ .count = 8,
++ .endpoint = 0x82,
++ .u = {
++ .bulk = {
++ .buffersize = 4096,
++ }
++ }
++ }
++ } },
++ }
++ },
++ .num_device_descs = 2,
++ .devices = {
++ { "TeVii S421 PCI",
++ { &dw2102_table[TEVII_S421], NULL },
++ { NULL },
++ },
++ { "TeVii S632 USB",
++ { &dw2102_table[TEVII_S632], NULL },
++ { NULL },
++ },
++ }
++};
++
+ static struct dvb_usb_device_properties t220_properties = {
+ .caps = DVB_USB_IS_AN_I2C_ADAPTER,
+ .usb_ctrl = DEVICE_SPECIFIC,
+@@ -2390,101 +2550,33 @@ static struct dvb_usb_device_properties tt_s2_4600_properties = {
+ static int dw2102_probe(struct usb_interface *intf,
+ const struct usb_device_id *id)
+ {
+- int retval = -ENOMEM;
+- struct dvb_usb_device_properties *p1100;
+- struct dvb_usb_device_properties *s660;
+- struct dvb_usb_device_properties *p7500;
+- struct dvb_usb_device_properties *s421;
+-
+- p1100 = kmemdup(&s6x0_properties,
+- sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
+- if (!p1100)
+- goto err0;
+-
+- /* copy default structure */
+- /* fill only different fields */
+- p1100->firmware = P1100_FIRMWARE;
+- p1100->devices[0] = d1100;
+- p1100->rc.core.rc_query = prof_rc_query;
+- p1100->rc.core.rc_codes = RC_MAP_TBS_NEC;
+- p1100->adapter->fe[0].frontend_attach = stv0288_frontend_attach;
+-
+- s660 = kmemdup(&s6x0_properties,
+- sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
+- if (!s660)
+- goto err1;
+-
+- s660->firmware = S660_FIRMWARE;
+- s660->num_device_descs = 3;
+- s660->devices[0] = d660;
+- s660->devices[1] = d480_1;
+- s660->devices[2] = d480_2;
+- s660->adapter->fe[0].frontend_attach = ds3000_frontend_attach;
+-
+- p7500 = kmemdup(&s6x0_properties,
+- sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
+- if (!p7500)
+- goto err2;
+-
+- p7500->firmware = P7500_FIRMWARE;
+- p7500->devices[0] = d7500;
+- p7500->rc.core.rc_query = prof_rc_query;
+- p7500->rc.core.rc_codes = RC_MAP_TBS_NEC;
+- p7500->adapter->fe[0].frontend_attach = prof_7500_frontend_attach;
+-
+-
+- s421 = kmemdup(&su3000_properties,
+- sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
+- if (!s421)
+- goto err3;
+-
+- s421->num_device_descs = 2;
+- s421->devices[0] = d421;
+- s421->devices[1] = d632;
+- s421->adapter->fe[0].frontend_attach = m88rs2000_frontend_attach;
+-
+- if (0 == dvb_usb_device_init(intf, &dw2102_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &dw2104_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &dw3101_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &s6x0_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, p1100,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, s660,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, p7500,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, s421,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &su3000_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &t220_properties,
+- THIS_MODULE, NULL, adapter_nr) ||
+- 0 == dvb_usb_device_init(intf, &tt_s2_4600_properties,
+- THIS_MODULE, NULL, adapter_nr)) {
+-
+- /* clean up copied properties */
+- kfree(s421);
+- kfree(p7500);
+- kfree(s660);
+- kfree(p1100);
++ if (!(dvb_usb_device_init(intf, &dw2102_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &dw2104_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &dw3101_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &s6x0_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &p1100_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &s660_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &p7500_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &s421_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &su3000_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &t220_properties,
++ THIS_MODULE, NULL, adapter_nr) &&
++ dvb_usb_device_init(intf, &tt_s2_4600_properties,
++ THIS_MODULE, NULL, adapter_nr))) {
+
+ return 0;
+ }
+
+- retval = -ENODEV;
+- kfree(s421);
+-err3:
+- kfree(p7500);
+-err2:
+- kfree(s660);
+-err1:
+- kfree(p1100);
+-err0:
+- return retval;
++ return -ENODEV;
+ }
+
+ static void dw2102_disconnect(struct usb_interface *intf)
+diff --git a/drivers/media/usb/dvb-usb/m920x.c b/drivers/media/usb/dvb-usb/m920x.c
+index 4bb5b82599a79..691e05833db19 100644
+--- a/drivers/media/usb/dvb-usb/m920x.c
++++ b/drivers/media/usb/dvb-usb/m920x.c
+@@ -274,6 +274,13 @@ static int m920x_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msg[], int nu
+ /* Should check for ack here, if we knew how. */
+ }
+ if (msg[i].flags & I2C_M_RD) {
++ char *read = kmalloc(1, GFP_KERNEL);
++ if (!read) {
++ ret = -ENOMEM;
++ kfree(read);
++ goto unlock;
++ }
++
+ for (j = 0; j < msg[i].len; j++) {
+ /* Last byte of transaction?
+ * Send STOP, otherwise send ACK. */
+@@ -281,9 +288,12 @@ static int m920x_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msg[], int nu
+
+ if ((ret = m920x_read(d->udev, M9206_I2C, 0x0,
+ 0x20 | stop,
+- &msg[i].buf[j], 1)) != 0)
++ read, 1)) != 0)
+ goto unlock;
++ msg[i].buf[j] = read[0];
+ }
++
++ kfree(read);
+ } else {
+ for (j = 0; j < msg[i].len; j++) {
+ /* Last byte of transaction? Then send STOP. */
+diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c
+index c1e0dccb74088..92d867fc519c4 100644
+--- a/drivers/media/usb/em28xx/em28xx-cards.c
++++ b/drivers/media/usb/em28xx/em28xx-cards.c
+@@ -3625,8 +3625,10 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev,
+
+ if (dev->is_audio_only) {
+ retval = em28xx_audio_setup(dev);
+- if (retval)
+- return -ENODEV;
++ if (retval) {
++ retval = -ENODEV;
++ goto err_deinit_media;
++ }
+ em28xx_init_extension(dev);
+
+ return 0;
+@@ -3645,7 +3647,7 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev,
+ dev_err(&dev->intf->dev,
+ "%s: em28xx_i2c_register bus 0 - error [%d]!\n",
+ __func__, retval);
+- return retval;
++ goto err_deinit_media;
+ }
+
+ /* register i2c bus 1 */
+@@ -3661,9 +3663,7 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev,
+ "%s: em28xx_i2c_register bus 1 - error [%d]!\n",
+ __func__, retval);
+
+- em28xx_i2c_unregister(dev, 0);
+-
+- return retval;
++ goto err_unreg_i2c;
+ }
+ }
+
+@@ -3671,6 +3671,12 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev,
+ em28xx_card_setup(dev);
+
+ return 0;
++
++err_unreg_i2c:
++ em28xx_i2c_unregister(dev, 0);
++err_deinit_media:
++ em28xx_unregister_media_device(dev);
++ return retval;
+ }
+
+ static int em28xx_duplicate_dev(struct em28xx *dev)
+@@ -3925,6 +3931,8 @@ static int em28xx_usb_probe(struct usb_interface *intf,
+ goto err_free;
+ }
+
++ kref_init(&dev->ref);
++
+ dev->devno = nr;
+ dev->model = id->driver_info;
+ dev->alt = -1;
+@@ -4025,6 +4033,8 @@ static int em28xx_usb_probe(struct usb_interface *intf,
+ }
+
+ if (dev->board.has_dual_ts && em28xx_duplicate_dev(dev) == 0) {
++ kref_init(&dev->dev_next->ref);
++
+ dev->dev_next->ts = SECONDARY_TS;
+ dev->dev_next->alt = -1;
+ dev->dev_next->is_audio_only = has_vendor_audio &&
+@@ -4079,12 +4089,8 @@ static int em28xx_usb_probe(struct usb_interface *intf,
+ em28xx_write_reg(dev, 0x0b, 0x82);
+ mdelay(100);
+ }
+-
+- kref_init(&dev->dev_next->ref);
+ }
+
+- kref_init(&dev->ref);
+-
+ request_modules(dev);
+
+ /*
+diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c
+index 584fa400cd7d8..c837cc528a335 100644
+--- a/drivers/media/usb/em28xx/em28xx-core.c
++++ b/drivers/media/usb/em28xx/em28xx-core.c
+@@ -89,7 +89,7 @@ int em28xx_read_reg_req_len(struct em28xx *dev, u8 req, u16 reg,
+ mutex_lock(&dev->ctrl_urb_lock);
+ ret = usb_control_msg(udev, pipe, req,
+ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- 0x0000, reg, dev->urb_buf, len, HZ);
++ 0x0000, reg, dev->urb_buf, len, 1000);
+ if (ret < 0) {
+ em28xx_regdbg("(pipe 0x%08x): IN: %02x %02x %02x %02x %02x %02x %02x %02x failed with error %i\n",
+ pipe,
+@@ -158,7 +158,7 @@ int em28xx_write_regs_req(struct em28xx *dev, u8 req, u16 reg, char *buf,
+ memcpy(dev->urb_buf, buf, len);
+ ret = usb_control_msg(udev, pipe, req,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- 0x0000, reg, dev->urb_buf, len, HZ);
++ 0x0000, reg, dev->urb_buf, len, 1000);
+ mutex_unlock(&dev->ctrl_urb_lock);
+
+ if (ret < 0) {
+@@ -1154,8 +1154,9 @@ int em28xx_suspend_extension(struct em28xx *dev)
+ dev_info(&dev->intf->dev, "Suspending extensions\n");
+ mutex_lock(&em28xx_devlist_mutex);
+ list_for_each_entry(ops, &em28xx_extension_devlist, next) {
+- if (ops->suspend)
+- ops->suspend(dev);
++ if (!ops->suspend)
++ continue;
++ ops->suspend(dev);
+ if (dev->dev_next)
+ ops->suspend(dev->dev_next);
+ }
+diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c
+index c742cc88fac5c..1fa6f10ee157b 100644
+--- a/drivers/media/usb/go7007/s2250-board.c
++++ b/drivers/media/usb/go7007/s2250-board.c
+@@ -504,6 +504,7 @@ static int s2250_probe(struct i2c_client *client,
+ u8 *data;
+ struct go7007 *go = i2c_get_adapdata(adapter);
+ struct go7007_usb *usb = go->hpi_context;
++ int err = -EIO;
+
+ audio = i2c_new_dummy_device(adapter, TLV320_ADDRESS >> 1);
+ if (IS_ERR(audio))
+@@ -532,11 +533,8 @@ static int s2250_probe(struct i2c_client *client,
+ V4L2_CID_HUE, -512, 511, 1, 0);
+ sd->ctrl_handler = &state->hdl;
+ if (state->hdl.error) {
+- int err = state->hdl.error;
+-
+- v4l2_ctrl_handler_free(&state->hdl);
+- kfree(state);
+- return err;
++ err = state->hdl.error;
++ goto fail;
+ }
+
+ state->std = V4L2_STD_NTSC;
+@@ -600,7 +598,7 @@ fail:
+ i2c_unregister_device(audio);
+ v4l2_ctrl_handler_free(&state->hdl);
+ kfree(state);
+- return -EIO;
++ return err;
+ }
+
+ static int s2250_remove(struct i2c_client *client)
+diff --git a/drivers/media/usb/hdpvr/hdpvr-video.c b/drivers/media/usb/hdpvr/hdpvr-video.c
+index 563128d117317..60e57e0f19272 100644
+--- a/drivers/media/usb/hdpvr/hdpvr-video.c
++++ b/drivers/media/usb/hdpvr/hdpvr-video.c
+@@ -308,7 +308,6 @@ static int hdpvr_start_streaming(struct hdpvr_device *dev)
+
+ dev->status = STATUS_STREAMING;
+
+- INIT_WORK(&dev->worker, hdpvr_transmit_buffers);
+ schedule_work(&dev->worker);
+
+ v4l2_dbg(MSG_BUFFER, hdpvr_debug, &dev->v4l2_dev,
+@@ -1165,6 +1164,9 @@ int hdpvr_register_videodev(struct hdpvr_device *dev, struct device *parent,
+ bool ac3 = dev->flags & HDPVR_FLAG_AC3_CAP;
+ int res;
+
++ // initialize dev->worker
++ INIT_WORK(&dev->worker, hdpvr_transmit_buffers);
++
+ dev->cur_std = V4L2_STD_525_60;
+ dev->width = 720;
+ dev->height = 480;
+diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+index d38dee1792e41..3915d551d59e7 100644
+--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
++++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+@@ -1467,7 +1467,7 @@ static int pvr2_upload_firmware1(struct pvr2_hdw *hdw)
+ for (address = 0; address < fwsize; address += 0x800) {
+ memcpy(fw_ptr, fw_entry->data + address, 0x800);
+ ret += usb_control_msg(hdw->usb_dev, pipe, 0xa0, 0x40, address,
+- 0, fw_ptr, 0x800, HZ);
++ 0, fw_ptr, 0x800, 1000);
+ }
+
+ trace_firmware("Upload done, releasing device's CPU");
+@@ -1605,7 +1605,7 @@ int pvr2_upload_firmware2(struct pvr2_hdw *hdw)
+ ((u32 *)fw_ptr)[icnt] = swab32(((u32 *)fw_ptr)[icnt]);
+
+ ret |= usb_bulk_msg(hdw->usb_dev, pipe, fw_ptr,bcnt,
+- &actual_length, HZ);
++ &actual_length, 1000);
+ ret |= (actual_length != bcnt);
+ if (ret) break;
+ fw_done += bcnt;
+@@ -3438,7 +3438,7 @@ void pvr2_hdw_cpufw_set_enabled(struct pvr2_hdw *hdw,
+ 0xa0,0xc0,
+ address,0,
+ hdw->fw_buffer+address,
+- 0x800,HZ);
++ 0x800,1000);
+ if (ret < 0) break;
+ }
+
+@@ -3977,7 +3977,7 @@ void pvr2_hdw_cpureset_assert(struct pvr2_hdw *hdw,int val)
+ /* Write the CPUCS register on the 8051. The lsb of the register
+ is the reset bit; a 1 asserts reset while a 0 clears it. */
+ pipe = usb_sndctrlpipe(hdw->usb_dev, 0);
+- ret = usb_control_msg(hdw->usb_dev,pipe,0xa0,0x40,0xe600,0,da,1,HZ);
++ ret = usb_control_msg(hdw->usb_dev,pipe,0xa0,0x40,0xe600,0,da,1,1000);
+ if (ret < 0) {
+ pvr2_trace(PVR2_TRACE_ERROR_LEGS,
+ "cpureset_assert(%d) error=%d",val,ret);
+diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
+index 3b0e4ed75d99c..acf18e2251a52 100644
+--- a/drivers/media/usb/s2255/s2255drv.c
++++ b/drivers/media/usb/s2255/s2255drv.c
+@@ -1882,7 +1882,7 @@ static long s2255_vendor_req(struct s2255_dev *dev, unsigned char Request,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE |
+ USB_DIR_IN,
+ Value, Index, buf,
+- TransferBufferLength, HZ * 5);
++ TransferBufferLength, USB_CTRL_SET_TIMEOUT);
+
+ if (r >= 0)
+ memcpy(TransferBuffer, buf, TransferBufferLength);
+@@ -1891,7 +1891,7 @@ static long s2255_vendor_req(struct s2255_dev *dev, unsigned char Request,
+ r = usb_control_msg(dev->udev, usb_sndctrlpipe(dev->udev, 0),
+ Request, USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+ Value, Index, buf,
+- TransferBufferLength, HZ * 5);
++ TransferBufferLength, USB_CTRL_SET_TIMEOUT);
+ }
+ kfree(buf);
+ return r;
+diff --git a/drivers/media/usb/stk1160/stk1160-core.c b/drivers/media/usb/stk1160/stk1160-core.c
+index b4f8bc5db1389..ce717502ea4c3 100644
+--- a/drivers/media/usb/stk1160/stk1160-core.c
++++ b/drivers/media/usb/stk1160/stk1160-core.c
+@@ -65,7 +65,7 @@ int stk1160_read_reg(struct stk1160 *dev, u16 reg, u8 *value)
+ return -ENOMEM;
+ ret = usb_control_msg(dev->udev, pipe, 0x00,
+ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- 0x00, reg, buf, sizeof(u8), HZ);
++ 0x00, reg, buf, sizeof(u8), 1000);
+ if (ret < 0) {
+ stk1160_err("read failed on reg 0x%x (%d)\n",
+ reg, ret);
+@@ -85,7 +85,7 @@ int stk1160_write_reg(struct stk1160 *dev, u16 reg, u16 value)
+
+ ret = usb_control_msg(dev->udev, pipe, 0x01,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- value, reg, NULL, 0, HZ);
++ value, reg, NULL, 0, 1000);
+ if (ret < 0) {
+ stk1160_err("write failed on reg 0x%x (%d)\n",
+ reg, ret);
+@@ -403,7 +403,7 @@ static void stk1160_disconnect(struct usb_interface *interface)
+ /* Here is the only place where isoc get released */
+ stk1160_uninit_isoc(dev);
+
+- stk1160_clear_queue(dev);
++ stk1160_clear_queue(dev, VB2_BUF_STATE_ERROR);
+
+ video_unregister_device(&dev->vdev);
+ v4l2_device_disconnect(&dev->v4l2_dev);
+diff --git a/drivers/media/usb/stk1160/stk1160-v4l.c b/drivers/media/usb/stk1160/stk1160-v4l.c
+index 6a4eb616d5160..1aa953469402f 100644
+--- a/drivers/media/usb/stk1160/stk1160-v4l.c
++++ b/drivers/media/usb/stk1160/stk1160-v4l.c
+@@ -258,7 +258,7 @@ out_uninit:
+ stk1160_uninit_isoc(dev);
+ out_stop_hw:
+ usb_set_interface(dev->udev, 0, 0);
+- stk1160_clear_queue(dev);
++ stk1160_clear_queue(dev, VB2_BUF_STATE_QUEUED);
+
+ mutex_unlock(&dev->v4l_lock);
+
+@@ -306,7 +306,7 @@ static int stk1160_stop_streaming(struct stk1160 *dev)
+
+ stk1160_stop_hw(dev);
+
+- stk1160_clear_queue(dev);
++ stk1160_clear_queue(dev, VB2_BUF_STATE_ERROR);
+
+ stk1160_dbg("streaming stopped\n");
+
+@@ -745,7 +745,7 @@ static const struct video_device v4l_template = {
+ /********************************************************************/
+
+ /* Must be called with both v4l_lock and vb_queue_lock hold */
+-void stk1160_clear_queue(struct stk1160 *dev)
++void stk1160_clear_queue(struct stk1160 *dev, enum vb2_buffer_state vb2_state)
+ {
+ struct stk1160_buffer *buf;
+ unsigned long flags;
+@@ -756,7 +756,7 @@ void stk1160_clear_queue(struct stk1160 *dev)
+ buf = list_first_entry(&dev->avail_bufs,
+ struct stk1160_buffer, list);
+ list_del(&buf->list);
+- vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
++ vb2_buffer_done(&buf->vb.vb2_buf, vb2_state);
+ stk1160_dbg("buffer [%p/%d] aborted\n",
+ buf, buf->vb.vb2_buf.index);
+ }
+@@ -766,7 +766,7 @@ void stk1160_clear_queue(struct stk1160 *dev)
+ buf = dev->isoc_ctl.buf;
+ dev->isoc_ctl.buf = NULL;
+
+- vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
++ vb2_buffer_done(&buf->vb.vb2_buf, vb2_state);
+ stk1160_dbg("buffer [%p/%d] aborted\n",
+ buf, buf->vb.vb2_buf.index);
+ }
+diff --git a/drivers/media/usb/stk1160/stk1160.h b/drivers/media/usb/stk1160/stk1160.h
+index a31ea1c80f255..a70963ce87533 100644
+--- a/drivers/media/usb/stk1160/stk1160.h
++++ b/drivers/media/usb/stk1160/stk1160.h
+@@ -166,7 +166,7 @@ struct regval {
+ int stk1160_vb2_setup(struct stk1160 *dev);
+ int stk1160_video_register(struct stk1160 *dev);
+ void stk1160_video_unregister(struct stk1160 *dev);
+-void stk1160_clear_queue(struct stk1160 *dev);
++void stk1160_clear_queue(struct stk1160 *dev, enum vb2_buffer_state vb2_state);
+
+ /* Provided by stk1160-video.c */
+ int stk1160_alloc_isoc(struct stk1160 *dev);
+diff --git a/drivers/media/usb/tm6000/tm6000-video.c b/drivers/media/usb/tm6000/tm6000-video.c
+index 3f650ede0c3dc..e293f6f3d1bc9 100644
+--- a/drivers/media/usb/tm6000/tm6000-video.c
++++ b/drivers/media/usb/tm6000/tm6000-video.c
+@@ -852,8 +852,7 @@ static int vidioc_querycap(struct file *file, void *priv,
+ struct tm6000_core *dev = ((struct tm6000_fh *)priv)->dev;
+
+ strscpy(cap->driver, "tm6000", sizeof(cap->driver));
+- strscpy(cap->card, "Trident TVMaster TM5600/6000/6010",
+- sizeof(cap->card));
++ strscpy(cap->card, "Trident TM5600/6000/6010", sizeof(cap->card));
+ usb_make_path(dev->udev, cap->bus_info, sizeof(cap->bus_info));
+ cap->capabilities = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_READWRITE |
+ V4L2_CAP_DEVICE_CAPS;
+diff --git a/drivers/media/usb/ttusb-dec/ttusb_dec.c b/drivers/media/usb/ttusb-dec/ttusb_dec.c
+index bfda46a36dc50..38822cedd93a9 100644
+--- a/drivers/media/usb/ttusb-dec/ttusb_dec.c
++++ b/drivers/media/usb/ttusb-dec/ttusb_dec.c
+@@ -327,7 +327,7 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command,
+ result = mutex_lock_interruptible(&dec->usb_mutex);
+ if (result) {
+ printk("%s: Failed to lock usb mutex.\n", __func__);
+- goto err;
++ goto err_free;
+ }
+
+ b[0] = 0xaa;
+@@ -349,7 +349,7 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command,
+ if (result) {
+ printk("%s: command bulk message failed: error %d\n",
+ __func__, result);
+- goto err;
++ goto err_mutex_unlock;
+ }
+
+ result = usb_bulk_msg(dec->udev, dec->result_pipe, b,
+@@ -358,7 +358,7 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command,
+ if (result) {
+ printk("%s: result bulk message failed: error %d\n",
+ __func__, result);
+- goto err;
++ goto err_mutex_unlock;
+ } else {
+ if (debug) {
+ printk(KERN_DEBUG "%s: result: %*ph\n",
+@@ -371,9 +371,9 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command,
+ memcpy(cmd_result, &b[4], b[3]);
+ }
+
+-err:
++err_mutex_unlock:
+ mutex_unlock(&dec->usb_mutex);
+-
++err_free:
+ kfree(b);
+ return result;
+ }
+diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c
+index 6acb8013de08b..c9d208677bcd8 100644
+--- a/drivers/media/usb/uvc/uvc_v4l2.c
++++ b/drivers/media/usb/uvc/uvc_v4l2.c
+@@ -472,10 +472,13 @@ static int uvc_v4l2_set_streamparm(struct uvc_streaming *stream,
+ uvc_simplify_fraction(&timeperframe.numerator,
+ &timeperframe.denominator, 8, 333);
+
+- if (parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
++ if (parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+ parm->parm.capture.timeperframe = timeperframe;
+- else
++ parm->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
++ } else {
+ parm->parm.output.timeperframe = timeperframe;
++ parm->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
++ }
+
+ return 0;
+ }
+diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
+index e16464606b140..1b4cc934109e8 100644
+--- a/drivers/media/usb/uvc/uvc_video.c
++++ b/drivers/media/usb/uvc/uvc_video.c
+@@ -115,6 +115,11 @@ int uvc_query_ctrl(struct uvc_device *dev, u8 query, u8 unit,
+ case 5: /* Invalid unit */
+ case 6: /* Invalid control */
+ case 7: /* Invalid Request */
++ /*
++ * The firmware has not properly implemented
++ * the control or there has been a HW error.
++ */
++ return -EIO;
+ case 8: /* Invalid value within range */
+ return -EINVAL;
+ default: /* reserved or unknown */
+@@ -1958,6 +1963,10 @@ static int uvc_video_start_transfer(struct uvc_streaming *stream,
+ if (ep == NULL)
+ return -EIO;
+
++ /* Reject broken descriptors. */
++ if (usb_endpoint_maxp(&ep->desc) == 0)
++ return -EIO;
++
+ ret = uvc_init_video_bulk(stream, ep, gfp_flags);
+ }
+
+diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h
+index cce5e38133cd3..c3ea6a53869f5 100644
+--- a/drivers/media/usb/uvc/uvcvideo.h
++++ b/drivers/media/usb/uvc/uvcvideo.h
+@@ -189,7 +189,7 @@
+ /* Maximum status buffer size in bytes of interrupt URB. */
+ #define UVC_MAX_STATUS_SIZE 16
+
+-#define UVC_CTRL_CONTROL_TIMEOUT 500
++#define UVC_CTRL_CONTROL_TIMEOUT 5000
+ #define UVC_CTRL_STREAMING_TIMEOUT 5000
+
+ /* Maximum allowed number of control mappings per device */
+diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+index 47aff3b197426..80aaf07b16f28 100644
+--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
++++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+@@ -744,10 +744,6 @@ static int put_v4l2_ext_controls32(struct v4l2_ext_controls *p64,
+ /*
+ * x86 is the only compat architecture with different struct alignment
+ * between 32-bit and 64-bit tasks.
+- *
+- * On all other architectures, v4l2_event32 and v4l2_event32_time32 are
+- * the same as v4l2_event and v4l2_event_time32, so we can use the native
+- * handlers, converting v4l2_event to v4l2_event_time32 if necessary.
+ */
+ struct v4l2_event32 {
+ __u32 type;
+@@ -765,21 +761,6 @@ struct v4l2_event32 {
+ __u32 reserved[8];
+ };
+
+-#ifdef CONFIG_COMPAT_32BIT_TIME
+-struct v4l2_event32_time32 {
+- __u32 type;
+- union {
+- compat_s64 value64;
+- __u8 data[64];
+- } u;
+- __u32 pending;
+- __u32 sequence;
+- struct old_timespec32 timestamp;
+- __u32 id;
+- __u32 reserved[8];
+-};
+-#endif
+-
+ static int put_v4l2_event32(struct v4l2_event *p64,
+ struct v4l2_event32 __user *p32)
+ {
+@@ -795,7 +776,22 @@ static int put_v4l2_event32(struct v4l2_event *p64,
+ return 0;
+ }
+
++#endif
++
+ #ifdef CONFIG_COMPAT_32BIT_TIME
++struct v4l2_event32_time32 {
++ __u32 type;
++ union {
++ compat_s64 value64;
++ __u8 data[64];
++ } u;
++ __u32 pending;
++ __u32 sequence;
++ struct old_timespec32 timestamp;
++ __u32 id;
++ __u32 reserved[8];
++};
++
+ static int put_v4l2_event32_time32(struct v4l2_event *p64,
+ struct v4l2_event32_time32 __user *p32)
+ {
+@@ -811,7 +807,6 @@ static int put_v4l2_event32_time32(struct v4l2_event *p64,
+ return 0;
+ }
+ #endif
+-#endif
+
+ struct v4l2_edid32 {
+ __u32 pad;
+@@ -873,9 +868,7 @@ static int put_v4l2_edid32(struct v4l2_edid *p64,
+ #define VIDIOC_QUERYBUF32_TIME32 _IOWR('V', 9, struct v4l2_buffer32_time32)
+ #define VIDIOC_QBUF32_TIME32 _IOWR('V', 15, struct v4l2_buffer32_time32)
+ #define VIDIOC_DQBUF32_TIME32 _IOWR('V', 17, struct v4l2_buffer32_time32)
+-#ifdef CONFIG_X86_64
+ #define VIDIOC_DQEVENT32_TIME32 _IOR ('V', 89, struct v4l2_event32_time32)
+-#endif
+ #define VIDIOC_PREPARE_BUF32_TIME32 _IOWR('V', 93, struct v4l2_buffer32_time32)
+ #endif
+
+@@ -929,10 +922,10 @@ unsigned int v4l2_compat_translate_cmd(unsigned int cmd)
+ #ifdef CONFIG_X86_64
+ case VIDIOC_DQEVENT32:
+ return VIDIOC_DQEVENT;
++#endif
+ #ifdef CONFIG_COMPAT_32BIT_TIME
+ case VIDIOC_DQEVENT32_TIME32:
+ return VIDIOC_DQEVENT;
+-#endif
+ #endif
+ }
+ return cmd;
+@@ -1025,10 +1018,10 @@ int v4l2_compat_put_user(void __user *arg, void *parg, unsigned int cmd)
+ #ifdef CONFIG_X86_64
+ case VIDIOC_DQEVENT32:
+ return put_v4l2_event32(parg, arg);
++#endif
+ #ifdef CONFIG_COMPAT_32BIT_TIME
+ case VIDIOC_DQEVENT32_TIME32:
+ return put_v4l2_event32_time32(parg, arg);
+-#endif
+ #endif
+ }
+ return 0;
+diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c
+index c4b5082849b66..45a76f40deeb3 100644
+--- a/drivers/media/v4l2-core/v4l2-ctrls-core.c
++++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c
+@@ -113,6 +113,7 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx,
+ struct v4l2_ctrl_mpeg2_quantisation *p_mpeg2_quant;
+ struct v4l2_ctrl_vp8_frame *p_vp8_frame;
+ struct v4l2_ctrl_fwht_params *p_fwht_params;
++ struct v4l2_ctrl_h264_scaling_matrix *p_h264_scaling_matrix;
+ void *p = ptr.p + idx * ctrl->elem_size;
+
+ if (ctrl->p_def.p_const)
+@@ -160,6 +161,15 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx,
+ p_fwht_params->flags = V4L2_FWHT_FL_PIXENC_YUV |
+ (2 << V4L2_FWHT_FL_COMPONENTS_NUM_OFFSET);
+ break;
++ case V4L2_CTRL_TYPE_H264_SCALING_MATRIX:
++ p_h264_scaling_matrix = p;
++ /*
++ * The default (flat) H.264 scaling matrix when none are
++ * specified in the bitstream, this is according to formulas
++ * (7-8) and (7-9) of the specification.
++ */
++ memset(p_h264_scaling_matrix, 16, sizeof(*p_h264_scaling_matrix));
++ break;
+ }
+ }
+
+diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
+index 05d5db3d85e58..7c596a85f34f5 100644
+--- a/drivers/media/v4l2-core/v4l2-ioctl.c
++++ b/drivers/media/v4l2-core/v4l2-ioctl.c
+@@ -279,8 +279,8 @@ static void v4l_print_format(const void *arg, bool write_only)
+ const struct v4l2_vbi_format *vbi;
+ const struct v4l2_sliced_vbi_format *sliced;
+ const struct v4l2_window *win;
+- const struct v4l2_sdr_format *sdr;
+ const struct v4l2_meta_format *meta;
++ u32 pixelformat;
+ u32 planes;
+ unsigned i;
+
+@@ -299,8 +299,9 @@ static void v4l_print_format(const void *arg, bool write_only)
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ mp = &p->fmt.pix_mp;
++ pixelformat = mp->pixelformat;
+ pr_cont(", width=%u, height=%u, format=%p4cc, field=%s, colorspace=%d, num_planes=%u, flags=0x%x, ycbcr_enc=%u, quantization=%u, xfer_func=%u\n",
+- mp->width, mp->height, &mp->pixelformat,
++ mp->width, mp->height, &pixelformat,
+ prt_names(mp->field, v4l2_field_names),
+ mp->colorspace, mp->num_planes, mp->flags,
+ mp->ycbcr_enc, mp->quantization, mp->xfer_func);
+@@ -343,14 +344,15 @@ static void v4l_print_format(const void *arg, bool write_only)
+ break;
+ case V4L2_BUF_TYPE_SDR_CAPTURE:
+ case V4L2_BUF_TYPE_SDR_OUTPUT:
+- sdr = &p->fmt.sdr;
+- pr_cont(", pixelformat=%p4cc\n", &sdr->pixelformat);
++ pixelformat = p->fmt.sdr.pixelformat;
++ pr_cont(", pixelformat=%p4cc\n", &pixelformat);
+ break;
+ case V4L2_BUF_TYPE_META_CAPTURE:
+ case V4L2_BUF_TYPE_META_OUTPUT:
+ meta = &p->fmt.meta;
++ pixelformat = meta->dataformat;
+ pr_cont(", dataformat=%p4cc, buffersize=%u\n",
+- &meta->dataformat, meta->buffersize);
++ &pixelformat, meta->buffersize);
+ break;
+ }
+ }
+@@ -869,7 +871,7 @@ static void v4l_print_default(const void *arg, bool write_only)
+ pr_cont("driver-specific ioctl\n");
+ }
+
+-static int check_ext_ctrls(struct v4l2_ext_controls *c, int allow_priv)
++static bool check_ext_ctrls(struct v4l2_ext_controls *c, unsigned long ioctl)
+ {
+ __u32 i;
+
+@@ -878,23 +880,41 @@ static int check_ext_ctrls(struct v4l2_ext_controls *c, int allow_priv)
+ for (i = 0; i < c->count; i++)
+ c->controls[i].reserved2[0] = 0;
+
+- /* V4L2_CID_PRIVATE_BASE cannot be used as control class
+- when using extended controls.
+- Only when passed in through VIDIOC_G_CTRL and VIDIOC_S_CTRL
+- is it allowed for backwards compatibility.
+- */
+- if (!allow_priv && c->which == V4L2_CID_PRIVATE_BASE)
+- return 0;
+- if (!c->which)
+- return 1;
++ switch (c->which) {
++ case V4L2_CID_PRIVATE_BASE:
++ /*
++ * V4L2_CID_PRIVATE_BASE cannot be used as control class
++ * when using extended controls.
++ * Only when passed in through VIDIOC_G_CTRL and VIDIOC_S_CTRL
++ * is it allowed for backwards compatibility.
++ */
++ if (ioctl == VIDIOC_G_CTRL || ioctl == VIDIOC_S_CTRL)
++ return false;
++ break;
++ case V4L2_CTRL_WHICH_DEF_VAL:
++ /* Default value cannot be changed */
++ if (ioctl == VIDIOC_S_EXT_CTRLS ||
++ ioctl == VIDIOC_TRY_EXT_CTRLS) {
++ c->error_idx = c->count;
++ return false;
++ }
++ return true;
++ case V4L2_CTRL_WHICH_CUR_VAL:
++ return true;
++ case V4L2_CTRL_WHICH_REQUEST_VAL:
++ c->error_idx = c->count;
++ return false;
++ }
++
+ /* Check that all controls are from the same control class. */
+ for (i = 0; i < c->count; i++) {
+ if (V4L2_CTRL_ID2WHICH(c->controls[i].id) != c->which) {
+- c->error_idx = i;
+- return 0;
++ c->error_idx = ioctl == VIDIOC_TRY_EXT_CTRLS ? i :
++ c->count;
++ return false;
+ }
+ }
+- return 1;
++ return true;
+ }
+
+ static int check_fmt(struct file *file, enum v4l2_buf_type type)
+@@ -2070,6 +2090,7 @@ static int v4l_prepare_buf(const struct v4l2_ioctl_ops *ops,
+ static int v4l_g_parm(const struct v4l2_ioctl_ops *ops,
+ struct file *file, void *fh, void *arg)
+ {
++ struct video_device *vfd = video_devdata(file);
+ struct v4l2_streamparm *p = arg;
+ v4l2_std_id std;
+ int ret = check_fmt(file, p->type);
+@@ -2081,7 +2102,8 @@ static int v4l_g_parm(const struct v4l2_ioctl_ops *ops,
+ if (p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+ p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+ return -EINVAL;
+- p->parm.capture.readbuffers = 2;
++ if (vfd->device_caps & V4L2_CAP_READWRITE)
++ p->parm.capture.readbuffers = 2;
+ ret = ops->vidioc_g_std(file, fh, &std);
+ if (ret == 0)
+ v4l2_video_std_frame_period(std, &p->parm.capture.timeperframe);
+@@ -2187,7 +2209,7 @@ static int v4l_g_ctrl(const struct v4l2_ioctl_ops *ops,
+ ctrls.controls = &ctrl;
+ ctrl.id = p->id;
+ ctrl.value = p->value;
+- if (check_ext_ctrls(&ctrls, 1)) {
++ if (check_ext_ctrls(&ctrls, VIDIOC_G_CTRL)) {
+ int ret = ops->vidioc_g_ext_ctrls(file, fh, &ctrls);
+
+ if (ret == 0)
+@@ -2206,6 +2228,7 @@ static int v4l_s_ctrl(const struct v4l2_ioctl_ops *ops,
+ test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL;
+ struct v4l2_ext_controls ctrls;
+ struct v4l2_ext_control ctrl;
++ int ret;
+
+ if (vfh && vfh->ctrl_handler)
+ return v4l2_s_ctrl(vfh, vfh->ctrl_handler, p);
+@@ -2221,9 +2244,11 @@ static int v4l_s_ctrl(const struct v4l2_ioctl_ops *ops,
+ ctrls.controls = &ctrl;
+ ctrl.id = p->id;
+ ctrl.value = p->value;
+- if (check_ext_ctrls(&ctrls, 1))
+- return ops->vidioc_s_ext_ctrls(file, fh, &ctrls);
+- return -EINVAL;
++ if (!check_ext_ctrls(&ctrls, VIDIOC_S_CTRL))
++ return -EINVAL;
++ ret = ops->vidioc_s_ext_ctrls(file, fh, &ctrls);
++ p->value = ctrl.value;
++ return ret;
+ }
+
+ static int v4l_g_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+@@ -2243,8 +2268,8 @@ static int v4l_g_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+ vfd, vfd->v4l2_dev->mdev, p);
+ if (ops->vidioc_g_ext_ctrls == NULL)
+ return -ENOTTY;
+- return check_ext_ctrls(p, 0) ? ops->vidioc_g_ext_ctrls(file, fh, p) :
+- -EINVAL;
++ return check_ext_ctrls(p, VIDIOC_G_EXT_CTRLS) ?
++ ops->vidioc_g_ext_ctrls(file, fh, p) : -EINVAL;
+ }
+
+ static int v4l_s_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+@@ -2264,8 +2289,8 @@ static int v4l_s_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+ vfd, vfd->v4l2_dev->mdev, p);
+ if (ops->vidioc_s_ext_ctrls == NULL)
+ return -ENOTTY;
+- return check_ext_ctrls(p, 0) ? ops->vidioc_s_ext_ctrls(file, fh, p) :
+- -EINVAL;
++ return check_ext_ctrls(p, VIDIOC_S_EXT_CTRLS) ?
++ ops->vidioc_s_ext_ctrls(file, fh, p) : -EINVAL;
+ }
+
+ static int v4l_try_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+@@ -2285,8 +2310,8 @@ static int v4l_try_ext_ctrls(const struct v4l2_ioctl_ops *ops,
+ vfd, vfd->v4l2_dev->mdev, p);
+ if (ops->vidioc_try_ext_ctrls == NULL)
+ return -ENOTTY;
+- return check_ext_ctrls(p, 0) ? ops->vidioc_try_ext_ctrls(file, fh, p) :
+- -EINVAL;
++ return check_ext_ctrls(p, VIDIOC_TRY_EXT_CTRLS) ?
++ ops->vidioc_try_ext_ctrls(file, fh, p) : -EINVAL;
+ }
+
+ /*
+diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c
+index e7f4bf5bc8dd7..3de683b5e06d0 100644
+--- a/drivers/media/v4l2-core/v4l2-mem2mem.c
++++ b/drivers/media/v4l2-core/v4l2-mem2mem.c
+@@ -585,19 +585,14 @@ int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+ }
+ EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs);
+
+-int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+- struct v4l2_buffer *buf)
++static void v4l2_m2m_adjust_mem_offset(struct vb2_queue *vq,
++ struct v4l2_buffer *buf)
+ {
+- struct vb2_queue *vq;
+- int ret = 0;
+- unsigned int i;
+-
+- vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
+- ret = vb2_querybuf(vq, buf);
+-
+ /* Adjust MMAP memory offsets for the CAPTURE queue */
+ if (buf->memory == V4L2_MEMORY_MMAP && V4L2_TYPE_IS_CAPTURE(vq->type)) {
+ if (V4L2_TYPE_IS_MULTIPLANAR(vq->type)) {
++ unsigned int i;
++
+ for (i = 0; i < buf->length; ++i)
+ buf->m.planes[i].m.mem_offset
+ += DST_QUEUE_OFF_BASE;
+@@ -605,8 +600,23 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+ buf->m.offset += DST_QUEUE_OFF_BASE;
+ }
+ }
++}
+
+- return ret;
++int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
++ struct v4l2_buffer *buf)
++{
++ struct vb2_queue *vq;
++ int ret;
++
++ vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
++ ret = vb2_querybuf(vq, buf);
++ if (ret)
++ return ret;
++
++ /* Adjust MMAP memory offsets for the CAPTURE queue */
++ v4l2_m2m_adjust_mem_offset(vq, buf);
++
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
+
+@@ -763,6 +773,9 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+ if (ret)
+ return ret;
+
++ /* Adjust MMAP memory offsets for the CAPTURE queue */
++ v4l2_m2m_adjust_mem_offset(vq, buf);
++
+ /*
+ * If the capture queue is streaming, but streaming hasn't started
+ * on the device, but was asked to stop, mark the previously queued
+@@ -784,9 +797,17 @@ int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+ struct v4l2_buffer *buf)
+ {
+ struct vb2_queue *vq;
++ int ret;
+
+ vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
+- return vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK);
++ ret = vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK);
++ if (ret)
++ return ret;
++
++ /* Adjust MMAP memory offsets for the CAPTURE queue */
++ v4l2_m2m_adjust_mem_offset(vq, buf);
++
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf);
+
+@@ -795,9 +816,17 @@ int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+ {
+ struct video_device *vdev = video_devdata(file);
+ struct vb2_queue *vq;
++ int ret;
+
+ vq = v4l2_m2m_get_vq(m2m_ctx, buf->type);
+- return vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf);
++ ret = vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf);
++ if (ret)
++ return ret;
++
++ /* Adjust MMAP memory offsets for the CAPTURE queue */
++ v4l2_m2m_adjust_mem_offset(vq, buf);
++
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf);
+
+diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
+index c267283b01fda..e749dcb3ddea9 100644
+--- a/drivers/memory/atmel-ebi.c
++++ b/drivers/memory/atmel-ebi.c
+@@ -544,20 +544,27 @@ static int atmel_ebi_probe(struct platform_device *pdev)
+ smc_np = of_parse_phandle(dev->of_node, "atmel,smc", 0);
+
+ ebi->smc.regmap = syscon_node_to_regmap(smc_np);
+- if (IS_ERR(ebi->smc.regmap))
+- return PTR_ERR(ebi->smc.regmap);
++ if (IS_ERR(ebi->smc.regmap)) {
++ ret = PTR_ERR(ebi->smc.regmap);
++ goto put_node;
++ }
+
+ ebi->smc.layout = atmel_hsmc_get_reg_layout(smc_np);
+- if (IS_ERR(ebi->smc.layout))
+- return PTR_ERR(ebi->smc.layout);
++ if (IS_ERR(ebi->smc.layout)) {
++ ret = PTR_ERR(ebi->smc.layout);
++ goto put_node;
++ }
+
+ ebi->smc.clk = of_clk_get(smc_np, 0);
+ if (IS_ERR(ebi->smc.clk)) {
+- if (PTR_ERR(ebi->smc.clk) != -ENOENT)
+- return PTR_ERR(ebi->smc.clk);
++ if (PTR_ERR(ebi->smc.clk) != -ENOENT) {
++ ret = PTR_ERR(ebi->smc.clk);
++ goto put_node;
++ }
+
+ ebi->smc.clk = NULL;
+ }
++ of_node_put(smc_np);
+ ret = clk_prepare_enable(ebi->smc.clk);
+ if (ret)
+ return ret;
+@@ -608,6 +615,10 @@ static int atmel_ebi_probe(struct platform_device *pdev)
+ }
+
+ return of_platform_populate(np, NULL, NULL, dev);
++
++put_node:
++ of_node_put(smc_np);
++ return ret;
+ }
+
+ static __maybe_unused int atmel_ebi_resume(struct device *dev)
+diff --git a/drivers/memory/emif.c b/drivers/memory/emif.c
+index 762d0c0f0716f..ecc78d6f89ed2 100644
+--- a/drivers/memory/emif.c
++++ b/drivers/memory/emif.c
+@@ -1025,7 +1025,7 @@ static struct emif_data *__init_or_module get_device_details(
+ temp = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
+ dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL);
+
+- if (!emif || !pd || !dev_info) {
++ if (!emif || !temp || !dev_info) {
+ dev_err(dev, "%s:%d: allocation error\n", __func__, __LINE__);
+ goto error;
+ }
+@@ -1117,7 +1117,7 @@ static int __init_or_module emif_probe(struct platform_device *pdev)
+ {
+ struct emif_data *emif;
+ struct resource *res;
+- int irq;
++ int irq, ret;
+
+ if (pdev->dev.of_node)
+ emif = of_get_memory_device_details(pdev->dev.of_node, &pdev->dev);
+@@ -1147,7 +1147,9 @@ static int __init_or_module emif_probe(struct platform_device *pdev)
+ emif_onetime_settings(emif);
+ emif_debugfs_init(emif);
+ disable_and_clear_all_interrupts(emif);
+- setup_interrupts(emif, irq);
++ ret = setup_interrupts(emif, irq);
++ if (ret)
++ goto error;
+
+ /* One-time actions taken on probing the first device */
+ if (!emif1) {
+diff --git a/drivers/memory/fsl_ifc.c b/drivers/memory/fsl_ifc.c
+index d062c2f8250f4..75a8c38df9394 100644
+--- a/drivers/memory/fsl_ifc.c
++++ b/drivers/memory/fsl_ifc.c
+@@ -263,7 +263,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev)
+
+ ret = fsl_ifc_ctrl_init(fsl_ifc_ctrl_dev);
+ if (ret < 0)
+- goto err;
++ goto err_unmap_nandirq;
+
+ init_waitqueue_head(&fsl_ifc_ctrl_dev->nand_wait);
+
+@@ -272,7 +272,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev)
+ if (ret != 0) {
+ dev_err(&dev->dev, "failed to install irq (%d)\n",
+ fsl_ifc_ctrl_dev->irq);
+- goto err_irq;
++ goto err_unmap_nandirq;
+ }
+
+ if (fsl_ifc_ctrl_dev->nand_irq) {
+@@ -281,17 +281,16 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev)
+ if (ret != 0) {
+ dev_err(&dev->dev, "failed to install irq (%d)\n",
+ fsl_ifc_ctrl_dev->nand_irq);
+- goto err_nandirq;
++ goto err_free_irq;
+ }
+ }
+
+ return 0;
+
+-err_nandirq:
+- free_irq(fsl_ifc_ctrl_dev->nand_irq, fsl_ifc_ctrl_dev);
+- irq_dispose_mapping(fsl_ifc_ctrl_dev->nand_irq);
+-err_irq:
++err_free_irq:
+ free_irq(fsl_ifc_ctrl_dev->irq, fsl_ifc_ctrl_dev);
++err_unmap_nandirq:
++ irq_dispose_mapping(fsl_ifc_ctrl_dev->nand_irq);
+ irq_dispose_mapping(fsl_ifc_ctrl_dev->irq);
+ err:
+ iounmap(fsl_ifc_ctrl_dev->gregs);
+diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c
+index 45eed659b0c6d..3a416705f61cb 100644
+--- a/drivers/memory/renesas-rpc-if.c
++++ b/drivers/memory/renesas-rpc-if.c
+@@ -160,10 +160,92 @@ static const struct regmap_access_table rpcif_volatile_table = {
+ .n_yes_ranges = ARRAY_SIZE(rpcif_volatile_ranges),
+ };
+
++
++/*
++ * Custom accessor functions to ensure SM[RW]DR[01] are always accessed with
++ * proper width. Requires rpcif.xfer_size to be correctly set before!
++ */
++static int rpcif_reg_read(void *context, unsigned int reg, unsigned int *val)
++{
++ struct rpcif *rpc = context;
++
++ switch (reg) {
++ case RPCIF_SMRDR0:
++ case RPCIF_SMWDR0:
++ switch (rpc->xfer_size) {
++ case 1:
++ *val = readb(rpc->base + reg);
++ return 0;
++
++ case 2:
++ *val = readw(rpc->base + reg);
++ return 0;
++
++ case 4:
++ case 8:
++ *val = readl(rpc->base + reg);
++ return 0;
++
++ default:
++ return -EILSEQ;
++ }
++
++ case RPCIF_SMRDR1:
++ case RPCIF_SMWDR1:
++ if (rpc->xfer_size != 8)
++ return -EILSEQ;
++ break;
++ }
++
++ *val = readl(rpc->base + reg);
++ return 0;
++
++}
++
++static int rpcif_reg_write(void *context, unsigned int reg, unsigned int val)
++{
++ struct rpcif *rpc = context;
++
++ switch (reg) {
++ case RPCIF_SMWDR0:
++ switch (rpc->xfer_size) {
++ case 1:
++ writeb(val, rpc->base + reg);
++ return 0;
++
++ case 2:
++ writew(val, rpc->base + reg);
++ return 0;
++
++ case 4:
++ case 8:
++ writel(val, rpc->base + reg);
++ return 0;
++
++ default:
++ return -EILSEQ;
++ }
++
++ case RPCIF_SMWDR1:
++ if (rpc->xfer_size != 8)
++ return -EILSEQ;
++ break;
++
++ case RPCIF_SMRDR0:
++ case RPCIF_SMRDR1:
++ return -EPERM;
++ }
++
++ writel(val, rpc->base + reg);
++ return 0;
++}
++
+ static const struct regmap_config rpcif_regmap_config = {
+ .reg_bits = 32,
+ .val_bits = 32,
+ .reg_stride = 4,
++ .reg_read = rpcif_reg_read,
++ .reg_write = rpcif_reg_write,
+ .fast_io = true,
+ .max_register = RPCIF_PHYINT,
+ .volatile_table = &rpcif_volatile_table,
+@@ -173,17 +255,15 @@ int rpcif_sw_init(struct rpcif *rpc, struct device *dev)
+ {
+ struct platform_device *pdev = to_platform_device(dev);
+ struct resource *res;
+- void __iomem *base;
+
+ rpc->dev = dev;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
+- base = devm_ioremap_resource(&pdev->dev, res);
+- if (IS_ERR(base))
+- return PTR_ERR(base);
++ rpc->base = devm_ioremap_resource(&pdev->dev, res);
++ if (IS_ERR(rpc->base))
++ return PTR_ERR(rpc->base);
+
+- rpc->regmap = devm_regmap_init_mmio(&pdev->dev, base,
+- &rpcif_regmap_config);
++ rpc->regmap = devm_regmap_init(&pdev->dev, NULL, rpc, &rpcif_regmap_config);
+ if (IS_ERR(rpc->regmap)) {
+ dev_err(&pdev->dev,
+ "failed to init regmap for rpcif, error %ld\n",
+@@ -194,7 +274,7 @@ int rpcif_sw_init(struct rpcif *rpc, struct device *dev)
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dirmap");
+ rpc->dirmap = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(rpc->dirmap))
+- rpc->dirmap = NULL;
++ return PTR_ERR(rpc->dirmap);
+ rpc->size = resource_size(res);
+
+ rpc->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
+@@ -354,20 +434,16 @@ void rpcif_prepare(struct rpcif *rpc, const struct rpcif_op *op, u64 *offs,
+ nbytes = op->data.nbytes;
+ rpc->xferlen = nbytes;
+
+- rpc->enable |= RPCIF_SMENR_SPIDE(rpcif_bits_set(rpc, nbytes)) |
+- RPCIF_SMENR_SPIDB(rpcif_bit_size(op->data.buswidth));
++ rpc->enable |= RPCIF_SMENR_SPIDB(rpcif_bit_size(op->data.buswidth));
+ }
+ }
+ EXPORT_SYMBOL(rpcif_prepare);
+
+ int rpcif_manual_xfer(struct rpcif *rpc)
+ {
+- u32 smenr, smcr, pos = 0, max = 4;
++ u32 smenr, smcr, pos = 0, max = rpc->bus_size == 2 ? 8 : 4;
+ int ret = 0;
+
+- if (rpc->bus_size == 2)
+- max = 8;
+-
+ pm_runtime_get_sync(rpc->dev);
+
+ regmap_update_bits(rpc->regmap, RPCIF_PHYCNT,
+@@ -378,37 +454,37 @@ int rpcif_manual_xfer(struct rpcif *rpc)
+ regmap_write(rpc->regmap, RPCIF_SMOPR, rpc->option);
+ regmap_write(rpc->regmap, RPCIF_SMDMCR, rpc->dummy);
+ regmap_write(rpc->regmap, RPCIF_SMDRENR, rpc->ddr);
++ regmap_write(rpc->regmap, RPCIF_SMADR, rpc->smadr);
+ smenr = rpc->enable;
+
+ switch (rpc->dir) {
+ case RPCIF_DATA_OUT:
+ while (pos < rpc->xferlen) {
+- u32 nbytes = rpc->xferlen - pos;
+- u32 data[2];
++ u32 bytes_left = rpc->xferlen - pos;
++ u32 nbytes, data[2];
+
+ smcr = rpc->smcr | RPCIF_SMCR_SPIE;
+- if (nbytes > max) {
+- nbytes = max;
++
++ /* nbytes may only be 1, 2, 4, or 8 */
++ nbytes = bytes_left >= max ? max : (1 << ilog2(bytes_left));
++ if (bytes_left > nbytes)
+ smcr |= RPCIF_SMCR_SSLKP;
+- }
++
++ smenr |= RPCIF_SMENR_SPIDE(rpcif_bits_set(rpc, nbytes));
++ regmap_write(rpc->regmap, RPCIF_SMENR, smenr);
++ rpc->xfer_size = nbytes;
+
+ memcpy(data, rpc->buffer + pos, nbytes);
+- if (nbytes > 4) {
++ if (nbytes == 8) {
+ regmap_write(rpc->regmap, RPCIF_SMWDR1,
+ data[0]);
+ regmap_write(rpc->regmap, RPCIF_SMWDR0,
+ data[1]);
+- } else if (nbytes > 2) {
++ } else {
+ regmap_write(rpc->regmap, RPCIF_SMWDR0,
+ data[0]);
+- } else {
+- regmap_write(rpc->regmap, RPCIF_SMWDR0,
+- data[0] << 16);
+ }
+
+- regmap_write(rpc->regmap, RPCIF_SMADR,
+- rpc->smadr + pos);
+- regmap_write(rpc->regmap, RPCIF_SMENR, smenr);
+ regmap_write(rpc->regmap, RPCIF_SMCR, smcr);
+ ret = wait_msg_xfer_end(rpc);
+ if (ret)
+@@ -448,33 +524,32 @@ int rpcif_manual_xfer(struct rpcif *rpc)
+ break;
+ }
+ while (pos < rpc->xferlen) {
+- u32 nbytes = rpc->xferlen - pos;
+- u32 data[2];
++ u32 bytes_left = rpc->xferlen - pos;
++ u32 nbytes, data[2];
+
+- if (nbytes > max)
+- nbytes = max;
++ /* nbytes may only be 1, 2, 4, or 8 */
++ nbytes = bytes_left >= max ? max : (1 << ilog2(bytes_left));
+
+ regmap_write(rpc->regmap, RPCIF_SMADR,
+ rpc->smadr + pos);
++ smenr &= ~RPCIF_SMENR_SPIDE(0xF);
++ smenr |= RPCIF_SMENR_SPIDE(rpcif_bits_set(rpc, nbytes));
+ regmap_write(rpc->regmap, RPCIF_SMENR, smenr);
+ regmap_write(rpc->regmap, RPCIF_SMCR,
+ rpc->smcr | RPCIF_SMCR_SPIE);
++ rpc->xfer_size = nbytes;
+ ret = wait_msg_xfer_end(rpc);
+ if (ret)
+ goto err_out;
+
+- if (nbytes > 4) {
++ if (nbytes == 8) {
+ regmap_read(rpc->regmap, RPCIF_SMRDR1,
+ &data[0]);
+ regmap_read(rpc->regmap, RPCIF_SMRDR0,
+ &data[1]);
+- } else if (nbytes > 2) {
++ } else {
+ regmap_read(rpc->regmap, RPCIF_SMRDR0,
+ &data[0]);
+- } else {
+- regmap_read(rpc->regmap, RPCIF_SMRDR0,
+- &data[0]);
+- data[0] >>= 16;
+ }
+ memcpy(rpc->buffer + pos, data, nbytes);
+
+@@ -536,6 +611,7 @@ static int rpcif_probe(struct platform_device *pdev)
+ struct platform_device *vdev;
+ struct device_node *flash;
+ const char *name;
++ int ret;
+
+ flash = of_get_next_child(pdev->dev.of_node, NULL);
+ if (!flash) {
+@@ -559,7 +635,14 @@ static int rpcif_probe(struct platform_device *pdev)
+ return -ENOMEM;
+ vdev->dev.parent = &pdev->dev;
+ platform_set_drvdata(pdev, vdev);
+- return platform_device_add(vdev);
++
++ ret = platform_device_add(vdev);
++ if (ret) {
++ platform_device_put(vdev);
++ return ret;
++ }
++
++ return 0;
+ }
+
+ static int rpcif_remove(struct platform_device *pdev)
+diff --git a/drivers/memory/tegra/tegra20-emc.c b/drivers/memory/tegra/tegra20-emc.c
+index c3462dbc8c22b..6fc90f2160e93 100644
+--- a/drivers/memory/tegra/tegra20-emc.c
++++ b/drivers/memory/tegra/tegra20-emc.c
+@@ -1117,4 +1117,5 @@ module_platform_driver(tegra_emc_driver);
+
+ MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>");
+ MODULE_DESCRIPTION("NVIDIA Tegra20 EMC driver");
++MODULE_SOFTDEP("pre: governor_simpleondemand");
+ MODULE_LICENSE("GPL v2");
+diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
+index acf36676e388d..487e4cc2951e0 100644
+--- a/drivers/memstick/core/ms_block.c
++++ b/drivers/memstick/core/ms_block.c
+@@ -1736,7 +1736,7 @@ static int msb_init_card(struct memstick_dev *card)
+ msb->pages_in_block = boot_block->attr.block_size * 2;
+ msb->block_size = msb->page_size * msb->pages_in_block;
+
+- if (msb->page_size > PAGE_SIZE) {
++ if ((size_t)msb->page_size > PAGE_SIZE) {
+ /* this isn't supported by linux at all, anyway*/
+ dbg("device page %d size isn't supported", msb->page_size);
+ return -EINVAL;
+diff --git a/drivers/memstick/host/jmb38x_ms.c b/drivers/memstick/host/jmb38x_ms.c
+index f9a93b0565e15..435d4c058b20e 100644
+--- a/drivers/memstick/host/jmb38x_ms.c
++++ b/drivers/memstick/host/jmb38x_ms.c
+@@ -882,7 +882,7 @@ static struct memstick_host *jmb38x_ms_alloc_host(struct jmb38x_ms *jm, int cnt)
+
+ iounmap(host->addr);
+ err_out_free:
+- kfree(msh);
++ memstick_free_host(msh);
+ return NULL;
+ }
+
+diff --git a/drivers/memstick/host/r592.c b/drivers/memstick/host/r592.c
+index e79a0218c492e..1d35d147552d4 100644
+--- a/drivers/memstick/host/r592.c
++++ b/drivers/memstick/host/r592.c
+@@ -838,15 +838,15 @@ static void r592_remove(struct pci_dev *pdev)
+ }
+ memstick_remove_host(dev->host);
+
++ if (dev->dummy_dma_page)
++ dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->dummy_dma_page,
++ dev->dummy_dma_page_physical_address);
++
+ free_irq(dev->irq, dev);
+ iounmap(dev->mmio);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ memstick_free_host(dev->host);
+-
+- if (dev->dummy_dma_page)
+- dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->dummy_dma_page,
+- dev->dummy_dma_page_physical_address);
+ }
+
+ #ifdef CONFIG_PM_SLEEP
+diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
+index ca0edab91aeb6..d2f3452455389 100644
+--- a/drivers/mfd/Kconfig
++++ b/drivers/mfd/Kconfig
+@@ -1194,6 +1194,7 @@ config MFD_SI476X_CORE
+ config MFD_SIMPLE_MFD_I2C
+ tristate
+ depends on I2C
++ select MFD_CORE
+ select REGMAP_I2C
+ help
+ This driver creates a single register map with the intention for it
+diff --git a/drivers/mfd/altera-sysmgr.c b/drivers/mfd/altera-sysmgr.c
+index 20cb294c75122..5d3715a28b28e 100644
+--- a/drivers/mfd/altera-sysmgr.c
++++ b/drivers/mfd/altera-sysmgr.c
+@@ -153,7 +153,7 @@ static int sysmgr_probe(struct platform_device *pdev)
+ if (!base)
+ return -ENOMEM;
+
+- sysmgr_config.max_register = resource_size(res) - 3;
++ sysmgr_config.max_register = resource_size(res) - 4;
+ regmap = devm_regmap_init_mmio(dev, base, &sysmgr_config);
+ }
+
+diff --git a/drivers/mfd/asic3.c b/drivers/mfd/asic3.c
+index 8d58c8df46cfb..56338f9dbd0ba 100644
+--- a/drivers/mfd/asic3.c
++++ b/drivers/mfd/asic3.c
+@@ -906,14 +906,14 @@ static int __init asic3_mfd_probe(struct platform_device *pdev,
+ ret = mfd_add_devices(&pdev->dev, pdev->id,
+ &asic3_cell_ds1wm, 1, mem, asic->irq_base, NULL);
+ if (ret < 0)
+- goto out;
++ goto out_unmap;
+ }
+
+ if (mem_sdio && (irq >= 0)) {
+ ret = mfd_add_devices(&pdev->dev, pdev->id,
+ &asic3_cell_mmc, 1, mem_sdio, irq, NULL);
+ if (ret < 0)
+- goto out;
++ goto out_unmap;
+ }
+
+ ret = 0;
+@@ -927,8 +927,12 @@ static int __init asic3_mfd_probe(struct platform_device *pdev,
+ ret = mfd_add_devices(&pdev->dev, 0,
+ asic3_cell_leds, ASIC3_NUM_LEDS, NULL, 0, NULL);
+ }
++ return ret;
+
+- out:
++out_unmap:
++ if (asic->tmio_cnf)
++ iounmap(asic->tmio_cnf);
++out:
+ return ret;
+ }
+
+diff --git a/drivers/mfd/atmel-flexcom.c b/drivers/mfd/atmel-flexcom.c
+index d2f5c073fdf31..559eb4d352b68 100644
+--- a/drivers/mfd/atmel-flexcom.c
++++ b/drivers/mfd/atmel-flexcom.c
+@@ -87,8 +87,7 @@ static const struct of_device_id atmel_flexcom_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, atmel_flexcom_of_match);
+
+-#ifdef CONFIG_PM_SLEEP
+-static int atmel_flexcom_resume(struct device *dev)
++static int __maybe_unused atmel_flexcom_resume_noirq(struct device *dev)
+ {
+ struct atmel_flexcom *ddata = dev_get_drvdata(dev);
+ int err;
+@@ -105,16 +104,16 @@ static int atmel_flexcom_resume(struct device *dev)
+
+ return 0;
+ }
+-#endif
+
+-static SIMPLE_DEV_PM_OPS(atmel_flexcom_pm_ops, NULL,
+- atmel_flexcom_resume);
++static const struct dev_pm_ops atmel_flexcom_pm_ops = {
++ .resume_noirq = atmel_flexcom_resume_noirq,
++};
+
+ static struct platform_driver atmel_flexcom_driver = {
+ .probe = atmel_flexcom_probe,
+ .driver = {
+ .name = "atmel_flexcom",
+- .pm = &atmel_flexcom_pm_ops,
++ .pm = pm_ptr(&atmel_flexcom_pm_ops),
+ .of_match_table = atmel_flexcom_of_match,
+ },
+ };
+diff --git a/drivers/mfd/dln2.c b/drivers/mfd/dln2.c
+index 83e676a096dc1..852129ea07666 100644
+--- a/drivers/mfd/dln2.c
++++ b/drivers/mfd/dln2.c
+@@ -50,6 +50,7 @@ enum dln2_handle {
+ DLN2_HANDLE_GPIO,
+ DLN2_HANDLE_I2C,
+ DLN2_HANDLE_SPI,
++ DLN2_HANDLE_ADC,
+ DLN2_HANDLES
+ };
+
+@@ -653,6 +654,7 @@ enum {
+ DLN2_ACPI_MATCH_GPIO = 0,
+ DLN2_ACPI_MATCH_I2C = 1,
+ DLN2_ACPI_MATCH_SPI = 2,
++ DLN2_ACPI_MATCH_ADC = 3,
+ };
+
+ static struct dln2_platform_data dln2_pdata_gpio = {
+@@ -683,6 +685,16 @@ static struct mfd_cell_acpi_match dln2_acpi_match_spi = {
+ .adr = DLN2_ACPI_MATCH_SPI,
+ };
+
++/* Only one ADC port supported */
++static struct dln2_platform_data dln2_pdata_adc = {
++ .handle = DLN2_HANDLE_ADC,
++ .port = 0,
++};
++
++static struct mfd_cell_acpi_match dln2_acpi_match_adc = {
++ .adr = DLN2_ACPI_MATCH_ADC,
++};
++
+ static const struct mfd_cell dln2_devs[] = {
+ {
+ .name = "dln2-gpio",
+@@ -702,6 +714,12 @@ static const struct mfd_cell dln2_devs[] = {
+ .platform_data = &dln2_pdata_spi,
+ .pdata_size = sizeof(struct dln2_platform_data),
+ },
++ {
++ .name = "dln2-adc",
++ .acpi_match = &dln2_acpi_match_adc,
++ .platform_data = &dln2_pdata_adc,
++ .pdata_size = sizeof(struct dln2_platform_data),
++ },
+ };
+
+ static void dln2_stop(struct dln2_dev *dln2)
+diff --git a/drivers/mfd/intel-lpss-acpi.c b/drivers/mfd/intel-lpss-acpi.c
+index 3f1d976eb67cb..f2ea6540a01e1 100644
+--- a/drivers/mfd/intel-lpss-acpi.c
++++ b/drivers/mfd/intel-lpss-acpi.c
+@@ -136,6 +136,7 @@ static int intel_lpss_acpi_probe(struct platform_device *pdev)
+ {
+ struct intel_lpss_platform_info *info;
+ const struct acpi_device_id *id;
++ int ret;
+
+ id = acpi_match_device(intel_lpss_acpi_ids, &pdev->dev);
+ if (!id)
+@@ -149,10 +150,14 @@ static int intel_lpss_acpi_probe(struct platform_device *pdev)
+ info->mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ info->irq = platform_get_irq(pdev, 0);
+
++ ret = intel_lpss_probe(&pdev->dev, info);
++ if (ret)
++ return ret;
++
+ pm_runtime_set_active(&pdev->dev);
+ pm_runtime_enable(&pdev->dev);
+
+- return intel_lpss_probe(&pdev->dev, info);
++ return 0;
+ }
+
+ static int intel_lpss_acpi_remove(struct platform_device *pdev)
+diff --git a/drivers/mfd/mc13xxx-core.c b/drivers/mfd/mc13xxx-core.c
+index 1abe7432aad82..e281a9202f110 100644
+--- a/drivers/mfd/mc13xxx-core.c
++++ b/drivers/mfd/mc13xxx-core.c
+@@ -323,8 +323,10 @@ int mc13xxx_adc_do_conversion(struct mc13xxx *mc13xxx, unsigned int mode,
+ adc1 |= MC13783_ADC1_ATOX;
+
+ dev_dbg(mc13xxx->dev, "%s: request irq\n", __func__);
+- mc13xxx_irq_request(mc13xxx, MC13XXX_IRQ_ADCDONE,
++ ret = mc13xxx_irq_request(mc13xxx, MC13XXX_IRQ_ADCDONE,
+ mc13xxx_handler_adcdone, __func__, &adcdone_data);
++ if (ret)
++ goto out;
+
+ mc13xxx_reg_write(mc13xxx, MC13XXX_ADC0, adc0);
+ mc13xxx_reg_write(mc13xxx, MC13XXX_ADC1, adc1);
+diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c
+index 79f5c6a18815a..684a011a63968 100644
+--- a/drivers/mfd/mfd-core.c
++++ b/drivers/mfd/mfd-core.c
+@@ -198,6 +198,7 @@ static int mfd_add_device(struct device *parent, int id,
+ if (of_device_is_compatible(np, cell->of_compatible)) {
+ /* Ignore 'disabled' devices error free */
+ if (!of_device_is_available(np)) {
++ of_node_put(np);
+ ret = 0;
+ goto fail_alias;
+ }
+@@ -205,6 +206,7 @@ static int mfd_add_device(struct device *parent, int id,
+ ret = mfd_match_of_node_to_dev(pdev, np, cell);
+ if (ret == -EAGAIN)
+ continue;
++ of_node_put(np);
+ if (ret)
+ goto fail_alias;
+
+diff --git a/drivers/mfd/motorola-cpcap.c b/drivers/mfd/motorola-cpcap.c
+index 6fb206da27298..265464b5d7cc5 100644
+--- a/drivers/mfd/motorola-cpcap.c
++++ b/drivers/mfd/motorola-cpcap.c
+@@ -202,6 +202,13 @@ static const struct of_device_id cpcap_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, cpcap_of_match);
+
++static const struct spi_device_id cpcap_spi_ids[] = {
++ { .name = "cpcap", },
++ { .name = "6556002", },
++ {},
++};
++MODULE_DEVICE_TABLE(spi, cpcap_spi_ids);
++
+ static const struct regmap_config cpcap_regmap_config = {
+ .reg_bits = 16,
+ .reg_stride = 4,
+@@ -342,6 +349,7 @@ static struct spi_driver cpcap_driver = {
+ .pm = &cpcap_pm,
+ },
+ .probe = cpcap_probe,
++ .id_table = cpcap_spi_ids,
+ };
+ module_spi_driver(cpcap_driver);
+
+diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c
+index 6b7956604a0f0..9890882db1ed3 100644
+--- a/drivers/mfd/sprd-sc27xx-spi.c
++++ b/drivers/mfd/sprd-sc27xx-spi.c
+@@ -236,6 +236,12 @@ static const struct of_device_id sprd_pmic_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, sprd_pmic_match);
+
++static const struct spi_device_id sprd_pmic_spi_ids[] = {
++ { .name = "sc2731", .driver_data = (unsigned long)&sc2731_data },
++ {},
++};
++MODULE_DEVICE_TABLE(spi, sprd_pmic_spi_ids);
++
+ static struct spi_driver sprd_pmic_driver = {
+ .driver = {
+ .name = "sc27xx-pmic",
+@@ -243,6 +249,7 @@ static struct spi_driver sprd_pmic_driver = {
+ .pm = &sprd_pmic_pm_ops,
+ },
+ .probe = sprd_pmic_probe,
++ .id_table = sprd_pmic_spi_ids,
+ };
+
+ static int __init sprd_pmic_init(void)
+diff --git a/drivers/mfd/tps65910.c b/drivers/mfd/tps65910.c
+index 6e105cca27d47..67e2707af4bce 100644
+--- a/drivers/mfd/tps65910.c
++++ b/drivers/mfd/tps65910.c
+@@ -436,15 +436,6 @@ static void tps65910_power_off(void)
+
+ tps65910 = dev_get_drvdata(&tps65910_i2c_client->dev);
+
+- /*
+- * The PWR_OFF bit needs to be set separately, before transitioning
+- * to the OFF state. It enables the "sequential" power-off mode on
+- * TPS65911, it's a NO-OP on TPS65910.
+- */
+- if (regmap_set_bits(tps65910->regmap, TPS65910_DEVCTRL,
+- DEVCTRL_PWR_OFF_MASK) < 0)
+- return;
+-
+ regmap_update_bits(tps65910->regmap, TPS65910_DEVCTRL,
+ DEVCTRL_DEV_OFF_MASK | DEVCTRL_DEV_ON_MASK,
+ DEVCTRL_DEV_OFF_MASK);
+@@ -504,6 +495,19 @@ static int tps65910_i2c_probe(struct i2c_client *i2c,
+ tps65910_sleepinit(tps65910, pmic_plat_data);
+
+ if (pmic_plat_data->pm_off && !pm_power_off) {
++ /*
++ * The PWR_OFF bit needs to be set separately, before
++ * transitioning to the OFF state. It enables the "sequential"
++ * power-off mode on TPS65911, it's a NO-OP on TPS65910.
++ */
++ ret = regmap_set_bits(tps65910->regmap, TPS65910_DEVCTRL,
++ DEVCTRL_PWR_OFF_MASK);
++ if (ret) {
++ dev_err(&i2c->dev, "failed to set power-off mode: %d\n",
++ ret);
++ return ret;
++ }
++
+ tps65910_i2c_client = i2c;
+ pm_power_off = tps65910_power_off;
+ }
+diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c
+index de6d44a158bba..3f514d77a843f 100644
+--- a/drivers/misc/cardreader/alcor_pci.c
++++ b/drivers/misc/cardreader/alcor_pci.c
+@@ -266,7 +266,7 @@ static int alcor_pci_probe(struct pci_dev *pdev,
+ if (!priv)
+ return -ENOMEM;
+
+- ret = ida_simple_get(&alcor_pci_idr, 0, 0, GFP_KERNEL);
++ ret = ida_alloc(&alcor_pci_idr, GFP_KERNEL);
+ if (ret < 0)
+ return ret;
+ priv->id = ret;
+@@ -280,7 +280,8 @@ static int alcor_pci_probe(struct pci_dev *pdev,
+ ret = pci_request_regions(pdev, DRV_NAME_ALCOR_PCI);
+ if (ret) {
+ dev_err(&pdev->dev, "Cannot request region\n");
+- return -ENOMEM;
++ ret = -ENOMEM;
++ goto error_free_ida;
+ }
+
+ if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
+@@ -324,6 +325,8 @@ static int alcor_pci_probe(struct pci_dev *pdev,
+
+ error_release_regions:
+ pci_release_regions(pdev);
++error_free_ida:
++ ida_free(&alcor_pci_idr, priv->id);
+ return ret;
+ }
+
+@@ -337,7 +340,7 @@ static void alcor_pci_remove(struct pci_dev *pdev)
+
+ mfd_remove_devices(&pdev->dev);
+
+- ida_simple_remove(&alcor_pci_idr, priv->id);
++ ida_free(&alcor_pci_idr, priv->id);
+
+ pci_release_regions(pdev);
+ pci_set_drvdata(pdev, NULL);
+diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c
+index baf83594a01d3..5121edb0d9eff 100644
+--- a/drivers/misc/cardreader/rtsx_pcr.c
++++ b/drivers/misc/cardreader/rtsx_pcr.c
+@@ -1803,8 +1803,6 @@ static int rtsx_pci_runtime_suspend(struct device *device)
+ mutex_lock(&pcr->pcr_mutex);
+ rtsx_pci_power_off(pcr, HOST_ENTER_S3);
+
+- free_irq(pcr->irq, (void *)pcr);
+-
+ mutex_unlock(&pcr->pcr_mutex);
+
+ pcr->is_runtime_suspended = true;
+@@ -1825,8 +1823,6 @@ static int rtsx_pci_runtime_resume(struct device *device)
+ mutex_lock(&pcr->pcr_mutex);
+
+ rtsx_pci_write_register(pcr, HOST_SLEEP_STATE, 0x03, 0x00);
+- rtsx_pci_acquire_irq(pcr);
+- synchronize_irq(pcr->irq);
+
+ if (pcr->ops->fetch_vendor_settings)
+ pcr->ops->fetch_vendor_settings(pcr);
+diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
+index 632325474233a..403243859dce9 100644
+--- a/drivers/misc/eeprom/at25.c
++++ b/drivers/misc/eeprom/at25.c
+@@ -17,8 +17,6 @@
+ #include <linux/spi/spi.h>
+ #include <linux/spi/eeprom.h>
+ #include <linux/property.h>
+-#include <linux/of.h>
+-#include <linux/of_device.h>
+ #include <linux/math.h>
+
+ /*
+@@ -32,6 +30,8 @@
+ */
+
+ #define FM25_SN_LEN 8 /* serial number length */
++#define EE_MAXADDRLEN 3 /* 24 bit addresses, up to 2 MBytes */
++
+ struct at25_data {
+ struct spi_device *spi;
+ struct mutex lock;
+@@ -40,6 +40,7 @@ struct at25_data {
+ struct nvmem_config nvmem_config;
+ struct nvmem_device *nvmem;
+ u8 sernum[FM25_SN_LEN];
++ u8 command[EE_MAXADDRLEN + 1];
+ };
+
+ #define AT25_WREN 0x06 /* latch the write enable */
+@@ -62,8 +63,6 @@ struct at25_data {
+
+ #define FM25_ID_LEN 9 /* ID length */
+
+-#define EE_MAXADDRLEN 3 /* 24 bit addresses, up to 2 MBytes */
+-
+ /* Specs often allow 5 msec for a page write, sometimes 20 msec;
+ * it's important to recover from write timeouts.
+ */
+@@ -78,7 +77,6 @@ static int at25_ee_read(void *priv, unsigned int offset,
+ {
+ struct at25_data *at25 = priv;
+ char *buf = val;
+- u8 command[EE_MAXADDRLEN + 1];
+ u8 *cp;
+ ssize_t status;
+ struct spi_transfer t[2];
+@@ -92,12 +90,15 @@ static int at25_ee_read(void *priv, unsigned int offset,
+ if (unlikely(!count))
+ return -EINVAL;
+
+- cp = command;
++ cp = at25->command;
+
+ instr = AT25_READ;
+ if (at25->chip.flags & EE_INSTR_BIT3_IS_ADDR)
+ if (offset >= (1U << (at25->addrlen * 8)))
+ instr |= AT25_INSTR_BIT3;
++
++ mutex_lock(&at25->lock);
++
+ *cp++ = instr;
+
+ /* 8/16/24-bit address is written MSB first */
+@@ -116,7 +117,7 @@ static int at25_ee_read(void *priv, unsigned int offset,
+ spi_message_init(&m);
+ memset(t, 0, sizeof(t));
+
+- t[0].tx_buf = command;
++ t[0].tx_buf = at25->command;
+ t[0].len = at25->addrlen + 1;
+ spi_message_add_tail(&t[0], &m);
+
+@@ -124,8 +125,6 @@ static int at25_ee_read(void *priv, unsigned int offset,
+ t[1].len = count;
+ spi_message_add_tail(&t[1], &m);
+
+- mutex_lock(&at25->lock);
+-
+ /* Read it all at once.
+ *
+ * REVISIT that's potentially a problem with large chips, if
+@@ -153,7 +152,7 @@ static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command,
+ spi_message_init(&m);
+ memset(t, 0, sizeof(t));
+
+- t[0].tx_buf = &command;
++ t[0].tx_buf = at25->command;
+ t[0].len = 1;
+ spi_message_add_tail(&t[0], &m);
+
+@@ -163,6 +162,8 @@ static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command,
+
+ mutex_lock(&at25->lock);
+
++ at25->command[0] = command;
++
+ status = spi_sync(at25->spi, &m);
+ dev_dbg(&at25->spi->dev, "read %d aux bytes --> %d\n", len, status);
+
+@@ -376,28 +377,31 @@ MODULE_DEVICE_TABLE(spi, at25_spi_ids);
+ static int at25_probe(struct spi_device *spi)
+ {
+ struct at25_data *at25 = NULL;
+- struct spi_eeprom chip;
+ int err;
+ int sr;
+ u8 id[FM25_ID_LEN];
+ u8 sernum[FM25_SN_LEN];
++ bool is_fram;
+ int i;
+- const struct of_device_id *match;
+- bool is_fram = 0;
+
+- match = of_match_device(of_match_ptr(at25_of_match), &spi->dev);
+- if (match && !strcmp(match->compatible, "cypress,fm25"))
+- is_fram = 1;
++ err = device_property_match_string(&spi->dev, "compatible", "cypress,fm25");
++ if (err >= 0)
++ is_fram = true;
++ else
++ is_fram = false;
++
++ at25 = devm_kzalloc(&spi->dev, sizeof(struct at25_data), GFP_KERNEL);
++ if (!at25)
++ return -ENOMEM;
+
+ /* Chip description */
+- if (!spi->dev.platform_data) {
+- if (!is_fram) {
+- err = at25_fw_to_chip(&spi->dev, &chip);
+- if (err)
+- return err;
+- }
+- } else
+- chip = *(struct spi_eeprom *)spi->dev.platform_data;
++ if (spi->dev.platform_data) {
++ memcpy(&at25->chip, spi->dev.platform_data, sizeof(at25->chip));
++ } else if (!is_fram) {
++ err = at25_fw_to_chip(&spi->dev, &at25->chip);
++ if (err)
++ return err;
++ }
+
+ /* Ping the chip ... the status register is pretty portable,
+ * unlike probing manufacturer IDs. We do expect that system
+@@ -409,12 +413,7 @@ static int at25_probe(struct spi_device *spi)
+ return -ENXIO;
+ }
+
+- at25 = devm_kzalloc(&spi->dev, sizeof(struct at25_data), GFP_KERNEL);
+- if (!at25)
+- return -ENOMEM;
+-
+ mutex_init(&at25->lock);
+- at25->chip = chip;
+ at25->spi = spi;
+ spi_set_drvdata(spi, at25);
+
+@@ -431,7 +430,7 @@ static int at25_probe(struct spi_device *spi)
+ dev_err(&spi->dev, "Error: unsupported size (id %02x)\n", id[7]);
+ return -ENODEV;
+ }
+- chip.byte_len = int_pow(2, id[7] - 0x21 + 4) * 1024;
++ at25->chip.byte_len = int_pow(2, id[7] - 0x21 + 4) * 1024;
+
+ if (at25->chip.byte_len > 64 * 1024)
+ at25->chip.flags |= EE_ADDR3;
+@@ -464,7 +463,7 @@ static int at25_probe(struct spi_device *spi)
+ at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM;
+ at25->nvmem_config.name = dev_name(&spi->dev);
+ at25->nvmem_config.dev = &spi->dev;
+- at25->nvmem_config.read_only = chip.flags & EE_READONLY;
++ at25->nvmem_config.read_only = at25->chip.flags & EE_READONLY;
+ at25->nvmem_config.root_only = true;
+ at25->nvmem_config.owner = THIS_MODULE;
+ at25->nvmem_config.compat = true;
+@@ -474,17 +473,18 @@ static int at25_probe(struct spi_device *spi)
+ at25->nvmem_config.priv = at25;
+ at25->nvmem_config.stride = 1;
+ at25->nvmem_config.word_size = 1;
+- at25->nvmem_config.size = chip.byte_len;
++ at25->nvmem_config.size = at25->chip.byte_len;
+
+ at25->nvmem = devm_nvmem_register(&spi->dev, &at25->nvmem_config);
+ if (IS_ERR(at25->nvmem))
+ return PTR_ERR(at25->nvmem);
+
+ dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n",
+- (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024),
+- (chip.byte_len < 1024) ? "Byte" : "KByte",
++ (at25->chip.byte_len < 1024) ?
++ at25->chip.byte_len : (at25->chip.byte_len / 1024),
++ (at25->chip.byte_len < 1024) ? "Byte" : "KByte",
+ at25->chip.name, is_fram ? "fram" : "eeprom",
+- (chip.flags & EE_READONLY) ? " (readonly)" : "",
++ (at25->chip.flags & EE_READONLY) ? " (readonly)" : "",
+ at25->chip.page_size);
+ return 0;
+ }
+diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c
+index bb9c4512c968c..9fbfe784d7101 100644
+--- a/drivers/misc/eeprom/ee1004.c
++++ b/drivers/misc/eeprom/ee1004.c
+@@ -114,6 +114,9 @@ static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf,
+ if (offset + count > EE1004_PAGE_SIZE)
+ count = EE1004_PAGE_SIZE - offset;
+
++ if (count > I2C_SMBUS_BLOCK_MAX)
++ count = I2C_SMBUS_BLOCK_MAX;
++
+ return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf);
+ }
+
+diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
+index ad6ced4546556..86d8fb8c0148b 100644
+--- a/drivers/misc/fastrpc.c
++++ b/drivers/misc/fastrpc.c
+@@ -719,16 +719,18 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
+ static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
+ {
+ u64 size = 0;
+- int i;
++ int oix;
+
+ size = ALIGN(metalen, FASTRPC_ALIGN);
+- for (i = 0; i < ctx->nscalars; i++) {
++ for (oix = 0; oix < ctx->nbufs; oix++) {
++ int i = ctx->olaps[oix].raix;
++
+ if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
+
+- if (ctx->olaps[i].offset == 0)
++ if (ctx->olaps[oix].offset == 0)
+ size = ALIGN(size, FASTRPC_ALIGN);
+
+- size += (ctx->olaps[i].mend - ctx->olaps[i].mstart);
++ size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
+ }
+ }
+
+@@ -1284,7 +1286,14 @@ static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
+ }
+
+ if (copy_to_user(argp, &bp, sizeof(bp))) {
+- dma_buf_put(buf->dmabuf);
++ /*
++ * The usercopy failed, but we can't do much about it, as
++ * dma_buf_fd() already called fd_install() and made the
++ * file descriptor accessible for the current process. It
++ * might already be closed and dmabuf no longer valid when
++ * we reach this point. Therefore "leak" the fd and rely on
++ * the process exit path to do any required cleanup.
++ */
+ return -EFAULT;
+ }
+
+diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c
+index 985f1f3dbd20f..0b46fd22c411b 100644
+--- a/drivers/misc/habanalabs/common/debugfs.c
++++ b/drivers/misc/habanalabs/common/debugfs.c
+@@ -856,6 +856,8 @@ static ssize_t hl_set_power_state(struct file *f, const char __user *buf,
+ pci_set_power_state(hdev->pdev, PCI_D0);
+ pci_restore_state(hdev->pdev);
+ rc = pci_enable_device(hdev->pdev);
++ if (rc < 0)
++ return rc;
+ } else if (value == 2) {
+ pci_save_state(hdev->pdev);
+ pci_disable_device(hdev->pdev);
+diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c
+index 8d2568c63f19e..a8e683964ab03 100644
+--- a/drivers/misc/habanalabs/common/firmware_if.c
++++ b/drivers/misc/habanalabs/common/firmware_if.c
+@@ -1703,6 +1703,9 @@ static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
+ return rc;
+ }
+
++ /* here we can mark the descriptor as valid as the content has been validated */
++ fw_loader->dynamic_loader.fw_desc_valid = true;
++
+ return 0;
+ }
+
+@@ -1759,7 +1762,13 @@ static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
+ return rc;
+ }
+
+- /* extract address copy the descriptor from */
++ /*
++ * extract address to copy the descriptor from
++ * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
++ * as invalid.
++ * it will be marked again as valid once validated
++ */
++ fw_loader->dynamic_loader.fw_desc_valid = false;
+ src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
+ response->ram_offset;
+ memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
+@@ -2239,6 +2248,9 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
+ dev_info(hdev->dev,
+ "Loading firmware to device, may take some time...\n");
+
++ /* initialize FW descriptor as invalid */
++ fw_loader->dynamic_loader.fw_desc_valid = false;
++
+ /*
+ * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
+ * It will be updated from FW after hl_fw_dynamic_request_descriptor().
+@@ -2325,7 +2337,8 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
+ return 0;
+
+ protocol_err:
+- fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
++ if (fw_loader->dynamic_loader.fw_desc_valid)
++ fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
+ le32_to_cpu(dyn_regs->cpu_boot_err1),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
+diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
+index bebebcb163ee8..dfcd87b98ca08 100644
+--- a/drivers/misc/habanalabs/common/habanalabs.h
++++ b/drivers/misc/habanalabs/common/habanalabs.h
+@@ -992,6 +992,7 @@ struct fw_response {
+ * @image_region: region to copy the FW image to
+ * @fw_image_size: size of FW image to load
+ * @wait_for_bl_timeout: timeout for waiting for boot loader to respond
++ * @fw_desc_valid: true if FW descriptor has been validated and hence the data can be used
+ */
+ struct dynamic_fw_load_mgr {
+ struct fw_response response;
+@@ -999,6 +1000,7 @@ struct dynamic_fw_load_mgr {
+ struct pci_mem_region *image_region;
+ size_t fw_image_size;
+ u32 wait_for_bl_timeout;
++ bool fw_desc_valid;
+ };
+
+ /**
+diff --git a/drivers/misc/habanalabs/common/mmu/mmu_v1.c b/drivers/misc/habanalabs/common/mmu/mmu_v1.c
+index 0f536f79dd9c9..e68e9f71c546a 100644
+--- a/drivers/misc/habanalabs/common/mmu/mmu_v1.c
++++ b/drivers/misc/habanalabs/common/mmu/mmu_v1.c
+@@ -467,7 +467,7 @@ static void hl_mmu_v1_fini(struct hl_device *hdev)
+ {
+ /* MMU H/W fini was already done in device hw_fini() */
+
+- if (!ZERO_OR_NULL_PTR(hdev->mmu_priv.hr.mmu_shadow_hop0)) {
++ if (!ZERO_OR_NULL_PTR(hdev->mmu_priv.dr.mmu_shadow_hop0)) {
+ kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0);
+ gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool);
+
+diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c
+index 67c5b452dd356..88b91ad8e5413 100644
+--- a/drivers/misc/kgdbts.c
++++ b/drivers/misc/kgdbts.c
+@@ -1070,10 +1070,10 @@ static int kgdbts_option_setup(char *opt)
+ {
+ if (strlen(opt) >= MAX_CONFIG_LEN) {
+ printk(KERN_ERR "kgdbts: config string too long\n");
+- return -ENOSPC;
++ return 1;
+ }
+ strcpy(config, opt);
+- return 0;
++ return 1;
+ }
+
+ __setup("kgdbts=", kgdbts_option_setup);
+diff --git a/drivers/misc/lattice-ecp3-config.c b/drivers/misc/lattice-ecp3-config.c
+index 0f54730c7ed56..98828030b5a4d 100644
+--- a/drivers/misc/lattice-ecp3-config.c
++++ b/drivers/misc/lattice-ecp3-config.c
+@@ -76,12 +76,12 @@ static void firmware_load(const struct firmware *fw, void *context)
+
+ if (fw == NULL) {
+ dev_err(&spi->dev, "Cannot load firmware, aborting\n");
+- return;
++ goto out;
+ }
+
+ if (fw->size == 0) {
+ dev_err(&spi->dev, "Error: Firmware size is 0!\n");
+- return;
++ goto out;
+ }
+
+ /* Fill dummy data (24 stuffing bits for commands) */
+@@ -103,7 +103,7 @@ static void firmware_load(const struct firmware *fw, void *context)
+ dev_err(&spi->dev,
+ "Error: No supported FPGA detected (JEDEC_ID=%08x)!\n",
+ jedec_id);
+- return;
++ goto out;
+ }
+
+ dev_info(&spi->dev, "FPGA %s detected\n", ecp3_dev[i].name);
+@@ -116,7 +116,7 @@ static void firmware_load(const struct firmware *fw, void *context)
+ buffer = kzalloc(fw->size + 8, GFP_KERNEL);
+ if (!buffer) {
+ dev_err(&spi->dev, "Error: Can't allocate memory!\n");
+- return;
++ goto out;
+ }
+
+ /*
+@@ -155,7 +155,7 @@ static void firmware_load(const struct firmware *fw, void *context)
+ "Error: Timeout waiting for FPGA to clear (status=%08x)!\n",
+ status);
+ kfree(buffer);
+- return;
++ goto out;
+ }
+
+ dev_info(&spi->dev, "Configuring the FPGA...\n");
+@@ -181,7 +181,7 @@ static void firmware_load(const struct firmware *fw, void *context)
+ release_firmware(fw);
+
+ kfree(buffer);
+-
++out:
+ complete(&data->fw_loaded);
+ }
+
+diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile
+index aa12097668d33..e2984ce51fe4d 100644
+--- a/drivers/misc/lkdtm/Makefile
++++ b/drivers/misc/lkdtm/Makefile
+@@ -20,7 +20,7 @@ CFLAGS_REMOVE_rodata.o += $(CC_FLAGS_LTO)
+
+ OBJCOPYFLAGS :=
+ OBJCOPYFLAGS_rodata_objcopy.o := \
+- --rename-section .noinstr.text=.rodata,alloc,readonly,load
++ --rename-section .noinstr.text=.rodata,alloc,readonly,load,contents
+ targets += rodata.o rodata_objcopy.o
+ $(obj)/rodata_objcopy.o: $(obj)/rodata.o FORCE
+ $(call if_changed,objcopy)
+diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c
+index be41843df75bc..cebcca6d6d3ef 100644
+--- a/drivers/misc/mei/hbm.c
++++ b/drivers/misc/mei/hbm.c
+@@ -672,10 +672,14 @@ static void mei_hbm_cl_dma_map_res(struct mei_device *dev,
+ if (!cl)
+ return;
+
+- dev_dbg(dev->dev, "cl dma map result = %d\n", res->status);
+- cl->status = res->status;
+- if (!cl->status)
++ if (res->status) {
++ dev_err(dev->dev, "cl dma map failed %d\n", res->status);
++ cl->status = -EFAULT;
++ } else {
++ dev_dbg(dev->dev, "cl dma map succeeded\n");
+ cl->dma_mapped = 1;
++ cl->status = 0;
++ }
+ wake_up(&cl->wait);
+ }
+
+@@ -698,10 +702,14 @@ static void mei_hbm_cl_dma_unmap_res(struct mei_device *dev,
+ if (!cl)
+ return;
+
+- dev_dbg(dev->dev, "cl dma unmap result = %d\n", res->status);
+- cl->status = res->status;
+- if (!cl->status)
++ if (res->status) {
++ dev_err(dev->dev, "cl dma unmap failed %d\n", res->status);
++ cl->status = -EFAULT;
++ } else {
++ dev_dbg(dev->dev, "cl dma unmap succeeded\n");
+ cl->dma_mapped = 0;
++ cl->status = 0;
++ }
+ wake_up(&cl->wait);
+ }
+
+diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
+index 67bb6a25fd0a0..64ce3f830262b 100644
+--- a/drivers/misc/mei/hw-me-regs.h
++++ b/drivers/misc/mei/hw-me-regs.h
+@@ -107,6 +107,7 @@
+ #define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */
+ #define MEI_DEV_ID_ADP_LP 0x7A60 /* Alder Lake Point LP */
+ #define MEI_DEV_ID_ADP_P 0x51E0 /* Alder Lake Point P */
++#define MEI_DEV_ID_ADP_N 0x54E0 /* Alder Lake Point N */
+
+ /*
+ * MEI HW Section
+@@ -120,6 +121,7 @@
+ #define PCI_CFG_HFS_2 0x48
+ #define PCI_CFG_HFS_3 0x60
+ # define PCI_CFG_HFS_3_FW_SKU_MSK 0x00000070
++# define PCI_CFG_HFS_3_FW_SKU_IGN 0x00000000
+ # define PCI_CFG_HFS_3_FW_SKU_SPS 0x00000060
+ #define PCI_CFG_HFS_4 0x64
+ #define PCI_CFG_HFS_5 0x68
+diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
+index d3a6c07286451..fbc4c95818645 100644
+--- a/drivers/misc/mei/hw-me.c
++++ b/drivers/misc/mei/hw-me.c
+@@ -1405,16 +1405,16 @@ static bool mei_me_fw_type_sps_4(const struct pci_dev *pdev)
+ .quirk_probe = mei_me_fw_type_sps_4
+
+ /**
+- * mei_me_fw_type_sps() - check for sps sku
++ * mei_me_fw_type_sps_ign() - check for sps or ign sku
+ *
+- * Read ME FW Status register to check for SPS Firmware.
+- * The SPS FW is only signaled in pci function 0
++ * Read ME FW Status register to check for SPS or IGN Firmware.
++ * The SPS/IGN FW is only signaled in pci function 0
+ *
+ * @pdev: pci device
+ *
+- * Return: true in case of SPS firmware
++ * Return: true in case of SPS/IGN firmware
+ */
+-static bool mei_me_fw_type_sps(const struct pci_dev *pdev)
++static bool mei_me_fw_type_sps_ign(const struct pci_dev *pdev)
+ {
+ u32 reg;
+ u32 fw_type;
+@@ -1427,14 +1427,15 @@ static bool mei_me_fw_type_sps(const struct pci_dev *pdev)
+
+ dev_dbg(&pdev->dev, "fw type is %d\n", fw_type);
+
+- return fw_type == PCI_CFG_HFS_3_FW_SKU_SPS;
++ return fw_type == PCI_CFG_HFS_3_FW_SKU_IGN ||
++ fw_type == PCI_CFG_HFS_3_FW_SKU_SPS;
+ }
+
+ #define MEI_CFG_KIND_ITOUCH \
+ .kind = "itouch"
+
+-#define MEI_CFG_FW_SPS \
+- .quirk_probe = mei_me_fw_type_sps
++#define MEI_CFG_FW_SPS_IGN \
++ .quirk_probe = mei_me_fw_type_sps_ign
+
+ #define MEI_CFG_FW_VER_SUPP \
+ .fw_ver_supported = 1
+@@ -1535,7 +1536,7 @@ static const struct mei_cfg mei_me_pch12_sps_cfg = {
+ MEI_CFG_PCH8_HFS,
+ MEI_CFG_FW_VER_SUPP,
+ MEI_CFG_DMA_128,
+- MEI_CFG_FW_SPS,
++ MEI_CFG_FW_SPS_IGN,
+ };
+
+ /* Cannon Lake itouch with quirk for SPS 5.0 and newer Firmware exclusion
+@@ -1545,7 +1546,7 @@ static const struct mei_cfg mei_me_pch12_itouch_sps_cfg = {
+ MEI_CFG_KIND_ITOUCH,
+ MEI_CFG_PCH8_HFS,
+ MEI_CFG_FW_VER_SUPP,
+- MEI_CFG_FW_SPS,
++ MEI_CFG_FW_SPS_IGN,
+ };
+
+ /* Tiger Lake and newer devices */
+@@ -1562,7 +1563,7 @@ static const struct mei_cfg mei_me_pch15_sps_cfg = {
+ MEI_CFG_FW_VER_SUPP,
+ MEI_CFG_DMA_128,
+ MEI_CFG_TRC,
+- MEI_CFG_FW_SPS,
++ MEI_CFG_FW_SPS_IGN,
+ };
+
+ /*
+diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c
+index a67f4f2d33a93..0706322154cbe 100644
+--- a/drivers/misc/mei/interrupt.c
++++ b/drivers/misc/mei/interrupt.c
+@@ -424,31 +424,26 @@ int mei_irq_read_handler(struct mei_device *dev,
+ list_for_each_entry(cl, &dev->file_list, link) {
+ if (mei_cl_hbm_equal(cl, mei_hdr)) {
+ cl_dbg(dev, cl, "got a message\n");
+- break;
++ ret = mei_cl_irq_read_msg(cl, mei_hdr, meta_hdr, cmpl_list);
++ goto reset_slots;
+ }
+ }
+
+ /* if no recipient cl was found we assume corrupted header */
+- if (&cl->link == &dev->file_list) {
+- /* A message for not connected fixed address clients
+- * should be silently discarded
+- * On power down client may be force cleaned,
+- * silently discard such messages
+- */
+- if (hdr_is_fixed(mei_hdr) ||
+- dev->dev_state == MEI_DEV_POWER_DOWN) {
+- mei_irq_discard_msg(dev, mei_hdr, mei_hdr->length);
+- ret = 0;
+- goto reset_slots;
+- }
+- dev_err(dev->dev, "no destination client found 0x%08X\n",
+- dev->rd_msg_hdr[0]);
+- ret = -EBADMSG;
+- goto end;
++ /* A message for not connected fixed address clients
++ * should be silently discarded
++ * On power down client may be force cleaned,
++ * silently discard such messages
++ */
++ if (hdr_is_fixed(mei_hdr) ||
++ dev->dev_state == MEI_DEV_POWER_DOWN) {
++ mei_irq_discard_msg(dev, mei_hdr, mei_hdr->length);
++ ret = 0;
++ goto reset_slots;
+ }
+-
+- ret = mei_cl_irq_read_msg(cl, mei_hdr, meta_hdr, cmpl_list);
+-
++ dev_err(dev->dev, "no destination client found 0x%08X\n", dev->rd_msg_hdr[0]);
++ ret = -EBADMSG;
++ goto end;
+
+ reset_slots:
+ /* reset the number of slots and header */
+diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
+index 3a45aaf002ac8..a738253dbd056 100644
+--- a/drivers/misc/mei/pci-me.c
++++ b/drivers/misc/mei/pci-me.c
+@@ -113,6 +113,7 @@ static const struct pci_device_id mei_me_pci_tbl[] = {
+ {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)},
+ {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_LP, MEI_ME_PCH15_CFG)},
+ {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_P, MEI_ME_PCH15_CFG)},
++ {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_N, MEI_ME_PCH15_CFG)},
+
+ /* required last entry */
+ {0, }
+diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
+index 431af5e8be2f8..c4fcb1ad6d4d4 100644
+--- a/drivers/mmc/core/block.c
++++ b/drivers/mmc/core/block.c
+@@ -1682,31 +1682,31 @@ static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)
+ struct mmc_card *card = mq->card;
+ struct mmc_host *host = card->host;
+ blk_status_t error = BLK_STS_OK;
+- int retries = 0;
+
+ do {
+ u32 status;
+ int err;
++ int retries = 0;
+
+- mmc_blk_rw_rq_prep(mqrq, card, 1, mq);
++ while (retries++ <= MMC_READ_SINGLE_RETRIES) {
++ mmc_blk_rw_rq_prep(mqrq, card, 1, mq);
+
+- mmc_wait_for_req(host, mrq);
++ mmc_wait_for_req(host, mrq);
+
+- err = mmc_send_status(card, &status);
+- if (err)
+- goto error_exit;
+-
+- if (!mmc_host_is_spi(host) &&
+- !mmc_ready_for_data(status)) {
+- err = mmc_blk_fix_state(card, req);
++ err = mmc_send_status(card, &status);
+ if (err)
+ goto error_exit;
+- }
+
+- if (mrq->cmd->error && retries++ < MMC_READ_SINGLE_RETRIES)
+- continue;
++ if (!mmc_host_is_spi(host) &&
++ !mmc_ready_for_data(status)) {
++ err = mmc_blk_fix_state(card, req);
++ if (err)
++ goto error_exit;
++ }
+
+- retries = 0;
++ if (!mrq->cmd->error)
++ break;
++ }
+
+ if (mrq->cmd->error ||
+ mrq->data->error ||
+@@ -1880,6 +1880,31 @@ static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
+ brq->data.error || brq->cmd.resp[0] & CMD_ERRORS;
+ }
+
++static int mmc_spi_err_check(struct mmc_card *card)
++{
++ u32 status = 0;
++ int err;
++
++ /*
++ * SPI does not have a TRAN state we have to wait on, instead the
++ * card is ready again when it no longer holds the line LOW.
++ * We still have to ensure two things here before we know the write
++ * was successful:
++ * 1. The card has not disconnected during busy and we actually read our
++ * own pull-up, thinking it was still connected, so ensure it
++ * still responds.
++ * 2. Check for any error bits, in particular R1_SPI_IDLE to catch a
++ * just reconnected card after being disconnected during busy.
++ */
++ err = __mmc_send_status(card, &status, 0);
++ if (err)
++ return err;
++ /* All R1 and R2 bits of SPI are errors in our case */
++ if (status)
++ return -EIO;
++ return 0;
++}
++
+ static int mmc_blk_busy_cb(void *cb_data, bool *busy)
+ {
+ struct mmc_blk_busy_data *data = cb_data;
+@@ -1903,9 +1928,16 @@ static int mmc_blk_card_busy(struct mmc_card *card, struct request *req)
+ struct mmc_blk_busy_data cb_data;
+ int err;
+
+- if (mmc_host_is_spi(card->host) || rq_data_dir(req) == READ)
++ if (rq_data_dir(req) == READ)
+ return 0;
+
++ if (mmc_host_is_spi(card->host)) {
++ err = mmc_spi_err_check(card);
++ if (err)
++ mqrq->brq.data.bytes_xfered = 0;
++ return err;
++ }
++
+ cb_data.card = card;
+ cb_data.status = 0;
+ err = __mmc_poll_for_busy(card, MMC_BLK_TIMEOUT_MS, &mmc_blk_busy_cb,
+@@ -2344,6 +2376,8 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
+ struct mmc_blk_data *md;
+ int devidx, ret;
+ char cap_str[10];
++ bool cache_enabled = false;
++ bool fua_enabled = false;
+
+ devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL);
+ if (devidx < 0) {
+@@ -2425,13 +2459,17 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
+ md->flags |= MMC_BLK_CMD23;
+ }
+
+- if (mmc_card_mmc(card) &&
+- md->flags & MMC_BLK_CMD23 &&
++ if (md->flags & MMC_BLK_CMD23 &&
+ ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) ||
+ card->ext_csd.rel_sectors)) {
+ md->flags |= MMC_BLK_REL_WR;
+- blk_queue_write_cache(md->queue.queue, true, true);
++ fua_enabled = true;
++ cache_enabled = true;
+ }
++ if (mmc_cache_enabled(card->host))
++ cache_enabled = true;
++
++ blk_queue_write_cache(md->queue.queue, cache_enabled, fua_enabled);
+
+ string_get_size((u64)size, 512, STRING_UNITS_2,
+ cap_str, sizeof(cap_str));
+diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
+index 240c5af793dce..368f10405e132 100644
+--- a/drivers/mmc/core/core.c
++++ b/drivers/mmc/core/core.c
+@@ -2264,7 +2264,7 @@ void mmc_start_host(struct mmc_host *host)
+ _mmc_detect_change(host, 0, false);
+ }
+
+-void mmc_stop_host(struct mmc_host *host)
++void __mmc_stop_host(struct mmc_host *host)
+ {
+ if (host->slot.cd_irq >= 0) {
+ mmc_gpio_set_cd_wake(host, false);
+@@ -2273,6 +2273,11 @@ void mmc_stop_host(struct mmc_host *host)
+
+ host->rescan_disable = 1;
+ cancel_delayed_work_sync(&host->detect);
++}
++
++void mmc_stop_host(struct mmc_host *host)
++{
++ __mmc_stop_host(host);
+
+ /* clear pm flags now and let card drivers set them as needed */
+ host->pm_flags = 0;
+diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
+index 7931a4f0137d2..f5f3f623ea492 100644
+--- a/drivers/mmc/core/core.h
++++ b/drivers/mmc/core/core.h
+@@ -70,6 +70,7 @@ static inline void mmc_delay(unsigned int ms)
+
+ void mmc_rescan(struct work_struct *work);
+ void mmc_start_host(struct mmc_host *host);
++void __mmc_stop_host(struct mmc_host *host);
+ void mmc_stop_host(struct mmc_host *host);
+
+ void _mmc_detect_change(struct mmc_host *host, unsigned long delay,
+diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
+index d4683b1d263fd..d739e2b631fe8 100644
+--- a/drivers/mmc/core/host.c
++++ b/drivers/mmc/core/host.c
+@@ -80,9 +80,18 @@ static void mmc_host_classdev_release(struct device *dev)
+ kfree(host);
+ }
+
++static int mmc_host_classdev_shutdown(struct device *dev)
++{
++ struct mmc_host *host = cls_dev_to_mmc_host(dev);
++
++ __mmc_stop_host(host);
++ return 0;
++}
++
+ static struct class mmc_host_class = {
+ .name = "mmc_host",
+ .dev_release = mmc_host_classdev_release,
++ .shutdown_pre = mmc_host_classdev_shutdown,
+ .pm = MMC_HOST_CLASS_DEV_PM_OPS,
+ };
+
+@@ -579,6 +588,16 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev)
+
+ EXPORT_SYMBOL(mmc_alloc_host);
+
++static int mmc_validate_host_caps(struct mmc_host *host)
++{
++ if (host->caps & MMC_CAP_SDIO_IRQ && !host->ops->enable_sdio_irq) {
++ dev_warn(host->parent, "missing ->enable_sdio_irq() ops\n");
++ return -EINVAL;
++ }
++
++ return 0;
++}
++
+ /**
+ * mmc_add_host - initialise host hardware
+ * @host: mmc host
+@@ -591,8 +610,9 @@ int mmc_add_host(struct mmc_host *host)
+ {
+ int err;
+
+- WARN_ON((host->caps & MMC_CAP_SDIO_IRQ) &&
+- !host->ops->enable_sdio_irq);
++ err = mmc_validate_host_caps(host);
++ if (err)
++ return err;
+
+ err = device_add(&host->class_dev);
+ if (err)
+diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
+index 29e58ffae3797..d805f84507198 100644
+--- a/drivers/mmc/core/mmc.c
++++ b/drivers/mmc/core/mmc.c
+@@ -1381,13 +1381,17 @@ static int mmc_select_hs400es(struct mmc_card *card)
+ goto out_err;
+ }
+
++ /*
++ * Bump to HS timing and frequency. Some cards don't handle
++ * SEND_STATUS reliably at the initial frequency.
++ */
+ mmc_set_timing(host, MMC_TIMING_MMC_HS);
++ mmc_set_bus_speed(card);
++
+ err = mmc_switch_status(card, true);
+ if (err)
+ goto out_err;
+
+- mmc_set_clock(host, card->ext_csd.hs_max_dtr);
+-
+ /* Switch card to DDR with strobe bit */
+ val = EXT_CSD_DDR_BUS_WIDTH_8 | EXT_CSD_BUS_WIDTH_STROBE;
+ err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+@@ -1445,7 +1449,7 @@ out_err:
+ static int mmc_select_hs200(struct mmc_card *card)
+ {
+ struct mmc_host *host = card->host;
+- unsigned int old_timing, old_signal_voltage;
++ unsigned int old_timing, old_signal_voltage, old_clock;
+ int err = -EINVAL;
+ u8 val;
+
+@@ -1476,8 +1480,17 @@ static int mmc_select_hs200(struct mmc_card *card)
+ false, true, MMC_CMD_RETRIES);
+ if (err)
+ goto err;
++
++ /*
++ * Bump to HS timing and frequency. Some cards don't handle
++ * SEND_STATUS reliably at the initial frequency.
++ * NB: We can't move to full (HS200) speeds until after we've
++ * successfully switched over.
++ */
+ old_timing = host->ios.timing;
++ old_clock = host->ios.clock;
+ mmc_set_timing(host, MMC_TIMING_MMC_HS200);
++ mmc_set_clock(card->host, card->ext_csd.hs_max_dtr);
+
+ /*
+ * For HS200, CRC errors are not a reliable way to know the
+@@ -1490,8 +1503,10 @@ static int mmc_select_hs200(struct mmc_card *card)
+ * mmc_select_timing() assumes timing has not changed if
+ * it is a switch error.
+ */
+- if (err == -EBADMSG)
++ if (err == -EBADMSG) {
++ mmc_set_clock(host, old_clock);
+ mmc_set_timing(host, old_timing);
++ }
+ }
+ err:
+ if (err) {
+diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
+index 4646b7a03db6b..44e134fa04afb 100644
+--- a/drivers/mmc/core/sd.c
++++ b/drivers/mmc/core/sd.c
+@@ -66,7 +66,7 @@ static const unsigned int sd_au_size[] = {
+ __res & __mask; \
+ })
+
+-#define SD_POWEROFF_NOTIFY_TIMEOUT_MS 2000
++#define SD_POWEROFF_NOTIFY_TIMEOUT_MS 1000
+ #define SD_WRITE_EXTR_SINGLE_TIMEOUT_MS 1000
+
+ struct sd_busy_data {
+@@ -1663,6 +1663,12 @@ static int sd_poweroff_notify(struct mmc_card *card)
+ goto out;
+ }
+
++ /* Find out when the command is completed. */
++ err = mmc_poll_for_busy(card, SD_WRITE_EXTR_SINGLE_TIMEOUT_MS, false,
++ MMC_BUSY_EXTR_SINGLE);
++ if (err)
++ goto out;
++
+ cb_data.card = card;
+ cb_data.reg_buf = reg_buf;
+ err = __mmc_poll_for_busy(card, SD_POWEROFF_NOTIFY_TIMEOUT_MS,
+diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c
+index 68edf7a615be5..5447c47157aa5 100644
+--- a/drivers/mmc/core/sdio.c
++++ b/drivers/mmc/core/sdio.c
+@@ -708,6 +708,8 @@ try_again:
+ if (host->ops->init_card)
+ host->ops->init_card(host, card);
+
++ card->ocr = ocr_card;
++
+ /*
+ * If the host and card support UHS-I mode request the card
+ * to switch to 1.8V signaling level. No 1.8v signalling if
+@@ -820,7 +822,7 @@ try_again:
+ goto mismatch;
+ }
+ }
+- card->ocr = ocr_card;
++
+ mmc_fixup_device(card, sdio_fixup_methods);
+
+ if (card->type == MMC_TYPE_SD_COMBO) {
+diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
+index 2a757c88f9d21..80de660027d89 100644
+--- a/drivers/mmc/host/davinci_mmc.c
++++ b/drivers/mmc/host/davinci_mmc.c
+@@ -1375,8 +1375,12 @@ static int davinci_mmcsd_suspend(struct device *dev)
+ static int davinci_mmcsd_resume(struct device *dev)
+ {
+ struct mmc_davinci_host *host = dev_get_drvdata(dev);
++ int ret;
++
++ ret = clk_enable(host->clk);
++ if (ret)
++ return ret;
+
+- clk_enable(host->clk);
+ mmc_davinci_reset_ctrl(host, 0);
+
+ return 0;
+diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
+index 380f9aa56eb26..1e8f1bb3cad7c 100644
+--- a/drivers/mmc/host/dw_mmc.c
++++ b/drivers/mmc/host/dw_mmc.c
+@@ -2086,7 +2086,8 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
+ * delayed. Allowing the transfer to take place
+ * avoids races and keeps things simple.
+ */
+- if (err != -ETIMEDOUT) {
++ if (err != -ETIMEDOUT &&
++ host->dir_status == DW_MCI_RECV_STATUS) {
+ state = STATE_SENDING_DATA;
+ continue;
+ }
+diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
+index 8f36536cb1b6d..58ab9d90bc8b9 100644
+--- a/drivers/mmc/host/meson-gx-mmc.c
++++ b/drivers/mmc/host/meson-gx-mmc.c
+@@ -173,6 +173,8 @@ struct meson_host {
+ int irq;
+
+ bool vqmmc_enabled;
++ bool needs_pre_post_req;
++
+ };
+
+ #define CMD_CFG_LENGTH_MASK GENMASK(8, 0)
+@@ -663,6 +665,8 @@ static void meson_mmc_request_done(struct mmc_host *mmc,
+ struct meson_host *host = mmc_priv(mmc);
+
+ host->cmd = NULL;
++ if (host->needs_pre_post_req)
++ meson_mmc_post_req(mmc, mrq, 0);
+ mmc_request_done(host->mmc, mrq);
+ }
+
+@@ -880,7 +884,7 @@ static int meson_mmc_validate_dram_access(struct mmc_host *mmc, struct mmc_data
+ static void meson_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
+ {
+ struct meson_host *host = mmc_priv(mmc);
+- bool needs_pre_post_req = mrq->data &&
++ host->needs_pre_post_req = mrq->data &&
+ !(mrq->data->host_cookie & SD_EMMC_PRE_REQ_DONE);
+
+ /*
+@@ -896,22 +900,19 @@ static void meson_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
+ }
+ }
+
+- if (needs_pre_post_req) {
++ if (host->needs_pre_post_req) {
+ meson_mmc_get_transfer_mode(mmc, mrq);
+ if (!meson_mmc_desc_chain_mode(mrq->data))
+- needs_pre_post_req = false;
++ host->needs_pre_post_req = false;
+ }
+
+- if (needs_pre_post_req)
++ if (host->needs_pre_post_req)
+ meson_mmc_pre_req(mmc, mrq);
+
+ /* Stop execution */
+ writel(0, host->regs + SD_EMMC_START);
+
+ meson_mmc_start_cmd(mmc, mrq->sbc ?: mrq->cmd);
+-
+- if (needs_pre_post_req)
+- meson_mmc_post_req(mmc, mrq, 0);
+ }
+
+ static void meson_mmc_read_resp(struct mmc_host *mmc, struct mmc_command *cmd)
+diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c
+index 7cd9c0ec2fcfe..28aa78aa08f3f 100644
+--- a/drivers/mmc/host/meson-mx-sdhc-mmc.c
++++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c
+@@ -135,6 +135,7 @@ static void meson_mx_sdhc_start_cmd(struct mmc_host *mmc,
+ struct mmc_command *cmd)
+ {
+ struct meson_mx_sdhc_host *host = mmc_priv(mmc);
++ bool manual_stop = false;
+ u32 ictl, send;
+ int pack_len;
+
+@@ -172,12 +173,27 @@ static void meson_mx_sdhc_start_cmd(struct mmc_host *mmc,
+ else
+ /* software flush: */
+ ictl |= MESON_SDHC_ICTL_DATA_XFER_OK;
++
++ /*
++ * Mimic the logic from the vendor driver where (only)
++ * SD_IO_RW_EXTENDED commands with more than one block set the
++ * MESON_SDHC_MISC_MANUAL_STOP bit. This fixes the firmware
++ * download in the brcmfmac driver for a BCM43362/1 card.
++ * Without this sdio_memcpy_toio() (with a size of 219557
++ * bytes) times out if MESON_SDHC_MISC_MANUAL_STOP is not set.
++ */
++ manual_stop = cmd->data->blocks > 1 &&
++ cmd->opcode == SD_IO_RW_EXTENDED;
+ } else {
+ pack_len = 0;
+
+ ictl |= MESON_SDHC_ICTL_RESP_OK;
+ }
+
++ regmap_update_bits(host->regmap, MESON_SDHC_MISC,
++ MESON_SDHC_MISC_MANUAL_STOP,
++ manual_stop ? MESON_SDHC_MISC_MANUAL_STOP : 0);
++
+ if (cmd->opcode == MMC_STOP_TRANSMISSION)
+ send |= MESON_SDHC_SEND_DATA_STOP;
+
+@@ -838,6 +854,11 @@ static int meson_mx_sdhc_probe(struct platform_device *pdev)
+ goto err_disable_pclk;
+
+ irq = platform_get_irq(pdev, 0);
++ if (irq < 0) {
++ ret = irq;
++ goto err_disable_pclk;
++ }
++
+ ret = devm_request_threaded_irq(dev, irq, meson_mx_sdhc_irq,
+ meson_mx_sdhc_irq_thread, IRQF_ONESHOT,
+ NULL, host);
+diff --git a/drivers/mmc/host/meson-mx-sdio.c b/drivers/mmc/host/meson-mx-sdio.c
+index d4a48916bfb67..3a19a05ef55a7 100644
+--- a/drivers/mmc/host/meson-mx-sdio.c
++++ b/drivers/mmc/host/meson-mx-sdio.c
+@@ -662,6 +662,11 @@ static int meson_mx_mmc_probe(struct platform_device *pdev)
+ }
+
+ irq = platform_get_irq(pdev, 0);
++ if (irq < 0) {
++ ret = irq;
++ goto error_free_mmc;
++ }
++
+ ret = devm_request_threaded_irq(host->controller_dev, irq,
+ meson_mx_mmc_irq,
+ meson_mx_mmc_irq_thread, IRQF_ONESHOT,
+diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c
+index f4c8e1a61f537..b431cdd27353b 100644
+--- a/drivers/mmc/host/mmc_spi.c
++++ b/drivers/mmc/host/mmc_spi.c
+@@ -1514,6 +1514,12 @@ static int mmc_spi_remove(struct spi_device *spi)
+ return 0;
+ }
+
++static const struct spi_device_id mmc_spi_dev_ids[] = {
++ { "mmc-spi-slot"},
++ { },
++};
++MODULE_DEVICE_TABLE(spi, mmc_spi_dev_ids);
++
+ static const struct of_device_id mmc_spi_of_match_table[] = {
+ { .compatible = "mmc-spi-slot", },
+ {},
+@@ -1525,6 +1531,7 @@ static struct spi_driver mmc_spi_driver = {
+ .name = "mmc_spi",
+ .of_match_table = mmc_spi_of_match_table,
+ },
++ .id_table = mmc_spi_dev_ids,
+ .probe = mmc_spi_probe,
+ .remove = mmc_spi_remove,
+ };
+diff --git a/drivers/mmc/host/mmci_stm32_sdmmc.c b/drivers/mmc/host/mmci_stm32_sdmmc.c
+index fdaa11f92fe6f..4cceb9bab0361 100644
+--- a/drivers/mmc/host/mmci_stm32_sdmmc.c
++++ b/drivers/mmc/host/mmci_stm32_sdmmc.c
+@@ -62,8 +62,8 @@ static int sdmmc_idma_validate_data(struct mmci_host *host,
+ * excepted the last element which has no constraint on idmasize
+ */
+ for_each_sg(data->sg, sg, data->sg_len - 1, i) {
+- if (!IS_ALIGNED(data->sg->offset, sizeof(u32)) ||
+- !IS_ALIGNED(data->sg->length, SDMMC_IDMA_BURST)) {
++ if (!IS_ALIGNED(sg->offset, sizeof(u32)) ||
++ !IS_ALIGNED(sg->length, SDMMC_IDMA_BURST)) {
+ dev_err(mmc_dev(host->mmc),
+ "unaligned scatterlist: ofst:%x length:%d\n",
+ data->sg->offset, data->sg->length);
+@@ -71,7 +71,7 @@ static int sdmmc_idma_validate_data(struct mmci_host *host,
+ }
+ }
+
+- if (!IS_ALIGNED(data->sg->offset, sizeof(u32))) {
++ if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
+ dev_err(mmc_dev(host->mmc),
+ "unaligned last scatterlist: ofst:%x length:%d\n",
+ data->sg->offset, data->sg->length);
+@@ -441,6 +441,8 @@ static int sdmmc_dlyb_phase_tuning(struct mmci_host *host, u32 opcode)
+ return -EINVAL;
+ }
+
++ writel_relaxed(0, dlyb->base + DLYB_CR);
++
+ phase = end_of_len - max_len / 2;
+ sdmmc_dlyb_set_cfgr(dlyb, dlyb->unit, phase, false);
+
+diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c
+index 6c9d38132f74c..b6eb75f4bbfc6 100644
+--- a/drivers/mmc/host/moxart-mmc.c
++++ b/drivers/mmc/host/moxart-mmc.c
+@@ -566,37 +566,37 @@ static int moxart_probe(struct platform_device *pdev)
+ if (!mmc) {
+ dev_err(dev, "mmc_alloc_host failed\n");
+ ret = -ENOMEM;
+- goto out;
++ goto out_mmc;
+ }
+
+ ret = of_address_to_resource(node, 0, &res_mmc);
+ if (ret) {
+ dev_err(dev, "of_address_to_resource failed\n");
+- goto out;
++ goto out_mmc;
+ }
+
+ irq = irq_of_parse_and_map(node, 0);
+ if (irq <= 0) {
+ dev_err(dev, "irq_of_parse_and_map failed\n");
+ ret = -EINVAL;
+- goto out;
++ goto out_mmc;
+ }
+
+ clk = devm_clk_get(dev, NULL);
+ if (IS_ERR(clk)) {
+ ret = PTR_ERR(clk);
+- goto out;
++ goto out_mmc;
+ }
+
+ reg_mmc = devm_ioremap_resource(dev, &res_mmc);
+ if (IS_ERR(reg_mmc)) {
+ ret = PTR_ERR(reg_mmc);
+- goto out;
++ goto out_mmc;
+ }
+
+ ret = mmc_of_parse(mmc);
+ if (ret)
+- goto out;
++ goto out_mmc;
+
+ host = mmc_priv(mmc);
+ host->mmc = mmc;
+@@ -621,6 +621,14 @@ static int moxart_probe(struct platform_device *pdev)
+ ret = -EPROBE_DEFER;
+ goto out;
+ }
++ if (!IS_ERR(host->dma_chan_tx)) {
++ dma_release_channel(host->dma_chan_tx);
++ host->dma_chan_tx = NULL;
++ }
++ if (!IS_ERR(host->dma_chan_rx)) {
++ dma_release_channel(host->dma_chan_rx);
++ host->dma_chan_rx = NULL;
++ }
+ dev_dbg(dev, "PIO mode transfer enabled\n");
+ host->have_dma = false;
+ } else {
+@@ -675,6 +683,11 @@ static int moxart_probe(struct platform_device *pdev)
+ return 0;
+
+ out:
++ if (!IS_ERR_OR_NULL(host->dma_chan_tx))
++ dma_release_channel(host->dma_chan_tx);
++ if (!IS_ERR_OR_NULL(host->dma_chan_rx))
++ dma_release_channel(host->dma_chan_rx);
++out_mmc:
+ if (mmc)
+ mmc_free_host(mmc);
+ return ret;
+@@ -687,17 +700,17 @@ static int moxart_remove(struct platform_device *pdev)
+
+ dev_set_drvdata(&pdev->dev, NULL);
+
+- if (!IS_ERR(host->dma_chan_tx))
++ if (!IS_ERR_OR_NULL(host->dma_chan_tx))
+ dma_release_channel(host->dma_chan_tx);
+- if (!IS_ERR(host->dma_chan_rx))
++ if (!IS_ERR_OR_NULL(host->dma_chan_rx))
+ dma_release_channel(host->dma_chan_rx);
+ mmc_remove_host(mmc);
+- mmc_free_host(mmc);
+
+ writel(0, host->base + REG_INTERRUPT_MASK);
+ writel(0, host->base + REG_POWER_CONTROL);
+ writel(readl(host->base + REG_CLOCK_CONTROL) | CLK_OFF,
+ host->base + REG_CLOCK_CONTROL);
++ mmc_free_host(mmc);
+
+ return 0;
+ }
+diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
+index b06b4dcb7c782..1ac92015992ed 100644
+--- a/drivers/mmc/host/mtk-sd.c
++++ b/drivers/mmc/host/mtk-sd.c
+@@ -8,6 +8,7 @@
+ #include <linux/clk.h>
+ #include <linux/delay.h>
+ #include <linux/dma-mapping.h>
++#include <linux/iopoll.h>
+ #include <linux/ioport.h>
+ #include <linux/irq.h>
+ #include <linux/of_address.h>
+@@ -627,12 +628,11 @@ static void msdc_reset_hw(struct msdc_host *host)
+ u32 val;
+
+ sdr_set_bits(host->base + MSDC_CFG, MSDC_CFG_RST);
+- while (readl(host->base + MSDC_CFG) & MSDC_CFG_RST)
+- cpu_relax();
++ readl_poll_timeout(host->base + MSDC_CFG, val, !(val & MSDC_CFG_RST), 0, 0);
+
+ sdr_set_bits(host->base + MSDC_FIFOCS, MSDC_FIFOCS_CLR);
+- while (readl(host->base + MSDC_FIFOCS) & MSDC_FIFOCS_CLR)
+- cpu_relax();
++ readl_poll_timeout(host->base + MSDC_FIFOCS, val,
++ !(val & MSDC_FIFOCS_CLR), 0, 0);
+
+ val = readl(host->base + MSDC_INT);
+ writel(val, host->base + MSDC_INT);
+@@ -805,8 +805,9 @@ static void msdc_gate_clock(struct msdc_host *host)
+ clk_disable_unprepare(host->h_clk);
+ }
+
+-static void msdc_ungate_clock(struct msdc_host *host)
++static int msdc_ungate_clock(struct msdc_host *host)
+ {
++ u32 val;
+ int ret;
+
+ clk_prepare_enable(host->h_clk);
+@@ -816,11 +817,11 @@ static void msdc_ungate_clock(struct msdc_host *host)
+ ret = clk_bulk_prepare_enable(MSDC_NR_CLOCKS, host->bulk_clks);
+ if (ret) {
+ dev_err(host->dev, "Cannot enable pclk/axi/ahb clock gates\n");
+- return;
++ return ret;
+ }
+
+- while (!(readl(host->base + MSDC_CFG) & MSDC_CFG_CKSTB))
+- cpu_relax();
++ return readl_poll_timeout(host->base + MSDC_CFG, val,
++ (val & MSDC_CFG_CKSTB), 1, 20000);
+ }
+
+ static void msdc_set_mclk(struct msdc_host *host, unsigned char timing, u32 hz)
+@@ -831,6 +832,7 @@ static void msdc_set_mclk(struct msdc_host *host, unsigned char timing, u32 hz)
+ u32 div;
+ u32 sclk;
+ u32 tune_reg = host->dev_comp->pad_tune_reg;
++ u32 val;
+
+ if (!hz) {
+ dev_dbg(host->dev, "set mclk to 0\n");
+@@ -911,8 +913,7 @@ static void msdc_set_mclk(struct msdc_host *host, unsigned char timing, u32 hz)
+ else
+ clk_prepare_enable(clk_get_parent(host->src_clk));
+
+- while (!(readl(host->base + MSDC_CFG) & MSDC_CFG_CKSTB))
+- cpu_relax();
++ readl_poll_timeout(host->base + MSDC_CFG, val, (val & MSDC_CFG_CKSTB), 0, 0);
+ sdr_set_bits(host->base + MSDC_CFG, MSDC_CFG_CKPDN);
+ mmc->actual_clock = sclk;
+ host->mclk = hz;
+@@ -1222,13 +1223,13 @@ static bool msdc_cmd_done(struct msdc_host *host, int events,
+ static inline bool msdc_cmd_is_ready(struct msdc_host *host,
+ struct mmc_request *mrq, struct mmc_command *cmd)
+ {
+- /* The max busy time we can endure is 20ms */
+- unsigned long tmo = jiffies + msecs_to_jiffies(20);
++ u32 val;
++ int ret;
+
+- while ((readl(host->base + SDC_STS) & SDC_STS_CMDBUSY) &&
+- time_before(jiffies, tmo))
+- cpu_relax();
+- if (readl(host->base + SDC_STS) & SDC_STS_CMDBUSY) {
++ /* The max busy time we can endure is 20ms */
++ ret = readl_poll_timeout_atomic(host->base + SDC_STS, val,
++ !(val & SDC_STS_CMDBUSY), 1, 20000);
++ if (ret) {
+ dev_err(host->dev, "CMD bus busy detected\n");
+ host->error |= REQ_CMD_BUSY;
+ msdc_cmd_done(host, MSDC_INT_CMDTMO, mrq, cmd);
+@@ -1236,12 +1237,10 @@ static inline bool msdc_cmd_is_ready(struct msdc_host *host,
+ }
+
+ if (mmc_resp_type(cmd) == MMC_RSP_R1B || cmd->data) {
+- tmo = jiffies + msecs_to_jiffies(20);
+ /* R1B or with data, should check SDCBUSY */
+- while ((readl(host->base + SDC_STS) & SDC_STS_SDCBUSY) &&
+- time_before(jiffies, tmo))
+- cpu_relax();
+- if (readl(host->base + SDC_STS) & SDC_STS_SDCBUSY) {
++ ret = readl_poll_timeout_atomic(host->base + SDC_STS, val,
++ !(val & SDC_STS_SDCBUSY), 1, 20000);
++ if (ret) {
+ dev_err(host->dev, "Controller busy detected\n");
+ host->error |= REQ_CMD_BUSY;
+ msdc_cmd_done(host, MSDC_INT_CMDTMO, mrq, cmd);
+@@ -1366,6 +1365,8 @@ static bool msdc_data_xfer_done(struct msdc_host *host, u32 events,
+ (MSDC_INT_XFER_COMPL | MSDC_INT_DATCRCERR | MSDC_INT_DATTMO
+ | MSDC_INT_DMA_BDCSERR | MSDC_INT_DMA_GPDCSERR
+ | MSDC_INT_DMA_PROTECT);
++ u32 val;
++ int ret;
+
+ spin_lock_irqsave(&host->lock, flags);
+ done = !host->data;
+@@ -1382,8 +1383,14 @@ static bool msdc_data_xfer_done(struct msdc_host *host, u32 events,
+ readl(host->base + MSDC_DMA_CFG));
+ sdr_set_field(host->base + MSDC_DMA_CTRL, MSDC_DMA_CTRL_STOP,
+ 1);
+- while (readl(host->base + MSDC_DMA_CFG) & MSDC_DMA_CFG_STS)
+- cpu_relax();
++
++ ret = readl_poll_timeout_atomic(host->base + MSDC_DMA_CFG, val,
++ !(val & MSDC_DMA_CFG_STS), 1, 20000);
++ if (ret) {
++ dev_dbg(host->dev, "DMA stop timed out\n");
++ return false;
++ }
++
+ sdr_clr_bits(host->base + MSDC_INTEN, data_ints_mask);
+ dev_dbg(host->dev, "DMA stop\n");
+
+@@ -2330,6 +2337,7 @@ static void msdc_cqe_enable(struct mmc_host *mmc)
+ static void msdc_cqe_disable(struct mmc_host *mmc, bool recovery)
+ {
+ struct msdc_host *host = mmc_priv(mmc);
++ unsigned int val = 0;
+
+ /* disable cmdq irq */
+ sdr_clr_bits(host->base + MSDC_INTEN, MSDC_INT_CMDQ);
+@@ -2339,6 +2347,9 @@ static void msdc_cqe_disable(struct mmc_host *mmc, bool recovery)
+ if (recovery) {
+ sdr_set_field(host->base + MSDC_DMA_CTRL,
+ MSDC_DMA_CTRL_STOP, 1);
++ if (WARN_ON(readl_poll_timeout(host->base + MSDC_DMA_CFG, val,
++ !(val & MSDC_DMA_CFG_STS), 1, 3000)))
++ return;
+ msdc_reset_hw(host);
+ }
+ }
+@@ -2593,7 +2604,11 @@ static int msdc_drv_probe(struct platform_device *pdev)
+ spin_lock_init(&host->lock);
+
+ platform_set_drvdata(pdev, mmc);
+- msdc_ungate_clock(host);
++ ret = msdc_ungate_clock(host);
++ if (ret) {
++ dev_err(&pdev->dev, "Cannot ungate clocks!\n");
++ goto release_mem;
++ }
+ msdc_init_hw(host);
+
+ if (mmc->caps2 & MMC_CAP2_CQE) {
+@@ -2752,8 +2767,12 @@ static int __maybe_unused msdc_runtime_resume(struct device *dev)
+ {
+ struct mmc_host *mmc = dev_get_drvdata(dev);
+ struct msdc_host *host = mmc_priv(mmc);
++ int ret;
++
++ ret = msdc_ungate_clock(host);
++ if (ret)
++ return ret;
+
+- msdc_ungate_clock(host);
+ msdc_restore_reg(host);
+ return 0;
+ }
+diff --git a/drivers/mmc/host/mxs-mmc.c b/drivers/mmc/host/mxs-mmc.c
+index 947581de78601..8c3655d3be961 100644
+--- a/drivers/mmc/host/mxs-mmc.c
++++ b/drivers/mmc/host/mxs-mmc.c
+@@ -552,6 +552,11 @@ static const struct of_device_id mxs_mmc_dt_ids[] = {
+ };
+ MODULE_DEVICE_TABLE(of, mxs_mmc_dt_ids);
+
++static void mxs_mmc_regulator_disable(void *regulator)
++{
++ regulator_disable(regulator);
++}
++
+ static int mxs_mmc_probe(struct platform_device *pdev)
+ {
+ struct device_node *np = pdev->dev.of_node;
+@@ -591,6 +596,11 @@ static int mxs_mmc_probe(struct platform_device *pdev)
+ "Failed to enable vmmc regulator: %d\n", ret);
+ goto out_mmc_free;
+ }
++
++ ret = devm_add_action_or_reset(&pdev->dev, mxs_mmc_regulator_disable,
++ reg_vmmc);
++ if (ret)
++ goto out_mmc_free;
+ }
+
+ ssp->clk = devm_clk_get(&pdev->dev, NULL);
+diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c
+index a4407f391f66a..ae689bf54686f 100644
+--- a/drivers/mmc/host/renesas_sdhi_core.c
++++ b/drivers/mmc/host/renesas_sdhi_core.c
+@@ -382,10 +382,10 @@ static void renesas_sdhi_hs400_complete(struct mmc_host *mmc)
+ SH_MOBILE_SDHI_SCC_TMPPORT2_HS400OSEL) |
+ sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT2));
+
+- /* Set the sampling clock selection range of HS400 mode */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL,
+ SH_MOBILE_SDHI_SCC_DTCNTL_TAPEN |
+- 0x4 << SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_SHIFT);
++ sd_scc_read32(host, priv,
++ SH_MOBILE_SDHI_SCC_DTCNTL));
+
+ /* Avoid bad TAP */
+ if (bad_taps & BIT(priv->tap_set)) {
+@@ -673,7 +673,7 @@ static int renesas_sdhi_execute_tuning(struct mmc_host *mmc, u32 opcode)
+
+ /* Issue CMD19 twice for each tap */
+ for (i = 0; i < 2 * priv->tap_num; i++) {
+- int cmd_error;
++ int cmd_error = 0;
+
+ /* Set sampling clock position */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TAPSET, i % priv->tap_num);
+diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c
+index 58cfaffa3c2d8..e1580f78c6b2d 100644
+--- a/drivers/mmc/host/rtsx_pci_sdmmc.c
++++ b/drivers/mmc/host/rtsx_pci_sdmmc.c
+@@ -38,10 +38,7 @@ struct realtek_pci_sdmmc {
+ bool double_clk;
+ bool eject;
+ bool initial_mode;
+- int power_state;
+-#define SDMMC_POWER_ON 1
+-#define SDMMC_POWER_OFF 0
+-
++ int prev_power_state;
+ int sg_count;
+ s32 cookie;
+ int cookie_sg_count;
+@@ -905,7 +902,7 @@ static int sd_set_bus_width(struct realtek_pci_sdmmc *host,
+ return err;
+ }
+
+-static int sd_power_on(struct realtek_pci_sdmmc *host)
++static int sd_power_on(struct realtek_pci_sdmmc *host, unsigned char power_mode)
+ {
+ struct rtsx_pcr *pcr = host->pcr;
+ struct mmc_host *mmc = host->mmc;
+@@ -913,9 +910,14 @@ static int sd_power_on(struct realtek_pci_sdmmc *host)
+ u32 val;
+ u8 test_mode;
+
+- if (host->power_state == SDMMC_POWER_ON)
++ if (host->prev_power_state == MMC_POWER_ON)
+ return 0;
+
++ if (host->prev_power_state == MMC_POWER_UP) {
++ rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, 0);
++ goto finish;
++ }
++
+ msleep(100);
+
+ rtsx_pci_init_cmd(pcr);
+@@ -936,10 +938,15 @@ static int sd_power_on(struct realtek_pci_sdmmc *host)
+ if (err < 0)
+ return err;
+
++ mdelay(1);
++
+ err = rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, SD_OUTPUT_EN);
+ if (err < 0)
+ return err;
+
++ /* send at least 74 clocks */
++ rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, SD_CLK_TOGGLE_EN);
++
+ if (PCI_PID(pcr) == PID_5261) {
+ /*
+ * If test mode is set switch to SD Express mandatorily,
+@@ -964,7 +971,8 @@ static int sd_power_on(struct realtek_pci_sdmmc *host)
+ }
+ }
+
+- host->power_state = SDMMC_POWER_ON;
++finish:
++ host->prev_power_state = power_mode;
+ return 0;
+ }
+
+@@ -973,7 +981,7 @@ static int sd_power_off(struct realtek_pci_sdmmc *host)
+ struct rtsx_pcr *pcr = host->pcr;
+ int err;
+
+- host->power_state = SDMMC_POWER_OFF;
++ host->prev_power_state = MMC_POWER_OFF;
+
+ rtsx_pci_init_cmd(pcr);
+
+@@ -999,7 +1007,7 @@ static int sd_set_power_mode(struct realtek_pci_sdmmc *host,
+ if (power_mode == MMC_POWER_OFF)
+ err = sd_power_off(host);
+ else
+- err = sd_power_on(host);
++ err = sd_power_on(host, power_mode);
+
+ return err;
+ }
+@@ -1482,10 +1490,11 @@ static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
+
+ host = mmc_priv(mmc);
+ host->pcr = pcr;
++ mmc->ios.power_delay_ms = 5;
+ host->mmc = mmc;
+ host->pdev = pdev;
+ host->cookie = -1;
+- host->power_state = SDMMC_POWER_OFF;
++ host->prev_power_state = MMC_POWER_OFF;
+ INIT_WORK(&host->work, sd_request);
+ platform_set_drvdata(pdev, host);
+ pcr->slots[RTSX_SD_CARD].p_dev = pdev;
+@@ -1495,12 +1504,12 @@ static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
+
+ realtek_init_host(host);
+
+- if (pcr->rtd3_en) {
+- pm_runtime_set_autosuspend_delay(&pdev->dev, 5000);
+- pm_runtime_use_autosuspend(&pdev->dev);
+- pm_runtime_enable(&pdev->dev);
+- }
+-
++ pm_runtime_no_callbacks(&pdev->dev);
++ pm_runtime_set_active(&pdev->dev);
++ pm_runtime_enable(&pdev->dev);
++ pm_runtime_set_autosuspend_delay(&pdev->dev, 200);
++ pm_runtime_mark_last_busy(&pdev->dev);
++ pm_runtime_use_autosuspend(&pdev->dev);
+
+ mmc_add_host(mmc);
+
+@@ -1521,11 +1530,6 @@ static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
+ pcr->slots[RTSX_SD_CARD].card_event = NULL;
+ mmc = host->mmc;
+
+- if (pcr->rtd3_en) {
+- pm_runtime_dont_use_autosuspend(&pdev->dev);
+- pm_runtime_disable(&pdev->dev);
+- }
+-
+ cancel_work_sync(&host->work);
+
+ mutex_lock(&host->host_mutex);
+@@ -1548,6 +1552,9 @@ static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
+
+ flush_work(&host->work);
+
++ pm_runtime_dont_use_autosuspend(&pdev->dev);
++ pm_runtime_disable(&pdev->dev);
++
+ mmc_free_host(mmc);
+
+ dev_dbg(&(pdev->dev),
+diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
+index e658f01742420..60f19369de845 100644
+--- a/drivers/mmc/host/sdhci-esdhc-imx.c
++++ b/drivers/mmc/host/sdhci-esdhc-imx.c
+@@ -300,7 +300,6 @@ static struct esdhc_soc_data usdhc_imx8qxp_data = {
+ .flags = ESDHC_FLAG_USDHC | ESDHC_FLAG_STD_TUNING
+ | ESDHC_FLAG_HAVE_CAP1 | ESDHC_FLAG_HS200
+ | ESDHC_FLAG_HS400 | ESDHC_FLAG_HS400_ES
+- | ESDHC_FLAG_CQHCI
+ | ESDHC_FLAG_STATE_LOST_IN_LPMODE
+ | ESDHC_FLAG_CLK_RATE_LOST_IN_PM_RUNTIME,
+ };
+@@ -309,7 +308,6 @@ static struct esdhc_soc_data usdhc_imx8mm_data = {
+ .flags = ESDHC_FLAG_USDHC | ESDHC_FLAG_STD_TUNING
+ | ESDHC_FLAG_HAVE_CAP1 | ESDHC_FLAG_HS200
+ | ESDHC_FLAG_HS400 | ESDHC_FLAG_HS400_ES
+- | ESDHC_FLAG_CQHCI
+ | ESDHC_FLAG_STATE_LOST_IN_LPMODE,
+ };
+
+diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
+index 50c71e0ba5e4e..ff9f5b63c337e 100644
+--- a/drivers/mmc/host/sdhci-msm.c
++++ b/drivers/mmc/host/sdhci-msm.c
+@@ -17,6 +17,7 @@
+ #include <linux/regulator/consumer.h>
+ #include <linux/interconnect.h>
+ #include <linux/pinctrl/consumer.h>
++#include <linux/reset.h>
+
+ #include "sdhci-pltfm.h"
+ #include "cqhci.h"
+@@ -2482,6 +2483,43 @@ static inline void sdhci_msm_get_of_property(struct platform_device *pdev,
+ of_property_read_u32(node, "qcom,dll-config", &msm_host->dll_config);
+ }
+
++static int sdhci_msm_gcc_reset(struct device *dev, struct sdhci_host *host)
++{
++ struct reset_control *reset;
++ int ret = 0;
++
++ reset = reset_control_get_optional_exclusive(dev, NULL);
++ if (IS_ERR(reset))
++ return dev_err_probe(dev, PTR_ERR(reset),
++ "unable to acquire core_reset\n");
++
++ if (!reset)
++ return ret;
++
++ ret = reset_control_assert(reset);
++ if (ret) {
++ reset_control_put(reset);
++ return dev_err_probe(dev, ret, "core_reset assert failed\n");
++ }
++
++ /*
++ * The hardware requirement for delay between assert/deassert
++ * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to
++ * ~125us (4/32768). To be on the safe side add 200us delay.
++ */
++ usleep_range(200, 210);
++
++ ret = reset_control_deassert(reset);
++ if (ret) {
++ reset_control_put(reset);
++ return dev_err_probe(dev, ret, "core_reset deassert failed\n");
++ }
++
++ usleep_range(200, 210);
++ reset_control_put(reset);
++
++ return ret;
++}
+
+ static int sdhci_msm_probe(struct platform_device *pdev)
+ {
+@@ -2529,6 +2567,10 @@ static int sdhci_msm_probe(struct platform_device *pdev)
+
+ msm_host->saved_tuning_phase = INVALID_TUNING_PHASE;
+
++ ret = sdhci_msm_gcc_reset(&pdev->dev, host);
++ if (ret)
++ goto pltfm_free;
++
+ /* Setup SDCC bus voter clock. */
+ msm_host->bus_clk = devm_clk_get(&pdev->dev, "bus");
+ if (!IS_ERR(msm_host->bus_clk)) {
+diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
+index a593b1fbd69e5..0f3658b36513c 100644
+--- a/drivers/mmc/host/sdhci-of-esdhc.c
++++ b/drivers/mmc/host/sdhci-of-esdhc.c
+@@ -524,12 +524,16 @@ static void esdhc_of_adma_workaround(struct sdhci_host *host, u32 intmask)
+
+ static int esdhc_of_enable_dma(struct sdhci_host *host)
+ {
++ int ret;
+ u32 value;
+ struct device *dev = mmc_dev(host->mmc);
+
+ if (of_device_is_compatible(dev->of_node, "fsl,ls1043a-esdhc") ||
+- of_device_is_compatible(dev->of_node, "fsl,ls1046a-esdhc"))
+- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
++ of_device_is_compatible(dev->of_node, "fsl,ls1046a-esdhc")) {
++ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
++ if (ret)
++ return ret;
++ }
+
+ value = sdhci_readl(host, ESDHC_DMA_SYSCTL);
+
+diff --git a/drivers/mmc/host/sdhci-omap.c b/drivers/mmc/host/sdhci-omap.c
+index 8f4d1f003f656..fd188b6d88f49 100644
+--- a/drivers/mmc/host/sdhci-omap.c
++++ b/drivers/mmc/host/sdhci-omap.c
+@@ -62,6 +62,8 @@
+ #define SDHCI_OMAP_IE 0x234
+ #define INT_CC_EN BIT(0)
+
++#define SDHCI_OMAP_ISE 0x238
++
+ #define SDHCI_OMAP_AC12 0x23c
+ #define AC12_V1V8_SIGEN BIT(19)
+ #define AC12_SCLK_SEL BIT(23)
+@@ -113,6 +115,8 @@ struct sdhci_omap_host {
+ u32 hctl;
+ u32 sysctl;
+ u32 capa;
++ u32 ie;
++ u32 ise;
+ };
+
+ static void sdhci_omap_start_clock(struct sdhci_omap_host *omap_host);
+@@ -682,7 +686,8 @@ static void sdhci_omap_set_power(struct sdhci_host *host, unsigned char mode,
+ {
+ struct mmc_host *mmc = host->mmc;
+
+- mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
++ if (!IS_ERR(mmc->supply.vmmc))
++ mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
+ }
+
+ static int sdhci_omap_enable_dma(struct sdhci_host *host)
+@@ -1244,14 +1249,23 @@ static void sdhci_omap_context_save(struct sdhci_omap_host *omap_host)
+ {
+ omap_host->con = sdhci_omap_readl(omap_host, SDHCI_OMAP_CON);
+ omap_host->hctl = sdhci_omap_readl(omap_host, SDHCI_OMAP_HCTL);
++ omap_host->sysctl = sdhci_omap_readl(omap_host, SDHCI_OMAP_SYSCTL);
+ omap_host->capa = sdhci_omap_readl(omap_host, SDHCI_OMAP_CAPA);
++ omap_host->ie = sdhci_omap_readl(omap_host, SDHCI_OMAP_IE);
++ omap_host->ise = sdhci_omap_readl(omap_host, SDHCI_OMAP_ISE);
+ }
+
++/* Order matters here, HCTL must be restored in two phases */
+ static void sdhci_omap_context_restore(struct sdhci_omap_host *omap_host)
+ {
+- sdhci_omap_writel(omap_host, SDHCI_OMAP_CON, omap_host->con);
+ sdhci_omap_writel(omap_host, SDHCI_OMAP_HCTL, omap_host->hctl);
+ sdhci_omap_writel(omap_host, SDHCI_OMAP_CAPA, omap_host->capa);
++ sdhci_omap_writel(omap_host, SDHCI_OMAP_HCTL, omap_host->hctl);
++
++ sdhci_omap_writel(omap_host, SDHCI_OMAP_SYSCTL, omap_host->sysctl);
++ sdhci_omap_writel(omap_host, SDHCI_OMAP_CON, omap_host->con);
++ sdhci_omap_writel(omap_host, SDHCI_OMAP_IE, omap_host->ie);
++ sdhci_omap_writel(omap_host, SDHCI_OMAP_ISE, omap_host->ise);
+ }
+
+ static int __maybe_unused sdhci_omap_suspend(struct device *dev)
+diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
+index d0f2edfe296c8..c2b26ada104d6 100644
+--- a/drivers/mmc/host/sdhci-pci-core.c
++++ b/drivers/mmc/host/sdhci-pci-core.c
+@@ -1951,6 +1951,7 @@ static const struct pci_device_id pci_ids[] = {
+ SDHCI_PCI_DEVICE(INTEL, JSL_SD, intel_byt_sd),
+ SDHCI_PCI_DEVICE(INTEL, LKF_EMMC, intel_glk_emmc),
+ SDHCI_PCI_DEVICE(INTEL, LKF_SD, intel_byt_sd),
++ SDHCI_PCI_DEVICE(INTEL, ADL_EMMC, intel_glk_emmc),
+ SDHCI_PCI_DEVICE(O2, 8120, o2),
+ SDHCI_PCI_DEVICE(O2, 8220, o2),
+ SDHCI_PCI_DEVICE(O2, 8221, o2),
+diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c
+index 4fd99c1e82ba3..ad50f16658fe2 100644
+--- a/drivers/mmc/host/sdhci-pci-gli.c
++++ b/drivers/mmc/host/sdhci-pci-gli.c
+@@ -12,6 +12,7 @@
+ #include <linux/pci.h>
+ #include <linux/mmc/mmc.h>
+ #include <linux/delay.h>
++#include <linux/of.h>
+ #include "sdhci.h"
+ #include "sdhci-pci.h"
+ #include "cqhci.h"
+@@ -116,6 +117,8 @@
+ #define PCI_GLI_9755_PECONF 0x44
+ #define PCI_GLI_9755_LFCLK GENMASK(14, 12)
+ #define PCI_GLI_9755_DMACLK BIT(29)
++#define PCI_GLI_9755_INVERT_CD BIT(30)
++#define PCI_GLI_9755_INVERT_WP BIT(31)
+
+ #define PCI_GLI_9755_CFG2 0x48
+ #define PCI_GLI_9755_CFG2_L1DLY GENMASK(28, 24)
+@@ -570,6 +573,14 @@ static void gl9755_hw_setting(struct sdhci_pci_slot *slot)
+ gl9755_wt_on(pdev);
+
+ pci_read_config_dword(pdev, PCI_GLI_9755_PECONF, &value);
++ /*
++ * Apple ARM64 platforms using these chips may have
++ * inverted CD/WP detection.
++ */
++ if (of_property_read_bool(pdev->dev.of_node, "cd-inverted"))
++ value |= PCI_GLI_9755_INVERT_CD;
++ if (of_property_read_bool(pdev->dev.of_node, "wp-inverted"))
++ value |= PCI_GLI_9755_INVERT_WP;
+ value &= ~PCI_GLI_9755_LFCLK;
+ value &= ~PCI_GLI_9755_DMACLK;
+ pci_write_config_dword(pdev, PCI_GLI_9755_PECONF, value);
+diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h
+index 8f90c4163bb5c..dcd99d5057ee1 100644
+--- a/drivers/mmc/host/sdhci-pci.h
++++ b/drivers/mmc/host/sdhci-pci.h
+@@ -59,6 +59,7 @@
+ #define PCI_DEVICE_ID_INTEL_JSL_SD 0x4df8
+ #define PCI_DEVICE_ID_INTEL_LKF_EMMC 0x98c4
+ #define PCI_DEVICE_ID_INTEL_LKF_SD 0x98f8
++#define PCI_DEVICE_ID_INTEL_ADL_EMMC 0x54c4
+
+ #define PCI_DEVICE_ID_SYSKONNECT_8000 0x8000
+ #define PCI_DEVICE_ID_VIA_95D0 0x95d0
+diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c
+index a5001875876b9..9762ffab2e236 100644
+--- a/drivers/mmc/host/sdhci-tegra.c
++++ b/drivers/mmc/host/sdhci-tegra.c
+@@ -356,23 +356,6 @@ static void tegra_sdhci_set_tap(struct sdhci_host *host, unsigned int tap)
+ }
+ }
+
+-static void tegra_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
+- struct mmc_ios *ios)
+-{
+- struct sdhci_host *host = mmc_priv(mmc);
+- u32 val;
+-
+- val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
+-
+- if (ios->enhanced_strobe)
+- val |= SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
+- else
+- val &= ~SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
+-
+- sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
+-
+-}
+-
+ static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask)
+ {
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+@@ -793,6 +776,32 @@ static void tegra_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+ }
+ }
+
++static void tegra_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
++ struct mmc_ios *ios)
++{
++ struct sdhci_host *host = mmc_priv(mmc);
++ u32 val;
++
++ val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
++
++ if (ios->enhanced_strobe) {
++ val |= SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
++ /*
++ * When CMD13 is sent from mmc_select_hs400es() after
++ * switching to HS400ES mode, the bus is operating at
++ * either MMC_HIGH_26_MAX_DTR or MMC_HIGH_52_MAX_DTR.
++ * To meet Tegra SDHCI requirement at HS400ES mode, force SDHCI
++ * interface clock to MMC_HS200_MAX_DTR (200 MHz) so that host
++ * controller CAR clock and the interface clock are rate matched.
++ */
++ tegra_sdhci_set_clock(host, MMC_HS200_MAX_DTR);
++ } else {
++ val &= ~SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
++ }
++
++ sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
++}
++
+ static unsigned int tegra_sdhci_get_max_clock(struct sdhci_host *host)
+ {
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
+index 666cee4c7f7c6..08e838400b526 100644
+--- a/drivers/mmc/host/sdhci-xenon.c
++++ b/drivers/mmc/host/sdhci-xenon.c
+@@ -241,16 +241,6 @@ static void xenon_voltage_switch(struct sdhci_host *host)
+ {
+ /* Wait for 5ms after set 1.8V signal enable bit */
+ usleep_range(5000, 5500);
+-
+- /*
+- * For some reason the controller's Host Control2 register reports
+- * the bit representing 1.8V signaling as 0 when read after it was
+- * written as 1. Subsequent read reports 1.
+- *
+- * Since this may cause some issues, do an empty read of the Host
+- * Control2 register here to circumvent this.
+- */
+- sdhci_readw(host, SDHCI_HOST_CONTROL2);
+ }
+
+ static unsigned int xenon_get_max_clock(struct sdhci_host *host)
+diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
+index 2d80a04e11d87..7728f26adb19f 100644
+--- a/drivers/mmc/host/sdhci.c
++++ b/drivers/mmc/host/sdhci.c
+@@ -771,7 +771,19 @@ static void sdhci_adma_table_pre(struct sdhci_host *host,
+ len -= offset;
+ }
+
+- BUG_ON(len > 65536);
++ /*
++ * The block layer forces a minimum segment size of PAGE_SIZE,
++ * so 'len' can be too big here if PAGE_SIZE >= 64KiB. Write
++ * multiple descriptors, noting that the ADMA table is sized
++ * for 4KiB chunks anyway, so it will be big enough.
++ */
++ while (len > host->max_adma) {
++ int n = 32 * 1024; /* 32KiB*/
++
++ __sdhci_adma_write_desc(host, &desc, addr, n, ADMA2_TRAN_VALID);
++ addr += n;
++ len -= n;
++ }
+
+ /* tran, valid */
+ if (len)
+@@ -3952,6 +3964,7 @@ struct sdhci_host *sdhci_alloc_host(struct device *dev,
+ * descriptor for each segment, plus 1 for a nop end descriptor.
+ */
+ host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1;
++ host->max_adma = 65536;
+
+ host->max_timeout_count = 0xE;
+
+@@ -4617,10 +4630,12 @@ int sdhci_setup_host(struct sdhci_host *host)
+ * be larger than 64 KiB though.
+ */
+ if (host->flags & SDHCI_USE_ADMA) {
+- if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
++ if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC) {
++ host->max_adma = 65532; /* 32-bit alignment */
+ mmc->max_seg_size = 65535;
+- else
++ } else {
+ mmc->max_seg_size = 65536;
++ }
+ } else {
+ mmc->max_seg_size = mmc->max_req_size;
+ }
+diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
+index e8d04e42a5afd..6c689be3e48f6 100644
+--- a/drivers/mmc/host/sdhci.h
++++ b/drivers/mmc/host/sdhci.h
+@@ -340,7 +340,8 @@ struct sdhci_adma2_64_desc {
+
+ /*
+ * Maximum segments assuming a 512KiB maximum requisition size and a minimum
+- * 4KiB page size.
++ * 4KiB page size. Note this also allows enough for multiple descriptors in
++ * case of PAGE_SIZE >= 64KiB.
+ */
+ #define SDHCI_MAX_SEGS 128
+
+@@ -543,6 +544,7 @@ struct sdhci_host {
+ unsigned int blocks; /* remaining PIO blocks */
+
+ int sg_count; /* Mapped sg entries */
++ int max_adma; /* Max. length in ADMA descriptor */
+
+ void *adma_table; /* ADMA descriptor table */
+ void *align_buffer; /* Bounce buffer */
+diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c
+index f654afbe8e83c..b4891bb266485 100644
+--- a/drivers/mmc/host/sdhci_am654.c
++++ b/drivers/mmc/host/sdhci_am654.c
+@@ -514,26 +514,6 @@ static const struct sdhci_am654_driver_data sdhci_j721e_4bit_drvdata = {
+ .flags = IOMUX_PRESENT,
+ };
+
+-static const struct sdhci_pltfm_data sdhci_am64_8bit_pdata = {
+- .ops = &sdhci_j721e_8bit_ops,
+- .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+-};
+-
+-static const struct sdhci_am654_driver_data sdhci_am64_8bit_drvdata = {
+- .pdata = &sdhci_am64_8bit_pdata,
+- .flags = DLL_PRESENT | DLL_CALIB,
+-};
+-
+-static const struct sdhci_pltfm_data sdhci_am64_4bit_pdata = {
+- .ops = &sdhci_j721e_4bit_ops,
+- .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+-};
+-
+-static const struct sdhci_am654_driver_data sdhci_am64_4bit_drvdata = {
+- .pdata = &sdhci_am64_4bit_pdata,
+- .flags = IOMUX_PRESENT,
+-};
+-
+ static const struct soc_device_attribute sdhci_am654_devices[] = {
+ { .family = "AM65X",
+ .revision = "SR1.0",
+@@ -759,11 +739,11 @@ static const struct of_device_id sdhci_am654_of_match[] = {
+ },
+ {
+ .compatible = "ti,am64-sdhci-8bit",
+- .data = &sdhci_am64_8bit_drvdata,
++ .data = &sdhci_j721e_8bit_drvdata,
+ },
+ {
+ .compatible = "ti,am64-sdhci-4bit",
+- .data = &sdhci_am64_4bit_drvdata,
++ .data = &sdhci_j721e_4bit_drvdata,
+ },
+ { /* sentinel */ }
+ };
+diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
+index 2702736a1c57d..ce6cb8be654ef 100644
+--- a/drivers/mmc/host/sunxi-mmc.c
++++ b/drivers/mmc/host/sunxi-mmc.c
+@@ -377,8 +377,9 @@ static void sunxi_mmc_init_idma_des(struct sunxi_mmc_host *host,
+ pdes[i].buf_addr_ptr1 =
+ cpu_to_le32(sg_dma_address(&data->sg[i]) >>
+ host->cfg->idma_des_shift);
+- pdes[i].buf_addr_ptr2 = cpu_to_le32((u32)next_desc >>
+- host->cfg->idma_des_shift);
++ pdes[i].buf_addr_ptr2 =
++ cpu_to_le32(next_desc >>
++ host->cfg->idma_des_shift);
+ }
+
+ pdes[0].config |= cpu_to_le32(SDXC_IDMAC_DES0_FD);
+diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c
+index e2affa52ef469..a5850d83908be 100644
+--- a/drivers/mmc/host/tmio_mmc_core.c
++++ b/drivers/mmc/host/tmio_mmc_core.c
+@@ -960,14 +960,8 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+ case MMC_POWER_OFF:
+ tmio_mmc_power_off(host);
+ /* For R-Car Gen2+, we need to reset SDHI specific SCC */
+- if (host->pdata->flags & TMIO_MMC_MIN_RCAR2) {
+- host->reset(host);
+-
+- if (host->native_hotplug)
+- tmio_mmc_enable_mmc_irqs(host,
+- TMIO_STAT_CARD_REMOVE |
+- TMIO_STAT_CARD_INSERT);
+- }
++ if (host->pdata->flags & TMIO_MMC_MIN_RCAR2)
++ tmio_mmc_reset(host);
+
+ host->set_clock(host, 0);
+ break;
+@@ -1175,6 +1169,7 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host)
+ if (mmc_can_gpio_cd(mmc))
+ _host->ops.get_cd = mmc_gpio_get_cd;
+
++ /* must be set before tmio_mmc_reset() */
+ _host->native_hotplug = !(mmc_can_gpio_cd(mmc) ||
+ mmc->caps & MMC_CAP_NEEDS_POLL ||
+ !mmc_card_is_removable(mmc));
+@@ -1295,10 +1290,6 @@ int tmio_mmc_host_runtime_resume(struct device *dev)
+ if (host->clk_cache)
+ host->set_clock(host, host->clk_cache);
+
+- if (host->native_hotplug)
+- tmio_mmc_enable_mmc_irqs(host,
+- TMIO_STAT_CARD_REMOVE | TMIO_STAT_CARD_INSERT);
+-
+ tmio_mmc_enable_dma(host, true);
+
+ return 0;
+diff --git a/drivers/most/most_usb.c b/drivers/most/most_usb.c
+index 2640c5b326a49..acabb7715b423 100644
+--- a/drivers/most/most_usb.c
++++ b/drivers/most/most_usb.c
+@@ -149,7 +149,8 @@ static inline int drci_rd_reg(struct usb_device *dev, u16 reg, u16 *buf)
+ retval = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
+ DRCI_READ_REQ, req_type,
+ 0x0000,
+- reg, dma_buf, sizeof(*dma_buf), 5 * HZ);
++ reg, dma_buf, sizeof(*dma_buf),
++ USB_CTRL_GET_TIMEOUT);
+ *buf = le16_to_cpu(*dma_buf);
+ kfree(dma_buf);
+
+@@ -176,7 +177,7 @@ static inline int drci_wr_reg(struct usb_device *dev, u16 reg, u16 data)
+ reg,
+ NULL,
+ 0,
+- 5 * HZ);
++ USB_CTRL_SET_TIMEOUT);
+ }
+
+ static inline int start_sync_ep(struct usb_device *usb_dev, u16 ep)
+diff --git a/drivers/mtd/chips/Kconfig b/drivers/mtd/chips/Kconfig
+index aef14990e5f7c..19726ebd973d0 100644
+--- a/drivers/mtd/chips/Kconfig
++++ b/drivers/mtd/chips/Kconfig
+@@ -55,12 +55,14 @@ choice
+ LITTLE_ENDIAN_BYTE, if the bytes are reversed.
+
+ config MTD_CFI_NOSWAP
++ depends on !ARCH_IXP4XX || CPU_BIG_ENDIAN
+ bool "NO"
+
+ config MTD_CFI_BE_BYTE_SWAP
+ bool "BIG_ENDIAN_BYTE"
+
+ config MTD_CFI_LE_BYTE_SWAP
++ depends on !ARCH_IXP4XX
+ bool "LITTLE_ENDIAN_BYTE"
+
+ endchoice
+diff --git a/drivers/mtd/devices/mchp23k256.c b/drivers/mtd/devices/mchp23k256.c
+index 77c872fd3d839..7d188cdff6a26 100644
+--- a/drivers/mtd/devices/mchp23k256.c
++++ b/drivers/mtd/devices/mchp23k256.c
+@@ -229,6 +229,19 @@ static const struct of_device_id mchp23k256_of_table[] = {
+ };
+ MODULE_DEVICE_TABLE(of, mchp23k256_of_table);
+
++static const struct spi_device_id mchp23k256_spi_ids[] = {
++ {
++ .name = "mchp23k256",
++ .driver_data = (kernel_ulong_t)&mchp23k256_caps,
++ },
++ {
++ .name = "mchp23lcv1024",
++ .driver_data = (kernel_ulong_t)&mchp23lcv1024_caps,
++ },
++ {}
++};
++MODULE_DEVICE_TABLE(spi, mchp23k256_spi_ids);
++
+ static struct spi_driver mchp23k256_driver = {
+ .driver = {
+ .name = "mchp23k256",
+@@ -236,6 +249,7 @@ static struct spi_driver mchp23k256_driver = {
+ },
+ .probe = mchp23k256_probe,
+ .remove = mchp23k256_remove,
++ .id_table = mchp23k256_spi_ids,
+ };
+
+ module_spi_driver(mchp23k256_driver);
+diff --git a/drivers/mtd/devices/mchp48l640.c b/drivers/mtd/devices/mchp48l640.c
+index 99400d0fb8c1e..fbd6b6bf908e5 100644
+--- a/drivers/mtd/devices/mchp48l640.c
++++ b/drivers/mtd/devices/mchp48l640.c
+@@ -357,6 +357,15 @@ static const struct of_device_id mchp48l640_of_table[] = {
+ };
+ MODULE_DEVICE_TABLE(of, mchp48l640_of_table);
+
++static const struct spi_device_id mchp48l640_spi_ids[] = {
++ {
++ .name = "48l640",
++ .driver_data = (kernel_ulong_t)&mchp48l640_caps,
++ },
++ {}
++};
++MODULE_DEVICE_TABLE(spi, mchp48l640_spi_ids);
++
+ static struct spi_driver mchp48l640_driver = {
+ .driver = {
+ .name = "mchp48l640",
+@@ -364,6 +373,7 @@ static struct spi_driver mchp48l640_driver = {
+ },
+ .probe = mchp48l640_probe,
+ .remove = mchp48l640_remove,
++ .id_table = mchp48l640_spi_ids,
+ };
+
+ module_spi_driver(mchp48l640_driver);
+diff --git a/drivers/mtd/devices/mtd_dataflash.c b/drivers/mtd/devices/mtd_dataflash.c
+index 9802e265fca80..2b317ed6c103f 100644
+--- a/drivers/mtd/devices/mtd_dataflash.c
++++ b/drivers/mtd/devices/mtd_dataflash.c
+@@ -96,6 +96,13 @@ struct dataflash {
+ struct mtd_info mtd;
+ };
+
++static const struct spi_device_id dataflash_dev_ids[] = {
++ { "at45" },
++ { "dataflash" },
++ { },
++};
++MODULE_DEVICE_TABLE(spi, dataflash_dev_ids);
++
+ #ifdef CONFIG_OF
+ static const struct of_device_id dataflash_dt_ids[] = {
+ { .compatible = "atmel,at45", },
+@@ -927,6 +934,7 @@ static struct spi_driver dataflash_driver = {
+ .name = "mtd_dataflash",
+ .of_match_table = of_match_ptr(dataflash_dt_ids),
+ },
++ .id_table = dataflash_dev_ids,
+
+ .probe = dataflash_probe,
+ .remove = dataflash_remove,
+diff --git a/drivers/mtd/devices/phram.c b/drivers/mtd/devices/phram.c
+index 6ed6c51fac69e..d503821a3e606 100644
+--- a/drivers/mtd/devices/phram.c
++++ b/drivers/mtd/devices/phram.c
+@@ -264,16 +264,20 @@ static int phram_setup(const char *val)
+ }
+ }
+
+- if (erasesize)
+- div_u64_rem(len, (uint32_t)erasesize, &rem);
+-
+ if (len == 0 || erasesize == 0 || erasesize > len
+- || erasesize > UINT_MAX || rem) {
++ || erasesize > UINT_MAX) {
+ parse_err("illegal erasesize or len\n");
+ ret = -EINVAL;
+ goto error;
+ }
+
++ div_u64_rem(len, (uint32_t)erasesize, &rem);
++ if (rem) {
++ parse_err("len is not multiple of erasesize\n");
++ ret = -EINVAL;
++ goto error;
++ }
++
+ ret = register_device(name, start, len, (uint32_t)erasesize);
+ if (ret)
+ goto error;
+diff --git a/drivers/mtd/hyperbus/rpc-if.c b/drivers/mtd/hyperbus/rpc-if.c
+index ecb050ba95cdf..dc164c18f8429 100644
+--- a/drivers/mtd/hyperbus/rpc-if.c
++++ b/drivers/mtd/hyperbus/rpc-if.c
+@@ -124,7 +124,9 @@ static int rpcif_hb_probe(struct platform_device *pdev)
+ if (!hyperbus)
+ return -ENOMEM;
+
+- rpcif_sw_init(&hyperbus->rpc, pdev->dev.parent);
++ error = rpcif_sw_init(&hyperbus->rpc, pdev->dev.parent);
++ if (error)
++ return error;
+
+ platform_set_drvdata(pdev, hyperbus);
+
+@@ -150,9 +152,9 @@ static int rpcif_hb_remove(struct platform_device *pdev)
+ {
+ struct rpcif_hyperbus *hyperbus = platform_get_drvdata(pdev);
+ int error = hyperbus_unregister_device(&hyperbus->hbdev);
+- struct rpcif *rpc = dev_get_drvdata(pdev->dev.parent);
+
+- rpcif_disable_rpm(rpc);
++ rpcif_disable_rpm(&hyperbus->rpc);
++
+ return error;
+ }
+
+diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
+index aaa164b977fe8..4945caa88345b 100644
+--- a/drivers/mtd/maps/Kconfig
++++ b/drivers/mtd/maps/Kconfig
+@@ -302,7 +302,7 @@ config MTD_DC21285
+
+ config MTD_IXP4XX
+ tristate "CFI Flash device mapped on Intel IXP4xx based systems"
+- depends on MTD_CFI && MTD_COMPLEX_MAPPINGS && ARCH_IXP4XX
++ depends on MTD_CFI && MTD_COMPLEX_MAPPINGS && ARCH_IXP4XX && MTD_CFI_ADV_OPTIONS
+ help
+ This enables MTD access to flash devices on platforms based
+ on Intel's IXP4xx family of network processors such as the
+diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
+index c8fd7f758938b..61f236e0378a6 100644
+--- a/drivers/mtd/mtdcore.c
++++ b/drivers/mtd/mtdcore.c
+@@ -546,6 +546,7 @@ static int mtd_nvmem_add(struct mtd_info *mtd)
+ config.stride = 1;
+ config.read_only = true;
+ config.root_only = true;
++ config.ignore_wp = true;
+ config.no_of_node = !of_device_is_compatible(node, "nvmem-cells");
+ config.priv = mtd;
+
+@@ -724,8 +725,6 @@ int del_mtd_device(struct mtd_info *mtd)
+
+ mutex_lock(&mtd_table_mutex);
+
+- debugfs_remove_recursive(mtd->dbg.dfs_dir);
+-
+ if (idr_find(&mtd_idr, mtd->index) != mtd) {
+ ret = -ENODEV;
+ goto out_error;
+@@ -741,6 +740,8 @@ int del_mtd_device(struct mtd_info *mtd)
+ mtd->index, mtd->name, mtd->usecount);
+ ret = -EBUSY;
+ } else {
++ debugfs_remove_recursive(mtd->dbg.dfs_dir);
++
+ /* Try to remove the NVMEM provider */
+ if (mtd->nvmem)
+ nvmem_unregister(mtd->nvmem);
+@@ -825,12 +826,12 @@ static struct nvmem_device *mtd_otp_nvmem_register(struct mtd_info *mtd,
+
+ /* OTP nvmem will be registered on the physical device */
+ config.dev = mtd->dev.parent;
+- /* just reuse the compatible as name */
+- config.name = compatible;
++ config.name = kasprintf(GFP_KERNEL, "%s-%s", dev_name(&mtd->dev), compatible);
+ config.id = NVMEM_DEVID_NONE;
+ config.owner = THIS_MODULE;
+ config.type = NVMEM_TYPE_OTP;
+ config.root_only = true;
++ config.ignore_wp = true;
+ config.reg_read = reg_read;
+ config.size = size;
+ config.of_node = np;
+@@ -842,6 +843,7 @@ static struct nvmem_device *mtd_otp_nvmem_register(struct mtd_info *mtd,
+ nvmem = NULL;
+
+ of_node_put(np);
++ kfree(config.name);
+
+ return nvmem;
+ }
+diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
+index 04af12b66110c..357661b62c94d 100644
+--- a/drivers/mtd/mtdpart.c
++++ b/drivers/mtd/mtdpart.c
+@@ -312,7 +312,7 @@ static int __mtd_del_partition(struct mtd_info *mtd)
+ if (err)
+ return err;
+
+- list_del(&child->part.node);
++ list_del(&mtd->part.node);
+ free_partition(mtd);
+
+ return 0;
+diff --git a/drivers/mtd/nand/onenand/generic.c b/drivers/mtd/nand/onenand/generic.c
+index 8b6f4da5d7201..a4b8b65fe15f5 100644
+--- a/drivers/mtd/nand/onenand/generic.c
++++ b/drivers/mtd/nand/onenand/generic.c
+@@ -53,7 +53,12 @@ static int generic_onenand_probe(struct platform_device *pdev)
+ }
+
+ info->onenand.mmcontrol = pdata ? pdata->mmcontrol : NULL;
+- info->onenand.irq = platform_get_irq(pdev, 0);
++
++ err = platform_get_irq(pdev, 0);
++ if (err < 0)
++ goto out_iounmap;
++
++ info->onenand.irq = err;
+
+ info->mtd.dev.parent = &pdev->dev;
+ info->mtd.priv = &info->onenand;
+diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c
+index ff1697f899ba6..13de39aa3288f 100644
+--- a/drivers/mtd/nand/raw/ams-delta.c
++++ b/drivers/mtd/nand/raw/ams-delta.c
+@@ -217,9 +217,8 @@ static int gpio_nand_setup_interface(struct nand_chip *this, int csline,
+
+ static int gpio_nand_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -370,6 +369,13 @@ static int gpio_nand_probe(struct platform_device *pdev)
+ /* Release write protection */
+ gpiod_set_value(priv->gpiod_nwp, 0);
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ /* Scan to find existence of the device */
+ err = nand_scan(this, 1);
+ if (err)
+diff --git a/drivers/mtd/nand/raw/arasan-nand-controller.c b/drivers/mtd/nand/raw/arasan-nand-controller.c
+index 9cbcc698c64d8..53bd10738418b 100644
+--- a/drivers/mtd/nand/raw/arasan-nand-controller.c
++++ b/drivers/mtd/nand/raw/arasan-nand-controller.c
+@@ -973,6 +973,21 @@ static int anfc_setup_interface(struct nand_chip *chip, int target,
+ nvddr = nand_get_nvddr_timings(conf);
+ if (IS_ERR(nvddr))
+ return PTR_ERR(nvddr);
++
++ /*
++ * The controller only supports data payload requests which are
++ * a multiple of 4. In practice, most data accesses are 4-byte
++ * aligned and this is not an issue. However, rounding up will
++ * simply be refused by the controller if we reached the end of
++ * the device *and* we are using the NV-DDR interface(!). In
++ * this situation, unaligned data requests ending at the device
++ * boundary will confuse the controller and cannot be performed.
++ *
++ * This is something that happens in nand_read_subpage() when
++ * selecting software ECC support and must be avoided.
++ */
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT)
++ return -ENOTSUPP;
+ } else {
+ sdr = nand_get_sdr_timings(conf);
+ if (IS_ERR(sdr))
+diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c
+index f3276ee9e4fe7..ddd93bc38ea6c 100644
+--- a/drivers/mtd/nand/raw/atmel/nand-controller.c
++++ b/drivers/mtd/nand/raw/atmel/nand-controller.c
+@@ -2060,13 +2060,15 @@ static int atmel_nand_controller_init(struct atmel_nand_controller *nc,
+ nc->mck = of_clk_get(dev->parent->of_node, 0);
+ if (IS_ERR(nc->mck)) {
+ dev_err(dev, "Failed to retrieve MCK clk\n");
+- return PTR_ERR(nc->mck);
++ ret = PTR_ERR(nc->mck);
++ goto out_release_dma;
+ }
+
+ np = of_parse_phandle(dev->parent->of_node, "atmel,smc", 0);
+ if (!np) {
+ dev_err(dev, "Missing or invalid atmel,smc property\n");
+- return -EINVAL;
++ ret = -EINVAL;
++ goto out_release_dma;
+ }
+
+ nc->smc = syscon_node_to_regmap(np);
+@@ -2074,10 +2076,16 @@ static int atmel_nand_controller_init(struct atmel_nand_controller *nc,
+ if (IS_ERR(nc->smc)) {
+ ret = PTR_ERR(nc->smc);
+ dev_err(dev, "Could not get SMC regmap (err = %d)\n", ret);
+- return ret;
++ goto out_release_dma;
+ }
+
+ return 0;
++
++out_release_dma:
++ if (nc->dmac)
++ dma_release_channel(nc->dmac);
++
++ return ret;
+ }
+
+ static int
+diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c
+index 99116896cfd6c..5aa3a06d740c7 100644
+--- a/drivers/mtd/nand/raw/au1550nd.c
++++ b/drivers/mtd/nand/raw/au1550nd.c
+@@ -239,9 +239,8 @@ static int au1550nd_exec_op(struct nand_chip *this,
+
+ static int au1550nd_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -310,6 +309,13 @@ static int au1550nd_probe(struct platform_device *pdev)
+ if (pd->devwidth)
+ this->options |= NAND_BUSWIDTH_16;
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ this->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ ret = nand_scan(this, 1);
+ if (ret) {
+ dev_err(&pdev->dev, "NAND scan failed with %d\n", ret);
+diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+index f75929783b941..aee78f5f4f156 100644
+--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c
++++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+@@ -2106,7 +2106,7 @@ static int brcmnand_read_by_pio(struct mtd_info *mtd, struct nand_chip *chip,
+ mtd->oobsize / trans,
+ host->hwcfg.sector_size_1k);
+
+- if (!ret) {
++ if (ret != -EBADMSG) {
+ *err_addr = brcmnand_get_uncorrecc_addr(ctrl);
+
+ if (*err_addr)
+diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c
+index 118da9944e3bc..45fec8c192aba 100644
+--- a/drivers/mtd/nand/raw/davinci_nand.c
++++ b/drivers/mtd/nand/raw/davinci_nand.c
+@@ -371,77 +371,6 @@ correct:
+ return corrected;
+ }
+
+-/**
+- * nand_read_page_hwecc_oob_first - hw ecc, read oob first
+- * @chip: nand chip info structure
+- * @buf: buffer to store read data
+- * @oob_required: caller requires OOB data read to chip->oob_poi
+- * @page: page number to read
+- *
+- * Hardware ECC for large page chips, require OOB to be read first. For this
+- * ECC mode, the write_page method is re-used from ECC_HW. These methods
+- * read/write ECC from the OOB area, unlike the ECC_HW_SYNDROME support with
+- * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from
+- * the data area, by overwriting the NAND manufacturer bad block markings.
+- */
+-static int nand_davinci_read_page_hwecc_oob_first(struct nand_chip *chip,
+- uint8_t *buf,
+- int oob_required, int page)
+-{
+- struct mtd_info *mtd = nand_to_mtd(chip);
+- int i, eccsize = chip->ecc.size, ret;
+- int eccbytes = chip->ecc.bytes;
+- int eccsteps = chip->ecc.steps;
+- uint8_t *p = buf;
+- uint8_t *ecc_code = chip->ecc.code_buf;
+- uint8_t *ecc_calc = chip->ecc.calc_buf;
+- unsigned int max_bitflips = 0;
+-
+- /* Read the OOB area first */
+- ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
+- if (ret)
+- return ret;
+-
+- ret = nand_read_page_op(chip, page, 0, NULL, 0);
+- if (ret)
+- return ret;
+-
+- ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
+- chip->ecc.total);
+- if (ret)
+- return ret;
+-
+- for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
+- int stat;
+-
+- chip->ecc.hwctl(chip, NAND_ECC_READ);
+-
+- ret = nand_read_data_op(chip, p, eccsize, false, false);
+- if (ret)
+- return ret;
+-
+- chip->ecc.calculate(chip, p, &ecc_calc[i]);
+-
+- stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
+- if (stat == -EBADMSG &&
+- (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
+- /* check for empty pages with bitflips */
+- stat = nand_check_erased_ecc_chunk(p, eccsize,
+- &ecc_code[i],
+- eccbytes, NULL, 0,
+- chip->ecc.strength);
+- }
+-
+- if (stat < 0) {
+- mtd->ecc_stats.failed++;
+- } else {
+- mtd->ecc_stats.corrected += stat;
+- max_bitflips = max_t(unsigned int, max_bitflips, stat);
+- }
+- }
+- return max_bitflips;
+-}
+-
+ /*----------------------------------------------------------------------*/
+
+ /* An ECC layout for using 4-bit ECC with small-page flash, storing
+@@ -651,7 +580,7 @@ static int davinci_nand_attach_chip(struct nand_chip *chip)
+ } else if (chunks == 4 || chunks == 8) {
+ mtd_set_ooblayout(mtd,
+ nand_get_large_page_ooblayout());
+- chip->ecc.read_page = nand_davinci_read_page_hwecc_oob_first;
++ chip->ecc.read_page = nand_read_page_hwecc_oob_first;
+ } else {
+ return -EIO;
+ }
+diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c
+index a3e66155ae405..6b2bda815b880 100644
+--- a/drivers/mtd/nand/raw/fsmc_nand.c
++++ b/drivers/mtd/nand/raw/fsmc_nand.c
+@@ -15,6 +15,7 @@
+
+ #include <linux/clk.h>
+ #include <linux/completion.h>
++#include <linux/delay.h>
+ #include <linux/dmaengine.h>
+ #include <linux/dma-direction.h>
+ #include <linux/dma-mapping.h>
+@@ -93,6 +94,14 @@
+
+ #define FSMC_BUSY_WAIT_TIMEOUT (1 * HZ)
+
++/*
++ * According to SPEAr300 Reference Manual (RM0082)
++ * TOUDEL = 7ns (Output delay from the flip-flops to the board)
++ * TINDEL = 5ns (Input delay from the board to the flipflop)
++ */
++#define TOUTDEL 7000
++#define TINDEL 5000
++
+ struct fsmc_nand_timings {
+ u8 tclr;
+ u8 tar;
+@@ -277,7 +286,7 @@ static int fsmc_calc_timings(struct fsmc_nand_data *host,
+ {
+ unsigned long hclk = clk_get_rate(host->clk);
+ unsigned long hclkn = NSEC_PER_SEC / hclk;
+- u32 thiz, thold, twait, tset;
++ u32 thiz, thold, twait, tset, twait_min;
+
+ if (sdrt->tRC_min < 30000)
+ return -EOPNOTSUPP;
+@@ -309,13 +318,6 @@ static int fsmc_calc_timings(struct fsmc_nand_data *host,
+ else if (tims->thold > FSMC_THOLD_MASK)
+ tims->thold = FSMC_THOLD_MASK;
+
+- twait = max(sdrt->tRP_min, sdrt->tWP_min);
+- tims->twait = DIV_ROUND_UP(twait / 1000, hclkn) - 1;
+- if (tims->twait == 0)
+- tims->twait = 1;
+- else if (tims->twait > FSMC_TWAIT_MASK)
+- tims->twait = FSMC_TWAIT_MASK;
+-
+ tset = max(sdrt->tCS_min - sdrt->tWP_min,
+ sdrt->tCEA_max - sdrt->tREA_max);
+ tims->tset = DIV_ROUND_UP(tset / 1000, hclkn) - 1;
+@@ -324,6 +326,21 @@ static int fsmc_calc_timings(struct fsmc_nand_data *host,
+ else if (tims->tset > FSMC_TSET_MASK)
+ tims->tset = FSMC_TSET_MASK;
+
++ /*
++ * According to SPEAr300 Reference Manual (RM0082) which gives more
++ * information related to FSMSC timings than the SPEAr600 one (RM0305),
++ * twait >= tCEA - (tset * TCLK) + TOUTDEL + TINDEL
++ */
++ twait_min = sdrt->tCEA_max - ((tims->tset + 1) * hclkn * 1000)
++ + TOUTDEL + TINDEL;
++ twait = max3(sdrt->tRP_min, sdrt->tWP_min, twait_min);
++
++ tims->twait = DIV_ROUND_UP(twait / 1000, hclkn) - 1;
++ if (tims->twait == 0)
++ tims->twait = 1;
++ else if (tims->twait > FSMC_TWAIT_MASK)
++ tims->twait = FSMC_TWAIT_MASK;
++
+ return 0;
+ }
+
+@@ -438,8 +455,10 @@ static int fsmc_correct_ecc1(struct nand_chip *chip,
+ unsigned char *read_ecc,
+ unsigned char *calc_ecc)
+ {
++ bool sm_order = chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER;
++
+ return ecc_sw_hamming_correct(buf, read_ecc, calc_ecc,
+- chip->ecc.size, false);
++ chip->ecc.size, sm_order);
+ }
+
+ /* Count the number of 0's in buff upto a max of max_bits */
+@@ -662,6 +681,9 @@ static int fsmc_exec_op(struct nand_chip *chip, const struct nand_operation *op,
+ instr->ctx.waitrdy.timeout_ms);
+ break;
+ }
++
++ if (instr->delay_ns)
++ ndelay(instr->delay_ns);
+ }
+
+ return ret;
+diff --git a/drivers/mtd/nand/raw/gpio.c b/drivers/mtd/nand/raw/gpio.c
+index fb7a086de35e5..fdf073d2e1b6c 100644
+--- a/drivers/mtd/nand/raw/gpio.c
++++ b/drivers/mtd/nand/raw/gpio.c
+@@ -163,9 +163,8 @@ static int gpio_nand_exec_op(struct nand_chip *chip,
+
+ static int gpio_nand_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -365,6 +364,13 @@ static int gpio_nand_probe(struct platform_device *pdev)
+ if (gpiomtd->nwp && !IS_ERR(gpiomtd->nwp))
+ gpiod_direction_output(gpiomtd->nwp, 1);
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ ret = nand_scan(chip, 1);
+ if (ret)
+ goto err_wp;
+diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+index 4d08e4ab5c1b6..b72b387c08ef7 100644
+--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
++++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+@@ -648,6 +648,7 @@ static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
+ const struct nand_sdr_timings *sdr)
+ {
+ struct gpmi_nfc_hardware_timing *hw = &this->hw;
++ struct resources *r = &this->resources;
+ unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
+ unsigned int period_ps, reference_period_ps;
+ unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
+@@ -671,6 +672,8 @@ static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
+ wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
+ }
+
++ hw->clk_rate = clk_round_rate(r->clock[0], hw->clk_rate);
++
+ /* SDR core timings are given in picoseconds */
+ period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
+
+@@ -713,14 +716,32 @@ static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
+ (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
+ }
+
+-static void gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
++static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
+ {
+ struct gpmi_nfc_hardware_timing *hw = &this->hw;
+ struct resources *r = &this->resources;
+ void __iomem *gpmi_regs = r->gpmi_regs;
+ unsigned int dll_wait_time_us;
++ int ret;
++
++ /* Clock dividers do NOT guarantee a clean clock signal on its output
++ * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8,
++ * all clock dividers provide these guarantee.
++ */
++ if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this))
++ clk_disable_unprepare(r->clock[0]);
+
+- clk_set_rate(r->clock[0], hw->clk_rate);
++ ret = clk_set_rate(r->clock[0], hw->clk_rate);
++ if (ret) {
++ dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret);
++ return ret;
++ }
++
++ if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) {
++ ret = clk_prepare_enable(r->clock[0]);
++ if (ret)
++ return ret;
++ }
+
+ writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
+ writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
+@@ -739,6 +760,8 @@ static void gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
+
+ /* Wait for the DLL to settle. */
+ udelay(dll_wait_time_us);
++
++ return 0;
+ }
+
+ static int gpmi_setup_interface(struct nand_chip *chip, int chipnr,
+@@ -1034,15 +1057,6 @@ static int gpmi_get_clks(struct gpmi_nand_data *this)
+ r->clock[i] = clk;
+ }
+
+- if (GPMI_IS_MX6(this))
+- /*
+- * Set the default value for the gpmi clock.
+- *
+- * If you want to use the ONFI nand which is in the
+- * Synchronous Mode, you should change the clock as you need.
+- */
+- clk_set_rate(r->clock[0], 22000000);
+-
+ return 0;
+
+ err_clock:
+@@ -2280,7 +2294,9 @@ static int gpmi_nfc_exec_op(struct nand_chip *chip,
+ */
+ if (this->hw.must_apply_timings) {
+ this->hw.must_apply_timings = false;
+- gpmi_nfc_apply_timings(this);
++ ret = gpmi_nfc_apply_timings(this);
++ if (ret)
++ goto out_pm;
+ }
+
+ dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs);
+@@ -2409,6 +2425,7 @@ unmap:
+
+ this->bch = false;
+
++out_pm:
+ pm_runtime_mark_last_busy(this->dev);
+ pm_runtime_put_autosuspend(this->dev);
+
+diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
+index efe0ffe4f1abc..9054559e52dda 100644
+--- a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
++++ b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
+@@ -68,9 +68,14 @@ static struct ingenic_ecc *ingenic_ecc_get(struct device_node *np)
+ struct ingenic_ecc *ecc;
+
+ pdev = of_find_device_by_node(np);
+- if (!pdev || !platform_get_drvdata(pdev))
++ if (!pdev)
+ return ERR_PTR(-EPROBE_DEFER);
+
++ if (!platform_get_drvdata(pdev)) {
++ put_device(&pdev->dev);
++ return ERR_PTR(-EPROBE_DEFER);
++ }
++
+ ecc = platform_get_drvdata(pdev);
+ clk_prepare_enable(ecc->clk);
+
+diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
+index 0e9d426fe4f2b..b18861bdcdc88 100644
+--- a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
++++ b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
+@@ -32,6 +32,7 @@ struct jz_soc_info {
+ unsigned long addr_offset;
+ unsigned long cmd_offset;
+ const struct mtd_ooblayout_ops *oob_layout;
++ bool oob_first;
+ };
+
+ struct ingenic_nand_cs {
+@@ -240,6 +241,9 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip)
+ if (chip->bbt_options & NAND_BBT_USE_FLASH)
+ chip->bbt_options |= NAND_BBT_NO_OOB;
+
++ if (nfc->soc_info->oob_first)
++ chip->ecc.read_page = nand_read_page_hwecc_oob_first;
++
+ /* For legacy reasons we use a different layout on the qi,lb60 board. */
+ if (of_machine_is_compatible("qi,lb60"))
+ mtd_set_ooblayout(mtd, &qi_lb60_ooblayout_ops);
+@@ -534,6 +538,7 @@ static const struct jz_soc_info jz4740_soc_info = {
+ .data_offset = 0x00000000,
+ .cmd_offset = 0x00008000,
+ .addr_offset = 0x00010000,
++ .oob_first = true,
+ };
+
+ static const struct jz_soc_info jz4725b_soc_info = {
+diff --git a/drivers/mtd/nand/raw/intel-nand-controller.c b/drivers/mtd/nand/raw/intel-nand-controller.c
+index b9784f3da7a11..7c1c80dae826a 100644
+--- a/drivers/mtd/nand/raw/intel-nand-controller.c
++++ b/drivers/mtd/nand/raw/intel-nand-controller.c
+@@ -609,6 +609,11 @@ static int ebu_nand_probe(struct platform_device *pdev)
+ dev_err(dev, "failed to get chip select: %d\n", ret);
+ return ret;
+ }
++ if (cs >= MAX_CS) {
++ dev_err(dev, "got invalid chip select: %d\n", cs);
++ return -EINVAL;
++ }
++
+ ebu_host->cs_num = cs;
+
+ resname = devm_kasprintf(dev, GFP_KERNEL, "nand_cs%d", cs);
+diff --git a/drivers/mtd/nand/raw/mpc5121_nfc.c b/drivers/mtd/nand/raw/mpc5121_nfc.c
+index bcd4a556c959c..5b9271b9c3265 100644
+--- a/drivers/mtd/nand/raw/mpc5121_nfc.c
++++ b/drivers/mtd/nand/raw/mpc5121_nfc.c
+@@ -291,7 +291,6 @@ static int ads5121_chipselect_init(struct mtd_info *mtd)
+ /* Control chips select signal on ADS5121 board */
+ static void ads5121_select_chip(struct nand_chip *nand, int chip)
+ {
+- struct mtd_info *mtd = nand_to_mtd(nand);
+ struct mpc5121_nfc_prv *prv = nand_get_controller_data(nand);
+ u8 v;
+
+@@ -605,9 +604,8 @@ static void mpc5121_nfc_free(struct device *dev, struct mtd_info *mtd)
+
+ static int mpc5121_nfc_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -772,6 +770,13 @@ static int mpc5121_nfc_probe(struct platform_device *op)
+ goto error;
+ }
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ /* Detect NAND chips */
+ retval = nand_scan(chip, be32_to_cpup(chips_no));
+ if (retval) {
+diff --git a/drivers/mtd/nand/raw/mtk_ecc.c b/drivers/mtd/nand/raw/mtk_ecc.c
+index c437d97debb8a..ec9d1fb07006f 100644
+--- a/drivers/mtd/nand/raw/mtk_ecc.c
++++ b/drivers/mtd/nand/raw/mtk_ecc.c
+@@ -43,6 +43,7 @@
+
+ struct mtk_ecc_caps {
+ u32 err_mask;
++ u32 err_shift;
+ const u8 *ecc_strength;
+ const u32 *ecc_regs;
+ u8 num_ecc_strength;
+@@ -76,7 +77,7 @@ static const u8 ecc_strength_mt2712[] = {
+ };
+
+ static const u8 ecc_strength_mt7622[] = {
+- 4, 6, 8, 10, 12, 14, 16
++ 4, 6, 8, 10, 12
+ };
+
+ enum mtk_ecc_regs {
+@@ -221,7 +222,7 @@ void mtk_ecc_get_stats(struct mtk_ecc *ecc, struct mtk_ecc_stats *stats,
+ for (i = 0; i < sectors; i++) {
+ offset = (i >> 2) << 2;
+ err = readl(ecc->regs + ECC_DECENUM0 + offset);
+- err = err >> ((i % 4) * 8);
++ err = err >> ((i % 4) * ecc->caps->err_shift);
+ err &= ecc->caps->err_mask;
+ if (err == ecc->caps->err_mask) {
+ /* uncorrectable errors */
+@@ -449,6 +450,7 @@ EXPORT_SYMBOL(mtk_ecc_get_parity_bits);
+
+ static const struct mtk_ecc_caps mtk_ecc_caps_mt2701 = {
+ .err_mask = 0x3f,
++ .err_shift = 8,
+ .ecc_strength = ecc_strength_mt2701,
+ .ecc_regs = mt2701_ecc_regs,
+ .num_ecc_strength = 20,
+@@ -459,6 +461,7 @@ static const struct mtk_ecc_caps mtk_ecc_caps_mt2701 = {
+
+ static const struct mtk_ecc_caps mtk_ecc_caps_mt2712 = {
+ .err_mask = 0x7f,
++ .err_shift = 8,
+ .ecc_strength = ecc_strength_mt2712,
+ .ecc_regs = mt2712_ecc_regs,
+ .num_ecc_strength = 23,
+@@ -468,10 +471,11 @@ static const struct mtk_ecc_caps mtk_ecc_caps_mt2712 = {
+ };
+
+ static const struct mtk_ecc_caps mtk_ecc_caps_mt7622 = {
+- .err_mask = 0x3f,
++ .err_mask = 0x1f,
++ .err_shift = 5,
+ .ecc_strength = ecc_strength_mt7622,
+ .ecc_regs = mt7622_ecc_regs,
+- .num_ecc_strength = 7,
++ .num_ecc_strength = 5,
+ .ecc_mode_shift = 4,
+ .parity_bits = 13,
+ .pg_irq_sel = 0,
+diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c
+index 3d6c6e8805207..881e768f636f8 100644
+--- a/drivers/mtd/nand/raw/nand_base.c
++++ b/drivers/mtd/nand/raw/nand_base.c
+@@ -335,16 +335,19 @@ static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs)
+ *
+ * Return: -EBUSY if the chip has been suspended, 0 otherwise
+ */
+-static int nand_get_device(struct nand_chip *chip)
++static void nand_get_device(struct nand_chip *chip)
+ {
+- mutex_lock(&chip->lock);
+- if (chip->suspended) {
++ /* Wait until the device is resumed. */
++ while (1) {
++ mutex_lock(&chip->lock);
++ if (!chip->suspended) {
++ mutex_lock(&chip->controller->lock);
++ return;
++ }
+ mutex_unlock(&chip->lock);
+- return -EBUSY;
+- }
+- mutex_lock(&chip->controller->lock);
+
+- return 0;
++ wait_event(chip->resume_wq, !chip->suspended);
++ }
+ }
+
+ /**
+@@ -573,9 +576,7 @@ static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs)
+ nand_erase_nand(chip, &einfo, 0);
+
+ /* Write bad block marker to OOB */
+- ret = nand_get_device(chip);
+- if (ret)
+- return ret;
++ nand_get_device(chip);
+
+ ret = nand_markbad_bbm(chip, ofs);
+ nand_release_device(chip);
+@@ -926,7 +927,7 @@ int nand_choose_best_sdr_timings(struct nand_chip *chip,
+ struct nand_sdr_timings *spec_timings)
+ {
+ const struct nand_controller_ops *ops = chip->controller->ops;
+- int best_mode = 0, mode, ret;
++ int best_mode = 0, mode, ret = -EOPNOTSUPP;
+
+ iface->type = NAND_SDR_IFACE;
+
+@@ -977,7 +978,7 @@ int nand_choose_best_nvddr_timings(struct nand_chip *chip,
+ struct nand_nvddr_timings *spec_timings)
+ {
+ const struct nand_controller_ops *ops = chip->controller->ops;
+- int best_mode = 0, mode, ret;
++ int best_mode = 0, mode, ret = -EOPNOTSUPP;
+
+ iface->type = NAND_NVDDR_IFACE;
+
+@@ -1837,7 +1838,7 @@ int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock)
+ NAND_OP_CMD(NAND_CMD_ERASE1, 0),
+ NAND_OP_ADDR(2, addrs, 0),
+ NAND_OP_CMD(NAND_CMD_ERASE2,
+- NAND_COMMON_TIMING_MS(conf, tWB_max)),
++ NAND_COMMON_TIMING_NS(conf, tWB_max)),
+ NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tBERS_max),
+ 0),
+ };
+@@ -3160,6 +3161,73 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
+ return max_bitflips;
+ }
+
++/**
++ * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC
++ * data read from OOB area
++ * @chip: nand chip info structure
++ * @buf: buffer to store read data
++ * @oob_required: caller requires OOB data read to chip->oob_poi
++ * @page: page number to read
++ *
++ * Hardware ECC for large page chips, which requires the ECC data to be
++ * extracted from the OOB before the actual data is read.
++ */
++int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf,
++ int oob_required, int page)
++{
++ struct mtd_info *mtd = nand_to_mtd(chip);
++ int i, eccsize = chip->ecc.size, ret;
++ int eccbytes = chip->ecc.bytes;
++ int eccsteps = chip->ecc.steps;
++ uint8_t *p = buf;
++ uint8_t *ecc_code = chip->ecc.code_buf;
++ unsigned int max_bitflips = 0;
++
++ /* Read the OOB area first */
++ ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
++ if (ret)
++ return ret;
++
++ /* Move read cursor to start of page */
++ ret = nand_change_read_column_op(chip, 0, NULL, 0, false);
++ if (ret)
++ return ret;
++
++ ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
++ chip->ecc.total);
++ if (ret)
++ return ret;
++
++ for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
++ int stat;
++
++ chip->ecc.hwctl(chip, NAND_ECC_READ);
++
++ ret = nand_read_data_op(chip, p, eccsize, false, false);
++ if (ret)
++ return ret;
++
++ stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
++ if (stat == -EBADMSG &&
++ (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
++ /* check for empty pages with bitflips */
++ stat = nand_check_erased_ecc_chunk(p, eccsize,
++ &ecc_code[i],
++ eccbytes, NULL, 0,
++ chip->ecc.strength);
++ }
++
++ if (stat < 0) {
++ mtd->ecc_stats.failed++;
++ } else {
++ mtd->ecc_stats.corrected += stat;
++ max_bitflips = max_t(unsigned int, max_bitflips, stat);
++ }
++ }
++ return max_bitflips;
++}
++EXPORT_SYMBOL_GPL(nand_read_page_hwecc_oob_first);
++
+ /**
+ * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
+ * @chip: nand chip info structure
+@@ -3756,9 +3824,7 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from,
+ ops->mode != MTD_OPS_RAW)
+ return -ENOTSUPP;
+
+- ret = nand_get_device(chip);
+- if (ret)
+- return ret;
++ nand_get_device(chip);
+
+ if (!ops->datbuf)
+ ret = nand_do_read_oob(chip, from, ops);
+@@ -4345,13 +4411,11 @@ static int nand_write_oob(struct mtd_info *mtd, loff_t to,
+ struct mtd_oob_ops *ops)
+ {
+ struct nand_chip *chip = mtd_to_nand(mtd);
+- int ret;
++ int ret = 0;
+
+ ops->retlen = 0;
+
+- ret = nand_get_device(chip);
+- if (ret)
+- return ret;
++ nand_get_device(chip);
+
+ switch (ops->mode) {
+ case MTD_OPS_PLACE_OOB:
+@@ -4411,9 +4475,7 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr,
+ return -EIO;
+
+ /* Grab the lock and see if the device is available */
+- ret = nand_get_device(chip);
+- if (ret)
+- return ret;
++ nand_get_device(chip);
+
+ /* Shift to get first page */
+ page = (int)(instr->addr >> chip->page_shift);
+@@ -4500,7 +4562,7 @@ static void nand_sync(struct mtd_info *mtd)
+ pr_debug("%s: called\n", __func__);
+
+ /* Grab the lock and see if the device is available */
+- WARN_ON(nand_get_device(chip));
++ nand_get_device(chip);
+ /* Release it and go back */
+ nand_release_device(chip);
+ }
+@@ -4517,9 +4579,7 @@ static int nand_block_isbad(struct mtd_info *mtd, loff_t offs)
+ int ret;
+
+ /* Select the NAND device */
+- ret = nand_get_device(chip);
+- if (ret)
+- return ret;
++ nand_get_device(chip);
+
+ nand_select_target(chip, chipnr);
+
+@@ -4590,6 +4650,8 @@ static void nand_resume(struct mtd_info *mtd)
+ __func__);
+ }
+ mutex_unlock(&chip->lock);
++
++ wake_up_all(&chip->resume_wq);
+ }
+
+ /**
+@@ -5367,6 +5429,7 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips,
+ chip->cur_cs = -1;
+
+ mutex_init(&chip->lock);
++ init_waitqueue_head(&chip->resume_wq);
+
+ /* Enforce the right timings for reset/detection */
+ chip->current_interface_config = nand_get_reset_interface_config();
+diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c
+index 66211c9311d2f..2c87c7d892058 100644
+--- a/drivers/mtd/nand/raw/orion_nand.c
++++ b/drivers/mtd/nand/raw/orion_nand.c
+@@ -85,9 +85,8 @@ static void orion_nand_read_buf(struct nand_chip *chip, uint8_t *buf, int len)
+
+ static int orion_nand_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -190,6 +189,13 @@ static int __init orion_nand_probe(struct platform_device *pdev)
+ return ret;
+ }
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ nc->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ ret = nand_scan(nc, 1);
+ if (ret)
+ goto no_dev;
+diff --git a/drivers/mtd/nand/raw/pasemi_nand.c b/drivers/mtd/nand/raw/pasemi_nand.c
+index 789f33312c15f..c176036453ed9 100644
+--- a/drivers/mtd/nand/raw/pasemi_nand.c
++++ b/drivers/mtd/nand/raw/pasemi_nand.c
+@@ -75,9 +75,8 @@ static int pasemi_device_ready(struct nand_chip *chip)
+
+ static int pasemi_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -154,6 +153,13 @@ static int pasemi_nand_probe(struct platform_device *ofdev)
+ /* Enable the following for a flash based bad block table */
+ chip->bbt_options = NAND_BBT_USE_FLASH;
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ /* Scan to find existence of the device */
+ err = nand_scan(chip, 1);
+ if (err)
+diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c
+index 8a91e069ee2e9..3c6f6aff649f8 100644
+--- a/drivers/mtd/nand/raw/pl35x-nand-controller.c
++++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c
+@@ -1062,7 +1062,7 @@ static int pl35x_nand_chip_init(struct pl35x_nandc *nfc,
+ chip->controller = &nfc->controller;
+ mtd = nand_to_mtd(chip);
+ mtd->dev.parent = nfc->dev;
+- nand_set_flash_node(chip, nfc->dev->of_node);
++ nand_set_flash_node(chip, np);
+ if (!mtd->name) {
+ mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
+ "%s", PL35X_NANDC_DRIVER_NAME);
+diff --git a/drivers/mtd/nand/raw/plat_nand.c b/drivers/mtd/nand/raw/plat_nand.c
+index 7711e1020c21c..0ee08c42cc35b 100644
+--- a/drivers/mtd/nand/raw/plat_nand.c
++++ b/drivers/mtd/nand/raw/plat_nand.c
+@@ -21,9 +21,8 @@ struct plat_nand_data {
+
+ static int plat_nand_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -94,6 +93,13 @@ static int plat_nand_probe(struct platform_device *pdev)
+ goto out;
+ }
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ data->chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ /* Scan to find existence of the device */
+ err = nand_scan(&data->chip, pdata->chip.nr_chips);
+ if (err)
+diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c
+index 04e6f7b267064..e972bee60e7c8 100644
+--- a/drivers/mtd/nand/raw/qcom_nandc.c
++++ b/drivers/mtd/nand/raw/qcom_nandc.c
+@@ -2,7 +2,6 @@
+ /*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ */
+-
+ #include <linux/clk.h>
+ #include <linux/slab.h>
+ #include <linux/bitops.h>
+@@ -2642,10 +2641,23 @@ static int qcom_nand_attach_chip(struct nand_chip *chip)
+ ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
+
+ mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
++ /* Free the initially allocated BAM transaction for reading the ONFI params */
++ if (nandc->props->is_bam)
++ free_bam_transaction(nandc);
+
+ nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
+ cwperpage);
+
++ /* Now allocate the BAM transaction based on updated max_cwperpage */
++ if (nandc->props->is_bam) {
++ nandc->bam_txn = alloc_bam_transaction(nandc);
++ if (!nandc->bam_txn) {
++ dev_err(nandc->dev,
++ "failed to allocate bam transaction\n");
++ return -ENOMEM;
++ }
++ }
++
+ /*
+ * DATA_UD_BYTES varies based on whether the read/write command protects
+ * spare data with ECC too. We protect spare data by default, so we set
+@@ -2946,17 +2958,6 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
+ if (ret)
+ return ret;
+
+- if (nandc->props->is_bam) {
+- free_bam_transaction(nandc);
+- nandc->bam_txn = alloc_bam_transaction(nandc);
+- if (!nandc->bam_txn) {
+- dev_err(nandc->dev,
+- "failed to allocate bam transaction\n");
+- nand_cleanup(chip);
+- return -ENOMEM;
+- }
+- }
+-
+ ret = mtd_device_parse_register(mtd, probes, NULL, NULL, 0);
+ if (ret)
+ nand_cleanup(chip);
+@@ -3063,10 +3064,6 @@ static int qcom_nandc_probe(struct platform_device *pdev)
+ if (dma_mapping_error(dev, nandc->base_dma))
+ return -ENXIO;
+
+- ret = qcom_nandc_alloc(nandc);
+- if (ret)
+- goto err_nandc_alloc;
+-
+ ret = clk_prepare_enable(nandc->core_clk);
+ if (ret)
+ goto err_core_clk;
+@@ -3075,6 +3072,10 @@ static int qcom_nandc_probe(struct platform_device *pdev)
+ if (ret)
+ goto err_aon_clk;
+
++ ret = qcom_nandc_alloc(nandc);
++ if (ret)
++ goto err_nandc_alloc;
++
+ ret = qcom_nandc_setup(nandc);
+ if (ret)
+ goto err_setup;
+@@ -3086,15 +3087,14 @@ static int qcom_nandc_probe(struct platform_device *pdev)
+ return 0;
+
+ err_setup:
++ qcom_nandc_unalloc(nandc);
++err_nandc_alloc:
+ clk_disable_unprepare(nandc->aon_clk);
+ err_aon_clk:
+ clk_disable_unprepare(nandc->core_clk);
+ err_core_clk:
+- qcom_nandc_unalloc(nandc);
+-err_nandc_alloc:
+ dma_unmap_resource(dev, res->start, resource_size(res),
+ DMA_BIDIRECTIONAL, 0);
+-
+ return ret;
+ }
+
+diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c
+index 13df4bdf792af..8f89e2d3d817f 100644
+--- a/drivers/mtd/nand/raw/sh_flctl.c
++++ b/drivers/mtd/nand/raw/sh_flctl.c
+@@ -384,7 +384,8 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
+ dma_addr_t dma_addr;
+ dma_cookie_t cookie;
+ uint32_t reg;
+- int ret;
++ int ret = 0;
++ unsigned long time_left;
+
+ if (dir == DMA_FROM_DEVICE) {
+ chan = flctl->chan_fifo0_rx;
+@@ -425,13 +426,14 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
+ goto out;
+ }
+
+- ret =
++ time_left =
+ wait_for_completion_timeout(&flctl->dma_complete,
+ msecs_to_jiffies(3000));
+
+- if (ret <= 0) {
++ if (time_left == 0) {
+ dmaengine_terminate_all(chan);
+ dev_err(&flctl->pdev->dev, "wait_for_completion_timeout\n");
++ ret = -ETIMEDOUT;
+ }
+
+ out:
+@@ -441,7 +443,7 @@ out:
+
+ dma_unmap_single(chan->device->dev, dma_addr, len, dir);
+
+- /* ret > 0 is success */
++ /* ret == 0 is success */
+ return ret;
+ }
+
+@@ -465,7 +467,7 @@ static void read_fiforeg(struct sh_flctl *flctl, int rlen, int offset)
+
+ /* initiate DMA transfer */
+ if (flctl->chan_fifo0_rx && rlen >= 32 &&
+- flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_FROM_DEVICE) > 0)
++ !flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_FROM_DEVICE))
+ goto convert; /* DMA success */
+
+ /* do polling transfer */
+@@ -524,7 +526,7 @@ static void write_ec_fiforeg(struct sh_flctl *flctl, int rlen,
+
+ /* initiate DMA transfer */
+ if (flctl->chan_fifo0_tx && rlen >= 32 &&
+- flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_TO_DEVICE) > 0)
++ !flctl_dma_fifo0_transfer(flctl, buf, rlen, DMA_TO_DEVICE))
+ return; /* DMA success */
+
+ /* do polling transfer */
+diff --git a/drivers/mtd/nand/raw/socrates_nand.c b/drivers/mtd/nand/raw/socrates_nand.c
+index 70f8305c9b6e1..fb39cc7ebce03 100644
+--- a/drivers/mtd/nand/raw/socrates_nand.c
++++ b/drivers/mtd/nand/raw/socrates_nand.c
+@@ -119,9 +119,8 @@ static int socrates_nand_device_ready(struct nand_chip *nand_chip)
+
+ static int socrates_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -175,6 +174,13 @@ static int socrates_nand_probe(struct platform_device *ofdev)
+ /* TODO: I have no idea what real delay is. */
+ nand_chip->legacy.chip_delay = 20; /* 20us command delay time */
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ nand_chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ dev_set_drvdata(&ofdev->dev, host);
+
+ res = nand_scan(nand_chip, 1);
+diff --git a/drivers/mtd/nand/raw/xway_nand.c b/drivers/mtd/nand/raw/xway_nand.c
+index 26751976e5026..236fd8c5a958f 100644
+--- a/drivers/mtd/nand/raw/xway_nand.c
++++ b/drivers/mtd/nand/raw/xway_nand.c
+@@ -148,9 +148,8 @@ static void xway_write_buf(struct nand_chip *chip, const u_char *buf, int len)
+
+ static int xway_attach_chip(struct nand_chip *chip)
+ {
+- chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
+-
+- if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
++ if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT &&
++ chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN)
+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
+
+ return 0;
+@@ -219,6 +218,13 @@ static int xway_nand_probe(struct platform_device *pdev)
+ | NAND_CON_SE_P | NAND_CON_WP_P | NAND_CON_PRE_P
+ | cs_flag, EBU_NAND_CON);
+
++ /*
++ * This driver assumes that the default ECC engine should be TYPE_SOFT.
++ * Set ->engine_type before registering the NAND devices in order to
++ * provide a driver specific default value.
++ */
++ data->chip.ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
++
+ /* Scan to find existence of the device */
+ err = nand_scan(&data->chip, 1);
+ if (err)
+diff --git a/drivers/mtd/parsers/qcomsmempart.c b/drivers/mtd/parsers/qcomsmempart.c
+index 06a818cd2433f..32ddfea701423 100644
+--- a/drivers/mtd/parsers/qcomsmempart.c
++++ b/drivers/mtd/parsers/qcomsmempart.c
+@@ -58,11 +58,11 @@ static int parse_qcomsmem_part(struct mtd_info *mtd,
+ const struct mtd_partition **pparts,
+ struct mtd_part_parser_data *data)
+ {
++ size_t len = SMEM_FLASH_PTABLE_HDR_LEN;
++ int ret, i, j, tmpparts, numparts = 0;
+ struct smem_flash_pentry *pentry;
+ struct smem_flash_ptable *ptable;
+- size_t len = SMEM_FLASH_PTABLE_HDR_LEN;
+ struct mtd_partition *parts;
+- int ret, i, numparts;
+ char *name, *c;
+
+ if (IS_ENABLED(CONFIG_MTD_SPI_NOR_USE_4K_SECTORS)
+@@ -87,8 +87,8 @@ static int parse_qcomsmem_part(struct mtd_info *mtd,
+ }
+
+ /* Ensure that # of partitions is less than the max we have allocated */
+- numparts = le32_to_cpu(ptable->numparts);
+- if (numparts > SMEM_FLASH_PTABLE_MAX_PARTS_V4) {
++ tmpparts = le32_to_cpu(ptable->numparts);
++ if (tmpparts > SMEM_FLASH_PTABLE_MAX_PARTS_V4) {
+ pr_err("Partition numbers exceed the max limit\n");
+ return -EINVAL;
+ }
+@@ -116,11 +116,17 @@ static int parse_qcomsmem_part(struct mtd_info *mtd,
+ return PTR_ERR(ptable);
+ }
+
++ for (i = 0; i < tmpparts; i++) {
++ pentry = &ptable->pentry[i];
++ if (pentry->name[0] != '\0')
++ numparts++;
++ }
++
+ parts = kcalloc(numparts, sizeof(*parts), GFP_KERNEL);
+ if (!parts)
+ return -ENOMEM;
+
+- for (i = 0; i < numparts; i++) {
++ for (i = 0, j = 0; i < tmpparts; i++) {
+ pentry = &ptable->pentry[i];
+ if (pentry->name[0] == '\0')
+ continue;
+@@ -135,24 +141,25 @@ static int parse_qcomsmem_part(struct mtd_info *mtd,
+ for (c = name; *c != '\0'; c++)
+ *c = tolower(*c);
+
+- parts[i].name = name;
+- parts[i].offset = le32_to_cpu(pentry->offset) * mtd->erasesize;
+- parts[i].mask_flags = pentry->attr;
+- parts[i].size = le32_to_cpu(pentry->length) * mtd->erasesize;
++ parts[j].name = name;
++ parts[j].offset = le32_to_cpu(pentry->offset) * mtd->erasesize;
++ parts[j].mask_flags = pentry->attr;
++ parts[j].size = le32_to_cpu(pentry->length) * mtd->erasesize;
+ pr_debug("%d: %s offs=0x%08x size=0x%08x attr:0x%08x\n",
+ i, pentry->name, le32_to_cpu(pentry->offset),
+ le32_to_cpu(pentry->length), pentry->attr);
++ j++;
+ }
+
+ pr_debug("SMEM partition table found: ver: %d len: %d\n",
+- le32_to_cpu(ptable->version), numparts);
++ le32_to_cpu(ptable->version), tmpparts);
+ *pparts = parts;
+
+ return numparts;
+
+ out_free_parts:
+- while (--i >= 0)
+- kfree(parts[i].name);
++ while (--j >= 0)
++ kfree(parts[j].name);
+ kfree(parts);
+ *pparts = NULL;
+
+@@ -166,6 +173,8 @@ static void parse_qcomsmem_cleanup(const struct mtd_partition *pparts,
+
+ for (i = 0; i < nr_parts; i++)
+ kfree(pparts[i].name);
++
++ kfree(pparts);
+ }
+
+ static const struct of_device_id qcomsmem_of_match_table[] = {
+diff --git a/drivers/mtd/spi-nor/controllers/hisi-sfc.c b/drivers/mtd/spi-nor/controllers/hisi-sfc.c
+index 47fbf1d1e5573..516e502694780 100644
+--- a/drivers/mtd/spi-nor/controllers/hisi-sfc.c
++++ b/drivers/mtd/spi-nor/controllers/hisi-sfc.c
+@@ -477,7 +477,6 @@ static int hisi_spi_nor_remove(struct platform_device *pdev)
+
+ hisi_spi_nor_unregister_all(host);
+ mutex_destroy(&host->lock);
+- clk_disable_unprepare(host->clk);
+ return 0;
+ }
+
+diff --git a/drivers/mtd/spi-nor/xilinx.c b/drivers/mtd/spi-nor/xilinx.c
+index 1138bdbf41998..75dd13a390404 100644
+--- a/drivers/mtd/spi-nor/xilinx.c
++++ b/drivers/mtd/spi-nor/xilinx.c
+@@ -66,7 +66,8 @@ static int xilinx_nor_setup(struct spi_nor *nor,
+ /* Flash in Power of 2 mode */
+ nor->page_size = (nor->page_size == 264) ? 256 : 512;
+ nor->mtd.writebufsize = nor->page_size;
+- nor->mtd.size = 8 * nor->page_size * nor->info->n_sectors;
++ nor->params->size = 8 * nor->page_size * nor->info->n_sectors;
++ nor->mtd.size = nor->params->size;
+ nor->mtd.erasesize = 8 * nor->page_size;
+ } else {
+ /* Flash in Default addressing mode */
+diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
+index a7e3eb9befb62..a32050fecabf3 100644
+--- a/drivers/mtd/ubi/build.c
++++ b/drivers/mtd/ubi/build.c
+@@ -351,9 +351,6 @@ static ssize_t dev_attribute_show(struct device *dev,
+ * we still can use 'ubi->ubi_num'.
+ */
+ ubi = container_of(dev, struct ubi_device, dev);
+- ubi = ubi_get_device(ubi->ubi_num);
+- if (!ubi)
+- return -ENODEV;
+
+ if (attr == &dev_eraseblock_size)
+ ret = sprintf(buf, "%d\n", ubi->leb_size);
+@@ -382,7 +379,6 @@ static ssize_t dev_attribute_show(struct device *dev,
+ else
+ ret = -EINVAL;
+
+- ubi_put_device(ubi);
+ return ret;
+ }
+
+@@ -979,9 +975,6 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
+ goto out_detach;
+ }
+
+- /* Make device "available" before it becomes accessible via sysfs */
+- ubi_devices[ubi_num] = ubi;
+-
+ err = uif_init(ubi);
+ if (err)
+ goto out_detach;
+@@ -1026,6 +1019,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
+ wake_up_process(ubi->bgt_thread);
+ spin_unlock(&ubi->wl_lock);
+
++ ubi_devices[ubi_num] = ubi;
+ ubi_notify_all(ubi, UBI_VOLUME_ADDED, NULL);
+ return ubi_num;
+
+@@ -1034,7 +1028,6 @@ out_debugfs:
+ out_uif:
+ uif_close(ubi);
+ out_detach:
+- ubi_devices[ubi_num] = NULL;
+ ubi_wl_close(ubi);
+ ubi_free_all_volumes(ubi);
+ vfree(ubi->vtbl);
+diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c
+index 022af59906aa9..6b5f1ffd961b9 100644
+--- a/drivers/mtd/ubi/fastmap.c
++++ b/drivers/mtd/ubi/fastmap.c
+@@ -468,7 +468,9 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
+ if (err == UBI_IO_FF_BITFLIPS)
+ scrub = 1;
+
+- add_aeb(ai, free, pnum, ec, scrub);
++ ret = add_aeb(ai, free, pnum, ec, scrub);
++ if (ret)
++ goto out;
+ continue;
+ } else if (err == 0 || err == UBI_IO_BITFLIPS) {
+ dbg_bld("Found non empty PEB:%i in pool", pnum);
+@@ -638,8 +640,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi,
+ if (fm_pos >= fm_size)
+ goto fail_bad;
+
+- add_aeb(ai, &ai->free, be32_to_cpu(fmec->pnum),
+- be32_to_cpu(fmec->ec), 0);
++ ret = add_aeb(ai, &ai->free, be32_to_cpu(fmec->pnum),
++ be32_to_cpu(fmec->ec), 0);
++ if (ret)
++ goto fail;
+ }
+
+ /* read EC values from used list */
+@@ -649,8 +653,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi,
+ if (fm_pos >= fm_size)
+ goto fail_bad;
+
+- add_aeb(ai, &used, be32_to_cpu(fmec->pnum),
+- be32_to_cpu(fmec->ec), 0);
++ ret = add_aeb(ai, &used, be32_to_cpu(fmec->pnum),
++ be32_to_cpu(fmec->ec), 0);
++ if (ret)
++ goto fail;
+ }
+
+ /* read EC values from scrub list */
+@@ -660,8 +666,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi,
+ if (fm_pos >= fm_size)
+ goto fail_bad;
+
+- add_aeb(ai, &used, be32_to_cpu(fmec->pnum),
+- be32_to_cpu(fmec->ec), 1);
++ ret = add_aeb(ai, &used, be32_to_cpu(fmec->pnum),
++ be32_to_cpu(fmec->ec), 1);
++ if (ret)
++ goto fail;
+ }
+
+ /* read EC values from erase list */
+@@ -671,8 +679,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi,
+ if (fm_pos >= fm_size)
+ goto fail_bad;
+
+- add_aeb(ai, &ai->erase, be32_to_cpu(fmec->pnum),
+- be32_to_cpu(fmec->ec), 1);
++ ret = add_aeb(ai, &ai->erase, be32_to_cpu(fmec->pnum),
++ be32_to_cpu(fmec->ec), 1);
++ if (ret)
++ goto fail;
+ }
+
+ ai->mean_ec = div_u64(ai->ec_sum, ai->ec_count);
+diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c
+index 139ee132bfbcf..1bc7b3a056046 100644
+--- a/drivers/mtd/ubi/vmt.c
++++ b/drivers/mtd/ubi/vmt.c
+@@ -56,16 +56,11 @@ static ssize_t vol_attribute_show(struct device *dev,
+ {
+ int ret;
+ struct ubi_volume *vol = container_of(dev, struct ubi_volume, dev);
+- struct ubi_device *ubi;
+-
+- ubi = ubi_get_device(vol->ubi->ubi_num);
+- if (!ubi)
+- return -ENODEV;
++ struct ubi_device *ubi = vol->ubi;
+
+ spin_lock(&ubi->volumes_lock);
+ if (!ubi->volumes[vol->vol_id]) {
+ spin_unlock(&ubi->volumes_lock);
+- ubi_put_device(ubi);
+ return -ENODEV;
+ }
+ /* Take a reference to prevent volume removal */
+@@ -103,7 +98,6 @@ static ssize_t vol_attribute_show(struct device *dev,
+ vol->ref_count -= 1;
+ ubi_assert(vol->ref_count >= 0);
+ spin_unlock(&ubi->volumes_lock);
+- ubi_put_device(ubi);
+ return ret;
+ }
+
+diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
+index f37b1c56f7c43..44a7d36446c52 100644
+--- a/drivers/net/Kconfig
++++ b/drivers/net/Kconfig
+@@ -81,7 +81,6 @@ config WIREGUARD
+ select CRYPTO
+ select CRYPTO_LIB_CURVE25519
+ select CRYPTO_LIB_CHACHA20POLY1305
+- select CRYPTO_LIB_BLAKE2S
+ select CRYPTO_CHACHA20_X86_64 if X86 && 64BIT
+ select CRYPTO_POLY1305_X86_64 if X86 && 64BIT
+ select CRYPTO_BLAKE2S_X86 if X86 && 64BIT
+@@ -150,7 +149,7 @@ config NET_FC
+
+ config IFB
+ tristate "Intermediate Functional Block support"
+- depends on NET_CLS_ACT
++ depends on NET_ACT_MIRRED || NFT_FWD_NETDEV
+ select NET_REDIRECT
+ help
+ This is an intermediate driver that allows sharing of
+diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c
+index 3c8f665c15580..28dccbc0e8d8f 100644
+--- a/drivers/net/arcnet/com20020-pci.c
++++ b/drivers/net/arcnet/com20020-pci.c
+@@ -138,6 +138,9 @@ static int com20020pci_probe(struct pci_dev *pdev,
+ return -ENOMEM;
+
+ ci = (struct com20020_pci_card_info *)id->driver_data;
++ if (!ci)
++ return -EINVAL;
++
+ priv->ci = ci;
+ mm = &ci->misc_map;
+
+diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c
+index 54e321a695ce9..98c915943f323 100644
+--- a/drivers/net/bareudp.c
++++ b/drivers/net/bareudp.c
+@@ -141,14 +141,14 @@ static int bareudp_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
+ skb_reset_network_header(skb);
+ skb_reset_mac_header(skb);
+
+- if (!IS_ENABLED(CONFIG_IPV6) || family == AF_INET)
++ if (!ipv6_mod_enabled() || family == AF_INET)
+ err = IP_ECN_decapsulate(oiph, skb);
+ else
+ err = IP6_ECN_decapsulate(oiph, skb);
+
+ if (unlikely(err)) {
+ if (log_ecn_error) {
+- if (!IS_ENABLED(CONFIG_IPV6) || family == AF_INET)
++ if (!ipv6_mod_enabled() || family == AF_INET)
+ net_info_ratelimited("non-ECT from %pI4 "
+ "with TOS=%#x\n",
+ &((struct iphdr *)oiph)->saddr,
+@@ -214,11 +214,12 @@ static struct socket *bareudp_create_sock(struct net *net, __be16 port)
+ int err;
+
+ memset(&udp_conf, 0, sizeof(udp_conf));
+-#if IS_ENABLED(CONFIG_IPV6)
+- udp_conf.family = AF_INET6;
+-#else
+- udp_conf.family = AF_INET;
+-#endif
++
++ if (ipv6_mod_enabled())
++ udp_conf.family = AF_INET6;
++ else
++ udp_conf.family = AF_INET;
++
+ udp_conf.local_udp_port = port;
+ /* Open UDP socket */
+ err = udp_sock_create(net, &udp_conf, &sock);
+@@ -441,7 +442,7 @@ static netdev_tx_t bareudp_xmit(struct sk_buff *skb, struct net_device *dev)
+ }
+
+ rcu_read_lock();
+- if (IS_ENABLED(CONFIG_IPV6) && info->mode & IP_TUNNEL_INFO_IPV6)
++ if (ipv6_mod_enabled() && info->mode & IP_TUNNEL_INFO_IPV6)
+ err = bareudp6_xmit_skb(skb, dev, bareudp, info);
+ else
+ err = bareudp_xmit_skb(skb, dev, bareudp, info);
+@@ -471,7 +472,7 @@ static int bareudp_fill_metadata_dst(struct net_device *dev,
+
+ use_cache = ip_tunnel_dst_cache_usable(skb, info);
+
+- if (!IS_ENABLED(CONFIG_IPV6) || ip_tunnel_info_af(info) == AF_INET) {
++ if (!ipv6_mod_enabled() || ip_tunnel_info_af(info) == AF_INET) {
+ struct rtable *rt;
+ __be32 saddr;
+
+diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c
+index 6006c2e8fa2bc..a86b1f71762ea 100644
+--- a/drivers/net/bonding/bond_3ad.c
++++ b/drivers/net/bonding/bond_3ad.c
+@@ -225,7 +225,7 @@ static inline int __check_agg_selection_timer(struct port *port)
+ if (bond == NULL)
+ return 0;
+
+- return BOND_AD_INFO(bond).agg_select_timer ? 1 : 0;
++ return atomic_read(&BOND_AD_INFO(bond).agg_select_timer) ? 1 : 0;
+ }
+
+ /**
+@@ -1021,8 +1021,8 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
+ if (port->aggregator &&
+ port->aggregator->is_active &&
+ !__port_is_enabled(port)) {
+-
+ __enable_port(port);
++ *update_slave_arr = true;
+ }
+ }
+ break;
+@@ -1779,6 +1779,7 @@ static void ad_agg_selection_logic(struct aggregator *agg,
+ port = port->next_port_in_aggregator) {
+ __enable_port(port);
+ }
++ *update_slave_arr = true;
+ }
+ }
+
+@@ -1994,7 +1995,7 @@ static void ad_marker_response_received(struct bond_marker *marker,
+ */
+ void bond_3ad_initiate_agg_selection(struct bonding *bond, int timeout)
+ {
+- BOND_AD_INFO(bond).agg_select_timer = timeout;
++ atomic_set(&BOND_AD_INFO(bond).agg_select_timer, timeout);
+ }
+
+ /**
+@@ -2277,6 +2278,28 @@ void bond_3ad_update_ad_actor_settings(struct bonding *bond)
+ spin_unlock_bh(&bond->mode_lock);
+ }
+
++/**
++ * bond_agg_timer_advance - advance agg_select_timer
++ * @bond: bonding structure
++ *
++ * Return true when agg_select_timer reaches 0.
++ */
++static bool bond_agg_timer_advance(struct bonding *bond)
++{
++ int val, nval;
++
++ while (1) {
++ val = atomic_read(&BOND_AD_INFO(bond).agg_select_timer);
++ if (!val)
++ return false;
++ nval = val - 1;
++ if (atomic_cmpxchg(&BOND_AD_INFO(bond).agg_select_timer,
++ val, nval) == val)
++ break;
++ }
++ return nval == 0;
++}
++
+ /**
+ * bond_3ad_state_machine_handler - handle state machines timeout
+ * @work: work context to fetch bonding struct to work on from
+@@ -2312,9 +2335,7 @@ void bond_3ad_state_machine_handler(struct work_struct *work)
+ if (!bond_has_slaves(bond))
+ goto re_arm;
+
+- /* check if agg_select_timer timer after initialize is timed out */
+- if (BOND_AD_INFO(bond).agg_select_timer &&
+- !(--BOND_AD_INFO(bond).agg_select_timer)) {
++ if (bond_agg_timer_advance(bond)) {
+ slave = bond_first_slave_rcu(bond);
+ port = slave ? &(SLAVE_AD_INFO(slave)->port) : NULL;
+
+diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
+index 7d3752cbf761d..bca36be884b8d 100644
+--- a/drivers/net/bonding/bond_alb.c
++++ b/drivers/net/bonding/bond_alb.c
+@@ -1502,14 +1502,14 @@ void bond_alb_monitor(struct work_struct *work)
+ struct slave *slave;
+
+ if (!bond_has_slaves(bond)) {
+- bond_info->tx_rebalance_counter = 0;
++ atomic_set(&bond_info->tx_rebalance_counter, 0);
+ bond_info->lp_counter = 0;
+ goto re_arm;
+ }
+
+ rcu_read_lock();
+
+- bond_info->tx_rebalance_counter++;
++ atomic_inc(&bond_info->tx_rebalance_counter);
+ bond_info->lp_counter++;
+
+ /* send learning packets */
+@@ -1531,7 +1531,7 @@ void bond_alb_monitor(struct work_struct *work)
+ }
+
+ /* rebalance tx traffic */
+- if (bond_info->tx_rebalance_counter >= BOND_TLB_REBALANCE_TICKS) {
++ if (atomic_read(&bond_info->tx_rebalance_counter) >= BOND_TLB_REBALANCE_TICKS) {
+ bond_for_each_slave_rcu(bond, slave, iter) {
+ tlb_clear_slave(bond, slave, 1);
+ if (slave == rcu_access_pointer(bond->curr_active_slave)) {
+@@ -1541,7 +1541,7 @@ void bond_alb_monitor(struct work_struct *work)
+ bond_info->unbalanced_load = 0;
+ }
+ }
+- bond_info->tx_rebalance_counter = 0;
++ atomic_set(&bond_info->tx_rebalance_counter, 0);
+ }
+
+ if (bond_info->rlb_enabled) {
+@@ -1611,7 +1611,8 @@ int bond_alb_init_slave(struct bonding *bond, struct slave *slave)
+ tlb_init_slave(slave);
+
+ /* order a rebalance ASAP */
+- bond->alb_info.tx_rebalance_counter = BOND_TLB_REBALANCE_TICKS;
++ atomic_set(&bond->alb_info.tx_rebalance_counter,
++ BOND_TLB_REBALANCE_TICKS);
+
+ if (bond->alb_info.rlb_enabled)
+ bond->alb_info.rlb_rebalance = 1;
+@@ -1648,7 +1649,8 @@ void bond_alb_handle_link_change(struct bonding *bond, struct slave *slave, char
+ rlb_clear_slave(bond, slave);
+ } else if (link == BOND_LINK_UP) {
+ /* order a rebalance ASAP */
+- bond_info->tx_rebalance_counter = BOND_TLB_REBALANCE_TICKS;
++ atomic_set(&bond_info->tx_rebalance_counter,
++ BOND_TLB_REBALANCE_TICKS);
+ if (bond->alb_info.rlb_enabled) {
+ bond->alb_info.rlb_rebalance = 1;
+ /* If the updelay module parameter is smaller than the
+diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
+index 77dc79a7f5748..2e75b7e8f70b3 100644
+--- a/drivers/net/bonding/bond_main.c
++++ b/drivers/net/bonding/bond_main.c
+@@ -1096,9 +1096,6 @@ static bool bond_should_notify_peers(struct bonding *bond)
+ slave = rcu_dereference(bond->curr_active_slave);
+ rcu_read_unlock();
+
+- netdev_dbg(bond->dev, "bond_should_notify_peers: slave %s\n",
+- slave ? slave->dev->name : "NULL");
+-
+ if (!slave || !bond->send_peer_notif ||
+ bond->send_peer_notif %
+ max(1, bond->params.peer_notif_delay) != 0 ||
+@@ -1106,6 +1103,9 @@ static bool bond_should_notify_peers(struct bonding *bond)
+ test_bit(__LINK_STATE_LINKWATCH_PENDING, &slave->dev->state))
+ return false;
+
++ netdev_dbg(bond->dev, "bond_should_notify_peers: slave %s\n",
++ slave ? slave->dev->name : "NULL");
++
+ return true;
+ }
+
+@@ -2377,10 +2377,9 @@ static int __bond_release_one(struct net_device *bond_dev,
+ bond_select_active_slave(bond);
+ }
+
+- if (!bond_has_slaves(bond)) {
+- bond_set_carrier(bond);
++ bond_set_carrier(bond);
++ if (!bond_has_slaves(bond))
+ eth_hw_addr_random(bond_dev);
+- }
+
+ unblock_netpoll_tx();
+ synchronize_rcu();
+@@ -3818,14 +3817,19 @@ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb, const v
+ return true;
+ }
+
+-static u32 bond_ip_hash(u32 hash, struct flow_keys *flow)
++static u32 bond_ip_hash(u32 hash, struct flow_keys *flow, int xmit_policy)
+ {
+ hash ^= (__force u32)flow_get_u32_dst(flow) ^
+ (__force u32)flow_get_u32_src(flow);
+ hash ^= (hash >> 16);
+ hash ^= (hash >> 8);
++
+ /* discard lowest hash bit to deal with the common even ports pattern */
+- return hash >> 1;
++ if (xmit_policy == BOND_XMIT_POLICY_LAYER34 ||
++ xmit_policy == BOND_XMIT_POLICY_ENCAP34)
++ return hash >> 1;
++
++ return hash;
+ }
+
+ /* Generate hash based on xmit policy. If @skb is given it is used to linearize
+@@ -3855,7 +3859,7 @@ static u32 __bond_xmit_hash(struct bonding *bond, struct sk_buff *skb, const voi
+ memcpy(&hash, &flow.ports.ports, sizeof(hash));
+ }
+
+- return bond_ip_hash(hash, &flow);
++ return bond_ip_hash(hash, &flow, bond->params.xmit_policy);
+ }
+
+ /**
+@@ -3872,8 +3876,8 @@ u32 bond_xmit_hash(struct bonding *bond, struct sk_buff *skb)
+ skb->l4_hash)
+ return skb->hash;
+
+- return __bond_xmit_hash(bond, skb, skb->head, skb->protocol,
+- skb->mac_header, skb->network_header,
++ return __bond_xmit_hash(bond, skb, skb->data, skb->protocol,
++ skb_mac_offset(skb), skb_network_offset(skb),
+ skb_headlen(skb));
+ }
+
+@@ -4843,25 +4847,39 @@ static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb,
+ struct bonding *bond = netdev_priv(bond_dev);
+ struct slave *slave = NULL;
+ struct list_head *iter;
++ bool xmit_suc = false;
++ bool skb_used = false;
+
+ bond_for_each_slave_rcu(bond, slave, iter) {
+- if (bond_is_last_slave(bond, slave))
+- break;
+- if (bond_slave_is_up(slave) && slave->link == BOND_LINK_UP) {
+- struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
++ struct sk_buff *skb2;
++
++ if (!(bond_slave_is_up(slave) && slave->link == BOND_LINK_UP))
++ continue;
+
++ if (bond_is_last_slave(bond, slave)) {
++ skb2 = skb;
++ skb_used = true;
++ } else {
++ skb2 = skb_clone(skb, GFP_ATOMIC);
+ if (!skb2) {
+ net_err_ratelimited("%s: Error: %s: skb_clone() failed\n",
+ bond_dev->name, __func__);
+ continue;
+ }
+- bond_dev_queue_xmit(bond, skb2, slave->dev);
+ }
++
++ if (bond_dev_queue_xmit(bond, skb2, slave->dev) == NETDEV_TX_OK)
++ xmit_suc = true;
+ }
+- if (slave && bond_slave_is_up(slave) && slave->link == BOND_LINK_UP)
+- return bond_dev_queue_xmit(bond, skb, slave->dev);
+
+- return bond_tx_drop(bond_dev, skb);
++ if (!skb_used)
++ dev_kfree_skb_any(skb);
++
++ if (xmit_suc)
++ return NETDEV_TX_OK;
++
++ atomic_long_inc(&bond_dev->tx_dropped);
++ return NET_XMIT_DROP;
+ }
+
+ /*------------------------- Device initialization ---------------------------*/
+@@ -4999,7 +5017,7 @@ static u32 bond_sk_hash_l34(struct sock *sk)
+ /* L4 */
+ memcpy(&hash, &flow.ports.ports, sizeof(hash));
+ /* L3 */
+- return bond_ip_hash(hash, &flow);
++ return bond_ip_hash(hash, &flow, BOND_XMIT_POLICY_LAYER34);
+ }
+
+ static struct net_device *__bond_sk_get_lower_dev(struct bonding *bond,
+diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
+index a8fde3bc458f6..b93337b5a7211 100644
+--- a/drivers/net/bonding/bond_options.c
++++ b/drivers/net/bonding/bond_options.c
+@@ -1526,7 +1526,7 @@ static int bond_option_ad_actor_system_set(struct bonding *bond,
+ mac = (u8 *)&newval->value;
+ }
+
+- if (!is_valid_ether_addr(mac))
++ if (is_multicast_ether_addr(mac))
+ goto err;
+
+ netdev_dbg(bond->dev, "Setting ad_actor_system to %pM\n", mac);
+diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c
+index fd07561da0348..6a6cdd0bb2585 100644
+--- a/drivers/net/bonding/bond_sysfs_slave.c
++++ b/drivers/net/bonding/bond_sysfs_slave.c
+@@ -108,15 +108,15 @@ static ssize_t ad_partner_oper_port_state_show(struct slave *slave, char *buf)
+ }
+ static SLAVE_ATTR_RO(ad_partner_oper_port_state);
+
+-static const struct slave_attribute *slave_attrs[] = {
+- &slave_attr_state,
+- &slave_attr_mii_status,
+- &slave_attr_link_failure_count,
+- &slave_attr_perm_hwaddr,
+- &slave_attr_queue_id,
+- &slave_attr_ad_aggregator_id,
+- &slave_attr_ad_actor_oper_port_state,
+- &slave_attr_ad_partner_oper_port_state,
++static const struct attribute *slave_attrs[] = {
++ &slave_attr_state.attr,
++ &slave_attr_mii_status.attr,
++ &slave_attr_link_failure_count.attr,
++ &slave_attr_perm_hwaddr.attr,
++ &slave_attr_queue_id.attr,
++ &slave_attr_ad_aggregator_id.attr,
++ &slave_attr_ad_actor_oper_port_state.attr,
++ &slave_attr_ad_partner_oper_port_state.attr,
+ NULL
+ };
+
+@@ -137,24 +137,10 @@ const struct sysfs_ops slave_sysfs_ops = {
+
+ int bond_sysfs_slave_add(struct slave *slave)
+ {
+- const struct slave_attribute **a;
+- int err;
+-
+- for (a = slave_attrs; *a; ++a) {
+- err = sysfs_create_file(&slave->kobj, &((*a)->attr));
+- if (err) {
+- kobject_put(&slave->kobj);
+- return err;
+- }
+- }
+-
+- return 0;
++ return sysfs_create_files(&slave->kobj, slave_attrs);
+ }
+
+ void bond_sysfs_slave_del(struct slave *slave)
+ {
+- const struct slave_attribute **a;
+-
+- for (a = slave_attrs; *a; ++a)
+- sysfs_remove_file(&slave->kobj, &((*a)->attr));
++ sysfs_remove_files(&slave->kobj, slave_attrs);
+ }
+diff --git a/drivers/net/can/dev/bittiming.c b/drivers/net/can/dev/bittiming.c
+index f49170eadd547..b1b5a82f08299 100644
+--- a/drivers/net/can/dev/bittiming.c
++++ b/drivers/net/can/dev/bittiming.c
+@@ -209,7 +209,7 @@ static int can_fixup_bittiming(struct net_device *dev, struct can_bittiming *bt,
+ const struct can_bittiming_const *btc)
+ {
+ struct can_priv *priv = netdev_priv(dev);
+- int tseg1, alltseg;
++ unsigned int tseg1, alltseg;
+ u64 brp64;
+
+ tseg1 = bt->prop_seg + bt->phase_seg1;
+diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c
+index 7734229aa0788..18d7bb99ec1bd 100644
+--- a/drivers/net/can/flexcan.c
++++ b/drivers/net/can/flexcan.c
+@@ -173,9 +173,9 @@
+
+ /* FLEXCAN interrupt flag register (IFLAG) bits */
+ /* Errata ERR005829 step7: Reserve first valid MB */
+-#define FLEXCAN_TX_MB_RESERVED_OFF_FIFO 8
+-#define FLEXCAN_TX_MB_RESERVED_OFF_TIMESTAMP 0
+-#define FLEXCAN_RX_MB_OFF_TIMESTAMP_FIRST (FLEXCAN_TX_MB_RESERVED_OFF_TIMESTAMP + 1)
++#define FLEXCAN_TX_MB_RESERVED_RX_FIFO 8
++#define FLEXCAN_TX_MB_RESERVED_RX_MAILBOX 0
++#define FLEXCAN_RX_MB_RX_MAILBOX_FIRST (FLEXCAN_TX_MB_RESERVED_RX_MAILBOX + 1)
+ #define FLEXCAN_IFLAG_MB(x) BIT_ULL(x)
+ #define FLEXCAN_IFLAG_RX_FIFO_OVERFLOW BIT(7)
+ #define FLEXCAN_IFLAG_RX_FIFO_WARN BIT(6)
+@@ -234,8 +234,8 @@
+ #define FLEXCAN_QUIRK_ENABLE_EACEN_RRS BIT(3)
+ /* Disable non-correctable errors interrupt and freeze mode */
+ #define FLEXCAN_QUIRK_DISABLE_MECR BIT(4)
+-/* Use timestamp based offloading */
+-#define FLEXCAN_QUIRK_USE_OFF_TIMESTAMP BIT(5)
++/* Use mailboxes (not FIFO) for RX path */
++#define FLEXCAN_QUIRK_USE_RX_MAILBOX BIT(5)
+ /* No interrupt for error passive */
+ #define FLEXCAN_QUIRK_BROKEN_PERR_STATE BIT(6)
+ /* default to BE register access */
+@@ -252,6 +252,12 @@
+ #define FLEXCAN_QUIRK_NR_IRQ_3 BIT(12)
+ /* Setup 16 mailboxes */
+ #define FLEXCAN_QUIRK_NR_MB_16 BIT(13)
++/* Device supports RX via mailboxes */
++#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX BIT(14)
++/* Device supports RTR reception via mailboxes */
++#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR BIT(15)
++/* Device supports RX via FIFO */
++#define FLEXCAN_QUIRK_SUPPPORT_RX_FIFO BIT(16)
+
+ /* Structure of the message buffer */
+ struct flexcan_mb {
+@@ -365,7 +371,7 @@ struct flexcan_priv {
+
+ struct clk *clk_ipg;
+ struct clk *clk_per;
+- const struct flexcan_devtype_data *devtype_data;
++ struct flexcan_devtype_data devtype_data;
+ struct regulator *reg_xceiver;
+ struct flexcan_stop_mode stm;
+
+@@ -382,59 +388,78 @@ struct flexcan_priv {
+
+ static const struct flexcan_devtype_data fsl_mcf5441x_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+- FLEXCAN_QUIRK_NR_IRQ_3 | FLEXCAN_QUIRK_NR_MB_16,
++ FLEXCAN_QUIRK_NR_IRQ_3 | FLEXCAN_QUIRK_NR_MB_16 |
++ FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+ };
+
+ static const struct flexcan_devtype_data fsl_p1010_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE |
+ FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+- FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN,
++ FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+ };
+
+ static const struct flexcan_devtype_data fsl_imx25_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE |
+- FLEXCAN_QUIRK_BROKEN_PERR_STATE,
++ FLEXCAN_QUIRK_BROKEN_PERR_STATE |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+ };
+
+ static const struct flexcan_devtype_data fsl_imx28_devtype_data = {
+- .quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE,
++ .quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+ };
+
+ static const struct flexcan_devtype_data fsl_imx6q_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+- FLEXCAN_QUIRK_USE_OFF_TIMESTAMP | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+- FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR,
++ FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
++ FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static const struct flexcan_devtype_data fsl_imx8qm_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+- FLEXCAN_QUIRK_USE_OFF_TIMESTAMP | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+- FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW,
++ FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
++ FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static struct flexcan_devtype_data fsl_imx8mp_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+- FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_OFF_TIMESTAMP |
++ FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX |
+ FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR |
+- FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC,
++ FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static const struct flexcan_devtype_data fsl_vf610_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+- FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_OFF_TIMESTAMP |
+- FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SUPPORT_ECC,
++ FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX |
++ FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SUPPORT_ECC |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static const struct flexcan_devtype_data fsl_ls1021a_r2_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+- FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_USE_OFF_TIMESTAMP,
++ FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_USE_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static const struct flexcan_devtype_data fsl_lx2160a_r1_devtype_data = {
+ .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+ FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+- FLEXCAN_QUIRK_USE_OFF_TIMESTAMP | FLEXCAN_QUIRK_SUPPORT_FD |
+- FLEXCAN_QUIRK_SUPPORT_ECC,
++ FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_SUPPORT_FD |
++ FLEXCAN_QUIRK_SUPPORT_ECC |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+ };
+
+ static const struct can_bittiming_const flexcan_bittiming_const = {
+@@ -596,7 +621,7 @@ static inline int flexcan_enter_stop_mode(struct flexcan_priv *priv)
+ priv->write(reg_mcr, &regs->mcr);
+
+ /* enable stop request */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW) {
+ ret = flexcan_stop_mode_enable_scfw(priv, true);
+ if (ret < 0)
+ return ret;
+@@ -615,7 +640,7 @@ static inline int flexcan_exit_stop_mode(struct flexcan_priv *priv)
+ int ret;
+
+ /* remove stop request */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW) {
+ ret = flexcan_stop_mode_enable_scfw(priv, false);
+ if (ret < 0)
+ return ret;
+@@ -1018,7 +1043,7 @@ static struct sk_buff *flexcan_mailbox_read(struct can_rx_offload *offload,
+
+ mb = flexcan_get_mb(priv, n);
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) {
+ u32 code;
+
+ do {
+@@ -1083,7 +1108,7 @@ static struct sk_buff *flexcan_mailbox_read(struct can_rx_offload *offload,
+ }
+
+ mark_as_read:
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX)
+ flexcan_write64(priv, FLEXCAN_IFLAG_MB(n), &regs->iflag1);
+ else
+ priv->write(FLEXCAN_IFLAG_RX_FIFO_AVAILABLE, &regs->iflag1);
+@@ -1109,7 +1134,7 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
+ enum can_state last_state = priv->can.state;
+
+ /* reception interrupt */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) {
+ u64 reg_iflag_rx;
+ int ret;
+
+@@ -1169,7 +1194,7 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
+
+ /* state change interrupt or broken error state quirk fix is enabled */
+ if ((reg_esr & FLEXCAN_ESR_ERR_STATE) ||
+- (priv->devtype_data->quirks & (FLEXCAN_QUIRK_BROKEN_WERR_STATE |
++ (priv->devtype_data.quirks & (FLEXCAN_QUIRK_BROKEN_WERR_STATE |
+ FLEXCAN_QUIRK_BROKEN_PERR_STATE)))
+ flexcan_irq_state(dev, reg_esr);
+
+@@ -1191,11 +1216,11 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
+ * (1): enabled if FLEXCAN_QUIRK_BROKEN_WERR_STATE is enabled
+ */
+ if ((last_state != priv->can.state) &&
+- (priv->devtype_data->quirks & FLEXCAN_QUIRK_BROKEN_PERR_STATE) &&
++ (priv->devtype_data.quirks & FLEXCAN_QUIRK_BROKEN_PERR_STATE) &&
+ !(priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)) {
+ switch (priv->can.state) {
+ case CAN_STATE_ERROR_ACTIVE:
+- if (priv->devtype_data->quirks &
++ if (priv->devtype_data.quirks &
+ FLEXCAN_QUIRK_BROKEN_WERR_STATE)
+ flexcan_error_irq_enable(priv);
+ else
+@@ -1423,26 +1448,26 @@ static int flexcan_rx_offload_setup(struct net_device *dev)
+ else
+ priv->mb_size = sizeof(struct flexcan_mb) + CAN_MAX_DLEN;
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_NR_MB_16)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_NR_MB_16)
+ priv->mb_count = 16;
+ else
+ priv->mb_count = (sizeof(priv->regs->mb[0]) / priv->mb_size) +
+ (sizeof(priv->regs->mb[1]) / priv->mb_size);
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX)
+ priv->tx_mb_reserved =
+- flexcan_get_mb(priv, FLEXCAN_TX_MB_RESERVED_OFF_TIMESTAMP);
++ flexcan_get_mb(priv, FLEXCAN_TX_MB_RESERVED_RX_MAILBOX);
+ else
+ priv->tx_mb_reserved =
+- flexcan_get_mb(priv, FLEXCAN_TX_MB_RESERVED_OFF_FIFO);
++ flexcan_get_mb(priv, FLEXCAN_TX_MB_RESERVED_RX_FIFO);
+ priv->tx_mb_idx = priv->mb_count - 1;
+ priv->tx_mb = flexcan_get_mb(priv, priv->tx_mb_idx);
+ priv->tx_mask = FLEXCAN_IFLAG_MB(priv->tx_mb_idx);
+
+ priv->offload.mailbox_read = flexcan_mailbox_read;
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
+- priv->offload.mb_first = FLEXCAN_RX_MB_OFF_TIMESTAMP_FIRST;
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) {
++ priv->offload.mb_first = FLEXCAN_RX_MB_RX_MAILBOX_FIRST;
+ priv->offload.mb_last = priv->mb_count - 2;
+
+ priv->rx_mask = GENMASK_ULL(priv->offload.mb_last,
+@@ -1506,7 +1531,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ if (err)
+ goto out_chip_disable;
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_ECC)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SUPPORT_ECC)
+ flexcan_ram_init(dev);
+
+ flexcan_set_bittiming(dev);
+@@ -1532,10 +1557,10 @@ static int flexcan_chip_start(struct net_device *dev)
+ /* MCR
+ *
+ * FIFO:
+- * - disable for timestamp mode
++ * - disable for mailbox mode
+ * - enable for FIFO mode
+ */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX)
+ reg_mcr &= ~FLEXCAN_MCR_FEN;
+ else
+ reg_mcr |= FLEXCAN_MCR_FEN;
+@@ -1586,7 +1611,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ * on most Flexcan cores, too. Otherwise we don't get
+ * any error warning or passive interrupts.
+ */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_BROKEN_WERR_STATE ||
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_BROKEN_WERR_STATE ||
+ priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)
+ reg_ctrl |= FLEXCAN_CTRL_ERR_MSK;
+ else
+@@ -1599,7 +1624,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ netdev_dbg(dev, "%s: writing ctrl=0x%08x", __func__, reg_ctrl);
+ priv->write(reg_ctrl, &regs->ctrl);
+
+- if ((priv->devtype_data->quirks & FLEXCAN_QUIRK_ENABLE_EACEN_RRS)) {
++ if ((priv->devtype_data.quirks & FLEXCAN_QUIRK_ENABLE_EACEN_RRS)) {
+ reg_ctrl2 = priv->read(&regs->ctrl2);
+ reg_ctrl2 |= FLEXCAN_CTRL2_EACEN | FLEXCAN_CTRL2_RRS;
+ priv->write(reg_ctrl2, &regs->ctrl2);
+@@ -1631,7 +1656,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ priv->write(reg_fdctrl, &regs->fdctrl);
+ }
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) {
+ for (i = priv->offload.mb_first; i <= priv->offload.mb_last; i++) {
+ mb = flexcan_get_mb(priv, i);
+ priv->write(FLEXCAN_MB_CODE_RX_EMPTY,
+@@ -1639,7 +1664,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ }
+ } else {
+ /* clear and invalidate unused mailboxes first */
+- for (i = FLEXCAN_TX_MB_RESERVED_OFF_FIFO; i < priv->mb_count; i++) {
++ for (i = FLEXCAN_TX_MB_RESERVED_RX_FIFO; i < priv->mb_count; i++) {
+ mb = flexcan_get_mb(priv, i);
+ priv->write(FLEXCAN_MB_CODE_RX_INACTIVE,
+ &mb->can_ctrl);
+@@ -1659,7 +1684,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ priv->write(0x0, &regs->rx14mask);
+ priv->write(0x0, &regs->rx15mask);
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_DISABLE_RXFG)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_DISABLE_RXFG)
+ priv->write(0x0, &regs->rxfgmask);
+
+ /* clear acceptance filters */
+@@ -1673,7 +1698,7 @@ static int flexcan_chip_start(struct net_device *dev)
+ * This also works around errata e5295 which generates false
+ * positive memory errors and put the device in freeze mode.
+ */
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_DISABLE_MECR) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_DISABLE_MECR) {
+ /* Follow the protocol as described in "Detection
+ * and Correction of Memory Errors" to write to
+ * MECR register (step 1 - 5)
+@@ -1799,7 +1824,7 @@ static int flexcan_open(struct net_device *dev)
+ if (err)
+ goto out_can_rx_offload_disable;
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
+ err = request_irq(priv->irq_boff,
+ flexcan_irq, IRQF_SHARED, dev->name, dev);
+ if (err)
+@@ -1845,7 +1870,7 @@ static int flexcan_close(struct net_device *dev)
+ netif_stop_queue(dev);
+ flexcan_chip_interrupts_disable(dev);
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
+ free_irq(priv->irq_err, dev);
+ free_irq(priv->irq_boff, dev);
+ }
+@@ -2051,9 +2076,9 @@ static int flexcan_setup_stop_mode(struct platform_device *pdev)
+
+ priv = netdev_priv(dev);
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW)
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW)
+ ret = flexcan_setup_stop_mode_scfw(pdev);
+- else if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR)
++ else if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR)
+ ret = flexcan_setup_stop_mode_gpr(pdev);
+ else
+ /* return 0 directly if doesn't support stop mode feature */
+@@ -2164,8 +2189,25 @@ static int flexcan_probe(struct platform_device *pdev)
+ return -ENODEV;
+
+ if ((devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_FD) &&
+- !(devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP)) {
+- dev_err(&pdev->dev, "CAN-FD mode doesn't work with FIFO mode!\n");
++ !((devtype_data->quirks &
++ (FLEXCAN_QUIRK_USE_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR |
++ FLEXCAN_QUIRK_SUPPPORT_RX_FIFO)) ==
++ (FLEXCAN_QUIRK_USE_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR))) {
++ dev_err(&pdev->dev, "CAN-FD mode doesn't work in RX-FIFO mode!\n");
++ return -EINVAL;
++ }
++
++ if ((devtype_data->quirks &
++ (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)) ==
++ FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR) {
++ dev_err(&pdev->dev,
++ "Quirks (0x%08x) inconsistent: RX_MAILBOX_RX supported but not RX_MAILBOX\n",
++ devtype_data->quirks);
+ return -EINVAL;
+ }
+
+@@ -2181,9 +2223,10 @@ static int flexcan_probe(struct platform_device *pdev)
+ dev->flags |= IFF_ECHO;
+
+ priv = netdev_priv(dev);
++ priv->devtype_data = *devtype_data;
+
+ if (of_property_read_bool(pdev->dev.of_node, "big-endian") ||
+- devtype_data->quirks & FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN) {
++ priv->devtype_data.quirks & FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN) {
+ priv->read = flexcan_read_be;
+ priv->write = flexcan_write_be;
+ } else {
+@@ -2202,10 +2245,9 @@ static int flexcan_probe(struct platform_device *pdev)
+ priv->clk_ipg = clk_ipg;
+ priv->clk_per = clk_per;
+ priv->clk_src = clk_src;
+- priv->devtype_data = devtype_data;
+ priv->reg_xceiver = reg_xceiver;
+
+- if (devtype_data->quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_NR_IRQ_3) {
+ priv->irq_boff = platform_get_irq(pdev, 1);
+ if (priv->irq_boff <= 0) {
+ err = -ENODEV;
+@@ -2218,7 +2260,7 @@ static int flexcan_probe(struct platform_device *pdev)
+ }
+ }
+
+- if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_FD) {
++ if (priv->devtype_data.quirks & FLEXCAN_QUIRK_SUPPORT_FD) {
+ priv->can.ctrlmode_supported |= CAN_CTRLMODE_FD |
+ CAN_CTRLMODE_FD_NON_ISO;
+ priv->can.bittiming_const = &flexcan_fd_bittiming_const;
+diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c
+index 78e27940b2aff..daee3652ac8b3 100644
+--- a/drivers/net/can/grcan.c
++++ b/drivers/net/can/grcan.c
+@@ -241,13 +241,14 @@ struct grcan_device_config {
+ .rxsize = GRCAN_DEFAULT_BUFFER_SIZE, \
+ }
+
+-#define GRCAN_TXBUG_SAFE_GRLIB_VERSION 0x4100
++#define GRCAN_TXBUG_SAFE_GRLIB_VERSION 4100
+ #define GRLIB_VERSION_MASK 0xffff
+
+ /* GRCAN private data structure */
+ struct grcan_priv {
+ struct can_priv can; /* must be the first member */
+ struct net_device *dev;
++ struct device *ofdev_dev;
+ struct napi_struct napi;
+
+ struct grcan_registers __iomem *regs; /* ioremap'ed registers */
+@@ -924,7 +925,7 @@ static void grcan_free_dma_buffers(struct net_device *dev)
+ struct grcan_priv *priv = netdev_priv(dev);
+ struct grcan_dma *dma = &priv->dma;
+
+- dma_free_coherent(&dev->dev, dma->base_size, dma->base_buf,
++ dma_free_coherent(priv->ofdev_dev, dma->base_size, dma->base_buf,
+ dma->base_handle);
+ memset(dma, 0, sizeof(*dma));
+ }
+@@ -949,7 +950,7 @@ static int grcan_allocate_dma_buffers(struct net_device *dev,
+
+ /* Extra GRCAN_BUFFER_ALIGNMENT to allow for alignment */
+ dma->base_size = lsize + ssize + GRCAN_BUFFER_ALIGNMENT;
+- dma->base_buf = dma_alloc_coherent(&dev->dev,
++ dma->base_buf = dma_alloc_coherent(priv->ofdev_dev,
+ dma->base_size,
+ &dma->base_handle,
+ GFP_KERNEL);
+@@ -1113,8 +1114,10 @@ static int grcan_close(struct net_device *dev)
+
+ priv->closing = true;
+ if (priv->need_txbug_workaround) {
++ spin_unlock_irqrestore(&priv->lock, flags);
+ del_timer_sync(&priv->hang_timer);
+ del_timer_sync(&priv->rr_timer);
++ spin_lock_irqsave(&priv->lock, flags);
+ }
+ netif_stop_queue(dev);
+ grcan_stop_hardware(dev);
+@@ -1134,7 +1137,7 @@ static int grcan_close(struct net_device *dev)
+ return 0;
+ }
+
+-static int grcan_transmit_catch_up(struct net_device *dev, int budget)
++static void grcan_transmit_catch_up(struct net_device *dev)
+ {
+ struct grcan_priv *priv = netdev_priv(dev);
+ unsigned long flags;
+@@ -1142,7 +1145,7 @@ static int grcan_transmit_catch_up(struct net_device *dev, int budget)
+
+ spin_lock_irqsave(&priv->lock, flags);
+
+- work_done = catch_up_echo_skb(dev, budget, true);
++ work_done = catch_up_echo_skb(dev, -1, true);
+ if (work_done) {
+ if (!priv->resetting && !priv->closing &&
+ !(priv->can.ctrlmode & CAN_CTRLMODE_LISTENONLY))
+@@ -1156,8 +1159,6 @@ static int grcan_transmit_catch_up(struct net_device *dev, int budget)
+ }
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+-
+- return work_done;
+ }
+
+ static int grcan_receive(struct net_device *dev, int budget)
+@@ -1239,19 +1240,13 @@ static int grcan_poll(struct napi_struct *napi, int budget)
+ struct net_device *dev = priv->dev;
+ struct grcan_registers __iomem *regs = priv->regs;
+ unsigned long flags;
+- int tx_work_done, rx_work_done;
+- int rx_budget = budget / 2;
+- int tx_budget = budget - rx_budget;
++ int work_done;
+
+- /* Half of the budget for receiving messages */
+- rx_work_done = grcan_receive(dev, rx_budget);
++ work_done = grcan_receive(dev, budget);
+
+- /* Half of the budget for transmitting messages as that can trigger echo
+- * frames being received
+- */
+- tx_work_done = grcan_transmit_catch_up(dev, tx_budget);
++ grcan_transmit_catch_up(dev);
+
+- if (rx_work_done < rx_budget && tx_work_done < tx_budget) {
++ if (work_done < budget) {
+ napi_complete(napi);
+
+ /* Guarantee no interference with a running reset that otherwise
+@@ -1268,7 +1263,7 @@ static int grcan_poll(struct napi_struct *napi, int budget)
+ spin_unlock_irqrestore(&priv->lock, flags);
+ }
+
+- return rx_work_done + tx_work_done;
++ return work_done;
+ }
+
+ /* Work tx bug by waiting while for the risky situation to clear. If that fails,
+@@ -1600,6 +1595,7 @@ static int grcan_setup_netdev(struct platform_device *ofdev,
+ memcpy(&priv->config, &grcan_module_config,
+ sizeof(struct grcan_device_config));
+ priv->dev = dev;
++ priv->ofdev_dev = &ofdev->dev;
+ priv->regs = base;
+ priv->can.bittiming_const = &grcan_bittiming_const;
+ priv->can.do_set_bittiming = grcan_set_bittiming;
+@@ -1652,6 +1648,7 @@ exit_free_candev:
+ static int grcan_probe(struct platform_device *ofdev)
+ {
+ struct device_node *np = ofdev->dev.of_node;
++ struct device_node *sysid_parent;
+ u32 sysid, ambafreq;
+ int irq, err;
+ void __iomem *base;
+@@ -1660,10 +1657,15 @@ static int grcan_probe(struct platform_device *ofdev)
+ /* Compare GRLIB version number with the first that does not
+ * have the tx bug (see start_xmit)
+ */
+- err = of_property_read_u32(np, "systemid", &sysid);
+- if (!err && ((sysid & GRLIB_VERSION_MASK)
+- >= GRCAN_TXBUG_SAFE_GRLIB_VERSION))
+- txbug = false;
++ sysid_parent = of_find_node_by_path("/ambapp0");
++ if (sysid_parent) {
++ of_node_get(sysid_parent);
++ err = of_property_read_u32(sysid_parent, "systemid", &sysid);
++ if (!err && ((sysid & GRLIB_VERSION_MASK) >=
++ GRCAN_TXBUG_SAFE_GRLIB_VERSION))
++ txbug = false;
++ of_node_put(sysid_parent);
++ }
+
+ err = of_property_read_u32(np, "freq", &ambafreq);
+ if (err) {
+diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c
+index 74d9899fc904c..eb74cdf26b88c 100644
+--- a/drivers/net/can/kvaser_pciefd.c
++++ b/drivers/net/can/kvaser_pciefd.c
+@@ -248,6 +248,9 @@ MODULE_DESCRIPTION("CAN driver for Kvaser CAN/PCIe devices");
+ #define KVASER_PCIEFD_SPACK_EWLR BIT(23)
+ #define KVASER_PCIEFD_SPACK_EPLR BIT(24)
+
++/* Kvaser KCAN_EPACK second word */
++#define KVASER_PCIEFD_EPACK_DIR_TX BIT(0)
++
+ struct kvaser_pciefd;
+
+ struct kvaser_pciefd_can {
+@@ -1285,7 +1288,10 @@ static int kvaser_pciefd_rx_error_frame(struct kvaser_pciefd_can *can,
+
+ can->err_rep_cnt++;
+ can->can.can_stats.bus_error++;
+- stats->rx_errors++;
++ if (p->header[1] & KVASER_PCIEFD_EPACK_DIR_TX)
++ stats->tx_errors++;
++ else
++ stats->rx_errors++;
+
+ can->bec.txerr = bec.txerr;
+ can->bec.rxerr = bec.rxerr;
+diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
+index 2470c47b2e315..bff33a619acde 100644
+--- a/drivers/net/can/m_can/m_can.c
++++ b/drivers/net/can/m_can/m_can.c
+@@ -204,16 +204,16 @@ enum m_can_reg {
+
+ /* Interrupts for version 3.0.x */
+ #define IR_ERR_LEC_30X (IR_STE | IR_FOE | IR_ACKE | IR_BE | IR_CRCE)
+-#define IR_ERR_BUS_30X (IR_ERR_LEC_30X | IR_WDI | IR_ELO | IR_BEU | \
+- IR_BEC | IR_TOO | IR_MRAF | IR_TSW | IR_TEFL | \
+- IR_RF1L | IR_RF0L)
++#define IR_ERR_BUS_30X (IR_ERR_LEC_30X | IR_WDI | IR_BEU | IR_BEC | \
++ IR_TOO | IR_MRAF | IR_TSW | IR_TEFL | IR_RF1L | \
++ IR_RF0L)
+ #define IR_ERR_ALL_30X (IR_ERR_STATE | IR_ERR_BUS_30X)
+
+ /* Interrupts for version >= 3.1.x */
+ #define IR_ERR_LEC_31X (IR_PED | IR_PEA)
+-#define IR_ERR_BUS_31X (IR_ERR_LEC_31X | IR_WDI | IR_ELO | IR_BEU | \
+- IR_BEC | IR_TOO | IR_MRAF | IR_TSW | IR_TEFL | \
+- IR_RF1L | IR_RF0L)
++#define IR_ERR_BUS_31X (IR_ERR_LEC_31X | IR_WDI | IR_BEU | IR_BEC | \
++ IR_TOO | IR_MRAF | IR_TSW | IR_TEFL | IR_RF1L | \
++ IR_RF0L)
+ #define IR_ERR_ALL_31X (IR_ERR_STATE | IR_ERR_BUS_31X)
+
+ /* Interrupt Line Select (ILS) */
+@@ -336,6 +336,9 @@ m_can_fifo_read(struct m_can_classdev *cdev,
+ u32 addr_offset = cdev->mcfg[MRAM_RXF0].off + fgi * RXF0_ELEMENT_SIZE +
+ offset;
+
++ if (val_count == 0)
++ return 0;
++
+ return cdev->ops->read_fifo(cdev, addr_offset, val, val_count);
+ }
+
+@@ -346,6 +349,9 @@ m_can_fifo_write(struct m_can_classdev *cdev,
+ u32 addr_offset = cdev->mcfg[MRAM_TXB].off + fpi * TXB_ELEMENT_SIZE +
+ offset;
+
++ if (val_count == 0)
++ return 0;
++
+ return cdev->ops->write_fifo(cdev, addr_offset, val, val_count);
+ }
+
+@@ -517,7 +523,7 @@ static int m_can_read_fifo(struct net_device *dev, u32 rxfs)
+ err = m_can_fifo_read(cdev, fgi, M_CAN_FIFO_DATA,
+ cf->data, DIV_ROUND_UP(cf->len, 4));
+ if (err)
+- goto out_fail;
++ goto out_free_skb;
+ }
+
+ /* acknowledge rx fifo 0 */
+@@ -532,6 +538,8 @@ static int m_can_read_fifo(struct net_device *dev, u32 rxfs)
+
+ return 0;
+
++out_free_skb:
++ kfree_skb(skb);
+ out_fail:
+ netdev_err(dev, "FIFO read returned %d\n", err);
+ return err;
+@@ -810,8 +818,6 @@ static void m_can_handle_other_err(struct net_device *dev, u32 irqstatus)
+ {
+ if (irqstatus & IR_WDI)
+ netdev_err(dev, "Message RAM Watchdog event due to missing READY\n");
+- if (irqstatus & IR_ELO)
+- netdev_err(dev, "Error Logging Overflow\n");
+ if (irqstatus & IR_BEU)
+ netdev_err(dev, "Bit Error Uncorrected\n");
+ if (irqstatus & IR_BEC)
+@@ -1494,20 +1500,32 @@ static int m_can_dev_setup(struct m_can_classdev *cdev)
+ case 30:
+ /* CAN_CTRLMODE_FD_NON_ISO is fixed with M_CAN IP v3.0.x */
+ can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO);
+- cdev->can.bittiming_const = &m_can_bittiming_const_30X;
+- cdev->can.data_bittiming_const = &m_can_data_bittiming_const_30X;
++ cdev->can.bittiming_const = cdev->bit_timing ?
++ cdev->bit_timing : &m_can_bittiming_const_30X;
++
++ cdev->can.data_bittiming_const = cdev->data_timing ?
++ cdev->data_timing :
++ &m_can_data_bittiming_const_30X;
+ break;
+ case 31:
+ /* CAN_CTRLMODE_FD_NON_ISO is fixed with M_CAN IP v3.1.x */
+ can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO);
+- cdev->can.bittiming_const = &m_can_bittiming_const_31X;
+- cdev->can.data_bittiming_const = &m_can_data_bittiming_const_31X;
++ cdev->can.bittiming_const = cdev->bit_timing ?
++ cdev->bit_timing : &m_can_bittiming_const_31X;
++
++ cdev->can.data_bittiming_const = cdev->data_timing ?
++ cdev->data_timing :
++ &m_can_data_bittiming_const_31X;
+ break;
+ case 32:
+ case 33:
+ /* Support both MCAN version v3.2.x and v3.3.0 */
+- cdev->can.bittiming_const = &m_can_bittiming_const_31X;
+- cdev->can.data_bittiming_const = &m_can_data_bittiming_const_31X;
++ cdev->can.bittiming_const = cdev->bit_timing ?
++ cdev->bit_timing : &m_can_bittiming_const_31X;
++
++ cdev->can.data_bittiming_const = cdev->data_timing ?
++ cdev->data_timing :
++ &m_can_data_bittiming_const_31X;
+
+ cdev->can.ctrlmode_supported |=
+ (m_can_niso_supported(cdev) ?
+@@ -1622,8 +1640,6 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev)
+ if (err)
+ goto out_fail;
+
+- can_put_echo_skb(skb, dev, 0, 0);
+-
+ if (cdev->can.ctrlmode & CAN_CTRLMODE_FD) {
+ cccr = m_can_read(cdev, M_CAN_CCCR);
+ cccr &= ~CCCR_CMR_MASK;
+@@ -1640,6 +1656,9 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev)
+ m_can_write(cdev, M_CAN_CCCR, cccr);
+ }
+ m_can_write(cdev, M_CAN_TXBTIE, 0x1);
++
++ can_put_echo_skb(skb, dev, 0, 0);
++
+ m_can_write(cdev, M_CAN_TXBAR, 0x1);
+ /* End of xmit function for version 3.0.x */
+ } else {
+diff --git a/drivers/net/can/m_can/m_can.h b/drivers/net/can/m_can/m_can.h
+index d18b515e6ccc7..2c5d409971686 100644
+--- a/drivers/net/can/m_can/m_can.h
++++ b/drivers/net/can/m_can/m_can.h
+@@ -85,6 +85,9 @@ struct m_can_classdev {
+ struct sk_buff *tx_skb;
+ struct phy *transceiver;
+
++ const struct can_bittiming_const *bit_timing;
++ const struct can_bittiming_const *data_timing;
++
+ struct m_can_ops *ops;
+
+ int version;
+diff --git a/drivers/net/can/m_can/m_can_pci.c b/drivers/net/can/m_can/m_can_pci.c
+index 89cc3d41e952b..8f184a852a0a7 100644
+--- a/drivers/net/can/m_can/m_can_pci.c
++++ b/drivers/net/can/m_can/m_can_pci.c
+@@ -18,7 +18,7 @@
+
+ #define M_CAN_PCI_MMIO_BAR 0
+
+-#define M_CAN_CLOCK_FREQ_EHL 100000000
++#define M_CAN_CLOCK_FREQ_EHL 200000000
+ #define CTL_CSR_INT_CTL_OFFSET 0x508
+
+ struct m_can_pci_priv {
+@@ -42,8 +42,13 @@ static u32 iomap_read_reg(struct m_can_classdev *cdev, int reg)
+ static int iomap_read_fifo(struct m_can_classdev *cdev, int offset, void *val, size_t val_count)
+ {
+ struct m_can_pci_priv *priv = cdev_to_priv(cdev);
++ void __iomem *src = priv->base + offset;
+
+- ioread32_rep(priv->base + offset, val, val_count);
++ while (val_count--) {
++ *(unsigned int *)val = ioread32(src);
++ val += 4;
++ src += 4;
++ }
+
+ return 0;
+ }
+@@ -61,8 +66,13 @@ static int iomap_write_fifo(struct m_can_classdev *cdev, int offset,
+ const void *val, size_t val_count)
+ {
+ struct m_can_pci_priv *priv = cdev_to_priv(cdev);
++ void __iomem *dst = priv->base + offset;
+
+- iowrite32_rep(priv->base + offset, val, val_count);
++ while (val_count--) {
++ iowrite32(*(unsigned int *)val, dst);
++ val += 4;
++ dst += 4;
++ }
+
+ return 0;
+ }
+diff --git a/drivers/net/can/m_can/tcan4x5x-regmap.c b/drivers/net/can/m_can/tcan4x5x-regmap.c
+index ca80dbaf7a3f5..26e212b8ca7a6 100644
+--- a/drivers/net/can/m_can/tcan4x5x-regmap.c
++++ b/drivers/net/can/m_can/tcan4x5x-regmap.c
+@@ -12,7 +12,7 @@
+ #define TCAN4X5X_SPI_INSTRUCTION_WRITE (0x61 << 24)
+ #define TCAN4X5X_SPI_INSTRUCTION_READ (0x41 << 24)
+
+-#define TCAN4X5X_MAX_REGISTER 0x8ffc
++#define TCAN4X5X_MAX_REGISTER 0x87fc
+
+ static int tcan4x5x_regmap_gather_write(void *context,
+ const void *reg, size_t reg_len,
+diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c
+index 92a54a5fd4c50..964c8a09226a9 100644
+--- a/drivers/net/can/pch_can.c
++++ b/drivers/net/can/pch_can.c
+@@ -692,11 +692,11 @@ static int pch_can_rx_normal(struct net_device *ndev, u32 obj_num, int quota)
+ cf->data[i + 1] = data_reg >> 8;
+ }
+
+- netif_receive_skb(skb);
+ rcv_pkts++;
+ stats->rx_packets++;
+ quota--;
+ stats->rx_bytes += cf->len;
++ netif_receive_skb(skb);
+
+ pch_fifo_thresh(priv, obj_num);
+ obj_num++;
+diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c
+index ff9d0f5ae0dd2..2f44c567ebd73 100644
+--- a/drivers/net/can/rcar/rcar_canfd.c
++++ b/drivers/net/can/rcar/rcar_canfd.c
+@@ -1640,8 +1640,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
+ ndev = alloc_candev(sizeof(*priv), RCANFD_FIFO_DEPTH);
+ if (!ndev) {
+ dev_err(&pdev->dev, "alloc_candev() failed\n");
+- err = -ENOMEM;
+- goto fail;
++ return -ENOMEM;
+ }
+ priv = netdev_priv(ndev);
+
+@@ -1721,22 +1720,22 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
+
+ netif_napi_add(ndev, &priv->napi, rcar_canfd_rx_poll,
+ RCANFD_NAPI_WEIGHT);
++ spin_lock_init(&priv->tx_lock);
++ devm_can_led_init(ndev);
++ gpriv->ch[priv->channel] = priv;
+ err = register_candev(ndev);
+ if (err) {
+ dev_err(&pdev->dev,
+ "register_candev() failed, error %d\n", err);
+ goto fail_candev;
+ }
+- spin_lock_init(&priv->tx_lock);
+- devm_can_led_init(ndev);
+- gpriv->ch[priv->channel] = priv;
+ dev_info(&pdev->dev, "device registered (channel %u)\n", priv->channel);
+ return 0;
+
+ fail_candev:
+ netif_napi_del(&priv->napi);
+- free_candev(ndev);
+ fail:
++ free_candev(ndev);
+ return err;
+ }
+
+diff --git a/drivers/net/can/sja1000/ems_pcmcia.c b/drivers/net/can/sja1000/ems_pcmcia.c
+index e21b169c14c01..4642b6d4aaf7b 100644
+--- a/drivers/net/can/sja1000/ems_pcmcia.c
++++ b/drivers/net/can/sja1000/ems_pcmcia.c
+@@ -234,7 +234,12 @@ static int ems_pcmcia_add_card(struct pcmcia_device *pdev, unsigned long base)
+ free_sja1000dev(dev);
+ }
+
+- err = request_irq(dev->irq, &ems_pcmcia_interrupt, IRQF_SHARED,
++ if (!card->channels) {
++ err = -ENODEV;
++ goto failure_cleanup;
++ }
++
++ err = request_irq(pdev->irq, &ems_pcmcia_interrupt, IRQF_SHARED,
+ DRV_NAME, card);
+ if (!err)
+ return 0;
+diff --git a/drivers/net/can/softing/softing_cs.c b/drivers/net/can/softing/softing_cs.c
+index 2e93ee7923739..e5c939b63fa65 100644
+--- a/drivers/net/can/softing/softing_cs.c
++++ b/drivers/net/can/softing/softing_cs.c
+@@ -293,7 +293,7 @@ static int softingcs_probe(struct pcmcia_device *pcmcia)
+ return 0;
+
+ platform_failed:
+- kfree(dev);
++ platform_device_put(pdev);
+ mem_failed:
+ pcmcia_bad:
+ pcmcia_failed:
+diff --git a/drivers/net/can/softing/softing_fw.c b/drivers/net/can/softing/softing_fw.c
+index 7e15368779931..32286f861a195 100644
+--- a/drivers/net/can/softing/softing_fw.c
++++ b/drivers/net/can/softing/softing_fw.c
+@@ -565,18 +565,19 @@ int softing_startstop(struct net_device *dev, int up)
+ if (ret < 0)
+ goto failed;
+ }
+- /* enable_error_frame */
+- /*
++
++ /* enable_error_frame
++ *
+ * Error reporting is switched off at the moment since
+ * the receiving of them is not yet 100% verified
+ * This should be enabled sooner or later
+- *
+- if (error_reporting) {
++ */
++ if (0 && error_reporting) {
+ ret = softing_fct_cmd(card, 51, "enable_error_frame");
+ if (ret < 0)
+ goto failed;
+ }
+- */
++
+ /* initialize interface */
+ iowrite16(1, &card->dpram[DPRAM_FCT_PARAM + 2]);
+ iowrite16(1, &card->dpram[DPRAM_FCT_PARAM + 4]);
+diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c
+index 673861ab665a4..3a0f022b15625 100644
+--- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c
++++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c
+@@ -1092,7 +1092,7 @@ static int mcp251xfd_chip_start(struct mcp251xfd_priv *priv)
+
+ err = mcp251xfd_chip_rx_int_enable(priv);
+ if (err)
+- return err;
++ goto out_chip_stop;
+
+ err = mcp251xfd_chip_ecc_init(priv);
+ if (err)
+@@ -1336,7 +1336,7 @@ mcp251xfd_tef_obj_read(const struct mcp251xfd_priv *priv,
+ len > tx_ring->obj_num ||
+ offset + len > tx_ring->obj_num)) {
+ netdev_err(priv->ndev,
+- "Trying to read to many TEF objects (max=%d, offset=%d, len=%d).\n",
++ "Trying to read too many TEF objects (max=%d, offset=%d, len=%d).\n",
+ tx_ring->obj_num, offset, len);
+ return -ERANGE;
+ }
+@@ -2290,8 +2290,10 @@ static irqreturn_t mcp251xfd_irq(int irq, void *dev_id)
+ * check will fail, too. So leave IRQ handler
+ * directly.
+ */
+- if (priv->can.state == CAN_STATE_BUS_OFF)
++ if (priv->can.state == CAN_STATE_BUS_OFF) {
++ can_rx_offload_threaded_irq_finish(&priv->offload);
+ return IRQ_HANDLED;
++ }
+ }
+
+ handled = IRQ_HANDLED;
+@@ -2623,7 +2625,7 @@ static int mcp251xfd_register_chip_detect(struct mcp251xfd_priv *priv)
+ if (!mcp251xfd_is_251X(priv) &&
+ priv->devtype_data.model != devtype_data->model) {
+ netdev_info(ndev,
+- "Detected %s, but firmware specifies a %s. Fixing up.",
++ "Detected %s, but firmware specifies a %s. Fixing up.\n",
+ __mcp251xfd_get_model_str(devtype_data->model),
+ mcp251xfd_get_model_str(priv));
+ }
+@@ -2660,7 +2662,7 @@ static int mcp251xfd_register_check_rx_int(struct mcp251xfd_priv *priv)
+ return 0;
+
+ netdev_info(priv->ndev,
+- "RX_INT active after softreset, disabling RX_INT support.");
++ "RX_INT active after softreset, disabling RX_INT support.\n");
+ devm_gpiod_put(&priv->spi->dev, priv->rx_int);
+ priv->rx_int = NULL;
+
+@@ -2704,7 +2706,7 @@ mcp251xfd_register_get_dev_id(const struct mcp251xfd_priv *priv,
+ out_kfree_buf_rx:
+ kfree(buf_rx);
+
+- return 0;
++ return err;
+ }
+
+ #define MCP251XFD_QUIRK_ACTIVE(quirk) \
+diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c
+index 2b5302e724353..c9552846fe257 100644
+--- a/drivers/net/can/usb/ems_usb.c
++++ b/drivers/net/can/usb/ems_usb.c
+@@ -823,7 +823,6 @@ static netdev_tx_t ems_usb_start_xmit(struct sk_buff *skb, struct net_device *ne
+
+ usb_unanchor_urb(urb);
+ usb_free_coherent(dev->udev, size, buf, urb->transfer_dma);
+- dev_kfree_skb(skb);
+
+ atomic_dec(&dev->active_tx_urbs);
+
+diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c
+index 96a13c770e4a1..cd4e7f356e488 100644
+--- a/drivers/net/can/usb/etas_es58x/es58x_core.c
++++ b/drivers/net/can/usb/etas_es58x/es58x_core.c
+@@ -664,7 +664,7 @@ int es58x_rx_err_msg(struct net_device *netdev, enum es58x_err error,
+ struct can_device_stats *can_stats = &can->can_stats;
+ struct can_frame *cf = NULL;
+ struct sk_buff *skb;
+- int ret;
++ int ret = 0;
+
+ if (!netif_running(netdev)) {
+ if (net_ratelimit())
+@@ -823,8 +823,6 @@ int es58x_rx_err_msg(struct net_device *netdev, enum es58x_err error,
+ can->state = CAN_STATE_BUS_OFF;
+ can_bus_off(netdev);
+ ret = can->do_set_mode(netdev, CAN_MODE_STOP);
+- if (ret)
+- return ret;
+ }
+ break;
+
+@@ -881,7 +879,7 @@ int es58x_rx_err_msg(struct net_device *netdev, enum es58x_err error,
+ ES58X_EVENT_BUSOFF, timestamp);
+ }
+
+- return 0;
++ return ret;
+ }
+
+ /**
+@@ -1796,7 +1794,7 @@ static int es58x_open(struct net_device *netdev)
+ struct es58x_device *es58x_dev = es58x_priv(netdev)->es58x_dev;
+ int ret;
+
+- if (atomic_inc_return(&es58x_dev->opened_channel_cnt) == 1) {
++ if (!es58x_dev->opened_channel_cnt) {
+ ret = es58x_alloc_rx_urbs(es58x_dev);
+ if (ret)
+ return ret;
+@@ -1814,12 +1812,13 @@ static int es58x_open(struct net_device *netdev)
+ if (ret)
+ goto free_urbs;
+
++ es58x_dev->opened_channel_cnt++;
+ netif_start_queue(netdev);
+
+ return ret;
+
+ free_urbs:
+- if (atomic_dec_and_test(&es58x_dev->opened_channel_cnt))
++ if (!es58x_dev->opened_channel_cnt)
+ es58x_free_urbs(es58x_dev);
+ netdev_err(netdev, "%s: Could not open the network device: %pe\n",
+ __func__, ERR_PTR(ret));
+@@ -1854,7 +1853,8 @@ static int es58x_stop(struct net_device *netdev)
+
+ es58x_flush_pending_tx_msg(netdev);
+
+- if (atomic_dec_and_test(&es58x_dev->opened_channel_cnt))
++ es58x_dev->opened_channel_cnt--;
++ if (!es58x_dev->opened_channel_cnt)
+ es58x_free_urbs(es58x_dev);
+
+ return 0;
+@@ -2223,7 +2223,6 @@ static struct es58x_device *es58x_init_es58x_dev(struct usb_interface *intf,
+ init_usb_anchor(&es58x_dev->tx_urbs_idle);
+ init_usb_anchor(&es58x_dev->tx_urbs_busy);
+ atomic_set(&es58x_dev->tx_urbs_idle_cnt, 0);
+- atomic_set(&es58x_dev->opened_channel_cnt, 0);
+ usb_set_intfdata(intf, es58x_dev);
+
+ es58x_dev->rx_pipe = usb_rcvbulkpipe(es58x_dev->udev,
+diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.h b/drivers/net/can/usb/etas_es58x/es58x_core.h
+index 826a15871573a..e5033cb5e6959 100644
+--- a/drivers/net/can/usb/etas_es58x/es58x_core.h
++++ b/drivers/net/can/usb/etas_es58x/es58x_core.h
+@@ -373,8 +373,6 @@ struct es58x_operators {
+ * queue wake/stop logic should prevent this URB from getting
+ * empty. Please refer to es58x_get_tx_urb() for more details.
+ * @tx_urbs_idle_cnt: number of urbs in @tx_urbs_idle.
+- * @opened_channel_cnt: number of channels opened (c.f. es58x_open()
+- * and es58x_stop()).
+ * @ktime_req_ns: kernel timestamp when es58x_set_realtime_diff_ns()
+ * was called.
+ * @realtime_diff_ns: difference in nanoseconds between the clocks of
+@@ -384,6 +382,10 @@ struct es58x_operators {
+ * in RX branches.
+ * @rx_max_packet_size: Maximum length of bulk-in URB.
+ * @num_can_ch: Number of CAN channel (i.e. number of elements of @netdev).
++ * @opened_channel_cnt: number of channels opened. Free of race
++ * conditions because its two users (net_device_ops:ndo_open()
++ * and net_device_ops:ndo_close()) guarantee that the network
++ * stack big kernel lock (a.k.a. rtnl_mutex) is being hold.
+ * @rx_cmd_buf_len: Length of @rx_cmd_buf.
+ * @rx_cmd_buf: The device might split the URB commands in an
+ * arbitrary amount of pieces. This buffer is used to concatenate
+@@ -406,7 +408,6 @@ struct es58x_device {
+ struct usb_anchor tx_urbs_busy;
+ struct usb_anchor tx_urbs_idle;
+ atomic_t tx_urbs_idle_cnt;
+- atomic_t opened_channel_cnt;
+
+ u64 ktime_req_ns;
+ s64 realtime_diff_ns;
+@@ -415,6 +416,7 @@ struct es58x_device {
+
+ u16 rx_max_packet_size;
+ u8 num_can_ch;
++ u8 opened_channel_cnt;
+
+ u16 rx_cmd_buf_len;
+ union es58x_urb_cmd rx_cmd_buf;
+diff --git a/drivers/net/can/usb/etas_es58x/es58x_fd.c b/drivers/net/can/usb/etas_es58x/es58x_fd.c
+index af042aa55f59e..26bf4775e884c 100644
+--- a/drivers/net/can/usb/etas_es58x/es58x_fd.c
++++ b/drivers/net/can/usb/etas_es58x/es58x_fd.c
+@@ -171,12 +171,11 @@ static int es58x_fd_rx_event_msg(struct net_device *netdev,
+ const struct es58x_fd_rx_event_msg *rx_event_msg;
+ int ret;
+
++ rx_event_msg = &es58x_fd_urb_cmd->rx_event_msg;
+ ret = es58x_check_msg_len(es58x_dev->dev, *rx_event_msg, msg_len);
+ if (ret)
+ return ret;
+
+- rx_event_msg = &es58x_fd_urb_cmd->rx_event_msg;
+-
+ return es58x_rx_err_msg(netdev, rx_event_msg->error_code,
+ rx_event_msg->event_code,
+ get_unaligned_le64(&rx_event_msg->timestamp));
+diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
+index 5e892bef46b00..d68d698f2606b 100644
+--- a/drivers/net/can/usb/gs_usb.c
++++ b/drivers/net/can/usb/gs_usb.c
+@@ -191,8 +191,8 @@ struct gs_can {
+ struct gs_usb {
+ struct gs_can *canch[GS_MAX_INTF];
+ struct usb_anchor rx_submitted;
+- atomic_t active_channels;
+ struct usb_device *udev;
++ u8 active_channels;
+ };
+
+ /* 'allocate' a tx context.
+@@ -321,7 +321,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
+
+ /* device reports out of range channel id */
+ if (hf->channel >= GS_MAX_INTF)
+- goto resubmit_urb;
++ goto device_detach;
+
+ dev = usbcan->canch[hf->channel];
+
+@@ -406,6 +406,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
+
+ /* USB failure take down all interfaces */
+ if (rc == -ENODEV) {
++ device_detach:
+ for (rc = 0; rc < GS_MAX_INTF; rc++) {
+ if (usbcan->canch[rc])
+ netif_device_detach(usbcan->canch[rc]->netdev);
+@@ -507,6 +508,8 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
+
+ hf->echo_id = idx;
+ hf->channel = dev->channel;
++ hf->flags = 0;
++ hf->reserved = 0;
+
+ cf = (struct can_frame *)skb->data;
+
+@@ -587,7 +590,7 @@ static int gs_can_open(struct net_device *netdev)
+ if (rc)
+ return rc;
+
+- if (atomic_add_return(1, &parent->active_channels) == 1) {
++ if (!parent->active_channels) {
+ for (i = 0; i < GS_MAX_RX_URBS; i++) {
+ struct urb *urb;
+ u8 *buf;
+@@ -688,6 +691,7 @@ static int gs_can_open(struct net_device *netdev)
+
+ dev->can.state = CAN_STATE_ERROR_ACTIVE;
+
++ parent->active_channels++;
+ if (!(dev->can.ctrlmode & CAN_CTRLMODE_LISTENONLY))
+ netif_start_queue(netdev);
+
+@@ -703,7 +707,8 @@ static int gs_can_close(struct net_device *netdev)
+ netif_stop_queue(netdev);
+
+ /* Stop polling */
+- if (atomic_dec_and_test(&parent->active_channels))
++ parent->active_channels--;
++ if (!parent->active_channels)
+ usb_kill_anchored_urbs(&parent->rx_submitted);
+
+ /* Stop sending URBs */
+@@ -982,8 +987,6 @@ static int gs_usb_probe(struct usb_interface *intf,
+
+ init_usb_anchor(&dev->rx_submitted);
+
+- atomic_set(&dev->active_channels, 0);
+-
+ usb_set_intfdata(intf, dev);
+ dev->udev = interface_to_usbdev(intf);
+
+diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c
+index 59ba7c7beec00..f7af1bf5ab46d 100644
+--- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c
++++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c
+@@ -28,10 +28,6 @@
+
+ #include "kvaser_usb.h"
+
+-/* Forward declaration */
+-static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg;
+-
+-#define CAN_USB_CLOCK 8000000
+ #define MAX_USBCAN_NET_DEVICES 2
+
+ /* Command header size */
+@@ -80,6 +76,12 @@ static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg;
+
+ #define CMD_LEAF_LOG_MESSAGE 106
+
++/* Leaf frequency options */
++#define KVASER_USB_LEAF_SWOPTION_FREQ_MASK 0x60
++#define KVASER_USB_LEAF_SWOPTION_FREQ_16_MHZ_CLK 0
++#define KVASER_USB_LEAF_SWOPTION_FREQ_32_MHZ_CLK BIT(5)
++#define KVASER_USB_LEAF_SWOPTION_FREQ_24_MHZ_CLK BIT(6)
++
+ /* error factors */
+ #define M16C_EF_ACKE BIT(0)
+ #define M16C_EF_CRCE BIT(1)
+@@ -340,6 +342,50 @@ struct kvaser_usb_err_summary {
+ };
+ };
+
++static const struct can_bittiming_const kvaser_usb_leaf_bittiming_const = {
++ .name = "kvaser_usb",
++ .tseg1_min = KVASER_USB_TSEG1_MIN,
++ .tseg1_max = KVASER_USB_TSEG1_MAX,
++ .tseg2_min = KVASER_USB_TSEG2_MIN,
++ .tseg2_max = KVASER_USB_TSEG2_MAX,
++ .sjw_max = KVASER_USB_SJW_MAX,
++ .brp_min = KVASER_USB_BRP_MIN,
++ .brp_max = KVASER_USB_BRP_MAX,
++ .brp_inc = KVASER_USB_BRP_INC,
++};
++
++static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg_8mhz = {
++ .clock = {
++ .freq = 8000000,
++ },
++ .timestamp_freq = 1,
++ .bittiming_const = &kvaser_usb_leaf_bittiming_const,
++};
++
++static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg_16mhz = {
++ .clock = {
++ .freq = 16000000,
++ },
++ .timestamp_freq = 1,
++ .bittiming_const = &kvaser_usb_leaf_bittiming_const,
++};
++
++static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg_24mhz = {
++ .clock = {
++ .freq = 24000000,
++ },
++ .timestamp_freq = 1,
++ .bittiming_const = &kvaser_usb_leaf_bittiming_const,
++};
++
++static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg_32mhz = {
++ .clock = {
++ .freq = 32000000,
++ },
++ .timestamp_freq = 1,
++ .bittiming_const = &kvaser_usb_leaf_bittiming_const,
++};
++
+ static void *
+ kvaser_usb_leaf_frame_to_cmd(const struct kvaser_usb_net_priv *priv,
+ const struct sk_buff *skb, int *frame_len,
+@@ -471,6 +517,27 @@ static int kvaser_usb_leaf_send_simple_cmd(const struct kvaser_usb *dev,
+ return rc;
+ }
+
++static void kvaser_usb_leaf_get_software_info_leaf(struct kvaser_usb *dev,
++ const struct leaf_cmd_softinfo *softinfo)
++{
++ u32 sw_options = le32_to_cpu(softinfo->sw_options);
++
++ dev->fw_version = le32_to_cpu(softinfo->fw_version);
++ dev->max_tx_urbs = le16_to_cpu(softinfo->max_outstanding_tx);
++
++ switch (sw_options & KVASER_USB_LEAF_SWOPTION_FREQ_MASK) {
++ case KVASER_USB_LEAF_SWOPTION_FREQ_16_MHZ_CLK:
++ dev->cfg = &kvaser_usb_leaf_dev_cfg_16mhz;
++ break;
++ case KVASER_USB_LEAF_SWOPTION_FREQ_24_MHZ_CLK:
++ dev->cfg = &kvaser_usb_leaf_dev_cfg_24mhz;
++ break;
++ case KVASER_USB_LEAF_SWOPTION_FREQ_32_MHZ_CLK:
++ dev->cfg = &kvaser_usb_leaf_dev_cfg_32mhz;
++ break;
++ }
++}
++
+ static int kvaser_usb_leaf_get_software_info_inner(struct kvaser_usb *dev)
+ {
+ struct kvaser_cmd cmd;
+@@ -486,14 +553,13 @@ static int kvaser_usb_leaf_get_software_info_inner(struct kvaser_usb *dev)
+
+ switch (dev->card_data.leaf.family) {
+ case KVASER_LEAF:
+- dev->fw_version = le32_to_cpu(cmd.u.leaf.softinfo.fw_version);
+- dev->max_tx_urbs =
+- le16_to_cpu(cmd.u.leaf.softinfo.max_outstanding_tx);
++ kvaser_usb_leaf_get_software_info_leaf(dev, &cmd.u.leaf.softinfo);
+ break;
+ case KVASER_USBCAN:
+ dev->fw_version = le32_to_cpu(cmd.u.usbcan.softinfo.fw_version);
+ dev->max_tx_urbs =
+ le16_to_cpu(cmd.u.usbcan.softinfo.max_outstanding_tx);
++ dev->cfg = &kvaser_usb_leaf_dev_cfg_8mhz;
+ break;
+ }
+
+@@ -1225,24 +1291,11 @@ static int kvaser_usb_leaf_init_card(struct kvaser_usb *dev)
+ {
+ struct kvaser_usb_dev_card_data *card_data = &dev->card_data;
+
+- dev->cfg = &kvaser_usb_leaf_dev_cfg;
+ card_data->ctrlmode_supported |= CAN_CTRLMODE_3_SAMPLES;
+
+ return 0;
+ }
+
+-static const struct can_bittiming_const kvaser_usb_leaf_bittiming_const = {
+- .name = "kvaser_usb",
+- .tseg1_min = KVASER_USB_TSEG1_MIN,
+- .tseg1_max = KVASER_USB_TSEG1_MAX,
+- .tseg2_min = KVASER_USB_TSEG2_MIN,
+- .tseg2_max = KVASER_USB_TSEG2_MAX,
+- .sjw_max = KVASER_USB_SJW_MAX,
+- .brp_min = KVASER_USB_BRP_MIN,
+- .brp_max = KVASER_USB_BRP_MAX,
+- .brp_inc = KVASER_USB_BRP_INC,
+-};
+-
+ static int kvaser_usb_leaf_set_bittiming(struct net_device *netdev)
+ {
+ struct kvaser_usb_net_priv *priv = netdev_priv(netdev);
+@@ -1348,11 +1401,3 @@ const struct kvaser_usb_dev_ops kvaser_usb_leaf_dev_ops = {
+ .dev_read_bulk_callback = kvaser_usb_leaf_read_bulk_callback,
+ .dev_frame_to_cmd = kvaser_usb_leaf_frame_to_cmd,
+ };
+-
+-static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg = {
+- .clock = {
+- .freq = CAN_USB_CLOCK,
+- },
+- .timestamp_freq = 1,
+- .bittiming_const = &kvaser_usb_leaf_bittiming_const,
+-};
+diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c
+index a1a154c08b7f7..023bd34d48e3c 100644
+--- a/drivers/net/can/usb/mcba_usb.c
++++ b/drivers/net/can/usb/mcba_usb.c
+@@ -33,10 +33,6 @@
+ #define MCBA_USB_RX_BUFF_SIZE 64
+ #define MCBA_USB_TX_BUFF_SIZE (sizeof(struct mcba_usb_msg))
+
+-/* MCBA endpoint numbers */
+-#define MCBA_USB_EP_IN 1
+-#define MCBA_USB_EP_OUT 1
+-
+ /* Microchip command id */
+ #define MBCA_CMD_RECEIVE_MESSAGE 0xE3
+ #define MBCA_CMD_I_AM_ALIVE_FROM_CAN 0xF5
+@@ -84,6 +80,8 @@ struct mcba_priv {
+ atomic_t free_ctx_cnt;
+ void *rxbuf[MCBA_MAX_RX_URBS];
+ dma_addr_t rxbuf_dma[MCBA_MAX_RX_URBS];
++ int rx_pipe;
++ int tx_pipe;
+ };
+
+ /* CAN frame */
+@@ -272,10 +270,8 @@ static netdev_tx_t mcba_usb_xmit(struct mcba_priv *priv,
+
+ memcpy(buf, usb_msg, MCBA_USB_TX_BUFF_SIZE);
+
+- usb_fill_bulk_urb(urb, priv->udev,
+- usb_sndbulkpipe(priv->udev, MCBA_USB_EP_OUT), buf,
+- MCBA_USB_TX_BUFF_SIZE, mcba_usb_write_bulk_callback,
+- ctx);
++ usb_fill_bulk_urb(urb, priv->udev, priv->tx_pipe, buf, MCBA_USB_TX_BUFF_SIZE,
++ mcba_usb_write_bulk_callback, ctx);
+
+ urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+ usb_anchor_urb(urb, &priv->tx_submitted);
+@@ -368,7 +364,6 @@ static netdev_tx_t mcba_usb_start_xmit(struct sk_buff *skb,
+ xmit_failed:
+ can_free_echo_skb(priv->netdev, ctx->ndx, NULL);
+ mcba_usb_free_ctx(ctx);
+- dev_kfree_skb(skb);
+ stats->tx_dropped++;
+
+ return NETDEV_TX_OK;
+@@ -611,7 +606,7 @@ static void mcba_usb_read_bulk_callback(struct urb *urb)
+ resubmit_urb:
+
+ usb_fill_bulk_urb(urb, priv->udev,
+- usb_rcvbulkpipe(priv->udev, MCBA_USB_EP_OUT),
++ priv->rx_pipe,
+ urb->transfer_buffer, MCBA_USB_RX_BUFF_SIZE,
+ mcba_usb_read_bulk_callback, priv);
+
+@@ -656,7 +651,7 @@ static int mcba_usb_start(struct mcba_priv *priv)
+ urb->transfer_dma = buf_dma;
+
+ usb_fill_bulk_urb(urb, priv->udev,
+- usb_rcvbulkpipe(priv->udev, MCBA_USB_EP_IN),
++ priv->rx_pipe,
+ buf, MCBA_USB_RX_BUFF_SIZE,
+ mcba_usb_read_bulk_callback, priv);
+ urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+@@ -810,6 +805,13 @@ static int mcba_usb_probe(struct usb_interface *intf,
+ struct mcba_priv *priv;
+ int err;
+ struct usb_device *usbdev = interface_to_usbdev(intf);
++ struct usb_endpoint_descriptor *in, *out;
++
++ err = usb_find_common_endpoints(intf->cur_altsetting, &in, &out, NULL, NULL);
++ if (err) {
++ dev_err(&intf->dev, "Can't find endpoints\n");
++ return err;
++ }
+
+ netdev = alloc_candev(sizeof(struct mcba_priv), MCBA_MAX_TX_URBS);
+ if (!netdev) {
+@@ -855,6 +857,9 @@ static int mcba_usb_probe(struct usb_interface *intf,
+ goto cleanup_free_candev;
+ }
+
++ priv->rx_pipe = usb_rcvbulkpipe(priv->udev, in->bEndpointAddress);
++ priv->tx_pipe = usb_sndbulkpipe(priv->udev, out->bEndpointAddress);
++
+ devm_can_led_init(netdev);
+
+ /* Start USB dev only if we have successfully registered CAN device */
+diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c
+index 837b3fecd71e9..af8d3dadbbb8b 100644
+--- a/drivers/net/can/usb/peak_usb/pcan_usb.c
++++ b/drivers/net/can/usb/peak_usb/pcan_usb.c
+@@ -841,14 +841,14 @@ static int pcan_usb_start(struct peak_usb_device *dev)
+ pdev->bec.rxerr = 0;
+ pdev->bec.txerr = 0;
+
+- /* be notified on error counter changes (if requested by user) */
+- if (dev->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING) {
+- err = pcan_usb_set_err_frame(dev, PCAN_USB_BERR_MASK);
+- if (err)
+- netdev_warn(dev->netdev,
+- "Asking for BERR reporting error %u\n",
+- err);
+- }
++ /* always ask the device for BERR reporting, to be able to switch from
++ * WARNING to PASSIVE state
++ */
++ err = pcan_usb_set_err_frame(dev, PCAN_USB_BERR_MASK);
++ if (err)
++ netdev_warn(dev->netdev,
++ "Asking for BERR reporting error %u\n",
++ err);
+
+ /* if revision greater than 3, can put silent mode on/off */
+ if (dev->device_rev > 3) {
+@@ -986,7 +986,6 @@ const struct peak_usb_adapter pcan_usb = {
+ .device_id = PCAN_USB_PRODUCT_ID,
+ .ctrl_count = 1,
+ .ctrlmode_supported = CAN_CTRLMODE_3_SAMPLES | CAN_CTRLMODE_LISTENONLY |
+- CAN_CTRLMODE_BERR_REPORTING |
+ CAN_CTRLMODE_CC_LEN8_DLC,
+ .clock = {
+ .freq = PCAN_USB_CRYSTAL_HZ / 2,
+diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c
+index d1b83bd1b3cb9..d4c8f934a1ce0 100644
+--- a/drivers/net/can/usb/usb_8dev.c
++++ b/drivers/net/can/usb/usb_8dev.c
+@@ -670,9 +670,20 @@ static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb,
+ atomic_inc(&priv->active_tx_urbs);
+
+ err = usb_submit_urb(urb, GFP_ATOMIC);
+- if (unlikely(err))
+- goto failed;
+- else if (atomic_read(&priv->active_tx_urbs) >= MAX_TX_URBS)
++ if (unlikely(err)) {
++ can_free_echo_skb(netdev, context->echo_index, NULL);
++
++ usb_unanchor_urb(urb);
++ usb_free_coherent(priv->udev, size, buf, urb->transfer_dma);
++
++ atomic_dec(&priv->active_tx_urbs);
++
++ if (err == -ENODEV)
++ netif_device_detach(netdev);
++ else
++ netdev_warn(netdev, "failed tx_urb %d\n", err);
++ stats->tx_dropped++;
++ } else if (atomic_read(&priv->active_tx_urbs) >= MAX_TX_URBS)
+ /* Slow down tx path */
+ netif_stop_queue(netdev);
+
+@@ -691,19 +702,6 @@ nofreecontext:
+
+ return NETDEV_TX_BUSY;
+
+-failed:
+- can_free_echo_skb(netdev, context->echo_index, NULL);
+-
+- usb_unanchor_urb(urb);
+- usb_free_coherent(priv->udev, size, buf, urb->transfer_dma);
+-
+- atomic_dec(&priv->active_tx_urbs);
+-
+- if (err == -ENODEV)
+- netif_device_detach(netdev);
+- else
+- netdev_warn(netdev, "failed tx_urb %d\n", err);
+-
+ nomembuf:
+ usb_free_urb(urb);
+
+diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c
+index 8861a7d875e7e..be5566168d0f3 100644
+--- a/drivers/net/can/vxcan.c
++++ b/drivers/net/can/vxcan.c
+@@ -148,7 +148,7 @@ static void vxcan_setup(struct net_device *dev)
+ dev->hard_header_len = 0;
+ dev->addr_len = 0;
+ dev->tx_queue_len = 0;
+- dev->flags = (IFF_NOARP|IFF_ECHO);
++ dev->flags = IFF_NOARP;
+ dev->netdev_ops = &vxcan_netdev_ops;
+ dev->needs_free_netdev = true;
+
+diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c
+index 3b883e607d8ba..a579b9b791ede 100644
+--- a/drivers/net/can/xilinx_can.c
++++ b/drivers/net/can/xilinx_can.c
+@@ -1762,7 +1762,12 @@ static int xcan_probe(struct platform_device *pdev)
+ spin_lock_init(&priv->tx_lock);
+
+ /* Get IRQ for the device */
+- ndev->irq = platform_get_irq(pdev, 0);
++ ret = platform_get_irq(pdev, 0);
++ if (ret < 0)
++ goto err_free;
++
++ ndev->irq = ret;
++
+ ndev->flags |= IFF_ECHO; /* We support local echo */
+
+ platform_set_drvdata(pdev, ndev);
+diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
+index a5f1aa911fe2a..9428aac4a6868 100644
+--- a/drivers/net/dsa/Kconfig
++++ b/drivers/net/dsa/Kconfig
+@@ -36,6 +36,7 @@ config NET_DSA_LANTIQ_GSWIP
+ config NET_DSA_MT7530
+ tristate "MediaTek MT753x and MT7621 Ethernet switch support"
+ select NET_DSA_TAG_MTK
++ select MEDIATEK_GE_PHY
+ help
+ This enables support for the MediaTek MT7530, MT7531, and MT7621
+ Ethernet switch chips.
+@@ -80,6 +81,7 @@ config NET_DSA_REALTEK_SMI
+
+ config NET_DSA_SMSC_LAN9303
+ tristate
++ depends on VLAN_8021Q || VLAN_8021Q=n
+ select NET_DSA_TAG_LAN9303
+ select REGMAP
+ help
+diff --git a/drivers/net/dsa/b53/b53_spi.c b/drivers/net/dsa/b53/b53_spi.c
+index 01e37b75471e1..2b88f03e52521 100644
+--- a/drivers/net/dsa/b53/b53_spi.c
++++ b/drivers/net/dsa/b53/b53_spi.c
+@@ -349,6 +349,19 @@ static const struct of_device_id b53_spi_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, b53_spi_of_match);
+
++static const struct spi_device_id b53_spi_ids[] = {
++ { .name = "bcm5325" },
++ { .name = "bcm5365" },
++ { .name = "bcm5395" },
++ { .name = "bcm5397" },
++ { .name = "bcm5398" },
++ { .name = "bcm53115" },
++ { .name = "bcm53125" },
++ { .name = "bcm53128" },
++ { /* sentinel */ }
++};
++MODULE_DEVICE_TABLE(spi, b53_spi_ids);
++
+ static struct spi_driver b53_spi_driver = {
+ .driver = {
+ .name = "b53-switch",
+@@ -357,6 +370,7 @@ static struct spi_driver b53_spi_driver = {
+ .probe = b53_spi_probe,
+ .remove = b53_spi_remove,
+ .shutdown = b53_spi_shutdown,
++ .id_table = b53_spi_ids,
+ };
+
+ module_spi_driver(b53_spi_driver);
+diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
+index 7578a5c38df59..b3a43a3d90e49 100644
+--- a/drivers/net/dsa/bcm_sf2.c
++++ b/drivers/net/dsa/bcm_sf2.c
+@@ -584,7 +584,7 @@ static int bcm_sf2_mdio_register(struct dsa_switch *ds)
+ get_device(&priv->master_mii_bus->dev);
+ priv->master_mii_dn = dn;
+
+- priv->slave_mii_bus = devm_mdiobus_alloc(ds->dev);
++ priv->slave_mii_bus = mdiobus_alloc();
+ if (!priv->slave_mii_bus) {
+ of_node_put(dn);
+ return -ENOMEM;
+@@ -644,8 +644,10 @@ static int bcm_sf2_mdio_register(struct dsa_switch *ds)
+ }
+
+ err = mdiobus_register(priv->slave_mii_bus);
+- if (err && dn)
++ if (err && dn) {
++ mdiobus_free(priv->slave_mii_bus);
+ of_node_put(dn);
++ }
+
+ return err;
+ }
+@@ -653,6 +655,7 @@ static int bcm_sf2_mdio_register(struct dsa_switch *ds)
+ static void bcm_sf2_mdio_unregister(struct bcm_sf2_priv *priv)
+ {
+ mdiobus_unregister(priv->slave_mii_bus);
++ mdiobus_free(priv->slave_mii_bus);
+ of_node_put(priv->master_mii_dn);
+ }
+
+@@ -793,6 +796,9 @@ static void bcm_sf2_sw_mac_link_down(struct dsa_switch *ds, int port,
+ struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
+ u32 reg, offset;
+
++ if (priv->wol_ports_mask & BIT(port))
++ return;
++
+ if (port != core_readl(priv, CORE_IMP0_PRT_ID)) {
+ if (priv->type == BCM4908_DEVICE_ID ||
+ priv->type == BCM7445_DEVICE_ID)
+diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c
+index a7e2fcf2df2c9..edbe5e7f1cb6b 100644
+--- a/drivers/net/dsa/bcm_sf2_cfp.c
++++ b/drivers/net/dsa/bcm_sf2_cfp.c
+@@ -567,14 +567,14 @@ static void bcm_sf2_cfp_slice_ipv6(struct bcm_sf2_priv *priv,
+ static struct cfp_rule *bcm_sf2_cfp_rule_find(struct bcm_sf2_priv *priv,
+ int port, u32 location)
+ {
+- struct cfp_rule *rule = NULL;
++ struct cfp_rule *rule;
+
+ list_for_each_entry(rule, &priv->cfp.rules_list, next) {
+ if (rule->port == port && rule->fs.location == location)
+- break;
++ return rule;
+ }
+
+- return rule;
++ return NULL;
+ }
+
+ static int bcm_sf2_cfp_rule_cmp(struct bcm_sf2_priv *priv, int port,
+diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c
+index 354655f9ed003..950a54ec4b59b 100644
+--- a/drivers/net/dsa/hirschmann/hellcreek.c
++++ b/drivers/net/dsa/hirschmann/hellcreek.c
+@@ -710,8 +710,9 @@ static int __hellcreek_fdb_add(struct hellcreek *hellcreek,
+ u16 meta = 0;
+
+ dev_dbg(hellcreek->dev, "Add static FDB entry: MAC=%pM, MASK=0x%02x, "
+- "OBT=%d, REPRIO_EN=%d, PRIO=%d\n", entry->mac, entry->portmask,
+- entry->is_obt, entry->reprio_en, entry->reprio_tc);
++ "OBT=%d, PASS_BLOCKED=%d, REPRIO_EN=%d, PRIO=%d\n", entry->mac,
++ entry->portmask, entry->is_obt, entry->pass_blocked,
++ entry->reprio_en, entry->reprio_tc);
+
+ /* Add mac address */
+ hellcreek_write(hellcreek, entry->mac[1] | (entry->mac[0] << 8), HR_FDBWDH);
+@@ -722,6 +723,8 @@ static int __hellcreek_fdb_add(struct hellcreek *hellcreek,
+ meta |= entry->portmask << HR_FDBWRM0_PORTMASK_SHIFT;
+ if (entry->is_obt)
+ meta |= HR_FDBWRM0_OBT;
++ if (entry->pass_blocked)
++ meta |= HR_FDBWRM0_PASS_BLOCKED;
+ if (entry->reprio_en) {
+ meta |= HR_FDBWRM0_REPRIO_EN;
+ meta |= entry->reprio_tc << HR_FDBWRM0_REPRIO_TC_SHIFT;
+@@ -1049,7 +1052,7 @@ static void hellcreek_setup_tc_identity_mapping(struct hellcreek *hellcreek)
+
+ static int hellcreek_setup_fdb(struct hellcreek *hellcreek)
+ {
+- static struct hellcreek_fdb_entry ptp = {
++ static struct hellcreek_fdb_entry l2_ptp = {
+ /* MAC: 01-1B-19-00-00-00 */
+ .mac = { 0x01, 0x1b, 0x19, 0x00, 0x00, 0x00 },
+ .portmask = 0x03, /* Management ports */
+@@ -1060,24 +1063,94 @@ static int hellcreek_setup_fdb(struct hellcreek *hellcreek)
+ .reprio_tc = 6, /* TC: 6 as per IEEE 802.1AS */
+ .reprio_en = 1,
+ };
+- static struct hellcreek_fdb_entry p2p = {
++ static struct hellcreek_fdb_entry udp4_ptp = {
++ /* MAC: 01-00-5E-00-01-81 */
++ .mac = { 0x01, 0x00, 0x5e, 0x00, 0x01, 0x81 },
++ .portmask = 0x03, /* Management ports */
++ .age = 0,
++ .is_obt = 0,
++ .pass_blocked = 0,
++ .is_static = 1,
++ .reprio_tc = 6,
++ .reprio_en = 1,
++ };
++ static struct hellcreek_fdb_entry udp6_ptp = {
++ /* MAC: 33-33-00-00-01-81 */
++ .mac = { 0x33, 0x33, 0x00, 0x00, 0x01, 0x81 },
++ .portmask = 0x03, /* Management ports */
++ .age = 0,
++ .is_obt = 0,
++ .pass_blocked = 0,
++ .is_static = 1,
++ .reprio_tc = 6,
++ .reprio_en = 1,
++ };
++ static struct hellcreek_fdb_entry l2_p2p = {
+ /* MAC: 01-80-C2-00-00-0E */
+ .mac = { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x0e },
+ .portmask = 0x03, /* Management ports */
+ .age = 0,
+ .is_obt = 0,
+- .pass_blocked = 0,
++ .pass_blocked = 1,
+ .is_static = 1,
+ .reprio_tc = 6, /* TC: 6 as per IEEE 802.1AS */
+ .reprio_en = 1,
+ };
++ static struct hellcreek_fdb_entry udp4_p2p = {
++ /* MAC: 01-00-5E-00-00-6B */
++ .mac = { 0x01, 0x00, 0x5e, 0x00, 0x00, 0x6b },
++ .portmask = 0x03, /* Management ports */
++ .age = 0,
++ .is_obt = 0,
++ .pass_blocked = 1,
++ .is_static = 1,
++ .reprio_tc = 6,
++ .reprio_en = 1,
++ };
++ static struct hellcreek_fdb_entry udp6_p2p = {
++ /* MAC: 33-33-00-00-00-6B */
++ .mac = { 0x33, 0x33, 0x00, 0x00, 0x00, 0x6b },
++ .portmask = 0x03, /* Management ports */
++ .age = 0,
++ .is_obt = 0,
++ .pass_blocked = 1,
++ .is_static = 1,
++ .reprio_tc = 6,
++ .reprio_en = 1,
++ };
++ static struct hellcreek_fdb_entry stp = {
++ /* MAC: 01-80-C2-00-00-00 */
++ .mac = { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x00 },
++ .portmask = 0x03, /* Management ports */
++ .age = 0,
++ .is_obt = 0,
++ .pass_blocked = 1,
++ .is_static = 1,
++ .reprio_tc = 6,
++ .reprio_en = 1,
++ };
+ int ret;
+
+ mutex_lock(&hellcreek->reg_lock);
+- ret = __hellcreek_fdb_add(hellcreek, &ptp);
++ ret = __hellcreek_fdb_add(hellcreek, &l2_ptp);
++ if (ret)
++ goto out;
++ ret = __hellcreek_fdb_add(hellcreek, &udp4_ptp);
++ if (ret)
++ goto out;
++ ret = __hellcreek_fdb_add(hellcreek, &udp6_ptp);
++ if (ret)
++ goto out;
++ ret = __hellcreek_fdb_add(hellcreek, &l2_p2p);
++ if (ret)
++ goto out;
++ ret = __hellcreek_fdb_add(hellcreek, &udp4_p2p);
++ if (ret)
++ goto out;
++ ret = __hellcreek_fdb_add(hellcreek, &udp6_p2p);
+ if (ret)
+ goto out;
+- ret = __hellcreek_fdb_add(hellcreek, &p2p);
++ ret = __hellcreek_fdb_add(hellcreek, &stp);
+ out:
+ mutex_unlock(&hellcreek->reg_lock);
+
+diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c
+index 89f920289ae21..0b6f29ee87b56 100644
+--- a/drivers/net/dsa/lan9303-core.c
++++ b/drivers/net/dsa/lan9303-core.c
+@@ -10,6 +10,7 @@
+ #include <linux/mii.h>
+ #include <linux/phy.h>
+ #include <linux/if_bridge.h>
++#include <linux/if_vlan.h>
+ #include <linux/etherdevice.h>
+
+ #include "lan9303.h"
+@@ -1083,21 +1084,27 @@ static void lan9303_adjust_link(struct dsa_switch *ds, int port,
+ static int lan9303_port_enable(struct dsa_switch *ds, int port,
+ struct phy_device *phy)
+ {
++ struct dsa_port *dp = dsa_to_port(ds, port);
+ struct lan9303 *chip = ds->priv;
+
+- if (!dsa_is_user_port(ds, port))
++ if (!dsa_port_is_user(dp))
+ return 0;
+
++ vlan_vid_add(dp->cpu_dp->master, htons(ETH_P_8021Q), port);
++
+ return lan9303_enable_processing_port(chip, port);
+ }
+
+ static void lan9303_port_disable(struct dsa_switch *ds, int port)
+ {
++ struct dsa_port *dp = dsa_to_port(ds, port);
+ struct lan9303 *chip = ds->priv;
+
+- if (!dsa_is_user_port(ds, port))
++ if (!dsa_port_is_user(dp))
+ return;
+
++ vlan_vid_del(dp->cpu_dp->master, htons(ETH_P_8021Q), port);
++
+ lan9303_disable_processing_port(chip, port);
+ lan9303_phy_write(ds, chip->phy_addr_base + port, MII_BMCR, BMCR_PDOWN);
+ }
+@@ -1309,7 +1316,7 @@ static int lan9303_probe_reset_gpio(struct lan9303 *chip,
+ struct device_node *np)
+ {
+ chip->reset_gpio = devm_gpiod_get_optional(chip->dev, "reset",
+- GPIOD_OUT_LOW);
++ GPIOD_OUT_HIGH);
+ if (IS_ERR(chip->reset_gpio))
+ return PTR_ERR(chip->reset_gpio);
+
+diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c
+index dbd4486a173ff..8a8f392813d82 100644
+--- a/drivers/net/dsa/lantiq_gswip.c
++++ b/drivers/net/dsa/lantiq_gswip.c
+@@ -497,8 +497,9 @@ static int gswip_mdio_rd(struct mii_bus *bus, int addr, int reg)
+ static int gswip_mdio(struct gswip_priv *priv, struct device_node *mdio_np)
+ {
+ struct dsa_switch *ds = priv->ds;
++ int err;
+
+- ds->slave_mii_bus = devm_mdiobus_alloc(priv->dev);
++ ds->slave_mii_bus = mdiobus_alloc();
+ if (!ds->slave_mii_bus)
+ return -ENOMEM;
+
+@@ -511,7 +512,11 @@ static int gswip_mdio(struct gswip_priv *priv, struct device_node *mdio_np)
+ ds->slave_mii_bus->parent = priv->dev;
+ ds->slave_mii_bus->phy_mask = ~ds->phys_mii_mask;
+
+- return of_mdiobus_register(ds->slave_mii_bus, mdio_np);
++ err = of_mdiobus_register(ds->slave_mii_bus, mdio_np);
++ if (err)
++ mdiobus_free(ds->slave_mii_bus);
++
++ return err;
+ }
+
+ static int gswip_pce_table_entry_read(struct gswip_priv *priv,
+@@ -1658,9 +1663,6 @@ static void gswip_phylink_mac_config(struct dsa_switch *ds, int port,
+ break;
+ case PHY_INTERFACE_MODE_RMII:
+ miicfg |= GSWIP_MII_CFG_MODE_RMIIM;
+-
+- /* Configure the RMII clock as output: */
+- miicfg |= GSWIP_MII_CFG_RMII_CLK;
+ break;
+ case PHY_INTERFACE_MODE_RGMII:
+ case PHY_INTERFACE_MODE_RGMII_ID:
+@@ -2170,8 +2172,10 @@ disable_switch:
+ gswip_mdio_mask(priv, GSWIP_MDIO_GLOB_ENABLE, 0, GSWIP_MDIO_GLOB);
+ dsa_unregister_switch(priv->ds);
+ mdio_bus:
+- if (mdio_np)
++ if (mdio_np) {
+ mdiobus_unregister(priv->ds->slave_mii_bus);
++ mdiobus_free(priv->ds->slave_mii_bus);
++ }
+ put_mdio_node:
+ of_node_put(mdio_np);
+ for (i = 0; i < priv->num_gphy_fw; i++)
+@@ -2195,6 +2199,7 @@ static int gswip_remove(struct platform_device *pdev)
+ if (priv->ds->slave_mii_bus) {
+ mdiobus_unregister(priv->ds->slave_mii_bus);
+ of_node_put(priv->ds->slave_mii_bus->dev.of_node);
++ mdiobus_free(priv->ds->slave_mii_bus);
+ }
+
+ for (i = 0; i < priv->num_gphy_fw; i++)
+diff --git a/drivers/net/dsa/microchip/ksz8795_spi.c b/drivers/net/dsa/microchip/ksz8795_spi.c
+index 866767b70d65b..b0a7dee27ffc9 100644
+--- a/drivers/net/dsa/microchip/ksz8795_spi.c
++++ b/drivers/net/dsa/microchip/ksz8795_spi.c
+@@ -124,12 +124,23 @@ static const struct of_device_id ksz8795_dt_ids[] = {
+ };
+ MODULE_DEVICE_TABLE(of, ksz8795_dt_ids);
+
++static const struct spi_device_id ksz8795_spi_ids[] = {
++ { "ksz8765" },
++ { "ksz8794" },
++ { "ksz8795" },
++ { "ksz8863" },
++ { "ksz8873" },
++ { },
++};
++MODULE_DEVICE_TABLE(spi, ksz8795_spi_ids);
++
+ static struct spi_driver ksz8795_spi_driver = {
+ .driver = {
+ .name = "ksz8795-switch",
+ .owner = THIS_MODULE,
+ .of_match_table = of_match_ptr(ksz8795_dt_ids),
+ },
++ .id_table = ksz8795_spi_ids,
+ .probe = ksz8795_spi_probe,
+ .remove = ksz8795_spi_remove,
+ .shutdown = ksz8795_spi_shutdown,
+diff --git a/drivers/net/dsa/microchip/ksz9477_spi.c b/drivers/net/dsa/microchip/ksz9477_spi.c
+index e3cb0e6c9f6f2..43addeabfc259 100644
+--- a/drivers/net/dsa/microchip/ksz9477_spi.c
++++ b/drivers/net/dsa/microchip/ksz9477_spi.c
+@@ -98,12 +98,24 @@ static const struct of_device_id ksz9477_dt_ids[] = {
+ };
+ MODULE_DEVICE_TABLE(of, ksz9477_dt_ids);
+
++static const struct spi_device_id ksz9477_spi_ids[] = {
++ { "ksz9477" },
++ { "ksz9897" },
++ { "ksz9893" },
++ { "ksz9563" },
++ { "ksz8563" },
++ { "ksz9567" },
++ { },
++};
++MODULE_DEVICE_TABLE(spi, ksz9477_spi_ids);
++
+ static struct spi_driver ksz9477_spi_driver = {
+ .driver = {
+ .name = "ksz9477-switch",
+ .owner = THIS_MODULE,
+ .of_match_table = of_match_ptr(ksz9477_dt_ids),
+ },
++ .id_table = ksz9477_spi_ids,
+ .probe = ksz9477_spi_probe,
+ .remove = ksz9477_spi_remove,
+ .shutdown = ksz9477_spi_shutdown,
+diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
+index 9890672a206d0..ca9e171cb2bb0 100644
+--- a/drivers/net/dsa/mt7530.c
++++ b/drivers/net/dsa/mt7530.c
+@@ -2066,7 +2066,7 @@ mt7530_setup_mdio(struct mt7530_priv *priv)
+ if (priv->irq)
+ mt7530_setup_mdio_irq(priv);
+
+- ret = mdiobus_register(bus);
++ ret = devm_mdiobus_register(dev, bus);
+ if (ret) {
+ dev_err(dev, "failed to register MDIO bus: %d\n", ret);
+ if (priv->irq)
+@@ -2216,6 +2216,7 @@ mt7530_setup(struct dsa_switch *ds)
+ ret = of_get_phy_mode(mac_np, &interface);
+ if (ret && ret != -ENODEV) {
+ of_node_put(mac_np);
++ of_node_put(phy_node);
+ return ret;
+ }
+ id = of_mdio_parse_addr(ds->dev, phy_node);
+@@ -2928,7 +2929,7 @@ mt753x_phylink_validate(struct dsa_switch *ds, int port,
+
+ phylink_set_port_modes(mask);
+
+- if (state->interface != PHY_INTERFACE_MODE_TRGMII ||
++ if (state->interface != PHY_INTERFACE_MODE_TRGMII &&
+ !phy_interface_mode_is_8023z(state->interface)) {
+ phylink_set(mask, 10baseT_Half);
+ phylink_set(mask, 10baseT_Full);
+diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
+index 8dadcae93c9b5..0830d7bb7a001 100644
+--- a/drivers/net/dsa/mv88e6xxx/chip.c
++++ b/drivers/net/dsa/mv88e6xxx/chip.c
+@@ -471,6 +471,12 @@ static int mv88e6xxx_port_ppu_updates(struct mv88e6xxx_chip *chip, int port)
+ u16 reg;
+ int err;
+
++ /* The 88e6250 family does not have the PHY detect bit. Instead,
++ * report whether the port is internal.
++ */
++ if (chip->info->family == MV88E6XXX_FAMILY_6250)
++ return port < chip->info->num_internal_phys;
++
+ err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &reg);
+ if (err) {
+ dev_err(chip->dev,
+@@ -640,7 +646,10 @@ static void mv88e6393x_phylink_validate(struct mv88e6xxx_chip *chip, int port,
+ unsigned long *mask,
+ struct phylink_link_state *state)
+ {
+- if (port == 0 || port == 9 || port == 10) {
++ bool is_6191x =
++ chip->info->prod_num == MV88E6XXX_PORT_SWITCH_ID_PROD_6191X;
++
++ if (((port == 0 || port == 9) && !is_6191x) || port == 10) {
+ phylink_set(mask, 10000baseT_Full);
+ phylink_set(mask, 10000baseKR_Full);
+ phylink_set(mask, 10000baseCR_Full);
+@@ -690,44 +699,48 @@ static void mv88e6xxx_mac_config(struct dsa_switch *ds, int port,
+ {
+ struct mv88e6xxx_chip *chip = ds->priv;
+ struct mv88e6xxx_port *p;
+- int err;
++ int err = 0;
+
+ p = &chip->ports[port];
+
+- /* FIXME: is this the correct test? If we're in fixed mode on an
+- * internal port, why should we process this any different from
+- * PHY mode? On the other hand, the port may be automedia between
+- * an internal PHY and the serdes...
+- */
+- if ((mode == MLO_AN_PHY) && mv88e6xxx_phy_is_internal(ds, port))
+- return;
+-
+ mv88e6xxx_reg_lock(chip);
+- /* In inband mode, the link may come up at any time while the link
+- * is not forced down. Force the link down while we reconfigure the
+- * interface mode.
+- */
+- if (mode == MLO_AN_INBAND && p->interface != state->interface &&
+- chip->info->ops->port_set_link)
+- chip->info->ops->port_set_link(chip, port, LINK_FORCED_DOWN);
+-
+- err = mv88e6xxx_port_config_interface(chip, port, state->interface);
+- if (err && err != -EOPNOTSUPP)
+- goto err_unlock;
+
+- err = mv88e6xxx_serdes_pcs_config(chip, port, mode, state->interface,
+- state->advertising);
+- /* FIXME: we should restart negotiation if something changed - which
+- * is something we get if we convert to using phylinks PCS operations.
+- */
+- if (err > 0)
+- err = 0;
++ if (mode != MLO_AN_PHY || !mv88e6xxx_phy_is_internal(ds, port)) {
++ /* In inband mode, the link may come up at any time while the
++ * link is not forced down. Force the link down while we
++ * reconfigure the interface mode.
++ */
++ if (mode == MLO_AN_INBAND &&
++ p->interface != state->interface &&
++ chip->info->ops->port_set_link)
++ chip->info->ops->port_set_link(chip, port,
++ LINK_FORCED_DOWN);
++
++ err = mv88e6xxx_port_config_interface(chip, port,
++ state->interface);
++ if (err && err != -EOPNOTSUPP)
++ goto err_unlock;
++
++ err = mv88e6xxx_serdes_pcs_config(chip, port, mode,
++ state->interface,
++ state->advertising);
++ /* FIXME: we should restart negotiation if something changed -
++ * which is something we get if we convert to using phylinks
++ * PCS operations.
++ */
++ if (err > 0)
++ err = 0;
++ }
+
+ /* Undo the forced down state above after completing configuration
+- * irrespective of its state on entry, which allows the link to come up.
++ * irrespective of its state on entry, which allows the link to come
++ * up in the in-band case where there is no separate SERDES. Also
++ * ensure that the link can come up if the PPU is in use and we are
++ * in PHY mode (we treat the PPU as an effective in-band mechanism.)
+ */
+- if (mode == MLO_AN_INBAND && p->interface != state->interface &&
+- chip->info->ops->port_set_link)
++ if (chip->info->ops->port_set_link &&
++ ((mode == MLO_AN_INBAND && p->interface != state->interface) ||
++ (mode == MLO_AN_PHY && mv88e6xxx_port_ppu_updates(chip, port))))
+ chip->info->ops->port_set_link(chip, port, LINK_UNFORCED);
+
+ p->interface = state->interface;
+@@ -750,13 +763,16 @@ static void mv88e6xxx_mac_link_down(struct dsa_switch *ds, int port,
+ ops = chip->info->ops;
+
+ mv88e6xxx_reg_lock(chip);
+- /* Internal PHYs propagate their configuration directly to the MAC.
+- * External PHYs depend on whether the PPU is enabled for this port.
++ /* Force the link down if we know the port may not be automatically
++ * updated by the switch or if we are using fixed-link mode.
+ */
+- if (((!mv88e6xxx_phy_is_internal(ds, port) &&
+- !mv88e6xxx_port_ppu_updates(chip, port)) ||
++ if ((!mv88e6xxx_port_ppu_updates(chip, port) ||
+ mode == MLO_AN_FIXED) && ops->port_sync_link)
+ err = ops->port_sync_link(chip, port, mode, false);
++
++ if (!err && ops->port_set_speed_duplex)
++ err = ops->port_set_speed_duplex(chip, port, SPEED_UNFORCED,
++ DUPLEX_UNFORCED);
+ mv88e6xxx_reg_unlock(chip);
+
+ if (err)
+@@ -777,11 +793,11 @@ static void mv88e6xxx_mac_link_up(struct dsa_switch *ds, int port,
+ ops = chip->info->ops;
+
+ mv88e6xxx_reg_lock(chip);
+- /* Internal PHYs propagate their configuration directly to the MAC.
+- * External PHYs depend on whether the PPU is enabled for this port.
++ /* Configure and force the link up if we know that the port may not
++ * automatically updated by the switch or if we are using fixed-link
++ * mode.
+ */
+- if ((!mv88e6xxx_phy_is_internal(ds, port) &&
+- !mv88e6xxx_port_ppu_updates(chip, port)) ||
++ if (!mv88e6xxx_port_ppu_updates(chip, port) ||
+ mode == MLO_AN_FIXED) {
+ /* FIXME: for an automedia port, should we force the link
+ * down here - what if the link comes up due to "other" media
+@@ -3400,7 +3416,7 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
+ return err;
+ }
+
+- bus = devm_mdiobus_alloc_size(chip->dev, sizeof(*mdio_bus));
++ bus = mdiobus_alloc_size(sizeof(*mdio_bus));
+ if (!bus)
+ return -ENOMEM;
+
+@@ -3425,14 +3441,14 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
+ if (!external) {
+ err = mv88e6xxx_g2_irq_mdio_setup(chip, bus);
+ if (err)
+- return err;
++ goto out;
+ }
+
+ err = of_mdiobus_register(bus, np);
+ if (err) {
+ dev_err(chip->dev, "Cannot register MDIO bus (%d)\n", err);
+ mv88e6xxx_g2_irq_mdio_free(chip, bus);
+- return err;
++ goto out;
+ }
+
+ if (external)
+@@ -3441,21 +3457,26 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
+ list_add(&mdio_bus->list, &chip->mdios);
+
+ return 0;
++
++out:
++ mdiobus_free(bus);
++ return err;
+ }
+
+ static void mv88e6xxx_mdios_unregister(struct mv88e6xxx_chip *chip)
+
+ {
+- struct mv88e6xxx_mdio_bus *mdio_bus;
++ struct mv88e6xxx_mdio_bus *mdio_bus, *p;
+ struct mii_bus *bus;
+
+- list_for_each_entry(mdio_bus, &chip->mdios, list) {
++ list_for_each_entry_safe(mdio_bus, p, &chip->mdios, list) {
+ bus = mdio_bus->bus;
+
+ if (!mdio_bus->external)
+ mv88e6xxx_g2_irq_mdio_free(chip, bus);
+
+ mdiobus_unregister(bus);
++ mdiobus_free(bus);
+ }
+ }
+
+@@ -3628,6 +3649,7 @@ static const struct mv88e6xxx_ops mv88e6097_ops = {
+ .port_sync_link = mv88e6185_port_sync_link,
+ .port_set_speed_duplex = mv88e6185_port_set_speed_duplex,
+ .port_tag_remap = mv88e6095_port_tag_remap,
++ .port_set_policy = mv88e6352_port_set_policy,
+ .port_set_frame_mode = mv88e6351_port_set_frame_mode,
+ .port_set_ucast_flood = mv88e6352_port_set_ucast_flood,
+ .port_set_mcast_flood = mv88e6352_port_set_mcast_flood,
+diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c
+index d9817b20ea641..ab41619a809b3 100644
+--- a/drivers/net/dsa/mv88e6xxx/port.c
++++ b/drivers/net/dsa/mv88e6xxx/port.c
+@@ -283,7 +283,7 @@ static int mv88e6xxx_port_set_speed_duplex(struct mv88e6xxx_chip *chip,
+ if (err)
+ return err;
+
+- if (speed)
++ if (speed != SPEED_UNFORCED)
+ dev_dbg(chip->dev, "p%d: Speed set to %d Mbps\n", port, speed);
+ else
+ dev_dbg(chip->dev, "p%d: Speed unforced\n", port);
+@@ -516,7 +516,7 @@ int mv88e6393x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
+ if (err)
+ return err;
+
+- if (speed)
++ if (speed != SPEED_UNFORCED)
+ dev_dbg(chip->dev, "p%d: Speed set to %d Mbps\n", port, speed);
+ else
+ dev_dbg(chip->dev, "p%d: Speed unforced\n", port);
+diff --git a/drivers/net/dsa/mv88e6xxx/port_hidden.c b/drivers/net/dsa/mv88e6xxx/port_hidden.c
+index b49d05f0e1179..7a9f9ff6dedf3 100644
+--- a/drivers/net/dsa/mv88e6xxx/port_hidden.c
++++ b/drivers/net/dsa/mv88e6xxx/port_hidden.c
+@@ -40,8 +40,9 @@ int mv88e6xxx_port_hidden_wait(struct mv88e6xxx_chip *chip)
+ {
+ int bit = __bf_shf(MV88E6XXX_PORT_RESERVED_1A_BUSY);
+
+- return mv88e6xxx_wait_bit(chip, MV88E6XXX_PORT_RESERVED_1A_CTRL_PORT,
+- MV88E6XXX_PORT_RESERVED_1A, bit, 0);
++ return mv88e6xxx_port_wait_bit(chip,
++ MV88E6XXX_PORT_RESERVED_1A_CTRL_PORT,
++ MV88E6XXX_PORT_RESERVED_1A, bit, 0);
+ }
+
+ int mv88e6xxx_port_hidden_read(struct mv88e6xxx_chip *chip, int block, int port,
+diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c
+index 6ea0036787986..2b05ead515cdc 100644
+--- a/drivers/net/dsa/mv88e6xxx/serdes.c
++++ b/drivers/net/dsa/mv88e6xxx/serdes.c
+@@ -50,11 +50,22 @@ static int mv88e6390_serdes_write(struct mv88e6xxx_chip *chip,
+ }
+
+ static int mv88e6xxx_serdes_pcs_get_state(struct mv88e6xxx_chip *chip,
+- u16 status, u16 lpa,
++ u16 ctrl, u16 status, u16 lpa,
+ struct phylink_link_state *state)
+ {
++ state->link = !!(status & MV88E6390_SGMII_PHY_STATUS_LINK);
++
+ if (status & MV88E6390_SGMII_PHY_STATUS_SPD_DPL_VALID) {
+- state->link = !!(status & MV88E6390_SGMII_PHY_STATUS_LINK);
++ /* The Spped and Duplex Resolved register is 1 if AN is enabled
++ * and complete, or if AN is disabled. So with disabled AN we
++ * still get here on link up. But we want to set an_complete
++ * only if AN was enabled, thus we look at BMCR_ANENABLE.
++ * (According to 802.3-2008 section 22.2.4.2.10, we should be
++ * able to get this same value from BMSR_ANEGCAPABLE, but tests
++ * show that these Marvell PHYs don't conform to this part of
++ * the specificaion - BMSR_ANEGCAPABLE is simply always 1.)
++ */
++ state->an_complete = !!(ctrl & BMCR_ANENABLE);
+ state->duplex = status &
+ MV88E6390_SGMII_PHY_STATUS_DUPLEX_FULL ?
+ DUPLEX_FULL : DUPLEX_HALF;
+@@ -81,6 +92,18 @@ static int mv88e6xxx_serdes_pcs_get_state(struct mv88e6xxx_chip *chip,
+ dev_err(chip->dev, "invalid PHY speed\n");
+ return -EINVAL;
+ }
++ } else if (state->link &&
++ state->interface != PHY_INTERFACE_MODE_SGMII) {
++ /* If Speed and Duplex Resolved register is 0 and link is up, it
++ * means that AN was enabled, but link partner had it disabled
++ * and the PHY invoked the Auto-Negotiation Bypass feature and
++ * linked anyway.
++ */
++ state->duplex = DUPLEX_FULL;
++ if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
++ state->speed = SPEED_2500;
++ else
++ state->speed = SPEED_1000;
+ } else {
+ state->link = false;
+ }
+@@ -168,9 +191,15 @@ int mv88e6352_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port,
+ int mv88e6352_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port,
+ int lane, struct phylink_link_state *state)
+ {
+- u16 lpa, status;
++ u16 lpa, status, ctrl;
+ int err;
+
++ err = mv88e6352_serdes_read(chip, MII_BMCR, &ctrl);
++ if (err) {
++ dev_err(chip->dev, "can't read Serdes PHY control: %d\n", err);
++ return err;
++ }
++
+ err = mv88e6352_serdes_read(chip, 0x11, &status);
+ if (err) {
+ dev_err(chip->dev, "can't read Serdes PHY status: %d\n", err);
+@@ -183,7 +212,7 @@ int mv88e6352_serdes_pcs_get_state(struct mv88e6xxx_chip *chip, int port,
+ return err;
+ }
+
+- return mv88e6xxx_serdes_pcs_get_state(chip, status, lpa, state);
++ return mv88e6xxx_serdes_pcs_get_state(chip, ctrl, status, lpa, state);
+ }
+
+ int mv88e6352_serdes_pcs_an_restart(struct mv88e6xxx_chip *chip, int port,
+@@ -801,7 +830,7 @@ int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane,
+ bool up)
+ {
+ u8 cmode = chip->ports[port].cmode;
+- int err = 0;
++ int err;
+
+ switch (cmode) {
+ case MV88E6XXX_PORT_STS_CMODE_SGMII:
+@@ -813,6 +842,9 @@ int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane,
+ case MV88E6XXX_PORT_STS_CMODE_RXAUI:
+ err = mv88e6390_serdes_power_10g(chip, lane, up);
+ break;
++ default:
++ err = -EINVAL;
++ break;
+ }
+
+ if (!err && up)
+@@ -883,9 +915,16 @@ int mv88e6390_serdes_pcs_config(struct mv88e6xxx_chip *chip, int port,
+ static int mv88e6390_serdes_pcs_get_state_sgmii(struct mv88e6xxx_chip *chip,
+ int port, int lane, struct phylink_link_state *state)
+ {
+- u16 lpa, status;
++ u16 lpa, status, ctrl;
+ int err;
+
++ err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6390_SGMII_BMCR, &ctrl);
++ if (err) {
++ dev_err(chip->dev, "can't read Serdes PHY control: %d\n", err);
++ return err;
++ }
++
+ err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS,
+ MV88E6390_SGMII_PHY_STATUS, &status);
+ if (err) {
+@@ -900,7 +939,7 @@ static int mv88e6390_serdes_pcs_get_state_sgmii(struct mv88e6xxx_chip *chip,
+ return err;
+ }
+
+- return mv88e6xxx_serdes_pcs_get_state(chip, status, lpa, state);
++ return mv88e6xxx_serdes_pcs_get_state(chip, ctrl, status, lpa, state);
+ }
+
+ static int mv88e6390_serdes_pcs_get_state_10g(struct mv88e6xxx_chip *chip,
+@@ -1271,9 +1310,31 @@ void mv88e6390_serdes_get_regs(struct mv88e6xxx_chip *chip, int port, void *_p)
+ }
+ }
+
+-static int mv88e6393x_serdes_port_errata(struct mv88e6xxx_chip *chip, int lane)
++static int mv88e6393x_serdes_power_lane(struct mv88e6xxx_chip *chip, int lane,
++ bool on)
+ {
+- u16 reg, pcs;
++ u16 reg;
++ int err;
++
++ err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_CTRL1, &reg);
++ if (err)
++ return err;
++
++ if (on)
++ reg &= ~(MV88E6393X_SERDES_CTRL1_TX_PDOWN |
++ MV88E6393X_SERDES_CTRL1_RX_PDOWN);
++ else
++ reg |= MV88E6393X_SERDES_CTRL1_TX_PDOWN |
++ MV88E6393X_SERDES_CTRL1_RX_PDOWN;
++
++ return mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_CTRL1, reg);
++}
++
++static int mv88e6393x_serdes_erratum_4_6(struct mv88e6xxx_chip *chip, int lane)
++{
++ u16 reg;
+ int err;
+
+ /* mv88e6393x family errata 4.6:
+@@ -1284,26 +1345,45 @@ static int mv88e6393x_serdes_port_errata(struct mv88e6xxx_chip *chip, int lane)
+ * It seems that after this workaround the SERDES is automatically
+ * powered up (the bit is cleared), so power it down.
+ */
+- if (lane == MV88E6393X_PORT0_LANE || lane == MV88E6393X_PORT9_LANE ||
+- lane == MV88E6393X_PORT10_LANE) {
+- err = mv88e6390_serdes_read(chip, lane,
+- MDIO_MMD_PHYXS,
+- MV88E6393X_SERDES_POC, &reg);
+- if (err)
+- return err;
++ err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_POC, &reg);
++ if (err)
++ return err;
+
+- reg &= ~MV88E6393X_SERDES_POC_PDOWN;
+- reg |= MV88E6393X_SERDES_POC_RESET;
++ reg &= ~MV88E6393X_SERDES_POC_PDOWN;
++ reg |= MV88E6393X_SERDES_POC_RESET;
+
+- err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS,
+- MV88E6393X_SERDES_POC, reg);
+- if (err)
+- return err;
++ err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_POC, reg);
++ if (err)
++ return err;
+
+- err = mv88e6390_serdes_power_sgmii(chip, lane, false);
+- if (err)
+- return err;
+- }
++ err = mv88e6390_serdes_power_sgmii(chip, lane, false);
++ if (err)
++ return err;
++
++ return mv88e6393x_serdes_power_lane(chip, lane, false);
++}
++
++int mv88e6393x_serdes_setup_errata(struct mv88e6xxx_chip *chip)
++{
++ int err;
++
++ err = mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT0_LANE);
++ if (err)
++ return err;
++
++ err = mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT9_LANE);
++ if (err)
++ return err;
++
++ return mv88e6393x_serdes_erratum_4_6(chip, MV88E6393X_PORT10_LANE);
++}
++
++static int mv88e6393x_serdes_erratum_4_8(struct mv88e6xxx_chip *chip, int lane)
++{
++ u16 reg, pcs;
++ int err;
+
+ /* mv88e6393x family errata 4.8:
+ * When a SERDES port is operating in 1000BASE-X or SGMII mode link may
+@@ -1334,38 +1414,152 @@ static int mv88e6393x_serdes_port_errata(struct mv88e6xxx_chip *chip, int lane)
+ MV88E6393X_ERRATA_4_8_REG, reg);
+ }
+
+-int mv88e6393x_serdes_setup_errata(struct mv88e6xxx_chip *chip)
++static int mv88e6393x_serdes_erratum_5_2(struct mv88e6xxx_chip *chip, int lane,
++ u8 cmode)
++{
++ static const struct {
++ u16 dev, reg, val, mask;
++ } fixes[] = {
++ { MDIO_MMD_VEND1, 0x8093, 0xcb5a, 0xffff },
++ { MDIO_MMD_VEND1, 0x8171, 0x7088, 0xffff },
++ { MDIO_MMD_VEND1, 0x80c9, 0x311a, 0xffff },
++ { MDIO_MMD_VEND1, 0x80a2, 0x8000, 0xff7f },
++ { MDIO_MMD_VEND1, 0x80a9, 0x0000, 0xfff0 },
++ { MDIO_MMD_VEND1, 0x80a3, 0x0000, 0xf8ff },
++ { MDIO_MMD_PHYXS, MV88E6393X_SERDES_POC,
++ MV88E6393X_SERDES_POC_RESET, MV88E6393X_SERDES_POC_RESET },
++ };
++ int err, i;
++ u16 reg;
++
++ /* mv88e6393x family errata 5.2:
++ * For optimal signal integrity the following sequence should be applied
++ * to SERDES operating in 10G mode. These registers only apply to 10G
++ * operation and have no effect on other speeds.
++ */
++ if (cmode != MV88E6393X_PORT_STS_CMODE_10GBASER)
++ return 0;
++
++ for (i = 0; i < ARRAY_SIZE(fixes); ++i) {
++ err = mv88e6390_serdes_read(chip, lane, fixes[i].dev,
++ fixes[i].reg, &reg);
++ if (err)
++ return err;
++
++ reg &= ~fixes[i].mask;
++ reg |= fixes[i].val;
++
++ err = mv88e6390_serdes_write(chip, lane, fixes[i].dev,
++ fixes[i].reg, reg);
++ if (err)
++ return err;
++ }
++
++ return 0;
++}
++
++static int mv88e6393x_serdes_fix_2500basex_an(struct mv88e6xxx_chip *chip,
++ int lane, u8 cmode, bool on)
+ {
++ u16 reg;
+ int err;
+
+- err = mv88e6393x_serdes_port_errata(chip, MV88E6393X_PORT0_LANE);
++ if (cmode != MV88E6XXX_PORT_STS_CMODE_2500BASEX)
++ return 0;
++
++ /* Inband AN is broken on Amethyst in 2500base-x mode when set by
++ * standard mechanism (via cmode).
++ * We can get around this by configuring the PCS mode to 1000base-x
++ * and then writing value 0x58 to register 1e.8000. (This must be done
++ * while SerDes receiver and transmitter are disabled, which is, when
++ * this function is called.)
++ * It seem that when we do this configuration to 2500base-x mode (by
++ * changing PCS mode to 1000base-x and frequency to 3.125 GHz from
++ * 1.25 GHz) and then configure to sgmii or 1000base-x, the device
++ * thinks that it already has SerDes at 1.25 GHz and does not change
++ * the 1e.8000 register, leaving SerDes at 3.125 GHz.
++ * To avoid this, change PCS mode back to 2500base-x when disabling
++ * SerDes from 2500base-x mode.
++ */
++ err = mv88e6390_serdes_read(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_POC, &reg);
++ if (err)
++ return err;
++
++ reg &= ~(MV88E6393X_SERDES_POC_PCS_MASK | MV88E6393X_SERDES_POC_AN);
++ if (on)
++ reg |= MV88E6393X_SERDES_POC_PCS_1000BASEX |
++ MV88E6393X_SERDES_POC_AN;
++ else
++ reg |= MV88E6393X_SERDES_POC_PCS_2500BASEX;
++ reg |= MV88E6393X_SERDES_POC_RESET;
++
++ err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_PHYXS,
++ MV88E6393X_SERDES_POC, reg);
+ if (err)
+ return err;
+
+- err = mv88e6393x_serdes_port_errata(chip, MV88E6393X_PORT9_LANE);
++ err = mv88e6390_serdes_write(chip, lane, MDIO_MMD_VEND1, 0x8000, 0x58);
+ if (err)
+ return err;
+
+- return mv88e6393x_serdes_port_errata(chip, MV88E6393X_PORT10_LANE);
++ return 0;
+ }
+
+ int mv88e6393x_serdes_power(struct mv88e6xxx_chip *chip, int port, int lane,
+ bool on)
+ {
+ u8 cmode = chip->ports[port].cmode;
++ int err;
+
+ if (port != 0 && port != 9 && port != 10)
+ return -EOPNOTSUPP;
+
++ if (on) {
++ err = mv88e6393x_serdes_erratum_4_8(chip, lane);
++ if (err)
++ return err;
++
++ err = mv88e6393x_serdes_erratum_5_2(chip, lane, cmode);
++ if (err)
++ return err;
++
++ err = mv88e6393x_serdes_fix_2500basex_an(chip, lane, cmode,
++ true);
++ if (err)
++ return err;
++
++ err = mv88e6393x_serdes_power_lane(chip, lane, true);
++ if (err)
++ return err;
++ }
++
+ switch (cmode) {
+ case MV88E6XXX_PORT_STS_CMODE_SGMII:
+ case MV88E6XXX_PORT_STS_CMODE_1000BASEX:
+ case MV88E6XXX_PORT_STS_CMODE_2500BASEX:
+- return mv88e6390_serdes_power_sgmii(chip, lane, on);
++ err = mv88e6390_serdes_power_sgmii(chip, lane, on);
++ break;
+ case MV88E6393X_PORT_STS_CMODE_5GBASER:
+ case MV88E6393X_PORT_STS_CMODE_10GBASER:
+- return mv88e6390_serdes_power_10g(chip, lane, on);
++ err = mv88e6390_serdes_power_10g(chip, lane, on);
++ break;
++ default:
++ err = -EINVAL;
++ break;
+ }
+
+- return 0;
++ if (err)
++ return err;
++
++ if (!on) {
++ err = mv88e6393x_serdes_power_lane(chip, lane, false);
++ if (err)
++ return err;
++
++ err = mv88e6393x_serdes_fix_2500basex_an(chip, lane, cmode,
++ false);
++ }
++
++ return err;
+ }
+diff --git a/drivers/net/dsa/mv88e6xxx/serdes.h b/drivers/net/dsa/mv88e6xxx/serdes.h
+index cbb3ba30caea9..8dd8ed225b459 100644
+--- a/drivers/net/dsa/mv88e6xxx/serdes.h
++++ b/drivers/net/dsa/mv88e6xxx/serdes.h
+@@ -93,6 +93,10 @@
+ #define MV88E6393X_SERDES_POC_PCS_MASK 0x0007
+ #define MV88E6393X_SERDES_POC_RESET BIT(15)
+ #define MV88E6393X_SERDES_POC_PDOWN BIT(5)
++#define MV88E6393X_SERDES_POC_AN BIT(3)
++#define MV88E6393X_SERDES_CTRL1 0xf003
++#define MV88E6393X_SERDES_CTRL1_TX_PDOWN BIT(9)
++#define MV88E6393X_SERDES_CTRL1_RX_PDOWN BIT(8)
+
+ #define MV88E6393X_ERRATA_4_8_REG 0xF074
+ #define MV88E6393X_ERRATA_4_8_BIT BIT(14)
+diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c
+index 341236dcbdb47..1513dfb523de7 100644
+--- a/drivers/net/dsa/ocelot/felix.c
++++ b/drivers/net/dsa/ocelot/felix.c
+@@ -290,8 +290,11 @@ static int felix_setup_mmio_filtering(struct felix *felix)
+ }
+ }
+
+- if (cpu < 0)
++ if (cpu < 0) {
++ kfree(tagging_rule);
++ kfree(redirect_rule);
+ return -EINVAL;
++ }
+
+ tagging_rule->key_type = OCELOT_VCAP_KEY_ETYPE;
+ *(__be16 *)tagging_rule->key.etype.etype.value = htons(ETH_P_1588);
+@@ -1368,12 +1371,12 @@ out:
+ static bool felix_rxtstamp(struct dsa_switch *ds, int port,
+ struct sk_buff *skb, unsigned int type)
+ {
+- u8 *extraction = skb->data - ETH_HLEN - OCELOT_TAG_LEN;
++ u32 tstamp_lo = OCELOT_SKB_CB(skb)->tstamp_lo;
+ struct skb_shared_hwtstamps *shhwtstamps;
+ struct ocelot *ocelot = ds->priv;
+- u32 tstamp_lo, tstamp_hi;
+ struct timespec64 ts;
+- u64 tstamp, val;
++ u32 tstamp_hi;
++ u64 tstamp;
+
+ /* If the "no XTR IRQ" workaround is in use, tell DSA to defer this skb
+ * for RX timestamping. Then free it, and poll for its copy through
+@@ -1388,9 +1391,6 @@ static bool felix_rxtstamp(struct dsa_switch *ds, int port,
+ ocelot_ptp_gettime64(&ocelot->ptp_info, &ts);
+ tstamp = ktime_set(ts.tv_sec, ts.tv_nsec);
+
+- ocelot_xfh_get_rew_val(extraction, &val);
+- tstamp_lo = (u32)val;
+-
+ tstamp_hi = tstamp >> 32;
+ if ((tstamp & 0xffffffff) < tstamp_lo)
+ tstamp_hi--;
+diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c
+index 11b42fd812e4a..a9c7ada890d88 100644
+--- a/drivers/net/dsa/ocelot/felix_vsc9959.c
++++ b/drivers/net/dsa/ocelot/felix_vsc9959.c
+@@ -1066,7 +1066,7 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot)
+ return PTR_ERR(hw);
+ }
+
+- bus = devm_mdiobus_alloc_size(dev, sizeof(*mdio_priv));
++ bus = mdiobus_alloc_size(sizeof(*mdio_priv));
+ if (!bus)
+ return -ENOMEM;
+
+@@ -1086,6 +1086,7 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot)
+ rc = mdiobus_register(bus);
+ if (rc < 0) {
+ dev_err(dev, "failed to register MDIO bus\n");
++ mdiobus_free(bus);
+ return rc;
+ }
+
+@@ -1135,6 +1136,7 @@ static void vsc9959_mdio_bus_free(struct ocelot *ocelot)
+ lynx_pcs_destroy(pcs);
+ }
+ mdiobus_unregister(felix->imdio);
++ mdiobus_free(felix->imdio);
+ }
+
+ static void vsc9959_sched_speed_set(struct ocelot *ocelot, int port,
+@@ -1453,7 +1455,7 @@ static int felix_pci_probe(struct pci_dev *pdev,
+
+ err = dsa_register_switch(ds);
+ if (err) {
+- dev_err(&pdev->dev, "Failed to register DSA switch: %d\n", err);
++ dev_err_probe(&pdev->dev, err, "Failed to register DSA switch\n");
+ goto err_register_ds;
+ }
+
+diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c
+index de1d34a1f1e47..05e4e75c01076 100644
+--- a/drivers/net/dsa/ocelot/seville_vsc9953.c
++++ b/drivers/net/dsa/ocelot/seville_vsc9953.c
+@@ -10,6 +10,7 @@
+ #include <linux/pcs-lynx.h>
+ #include <linux/dsa/ocelot.h>
+ #include <linux/iopoll.h>
++#include <linux/of_mdio.h>
+ #include "felix.h"
+
+ #define MSCC_MIIM_CMD_OPR_WRITE BIT(1)
+@@ -1110,7 +1111,7 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot)
+ snprintf(bus->id, MII_BUS_ID_SIZE, "%s-imdio", dev_name(dev));
+
+ /* Needed in order to initialize the bus mutex lock */
+- rc = mdiobus_register(bus);
++ rc = devm_of_mdiobus_register(dev, bus, NULL);
+ if (rc < 0) {
+ dev_err(dev, "failed to register MDIO bus\n");
+ return rc;
+@@ -1162,7 +1163,8 @@ static void vsc9953_mdio_bus_free(struct ocelot *ocelot)
+ mdio_device_free(pcs->mdio);
+ lynx_pcs_destroy(pcs);
+ }
+- mdiobus_unregister(felix->imdio);
++
++ /* mdiobus_unregister and mdiobus_free handled by devres */
+ }
+
+ static const struct felix_info seville_info_vsc9953 = {
+diff --git a/drivers/net/dsa/qca/ar9331.c b/drivers/net/dsa/qca/ar9331.c
+index a6bfb6abc51a7..5d476f452396c 100644
+--- a/drivers/net/dsa/qca/ar9331.c
++++ b/drivers/net/dsa/qca/ar9331.c
+@@ -378,7 +378,7 @@ static int ar9331_sw_mbus_init(struct ar9331_sw_priv *priv)
+ if (!mnp)
+ return -ENODEV;
+
+- ret = of_mdiobus_register(mbus, mnp);
++ ret = devm_of_mdiobus_register(dev, mbus, mnp);
+ of_node_put(mnp);
+ if (ret)
+ return ret;
+@@ -1093,7 +1093,6 @@ static void ar9331_sw_remove(struct mdio_device *mdiodev)
+ }
+
+ irq_domain_remove(priv->irqdomain);
+- mdiobus_unregister(priv->mbus);
+ dsa_unregister_switch(&priv->ds);
+
+ reset_control_assert(priv->sw_reset);
+diff --git a/drivers/net/dsa/rtl8366.c b/drivers/net/dsa/rtl8366.c
+index 75897a3690969..ffbe5b6b2655b 100644
+--- a/drivers/net/dsa/rtl8366.c
++++ b/drivers/net/dsa/rtl8366.c
+@@ -457,7 +457,7 @@ int rtl8366_vlan_del(struct dsa_switch *ds, int port,
+ * anymore then clear the whole member
+ * config so it can be reused.
+ */
+- if (!vlanmc.member && vlanmc.untag) {
++ if (!vlanmc.member) {
+ vlanmc.vid = 0;
+ vlanmc.priority = 0;
+ vlanmc.fid = 0;
+diff --git a/drivers/net/dsa/rtl8366rb.c b/drivers/net/dsa/rtl8366rb.c
+index a89093bc6c6ad..9e3b572ed999e 100644
+--- a/drivers/net/dsa/rtl8366rb.c
++++ b/drivers/net/dsa/rtl8366rb.c
+@@ -1350,7 +1350,7 @@ static int rtl8366rb_set_mc_index(struct realtek_smi *smi, int port, int index)
+
+ static bool rtl8366rb_is_vlan_valid(struct realtek_smi *smi, unsigned int vlan)
+ {
+- unsigned int max = RTL8366RB_NUM_VLANS;
++ unsigned int max = RTL8366RB_NUM_VLANS - 1;
+
+ if (smi->vlan4k_enabled)
+ max = RTL8366RB_NUM_VIDS - 1;
+diff --git a/drivers/net/ethernet/8390/mcf8390.c b/drivers/net/ethernet/8390/mcf8390.c
+index 4ad8031ab6695..065fdbe66c425 100644
+--- a/drivers/net/ethernet/8390/mcf8390.c
++++ b/drivers/net/ethernet/8390/mcf8390.c
+@@ -406,12 +406,12 @@ static int mcf8390_init(struct net_device *dev)
+ static int mcf8390_probe(struct platform_device *pdev)
+ {
+ struct net_device *dev;
+- struct resource *mem, *irq;
++ struct resource *mem;
+ resource_size_t msize;
+- int ret;
++ int ret, irq;
+
+- irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+- if (irq == NULL) {
++ irq = platform_get_irq(pdev, 0);
++ if (irq < 0) {
+ dev_err(&pdev->dev, "no IRQ specified?\n");
+ return -ENXIO;
+ }
+@@ -434,7 +434,7 @@ static int mcf8390_probe(struct platform_device *pdev)
+ SET_NETDEV_DEV(dev, &pdev->dev);
+ platform_set_drvdata(pdev, dev);
+
+- dev->irq = irq->start;
++ dev->irq = irq;
+ dev->base_addr = mem->start;
+
+ ret = mcf8390_init(dev);
+diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
+index 412ae3e43ffb7..35ac6fe7529c5 100644
+--- a/drivers/net/ethernet/Kconfig
++++ b/drivers/net/ethernet/Kconfig
+@@ -34,15 +34,6 @@ source "drivers/net/ethernet/apple/Kconfig"
+ source "drivers/net/ethernet/aquantia/Kconfig"
+ source "drivers/net/ethernet/arc/Kconfig"
+ source "drivers/net/ethernet/atheros/Kconfig"
+-source "drivers/net/ethernet/broadcom/Kconfig"
+-source "drivers/net/ethernet/brocade/Kconfig"
+-source "drivers/net/ethernet/cadence/Kconfig"
+-source "drivers/net/ethernet/calxeda/Kconfig"
+-source "drivers/net/ethernet/cavium/Kconfig"
+-source "drivers/net/ethernet/chelsio/Kconfig"
+-source "drivers/net/ethernet/cirrus/Kconfig"
+-source "drivers/net/ethernet/cisco/Kconfig"
+-source "drivers/net/ethernet/cortina/Kconfig"
+
+ config CX_ECAT
+ tristate "Beckhoff CX5020 EtherCAT master support"
+@@ -56,6 +47,14 @@ config CX_ECAT
+ To compile this driver as a module, choose M here. The module
+ will be called ec_bhf.
+
++source "drivers/net/ethernet/broadcom/Kconfig"
++source "drivers/net/ethernet/cadence/Kconfig"
++source "drivers/net/ethernet/calxeda/Kconfig"
++source "drivers/net/ethernet/cavium/Kconfig"
++source "drivers/net/ethernet/chelsio/Kconfig"
++source "drivers/net/ethernet/cirrus/Kconfig"
++source "drivers/net/ethernet/cisco/Kconfig"
++source "drivers/net/ethernet/cortina/Kconfig"
+ source "drivers/net/ethernet/davicom/Kconfig"
+
+ config DNET
+@@ -82,7 +81,6 @@ source "drivers/net/ethernet/huawei/Kconfig"
+ source "drivers/net/ethernet/i825xx/Kconfig"
+ source "drivers/net/ethernet/ibm/Kconfig"
+ source "drivers/net/ethernet/intel/Kconfig"
+-source "drivers/net/ethernet/microsoft/Kconfig"
+ source "drivers/net/ethernet/xscale/Kconfig"
+
+ config JME
+@@ -125,8 +123,9 @@ source "drivers/net/ethernet/mediatek/Kconfig"
+ source "drivers/net/ethernet/mellanox/Kconfig"
+ source "drivers/net/ethernet/micrel/Kconfig"
+ source "drivers/net/ethernet/microchip/Kconfig"
+-source "drivers/net/ethernet/moxa/Kconfig"
+ source "drivers/net/ethernet/mscc/Kconfig"
++source "drivers/net/ethernet/microsoft/Kconfig"
++source "drivers/net/ethernet/moxa/Kconfig"
+ source "drivers/net/ethernet/myricom/Kconfig"
+
+ config FEALNX
+@@ -138,10 +137,10 @@ config FEALNX
+ Say Y here to support the Myson MTD-800 family of PCI-based Ethernet
+ cards. <http://www.myson.com.tw/>
+
++source "drivers/net/ethernet/ni/Kconfig"
+ source "drivers/net/ethernet/natsemi/Kconfig"
+ source "drivers/net/ethernet/neterion/Kconfig"
+ source "drivers/net/ethernet/netronome/Kconfig"
+-source "drivers/net/ethernet/ni/Kconfig"
+ source "drivers/net/ethernet/8390/Kconfig"
+ source "drivers/net/ethernet/nvidia/Kconfig"
+ source "drivers/net/ethernet/nxp/Kconfig"
+@@ -161,6 +160,7 @@ source "drivers/net/ethernet/packetengines/Kconfig"
+ source "drivers/net/ethernet/pasemi/Kconfig"
+ source "drivers/net/ethernet/pensando/Kconfig"
+ source "drivers/net/ethernet/qlogic/Kconfig"
++source "drivers/net/ethernet/brocade/Kconfig"
+ source "drivers/net/ethernet/qualcomm/Kconfig"
+ source "drivers/net/ethernet/rdc/Kconfig"
+ source "drivers/net/ethernet/realtek/Kconfig"
+@@ -168,10 +168,10 @@ source "drivers/net/ethernet/renesas/Kconfig"
+ source "drivers/net/ethernet/rocker/Kconfig"
+ source "drivers/net/ethernet/samsung/Kconfig"
+ source "drivers/net/ethernet/seeq/Kconfig"
+-source "drivers/net/ethernet/sfc/Kconfig"
+ source "drivers/net/ethernet/sgi/Kconfig"
+ source "drivers/net/ethernet/silan/Kconfig"
+ source "drivers/net/ethernet/sis/Kconfig"
++source "drivers/net/ethernet/sfc/Kconfig"
+ source "drivers/net/ethernet/smsc/Kconfig"
+ source "drivers/net/ethernet/socionext/Kconfig"
+ source "drivers/net/ethernet/stmicro/Kconfig"
+diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c
+index 1c00d719e5d76..804b37c76b1e6 100644
+--- a/drivers/net/ethernet/altera/altera_tse_main.c
++++ b/drivers/net/ethernet/altera/altera_tse_main.c
+@@ -1430,16 +1430,19 @@ static int altera_tse_probe(struct platform_device *pdev)
+ priv->rxdescmem_busaddr = dma_res->start;
+
+ } else {
++ ret = -ENODEV;
+ goto err_free_netdev;
+ }
+
+- if (!dma_set_mask(priv->device, DMA_BIT_MASK(priv->dmaops->dmamask)))
++ if (!dma_set_mask(priv->device, DMA_BIT_MASK(priv->dmaops->dmamask))) {
+ dma_set_coherent_mask(priv->device,
+ DMA_BIT_MASK(priv->dmaops->dmamask));
+- else if (!dma_set_mask(priv->device, DMA_BIT_MASK(32)))
++ } else if (!dma_set_mask(priv->device, DMA_BIT_MASK(32))) {
+ dma_set_coherent_mask(priv->device, DMA_BIT_MASK(32));
+- else
++ } else {
++ ret = -EIO;
+ goto err_free_netdev;
++ }
+
+ /* MAC address space */
+ ret = request_and_map(pdev, "control_port", &control_port,
+diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
+index 0e43000614abd..8f08e0bae3004 100644
+--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
++++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
+@@ -1288,26 +1288,22 @@ static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
+
+ static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
+ {
+- struct ena_tx_buffer *tx_info = NULL;
++ struct ena_tx_buffer *tx_info;
+
+- if (likely(req_id < tx_ring->ring_size)) {
+- tx_info = &tx_ring->tx_buffer_info[req_id];
+- if (likely(tx_info->skb))
+- return 0;
+- }
++ tx_info = &tx_ring->tx_buffer_info[req_id];
++ if (likely(tx_info->skb))
++ return 0;
+
+ return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
+ }
+
+ static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)
+ {
+- struct ena_tx_buffer *tx_info = NULL;
++ struct ena_tx_buffer *tx_info;
+
+- if (likely(req_id < xdp_ring->ring_size)) {
+- tx_info = &xdp_ring->tx_buffer_info[req_id];
+- if (likely(tx_info->xdpf))
+- return 0;
+- }
++ tx_info = &xdp_ring->tx_buffer_info[req_id];
++ if (likely(tx_info->xdpf))
++ return 0;
+
+ return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);
+ }
+@@ -1332,9 +1328,14 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
+
+ rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
+ &req_id);
+- if (rc)
++ if (rc) {
++ if (unlikely(rc == -EINVAL))
++ handle_invalid_req_id(tx_ring, req_id, NULL,
++ false);
+ break;
++ }
+
++ /* validate that the request id points to a valid skb */
+ rc = validate_tx_req_id(tx_ring, req_id);
+ if (rc)
+ break;
+@@ -1427,6 +1428,7 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
+ u16 *next_to_clean)
+ {
+ struct ena_rx_buffer *rx_info;
++ struct ena_adapter *adapter;
+ u16 len, req_id, buf = 0;
+ struct sk_buff *skb;
+ void *page_addr;
+@@ -1439,8 +1441,14 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
+ rx_info = &rx_ring->rx_buffer_info[req_id];
+
+ if (unlikely(!rx_info->page)) {
+- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+- "Page is NULL\n");
++ adapter = rx_ring->adapter;
++ netif_err(adapter, rx_err, rx_ring->netdev,
++ "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);
++ ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);
++ adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
++ /* Make sure reset reason is set before triggering the reset */
++ smp_mb__before_atomic();
++ set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+ return NULL;
+ }
+
+@@ -1896,9 +1904,14 @@ static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
+
+ rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,
+ &req_id);
+- if (rc)
++ if (rc) {
++ if (unlikely(rc == -EINVAL))
++ handle_invalid_req_id(xdp_ring, req_id, NULL,
++ true);
+ break;
++ }
+
++ /* validate that the request id points to a valid xdp_frame */
+ rc = validate_xdp_req_id(xdp_ring, req_id);
+ if (rc)
+ break;
+@@ -4013,10 +4026,6 @@ static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
+ max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
+ /* 1 IRQ for mgmnt and 1 IRQs for each IO direction */
+ max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
+- if (unlikely(!max_num_io_queues)) {
+- dev_err(&pdev->dev, "The device doesn't have io queues\n");
+- return -EFAULT;
+- }
+
+ return max_num_io_queues;
+ }
+diff --git a/drivers/net/ethernet/amd/Kconfig b/drivers/net/ethernet/amd/Kconfig
+index 4786f0504691d..899c8a2a34b6b 100644
+--- a/drivers/net/ethernet/amd/Kconfig
++++ b/drivers/net/ethernet/amd/Kconfig
+@@ -168,7 +168,7 @@ config SUNLANCE
+
+ config AMD_XGBE
+ tristate "AMD 10GbE Ethernet driver"
+- depends on ((OF_NET && OF_ADDRESS) || ACPI || PCI) && HAS_IOMEM
++ depends on (OF_ADDRESS || ACPI || PCI) && HAS_IOMEM
+ depends on X86 || ARM64 || COMPILE_TEST
+ depends on PTP_1588_CLOCK_OPTIONAL
+ select BITREVERSE
+diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
+index b2cd3bdba9f89..533b8519ec352 100644
+--- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h
++++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
+@@ -1331,6 +1331,10 @@
+ #define MDIO_VEND2_PMA_CDR_CONTROL 0x8056
+ #endif
+
++#ifndef MDIO_VEND2_PMA_MISC_CTRL0
++#define MDIO_VEND2_PMA_MISC_CTRL0 0x8090
++#endif
++
+ #ifndef MDIO_CTRL1_SPEED1G
+ #define MDIO_CTRL1_SPEED1G (MDIO_CTRL1_SPEED10G & ~BMCR_SPEED100)
+ #endif
+@@ -1389,6 +1393,10 @@
+ #define XGBE_PMA_RX_RST_0_RESET_ON 0x10
+ #define XGBE_PMA_RX_RST_0_RESET_OFF 0x00
+
++#define XGBE_PMA_PLL_CTRL_MASK BIT(15)
++#define XGBE_PMA_PLL_CTRL_ENABLE BIT(15)
++#define XGBE_PMA_PLL_CTRL_DISABLE 0x0000
++
+ /* Bit setting and getting macros
+ * The get macro will extract the current bit field value from within
+ * the variable
+diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+index 17a585adfb49c..e6883d52d230c 100644
+--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
++++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+@@ -721,7 +721,9 @@ static void xgbe_stop_timers(struct xgbe_prv_data *pdata)
+ if (!channel->tx_ring)
+ break;
+
++ /* Deactivate the Tx timer */
+ del_timer_sync(&channel->tx_timer);
++ channel->tx_timer_active = 0;
+ }
+ }
+
+@@ -2555,6 +2557,14 @@ read_again:
+ buf2_len = xgbe_rx_buf2_len(rdata, packet, len);
+ len += buf2_len;
+
++ if (buf2_len > rdata->rx.buf.dma_len) {
++ /* Hardware inconsistency within the descriptors
++ * that has resulted in a length underflow.
++ */
++ error = 1;
++ goto skip_data;
++ }
++
+ if (!skb) {
+ skb = xgbe_create_skb(pdata, napi, rdata,
+ buf1_len);
+@@ -2584,8 +2594,10 @@ skip_data:
+ if (!last || context_next)
+ goto read_again;
+
+- if (!skb)
++ if (!skb || error) {
++ dev_kfree_skb(skb);
+ goto next_packet;
++ }
+
+ /* Be sure we don't exceed the configured MTU */
+ max_len = netdev->mtu + ETH_HLEN;
+diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+index 90cb55eb54665..014513ce00a14 100644
+--- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
++++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+@@ -418,6 +418,9 @@ static void xgbe_pci_remove(struct pci_dev *pdev)
+
+ pci_free_irq_vectors(pdata->pcidev);
+
++ /* Disable all interrupts in the hardware */
++ XP_IOWRITE(pdata, XP_INT_EN, 0x0);
++
+ xgbe_free_pdata(pdata);
+ }
+
+diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
+index 18e48b3bc402b..213769054391c 100644
+--- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
++++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
+@@ -1977,12 +1977,26 @@ static void xgbe_phy_rx_reset(struct xgbe_prv_data *pdata)
+ }
+ }
+
++static void xgbe_phy_pll_ctrl(struct xgbe_prv_data *pdata, bool enable)
++{
++ XMDIO_WRITE_BITS(pdata, MDIO_MMD_PMAPMD, MDIO_VEND2_PMA_MISC_CTRL0,
++ XGBE_PMA_PLL_CTRL_MASK,
++ enable ? XGBE_PMA_PLL_CTRL_ENABLE
++ : XGBE_PMA_PLL_CTRL_DISABLE);
++
++ /* Wait for command to complete */
++ usleep_range(100, 200);
++}
++
+ static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata,
+ unsigned int cmd, unsigned int sub_cmd)
+ {
+ unsigned int s0 = 0;
+ unsigned int wait;
+
++ /* Disable PLL re-initialization during FW command processing */
++ xgbe_phy_pll_ctrl(pdata, false);
++
+ /* Log if a previous command did not complete */
+ if (XP_IOREAD_BITS(pdata, XP_DRIVER_INT_RO, STATUS)) {
+ netif_dbg(pdata, link, pdata->netdev,
+@@ -2003,7 +2017,7 @@ static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata,
+ wait = XGBE_RATECHANGE_COUNT;
+ while (wait--) {
+ if (!XP_IOREAD_BITS(pdata, XP_DRIVER_INT_RO, STATUS))
+- return;
++ goto reenable_pll;
+
+ usleep_range(1000, 2000);
+ }
+@@ -2013,6 +2027,10 @@ static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata,
+
+ /* Reset on error */
+ xgbe_phy_rx_reset(pdata);
++
++reenable_pll:
++ /* Enable PLL re-initialization */
++ xgbe_phy_pll_ctrl(pdata, true);
+ }
+
+ static void xgbe_phy_rrc(struct xgbe_prv_data *pdata)
+diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+index 5f1fc6582d74a..78c7cbc372b05 100644
+--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
++++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+@@ -696,6 +696,12 @@ static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
+ buf_pool->rx_skb[skb_index] = NULL;
+
+ datalen = xgene_enet_get_data_len(le64_to_cpu(raw_desc->m1));
++
++ /* strip off CRC as HW isn't doing this */
++ nv = GET_VAL(NV, le64_to_cpu(raw_desc->m0));
++ if (!nv)
++ datalen -= 4;
++
+ skb_put(skb, datalen);
+ prefetch(skb->data - NET_IP_ALIGN);
+ skb->protocol = eth_type_trans(skb, ndev);
+@@ -717,12 +723,8 @@ static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
+ }
+ }
+
+- nv = GET_VAL(NV, le64_to_cpu(raw_desc->m0));
+- if (!nv) {
+- /* strip off CRC as HW isn't doing this */
+- datalen -= 4;
++ if (!nv)
+ goto skip_jumbo;
+- }
+
+ slots = page_pool->slots - 1;
+ head = page_pool->head;
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_common.h b/drivers/net/ethernet/aquantia/atlantic/aq_common.h
+index 23b2d390fcdda..ace691d7cd759 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_common.h
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_common.h
+@@ -40,10 +40,12 @@
+
+ #define AQ_DEVICE_ID_AQC113DEV 0x00C0
+ #define AQ_DEVICE_ID_AQC113CS 0x94C0
++#define AQ_DEVICE_ID_AQC113CA 0x34C0
+ #define AQ_DEVICE_ID_AQC114CS 0x93C0
+ #define AQ_DEVICE_ID_AQC113 0x04C0
+ #define AQ_DEVICE_ID_AQC113C 0x14C0
+ #define AQ_DEVICE_ID_AQC115C 0x12C0
++#define AQ_DEVICE_ID_AQC116C 0x11C0
+
+ #define HW_ATL_NIC_NAME "Marvell (aQuantia) AQtion 10Gbit Network Adapter"
+
+@@ -53,20 +55,19 @@
+
+ #define AQ_NIC_RATE_10G BIT(0)
+ #define AQ_NIC_RATE_5G BIT(1)
+-#define AQ_NIC_RATE_5GSR BIT(2)
+-#define AQ_NIC_RATE_2G5 BIT(3)
+-#define AQ_NIC_RATE_1G BIT(4)
+-#define AQ_NIC_RATE_100M BIT(5)
+-#define AQ_NIC_RATE_10M BIT(6)
+-#define AQ_NIC_RATE_1G_HALF BIT(7)
+-#define AQ_NIC_RATE_100M_HALF BIT(8)
+-#define AQ_NIC_RATE_10M_HALF BIT(9)
++#define AQ_NIC_RATE_2G5 BIT(2)
++#define AQ_NIC_RATE_1G BIT(3)
++#define AQ_NIC_RATE_100M BIT(4)
++#define AQ_NIC_RATE_10M BIT(5)
++#define AQ_NIC_RATE_1G_HALF BIT(6)
++#define AQ_NIC_RATE_100M_HALF BIT(7)
++#define AQ_NIC_RATE_10M_HALF BIT(8)
+
+-#define AQ_NIC_RATE_EEE_10G BIT(10)
+-#define AQ_NIC_RATE_EEE_5G BIT(11)
+-#define AQ_NIC_RATE_EEE_2G5 BIT(12)
+-#define AQ_NIC_RATE_EEE_1G BIT(13)
+-#define AQ_NIC_RATE_EEE_100M BIT(14)
++#define AQ_NIC_RATE_EEE_10G BIT(9)
++#define AQ_NIC_RATE_EEE_5G BIT(10)
++#define AQ_NIC_RATE_EEE_2G5 BIT(11)
++#define AQ_NIC_RATE_EEE_1G BIT(12)
++#define AQ_NIC_RATE_EEE_100M BIT(13)
+ #define AQ_NIC_RATE_EEE_MSK (AQ_NIC_RATE_EEE_10G |\
+ AQ_NIC_RATE_EEE_5G |\
+ AQ_NIC_RATE_EEE_2G5 |\
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
+index bed481816ea31..7442850ca95f0 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
+@@ -80,6 +80,8 @@ struct aq_hw_link_status_s {
+ };
+
+ struct aq_stats_s {
++ u64 brc;
++ u64 btc;
+ u64 uprc;
+ u64 mprc;
+ u64 bprc;
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
+index 6c049864dac08..fbb1e05d58783 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
+@@ -480,8 +480,8 @@ int aq_nic_start(struct aq_nic_s *self)
+ if (err < 0)
+ goto err_exit;
+
+- for (i = 0U, aq_vec = self->aq_vec[0];
+- self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
++ for (i = 0U; self->aq_vecs > i; ++i) {
++ aq_vec = self->aq_vec[i];
+ err = aq_vec_start(aq_vec);
+ if (err < 0)
+ goto err_exit;
+@@ -511,8 +511,8 @@ int aq_nic_start(struct aq_nic_s *self)
+ mod_timer(&self->polling_timer, jiffies +
+ AQ_CFG_POLLING_TIMER_INTERVAL);
+ } else {
+- for (i = 0U, aq_vec = self->aq_vec[0];
+- self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
++ for (i = 0U; self->aq_vecs > i; ++i) {
++ aq_vec = self->aq_vec[i];
+ err = aq_pci_func_alloc_irq(self, i, self->ndev->name,
+ aq_vec_isr, aq_vec,
+ aq_vec_get_affinity_mask(aq_vec));
+@@ -903,8 +903,14 @@ u64 *aq_nic_get_stats(struct aq_nic_s *self, u64 *data)
+ data[++i] = stats->mbtc;
+ data[++i] = stats->bbrc;
+ data[++i] = stats->bbtc;
+- data[++i] = stats->ubrc + stats->mbrc + stats->bbrc;
+- data[++i] = stats->ubtc + stats->mbtc + stats->bbtc;
++ if (stats->brc)
++ data[++i] = stats->brc;
++ else
++ data[++i] = stats->ubrc + stats->mbrc + stats->bbrc;
++ if (stats->btc)
++ data[++i] = stats->btc;
++ else
++ data[++i] = stats->ubtc + stats->mbtc + stats->bbtc;
+ data[++i] = stats->dma_pkt_rc;
+ data[++i] = stats->dma_pkt_tc;
+ data[++i] = stats->dma_oct_rc;
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
+index d4b1976ee69b9..831833911a525 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
+@@ -49,6 +49,8 @@ static const struct pci_device_id aq_pci_tbl[] = {
+ { PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC113), },
+ { PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC113C), },
+ { PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC115C), },
++ { PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC113CA), },
++ { PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC116C), },
+
+ {}
+ };
+@@ -85,7 +87,10 @@ static const struct aq_board_revision_s hw_atl_boards[] = {
+ { AQ_DEVICE_ID_AQC113CS, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc113, },
+ { AQ_DEVICE_ID_AQC114CS, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc113, },
+ { AQ_DEVICE_ID_AQC113C, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc113, },
+- { AQ_DEVICE_ID_AQC115C, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc113, },
++ { AQ_DEVICE_ID_AQC115C, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc115c, },
++ { AQ_DEVICE_ID_AQC113CA, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc113, },
++ { AQ_DEVICE_ID_AQC116C, AQ_HWREV_ANY, &hw_atl2_ops, &hw_atl2_caps_aqc116c, },
++
+ };
+
+ MODULE_DEVICE_TABLE(pci, aq_pci_tbl);
+@@ -439,7 +444,7 @@ err_exit:
+
+ static int aq_pm_freeze(struct device *dev)
+ {
+- return aq_suspend_common(dev, false);
++ return aq_suspend_common(dev, true);
+ }
+
+ static int aq_pm_suspend_poweroff(struct device *dev)
+@@ -449,7 +454,7 @@ static int aq_pm_suspend_poweroff(struct device *dev)
+
+ static int aq_pm_thaw(struct device *dev)
+ {
+- return atl_resume_common(dev, false);
++ return atl_resume_common(dev, true);
+ }
+
+ static int aq_pm_resume_restore(struct device *dev)
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
+index 24122ccda614c..e9c6f1fa0b1a7 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
+@@ -345,7 +345,6 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
+ int budget)
+ {
+ struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);
+- bool is_rsc_completed = true;
+ int err = 0;
+
+ for (; (self->sw_head != self->hw_head) && budget;
+@@ -363,8 +362,17 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
+ continue;
+
+ if (!buff->is_eop) {
++ unsigned int frag_cnt = 0U;
+ buff_ = buff;
+ do {
++ bool is_rsc_completed = true;
++
++ if (buff_->next >= self->size) {
++ err = -EIO;
++ goto err_exit;
++ }
++
++ frag_cnt++;
+ next_ = buff_->next,
+ buff_ = &self->buff_ring[next_];
+ is_rsc_completed =
+@@ -372,22 +380,25 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
+ next_,
+ self->hw_head);
+
+- if (unlikely(!is_rsc_completed))
+- break;
++ if (unlikely(!is_rsc_completed) ||
++ frag_cnt > MAX_SKB_FRAGS) {
++ err = 0;
++ goto err_exit;
++ }
+
+ buff->is_error |= buff_->is_error;
+ buff->is_cso_err |= buff_->is_cso_err;
+
+ } while (!buff_->is_eop);
+
+- if (!is_rsc_completed) {
+- err = 0;
+- goto err_exit;
+- }
+ if (buff->is_error ||
+ (buff->is_lro && buff->is_cso_err)) {
+ buff_ = buff;
+ do {
++ if (buff_->next >= self->size) {
++ err = -EIO;
++ goto err_exit;
++ }
+ next_ = buff_->next,
+ buff_ = &self->buff_ring[next_];
+
+@@ -437,7 +448,7 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
+ ALIGN(hdr_len, sizeof(long)));
+
+ if (buff->len - hdr_len > 0) {
+- skb_add_rx_frag(skb, 0, buff->rxdata.page,
++ skb_add_rx_frag(skb, i++, buff->rxdata.page,
+ buff->rxdata.pg_off + hdr_len,
+ buff->len - hdr_len,
+ AQ_CFG_RX_FRAME_MAX);
+@@ -446,7 +457,6 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
+
+ if (!buff->is_eop) {
+ buff_ = buff;
+- i = 1U;
+ do {
+ next_ = buff_->next;
+ buff_ = &self->buff_ring[next_];
+diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
+index d281322d7dd29..6ab1f3212d246 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
++++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
+@@ -43,8 +43,8 @@ static int aq_vec_poll(struct napi_struct *napi, int budget)
+ if (!self) {
+ err = -EINVAL;
+ } else {
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
+ ring[AQ_VEC_RX_ID].stats.rx.polls++;
+ u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
+@@ -182,8 +182,8 @@ int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
+ self->aq_hw_ops = aq_hw_ops;
+ self->aq_hw = aq_hw;
+
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX);
+ if (err < 0)
+ goto err_exit;
+@@ -224,8 +224,8 @@ int aq_vec_start(struct aq_vec_s *self)
+ unsigned int i = 0U;
+ int err = 0;
+
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
+ &ring[AQ_VEC_TX_ID]);
+ if (err < 0)
+@@ -248,8 +248,8 @@ void aq_vec_stop(struct aq_vec_s *self)
+ struct aq_ring_s *ring = NULL;
+ unsigned int i = 0U;
+
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
+ &ring[AQ_VEC_TX_ID]);
+
+@@ -268,8 +268,8 @@ void aq_vec_deinit(struct aq_vec_s *self)
+ if (!self)
+ goto err_exit;
+
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
+ aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
+ }
+@@ -297,8 +297,8 @@ void aq_vec_ring_free(struct aq_vec_s *self)
+ if (!self)
+ goto err_exit;
+
+- for (i = 0U, ring = self->ring[0];
+- self->tx_rings > i; ++i, ring = self->ring[i]) {
++ for (i = 0U; self->tx_rings > i; ++i) {
++ ring = self->ring[i];
+ aq_ring_free(&ring[AQ_VEC_TX_ID]);
+ if (i < self->rx_rings)
+ aq_ring_free(&ring[AQ_VEC_RX_ID]);
+@@ -362,9 +362,6 @@ unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u
+ {
+ unsigned int count;
+
+- WARN_ONCE(!aq_vec_is_valid_tc(self, tc),
+- "Invalid tc %u (#rx=%u, #tx=%u)\n",
+- tc, self->rx_rings, self->tx_rings);
+ if (!aq_vec_is_valid_tc(self, tc))
+ return 0;
+
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
+index 9f1b15077e7d6..45c17c585d743 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
+@@ -889,6 +889,13 @@ int hw_atl_b0_hw_ring_tx_head_update(struct aq_hw_s *self,
+ err = -ENXIO;
+ goto err_exit;
+ }
++
++ /* Validate that the new hw_head_ is reasonable. */
++ if (hw_head_ >= ring->size) {
++ err = -ENXIO;
++ goto err_exit;
++ }
++
+ ring->hw_head = hw_head_;
+ err = aq_hw_err_from_flags(self);
+
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
+index 404cbf60d3f2f..65b9e5846be45 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
+@@ -559,6 +559,11 @@ int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
+ goto err_exit;
+
+ if (fw.len == 0xFFFFU) {
++ if (sw.len > sizeof(self->rpc)) {
++ printk(KERN_INFO "Invalid sw len: %x\n", sw.len);
++ err = -EINVAL;
++ goto err_exit;
++ }
+ err = hw_atl_utils_fw_rpc_call(self, sw.len);
+ if (err < 0)
+ goto err_exit;
+@@ -567,6 +572,11 @@ int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
+
+ if (rpc) {
+ if (fw.len) {
++ if (fw.len > sizeof(self->rpc)) {
++ printk(KERN_INFO "Invalid fw len: %x\n", fw.len);
++ err = -EINVAL;
++ goto err_exit;
++ }
+ err =
+ hw_atl_utils_fw_downld_dwords(self,
+ self->rpc_addr,
+@@ -857,12 +867,20 @@ static int hw_atl_fw1x_deinit(struct aq_hw_s *self)
+ int hw_atl_utils_update_stats(struct aq_hw_s *self)
+ {
+ struct aq_stats_s *cs = &self->curr_stats;
++ struct aq_stats_s curr_stats = *cs;
+ struct hw_atl_utils_mbox mbox;
++ bool corrupted_stats = false;
+
+ hw_atl_utils_mpi_read_stats(self, &mbox);
+
+-#define AQ_SDELTA(_N_) (self->curr_stats._N_ += \
+- mbox.stats._N_ - self->last_stats._N_)
++#define AQ_SDELTA(_N_) \
++do { \
++ if (!corrupted_stats && \
++ ((s64)(mbox.stats._N_ - self->last_stats._N_)) >= 0) \
++ curr_stats._N_ += mbox.stats._N_ - self->last_stats._N_; \
++ else \
++ corrupted_stats = true; \
++} while (0)
+
+ if (self->aq_link_status.mbps) {
+ AQ_SDELTA(uprc);
+@@ -882,6 +900,9 @@ int hw_atl_utils_update_stats(struct aq_hw_s *self)
+ AQ_SDELTA(bbrc);
+ AQ_SDELTA(bbtc);
+ AQ_SDELTA(dpc);
++
++ if (!corrupted_stats)
++ *cs = curr_stats;
+ }
+ #undef AQ_SDELTA
+
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
+index ee0c22d049354..05086f0040fd9 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
+@@ -132,9 +132,6 @@ static enum hw_atl_fw2x_rate link_speed_mask_2fw2x_ratemask(u32 speed)
+ if (speed & AQ_NIC_RATE_5G)
+ rate |= FW2X_RATE_5G;
+
+- if (speed & AQ_NIC_RATE_5GSR)
+- rate |= FW2X_RATE_5G;
+-
+ if (speed & AQ_NIC_RATE_2G5)
+ rate |= FW2X_RATE_2G5;
+
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c
+index 92f64048bf691..c76ccdc77ba60 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.c
+@@ -65,11 +65,25 @@ const struct aq_hw_caps_s hw_atl2_caps_aqc113 = {
+ AQ_NIC_RATE_5G |
+ AQ_NIC_RATE_2G5 |
+ AQ_NIC_RATE_1G |
+- AQ_NIC_RATE_1G_HALF |
+ AQ_NIC_RATE_100M |
+- AQ_NIC_RATE_100M_HALF |
+- AQ_NIC_RATE_10M |
+- AQ_NIC_RATE_10M_HALF,
++ AQ_NIC_RATE_10M,
++};
++
++const struct aq_hw_caps_s hw_atl2_caps_aqc115c = {
++ DEFAULT_BOARD_BASIC_CAPABILITIES,
++ .media_type = AQ_HW_MEDIA_TYPE_TP,
++ .link_speed_msk = AQ_NIC_RATE_2G5 |
++ AQ_NIC_RATE_1G |
++ AQ_NIC_RATE_100M |
++ AQ_NIC_RATE_10M,
++};
++
++const struct aq_hw_caps_s hw_atl2_caps_aqc116c = {
++ DEFAULT_BOARD_BASIC_CAPABILITIES,
++ .media_type = AQ_HW_MEDIA_TYPE_TP,
++ .link_speed_msk = AQ_NIC_RATE_1G |
++ AQ_NIC_RATE_100M |
++ AQ_NIC_RATE_10M,
+ };
+
+ static u32 hw_atl2_sem_act_rslvr_get(struct aq_hw_s *self)
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.h
+index de8723f1c28a1..346f0dc9912e5 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.h
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2.h
+@@ -9,6 +9,8 @@
+ #include "aq_common.h"
+
+ extern const struct aq_hw_caps_s hw_atl2_caps_aqc113;
++extern const struct aq_hw_caps_s hw_atl2_caps_aqc115c;
++extern const struct aq_hw_caps_s hw_atl2_caps_aqc116c;
+ extern const struct aq_hw_ops hw_atl2_ops;
+
+ #endif /* HW_ATL2_H */
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils.h
+index b66fa346581ce..6bad64c77b87c 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils.h
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils.h
+@@ -239,7 +239,8 @@ struct version_s {
+ u8 minor;
+ u16 build;
+ } phy;
+- u32 rsvd;
++ u32 drv_iface_ver:4;
++ u32 rsvd:28;
+ };
+
+ struct link_status_s {
+@@ -424,7 +425,7 @@ struct cable_diag_status_s {
+ u16 rsvd2;
+ };
+
+-struct statistics_s {
++struct statistics_a0_s {
+ struct {
+ u32 link_up;
+ u32 link_down;
+@@ -457,6 +458,33 @@ struct statistics_s {
+ u32 reserve_fw_gap;
+ };
+
++struct __packed statistics_b0_s {
++ u64 rx_good_octets;
++ u64 rx_pause_frames;
++ u64 rx_good_frames;
++ u64 rx_errors;
++ u64 rx_unicast_frames;
++ u64 rx_multicast_frames;
++ u64 rx_broadcast_frames;
++
++ u64 tx_good_octets;
++ u64 tx_pause_frames;
++ u64 tx_good_frames;
++ u64 tx_errors;
++ u64 tx_unicast_frames;
++ u64 tx_multicast_frames;
++ u64 tx_broadcast_frames;
++
++ u32 main_loop_cycles;
++};
++
++struct __packed statistics_s {
++ union __packed {
++ struct statistics_a0_s a0;
++ struct statistics_b0_s b0;
++ };
++};
++
+ struct filter_caps_s {
+ u8 l2_filters_base_index:6;
+ u8 flexible_filter_mask:2;
+@@ -545,7 +573,7 @@ struct management_status_s {
+ u32 rsvd5;
+ };
+
+-struct fw_interface_out {
++struct __packed fw_interface_out {
+ struct transaction_counter_s transaction_id;
+ struct version_s version;
+ struct link_status_s link_status;
+@@ -569,7 +597,6 @@ struct fw_interface_out {
+ struct core_dump_s core_dump;
+ u32 rsvd11;
+ struct statistics_s stats;
+- u32 rsvd12;
+ struct filter_caps_s filter_caps;
+ struct device_caps_s device_caps;
+ u32 rsvd13;
+@@ -592,6 +619,9 @@ struct fw_interface_out {
+ #define AQ_HOST_MODE_LOW_POWER 3U
+ #define AQ_HOST_MODE_SHUTDOWN 4U
+
++#define AQ_A2_FW_INTERFACE_A0 0
++#define AQ_A2_FW_INTERFACE_B0 1
++
+ int hw_atl2_utils_initfw(struct aq_hw_s *self, const struct aq_fw_ops **fw_ops);
+
+ int hw_atl2_utils_soft_reset(struct aq_hw_s *self);
+diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c
+index dd259c8f2f4f3..58d426dda3edb 100644
+--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c
++++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl2/hw_atl2_utils_fw.c
+@@ -84,7 +84,7 @@ static int hw_atl2_shared_buffer_read_block(struct aq_hw_s *self,
+ if (cnt > AQ_A2_FW_READ_TRY_MAX)
+ return -ETIME;
+ if (tid1.transaction_cnt_a != tid1.transaction_cnt_b)
+- udelay(1);
++ mdelay(1);
+ } while (tid1.transaction_cnt_a != tid1.transaction_cnt_b);
+
+ hw_atl2_mif_shared_buf_read(self, offset, (u32 *)data, dwords);
+@@ -154,7 +154,7 @@ static void a2_link_speed_mask2fw(u32 speed,
+ {
+ link_options->rate_10G = !!(speed & AQ_NIC_RATE_10G);
+ link_options->rate_5G = !!(speed & AQ_NIC_RATE_5G);
+- link_options->rate_N5G = !!(speed & AQ_NIC_RATE_5GSR);
++ link_options->rate_N5G = link_options->rate_5G;
+ link_options->rate_2P5G = !!(speed & AQ_NIC_RATE_2G5);
+ link_options->rate_N2P5G = link_options->rate_2P5G;
+ link_options->rate_1G = !!(speed & AQ_NIC_RATE_1G);
+@@ -192,8 +192,6 @@ static u32 a2_fw_lkp_to_mask(struct lkp_link_caps_s *lkp_link_caps)
+ rate |= AQ_NIC_RATE_10G;
+ if (lkp_link_caps->rate_5G)
+ rate |= AQ_NIC_RATE_5G;
+- if (lkp_link_caps->rate_N5G)
+- rate |= AQ_NIC_RATE_5GSR;
+ if (lkp_link_caps->rate_2P5G)
+ rate |= AQ_NIC_RATE_2G5;
+ if (lkp_link_caps->rate_1G)
+@@ -335,15 +333,22 @@ static int aq_a2_fw_get_mac_permanent(struct aq_hw_s *self, u8 *mac)
+ return 0;
+ }
+
+-static int aq_a2_fw_update_stats(struct aq_hw_s *self)
++static void aq_a2_fill_a0_stats(struct aq_hw_s *self,
++ struct statistics_s *stats)
+ {
+ struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv;
+- struct statistics_s stats;
+-
+- hw_atl2_shared_buffer_read_safe(self, stats, &stats);
+-
+-#define AQ_SDELTA(_N_, _F_) (self->curr_stats._N_ += \
+- stats.msm._F_ - priv->last_stats.msm._F_)
++ struct aq_stats_s *cs = &self->curr_stats;
++ struct aq_stats_s curr_stats = *cs;
++ bool corrupted_stats = false;
++
++#define AQ_SDELTA(_N, _F) \
++do { \
++ if (!corrupted_stats && \
++ ((s64)(stats->a0.msm._F - priv->last_stats.a0.msm._F)) >= 0) \
++ curr_stats._N += stats->a0.msm._F - priv->last_stats.a0.msm._F;\
++ else \
++ corrupted_stats = true; \
++} while (0)
+
+ if (self->aq_link_status.mbps) {
+ AQ_SDELTA(uprc, rx_unicast_frames);
+@@ -362,17 +367,76 @@ static int aq_a2_fw_update_stats(struct aq_hw_s *self)
+ AQ_SDELTA(mbtc, tx_multicast_octets);
+ AQ_SDELTA(bbrc, rx_broadcast_octets);
+ AQ_SDELTA(bbtc, tx_broadcast_octets);
++
++ if (!corrupted_stats)
++ *cs = curr_stats;
+ }
+ #undef AQ_SDELTA
+- self->curr_stats.dma_pkt_rc =
+- hw_atl_stats_rx_dma_good_pkt_counter_get(self);
+- self->curr_stats.dma_pkt_tc =
+- hw_atl_stats_tx_dma_good_pkt_counter_get(self);
+- self->curr_stats.dma_oct_rc =
+- hw_atl_stats_rx_dma_good_octet_counter_get(self);
+- self->curr_stats.dma_oct_tc =
+- hw_atl_stats_tx_dma_good_octet_counter_get(self);
+- self->curr_stats.dpc = hw_atl_rpb_rx_dma_drop_pkt_cnt_get(self);
++
++}
++
++static void aq_a2_fill_b0_stats(struct aq_hw_s *self,
++ struct statistics_s *stats)
++{
++ struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv;
++ struct aq_stats_s *cs = &self->curr_stats;
++ struct aq_stats_s curr_stats = *cs;
++ bool corrupted_stats = false;
++
++#define AQ_SDELTA(_N, _F) \
++do { \
++ if (!corrupted_stats && \
++ ((s64)(stats->b0._F - priv->last_stats.b0._F)) >= 0) \
++ curr_stats._N += stats->b0._F - priv->last_stats.b0._F; \
++ else \
++ corrupted_stats = true; \
++} while (0)
++
++ if (self->aq_link_status.mbps) {
++ AQ_SDELTA(uprc, rx_unicast_frames);
++ AQ_SDELTA(mprc, rx_multicast_frames);
++ AQ_SDELTA(bprc, rx_broadcast_frames);
++ AQ_SDELTA(erpr, rx_errors);
++ AQ_SDELTA(brc, rx_good_octets);
++
++ AQ_SDELTA(uptc, tx_unicast_frames);
++ AQ_SDELTA(mptc, tx_multicast_frames);
++ AQ_SDELTA(bptc, tx_broadcast_frames);
++ AQ_SDELTA(erpt, tx_errors);
++ AQ_SDELTA(btc, tx_good_octets);
++
++ if (!corrupted_stats)
++ *cs = curr_stats;
++ }
++#undef AQ_SDELTA
++}
++
++static int aq_a2_fw_update_stats(struct aq_hw_s *self)
++{
++ struct hw_atl2_priv *priv = (struct hw_atl2_priv *)self->priv;
++ struct aq_stats_s *cs = &self->curr_stats;
++ struct statistics_s stats;
++ struct version_s version;
++ int err;
++
++ err = hw_atl2_shared_buffer_read_safe(self, version, &version);
++ if (err)
++ return err;
++
++ err = hw_atl2_shared_buffer_read_safe(self, stats, &stats);
++ if (err)
++ return err;
++
++ if (version.drv_iface_ver == AQ_A2_FW_INTERFACE_A0)
++ aq_a2_fill_a0_stats(self, &stats);
++ else
++ aq_a2_fill_b0_stats(self, &stats);
++
++ cs->dma_pkt_rc = hw_atl_stats_rx_dma_good_pkt_counter_get(self);
++ cs->dma_pkt_tc = hw_atl_stats_tx_dma_good_pkt_counter_get(self);
++ cs->dma_oct_rc = hw_atl_stats_rx_dma_good_octet_counter_get(self);
++ cs->dma_oct_tc = hw_atl_stats_tx_dma_good_octet_counter_get(self);
++ cs->dpc = hw_atl_rpb_rx_dma_drop_pkt_cnt_get(self);
+
+ memcpy(&priv->last_stats, &stats, sizeof(stats));
+
+@@ -499,9 +563,9 @@ u32 hw_atl2_utils_get_fw_version(struct aq_hw_s *self)
+ hw_atl2_shared_buffer_read_safe(self, version, &version);
+
+ /* A2 FW version is stored in reverse order */
+- return version.mac.major << 24 |
+- version.mac.minor << 16 |
+- version.mac.build;
++ return version.bundle.major << 24 |
++ version.bundle.minor << 16 |
++ version.bundle.build;
+ }
+
+ int hw_atl2_utils_get_action_resolve_table_caps(struct aq_hw_s *self,
+diff --git a/drivers/net/ethernet/arc/Kconfig b/drivers/net/ethernet/arc/Kconfig
+index 92a79c4ffa2c7..0a67612af2281 100644
+--- a/drivers/net/ethernet/arc/Kconfig
++++ b/drivers/net/ethernet/arc/Kconfig
+@@ -26,7 +26,7 @@ config ARC_EMAC_CORE
+ config ARC_EMAC
+ tristate "ARC EMAC support"
+ select ARC_EMAC_CORE
+- depends on OF_IRQ && OF_NET
++ depends on OF_IRQ
+ depends on ARC || COMPILE_TEST
+ help
+ On some legacy ARC (Synopsys) FPGA boards such as ARCAngel4/ML50x
+@@ -36,7 +36,7 @@ config ARC_EMAC
+ config EMAC_ROCKCHIP
+ tristate "Rockchip EMAC support"
+ select ARC_EMAC_CORE
+- depends on OF_IRQ && OF_NET && REGULATOR
++ depends on OF_IRQ && REGULATOR
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
+ help
+ Support for Rockchip RK3036/RK3066/RK3188 EMAC ethernet controllers.
+diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c
+index 02ae98aabf91c..416a5c99db5a2 100644
+--- a/drivers/net/ethernet/atheros/ag71xx.c
++++ b/drivers/net/ethernet/atheros/ag71xx.c
+@@ -1915,15 +1915,12 @@ static int ag71xx_probe(struct platform_device *pdev)
+ ag->mac_reset = devm_reset_control_get(&pdev->dev, "mac");
+ if (IS_ERR(ag->mac_reset)) {
+ netif_err(ag, probe, ndev, "missing mac reset\n");
+- err = PTR_ERR(ag->mac_reset);
+- goto err_free;
++ return PTR_ERR(ag->mac_reset);
+ }
+
+ ag->mac_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+- if (!ag->mac_base) {
+- err = -ENOMEM;
+- goto err_free;
+- }
++ if (!ag->mac_base)
++ return -ENOMEM;
+
+ ndev->irq = platform_get_irq(pdev, 0);
+ err = devm_request_irq(&pdev->dev, ndev->irq, ag71xx_interrupt,
+@@ -1931,7 +1928,7 @@ static int ag71xx_probe(struct platform_device *pdev)
+ if (err) {
+ netif_err(ag, probe, ndev, "unable to request IRQ %d\n",
+ ndev->irq);
+- goto err_free;
++ return err;
+ }
+
+ ndev->netdev_ops = &ag71xx_netdev_ops;
+@@ -1959,10 +1956,8 @@ static int ag71xx_probe(struct platform_device *pdev)
+ ag->stop_desc = dmam_alloc_coherent(&pdev->dev,
+ sizeof(struct ag71xx_desc),
+ &ag->stop_desc_dma, GFP_KERNEL);
+- if (!ag->stop_desc) {
+- err = -ENOMEM;
+- goto err_free;
+- }
++ if (!ag->stop_desc)
++ return -ENOMEM;
+
+ ag->stop_desc->data = 0;
+ ag->stop_desc->ctrl = 0;
+@@ -1977,7 +1972,7 @@ static int ag71xx_probe(struct platform_device *pdev)
+ err = of_get_phy_mode(np, &ag->phy_if_mode);
+ if (err) {
+ netif_err(ag, probe, ndev, "missing phy-mode property in DT\n");
+- goto err_free;
++ return err;
+ }
+
+ netif_napi_add(ndev, &ag->napi, ag71xx_poll, AG71XX_NAPI_WEIGHT);
+@@ -1985,7 +1980,7 @@ static int ag71xx_probe(struct platform_device *pdev)
+ err = clk_prepare_enable(ag->clk_eth);
+ if (err) {
+ netif_err(ag, probe, ndev, "Failed to enable eth clk.\n");
+- goto err_free;
++ return err;
+ }
+
+ ag71xx_wr(ag, AG71XX_REG_MAC_CFG1, 0);
+@@ -2021,8 +2016,6 @@ err_mdio_remove:
+ ag71xx_mdio_remove(ag);
+ err_put_clk:
+ clk_disable_unprepare(ag->clk_eth);
+-err_free:
+- free_netdev(ndev);
+ return err;
+ }
+
+diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
+index 4ea157efca868..98a8698a32017 100644
+--- a/drivers/net/ethernet/atheros/alx/main.c
++++ b/drivers/net/ethernet/atheros/alx/main.c
+@@ -1181,8 +1181,11 @@ static int alx_change_mtu(struct net_device *netdev, int mtu)
+ alx->hw.mtu = mtu;
+ alx->rxbuf_size = max(max_frame, ALX_DEF_RXBUF_SIZE);
+ netdev_update_features(netdev);
+- if (netif_running(netdev))
++ if (netif_running(netdev)) {
++ mutex_lock(&alx->mtx);
+ alx_reinit(alx);
++ mutex_unlock(&alx->mtx);
++ }
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+index 3b51b172b3172..5cbd815c737e7 100644
+--- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
++++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+@@ -900,7 +900,7 @@ static void atl1c_clean_tx_ring(struct atl1c_adapter *adapter,
+ atl1c_clean_buffer(pdev, buffer_info);
+ }
+
+- netdev_reset_queue(adapter->netdev);
++ netdev_tx_reset_queue(netdev_get_tx_queue(adapter->netdev, queue));
+
+ /* Zero out Tx-buffers */
+ memset(tpd_ring->desc, 0, sizeof(struct atl1c_tpd_desc) *
+diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c
+index 02a569500234c..376f81796a293 100644
+--- a/drivers/net/ethernet/broadcom/bcm4908_enet.c
++++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c
+@@ -708,7 +708,9 @@ static int bcm4908_enet_probe(struct platform_device *pdev)
+
+ enet->irq_tx = platform_get_irq_byname(pdev, "tx");
+
+- dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
++ err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
++ if (err)
++ return err;
+
+ err = bcm4908_enet_dma_alloc(enet);
+ if (err)
+diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
+index 7fa1b695400d7..ae541a9d1eeed 100644
+--- a/drivers/net/ethernet/broadcom/bcmsysport.c
++++ b/drivers/net/ethernet/broadcom/bcmsysport.c
+@@ -1309,11 +1309,11 @@ static netdev_tx_t bcm_sysport_xmit(struct sk_buff *skb,
+ struct bcm_sysport_priv *priv = netdev_priv(dev);
+ struct device *kdev = &priv->pdev->dev;
+ struct bcm_sysport_tx_ring *ring;
++ unsigned long flags, desc_flags;
+ struct bcm_sysport_cb *cb;
+ struct netdev_queue *txq;
+ u32 len_status, addr_lo;
+ unsigned int skb_len;
+- unsigned long flags;
+ dma_addr_t mapping;
+ u16 queue;
+ int ret;
+@@ -1373,8 +1373,10 @@ static netdev_tx_t bcm_sysport_xmit(struct sk_buff *skb,
+ ring->desc_count--;
+
+ /* Ports are latched, so write upper address first */
++ spin_lock_irqsave(&priv->desc_lock, desc_flags);
+ tdma_writel(priv, len_status, TDMA_WRITE_PORT_HI(ring->index));
+ tdma_writel(priv, addr_lo, TDMA_WRITE_PORT_LO(ring->index));
++ spin_unlock_irqrestore(&priv->desc_lock, desc_flags);
+
+ /* Check ring space and update SW control flow */
+ if (ring->desc_count == 0)
+@@ -2013,6 +2015,7 @@ static int bcm_sysport_open(struct net_device *dev)
+ }
+
+ /* Initialize both hardware and software ring */
++ spin_lock_init(&priv->desc_lock);
+ for (i = 0; i < dev->num_tx_queues; i++) {
+ ret = bcm_sysport_init_tx_ring(priv, i);
+ if (ret) {
+@@ -2582,8 +2585,10 @@ static int bcm_sysport_probe(struct platform_device *pdev)
+ device_set_wakeup_capable(&pdev->dev, 1);
+
+ priv->wol_clk = devm_clk_get_optional(&pdev->dev, "sw_sysportwol");
+- if (IS_ERR(priv->wol_clk))
+- return PTR_ERR(priv->wol_clk);
++ if (IS_ERR(priv->wol_clk)) {
++ ret = PTR_ERR(priv->wol_clk);
++ goto err_deregister_fixed_link;
++ }
+
+ /* Set the needed headroom once and for all */
+ BUILD_BUG_ON(sizeof(struct bcm_tsb) != 8);
+diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h
+index 984f76e74b43e..16b73bb9acc78 100644
+--- a/drivers/net/ethernet/broadcom/bcmsysport.h
++++ b/drivers/net/ethernet/broadcom/bcmsysport.h
+@@ -711,6 +711,7 @@ struct bcm_sysport_priv {
+ int wol_irq;
+
+ /* Transmit rings */
++ spinlock_t desc_lock;
+ struct bcm_sysport_tx_ring *tx_rings;
+
+ /* Receive queue */
+diff --git a/drivers/net/ethernet/broadcom/bgmac-platform.c b/drivers/net/ethernet/broadcom/bgmac-platform.c
+index df8ff839cc621..94eb3a42158e9 100644
+--- a/drivers/net/ethernet/broadcom/bgmac-platform.c
++++ b/drivers/net/ethernet/broadcom/bgmac-platform.c
+@@ -172,6 +172,7 @@ static int bgmac_probe(struct platform_device *pdev)
+ {
+ struct device_node *np = pdev->dev.of_node;
+ struct bgmac *bgmac;
++ struct resource *regs;
+ int ret;
+
+ bgmac = bgmac_alloc(&pdev->dev);
+@@ -208,15 +209,23 @@ static int bgmac_probe(struct platform_device *pdev)
+ if (IS_ERR(bgmac->plat.base))
+ return PTR_ERR(bgmac->plat.base);
+
+- bgmac->plat.idm_base = devm_platform_ioremap_resource_byname(pdev, "idm_base");
+- if (IS_ERR(bgmac->plat.idm_base))
+- return PTR_ERR(bgmac->plat.idm_base);
+- else
++ /* The idm_base resource is optional for some platforms */
++ regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "idm_base");
++ if (regs) {
++ bgmac->plat.idm_base = devm_ioremap_resource(&pdev->dev, regs);
++ if (IS_ERR(bgmac->plat.idm_base))
++ return PTR_ERR(bgmac->plat.idm_base);
+ bgmac->feature_flags &= ~BGMAC_FEAT_IDM_MASK;
++ }
+
+- bgmac->plat.nicpm_base = devm_platform_ioremap_resource_byname(pdev, "nicpm_base");
+- if (IS_ERR(bgmac->plat.nicpm_base))
+- return PTR_ERR(bgmac->plat.nicpm_base);
++ /* The nicpm_base resource is optional for some platforms */
++ regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nicpm_base");
++ if (regs) {
++ bgmac->plat.nicpm_base = devm_ioremap_resource(&pdev->dev,
++ regs);
++ if (IS_ERR(bgmac->plat.nicpm_base))
++ return PTR_ERR(bgmac->plat.nicpm_base);
++ }
+
+ bgmac->read = platform_bgmac_read;
+ bgmac->write = platform_bgmac_write;
+diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
+index 8c83973adca57..9d70d908c0646 100644
+--- a/drivers/net/ethernet/broadcom/bnx2.c
++++ b/drivers/net/ethernet/broadcom/bnx2.c
+@@ -8212,7 +8212,7 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev)
+ rc = dma_set_coherent_mask(&pdev->dev, persist_dma_mask);
+ if (rc) {
+ dev_err(&pdev->dev,
+- "pci_set_consistent_dma_mask failed, aborting\n");
++ "dma_set_coherent_mask failed, aborting\n");
+ goto err_out_unmap;
+ }
+ } else if ((rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) != 0) {
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
+index e789430f407c3..9e79bcfb365fa 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
+@@ -1850,6 +1850,14 @@ struct bnx2x {
+
+ /* Vxlan/Geneve related information */
+ u16 udp_tunnel_ports[BNX2X_UDP_PORT_MAX];
++
++#define FW_CAP_INVALIDATE_VF_FP_HSI BIT(0)
++ u32 fw_cap;
++
++ u32 fw_major;
++ u32 fw_minor;
++ u32 fw_rev;
++ u32 fw_eng;
+ };
+
+ /* Tx queues may be less or equal to Rx queues */
+@@ -2525,5 +2533,4 @@ void bnx2x_register_phc(struct bnx2x *bp);
+ * Meant for implicit re-load flows.
+ */
+ int bnx2x_vlan_reconfigure_vid(struct bnx2x *bp);
+-
+ #endif /* bnx2x.h */
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+index b5d954cb409ae..198e041d84109 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+@@ -2363,26 +2363,30 @@ int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
+ /* is another pf loaded on this engine? */
+ if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
+ load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
+- /* build my FW version dword */
+- u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
+- (BCM_5710_FW_MINOR_VERSION << 8) +
+- (BCM_5710_FW_REVISION_VERSION << 16) +
+- (BCM_5710_FW_ENGINEERING_VERSION << 24);
++ u8 loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng;
++ u32 loaded_fw;
+
+ /* read loaded FW from chip */
+- u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
++ loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
+
+- DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
+- loaded_fw, my_fw);
++ loaded_fw_major = loaded_fw & 0xff;
++ loaded_fw_minor = (loaded_fw >> 8) & 0xff;
++ loaded_fw_rev = (loaded_fw >> 16) & 0xff;
++ loaded_fw_eng = (loaded_fw >> 24) & 0xff;
++
++ DP(BNX2X_MSG_SP, "loaded fw 0x%x major 0x%x minor 0x%x rev 0x%x eng 0x%x\n",
++ loaded_fw, loaded_fw_major, loaded_fw_minor, loaded_fw_rev, loaded_fw_eng);
+
+ /* abort nic load if version mismatch */
+- if (my_fw != loaded_fw) {
++ if (loaded_fw_major != BCM_5710_FW_MAJOR_VERSION ||
++ loaded_fw_minor != BCM_5710_FW_MINOR_VERSION ||
++ loaded_fw_eng != BCM_5710_FW_ENGINEERING_VERSION ||
++ loaded_fw_rev < BCM_5710_FW_REVISION_VERSION_V15) {
+ if (print_err)
+- BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
+- loaded_fw, my_fw);
++ BNX2X_ERR("loaded FW incompatible. Aborting\n");
+ else
+- BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
+- loaded_fw, my_fw);
++ BNX2X_DEV_INFO("loaded FW incompatible, possibly due to MF UNDI\n");
++
+ return -EBUSY;
+ }
+ }
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_fw_defs.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_fw_defs.h
+index 3f8435208bf49..a84d015da5dfa 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_fw_defs.h
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_fw_defs.h
+@@ -241,6 +241,8 @@
+ IRO[221].m2))
+ #define XSTORM_VF_TO_PF_OFFSET(funcId) \
+ (IRO[48].base + ((funcId) * IRO[48].m1))
++#define XSTORM_ETH_FUNCTION_INFO_FP_HSI_VALID_E2_OFFSET(fid) \
++ (IRO[386].base + ((fid) * IRO[386].m1))
+ #define COMMON_ASM_INVALID_ASSERT_OPCODE 0x0
+
+ /* eth hsi version */
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
+index 622fadc50316e..611efee758340 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
+@@ -3024,7 +3024,8 @@ struct afex_stats {
+
+ #define BCM_5710_FW_MAJOR_VERSION 7
+ #define BCM_5710_FW_MINOR_VERSION 13
+-#define BCM_5710_FW_REVISION_VERSION 15
++#define BCM_5710_FW_REVISION_VERSION 21
++#define BCM_5710_FW_REVISION_VERSION_V15 15
+ #define BCM_5710_FW_ENGINEERING_VERSION 0
+ #define BCM_5710_FW_COMPILE_FLAGS 1
+
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
+index 1835d2e451c01..fc7fce642666c 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
+@@ -635,11 +635,13 @@ static int bnx2x_ilt_client_mem_op(struct bnx2x *bp, int cli_num,
+ {
+ int i, rc;
+ struct bnx2x_ilt *ilt = BP_ILT(bp);
+- struct ilt_client_info *ilt_cli = &ilt->clients[cli_num];
++ struct ilt_client_info *ilt_cli;
+
+ if (!ilt || !ilt->lines)
+ return -1;
+
++ ilt_cli = &ilt->clients[cli_num];
++
+ if (ilt_cli->flags & (ILT_CLIENT_SKIP_INIT | ILT_CLIENT_SKIP_MEM))
+ return 0;
+
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+index ae87296ae1ffa..553f3de939574 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+@@ -74,9 +74,19 @@
+ __stringify(BCM_5710_FW_MINOR_VERSION) "." \
+ __stringify(BCM_5710_FW_REVISION_VERSION) "." \
+ __stringify(BCM_5710_FW_ENGINEERING_VERSION)
++
++#define FW_FILE_VERSION_V15 \
++ __stringify(BCM_5710_FW_MAJOR_VERSION) "." \
++ __stringify(BCM_5710_FW_MINOR_VERSION) "." \
++ __stringify(BCM_5710_FW_REVISION_VERSION_V15) "." \
++ __stringify(BCM_5710_FW_ENGINEERING_VERSION)
++
+ #define FW_FILE_NAME_E1 "bnx2x/bnx2x-e1-" FW_FILE_VERSION ".fw"
+ #define FW_FILE_NAME_E1H "bnx2x/bnx2x-e1h-" FW_FILE_VERSION ".fw"
+ #define FW_FILE_NAME_E2 "bnx2x/bnx2x-e2-" FW_FILE_VERSION ".fw"
++#define FW_FILE_NAME_E1_V15 "bnx2x/bnx2x-e1-" FW_FILE_VERSION_V15 ".fw"
++#define FW_FILE_NAME_E1H_V15 "bnx2x/bnx2x-e1h-" FW_FILE_VERSION_V15 ".fw"
++#define FW_FILE_NAME_E2_V15 "bnx2x/bnx2x-e2-" FW_FILE_VERSION_V15 ".fw"
+
+ /* Time in jiffies before concluding the transmitter is hung */
+ #define TX_TIMEOUT (5*HZ)
+@@ -90,6 +100,9 @@ MODULE_LICENSE("GPL");
+ MODULE_FIRMWARE(FW_FILE_NAME_E1);
+ MODULE_FIRMWARE(FW_FILE_NAME_E1H);
+ MODULE_FIRMWARE(FW_FILE_NAME_E2);
++MODULE_FIRMWARE(FW_FILE_NAME_E1_V15);
++MODULE_FIRMWARE(FW_FILE_NAME_E1H_V15);
++MODULE_FIRMWARE(FW_FILE_NAME_E2_V15);
+
+ int bnx2x_num_queues;
+ module_param_named(num_queues, bnx2x_num_queues, int, 0444);
+@@ -747,9 +760,7 @@ static int bnx2x_mc_assert(struct bnx2x *bp)
+ CHIP_IS_E1(bp) ? "everest1" :
+ CHIP_IS_E1H(bp) ? "everest1h" :
+ CHIP_IS_E2(bp) ? "everest2" : "everest3",
+- BCM_5710_FW_MAJOR_VERSION,
+- BCM_5710_FW_MINOR_VERSION,
+- BCM_5710_FW_REVISION_VERSION);
++ bp->fw_major, bp->fw_minor, bp->fw_rev);
+
+ return rc;
+ }
+@@ -13311,16 +13322,11 @@ static int bnx2x_check_firmware(struct bnx2x *bp)
+ /* Check FW version */
+ offset = be32_to_cpu(fw_hdr->fw_version.offset);
+ fw_ver = firmware->data + offset;
+- if ((fw_ver[0] != BCM_5710_FW_MAJOR_VERSION) ||
+- (fw_ver[1] != BCM_5710_FW_MINOR_VERSION) ||
+- (fw_ver[2] != BCM_5710_FW_REVISION_VERSION) ||
+- (fw_ver[3] != BCM_5710_FW_ENGINEERING_VERSION)) {
++ if (fw_ver[0] != bp->fw_major || fw_ver[1] != bp->fw_minor ||
++ fw_ver[2] != bp->fw_rev || fw_ver[3] != bp->fw_eng) {
+ BNX2X_ERR("Bad FW version:%d.%d.%d.%d. Should be %d.%d.%d.%d\n",
+- fw_ver[0], fw_ver[1], fw_ver[2], fw_ver[3],
+- BCM_5710_FW_MAJOR_VERSION,
+- BCM_5710_FW_MINOR_VERSION,
+- BCM_5710_FW_REVISION_VERSION,
+- BCM_5710_FW_ENGINEERING_VERSION);
++ fw_ver[0], fw_ver[1], fw_ver[2], fw_ver[3],
++ bp->fw_major, bp->fw_minor, bp->fw_rev, bp->fw_eng);
+ return -EINVAL;
+ }
+
+@@ -13400,32 +13406,49 @@ do { \
+
+ static int bnx2x_init_firmware(struct bnx2x *bp)
+ {
+- const char *fw_file_name;
++ const char *fw_file_name, *fw_file_name_v15;
+ struct bnx2x_fw_file_hdr *fw_hdr;
+ int rc;
+
+ if (bp->firmware)
+ return 0;
+
+- if (CHIP_IS_E1(bp))
++ if (CHIP_IS_E1(bp)) {
+ fw_file_name = FW_FILE_NAME_E1;
+- else if (CHIP_IS_E1H(bp))
++ fw_file_name_v15 = FW_FILE_NAME_E1_V15;
++ } else if (CHIP_IS_E1H(bp)) {
+ fw_file_name = FW_FILE_NAME_E1H;
+- else if (!CHIP_IS_E1x(bp))
++ fw_file_name_v15 = FW_FILE_NAME_E1H_V15;
++ } else if (!CHIP_IS_E1x(bp)) {
+ fw_file_name = FW_FILE_NAME_E2;
+- else {
++ fw_file_name_v15 = FW_FILE_NAME_E2_V15;
++ } else {
+ BNX2X_ERR("Unsupported chip revision\n");
+ return -EINVAL;
+ }
++
+ BNX2X_DEV_INFO("Loading %s\n", fw_file_name);
+
+ rc = request_firmware(&bp->firmware, fw_file_name, &bp->pdev->dev);
+ if (rc) {
+- BNX2X_ERR("Can't load firmware file %s\n",
+- fw_file_name);
+- goto request_firmware_exit;
++ BNX2X_DEV_INFO("Trying to load older fw %s\n", fw_file_name_v15);
++
++ /* try to load prev version */
++ rc = request_firmware(&bp->firmware, fw_file_name_v15, &bp->pdev->dev);
++
++ if (rc)
++ goto request_firmware_exit;
++
++ bp->fw_rev = BCM_5710_FW_REVISION_VERSION_V15;
++ } else {
++ bp->fw_cap |= FW_CAP_INVALIDATE_VF_FP_HSI;
++ bp->fw_rev = BCM_5710_FW_REVISION_VERSION;
+ }
+
++ bp->fw_major = BCM_5710_FW_MAJOR_VERSION;
++ bp->fw_minor = BCM_5710_FW_MINOR_VERSION;
++ bp->fw_eng = BCM_5710_FW_ENGINEERING_VERSION;
++
+ rc = bnx2x_check_firmware(bp);
+ if (rc) {
+ BNX2X_ERR("Corrupt firmware file %s\n", fw_file_name);
+@@ -14135,10 +14158,6 @@ static int bnx2x_eeh_nic_unload(struct bnx2x *bp)
+
+ /* Stop Tx */
+ bnx2x_tx_disable(bp);
+- /* Delete all NAPI objects */
+- bnx2x_del_all_napi(bp);
+- if (CNIC_LOADED(bp))
+- bnx2x_del_all_napi_cnic(bp);
+ netdev_reset_tc(bp->dev);
+
+ del_timer_sync(&bp->timer);
+@@ -14243,6 +14262,11 @@ static pci_ers_result_t bnx2x_io_slot_reset(struct pci_dev *pdev)
+ bnx2x_drain_tx_queues(bp);
+ bnx2x_send_unload_req(bp, UNLOAD_RECOVERY);
+ bnx2x_netif_stop(bp, 1);
++ bnx2x_del_all_napi(bp);
++
++ if (CNIC_LOADED(bp))
++ bnx2x_del_all_napi_cnic(bp);
++
+ bnx2x_free_irq(bp);
+
+ /* Report UNLOAD_DONE to MCP */
+diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+index 6fbf735fca31c..5613957314501 100644
+--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
++++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+@@ -758,9 +758,18 @@ static void bnx2x_vf_igu_reset(struct bnx2x *bp, struct bnx2x_virtf *vf)
+
+ void bnx2x_vf_enable_access(struct bnx2x *bp, u8 abs_vfid)
+ {
++ u16 abs_fid;
++
++ abs_fid = FW_VF_HANDLE(abs_vfid);
++
+ /* set the VF-PF association in the FW */
+- storm_memset_vf_to_pf(bp, FW_VF_HANDLE(abs_vfid), BP_FUNC(bp));
+- storm_memset_func_en(bp, FW_VF_HANDLE(abs_vfid), 1);
++ storm_memset_vf_to_pf(bp, abs_fid, BP_FUNC(bp));
++ storm_memset_func_en(bp, abs_fid, 1);
++
++ /* Invalidate fp_hsi version for vfs */
++ if (bp->fw_cap & FW_CAP_INVALIDATE_VF_FP_HSI)
++ REG_WR8(bp, BAR_XSTRORM_INTMEM +
++ XSTORM_ETH_FUNCTION_INFO_FP_HSI_VALID_E2_OFFSET(abs_fid), 0);
+
+ /* clear vf errors*/
+ bnx2x_vf_semi_clear_err(bp, abs_vfid);
+diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile
+index c6ef7ec2c1151..2bc2b707d6eee 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/Makefile
++++ b/drivers/net/ethernet/broadcom/bnxt/Makefile
+@@ -1,6 +1,6 @@
+ # SPDX-License-Identifier: GPL-2.0-only
+ obj-$(CONFIG_BNXT) += bnxt_en.o
+
+-bnxt_en-y := bnxt.o bnxt_hwrm.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_ptp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o
++bnxt_en-y := bnxt.o bnxt_hwrm.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_ptp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_coredump.o
+ bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o
+ bnxt_en-$(CONFIG_DEBUG_FS) += bnxt_debugfs.o
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+index 62f84cc91e4d1..cb53149455890 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+@@ -2699,6 +2699,10 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget)
+ u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
+ struct bnxt_cp_ring_info *cpr2;
+
++ /* No more budget for RX work */
++ if (budget && work_done >= budget && idx == BNXT_RX_HDL)
++ break;
++
+ cpr2 = cpr->cp_ring_arr[idx];
+ work_done += __bnxt_poll_work(bp, cpr2,
+ budget - work_done);
+@@ -3234,6 +3238,7 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp)
+ }
+ qidx = bp->tc_to_qidx[j];
+ ring->queue_id = bp->q_info[qidx].queue_id;
++ spin_lock_init(&txr->xdp_tx_lock);
+ if (i < bp->tx_nr_rings_xdp)
+ continue;
+ if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
+@@ -4757,8 +4762,10 @@ static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
+ return rc;
+
+ req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
+- req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
+- req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
++ if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
++ req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
++ req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
++ }
+ req->mask = cpu_to_le32(vnic->rx_mask);
+ return hwrm_req_send_silent(bp, req);
+ }
+@@ -8004,6 +8011,12 @@ static int bnxt_hwrm_ver_get(struct bnxt *bp)
+ bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
+ if (!bp->hwrm_cmd_timeout)
+ bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
++ bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
++ if (!bp->hwrm_cmd_max_timeout)
++ bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
++ else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
++ netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
++ bp->hwrm_cmd_max_timeout / 1000);
+
+ if (resp->hwrm_intf_maj_8b >= 1) {
+ bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
+@@ -8609,6 +8622,9 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
+ vnic->uc_filter_count = 1;
+
+ vnic->rx_mask = 0;
++ if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
++ goto skip_rx_mask;
++
+ if (bp->dev->flags & IFF_BROADCAST)
+ vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
+
+@@ -8618,7 +8634,7 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
+ if (bp->dev->flags & IFF_ALLMULTI) {
+ vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
+ vnic->mc_list_count = 0;
+- } else {
++ } else if (bp->dev->flags & IFF_MULTICAST) {
+ u32 mask = 0;
+
+ bnxt_mc_list_updated(bp, &mask);
+@@ -8629,6 +8645,7 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
+ if (rc)
+ goto err_out;
+
++skip_rx_mask:
+ rc = bnxt_hwrm_set_coal(bp);
+ if (rc)
+ netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
+@@ -10234,6 +10251,12 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
+ if (irq_re_init)
+ udp_tunnel_nic_reset_ntf(bp->dev);
+
++ if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
++ if (!static_key_enabled(&bnxt_xdp_locking_key))
++ static_branch_enable(&bnxt_xdp_locking_key);
++ } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
++ static_branch_disable(&bnxt_xdp_locking_key);
++ }
+ set_bit(BNXT_STATE_OPEN, &bp->state);
+ bnxt_enable_int(bp);
+ /* Enable TX queues */
+@@ -10289,13 +10312,15 @@ int bnxt_half_open_nic(struct bnxt *bp)
+ goto half_open_err;
+ }
+
+- rc = bnxt_alloc_mem(bp, false);
++ rc = bnxt_alloc_mem(bp, true);
+ if (rc) {
+ netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
+ goto half_open_err;
+ }
+- rc = bnxt_init_nic(bp, false);
++ set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
++ rc = bnxt_init_nic(bp, true);
+ if (rc) {
++ clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
+ netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
+ goto half_open_err;
+ }
+@@ -10303,7 +10328,7 @@ int bnxt_half_open_nic(struct bnxt *bp)
+
+ half_open_err:
+ bnxt_free_skbs(bp);
+- bnxt_free_mem(bp, false);
++ bnxt_free_mem(bp, true);
+ dev_close(bp->dev);
+ return rc;
+ }
+@@ -10313,9 +10338,10 @@ half_open_err:
+ */
+ void bnxt_half_close_nic(struct bnxt *bp)
+ {
+- bnxt_hwrm_resource_free(bp, false, false);
++ bnxt_hwrm_resource_free(bp, false, true);
+ bnxt_free_skbs(bp);
+- bnxt_free_mem(bp, false);
++ bnxt_free_mem(bp, true);
++ clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
+ }
+
+ static void bnxt_reenable_sriov(struct bnxt *bp)
+@@ -10731,7 +10757,7 @@ static void bnxt_set_rx_mode(struct net_device *dev)
+ if (dev->flags & IFF_ALLMULTI) {
+ mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
+ vnic->mc_list_count = 0;
+- } else {
++ } else if (dev->flags & IFF_MULTICAST) {
+ mc_update = bnxt_mc_list_updated(bp, &mask);
+ }
+
+@@ -10799,9 +10825,10 @@ skip_uc:
+ !bnxt_promisc_ok(bp))
+ vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
+ rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
+- if (rc && vnic->mc_list_count) {
++ if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
+ netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
+ rc);
++ vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
+ vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
+ vnic->mc_list_count = 0;
+ rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
+@@ -10858,7 +10885,7 @@ static bool bnxt_rfs_capable(struct bnxt *bp)
+
+ if (bp->flags & BNXT_FLAG_CHIP_P5)
+ return bnxt_rfs_supported(bp);
+- if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
++ if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
+ return false;
+
+ vnics = 1 + bp->rx_nr_rings;
+@@ -13064,10 +13091,9 @@ static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
+ goto init_dflt_ring_err;
+
+ bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+- if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
+- bp->flags |= BNXT_FLAG_RFS;
+- bp->dev->features |= NETIF_F_NTUPLE;
+- }
++
++ bnxt_set_dflt_rfs(bp);
++
+ init_dflt_ring_err:
+ bnxt_ulp_irq_restart(bp, rc);
+ return rc;
+@@ -13370,7 +13396,9 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+ }
+
+ bnxt_inv_fw_health_reg(bp);
+- bnxt_dl_register(bp);
++ rc = bnxt_dl_register(bp);
++ if (rc)
++ goto init_err_dl;
+
+ rc = register_netdev(dev);
+ if (rc)
+@@ -13390,6 +13418,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ init_err_cleanup:
+ bnxt_dl_unregister(bp);
++init_err_dl:
+ bnxt_shutdown_tc(bp);
+ bnxt_clear_int_mode(bp);
+
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+index 19fe6478e9b4b..e5874c829226e 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+@@ -584,7 +584,8 @@ struct nqe_cn {
+ #define BNXT_MAX_MTU 9500
+ #define BNXT_MAX_PAGE_MODE_MTU \
+ ((unsigned int)PAGE_SIZE - VLAN_ETH_HLEN - NET_IP_ALIGN - \
+- XDP_PACKET_HEADROOM)
++ XDP_PACKET_HEADROOM - \
++ SKB_DATA_ALIGN((unsigned int)sizeof(struct skb_shared_info)))
+
+ #define BNXT_MIN_PKT_SIZE 52
+
+@@ -791,6 +792,8 @@ struct bnxt_tx_ring_info {
+ u32 dev_state;
+
+ struct bnxt_ring_struct tx_ring_struct;
++ /* Synchronize simultaneous xdp_xmit on same ring */
++ spinlock_t xdp_tx_lock;
+ };
+
+ #define BNXT_LEGACY_COAL_CMPL_PARAMS \
+@@ -1840,6 +1843,7 @@ struct bnxt {
+ #define BNXT_STATE_DRV_REGISTERED 7
+ #define BNXT_STATE_PCI_CHANNEL_IO_FROZEN 8
+ #define BNXT_STATE_NAPI_DISABLED 9
++#define BNXT_STATE_HALF_OPEN 15 /* For offline ethtool tests */
+
+ #define BNXT_NO_FW_ACCESS(bp) \
+ (test_bit(BNXT_STATE_FW_FATAL_COND, &(bp)->state) || \
+@@ -1901,7 +1905,8 @@ struct bnxt {
+
+ u16 hwrm_max_req_len;
+ u16 hwrm_max_ext_req_len;
+- int hwrm_cmd_timeout;
++ unsigned int hwrm_cmd_timeout;
++ unsigned int hwrm_cmd_max_timeout;
+ struct mutex hwrm_cmd_lock; /* serialize hwrm messages */
+ struct hwrm_ver_get_output ver_resp;
+ #define FW_VER_STR_LEN 32
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c
+new file mode 100644
+index 0000000000000..156f76bcea7eb
+--- /dev/null
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.c
+@@ -0,0 +1,372 @@
++/* Broadcom NetXtreme-C/E network driver.
++ *
++ * Copyright (c) 2021 Broadcom Limited
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation.
++ */
++
++#include <linux/types.h>
++#include <linux/errno.h>
++#include <linux/pci.h>
++#include "bnxt_hsi.h"
++#include "bnxt.h"
++#include "bnxt_hwrm.h"
++#include "bnxt_coredump.h"
++
++static int bnxt_hwrm_dbg_dma_data(struct bnxt *bp, void *msg,
++ struct bnxt_hwrm_dbg_dma_info *info)
++{
++ struct hwrm_dbg_cmn_input *cmn_req = msg;
++ __le16 *seq_ptr = msg + info->seq_off;
++ struct hwrm_dbg_cmn_output *cmn_resp;
++ u16 seq = 0, len, segs_off;
++ dma_addr_t dma_handle;
++ void *dma_buf, *resp;
++ int rc, off = 0;
++
++ dma_buf = hwrm_req_dma_slice(bp, msg, info->dma_len, &dma_handle);
++ if (!dma_buf) {
++ hwrm_req_drop(bp, msg);
++ return -ENOMEM;
++ }
++
++ hwrm_req_timeout(bp, msg, bp->hwrm_cmd_max_timeout);
++ cmn_resp = hwrm_req_hold(bp, msg);
++ resp = cmn_resp;
++
++ segs_off = offsetof(struct hwrm_dbg_coredump_list_output,
++ total_segments);
++ cmn_req->host_dest_addr = cpu_to_le64(dma_handle);
++ cmn_req->host_buf_len = cpu_to_le32(info->dma_len);
++ while (1) {
++ *seq_ptr = cpu_to_le16(seq);
++ rc = hwrm_req_send(bp, msg);
++ if (rc)
++ break;
++
++ len = le16_to_cpu(*((__le16 *)(resp + info->data_len_off)));
++ if (!seq &&
++ cmn_req->req_type == cpu_to_le16(HWRM_DBG_COREDUMP_LIST)) {
++ info->segs = le16_to_cpu(*((__le16 *)(resp +
++ segs_off)));
++ if (!info->segs) {
++ rc = -EIO;
++ break;
++ }
++
++ info->dest_buf_size = info->segs *
++ sizeof(struct coredump_segment_record);
++ info->dest_buf = kmalloc(info->dest_buf_size,
++ GFP_KERNEL);
++ if (!info->dest_buf) {
++ rc = -ENOMEM;
++ break;
++ }
++ }
++
++ if (info->dest_buf) {
++ if ((info->seg_start + off + len) <=
++ BNXT_COREDUMP_BUF_LEN(info->buf_len)) {
++ memcpy(info->dest_buf + off, dma_buf, len);
++ } else {
++ rc = -ENOBUFS;
++ break;
++ }
++ }
++
++ if (cmn_req->req_type ==
++ cpu_to_le16(HWRM_DBG_COREDUMP_RETRIEVE))
++ info->dest_buf_size += len;
++
++ if (!(cmn_resp->flags & HWRM_DBG_CMN_FLAGS_MORE))
++ break;
++
++ seq++;
++ off += len;
++ }
++ hwrm_req_drop(bp, msg);
++ return rc;
++}
++
++static int bnxt_hwrm_dbg_coredump_list(struct bnxt *bp,
++ struct bnxt_coredump *coredump)
++{
++ struct bnxt_hwrm_dbg_dma_info info = {NULL};
++ struct hwrm_dbg_coredump_list_input *req;
++ int rc;
++
++ rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_LIST);
++ if (rc)
++ return rc;
++
++ info.dma_len = COREDUMP_LIST_BUF_LEN;
++ info.seq_off = offsetof(struct hwrm_dbg_coredump_list_input, seq_no);
++ info.data_len_off = offsetof(struct hwrm_dbg_coredump_list_output,
++ data_len);
++
++ rc = bnxt_hwrm_dbg_dma_data(bp, req, &info);
++ if (!rc) {
++ coredump->data = info.dest_buf;
++ coredump->data_size = info.dest_buf_size;
++ coredump->total_segs = info.segs;
++ }
++ return rc;
++}
++
++static int bnxt_hwrm_dbg_coredump_initiate(struct bnxt *bp, u16 component_id,
++ u16 segment_id)
++{
++ struct hwrm_dbg_coredump_initiate_input *req;
++ int rc;
++
++ rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_INITIATE);
++ if (rc)
++ return rc;
++
++ hwrm_req_timeout(bp, req, bp->hwrm_cmd_max_timeout);
++ req->component_id = cpu_to_le16(component_id);
++ req->segment_id = cpu_to_le16(segment_id);
++
++ return hwrm_req_send(bp, req);
++}
++
++static int bnxt_hwrm_dbg_coredump_retrieve(struct bnxt *bp, u16 component_id,
++ u16 segment_id, u32 *seg_len,
++ void *buf, u32 buf_len, u32 offset)
++{
++ struct hwrm_dbg_coredump_retrieve_input *req;
++ struct bnxt_hwrm_dbg_dma_info info = {NULL};
++ int rc;
++
++ rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_RETRIEVE);
++ if (rc)
++ return rc;
++
++ req->component_id = cpu_to_le16(component_id);
++ req->segment_id = cpu_to_le16(segment_id);
++
++ info.dma_len = COREDUMP_RETRIEVE_BUF_LEN;
++ info.seq_off = offsetof(struct hwrm_dbg_coredump_retrieve_input,
++ seq_no);
++ info.data_len_off = offsetof(struct hwrm_dbg_coredump_retrieve_output,
++ data_len);
++ if (buf) {
++ info.dest_buf = buf + offset;
++ info.buf_len = buf_len;
++ info.seg_start = offset;
++ }
++
++ rc = bnxt_hwrm_dbg_dma_data(bp, req, &info);
++ if (!rc)
++ *seg_len = info.dest_buf_size;
++
++ return rc;
++}
++
++static void
++bnxt_fill_coredump_seg_hdr(struct bnxt *bp,
++ struct bnxt_coredump_segment_hdr *seg_hdr,
++ struct coredump_segment_record *seg_rec, u32 seg_len,
++ int status, u32 duration, u32 instance)
++{
++ memset(seg_hdr, 0, sizeof(*seg_hdr));
++ memcpy(seg_hdr->signature, "sEgM", 4);
++ if (seg_rec) {
++ seg_hdr->component_id = (__force __le32)seg_rec->component_id;
++ seg_hdr->segment_id = (__force __le32)seg_rec->segment_id;
++ seg_hdr->low_version = seg_rec->version_low;
++ seg_hdr->high_version = seg_rec->version_hi;
++ } else {
++ /* For hwrm_ver_get response Component id = 2
++ * and Segment id = 0
++ */
++ seg_hdr->component_id = cpu_to_le32(2);
++ seg_hdr->segment_id = 0;
++ }
++ seg_hdr->function_id = cpu_to_le16(bp->pdev->devfn);
++ seg_hdr->length = cpu_to_le32(seg_len);
++ seg_hdr->status = cpu_to_le32(status);
++ seg_hdr->duration = cpu_to_le32(duration);
++ seg_hdr->data_offset = cpu_to_le32(sizeof(*seg_hdr));
++ seg_hdr->instance = cpu_to_le32(instance);
++}
++
++static void
++bnxt_fill_coredump_record(struct bnxt *bp, struct bnxt_coredump_record *record,
++ time64_t start, s16 start_utc, u16 total_segs,
++ int status)
++{
++ time64_t end = ktime_get_real_seconds();
++ u32 os_ver_major = 0, os_ver_minor = 0;
++ struct tm tm;
++
++ time64_to_tm(start, 0, &tm);
++ memset(record, 0, sizeof(*record));
++ memcpy(record->signature, "cOrE", 4);
++ record->flags = 0;
++ record->low_version = 0;
++ record->high_version = 1;
++ record->asic_state = 0;
++ strscpy(record->system_name, utsname()->nodename,
++ sizeof(record->system_name));
++ record->year = cpu_to_le16(tm.tm_year + 1900);
++ record->month = cpu_to_le16(tm.tm_mon + 1);
++ record->day = cpu_to_le16(tm.tm_mday);
++ record->hour = cpu_to_le16(tm.tm_hour);
++ record->minute = cpu_to_le16(tm.tm_min);
++ record->second = cpu_to_le16(tm.tm_sec);
++ record->utc_bias = cpu_to_le16(start_utc);
++ strcpy(record->commandline, "ethtool -w");
++ record->total_segments = cpu_to_le32(total_segs);
++
++ if (sscanf(utsname()->release, "%u.%u", &os_ver_major, &os_ver_minor) != 2)
++ netdev_warn(bp->dev, "Unknown OS release in coredump\n");
++ record->os_ver_major = cpu_to_le32(os_ver_major);
++ record->os_ver_minor = cpu_to_le32(os_ver_minor);
++
++ strscpy(record->os_name, utsname()->sysname, sizeof(record->os_name));
++ time64_to_tm(end, 0, &tm);
++ record->end_year = cpu_to_le16(tm.tm_year + 1900);
++ record->end_month = cpu_to_le16(tm.tm_mon + 1);
++ record->end_day = cpu_to_le16(tm.tm_mday);
++ record->end_hour = cpu_to_le16(tm.tm_hour);
++ record->end_minute = cpu_to_le16(tm.tm_min);
++ record->end_second = cpu_to_le16(tm.tm_sec);
++ record->end_utc_bias = cpu_to_le16(sys_tz.tz_minuteswest * 60);
++ record->asic_id1 = cpu_to_le32(bp->chip_num << 16 |
++ bp->ver_resp.chip_rev << 8 |
++ bp->ver_resp.chip_metal);
++ record->asic_id2 = 0;
++ record->coredump_status = cpu_to_le32(status);
++ record->ioctl_low_version = 0;
++ record->ioctl_high_version = 0;
++}
++
++static int __bnxt_get_coredump(struct bnxt *bp, void *buf, u32 *dump_len)
++{
++ u32 ver_get_resp_len = sizeof(struct hwrm_ver_get_output);
++ u32 offset = 0, seg_hdr_len, seg_record_len, buf_len = 0;
++ struct coredump_segment_record *seg_record = NULL;
++ struct bnxt_coredump_segment_hdr seg_hdr;
++ struct bnxt_coredump coredump = {NULL};
++ time64_t start_time;
++ u16 start_utc;
++ int rc = 0, i;
++
++ if (buf)
++ buf_len = *dump_len;
++
++ start_time = ktime_get_real_seconds();
++ start_utc = sys_tz.tz_minuteswest * 60;
++ seg_hdr_len = sizeof(seg_hdr);
++
++ /* First segment should be hwrm_ver_get response */
++ *dump_len = seg_hdr_len + ver_get_resp_len;
++ if (buf) {
++ bnxt_fill_coredump_seg_hdr(bp, &seg_hdr, NULL, ver_get_resp_len,
++ 0, 0, 0);
++ memcpy(buf + offset, &seg_hdr, seg_hdr_len);
++ offset += seg_hdr_len;
++ memcpy(buf + offset, &bp->ver_resp, ver_get_resp_len);
++ offset += ver_get_resp_len;
++ }
++
++ rc = bnxt_hwrm_dbg_coredump_list(bp, &coredump);
++ if (rc) {
++ netdev_err(bp->dev, "Failed to get coredump segment list\n");
++ goto err;
++ }
++
++ *dump_len += seg_hdr_len * coredump.total_segs;
++
++ seg_record = (struct coredump_segment_record *)coredump.data;
++ seg_record_len = sizeof(*seg_record);
++
++ for (i = 0; i < coredump.total_segs; i++) {
++ u16 comp_id = le16_to_cpu(seg_record->component_id);
++ u16 seg_id = le16_to_cpu(seg_record->segment_id);
++ u32 duration = 0, seg_len = 0;
++ unsigned long start, end;
++
++ if (buf && ((offset + seg_hdr_len) >
++ BNXT_COREDUMP_BUF_LEN(buf_len))) {
++ rc = -ENOBUFS;
++ goto err;
++ }
++
++ start = jiffies;
++
++ rc = bnxt_hwrm_dbg_coredump_initiate(bp, comp_id, seg_id);
++ if (rc) {
++ netdev_err(bp->dev,
++ "Failed to initiate coredump for seg = %d\n",
++ seg_record->segment_id);
++ goto next_seg;
++ }
++
++ /* Write segment data into the buffer */
++ rc = bnxt_hwrm_dbg_coredump_retrieve(bp, comp_id, seg_id,
++ &seg_len, buf, buf_len,
++ offset + seg_hdr_len);
++ if (rc && rc == -ENOBUFS)
++ goto err;
++ else if (rc)
++ netdev_err(bp->dev,
++ "Failed to retrieve coredump for seg = %d\n",
++ seg_record->segment_id);
++
++next_seg:
++ end = jiffies;
++ duration = jiffies_to_msecs(end - start);
++ bnxt_fill_coredump_seg_hdr(bp, &seg_hdr, seg_record, seg_len,
++ rc, duration, 0);
++
++ if (buf) {
++ /* Write segment header into the buffer */
++ memcpy(buf + offset, &seg_hdr, seg_hdr_len);
++ offset += seg_hdr_len + seg_len;
++ }
++
++ *dump_len += seg_len;
++ seg_record =
++ (struct coredump_segment_record *)((u8 *)seg_record +
++ seg_record_len);
++ }
++
++err:
++ if (buf)
++ bnxt_fill_coredump_record(bp, buf + offset, start_time,
++ start_utc, coredump.total_segs + 1,
++ rc);
++ kfree(coredump.data);
++ *dump_len += sizeof(struct bnxt_coredump_record);
++ if (rc == -ENOBUFS)
++ netdev_err(bp->dev, "Firmware returned large coredump buffer\n");
++ return rc;
++}
++
++int bnxt_get_coredump(struct bnxt *bp, u16 dump_type, void *buf, u32 *dump_len)
++{
++ if (dump_type == BNXT_DUMP_CRASH) {
++#ifdef CONFIG_TEE_BNXT_FW
++ return tee_bnxt_copy_coredump(buf, 0, *dump_len);
++#else
++ return -EOPNOTSUPP;
++#endif
++ } else {
++ return __bnxt_get_coredump(bp, buf, dump_len);
++ }
++}
++
++u32 bnxt_get_coredump_length(struct bnxt *bp, u16 dump_type)
++{
++ u32 len = 0;
++
++ if (dump_type == BNXT_DUMP_CRASH)
++ len = BNXT_CRASH_DUMP_LEN;
++ else
++ __bnxt_get_coredump(bp, NULL, &len);
++ return len;
++}
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.h
+index 09c22f8fe3991..b1a1b2fffb194 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_coredump.h
+@@ -10,6 +10,10 @@
+ #ifndef BNXT_COREDUMP_H
+ #define BNXT_COREDUMP_H
+
++#include <linux/utsname.h>
++#include <linux/time.h>
++#include <linux/rtc.h>
++
+ struct bnxt_coredump_segment_hdr {
+ __u8 signature[4];
+ __le32 component_id;
+@@ -63,4 +67,51 @@ struct bnxt_coredump_record {
+ __u8 ioctl_high_version;
+ __le16 rsvd3[313];
+ };
++
++#define BNXT_CRASH_DUMP_LEN (8 << 20)
++
++#define COREDUMP_LIST_BUF_LEN 2048
++#define COREDUMP_RETRIEVE_BUF_LEN 4096
++
++struct bnxt_coredump {
++ void *data;
++ int data_size;
++ u16 total_segs;
++};
++
++#define BNXT_COREDUMP_BUF_LEN(len) ((len) - sizeof(struct bnxt_coredump_record))
++
++struct bnxt_hwrm_dbg_dma_info {
++ void *dest_buf;
++ int dest_buf_size;
++ u16 dma_len;
++ u16 seq_off;
++ u16 data_len_off;
++ u16 segs;
++ u32 seg_start;
++ u32 buf_len;
++};
++
++struct hwrm_dbg_cmn_input {
++ __le16 req_type;
++ __le16 cmpl_ring;
++ __le16 seq_id;
++ __le16 target_id;
++ __le64 resp_addr;
++ __le64 host_dest_addr;
++ __le32 host_buf_len;
++};
++
++struct hwrm_dbg_cmn_output {
++ __le16 error_code;
++ __le16 req_type;
++ __le16 seq_id;
++ __le16 resp_len;
++ u8 flags;
++ #define HWRM_DBG_CMN_FLAGS_MORE 1
++};
++
++int bnxt_get_coredump(struct bnxt *bp, u16 dump_type, void *buf, u32 *dump_len);
++u32 bnxt_get_coredump_length(struct bnxt *bp, u16 dump_type);
++
+ #endif
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c
+index 9576547df4aba..2a80882971e3d 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c
+@@ -134,7 +134,7 @@ void bnxt_dl_fw_reporters_create(struct bnxt *bp)
+ {
+ struct bnxt_fw_health *health = bp->fw_health;
+
+- if (!bp->dl || !health)
++ if (!health)
+ return;
+
+ if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) || health->fw_reset_reporter)
+@@ -188,7 +188,7 @@ void bnxt_dl_fw_reporters_destroy(struct bnxt *bp, bool all)
+ {
+ struct bnxt_fw_health *health = bp->fw_health;
+
+- if (!bp->dl || !health)
++ if (!health)
+ return;
+
+ if ((all || !(bp->fw_cap & BNXT_FW_CAP_HOT_RESET)) &&
+@@ -781,6 +781,7 @@ int bnxt_dl_register(struct bnxt *bp)
+ {
+ const struct devlink_ops *devlink_ops;
+ struct devlink_port_attrs attrs = {};
++ struct bnxt_dl *bp_dl;
+ struct devlink *dl;
+ int rc;
+
+@@ -795,7 +796,9 @@ int bnxt_dl_register(struct bnxt *bp)
+ return -ENOMEM;
+ }
+
+- bnxt_link_bp_to_dl(bp, dl);
++ bp->dl = dl;
++ bp_dl = devlink_priv(dl);
++ bp_dl->bp = bp;
+
+ /* Add switchdev eswitch mode setting, if SRIOV supported */
+ if (pci_find_ext_capability(bp->pdev, PCI_EXT_CAP_ID_SRIOV) &&
+@@ -833,7 +836,6 @@ err_dl_port_unreg:
+ err_dl_unreg:
+ devlink_unregister(dl);
+ err_dl_free:
+- bnxt_link_bp_to_dl(bp, NULL);
+ devlink_free(dl);
+ return rc;
+ }
+@@ -842,9 +844,6 @@ void bnxt_dl_unregister(struct bnxt *bp)
+ {
+ struct devlink *dl = bp->dl;
+
+- if (!dl)
+- return;
+-
+ if (BNXT_PF(bp)) {
+ bnxt_dl_params_unregister(bp);
+ devlink_port_unregister(&bp->dl_port);
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.h
+index d889f240da2b2..406dc655a5fc9 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.h
+@@ -20,19 +20,6 @@ static inline struct bnxt *bnxt_get_bp_from_dl(struct devlink *dl)
+ return ((struct bnxt_dl *)devlink_priv(dl))->bp;
+ }
+
+-/* To clear devlink pointer from bp, pass NULL dl */
+-static inline void bnxt_link_bp_to_dl(struct bnxt *bp, struct devlink *dl)
+-{
+- bp->dl = dl;
+-
+- /* add a back pointer in dl to bp */
+- if (dl) {
+- struct bnxt_dl *bp_dl = devlink_priv(dl);
+-
+- bp_dl->bp = bp;
+- }
+-}
+-
+ #define NVM_OFF_MSIX_VEC_PER_PF_MAX 108
+ #define NVM_OFF_MSIX_VEC_PER_PF_MIN 114
+ #define NVM_OFF_IGNORE_ARI 164
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+index 7260910e75fb2..0f276ce2d1eb7 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+@@ -25,15 +25,13 @@
+ #include "bnxt_hsi.h"
+ #include "bnxt.h"
+ #include "bnxt_hwrm.h"
++#include "bnxt_ulp.h"
+ #include "bnxt_xdp.h"
+ #include "bnxt_ptp.h"
+ #include "bnxt_ethtool.h"
+ #include "bnxt_nvm_defs.h" /* NVRAM content constant and structure defs */
+ #include "bnxt_fw_hdr.h" /* Firmware hdr constant and structure defs */
+ #include "bnxt_coredump.h"
+-#define FLASH_NVRAM_TIMEOUT ((HWRM_CMD_TIMEOUT) * 100)
+-#define FLASH_PACKAGE_TIMEOUT ((HWRM_CMD_TIMEOUT) * 200)
+-#define INSTALL_PACKAGE_TIMEOUT ((HWRM_CMD_TIMEOUT) * 200)
+
+ static u32 bnxt_get_msglevel(struct net_device *dev)
+ {
+@@ -1945,6 +1943,9 @@ static int bnxt_get_fecparam(struct net_device *dev,
+ case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
+ fec->active_fec |= ETHTOOL_FEC_LLRS;
+ break;
++ case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
++ fec->active_fec |= ETHTOOL_FEC_OFF;
++ break;
+ }
+ return 0;
+ }
+@@ -2073,9 +2074,7 @@ static int bnxt_set_pauseparam(struct net_device *dev,
+ }
+
+ link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
+- if (bp->hwrm_spec_code >= 0x10201)
+- link_info->req_flow_ctrl =
+- PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
++ link_info->req_flow_ctrl = 0;
+ } else {
+ /* when transition from auto pause to force pause,
+ * force a link change
+@@ -2167,7 +2166,7 @@ static int bnxt_flash_nvram(struct net_device *dev, u16 dir_type,
+ req->host_src_addr = cpu_to_le64(dma_handle);
+ }
+
+- hwrm_req_timeout(bp, req, FLASH_NVRAM_TIMEOUT);
++ hwrm_req_timeout(bp, req, bp->hwrm_cmd_max_timeout);
+ req->dir_type = cpu_to_le16(dir_type);
+ req->dir_ordinal = cpu_to_le16(dir_ordinal);
+ req->dir_ext = cpu_to_le16(dir_ext);
+@@ -2508,8 +2507,8 @@ int bnxt_flash_package_from_fw_obj(struct net_device *dev, const struct firmware
+ return rc;
+ }
+
+- hwrm_req_timeout(bp, modify, FLASH_PACKAGE_TIMEOUT);
+- hwrm_req_timeout(bp, install, INSTALL_PACKAGE_TIMEOUT);
++ hwrm_req_timeout(bp, modify, bp->hwrm_cmd_max_timeout);
++ hwrm_req_timeout(bp, install, bp->hwrm_cmd_max_timeout);
+
+ hwrm_req_hold(bp, modify);
+ modify->host_src_addr = cpu_to_le64(dma_handle);
+@@ -3408,7 +3407,7 @@ static int bnxt_run_loopback(struct bnxt *bp)
+ if (!skb)
+ return -ENOMEM;
+ data = skb_put(skb, pkt_size);
+- eth_broadcast_addr(data);
++ ether_addr_copy(&data[i], bp->dev->dev_addr);
+ i += ETH_ALEN;
+ ether_addr_copy(&data[i], bp->dev->dev_addr);
+ i += ETH_ALEN;
+@@ -3502,9 +3501,12 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest,
+ if (!offline) {
+ bnxt_run_fw_tests(bp, test_mask, &test_results);
+ } else {
+- rc = bnxt_close_nic(bp, false, false);
+- if (rc)
++ bnxt_ulp_stop(bp);
++ rc = bnxt_close_nic(bp, true, false);
++ if (rc) {
++ bnxt_ulp_start(bp, rc);
+ return;
++ }
+ bnxt_run_fw_tests(bp, test_mask, &test_results);
+
+ buf[BNXT_MACLPBK_TEST_IDX] = 1;
+@@ -3514,6 +3516,7 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest,
+ if (rc) {
+ bnxt_hwrm_mac_loopback(bp, false);
+ etest->flags |= ETH_TEST_FL_FAILED;
++ bnxt_ulp_start(bp, rc);
+ return;
+ }
+ if (bnxt_run_loopback(bp))
+@@ -3539,7 +3542,8 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest,
+ }
+ bnxt_hwrm_phy_loopback(bp, false, false);
+ bnxt_half_close_nic(bp);
+- rc = bnxt_open_nic(bp, false, true);
++ rc = bnxt_open_nic(bp, true, true);
++ bnxt_ulp_start(bp, rc);
+ }
+ if (rc || bnxt_test_irq(bp)) {
+ buf[BNXT_IRQ_TEST_IDX] = 1;
+@@ -3609,337 +3613,6 @@ static int bnxt_reset(struct net_device *dev, u32 *flags)
+ return 0;
+ }
+
+-static int bnxt_hwrm_dbg_dma_data(struct bnxt *bp, void *msg,
+- struct bnxt_hwrm_dbg_dma_info *info)
+-{
+- struct hwrm_dbg_cmn_input *cmn_req = msg;
+- __le16 *seq_ptr = msg + info->seq_off;
+- struct hwrm_dbg_cmn_output *cmn_resp;
+- u16 seq = 0, len, segs_off;
+- dma_addr_t dma_handle;
+- void *dma_buf, *resp;
+- int rc, off = 0;
+-
+- dma_buf = hwrm_req_dma_slice(bp, msg, info->dma_len, &dma_handle);
+- if (!dma_buf) {
+- hwrm_req_drop(bp, msg);
+- return -ENOMEM;
+- }
+-
+- hwrm_req_timeout(bp, msg, HWRM_COREDUMP_TIMEOUT);
+- cmn_resp = hwrm_req_hold(bp, msg);
+- resp = cmn_resp;
+-
+- segs_off = offsetof(struct hwrm_dbg_coredump_list_output,
+- total_segments);
+- cmn_req->host_dest_addr = cpu_to_le64(dma_handle);
+- cmn_req->host_buf_len = cpu_to_le32(info->dma_len);
+- while (1) {
+- *seq_ptr = cpu_to_le16(seq);
+- rc = hwrm_req_send(bp, msg);
+- if (rc)
+- break;
+-
+- len = le16_to_cpu(*((__le16 *)(resp + info->data_len_off)));
+- if (!seq &&
+- cmn_req->req_type == cpu_to_le16(HWRM_DBG_COREDUMP_LIST)) {
+- info->segs = le16_to_cpu(*((__le16 *)(resp +
+- segs_off)));
+- if (!info->segs) {
+- rc = -EIO;
+- break;
+- }
+-
+- info->dest_buf_size = info->segs *
+- sizeof(struct coredump_segment_record);
+- info->dest_buf = kmalloc(info->dest_buf_size,
+- GFP_KERNEL);
+- if (!info->dest_buf) {
+- rc = -ENOMEM;
+- break;
+- }
+- }
+-
+- if (info->dest_buf) {
+- if ((info->seg_start + off + len) <=
+- BNXT_COREDUMP_BUF_LEN(info->buf_len)) {
+- memcpy(info->dest_buf + off, dma_buf, len);
+- } else {
+- rc = -ENOBUFS;
+- break;
+- }
+- }
+-
+- if (cmn_req->req_type ==
+- cpu_to_le16(HWRM_DBG_COREDUMP_RETRIEVE))
+- info->dest_buf_size += len;
+-
+- if (!(cmn_resp->flags & HWRM_DBG_CMN_FLAGS_MORE))
+- break;
+-
+- seq++;
+- off += len;
+- }
+- hwrm_req_drop(bp, msg);
+- return rc;
+-}
+-
+-static int bnxt_hwrm_dbg_coredump_list(struct bnxt *bp,
+- struct bnxt_coredump *coredump)
+-{
+- struct bnxt_hwrm_dbg_dma_info info = {NULL};
+- struct hwrm_dbg_coredump_list_input *req;
+- int rc;
+-
+- rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_LIST);
+- if (rc)
+- return rc;
+-
+- info.dma_len = COREDUMP_LIST_BUF_LEN;
+- info.seq_off = offsetof(struct hwrm_dbg_coredump_list_input, seq_no);
+- info.data_len_off = offsetof(struct hwrm_dbg_coredump_list_output,
+- data_len);
+-
+- rc = bnxt_hwrm_dbg_dma_data(bp, req, &info);
+- if (!rc) {
+- coredump->data = info.dest_buf;
+- coredump->data_size = info.dest_buf_size;
+- coredump->total_segs = info.segs;
+- }
+- return rc;
+-}
+-
+-static int bnxt_hwrm_dbg_coredump_initiate(struct bnxt *bp, u16 component_id,
+- u16 segment_id)
+-{
+- struct hwrm_dbg_coredump_initiate_input *req;
+- int rc;
+-
+- rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_INITIATE);
+- if (rc)
+- return rc;
+-
+- hwrm_req_timeout(bp, req, HWRM_COREDUMP_TIMEOUT);
+- req->component_id = cpu_to_le16(component_id);
+- req->segment_id = cpu_to_le16(segment_id);
+-
+- return hwrm_req_send(bp, req);
+-}
+-
+-static int bnxt_hwrm_dbg_coredump_retrieve(struct bnxt *bp, u16 component_id,
+- u16 segment_id, u32 *seg_len,
+- void *buf, u32 buf_len, u32 offset)
+-{
+- struct hwrm_dbg_coredump_retrieve_input *req;
+- struct bnxt_hwrm_dbg_dma_info info = {NULL};
+- int rc;
+-
+- rc = hwrm_req_init(bp, req, HWRM_DBG_COREDUMP_RETRIEVE);
+- if (rc)
+- return rc;
+-
+- req->component_id = cpu_to_le16(component_id);
+- req->segment_id = cpu_to_le16(segment_id);
+-
+- info.dma_len = COREDUMP_RETRIEVE_BUF_LEN;
+- info.seq_off = offsetof(struct hwrm_dbg_coredump_retrieve_input,
+- seq_no);
+- info.data_len_off = offsetof(struct hwrm_dbg_coredump_retrieve_output,
+- data_len);
+- if (buf) {
+- info.dest_buf = buf + offset;
+- info.buf_len = buf_len;
+- info.seg_start = offset;
+- }
+-
+- rc = bnxt_hwrm_dbg_dma_data(bp, req, &info);
+- if (!rc)
+- *seg_len = info.dest_buf_size;
+-
+- return rc;
+-}
+-
+-static void
+-bnxt_fill_coredump_seg_hdr(struct bnxt *bp,
+- struct bnxt_coredump_segment_hdr *seg_hdr,
+- struct coredump_segment_record *seg_rec, u32 seg_len,
+- int status, u32 duration, u32 instance)
+-{
+- memset(seg_hdr, 0, sizeof(*seg_hdr));
+- memcpy(seg_hdr->signature, "sEgM", 4);
+- if (seg_rec) {
+- seg_hdr->component_id = (__force __le32)seg_rec->component_id;
+- seg_hdr->segment_id = (__force __le32)seg_rec->segment_id;
+- seg_hdr->low_version = seg_rec->version_low;
+- seg_hdr->high_version = seg_rec->version_hi;
+- } else {
+- /* For hwrm_ver_get response Component id = 2
+- * and Segment id = 0
+- */
+- seg_hdr->component_id = cpu_to_le32(2);
+- seg_hdr->segment_id = 0;
+- }
+- seg_hdr->function_id = cpu_to_le16(bp->pdev->devfn);
+- seg_hdr->length = cpu_to_le32(seg_len);
+- seg_hdr->status = cpu_to_le32(status);
+- seg_hdr->duration = cpu_to_le32(duration);
+- seg_hdr->data_offset = cpu_to_le32(sizeof(*seg_hdr));
+- seg_hdr->instance = cpu_to_le32(instance);
+-}
+-
+-static void
+-bnxt_fill_coredump_record(struct bnxt *bp, struct bnxt_coredump_record *record,
+- time64_t start, s16 start_utc, u16 total_segs,
+- int status)
+-{
+- time64_t end = ktime_get_real_seconds();
+- u32 os_ver_major = 0, os_ver_minor = 0;
+- struct tm tm;
+-
+- time64_to_tm(start, 0, &tm);
+- memset(record, 0, sizeof(*record));
+- memcpy(record->signature, "cOrE", 4);
+- record->flags = 0;
+- record->low_version = 0;
+- record->high_version = 1;
+- record->asic_state = 0;
+- strlcpy(record->system_name, utsname()->nodename,
+- sizeof(record->system_name));
+- record->year = cpu_to_le16(tm.tm_year + 1900);
+- record->month = cpu_to_le16(tm.tm_mon + 1);
+- record->day = cpu_to_le16(tm.tm_mday);
+- record->hour = cpu_to_le16(tm.tm_hour);
+- record->minute = cpu_to_le16(tm.tm_min);
+- record->second = cpu_to_le16(tm.tm_sec);
+- record->utc_bias = cpu_to_le16(start_utc);
+- strcpy(record->commandline, "ethtool -w");
+- record->total_segments = cpu_to_le32(total_segs);
+-
+- sscanf(utsname()->release, "%u.%u", &os_ver_major, &os_ver_minor);
+- record->os_ver_major = cpu_to_le32(os_ver_major);
+- record->os_ver_minor = cpu_to_le32(os_ver_minor);
+-
+- strlcpy(record->os_name, utsname()->sysname, 32);
+- time64_to_tm(end, 0, &tm);
+- record->end_year = cpu_to_le16(tm.tm_year + 1900);
+- record->end_month = cpu_to_le16(tm.tm_mon + 1);
+- record->end_day = cpu_to_le16(tm.tm_mday);
+- record->end_hour = cpu_to_le16(tm.tm_hour);
+- record->end_minute = cpu_to_le16(tm.tm_min);
+- record->end_second = cpu_to_le16(tm.tm_sec);
+- record->end_utc_bias = cpu_to_le16(sys_tz.tz_minuteswest * 60);
+- record->asic_id1 = cpu_to_le32(bp->chip_num << 16 |
+- bp->ver_resp.chip_rev << 8 |
+- bp->ver_resp.chip_metal);
+- record->asic_id2 = 0;
+- record->coredump_status = cpu_to_le32(status);
+- record->ioctl_low_version = 0;
+- record->ioctl_high_version = 0;
+-}
+-
+-static int bnxt_get_coredump(struct bnxt *bp, void *buf, u32 *dump_len)
+-{
+- u32 ver_get_resp_len = sizeof(struct hwrm_ver_get_output);
+- u32 offset = 0, seg_hdr_len, seg_record_len, buf_len = 0;
+- struct coredump_segment_record *seg_record = NULL;
+- struct bnxt_coredump_segment_hdr seg_hdr;
+- struct bnxt_coredump coredump = {NULL};
+- time64_t start_time;
+- u16 start_utc;
+- int rc = 0, i;
+-
+- if (buf)
+- buf_len = *dump_len;
+-
+- start_time = ktime_get_real_seconds();
+- start_utc = sys_tz.tz_minuteswest * 60;
+- seg_hdr_len = sizeof(seg_hdr);
+-
+- /* First segment should be hwrm_ver_get response */
+- *dump_len = seg_hdr_len + ver_get_resp_len;
+- if (buf) {
+- bnxt_fill_coredump_seg_hdr(bp, &seg_hdr, NULL, ver_get_resp_len,
+- 0, 0, 0);
+- memcpy(buf + offset, &seg_hdr, seg_hdr_len);
+- offset += seg_hdr_len;
+- memcpy(buf + offset, &bp->ver_resp, ver_get_resp_len);
+- offset += ver_get_resp_len;
+- }
+-
+- rc = bnxt_hwrm_dbg_coredump_list(bp, &coredump);
+- if (rc) {
+- netdev_err(bp->dev, "Failed to get coredump segment list\n");
+- goto err;
+- }
+-
+- *dump_len += seg_hdr_len * coredump.total_segs;
+-
+- seg_record = (struct coredump_segment_record *)coredump.data;
+- seg_record_len = sizeof(*seg_record);
+-
+- for (i = 0; i < coredump.total_segs; i++) {
+- u16 comp_id = le16_to_cpu(seg_record->component_id);
+- u16 seg_id = le16_to_cpu(seg_record->segment_id);
+- u32 duration = 0, seg_len = 0;
+- unsigned long start, end;
+-
+- if (buf && ((offset + seg_hdr_len) >
+- BNXT_COREDUMP_BUF_LEN(buf_len))) {
+- rc = -ENOBUFS;
+- goto err;
+- }
+-
+- start = jiffies;
+-
+- rc = bnxt_hwrm_dbg_coredump_initiate(bp, comp_id, seg_id);
+- if (rc) {
+- netdev_err(bp->dev,
+- "Failed to initiate coredump for seg = %d\n",
+- seg_record->segment_id);
+- goto next_seg;
+- }
+-
+- /* Write segment data into the buffer */
+- rc = bnxt_hwrm_dbg_coredump_retrieve(bp, comp_id, seg_id,
+- &seg_len, buf, buf_len,
+- offset + seg_hdr_len);
+- if (rc && rc == -ENOBUFS)
+- goto err;
+- else if (rc)
+- netdev_err(bp->dev,
+- "Failed to retrieve coredump for seg = %d\n",
+- seg_record->segment_id);
+-
+-next_seg:
+- end = jiffies;
+- duration = jiffies_to_msecs(end - start);
+- bnxt_fill_coredump_seg_hdr(bp, &seg_hdr, seg_record, seg_len,
+- rc, duration, 0);
+-
+- if (buf) {
+- /* Write segment header into the buffer */
+- memcpy(buf + offset, &seg_hdr, seg_hdr_len);
+- offset += seg_hdr_len + seg_len;
+- }
+-
+- *dump_len += seg_len;
+- seg_record =
+- (struct coredump_segment_record *)((u8 *)seg_record +
+- seg_record_len);
+- }
+-
+-err:
+- if (buf)
+- bnxt_fill_coredump_record(bp, buf + offset, start_time,
+- start_utc, coredump.total_segs + 1,
+- rc);
+- kfree(coredump.data);
+- *dump_len += sizeof(struct bnxt_coredump_record);
+- if (rc == -ENOBUFS)
+- netdev_err(bp->dev, "Firmware returned large coredump buffer\n");
+- return rc;
+-}
+-
+ static int bnxt_set_dump(struct net_device *dev, struct ethtool_dump *dump)
+ {
+ struct bnxt *bp = netdev_priv(dev);
+@@ -3971,10 +3644,7 @@ static int bnxt_get_dump_flag(struct net_device *dev, struct ethtool_dump *dump)
+ bp->ver_resp.hwrm_fw_rsvd_8b;
+
+ dump->flag = bp->dump_flag;
+- if (bp->dump_flag == BNXT_DUMP_CRASH)
+- dump->len = BNXT_CRASH_DUMP_LEN;
+- else
+- bnxt_get_coredump(bp, NULL, &dump->len);
++ dump->len = bnxt_get_coredump_length(bp, bp->dump_flag);
+ return 0;
+ }
+
+@@ -3989,15 +3659,7 @@ static int bnxt_get_dump_data(struct net_device *dev, struct ethtool_dump *dump,
+ memset(buf, 0, dump->len);
+
+ dump->flag = bp->dump_flag;
+- if (dump->flag == BNXT_DUMP_CRASH) {
+-#ifdef CONFIG_TEE_BNXT_FW
+- return tee_bnxt_copy_coredump(buf, 0, dump->len);
+-#endif
+- } else {
+- return bnxt_get_coredump(bp, buf, &dump->len);
+- }
+-
+- return 0;
++ return bnxt_get_coredump(bp, dump->flag, buf, &dump->len);
+ }
+
+ static int bnxt_get_ts_info(struct net_device *dev,
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
+index 0a57cb6a4a4bf..11a719f98defd 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
+@@ -22,49 +22,6 @@ struct bnxt_led_cfg {
+ u8 rsvd;
+ };
+
+-#define COREDUMP_LIST_BUF_LEN 2048
+-#define COREDUMP_RETRIEVE_BUF_LEN 4096
+-
+-struct bnxt_coredump {
+- void *data;
+- int data_size;
+- u16 total_segs;
+-};
+-
+-#define BNXT_COREDUMP_BUF_LEN(len) ((len) - sizeof(struct bnxt_coredump_record))
+-
+-struct bnxt_hwrm_dbg_dma_info {
+- void *dest_buf;
+- int dest_buf_size;
+- u16 dma_len;
+- u16 seq_off;
+- u16 data_len_off;
+- u16 segs;
+- u32 seg_start;
+- u32 buf_len;
+-};
+-
+-struct hwrm_dbg_cmn_input {
+- __le16 req_type;
+- __le16 cmpl_ring;
+- __le16 seq_id;
+- __le16 target_id;
+- __le64 resp_addr;
+- __le64 host_dest_addr;
+- __le32 host_buf_len;
+-};
+-
+-struct hwrm_dbg_cmn_output {
+- __le16 error_code;
+- __le16 req_type;
+- __le16 seq_id;
+- __le16 resp_len;
+- u8 flags;
+- #define HWRM_DBG_CMN_FLAGS_MORE 1
+-};
+-
+-#define BNXT_CRASH_DUMP_LEN (8 << 20)
+-
+ #define BNXT_LED_DFLT_ENA \
+ (PORT_LED_CFG_REQ_ENABLES_LED0_ID | \
+ PORT_LED_CFG_REQ_ENABLES_LED0_STATE | \
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c
+index bb7327b82d0b2..3a0eeb3737767 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c
+@@ -496,7 +496,7 @@ static int __hwrm_send(struct bnxt *bp, struct bnxt_hwrm_ctx *ctx)
+ }
+
+ /* Limit timeout to an upper limit */
+- timeout = min_t(uint, ctx->timeout, HWRM_CMD_MAX_TIMEOUT);
++ timeout = min(ctx->timeout, bp->hwrm_cmd_max_timeout ?: HWRM_CMD_MAX_TIMEOUT);
+ /* convert timeout to usec */
+ timeout *= 1000;
+
+@@ -595,18 +595,24 @@ timeout_abort:
+
+ /* Last byte of resp contains valid bit */
+ valid = ((u8 *)ctx->resp) + len - 1;
+- for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
++ for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; ) {
+ /* make sure we read from updated DMA memory */
+ dma_rmb();
+ if (*valid)
+ break;
+- usleep_range(1, 5);
++ if (j < 10) {
++ udelay(1);
++ j++;
++ } else {
++ usleep_range(20, 30);
++ j += 20;
++ }
+ }
+
+ if (j >= HWRM_VALID_BIT_DELAY_USEC) {
+ if (!(ctx->flags & BNXT_HWRM_CTX_SILENT))
+ netdev_err(bp->dev, "Error (timeout: %u) msg {0x%x 0x%x} len:%d v:%d\n",
+- hwrm_total_timeout(i),
++ hwrm_total_timeout(i) + j,
+ le16_to_cpu(ctx->req->req_type),
+ le16_to_cpu(ctx->req->seq_id), len,
+ *valid);
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.h
+index 4d17f0d5363bb..380ef69afb51b 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.h
+@@ -58,11 +58,10 @@ void hwrm_update_token(struct bnxt *bp, u16 seq, enum bnxt_hwrm_wait_state s);
+
+ #define BNXT_HWRM_MAX_REQ_LEN (bp->hwrm_max_req_len)
+ #define BNXT_HWRM_SHORT_REQ_LEN sizeof(struct hwrm_short_input)
+-#define HWRM_CMD_MAX_TIMEOUT 40000
++#define HWRM_CMD_MAX_TIMEOUT 40000U
+ #define SHORT_HWRM_CMD_TIMEOUT 20
+ #define HWRM_CMD_TIMEOUT (bp->hwrm_cmd_timeout)
+ #define HWRM_RESET_TIMEOUT ((HWRM_CMD_TIMEOUT) * 4)
+-#define HWRM_COREDUMP_TIMEOUT ((HWRM_CMD_TIMEOUT) * 12)
+ #define BNXT_HWRM_TARGET 0xffff
+ #define BNXT_HWRM_NO_CMPL_RING -1
+ #define BNXT_HWRM_REQ_MAX_SIZE 128
+@@ -95,7 +94,7 @@ static inline unsigned int hwrm_total_timeout(unsigned int n)
+ }
+
+
+-#define HWRM_VALID_BIT_DELAY_USEC 150
++#define HWRM_VALID_BIT_DELAY_USEC 50000
+
+ static inline bool bnxt_cfa_hwrm_message(u16 req_type)
+ {
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
+index f0aa480799ca4..62a931de5b1a7 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
+@@ -331,7 +331,7 @@ static int bnxt_ptp_enable(struct ptp_clock_info *ptp_info,
+ struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg,
+ ptp_info);
+ struct bnxt *bp = ptp->bp;
+- u8 pin_id;
++ int pin_id;
+ int rc;
+
+ switch (rq->type) {
+@@ -339,6 +339,8 @@ static int bnxt_ptp_enable(struct ptp_clock_info *ptp_info,
+ /* Configure an External PPS IN */
+ pin_id = ptp_find_pin(ptp->ptp_clock, PTP_PF_EXTTS,
+ rq->extts.index);
++ if (!TSIO_PIN_VALID(pin_id))
++ return -EOPNOTSUPP;
+ if (!on)
+ break;
+ rc = bnxt_ptp_cfg_pin(bp, pin_id, BNXT_PPS_PIN_PPS_IN);
+@@ -352,6 +354,8 @@ static int bnxt_ptp_enable(struct ptp_clock_info *ptp_info,
+ /* Configure a Periodic PPS OUT */
+ pin_id = ptp_find_pin(ptp->ptp_clock, PTP_PF_PEROUT,
+ rq->perout.index);
++ if (!TSIO_PIN_VALID(pin_id))
++ return -EOPNOTSUPP;
+ if (!on)
+ break;
+
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h
+index fa5f05708e6df..c3cd51e672e7b 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h
+@@ -28,7 +28,7 @@ struct pps_pin {
+ u8 state;
+ };
+
+-#define TSIO_PIN_VALID(pin) ((pin) < (BNXT_MAX_TSIO_PINS))
++#define TSIO_PIN_VALID(pin) ((pin) >= 0 && (pin) < (BNXT_MAX_TSIO_PINS))
+
+ #define EVENT_DATA2_PPS_EVENT_TYPE(data2) \
+ ((data2) & ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE)
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
+index e6a4a768b10b2..1471b6130a2b9 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
+@@ -1868,7 +1868,7 @@ static int bnxt_tc_setup_indr_block_cb(enum tc_setup_type type,
+ struct flow_cls_offload *flower = type_data;
+ struct bnxt *bp = priv->bp;
+
+- if (flower->common.chain_index)
++ if (!tc_cls_can_offload_and_chain0(bp->dev, type_data))
+ return -EOPNOTSUPP;
+
+ switch (type) {
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+index c8083df5e0ab8..148b58f3468b3 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+@@ -20,6 +20,8 @@
+ #include "bnxt.h"
+ #include "bnxt_xdp.h"
+
++DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
++
+ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
+ struct bnxt_tx_ring_info *txr,
+ dma_addr_t mapping, u32 len)
+@@ -227,11 +229,16 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
+ ring = smp_processor_id() % bp->tx_nr_rings_xdp;
+ txr = &bp->tx_ring[ring];
+
++ if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
++ return -EINVAL;
++
++ if (static_branch_unlikely(&bnxt_xdp_locking_key))
++ spin_lock(&txr->xdp_tx_lock);
++
+ for (i = 0; i < num_frames; i++) {
+ struct xdp_frame *xdp = frames[i];
+
+- if (!txr || !bnxt_tx_avail(bp, txr) ||
+- !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP))
++ if (!bnxt_tx_avail(bp, txr))
+ break;
+
+ mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
+@@ -250,6 +257,9 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
+ bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
+ }
+
++ if (static_branch_unlikely(&bnxt_xdp_locking_key))
++ spin_unlock(&txr->xdp_tx_lock);
++
+ return nxmit;
+ }
+
+diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
+index 0df40c3beb050..067bb5e821f54 100644
+--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
++++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h
+@@ -10,6 +10,8 @@
+ #ifndef BNXT_XDP_H
+ #define BNXT_XDP_H
+
++DECLARE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
++
+ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
+ struct bnxt_tx_ring_info *txr,
+ dma_addr_t mapping, u32 len);
+diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+index 23c7595d2a1d3..ea13917537526 100644
+--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
++++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+@@ -1991,6 +1991,11 @@ static struct sk_buff *bcmgenet_add_tsb(struct net_device *dev,
+ return skb;
+ }
+
++static void bcmgenet_hide_tsb(struct sk_buff *skb)
++{
++ __skb_pull(skb, sizeof(struct status_64));
++}
++
+ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
+ {
+ struct bcmgenet_priv *priv = netdev_priv(dev);
+@@ -2097,6 +2102,8 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
+ }
+
+ GENET_CB(skb)->last_cb = tx_cb_ptr;
++
++ bcmgenet_hide_tsb(skb);
+ skb_tx_timestamp(skb);
+
+ /* Decrement total BD count and advance our write pointer */
+@@ -2243,8 +2250,10 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
+ dma_length_status = status->length_status;
+ if (dev->features & NETIF_F_RXCSUM) {
+ rx_csum = (__force __be16)(status->rx_csum & 0xffff);
+- skb->csum = (__force __wsum)ntohs(rx_csum);
+- skb->ip_summed = CHECKSUM_COMPLETE;
++ if (rx_csum) {
++ skb->csum = (__force __wsum)ntohs(rx_csum);
++ skb->ip_summed = CHECKSUM_COMPLETE;
++ }
+ }
+
+ /* DMA flags and length are still valid no matter how
+@@ -3941,6 +3950,10 @@ static int bcmgenet_probe(struct platform_device *pdev)
+ goto err;
+ }
+ priv->wol_irq = platform_get_irq_optional(pdev, 2);
++ if (priv->wol_irq == -EPROBE_DEFER) {
++ err = priv->wol_irq;
++ goto err;
++ }
+
+ priv->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(priv->base)) {
+@@ -3966,10 +3979,12 @@ static int bcmgenet_probe(struct platform_device *pdev)
+
+ /* Request the WOL interrupt and advertise suspend if available */
+ priv->wol_irq_disabled = true;
+- err = devm_request_irq(&pdev->dev, priv->wol_irq, bcmgenet_wol_isr, 0,
+- dev->name, priv);
+- if (!err)
+- device_set_wakeup_capable(&pdev->dev, 1);
++ if (priv->wol_irq > 0) {
++ err = devm_request_irq(&pdev->dev, priv->wol_irq,
++ bcmgenet_wol_isr, 0, dev->name, priv);
++ if (!err)
++ device_set_wakeup_capable(&pdev->dev, 1);
++ }
+
+ /* Set the needed headroom to account for any possible
+ * features enabling/disabling at runtime
+diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
+index e31a5a397f114..f55d9d9c01a85 100644
+--- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
++++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
+@@ -40,6 +40,13 @@
+ void bcmgenet_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
+ {
+ struct bcmgenet_priv *priv = netdev_priv(dev);
++ struct device *kdev = &priv->pdev->dev;
++
++ if (!device_can_wakeup(kdev)) {
++ wol->supported = 0;
++ wol->wolopts = 0;
++ return;
++ }
+
+ wol->supported = WAKE_MAGIC | WAKE_MAGICSECURE | WAKE_FILTER;
+ wol->wolopts = priv->wolopts;
+diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
+index d13fb1d318215..2fd3dd4b8b815 100644
+--- a/drivers/net/ethernet/cadence/macb_main.c
++++ b/drivers/net/ethernet/cadence/macb_main.c
+@@ -1250,7 +1250,6 @@ static void gem_rx_refill(struct macb_queue *queue)
+ /* Make hw descriptor updates visible to CPU */
+ rmb();
+
+- queue->rx_prepared_head++;
+ desc = macb_rx_desc(queue, entry);
+
+ if (!queue->rx_skbuff[entry]) {
+@@ -1289,6 +1288,7 @@ static void gem_rx_refill(struct macb_queue *queue)
+ dma_wmb();
+ desc->addr &= ~MACB_BIT(RX_USED);
+ }
++ queue->rx_prepared_head++;
+ }
+
+ /* Make descriptor updates visible to hardware */
+@@ -1606,7 +1606,14 @@ static int macb_poll(struct napi_struct *napi, int budget)
+ if (work_done < budget) {
+ napi_complete_done(napi, work_done);
+
+- /* Packets received while interrupts were disabled */
++ /* RSR bits only seem to propagate to raise interrupts when
++ * interrupts are enabled at the time, so if bits are already
++ * set due to packets received while interrupts were disabled,
++ * they will not cause another interrupt to be generated when
++ * interrupts are re-enabled.
++ * Check for this case here. This has been seen to happen
++ * around 30% of the time under heavy network load.
++ */
+ status = macb_readl(bp, RSR);
+ if (status) {
+ if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
+@@ -1614,6 +1621,22 @@ static int macb_poll(struct napi_struct *napi, int budget)
+ napi_reschedule(napi);
+ } else {
+ queue_writel(queue, IER, bp->rx_intr_mask);
++
++ /* In rare cases, packets could have been received in
++ * the window between the check above and re-enabling
++ * interrupts. Therefore, a double-check is required
++ * to avoid losing a wakeup. This can potentially race
++ * with the interrupt handler doing the same actions
++ * if an interrupt is raised just after enabling them,
++ * but this should be harmless.
++ */
++ status = macb_readl(bp, RSR);
++ if (unlikely(status)) {
++ queue_writel(queue, IDR, bp->rx_intr_mask);
++ if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
++ queue_writel(queue, ISR, MACB_BIT(RCOMP));
++ napi_schedule(napi);
++ }
+ }
+ }
+
+@@ -1666,6 +1689,7 @@ static void macb_tx_restart(struct macb_queue *queue)
+ unsigned int head = queue->tx_head;
+ unsigned int tail = queue->tx_tail;
+ struct macb *bp = queue->bp;
++ unsigned int head_idx, tbqp;
+
+ if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
+ queue_writel(queue, ISR, MACB_BIT(TXUBR));
+@@ -1673,6 +1697,13 @@ static void macb_tx_restart(struct macb_queue *queue)
+ if (head == tail)
+ return;
+
++ tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp);
++ tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp));
++ head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, head));
++
++ if (tbqp == head_idx)
++ return;
++
+ macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
+ }
+
+@@ -4739,7 +4770,7 @@ static int macb_probe(struct platform_device *pdev)
+
+ #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+ if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {
+- dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
++ dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));
+ bp->hw_dma_cap |= HW_DMA_CAP_64B;
+ }
+ #endif
+diff --git a/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c b/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c
+index 7ff31d1026fb2..e0d34e64fc6cb 100644
+--- a/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c
++++ b/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c
+@@ -3678,6 +3678,8 @@ int t3_prep_adapter(struct adapter *adapter, const struct adapter_info *ai,
+ MAC_STATS_ACCUM_SECS : (MAC_STATS_ACCUM_SECS * 10);
+ adapter->params.pci.vpd_cap_addr =
+ pci_find_capability(adapter->pdev, PCI_CAP_ID_VPD);
++ if (!adapter->params.pci.vpd_cap_addr)
++ return -ENODEV;
+ ret = get_vpd_params(adapter, &adapter->params.vpd);
+ if (ret < 0)
+ return ret;
+diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+index 5903bdb78916f..129352bbe1143 100644
+--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
++++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+@@ -2015,12 +2015,15 @@ static int cxgb4_get_module_info(struct net_device *dev,
+ if (ret)
+ return ret;
+
+- if (!sff8472_comp || (sff_diag_type & 4)) {
++ if (!sff8472_comp || (sff_diag_type & SFP_DIAG_ADDRMODE)) {
+ modinfo->type = ETH_MODULE_SFF_8079;
+ modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+ } else {
+ modinfo->type = ETH_MODULE_SFF_8472;
+- modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
++ if (sff_diag_type & SFP_DIAG_IMPLEMENTED)
++ modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
++ else
++ modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN / 2;
+ }
+ break;
+
+diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+index 64144b6171d72..b1c9f65ab10f6 100644
+--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
++++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+@@ -2793,14 +2793,14 @@ int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
+ goto out;
+ na = ret;
+
+- memcpy(p->id, vpd + id, min_t(int, id_len, ID_LEN));
++ memcpy(p->id, vpd + id, min_t(unsigned int, id_len, ID_LEN));
+ strim(p->id);
+- memcpy(p->sn, vpd + sn, min_t(int, sn_len, SERNUM_LEN));
++ memcpy(p->sn, vpd + sn, min_t(unsigned int, sn_len, SERNUM_LEN));
+ strim(p->sn);
+- memcpy(p->pn, vpd + pn, min_t(int, pn_len, PN_LEN));
++ memcpy(p->pn, vpd + pn, min_t(unsigned int, pn_len, PN_LEN));
+ strim(p->pn);
+- memcpy(p->na, vpd + na, min_t(int, na_len, MACADDR_LEN));
+- strim((char *)p->na);
++ memcpy(p->na, vpd + na, min_t(unsigned int, na_len, MACADDR_LEN));
++ strim(p->na);
+
+ out:
+ vfree(vpd);
+diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
+index 002fc62ea7262..63bc956d20376 100644
+--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
++++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
+@@ -293,6 +293,8 @@ enum {
+ #define I2C_PAGE_SIZE 0x100
+ #define SFP_DIAG_TYPE_ADDR 0x5c
+ #define SFP_DIAG_TYPE_LEN 0x1
++#define SFP_DIAG_ADDRMODE BIT(2)
++#define SFP_DIAG_IMPLEMENTED BIT(6)
+ #define SFF_8472_COMP_ADDR 0x5e
+ #define SFF_8472_COMP_LEN 0x1
+ #define SFF_REV_ADDR 0x1
+diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+index bcad69c480740..4af5561cbfc54 100644
+--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
++++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+@@ -870,7 +870,7 @@ static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
+ * created only after 3 way handshake is done.
+ */
+ sock_orphan(child);
+- percpu_counter_inc((child)->sk_prot->orphan_count);
++ INC_ORPHAN_COUNT(child);
+ chtls_release_resources(child);
+ chtls_conn_done(child);
+ } else {
+diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
+index b1161bdeda4dc..f61ca657601ca 100644
+--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
++++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
+@@ -95,7 +95,7 @@ struct deferred_skb_cb {
+ #define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok)
+ #define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok)
+ #define SACK_OK(tp) ((tp)->rx_opt.sack_ok)
+-#define INC_ORPHAN_COUNT(sk) percpu_counter_inc((sk)->sk_prot->orphan_count)
++#define INC_ORPHAN_COUNT(sk) this_cpu_inc(*(sk)->sk_prot->orphan_count)
+
+ /* TLS SKB */
+ #define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld)
+diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c
+index d04a6c1634452..da8d10475a08e 100644
+--- a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c
++++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c
+@@ -32,6 +32,7 @@
+
+ #include <linux/tcp.h>
+ #include <linux/ipv6.h>
++#include <net/inet_ecn.h>
+ #include <net/route.h>
+ #include <net/ip6_route.h>
+
+@@ -99,7 +100,7 @@ cxgb_find_route(struct cxgb4_lld_info *lldi,
+
+ rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
+ peer_port, local_port, IPPROTO_TCP,
+- tos, 0);
++ tos & ~INET_ECN_MASK, 0);
+ if (IS_ERR(rt))
+ return NULL;
+ n = dst_neigh_lookup(&rt->dst, &peer_ip);
+diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c
+index 6e745ca4c4333..012ca11a38cc1 100644
+--- a/drivers/net/ethernet/cortina/gemini.c
++++ b/drivers/net/ethernet/cortina/gemini.c
+@@ -305,21 +305,21 @@ static void gmac_speed_set(struct net_device *netdev)
+ switch (phydev->speed) {
+ case 1000:
+ status.bits.speed = GMAC_SPEED_1000;
+- if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
++ if (phy_interface_mode_is_rgmii(phydev->interface))
+ status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
+ netdev_dbg(netdev, "connect %s to RGMII @ 1Gbit\n",
+ phydev_name(phydev));
+ break;
+ case 100:
+ status.bits.speed = GMAC_SPEED_100;
+- if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
++ if (phy_interface_mode_is_rgmii(phydev->interface))
+ status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+ netdev_dbg(netdev, "connect %s to RGMII @ 100 Mbit\n",
+ phydev_name(phydev));
+ break;
+ case 10:
+ status.bits.speed = GMAC_SPEED_10;
+- if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
++ if (phy_interface_mode_is_rgmii(phydev->interface))
+ status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+ netdev_dbg(netdev, "connect %s to RGMII @ 10 Mbit\n",
+ phydev_name(phydev));
+@@ -389,6 +389,9 @@ static int gmac_setup_phy(struct net_device *netdev)
+ status.bits.mii_rmii = GMAC_PHY_GMII;
+ break;
+ case PHY_INTERFACE_MODE_RGMII:
++ case PHY_INTERFACE_MODE_RGMII_ID:
++ case PHY_INTERFACE_MODE_RGMII_TXID:
++ case PHY_INTERFACE_MODE_RGMII_RXID:
+ netdev_dbg(netdev,
+ "RGMII: set GMAC0 and GMAC1 to MII/RGMII mode\n");
+ status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+diff --git a/drivers/net/ethernet/dec/tulip/de4x5.c b/drivers/net/ethernet/dec/tulip/de4x5.c
+index 36ab4cbf2ad08..b9d967e419387 100644
+--- a/drivers/net/ethernet/dec/tulip/de4x5.c
++++ b/drivers/net/ethernet/dec/tulip/de4x5.c
+@@ -4708,6 +4708,10 @@ type3_infoblock(struct net_device *dev, u_char count, u_char *p)
+ lp->ibn = 3;
+ lp->active = *p++;
+ if (MOTO_SROM_BUG) lp->active = 0;
++ /* if (MOTO_SROM_BUG) statement indicates lp->active could
++ * be 8 (i.e. the size of array lp->phy) */
++ if (WARN_ON(lp->active >= ARRAY_SIZE(lp->phy)))
++ return -EINVAL;
+ lp->phy[lp->active].gep = (*p ? p : NULL); p += (2 * (*p) + 1);
+ lp->phy[lp->active].rst = (*p ? p : NULL); p += (2 * (*p) + 1);
+ lp->phy[lp->active].mc = get_unaligned_le16(p); p += 2;
+@@ -4999,19 +5003,23 @@ mii_get_phy(struct net_device *dev)
+ }
+ if ((j == limit) && (i < DE4X5_MAX_MII)) {
+ for (k=0; k < DE4X5_MAX_PHY && lp->phy[k].id; k++);
+- lp->phy[k].addr = i;
+- lp->phy[k].id = id;
+- lp->phy[k].spd.reg = GENERIC_REG; /* ANLPA register */
+- lp->phy[k].spd.mask = GENERIC_MASK; /* 100Mb/s technologies */
+- lp->phy[k].spd.value = GENERIC_VALUE; /* TX & T4, H/F Duplex */
+- lp->mii_cnt++;
+- lp->active++;
+- printk("%s: Using generic MII device control. If the board doesn't operate,\nplease mail the following dump to the author:\n", dev->name);
+- j = de4x5_debug;
+- de4x5_debug |= DEBUG_MII;
+- de4x5_dbg_mii(dev, k);
+- de4x5_debug = j;
+- printk("\n");
++ if (k < DE4X5_MAX_PHY) {
++ lp->phy[k].addr = i;
++ lp->phy[k].id = id;
++ lp->phy[k].spd.reg = GENERIC_REG; /* ANLPA register */
++ lp->phy[k].spd.mask = GENERIC_MASK; /* 100Mb/s technologies */
++ lp->phy[k].spd.value = GENERIC_VALUE; /* TX & T4, H/F Duplex */
++ lp->mii_cnt++;
++ lp->active++;
++ printk("%s: Using generic MII device control. If the board doesn't operate,\nplease mail the following dump to the author:\n", dev->name);
++ j = de4x5_debug;
++ de4x5_debug |= DEBUG_MII;
++ de4x5_dbg_mii(dev, k);
++ de4x5_debug = j;
++ printk("\n");
++ } else {
++ goto purgatory;
++ }
+ }
+ }
+ purgatory:
+diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c
+index fcedd733bacbf..834a3f8c80da0 100644
+--- a/drivers/net/ethernet/dec/tulip/tulip_core.c
++++ b/drivers/net/ethernet/dec/tulip/tulip_core.c
+@@ -1398,8 +1398,10 @@ static int tulip_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ /* alloc_etherdev ensures aligned and zeroed private structures */
+ dev = alloc_etherdev (sizeof (*tp));
+- if (!dev)
++ if (!dev) {
++ pci_disable_device(pdev);
+ return -ENOMEM;
++ }
+
+ SET_NETDEV_DEV(dev, &pdev->dev);
+ if (pci_resource_len (pdev, 0) < tulip_tbl[chip_idx].io_size) {
+@@ -1778,6 +1780,7 @@ err_out_free_res:
+
+ err_out_free_netdev:
+ free_netdev (dev);
++ pci_disable_device(pdev);
+ return -ENODEV;
+ }
+
+diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c
+index 85b99099c6b94..5babcf05bc2f1 100644
+--- a/drivers/net/ethernet/dec/tulip/winbond-840.c
++++ b/drivers/net/ethernet/dec/tulip/winbond-840.c
+@@ -877,7 +877,7 @@ static void init_registers(struct net_device *dev)
+ 8000 16 longwords 0200 2 longwords 2000 32 longwords
+ C000 32 longwords 0400 4 longwords */
+
+-#if defined (__i386__) && !defined(MODULE)
++#if defined (__i386__) && !defined(MODULE) && !defined(CONFIG_UML)
+ /* When not a module we can work around broken '486 PCI boards. */
+ if (boot_cpu_data.x86 <= 4) {
+ i |= 0x4800;
+diff --git a/drivers/net/ethernet/ezchip/Kconfig b/drivers/net/ethernet/ezchip/Kconfig
+index 38aa824efb25d..9241b9b1c7a36 100644
+--- a/drivers/net/ethernet/ezchip/Kconfig
++++ b/drivers/net/ethernet/ezchip/Kconfig
+@@ -18,7 +18,7 @@ if NET_VENDOR_EZCHIP
+
+ config EZCHIP_NPS_MANAGEMENT_ENET
+ tristate "EZchip NPS management enet support"
+- depends on OF_IRQ && OF_NET
++ depends on OF_IRQ
+ depends on HAS_IOMEM
+ help
+ Simple LAN device for debug or management purposes.
+diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c
+index ff76e401a014b..e1df2dc810a28 100644
+--- a/drivers/net/ethernet/faraday/ftgmac100.c
++++ b/drivers/net/ethernet/faraday/ftgmac100.c
+@@ -1817,11 +1817,6 @@ static int ftgmac100_probe(struct platform_device *pdev)
+ priv->rxdes0_edorr_mask = BIT(30);
+ priv->txdes0_edotr_mask = BIT(30);
+ priv->is_aspeed = true;
+- /* Disable ast2600 problematic HW arbitration */
+- if (of_device_is_compatible(np, "aspeed,ast2600-mac")) {
+- iowrite32(FTGMAC100_TM_DEFAULT,
+- priv->base + FTGMAC100_OFFSET_TM);
+- }
+ } else {
+ priv->rxdes0_edorr_mask = BIT(15);
+ priv->txdes0_edotr_mask = BIT(15);
+@@ -1893,6 +1888,11 @@ static int ftgmac100_probe(struct platform_device *pdev)
+ err = ftgmac100_setup_clk(priv);
+ if (err)
+ goto err_phy_connect;
++
++ /* Disable ast2600 problematic HW arbitration */
++ if (of_device_is_compatible(np, "aspeed,ast2600-mac"))
++ iowrite32(FTGMAC100_TM_DEFAULT,
++ priv->base + FTGMAC100_OFFSET_TM);
+ }
+
+ /* Default ring sizes */
+diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c
+index 25c91b3c5fd30..819266d463b07 100644
+--- a/drivers/net/ethernet/fealnx.c
++++ b/drivers/net/ethernet/fealnx.c
+@@ -857,7 +857,7 @@ static int netdev_open(struct net_device *dev)
+ np->bcrvalue |= 0x04; /* big-endian */
+ #endif
+
+-#if defined(__i386__) && !defined(MODULE)
++#if defined(__i386__) && !defined(MODULE) && !defined(CONFIG_UML)
+ if (boot_cpu_data.x86 <= 4)
+ np->crvalue = 0xa00;
+ else
+diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
+index 763d2c7b5fb1a..5750f9a56393a 100644
+--- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
++++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
+@@ -489,11 +489,15 @@ static int dpaa_get_ts_info(struct net_device *net_dev,
+ info->phc_index = -1;
+
+ fman_node = of_get_parent(mac_node);
+- if (fman_node)
++ if (fman_node) {
+ ptp_node = of_parse_phandle(fman_node, "ptimer-handle", 0);
++ of_node_put(fman_node);
++ }
+
+- if (ptp_node)
++ if (ptp_node) {
+ ptp_dev = of_find_device_by_node(ptp_node);
++ of_node_put(ptp_node);
++ }
+
+ if (ptp_dev)
+ ptp = platform_get_drvdata(ptp_dev);
+diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+index 7065c71ed7b86..8b7a29e1e221b 100644
+--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
++++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+@@ -4329,7 +4329,7 @@ static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev)
+ }
+
+ INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp);
+-
++ mutex_init(&priv->onestep_tstamp_lock);
+ skb_queue_head_init(&priv->tx_skbs);
+
+ priv->rx_copybreak = DPAA2_ETH_DEFAULT_COPYBREAK;
+@@ -4511,12 +4511,12 @@ static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev)
+ #ifdef CONFIG_DEBUG_FS
+ dpaa2_dbg_remove(priv);
+ #endif
++
++ unregister_netdev(net_dev);
+ rtnl_lock();
+ dpaa2_eth_disconnect_mac(priv);
+ rtnl_unlock();
+
+- unregister_netdev(net_dev);
+-
+ dpaa2_eth_dl_port_del(priv);
+ dpaa2_eth_dl_traps_unregister(priv);
+ dpaa2_eth_dl_unregister(priv);
+@@ -4538,10 +4538,12 @@ static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev)
+
+ fsl_mc_portal_free(priv->mc_io);
+
+- free_netdev(net_dev);
++ destroy_workqueue(priv->dpaa2_ptp_wq);
+
+ dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name);
+
++ free_netdev(net_dev);
++
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
+index 32b5faa87bb8d..208a3459f2e29 100644
+--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
++++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
+@@ -168,7 +168,7 @@ static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev)
+ base = of_iomap(node, 0);
+ if (!base) {
+ err = -ENOMEM;
+- goto err_close;
++ goto err_put;
+ }
+
+ err = fsl_mc_allocate_irqs(mc_dev);
+@@ -212,6 +212,8 @@ err_free_mc_irq:
+ fsl_mc_free_irqs(mc_dev);
+ err_unmap:
+ iounmap(base);
++err_put:
++ of_node_put(node);
+ err_close:
+ dprtc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
+ err_free_mcp:
+diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c
+index d6eefbbf163fa..cacd454ac696c 100644
+--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c
++++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-flower.c
+@@ -532,6 +532,7 @@ static int dpaa2_switch_flower_parse_mirror_key(struct flow_cls_offload *cls,
+ struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+ struct flow_dissector *dissector = rule->match.dissector;
+ struct netlink_ext_ack *extack = cls->common.extack;
++ int ret = -EOPNOTSUPP;
+
+ if (dissector->used_keys &
+ ~(BIT(FLOW_DISSECTOR_KEY_BASIC) |
+@@ -561,9 +562,10 @@ static int dpaa2_switch_flower_parse_mirror_key(struct flow_cls_offload *cls,
+ }
+
+ *vlan = (u16)match.key->vlan_id;
++ ret = 0;
+ }
+
+- return 0;
++ return ret;
+ }
+
+ static int
+diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
+index 910b9f722504a..d62c188c87480 100644
+--- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
++++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
+@@ -672,7 +672,10 @@ static int enetc_get_ts_info(struct net_device *ndev,
+ #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
+ info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE |
+ SOF_TIMESTAMPING_RX_HARDWARE |
+- SOF_TIMESTAMPING_RAW_HARDWARE;
++ SOF_TIMESTAMPING_RAW_HARDWARE |
++ SOF_TIMESTAMPING_TX_SOFTWARE |
++ SOF_TIMESTAMPING_RX_SOFTWARE |
++ SOF_TIMESTAMPING_SOFTWARE;
+
+ info->tx_types = (1 << HWTSTAMP_TX_OFF) |
+ (1 << HWTSTAMP_TX_ON) |
+diff --git a/drivers/net/ethernet/freescale/enetc/enetc_qos.c b/drivers/net/ethernet/freescale/enetc/enetc_qos.c
+index 4577226d3c6ad..d779dde522c86 100644
+--- a/drivers/net/ethernet/freescale/enetc/enetc_qos.c
++++ b/drivers/net/ethernet/freescale/enetc/enetc_qos.c
+@@ -45,6 +45,7 @@ void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed)
+ | pspeed);
+ }
+
++#define ENETC_QOS_ALIGN 64
+ static int enetc_setup_taprio(struct net_device *ndev,
+ struct tc_taprio_qopt_offload *admin_conf)
+ {
+@@ -52,10 +53,11 @@ static int enetc_setup_taprio(struct net_device *ndev,
+ struct enetc_cbd cbd = {.cmd = 0};
+ struct tgs_gcl_conf *gcl_config;
+ struct tgs_gcl_data *gcl_data;
++ dma_addr_t dma, dma_align;
+ struct gce *gce;
+- dma_addr_t dma;
+ u16 data_size;
+ u16 gcl_len;
++ void *tmp;
+ u32 tge;
+ int err;
+ int i;
+@@ -82,9 +84,16 @@ static int enetc_setup_taprio(struct net_device *ndev,
+ gcl_config = &cbd.gcl_conf;
+
+ data_size = struct_size(gcl_data, entry, gcl_len);
+- gcl_data = kzalloc(data_size, __GFP_DMA | GFP_KERNEL);
+- if (!gcl_data)
++ tmp = dma_alloc_coherent(&priv->si->pdev->dev,
++ data_size + ENETC_QOS_ALIGN,
++ &dma, GFP_KERNEL);
++ if (!tmp) {
++ dev_err(&priv->si->pdev->dev,
++ "DMA mapping of taprio gate list failed!\n");
+ return -ENOMEM;
++ }
++ dma_align = ALIGN(dma, ENETC_QOS_ALIGN);
++ gcl_data = (struct tgs_gcl_data *)PTR_ALIGN(tmp, ENETC_QOS_ALIGN);
+
+ gce = (struct gce *)(gcl_data + 1);
+
+@@ -110,16 +119,8 @@ static int enetc_setup_taprio(struct net_device *ndev,
+ cbd.length = cpu_to_le16(data_size);
+ cbd.status_flags = 0;
+
+- dma = dma_map_single(&priv->si->pdev->dev, gcl_data,
+- data_size, DMA_TO_DEVICE);
+- if (dma_mapping_error(&priv->si->pdev->dev, dma)) {
+- netdev_err(priv->si->ndev, "DMA mapping failed!\n");
+- kfree(gcl_data);
+- return -ENOMEM;
+- }
+-
+- cbd.addr[0] = cpu_to_le32(lower_32_bits(dma));
+- cbd.addr[1] = cpu_to_le32(upper_32_bits(dma));
++ cbd.addr[0] = cpu_to_le32(lower_32_bits(dma_align));
++ cbd.addr[1] = cpu_to_le32(upper_32_bits(dma_align));
+ cbd.cls = BDCR_CMD_PORT_GCL;
+ cbd.status_flags = 0;
+
+@@ -132,8 +133,8 @@ static int enetc_setup_taprio(struct net_device *ndev,
+ ENETC_QBV_PTGCR_OFFSET,
+ tge & (~ENETC_QBV_TGE));
+
+- dma_unmap_single(&priv->si->pdev->dev, dma, data_size, DMA_TO_DEVICE);
+- kfree(gcl_data);
++ dma_free_coherent(&priv->si->pdev->dev, data_size + ENETC_QOS_ALIGN,
++ tmp, dma);
+
+ return err;
+ }
+@@ -463,8 +464,9 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv,
+ struct enetc_cbd cbd = {.cmd = 0};
+ struct streamid_data *si_data;
+ struct streamid_conf *si_conf;
++ dma_addr_t dma, dma_align;
+ u16 data_size;
+- dma_addr_t dma;
++ void *tmp;
+ int port;
+ int err;
+
+@@ -485,19 +487,20 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv,
+ cbd.status_flags = 0;
+
+ data_size = sizeof(struct streamid_data);
+- si_data = kzalloc(data_size, __GFP_DMA | GFP_KERNEL);
+- cbd.length = cpu_to_le16(data_size);
+-
+- dma = dma_map_single(&priv->si->pdev->dev, si_data,
+- data_size, DMA_FROM_DEVICE);
+- if (dma_mapping_error(&priv->si->pdev->dev, dma)) {
+- netdev_err(priv->si->ndev, "DMA mapping failed!\n");
+- kfree(si_data);
++ tmp = dma_alloc_coherent(&priv->si->pdev->dev,
++ data_size + ENETC_QOS_ALIGN,
++ &dma, GFP_KERNEL);
++ if (!tmp) {
++ dev_err(&priv->si->pdev->dev,
++ "DMA mapping of stream identify failed!\n");
+ return -ENOMEM;
+ }
++ dma_align = ALIGN(dma, ENETC_QOS_ALIGN);
++ si_data = (struct streamid_data *)PTR_ALIGN(tmp, ENETC_QOS_ALIGN);
+
+- cbd.addr[0] = cpu_to_le32(lower_32_bits(dma));
+- cbd.addr[1] = cpu_to_le32(upper_32_bits(dma));
++ cbd.length = cpu_to_le16(data_size);
++ cbd.addr[0] = cpu_to_le32(lower_32_bits(dma_align));
++ cbd.addr[1] = cpu_to_le32(upper_32_bits(dma_align));
+ eth_broadcast_addr(si_data->dmac);
+ si_data->vid_vidm_tg = (ENETC_CBDR_SID_VID_MASK
+ + ((0x3 << 14) | ENETC_CBDR_SID_VIDM));
+@@ -512,12 +515,10 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv,
+
+ err = enetc_send_cmd(priv->si, &cbd);
+ if (err)
+- return -EINVAL;
++ goto out;
+
+- if (!enable) {
+- kfree(si_data);
+- return 0;
+- }
++ if (!enable)
++ goto out;
+
+ /* Enable the entry overwrite again incase space flushed by hardware */
+ memset(&cbd, 0, sizeof(cbd));
+@@ -539,8 +540,8 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv,
+
+ cbd.length = cpu_to_le16(data_size);
+
+- cbd.addr[0] = cpu_to_le32(lower_32_bits(dma));
+- cbd.addr[1] = cpu_to_le32(upper_32_bits(dma));
++ cbd.addr[0] = cpu_to_le32(lower_32_bits(dma_align));
++ cbd.addr[1] = cpu_to_le32(upper_32_bits(dma_align));
+
+ /* VIDM default to be 1.
+ * VID Match. If set (b1) then the VID must match, otherwise
+@@ -560,7 +561,9 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv,
+ }
+
+ err = enetc_send_cmd(priv->si, &cbd);
+- kfree(si_data);
++out:
++ dma_free_coherent(&priv->si->pdev->dev, data_size + ENETC_QOS_ALIGN,
++ tmp, dma);
+
+ return err;
+ }
+@@ -629,8 +632,9 @@ static int enetc_streamcounter_hw_get(struct enetc_ndev_priv *priv,
+ {
+ struct enetc_cbd cbd = { .cmd = 2 };
+ struct sfi_counter_data *data_buf;
+- dma_addr_t dma;
++ dma_addr_t dma, dma_align;
+ u16 data_size;
++ void *tmp;
+ int err;
+
+ cbd.index = cpu_to_le16((u16)index);
+@@ -639,19 +643,19 @@ static int enetc_streamcounter_hw_get(struct enetc_ndev_priv *priv,
+ cbd.status_flags = 0;
+
+ data_size = sizeof(struct sfi_counter_data);
+- data_buf = kzalloc(data_size, __GFP_DMA | GFP_KERNEL);
+- if (!data_buf)
++ tmp = dma_alloc_coherent(&priv->si->pdev->dev,
++ data_size + ENETC_QOS_ALIGN,
++ &dma, GFP_KERNEL);
++ if (!tmp) {
++ dev_err(&priv->si->pdev->dev,
++ "DMA mapping of stream counter failed!\n");
+ return -ENOMEM;
+-
+- dma = dma_map_single(&priv->si->pdev->dev, data_buf,
+- data_size, DMA_FROM_DEVICE);
+- if (dma_mapping_error(&priv->si->pdev->dev, dma)) {
+- netdev_err(priv->si->ndev, "DMA mapping failed!\n");
+- err = -ENOMEM;
+- goto exit;
+ }
+- cbd.addr[0] = cpu_to_le32(lower_32_bits(dma));
+- cbd.addr[1] = cpu_to_le32(upper_32_bits(dma));
++ dma_align = ALIGN(dma, ENETC_QOS_ALIGN);
++ data_buf = (struct sfi_counter_data *)PTR_ALIGN(tmp, ENETC_QOS_ALIGN);
++
++ cbd.addr[0] = cpu_to_le32(lower_32_bits(dma_align));
++ cbd.addr[1] = cpu_to_le32(upper_32_bits(dma_align));
+
+ cbd.length = cpu_to_le16(data_size);
+
+@@ -680,7 +684,9 @@ static int enetc_streamcounter_hw_get(struct enetc_ndev_priv *priv,
+ data_buf->flow_meter_dropl;
+
+ exit:
+- kfree(data_buf);
++ dma_free_coherent(&priv->si->pdev->dev, data_size + ENETC_QOS_ALIGN,
++ tmp, dma);
++
+ return err;
+ }
+
+@@ -719,9 +725,10 @@ static int enetc_streamgate_hw_set(struct enetc_ndev_priv *priv,
+ struct sgcl_conf *sgcl_config;
+ struct sgcl_data *sgcl_data;
+ struct sgce *sgce;
+- dma_addr_t dma;
++ dma_addr_t dma, dma_align;
+ u16 data_size;
+ int err, i;
++ void *tmp;
+ u64 now;
+
+ cbd.index = cpu_to_le16(sgi->index);
+@@ -768,24 +775,20 @@ static int enetc_streamgate_hw_set(struct enetc_ndev_priv *priv,
+ sgcl_config->acl_len = (sgi->num_entries - 1) & 0x3;
+
+ data_size = struct_size(sgcl_data, sgcl, sgi->num_entries);
+-
+- sgcl_data = kzalloc(data_size, __GFP_DMA | GFP_KERNEL);
+- if (!sgcl_data)
+- return -ENOMEM;
+-
+- cbd.length = cpu_to_le16(data_size);
+-
+- dma = dma_map_single(&priv->si->pdev->dev,
+- sgcl_data, data_size,
+- DMA_FROM_DEVICE);
+- if (dma_mapping_error(&priv->si->pdev->dev, dma)) {
+- netdev_err(priv->si->ndev, "DMA mapping failed!\n");
+- kfree(sgcl_data);
++ tmp = dma_alloc_coherent(&priv->si->pdev->dev,
++ data_size + ENETC_QOS_ALIGN,
++ &dma, GFP_KERNEL);
++ if (!tmp) {
++ dev_err(&priv->si->pdev->dev,
++ "DMA mapping of stream counter failed!\n");
+ return -ENOMEM;
+ }
++ dma_align = ALIGN(dma, ENETC_QOS_ALIGN);
++ sgcl_data = (struct sgcl_data *)PTR_ALIGN(tmp, ENETC_QOS_ALIGN);
+
+- cbd.addr[0] = cpu_to_le32(lower_32_bits(dma));
+- cbd.addr[1] = cpu_to_le32(upper_32_bits(dma));
++ cbd.length = cpu_to_le16(data_size);
++ cbd.addr[0] = cpu_to_le32(lower_32_bits(dma_align));
++ cbd.addr[1] = cpu_to_le32(upper_32_bits(dma_align));
+
+ sgce = &sgcl_data->sgcl[0];
+
+@@ -840,7 +843,8 @@ static int enetc_streamgate_hw_set(struct enetc_ndev_priv *priv,
+ err = enetc_send_cmd(priv->si, &cbd);
+
+ exit:
+- kfree(sgcl_data);
++ dma_free_coherent(&priv->si->pdev->dev, data_size + ENETC_QOS_ALIGN,
++ tmp, dma);
+
+ return err;
+ }
+diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
+index 7b4961daa2540..ed7301b691694 100644
+--- a/drivers/net/ethernet/freescale/fec.h
++++ b/drivers/net/ethernet/freescale/fec.h
+@@ -377,6 +377,9 @@ struct bufdesc_ex {
+ #define FEC_ENET_WAKEUP ((uint)0x00020000) /* Wakeup request */
+ #define FEC_ENET_TXF (FEC_ENET_TXF_0 | FEC_ENET_TXF_1 | FEC_ENET_TXF_2)
+ #define FEC_ENET_RXF (FEC_ENET_RXF_0 | FEC_ENET_RXF_1 | FEC_ENET_RXF_2)
++#define FEC_ENET_RXF_GET(X) (((X) == 0) ? FEC_ENET_RXF_0 : \
++ (((X) == 1) ? FEC_ENET_RXF_1 : \
++ FEC_ENET_RXF_2))
+ #define FEC_ENET_TS_AVAIL ((uint)0x00010000)
+ #define FEC_ENET_TS_TIMER ((uint)0x00008000)
+
+diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
+index ec87b370bba1f..67eb9b671244b 100644
+--- a/drivers/net/ethernet/freescale/fec_main.c
++++ b/drivers/net/ethernet/freescale/fec_main.c
+@@ -1480,7 +1480,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
+ break;
+ pkt_received++;
+
+- writel(FEC_ENET_RXF, fep->hwp + FEC_IEVENT);
++ writel(FEC_ENET_RXF_GET(queue_id), fep->hwp + FEC_IEVENT);
+
+ /* Check for errors. */
+ status ^= BD_ENET_RX_LAST;
+@@ -3726,7 +3726,7 @@ static int fec_enet_init_stop_mode(struct fec_enet_private *fep,
+ ARRAY_SIZE(out_val));
+ if (ret) {
+ dev_dbg(&fep->pdev->dev, "no stop mode property\n");
+- return ret;
++ goto out;
+ }
+
+ fep->stop_gpr.gpr = syscon_node_to_regmap(gpr_np);
+diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c
+index d9baac0dbc7d0..4c9d05c45c033 100644
+--- a/drivers/net/ethernet/freescale/fman/fman_port.c
++++ b/drivers/net/ethernet/freescale/fman/fman_port.c
+@@ -1805,7 +1805,7 @@ static int fman_port_probe(struct platform_device *of_dev)
+ fman = dev_get_drvdata(&fm_pdev->dev);
+ if (!fman) {
+ err = -EINVAL;
+- goto return_err;
++ goto put_device;
+ }
+
+ err = of_property_read_u32(port_node, "cell-index", &val);
+@@ -1813,7 +1813,7 @@ static int fman_port_probe(struct platform_device *of_dev)
+ dev_err(port->dev, "%s: reading cell-index for %pOF failed\n",
+ __func__, port_node);
+ err = -EINVAL;
+- goto return_err;
++ goto put_device;
+ }
+ port_id = (u8)val;
+ port->dts_params.id = port_id;
+@@ -1847,7 +1847,7 @@ static int fman_port_probe(struct platform_device *of_dev)
+ } else {
+ dev_err(port->dev, "%s: Illegal port type\n", __func__);
+ err = -EINVAL;
+- goto return_err;
++ goto put_device;
+ }
+
+ port->dts_params.type = port_type;
+@@ -1861,7 +1861,7 @@ static int fman_port_probe(struct platform_device *of_dev)
+ dev_err(port->dev, "%s: incorrect qman-channel-id\n",
+ __func__);
+ err = -EINVAL;
+- goto return_err;
++ goto put_device;
+ }
+ port->dts_params.qman_channel_id = qman_channel_id;
+ }
+@@ -1871,7 +1871,7 @@ static int fman_port_probe(struct platform_device *of_dev)
+ dev_err(port->dev, "%s: of_address_to_resource() failed\n",
+ __func__);
+ err = -ENOMEM;
+- goto return_err;
++ goto put_device;
+ }
+
+ port->dts_params.fman = fman;
+@@ -1896,6 +1896,8 @@ static int fman_port_probe(struct platform_device *of_dev)
+
+ return 0;
+
++put_device:
++ put_device(&fm_pdev->dev);
+ return_err:
+ of_node_put(port_node);
+ free_port:
+diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c
+index d9fc5c456bf3e..39ae965cd4f64 100644
+--- a/drivers/net/ethernet/freescale/fman/mac.c
++++ b/drivers/net/ethernet/freescale/fman/mac.c
+@@ -94,14 +94,17 @@ static void mac_exception(void *handle, enum fman_mac_exceptions ex)
+ __func__, ex);
+ }
+
+-static void set_fman_mac_params(struct mac_device *mac_dev,
+- struct fman_mac_params *params)
++static int set_fman_mac_params(struct mac_device *mac_dev,
++ struct fman_mac_params *params)
+ {
+ struct mac_priv_s *priv = mac_dev->priv;
+
+ params->base_addr = (typeof(params->base_addr))
+ devm_ioremap(priv->dev, mac_dev->res->start,
+ resource_size(mac_dev->res));
++ if (!params->base_addr)
++ return -ENOMEM;
++
+ memcpy(&params->addr, mac_dev->addr, sizeof(mac_dev->addr));
+ params->max_speed = priv->max_speed;
+ params->phy_if = mac_dev->phy_if;
+@@ -112,6 +115,8 @@ static void set_fman_mac_params(struct mac_device *mac_dev,
+ params->event_cb = mac_exception;
+ params->dev_id = mac_dev;
+ params->internal_phy_node = priv->internal_phy_node;
++
++ return 0;
+ }
+
+ static int tgec_initialization(struct mac_device *mac_dev)
+@@ -123,7 +128,9 @@ static int tgec_initialization(struct mac_device *mac_dev)
+
+ priv = mac_dev->priv;
+
+- set_fman_mac_params(mac_dev, &params);
++ err = set_fman_mac_params(mac_dev, &params);
++ if (err)
++ goto _return;
+
+ mac_dev->fman_mac = tgec_config(&params);
+ if (!mac_dev->fman_mac) {
+@@ -169,7 +176,9 @@ static int dtsec_initialization(struct mac_device *mac_dev)
+
+ priv = mac_dev->priv;
+
+- set_fman_mac_params(mac_dev, &params);
++ err = set_fman_mac_params(mac_dev, &params);
++ if (err)
++ goto _return;
+
+ mac_dev->fman_mac = dtsec_config(&params);
+ if (!mac_dev->fman_mac) {
+@@ -218,7 +227,9 @@ static int memac_initialization(struct mac_device *mac_dev)
+
+ priv = mac_dev->priv;
+
+- set_fman_mac_params(mac_dev, &params);
++ err = set_fman_mac_params(mac_dev, &params);
++ if (err)
++ goto _return;
+
+ if (priv->max_speed == SPEED_10000)
+ params.phy_if = PHY_INTERFACE_MODE_XGMII;
+diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c
+index 7b32ed29bf4cb..8c17fe5d66ed4 100644
+--- a/drivers/net/ethernet/freescale/gianfar_ethtool.c
++++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c
+@@ -1460,6 +1460,7 @@ static int gfar_get_ts_info(struct net_device *dev,
+ ptp_node = of_find_compatible_node(NULL, NULL, "fsl,etsec-ptp");
+ if (ptp_node) {
+ ptp_dev = of_find_device_by_node(ptp_node);
++ of_node_put(ptp_node);
+ if (ptp_dev)
+ ptp = platform_get_drvdata(ptp_dev);
+ }
+diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c
+index 0b68852379da5..27d07468765f0 100644
+--- a/drivers/net/ethernet/freescale/xgmac_mdio.c
++++ b/drivers/net/ethernet/freescale/xgmac_mdio.c
+@@ -52,6 +52,7 @@ struct tgec_mdio_controller {
+ struct mdio_fsl_priv {
+ struct tgec_mdio_controller __iomem *mdio_base;
+ bool is_little_endian;
++ bool has_a009885;
+ bool has_a011043;
+ };
+
+@@ -187,10 +188,10 @@ static int xgmac_mdio_read(struct mii_bus *bus, int phy_id, int regnum)
+ {
+ struct mdio_fsl_priv *priv = (struct mdio_fsl_priv *)bus->priv;
+ struct tgec_mdio_controller __iomem *regs = priv->mdio_base;
++ unsigned long flags;
+ uint16_t dev_addr;
+ uint32_t mdio_stat;
+ uint32_t mdio_ctl;
+- uint16_t value;
+ int ret;
+ bool endian = priv->is_little_endian;
+
+@@ -222,12 +223,18 @@ static int xgmac_mdio_read(struct mii_bus *bus, int phy_id, int regnum)
+ return ret;
+ }
+
++ if (priv->has_a009885)
++ /* Once the operation completes, i.e. MDIO_STAT_BSY clears, we
++ * must read back the data register within 16 MDC cycles.
++ */
++ local_irq_save(flags);
++
+ /* Initiate the read */
+ xgmac_write32(mdio_ctl | MDIO_CTL_READ, &regs->mdio_ctl, endian);
+
+ ret = xgmac_wait_until_done(&bus->dev, regs, endian);
+ if (ret)
+- return ret;
++ goto irq_restore;
+
+ /* Return all Fs if nothing was there */
+ if ((xgmac_read32(&regs->mdio_stat, endian) & MDIO_STAT_RD_ER) &&
+@@ -235,13 +242,17 @@ static int xgmac_mdio_read(struct mii_bus *bus, int phy_id, int regnum)
+ dev_dbg(&bus->dev,
+ "Error while reading PHY%d reg at %d.%hhu\n",
+ phy_id, dev_addr, regnum);
+- return 0xffff;
++ ret = 0xffff;
++ } else {
++ ret = xgmac_read32(&regs->mdio_data, endian) & 0xffff;
++ dev_dbg(&bus->dev, "read %04x\n", ret);
+ }
+
+- value = xgmac_read32(&regs->mdio_data, endian) & 0xffff;
+- dev_dbg(&bus->dev, "read %04x\n", value);
++irq_restore:
++ if (priv->has_a009885)
++ local_irq_restore(flags);
+
+- return value;
++ return ret;
+ }
+
+ static int xgmac_mdio_probe(struct platform_device *pdev)
+@@ -288,6 +299,8 @@ static int xgmac_mdio_probe(struct platform_device *pdev)
+ priv->is_little_endian = device_property_read_bool(&pdev->dev,
+ "little-endian");
+
++ priv->has_a009885 = device_property_read_bool(&pdev->dev,
++ "fsl,erratum-a009885");
+ priv->has_a011043 = device_property_read_bool(&pdev->dev,
+ "fsl,erratum-a011043");
+
+@@ -319,9 +332,10 @@ err_ioremap:
+ static int xgmac_mdio_remove(struct platform_device *pdev)
+ {
+ struct mii_bus *bus = platform_get_drvdata(pdev);
++ struct mdio_fsl_priv *priv = bus->priv;
+
+ mdiobus_unregister(bus);
+- iounmap(bus->priv);
++ iounmap(priv->mdio_base);
+ mdiobus_free(bus);
+
+ return 0;
+diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
+index 92dc18a4bcc41..b1273dce4795b 100644
+--- a/drivers/net/ethernet/google/gve/gve.h
++++ b/drivers/net/ethernet/google/gve/gve.h
+@@ -30,7 +30,7 @@
+ #define GVE_MIN_MSIX 3
+
+ /* Numbers of gve tx/rx stats in stats report. */
+-#define GVE_TX_STATS_REPORT_NUM 5
++#define GVE_TX_STATS_REPORT_NUM 6
+ #define GVE_RX_STATS_REPORT_NUM 2
+
+ /* Interval to schedule a stats report update, 20000ms. */
+@@ -224,11 +224,6 @@ struct gve_tx_iovec {
+ u32 iov_padding; /* padding associated with this segment */
+ };
+
+-struct gve_tx_dma_buf {
+- DEFINE_DMA_UNMAP_ADDR(dma);
+- DEFINE_DMA_UNMAP_LEN(len);
+-};
+-
+ /* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc
+ * ring entry but only used for a pkt_desc not a seg_desc
+ */
+@@ -236,7 +231,10 @@ struct gve_tx_buffer_state {
+ struct sk_buff *skb; /* skb for this pkt */
+ union {
+ struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
+- struct gve_tx_dma_buf buf;
++ struct {
++ DEFINE_DMA_UNMAP_ADDR(dma);
++ DEFINE_DMA_UNMAP_LEN(len);
++ };
+ };
+ };
+
+@@ -280,7 +278,8 @@ struct gve_tx_pending_packet_dqo {
+ * All others correspond to `skb`'s frags and should be unmapped with
+ * `dma_unmap_page`.
+ */
+- struct gve_tx_dma_buf bufs[MAX_SKB_FRAGS + 1];
++ DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]);
++ DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]);
+ u16 num_bufs;
+
+ /* Linked list index to next element in the list, or -1 if none */
+@@ -414,7 +413,9 @@ struct gve_tx_ring {
+ u32 q_num ____cacheline_aligned; /* queue idx */
+ u32 stop_queue; /* count of queue stops */
+ u32 wake_queue; /* count of queue wakes */
++ u32 queue_timeout; /* count of queue timeouts */
+ u32 ntfy_id; /* notification block index */
++ u32 last_kick_msec; /* Last time the queue was kicked */
+ dma_addr_t bus; /* dma address of the descr ring */
+ dma_addr_t q_resources_bus; /* dma address of the queue resources */
+ dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */
+@@ -814,7 +815,7 @@ static inline bool gve_is_gqi(struct gve_priv *priv)
+ /* buffers */
+ int gve_alloc_page(struct gve_priv *priv, struct device *dev,
+ struct page **page, dma_addr_t *dma,
+- enum dma_data_direction);
++ enum dma_data_direction, gfp_t gfp_flags);
+ void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
+ enum dma_data_direction);
+ /* tx handling */
+diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
+index f089d33dd48e0..ce507464f3d62 100644
+--- a/drivers/net/ethernet/google/gve/gve_adminq.c
++++ b/drivers/net/ethernet/google/gve/gve_adminq.c
+@@ -281,7 +281,7 @@ static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
+ */
+ static int gve_adminq_kick_and_wait(struct gve_priv *priv)
+ {
+- u32 tail, head;
++ int tail, head;
+ int i;
+
+ tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
+diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
+index 47c3d8f313fcf..3953f6f7a4273 100644
+--- a/drivers/net/ethernet/google/gve/gve_adminq.h
++++ b/drivers/net/ethernet/google/gve/gve_adminq.h
+@@ -270,6 +270,7 @@ enum gve_stat_names {
+ TX_LAST_COMPLETION_PROCESSED = 5,
+ RX_NEXT_EXPECTED_SEQUENCE = 6,
+ RX_BUFFERS_POSTED = 7,
++ TX_TIMEOUT_CNT = 8,
+ // stats from NIC
+ RX_QUEUE_DROP_CNT = 65,
+ RX_NO_BUFFERS_POSTED = 66,
+diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
+index bf8a4a7c43f78..68552848d3888 100644
+--- a/drivers/net/ethernet/google/gve/gve_main.c
++++ b/drivers/net/ethernet/google/gve/gve_main.c
+@@ -24,6 +24,9 @@
+ #define GVE_VERSION "1.0.0"
+ #define GVE_VERSION_PREFIX "GVE-"
+
++// Minimum amount of time between queue kicks in msec (10 seconds)
++#define MIN_TX_TIMEOUT_GAP (1000 * 10)
++
+ const char gve_version_str[] = GVE_VERSION;
+ static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
+
+@@ -743,9 +746,9 @@ static void gve_free_rings(struct gve_priv *priv)
+
+ int gve_alloc_page(struct gve_priv *priv, struct device *dev,
+ struct page **page, dma_addr_t *dma,
+- enum dma_data_direction dir)
++ enum dma_data_direction dir, gfp_t gfp_flags)
+ {
+- *page = alloc_page(GFP_KERNEL);
++ *page = alloc_page(gfp_flags);
+ if (!*page) {
+ priv->page_alloc_fail++;
+ return -ENOMEM;
+@@ -789,7 +792,7 @@ static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
+ for (i = 0; i < pages; i++) {
+ err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i],
+ &qpl->page_buses[i],
+- gve_qpl_dma_dir(priv, id));
++ gve_qpl_dma_dir(priv, id), GFP_KERNEL);
+ /* caller handles clean up */
+ if (err)
+ return -ENOMEM;
+@@ -1116,9 +1119,47 @@ static void gve_turnup(struct gve_priv *priv)
+
+ static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue)
+ {
+- struct gve_priv *priv = netdev_priv(dev);
++ struct gve_notify_block *block;
++ struct gve_tx_ring *tx = NULL;
++ struct gve_priv *priv;
++ u32 last_nic_done;
++ u32 current_time;
++ u32 ntfy_idx;
++
++ netdev_info(dev, "Timeout on tx queue, %d", txqueue);
++ priv = netdev_priv(dev);
++ if (txqueue > priv->tx_cfg.num_queues)
++ goto reset;
++
++ ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue);
++ if (ntfy_idx >= priv->num_ntfy_blks)
++ goto reset;
+
++ block = &priv->ntfy_blocks[ntfy_idx];
++ tx = block->tx;
++
++ current_time = jiffies_to_msecs(jiffies);
++ if (tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time)
++ goto reset;
++
++ /* Check to see if there are missed completions, which will allow us to
++ * kick the queue.
++ */
++ last_nic_done = gve_tx_load_event_counter(priv, tx);
++ if (last_nic_done - tx->done) {
++ netdev_info(dev, "Kicking queue %d", txqueue);
++ iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
++ napi_schedule(&block->napi);
++ tx->last_kick_msec = current_time;
++ goto out;
++ } // Else reset.
++
++reset:
+ gve_schedule_reset(priv);
++
++out:
++ if (tx)
++ tx->queue_timeout++;
+ priv->tx_timeo_cnt++;
+ }
+
+@@ -1247,6 +1288,11 @@ void gve_handle_report_stats(struct gve_priv *priv)
+ .value = cpu_to_be64(last_completion),
+ .queue_id = cpu_to_be32(idx),
+ };
++ stats[stats_idx++] = (struct stats) {
++ .stat_name = cpu_to_be32(TX_TIMEOUT_CNT),
++ .value = cpu_to_be64(priv->tx[idx].queue_timeout),
++ .queue_id = cpu_to_be32(idx),
++ };
+ }
+ }
+ /* rx stats */
+diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
+index 94941d4e47449..97431969a488f 100644
+--- a/drivers/net/ethernet/google/gve/gve_rx.c
++++ b/drivers/net/ethernet/google/gve/gve_rx.c
+@@ -79,7 +79,8 @@ static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
+ dma_addr_t dma;
+ int err;
+
+- err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE);
++ err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE,
++ GFP_ATOMIC);
+ if (err)
+ return err;
+
+@@ -449,6 +450,7 @@ static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
+ skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
+ gve_rss_type(rx_desc->flags_seq));
+
++ skb_record_rx_queue(skb, rx->q_num);
+ if (skb_is_nonlinear(skb))
+ napi_gro_frags(napi);
+ else
+@@ -514,8 +516,13 @@ static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
+
+ gve_rx_free_buffer(dev, page_info, data_slot);
+ page_info->page = NULL;
+- if (gve_rx_alloc_buffer(priv, dev, page_info, data_slot))
++ if (gve_rx_alloc_buffer(priv, dev, page_info,
++ data_slot)) {
++ u64_stats_update_begin(&rx->statss);
++ rx->rx_buf_alloc_fail++;
++ u64_stats_update_end(&rx->statss);
+ break;
++ }
+ }
+ }
+ fill_cnt++;
+diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+index 8500621b2cd41..7b18b4fd9e548 100644
+--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
++++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+@@ -157,7 +157,7 @@ static int gve_alloc_page_dqo(struct gve_priv *priv,
+ int err;
+
+ err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page,
+- &buf_state->addr, DMA_FROM_DEVICE);
++ &buf_state->addr, DMA_FROM_DEVICE, GFP_KERNEL);
+ if (err)
+ return err;
+
+diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c
+index 665ac795a1adf..9922ce46a6351 100644
+--- a/drivers/net/ethernet/google/gve/gve_tx.c
++++ b/drivers/net/ethernet/google/gve/gve_tx.c
+@@ -303,15 +303,15 @@ static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
+ static void gve_tx_unmap_buf(struct device *dev, struct gve_tx_buffer_state *info)
+ {
+ if (info->skb) {
+- dma_unmap_single(dev, dma_unmap_addr(&info->buf, dma),
+- dma_unmap_len(&info->buf, len),
++ dma_unmap_single(dev, dma_unmap_addr(info, dma),
++ dma_unmap_len(info, len),
+ DMA_TO_DEVICE);
+- dma_unmap_len_set(&info->buf, len, 0);
++ dma_unmap_len_set(info, len, 0);
+ } else {
+- dma_unmap_page(dev, dma_unmap_addr(&info->buf, dma),
+- dma_unmap_len(&info->buf, len),
++ dma_unmap_page(dev, dma_unmap_addr(info, dma),
++ dma_unmap_len(info, len),
+ DMA_TO_DEVICE);
+- dma_unmap_len_set(&info->buf, len, 0);
++ dma_unmap_len_set(info, len, 0);
+ }
+ }
+
+@@ -491,7 +491,6 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
+ struct gve_tx_buffer_state *info;
+ bool is_gso = skb_is_gso(skb);
+ u32 idx = tx->req & tx->mask;
+- struct gve_tx_dma_buf *buf;
+ u64 addr;
+ u32 len;
+ int i;
+@@ -515,9 +514,8 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
+ tx->dma_mapping_error++;
+ goto drop;
+ }
+- buf = &info->buf;
+- dma_unmap_len_set(buf, len, len);
+- dma_unmap_addr_set(buf, dma, addr);
++ dma_unmap_len_set(info, len, len);
++ dma_unmap_addr_set(info, dma, addr);
+
+ payload_nfrags = shinfo->nr_frags;
+ if (hlen < len) {
+@@ -549,10 +547,9 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
+ tx->dma_mapping_error++;
+ goto unmap_drop;
+ }
+- buf = &tx->info[idx].buf;
+ tx->info[idx].skb = NULL;
+- dma_unmap_len_set(buf, len, len);
+- dma_unmap_addr_set(buf, dma, addr);
++ dma_unmap_len_set(&tx->info[idx], len, len);
++ dma_unmap_addr_set(&tx->info[idx], dma, addr);
+
+ gve_tx_fill_seg_desc(seg_desc, skb, is_gso, len, addr);
+ }
+diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+index 05ddb6a75c38f..ec394d9916681 100644
+--- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c
++++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+@@ -85,18 +85,16 @@ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx)
+ int j;
+
+ for (j = 0; j < cur_state->num_bufs; j++) {
+- struct gve_tx_dma_buf *buf = &cur_state->bufs[j];
+-
+ if (j == 0) {
+ dma_unmap_single(tx->dev,
+- dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len),
+- DMA_TO_DEVICE);
++ dma_unmap_addr(cur_state, dma[j]),
++ dma_unmap_len(cur_state, len[j]),
++ DMA_TO_DEVICE);
+ } else {
+ dma_unmap_page(tx->dev,
+- dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len),
+- DMA_TO_DEVICE);
++ dma_unmap_addr(cur_state, dma[j]),
++ dma_unmap_len(cur_state, len[j]),
++ DMA_TO_DEVICE);
+ }
+ }
+ if (cur_state->skb) {
+@@ -457,15 +455,15 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+ const bool is_gso = skb_is_gso(skb);
+ u32 desc_idx = tx->dqo_tx.tail;
+
+- struct gve_tx_pending_packet_dqo *pending_packet;
++ struct gve_tx_pending_packet_dqo *pkt;
+ struct gve_tx_metadata_dqo metadata;
+ s16 completion_tag;
+ int i;
+
+- pending_packet = gve_alloc_pending_packet(tx);
+- pending_packet->skb = skb;
+- pending_packet->num_bufs = 0;
+- completion_tag = pending_packet - tx->dqo.pending_packets;
++ pkt = gve_alloc_pending_packet(tx);
++ pkt->skb = skb;
++ pkt->num_bufs = 0;
++ completion_tag = pkt - tx->dqo.pending_packets;
+
+ gve_extract_tx_metadata_dqo(skb, &metadata);
+ if (is_gso) {
+@@ -493,8 +491,6 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+
+ /* Map the linear portion of skb */
+ {
+- struct gve_tx_dma_buf *buf =
+- &pending_packet->bufs[pending_packet->num_bufs];
+ u32 len = skb_headlen(skb);
+ dma_addr_t addr;
+
+@@ -502,9 +498,9 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+ if (unlikely(dma_mapping_error(tx->dev, addr)))
+ goto err;
+
+- dma_unmap_len_set(buf, len, len);
+- dma_unmap_addr_set(buf, dma, addr);
+- ++pending_packet->num_bufs;
++ dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
++ dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
++ ++pkt->num_bufs;
+
+ gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr,
+ completion_tag,
+@@ -512,8 +508,6 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+ }
+
+ for (i = 0; i < shinfo->nr_frags; i++) {
+- struct gve_tx_dma_buf *buf =
+- &pending_packet->bufs[pending_packet->num_bufs];
+ const skb_frag_t *frag = &shinfo->frags[i];
+ bool is_eop = i == (shinfo->nr_frags - 1);
+ u32 len = skb_frag_size(frag);
+@@ -523,9 +517,9 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+ if (unlikely(dma_mapping_error(tx->dev, addr)))
+ goto err;
+
+- dma_unmap_len_set(buf, len, len);
+- dma_unmap_addr_set(buf, dma, addr);
+- ++pending_packet->num_bufs;
++ dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
++ dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
++ ++pkt->num_bufs;
+
+ gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr,
+ completion_tag, is_eop, is_gso);
+@@ -552,22 +546,23 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
+ return 0;
+
+ err:
+- for (i = 0; i < pending_packet->num_bufs; i++) {
+- struct gve_tx_dma_buf *buf = &pending_packet->bufs[i];
+-
++ for (i = 0; i < pkt->num_bufs; i++) {
+ if (i == 0) {
+- dma_unmap_single(tx->dev, dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len),
++ dma_unmap_single(tx->dev,
++ dma_unmap_addr(pkt, dma[i]),
++ dma_unmap_len(pkt, len[i]),
+ DMA_TO_DEVICE);
+ } else {
+- dma_unmap_page(tx->dev, dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len), DMA_TO_DEVICE);
++ dma_unmap_page(tx->dev,
++ dma_unmap_addr(pkt, dma[i]),
++ dma_unmap_len(pkt, len[i]),
++ DMA_TO_DEVICE);
+ }
+ }
+
+- pending_packet->skb = NULL;
+- pending_packet->num_bufs = 0;
+- gve_free_pending_packet(tx, pending_packet);
++ pkt->skb = NULL;
++ pkt->num_bufs = 0;
++ gve_free_pending_packet(tx, pkt);
+
+ return -1;
+ }
+@@ -725,12 +720,12 @@ static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list,
+
+ static void remove_from_list(struct gve_tx_ring *tx,
+ struct gve_index_list *list,
+- struct gve_tx_pending_packet_dqo *pending_packet)
++ struct gve_tx_pending_packet_dqo *pkt)
+ {
+ s16 prev_index, next_index;
+
+- prev_index = pending_packet->prev;
+- next_index = pending_packet->next;
++ prev_index = pkt->prev;
++ next_index = pkt->next;
+
+ if (prev_index == -1) {
+ /* Node is head */
+@@ -747,21 +742,18 @@ static void remove_from_list(struct gve_tx_ring *tx,
+ }
+
+ static void gve_unmap_packet(struct device *dev,
+- struct gve_tx_pending_packet_dqo *pending_packet)
++ struct gve_tx_pending_packet_dqo *pkt)
+ {
+- struct gve_tx_dma_buf *buf;
+ int i;
+
+ /* SKB linear portion is guaranteed to be mapped */
+- buf = &pending_packet->bufs[0];
+- dma_unmap_single(dev, dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len), DMA_TO_DEVICE);
+- for (i = 1; i < pending_packet->num_bufs; i++) {
+- buf = &pending_packet->bufs[i];
+- dma_unmap_page(dev, dma_unmap_addr(buf, dma),
+- dma_unmap_len(buf, len), DMA_TO_DEVICE);
++ dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]),
++ dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE);
++ for (i = 1; i < pkt->num_bufs; i++) {
++ dma_unmap_page(dev, dma_unmap_addr(pkt, dma[i]),
++ dma_unmap_len(pkt, len[i]), DMA_TO_DEVICE);
+ }
+- pending_packet->num_bufs = 0;
++ pkt->num_bufs = 0;
+ }
+
+ /* Completion types and expected behavior:
+diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+index 23d9cbf262c32..740850b64aff5 100644
+--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
++++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+@@ -400,6 +400,10 @@ static void hns_dsaf_ge_srst_by_port(struct dsaf_device *dsaf_dev, u32 port,
+ return;
+
+ if (!HNS_DSAF_IS_DEBUG(dsaf_dev)) {
++ /* DSAF_MAX_PORT_NUM is 6, but DSAF_GE_NUM is 8.
++ We need check to prevent array overflow */
++ if (port >= DSAF_MAX_PORT_NUM)
++ return;
+ reg_val_1 = 0x1 << port;
+ port_rst_off = dsaf_dev->mac_cb[port]->port_rst_off;
+ /* there is difference between V1 and V2 in register.*/
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+index d701451596c82..9204f5ecd4151 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
++++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+@@ -527,6 +527,8 @@ struct hnae3_ae_dev {
+ * Get 1588 rx hwstamp
+ * get_ts_info
+ * Get phc info
++ * clean_vf_config
++ * Clean residual vf info after disable sriov
+ */
+ struct hnae3_ae_ops {
+ int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev);
+@@ -720,6 +722,7 @@ struct hnae3_ae_ops {
+ struct ethtool_ts_info *info);
+ int (*get_link_diagnosis_info)(struct hnae3_handle *handle,
+ u32 *status_code);
++ void (*clean_vf_config)(struct hnae3_ae_dev *ae_dev, int num_vfs);
+ };
+
+ struct hnae3_dcb_ops {
+@@ -830,6 +833,9 @@ struct hnae3_handle {
+
+ u8 netdev_flags;
+ struct dentry *hnae3_dbgfs;
++ /* protects concurrent contention between debugfs commands */
++ struct mutex dbgfs_lock;
++ char **dbgfs_buf;
+
+ /* Network interface message level enabled bits */
+ u32 msg_enable;
+@@ -850,6 +856,20 @@ struct hnae3_handle {
+ #define hnae3_get_bit(origin, shift) \
+ hnae3_get_field(origin, 0x1 << (shift), shift)
+
++#define HNAE3_FORMAT_MAC_ADDR_LEN 18
++#define HNAE3_FORMAT_MAC_ADDR_OFFSET_0 0
++#define HNAE3_FORMAT_MAC_ADDR_OFFSET_4 4
++#define HNAE3_FORMAT_MAC_ADDR_OFFSET_5 5
++
++static inline void hnae3_format_mac_addr(char *format_mac_addr,
++ const u8 *mac_addr)
++{
++ snprintf(format_mac_addr, HNAE3_FORMAT_MAC_ADDR_LEN, "%02x:**:**:**:%02x:%02x",
++ mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_0],
++ mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_4],
++ mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_5]);
++}
++
+ int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev);
+ void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev);
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
+index e54f96251fea9..15ce1a33649ee 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
+@@ -1021,7 +1021,8 @@ static ssize_t hns3_dbg_read(struct file *filp, char __user *buffer,
+ if (ret)
+ return ret;
+
+- save_buf = &hns3_dbg_cmd[index].buf;
++ mutex_lock(&handle->dbgfs_lock);
++ save_buf = &handle->dbgfs_buf[index];
+
+ if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state) ||
+ test_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) {
+@@ -1033,15 +1034,15 @@ static ssize_t hns3_dbg_read(struct file *filp, char __user *buffer,
+ read_buf = *save_buf;
+ } else {
+ read_buf = kvzalloc(hns3_dbg_cmd[index].buf_len, GFP_KERNEL);
+- if (!read_buf)
+- return -ENOMEM;
++ if (!read_buf) {
++ ret = -ENOMEM;
++ goto out;
++ }
+
+ /* save the buffer addr until the last read operation */
+ *save_buf = read_buf;
+- }
+
+- /* get data ready for the first time to read */
+- if (!*ppos) {
++ /* get data ready for the first time to read */
+ ret = hns3_dbg_read_cmd(dbg_data, hns3_dbg_cmd[index].cmd,
+ read_buf, hns3_dbg_cmd[index].buf_len);
+ if (ret)
+@@ -1050,8 +1051,10 @@ static ssize_t hns3_dbg_read(struct file *filp, char __user *buffer,
+
+ size = simple_read_from_buffer(buffer, count, ppos, read_buf,
+ strlen(read_buf));
+- if (size > 0)
++ if (size > 0) {
++ mutex_unlock(&handle->dbgfs_lock);
+ return size;
++ }
+
+ out:
+ /* free the buffer for the last read operation */
+@@ -1060,6 +1063,7 @@ out:
+ *save_buf = NULL;
+ }
+
++ mutex_unlock(&handle->dbgfs_lock);
+ return ret;
+ }
+
+@@ -1123,6 +1127,13 @@ int hns3_dbg_init(struct hnae3_handle *handle)
+ int ret;
+ u32 i;
+
++ handle->dbgfs_buf = devm_kcalloc(&handle->pdev->dev,
++ ARRAY_SIZE(hns3_dbg_cmd),
++ sizeof(*handle->dbgfs_buf),
++ GFP_KERNEL);
++ if (!handle->dbgfs_buf)
++ return -ENOMEM;
++
+ hns3_dbg_dentry[HNS3_DBG_DENTRY_COMMON].dentry =
+ debugfs_create_dir(name, hns3_dbgfs_root);
+ handle->hnae3_dbgfs = hns3_dbg_dentry[HNS3_DBG_DENTRY_COMMON].dentry;
+@@ -1132,6 +1143,8 @@ int hns3_dbg_init(struct hnae3_handle *handle)
+ debugfs_create_dir(hns3_dbg_dentry[i].name,
+ handle->hnae3_dbgfs);
+
++ mutex_init(&handle->dbgfs_lock);
++
+ for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd); i++) {
+ if ((hns3_dbg_cmd[i].cmd == HNAE3_DBG_CMD_TM_NODES &&
+ ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2) ||
+@@ -1158,6 +1171,7 @@ int hns3_dbg_init(struct hnae3_handle *handle)
+ return 0;
+
+ out:
++ mutex_destroy(&handle->dbgfs_lock);
+ debugfs_remove_recursive(handle->hnae3_dbgfs);
+ handle->hnae3_dbgfs = NULL;
+ return ret;
+@@ -1168,11 +1182,12 @@ void hns3_dbg_uninit(struct hnae3_handle *handle)
+ u32 i;
+
+ for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd); i++)
+- if (hns3_dbg_cmd[i].buf) {
+- kvfree(hns3_dbg_cmd[i].buf);
+- hns3_dbg_cmd[i].buf = NULL;
++ if (handle->dbgfs_buf[i]) {
++ kvfree(handle->dbgfs_buf[i]);
++ handle->dbgfs_buf[i] = NULL;
+ }
+
++ mutex_destroy(&handle->dbgfs_lock);
+ debugfs_remove_recursive(handle->hnae3_dbgfs);
+ handle->hnae3_dbgfs = NULL;
+ }
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h
+index bd8801065e024..814f7491ca08d 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h
+@@ -47,7 +47,6 @@ struct hns3_dbg_cmd_info {
+ enum hnae3_dbg_cmd cmd;
+ enum hns3_dbg_dentry_type dentry;
+ u32 buf_len;
+- char *buf;
+ int (*init)(struct hnae3_handle *handle, unsigned int cmd);
+ };
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+index 4b886a13e0797..818a028703c65 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+@@ -2255,6 +2255,8 @@ out_err_tx_ok:
+
+ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
+ {
++ char format_mac_addr_perm[HNAE3_FORMAT_MAC_ADDR_LEN];
++ char format_mac_addr_sa[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hnae3_handle *h = hns3_get_handle(netdev);
+ struct sockaddr *mac_addr = p;
+ int ret;
+@@ -2263,8 +2265,9 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
+ return -EADDRNOTAVAIL;
+
+ if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
+- netdev_info(netdev, "already using mac address %pM\n",
+- mac_addr->sa_data);
++ hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data);
++ netdev_info(netdev, "already using mac address %s\n",
++ format_mac_addr_sa);
+ return 0;
+ }
+
+@@ -2273,8 +2276,10 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
+ */
+ if (!hns3_is_phys_func(h->pdev) &&
+ !is_zero_ether_addr(netdev->perm_addr)) {
+- netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
+- netdev->perm_addr, mac_addr->sa_data);
++ hnae3_format_mac_addr(format_mac_addr_perm, netdev->perm_addr);
++ hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data);
++ netdev_err(netdev, "has permanent MAC %s, user MAC %s not allow\n",
++ format_mac_addr_perm, format_mac_addr_sa);
+ return -EPERM;
+ }
+
+@@ -2836,14 +2841,16 @@ static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
+ static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
+ {
+ struct hnae3_handle *h = hns3_get_handle(netdev);
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+
+ if (!h->ae_algo->ops->set_vf_mac)
+ return -EOPNOTSUPP;
+
+ if (is_multicast_ether_addr(mac)) {
++ hnae3_format_mac_addr(format_mac_addr, mac);
+ netdev_err(netdev,
+- "Invalid MAC:%pM specified. Could not set MAC\n",
+- mac);
++ "Invalid MAC:%s specified. Could not set MAC\n",
++ format_mac_addr);
+ return -EINVAL;
+ }
+
+@@ -2947,6 +2954,21 @@ static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ return ret;
+ }
+
++/**
++ * hns3_clean_vf_config
++ * @pdev: pointer to a pci_dev structure
++ * @num_vfs: number of VFs allocated
++ *
++ * Clean residual vf config after disable sriov
++ **/
++static void hns3_clean_vf_config(struct pci_dev *pdev, int num_vfs)
++{
++ struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
++
++ if (ae_dev->ops->clean_vf_config)
++ ae_dev->ops->clean_vf_config(ae_dev, num_vfs);
++}
++
+ /* hns3_remove - Device removal routine
+ * @pdev: PCI device information struct
+ */
+@@ -2985,7 +3007,10 @@ static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
+ else
+ return num_vfs;
+ } else if (!pci_vfs_assigned(pdev)) {
++ int num_vfs_pre = pci_num_vf(pdev);
++
+ pci_disable_sriov(pdev);
++ hns3_clean_vf_config(pdev, num_vfs_pre);
+ } else {
+ dev_warn(&pdev->dev,
+ "Unable to free VFs because some are assigned to VMs.\n");
+@@ -4927,6 +4952,7 @@ static void hns3_uninit_all_ring(struct hns3_nic_priv *priv)
+ static int hns3_init_mac_addr(struct net_device *netdev)
+ {
+ struct hns3_nic_priv *priv = netdev_priv(netdev);
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hnae3_handle *h = priv->ae_handle;
+ u8 mac_addr_temp[ETH_ALEN];
+ int ret = 0;
+@@ -4937,8 +4963,9 @@ static int hns3_init_mac_addr(struct net_device *netdev)
+ /* Check if the MAC address is valid, if not get a random one */
+ if (!is_valid_ether_addr(mac_addr_temp)) {
+ eth_hw_addr_random(netdev);
+- dev_warn(priv->dev, "using random MAC address %pM\n",
+- netdev->dev_addr);
++ hnae3_format_mac_addr(format_mac_addr, netdev->dev_addr);
++ dev_warn(priv->dev, "using random MAC address %s\n",
++ format_mac_addr);
+ } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) {
+ ether_addr_copy(netdev->dev_addr, mac_addr_temp);
+ ether_addr_copy(netdev->perm_addr, mac_addr_temp);
+@@ -4990,8 +5017,10 @@ static void hns3_client_stop(struct hnae3_handle *handle)
+ static void hns3_info_show(struct hns3_nic_priv *priv)
+ {
+ struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+
+- dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr);
++ hnae3_format_mac_addr(format_mac_addr, priv->netdev->dev_addr);
++ dev_info(priv->dev, "MAC address: %s\n", format_mac_addr);
+ dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps);
+ dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size);
+ dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size);
+@@ -5063,6 +5092,13 @@ static void hns3_state_init(struct hnae3_handle *handle)
+ set_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state);
+ }
+
++static void hns3_state_uninit(struct hnae3_handle *handle)
++{
++ struct hns3_nic_priv *priv = handle->priv;
++
++ clear_bit(HNS3_NIC_STATE_INITED, &priv->state);
++}
++
+ static int hns3_client_init(struct hnae3_handle *handle)
+ {
+ struct pci_dev *pdev = handle->pdev;
+@@ -5180,7 +5216,9 @@ static int hns3_client_init(struct hnae3_handle *handle)
+ return ret;
+
+ out_reg_netdev_fail:
++ hns3_state_uninit(handle);
+ hns3_dbg_uninit(handle);
++ hns3_client_stop(handle);
+ out_client_start:
+ hns3_free_rx_cpu_rmap(netdev);
+ hns3_nic_uninit_irq(priv);
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+index 5ebd96f6833d6..526fb56c84f24 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+@@ -985,6 +985,7 @@ static int hns3_set_reset(struct net_device *netdev, u32 *flags)
+ struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev);
+ const struct hnae3_ae_ops *ops = h->ae_algo->ops;
+ const struct hns3_reset_type_map *rst_type_map;
++ enum ethtool_reset_flags rst_flags;
+ u32 i, size;
+
+ if (ops->ae_dev_resetting && ops->ae_dev_resetting(h))
+@@ -1004,6 +1005,7 @@ static int hns3_set_reset(struct net_device *netdev, u32 *flags)
+ for (i = 0; i < size; i++) {
+ if (rst_type_map[i].rst_flags == *flags) {
+ rst_type = rst_type_map[i].rst_type;
++ rst_flags = rst_type_map[i].rst_flags;
+ break;
+ }
+ }
+@@ -1019,6 +1021,8 @@ static int hns3_set_reset(struct net_device *netdev, u32 *flags)
+
+ ops->reset_event(h->pdev, h);
+
++ *flags &= ~rst_flags;
++
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
+index 91cb578f56b80..375ebf105a9aa 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
+@@ -129,7 +129,7 @@ static int hclge_ets_sch_mode_validate(struct hclge_dev *hdev,
+ u32 total_ets_bw = 0;
+ u8 i;
+
+- for (i = 0; i < hdev->tc_max; i++) {
++ for (i = 0; i < HNAE3_MAX_TC; i++) {
+ switch (ets->tc_tsa[i]) {
+ case IEEE_8021QAZ_TSA_STRICT:
+ if (hdev->tm_info.tc_info[i].tc_sch_mode !=
+@@ -286,28 +286,24 @@ err_out:
+
+ static int hclge_ieee_getpfc(struct hnae3_handle *h, struct ieee_pfc *pfc)
+ {
+- u64 requests[HNAE3_MAX_TC], indications[HNAE3_MAX_TC];
+ struct hclge_vport *vport = hclge_get_vport(h);
+ struct hclge_dev *hdev = vport->back;
+ int ret;
+- u8 i;
+
+ memset(pfc, 0, sizeof(*pfc));
+ pfc->pfc_cap = hdev->pfc_max;
+ pfc->pfc_en = hdev->tm_info.pfc_en;
+
+- ret = hclge_pfc_tx_stats_get(hdev, requests);
+- if (ret)
++ ret = hclge_mac_update_stats(hdev);
++ if (ret) {
++ dev_err(&hdev->pdev->dev,
++ "failed to update MAC stats, ret = %d.\n", ret);
+ return ret;
++ }
+
+- ret = hclge_pfc_rx_stats_get(hdev, indications);
+- if (ret)
+- return ret;
++ hclge_pfc_tx_stats_get(hdev, pfc->requests);
++ hclge_pfc_rx_stats_get(hdev, pfc->indications);
+
+- for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+- pfc->requests[i] = requests[i];
+- pfc->indications[i] = indications[i];
+- }
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+index d891390d492f6..892f2f12c54c6 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+@@ -26,8 +26,6 @@
+ #include "hclge_devlink.h"
+
+ #define HCLGE_NAME "hclge"
+-#define HCLGE_STATS_READ(p, offset) (*(u64 *)((u8 *)(p) + (offset)))
+-#define HCLGE_MAC_STATS_FIELD_OFF(f) (offsetof(struct hclge_mac_stats, f))
+
+ #define HCLGE_BUF_SIZE_UNIT 256U
+ #define HCLGE_BUF_MUL_BY 2
+@@ -548,7 +546,7 @@ static int hclge_mac_query_reg_num(struct hclge_dev *hdev, u32 *desc_num)
+ return 0;
+ }
+
+-static int hclge_mac_update_stats(struct hclge_dev *hdev)
++int hclge_mac_update_stats(struct hclge_dev *hdev)
+ {
+ u32 desc_num;
+ int ret;
+@@ -1865,6 +1863,7 @@ static int hclge_alloc_vport(struct hclge_dev *hdev)
+ vport->vf_info.link_state = IFLA_VF_LINK_STATE_AUTO;
+ vport->mps = HCLGE_MAC_DEFAULT_FRAME;
+ vport->port_base_vlan_cfg.state = HNAE3_PORT_BASE_VLAN_DISABLE;
++ vport->port_base_vlan_cfg.tbl_sta = true;
+ vport->rxvlan_cfg.rx_vlan_offload_en = true;
+ vport->req_vlan_fltr_en = true;
+ INIT_LIST_HEAD(&vport->vlan_list);
+@@ -2498,7 +2497,7 @@ static int hclge_init_roce_base_info(struct hclge_vport *vport)
+ if (hdev->num_msi < hdev->num_nic_msi + hdev->num_roce_msi)
+ return -EINVAL;
+
+- roce->rinfo.base_vector = hdev->roce_base_vector;
++ roce->rinfo.base_vector = hdev->num_nic_msi;
+
+ roce->rinfo.netdev = nic->kinfo.netdev;
+ roce->rinfo.roce_io_base = hdev->hw.io_base;
+@@ -2534,10 +2533,6 @@ static int hclge_init_msi(struct hclge_dev *hdev)
+ hdev->num_msi = vectors;
+ hdev->num_msi_left = vectors;
+
+- hdev->base_msi_vector = pdev->irq;
+- hdev->roce_base_vector = hdev->base_msi_vector +
+- hdev->num_nic_msi;
+-
+ hdev->vector_status = devm_kcalloc(&pdev->dev, hdev->num_msi,
+ sizeof(u16), GFP_KERNEL);
+ if (!hdev->vector_status) {
+@@ -8575,6 +8570,7 @@ int hclge_update_mac_list(struct hclge_vport *vport,
+ enum HCLGE_MAC_ADDR_TYPE mac_type,
+ const unsigned char *addr)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ struct hclge_mac_node *mac_node;
+ struct list_head *list;
+@@ -8599,9 +8595,10 @@ int hclge_update_mac_list(struct hclge_vport *vport,
+ /* if this address is never added, unnecessary to delete */
+ if (state == HCLGE_MAC_TO_DEL) {
+ spin_unlock_bh(&vport->mac_list_lock);
++ hnae3_format_mac_addr(format_mac_addr, addr);
+ dev_err(&hdev->pdev->dev,
+- "failed to delete address %pM from mac list\n",
+- addr);
++ "failed to delete address %s from mac list\n",
++ format_mac_addr);
+ return -ENOENT;
+ }
+
+@@ -8634,6 +8631,7 @@ static int hclge_add_uc_addr(struct hnae3_handle *handle,
+ int hclge_add_uc_addr_common(struct hclge_vport *vport,
+ const unsigned char *addr)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ struct hclge_mac_vlan_tbl_entry_cmd req;
+ struct hclge_desc desc;
+@@ -8644,9 +8642,10 @@ int hclge_add_uc_addr_common(struct hclge_vport *vport,
+ if (is_zero_ether_addr(addr) ||
+ is_broadcast_ether_addr(addr) ||
+ is_multicast_ether_addr(addr)) {
++ hnae3_format_mac_addr(format_mac_addr, addr);
+ dev_err(&hdev->pdev->dev,
+- "Set_uc mac err! invalid mac:%pM. is_zero:%d,is_br=%d,is_mul=%d\n",
+- addr, is_zero_ether_addr(addr),
++ "Set_uc mac err! invalid mac:%s. is_zero:%d,is_br=%d,is_mul=%d\n",
++ format_mac_addr, is_zero_ether_addr(addr),
+ is_broadcast_ether_addr(addr),
+ is_multicast_ether_addr(addr));
+ return -EINVAL;
+@@ -8703,6 +8702,7 @@ static int hclge_rm_uc_addr(struct hnae3_handle *handle,
+ int hclge_rm_uc_addr_common(struct hclge_vport *vport,
+ const unsigned char *addr)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ struct hclge_mac_vlan_tbl_entry_cmd req;
+ int ret;
+@@ -8711,8 +8711,9 @@ int hclge_rm_uc_addr_common(struct hclge_vport *vport,
+ if (is_zero_ether_addr(addr) ||
+ is_broadcast_ether_addr(addr) ||
+ is_multicast_ether_addr(addr)) {
+- dev_dbg(&hdev->pdev->dev, "Remove mac err! invalid mac:%pM.\n",
+- addr);
++ hnae3_format_mac_addr(format_mac_addr, addr);
++ dev_dbg(&hdev->pdev->dev, "Remove mac err! invalid mac:%s.\n",
++ format_mac_addr);
+ return -EINVAL;
+ }
+
+@@ -8720,12 +8721,11 @@ int hclge_rm_uc_addr_common(struct hclge_vport *vport,
+ hnae3_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+ hclge_prepare_mac_addr(&req, addr, false);
+ ret = hclge_remove_mac_vlan_tbl(vport, &req);
+- if (!ret) {
++ if (!ret || ret == -ENOENT) {
+ mutex_lock(&hdev->vport_lock);
+ hclge_update_umv_space(vport, true);
+ mutex_unlock(&hdev->vport_lock);
+- } else if (ret == -ENOENT) {
+- ret = 0;
++ return 0;
+ }
+
+ return ret;
+@@ -8743,6 +8743,7 @@ static int hclge_add_mc_addr(struct hnae3_handle *handle,
+ int hclge_add_mc_addr_common(struct hclge_vport *vport,
+ const unsigned char *addr)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ struct hclge_mac_vlan_tbl_entry_cmd req;
+ struct hclge_desc desc[3];
+@@ -8750,9 +8751,10 @@ int hclge_add_mc_addr_common(struct hclge_vport *vport,
+
+ /* mac addr check */
+ if (!is_multicast_ether_addr(addr)) {
++ hnae3_format_mac_addr(format_mac_addr, addr);
+ dev_err(&hdev->pdev->dev,
+- "Add mc mac err! invalid mac:%pM.\n",
+- addr);
++ "Add mc mac err! invalid mac:%s.\n",
++ format_mac_addr);
+ return -EINVAL;
+ }
+ memset(&req, 0, sizeof(req));
+@@ -8788,6 +8790,7 @@ static int hclge_rm_mc_addr(struct hnae3_handle *handle,
+ int hclge_rm_mc_addr_common(struct hclge_vport *vport,
+ const unsigned char *addr)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ struct hclge_mac_vlan_tbl_entry_cmd req;
+ enum hclge_cmd_status status;
+@@ -8795,9 +8798,10 @@ int hclge_rm_mc_addr_common(struct hclge_vport *vport,
+
+ /* mac addr check */
+ if (!is_multicast_ether_addr(addr)) {
++ hnae3_format_mac_addr(format_mac_addr, addr);
+ dev_dbg(&hdev->pdev->dev,
+- "Remove mc mac err! invalid mac:%pM.\n",
+- addr);
++ "Remove mc mac err! invalid mac:%s.\n",
++ format_mac_addr);
+ return -EINVAL;
+ }
+
+@@ -9263,16 +9267,18 @@ static int hclge_set_vf_mac(struct hnae3_handle *handle, int vf,
+ u8 *mac_addr)
+ {
+ struct hclge_vport *vport = hclge_get_vport(handle);
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+
+ vport = hclge_get_vf_vport(hdev, vf);
+ if (!vport)
+ return -EINVAL;
+
++ hnae3_format_mac_addr(format_mac_addr, mac_addr);
+ if (ether_addr_equal(mac_addr, vport->vf_info.mac)) {
+ dev_info(&hdev->pdev->dev,
+- "Specified MAC(=%pM) is same as before, no change committed!\n",
+- mac_addr);
++ "Specified MAC(=%s) is same as before, no change committed!\n",
++ format_mac_addr);
+ return 0;
+ }
+
+@@ -9284,15 +9290,20 @@ static int hclge_set_vf_mac(struct hnae3_handle *handle, int vf,
+
+ ether_addr_copy(vport->vf_info.mac, mac_addr);
+
++ /* there is a timewindow for PF to know VF unalive, it may
++ * cause send mailbox fail, but it doesn't matter, VF will
++ * query it when reinit.
++ */
+ if (test_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state)) {
+ dev_info(&hdev->pdev->dev,
+- "MAC of VF %d has been set to %pM, and it will be reinitialized!\n",
+- vf, mac_addr);
+- return hclge_inform_reset_assert_to_vf(vport);
++ "MAC of VF %d has been set to %s, and it will be reinitialized!\n",
++ vf, format_mac_addr);
++ (void)hclge_inform_reset_assert_to_vf(vport);
++ return 0;
+ }
+
+- dev_info(&hdev->pdev->dev, "MAC of VF %d has been set to %pM\n",
+- vf, mac_addr);
++ dev_info(&hdev->pdev->dev, "MAC of VF %d has been set to %s\n",
++ vf, format_mac_addr);
+ return 0;
+ }
+
+@@ -9396,6 +9407,7 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p,
+ {
+ const unsigned char *new_addr = (const unsigned char *)p;
+ struct hclge_vport *vport = hclge_get_vport(handle);
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclge_dev *hdev = vport->back;
+ unsigned char *old_addr = NULL;
+ int ret;
+@@ -9404,9 +9416,10 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p,
+ if (is_zero_ether_addr(new_addr) ||
+ is_broadcast_ether_addr(new_addr) ||
+ is_multicast_ether_addr(new_addr)) {
++ hnae3_format_mac_addr(format_mac_addr, new_addr);
+ dev_err(&hdev->pdev->dev,
+- "change uc mac err! invalid mac: %pM.\n",
+- new_addr);
++ "change uc mac err! invalid mac: %s.\n",
++ format_mac_addr);
+ return -EINVAL;
+ }
+
+@@ -9424,9 +9437,10 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p,
+ spin_lock_bh(&vport->mac_list_lock);
+ ret = hclge_update_mac_node_for_dev_addr(vport, old_addr, new_addr);
+ if (ret) {
++ hnae3_format_mac_addr(format_mac_addr, new_addr);
+ dev_err(&hdev->pdev->dev,
+- "failed to change the mac addr:%pM, ret = %d\n",
+- new_addr, ret);
++ "failed to change the mac addr:%s, ret = %d\n",
++ format_mac_addr, ret);
+ spin_unlock_bh(&vport->mac_list_lock);
+
+ if (!is_first)
+@@ -10084,19 +10098,28 @@ static void hclge_add_vport_vlan_table(struct hclge_vport *vport, u16 vlan_id,
+ bool writen_to_tbl)
+ {
+ struct hclge_vport_vlan_cfg *vlan, *tmp;
++ struct hclge_dev *hdev = vport->back;
+
+- list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node)
+- if (vlan->vlan_id == vlan_id)
++ mutex_lock(&hdev->vport_lock);
++
++ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
++ if (vlan->vlan_id == vlan_id) {
++ mutex_unlock(&hdev->vport_lock);
+ return;
++ }
++ }
+
+ vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
+- if (!vlan)
++ if (!vlan) {
++ mutex_unlock(&hdev->vport_lock);
+ return;
++ }
+
+ vlan->hd_tbl_status = writen_to_tbl;
+ vlan->vlan_id = vlan_id;
+
+ list_add_tail(&vlan->node, &vport->vlan_list);
++ mutex_unlock(&hdev->vport_lock);
+ }
+
+ static int hclge_add_vport_all_vlan_table(struct hclge_vport *vport)
+@@ -10105,6 +10128,8 @@ static int hclge_add_vport_all_vlan_table(struct hclge_vport *vport)
+ struct hclge_dev *hdev = vport->back;
+ int ret;
+
++ mutex_lock(&hdev->vport_lock);
++
+ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
+ if (!vlan->hd_tbl_status) {
+ ret = hclge_set_vlan_filter_hw(hdev, htons(ETH_P_8021Q),
+@@ -10114,12 +10139,16 @@ static int hclge_add_vport_all_vlan_table(struct hclge_vport *vport)
+ dev_err(&hdev->pdev->dev,
+ "restore vport vlan list failed, ret=%d\n",
+ ret);
++
++ mutex_unlock(&hdev->vport_lock);
+ return ret;
+ }
+ }
+ vlan->hd_tbl_status = true;
+ }
+
++ mutex_unlock(&hdev->vport_lock);
++
+ return 0;
+ }
+
+@@ -10129,6 +10158,8 @@ static void hclge_rm_vport_vlan_table(struct hclge_vport *vport, u16 vlan_id,
+ struct hclge_vport_vlan_cfg *vlan, *tmp;
+ struct hclge_dev *hdev = vport->back;
+
++ mutex_lock(&hdev->vport_lock);
++
+ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
+ if (vlan->vlan_id == vlan_id) {
+ if (is_write_tbl && vlan->hd_tbl_status)
+@@ -10143,6 +10174,8 @@ static void hclge_rm_vport_vlan_table(struct hclge_vport *vport, u16 vlan_id,
+ break;
+ }
+ }
++
++ mutex_unlock(&hdev->vport_lock);
+ }
+
+ void hclge_rm_vport_all_vlan_table(struct hclge_vport *vport, bool is_del_list)
+@@ -10150,6 +10183,8 @@ void hclge_rm_vport_all_vlan_table(struct hclge_vport *vport, bool is_del_list)
+ struct hclge_vport_vlan_cfg *vlan, *tmp;
+ struct hclge_dev *hdev = vport->back;
+
++ mutex_lock(&hdev->vport_lock);
++
+ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
+ if (vlan->hd_tbl_status)
+ hclge_set_vlan_filter_hw(hdev,
+@@ -10165,6 +10200,7 @@ void hclge_rm_vport_all_vlan_table(struct hclge_vport *vport, bool is_del_list)
+ }
+ }
+ clear_bit(vport->vport_id, hdev->vf_vlan_full);
++ mutex_unlock(&hdev->vport_lock);
+ }
+
+ void hclge_uninit_vport_vlan_table(struct hclge_dev *hdev)
+@@ -10173,6 +10209,8 @@ void hclge_uninit_vport_vlan_table(struct hclge_dev *hdev)
+ struct hclge_vport *vport;
+ int i;
+
++ mutex_lock(&hdev->vport_lock);
++
+ for (i = 0; i < hdev->num_alloc_vport; i++) {
+ vport = &hdev->vport[i];
+ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
+@@ -10180,37 +10218,61 @@ void hclge_uninit_vport_vlan_table(struct hclge_dev *hdev)
+ kfree(vlan);
+ }
+ }
++
++ mutex_unlock(&hdev->vport_lock);
+ }
+
+-void hclge_restore_vport_vlan_table(struct hclge_vport *vport)
++void hclge_restore_vport_port_base_vlan_config(struct hclge_dev *hdev)
+ {
+- struct hclge_vport_vlan_cfg *vlan, *tmp;
+- struct hclge_dev *hdev = vport->back;
++ struct hclge_vlan_info *vlan_info;
++ struct hclge_vport *vport;
+ u16 vlan_proto;
+ u16 vlan_id;
+ u16 state;
++ int vf_id;
+ int ret;
+
+- vlan_proto = vport->port_base_vlan_cfg.vlan_info.vlan_proto;
+- vlan_id = vport->port_base_vlan_cfg.vlan_info.vlan_tag;
+- state = vport->port_base_vlan_cfg.state;
++ /* PF should restore all vfs port base vlan */
++ for (vf_id = 0; vf_id < hdev->num_alloc_vfs; vf_id++) {
++ vport = &hdev->vport[vf_id + HCLGE_VF_VPORT_START_NUM];
++ vlan_info = vport->port_base_vlan_cfg.tbl_sta ?
++ &vport->port_base_vlan_cfg.vlan_info :
++ &vport->port_base_vlan_cfg.old_vlan_info;
+
+- if (state != HNAE3_PORT_BASE_VLAN_DISABLE) {
+- clear_bit(vport->vport_id, hdev->vlan_table[vlan_id]);
+- hclge_set_vlan_filter_hw(hdev, htons(vlan_proto),
+- vport->vport_id, vlan_id,
+- false);
+- return;
++ vlan_id = vlan_info->vlan_tag;
++ vlan_proto = vlan_info->vlan_proto;
++ state = vport->port_base_vlan_cfg.state;
++
++ if (state != HNAE3_PORT_BASE_VLAN_DISABLE) {
++ clear_bit(vport->vport_id, hdev->vlan_table[vlan_id]);
++ ret = hclge_set_vlan_filter_hw(hdev, htons(vlan_proto),
++ vport->vport_id,
++ vlan_id, false);
++ vport->port_base_vlan_cfg.tbl_sta = ret == 0;
++ }
+ }
++}
+
+- list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
+- ret = hclge_set_vlan_filter_hw(hdev, htons(ETH_P_8021Q),
+- vport->vport_id,
+- vlan->vlan_id, false);
+- if (ret)
+- break;
+- vlan->hd_tbl_status = true;
++void hclge_restore_vport_vlan_table(struct hclge_vport *vport)
++{
++ struct hclge_vport_vlan_cfg *vlan, *tmp;
++ struct hclge_dev *hdev = vport->back;
++ int ret;
++
++ mutex_lock(&hdev->vport_lock);
++
++ if (vport->port_base_vlan_cfg.state == HNAE3_PORT_BASE_VLAN_DISABLE) {
++ list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) {
++ ret = hclge_set_vlan_filter_hw(hdev, htons(ETH_P_8021Q),
++ vport->vport_id,
++ vlan->vlan_id, false);
++ if (ret)
++ break;
++ vlan->hd_tbl_status = true;
++ }
+ }
++
++ mutex_unlock(&hdev->vport_lock);
+ }
+
+ /* For global reset and imp reset, hardware will clear the mac table,
+@@ -10250,6 +10312,7 @@ static void hclge_restore_hw_table(struct hclge_dev *hdev)
+ struct hnae3_handle *handle = &vport->nic;
+
+ hclge_restore_mac_table_common(vport);
++ hclge_restore_vport_port_base_vlan_config(hdev);
+ hclge_restore_vport_vlan_table(vport);
+ set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state);
+ hclge_restore_fd_entries(handle);
+@@ -10306,6 +10369,8 @@ static int hclge_update_vlan_filter_entries(struct hclge_vport *vport,
+ false);
+ }
+
++ vport->port_base_vlan_cfg.tbl_sta = false;
++
+ /* force add VLAN 0 */
+ ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, false, 0);
+ if (ret)
+@@ -10392,7 +10457,9 @@ out:
+ else
+ nic->port_base_vlan_state = HNAE3_PORT_BASE_VLAN_ENABLE;
+
++ vport->port_base_vlan_cfg.old_vlan_info = *old_vlan_info;
+ vport->port_base_vlan_cfg.vlan_info = *vlan_info;
++ vport->port_base_vlan_cfg.tbl_sta = true;
+ hclge_set_vport_vlan_fltr_change(vport);
+
+ return 0;
+@@ -10460,14 +10527,17 @@ static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid,
+ return ret;
+ }
+
+- /* for DEVICE_VERSION_V3, vf doesn't need to know about the port based
++ /* there is a timewindow for PF to know VF unalive, it may
++ * cause send mailbox fail, but it doesn't matter, VF will
++ * query it when reinit.
++ * for DEVICE_VERSION_V3, vf doesn't need to know about the port based
+ * VLAN state.
+ */
+ if (ae_dev->dev_version < HNAE3_DEVICE_VERSION_V3 &&
+ test_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state))
+- hclge_push_vf_port_base_vlan_info(&hdev->vport[0],
+- vport->vport_id, state,
+- &vlan_info);
++ (void)hclge_push_vf_port_base_vlan_info(&hdev->vport[0],
++ vport->vport_id,
++ state, &vlan_info);
+
+ return 0;
+ }
+@@ -10525,11 +10595,11 @@ int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto,
+ }
+
+ if (!ret) {
+- if (is_kill)
+- hclge_rm_vport_vlan_table(vport, vlan_id, false);
+- else
++ if (!is_kill)
+ hclge_add_vport_vlan_table(vport, vlan_id,
+ writen_to_tbl);
++ else if (is_kill && vlan_id != 0)
++ hclge_rm_vport_vlan_table(vport, vlan_id, false);
+ } else if (is_kill) {
+ /* when remove hw vlan filter failed, record the vlan id,
+ * and try to remove it from hw later, to be consistence
+@@ -12103,8 +12173,8 @@ static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
+ hclge_misc_irq_uninit(hdev);
+ hclge_devlink_uninit(hdev);
+ hclge_pci_uninit(hdev);
+- mutex_destroy(&hdev->vport_lock);
+ hclge_uninit_vport_vlan_table(hdev);
++ mutex_destroy(&hdev->vport_lock);
+ ae_dev->priv = NULL;
+ }
+
+@@ -12917,6 +12987,55 @@ static int hclge_get_link_diagnosis_info(struct hnae3_handle *handle,
+ return 0;
+ }
+
++/* After disable sriov, VF still has some config and info need clean,
++ * which configed by PF.
++ */
++static void hclge_clear_vport_vf_info(struct hclge_vport *vport, int vfid)
++{
++ struct hclge_dev *hdev = vport->back;
++ struct hclge_vlan_info vlan_info;
++ int ret;
++
++ /* after disable sriov, clean VF rate configured by PF */
++ ret = hclge_tm_qs_shaper_cfg(vport, 0);
++ if (ret)
++ dev_err(&hdev->pdev->dev,
++ "failed to clean vf%d rate config, ret = %d\n",
++ vfid, ret);
++
++ vlan_info.vlan_tag = 0;
++ vlan_info.qos = 0;
++ vlan_info.vlan_proto = ETH_P_8021Q;
++ ret = hclge_update_port_base_vlan_cfg(vport,
++ HNAE3_PORT_BASE_VLAN_DISABLE,
++ &vlan_info);
++ if (ret)
++ dev_err(&hdev->pdev->dev,
++ "failed to clean vf%d port base vlan, ret = %d\n",
++ vfid, ret);
++
++ ret = hclge_set_vf_spoofchk_hw(hdev, vport->vport_id, false);
++ if (ret)
++ dev_err(&hdev->pdev->dev,
++ "failed to clean vf%d spoof config, ret = %d\n",
++ vfid, ret);
++
++ memset(&vport->vf_info, 0, sizeof(vport->vf_info));
++}
++
++static void hclge_clean_vport_config(struct hnae3_ae_dev *ae_dev, int num_vfs)
++{
++ struct hclge_dev *hdev = ae_dev->priv;
++ struct hclge_vport *vport;
++ int i;
++
++ for (i = 0; i < num_vfs; i++) {
++ vport = &hdev->vport[i + HCLGE_VF_VPORT_START_NUM];
++
++ hclge_clear_vport_vf_info(vport, i);
++ }
++}
++
+ static const struct hnae3_ae_ops hclge_ops = {
+ .init_ae_dev = hclge_init_ae_dev,
+ .uninit_ae_dev = hclge_uninit_ae_dev,
+@@ -13018,6 +13137,7 @@ static const struct hnae3_ae_ops hclge_ops = {
+ .get_rx_hwts = hclge_ptp_get_rx_hwts,
+ .get_ts_info = hclge_ptp_get_ts_info,
+ .get_link_diagnosis_info = hclge_get_link_diagnosis_info,
++ .clean_vf_config = hclge_clean_vport_config,
+ };
+
+ static struct hnae3_ae_algo ae_algo = {
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+index 69cd8f87b4c86..4d6dbfe0be7a2 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+@@ -824,6 +824,9 @@ struct hclge_vf_vlan_cfg {
+ (y) = (_k_ ^ ~_v_) & (_k_); \
+ } while (0)
+
++#define HCLGE_MAC_STATS_FIELD_OFF(f) (offsetof(struct hclge_mac_stats, f))
++#define HCLGE_STATS_READ(p, offset) (*(u64 *)((u8 *)(p) + (offset)))
++
+ #define HCLGE_MAC_TNL_LOG_SIZE 8
+ #define HCLGE_VPORT_NUM 256
+ struct hclge_dev {
+@@ -876,12 +879,10 @@ struct hclge_dev {
+ u16 num_msi;
+ u16 num_msi_left;
+ u16 num_msi_used;
+- u32 base_msi_vector;
+ u16 *vector_status;
+ int *vector_irq;
+ u16 num_nic_msi; /* Num of nic vectors for this PF */
+ u16 num_roce_msi; /* Num of roce vectors for this PF */
+- int roce_base_vector;
+
+ unsigned long service_timer_period;
+ unsigned long service_timer_previous;
+@@ -999,7 +1000,9 @@ struct hclge_vlan_info {
+
+ struct hclge_port_base_vlan_config {
+ u16 state;
++ bool tbl_sta;
+ struct hclge_vlan_info vlan_info;
++ struct hclge_vlan_info old_vlan_info;
+ };
+
+ struct hclge_vf_info {
+@@ -1054,6 +1057,7 @@ struct hclge_vport {
+ spinlock_t mac_list_lock; /* protect mac address need to add/detele */
+ struct list_head uc_mac_list; /* Store VF unicast table */
+ struct list_head mc_mac_list; /* Store VF multicast table */
++
+ struct list_head vlan_list; /* Store VF vlan table */
+ };
+
+@@ -1123,6 +1127,7 @@ void hclge_rm_vport_all_mac_table(struct hclge_vport *vport, bool is_del_list,
+ void hclge_rm_vport_all_vlan_table(struct hclge_vport *vport, bool is_del_list);
+ void hclge_uninit_vport_vlan_table(struct hclge_dev *hdev);
+ void hclge_restore_mac_table_common(struct hclge_vport *vport);
++void hclge_restore_vport_port_base_vlan_config(struct hclge_dev *hdev);
+ void hclge_restore_vport_vlan_table(struct hclge_vport *vport);
+ int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state,
+ struct hclge_vlan_info *vlan_info);
+@@ -1138,4 +1143,5 @@ void hclge_inform_vf_promisc_info(struct hclge_vport *vport);
+ int hclge_dbg_dump_rst_info(struct hclge_dev *hdev, char *buf, int len);
+ int hclge_push_vf_link_status(struct hclge_vport *vport);
+ int hclge_enable_vport_vlan_filter(struct hclge_vport *vport, bool request_en);
++int hclge_mac_update_stats(struct hclge_dev *hdev);
+ #endif
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+index 65d78ee4d65a0..4a5b11b6fed3f 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+@@ -93,6 +93,13 @@ static int hclge_send_mbx_msg(struct hclge_vport *vport, u8 *msg, u16 msg_len,
+ enum hclge_cmd_status status;
+ struct hclge_desc desc;
+
++ if (msg_len > HCLGE_MBX_MAX_MSG_SIZE) {
++ dev_err(&hdev->pdev->dev,
++ "msg data length(=%u) exceeds maximum(=%u)\n",
++ msg_len, HCLGE_MBX_MAX_MSG_SIZE);
++ return -EMSGSIZE;
++ }
++
+ resp_pf_to_vf = (struct hclge_mbx_pf_to_vf_cmd *)desc.data;
+
+ hclge_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_PF_TO_VF, false);
+@@ -175,7 +182,7 @@ static int hclge_get_ring_chain_from_mbx(
+ ring_num = req->msg.ring_num;
+
+ if (ring_num > HCLGE_MBX_MAX_RING_CHAIN_PARAM_NUM)
+- return -ENOMEM;
++ return -EINVAL;
+
+ for (i = 0; i < ring_num; i++) {
+ if (req->msg.param[i].tqp_index >= vport->nic.kinfo.rss_size) {
+@@ -586,9 +593,9 @@ static int hclge_set_vf_mtu(struct hclge_vport *vport,
+ return hclge_set_vport_mtu(vport, mtu);
+ }
+
+-static void hclge_get_queue_id_in_pf(struct hclge_vport *vport,
+- struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+- struct hclge_respond_to_vf_msg *resp_msg)
++static int hclge_get_queue_id_in_pf(struct hclge_vport *vport,
++ struct hclge_mbx_vf_to_pf_cmd *mbx_req,
++ struct hclge_respond_to_vf_msg *resp_msg)
+ {
+ struct hnae3_handle *handle = &vport->nic;
+ struct hclge_dev *hdev = vport->back;
+@@ -598,17 +605,18 @@ static void hclge_get_queue_id_in_pf(struct hclge_vport *vport,
+ if (queue_id >= handle->kinfo.num_tqps) {
+ dev_err(&hdev->pdev->dev, "Invalid queue id(%u) from VF %u\n",
+ queue_id, mbx_req->mbx_src_vfid);
+- return;
++ return -EINVAL;
+ }
+
+ qid_in_pf = hclge_covert_handle_qid_global(&vport->nic, queue_id);
+ memcpy(resp_msg->data, &qid_in_pf, sizeof(qid_in_pf));
+ resp_msg->len = sizeof(qid_in_pf);
++ return 0;
+ }
+
+-static void hclge_get_rss_key(struct hclge_vport *vport,
+- struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+- struct hclge_respond_to_vf_msg *resp_msg)
++static int hclge_get_rss_key(struct hclge_vport *vport,
++ struct hclge_mbx_vf_to_pf_cmd *mbx_req,
++ struct hclge_respond_to_vf_msg *resp_msg)
+ {
+ #define HCLGE_RSS_MBX_RESP_LEN 8
+ struct hclge_dev *hdev = vport->back;
+@@ -624,13 +632,14 @@ static void hclge_get_rss_key(struct hclge_vport *vport,
+ dev_warn(&hdev->pdev->dev,
+ "failed to get the rss hash key, the index(%u) invalid !\n",
+ index);
+- return;
++ return -EINVAL;
+ }
+
+ memcpy(resp_msg->data,
+ &hdev->vport[0].rss_hash_key[index * HCLGE_RSS_MBX_RESP_LEN],
+ HCLGE_RSS_MBX_RESP_LEN);
+ resp_msg->len = HCLGE_RSS_MBX_RESP_LEN;
++ return 0;
+ }
+
+ static void hclge_link_fail_parse(struct hclge_dev *hdev, u8 link_fail_code)
+@@ -805,10 +814,10 @@ void hclge_mbx_handler(struct hclge_dev *hdev)
+ "VF fail(%d) to set mtu\n", ret);
+ break;
+ case HCLGE_MBX_GET_QID_IN_PF:
+- hclge_get_queue_id_in_pf(vport, req, &resp_msg);
++ ret = hclge_get_queue_id_in_pf(vport, req, &resp_msg);
+ break;
+ case HCLGE_MBX_GET_RSS_KEY:
+- hclge_get_rss_key(vport, req, &resp_msg);
++ ret = hclge_get_rss_key(vport, req, &resp_msg);
+ break;
+ case HCLGE_MBX_GET_LINK_MODE:
+ hclge_get_link_mode(vport, req);
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+index 95074e91a8466..429652a8cde16 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+@@ -113,50 +113,50 @@ static int hclge_shaper_para_calc(u32 ir, u8 shaper_level,
+ return 0;
+ }
+
+-static int hclge_pfc_stats_get(struct hclge_dev *hdev,
+- enum hclge_opcode_type opcode, u64 *stats)
+-{
+- struct hclge_desc desc[HCLGE_TM_PFC_PKT_GET_CMD_NUM];
+- int ret, i, j;
+-
+- if (!(opcode == HCLGE_OPC_QUERY_PFC_RX_PKT_CNT ||
+- opcode == HCLGE_OPC_QUERY_PFC_TX_PKT_CNT))
+- return -EINVAL;
+-
+- for (i = 0; i < HCLGE_TM_PFC_PKT_GET_CMD_NUM - 1; i++) {
+- hclge_cmd_setup_basic_desc(&desc[i], opcode, true);
+- desc[i].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+- }
+-
+- hclge_cmd_setup_basic_desc(&desc[i], opcode, true);
++static const u16 hclge_pfc_tx_stats_offset[] = {
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri0_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri1_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri2_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri3_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri4_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri5_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri6_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri7_pkt_num)
++};
+
+- ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_TM_PFC_PKT_GET_CMD_NUM);
+- if (ret)
+- return ret;
++static const u16 hclge_pfc_rx_stats_offset[] = {
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri0_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri1_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri2_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri3_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri4_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri5_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri6_pkt_num),
++ HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri7_pkt_num)
++};
+
+- for (i = 0; i < HCLGE_TM_PFC_PKT_GET_CMD_NUM; i++) {
+- struct hclge_pfc_stats_cmd *pfc_stats =
+- (struct hclge_pfc_stats_cmd *)desc[i].data;
++static void hclge_pfc_stats_get(struct hclge_dev *hdev, bool tx, u64 *stats)
++{
++ const u16 *offset;
++ int i;
+
+- for (j = 0; j < HCLGE_TM_PFC_NUM_GET_PER_CMD; j++) {
+- u32 index = i * HCLGE_TM_PFC_PKT_GET_CMD_NUM + j;
++ if (tx)
++ offset = hclge_pfc_tx_stats_offset;
++ else
++ offset = hclge_pfc_rx_stats_offset;
+
+- if (index < HCLGE_MAX_TC_NUM)
+- stats[index] =
+- le64_to_cpu(pfc_stats->pkt_num[j]);
+- }
+- }
+- return 0;
++ for (i = 0; i < HCLGE_MAX_TC_NUM; i++)
++ stats[i] = HCLGE_STATS_READ(&hdev->mac_stats, offset[i]);
+ }
+
+-int hclge_pfc_rx_stats_get(struct hclge_dev *hdev, u64 *stats)
++void hclge_pfc_rx_stats_get(struct hclge_dev *hdev, u64 *stats)
+ {
+- return hclge_pfc_stats_get(hdev, HCLGE_OPC_QUERY_PFC_RX_PKT_CNT, stats);
++ hclge_pfc_stats_get(hdev, false, stats);
+ }
+
+-int hclge_pfc_tx_stats_get(struct hclge_dev *hdev, u64 *stats)
++void hclge_pfc_tx_stats_get(struct hclge_dev *hdev, u64 *stats)
+ {
+- return hclge_pfc_stats_get(hdev, HCLGE_OPC_QUERY_PFC_TX_PKT_CNT, stats);
++ hclge_pfc_stats_get(hdev, true, stats);
+ }
+
+ int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx)
+@@ -1123,7 +1123,6 @@ static int hclge_tm_pri_tc_base_dwrr_cfg(struct hclge_dev *hdev)
+
+ static int hclge_tm_ets_tc_dwrr_cfg(struct hclge_dev *hdev)
+ {
+-#define DEFAULT_TC_WEIGHT 1
+ #define DEFAULT_TC_OFFSET 14
+
+ struct hclge_ets_tc_weight_cmd *ets_weight;
+@@ -1136,13 +1135,7 @@ static int hclge_tm_ets_tc_dwrr_cfg(struct hclge_dev *hdev)
+ for (i = 0; i < HNAE3_MAX_TC; i++) {
+ struct hclge_pg_info *pg_info;
+
+- ets_weight->tc_weight[i] = DEFAULT_TC_WEIGHT;
+-
+- if (!(hdev->hw_tc_map & BIT(i)))
+- continue;
+-
+- pg_info =
+- &hdev->tm_info.pg_info[hdev->tm_info.tc_info[i].pgid];
++ pg_info = &hdev->tm_info.pg_info[hdev->tm_info.tc_info[i].pgid];
+ ets_weight->tc_weight[i] = pg_info->tc_dwrr[i];
+ }
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
+index 2ee9b795f71dc..1db7f40b45255 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
+@@ -228,8 +228,8 @@ int hclge_tm_dwrr_cfg(struct hclge_dev *hdev);
+ int hclge_tm_init_hw(struct hclge_dev *hdev, bool init);
+ int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx);
+ int hclge_pause_addr_cfg(struct hclge_dev *hdev, const u8 *mac_addr);
+-int hclge_pfc_rx_stats_get(struct hclge_dev *hdev, u64 *stats);
+-int hclge_pfc_tx_stats_get(struct hclge_dev *hdev, u64 *stats);
++void hclge_pfc_rx_stats_get(struct hclge_dev *hdev, u64 *stats);
++void hclge_pfc_tx_stats_get(struct hclge_dev *hdev, u64 *stats);
+ int hclge_tm_qs_shaper_cfg(struct hclge_vport *vport, int max_tx_rate);
+ int hclge_tm_get_qset_num(struct hclge_dev *hdev, u16 *qset_num);
+ int hclge_tm_get_pri_num(struct hclge_dev *hdev, u8 *pri_num);
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+index cf00ad7bb881f..21678c12afa26 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+@@ -703,9 +703,9 @@ static int hclgevf_set_rss_tc_mode(struct hclgevf_dev *hdev, u16 rss_size)
+ roundup_size = ilog2(roundup_size);
+
+ for (i = 0; i < HCLGEVF_MAX_TC_NUM; i++) {
+- tc_valid[i] = !!(hdev->hw_tc_map & BIT(i));
++ tc_valid[i] = 1;
+ tc_size[i] = roundup_size;
+- tc_offset[i] = rss_size * i;
++ tc_offset[i] = (hdev->hw_tc_map & BIT(i)) ? rss_size * i : 0;
+ }
+
+ hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_RSS_TC_MODE, false);
+@@ -1514,15 +1514,18 @@ static void hclgevf_config_mac_list(struct hclgevf_dev *hdev,
+ struct list_head *list,
+ enum HCLGEVF_MAC_ADDR_TYPE mac_type)
+ {
++ char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];
+ struct hclgevf_mac_addr_node *mac_node, *tmp;
+ int ret;
+
+ list_for_each_entry_safe(mac_node, tmp, list, node) {
+ ret = hclgevf_add_del_mac_addr(hdev, mac_node, mac_type);
+ if (ret) {
++ hnae3_format_mac_addr(format_mac_addr,
++ mac_node->mac_addr);
+ dev_err(&hdev->pdev->dev,
+- "failed to configure mac %pM, state = %d, ret = %d\n",
+- mac_node->mac_addr, mac_node->state, ret);
++ "failed to configure mac %s, state = %d, ret = %d\n",
++ format_mac_addr, mac_node->state, ret);
+ return;
+ }
+ if (mac_node->state == HCLGEVF_MAC_TO_ADD) {
+@@ -2496,8 +2499,7 @@ static irqreturn_t hclgevf_misc_irq_handle(int irq, void *data)
+ break;
+ }
+
+- if (event_cause != HCLGEVF_VECTOR0_EVENT_OTHER)
+- hclgevf_enable_vector(&hdev->misc_vector, true);
++ hclgevf_enable_vector(&hdev->misc_vector, true);
+
+ return IRQ_HANDLED;
+ }
+@@ -2557,7 +2559,7 @@ static int hclgevf_init_roce_base_info(struct hclgevf_dev *hdev)
+ hdev->num_msi_left == 0)
+ return -EINVAL;
+
+- roce->rinfo.base_vector = hdev->roce_base_vector;
++ roce->rinfo.base_vector = hdev->roce_base_msix_offset;
+
+ roce->rinfo.netdev = nic->kinfo.netdev;
+ roce->rinfo.roce_io_base = hdev->hw.io_base;
+@@ -2823,9 +2825,6 @@ static int hclgevf_init_msi(struct hclgevf_dev *hdev)
+ hdev->num_msi = vectors;
+ hdev->num_msi_left = vectors;
+
+- hdev->base_msi_vector = pdev->irq;
+- hdev->roce_base_vector = pdev->irq + hdev->roce_base_msix_offset;
+-
+ hdev->vector_status = devm_kcalloc(&pdev->dev, hdev->num_msi,
+ sizeof(u16), GFP_KERNEL);
+ if (!hdev->vector_status) {
+@@ -3013,7 +3012,10 @@ static void hclgevf_uninit_client_instance(struct hnae3_client *client,
+
+ /* un-init roce, if it exists */
+ if (hdev->roce_client) {
++ while (test_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state))
++ msleep(HCLGEVF_WAIT_RESET_DONE);
+ clear_bit(HCLGEVF_STATE_ROCE_REGISTERED, &hdev->state);
++
+ hdev->roce_client->ops->uninit_instance(&hdev->roce, 0);
+ hdev->roce_client = NULL;
+ hdev->roce.client = NULL;
+@@ -3022,6 +3024,8 @@ static void hclgevf_uninit_client_instance(struct hnae3_client *client,
+ /* un-init nic/unic, if this was not called by roce client */
+ if (client->ops->uninit_instance && hdev->nic_client &&
+ client->type != HNAE3_CLIENT_ROCE) {
++ while (test_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state))
++ msleep(HCLGEVF_WAIT_RESET_DONE);
+ clear_bit(HCLGEVF_STATE_NIC_REGISTERED, &hdev->state);
+
+ client->ops->uninit_instance(&hdev->nic, 0);
+@@ -3340,6 +3344,11 @@ static int hclgevf_reset_hdev(struct hclgevf_dev *hdev)
+ return ret;
+ }
+
++ /* get current port based vlan state from PF */
++ ret = hclgevf_get_port_base_vlan_filter_state(hdev);
++ if (ret)
++ return ret;
++
+ set_bit(HCLGEVF_STATE_PROMISC_CHANGED, &hdev->state);
+
+ hclgevf_init_rxd_adv_layout(hdev);
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
+index 28288d7e33032..f6f736c0091c0 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
+@@ -109,6 +109,8 @@
+ #define HCLGEVF_VF_RST_ING 0x07008
+ #define HCLGEVF_VF_RST_ING_BIT BIT(16)
+
++#define HCLGEVF_WAIT_RESET_DONE 100
++
+ #define HCLGEVF_RSS_IND_TBL_SIZE 512
+ #define HCLGEVF_RSS_SET_BITMAP_MSK 0xffff
+ #define HCLGEVF_RSS_KEY_SIZE 40
+@@ -308,8 +310,6 @@ struct hclgevf_dev {
+ u16 num_nic_msix; /* Num of nic vectors for this VF */
+ u16 num_roce_msix; /* Num of roce vectors for this VF */
+ u16 roce_base_msix_offset;
+- int roce_base_vector;
+- u32 base_msi_vector;
+ u16 *vector_status;
+ int *vector_irq;
+
+diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
+index fdc66fae09601..c5ac6ecf36e10 100644
+--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
++++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
+@@ -114,7 +114,8 @@ int hclgevf_send_mbx_msg(struct hclgevf_dev *hdev,
+
+ memcpy(&req->msg, send_msg, sizeof(struct hclge_vf_to_pf_msg));
+
+- trace_hclge_vf_mbx_send(hdev, req);
++ if (test_bit(HCLGEVF_STATE_NIC_REGISTERED, &hdev->state))
++ trace_hclge_vf_mbx_send(hdev, req);
+
+ /* synchronous send */
+ if (need_resp) {
+diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
+index 7f0f1aa3cedd9..0c1b0a91b1aed 100644
+--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
++++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
+@@ -772,7 +772,7 @@ struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+ /* If we only have one page, still need to get shadown wqe when
+ * wqe rolling-over page
+ */
+- if (curr_pg != end_pg || MASKED_WQE_IDX(wq, end_prod_idx) < *prod_idx) {
++ if (curr_pg != end_pg || end_prod_idx < *prod_idx) {
+ void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
+
+ copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
+@@ -842,7 +842,10 @@ struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+
+ *cons_idx = curr_cons_idx;
+
+- if (curr_pg != end_pg) {
++ /* If we only have one page, still need to get shadown wqe when
++ * wqe rolling-over page
++ */
++ if (curr_pg != end_pg || end_cons_idx < curr_cons_idx) {
+ void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
+
+ copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);
+diff --git a/drivers/net/ethernet/i825xx/sni_82596.c b/drivers/net/ethernet/i825xx/sni_82596.c
+index 27937c5d79567..daec9ce04531b 100644
+--- a/drivers/net/ethernet/i825xx/sni_82596.c
++++ b/drivers/net/ethernet/i825xx/sni_82596.c
+@@ -117,9 +117,10 @@ static int sni_82596_probe(struct platform_device *dev)
+ netdevice->dev_addr[5] = readb(eth_addr + 0x06);
+ iounmap(eth_addr);
+
+- if (!netdevice->irq) {
++ if (netdevice->irq < 0) {
+ printk(KERN_ERR "%s: IRQ not found for i82596 at 0x%lx\n",
+ __FILE__, netdevice->base_addr);
++ retval = netdevice->irq;
+ goto probe_failed;
+ }
+
+diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
+index 6aa6ff89a7651..b262aa84b6a24 100644
+--- a/drivers/net/ethernet/ibm/ibmvnic.c
++++ b/drivers/net/ethernet/ibm/ibmvnic.c
+@@ -108,6 +108,7 @@ static int init_crq_queue(struct ibmvnic_adapter *adapter);
+ static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
+ static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
+ struct ibmvnic_sub_crq_queue *tx_scrq);
++static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter);
+
+ struct ibmvnic_stat {
+ char name[ETH_GSTRING_LEN];
+@@ -1245,10 +1246,19 @@ static int __ibmvnic_open(struct net_device *netdev)
+ rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
+ if (rc) {
+ ibmvnic_napi_disable(adapter);
+- release_resources(adapter);
++ ibmvnic_disable_irqs(adapter);
+ return rc;
+ }
+
++ adapter->tx_queues_active = true;
++
++ /* Since queues were stopped until now, there shouldn't be any
++ * one in ibmvnic_complete_tx() or ibmvnic_xmit() so maybe we
++ * don't need the synchronize_rcu()? Leaving it for consistency
++ * with setting ->tx_queues_active = false.
++ */
++ synchronize_rcu();
++
+ netif_tx_start_all_queues(netdev);
+
+ if (prev_state == VNIC_CLOSED) {
+@@ -1295,7 +1305,6 @@ static int ibmvnic_open(struct net_device *netdev)
+ rc = init_resources(adapter);
+ if (rc) {
+ netdev_err(netdev, "failed to initialize resources\n");
+- release_resources(adapter);
+ goto out;
+ }
+ }
+@@ -1312,6 +1321,11 @@ out:
+ adapter->state = VNIC_OPEN;
+ rc = 0;
+ }
++
++ if (rc) {
++ release_resources(adapter);
++ }
++
+ return rc;
+ }
+
+@@ -1417,6 +1431,14 @@ static void ibmvnic_cleanup(struct net_device *netdev)
+ struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+
+ /* ensure that transmissions are stopped if called by do_reset */
++
++ adapter->tx_queues_active = false;
++
++ /* Ensure complete_tx() and ibmvnic_xmit() see ->tx_queues_active
++ * update so they don't restart a queue after we stop it below.
++ */
++ synchronize_rcu();
++
+ if (test_bit(0, &adapter->resetting))
+ netif_tx_disable(netdev);
+ else
+@@ -1657,14 +1679,21 @@ static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
+ tx_buff->skb = NULL;
+ adapter->netdev->stats.tx_dropped++;
+ }
++
+ ind_bufp->index = 0;
++
+ if (atomic_sub_return(entries, &tx_scrq->used) <=
+ (adapter->req_tx_entries_per_subcrq / 2) &&
+- __netif_subqueue_stopped(adapter->netdev, queue_num) &&
+- !test_bit(0, &adapter->resetting)) {
+- netif_wake_subqueue(adapter->netdev, queue_num);
+- netdev_dbg(adapter->netdev, "Started queue %d\n",
+- queue_num);
++ __netif_subqueue_stopped(adapter->netdev, queue_num)) {
++ rcu_read_lock();
++
++ if (adapter->tx_queues_active) {
++ netif_wake_subqueue(adapter->netdev, queue_num);
++ netdev_dbg(adapter->netdev, "Started queue %d\n",
++ queue_num);
++ }
++
++ rcu_read_unlock();
+ }
+ }
+
+@@ -1719,13 +1748,12 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
+ int index = 0;
+ u8 proto = 0;
+
+- tx_scrq = adapter->tx_scrq[queue_num];
+- txq = netdev_get_tx_queue(netdev, queue_num);
+- ind_bufp = &tx_scrq->ind_buf;
+-
+- if (test_bit(0, &adapter->resetting)) {
+- if (!netif_subqueue_stopped(netdev, skb))
+- netif_stop_subqueue(netdev, queue_num);
++ /* If a reset is in progress, drop the packet since
++ * the scrqs may get torn down. Otherwise use the
++ * rcu to ensure reset waits for us to complete.
++ */
++ rcu_read_lock();
++ if (!adapter->tx_queues_active) {
+ dev_kfree_skb_any(skb);
+
+ tx_send_failed++;
+@@ -1734,6 +1762,10 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
+ goto out;
+ }
+
++ tx_scrq = adapter->tx_scrq[queue_num];
++ txq = netdev_get_tx_queue(netdev, queue_num);
++ ind_bufp = &tx_scrq->ind_buf;
++
+ if (ibmvnic_xmit_workarounds(skb, netdev)) {
+ tx_dropped++;
+ tx_send_failed++;
+@@ -1741,6 +1773,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
+ ibmvnic_tx_scrq_flush(adapter, tx_scrq);
+ goto out;
+ }
++
+ if (skb_is_gso(skb))
+ tx_pool = &adapter->tso_pool[queue_num];
+ else
+@@ -1895,6 +1928,7 @@ tx_err:
+ netif_carrier_off(netdev);
+ }
+ out:
++ rcu_read_unlock();
+ netdev->stats.tx_dropped += tx_dropped;
+ netdev->stats.tx_bytes += tx_bytes;
+ netdev->stats.tx_packets += tx_packets;
+@@ -2426,6 +2460,7 @@ static void __ibmvnic_reset(struct work_struct *work)
+ struct ibmvnic_rwi *rwi;
+ unsigned long flags;
+ u32 reset_state;
++ int num_fails = 0;
+ int rc = 0;
+
+ adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);
+@@ -2479,11 +2514,23 @@ static void __ibmvnic_reset(struct work_struct *work)
+ rc = do_hard_reset(adapter, rwi, reset_state);
+ rtnl_unlock();
+ }
+- if (rc) {
+- /* give backing device time to settle down */
++ if (rc)
++ num_fails++;
++ else
++ num_fails = 0;
++
++ /* If auto-priority-failover is enabled we can get
++ * back to back failovers during resets, resulting
++ * in at least two failed resets (from high-priority
++ * backing device to low-priority one and then back)
++ * If resets continue to fail beyond that, give the
++ * adapter some time to settle down before retrying.
++ */
++ if (num_fails >= 3) {
+ netdev_dbg(adapter->netdev,
+- "[S:%s] Hard reset failed, waiting 60 secs\n",
+- adapter_state_to_string(adapter->state));
++ "[S:%s] Hard reset failed %d times, waiting 60 secs\n",
++ adapter_state_to_string(adapter->state),
++ num_fails);
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(60 * HZ);
+ }
+@@ -2541,12 +2588,23 @@ static void __ibmvnic_delayed_reset(struct work_struct *work)
+ __ibmvnic_reset(&adapter->ibmvnic_reset);
+ }
+
++static void flush_reset_queue(struct ibmvnic_adapter *adapter)
++{
++ struct list_head *entry, *tmp_entry;
++
++ if (!list_empty(&adapter->rwi_list)) {
++ list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) {
++ list_del(entry);
++ kfree(list_entry(entry, struct ibmvnic_rwi, list));
++ }
++ }
++}
++
+ static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
+ enum ibmvnic_reset_reason reason)
+ {
+- struct list_head *entry, *tmp_entry;
+- struct ibmvnic_rwi *rwi, *tmp;
+ struct net_device *netdev = adapter->netdev;
++ struct ibmvnic_rwi *rwi, *tmp;
+ unsigned long flags;
+ int ret;
+
+@@ -2567,7 +2625,7 @@ static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
+
+ if (adapter->state == VNIC_PROBING) {
+ netdev_warn(netdev, "Adapter reset during probe\n");
+- adapter->init_done_rc = EAGAIN;
++ adapter->init_done_rc = -EAGAIN;
+ ret = EAGAIN;
+ goto err;
+ }
+@@ -2589,10 +2647,9 @@ static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
+ /* if we just received a transport event,
+ * flush reset queue and process this reset
+ */
+- if (adapter->force_reset_recovery && !list_empty(&adapter->rwi_list)) {
+- list_for_each_safe(entry, tmp_entry, &adapter->rwi_list)
+- list_del(entry);
+- }
++ if (adapter->force_reset_recovery)
++ flush_reset_queue(adapter);
++
+ rwi->reset_reason = reason;
+ list_add_tail(&rwi->list, &adapter->rwi_list);
+ netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",
+@@ -2904,13 +2961,8 @@ static void ibmvnic_get_ringparam(struct net_device *netdev,
+ {
+ struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+
+- if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
+- ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
+- ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
+- } else {
+- ring->rx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
+- ring->tx_max_pending = IBMVNIC_MAX_QUEUE_SZ;
+- }
++ ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
++ ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
+ ring->rx_mini_max_pending = 0;
+ ring->rx_jumbo_max_pending = 0;
+ ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
+@@ -2923,23 +2975,21 @@ static int ibmvnic_set_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *ring)
+ {
+ struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+- int ret;
+
+- ret = 0;
++ if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq ||
++ ring->tx_pending > adapter->max_tx_entries_per_subcrq) {
++ netdev_err(netdev, "Invalid request.\n");
++ netdev_err(netdev, "Max tx buffers = %llu\n",
++ adapter->max_rx_add_entries_per_subcrq);
++ netdev_err(netdev, "Max rx buffers = %llu\n",
++ adapter->max_tx_entries_per_subcrq);
++ return -EINVAL;
++ }
++
+ adapter->desired.rx_entries = ring->rx_pending;
+ adapter->desired.tx_entries = ring->tx_pending;
+
+- ret = wait_for_reset(adapter);
+-
+- if (!ret &&
+- (adapter->req_rx_add_entries_per_subcrq != ring->rx_pending ||
+- adapter->req_tx_entries_per_subcrq != ring->tx_pending))
+- netdev_info(netdev,
+- "Could not match full ringsize request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
+- ring->rx_pending, ring->tx_pending,
+- adapter->req_rx_add_entries_per_subcrq,
+- adapter->req_tx_entries_per_subcrq);
+- return ret;
++ return wait_for_reset(adapter);
+ }
+
+ static void ibmvnic_get_channels(struct net_device *netdev,
+@@ -2947,14 +2997,8 @@ static void ibmvnic_get_channels(struct net_device *netdev,
+ {
+ struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+
+- if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) {
+- channels->max_rx = adapter->max_rx_queues;
+- channels->max_tx = adapter->max_tx_queues;
+- } else {
+- channels->max_rx = IBMVNIC_MAX_QUEUES;
+- channels->max_tx = IBMVNIC_MAX_QUEUES;
+- }
+-
++ channels->max_rx = adapter->max_rx_queues;
++ channels->max_tx = adapter->max_tx_queues;
+ channels->max_other = 0;
+ channels->max_combined = 0;
+ channels->rx_count = adapter->req_rx_queues;
+@@ -2967,22 +3011,11 @@ static int ibmvnic_set_channels(struct net_device *netdev,
+ struct ethtool_channels *channels)
+ {
+ struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+- int ret;
+
+- ret = 0;
+ adapter->desired.rx_queues = channels->rx_count;
+ adapter->desired.tx_queues = channels->tx_count;
+
+- ret = wait_for_reset(adapter);
+-
+- if (!ret &&
+- (adapter->req_rx_queues != channels->rx_count ||
+- adapter->req_tx_queues != channels->tx_count))
+- netdev_info(netdev,
+- "Could not match full channels request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n",
+- channels->rx_count, channels->tx_count,
+- adapter->req_rx_queues, adapter->req_tx_queues);
+- return ret;
++ return wait_for_reset(adapter);
+ }
+
+ static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+@@ -2990,43 +3023,32 @@ static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+ struct ibmvnic_adapter *adapter = netdev_priv(dev);
+ int i;
+
+- switch (stringset) {
+- case ETH_SS_STATS:
+- for (i = 0; i < ARRAY_SIZE(ibmvnic_stats);
+- i++, data += ETH_GSTRING_LEN)
+- memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
++ if (stringset != ETH_SS_STATS)
++ return;
+
+- for (i = 0; i < adapter->req_tx_queues; i++) {
+- snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
+- data += ETH_GSTRING_LEN;
++ for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++, data += ETH_GSTRING_LEN)
++ memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
+
+- snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
+- data += ETH_GSTRING_LEN;
++ for (i = 0; i < adapter->req_tx_queues; i++) {
++ snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
++ data += ETH_GSTRING_LEN;
+
+- snprintf(data, ETH_GSTRING_LEN,
+- "tx%d_dropped_packets", i);
+- data += ETH_GSTRING_LEN;
+- }
++ snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
++ data += ETH_GSTRING_LEN;
+
+- for (i = 0; i < adapter->req_rx_queues; i++) {
+- snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
+- data += ETH_GSTRING_LEN;
++ snprintf(data, ETH_GSTRING_LEN, "tx%d_dropped_packets", i);
++ data += ETH_GSTRING_LEN;
++ }
+
+- snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
+- data += ETH_GSTRING_LEN;
++ for (i = 0; i < adapter->req_rx_queues; i++) {
++ snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
++ data += ETH_GSTRING_LEN;
+
+- snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
+- data += ETH_GSTRING_LEN;
+- }
+- break;
++ snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
++ data += ETH_GSTRING_LEN;
+
+- case ETH_SS_PRIV_FLAGS:
+- for (i = 0; i < ARRAY_SIZE(ibmvnic_priv_flags); i++)
+- strcpy(data + i * ETH_GSTRING_LEN,
+- ibmvnic_priv_flags[i]);
+- break;
+- default:
+- return;
++ snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
++ data += ETH_GSTRING_LEN;
+ }
+ }
+
+@@ -3039,8 +3061,6 @@ static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
+ return ARRAY_SIZE(ibmvnic_stats) +
+ adapter->req_tx_queues * NUM_TX_STATS +
+ adapter->req_rx_queues * NUM_RX_STATS;
+- case ETH_SS_PRIV_FLAGS:
+- return ARRAY_SIZE(ibmvnic_priv_flags);
+ default:
+ return -EOPNOTSUPP;
+ }
+@@ -3093,26 +3113,6 @@ static void ibmvnic_get_ethtool_stats(struct net_device *dev,
+ }
+ }
+
+-static u32 ibmvnic_get_priv_flags(struct net_device *netdev)
+-{
+- struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+-
+- return adapter->priv_flags;
+-}
+-
+-static int ibmvnic_set_priv_flags(struct net_device *netdev, u32 flags)
+-{
+- struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+- bool which_maxes = !!(flags & IBMVNIC_USE_SERVER_MAXES);
+-
+- if (which_maxes)
+- adapter->priv_flags |= IBMVNIC_USE_SERVER_MAXES;
+- else
+- adapter->priv_flags &= ~IBMVNIC_USE_SERVER_MAXES;
+-
+- return 0;
+-}
+-
+ static const struct ethtool_ops ibmvnic_ethtool_ops = {
+ .get_drvinfo = ibmvnic_get_drvinfo,
+ .get_msglevel = ibmvnic_get_msglevel,
+@@ -3126,8 +3126,6 @@ static const struct ethtool_ops ibmvnic_ethtool_ops = {
+ .get_sset_count = ibmvnic_get_sset_count,
+ .get_ethtool_stats = ibmvnic_get_ethtool_stats,
+ .get_link_ksettings = ibmvnic_get_link_ksettings,
+- .get_priv_flags = ibmvnic_get_priv_flags,
+- .set_priv_flags = ibmvnic_set_priv_flags,
+ };
+
+ /* Routines for managing CRQs/sCRQs */
+@@ -3456,9 +3454,15 @@ restart_loop:
+ (adapter->req_tx_entries_per_subcrq / 2) &&
+ __netif_subqueue_stopped(adapter->netdev,
+ scrq->pool_index)) {
+- netif_wake_subqueue(adapter->netdev, scrq->pool_index);
+- netdev_dbg(adapter->netdev, "Started queue %d\n",
+- scrq->pool_index);
++ rcu_read_lock();
++ if (adapter->tx_queues_active) {
++ netif_wake_subqueue(adapter->netdev,
++ scrq->pool_index);
++ netdev_dbg(adapter->netdev,
++ "Started queue %d\n",
++ scrq->pool_index);
++ }
++ rcu_read_unlock();
+ }
+ }
+
+@@ -3664,11 +3668,25 @@ static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)
+ struct device *dev = &adapter->vdev->dev;
+ union ibmvnic_crq crq;
+ int max_entries;
++ int cap_reqs;
++
++ /* We send out 6 or 7 REQUEST_CAPABILITY CRQs below (depending on
++ * the PROMISC flag). Initialize this count upfront. When the tasklet
++ * receives a response to all of these, it will send the next protocol
++ * message (QUERY_IP_OFFLOAD).
++ */
++ if (!(adapter->netdev->flags & IFF_PROMISC) ||
++ adapter->promisc_supported)
++ cap_reqs = 7;
++ else
++ cap_reqs = 6;
+
+ if (!retry) {
+ /* Sub-CRQ entries are 32 byte long */
+ int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);
+
++ atomic_set(&adapter->running_cap_crqs, cap_reqs);
++
+ if (adapter->min_tx_entries_per_subcrq > entries_page ||
+ adapter->min_rx_add_entries_per_subcrq > entries_page) {
+ dev_err(dev, "Fatal, invalid entries per sub-crq\n");
+@@ -3729,44 +3747,45 @@ static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)
+ adapter->opt_rx_comp_queues;
+
+ adapter->req_rx_add_queues = adapter->max_rx_add_queues;
++ } else {
++ atomic_add(cap_reqs, &adapter->running_cap_crqs);
+ }
+-
+ memset(&crq, 0, sizeof(crq));
+ crq.request_capability.first = IBMVNIC_CRQ_CMD;
+ crq.request_capability.cmd = REQUEST_CAPABILITY;
+
+ crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);
+ crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);
+ crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);
+ crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ crq.request_capability.capability =
+ cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);
+ crq.request_capability.number =
+ cpu_to_be64(adapter->req_tx_entries_per_subcrq);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ crq.request_capability.capability =
+ cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);
+ crq.request_capability.number =
+ cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ crq.request_capability.capability = cpu_to_be16(REQ_MTU);
+ crq.request_capability.number = cpu_to_be64(adapter->req_mtu);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+
+ if (adapter->netdev->flags & IFF_PROMISC) {
+@@ -3774,16 +3793,21 @@ static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)
+ crq.request_capability.capability =
+ cpu_to_be16(PROMISC_REQUESTED);
+ crq.request_capability.number = cpu_to_be64(1);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+ }
+ } else {
+ crq.request_capability.capability =
+ cpu_to_be16(PROMISC_REQUESTED);
+ crq.request_capability.number = cpu_to_be64(0);
+- atomic_inc(&adapter->running_cap_crqs);
++ cap_reqs--;
+ ibmvnic_send_crq(adapter, &crq);
+ }
++
++ /* Keep at end to catch any discrepancy between expected and actual
++ * CRQs sent.
++ */
++ WARN_ON(cap_reqs != 0);
+ }
+
+ static int pending_scrq(struct ibmvnic_adapter *adapter,
+@@ -4177,118 +4201,132 @@ static void send_query_map(struct ibmvnic_adapter *adapter)
+ static void send_query_cap(struct ibmvnic_adapter *adapter)
+ {
+ union ibmvnic_crq crq;
++ int cap_reqs;
++
++ /* We send out 25 QUERY_CAPABILITY CRQs below. Initialize this count
++ * upfront. When the tasklet receives a response to all of these, it
++ * can send out the next protocol messaage (REQUEST_CAPABILITY).
++ */
++ cap_reqs = 25;
++
++ atomic_set(&adapter->running_cap_crqs, cap_reqs);
+
+- atomic_set(&adapter->running_cap_crqs, 0);
+ memset(&crq, 0, sizeof(crq));
+ crq.query_capability.first = IBMVNIC_CRQ_CMD;
+ crq.query_capability.cmd = QUERY_CAPABILITY;
+
+ crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MIN_MTU);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_MTU);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability =
+ cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);
+- atomic_inc(&adapter->running_cap_crqs);
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
+
+ crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);
+- atomic_inc(&adapter->running_cap_crqs);
++
+ ibmvnic_send_crq(adapter, &crq);
++ cap_reqs--;
++
++ /* Keep at end to catch any discrepancy between expected and actual
++ * CRQs sent.
++ */
++ WARN_ON(cap_reqs != 0);
+ }
+
+ static void send_query_ip_offload(struct ibmvnic_adapter *adapter)
+@@ -4593,6 +4631,8 @@ static void handle_request_cap_rsp(union ibmvnic_crq *crq,
+ char *name;
+
+ atomic_dec(&adapter->running_cap_crqs);
++ netdev_dbg(adapter->netdev, "Outstanding request-caps: %d\n",
++ atomic_read(&adapter->running_cap_crqs));
+ switch (be16_to_cpu(crq->request_capability_rsp.capability)) {
+ case REQ_TX_QUEUES:
+ req_value = &adapter->req_tx_queues;
+@@ -5069,11 +5109,6 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
+ */
+ adapter->login_pending = false;
+
+- if (!completion_done(&adapter->init_done)) {
+- complete(&adapter->init_done);
+- adapter->init_done_rc = -EIO;
+- }
+-
+ if (adapter->state == VNIC_DOWN)
+ rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);
+ else
+@@ -5094,6 +5129,13 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
+ rc);
+ adapter->failover_pending = false;
+ }
++
++ if (!completion_done(&adapter->init_done)) {
++ if (!adapter->init_done_rc)
++ adapter->init_done_rc = -EAGAIN;
++ complete(&adapter->init_done);
++ }
++
+ break;
+ case IBMVNIC_CRQ_INIT_COMPLETE:
+ dev_info(dev, "Partner initialization complete\n");
+@@ -5114,6 +5156,13 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
+ adapter->fw_done_rc = -EIO;
+ complete(&adapter->fw_done);
+ }
++
++ /* if we got here during crq-init, retry crq-init */
++ if (!completion_done(&adapter->init_done)) {
++ adapter->init_done_rc = -EAGAIN;
++ complete(&adapter->init_done);
++ }
++
+ if (!completion_done(&adapter->stats_done))
+ complete(&adapter->stats_done);
+ if (test_bit(0, &adapter->resetting))
+@@ -5268,12 +5317,6 @@ static void ibmvnic_tasklet(struct tasklet_struct *t)
+ ibmvnic_handle_crq(crq, adapter);
+ crq->generic.first = 0;
+ }
+-
+- /* remain in tasklet until all
+- * capabilities responses are received
+- */
+- if (!adapter->wait_capability)
+- done = true;
+ }
+ /* if capabilities CRQ's were sent in this tasklet, the following
+ * tasklet must wait until all responses are received
+@@ -5414,6 +5457,9 @@ static int init_crq_queue(struct ibmvnic_adapter *adapter)
+ crq->cur = 0;
+ spin_lock_init(&crq->lock);
+
++ /* process any CRQs that were queued before we enabled interrupts */
++ tasklet_schedule(&adapter->tasklet);
++
+ return retrc;
+
+ req_irq_failed:
+@@ -5558,7 +5604,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
+ }
+
+ rc = ibmvnic_reset_init(adapter, false);
+- } while (rc == EAGAIN);
++ } while (rc == -EAGAIN);
+
+ /* We are ignoring the error from ibmvnic_reset_init() assuming that the
+ * partner is not ready. CRQ is not active. When the partner becomes
+@@ -5581,12 +5627,6 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
+ goto ibmvnic_dev_file_err;
+
+ netif_carrier_off(netdev);
+- rc = register_netdev(netdev);
+- if (rc) {
+- dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
+- goto ibmvnic_register_fail;
+- }
+- dev_info(&dev->dev, "ibmvnic registered\n");
+
+ if (init_success) {
+ adapter->state = VNIC_PROBED;
+@@ -5599,6 +5639,14 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
+
+ adapter->wait_for_reset = false;
+ adapter->last_reset_time = jiffies;
++
++ rc = register_netdev(netdev);
++ if (rc) {
++ dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
++ goto ibmvnic_register_fail;
++ }
++ dev_info(&dev->dev, "ibmvnic registered\n");
++
+ return 0;
+
+ ibmvnic_register_fail:
+@@ -5687,10 +5735,14 @@ static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
+ be64_to_cpu(session_token));
+ rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
+ H_SESSION_ERR_DETECTED, session_token, 0, 0);
+- if (rc)
++ if (rc) {
+ netdev_err(netdev,
+ "H_VIOCTL initiated failover failed, rc %ld\n",
+ rc);
++ goto last_resort;
++ }
++
++ return count;
+
+ last_resort:
+ netdev_dbg(netdev, "Trying to send CRQ_CMD, the last resort\n");
+diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
+index 22df602323bc0..1a9ed9202654f 100644
+--- a/drivers/net/ethernet/ibm/ibmvnic.h
++++ b/drivers/net/ethernet/ibm/ibmvnic.h
+@@ -43,11 +43,6 @@
+
+ #define IBMVNIC_RESET_DELAY 100
+
+-static const char ibmvnic_priv_flags[][ETH_GSTRING_LEN] = {
+-#define IBMVNIC_USE_SERVER_MAXES 0x1
+- "use-server-maxes"
+-};
+-
+ struct ibmvnic_login_buffer {
+ __be32 len;
+ __be32 version;
+@@ -885,7 +880,6 @@ struct ibmvnic_adapter {
+ struct ibmvnic_control_ip_offload_buffer ip_offload_ctrl;
+ dma_addr_t ip_offload_ctrl_tok;
+ u32 msg_enable;
+- u32 priv_flags;
+
+ /* Vital Product Data (VPD) */
+ struct ibmvnic_vpd *vpd;
+@@ -1002,11 +996,14 @@ struct ibmvnic_adapter {
+ struct work_struct ibmvnic_reset;
+ struct delayed_work ibmvnic_delayed_reset;
+ unsigned long resetting;
+- bool napi_enabled, from_passive_init;
+- bool login_pending;
+ /* last device reset time */
+ unsigned long last_reset_time;
+
++ bool napi_enabled;
++ bool from_passive_init;
++ bool login_pending;
++ /* protected by rcu */
++ bool tx_queues_active;
+ bool failover_pending;
+ bool force_reset_recovery;
+
+diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c
+index 09ae1939e6db4..36d52246bdc66 100644
+--- a/drivers/net/ethernet/intel/e100.c
++++ b/drivers/net/ethernet/intel/e100.c
+@@ -3003,9 +3003,10 @@ static void __e100_shutdown(struct pci_dev *pdev, bool *enable_wake)
+ struct net_device *netdev = pci_get_drvdata(pdev);
+ struct nic *nic = netdev_priv(netdev);
+
++ netif_device_detach(netdev);
++
+ if (netif_running(netdev))
+ e100_down(nic);
+- netif_device_detach(netdev);
+
+ if ((nic->flags & wol_magic) | e100_asf(nic)) {
+ /* enable reverse auto-negotiation */
+@@ -3022,7 +3023,7 @@ static void __e100_shutdown(struct pci_dev *pdev, bool *enable_wake)
+ *enable_wake = false;
+ }
+
+- pci_clear_master(pdev);
++ pci_disable_device(pdev);
+ }
+
+ static int __e100_power_off(struct pci_dev *pdev, bool wake)
+@@ -3042,8 +3043,6 @@ static int __maybe_unused e100_suspend(struct device *dev_d)
+
+ __e100_shutdown(to_pci_dev(dev_d), &wake);
+
+- device_wakeup_disable(dev_d);
+-
+ return 0;
+ }
+
+@@ -3051,6 +3050,14 @@ static int __maybe_unused e100_resume(struct device *dev_d)
+ {
+ struct net_device *netdev = dev_get_drvdata(dev_d);
+ struct nic *nic = netdev_priv(netdev);
++ int err;
++
++ err = pci_enable_device(to_pci_dev(dev_d));
++ if (err) {
++ netdev_err(netdev, "Resume cannot enable PCI device, aborting\n");
++ return err;
++ }
++ pci_set_master(to_pci_dev(dev_d));
+
+ /* disable reverse auto-negotiation */
+ if (nic->phy == phy_82552_v) {
+@@ -3062,10 +3069,11 @@ static int __maybe_unused e100_resume(struct device *dev_d)
+ smartspeed & ~(E100_82552_REV_ANEG));
+ }
+
+- netif_device_attach(netdev);
+ if (netif_running(netdev))
+ e100_up(nic);
+
++ netif_device_attach(netdev);
++
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/intel/e1000e/e1000.h b/drivers/net/ethernet/intel/e1000e/e1000.h
+index 3178efd980066..eb6961d901c16 100644
+--- a/drivers/net/ethernet/intel/e1000e/e1000.h
++++ b/drivers/net/ethernet/intel/e1000e/e1000.h
+@@ -114,7 +114,8 @@ enum e1000_boards {
+ board_pch_lpt,
+ board_pch_spt,
+ board_pch_cnp,
+- board_pch_tgp
++ board_pch_tgp,
++ board_pch_adp
+ };
+
+ struct e1000_ps_page {
+@@ -501,6 +502,7 @@ extern const struct e1000_info e1000_pch_lpt_info;
+ extern const struct e1000_info e1000_pch_spt_info;
+ extern const struct e1000_info e1000_pch_cnp_info;
+ extern const struct e1000_info e1000_pch_tgp_info;
++extern const struct e1000_info e1000_pch_adp_info;
+ extern const struct e1000_info e1000_es2_info;
+
+ void e1000e_ptp_init(struct e1000_adapter *adapter);
+diff --git a/drivers/net/ethernet/intel/e1000e/hw.h b/drivers/net/ethernet/intel/e1000e/hw.h
+index bcf680e838113..13382df2f2eff 100644
+--- a/drivers/net/ethernet/intel/e1000e/hw.h
++++ b/drivers/net/ethernet/intel/e1000e/hw.h
+@@ -630,6 +630,7 @@ struct e1000_phy_info {
+ bool disable_polarity_correction;
+ bool is_mdix;
+ bool polarity_correction;
++ bool reset_disable;
+ bool speed_downgraded;
+ bool autoneg_wait_to_complete;
+ };
+diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c
+index 5e4fc9b4e2adb..e6c8e6d5234f8 100644
+--- a/drivers/net/ethernet/intel/e1000e/ich8lan.c
++++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c
+@@ -1009,8 +1009,8 @@ static s32 e1000_platform_pm_pch_lpt(struct e1000_hw *hw, bool link)
+ {
+ u32 reg = link << (E1000_LTRV_REQ_SHIFT + E1000_LTRV_NOSNOOP_SHIFT) |
+ link << E1000_LTRV_REQ_SHIFT | E1000_LTRV_SEND;
+- u16 max_ltr_enc_d = 0; /* maximum LTR decoded by platform */
+- u16 lat_enc_d = 0; /* latency decoded */
++ u32 max_ltr_enc_d = 0; /* maximum LTR decoded by platform */
++ u32 lat_enc_d = 0; /* latency decoded */
+ u16 lat_enc = 0; /* latency encoded */
+
+ if (link) {
+@@ -2050,6 +2050,10 @@ static s32 e1000_check_reset_block_ich8lan(struct e1000_hw *hw)
+ bool blocked = false;
+ int i = 0;
+
++ /* Check the PHY (LCD) reset flag */
++ if (hw->phy.reset_disable)
++ return true;
++
+ while ((blocked = !(er32(FWSM) & E1000_ICH_FWSM_RSPCIPHY)) &&
+ (i++ < 30))
+ usleep_range(10000, 11000);
+@@ -4136,9 +4140,9 @@ static s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw)
+ return ret_val;
+
+ if (!(data & valid_csum_mask)) {
+- e_dbg("NVM Checksum Invalid\n");
++ e_dbg("NVM Checksum valid bit not set\n");
+
+- if (hw->mac.type < e1000_pch_cnp) {
++ if (hw->mac.type < e1000_pch_tgp) {
+ data |= valid_csum_mask;
+ ret_val = e1000_write_nvm(hw, word, 1, &data);
+ if (ret_val)
+@@ -6021,3 +6025,23 @@ const struct e1000_info e1000_pch_tgp_info = {
+ .phy_ops = &ich8_phy_ops,
+ .nvm_ops = &spt_nvm_ops,
+ };
++
++const struct e1000_info e1000_pch_adp_info = {
++ .mac = e1000_pch_adp,
++ .flags = FLAG_IS_ICH
++ | FLAG_HAS_WOL
++ | FLAG_HAS_HW_TIMESTAMP
++ | FLAG_HAS_CTRLEXT_ON_LOAD
++ | FLAG_HAS_AMT
++ | FLAG_HAS_FLASH
++ | FLAG_HAS_JUMBO_FRAMES
++ | FLAG_APME_IN_WUC,
++ .flags2 = FLAG2_HAS_PHY_STATS
++ | FLAG2_HAS_EEE,
++ .pba = 26,
++ .max_hw_frame_size = 9022,
++ .get_variants = e1000_get_variants_ich8lan,
++ .mac_ops = &ich8_mac_ops,
++ .phy_ops = &ich8_phy_ops,
++ .nvm_ops = &spt_nvm_ops,
++};
+diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.h b/drivers/net/ethernet/intel/e1000e/ich8lan.h
+index 2504b11c3169f..638a3ddd7ada8 100644
+--- a/drivers/net/ethernet/intel/e1000e/ich8lan.h
++++ b/drivers/net/ethernet/intel/e1000e/ich8lan.h
+@@ -271,6 +271,7 @@
+ #define I217_CGFREG_ENABLE_MTA_RESET 0x0002
+ #define I217_MEMPWR PHY_REG(772, 26)
+ #define I217_MEMPWR_DISABLE_SMB_RELEASE 0x0010
++#define I217_MEMPWR_MOEM 0x1000
+
+ /* Receive Address Initial CRC Calculation */
+ #define E1000_PCH_RAICC(_n) (0x05F50 + ((_n) * 4))
+diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
+index ebcb2a30add09..ce48e630fe550 100644
+--- a/drivers/net/ethernet/intel/e1000e/netdev.c
++++ b/drivers/net/ethernet/intel/e1000e/netdev.c
+@@ -52,6 +52,7 @@ static const struct e1000_info *e1000_info_tbl[] = {
+ [board_pch_spt] = &e1000_pch_spt_info,
+ [board_pch_cnp] = &e1000_pch_cnp_info,
+ [board_pch_tgp] = &e1000_pch_tgp_info,
++ [board_pch_adp] = &e1000_pch_adp_info,
+ };
+
+ struct e1000_reg_info {
+@@ -6346,7 +6347,8 @@ static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter)
+ u32 mac_data;
+ u16 phy_data;
+
+- if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) {
++ if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&
++ hw->mac.type >= e1000_pch_adp) {
+ /* Request ME configure the device for S0ix */
+ mac_data = er32(H2ME);
+ mac_data |= E1000_H2ME_START_DPG;
+@@ -6495,7 +6497,8 @@ static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter)
+ u16 phy_data;
+ u32 i = 0;
+
+- if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) {
++ if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&
++ hw->mac.type >= e1000_pch_adp) {
+ /* Request ME unconfigure the device from S0ix */
+ mac_data = er32(H2ME);
+ mac_data &= ~E1000_H2ME_START_DPG;
+@@ -6989,8 +6992,21 @@ static __maybe_unused int e1000e_pm_suspend(struct device *dev)
+ struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));
+ struct e1000_adapter *adapter = netdev_priv(netdev);
+ struct pci_dev *pdev = to_pci_dev(dev);
++ struct e1000_hw *hw = &adapter->hw;
++ u16 phy_data;
+ int rc;
+
++ if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&
++ hw->mac.type >= e1000_pch_adp) {
++ /* Mask OEM Bits / Gig Disable / Restart AN (772_26[12] = 1) */
++ e1e_rphy(hw, I217_MEMPWR, &phy_data);
++ phy_data |= I217_MEMPWR_MOEM;
++ e1e_wphy(hw, I217_MEMPWR, phy_data);
++
++ /* Disable LCD reset */
++ hw->phy.reset_disable = true;
++ }
++
+ e1000e_flush_lpic(pdev);
+
+ e1000e_pm_freeze(dev);
+@@ -7012,6 +7028,8 @@ static __maybe_unused int e1000e_pm_resume(struct device *dev)
+ struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));
+ struct e1000_adapter *adapter = netdev_priv(netdev);
+ struct pci_dev *pdev = to_pci_dev(dev);
++ struct e1000_hw *hw = &adapter->hw;
++ u16 phy_data;
+ int rc;
+
+ /* Introduce S0ix implementation */
+@@ -7022,6 +7040,17 @@ static __maybe_unused int e1000e_pm_resume(struct device *dev)
+ if (rc)
+ return rc;
+
++ if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&
++ hw->mac.type >= e1000_pch_adp) {
++ /* Unmask OEM Bits / Gig Disable / Restart AN 772_26[12] = 0 */
++ e1e_rphy(hw, I217_MEMPWR, &phy_data);
++ phy_data &= ~I217_MEMPWR_MOEM;
++ e1e_wphy(hw, I217_MEMPWR, phy_data);
++
++ /* Enable LCD reset */
++ hw->phy.reset_disable = false;
++ }
++
+ return e1000e_pm_thaw(dev);
+ }
+
+@@ -7903,22 +7932,22 @@ static const struct pci_device_id e1000_pci_tbl[] = {
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_tgp },
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_tgp },
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21), board_pch_tgp },
+- { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21), board_pch_tgp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21), board_pch_adp },
++ { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21), board_pch_adp },
+
+ { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
+ };
+diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
+index 39fb3d57c0574..56a3a6d1dbe41 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e.h
++++ b/drivers/net/ethernet/intel/i40e/i40e.h
+@@ -144,6 +144,7 @@ enum i40e_state_t {
+ __I40E_VIRTCHNL_OP_PENDING,
+ __I40E_RECOVERY_MODE,
+ __I40E_VF_RESETS_DISABLED, /* disable resets during i40e_remove */
++ __I40E_IN_REMOVE,
+ __I40E_VFS_RELEASING,
+ /* This must be last as it determines the size of the BITMAP */
+ __I40E_STATE_SIZE__,
+@@ -161,6 +162,7 @@ enum i40e_vsi_state_t {
+ __I40E_VSI_OVERFLOW_PROMISC,
+ __I40E_VSI_REINIT_REQUESTED,
+ __I40E_VSI_DOWN_REQUESTED,
++ __I40E_VSI_RELEASING,
+ /* This must be last as it determines the size of the BITMAP */
+ __I40E_VSI_STATE_SIZE__,
+ };
+@@ -173,7 +175,6 @@ enum i40e_interrupt_policy {
+
+ struct i40e_lump_tracking {
+ u16 num_entries;
+- u16 search_hint;
+ u16 list[0];
+ #define I40E_PILE_VALID_BIT 0x8000
+ #define I40E_IWARP_IRQ_PILE_ID (I40E_PILE_VALID_BIT - 2)
+@@ -847,12 +848,12 @@ struct i40e_vsi {
+ struct rtnl_link_stats64 net_stats_offsets;
+ struct i40e_eth_stats eth_stats;
+ struct i40e_eth_stats eth_stats_offsets;
+- u32 tx_restart;
+- u32 tx_busy;
++ u64 tx_restart;
++ u64 tx_busy;
+ u64 tx_linearize;
+ u64 tx_force_wb;
+- u32 rx_buf_failed;
+- u32 rx_page_failed;
++ u64 rx_buf_failed;
++ u64 rx_page_failed;
+
+ /* These are containers of ring pointers, allocated at run-time */
+ struct i40e_ring **rx_rings;
+@@ -1247,6 +1248,7 @@ void i40e_ptp_restore_hw_time(struct i40e_pf *pf);
+ void i40e_ptp_init(struct i40e_pf *pf);
+ void i40e_ptp_stop(struct i40e_pf *pf);
+ int i40e_ptp_alloc_pins(struct i40e_pf *pf);
++int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset);
+ int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi);
+ i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf);
+ i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf);
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+index 291e61ac3e448..9db5001297c7e 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
++++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+@@ -240,7 +240,7 @@ static void i40e_dbg_dump_vsi_seid(struct i40e_pf *pf, int seid)
+ (unsigned long int)vsi->net_stats_offsets.rx_compressed,
+ (unsigned long int)vsi->net_stats_offsets.tx_compressed);
+ dev_info(&pf->pdev->dev,
+- " tx_restart = %d, tx_busy = %d, rx_buf_failed = %d, rx_page_failed = %d\n",
++ " tx_restart = %llu, tx_busy = %llu, rx_buf_failed = %llu, rx_page_failed = %llu\n",
+ vsi->tx_restart, vsi->tx_busy,
+ vsi->rx_buf_failed, vsi->rx_page_failed);
+ rcu_read_lock();
+@@ -553,6 +553,14 @@ static void i40e_dbg_dump_desc(int cnt, int vsi_seid, int ring_id, int desc_n,
+ dev_info(&pf->pdev->dev, "vsi %d not found\n", vsi_seid);
+ return;
+ }
++ if (vsi->type != I40E_VSI_MAIN &&
++ vsi->type != I40E_VSI_FDIR &&
++ vsi->type != I40E_VSI_VMDQ2) {
++ dev_info(&pf->pdev->dev,
++ "vsi %d type %d descriptor rings not available\n",
++ vsi_seid, vsi->type);
++ return;
++ }
+ if (type == RING_TYPE_XDP && !i40e_enabled_xdp_vsi(vsi)) {
+ dev_info(&pf->pdev->dev, "XDP not enabled on VSI %d\n", vsi_seid);
+ return;
+@@ -734,10 +742,8 @@ static void i40e_dbg_dump_vf(struct i40e_pf *pf, int vf_id)
+ vsi = pf->vsi[vf->lan_vsi_idx];
+ dev_info(&pf->pdev->dev, "vf %2d: VSI id=%d, seid=%d, qps=%d\n",
+ vf_id, vf->lan_vsi_id, vsi->seid, vf->num_queue_pairs);
+- dev_info(&pf->pdev->dev, " num MDD=%lld, invalid msg=%lld, valid msg=%lld\n",
+- vf->num_mdd_events,
+- vf->num_invalid_msgs,
+- vf->num_valid_msgs);
++ dev_info(&pf->pdev->dev, " num MDD=%lld\n",
++ vf->num_mdd_events);
+ } else {
+ dev_info(&pf->pdev->dev, "invalid VF id %d\n", vf_id);
+ }
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
+index e04b540cedc85..29387f0814e9b 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
++++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
+@@ -99,6 +99,24 @@ MODULE_LICENSE("GPL v2");
+
+ static struct workqueue_struct *i40e_wq;
+
++static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
++ struct net_device *netdev, int delta)
++{
++ struct netdev_hw_addr *ha;
++
++ if (!f || !netdev)
++ return;
++
++ netdev_for_each_mc_addr(ha, netdev) {
++ if (ether_addr_equal(ha->addr, f->macaddr)) {
++ ha->refcount += delta;
++ if (ha->refcount <= 0)
++ ha->refcount = 1;
++ break;
++ }
++ }
++}
++
+ /**
+ * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
+ * @hw: pointer to the HW structure
+@@ -178,10 +196,6 @@ int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
+ * @id: an owner id to stick on the items assigned
+ *
+ * Returns the base item index of the lump, or negative for error
+- *
+- * The search_hint trick and lack of advanced fit-finding only work
+- * because we're highly likely to have all the same size lump requests.
+- * Linear search time and any fragmentation should be minimal.
+ **/
+ static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
+ u16 needed, u16 id)
+@@ -196,8 +210,21 @@ static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
+ return -EINVAL;
+ }
+
+- /* start the linear search with an imperfect hint */
+- i = pile->search_hint;
++ /* Allocate last queue in the pile for FDIR VSI queue
++ * so it doesn't fragment the qp_pile
++ */
++ if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
++ if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
++ dev_err(&pf->pdev->dev,
++ "Cannot allocate queue %d for I40E_VSI_FDIR\n",
++ pile->num_entries - 1);
++ return -ENOMEM;
++ }
++ pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
++ return pile->num_entries - 1;
++ }
++
++ i = 0;
+ while (i < pile->num_entries) {
+ /* skip already allocated entries */
+ if (pile->list[i] & I40E_PILE_VALID_BIT) {
+@@ -216,7 +243,6 @@ static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
+ for (j = 0; j < needed; j++)
+ pile->list[i+j] = id | I40E_PILE_VALID_BIT;
+ ret = i;
+- pile->search_hint = i + j;
+ break;
+ }
+
+@@ -239,7 +265,7 @@ static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
+ {
+ int valid_id = (id | I40E_PILE_VALID_BIT);
+ int count = 0;
+- int i;
++ u16 i;
+
+ if (!pile || index >= pile->num_entries)
+ return -EINVAL;
+@@ -251,8 +277,6 @@ static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
+ count++;
+ }
+
+- if (count && index < pile->search_hint)
+- pile->search_hint = index;
+
+ return count;
+ }
+@@ -754,9 +778,9 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
+ struct rtnl_link_stats64 *ns; /* netdev stats */
+ struct i40e_eth_stats *oes;
+ struct i40e_eth_stats *es; /* device's eth stats */
+- u32 tx_restart, tx_busy;
++ u64 tx_restart, tx_busy;
+ struct i40e_ring *p;
+- u32 rx_page, rx_buf;
++ u64 rx_page, rx_buf;
+ u64 bytes, packets;
+ unsigned int start;
+ u64 tx_linearize;
+@@ -1790,6 +1814,7 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
+ bool is_add)
+ {
+ struct i40e_pf *pf = vsi->back;
++ u16 num_tc_qps = 0;
+ u16 sections = 0;
+ u8 netdev_tc = 0;
+ u16 numtc = 1;
+@@ -1797,13 +1822,33 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
+ u8 offset;
+ u16 qmap;
+ int i;
+- u16 num_tc_qps = 0;
+
+ sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
+ offset = 0;
++ /* zero out queue mapping, it will get updated on the end of the function */
++ memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
++
++ if (vsi->type == I40E_VSI_MAIN) {
++ /* This code helps add more queue to the VSI if we have
++ * more cores than RSS can support, the higher cores will
++ * be served by ATR or other filters. Furthermore, the
++ * non-zero req_queue_pairs says that user requested a new
++ * queue count via ethtool's set_channels, so use this
++ * value for queues distribution across traffic classes
++ */
++ if (vsi->req_queue_pairs > 0)
++ vsi->num_queue_pairs = vsi->req_queue_pairs;
++ else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
++ vsi->num_queue_pairs = pf->num_lan_msix;
++ }
+
+ /* Number of queues per enabled TC */
+- num_tc_qps = vsi->alloc_queue_pairs;
++ if (vsi->type == I40E_VSI_MAIN ||
++ (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
++ num_tc_qps = vsi->num_queue_pairs;
++ else
++ num_tc_qps = vsi->alloc_queue_pairs;
++
+ if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
+ /* Find numtc from enabled TC bitmap */
+ for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
+@@ -1881,15 +1926,11 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
+ }
+ ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
+ }
+-
+- /* Set actual Tx/Rx queue pairs */
+- vsi->num_queue_pairs = offset;
+- if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
+- if (vsi->req_queue_pairs > 0)
+- vsi->num_queue_pairs = vsi->req_queue_pairs;
+- else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+- vsi->num_queue_pairs = pf->num_lan_msix;
+- }
++ /* Do not change previously set num_queue_pairs for PFs and VFs*/
++ if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
++ (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
++ (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
++ vsi->num_queue_pairs = offset;
+
+ /* Scheduler section valid can only be set for ADD VSI */
+ if (is_add) {
+@@ -2019,6 +2060,7 @@ static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
+ hlist_for_each_entry_safe(new, h, from, hlist) {
+ /* We can simply free the wrapper structure */
+ hlist_del(&new->hlist);
++ netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
+ kfree(new);
+ }
+ }
+@@ -2366,6 +2408,10 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
+ &tmp_add_list,
+ &tmp_del_list,
+ vlan_filters);
++
++ hlist_for_each_entry(new, &tmp_add_list, hlist)
++ netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
++
+ if (retval)
+ goto err_no_memory_locked;
+
+@@ -2498,6 +2544,7 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
+ if (new->f->state == I40E_FILTER_NEW)
+ new->f->state = new->state;
+ hlist_del(&new->hlist);
++ netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
+ kfree(new);
+ }
+ spin_unlock_bh(&vsi->mac_filter_hash_lock);
+@@ -2623,7 +2670,8 @@ static void i40e_sync_filters_subtask(struct i40e_pf *pf)
+
+ for (v = 0; v < pf->num_alloc_vsi; v++) {
+ if (pf->vsi[v] &&
+- (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
++ (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
++ !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
+ int ret = i40e_sync_vsi_filters(pf->vsi[v]);
+
+ if (ret) {
+@@ -5426,6 +5474,58 @@ static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
+ sizeof(vsi->info.tc_mapping));
+ }
+
++/**
++ * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
++ * @vsi: the VSI being reconfigured
++ * @vsi_offset: offset from main VF VSI
++ */
++int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
++{
++ struct i40e_vsi_context ctxt = {};
++ struct i40e_pf *pf;
++ struct i40e_hw *hw;
++ int ret;
++
++ if (!vsi)
++ return I40E_ERR_PARAM;
++ pf = vsi->back;
++ hw = &pf->hw;
++
++ ctxt.seid = vsi->seid;
++ ctxt.pf_num = hw->pf_id;
++ ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
++ ctxt.uplink_seid = vsi->uplink_seid;
++ ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
++ ctxt.flags = I40E_AQ_VSI_TYPE_VF;
++ ctxt.info = vsi->info;
++
++ i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
++ false);
++ if (vsi->reconfig_rss) {
++ vsi->rss_size = min_t(int, pf->alloc_rss_size,
++ vsi->num_queue_pairs);
++ ret = i40e_vsi_config_rss(vsi);
++ if (ret) {
++ dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
++ return ret;
++ }
++ vsi->reconfig_rss = false;
++ }
++
++ ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
++ if (ret) {
++ dev_info(&pf->pdev->dev, "Update vsi config failed, err %s aq_err %s\n",
++ i40e_stat_str(hw, ret),
++ i40e_aq_str(hw, hw->aq.asq_last_status));
++ return ret;
++ }
++ /* update the local VSI info with updated queue map */
++ i40e_vsi_update_queue_map(vsi, &ctxt);
++ vsi->info.valid_sections = 0;
++
++ return ret;
++}
++
+ /**
+ * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
+ * @vsi: VSI to be configured
+@@ -5716,24 +5816,6 @@ static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
+ INIT_LIST_HEAD(&vsi->ch_list);
+ }
+
+-/**
+- * i40e_is_any_channel - channel exist or not
+- * @vsi: ptr to VSI to which channels are associated with
+- *
+- * Returns true or false if channel(s) exist for associated VSI or not
+- **/
+-static bool i40e_is_any_channel(struct i40e_vsi *vsi)
+-{
+- struct i40e_channel *ch, *ch_tmp;
+-
+- list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
+- if (ch->initialized)
+- return true;
+- }
+-
+- return false;
+-}
+-
+ /**
+ * i40e_get_max_queues_for_channel
+ * @vsi: ptr to VSI to which channels are associated with
+@@ -6240,26 +6322,15 @@ int i40e_create_queue_channel(struct i40e_vsi *vsi,
+ /* By default we are in VEPA mode, if this is the first VF/VMDq
+ * VSI to be added switch to VEB mode.
+ */
+- if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
+- (!i40e_is_any_channel(vsi))) {
+- if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
+- dev_dbg(&pf->pdev->dev,
+- "Failed to create channel. Override queues (%u) not power of 2\n",
+- vsi->tc_config.tc_info[0].qcount);
+- return -EINVAL;
+- }
+
+- if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
+- pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
++ if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
++ pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+
+- if (vsi->type == I40E_VSI_MAIN) {
+- if (pf->flags & I40E_FLAG_TC_MQPRIO)
+- i40e_do_reset(pf, I40E_PF_RESET_FLAG,
+- true);
+- else
+- i40e_do_reset_safe(pf,
+- I40E_PF_RESET_FLAG);
+- }
++ if (vsi->type == I40E_VSI_MAIN) {
++ if (pf->flags & I40E_FLAG_TC_MQPRIO)
++ i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
++ else
++ i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
+ }
+ /* now onwards for main VSI, number of queues will be value
+ * of TC0's queue count
+@@ -7464,42 +7535,43 @@ static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
+ static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
+ struct i40e_fwd_adapter *fwd)
+ {
++ struct i40e_channel *ch = NULL, *ch_tmp, *iter;
+ int ret = 0, num_tc = 1, i, aq_err;
+- struct i40e_channel *ch, *ch_tmp;
+ struct i40e_pf *pf = vsi->back;
+ struct i40e_hw *hw = &pf->hw;
+
+- if (list_empty(&vsi->macvlan_list))
+- return -EINVAL;
+-
+ /* Go through the list and find an available channel */
+- list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
+- if (!i40e_is_channel_macvlan(ch)) {
+- ch->fwd = fwd;
++ list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
++ if (!i40e_is_channel_macvlan(iter)) {
++ iter->fwd = fwd;
+ /* record configuration for macvlan interface in vdev */
+ for (i = 0; i < num_tc; i++)
+ netdev_bind_sb_channel_queue(vsi->netdev, vdev,
+ i,
+- ch->num_queue_pairs,
+- ch->base_queue);
+- for (i = 0; i < ch->num_queue_pairs; i++) {
++ iter->num_queue_pairs,
++ iter->base_queue);
++ for (i = 0; i < iter->num_queue_pairs; i++) {
+ struct i40e_ring *tx_ring, *rx_ring;
+ u16 pf_q;
+
+- pf_q = ch->base_queue + i;
++ pf_q = iter->base_queue + i;
+
+ /* Get to TX ring ptr */
+ tx_ring = vsi->tx_rings[pf_q];
+- tx_ring->ch = ch;
++ tx_ring->ch = iter;
+
+ /* Get the RX ring ptr */
+ rx_ring = vsi->rx_rings[pf_q];
+- rx_ring->ch = ch;
++ rx_ring->ch = iter;
+ }
++ ch = iter;
+ break;
+ }
+ }
+
++ if (!ch)
++ return -EINVAL;
++
+ /* Guarantee all rings are updated before we update the
+ * MAC address filter.
+ */
+@@ -7912,12 +7984,20 @@ config_tc:
+ vsi->seid);
+ need_reset = true;
+ goto exit;
+- } else {
+- dev_info(&vsi->back->pdev->dev,
+- "Setup channel (id:%u) utilizing num_queues %d\n",
+- vsi->seid, vsi->tc_config.tc_info[0].qcount);
++ } else if (enabled_tc &&
++ (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
++ netdev_info(netdev,
++ "Failed to create channel. Override queues (%u) not power of 2\n",
++ vsi->tc_config.tc_info[0].qcount);
++ ret = -EINVAL;
++ need_reset = true;
++ goto exit;
+ }
+
++ dev_info(&vsi->back->pdev->dev,
++ "Setup channel (id:%u) utilizing num_queues %d\n",
++ vsi->seid, vsi->tc_config.tc_info[0].qcount);
++
+ if (pf->flags & I40E_FLAG_TC_MQPRIO) {
+ if (vsi->mqprio_qopt.max_rate[0]) {
+ u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
+@@ -8482,9 +8562,8 @@ static int i40e_configure_clsflower(struct i40e_vsi *vsi,
+ err = i40e_add_del_cloud_filter(vsi, filter, true);
+
+ if (err) {
+- dev_err(&pf->pdev->dev,
+- "Failed to add cloud filter, err %s\n",
+- i40e_stat_str(&pf->hw, err));
++ dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
++ err);
+ goto err;
+ }
+
+@@ -8668,6 +8747,27 @@ int i40e_open(struct net_device *netdev)
+ return 0;
+ }
+
++/**
++ * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
++ * @vsi: vsi structure
++ *
++ * This updates netdev's number of tx/rx queues
++ *
++ * Returns status of setting tx/rx queues
++ **/
++static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
++{
++ int ret;
++
++ ret = netif_set_real_num_rx_queues(vsi->netdev,
++ vsi->num_queue_pairs);
++ if (ret)
++ return ret;
++
++ return netif_set_real_num_tx_queues(vsi->netdev,
++ vsi->num_queue_pairs);
++}
++
+ /**
+ * i40e_vsi_open -
+ * @vsi: the VSI to open
+@@ -8704,13 +8804,7 @@ int i40e_vsi_open(struct i40e_vsi *vsi)
+ goto err_setup_rx;
+
+ /* Notify the stack of the actual queue counts. */
+- err = netif_set_real_num_tx_queues(vsi->netdev,
+- vsi->num_queue_pairs);
+- if (err)
+- goto err_set_queues;
+-
+- err = netif_set_real_num_rx_queues(vsi->netdev,
+- vsi->num_queue_pairs);
++ err = i40e_netif_set_realnum_tx_rx_queues(vsi);
+ if (err)
+ goto err_set_queues;
+
+@@ -10487,15 +10581,9 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
+ }
+ i40e_get_oem_version(&pf->hw);
+
+- if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
+- ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
+- hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
+- /* The following delay is necessary for 4.33 firmware and older
+- * to recover after EMP reset. 200 ms should suffice but we
+- * put here 300 ms to be sure that FW is ready to operate
+- * after reset.
+- */
+- mdelay(300);
++ if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
++ /* The following delay is necessary for firmware update. */
++ mdelay(1000);
+ }
+
+ /* re-verify the eeprom if we just had an EMP reset */
+@@ -10766,6 +10854,9 @@ static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
+ bool lock_acquired)
+ {
+ int ret;
++
++ if (test_bit(__I40E_IN_REMOVE, pf->state))
++ return;
+ /* Now we wait for GRST to settle out.
+ * We don't have to delete the VEBs or VSIs from the hw switch
+ * because the reset will make them disappear.
+@@ -11705,7 +11796,6 @@ static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
+ return -ENOMEM;
+
+ pf->irq_pile->num_entries = vectors;
+- pf->irq_pile->search_hint = 0;
+
+ /* track first vector for misc interrupts, ignore return */
+ (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
+@@ -12126,6 +12216,8 @@ int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
+
+ vsi->req_queue_pairs = queue_count;
+ i40e_prep_for_reset(pf);
++ if (test_bit(__I40E_IN_REMOVE, pf->state))
++ return pf->alloc_rss_size;
+
+ pf->alloc_rss_size = new_rss_size;
+
+@@ -12508,7 +12600,6 @@ static int i40e_sw_init(struct i40e_pf *pf)
+ goto sw_init_done;
+ }
+ pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
+- pf->qp_pile->search_hint = 0;
+
+ pf->tx_timeout_recovery_level = 1;
+
+@@ -12953,6 +13044,10 @@ static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
+ if (need_reset)
+ i40e_prep_for_reset(pf);
+
++ /* VSI shall be deleted in a moment, just return EINVAL */
++ if (test_bit(__I40E_IN_REMOVE, pf->state))
++ return -EINVAL;
++
+ old_prog = xchg(&vsi->xdp_prog, prog);
+
+ if (need_reset) {
+@@ -13771,7 +13866,7 @@ int i40e_vsi_release(struct i40e_vsi *vsi)
+ dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
+ return -ENODEV;
+ }
+-
++ set_bit(__I40E_VSI_RELEASING, vsi->state);
+ uplink_seid = vsi->uplink_seid;
+ if (vsi->type != I40E_VSI_SRIOV) {
+ if (vsi->netdev_registered) {
+@@ -14101,6 +14196,9 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
+ case I40E_VSI_MAIN:
+ case I40E_VSI_VMDQ2:
+ ret = i40e_config_netdev(vsi);
++ if (ret)
++ goto err_netdev;
++ ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
+ if (ret)
+ goto err_netdev;
+ ret = register_netdev(vsi->netdev);
+@@ -15403,8 +15501,8 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
+ hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
+- dev_info(&pdev->dev,
+- "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n",
++ dev_dbg(&pdev->dev,
++ "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
+ hw->aq.api_maj_ver,
+ hw->aq.api_min_ver,
+ I40E_FW_API_VERSION_MAJOR,
+@@ -15840,8 +15938,13 @@ static void i40e_remove(struct pci_dev *pdev)
+ i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
+ i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
+
+- while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
++ /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
++ * flags, once they are set, i40e_rebuild should not be called as
++ * i40e_prep_for_reset always returns early.
++ */
++ while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
+ usleep_range(1000, 2000);
++ set_bit(__I40E_IN_REMOVE, pf->state);
+
+ if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
+ set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
+@@ -16040,6 +16143,9 @@ static void i40e_pci_error_reset_done(struct pci_dev *pdev)
+ {
+ struct i40e_pf *pf = pci_get_drvdata(pdev);
+
++ if (test_bit(__I40E_IN_REMOVE, pf->state))
++ return;
++
+ i40e_reset_and_rebuild(pf, false, false);
+ }
+
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_register.h b/drivers/net/ethernet/intel/i40e/i40e_register.h
+index 8d0588a27a053..1908eed4fa5ee 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_register.h
++++ b/drivers/net/ethernet/intel/i40e/i40e_register.h
+@@ -413,6 +413,9 @@
+ #define I40E_VFINT_DYN_CTLN(_INTVF) (0x00024800 + ((_INTVF) * 4)) /* _i=0...511 */ /* Reset: VFR */
+ #define I40E_VFINT_DYN_CTLN_CLEARPBA_SHIFT 1
+ #define I40E_VFINT_DYN_CTLN_CLEARPBA_MASK I40E_MASK(0x1, I40E_VFINT_DYN_CTLN_CLEARPBA_SHIFT)
++#define I40E_VFINT_ICR0_ADMINQ_SHIFT 30
++#define I40E_VFINT_ICR0_ADMINQ_MASK I40E_MASK(0x1, I40E_VFINT_ICR0_ADMINQ_SHIFT)
++#define I40E_VFINT_ICR0_ENA(_VF) (0x0002C000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: CORER */
+ #define I40E_VPINT_AEQCTL(_VF) (0x0002B800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: CORER */
+ #define I40E_VPINT_AEQCTL_MSIX_INDX_SHIFT 0
+ #define I40E_VPINT_AEQCTL_ITR_INDX_SHIFT 11
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+index 472f56b360b8c..babf8b7fa7678 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
++++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+@@ -183,17 +183,18 @@ void i40e_vc_notify_vf_reset(struct i40e_vf *vf)
+ /***********************misc routines*****************************/
+
+ /**
+- * i40e_vc_disable_vf
++ * i40e_vc_reset_vf
+ * @vf: pointer to the VF info
+- *
+- * Disable the VF through a SW reset.
++ * @notify_vf: notify vf about reset or not
++ * Reset VF handler.
+ **/
+-static inline void i40e_vc_disable_vf(struct i40e_vf *vf)
++static void i40e_vc_reset_vf(struct i40e_vf *vf, bool notify_vf)
+ {
+ struct i40e_pf *pf = vf->pf;
+ int i;
+
+- i40e_vc_notify_vf_reset(vf);
++ if (notify_vf)
++ i40e_vc_notify_vf_reset(vf);
+
+ /* We want to ensure that an actual reset occurs initiated after this
+ * function was called. However, we do not want to wait forever, so
+@@ -211,9 +212,14 @@ static inline void i40e_vc_disable_vf(struct i40e_vf *vf)
+ usleep_range(10000, 20000);
+ }
+
+- dev_warn(&vf->pf->pdev->dev,
+- "Failed to initiate reset for VF %d after 200 milliseconds\n",
+- vf->vf_id);
++ if (notify_vf)
++ dev_warn(&vf->pf->pdev->dev,
++ "Failed to initiate reset for VF %d after 200 milliseconds\n",
++ vf->vf_id);
++ else
++ dev_dbg(&vf->pf->pdev->dev,
++ "Failed to initiate reset for VF %d after 200 milliseconds\n",
++ vf->vf_id);
+ }
+
+ /**
+@@ -674,14 +680,13 @@ static int i40e_config_vsi_rx_queue(struct i40e_vf *vf, u16 vsi_id,
+ u16 vsi_queue_id,
+ struct virtchnl_rxq_info *info)
+ {
++ u16 pf_queue_id = i40e_vc_get_pf_queue_id(vf, vsi_id, vsi_queue_id);
+ struct i40e_pf *pf = vf->pf;
++ struct i40e_vsi *vsi = pf->vsi[vf->lan_vsi_idx];
+ struct i40e_hw *hw = &pf->hw;
+ struct i40e_hmc_obj_rxq rx_ctx;
+- u16 pf_queue_id;
+ int ret = 0;
+
+- pf_queue_id = i40e_vc_get_pf_queue_id(vf, vsi_id, vsi_queue_id);
+-
+ /* clear the context structure first */
+ memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
+
+@@ -719,6 +724,10 @@ static int i40e_config_vsi_rx_queue(struct i40e_vf *vf, u16 vsi_id,
+ }
+ rx_ctx.rxmax = info->max_pkt_size;
+
++ /* if port VLAN is configured increase the max packet size */
++ if (vsi->info.pvid)
++ rx_ctx.rxmax += VLAN_HLEN;
++
+ /* enable 32bytes desc always */
+ rx_ctx.dsize = 1;
+
+@@ -1367,6 +1376,32 @@ static i40e_status i40e_config_vf_promiscuous_mode(struct i40e_vf *vf,
+ return aq_ret;
+ }
+
++/**
++ * i40e_sync_vfr_reset
++ * @hw: pointer to hw struct
++ * @vf_id: VF identifier
++ *
++ * Before trigger hardware reset, we need to know if no other process has
++ * reserved the hardware for any reset operations. This check is done by
++ * examining the status of the RSTAT1 register used to signal the reset.
++ **/
++static int i40e_sync_vfr_reset(struct i40e_hw *hw, int vf_id)
++{
++ u32 reg;
++ int i;
++
++ for (i = 0; i < I40E_VFR_WAIT_COUNT; i++) {
++ reg = rd32(hw, I40E_VFINT_ICR0_ENA(vf_id)) &
++ I40E_VFINT_ICR0_ADMINQ_MASK;
++ if (reg)
++ return 0;
++
++ usleep_range(100, 200);
++ }
++
++ return -EAGAIN;
++}
++
+ /**
+ * i40e_trigger_vf_reset
+ * @vf: pointer to the VF structure
+@@ -1381,9 +1416,11 @@ static void i40e_trigger_vf_reset(struct i40e_vf *vf, bool flr)
+ struct i40e_pf *pf = vf->pf;
+ struct i40e_hw *hw = &pf->hw;
+ u32 reg, reg_idx, bit_idx;
++ bool vf_active;
++ u32 radq;
+
+ /* warn the VF */
+- clear_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states);
++ vf_active = test_and_clear_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states);
+
+ /* Disable VF's configuration API during reset. The flag is re-enabled
+ * in i40e_alloc_vf_res(), when it's safe again to access VF's VSI.
+@@ -1397,7 +1434,19 @@ static void i40e_trigger_vf_reset(struct i40e_vf *vf, bool flr)
+ * just need to clean up, so don't hit the VFRTRIG register.
+ */
+ if (!flr) {
+- /* reset VF using VPGEN_VFRTRIG reg */
++ /* Sync VFR reset before trigger next one */
++ radq = rd32(hw, I40E_VFINT_ICR0_ENA(vf->vf_id)) &
++ I40E_VFINT_ICR0_ADMINQ_MASK;
++ if (vf_active && !radq)
++ /* waiting for finish reset by virtual driver */
++ if (i40e_sync_vfr_reset(hw, vf->vf_id))
++ dev_info(&pf->pdev->dev,
++ "Reset VF %d never finished\n",
++ vf->vf_id);
++
++ /* Reset VF using VPGEN_VFRTRIG reg. It is also setting
++ * in progress state in rstat1 register.
++ */
+ reg = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_id));
+ reg |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
+ wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_id), reg);
+@@ -1893,25 +1942,6 @@ static int i40e_vc_send_msg_to_vf(struct i40e_vf *vf, u32 v_opcode,
+ hw = &pf->hw;
+ abs_vf_id = vf->vf_id + hw->func_caps.vf_base_id;
+
+- /* single place to detect unsuccessful return values */
+- if (v_retval) {
+- vf->num_invalid_msgs++;
+- dev_info(&pf->pdev->dev, "VF %d failed opcode %d, retval: %d\n",
+- vf->vf_id, v_opcode, v_retval);
+- if (vf->num_invalid_msgs >
+- I40E_DEFAULT_NUM_INVALID_MSGS_ALLOWED) {
+- dev_err(&pf->pdev->dev,
+- "Number of invalid messages exceeded for VF %d\n",
+- vf->vf_id);
+- dev_err(&pf->pdev->dev, "Use PF Control I/F to enable the VF\n");
+- set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
+- }
+- } else {
+- vf->num_valid_msgs++;
+- /* reset the invalid counter, if a valid message is received. */
+- vf->num_invalid_msgs = 0;
+- }
+-
+ aq_ret = i40e_aq_send_msg_to_vf(hw, abs_vf_id, v_opcode, v_retval,
+ msg, msglen, NULL);
+ if (aq_ret) {
+@@ -1939,6 +1969,32 @@ static int i40e_vc_send_resp_to_vf(struct i40e_vf *vf,
+ return i40e_vc_send_msg_to_vf(vf, opcode, retval, NULL, 0);
+ }
+
++/**
++ * i40e_sync_vf_state
++ * @vf: pointer to the VF info
++ * @state: VF state
++ *
++ * Called from a VF message to synchronize the service with a potential
++ * VF reset state
++ **/
++static bool i40e_sync_vf_state(struct i40e_vf *vf, enum i40e_vf_states state)
++{
++ int i;
++
++ /* When handling some messages, it needs VF state to be set.
++ * It is possible that this flag is cleared during VF reset,
++ * so there is a need to wait until the end of the reset to
++ * handle the request message correctly.
++ */
++ for (i = 0; i < I40E_VF_STATE_WAIT_COUNT; i++) {
++ if (test_bit(state, &vf->vf_states))
++ return true;
++ usleep_range(10000, 20000);
++ }
++
++ return test_bit(state, &vf->vf_states);
++}
++
+ /**
+ * i40e_vc_get_version_msg
+ * @vf: pointer to the VF info
+@@ -1999,7 +2055,7 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
+ size_t len = 0;
+ int ret;
+
+- if (!test_bit(I40E_VF_STATE_INIT, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_INIT)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -2105,20 +2161,6 @@ err:
+ return ret;
+ }
+
+-/**
+- * i40e_vc_reset_vf_msg
+- * @vf: pointer to the VF info
+- *
+- * called from the VF to reset itself,
+- * unlike other virtchnl messages, PF driver
+- * doesn't send the response back to the VF
+- **/
+-static void i40e_vc_reset_vf_msg(struct i40e_vf *vf)
+-{
+- if (test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states))
+- i40e_reset_vf(vf, false);
+-}
+-
+ /**
+ * i40e_vc_config_promiscuous_mode_msg
+ * @vf: pointer to the VF info
+@@ -2136,7 +2178,7 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf, u8 *msg)
+ bool allmulti = false;
+ bool alluni = false;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err_out;
+ }
+@@ -2217,13 +2259,14 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg)
+ struct virtchnl_vsi_queue_config_info *qci =
+ (struct virtchnl_vsi_queue_config_info *)msg;
+ struct virtchnl_queue_pair_info *qpi;
+- struct i40e_pf *pf = vf->pf;
+ u16 vsi_id, vsi_queue_id = 0;
+- u16 num_qps_all = 0;
++ struct i40e_pf *pf = vf->pf;
+ i40e_status aq_ret = 0;
+ int i, j = 0, idx = 0;
++ struct i40e_vsi *vsi;
++ u16 num_qps_all = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto error_param;
+ }
+@@ -2310,9 +2353,15 @@ static int i40e_vc_config_queues_msg(struct i40e_vf *vf, u8 *msg)
+ pf->vsi[vf->lan_vsi_idx]->num_queue_pairs =
+ qci->num_queue_pairs;
+ } else {
+- for (i = 0; i < vf->num_tc; i++)
+- pf->vsi[vf->ch[i].vsi_idx]->num_queue_pairs =
+- vf->ch[i].num_qps;
++ for (i = 0; i < vf->num_tc; i++) {
++ vsi = pf->vsi[vf->ch[i].vsi_idx];
++ vsi->num_queue_pairs = vf->ch[i].num_qps;
++
++ if (i40e_update_adq_vsi_queues(vsi, i)) {
++ aq_ret = I40E_ERR_CONFIG;
++ goto error_param;
++ }
++ }
+ }
+
+ error_param:
+@@ -2366,7 +2415,7 @@ static int i40e_vc_config_irq_map_msg(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ int i;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto error_param;
+ }
+@@ -2538,7 +2587,7 @@ static int i40e_vc_disable_queues_msg(struct i40e_vf *vf, u8 *msg)
+ struct i40e_pf *pf = vf->pf;
+ i40e_status aq_ret = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto error_param;
+ }
+@@ -2570,6 +2619,59 @@ error_param:
+ aq_ret);
+ }
+
++/**
++ * i40e_check_enough_queue - find big enough queue number
++ * @vf: pointer to the VF info
++ * @needed: the number of items needed
++ *
++ * Returns the base item index of the queue, or negative for error
++ **/
++static int i40e_check_enough_queue(struct i40e_vf *vf, u16 needed)
++{
++ unsigned int i, cur_queues, more, pool_size;
++ struct i40e_lump_tracking *pile;
++ struct i40e_pf *pf = vf->pf;
++ struct i40e_vsi *vsi;
++
++ vsi = pf->vsi[vf->lan_vsi_idx];
++ cur_queues = vsi->alloc_queue_pairs;
++
++ /* if current allocated queues are enough for need */
++ if (cur_queues >= needed)
++ return vsi->base_queue;
++
++ pile = pf->qp_pile;
++ if (cur_queues > 0) {
++ /* if the allocated queues are not zero
++ * just check if there are enough queues for more
++ * behind the allocated queues.
++ */
++ more = needed - cur_queues;
++ for (i = vsi->base_queue + cur_queues;
++ i < pile->num_entries; i++) {
++ if (pile->list[i] & I40E_PILE_VALID_BIT)
++ break;
++
++ if (more-- == 1)
++ /* there is enough */
++ return vsi->base_queue;
++ }
++ }
++
++ pool_size = 0;
++ for (i = 0; i < pile->num_entries; i++) {
++ if (pile->list[i] & I40E_PILE_VALID_BIT) {
++ pool_size = 0;
++ continue;
++ }
++ if (needed <= ++pool_size)
++ /* there is enough */
++ return i;
++ }
++
++ return -ENOMEM;
++}
++
+ /**
+ * i40e_vc_request_queues_msg
+ * @vf: pointer to the VF info
+@@ -2588,7 +2690,7 @@ static int i40e_vc_request_queues_msg(struct i40e_vf *vf, u8 *msg)
+ u8 cur_pairs = vf->num_queue_pairs;
+ struct i40e_pf *pf = vf->pf;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states))
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE))
+ return -EINVAL;
+
+ if (req_pairs > I40E_MAX_VF_QUEUES) {
+@@ -2604,11 +2706,16 @@ static int i40e_vc_request_queues_msg(struct i40e_vf *vf, u8 *msg)
+ req_pairs - cur_pairs,
+ pf->queues_left);
+ vfres->num_queue_pairs = pf->queues_left + cur_pairs;
++ } else if (i40e_check_enough_queue(vf, req_pairs) < 0) {
++ dev_warn(&pf->pdev->dev,
++ "VF %d requested %d more queues, but there is not enough for it.\n",
++ vf->vf_id,
++ req_pairs - cur_pairs);
++ vfres->num_queue_pairs = cur_pairs;
+ } else {
+ /* successful request */
+ vf->num_req_queues = req_pairs;
+- i40e_vc_notify_vf_reset(vf);
+- i40e_reset_vf(vf, false);
++ i40e_vc_reset_vf(vf, true);
+ return 0;
+ }
+
+@@ -2634,7 +2741,7 @@ static int i40e_vc_get_stats_msg(struct i40e_vf *vf, u8 *msg)
+
+ memset(&stats, 0, sizeof(struct i40e_eth_stats));
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto error_param;
+ }
+@@ -2751,7 +2858,7 @@ static int i40e_vc_add_mac_addr_msg(struct i40e_vf *vf, u8 *msg)
+ i40e_status ret = 0;
+ int i;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) ||
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) ||
+ !i40e_vc_isvalid_vsi_id(vf, al->vsi_id)) {
+ ret = I40E_ERR_PARAM;
+ goto error_param;
+@@ -2802,8 +2909,8 @@ static int i40e_vc_add_mac_addr_msg(struct i40e_vf *vf, u8 *msg)
+
+ error_param:
+ /* send the response to the VF */
+- return i40e_vc_send_resp_to_vf(vf, VIRTCHNL_OP_ADD_ETH_ADDR,
+- ret);
++ return i40e_vc_send_msg_to_vf(vf, VIRTCHNL_OP_ADD_ETH_ADDR,
++ ret, NULL, 0);
+ }
+
+ /**
+@@ -2823,7 +2930,7 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg)
+ i40e_status ret = 0;
+ int i;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) ||
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) ||
+ !i40e_vc_isvalid_vsi_id(vf, al->vsi_id)) {
+ ret = I40E_ERR_PARAM;
+ goto error_param;
+@@ -2967,7 +3074,7 @@ static int i40e_vc_remove_vlan_msg(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ int i;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) ||
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) ||
+ !i40e_vc_isvalid_vsi_id(vf, vfl->vsi_id)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto error_param;
+@@ -3087,9 +3194,9 @@ static int i40e_vc_config_rss_key(struct i40e_vf *vf, u8 *msg)
+ struct i40e_vsi *vsi = NULL;
+ i40e_status aq_ret = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) ||
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) ||
+ !i40e_vc_isvalid_vsi_id(vf, vrk->vsi_id) ||
+- (vrk->key_len != I40E_HKEY_ARRAY_SIZE)) {
++ vrk->key_len != I40E_HKEY_ARRAY_SIZE) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3118,9 +3225,9 @@ static int i40e_vc_config_rss_lut(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ u16 i;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states) ||
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE) ||
+ !i40e_vc_isvalid_vsi_id(vf, vrl->vsi_id) ||
+- (vrl->lut_entries != I40E_VF_HLUT_ARRAY_SIZE)) {
++ vrl->lut_entries != I40E_VF_HLUT_ARRAY_SIZE) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3153,7 +3260,7 @@ static int i40e_vc_get_rss_hena(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ int len = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3189,7 +3296,7 @@ static int i40e_vc_set_rss_hena(struct i40e_vf *vf, u8 *msg)
+ struct i40e_hw *hw = &pf->hw;
+ i40e_status aq_ret = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3214,7 +3321,7 @@ static int i40e_vc_enable_vlan_stripping(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ struct i40e_vsi *vsi;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3240,7 +3347,7 @@ static int i40e_vc_disable_vlan_stripping(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ struct i40e_vsi *vsi;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3467,7 +3574,7 @@ static int i40e_vc_del_cloud_filter(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ int i, ret;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3598,7 +3705,7 @@ static int i40e_vc_add_cloud_filter(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ int i, ret;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err_out;
+ }
+@@ -3707,7 +3814,7 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg)
+ i40e_status aq_ret = 0;
+ u64 speed = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3796,15 +3903,9 @@ static int i40e_vc_add_qch_msg(struct i40e_vf *vf, u8 *msg)
+
+ /* set this flag only after making sure all inputs are sane */
+ vf->adq_enabled = true;
+- /* num_req_queues is set when user changes number of queues via ethtool
+- * and this causes issue for default VSI(which depends on this variable)
+- * when ADq is enabled, hence reset it.
+- */
+- vf->num_req_queues = 0;
+
+ /* reset the VF in order to allocate resources */
+- i40e_vc_notify_vf_reset(vf);
+- i40e_reset_vf(vf, false);
++ i40e_vc_reset_vf(vf, true);
+
+ return I40E_SUCCESS;
+
+@@ -3824,7 +3925,7 @@ static int i40e_vc_del_qch_msg(struct i40e_vf *vf, u8 *msg)
+ struct i40e_pf *pf = vf->pf;
+ i40e_status aq_ret = 0;
+
+- if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
++ if (!i40e_sync_vf_state(vf, I40E_VF_STATE_ACTIVE)) {
+ aq_ret = I40E_ERR_PARAM;
+ goto err;
+ }
+@@ -3844,8 +3945,7 @@ static int i40e_vc_del_qch_msg(struct i40e_vf *vf, u8 *msg)
+ }
+
+ /* reset the VF in order to allocate resources */
+- i40e_vc_notify_vf_reset(vf);
+- i40e_reset_vf(vf, false);
++ i40e_vc_reset_vf(vf, true);
+
+ return I40E_SUCCESS;
+
+@@ -3907,7 +4007,7 @@ int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode,
+ i40e_vc_notify_vf_link_state(vf);
+ break;
+ case VIRTCHNL_OP_RESET_VF:
+- i40e_vc_reset_vf_msg(vf);
++ i40e_vc_reset_vf(vf, false);
+ ret = 0;
+ break;
+ case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
+@@ -4161,7 +4261,7 @@ int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
+ /* Force the VF interface down so it has to bring up with new MAC
+ * address
+ */
+- i40e_vc_disable_vf(vf);
++ i40e_vc_reset_vf(vf, true);
+ dev_info(&pf->pdev->dev, "Bring down and up the VF interface to make this change effective.\n");
+
+ error_param:
+@@ -4169,34 +4269,6 @@ error_param:
+ return ret;
+ }
+
+-/**
+- * i40e_vsi_has_vlans - True if VSI has configured VLANs
+- * @vsi: pointer to the vsi
+- *
+- * Check if a VSI has configured any VLANs. False if we have a port VLAN or if
+- * we have no configured VLANs. Do not call while holding the
+- * mac_filter_hash_lock.
+- */
+-static bool i40e_vsi_has_vlans(struct i40e_vsi *vsi)
+-{
+- bool have_vlans;
+-
+- /* If we have a port VLAN, then the VSI cannot have any VLANs
+- * configured, as all MAC/VLAN filters will be assigned to the PVID.
+- */
+- if (vsi->info.pvid)
+- return false;
+-
+- /* Since we don't have a PVID, we know that if the device is in VLAN
+- * mode it must be because of a VLAN filter configured on this VSI.
+- */
+- spin_lock_bh(&vsi->mac_filter_hash_lock);
+- have_vlans = i40e_is_vsi_in_vlan(vsi);
+- spin_unlock_bh(&vsi->mac_filter_hash_lock);
+-
+- return have_vlans;
+-}
+-
+ /**
+ * i40e_ndo_set_vf_port_vlan
+ * @netdev: network interface device structure
+@@ -4253,19 +4325,9 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id,
+ /* duplicate request, so just return success */
+ goto error_pvid;
+
+- if (i40e_vsi_has_vlans(vsi)) {
+- dev_err(&pf->pdev->dev,
+- "VF %d has already configured VLAN filters and the administrator is requesting a port VLAN override.\nPlease unload and reload the VF driver for this change to take effect.\n",
+- vf_id);
+- /* Administrator Error - knock the VF offline until he does
+- * the right thing by reconfiguring his network correctly
+- * and then reloading the VF driver.
+- */
+- i40e_vc_disable_vf(vf);
+- /* During reset the VF got a new VSI, so refresh the pointer. */
+- vsi = pf->vsi[vf->lan_vsi_idx];
+- }
+-
++ i40e_vc_reset_vf(vf, true);
++ /* During reset the VF got a new VSI, so refresh a pointer. */
++ vsi = pf->vsi[vf->lan_vsi_idx];
+ /* Locked once because multiple functions below iterate list */
+ spin_lock_bh(&vsi->mac_filter_hash_lock);
+
+@@ -4641,7 +4703,7 @@ int i40e_ndo_set_vf_trust(struct net_device *netdev, int vf_id, bool setting)
+ goto out;
+
+ vf->trusted = setting;
+- i40e_vc_disable_vf(vf);
++ i40e_vc_reset_vf(vf, true);
+ dev_info(&pf->pdev->dev, "VF %u is now %strusted\n",
+ vf_id, setting ? "" : "un");
+
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
+index 091e32c1bb46f..a554d0a0b09bd 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
++++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
+@@ -10,14 +10,15 @@
+
+ #define I40E_VIRTCHNL_SUPPORTED_QTYPES 2
+
+-#define I40E_DEFAULT_NUM_INVALID_MSGS_ALLOWED 10
+-
+ #define I40E_VLAN_PRIORITY_SHIFT 13
+ #define I40E_VLAN_MASK 0xFFF
+ #define I40E_PRIORITY_MASK 0xE000
+
+ #define I40E_MAX_VF_PROMISC_FLAGS 3
+
++#define I40E_VF_STATE_WAIT_COUNT 20
++#define I40E_VFR_WAIT_COUNT 100
++
+ /* Various queue ctrls */
+ enum i40e_queue_ctrl {
+ I40E_QUEUE_CTRL_UNKNOWN = 0,
+@@ -89,9 +90,6 @@ struct i40e_vf {
+ u8 num_queue_pairs; /* num of qps assigned to VF vsis */
+ u8 num_req_queues; /* num of requested qps */
+ u64 num_mdd_events; /* num of mdd events detected */
+- /* num of continuous malformed or invalid msgs detected */
+- u64 num_invalid_msgs;
+- u64 num_valid_msgs; /* num of valid msgs detected */
+
+ unsigned long vf_caps; /* vf's adv. capabilities */
+ unsigned long vf_states; /* vf's runtime states */
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+index e7e778ca074c0..3f27a8ebe2ec2 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
++++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+@@ -243,21 +243,25 @@ no_buffers:
+ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
+ struct xdp_buff *xdp)
+ {
++ unsigned int totalsize = xdp->data_end - xdp->data_meta;
+ unsigned int metasize = xdp->data - xdp->data_meta;
+- unsigned int datasize = xdp->data_end - xdp->data;
+ struct sk_buff *skb;
+
++ net_prefetch(xdp->data_meta);
++
+ /* allocate a skb to store the frags */
+- skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
+- xdp->data_end - xdp->data_hard_start,
++ skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize,
+ GFP_ATOMIC | __GFP_NOWARN);
+ if (unlikely(!skb))
+ goto out;
+
+- skb_reserve(skb, xdp->data - xdp->data_hard_start);
+- memcpy(__skb_put(skb, datasize), xdp->data, datasize);
+- if (metasize)
++ memcpy(__skb_put(skb, totalsize), xdp->data_meta,
++ ALIGN(totalsize, sizeof(long)));
++
++ if (metasize) {
+ skb_metadata_set(skb, metasize);
++ __skb_pull(skb, metasize);
++ }
+
+ out:
+ xsk_buff_free(xdp);
+diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h
+index 68c80f04113c8..9a122aea69793 100644
+--- a/drivers/net/ethernet/intel/iavf/iavf.h
++++ b/drivers/net/ethernet/intel/iavf/iavf.h
+@@ -39,6 +39,7 @@
+ #include "iavf_txrx.h"
+ #include "iavf_fdir.h"
+ #include "iavf_adv_rss.h"
++#include <linux/bitmap.h>
+
+ #define DEFAULT_DEBUG_LEVEL_SHIFT 3
+ #define PFX "iavf: "
+@@ -177,6 +178,7 @@ enum iavf_state_t {
+ __IAVF_INIT_VERSION_CHECK, /* aq msg sent, awaiting reply */
+ __IAVF_INIT_GET_RESOURCES, /* aq msg sent, awaiting reply */
+ __IAVF_INIT_SW, /* got resources, setting up structs */
++ __IAVF_INIT_FAILED, /* init failed, restarting procedure */
+ __IAVF_RESETTING, /* in reset */
+ __IAVF_COMM_FAILED, /* communication with PF failed */
+ /* Below here, watchdog is running */
+@@ -186,6 +188,10 @@ enum iavf_state_t {
+ __IAVF_RUNNING, /* opened, working */
+ };
+
++enum iavf_critical_section_t {
++ __IAVF_IN_REMOVE_TASK, /* device being removed */
++};
++
+ #define IAVF_CLOUD_FIELD_OMAC 0x01
+ #define IAVF_CLOUD_FIELD_IMAC 0x02
+ #define IAVF_CLOUD_FIELD_IVLAN 0x04
+@@ -225,14 +231,12 @@ struct iavf_adapter {
+ struct work_struct reset_task;
+ struct work_struct adminq_task;
+ struct delayed_work client_task;
+- struct delayed_work init_task;
+ wait_queue_head_t down_waitqueue;
+ struct iavf_q_vector *q_vectors;
+ struct list_head vlan_filter_list;
+ struct list_head mac_filter_list;
+ struct mutex crit_lock;
+ struct mutex client_lock;
+- struct mutex remove_lock;
+ /* Lock to protect accesses to MAC and VLAN lists */
+ spinlock_t mac_vlan_list_lock;
+ char misc_vector_name[IFNAMSIZ + 9];
+@@ -270,6 +274,7 @@ struct iavf_adapter {
+ #define IAVF_FLAG_LEGACY_RX BIT(15)
+ #define IAVF_FLAG_REINIT_ITR_NEEDED BIT(16)
+ #define IAVF_FLAG_QUEUES_DISABLED BIT(17)
++#define IAVF_FLAG_SETUP_NETDEV_FEATURES BIT(18)
+ /* duplicates for common code */
+ #define IAVF_FLAG_DCB_ENABLED 0
+ /* flags for admin queue service task */
+@@ -304,6 +309,7 @@ struct iavf_adapter {
+ #define IAVF_FLAG_AQ_DEL_FDIR_FILTER BIT(26)
+ #define IAVF_FLAG_AQ_ADD_ADV_RSS_CFG BIT(27)
+ #define IAVF_FLAG_AQ_DEL_ADV_RSS_CFG BIT(28)
++#define IAVF_FLAG_AQ_REQUEST_STATS BIT(29)
+
+ /* OS defined structs */
+ struct net_device *netdev;
+@@ -312,6 +318,7 @@ struct iavf_adapter {
+ struct iavf_hw hw; /* defined in iavf_type.h */
+
+ enum iavf_state_t state;
++ enum iavf_state_t last_state;
+ unsigned long crit_section;
+
+ struct delayed_work watchdog_task;
+@@ -393,10 +400,56 @@ struct iavf_device {
+ extern char iavf_driver_name[];
+ extern struct workqueue_struct *iavf_wq;
+
++static inline const char *iavf_state_str(enum iavf_state_t state)
++{
++ switch (state) {
++ case __IAVF_STARTUP:
++ return "__IAVF_STARTUP";
++ case __IAVF_REMOVE:
++ return "__IAVF_REMOVE";
++ case __IAVF_INIT_VERSION_CHECK:
++ return "__IAVF_INIT_VERSION_CHECK";
++ case __IAVF_INIT_GET_RESOURCES:
++ return "__IAVF_INIT_GET_RESOURCES";
++ case __IAVF_INIT_SW:
++ return "__IAVF_INIT_SW";
++ case __IAVF_INIT_FAILED:
++ return "__IAVF_INIT_FAILED";
++ case __IAVF_RESETTING:
++ return "__IAVF_RESETTING";
++ case __IAVF_COMM_FAILED:
++ return "__IAVF_COMM_FAILED";
++ case __IAVF_DOWN:
++ return "__IAVF_DOWN";
++ case __IAVF_DOWN_PENDING:
++ return "__IAVF_DOWN_PENDING";
++ case __IAVF_TESTING:
++ return "__IAVF_TESTING";
++ case __IAVF_RUNNING:
++ return "__IAVF_RUNNING";
++ default:
++ return "__IAVF_UNKNOWN_STATE";
++ }
++}
++
++static inline void iavf_change_state(struct iavf_adapter *adapter,
++ enum iavf_state_t state)
++{
++ if (adapter->state != state) {
++ adapter->last_state = adapter->state;
++ adapter->state = state;
++ }
++ dev_dbg(&adapter->pdev->dev,
++ "state transition from:%s to:%s\n",
++ iavf_state_str(adapter->last_state),
++ iavf_state_str(adapter->state));
++}
++
+ int iavf_up(struct iavf_adapter *adapter);
+ void iavf_down(struct iavf_adapter *adapter);
+ int iavf_process_config(struct iavf_adapter *adapter);
+ void iavf_schedule_reset(struct iavf_adapter *adapter);
++void iavf_schedule_request_stats(struct iavf_adapter *adapter);
+ void iavf_reset(struct iavf_adapter *adapter);
+ void iavf_set_ethtool_ops(struct net_device *netdev);
+ void iavf_update_stats(struct iavf_adapter *adapter);
+@@ -454,4 +507,5 @@ void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter);
+ void iavf_del_adv_rss_cfg(struct iavf_adapter *adapter);
+ struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
+ const u8 *macaddr);
++int iavf_lock_timeout(struct mutex *lock, unsigned int msecs);
+ #endif /* _IAVF_H_ */
+diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
+index 5a359a0a20ecc..461f5237a2f88 100644
+--- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
++++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
+@@ -354,6 +354,9 @@ static void iavf_get_ethtool_stats(struct net_device *netdev,
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ unsigned int i;
+
++ /* Explicitly request stats refresh */
++ iavf_schedule_request_stats(adapter);
++
+ iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats);
+
+ rcu_read_lock();
+@@ -612,23 +615,44 @@ static int iavf_set_ringparam(struct net_device *netdev,
+ if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
+ return -EINVAL;
+
+- new_tx_count = clamp_t(u32, ring->tx_pending,
+- IAVF_MIN_TXD,
+- IAVF_MAX_TXD);
+- new_tx_count = ALIGN(new_tx_count, IAVF_REQ_DESCRIPTOR_MULTIPLE);
++ if (ring->tx_pending > IAVF_MAX_TXD ||
++ ring->tx_pending < IAVF_MIN_TXD ||
++ ring->rx_pending > IAVF_MAX_RXD ||
++ ring->rx_pending < IAVF_MIN_RXD) {
++ netdev_err(netdev, "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d] (increment %d)\n",
++ ring->tx_pending, ring->rx_pending, IAVF_MIN_TXD,
++ IAVF_MAX_RXD, IAVF_REQ_DESCRIPTOR_MULTIPLE);
++ return -EINVAL;
++ }
++
++ new_tx_count = ALIGN(ring->tx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE);
++ if (new_tx_count != ring->tx_pending)
++ netdev_info(netdev, "Requested Tx descriptor count rounded up to %d\n",
++ new_tx_count);
+
+- new_rx_count = clamp_t(u32, ring->rx_pending,
+- IAVF_MIN_RXD,
+- IAVF_MAX_RXD);
+- new_rx_count = ALIGN(new_rx_count, IAVF_REQ_DESCRIPTOR_MULTIPLE);
++ new_rx_count = ALIGN(ring->rx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE);
++ if (new_rx_count != ring->rx_pending)
++ netdev_info(netdev, "Requested Rx descriptor count rounded up to %d\n",
++ new_rx_count);
+
+ /* if nothing to do return success */
+ if ((new_tx_count == adapter->tx_desc_count) &&
+- (new_rx_count == adapter->rx_desc_count))
++ (new_rx_count == adapter->rx_desc_count)) {
++ netdev_dbg(netdev, "Nothing to change, descriptor count is same as requested\n");
+ return 0;
++ }
++
++ if (new_tx_count != adapter->tx_desc_count) {
++ netdev_dbg(netdev, "Changing Tx descriptor count from %d to %d\n",
++ adapter->tx_desc_count, new_tx_count);
++ adapter->tx_desc_count = new_tx_count;
++ }
+
+- adapter->tx_desc_count = new_tx_count;
+- adapter->rx_desc_count = new_rx_count;
++ if (new_rx_count != adapter->rx_desc_count) {
++ netdev_dbg(netdev, "Changing Rx descriptor count from %d to %d\n",
++ adapter->rx_desc_count, new_rx_count);
++ adapter->rx_desc_count = new_rx_count;
++ }
+
+ if (netif_running(netdev)) {
+ adapter->flags |= IAVF_FLAG_RESET_NEEDED;
+@@ -723,12 +747,31 @@ static int iavf_get_per_queue_coalesce(struct net_device *netdev, u32 queue,
+ *
+ * Change the ITR settings for a specific queue.
+ **/
+-static void iavf_set_itr_per_queue(struct iavf_adapter *adapter,
+- struct ethtool_coalesce *ec, int queue)
++static int iavf_set_itr_per_queue(struct iavf_adapter *adapter,
++ struct ethtool_coalesce *ec, int queue)
+ {
+ struct iavf_ring *rx_ring = &adapter->rx_rings[queue];
+ struct iavf_ring *tx_ring = &adapter->tx_rings[queue];
+ struct iavf_q_vector *q_vector;
++ u16 itr_setting;
++
++ itr_setting = rx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
++
++ if (ec->rx_coalesce_usecs != itr_setting &&
++ ec->use_adaptive_rx_coalesce) {
++ netif_info(adapter, drv, adapter->netdev,
++ "Rx interrupt throttling cannot be changed if adaptive-rx is enabled\n");
++ return -EINVAL;
++ }
++
++ itr_setting = tx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
++
++ if (ec->tx_coalesce_usecs != itr_setting &&
++ ec->use_adaptive_tx_coalesce) {
++ netif_info(adapter, drv, adapter->netdev,
++ "Tx interrupt throttling cannot be changed if adaptive-tx is enabled\n");
++ return -EINVAL;
++ }
+
+ rx_ring->itr_setting = ITR_REG_ALIGN(ec->rx_coalesce_usecs);
+ tx_ring->itr_setting = ITR_REG_ALIGN(ec->tx_coalesce_usecs);
+@@ -751,6 +794,7 @@ static void iavf_set_itr_per_queue(struct iavf_adapter *adapter,
+ * the Tx and Rx ITR values based on the values we have entered
+ * into the q_vector, no need to write the values now.
+ */
++ return 0;
+ }
+
+ /**
+@@ -792,9 +836,11 @@ static int __iavf_set_coalesce(struct net_device *netdev,
+ */
+ if (queue < 0) {
+ for (i = 0; i < adapter->num_active_queues; i++)
+- iavf_set_itr_per_queue(adapter, ec, i);
++ if (iavf_set_itr_per_queue(adapter, ec, i))
++ return -EINVAL;
+ } else if (queue < adapter->num_active_queues) {
+- iavf_set_itr_per_queue(adapter, ec, queue);
++ if (iavf_set_itr_per_queue(adapter, ec, queue))
++ return -EINVAL;
+ } else {
+ netif_info(adapter, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
+ adapter->num_active_queues - 1);
+@@ -1776,6 +1822,7 @@ static int iavf_set_channels(struct net_device *netdev,
+ {
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u32 num_req = ch->combined_count;
++ int i;
+
+ if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
+ adapter->num_tc) {
+@@ -1786,7 +1833,7 @@ static int iavf_set_channels(struct net_device *netdev,
+ /* All of these should have already been checked by ethtool before this
+ * even gets to us, but just to be sure.
+ */
+- if (num_req > adapter->vsi_res->num_queue_pairs)
++ if (num_req == 0 || num_req > adapter->vsi_res->num_queue_pairs)
+ return -EINVAL;
+
+ if (num_req == adapter->num_active_queues)
+@@ -1798,6 +1845,20 @@ static int iavf_set_channels(struct net_device *netdev,
+ adapter->num_req_queues = num_req;
+ adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
+ iavf_schedule_reset(adapter);
++
++ /* wait for the reset is done */
++ for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
++ msleep(IAVF_RESET_WAIT_MS);
++ if (adapter->flags & IAVF_FLAG_RESET_PENDING)
++ continue;
++ break;
++ }
++ if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
++ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
++ adapter->num_active_queues = num_req;
++ return -EOPNOTSUPP;
++ }
++
+ return 0;
+ }
+
+@@ -1844,14 +1905,13 @@ static int iavf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
+
+ if (hfunc)
+ *hfunc = ETH_RSS_HASH_TOP;
+- if (!indir)
+- return 0;
+-
+- memcpy(key, adapter->rss_key, adapter->rss_key_size);
++ if (key)
++ memcpy(key, adapter->rss_key, adapter->rss_key_size);
+
+- /* Each 32 bits pointed by 'indir' is stored with a lut entry */
+- for (i = 0; i < adapter->rss_lut_size; i++)
+- indir[i] = (u32)adapter->rss_lut[i];
++ if (indir)
++ /* Each 32 bits pointed by 'indir' is stored with a lut entry */
++ for (i = 0; i < adapter->rss_lut_size; i++)
++ indir[i] = (u32)adapter->rss_lut[i];
+
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
+index cada4e0e40b48..ca74824d40b85 100644
+--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
++++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
+@@ -14,7 +14,7 @@
+ static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
+ static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
+ static int iavf_close(struct net_device *netdev);
+-static int iavf_init_get_resources(struct iavf_adapter *adapter);
++static void iavf_init_get_resources(struct iavf_adapter *adapter);
+ static int iavf_check_reset_complete(struct iavf_hw *hw);
+
+ char iavf_driver_name[] = "iavf";
+@@ -51,6 +51,15 @@ MODULE_LICENSE("GPL v2");
+ static const struct net_device_ops iavf_netdev_ops;
+ struct workqueue_struct *iavf_wq;
+
++/**
++ * iavf_pdev_to_adapter - go from pci_dev to adapter
++ * @pdev: pci_dev pointer
++ */
++static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
++{
++ return netdev_priv(pci_get_drvdata(pdev));
++}
++
+ /**
+ * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
+ * @hw: pointer to the HW structure
+@@ -138,7 +147,7 @@ enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
+ *
+ * Returns 0 on success, negative on failure
+ **/
+-static int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
++int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
+ {
+ unsigned int wait, delay = 10;
+
+@@ -165,6 +174,19 @@ void iavf_schedule_reset(struct iavf_adapter *adapter)
+ }
+ }
+
++/**
++ * iavf_schedule_request_stats - Set the flags and schedule statistics request
++ * @adapter: board private structure
++ *
++ * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly
++ * request and refresh ethtool stats
++ **/
++void iavf_schedule_request_stats(struct iavf_adapter *adapter)
++{
++ adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS;
++ mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
++}
++
+ /**
+ * iavf_tx_timeout - Respond to a Tx Hang
+ * @netdev: network interface device structure
+@@ -280,8 +302,9 @@ static irqreturn_t iavf_msix_aq(int irq, void *data)
+ rd32(hw, IAVF_VFINT_ICR01);
+ rd32(hw, IAVF_VFINT_ICR0_ENA1);
+
+- /* schedule work on the private workqueue */
+- queue_work(iavf_wq, &adapter->adminq_task);
++ if (adapter->state != __IAVF_REMOVE)
++ /* schedule work on the private workqueue */
++ queue_work(iavf_wq, &adapter->adminq_task);
+
+ return IRQ_HANDLED;
+ }
+@@ -687,6 +710,21 @@ static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan)
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ }
+
++/**
++ * iavf_restore_filters
++ * @adapter: board private structure
++ *
++ * Restore existing non MAC filters when VF netdev comes back up
++ **/
++static void iavf_restore_filters(struct iavf_adapter *adapter)
++{
++ u16 vid;
++
++ /* re-add all VLAN filters */
++ for_each_set_bit(vid, adapter->vsi.active_vlans, VLAN_N_VID)
++ iavf_add_vlan(adapter, vid);
++}
++
+ /**
+ * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
+ * @netdev: network device struct
+@@ -700,8 +738,11 @@ static int iavf_vlan_rx_add_vid(struct net_device *netdev,
+
+ if (!VLAN_ALLOWED(adapter))
+ return -EIO;
++
+ if (iavf_add_vlan(adapter, vid) == NULL)
+ return -ENOMEM;
++
++ set_bit(vid, adapter->vsi.active_vlans);
+ return 0;
+ }
+
+@@ -716,11 +757,10 @@ static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
+ {
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+- if (VLAN_ALLOWED(adapter)) {
+- iavf_del_vlan(adapter, vid);
+- return 0;
+- }
+- return -EIO;
++ iavf_del_vlan(adapter, vid);
++ clear_bit(vid, adapter->vsi.active_vlans);
++
++ return 0;
+ }
+
+ /**
+@@ -960,7 +1000,7 @@ static void iavf_configure(struct iavf_adapter *adapter)
+ **/
+ static void iavf_up_complete(struct iavf_adapter *adapter)
+ {
+- adapter->state = __IAVF_RUNNING;
++ iavf_change_state(adapter, __IAVF_RUNNING);
+ clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+
+ iavf_napi_enable_all(adapter);
+@@ -1033,8 +1073,7 @@ void iavf_down(struct iavf_adapter *adapter)
+ rss->state = IAVF_ADV_RSS_DEL_REQUEST;
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+- if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
+- adapter->state != __IAVF_RESETTING) {
++ if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) {
+ /* cancel any current operation */
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ /* Schedule operations to close down the HW. Don't wait
+@@ -1630,8 +1669,7 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter)
+ iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
+ return 0;
+ }
+-
+- if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) &&
++ if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) ||
+ (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
+ iavf_set_promiscuous(adapter, 0);
+ return 0;
+@@ -1679,6 +1717,11 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter)
+ iavf_del_adv_rss_cfg(adapter);
+ return 0;
+ }
++ if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
++ iavf_request_stats(adapter);
++ return 0;
++ }
++
+ return -EAGAIN;
+ }
+
+@@ -1688,9 +1731,9 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter)
+ *
+ * Function process __IAVF_STARTUP driver state.
+ * When success the state is changed to __IAVF_INIT_VERSION_CHECK
+- * when fails it returns -EAGAIN
++ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+-static int iavf_startup(struct iavf_adapter *adapter)
++static void iavf_startup(struct iavf_adapter *adapter)
+ {
+ struct pci_dev *pdev = adapter->pdev;
+ struct iavf_hw *hw = &adapter->hw;
+@@ -1729,9 +1772,10 @@ static int iavf_startup(struct iavf_adapter *adapter)
+ iavf_shutdown_adminq(hw);
+ goto err;
+ }
+- adapter->state = __IAVF_INIT_VERSION_CHECK;
++ iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
++ return;
+ err:
+- return err;
++ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+ }
+
+ /**
+@@ -1740,9 +1784,9 @@ err:
+ *
+ * Function process __IAVF_INIT_VERSION_CHECK driver state.
+ * When success the state is changed to __IAVF_INIT_GET_RESOURCES
+- * when fails it returns -EAGAIN
++ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+-static int iavf_init_version_check(struct iavf_adapter *adapter)
++static void iavf_init_version_check(struct iavf_adapter *adapter)
+ {
+ struct pci_dev *pdev = adapter->pdev;
+ struct iavf_hw *hw = &adapter->hw;
+@@ -1753,7 +1797,7 @@ static int iavf_init_version_check(struct iavf_adapter *adapter)
+ if (!iavf_asq_done(hw)) {
+ dev_err(&pdev->dev, "Admin queue command never completed\n");
+ iavf_shutdown_adminq(hw);
+- adapter->state = __IAVF_STARTUP;
++ iavf_change_state(adapter, __IAVF_STARTUP);
+ goto err;
+ }
+
+@@ -1776,10 +1820,10 @@ static int iavf_init_version_check(struct iavf_adapter *adapter)
+ err);
+ goto err;
+ }
+- adapter->state = __IAVF_INIT_GET_RESOURCES;
+-
++ iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
++ return;
+ err:
+- return err;
++ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+ }
+
+ /**
+@@ -1789,9 +1833,9 @@ err:
+ * Function process __IAVF_INIT_GET_RESOURCES driver state and
+ * finishes driver initialization procedure.
+ * When success the state is changed to __IAVF_DOWN
+- * when fails it returns -EAGAIN
++ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+-static int iavf_init_get_resources(struct iavf_adapter *adapter)
++static void iavf_init_get_resources(struct iavf_adapter *adapter)
+ {
+ struct net_device *netdev = adapter->netdev;
+ struct pci_dev *pdev = adapter->pdev;
+@@ -1819,7 +1863,7 @@ static int iavf_init_get_resources(struct iavf_adapter *adapter)
+ */
+ iavf_shutdown_adminq(hw);
+ dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
+- return 0;
++ return;
+ }
+ if (err) {
+ dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
+@@ -1893,7 +1937,7 @@ static int iavf_init_get_resources(struct iavf_adapter *adapter)
+ if (netdev->features & NETIF_F_GRO)
+ dev_info(&pdev->dev, "GRO is enabled\n");
+
+- adapter->state = __IAVF_DOWN;
++ iavf_change_state(adapter, __IAVF_DOWN);
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+ rtnl_unlock();
+
+@@ -1911,7 +1955,7 @@ static int iavf_init_get_resources(struct iavf_adapter *adapter)
+ else
+ iavf_init_rss(adapter);
+
+- return err;
++ return;
+ err_mem:
+ iavf_free_rss(adapter);
+ err_register:
+@@ -1922,7 +1966,7 @@ err_alloc:
+ kfree(adapter->vf_res);
+ adapter->vf_res = NULL;
+ err:
+- return err;
++ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+ }
+
+ /**
+@@ -1937,14 +1981,80 @@ static void iavf_watchdog_task(struct work_struct *work)
+ struct iavf_hw *hw = &adapter->hw;
+ u32 reg_val;
+
+- if (!mutex_trylock(&adapter->crit_lock))
++ if (!mutex_trylock(&adapter->crit_lock)) {
++ if (adapter->state == __IAVF_REMOVE)
++ return;
++
+ goto restart_watchdog;
++ }
+
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
+- adapter->state = __IAVF_COMM_FAILED;
++ iavf_change_state(adapter, __IAVF_COMM_FAILED);
++
++ if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
++ adapter->aq_required = 0;
++ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
++ mutex_unlock(&adapter->crit_lock);
++ queue_work(iavf_wq, &adapter->reset_task);
++ return;
++ }
+
+ switch (adapter->state) {
++ case __IAVF_STARTUP:
++ iavf_startup(adapter);
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq, &adapter->watchdog_task,
++ msecs_to_jiffies(30));
++ return;
++ case __IAVF_INIT_VERSION_CHECK:
++ iavf_init_version_check(adapter);
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq, &adapter->watchdog_task,
++ msecs_to_jiffies(30));
++ return;
++ case __IAVF_INIT_GET_RESOURCES:
++ iavf_init_get_resources(adapter);
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq, &adapter->watchdog_task,
++ msecs_to_jiffies(1));
++ return;
++ case __IAVF_INIT_FAILED:
++ if (test_bit(__IAVF_IN_REMOVE_TASK,
++ &adapter->crit_section)) {
++ /* Do not update the state and do not reschedule
++ * watchdog task, iavf_remove should handle this state
++ * as it can loop forever
++ */
++ mutex_unlock(&adapter->crit_lock);
++ return;
++ }
++ if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
++ dev_err(&adapter->pdev->dev,
++ "Failed to communicate with PF; waiting before retry\n");
++ adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
++ iavf_shutdown_adminq(hw);
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq,
++ &adapter->watchdog_task, (5 * HZ));
++ return;
++ }
++ /* Try again from failed step*/
++ iavf_change_state(adapter, adapter->last_state);
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ);
++ return;
+ case __IAVF_COMM_FAILED:
++ if (test_bit(__IAVF_IN_REMOVE_TASK,
++ &adapter->crit_section)) {
++ /* Set state to __IAVF_INIT_FAILED and perform remove
++ * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
++ * doesn't bring the state back to __IAVF_COMM_FAILED.
++ */
++ iavf_change_state(adapter, __IAVF_INIT_FAILED);
++ adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
++ mutex_unlock(&adapter->crit_lock);
++ return;
++ }
+ reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
+ IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
+ if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
+@@ -1952,23 +2062,20 @@ static void iavf_watchdog_task(struct work_struct *work)
+ /* A chance for redemption! */
+ dev_err(&adapter->pdev->dev,
+ "Hardware came out of reset. Attempting reinit.\n");
+- adapter->state = __IAVF_STARTUP;
+- adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
+- queue_delayed_work(iavf_wq, &adapter->init_task, 10);
+- mutex_unlock(&adapter->crit_lock);
+- /* Don't reschedule the watchdog, since we've restarted
+- * the init task. When init_task contacts the PF and
++ /* When init task contacts the PF and
+ * gets everything set up again, it'll restart the
+ * watchdog for us. Down, boy. Sit. Stay. Woof.
+ */
+- return;
++ iavf_change_state(adapter, __IAVF_STARTUP);
++ adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
+ }
+ adapter->aq_required = 0;
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
++ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(iavf_wq,
+ &adapter->watchdog_task,
+ msecs_to_jiffies(10));
+- goto watchdog_done;
++ return;
+ case __IAVF_RESETTING:
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
+@@ -1991,15 +2098,16 @@ static void iavf_watchdog_task(struct work_struct *work)
+ adapter->state == __IAVF_RUNNING)
+ iavf_request_stats(adapter);
+ }
++ if (adapter->state == __IAVF_RUNNING)
++ iavf_detect_recover_hung(&adapter->vsi);
+ break;
+ case __IAVF_REMOVE:
++ default:
+ mutex_unlock(&adapter->crit_lock);
+ return;
+- default:
+- goto restart_watchdog;
+ }
+
+- /* check for hw reset */
++ /* check for hw reset */
+ reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
+ if (!reg_val) {
+ adapter->flags |= IAVF_FLAG_RESET_PENDING;
+@@ -2007,24 +2115,31 @@ static void iavf_watchdog_task(struct work_struct *work)
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
+ queue_work(iavf_wq, &adapter->reset_task);
+- goto watchdog_done;
++ mutex_unlock(&adapter->crit_lock);
++ queue_delayed_work(iavf_wq,
++ &adapter->watchdog_task, HZ * 2);
++ return;
+ }
+
+ schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
+-watchdog_done:
+- if (adapter->state == __IAVF_RUNNING ||
+- adapter->state == __IAVF_COMM_FAILED)
+- iavf_detect_recover_hung(&adapter->vsi);
+ mutex_unlock(&adapter->crit_lock);
+ restart_watchdog:
++ if (adapter->state >= __IAVF_DOWN)
++ queue_work(iavf_wq, &adapter->adminq_task);
+ if (adapter->aq_required)
+ queue_delayed_work(iavf_wq, &adapter->watchdog_task,
+ msecs_to_jiffies(20));
+ else
+ queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
+- queue_work(iavf_wq, &adapter->adminq_task);
+ }
+
++/**
++ * iavf_disable_vf - disable VF
++ * @adapter: board private structure
++ *
++ * Set communication failed flag and free all resources.
++ * NOTE: This function is expected to be called with crit_lock being held.
++ **/
+ static void iavf_disable_vf(struct iavf_adapter *adapter)
+ {
+ struct iavf_mac_filter *f, *ftmp;
+@@ -2074,14 +2189,13 @@ static void iavf_disable_vf(struct iavf_adapter *adapter)
+
+ iavf_free_misc_irq(adapter);
+ iavf_reset_interrupt_capability(adapter);
+- iavf_free_queues(adapter);
+ iavf_free_q_vectors(adapter);
++ iavf_free_queues(adapter);
+ memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
+ iavf_shutdown_adminq(&adapter->hw);
+ adapter->netdev->flags &= ~IFF_UP;
+- mutex_unlock(&adapter->crit_lock);
+ adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
+- adapter->state = __IAVF_DOWN;
++ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
+ }
+@@ -2103,7 +2217,6 @@ static void iavf_reset_task(struct work_struct *work)
+ struct net_device *netdev = adapter->netdev;
+ struct iavf_hw *hw = &adapter->hw;
+ struct iavf_mac_filter *f, *ftmp;
+- struct iavf_vlan_filter *vlf;
+ struct iavf_cloud_filter *cf;
+ u32 reg_val;
+ int i = 0, err;
+@@ -2112,13 +2225,13 @@ static void iavf_reset_task(struct work_struct *work)
+ /* When device is being removed it doesn't make sense to run the reset
+ * task, just return in such a case.
+ */
+- if (mutex_is_locked(&adapter->remove_lock))
+- return;
++ if (!mutex_trylock(&adapter->crit_lock)) {
++ if (adapter->state != __IAVF_REMOVE)
++ queue_work(iavf_wq, &adapter->reset_task);
+
+- if (iavf_lock_timeout(&adapter->crit_lock, 200)) {
+- schedule_work(&adapter->reset_task);
+ return;
+ }
++
+ while (!mutex_trylock(&adapter->client_lock))
+ usleep_range(500, 1000);
+ if (CLIENT_ENABLED(adapter)) {
+@@ -2166,12 +2279,14 @@ static void iavf_reset_task(struct work_struct *work)
+ }
+
+ pci_set_master(adapter->pdev);
++ pci_restore_msi_state(adapter->pdev);
+
+ if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
+ dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
+ reg_val);
+ iavf_disable_vf(adapter);
+ mutex_unlock(&adapter->client_lock);
++ mutex_unlock(&adapter->crit_lock);
+ return; /* Do not attempt to reinit. It's dead, Jim. */
+ }
+
+@@ -2180,8 +2295,7 @@ continue_reset:
+ * ndo_open() returning, so we can't assume it means all our open
+ * tasks have finished, since we're not holding the rtnl_lock here.
+ */
+- running = ((adapter->state == __IAVF_RUNNING) ||
+- (adapter->state == __IAVF_RESETTING));
++ running = adapter->state == __IAVF_RUNNING;
+
+ if (running) {
+ netif_carrier_off(netdev);
+@@ -2191,7 +2305,7 @@ continue_reset:
+ }
+ iavf_irq_disable(adapter);
+
+- adapter->state = __IAVF_RESETTING;
++ iavf_change_state(adapter, __IAVF_RESETTING);
+ adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
+
+ /* free the Tx/Rx rings and descriptors, might be better to just
+@@ -2243,11 +2357,6 @@ continue_reset:
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ f->add = true;
+ }
+- /* re-add all VLAN filters */
+- list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
+- vlf->add = true;
+- }
+-
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ /* check if TCs are running and re-add all cloud filters */
+@@ -2261,7 +2370,6 @@ continue_reset:
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
+- adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
+ iavf_misc_irq_enable(adapter);
+
+@@ -2291,11 +2399,14 @@ continue_reset:
+
+ iavf_configure(adapter);
+
++ /* iavf_up_complete() will switch device back
++ * to __IAVF_RUNNING
++ */
+ iavf_up_complete(adapter);
+
+ iavf_irq_enable(adapter, true);
+ } else {
+- adapter->state = __IAVF_DOWN;
++ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ }
+ mutex_unlock(&adapter->client_lock);
+@@ -2305,6 +2416,8 @@ continue_reset:
+ reset_err:
+ mutex_unlock(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
++ if (running)
++ iavf_change_state(adapter, __IAVF_RUNNING);
+ dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
+ iavf_close(netdev);
+ }
+@@ -2327,13 +2440,19 @@ static void iavf_adminq_task(struct work_struct *work)
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
+ goto out;
+
++ if (!mutex_trylock(&adapter->crit_lock)) {
++ if (adapter->state == __IAVF_REMOVE)
++ return;
++
++ queue_work(iavf_wq, &adapter->adminq_task);
++ goto out;
++ }
++
+ event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
+ event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
+ if (!event.msg_buf)
+ goto out;
+
+- if (iavf_lock_timeout(&adapter->crit_lock, 200))
+- goto freedom;
+ do {
+ ret = iavf_clean_arq_element(hw, &event, &pending);
+ v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
+@@ -2349,6 +2468,18 @@ static void iavf_adminq_task(struct work_struct *work)
+ } while (pending);
+ mutex_unlock(&adapter->crit_lock);
+
++ if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES)) {
++ if (adapter->netdev_registered ||
++ !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
++ struct net_device *netdev = adapter->netdev;
++
++ rtnl_lock();
++ netdev_update_features(netdev);
++ rtnl_unlock();
++ }
++
++ adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
++ }
+ if ((adapter->flags &
+ (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
+ adapter->state == __IAVF_RESETTING)
+@@ -2356,7 +2487,7 @@ static void iavf_adminq_task(struct work_struct *work)
+
+ /* check for error indications */
+ val = rd32(hw, hw->aq.arq.len);
+- if (val == 0xdeadbeef) /* indicates device in reset */
++ if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
+ goto freedom;
+ oldval = val;
+ if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
+@@ -2624,8 +2755,11 @@ static int iavf_validate_ch_config(struct iavf_adapter *adapter,
+ total_max_rate += tx_rate;
+ num_qps += mqprio_qopt->qopt.count[i];
+ }
+- if (num_qps > IAVF_MAX_REQ_QUEUES)
++ if (num_qps > adapter->num_active_queues) {
++ dev_err(&adapter->pdev->dev,
++ "Cannot support requested number of queues\n");
+ return -EINVAL;
++ }
+
+ ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
+ return ret;
+@@ -3041,8 +3175,10 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter,
+ return -ENOMEM;
+
+ while (!mutex_trylock(&adapter->crit_lock)) {
+- if (--count == 0)
+- goto err;
++ if (--count == 0) {
++ kfree(filter);
++ return err;
++ }
+ udelay(1);
+ }
+
+@@ -3053,11 +3189,11 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter,
+ /* start out with flow type and eth type IPv4 to begin with */
+ filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
+ err = iavf_parse_cls_flower(adapter, cls_flower, filter);
+- if (err < 0)
++ if (err)
+ goto err;
+
+ err = iavf_handle_tclass(adapter, tc, filter);
+- if (err < 0)
++ if (err)
+ goto err;
+
+ /* add filter to the list */
+@@ -3226,6 +3362,13 @@ static int iavf_open(struct net_device *netdev)
+ goto err_unlock;
+ }
+
++ if (adapter->state == __IAVF_RUNNING &&
++ !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
++ dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
++ err = 0;
++ goto err_unlock;
++ }
++
+ /* allocate transmit descriptors */
+ err = iavf_setup_all_tx_resources(adapter);
+ if (err)
+@@ -3247,6 +3390,9 @@ static int iavf_open(struct net_device *netdev)
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
++ /* Restore VLAN filters that were removed with IFF_DOWN */
++ iavf_restore_filters(adapter);
++
+ iavf_configure(adapter);
+
+ iavf_up_complete(adapter);
+@@ -3286,18 +3432,19 @@ static int iavf_close(struct net_device *netdev)
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int status;
+
+- if (adapter->state <= __IAVF_DOWN_PENDING)
+- return 0;
++ mutex_lock(&adapter->crit_lock);
+
+- while (!mutex_trylock(&adapter->crit_lock))
+- usleep_range(500, 1000);
++ if (adapter->state <= __IAVF_DOWN_PENDING) {
++ mutex_unlock(&adapter->crit_lock);
++ return 0;
++ }
+
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+ if (CLIENT_ENABLED(adapter))
+ adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
+
+ iavf_down(adapter);
+- adapter->state = __IAVF_DOWN_PENDING;
++ iavf_change_state(adapter, __IAVF_DOWN_PENDING);
+ iavf_free_traffic_irqs(adapter);
+
+ mutex_unlock(&adapter->crit_lock);
+@@ -3337,8 +3484,11 @@ static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
+ iavf_notify_client_l2_params(&adapter->vsi);
+ adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
+ }
+- adapter->flags |= IAVF_FLAG_RESET_NEEDED;
+- queue_work(iavf_wq, &adapter->reset_task);
++
++ if (netif_running(netdev)) {
++ adapter->flags |= IAVF_FLAG_RESET_NEEDED;
++ queue_work(iavf_wq, &adapter->reset_task);
++ }
+
+ return 0;
+ }
+@@ -3354,11 +3504,16 @@ static int iavf_set_features(struct net_device *netdev,
+ {
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+- /* Don't allow changing VLAN_RX flag when adapter is not capable
+- * of VLAN offload
++ /* Don't allow enabling VLAN features when adapter is not capable
++ * of VLAN offload/filtering
+ */
+ if (!VLAN_ALLOWED(adapter)) {
+- if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX)
++ netdev->hw_features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
++ NETIF_F_HW_VLAN_CTAG_TX |
++ NETIF_F_HW_VLAN_CTAG_FILTER);
++ if (features & (NETIF_F_HW_VLAN_CTAG_RX |
++ NETIF_F_HW_VLAN_CTAG_TX |
++ NETIF_F_HW_VLAN_CTAG_FILTER))
+ return -EINVAL;
+ } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
+ if (features & NETIF_F_HW_VLAN_CTAG_RX)
+@@ -3442,7 +3597,8 @@ static netdev_features_t iavf_fix_features(struct net_device *netdev,
+ {
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+- if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
++ if (adapter->vf_res &&
++ !(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
+ features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
+ NETIF_F_HW_VLAN_CTAG_RX |
+ NETIF_F_HW_VLAN_CTAG_FILTER);
+@@ -3630,72 +3786,14 @@ int iavf_process_config(struct iavf_adapter *adapter)
+ return 0;
+ }
+
+-/**
+- * iavf_init_task - worker thread to perform delayed initialization
+- * @work: pointer to work_struct containing our data
+- *
+- * This task completes the work that was begun in probe. Due to the nature
+- * of VF-PF communications, we may need to wait tens of milliseconds to get
+- * responses back from the PF. Rather than busy-wait in probe and bog down the
+- * whole system, we'll do it in a task so we can sleep.
+- * This task only runs during driver init. Once we've established
+- * communications with the PF driver and set up our netdev, the watchdog
+- * takes over.
+- **/
+-static void iavf_init_task(struct work_struct *work)
+-{
+- struct iavf_adapter *adapter = container_of(work,
+- struct iavf_adapter,
+- init_task.work);
+- struct iavf_hw *hw = &adapter->hw;
+-
+- if (iavf_lock_timeout(&adapter->crit_lock, 5000)) {
+- dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
+- return;
+- }
+- switch (adapter->state) {
+- case __IAVF_STARTUP:
+- if (iavf_startup(adapter) < 0)
+- goto init_failed;
+- break;
+- case __IAVF_INIT_VERSION_CHECK:
+- if (iavf_init_version_check(adapter) < 0)
+- goto init_failed;
+- break;
+- case __IAVF_INIT_GET_RESOURCES:
+- if (iavf_init_get_resources(adapter) < 0)
+- goto init_failed;
+- goto out;
+- default:
+- goto init_failed;
+- }
+-
+- queue_delayed_work(iavf_wq, &adapter->init_task,
+- msecs_to_jiffies(30));
+- goto out;
+-init_failed:
+- if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
+- dev_err(&adapter->pdev->dev,
+- "Failed to communicate with PF; waiting before retry\n");
+- adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
+- iavf_shutdown_adminq(hw);
+- adapter->state = __IAVF_STARTUP;
+- queue_delayed_work(iavf_wq, &adapter->init_task, HZ * 5);
+- goto out;
+- }
+- queue_delayed_work(iavf_wq, &adapter->init_task, HZ);
+-out:
+- mutex_unlock(&adapter->crit_lock);
+-}
+-
+ /**
+ * iavf_shutdown - Shutdown the device in preparation for a reboot
+ * @pdev: pci device structure
+ **/
+ static void iavf_shutdown(struct pci_dev *pdev)
+ {
+- struct net_device *netdev = pci_get_drvdata(pdev);
+- struct iavf_adapter *adapter = netdev_priv(netdev);
++ struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
++ struct net_device *netdev = adapter->netdev;
+
+ netif_device_detach(netdev);
+
+@@ -3705,7 +3803,7 @@ static void iavf_shutdown(struct pci_dev *pdev)
+ if (iavf_lock_timeout(&adapter->crit_lock, 5000))
+ dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
+ /* Prevent the watchdog from running. */
+- adapter->state = __IAVF_REMOVE;
++ iavf_change_state(adapter, __IAVF_REMOVE);
+ adapter->aq_required = 0;
+ mutex_unlock(&adapter->crit_lock);
+
+@@ -3778,7 +3876,7 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ hw->back = adapter;
+
+ adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
+- adapter->state = __IAVF_STARTUP;
++ iavf_change_state(adapter, __IAVF_STARTUP);
+
+ /* Call save state here because it relies on the adapter struct. */
+ pci_save_state(pdev);
+@@ -3803,7 +3901,6 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ */
+ mutex_init(&adapter->crit_lock);
+ mutex_init(&adapter->client_lock);
+- mutex_init(&adapter->remove_lock);
+ mutex_init(&hw->aq.asq_mutex);
+ mutex_init(&hw->aq.arq_mutex);
+
+@@ -3822,8 +3919,7 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
+ INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
+ INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
+- INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task);
+- queue_delayed_work(iavf_wq, &adapter->init_task,
++ queue_delayed_work(iavf_wq, &adapter->watchdog_task,
+ msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
+
+ /* Setup the wait queue for indicating transition to down status */
+@@ -3880,10 +3976,11 @@ static int __maybe_unused iavf_suspend(struct device *dev_d)
+ static int __maybe_unused iavf_resume(struct device *dev_d)
+ {
+ struct pci_dev *pdev = to_pci_dev(dev_d);
+- struct net_device *netdev = pci_get_drvdata(pdev);
+- struct iavf_adapter *adapter = netdev_priv(netdev);
++ struct iavf_adapter *adapter;
+ u32 err;
+
++ adapter = iavf_pdev_to_adapter(pdev);
++
+ pci_set_master(pdev);
+
+ rtnl_lock();
+@@ -3902,7 +3999,7 @@ static int __maybe_unused iavf_resume(struct device *dev_d)
+
+ queue_work(iavf_wq, &adapter->reset_task);
+
+- netif_device_attach(netdev);
++ netif_device_attach(adapter->netdev);
+
+ return err;
+ }
+@@ -3918,8 +4015,8 @@ static int __maybe_unused iavf_resume(struct device *dev_d)
+ **/
+ static void iavf_remove(struct pci_dev *pdev)
+ {
+- struct net_device *netdev = pci_get_drvdata(pdev);
+- struct iavf_adapter *adapter = netdev_priv(netdev);
++ struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
++ struct net_device *netdev = adapter->netdev;
+ struct iavf_fdir_fltr *fdir, *fdirtmp;
+ struct iavf_vlan_filter *vlf, *vlftmp;
+ struct iavf_adv_rss *rss, *rsstmp;
+@@ -3927,14 +4024,37 @@ static void iavf_remove(struct pci_dev *pdev)
+ struct iavf_cloud_filter *cf, *cftmp;
+ struct iavf_hw *hw = &adapter->hw;
+ int err;
+- /* Indicate we are in remove and not to run reset_task */
+- mutex_lock(&adapter->remove_lock);
+- cancel_delayed_work_sync(&adapter->init_task);
+- cancel_work_sync(&adapter->reset_task);
+- cancel_delayed_work_sync(&adapter->client_task);
++
++ /* When reboot/shutdown is in progress no need to do anything
++ * as the adapter is already REMOVE state that was set during
++ * iavf_shutdown() callback.
++ */
++ if (adapter->state == __IAVF_REMOVE)
++ return;
++
++ set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section);
++ /* Wait until port initialization is complete.
++ * There are flows where register/unregister netdev may race.
++ */
++ while (1) {
++ mutex_lock(&adapter->crit_lock);
++ if (adapter->state == __IAVF_RUNNING ||
++ adapter->state == __IAVF_DOWN ||
++ adapter->state == __IAVF_INIT_FAILED) {
++ mutex_unlock(&adapter->crit_lock);
++ break;
++ }
++
++ mutex_unlock(&adapter->crit_lock);
++ usleep_range(500, 1000);
++ }
++ cancel_delayed_work_sync(&adapter->watchdog_task);
++
+ if (adapter->netdev_registered) {
+- unregister_netdev(netdev);
++ rtnl_lock();
++ unregister_netdevice(netdev);
+ adapter->netdev_registered = false;
++ rtnl_unlock();
+ }
+ if (CLIENT_ALLOWED(adapter)) {
+ err = iavf_lan_del_device(adapter);
+@@ -3943,6 +4063,10 @@ static void iavf_remove(struct pci_dev *pdev)
+ err);
+ }
+
++ mutex_lock(&adapter->crit_lock);
++ dev_info(&adapter->pdev->dev, "Remove device\n");
++ iavf_change_state(adapter, __IAVF_REMOVE);
++
+ iavf_request_reset(adapter);
+ msleep(50);
+ /* If the FW isn't responding, kick it once, but only once. */
+@@ -3950,24 +4074,24 @@ static void iavf_remove(struct pci_dev *pdev)
+ iavf_request_reset(adapter);
+ msleep(50);
+ }
+- if (iavf_lock_timeout(&adapter->crit_lock, 5000))
+- dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
+
++ iavf_misc_irq_disable(adapter);
+ /* Shut down all the garbage mashers on the detention level */
+- adapter->state = __IAVF_REMOVE;
++ cancel_work_sync(&adapter->reset_task);
++ cancel_delayed_work_sync(&adapter->watchdog_task);
++ cancel_work_sync(&adapter->adminq_task);
++ cancel_delayed_work_sync(&adapter->client_task);
++
+ adapter->aq_required = 0;
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
++
+ iavf_free_all_tx_resources(adapter);
+ iavf_free_all_rx_resources(adapter);
+- iavf_misc_irq_disable(adapter);
+ iavf_free_misc_irq(adapter);
++
+ iavf_reset_interrupt_capability(adapter);
+ iavf_free_q_vectors(adapter);
+
+- cancel_delayed_work_sync(&adapter->watchdog_task);
+-
+- cancel_work_sync(&adapter->adminq_task);
+-
+ iavf_free_rss(adapter);
+
+ if (hw->aq.asq.count)
+@@ -3979,8 +4103,6 @@ static void iavf_remove(struct pci_dev *pdev)
+ mutex_destroy(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
+ mutex_destroy(&adapter->crit_lock);
+- mutex_unlock(&adapter->remove_lock);
+- mutex_destroy(&adapter->remove_lock);
+
+ iounmap(hw->hw_addr);
+ pci_release_regions(pdev);
+diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
+index 3c735968e1b85..7013769fc0389 100644
+--- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
++++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
+@@ -607,7 +607,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter)
+ if (f->add)
+ count++;
+ }
+- if (!count) {
++ if (!count || !VLAN_ALLOWED(adapter)) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+@@ -673,9 +673,19 @@ void iavf_del_vlans(struct iavf_adapter *adapter)
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+- list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+- if (f->remove)
++ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
++ /* since VLAN capabilities are not allowed, we dont want to send
++ * a VLAN delete request because it will most likely fail and
++ * create unnecessary errors/noise, so just free the VLAN
++ * filters marked for removal to enable bailing out before
++ * sending a virtchnl message
++ */
++ if (f->remove && !VLAN_ALLOWED(adapter)) {
++ list_del(&f->list);
++ kfree(f);
++ } else if (f->remove) {
+ count++;
++ }
+ }
+ if (!count) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+@@ -784,6 +794,8 @@ void iavf_request_stats(struct iavf_adapter *adapter)
+ /* no error message, this isn't crucial */
+ return;
+ }
++
++ adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_STATS;
+ adapter->current_op = VIRTCHNL_OP_GET_STATS;
+ vqs.vsi_id = adapter->vsi_res->vsi_id;
+ /* queue maps are ignored for this message - only the vsi is used */
+@@ -1448,6 +1460,22 @@ void iavf_request_reset(struct iavf_adapter *adapter)
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ }
+
++/**
++ * iavf_netdev_features_vlan_strip_set - update vlan strip status
++ * @netdev: ptr to netdev being adjusted
++ * @enable: enable or disable vlan strip
++ *
++ * Helper function to change vlan strip status in netdev->features.
++ */
++static void iavf_netdev_features_vlan_strip_set(struct net_device *netdev,
++ const bool enable)
++{
++ if (enable)
++ netdev->features |= NETIF_F_HW_VLAN_CTAG_RX;
++ else
++ netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
++}
++
+ /**
+ * iavf_virtchnl_completion
+ * @adapter: adapter structure
+@@ -1671,8 +1699,18 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ }
+ break;
+ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
++ dev_warn(&adapter->pdev->dev, "Changing VLAN Stripping is not allowed when Port VLAN is configured\n");
++ /* Vlan stripping could not be enabled by ethtool.
++ * Disable it in netdev->features.
++ */
++ iavf_netdev_features_vlan_strip_set(netdev, false);
++ break;
+ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
+ dev_warn(&adapter->pdev->dev, "Changing VLAN Stripping is not allowed when Port VLAN is configured\n");
++ /* Vlan stripping could not be disabled by ethtool.
++ * Enable it in netdev->features.
++ */
++ iavf_netdev_features_vlan_strip_set(netdev, true);
+ break;
+ default:
+ dev_err(&adapter->pdev->dev, "PF returned error %d (%s) to our request %d\n",
+@@ -1722,8 +1760,25 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ }
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ iavf_add_filter(adapter, adapter->hw.mac.addr);
++
++ if (VLAN_ALLOWED(adapter)) {
++ if (!list_empty(&adapter->vlan_filter_list)) {
++ struct iavf_vlan_filter *vlf;
++
++ /* re-add all VLAN filters over virtchnl */
++ list_for_each_entry(vlf,
++ &adapter->vlan_filter_list,
++ list)
++ vlf->add = true;
++
++ adapter->aq_required |=
++ IAVF_FLAG_AQ_ADD_VLAN_FILTER;
++ }
++ }
++
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ iavf_process_config(adapter);
++ adapter->flags |= IAVF_FLAG_SETUP_NETDEV_FEATURES;
+ }
+ break;
+ case VIRTCHNL_OP_ENABLE_QUEUES:
+@@ -1735,7 +1790,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ iavf_free_all_tx_resources(adapter);
+ iavf_free_all_rx_resources(adapter);
+ if (adapter->state == __IAVF_DOWN_PENDING) {
+- adapter->state = __IAVF_DOWN;
++ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ }
+ break;
+@@ -1889,6 +1944,20 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ spin_unlock_bh(&adapter->adv_rss_lock);
+ }
+ break;
++ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
++ /* PF enabled vlan strip on this VF.
++ * Update netdev->features if needed to be in sync with ethtool.
++ */
++ if (!v_retval)
++ iavf_netdev_features_vlan_strip_set(netdev, true);
++ break;
++ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
++ /* PF disabled vlan strip on this VF.
++ * Update netdev->features if needed to be in sync with ethtool.
++ */
++ if (!v_retval)
++ iavf_netdev_features_vlan_strip_set(netdev, false);
++ break;
+ default:
+ if (adapter->current_op && (v_opcode != adapter->current_op))
+ dev_warn(&adapter->pdev->dev, "Expected response %d from PF, received %d\n",
+diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
+index 3c4f08d20414e..89bca2ed895a0 100644
+--- a/drivers/net/ethernet/intel/ice/ice.h
++++ b/drivers/net/ethernet/intel/ice/ice.h
+@@ -139,13 +139,10 @@
+ #define ice_for_each_q_vector(vsi, i) \
+ for ((i) = 0; (i) < (vsi)->num_q_vectors; (i)++)
+
+-#define ICE_UCAST_PROMISC_BITS (ICE_PROMISC_UCAST_TX | ICE_PROMISC_MCAST_TX | \
+- ICE_PROMISC_UCAST_RX | ICE_PROMISC_MCAST_RX)
++#define ICE_UCAST_PROMISC_BITS (ICE_PROMISC_UCAST_TX | ICE_PROMISC_UCAST_RX)
+
+ #define ICE_UCAST_VLAN_PROMISC_BITS (ICE_PROMISC_UCAST_TX | \
+- ICE_PROMISC_MCAST_TX | \
+ ICE_PROMISC_UCAST_RX | \
+- ICE_PROMISC_MCAST_RX | \
+ ICE_PROMISC_VLAN_TX | \
+ ICE_PROMISC_VLAN_RX)
+
+@@ -234,7 +231,6 @@ enum ice_pf_state {
+ ICE_VFLR_EVENT_PENDING,
+ ICE_FLTR_OVERFLOW_PROMISC,
+ ICE_VF_DIS,
+- ICE_VF_DEINIT_IN_PROGRESS,
+ ICE_CFG_BUSY,
+ ICE_SERVICE_SCHED,
+ ICE_SERVICE_DIS,
+@@ -245,6 +241,7 @@ enum ice_pf_state {
+ ICE_LINK_DEFAULT_OVERRIDE_PENDING,
+ ICE_PHY_INIT_COMPLETE,
+ ICE_FD_VF_FLUSH_CTX, /* set at FD Rx IRQ or timeout */
++ ICE_AUX_ERR_PENDING,
+ ICE_STATE_NBITS /* must be last */
+ };
+
+@@ -306,10 +303,6 @@ struct ice_vsi {
+ spinlock_t arfs_lock; /* protects aRFS hash table and filter state */
+ atomic_t *arfs_last_fltr_id;
+
+- /* devlink port data */
+- struct devlink_port devlink_port;
+- bool devlink_port_registered;
+-
+ u16 max_frame;
+ u16 rx_buf_len;
+
+@@ -405,6 +398,8 @@ enum ice_pf_flags {
+ ICE_FLAG_VF_TRUE_PROMISC_ENA,
+ ICE_FLAG_MDD_AUTO_RESET_VF,
+ ICE_FLAG_LINK_LENIENT_MODE_ENA,
++ ICE_FLAG_PLUG_AUX_DEV,
++ ICE_FLAG_MTU_CHANGED,
+ ICE_PF_FLAGS_NBITS /* must be last */
+ };
+
+@@ -421,6 +416,9 @@ struct ice_pf {
+ struct devlink_region *nvm_region;
+ struct devlink_region *devcaps_region;
+
++ /* devlink port data */
++ struct devlink_port devlink_port;
++
+ /* OS reserved IRQ details */
+ struct msix_entry *msix_entries;
+ struct ice_res_tracker *irq_tracker;
+@@ -454,6 +452,7 @@ struct ice_pf {
+ struct mutex avail_q_mutex; /* protects access to avail_[rx|tx]qs */
+ struct mutex sw_mutex; /* lock for protecting VSI alloc flow */
+ struct mutex tc_mutex; /* lock to protect TC changes */
++ struct mutex adev_mutex; /* lock to protect aux device access */
+ u32 msg_enable;
+ struct ice_ptp ptp;
+ u16 num_rdma_msix; /* Total MSIX vectors for RDMA driver */
+@@ -467,6 +466,7 @@ struct ice_pf {
+ wait_queue_head_t reset_wait_queue;
+
+ u32 hw_csum_rx_error;
++ u32 oicr_err_reg;
+ u16 oicr_idx; /* Other interrupt cause MSIX vector index */
+ u16 num_avail_sw_msix; /* remaining MSIX SW vectors left unclaimed */
+ u16 max_pf_txqs; /* Total Tx queues PF wide */
+@@ -553,7 +553,7 @@ static inline struct ice_pf *ice_netdev_to_pf(struct net_device *netdev)
+
+ static inline bool ice_is_xdp_ena_vsi(struct ice_vsi *vsi)
+ {
+- return !!vsi->xdp_prog;
++ return !!READ_ONCE(vsi->xdp_prog);
+ }
+
+ static inline void ice_set_ring_xdp(struct ice_ring *ring)
+@@ -696,7 +696,7 @@ static inline void ice_set_rdma_cap(struct ice_pf *pf)
+ if (pf->hw.func_caps.common_cap.rdma && pf->num_rdma_msix) {
+ set_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+ set_bit(ICE_FLAG_AUX_ENA, pf->flags);
+- ice_plug_aux_dev(pf);
++ set_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags);
+ }
+ }
+
+@@ -706,7 +706,16 @@ static inline void ice_set_rdma_cap(struct ice_pf *pf)
+ */
+ static inline void ice_clear_rdma_cap(struct ice_pf *pf)
+ {
+- ice_unplug_aux_dev(pf);
++ /* We can directly unplug aux device here only if the flag bit
++ * ICE_FLAG_PLUG_AUX_DEV is not set because ice_unplug_aux_dev()
++ * could race with ice_plug_aux_dev() called from
++ * ice_service_task(). In this case we only clear that bit now and
++ * aux device will be unplugged later once ice_plug_aux_device()
++ * called from ice_service_task() finishes (see ice_service_task()).
++ */
++ if (!test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags))
++ ice_unplug_aux_dev(pf);
++
+ clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+ clear_bit(ICE_FLAG_AUX_ENA, pf->flags);
+ }
+diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c
+index c36057efc7ae3..533a953f15acb 100644
+--- a/drivers/net/ethernet/intel/ice/ice_base.c
++++ b/drivers/net/ethernet/intel/ice/ice_base.c
+@@ -115,6 +115,8 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx)
+ q_vector->rx.itr_setting = ICE_DFLT_RX_ITR;
+ q_vector->tx.itr_mode = ITR_DYNAMIC;
+ q_vector->rx.itr_mode = ITR_DYNAMIC;
++ q_vector->tx.type = ICE_TX_CONTAINER;
++ q_vector->rx.type = ICE_RX_CONTAINER;
+
+ if (vsi->type == ICE_VSI_VF)
+ goto out;
+@@ -909,7 +911,7 @@ ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src,
+ } else if (status == ICE_ERR_DOES_NOT_EXIST) {
+ dev_dbg(ice_pf_to_dev(vsi->back), "LAN Tx queues do not exist, nothing to disable\n");
+ } else if (status) {
+- dev_err(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %s\n",
++ dev_dbg(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %s\n",
+ ice_stat_str(status));
+ return -ENODEV;
+ }
+diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c
+index df5ad4de1f00e..3de6f16f985ab 100644
+--- a/drivers/net/ethernet/intel/ice/ice_common.c
++++ b/drivers/net/ethernet/intel/ice/ice_common.c
+@@ -3270,9 +3270,10 @@ ice_cfg_phy_fec(struct ice_port_info *pi, struct ice_aqc_set_phy_cfg_data *cfg,
+
+ if (fec == ICE_FEC_AUTO && ice_fw_supports_link_override(hw) &&
+ !ice_fw_supports_report_dflt_cfg(hw)) {
+- struct ice_link_default_override_tlv tlv;
++ struct ice_link_default_override_tlv tlv = { 0 };
+
+- if (ice_get_link_default_override(&tlv, pi))
++ status = ice_get_link_default_override(&tlv, pi);
++ if (status)
+ goto out;
+
+ if (!(tlv.options & ICE_LINK_OVERRIDE_STRICT_MODE) &&
+diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
+index da7288bdc9a3f..2ec5d5cb72803 100644
+--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
++++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
+@@ -526,60 +526,115 @@ void ice_devlink_unregister(struct ice_pf *pf)
+ }
+
+ /**
+- * ice_devlink_create_port - Create a devlink port for this VSI
+- * @vsi: the VSI to create a port for
++ * ice_devlink_create_pf_port - Create a devlink port for this PF
++ * @pf: the PF to create a devlink port for
+ *
+- * Create and register a devlink_port for this VSI.
++ * Create and register a devlink_port for this PF.
+ *
+ * Return: zero on success or an error code on failure.
+ */
+-int ice_devlink_create_port(struct ice_vsi *vsi)
++int ice_devlink_create_pf_port(struct ice_pf *pf)
+ {
+ struct devlink_port_attrs attrs = {};
+- struct ice_port_info *pi;
++ struct devlink_port *devlink_port;
+ struct devlink *devlink;
++ struct ice_vsi *vsi;
+ struct device *dev;
+- struct ice_pf *pf;
+ int err;
+
+- /* Currently we only create devlink_port instances for PF VSIs */
+- if (vsi->type != ICE_VSI_PF)
+- return -EINVAL;
+-
+- pf = vsi->back;
+- devlink = priv_to_devlink(pf);
+ dev = ice_pf_to_dev(pf);
+- pi = pf->hw.port_info;
++
++ devlink_port = &pf->devlink_port;
++
++ vsi = ice_get_main_vsi(pf);
++ if (!vsi)
++ return -EIO;
+
+ attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
+- attrs.phys.port_number = pi->lport;
+- devlink_port_attrs_set(&vsi->devlink_port, &attrs);
+- err = devlink_port_register(devlink, &vsi->devlink_port, vsi->idx);
++ attrs.phys.port_number = pf->hw.bus.func;
++ devlink_port_attrs_set(devlink_port, &attrs);
++ devlink = priv_to_devlink(pf);
++
++ err = devlink_port_register(devlink, devlink_port, vsi->idx);
+ if (err) {
+- dev_err(dev, "devlink_port_register failed: %d\n", err);
++ dev_err(dev, "Failed to create devlink port for PF %d, error %d\n",
++ pf->hw.pf_id, err);
+ return err;
+ }
+
+- vsi->devlink_port_registered = true;
++ return 0;
++}
++
++/**
++ * ice_devlink_destroy_pf_port - Destroy the devlink_port for this PF
++ * @pf: the PF to cleanup
++ *
++ * Unregisters the devlink_port structure associated with this PF.
++ */
++void ice_devlink_destroy_pf_port(struct ice_pf *pf)
++{
++ struct devlink_port *devlink_port;
++
++ devlink_port = &pf->devlink_port;
++
++ devlink_port_type_clear(devlink_port);
++ devlink_port_unregister(devlink_port);
++}
++
++/**
++ * ice_devlink_create_vf_port - Create a devlink port for this VF
++ * @vf: the VF to create a port for
++ *
++ * Create and register a devlink_port for this VF.
++ *
++ * Return: zero on success or an error code on failure.
++ */
++int ice_devlink_create_vf_port(struct ice_vf *vf)
++{
++ struct devlink_port_attrs attrs = {};
++ struct devlink_port *devlink_port;
++ struct devlink *devlink;
++ struct ice_vsi *vsi;
++ struct device *dev;
++ struct ice_pf *pf;
++ int err;
++
++ pf = vf->pf;
++ dev = ice_pf_to_dev(pf);
++ vsi = ice_get_vf_vsi(vf);
++ devlink_port = &vf->devlink_port;
++
++ attrs.flavour = DEVLINK_PORT_FLAVOUR_PCI_VF;
++ attrs.pci_vf.pf = pf->hw.bus.func;
++ attrs.pci_vf.vf = vf->vf_id;
++
++ devlink_port_attrs_set(devlink_port, &attrs);
++ devlink = priv_to_devlink(pf);
++
++ err = devlink_port_register(devlink, devlink_port, vsi->idx);
++ if (err) {
++ dev_err(dev, "Failed to create devlink port for VF %d, error %d\n",
++ vf->vf_id, err);
++ return err;
++ }
+
+ return 0;
+ }
+
+ /**
+- * ice_devlink_destroy_port - Destroy the devlink_port for this VSI
+- * @vsi: the VSI to cleanup
++ * ice_devlink_destroy_vf_port - Destroy the devlink_port for this VF
++ * @vf: the VF to cleanup
+ *
+- * Unregisters the devlink_port structure associated with this VSI.
++ * Unregisters the devlink_port structure associated with this VF.
+ */
+-void ice_devlink_destroy_port(struct ice_vsi *vsi)
++void ice_devlink_destroy_vf_port(struct ice_vf *vf)
+ {
+- if (!vsi->devlink_port_registered)
+- return;
++ struct devlink_port *devlink_port;
+
+- devlink_port_type_clear(&vsi->devlink_port);
+- devlink_port_unregister(&vsi->devlink_port);
++ devlink_port = &vf->devlink_port;
+
+- vsi->devlink_port_registered = false;
++ devlink_port_type_clear(devlink_port);
++ devlink_port_unregister(devlink_port);
+ }
+
+ /**
+diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.h b/drivers/net/ethernet/intel/ice/ice_devlink.h
+index e07e74426bde8..e30284ccbed4c 100644
+--- a/drivers/net/ethernet/intel/ice/ice_devlink.h
++++ b/drivers/net/ethernet/intel/ice/ice_devlink.h
+@@ -8,8 +8,10 @@ struct ice_pf *ice_allocate_pf(struct device *dev);
+
+ int ice_devlink_register(struct ice_pf *pf);
+ void ice_devlink_unregister(struct ice_pf *pf);
+-int ice_devlink_create_port(struct ice_vsi *vsi);
+-void ice_devlink_destroy_port(struct ice_vsi *vsi);
++int ice_devlink_create_pf_port(struct ice_pf *pf);
++void ice_devlink_destroy_pf_port(struct ice_pf *pf);
++int ice_devlink_create_vf_port(struct ice_vf *vf);
++void ice_devlink_destroy_vf_port(struct ice_vf *vf);
+
+ void ice_devlink_init_regions(struct ice_pf *pf);
+ void ice_devlink_destroy_regions(struct ice_pf *pf);
+diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
+index c451cf401e635..19f115402969e 100644
+--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
++++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
+@@ -2275,7 +2275,7 @@ ice_set_link_ksettings(struct net_device *netdev,
+ goto done;
+ }
+
+- curr_link_speed = pi->phy.link_info.link_speed;
++ curr_link_speed = pi->phy.curr_user_speed_req;
+ adv_link_speed = ice_ksettings_find_adv_link_speed(ks);
+
+ /* If speed didn't get set, set it to what it currently is.
+@@ -3466,15 +3466,9 @@ static int ice_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
+ return 0;
+ }
+
+-enum ice_container_type {
+- ICE_RX_CONTAINER,
+- ICE_TX_CONTAINER,
+-};
+-
+ /**
+ * ice_get_rc_coalesce - get ITR values for specific ring container
+ * @ec: ethtool structure to fill with driver's coalesce settings
+- * @c_type: container type, Rx or Tx
+ * @rc: ring container that the ITR values will come from
+ *
+ * Query the device for ice_ring_container specific ITR values. This is
+@@ -3484,13 +3478,12 @@ enum ice_container_type {
+ * Returns 0 on success, negative otherwise.
+ */
+ static int
+-ice_get_rc_coalesce(struct ethtool_coalesce *ec, enum ice_container_type c_type,
+- struct ice_ring_container *rc)
++ice_get_rc_coalesce(struct ethtool_coalesce *ec, struct ice_ring_container *rc)
+ {
+ if (!rc->ring)
+ return -EINVAL;
+
+- switch (c_type) {
++ switch (rc->type) {
+ case ICE_RX_CONTAINER:
+ ec->use_adaptive_rx_coalesce = ITR_IS_DYNAMIC(rc);
+ ec->rx_coalesce_usecs = rc->itr_setting;
+@@ -3501,7 +3494,7 @@ ice_get_rc_coalesce(struct ethtool_coalesce *ec, enum ice_container_type c_type,
+ ec->tx_coalesce_usecs = rc->itr_setting;
+ break;
+ default:
+- dev_dbg(ice_pf_to_dev(rc->ring->vsi->back), "Invalid c_type %d\n", c_type);
++ dev_dbg(ice_pf_to_dev(rc->ring->vsi->back), "Invalid c_type %d\n", rc->type);
+ return -EINVAL;
+ }
+
+@@ -3522,18 +3515,18 @@ static int
+ ice_get_q_coalesce(struct ice_vsi *vsi, struct ethtool_coalesce *ec, int q_num)
+ {
+ if (q_num < vsi->num_rxq && q_num < vsi->num_txq) {
+- if (ice_get_rc_coalesce(ec, ICE_RX_CONTAINER,
++ if (ice_get_rc_coalesce(ec,
+ &vsi->rx_rings[q_num]->q_vector->rx))
+ return -EINVAL;
+- if (ice_get_rc_coalesce(ec, ICE_TX_CONTAINER,
++ if (ice_get_rc_coalesce(ec,
+ &vsi->tx_rings[q_num]->q_vector->tx))
+ return -EINVAL;
+ } else if (q_num < vsi->num_rxq) {
+- if (ice_get_rc_coalesce(ec, ICE_RX_CONTAINER,
++ if (ice_get_rc_coalesce(ec,
+ &vsi->rx_rings[q_num]->q_vector->rx))
+ return -EINVAL;
+ } else if (q_num < vsi->num_txq) {
+- if (ice_get_rc_coalesce(ec, ICE_TX_CONTAINER,
++ if (ice_get_rc_coalesce(ec,
+ &vsi->tx_rings[q_num]->q_vector->tx))
+ return -EINVAL;
+ } else {
+@@ -3585,7 +3578,6 @@ ice_get_per_q_coalesce(struct net_device *netdev, u32 q_num,
+
+ /**
+ * ice_set_rc_coalesce - set ITR values for specific ring container
+- * @c_type: container type, Rx or Tx
+ * @ec: ethtool structure from user to update ITR settings
+ * @rc: ring container that the ITR values will come from
+ * @vsi: VSI associated to the ring container
+@@ -3597,10 +3589,10 @@ ice_get_per_q_coalesce(struct net_device *netdev, u32 q_num,
+ * Returns 0 on success, negative otherwise.
+ */
+ static int
+-ice_set_rc_coalesce(enum ice_container_type c_type, struct ethtool_coalesce *ec,
++ice_set_rc_coalesce(struct ethtool_coalesce *ec,
+ struct ice_ring_container *rc, struct ice_vsi *vsi)
+ {
+- const char *c_type_str = (c_type == ICE_RX_CONTAINER) ? "rx" : "tx";
++ const char *c_type_str = (rc->type == ICE_RX_CONTAINER) ? "rx" : "tx";
+ u32 use_adaptive_coalesce, coalesce_usecs;
+ struct ice_pf *pf = vsi->back;
+ u16 itr_setting;
+@@ -3608,7 +3600,7 @@ ice_set_rc_coalesce(enum ice_container_type c_type, struct ethtool_coalesce *ec,
+ if (!rc->ring)
+ return -EINVAL;
+
+- switch (c_type) {
++ switch (rc->type) {
+ case ICE_RX_CONTAINER:
+ if (ec->rx_coalesce_usecs_high > ICE_MAX_INTRL ||
+ (ec->rx_coalesce_usecs_high &&
+@@ -3641,7 +3633,7 @@ ice_set_rc_coalesce(enum ice_container_type c_type, struct ethtool_coalesce *ec,
+ break;
+ default:
+ dev_dbg(ice_pf_to_dev(pf), "Invalid container type %d\n",
+- c_type);
++ rc->type);
+ return -EINVAL;
+ }
+
+@@ -3690,22 +3682,22 @@ static int
+ ice_set_q_coalesce(struct ice_vsi *vsi, struct ethtool_coalesce *ec, int q_num)
+ {
+ if (q_num < vsi->num_rxq && q_num < vsi->num_txq) {
+- if (ice_set_rc_coalesce(ICE_RX_CONTAINER, ec,
++ if (ice_set_rc_coalesce(ec,
+ &vsi->rx_rings[q_num]->q_vector->rx,
+ vsi))
+ return -EINVAL;
+
+- if (ice_set_rc_coalesce(ICE_TX_CONTAINER, ec,
++ if (ice_set_rc_coalesce(ec,
+ &vsi->tx_rings[q_num]->q_vector->tx,
+ vsi))
+ return -EINVAL;
+ } else if (q_num < vsi->num_rxq) {
+- if (ice_set_rc_coalesce(ICE_RX_CONTAINER, ec,
++ if (ice_set_rc_coalesce(ec,
+ &vsi->rx_rings[q_num]->q_vector->rx,
+ vsi))
+ return -EINVAL;
+ } else if (q_num < vsi->num_txq) {
+- if (ice_set_rc_coalesce(ICE_TX_CONTAINER, ec,
++ if (ice_set_rc_coalesce(ec,
+ &vsi->tx_rings[q_num]->q_vector->tx,
+ vsi))
+ return -EINVAL;
+diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c
+index adcc9a251595a..1dd3622991c65 100644
+--- a/drivers/net/ethernet/intel/ice/ice_idc.c
++++ b/drivers/net/ethernet/intel/ice/ice_idc.c
+@@ -34,14 +34,20 @@ void ice_send_event_to_aux(struct ice_pf *pf, struct iidc_event *event)
+ {
+ struct iidc_auxiliary_drv *iadrv;
+
+- if (!pf->adev)
++ if (WARN_ON_ONCE(!in_task()))
+ return;
+
++ mutex_lock(&pf->adev_mutex);
++ if (!pf->adev)
++ goto finish;
++
+ device_lock(&pf->adev->dev);
+ iadrv = ice_get_auxiliary_drv(pf);
+ if (iadrv && iadrv->event_handler)
+ iadrv->event_handler(pf, event);
+ device_unlock(&pf->adev->dev);
++finish:
++ mutex_unlock(&pf->adev_mutex);
+ }
+
+ /**
+@@ -282,7 +288,6 @@ int ice_plug_aux_dev(struct ice_pf *pf)
+ return -ENOMEM;
+
+ adev = &iadev->adev;
+- pf->adev = adev;
+ iadev->pf = pf;
+
+ adev->id = pf->aux_idx;
+@@ -292,18 +297,20 @@ int ice_plug_aux_dev(struct ice_pf *pf)
+
+ ret = auxiliary_device_init(adev);
+ if (ret) {
+- pf->adev = NULL;
+ kfree(iadev);
+ return ret;
+ }
+
+ ret = auxiliary_device_add(adev);
+ if (ret) {
+- pf->adev = NULL;
+ auxiliary_device_uninit(adev);
+ return ret;
+ }
+
++ mutex_lock(&pf->adev_mutex);
++ pf->adev = adev;
++ mutex_unlock(&pf->adev_mutex);
++
+ return 0;
+ }
+
+@@ -312,12 +319,17 @@ int ice_plug_aux_dev(struct ice_pf *pf)
+ */
+ void ice_unplug_aux_dev(struct ice_pf *pf)
+ {
+- if (!pf->adev)
+- return;
++ struct auxiliary_device *adev;
+
+- auxiliary_device_delete(pf->adev);
+- auxiliary_device_uninit(pf->adev);
++ mutex_lock(&pf->adev_mutex);
++ adev = pf->adev;
+ pf->adev = NULL;
++ mutex_unlock(&pf->adev_mutex);
++
++ if (adev) {
++ auxiliary_device_delete(adev);
++ auxiliary_device_uninit(adev);
++ }
+ }
+
+ /**
+diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c
+index e375ac849aecd..4f954db01b929 100644
+--- a/drivers/net/ethernet/intel/ice/ice_lag.c
++++ b/drivers/net/ethernet/intel/ice/ice_lag.c
+@@ -204,17 +204,39 @@ ice_lag_unlink(struct ice_lag *lag,
+ lag->upper_netdev = NULL;
+ }
+
+- if (lag->peer_netdev) {
+- dev_put(lag->peer_netdev);
+- lag->peer_netdev = NULL;
+- }
+-
++ lag->peer_netdev = NULL;
+ ice_set_sriov_cap(pf);
+ ice_set_rdma_cap(pf);
+ lag->bonded = false;
+ lag->role = ICE_LAG_NONE;
+ }
+
++/**
++ * ice_lag_unregister - handle netdev unregister events
++ * @lag: LAG info struct
++ * @netdev: netdev reporting the event
++ */
++static void ice_lag_unregister(struct ice_lag *lag, struct net_device *netdev)
++{
++ struct ice_pf *pf = lag->pf;
++
++ /* check to see if this event is for this netdev
++ * check that we are in an aggregate
++ */
++ if (netdev != lag->netdev || !lag->bonded)
++ return;
++
++ if (lag->upper_netdev) {
++ dev_put(lag->upper_netdev);
++ lag->upper_netdev = NULL;
++ ice_set_sriov_cap(pf);
++ ice_set_rdma_cap(pf);
++ }
++ /* perform some cleanup in case we come back */
++ lag->bonded = false;
++ lag->role = ICE_LAG_NONE;
++}
++
+ /**
+ * ice_lag_changeupper_event - handle LAG changeupper event
+ * @lag: LAG info struct
+@@ -307,7 +329,7 @@ ice_lag_event_handler(struct notifier_block *notif_blk, unsigned long event,
+ ice_lag_info_event(lag, ptr);
+ break;
+ case NETDEV_UNREGISTER:
+- ice_lag_unlink(lag, ptr);
++ ice_lag_unregister(lag, netdev);
+ break;
+ default:
+ break;
+diff --git a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h
+index 80736e0ec0dca..3f635fdbfaff9 100644
+--- a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h
++++ b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h
+@@ -528,6 +528,7 @@ struct ice_tx_ctx_desc {
+ (0x3FFFFULL << ICE_TXD_CTX_QW1_TSO_LEN_S)
+
+ #define ICE_TXD_CTX_QW1_MSS_S 50
++#define ICE_TXD_CTX_MIN_MSS 64
+
+ enum ice_tx_ctx_desc_cmd_bits {
+ ICE_TX_CTX_DESC_TSO = 0x01,
+diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
+index b718e196af2a4..4417238b0e64f 100644
+--- a/drivers/net/ethernet/intel/ice/ice_lib.c
++++ b/drivers/net/ethernet/intel/ice/ice_lib.c
+@@ -83,8 +83,13 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi)
+ if (!vsi->rx_rings)
+ goto err_rings;
+
+- /* XDP will have vsi->alloc_txq Tx queues as well, so double the size */
+- vsi->txq_map = devm_kcalloc(dev, (2 * vsi->alloc_txq),
++ /* txq_map needs to have enough space to track both Tx (stack) rings
++ * and XDP rings; at this point vsi->num_xdp_txq might not be set,
++ * so use num_possible_cpus() as we want to always provide XDP ring
++ * per CPU, regardless of queue count settings from user that might
++ * have come from ethtool's set_channels() callback;
++ */
++ vsi->txq_map = devm_kcalloc(dev, (vsi->alloc_txq + num_possible_cpus()),
+ sizeof(*vsi->txq_map), GFP_KERNEL);
+
+ if (!vsi->txq_map)
+@@ -1301,6 +1306,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi)
+ ring->tx_tstamps = &pf->ptp.port.tx;
+ ring->dev = dev;
+ ring->count = vsi->num_tx_desc;
++ ring->txq_teid = ICE_INVAL_TEID;
+ WRITE_ONCE(vsi->tx_rings[i], ring);
+ }
+
+@@ -1516,6 +1522,12 @@ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi)
+ if (status)
+ dev_dbg(dev, "ice_add_rss_cfg failed for sctp6 flow, vsi = %d, error = %s\n",
+ vsi_num, ice_stat_str(status));
++
++ status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_ESP_SPI,
++ ICE_FLOW_SEG_HDR_ESP);
++ if (status)
++ dev_dbg(dev, "ice_add_rss_cfg failed for esp/spi flow, vsi = %d, error = %d\n",
++ vsi_num, status);
+ }
+
+ /**
+@@ -2860,7 +2872,8 @@ int ice_vsi_release(struct ice_vsi *vsi)
+ clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
+ }
+
+- ice_devlink_destroy_port(vsi);
++ if (vsi->type == ICE_VSI_PF)
++ ice_devlink_destroy_pf_port(pf);
+
+ if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
+ ice_rss_clean(vsi);
+@@ -2911,6 +2924,8 @@ int ice_vsi_release(struct ice_vsi *vsi)
+ }
+ }
+
++ if (ice_is_vsi_dflt_vsi(pf->first_sw, vsi))
++ ice_clear_dflt_vsi(pf->first_sw);
+ ice_fltr_remove_all(vsi);
+ ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx);
+ err = ice_rm_vsi_rdma_cfg(vsi->port_info, vsi->idx);
+@@ -2965,8 +2980,8 @@ ice_vsi_rebuild_get_coalesce(struct ice_vsi *vsi,
+ ice_for_each_q_vector(vsi, i) {
+ struct ice_q_vector *q_vector = vsi->q_vectors[i];
+
+- coalesce[i].itr_tx = q_vector->tx.itr_setting;
+- coalesce[i].itr_rx = q_vector->rx.itr_setting;
++ coalesce[i].itr_tx = q_vector->tx.itr_settings;
++ coalesce[i].itr_rx = q_vector->rx.itr_settings;
+ coalesce[i].intrl = q_vector->intrl;
+
+ if (i < vsi->num_txq)
+@@ -3022,21 +3037,21 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi,
+ */
+ if (i < vsi->alloc_rxq && coalesce[i].rx_valid) {
+ rc = &vsi->q_vectors[i]->rx;
+- rc->itr_setting = coalesce[i].itr_rx;
++ rc->itr_settings = coalesce[i].itr_rx;
+ ice_write_itr(rc, rc->itr_setting);
+ } else if (i < vsi->alloc_rxq) {
+ rc = &vsi->q_vectors[i]->rx;
+- rc->itr_setting = coalesce[0].itr_rx;
++ rc->itr_settings = coalesce[0].itr_rx;
+ ice_write_itr(rc, rc->itr_setting);
+ }
+
+ if (i < vsi->alloc_txq && coalesce[i].tx_valid) {
+ rc = &vsi->q_vectors[i]->tx;
+- rc->itr_setting = coalesce[i].itr_tx;
++ rc->itr_settings = coalesce[i].itr_tx;
+ ice_write_itr(rc, rc->itr_setting);
+ } else if (i < vsi->alloc_txq) {
+ rc = &vsi->q_vectors[i]->tx;
+- rc->itr_setting = coalesce[0].itr_tx;
++ rc->itr_settings = coalesce[0].itr_tx;
+ ice_write_itr(rc, rc->itr_setting);
+ }
+
+@@ -3050,12 +3065,12 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi,
+ for (; i < vsi->num_q_vectors; i++) {
+ /* transmit */
+ rc = &vsi->q_vectors[i]->tx;
+- rc->itr_setting = coalesce[0].itr_tx;
++ rc->itr_settings = coalesce[0].itr_tx;
+ ice_write_itr(rc, rc->itr_setting);
+
+ /* receive */
+ rc = &vsi->q_vectors[i]->rx;
+- rc->itr_setting = coalesce[0].itr_rx;
++ rc->itr_settings = coalesce[0].itr_rx;
+ ice_write_itr(rc, rc->itr_setting);
+
+ vsi->q_vectors[i]->intrl = coalesce[0].intrl;
+diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
+index 06fa93e597fbc..188abf36a5b23 100644
+--- a/drivers/net/ethernet/intel/ice/ice_main.c
++++ b/drivers/net/ethernet/intel/ice/ice_main.c
+@@ -1679,7 +1679,9 @@ static void ice_handle_mdd_event(struct ice_pf *pf)
+ * reset, so print the event prior to reset.
+ */
+ ice_print_vf_rx_mdd_event(vf);
++ mutex_lock(&pf->vf[i].cfg_lock);
+ ice_reset_vf(&pf->vf[i], false);
++ mutex_unlock(&pf->vf[i].cfg_lock);
+ }
+ }
+ }
+@@ -2141,6 +2143,43 @@ static void ice_service_task(struct work_struct *work)
+ return;
+ }
+
++ if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) {
++ struct iidc_event *event;
++
++ event = kzalloc(sizeof(*event), GFP_KERNEL);
++ if (event) {
++ set_bit(IIDC_EVENT_CRIT_ERR, event->type);
++ /* report the entire OICR value to AUX driver */
++ swap(event->reg, pf->oicr_err_reg);
++ ice_send_event_to_aux(pf, event);
++ kfree(event);
++ }
++ }
++
++ if (test_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags)) {
++ /* Plug aux device per request */
++ ice_plug_aux_dev(pf);
++
++ /* Mark plugging as done but check whether unplug was
++ * requested during ice_plug_aux_dev() call
++ * (e.g. from ice_clear_rdma_cap()) and if so then
++ * plug aux device.
++ */
++ if (!test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags))
++ ice_unplug_aux_dev(pf);
++ }
++
++ if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) {
++ struct iidc_event *event;
++
++ event = kzalloc(sizeof(*event), GFP_KERNEL);
++ if (event) {
++ set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type);
++ ice_send_event_to_aux(pf, event);
++ kfree(event);
++ }
++ }
++
+ ice_clean_adminq_subtask(pf);
+ ice_check_media_subtask(pf);
+ ice_check_for_hang_subtask(pf);
+@@ -2497,7 +2536,18 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog)
+ ice_stat_str(status));
+ goto clear_xdp_rings;
+ }
+- ice_vsi_assign_bpf_prog(vsi, prog);
++
++ /* assign the prog only when it's not already present on VSI;
++ * this flow is a subject of both ethtool -L and ndo_bpf flows;
++ * VSI rebuild that happens under ethtool -L can expose us to
++ * the bpf_prog refcount issues as we would be swapping same
++ * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put
++ * on it as it would be treated as an 'old_prog'; for ndo_bpf
++ * this is not harmful as dev_xdp_install bumps the refcount
++ * before calling the op exposed by the driver;
++ */
++ if (!ice_is_xdp_ena_vsi(vsi))
++ ice_vsi_assign_bpf_prog(vsi, prog);
+
+ return 0;
+ clear_xdp_rings:
+@@ -2562,8 +2612,10 @@ free_qmap:
+
+ for (i = 0; i < vsi->num_xdp_txq; i++)
+ if (vsi->xdp_rings[i]) {
+- if (vsi->xdp_rings[i]->desc)
++ if (vsi->xdp_rings[i]->desc) {
++ synchronize_rcu();
+ ice_free_tx_ring(vsi->xdp_rings[i]);
++ }
+ kfree_rcu(vsi->xdp_rings[i], rcu);
+ vsi->xdp_rings[i] = NULL;
+ }
+@@ -2643,6 +2695,11 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
+ if (xdp_ring_err)
+ NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed");
+ } else {
++ /* safe to call even when prog == vsi->xdp_prog as
++ * dev_xdp_install in net/core/dev.c incremented prog's
++ * refcount so corresponding bpf_prog_put won't cause
++ * underflow
++ */
+ ice_vsi_assign_bpf_prog(vsi, prog);
+ }
+
+@@ -2839,17 +2896,9 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
+
+ #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M)
+ if (oicr & ICE_AUX_CRIT_ERR) {
+- struct iidc_event *event;
+-
++ pf->oicr_err_reg |= oicr;
++ set_bit(ICE_AUX_ERR_PENDING, pf->state);
+ ena_mask &= ~ICE_AUX_CRIT_ERR;
+- event = kzalloc(sizeof(*event), GFP_KERNEL);
+- if (event) {
+- set_bit(IIDC_EVENT_CRIT_ERR, event->type);
+- /* report the entire OICR value to AUX driver */
+- event->reg = oicr;
+- ice_send_event_to_aux(pf, event);
+- kfree(event);
+- }
+ }
+
+ /* Report any remaining unexpected interrupts */
+@@ -3398,6 +3447,7 @@ u16 ice_get_avail_rxq_count(struct ice_pf *pf)
+ static void ice_deinit_pf(struct ice_pf *pf)
+ {
+ ice_service_task_stop(pf);
++ mutex_destroy(&pf->adev_mutex);
+ mutex_destroy(&pf->sw_mutex);
+ mutex_destroy(&pf->tc_mutex);
+ mutex_destroy(&pf->avail_q_mutex);
+@@ -3478,6 +3528,7 @@ static int ice_init_pf(struct ice_pf *pf)
+
+ mutex_init(&pf->sw_mutex);
+ mutex_init(&pf->tc_mutex);
++ mutex_init(&pf->adev_mutex);
+
+ INIT_HLIST_HEAD(&pf->aq_wait_list);
+ spin_lock_init(&pf->aq_wait_lock);
+@@ -4170,11 +4221,11 @@ static int ice_register_netdev(struct ice_pf *pf)
+ set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
+ netif_carrier_off(vsi->netdev);
+ netif_tx_stop_all_queues(vsi->netdev);
+- err = ice_devlink_create_port(vsi);
++ err = ice_devlink_create_pf_port(pf);
+ if (err)
+ goto err_devlink_create;
+
+- devlink_port_type_eth_set(&vsi->devlink_port, vsi->netdev);
++ devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev);
+
+ return 0;
+ err_devlink_create:
+@@ -4600,9 +4651,6 @@ static void ice_remove(struct pci_dev *pdev)
+ struct ice_pf *pf = pci_get_drvdata(pdev);
+ int i;
+
+- if (!pf)
+- return;
+-
+ for (i = 0; i < ICE_MAX_RESET_WAIT; i++) {
+ if (!ice_is_reset_in_progress(pf->state))
+ break;
+@@ -5608,6 +5656,10 @@ static int ice_up_complete(struct ice_vsi *vsi)
+ netif_carrier_on(vsi->netdev);
+ }
+
++ /* Perform an initial read of the statistics registers now to
++ * set the baseline so counters are ready when interface is up
++ */
++ ice_update_eth_stats(vsi);
+ ice_service_task_schedule(pf);
+
+ return 0;
+@@ -6511,7 +6563,6 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu)
+ struct ice_netdev_priv *np = netdev_priv(netdev);
+ struct ice_vsi *vsi = np->vsi;
+ struct ice_pf *pf = vsi->back;
+- struct iidc_event *event;
+ u8 count = 0;
+ int err = 0;
+
+@@ -6546,14 +6597,6 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu)
+ return -EBUSY;
+ }
+
+- event = kzalloc(sizeof(*event), GFP_KERNEL);
+- if (!event)
+- return -ENOMEM;
+-
+- set_bit(IIDC_EVENT_BEFORE_MTU_CHANGE, event->type);
+- ice_send_event_to_aux(pf, event);
+- clear_bit(IIDC_EVENT_BEFORE_MTU_CHANGE, event->type);
+-
+ netdev->mtu = (unsigned int)new_mtu;
+
+ /* if VSI is up, bring it down and then back up */
+@@ -6561,21 +6604,18 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu)
+ err = ice_down(vsi);
+ if (err) {
+ netdev_err(netdev, "change MTU if_down err %d\n", err);
+- goto event_after;
++ return err;
+ }
+
+ err = ice_up(vsi);
+ if (err) {
+ netdev_err(netdev, "change MTU if_up err %d\n", err);
+- goto event_after;
++ return err;
+ }
+ }
+
+ netdev_dbg(netdev, "changed MTU to %d\n", new_mtu);
+-event_after:
+- set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type);
+- ice_send_event_to_aux(pf, event);
+- kfree(event);
++ set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
+
+ return err;
+ }
+@@ -7190,6 +7230,7 @@ ice_features_check(struct sk_buff *skb,
+ struct net_device __always_unused *netdev,
+ netdev_features_t features)
+ {
++ bool gso = skb_is_gso(skb);
+ size_t len;
+
+ /* No point in doing any of this if neither checksum nor GSO are
+@@ -7202,24 +7243,32 @@ ice_features_check(struct sk_buff *skb,
+ /* We cannot support GSO if the MSS is going to be less than
+ * 64 bytes. If it is then we need to drop support for GSO.
+ */
+- if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
++ if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS))
+ features &= ~NETIF_F_GSO_MASK;
+
+- len = skb_network_header(skb) - skb->data;
++ len = skb_network_offset(skb);
+ if (len > ICE_TXD_MACLEN_MAX || len & 0x1)
+ goto out_rm_features;
+
+- len = skb_transport_header(skb) - skb_network_header(skb);
++ len = skb_network_header_len(skb);
+ if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
+ goto out_rm_features;
+
+ if (skb->encapsulation) {
+- len = skb_inner_network_header(skb) - skb_transport_header(skb);
+- if (len > ICE_TXD_L4LEN_MAX || len & 0x1)
+- goto out_rm_features;
++ /* this must work for VXLAN frames AND IPIP/SIT frames, and in
++ * the case of IPIP frames, the transport header pointer is
++ * after the inner header! So check to make sure that this
++ * is a GRE or UDP_TUNNEL frame before doing that math.
++ */
++ if (gso && (skb_shinfo(skb)->gso_type &
++ (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) {
++ len = skb_inner_network_header(skb) -
++ skb_transport_header(skb);
++ if (len > ICE_TXD_L4LEN_MAX || len & 0x1)
++ goto out_rm_features;
++ }
+
+- len = skb_inner_transport_header(skb) -
+- skb_inner_network_header(skb);
++ len = skb_inner_network_header_len(skb);
+ if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
+ goto out_rm_features;
+ }
+diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c
+index d1ef3d48a4b03..9b50e9e6042a5 100644
+--- a/drivers/net/ethernet/intel/ice/ice_ptp.c
++++ b/drivers/net/ethernet/intel/ice/ice_ptp.c
+@@ -254,12 +254,19 @@ ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts)
+ * This function must be called periodically to ensure that the cached value
+ * is never more than 2 seconds old. It must also be called whenever the PHC
+ * time has been changed.
++ *
++ * Return:
++ * * 0 - OK, successfully updated
++ * * -EAGAIN - PF was busy, need to reschedule the update
+ */
+-static void ice_ptp_update_cached_phctime(struct ice_pf *pf)
++static int ice_ptp_update_cached_phctime(struct ice_pf *pf)
+ {
+ u64 systime;
+ int i;
+
++ if (test_and_set_bit(ICE_CFG_BUSY, pf->state))
++ return -EAGAIN;
++
+ /* Read the current PHC time */
+ systime = ice_ptp_read_src_clk_reg(pf, NULL);
+
+@@ -282,6 +289,9 @@ static void ice_ptp_update_cached_phctime(struct ice_pf *pf)
+ WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
+ }
+ }
++ clear_bit(ICE_CFG_BUSY, pf->state);
++
++ return 0;
+ }
+
+ /**
+@@ -459,7 +469,7 @@ static int ice_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm)
+ scaled_ppm = -scaled_ppm;
+ }
+
+- while ((u64)scaled_ppm > div_u64(U64_MAX, incval)) {
++ while ((u64)scaled_ppm > div64_u64(U64_MAX, incval)) {
+ /* handle overflow by scaling down the scaled_ppm and
+ * the divisor, losing some precision
+ */
+@@ -846,9 +856,12 @@ exit:
+ static int ice_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta)
+ {
+ struct timespec64 now, then;
++ int ret;
+
+ then = ns_to_timespec64(delta);
+- ice_ptp_gettimex64(info, &now, NULL);
++ ret = ice_ptp_gettimex64(info, &now, NULL);
++ if (ret)
++ return ret;
+ now = timespec64_add(now, then);
+
+ return ice_ptp_settime64(info, (const struct timespec64 *)&now);
+@@ -1182,19 +1195,16 @@ static void ice_ptp_tx_tstamp_work(struct kthread_work *work)
+ if (err)
+ continue;
+
+- /* Check if the timestamp is valid */
+- if (!(raw_tstamp & ICE_PTP_TS_VALID))
++ /* Check if the timestamp is invalid or stale */
++ if (!(raw_tstamp & ICE_PTP_TS_VALID) ||
++ raw_tstamp == tx->tstamps[idx].cached_tstamp)
+ continue;
+
+- /* clear the timestamp register, so that it won't show valid
+- * again when re-used.
+- */
+- ice_clear_phy_tstamp(hw, tx->quad, phy_idx);
+-
+ /* The timestamp is valid, so we'll go ahead and clear this
+ * index and then send the timestamp up to the stack.
+ */
+ spin_lock(&tx->lock);
++ tx->tstamps[idx].cached_tstamp = raw_tstamp;
+ clear_bit(idx, tx->in_use);
+ skb = tx->tstamps[idx].skb;
+ tx->tstamps[idx].skb = NULL;
+@@ -1375,6 +1385,7 @@ ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
+
+ /**
+ * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped
++ * @hw: pointer to the hw struct
+ * @tx: PTP Tx tracker to clean up
+ *
+ * Loop through the Tx timestamp requests and see if any of them have been
+@@ -1383,7 +1394,7 @@ ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
+ * timestamp will never be captured. This might happen if the packet gets
+ * discarded before it reaches the PHY timestamping block.
+ */
+-static void ice_ptp_tx_tstamp_cleanup(struct ice_ptp_tx *tx)
++static void ice_ptp_tx_tstamp_cleanup(struct ice_hw *hw, struct ice_ptp_tx *tx)
+ {
+ u8 idx;
+
+@@ -1392,11 +1403,16 @@ static void ice_ptp_tx_tstamp_cleanup(struct ice_ptp_tx *tx)
+
+ for_each_set_bit(idx, tx->in_use, tx->len) {
+ struct sk_buff *skb;
++ u64 raw_tstamp;
+
+ /* Check if this SKB has been waiting for too long */
+ if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ))
+ continue;
+
++ /* Read tstamp to be able to use this register again */
++ ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset,
++ &raw_tstamp);
++
+ spin_lock(&tx->lock);
+ skb = tx->tstamps[idx].skb;
+ tx->tstamps[idx].skb = NULL;
+@@ -1412,17 +1428,18 @@ static void ice_ptp_periodic_work(struct kthread_work *work)
+ {
+ struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work);
+ struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp);
++ int err;
+
+ if (!test_bit(ICE_FLAG_PTP, pf->flags))
+ return;
+
+- ice_ptp_update_cached_phctime(pf);
++ err = ice_ptp_update_cached_phctime(pf);
+
+- ice_ptp_tx_tstamp_cleanup(&pf->ptp.port.tx);
++ ice_ptp_tx_tstamp_cleanup(&pf->hw, &pf->ptp.port.tx);
+
+- /* Run twice a second */
++ /* Run twice a second or reschedule if phc update failed */
+ kthread_queue_delayed_work(ptp->kworker, &ptp->work,
+- msecs_to_jiffies(500));
++ msecs_to_jiffies(err ? 10 : 500));
+ }
+
+ /**
+diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h
+index e1c787bd5b967..8cdd6f7046b73 100644
+--- a/drivers/net/ethernet/intel/ice/ice_ptp.h
++++ b/drivers/net/ethernet/intel/ice/ice_ptp.h
+@@ -46,15 +46,21 @@ struct ice_perout_channel {
+ * struct ice_tx_tstamp - Tracking for a single Tx timestamp
+ * @skb: pointer to the SKB for this timestamp request
+ * @start: jiffies when the timestamp was first requested
++ * @cached_tstamp: last read timestamp
+ *
+ * This structure tracks a single timestamp request. The SKB pointer is
+ * provided when initiating a request. The start time is used to ensure that
+ * we discard old requests that were not fulfilled within a 2 second time
+ * window.
++ * Timestamp values in the PHY are read only and do not get cleared except at
++ * hardware reset or when a new timestamp value is captured. The cached_tstamp
++ * field is used to detect the case where a new timestamp has not yet been
++ * captured, ensuring that we avoid sending stale timestamp data to the stack.
+ */
+ struct ice_tx_tstamp {
+ struct sk_buff *skb;
+ unsigned long start;
++ u64 cached_tstamp;
+ };
+
+ /**
+diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h
+index 1e46e80f3d6f8..4adc3dff04ba7 100644
+--- a/drivers/net/ethernet/intel/ice/ice_txrx.h
++++ b/drivers/net/ethernet/intel/ice/ice_txrx.h
+@@ -164,17 +164,10 @@ struct ice_tx_offload_params {
+ };
+
+ struct ice_rx_buf {
+- union {
+- struct {
+- dma_addr_t dma;
+- struct page *page;
+- unsigned int page_offset;
+- u16 pagecnt_bias;
+- };
+- struct {
+- struct xdp_buff *xdp;
+- };
+- };
++ dma_addr_t dma;
++ struct page *page;
++ unsigned int page_offset;
++ u16 pagecnt_bias;
+ };
+
+ struct ice_q_stats {
+@@ -270,6 +263,7 @@ struct ice_ring {
+ union {
+ struct ice_tx_buf *tx_buf;
+ struct ice_rx_buf *rx_buf;
++ struct xdp_buff **xdp_buf;
+ };
+ /* CL2 - 2nd cacheline starts here */
+ u16 q_index; /* Queue number of ring */
+@@ -338,6 +332,11 @@ static inline bool ice_ring_is_xdp(struct ice_ring *ring)
+ return !!(ring->flags & ICE_TX_FLAGS_RING_XDP);
+ }
+
++enum ice_container_type {
++ ICE_RX_CONTAINER,
++ ICE_TX_CONTAINER,
++};
++
+ struct ice_ring_container {
+ /* head of linked-list of rings */
+ struct ice_ring *ring;
+@@ -346,9 +345,15 @@ struct ice_ring_container {
+ /* this matches the maximum number of ITR bits, but in usec
+ * values, so it is shifted left one bit (bit zero is ignored)
+ */
+- u16 itr_setting:13;
+- u16 itr_reserved:2;
+- u16 itr_mode:1;
++ union {
++ struct {
++ u16 itr_setting:13;
++ u16 itr_reserved:2;
++ u16 itr_mode:1;
++ };
++ u16 itr_settings;
++ };
++ enum ice_container_type type;
+ };
+
+ struct ice_coalesce_stored {
+diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+index e93430ab37f1e..9d4d58757e040 100644
+--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
++++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+@@ -251,7 +251,7 @@ ice_vc_hash_field_match_type ice_vc_hash_field_list_comms[] = {
+ * ice_get_vf_vsi - get VF's VSI based on the stored index
+ * @vf: VF used to get VSI
+ */
+-static struct ice_vsi *ice_get_vf_vsi(struct ice_vf *vf)
++struct ice_vsi *ice_get_vf_vsi(struct ice_vf *vf)
+ {
+ return vf->pf->vsi[vf->lan_vsi_idx];
+ }
+@@ -615,8 +615,6 @@ void ice_free_vfs(struct ice_pf *pf)
+ struct ice_hw *hw = &pf->hw;
+ unsigned int tmp, i;
+
+- set_bit(ICE_VF_DEINIT_IN_PROGRESS, pf->state);
+-
+ if (!pf->vf)
+ return;
+
+@@ -632,21 +630,26 @@ void ice_free_vfs(struct ice_pf *pf)
+ else
+ dev_warn(dev, "VFs are assigned - not disabling SR-IOV\n");
+
+- /* Avoid wait time by stopping all VFs at the same time */
+- ice_for_each_vf(pf, i)
+- if (test_bit(ICE_VF_STATE_QS_ENA, pf->vf[i].vf_states))
+- ice_dis_vf_qs(&pf->vf[i]);
+-
+ tmp = pf->num_alloc_vfs;
+ pf->num_qps_per_vf = 0;
+ pf->num_alloc_vfs = 0;
+ for (i = 0; i < tmp; i++) {
+- if (test_bit(ICE_VF_STATE_INIT, pf->vf[i].vf_states)) {
++ struct ice_vf *vf = &pf->vf[i];
++
++ mutex_lock(&vf->cfg_lock);
++
++ ice_dis_vf_qs(vf);
++
++ if (test_bit(ICE_VF_STATE_INIT, vf->vf_states)) {
+ /* disable VF qp mappings and set VF disable state */
+- ice_dis_vf_mappings(&pf->vf[i]);
+- set_bit(ICE_VF_STATE_DIS, pf->vf[i].vf_states);
+- ice_free_vf_res(&pf->vf[i]);
++ ice_dis_vf_mappings(vf);
++ set_bit(ICE_VF_STATE_DIS, vf->vf_states);
++ ice_free_vf_res(vf);
+ }
++
++ mutex_unlock(&vf->cfg_lock);
++
++ mutex_destroy(&vf->cfg_lock);
+ }
+
+ if (ice_sriov_free_msix_res(pf))
+@@ -682,7 +685,6 @@ void ice_free_vfs(struct ice_pf *pf)
+ i);
+
+ clear_bit(ICE_VF_DIS, pf->state);
+- clear_bit(ICE_VF_DEINIT_IN_PROGRESS, pf->state);
+ clear_bit(ICE_FLAG_SRIOV_ENA, pf->flags);
+ }
+
+@@ -1566,10 +1568,13 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
+ ice_for_each_vf(pf, v) {
+ vf = &pf->vf[v];
+
++ mutex_lock(&vf->cfg_lock);
++
+ vf->driver_caps = 0;
+ ice_vc_set_default_allowlist(vf);
+
+ ice_vf_fdir_exit(vf);
++ ice_vf_fdir_init(vf);
+ /* clean VF control VSI when resetting VFs since it should be
+ * setup only when VF creates its first FDIR rule.
+ */
+@@ -1579,6 +1584,8 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
+ ice_vf_pre_vsi_rebuild(vf);
+ ice_vf_rebuild_vsi(vf);
+ ice_vf_post_vsi_rebuild(vf);
++
++ mutex_unlock(&vf->cfg_lock);
+ }
+
+ ice_flush(hw);
+@@ -1625,6 +1632,8 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)
+ u32 reg;
+ int i;
+
++ lockdep_assert_held(&vf->cfg_lock);
++
+ dev = ice_pf_to_dev(pf);
+
+ if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) {
+@@ -1645,8 +1654,7 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)
+
+ vsi = ice_get_vf_vsi(vf);
+
+- if (test_bit(ICE_VF_STATE_QS_ENA, vf->vf_states))
+- ice_dis_vf_qs(vf);
++ ice_dis_vf_qs(vf);
+
+ /* Call Disable LAN Tx queue AQ whether or not queues are
+ * enabled. This is needed for successful completion of VFR.
+@@ -1697,6 +1705,7 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)
+ }
+
+ ice_vf_fdir_exit(vf);
++ ice_vf_fdir_init(vf);
+ /* clean VF control VSI when resetting VF since it should be setup
+ * only when VF creates its first FDIR rule.
+ */
+@@ -1894,6 +1903,8 @@ static void ice_set_dflt_settings_vfs(struct ice_pf *pf)
+ */
+ ice_vf_ctrl_invalidate_vsi(vf);
+ ice_vf_fdir_init(vf);
++
++ mutex_init(&vf->cfg_lock);
+ }
+ }
+
+@@ -2109,9 +2120,12 @@ void ice_process_vflr_event(struct ice_pf *pf)
+ bit_idx = (hw->func_caps.vf_base_id + vf_id) % 32;
+ /* read GLGEN_VFLRSTAT register to find out the flr VFs */
+ reg = rd32(hw, GLGEN_VFLRSTAT(reg_idx));
+- if (reg & BIT(bit_idx))
++ if (reg & BIT(bit_idx)) {
+ /* GLGEN_VFLRSTAT bit will be cleared in ice_reset_vf */
++ mutex_lock(&vf->cfg_lock);
+ ice_reset_vf(vf, true);
++ mutex_unlock(&vf->cfg_lock);
++ }
+ }
+ }
+
+@@ -2188,7 +2202,9 @@ ice_vf_lan_overflow_event(struct ice_pf *pf, struct ice_rq_event_info *event)
+ if (!vf)
+ return;
+
++ mutex_lock(&vf->cfg_lock);
+ ice_vc_reset_vf(vf);
++ mutex_unlock(&vf->cfg_lock);
+ }
+
+ /**
+@@ -2218,24 +2234,6 @@ ice_vc_send_msg_to_vf(struct ice_vf *vf, u32 v_opcode,
+
+ dev = ice_pf_to_dev(pf);
+
+- /* single place to detect unsuccessful return values */
+- if (v_retval) {
+- vf->num_inval_msgs++;
+- dev_info(dev, "VF %d failed opcode %d, retval: %d\n", vf->vf_id,
+- v_opcode, v_retval);
+- if (vf->num_inval_msgs > ICE_DFLT_NUM_INVAL_MSGS_ALLOWED) {
+- dev_err(dev, "Number of invalid messages exceeded for VF %d\n",
+- vf->vf_id);
+- dev_err(dev, "Use PF Control I/F to enable the VF\n");
+- set_bit(ICE_VF_STATE_DIS, vf->vf_states);
+- return -EIO;
+- }
+- } else {
+- vf->num_valid_msgs++;
+- /* reset the invalid counter, if a valid message is received. */
+- vf->num_inval_msgs = 0;
+- }
+-
+ aq_ret = ice_aq_send_msg_to_vf(&pf->hw, vf->vf_id, v_opcode, v_retval,
+ msg, msglen, NULL);
+ if (aq_ret && pf->hw.mailboxq.sq_last_status != ICE_AQ_RC_ENOSYS) {
+@@ -2954,6 +2952,7 @@ bool ice_is_any_vf_in_promisc(struct ice_pf *pf)
+ static int ice_vc_cfg_promiscuous_mode_msg(struct ice_vf *vf, u8 *msg)
+ {
+ enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS;
++ enum ice_status mcast_status = 0, ucast_status = 0;
+ bool rm_promisc, alluni = false, allmulti = false;
+ struct virtchnl_promisc_info *info =
+ (struct virtchnl_promisc_info *)msg;
+@@ -3043,52 +3042,51 @@ static int ice_vc_cfg_promiscuous_mode_msg(struct ice_vf *vf, u8 *msg)
+ goto error_param;
+ }
+ } else {
+- enum ice_status status;
+- u8 promisc_m;
+-
+- if (alluni) {
+- if (vf->port_vlan_info || vsi->num_vlan)
+- promisc_m = ICE_UCAST_VLAN_PROMISC_BITS;
+- else
+- promisc_m = ICE_UCAST_PROMISC_BITS;
+- } else if (allmulti) {
+- if (vf->port_vlan_info || vsi->num_vlan)
+- promisc_m = ICE_MCAST_VLAN_PROMISC_BITS;
+- else
+- promisc_m = ICE_MCAST_PROMISC_BITS;
++ u8 mcast_m, ucast_m;
++
++ if (vf->port_vlan_info || vsi->num_vlan > 1) {
++ mcast_m = ICE_MCAST_VLAN_PROMISC_BITS;
++ ucast_m = ICE_UCAST_VLAN_PROMISC_BITS;
+ } else {
+- if (vf->port_vlan_info || vsi->num_vlan)
+- promisc_m = ICE_UCAST_VLAN_PROMISC_BITS;
+- else
+- promisc_m = ICE_UCAST_PROMISC_BITS;
++ mcast_m = ICE_MCAST_PROMISC_BITS;
++ ucast_m = ICE_UCAST_PROMISC_BITS;
+ }
+
+- /* Configure multicast/unicast with or without VLAN promiscuous
+- * mode
+- */
+- status = ice_vf_set_vsi_promisc(vf, vsi, promisc_m, rm_promisc);
+- if (status) {
+- dev_err(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d failed, error: %s\n",
+- rm_promisc ? "dis" : "en", vf->vf_id,
+- ice_stat_str(status));
+- v_ret = ice_err_to_virt_err(status);
+- goto error_param;
+- } else {
+- dev_dbg(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d succeeded\n",
+- rm_promisc ? "dis" : "en", vf->vf_id);
++ ucast_status = ice_vf_set_vsi_promisc(vf, vsi, ucast_m,
++ !alluni);
++ if (ucast_status) {
++ dev_err(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d failed\n",
++ alluni ? "en" : "dis", vf->vf_id);
++ v_ret = ice_err_to_virt_err(ucast_status);
++ }
++
++ mcast_status = ice_vf_set_vsi_promisc(vf, vsi, mcast_m,
++ !allmulti);
++ if (mcast_status) {
++ dev_err(dev, "%sable Tx/Rx filter promiscuous mode on VF-%d failed\n",
++ allmulti ? "en" : "dis", vf->vf_id);
++ v_ret = ice_err_to_virt_err(mcast_status);
+ }
+ }
+
+- if (allmulti &&
+- !test_and_set_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states))
+- dev_info(dev, "VF %u successfully set multicast promiscuous mode\n", vf->vf_id);
+- else if (!allmulti && test_and_clear_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states))
+- dev_info(dev, "VF %u successfully unset multicast promiscuous mode\n", vf->vf_id);
++ if (!mcast_status) {
++ if (allmulti &&
++ !test_and_set_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states))
++ dev_info(dev, "VF %u successfully set multicast promiscuous mode\n",
++ vf->vf_id);
++ else if (!allmulti && test_and_clear_bit(ICE_VF_STATE_MC_PROMISC, vf->vf_states))
++ dev_info(dev, "VF %u successfully unset multicast promiscuous mode\n",
++ vf->vf_id);
++ }
+
+- if (alluni && !test_and_set_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states))
+- dev_info(dev, "VF %u successfully set unicast promiscuous mode\n", vf->vf_id);
+- else if (!alluni && test_and_clear_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states))
+- dev_info(dev, "VF %u successfully unset unicast promiscuous mode\n", vf->vf_id);
++ if (!ucast_status) {
++ if (alluni && !test_and_set_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states))
++ dev_info(dev, "VF %u successfully set unicast promiscuous mode\n",
++ vf->vf_id);
++ else if (!alluni && test_and_clear_bit(ICE_VF_STATE_UC_PROMISC, vf->vf_states))
++ dev_info(dev, "VF %u successfully unset unicast promiscuous mode\n",
++ vf->vf_id);
++ }
+
+ error_param:
+ return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE,
+@@ -3337,9 +3335,9 @@ static int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg)
+ goto error_param;
+ }
+
+- /* Skip queue if not enabled */
+ if (!test_bit(vf_q_id, vf->txq_ena))
+- continue;
++ dev_dbg(ice_pf_to_dev(vsi->back), "Queue %u on VSI %u is not enabled, but stopping it anyway\n",
++ vf_q_id, vsi->vsi_num);
+
+ ice_fill_txq_meta(vsi, ring, &txq_meta);
+
+@@ -3762,6 +3760,7 @@ ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi,
+ struct device *dev = ice_pf_to_dev(vf->pf);
+ u8 *mac_addr = vc_ether_addr->addr;
+ enum ice_status status;
++ int ret = 0;
+
+ /* device MAC already added */
+ if (ether_addr_equal(mac_addr, vf->dev_lan_addr.addr))
+@@ -3774,20 +3773,23 @@ ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi,
+
+ status = ice_fltr_add_mac(vsi, mac_addr, ICE_FWD_TO_VSI);
+ if (status == ICE_ERR_ALREADY_EXISTS) {
+- dev_err(dev, "MAC %pM already exists for VF %d\n", mac_addr,
++ dev_dbg(dev, "MAC %pM already exists for VF %d\n", mac_addr,
+ vf->vf_id);
+- return -EEXIST;
++ /* don't return since we might need to update
++ * the primary MAC in ice_vfhw_mac_add() below
++ */
++ ret = -EEXIST;
+ } else if (status) {
+ dev_err(dev, "Failed to add MAC %pM for VF %d\n, error %s\n",
+ mac_addr, vf->vf_id, ice_stat_str(status));
+ return -EIO;
++ } else {
++ vf->num_mac++;
+ }
+
+ ice_vfhw_mac_add(vf, vc_ether_addr);
+
+- vf->num_mac++;
+-
+- return 0;
++ return ret;
+ }
+
+ /**
+@@ -4078,6 +4080,8 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos,
+ return 0;
+ }
+
++ mutex_lock(&vf->cfg_lock);
++
+ vf->port_vlan_info = vlanprio;
+
+ if (vf->port_vlan_info)
+@@ -4087,6 +4091,7 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos,
+ dev_info(dev, "Clearing port VLAN on VF %d\n", vf_id);
+
+ ice_vc_reset_vf(vf);
++ mutex_unlock(&vf->cfg_lock);
+
+ return 0;
+ }
+@@ -4418,10 +4423,6 @@ void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event)
+ struct device *dev;
+ int err = 0;
+
+- /* if de-init is underway, don't process messages from VF */
+- if (test_bit(ICE_VF_DEINIT_IN_PROGRESS, pf->state))
+- return;
+-
+ dev = ice_pf_to_dev(pf);
+ if (ice_validate_vf_id(pf, vf_id)) {
+ err = -EINVAL;
+@@ -4461,6 +4462,15 @@ error_handler:
+ return;
+ }
+
++ /* VF is being configured in another context that triggers a VFR, so no
++ * need to process this message
++ */
++ if (!mutex_trylock(&vf->cfg_lock)) {
++ dev_info(dev, "VF %u is being configured in another context that will trigger a VFR, so there is no need to handle this message\n",
++ vf->vf_id);
++ return;
++ }
++
+ switch (v_opcode) {
+ case VIRTCHNL_OP_VERSION:
+ err = ice_vc_get_ver_msg(vf, msg);
+@@ -4549,6 +4559,8 @@ error_handler:
+ dev_info(dev, "PF failed to honor VF %d, opcode %d, error %d\n",
+ vf_id, v_opcode, err);
+ }
++
++ mutex_unlock(&vf->cfg_lock);
+ }
+
+ /**
+@@ -4664,6 +4676,8 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
+ return -EINVAL;
+ }
+
++ mutex_lock(&vf->cfg_lock);
++
+ /* VF is notified of its new MAC via the PF's response to the
+ * VIRTCHNL_OP_GET_VF_RESOURCES message after the VF has been reset
+ */
+@@ -4682,6 +4696,7 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
+ }
+
+ ice_vc_reset_vf(vf);
++ mutex_unlock(&vf->cfg_lock);
+ return 0;
+ }
+
+@@ -4711,11 +4726,15 @@ int ice_set_vf_trust(struct net_device *netdev, int vf_id, bool trusted)
+ if (trusted == vf->trusted)
+ return 0;
+
++ mutex_lock(&vf->cfg_lock);
++
+ vf->trusted = trusted;
+ ice_vc_reset_vf(vf);
+ dev_info(ice_pf_to_dev(pf), "VF %u is now %strusted\n",
+ vf_id, trusted ? "" : "un");
+
++ mutex_unlock(&vf->cfg_lock);
++
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
+index 842cb077df861..532f57f014673 100644
+--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
++++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
+@@ -14,7 +14,6 @@
+ #define ICE_MAX_MACADDR_PER_VF 18
+
+ /* Malicious Driver Detection */
+-#define ICE_DFLT_NUM_INVAL_MSGS_ALLOWED 10
+ #define ICE_MDD_EVENTS_THRESHOLD 30
+
+ /* Static VF transaction/status register def */
+@@ -74,6 +73,11 @@ struct ice_mdd_vf_events {
+ struct ice_vf {
+ struct ice_pf *pf;
+
++ /* Used during virtchnl message handling and NDO ops against the VF
++ * that will trigger a VFR
++ */
++ struct mutex cfg_lock;
++
+ u16 vf_id; /* VF ID in the PF space */
+ u16 lan_vsi_idx; /* index into PF struct */
+ u16 ctrl_vsi_idx;
+@@ -102,8 +106,6 @@ struct ice_vf {
+ unsigned int tx_rate; /* Tx bandwidth limit in Mbps */
+ DECLARE_BITMAP(vf_states, ICE_VF_STATES_NBITS); /* VF runtime states */
+
+- u64 num_inval_msgs; /* number of continuous invalid msgs */
+- u64 num_valid_msgs; /* number of valid msgs detected */
+ unsigned long vf_caps; /* VF's adv. capabilities */
+ u8 num_req_qs; /* num of queue pairs requested by VF */
+ u16 num_mac;
+@@ -111,9 +113,13 @@ struct ice_vf {
+ struct ice_mdd_vf_events mdd_rx_events;
+ struct ice_mdd_vf_events mdd_tx_events;
+ DECLARE_BITMAP(opcodes_allowlist, VIRTCHNL_OP_MAX);
++
++ /* devlink port data */
++ struct devlink_port devlink_port;
+ };
+
+ #ifdef CONFIG_PCI_IOV
++struct ice_vsi *ice_get_vf_vsi(struct ice_vf *vf);
+ void ice_process_vflr_event(struct ice_pf *pf);
+ int ice_sriov_configure(struct pci_dev *pdev, int num_vfs);
+ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac);
+@@ -171,6 +177,11 @@ static inline void ice_print_vfs_mdd_events(struct ice_pf *pf) { }
+ static inline void ice_print_vf_rx_mdd_event(struct ice_vf *vf) { }
+ static inline void ice_restore_all_vfs_msi_state(struct pci_dev *pdev) { }
+
++static inline struct ice_vsi *ice_get_vf_vsi(struct ice_vf *vf)
++{
++ return NULL;
++}
++
+ static inline bool
+ ice_is_malicious_vf(struct ice_pf __always_unused *pf,
+ struct ice_rq_event_info __always_unused *event,
+diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
+index 5a9f61deeb38d..5581747947e57 100644
+--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
++++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
+@@ -36,8 +36,10 @@ static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
+ static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
+ {
+ ice_clean_tx_ring(vsi->tx_rings[q_idx]);
+- if (ice_is_xdp_ena_vsi(vsi))
++ if (ice_is_xdp_ena_vsi(vsi)) {
++ synchronize_rcu();
+ ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
++ }
+ ice_clean_rx_ring(vsi->rx_rings[q_idx]);
+ }
+
+@@ -364,7 +366,7 @@ bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count)
+ {
+ union ice_32b_rx_flex_desc *rx_desc;
+ u16 ntu = rx_ring->next_to_use;
+- struct ice_rx_buf *rx_buf;
++ struct xdp_buff **xdp;
+ bool ok = true;
+ dma_addr_t dma;
+
+@@ -372,26 +374,26 @@ bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count)
+ return true;
+
+ rx_desc = ICE_RX_DESC(rx_ring, ntu);
+- rx_buf = &rx_ring->rx_buf[ntu];
++ xdp = &rx_ring->xdp_buf[ntu];
+
+ do {
+- rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_pool);
+- if (!rx_buf->xdp) {
++ *xdp = xsk_buff_alloc(rx_ring->xsk_pool);
++ if (!*xdp) {
+ ok = false;
+ break;
+ }
+
+- dma = xsk_buff_xdp_get_dma(rx_buf->xdp);
++ dma = xsk_buff_xdp_get_dma(*xdp);
+ rx_desc->read.pkt_addr = cpu_to_le64(dma);
+ rx_desc->wb.status_error0 = 0;
+
+ rx_desc++;
+- rx_buf++;
++ xdp++;
+ ntu++;
+
+ if (unlikely(ntu == rx_ring->count)) {
+ rx_desc = ICE_RX_DESC(rx_ring, 0);
+- rx_buf = rx_ring->rx_buf;
++ xdp = rx_ring->xdp_buf;
+ ntu = 0;
+ }
+ } while (--count);
+@@ -421,19 +423,19 @@ static void ice_bump_ntc(struct ice_ring *rx_ring)
+ /**
+ * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
+ * @rx_ring: Rx ring
+- * @rx_buf: zero-copy Rx buffer
++ * @xdp_arr: Pointer to the SW ring of xdp_buff pointers
+ *
+ * This function allocates a new skb from a zero-copy Rx buffer.
+ *
+ * Returns the skb on success, NULL on failure.
+ */
+ static struct sk_buff *
+-ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
++ice_construct_skb_zc(struct ice_ring *rx_ring, struct xdp_buff **xdp_arr)
+ {
+- unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta;
+- unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data;
+- unsigned int datasize_hard = rx_buf->xdp->data_end -
+- rx_buf->xdp->data_hard_start;
++ struct xdp_buff *xdp = *xdp_arr;
++ unsigned int metasize = xdp->data - xdp->data_meta;
++ unsigned int datasize = xdp->data_end - xdp->data;
++ unsigned int datasize_hard = xdp->data_end - xdp->data_hard_start;
+ struct sk_buff *skb;
+
+ skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,
+@@ -441,13 +443,13 @@ ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
+ if (unlikely(!skb))
+ return NULL;
+
+- skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start);
+- memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize);
++ skb_reserve(skb, xdp->data - xdp->data_hard_start);
++ memcpy(__skb_put(skb, datasize), xdp->data, datasize);
+ if (metasize)
+ skb_metadata_set(skb, metasize);
+
+- xsk_buff_free(rx_buf->xdp);
+- rx_buf->xdp = NULL;
++ xsk_buff_free(xdp);
++ *xdp_arr = NULL;
+ return skb;
+ }
+
+@@ -521,7 +523,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
+ while (likely(total_rx_packets < (unsigned int)budget)) {
+ union ice_32b_rx_flex_desc *rx_desc;
+ unsigned int size, xdp_res = 0;
+- struct ice_rx_buf *rx_buf;
++ struct xdp_buff **xdp;
+ struct sk_buff *skb;
+ u16 stat_err_bits;
+ u16 vlan_tag = 0;
+@@ -544,18 +546,18 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
+ if (!size)
+ break;
+
+- rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
+- rx_buf->xdp->data_end = rx_buf->xdp->data + size;
+- xsk_buff_dma_sync_for_cpu(rx_buf->xdp, rx_ring->xsk_pool);
++ xdp = &rx_ring->xdp_buf[rx_ring->next_to_clean];
++ (*xdp)->data_end = (*xdp)->data + size;
++ xsk_buff_dma_sync_for_cpu(*xdp, rx_ring->xsk_pool);
+
+- xdp_res = ice_run_xdp_zc(rx_ring, rx_buf->xdp);
++ xdp_res = ice_run_xdp_zc(rx_ring, *xdp);
+ if (xdp_res) {
+ if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))
+ xdp_xmit |= xdp_res;
+ else
+- xsk_buff_free(rx_buf->xdp);
++ xsk_buff_free(*xdp);
+
+- rx_buf->xdp = NULL;
++ *xdp = NULL;
+ total_rx_bytes += size;
+ total_rx_packets++;
+ cleaned_count++;
+@@ -565,7 +567,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
+ }
+
+ /* XDP_PASS path */
+- skb = ice_construct_skb_zc(rx_ring, rx_buf);
++ skb = ice_construct_skb_zc(rx_ring, xdp);
+ if (!skb) {
+ rx_ring->rx_stats.alloc_buf_failed++;
+ break;
+@@ -759,7 +761,7 @@ ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,
+ struct ice_vsi *vsi = np->vsi;
+ struct ice_ring *ring;
+
+- if (test_bit(ICE_DOWN, vsi->state))
++ if (test_bit(ICE_VSI_DOWN, vsi->state))
+ return -ENETDOWN;
+
+ if (!ice_is_xdp_ena_vsi(vsi))
+@@ -810,15 +812,15 @@ bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
+ */
+ void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)
+ {
+- u16 i;
+-
+- for (i = 0; i < rx_ring->count; i++) {
+- struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];
++ u16 count_mask = rx_ring->count - 1;
++ u16 ntc = rx_ring->next_to_clean;
++ u16 ntu = rx_ring->next_to_use;
+
+- if (!rx_buf->xdp)
+- continue;
++ for ( ; ntc != ntu; ntc = (ntc + 1) & count_mask) {
++ struct xdp_buff **xdp = &rx_ring->xdp_buf[ntc];
+
+- rx_buf->xdp = NULL;
++ xsk_buff_free(*xdp);
++ *xdp = NULL;
+ }
+ }
+
+diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
+index fb1029352c3e7..3cbb5a89b336f 100644
+--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
++++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
+@@ -961,10 +961,6 @@ static int igb_set_ringparam(struct net_device *netdev,
+ memcpy(&temp_ring[i], adapter->rx_ring[i],
+ sizeof(struct igb_ring));
+
+- /* Clear copied XDP RX-queue info */
+- memset(&temp_ring[i].xdp_rxq, 0,
+- sizeof(temp_ring[i].xdp_rxq));
+-
+ temp_ring[i].count = new_rx_count;
+ err = igb_setup_rx_resources(&temp_ring[i]);
+ if (err) {
+diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
+index 751de06019a0e..b883033514843 100644
+--- a/drivers/net/ethernet/intel/igb/igb_main.c
++++ b/drivers/net/ethernet/intel/igb/igb_main.c
+@@ -4345,7 +4345,18 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring)
+ {
+ struct igb_adapter *adapter = netdev_priv(rx_ring->netdev);
+ struct device *dev = rx_ring->dev;
+- int size;
++ int size, res;
++
++ /* XDP RX-queue info */
++ if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
++ xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
++ res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
++ rx_ring->queue_index, 0);
++ if (res < 0) {
++ dev_err(dev, "Failed to register xdp_rxq index %u\n",
++ rx_ring->queue_index);
++ return res;
++ }
+
+ size = sizeof(struct igb_rx_buffer) * rx_ring->count;
+
+@@ -4368,14 +4379,10 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring)
+
+ rx_ring->xdp_prog = adapter->xdp_prog;
+
+- /* XDP RX-queue info */
+- if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
+- rx_ring->queue_index, 0) < 0)
+- goto err;
+-
+ return 0;
+
+ err:
++ xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
+ vfree(rx_ring->rx_buffer_info);
+ rx_ring->rx_buffer_info = NULL;
+ dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
+@@ -5498,7 +5505,8 @@ static void igb_watchdog_task(struct work_struct *work)
+ break;
+ }
+
+- if (adapter->link_speed != SPEED_1000)
++ if (adapter->link_speed != SPEED_1000 ||
++ !hw->phy.ops.read_reg)
+ goto no_wait;
+
+ /* wait for Remote receiver status OK */
+@@ -7641,6 +7649,20 @@ static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
+ struct vf_mac_filter *entry = NULL;
+ int ret = 0;
+
++ if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
++ !vf_data->trusted) {
++ dev_warn(&pdev->dev,
++ "VF %d requested MAC filter but is administratively denied\n",
++ vf);
++ return -EINVAL;
++ }
++ if (!is_valid_ether_addr(addr)) {
++ dev_warn(&pdev->dev,
++ "VF %d attempted to set invalid MAC filter\n",
++ vf);
++ return -EINVAL;
++ }
++
+ switch (info) {
+ case E1000_VF_MAC_FILTER_CLR:
+ /* remove all unicast MAC filters related to the current VF */
+@@ -7654,20 +7676,6 @@ static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
+ }
+ break;
+ case E1000_VF_MAC_FILTER_ADD:
+- if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
+- !vf_data->trusted) {
+- dev_warn(&pdev->dev,
+- "VF %d requested MAC filter but is administratively denied\n",
+- vf);
+- return -EINVAL;
+- }
+- if (!is_valid_ether_addr(addr)) {
+- dev_warn(&pdev->dev,
+- "VF %d attempted to set invalid MAC filter\n",
+- vf);
+- return -EINVAL;
+- }
+-
+ /* try to find empty slot in the list */
+ list_for_each(pos, &adapter->vf_macs.l) {
+ entry = list_entry(pos, struct vf_mac_filter, l);
+@@ -8019,7 +8027,7 @@ static int igb_poll(struct napi_struct *napi, int budget)
+ if (likely(napi_complete_done(napi, work_done)))
+ igb_ring_irq_enable(q_vector);
+
+- return min(work_done, budget - 1);
++ return work_done;
+ }
+
+ /**
+@@ -9247,7 +9255,7 @@ static int __maybe_unused igb_suspend(struct device *dev)
+ return __igb_shutdown(to_pci_dev(dev), NULL, 0);
+ }
+
+-static int __maybe_unused igb_resume(struct device *dev)
++static int __maybe_unused __igb_resume(struct device *dev, bool rpm)
+ {
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct net_device *netdev = pci_get_drvdata(pdev);
+@@ -9290,17 +9298,24 @@ static int __maybe_unused igb_resume(struct device *dev)
+
+ wr32(E1000_WUS, ~0);
+
+- rtnl_lock();
++ if (!rpm)
++ rtnl_lock();
+ if (!err && netif_running(netdev))
+ err = __igb_open(netdev, true);
+
+ if (!err)
+ netif_device_attach(netdev);
+- rtnl_unlock();
++ if (!rpm)
++ rtnl_unlock();
+
+ return err;
+ }
+
++static int __maybe_unused igb_resume(struct device *dev)
++{
++ return __igb_resume(dev, false);
++}
++
+ static int __maybe_unused igb_runtime_idle(struct device *dev)
+ {
+ struct net_device *netdev = dev_get_drvdata(dev);
+@@ -9319,7 +9334,7 @@ static int __maybe_unused igb_runtime_suspend(struct device *dev)
+
+ static int __maybe_unused igb_runtime_resume(struct device *dev)
+ {
+- return igb_resume(dev);
++ return __igb_resume(dev, true);
+ }
+
+ static void igb_shutdown(struct pci_dev *pdev)
+@@ -9435,7 +9450,7 @@ static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
+ * @pdev: Pointer to PCI device
+ *
+ * Restart the card from scratch, as if from a cold-boot. Implementation
+- * resembles the first-half of the igb_resume routine.
++ * resembles the first-half of the __igb_resume routine.
+ **/
+ static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
+ {
+@@ -9475,7 +9490,7 @@ static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
+ *
+ * This callback is called when the error recovery driver tells us that
+ * its OK to resume normal operation. Implementation resembles the
+- * second-half of the igb_resume routine.
++ * second-half of the __igb_resume routine.
+ */
+ static void igb_io_resume(struct pci_dev *pdev)
+ {
+diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
+index d32e72d953c8d..d051918dfdff9 100644
+--- a/drivers/net/ethernet/intel/igbvf/netdev.c
++++ b/drivers/net/ethernet/intel/igbvf/netdev.c
+@@ -2861,6 +2861,7 @@ static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ return 0;
+
+ err_hw_init:
++ netif_napi_del(&adapter->rx_ring->napi);
+ kfree(adapter->tx_ring);
+ kfree(adapter->rx_ring);
+ err_sw_init:
+diff --git a/drivers/net/ethernet/intel/igc/igc_i225.c b/drivers/net/ethernet/intel/igc/igc_i225.c
+index b2ef9fde97b38..a0e2a404d5355 100644
+--- a/drivers/net/ethernet/intel/igc/igc_i225.c
++++ b/drivers/net/ethernet/intel/igc/igc_i225.c
+@@ -156,8 +156,15 @@ void igc_release_swfw_sync_i225(struct igc_hw *hw, u16 mask)
+ {
+ u32 swfw_sync;
+
+- while (igc_get_hw_semaphore_i225(hw))
+- ; /* Empty */
++ /* Releasing the resource requires first getting the HW semaphore.
++ * If we fail to get the semaphore, there is nothing we can do,
++ * except log an error and quit. We are not allowed to hang here
++ * indefinitely, as it may cause denial of service or system crash.
++ */
++ if (igc_get_hw_semaphore_i225(hw)) {
++ hw_dbg("Failed to release SW_FW_SYNC.\n");
++ return;
++ }
+
+ swfw_sync = rd32(IGC_SW_FW_SYNC);
+ swfw_sync &= ~mask;
+@@ -636,7 +643,7 @@ s32 igc_set_ltr_i225(struct igc_hw *hw, bool link)
+ ltrv = rd32(IGC_LTRMAXV);
+ if (ltr_max != (ltrv & IGC_LTRMAXV_LTRV_MASK)) {
+ ltrv = IGC_LTRMAXV_LSNP_REQ | ltr_max |
+- (scale_min << IGC_LTRMAXV_SCALE_SHIFT);
++ (scale_max << IGC_LTRMAXV_SCALE_SHIFT);
+ wr32(IGC_LTRMAXV, ltrv);
+ }
+ }
+diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
+index 0e19b4d02e628..f99819fc559d9 100644
+--- a/drivers/net/ethernet/intel/igc/igc_main.c
++++ b/drivers/net/ethernet/intel/igc/igc_main.c
+@@ -504,6 +504,9 @@ int igc_setup_rx_resources(struct igc_ring *rx_ring)
+ u8 index = rx_ring->queue_index;
+ int size, desc_len, res;
+
++ /* XDP RX-queue info */
++ if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
++ xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
+ res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
+ rx_ring->q_vector->napi.napi_id);
+ if (res < 0) {
+@@ -2434,21 +2437,24 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
+ static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
+ struct xdp_buff *xdp)
+ {
++ unsigned int totalsize = xdp->data_end - xdp->data_meta;
+ unsigned int metasize = xdp->data - xdp->data_meta;
+- unsigned int datasize = xdp->data_end - xdp->data;
+- unsigned int totalsize = metasize + datasize;
+ struct sk_buff *skb;
+
+- skb = __napi_alloc_skb(&ring->q_vector->napi,
+- xdp->data_end - xdp->data_hard_start,
++ net_prefetch(xdp->data_meta);
++
++ skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
+ GFP_ATOMIC | __GFP_NOWARN);
+ if (unlikely(!skb))
+ return NULL;
+
+- skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
+- memcpy(__skb_put(skb, totalsize), xdp->data_meta, totalsize);
+- if (metasize)
++ memcpy(__skb_put(skb, totalsize), xdp->data_meta,
++ ALIGN(totalsize, sizeof(long)));
++
++ if (metasize) {
+ skb_metadata_set(skb, metasize);
++ __skb_pull(skb, metasize);
++ }
+
+ return skb;
+ }
+@@ -5466,6 +5472,9 @@ static irqreturn_t igc_intr_msi(int irq, void *data)
+ mod_timer(&adapter->watchdog_timer, jiffies + 1);
+ }
+
++ if (icr & IGC_ICR_TS)
++ igc_tsync_interrupt(adapter);
++
+ napi_schedule(&q_vector->napi);
+
+ return IRQ_HANDLED;
+@@ -5509,6 +5518,9 @@ static irqreturn_t igc_intr(int irq, void *data)
+ mod_timer(&adapter->watchdog_timer, jiffies + 1);
+ }
+
++ if (icr & IGC_ICR_TS)
++ igc_tsync_interrupt(adapter);
++
+ napi_schedule(&q_vector->napi);
+
+ return IRQ_HANDLED;
+diff --git a/drivers/net/ethernet/intel/igc/igc_phy.c b/drivers/net/ethernet/intel/igc/igc_phy.c
+index 5cad31c3c7b09..6961f65d36b9a 100644
+--- a/drivers/net/ethernet/intel/igc/igc_phy.c
++++ b/drivers/net/ethernet/intel/igc/igc_phy.c
+@@ -581,7 +581,7 @@ static s32 igc_read_phy_reg_mdic(struct igc_hw *hw, u32 offset, u16 *data)
+ * the lower time out
+ */
+ for (i = 0; i < IGC_GEN_POLL_TIMEOUT; i++) {
+- usleep_range(500, 1000);
++ udelay(50);
+ mdic = rd32(IGC_MDIC);
+ if (mdic & IGC_MDIC_READY)
+ break;
+@@ -638,7 +638,7 @@ static s32 igc_write_phy_reg_mdic(struct igc_hw *hw, u32 offset, u16 data)
+ * the lower time out
+ */
+ for (i = 0; i < IGC_GEN_POLL_TIMEOUT; i++) {
+- usleep_range(500, 1000);
++ udelay(50);
+ mdic = rd32(IGC_MDIC);
+ if (mdic & IGC_MDIC_READY)
+ break;
+@@ -746,8 +746,6 @@ s32 igc_write_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 data)
+ if (ret_val)
+ return ret_val;
+ ret_val = igc_write_phy_reg_mdic(hw, offset, data);
+- if (ret_val)
+- return ret_val;
+ hw->phy.ops.release(hw);
+ } else {
+ ret_val = igc_write_xmdio_reg(hw, (u16)offset, dev_addr,
+@@ -779,8 +777,6 @@ s32 igc_read_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 *data)
+ if (ret_val)
+ return ret_val;
+ ret_val = igc_read_phy_reg_mdic(hw, offset, data);
+- if (ret_val)
+- return ret_val;
+ hw->phy.ops.release(hw);
+ } else {
+ ret_val = igc_read_xmdio_reg(hw, (u16)offset, dev_addr,
+diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c
+index 0f021909b430a..8e521f99b80ae 100644
+--- a/drivers/net/ethernet/intel/igc/igc_ptp.c
++++ b/drivers/net/ethernet/intel/igc/igc_ptp.c
+@@ -768,12 +768,25 @@ int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
+ */
+ static bool igc_is_crosststamp_supported(struct igc_adapter *adapter)
+ {
+- return IS_ENABLED(CONFIG_X86_TSC) ? pcie_ptm_enabled(adapter->pdev) : false;
++ if (!IS_ENABLED(CONFIG_X86_TSC))
++ return false;
++
++ /* FIXME: it was noticed that enabling support for PCIe PTM in
++ * some i225-V models could cause lockups when bringing the
++ * interface up/down. There should be no downsides to
++ * disabling crosstimestamping support for i225-V, as it
++ * doesn't have any PTP support. That way we gain some time
++ * while root causing the issue.
++ */
++ if (adapter->pdev->device == IGC_DEV_ID_I225_V)
++ return false;
++
++ return pcie_ptm_enabled(adapter->pdev);
+ }
+
+ static struct system_counterval_t igc_device_tstamp_to_system(u64 tstamp)
+ {
+-#if IS_ENABLED(CONFIG_X86_TSC)
++#if IS_ENABLED(CONFIG_X86_TSC) && !defined(CONFIG_UML)
+ return convert_art_ns_to_tsc(tstamp);
+ #else
+ return (struct system_counterval_t) { };
+@@ -983,6 +996,17 @@ static void igc_ptp_time_restore(struct igc_adapter *adapter)
+ igc_ptp_write_i225(adapter, &ts);
+ }
+
++static void igc_ptm_stop(struct igc_adapter *adapter)
++{
++ struct igc_hw *hw = &adapter->hw;
++ u32 ctrl;
++
++ ctrl = rd32(IGC_PTM_CTRL);
++ ctrl &= ~IGC_PTM_CTRL_EN;
++
++ wr32(IGC_PTM_CTRL, ctrl);
++}
++
+ /**
+ * igc_ptp_suspend - Disable PTP work items and prepare for suspend
+ * @adapter: Board private structure
+@@ -1000,8 +1024,10 @@ void igc_ptp_suspend(struct igc_adapter *adapter)
+ adapter->ptp_tx_skb = NULL;
+ clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state);
+
+- if (pci_device_is_present(adapter->pdev))
++ if (pci_device_is_present(adapter->pdev)) {
+ igc_ptp_time_save(adapter);
++ igc_ptm_stop(adapter);
++ }
+ }
+
+ /**
+diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
+index e596e1a9fc757..69d11ff7677d6 100644
+--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
++++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
+@@ -903,7 +903,8 @@ int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf)
+ /* Tx IPsec offload doesn't seem to work on this
+ * device, so block these requests for now.
+ */
+- if (!(sam->flags & XFRM_OFFLOAD_INBOUND)) {
++ sam->flags = sam->flags & ~XFRM_OFFLOAD_IPV6;
++ if (sam->flags != XFRM_OFFLOAD_INBOUND) {
+ err = -EOPNOTSUPP;
+ goto err_out;
+ }
+diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+index 13c4782b920a7..750b02bb2fdc2 100644
+--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
++++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+@@ -5526,6 +5526,10 @@ static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
+ if (!speed && hw->mac.ops.get_link_capabilities) {
+ ret = hw->mac.ops.get_link_capabilities(hw, &speed,
+ &autoneg);
++ /* remove NBASE-T speeds from default autonegotiation
++ * to accommodate broken network switches in the field
++ * which cannot cope with advertised NBASE-T speeds
++ */
+ speed &= ~(IXGBE_LINK_SPEED_5GB_FULL |
+ IXGBE_LINK_SPEED_2_5GB_FULL);
+ }
+diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+index 9724ffb165189..e4b50c7781ffa 100644
+--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
++++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+@@ -3405,6 +3405,9 @@ static s32 ixgbe_reset_hw_X550em(struct ixgbe_hw *hw)
+ /* flush pending Tx transactions */
+ ixgbe_clear_tx_pending(hw);
+
++ /* set MDIO speed before talking to the PHY in case it's the 1st time */
++ ixgbe_set_mdio_speed(hw);
++
+ /* PHY ops must be identified and initialized prior to reset */
+ status = hw->phy.ops.init(hw);
+ if (status == IXGBE_ERR_SFP_NOT_SUPPORTED ||
+diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+index b1d22e4d5ec9c..b399b9c147172 100644
+--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
++++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+@@ -201,26 +201,28 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)
+ }
+
+ static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring,
+- struct ixgbe_rx_buffer *bi)
++ const struct xdp_buff *xdp)
+ {
+- unsigned int metasize = bi->xdp->data - bi->xdp->data_meta;
+- unsigned int datasize = bi->xdp->data_end - bi->xdp->data;
++ unsigned int totalsize = xdp->data_end - xdp->data_meta;
++ unsigned int metasize = xdp->data - xdp->data_meta;
+ struct sk_buff *skb;
+
++ net_prefetch(xdp->data_meta);
++
+ /* allocate a skb to store the frags */
+- skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
+- bi->xdp->data_end - bi->xdp->data_hard_start,
++ skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize,
+ GFP_ATOMIC | __GFP_NOWARN);
+ if (unlikely(!skb))
+ return NULL;
+
+- skb_reserve(skb, bi->xdp->data - bi->xdp->data_hard_start);
+- memcpy(__skb_put(skb, datasize), bi->xdp->data, datasize);
+- if (metasize)
++ memcpy(__skb_put(skb, totalsize), xdp->data_meta,
++ ALIGN(totalsize, sizeof(long)));
++
++ if (metasize) {
+ skb_metadata_set(skb, metasize);
++ __skb_pull(skb, metasize);
++ }
+
+- xsk_buff_free(bi->xdp);
+- bi->xdp = NULL;
+ return skb;
+ }
+
+@@ -311,12 +313,15 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
+ }
+
+ /* XDP_PASS path */
+- skb = ixgbe_construct_skb_zc(rx_ring, bi);
++ skb = ixgbe_construct_skb_zc(rx_ring, bi->xdp);
+ if (!skb) {
+ rx_ring->rx_stats.alloc_rx_buff_failed++;
+ break;
+ }
+
++ xsk_buff_free(bi->xdp);
++ bi->xdp = NULL;
++
+ cleaned_count++;
+ ixgbe_inc_ntc(rx_ring);
+
+@@ -388,12 +393,14 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)
+ u32 cmd_type;
+
+ while (budget-- > 0) {
+- if (unlikely(!ixgbe_desc_unused(xdp_ring)) ||
+- !netif_carrier_ok(xdp_ring->netdev)) {
++ if (unlikely(!ixgbe_desc_unused(xdp_ring))) {
+ work_done = false;
+ break;
+ }
+
++ if (!netif_carrier_ok(xdp_ring->netdev))
++ break;
++
+ if (!xsk_tx_peek_desc(pool, &desc))
+ break;
+
+diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+index c714e1ecd3089..7ef2e1241a76e 100644
+--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
++++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+@@ -1984,14 +1984,15 @@ static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter,
+ if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX)
+ return;
+
+- set_ring_build_skb_enabled(rx_ring);
++ if (PAGE_SIZE < 8192)
++ if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB)
++ set_ring_uses_large_buffer(rx_ring);
+
+- if (PAGE_SIZE < 8192) {
+- if (max_frame <= IXGBEVF_MAX_FRAME_BUILD_SKB)
+- return;
++ /* 82599 can't rely on RXDCTL.RLPML to restrict the size of the frame */
++ if (adapter->hw.mac.type == ixgbe_mac_82599_vf && !ring_uses_large_buffer(rx_ring))
++ return;
+
+- set_ring_uses_large_buffer(rx_ring);
+- }
++ set_ring_build_skb_enabled(rx_ring);
+ }
+
+ /**
+diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c
+index fb78f17d734fe..b02f796b5422f 100644
+--- a/drivers/net/ethernet/lantiq_xrx200.c
++++ b/drivers/net/ethernet/lantiq_xrx200.c
+@@ -209,7 +209,7 @@ static int xrx200_hw_receive(struct xrx200_chan *ch)
+ skb->protocol = eth_type_trans(skb, net_dev);
+ netif_receive_skb(skb);
+ net_dev->stats.rx_packets++;
+- net_dev->stats.rx_bytes += len - ETH_FCS_LEN;
++ net_dev->stats.rx_bytes += len;
+
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/litex/Kconfig b/drivers/net/ethernet/litex/Kconfig
+index 63bf01d28f0cf..04345b929d8e5 100644
+--- a/drivers/net/ethernet/litex/Kconfig
++++ b/drivers/net/ethernet/litex/Kconfig
+@@ -17,7 +17,7 @@ if NET_VENDOR_LITEX
+
+ config LITEX_LITEETH
+ tristate "LiteX Ethernet support"
+- depends on OF_NET
++ depends on OF && HAS_IOMEM
+ help
+ If you wish to compile a kernel for hardware with a LiteX LiteEth
+ device then you should answer Y to this.
+diff --git a/drivers/net/ethernet/litex/litex_liteeth.c b/drivers/net/ethernet/litex/litex_liteeth.c
+index a9bdbf0dcfe1e..5bb1cc8a2ce13 100644
+--- a/drivers/net/ethernet/litex/litex_liteeth.c
++++ b/drivers/net/ethernet/litex/litex_liteeth.c
+@@ -289,7 +289,6 @@ static int liteeth_remove(struct platform_device *pdev)
+ struct net_device *netdev = platform_get_drvdata(pdev);
+
+ unregister_netdev(netdev);
+- free_netdev(netdev);
+
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c
+index 28d5ad296646a..90fd5588e20dd 100644
+--- a/drivers/net/ethernet/marvell/mv643xx_eth.c
++++ b/drivers/net/ethernet/marvell/mv643xx_eth.c
+@@ -2700,6 +2700,16 @@ MODULE_DEVICE_TABLE(of, mv643xx_eth_shared_ids);
+
+ static struct platform_device *port_platdev[3];
+
++static void mv643xx_eth_shared_of_remove(void)
++{
++ int n;
++
++ for (n = 0; n < 3; n++) {
++ platform_device_del(port_platdev[n]);
++ port_platdev[n] = NULL;
++ }
++}
++
+ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev,
+ struct device_node *pnp)
+ {
+@@ -2736,7 +2746,9 @@ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev,
+ return -EINVAL;
+ }
+
+- of_get_mac_address(pnp, ppd.mac_addr);
++ ret = of_get_mac_address(pnp, ppd.mac_addr);
++ if (ret == -EPROBE_DEFER)
++ return ret;
+
+ mv643xx_eth_property(pnp, "tx-queue-size", ppd.tx_queue_size);
+ mv643xx_eth_property(pnp, "tx-sram-addr", ppd.tx_sram_addr);
+@@ -2800,21 +2812,13 @@ static int mv643xx_eth_shared_of_probe(struct platform_device *pdev)
+ ret = mv643xx_eth_shared_of_add_port(pdev, pnp);
+ if (ret) {
+ of_node_put(pnp);
++ mv643xx_eth_shared_of_remove();
+ return ret;
+ }
+ }
+ return 0;
+ }
+
+-static void mv643xx_eth_shared_of_remove(void)
+-{
+- int n;
+-
+- for (n = 0; n < 3; n++) {
+- platform_device_del(port_platdev[n]);
+- port_platdev[n] = NULL;
+- }
+-}
+ #else
+ static inline int mv643xx_eth_shared_of_probe(struct platform_device *pdev)
+ {
+diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c
+index 62a97c46fba05..ef878973b8597 100644
+--- a/drivers/net/ethernet/marvell/mvmdio.c
++++ b/drivers/net/ethernet/marvell/mvmdio.c
+@@ -429,12 +429,14 @@ static const struct of_device_id orion_mdio_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, orion_mdio_match);
+
++#ifdef CONFIG_ACPI
+ static const struct acpi_device_id orion_mdio_acpi_match[] = {
+ { "MRVL0100", BUS_TYPE_SMI },
+ { "MRVL0101", BUS_TYPE_XSMI },
+ { },
+ };
+ MODULE_DEVICE_TABLE(acpi, orion_mdio_acpi_match);
++#endif
+
+ static struct platform_driver orion_mdio_driver = {
+ .probe = orion_mdio_probe,
+diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+index d5c92e43f89e6..2baa909290b3b 100644
+--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
++++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+@@ -1605,7 +1605,7 @@ static void mvpp22_gop_fca_set_periodic_timer(struct mvpp2_port *port)
+ mvpp22_gop_fca_enable_periodic(port, true);
+ }
+
+-static int mvpp22_gop_init(struct mvpp2_port *port)
++static int mvpp22_gop_init(struct mvpp2_port *port, phy_interface_t interface)
+ {
+ struct mvpp2 *priv = port->priv;
+ u32 val;
+@@ -1613,7 +1613,7 @@ static int mvpp22_gop_init(struct mvpp2_port *port)
+ if (!priv->sysctrl_base)
+ return 0;
+
+- switch (port->phy_interface) {
++ switch (interface) {
+ case PHY_INTERFACE_MODE_RGMII:
+ case PHY_INTERFACE_MODE_RGMII_ID:
+ case PHY_INTERFACE_MODE_RGMII_RXID:
+@@ -1743,15 +1743,15 @@ static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
+ * lanes by the physical layer. This is why configurations like
+ * "PPv2 (2500BaseX) - COMPHY (2500SGMII)" are valid.
+ */
+-static int mvpp22_comphy_init(struct mvpp2_port *port)
++static int mvpp22_comphy_init(struct mvpp2_port *port,
++ phy_interface_t interface)
+ {
+ int ret;
+
+ if (!port->comphy)
+ return 0;
+
+- ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET,
+- port->phy_interface);
++ ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET, interface);
+ if (ret)
+ return ret;
+
+@@ -2172,7 +2172,8 @@ static void mvpp22_pcs_reset_assert(struct mvpp2_port *port)
+ writel(val & ~MVPP22_XPCS_CFG0_RESET_DIS, xpcs + MVPP22_XPCS_CFG0);
+ }
+
+-static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port)
++static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port,
++ phy_interface_t interface)
+ {
+ struct mvpp2 *priv = port->priv;
+ void __iomem *mpcs, *xpcs;
+@@ -2184,7 +2185,7 @@ static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port)
+ mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id);
+ xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id);
+
+- switch (port->phy_interface) {
++ switch (interface) {
+ case PHY_INTERFACE_MODE_10GBASER:
+ val = readl(mpcs + MVPP22_MPCS_CLK_RESET);
+ val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX |
+@@ -2959,11 +2960,11 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
+ mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size);
+
+ if (priv->percpu_pools) {
+- err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0);
++ err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->logic_rxq, 0);
+ if (err < 0)
+ goto err_free_dma;
+
+- err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0);
++ err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->logic_rxq, 0);
+ if (err < 0)
+ goto err_unregister_rxq_short;
+
+@@ -4529,7 +4530,8 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
+ return rx_done;
+ }
+
+-static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
++static void mvpp22_mode_reconfigure(struct mvpp2_port *port,
++ phy_interface_t interface)
+ {
+ u32 ctrl3;
+
+@@ -4540,18 +4542,18 @@ static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
+ mvpp22_pcs_reset_assert(port);
+
+ /* comphy reconfiguration */
+- mvpp22_comphy_init(port);
++ mvpp22_comphy_init(port, interface);
+
+ /* gop reconfiguration */
+- mvpp22_gop_init(port);
++ mvpp22_gop_init(port, interface);
+
+- mvpp22_pcs_reset_deassert(port);
++ mvpp22_pcs_reset_deassert(port, interface);
+
+ if (mvpp2_port_supports_xlg(port)) {
+ ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG);
+ ctrl3 &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
+
+- if (mvpp2_is_xlg(port->phy_interface))
++ if (mvpp2_is_xlg(interface))
+ ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_10G;
+ else
+ ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
+@@ -4559,7 +4561,7 @@ static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
+ writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG);
+ }
+
+- if (mvpp2_port_supports_xlg(port) && mvpp2_is_xlg(port->phy_interface))
++ if (mvpp2_port_supports_xlg(port) && mvpp2_is_xlg(interface))
+ mvpp2_xlg_max_rx_size_set(port);
+ else
+ mvpp2_gmac_max_rx_size_set(port);
+@@ -4579,7 +4581,7 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
+ mvpp2_interrupts_enable(port);
+
+ if (port->priv->hw_version >= MVPP22)
+- mvpp22_mode_reconfigure(port);
++ mvpp22_mode_reconfigure(port, port->phy_interface);
+
+ if (port->phylink) {
+ phylink_start(port->phylink);
+@@ -5015,11 +5017,13 @@ static int mvpp2_change_mtu(struct net_device *dev, int mtu)
+ mtu = ALIGN(MVPP2_RX_PKT_SIZE(mtu), 8);
+ }
+
++ if (port->xdp_prog && mtu > MVPP2_MAX_RX_BUF_SIZE) {
++ netdev_err(dev, "Illegal MTU value %d (> %d) for XDP mode\n",
++ mtu, (int)MVPP2_MAX_RX_BUF_SIZE);
++ return -EINVAL;
++ }
++
+ if (MVPP2_RX_PKT_SIZE(mtu) > MVPP2_BM_LONG_PKT_SIZE) {
+- if (port->xdp_prog) {
+- netdev_err(dev, "Jumbo frames are not supported with XDP\n");
+- return -EINVAL;
+- }
+ if (priv->percpu_pools) {
+ netdev_warn(dev, "mtu %d too high, switching to shared buffers", mtu);
+ mvpp2_bm_switch_buffers(priv, false);
+@@ -5305,8 +5309,8 @@ static int mvpp2_xdp_setup(struct mvpp2_port *port, struct netdev_bpf *bpf)
+ bool running = netif_running(port->dev);
+ bool reset = !prog != !port->xdp_prog;
+
+- if (port->dev->mtu > ETH_DATA_LEN) {
+- NL_SET_ERR_MSG_MOD(bpf->extack, "XDP is not supported with jumbo frames enabled");
++ if (port->dev->mtu > MVPP2_MAX_RX_BUF_SIZE) {
++ NL_SET_ERR_MSG_MOD(bpf->extack, "MTU too large for XDP");
+ return -EOPNOTSUPP;
+ }
+
+@@ -6477,6 +6481,9 @@ static int mvpp2__mac_prepare(struct phylink_config *config, unsigned int mode,
+ mvpp22_gop_mask_irq(port);
+
+ phy_power_off(port->comphy);
++
++ /* Reconfigure the serdes lanes */
++ mvpp22_mode_reconfigure(port, interface);
+ }
+ }
+
+@@ -6531,9 +6538,6 @@ static int mvpp2_mac_finish(struct phylink_config *config, unsigned int mode,
+ port->phy_interface != interface) {
+ port->phy_interface = interface;
+
+- /* Reconfigure the serdes lanes */
+- mvpp22_mode_reconfigure(port);
+-
+ /* Unmask interrupts */
+ mvpp22_gop_unmask_irq(port);
+ }
+@@ -6960,7 +6964,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
+ * driver does this, we can remove this code.
+ */
+ if (port->comphy) {
+- err = mvpp22_comphy_init(port);
++ err = mvpp22_comphy_init(port, port->phy_interface);
+ if (err == 0)
+ phy_power_off(port->comphy);
+ }
+@@ -7453,7 +7457,7 @@ static int mvpp2_probe(struct platform_device *pdev)
+
+ shared = num_present_cpus() - priv->nthreads;
+ if (shared > 0)
+- bitmap_fill(&priv->lock_map,
++ bitmap_set(&priv->lock_map, 0,
+ min_t(int, shared, MVPP2_MAX_THREADS));
+
+ for (i = 0; i < MVPP2_MAX_THREADS; i++) {
+diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig
+index 3f982ccf2c85f..639893d870550 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/Kconfig
++++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig
+@@ -31,6 +31,7 @@ config NDC_DIS_DYNAMIC_CACHING
+ config OCTEONTX2_PF
+ tristate "Marvell OcteonTX2 NIC Physical Function driver"
+ select OCTEONTX2_MBOX
++ select NET_DEVLINK
+ depends on (64BIT && COMPILE_TEST) || ARM64
+ depends on PCI
+ depends on PTP_1588_CLOCK_OPTIONAL
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+index 34a089b71e554..6b335139abe7f 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+@@ -838,9 +838,6 @@ void cgx_lmac_ptp_config(void *cgxd, int lmac_id, bool enable)
+ if (!cgx)
+ return;
+
+- if (is_dev_rpm(cgx))
+- return;
+-
+ if (enable) {
+ /* Enable inbound PTP timestamping */
+ cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
+@@ -1545,9 +1542,11 @@ static int cgx_lmac_exit(struct cgx *cgx)
+ static void cgx_populate_features(struct cgx *cgx)
+ {
+ if (is_dev_rpm(cgx))
+- cgx->hw_features = (RVU_MAC_RPM | RVU_LMAC_FEAT_FC);
++ cgx->hw_features = (RVU_LMAC_FEAT_DMACF | RVU_MAC_RPM |
++ RVU_LMAC_FEAT_FC | RVU_LMAC_FEAT_PTP);
+ else
+- cgx->hw_features = (RVU_LMAC_FEAT_FC | RVU_LMAC_FEAT_PTP);
++ cgx->hw_features = (RVU_LMAC_FEAT_FC | RVU_LMAC_FEAT_HIGIG2 |
++ RVU_LMAC_FEAT_PTP | RVU_LMAC_FEAT_DMACF);
+ }
+
+ static struct mac_ops cgx_mac_ops = {
+@@ -1571,6 +1570,9 @@ static struct mac_ops cgx_mac_ops = {
+ .mac_get_pause_frm_status = cgx_lmac_get_pause_frm_status,
+ .mac_enadis_pause_frm = cgx_lmac_enadis_pause_frm,
+ .mac_pause_frm_config = cgx_lmac_pause_frm_config,
++ .mac_enadis_ptp_config = cgx_lmac_ptp_config,
++ .mac_rx_tx_enable = cgx_lmac_rx_tx_enable,
++ .mac_tx_enable = cgx_lmac_tx_enable,
+ };
+
+ static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
+index c38306b3384a7..b33e7d1d0851c 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
+@@ -102,6 +102,14 @@ struct mac_ops {
+ void (*mac_pause_frm_config)(void *cgxd,
+ int lmac_id,
+ bool enable);
++
++ /* Enable/Disable Inbound PTP */
++ void (*mac_enadis_ptp_config)(void *cgxd,
++ int lmac_id,
++ bool enable);
++
++ int (*mac_rx_tx_enable)(void *cgxd, int lmac_id, bool enable);
++ int (*mac_tx_enable)(void *cgxd, int lmac_id, bool enable);
+ };
+
+ struct cgx {
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+index 154877706a0e1..c6643c7db1fc4 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+@@ -84,7 +84,7 @@ struct mbox_msghdr {
+ #define OTX2_MBOX_REQ_SIG (0xdead)
+ #define OTX2_MBOX_RSP_SIG (0xbeef)
+ u16 sig; /* Signature, for validating corrupted msgs */
+-#define OTX2_MBOX_VERSION (0x0009)
++#define OTX2_MBOX_VERSION (0x000a)
+ u16 ver; /* Version of msg's structure for this ID */
+ u16 next_msgoff; /* Offset of next msg within mailbox region */
+ int rc; /* Msg process'ed response code */
+@@ -154,23 +154,23 @@ M(CGX_PTP_RX_ENABLE, 0x20C, cgx_ptp_rx_enable, msg_req, msg_rsp) \
+ M(CGX_PTP_RX_DISABLE, 0x20D, cgx_ptp_rx_disable, msg_req, msg_rsp) \
+ M(CGX_CFG_PAUSE_FRM, 0x20E, cgx_cfg_pause_frm, cgx_pause_frm_cfg, \
+ cgx_pause_frm_cfg) \
+-M(CGX_FEC_SET, 0x210, cgx_set_fec_param, fec_mode, fec_mode) \
+-M(CGX_FEC_STATS, 0x211, cgx_fec_stats, msg_req, cgx_fec_stats_rsp) \
+-M(CGX_GET_PHY_FEC_STATS, 0x212, cgx_get_phy_fec_stats, msg_req, msg_rsp) \
+-M(CGX_FW_DATA_GET, 0x213, cgx_get_aux_link_info, msg_req, cgx_fw_data) \
+-M(CGX_SET_LINK_MODE, 0x214, cgx_set_link_mode, cgx_set_link_mode_req,\
+- cgx_set_link_mode_rsp) \
+-M(CGX_FEATURES_GET, 0x215, cgx_features_get, msg_req, \
+- cgx_features_info_msg) \
+-M(RPM_STATS, 0x216, rpm_stats, msg_req, rpm_stats_rsp) \
+-M(CGX_MAC_ADDR_ADD, 0x217, cgx_mac_addr_add, cgx_mac_addr_add_req, \
+- cgx_mac_addr_add_rsp) \
+-M(CGX_MAC_ADDR_DEL, 0x218, cgx_mac_addr_del, cgx_mac_addr_del_req, \
++M(CGX_FW_DATA_GET, 0x20F, cgx_get_aux_link_info, msg_req, cgx_fw_data) \
++M(CGX_FEC_SET, 0x210, cgx_set_fec_param, fec_mode, fec_mode) \
++M(CGX_MAC_ADDR_ADD, 0x211, cgx_mac_addr_add, cgx_mac_addr_add_req, \
++ cgx_mac_addr_add_rsp) \
++M(CGX_MAC_ADDR_DEL, 0x212, cgx_mac_addr_del, cgx_mac_addr_del_req, \
+ msg_rsp) \
+-M(CGX_MAC_MAX_ENTRIES_GET, 0x219, cgx_mac_max_entries_get, msg_req, \
++M(CGX_MAC_MAX_ENTRIES_GET, 0x213, cgx_mac_max_entries_get, msg_req, \
+ cgx_max_dmac_entries_get_rsp) \
+-M(CGX_MAC_ADDR_RESET, 0x21A, cgx_mac_addr_reset, msg_req, msg_rsp) \
+-M(CGX_MAC_ADDR_UPDATE, 0x21B, cgx_mac_addr_update, cgx_mac_addr_update_req, \
++M(CGX_FEC_STATS, 0x217, cgx_fec_stats, msg_req, cgx_fec_stats_rsp) \
++M(CGX_SET_LINK_MODE, 0x218, cgx_set_link_mode, cgx_set_link_mode_req,\
++ cgx_set_link_mode_rsp) \
++M(CGX_GET_PHY_FEC_STATS, 0x219, cgx_get_phy_fec_stats, msg_req, msg_rsp) \
++M(CGX_FEATURES_GET, 0x21B, cgx_features_get, msg_req, \
++ cgx_features_info_msg) \
++M(RPM_STATS, 0x21C, rpm_stats, msg_req, rpm_stats_rsp) \
++M(CGX_MAC_ADDR_RESET, 0x21D, cgx_mac_addr_reset, msg_req, msg_rsp) \
++M(CGX_MAC_ADDR_UPDATE, 0x21E, cgx_mac_addr_update, cgx_mac_addr_update_req, \
+ msg_rsp) \
+ /* NPA mbox IDs (range 0x400 - 0x5FF) */ \
+ M(NPA_LF_ALLOC, 0x400, npa_lf_alloc, \
+@@ -229,6 +229,8 @@ M(NPC_DELETE_FLOW, 0x600e, npc_delete_flow, \
+ M(NPC_MCAM_READ_ENTRY, 0x600f, npc_mcam_read_entry, \
+ npc_mcam_read_entry_req, \
+ npc_mcam_read_entry_rsp) \
++M(NPC_SET_PKIND, 0x6010, npc_set_pkind, \
++ npc_set_pkind, msg_rsp) \
+ M(NPC_MCAM_READ_BASE_RULE, 0x6011, npc_read_base_steer_rule, \
+ msg_req, npc_mcam_read_base_rule_rsp) \
+ M(NPC_MCAM_GET_STATS, 0x6012, npc_mcam_entry_stats, \
+@@ -575,10 +577,13 @@ struct cgx_mac_addr_update_req {
+ };
+
+ #define RVU_LMAC_FEAT_FC BIT_ULL(0) /* pause frames */
+-#define RVU_LMAC_FEAT_PTP BIT_ULL(1) /* precision time protocol */
+-#define RVU_MAC_VERSION BIT_ULL(2)
+-#define RVU_MAC_CGX BIT_ULL(3)
+-#define RVU_MAC_RPM BIT_ULL(4)
++#define RVU_LMAC_FEAT_HIGIG2 BIT_ULL(1)
++ /* flow control from physical link higig2 messages */
++#define RVU_LMAC_FEAT_PTP BIT_ULL(2) /* precison time protocol */
++#define RVU_LMAC_FEAT_DMACF BIT_ULL(3) /* DMAC FILTER */
++#define RVU_MAC_VERSION BIT_ULL(4)
++#define RVU_MAC_CGX BIT_ULL(5)
++#define RVU_MAC_RPM BIT_ULL(6)
+
+ struct cgx_features_info_msg {
+ struct mbox_msghdr hdr;
+@@ -593,6 +598,22 @@ struct rpm_stats_rsp {
+ u64 tx_stats[RPM_TX_STATS_COUNT];
+ };
+
++struct npc_set_pkind {
++ struct mbox_msghdr hdr;
++#define OTX2_PRIV_FLAGS_DEFAULT BIT_ULL(0)
++#define OTX2_PRIV_FLAGS_CUSTOM BIT_ULL(63)
++ u64 mode;
++#define PKIND_TX BIT_ULL(0)
++#define PKIND_RX BIT_ULL(1)
++ u8 dir;
++ u8 pkind; /* valid only in case custom flag */
++ u8 var_len_off; /* Offset of custom header length field.
++ * Valid only for pkind NPC_RX_CUSTOM_PRE_L2_PKIND
++ */
++ u8 var_len_off_mask; /* Mask for length with in offset */
++ u8 shift_dir; /* shift direction to get length of the header at var_len_off */
++};
++
+ /* NPA mbox message formats */
+
+ /* NPA mailbox error codes
+@@ -698,6 +719,9 @@ enum nix_af_status {
+ NIX_AF_ERR_INVALID_BANDPROF = -426,
+ NIX_AF_ERR_IPOLICER_NOTSUPP = -427,
+ NIX_AF_ERR_BANDPROF_INVAL_REQ = -428,
++ NIX_AF_ERR_CQ_CTX_WRITE_ERR = -429,
++ NIX_AF_ERR_AQ_CTX_RETRY_WRITE = -430,
++ NIX_AF_ERR_LINK_CREDITS = -431,
+ };
+
+ /* For NIX RX vtag action */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
+index 3a819b24accc6..6e1192f526089 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
+@@ -31,9 +31,9 @@ enum npc_kpu_la_ltype {
+ NPC_LT_LA_HIGIG2_ETHER,
+ NPC_LT_LA_IH_NIX_HIGIG2_ETHER,
+ NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_LT_LA_CH_LEN_90B_ETHER,
+ NPC_LT_LA_CPT_HDR,
+ NPC_LT_LA_CUSTOM_L2_24B_ETHER,
++ NPC_LT_LA_CUSTOM_PRE_L2_ETHER,
+ NPC_LT_LA_CUSTOM0 = 0xE,
+ NPC_LT_LA_CUSTOM1 = 0xF,
+ };
+@@ -148,10 +148,11 @@ enum npc_kpu_lh_ltype {
+ * Software assigns pkind for each incoming port such as CGX
+ * Ethernet interfaces, LBK interfaces, etc.
+ */
+-#define NPC_UNRESERVED_PKIND_COUNT NPC_RX_VLAN_EXDSA_PKIND
++#define NPC_UNRESERVED_PKIND_COUNT NPC_RX_CUSTOM_PRE_L2_PKIND
+
+ enum npc_pkind_type {
+ NPC_RX_LBK_PKIND = 0ULL,
++ NPC_RX_CUSTOM_PRE_L2_PKIND = 55ULL,
+ NPC_RX_VLAN_EXDSA_PKIND = 56ULL,
+ NPC_RX_CHLEN24B_PKIND = 57ULL,
+ NPC_RX_CPT_HDR_PKIND,
+@@ -162,6 +163,10 @@ enum npc_pkind_type {
+ NPC_TX_DEF_PKIND, /* NIX-TX PKIND */
+ };
+
++enum npc_interface_type {
++ NPC_INTF_MODE_DEF,
++};
++
+ /* list of known and supported fields in packet header and
+ * fields present in key structure.
+ */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
+index 588822a0cf21e..695123e32ba85 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
+@@ -176,9 +176,8 @@ enum npc_kpu_parser_state {
+ NPC_S_KPU1_EXDSA,
+ NPC_S_KPU1_HIGIG2,
+ NPC_S_KPU1_IH_NIX_HIGIG2,
+- NPC_S_KPU1_CUSTOM_L2_90B,
++ NPC_S_KPU1_CUSTOM_PRE_L2,
+ NPC_S_KPU1_CPT_HDR,
+- NPC_S_KPU1_CUSTOM_L2_24B,
+ NPC_S_KPU1_VLAN_EXDSA,
+ NPC_S_KPU2_CTAG,
+ NPC_S_KPU2_CTAG2,
+@@ -187,7 +186,8 @@ enum npc_kpu_parser_state {
+ NPC_S_KPU2_ETAG,
+ NPC_S_KPU2_PREHEADER,
+ NPC_S_KPU2_EXDSA,
+- NPC_S_KPU2_NGIO,
++ NPC_S_KPU2_CPT_CTAG,
++ NPC_S_KPU2_CPT_QINQ,
+ NPC_S_KPU3_CTAG,
+ NPC_S_KPU3_STAG,
+ NPC_S_KPU3_QINQ,
+@@ -212,6 +212,7 @@ enum npc_kpu_parser_state {
+ NPC_S_KPU5_NSH,
+ NPC_S_KPU5_CPT_IP,
+ NPC_S_KPU5_CPT_IP6,
++ NPC_S_KPU5_NGIO,
+ NPC_S_KPU6_IP6_EXT,
+ NPC_S_KPU6_IP6_HOP_DEST,
+ NPC_S_KPU6_IP6_ROUT,
+@@ -979,8 +980,8 @@ static struct npc_kpu_profile_action ikpu_action_entries[] = {
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 12, 16, 20, 0, 0,
+- NPC_S_KPU1_ETHER, 0, 0,
+- NPC_LID_LA, NPC_LT_NA,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0, 1,
++ NPC_LID_LA, NPC_LT_LA_CUSTOM_PRE_L2_ETHER,
+ 0,
+ 0, 0, 0, 0,
+
+@@ -996,27 +997,27 @@ static struct npc_kpu_profile_action ikpu_action_entries[] = {
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 36, 40, 44, 0, 0,
+- NPC_S_KPU1_CUSTOM_L2_24B, 0, 0,
+- NPC_LID_LA, NPC_LT_NA,
++ 12, 16, 20, 0, 0,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 24, 1,
++ NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+ 0,
+ 0, 0, 0, 0,
+
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 40, 54, 58, 0, 0,
+- NPC_S_KPU1_CPT_HDR, 0, 0,
++ 12, 16, 20, 0, 0,
++ NPC_S_KPU1_CPT_HDR, 40, 0,
+ NPC_LID_LA, NPC_LT_NA,
+ 0,
+- 0, 0, 0, 0,
++ 7, 7, 0, 0,
+
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 102, 106, 110, 0, 0,
+- NPC_S_KPU1_CUSTOM_L2_90B, 0, 0,
+- NPC_LID_LA, NPC_LT_NA,
++ 12, 16, 20, 0, 0,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 90, 1,
++ NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+ 0,
+ 0, 0, 0, 0,
+
+@@ -1116,15 +1117,6 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ 0x0000,
+ },
+- {
+- NPC_S_KPU1_ETHER, 0xff,
+- NPC_ETYPE_CTAG,
+- 0xffff,
+- NPC_ETYPE_NGIO,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- },
+ {
+ NPC_S_KPU1_ETHER, 0xff,
+ NPC_ETYPE_CTAG,
+@@ -1711,7 +1703,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_IP,
+ 0xffff,
+ 0x0000,
+@@ -1720,7 +1712,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_IP6,
+ 0xffff,
+ 0x0000,
+@@ -1729,7 +1721,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_ARP,
+ 0xffff,
+ 0x0000,
+@@ -1738,7 +1730,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_RARP,
+ 0xffff,
+ 0x0000,
+@@ -1747,7 +1739,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_PTP,
+ 0xffff,
+ 0x0000,
+@@ -1756,7 +1748,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_FCOE,
+ 0xffff,
+ 0x0000,
+@@ -1765,7 +1757,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_CTAG,
+ 0xffff,
+ NPC_ETYPE_CTAG,
+@@ -1774,7 +1766,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_CTAG,
+ 0xffff,
+ 0x0000,
+@@ -1783,7 +1775,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_SBTAG,
+ 0xffff,
+ 0x0000,
+@@ -1792,7 +1784,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_QINQ,
+ 0xffff,
+ 0x0000,
+@@ -1801,7 +1793,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_ETAG,
+ 0xffff,
+ 0x0000,
+@@ -1810,7 +1802,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_MPLSU,
+ 0xffff,
+ 0x0000,
+@@ -1819,7 +1811,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_MPLSM,
+ 0xffff,
+ 0x0000,
+@@ -1828,7 +1820,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ NPC_ETYPE_NSH,
+ 0xffff,
+ 0x0000,
+@@ -1837,7 +1829,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_90B, 0xff,
++ NPC_S_KPU1_CUSTOM_PRE_L2, 0xff,
+ 0x0000,
+ 0x0000,
+ 0x0000,
+@@ -1847,150 +1839,24 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ },
+ {
+ NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+ NPC_ETYPE_IP,
+ 0xffff,
+ 0x0000,
+ 0x0000,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- NPC_ETYPE_IP6,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- NPC_ETYPE_CTAG,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- NPC_ETYPE_QINQ,
+- 0xffff,
+ 0x0000,
+ 0x0000,
+ },
+ {
+ NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- NPC_ETYPE_IP,
+- 0xffff,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+ NPC_ETYPE_IP6,
+ 0xffff,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0xffff,
+- 0x0000,
+ 0x0000,
+- NPC_ETYPE_CTAG,
+- 0xffff,
+- },
+- {
+- NPC_S_KPU1_CPT_HDR, 0xff,
+ 0x0000,
+- 0xffff,
+ 0x0000,
+ 0x0000,
+- NPC_ETYPE_QINQ,
+- 0xffff,
+ },
+ {
+ NPC_S_KPU1_CPT_HDR, 0xff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_IP,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_IP6,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_ARP,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_RARP,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_PTP,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_FCOE,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_CTAG,
+- 0xffff,
+- NPC_ETYPE_CTAG,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+ NPC_ETYPE_CTAG,
+ 0xffff,
+ 0x0000,
+@@ -1999,16 +1865,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_SBTAG,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
++ NPC_S_KPU1_CPT_HDR, 0xff,
+ NPC_ETYPE_QINQ,
+ 0xffff,
+ 0x0000,
+@@ -2016,51 +1873,6 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+ 0x0000,
+ 0x0000,
+ },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_ETAG,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_MPLSU,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_MPLSM,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- NPC_ETYPE_NSH,
+- 0xffff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+- {
+- NPC_S_KPU1_CUSTOM_L2_24B, 0xff,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- 0x0000,
+- },
+ {
+ NPC_S_KPU1_VLAN_EXDSA, 0xff,
+ NPC_ETYPE_CTAG,
+@@ -2165,6 +1977,15 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = {
+ 0x0000,
+ 0x0000,
+ },
++ {
++ NPC_S_KPU2_CTAG, 0xff,
++ NPC_ETYPE_NGIO,
++ 0xffff,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ },
+ {
+ NPC_S_KPU2_CTAG, 0xff,
+ NPC_ETYPE_PPPOE,
+@@ -3057,11 +2878,38 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = {
+ 0x0000,
+ },
+ {
+- NPC_S_KPU2_NGIO, 0xff,
++ NPC_S_KPU2_CPT_CTAG, 0xff,
++ NPC_ETYPE_IP,
++ 0xffff,
+ 0x0000,
+ 0x0000,
+ 0x0000,
+ 0x0000,
++ },
++ {
++ NPC_S_KPU2_CPT_CTAG, 0xff,
++ NPC_ETYPE_IP6,
++ 0xffff,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ },
++ {
++ NPC_S_KPU2_CPT_QINQ, 0xff,
++ NPC_ETYPE_CTAG,
++ 0xffff,
++ NPC_ETYPE_IP,
++ 0xffff,
++ 0x0000,
++ 0x0000,
++ },
++ {
++ NPC_S_KPU2_CPT_QINQ, 0xff,
++ NPC_ETYPE_CTAG,
++ 0xffff,
++ NPC_ETYPE_IP6,
++ 0xffff,
+ 0x0000,
+ 0x0000,
+ },
+@@ -5348,6 +5196,15 @@ static struct npc_kpu_profile_cam kpu5_cam_entries[] = {
+ 0x0000,
+ 0x0000,
+ },
++ {
++ NPC_S_KPU5_NGIO, 0xff,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ 0x0000,
++ },
+ {
+ NPC_S_NA, 0X00,
+ 0x0000,
+@@ -8642,14 +8499,6 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
+ 0,
+ 0, 0, 0, 0,
+ },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 8, 12, 0, 0, 0,
+- NPC_S_KPU2_NGIO, 12, 1,
+- NPC_LID_LA, NPC_LT_LA_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 12, 0, 0, 0,
+@@ -9192,159 +9041,127 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 0, 6, 3, 0,
+- NPC_S_KPU5_IP, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_IP, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 6, 0, 0, 3, 0,
+- NPC_S_KPU5_IP6, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_IP6, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 0, 0, 0, 3, 0,
+- NPC_S_KPU5_ARP, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_ARP, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 0, 0, 0, 3, 0,
+- NPC_S_KPU5_RARP, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_RARP, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 0, 0, 0, 3, 0,
+- NPC_S_KPU5_PTP, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_PTP, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 0, 0, 0, 3, 0,
+- NPC_S_KPU5_FCOE, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
++ NPC_S_KPU5_FCOE, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 12, 0, 0, 0,
+- NPC_S_KPU2_CTAG2, 102, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
++ NPC_S_KPU2_CTAG2, 12, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 4, 8, 0, 0, 0,
+- NPC_S_KPU2_CTAG, 102, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
++ NPC_S_KPU2_CTAG, 12, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 4, 8, 22, 0, 0,
+- NPC_S_KPU2_SBTAG, 102, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
++ NPC_S_KPU2_SBTAG, 12, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 4, 8, 0, 0, 0,
+- NPC_S_KPU2_QINQ, 102, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
++ NPC_S_KPU2_QINQ, 12, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 12, 26, 0, 0,
+- NPC_S_KPU2_ETAG, 102, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ETAG,
++ NPC_S_KPU2_ETAG, 12, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 2, 6, 10, 2, 0,
+- NPC_S_KPU4_MPLS, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_L_WITH_MPLS,
++ NPC_S_KPU4_MPLS, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 2, 6, 10, 2, 0,
+- NPC_S_KPU4_MPLS, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_L_WITH_MPLS,
++ NPC_S_KPU4_MPLS, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
++ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 2, 0, 0, 2, 0,
+- NPC_S_KPU4_NSH, 104, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_L_WITH_NSH,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 0, 1,
+- NPC_S_NA, 0, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER,
+- NPC_F_LA_L_UNK_ETYPE,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 8, 0, 6, 3, 0,
+- NPC_S_KPU5_CPT_IP, 56, 1,
+- NPC_LID_LA, NPC_LT_LA_CPT_HDR,
++ NPC_S_KPU4_NSH, 14, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 6, 0, 0, 3, 0,
+- NPC_S_KPU5_CPT_IP6, 56, 1,
+- NPC_LID_LA, NPC_LT_LA_CPT_HDR,
++ 0, 0, 0, 0, 1,
++ NPC_S_NA, 0, 0,
++ NPC_LID_LA, NPC_LT_NA,
+ 0,
+ 0, 0, 0, 0,
+ },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 4, 8, 0, 0, 0,
+- NPC_S_KPU2_CTAG, 54, 1,
+- NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 4, 8, 0, 0, 0,
+- NPC_S_KPU2_QINQ, 54, 1,
+- NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 0, 6, 3, 0,
+- NPC_S_KPU5_CPT_IP, 60, 1,
++ NPC_S_KPU5_CPT_IP, 14, 1,
+ NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+ 0,
+ 0, 0, 0, 0,
+@@ -9352,7 +9169,7 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 6, 0, 0, 3, 0,
+- NPC_S_KPU5_CPT_IP6, 60, 1,
++ NPC_S_KPU5_CPT_IP6, 14, 1,
+ NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+ 0,
+ 0, 0, 0, 0,
+@@ -9360,7 +9177,7 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 4, 8, 0, 0, 0,
+- NPC_S_KPU2_CTAG, 58, 1,
++ NPC_S_KPU2_CPT_CTAG, 12, 1,
+ NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+ NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+ 0, 0, 0, 0,
+@@ -9368,139 +9185,11 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 4, 8, 0, 0, 0,
+- NPC_S_KPU2_QINQ, 58, 1,
++ NPC_S_KPU2_CPT_QINQ, 12, 1,
+ NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+ NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+ 0, 0, 0, 0,
+ },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 0, 1,
+- NPC_S_NA, 0, 1,
+- NPC_LID_LA, NPC_LT_LA_CPT_HDR,
+- NPC_F_LA_L_UNK_ETYPE,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 8, 0, 6, 3, 0,
+- NPC_S_KPU5_IP, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 6, 0, 0, 3, 0,
+- NPC_S_KPU5_IP6, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 3, 0,
+- NPC_S_KPU5_ARP, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 3, 0,
+- NPC_S_KPU5_RARP, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 3, 0,
+- NPC_S_KPU5_PTP, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 3, 0,
+- NPC_S_KPU5_FCOE, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- 0,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 8, 12, 0, 0, 0,
+- NPC_S_KPU2_CTAG2, 36, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 4, 8, 0, 0, 0,
+- NPC_S_KPU2_CTAG, 36, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 4, 8, 22, 0, 0,
+- NPC_S_KPU2_SBTAG, 36, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 4, 8, 0, 0, 0,
+- NPC_S_KPU2_QINQ, 36, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 8, 12, 26, 0, 0,
+- NPC_S_KPU2_ETAG, 36, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ETAG,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 2, 6, 10, 2, 0,
+- NPC_S_KPU4_MPLS, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_L_WITH_MPLS,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 2, 6, 10, 2, 0,
+- NPC_S_KPU4_MPLS, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_L_WITH_MPLS,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 2, 0, 0, 2, 0,
+- NPC_S_KPU4_NSH, 38, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_L_WITH_NSH,
+- 0, 0, 0, 0,
+- },
+- {
+- NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 0, 1,
+- NPC_S_NA, 0, 1,
+- NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER,
+- NPC_F_LA_L_UNK_ETYPE,
+- 0, 0, 0, 0,
+- },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 12, 0, 0, 1, 0,
+@@ -9594,6 +9283,14 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = {
+ 0,
+ 0, 0, 0, 0,
+ },
++ {
++ NPC_ERRLEV_RE, NPC_EC_NOERR,
++ 0, 0, 0, 2, 0,
++ NPC_S_KPU5_NGIO, 6, 1,
++ NPC_LID_LB, NPC_LT_LB_CTAG,
++ 0,
++ 0, 0, 0, 0,
++ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+ 8, 0, 6, 2, 0,
+@@ -10388,12 +10085,36 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = {
+ },
+ {
+ NPC_ERRLEV_RE, NPC_EC_NOERR,
+- 0, 0, 0, 0, 1,
+- NPC_S_NA, 0, 1,
+- NPC_LID_LC, NPC_LT_LC_NGIO,
++ 8, 0, 6, 2, 0,
++ NPC_S_KPU5_CPT_IP, 6, 1,
++ NPC_LID_LB, NPC_LT_LB_CTAG,
++ 0,
++ 0, 0, 0, 0,
++ },
++ {
++ NPC_ERRLEV_RE, NPC_EC_NOERR,
++ 6, 0, 0, 2, 0,
++ NPC_S_KPU5_CPT_IP6, 6, 1,
++ NPC_LID_LB, NPC_LT_LB_CTAG,
+ 0,
+ 0, 0, 0, 0,
+ },
++ {
++ NPC_ERRLEV_RE, NPC_EC_NOERR,
++ 8, 0, 6, 2, 0,
++ NPC_S_KPU5_CPT_IP, 10, 1,
++ NPC_LID_LB, NPC_LT_LB_STAG_QINQ,
++ NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG,
++ 0, 0, 0, 0,
++ },
++ {
++ NPC_ERRLEV_RE, NPC_EC_NOERR,
++ 6, 0, 0, 2, 0,
++ NPC_S_KPU5_CPT_IP6, 10, 1,
++ NPC_LID_LB, NPC_LT_LB_STAG_QINQ,
++ NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG,
++ 0, 0, 0, 0,
++ },
+ {
+ NPC_ERRLEV_LB, NPC_EC_L2_K3,
+ 0, 0, 0, 0, 1,
+@@ -12425,6 +12146,14 @@ static struct npc_kpu_profile_action kpu5_action_entries[] = {
+ 0,
+ 0, 0, 0, 0,
+ },
++ {
++ NPC_ERRLEV_RE, NPC_EC_NOERR,
++ 0, 0, 0, 0, 1,
++ NPC_S_NA, 0, 1,
++ NPC_LID_LC, NPC_LT_LC_NGIO,
++ 0,
++ 0, 0, 0, 0,
++ },
+ {
+ NPC_ERRLEV_LC, NPC_EC_UNK,
+ 0, 0, 0, 0, 1,
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
+index 9b8e59f4c206d..77cb52b80c60f 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
+@@ -85,6 +85,8 @@ struct ptp *ptp_get(void)
+ /* Check driver is bound to PTP block */
+ if (!ptp)
+ ptp = ERR_PTR(-EPROBE_DEFER);
++ else
++ pci_dev_get(ptp->pdev);
+
+ return ptp;
+ }
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
+index 07b0eafccad87..9ea2f6ac38ec1 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
+@@ -29,6 +29,9 @@ static struct mac_ops rpm_mac_ops = {
+ .mac_get_pause_frm_status = rpm_lmac_get_pause_frm_status,
+ .mac_enadis_pause_frm = rpm_lmac_enadis_pause_frm,
+ .mac_pause_frm_config = rpm_lmac_pause_frm_config,
++ .mac_enadis_ptp_config = rpm_lmac_ptp_config,
++ .mac_rx_tx_enable = rpm_lmac_rx_tx_enable,
++ .mac_tx_enable = rpm_lmac_tx_enable,
+ };
+
+ struct mac_ops *rpm_get_mac_ops(void)
+@@ -53,6 +56,43 @@ int rpm_get_nr_lmacs(void *rpmd)
+ return hweight8(rpm_read(rpm, 0, CGXX_CMRX_RX_LMACS) & 0xFULL);
+ }
+
++int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable)
++{
++ rpm_t *rpm = rpmd;
++ u64 cfg, last;
++
++ if (!is_lmac_valid(rpm, lmac_id))
++ return -ENODEV;
++
++ cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG);
++ last = cfg;
++ if (enable)
++ cfg |= RPM_TX_EN;
++ else
++ cfg &= ~(RPM_TX_EN);
++
++ if (cfg != last)
++ rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg);
++ return !!(last & RPM_TX_EN);
++}
++
++int rpm_lmac_rx_tx_enable(void *rpmd, int lmac_id, bool enable)
++{
++ rpm_t *rpm = rpmd;
++ u64 cfg;
++
++ if (!is_lmac_valid(rpm, lmac_id))
++ return -ENODEV;
++
++ cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG);
++ if (enable)
++ cfg |= RPM_RX_EN | RPM_TX_EN;
++ else
++ cfg &= ~(RPM_RX_EN | RPM_TX_EN);
++ rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg);
++ return 0;
++}
++
+ void rpm_lmac_enadis_rx_pause_fwding(void *rpmd, int lmac_id, bool enable)
+ {
+ rpm_t *rpm = rpmd;
+@@ -251,22 +291,35 @@ int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable)
+ if (!rpm || lmac_id >= rpm->lmac_count)
+ return -ENODEV;
+ lmac_type = rpm->mac_ops->get_lmac_type(rpm, lmac_id);
+- if (lmac_type == LMAC_MODE_100G_R) {
+- cfg = rpm_read(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1);
+-
+- if (enable)
+- cfg |= RPMX_MTI_PCS_LBK;
+- else
+- cfg &= ~RPMX_MTI_PCS_LBK;
+- rpm_write(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1, cfg);
+- } else {
+- cfg = rpm_read(rpm, lmac_id, RPMX_MTI_LPCSX_CONTROL1);
+- if (enable)
+- cfg |= RPMX_MTI_PCS_LBK;
+- else
+- cfg &= ~RPMX_MTI_PCS_LBK;
+- rpm_write(rpm, lmac_id, RPMX_MTI_LPCSX_CONTROL1, cfg);
++
++ if (lmac_type == LMAC_MODE_QSGMII || lmac_type == LMAC_MODE_SGMII) {
++ dev_err(&rpm->pdev->dev, "loopback not supported for LPC mode\n");
++ return 0;
+ }
+
++ cfg = rpm_read(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1);
++
++ if (enable)
++ cfg |= RPMX_MTI_PCS_LBK;
++ else
++ cfg &= ~RPMX_MTI_PCS_LBK;
++ rpm_write(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1, cfg);
++
+ return 0;
+ }
++
++void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable)
++{
++ rpm_t *rpm = rpmd;
++ u64 cfg;
++
++ if (!is_lmac_valid(rpm, lmac_id))
++ return;
++
++ cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_CFG);
++ if (enable)
++ cfg |= RPMX_RX_TS_PREPEND;
++ else
++ cfg &= ~RPMX_RX_TS_PREPEND;
++ rpm_write(rpm, lmac_id, RPMX_CMRX_CFG, cfg);
++}
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
+index f0b069442dccb..ff580311edd03 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
+@@ -14,6 +14,8 @@
+ #define PCI_DEVID_CN10K_RPM 0xA060
+
+ /* Registers */
++#define RPMX_CMRX_CFG 0x00
++#define RPMX_RX_TS_PREPEND BIT_ULL(22)
+ #define RPMX_CMRX_SW_INT 0x180
+ #define RPMX_CMRX_SW_INT_W1S 0x188
+ #define RPMX_CMRX_SW_INT_ENA_W1S 0x198
+@@ -41,6 +43,8 @@
+ #define RPMX_MTI_STAT_DATA_HI_CDC 0x10038
+
+ #define RPM_LMAC_FWI 0xa
++#define RPM_TX_EN BIT_ULL(0)
++#define RPM_RX_EN BIT_ULL(1)
+
+ /* Function Declarations */
+ int rpm_get_nr_lmacs(void *rpmd);
+@@ -54,4 +58,7 @@ int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause,
+ u8 rx_pause);
+ int rpm_get_tx_stats(void *rpmd, int lmac_id, int idx, u64 *tx_stat);
+ int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat);
++void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable);
++int rpm_lmac_rx_tx_enable(void *rpmd, int lmac_id, bool enable);
++int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable);
+ #endif /* RPM_H */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+index 35836903b7fbc..11ef46e72ddd9 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+@@ -520,8 +520,11 @@ static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
+
+ rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
+ err = rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
+- if (err)
+- dev_err(rvu->dev, "HW block:%d reset failed\n", blkaddr);
++ if (err) {
++ dev_err(rvu->dev, "HW block:%d reset timeout retrying again\n", blkaddr);
++ while (rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true) == -EBUSY)
++ ;
++ }
+ }
+
+ static void rvu_reset_all_blocks(struct rvu *rvu)
+@@ -2281,7 +2284,7 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
+ goto free_regions;
+ break;
+ default:
+- return err;
++ goto free_regions;
+ }
+
+ mw->mbox_wq = alloc_workqueue(name,
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+index 1d9411232f1da..a7213db38804b 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+@@ -220,6 +220,7 @@ struct rvu_pfvf {
+ u16 maxlen;
+ u16 minlen;
+
++ bool hw_rx_tstamp_en; /* Is rx_tstamp enabled */
+ u8 mac_addr[ETH_ALEN]; /* MAC address of this PF/VF */
+ u8 default_mac[ETH_ALEN]; /* MAC address from FWdata */
+
+@@ -237,6 +238,7 @@ struct rvu_pfvf {
+ bool cgx_in_use; /* this PF/VF using CGX? */
+ int cgx_users; /* number of cgx users - used only by PFs */
+
++ int intf_mode;
+ u8 nix_blkaddr; /* BLKADDR_NIX0/1 assigned to this PF */
+ u8 nix_rx_intf; /* NIX0_RX/NIX1_RX interface to NPC */
+ u8 nix_tx_intf; /* NIX0_TX/NIX1_TX interface to NPC */
+@@ -794,10 +796,12 @@ void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ int blkaddr, u16 src, struct mcam_entry *entry,
+ u8 *intf, u8 *ena);
++bool is_cgx_config_permitted(struct rvu *rvu, u16 pcifunc);
+ bool is_mac_feature_supported(struct rvu *rvu, int pf, int feature);
+ u32 rvu_cgx_get_fifolen(struct rvu *rvu);
+ void *rvu_first_cgx_pdata(struct rvu *rvu);
+ int cgxlmac_to_pf(struct rvu *rvu, int cgx_id, int lmac_id);
++int rvu_cgx_config_tx(void *cgxd, int lmac_id, bool enable);
+
+ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf,
+ int type);
+@@ -827,4 +831,7 @@ void rvu_switch_enable(struct rvu *rvu);
+ void rvu_switch_disable(struct rvu *rvu);
+ void rvu_switch_update_rules(struct rvu *rvu, u16 pcifunc);
+
++int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
++ u64 pkind, u8 var_len_off, u8 var_len_off_mask,
++ u8 shift_dir);
+ #endif /* RVU_H */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
+index 81e8ea9ee30ea..28ff67819566c 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
+@@ -411,7 +411,7 @@ int rvu_cgx_exit(struct rvu *rvu)
+ * VF's of mapped PF and other PFs are not allowed. This fn() checks
+ * whether a PFFUNC is permitted to do the config or not.
+ */
+-static bool is_cgx_config_permitted(struct rvu *rvu, u16 pcifunc)
++inline bool is_cgx_config_permitted(struct rvu *rvu, u16 pcifunc)
+ {
+ if ((pcifunc & RVU_PFVF_FUNC_MASK) ||
+ !is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc)))
+@@ -442,16 +442,26 @@ void rvu_cgx_enadis_rx_bp(struct rvu *rvu, int pf, bool enable)
+ int rvu_cgx_config_rxtx(struct rvu *rvu, u16 pcifunc, bool start)
+ {
+ int pf = rvu_get_pf(pcifunc);
++ struct mac_ops *mac_ops;
+ u8 cgx_id, lmac_id;
++ void *cgxd;
+
+ if (!is_cgx_config_permitted(rvu, pcifunc))
+ return LMAC_AF_ERR_PERM_DENIED;
+
+ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
++ cgxd = rvu_cgx_pdata(cgx_id, rvu);
++ mac_ops = get_mac_ops(cgxd);
+
+- cgx_lmac_rx_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, start);
++ return mac_ops->mac_rx_tx_enable(cgxd, lmac_id, start);
++}
+
+- return 0;
++int rvu_cgx_config_tx(void *cgxd, int lmac_id, bool enable)
++{
++ struct mac_ops *mac_ops;
++
++ mac_ops = get_mac_ops(cgxd);
++ return mac_ops->mac_tx_enable(cgxd, lmac_id, enable);
+ }
+
+ void rvu_cgx_disable_dmac_entries(struct rvu *rvu, u16 pcifunc)
+@@ -694,7 +704,9 @@ int rvu_mbox_handler_cgx_promisc_disable(struct rvu *rvu, struct msg_req *req,
+
+ static int rvu_cgx_ptp_rx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
+ {
++ struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
+ int pf = rvu_get_pf(pcifunc);
++ struct mac_ops *mac_ops;
+ u8 cgx_id, lmac_id;
+ void *cgxd;
+
+@@ -711,13 +723,16 @@ static int rvu_cgx_ptp_rx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
+ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
+ cgxd = rvu_cgx_pdata(cgx_id, rvu);
+
+- cgx_lmac_ptp_config(cgxd, lmac_id, enable);
++ mac_ops = get_mac_ops(cgxd);
++ mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, true);
+ /* If PTP is enabled then inform NPC that packets to be
+ * parsed by this PF will have their data shifted by 8 bytes
+ * and if PTP is disabled then no shift is required
+ */
+ if (npc_config_ts_kpuaction(rvu, pf, pcifunc, enable))
+ return -EINVAL;
++ /* This flag is required to clean up CGX conf if app gets killed */
++ pfvf->hw_rx_tstamp_en = enable;
+
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
+index 49d822a98adab..f001579569a2b 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
+@@ -1131,6 +1131,8 @@ static void print_nix_cn10k_sq_ctx(struct seq_file *m,
+ seq_printf(m, "W3: head_offset\t\t\t%d\nW3: smenq_next_sqb_vld\t\t%d\n\n",
+ sq_ctx->head_offset, sq_ctx->smenq_next_sqb_vld);
+
++ seq_printf(m, "W3: smq_next_sq_vld\t\t%d\nW3: smq_pend\t\t\t%d\n",
++ sq_ctx->smq_next_sq_vld, sq_ctx->smq_pend);
+ seq_printf(m, "W4: next_sqb \t\t\t%llx\n\n", sq_ctx->next_sqb);
+ seq_printf(m, "W5: tail_sqb \t\t\t%llx\n\n", sq_ctx->tail_sqb);
+ seq_printf(m, "W6: smenq_sqb \t\t\t%llx\n\n", sq_ctx->smenq_sqb);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+index 6970540dc4709..603361c94786a 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+@@ -28,6 +28,7 @@ static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req,
+ static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc);
+ static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw,
+ u32 leaf_prof);
++static const char *nix_get_ctx_name(int ctype);
+
+ enum mc_tbl_sz {
+ MC_TBL_SZ_256,
+@@ -511,11 +512,11 @@ static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
+ cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
+ lmac_chan_cnt = cfg & 0xFF;
+
+- cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
+- sdp_chan_cnt = cfg & 0xFFF;
+-
+ cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt;
+ lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF);
++
++ cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
++ sdp_chan_cnt = cfg & 0xFFF;
+ sdp_bpid_cnt = hw->sdp_links * sdp_chan_cnt;
+
+ pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
+@@ -1061,10 +1062,68 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw,
+ return 0;
+ }
+
++static int rvu_nix_verify_aq_ctx(struct rvu *rvu, struct nix_hw *nix_hw,
++ struct nix_aq_enq_req *req, u8 ctype)
++{
++ struct nix_cn10k_aq_enq_req aq_req;
++ struct nix_cn10k_aq_enq_rsp aq_rsp;
++ int rc, word;
++
++ if (req->ctype != NIX_AQ_CTYPE_CQ)
++ return 0;
++
++ rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp,
++ req->hdr.pcifunc, ctype, req->qidx);
++ if (rc) {
++ dev_err(rvu->dev,
++ "%s: Failed to fetch %s%d context of PFFUNC 0x%x\n",
++ __func__, nix_get_ctx_name(ctype), req->qidx,
++ req->hdr.pcifunc);
++ return rc;
++ }
++
++ /* Make copy of original context & mask which are required
++ * for resubmission
++ */
++ memcpy(&aq_req.cq_mask, &req->cq_mask, sizeof(struct nix_cq_ctx_s));
++ memcpy(&aq_req.cq, &req->cq, sizeof(struct nix_cq_ctx_s));
++
++ /* exclude fields which HW can update */
++ aq_req.cq_mask.cq_err = 0;
++ aq_req.cq_mask.wrptr = 0;
++ aq_req.cq_mask.tail = 0;
++ aq_req.cq_mask.head = 0;
++ aq_req.cq_mask.avg_level = 0;
++ aq_req.cq_mask.update_time = 0;
++ aq_req.cq_mask.substream = 0;
++
++ /* Context mask (cq_mask) holds mask value of fields which
++ * are changed in AQ WRITE operation.
++ * for example cq.drop = 0xa;
++ * cq_mask.drop = 0xff;
++ * Below logic performs '&' between cq and cq_mask so that non
++ * updated fields are masked out for request and response
++ * comparison
++ */
++ for (word = 0; word < sizeof(struct nix_cq_ctx_s) / sizeof(u64);
++ word++) {
++ *(u64 *)((u8 *)&aq_rsp.cq + word * 8) &=
++ (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8));
++ *(u64 *)((u8 *)&aq_req.cq + word * 8) &=
++ (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8));
++ }
++
++ if (memcmp(&aq_req.cq, &aq_rsp.cq, sizeof(struct nix_cq_ctx_s)))
++ return NIX_AF_ERR_AQ_CTX_RETRY_WRITE;
++
++ return 0;
++}
++
+ static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
+ struct nix_aq_enq_rsp *rsp)
+ {
+ struct nix_hw *nix_hw;
++ int err, retries = 5;
+ int blkaddr;
+
+ blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
+@@ -1075,7 +1134,24 @@ static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
+ if (!nix_hw)
+ return NIX_AF_ERR_INVALID_NIXBLK;
+
+- return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp);
++retry:
++ err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp);
++
++ /* HW errata 'AQ Modification to CQ could be discarded on heavy traffic'
++ * As a work around perfrom CQ context read after each AQ write. If AQ
++ * read shows AQ write is not updated perform AQ write again.
++ */
++ if (!err && req->op == NIX_AQ_INSTOP_WRITE) {
++ err = rvu_nix_verify_aq_ctx(rvu, nix_hw, req, NIX_AQ_CTYPE_CQ);
++ if (err == NIX_AF_ERR_AQ_CTX_RETRY_WRITE) {
++ if (retries--)
++ goto retry;
++ else
++ return NIX_AF_ERR_CQ_CTX_WRITE_ERR;
++ }
++ }
++
++ return err;
+ }
+
+ static const char *nix_get_ctx_name(int ctype)
+@@ -1992,8 +2068,8 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr,
+ /* enable cgx tx if disabled */
+ if (is_pf_cgxmapped(rvu, pf)) {
+ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
+- restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
+- lmac_id, true);
++ restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu),
++ lmac_id, true);
+ }
+
+ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
+@@ -2016,7 +2092,7 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr,
+ rvu_cgx_enadis_rx_bp(rvu, pf, true);
+ /* restore cgx tx state */
+ if (restore_tx_en)
+- cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
++ rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
+ return err;
+ }
+
+@@ -3802,7 +3878,7 @@ nix_config_link_credits(struct rvu *rvu, int blkaddr, int link,
+ /* Enable cgx tx if disabled for credits to be back */
+ if (is_pf_cgxmapped(rvu, pf)) {
+ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
+- restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
++ restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu),
+ lmac_id, true);
+ }
+
+@@ -3815,8 +3891,8 @@ nix_config_link_credits(struct rvu *rvu, int blkaddr, int link,
+ NIX_AF_TL1X_SW_XOFF(schq), BIT_ULL(0));
+ }
+
+- rc = -EBUSY;
+- poll_tmo = jiffies + usecs_to_jiffies(10000);
++ rc = NIX_AF_ERR_LINK_CREDITS;
++ poll_tmo = jiffies + usecs_to_jiffies(200000);
+ /* Wait for credits to return */
+ do {
+ if (time_after(jiffies, poll_tmo))
+@@ -3842,7 +3918,7 @@ exit:
+
+ /* Restore state of cgx tx */
+ if (restore_tx_en)
+- cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
++ rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
+
+ mutex_unlock(&rvu->rsrc_lock);
+ return rc;
+@@ -4443,6 +4519,10 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
+ {
+ struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
+ struct hwctx_disable_req ctx_req;
++ int pf = rvu_get_pf(pcifunc);
++ struct mac_ops *mac_ops;
++ u8 cgx_id, lmac_id;
++ void *cgxd;
+ int err;
+
+ ctx_req.hdr.pcifunc = pcifunc;
+@@ -4479,6 +4559,22 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
+ dev_err(rvu->dev, "CQ ctx disable failed\n");
+ }
+
++ /* reset HW config done for Switch headers */
++ rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT,
++ (PKIND_TX | PKIND_RX), 0, 0, 0, 0);
++
++ /* Disabling CGX and NPC config done for PTP */
++ if (pfvf->hw_rx_tstamp_en) {
++ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
++ cgxd = rvu_cgx_pdata(cgx_id, rvu);
++ mac_ops = get_mac_ops(cgxd);
++ mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, false);
++ /* Undo NPC config done for PTP */
++ if (npc_config_ts_kpuaction(rvu, pf, pcifunc, false))
++ dev_err(rvu->dev, "NPC config for PTP failed\n");
++ pfvf->hw_rx_tstamp_en = false;
++ }
++
+ nix_ctx_free(rvu, pfvf);
+
+ nix_free_all_bandprof(rvu, pcifunc);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+index 5efb4174e82df..c4a46b295d401 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+@@ -402,6 +402,7 @@ static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam,
+ int blkaddr, int index, struct mcam_entry *entry,
+ bool *enable)
+ {
++ struct rvu_npc_mcam_rule *rule;
+ u16 owner, target_func;
+ struct rvu_pfvf *pfvf;
+ u64 rx_action;
+@@ -423,6 +424,12 @@ static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam,
+ test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
+ *enable = false;
+
++ /* fix up not needed for the rules added by user(ntuple filters) */
++ list_for_each_entry(rule, &mcam->mcam_rules, list) {
++ if (rule->entry == index)
++ return;
++ }
++
+ /* copy VF default entry action to the VF mcam entry */
+ rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr,
+ target_func);
+@@ -489,8 +496,8 @@ static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ }
+
+ /* PF installing VF rule */
+- if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries)
+- npc_fixup_vf_rule(rvu, mcam, blkaddr, index, entry, &enable);
++ if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries)
++ npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable);
+
+ /* Set 'action' */
+ rvu_write64(rvu, blkaddr,
+@@ -598,7 +605,7 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
+ struct npc_install_flow_req req = { 0 };
+ struct npc_install_flow_rsp rsp = { 0 };
+ struct npc_mcam *mcam = &rvu->hw->mcam;
+- struct nix_rx_action action;
++ struct nix_rx_action action = { 0 };
+ int blkaddr, index;
+
+ /* AF's and SDP VFs work in promiscuous mode */
+@@ -619,7 +626,6 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
+ *(u64 *)&action = npc_get_mcam_action(rvu, mcam,
+ blkaddr, index);
+ } else {
+- *(u64 *)&action = 0x00;
+ action.op = NIX_RX_ACTIONOP_UCAST;
+ action.pf_func = pcifunc;
+ }
+@@ -650,7 +656,7 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
+ struct npc_mcam *mcam = &rvu->hw->mcam;
+ struct rvu_hwinfo *hw = rvu->hw;
+ int blkaddr, ucast_idx, index;
+- struct nix_rx_action action;
++ struct nix_rx_action action = { 0 };
+ u64 relaxed_mask;
+
+ if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc))
+@@ -678,14 +684,14 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
+ blkaddr, ucast_idx);
+
+ if (action.op != NIX_RX_ACTIONOP_RSS) {
+- *(u64 *)&action = 0x00;
++ *(u64 *)&action = 0;
+ action.op = NIX_RX_ACTIONOP_UCAST;
+ }
+
+ /* RX_ACTION set to MCAST for CGX PF's */
+ if (hw->cap.nix_rx_multicast && pfvf->use_mce_list &&
+ is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) {
+- *(u64 *)&action = 0x00;
++ *(u64 *)&action = 0;
+ action.op = NIX_RX_ACTIONOP_MCAST;
+ pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
+ action.index = pfvf->promisc_mce_idx;
+@@ -825,7 +831,7 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
+ struct rvu_hwinfo *hw = rvu->hw;
+ int blkaddr, ucast_idx, index;
+ u8 mac_addr[ETH_ALEN] = { 0 };
+- struct nix_rx_action action;
++ struct nix_rx_action action = { 0 };
+ struct rvu_pfvf *pfvf;
+ u16 vf_func;
+
+@@ -854,14 +860,14 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
+ blkaddr, ucast_idx);
+
+ if (action.op != NIX_RX_ACTIONOP_RSS) {
+- *(u64 *)&action = 0x00;
++ *(u64 *)&action = 0;
+ action.op = NIX_RX_ACTIONOP_UCAST;
+ action.pf_func = pcifunc;
+ }
+
+ /* RX_ACTION set to MCAST for CGX PF's */
+ if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) {
+- *(u64 *)&action = 0x00;
++ *(u64 *)&action = 0;
+ action.op = NIX_RX_ACTIONOP_MCAST;
+ action.index = pfvf->mcast_mce_idx;
+ }
+@@ -916,7 +922,8 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ int blkaddr, u16 pcifunc, u64 rx_action)
+ {
+ int actindex, index, bank, entry;
+- bool enable;
++ struct rvu_npc_mcam_rule *rule;
++ bool enable, update;
+
+ if (!(pcifunc & RVU_PFVF_FUNC_MASK))
+ return;
+@@ -924,6 +931,14 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ mutex_lock(&mcam->lock);
+ for (index = 0; index < mcam->bmap_entries; index++) {
+ if (mcam->entry2target_pffunc[index] == pcifunc) {
++ update = true;
++ /* update not needed for the rules added via ntuple filters */
++ list_for_each_entry(rule, &mcam->mcam_rules, list) {
++ if (rule->entry == index)
++ update = false;
++ }
++ if (!update)
++ continue;
+ bank = npc_get_bank(mcam, index);
+ actindex = index;
+ entry = index & (mcam->banksize - 1);
+@@ -3167,6 +3182,102 @@ int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req,
+ return 0;
+ }
+
++static int
++npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind,
++ u8 var_len_off, u8 var_len_off_mask, u8 shift_dir)
++{
++ struct npc_kpu_action0 *act0;
++ u8 shift_count = 0;
++ int blkaddr;
++ u64 val;
++
++ if (!var_len_off_mask)
++ return -EINVAL;
++
++ if (var_len_off_mask != 0xff) {
++ if (shift_dir)
++ shift_count = __ffs(var_len_off_mask);
++ else
++ shift_count = (8 - __fls(var_len_off_mask));
++ }
++ blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);
++ if (blkaddr < 0) {
++ dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
++ return -EINVAL;
++ }
++ val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));
++ act0 = (struct npc_kpu_action0 *)&val;
++ act0->var_len_shift = shift_count;
++ act0->var_len_right = shift_dir;
++ act0->var_len_mask = var_len_off_mask;
++ act0->var_len_offset = var_len_off;
++ rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);
++ return 0;
++}
++
++int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
++ u64 pkind, u8 var_len_off, u8 var_len_off_mask,
++ u8 shift_dir)
++
++{
++ struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
++ int blkaddr, nixlf, rc, intf_mode;
++ int pf = rvu_get_pf(pcifunc);
++ u64 rxpkind, txpkind;
++ u8 cgx_id, lmac_id;
++
++ /* use default pkind to disable edsa/higig */
++ rxpkind = rvu_npc_get_pkind(rvu, pf);
++ txpkind = NPC_TX_DEF_PKIND;
++ intf_mode = NPC_INTF_MODE_DEF;
++
++ if (mode & OTX2_PRIV_FLAGS_CUSTOM) {
++ if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) {
++ rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind,
++ var_len_off,
++ var_len_off_mask,
++ shift_dir);
++ if (rc)
++ return rc;
++ }
++ rxpkind = pkind;
++ txpkind = pkind;
++ }
++
++ if (dir & PKIND_RX) {
++ /* rx pkind set req valid only for cgx mapped PFs */
++ if (!is_cgx_config_permitted(rvu, pcifunc))
++ return 0;
++ rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
++
++ rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id,
++ rxpkind);
++ if (rc)
++ return rc;
++ }
++
++ if (dir & PKIND_TX) {
++ /* Tx pkind set request valid if PCIFUNC has NIXLF attached */
++ rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
++ if (rc)
++ return rc;
++
++ rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf),
++ txpkind);
++ }
++
++ pfvf->intf_mode = intf_mode;
++ return 0;
++}
++
++int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req,
++ struct msg_rsp *rsp)
++{
++ return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode,
++ req->dir, req->pkind, req->var_len_off,
++ req->var_len_off_mask, req->shift_dir);
++}
++
+ int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu,
+ struct msg_req *req,
+ struct npc_mcam_read_base_rule_rsp *rsp)
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
+index 51ddc7b81d0bd..ca404d51d9f56 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
+@@ -1098,14 +1098,6 @@ find_rule:
+ write_req.cntr = rule->cntr;
+ }
+
+- err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
+- &write_rsp);
+- if (err) {
+- rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
+- if (new)
+- kfree(rule);
+- return err;
+- }
+ /* update rule */
+ memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
+ memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
+@@ -1129,6 +1121,18 @@ find_rule:
+ if (req->default_rule)
+ pfvf->def_ucast_rule = rule;
+
++ /* write to mcam entry registers */
++ err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
++ &write_rsp);
++ if (err) {
++ rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
++ if (new) {
++ list_del(&rule->list);
++ kfree(rule);
++ }
++ return err;
++ }
++
+ /* VF's MAC address is being changed via PF */
+ if (pf_set_vfs_mac) {
+ ether_addr_copy(pfvf->default_mac, req->packet.dmac);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+index a51ecd771d075..637450de189c8 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+@@ -591,6 +591,7 @@ static inline void __cn10k_aura_freeptr(struct otx2_nic *pfvf, u64 aura,
+ size++;
+ tar_addr |= ((size - 1) & 0x7) << 4;
+ }
++ dma_wmb();
+ memcpy((u64 *)lmt_info->lmt_addr, ptrs, sizeof(u64) * num_ptrs);
+ /* Perform LMTST flush */
+ cn10k_lmt_flush(val, tar_addr);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+index 53df7fff92c40..b1894d4045b8d 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+@@ -386,7 +386,12 @@ static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
+ dst_mdev->msg_size = mbox_hdr->msg_size;
+ dst_mdev->num_msgs = num_msgs;
+ err = otx2_sync_mbox_msg(dst_mbox);
+- if (err) {
++ /* Error code -EIO indicate there is a communication failure
++ * to the AF. Rest of the error codes indicate that AF processed
++ * VF messages and set the error codes in response messages
++ * (if any) so simply forward responses to VF.
++ */
++ if (err == -EIO) {
+ dev_warn(pf->dev,
+ "AF not responding to VF%d messages\n", vf);
+ /* restore PF mbase and exit */
+@@ -1493,6 +1498,44 @@ static void otx2_free_hw_resources(struct otx2_nic *pf)
+ mutex_unlock(&mbox->lock);
+ }
+
++static void otx2_do_set_rx_mode(struct otx2_nic *pf)
++{
++ struct net_device *netdev = pf->netdev;
++ struct nix_rx_mode *req;
++ bool promisc = false;
++
++ if (!(netdev->flags & IFF_UP))
++ return;
++
++ if ((netdev->flags & IFF_PROMISC) ||
++ (netdev_uc_count(netdev) > OTX2_MAX_UNICAST_FLOWS)) {
++ promisc = true;
++ }
++
++ /* Write unicast address to mcam entries or del from mcam */
++ if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
++ __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
++
++ mutex_lock(&pf->mbox.lock);
++ req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
++ if (!req) {
++ mutex_unlock(&pf->mbox.lock);
++ return;
++ }
++
++ req->mode = NIX_RX_MODE_UCAST;
++
++ if (promisc)
++ req->mode |= NIX_RX_MODE_PROMISC;
++ if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
++ req->mode |= NIX_RX_MODE_ALLMULTI;
++
++ req->mode |= NIX_RX_MODE_USE_MCE;
++
++ otx2_sync_mbox_msg(&pf->mbox);
++ mutex_unlock(&pf->mbox.lock);
++}
++
+ int otx2_open(struct net_device *netdev)
+ {
+ struct otx2_nic *pf = netdev_priv(netdev);
+@@ -1646,6 +1689,8 @@ int otx2_open(struct net_device *netdev)
+ if (err)
+ goto err_tx_stop_queues;
+
++ otx2_do_set_rx_mode(pf);
++
+ return 0;
+
+ err_tx_stop_queues:
+@@ -1791,43 +1836,11 @@ static void otx2_set_rx_mode(struct net_device *netdev)
+ queue_work(pf->otx2_wq, &pf->rx_mode_work);
+ }
+
+-static void otx2_do_set_rx_mode(struct work_struct *work)
++static void otx2_rx_mode_wrk_handler(struct work_struct *work)
+ {
+ struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
+- struct net_device *netdev = pf->netdev;
+- struct nix_rx_mode *req;
+- bool promisc = false;
+-
+- if (!(netdev->flags & IFF_UP))
+- return;
+-
+- if ((netdev->flags & IFF_PROMISC) ||
+- (netdev_uc_count(netdev) > OTX2_MAX_UNICAST_FLOWS)) {
+- promisc = true;
+- }
+-
+- /* Write unicast address to mcam entries or del from mcam */
+- if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
+- __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
+
+- mutex_lock(&pf->mbox.lock);
+- req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
+- if (!req) {
+- mutex_unlock(&pf->mbox.lock);
+- return;
+- }
+-
+- req->mode = NIX_RX_MODE_UCAST;
+-
+- if (promisc)
+- req->mode |= NIX_RX_MODE_PROMISC;
+- if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
+- req->mode |= NIX_RX_MODE_ALLMULTI;
+-
+- req->mode |= NIX_RX_MODE_USE_MCE;
+-
+- otx2_sync_mbox_msg(&pf->mbox);
+- mutex_unlock(&pf->mbox.lock);
++ otx2_do_set_rx_mode(pf);
+ }
+
+ static int otx2_set_features(struct net_device *netdev,
+@@ -2358,7 +2371,7 @@ static int otx2_wq_init(struct otx2_nic *pf)
+ if (!pf->otx2_wq)
+ return -ENOMEM;
+
+- INIT_WORK(&pf->rx_mode_work, otx2_do_set_rx_mode);
++ INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler);
+ INIT_WORK(&pf->reset_task, otx2_reset_task);
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c
+index 44c670807fb3c..656c68cfd7ec6 100644
+--- a/drivers/net/ethernet/marvell/prestera/prestera_main.c
++++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c
+@@ -54,12 +54,14 @@ int prestera_port_pvid_set(struct prestera_port *port, u16 vid)
+ struct prestera_port *prestera_port_find_by_hwid(struct prestera_switch *sw,
+ u32 dev_id, u32 hw_id)
+ {
+- struct prestera_port *port = NULL;
++ struct prestera_port *port = NULL, *tmp;
+
+ read_lock(&sw->port_list_lock);
+- list_for_each_entry(port, &sw->port_list, list) {
+- if (port->dev_id == dev_id && port->hw_id == hw_id)
++ list_for_each_entry(tmp, &sw->port_list, list) {
++ if (tmp->dev_id == dev_id && tmp->hw_id == hw_id) {
++ port = tmp;
+ break;
++ }
+ }
+ read_unlock(&sw->port_list_lock);
+
+@@ -68,12 +70,14 @@ struct prestera_port *prestera_port_find_by_hwid(struct prestera_switch *sw,
+
+ struct prestera_port *prestera_find_port(struct prestera_switch *sw, u32 id)
+ {
+- struct prestera_port *port = NULL;
++ struct prestera_port *port = NULL, *tmp;
+
+ read_lock(&sw->port_list_lock);
+- list_for_each_entry(port, &sw->port_list, list) {
+- if (port->id == id)
++ list_for_each_entry(tmp, &sw->port_list, list) {
++ if (tmp->id == id) {
++ port = tmp;
+ break;
++ }
+ }
+ read_unlock(&sw->port_list_lock);
+
+@@ -488,6 +492,7 @@ static int prestera_switch_set_base_mac_addr(struct prestera_switch *sw)
+ dev_info(prestera_dev(sw), "using random base mac address\n");
+ }
+ of_node_put(base_mac_np);
++ of_node_put(np);
+
+ return prestera_hw_switch_mac_set(sw, sw->base_mac);
+ }
+@@ -703,23 +708,27 @@ static int prestera_netdev_port_event(struct net_device *lower,
+ struct net_device *dev,
+ unsigned long event, void *ptr)
+ {
+- struct netdev_notifier_changeupper_info *info = ptr;
++ struct netdev_notifier_info *info = ptr;
++ struct netdev_notifier_changeupper_info *cu_info;
+ struct prestera_port *port = netdev_priv(dev);
+ struct netlink_ext_ack *extack;
+ struct net_device *upper;
+
+- extack = netdev_notifier_info_to_extack(&info->info);
+- upper = info->upper_dev;
++ extack = netdev_notifier_info_to_extack(info);
++ cu_info = container_of(info,
++ struct netdev_notifier_changeupper_info,
++ info);
+
+ switch (event) {
+ case NETDEV_PRECHANGEUPPER:
++ upper = cu_info->upper_dev;
+ if (!netif_is_bridge_master(upper) &&
+ !netif_is_lag_master(upper)) {
+ NL_SET_ERR_MSG_MOD(extack, "Unknown upper device type");
+ return -EINVAL;
+ }
+
+- if (!info->linking)
++ if (!cu_info->linking)
+ break;
+
+ if (netdev_has_any_upper_dev(upper)) {
+@@ -728,7 +737,7 @@ static int prestera_netdev_port_event(struct net_device *lower,
+ }
+
+ if (netif_is_lag_master(upper) &&
+- !prestera_lag_master_check(upper, info->upper_info, extack))
++ !prestera_lag_master_check(upper, cu_info->upper_info, extack))
+ return -EOPNOTSUPP;
+ if (netif_is_lag_master(upper) && vlan_uses_dev(dev)) {
+ NL_SET_ERR_MSG_MOD(extack,
+@@ -744,14 +753,15 @@ static int prestera_netdev_port_event(struct net_device *lower,
+ break;
+
+ case NETDEV_CHANGEUPPER:
++ upper = cu_info->upper_dev;
+ if (netif_is_bridge_master(upper)) {
+- if (info->linking)
++ if (cu_info->linking)
+ return prestera_bridge_port_join(upper, port,
+ extack);
+ else
+ prestera_bridge_port_leave(upper, port);
+ } else if (netif_is_lag_master(upper)) {
+- if (info->linking)
++ if (cu_info->linking)
+ return prestera_lag_port_add(port, upper);
+ else
+ prestera_lag_port_del(port);
+diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
+index 3ce6ccd0f5394..b4599fe4ca8da 100644
+--- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
++++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
+@@ -497,8 +497,8 @@ int prestera_bridge_port_join(struct net_device *br_dev,
+
+ br_port = prestera_bridge_port_add(bridge, port->dev);
+ if (IS_ERR(br_port)) {
+- err = PTR_ERR(br_port);
+- goto err_brport_create;
++ prestera_bridge_put(bridge);
++ return PTR_ERR(br_port);
+ }
+
+ err = switchdev_bridge_port_offload(br_port->dev, port->dev, NULL,
+@@ -519,8 +519,6 @@ err_port_join:
+ switchdev_bridge_port_unoffload(br_port->dev, NULL, NULL, NULL);
+ err_switchdev_offload:
+ prestera_bridge_port_put(br_port);
+-err_brport_create:
+- prestera_bridge_put(bridge);
+ return err;
+ }
+
+@@ -1124,7 +1122,7 @@ static int prestera_switchdev_blk_event(struct notifier_block *unused,
+ prestera_port_obj_attr_set);
+ break;
+ default:
+- err = -EOPNOTSUPP;
++ return NOTIFY_DONE;
+ }
+
+ return notifier_from_errno(err);
+diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+index 398c23cec8151..ee1fd472e9252 100644
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -91,46 +91,53 @@ static int mtk_mdio_busy_wait(struct mtk_eth *eth)
+ }
+
+ dev_err(eth->dev, "mdio: MDIO timeout\n");
+- return -1;
++ return -ETIMEDOUT;
+ }
+
+-static u32 _mtk_mdio_write(struct mtk_eth *eth, u32 phy_addr,
+- u32 phy_register, u32 write_data)
++static int _mtk_mdio_write(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg,
++ u32 write_data)
+ {
+- if (mtk_mdio_busy_wait(eth))
+- return -1;
++ int ret;
+
+- write_data &= 0xffff;
++ ret = mtk_mdio_busy_wait(eth);
++ if (ret < 0)
++ return ret;
+
+- mtk_w32(eth, PHY_IAC_ACCESS | PHY_IAC_START | PHY_IAC_WRITE |
+- (phy_register << PHY_IAC_REG_SHIFT) |
+- (phy_addr << PHY_IAC_ADDR_SHIFT) | write_data,
++ mtk_w32(eth, PHY_IAC_ACCESS |
++ PHY_IAC_START_C22 |
++ PHY_IAC_CMD_WRITE |
++ PHY_IAC_REG(phy_reg) |
++ PHY_IAC_ADDR(phy_addr) |
++ PHY_IAC_DATA(write_data),
+ MTK_PHY_IAC);
+
+- if (mtk_mdio_busy_wait(eth))
+- return -1;
++ ret = mtk_mdio_busy_wait(eth);
++ if (ret < 0)
++ return ret;
+
+ return 0;
+ }
+
+-static u32 _mtk_mdio_read(struct mtk_eth *eth, int phy_addr, int phy_reg)
++static int _mtk_mdio_read(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg)
+ {
+- u32 d;
++ int ret;
+
+- if (mtk_mdio_busy_wait(eth))
+- return 0xffff;
++ ret = mtk_mdio_busy_wait(eth);
++ if (ret < 0)
++ return ret;
+
+- mtk_w32(eth, PHY_IAC_ACCESS | PHY_IAC_START | PHY_IAC_READ |
+- (phy_reg << PHY_IAC_REG_SHIFT) |
+- (phy_addr << PHY_IAC_ADDR_SHIFT),
++ mtk_w32(eth, PHY_IAC_ACCESS |
++ PHY_IAC_START_C22 |
++ PHY_IAC_CMD_C22_READ |
++ PHY_IAC_REG(phy_reg) |
++ PHY_IAC_ADDR(phy_addr),
+ MTK_PHY_IAC);
+
+- if (mtk_mdio_busy_wait(eth))
+- return 0xffff;
+-
+- d = mtk_r32(eth, MTK_PHY_IAC) & 0xffff;
++ ret = mtk_mdio_busy_wait(eth);
++ if (ret < 0)
++ return ret;
+
+- return d;
++ return mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_DATA_MASK;
+ }
+
+ static int mtk_mdio_write(struct mii_bus *bus, int phy_addr,
+@@ -217,7 +224,7 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
+ phylink_config);
+ struct mtk_eth *eth = mac->hw;
+ u32 mcr_cur, mcr_new, sid, i;
+- int val, ge_mode, err;
++ int val, ge_mode, err = 0;
+
+ /* MT76x8 has no hardware settings between for the MAC */
+ if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
+diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+index 5ef70dd8b49c6..f2d90639d7ed1 100644
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -341,11 +341,17 @@
+ /* PHY Indirect Access Control registers */
+ #define MTK_PHY_IAC 0x10004
+ #define PHY_IAC_ACCESS BIT(31)
+-#define PHY_IAC_READ BIT(19)
+-#define PHY_IAC_WRITE BIT(18)
+-#define PHY_IAC_START BIT(16)
+-#define PHY_IAC_ADDR_SHIFT 20
+-#define PHY_IAC_REG_SHIFT 25
++#define PHY_IAC_REG_MASK GENMASK(29, 25)
++#define PHY_IAC_REG(x) FIELD_PREP(PHY_IAC_REG_MASK, (x))
++#define PHY_IAC_ADDR_MASK GENMASK(24, 20)
++#define PHY_IAC_ADDR(x) FIELD_PREP(PHY_IAC_ADDR_MASK, (x))
++#define PHY_IAC_CMD_MASK GENMASK(19, 18)
++#define PHY_IAC_CMD_WRITE FIELD_PREP(PHY_IAC_CMD_MASK, 1)
++#define PHY_IAC_CMD_C22_READ FIELD_PREP(PHY_IAC_CMD_MASK, 2)
++#define PHY_IAC_START_MASK GENMASK(17, 16)
++#define PHY_IAC_START_C22 FIELD_PREP(PHY_IAC_START_MASK, 1)
++#define PHY_IAC_DATA_MASK GENMASK(15, 0)
++#define PHY_IAC_DATA(x) FIELD_PREP(PHY_IAC_DATA_MASK, (x))
+ #define PHY_IAC_TIMEOUT HZ
+
+ #define MTK_MAC_MISC 0x1000c
+diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
+index 3ad10c793308e..66298e2235c91 100644
+--- a/drivers/net/ethernet/mediatek/mtk_ppe.c
++++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
+@@ -395,7 +395,7 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
+ static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
+ int i, k;
+
+- memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(ppe->foe_table));
++ memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(*ppe->foe_table));
+
+ if (!IS_ENABLED(CONFIG_SOC_MT7621))
+ return;
+diff --git a/drivers/net/ethernet/mediatek/mtk_sgmii.c b/drivers/net/ethernet/mediatek/mtk_sgmii.c
+index 32d83421226a2..5897940a418b6 100644
+--- a/drivers/net/ethernet/mediatek/mtk_sgmii.c
++++ b/drivers/net/ethernet/mediatek/mtk_sgmii.c
+@@ -26,6 +26,7 @@ int mtk_sgmii_init(struct mtk_sgmii *ss, struct device_node *r, u32 ana_rgc3)
+ break;
+
+ ss->regmap[i] = syscon_node_to_regmap(np);
++ of_node_put(np);
+ if (IS_ERR(ss->regmap[i]))
+ return PTR_ERR(ss->regmap[i]);
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
+index ef518b1040f72..fde521b1eecad 100644
+--- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
++++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
+@@ -663,7 +663,7 @@ void __init mlx4_en_init_ptys2ethtool_map(void)
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_1000BASE_T, SPEED_1000,
+ ETHTOOL_LINK_MODE_1000baseT_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_1000BASE_CX_SGMII, SPEED_1000,
+- ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
++ ETHTOOL_LINK_MODE_1000baseX_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_1000BASE_KX, SPEED_1000,
+ ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_10GBASE_T, SPEED_10000,
+@@ -675,9 +675,9 @@ void __init mlx4_en_init_ptys2ethtool_map(void)
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_10GBASE_KR, SPEED_10000,
+ ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_10GBASE_CR, SPEED_10000,
+- ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
++ ETHTOOL_LINK_MODE_10000baseCR_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_10GBASE_SR, SPEED_10000,
+- ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
++ ETHTOOL_LINK_MODE_10000baseSR_Full_BIT);
+ MLX4_BUILD_PTYS2ETHTOOL_CONFIG(MLX4_20GBASE_KR2, SPEED_20000,
+ ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT,
+ ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT);
+diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+index 8af7f28273225..3bd3603873e32 100644
+--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
++++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+@@ -2286,9 +2286,14 @@ int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv,
+ bool carry_xdp_prog)
+ {
+ struct bpf_prog *xdp_prog;
+- int i, t;
++ int i, t, ret;
+
+- mlx4_en_copy_priv(tmp, priv, prof);
++ ret = mlx4_en_copy_priv(tmp, priv, prof);
++ if (ret) {
++ en_warn(priv, "%s: mlx4_en_copy_priv() failed, return\n",
++ __func__);
++ return ret;
++ }
+
+ if (mlx4_en_alloc_resources(tmp)) {
+ en_warn(priv,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+index db5dfff585c99..e06a6104e91fe 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+@@ -130,11 +130,8 @@ static int cmd_alloc_index(struct mlx5_cmd *cmd)
+
+ static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
+ {
+- unsigned long flags;
+-
+- spin_lock_irqsave(&cmd->alloc_lock, flags);
++ lockdep_assert_held(&cmd->alloc_lock);
+ set_bit(idx, &cmd->bitmask);
+- spin_unlock_irqrestore(&cmd->alloc_lock, flags);
+ }
+
+ static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
+@@ -144,13 +141,21 @@ static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
+
+ static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
+ {
++ struct mlx5_cmd *cmd = ent->cmd;
++ unsigned long flags;
++
++ spin_lock_irqsave(&cmd->alloc_lock, flags);
+ if (!refcount_dec_and_test(&ent->refcnt))
+- return;
++ goto out;
+
+- if (ent->idx >= 0)
+- cmd_free_index(ent->cmd, ent->idx);
++ if (ent->idx >= 0) {
++ cmd_free_index(cmd, ent->idx);
++ up(ent->page_queue ? &cmd->pages_sem : &cmd->sem);
++ }
+
+ cmd_free_ent(ent);
++out:
++ spin_unlock_irqrestore(&cmd->alloc_lock, flags);
+ }
+
+ static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
+@@ -334,6 +339,9 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
+ case MLX5_CMD_OP_PAGE_FAULT_RESUME:
+ case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
+ case MLX5_CMD_OP_DEALLOC_SF:
++ case MLX5_CMD_OP_DESTROY_UCTX:
++ case MLX5_CMD_OP_DESTROY_UMEM:
++ case MLX5_CMD_OP_MODIFY_RQT:
+ return MLX5_CMD_STAT_OK;
+
+ case MLX5_CMD_OP_QUERY_HCA_CAP:
+@@ -439,7 +447,6 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
+ case MLX5_CMD_OP_MODIFY_TIS:
+ case MLX5_CMD_OP_QUERY_TIS:
+ case MLX5_CMD_OP_CREATE_RQT:
+- case MLX5_CMD_OP_MODIFY_RQT:
+ case MLX5_CMD_OP_QUERY_RQT:
+
+ case MLX5_CMD_OP_CREATE_FLOW_TABLE:
+@@ -459,9 +466,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
+ case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
+ case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
+ case MLX5_CMD_OP_CREATE_UCTX:
+- case MLX5_CMD_OP_DESTROY_UCTX:
+ case MLX5_CMD_OP_CREATE_UMEM:
+- case MLX5_CMD_OP_DESTROY_UMEM:
+ case MLX5_CMD_OP_ALLOC_MEMIC:
+ case MLX5_CMD_OP_MODIFY_XRQ:
+ case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
+@@ -895,25 +900,6 @@ static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
+ return cmd->allowed_opcode == opcode;
+ }
+
+-static int cmd_alloc_index_retry(struct mlx5_cmd *cmd)
+-{
+- unsigned long alloc_end = jiffies + msecs_to_jiffies(1000);
+- int idx;
+-
+-retry:
+- idx = cmd_alloc_index(cmd);
+- if (idx < 0 && time_before(jiffies, alloc_end)) {
+- /* Index allocation can fail on heavy load of commands. This is a temporary
+- * situation as the current command already holds the semaphore, meaning that
+- * another command completion is being handled and it is expected to release
+- * the entry index soon.
+- */
+- cpu_relax();
+- goto retry;
+- }
+- return idx;
+-}
+-
+ bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
+ {
+ return pci_channel_offline(dev->pdev) ||
+@@ -938,7 +924,7 @@ static void cmd_work_handler(struct work_struct *work)
+ sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
+ down(sem);
+ if (!ent->page_queue) {
+- alloc_ret = cmd_alloc_index_retry(cmd);
++ alloc_ret = cmd_alloc_index(cmd);
+ if (alloc_ret < 0) {
+ mlx5_core_err_rl(dev, "failed to allocate command entry\n");
+ if (ent->callback) {
+@@ -1594,8 +1580,6 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force
+ vector = vec & 0xffffffff;
+ for (i = 0; i < (1 << cmd->log_sz); i++) {
+ if (test_bit(i, &vector)) {
+- struct semaphore *sem;
+-
+ ent = cmd->ent_arr[i];
+
+ /* if we already completed the command, ignore it */
+@@ -1618,10 +1602,6 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force
+ dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
+ cmd_ent_put(ent);
+
+- if (ent->page_queue)
+- sem = &cmd->pages_sem;
+- else
+- sem = &cmd->sem;
+ ent->ts2 = ktime_get_ns();
+ memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
+ dump_command(dev, ent, 0);
+@@ -1675,7 +1655,6 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force
+ */
+ complete(&ent->done);
+ }
+- up(sem);
+ }
+ }
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+index 02e77ffe5c3e4..5371ad0a12eb5 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+@@ -164,13 +164,14 @@ int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
+ MLX5_SET(destroy_cq_in, in, cqn, cq->cqn);
+ MLX5_SET(destroy_cq_in, in, uid, cq->uid);
+ err = mlx5_cmd_exec_in(dev, destroy_cq, in);
++ if (err)
++ return err;
+
+ synchronize_irq(cq->irqn);
+-
+ mlx5_cq_put(cq);
+ wait_for_completion(&cq->free);
+
+- return err;
++ return 0;
+ }
+ EXPORT_SYMBOL(mlx5_core_destroy_cq);
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
+index 07c8d9811bc81..10d195042ab55 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
+@@ -507,6 +507,8 @@ void mlx5_debug_cq_remove(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
+ if (!mlx5_debugfs_root)
+ return;
+
+- if (cq->dbg)
++ if (cq->dbg) {
+ rem_res_tree(cq->dbg);
++ cq->dbg = NULL;
++ }
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+index dcf9f27ba2efd..7d56a927081d0 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+@@ -625,7 +625,6 @@ static int mlx5_devlink_eth_param_register(struct devlink *devlink)
+ devlink_param_driverinit_value_set(devlink,
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
+ value);
+- devlink_param_publish(devlink, &enable_eth_param);
+ return 0;
+ }
+
+@@ -636,7 +635,6 @@ static void mlx5_devlink_eth_param_unregister(struct devlink *devlink)
+ if (!mlx5_eth_supported(dev))
+ return;
+
+- devlink_param_unpublish(devlink, &enable_eth_param);
+ devlink_param_unregister(devlink, &enable_eth_param);
+ }
+
+@@ -672,7 +670,6 @@ static int mlx5_devlink_rdma_param_register(struct devlink *devlink)
+ devlink_param_driverinit_value_set(devlink,
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
+ value);
+- devlink_param_publish(devlink, &enable_rdma_param);
+ return 0;
+ }
+
+@@ -681,7 +678,6 @@ static void mlx5_devlink_rdma_param_unregister(struct devlink *devlink)
+ if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND))
+ return;
+
+- devlink_param_unpublish(devlink, &enable_rdma_param);
+ devlink_param_unregister(devlink, &enable_rdma_param);
+ }
+
+@@ -706,7 +702,6 @@ static int mlx5_devlink_vnet_param_register(struct devlink *devlink)
+ devlink_param_driverinit_value_set(devlink,
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
+ value);
+- devlink_param_publish(devlink, &enable_rdma_param);
+ return 0;
+ }
+
+@@ -717,7 +712,6 @@ static void mlx5_devlink_vnet_param_unregister(struct devlink *devlink)
+ if (!mlx5_vnet_supported(dev))
+ return;
+
+- devlink_param_unpublish(devlink, &enable_vnet_param);
+ devlink_param_unregister(devlink, &enable_vnet_param);
+ }
+
+@@ -808,7 +802,6 @@ int mlx5_devlink_register(struct devlink *devlink)
+ if (err)
+ goto params_reg_err;
+ mlx5_devlink_set_params_init_values(devlink);
+- devlink_params_publish(devlink);
+
+ err = mlx5_devlink_auxdev_params_register(devlink);
+ if (err)
+@@ -818,6 +811,7 @@ int mlx5_devlink_register(struct devlink *devlink)
+ if (err)
+ goto traps_reg_err;
+
++ devlink_params_publish(devlink);
+ return 0;
+
+ traps_reg_err:
+@@ -832,9 +826,9 @@ params_reg_err:
+
+ void mlx5_devlink_unregister(struct devlink *devlink)
+ {
++ devlink_params_unpublish(devlink);
+ mlx5_devlink_traps_unregister(devlink);
+ mlx5_devlink_auxdev_params_unregister(devlink);
+- devlink_params_unpublish(devlink);
+ devlink_params_unregister(devlink, mlx5_devlink_params,
+ ARRAY_SIZE(mlx5_devlink_params));
+ devlink_unregister(devlink);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c
+index ed4fb79b4db76..75b6060f7a9ae 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c
+@@ -31,6 +31,7 @@ static const char *const mlx5_rsc_sgmt_name[] = {
+ struct mlx5_rsc_dump {
+ u32 pdn;
+ struct mlx5_core_mkey mkey;
++ u32 number_of_menu_items;
+ u16 fw_segment_type[MLX5_SGMT_TYPE_NUM];
+ };
+
+@@ -50,21 +51,37 @@ static int mlx5_rsc_dump_sgmt_get_by_name(char *name)
+ return -EINVAL;
+ }
+
+-static void mlx5_rsc_dump_read_menu_sgmt(struct mlx5_rsc_dump *rsc_dump, struct page *page)
++#define MLX5_RSC_DUMP_MENU_HEADER_SIZE (MLX5_ST_SZ_BYTES(resource_dump_info_segment) + \
++ MLX5_ST_SZ_BYTES(resource_dump_command_segment) + \
++ MLX5_ST_SZ_BYTES(resource_dump_menu_segment))
++
++static int mlx5_rsc_dump_read_menu_sgmt(struct mlx5_rsc_dump *rsc_dump, struct page *page,
++ int read_size, int start_idx)
+ {
+ void *data = page_address(page);
+ enum mlx5_sgmt_type sgmt_idx;
+ int num_of_items;
+ char *sgmt_name;
+ void *member;
++ int size = 0;
+ void *menu;
+ int i;
+
+- menu = MLX5_ADDR_OF(menu_resource_dump_response, data, menu);
+- num_of_items = MLX5_GET(resource_dump_menu_segment, menu, num_of_records);
++ if (!start_idx) {
++ menu = MLX5_ADDR_OF(menu_resource_dump_response, data, menu);
++ rsc_dump->number_of_menu_items = MLX5_GET(resource_dump_menu_segment, menu,
++ num_of_records);
++ size = MLX5_RSC_DUMP_MENU_HEADER_SIZE;
++ data += size;
++ }
++ num_of_items = rsc_dump->number_of_menu_items;
++
++ for (i = 0; start_idx + i < num_of_items; i++) {
++ size += MLX5_ST_SZ_BYTES(resource_dump_menu_record);
++ if (size >= read_size)
++ return start_idx + i;
+
+- for (i = 0; i < num_of_items; i++) {
+- member = MLX5_ADDR_OF(resource_dump_menu_segment, menu, record[i]);
++ member = data + MLX5_ST_SZ_BYTES(resource_dump_menu_record) * i;
+ sgmt_name = MLX5_ADDR_OF(resource_dump_menu_record, member, segment_name);
+ sgmt_idx = mlx5_rsc_dump_sgmt_get_by_name(sgmt_name);
+ if (sgmt_idx == -EINVAL)
+@@ -72,6 +89,7 @@ static void mlx5_rsc_dump_read_menu_sgmt(struct mlx5_rsc_dump *rsc_dump, struct
+ rsc_dump->fw_segment_type[sgmt_idx] = MLX5_GET(resource_dump_menu_record,
+ member, segment_type);
+ }
++ return 0;
+ }
+
+ static int mlx5_rsc_dump_trigger(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd,
+@@ -168,6 +186,7 @@ static int mlx5_rsc_dump_menu(struct mlx5_core_dev *dev)
+ struct mlx5_rsc_dump_cmd *cmd = NULL;
+ struct mlx5_rsc_key key = {};
+ struct page *page;
++ int start_idx = 0;
+ int size;
+ int err;
+
+@@ -189,7 +208,7 @@ static int mlx5_rsc_dump_menu(struct mlx5_core_dev *dev)
+ if (err < 0)
+ goto destroy_cmd;
+
+- mlx5_rsc_dump_read_menu_sgmt(dev->rsc_dump, page);
++ start_idx = mlx5_rsc_dump_read_menu_sgmt(dev->rsc_dump, page, size, start_idx);
+
+ } while (err > 0);
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
+index 03a7a4ce5cd5e..7204bc86e4741 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
+@@ -244,6 +244,17 @@ enum mlx5e_priv_flag {
+
+ #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
+
++enum packet_merge {
++ MLX5E_PACKET_MERGE_NONE,
++ MLX5E_PACKET_MERGE_LRO,
++ MLX5E_PACKET_MERGE_SHAMPO,
++};
++
++struct mlx5e_packet_merge_param {
++ enum packet_merge type;
++ u32 timeout;
++};
++
+ struct mlx5e_params {
+ u8 log_sq_size;
+ u8 rq_wq_type;
+@@ -258,13 +269,12 @@ struct mlx5e_params {
+ bool tunneled_offload_en;
+ struct dim_cq_moder rx_cq_moderation;
+ struct dim_cq_moder tx_cq_moderation;
+- bool lro_en;
++ struct mlx5e_packet_merge_param packet_merge;
+ u8 tx_min_inline_mode;
+ bool vlan_strip_disable;
+ bool scatter_fcs_en;
+ bool rx_dim_enabled;
+ bool tx_dim_enabled;
+- u32 lro_timeout;
+ u32 pflags;
+ struct bpf_prog *xdp_prog;
+ struct mlx5e_xsk *xsk;
+@@ -717,6 +727,8 @@ struct mlx5e_channel {
+ DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
+ int ix;
+ int cpu;
++ /* Sync between icosq recovery and XSK enable/disable. */
++ struct mutex icosq_recovery_lock;
+ };
+
+ struct mlx5e_ptp;
+@@ -944,9 +956,6 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param);
+ void mlx5e_destroy_rq(struct mlx5e_rq *rq);
+
+ struct mlx5e_sq_param;
+-int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
+- struct mlx5e_sq_param *param, struct mlx5e_icosq *sq);
+-void mlx5e_close_icosq(struct mlx5e_icosq *sq);
+ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
+ struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
+ struct mlx5e_xdpsq *sq, bool is_redirect);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
+index 018262d0164b3..3aaf3c2752feb 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
+@@ -30,6 +30,8 @@ void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv);
+ void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq);
+ void mlx5e_reporter_rq_cqe_err(struct mlx5e_rq *rq);
+ void mlx5e_reporter_rx_timeout(struct mlx5e_rq *rq);
++void mlx5e_reporter_icosq_suspend_recovery(struct mlx5e_channel *c);
++void mlx5e_reporter_icosq_resume_recovery(struct mlx5e_channel *c);
+
+ #define MLX5E_REPORTER_PER_Q_MAX_LEN 256
+ #define MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC 2000
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+index 3cbb596821e89..15f441a1b80c2 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+@@ -87,7 +87,8 @@ bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
+ u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk),
+ mlx5e_rx_get_linear_frag_sz(params, NULL));
+
+- return !params->lro_en && linear_frag_sz <= PAGE_SIZE;
++ return params->packet_merge.type == MLX5E_PACKET_MERGE_NONE &&
++ linear_frag_sz <= PAGE_SIZE;
+ }
+
+ bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
+@@ -164,19 +165,8 @@ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
+ mlx5e_rx_is_linear_skb(params, xsk) :
+ mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
+
+- return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0;
+-}
+-
+-struct mlx5e_lro_param mlx5e_get_lro_param(struct mlx5e_params *params)
+-{
+- struct mlx5e_lro_param lro_param;
+-
+- lro_param = (struct mlx5e_lro_param) {
+- .enabled = params->lro_en,
+- .timeout = params->lro_timeout,
+- };
+-
+- return lro_param;
++ return is_linear_skb || params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO ?
++ mlx5e_get_linear_rq_headroom(params, xsk) : 0;
+ }
+
+ u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
+@@ -485,10 +475,11 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
+
+ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
+ {
++ bool lro_en = params->packet_merge.type == MLX5E_PACKET_MERGE_LRO;
+ bool ro = pcie_relaxed_ordering_enabled(mdev->pdev) &&
+ MLX5_CAP_GEN(mdev, relaxed_ordering_write);
+
+- return ro && params->lro_en ?
++ return ro && lro_en ?
+ MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+index 879ad46d754e1..e9593f5f06610 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+@@ -11,11 +11,6 @@ struct mlx5e_xsk_param {
+ u16 chunk_size;
+ };
+
+-struct mlx5e_lro_param {
+- bool enabled;
+- u32 timeout;
+-};
+-
+ struct mlx5e_cq_param {
+ u32 cqc[MLX5_ST_SZ_DW(cqc)];
+ struct mlx5_wq_param wq;
+@@ -125,7 +120,6 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
+ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
+ struct mlx5e_params *params,
+ struct mlx5e_xsk_param *xsk);
+-struct mlx5e_lro_param mlx5e_get_lro_param(struct mlx5e_params *params);
+
+ /* Build queue parameters */
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
+index 673f1c82d3815..c9d5d8d93994d 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
+@@ -309,8 +309,8 @@ int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv,
+ if (err)
+ return err;
+
+- err = update_buffer_lossy(max_mtu, curr_pfc_en, prio2buffer, port_buff_cell_sz,
+- xoff, &port_buffer, &update_buffer);
++ err = update_buffer_lossy(max_mtu, curr_pfc_en, prio2buffer, xoff,
++ port_buff_cell_sz, &port_buffer, &update_buffer);
+ if (err)
+ return err;
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
+index e8a8d78e3e4d5..965838893432d 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c
+@@ -553,7 +553,8 @@ static int mlx5e_htb_convert_rate(struct mlx5e_priv *priv, u64 rate,
+
+ static void mlx5e_htb_convert_ceil(struct mlx5e_priv *priv, u64 ceil, u32 *max_average_bw)
+ {
+- *max_average_bw = div_u64(ceil, BYTES_IN_MBIT);
++ /* Hardware treats 0 as "unlimited", set at least 1. */
++ *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1);
+
+ qos_dbg(priv->mdev, "Convert: ceil %llu -> max_average_bw %u\n",
+ ceil, *max_average_bw);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c
+index 9c076aa20306a..b6f5c1bcdbcd4 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c
+@@ -183,18 +183,7 @@ void mlx5e_rep_bond_unslave(struct mlx5_eswitch *esw,
+
+ static bool mlx5e_rep_is_lag_netdev(struct net_device *netdev)
+ {
+- struct mlx5e_rep_priv *rpriv;
+- struct mlx5e_priv *priv;
+-
+- /* A given netdev is not a representor or not a slave of LAG configuration */
+- if (!mlx5e_eswitch_rep(netdev) || !netif_is_lag_port(netdev))
+- return false;
+-
+- priv = netdev_priv(netdev);
+- rpriv = priv->ppriv;
+-
+- /* Egress acl forward to vport is supported only non-uplink representor */
+- return rpriv->rep->vport != MLX5_VPORT_UPLINK;
++ return netif_is_lag_port(netdev) && mlx5e_eswitch_vf_rep(netdev);
+ }
+
+ static void mlx5e_rep_changelowerstate_event(struct net_device *netdev, void *ptr)
+@@ -210,9 +199,6 @@ static void mlx5e_rep_changelowerstate_event(struct net_device *netdev, void *pt
+ u16 fwd_vport_num;
+ int err;
+
+- if (!mlx5e_rep_is_lag_netdev(netdev))
+- return;
+-
+ info = ptr;
+ lag_info = info->lower_state_info;
+ /* This is not an event of a representor becoming active slave */
+@@ -266,9 +252,6 @@ static void mlx5e_rep_changeupper_event(struct net_device *netdev, void *ptr)
+ struct net_device *lag_dev;
+ struct mlx5e_priv *priv;
+
+- if (!mlx5e_rep_is_lag_netdev(netdev))
+- return;
+-
+ priv = netdev_priv(netdev);
+ rpriv = priv->ppriv;
+ lag_dev = info->upper_dev;
+@@ -293,6 +276,19 @@ static int mlx5e_rep_esw_bond_netevent(struct notifier_block *nb,
+ unsigned long event, void *ptr)
+ {
+ struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
++ struct mlx5e_rep_priv *rpriv;
++ struct mlx5e_rep_bond *bond;
++ struct mlx5e_priv *priv;
++
++ if (!mlx5e_rep_is_lag_netdev(netdev))
++ return NOTIFY_DONE;
++
++ bond = container_of(nb, struct mlx5e_rep_bond, nb);
++ priv = netdev_priv(netdev);
++ rpriv = mlx5_eswitch_get_uplink_priv(priv->mdev->priv.eswitch, REP_ETH);
++ /* Verify VF representor is on the same device of the bond handling the netevent. */
++ if (rpriv->uplink_priv.bond != bond)
++ return NOTIFY_DONE;
+
+ switch (event) {
+ case NETDEV_CHANGELOWERSTATE:
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
+index c6d2f8c78db71..48dc121b2cb4c 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
+@@ -491,7 +491,7 @@ void mlx5e_rep_bridge_init(struct mlx5e_priv *priv)
+ }
+
+ br_offloads->netdev_nb.notifier_call = mlx5_esw_bridge_switchdev_port_event;
+- err = register_netdevice_notifier(&br_offloads->netdev_nb);
++ err = register_netdevice_notifier_net(&init_net, &br_offloads->netdev_nb);
+ if (err) {
+ esw_warn(mdev, "Failed to register bridge offloads netdevice notifier (err=%d)\n",
+ err);
+@@ -509,7 +509,9 @@ err_register_swdev_blk:
+ err_register_swdev:
+ destroy_workqueue(br_offloads->wq);
+ err_alloc_wq:
++ rtnl_lock();
+ mlx5_esw_bridge_cleanup(esw);
++ rtnl_unlock();
+ }
+
+ void mlx5e_rep_bridge_cleanup(struct mlx5e_priv *priv)
+@@ -524,7 +526,7 @@ void mlx5e_rep_bridge_cleanup(struct mlx5e_priv *priv)
+ return;
+
+ cancel_delayed_work_sync(&br_offloads->update_work);
+- unregister_netdevice_notifier(&br_offloads->netdev_nb);
++ unregister_netdevice_notifier_net(&init_net, &br_offloads->netdev_nb);
+ unregister_switchdev_blocking_notifier(&br_offloads->nb_blk);
+ unregister_switchdev_notifier(&br_offloads->nb);
+ destroy_workqueue(br_offloads->wq);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
+index de03684528bbf..8451940c16ab9 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
+@@ -647,9 +647,7 @@ static void mlx5e_restore_skb_sample(struct mlx5e_priv *priv, struct sk_buff *sk
+ "Failed to restore tunnel info for sampled packet\n");
+ return;
+ }
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ mlx5e_tc_sample_skb(skb, mapped_obj);
+-#endif /* CONFIG_MLX5_TC_SAMPLE */
+ mlx5_rep_tc_post_napi_receive(tc_priv);
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+index 0eb125316fe20..e329158fdc555 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+@@ -59,6 +59,7 @@ static void mlx5e_reset_icosq_cc_pc(struct mlx5e_icosq *icosq)
+
+ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
+ {
++ struct mlx5e_rq *xskrq = NULL;
+ struct mlx5_core_dev *mdev;
+ struct mlx5e_icosq *icosq;
+ struct net_device *dev;
+@@ -67,7 +68,13 @@ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
+ int err;
+
+ icosq = ctx;
++
++ mutex_lock(&icosq->channel->icosq_recovery_lock);
++
++ /* mlx5e_close_rq cancels this work before RQ and ICOSQ are killed. */
+ rq = &icosq->channel->rq;
++ if (test_bit(MLX5E_RQ_STATE_ENABLED, &icosq->channel->xskrq.state))
++ xskrq = &icosq->channel->xskrq;
+ mdev = icosq->channel->mdev;
+ dev = icosq->channel->netdev;
+ err = mlx5_core_query_sq_state(mdev, icosq->sqn, &state);
+@@ -81,6 +88,9 @@ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
+ goto out;
+
+ mlx5e_deactivate_rq(rq);
++ if (xskrq)
++ mlx5e_deactivate_rq(xskrq);
++
+ err = mlx5e_wait_for_icosq_flush(icosq);
+ if (err)
+ goto out;
+@@ -94,15 +104,28 @@ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
+ goto out;
+
+ mlx5e_reset_icosq_cc_pc(icosq);
++
+ mlx5e_free_rx_in_progress_descs(rq);
++ if (xskrq)
++ mlx5e_free_rx_in_progress_descs(xskrq);
++
+ clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state);
+ mlx5e_activate_icosq(icosq);
+- mlx5e_activate_rq(rq);
+
++ mlx5e_activate_rq(rq);
+ rq->stats->recover++;
++
++ if (xskrq) {
++ mlx5e_activate_rq(xskrq);
++ xskrq->stats->recover++;
++ }
++
++ mutex_unlock(&icosq->channel->icosq_recovery_lock);
++
+ return 0;
+ out:
+ clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state);
++ mutex_unlock(&icosq->channel->icosq_recovery_lock);
+ return err;
+ }
+
+@@ -703,6 +726,16 @@ void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq)
+ mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx);
+ }
+
++void mlx5e_reporter_icosq_suspend_recovery(struct mlx5e_channel *c)
++{
++ mutex_lock(&c->icosq_recovery_lock);
++}
++
++void mlx5e_reporter_icosq_resume_recovery(struct mlx5e_channel *c)
++{
++ mutex_unlock(&c->icosq_recovery_lock);
++}
++
+ static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = {
+ .name = "rx",
+ .recover = mlx5e_rx_reporter_recover,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
+index bb682fd751c98..8024599994642 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
+@@ -463,6 +463,14 @@ static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fms
+ return mlx5e_health_fmsg_named_obj_nest_end(fmsg);
+ }
+
++static int mlx5e_tx_reporter_timeout_dump(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg,
++ void *ctx)
++{
++ struct mlx5e_tx_timeout_ctx *to_ctx = ctx;
++
++ return mlx5e_tx_reporter_dump_sq(priv, fmsg, to_ctx->sq);
++}
++
+ static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv,
+ struct devlink_fmsg *fmsg)
+ {
+@@ -558,7 +566,7 @@ int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq)
+ to_ctx.sq = sq;
+ err_ctx.ctx = &to_ctx;
+ err_ctx.recover = mlx5e_tx_reporter_timeout_recover;
+- err_ctx.dump = mlx5e_tx_reporter_dump_sq;
++ err_ctx.dump = mlx5e_tx_reporter_timeout_dump;
+ snprintf(err_str, sizeof(err_str),
+ "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u",
+ sq->ch_ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
+index 625cd49ef96c5..7b55b14d47ef7 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
+@@ -127,7 +127,7 @@ mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
+
+ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss,
+ enum mlx5_traffic_types tt,
+- const struct mlx5e_lro_param *init_lro_param,
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ bool inner)
+ {
+ struct mlx5e_rss_params_traffic_type rss_tt;
+@@ -161,7 +161,7 @@ static int mlx5e_rss_create_tir(struct mlx5e_rss *rss,
+ rqtn = mlx5e_rqt_get_rqtn(&rss->rqt);
+ mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn,
+ rqtn, rss->inner_ft_support);
+- mlx5e_tir_builder_build_lro(builder, init_lro_param);
++ mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param);
+ rss_tt = mlx5e_rss_get_tt_config(rss, tt);
+ mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
+
+@@ -198,14 +198,14 @@ static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types
+ }
+
+ static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss,
+- const struct mlx5e_lro_param *init_lro_param,
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ bool inner)
+ {
+ enum mlx5_traffic_types tt, max_tt;
+ int err;
+
+ for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
+- err = mlx5e_rss_create_tir(rss, tt, init_lro_param, inner);
++ err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
+ if (err)
+ goto err_destroy_tirs;
+ }
+@@ -297,7 +297,7 @@ int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
+
+ int mlx5e_rss_init(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
+ bool inner_ft_support, u32 drop_rqn,
+- const struct mlx5e_lro_param *init_lro_param)
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param)
+ {
+ int err;
+
+@@ -305,12 +305,12 @@ int mlx5e_rss_init(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
+ if (err)
+ goto err_out;
+
+- err = mlx5e_rss_create_tirs(rss, init_lro_param, false);
++ err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, false);
+ if (err)
+ goto err_destroy_rqt;
+
+ if (inner_ft_support) {
+- err = mlx5e_rss_create_tirs(rss, init_lro_param, true);
++ err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, true);
+ if (err)
+ goto err_destroy_tirs;
+ }
+@@ -372,7 +372,7 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
+ */
+ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss,
+ enum mlx5_traffic_types tt,
+- const struct mlx5e_lro_param *init_lro_param,
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ bool inner, u32 *tirn)
+ {
+ struct mlx5e_tir *tir;
+@@ -381,7 +381,7 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss,
+ if (!tir) { /* TIR doesn't exist, create one */
+ int err;
+
+- err = mlx5e_rss_create_tir(rss, tt, init_lro_param, inner);
++ err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
+ if (err)
+ return err;
+ tir = rss_get_tir(rss, tt, inner);
+@@ -418,7 +418,8 @@ void mlx5e_rss_disable(struct mlx5e_rss *rss)
+ mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err);
+ }
+
+-int mlx5e_rss_lro_set_param(struct mlx5e_rss *rss, struct mlx5e_lro_param *lro_param)
++int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss,
++ struct mlx5e_packet_merge_param *pkt_merge_param)
+ {
+ struct mlx5e_tir_builder *builder;
+ enum mlx5_traffic_types tt;
+@@ -428,7 +429,7 @@ int mlx5e_rss_lro_set_param(struct mlx5e_rss *rss, struct mlx5e_lro_param *lro_p
+ if (!builder)
+ return -ENOMEM;
+
+- mlx5e_tir_builder_build_lro(builder, lro_param);
++ mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
+
+ final_err = 0;
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
+index d522a10dadf33..c6b2164163440 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
+@@ -17,7 +17,7 @@ struct mlx5e_rss *mlx5e_rss_alloc(void);
+ void mlx5e_rss_free(struct mlx5e_rss *rss);
+ int mlx5e_rss_init(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
+ bool inner_ft_support, u32 drop_rqn,
+- const struct mlx5e_lro_param *init_lro_param);
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param);
+ int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev,
+ bool inner_ft_support, u32 drop_rqn);
+ int mlx5e_rss_cleanup(struct mlx5e_rss *rss);
+@@ -30,13 +30,14 @@ u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
+ bool inner);
+ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss,
+ enum mlx5_traffic_types tt,
+- const struct mlx5e_lro_param *init_lro_param,
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ bool inner, u32 *tirn);
+
+ void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns);
+ void mlx5e_rss_disable(struct mlx5e_rss *rss);
+
+-int mlx5e_rss_lro_set_param(struct mlx5e_rss *rss, struct mlx5e_lro_param *lro_param);
++int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss,
++ struct mlx5e_packet_merge_param *pkt_merge_param);
+ int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc);
+ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir,
+ const u8 *key, const u8 *hfunc,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
+index 13056cb9757d4..0015a81eb9a17 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
+@@ -13,6 +13,9 @@ struct mlx5e_rx_res {
+ unsigned int max_nch;
+ u32 drop_rqn;
+
++ struct mlx5e_packet_merge_param pkt_merge_param;
++ struct rw_semaphore pkt_merge_param_sem;
++
+ struct mlx5e_rss *rss[MLX5E_MAX_NUM_RSS];
+ bool rss_active;
+ u32 rss_rqns[MLX5E_INDIR_RQT_SIZE];
+@@ -34,7 +37,7 @@ struct mlx5e_rx_res {
+ /* API for rx_res_rss_* */
+
+ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res,
+- const struct mlx5e_lro_param *init_lro_param,
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ unsigned int init_nch)
+ {
+ bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT;
+@@ -49,7 +52,7 @@ static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res,
+ return -ENOMEM;
+
+ err = mlx5e_rss_init(rss, res->mdev, inner_ft_support, res->drop_rqn,
+- init_lro_param);
++ init_pkt_merge_param);
+ if (err)
+ goto err_rss_free;
+
+@@ -275,7 +278,7 @@ struct mlx5e_rx_res *mlx5e_rx_res_alloc(void)
+ }
+
+ static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res,
+- const struct mlx5e_lro_param *init_lro_param)
++ const struct mlx5e_packet_merge_param *init_pkt_merge_param)
+ {
+ bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT;
+ struct mlx5e_tir_builder *builder;
+@@ -306,7 +309,7 @@ static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res,
+ mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn,
+ mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
+ inner_ft_support);
+- mlx5e_tir_builder_build_lro(builder, init_lro_param);
++ mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param);
+ mlx5e_tir_builder_build_direct(builder);
+
+ err = mlx5e_tir_init(&res->channels[ix].direct_tir, builder, res->mdev, true);
+@@ -336,7 +339,7 @@ static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res,
+ mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn,
+ mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
+ inner_ft_support);
+- mlx5e_tir_builder_build_lro(builder, init_lro_param);
++ mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param);
+ mlx5e_tir_builder_build_direct(builder);
+
+ err = mlx5e_tir_init(&res->channels[ix].xsk_tir, builder, res->mdev, true);
+@@ -392,6 +395,7 @@ static int mlx5e_rx_res_ptp_init(struct mlx5e_rx_res *res)
+ if (err)
+ goto out;
+
++ /* Separated from the channels RQs, does not share pkt_merge state with them */
+ mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn,
+ mlx5e_rqt_get_rqtn(&res->ptp.rqt),
+ inner_ft_support);
+@@ -437,7 +441,7 @@ static void mlx5e_rx_res_ptp_destroy(struct mlx5e_rx_res *res)
+
+ int mlx5e_rx_res_init(struct mlx5e_rx_res *res, struct mlx5_core_dev *mdev,
+ enum mlx5e_rx_res_features features, unsigned int max_nch,
+- u32 drop_rqn, const struct mlx5e_lro_param *init_lro_param,
++ u32 drop_rqn, const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ unsigned int init_nch)
+ {
+ int err;
+@@ -447,11 +451,14 @@ int mlx5e_rx_res_init(struct mlx5e_rx_res *res, struct mlx5_core_dev *mdev,
+ res->max_nch = max_nch;
+ res->drop_rqn = drop_rqn;
+
+- err = mlx5e_rx_res_rss_init_def(res, init_lro_param, init_nch);
++ res->pkt_merge_param = *init_pkt_merge_param;
++ init_rwsem(&res->pkt_merge_param_sem);
++
++ err = mlx5e_rx_res_rss_init_def(res, init_pkt_merge_param, init_nch);
+ if (err)
+ goto err_out;
+
+- err = mlx5e_rx_res_channels_init(res, init_lro_param);
++ err = mlx5e_rx_res_channels_init(res, init_pkt_merge_param);
+ if (err)
+ goto err_rss_destroy;
+
+@@ -513,7 +520,7 @@ u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res)
+ return mlx5e_tir_get_tirn(&res->ptp.tir);
+ }
+
+-u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int ix)
++static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int ix)
+ {
+ return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt);
+ }
+@@ -645,7 +652,8 @@ int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix)
+ return err;
+ }
+
+-int mlx5e_rx_res_lro_set_param(struct mlx5e_rx_res *res, struct mlx5e_lro_param *lro_param)
++int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res,
++ struct mlx5e_packet_merge_param *pkt_merge_param)
+ {
+ struct mlx5e_tir_builder *builder;
+ int err, final_err;
+@@ -655,7 +663,10 @@ int mlx5e_rx_res_lro_set_param(struct mlx5e_rx_res *res, struct mlx5e_lro_param
+ if (!builder)
+ return -ENOMEM;
+
+- mlx5e_tir_builder_build_lro(builder, lro_param);
++ down_write(&res->pkt_merge_param_sem);
++ res->pkt_merge_param = *pkt_merge_param;
++
++ mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
+
+ final_err = 0;
+
+@@ -665,7 +676,7 @@ int mlx5e_rx_res_lro_set_param(struct mlx5e_rx_res *res, struct mlx5e_lro_param
+ if (!rss)
+ continue;
+
+- err = mlx5e_rss_lro_set_param(rss, lro_param);
++ err = mlx5e_rss_packet_merge_set_param(rss, pkt_merge_param);
+ if (err)
+ final_err = final_err ? : err;
+ }
+@@ -673,13 +684,14 @@ int mlx5e_rx_res_lro_set_param(struct mlx5e_rx_res *res, struct mlx5e_lro_param
+ for (ix = 0; ix < res->max_nch; ix++) {
+ err = mlx5e_tir_modify(&res->channels[ix].direct_tir, builder);
+ if (err) {
+- mlx5_core_warn(res->mdev, "Failed to update LRO state of direct TIR %#x for channel %u: err = %d\n",
++ mlx5_core_warn(res->mdev, "Failed to update packet merge state of direct TIR %#x for channel %u: err = %d\n",
+ mlx5e_tir_get_tirn(&res->channels[ix].direct_tir), ix, err);
+ if (!final_err)
+ final_err = err;
+ }
+ }
+
++ up_write(&res->pkt_merge_param_sem);
+ mlx5e_tir_builder_free(builder);
+ return final_err;
+ }
+@@ -688,3 +700,31 @@ struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *
+ {
+ return mlx5e_rss_get_hash(res->rss[0]);
+ }
++
++int mlx5e_rx_res_tls_tir_create(struct mlx5e_rx_res *res, unsigned int rxq,
++ struct mlx5e_tir *tir)
++{
++ bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT;
++ struct mlx5e_tir_builder *builder;
++ u32 rqtn;
++ int err;
++
++ builder = mlx5e_tir_builder_alloc(false);
++ if (!builder)
++ return -ENOMEM;
++
++ rqtn = mlx5e_rx_res_get_rqtn_direct(res, rxq);
++
++ mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, rqtn,
++ inner_ft_support);
++ mlx5e_tir_builder_build_direct(builder);
++ mlx5e_tir_builder_build_tls(builder);
++ down_read(&res->pkt_merge_param_sem);
++ mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param);
++ err = mlx5e_tir_init(tir, builder, res->mdev, false);
++ up_read(&res->pkt_merge_param_sem);
++
++ mlx5e_tir_builder_free(builder);
++
++ return err;
++}
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
+index 4a15942d79f7d..b39b20a720e0f 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
+@@ -25,7 +25,7 @@ enum mlx5e_rx_res_features {
+ struct mlx5e_rx_res *mlx5e_rx_res_alloc(void);
+ int mlx5e_rx_res_init(struct mlx5e_rx_res *res, struct mlx5_core_dev *mdev,
+ enum mlx5e_rx_res_features features, unsigned int max_nch,
+- u32 drop_rqn, const struct mlx5e_lro_param *init_lro_param,
++ u32 drop_rqn, const struct mlx5e_packet_merge_param *init_pkt_merge_param,
+ unsigned int init_nch);
+ void mlx5e_rx_res_destroy(struct mlx5e_rx_res *res);
+ void mlx5e_rx_res_free(struct mlx5e_rx_res *res);
+@@ -37,9 +37,6 @@ u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types
+ u32 mlx5e_rx_res_get_tirn_rss_inner(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt);
+ u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res);
+
+-/* RQTN getters for modules that create their own TIRs */
+-u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int ix);
+-
+ /* Activate/deactivate API */
+ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs);
+ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res);
+@@ -57,7 +54,8 @@ int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx,
+ u8 mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt);
+ int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt,
+ u8 rx_hash_fields);
+-int mlx5e_rx_res_lro_set_param(struct mlx5e_rx_res *res, struct mlx5e_lro_param *lro_param);
++int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res,
++ struct mlx5e_packet_merge_param *pkt_merge_param);
+
+ int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 *rss_idx, unsigned int init_nch);
+ int mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx);
+@@ -68,4 +66,7 @@ struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx);
+ /* Workaround for hairpin */
+ struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *res);
+
++/* Accel TIRs */
++int mlx5e_rx_res_tls_tir_create(struct mlx5e_rx_res *res, unsigned int rxq,
++ struct mlx5e_tir *tir);
+ #endif /* __MLX5_EN_RX_RES_H__ */
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.h
+index db0146df9b303..9ef8a49d78014 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.h
+@@ -19,6 +19,8 @@ struct mlx5e_sample_attr {
+ struct mlx5e_sample_flow *sample_flow;
+ };
+
++#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
++
+ void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj);
+
+ struct mlx5_flow_handle *
+@@ -38,4 +40,29 @@ mlx5e_tc_sample_init(struct mlx5_eswitch *esw, struct mlx5e_post_act *post_act);
+ void
+ mlx5e_tc_sample_cleanup(struct mlx5e_tc_psample *tc_psample);
+
++#else /* CONFIG_MLX5_TC_SAMPLE */
++
++static inline struct mlx5_flow_handle *
++mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
++ struct mlx5_flow_spec *spec,
++ struct mlx5_flow_attr *attr,
++ u32 tunnel_id)
++{ return ERR_PTR(-EOPNOTSUPP); }
++
++static inline void
++mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample *tc_psample,
++ struct mlx5_flow_handle *rule,
++ struct mlx5_flow_attr *attr) {}
++
++static inline struct mlx5e_tc_psample *
++mlx5e_tc_sample_init(struct mlx5_eswitch *esw, struct mlx5e_post_act *post_act)
++{ return ERR_PTR(-EOPNOTSUPP); }
++
++static inline void
++mlx5e_tc_sample_cleanup(struct mlx5e_tc_psample *tc_psample) {}
++
++static inline void
++mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj) {}
++
++#endif /* CONFIG_MLX5_TC_SAMPLE */
+ #endif /* __MLX5_EN_TC_SAMPLE_H__ */
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
+index 6c949abcd2e14..d4b7b4d73b08e 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
+@@ -1356,9 +1356,13 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
+ int
+ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
+ struct mlx5_flow_attr *attr,
++ struct mlx5e_tc_mod_hdr_acts *mod_acts,
+ const struct flow_action_entry *act,
+ struct netlink_ext_ack *extack)
+ {
++ bool clear_action = act->ct.action & TCA_CT_ACT_CLEAR;
++ int err;
++
+ if (!priv) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "offload of ct action isn't available");
+@@ -1369,6 +1373,17 @@ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
+ attr->ct_attr.ct_action = act->ct.action;
+ attr->ct_attr.nf_ft = act->ct.flow_table;
+
++ if (!clear_action)
++ goto out;
++
++ err = mlx5_tc_ct_entry_set_registers(priv, mod_acts, 0, 0, 0, 0);
++ if (err) {
++ NL_SET_ERR_MSG_MOD(extack, "Failed to set registers for ct clear");
++ return err;
++ }
++ attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
++
++out:
+ return 0;
+ }
+
+@@ -1684,6 +1699,8 @@ mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg)
+ static void
+ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
+ {
++ struct mlx5e_priv *priv;
++
+ if (!refcount_dec_and_test(&ft->refcount))
+ return;
+
+@@ -1693,6 +1710,8 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
+ rhashtable_free_and_destroy(&ft->ct_entries_ht,
+ mlx5_tc_ct_flush_ft_entry,
+ ct_priv);
++ priv = netdev_priv(ct_priv->netdev);
++ flush_workqueue(priv->wq);
+ mlx5_tc_ct_free_pre_ct_tables(ft);
+ mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
+ kfree(ft);
+@@ -1898,23 +1917,16 @@ __mlx5_tc_ct_flow_offload_clear(struct mlx5_tc_ct_priv *ct_priv,
+
+ memcpy(pre_ct_attr, attr, attr_sz);
+
+- err = mlx5_tc_ct_entry_set_registers(ct_priv, mod_acts, 0, 0, 0, 0);
+- if (err) {
+- ct_dbg("Failed to set register for ct clear");
+- goto err_set_registers;
+- }
+-
+ mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type,
+ mod_acts->num_actions,
+ mod_acts->actions);
+ if (IS_ERR(mod_hdr)) {
+ err = PTR_ERR(mod_hdr);
+ ct_dbg("Failed to add create ct clear mod hdr");
+- goto err_set_registers;
++ goto err_mod_hdr;
+ }
+
+ pre_ct_attr->modify_hdr = mod_hdr;
+- pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+
+ rule = mlx5_tc_rule_insert(priv, orig_spec, pre_ct_attr);
+ if (IS_ERR(rule)) {
+@@ -1930,7 +1942,7 @@ __mlx5_tc_ct_flow_offload_clear(struct mlx5_tc_ct_priv *ct_priv,
+
+ err_insert:
+ mlx5_modify_header_dealloc(priv->mdev, mod_hdr);
+-err_set_registers:
++err_mod_hdr:
+ netdev_warn(priv->netdev,
+ "Failed to offload ct clear flow, err %d\n", err);
+ kfree(pre_ct_attr);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
+index 363329f4aac61..99662af1e41a7 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
+@@ -110,6 +110,7 @@ int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec);
+ int
+ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
+ struct mlx5_flow_attr *attr,
++ struct mlx5e_tc_mod_hdr_acts *mod_acts,
+ const struct flow_action_entry *act,
+ struct netlink_ext_ack *extack);
+
+@@ -172,6 +173,7 @@ mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec)
+ static inline int
+ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
+ struct mlx5_flow_attr *attr,
++ struct mlx5e_tc_mod_hdr_acts *mod_acts,
+ const struct flow_action_entry *act,
+ struct netlink_ext_ack *extack)
+ {
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
+index d1599b7b944bf..c340bf90354a0 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
+@@ -102,6 +102,7 @@ struct mlx5e_tc_flow {
+ refcount_t refcnt;
+ struct rcu_head rcu_head;
+ struct completion init_done;
++ struct completion del_hw_done;
+ int tunnel_id; /* the mapped tunnel id of this flow */
+ struct mlx5_flow_attr *attr;
+ };
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c
+index 4a13ef561587d..cf03297c84710 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c
+@@ -1,6 +1,7 @@
+ /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+ /* Copyright (c) 2018 Mellanox Technologies. */
+
++#include <net/inet_ecn.h>
+ #include <net/vxlan.h>
+ #include <net/gre.h>
+ #include <net/geneve.h>
+@@ -229,7 +230,7 @@ int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
+ int err;
+
+ /* add the IP fields */
+- attr.fl.fl4.flowi4_tos = tun_key->tos;
++ attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK;
+ attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
+ attr.fl.fl4.saddr = tun_key->u.ipv4.src;
+ attr.ttl = tun_key->ttl;
+@@ -344,7 +345,7 @@ int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
+ int err;
+
+ /* add the IP fields */
+- attr.fl.fl4.flowi4_tos = tun_key->tos;
++ attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK;
+ attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
+ attr.fl.fl4.saddr = tun_key->u.ipv4.src;
+ attr.ttl = tun_key->ttl;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
+index 1c44c6c345f5d..700c463ea3679 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
+@@ -221,8 +221,14 @@ static void mlx5e_take_tmp_flow(struct mlx5e_tc_flow *flow,
+ struct list_head *flow_list,
+ int index)
+ {
+- if (IS_ERR(mlx5e_flow_get(flow)))
++ if (IS_ERR(mlx5e_flow_get(flow))) {
++ /* Flow is being deleted concurrently. Wait for it to be
++ * unoffloaded from hardware, otherwise deleting encap will
++ * fail.
++ */
++ wait_for_completion(&flow->del_hw_done);
+ return;
++ }
+ wait_for_completion(&flow->init_done);
+
+ flow->tmp_entry_index = index;
+@@ -1538,6 +1544,8 @@ mlx5e_init_fib_work_ipv4(struct mlx5e_priv *priv,
+ struct net_device *fib_dev;
+
+ fen_info = container_of(info, struct fib_entry_notifier_info, info);
++ if (fen_info->fi->nh)
++ return NULL;
+ fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev;
+ if (!fib_dev || fib_dev->netdev_ops != &mlx5e_netdev_ops ||
+ fen_info->dst_len != 32)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_mplsoudp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_mplsoudp.c
+index 60952b33b5688..d2333310b56fe 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_mplsoudp.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_mplsoudp.c
+@@ -60,37 +60,31 @@ static int parse_tunnel(struct mlx5e_priv *priv,
+ void *headers_v)
+ {
+ struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+- struct flow_match_enc_keyid enc_keyid;
+ struct flow_match_mpls match;
+ void *misc2_c;
+ void *misc2_v;
+
+- misc2_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+- misc_parameters_2);
+- misc2_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+- misc_parameters_2);
+-
+- if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS))
+- return 0;
+-
+- if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID))
+- return 0;
+-
+- flow_rule_match_enc_keyid(rule, &enc_keyid);
+-
+- if (!enc_keyid.mask->keyid)
+- return 0;
+-
+ if (!MLX5_CAP_ETH(priv->mdev, tunnel_stateless_mpls_over_udp) &&
+ !(MLX5_CAP_GEN(priv->mdev, flex_parser_protocols) & MLX5_FLEX_PROTO_CW_MPLS_UDP))
+ return -EOPNOTSUPP;
+
++ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID))
++ return -EOPNOTSUPP;
++
++ if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS))
++ return 0;
++
+ flow_rule_match_mpls(rule, &match);
+
+ /* Only support matching the first LSE */
+ if (match.mask->used_lses != 1)
+ return -EOPNOTSUPP;
+
++ misc2_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
++ misc_parameters_2);
++ misc2_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
++ misc_parameters_2);
++
+ MLX5_SET(fte_match_set_misc2, misc2_c,
+ outer_first_mpls_over_udp.mpls_label,
+ match.mask->ls[0].mpls_label);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tir.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tir.c
+index de936dc4bc483..a1afb8585e37f 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tir.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tir.c
+@@ -70,24 +70,24 @@ void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn,
+ MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support);
+ }
+
+-void mlx5e_tir_builder_build_lro(struct mlx5e_tir_builder *builder,
+- const struct mlx5e_lro_param *lro_param)
++void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder,
++ const struct mlx5e_packet_merge_param *pkt_merge_param)
+ {
+ void *tirc = mlx5e_tir_builder_get_tirc(builder);
+ const unsigned int rough_max_l2_l3_hdr_sz = 256;
+
+ if (builder->modify)
+- MLX5_SET(modify_tir_in, builder->in, bitmask.lro, 1);
++ MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1);
+
+- if (!lro_param->enabled)
++ if (pkt_merge_param->type == MLX5E_PACKET_MERGE_NONE)
+ return;
+
+- MLX5_SET(tirc, tirc, lro_enable_mask,
+- MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
+- MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
++ MLX5_SET(tirc, tirc, packet_merge_mask,
++ MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO |
++ MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO);
+ MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
+ (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8);
+- MLX5_SET(tirc, tirc, lro_timeout_period_usecs, lro_param->timeout);
++ MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout);
+ }
+
+ static int mlx5e_hfunc_to_hw(u8 hfunc)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tir.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tir.h
+index e45149a78ed9d..857a84bcd53af 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tir.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tir.h
+@@ -18,7 +18,7 @@ struct mlx5e_rss_params_traffic_type {
+ };
+
+ struct mlx5e_tir_builder;
+-struct mlx5e_lro_param;
++struct mlx5e_packet_merge_param;
+
+ struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify);
+ void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder);
+@@ -27,8 +27,8 @@ void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder);
+ void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn);
+ void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn,
+ u32 rqtn, bool inner_ft_support);
+-void mlx5e_tir_builder_build_lro(struct mlx5e_tir_builder *builder,
+- const struct mlx5e_lro_param *lro_param);
++void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder,
++ const struct mlx5e_packet_merge_param *pkt_merge_param);
+ void mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder *builder,
+ const struct mlx5e_rss_params_hash *rss_hash,
+ const struct mlx5e_rss_params_traffic_type *rss_tt,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+index 7b562d2c8a196..279cd8f4e79f7 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+@@ -11,13 +11,13 @@ static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv,
+ {
+ struct device *dev = mlx5_core_dma_dev(priv->mdev);
+
+- return xsk_pool_dma_map(pool, dev, 0);
++ return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
+ }
+
+ static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
+ struct xsk_buff_pool *pool)
+ {
+- return xsk_pool_dma_unmap(pool, 0);
++ return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
+ }
+
+ static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+index 538bc2419bd83..8526a5fbbf0bf 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+@@ -4,6 +4,7 @@
+ #include "setup.h"
+ #include "en/params.h"
+ #include "en/txrx.h"
++#include "en/health.h"
+
+ /* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may
+ * change unexpectedly, and mlx5e has a minimum valid stride size for striding
+@@ -170,7 +171,13 @@ void mlx5e_close_xsk(struct mlx5e_channel *c)
+
+ void mlx5e_activate_xsk(struct mlx5e_channel *c)
+ {
++ /* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid
++ * activating XSKRQ in the middle of recovery.
++ */
++ mlx5e_reporter_icosq_suspend_recovery(c);
+ set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
++ mlx5e_reporter_icosq_resume_recovery(c);
++
+ /* TX queue is created active. */
+
+ spin_lock_bh(&c->async_icosq_lock);
+@@ -180,6 +187,13 @@ void mlx5e_activate_xsk(struct mlx5e_channel *c)
+
+ void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
+ {
+- mlx5e_deactivate_rq(&c->xskrq);
++ /* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the
++ * middle of recovery. Suspend the recovery to avoid it.
++ */
++ mlx5e_reporter_icosq_suspend_recovery(c);
++ clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
++ mlx5e_reporter_icosq_resume_recovery(c);
++ synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
++
+ /* TX queue is disabled on close. */
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+index fb5397324aa4f..b56fea142c246 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+@@ -157,11 +157,20 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb,
+ /* Tunnel mode */
+ if (mode == XFRM_MODE_TUNNEL) {
+ eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
+- eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
+ if (xo->proto == IPPROTO_IPV6)
+ eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
+- if (inner_ip_hdr(skb)->protocol == IPPROTO_UDP)
++
++ switch (xo->inner_ipproto) {
++ case IPPROTO_UDP:
+ eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
++ fallthrough;
++ case IPPROTO_TCP:
++ /* IP | ESP | IP | [TCP | UDP] */
++ eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
++ break;
++ default:
++ break;
++ }
+ return;
+ }
+
+@@ -191,7 +200,7 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb,
+ eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
+ eseg->swp_inner_l4_offset =
+ (skb->csum_start + skb->head - skb->data) / 2;
+- if (skb->protocol == htons(ETH_P_IPV6))
++ if (inner_ip_hdr(skb)->version == 6)
+ eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
+ break;
+ default:
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
+index 5120a59361e6a..428881e0adcbe 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
+@@ -127,6 +127,28 @@ out_disable:
+ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
+ }
+
++static inline bool
++mlx5e_ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb,
++ struct mlx5_wqe_eth_seg *eseg)
++{
++ u8 inner_ipproto;
++
++ if (!mlx5e_ipsec_eseg_meta(eseg))
++ return false;
++
++ eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
++ inner_ipproto = xfrm_offload(skb)->inner_ipproto;
++ if (inner_ipproto) {
++ eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM;
++ if (inner_ipproto == IPPROTO_TCP || inner_ipproto == IPPROTO_UDP)
++ eseg->cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM;
++ } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
++ eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
++ sq->stats->csum_partial_inner++;
++ }
++
++ return true;
++}
+ #else
+ static inline
+ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+@@ -143,6 +165,13 @@ static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false;
+ static inline netdev_features_t
+ mlx5e_ipsec_feature_check(struct sk_buff *skb, netdev_features_t features)
+ { return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); }
++
++static inline bool
++mlx5e_ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb,
++ struct mlx5_wqe_eth_seg *eseg)
++{
++ return false;
++}
+ #endif /* CONFIG_MLX5_EN_IPSEC */
+
+ #endif /* __MLX5E_IPSEC_RXTX_H__ */
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
+index 62abce008c7b8..15711814d2d28 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
+@@ -55,6 +55,7 @@ struct mlx5e_ktls_offload_context_rx {
+ DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
+
+ /* resync */
++ spinlock_t lock; /* protects resync fields */
+ struct mlx5e_ktls_rx_resync_ctx resync;
+ struct list_head list;
+ };
+@@ -99,25 +100,6 @@ mlx5e_ktls_rx_resync_create_resp_list(void)
+ return resp_list;
+ }
+
+-static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir, u32 rqtn)
+-{
+- struct mlx5e_tir_builder *builder;
+- int err;
+-
+- builder = mlx5e_tir_builder_alloc(false);
+- if (!builder)
+- return -ENOMEM;
+-
+- mlx5e_tir_builder_build_rqt(builder, mdev->mlx5e_res.hw_objs.td.tdn, rqtn, false);
+- mlx5e_tir_builder_build_direct(builder);
+- mlx5e_tir_builder_build_tls(builder);
+- err = mlx5e_tir_init(tir, builder, mdev, false);
+-
+- mlx5e_tir_builder_free(builder);
+-
+- return err;
+-}
+-
+ static void accel_rule_handle_work(struct work_struct *work)
+ {
+ struct mlx5e_ktls_offload_context_rx *priv_rx;
+@@ -386,14 +368,18 @@ static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_r
+ struct mlx5e_icosq *sq;
+ bool trigger_poll;
+
+- memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq));
+-
+ sq = &c->async_icosq;
+ ktls_resync = sq->ktls_resync;
++ trigger_poll = false;
+
+ spin_lock_bh(&ktls_resync->lock);
+- list_add_tail(&priv_rx->list, &ktls_resync->list);
+- trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
++ spin_lock_bh(&priv_rx->lock);
++ memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq));
++ if (list_empty(&priv_rx->list)) {
++ list_add_tail(&priv_rx->list, &ktls_resync->list);
++ trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
++ }
++ spin_unlock_bh(&priv_rx->lock);
+ spin_unlock_bh(&ktls_resync->lock);
+
+ if (!trigger_poll)
+@@ -604,7 +590,6 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
+ struct mlx5_core_dev *mdev;
+ struct mlx5e_priv *priv;
+ int rxq, err;
+- u32 rqtn;
+
+ tls_ctx = tls_get_ctx(sk);
+ priv = netdev_priv(netdev);
+@@ -617,6 +602,8 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
+ if (err)
+ goto err_create_key;
+
++ INIT_LIST_HEAD(&priv_rx->list);
++ spin_lock_init(&priv_rx->lock);
+ priv_rx->crypto_info =
+ *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+
+@@ -628,9 +615,7 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
+ priv_rx->sw_stats = &priv->tls->sw_stats;
+ mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx);
+
+- rqtn = mlx5e_rx_res_get_rqtn_direct(priv->rx_res, rxq);
+-
+- err = mlx5e_ktls_create_tir(mdev, &priv_rx->tir, rqtn);
++ err = mlx5e_rx_res_tls_tir_create(priv->rx_res, rxq, &priv_rx->tir);
+ if (err)
+ goto err_create_tir;
+
+@@ -730,10 +715,14 @@ bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget)
+ priv_rx = list_first_entry(&local_list,
+ struct mlx5e_ktls_offload_context_rx,
+ list);
++ spin_lock(&priv_rx->lock);
+ cseg = post_static_params(sq, priv_rx);
+- if (IS_ERR(cseg))
++ if (IS_ERR(cseg)) {
++ spin_unlock(&priv_rx->lock);
+ break;
+- list_del(&priv_rx->list);
++ }
++ list_del_init(&priv_rx->list);
++ spin_unlock(&priv_rx->lock);
+ db_cseg = cseg;
+ }
+ if (db_cseg)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+index a4c8d8d00d5a4..72e08559e0d05 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+@@ -1198,6 +1198,16 @@ static int mlx5e_trust_initialize(struct mlx5e_priv *priv)
+ if (err)
+ return err;
+
++ if (priv->dcbx_dp.trust_state == MLX5_QPTS_TRUST_PCP && priv->dcbx.dscp_app_cnt) {
++ /*
++ * Align the driver state with the register state.
++ * Temporary state change is required to enable the app list reset.
++ */
++ priv->dcbx_dp.trust_state = MLX5_QPTS_TRUST_DSCP;
++ mlx5e_dcbnl_delete_app(priv);
++ priv->dcbx_dp.trust_state = MLX5_QPTS_TRUST_PCP;
++ }
++
+ mlx5e_params_calc_trust_tx_min_inline_mode(priv->mdev, &priv->channels.params,
+ priv->dcbx_dp.trust_state);
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+index 9d451b8ee467c..2d3cd237355a6 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+@@ -1754,7 +1754,7 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev,
+ if (size_read < 0) {
+ netdev_err(priv->netdev, "%s: mlx5_query_eeprom failed:0x%x\n",
+ __func__, size_read);
+- return 0;
++ return size_read;
+ }
+
+ i += size_read;
+@@ -1954,8 +1954,8 @@ static int set_pflag_rx_striding_rq(struct net_device *netdev, bool enable)
+ return -EOPNOTSUPP;
+ if (!mlx5e_striding_rq_possible(mdev, &priv->channels.params))
+ return -EINVAL;
+- } else if (priv->channels.params.lro_en) {
+- netdev_warn(netdev, "Can't set legacy RQ with LRO, disable LRO first\n");
++ } else if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
++ netdev_warn(netdev, "Can't set legacy RQ with HW-GRO/LRO, disable them first\n");
+ return -EINVAL;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
+index 03693fa74a704..d32b70c62c949 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
+@@ -411,7 +411,7 @@ static int flow_get_tirn(struct mlx5e_priv *priv,
+ u32 rss_context, u32 *tirn)
+ {
+ if (fs->flow_type & FLOW_RSS) {
+- struct mlx5e_lro_param lro_param;
++ struct mlx5e_packet_merge_param pkt_merge_param;
+ struct mlx5e_rss *rss;
+ u32 flow_type;
+ int err;
+@@ -426,8 +426,8 @@ static int flow_get_tirn(struct mlx5e_priv *priv,
+ if (tt < 0)
+ return -EINVAL;
+
+- lro_param = mlx5e_get_lro_param(&priv->channels.params);
+- err = mlx5e_rss_obtain_tirn(rss, tt, &lro_param, false, tirn);
++ pkt_merge_param = priv->channels.params.packet_merge;
++ err = mlx5e_rss_obtain_tirn(rss, tt, &pkt_merge_param, false, tirn);
+ if (err)
+ return err;
+ eth_rule->rss = rss;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+index 41ef6eb70a585..7efb898e9f96f 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+@@ -911,8 +911,6 @@ void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
+ void mlx5e_close_rq(struct mlx5e_rq *rq)
+ {
+ cancel_work_sync(&rq->dim.work);
+- if (rq->icosq)
+- cancel_work_sync(&rq->icosq->recover_work);
+ cancel_work_sync(&rq->recover_work);
+ mlx5e_destroy_rq(rq);
+ mlx5e_free_rx_descs(rq);
+@@ -1038,9 +1036,20 @@ static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
+ mlx5e_reporter_icosq_cqe_err(sq);
+ }
+
++static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
++{
++ struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
++ recover_work);
++
++ /* Not implemented yet. */
++
++ netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n");
++}
++
+ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
+ struct mlx5e_sq_param *param,
+- struct mlx5e_icosq *sq)
++ struct mlx5e_icosq *sq,
++ work_func_t recover_work_func)
+ {
+ void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
+ struct mlx5_core_dev *mdev = c->mdev;
+@@ -1061,7 +1070,7 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
+ if (err)
+ goto err_sq_wq_destroy;
+
+- INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work);
++ INIT_WORK(&sq->recover_work, recover_work_func);
+
+ return 0;
+
+@@ -1399,13 +1408,14 @@ void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
+ mlx5e_reporter_tx_err_cqe(sq);
+ }
+
+-int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
+- struct mlx5e_sq_param *param, struct mlx5e_icosq *sq)
++static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
++ struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
++ work_func_t recover_work_func)
+ {
+ struct mlx5e_create_sq_param csp = {};
+ int err;
+
+- err = mlx5e_alloc_icosq(c, param, sq);
++ err = mlx5e_alloc_icosq(c, param, sq, recover_work_func);
+ if (err)
+ return err;
+
+@@ -1444,7 +1454,7 @@ void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
+ synchronize_net(); /* Sync with NAPI. */
+ }
+
+-void mlx5e_close_icosq(struct mlx5e_icosq *sq)
++static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
+ {
+ struct mlx5e_channel *c = sq->channel;
+
+@@ -1871,11 +1881,15 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
+
+ spin_lock_init(&c->async_icosq_lock);
+
+- err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq);
++ err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq,
++ mlx5e_async_icosq_err_cqe_work);
+ if (err)
+ goto err_close_xdpsq_cq;
+
+- err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
++ mutex_init(&c->icosq_recovery_lock);
++
++ err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq,
++ mlx5e_icosq_err_cqe_work);
+ if (err)
+ goto err_close_async_icosq;
+
+@@ -1943,9 +1957,12 @@ static void mlx5e_close_queues(struct mlx5e_channel *c)
+ mlx5e_close_xdpsq(&c->xdpsq);
+ if (c->xdp)
+ mlx5e_close_xdpsq(&c->rq_xdpsq);
++ /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */
++ cancel_work_sync(&c->icosq.recover_work);
+ mlx5e_close_rq(&c->rq);
+ mlx5e_close_sqs(c);
+ mlx5e_close_icosq(&c->icosq);
++ mutex_destroy(&c->icosq_recovery_lock);
+ mlx5e_close_icosq(&c->async_icosq);
+ if (c->xdp)
+ mlx5e_close_cq(&c->rq_xdpsq.cq);
+@@ -2185,17 +2202,14 @@ void mlx5e_close_channels(struct mlx5e_channels *chs)
+ chs->num = 0;
+ }
+
+-static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
++static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
+ {
+ struct mlx5e_rx_res *res = priv->rx_res;
+- struct mlx5e_lro_param lro_param;
+
+- lro_param = mlx5e_get_lro_param(&priv->channels.params);
+-
+- return mlx5e_rx_res_lro_set_param(res, &lro_param);
++ return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge);
+ }
+
+-static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_lro);
++static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);
+
+ static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
+ struct mlx5e_params *params, u16 mtu)
+@@ -3270,16 +3284,25 @@ static int set_feature_lro(struct net_device *netdev, bool enable)
+ }
+
+ new_params = *cur_params;
+- new_params.lro_en = enable;
+
+- if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
+- if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
+- mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
+- reset = false;
++ if (enable)
++ new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO;
++ else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)
++ new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
++ else
++ goto out;
++
++ if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO &&
++ new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) {
++ if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
++ if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
++ mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
++ reset = false;
++ }
+ }
+
+ err = mlx5e_safe_switch_params(priv, &new_params,
+- mlx5e_modify_tirs_lro_ctx, NULL, reset);
++ mlx5e_modify_tirs_packet_merge_ctx, NULL, reset);
+ out:
+ mutex_unlock(&priv->state_lock);
+ return err;
+@@ -3427,12 +3450,11 @@ static int set_feature_arfs(struct net_device *netdev, bool enable)
+
+ static int mlx5e_handle_feature(struct net_device *netdev,
+ netdev_features_t *features,
+- netdev_features_t wanted_features,
+ netdev_features_t feature,
+ mlx5e_feature_handler feature_handler)
+ {
+- netdev_features_t changes = wanted_features ^ netdev->features;
+- bool enable = !!(wanted_features & feature);
++ netdev_features_t changes = *features ^ netdev->features;
++ bool enable = !!(*features & feature);
+ int err;
+
+ if (!(changes & feature))
+@@ -3440,22 +3462,22 @@ static int mlx5e_handle_feature(struct net_device *netdev,
+
+ err = feature_handler(netdev, enable);
+ if (err) {
++ MLX5E_SET_FEATURE(features, feature, !enable);
+ netdev_err(netdev, "%s feature %pNF failed, err %d\n",
+ enable ? "Enable" : "Disable", &feature, err);
+ return err;
+ }
+
+- MLX5E_SET_FEATURE(features, feature, enable);
+ return 0;
+ }
+
+ int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
+ {
+- netdev_features_t oper_features = netdev->features;
++ netdev_features_t oper_features = features;
+ int err = 0;
+
+ #define MLX5E_HANDLE_FEATURE(feature, handler) \
+- mlx5e_handle_feature(netdev, &oper_features, features, feature, handler)
++ mlx5e_handle_feature(netdev, &oper_features, feature, handler)
+
+ err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
+ err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
+@@ -3520,6 +3542,13 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
+ }
+ }
+
++ if (params->xdp_prog) {
++ if (features & NETIF_F_LRO) {
++ netdev_warn(netdev, "LRO is incompatible with XDP\n");
++ features &= ~NETIF_F_LRO;
++ }
++ }
++
+ if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
+ features &= ~NETIF_F_RXHASH;
+ if (netdev->features & NETIF_F_RXHASH)
+@@ -3606,7 +3635,7 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
+ goto out;
+ }
+
+- if (params->lro_en)
++ if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
+ reset = false;
+
+ if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
+@@ -4063,8 +4092,8 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
+ struct net_device *netdev = priv->netdev;
+ struct mlx5e_params new_params;
+
+- if (priv->channels.params.lro_en) {
+- netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
++ if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
++ netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
+ return -EINVAL;
+ }
+
+@@ -4321,9 +4350,10 @@ void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16
+ params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
+ /* No XSK params: checking the availability of striding RQ in general. */
+ if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
+- params->lro_en = !slow_pci_heuristic(mdev);
++ params->packet_merge.type = slow_pci_heuristic(mdev) ?
++ MLX5E_PACKET_MERGE_NONE : MLX5E_PACKET_MERGE_LRO;
+ }
+- params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
++ params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
+
+ /* CQ moderation params */
+ rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
+@@ -4462,15 +4492,22 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
+ }
+
+ if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
+- netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
+- netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
+- netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL;
++ netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
++ NETIF_F_GSO_UDP_TUNNEL_CSUM;
++ netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
++ NETIF_F_GSO_UDP_TUNNEL_CSUM;
++ netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
++ netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
++ NETIF_F_GSO_UDP_TUNNEL_CSUM;
+ }
+
+ if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
+- netdev->hw_features |= NETIF_F_GSO_GRE;
+- netdev->hw_enc_features |= NETIF_F_GSO_GRE;
+- netdev->gso_partial_features |= NETIF_F_GSO_GRE;
++ netdev->hw_features |= NETIF_F_GSO_GRE |
++ NETIF_F_GSO_GRE_CSUM;
++ netdev->hw_enc_features |= NETIF_F_GSO_GRE |
++ NETIF_F_GSO_GRE_CSUM;
++ netdev->gso_partial_features |= NETIF_F_GSO_GRE |
++ NETIF_F_GSO_GRE_CSUM;
+ }
+
+ if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
+@@ -4608,7 +4645,6 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
+ {
+ struct mlx5_core_dev *mdev = priv->mdev;
+ enum mlx5e_rx_res_features features;
+- struct mlx5e_lro_param lro_param;
+ int err;
+
+ priv->rx_res = mlx5e_rx_res_alloc();
+@@ -4626,9 +4662,9 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
+ features = MLX5E_RX_RES_FEATURE_XSK | MLX5E_RX_RES_FEATURE_PTP;
+ if (priv->channels.params.tunneled_offload_en)
+ features |= MLX5E_RX_RES_FEATURE_INNER_FT;
+- lro_param = mlx5e_get_lro_param(&priv->channels.params);
+ err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features,
+- priv->max_nch, priv->drop_rq.rqn, &lro_param,
++ priv->max_nch, priv->drop_rq.rqn,
++ &priv->channels.params.packet_merge,
+ priv->channels.params.num_channels);
+ if (err)
+ goto err_close_drop_rq;
+@@ -4885,6 +4921,7 @@ mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *prof
+ }
+
+ netif_carrier_off(netdev);
++ netif_tx_disable(netdev);
+ dev_net_set(netdev, mlx5_core_net(mdev));
+
+ return netdev;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+index 0684ac6699b2d..161b60e1139b3 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+@@ -50,6 +50,7 @@
+ #include "fs_core.h"
+ #include "lib/mlx5.h"
+ #include "lib/devcom.h"
++#include "lib/vxlan.h"
+ #define CREATE_TRACE_POINTS
+ #include "diag/en_rep_tracepoint.h"
+ #include "en_accel/ipsec.h"
+@@ -793,7 +794,6 @@ int mlx5e_rep_bond_update(struct mlx5e_priv *priv, bool cleanup)
+ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
+ {
+ struct mlx5_core_dev *mdev = priv->mdev;
+- struct mlx5e_lro_param lro_param;
+ int err;
+
+ priv->rx_res = mlx5e_rx_res_alloc();
+@@ -808,9 +808,9 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
+ return err;
+ }
+
+- lro_param = mlx5e_get_lro_param(&priv->channels.params);
+ err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, 0,
+- priv->max_nch, priv->drop_rq.rqn, &lro_param,
++ priv->max_nch, priv->drop_rq.rqn,
++ &priv->channels.params.packet_merge,
+ priv->channels.params.num_channels);
+ if (err)
+ goto err_close_drop_rq;
+@@ -1017,6 +1017,7 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv)
+ rtnl_lock();
+ if (netif_running(netdev))
+ mlx5e_open(netdev);
++ udp_tunnel_nic_reset_ntf(priv->netdev);
+ netif_device_attach(netdev);
+ rtnl_unlock();
+ }
+@@ -1038,6 +1039,7 @@ static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv)
+ mlx5_notifier_unregister(mdev, &priv->events_nb);
+ mlx5e_rep_tc_disable(priv);
+ mlx5_lag_remove_netdev(mdev, priv->netdev);
++ mlx5_vxlan_reset_to_default(mdev->vxlan);
+ }
+
+ static MLX5E_DEFINE_STATS_GRP(sw_rep, 0);
+@@ -1070,6 +1072,10 @@ static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = {
+ &MLX5E_STATS_GRP(pme),
+ &MLX5E_STATS_GRP(channels),
+ &MLX5E_STATS_GRP(per_port_buff_congest),
++#ifdef CONFIG_MLX5_EN_IPSEC
++ &MLX5E_STATS_GRP(ipsec_sw),
++ &MLX5E_STATS_GRP(ipsec_hw),
++#endif
+ };
+
+ static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+index 29a6586ef28dc..d2de1e6c514c1 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+@@ -271,8 +271,8 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
+ if (unlikely(!dma_info->page))
+ return -ENOMEM;
+
+- dma_info->addr = dma_map_page(rq->pdev, dma_info->page, 0,
+- PAGE_SIZE, rq->buff.map_dir);
++ dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE,
++ rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
+ page_pool_recycle_direct(rq->page_pool, dma_info->page);
+ dma_info->page = NULL;
+@@ -293,7 +293,8 @@ static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
+
+ void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
+ {
+- dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
++ dma_unmap_page_attrs(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir,
++ DMA_ATTR_SKIP_CPU_SYNC);
+ }
+
+ void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
+@@ -986,7 +987,8 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
+ }
+
+ /* True when explicitly set via priv flag, or XDP prog is loaded */
+- if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state))
++ if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state) ||
++ get_cqe_tls_offload(cqe))
+ goto csum_unnecessary;
+
+ /* CQE csum doesn't cover padding octets in short ethernet
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+index 129ff7e0d65cc..60a4ac0ca76da 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+@@ -248,7 +248,6 @@ get_ct_priv(struct mlx5e_priv *priv)
+ return priv->fs.tc.ct;
+ }
+
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ static struct mlx5e_tc_psample *
+ get_sample_priv(struct mlx5e_priv *priv)
+ {
+@@ -265,7 +264,6 @@ get_sample_priv(struct mlx5e_priv *priv)
+
+ return NULL;
+ }
+-#endif
+
+ struct mlx5_flow_handle *
+ mlx5_tc_rule_insert(struct mlx5e_priv *priv,
+@@ -1148,11 +1146,9 @@ mlx5e_tc_offload_fdb_rules(struct mlx5_eswitch *esw,
+ rule = mlx5_tc_ct_flow_offload(get_ct_priv(flow->priv),
+ flow, spec, attr,
+ mod_hdr_acts);
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ } else if (flow_flag_test(flow, SAMPLE)) {
+ rule = mlx5e_tc_sample_offload(get_sample_priv(flow->priv), spec, attr,
+ mlx5e_tc_get_flow_tun_id(flow));
+-#endif
+ } else {
+ rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
+ }
+@@ -1183,23 +1179,16 @@ void mlx5e_tc_unoffload_fdb_rules(struct mlx5_eswitch *esw,
+ if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)
+ goto offload_rule_0;
+
+- if (flow_flag_test(flow, CT)) {
+- mlx5_tc_ct_delete_flow(get_ct_priv(flow->priv), flow, attr);
+- return;
+- }
+-
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+- if (flow_flag_test(flow, SAMPLE)) {
+- mlx5e_tc_sample_unoffload(get_sample_priv(flow->priv), flow->rule[0], attr);
+- return;
+- }
+-#endif
+-
+ if (attr->esw_attr->split_count)
+ mlx5_eswitch_del_fwd_rule(esw, flow->rule[1], attr);
+
++ if (flow_flag_test(flow, CT))
++ mlx5_tc_ct_delete_flow(get_ct_priv(flow->priv), flow, attr);
++ else if (flow_flag_test(flow, SAMPLE))
++ mlx5e_tc_sample_unoffload(get_sample_priv(flow->priv), flow->rule[0], attr);
++ else
+ offload_rule_0:
+- mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], attr);
++ mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], attr);
+ }
+
+ struct mlx5_flow_handle *
+@@ -1544,6 +1533,7 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
+ else
+ mlx5e_tc_unoffload_fdb_rules(esw, flow, attr);
+ }
++ complete_all(&flow->del_hw_done);
+
+ if (mlx5_flow_has_geneve_opt(flow))
+ mlx5_geneve_tlv_option_del(priv->mdev->geneve);
+@@ -1884,6 +1874,111 @@ u8 mlx5e_tc_get_ip_version(struct mlx5_flow_spec *spec, bool outer)
+ return ip_version;
+ }
+
++/* Tunnel device follows RFC 6040, see include/net/inet_ecn.h.
++ * And changes inner ip_ecn depending on inner and outer ip_ecn as follows:
++ * +---------+----------------------------------------+
++ * |Arriving | Arriving Outer Header |
++ * | Inner +---------+---------+---------+----------+
++ * | Header | Not-ECT | ECT(0) | ECT(1) | CE |
++ * +---------+---------+---------+---------+----------+
++ * | Not-ECT | Not-ECT | Not-ECT | Not-ECT | <drop> |
++ * | ECT(0) | ECT(0) | ECT(0) | ECT(1) | CE* |
++ * | ECT(1) | ECT(1) | ECT(1) | ECT(1)* | CE* |
++ * | CE | CE | CE | CE | CE |
++ * +---------+---------+---------+---------+----------+
++ *
++ * Tc matches on inner after decapsulation on tunnel device, but hw offload matches
++ * the inner ip_ecn value before hardware decap action.
++ *
++ * Cells marked are changed from original inner packet ip_ecn value during decap, and
++ * so matching those values on inner ip_ecn before decap will fail.
++ *
++ * The following helper allows offload when inner ip_ecn won't be changed by outer ip_ecn,
++ * except for the outer ip_ecn = CE, where in all cases inner ip_ecn will be changed to CE,
++ * and such we can drop the inner ip_ecn=CE match.
++ */
++
++static int mlx5e_tc_verify_tunnel_ecn(struct mlx5e_priv *priv,
++ struct flow_cls_offload *f,
++ bool *match_inner_ecn)
++{
++ u8 outer_ecn_mask = 0, outer_ecn_key = 0, inner_ecn_mask = 0, inner_ecn_key = 0;
++ struct flow_rule *rule = flow_cls_offload_flow_rule(f);
++ struct netlink_ext_ack *extack = f->common.extack;
++ struct flow_match_ip match;
++
++ *match_inner_ecn = true;
++
++ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
++ flow_rule_match_enc_ip(rule, &match);
++ outer_ecn_key = match.key->tos & INET_ECN_MASK;
++ outer_ecn_mask = match.mask->tos & INET_ECN_MASK;
++ }
++
++ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
++ flow_rule_match_ip(rule, &match);
++ inner_ecn_key = match.key->tos & INET_ECN_MASK;
++ inner_ecn_mask = match.mask->tos & INET_ECN_MASK;
++ }
++
++ if (outer_ecn_mask != 0 && outer_ecn_mask != INET_ECN_MASK) {
++ NL_SET_ERR_MSG_MOD(extack, "Partial match on enc_tos ecn bits isn't supported");
++ netdev_warn(priv->netdev, "Partial match on enc_tos ecn bits isn't supported");
++ return -EOPNOTSUPP;
++ }
++
++ if (!outer_ecn_mask) {
++ if (!inner_ecn_mask)
++ return 0;
++
++ NL_SET_ERR_MSG_MOD(extack,
++ "Matching on tos ecn bits without also matching enc_tos ecn bits isn't supported");
++ netdev_warn(priv->netdev,
++ "Matching on tos ecn bits without also matching enc_tos ecn bits isn't supported");
++ return -EOPNOTSUPP;
++ }
++
++ if (inner_ecn_mask && inner_ecn_mask != INET_ECN_MASK) {
++ NL_SET_ERR_MSG_MOD(extack,
++ "Partial match on tos ecn bits with match on enc_tos ecn bits isn't supported");
++ netdev_warn(priv->netdev,
++ "Partial match on tos ecn bits with match on enc_tos ecn bits isn't supported");
++ return -EOPNOTSUPP;
++ }
++
++ if (!inner_ecn_mask)
++ return 0;
++
++ /* Both inner and outer have full mask on ecn */
++
++ if (outer_ecn_key == INET_ECN_ECT_1) {
++ /* inner ecn might change by DECAP action */
++
++ NL_SET_ERR_MSG_MOD(extack, "Match on enc_tos ecn = ECT(1) isn't supported");
++ netdev_warn(priv->netdev, "Match on enc_tos ecn = ECT(1) isn't supported");
++ return -EOPNOTSUPP;
++ }
++
++ if (outer_ecn_key != INET_ECN_CE)
++ return 0;
++
++ if (inner_ecn_key != INET_ECN_CE) {
++ /* Can't happen in software, as packet ecn will be changed to CE after decap */
++ NL_SET_ERR_MSG_MOD(extack,
++ "Match on tos enc_tos ecn = CE while match on tos ecn != CE isn't supported");
++ netdev_warn(priv->netdev,
++ "Match on tos enc_tos ecn = CE while match on tos ecn != CE isn't supported");
++ return -EOPNOTSUPP;
++ }
++
++ /* outer ecn = CE, inner ecn = CE, as decap will change inner ecn to CE in anycase,
++ * drop match on inner ecn
++ */
++ *match_inner_ecn = false;
++
++ return 0;
++}
++
+ static int parse_tunnel_attr(struct mlx5e_priv *priv,
+ struct mlx5e_tc_flow *flow,
+ struct mlx5_flow_spec *spec,
+@@ -2077,6 +2172,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
+ struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+ struct flow_dissector *dissector = rule->match.dissector;
+ enum fs_flow_table_type fs_type;
++ bool match_inner_ecn = true;
+ u16 addr_type = 0;
+ u8 ip_proto = 0;
+ u8 *match_level;
+@@ -2130,6 +2226,10 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
+ headers_c = get_match_inner_headers_criteria(spec);
+ headers_v = get_match_inner_headers_value(spec);
+ }
++
++ err = mlx5e_tc_verify_tunnel_ecn(priv, f, &match_inner_ecn);
++ if (err)
++ return err;
+ }
+
+ err = mlx5e_flower_parse_meta(filter_dev, f);
+@@ -2191,6 +2291,17 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
+ match.key->vlan_priority);
+
+ *match_level = MLX5_MATCH_L2;
++
++ if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN) &&
++ match.mask->vlan_eth_type &&
++ MLX5_CAP_FLOWTABLE_TYPE(priv->mdev,
++ ft_field_support.outer_second_vid,
++ fs_type)) {
++ MLX5_SET(fte_match_set_misc, misc_c,
++ outer_second_cvlan_tag, 1);
++ spec->match_criteria_enable |=
++ MLX5_MATCH_MISC_PARAMETERS;
++ }
+ }
+ } else if (*match_level != MLX5_MATCH_NONE) {
+ /* cvlan_tag enabled in match criteria and
+@@ -2351,10 +2462,12 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
+ struct flow_match_ip match;
+
+ flow_rule_match_ip(rule, &match);
+- MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
+- match.mask->tos & 0x3);
+- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
+- match.key->tos & 0x3);
++ if (match_inner_ecn) {
++ MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
++ match.mask->tos & 0x3);
++ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
++ match.key->tos & 0x3);
++ }
+
+ MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
+ match.mask->tos >> 2);
+@@ -3457,7 +3570,9 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv,
+ attr->dest_chain = act->chain_index;
+ break;
+ case FLOW_ACTION_CT:
+- err = mlx5_tc_ct_parse_action(get_ct_priv(priv), attr, act, extack);
++ err = mlx5_tc_ct_parse_action(get_ct_priv(priv), attr,
++ &parse_attr->mod_hdr_acts,
++ act, extack);
+ if (err)
+ return err;
+
+@@ -4008,7 +4123,9 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv,
+ NL_SET_ERR_MSG_MOD(extack, "Sample action with connection tracking is not supported");
+ return -EOPNOTSUPP;
+ }
+- err = mlx5_tc_ct_parse_action(get_ct_priv(priv), attr, act, extack);
++ err = mlx5_tc_ct_parse_action(get_ct_priv(priv), attr,
++ &parse_attr->mod_hdr_acts,
++ act, extack);
+ if (err)
+ return err;
+
+@@ -4222,6 +4339,7 @@ mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size,
+ INIT_LIST_HEAD(&flow->l3_to_l2_reformat);
+ refcount_set(&flow->refcnt, 1);
+ init_completion(&flow->init_done);
++ init_completion(&flow->del_hw_done);
+
+ *__flow = flow;
+ *__parse_attr = parse_attr;
+@@ -5008,9 +5126,7 @@ int mlx5e_tc_esw_init(struct rhashtable *tc_ht)
+ MLX5_FLOW_NAMESPACE_FDB,
+ uplink_priv->post_act);
+
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ uplink_priv->tc_psample = mlx5e_tc_sample_init(esw, uplink_priv->post_act);
+-#endif
+
+ mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
+
+@@ -5054,9 +5170,7 @@ err_ht_init:
+ err_enc_opts_mapping:
+ mapping_destroy(uplink_priv->tunnel_mapping);
+ err_tun_mapping:
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ mlx5e_tc_sample_cleanup(uplink_priv->tc_psample);
+-#endif
+ mlx5_tc_ct_clean(uplink_priv->ct_priv);
+ netdev_warn(priv->netdev,
+ "Failed to initialize tc (eswitch), err: %d", err);
+@@ -5076,9 +5190,7 @@ void mlx5e_tc_esw_cleanup(struct rhashtable *tc_ht)
+ mapping_destroy(uplink_priv->tunnel_enc_opts_mapping);
+ mapping_destroy(uplink_priv->tunnel_mapping);
+
+-#if IS_ENABLED(CONFIG_MLX5_TC_SAMPLE)
+ mlx5e_tc_sample_cleanup(uplink_priv->tc_psample);
+-#endif
+ mlx5_tc_ct_clean(uplink_priv->ct_priv);
+ mlx5e_tc_post_act_destroy(uplink_priv->post_act);
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+index 188994d091c54..7fd33b356cc8d 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+@@ -38,6 +38,7 @@
+ #include "en/txrx.h"
+ #include "ipoib/ipoib.h"
+ #include "en_accel/en_accel.h"
++#include "en_accel/ipsec_rxtx.h"
+ #include "en/ptp.h"
+
+ static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)
+@@ -213,30 +214,13 @@ static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs)
+ memcpy(&vhdr->h_vlan_encapsulated_proto, skb->data + cpy1_sz, cpy2_sz);
+ }
+
+-static void
+-ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb,
+- struct mlx5_wqe_eth_seg *eseg)
+-{
+- struct xfrm_offload *xo = xfrm_offload(skb);
+-
+- eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
+- if (xo->inner_ipproto) {
+- eseg->cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM | MLX5_ETH_WQE_L3_INNER_CSUM;
+- } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+- eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
+- sq->stats->csum_partial_inner++;
+- }
+-}
+-
+ static inline void
+ mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb,
+ struct mlx5e_accel_tx_state *accel,
+ struct mlx5_wqe_eth_seg *eseg)
+ {
+- if (unlikely(mlx5e_ipsec_eseg_meta(eseg))) {
+- ipsec_txwqe_build_eseg_csum(sq, skb, eseg);
++ if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg)))
+ return;
+- }
+
+ if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+ eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+index 7e221038df8d5..317d76b97c42a 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+@@ -1385,6 +1385,8 @@ struct mlx5_esw_bridge_offloads *mlx5_esw_bridge_init(struct mlx5_eswitch *esw)
+ {
+ struct mlx5_esw_bridge_offloads *br_offloads;
+
++ ASSERT_RTNL();
++
+ br_offloads = kvzalloc(sizeof(*br_offloads), GFP_KERNEL);
+ if (!br_offloads)
+ return ERR_PTR(-ENOMEM);
+@@ -1401,6 +1403,8 @@ void mlx5_esw_bridge_cleanup(struct mlx5_eswitch *esw)
+ {
+ struct mlx5_esw_bridge_offloads *br_offloads = esw->br_offloads;
+
++ ASSERT_RTNL();
++
+ if (!br_offloads)
+ return;
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
+index 3401188e0a602..51ac24e6ec3c3 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
+@@ -21,7 +21,7 @@ DECLARE_EVENT_CLASS(mlx5_esw_bridge_fdb_template,
+ __field(unsigned int, used)
+ ),
+ TP_fast_assign(
+- strncpy(__entry->dev_name,
++ strscpy(__entry->dev_name,
+ netdev_name(fdb->dev),
+ IFNAMSIZ);
+ memcpy(__entry->addr, fdb->key.addr, ETH_ALEN);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
+index df277a6cddc0b..0c4c743ca31e1 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
+@@ -431,7 +431,7 @@ int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
+ int err = 0;
+
+ if (!mlx5_esw_allowed(esw))
+- return -EPERM;
++ return vlan ? -EPERM : 0;
+
+ if (vlan || qos)
+ set_flags = SET_VLAN_STRIP | SET_VLAN_INSERT;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+index c6cc67cb4f6ad..d377ddc70fc70 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+@@ -130,7 +130,7 @@ static u32 esw_qos_calculate_min_rate_divider(struct mlx5_eswitch *esw,
+ /* If vports min rate divider is 0 but their group has bw_share configured, then
+ * need to set bw_share for vports to minimal value.
+ */
+- if (!group_level && !max_guarantee && group->bw_share)
++ if (!group_level && !max_guarantee && group && group->bw_share)
+ return 1;
+ return 0;
+ }
+@@ -423,7 +423,7 @@ static int esw_qos_vport_update_group(struct mlx5_eswitch *esw,
+ return err;
+
+ /* Recalculate bw share weights of old and new groups */
+- if (vport->qos.bw_share) {
++ if (vport->qos.bw_share || new_group->bw_share) {
+ esw_qos_normalize_vports_min_rate(esw, curr_group, extack);
+ esw_qos_normalize_vports_min_rate(esw, new_group, extack);
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+index ec136b4992045..51a8cecc4a7ce 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+@@ -1305,12 +1305,17 @@ abort:
+ */
+ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
+ {
++ bool toggle_lag;
+ int ret;
+
+ if (!mlx5_esw_allowed(esw))
+ return 0;
+
+- mlx5_lag_disable_change(esw->dev);
++ toggle_lag = esw->mode == MLX5_ESWITCH_NONE;
++
++ if (toggle_lag)
++ mlx5_lag_disable_change(esw->dev);
++
+ down_write(&esw->mode_lock);
+ if (esw->mode == MLX5_ESWITCH_NONE) {
+ ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs);
+@@ -1324,7 +1329,10 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
+ esw->esw_funcs.num_vfs = num_vfs;
+ }
+ up_write(&esw->mode_lock);
+- mlx5_lag_enable_change(esw->dev);
++
++ if (toggle_lag)
++ mlx5_lag_enable_change(esw->dev);
++
+ return ret;
+ }
+
+@@ -1572,6 +1580,11 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
+ esw->enabled_vports = 0;
+ esw->mode = MLX5_ESWITCH_NONE;
+ esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE;
++ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) &&
++ MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))
++ esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC;
++ else
++ esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+
+ dev->priv.eswitch = esw;
+ BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head);
+@@ -1934,7 +1947,7 @@ free_out:
+ return err;
+ }
+
+-u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev)
++u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev)
+ {
+ struct mlx5_eswitch *esw = dev->priv.eswitch;
+
+@@ -1948,7 +1961,7 @@ mlx5_eswitch_get_encap_mode(const struct mlx5_core_dev *dev)
+ struct mlx5_eswitch *esw;
+
+ esw = dev->priv.eswitch;
+- return mlx5_esw_allowed(esw) ? esw->offloads.encap :
++ return (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS) ? esw->offloads.encap :
+ DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+ }
+ EXPORT_SYMBOL(mlx5_eswitch_get_encap_mode);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+index 0d461e38add37..3194cdcd2f630 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+@@ -2471,6 +2471,7 @@ static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
+ struct mlx5_eswitch *esw = master->priv.eswitch;
+ struct mlx5_flow_table_attr ft_attr = {
+ .max_fte = 1, .prio = 0, .level = 0,
++ .flags = MLX5_FLOW_TABLE_OTHER_VPORT,
+ };
+ struct mlx5_flow_namespace *egress_ns;
+ struct mlx5_flow_table *acl;
+@@ -2783,10 +2784,6 @@ bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
+ if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
+ return false;
+
+- if (mlx5_core_is_ecpf_esw_manager(esw->dev) ||
+- mlx5_ecpf_vport_exists(esw->dev))
+- return false;
+-
+ return true;
+ }
+
+@@ -3141,12 +3138,6 @@ int esw_offloads_enable(struct mlx5_eswitch *esw)
+ u64 mapping_id;
+ int err;
+
+- if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) &&
+- MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap))
+- esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC;
+- else
+- esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+-
+ mutex_init(&esw->offloads.termtbl_mutex);
+ mlx5_rdma_enable_roce(esw->dev);
+
+@@ -3244,7 +3235,6 @@ void esw_offloads_disable(struct mlx5_eswitch *esw)
+ esw_offloads_metadata_uninit(esw);
+ mlx5_rdma_disable_roce(esw->dev);
+ mutex_destroy(&esw->offloads.termtbl_mutex);
+- esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
+ }
+
+ static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
+@@ -3588,7 +3578,7 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
+ *encap = esw->offloads.encap;
+ unlock:
+ up_write(&esw->mode_lock);
+- return 0;
++ return err;
+ }
+
+ static bool
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+index fe501ba88bea9..00834c914dc64 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+@@ -2041,6 +2041,8 @@ void mlx5_del_flow_rules(struct mlx5_flow_handle *handle)
+ fte->node.del_hw_func = NULL;
+ up_write_ref_node(&fte->node, false);
+ tree_put_node(&fte->node, false);
++ } else {
++ up_write_ref_node(&fte->node, false);
+ }
+ kfree(handle);
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+index 106b50e42b464..8c2b249949b97 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+@@ -111,6 +111,28 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
+ }
+ }
+
++static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
++{
++ struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
++
++ del_timer_sync(&fw_reset->timer);
++}
++
++static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
++{
++ struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
++
++ if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
++ mlx5_core_warn(dev, "Reset request was already cleared\n");
++ return -EALREADY;
++ }
++
++ mlx5_stop_sync_reset_poll(dev);
++ if (poll_health)
++ mlx5_start_health_poll(dev);
++ return 0;
++}
++
+ static void mlx5_sync_reset_reload_work(struct work_struct *work)
+ {
+ struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
+@@ -118,6 +140,7 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work)
+ struct mlx5_core_dev *dev = fw_reset->dev;
+ int err;
+
++ mlx5_sync_reset_clear_reset_requested(dev, false);
+ mlx5_enter_error_state(dev, true);
+ mlx5_unload_one(dev);
+ err = mlx5_health_wait_pci_up(dev);
+@@ -127,23 +150,6 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work)
+ mlx5_fw_reset_complete_reload(dev);
+ }
+
+-static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
+-{
+- struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
+-
+- del_timer(&fw_reset->timer);
+-}
+-
+-static void mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
+-{
+- struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
+-
+- mlx5_stop_sync_reset_poll(dev);
+- clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags);
+- if (poll_health)
+- mlx5_start_health_poll(dev);
+-}
+-
+ #define MLX5_RESET_POLL_INTERVAL (HZ / 10)
+ static void poll_sync_reset(struct timer_list *t)
+ {
+@@ -158,7 +164,6 @@ static void poll_sync_reset(struct timer_list *t)
+
+ if (fatal_error) {
+ mlx5_core_warn(dev, "Got Device Reset\n");
+- mlx5_sync_reset_clear_reset_requested(dev, false);
+ queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
+ return;
+ }
+@@ -185,13 +190,17 @@ static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
+ return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
+ }
+
+-static void mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
++static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
+ {
+ struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
+
++ if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
++ mlx5_core_warn(dev, "Reset request was already set\n");
++ return -EALREADY;
++ }
+ mlx5_stop_health_poll(dev, true);
+- set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags);
+ mlx5_start_sync_reset_poll(dev);
++ return 0;
+ }
+
+ static void mlx5_fw_live_patch_event(struct work_struct *work)
+@@ -220,7 +229,9 @@ static void mlx5_sync_reset_request_event(struct work_struct *work)
+ err ? "Failed" : "Sent");
+ return;
+ }
+- mlx5_sync_reset_set_reset_requested(dev);
++ if (mlx5_sync_reset_set_reset_requested(dev))
++ return;
++
+ err = mlx5_fw_reset_set_reset_sync_ack(dev);
+ if (err)
+ mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
+@@ -320,7 +331,8 @@ static void mlx5_sync_reset_now_event(struct work_struct *work)
+ struct mlx5_core_dev *dev = fw_reset->dev;
+ int err;
+
+- mlx5_sync_reset_clear_reset_requested(dev, false);
++ if (mlx5_sync_reset_clear_reset_requested(dev, false))
++ return;
+
+ mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
+
+@@ -349,10 +361,8 @@ static void mlx5_sync_reset_abort_event(struct work_struct *work)
+ reset_abort_work);
+ struct mlx5_core_dev *dev = fw_reset->dev;
+
+- if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
++ if (mlx5_sync_reset_clear_reset_requested(dev, true))
+ return;
+-
+- mlx5_sync_reset_clear_reset_requested(dev, true);
+ mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+index 269ebb53eda67..cfde0a45b8b8a 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+@@ -67,7 +67,7 @@ static void mlx5i_build_nic_params(struct mlx5_core_dev *mdev,
+ MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
+ MLX5I_PARAMS_DEFAULT_LOG_RQ_SIZE;
+
+- params->lro_en = false;
++ params->packet_merge.type = MLX5E_PACKET_MERGE_NONE;
+ params->hard_mtu = MLX5_IB_GRH_BYTES + MLX5_IPOIB_HARD_LEN;
+ params->tunneled_offload_en = false;
+ }
+@@ -353,7 +353,6 @@ static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv)
+ static int mlx5i_init_rx(struct mlx5e_priv *priv)
+ {
+ struct mlx5_core_dev *mdev = priv->mdev;
+- struct mlx5e_lro_param lro_param;
+ int err;
+
+ priv->rx_res = mlx5e_rx_res_alloc();
+@@ -368,9 +367,9 @@ static int mlx5i_init_rx(struct mlx5e_priv *priv)
+ goto err_destroy_q_counters;
+ }
+
+- lro_param = mlx5e_get_lro_param(&priv->channels.params);
+ err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, 0,
+- priv->max_nch, priv->drop_rq.rqn, &lro_param,
++ priv->max_nch, priv->drop_rq.rqn,
++ &priv->channels.params.packet_merge,
+ priv->channels.params.num_channels);
+ if (err)
+ goto err_close_drop_rq;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag.c
+index d2105c1635c34..c19d9327095b2 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/lag.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/lag.c
+@@ -565,6 +565,7 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
+ bool is_bonded, is_in_lag, mode_supported;
+ int bond_status = 0;
+ int num_slaves = 0;
++ int changed = 0;
+ int idx;
+
+ if (!netif_is_lag_master(upper))
+@@ -601,27 +602,27 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
+ */
+ is_in_lag = num_slaves == MLX5_MAX_PORTS && bond_status == 0x3;
+
+- if (!mlx5_lag_is_ready(ldev) && is_in_lag) {
+- NL_SET_ERR_MSG_MOD(info->info.extack,
+- "Can't activate LAG offload, PF is configured with more than 64 VFs");
+- return 0;
+- }
+-
+ /* Lag mode must be activebackup or hash. */
+ mode_supported = tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP ||
+ tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH;
+
+- if (is_in_lag && !mode_supported)
+- NL_SET_ERR_MSG_MOD(info->info.extack,
+- "Can't activate LAG offload, TX type isn't supported");
+-
+ is_bonded = is_in_lag && mode_supported;
+ if (tracker->is_bonded != is_bonded) {
+ tracker->is_bonded = is_bonded;
+- return 1;
++ changed = 1;
+ }
+
+- return 0;
++ if (!is_in_lag)
++ return changed;
++
++ if (!mlx5_lag_is_ready(ldev))
++ NL_SET_ERR_MSG_MOD(info->info.extack,
++ "Can't activate LAG offload, PF is configured with more than 64 VFs");
++ else if (!mode_supported)
++ NL_SET_ERR_MSG_MOD(info->info.extack,
++ "Can't activate LAG offload, TX type isn't supported");
++
++ return changed;
+ }
+
+ static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
+@@ -664,9 +665,6 @@ static int mlx5_lag_netdev_event(struct notifier_block *this,
+
+ ldev = container_of(this, struct mlx5_lag, nb);
+
+- if (!mlx5_lag_is_ready(ldev) && event == NETDEV_CHANGELOWERSTATE)
+- return NOTIFY_DONE;
+-
+ tracker = ldev->tracker;
+
+ switch (event) {
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c
+index 21fdaf708f1fe..81786a9a424c8 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c
+@@ -100,6 +100,14 @@ static void mlx5_lag_fib_event_flush(struct notifier_block *nb)
+ flush_workqueue(mp->wq);
+ }
+
++static void mlx5_lag_fib_set(struct lag_mp *mp, struct fib_info *fi, u32 dst, int dst_len)
++{
++ mp->fib.mfi = fi;
++ mp->fib.priority = fi->fib_priority;
++ mp->fib.dst = dst;
++ mp->fib.dst_len = dst_len;
++}
++
+ struct mlx5_fib_event_work {
+ struct work_struct work;
+ struct mlx5_lag *ldev;
+@@ -110,10 +118,10 @@ struct mlx5_fib_event_work {
+ };
+ };
+
+-static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev,
+- unsigned long event,
+- struct fib_info *fi)
++static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event,
++ struct fib_entry_notifier_info *fen_info)
+ {
++ struct fib_info *fi = fen_info->fi;
+ struct lag_mp *mp = &ldev->lag_mp;
+ struct fib_nh *fib_nh0, *fib_nh1;
+ unsigned int nhs;
+@@ -121,11 +129,17 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev,
+ /* Handle delete event */
+ if (event == FIB_EVENT_ENTRY_DEL) {
+ /* stop track */
+- if (mp->mfi == fi)
+- mp->mfi = NULL;
++ if (mp->fib.mfi == fi)
++ mp->fib.mfi = NULL;
+ return;
+ }
+
++ /* Handle multipath entry with lower priority value */
++ if (mp->fib.mfi && mp->fib.mfi != fi &&
++ (mp->fib.dst != fen_info->dst || mp->fib.dst_len != fen_info->dst_len) &&
++ fi->fib_priority >= mp->fib.priority)
++ return;
++
+ /* Handle add/replace event */
+ nhs = fib_info_num_path(fi);
+ if (nhs == 1) {
+@@ -135,12 +149,13 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev,
+ int i = mlx5_lag_dev_get_netdev_idx(ldev, nh_dev);
+
+ if (i < 0)
+- i = MLX5_LAG_NORMAL_AFFINITY;
+- else
+- ++i;
++ return;
+
++ i++;
+ mlx5_lag_set_port_affinity(ldev, i);
++ mlx5_lag_fib_set(mp, fi, fen_info->dst, fen_info->dst_len);
+ }
++
+ return;
+ }
+
+@@ -160,7 +175,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev,
+ }
+
+ /* First time we see multipath route */
+- if (!mp->mfi && !__mlx5_lag_is_active(ldev)) {
++ if (!mp->fib.mfi && !__mlx5_lag_is_active(ldev)) {
+ struct lag_tracker tracker;
+
+ tracker = ldev->tracker;
+@@ -168,7 +183,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev,
+ }
+
+ mlx5_lag_set_port_affinity(ldev, MLX5_LAG_NORMAL_AFFINITY);
+- mp->mfi = fi;
++ mlx5_lag_fib_set(mp, fi, fen_info->dst, fen_info->dst_len);
+ }
+
+ static void mlx5_lag_fib_nexthop_event(struct mlx5_lag *ldev,
+@@ -179,7 +194,7 @@ static void mlx5_lag_fib_nexthop_event(struct mlx5_lag *ldev,
+ struct lag_mp *mp = &ldev->lag_mp;
+
+ /* Check the nh event is related to the route */
+- if (!mp->mfi || mp->mfi != fi)
++ if (!mp->fib.mfi || mp->fib.mfi != fi)
+ return;
+
+ /* nh added/removed */
+@@ -209,7 +224,7 @@ static void mlx5_lag_fib_update(struct work_struct *work)
+ case FIB_EVENT_ENTRY_REPLACE:
+ case FIB_EVENT_ENTRY_DEL:
+ mlx5_lag_fib_route_event(ldev, fib_work->event,
+- fib_work->fen_info.fi);
++ &fib_work->fen_info);
+ fib_info_put(fib_work->fen_info.fi);
+ break;
+ case FIB_EVENT_NH_ADD:
+@@ -268,10 +283,8 @@ static int mlx5_lag_fib_event(struct notifier_block *nb,
+ fen_info = container_of(info, struct fib_entry_notifier_info,
+ info);
+ fi = fen_info->fi;
+- if (fi->nh) {
+- NL_SET_ERR_MSG_MOD(info->extack, "IPv4 route with nexthop objects is not supported");
+- return notifier_from_errno(-EINVAL);
+- }
++ if (fi->nh)
++ return NOTIFY_DONE;
+ fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev;
+ if (fib_dev != ldev->pf[MLX5_LAG_P1].netdev &&
+ fib_dev != ldev->pf[MLX5_LAG_P2].netdev) {
+@@ -310,7 +323,7 @@ void mlx5_lag_mp_reset(struct mlx5_lag *ldev)
+ /* Clear mfi, as it might become stale when a route delete event
+ * has been missed, see mlx5_lag_fib_route_event().
+ */
+- ldev->lag_mp.mfi = NULL;
++ ldev->lag_mp.fib.mfi = NULL;
+ }
+
+ int mlx5_lag_mp_init(struct mlx5_lag *ldev)
+@@ -321,7 +334,7 @@ int mlx5_lag_mp_init(struct mlx5_lag *ldev)
+ /* always clear mfi, as it might become stale when a route delete event
+ * has been missed
+ */
+- mp->mfi = NULL;
++ mp->fib.mfi = NULL;
+
+ if (mp->fib_nb.notifier_call)
+ return 0;
+@@ -351,5 +364,5 @@ void mlx5_lag_mp_cleanup(struct mlx5_lag *ldev)
+ unregister_fib_notifier(&init_net, &mp->fib_nb);
+ destroy_workqueue(mp->wq);
+ mp->fib_nb.notifier_call = NULL;
+- mp->mfi = NULL;
++ mp->fib.mfi = NULL;
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.h b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.h
+index dea199e79beda..b3a7f18b9e303 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.h
+@@ -15,7 +15,12 @@ enum mlx5_lag_port_affinity {
+
+ struct lag_mp {
+ struct notifier_block fib_nb;
+- struct fib_info *mfi; /* used in tracking fib events */
++ struct {
++ const void *mfi; /* used in tracking fib events */
++ u32 priority;
++ u32 dst;
++ int dst_len;
++ } fib;
+ struct workqueue_struct *wq;
+ };
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
+index 97e5845b4cfdd..df58cba37930a 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
+@@ -124,6 +124,10 @@ u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
+ if (mlx5_chains_ignore_flow_level_supported(chains))
+ return UINT_MAX;
+
++ if (!chains->dev->priv.eswitch ||
++ chains->dev->priv.eswitch->mode != MLX5_ESWITCH_OFFLOADS)
++ return 1;
++
+ /* We should get here only for eswitch case */
+ return FDB_TC_MAX_PRIO;
+ }
+@@ -208,7 +212,7 @@ static int
+ create_chain_restore(struct fs_chain *chain)
+ {
+ struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch;
+- char modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)];
++ u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+ struct mlx5_fs_chains *chains = chain->chains;
+ enum mlx5e_tc_attr_to_reg chain_to_reg;
+ struct mlx5_modify_hdr *mod_hdr;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
+index 79482824c64ff..4ed7409942791 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
+@@ -97,6 +97,8 @@ enum {
+ MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
+ };
+
++#define LOG_MAX_SUPPORTED_QPS 0xff
++
+ static struct mlx5_profile profile[] = {
+ [0] = {
+ .mask = 0,
+@@ -108,7 +110,7 @@ static struct mlx5_profile profile[] = {
+ [2] = {
+ .mask = MLX5_PROF_MASK_QP_SIZE |
+ MLX5_PROF_MASK_MR_CACHE,
+- .log_max_qp = 18,
++ .log_max_qp = LOG_MAX_SUPPORTED_QPS,
+ .mr_cache[0] = {
+ .size = 500,
+ .limit = 250
+@@ -513,7 +515,9 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
+ to_fw_pkey_sz(dev, 128));
+
+ /* Check log_max_qp from HCA caps to set in current profile */
+- if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) {
++ if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) {
++ prof->log_max_qp = min_t(u8, 17, MLX5_CAP_GEN_MAX(dev, log_max_qp));
++ } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) {
+ mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n",
+ prof->log_max_qp,
+ MLX5_CAP_GEN_MAX(dev, log_max_qp));
+@@ -1756,10 +1760,12 @@ static const struct pci_device_id mlx5_core_pci_table[] = {
+ { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */
+ { PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */
+ { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */
++ { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */
+ { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */
+ { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */
+ { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */
+ { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */
++ { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */
+ { 0, }
+ };
+
+@@ -1773,12 +1779,13 @@ void mlx5_disable_device(struct mlx5_core_dev *dev)
+
+ int mlx5_recover_device(struct mlx5_core_dev *dev)
+ {
+- int ret = -EIO;
++ if (!mlx5_core_is_sf(dev)) {
++ mlx5_pci_disable_device(dev);
++ if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED)
++ return -EIO;
++ }
+
+- mlx5_pci_disable_device(dev);
+- if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED)
+- ret = mlx5_load_one(dev);
+- return ret;
++ return mlx5_load_one(dev);
+ }
+
+ static struct pci_driver mlx5_core_driver = {
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
+index 763c83a023809..11f3649fdaab1 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
+@@ -346,8 +346,8 @@ static struct mlx5_irq *irq_pool_request_affinity(struct mlx5_irq_pool *pool,
+ new_irq = irq_pool_create_irq(pool, affinity);
+ if (IS_ERR(new_irq)) {
+ if (!least_loaded_irq) {
+- mlx5_core_err(pool->dev, "Didn't find IRQ for cpu = %u\n",
+- cpumask_first(affinity));
++ mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n",
++ PTR_ERR(new_irq));
+ mutex_unlock(&pool->lock);
+ return new_irq;
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
+index 1ef2b6a848c10..fd79860de723b 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
+@@ -406,23 +406,24 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev,
+
+ switch (module_id) {
+ case MLX5_MODULE_ID_SFP:
+- mlx5_sfp_eeprom_params_set(&query.i2c_address, &query.page, &query.offset);
++ mlx5_sfp_eeprom_params_set(&query.i2c_address, &query.page, &offset);
+ break;
+ case MLX5_MODULE_ID_QSFP:
+ case MLX5_MODULE_ID_QSFP_PLUS:
+ case MLX5_MODULE_ID_QSFP28:
+- mlx5_qsfp_eeprom_params_set(&query.i2c_address, &query.page, &query.offset);
++ mlx5_qsfp_eeprom_params_set(&query.i2c_address, &query.page, &offset);
+ break;
+ default:
+ mlx5_core_err(dev, "Module ID not recognized: 0x%x\n", module_id);
+ return -EINVAL;
+ }
+
+- if (query.offset + size > MLX5_EEPROM_PAGE_LENGTH)
++ if (offset + size > MLX5_EEPROM_PAGE_LENGTH)
+ /* Cross pages read, read until offset 256 in low page */
+- size -= offset + size - MLX5_EEPROM_PAGE_LENGTH;
++ size = MLX5_EEPROM_PAGE_LENGTH - offset;
+
+ query.size = size;
++ query.offset = offset;
+
+ return mlx5_query_mcia(dev, &query, data);
+ }
+@@ -432,35 +433,12 @@ int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev,
+ struct mlx5_module_eeprom_query_params *params,
+ u8 *data)
+ {
+- u8 module_id;
+ int err;
+
+ err = mlx5_query_module_num(dev, &params->module_number);
+ if (err)
+ return err;
+
+- err = mlx5_query_module_id(dev, params->module_number, &module_id);
+- if (err)
+- return err;
+-
+- switch (module_id) {
+- case MLX5_MODULE_ID_SFP:
+- if (params->page > 0)
+- return -EINVAL;
+- break;
+- case MLX5_MODULE_ID_QSFP:
+- case MLX5_MODULE_ID_QSFP28:
+- case MLX5_MODULE_ID_QSFP_PLUS:
+- if (params->page > 3)
+- return -EINVAL;
+- break;
+- case MLX5_MODULE_ID_DSFP:
+- break;
+- default:
+- mlx5_core_err(dev, "Module ID not recognized: 0x%x\n", module_id);
+- return -EINVAL;
+- }
+-
+ if (params->i2c_address != MLX5_I2C_ADDR_HIGH &&
+ params->i2c_address != MLX5_I2C_ADDR_LOW) {
+ mlx5_core_err(dev, "I2C address not recognized: 0x%x\n", params->i2c_address);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c
+index 871c2fbe18d39..64bbc18332d56 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c
+@@ -28,10 +28,7 @@ bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev)
+ {
+ struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
+
+- if (!mlx5_sf_dev_supported(dev))
+- return false;
+-
+- return !xa_empty(&table->devices);
++ return table && !xa_empty(&table->devices);
+ }
+
+ static ssize_t sfnum_show(struct device *dev, struct device_attribute *attr, char *buf)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
+index a5b9f65db23c6..897c7f8521238 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
+@@ -846,7 +846,8 @@ struct mlx5dr_action *
+ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn,
+ struct mlx5dr_action_dest *dests,
+ u32 num_of_dests,
+- bool ignore_flow_level)
++ bool ignore_flow_level,
++ u32 flow_source)
+ {
+ struct mlx5dr_cmd_flow_destination_hw_info *hw_dests;
+ struct mlx5dr_action **ref_actions;
+@@ -914,7 +915,8 @@ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn,
+ reformat_req,
+ &action->dest_tbl->fw_tbl.id,
+ &action->dest_tbl->fw_tbl.group_id,
+- ignore_flow_level);
++ ignore_flow_level,
++ flow_source);
+ if (ret)
+ goto free_action;
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
+index 0fe159809ba15..ea1b8ca5bf3aa 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
+@@ -2,6 +2,7 @@
+ /* Copyright (c) 2019 Mellanox Technologies. */
+
+ #include <linux/mlx5/eswitch.h>
++#include <linux/err.h>
+ #include "dr_types.h"
+
+ #define DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, dmn_type) \
+@@ -75,9 +76,9 @@ static int dr_domain_init_resources(struct mlx5dr_domain *dmn)
+ }
+
+ dmn->uar = mlx5_get_uars_page(dmn->mdev);
+- if (!dmn->uar) {
++ if (IS_ERR(dmn->uar)) {
+ mlx5dr_err(dmn, "Couldn't allocate UAR\n");
+- ret = -ENOMEM;
++ ret = PTR_ERR(dmn->uar);
+ goto clean_pd;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c
+index 0d6f86eb248b9..c74083de1801b 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c
+@@ -104,7 +104,8 @@ int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
+ bool reformat_req,
+ u32 *tbl_id,
+ u32 *group_id,
+- bool ignore_flow_level)
++ bool ignore_flow_level,
++ u32 flow_source)
+ {
+ struct mlx5dr_cmd_create_flow_table_attr ft_attr = {};
+ struct mlx5dr_cmd_fte_info fte_info = {};
+@@ -139,6 +140,7 @@ int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
+ fte_info.val = val;
+ fte_info.dest_arr = dest;
+ fte_info.ignore_flow_level = ignore_flow_level;
++ fte_info.flow_context.flow_source = flow_source;
+
+ ret = mlx5dr_cmd_set_fte(dmn->mdev, 0, 0, &ft_info, *group_id, &fte_info);
+ if (ret) {
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
+index 66c24767e3b00..8ad8d73e17f06 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
+@@ -4,7 +4,6 @@
+ #include "dr_types.h"
+
+ #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
+-#define DR_ICM_SYNC_THRESHOLD_POOL (64 * 1024 * 1024)
+
+ struct mlx5dr_icm_pool {
+ enum mlx5dr_icm_type icm_type;
+@@ -136,37 +135,35 @@ static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
+ kvfree(icm_mr);
+ }
+
+-static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
++static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy)
+ {
+- chunk->ste_arr = kvzalloc(chunk->num_of_entries *
+- sizeof(chunk->ste_arr[0]), GFP_KERNEL);
+- if (!chunk->ste_arr)
+- return -ENOMEM;
+-
+- chunk->hw_ste_arr = kvzalloc(chunk->num_of_entries *
+- DR_STE_SIZE_REDUCED, GFP_KERNEL);
+- if (!chunk->hw_ste_arr)
+- goto out_free_ste_arr;
+-
+- chunk->miss_list = kvmalloc(chunk->num_of_entries *
+- sizeof(chunk->miss_list[0]), GFP_KERNEL);
+- if (!chunk->miss_list)
+- goto out_free_hw_ste_arr;
++ /* We support only one type of STE size, both for ConnectX-5 and later
++ * devices. Once the support for match STE which has a larger tag is
++ * added (32B instead of 16B), the STE size for devices later than
++ * ConnectX-5 needs to account for that.
++ */
++ return DR_STE_SIZE_REDUCED;
++}
+
+- return 0;
++static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset)
++{
++ struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem;
++ int index = offset / DR_STE_SIZE;
+
+-out_free_hw_ste_arr:
+- kvfree(chunk->hw_ste_arr);
+-out_free_ste_arr:
+- kvfree(chunk->ste_arr);
+- return -ENOMEM;
++ chunk->ste_arr = &buddy->ste_arr[index];
++ chunk->miss_list = &buddy->miss_list[index];
++ chunk->hw_ste_arr = buddy->hw_ste_arr +
++ index * dr_icm_buddy_get_ste_size(buddy);
+ }
+
+ static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
+ {
+- kvfree(chunk->miss_list);
+- kvfree(chunk->hw_ste_arr);
+- kvfree(chunk->ste_arr);
++ struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem;
++
++ memset(chunk->hw_ste_arr, 0,
++ chunk->num_of_entries * dr_icm_buddy_get_ste_size(buddy));
++ memset(chunk->ste_arr, 0,
++ chunk->num_of_entries * sizeof(chunk->ste_arr[0]));
+ }
+
+ static enum mlx5dr_icm_type
+@@ -189,6 +186,44 @@ static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk,
+ kvfree(chunk);
+ }
+
++static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy)
++{
++ int num_of_entries =
++ mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz);
++
++ buddy->ste_arr = kvcalloc(num_of_entries,
++ sizeof(struct mlx5dr_ste), GFP_KERNEL);
++ if (!buddy->ste_arr)
++ return -ENOMEM;
++
++ /* Preallocate full STE size on non-ConnectX-5 devices since
++ * we need to support both full and reduced with the same cache.
++ */
++ buddy->hw_ste_arr = kvcalloc(num_of_entries,
++ dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL);
++ if (!buddy->hw_ste_arr)
++ goto free_ste_arr;
++
++ buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL);
++ if (!buddy->miss_list)
++ goto free_hw_ste_arr;
++
++ return 0;
++
++free_hw_ste_arr:
++ kvfree(buddy->hw_ste_arr);
++free_ste_arr:
++ kvfree(buddy->ste_arr);
++ return -ENOMEM;
++}
++
++static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy)
++{
++ kvfree(buddy->ste_arr);
++ kvfree(buddy->hw_ste_arr);
++ kvfree(buddy->miss_list);
++}
++
+ static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool)
+ {
+ struct mlx5dr_icm_buddy_mem *buddy;
+@@ -208,11 +243,19 @@ static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool)
+ buddy->icm_mr = icm_mr;
+ buddy->pool = pool;
+
++ if (pool->icm_type == DR_ICM_TYPE_STE) {
++ /* Reduce allocations by preallocating and reusing the STE structures */
++ if (dr_icm_buddy_init_ste_cache(buddy))
++ goto err_cleanup_buddy;
++ }
++
+ /* add it to the -start- of the list in order to search in it first */
+ list_add(&buddy->list_node, &pool->buddy_mem_list);
+
+ return 0;
+
++err_cleanup_buddy:
++ mlx5dr_buddy_cleanup(buddy);
+ err_free_buddy:
+ kvfree(buddy);
+ free_mr:
+@@ -234,6 +277,9 @@ static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy)
+
+ mlx5dr_buddy_cleanup(buddy);
+
++ if (buddy->pool->icm_type == DR_ICM_TYPE_STE)
++ dr_icm_buddy_cleanup_ste_cache(buddy);
++
+ kvfree(buddy);
+ }
+
+@@ -261,34 +307,30 @@ dr_icm_chunk_create(struct mlx5dr_icm_pool *pool,
+ chunk->byte_size =
+ mlx5dr_icm_pool_chunk_size_to_byte(chunk_size, pool->icm_type);
+ chunk->seg = seg;
++ chunk->buddy_mem = buddy_mem_pool;
+
+- if (pool->icm_type == DR_ICM_TYPE_STE && dr_icm_chunk_ste_init(chunk)) {
+- mlx5dr_err(pool->dmn,
+- "Failed to init ste arrays (order: %d)\n",
+- chunk_size);
+- goto out_free_chunk;
+- }
++ if (pool->icm_type == DR_ICM_TYPE_STE)
++ dr_icm_chunk_ste_init(chunk, offset);
+
+ buddy_mem_pool->used_memory += chunk->byte_size;
+- chunk->buddy_mem = buddy_mem_pool;
+ INIT_LIST_HEAD(&chunk->chunk_list);
+
+ /* chunk now is part of the used_list */
+ list_add_tail(&chunk->chunk_list, &buddy_mem_pool->used_list);
+
+ return chunk;
+-
+-out_free_chunk:
+- kvfree(chunk);
+- return NULL;
+ }
+
+ static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool)
+ {
+- if (pool->hot_memory_size > DR_ICM_SYNC_THRESHOLD_POOL)
+- return true;
++ int allow_hot_size;
++
++ /* sync when hot memory reaches half of the pool size */
++ allow_hot_size =
++ mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
++ pool->icm_type) / 2;
+
+- return false;
++ return pool->hot_memory_size > allow_hot_size;
+ }
+
+ static int dr_icm_pool_sync_all_buddy_pools(struct mlx5dr_icm_pool *pool)
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c
+index b5409cc021d33..a19e8157c1002 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c
+@@ -13,18 +13,6 @@ static bool dr_mask_is_dmac_set(struct mlx5dr_match_spec *spec)
+ return (spec->dmac_47_16 || spec->dmac_15_0);
+ }
+
+-static bool dr_mask_is_src_addr_set(struct mlx5dr_match_spec *spec)
+-{
+- return (spec->src_ip_127_96 || spec->src_ip_95_64 ||
+- spec->src_ip_63_32 || spec->src_ip_31_0);
+-}
+-
+-static bool dr_mask_is_dst_addr_set(struct mlx5dr_match_spec *spec)
+-{
+- return (spec->dst_ip_127_96 || spec->dst_ip_95_64 ||
+- spec->dst_ip_63_32 || spec->dst_ip_31_0);
+-}
+-
+ static bool dr_mask_is_l3_base_set(struct mlx5dr_match_spec *spec)
+ {
+ return (spec->ip_protocol || spec->frag || spec->tcp_flags ||
+@@ -480,11 +468,11 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher,
+ &mask, inner, rx);
+
+ if (outer_ipv == DR_RULE_IPV6) {
+- if (dr_mask_is_dst_addr_set(&mask.outer))
++ if (DR_MASK_IS_DST_IP_SET(&mask.outer))
+ mlx5dr_ste_build_eth_l3_ipv6_dst(ste_ctx, &sb[idx++],
+ &mask, inner, rx);
+
+- if (dr_mask_is_src_addr_set(&mask.outer))
++ if (DR_MASK_IS_SRC_IP_SET(&mask.outer))
+ mlx5dr_ste_build_eth_l3_ipv6_src(ste_ctx, &sb[idx++],
+ &mask, inner, rx);
+
+@@ -580,11 +568,11 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher,
+ &mask, inner, rx);
+
+ if (inner_ipv == DR_RULE_IPV6) {
+- if (dr_mask_is_dst_addr_set(&mask.inner))
++ if (DR_MASK_IS_DST_IP_SET(&mask.inner))
+ mlx5dr_ste_build_eth_l3_ipv6_dst(ste_ctx, &sb[idx++],
+ &mask, inner, rx);
+
+- if (dr_mask_is_src_addr_set(&mask.inner))
++ if (DR_MASK_IS_SRC_IP_SET(&mask.inner))
+ mlx5dr_ste_build_eth_l3_ipv6_src(ste_ctx, &sb[idx++],
+ &mask, inner, rx);
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
+index 1cdfe4fccc7a9..01246a1ae7d13 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
+@@ -602,12 +602,34 @@ int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
+ used_hw_action_num);
+ }
+
++static int dr_ste_build_pre_check_spec(struct mlx5dr_domain *dmn,
++ struct mlx5dr_match_spec *spec)
++{
++ if (spec->ip_version) {
++ if (spec->ip_version != 0xf) {
++ mlx5dr_err(dmn,
++ "Partial ip_version mask with src/dst IP is not supported\n");
++ return -EINVAL;
++ }
++ } else if (spec->ethertype != 0xffff &&
++ (DR_MASK_IS_SRC_IP_SET(spec) || DR_MASK_IS_DST_IP_SET(spec))) {
++ mlx5dr_err(dmn,
++ "Partial/no ethertype mask with src/dst IP is not supported\n");
++ return -EINVAL;
++ }
++
++ return 0;
++}
++
+ int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
+ u8 match_criteria,
+ struct mlx5dr_match_param *mask,
+ struct mlx5dr_match_param *value)
+ {
+- if (!value && (match_criteria & DR_MATCHER_CRITERIA_MISC)) {
++ if (value)
++ return 0;
++
++ if (match_criteria & DR_MATCHER_CRITERIA_MISC) {
+ if (mask->misc.source_port && mask->misc.source_port != 0xffff) {
+ mlx5dr_err(dmn,
+ "Partial mask source_port is not supported\n");
+@@ -621,6 +643,14 @@ int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
+ }
+ }
+
++ if ((match_criteria & DR_MATCHER_CRITERIA_OUTER) &&
++ dr_ste_build_pre_check_spec(dmn, &mask->outer))
++ return -EINVAL;
++
++ if ((match_criteria & DR_MATCHER_CRITERIA_INNER) &&
++ dr_ste_build_pre_check_spec(dmn, &mask->inner))
++ return -EINVAL;
++
+ return 0;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
+index b20e8aabb861b..bc206836af6ac 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
+@@ -740,6 +740,16 @@ struct mlx5dr_match_param {
+ (_misc3)->icmpv4_code || \
+ (_misc3)->icmpv4_header_data)
+
++#define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \
++ (_spec)->src_ip_95_64 || \
++ (_spec)->src_ip_63_32 || \
++ (_spec)->src_ip_31_0)
++
++#define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \
++ (_spec)->dst_ip_95_64 || \
++ (_spec)->dst_ip_63_32 || \
++ (_spec)->dst_ip_31_0)
++
+ struct mlx5dr_esw_caps {
+ u64 drop_icm_address_rx;
+ u64 drop_icm_address_tx;
+@@ -1384,7 +1394,8 @@ int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
+ bool reformat_req,
+ u32 *tbl_id,
+ u32 *group_id,
+- bool ignore_flow_level);
++ bool ignore_flow_level,
++ u32 flow_source);
+ void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id,
+ u32 group_id);
+ #endif /* _DR_TYPES_H_ */
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c
+index 7e58f4e594b74..ae4597118f8b6 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c
+@@ -492,11 +492,13 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns,
+ } else if (num_term_actions > 1) {
+ bool ignore_flow_level =
+ !!(fte->action.flags & FLOW_ACT_IGNORE_FLOW_LEVEL);
++ u32 flow_source = fte->flow_context.flow_source;
+
+ tmp_action = mlx5dr_action_create_mult_dest_tbl(domain,
+ term_actions,
+ num_term_actions,
+- ignore_flow_level);
++ ignore_flow_level,
++ flow_source);
+ if (!tmp_action) {
+ err = -EOPNOTSUPP;
+ goto free_actions;
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
+index c5a8b16019991..7806e5c05b677 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h
+@@ -96,7 +96,8 @@ struct mlx5dr_action *
+ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn,
+ struct mlx5dr_action_dest *dests,
+ u32 num_of_dests,
+- bool ignore_flow_level);
++ bool ignore_flow_level,
++ u32 flow_source);
+
+ struct mlx5dr_action *mlx5dr_action_create_drop(void);
+
+@@ -160,6 +161,11 @@ struct mlx5dr_icm_buddy_mem {
+ * sync_ste command sets them free.
+ */
+ struct list_head hot_list;
++
++ /* Memory optimisation */
++ struct mlx5dr_ste *ste_arr;
++ struct list_head *miss_list;
++ u8 *hw_ste_arr;
+ };
+
+ int mlx5dr_buddy_init(struct mlx5dr_icm_buddy_mem *buddy,
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
+index 392ce3cb27f72..51b260d54237e 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h
++++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
+@@ -935,6 +935,18 @@ static inline int mlxsw_cmd_sw2hw_rdq(struct mlxsw_core *mlxsw_core,
+ */
+ MLXSW_ITEM32(cmd_mbox, sw2hw_dq, cq, 0x00, 24, 8);
+
++enum mlxsw_cmd_mbox_sw2hw_dq_sdq_lp {
++ MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_WQE,
++ MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_IGNORE_WQE,
++};
++
++/* cmd_mbox_sw2hw_dq_sdq_lp
++ * SDQ local Processing
++ * 0: local processing by wqe.lp
++ * 1: local processing (ignoring wqe.lp)
++ */
++MLXSW_ITEM32(cmd_mbox, sw2hw_dq, sdq_lp, 0x00, 23, 1);
++
+ /* cmd_mbox_sw2hw_dq_sdq_tclass
+ * SDQ: CPU Egress TClass
+ * RDQ: Reserved
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/i2c.c b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
+index 939b692ffc335..ce843ea914646 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/i2c.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
+@@ -650,6 +650,7 @@ static int mlxsw_i2c_probe(struct i2c_client *client,
+ return 0;
+
+ errout:
++ mutex_destroy(&mlxsw_i2c->cmd.lock);
+ i2c_set_clientdata(client, NULL);
+
+ return err;
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
+index fcace73eae40f..d9f9cbba62465 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
+@@ -285,6 +285,7 @@ static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
+ struct mlxsw_pci_queue *q)
+ {
+ int tclass;
++ int lp;
+ int i;
+ int err;
+
+@@ -292,9 +293,12 @@ static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
+ q->consumer_counter = 0;
+ tclass = q->num == MLXSW_PCI_SDQ_EMAD_INDEX ? MLXSW_PCI_SDQ_EMAD_TC :
+ MLXSW_PCI_SDQ_CTL_TC;
++ lp = q->num == MLXSW_PCI_SDQ_EMAD_INDEX ? MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_IGNORE_WQE :
++ MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_WQE;
+
+ /* Set CQ of same number of this SDQ. */
+ mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);
++ mlxsw_cmd_mbox_sw2hw_dq_sdq_lp_set(mbox, lp);
+ mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, tclass);
+ mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
+ for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
+@@ -1678,7 +1682,7 @@ static int mlxsw_pci_skb_transmit(void *bus_priv, struct sk_buff *skb,
+
+ wqe = elem_info->elem;
+ mlxsw_pci_wqe_c_set(wqe, 1); /* always report completion */
+- mlxsw_pci_wqe_lp_set(wqe, !!tx_info->is_emad);
++ mlxsw_pci_wqe_lp_set(wqe, 0);
+ mlxsw_pci_wqe_type_set(wqe, MLXSW_PCI_WQE_TYPE_ETHERNET);
+
+ err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
+@@ -1973,6 +1977,7 @@ int mlxsw_pci_driver_register(struct pci_driver *pci_driver)
+ {
+ pci_driver->probe = mlxsw_pci_probe;
+ pci_driver->remove = mlxsw_pci_remove;
++ pci_driver->shutdown = mlxsw_pci_remove;
+ return pci_register_driver(pci_driver);
+ }
+ EXPORT_SYMBOL(mlxsw_pci_driver_register);
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+index 250c5a24264dc..edfdd44de579c 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+@@ -2131,7 +2131,7 @@ static void mlxsw_sp_pude_event_func(const struct mlxsw_reg_info *reg,
+ max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
+ local_port = mlxsw_reg_pude_local_port_get(pude_pl);
+
+- if (WARN_ON_ONCE(local_port >= max_ports))
++ if (WARN_ON_ONCE(!local_port || local_port >= max_ports))
+ return;
+ mlxsw_sp_port = mlxsw_sp->ports[local_port];
+ if (!mlxsw_sp_port)
+diff --git a/drivers/net/ethernet/micrel/Kconfig b/drivers/net/ethernet/micrel/Kconfig
+index 93df3049cdc05..1b632cdd76309 100644
+--- a/drivers/net/ethernet/micrel/Kconfig
++++ b/drivers/net/ethernet/micrel/Kconfig
+@@ -39,6 +39,7 @@ config KS8851
+ config KS8851_MLL
+ tristate "Micrel KS8851 MLL"
+ depends on HAS_IOMEM
++ depends on PTP_1588_CLOCK_OPTIONAL
+ select MII
+ select CRC32
+ select EEPROM_93CX6
+diff --git a/drivers/net/ethernet/micrel/ks8851_par.c b/drivers/net/ethernet/micrel/ks8851_par.c
+index 2e8fcce50f9d1..c6f517c07bb9a 100644
+--- a/drivers/net/ethernet/micrel/ks8851_par.c
++++ b/drivers/net/ethernet/micrel/ks8851_par.c
+@@ -321,6 +321,8 @@ static int ks8851_probe_par(struct platform_device *pdev)
+ return ret;
+
+ netdev->irq = platform_get_irq(pdev, 0);
++ if (netdev->irq < 0)
++ return netdev->irq;
+
+ return ks8851_probe_common(netdev, dev, msg_enable);
+ }
+diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c
+index 4d5a5d6595b3b..d64ce65a3c174 100644
+--- a/drivers/net/ethernet/microchip/lan743x_main.c
++++ b/drivers/net/ethernet/microchip/lan743x_main.c
+@@ -914,8 +914,7 @@ static int lan743x_phy_reset(struct lan743x_adapter *adapter)
+ }
+
+ static void lan743x_phy_update_flowcontrol(struct lan743x_adapter *adapter,
+- u8 duplex, u16 local_adv,
+- u16 remote_adv)
++ u16 local_adv, u16 remote_adv)
+ {
+ struct lan743x_phy *phy = &adapter->phy;
+ u8 cap;
+@@ -943,7 +942,6 @@ static void lan743x_phy_link_status_change(struct net_device *netdev)
+
+ phy_print_status(phydev);
+ if (phydev->state == PHY_RUNNING) {
+- struct ethtool_link_ksettings ksettings;
+ int remote_advertisement = 0;
+ int local_advertisement = 0;
+
+@@ -980,18 +978,14 @@ static void lan743x_phy_link_status_change(struct net_device *netdev)
+ }
+ lan743x_csr_write(adapter, MAC_CR, data);
+
+- memset(&ksettings, 0, sizeof(ksettings));
+- phy_ethtool_get_link_ksettings(netdev, &ksettings);
+ local_advertisement =
+ linkmode_adv_to_mii_adv_t(phydev->advertising);
+ remote_advertisement =
+ linkmode_adv_to_mii_adv_t(phydev->lp_advertising);
+
+- lan743x_phy_update_flowcontrol(adapter,
+- ksettings.base.duplex,
+- local_advertisement,
++ lan743x_phy_update_flowcontrol(adapter, local_advertisement,
+ remote_advertisement);
+- lan743x_ptp_update_latency(adapter, ksettings.base.speed);
++ lan743x_ptp_update_latency(adapter, phydev->speed);
+ }
+ }
+
+diff --git a/drivers/net/ethernet/microchip/sparx5/Kconfig b/drivers/net/ethernet/microchip/sparx5/Kconfig
+index 7bdbb2d09a148..cc5e48e1bb4c3 100644
+--- a/drivers/net/ethernet/microchip/sparx5/Kconfig
++++ b/drivers/net/ethernet/microchip/sparx5/Kconfig
+@@ -4,6 +4,8 @@ config SPARX5_SWITCH
+ depends on HAS_IOMEM
+ depends on OF
+ depends on ARCH_SPARX5 || COMPILE_TEST
++ depends on PTP_1588_CLOCK_OPTIONAL
++ depends on BRIDGE || BRIDGE=n
+ select PHYLINK
+ select PHY_SPARX5_SERDES
+ select RESET_CONTROLLER
+diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
+index 59783fc46a7b9..10b866e9f7266 100644
+--- a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
++++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
+@@ -1103,7 +1103,7 @@ void sparx5_get_stats64(struct net_device *ndev,
+ stats->tx_carrier_errors = portstats[spx5_stats_tx_csense_cnt];
+ stats->tx_window_errors = portstats[spx5_stats_tx_late_coll_cnt];
+ stats->rx_dropped = portstats[spx5_stats_ana_ac_port_stat_lsb_cnt];
+- for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx, ++stats)
++ for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx)
+ stats->rx_dropped += portstats[spx5_stats_green_p0_rx_port_drop
+ + idx];
+ stats->tx_dropped = portstats[spx5_stats_tx_local_drop];
+diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c
+index 7436f62fa1525..174ad95e746a3 100644
+--- a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c
++++ b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c
+@@ -420,6 +420,8 @@ static int sparx5_fdma_tx_alloc(struct sparx5 *sparx5)
+ db_hw->dataptr = phys;
+ db_hw->status = 0;
+ db = devm_kzalloc(sparx5->dev, sizeof(*db), GFP_KERNEL);
++ if (!db)
++ return -ENOMEM;
+ db->cpu_addr = cpu_addr;
+ list_add_tail(&db->list, &tx->db_list);
+ }
+diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
+index dc7e5ea6ec158..148d431fcde42 100644
+--- a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
++++ b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
+@@ -145,9 +145,9 @@ static void sparx5_xtr_grp(struct sparx5 *sparx5, u8 grp, bool byte_swap)
+ skb_put(skb, byte_cnt - ETH_FCS_LEN);
+ eth_skb_pad(skb);
+ skb->protocol = eth_type_trans(skb, netdev);
+- netif_rx(skb);
+ netdev->stats.rx_bytes += skb->len;
+ netdev->stats.rx_packets++;
++ netif_rx(skb);
+ }
+
+ static int sparx5_inject(struct sparx5 *sparx5,
+diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
+index 4ce490a25f332..8e56ffa1c4f7a 100644
+--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
++++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
+@@ -58,16 +58,6 @@ int sparx5_vlan_vid_add(struct sparx5_port *port, u16 vid, bool pvid,
+ struct sparx5 *sparx5 = port->sparx5;
+ int ret;
+
+- /* Make the port a member of the VLAN */
+- set_bit(port->portno, sparx5->vlan_mask[vid]);
+- ret = sparx5_vlant_set_mask(sparx5, vid);
+- if (ret)
+- return ret;
+-
+- /* Default ingress vlan classification */
+- if (pvid)
+- port->pvid = vid;
+-
+ /* Untagged egress vlan classification */
+ if (untagged && port->vid != vid) {
+ if (port->vid) {
+@@ -79,6 +69,16 @@ int sparx5_vlan_vid_add(struct sparx5_port *port, u16 vid, bool pvid,
+ port->vid = vid;
+ }
+
++ /* Make the port a member of the VLAN */
++ set_bit(port->portno, sparx5->vlan_mask[vid]);
++ ret = sparx5_vlant_set_mask(sparx5, vid);
++ if (ret)
++ return ret;
++
++ /* Default ingress vlan classification */
++ if (pvid)
++ port->pvid = vid;
++
+ sparx5_vlan_port_apply(sparx5, port);
+
+ return 0;
+diff --git a/drivers/net/ethernet/mscc/Kconfig b/drivers/net/ethernet/mscc/Kconfig
+index b6a73d151dec1..8dd8c7f425d2c 100644
+--- a/drivers/net/ethernet/mscc/Kconfig
++++ b/drivers/net/ethernet/mscc/Kconfig
+@@ -28,7 +28,7 @@ config MSCC_OCELOT_SWITCH
+ depends on BRIDGE || BRIDGE=n
+ depends on NET_SWITCHDEV
+ depends on HAS_IOMEM
+- depends on OF_NET
++ depends on OF
+ select MSCC_OCELOT_SWITCH_LIB
+ select GENERIC_PHY
+ help
+diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c
+index a08e4f530c1c1..a59300d9e0000 100644
+--- a/drivers/net/ethernet/mscc/ocelot.c
++++ b/drivers/net/ethernet/mscc/ocelot.c
+@@ -555,7 +555,10 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port,
+
+ ocelot_write_rix(ocelot, 0, ANA_POL_FLOWC, port);
+
+- ocelot_fields_write(ocelot, port, SYS_PAUSE_CFG_PAUSE_ENA, tx_pause);
++ /* Don't attempt to send PAUSE frames on the NPI port, it's broken */
++ if (port != ocelot->npi)
++ ocelot_fields_write(ocelot, port, SYS_PAUSE_CFG_PAUSE_ENA,
++ tx_pause);
+
+ /* Undo the effects of ocelot_phylink_mac_link_down:
+ * enable MAC module
+@@ -1175,12 +1178,6 @@ int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr)
+ switch (cfg.rx_filter) {
+ case HWTSTAMP_FILTER_NONE:
+ break;
+- case HWTSTAMP_FILTER_ALL:
+- case HWTSTAMP_FILTER_SOME:
+- case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+- case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+- case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+- case HWTSTAMP_FILTER_NTP_ALL:
+ case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+ case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+ case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+@@ -1218,12 +1215,11 @@ void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data)
+ }
+ EXPORT_SYMBOL(ocelot_get_strings);
+
++/* Caller must hold &ocelot->stats_lock */
+ static void ocelot_update_stats(struct ocelot *ocelot)
+ {
+ int i, j;
+
+- mutex_lock(&ocelot->stats_lock);
+-
+ for (i = 0; i < ocelot->num_phys_ports; i++) {
+ /* Configure the port to read the stats from */
+ ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(i), SYS_STAT_CFG);
+@@ -1242,8 +1238,6 @@ static void ocelot_update_stats(struct ocelot *ocelot)
+ ~(u64)U32_MAX) + val;
+ }
+ }
+-
+- mutex_unlock(&ocelot->stats_lock);
+ }
+
+ static void ocelot_check_stats_work(struct work_struct *work)
+@@ -1252,7 +1246,9 @@ static void ocelot_check_stats_work(struct work_struct *work)
+ struct ocelot *ocelot = container_of(del_work, struct ocelot,
+ stats_work);
+
++ mutex_lock(&ocelot->stats_lock);
+ ocelot_update_stats(ocelot);
++ mutex_unlock(&ocelot->stats_lock);
+
+ queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
+ OCELOT_STATS_CHECK_DELAY);
+@@ -1262,12 +1258,16 @@ void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data)
+ {
+ int i;
+
++ mutex_lock(&ocelot->stats_lock);
++
+ /* check and update now */
+ ocelot_update_stats(ocelot);
+
+ /* Copy all counters */
+ for (i = 0; i < ocelot->num_stats; i++)
+ *data++ = ocelot->stats[port * ocelot->num_stats + i];
++
++ mutex_unlock(&ocelot->stats_lock);
+ }
+ EXPORT_SYMBOL(ocelot_get_ethtool_stats);
+
+@@ -1299,14 +1299,16 @@ int ocelot_get_ts_info(struct ocelot *ocelot, int port,
+ SOF_TIMESTAMPING_RAW_HARDWARE;
+ info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON) |
+ BIT(HWTSTAMP_TX_ONESTEP_SYNC);
+- info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | BIT(HWTSTAMP_FILTER_ALL);
++ info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
++ BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
++ BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
++ BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
+
+ return 0;
+ }
+ EXPORT_SYMBOL(ocelot_get_ts_info);
+
+-static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond,
+- bool only_active_ports)
++static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond)
+ {
+ u32 mask = 0;
+ int port;
+@@ -1317,12 +1319,8 @@ static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond,
+ if (!ocelot_port)
+ continue;
+
+- if (ocelot_port->bond == bond) {
+- if (only_active_ports && !ocelot_port->lag_tx_active)
+- continue;
+-
++ if (ocelot_port->bond == bond)
+ mask |= BIT(port);
+- }
+ }
+
+ return mask;
+@@ -1409,10 +1407,8 @@ void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot)
+ mask = ocelot_get_bridge_fwd_mask(ocelot, port, bridge);
+ mask |= cpu_fwd_mask;
+ mask &= ~BIT(port);
+- if (bond) {
+- mask &= ~ocelot_get_bond_mask(ocelot, bond,
+- false);
+- }
++ if (bond)
++ mask &= ~ocelot_get_bond_mask(ocelot, bond);
+ } else {
+ /* Standalone ports forward only to DSA tag_8021q CPU
+ * ports (if those exist), or to the hardware CPU port
+@@ -1730,13 +1726,17 @@ static void ocelot_set_aggr_pgids(struct ocelot *ocelot)
+ if (!bond || (visited & BIT(lag)))
+ continue;
+
+- bond_mask = ocelot_get_bond_mask(ocelot, bond, true);
++ bond_mask = ocelot_get_bond_mask(ocelot, bond);
+
+ for_each_set_bit(port, &bond_mask, ocelot->num_phys_ports) {
++ struct ocelot_port *ocelot_port = ocelot->ports[port];
++
+ // Destination mask
+ ocelot_write_rix(ocelot, bond_mask,
+ ANA_PGID_PGID, port);
+- aggr_idx[num_active_ports++] = port;
++
++ if (ocelot_port->lag_tx_active)
++ aggr_idx[num_active_ports++] = port;
+ }
+
+ for_each_aggr_pgid(ocelot, i) {
+@@ -1785,8 +1785,7 @@ static void ocelot_setup_logical_port_ids(struct ocelot *ocelot)
+
+ bond = ocelot_port->bond;
+ if (bond) {
+- int lag = __ffs(ocelot_get_bond_mask(ocelot, bond,
+- false));
++ int lag = __ffs(ocelot_get_bond_mask(ocelot, bond));
+
+ ocelot_rmw_gix(ocelot,
+ ANA_PORT_PORT_CFG_PORTID_VAL(lag),
+@@ -1933,6 +1932,8 @@ static void ocelot_port_set_mcast_flood(struct ocelot *ocelot, int port,
+ val = BIT(port);
+
+ ocelot_rmw_rix(ocelot, val, BIT(port), ANA_PGID_PGID, PGID_MC);
++ ocelot_rmw_rix(ocelot, val, BIT(port), ANA_PGID_PGID, PGID_MCIPV4);
++ ocelot_rmw_rix(ocelot, val, BIT(port), ANA_PGID_PGID, PGID_MCIPV6);
+ }
+
+ static void ocelot_port_set_bcast_flood(struct ocelot *ocelot, int port,
+diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c
+index 8b843d3c9189a..a3a5ad5dbb0e0 100644
+--- a/drivers/net/ethernet/mscc/ocelot_flower.c
++++ b/drivers/net/ethernet/mscc/ocelot_flower.c
+@@ -54,6 +54,12 @@ static int ocelot_chain_to_block(int chain, bool ingress)
+ */
+ static int ocelot_chain_to_lookup(int chain)
+ {
++ /* Backwards compatibility with older, single-chain tc-flower
++ * offload support in Ocelot
++ */
++ if (chain == 0)
++ return 0;
++
+ return (chain / VCAP_LOOKUP) % 10;
+ }
+
+@@ -62,7 +68,15 @@ static int ocelot_chain_to_lookup(int chain)
+ */
+ static int ocelot_chain_to_pag(int chain)
+ {
+- int lookup = ocelot_chain_to_lookup(chain);
++ int lookup;
++
++ /* Backwards compatibility with older, single-chain tc-flower
++ * offload support in Ocelot
++ */
++ if (chain == 0)
++ return 0;
++
++ lookup = ocelot_chain_to_lookup(chain);
+
+ /* calculate PAG value as chain index relative to the first PAG */
+ return chain - VCAP_IS2_CHAIN(lookup, 0);
+@@ -192,9 +206,10 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port,
+ filter->type = OCELOT_VCAP_FILTER_OFFLOAD;
+ break;
+ case FLOW_ACTION_TRAP:
+- if (filter->block_id != VCAP_IS2) {
++ if (filter->block_id != VCAP_IS2 ||
++ filter->lookup != 0) {
+ NL_SET_ERR_MSG_MOD(extack,
+- "Trap action can only be offloaded to VCAP IS2");
++ "Trap action can only be offloaded to VCAP IS2 lookup 0");
+ return -EOPNOTSUPP;
+ }
+ if (filter->goto_target != -1) {
+@@ -467,13 +482,6 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress,
+ return -EOPNOTSUPP;
+ }
+
+- if (filter->block_id == VCAP_IS1 &&
+- !is_zero_ether_addr(match.mask->dst)) {
+- NL_SET_ERR_MSG_MOD(extack,
+- "Key type S1_NORMAL cannot match on destination MAC");
+- return -EOPNOTSUPP;
+- }
+-
+ /* The hw support mac matches only for MAC_ETYPE key,
+ * therefore if other matches(port, tcp flags, etc) are added
+ * then just bail out
+@@ -488,6 +496,14 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress,
+ return -EOPNOTSUPP;
+
+ flow_rule_match_eth_addrs(rule, &match);
++
++ if (filter->block_id == VCAP_IS1 &&
++ !is_zero_ether_addr(match.mask->dst)) {
++ NL_SET_ERR_MSG_MOD(extack,
++ "Key type S1_NORMAL cannot match on destination MAC");
++ return -EOPNOTSUPP;
++ }
++
+ filter->key_type = OCELOT_VCAP_KEY_ETYPE;
+ ether_addr_copy(filter->key.etype.dmac.value,
+ match.key->dst);
+diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c
+index 2545727fd5b2f..c08c56e07b1d3 100644
+--- a/drivers/net/ethernet/mscc/ocelot_net.c
++++ b/drivers/net/ethernet/mscc/ocelot_net.c
+@@ -1168,7 +1168,7 @@ static int ocelot_netdevice_bridge_join(struct net_device *dev,
+ ocelot_port_bridge_join(ocelot, port, bridge);
+
+ err = switchdev_bridge_port_offload(brport_dev, dev, priv,
+- &ocelot_netdevice_nb,
++ &ocelot_switchdev_nb,
+ &ocelot_switchdev_blocking_nb,
+ false, extack);
+ if (err)
+@@ -1182,7 +1182,7 @@ static int ocelot_netdevice_bridge_join(struct net_device *dev,
+
+ err_switchdev_sync:
+ switchdev_bridge_port_unoffload(brport_dev, priv,
+- &ocelot_netdevice_nb,
++ &ocelot_switchdev_nb,
+ &ocelot_switchdev_blocking_nb);
+ err_switchdev_offload:
+ ocelot_port_bridge_leave(ocelot, port, bridge);
+@@ -1195,7 +1195,7 @@ static void ocelot_netdevice_pre_bridge_leave(struct net_device *dev,
+ struct ocelot_port_private *priv = netdev_priv(dev);
+
+ switchdev_bridge_port_unoffload(brport_dev, priv,
+- &ocelot_netdevice_nb,
++ &ocelot_switchdev_nb,
+ &ocelot_switchdev_blocking_nb);
+ }
+
+diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.c b/drivers/net/ethernet/mscc/ocelot_vcap.c
+index 99d7376a70a74..732a4ef22518d 100644
+--- a/drivers/net/ethernet/mscc/ocelot_vcap.c
++++ b/drivers/net/ethernet/mscc/ocelot_vcap.c
+@@ -373,7 +373,6 @@ static void is2_entry_set(struct ocelot *ocelot, int ix,
+ OCELOT_VCAP_BIT_0);
+ vcap_key_set(vcap, &data, VCAP_IS2_HK_IGR_PORT_MASK, 0,
+ ~filter->ingress_port_mask);
+- vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_FIRST, OCELOT_VCAP_BIT_ANY);
+ vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_HOST_MATCH,
+ OCELOT_VCAP_BIT_ANY);
+ vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L2_MC, filter->dmac_mc);
+@@ -1153,6 +1152,8 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot,
+ struct ocelot_vcap_filter *tmp;
+
+ tmp = ocelot_vcap_block_find_filter_by_index(block, i);
++ /* Read back the filter's counters before moving it */
++ vcap_entry_get(ocelot, i - 1, tmp);
+ vcap_entry_set(ocelot, i, tmp);
+ }
+
+@@ -1192,7 +1193,11 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot,
+ struct ocelot_vcap_filter del_filter;
+ int i, index;
+
++ /* Need to inherit the block_id so that vcap_entry_set()
++ * does not get confused and knows where to install it.
++ */
+ memset(&del_filter, 0, sizeof(del_filter));
++ del_filter.block_id = filter->block_id;
+
+ /* Gets index of the filter */
+ index = ocelot_vcap_block_get_filter_index(block, filter);
+@@ -1207,6 +1212,8 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot,
+ struct ocelot_vcap_filter *tmp;
+
+ tmp = ocelot_vcap_block_find_filter_by_index(block, i);
++ /* Read back the filter's counters before moving it */
++ vcap_entry_get(ocelot, i + 1, tmp);
+ vcap_entry_set(ocelot, i, tmp);
+ }
+
+diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
+index c1a75b08ced7e..052696ce50963 100644
+--- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
++++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
+@@ -2900,11 +2900,9 @@ static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb,
+ status = myri10ge_xmit(curr, dev);
+ if (status != 0) {
+ dev_kfree_skb_any(curr);
+- if (segs != NULL) {
+- curr = segs;
+- segs = next;
++ skb_list_walk_safe(next, curr, next) {
+ curr->next = NULL;
+- dev_kfree_skb_any(segs);
++ dev_kfree_skb_any(curr);
+ }
+ goto drop;
+ }
+diff --git a/drivers/net/ethernet/natsemi/xtsonic.c b/drivers/net/ethernet/natsemi/xtsonic.c
+index ca4686094701c..0a02d8bd0a3e5 100644
+--- a/drivers/net/ethernet/natsemi/xtsonic.c
++++ b/drivers/net/ethernet/natsemi/xtsonic.c
+@@ -120,7 +120,7 @@ static const struct net_device_ops xtsonic_netdev_ops = {
+ .ndo_set_mac_address = eth_mac_addr,
+ };
+
+-static int __init sonic_probe1(struct net_device *dev)
++static int sonic_probe1(struct net_device *dev)
+ {
+ unsigned int silicon_revision;
+ struct sonic_local *lp = netdev_priv(dev);
+diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
+index ab70179728f63..babd374333f34 100644
+--- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
++++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
+@@ -922,8 +922,8 @@ nfp_tunnel_add_shared_mac(struct nfp_app *app, struct net_device *netdev,
+ int port, bool mod)
+ {
+ struct nfp_flower_priv *priv = app->priv;
+- int ida_idx = NFP_MAX_MAC_INDEX, err;
+ struct nfp_tun_offloaded_mac *entry;
++ int ida_idx = -1, err;
+ u16 nfp_mac_idx = 0;
+
+ entry = nfp_tunnel_lookup_offloaded_macs(app, netdev->dev_addr);
+@@ -997,7 +997,7 @@ err_remove_hash:
+ err_free_entry:
+ kfree(entry);
+ err_free_ida:
+- if (ida_idx != NFP_MAX_MAC_INDEX)
++ if (ida_idx != -1)
+ ida_simple_remove(&priv->tun.mac_off_ids, ida_idx);
+
+ return err;
+@@ -1011,6 +1011,7 @@ nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
+ struct nfp_flower_repr_priv *repr_priv;
+ struct nfp_tun_offloaded_mac *entry;
+ struct nfp_repr *repr;
++ u16 nfp_mac_idx;
+ int ida_idx;
+
+ entry = nfp_tunnel_lookup_offloaded_macs(app, mac);
+@@ -1029,8 +1030,6 @@ nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
+ entry->bridge_count--;
+
+ if (!entry->bridge_count && entry->ref_count) {
+- u16 nfp_mac_idx;
+-
+ nfp_mac_idx = entry->index & ~NFP_TUN_PRE_TUN_IDX_BIT;
+ if (__nfp_tunnel_offload_mac(app, mac, nfp_mac_idx,
+ false)) {
+@@ -1046,7 +1045,6 @@ nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
+
+ /* If MAC is now used by 1 repr set the offloaded MAC index to port. */
+ if (entry->ref_count == 1 && list_is_singular(&entry->repr_list)) {
+- u16 nfp_mac_idx;
+ int port, err;
+
+ repr_priv = list_first_entry(&entry->repr_list,
+@@ -1074,8 +1072,14 @@ nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
+ WARN_ON_ONCE(rhashtable_remove_fast(&priv->tun.offloaded_macs,
+ &entry->ht_node,
+ offloaded_macs_params));
++
++ if (nfp_flower_is_supported_bridge(netdev))
++ nfp_mac_idx = entry->index & ~NFP_TUN_PRE_TUN_IDX_BIT;
++ else
++ nfp_mac_idx = entry->index;
++
+ /* If MAC has global ID then extract and free the ida entry. */
+- if (nfp_tunnel_is_mac_idx_global(entry->index)) {
++ if (nfp_tunnel_is_mac_idx_global(nfp_mac_idx)) {
+ ida_idx = nfp_tunnel_get_ida_from_global_mac_idx(entry->index);
+ ida_simple_remove(&priv->tun.mac_off_ids, ida_idx);
+ }
+diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
+index df203738511bf..0b1865e9f0b59 100644
+--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
++++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
+@@ -565,7 +565,6 @@ struct nfp_net_dp {
+ * @exn_name: Name for Exception interrupt
+ * @shared_handler: Handler for shared interrupts
+ * @shared_name: Name for shared interrupt
+- * @me_freq_mhz: ME clock_freq (MHz)
+ * @reconfig_lock: Protects @reconfig_posted, @reconfig_timer_active,
+ * @reconfig_sync_present and HW reconfiguration request
+ * regs/machinery from async requests (sync must take
+@@ -650,8 +649,6 @@ struct nfp_net {
+ irq_handler_t shared_handler;
+ char shared_name[IFNAMSIZ + 8];
+
+- u32 me_freq_mhz;
+-
+ bool link_up;
+ spinlock_t link_status_lock;
+
+diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+index 5bfa22accf2c9..850bfdf83d0a4 100644
+--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
++++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+@@ -2067,7 +2067,7 @@ static int nfp_net_poll(struct napi_struct *napi, int budget)
+ if (napi_complete_done(napi, pkts_polled))
+ nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);
+
+- if (r_vec->nfp_net->rx_coalesce_adapt_on) {
++ if (r_vec->nfp_net->rx_coalesce_adapt_on && r_vec->rx_ring) {
+ struct dim_sample dim_sample = {};
+ unsigned int start;
+ u64 pkts, bytes;
+@@ -2082,7 +2082,7 @@ static int nfp_net_poll(struct napi_struct *napi, int budget)
+ net_dim(&r_vec->rx_dim, dim_sample);
+ }
+
+- if (r_vec->nfp_net->tx_coalesce_adapt_on) {
++ if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {
+ struct dim_sample dim_sample = {};
+ unsigned int start;
+ u64 pkts, bytes;
+@@ -3016,10 +3016,8 @@ static void nfp_net_rx_dim_work(struct work_struct *work)
+
+ /* copy RX interrupt coalesce parameters */
+ value = (moder.pkts << 16) | (factor * moder.usec);
+- rtnl_lock();
+ nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(r_vec->rx_ring->idx), value);
+ (void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);
+- rtnl_unlock();
+
+ dim->state = DIM_START_MEASURE;
+ }
+@@ -3047,10 +3045,8 @@ static void nfp_net_tx_dim_work(struct work_struct *work)
+
+ /* copy TX interrupt coalesce parameters */
+ value = (moder.pkts << 16) | (factor * moder.usec);
+- rtnl_lock();
+ nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(r_vec->tx_ring->idx), value);
+ (void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);
+- rtnl_unlock();
+
+ dim->state = DIM_START_MEASURE;
+ }
+diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+index 0685ece1f155d..be1a358baadb9 100644
+--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
++++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+@@ -1343,7 +1343,7 @@ static int nfp_net_set_coalesce(struct net_device *netdev,
+ * ME timestamp ticks. There are 16 ME clock cycles for each timestamp
+ * count.
+ */
+- factor = nn->me_freq_mhz / 16;
++ factor = nn->tlv_caps.me_freq_mhz / 16;
+
+ /* Each pair of (usecs, max_frames) fields specifies that interrupts
+ * should be coalesced until
+diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
+index d7ac0307797fd..34c0d2ddf9ef6 100644
+--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
++++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
+@@ -803,8 +803,10 @@ int nfp_cpp_area_cache_add(struct nfp_cpp *cpp, size_t size)
+ return -ENOMEM;
+
+ cache = kzalloc(sizeof(*cache), GFP_KERNEL);
+- if (!cache)
++ if (!cache) {
++ nfp_cpp_area_free(area);
+ return -ENOMEM;
++ }
+
+ cache->id = 0;
+ cache->addr = 0;
+diff --git a/drivers/net/ethernet/nxp/lpc_eth.c b/drivers/net/ethernet/nxp/lpc_eth.c
+index c910fa2f40a4b..919140522885d 100644
+--- a/drivers/net/ethernet/nxp/lpc_eth.c
++++ b/drivers/net/ethernet/nxp/lpc_eth.c
+@@ -1469,6 +1469,7 @@ static int lpc_eth_drv_resume(struct platform_device *pdev)
+ {
+ struct net_device *ndev = platform_get_drvdata(pdev);
+ struct netdata_local *pldat;
++ int ret;
+
+ if (device_may_wakeup(&pdev->dev))
+ disable_irq_wake(ndev->irq);
+@@ -1478,7 +1479,9 @@ static int lpc_eth_drv_resume(struct platform_device *pdev)
+ pldat = netdev_priv(ndev);
+
+ /* Enable interface clock */
+- clk_enable(pldat->clk);
++ ret = clk_enable(pldat->clk);
++ if (ret)
++ return ret;
+
+ /* Reset and initialize */
+ __lpc_eth_reset(pldat);
+diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+index 7e296fa71b368..d324c292318b3 100644
+--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
++++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+@@ -255,7 +255,7 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ err = ionic_map_bars(ionic);
+ if (err)
+- goto err_out_pci_disable_device;
++ goto err_out_pci_release_regions;
+
+ /* Configure the device */
+ err = ionic_setup(ionic);
+@@ -331,6 +331,9 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ goto err_out_deregister_lifs;
+ }
+
++ mod_timer(&ionic->watchdog_timer,
++ round_jiffies(jiffies + ionic->watchdog_period));
++
+ return 0;
+
+ err_out_deregister_lifs:
+@@ -348,7 +351,6 @@ err_out_port_reset:
+ err_out_reset:
+ ionic_reset(ionic);
+ err_out_teardown:
+- del_timer_sync(&ionic->watchdog_timer);
+ pci_clear_master(pdev);
+ /* Don't fail the probe for these errors, keep
+ * the hw interface around for inspection
+@@ -357,6 +359,7 @@ err_out_teardown:
+
+ err_out_unmap_bars:
+ ionic_unmap_bars(ionic);
++err_out_pci_release_regions:
+ pci_release_regions(pdev);
+ err_out_pci_disable_device:
+ pci_disable_device(pdev);
+diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
+index 0d6858ab511c6..b778d8264bca0 100644
+--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.c
++++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
+@@ -122,9 +122,6 @@ int ionic_dev_setup(struct ionic *ionic)
+ idev->fw_generation = IONIC_FW_STS_F_GENERATION &
+ ioread8(&idev->dev_info_regs->fw_status);
+
+- mod_timer(&ionic->watchdog_timer,
+- round_jiffies(jiffies + ionic->watchdog_period));
+-
+ idev->db_pages = bar->vaddr;
+ idev->phy_db_pages = bar->bus_addr;
+
+@@ -132,6 +129,16 @@ int ionic_dev_setup(struct ionic *ionic)
+ }
+
+ /* Devcmd Interface */
++bool ionic_is_fw_running(struct ionic_dev *idev)
++{
++ u8 fw_status = ioread8(&idev->dev_info_regs->fw_status);
++
++ /* firmware is useful only if the running bit is set and
++ * fw_status != 0xff (bad PCI read)
++ */
++ return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
++}
++
+ int ionic_heartbeat_check(struct ionic *ionic)
+ {
+ struct ionic_dev *idev = &ionic->idev;
+@@ -155,13 +162,10 @@ do_check_time:
+ goto do_check_time;
+ }
+
+- /* firmware is useful only if the running bit is set and
+- * fw_status != 0xff (bad PCI read)
+- * If fw_status is not ready don't bother with the generation.
+- */
+ fw_status = ioread8(&idev->dev_info_regs->fw_status);
+
+- if (fw_status == 0xff || !(fw_status & IONIC_FW_STS_F_RUNNING)) {
++ /* If fw_status is not ready don't bother with the generation */
++ if (!ionic_is_fw_running(idev)) {
+ fw_status_ready = false;
+ } else {
+ fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
+diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+index 8311086fb1f49..922bb6c9e01d5 100644
+--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
++++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+@@ -357,5 +357,6 @@ void ionic_q_rewind(struct ionic_queue *q, struct ionic_desc_info *start);
+ void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
+ unsigned int stop_index);
+ int ionic_heartbeat_check(struct ionic *ionic);
++bool ionic_is_fw_running(struct ionic_dev *idev);
+
+ #endif /* _IONIC_DEV_H_ */
+diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+index 7f3322ce044c7..6ac507ddf09af 100644
+--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
++++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+@@ -3283,7 +3283,7 @@ int ionic_lif_init(struct ionic_lif *lif)
+ return -EINVAL;
+ }
+
+- lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
++ lif->dbid_inuse = bitmap_zalloc(lif->dbid_count, GFP_KERNEL);
+ if (!lif->dbid_inuse) {
+ dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
+ return -ENOMEM;
+diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c
+index 6f07bf509efed..480f85bc17f99 100644
+--- a/drivers/net/ethernet/pensando/ionic/ionic_main.c
++++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c
+@@ -328,10 +328,10 @@ int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
+
+ static void ionic_dev_cmd_clean(struct ionic *ionic)
+ {
+- union __iomem ionic_dev_cmd_regs *regs = ionic->idev.dev_cmd_regs;
++ struct ionic_dev *idev = &ionic->idev;
+
+- iowrite32(0, &regs->doorbell);
+- memset_io(&regs->cmd, 0, sizeof(regs->cmd));
++ iowrite32(0, &idev->dev_cmd_regs->doorbell);
++ memset_io(&idev->dev_cmd_regs->cmd, 0, sizeof(idev->dev_cmd_regs->cmd));
+ }
+
+ int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds)
+@@ -488,6 +488,9 @@ int ionic_reset(struct ionic *ionic)
+ struct ionic_dev *idev = &ionic->idev;
+ int err;
+
++ if (!ionic_is_fw_running(idev))
++ return 0;
++
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_reset(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+@@ -560,15 +563,17 @@ int ionic_port_init(struct ionic *ionic)
+ int ionic_port_reset(struct ionic *ionic)
+ {
+ struct ionic_dev *idev = &ionic->idev;
+- int err;
++ int err = 0;
+
+ if (!idev->port_info)
+ return 0;
+
+- mutex_lock(&ionic->dev_cmd_lock);
+- ionic_dev_cmd_port_reset(idev);
+- err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+- mutex_unlock(&ionic->dev_cmd_lock);
++ if (ionic_is_fw_running(idev)) {
++ mutex_lock(&ionic->dev_cmd_lock);
++ ionic_dev_cmd_port_reset(idev);
++ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
++ mutex_unlock(&ionic->dev_cmd_lock);
++ }
+
+ dma_free_coherent(ionic->dev, idev->port_info_sz,
+ idev->port_info, idev->port_info_pa);
+@@ -576,9 +581,6 @@ int ionic_port_reset(struct ionic *ionic)
+ idev->port_info = NULL;
+ idev->port_info_pa = 0;
+
+- if (err)
+- dev_err(ionic->dev, "Failed to reset port\n");
+-
+ return err;
+ }
+
+diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+index ed2b6fe5a78d3..3eb05376e7c3e 100644
+--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
++++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+@@ -2982,12 +2982,16 @@ static int qed_iov_pre_update_vport(struct qed_hwfn *hwfn,
+ u8 mask = QED_ACCEPT_UCAST_UNMATCHED | QED_ACCEPT_MCAST_UNMATCHED;
+ struct qed_filter_accept_flags *flags = &params->accept_flags;
+ struct qed_public_vf_info *vf_info;
++ u16 tlv_mask;
++
++ tlv_mask = BIT(QED_IOV_VP_UPDATE_ACCEPT_PARAM) |
++ BIT(QED_IOV_VP_UPDATE_ACCEPT_ANY_VLAN);
+
+ /* Untrusted VFs can't even be trusted to know that fact.
+ * Simply indicate everything is configured fine, and trace
+ * configuration 'behind their back'.
+ */
+- if (!(*tlvs & BIT(QED_IOV_VP_UPDATE_ACCEPT_PARAM)))
++ if (!(*tlvs & tlv_mask))
+ return 0;
+
+ vf_info = qed_iov_get_public_vf_info(hwfn, vfid, true);
+@@ -3004,6 +3008,13 @@ static int qed_iov_pre_update_vport(struct qed_hwfn *hwfn,
+ flags->tx_accept_filter &= ~mask;
+ }
+
++ if (params->update_accept_any_vlan_flg) {
++ vf_info->accept_any_vlan = params->accept_any_vlan;
++
++ if (vf_info->forced_vlan && !vf_info->is_trusted_configured)
++ params->accept_any_vlan = false;
++ }
++
+ return 0;
+ }
+
+@@ -3778,11 +3789,11 @@ bool qed_iov_mark_vf_flr(struct qed_hwfn *p_hwfn, u32 *p_disabled_vfs)
+ return found;
+ }
+
+-static void qed_iov_get_link(struct qed_hwfn *p_hwfn,
+- u16 vfid,
+- struct qed_mcp_link_params *p_params,
+- struct qed_mcp_link_state *p_link,
+- struct qed_mcp_link_capabilities *p_caps)
++static int qed_iov_get_link(struct qed_hwfn *p_hwfn,
++ u16 vfid,
++ struct qed_mcp_link_params *p_params,
++ struct qed_mcp_link_state *p_link,
++ struct qed_mcp_link_capabilities *p_caps)
+ {
+ struct qed_vf_info *p_vf = qed_iov_get_vf_info(p_hwfn,
+ vfid,
+@@ -3790,7 +3801,7 @@ static void qed_iov_get_link(struct qed_hwfn *p_hwfn,
+ struct qed_bulletin_content *p_bulletin;
+
+ if (!p_vf)
+- return;
++ return -EINVAL;
+
+ p_bulletin = p_vf->bulletin.p_virt;
+
+@@ -3800,6 +3811,7 @@ static void qed_iov_get_link(struct qed_hwfn *p_hwfn,
+ __qed_vf_get_link_state(p_hwfn, p_link, p_bulletin);
+ if (p_caps)
+ __qed_vf_get_link_caps(p_hwfn, p_caps, p_bulletin);
++ return 0;
+ }
+
+ static int
+@@ -4658,6 +4670,7 @@ static int qed_get_vf_config(struct qed_dev *cdev,
+ struct qed_public_vf_info *vf_info;
+ struct qed_mcp_link_state link;
+ u32 tx_rate;
++ int ret;
+
+ /* Sanitize request */
+ if (IS_VF(cdev))
+@@ -4671,7 +4684,9 @@ static int qed_get_vf_config(struct qed_dev *cdev,
+
+ vf_info = qed_iov_get_public_vf_info(hwfn, vf_id, true);
+
+- qed_iov_get_link(hwfn, vf_id, NULL, &link, NULL);
++ ret = qed_iov_get_link(hwfn, vf_id, NULL, &link, NULL);
++ if (ret)
++ return ret;
+
+ /* Fill information about VF */
+ ivi->vf = vf_id;
+@@ -4687,6 +4702,7 @@ static int qed_get_vf_config(struct qed_dev *cdev,
+ tx_rate = vf_info->tx_rate;
+ ivi->max_tx_rate = tx_rate ? tx_rate : link.speed;
+ ivi->min_tx_rate = qed_iov_get_vf_min_rate(hwfn, vf_id);
++ ivi->trusted = vf_info->is_trusted_request;
+
+ return 0;
+ }
+@@ -5117,6 +5133,12 @@ static void qed_iov_handle_trust_change(struct qed_hwfn *hwfn)
+
+ params.update_ctl_frame_check = 1;
+ params.mac_chk_en = !vf_info->is_trusted_configured;
++ params.update_accept_any_vlan_flg = 0;
++
++ if (vf_info->accept_any_vlan && vf_info->forced_vlan) {
++ params.update_accept_any_vlan_flg = 1;
++ params.accept_any_vlan = vf_info->accept_any_vlan;
++ }
+
+ if (vf_info->rx_accept_mode & mask) {
+ flags->update_rx_mode_config = 1;
+@@ -5132,13 +5154,20 @@ static void qed_iov_handle_trust_change(struct qed_hwfn *hwfn)
+ if (!vf_info->is_trusted_configured) {
+ flags->rx_accept_filter &= ~mask;
+ flags->tx_accept_filter &= ~mask;
++ params.accept_any_vlan = false;
+ }
+
+ if (flags->update_rx_mode_config ||
+ flags->update_tx_mode_config ||
+- params.update_ctl_frame_check)
++ params.update_ctl_frame_check ||
++ params.update_accept_any_vlan_flg) {
++ DP_VERBOSE(hwfn, QED_MSG_IOV,
++ "vport update config for %s VF[abs 0x%x rel 0x%x]\n",
++ vf_info->is_trusted_configured ? "trusted" : "untrusted",
++ vf->abs_vf_id, vf->relative_vf_id);
+ qed_sp_vport_update(hwfn, &params,
+ QED_SPQ_MODE_EBLOCK, NULL);
++ }
+ }
+ }
+
+diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.h b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+index eacd6457f195c..7ff23ef8ccc17 100644
+--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.h
++++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+@@ -62,6 +62,7 @@ struct qed_public_vf_info {
+ bool is_trusted_request;
+ u8 rx_accept_mode;
+ u8 tx_accept_mode;
++ bool accept_any_vlan;
+ };
+
+ struct qed_iov_vf_init_params {
+diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.c b/drivers/net/ethernet/qlogic/qed/qed_vf.c
+index 72a38d53d33f6..e2a5a6a373cbe 100644
+--- a/drivers/net/ethernet/qlogic/qed/qed_vf.c
++++ b/drivers/net/ethernet/qlogic/qed/qed_vf.c
+@@ -513,6 +513,9 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
+ p_iov->bulletin.size,
+ &p_iov->bulletin.phys,
+ GFP_KERNEL);
++ if (!p_iov->bulletin.p_virt)
++ goto free_pf2vf_reply;
++
+ DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+ "VF's bulletin Board [%p virt 0x%llx phys 0x%08x bytes]\n",
+ p_iov->bulletin.p_virt,
+@@ -552,6 +555,10 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
+
+ return rc;
+
++free_pf2vf_reply:
++ dma_free_coherent(&p_hwfn->cdev->pdev->dev,
++ sizeof(union pfvf_tlvs),
++ p_iov->pf2vf_reply, p_iov->pf2vf_reply_phys);
+ free_vf2pf_request:
+ dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+ sizeof(union vfpf_tlvs),
+diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c
+index 065e9004598ee..17f895250e041 100644
+--- a/drivers/net/ethernet/qlogic/qede/qede_fp.c
++++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c
+@@ -747,6 +747,9 @@ qede_build_skb(struct qede_rx_queue *rxq,
+ buf = page_address(bd->data) + bd->page_offset;
+ skb = build_skb(buf, rxq->rx_buf_seg_size);
+
++ if (unlikely(!skb))
++ return NULL;
++
+ skb_reserve(skb, pad);
+ skb_put(skb, len);
+
+@@ -1643,6 +1646,13 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+ data_split = true;
+ }
+ } else {
++ if (unlikely(skb->len > ETH_TX_MAX_NON_LSO_PKT_LEN)) {
++ DP_ERR(edev, "Unexpected non LSO skb length = 0x%x\n", skb->len);
++ qede_free_failed_tx_pkt(txq, first_bd, 0, false);
++ qede_update_tx_producer(txq);
++ return NETDEV_TX_OK;
++ }
++
+ val |= ((skb->len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) <<
+ ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT);
+ }
+diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
+index 9837bdb89cd40..ee4c3bd28a934 100644
+--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
++++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
+@@ -1176,19 +1176,17 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
+ edev->devlink = qed_ops->common->devlink_register(cdev);
+ if (IS_ERR(edev->devlink)) {
+ DP_NOTICE(edev, "Cannot register devlink\n");
++ rc = PTR_ERR(edev->devlink);
+ edev->devlink = NULL;
+- /* Go on, we can live without devlink */
++ goto err3;
+ }
+ } else {
+ struct net_device *ndev = pci_get_drvdata(pdev);
++ struct qed_devlink *qdl;
+
+ edev = netdev_priv(ndev);
+-
+- if (edev->devlink) {
+- struct qed_devlink *qdl = devlink_priv(edev->devlink);
+-
+- qdl->cdev = cdev;
+- }
++ qdl = devlink_priv(edev->devlink);
++ qdl->cdev = cdev;
+ edev->cdev = cdev;
+ memset(&edev->stats, 0, sizeof(edev->stats));
+ memcpy(&edev->dev_info, &dev_info, sizeof(dev_info));
+diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c
+index c00ad57575eab..40d14d80f6f1f 100644
+--- a/drivers/net/ethernet/qlogic/qla3xxx.c
++++ b/drivers/net/ethernet/qlogic/qla3xxx.c
+@@ -3478,20 +3478,19 @@ static int ql_adapter_up(struct ql3_adapter *qdev)
+
+ spin_lock_irqsave(&qdev->hw_lock, hw_flags);
+
+- err = ql_wait_for_drvr_lock(qdev);
+- if (err) {
+- err = ql_adapter_initialize(qdev);
+- if (err) {
+- netdev_err(ndev, "Unable to initialize adapter\n");
+- goto err_init;
+- }
+- netdev_err(ndev, "Releasing driver lock\n");
+- ql_sem_unlock(qdev, QL_DRVR_SEM_MASK);
+- } else {
++ if (!ql_wait_for_drvr_lock(qdev)) {
+ netdev_err(ndev, "Could not acquire driver lock\n");
++ err = -ENODEV;
+ goto err_lock;
+ }
+
++ err = ql_adapter_initialize(qdev);
++ if (err) {
++ netdev_err(ndev, "Unable to initialize adapter\n");
++ goto err_init;
++ }
++ ql_sem_unlock(qdev, QL_DRVR_SEM_MASK);
++
+ spin_unlock_irqrestore(&qdev->hw_lock, hw_flags);
+
+ set_bit(QL_ADAPTER_UP, &qdev->flags);
+@@ -3613,7 +3612,8 @@ static void ql_reset_work(struct work_struct *work)
+ qdev->mem_map_registers;
+ unsigned long hw_flags;
+
+- if (test_bit((QL_RESET_PER_SCSI | QL_RESET_START), &qdev->flags)) {
++ if (test_bit(QL_RESET_PER_SCSI, &qdev->flags) ||
++ test_bit(QL_RESET_START, &qdev->flags)) {
+ clear_bit(QL_LINK_MASTER, &qdev->flags);
+
+ /*
+diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
+index d51bac7ba5afa..bd06076803295 100644
+--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
++++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
+@@ -1077,8 +1077,14 @@ static int qlcnic_83xx_add_rings(struct qlcnic_adapter *adapter)
+ sds_mbx_size = sizeof(struct qlcnic_sds_mbx);
+ context_id = recv_ctx->context_id;
+ num_sds = adapter->drv_sds_rings - QLCNIC_MAX_SDS_RINGS;
+- ahw->hw_ops->alloc_mbx_args(&cmd, adapter,
+- QLCNIC_CMD_ADD_RCV_RINGS);
++ err = ahw->hw_ops->alloc_mbx_args(&cmd, adapter,
++ QLCNIC_CMD_ADD_RCV_RINGS);
++ if (err) {
++ dev_err(&adapter->pdev->dev,
++ "Failed to alloc mbx args %d\n", err);
++ return err;
++ }
++
+ cmd.req.arg[1] = 0 | (num_sds << 8) | (context_id << 16);
+
+ /* set up status rings, mbx 2-81 */
+diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.h
+index 5d79ee4370bcd..7519773eaca6e 100644
+--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.h
++++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.h
+@@ -51,7 +51,7 @@ static inline int qlcnic_dcb_get_hw_capability(struct qlcnic_dcb *dcb)
+ if (dcb && dcb->ops->get_hw_capability)
+ return dcb->ops->get_hw_capability(dcb);
+
+- return 0;
++ return -EOPNOTSUPP;
+ }
+
+ static inline void qlcnic_dcb_free(struct qlcnic_dcb *dcb)
+@@ -65,7 +65,7 @@ static inline int qlcnic_dcb_attach(struct qlcnic_dcb *dcb)
+ if (dcb && dcb->ops->attach)
+ return dcb->ops->attach(dcb);
+
+- return 0;
++ return -EOPNOTSUPP;
+ }
+
+ static inline int
+@@ -74,7 +74,7 @@ qlcnic_dcb_query_hw_capability(struct qlcnic_dcb *dcb, char *buf)
+ if (dcb && dcb->ops->query_hw_capability)
+ return dcb->ops->query_hw_capability(dcb, buf);
+
+- return 0;
++ return -EOPNOTSUPP;
+ }
+
+ static inline void qlcnic_dcb_get_info(struct qlcnic_dcb *dcb)
+@@ -89,7 +89,7 @@ qlcnic_dcb_query_cee_param(struct qlcnic_dcb *dcb, char *buf, u8 type)
+ if (dcb && dcb->ops->query_cee_param)
+ return dcb->ops->query_cee_param(dcb, buf, type);
+
+- return 0;
++ return -EOPNOTSUPP;
+ }
+
+ static inline int qlcnic_dcb_get_cee_cfg(struct qlcnic_dcb *dcb)
+@@ -97,7 +97,7 @@ static inline int qlcnic_dcb_get_cee_cfg(struct qlcnic_dcb *dcb)
+ if (dcb && dcb->ops->get_cee_cfg)
+ return dcb->ops->get_cee_cfg(dcb);
+
+- return 0;
++ return -EOPNOTSUPP;
+ }
+
+ static inline void qlcnic_dcb_aen_handler(struct qlcnic_dcb *dcb, void *msg)
+diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
+index 7160b42f51ddd..d0111cb3b40e1 100644
+--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
++++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
+@@ -201,7 +201,7 @@ int qlcnic_sriov_get_vf_vport_info(struct qlcnic_adapter *,
+ struct qlcnic_info *, u16);
+ int qlcnic_sriov_cfg_vf_guest_vlan(struct qlcnic_adapter *, u16, u8);
+ void qlcnic_sriov_free_vlans(struct qlcnic_adapter *);
+-void qlcnic_sriov_alloc_vlans(struct qlcnic_adapter *);
++int qlcnic_sriov_alloc_vlans(struct qlcnic_adapter *);
+ bool qlcnic_sriov_check_any_vlan(struct qlcnic_vf_info *);
+ void qlcnic_sriov_del_vlan_id(struct qlcnic_sriov *,
+ struct qlcnic_vf_info *, u16);
+diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
+index dd03be3fc82a9..42a44c97572ae 100644
+--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
++++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
+@@ -432,7 +432,7 @@ static int qlcnic_sriov_set_guest_vlan_mode(struct qlcnic_adapter *adapter,
+ struct qlcnic_cmd_args *cmd)
+ {
+ struct qlcnic_sriov *sriov = adapter->ahw->sriov;
+- int i, num_vlans;
++ int i, num_vlans, ret;
+ u16 *vlans;
+
+ if (sriov->allowed_vlans)
+@@ -443,7 +443,9 @@ static int qlcnic_sriov_set_guest_vlan_mode(struct qlcnic_adapter *adapter,
+ dev_info(&adapter->pdev->dev, "Number of allowed Guest VLANs = %d\n",
+ sriov->num_allowed_vlans);
+
+- qlcnic_sriov_alloc_vlans(adapter);
++ ret = qlcnic_sriov_alloc_vlans(adapter);
++ if (ret)
++ return ret;
+
+ if (!sriov->any_vlan)
+ return 0;
+@@ -2154,7 +2156,7 @@ static int qlcnic_sriov_vf_resume(struct qlcnic_adapter *adapter)
+ return err;
+ }
+
+-void qlcnic_sriov_alloc_vlans(struct qlcnic_adapter *adapter)
++int qlcnic_sriov_alloc_vlans(struct qlcnic_adapter *adapter)
+ {
+ struct qlcnic_sriov *sriov = adapter->ahw->sriov;
+ struct qlcnic_vf_info *vf;
+@@ -2164,7 +2166,11 @@ void qlcnic_sriov_alloc_vlans(struct qlcnic_adapter *adapter)
+ vf = &sriov->vf_info[i];
+ vf->sriov_vlans = kcalloc(sriov->num_allowed_vlans,
+ sizeof(*vf->sriov_vlans), GFP_KERNEL);
++ if (!vf->sriov_vlans)
++ return -ENOMEM;
+ }
++
++ return 0;
+ }
+
+ void qlcnic_sriov_free_vlans(struct qlcnic_adapter *adapter)
+diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
+index 447720b93e5ab..e90fa97c0ae6c 100644
+--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
++++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
+@@ -597,7 +597,9 @@ static int __qlcnic_pci_sriov_enable(struct qlcnic_adapter *adapter,
+ if (err)
+ goto del_flr_queue;
+
+- qlcnic_sriov_alloc_vlans(adapter);
++ err = qlcnic_sriov_alloc_vlans(adapter);
++ if (err)
++ goto del_flr_queue;
+
+ return err;
+
+diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
+index 0f85f2d97b18d..4e08b7219403c 100644
+--- a/drivers/net/ethernet/renesas/ravb_main.c
++++ b/drivers/net/ethernet/renesas/ravb_main.c
+@@ -30,8 +30,7 @@
+ #include <linux/spinlock.h>
+ #include <linux/sys_soc.h>
+ #include <linux/reset.h>
+-
+-#include <asm/div64.h>
++#include <linux/math64.h>
+
+ #include "ravb.h"
+
+@@ -2061,8 +2060,7 @@ static int ravb_set_gti(struct net_device *ndev)
+ if (!rate)
+ return -EINVAL;
+
+- inc = 1000000000ULL << 20;
+- do_div(inc, rate);
++ inc = div64_ul(1000000000ULL << 20, rate);
+
+ if (inc < GTI_TIV_MIN || inc > GTI_TIV_MAX) {
+ dev_err(dev, "gti.tiv increment 0x%llx is outside the range 0x%x - 0x%x\n",
+diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
+index 3e1ca7a8d0295..bc70c6abd6a5b 100644
+--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
++++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
+@@ -2783,7 +2783,8 @@ static void ofdpa_fib4_abort(struct rocker *rocker)
+ if (!ofdpa_port)
+ continue;
+ nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
+- ofdpa_flow_tbl_del(ofdpa_port, OFDPA_OP_FLAG_REMOVE,
++ ofdpa_flow_tbl_del(ofdpa_port,
++ OFDPA_OP_FLAG_REMOVE | OFDPA_OP_FLAG_NOWAIT,
+ flow_entry);
+ }
+ spin_unlock_irqrestore(&ofdpa->flow_tbl_lock, flags);
+diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
+index 6781aa636d588..1b415fe6f9b9f 100644
+--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
++++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
+@@ -2282,18 +2282,18 @@ static int __init sxgbe_cmdline_opt(char *str)
+ char *opt;
+
+ if (!str || !*str)
+- return -EINVAL;
++ return 1;
+ while ((opt = strsep(&str, ",")) != NULL) {
+ if (!strncmp(opt, "eee_timer:", 10)) {
+ if (kstrtoint(opt + 10, 0, &eee_timer))
+ goto err;
+ }
+ }
+- return 0;
++ return 1;
+
+ err:
+ pr_err("%s: ERROR broken module parameter conversion\n", __func__);
+- return -EINVAL;
++ return 1;
+ }
+
+ __setup("sxgbeeth=", sxgbe_cmdline_opt);
+diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
+index e7e2223aebbf5..f5a4d8f4fd114 100644
+--- a/drivers/net/ethernet/sfc/ef10.c
++++ b/drivers/net/ethernet/sfc/ef10.c
+@@ -3579,6 +3579,11 @@ static int efx_ef10_mtd_probe(struct efx_nic *efx)
+ n_parts++;
+ }
+
++ if (!n_parts) {
++ kfree(parts);
++ return 0;
++ }
++
+ rc = efx_mtd_add(efx, &parts[0].common, n_parts, sizeof(*parts));
+ fail:
+ if (rc)
+diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c
+index 518268ce20644..d35cafd422b1c 100644
+--- a/drivers/net/ethernet/sfc/ef100_nic.c
++++ b/drivers/net/ethernet/sfc/ef100_nic.c
+@@ -609,6 +609,9 @@ static size_t ef100_update_stats(struct efx_nic *efx,
+ ef100_common_stat_mask(mask);
+ ef100_ethtool_stat_mask(mask);
+
++ if (!mc_stats)
++ return 0;
++
+ efx_nic_copy_stats(efx, mc_stats);
+ efx_nic_update_stats(ef100_stat_desc, EF100_STAT_COUNT, mask,
+ stats, mc_stats, false);
+diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c
+index 3dbea028b325c..d5f2ccd3bca44 100644
+--- a/drivers/net/ethernet/sfc/efx_channels.c
++++ b/drivers/net/ethernet/sfc/efx_channels.c
+@@ -763,9 +763,90 @@ void efx_remove_channels(struct efx_nic *efx)
+ kfree(efx->xdp_tx_queues);
+ }
+
++static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
++ struct efx_tx_queue *tx_queue)
++{
++ if (xdp_queue_number >= efx->xdp_tx_queue_count)
++ return -EINVAL;
++
++ netif_dbg(efx, drv, efx->net_dev,
++ "Channel %u TXQ %u is XDP %u, HW %u\n",
++ tx_queue->channel->channel, tx_queue->label,
++ xdp_queue_number, tx_queue->queue);
++ efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
++ return 0;
++}
++
++static void efx_set_xdp_channels(struct efx_nic *efx)
++{
++ struct efx_tx_queue *tx_queue;
++ struct efx_channel *channel;
++ unsigned int next_queue = 0;
++ int xdp_queue_number = 0;
++ int rc;
++
++ /* We need to mark which channels really have RX and TX
++ * queues, and adjust the TX queue numbers if we have separate
++ * RX-only and TX-only channels.
++ */
++ efx_for_each_channel(channel, efx) {
++ if (channel->channel < efx->tx_channel_offset)
++ continue;
++
++ if (efx_channel_is_xdp_tx(channel)) {
++ efx_for_each_channel_tx_queue(tx_queue, channel) {
++ tx_queue->queue = next_queue++;
++ rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
++ tx_queue);
++ if (rc == 0)
++ xdp_queue_number++;
++ }
++ } else {
++ efx_for_each_channel_tx_queue(tx_queue, channel) {
++ tx_queue->queue = next_queue++;
++ netif_dbg(efx, drv, efx->net_dev,
++ "Channel %u TXQ %u is HW %u\n",
++ channel->channel, tx_queue->label,
++ tx_queue->queue);
++ }
++
++ /* If XDP is borrowing queues from net stack, it must
++ * use the queue with no csum offload, which is the
++ * first one of the channel
++ * (note: tx_queue_by_type is not initialized yet)
++ */
++ if (efx->xdp_txq_queues_mode ==
++ EFX_XDP_TX_QUEUES_BORROWED) {
++ tx_queue = &channel->tx_queue[0];
++ rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
++ tx_queue);
++ if (rc == 0)
++ xdp_queue_number++;
++ }
++ }
++ }
++ WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
++ xdp_queue_number != efx->xdp_tx_queue_count);
++ WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
++ xdp_queue_number > efx->xdp_tx_queue_count);
++
++ /* If we have more CPUs than assigned XDP TX queues, assign the already
++ * existing queues to the exceeding CPUs
++ */
++ next_queue = 0;
++ while (xdp_queue_number < efx->xdp_tx_queue_count) {
++ tx_queue = efx->xdp_tx_queues[next_queue++];
++ rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
++ if (rc == 0)
++ xdp_queue_number++;
++ }
++}
++
+ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
+ {
+- struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
++ struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
++ *ptp_channel = efx_ptp_channel(efx);
++ struct efx_ptp_data *ptp_data = efx->ptp_data;
+ unsigned int i, next_buffer_table = 0;
+ u32 old_rxq_entries, old_txq_entries;
+ int rc, rc2;
+@@ -818,11 +899,8 @@ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
+ old_txq_entries = efx->txq_entries;
+ efx->rxq_entries = rxq_entries;
+ efx->txq_entries = txq_entries;
+- for (i = 0; i < efx->n_channels; i++) {
+- channel = efx->channel[i];
+- efx->channel[i] = other_channel[i];
+- other_channel[i] = channel;
+- }
++ for (i = 0; i < efx->n_channels; i++)
++ swap(efx->channel[i], other_channel[i]);
+
+ /* Restart buffer table allocation */
+ efx->next_buffer_table = next_buffer_table;
+@@ -837,7 +915,9 @@ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
+ efx_init_napi_channel(efx->channel[i]);
+ }
+
++ efx_set_xdp_channels(efx);
+ out:
++ efx->ptp_data = NULL;
+ /* Destroy unused channel structures */
+ for (i = 0; i < efx->n_channels; i++) {
+ channel = other_channel[i];
+@@ -848,6 +928,7 @@ out:
+ }
+ }
+
++ efx->ptp_data = ptp_data;
+ rc2 = efx_soft_enable_interrupts(efx);
+ if (rc2) {
+ rc = rc ? rc : rc2;
+@@ -864,34 +945,15 @@ rollback:
+ /* Swap back */
+ efx->rxq_entries = old_rxq_entries;
+ efx->txq_entries = old_txq_entries;
+- for (i = 0; i < efx->n_channels; i++) {
+- channel = efx->channel[i];
+- efx->channel[i] = other_channel[i];
+- other_channel[i] = channel;
+- }
++ for (i = 0; i < efx->n_channels; i++)
++ swap(efx->channel[i], other_channel[i]);
++ efx_ptp_update_channel(efx, ptp_channel);
+ goto out;
+ }
+
+-static inline int
+-efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
+- struct efx_tx_queue *tx_queue)
+-{
+- if (xdp_queue_number >= efx->xdp_tx_queue_count)
+- return -EINVAL;
+-
+- netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n",
+- tx_queue->channel->channel, tx_queue->label,
+- xdp_queue_number, tx_queue->queue);
+- efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
+- return 0;
+-}
+-
+ int efx_set_channels(struct efx_nic *efx)
+ {
+- struct efx_tx_queue *tx_queue;
+ struct efx_channel *channel;
+- unsigned int next_queue = 0;
+- int xdp_queue_number;
+ int rc;
+
+ efx->tx_channel_offset =
+@@ -909,61 +971,14 @@ int efx_set_channels(struct efx_nic *efx)
+ return -ENOMEM;
+ }
+
+- /* We need to mark which channels really have RX and TX
+- * queues, and adjust the TX queue numbers if we have separate
+- * RX-only and TX-only channels.
+- */
+- xdp_queue_number = 0;
+ efx_for_each_channel(channel, efx) {
+ if (channel->channel < efx->n_rx_channels)
+ channel->rx_queue.core_index = channel->channel;
+ else
+ channel->rx_queue.core_index = -1;
+-
+- if (channel->channel >= efx->tx_channel_offset) {
+- if (efx_channel_is_xdp_tx(channel)) {
+- efx_for_each_channel_tx_queue(tx_queue, channel) {
+- tx_queue->queue = next_queue++;
+- rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
+- if (rc == 0)
+- xdp_queue_number++;
+- }
+- } else {
+- efx_for_each_channel_tx_queue(tx_queue, channel) {
+- tx_queue->queue = next_queue++;
+- netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n",
+- channel->channel, tx_queue->label,
+- tx_queue->queue);
+- }
+-
+- /* If XDP is borrowing queues from net stack, it must use the queue
+- * with no csum offload, which is the first one of the channel
+- * (note: channel->tx_queue_by_type is not initialized yet)
+- */
+- if (efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_BORROWED) {
+- tx_queue = &channel->tx_queue[0];
+- rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
+- if (rc == 0)
+- xdp_queue_number++;
+- }
+- }
+- }
+ }
+- WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
+- xdp_queue_number != efx->xdp_tx_queue_count);
+- WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
+- xdp_queue_number > efx->xdp_tx_queue_count);
+
+- /* If we have more CPUs than assigned XDP TX queues, assign the already
+- * existing queues to the exceeding CPUs
+- */
+- next_queue = 0;
+- while (xdp_queue_number < efx->xdp_tx_queue_count) {
+- tx_queue = efx->xdp_tx_queues[next_queue++];
+- rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
+- if (rc == 0)
+- xdp_queue_number++;
+- }
++ efx_set_xdp_channels(efx);
+
+ rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
+ if (rc)
+@@ -1107,7 +1122,7 @@ void efx_start_channels(struct efx_nic *efx)
+ struct efx_rx_queue *rx_queue;
+ struct efx_channel *channel;
+
+- efx_for_each_channel(channel, efx) {
++ efx_for_each_channel_rev(channel, efx) {
+ efx_for_each_channel_tx_queue(tx_queue, channel) {
+ efx_init_tx_queue(tx_queue);
+ atomic_inc(&efx->active_queues);
+diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c
+index bf1443539a1a4..bd552c7dffcb1 100644
+--- a/drivers/net/ethernet/sfc/ethtool_common.c
++++ b/drivers/net/ethernet/sfc/ethtool_common.c
+@@ -563,20 +563,14 @@ int efx_ethtool_get_link_ksettings(struct net_device *net_dev,
+ {
+ struct efx_nic *efx = netdev_priv(net_dev);
+ struct efx_link_state *link_state = &efx->link_state;
+- u32 supported;
+
+ mutex_lock(&efx->mac_lock);
+ efx_mcdi_phy_get_link_ksettings(efx, cmd);
+ mutex_unlock(&efx->mac_lock);
+
+ /* Both MACs support pause frames (bidirectional and respond-only) */
+- ethtool_convert_link_mode_to_legacy_u32(&supported,
+- cmd->link_modes.supported);
+-
+- supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+-
+- ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+- supported);
++ ethtool_link_ksettings_add_link_mode(cmd, supported, Pause);
++ ethtool_link_ksettings_add_link_mode(cmd, supported, Asym_Pause);
+
+ if (LOOPBACK_INTERNAL(efx)) {
+ cmd->base.speed = link_state->speed;
+diff --git a/drivers/net/ethernet/sfc/falcon/rx.c b/drivers/net/ethernet/sfc/falcon/rx.c
+index 966f13e7475dd..0c6cc21913693 100644
+--- a/drivers/net/ethernet/sfc/falcon/rx.c
++++ b/drivers/net/ethernet/sfc/falcon/rx.c
+@@ -110,6 +110,8 @@ static struct page *ef4_reuse_page(struct ef4_rx_queue *rx_queue)
+ struct ef4_rx_page_state *state;
+ unsigned index;
+
++ if (unlikely(!rx_queue->page_ring))
++ return NULL;
+ index = rx_queue->page_remove & rx_queue->page_ptr_mask;
+ page = rx_queue->page_ring[index];
+ if (page == NULL)
+@@ -293,6 +295,9 @@ static void ef4_recycle_rx_pages(struct ef4_channel *channel,
+ {
+ struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);
+
++ if (unlikely(!rx_queue->page_ring))
++ return;
++
+ do {
+ ef4_recycle_rx_page(channel, rx_buf);
+ rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);
+@@ -728,7 +733,10 @@ static void ef4_init_rx_recycle_ring(struct ef4_nic *efx,
+ efx->rx_bufs_per_page);
+ rx_queue->page_ring = kcalloc(page_ring_size,
+ sizeof(*rx_queue->page_ring), GFP_KERNEL);
+- rx_queue->page_ptr_mask = page_ring_size - 1;
++ if (!rx_queue->page_ring)
++ rx_queue->page_ptr_mask = 0;
++ else
++ rx_queue->page_ptr_mask = page_ring_size - 1;
+ }
+
+ void ef4_init_rx_queue(struct ef4_rx_queue *rx_queue)
+diff --git a/drivers/net/ethernet/sfc/mcdi.c b/drivers/net/ethernet/sfc/mcdi.c
+index be6bfd6b7ec75..50baf62b2cbc6 100644
+--- a/drivers/net/ethernet/sfc/mcdi.c
++++ b/drivers/net/ethernet/sfc/mcdi.c
+@@ -163,9 +163,9 @@ static void efx_mcdi_send_request(struct efx_nic *efx, unsigned cmd,
+ /* Serialise with efx_mcdi_ev_cpl() and efx_mcdi_ev_death() */
+ spin_lock_bh(&mcdi->iface_lock);
+ ++mcdi->seqno;
++ seqno = mcdi->seqno & SEQ_MASK;
+ spin_unlock_bh(&mcdi->iface_lock);
+
+- seqno = mcdi->seqno & SEQ_MASK;
+ xflags = 0;
+ if (mcdi->mode == MCDI_MODE_EVENTS)
+ xflags |= MCDI_HEADER_XFLAGS_EVREQ;
+diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
+index 797e51802ccbb..725b0f38813a9 100644
+--- a/drivers/net/ethernet/sfc/ptp.c
++++ b/drivers/net/ethernet/sfc/ptp.c
+@@ -45,6 +45,7 @@
+ #include "farch_regs.h"
+ #include "tx.h"
+ #include "nic.h" /* indirectly includes ptp.h */
++#include "efx_channels.h"
+
+ /* Maximum number of events expected to make up a PTP event */
+ #define MAX_EVENT_FRAGS 3
+@@ -541,6 +542,12 @@ struct efx_channel *efx_ptp_channel(struct efx_nic *efx)
+ return efx->ptp_data ? efx->ptp_data->channel : NULL;
+ }
+
++void efx_ptp_update_channel(struct efx_nic *efx, struct efx_channel *channel)
++{
++ if (efx->ptp_data)
++ efx->ptp_data->channel = channel;
++}
++
+ static u32 last_sync_timestamp_major(struct efx_nic *efx)
+ {
+ struct efx_channel *channel = efx_ptp_channel(efx);
+@@ -1443,6 +1450,11 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
+ int rc = 0;
+ unsigned int pos;
+
++ if (efx->ptp_data) {
++ efx->ptp_data->channel = channel;
++ return 0;
++ }
++
+ ptp = kzalloc(sizeof(struct efx_ptp_data), GFP_KERNEL);
+ efx->ptp_data = ptp;
+ if (!efx->ptp_data)
+@@ -2179,7 +2191,7 @@ static const struct efx_channel_type efx_ptp_channel_type = {
+ .pre_probe = efx_ptp_probe_channel,
+ .post_remove = efx_ptp_remove_channel,
+ .get_name = efx_ptp_get_channel_name,
+- /* no copy operation; there is no need to reallocate this channel */
++ .copy = efx_copy_channel,
+ .receive_skb = efx_ptp_rx,
+ .want_txqs = efx_ptp_want_txqs,
+ .keep_eventq = false,
+diff --git a/drivers/net/ethernet/sfc/ptp.h b/drivers/net/ethernet/sfc/ptp.h
+index 9855e8c9e544d..7b1ef7002b3f0 100644
+--- a/drivers/net/ethernet/sfc/ptp.h
++++ b/drivers/net/ethernet/sfc/ptp.h
+@@ -16,6 +16,7 @@ struct ethtool_ts_info;
+ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel);
+ void efx_ptp_defer_probe_with_channel(struct efx_nic *efx);
+ struct efx_channel *efx_ptp_channel(struct efx_nic *efx);
++void efx_ptp_update_channel(struct efx_nic *efx, struct efx_channel *channel);
+ void efx_ptp_remove(struct efx_nic *efx);
+ int efx_ptp_set_ts_config(struct efx_nic *efx, struct ifreq *ifr);
+ int efx_ptp_get_ts_config(struct efx_nic *efx, struct ifreq *ifr);
+diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c
+index 68fc7d317693b..b925de9b43028 100644
+--- a/drivers/net/ethernet/sfc/rx_common.c
++++ b/drivers/net/ethernet/sfc/rx_common.c
+@@ -45,6 +45,8 @@ static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
+ unsigned int index;
+ struct page *page;
+
++ if (unlikely(!rx_queue->page_ring))
++ return NULL;
+ index = rx_queue->page_remove & rx_queue->page_ptr_mask;
+ page = rx_queue->page_ring[index];
+ if (page == NULL)
+@@ -114,6 +116,9 @@ void efx_recycle_rx_pages(struct efx_channel *channel,
+ {
+ struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
+
++ if (unlikely(!rx_queue->page_ring))
++ return;
++
+ do {
+ efx_recycle_rx_page(channel, rx_buf);
+ rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
+@@ -150,7 +155,10 @@ static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)
+ efx->rx_bufs_per_page);
+ rx_queue->page_ring = kcalloc(page_ring_size,
+ sizeof(*rx_queue->page_ring), GFP_KERNEL);
+- rx_queue->page_ptr_mask = page_ring_size - 1;
++ if (!rx_queue->page_ring)
++ rx_queue->page_ptr_mask = 0;
++ else
++ rx_queue->page_ptr_mask = page_ring_size - 1;
+ }
+
+ static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)
+@@ -158,6 +166,9 @@ static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)
+ struct efx_nic *efx = rx_queue->efx;
+ int i;
+
++ if (unlikely(!rx_queue->page_ring))
++ return;
++
+ /* Unmap and release the pages in the recycle ring. Remove the ring. */
+ for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
+ struct page *page = rx_queue->page_ring[i];
+diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
+index d16e031e95f44..6983799e1c05d 100644
+--- a/drivers/net/ethernet/sfc/tx.c
++++ b/drivers/net/ethernet/sfc/tx.c
+@@ -443,6 +443,9 @@ int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs,
+ if (unlikely(!tx_queue))
+ return -EINVAL;
+
++ if (!tx_queue->initialised)
++ return -EINVAL;
++
+ if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED)
+ HARD_TX_LOCK(efx->net_dev, tx_queue->core_txq, cpu);
+
+diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c
+index d530cde2b8648..9bc8281b7f5bd 100644
+--- a/drivers/net/ethernet/sfc/tx_common.c
++++ b/drivers/net/ethernet/sfc/tx_common.c
+@@ -101,6 +101,8 @@ void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
+ netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
+ "shutting down TX queue %d\n", tx_queue->queue);
+
++ tx_queue->initialised = false;
++
+ if (!tx_queue->buffer)
+ return;
+
+diff --git a/drivers/net/ethernet/smsc/smc911x.c b/drivers/net/ethernet/smsc/smc911x.c
+index b008b4e8a2a5a..0641a1d392b86 100644
+--- a/drivers/net/ethernet/smsc/smc911x.c
++++ b/drivers/net/ethernet/smsc/smc911x.c
+@@ -2070,6 +2070,11 @@ static int smc911x_drv_probe(struct platform_device *pdev)
+
+ ndev->dma = (unsigned char)-1;
+ ndev->irq = platform_get_irq(pdev, 0);
++ if (ndev->irq < 0) {
++ ret = ndev->irq;
++ goto release_both;
++ }
++
+ lp = netdev_priv(ndev);
+ lp->netdev = ndev;
+ #ifdef SMC_DYNAMIC_BUS_CONFIG
+diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
+index 199a973392806..592e191adbf7d 100644
+--- a/drivers/net/ethernet/smsc/smsc911x.c
++++ b/drivers/net/ethernet/smsc/smsc911x.c
+@@ -2429,7 +2429,7 @@ static int smsc911x_drv_probe(struct platform_device *pdev)
+ if (irq == -EPROBE_DEFER) {
+ retval = -EPROBE_DEFER;
+ goto out_0;
+- } else if (irq <= 0) {
++ } else if (irq < 0) {
+ pr_warn("Could not allocate irq resource\n");
+ retval = -ENODEV;
+ goto out_0;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c
+index cd478d2cd871a..00f6d347eaf75 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c
++++ b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c
+@@ -57,10 +57,6 @@
+ #define TSE_PCS_USE_SGMII_ENA BIT(0)
+ #define TSE_PCS_IF_USE_SGMII 0x03
+
+-#define SGMII_ADAPTER_CTRL_REG 0x00
+-#define SGMII_ADAPTER_DISABLE 0x0001
+-#define SGMII_ADAPTER_ENABLE 0x0000
+-
+ #define AUTONEGO_LINK_TIMER 20
+
+ static int tse_pcs_reset(void __iomem *base, struct tse_pcs *pcs)
+@@ -202,12 +198,8 @@ void tse_pcs_fix_mac_speed(struct tse_pcs *pcs, struct phy_device *phy_dev,
+ unsigned int speed)
+ {
+ void __iomem *tse_pcs_base = pcs->tse_pcs_base;
+- void __iomem *sgmii_adapter_base = pcs->sgmii_adapter_base;
+ u32 val;
+
+- writew(SGMII_ADAPTER_ENABLE,
+- sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG);
+-
+ pcs->autoneg = phy_dev->autoneg;
+
+ if (phy_dev->autoneg == AUTONEG_ENABLE) {
+diff --git a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h
+index 442812c0a4bdc..694ac25ef426b 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h
++++ b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h
+@@ -10,6 +10,10 @@
+ #include <linux/phy.h>
+ #include <linux/timer.h>
+
++#define SGMII_ADAPTER_CTRL_REG 0x00
++#define SGMII_ADAPTER_ENABLE 0x0000
++#define SGMII_ADAPTER_DISABLE 0x0001
++
+ struct tse_pcs {
+ struct device *dev;
+ void __iomem *tse_pcs_base;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+index 8e8778cfbbadd..6f87e296a410f 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+@@ -454,6 +454,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev,
+ plat->has_gmac4 = 1;
+ plat->force_sf_dma_mode = 0;
+ plat->tso_en = 1;
++ plat->sph_disable = 1;
+
+ /* Multiplying factor to the clk_eee_i clock time
+ * period to make it closer to 100 ns. This value
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
+index 5c74b6279d690..6b1d9e8879f46 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
+@@ -113,8 +113,10 @@ static void rgmii_updatel(struct qcom_ethqos *ethqos,
+ rgmii_writel(ethqos, temp, offset);
+ }
+
+-static void rgmii_dump(struct qcom_ethqos *ethqos)
++static void rgmii_dump(void *priv)
+ {
++ struct qcom_ethqos *ethqos = priv;
++
+ dev_dbg(&ethqos->pdev->dev, "Rgmii register dump\n");
+ dev_dbg(&ethqos->pdev->dev, "RGMII_IO_MACRO_CONFIG: %x\n",
+ rgmii_readl(ethqos, RGMII_IO_MACRO_CONFIG));
+@@ -499,6 +501,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
+
+ plat_dat->bsp_priv = ethqos;
+ plat_dat->fix_mac_speed = ethqos_fix_mac_speed;
++ plat_dat->dump_debug_regs = rgmii_dump;
+ plat_dat->has_gmac4 = 1;
+ plat_dat->pmt = 1;
+ plat_dat->tso_en = of_property_read_bool(np, "snps,tso");
+@@ -507,8 +510,6 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
+ if (ret)
+ goto err_clk;
+
+- rgmii_dump(ethqos);
+-
+ return ret;
+
+ err_clk:
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+index 6924a6aacbd53..c469abc91fa1b 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+@@ -33,6 +33,7 @@ struct rk_gmac_ops {
+ void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed);
+ void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed);
+ void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv);
++ bool regs_valid;
+ u32 regs[];
+ };
+
+@@ -1092,6 +1093,7 @@ static const struct rk_gmac_ops rk3568_ops = {
+ .set_to_rmii = rk3568_set_to_rmii,
+ .set_rgmii_speed = rk3568_set_gmac_speed,
+ .set_rmii_speed = rk3568_set_gmac_speed,
++ .regs_valid = true,
+ .regs = {
+ 0xfe2a0000, /* gmac0 */
+ 0xfe010000, /* gmac1 */
+@@ -1383,7 +1385,7 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
+ * to be distinguished.
+ */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+- if (res) {
++ if (res && ops->regs_valid) {
+ int i = 0;
+
+ while (ops->regs[i]) {
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
+index 85208128f135c..6b447d8f0bd8a 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
+@@ -18,9 +18,6 @@
+
+ #include "altr_tse_pcs.h"
+
+-#define SGMII_ADAPTER_CTRL_REG 0x00
+-#define SGMII_ADAPTER_DISABLE 0x0001
+-
+ #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_GMII_MII 0x0
+ #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RGMII 0x1
+ #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RMII 0x2
+@@ -62,14 +59,13 @@ static void socfpga_dwmac_fix_mac_speed(void *priv, unsigned int speed)
+ {
+ struct socfpga_dwmac *dwmac = (struct socfpga_dwmac *)priv;
+ void __iomem *splitter_base = dwmac->splitter_base;
+- void __iomem *tse_pcs_base = dwmac->pcs.tse_pcs_base;
+ void __iomem *sgmii_adapter_base = dwmac->pcs.sgmii_adapter_base;
+ struct device *dev = dwmac->dev;
+ struct net_device *ndev = dev_get_drvdata(dev);
+ struct phy_device *phy_dev = ndev->phydev;
+ u32 val;
+
+- if ((tse_pcs_base) && (sgmii_adapter_base))
++ if (sgmii_adapter_base)
+ writew(SGMII_ADAPTER_DISABLE,
+ sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG);
+
+@@ -93,8 +89,11 @@ static void socfpga_dwmac_fix_mac_speed(void *priv, unsigned int speed)
+ writel(val, splitter_base + EMAC_SPLITTER_CTRL_REG);
+ }
+
+- if (tse_pcs_base && sgmii_adapter_base)
++ if (phy_dev && sgmii_adapter_base) {
++ writew(SGMII_ADAPTER_ENABLE,
++ sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG);
+ tse_pcs_fix_mac_speed(&dwmac->pcs, phy_dev, speed);
++ }
+ }
+
+ static int socfpga_dwmac_parse_data(struct socfpga_dwmac *dwmac, struct device *dev)
+@@ -485,8 +484,28 @@ static int socfpga_dwmac_resume(struct device *dev)
+ }
+ #endif /* CONFIG_PM_SLEEP */
+
+-static SIMPLE_DEV_PM_OPS(socfpga_dwmac_pm_ops, stmmac_suspend,
+- socfpga_dwmac_resume);
++static int __maybe_unused socfpga_dwmac_runtime_suspend(struct device *dev)
++{
++ struct net_device *ndev = dev_get_drvdata(dev);
++ struct stmmac_priv *priv = netdev_priv(ndev);
++
++ stmmac_bus_clks_config(priv, false);
++
++ return 0;
++}
++
++static int __maybe_unused socfpga_dwmac_runtime_resume(struct device *dev)
++{
++ struct net_device *ndev = dev_get_drvdata(dev);
++ struct stmmac_priv *priv = netdev_priv(ndev);
++
++ return stmmac_bus_clks_config(priv, true);
++}
++
++static const struct dev_pm_ops socfpga_dwmac_pm_ops = {
++ SET_SYSTEM_SLEEP_PM_OPS(stmmac_suspend, socfpga_dwmac_resume)
++ SET_RUNTIME_PM_OPS(socfpga_dwmac_runtime_suspend, socfpga_dwmac_runtime_resume, NULL)
++};
+
+ static const struct socfpga_dwmac_ops socfpga_gen5_ops = {
+ .set_phy_mode = socfpga_gen5_set_phy_mode,
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+index 4422baeed3d89..06e2af9387d7c 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+@@ -756,7 +756,7 @@ static int sun8i_dwmac_reset(struct stmmac_priv *priv)
+
+ if (err) {
+ dev_err(priv->device, "EMAC reset timeout\n");
+- return -EFAULT;
++ return err;
+ }
+ return 0;
+ }
+@@ -916,6 +916,7 @@ static int sun8i_dwmac_register_mdio_mux(struct stmmac_priv *priv)
+
+ ret = mdio_mux_init(priv->device, mdio_mux, mdio_mux_syscon_switch_fn,
+ &gmac->mux_handle, priv, priv->mii);
++ of_node_put(mdio_mux);
+ return ret;
+ }
+
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c
+index d046e33b8a297..c27441c08dd6f 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c
+@@ -22,21 +22,21 @@
+ #define ETHER_CLK_SEL_RMII_CLK_EN BIT(2)
+ #define ETHER_CLK_SEL_RMII_CLK_RST BIT(3)
+ #define ETHER_CLK_SEL_DIV_SEL_2 BIT(4)
+-#define ETHER_CLK_SEL_DIV_SEL_20 BIT(0)
++#define ETHER_CLK_SEL_DIV_SEL_20 0
+ #define ETHER_CLK_SEL_FREQ_SEL_125M (BIT(9) | BIT(8))
+ #define ETHER_CLK_SEL_FREQ_SEL_50M BIT(9)
+ #define ETHER_CLK_SEL_FREQ_SEL_25M BIT(8)
+-#define ETHER_CLK_SEL_FREQ_SEL_2P5M BIT(0)
+-#define ETHER_CLK_SEL_TX_CLK_EXT_SEL_IN BIT(0)
++#define ETHER_CLK_SEL_FREQ_SEL_2P5M 0
++#define ETHER_CLK_SEL_TX_CLK_EXT_SEL_IN 0
+ #define ETHER_CLK_SEL_TX_CLK_EXT_SEL_TXC BIT(10)
+ #define ETHER_CLK_SEL_TX_CLK_EXT_SEL_DIV BIT(11)
+-#define ETHER_CLK_SEL_RX_CLK_EXT_SEL_IN BIT(0)
++#define ETHER_CLK_SEL_RX_CLK_EXT_SEL_IN 0
+ #define ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC BIT(12)
+ #define ETHER_CLK_SEL_RX_CLK_EXT_SEL_DIV BIT(13)
+-#define ETHER_CLK_SEL_TX_CLK_O_TX_I BIT(0)
++#define ETHER_CLK_SEL_TX_CLK_O_TX_I 0
+ #define ETHER_CLK_SEL_TX_CLK_O_RMII_I BIT(14)
+ #define ETHER_CLK_SEL_TX_O_E_N_IN BIT(15)
+-#define ETHER_CLK_SEL_RMII_CLK_SEL_IN BIT(0)
++#define ETHER_CLK_SEL_RMII_CLK_SEL_IN 0
+ #define ETHER_CLK_SEL_RMII_CLK_SEL_RX_C BIT(16)
+
+ #define ETHER_CLK_SEL_RX_TX_CLK_EN (ETHER_CLK_SEL_RX_CLK_EN | ETHER_CLK_SEL_TX_CLK_EN)
+@@ -49,13 +49,15 @@ struct visconti_eth {
+ void __iomem *reg;
+ u32 phy_intf_sel;
+ struct clk *phy_ref_clk;
++ struct device *dev;
+ spinlock_t lock; /* lock to protect register update */
+ };
+
+ static void visconti_eth_fix_mac_speed(void *priv, unsigned int speed)
+ {
+ struct visconti_eth *dwmac = priv;
+- unsigned int val, clk_sel_val;
++ struct net_device *netdev = dev_get_drvdata(dwmac->dev);
++ unsigned int val, clk_sel_val = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dwmac->lock, flags);
+@@ -85,7 +87,9 @@ static void visconti_eth_fix_mac_speed(void *priv, unsigned int speed)
+ break;
+ default:
+ /* No bit control */
+- break;
++ netdev_err(netdev, "Unsupported speed request (%d)", speed);
++ spin_unlock_irqrestore(&dwmac->lock, flags);
++ return;
+ }
+
+ writel(val, dwmac->reg + MAC_CTRL_REG);
+@@ -96,31 +100,41 @@ static void visconti_eth_fix_mac_speed(void *priv, unsigned int speed)
+ val |= ETHER_CLK_SEL_TX_O_E_N_IN;
+ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+
++ /* Set Clock-Mux, Start clock, Set TX_O direction */
+ switch (dwmac->phy_intf_sel) {
+ case ETHER_CONFIG_INTF_RGMII:
+ val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
++
++ val |= ETHER_CLK_SEL_RX_TX_CLK_EN;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
++
++ val &= ~ETHER_CLK_SEL_TX_O_E_N_IN;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+ break;
+ case ETHER_CONFIG_INTF_RMII:
+ val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_DIV |
+- ETHER_CLK_SEL_TX_CLK_EXT_SEL_TXC | ETHER_CLK_SEL_TX_O_E_N_IN |
++ ETHER_CLK_SEL_TX_CLK_EXT_SEL_DIV | ETHER_CLK_SEL_TX_O_E_N_IN |
+ ETHER_CLK_SEL_RMII_CLK_SEL_RX_C;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
++
++ val |= ETHER_CLK_SEL_RMII_CLK_RST;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
++
++ val |= ETHER_CLK_SEL_RMII_CLK_EN | ETHER_CLK_SEL_RX_TX_CLK_EN;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+ break;
+ case ETHER_CONFIG_INTF_MII:
+ default:
+ val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC |
+- ETHER_CLK_SEL_TX_CLK_EXT_SEL_DIV | ETHER_CLK_SEL_TX_O_E_N_IN |
+- ETHER_CLK_SEL_RMII_CLK_EN;
++ ETHER_CLK_SEL_TX_CLK_EXT_SEL_TXC | ETHER_CLK_SEL_TX_O_E_N_IN;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
++
++ val |= ETHER_CLK_SEL_RX_TX_CLK_EN;
++ writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+ break;
+ }
+
+- /* Start clock */
+- writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+- val |= ETHER_CLK_SEL_RX_TX_CLK_EN;
+- writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+-
+- val &= ~ETHER_CLK_SEL_TX_O_E_N_IN;
+- writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL);
+-
+ spin_unlock_irqrestore(&dwmac->lock, flags);
+ }
+
+@@ -220,6 +234,7 @@ static int visconti_eth_dwmac_probe(struct platform_device *pdev)
+
+ spin_lock_init(&dwmac->lock);
+ dwmac->reg = stmmac_res.addr;
++ dwmac->dev = &pdev->dev;
+ plat_dat->bsp_priv = dwmac;
+ plat_dat->fix_mac_speed = visconti_eth_fix_mac_speed;
+
+diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
+index 1914ad698cab2..acd70b9a3173c 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
++++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
+@@ -150,6 +150,7 @@
+
+ #define NUM_DWMAC100_DMA_REGS 9
+ #define NUM_DWMAC1000_DMA_REGS 23
++#define NUM_DWMAC4_DMA_REGS 27
+
+ void dwmac_enable_dma_transmission(void __iomem *ioaddr);
+ void dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+index 43eead726886a..05b5371ca036b 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+@@ -172,6 +172,19 @@ struct stmmac_flow_entry {
+ int is_l4;
+ };
+
++/* Rx Frame Steering */
++enum stmmac_rfs_type {
++ STMMAC_RFS_T_VLAN,
++ STMMAC_RFS_T_MAX,
++};
++
++struct stmmac_rfs_entry {
++ unsigned long cookie;
++ int in_use;
++ int type;
++ int tc;
++};
++
+ struct stmmac_priv {
+ /* Frequently used values are kept adjacent for cache effect */
+ u32 tx_coal_frames[MTL_MAX_TX_QUEUES];
+@@ -289,6 +302,10 @@ struct stmmac_priv {
+ struct stmmac_tc_entry *tc_entries;
+ unsigned int flow_entries_max;
+ struct stmmac_flow_entry *flow_entries;
++ unsigned int rfs_entries_max[STMMAC_RFS_T_MAX];
++ unsigned int rfs_entries_cnt[STMMAC_RFS_T_MAX];
++ unsigned int rfs_entries_total;
++ struct stmmac_rfs_entry *rfs_entries;
+
+ /* Pulse Per Second output */
+ struct stmmac_pps_cfg pps[STMMAC_PPS_MAX];
+@@ -314,10 +331,11 @@ int stmmac_mdio_reset(struct mii_bus *mii);
+ int stmmac_xpcs_setup(struct mii_bus *mii);
+ void stmmac_set_ethtool_ops(struct net_device *netdev);
+
++int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags);
+ void stmmac_ptp_register(struct stmmac_priv *priv);
+ void stmmac_ptp_unregister(struct stmmac_priv *priv);
+-int stmmac_open(struct net_device *dev);
+-int stmmac_release(struct net_device *dev);
++int stmmac_xdp_open(struct net_device *dev);
++void stmmac_xdp_release(struct net_device *dev);
+ int stmmac_resume(struct device *dev);
+ int stmmac_suspend(struct device *dev);
+ int stmmac_dvr_remove(struct device *dev);
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+index d89455803beda..8f563b446d5ca 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+@@ -21,10 +21,18 @@
+ #include "dwxgmac2.h"
+
+ #define REG_SPACE_SIZE 0x1060
++#define GMAC4_REG_SPACE_SIZE 0x116C
+ #define MAC100_ETHTOOL_NAME "st_mac100"
+ #define GMAC_ETHTOOL_NAME "st_gmac"
+ #define XGMAC_ETHTOOL_NAME "st_xgmac"
+
++/* Same as DMA_CHAN_BASE_ADDR defined in dwmac4_dma.h
++ *
++ * It is here because dwmac_dma.h and dwmac4_dam.h can not be included at the
++ * same time due to the conflicting macro names.
++ */
++#define GMAC4_DMA_CHAN_BASE_ADDR 0x00001100
++
+ #define ETHTOOL_DMA_OFFSET 55
+
+ struct stmmac_stats {
+@@ -435,6 +443,8 @@ static int stmmac_ethtool_get_regs_len(struct net_device *dev)
+
+ if (priv->plat->has_xgmac)
+ return XGMAC_REGSIZE * 4;
++ else if (priv->plat->has_gmac4)
++ return GMAC4_REG_SPACE_SIZE;
+ return REG_SPACE_SIZE;
+ }
+
+@@ -447,8 +457,13 @@ static void stmmac_ethtool_gregs(struct net_device *dev,
+ stmmac_dump_mac_regs(priv, priv->hw, reg_space);
+ stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space);
+
+- if (!priv->plat->has_xgmac) {
+- /* Copy DMA registers to where ethtool expects them */
++ /* Copy DMA registers to where ethtool expects them */
++ if (priv->plat->has_gmac4) {
++ /* GMAC4 dumps its DMA registers at its DMA_CHAN_BASE_ADDR */
++ memcpy(&reg_space[ETHTOOL_DMA_OFFSET],
++ &reg_space[GMAC4_DMA_CHAN_BASE_ADDR / 4],
++ NUM_DWMAC4_DMA_REGS * 4);
++ } else if (!priv->plat->has_xgmac) {
+ memcpy(&reg_space[ETHTOOL_DMA_OFFSET],
+ &reg_space[DMA_BUS_MODE / 4],
+ NUM_DWMAC1000_DMA_REGS * 4);
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
+index 074e2cdfb0fa6..d68ef72dcdde0 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
+@@ -71,9 +71,9 @@ static int init_systime(void __iomem *ioaddr, u32 sec, u32 nsec)
+ writel(value, ioaddr + PTP_TCR);
+
+ /* wait for present system time initialize to complete */
+- return readl_poll_timeout(ioaddr + PTP_TCR, value,
++ return readl_poll_timeout_atomic(ioaddr + PTP_TCR, value,
+ !(value & PTP_TCR_TSINIT),
+- 10000, 100000);
++ 10, 100000);
+ }
+
+ static int config_addend(void __iomem *ioaddr, u32 addend)
+@@ -145,15 +145,20 @@ static int adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec,
+
+ static void get_systime(void __iomem *ioaddr, u64 *systime)
+ {
+- u64 ns;
+-
+- /* Get the TSSS value */
+- ns = readl(ioaddr + PTP_STNSR);
+- /* Get the TSS and convert sec time value to nanosecond */
+- ns += readl(ioaddr + PTP_STSR) * 1000000000ULL;
++ u64 ns, sec0, sec1;
++
++ /* Get the TSS value */
++ sec1 = readl_relaxed(ioaddr + PTP_STSR);
++ do {
++ sec0 = sec1;
++ /* Get the TSSS value */
++ ns = readl_relaxed(ioaddr + PTP_STNSR);
++ /* Get the TSS value */
++ sec1 = readl_relaxed(ioaddr + PTP_STSR);
++ } while (sec0 != sec1);
+
+ if (systime)
+- *systime = ns;
++ *systime = ns + (sec1 * 1000000000ULL);
+ }
+
+ static void get_ptptime(void __iomem *ptpaddr, u64 *ptp_time)
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+index 3d67d1fa36906..9c1e19ea6fcd0 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+@@ -50,6 +50,13 @@
+ #include "dwxgmac2.h"
+ #include "hwif.h"
+
++/* As long as the interface is active, we keep the timestamping counter enabled
++ * with fine resolution and binary rollover. This avoid non-monotonic behavior
++ * (clock jumps) when changing timestamping settings at runtime.
++ */
++#define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \
++ PTP_TCR_TSCTRLSSR)
++
+ #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
+ #define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
+
+@@ -393,7 +400,7 @@ static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)
+ * Description: this function is to verify and enter in LPI mode in case of
+ * EEE.
+ */
+-static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
++static int stmmac_enable_eee_mode(struct stmmac_priv *priv)
+ {
+ u32 tx_cnt = priv->plat->tx_queues_to_use;
+ u32 queue;
+@@ -403,13 +410,14 @@ static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
+ struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
+
+ if (tx_q->dirty_tx != tx_q->cur_tx)
+- return; /* still unfinished work */
++ return -EBUSY; /* still unfinished work */
+ }
+
+ /* Check and enter in LPI mode */
+ if (!priv->tx_path_in_lpi_mode)
+ stmmac_set_eee_mode(priv, priv->hw,
+ priv->plat->en_tx_lpi_clockgating);
++ return 0;
+ }
+
+ /**
+@@ -441,8 +449,8 @@ static void stmmac_eee_ctrl_timer(struct timer_list *t)
+ {
+ struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
+
+- stmmac_enable_eee_mode(priv);
+- mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
++ if (stmmac_enable_eee_mode(priv))
++ mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
+ }
+
+ /**
+@@ -511,6 +519,14 @@ bool stmmac_eee_init(struct stmmac_priv *priv)
+ return true;
+ }
+
++static inline u32 stmmac_cdc_adjust(struct stmmac_priv *priv)
++{
++ /* Correct the clk domain crossing(CDC) error */
++ if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate)
++ return (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate;
++ return 0;
++}
++
+ /* stmmac_get_tx_hwtstamp - get HW TX timestamps
+ * @priv: driver private structure
+ * @p : descriptor pointer
+@@ -524,7 +540,6 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
+ {
+ struct skb_shared_hwtstamps shhwtstamp;
+ bool found = false;
+- s64 adjust = 0;
+ u64 ns = 0;
+
+ if (!priv->hwts_tx_en)
+@@ -543,12 +558,7 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
+ }
+
+ if (found) {
+- /* Correct the clk domain crossing(CDC) error */
+- if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) {
+- adjust += -(2 * (NSEC_PER_SEC /
+- priv->plat->clk_ptp_rate));
+- ns += adjust;
+- }
++ ns -= stmmac_cdc_adjust(priv);
+
+ memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
+ shhwtstamp.hwtstamp = ns_to_ktime(ns);
+@@ -573,7 +583,6 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
+ {
+ struct skb_shared_hwtstamps *shhwtstamp = NULL;
+ struct dma_desc *desc = p;
+- u64 adjust = 0;
+ u64 ns = 0;
+
+ if (!priv->hwts_rx_en)
+@@ -586,11 +595,7 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
+ if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
+ stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
+
+- /* Correct the clk domain crossing(CDC) error */
+- if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) {
+- adjust += 2 * (NSEC_PER_SEC / priv->plat->clk_ptp_rate);
+- ns -= adjust;
+- }
++ ns -= stmmac_cdc_adjust(priv);
+
+ netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
+ shhwtstamp = skb_hwtstamps(skb);
+@@ -616,8 +621,6 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
+ {
+ struct stmmac_priv *priv = netdev_priv(dev);
+ struct hwtstamp_config config;
+- struct timespec64 now;
+- u64 temp = 0;
+ u32 ptp_v2 = 0;
+ u32 tstamp_all = 0;
+ u32 ptp_over_ipv4_udp = 0;
+@@ -626,11 +629,6 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
+ u32 snap_type_sel = 0;
+ u32 ts_master_en = 0;
+ u32 ts_event_en = 0;
+- u32 sec_inc = 0;
+- u32 value = 0;
+- bool xmac;
+-
+- xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
+
+ if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
+ netdev_alert(priv->dev, "No support for HW time stamping\n");
+@@ -792,42 +790,17 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
+ priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
+ priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
+
+- if (!priv->hwts_tx_en && !priv->hwts_rx_en)
+- stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
+- else {
+- value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
+- tstamp_all | ptp_v2 | ptp_over_ethernet |
+- ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
+- ts_master_en | snap_type_sel);
+- stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
+-
+- /* program Sub Second Increment reg */
+- stmmac_config_sub_second_increment(priv,
+- priv->ptpaddr, priv->plat->clk_ptp_rate,
+- xmac, &sec_inc);
+- temp = div_u64(1000000000ULL, sec_inc);
+-
+- /* Store sub second increment and flags for later use */
+- priv->sub_second_inc = sec_inc;
+- priv->systime_flags = value;
+-
+- /* calculate default added value:
+- * formula is :
+- * addend = (2^32)/freq_div_ratio;
+- * where, freq_div_ratio = 1e9ns/sec_inc
+- */
+- temp = (u64)(temp << 32);
+- priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
+- stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
+-
+- /* initialize system time */
+- ktime_get_real_ts64(&now);
++ priv->systime_flags = STMMAC_HWTS_ACTIVE;
+
+- /* lower 32 bits of tv_sec are safe until y2106 */
+- stmmac_init_systime(priv, priv->ptpaddr,
+- (u32)now.tv_sec, now.tv_nsec);
++ if (priv->hwts_tx_en || priv->hwts_rx_en) {
++ priv->systime_flags |= tstamp_all | ptp_v2 |
++ ptp_over_ethernet | ptp_over_ipv6_udp |
++ ptp_over_ipv4_udp | ts_event_en |
++ ts_master_en | snap_type_sel;
+ }
+
++ stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
++
+ memcpy(&priv->tstamp_config, &config, sizeof(config));
+
+ return copy_to_user(ifr->ifr_data, &config,
+@@ -855,6 +828,66 @@ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
+ sizeof(*config)) ? -EFAULT : 0;
+ }
+
++/**
++ * stmmac_init_tstamp_counter - init hardware timestamping counter
++ * @priv: driver private structure
++ * @systime_flags: timestamping flags
++ * Description:
++ * Initialize hardware counter for packet timestamping.
++ * This is valid as long as the interface is open and not suspended.
++ * Will be rerun after resuming from suspend, case in which the timestamping
++ * flags updated by stmmac_hwtstamp_set() also need to be restored.
++ */
++int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)
++{
++ bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
++ struct timespec64 now;
++ u32 sec_inc = 0;
++ u64 temp = 0;
++ int ret;
++
++ if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
++ return -EOPNOTSUPP;
++
++ ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
++ if (ret < 0) {
++ netdev_warn(priv->dev,
++ "failed to enable PTP reference clock: %pe\n",
++ ERR_PTR(ret));
++ return ret;
++ }
++
++ stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
++ priv->systime_flags = systime_flags;
++
++ /* program Sub Second Increment reg */
++ stmmac_config_sub_second_increment(priv, priv->ptpaddr,
++ priv->plat->clk_ptp_rate,
++ xmac, &sec_inc);
++ temp = div_u64(1000000000ULL, sec_inc);
++
++ /* Store sub second increment for later use */
++ priv->sub_second_inc = sec_inc;
++
++ /* calculate default added value:
++ * formula is :
++ * addend = (2^32)/freq_div_ratio;
++ * where, freq_div_ratio = 1e9ns/sec_inc
++ */
++ temp = (u64)(temp << 32);
++ priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
++ stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
++
++ /* initialize system time */
++ ktime_get_real_ts64(&now);
++
++ /* lower 32 bits of tv_sec are safe until y2106 */
++ stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);
++
++ return 0;
++}
++EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter);
++
+ /**
+ * stmmac_init_ptp - init PTP
+ * @priv: driver private structure
+@@ -865,9 +898,14 @@ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
+ static int stmmac_init_ptp(struct stmmac_priv *priv)
+ {
+ bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
++ int ret;
+
+- if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
+- return -EOPNOTSUPP;
++ if (priv->plat->ptp_clk_freq_config)
++ priv->plat->ptp_clk_freq_config(priv);
++
++ ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);
++ if (ret)
++ return ret;
+
+ priv->adv_ts = 0;
+ /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
+@@ -887,8 +925,6 @@ static int stmmac_init_ptp(struct stmmac_priv *priv)
+ priv->hwts_tx_en = 0;
+ priv->hwts_rx_en = 0;
+
+- stmmac_ptp_register(priv);
+-
+ return 0;
+ }
+
+@@ -2232,6 +2268,23 @@ static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
+ stmmac_stop_tx(priv, priv->ioaddr, chan);
+ }
+
++static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
++{
++ u32 rx_channels_count = priv->plat->rx_queues_to_use;
++ u32 tx_channels_count = priv->plat->tx_queues_to_use;
++ u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
++ u32 chan;
++
++ for (chan = 0; chan < dma_csr_ch; chan++) {
++ struct stmmac_channel *ch = &priv->channel[chan];
++ unsigned long flags;
++
++ spin_lock_irqsave(&ch->lock, flags);
++ stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
++ spin_unlock_irqrestore(&ch->lock, flags);
++ }
++}
++
+ /**
+ * stmmac_start_all_dma - start all RX and TX DMA channels
+ * @priv: driver private structure
+@@ -2602,8 +2655,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
+
+ if (priv->eee_enabled && !priv->tx_path_in_lpi_mode &&
+ priv->eee_sw_timer_en) {
+- stmmac_enable_eee_mode(priv);
+- mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
++ if (stmmac_enable_eee_mode(priv))
++ mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
+ }
+
+ /* We still have pending packets, let's call for a new scheduling */
+@@ -2867,8 +2920,10 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv)
+ stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
+
+ /* DMA CSR Channel configuration */
+- for (chan = 0; chan < dma_csr_ch; chan++)
++ for (chan = 0; chan < dma_csr_ch; chan++) {
+ stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
++ stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
++ }
+
+ /* DMA RX Channel Configuration */
+ for (chan = 0; chan < rx_channels_count; chan++) {
+@@ -3203,7 +3258,7 @@ static int stmmac_fpe_start_wq(struct stmmac_priv *priv)
+ /**
+ * stmmac_hw_setup - setup mac in a usable state.
+ * @dev : pointer to the device structure.
+- * @init_ptp: initialize PTP if set
++ * @ptp_register: register PTP if set
+ * Description:
+ * this is the main function to setup the HW in a usable state because the
+ * dma engine is reset, the core registers are configured (e.g. AXI,
+@@ -3213,7 +3268,7 @@ static int stmmac_fpe_start_wq(struct stmmac_priv *priv)
+ * 0 on success and an appropriate (-)ve integer as defined in errno.h
+ * file on failure.
+ */
+-static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
++static int stmmac_hw_setup(struct net_device *dev, bool ptp_register)
+ {
+ struct stmmac_priv *priv = netdev_priv(dev);
+ u32 rx_cnt = priv->plat->rx_queues_to_use;
+@@ -3270,17 +3325,13 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
+
+ stmmac_mmc_setup(priv);
+
+- if (init_ptp) {
+- ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
+- if (ret < 0)
+- netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
+-
+- ret = stmmac_init_ptp(priv);
+- if (ret == -EOPNOTSUPP)
+- netdev_warn(priv->dev, "PTP not supported by HW\n");
+- else if (ret)
+- netdev_warn(priv->dev, "PTP init failed\n");
+- }
++ ret = stmmac_init_ptp(priv);
++ if (ret == -EOPNOTSUPP)
++ netdev_warn(priv->dev, "PTP not supported by HW\n");
++ else if (ret)
++ netdev_warn(priv->dev, "PTP init failed\n");
++ else if (ptp_register)
++ stmmac_ptp_register(priv);
+
+ priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
+
+@@ -3635,7 +3686,7 @@ static int stmmac_request_irq(struct net_device *dev)
+ * 0 on success and an appropriate (-)ve integer as defined in errno.h
+ * file on failure.
+ */
+-int stmmac_open(struct net_device *dev)
++static int stmmac_open(struct net_device *dev)
+ {
+ struct stmmac_priv *priv = netdev_priv(dev);
+ int mode = priv->plat->phy_interface;
+@@ -3724,6 +3775,7 @@ int stmmac_open(struct net_device *dev)
+
+ stmmac_enable_all_queues(priv);
+ netif_tx_start_all_queues(priv->dev);
++ stmmac_enable_all_dma_irq(priv);
+
+ return 0;
+
+@@ -3759,11 +3811,13 @@ static void stmmac_fpe_stop_wq(struct stmmac_priv *priv)
+ * Description:
+ * This is the stop entry point of the driver.
+ */
+-int stmmac_release(struct net_device *dev)
++static int stmmac_release(struct net_device *dev)
+ {
+ struct stmmac_priv *priv = netdev_priv(dev);
+ u32 chan;
+
++ netif_tx_disable(dev);
++
+ if (device_may_wakeup(priv->device))
+ phylink_speed_down(priv->phylink, false);
+ /* Stop and disconnect the PHY */
+@@ -5499,8 +5553,6 @@ static int stmmac_set_features(struct net_device *netdev,
+ netdev_features_t features)
+ {
+ struct stmmac_priv *priv = netdev_priv(netdev);
+- bool sph_en;
+- u32 chan;
+
+ /* Keep the COE Type in case of csum is supporting */
+ if (features & NETIF_F_RXCSUM)
+@@ -5512,10 +5564,13 @@ static int stmmac_set_features(struct net_device *netdev,
+ */
+ stmmac_rx_ipc(priv, priv->hw);
+
+- sph_en = (priv->hw->rx_csum > 0) && priv->sph;
++ if (priv->sph_cap) {
++ bool sph_en = (priv->hw->rx_csum > 0) && priv->sph;
++ u32 chan;
+
+- for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
+- stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
++ for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
++ stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
++ }
+
+ return 0;
+ }
+@@ -6421,6 +6476,143 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
+ spin_unlock_irqrestore(&ch->lock, flags);
+ }
+
++void stmmac_xdp_release(struct net_device *dev)
++{
++ struct stmmac_priv *priv = netdev_priv(dev);
++ u32 chan;
++
++ /* Disable NAPI process */
++ stmmac_disable_all_queues(priv);
++
++ for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
++ hrtimer_cancel(&priv->tx_queue[chan].txtimer);
++
++ /* Free the IRQ lines */
++ stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
++
++ /* Stop TX/RX DMA channels */
++ stmmac_stop_all_dma(priv);
++
++ /* Release and free the Rx/Tx resources */
++ free_dma_desc_resources(priv);
++
++ /* Disable the MAC Rx/Tx */
++ stmmac_mac_set(priv, priv->ioaddr, false);
++
++ /* set trans_start so we don't get spurious
++ * watchdogs during reset
++ */
++ netif_trans_update(dev);
++ netif_carrier_off(dev);
++}
++
++int stmmac_xdp_open(struct net_device *dev)
++{
++ struct stmmac_priv *priv = netdev_priv(dev);
++ u32 rx_cnt = priv->plat->rx_queues_to_use;
++ u32 tx_cnt = priv->plat->tx_queues_to_use;
++ u32 dma_csr_ch = max(rx_cnt, tx_cnt);
++ struct stmmac_rx_queue *rx_q;
++ struct stmmac_tx_queue *tx_q;
++ u32 buf_size;
++ bool sph_en;
++ u32 chan;
++ int ret;
++
++ ret = alloc_dma_desc_resources(priv);
++ if (ret < 0) {
++ netdev_err(dev, "%s: DMA descriptors allocation failed\n",
++ __func__);
++ goto dma_desc_error;
++ }
++
++ ret = init_dma_desc_rings(dev, GFP_KERNEL);
++ if (ret < 0) {
++ netdev_err(dev, "%s: DMA descriptors initialization failed\n",
++ __func__);
++ goto init_error;
++ }
++
++ /* DMA CSR Channel configuration */
++ for (chan = 0; chan < dma_csr_ch; chan++) {
++ stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
++ stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
++ }
++
++ /* Adjust Split header */
++ sph_en = (priv->hw->rx_csum > 0) && priv->sph;
++
++ /* DMA RX Channel Configuration */
++ for (chan = 0; chan < rx_cnt; chan++) {
++ rx_q = &priv->rx_queue[chan];
++
++ stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
++ rx_q->dma_rx_phy, chan);
++
++ rx_q->rx_tail_addr = rx_q->dma_rx_phy +
++ (rx_q->buf_alloc_num *
++ sizeof(struct dma_desc));
++ stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
++ rx_q->rx_tail_addr, chan);
++
++ if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
++ buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
++ stmmac_set_dma_bfsize(priv, priv->ioaddr,
++ buf_size,
++ rx_q->queue_index);
++ } else {
++ stmmac_set_dma_bfsize(priv, priv->ioaddr,
++ priv->dma_buf_sz,
++ rx_q->queue_index);
++ }
++
++ stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
++ }
++
++ /* DMA TX Channel Configuration */
++ for (chan = 0; chan < tx_cnt; chan++) {
++ tx_q = &priv->tx_queue[chan];
++
++ stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
++ tx_q->dma_tx_phy, chan);
++
++ tx_q->tx_tail_addr = tx_q->dma_tx_phy;
++ stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
++ tx_q->tx_tail_addr, chan);
++
++ hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
++ tx_q->txtimer.function = stmmac_tx_timer;
++ }
++
++ /* Enable the MAC Rx/Tx */
++ stmmac_mac_set(priv, priv->ioaddr, true);
++
++ /* Start Rx & Tx DMA Channels */
++ stmmac_start_all_dma(priv);
++
++ ret = stmmac_request_irq(dev);
++ if (ret)
++ goto irq_error;
++
++ /* Enable NAPI process*/
++ stmmac_enable_all_queues(priv);
++ netif_carrier_on(dev);
++ netif_tx_start_all_queues(dev);
++ stmmac_enable_all_dma_irq(priv);
++
++ return 0;
++
++irq_error:
++ for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
++ hrtimer_cancel(&priv->tx_queue[chan].txtimer);
++
++ stmmac_hw_teardown(dev);
++init_error:
++ free_dma_desc_resources(priv);
++dma_desc_error:
++ return ret;
++}
++
+ int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
+ {
+ struct stmmac_priv *priv = netdev_priv(dev);
+@@ -6889,7 +7081,7 @@ int stmmac_dvr_probe(struct device *device,
+ dev_info(priv->device, "TSO feature enabled\n");
+ }
+
+- if (priv->dma_cap.sphen) {
++ if (priv->dma_cap.sphen && !priv->plat->sph_disable) {
+ ndev->hw_features |= NETIF_F_GRO;
+ priv->sph_cap = true;
+ priv->sph = priv->sph_cap;
+@@ -7044,6 +7236,9 @@ int stmmac_dvr_probe(struct device *device,
+ stmmac_init_fs(ndev);
+ #endif
+
++ if (priv->plat->dump_debug_regs)
++ priv->plat->dump_debug_regs(priv->plat->bsp_priv);
++
+ /* Let pm_runtime_put() disable the clocks.
+ * If CONFIG_PM is not enabled, the clocks will stay powered.
+ */
+@@ -7083,6 +7278,10 @@ int stmmac_dvr_remove(struct device *dev)
+
+ netdev_info(priv->dev, "%s: removing driver", __func__);
+
++ pm_runtime_get_sync(dev);
++ pm_runtime_disable(dev);
++ pm_runtime_put_noidle(dev);
++
+ stmmac_stop_all_dma(priv);
+ stmmac_mac_set(priv, priv->ioaddr, false);
+ netif_carrier_off(ndev);
+@@ -7101,8 +7300,6 @@ int stmmac_dvr_remove(struct device *dev)
+ if (priv->plat->stmmac_rst)
+ reset_control_assert(priv->plat->stmmac_rst);
+ reset_control_assert(priv->plat->stmmac_ahb_rst);
+- pm_runtime_put(dev);
+- pm_runtime_disable(dev);
+ if (priv->hw->pcs != STMMAC_PCS_TBI &&
+ priv->hw->pcs != STMMAC_PCS_RTBI)
+ stmmac_mdio_unregister(ndev);
+@@ -7280,6 +7477,7 @@ int stmmac_resume(struct device *dev)
+ stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
+
+ stmmac_enable_all_queues(priv);
++ stmmac_enable_all_dma_irq(priv);
+
+ mutex_unlock(&priv->lock);
+ rtnl_unlock();
+@@ -7296,7 +7494,7 @@ static int __init stmmac_cmdline_opt(char *str)
+ char *opt;
+
+ if (!str || !*str)
+- return -EINVAL;
++ return 1;
+ while ((opt = strsep(&str, ",")) != NULL) {
+ if (!strncmp(opt, "debug:", 6)) {
+ if (kstrtoint(opt + 6, 0, &debug))
+@@ -7327,11 +7525,11 @@ static int __init stmmac_cmdline_opt(char *str)
+ goto err;
+ }
+ }
+- return 0;
++ return 1;
+
+ err:
+ pr_err("%s: ERROR broken module parameter conversion", __func__);
+- return -EINVAL;
++ return 1;
+ }
+
+ __setup("stmmaceth=", stmmac_cmdline_opt);
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
+index fcf17d8a0494b..644bb54f5f020 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
+@@ -181,7 +181,7 @@ static int stmmac_pci_probe(struct pci_dev *pdev,
+ return -ENOMEM;
+
+ /* Enable pci device */
+- ret = pci_enable_device(pdev);
++ ret = pcim_enable_device(pdev);
+ if (ret) {
+ dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n",
+ __func__);
+@@ -241,8 +241,6 @@ static void stmmac_pci_remove(struct pci_dev *pdev)
+ pcim_iounmap_regions(pdev, BIT(i));
+ break;
+ }
+-
+- pci_disable_device(pdev);
+ }
+
+ static int __maybe_unused stmmac_pci_suspend(struct device *dev)
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+index 232ac98943cd0..11e1055e8260f 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+@@ -431,8 +431,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
+ plat->phylink_node = np;
+
+ /* Get max speed of operation from device tree */
+- if (of_property_read_u32(np, "max-speed", &plat->max_speed))
+- plat->max_speed = -1;
++ of_property_read_u32(np, "max-speed", &plat->max_speed);
+
+ plat->bus_id = of_alias_get_id(np, "ethernet");
+ if (plat->bus_id < 0)
+@@ -816,7 +815,7 @@ static int __maybe_unused stmmac_pltfr_noirq_resume(struct device *dev)
+ if (ret)
+ return ret;
+
+- clk_prepare_enable(priv->plat->clk_ptp_ref);
++ stmmac_init_tstamp_counter(priv, priv->systime_flags);
+ }
+
+ return 0;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+index 580cc035536bd..ac8bc1c8614d3 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+@@ -102,7 +102,7 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta)
+ time.tv_nsec = priv->plat->est->btr_reserve[0];
+ time.tv_sec = priv->plat->est->btr_reserve[1];
+ basetime = timespec64_to_ktime(time);
+- cycle_time = priv->plat->est->ctr[1] * NSEC_PER_SEC +
++ cycle_time = (u64)priv->plat->est->ctr[1] * NSEC_PER_SEC +
+ priv->plat->est->ctr[0];
+ time = stmmac_calc_tas_basetime(basetime,
+ current_time_ns,
+@@ -297,9 +297,6 @@ void stmmac_ptp_register(struct stmmac_priv *priv)
+ {
+ int i;
+
+- if (priv->plat->ptp_clk_freq_config)
+- priv->plat->ptp_clk_freq_config(priv);
+-
+ for (i = 0; i < priv->dma_cap.pps_out_num; i++) {
+ if (i >= STMMAC_PPS_MAX)
+ break;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
+index 8160087ee92f2..d0a2b289f4603 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
+@@ -232,11 +232,33 @@ static int tc_setup_cls_u32(struct stmmac_priv *priv,
+ }
+ }
+
++static int tc_rfs_init(struct stmmac_priv *priv)
++{
++ int i;
++
++ priv->rfs_entries_max[STMMAC_RFS_T_VLAN] = 8;
++
++ for (i = 0; i < STMMAC_RFS_T_MAX; i++)
++ priv->rfs_entries_total += priv->rfs_entries_max[i];
++
++ priv->rfs_entries = devm_kcalloc(priv->device,
++ priv->rfs_entries_total,
++ sizeof(*priv->rfs_entries),
++ GFP_KERNEL);
++ if (!priv->rfs_entries)
++ return -ENOMEM;
++
++ dev_info(priv->device, "Enabled RFS Flow TC (entries=%d)\n",
++ priv->rfs_entries_total);
++
++ return 0;
++}
++
+ static int tc_init(struct stmmac_priv *priv)
+ {
+ struct dma_features *dma_cap = &priv->dma_cap;
+ unsigned int count;
+- int i;
++ int ret, i;
+
+ if (dma_cap->l3l4fnum) {
+ priv->flow_entries_max = dma_cap->l3l4fnum;
+@@ -250,10 +272,14 @@ static int tc_init(struct stmmac_priv *priv)
+ for (i = 0; i < priv->flow_entries_max; i++)
+ priv->flow_entries[i].idx = i;
+
+- dev_info(priv->device, "Enabled Flow TC (entries=%d)\n",
++ dev_info(priv->device, "Enabled L3L4 Flow TC (entries=%d)\n",
+ priv->flow_entries_max);
+ }
+
++ ret = tc_rfs_init(priv);
++ if (ret)
++ return -ENOMEM;
++
+ if (!priv->plat->fpe_cfg) {
+ priv->plat->fpe_cfg = devm_kzalloc(priv->device,
+ sizeof(*priv->plat->fpe_cfg),
+@@ -607,16 +633,45 @@ static int tc_del_flow(struct stmmac_priv *priv,
+ return ret;
+ }
+
++static struct stmmac_rfs_entry *tc_find_rfs(struct stmmac_priv *priv,
++ struct flow_cls_offload *cls,
++ bool get_free)
++{
++ int i;
++
++ for (i = 0; i < priv->rfs_entries_total; i++) {
++ struct stmmac_rfs_entry *entry = &priv->rfs_entries[i];
++
++ if (entry->cookie == cls->cookie)
++ return entry;
++ if (get_free && entry->in_use == false)
++ return entry;
++ }
++
++ return NULL;
++}
++
+ #define VLAN_PRIO_FULL_MASK (0x07)
+
+ static int tc_add_vlan_flow(struct stmmac_priv *priv,
+ struct flow_cls_offload *cls)
+ {
++ struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+ struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+ struct flow_dissector *dissector = rule->match.dissector;
+ int tc = tc_classid_to_hwtc(priv->dev, cls->classid);
+ struct flow_match_vlan match;
+
++ if (!entry) {
++ entry = tc_find_rfs(priv, cls, true);
++ if (!entry)
++ return -ENOENT;
++ }
++
++ if (priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN] >=
++ priv->rfs_entries_max[STMMAC_RFS_T_VLAN])
++ return -ENOENT;
++
+ /* Nothing to do here */
+ if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN))
+ return -EINVAL;
+@@ -638,6 +693,12 @@ static int tc_add_vlan_flow(struct stmmac_priv *priv,
+
+ prio = BIT(match.key->vlan_priority);
+ stmmac_rx_queue_prio(priv, priv->hw, prio, tc);
++
++ entry->in_use = true;
++ entry->cookie = cls->cookie;
++ entry->tc = tc;
++ entry->type = STMMAC_RFS_T_VLAN;
++ priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]++;
+ }
+
+ return 0;
+@@ -646,20 +707,19 @@ static int tc_add_vlan_flow(struct stmmac_priv *priv,
+ static int tc_del_vlan_flow(struct stmmac_priv *priv,
+ struct flow_cls_offload *cls)
+ {
+- struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+- struct flow_dissector *dissector = rule->match.dissector;
+- int tc = tc_classid_to_hwtc(priv->dev, cls->classid);
++ struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+
+- /* Nothing to do here */
+- if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN))
+- return -EINVAL;
++ if (!entry || !entry->in_use || entry->type != STMMAC_RFS_T_VLAN)
++ return -ENOENT;
+
+- if (tc < 0) {
+- netdev_err(priv->dev, "Invalid traffic class\n");
+- return -EINVAL;
+- }
++ stmmac_rx_queue_prio(priv, priv->hw, 0, entry->tc);
++
++ entry->in_use = false;
++ entry->cookie = 0;
++ entry->tc = 0;
++ entry->type = 0;
+
+- stmmac_rx_queue_prio(priv, priv->hw, 0, tc);
++ priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]--;
+
+ return 0;
+ }
+@@ -786,8 +846,6 @@ static int tc_setup_taprio(struct stmmac_priv *priv,
+ goto disable;
+ if (qopt->num_entries >= dep)
+ return -EINVAL;
+- if (!qopt->base_time)
+- return -ERANGE;
+ if (!qopt->cycle_time)
+ return -ERANGE;
+
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c
+index 2a616c6f7cd0e..9d4d8c3dad0a3 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c
+@@ -119,7 +119,7 @@ int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog,
+
+ need_update = !!priv->xdp_prog != !!prog;
+ if (if_running && need_update)
+- stmmac_release(dev);
++ stmmac_xdp_release(dev);
+
+ old_prog = xchg(&priv->xdp_prog, prog);
+ if (old_prog)
+@@ -129,7 +129,7 @@ int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog,
+ priv->sph = priv->sph_cap && !stmmac_xdp_is_enabled(priv);
+
+ if (if_running && need_update)
+- stmmac_open(dev);
++ stmmac_xdp_open(dev);
+
+ return 0;
+ }
+diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c
+index 62f81b0d14ed8..b05ee2e0e3051 100644
+--- a/drivers/net/ethernet/sun/sunhme.c
++++ b/drivers/net/ethernet/sun/sunhme.c
+@@ -3139,7 +3139,7 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
+ if (err) {
+ printk(KERN_ERR "happymeal(PCI): Cannot register net device, "
+ "aborting.\n");
+- goto err_out_iounmap;
++ goto err_out_free_coherent;
+ }
+
+ pci_set_drvdata(pdev, hp);
+@@ -3172,6 +3172,10 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
+
+ return 0;
+
++err_out_free_coherent:
++ dma_free_coherent(hp->dma_dev, PAGE_SIZE,
++ hp->happy_block, hp->hblock_dvma);
++
+ err_out_iounmap:
+ iounmap(hp->gregs);
+
+diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
+index 66f7ddd9b1f99..e226ecd95a2cc 100644
+--- a/drivers/net/ethernet/ti/cpsw.c
++++ b/drivers/net/ethernet/ti/cpsw.c
+@@ -349,7 +349,7 @@ static void cpsw_rx_handler(void *token, int len, int status)
+ struct cpsw_common *cpsw = ndev_to_cpsw(xmeta->ndev);
+ int pkt_size = cpsw->rx_packet_max;
+ int ret = 0, port, ch = xmeta->ch;
+- int headroom = CPSW_HEADROOM;
++ int headroom = CPSW_HEADROOM_NA;
+ struct net_device *ndev = xmeta->ndev;
+ struct cpsw_priv *priv;
+ struct page_pool *pool;
+@@ -392,7 +392,7 @@ static void cpsw_rx_handler(void *token, int len, int status)
+ }
+
+ if (priv->xdp_prog) {
+- int headroom = CPSW_HEADROOM, size = len;
++ int size = len;
+
+ xdp_init_buff(&xdp, PAGE_SIZE, &priv->xdp_rxq[ch]);
+ if (status & CPDMA_RX_VLAN_ENCAP) {
+@@ -442,7 +442,7 @@ requeue:
+ xmeta->ndev = ndev;
+ xmeta->ch = ch;
+
+- dma = page_pool_get_dma_addr(new_page) + CPSW_HEADROOM;
++ dma = page_pool_get_dma_addr(new_page) + CPSW_HEADROOM_NA;
+ ret = cpdma_chan_submit_mapped(cpsw->rxv[ch].ch, new_page, dma,
+ pkt_size, 0);
+ if (ret < 0) {
+diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c
+index 0c75e0576ee1f..1ef0aaef5c61c 100644
+--- a/drivers/net/ethernet/ti/cpsw_ale.c
++++ b/drivers/net/ethernet/ti/cpsw_ale.c
+@@ -1299,10 +1299,8 @@ struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params)
+ if (!ale)
+ return ERR_PTR(-ENOMEM);
+
+- ale->p0_untag_vid_mask =
+- devm_kmalloc_array(params->dev, BITS_TO_LONGS(VLAN_N_VID),
+- sizeof(unsigned long),
+- GFP_KERNEL);
++ ale->p0_untag_vid_mask = devm_bitmap_zalloc(params->dev, VLAN_N_VID,
++ GFP_KERNEL);
+ if (!ale->p0_untag_vid_mask)
+ return ERR_PTR(-ENOMEM);
+
+diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c
+index 158c8d3793f43..b5bae6324970a 100644
+--- a/drivers/net/ethernet/ti/cpsw_ethtool.c
++++ b/drivers/net/ethernet/ti/cpsw_ethtool.c
+@@ -364,11 +364,9 @@ int cpsw_ethtool_op_begin(struct net_device *ndev)
+ struct cpsw_common *cpsw = priv->cpsw;
+ int ret;
+
+- ret = pm_runtime_get_sync(cpsw->dev);
+- if (ret < 0) {
++ ret = pm_runtime_resume_and_get(cpsw->dev);
++ if (ret < 0)
+ cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
+- pm_runtime_put_noidle(cpsw->dev);
+- }
+
+ return ret;
+ }
+diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c
+index 7968f24d99c85..277c91d135708 100644
+--- a/drivers/net/ethernet/ti/cpsw_new.c
++++ b/drivers/net/ethernet/ti/cpsw_new.c
+@@ -283,7 +283,7 @@ static void cpsw_rx_handler(void *token, int len, int status)
+ {
+ struct page *new_page, *page = token;
+ void *pa = page_address(page);
+- int headroom = CPSW_HEADROOM;
++ int headroom = CPSW_HEADROOM_NA;
+ struct cpsw_meta_xdp *xmeta;
+ struct cpsw_common *cpsw;
+ struct net_device *ndev;
+@@ -336,7 +336,7 @@ static void cpsw_rx_handler(void *token, int len, int status)
+ }
+
+ if (priv->xdp_prog) {
+- int headroom = CPSW_HEADROOM, size = len;
++ int size = len;
+
+ xdp_init_buff(&xdp, PAGE_SIZE, &priv->xdp_rxq[ch]);
+ if (status & CPDMA_RX_VLAN_ENCAP) {
+@@ -386,7 +386,7 @@ requeue:
+ xmeta->ndev = ndev;
+ xmeta->ch = ch;
+
+- dma = page_pool_get_dma_addr(new_page) + CPSW_HEADROOM;
++ dma = page_pool_get_dma_addr(new_page) + CPSW_HEADROOM_NA;
+ ret = cpdma_chan_submit_mapped(cpsw->rxv[ch].ch, new_page, dma,
+ pkt_size, 0);
+ if (ret < 0) {
+@@ -1246,8 +1246,10 @@ static int cpsw_probe_dt(struct cpsw_common *cpsw)
+ data->slave_data = devm_kcalloc(dev, CPSW_SLAVE_PORTS_NUM,
+ sizeof(struct cpsw_slave_data),
+ GFP_KERNEL);
+- if (!data->slave_data)
++ if (!data->slave_data) {
++ of_node_put(tmp_node);
+ return -ENOMEM;
++ }
+
+ /* Populate all the child nodes here...
+ */
+@@ -1341,6 +1343,7 @@ static int cpsw_probe_dt(struct cpsw_common *cpsw)
+
+ err_node_put:
+ of_node_put(port_np);
++ of_node_put(tmp_node);
+ return ret;
+ }
+
+diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c
+index ecc2a6b7e28f2..f8e591d69d2cb 100644
+--- a/drivers/net/ethernet/ti/cpsw_priv.c
++++ b/drivers/net/ethernet/ti/cpsw_priv.c
+@@ -1120,7 +1120,7 @@ int cpsw_fill_rx_channels(struct cpsw_priv *priv)
+ xmeta->ndev = priv->ndev;
+ xmeta->ch = ch;
+
+- dma = page_pool_get_dma_addr(page) + CPSW_HEADROOM;
++ dma = page_pool_get_dma_addr(page) + CPSW_HEADROOM_NA;
+ ret = cpdma_chan_idle_submit_mapped(cpsw->rxv[ch].ch,
+ page, dma,
+ cpsw->rx_packet_max,
+@@ -1144,7 +1144,7 @@ int cpsw_fill_rx_channels(struct cpsw_priv *priv)
+ static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw,
+ int size)
+ {
+- struct page_pool_params pp_params;
++ struct page_pool_params pp_params = {};
+ struct page_pool *pool;
+
+ pp_params.order = 0;
+diff --git a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c
+index 43222a34cba06..f9514518700eb 100644
+--- a/drivers/net/ethernet/ti/cpts.c
++++ b/drivers/net/ethernet/ti/cpts.c
+@@ -568,7 +568,9 @@ int cpts_register(struct cpts *cpts)
+ for (i = 0; i < CPTS_MAX_EVENTS; i++)
+ list_add(&cpts->pool_data[i].list, &cpts->pool);
+
+- clk_enable(cpts->refclk);
++ err = clk_enable(cpts->refclk);
++ if (err)
++ return err;
+
+ cpts_write32(cpts, CPTS_EN, control);
+ cpts_write32(cpts, TS_PEND_EN, int_enable);
+diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
+index e8291d8488391..d243ca5dfde00 100644
+--- a/drivers/net/ethernet/ti/davinci_emac.c
++++ b/drivers/net/ethernet/ti/davinci_emac.c
+@@ -420,8 +420,20 @@ static int emac_set_coalesce(struct net_device *ndev,
+ u32 int_ctrl, num_interrupts = 0;
+ u32 prescale = 0, addnl_dvdr = 1, coal_intvl = 0;
+
+- if (!coal->rx_coalesce_usecs)
+- return -EINVAL;
++ if (!coal->rx_coalesce_usecs) {
++ priv->coal_intvl = 0;
++
++ switch (priv->version) {
++ case EMAC_VERSION_2:
++ emac_ctrl_write(EMAC_DM646X_CMINTCTRL, 0);
++ break;
++ default:
++ emac_ctrl_write(EMAC_CTRL_EWINTTCNT, 0);
++ break;
++ }
++
++ return 0;
++ }
+
+ coal_intvl = coal->rx_coalesce_usecs;
+
+diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c
+index 463094ced104a..2ab29efa6b6e4 100644
+--- a/drivers/net/ethernet/xilinx/ll_temac_main.c
++++ b/drivers/net/ethernet/xilinx/ll_temac_main.c
+@@ -1427,6 +1427,8 @@ static int temac_probe(struct platform_device *pdev)
+ lp->indirect_lock = devm_kmalloc(&pdev->dev,
+ sizeof(*lp->indirect_lock),
+ GFP_KERNEL);
++ if (!lp->indirect_lock)
++ return -ENOMEM;
+ spin_lock_init(lp->indirect_lock);
+ }
+
+diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+index 871b5ec3183d6..fbbbcfe0e891e 100644
+--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
++++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+@@ -41,8 +41,9 @@
+ #include "xilinx_axienet.h"
+
+ /* Descriptors defines for Tx and Rx DMA */
+-#define TX_BD_NUM_DEFAULT 64
++#define TX_BD_NUM_DEFAULT 128
+ #define RX_BD_NUM_DEFAULT 1024
++#define TX_BD_NUM_MIN (MAX_SKB_FRAGS + 1)
+ #define TX_BD_NUM_MAX 4096
+ #define RX_BD_NUM_MAX 4096
+
+@@ -496,7 +497,8 @@ static void axienet_setoptions(struct net_device *ndev, u32 options)
+
+ static int __axienet_device_reset(struct axienet_local *lp)
+ {
+- u32 timeout;
++ u32 value;
++ int ret;
+
+ /* Reset Axi DMA. This would reset Axi Ethernet core as well. The reset
+ * process of Axi DMA takes a while to complete as all pending
+@@ -506,15 +508,23 @@ static int __axienet_device_reset(struct axienet_local *lp)
+ * they both reset the entire DMA core, so only one needs to be used.
+ */
+ axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, XAXIDMA_CR_RESET_MASK);
+- timeout = DELAY_OF_ONE_MILLISEC;
+- while (axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET) &
+- XAXIDMA_CR_RESET_MASK) {
+- udelay(1);
+- if (--timeout == 0) {
+- netdev_err(lp->ndev, "%s: DMA reset timeout!\n",
+- __func__);
+- return -ETIMEDOUT;
+- }
++ ret = read_poll_timeout(axienet_dma_in32, value,
++ !(value & XAXIDMA_CR_RESET_MASK),
++ DELAY_OF_ONE_MILLISEC, 50000, false, lp,
++ XAXIDMA_TX_CR_OFFSET);
++ if (ret) {
++ dev_err(lp->dev, "%s: DMA reset timeout!\n", __func__);
++ return ret;
++ }
++
++ /* Wait for PhyRstCmplt bit to be set, indicating the PHY reset has finished */
++ ret = read_poll_timeout(axienet_ior, value,
++ value & XAE_INT_PHYRSTCMPLT_MASK,
++ DELAY_OF_ONE_MILLISEC, 50000, false, lp,
++ XAE_IS_OFFSET);
++ if (ret) {
++ dev_err(lp->dev, "%s: timeout waiting for PhyRstCmplt\n", __func__);
++ return ret;
+ }
+
+ return 0;
+@@ -623,6 +633,8 @@ static int axienet_free_tx_chain(struct net_device *ndev, u32 first_bd,
+ if (nr_bds == -1 && !(status & XAXIDMA_BD_STS_COMPLETE_MASK))
+ break;
+
++ /* Ensure we see complete descriptor update */
++ dma_rmb();
+ phys = desc_get_phys_addr(lp, cur_p);
+ dma_unmap_single(ndev->dev.parent, phys,
+ (cur_p->cntrl & XAXIDMA_BD_CTRL_LENGTH_MASK),
+@@ -631,13 +643,15 @@ static int axienet_free_tx_chain(struct net_device *ndev, u32 first_bd,
+ if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK))
+ dev_consume_skb_irq(cur_p->skb);
+
+- cur_p->cntrl = 0;
+ cur_p->app0 = 0;
+ cur_p->app1 = 0;
+ cur_p->app2 = 0;
+ cur_p->app4 = 0;
+- cur_p->status = 0;
+ cur_p->skb = NULL;
++ /* ensure our transmit path and device don't prematurely see status cleared */
++ wmb();
++ cur_p->cntrl = 0;
++ cur_p->status = 0;
+
+ if (sizep)
+ *sizep += status & XAXIDMA_BD_STS_ACTUAL_LEN_MASK;
+@@ -646,6 +660,32 @@ static int axienet_free_tx_chain(struct net_device *ndev, u32 first_bd,
+ return i;
+ }
+
++/**
++ * axienet_check_tx_bd_space - Checks if a BD/group of BDs are currently busy
++ * @lp: Pointer to the axienet_local structure
++ * @num_frag: The number of BDs to check for
++ *
++ * Return: 0, on success
++ * NETDEV_TX_BUSY, if any of the descriptors are not free
++ *
++ * This function is invoked before BDs are allocated and transmission starts.
++ * This function returns 0 if a BD or group of BDs can be allocated for
++ * transmission. If the BD or any of the BDs are not free the function
++ * returns a busy status. This is invoked from axienet_start_xmit.
++ */
++static inline int axienet_check_tx_bd_space(struct axienet_local *lp,
++ int num_frag)
++{
++ struct axidma_bd *cur_p;
++
++ /* Ensure we see all descriptor updates from device or TX IRQ path */
++ rmb();
++ cur_p = &lp->tx_bd_v[(lp->tx_bd_tail + num_frag) % lp->tx_bd_num];
++ if (cur_p->cntrl)
++ return NETDEV_TX_BUSY;
++ return 0;
++}
++
+ /**
+ * axienet_start_xmit_done - Invoked once a transmit is completed by the
+ * Axi DMA Tx channel.
+@@ -675,30 +715,8 @@ static void axienet_start_xmit_done(struct net_device *ndev)
+ /* Matches barrier in axienet_start_xmit */
+ smp_mb();
+
+- netif_wake_queue(ndev);
+-}
+-
+-/**
+- * axienet_check_tx_bd_space - Checks if a BD/group of BDs are currently busy
+- * @lp: Pointer to the axienet_local structure
+- * @num_frag: The number of BDs to check for
+- *
+- * Return: 0, on success
+- * NETDEV_TX_BUSY, if any of the descriptors are not free
+- *
+- * This function is invoked before BDs are allocated and transmission starts.
+- * This function returns 0 if a BD or group of BDs can be allocated for
+- * transmission. If the BD or any of the BDs are not free the function
+- * returns a busy status. This is invoked from axienet_start_xmit.
+- */
+-static inline int axienet_check_tx_bd_space(struct axienet_local *lp,
+- int num_frag)
+-{
+- struct axidma_bd *cur_p;
+- cur_p = &lp->tx_bd_v[(lp->tx_bd_tail + num_frag) % lp->tx_bd_num];
+- if (cur_p->status & XAXIDMA_BD_STS_ALL_MASK)
+- return NETDEV_TX_BUSY;
+- return 0;
++ if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1))
++ netif_wake_queue(ndev);
+ }
+
+ /**
+@@ -730,20 +748,15 @@ axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+ num_frag = skb_shinfo(skb)->nr_frags;
+ cur_p = &lp->tx_bd_v[lp->tx_bd_tail];
+
+- if (axienet_check_tx_bd_space(lp, num_frag)) {
+- if (netif_queue_stopped(ndev))
+- return NETDEV_TX_BUSY;
+-
++ if (axienet_check_tx_bd_space(lp, num_frag + 1)) {
++ /* Should not happen as last start_xmit call should have
++ * checked for sufficient space and queue should only be
++ * woken when sufficient space is available.
++ */
+ netif_stop_queue(ndev);
+-
+- /* Matches barrier in axienet_start_xmit_done */
+- smp_mb();
+-
+- /* Space might have just been freed - check again */
+- if (axienet_check_tx_bd_space(lp, num_frag))
+- return NETDEV_TX_BUSY;
+-
+- netif_wake_queue(ndev);
++ if (net_ratelimit())
++ netdev_warn(ndev, "TX ring unexpectedly full\n");
++ return NETDEV_TX_BUSY;
+ }
+
+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
+@@ -804,6 +817,18 @@ axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+ if (++lp->tx_bd_tail >= lp->tx_bd_num)
+ lp->tx_bd_tail = 0;
+
++ /* Stop queue if next transmit may not have space */
++ if (axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) {
++ netif_stop_queue(ndev);
++
++ /* Matches barrier in axienet_start_xmit_done */
++ smp_mb();
++
++ /* Space might have just been freed - check again */
++ if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1))
++ netif_wake_queue(ndev);
++ }
++
+ return NETDEV_TX_OK;
+ }
+
+@@ -832,44 +857,53 @@ static void axienet_recv(struct net_device *ndev)
+ while ((cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) {
+ dma_addr_t phys;
+
+- tail_p = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * lp->rx_bd_ci;
+-
+- phys = desc_get_phys_addr(lp, cur_p);
+- dma_unmap_single(ndev->dev.parent, phys, lp->max_frm_size,
+- DMA_FROM_DEVICE);
++ /* Ensure we see complete descriptor update */
++ dma_rmb();
+
+ skb = cur_p->skb;
+ cur_p->skb = NULL;
+- length = cur_p->app4 & 0x0000FFFF;
+-
+- skb_put(skb, length);
+- skb->protocol = eth_type_trans(skb, ndev);
+- /*skb_checksum_none_assert(skb);*/
+- skb->ip_summed = CHECKSUM_NONE;
+-
+- /* if we're doing Rx csum offload, set it up */
+- if (lp->features & XAE_FEATURE_FULL_RX_CSUM) {
+- csumstatus = (cur_p->app2 &
+- XAE_FULL_CSUM_STATUS_MASK) >> 3;
+- if ((csumstatus == XAE_IP_TCP_CSUM_VALIDATED) ||
+- (csumstatus == XAE_IP_UDP_CSUM_VALIDATED)) {
+- skb->ip_summed = CHECKSUM_UNNECESSARY;
++
++ /* skb could be NULL if a previous pass already received the
++ * packet for this slot in the ring, but failed to refill it
++ * with a newly allocated buffer. In this case, don't try to
++ * receive it again.
++ */
++ if (likely(skb)) {
++ length = cur_p->app4 & 0x0000FFFF;
++
++ phys = desc_get_phys_addr(lp, cur_p);
++ dma_unmap_single(ndev->dev.parent, phys, lp->max_frm_size,
++ DMA_FROM_DEVICE);
++
++ skb_put(skb, length);
++ skb->protocol = eth_type_trans(skb, ndev);
++ /*skb_checksum_none_assert(skb);*/
++ skb->ip_summed = CHECKSUM_NONE;
++
++ /* if we're doing Rx csum offload, set it up */
++ if (lp->features & XAE_FEATURE_FULL_RX_CSUM) {
++ csumstatus = (cur_p->app2 &
++ XAE_FULL_CSUM_STATUS_MASK) >> 3;
++ if (csumstatus == XAE_IP_TCP_CSUM_VALIDATED ||
++ csumstatus == XAE_IP_UDP_CSUM_VALIDATED) {
++ skb->ip_summed = CHECKSUM_UNNECESSARY;
++ }
++ } else if ((lp->features & XAE_FEATURE_PARTIAL_RX_CSUM) != 0 &&
++ skb->protocol == htons(ETH_P_IP) &&
++ skb->len > 64) {
++ skb->csum = be32_to_cpu(cur_p->app3 & 0xFFFF);
++ skb->ip_summed = CHECKSUM_COMPLETE;
+ }
+- } else if ((lp->features & XAE_FEATURE_PARTIAL_RX_CSUM) != 0 &&
+- skb->protocol == htons(ETH_P_IP) &&
+- skb->len > 64) {
+- skb->csum = be32_to_cpu(cur_p->app3 & 0xFFFF);
+- skb->ip_summed = CHECKSUM_COMPLETE;
+- }
+
+- netif_rx(skb);
++ netif_rx(skb);
+
+- size += length;
+- packets++;
++ size += length;
++ packets++;
++ }
+
+ new_skb = netdev_alloc_skb_ip_align(ndev, lp->max_frm_size);
+ if (!new_skb)
+- return;
++ break;
+
+ phys = dma_map_single(ndev->dev.parent, new_skb->data,
+ lp->max_frm_size,
+@@ -878,7 +912,7 @@ static void axienet_recv(struct net_device *ndev)
+ if (net_ratelimit())
+ netdev_err(ndev, "RX DMA mapping error\n");
+ dev_kfree_skb(new_skb);
+- return;
++ break;
+ }
+ desc_set_phys_addr(lp, phys, cur_p);
+
+@@ -886,6 +920,11 @@ static void axienet_recv(struct net_device *ndev)
+ cur_p->status = 0;
+ cur_p->skb = new_skb;
+
++ /* Only update tail_p to mark this slot as usable after it has
++ * been successfully refilled.
++ */
++ tail_p = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * lp->rx_bd_ci;
++
+ if (++lp->rx_bd_ci >= lp->rx_bd_num)
+ lp->rx_bd_ci = 0;
+ cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
+@@ -1346,7 +1385,8 @@ static int axienet_ethtools_set_ringparam(struct net_device *ndev,
+ if (ering->rx_pending > RX_BD_NUM_MAX ||
+ ering->rx_mini_pending ||
+ ering->rx_jumbo_pending ||
+- ering->rx_pending > TX_BD_NUM_MAX)
++ ering->tx_pending < TX_BD_NUM_MIN ||
++ ering->tx_pending > TX_BD_NUM_MAX)
+ return -EINVAL;
+
+ if (netif_running(ndev))
+@@ -2082,15 +2122,19 @@ static int axienet_probe(struct platform_device *pdev)
+ lp->coalesce_count_rx = XAXIDMA_DFT_RX_THRESHOLD;
+ lp->coalesce_count_tx = XAXIDMA_DFT_TX_THRESHOLD;
+
+- lp->phy_node = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
+- if (lp->phy_node) {
+- ret = axienet_mdio_setup(lp);
+- if (ret)
+- dev_warn(&pdev->dev,
+- "error registering MDIO bus: %d\n", ret);
+- }
++ /* Reset core now that clocks are enabled, prior to accessing MDIO */
++ ret = __axienet_device_reset(lp);
++ if (ret)
++ goto cleanup_clk;
++
++ ret = axienet_mdio_setup(lp);
++ if (ret)
++ dev_warn(&pdev->dev,
++ "error registering MDIO bus: %d\n", ret);
++
+ if (lp->phy_mode == PHY_INTERFACE_MODE_SGMII ||
+ lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX) {
++ lp->phy_node = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
+ if (!lp->phy_node) {
+ dev_err(&pdev->dev, "phy-handle required for 1000BaseX/SGMII\n");
+ ret = -EINVAL;
+diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
+index b780aad3550aa..97c1d1ecba34c 100644
+--- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c
++++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
+@@ -822,10 +822,10 @@ static int xemaclite_mdio_write(struct mii_bus *bus, int phy_id, int reg,
+ static int xemaclite_mdio_setup(struct net_local *lp, struct device *dev)
+ {
+ struct mii_bus *bus;
+- int rc;
+ struct resource res;
+ struct device_node *np = of_get_parent(lp->phy_node);
+ struct device_node *npp;
++ int rc, ret;
+
+ /* Don't register the MDIO bus if the phy_node or its parent node
+ * can't be found.
+@@ -835,8 +835,14 @@ static int xemaclite_mdio_setup(struct net_local *lp, struct device *dev)
+ return -ENODEV;
+ }
+ npp = of_get_parent(np);
+-
+- of_address_to_resource(npp, 0, &res);
++ ret = of_address_to_resource(npp, 0, &res);
++ of_node_put(npp);
++ if (ret) {
++ dev_err(dev, "%s resource error!\n",
++ dev->of_node->full_name);
++ of_node_put(np);
++ return ret;
++ }
+ if (lp->ndev->mem_start != res.start) {
+ struct phy_device *phydev;
+ phydev = of_phy_find_device(lp->phy_node);
+@@ -845,6 +851,7 @@ static int xemaclite_mdio_setup(struct net_local *lp, struct device *dev)
+ "MDIO of the phy is not registered yet\n");
+ else
+ put_device(&phydev->mdio.dev);
++ of_node_put(np);
+ return 0;
+ }
+
+@@ -857,6 +864,7 @@ static int xemaclite_mdio_setup(struct net_local *lp, struct device *dev)
+ bus = mdiobus_alloc();
+ if (!bus) {
+ dev_err(dev, "Failed to allocate mdiobus\n");
++ of_node_put(np);
+ return -ENOMEM;
+ }
+
+@@ -869,6 +877,7 @@ static int xemaclite_mdio_setup(struct net_local *lp, struct device *dev)
+ bus->parent = dev;
+
+ rc = of_mdiobus_register(bus, np);
++ of_node_put(np);
+ if (rc) {
+ dev_err(dev, "Failed to register mdio bus.\n");
+ goto err_register;
+@@ -925,8 +934,6 @@ static int xemaclite_open(struct net_device *dev)
+ xemaclite_disable_interrupts(lp);
+
+ if (lp->phy_node) {
+- u32 bmcr;
+-
+ lp->phy_dev = of_phy_connect(lp->ndev, lp->phy_node,
+ xemaclite_adjust_link, 0,
+ PHY_INTERFACE_MODE_MII);
+@@ -937,19 +944,6 @@ static int xemaclite_open(struct net_device *dev)
+
+ /* EmacLite doesn't support giga-bit speeds */
+ phy_set_max_speed(lp->phy_dev, SPEED_100);
+-
+- /* Don't advertise 1000BASE-T Full/Half duplex speeds */
+- phy_write(lp->phy_dev, MII_CTRL1000, 0);
+-
+- /* Advertise only 10 and 100mbps full/half duplex speeds */
+- phy_write(lp->phy_dev, MII_ADVERTISE, ADVERTISE_ALL |
+- ADVERTISE_CSMA);
+-
+- /* Restart auto negotiation */
+- bmcr = phy_read(lp->phy_dev, MII_BMCR);
+- bmcr |= (BMCR_ANENABLE | BMCR_ANRESTART);
+- phy_write(lp->phy_dev, MII_BMCR, bmcr);
+-
+ phy_start(lp->phy_dev);
+ }
+
+@@ -1185,7 +1179,7 @@ static int xemaclite_of_probe(struct platform_device *ofdev)
+ if (rc) {
+ dev_err(dev,
+ "Cannot register network device, aborting\n");
+- goto error;
++ goto put_node;
+ }
+
+ dev_info(dev,
+@@ -1193,6 +1187,8 @@ static int xemaclite_of_probe(struct platform_device *ofdev)
+ (unsigned long __force)ndev->mem_start, lp->base_addr, ndev->irq);
+ return 0;
+
++put_node:
++ of_node_put(lp->phy_node);
+ error:
+ free_netdev(ndev);
+ return rc;
+diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c
+index 185c8a3986816..1d1808afd5295 100644
+--- a/drivers/net/fjes/fjes_main.c
++++ b/drivers/net/fjes/fjes_main.c
+@@ -1261,6 +1261,11 @@ static int fjes_probe(struct platform_device *plat_dev)
+ hw->hw_res.start = res->start;
+ hw->hw_res.size = resource_size(res);
+ hw->hw_res.irq = platform_get_irq(plat_dev, 0);
++ if (hw->hw_res.irq < 0) {
++ err = hw->hw_res.irq;
++ goto err_free_control_wq;
++ }
++
+ err = fjes_hw_init(&adapter->hw);
+ if (err)
+ goto err_free_control_wq;
+diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c
+index 6192244b304ab..36a9fbb704029 100644
+--- a/drivers/net/hamradio/6pack.c
++++ b/drivers/net/hamradio/6pack.c
+@@ -306,7 +306,6 @@ static void sp_setup(struct net_device *dev)
+ {
+ /* Finish setting up the DEVICE info. */
+ dev->netdev_ops = &sp_netdev_ops;
+- dev->needs_free_netdev = true;
+ dev->mtu = SIXP_MTU;
+ dev->hard_header_len = AX25_MAX_HEADER_LEN;
+ dev->header_ops = &ax25_header_ops;
+@@ -669,14 +668,16 @@ static void sixpack_close(struct tty_struct *tty)
+ */
+ netif_stop_queue(sp->dev);
+
++ unregister_netdev(sp->dev);
++
+ del_timer_sync(&sp->tx_t);
+ del_timer_sync(&sp->resync_t);
+
+- /* Free all 6pack frame buffers. */
++ /* Free all 6pack frame buffers after unreg. */
+ kfree(sp->rbuff);
+ kfree(sp->xbuff);
+
+- unregister_netdev(sp->dev);
++ free_netdev(sp->dev);
+ }
+
+ /* Perform I/O control on an active 6pack channel. */
+diff --git a/drivers/net/hamradio/mkiss.c b/drivers/net/hamradio/mkiss.c
+index 8666110bec555..763d435a9564c 100644
+--- a/drivers/net/hamradio/mkiss.c
++++ b/drivers/net/hamradio/mkiss.c
+@@ -31,6 +31,8 @@
+
+ #define AX_MTU 236
+
++/* some arch define END as assembly function ending, just undef it */
++#undef END
+ /* SLIP/KISS protocol characters. */
+ #define END 0300 /* indicates end of frame */
+ #define ESC 0333 /* indicates byte stuffing */
+@@ -792,13 +794,14 @@ static void mkiss_close(struct tty_struct *tty)
+ */
+ netif_stop_queue(ax->dev);
+
+- /* Free all AX25 frame buffers. */
++ unregister_netdev(ax->dev);
++
++ /* Free all AX25 frame buffers after unreg. */
+ kfree(ax->rbuff);
+ kfree(ax->xbuff);
+
+ ax->tty = NULL;
+
+- unregister_netdev(ax->dev);
+ free_netdev(ax->dev);
+ }
+
+diff --git a/drivers/net/hamradio/yam.c b/drivers/net/hamradio/yam.c
+index 6ddacbdb224ba..528d57a435394 100644
+--- a/drivers/net/hamradio/yam.c
++++ b/drivers/net/hamradio/yam.c
+@@ -950,9 +950,7 @@ static int yam_siocdevprivate(struct net_device *dev, struct ifreq *ifr, void __
+ ym = memdup_user(data, sizeof(struct yamdrv_ioctl_mcs));
+ if (IS_ERR(ym))
+ return PTR_ERR(ym);
+- if (ym->cmd != SIOCYAMSMCS)
+- return -EINVAL;
+- if (ym->bitrate > YAM_MAXBITRATE) {
++ if (ym->cmd != SIOCYAMSMCS || ym->bitrate > YAM_MAXBITRATE) {
+ kfree(ym);
+ return -EINVAL;
+ }
+diff --git a/drivers/net/hippi/rrunner.c b/drivers/net/hippi/rrunner.c
+index 7661dbb31162b..50e4bea46d673 100644
+--- a/drivers/net/hippi/rrunner.c
++++ b/drivers/net/hippi/rrunner.c
+@@ -1353,7 +1353,9 @@ static int rr_close(struct net_device *dev)
+
+ rrpriv->fw_running = 0;
+
++ spin_unlock_irqrestore(&rrpriv->lock, flags);
+ del_timer_sync(&rrpriv->timer);
++ spin_lock_irqsave(&rrpriv->lock, flags);
+
+ writel(0, &regs->TxPi);
+ writel(0, &regs->IpRxPi);
+diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
+index 382bebc2420df..bdfcf75f08277 100644
+--- a/drivers/net/hyperv/netvsc_drv.c
++++ b/drivers/net/hyperv/netvsc_drv.c
+@@ -1586,6 +1586,9 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
+ pcpu_sum = kvmalloc_array(num_possible_cpus(),
+ sizeof(struct netvsc_ethtool_pcpu_stats),
+ GFP_KERNEL);
++ if (!pcpu_sum)
++ return;
++
+ netvsc_get_pcpu_stats(dev, pcpu_sum);
+ for_each_present_cpu(cpu) {
+ struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];
+diff --git a/drivers/net/ieee802154/at86rf230.c b/drivers/net/ieee802154/at86rf230.c
+index 7d67f41387f55..4f5ef8a9a9a87 100644
+--- a/drivers/net/ieee802154/at86rf230.c
++++ b/drivers/net/ieee802154/at86rf230.c
+@@ -100,6 +100,7 @@ struct at86rf230_local {
+ unsigned long cal_timeout;
+ bool is_tx;
+ bool is_tx_from_off;
++ bool was_tx;
+ u8 tx_retry;
+ struct sk_buff *tx_skb;
+ struct at86rf230_state_change tx;
+@@ -343,7 +344,11 @@ at86rf230_async_error_recover_complete(void *context)
+ if (ctx->free)
+ kfree(ctx);
+
+- ieee802154_wake_queue(lp->hw);
++ if (lp->was_tx) {
++ lp->was_tx = 0;
++ dev_kfree_skb_any(lp->tx_skb);
++ ieee802154_wake_queue(lp->hw);
++ }
+ }
+
+ static void
+@@ -352,7 +357,11 @@ at86rf230_async_error_recover(void *context)
+ struct at86rf230_state_change *ctx = context;
+ struct at86rf230_local *lp = ctx->lp;
+
+- lp->is_tx = 0;
++ if (lp->is_tx) {
++ lp->was_tx = 1;
++ lp->is_tx = 0;
++ }
++
+ at86rf230_async_state_change(lp, ctx, STATE_RX_AACK_ON,
+ at86rf230_async_error_recover_complete);
+ }
+diff --git a/drivers/net/ieee802154/atusb.c b/drivers/net/ieee802154/atusb.c
+index 23ee0b14cbfa1..2f5e7b31032aa 100644
+--- a/drivers/net/ieee802154/atusb.c
++++ b/drivers/net/ieee802154/atusb.c
+@@ -93,7 +93,9 @@ static int atusb_control_msg(struct atusb *atusb, unsigned int pipe,
+
+ ret = usb_control_msg(usb_dev, pipe, request, requesttype,
+ value, index, data, size, timeout);
+- if (ret < 0) {
++ if (ret < size) {
++ ret = ret < 0 ? ret : -ENODATA;
++
+ atusb->err = ret;
+ dev_err(&usb_dev->dev,
+ "%s: req 0x%02x val 0x%x idx 0x%x, error %d\n",
+@@ -861,9 +863,9 @@ static int atusb_get_and_show_build(struct atusb *atusb)
+ if (!build)
+ return -ENOMEM;
+
+- ret = atusb_control_msg(atusb, usb_rcvctrlpipe(usb_dev, 0),
+- ATUSB_BUILD, ATUSB_REQ_FROM_DEV, 0, 0,
+- build, ATUSB_BUILD_SIZE, 1000);
++ /* We cannot call atusb_control_msg() here, since this request may read various length data */
++ ret = usb_control_msg(atusb->usb_dev, usb_rcvctrlpipe(usb_dev, 0), ATUSB_BUILD,
++ ATUSB_REQ_FROM_DEV, 0, 0, build, ATUSB_BUILD_SIZE, 1000);
+ if (ret >= 0) {
+ build[ret] = 0;
+ dev_info(&usb_dev->dev, "Firmware: build %s\n", build);
+diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c
+index 3a2824f24caa8..96592a20c61ff 100644
+--- a/drivers/net/ieee802154/ca8210.c
++++ b/drivers/net/ieee802154/ca8210.c
+@@ -1771,6 +1771,7 @@ static int ca8210_async_xmit_complete(
+ status
+ );
+ if (status != MAC_TRANSACTION_OVERFLOW) {
++ dev_kfree_skb_any(priv->tx_skb);
+ ieee802154_wake_queue(priv->hw);
+ return 0;
+ }
+@@ -2976,8 +2977,8 @@ static void ca8210_hw_setup(struct ieee802154_hw *ca8210_hw)
+ ca8210_hw->phy->cca.opt = NL802154_CCA_OPT_ENERGY_CARRIER_AND;
+ ca8210_hw->phy->cca_ed_level = -9800;
+ ca8210_hw->phy->symbol_duration = 16;
+- ca8210_hw->phy->lifs_period = 40;
+- ca8210_hw->phy->sifs_period = 12;
++ ca8210_hw->phy->lifs_period = 40 * ca8210_hw->phy->symbol_duration;
++ ca8210_hw->phy->sifs_period = 12 * ca8210_hw->phy->symbol_duration;
+ ca8210_hw->flags =
+ IEEE802154_HW_AFILT |
+ IEEE802154_HW_OMIT_CKSUM |
+diff --git a/drivers/net/ieee802154/mac802154_hwsim.c b/drivers/net/ieee802154/mac802154_hwsim.c
+index 8caa61ec718f5..36f1c5aa98fc6 100644
+--- a/drivers/net/ieee802154/mac802154_hwsim.c
++++ b/drivers/net/ieee802154/mac802154_hwsim.c
+@@ -786,6 +786,7 @@ static int hwsim_add_one(struct genl_info *info, struct device *dev,
+ goto err_pib;
+ }
+
++ pib->channel = 13;
+ rcu_assign_pointer(phy->pib, pib);
+ phy->idx = idx;
+ INIT_LIST_HEAD(&phy->edges);
+diff --git a/drivers/net/ieee802154/mcr20a.c b/drivers/net/ieee802154/mcr20a.c
+index 8dc04e2590b18..383231b854642 100644
+--- a/drivers/net/ieee802154/mcr20a.c
++++ b/drivers/net/ieee802154/mcr20a.c
+@@ -976,8 +976,8 @@ static void mcr20a_hw_setup(struct mcr20a_local *lp)
+ dev_dbg(printdev(lp), "%s\n", __func__);
+
+ phy->symbol_duration = 16;
+- phy->lifs_period = 40;
+- phy->sifs_period = 12;
++ phy->lifs_period = 40 * phy->symbol_duration;
++ phy->sifs_period = 12 * phy->symbol_duration;
+
+ hw->flags = IEEE802154_HW_TX_OMIT_CKSUM |
+ IEEE802154_HW_AFILT |
+diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
+index e9258a9f3702c..31bc02421dd4e 100644
+--- a/drivers/net/ifb.c
++++ b/drivers/net/ifb.c
+@@ -76,7 +76,9 @@ static void ifb_ri_tasklet(struct tasklet_struct *t)
+
+ while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
+ skb->redirected = 0;
++#ifdef CONFIG_NET_CLS_ACT
+ skb->tc_skip_classify = 1;
++#endif
+
+ u64_stats_update_begin(&txp->tsync);
+ txp->tx_packets++;
+diff --git a/drivers/net/ipa/Kconfig b/drivers/net/ipa/Kconfig
+index d037682fb7adb..6782c2cbf542f 100644
+--- a/drivers/net/ipa/Kconfig
++++ b/drivers/net/ipa/Kconfig
+@@ -2,7 +2,9 @@ config QCOM_IPA
+ tristate "Qualcomm IPA support"
+ depends on NET && QCOM_SMEM
+ depends on ARCH_QCOM || COMPILE_TEST
++ depends on INTERCONNECT
+ depends on QCOM_RPROC_COMMON || (QCOM_RPROC_COMMON=n && COMPILE_TEST)
++ depends on QCOM_AOSS_QMP || QCOM_AOSS_QMP=n
+ select QCOM_MDT_LOADER if ARCH_QCOM
+ select QCOM_SCM
+ select QCOM_QMI_HELPERS
+diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c
+index a2fcdb1abdb96..a734e5576729e 100644
+--- a/drivers/net/ipa/gsi.c
++++ b/drivers/net/ipa/gsi.c
+@@ -1370,9 +1370,10 @@ static void gsi_evt_ring_rx_update(struct gsi_evt_ring *evt_ring, u32 index)
+ struct gsi_event *event_done;
+ struct gsi_event *event;
+ struct gsi_trans *trans;
++ u32 trans_count = 0;
+ u32 byte_count = 0;
+- u32 old_index;
+ u32 event_avail;
++ u32 old_index;
+
+ trans_info = &channel->trans_info;
+
+@@ -1393,6 +1394,7 @@ static void gsi_evt_ring_rx_update(struct gsi_evt_ring *evt_ring, u32 index)
+ do {
+ trans->len = __le16_to_cpu(event->len);
+ byte_count += trans->len;
++ trans_count++;
+
+ /* Move on to the next event and transaction */
+ if (--event_avail)
+@@ -1404,7 +1406,7 @@ static void gsi_evt_ring_rx_update(struct gsi_evt_ring *evt_ring, u32 index)
+
+ /* We record RX bytes when they are received */
+ channel->byte_count += byte_count;
+- channel->trans_count++;
++ channel->trans_count += trans_count;
+ }
+
+ /* Initialize a ring, including allocating DMA memory for its entries */
+diff --git a/drivers/net/ipa/ipa_cmd.c b/drivers/net/ipa/ipa_cmd.c
+index cff51731195aa..d57472ea077f2 100644
+--- a/drivers/net/ipa/ipa_cmd.c
++++ b/drivers/net/ipa/ipa_cmd.c
+@@ -661,22 +661,6 @@ void ipa_cmd_pipeline_clear_wait(struct ipa *ipa)
+ wait_for_completion(&ipa->completion);
+ }
+
+-void ipa_cmd_pipeline_clear(struct ipa *ipa)
+-{
+- u32 count = ipa_cmd_pipeline_clear_count();
+- struct gsi_trans *trans;
+-
+- trans = ipa_cmd_trans_alloc(ipa, count);
+- if (trans) {
+- ipa_cmd_pipeline_clear_add(trans);
+- gsi_trans_commit_wait(trans);
+- ipa_cmd_pipeline_clear_wait(ipa);
+- } else {
+- dev_err(&ipa->pdev->dev,
+- "error allocating %u entry tag transaction\n", count);
+- }
+-}
+-
+ static struct ipa_cmd_info *
+ ipa_cmd_info_alloc(struct ipa_endpoint *endpoint, u32 tre_count)
+ {
+diff --git a/drivers/net/ipa/ipa_cmd.h b/drivers/net/ipa/ipa_cmd.h
+index 69cd085d427db..05ed7e42e1842 100644
+--- a/drivers/net/ipa/ipa_cmd.h
++++ b/drivers/net/ipa/ipa_cmd.h
+@@ -163,12 +163,6 @@ u32 ipa_cmd_pipeline_clear_count(void);
+ */
+ void ipa_cmd_pipeline_clear_wait(struct ipa *ipa);
+
+-/**
+- * ipa_cmd_pipeline_clear() - Clear the hardware pipeline
+- * @ipa: - IPA pointer
+- */
+-void ipa_cmd_pipeline_clear(struct ipa *ipa);
+-
+ /**
+ * ipa_cmd_trans_alloc() - Allocate a transaction for the command TX endpoint
+ * @ipa: IPA pointer
+diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c
+index 5528d97110d56..87e42db1b61e6 100644
+--- a/drivers/net/ipa/ipa_endpoint.c
++++ b/drivers/net/ipa/ipa_endpoint.c
+@@ -853,6 +853,7 @@ static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint,
+ u32 offset;
+ u32 val;
+
++ /* This should only be changed when HOL_BLOCK_EN is disabled */
+ offset = IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(endpoint_id);
+ val = hol_block_timer_val(ipa, microseconds);
+ iowrite32(val, ipa->reg_virt + offset);
+@@ -868,6 +869,9 @@ ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, bool enable)
+ val = enable ? HOL_BLOCK_EN_FMASK : 0;
+ offset = IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(endpoint_id);
+ iowrite32(val, endpoint->ipa->reg_virt + offset);
++ /* When enabling, the register must be written twice for IPA v4.5+ */
++ if (enable && endpoint->ipa->version >= IPA_VERSION_4_5)
++ iowrite32(val, endpoint->ipa->reg_virt + offset);
+ }
+
+ void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
+@@ -880,6 +884,7 @@ void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
+ if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM)
+ continue;
+
++ ipa_endpoint_init_hol_block_enable(endpoint, false);
+ ipa_endpoint_init_hol_block_timer(endpoint, 0);
+ ipa_endpoint_init_hol_block_enable(endpoint, true);
+ }
+@@ -1062,27 +1067,38 @@ static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint, bool add_one)
+ {
+ struct gsi *gsi;
+ u32 backlog;
++ int delta;
+
+- if (!endpoint->replenish_enabled) {
++ if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags)) {
+ if (add_one)
+ atomic_inc(&endpoint->replenish_saved);
+ return;
+ }
+
++ /* If already active, just update the backlog */
++ if (test_and_set_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags)) {
++ if (add_one)
++ atomic_inc(&endpoint->replenish_backlog);
++ return;
++ }
++
+ while (atomic_dec_not_zero(&endpoint->replenish_backlog))
+ if (ipa_endpoint_replenish_one(endpoint))
+ goto try_again_later;
++
++ clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
++
+ if (add_one)
+ atomic_inc(&endpoint->replenish_backlog);
+
+ return;
+
+ try_again_later:
+- /* The last one didn't succeed, so fix the backlog */
+- backlog = atomic_inc_return(&endpoint->replenish_backlog);
++ clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
+
+- if (add_one)
+- atomic_inc(&endpoint->replenish_backlog);
++ /* The last one didn't succeed, so fix the backlog */
++ delta = add_one ? 2 : 1;
++ backlog = atomic_add_return(delta, &endpoint->replenish_backlog);
+
+ /* Whenever a receive buffer transaction completes we'll try to
+ * replenish again. It's unlikely, but if we fail to supply even
+@@ -1102,7 +1118,7 @@ static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint)
+ u32 max_backlog;
+ u32 saved;
+
+- endpoint->replenish_enabled = true;
++ set_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
+ while ((saved = atomic_xchg(&endpoint->replenish_saved, 0)))
+ atomic_add(saved, &endpoint->replenish_backlog);
+
+@@ -1116,7 +1132,7 @@ static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint)
+ {
+ u32 backlog;
+
+- endpoint->replenish_enabled = false;
++ clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
+ while ((backlog = atomic_xchg(&endpoint->replenish_backlog, 0)))
+ atomic_add(backlog, &endpoint->replenish_saved);
+ }
+@@ -1631,8 +1647,6 @@ void ipa_endpoint_suspend(struct ipa *ipa)
+ if (ipa->modem_netdev)
+ ipa_modem_suspend(ipa->modem_netdev);
+
+- ipa_cmd_pipeline_clear(ipa);
+-
+ ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
+ ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
+ }
+@@ -1663,7 +1677,8 @@ static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint)
+ /* RX transactions require a single TRE, so the maximum
+ * backlog is the same as the maximum outstanding TREs.
+ */
+- endpoint->replenish_enabled = false;
++ clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
++ clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
+ atomic_set(&endpoint->replenish_saved,
+ gsi_channel_tre_max(gsi, endpoint->channel_id));
+ atomic_set(&endpoint->replenish_backlog, 0);
+diff --git a/drivers/net/ipa/ipa_endpoint.h b/drivers/net/ipa/ipa_endpoint.h
+index 0a859d10312dc..0313cdc607de3 100644
+--- a/drivers/net/ipa/ipa_endpoint.h
++++ b/drivers/net/ipa/ipa_endpoint.h
+@@ -40,6 +40,19 @@ enum ipa_endpoint_name {
+
+ #define IPA_ENDPOINT_MAX 32 /* Max supported by driver */
+
++/**
++ * enum ipa_replenish_flag: RX buffer replenish flags
++ *
++ * @IPA_REPLENISH_ENABLED: Whether receive buffer replenishing is enabled
++ * @IPA_REPLENISH_ACTIVE: Whether replenishing is underway
++ * @IPA_REPLENISH_COUNT: Number of defined replenish flags
++ */
++enum ipa_replenish_flag {
++ IPA_REPLENISH_ENABLED,
++ IPA_REPLENISH_ACTIVE,
++ IPA_REPLENISH_COUNT, /* Number of flags (must be last) */
++};
++
+ /**
+ * struct ipa_endpoint - IPA endpoint information
+ * @ipa: IPA pointer
+@@ -51,7 +64,7 @@ enum ipa_endpoint_name {
+ * @trans_tre_max: Maximum number of TRE descriptors per transaction
+ * @evt_ring_id: GSI event ring used by the endpoint
+ * @netdev: Network device pointer, if endpoint uses one
+- * @replenish_enabled: Whether receive buffer replenishing is enabled
++ * @replenish_flags: Replenishing state flags
+ * @replenish_ready: Number of replenish transactions without doorbell
+ * @replenish_saved: Replenish requests held while disabled
+ * @replenish_backlog: Number of buffers needed to fill hardware queue
+@@ -72,7 +85,7 @@ struct ipa_endpoint {
+ struct net_device *netdev;
+
+ /* Receive buffer replenishing for RX endpoints */
+- bool replenish_enabled;
++ DECLARE_BITMAP(replenish_flags, IPA_REPLENISH_COUNT);
+ u32 replenish_ready;
+ atomic_t replenish_saved;
+ atomic_t replenish_backlog;
+diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c
+index cdfa98a76e1f4..a448ec198bee1 100644
+--- a/drivers/net/ipa/ipa_main.c
++++ b/drivers/net/ipa/ipa_main.c
+@@ -28,6 +28,7 @@
+ #include "ipa_reg.h"
+ #include "ipa_mem.h"
+ #include "ipa_table.h"
++#include "ipa_smp2p.h"
+ #include "ipa_modem.h"
+ #include "ipa_uc.h"
+ #include "ipa_interrupt.h"
+@@ -801,6 +802,11 @@ static int ipa_remove(struct platform_device *pdev)
+ struct device *dev = &pdev->dev;
+ int ret;
+
++ /* Prevent the modem from triggering a call to ipa_setup(). This
++ * also ensures a modem-initiated setup that's underway completes.
++ */
++ ipa_smp2p_irq_disable_setup(ipa);
++
+ ret = pm_runtime_get_sync(dev);
+ if (WARN_ON(ret < 0))
+ goto out_power_put;
+diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c
+index ad116bcc0580e..d0ab4d70c303b 100644
+--- a/drivers/net/ipa/ipa_modem.c
++++ b/drivers/net/ipa/ipa_modem.c
+@@ -339,9 +339,6 @@ int ipa_modem_stop(struct ipa *ipa)
+ if (state != IPA_MODEM_STATE_RUNNING)
+ return -EBUSY;
+
+- /* Prevent the modem from triggering a call to ipa_setup() */
+- ipa_smp2p_disable(ipa);
+-
+ /* Clean up the netdev and endpoints if it was started */
+ if (netdev) {
+ struct ipa_priv *priv = netdev_priv(netdev);
+@@ -369,6 +366,9 @@ static void ipa_modem_crashed(struct ipa *ipa)
+ struct device *dev = &ipa->pdev->dev;
+ int ret;
+
++ /* Prevent the modem from triggering a call to ipa_setup() */
++ ipa_smp2p_irq_disable_setup(ipa);
++
+ ret = pm_runtime_get_sync(dev);
+ if (ret < 0) {
+ dev_err(dev, "error %d getting power to handle crash\n", ret);
+diff --git a/drivers/net/ipa/ipa_power.c b/drivers/net/ipa/ipa_power.c
+index b1c6c0fcb654f..f2989aac47a62 100644
+--- a/drivers/net/ipa/ipa_power.c
++++ b/drivers/net/ipa/ipa_power.c
+@@ -11,6 +11,8 @@
+ #include <linux/pm_runtime.h>
+ #include <linux/bitops.h>
+
++#include "linux/soc/qcom/qcom_aoss.h"
++
+ #include "ipa.h"
+ #include "ipa_power.h"
+ #include "ipa_endpoint.h"
+@@ -64,6 +66,7 @@ enum ipa_power_flag {
+ * struct ipa_power - IPA power management information
+ * @dev: IPA device pointer
+ * @core: IPA core clock
++ * @qmp: QMP handle for AOSS communication
+ * @spinlock: Protects modem TX queue enable/disable
+ * @flags: Boolean state flags
+ * @interconnect_count: Number of elements in interconnect[]
+@@ -72,6 +75,7 @@ enum ipa_power_flag {
+ struct ipa_power {
+ struct device *dev;
+ struct clk *core;
++ struct qmp *qmp;
+ spinlock_t spinlock; /* used with STOPPED/STARTED power flags */
+ DECLARE_BITMAP(flags, IPA_POWER_FLAG_COUNT);
+ u32 interconnect_count;
+@@ -382,6 +386,47 @@ void ipa_power_modem_queue_active(struct ipa *ipa)
+ clear_bit(IPA_POWER_FLAG_STARTED, ipa->power->flags);
+ }
+
++static int ipa_power_retention_init(struct ipa_power *power)
++{
++ struct qmp *qmp = qmp_get(power->dev);
++
++ if (IS_ERR(qmp)) {
++ if (PTR_ERR(qmp) == -EPROBE_DEFER)
++ return -EPROBE_DEFER;
++
++ /* We assume any other error means it's not defined/needed */
++ qmp = NULL;
++ }
++ power->qmp = qmp;
++
++ return 0;
++}
++
++static void ipa_power_retention_exit(struct ipa_power *power)
++{
++ qmp_put(power->qmp);
++ power->qmp = NULL;
++}
++
++/* Control register retention on power collapse */
++void ipa_power_retention(struct ipa *ipa, bool enable)
++{
++ static const char fmt[] = "{ class: bcm, res: ipa_pc, val: %c }";
++ struct ipa_power *power = ipa->power;
++ char buf[36]; /* Exactly enough for fmt[]; size a multiple of 4 */
++ int ret;
++
++ if (!power->qmp)
++ return; /* Not needed on this platform */
++
++ (void)snprintf(buf, sizeof(buf), fmt, enable ? '1' : '0');
++
++ ret = qmp_send(power->qmp, buf, sizeof(buf));
++ if (ret)
++ dev_err(power->dev, "error %d sending QMP %sable request\n",
++ ret, enable ? "en" : "dis");
++}
++
+ int ipa_power_setup(struct ipa *ipa)
+ {
+ int ret;
+@@ -438,12 +483,18 @@ ipa_power_init(struct device *dev, const struct ipa_power_data *data)
+ if (ret)
+ goto err_kfree;
+
++ ret = ipa_power_retention_init(power);
++ if (ret)
++ goto err_interconnect_exit;
++
+ pm_runtime_set_autosuspend_delay(dev, IPA_AUTOSUSPEND_DELAY);
+ pm_runtime_use_autosuspend(dev);
+ pm_runtime_enable(dev);
+
+ return power;
+
++err_interconnect_exit:
++ ipa_interconnect_exit(power);
+ err_kfree:
+ kfree(power);
+ err_clk_put:
+@@ -460,6 +511,7 @@ void ipa_power_exit(struct ipa_power *power)
+
+ pm_runtime_disable(dev);
+ pm_runtime_dont_use_autosuspend(dev);
++ ipa_power_retention_exit(power);
+ ipa_interconnect_exit(power);
+ kfree(power);
+ clk_put(clk);
+diff --git a/drivers/net/ipa/ipa_power.h b/drivers/net/ipa/ipa_power.h
+index 2151805d7fbb0..6f84f057a2095 100644
+--- a/drivers/net/ipa/ipa_power.h
++++ b/drivers/net/ipa/ipa_power.h
+@@ -40,6 +40,13 @@ void ipa_power_modem_queue_wake(struct ipa *ipa);
+ */
+ void ipa_power_modem_queue_active(struct ipa *ipa);
+
++/**
++ * ipa_power_retention() - Control register retention on power collapse
++ * @ipa: IPA pointer
++ * @enable: Whether retention should be enabled or disabled
++ */
++void ipa_power_retention(struct ipa *ipa, bool enable);
++
+ /**
+ * ipa_power_setup() - Set up IPA power management
+ * @ipa: IPA pointer
+diff --git a/drivers/net/ipa/ipa_resource.c b/drivers/net/ipa/ipa_resource.c
+index e3da95d694099..06cec71993823 100644
+--- a/drivers/net/ipa/ipa_resource.c
++++ b/drivers/net/ipa/ipa_resource.c
+@@ -52,7 +52,7 @@ static bool ipa_resource_limits_valid(struct ipa *ipa,
+ return false;
+ }
+
+- group_count = data->rsrc_group_src_count;
++ group_count = data->rsrc_group_dst_count;
+ if (!group_count || group_count > IPA_RESOURCE_GROUP_MAX)
+ return false;
+
+diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c
+index df7639c39d716..2112336120391 100644
+--- a/drivers/net/ipa/ipa_smp2p.c
++++ b/drivers/net/ipa/ipa_smp2p.c
+@@ -53,7 +53,7 @@
+ * @setup_ready_irq: IPA interrupt triggered by modem to signal GSI ready
+ * @power_on: Whether IPA power is on
+ * @notified: Whether modem has been notified of power state
+- * @disabled: Whether setup ready interrupt handling is disabled
++ * @setup_disabled: Whether setup ready interrupt handler is disabled
+ * @mutex: Mutex protecting ready-interrupt/shutdown interlock
+ * @panic_notifier: Panic notifier structure
+ */
+@@ -67,7 +67,7 @@ struct ipa_smp2p {
+ u32 setup_ready_irq;
+ bool power_on;
+ bool notified;
+- bool disabled;
++ bool setup_disabled;
+ struct mutex mutex;
+ struct notifier_block panic_notifier;
+ };
+@@ -155,11 +155,9 @@ static irqreturn_t ipa_smp2p_modem_setup_ready_isr(int irq, void *dev_id)
+ struct device *dev;
+ int ret;
+
+- mutex_lock(&smp2p->mutex);
+-
+- if (smp2p->disabled)
+- goto out_mutex_unlock;
+- smp2p->disabled = true; /* If any others arrive, ignore them */
++ /* Ignore any (spurious) interrupts received after the first */
++ if (smp2p->ipa->setup_complete)
++ return IRQ_HANDLED;
+
+ /* Power needs to be active for setup */
+ dev = &smp2p->ipa->pdev->dev;
+@@ -176,8 +174,6 @@ static irqreturn_t ipa_smp2p_modem_setup_ready_isr(int irq, void *dev_id)
+ out_power_put:
+ pm_runtime_mark_last_busy(dev);
+ (void)pm_runtime_put_autosuspend(dev);
+-out_mutex_unlock:
+- mutex_unlock(&smp2p->mutex);
+
+ return IRQ_HANDLED;
+ }
+@@ -313,7 +309,7 @@ void ipa_smp2p_exit(struct ipa *ipa)
+ kfree(smp2p);
+ }
+
+-void ipa_smp2p_disable(struct ipa *ipa)
++void ipa_smp2p_irq_disable_setup(struct ipa *ipa)
+ {
+ struct ipa_smp2p *smp2p = ipa->smp2p;
+
+@@ -322,7 +318,10 @@ void ipa_smp2p_disable(struct ipa *ipa)
+
+ mutex_lock(&smp2p->mutex);
+
+- smp2p->disabled = true;
++ if (!smp2p->setup_disabled) {
++ disable_irq(smp2p->setup_ready_irq);
++ smp2p->setup_disabled = true;
++ }
+
+ mutex_unlock(&smp2p->mutex);
+ }
+diff --git a/drivers/net/ipa/ipa_smp2p.h b/drivers/net/ipa/ipa_smp2p.h
+index 99a9567896388..59cee31a73836 100644
+--- a/drivers/net/ipa/ipa_smp2p.h
++++ b/drivers/net/ipa/ipa_smp2p.h
+@@ -27,13 +27,12 @@ int ipa_smp2p_init(struct ipa *ipa, bool modem_init);
+ void ipa_smp2p_exit(struct ipa *ipa);
+
+ /**
+- * ipa_smp2p_disable() - Prevent "ipa-setup-ready" interrupt handling
++ * ipa_smp2p_irq_disable_setup() - Disable the "setup ready" interrupt
+ * @ipa: IPA pointer
+ *
+- * Prevent handling of the "setup ready" interrupt from the modem.
+- * This is used before initiating shutdown of the driver.
++ * Disable the "ipa-setup-ready" interrupt from the modem.
+ */
+-void ipa_smp2p_disable(struct ipa *ipa);
++void ipa_smp2p_irq_disable_setup(struct ipa *ipa);
+
+ /**
+ * ipa_smp2p_notify_reset() - Reset modem notification state
+diff --git a/drivers/net/ipa/ipa_uc.c b/drivers/net/ipa/ipa_uc.c
+index 856e55a080a7f..fe11910518d95 100644
+--- a/drivers/net/ipa/ipa_uc.c
++++ b/drivers/net/ipa/ipa_uc.c
+@@ -11,6 +11,7 @@
+
+ #include "ipa.h"
+ #include "ipa_uc.h"
++#include "ipa_power.h"
+
+ /**
+ * DOC: The IPA embedded microcontroller
+@@ -154,6 +155,7 @@ static void ipa_uc_response_hdlr(struct ipa *ipa, enum ipa_irq_id irq_id)
+ case IPA_UC_RESPONSE_INIT_COMPLETED:
+ if (ipa->uc_powered) {
+ ipa->uc_loaded = true;
++ ipa_power_retention(ipa, true);
+ pm_runtime_mark_last_busy(dev);
+ (void)pm_runtime_put_autosuspend(dev);
+ ipa->uc_powered = false;
+@@ -184,6 +186,9 @@ void ipa_uc_deconfig(struct ipa *ipa)
+
+ ipa_interrupt_remove(ipa->interrupt, IPA_IRQ_UC_1);
+ ipa_interrupt_remove(ipa->interrupt, IPA_IRQ_UC_0);
++ if (ipa->uc_loaded)
++ ipa_power_retention(ipa, false);
++
+ if (!ipa->uc_powered)
+ return;
+
+diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
+index 93dc48b9b4f24..e53b40359fd10 100644
+--- a/drivers/net/macsec.c
++++ b/drivers/net/macsec.c
+@@ -3870,6 +3870,18 @@ static void macsec_common_dellink(struct net_device *dev, struct list_head *head
+ struct macsec_dev *macsec = macsec_priv(dev);
+ struct net_device *real_dev = macsec->real_dev;
+
++ /* If h/w offloading is available, propagate to the device */
++ if (macsec_is_offloaded(macsec)) {
++ const struct macsec_ops *ops;
++ struct macsec_context ctx;
++
++ ops = macsec_get_ops(netdev_priv(dev), &ctx);
++ if (ops) {
++ ctx.secy = &macsec->secy;
++ macsec_offload(ops->mdo_del_secy, &ctx);
++ }
++ }
++
+ unregister_netdevice_queue(dev, head);
+ list_del_rcu(&macsec->secys);
+ macsec_del_dev(macsec);
+@@ -3884,18 +3896,6 @@ static void macsec_dellink(struct net_device *dev, struct list_head *head)
+ struct net_device *real_dev = macsec->real_dev;
+ struct macsec_rxh_data *rxd = macsec_data_rtnl(real_dev);
+
+- /* If h/w offloading is available, propagate to the device */
+- if (macsec_is_offloaded(macsec)) {
+- const struct macsec_ops *ops;
+- struct macsec_context ctx;
+-
+- ops = macsec_get_ops(netdev_priv(dev), &ctx);
+- if (ops) {
+- ctx.secy = &macsec->secy;
+- macsec_offload(ops->mdo_del_secy, &ctx);
+- }
+- }
+-
+ macsec_common_dellink(dev, head);
+
+ if (list_empty(&rxd->secys)) {
+@@ -4018,6 +4018,15 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
+ !macsec_check_offload(macsec->offload, macsec))
+ return -EOPNOTSUPP;
+
++ /* send_sci must be set to true when transmit sci explicitly is set */
++ if ((data && data[IFLA_MACSEC_SCI]) &&
++ (data && data[IFLA_MACSEC_INC_SCI])) {
++ u8 send_sci = !!nla_get_u8(data[IFLA_MACSEC_INC_SCI]);
++
++ if (!send_sci)
++ return -EINVAL;
++ }
++
+ if (data && data[IFLA_MACSEC_ICV_LEN])
+ icv_len = nla_get_u8(data[IFLA_MACSEC_ICV_LEN]);
+ mtu = real_dev->mtu - icv_len - macsec_extra_len(true);
+diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
+index 35f46ad040b0d..a9a515cf5a460 100644
+--- a/drivers/net/macvlan.c
++++ b/drivers/net/macvlan.c
+@@ -460,8 +460,10 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
+ return RX_HANDLER_CONSUMED;
+ *pskb = skb;
+ eth = eth_hdr(skb);
+- if (macvlan_forward_source(skb, port, eth->h_source))
++ if (macvlan_forward_source(skb, port, eth->h_source)) {
++ kfree_skb(skb);
+ return RX_HANDLER_CONSUMED;
++ }
+ src = macvlan_hash_lookup(port, eth->h_source);
+ if (src && src->mode != MACVLAN_MODE_VEPA &&
+ src->mode != MACVLAN_MODE_BRIDGE) {
+@@ -480,8 +482,10 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
+ return RX_HANDLER_PASS;
+ }
+
+- if (macvlan_forward_source(skb, port, eth->h_source))
++ if (macvlan_forward_source(skb, port, eth->h_source)) {
++ kfree_skb(skb);
+ return RX_HANDLER_CONSUMED;
++ }
+ if (macvlan_passthru(port))
+ vlan = list_first_or_null_rcu(&port->vlans,
+ struct macvlan_dev, list);
+diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
+index 694e2f5dbbe59..39801c31e5071 100644
+--- a/drivers/net/macvtap.c
++++ b/drivers/net/macvtap.c
+@@ -133,11 +133,17 @@ static void macvtap_setup(struct net_device *dev)
+ dev->tx_queue_len = TUN_READQ_SIZE;
+ }
+
++static struct net *macvtap_link_net(const struct net_device *dev)
++{
++ return dev_net(macvlan_dev_real_dev(dev));
++}
++
+ static struct rtnl_link_ops macvtap_link_ops __read_mostly = {
+ .kind = "macvtap",
+ .setup = macvtap_setup,
+ .newlink = macvtap_newlink,
+ .dellink = macvtap_dellink,
++ .get_link_net = macvtap_link_net,
+ .priv_size = sizeof(struct macvtap_dev),
+ };
+
+diff --git a/drivers/net/mdio/fwnode_mdio.c b/drivers/net/mdio/fwnode_mdio.c
+index 1becb1a731f67..1c1584fca6327 100644
+--- a/drivers/net/mdio/fwnode_mdio.c
++++ b/drivers/net/mdio/fwnode_mdio.c
+@@ -43,6 +43,11 @@ int fwnode_mdiobus_phy_device_register(struct mii_bus *mdio,
+ int rc;
+
+ rc = fwnode_irq_get(child, 0);
++ /* Don't wait forever if the IRQ provider doesn't become available,
++ * just fall back to poll mode
++ */
++ if (rc == -EPROBE_DEFER)
++ rc = driver_deferred_probe_check_state(&phy->mdio.dev);
+ if (rc == -EPROBE_DEFER)
+ return rc;
+
+diff --git a/drivers/net/mdio/mdio-aspeed.c b/drivers/net/mdio/mdio-aspeed.c
+index cad820568f751..e2273588c75b6 100644
+--- a/drivers/net/mdio/mdio-aspeed.c
++++ b/drivers/net/mdio/mdio-aspeed.c
+@@ -61,6 +61,13 @@ static int aspeed_mdio_read(struct mii_bus *bus, int addr, int regnum)
+
+ iowrite32(ctrl, ctx->base + ASPEED_MDIO_CTRL);
+
++ rc = readl_poll_timeout(ctx->base + ASPEED_MDIO_CTRL, ctrl,
++ !(ctrl & ASPEED_MDIO_CTRL_FIRE),
++ ASPEED_MDIO_INTERVAL_US,
++ ASPEED_MDIO_TIMEOUT_US);
++ if (rc < 0)
++ return rc;
++
+ rc = readl_poll_timeout(ctx->base + ASPEED_MDIO_DATA, data,
+ data & ASPEED_MDIO_DATA_IDLE,
+ ASPEED_MDIO_INTERVAL_US,
+@@ -141,6 +148,7 @@ static const struct of_device_id aspeed_mdio_of_match[] = {
+ { .compatible = "aspeed,ast2600-mdio", },
+ { },
+ };
++MODULE_DEVICE_TABLE(of, aspeed_mdio_of_match);
+
+ static struct platform_driver aspeed_mdio_driver = {
+ .driver = {
+diff --git a/drivers/net/mdio/mdio-ipq4019.c b/drivers/net/mdio/mdio-ipq4019.c
+index 5f4cd24a0241d..4eba5a91075c0 100644
+--- a/drivers/net/mdio/mdio-ipq4019.c
++++ b/drivers/net/mdio/mdio-ipq4019.c
+@@ -200,7 +200,11 @@ static int ipq_mdio_reset(struct mii_bus *bus)
+ if (ret)
+ return ret;
+
+- return clk_prepare_enable(priv->mdio_clk);
++ ret = clk_prepare_enable(priv->mdio_clk);
++ if (ret == 0)
++ mdelay(10);
++
++ return ret;
+ }
+
+ static int ipq4019_mdio_probe(struct platform_device *pdev)
+diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c
+index 17f98f609ec82..5070ca2f2637a 100644
+--- a/drivers/net/mdio/mdio-mscc-miim.c
++++ b/drivers/net/mdio/mdio-mscc-miim.c
+@@ -76,6 +76,9 @@ static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum)
+ u32 val;
+ int ret;
+
++ if (regnum & MII_ADDR_C45)
++ return -EOPNOTSUPP;
++
+ ret = mscc_miim_wait_pending(bus);
+ if (ret)
+ goto out;
+@@ -105,6 +108,9 @@ static int mscc_miim_write(struct mii_bus *bus, int mii_id,
+ struct mscc_miim_dev *miim = bus->priv;
+ int ret;
+
++ if (regnum & MII_ADDR_C45)
++ return -EOPNOTSUPP;
++
+ ret = mscc_miim_wait_pending(bus);
+ if (ret < 0)
+ goto out;
+diff --git a/drivers/net/mdio/mdio-mux-bcm6368.c b/drivers/net/mdio/mdio-mux-bcm6368.c
+index 6dcbf987d61b5..8b444a8eb6b55 100644
+--- a/drivers/net/mdio/mdio-mux-bcm6368.c
++++ b/drivers/net/mdio/mdio-mux-bcm6368.c
+@@ -115,7 +115,7 @@ static int bcm6368_mdiomux_probe(struct platform_device *pdev)
+ md->mii_bus = devm_mdiobus_alloc(&pdev->dev);
+ if (!md->mii_bus) {
+ dev_err(&pdev->dev, "mdiomux bus alloc failed\n");
+- return ENOMEM;
++ return -ENOMEM;
+ }
+
+ bus = md->mii_bus;
+diff --git a/drivers/net/netdevsim/bpf.c b/drivers/net/netdevsim/bpf.c
+index 90aafb56f1409..a438202129323 100644
+--- a/drivers/net/netdevsim/bpf.c
++++ b/drivers/net/netdevsim/bpf.c
+@@ -514,6 +514,7 @@ nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
+ goto err_free;
+ key = nmap->entry[i].key;
+ *key = i;
++ memset(nmap->entry[i].value, 0, offmap->map.value_size);
+ }
+ }
+
+diff --git a/drivers/net/netdevsim/ethtool.c b/drivers/net/netdevsim/ethtool.c
+index b03a0513eb7e7..2e7c1cc16cb93 100644
+--- a/drivers/net/netdevsim/ethtool.c
++++ b/drivers/net/netdevsim/ethtool.c
+@@ -77,7 +77,10 @@ static int nsim_set_ringparam(struct net_device *dev,
+ {
+ struct netdevsim *ns = netdev_priv(dev);
+
+- memcpy(&ns->ethtool.ring, ring, sizeof(ns->ethtool.ring));
++ ns->ethtool.ring.rx_pending = ring->rx_pending;
++ ns->ethtool.ring.rx_jumbo_pending = ring->rx_jumbo_pending;
++ ns->ethtool.ring.rx_mini_pending = ring->rx_mini_pending;
++ ns->ethtool.ring.tx_pending = ring->tx_pending;
+ return 0;
+ }
+
+diff --git a/drivers/net/netdevsim/fib.c b/drivers/net/netdevsim/fib.c
+index 4300261e2f9e7..378ee779061c3 100644
+--- a/drivers/net/netdevsim/fib.c
++++ b/drivers/net/netdevsim/fib.c
+@@ -623,14 +623,14 @@ static int nsim_fib6_rt_append(struct nsim_fib_data *data,
+ if (err)
+ goto err_fib6_rt_nh_del;
+
+- fib6_event->rt_arr[i]->trap = true;
++ WRITE_ONCE(fib6_event->rt_arr[i]->trap, true);
+ }
+
+ return 0;
+
+ err_fib6_rt_nh_del:
+ for (i--; i >= 0; i--) {
+- fib6_event->rt_arr[i]->trap = false;
++ WRITE_ONCE(fib6_event->rt_arr[i]->trap, false);
+ nsim_fib6_rt_nh_del(fib6_rt, fib6_event->rt_arr[i]);
+ }
+ return err;
+diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
+index bdac087058b26..5ae39d236b307 100644
+--- a/drivers/net/phy/at803x.c
++++ b/drivers/net/phy/at803x.c
+@@ -666,25 +666,7 @@ static int at803x_probe(struct phy_device *phydev)
+ return ret;
+ }
+
+- /* Some bootloaders leave the fiber page selected.
+- * Switch to the copper page, as otherwise we read
+- * the PHY capabilities from the fiber side.
+- */
+- if (phydev->drv->phy_id == ATH8031_PHY_ID) {
+- phy_lock_mdio_bus(phydev);
+- ret = at803x_write_page(phydev, AT803X_PAGE_COPPER);
+- phy_unlock_mdio_bus(phydev);
+- if (ret)
+- goto err;
+- }
+-
+ return 0;
+-
+-err:
+- if (priv->vddio)
+- regulator_disable(priv->vddio);
+-
+- return ret;
+ }
+
+ static void at803x_remove(struct phy_device *phydev)
+@@ -785,6 +767,22 @@ static int at803x_config_init(struct phy_device *phydev)
+ {
+ int ret;
+
++ if (phydev->drv->phy_id == ATH8031_PHY_ID) {
++ /* Some bootloaders leave the fiber page selected.
++ * Switch to the copper page, as otherwise we read
++ * the PHY capabilities from the fiber side.
++ */
++ phy_lock_mdio_bus(phydev);
++ ret = at803x_write_page(phydev, AT803X_PAGE_COPPER);
++ phy_unlock_mdio_bus(phydev);
++ if (ret)
++ return ret;
++
++ ret = at8031_pll_config(phydev);
++ if (ret < 0)
++ return ret;
++ }
++
+ /* The RX and TX delay default is:
+ * after HW reset: RX delay enabled and TX delay disabled
+ * after SW reset: RX delay enabled, while TX delay retains the
+@@ -814,12 +812,6 @@ static int at803x_config_init(struct phy_device *phydev)
+ if (ret < 0)
+ return ret;
+
+- if (phydev->drv->phy_id == ATH8031_PHY_ID) {
+- ret = at8031_pll_config(phydev);
+- if (ret < 0)
+- return ret;
+- }
+-
+ /* Ar803x extended next page bit is enabled by default. Cisco
+ * multigig switches read this bit and attempt to negotiate 10Gbps
+ * rates even if the next page bit is disabled. This is incorrect
+diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
+index 83aea5c5cd03c..b330efb98209b 100644
+--- a/drivers/net/phy/broadcom.c
++++ b/drivers/net/phy/broadcom.c
+@@ -11,6 +11,7 @@
+ */
+
+ #include "bcm-phy-lib.h"
++#include <linux/delay.h>
+ #include <linux/module.h>
+ #include <linux/phy.h>
+ #include <linux/brcmphy.h>
+@@ -553,6 +554,26 @@ static int brcm_fet_config_init(struct phy_device *phydev)
+ if (err < 0)
+ return err;
+
++ /* The datasheet indicates the PHY needs up to 1us to complete a reset,
++ * build some slack here.
++ */
++ usleep_range(1000, 2000);
++
++ /* The PHY requires 65 MDC clock cycles to complete a write operation
++ * and turnaround the line properly.
++ *
++ * We ignore -EIO here as the MDIO controller (e.g.: mdio-bcm-unimac)
++ * may flag the lack of turn-around as a read failure. This is
++ * particularly true with this combination since the MDIO controller
++ * only used 64 MDC cycles. This is not a critical failure in this
++ * specific case and it has no functional impact otherwise, so we let
++ * that one go through. If there is a genuine bus error, the next read
++ * of MII_BRCM_FET_INTREG will error out.
++ */
++ err = phy_read(phydev, MII_BMCR);
++ if (err < 0 && err != -EIO)
++ return err;
++
+ reg = phy_read(phydev, MII_BRCM_FET_INTREG);
+ if (reg < 0)
+ return reg;
+@@ -768,6 +789,7 @@ static struct phy_driver broadcom_drivers[] = {
+ .phy_id_mask = 0xfffffff0,
+ .name = "Broadcom BCM54616S",
+ /* PHY_GBIT_FEATURES */
++ .soft_reset = genphy_soft_reset,
+ .config_init = bcm54xx_config_init,
+ .config_aneg = bcm54616s_config_aneg,
+ .config_intr = bcm_phy_config_intr,
+diff --git a/drivers/net/phy/dp83822.c b/drivers/net/phy/dp83822.c
+index 211b5476a6f51..ce17b2af3218f 100644
+--- a/drivers/net/phy/dp83822.c
++++ b/drivers/net/phy/dp83822.c
+@@ -274,7 +274,7 @@ static int dp83822_config_intr(struct phy_device *phydev)
+ if (err < 0)
+ return err;
+
+- err = phy_write(phydev, MII_DP83822_MISR1, 0);
++ err = phy_write(phydev, MII_DP83822_MISR2, 0);
+ if (err < 0)
+ return err;
+
+diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
+index 4fcfca4e17021..4d726ee03ce20 100644
+--- a/drivers/net/phy/marvell.c
++++ b/drivers/net/phy/marvell.c
+@@ -189,6 +189,8 @@
+ #define MII_88E1510_GEN_CTRL_REG_1_MODE_RGMII_SGMII 0x4
+ #define MII_88E1510_GEN_CTRL_REG_1_RESET 0x8000 /* Soft reset */
+
++#define MII_88E1510_MSCR_2 0x15
++
+ #define MII_VCT5_TX_RX_MDI0_COUPLING 0x10
+ #define MII_VCT5_TX_RX_MDI1_COUPLING 0x11
+ #define MII_VCT5_TX_RX_MDI2_COUPLING 0x12
+@@ -551,9 +553,9 @@ static int m88e1121_config_aneg_rgmii_delays(struct phy_device *phydev)
+ else
+ mscr = 0;
+
+- return phy_modify_paged(phydev, MII_MARVELL_MSCR_PAGE,
+- MII_88E1121_PHY_MSCR_REG,
+- MII_88E1121_PHY_MSCR_DELAY_MASK, mscr);
++ return phy_modify_paged_changed(phydev, MII_MARVELL_MSCR_PAGE,
++ MII_88E1121_PHY_MSCR_REG,
++ MII_88E1121_PHY_MSCR_DELAY_MASK, mscr);
+ }
+
+ static int m88e1121_config_aneg(struct phy_device *phydev)
+@@ -567,11 +569,13 @@ static int m88e1121_config_aneg(struct phy_device *phydev)
+ return err;
+ }
+
++ changed = err;
++
+ err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
+ if (err < 0)
+ return err;
+
+- changed = err;
++ changed |= err;
+
+ err = genphy_config_aneg(phydev);
+ if (err < 0)
+@@ -1211,16 +1215,15 @@ static int m88e1118_config_aneg(struct phy_device *phydev)
+ {
+ int err;
+
+- err = genphy_soft_reset(phydev);
++ err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
+ if (err < 0)
+ return err;
+
+- err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
++ err = genphy_config_aneg(phydev);
+ if (err < 0)
+ return err;
+
+- err = genphy_config_aneg(phydev);
+- return 0;
++ return genphy_soft_reset(phydev);
+ }
+
+ static int m88e1118_config_init(struct phy_device *phydev)
+@@ -1242,6 +1245,12 @@ static int m88e1118_config_init(struct phy_device *phydev)
+ if (err < 0)
+ return err;
+
++ if (phy_interface_is_rgmii(phydev)) {
++ err = m88e1121_config_aneg_rgmii_delays(phydev);
++ if (err < 0)
++ return err;
++ }
++
+ /* Adjust LED Control */
+ if (phydev->dev_flags & MARVELL_PHY_M1118_DNS323_LEDS)
+ err = phy_write(phydev, 0x10, 0x1100);
+@@ -1684,8 +1693,8 @@ static int marvell_suspend(struct phy_device *phydev)
+ int err;
+
+ /* Suspend the fiber mode first */
+- if (!linkmode_test_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
+- phydev->supported)) {
++ if (linkmode_test_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
++ phydev->supported)) {
+ err = marvell_set_page(phydev, MII_MARVELL_FIBER_PAGE);
+ if (err < 0)
+ goto error;
+@@ -1719,8 +1728,8 @@ static int marvell_resume(struct phy_device *phydev)
+ int err;
+
+ /* Resume the fiber mode first */
+- if (!linkmode_test_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
+- phydev->supported)) {
++ if (linkmode_test_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
++ phydev->supported)) {
+ err = marvell_set_page(phydev, MII_MARVELL_FIBER_PAGE);
+ if (err < 0)
+ goto error;
+@@ -1932,6 +1941,58 @@ static void marvell_get_stats(struct phy_device *phydev,
+ data[i] = marvell_get_stat(phydev, i);
+ }
+
++static int m88e1510_loopback(struct phy_device *phydev, bool enable)
++{
++ int err;
++
++ if (enable) {
++ u16 bmcr_ctl = 0, mscr2_ctl = 0;
++
++ if (phydev->speed == SPEED_1000)
++ bmcr_ctl = BMCR_SPEED1000;
++ else if (phydev->speed == SPEED_100)
++ bmcr_ctl = BMCR_SPEED100;
++
++ if (phydev->duplex == DUPLEX_FULL)
++ bmcr_ctl |= BMCR_FULLDPLX;
++
++ err = phy_write(phydev, MII_BMCR, bmcr_ctl);
++ if (err < 0)
++ return err;
++
++ if (phydev->speed == SPEED_1000)
++ mscr2_ctl = BMCR_SPEED1000;
++ else if (phydev->speed == SPEED_100)
++ mscr2_ctl = BMCR_SPEED100;
++
++ err = phy_modify_paged(phydev, MII_MARVELL_MSCR_PAGE,
++ MII_88E1510_MSCR_2, BMCR_SPEED1000 |
++ BMCR_SPEED100, mscr2_ctl);
++ if (err < 0)
++ return err;
++
++ /* Need soft reset to have speed configuration takes effect */
++ err = genphy_soft_reset(phydev);
++ if (err < 0)
++ return err;
++
++ /* FIXME: Based on trial and error test, it seem 1G need to have
++ * delay between soft reset and loopback enablement.
++ */
++ if (phydev->speed == SPEED_1000)
++ msleep(1000);
++
++ return phy_modify(phydev, MII_BMCR, BMCR_LOOPBACK,
++ BMCR_LOOPBACK);
++ } else {
++ err = phy_modify(phydev, MII_BMCR, BMCR_LOOPBACK, 0);
++ if (err < 0)
++ return err;
++
++ return phy_config_aneg(phydev);
++ }
++}
++
+ static int marvell_vct5_wait_complete(struct phy_device *phydev)
+ {
+ int i;
+@@ -3078,7 +3139,7 @@ static struct phy_driver marvell_drivers[] = {
+ .get_sset_count = marvell_get_sset_count,
+ .get_strings = marvell_get_strings,
+ .get_stats = marvell_get_stats,
+- .set_loopback = genphy_loopback,
++ .set_loopback = m88e1510_loopback,
+ .get_tunable = m88e1011_get_tunable,
+ .set_tunable = m88e1011_set_tunable,
+ .cable_test_start = marvell_vct7_cable_test_start,
+diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c
+index bd310e8d5e43d..df33637c5269a 100644
+--- a/drivers/net/phy/marvell10g.c
++++ b/drivers/net/phy/marvell10g.c
+@@ -789,7 +789,7 @@ static int mv3310_read_status_copper(struct phy_device *phydev)
+
+ cssr1 = phy_read_mmd(phydev, MDIO_MMD_PCS, MV_PCS_CSSR1);
+ if (cssr1 < 0)
+- return val;
++ return cssr1;
+
+ /* If the link settings are not resolved, mark the link down */
+ if (!(cssr1 & MV_PCS_CSSR1_RESOLVED)) {
+diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
+index 6865d9319197f..8dc6e6269c65e 100644
+--- a/drivers/net/phy/mdio_bus.c
++++ b/drivers/net/phy/mdio_bus.c
+@@ -591,7 +591,7 @@ int __mdiobus_register(struct mii_bus *bus, struct module *owner)
+ mdiobus_setup_mdiodev_from_board_info(bus, mdiobus_create_device);
+
+ bus->state = MDIOBUS_REGISTERED;
+- pr_info("%s: probed\n", bus->name);
++ dev_dbg(&bus->dev, "probed\n");
+ return 0;
+
+ error:
+diff --git a/drivers/net/phy/mediatek-ge.c b/drivers/net/phy/mediatek-ge.c
+index b7a5ae20edd53..68ee434f9dea3 100644
+--- a/drivers/net/phy/mediatek-ge.c
++++ b/drivers/net/phy/mediatek-ge.c
+@@ -55,9 +55,6 @@ static int mt7530_phy_config_init(struct phy_device *phydev)
+
+ static int mt7531_phy_config_init(struct phy_device *phydev)
+ {
+- if (phydev->interface != PHY_INTERFACE_MODE_INTERNAL)
+- return -EINVAL;
+-
+ mtk_gephy_config_init(phydev);
+
+ /* PHY link down power saving enable */
+diff --git a/drivers/net/phy/meson-gxl.c b/drivers/net/phy/meson-gxl.c
+index 7e7904fee1d97..73f7962a37d33 100644
+--- a/drivers/net/phy/meson-gxl.c
++++ b/drivers/net/phy/meson-gxl.c
+@@ -30,8 +30,12 @@
+ #define INTSRC_LINK_DOWN BIT(4)
+ #define INTSRC_REMOTE_FAULT BIT(5)
+ #define INTSRC_ANEG_COMPLETE BIT(6)
++#define INTSRC_ENERGY_DETECT BIT(7)
+ #define INTSRC_MASK 30
+
++#define INT_SOURCES (INTSRC_LINK_DOWN | INTSRC_ANEG_COMPLETE | \
++ INTSRC_ENERGY_DETECT)
++
+ #define BANK_ANALOG_DSP 0
+ #define BANK_WOL 1
+ #define BANK_BIST 3
+@@ -200,7 +204,6 @@ static int meson_gxl_ack_interrupt(struct phy_device *phydev)
+
+ static int meson_gxl_config_intr(struct phy_device *phydev)
+ {
+- u16 val;
+ int ret;
+
+ if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+@@ -209,16 +212,9 @@ static int meson_gxl_config_intr(struct phy_device *phydev)
+ if (ret)
+ return ret;
+
+- val = INTSRC_ANEG_PR
+- | INTSRC_PARALLEL_FAULT
+- | INTSRC_ANEG_LP_ACK
+- | INTSRC_LINK_DOWN
+- | INTSRC_REMOTE_FAULT
+- | INTSRC_ANEG_COMPLETE;
+- ret = phy_write(phydev, INTSRC_MASK, val);
++ ret = phy_write(phydev, INTSRC_MASK, INT_SOURCES);
+ } else {
+- val = 0;
+- ret = phy_write(phydev, INTSRC_MASK, val);
++ ret = phy_write(phydev, INTSRC_MASK, 0);
+
+ /* Ack any pending IRQ */
+ ret = meson_gxl_ack_interrupt(phydev);
+@@ -237,10 +233,23 @@ static irqreturn_t meson_gxl_handle_interrupt(struct phy_device *phydev)
+ return IRQ_NONE;
+ }
+
++ irq_status &= INT_SOURCES;
++
+ if (irq_status == 0)
+ return IRQ_NONE;
+
+- phy_trigger_machine(phydev);
++ /* Aneg-complete interrupt is used for link-up detection */
++ if (phydev->autoneg == AUTONEG_ENABLE &&
++ irq_status == INTSRC_ENERGY_DETECT)
++ return IRQ_HANDLED;
++
++ /* Give PHY some time before MAC starts sending data. This works
++ * around an issue where network doesn't come up properly.
++ */
++ if (!(irq_status & INTSRC_LINK_DOWN))
++ phy_queue_state_machine(phydev, msecs_to_jiffies(100));
++ else
++ phy_trigger_machine(phydev);
+
+ return IRQ_HANDLED;
+ }
+diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
+index 5c928f827173c..64d829ed98876 100644
+--- a/drivers/net/phy/micrel.c
++++ b/drivers/net/phy/micrel.c
+@@ -863,9 +863,9 @@ static int ksz9031_config_init(struct phy_device *phydev)
+ MII_KSZ9031RN_TX_DATA_PAD_SKEW, 4,
+ tx_data_skews, 4, &update);
+
+- if (update && phydev->interface != PHY_INTERFACE_MODE_RGMII)
++ if (update && !phy_interface_is_rgmii(phydev))
+ phydev_warn(phydev,
+- "*-skew-ps values should be used only with phy-mode = \"rgmii\"\n");
++ "*-skew-ps values should be used only with RGMII PHY modes\n");
+
+ /* Silicon Errata Sheet (DS80000691D or DS80000692D):
+ * When the device links in the 1000BASE-T slave mode only,
+@@ -1544,11 +1544,12 @@ static struct phy_driver ksphy_driver[] = {
+ .name = "Micrel KS8737",
+ /* PHY_BASIC_FEATURES */
+ .driver_data = &ks8737_type,
++ .probe = kszphy_probe,
+ .config_init = kszphy_config_init,
+ .config_intr = kszphy_config_intr,
+ .handle_interrupt = kszphy_handle_interrupt,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8021,
+ .phy_id_mask = 0x00ffffff,
+@@ -1562,8 +1563,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8031,
+ .phy_id_mask = 0x00ffffff,
+@@ -1577,8 +1578,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8041,
+ .phy_id_mask = MICREL_PHY_ID_MASK,
+@@ -1593,8 +1594,9 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ /* No suspend/resume callbacks because of errata DS80000700A,
++ * receiver error following software power down.
++ */
+ }, {
+ .phy_id = PHY_ID_KSZ8041RNLI,
+ .phy_id_mask = MICREL_PHY_ID_MASK,
+@@ -1608,8 +1610,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .name = "Micrel KSZ8051",
+ /* PHY_BASIC_FEATURES */
+@@ -1622,8 +1624,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+ .match_phy_device = ksz8051_match_phy_device,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8001,
+ .name = "Micrel KSZ8001 or KS8721",
+@@ -1637,8 +1639,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8081,
+ .name = "Micrel KSZ8081 or KSZ8091",
+@@ -1684,8 +1686,8 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
+- .resume = genphy_resume,
++ .suspend = kszphy_suspend,
++ .resume = kszphy_resume,
+ .read_mmd = genphy_read_mmd_unsupported,
+ .write_mmd = genphy_write_mmd_unsupported,
+ }, {
+@@ -1703,7 +1705,7 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
++ .suspend = kszphy_suspend,
+ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_LAN8814,
+@@ -1731,7 +1733,7 @@ static struct phy_driver ksphy_driver[] = {
+ .get_sset_count = kszphy_get_sset_count,
+ .get_strings = kszphy_get_strings,
+ .get_stats = kszphy_get_stats,
+- .suspend = genphy_suspend,
++ .suspend = kszphy_suspend,
+ .resume = kszphy_resume,
+ }, {
+ .phy_id = PHY_ID_KSZ8873MLL,
+diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c
+index 6e32da28e138f..f2e3a67198dd6 100644
+--- a/drivers/net/phy/mscc/mscc_main.c
++++ b/drivers/net/phy/mscc/mscc_main.c
+@@ -2685,3 +2685,6 @@ MODULE_DEVICE_TABLE(mdio, vsc85xx_tbl);
+ MODULE_DESCRIPTION("Microsemi VSC85xx PHY driver");
+ MODULE_AUTHOR("Nagaraju Lakkaraju");
+ MODULE_LICENSE("Dual MIT/GPL");
++
++MODULE_FIRMWARE(MSCC_VSC8584_REVB_INT8051_FW);
++MODULE_FIRMWARE(MSCC_VSC8574_REVB_INT8051_FW);
+diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c
+index 2870c33b8975d..271fc01f7f7fd 100644
+--- a/drivers/net/phy/phy-core.c
++++ b/drivers/net/phy/phy-core.c
+@@ -162,11 +162,11 @@ static const struct phy_setting settings[] = {
+ PHY_SETTING( 2500, FULL, 2500baseT_Full ),
+ PHY_SETTING( 2500, FULL, 2500baseX_Full ),
+ /* 1G */
+- PHY_SETTING( 1000, FULL, 1000baseKX_Full ),
+ PHY_SETTING( 1000, FULL, 1000baseT_Full ),
+ PHY_SETTING( 1000, HALF, 1000baseT_Half ),
+ PHY_SETTING( 1000, FULL, 1000baseT1_Full ),
+ PHY_SETTING( 1000, FULL, 1000baseX_Full ),
++ PHY_SETTING( 1000, FULL, 1000baseKX_Full ),
+ /* 100M */
+ PHY_SETTING( 100, FULL, 100baseT_Full ),
+ PHY_SETTING( 100, FULL, 100baseT1_Full ),
+diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
+index a3bfb156c83d7..f122026c46826 100644
+--- a/drivers/net/phy/phy.c
++++ b/drivers/net/phy/phy.c
+@@ -815,7 +815,12 @@ int phy_ethtool_ksettings_set(struct phy_device *phydev,
+ phydev->mdix_ctrl = cmd->base.eth_tp_mdix_ctrl;
+
+ /* Restart the PHY */
+- _phy_start_aneg(phydev);
++ if (phy_is_started(phydev)) {
++ phydev->state = PHY_UP;
++ phy_trigger_machine(phydev);
++ } else {
++ _phy_start_aneg(phydev);
++ }
+
+ mutex_unlock(&phydev->lock);
+ return 0;
+@@ -965,8 +970,13 @@ static irqreturn_t phy_interrupt(int irq, void *phy_dat)
+ {
+ struct phy_device *phydev = phy_dat;
+ struct phy_driver *drv = phydev->drv;
++ irqreturn_t ret;
+
+- return drv->handle_interrupt(phydev);
++ mutex_lock(&phydev->lock);
++ ret = drv->handle_interrupt(phydev);
++ mutex_unlock(&phydev->lock);
++
++ return ret;
+ }
+
+ /**
+diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
+index 4f9990b47a377..28f4a383aba72 100644
+--- a/drivers/net/phy/phy_device.c
++++ b/drivers/net/phy/phy_device.c
+@@ -1746,6 +1746,9 @@ void phy_detach(struct phy_device *phydev)
+ phy_driver_is_genphy_10g(phydev))
+ device_release_driver(&phydev->mdio.dev);
+
++ /* Assert the reset signal */
++ phy_device_reset(phydev, 1);
++
+ /*
+ * The phydev might go away on the put_device() below, so avoid
+ * a use-after-free bug by reading the underlying bus first.
+@@ -1757,9 +1760,6 @@ void phy_detach(struct phy_device *phydev)
+ ndev_owner = dev->dev.parent->driver->owner;
+ if (ndev_owner != bus->owner)
+ module_put(bus->owner);
+-
+- /* Assert the reset signal */
+- phy_device_reset(phydev, 1);
+ }
+ EXPORT_SYMBOL(phy_detach);
+
+diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
+index 0a0abe8e4be0b..fef1416dcee4c 100644
+--- a/drivers/net/phy/phylink.c
++++ b/drivers/net/phy/phylink.c
+@@ -657,6 +657,7 @@ static void phylink_resolve(struct work_struct *w)
+ struct phylink_link_state link_state;
+ struct net_device *ndev = pl->netdev;
+ bool mac_config = false;
++ bool retrigger = false;
+ bool cur_link_state;
+
+ mutex_lock(&pl->state_mutex);
+@@ -670,6 +671,7 @@ static void phylink_resolve(struct work_struct *w)
+ link_state.link = false;
+ } else if (pl->mac_link_dropped) {
+ link_state.link = false;
++ retrigger = true;
+ } else {
+ switch (pl->cur_link_an_mode) {
+ case MLO_AN_PHY:
+@@ -686,6 +688,19 @@ static void phylink_resolve(struct work_struct *w)
+ case MLO_AN_INBAND:
+ phylink_mac_pcs_get_state(pl, &link_state);
+
++ /* The PCS may have a latching link-fail indicator.
++ * If the link was up, bring the link down and
++ * re-trigger the resolve. Otherwise, re-read the
++ * PCS state to get the current status of the link.
++ */
++ if (!link_state.link) {
++ if (cur_link_state)
++ retrigger = true;
++ else
++ phylink_mac_pcs_get_state(pl,
++ &link_state);
++ }
++
+ /* If we have a phy, the "up" state is the union of
+ * both the PHY and the MAC
+ */
+@@ -694,6 +709,15 @@ static void phylink_resolve(struct work_struct *w)
+
+ /* Only update if the PHY link is up */
+ if (pl->phydev && pl->phy_state.link) {
++ /* If the interface has changed, force a
++ * link down event if the link isn't already
++ * down, and re-resolve.
++ */
++ if (link_state.interface !=
++ pl->phy_state.interface) {
++ retrigger = true;
++ link_state.link = false;
++ }
+ link_state.interface = pl->phy_state.interface;
+
+ /* If we have a PHY, we need to update with
+@@ -736,7 +760,7 @@ static void phylink_resolve(struct work_struct *w)
+ else
+ phylink_link_up(pl, link_state);
+ }
+- if (!link_state.link && pl->mac_link_dropped) {
++ if (!link_state.link && retrigger) {
+ pl->mac_link_dropped = false;
+ queue_work(system_power_efficient_wq, &pl->resolve);
+ }
+@@ -1333,7 +1357,10 @@ void phylink_suspend(struct phylink *pl, bool mac_wol)
+ * but one would hope all packets have been sent. This
+ * also means phylink_resolve() will do nothing.
+ */
+- netif_carrier_off(pl->netdev);
++ if (pl->netdev)
++ netif_carrier_off(pl->netdev);
++ else
++ pl->old_link_state = false;
+
+ /* We do not call mac_link_down() here as we want the
+ * link to remain up to receive the WoL packets.
+@@ -1724,7 +1751,7 @@ int phylink_ethtool_set_pauseparam(struct phylink *pl,
+ return -EOPNOTSUPP;
+
+ if (!phylink_test(pl->supported, Asym_Pause) &&
+- !pause->autoneg && pause->rx_pause != pause->tx_pause)
++ pause->rx_pause != pause->tx_pause)
+ return -EINVAL;
+
+ pause_state = 0;
+diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
+index 7362f8c3271c9..4369d6249e7bb 100644
+--- a/drivers/net/phy/sfp-bus.c
++++ b/drivers/net/phy/sfp-bus.c
+@@ -74,6 +74,12 @@ static const struct sfp_quirk sfp_quirks[] = {
+ .vendor = "HUAWEI",
+ .part = "MA5671A",
+ .modes = sfp_quirk_2500basex,
++ }, {
++ // Lantech 8330-262D-E can operate at 2500base-X, but
++ // incorrectly report 2500MBd NRZ in their EEPROM
++ .vendor = "Lantech",
++ .part = "8330-262D-E",
++ .modes = sfp_quirk_2500basex,
+ }, {
+ .vendor = "UBNT",
+ .part = "UF-INSTANT",
+@@ -651,6 +657,11 @@ struct sfp_bus *sfp_bus_find_fwnode(struct fwnode_handle *fwnode)
+ else if (ret < 0)
+ return ERR_PTR(ret);
+
++ if (!fwnode_device_is_available(ref.fwnode)) {
++ fwnode_handle_put(ref.fwnode);
++ return NULL;
++ }
++
+ bus = sfp_bus_get(ref.fwnode);
+ fwnode_handle_put(ref.fwnode);
+ if (!bus)
+diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
+index ab77a9f439ef9..90dfefc1f5f8d 100644
+--- a/drivers/net/phy/sfp.c
++++ b/drivers/net/phy/sfp.c
+@@ -250,6 +250,7 @@ struct sfp {
+ struct sfp_eeprom_id id;
+ unsigned int module_power_mW;
+ unsigned int module_t_start_up;
++ bool tx_fault_ignore;
+
+ #if IS_ENABLED(CONFIG_HWMON)
+ struct sfp_diag diag;
+@@ -1641,17 +1642,20 @@ static int sfp_sm_probe_for_phy(struct sfp *sfp)
+ static int sfp_module_parse_power(struct sfp *sfp)
+ {
+ u32 power_mW = 1000;
++ bool supports_a2;
+
+ if (sfp->id.ext.options & cpu_to_be16(SFP_OPTIONS_POWER_DECL))
+ power_mW = 1500;
+ if (sfp->id.ext.options & cpu_to_be16(SFP_OPTIONS_HIGH_POWER_LEVEL))
+ power_mW = 2000;
+
++ supports_a2 = sfp->id.ext.sff8472_compliance !=
++ SFP_SFF8472_COMPLIANCE_NONE ||
++ sfp->id.ext.diagmon & SFP_DIAGMON_DDM;
++
+ if (power_mW > sfp->max_power_mW) {
+ /* Module power specification exceeds the allowed maximum. */
+- if (sfp->id.ext.sff8472_compliance ==
+- SFP_SFF8472_COMPLIANCE_NONE &&
+- !(sfp->id.ext.diagmon & SFP_DIAGMON_DDM)) {
++ if (!supports_a2) {
+ /* The module appears not to implement bus address
+ * 0xa2, so assume that the module powers up in the
+ * indicated mode.
+@@ -1668,11 +1672,25 @@ static int sfp_module_parse_power(struct sfp *sfp)
+ }
+ }
+
++ if (power_mW <= 1000) {
++ /* Modules below 1W do not require a power change sequence */
++ sfp->module_power_mW = power_mW;
++ return 0;
++ }
++
++ if (!supports_a2) {
++ /* The module power level is below the host maximum and the
++ * module appears not to implement bus address 0xa2, so assume
++ * that the module powers up in the indicated mode.
++ */
++ return 0;
++ }
++
+ /* If the module requires a higher power mode, but also requires
+ * an address change sequence, warn the user that the module may
+ * not be functional.
+ */
+- if (sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE && power_mW > 1000) {
++ if (sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE) {
+ dev_warn(sfp->dev,
+ "Address Change Sequence not supported but module requires %u.%uW, module may not be functional\n",
+ power_mW / 1000, (power_mW / 100) % 10);
+@@ -1928,6 +1946,12 @@ static int sfp_sm_mod_probe(struct sfp *sfp, bool report)
+ else
+ sfp->module_t_start_up = T_START_UP;
+
++ if (!memcmp(id.base.vendor_name, "HUAWEI ", 16) &&
++ !memcmp(id.base.vendor_pn, "MA5671A ", 16))
++ sfp->tx_fault_ignore = true;
++ else
++ sfp->tx_fault_ignore = false;
++
+ return 0;
+ }
+
+@@ -2380,7 +2404,10 @@ static void sfp_check_state(struct sfp *sfp)
+ mutex_lock(&sfp->st_mutex);
+ state = sfp_get_state(sfp);
+ changed = state ^ sfp->state;
+- changed &= SFP_F_PRESENT | SFP_F_LOS | SFP_F_TX_FAULT;
++ if (sfp->tx_fault_ignore)
++ changed &= SFP_F_PRESENT | SFP_F_LOS;
++ else
++ changed &= SFP_F_PRESENT | SFP_F_LOS | SFP_F_TX_FAULT;
+
+ for (i = 0; i < GPIO_MAX; i++)
+ if (changed & BIT(i))
+diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
+index fb52cd175b45d..829d6ada1704c 100644
+--- a/drivers/net/ppp/ppp_generic.c
++++ b/drivers/net/ppp/ppp_generic.c
+@@ -69,6 +69,8 @@
+ #define MPHDRLEN 6 /* multilink protocol header length */
+ #define MPHDRLEN_SSN 4 /* ditto with short sequence numbers */
+
++#define PPP_PROTO_LEN 2
++
+ /*
+ * An instance of /dev/ppp can be associated with either a ppp
+ * interface unit or a ppp channel. In both cases, file->private_data
+@@ -497,6 +499,9 @@ static ssize_t ppp_write(struct file *file, const char __user *buf,
+
+ if (!pf)
+ return -ENXIO;
++ /* All PPP packets should start with the 2-byte protocol */
++ if (count < PPP_PROTO_LEN)
++ return -EINVAL;
+ ret = -ENOMEM;
+ skb = alloc_skb(count + pf->hdrlen, GFP_KERNEL);
+ if (!skb)
+@@ -1764,7 +1769,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
+ }
+
+ ++ppp->stats64.tx_packets;
+- ppp->stats64.tx_bytes += skb->len - 2;
++ ppp->stats64.tx_bytes += skb->len - PPP_PROTO_LEN;
+
+ switch (proto) {
+ case PPP_IP:
+diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
+index 3619520340b74..e172743948ed7 100644
+--- a/drivers/net/ppp/pppoe.c
++++ b/drivers/net/ppp/pppoe.c
+@@ -988,6 +988,7 @@ static int pppoe_fill_forward_path(struct net_device_path_ctx *ctx,
+ path->encap.proto = htons(ETH_P_PPP_SES);
+ path->encap.id = be16_to_cpu(po->num);
+ memcpy(path->encap.h_dest, po->pppoe_pa.remote, ETH_ALEN);
++ memcpy(ctx->daddr, po->pppoe_pa.remote, ETH_ALEN);
+ path->dev = ctx->dev;
+ ctx->dev = dev;
+
+diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c
+index 5435b5689ce6b..2a3892528ec36 100644
+--- a/drivers/net/slip/slip.c
++++ b/drivers/net/slip/slip.c
+@@ -469,7 +469,7 @@ static void sl_tx_timeout(struct net_device *dev, unsigned int txqueue)
+ spin_lock(&sl->lock);
+
+ if (netif_queue_stopped(dev)) {
+- if (!netif_running(dev))
++ if (!netif_running(dev) || !sl->tty)
+ goto out;
+
+ /* May be we must check transmitter timeout here ?
+diff --git a/drivers/net/slip/slip.h b/drivers/net/slip/slip.h
+index c420e59485221..3d7f88b330c1e 100644
+--- a/drivers/net/slip/slip.h
++++ b/drivers/net/slip/slip.h
+@@ -40,6 +40,8 @@
+ insmod -oslip_maxdev=nnn */
+ #define SL_MTU 296 /* 296; I am used to 600- FvK */
+
++/* some arch define END as assembly function ending, just undef it */
++#undef END
+ /* SLIP protocol characters. */
+ #define END 0300 /* indicates end of frame */
+ #define ESC 0333 /* indicates byte stuffing */
+diff --git a/drivers/net/tap.c b/drivers/net/tap.c
+index 8e3a28ba6b282..ba2ef5437e167 100644
+--- a/drivers/net/tap.c
++++ b/drivers/net/tap.c
+@@ -1198,7 +1198,8 @@ static int tap_sendmsg(struct socket *sock, struct msghdr *m,
+ struct xdp_buff *xdp;
+ int i;
+
+- if (ctl && (ctl->type == TUN_MSG_PTR)) {
++ if (m->msg_controllen == sizeof(struct tun_msg_ctl) &&
++ ctl && ctl->type == TUN_MSG_PTR) {
+ for (i = 0; i < ctl->num; i++) {
+ xdp = &((struct xdp_buff *)ctl->ptr)[i];
+ tap_get_user_xdp(q, xdp);
+diff --git a/drivers/net/tun.c b/drivers/net/tun.c
+index fecc9a1d293ae..02de8d998bfa4 100644
+--- a/drivers/net/tun.c
++++ b/drivers/net/tun.c
+@@ -209,6 +209,9 @@ struct tun_struct {
+ struct tun_prog __rcu *steering_prog;
+ struct tun_prog __rcu *filter_prog;
+ struct ethtool_link_ksettings link_ksettings;
++ /* init args */
++ struct file *file;
++ struct ifreq *ifr;
+ };
+
+ struct veth {
+@@ -216,6 +219,9 @@ struct veth {
+ __be16 h_vlan_TCI;
+ };
+
++static void tun_flow_init(struct tun_struct *tun);
++static void tun_flow_uninit(struct tun_struct *tun);
++
+ static int tun_napi_receive(struct napi_struct *napi, int budget)
+ {
+ struct tun_file *tfile = container_of(napi, struct tun_file, napi);
+@@ -953,6 +959,49 @@ static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
+
+ static const struct ethtool_ops tun_ethtool_ops;
+
++static int tun_net_init(struct net_device *dev)
++{
++ struct tun_struct *tun = netdev_priv(dev);
++ struct ifreq *ifr = tun->ifr;
++ int err;
++
++ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
++ if (!dev->tstats)
++ return -ENOMEM;
++
++ spin_lock_init(&tun->lock);
++
++ err = security_tun_dev_alloc_security(&tun->security);
++ if (err < 0) {
++ free_percpu(dev->tstats);
++ return err;
++ }
++
++ tun_flow_init(tun);
++
++ dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
++ TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
++ NETIF_F_HW_VLAN_STAG_TX;
++ dev->features = dev->hw_features | NETIF_F_LLTX;
++ dev->vlan_features = dev->features &
++ ~(NETIF_F_HW_VLAN_CTAG_TX |
++ NETIF_F_HW_VLAN_STAG_TX);
++
++ tun->flags = (tun->flags & ~TUN_FEATURES) |
++ (ifr->ifr_flags & TUN_FEATURES);
++
++ INIT_LIST_HEAD(&tun->disabled);
++ err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI,
++ ifr->ifr_flags & IFF_NAPI_FRAGS, false);
++ if (err < 0) {
++ tun_flow_uninit(tun);
++ security_tun_dev_free_security(tun->security);
++ free_percpu(dev->tstats);
++ return err;
++ }
++ return 0;
++}
++
+ /* Net device detach from fd. */
+ static void tun_net_uninit(struct net_device *dev)
+ {
+@@ -1010,6 +1059,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
+ {
+ struct tun_struct *tun = netdev_priv(dev);
+ int txq = skb->queue_mapping;
++ struct netdev_queue *queue;
+ struct tun_file *tfile;
+ int len = skb->len;
+
+@@ -1054,6 +1104,10 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
+ if (ptr_ring_produce(&tfile->tx_ring, skb))
+ goto drop;
+
++ /* NETIF_F_LLTX requires to do our own update of trans_start */
++ queue = netdev_get_tx_queue(dev, txq);
++ queue->trans_start = jiffies;
++
+ /* Notify and wake up reader process */
+ if (tfile->flags & TUN_FASYNC)
+ kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
+@@ -1164,6 +1218,7 @@ static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
+ }
+
+ static const struct net_device_ops tun_netdev_ops = {
++ .ndo_init = tun_net_init,
+ .ndo_uninit = tun_net_uninit,
+ .ndo_open = tun_net_open,
+ .ndo_stop = tun_net_close,
+@@ -1247,6 +1302,7 @@ static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
+ }
+
+ static const struct net_device_ops tap_netdev_ops = {
++ .ndo_init = tun_net_init,
+ .ndo_uninit = tun_net_uninit,
+ .ndo_open = tun_net_open,
+ .ndo_stop = tun_net_close,
+@@ -1287,7 +1343,7 @@ static void tun_flow_uninit(struct tun_struct *tun)
+ #define MAX_MTU 65535
+
+ /* Initialize net device. */
+-static void tun_net_init(struct net_device *dev)
++static void tun_net_initialize(struct net_device *dev)
+ {
+ struct tun_struct *tun = netdev_priv(dev);
+
+@@ -2201,11 +2257,6 @@ static void tun_free_netdev(struct net_device *dev)
+ BUG_ON(!(list_empty(&tun->disabled)));
+
+ free_percpu(dev->tstats);
+- /* We clear tstats so that tun_set_iff() can tell if
+- * tun_free_netdev() has been called from register_netdevice().
+- */
+- dev->tstats = NULL;
+-
+ tun_flow_uninit(tun);
+ security_tun_dev_free_security(tun->security);
+ __tun_set_ebpf(tun, &tun->steering_prog, NULL);
+@@ -2438,7 +2489,8 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
+ if (!tun)
+ return -EBADFD;
+
+- if (ctl && (ctl->type == TUN_MSG_PTR)) {
++ if (m->msg_controllen == sizeof(struct tun_msg_ctl) &&
++ ctl && ctl->type == TUN_MSG_PTR) {
+ struct tun_page tpage;
+ int n = ctl->num;
+ int flush = 0;
+@@ -2711,41 +2763,16 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
+ tun->rx_batched = 0;
+ RCU_INIT_POINTER(tun->steering_prog, NULL);
+
+- dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+- if (!dev->tstats) {
+- err = -ENOMEM;
+- goto err_free_dev;
+- }
++ tun->ifr = ifr;
++ tun->file = file;
+
+- spin_lock_init(&tun->lock);
+-
+- err = security_tun_dev_alloc_security(&tun->security);
+- if (err < 0)
+- goto err_free_stat;
+-
+- tun_net_init(dev);
+- tun_flow_init(tun);
+-
+- dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
+- TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
+- NETIF_F_HW_VLAN_STAG_TX;
+- dev->features = dev->hw_features | NETIF_F_LLTX;
+- dev->vlan_features = dev->features &
+- ~(NETIF_F_HW_VLAN_CTAG_TX |
+- NETIF_F_HW_VLAN_STAG_TX);
+-
+- tun->flags = (tun->flags & ~TUN_FEATURES) |
+- (ifr->ifr_flags & TUN_FEATURES);
+-
+- INIT_LIST_HEAD(&tun->disabled);
+- err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI,
+- ifr->ifr_flags & IFF_NAPI_FRAGS, false);
+- if (err < 0)
+- goto err_free_flow;
++ tun_net_initialize(dev);
+
+ err = register_netdevice(tun->dev);
+- if (err < 0)
+- goto err_detach;
++ if (err < 0) {
++ free_netdev(dev);
++ return err;
++ }
+ /* free_netdev() won't check refcnt, to avoid race
+ * with dev_put() we need publish tun after registration.
+ */
+@@ -2762,24 +2789,6 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
+
+ strcpy(ifr->ifr_name, tun->dev->name);
+ return 0;
+-
+-err_detach:
+- tun_detach_all(dev);
+- /* We are here because register_netdevice() has failed.
+- * If register_netdevice() already called tun_free_netdev()
+- * while dealing with the error, dev->stats has been cleared.
+- */
+- if (!dev->tstats)
+- goto err_free_dev;
+-
+-err_free_flow:
+- tun_flow_uninit(tun);
+- security_tun_dev_free_security(tun->security);
+-err_free_stat:
+- free_percpu(dev->tstats);
+-err_free_dev:
+- free_netdev(dev);
+- return err;
+ }
+
+ static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)
+diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c
+index 73b97f4cc1ec1..e8d49886d6953 100644
+--- a/drivers/net/usb/aqc111.c
++++ b/drivers/net/usb/aqc111.c
+@@ -1102,10 +1102,15 @@ static int aqc111_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
+ if (start_of_descs != desc_offset)
+ goto err;
+
+- /* self check desc_offset from header*/
+- if (desc_offset >= skb_len)
++ /* self check desc_offset from header and make sure that the
++ * bounds of the metadata array are inside the SKB
++ */
++ if (pkt_count * 2 + desc_offset >= skb_len)
+ goto err;
+
++ /* Packets must not overlap the metadata array */
++ skb_trim(skb, desc_offset);
++
+ if (pkt_count == 0)
+ goto err;
+
+diff --git a/drivers/net/usb/asix.h b/drivers/net/usb/asix.h
+index 2a1e31defe718..4334aafab59a4 100644
+--- a/drivers/net/usb/asix.h
++++ b/drivers/net/usb/asix.h
+@@ -192,8 +192,8 @@ extern const struct driver_info ax88172a_info;
+ /* ASIX specific flags */
+ #define FLAG_EEPROM_MAC (1UL << 0) /* init device MAC from eeprom */
+
+-int asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
+- u16 size, void *data, int in_pm);
++int __must_check asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
++ u16 size, void *data, int in_pm);
+
+ int asix_write_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
+ u16 size, void *data, int in_pm);
+diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c
+index 38cda590895cc..f39188b7717ae 100644
+--- a/drivers/net/usb/asix_common.c
++++ b/drivers/net/usb/asix_common.c
+@@ -9,8 +9,10 @@
+
+ #include "asix.h"
+
+-int asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
+- u16 size, void *data, int in_pm)
++#define AX_HOST_EN_RETRIES 30
++
++int __must_check asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
++ u16 size, void *data, int in_pm)
+ {
+ int ret;
+ int (*fn)(struct usbnet *, u8, u8, u16, u16, void *, u16);
+@@ -25,9 +27,12 @@ int asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
+ ret = fn(dev, cmd, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+ value, index, data, size);
+
+- if (unlikely(ret < 0))
++ if (unlikely(ret < size)) {
++ ret = ret < 0 ? ret : -ENODATA;
++
+ netdev_warn(dev->net, "Failed to read reg index 0x%04x: %d\n",
+ index, ret);
++ }
+
+ return ret;
+ }
+@@ -68,7 +73,7 @@ static int asix_check_host_enable(struct usbnet *dev, int in_pm)
+ int i, ret;
+ u8 smsr;
+
+- for (i = 0; i < 30; ++i) {
++ for (i = 0; i < AX_HOST_EN_RETRIES; ++i) {
+ ret = asix_set_sw_mii(dev, in_pm);
+ if (ret == -ENODEV || ret == -ETIMEDOUT)
+ break;
+@@ -83,7 +88,7 @@ static int asix_check_host_enable(struct usbnet *dev, int in_pm)
+ break;
+ }
+
+- return ret;
++ return i >= AX_HOST_EN_RETRIES ? -ETIMEDOUT : ret;
+ }
+
+ static void reset_asix_rx_fixup_info(struct asix_rx_fixup_info *rx)
+@@ -577,8 +582,12 @@ int asix_mdio_read_nopm(struct net_device *netdev, int phy_id, int loc)
+ return ret;
+ }
+
+- asix_read_cmd(dev, AX_CMD_READ_MII_REG, phy_id,
+- (__u16)loc, 2, &res, 1);
++ ret = asix_read_cmd(dev, AX_CMD_READ_MII_REG, phy_id,
++ (__u16)loc, 2, &res, 1);
++ if (ret < 0) {
++ mutex_unlock(&dev->phy_mutex);
++ return ret;
++ }
+ asix_set_hw_mii(dev, 1);
+ mutex_unlock(&dev->phy_mutex);
+
+diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c
+index 30821f6a6d7ac..bd8f8619ad6f2 100644
+--- a/drivers/net/usb/asix_devices.c
++++ b/drivers/net/usb/asix_devices.c
+@@ -755,7 +755,12 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
+ priv->phy_addr = ret;
+ priv->embd_phy = ((priv->phy_addr & 0x1f) == 0x10);
+
+- asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0, 0, 1, &chipcode, 0);
++ ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0, 0, 1, &chipcode, 0);
++ if (ret < 0) {
++ netdev_dbg(dev->net, "Failed to read STATMNGSTS_REG: %d\n", ret);
++ return ret;
++ }
++
+ chipcode &= AX_CHIPCODE_MASK;
+
+ ret = (chipcode == AX_AX88772_CHIPCODE) ? ax88772_hw_reset(dev, 0) :
+@@ -920,11 +925,21 @@ static int ax88178_reset(struct usbnet *dev)
+ int gpio0 = 0;
+ u32 phyid;
+
+- asix_read_cmd(dev, AX_CMD_READ_GPIOS, 0, 0, 1, &status, 0);
++ ret = asix_read_cmd(dev, AX_CMD_READ_GPIOS, 0, 0, 1, &status, 0);
++ if (ret < 0) {
++ netdev_dbg(dev->net, "Failed to read GPIOS: %d\n", ret);
++ return ret;
++ }
++
+ netdev_dbg(dev->net, "GPIO Status: 0x%04x\n", status);
+
+ asix_write_cmd(dev, AX_CMD_WRITE_ENABLE, 0, 0, 0, NULL, 0);
+- asix_read_cmd(dev, AX_CMD_READ_EEPROM, 0x0017, 0, 2, &eeprom, 0);
++ ret = asix_read_cmd(dev, AX_CMD_READ_EEPROM, 0x0017, 0, 2, &eeprom, 0);
++ if (ret < 0) {
++ netdev_dbg(dev->net, "Failed to read EEPROM: %d\n", ret);
++ return ret;
++ }
++
+ asix_write_cmd(dev, AX_CMD_WRITE_DISABLE, 0, 0, 0, NULL, 0);
+
+ netdev_dbg(dev->net, "EEPROM index 0x17 is 0x%04x\n", eeprom);
+diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c
+index f25448a088707..d5ce642200e8e 100644
+--- a/drivers/net/usb/ax88179_178a.c
++++ b/drivers/net/usb/ax88179_178a.c
+@@ -1467,58 +1467,68 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
+ u16 hdr_off;
+ u32 *pkt_hdr;
+
+- /* This check is no longer done by usbnet */
+- if (skb->len < dev->net->hard_header_len)
++ /* At the end of the SKB, there's a header telling us how many packets
++ * are bundled into this buffer and where we can find an array of
++ * per-packet metadata (which contains elements encoded into u16).
++ */
++ if (skb->len < 4)
+ return 0;
+-
+ skb_trim(skb, skb->len - 4);
+ rx_hdr = get_unaligned_le32(skb_tail_pointer(skb));
+-
+ pkt_cnt = (u16)rx_hdr;
+ hdr_off = (u16)(rx_hdr >> 16);
++
++ if (pkt_cnt == 0)
++ return 0;
++
++ /* Make sure that the bounds of the metadata array are inside the SKB
++ * (and in front of the counter at the end).
++ */
++ if (pkt_cnt * 2 + hdr_off > skb->len)
++ return 0;
+ pkt_hdr = (u32 *)(skb->data + hdr_off);
+
+- while (pkt_cnt--) {
++ /* Packets must not overlap the metadata array */
++ skb_trim(skb, hdr_off);
++
++ for (; ; pkt_cnt--, pkt_hdr++) {
+ u16 pkt_len;
+
+ le32_to_cpus(pkt_hdr);
+ pkt_len = (*pkt_hdr >> 16) & 0x1fff;
+
+- /* Check CRC or runt packet */
+- if ((*pkt_hdr & AX_RXHDR_CRC_ERR) ||
+- (*pkt_hdr & AX_RXHDR_DROP_ERR)) {
+- skb_pull(skb, (pkt_len + 7) & 0xFFF8);
+- pkt_hdr++;
+- continue;
+- }
+-
+- if (pkt_cnt == 0) {
+- skb->len = pkt_len;
+- /* Skip IP alignment pseudo header */
+- skb_pull(skb, 2);
+- skb_set_tail_pointer(skb, skb->len);
+- skb->truesize = pkt_len + sizeof(struct sk_buff);
+- ax88179_rx_checksum(skb, pkt_hdr);
+- return 1;
+- }
++ if (pkt_len > skb->len)
++ return 0;
+
+- ax_skb = skb_clone(skb, GFP_ATOMIC);
+- if (ax_skb) {
++ /* Check CRC or runt packet */
++ if (((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) == 0) &&
++ pkt_len >= 2 + ETH_HLEN) {
++ bool last = (pkt_cnt == 0);
++
++ if (last) {
++ ax_skb = skb;
++ } else {
++ ax_skb = skb_clone(skb, GFP_ATOMIC);
++ if (!ax_skb)
++ return 0;
++ }
+ ax_skb->len = pkt_len;
+ /* Skip IP alignment pseudo header */
+ skb_pull(ax_skb, 2);
+ skb_set_tail_pointer(ax_skb, ax_skb->len);
+ ax_skb->truesize = pkt_len + sizeof(struct sk_buff);
+ ax88179_rx_checksum(ax_skb, pkt_hdr);
++
++ if (last)
++ return 1;
++
+ usbnet_skb_return(dev, ax_skb);
+- } else {
+- return 0;
+ }
+
+- skb_pull(skb, (pkt_len + 7) & 0xFFF8);
+- pkt_hdr++;
++ /* Trim this packet away from the SKB */
++ if (!skb_pull(skb, (pkt_len + 7) & 0xFFF8))
++ return 0;
+ }
+- return 1;
+ }
+
+ static struct sk_buff *
+diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c
+index eb3817d70f2b8..9b4dfa3001d6e 100644
+--- a/drivers/net/usb/cdc_ether.c
++++ b/drivers/net/usb/cdc_ether.c
+@@ -583,6 +583,11 @@ static const struct usb_device_id products[] = {
+ .bInterfaceSubClass = USB_CDC_SUBCLASS_ETHERNET, \
+ .bInterfaceProtocol = USB_CDC_PROTO_NONE
+
++#define ZAURUS_FAKE_INTERFACE \
++ .bInterfaceClass = USB_CLASS_COMM, \
++ .bInterfaceSubClass = USB_CDC_SUBCLASS_MDLM, \
++ .bInterfaceProtocol = USB_CDC_PROTO_NONE
++
+ /* SA-1100 based Sharp Zaurus ("collie"), or compatible;
+ * wire-incompatible with true CDC Ethernet implementations.
+ * (And, it seems, needlessly so...)
+@@ -636,6 +641,13 @@ static const struct usb_device_id products[] = {
+ .idProduct = 0x9032, /* SL-6000 */
+ ZAURUS_MASTER_INTERFACE,
+ .driver_info = 0,
++}, {
++ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
++ | USB_DEVICE_ID_MATCH_DEVICE,
++ .idVendor = 0x04DD,
++ .idProduct = 0x9032, /* SL-6000 */
++ ZAURUS_FAKE_INTERFACE,
++ .driver_info = 0,
+ }, {
+ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
+ | USB_DEVICE_ID_MATCH_DEVICE,
+diff --git a/drivers/net/usb/cdc_mbim.c b/drivers/net/usb/cdc_mbim.c
+index 82bb5ed94c485..c0b8b4aa78f37 100644
+--- a/drivers/net/usb/cdc_mbim.c
++++ b/drivers/net/usb/cdc_mbim.c
+@@ -659,6 +659,11 @@ static const struct usb_device_id mbim_devs[] = {
+ .driver_info = (unsigned long)&cdc_mbim_info_avoid_altsetting_toggle,
+ },
+
++ /* Telit FN990 */
++ { USB_DEVICE_AND_INTERFACE_INFO(0x1bc7, 0x1071, USB_CLASS_COMM, USB_CDC_SUBCLASS_MBIM, USB_CDC_PROTO_NONE),
++ .driver_info = (unsigned long)&cdc_mbim_info_avoid_altsetting_toggle,
++ },
++
+ /* default entry */
+ { USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_MBIM, USB_CDC_PROTO_NONE),
+ .driver_info = (unsigned long)&cdc_mbim_info_zlp,
+diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c
+index 24753a4da7e60..15f91d691bba3 100644
+--- a/drivers/net/usb/cdc_ncm.c
++++ b/drivers/net/usb/cdc_ncm.c
+@@ -181,6 +181,8 @@ static u32 cdc_ncm_check_tx_max(struct usbnet *dev, u32 new_tx)
+ min = ctx->max_datagram_size + ctx->max_ndp_size + sizeof(struct usb_cdc_ncm_nth32);
+
+ max = min_t(u32, CDC_NCM_NTB_MAX_SIZE_TX, le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize));
++ if (max == 0)
++ max = CDC_NCM_NTB_MAX_SIZE_TX; /* dwNtbOutMaxSize not set */
+
+ /* some devices set dwNtbOutMaxSize too low for the above default */
+ min = min(min, max);
+@@ -1713,10 +1715,10 @@ int cdc_ncm_rx_fixup(struct usbnet *dev, struct sk_buff *skb_in)
+ {
+ struct sk_buff *skb;
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
+- int len;
++ unsigned int len;
+ int nframes;
+ int x;
+- int offset;
++ unsigned int offset;
+ union {
+ struct usb_cdc_ncm_ndp16 *ndp16;
+ struct usb_cdc_ncm_ndp32 *ndp32;
+@@ -1788,8 +1790,8 @@ next_ndp:
+ break;
+ }
+
+- /* sanity checking */
+- if (((offset + len) > skb_in->len) ||
++ /* sanity checking - watch out for integer wrap*/
++ if ((offset > skb_in->len) || (len > skb_in->len - offset) ||
+ (len > ctx->rx_max) || (len < ETH_HLEN)) {
+ netif_dbg(dev, rx_err, dev->net,
+ "invalid frame detected (ignored) offset[%u]=%u, length=%u, skb=%p\n",
+diff --git a/drivers/net/usb/ipheth.c b/drivers/net/usb/ipheth.c
+index 06e2181e58108..d56e276e4d805 100644
+--- a/drivers/net/usb/ipheth.c
++++ b/drivers/net/usb/ipheth.c
+@@ -121,7 +121,7 @@ static int ipheth_alloc_urbs(struct ipheth_device *iphone)
+ if (tx_buf == NULL)
+ goto free_rx_urb;
+
+- rx_buf = usb_alloc_coherent(iphone->udev, IPHETH_BUF_SIZE,
++ rx_buf = usb_alloc_coherent(iphone->udev, IPHETH_BUF_SIZE + IPHETH_IP_ALIGN,
+ GFP_KERNEL, &rx_urb->transfer_dma);
+ if (rx_buf == NULL)
+ goto free_tx_buf;
+@@ -146,7 +146,7 @@ error_nomem:
+
+ static void ipheth_free_urbs(struct ipheth_device *iphone)
+ {
+- usb_free_coherent(iphone->udev, IPHETH_BUF_SIZE, iphone->rx_buf,
++ usb_free_coherent(iphone->udev, IPHETH_BUF_SIZE + IPHETH_IP_ALIGN, iphone->rx_buf,
+ iphone->rx_urb->transfer_dma);
+ usb_free_coherent(iphone->udev, IPHETH_BUF_SIZE, iphone->tx_buf,
+ iphone->tx_urb->transfer_dma);
+@@ -317,7 +317,7 @@ static int ipheth_rx_submit(struct ipheth_device *dev, gfp_t mem_flags)
+
+ usb_fill_bulk_urb(dev->rx_urb, udev,
+ usb_rcvbulkpipe(udev, dev->bulk_in),
+- dev->rx_buf, IPHETH_BUF_SIZE,
++ dev->rx_buf, IPHETH_BUF_SIZE + IPHETH_IP_ALIGN,
+ ipheth_rcvbulk_callback,
+ dev);
+ dev->rx_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
+index 63cd72c5f580c..3e1a83a22fdd6 100644
+--- a/drivers/net/usb/lan78xx.c
++++ b/drivers/net/usb/lan78xx.c
+@@ -76,6 +76,8 @@
+ #define LAN7801_USB_PRODUCT_ID (0x7801)
+ #define LAN78XX_EEPROM_MAGIC (0x78A5)
+ #define LAN78XX_OTP_MAGIC (0x78F3)
++#define AT29M2AF_USB_VENDOR_ID (0x07C9)
++#define AT29M2AF_USB_PRODUCT_ID (0x0012)
+
+ #define MII_READ 1
+ #define MII_WRITE 0
+@@ -2228,7 +2230,7 @@ static int lan78xx_phy_init(struct lan78xx_net *dev)
+ if (dev->domain_data.phyirq > 0)
+ phydev->irq = dev->domain_data.phyirq;
+ else
+- phydev->irq = 0;
++ phydev->irq = PHY_POLL;
+ netdev_dbg(dev->net, "phydev->irq = %d\n", phydev->irq);
+
+ /* set to AUTOMDIX */
+@@ -4734,6 +4736,10 @@ static const struct usb_device_id products[] = {
+ /* LAN7801 USB Gigabit Ethernet Device */
+ USB_DEVICE(LAN78XX_USB_VENDOR_ID, LAN7801_USB_PRODUCT_ID),
+ },
++ {
++ /* ATM2-AF USB Gigabit Ethernet Device */
++ USB_DEVICE(AT29M2AF_USB_VENDOR_ID, AT29M2AF_USB_PRODUCT_ID),
++ },
+ {},
+ };
+ MODULE_DEVICE_TABLE(usb, products);
+diff --git a/drivers/net/usb/mcs7830.c b/drivers/net/usb/mcs7830.c
+index 66866bef25df7..a31a3b9cbd58d 100644
+--- a/drivers/net/usb/mcs7830.c
++++ b/drivers/net/usb/mcs7830.c
+@@ -108,8 +108,16 @@ static const char driver_name[] = "MOSCHIP usb-ethernet driver";
+
+ static int mcs7830_get_reg(struct usbnet *dev, u16 index, u16 size, void *data)
+ {
+- return usbnet_read_cmd(dev, MCS7830_RD_BREQ, MCS7830_RD_BMREQ,
+- 0x0000, index, data, size);
++ int ret;
++
++ ret = usbnet_read_cmd(dev, MCS7830_RD_BREQ, MCS7830_RD_BMREQ,
++ 0x0000, index, data, size);
++ if (ret < 0)
++ return ret;
++ else if (ret < size)
++ return -ENODATA;
++
++ return ret;
+ }
+
+ static int mcs7830_set_reg(struct usbnet *dev, u16 index, u16 size, const void *data)
+diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c
+index 6a92a3fef75e5..cd063f45785b7 100644
+--- a/drivers/net/usb/pegasus.c
++++ b/drivers/net/usb/pegasus.c
+@@ -493,11 +493,11 @@ static void read_bulk_callback(struct urb *urb)
+ goto goon;
+
+ rx_status = buf[count - 2];
+- if (rx_status & 0x1e) {
++ if (rx_status & 0x1c) {
+ netif_dbg(pegasus, rx_err, net,
+ "RX packet error %x\n", rx_status);
+ net->stats.rx_errors++;
+- if (rx_status & 0x06) /* long or runt */
++ if (rx_status & 0x04) /* runt */
+ net->stats.rx_length_errors++;
+ if (rx_status & 0x08)
+ net->stats.rx_crc_errors++;
+diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
+index 33ada2c59952e..0c7f02ca6822b 100644
+--- a/drivers/net/usb/qmi_wwan.c
++++ b/drivers/net/usb/qmi_wwan.c
+@@ -1395,6 +1395,8 @@ static const struct usb_device_id products[] = {
+ {QMI_FIXED_INTF(0x413c, 0x81d7, 0)}, /* Dell Wireless 5821e */
+ {QMI_FIXED_INTF(0x413c, 0x81d7, 1)}, /* Dell Wireless 5821e preproduction config */
+ {QMI_FIXED_INTF(0x413c, 0x81e0, 0)}, /* Dell Wireless 5821e with eSIM support*/
++ {QMI_FIXED_INTF(0x413c, 0x81e4, 0)}, /* Dell Wireless 5829e with eSIM support*/
++ {QMI_FIXED_INTF(0x413c, 0x81e6, 0)}, /* Dell Wireless 5829e */
+ {QMI_FIXED_INTF(0x03f0, 0x4e1d, 8)}, /* HP lt4111 LTE/EV-DO/HSPA+ Gobi 4G Module */
+ {QMI_FIXED_INTF(0x03f0, 0x9d1d, 1)}, /* HP lt4120 Snapdragon X5 LTE */
+ {QMI_FIXED_INTF(0x22de, 0x9061, 3)}, /* WeTelecom WPD-600N */
+diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
+index f329e39100a7d..d467a9f3bb44d 100644
+--- a/drivers/net/usb/r8152.c
++++ b/drivers/net/usb/r8152.c
+@@ -32,7 +32,7 @@
+ #define NETNEXT_VERSION "12"
+
+ /* Information for net */
+-#define NET_VERSION "11"
++#define NET_VERSION "12"
+
+ #define DRIVER_VERSION "v1." NETNEXT_VERSION "." NET_VERSION
+ #define DRIVER_AUTHOR "Realtek linux nic maintainers <nic_swsd@realtek.com>"
+@@ -4016,6 +4016,11 @@ static void rtl_clear_bp(struct r8152 *tp, u16 type)
+ ocp_write_word(tp, type, PLA_BP_BA, 0);
+ }
+
++static inline void rtl_reset_ocp_base(struct r8152 *tp)
++{
++ tp->ocp_base = -1;
++}
++
+ static int rtl_phy_patch_request(struct r8152 *tp, bool request, bool wait)
+ {
+ u16 data, check;
+@@ -4087,8 +4092,6 @@ static int rtl_post_ram_code(struct r8152 *tp, u16 key_addr, bool wait)
+
+ rtl_phy_patch_request(tp, false, wait);
+
+- ocp_write_word(tp, MCU_TYPE_PLA, PLA_OCP_GPHY_BASE, tp->ocp_base);
+-
+ return 0;
+ }
+
+@@ -4800,6 +4803,8 @@ static void rtl_ram_code_speed_up(struct r8152 *tp, struct fw_phy_speed_up *phy,
+ u32 len;
+ u8 *data;
+
++ rtl_reset_ocp_base(tp);
++
+ if (sram_read(tp, SRAM_GPHY_FW_VER) >= __le16_to_cpu(phy->version)) {
+ dev_dbg(&tp->intf->dev, "PHY firmware has been the newest\n");
+ return;
+@@ -4845,7 +4850,8 @@ static void rtl_ram_code_speed_up(struct r8152 *tp, struct fw_phy_speed_up *phy,
+ }
+ }
+
+- ocp_write_word(tp, MCU_TYPE_PLA, PLA_OCP_GPHY_BASE, tp->ocp_base);
++ rtl_reset_ocp_base(tp);
++
+ rtl_phy_patch_request(tp, false, wait);
+
+ if (sram_read(tp, SRAM_GPHY_FW_VER) == __le16_to_cpu(phy->version))
+@@ -4861,6 +4867,8 @@ static int rtl8152_fw_phy_ver(struct r8152 *tp, struct fw_phy_ver *phy_ver)
+ ver_addr = __le16_to_cpu(phy_ver->ver.addr);
+ ver = __le16_to_cpu(phy_ver->ver.data);
+
++ rtl_reset_ocp_base(tp);
++
+ if (sram_read(tp, ver_addr) >= ver) {
+ dev_dbg(&tp->intf->dev, "PHY firmware has been the newest\n");
+ return 0;
+@@ -4877,6 +4885,8 @@ static void rtl8152_fw_phy_fixup(struct r8152 *tp, struct fw_phy_fixup *fix)
+ {
+ u16 addr, data;
+
++ rtl_reset_ocp_base(tp);
++
+ addr = __le16_to_cpu(fix->setting.addr);
+ data = ocp_reg_read(tp, addr);
+
+@@ -4908,6 +4918,8 @@ static void rtl8152_fw_phy_union_apply(struct r8152 *tp, struct fw_phy_union *ph
+ u32 length;
+ int i, num;
+
++ rtl_reset_ocp_base(tp);
++
+ num = phy->pre_num;
+ for (i = 0; i < num; i++)
+ sram_write(tp, __le16_to_cpu(phy->pre_set[i].addr),
+@@ -4938,6 +4950,8 @@ static void rtl8152_fw_phy_nc_apply(struct r8152 *tp, struct fw_phy_nc *phy)
+ u32 length, i, num;
+ __le16 *data;
+
++ rtl_reset_ocp_base(tp);
++
+ mode_reg = __le16_to_cpu(phy->mode_reg);
+ sram_write(tp, mode_reg, __le16_to_cpu(phy->mode_pre));
+ sram_write(tp, __le16_to_cpu(phy->ba_reg),
+@@ -5107,6 +5121,7 @@ post_fw:
+ if (rtl_fw->post_fw)
+ rtl_fw->post_fw(tp);
+
++ rtl_reset_ocp_base(tp);
+ strscpy(rtl_fw->version, fw_hdr->version, RTL_VER_SIZE);
+ dev_info(&tp->intf->dev, "load %s successfully\n", rtl_fw->version);
+ }
+@@ -6584,6 +6599,21 @@ static bool rtl8153_in_nway(struct r8152 *tp)
+ return true;
+ }
+
++static void r8156_mdio_force_mode(struct r8152 *tp)
++{
++ u16 data;
++
++ /* Select force mode through 0xa5b4 bit 15
++ * 0: MDIO force mode
++ * 1: MMD force mode
++ */
++ data = ocp_reg_read(tp, 0xa5b4);
++ if (data & BIT(15)) {
++ data &= ~BIT(15);
++ ocp_reg_write(tp, 0xa5b4, data);
++ }
++}
++
+ static void set_carrier(struct r8152 *tp)
+ {
+ struct net_device *netdev = tp->netdev;
+@@ -8016,6 +8046,7 @@ static void r8156_init(struct r8152 *tp)
+ ocp_data |= ACT_ODMA;
+ ocp_write_byte(tp, MCU_TYPE_USB, USB_BMU_CONFIG, ocp_data);
+
++ r8156_mdio_force_mode(tp);
+ rtl_tally_reset(tp);
+
+ tp->coalesce = 15000; /* 15 us */
+@@ -8145,6 +8176,7 @@ static void r8156b_init(struct r8152 *tp)
+ ocp_data &= ~(RX_AGG_DISABLE | RX_ZERO_EN);
+ ocp_write_word(tp, MCU_TYPE_USB, USB_USB_CTRL, ocp_data);
+
++ r8156_mdio_force_mode(tp);
+ rtl_tally_reset(tp);
+
+ tp->coalesce = 15000; /* 15 us */
+@@ -8467,6 +8499,8 @@ static int rtl8152_resume(struct usb_interface *intf)
+
+ mutex_lock(&tp->control);
+
++ rtl_reset_ocp_base(tp);
++
+ if (test_bit(SELECTIVE_SUSPEND, &tp->flags))
+ ret = rtl8152_runtime_resume(tp);
+ else
+@@ -8482,6 +8516,7 @@ static int rtl8152_reset_resume(struct usb_interface *intf)
+ struct r8152 *tp = usb_get_intfdata(intf);
+
+ clear_bit(SELECTIVE_SUSPEND, &tp->flags);
++ rtl_reset_ocp_base(tp);
+ tp->rtl_ops.init(tp);
+ queue_delayed_work(system_long_wq, &tp->hw_phy_work, 0);
+ set_ethernet_addr(tp, true);
+diff --git a/drivers/net/usb/rndis_host.c b/drivers/net/usb/rndis_host.c
+index 85a8b96e39a65..bedd36ab5cf01 100644
+--- a/drivers/net/usb/rndis_host.c
++++ b/drivers/net/usb/rndis_host.c
+@@ -608,6 +608,11 @@ static const struct usb_device_id products [] = {
+ USB_DEVICE_AND_INTERFACE_INFO(0x1630, 0x0042,
+ USB_CLASS_COMM, 2 /* ACM */, 0x0ff),
+ .driver_info = (unsigned long) &rndis_poll_status_info,
++}, {
++ /* Hytera Communications DMR radios' "Radio to PC Network" */
++ USB_VENDOR_AND_INTERFACE_INFO(0x238b,
++ USB_CLASS_COMM, 2 /* ACM */, 0x0ff),
++ .driver_info = (unsigned long)&rndis_info,
+ }, {
+ /* RNDIS is MSFT's un-official variant of CDC ACM */
+ USB_INTERFACE_INFO(USB_CLASS_COMM, 2 /* ACM */, 0x0ff),
+diff --git a/drivers/net/usb/smsc95xx.c b/drivers/net/usb/smsc95xx.c
+index 26b1bd8e845b4..eb0d325e92b76 100644
+--- a/drivers/net/usb/smsc95xx.c
++++ b/drivers/net/usb/smsc95xx.c
+@@ -84,9 +84,10 @@ static int __must_check __smsc95xx_read_reg(struct usbnet *dev, u32 index,
+ ret = fn(dev, USB_VENDOR_REQUEST_READ_REGISTER, USB_DIR_IN
+ | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+ 0, index, &buf, 4);
+- if (unlikely(ret < 0)) {
+- netdev_warn(dev->net, "Failed to read reg index 0x%08x: %d\n",
+- index, ret);
++ if (ret < 0) {
++ if (ret != -ENODEV)
++ netdev_warn(dev->net, "Failed to read reg index 0x%08x: %d\n",
++ index, ret);
+ return ret;
+ }
+
+@@ -116,7 +117,7 @@ static int __must_check __smsc95xx_write_reg(struct usbnet *dev, u32 index,
+ ret = fn(dev, USB_VENDOR_REQUEST_WRITE_REGISTER, USB_DIR_OUT
+ | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+ 0, index, &buf, 4);
+- if (unlikely(ret < 0))
++ if (ret < 0 && ret != -ENODEV)
+ netdev_warn(dev->net, "Failed to write reg index 0x%08x: %d\n",
+ index, ret);
+
+@@ -159,6 +160,9 @@ static int __must_check __smsc95xx_phy_wait_not_busy(struct usbnet *dev,
+ do {
+ ret = __smsc95xx_read_reg(dev, MII_ADDR, &val, in_pm);
+ if (ret < 0) {
++ /* Ignore -ENODEV error during disconnect() */
++ if (ret == -ENODEV)
++ return 0;
+ netdev_warn(dev->net, "Error reading MII_ACCESS\n");
+ return ret;
+ }
+@@ -194,7 +198,8 @@ static int __smsc95xx_mdio_read(struct usbnet *dev, int phy_id, int idx,
+ addr = mii_address_cmd(phy_id, idx, MII_READ_ | MII_BUSY_);
+ ret = __smsc95xx_write_reg(dev, MII_ADDR, addr, in_pm);
+ if (ret < 0) {
+- netdev_warn(dev->net, "Error writing MII_ADDR\n");
++ if (ret != -ENODEV)
++ netdev_warn(dev->net, "Error writing MII_ADDR\n");
+ goto done;
+ }
+
+@@ -206,7 +211,8 @@ static int __smsc95xx_mdio_read(struct usbnet *dev, int phy_id, int idx,
+
+ ret = __smsc95xx_read_reg(dev, MII_DATA, &val, in_pm);
+ if (ret < 0) {
+- netdev_warn(dev->net, "Error reading MII_DATA\n");
++ if (ret != -ENODEV)
++ netdev_warn(dev->net, "Error reading MII_DATA\n");
+ goto done;
+ }
+
+@@ -214,6 +220,10 @@ static int __smsc95xx_mdio_read(struct usbnet *dev, int phy_id, int idx,
+
+ done:
+ mutex_unlock(&dev->phy_mutex);
++
++ /* Ignore -ENODEV error during disconnect() */
++ if (ret == -ENODEV)
++ return 0;
+ return ret;
+ }
+
+@@ -235,7 +245,8 @@ static void __smsc95xx_mdio_write(struct usbnet *dev, int phy_id,
+ val = regval;
+ ret = __smsc95xx_write_reg(dev, MII_DATA, val, in_pm);
+ if (ret < 0) {
+- netdev_warn(dev->net, "Error writing MII_DATA\n");
++ if (ret != -ENODEV)
++ netdev_warn(dev->net, "Error writing MII_DATA\n");
+ goto done;
+ }
+
+@@ -243,7 +254,8 @@ static void __smsc95xx_mdio_write(struct usbnet *dev, int phy_id,
+ addr = mii_address_cmd(phy_id, idx, MII_WRITE_ | MII_BUSY_);
+ ret = __smsc95xx_write_reg(dev, MII_ADDR, addr, in_pm);
+ if (ret < 0) {
+- netdev_warn(dev->net, "Error writing MII_ADDR\n");
++ if (ret != -ENODEV)
++ netdev_warn(dev->net, "Error writing MII_ADDR\n");
+ goto done;
+ }
+
+@@ -1049,6 +1061,14 @@ static const struct net_device_ops smsc95xx_netdev_ops = {
+ .ndo_set_features = smsc95xx_set_features,
+ };
+
++static void smsc95xx_handle_link_change(struct net_device *net)
++{
++ struct usbnet *dev = netdev_priv(net);
++
++ phy_print_status(net->phydev);
++ usbnet_defer_kevent(dev, EVENT_LINK_CHANGE);
++}
++
+ static int smsc95xx_bind(struct usbnet *dev, struct usb_interface *intf)
+ {
+ struct smsc95xx_priv *pdata;
+@@ -1153,6 +1173,17 @@ static int smsc95xx_bind(struct usbnet *dev, struct usb_interface *intf)
+ dev->net->min_mtu = ETH_MIN_MTU;
+ dev->net->max_mtu = ETH_DATA_LEN;
+ dev->hard_mtu = dev->net->mtu + dev->net->hard_header_len;
++
++ ret = phy_connect_direct(dev->net, pdata->phydev,
++ &smsc95xx_handle_link_change,
++ PHY_INTERFACE_MODE_MII);
++ if (ret) {
++ netdev_err(dev->net, "can't attach PHY to %s\n", pdata->mdiobus->id);
++ goto unregister_mdio;
++ }
++
++ phy_attached_info(dev->net->phydev);
++
+ return 0;
+
+ unregister_mdio:
+@@ -1170,47 +1201,25 @@ static void smsc95xx_unbind(struct usbnet *dev, struct usb_interface *intf)
+ {
+ struct smsc95xx_priv *pdata = dev->driver_priv;
+
++ phy_disconnect(dev->net->phydev);
+ mdiobus_unregister(pdata->mdiobus);
+ mdiobus_free(pdata->mdiobus);
+ netif_dbg(dev, ifdown, dev->net, "free pdata\n");
+ kfree(pdata);
+ }
+
+-static void smsc95xx_handle_link_change(struct net_device *net)
+-{
+- struct usbnet *dev = netdev_priv(net);
+-
+- phy_print_status(net->phydev);
+- usbnet_defer_kevent(dev, EVENT_LINK_CHANGE);
+-}
+-
+ static int smsc95xx_start_phy(struct usbnet *dev)
+ {
+- struct smsc95xx_priv *pdata = dev->driver_priv;
+- struct net_device *net = dev->net;
+- int ret;
+-
+- ret = smsc95xx_reset(dev);
+- if (ret < 0)
+- return ret;
++ phy_start(dev->net->phydev);
+
+- ret = phy_connect_direct(net, pdata->phydev,
+- &smsc95xx_handle_link_change,
+- PHY_INTERFACE_MODE_MII);
+- if (ret) {
+- netdev_err(net, "can't attach PHY to %s\n", pdata->mdiobus->id);
+- return ret;
+- }
+-
+- phy_attached_info(net->phydev);
+- phy_start(net->phydev);
+ return 0;
+ }
+
+-static int smsc95xx_disconnect_phy(struct usbnet *dev)
++static int smsc95xx_stop(struct usbnet *dev)
+ {
+- phy_stop(dev->net->phydev);
+- phy_disconnect(dev->net->phydev);
++ if (dev->net->phydev)
++ phy_stop(dev->net->phydev);
++
+ return 0;
+ }
+
+@@ -1964,8 +1973,9 @@ static const struct driver_info smsc95xx_info = {
+ .bind = smsc95xx_bind,
+ .unbind = smsc95xx_unbind,
+ .link_reset = smsc95xx_link_reset,
+- .reset = smsc95xx_start_phy,
+- .stop = smsc95xx_disconnect_phy,
++ .reset = smsc95xx_reset,
++ .check_connect = smsc95xx_start_phy,
++ .stop = smsc95xx_stop,
+ .rx_fixup = smsc95xx_rx_fixup,
+ .tx_fixup = smsc95xx_tx_fixup,
+ .status = smsc95xx_status,
+diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c
+index 6516a37893e27..0c50f24671da3 100644
+--- a/drivers/net/usb/sr9700.c
++++ b/drivers/net/usb/sr9700.c
+@@ -410,7 +410,7 @@ static int sr9700_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
+ /* ignore the CRC length */
+ len = (skb->data[1] | (skb->data[2] << 8)) - 4;
+
+- if (len > ETH_FRAME_LEN)
++ if (len > ETH_FRAME_LEN || len > skb->len)
+ return 0;
+
+ /* the last packet of current skb */
+diff --git a/drivers/net/usb/zaurus.c b/drivers/net/usb/zaurus.c
+index 8e717a0b559b3..7984f2157d222 100644
+--- a/drivers/net/usb/zaurus.c
++++ b/drivers/net/usb/zaurus.c
+@@ -256,6 +256,11 @@ static const struct usb_device_id products [] = {
+ .bInterfaceSubClass = USB_CDC_SUBCLASS_ETHERNET, \
+ .bInterfaceProtocol = USB_CDC_PROTO_NONE
+
++#define ZAURUS_FAKE_INTERFACE \
++ .bInterfaceClass = USB_CLASS_COMM, \
++ .bInterfaceSubClass = USB_CDC_SUBCLASS_MDLM, \
++ .bInterfaceProtocol = USB_CDC_PROTO_NONE
++
+ /* SA-1100 based Sharp Zaurus ("collie"), or compatible. */
+ {
+ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
+@@ -313,6 +318,13 @@ static const struct usb_device_id products [] = {
+ .idProduct = 0x9032, /* SL-6000 */
+ ZAURUS_MASTER_INTERFACE,
+ .driver_info = ZAURUS_PXA_INFO,
++}, {
++ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
++ | USB_DEVICE_ID_MATCH_DEVICE,
++ .idVendor = 0x04DD,
++ .idProduct = 0x9032, /* SL-6000 */
++ ZAURUS_FAKE_INTERFACE,
++ .driver_info = (unsigned long)&bogus_mdlm_info,
+ }, {
+ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
+ | USB_DEVICE_ID_MATCH_DEVICE,
+diff --git a/drivers/net/veth.c b/drivers/net/veth.c
+index 50eb43e5bf459..64fa8e9c0a22b 100644
+--- a/drivers/net/veth.c
++++ b/drivers/net/veth.c
+@@ -272,9 +272,10 @@ static void __veth_xdp_flush(struct veth_rq *rq)
+ {
+ /* Write ptr_ring before reading rx_notify_masked */
+ smp_mb();
+- if (!rq->rx_notify_masked) {
+- rq->rx_notify_masked = true;
+- napi_schedule(&rq->xdp_napi);
++ if (!READ_ONCE(rq->rx_notify_masked) &&
++ napi_schedule_prep(&rq->xdp_napi)) {
++ WRITE_ONCE(rq->rx_notify_masked, true);
++ __napi_schedule(&rq->xdp_napi);
+ }
+ }
+
+@@ -326,7 +327,7 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
+
+ rcu_read_lock();
+ rcv = rcu_dereference(priv->peer);
+- if (unlikely(!rcv)) {
++ if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) {
+ kfree_skb(skb);
+ goto drop;
+ }
+@@ -342,7 +343,6 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
+ */
+ use_napi = rcu_access_pointer(rq->napi) &&
+ veth_skb_is_eligible_for_gro(dev, rcv, skb);
+- skb_record_rx_queue(skb, rxq);
+ }
+
+ skb_tx_timestamp(skb);
+@@ -879,8 +879,12 @@ static int veth_xdp_rcv(struct veth_rq *rq, int budget,
+
+ stats->xdp_bytes += skb->len;
+ skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
+- if (skb)
+- napi_gro_receive(&rq->xdp_napi, skb);
++ if (skb) {
++ if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC))
++ netif_receive_skb(skb);
++ else
++ napi_gro_receive(&rq->xdp_napi, skb);
++ }
+ }
+ done++;
+ }
+@@ -916,8 +920,10 @@ static int veth_poll(struct napi_struct *napi, int budget)
+ /* Write rx_notify_masked before reading ptr_ring */
+ smp_store_mb(rq->rx_notify_masked, false);
+ if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) {
+- rq->rx_notify_masked = true;
+- napi_schedule(&rq->xdp_napi);
++ if (napi_schedule_prep(&rq->xdp_napi)) {
++ WRITE_ONCE(rq->rx_notify_masked, true);
++ __napi_schedule(&rq->xdp_napi);
++ }
+ }
+ }
+
+diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
+index 4ad25a8b0870c..73aba760e10c6 100644
+--- a/drivers/net/virtio_net.c
++++ b/drivers/net/virtio_net.c
+@@ -965,6 +965,24 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
+ * xdp.data_meta were adjusted
+ */
+ len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
++
++ /* recalculate headroom if xdp.data or xdp_data_meta
++ * were adjusted, note that offset should always point
++ * to the start of the reserved bytes for virtio_net
++ * header which are followed by xdp.data, that means
++ * that offset is equal to the headroom (when buf is
++ * starting at the beginning of the page, otherwise
++ * there is a base offset inside the page) but it's used
++ * with a different starting point (buf start) than
++ * xdp.data (buf start + vnet hdr size). If xdp.data or
++ * data_meta were adjusted by the xdp prog then the
++ * headroom size has changed and so has the offset, we
++ * can use data_hard_start, which points at buf start +
++ * vnet hdr size, to calculate the new headroom and use
++ * it later to compute buf start in page_to_skb()
++ */
++ headroom = xdp.data - xdp.data_hard_start - metasize;
++
+ /* We can only create skb based on xdp_page. */
+ if (unlikely(xdp_page != page)) {
+ rcu_read_unlock();
+@@ -972,7 +990,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
+ head_skb = page_to_skb(vi, rq, xdp_page, offset,
+ len, PAGE_SIZE, false,
+ metasize,
+- VIRTIO_XDP_HEADROOM);
++ headroom);
+ return head_skb;
+ }
+ break;
+diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
+index 8799854bacb29..bc3192cf48e3e 100644
+--- a/drivers/net/vmxnet3/vmxnet3_drv.c
++++ b/drivers/net/vmxnet3/vmxnet3_drv.c
+@@ -589,6 +589,7 @@ vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
+ if (dma_mapping_error(&adapter->pdev->dev,
+ rbi->dma_addr)) {
+ dev_kfree_skb_any(rbi->skb);
++ rbi->skb = NULL;
+ rq->stats.rx_buf_alloc_failure++;
+ break;
+ }
+@@ -613,6 +614,7 @@ vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
+ if (dma_mapping_error(&adapter->pdev->dev,
+ rbi->dma_addr)) {
+ put_page(rbi->page);
++ rbi->page = NULL;
+ rq->stats.rx_buf_alloc_failure++;
+ break;
+ }
+@@ -1666,6 +1668,10 @@ vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
+ u32 i, ring_idx;
+ struct Vmxnet3_RxDesc *rxd;
+
++ /* ring has already been cleaned up */
++ if (!rq->rx_ring[0].base)
++ return;
++
+ for (ring_idx = 0; ring_idx < 2; ring_idx++) {
+ for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
+ #ifdef __BIG_ENDIAN_BITFIELD
+@@ -3261,7 +3267,7 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
+
+ #ifdef CONFIG_PCI_MSI
+ if (adapter->intr.type == VMXNET3_IT_MSIX) {
+- int i, nvec;
++ int i, nvec, nvec_allocated;
+
+ nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
+ 1 : adapter->num_tx_queues;
+@@ -3274,14 +3280,15 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
+ for (i = 0; i < nvec; i++)
+ adapter->intr.msix_entries[i].entry = i;
+
+- nvec = vmxnet3_acquire_msix_vectors(adapter, nvec);
+- if (nvec < 0)
++ nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec);
++ if (nvec_allocated < 0)
+ goto msix_err;
+
+ /* If we cannot allocate one MSIx vector per queue
+ * then limit the number of rx queues to 1
+ */
+- if (nvec == VMXNET3_LINUX_MIN_MSIX_VECT) {
++ if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT &&
++ nvec != VMXNET3_LINUX_MIN_MSIX_VECT) {
+ if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
+ || adapter->num_rx_queues != 1) {
+ adapter->share_intr = VMXNET3_INTR_TXSHARE;
+@@ -3291,14 +3298,14 @@ vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
+ }
+ }
+
+- adapter->intr.num_intrs = nvec;
++ adapter->intr.num_intrs = nvec_allocated;
+ return;
+
+ msix_err:
+ /* If we cannot allocate MSIx vectors use only one rx queue */
+ dev_info(&adapter->pdev->dev,
+ "Failed to enable MSI-X, error %d. "
+- "Limiting #rx queues to 1, try MSI.\n", nvec);
++ "Limiting #rx queues to 1, try MSI.\n", nvec_allocated);
+
+ adapter->intr.type = VMXNET3_IT_MSI;
+ }
+diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
+index 662e261173539..091dd7caf10cc 100644
+--- a/drivers/net/vrf.c
++++ b/drivers/net/vrf.c
+@@ -35,6 +35,7 @@
+ #include <net/l3mdev.h>
+ #include <net/fib_rules.h>
+ #include <net/netns/generic.h>
++#include <net/netfilter/nf_conntrack.h>
+
+ #define DRV_NAME "vrf"
+ #define DRV_VERSION "1.1"
+@@ -424,12 +425,26 @@ static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
+ return NETDEV_TX_OK;
+ }
+
++static void vrf_nf_set_untracked(struct sk_buff *skb)
++{
++ if (skb_get_nfct(skb) == 0)
++ nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
++}
++
++static void vrf_nf_reset_ct(struct sk_buff *skb)
++{
++ if (skb_get_nfct(skb) == IP_CT_UNTRACKED)
++ nf_reset_ct(skb);
++}
++
+ #if IS_ENABLED(CONFIG_IPV6)
+ static int vrf_ip6_local_out(struct net *net, struct sock *sk,
+ struct sk_buff *skb)
+ {
+ int err;
+
++ vrf_nf_reset_ct(skb);
++
+ err = nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT, net,
+ sk, skb, NULL, skb_dst(skb)->dev, dst_output);
+
+@@ -482,6 +497,7 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
+ /* strip the ethernet header added for pass through VRF device */
+ __skb_pull(skb, skb_network_offset(skb));
+
++ memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
+ ret = vrf_ip6_local_out(net, skb->sk, skb);
+ if (unlikely(net_xmit_eval(ret)))
+ dev->stats.tx_errors++;
+@@ -508,6 +524,8 @@ static int vrf_ip_local_out(struct net *net, struct sock *sk,
+ {
+ int err;
+
++ vrf_nf_reset_ct(skb);
++
+ err = nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, net, sk,
+ skb, NULL, skb_dst(skb)->dev, dst_output);
+ if (likely(err == 1))
+@@ -562,6 +580,7 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
+ RT_SCOPE_LINK);
+ }
+
++ memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
+ ret = vrf_ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
+ if (unlikely(net_xmit_eval(ret)))
+ vrf_dev->stats.tx_errors++;
+@@ -626,8 +645,7 @@ static void vrf_finish_direct(struct sk_buff *skb)
+ skb_pull(skb, ETH_HLEN);
+ }
+
+- /* reset skb device */
+- nf_reset_ct(skb);
++ vrf_nf_reset_ct(skb);
+ }
+
+ #if IS_ENABLED(CONFIG_IPV6)
+@@ -641,7 +659,7 @@ static int vrf_finish_output6(struct net *net, struct sock *sk,
+ struct neighbour *neigh;
+ int ret;
+
+- nf_reset_ct(skb);
++ vrf_nf_reset_ct(skb);
+
+ skb->protocol = htons(ETH_P_IPV6);
+ skb->dev = dev;
+@@ -772,6 +790,8 @@ static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
+ if (rt6_need_strict(&ipv6_hdr(skb)->daddr))
+ return skb;
+
++ vrf_nf_set_untracked(skb);
++
+ if (qdisc_tx_is_default(vrf_dev) ||
+ IP6CB(skb)->flags & IP6SKB_XFRM_TRANSFORMED)
+ return vrf_ip6_out_direct(vrf_dev, sk, skb);
+@@ -858,7 +878,7 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s
+ struct neighbour *neigh;
+ bool is_v6gw = false;
+
+- nf_reset_ct(skb);
++ vrf_nf_reset_ct(skb);
+
+ /* Be paranoid, rather than too clever. */
+ if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
+@@ -1001,6 +1021,8 @@ static struct sk_buff *vrf_ip_out(struct net_device *vrf_dev,
+ ipv4_is_lbcast(ip_hdr(skb)->daddr))
+ return skb;
+
++ vrf_nf_set_untracked(skb);
++
+ if (qdisc_tx_is_default(vrf_dev) ||
+ IPCB(skb)->flags & IPSKB_XFRM_TRANSFORMED)
+ return vrf_ip_out_direct(vrf_dev, sk, skb);
+@@ -1243,6 +1265,7 @@ static int vrf_prepare_mac_header(struct sk_buff *skb,
+ eth = (struct ethhdr *)skb->data;
+
+ skb_reset_mac_header(skb);
++ skb_reset_mac_len(skb);
+
+ /* we set the ethernet destination and the source addresses to the
+ * address of the VRF device.
+@@ -1272,9 +1295,9 @@ static int vrf_prepare_mac_header(struct sk_buff *skb,
+ */
+ static int vrf_add_mac_header_if_unset(struct sk_buff *skb,
+ struct net_device *vrf_dev,
+- u16 proto)
++ u16 proto, struct net_device *orig_dev)
+ {
+- if (skb_mac_header_was_set(skb))
++ if (skb_mac_header_was_set(skb) && dev_has_header(orig_dev))
+ return 0;
+
+ return vrf_prepare_mac_header(skb, vrf_dev, proto);
+@@ -1380,6 +1403,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
+
+ /* if packet is NDISC then keep the ingress interface */
+ if (!is_ndisc) {
++ struct net_device *orig_dev = skb->dev;
++
+ vrf_rx_stats(vrf_dev, skb->len);
+ skb->dev = vrf_dev;
+ skb->skb_iif = vrf_dev->ifindex;
+@@ -1388,7 +1413,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
+ int err;
+
+ err = vrf_add_mac_header_if_unset(skb, vrf_dev,
+- ETH_P_IPV6);
++ ETH_P_IPV6,
++ orig_dev);
+ if (likely(!err)) {
+ skb_push(skb, skb->mac_len);
+ dev_queue_xmit_nit(skb, vrf_dev);
+@@ -1418,6 +1444,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
+ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev,
+ struct sk_buff *skb)
+ {
++ struct net_device *orig_dev = skb->dev;
++
+ skb->dev = vrf_dev;
+ skb->skb_iif = vrf_dev->ifindex;
+ IPCB(skb)->flags |= IPSKB_L3SLAVE;
+@@ -1438,7 +1466,8 @@ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev,
+ if (!list_empty(&vrf_dev->ptype_all)) {
+ int err;
+
+- err = vrf_add_mac_header_if_unset(skb, vrf_dev, ETH_P_IP);
++ err = vrf_add_mac_header_if_unset(skb, vrf_dev, ETH_P_IP,
++ orig_dev);
+ if (likely(!err)) {
+ skb_push(skb, skb->mac_len);
+ dev_queue_xmit_nit(skb, vrf_dev);
+diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
+index 141635a35c28a..129e270e9a7cd 100644
+--- a/drivers/net/vxlan.c
++++ b/drivers/net/vxlan.c
+@@ -711,11 +711,11 @@ static int vxlan_fdb_append(struct vxlan_fdb *f,
+
+ rd = kmalloc(sizeof(*rd), GFP_ATOMIC);
+ if (rd == NULL)
+- return -ENOBUFS;
++ return -ENOMEM;
+
+ if (dst_cache_init(&rd->dst_cache, GFP_ATOMIC)) {
+ kfree(rd);
+- return -ENOBUFS;
++ return -ENOMEM;
+ }
+
+ rd->remote_ip = *ip;
+diff --git a/drivers/net/wireguard/allowedips.c b/drivers/net/wireguard/allowedips.c
+index b7197e80f2264..9a4c8ff32d9dd 100644
+--- a/drivers/net/wireguard/allowedips.c
++++ b/drivers/net/wireguard/allowedips.c
+@@ -163,7 +163,7 @@ static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key,
+ return exact;
+ }
+
+-static inline void connect_node(struct allowedips_node **parent, u8 bit, struct allowedips_node *node)
++static inline void connect_node(struct allowedips_node __rcu **parent, u8 bit, struct allowedips_node *node)
+ {
+ node->parent_bit_packed = (unsigned long)parent | bit;
+ rcu_assign_pointer(*parent, node);
+diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c
+index 551ddaaaf5400..5eaef79c06e16 100644
+--- a/drivers/net/wireguard/device.c
++++ b/drivers/net/wireguard/device.c
+@@ -19,6 +19,7 @@
+ #include <linux/if_arp.h>
+ #include <linux/icmp.h>
+ #include <linux/suspend.h>
++#include <net/dst_metadata.h>
+ #include <net/icmp.h>
+ #include <net/rtnetlink.h>
+ #include <net/ip_tunnels.h>
+@@ -98,6 +99,7 @@ static int wg_stop(struct net_device *dev)
+ {
+ struct wg_device *wg = netdev_priv(dev);
+ struct wg_peer *peer;
++ struct sk_buff *skb;
+
+ mutex_lock(&wg->device_update_lock);
+ list_for_each_entry(peer, &wg->peer_list, peer_list) {
+@@ -108,7 +110,9 @@ static int wg_stop(struct net_device *dev)
+ wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
+ }
+ mutex_unlock(&wg->device_update_lock);
+- skb_queue_purge(&wg->incoming_handshakes);
++ while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
++ kfree_skb(skb);
++ atomic_set(&wg->handshake_queue_len, 0);
+ wg_socket_reinit(wg, NULL, NULL);
+ return 0;
+ }
+@@ -149,7 +153,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
+ goto err_peer;
+ }
+
+- mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
++ mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
+
+ __skb_queue_head_init(&packets);
+ if (!skb_is_gso(skb)) {
+@@ -235,14 +239,13 @@ static void wg_destruct(struct net_device *dev)
+ destroy_workqueue(wg->handshake_receive_wq);
+ destroy_workqueue(wg->handshake_send_wq);
+ destroy_workqueue(wg->packet_crypt_wq);
+- wg_packet_queue_free(&wg->decrypt_queue);
+- wg_packet_queue_free(&wg->encrypt_queue);
++ wg_packet_queue_free(&wg->handshake_queue, true);
++ wg_packet_queue_free(&wg->decrypt_queue, false);
++ wg_packet_queue_free(&wg->encrypt_queue, false);
+ rcu_barrier(); /* Wait for all the peers to be actually freed. */
+ wg_ratelimiter_uninit();
+ memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
+- skb_queue_purge(&wg->incoming_handshakes);
+ free_percpu(dev->tstats);
+- free_percpu(wg->incoming_handshakes_worker);
+ kvfree(wg->index_hashtable);
+ kvfree(wg->peer_hashtable);
+ mutex_unlock(&wg->device_update_lock);
+@@ -298,7 +301,6 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
+ init_rwsem(&wg->static_identity.lock);
+ mutex_init(&wg->socket_update_lock);
+ mutex_init(&wg->device_update_lock);
+- skb_queue_head_init(&wg->incoming_handshakes);
+ wg_allowedips_init(&wg->peer_allowedips);
+ wg_cookie_checker_init(&wg->cookie_checker, wg);
+ INIT_LIST_HEAD(&wg->peer_list);
+@@ -316,16 +318,10 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
+ if (!dev->tstats)
+ goto err_free_index_hashtable;
+
+- wg->incoming_handshakes_worker =
+- wg_packet_percpu_multicore_worker_alloc(
+- wg_packet_handshake_receive_worker, wg);
+- if (!wg->incoming_handshakes_worker)
+- goto err_free_tstats;
+-
+ wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
+ WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
+ if (!wg->handshake_receive_wq)
+- goto err_free_incoming_handshakes;
++ goto err_free_tstats;
+
+ wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
+ WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
+@@ -347,10 +343,15 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
+ if (ret < 0)
+ goto err_free_encrypt_queue;
+
+- ret = wg_ratelimiter_init();
++ ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
++ MAX_QUEUED_INCOMING_HANDSHAKES);
+ if (ret < 0)
+ goto err_free_decrypt_queue;
+
++ ret = wg_ratelimiter_init();
++ if (ret < 0)
++ goto err_free_handshake_queue;
++
+ ret = register_netdevice(dev);
+ if (ret < 0)
+ goto err_uninit_ratelimiter;
+@@ -367,18 +368,18 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
+
+ err_uninit_ratelimiter:
+ wg_ratelimiter_uninit();
++err_free_handshake_queue:
++ wg_packet_queue_free(&wg->handshake_queue, false);
+ err_free_decrypt_queue:
+- wg_packet_queue_free(&wg->decrypt_queue);
++ wg_packet_queue_free(&wg->decrypt_queue, false);
+ err_free_encrypt_queue:
+- wg_packet_queue_free(&wg->encrypt_queue);
++ wg_packet_queue_free(&wg->encrypt_queue, false);
+ err_destroy_packet_crypt:
+ destroy_workqueue(wg->packet_crypt_wq);
+ err_destroy_handshake_send:
+ destroy_workqueue(wg->handshake_send_wq);
+ err_destroy_handshake_receive:
+ destroy_workqueue(wg->handshake_receive_wq);
+-err_free_incoming_handshakes:
+- free_percpu(wg->incoming_handshakes_worker);
+ err_free_tstats:
+ free_percpu(dev->tstats);
+ err_free_index_hashtable:
+@@ -398,6 +399,7 @@ static struct rtnl_link_ops link_ops __read_mostly = {
+ static void wg_netns_pre_exit(struct net *net)
+ {
+ struct wg_device *wg;
++ struct wg_peer *peer;
+
+ rtnl_lock();
+ list_for_each_entry(wg, &device_list, device_list) {
+@@ -407,6 +409,8 @@ static void wg_netns_pre_exit(struct net *net)
+ mutex_lock(&wg->device_update_lock);
+ rcu_assign_pointer(wg->creating_net, NULL);
+ wg_socket_reinit(wg, NULL, NULL);
++ list_for_each_entry(peer, &wg->peer_list, peer_list)
++ wg_socket_clear_peer_endpoint_src(peer);
+ mutex_unlock(&wg->device_update_lock);
+ }
+ }
+diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h
+index 854bc3d97150e..43c7cebbf50b0 100644
+--- a/drivers/net/wireguard/device.h
++++ b/drivers/net/wireguard/device.h
+@@ -39,21 +39,18 @@ struct prev_queue {
+
+ struct wg_device {
+ struct net_device *dev;
+- struct crypt_queue encrypt_queue, decrypt_queue;
++ struct crypt_queue encrypt_queue, decrypt_queue, handshake_queue;
+ struct sock __rcu *sock4, *sock6;
+ struct net __rcu *creating_net;
+ struct noise_static_identity static_identity;
+- struct workqueue_struct *handshake_receive_wq, *handshake_send_wq;
+- struct workqueue_struct *packet_crypt_wq;
+- struct sk_buff_head incoming_handshakes;
+- int incoming_handshake_cpu;
+- struct multicore_worker __percpu *incoming_handshakes_worker;
++ struct workqueue_struct *packet_crypt_wq,*handshake_receive_wq, *handshake_send_wq;
+ struct cookie_checker cookie_checker;
+ struct pubkey_hashtable *peer_hashtable;
+ struct index_hashtable *index_hashtable;
+ struct allowedips peer_allowedips;
+ struct mutex device_update_lock, socket_update_lock;
+ struct list_head device_list, peer_list;
++ atomic_t handshake_queue_len;
+ unsigned int num_peers, device_update_gen;
+ u32 fwmark;
+ u16 incoming_port;
+diff --git a/drivers/net/wireguard/noise.c b/drivers/net/wireguard/noise.c
+index c0cfd9b36c0b5..720952b92e784 100644
+--- a/drivers/net/wireguard/noise.c
++++ b/drivers/net/wireguard/noise.c
+@@ -302,6 +302,41 @@ void wg_noise_set_static_identity_private_key(
+ static_identity->static_public, private_key);
+ }
+
++static void hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, const size_t keylen)
++{
++ struct blake2s_state state;
++ u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 };
++ u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32));
++ int i;
++
++ if (keylen > BLAKE2S_BLOCK_SIZE) {
++ blake2s_init(&state, BLAKE2S_HASH_SIZE);
++ blake2s_update(&state, key, keylen);
++ blake2s_final(&state, x_key);
++ } else
++ memcpy(x_key, key, keylen);
++
++ for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
++ x_key[i] ^= 0x36;
++
++ blake2s_init(&state, BLAKE2S_HASH_SIZE);
++ blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
++ blake2s_update(&state, in, inlen);
++ blake2s_final(&state, i_hash);
++
++ for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
++ x_key[i] ^= 0x5c ^ 0x36;
++
++ blake2s_init(&state, BLAKE2S_HASH_SIZE);
++ blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
++ blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE);
++ blake2s_final(&state, i_hash);
++
++ memcpy(out, i_hash, BLAKE2S_HASH_SIZE);
++ memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE);
++ memzero_explicit(i_hash, BLAKE2S_HASH_SIZE);
++}
++
+ /* This is Hugo Krawczyk's HKDF:
+ * - https://eprint.iacr.org/2010/264.pdf
+ * - https://tools.ietf.org/html/rfc5869
+@@ -322,14 +357,14 @@ static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data,
+ ((third_len || third_dst) && (!second_len || !second_dst))));
+
+ /* Extract entropy from data into secret */
+- blake2s256_hmac(secret, data, chaining_key, data_len, NOISE_HASH_LEN);
++ hmac(secret, data, chaining_key, data_len, NOISE_HASH_LEN);
+
+ if (!first_dst || !first_len)
+ goto out;
+
+ /* Expand first key: key = secret, data = 0x1 */
+ output[0] = 1;
+- blake2s256_hmac(output, output, secret, 1, BLAKE2S_HASH_SIZE);
++ hmac(output, output, secret, 1, BLAKE2S_HASH_SIZE);
+ memcpy(first_dst, output, first_len);
+
+ if (!second_dst || !second_len)
+@@ -337,8 +372,7 @@ static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data,
+
+ /* Expand second key: key = secret, data = first-key || 0x2 */
+ output[BLAKE2S_HASH_SIZE] = 2;
+- blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1,
+- BLAKE2S_HASH_SIZE);
++ hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE);
+ memcpy(second_dst, output, second_len);
+
+ if (!third_dst || !third_len)
+@@ -346,8 +380,7 @@ static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data,
+
+ /* Expand third key: key = secret, data = second-key || 0x3 */
+ output[BLAKE2S_HASH_SIZE] = 3;
+- blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1,
+- BLAKE2S_HASH_SIZE);
++ hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE);
+ memcpy(third_dst, output, third_len);
+
+ out:
+diff --git a/drivers/net/wireguard/queueing.c b/drivers/net/wireguard/queueing.c
+index 48e7b982a3073..8084e7408c0ae 100644
+--- a/drivers/net/wireguard/queueing.c
++++ b/drivers/net/wireguard/queueing.c
+@@ -4,6 +4,7 @@
+ */
+
+ #include "queueing.h"
++#include <linux/skb_array.h>
+
+ struct multicore_worker __percpu *
+ wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr)
+@@ -38,11 +39,11 @@ int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
+ return 0;
+ }
+
+-void wg_packet_queue_free(struct crypt_queue *queue)
++void wg_packet_queue_free(struct crypt_queue *queue, bool purge)
+ {
+ free_percpu(queue->worker);
+- WARN_ON(!__ptr_ring_empty(&queue->ring));
+- ptr_ring_cleanup(&queue->ring, NULL);
++ WARN_ON(!purge && !__ptr_ring_empty(&queue->ring));
++ ptr_ring_cleanup(&queue->ring, purge ? __skb_array_destroy_skb : NULL);
+ }
+
+ #define NEXT(skb) ((skb)->prev)
+diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h
+index 4ef2944a68bc9..e2388107f7fdc 100644
+--- a/drivers/net/wireguard/queueing.h
++++ b/drivers/net/wireguard/queueing.h
+@@ -23,7 +23,7 @@ struct sk_buff;
+ /* queueing.c APIs: */
+ int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
+ unsigned int len);
+-void wg_packet_queue_free(struct crypt_queue *queue);
++void wg_packet_queue_free(struct crypt_queue *queue, bool purge);
+ struct multicore_worker __percpu *
+ wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr);
+
+diff --git a/drivers/net/wireguard/ratelimiter.c b/drivers/net/wireguard/ratelimiter.c
+index 3fedd1d21f5ee..dd55e5c26f468 100644
+--- a/drivers/net/wireguard/ratelimiter.c
++++ b/drivers/net/wireguard/ratelimiter.c
+@@ -176,12 +176,12 @@ int wg_ratelimiter_init(void)
+ (1U << 14) / sizeof(struct hlist_head)));
+ max_entries = table_size * 8;
+
+- table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL);
++ table_v4 = kvcalloc(table_size, sizeof(*table_v4), GFP_KERNEL);
+ if (unlikely(!table_v4))
+ goto err_kmemcache;
+
+ #if IS_ENABLED(CONFIG_IPV6)
+- table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL);
++ table_v6 = kvcalloc(table_size, sizeof(*table_v6), GFP_KERNEL);
+ if (unlikely(!table_v6)) {
+ kvfree(table_v4);
+ goto err_kmemcache;
+diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c
+index 7dc84bcca2613..7b8df406c7737 100644
+--- a/drivers/net/wireguard/receive.c
++++ b/drivers/net/wireguard/receive.c
+@@ -116,8 +116,8 @@ static void wg_receive_handshake_packet(struct wg_device *wg,
+ return;
+ }
+
+- under_load = skb_queue_len(&wg->incoming_handshakes) >=
+- MAX_QUEUED_INCOMING_HANDSHAKES / 8;
++ under_load = atomic_read(&wg->handshake_queue_len) >=
++ MAX_QUEUED_INCOMING_HANDSHAKES / 8;
+ if (under_load) {
+ last_under_load = ktime_get_coarse_boottime_ns();
+ } else if (last_under_load) {
+@@ -212,13 +212,14 @@ static void wg_receive_handshake_packet(struct wg_device *wg,
+
+ void wg_packet_handshake_receive_worker(struct work_struct *work)
+ {
+- struct wg_device *wg = container_of(work, struct multicore_worker,
+- work)->ptr;
++ struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr;
++ struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue);
+ struct sk_buff *skb;
+
+- while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) {
++ while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
+ wg_receive_handshake_packet(wg, skb);
+ dev_kfree_skb(skb);
++ atomic_dec(&wg->handshake_queue_len);
+ cond_resched();
+ }
+ }
+@@ -553,22 +554,28 @@ void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb)
+ case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION):
+ case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE):
+ case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): {
+- int cpu;
+-
+- if (skb_queue_len(&wg->incoming_handshakes) >
+- MAX_QUEUED_INCOMING_HANDSHAKES ||
+- unlikely(!rng_is_initialized())) {
++ int cpu, ret = -EBUSY;
++
++ if (unlikely(!rng_is_initialized()))
++ goto drop;
++ if (atomic_read(&wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) {
++ if (spin_trylock_bh(&wg->handshake_queue.ring.producer_lock)) {
++ ret = __ptr_ring_produce(&wg->handshake_queue.ring, skb);
++ spin_unlock_bh(&wg->handshake_queue.ring.producer_lock);
++ }
++ } else
++ ret = ptr_ring_produce_bh(&wg->handshake_queue.ring, skb);
++ if (ret) {
++ drop:
+ net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n",
+ wg->dev->name, skb);
+ goto err;
+ }
+- skb_queue_tail(&wg->incoming_handshakes, skb);
+- /* Queues up a call to packet_process_queued_handshake_
+- * packets(skb):
+- */
+- cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu);
++ atomic_inc(&wg->handshake_queue_len);
++ cpu = wg_cpumask_next_online(&wg->handshake_queue.last_cpu);
++ /* Queues up a call to packet_process_queued_handshake_packets(skb): */
+ queue_work_on(cpu, wg->handshake_receive_wq,
+- &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work);
++ &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work);
+ break;
+ }
+ case cpu_to_le32(MESSAGE_DATA):
+diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c
+index 8c496b7471082..0414d7a6ce741 100644
+--- a/drivers/net/wireguard/socket.c
++++ b/drivers/net/wireguard/socket.c
+@@ -160,6 +160,7 @@ out:
+ rcu_read_unlock_bh();
+ return ret;
+ #else
++ kfree_skb(skb);
+ return -EAFNOSUPPORT;
+ #endif
+ }
+@@ -241,7 +242,7 @@ int wg_socket_endpoint_from_skb(struct endpoint *endpoint,
+ endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr;
+ endpoint->src4.s_addr = ip_hdr(skb)->daddr;
+ endpoint->src_if4 = skb->skb_iif;
+- } else if (skb->protocol == htons(ETH_P_IPV6)) {
++ } else if (IS_ENABLED(CONFIG_IPV6) && skb->protocol == htons(ETH_P_IPV6)) {
+ endpoint->addr6.sin6_family = AF_INET6;
+ endpoint->addr6.sin6_port = udp_hdr(skb)->source;
+ endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr;
+@@ -284,7 +285,7 @@ void wg_socket_set_peer_endpoint(struct wg_peer *peer,
+ peer->endpoint.addr4 = endpoint->addr4;
+ peer->endpoint.src4 = endpoint->src4;
+ peer->endpoint.src_if4 = endpoint->src_if4;
+- } else if (endpoint->addr.sa_family == AF_INET6) {
++ } else if (IS_ENABLED(CONFIG_IPV6) && endpoint->addr.sa_family == AF_INET6) {
+ peer->endpoint.addr6 = endpoint->addr6;
+ peer->endpoint.src6 = endpoint->src6;
+ } else {
+@@ -308,7 +309,7 @@ void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer)
+ {
+ write_lock_bh(&peer->endpoint_lock);
+ memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6));
+- dst_cache_reset(&peer->endpoint_cache);
++ dst_cache_reset_now(&peer->endpoint_cache);
+ write_unlock_bh(&peer->endpoint_lock);
+ }
+
+diff --git a/drivers/net/wireless/ath/ar5523/ar5523.c b/drivers/net/wireless/ath/ar5523/ar5523.c
+index 49cc4b7ed5163..1baec4b412c8d 100644
+--- a/drivers/net/wireless/ath/ar5523/ar5523.c
++++ b/drivers/net/wireless/ath/ar5523/ar5523.c
+@@ -153,6 +153,10 @@ static void ar5523_cmd_rx_cb(struct urb *urb)
+ ar5523_err(ar, "Invalid reply to WDCMSG_TARGET_START");
+ return;
+ }
++ if (!cmd->odata) {
++ ar5523_err(ar, "Unexpected WDCMSG_TARGET_START reply");
++ return;
++ }
+ memcpy(cmd->odata, hdr + 1, sizeof(u32));
+ cmd->olen = sizeof(u32);
+ cmd->res = 0;
+diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
+index 2f9be182fbfbb..58e86e662ab83 100644
+--- a/drivers/net/wireless/ath/ath10k/core.c
++++ b/drivers/net/wireless/ath/ath10k/core.c
+@@ -89,6 +89,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = true,
+ .dynamic_sar_support = false,
+ },
+@@ -124,6 +125,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = true,
+ .dynamic_sar_support = false,
+ },
+@@ -160,6 +162,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -190,6 +193,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .num_wds_entries = 0x20,
+ .uart_pin_workaround = true,
+ .tx_stats_over_pktlog = false,
++ .credit_size_workaround = false,
+ .bmi_large_size_download = true,
+ .supports_peer_stats_info = true,
+ .dynamic_sar_support = true,
+@@ -226,6 +230,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -261,6 +266,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -296,6 +302,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -334,6 +341,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = true,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .supports_peer_stats_info = true,
+ .dynamic_sar_support = true,
+@@ -376,6 +384,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -424,6 +433,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -469,6 +479,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -504,6 +515,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -541,6 +553,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = true,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -570,6 +583,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .ast_skid_limit = 0x10,
+ .num_wds_entries = 0x20,
+ .uart_pin_workaround = true,
++ .credit_size_workaround = true,
+ .dynamic_sar_support = false,
+ },
+ {
+@@ -611,6 +625,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = false,
+ .hw_filter_reset_required = true,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = false,
+ },
+@@ -639,6 +654,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
+ .rri_on_ddr = true,
+ .hw_filter_reset_required = false,
+ .fw_diag_ce_download = false,
++ .credit_size_workaround = false,
+ .tx_stats_over_pktlog = false,
+ .dynamic_sar_support = true,
+ },
+@@ -714,6 +730,7 @@ static void ath10k_send_suspend_complete(struct ath10k *ar)
+
+ static int ath10k_init_sdio(struct ath10k *ar, enum ath10k_firmware_mode mode)
+ {
++ bool mtu_workaround = ar->hw_params.credit_size_workaround;
+ int ret;
+ u32 param = 0;
+
+@@ -731,7 +748,7 @@ static int ath10k_init_sdio(struct ath10k *ar, enum ath10k_firmware_mode mode)
+
+ param |= HI_ACS_FLAGS_SDIO_REDUCE_TX_COMPL_SET;
+
+- if (mode == ATH10K_FIRMWARE_MODE_NORMAL)
++ if (mode == ATH10K_FIRMWARE_MODE_NORMAL && !mtu_workaround)
+ param |= HI_ACS_FLAGS_ALT_DATA_CREDIT_SIZE;
+ else
+ param &= ~HI_ACS_FLAGS_ALT_DATA_CREDIT_SIZE;
+@@ -2690,9 +2707,16 @@ static int ath10k_core_copy_target_iram(struct ath10k *ar)
+ int i, ret;
+ u32 len, remaining_len;
+
+- hw_mem = ath10k_coredump_get_mem_layout(ar);
++ /* copy target iram feature must work also when
++ * ATH10K_FW_CRASH_DUMP_RAM_DATA is disabled, so
++ * _ath10k_coredump_get_mem_layout() to accomplist that
++ */
++ hw_mem = _ath10k_coredump_get_mem_layout(ar);
+ if (!hw_mem)
+- return -ENOMEM;
++ /* if CONFIG_DEV_COREDUMP is disabled we get NULL, then
++ * just silently disable the feature by doing nothing
++ */
++ return 0;
+
+ for (i = 0; i < hw_mem->region_table.size; i++) {
+ tmp = &hw_mem->region_table.regions[i];
+diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c
+index 7eb72290a925c..55e7e11d06d94 100644
+--- a/drivers/net/wireless/ath/ath10k/coredump.c
++++ b/drivers/net/wireless/ath/ath10k/coredump.c
+@@ -1447,11 +1447,17 @@ static u32 ath10k_coredump_get_ramdump_size(struct ath10k *ar)
+
+ const struct ath10k_hw_mem_layout *ath10k_coredump_get_mem_layout(struct ath10k *ar)
+ {
+- int i;
+-
+ if (!test_bit(ATH10K_FW_CRASH_DUMP_RAM_DATA, &ath10k_coredump_mask))
+ return NULL;
+
++ return _ath10k_coredump_get_mem_layout(ar);
++}
++EXPORT_SYMBOL(ath10k_coredump_get_mem_layout);
++
++const struct ath10k_hw_mem_layout *_ath10k_coredump_get_mem_layout(struct ath10k *ar)
++{
++ int i;
++
+ if (WARN_ON(ar->target_version == 0))
+ return NULL;
+
+@@ -1464,7 +1470,6 @@ const struct ath10k_hw_mem_layout *ath10k_coredump_get_mem_layout(struct ath10k
+
+ return NULL;
+ }
+-EXPORT_SYMBOL(ath10k_coredump_get_mem_layout);
+
+ struct ath10k_fw_crash_data *ath10k_coredump_new(struct ath10k *ar)
+ {
+diff --git a/drivers/net/wireless/ath/ath10k/coredump.h b/drivers/net/wireless/ath/ath10k/coredump.h
+index 42404e246e0e9..240d705150888 100644
+--- a/drivers/net/wireless/ath/ath10k/coredump.h
++++ b/drivers/net/wireless/ath/ath10k/coredump.h
+@@ -176,6 +176,7 @@ int ath10k_coredump_register(struct ath10k *ar);
+ void ath10k_coredump_unregister(struct ath10k *ar);
+ void ath10k_coredump_destroy(struct ath10k *ar);
+
++const struct ath10k_hw_mem_layout *_ath10k_coredump_get_mem_layout(struct ath10k *ar);
+ const struct ath10k_hw_mem_layout *ath10k_coredump_get_mem_layout(struct ath10k *ar);
+
+ #else /* CONFIG_DEV_COREDUMP */
+@@ -214,6 +215,12 @@ ath10k_coredump_get_mem_layout(struct ath10k *ar)
+ return NULL;
+ }
+
++static inline const struct ath10k_hw_mem_layout *
++_ath10k_coredump_get_mem_layout(struct ath10k *ar)
++{
++ return NULL;
++}
++
+ #endif /* CONFIG_DEV_COREDUMP */
+
+ #endif /* _COREDUMP_H_ */
+diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
+index d6b8bdcef4160..b793eac2cfac8 100644
+--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
++++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
+@@ -147,6 +147,9 @@ void ath10k_htt_tx_dec_pending(struct ath10k_htt *htt)
+ htt->num_pending_tx--;
+ if (htt->num_pending_tx == htt->max_num_pending_tx - 1)
+ ath10k_mac_tx_unlock(htt->ar, ATH10K_TX_PAUSE_Q_FULL);
++
++ if (htt->num_pending_tx == 0)
++ wake_up(&htt->empty_tx_wq);
+ }
+
+ int ath10k_htt_tx_inc_pending(struct ath10k_htt *htt)
+diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
+index 6b03c7787e36a..591ef7416b613 100644
+--- a/drivers/net/wireless/ath/ath10k/hw.h
++++ b/drivers/net/wireless/ath/ath10k/hw.h
+@@ -618,6 +618,9 @@ struct ath10k_hw_params {
+ */
+ bool uart_pin_workaround;
+
++ /* Workaround for the credit size calculation */
++ bool credit_size_workaround;
++
+ /* tx stats support over pktlog */
+ bool tx_stats_over_pktlog;
+
+diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
+index c272b290fa73d..1f73fbfee0c06 100644
+--- a/drivers/net/wireless/ath/ath10k/mac.c
++++ b/drivers/net/wireless/ath/ath10k/mac.c
+@@ -993,8 +993,12 @@ static void ath10k_mac_vif_beacon_cleanup(struct ath10k_vif *arvif)
+ ath10k_mac_vif_beacon_free(arvif);
+
+ if (arvif->beacon_buf) {
+- dma_free_coherent(ar->dev, IEEE80211_MAX_FRAME_LEN,
+- arvif->beacon_buf, arvif->beacon_paddr);
++ if (ar->bus_param.dev_type == ATH10K_DEV_TYPE_HL)
++ kfree(arvif->beacon_buf);
++ else
++ dma_free_coherent(ar->dev, IEEE80211_MAX_FRAME_LEN,
++ arvif->beacon_buf,
++ arvif->beacon_paddr);
+ arvif->beacon_buf = NULL;
+ }
+ }
+@@ -1048,7 +1052,7 @@ static int ath10k_monitor_vdev_start(struct ath10k *ar, int vdev_id)
+ arg.channel.min_power = 0;
+ arg.channel.max_power = channel->max_power * 2;
+ arg.channel.max_reg_power = channel->max_reg_power * 2;
+- arg.channel.max_antenna_gain = channel->max_antenna_gain * 2;
++ arg.channel.max_antenna_gain = channel->max_antenna_gain;
+
+ reinit_completion(&ar->vdev_setup_done);
+ reinit_completion(&ar->vdev_delete_done);
+@@ -1494,7 +1498,7 @@ static int ath10k_vdev_start_restart(struct ath10k_vif *arvif,
+ arg.channel.min_power = 0;
+ arg.channel.max_power = chandef->chan->max_power * 2;
+ arg.channel.max_reg_power = chandef->chan->max_reg_power * 2;
+- arg.channel.max_antenna_gain = chandef->chan->max_antenna_gain * 2;
++ arg.channel.max_antenna_gain = chandef->chan->max_antenna_gain;
+
+ if (arvif->vdev_type == WMI_VDEV_TYPE_AP) {
+ arg.ssid = arvif->u.ap.ssid;
+@@ -3422,7 +3426,7 @@ static int ath10k_update_channel_list(struct ath10k *ar)
+ ch->min_power = 0;
+ ch->max_power = channel->max_power * 2;
+ ch->max_reg_power = channel->max_reg_power * 2;
+- ch->max_antenna_gain = channel->max_antenna_gain * 2;
++ ch->max_antenna_gain = channel->max_antenna_gain;
+ ch->reg_class_id = 0; /* FIXME */
+
+ /* FIXME: why use only legacy modes, why not any
+@@ -5576,10 +5580,25 @@ static int ath10k_add_interface(struct ieee80211_hw *hw,
+ if (vif->type == NL80211_IFTYPE_ADHOC ||
+ vif->type == NL80211_IFTYPE_MESH_POINT ||
+ vif->type == NL80211_IFTYPE_AP) {
+- arvif->beacon_buf = dma_alloc_coherent(ar->dev,
+- IEEE80211_MAX_FRAME_LEN,
+- &arvif->beacon_paddr,
+- GFP_ATOMIC);
++ if (ar->bus_param.dev_type == ATH10K_DEV_TYPE_HL) {
++ arvif->beacon_buf = kmalloc(IEEE80211_MAX_FRAME_LEN,
++ GFP_KERNEL);
++
++ /* Using a kernel pointer in place of a dma_addr_t
++ * token can lead to undefined behavior if that
++ * makes it into cache management functions. Use a
++ * known-invalid address token instead, which
++ * avoids the warning and makes it easier to catch
++ * bugs if it does end up getting used.
++ */
++ arvif->beacon_paddr = DMA_MAPPING_ERROR;
++ } else {
++ arvif->beacon_buf =
++ dma_alloc_coherent(ar->dev,
++ IEEE80211_MAX_FRAME_LEN,
++ &arvif->beacon_paddr,
++ GFP_ATOMIC);
++ }
+ if (!arvif->beacon_buf) {
+ ret = -ENOMEM;
+ ath10k_warn(ar, "failed to allocate beacon buffer: %d\n",
+@@ -5794,8 +5813,12 @@ err_vdev_delete:
+
+ err:
+ if (arvif->beacon_buf) {
+- dma_free_coherent(ar->dev, IEEE80211_MAX_FRAME_LEN,
+- arvif->beacon_buf, arvif->beacon_paddr);
++ if (ar->bus_param.dev_type == ATH10K_DEV_TYPE_HL)
++ kfree(arvif->beacon_buf);
++ else
++ dma_free_coherent(ar->dev, IEEE80211_MAX_FRAME_LEN,
++ arvif->beacon_buf,
++ arvif->beacon_paddr);
+ arvif->beacon_buf = NULL;
+ }
+
+diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c
+index 07e478f9a808c..80fcb917fe4e1 100644
+--- a/drivers/net/wireless/ath/ath10k/qmi.c
++++ b/drivers/net/wireless/ath/ath10k/qmi.c
+@@ -864,7 +864,8 @@ static void ath10k_qmi_event_server_exit(struct ath10k_qmi *qmi)
+
+ ath10k_qmi_remove_msa_permission(qmi);
+ ath10k_core_free_board_files(ar);
+- if (!test_bit(ATH10K_SNOC_FLAG_UNREGISTERING, &ar_snoc->flags))
++ if (!test_bit(ATH10K_SNOC_FLAG_UNREGISTERING, &ar_snoc->flags) &&
++ !test_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags))
+ ath10k_snoc_fw_crashed_dump(ar);
+
+ ath10k_snoc_fw_indication(ar, ATH10K_QMI_EVENT_FW_DOWN_IND);
+diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c
+index b746052737e0b..eb705214f3f0a 100644
+--- a/drivers/net/wireless/ath/ath10k/sdio.c
++++ b/drivers/net/wireless/ath/ath10k/sdio.c
+@@ -1363,8 +1363,11 @@ static void ath10k_rx_indication_async_work(struct work_struct *work)
+ ep->ep_ops.ep_rx_complete(ar, skb);
+ }
+
+- if (test_bit(ATH10K_FLAG_CORE_REGISTERED, &ar->dev_flags))
++ if (test_bit(ATH10K_FLAG_CORE_REGISTERED, &ar->dev_flags)) {
++ local_bh_disable();
+ napi_schedule(&ar->napi);
++ local_bh_enable();
++ }
+ }
+
+ static int ath10k_sdio_read_rtc_state(struct ath10k_sdio *ar_sdio, unsigned char *state)
+diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c
+index ea00fbb156015..f79dd9a716906 100644
+--- a/drivers/net/wireless/ath/ath10k/snoc.c
++++ b/drivers/net/wireless/ath/ath10k/snoc.c
+@@ -12,6 +12,7 @@
+ #include <linux/platform_device.h>
+ #include <linux/property.h>
+ #include <linux/regulator/consumer.h>
++#include <linux/remoteproc/qcom_rproc.h>
+ #include <linux/of_address.h>
+ #include <linux/iommu.h>
+
+@@ -1477,6 +1478,74 @@ void ath10k_snoc_fw_crashed_dump(struct ath10k *ar)
+ mutex_unlock(&ar->dump_mutex);
+ }
+
++static int ath10k_snoc_modem_notify(struct notifier_block *nb, unsigned long action,
++ void *data)
++{
++ struct ath10k_snoc *ar_snoc = container_of(nb, struct ath10k_snoc, nb);
++ struct ath10k *ar = ar_snoc->ar;
++ struct qcom_ssr_notify_data *notify_data = data;
++
++ switch (action) {
++ case QCOM_SSR_BEFORE_POWERUP:
++ ath10k_dbg(ar, ATH10K_DBG_SNOC, "received modem starting event\n");
++ clear_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags);
++ break;
++
++ case QCOM_SSR_AFTER_POWERUP:
++ ath10k_dbg(ar, ATH10K_DBG_SNOC, "received modem running event\n");
++ break;
++
++ case QCOM_SSR_BEFORE_SHUTDOWN:
++ ath10k_dbg(ar, ATH10K_DBG_SNOC, "received modem %s event\n",
++ notify_data->crashed ? "crashed" : "stopping");
++ if (!notify_data->crashed)
++ set_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags);
++ else
++ clear_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags);
++ break;
++
++ case QCOM_SSR_AFTER_SHUTDOWN:
++ ath10k_dbg(ar, ATH10K_DBG_SNOC, "received modem offline event\n");
++ break;
++
++ default:
++ ath10k_err(ar, "received unrecognized event %lu\n", action);
++ break;
++ }
++
++ return NOTIFY_OK;
++}
++
++static int ath10k_modem_init(struct ath10k *ar)
++{
++ struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
++ void *notifier;
++ int ret;
++
++ ar_snoc->nb.notifier_call = ath10k_snoc_modem_notify;
++
++ notifier = qcom_register_ssr_notifier("mpss", &ar_snoc->nb);
++ if (IS_ERR(notifier)) {
++ ret = PTR_ERR(notifier);
++ ath10k_err(ar, "failed to initialize modem notifier: %d\n", ret);
++ return ret;
++ }
++
++ ar_snoc->notifier = notifier;
++
++ return 0;
++}
++
++static void ath10k_modem_deinit(struct ath10k *ar)
++{
++ int ret;
++ struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
++
++ ret = qcom_unregister_ssr_notifier(ar_snoc->notifier, &ar_snoc->nb);
++ if (ret)
++ ath10k_err(ar, "error %d unregistering notifier\n", ret);
++}
++
+ static int ath10k_setup_msa_resources(struct ath10k *ar, u32 msa_size)
+ {
+ struct device *dev = ar->dev;
+@@ -1487,11 +1556,11 @@ static int ath10k_setup_msa_resources(struct ath10k *ar, u32 msa_size)
+ node = of_parse_phandle(dev->of_node, "memory-region", 0);
+ if (node) {
+ ret = of_address_to_resource(node, 0, &r);
++ of_node_put(node);
+ if (ret) {
+ dev_err(dev, "failed to resolve msa fixed region\n");
+ return ret;
+ }
+- of_node_put(node);
+
+ ar->msa.paddr = r.start;
+ ar->msa.mem_size = resource_size(&r);
+@@ -1740,10 +1809,17 @@ static int ath10k_snoc_probe(struct platform_device *pdev)
+ goto err_fw_deinit;
+ }
+
++ ret = ath10k_modem_init(ar);
++ if (ret)
++ goto err_qmi_deinit;
++
+ ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc probe\n");
+
+ return 0;
+
++err_qmi_deinit:
++ ath10k_qmi_deinit(ar);
++
+ err_fw_deinit:
+ ath10k_fw_deinit(ar);
+
+@@ -1771,6 +1847,7 @@ static int ath10k_snoc_free_resources(struct ath10k *ar)
+ ath10k_fw_deinit(ar);
+ ath10k_snoc_free_irq(ar);
+ ath10k_snoc_release_resource(ar);
++ ath10k_modem_deinit(ar);
+ ath10k_qmi_deinit(ar);
+ ath10k_core_destroy(ar);
+
+diff --git a/drivers/net/wireless/ath/ath10k/snoc.h b/drivers/net/wireless/ath/ath10k/snoc.h
+index 5095d1893681b..d4bce17076960 100644
+--- a/drivers/net/wireless/ath/ath10k/snoc.h
++++ b/drivers/net/wireless/ath/ath10k/snoc.h
+@@ -6,6 +6,8 @@
+ #ifndef _SNOC_H_
+ #define _SNOC_H_
+
++#include <linux/notifier.h>
++
+ #include "hw.h"
+ #include "ce.h"
+ #include "qmi.h"
+@@ -45,6 +47,7 @@ struct ath10k_snoc_ce_irq {
+ enum ath10k_snoc_flags {
+ ATH10K_SNOC_FLAG_REGISTERED,
+ ATH10K_SNOC_FLAG_UNREGISTERING,
++ ATH10K_SNOC_FLAG_MODEM_STOPPED,
+ ATH10K_SNOC_FLAG_RECOVERY,
+ ATH10K_SNOC_FLAG_8BIT_HOST_CAP_QUIRK,
+ };
+@@ -75,6 +78,8 @@ struct ath10k_snoc {
+ struct clk_bulk_data *clks;
+ size_t num_clks;
+ struct ath10k_qmi *qmi;
++ struct notifier_block nb;
++ void *notifier;
+ unsigned long flags;
+ bool xo_cal_supported;
+ u32 xo_cal_data;
+diff --git a/drivers/net/wireless/ath/ath10k/txrx.c b/drivers/net/wireless/ath/ath10k/txrx.c
+index 7c9ea0c073d8b..6f8b642188941 100644
+--- a/drivers/net/wireless/ath/ath10k/txrx.c
++++ b/drivers/net/wireless/ath/ath10k/txrx.c
+@@ -82,8 +82,6 @@ int ath10k_txrx_tx_unref(struct ath10k_htt *htt,
+ flags = skb_cb->flags;
+ ath10k_htt_tx_free_msdu_id(htt, tx_done->msdu_id);
+ ath10k_htt_tx_dec_pending(htt);
+- if (htt->num_pending_tx == 0)
+- wake_up(&htt->empty_tx_wq);
+ spin_unlock_bh(&htt->tx_lock);
+
+ rcu_read_lock();
+diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c
+index 19b9c27e30e20..3d98f19c6ec8a 100644
+--- a/drivers/net/wireless/ath/ath10k/usb.c
++++ b/drivers/net/wireless/ath/ath10k/usb.c
+@@ -525,7 +525,7 @@ static int ath10k_usb_submit_ctrl_in(struct ath10k *ar,
+ req,
+ USB_DIR_IN | USB_TYPE_VENDOR |
+ USB_RECIP_DEVICE, value, index, buf,
+- size, 2 * HZ);
++ size, 2000);
+
+ if (ret < 0) {
+ ath10k_warn(ar, "Failed to read usb control message: %d\n",
+@@ -853,6 +853,11 @@ static int ath10k_usb_setup_pipe_resources(struct ath10k *ar,
+ le16_to_cpu(endpoint->wMaxPacketSize),
+ endpoint->bInterval);
+ }
++
++ /* Ignore broken descriptors. */
++ if (usb_endpoint_maxp(endpoint) == 0)
++ continue;
++
+ urbcount = 0;
+
+ pipe_num =
+diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
+index b8a4bbfe10b87..7c1c2658cb5f8 100644
+--- a/drivers/net/wireless/ath/ath10k/wmi.c
++++ b/drivers/net/wireless/ath/ath10k/wmi.c
+@@ -2610,6 +2610,10 @@ int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
+ if (ieee80211_is_beacon(hdr->frame_control))
+ ath10k_mac_handle_beacon(ar, skb);
+
++ if (ieee80211_is_beacon(hdr->frame_control) ||
++ ieee80211_is_probe_resp(hdr->frame_control))
++ status->boottime_ns = ktime_get_boottime_ns();
++
+ ath10k_dbg(ar, ATH10K_DBG_MGMT,
+ "event mgmt rx skb %pK len %d ftype %02x stype %02x\n",
+ skb, skb->len,
+diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
+index 41c1a3d339c25..01bfd09a9d88c 100644
+--- a/drivers/net/wireless/ath/ath10k/wmi.h
++++ b/drivers/net/wireless/ath/ath10k/wmi.h
+@@ -2066,7 +2066,9 @@ struct wmi_channel {
+ union {
+ __le32 reginfo1;
+ struct {
++ /* note: power unit is 1 dBm */
+ u8 antenna_max;
++ /* note: power unit is 0.5 dBm */
+ u8 max_tx_power;
+ } __packed;
+ } __packed;
+@@ -2086,6 +2088,7 @@ struct wmi_channel_arg {
+ u32 min_power;
+ u32 max_power;
+ u32 max_reg_power;
++ /* note: power unit is 1 dBm */
+ u32 max_antenna_gain;
+ u32 reg_class_id;
+ enum wmi_phy_mode mode;
+diff --git a/drivers/net/wireless/ath/ath10k/wow.c b/drivers/net/wireless/ath/ath10k/wow.c
+index 7d65c115669fe..20b9aa8ddf7d5 100644
+--- a/drivers/net/wireless/ath/ath10k/wow.c
++++ b/drivers/net/wireless/ath/ath10k/wow.c
+@@ -337,14 +337,15 @@ static int ath10k_vif_wow_set_wakeups(struct ath10k_vif *arvif,
+ if (patterns[i].mask[j / 8] & BIT(j % 8))
+ bitmask[j] = 0xff;
+ old_pattern.mask = bitmask;
+- new_pattern = old_pattern;
+
+ if (ar->wmi.rx_decap_mode == ATH10K_HW_TXRX_NATIVE_WIFI) {
+- if (patterns[i].pkt_offset < ETH_HLEN)
++ if (patterns[i].pkt_offset < ETH_HLEN) {
+ ath10k_wow_convert_8023_to_80211(&new_pattern,
+ &old_pattern);
+- else
++ } else {
++ new_pattern = old_pattern;
+ new_pattern.pkt_offset += WOW_HDR_LEN - ETH_HLEN;
++ }
+ }
+
+ if (WARN_ON(new_pattern.pattern_len > WOW_MAX_PATTERN_SIZE))
+diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
+index 8c9c781afc3e5..24bd0520926bf 100644
+--- a/drivers/net/wireless/ath/ath11k/ahb.c
++++ b/drivers/net/wireless/ath/ath11k/ahb.c
+@@ -175,8 +175,11 @@ static void __ath11k_ahb_ext_irq_disable(struct ath11k_base *ab)
+
+ ath11k_ahb_ext_grp_disable(irq_grp);
+
+- napi_synchronize(&irq_grp->napi);
+- napi_disable(&irq_grp->napi);
++ if (irq_grp->napi_enabled) {
++ napi_synchronize(&irq_grp->napi);
++ napi_disable(&irq_grp->napi);
++ irq_grp->napi_enabled = false;
++ }
+ }
+ }
+
+@@ -206,13 +209,13 @@ static void ath11k_ahb_clearbit32(struct ath11k_base *ab, u8 bit, u32 offset)
+
+ static void ath11k_ahb_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
+ {
+- const struct ce_pipe_config *ce_config;
++ const struct ce_attr *ce_attr;
+
+- ce_config = &ab->hw_params.target_ce_config[ce_id];
+- if (__le32_to_cpu(ce_config->pipedir) & PIPEDIR_OUT)
++ ce_attr = &ab->hw_params.host_ce_config[ce_id];
++ if (ce_attr->src_nentries)
+ ath11k_ahb_setbit32(ab, ce_id, CE_HOST_IE_ADDRESS);
+
+- if (__le32_to_cpu(ce_config->pipedir) & PIPEDIR_IN) {
++ if (ce_attr->dest_nentries) {
+ ath11k_ahb_setbit32(ab, ce_id, CE_HOST_IE_2_ADDRESS);
+ ath11k_ahb_setbit32(ab, ce_id + CE_HOST_IE_3_SHIFT,
+ CE_HOST_IE_3_ADDRESS);
+@@ -221,13 +224,13 @@ static void ath11k_ahb_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
+
+ static void ath11k_ahb_ce_irq_disable(struct ath11k_base *ab, u16 ce_id)
+ {
+- const struct ce_pipe_config *ce_config;
++ const struct ce_attr *ce_attr;
+
+- ce_config = &ab->hw_params.target_ce_config[ce_id];
+- if (__le32_to_cpu(ce_config->pipedir) & PIPEDIR_OUT)
++ ce_attr = &ab->hw_params.host_ce_config[ce_id];
++ if (ce_attr->src_nentries)
+ ath11k_ahb_clearbit32(ab, ce_id, CE_HOST_IE_ADDRESS);
+
+- if (__le32_to_cpu(ce_config->pipedir) & PIPEDIR_IN) {
++ if (ce_attr->dest_nentries) {
+ ath11k_ahb_clearbit32(ab, ce_id, CE_HOST_IE_2_ADDRESS);
+ ath11k_ahb_clearbit32(ab, ce_id + CE_HOST_IE_3_SHIFT,
+ CE_HOST_IE_3_ADDRESS);
+@@ -300,7 +303,10 @@ static void ath11k_ahb_ext_irq_enable(struct ath11k_base *ab)
+ for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
+ struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
+
+- napi_enable(&irq_grp->napi);
++ if (!irq_grp->napi_enabled) {
++ napi_enable(&irq_grp->napi);
++ irq_grp->napi_enabled = true;
++ }
+ ath11k_ahb_ext_grp_enable(irq_grp);
+ }
+ }
+@@ -385,6 +391,8 @@ static void ath11k_ahb_free_ext_irq(struct ath11k_base *ab)
+
+ for (j = 0; j < irq_grp->num_irq; j++)
+ free_irq(ab->irq_num[irq_grp->irqs[j]], irq_grp);
++
++ netif_napi_del(&irq_grp->napi);
+ }
+ }
+
+diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c
+index 969bf1a590d99..7dcf6b13f7949 100644
+--- a/drivers/net/wireless/ath/ath11k/core.c
++++ b/drivers/net/wireless/ath/ath11k/core.c
+@@ -347,11 +347,26 @@ static int ath11k_core_create_board_name(struct ath11k_base *ab, char *name,
+ scnprintf(variant, sizeof(variant), ",variant=%s",
+ ab->qmi.target.bdf_ext);
+
+- scnprintf(name, name_len,
+- "bus=%s,qmi-chip-id=%d,qmi-board-id=%d%s",
+- ath11k_bus_str(ab->hif.bus),
+- ab->qmi.target.chip_id,
+- ab->qmi.target.board_id, variant);
++ switch (ab->id.bdf_search) {
++ case ATH11K_BDF_SEARCH_BUS_AND_BOARD:
++ scnprintf(name, name_len,
++ "bus=%s,vendor=%04x,device=%04x,subsystem-vendor=%04x,subsystem-device=%04x,qmi-chip-id=%d,qmi-board-id=%d%s",
++ ath11k_bus_str(ab->hif.bus),
++ ab->id.vendor, ab->id.device,
++ ab->id.subsystem_vendor,
++ ab->id.subsystem_device,
++ ab->qmi.target.chip_id,
++ ab->qmi.target.board_id,
++ variant);
++ break;
++ default:
++ scnprintf(name, name_len,
++ "bus=%s,qmi-chip-id=%d,qmi-board-id=%d%s",
++ ath11k_bus_str(ab->hif.bus),
++ ab->qmi.target.chip_id,
++ ab->qmi.target.board_id, variant);
++ break;
++ }
+
+ ath11k_dbg(ab, ATH11K_DBG_BOOT, "boot using board name '%s'\n", name);
+
+@@ -588,7 +603,7 @@ static int ath11k_core_fetch_board_data_api_1(struct ath11k_base *ab,
+ return 0;
+ }
+
+-#define BOARD_NAME_SIZE 100
++#define BOARD_NAME_SIZE 200
+ int ath11k_core_fetch_bdf(struct ath11k_base *ab, struct ath11k_board_data *bd)
+ {
+ char boardname[BOARD_NAME_SIZE];
+diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h
+index 018fb2385f2a3..caa8f6eba0097 100644
+--- a/drivers/net/wireless/ath/ath11k/core.h
++++ b/drivers/net/wireless/ath/ath11k/core.h
+@@ -47,6 +47,11 @@ enum ath11k_supported_bw {
+ ATH11K_BW_160 = 3,
+ };
+
++enum ath11k_bdf_search {
++ ATH11K_BDF_SEARCH_DEFAULT,
++ ATH11K_BDF_SEARCH_BUS_AND_BOARD,
++};
++
+ enum wme_ac {
+ WME_AC_BE,
+ WME_AC_BK,
+@@ -132,6 +137,7 @@ struct ath11k_ext_irq_grp {
+ u32 num_irq;
+ u32 grp_id;
+ u64 timestamp;
++ bool napi_enabled;
+ struct napi_struct napi;
+ struct net_device napi_ndev;
+ };
+@@ -701,7 +707,6 @@ struct ath11k_base {
+ u32 wlan_init_status;
+ int irq_num[ATH11K_IRQ_NUM_MAX];
+ struct ath11k_ext_irq_grp ext_irq_grp[ATH11K_EXT_IRQ_GRP_NUM_MAX];
+- struct napi_struct *napi;
+ struct ath11k_targ_cap target_caps;
+ u32 ext_service_bitmap[WMI_SERVICE_EXT_BM_SIZE];
+ bool pdevs_macaddr_valid;
+@@ -747,6 +752,14 @@ struct ath11k_base {
+
+ struct completion htc_suspend;
+
++ struct {
++ enum ath11k_bdf_search bdf_search;
++ u32 vendor;
++ u32 device;
++ u32 subsystem_vendor;
++ u32 subsystem_device;
++ } id;
++
+ /* must be last */
+ u8 drv_priv[0] __aligned(sizeof(void *));
+ };
+diff --git a/drivers/net/wireless/ath/ath11k/dbring.c b/drivers/net/wireless/ath/ath11k/dbring.c
+index 5e1f5437b4185..fd98ba5b1130b 100644
+--- a/drivers/net/wireless/ath/ath11k/dbring.c
++++ b/drivers/net/wireless/ath/ath11k/dbring.c
+@@ -8,8 +8,7 @@
+
+ static int ath11k_dbring_bufs_replenish(struct ath11k *ar,
+ struct ath11k_dbring *ring,
+- struct ath11k_dbring_element *buff,
+- gfp_t gfp)
++ struct ath11k_dbring_element *buff)
+ {
+ struct ath11k_base *ab = ar->ab;
+ struct hal_srng *srng;
+@@ -35,7 +34,7 @@ static int ath11k_dbring_bufs_replenish(struct ath11k *ar,
+ goto err;
+
+ spin_lock_bh(&ring->idr_lock);
+- buf_id = idr_alloc(&ring->bufs_idr, buff, 0, ring->bufs_max, gfp);
++ buf_id = idr_alloc(&ring->bufs_idr, buff, 0, ring->bufs_max, GFP_ATOMIC);
+ spin_unlock_bh(&ring->idr_lock);
+ if (buf_id < 0) {
+ ret = -ENOBUFS;
+@@ -72,8 +71,7 @@ err:
+ }
+
+ static int ath11k_dbring_fill_bufs(struct ath11k *ar,
+- struct ath11k_dbring *ring,
+- gfp_t gfp)
++ struct ath11k_dbring *ring)
+ {
+ struct ath11k_dbring_element *buff;
+ struct hal_srng *srng;
+@@ -92,11 +90,11 @@ static int ath11k_dbring_fill_bufs(struct ath11k *ar,
+ size = sizeof(*buff) + ring->buf_sz + align - 1;
+
+ while (num_remain > 0) {
+- buff = kzalloc(size, gfp);
++ buff = kzalloc(size, GFP_ATOMIC);
+ if (!buff)
+ break;
+
+- ret = ath11k_dbring_bufs_replenish(ar, ring, buff, gfp);
++ ret = ath11k_dbring_bufs_replenish(ar, ring, buff);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to replenish db ring num_remain %d req_ent %d\n",
+ num_remain, req_entries);
+@@ -176,7 +174,7 @@ int ath11k_dbring_buf_setup(struct ath11k *ar,
+ ring->hp_addr = ath11k_hal_srng_get_hp_addr(ar->ab, srng);
+ ring->tp_addr = ath11k_hal_srng_get_tp_addr(ar->ab, srng);
+
+- ret = ath11k_dbring_fill_bufs(ar, ring, GFP_KERNEL);
++ ret = ath11k_dbring_fill_bufs(ar, ring);
+
+ return ret;
+ }
+@@ -322,7 +320,7 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab,
+ }
+
+ memset(buff, 0, size);
+- ath11k_dbring_bufs_replenish(ar, ring, buff, GFP_ATOMIC);
++ ath11k_dbring_bufs_replenish(ar, ring, buff);
+ }
+
+ spin_unlock_bh(&srng->lock);
+diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h
+index ee768ccce46e1..d3e50e34f23dd 100644
+--- a/drivers/net/wireless/ath/ath11k/dp.h
++++ b/drivers/net/wireless/ath/ath11k/dp.h
+@@ -515,7 +515,8 @@ struct htt_ppdu_stats_cfg_cmd {
+ } __packed;
+
+ #define HTT_PPDU_STATS_CFG_MSG_TYPE GENMASK(7, 0)
+-#define HTT_PPDU_STATS_CFG_PDEV_ID GENMASK(15, 8)
++#define HTT_PPDU_STATS_CFG_SOC_STATS BIT(8)
++#define HTT_PPDU_STATS_CFG_PDEV_ID GENMASK(15, 9)
+ #define HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK GENMASK(31, 16)
+
+ enum htt_ppdu_stats_tag_type {
+diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
+index 9a224817630ae..0ae6bebff801d 100644
+--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
++++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
+@@ -2337,8 +2337,10 @@ static void ath11k_dp_rx_h_ppdu(struct ath11k *ar, struct hal_rx_desc *rx_desc,
+ channel_num = meta_data;
+ center_freq = meta_data >> 16;
+
+- if (center_freq >= 5935 && center_freq <= 7105) {
++ if (center_freq >= ATH11K_MIN_6G_FREQ &&
++ center_freq <= ATH11K_MAX_6G_FREQ) {
+ rx_status->band = NL80211_BAND_6GHZ;
++ rx_status->freq = center_freq;
+ } else if (channel_num >= 1 && channel_num <= 14) {
+ rx_status->band = NL80211_BAND_2GHZ;
+ } else if (channel_num >= 36 && channel_num <= 173) {
+@@ -2356,8 +2358,9 @@ static void ath11k_dp_rx_h_ppdu(struct ath11k *ar, struct hal_rx_desc *rx_desc,
+ rx_desc, sizeof(struct hal_rx_desc));
+ }
+
+- rx_status->freq = ieee80211_channel_to_frequency(channel_num,
+- rx_status->band);
++ if (rx_status->band != NL80211_BAND_6GHZ)
++ rx_status->freq = ieee80211_channel_to_frequency(channel_num,
++ rx_status->band);
+
+ ath11k_dp_rx_h_rate(ar, rx_desc, rx_status);
+ }
+@@ -3310,7 +3313,7 @@ static int ath11k_dp_rx_h_defrag_reo_reinject(struct ath11k *ar, struct dp_rx_ti
+
+ paddr = dma_map_single(ab->dev, defrag_skb->data,
+ defrag_skb->len + skb_tailroom(defrag_skb),
+- DMA_FROM_DEVICE);
++ DMA_TO_DEVICE);
+ if (dma_mapping_error(ab->dev, paddr))
+ return -ENOMEM;
+
+@@ -3375,7 +3378,7 @@ err_free_idr:
+ spin_unlock_bh(&rx_refill_ring->idr_lock);
+ err_unmap_dma:
+ dma_unmap_single(ab->dev, paddr, defrag_skb->len + skb_tailroom(defrag_skb),
+- DMA_FROM_DEVICE);
++ DMA_TO_DEVICE);
+ return ret;
+ }
+
+diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c
+index 8bba5234f81fc..bb8744ccfa00c 100644
+--- a/drivers/net/wireless/ath/ath11k/dp_tx.c
++++ b/drivers/net/wireless/ath/ath11k/dp_tx.c
+@@ -895,7 +895,7 @@ int ath11k_dp_tx_htt_h2t_ppdu_stats_req(struct ath11k *ar, u32 mask)
+ cmd->msg = FIELD_PREP(HTT_PPDU_STATS_CFG_MSG_TYPE,
+ HTT_H2T_MSG_TYPE_PPDU_STATS_CFG);
+
+- pdev_mask = 1 << (i + 1);
++ pdev_mask = 1 << (ar->pdev_idx + i);
+ cmd->msg |= FIELD_PREP(HTT_PPDU_STATS_CFG_PDEV_ID, pdev_mask);
+ cmd->msg |= FIELD_PREP(HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK, mask);
+
+diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c
+index eaa0edca55761..5dbf5596c9e8e 100644
+--- a/drivers/net/wireless/ath/ath11k/hal.c
++++ b/drivers/net/wireless/ath/ath11k/hal.c
+@@ -947,6 +947,7 @@ int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type,
+ srng->msi_data = params->msi_data;
+ srng->initialized = 1;
+ spin_lock_init(&srng->lock);
++ lockdep_set_class(&srng->lock, hal->srng_key + ring_id);
+
+ for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) {
+ srng->hwreg_base[i] = srng_config->reg_start[i] +
+@@ -1233,6 +1234,24 @@ static int ath11k_hal_srng_create_config(struct ath11k_base *ab)
+ return 0;
+ }
+
++static void ath11k_hal_register_srng_key(struct ath11k_base *ab)
++{
++ struct ath11k_hal *hal = &ab->hal;
++ u32 ring_id;
++
++ for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
++ lockdep_register_key(hal->srng_key + ring_id);
++}
++
++static void ath11k_hal_unregister_srng_key(struct ath11k_base *ab)
++{
++ struct ath11k_hal *hal = &ab->hal;
++ u32 ring_id;
++
++ for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
++ lockdep_unregister_key(hal->srng_key + ring_id);
++}
++
+ int ath11k_hal_srng_init(struct ath11k_base *ab)
+ {
+ struct ath11k_hal *hal = &ab->hal;
+@@ -1252,6 +1271,8 @@ int ath11k_hal_srng_init(struct ath11k_base *ab)
+ if (ret)
+ goto err_free_cont_rdp;
+
++ ath11k_hal_register_srng_key(ab);
++
+ return 0;
+
+ err_free_cont_rdp:
+@@ -1266,6 +1287,7 @@ void ath11k_hal_srng_deinit(struct ath11k_base *ab)
+ {
+ struct ath11k_hal *hal = &ab->hal;
+
++ ath11k_hal_unregister_srng_key(ab);
+ ath11k_hal_free_cont_rdp(ab);
+ ath11k_hal_free_cont_wrp(ab);
+ kfree(hal->srng_config);
+diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h
+index 35ed3a14e200a..7fdcd8bbf7e98 100644
+--- a/drivers/net/wireless/ath/ath11k/hal.h
++++ b/drivers/net/wireless/ath/ath11k/hal.h
+@@ -901,6 +901,8 @@ struct ath11k_hal {
+ /* shadow register configuration */
+ u32 shadow_reg_addr[HAL_SHADOW_NUM_REGS];
+ int num_shadow_reg_configured;
++
++ struct lock_class_key srng_key[HAL_SRNG_RING_ID_MAX];
+ };
+
+ u32 ath11k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid);
+diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c
+index d9596903b0a58..3e92cc7cfe4c9 100644
+--- a/drivers/net/wireless/ath/ath11k/hw.c
++++ b/drivers/net/wireless/ath/ath11k/hw.c
+@@ -1015,8 +1015,6 @@ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_ipq8074 = {
+ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qca6390 = {
+ .tx = {
+ ATH11K_TX_RING_MASK_0,
+- ATH11K_TX_RING_MASK_1,
+- ATH11K_TX_RING_MASK_2,
+ },
+ .rx_mon_status = {
+ 0, 0, 0, 0,
+diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c
+index e9b3689331ec2..07004564a3ec5 100644
+--- a/drivers/net/wireless/ath/ath11k/mac.c
++++ b/drivers/net/wireless/ath/ath11k/mac.c
+@@ -1,6 +1,7 @@
+ // SPDX-License-Identifier: BSD-3-Clause-Clear
+ /*
+ * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
++ * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+ #include <net/mac80211.h>
+@@ -767,11 +768,15 @@ static int ath11k_mac_setup_bcn_tmpl(struct ath11k_vif *arvif)
+
+ if (cfg80211_find_ie(WLAN_EID_RSN, ies, (skb_tail_pointer(bcn) - ies)))
+ arvif->rsnie_present = true;
++ else
++ arvif->rsnie_present = false;
+
+ if (cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
+ WLAN_OUI_TYPE_MICROSOFT_WPA,
+ ies, (skb_tail_pointer(bcn) - ies)))
+ arvif->wpaie_present = true;
++ else
++ arvif->wpaie_present = false;
+
+ ret = ath11k_wmi_bcn_tmpl(ar, arvif->vdev_id, &offs, bcn);
+
+@@ -2151,6 +2156,19 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw,
+ if (ret)
+ ath11k_warn(ar->ab, "failed to update bcn template: %d\n",
+ ret);
++ if (vif->bss_conf.he_support) {
++ ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
++ WMI_VDEV_PARAM_BA_MODE,
++ WMI_BA_MODE_BUFFER_SIZE_256);
++ if (ret)
++ ath11k_warn(ar->ab,
++ "failed to set BA BUFFER SIZE 256 for vdev: %d\n",
++ arvif->vdev_id);
++ else
++ ath11k_dbg(ar->ab, ATH11K_DBG_MAC,
++ "Set BA BUFFER SIZE 256 for VDEV: %d\n",
++ arvif->vdev_id);
++ }
+ }
+
+ if (changed & (BSS_CHANGED_BEACON_INFO | BSS_CHANGED_BEACON)) {
+@@ -2186,14 +2204,6 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw,
+
+ if (arvif->is_up && vif->bss_conf.he_support &&
+ vif->bss_conf.he_oper.params) {
+- ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
+- WMI_VDEV_PARAM_BA_MODE,
+- WMI_BA_MODE_BUFFER_SIZE_256);
+- if (ret)
+- ath11k_warn(ar->ab,
+- "failed to set BA BUFFER SIZE 256 for vdev: %d\n",
+- arvif->vdev_id);
+-
+ param_id = WMI_VDEV_PARAM_HEOPS_0_31;
+ param_value = vif->bss_conf.he_oper.params;
+ ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
+@@ -2576,9 +2586,12 @@ static int ath11k_mac_op_hw_scan(struct ieee80211_hw *hw,
+ arg.scan_id = ATH11K_SCAN_ID;
+
+ if (req->ie_len) {
++ arg.extraie.ptr = kmemdup(req->ie, req->ie_len, GFP_KERNEL);
++ if (!arg.extraie.ptr) {
++ ret = -ENOMEM;
++ goto exit;
++ }
+ arg.extraie.len = req->ie_len;
+- arg.extraie.ptr = kzalloc(req->ie_len, GFP_KERNEL);
+- memcpy(arg.extraie.ptr, req->ie, req->ie_len);
+ }
+
+ if (req->n_ssids) {
+@@ -2655,9 +2668,7 @@ static int ath11k_install_key(struct ath11k_vif *arvif,
+ return 0;
+
+ if (cmd == DISABLE_KEY) {
+- /* TODO: Check if FW expects value other than NONE for del */
+- /* arg.key_cipher = WMI_CIPHER_NONE; */
+- arg.key_len = 0;
++ arg.key_cipher = WMI_CIPHER_NONE;
+ arg.key_data = NULL;
+ goto install;
+ }
+@@ -2789,7 +2800,7 @@ static int ath11k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+ /* flush the fragments cache during key (re)install to
+ * ensure all frags in the new frag list belong to the same key.
+ */
+- if (peer && cmd == SET_KEY)
++ if (peer && sta && cmd == SET_KEY)
+ ath11k_peer_frags_flush(ar, peer);
+ spin_unlock_bh(&ab->base_lock);
+
+@@ -4131,23 +4142,32 @@ static int __ath11k_set_antenna(struct ath11k *ar, u32 tx_ant, u32 rx_ant)
+ return 0;
+ }
+
+-int ath11k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx)
++static void ath11k_mac_tx_mgmt_free(struct ath11k *ar, int buf_id)
+ {
+- struct sk_buff *msdu = skb;
++ struct sk_buff *msdu;
+ struct ieee80211_tx_info *info;
+- struct ath11k *ar = ctx;
+- struct ath11k_base *ab = ar->ab;
+
+ spin_lock_bh(&ar->txmgmt_idr_lock);
+- idr_remove(&ar->txmgmt_idr, buf_id);
++ msdu = idr_remove(&ar->txmgmt_idr, buf_id);
+ spin_unlock_bh(&ar->txmgmt_idr_lock);
+- dma_unmap_single(ab->dev, ATH11K_SKB_CB(msdu)->paddr, msdu->len,
++
++ if (!msdu)
++ return;
++
++ dma_unmap_single(ar->ab->dev, ATH11K_SKB_CB(msdu)->paddr, msdu->len,
+ DMA_TO_DEVICE);
+
+ info = IEEE80211_SKB_CB(msdu);
+ memset(&info->status, 0, sizeof(info->status));
+
+ ieee80211_free_txskb(ar->hw, msdu);
++}
++
++int ath11k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx)
++{
++ struct ath11k *ar = ctx;
++
++ ath11k_mac_tx_mgmt_free(ar, buf_id);
+
+ return 0;
+ }
+@@ -4156,17 +4176,10 @@ static int ath11k_mac_vif_txmgmt_idr_remove(int buf_id, void *skb, void *ctx)
+ {
+ struct ieee80211_vif *vif = ctx;
+ struct ath11k_skb_cb *skb_cb = ATH11K_SKB_CB((struct sk_buff *)skb);
+- struct sk_buff *msdu = skb;
+ struct ath11k *ar = skb_cb->ar;
+- struct ath11k_base *ab = ar->ab;
+
+- if (skb_cb->vif == vif) {
+- spin_lock_bh(&ar->txmgmt_idr_lock);
+- idr_remove(&ar->txmgmt_idr, buf_id);
+- spin_unlock_bh(&ar->txmgmt_idr_lock);
+- dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len,
+- DMA_TO_DEVICE);
+- }
++ if (skb_cb->vif == vif)
++ ath11k_mac_tx_mgmt_free(ar, buf_id);
+
+ return 0;
+ }
+@@ -4181,6 +4194,8 @@ static int ath11k_mac_mgmt_tx_wmi(struct ath11k *ar, struct ath11k_vif *arvif,
+ int buf_id;
+ int ret;
+
++ ATH11K_SKB_CB(skb)->ar = ar;
++
+ spin_lock_bh(&ar->txmgmt_idr_lock);
+ buf_id = idr_alloc(&ar->txmgmt_idr, skb, 0,
+ ATH11K_TX_MGMT_NUM_PENDING_MAX, GFP_ATOMIC);
+@@ -6590,7 +6605,7 @@ static int __ath11k_mac_register(struct ath11k *ar)
+ ar->hw->wiphy->interface_modes &= ~BIT(NL80211_IFTYPE_MONITOR);
+
+ /* Apply the regd received during initialization */
+- ret = ath11k_regd_update(ar, true);
++ ret = ath11k_regd_update(ar);
+ if (ret) {
+ ath11k_err(ar->ab, "ath11k regd update failed: %d\n", ret);
+ goto err_unregister_hw;
+diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c
+index 26c7ae242db67..f2149241fb131 100644
+--- a/drivers/net/wireless/ath/ath11k/mhi.c
++++ b/drivers/net/wireless/ath/ath11k/mhi.c
+@@ -519,7 +519,7 @@ static int ath11k_mhi_set_state(struct ath11k_pci *ab_pci,
+ ret = 0;
+ break;
+ case ATH11K_MHI_POWER_ON:
+- ret = mhi_async_power_up(ab_pci->mhi_ctrl);
++ ret = mhi_sync_power_up(ab_pci->mhi_ctrl);
+ break;
+ case ATH11K_MHI_POWER_OFF:
+ mhi_power_down(ab_pci->mhi_ctrl, true);
+@@ -533,7 +533,11 @@ static int ath11k_mhi_set_state(struct ath11k_pci *ab_pci,
+ ret = mhi_pm_suspend(ab_pci->mhi_ctrl);
+ break;
+ case ATH11K_MHI_RESUME:
+- ret = mhi_pm_resume(ab_pci->mhi_ctrl);
++ /* Do force MHI resume as some devices like QCA6390, WCN6855
++ * are not in M3 state but they are functional. So just ignore
++ * the MHI state while resuming.
++ */
++ ret = mhi_pm_resume_force(ab_pci->mhi_ctrl);
+ break;
+ case ATH11K_MHI_TRIGGER_RDDM:
+ ret = mhi_force_rddm_mode(ab_pci->mhi_ctrl);
+diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
+index 5abb38cc3b55f..353a2d669fcd5 100644
+--- a/drivers/net/wireless/ath/ath11k/pci.c
++++ b/drivers/net/wireless/ath/ath11k/pci.c
+@@ -632,8 +632,11 @@ static void __ath11k_pci_ext_irq_disable(struct ath11k_base *sc)
+
+ ath11k_pci_ext_grp_disable(irq_grp);
+
+- napi_synchronize(&irq_grp->napi);
+- napi_disable(&irq_grp->napi);
++ if (irq_grp->napi_enabled) {
++ napi_synchronize(&irq_grp->napi);
++ napi_disable(&irq_grp->napi);
++ irq_grp->napi_enabled = false;
++ }
+ }
+ }
+
+@@ -652,7 +655,10 @@ static void ath11k_pci_ext_irq_enable(struct ath11k_base *ab)
+ for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
+ struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
+
+- napi_enable(&irq_grp->napi);
++ if (!irq_grp->napi_enabled) {
++ napi_enable(&irq_grp->napi);
++ irq_grp->napi_enabled = true;
++ }
+ ath11k_pci_ext_grp_enable(irq_grp);
+ }
+ }
+@@ -1218,6 +1224,15 @@ static int ath11k_pci_probe(struct pci_dev *pdev,
+ goto err_free_core;
+ }
+
++ ath11k_dbg(ab, ATH11K_DBG_BOOT, "pci probe %04x:%04x %04x:%04x\n",
++ pdev->vendor, pdev->device,
++ pdev->subsystem_vendor, pdev->subsystem_device);
++
++ ab->id.vendor = pdev->vendor;
++ ab->id.device = pdev->device;
++ ab->id.subsystem_vendor = pdev->subsystem_vendor;
++ ab->id.subsystem_device = pdev->subsystem_device;
++
+ switch (pci_dev->device) {
+ case QCA6390_DEVICE_ID:
+ ath11k_pci_read_hw_version(ab, &soc_hw_version_major,
+@@ -1240,6 +1255,7 @@ static int ath11k_pci_probe(struct pci_dev *pdev,
+ ab->hw_rev = ATH11K_HW_QCN9074_HW10;
+ break;
+ case WCN6855_DEVICE_ID:
++ ab->id.bdf_search = ATH11K_BDF_SEARCH_BUS_AND_BOARD;
+ ath11k_pci_read_hw_version(ab, &soc_hw_version_major,
+ &soc_hw_version_minor);
+ switch (soc_hw_version_major) {
+@@ -1366,6 +1382,11 @@ static __maybe_unused int ath11k_pci_pm_suspend(struct device *dev)
+ struct ath11k_base *ab = dev_get_drvdata(dev);
+ int ret;
+
++ if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) {
++ ath11k_dbg(ab, ATH11K_DBG_BOOT, "boot skipping pci suspend as qmi is not initialised\n");
++ return 0;
++ }
++
+ ret = ath11k_core_suspend(ab);
+ if (ret)
+ ath11k_warn(ab, "failed to suspend core: %d\n", ret);
+@@ -1378,6 +1399,11 @@ static __maybe_unused int ath11k_pci_pm_resume(struct device *dev)
+ struct ath11k_base *ab = dev_get_drvdata(dev);
+ int ret;
+
++ if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) {
++ ath11k_dbg(ab, ATH11K_DBG_BOOT, "boot skipping pci resume as qmi is not initialised\n");
++ return 0;
++ }
++
+ ret = ath11k_core_resume(ab);
+ if (ret)
+ ath11k_warn(ab, "failed to resume core: %d\n", ret);
+diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c
+index b5e34d670715e..e4a65513a1bfd 100644
+--- a/drivers/net/wireless/ath/ath11k/qmi.c
++++ b/drivers/net/wireless/ath/ath11k/qmi.c
+@@ -1770,7 +1770,7 @@ static int ath11k_qmi_alloc_target_mem_chunk(struct ath11k_base *ab)
+ chunk->vaddr = dma_alloc_coherent(ab->dev,
+ chunk->size,
+ &chunk->paddr,
+- GFP_KERNEL);
++ GFP_KERNEL | __GFP_NOWARN);
+ if (!chunk->vaddr) {
+ if (ab->qmi.mem_seg_count <= ATH11K_QMI_FW_MEM_REQ_SEGMENT_CNT) {
+ ath11k_dbg(ab, ATH11K_DBG_QMI,
+@@ -2707,8 +2707,10 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work)
+ list_del(&event->list);
+ spin_unlock(&qmi->event_lock);
+
+- if (test_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags))
++ if (test_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags)) {
++ kfree(event);
+ return;
++ }
+
+ switch (event->type) {
+ case ATH11K_QMI_EVENT_SERVER_ARRIVE:
+diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c
+index e1a1df169034b..f793324ad0b73 100644
+--- a/drivers/net/wireless/ath/ath11k/reg.c
++++ b/drivers/net/wireless/ath/ath11k/reg.c
+@@ -198,7 +198,7 @@ static void ath11k_copy_regd(struct ieee80211_regdomain *regd_orig,
+ sizeof(struct ieee80211_reg_rule));
+ }
+
+-int ath11k_regd_update(struct ath11k *ar, bool init)
++int ath11k_regd_update(struct ath11k *ar)
+ {
+ struct ieee80211_regdomain *regd, *regd_copy = NULL;
+ int ret, regd_len, pdev_id;
+@@ -209,7 +209,10 @@ int ath11k_regd_update(struct ath11k *ar, bool init)
+
+ spin_lock_bh(&ab->base_lock);
+
+- if (init) {
++ /* Prefer the latest regd update over default if it's available */
++ if (ab->new_regd[pdev_id]) {
++ regd = ab->new_regd[pdev_id];
++ } else {
+ /* Apply the regd received during init through
+ * WMI_REG_CHAN_LIST_CC event. In case of failure to
+ * receive the regd, initialize with a default world
+@@ -222,8 +225,6 @@ int ath11k_regd_update(struct ath11k *ar, bool init)
+ "failed to receive default regd during init\n");
+ regd = (struct ieee80211_regdomain *)&ath11k_world_regd;
+ }
+- } else {
+- regd = ab->new_regd[pdev_id];
+ }
+
+ if (!regd) {
+@@ -458,6 +459,9 @@ ath11k_reg_adjust_bw(u16 start_freq, u16 end_freq, u16 max_bw)
+ {
+ u16 bw;
+
++ if (end_freq <= start_freq)
++ return 0;
++
+ bw = end_freq - start_freq;
+ bw = min_t(u16, bw, max_bw);
+
+@@ -465,8 +469,10 @@ ath11k_reg_adjust_bw(u16 start_freq, u16 end_freq, u16 max_bw)
+ bw = 80;
+ else if (bw >= 40 && bw < 80)
+ bw = 40;
+- else if (bw < 40)
++ else if (bw >= 20 && bw < 40)
+ bw = 20;
++ else
++ bw = 0;
+
+ return bw;
+ }
+@@ -490,73 +496,77 @@ ath11k_reg_update_weather_radar_band(struct ath11k_base *ab,
+ struct cur_reg_rule *reg_rule,
+ u8 *rule_idx, u32 flags, u16 max_bw)
+ {
++ u32 start_freq;
+ u32 end_freq;
+ u16 bw;
+ u8 i;
+
+ i = *rule_idx;
+
++ /* there might be situations when even the input rule must be dropped */
++ i--;
++
++ /* frequencies below weather radar */
+ bw = ath11k_reg_adjust_bw(reg_rule->start_freq,
+ ETSI_WEATHER_RADAR_BAND_LOW, max_bw);
++ if (bw > 0) {
++ i++;
+
+- ath11k_reg_update_rule(regd->reg_rules + i, reg_rule->start_freq,
+- ETSI_WEATHER_RADAR_BAND_LOW, bw,
+- reg_rule->ant_gain, reg_rule->reg_power,
+- flags);
++ ath11k_reg_update_rule(regd->reg_rules + i,
++ reg_rule->start_freq,
++ ETSI_WEATHER_RADAR_BAND_LOW, bw,
++ reg_rule->ant_gain, reg_rule->reg_power,
++ flags);
+
+- ath11k_dbg(ab, ATH11K_DBG_REG,
+- "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
+- i + 1, reg_rule->start_freq, ETSI_WEATHER_RADAR_BAND_LOW,
+- bw, reg_rule->ant_gain, reg_rule->reg_power,
+- regd->reg_rules[i].dfs_cac_ms,
+- flags);
+-
+- if (reg_rule->end_freq > ETSI_WEATHER_RADAR_BAND_HIGH)
+- end_freq = ETSI_WEATHER_RADAR_BAND_HIGH;
+- else
+- end_freq = reg_rule->end_freq;
++ ath11k_dbg(ab, ATH11K_DBG_REG,
++ "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
++ i + 1, reg_rule->start_freq,
++ ETSI_WEATHER_RADAR_BAND_LOW, bw, reg_rule->ant_gain,
++ reg_rule->reg_power, regd->reg_rules[i].dfs_cac_ms,
++ flags);
++ }
+
+- bw = ath11k_reg_adjust_bw(ETSI_WEATHER_RADAR_BAND_LOW, end_freq,
+- max_bw);
++ /* weather radar frequencies */
++ start_freq = max_t(u32, reg_rule->start_freq,
++ ETSI_WEATHER_RADAR_BAND_LOW);
++ end_freq = min_t(u32, reg_rule->end_freq, ETSI_WEATHER_RADAR_BAND_HIGH);
+
+- i++;
++ bw = ath11k_reg_adjust_bw(start_freq, end_freq, max_bw);
++ if (bw > 0) {
++ i++;
+
+- ath11k_reg_update_rule(regd->reg_rules + i,
+- ETSI_WEATHER_RADAR_BAND_LOW, end_freq, bw,
+- reg_rule->ant_gain, reg_rule->reg_power,
+- flags);
++ ath11k_reg_update_rule(regd->reg_rules + i, start_freq,
++ end_freq, bw, reg_rule->ant_gain,
++ reg_rule->reg_power, flags);
+
+- regd->reg_rules[i].dfs_cac_ms = ETSI_WEATHER_RADAR_BAND_CAC_TIMEOUT;
++ regd->reg_rules[i].dfs_cac_ms = ETSI_WEATHER_RADAR_BAND_CAC_TIMEOUT;
+
+- ath11k_dbg(ab, ATH11K_DBG_REG,
+- "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
+- i + 1, ETSI_WEATHER_RADAR_BAND_LOW, end_freq,
+- bw, reg_rule->ant_gain, reg_rule->reg_power,
+- regd->reg_rules[i].dfs_cac_ms,
+- flags);
+-
+- if (end_freq == reg_rule->end_freq) {
+- regd->n_reg_rules--;
+- *rule_idx = i;
+- return;
++ ath11k_dbg(ab, ATH11K_DBG_REG,
++ "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
++ i + 1, start_freq, end_freq, bw,
++ reg_rule->ant_gain, reg_rule->reg_power,
++ regd->reg_rules[i].dfs_cac_ms, flags);
+ }
+
++ /* frequencies above weather radar */
+ bw = ath11k_reg_adjust_bw(ETSI_WEATHER_RADAR_BAND_HIGH,
+ reg_rule->end_freq, max_bw);
++ if (bw > 0) {
++ i++;
+
+- i++;
+-
+- ath11k_reg_update_rule(regd->reg_rules + i, ETSI_WEATHER_RADAR_BAND_HIGH,
+- reg_rule->end_freq, bw,
+- reg_rule->ant_gain, reg_rule->reg_power,
+- flags);
++ ath11k_reg_update_rule(regd->reg_rules + i,
++ ETSI_WEATHER_RADAR_BAND_HIGH,
++ reg_rule->end_freq, bw,
++ reg_rule->ant_gain, reg_rule->reg_power,
++ flags);
+
+- ath11k_dbg(ab, ATH11K_DBG_REG,
+- "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
+- i + 1, ETSI_WEATHER_RADAR_BAND_HIGH, reg_rule->end_freq,
+- bw, reg_rule->ant_gain, reg_rule->reg_power,
+- regd->reg_rules[i].dfs_cac_ms,
+- flags);
++ ath11k_dbg(ab, ATH11K_DBG_REG,
++ "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
++ i + 1, ETSI_WEATHER_RADAR_BAND_HIGH,
++ reg_rule->end_freq, bw, reg_rule->ant_gain,
++ reg_rule->reg_power, regd->reg_rules[i].dfs_cac_ms,
++ flags);
++ }
+
+ *rule_idx = i;
+ }
+@@ -683,7 +693,7 @@ void ath11k_regd_update_work(struct work_struct *work)
+ regd_update_work);
+ int ret;
+
+- ret = ath11k_regd_update(ar, false);
++ ret = ath11k_regd_update(ar);
+ if (ret) {
+ /* Firmware has already moved to the new regd. We need
+ * to maintain channel consistency across FW, Host driver
+diff --git a/drivers/net/wireless/ath/ath11k/reg.h b/drivers/net/wireless/ath/ath11k/reg.h
+index 65d56d44796f6..5fb9dc03a74e8 100644
+--- a/drivers/net/wireless/ath/ath11k/reg.h
++++ b/drivers/net/wireless/ath/ath11k/reg.h
+@@ -31,6 +31,6 @@ void ath11k_regd_update_work(struct work_struct *work);
+ struct ieee80211_regdomain *
+ ath11k_reg_build_regd(struct ath11k_base *ab,
+ struct cur_regulatory_info *reg_info, bool intersect);
+-int ath11k_regd_update(struct ath11k *ar, bool init);
++int ath11k_regd_update(struct ath11k *ar);
+ int ath11k_reg_update_chan_list(struct ath11k *ar);
+ #endif
+diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
+index 6c253eae9d069..b11070cf159cc 100644
+--- a/drivers/net/wireless/ath/ath11k/wmi.c
++++ b/drivers/net/wireless/ath/ath11k/wmi.c
+@@ -1339,6 +1339,7 @@ int ath11k_wmi_pdev_bss_chan_info_request(struct ath11k *ar,
+ WMI_TAG_PDEV_BSS_CHAN_INFO_REQUEST) |
+ FIELD_PREP(WMI_TLV_LEN, sizeof(*cmd) - TLV_HDR_SIZE);
+ cmd->req_type = type;
++ cmd->pdev_id = ar->pdev->pdev_id;
+
+ ath11k_dbg(ar->ab, ATH11K_DBG_WMI,
+ "WMI bss chan info req type %d\n", type);
+@@ -1670,7 +1671,8 @@ int ath11k_wmi_vdev_install_key(struct ath11k *ar,
+ tlv = (struct wmi_tlv *)(skb->data + sizeof(*cmd));
+ tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) |
+ FIELD_PREP(WMI_TLV_LEN, key_len_aligned);
+- memcpy(tlv->value, (u8 *)arg->key_data, key_len_aligned);
++ if (arg->key_data)
++ memcpy(tlv->value, (u8 *)arg->key_data, key_len_aligned);
+
+ ret = ath11k_wmi_cmd_send(wmi, skb, WMI_VDEV_INSTALL_KEY_CMDID);
+ if (ret) {
+@@ -2050,7 +2052,7 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar,
+ void *ptr;
+ int i, ret, len;
+ u32 *tmp_ptr;
+- u8 extraie_len_with_pad = 0;
++ u16 extraie_len_with_pad = 0;
+ struct hint_short_ssid *s_ssid = NULL;
+ struct hint_bssid *hint_bssid = NULL;
+
+@@ -2069,7 +2071,7 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar,
+ len += sizeof(*bssid) * params->num_bssid;
+
+ len += TLV_HDR_SIZE;
+- if (params->extraie.len)
++ if (params->extraie.len && params->extraie.len <= 0xFFFF)
+ extraie_len_with_pad =
+ roundup(params->extraie.len, sizeof(u32));
+ len += extraie_len_with_pad;
+@@ -2176,7 +2178,7 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar,
+ FIELD_PREP(WMI_TLV_LEN, len);
+ ptr += TLV_HDR_SIZE;
+
+- if (params->extraie.len)
++ if (extraie_len_with_pad)
+ memcpy(ptr, params->extraie.ptr,
+ params->extraie.len);
+
+@@ -5792,6 +5794,17 @@ static int ath11k_reg_chan_list_event(struct ath11k_base *ab, struct sk_buff *sk
+
+ pdev_idx = reg_info->phy_id;
+
++ /* Avoid default reg rule updates sent during FW recovery if
++ * it is already available
++ */
++ spin_lock(&ab->base_lock);
++ if (test_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags) &&
++ ab->default_regd[pdev_idx]) {
++ spin_unlock(&ab->base_lock);
++ goto mem_free;
++ }
++ spin_unlock(&ab->base_lock);
++
+ if (pdev_idx >= ab->num_radios) {
+ /* Process the event for phy0 only if single_pdev_only
+ * is true. If pdev_idx is valid but not 0, discard the
+@@ -5829,10 +5842,10 @@ static int ath11k_reg_chan_list_event(struct ath11k_base *ab, struct sk_buff *sk
+ }
+
+ spin_lock(&ab->base_lock);
+- if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) {
+- /* Once mac is registered, ar is valid and all CC events from
+- * fw is considered to be received due to user requests
+- * currently.
++ if (ab->default_regd[pdev_idx]) {
++ /* The initial rules from FW after WMI Init is to build
++ * the default regd. From then on, any rules updated for
++ * the pdev could be due to user reg changes.
+ * Free previously built regd before assigning the newly
+ * generated regd to ar. NULL pointer handling will be
+ * taken care by kfree itself.
+@@ -5840,15 +5853,11 @@ static int ath11k_reg_chan_list_event(struct ath11k_base *ab, struct sk_buff *sk
+ ar = ab->pdevs[pdev_idx].ar;
+ kfree(ab->new_regd[pdev_idx]);
+ ab->new_regd[pdev_idx] = regd;
+- ieee80211_queue_work(ar->hw, &ar->regd_update_work);
++ queue_work(ab->workqueue, &ar->regd_update_work);
+ } else {
+- /* Multiple events for the same *ar is not expected. But we
+- * can still clear any previously stored default_regd if we
+- * are receiving this event for the same radio by mistake.
+- * NULL pointer handling will be taken care by kfree itself.
++ /* This regd would be applied during mac registration and is
++ * held constant throughout for regd intersection purpose
+ */
+- kfree(ab->default_regd[pdev_idx]);
+- /* This regd would be applied during mac registration */
+ ab->default_regd[pdev_idx] = regd;
+ }
+ ab->dfs_region = reg_info->dfs_region;
+@@ -6119,8 +6128,10 @@ static void ath11k_mgmt_rx_event(struct ath11k_base *ab, struct sk_buff *skb)
+ if (rx_ev.status & WMI_RX_STATUS_ERR_MIC)
+ status->flag |= RX_FLAG_MMIC_ERROR;
+
+- if (rx_ev.chan_freq >= ATH11K_MIN_6G_FREQ) {
++ if (rx_ev.chan_freq >= ATH11K_MIN_6G_FREQ &&
++ rx_ev.chan_freq <= ATH11K_MAX_6G_FREQ) {
+ status->band = NL80211_BAND_6GHZ;
++ status->freq = rx_ev.chan_freq;
+ } else if (rx_ev.channel >= 1 && rx_ev.channel <= 14) {
+ status->band = NL80211_BAND_2GHZ;
+ } else if (rx_ev.channel >= 36 && rx_ev.channel <= ATH11K_MAX_5G_CHAN) {
+@@ -6141,8 +6152,10 @@ static void ath11k_mgmt_rx_event(struct ath11k_base *ab, struct sk_buff *skb)
+
+ sband = &ar->mac.sbands[status->band];
+
+- status->freq = ieee80211_channel_to_frequency(rx_ev.channel,
+- status->band);
++ if (status->band != NL80211_BAND_6GHZ)
++ status->freq = ieee80211_channel_to_frequency(rx_ev.channel,
++ status->band);
++
+ status->signal = rx_ev.snr + ATH11K_DEFAULT_NOISE_FLOOR;
+ status->rate_idx = ath11k_mac_bitrate_to_idx(sband, rx_ev.rate / 100);
+
+@@ -6301,6 +6314,8 @@ static void ath11k_scan_event(struct ath11k_base *ab, struct sk_buff *skb)
+ ath11k_wmi_event_scan_start_failed(ar);
+ break;
+ case WMI_SCAN_EVENT_DEQUEUED:
++ __ath11k_mac_scan_finish(ar);
++ break;
+ case WMI_SCAN_EVENT_PREEMPTED:
+ case WMI_SCAN_EVENT_RESTARTED:
+ case WMI_SCAN_EVENT_FOREIGN_CHAN_EXIT:
+diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h
+index d35c47e0b19d4..0b7d337b36930 100644
+--- a/drivers/net/wireless/ath/ath11k/wmi.h
++++ b/drivers/net/wireless/ath/ath11k/wmi.h
+@@ -2960,6 +2960,7 @@ struct wmi_pdev_bss_chan_info_req_cmd {
+ u32 tlv_header;
+ /* ref wmi_bss_chan_info_req_type */
+ u32 req_type;
++ u32 pdev_id;
+ } __packed;
+
+ struct wmi_ap_ps_peer_cmd {
+@@ -4056,7 +4057,6 @@ struct wmi_vdev_stopped_event {
+ } __packed;
+
+ struct wmi_pdev_bss_chan_info_event {
+- u32 pdev_id;
+ u32 freq; /* Units in MHz */
+ u32 noise_floor; /* units are dBm */
+ /* rx clear - how often the channel was unused */
+@@ -4074,6 +4074,7 @@ struct wmi_pdev_bss_chan_info_event {
+ /*rx_cycle cnt for my bss in 64bits format */
+ u32 rx_bss_cycle_count_low;
+ u32 rx_bss_cycle_count_high;
++ u32 pdev_id;
+ } __packed;
+
+ #define WMI_VDEV_INSTALL_KEY_COMPL_STATUS_SUCCESS 0
+diff --git a/drivers/net/wireless/ath/ath5k/eeprom.c b/drivers/net/wireless/ath/ath5k/eeprom.c
+index 1fbc2c19848f2..d444b3d70ba2e 100644
+--- a/drivers/net/wireless/ath/ath5k/eeprom.c
++++ b/drivers/net/wireless/ath/ath5k/eeprom.c
+@@ -746,6 +746,9 @@ ath5k_eeprom_convert_pcal_info_5111(struct ath5k_hw *ah, int mode,
+ }
+ }
+
++ if (idx == AR5K_EEPROM_N_PD_CURVES)
++ goto err_out;
++
+ ee->ee_pd_gains[mode] = 1;
+
+ pd = &chinfo[pier].pd_curves[idx];
+diff --git a/drivers/net/wireless/ath/ath6kl/usb.c b/drivers/net/wireless/ath/ath6kl/usb.c
+index 5372e948e761d..aba70f35e574b 100644
+--- a/drivers/net/wireless/ath/ath6kl/usb.c
++++ b/drivers/net/wireless/ath/ath6kl/usb.c
+@@ -340,6 +340,11 @@ static int ath6kl_usb_setup_pipe_resources(struct ath6kl_usb *ar_usb)
+ le16_to_cpu(endpoint->wMaxPacketSize),
+ endpoint->bInterval);
+ }
++
++ /* Ignore broken descriptors. */
++ if (usb_endpoint_maxp(endpoint) == 0)
++ continue;
++
+ urbcount = 0;
+
+ pipe_num =
+@@ -907,7 +912,7 @@ static int ath6kl_usb_submit_ctrl_in(struct ath6kl_usb *ar_usb,
+ req,
+ USB_DIR_IN | USB_TYPE_VENDOR |
+ USB_RECIP_DEVICE, value, index, buf,
+- size, 2 * HZ);
++ size, 2000);
+
+ if (ret < 0) {
+ ath6kl_warn("Failed to read usb control message: %d\n", ret);
+diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
+index 860da13bfb6ac..f06eec99de688 100644
+--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
++++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
+@@ -590,6 +590,13 @@ static void ath9k_hif_usb_rx_stream(struct hif_device_usb *hif_dev,
+ return;
+ }
+
++ if (pkt_len > 2 * MAX_RX_BUF_SIZE) {
++ dev_err(&hif_dev->udev->dev,
++ "ath9k_htc: invalid pkt_len (%x)\n", pkt_len);
++ RX_STAT_INC(skb_dropped);
++ return;
++ }
++
+ pad_len = 4 - (pkt_len & 0x3);
+ if (pad_len == 4)
+ pad_len = 0;
+diff --git a/drivers/net/wireless/ath/ath9k/htc.h b/drivers/net/wireless/ath/ath9k/htc.h
+index 0a1634238e673..6b45e63fae4ba 100644
+--- a/drivers/net/wireless/ath/ath9k/htc.h
++++ b/drivers/net/wireless/ath/ath9k/htc.h
+@@ -281,6 +281,7 @@ struct ath9k_htc_rxbuf {
+ struct ath9k_htc_rx {
+ struct list_head rxbuf;
+ spinlock_t rxbuflock;
++ bool initialized;
+ };
+
+ #define ATH9K_HTC_TX_CLEANUP_INTERVAL 50 /* ms */
+@@ -305,6 +306,7 @@ struct ath9k_htc_tx {
+ DECLARE_BITMAP(tx_slot, MAX_TX_BUF_NUM);
+ struct timer_list cleanup_timer;
+ spinlock_t tx_lock;
++ bool initialized;
+ };
+
+ struct ath9k_htc_tx_ctl {
+diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
+index 8e69e8989f6d3..6a850a0bfa8ad 100644
+--- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
++++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
+@@ -813,6 +813,11 @@ int ath9k_tx_init(struct ath9k_htc_priv *priv)
+ skb_queue_head_init(&priv->tx.data_vi_queue);
+ skb_queue_head_init(&priv->tx.data_vo_queue);
+ skb_queue_head_init(&priv->tx.tx_failed);
++
++ /* Allow ath9k_wmi_event_tasklet(WMI_TXSTATUS_EVENTID) to operate. */
++ smp_wmb();
++ priv->tx.initialized = true;
++
+ return 0;
+ }
+
+@@ -1130,6 +1135,10 @@ void ath9k_htc_rxep(void *drv_priv, struct sk_buff *skb,
+ struct ath9k_htc_rxbuf *rxbuf = NULL, *tmp_buf = NULL;
+ unsigned long flags;
+
++ /* Check if ath9k_rx_init() completed. */
++ if (!data_race(priv->rx.initialized))
++ goto err;
++
+ spin_lock_irqsave(&priv->rx.rxbuflock, flags);
+ list_for_each_entry(tmp_buf, &priv->rx.rxbuf, list) {
+ if (!tmp_buf->in_process) {
+@@ -1185,6 +1194,10 @@ int ath9k_rx_init(struct ath9k_htc_priv *priv)
+ list_add_tail(&rxbuf->list, &priv->rx.rxbuf);
+ }
+
++ /* Allow ath9k_htc_rxep() to operate. */
++ smp_wmb();
++ priv->rx.initialized = true;
++
+ return 0;
+
+ err:
+diff --git a/drivers/net/wireless/ath/ath9k/htc_hst.c b/drivers/net/wireless/ath/ath9k/htc_hst.c
+index 510e61e97dbcb..994ec48b2f669 100644
+--- a/drivers/net/wireless/ath/ath9k/htc_hst.c
++++ b/drivers/net/wireless/ath/ath9k/htc_hst.c
+@@ -30,6 +30,7 @@ static int htc_issue_send(struct htc_target *target, struct sk_buff* skb,
+ hdr->endpoint_id = epid;
+ hdr->flags = flags;
+ hdr->payload_len = cpu_to_be16(len);
++ memset(hdr->control, 0, sizeof(hdr->control));
+
+ status = target->hif->send(target->hif_dev, endpoint->ul_pipeid, skb);
+
+@@ -272,6 +273,10 @@ int htc_connect_service(struct htc_target *target,
+ conn_msg->dl_pipeid = endpoint->dl_pipeid;
+ conn_msg->ul_pipeid = endpoint->ul_pipeid;
+
++ /* To prevent infoleak */
++ conn_msg->svc_meta_len = 0;
++ conn_msg->pad = 0;
++
+ ret = htc_issue_send(target, skb, skb->len, 0, ENDPOINT0);
+ if (ret)
+ goto err;
+diff --git a/drivers/net/wireless/ath/ath9k/main.c b/drivers/net/wireless/ath/ath9k/main.c
+index 139831539da37..e2791d45f5f59 100644
+--- a/drivers/net/wireless/ath/ath9k/main.c
++++ b/drivers/net/wireless/ath/ath9k/main.c
+@@ -533,8 +533,10 @@ irqreturn_t ath_isr(int irq, void *dev)
+ ath9k_debug_sync_cause(sc, sync_cause);
+ status &= ah->imask; /* discard unasked-for bits */
+
+- if (test_bit(ATH_OP_HW_RESET, &common->op_flags))
++ if (test_bit(ATH_OP_HW_RESET, &common->op_flags)) {
++ ath9k_hw_kill_interrupts(sc->sc_ah);
+ return IRQ_HANDLED;
++ }
+
+ /*
+ * If there are no status bits set, then this interrupt was not
+@@ -837,7 +839,7 @@ static bool ath9k_txq_list_has_key(struct list_head *txq_list, u32 keyix)
+ continue;
+
+ txinfo = IEEE80211_SKB_CB(bf->bf_mpdu);
+- fi = (struct ath_frame_info *)&txinfo->rate_driver_data[0];
++ fi = (struct ath_frame_info *)&txinfo->status.status_driver_data[0];
+ if (fi->keyix == keyix)
+ return true;
+ }
+diff --git a/drivers/net/wireless/ath/ath9k/wmi.c b/drivers/net/wireless/ath/ath9k/wmi.c
+index fe29ad4b9023c..f315c54bd3ac0 100644
+--- a/drivers/net/wireless/ath/ath9k/wmi.c
++++ b/drivers/net/wireless/ath/ath9k/wmi.c
+@@ -169,6 +169,10 @@ void ath9k_wmi_event_tasklet(struct tasklet_struct *t)
+ &wmi->drv_priv->fatal_work);
+ break;
+ case WMI_TXSTATUS_EVENTID:
++ /* Check if ath9k_tx_init() completed. */
++ if (!data_race(priv->tx.initialized))
++ break;
++
+ spin_lock_bh(&priv->tx.tx_lock);
+ if (priv->tx.flags & ATH9K_HTC_OP_TX_DRAIN) {
+ spin_unlock_bh(&priv->tx.tx_lock);
+diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
+index 5691bd6eb82c2..6555abf02f18b 100644
+--- a/drivers/net/wireless/ath/ath9k/xmit.c
++++ b/drivers/net/wireless/ath/ath9k/xmit.c
+@@ -141,8 +141,8 @@ static struct ath_frame_info *get_frame_info(struct sk_buff *skb)
+ {
+ struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
+ BUILD_BUG_ON(sizeof(struct ath_frame_info) >
+- sizeof(tx_info->rate_driver_data));
+- return (struct ath_frame_info *) &tx_info->rate_driver_data[0];
++ sizeof(tx_info->status.status_driver_data));
++ return (struct ath_frame_info *) &tx_info->status.status_driver_data[0];
+ }
+
+ static void ath_send_bar(struct ath_atx_tid *tid, u16 seqno)
+@@ -2501,6 +2501,16 @@ skip_tx_complete:
+ spin_unlock_irqrestore(&sc->tx.txbuflock, flags);
+ }
+
++static void ath_clear_tx_status(struct ieee80211_tx_info *tx_info)
++{
++ void *ptr = &tx_info->status;
++
++ memset(ptr + sizeof(tx_info->status.rates), 0,
++ sizeof(tx_info->status) -
++ sizeof(tx_info->status.rates) -
++ sizeof(tx_info->status.status_driver_data));
++}
++
+ static void ath_tx_rc_status(struct ath_softc *sc, struct ath_buf *bf,
+ struct ath_tx_status *ts, int nframes, int nbad,
+ int txok)
+@@ -2512,6 +2522,8 @@ static void ath_tx_rc_status(struct ath_softc *sc, struct ath_buf *bf,
+ struct ath_hw *ah = sc->sc_ah;
+ u8 i, tx_rateindex;
+
++ ath_clear_tx_status(tx_info);
++
+ if (txok)
+ tx_info->status.ack_signal = ts->ts_rssi;
+
+@@ -2526,6 +2538,13 @@ static void ath_tx_rc_status(struct ath_softc *sc, struct ath_buf *bf,
+ tx_info->status.ampdu_len = nframes;
+ tx_info->status.ampdu_ack_len = nframes - nbad;
+
++ tx_info->status.rates[tx_rateindex].count = ts->ts_longretry + 1;
++
++ for (i = tx_rateindex + 1; i < hw->max_rates; i++) {
++ tx_info->status.rates[i].count = 0;
++ tx_info->status.rates[i].idx = -1;
++ }
++
+ if ((ts->ts_status & ATH9K_TXERR_FILT) == 0 &&
+ (tx_info->flags & IEEE80211_TX_CTL_NO_ACK) == 0) {
+ /*
+@@ -2547,16 +2566,6 @@ static void ath_tx_rc_status(struct ath_softc *sc, struct ath_buf *bf,
+ tx_info->status.rates[tx_rateindex].count =
+ hw->max_rate_tries;
+ }
+-
+- for (i = tx_rateindex + 1; i < hw->max_rates; i++) {
+- tx_info->status.rates[i].count = 0;
+- tx_info->status.rates[i].idx = -1;
+- }
+-
+- tx_info->status.rates[tx_rateindex].count = ts->ts_longretry + 1;
+-
+- /* we report airtime in ath_tx_count_airtime(), don't report twice */
+- tx_info->status.tx_time = 0;
+ }
+
+ static void ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
+diff --git a/drivers/net/wireless/ath/carl9170/main.c b/drivers/net/wireless/ath/carl9170/main.c
+index cca3b086aa701..a87476383c540 100644
+--- a/drivers/net/wireless/ath/carl9170/main.c
++++ b/drivers/net/wireless/ath/carl9170/main.c
+@@ -1915,7 +1915,7 @@ static int carl9170_parse_eeprom(struct ar9170 *ar)
+ WARN_ON(!(tx_streams >= 1 && tx_streams <=
+ IEEE80211_HT_MCS_TX_MAX_STREAMS));
+
+- tx_params = (tx_streams - 1) <<
++ tx_params |= (tx_streams - 1) <<
+ IEEE80211_HT_MCS_TX_MAX_STREAMS_SHIFT;
+
+ carl9170_band_2GHz.ht_cap.mcs.tx_params |= tx_params;
+diff --git a/drivers/net/wireless/ath/dfs_pattern_detector.c b/drivers/net/wireless/ath/dfs_pattern_detector.c
+index 80390495ea250..75cb53a3ec15e 100644
+--- a/drivers/net/wireless/ath/dfs_pattern_detector.c
++++ b/drivers/net/wireless/ath/dfs_pattern_detector.c
+@@ -183,10 +183,12 @@ static void channel_detector_exit(struct dfs_pattern_detector *dpd,
+ if (cd == NULL)
+ return;
+ list_del(&cd->head);
+- for (i = 0; i < dpd->num_radar_types; i++) {
+- struct pri_detector *de = cd->detectors[i];
+- if (de != NULL)
+- de->exit(de);
++ if (cd->detectors) {
++ for (i = 0; i < dpd->num_radar_types; i++) {
++ struct pri_detector *de = cd->detectors[i];
++ if (de != NULL)
++ de->exit(de);
++ }
+ }
+ kfree(cd->detectors);
+ kfree(cd);
+diff --git a/drivers/net/wireless/ath/regd.c b/drivers/net/wireless/ath/regd.c
+index b2400e2417a55..f15e7bd690b5b 100644
+--- a/drivers/net/wireless/ath/regd.c
++++ b/drivers/net/wireless/ath/regd.c
+@@ -667,14 +667,14 @@ ath_regd_init_wiphy(struct ath_regulatory *reg,
+
+ /*
+ * Some users have reported their EEPROM programmed with
+- * 0x8000 or 0x0 set, this is not a supported regulatory
+- * domain but since we have more than one user with it we
+- * need a solution for them. We default to 0x64, which is
+- * the default Atheros world regulatory domain.
++ * 0x8000 set, this is not a supported regulatory domain
++ * but since we have more than one user with it we need
++ * a solution for them. We default to 0x64, which is the
++ * default Atheros world regulatory domain.
+ */
+ static void ath_regd_sanitize(struct ath_regulatory *reg)
+ {
+- if (reg->current_rd != COUNTRY_ERD_FLAG && reg->current_rd != 0)
++ if (reg->current_rd != COUNTRY_ERD_FLAG)
+ return;
+ printk(KERN_DEBUG "ath: EEPROM regdomain sanitized\n");
+ reg->current_rd = 0x64;
+diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c
+index 8e1dbfda65386..e1a35c2eadb6c 100644
+--- a/drivers/net/wireless/ath/wcn36xx/dxe.c
++++ b/drivers/net/wireless/ath/wcn36xx/dxe.c
+@@ -272,6 +272,21 @@ static int wcn36xx_dxe_enable_ch_int(struct wcn36xx *wcn, u16 wcn_ch)
+ return 0;
+ }
+
++static void wcn36xx_dxe_disable_ch_int(struct wcn36xx *wcn, u16 wcn_ch)
++{
++ int reg_data = 0;
++
++ wcn36xx_dxe_read_register(wcn,
++ WCN36XX_DXE_INT_MASK_REG,
++ &reg_data);
++
++ reg_data &= ~wcn_ch;
++
++ wcn36xx_dxe_write_register(wcn,
++ WCN36XX_DXE_INT_MASK_REG,
++ (int)reg_data);
++}
++
+ static int wcn36xx_dxe_fill_skb(struct device *dev,
+ struct wcn36xx_dxe_ctl *ctl,
+ gfp_t gfp)
+@@ -403,8 +418,21 @@ static void reap_tx_dxes(struct wcn36xx *wcn, struct wcn36xx_dxe_ch *ch)
+ dma_unmap_single(wcn->dev, ctl->desc->src_addr_l,
+ ctl->skb->len, DMA_TO_DEVICE);
+ info = IEEE80211_SKB_CB(ctl->skb);
+- if (!(info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)) {
+- /* Keep frame until TX status comes */
++ if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
++ if (info->flags & IEEE80211_TX_CTL_NO_ACK) {
++ info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
++ ieee80211_tx_status_irqsafe(wcn->hw, ctl->skb);
++ } else {
++ /* Wait for the TX ack indication or timeout... */
++ spin_lock(&wcn->dxe_lock);
++ if (WARN_ON(wcn->tx_ack_skb))
++ ieee80211_free_txskb(wcn->hw, wcn->tx_ack_skb);
++ wcn->tx_ack_skb = ctl->skb; /* Tracking ref */
++ mod_timer(&wcn->tx_ack_timer, jiffies + HZ / 10);
++ spin_unlock(&wcn->dxe_lock);
++ }
++ /* do not free, ownership transferred to mac80211 status cb */
++ } else {
+ ieee80211_free_txskb(wcn->hw, ctl->skb);
+ }
+
+@@ -426,7 +454,6 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
+ {
+ struct wcn36xx *wcn = (struct wcn36xx *)dev;
+ int int_src, int_reason;
+- bool transmitted = false;
+
+ wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_INT_SRC_RAW_REG, &int_src);
+
+@@ -466,7 +493,6 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
+ if (int_reason & (WCN36XX_CH_STAT_INT_DONE_MASK |
+ WCN36XX_CH_STAT_INT_ED_MASK)) {
+ reap_tx_dxes(wcn, &wcn->dxe_tx_h_ch);
+- transmitted = true;
+ }
+ }
+
+@@ -479,7 +505,6 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
+ WCN36XX_DXE_0_INT_CLR,
+ WCN36XX_INT_MASK_CHAN_TX_L);
+
+-
+ if (int_reason & WCN36XX_CH_STAT_INT_ERR_MASK ) {
+ wcn36xx_dxe_write_register(wcn,
+ WCN36XX_DXE_0_INT_ERR_CLR,
+@@ -507,25 +532,8 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
+ if (int_reason & (WCN36XX_CH_STAT_INT_DONE_MASK |
+ WCN36XX_CH_STAT_INT_ED_MASK)) {
+ reap_tx_dxes(wcn, &wcn->dxe_tx_l_ch);
+- transmitted = true;
+- }
+- }
+-
+- spin_lock(&wcn->dxe_lock);
+- if (wcn->tx_ack_skb && transmitted) {
+- struct ieee80211_tx_info *info = IEEE80211_SKB_CB(wcn->tx_ack_skb);
+-
+- /* TX complete, no need to wait for 802.11 ack indication */
+- if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS &&
+- info->flags & IEEE80211_TX_CTL_NO_ACK) {
+- info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+- del_timer(&wcn->tx_ack_timer);
+- ieee80211_tx_status_irqsafe(wcn->hw, wcn->tx_ack_skb);
+- wcn->tx_ack_skb = NULL;
+- ieee80211_wake_queues(wcn->hw);
+ }
+ }
+- spin_unlock(&wcn->dxe_lock);
+
+ return IRQ_HANDLED;
+ }
+@@ -613,6 +621,10 @@ static int wcn36xx_rx_handle_packets(struct wcn36xx *wcn,
+ dxe = ctl->desc;
+
+ while (!(READ_ONCE(dxe->ctrl) & WCN36xx_DXE_CTRL_VLD)) {
++ /* do not read until we own DMA descriptor */
++ dma_rmb();
++
++ /* read/modify DMA descriptor */
+ skb = ctl->skb;
+ dma_addr = dxe->dst_addr_l;
+ ret = wcn36xx_dxe_fill_skb(wcn->dev, ctl, GFP_ATOMIC);
+@@ -623,9 +635,15 @@ static int wcn36xx_rx_handle_packets(struct wcn36xx *wcn,
+ dma_unmap_single(wcn->dev, dma_addr, WCN36XX_PKT_SIZE,
+ DMA_FROM_DEVICE);
+ wcn36xx_rx_skb(wcn, skb);
+- } /* else keep old skb not submitted and use it for rx DMA */
++ }
++ /* else keep old skb not submitted and reuse it for rx DMA
++ * (dropping the packet that it contained)
++ */
+
++ /* flush descriptor changes before re-marking as valid */
++ dma_wmb();
+ dxe->ctrl = ctrl;
++
+ ctl = ctl->next;
+ dxe = ctl->desc;
+ }
+@@ -866,7 +884,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
+ WCN36XX_DXE_WQ_TX_L);
+
+ wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, &reg_data);
+- wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L);
+
+ /***************************************/
+ /* Init descriptors for TX HIGH channel */
+@@ -890,9 +907,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
+
+ wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, &reg_data);
+
+- /* Enable channel interrupts */
+- wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H);
+-
+ /***************************************/
+ /* Init descriptors for RX LOW channel */
+ /***************************************/
+@@ -902,7 +916,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
+ goto out_err_rxl_ch;
+ }
+
+-
+ /* For RX we need to preallocated buffers */
+ wcn36xx_dxe_ch_alloc_skb(wcn, &wcn->dxe_rx_l_ch);
+
+@@ -925,9 +938,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
+ WCN36XX_DXE_REG_CTL_RX_L,
+ WCN36XX_DXE_CH_DEFAULT_CTL_RX_L);
+
+- /* Enable channel interrupts */
+- wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L);
+-
+ /***************************************/
+ /* Init descriptors for RX HIGH channel */
+ /***************************************/
+@@ -959,15 +969,18 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
+ WCN36XX_DXE_REG_CTL_RX_H,
+ WCN36XX_DXE_CH_DEFAULT_CTL_RX_H);
+
+- /* Enable channel interrupts */
+- wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H);
+-
+ ret = wcn36xx_dxe_request_irqs(wcn);
+ if (ret < 0)
+ goto out_err_irq;
+
+ timer_setup(&wcn->tx_ack_timer, wcn36xx_dxe_tx_timer, 0);
+
++ /* Enable channel interrupts */
++ wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L);
++ wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H);
++ wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L);
++ wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H);
++
+ return 0;
+
+ out_err_irq:
+@@ -984,6 +997,14 @@ out_err_txh_ch:
+
+ void wcn36xx_dxe_deinit(struct wcn36xx *wcn)
+ {
++ int reg_data = 0;
++
++ /* Disable channel interrupts */
++ wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H);
++ wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L);
++ wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H);
++ wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L);
++
+ free_irq(wcn->tx_irq, wcn);
+ free_irq(wcn->rx_irq, wcn);
+ del_timer(&wcn->tx_ack_timer);
+@@ -993,6 +1014,15 @@ void wcn36xx_dxe_deinit(struct wcn36xx *wcn)
+ wcn->tx_ack_skb = NULL;
+ }
+
++ /* Put the DXE block into reset before freeing memory */
++ reg_data = WCN36XX_DXE_REG_RESET;
++ wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_REG_CSR_RESET, reg_data);
++
+ wcn36xx_dxe_ch_free_skbs(wcn, &wcn->dxe_rx_l_ch);
+ wcn36xx_dxe_ch_free_skbs(wcn, &wcn->dxe_rx_h_ch);
++
++ wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_l_ch);
++ wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_h_ch);
++ wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_l_ch);
++ wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_h_ch);
+ }
+diff --git a/drivers/net/wireless/ath/wcn36xx/hal.h b/drivers/net/wireless/ath/wcn36xx/hal.h
+index 455143c4164ee..de3bca043c2b3 100644
+--- a/drivers/net/wireless/ath/wcn36xx/hal.h
++++ b/drivers/net/wireless/ath/wcn36xx/hal.h
+@@ -359,6 +359,8 @@ enum wcn36xx_hal_host_msg_type {
+ WCN36XX_HAL_START_SCAN_OFFLOAD_RSP = 205,
+ WCN36XX_HAL_STOP_SCAN_OFFLOAD_REQ = 206,
+ WCN36XX_HAL_STOP_SCAN_OFFLOAD_RSP = 207,
++ WCN36XX_HAL_UPDATE_CHANNEL_LIST_REQ = 208,
++ WCN36XX_HAL_UPDATE_CHANNEL_LIST_RSP = 209,
+ WCN36XX_HAL_SCAN_OFFLOAD_IND = 210,
+
+ WCN36XX_HAL_AVOID_FREQ_RANGE_IND = 233,
+@@ -1353,6 +1355,36 @@ struct wcn36xx_hal_stop_scan_offload_rsp_msg {
+ u32 status;
+ } __packed;
+
++#define WCN36XX_HAL_CHAN_REG1_MIN_PWR_MASK 0x000000ff
++#define WCN36XX_HAL_CHAN_REG1_MAX_PWR_MASK 0x0000ff00
++#define WCN36XX_HAL_CHAN_REG1_REG_PWR_MASK 0x00ff0000
++#define WCN36XX_HAL_CHAN_REG1_CLASS_ID_MASK 0xff000000
++#define WCN36XX_HAL_CHAN_REG2_ANT_GAIN_MASK 0x000000ff
++#define WCN36XX_HAL_CHAN_INFO_FLAG_PASSIVE BIT(7)
++#define WCN36XX_HAL_CHAN_INFO_FLAG_DFS BIT(10)
++#define WCN36XX_HAL_CHAN_INFO_FLAG_HT BIT(11)
++#define WCN36XX_HAL_CHAN_INFO_FLAG_VHT BIT(12)
++#define WCN36XX_HAL_CHAN_INFO_PHY_11A 0
++#define WCN36XX_HAL_CHAN_INFO_PHY_11BG 1
++#define WCN36XX_HAL_DEFAULT_ANT_GAIN 6
++#define WCN36XX_HAL_DEFAULT_MIN_POWER 6
++
++struct wcn36xx_hal_channel_param {
++ u32 mhz;
++ u32 band_center_freq1;
++ u32 band_center_freq2;
++ u32 channel_info;
++ u32 reg_info_1;
++ u32 reg_info_2;
++} __packed;
++
++struct wcn36xx_hal_update_channel_list_req_msg {
++ struct wcn36xx_hal_msg_header header;
++
++ u8 num_channel;
++ struct wcn36xx_hal_channel_param channels[80];
++} __packed;
++
+ enum wcn36xx_hal_rate_index {
+ HW_RATE_INDEX_1MBPS = 0x82,
+ HW_RATE_INDEX_2MBPS = 0x84,
+diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c
+index ec913ec991f3f..d51a783301359 100644
+--- a/drivers/net/wireless/ath/wcn36xx/main.c
++++ b/drivers/net/wireless/ath/wcn36xx/main.c
+@@ -135,7 +135,9 @@ static struct ieee80211_supported_band wcn_band_2ghz = {
+ .cap = IEEE80211_HT_CAP_GRN_FLD |
+ IEEE80211_HT_CAP_SGI_20 |
+ IEEE80211_HT_CAP_DSSSCCK40 |
+- IEEE80211_HT_CAP_LSIG_TXOP_PROT,
++ IEEE80211_HT_CAP_LSIG_TXOP_PROT |
++ IEEE80211_HT_CAP_SGI_40 |
++ IEEE80211_HT_CAP_SUP_WIDTH_20_40,
+ .ht_supported = true,
+ .ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K,
+ .ampdu_density = IEEE80211_HT_MPDU_DENSITY_16,
+@@ -398,6 +400,7 @@ static void wcn36xx_change_opchannel(struct wcn36xx *wcn, int ch)
+ static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed)
+ {
+ struct wcn36xx *wcn = hw->priv;
++ int ret;
+
+ wcn36xx_dbg(WCN36XX_DBG_MAC, "mac config changed 0x%08x\n", changed);
+
+@@ -413,17 +416,31 @@ static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed)
+ * want to receive/transmit regular data packets, then
+ * simply stop the scan session and exit PS mode.
+ */
+- wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN,
+- wcn->sw_scan_vif);
+- wcn->sw_scan_channel = 0;
++ if (wcn->sw_scan_channel)
++ wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel);
++ if (wcn->sw_scan_init) {
++ wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN,
++ wcn->sw_scan_vif);
++ }
+ } else if (wcn->sw_scan) {
+ /* A scan is ongoing, do not change the operating
+ * channel, but start a scan session on the channel.
+ */
+- wcn36xx_smd_init_scan(wcn, HAL_SYS_MODE_SCAN,
+- wcn->sw_scan_vif);
++ if (wcn->sw_scan_channel)
++ wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel);
++ if (!wcn->sw_scan_init) {
++ /* This can fail if we are unable to notify the
++ * operating channel.
++ */
++ ret = wcn36xx_smd_init_scan(wcn,
++ HAL_SYS_MODE_SCAN,
++ wcn->sw_scan_vif);
++ if (ret) {
++ mutex_unlock(&wcn->conf_mutex);
++ return -EIO;
++ }
++ }
+ wcn36xx_smd_start_scan(wcn, ch);
+- wcn->sw_scan_channel = ch;
+ } else {
+ wcn36xx_change_opchannel(wcn, ch);
+ }
+@@ -569,12 +586,14 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+ if (IEEE80211_KEY_FLAG_PAIRWISE & key_conf->flags) {
+ sta_priv->is_data_encrypted = true;
+ /* Reconfigure bss with encrypt_type */
+- if (NL80211_IFTYPE_STATION == vif->type)
++ if (NL80211_IFTYPE_STATION == vif->type) {
+ wcn36xx_smd_config_bss(wcn,
+ vif,
+ sta,
+ sta->addr,
+ true);
++ wcn36xx_smd_config_sta(wcn, vif, sta);
++ }
+
+ wcn36xx_smd_set_stakey(wcn,
+ vif_priv->encrypt_type,
+@@ -604,15 +623,6 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+ }
+ }
+ }
+- /* FIXME: Only enable bmps support when encryption is enabled.
+- * For any reasons, when connected to open/no-security BSS,
+- * the wcn36xx controller in bmps mode does not forward
+- * 'wake-up' beacons despite AP sends DTIM with station AID.
+- * It could be due to a firmware issue or to the way driver
+- * configure the station.
+- */
+- if (vif->type == NL80211_IFTYPE_STATION)
+- vif_priv->allow_bmps = true;
+ break;
+ case DISABLE_KEY:
+ if (!(IEEE80211_KEY_FLAG_PAIRWISE & key_conf->flags)) {
+@@ -676,6 +686,7 @@ static int wcn36xx_hw_scan(struct ieee80211_hw *hw,
+
+ mutex_unlock(&wcn->scan_lock);
+
++ wcn36xx_smd_update_channel_list(wcn, &hw_req->req);
+ return wcn36xx_smd_start_hw_scan(wcn, vif, &hw_req->req);
+ }
+
+@@ -717,7 +728,12 @@ static void wcn36xx_sw_scan_complete(struct ieee80211_hw *hw,
+ struct wcn36xx *wcn = hw->priv;
+
+ /* ensure that any scan session is finished */
+- wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN, wcn->sw_scan_vif);
++ if (wcn->sw_scan_channel)
++ wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel);
++ if (wcn->sw_scan_init) {
++ wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN,
++ wcn->sw_scan_vif);
++ }
+ wcn->sw_scan = false;
+ wcn->sw_scan_opchannel = 0;
+ }
+@@ -913,7 +929,6 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
+ vif->addr,
+ bss_conf->aid);
+ vif_priv->sta_assoc = false;
+- vif_priv->allow_bmps = false;
+ wcn36xx_smd_set_link_st(wcn,
+ bss_conf->bssid,
+ vif->addr,
+@@ -1123,6 +1138,13 @@ static int wcn36xx_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wow)
+ goto out;
+ ret = wcn36xx_smd_wlan_host_suspend_ind(wcn);
+ }
++
++ /* Disable IRQ, we don't want to handle any packet before mac80211 is
++ * resumed and ready to receive packets.
++ */
++ disable_irq(wcn->tx_irq);
++ disable_irq(wcn->rx_irq);
++
+ out:
+ mutex_unlock(&wcn->conf_mutex);
+ return ret;
+@@ -1145,6 +1167,10 @@ static int wcn36xx_resume(struct ieee80211_hw *hw)
+ wcn36xx_smd_ipv6_ns_offload(wcn, vif, false);
+ wcn36xx_smd_arp_offload(wcn, vif, false);
+ }
++
++ enable_irq(wcn->tx_irq);
++ enable_irq(wcn->rx_irq);
++
+ mutex_unlock(&wcn->conf_mutex);
+
+ return 0;
+@@ -1338,7 +1364,6 @@ static int wcn36xx_init_ieee80211(struct wcn36xx *wcn)
+ ieee80211_hw_set(wcn->hw, HAS_RATE_CONTROL);
+ ieee80211_hw_set(wcn->hw, SINGLE_SCAN_ON_ALL_BANDS);
+ ieee80211_hw_set(wcn->hw, REPORTS_TX_ACK_STATUS);
+- ieee80211_hw_set(wcn->hw, CONNECTION_MONITOR);
+
+ wcn->hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
+ BIT(NL80211_IFTYPE_AP) |
+@@ -1449,6 +1474,9 @@ static int wcn36xx_platform_get_resources(struct wcn36xx *wcn,
+ if (iris_node) {
+ if (of_device_is_compatible(iris_node, "qcom,wcn3620"))
+ wcn->rf_id = RF_IRIS_WCN3620;
++ if (of_device_is_compatible(iris_node, "qcom,wcn3660") ||
++ of_device_is_compatible(iris_node, "qcom,wcn3660b"))
++ wcn->rf_id = RF_IRIS_WCN3660;
+ if (of_device_is_compatible(iris_node, "qcom,wcn3680"))
+ wcn->rf_id = RF_IRIS_WCN3680;
+ of_node_put(iris_node);
+diff --git a/drivers/net/wireless/ath/wcn36xx/pmc.c b/drivers/net/wireless/ath/wcn36xx/pmc.c
+index 2d0780fefd477..2936aaf532738 100644
+--- a/drivers/net/wireless/ath/wcn36xx/pmc.c
++++ b/drivers/net/wireless/ath/wcn36xx/pmc.c
+@@ -23,10 +23,7 @@ int wcn36xx_pmc_enter_bmps_state(struct wcn36xx *wcn,
+ {
+ int ret = 0;
+ struct wcn36xx_vif *vif_priv = wcn36xx_vif_to_priv(vif);
+-
+- if (!vif_priv->allow_bmps)
+- return -ENOTSUPP;
+-
++ /* TODO: Make sure the TX chain clean */
+ ret = wcn36xx_smd_enter_bmps(wcn, vif);
+ if (!ret) {
+ wcn36xx_dbg(WCN36XX_DBG_PMC, "Entered BMPS\n");
+diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c
+index 57fa857b290b7..c056fae1d6418 100644
+--- a/drivers/net/wireless/ath/wcn36xx/smd.c
++++ b/drivers/net/wireless/ath/wcn36xx/smd.c
+@@ -16,6 +16,7 @@
+
+ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
++#include <linux/bitfield.h>
+ #include <linux/etherdevice.h>
+ #include <linux/firmware.h>
+ #include <linux/bitops.h>
+@@ -720,6 +721,7 @@ int wcn36xx_smd_init_scan(struct wcn36xx *wcn, enum wcn36xx_hal_sys_mode mode,
+ wcn36xx_err("hal_init_scan response failed err=%d\n", ret);
+ goto out;
+ }
++ wcn->sw_scan_init = true;
+ out:
+ mutex_unlock(&wcn->hal_mutex);
+ return ret;
+@@ -750,6 +752,7 @@ int wcn36xx_smd_start_scan(struct wcn36xx *wcn, u8 scan_channel)
+ wcn36xx_err("hal_start_scan response failed err=%d\n", ret);
+ goto out;
+ }
++ wcn->sw_scan_channel = scan_channel;
+ out:
+ mutex_unlock(&wcn->hal_mutex);
+ return ret;
+@@ -780,6 +783,7 @@ int wcn36xx_smd_end_scan(struct wcn36xx *wcn, u8 scan_channel)
+ wcn36xx_err("hal_end_scan response failed err=%d\n", ret);
+ goto out;
+ }
++ wcn->sw_scan_channel = 0;
+ out:
+ mutex_unlock(&wcn->hal_mutex);
+ return ret;
+@@ -821,6 +825,7 @@ int wcn36xx_smd_finish_scan(struct wcn36xx *wcn,
+ wcn36xx_err("hal_finish_scan response failed err=%d\n", ret);
+ goto out;
+ }
++ wcn->sw_scan_init = false;
+ out:
+ mutex_unlock(&wcn->hal_mutex);
+ return ret;
+@@ -927,6 +932,86 @@ out:
+ return ret;
+ }
+
++int wcn36xx_smd_update_channel_list(struct wcn36xx *wcn, struct cfg80211_scan_request *req)
++{
++ struct wcn36xx_hal_update_channel_list_req_msg *msg_body;
++ int ret, i;
++
++ msg_body = kzalloc(sizeof(*msg_body), GFP_KERNEL);
++ if (!msg_body)
++ return -ENOMEM;
++
++ INIT_HAL_MSG((*msg_body), WCN36XX_HAL_UPDATE_CHANNEL_LIST_REQ);
++
++ msg_body->num_channel = min_t(u8, req->n_channels, ARRAY_SIZE(msg_body->channels));
++ for (i = 0; i < msg_body->num_channel; i++) {
++ struct wcn36xx_hal_channel_param *param = &msg_body->channels[i];
++ u32 min_power = WCN36XX_HAL_DEFAULT_MIN_POWER;
++ u32 ant_gain = WCN36XX_HAL_DEFAULT_ANT_GAIN;
++
++ param->mhz = req->channels[i]->center_freq;
++ param->band_center_freq1 = req->channels[i]->center_freq;
++ param->band_center_freq2 = 0;
++
++ if (req->channels[i]->flags & IEEE80211_CHAN_NO_IR)
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_FLAG_PASSIVE;
++
++ if (req->channels[i]->flags & IEEE80211_CHAN_RADAR)
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_FLAG_DFS;
++
++ if (req->channels[i]->band == NL80211_BAND_5GHZ) {
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_FLAG_HT;
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_FLAG_VHT;
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_PHY_11A;
++ } else {
++ param->channel_info |= WCN36XX_HAL_CHAN_INFO_PHY_11BG;
++ }
++
++ if (min_power > req->channels[i]->max_power)
++ min_power = req->channels[i]->max_power;
++
++ if (req->channels[i]->max_antenna_gain)
++ ant_gain = req->channels[i]->max_antenna_gain;
++
++ u32p_replace_bits(&param->reg_info_1, min_power,
++ WCN36XX_HAL_CHAN_REG1_MIN_PWR_MASK);
++ u32p_replace_bits(&param->reg_info_1, req->channels[i]->max_power,
++ WCN36XX_HAL_CHAN_REG1_MAX_PWR_MASK);
++ u32p_replace_bits(&param->reg_info_1, req->channels[i]->max_reg_power,
++ WCN36XX_HAL_CHAN_REG1_REG_PWR_MASK);
++ u32p_replace_bits(&param->reg_info_1, 0,
++ WCN36XX_HAL_CHAN_REG1_CLASS_ID_MASK);
++ u32p_replace_bits(&param->reg_info_2, ant_gain,
++ WCN36XX_HAL_CHAN_REG2_ANT_GAIN_MASK);
++
++ wcn36xx_dbg(WCN36XX_DBG_HAL,
++ "%s: freq=%u, channel_info=%08x, reg_info1=%08x, reg_info2=%08x\n",
++ __func__, param->mhz, param->channel_info, param->reg_info_1,
++ param->reg_info_2);
++ }
++
++ mutex_lock(&wcn->hal_mutex);
++
++ PREPARE_HAL_BUF(wcn->hal_buf, (*msg_body));
++
++ ret = wcn36xx_smd_send_and_wait(wcn, msg_body->header.len);
++ if (ret) {
++ wcn36xx_err("Sending hal_update_channel_list failed\n");
++ goto out;
++ }
++
++ ret = wcn36xx_smd_rsp_status_check(wcn->hal_buf, wcn->hal_rsp_len);
++ if (ret) {
++ wcn36xx_err("hal_update_channel_list response failed err=%d\n", ret);
++ goto out;
++ }
++
++out:
++ kfree(msg_body);
++ mutex_unlock(&wcn->hal_mutex);
++ return ret;
++}
++
+ static int wcn36xx_smd_switch_channel_rsp(void *buf, size_t len)
+ {
+ struct wcn36xx_hal_switch_channel_rsp_msg *rsp;
+@@ -2594,7 +2679,7 @@ static int wcn36xx_smd_missed_beacon_ind(struct wcn36xx *wcn,
+ wcn36xx_dbg(WCN36XX_DBG_HAL, "beacon missed bss_index %d\n",
+ tmp->bss_index);
+ vif = wcn36xx_priv_to_vif(tmp);
+- ieee80211_connection_loss(vif);
++ ieee80211_beacon_loss(vif);
+ }
+ return 0;
+ }
+@@ -2609,7 +2694,7 @@ static int wcn36xx_smd_missed_beacon_ind(struct wcn36xx *wcn,
+ wcn36xx_dbg(WCN36XX_DBG_HAL, "beacon missed bss_index %d\n",
+ rsp->bss_index);
+ vif = wcn36xx_priv_to_vif(tmp);
+- ieee80211_connection_loss(vif);
++ ieee80211_beacon_loss(vif);
+ return 0;
+ }
+ }
+@@ -2623,30 +2708,52 @@ static int wcn36xx_smd_delete_sta_context_ind(struct wcn36xx *wcn,
+ size_t len)
+ {
+ struct wcn36xx_hal_delete_sta_context_ind_msg *rsp = buf;
+- struct wcn36xx_vif *tmp;
++ struct wcn36xx_vif *vif_priv;
++ struct ieee80211_vif *vif;
++ struct ieee80211_bss_conf *bss_conf;
+ struct ieee80211_sta *sta;
++ bool found = false;
+
+ if (len != sizeof(*rsp)) {
+ wcn36xx_warn("Corrupted delete sta indication\n");
+ return -EIO;
+ }
+
+- wcn36xx_dbg(WCN36XX_DBG_HAL, "delete station indication %pM index %d\n",
+- rsp->addr2, rsp->sta_id);
++ wcn36xx_dbg(WCN36XX_DBG_HAL,
++ "delete station indication %pM index %d reason %d\n",
++ rsp->addr2, rsp->sta_id, rsp->reason_code);
+
+- list_for_each_entry(tmp, &wcn->vif_list, list) {
++ list_for_each_entry(vif_priv, &wcn->vif_list, list) {
+ rcu_read_lock();
+- sta = ieee80211_find_sta(wcn36xx_priv_to_vif(tmp), rsp->addr2);
+- if (sta)
+- ieee80211_report_low_ack(sta, 0);
++ vif = wcn36xx_priv_to_vif(vif_priv);
++
++ if (vif->type == NL80211_IFTYPE_STATION) {
++ /* We could call ieee80211_find_sta too, but checking
++ * bss_conf is clearer.
++ */
++ bss_conf = &vif->bss_conf;
++ if (vif_priv->sta_assoc &&
++ !memcmp(bss_conf->bssid, rsp->addr2, ETH_ALEN)) {
++ found = true;
++ wcn36xx_dbg(WCN36XX_DBG_HAL,
++ "connection loss bss_index %d\n",
++ vif_priv->bss_index);
++ ieee80211_connection_loss(vif);
++ }
++ } else {
++ sta = ieee80211_find_sta(vif, rsp->addr2);
++ if (sta) {
++ found = true;
++ ieee80211_report_low_ack(sta, 0);
++ }
++ }
++
+ rcu_read_unlock();
+- if (sta)
++ if (found)
+ return 0;
+ }
+
+- wcn36xx_warn("STA with addr %pM and index %d not found\n",
+- rsp->addr2,
+- rsp->sta_id);
++ wcn36xx_warn("BSS or STA with addr %pM not found\n", rsp->addr2);
+ return -ENOENT;
+ }
+
+@@ -3060,6 +3167,7 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev,
+ case WCN36XX_HAL_GTK_OFFLOAD_RSP:
+ case WCN36XX_HAL_GTK_OFFLOAD_GETINFO_RSP:
+ case WCN36XX_HAL_HOST_RESUME_RSP:
++ case WCN36XX_HAL_UPDATE_CHANNEL_LIST_RSP:
+ memcpy(wcn->hal_buf, buf, len);
+ wcn->hal_rsp_len = len;
+ complete(&wcn->hal_rsp_compl);
+diff --git a/drivers/net/wireless/ath/wcn36xx/smd.h b/drivers/net/wireless/ath/wcn36xx/smd.h
+index d8bded03945d4..d3774568d885e 100644
+--- a/drivers/net/wireless/ath/wcn36xx/smd.h
++++ b/drivers/net/wireless/ath/wcn36xx/smd.h
+@@ -70,6 +70,7 @@ int wcn36xx_smd_update_scan_params(struct wcn36xx *wcn, u8 *channels, size_t cha
+ int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
+ struct cfg80211_scan_request *req);
+ int wcn36xx_smd_stop_hw_scan(struct wcn36xx *wcn);
++int wcn36xx_smd_update_channel_list(struct wcn36xx *wcn, struct cfg80211_scan_request *req);
+ int wcn36xx_smd_add_sta_self(struct wcn36xx *wcn, struct ieee80211_vif *vif);
+ int wcn36xx_smd_delete_sta_self(struct wcn36xx *wcn, u8 *addr);
+ int wcn36xx_smd_delete_sta(struct wcn36xx *wcn, u8 sta_index);
+diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.c b/drivers/net/wireless/ath/wcn36xx/txrx.c
+index cab196bb38cd4..f33e7228a1010 100644
+--- a/drivers/net/wireless/ath/wcn36xx/txrx.c
++++ b/drivers/net/wireless/ath/wcn36xx/txrx.c
+@@ -31,6 +31,13 @@ struct wcn36xx_rate {
+ enum rate_info_bw bw;
+ };
+
++/* Buffer descriptor rx_ch field is limited to 5-bit (4+1), a mapping is used
++ * for 11A Channels.
++ */
++static const u8 ab_rx_ch_map[] = { 36, 40, 44, 48, 52, 56, 60, 64, 100, 104,
++ 108, 112, 116, 120, 124, 128, 132, 136, 140,
++ 149, 153, 157, 161, 165, 144 };
++
+ static const struct wcn36xx_rate wcn36xx_rate_table[] = {
+ /* 11b rates */
+ { 10, 0, RX_ENC_LEGACY, 0, RATE_INFO_BW_20 },
+@@ -230,7 +237,6 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb)
+ const struct wcn36xx_rate *rate;
+ struct ieee80211_hdr *hdr;
+ struct wcn36xx_rx_bd *bd;
+- struct ieee80211_supported_band *sband;
+ u16 fc, sn;
+
+ /*
+@@ -252,8 +258,6 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb)
+ fc = __le16_to_cpu(hdr->frame_control);
+ sn = IEEE80211_SEQ_TO_SN(__le16_to_cpu(hdr->seq_ctrl));
+
+- status.freq = WCN36XX_CENTER_FREQ(wcn);
+- status.band = WCN36XX_BAND(wcn);
+ status.mactime = 10;
+ status.signal = -get_rssi0(bd);
+ status.antenna = 1;
+@@ -265,18 +269,36 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb)
+
+ wcn36xx_dbg(WCN36XX_DBG_RX, "status.flags=%x\n", status.flag);
+
++ if (bd->scan_learn) {
++ /* If packet originate from hardware scanning, extract the
++ * band/channel from bd descriptor.
++ */
++ u8 hwch = (bd->reserved0 << 4) + bd->rx_ch;
++
++ if (bd->rf_band != 1 && hwch <= sizeof(ab_rx_ch_map) && hwch >= 1) {
++ status.band = NL80211_BAND_5GHZ;
++ status.freq = ieee80211_channel_to_frequency(ab_rx_ch_map[hwch - 1],
++ status.band);
++ } else {
++ status.band = NL80211_BAND_2GHZ;
++ status.freq = ieee80211_channel_to_frequency(hwch, status.band);
++ }
++ } else {
++ status.band = WCN36XX_BAND(wcn);
++ status.freq = WCN36XX_CENTER_FREQ(wcn);
++ }
++
+ if (bd->rate_id < ARRAY_SIZE(wcn36xx_rate_table)) {
+ rate = &wcn36xx_rate_table[bd->rate_id];
+ status.encoding = rate->encoding;
+ status.enc_flags = rate->encoding_flags;
+ status.bw = rate->bw;
+ status.rate_idx = rate->mcs_or_legacy_index;
+- sband = wcn->hw->wiphy->bands[status.band];
+ status.nss = 1;
+
+ if (status.band == NL80211_BAND_5GHZ &&
+ status.encoding == RX_ENC_LEGACY &&
+- status.rate_idx >= sband->n_bitrates) {
++ status.rate_idx >= 4) {
+ /* no dsss rates in 5Ghz rates table */
+ status.rate_idx -= 4;
+ }
+@@ -321,8 +343,6 @@ static void wcn36xx_set_tx_pdu(struct wcn36xx_tx_bd *bd,
+ bd->pdu.mpdu_header_off;
+ bd->pdu.mpdu_len = len;
+ bd->pdu.tid = tid;
+- /* Use seq number generated by mac80211 */
+- bd->pdu.bd_ssn = WCN36XX_TXBD_SSN_FILL_HOST;
+ }
+
+ static inline struct wcn36xx_vif *get_vif_by_addr(struct wcn36xx *wcn,
+@@ -419,6 +439,9 @@ static void wcn36xx_set_tx_data(struct wcn36xx_tx_bd *bd,
+ tid = ieee80211_get_tid(hdr);
+ /* TID->QID is one-to-one mapping */
+ bd->queue_id = tid;
++ bd->pdu.bd_ssn = WCN36XX_TXBD_SSN_FILL_DPU_QOS;
++ } else {
++ bd->pdu.bd_ssn = WCN36XX_TXBD_SSN_FILL_DPU_NON_QOS;
+ }
+
+ if (info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT ||
+@@ -429,6 +452,9 @@ static void wcn36xx_set_tx_data(struct wcn36xx_tx_bd *bd,
+ if (ieee80211_is_any_nullfunc(hdr->frame_control)) {
+ /* Don't use a regular queue for null packet (no ampdu) */
+ bd->queue_id = WCN36XX_TX_U_WQ_ID;
++ bd->bd_rate = WCN36XX_BD_RATE_CTRL;
++ if (ieee80211_is_qos_nullfunc(hdr->frame_control))
++ bd->pdu.bd_ssn = WCN36XX_TXBD_SSN_FILL_HOST;
+ }
+
+ if (bcast) {
+@@ -488,6 +514,8 @@ static void wcn36xx_set_tx_mgmt(struct wcn36xx_tx_bd *bd,
+ bd->queue_id = WCN36XX_TX_U_WQ_ID;
+ *vif_priv = __vif_priv;
+
++ bd->pdu.bd_ssn = WCN36XX_TXBD_SSN_FILL_DPU_NON_QOS;
++
+ wcn36xx_set_tx_pdu(bd,
+ ieee80211_is_data_qos(hdr->frame_control) ?
+ sizeof(struct ieee80211_qos_hdr) :
+@@ -502,10 +530,11 @@ int wcn36xx_start_tx(struct wcn36xx *wcn,
+ struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+ struct wcn36xx_vif *vif_priv = NULL;
+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+- unsigned long flags;
+ bool is_low = ieee80211_is_data(hdr->frame_control);
+ bool bcast = is_broadcast_ether_addr(hdr->addr1) ||
+ is_multicast_ether_addr(hdr->addr1);
++ bool ack_ind = (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) &&
++ !(info->flags & IEEE80211_TX_CTL_NO_ACK);
+ struct wcn36xx_tx_bd bd;
+ int ret;
+
+@@ -521,30 +550,16 @@ int wcn36xx_start_tx(struct wcn36xx *wcn,
+
+ bd.dpu_rf = WCN36XX_BMU_WQ_TX;
+
+- if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
++ if (unlikely(ack_ind)) {
+ wcn36xx_dbg(WCN36XX_DBG_DXE, "TX_ACK status requested\n");
+
+- spin_lock_irqsave(&wcn->dxe_lock, flags);
+- if (wcn->tx_ack_skb) {
+- spin_unlock_irqrestore(&wcn->dxe_lock, flags);
+- wcn36xx_warn("tx_ack_skb already set\n");
+- return -EINVAL;
+- }
+-
+- wcn->tx_ack_skb = skb;
+- spin_unlock_irqrestore(&wcn->dxe_lock, flags);
+-
+ /* Only one at a time is supported by fw. Stop the TX queues
+ * until the ack status gets back.
+ */
+ ieee80211_stop_queues(wcn->hw);
+
+- /* TX watchdog if no TX irq or ack indication received */
+- mod_timer(&wcn->tx_ack_timer, jiffies + HZ / 10);
+-
+ /* Request ack indication from the firmware */
+- if (!(info->flags & IEEE80211_TX_CTL_NO_ACK))
+- bd.tx_comp = 1;
++ bd.tx_comp = 1;
+ }
+
+ /* Data frames served first*/
+@@ -558,14 +573,8 @@ int wcn36xx_start_tx(struct wcn36xx *wcn,
+ bd.tx_bd_sign = 0xbdbdbdbd;
+
+ ret = wcn36xx_dxe_tx_frame(wcn, vif_priv, &bd, skb, is_low);
+- if (ret && (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)) {
+- /* If the skb has not been transmitted,
+- * don't keep a reference to it.
+- */
+- spin_lock_irqsave(&wcn->dxe_lock, flags);
+- wcn->tx_ack_skb = NULL;
+- spin_unlock_irqrestore(&wcn->dxe_lock, flags);
+-
++ if (unlikely(ret && ack_ind)) {
++ /* If the skb has not been transmitted, resume TX queue */
+ ieee80211_wake_queues(wcn->hw);
+ }
+
+diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.h b/drivers/net/wireless/ath/wcn36xx/txrx.h
+index 032216e82b2be..b54311ffde9c5 100644
+--- a/drivers/net/wireless/ath/wcn36xx/txrx.h
++++ b/drivers/net/wireless/ath/wcn36xx/txrx.h
+@@ -110,7 +110,8 @@ struct wcn36xx_rx_bd {
+ /* 0x44 */
+ u32 exp_seq_num:12;
+ u32 cur_seq_num:12;
+- u32 fr_type_subtype:8;
++ u32 rf_band:2;
++ u32 fr_type_subtype:6;
+
+ /* 0x48 */
+ u32 msdu_size:16;
+diff --git a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
+index add6e527e8330..597f740f3c256 100644
+--- a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
++++ b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
+@@ -97,6 +97,7 @@ enum wcn36xx_ampdu_state {
+
+ #define RF_UNKNOWN 0x0000
+ #define RF_IRIS_WCN3620 0x3620
++#define RF_IRIS_WCN3660 0x3660
+ #define RF_IRIS_WCN3680 0x3680
+
+ static inline void buff_to_be(u32 *buf, size_t len)
+@@ -128,7 +129,6 @@ struct wcn36xx_vif {
+ enum wcn36xx_hal_bss_type bss_type;
+
+ /* Power management */
+- bool allow_bmps;
+ enum wcn36xx_power_state pw_state;
+
+ u8 bss_index;
+@@ -247,6 +247,7 @@ struct wcn36xx {
+ struct cfg80211_scan_request *scan_req;
+ bool sw_scan;
+ u8 sw_scan_opchannel;
++ bool sw_scan_init;
+ u8 sw_scan_channel;
+ struct ieee80211_vif *sw_scan_vif;
+ struct mutex scan_lock;
+diff --git a/drivers/net/wireless/broadcom/b43/phy_g.c b/drivers/net/wireless/broadcom/b43/phy_g.c
+index d5a1a5c582366..ac72ca39e409b 100644
+--- a/drivers/net/wireless/broadcom/b43/phy_g.c
++++ b/drivers/net/wireless/broadcom/b43/phy_g.c
+@@ -2297,7 +2297,7 @@ static u8 b43_gphy_aci_scan(struct b43_wldev *dev)
+ b43_phy_mask(dev, B43_PHY_G_CRS, 0x7FFF);
+ b43_set_all_gains(dev, 3, 8, 1);
+
+- start = (channel - 5 > 0) ? channel - 5 : 1;
++ start = (channel > 5) ? channel - 5 : 1;
+ end = (channel + 5 < 14) ? channel + 5 : 13;
+
+ for (i = start; i <= end; i++) {
+diff --git a/drivers/net/wireless/broadcom/b43legacy/radio.c b/drivers/net/wireless/broadcom/b43legacy/radio.c
+index 06891b4f837b9..fdf78c10a05c2 100644
+--- a/drivers/net/wireless/broadcom/b43legacy/radio.c
++++ b/drivers/net/wireless/broadcom/b43legacy/radio.c
+@@ -283,7 +283,7 @@ u8 b43legacy_radio_aci_scan(struct b43legacy_wldev *dev)
+ & 0x7FFF);
+ b43legacy_set_all_gains(dev, 3, 8, 1);
+
+- start = (channel - 5 > 0) ? channel - 5 : 1;
++ start = (channel > 5) ? channel - 5 : 1;
+ end = (channel + 5 < 14) ? channel + 5 : 13;
+
+ for (i = start; i <= end; i++) {
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
+index 6d5188b78f2de..0af452dca7664 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
+@@ -75,6 +75,16 @@ static const struct dmi_system_id dmi_platform_data[] = {
+ },
+ .driver_data = (void *)&acepc_t8_data,
+ },
++ {
++ /* Cyberbook T116 rugged tablet */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Default string"),
++ DMI_EXACT_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "20170531"),
++ },
++ /* The factory image nvram file is identical to the ACEPC T8 one */
++ .driver_data = (void *)&acepc_t8_data,
++ },
+ {
+ /* Match for the GPDwin which unfortunately uses somewhat
+ * generic dmi strings, which is why we test for 4 strings.
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
+index 0eb13e5df5177..dcbe55b56e437 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
+@@ -207,6 +207,8 @@ static int brcmf_init_nvram_parser(struct nvram_parser *nvp,
+ size = BRCMF_FW_MAX_NVRAM_SIZE;
+ else
+ size = data_len;
++ /* Add space for properties we may add */
++ size += strlen(BRCMF_FW_DEFAULT_BOARDREV) + 1;
+ /* Alloc for extra 0 byte + roundup by 4 + length field */
+ size += 1 + 3 + sizeof(u32);
+ nvp->nvram = kzalloc(size, GFP_KERNEL);
+@@ -693,7 +695,7 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
+ {
+ struct brcmf_fw_item *first = &req->items[0];
+ struct brcmf_fw *fwctx;
+- char *alt_path;
++ char *alt_path = NULL;
+ int ret;
+
+ brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(dev));
+@@ -712,7 +714,9 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
+ fwctx->done = fw_cb;
+
+ /* First try alternative board-specific path if any */
+- alt_path = brcm_alt_fw_path(first->path, fwctx->req->board_type);
++ if (fwctx->req->board_type)
++ alt_path = brcm_alt_fw_path(first->path,
++ fwctx->req->board_type);
+ if (alt_path) {
+ ret = request_firmware_nowait(THIS_MODULE, true, alt_path,
+ fwctx->dev, GFP_KERNEL, fwctx,
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+index 8b149996fc000..3ff4997e1c97a 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+@@ -12,6 +12,7 @@
+ #include <linux/interrupt.h>
+ #include <linux/bcma/bcma.h>
+ #include <linux/sched.h>
++#include <linux/io.h>
+ #include <asm/unaligned.h>
+
+ #include <soc.h>
+@@ -59,6 +60,13 @@ BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie");
+ BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie");
+ BRCMF_FW_DEF(4371, "brcmfmac4371-pcie");
+
++/* firmware config files */
++MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.txt");
++MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt");
++
++/* per-board firmware binaries */
++MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.bin");
++
+ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
+ BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
+ BRCMF_FW_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C),
+@@ -447,47 +455,6 @@ brcmf_pcie_write_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
+ }
+
+
+-static void
+-brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
+- void *srcaddr, u32 len)
+-{
+- void __iomem *address = devinfo->tcm + mem_offset;
+- __le32 *src32;
+- __le16 *src16;
+- u8 *src8;
+-
+- if (((ulong)address & 4) || ((ulong)srcaddr & 4) || (len & 4)) {
+- if (((ulong)address & 2) || ((ulong)srcaddr & 2) || (len & 2)) {
+- src8 = (u8 *)srcaddr;
+- while (len) {
+- iowrite8(*src8, address);
+- address++;
+- src8++;
+- len--;
+- }
+- } else {
+- len = len / 2;
+- src16 = (__le16 *)srcaddr;
+- while (len) {
+- iowrite16(le16_to_cpu(*src16), address);
+- address += 2;
+- src16++;
+- len--;
+- }
+- }
+- } else {
+- len = len / 4;
+- src32 = (__le32 *)srcaddr;
+- while (len) {
+- iowrite32(le32_to_cpu(*src32), address);
+- address += 4;
+- src32++;
+- len--;
+- }
+- }
+-}
+-
+-
+ static void
+ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
+ void *dstaddr, u32 len)
+@@ -1348,6 +1315,18 @@ static void brcmf_pcie_down(struct device *dev)
+ {
+ }
+
++static int brcmf_pcie_preinit(struct device *dev)
++{
++ struct brcmf_bus *bus_if = dev_get_drvdata(dev);
++ struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie;
++
++ brcmf_dbg(PCIE, "Enter\n");
++
++ brcmf_pcie_intr_enable(buspub->devinfo);
++ brcmf_pcie_hostready(buspub->devinfo);
++
++ return 0;
++}
+
+ static int brcmf_pcie_tx(struct device *dev, struct sk_buff *skb)
+ {
+@@ -1456,6 +1435,7 @@ static int brcmf_pcie_reset(struct device *dev)
+ }
+
+ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = {
++ .preinit = brcmf_pcie_preinit,
+ .txdata = brcmf_pcie_tx,
+ .stop = brcmf_pcie_down,
+ .txctl = brcmf_pcie_tx_ctlpkt,
+@@ -1563,8 +1543,8 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo,
+ return err;
+
+ brcmf_dbg(PCIE, "Download FW %s\n", devinfo->fw_name);
+- brcmf_pcie_copy_mem_todev(devinfo, devinfo->ci->rambase,
+- (void *)fw->data, fw->size);
++ memcpy_toio(devinfo->tcm + devinfo->ci->rambase,
++ (void *)fw->data, fw->size);
+
+ resetintr = get_unaligned_le32(fw->data);
+ release_firmware(fw);
+@@ -1578,7 +1558,7 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo,
+ brcmf_dbg(PCIE, "Download NVRAM %s\n", devinfo->nvram_name);
+ address = devinfo->ci->rambase + devinfo->ci->ramsize -
+ nvram_len;
+- brcmf_pcie_copy_mem_todev(devinfo, address, nvram, nvram_len);
++ memcpy_toio(devinfo->tcm + address, nvram, nvram_len);
+ brcmf_fw_nvram_free(nvram);
+ } else {
+ brcmf_dbg(PCIE, "No matching NVRAM file found %s\n",
+@@ -1777,6 +1757,8 @@ static void brcmf_pcie_setup(struct device *dev, int ret,
+ ret = brcmf_chip_get_raminfo(devinfo->ci);
+ if (ret) {
+ brcmf_err(bus, "Failed to get RAM info\n");
++ release_firmware(fw);
++ brcmf_fw_nvram_free(nvram);
+ goto fail;
+ }
+
+@@ -1826,9 +1808,6 @@ static void brcmf_pcie_setup(struct device *dev, int ret,
+
+ init_waitqueue_head(&devinfo->mbdata_resp_wait);
+
+- brcmf_pcie_intr_enable(devinfo);
+- brcmf_pcie_hostready(devinfo);
+-
+ ret = brcmf_attach(&devinfo->pdev->dev);
+ if (ret)
+ goto fail;
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+index 8effeb7a7269b..f7961b22e0518 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+@@ -557,7 +557,7 @@ enum brcmf_sdio_frmtype {
+ BRCMF_SDIO_FT_SUB,
+ };
+
+-#define SDIOD_DRVSTR_KEY(chip, pmu) (((chip) << 16) | (pmu))
++#define SDIOD_DRVSTR_KEY(chip, pmu) (((unsigned int)(chip) << 16) | (pmu))
+
+ /* SDIO Pad drive strength to select value mappings */
+ struct sdiod_drive_str {
+@@ -629,7 +629,6 @@ BRCMF_FW_CLM_DEF(43752, "brcmfmac43752-sdio");
+
+ /* firmware config files */
+ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-sdio.*.txt");
+-MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt");
+
+ /* per-board firmware binaries */
+ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-sdio.*.bin");
+diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/dvm/mac80211.c
+index 75e7665773c52..90fe4adca4926 100644
+--- a/drivers/net/wireless/intel/iwlwifi/dvm/mac80211.c
++++ b/drivers/net/wireless/intel/iwlwifi/dvm/mac80211.c
+@@ -304,7 +304,7 @@ static int iwlagn_mac_start(struct ieee80211_hw *hw)
+
+ priv->is_open = 1;
+ IWL_DEBUG_MAC80211(priv, "leave\n");
+- return 0;
++ return ret;
+ }
+
+ static void iwlagn_mac_stop(struct ieee80211_hw *hw)
+diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+index 1efac0b2a94d7..9e00d1d7e1468 100644
+--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
++++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+@@ -1,7 +1,7 @@
+ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+ /*
+ * Copyright (C) 2017 Intel Deutschland GmbH
+- * Copyright (C) 2019-2021 Intel Corporation
++ * Copyright (C) 2019-2022 Intel Corporation
+ */
+ #include <linux/uuid.h>
+ #include "iwl-drv.h"
+@@ -814,10 +814,11 @@ bool iwl_sar_geo_support(struct iwl_fw_runtime *fwrt)
+ * only one using version 36, so skip this version entirely.
+ */
+ return IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) >= 38 ||
+- IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 17 ||
+- (IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 29 &&
+- ((fwrt->trans->hw_rev & CSR_HW_REV_TYPE_MSK) ==
+- CSR_HW_REV_TYPE_7265D));
++ (IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 17 &&
++ fwrt->trans->hw_rev != CSR_HW_REV_TYPE_3160) ||
++ (IWL_UCODE_SERIAL(fwrt->fw->ucode_ver) == 29 &&
++ ((fwrt->trans->hw_rev & CSR_HW_REV_TYPE_MSK) ==
++ CSR_HW_REV_TYPE_7265D));
+ }
+ IWL_EXPORT_SYMBOL(iwl_sar_geo_support);
+
+diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+index 6dcafd0a3d4b1..b00cf92c8965a 100644
+--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
++++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+@@ -1532,8 +1532,6 @@ iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
+ return -EBUSY;
+
+ range->range_data_size = reg->dev_addr.size;
+- iwl_write_prph_no_grab(fwrt->trans, DBGI_SRAM_TARGET_ACCESS_CFG,
+- DBGI_SRAM_TARGET_ACCESS_CFG_RESET_ADDRESS_MSK);
+ for (i = 0; i < (le32_to_cpu(reg->dev_addr.size) / 4); i++) {
+ prph_data = iwl_read_prph(fwrt->trans, (i % 2) ?
+ DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
+diff --git a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
+index dde22bdc87039..069fcbc46d2ba 100644
+--- a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
++++ b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
+@@ -284,16 +284,19 @@ int iwl_pnvm_load(struct iwl_trans *trans,
+ /* First attempt to get the PNVM from BIOS */
+ package = iwl_uefi_get_pnvm(trans, &len);
+ if (!IS_ERR_OR_NULL(package)) {
+- data = kmemdup(package->data, len, GFP_KERNEL);
++ if (len >= sizeof(*package)) {
++ /* we need only the data */
++ len -= sizeof(*package);
++ data = kmemdup(package->data, len, GFP_KERNEL);
++ } else {
++ data = NULL;
++ }
+
+ /* free package regardless of whether kmemdup succeeded */
+ kfree(package);
+
+- if (data) {
+- /* we need only the data size */
+- len -= sizeof(*package);
++ if (data)
+ goto parse;
+- }
+ }
+
+ /* If it's not available, try from the filesystem */
+diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+index c875bf35533ce..009dd4be597b0 100644
+--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
++++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+@@ -86,6 +86,7 @@ static void *iwl_uefi_reduce_power_section(struct iwl_trans *trans,
+ if (len < tlv_len) {
+ IWL_ERR(trans, "invalid TLV len: %zd/%u\n",
+ len, tlv_len);
++ kfree(reduce_power_data);
+ reduce_power_data = ERR_PTR(-EINVAL);
+ goto out;
+ }
+@@ -105,6 +106,7 @@ static void *iwl_uefi_reduce_power_section(struct iwl_trans *trans,
+ IWL_DEBUG_FW(trans,
+ "Couldn't allocate (more) reduce_power_data\n");
+
++ kfree(reduce_power_data);
+ reduce_power_data = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+@@ -134,6 +136,10 @@ static void *iwl_uefi_reduce_power_section(struct iwl_trans *trans,
+ done:
+ if (!size) {
+ IWL_DEBUG_FW(trans, "Empty REDUCE_POWER, skipping.\n");
++ /* Better safe than sorry, but 'reduce_power_data' should
++ * always be NULL if !size.
++ */
++ kfree(reduce_power_data);
+ reduce_power_data = ERR_PTR(-ENOENT);
+ goto out;
+ }
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+index cf796403c45c0..c8dff76ac03c1 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+@@ -1,6 +1,6 @@
+ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+ /*
+- * Copyright (C) 2005-2014, 2018-2021 Intel Corporation
++ * Copyright (C) 2005-2014, 2018-2022 Intel Corporation
+ * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
+ * Copyright (C) 2016 Intel Deutschland GmbH
+ */
+@@ -104,9 +104,10 @@
+ /* GIO Chicken Bits (PCI Express bus link power management) */
+ #define CSR_GIO_CHICKEN_BITS (CSR_BASE+0x100)
+
+-/* Doorbell NMI (since Bz) */
++/* Doorbell - since Bz
++ * connected to UREG_DOORBELL_TO_ISR6 (lower 16 bits only)
++ */
+ #define CSR_DOORBELL_VECTOR (CSR_BASE + 0x130)
+-#define CSR_DOORBELL_VECTOR_NMI BIT(1)
+
+ /* host chicken bits */
+ #define CSR_HOST_CHICKEN (CSR_BASE + 0x204)
+@@ -318,6 +319,7 @@ enum {
+ #define CSR_HW_REV_TYPE_2x00 (0x0000100)
+ #define CSR_HW_REV_TYPE_105 (0x0000110)
+ #define CSR_HW_REV_TYPE_135 (0x0000120)
++#define CSR_HW_REV_TYPE_3160 (0x0000164)
+ #define CSR_HW_REV_TYPE_7265D (0x0000210)
+ #define CSR_HW_REV_TYPE_NONE (0x00001F0)
+ #define CSR_HW_REV_TYPE_QNJ (0x0000360)
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
+index 125479b5c0d61..fc4197bf2478e 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
+@@ -322,7 +322,7 @@ void iwl_dbg_tlv_del_timers(struct iwl_trans *trans)
+ struct iwl_dbg_tlv_timer_node *node, *tmp;
+
+ list_for_each_entry_safe(node, tmp, timer_list, list) {
+- del_timer(&node->timer);
++ del_timer_sync(&node->timer);
+ list_del(&node->list);
+ kfree(node);
+ }
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+index 77124b8b235ee..524b0ad873578 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+@@ -131,6 +131,9 @@ static void iwl_dealloc_ucode(struct iwl_drv *drv)
+
+ for (i = 0; i < IWL_UCODE_TYPE_MAX; i++)
+ iwl_free_fw_img(drv, drv->fw.img + i);
++
++ /* clear the data for the aborted load case */
++ memset(&drv->fw, 0, sizeof(drv->fw));
+ }
+
+ static int iwl_alloc_fw_desc(struct iwl_drv *drv, struct fw_desc *desc,
+@@ -1271,23 +1274,31 @@ _iwl_op_mode_start(struct iwl_drv *drv, struct iwlwifi_opmode_table *op)
+ const struct iwl_op_mode_ops *ops = op->ops;
+ struct dentry *dbgfs_dir = NULL;
+ struct iwl_op_mode *op_mode = NULL;
++ int retry, max_retry = !!iwlwifi_mod_params.fw_restart * IWL_MAX_INIT_RETRY;
++
++ for (retry = 0; retry <= max_retry; retry++) {
+
+ #ifdef CONFIG_IWLWIFI_DEBUGFS
+- drv->dbgfs_op_mode = debugfs_create_dir(op->name,
+- drv->dbgfs_drv);
+- dbgfs_dir = drv->dbgfs_op_mode;
++ drv->dbgfs_op_mode = debugfs_create_dir(op->name,
++ drv->dbgfs_drv);
++ dbgfs_dir = drv->dbgfs_op_mode;
+ #endif
+
+- op_mode = ops->start(drv->trans, drv->trans->cfg, &drv->fw, dbgfs_dir);
++ op_mode = ops->start(drv->trans, drv->trans->cfg,
++ &drv->fw, dbgfs_dir);
++
++ if (op_mode)
++ return op_mode;
++
++ IWL_ERR(drv, "retry init count %d\n", retry);
+
+ #ifdef CONFIG_IWLWIFI_DEBUGFS
+- if (!op_mode) {
+ debugfs_remove_recursive(drv->dbgfs_op_mode);
+ drv->dbgfs_op_mode = NULL;
+- }
+ #endif
++ }
+
+- return op_mode;
++ return NULL;
+ }
+
+ static void _iwl_op_mode_stop(struct iwl_drv *drv)
+@@ -1325,6 +1336,7 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
+ int i;
+ bool load_module = false;
+ bool usniffer_images = false;
++ bool failure = true;
+
+ fw->ucode_capa.max_probe_length = IWL_DEFAULT_MAX_PROBE_LENGTH;
+ fw->ucode_capa.standard_phy_calibration_size =
+@@ -1585,15 +1597,9 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
+ * else from proceeding if the module fails to load
+ * or hangs loading.
+ */
+- if (load_module) {
++ if (load_module)
+ request_module("%s", op->name);
+-#ifdef CONFIG_IWLWIFI_OPMODE_MODULAR
+- if (err)
+- IWL_ERR(drv,
+- "failed to load module %s (error %d), is dynamic loading enabled?\n",
+- op->name, err);
+-#endif
+- }
++ failure = false;
+ goto free;
+
+ try_again:
+@@ -1608,7 +1614,12 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
+ out_unbind:
+ complete(&drv->request_firmware_complete);
+ device_release_driver(drv->trans->dev);
++ /* drv has just been freed by the release */
++ failure = false;
+ free:
++ if (failure)
++ iwl_dealloc_ucode(drv);
++
+ if (pieces) {
+ for (i = 0; i < ARRAY_SIZE(pieces->img); i++)
+ kfree(pieces->img[i].sec);
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.h b/drivers/net/wireless/intel/iwlwifi/iwl-drv.h
+index b6442df0c6439..56f2fd3b94906 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.h
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.h
+@@ -90,4 +90,7 @@ void iwl_drv_stop(struct iwl_drv *drv);
+ #define IWL_EXPORT_SYMBOL(sym)
+ #endif
+
++/* max retry for init flow */
++#define IWL_MAX_INIT_RETRY 2
++
+ #endif /* __iwl_drv_h__ */
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+index 2517c4ae07ab3..5e76ab6c8ad0a 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+@@ -218,7 +218,7 @@ void iwl_force_nmi(struct iwl_trans *trans)
+ UREG_DOORBELL_TO_ISR6_NMI_BIT);
+ else
+ iwl_write32(trans, CSR_DOORBELL_VECTOR,
+- CSR_DOORBELL_VECTOR_NMI);
++ UREG_DOORBELL_TO_ISR6_NMI_BIT);
+ }
+ IWL_EXPORT_SYMBOL(iwl_force_nmi);
+
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+index 475f951d4b1ef..fc40cca096c26 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+@@ -541,8 +541,7 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = {
+ .has_he = true,
+ .he_cap_elem = {
+ .mac_cap_info[0] =
+- IEEE80211_HE_MAC_CAP0_HTC_HE |
+- IEEE80211_HE_MAC_CAP0_TWT_REQ,
++ IEEE80211_HE_MAC_CAP0_HTC_HE,
+ .mac_cap_info[1] =
+ IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+ IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+index d0a7d58336a9e..6c4f1c949541a 100644
+--- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
++++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+@@ -350,8 +350,6 @@
+ #define WFPM_GP2 0xA030B4
+
+ /* DBGI SRAM Register details */
+-#define DBGI_SRAM_TARGET_ACCESS_CFG 0x00A2E14C
+-#define DBGI_SRAM_TARGET_ACCESS_CFG_RESET_ADDRESS_MSK 0x10000
+ #define DBGI_SRAM_TARGET_ACCESS_RDATA_LSB 0x00A2E154
+ #define DBGI_SRAM_TARGET_ACCESS_RDATA_MSB 0x00A2E158
+
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+index 9f706fffb5922..00ca17f3b263c 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+@@ -2336,7 +2336,6 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
+ iwl_fw_dbg_collect_desc(&mvm->fwrt, &iwl_dump_desc_assert,
+ false, 0);
+ ret = 1;
+- mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
+ goto err;
+ }
+
+@@ -2385,6 +2384,7 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
+ }
+ }
+
++ /* after the successful handshake, we're out of D3 */
+ mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
+
+ /*
+@@ -2455,6 +2455,9 @@ out:
+ */
+ set_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status);
+
++ /* regardless of what happened, we're now out of D3 */
++ mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
++
+ return 1;
+ }
+
+@@ -2496,7 +2499,9 @@ static int iwl_mvm_d3_test_open(struct inode *inode, struct file *file)
+
+ /* start pseudo D3 */
+ rtnl_lock();
++ wiphy_lock(mvm->hw->wiphy);
+ err = __iwl_mvm_suspend(mvm->hw, mvm->hw->wiphy->wowlan_config, true);
++ wiphy_unlock(mvm->hw->wiphy);
+ rtnl_unlock();
+ if (err > 0)
+ err = -EINVAL;
+@@ -2552,7 +2557,9 @@ static int iwl_mvm_d3_test_release(struct inode *inode, struct file *file)
+ iwl_fw_dbg_read_d3_debug_data(&mvm->fwrt);
+
+ rtnl_lock();
++ wiphy_lock(mvm->hw->wiphy);
+ __iwl_mvm_resume(mvm, true);
++ wiphy_unlock(mvm->hw->wiphy);
+ rtnl_unlock();
+
+ iwl_mvm_resume_tcm(mvm);
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+index 5dc39fbb74d67..d398a06b26567 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+@@ -5,6 +5,7 @@
+ * Copyright (C) 2016-2017 Intel Deutschland GmbH
+ */
+ #include <linux/vmalloc.h>
++#include <linux/err.h>
+ #include <linux/ieee80211.h>
+ #include <linux/netdevice.h>
+
+@@ -2044,7 +2045,6 @@ void iwl_mvm_sta_add_debugfs(struct ieee80211_hw *hw,
+ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
+ {
+ struct dentry *bcast_dir __maybe_unused;
+- char buf[100];
+
+ spin_lock_init(&mvm->drv_stats_lock);
+
+@@ -2140,6 +2140,11 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
+ * Create a symlink with mac80211. It will be removed when mac80211
+ * exists (before the opmode exists which removes the target.)
+ */
+- snprintf(buf, 100, "../../%pd2", mvm->debugfs_dir->d_parent);
+- debugfs_create_symlink("iwlwifi", mvm->hw->wiphy->debugfsdir, buf);
++ if (!IS_ERR(mvm->debugfs_dir)) {
++ char buf[100];
++
++ snprintf(buf, 100, "../../%pd2", mvm->debugfs_dir->d_parent);
++ debugfs_create_symlink("iwlwifi", mvm->hw->wiphy->debugfsdir,
++ buf);
++ }
+ }
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+index 03e5bf5cb9094..bb5fff8174435 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+@@ -499,7 +499,7 @@ iwl_mvm_ftm_put_target(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+ rcu_read_lock();
+
+ sta = rcu_dereference(mvm->fw_id_to_mac_id[mvmvif->ap_sta_id]);
+- if (sta->mfp)
++ if (sta->mfp && (peer->ftm.trigger_based || peer->ftm.non_trigger_based))
+ FTM_PUT_FLAG(PMF);
+
+ rcu_read_unlock();
+@@ -1054,7 +1054,7 @@ static void iwl_mvm_ftm_rtt_smoothing(struct iwl_mvm *mvm,
+ overshoot = IWL_MVM_FTM_INITIATOR_SMOOTH_OVERSHOOT;
+ alpha = IWL_MVM_FTM_INITIATOR_SMOOTH_ALPHA;
+
+- rtt_avg = (alpha * rtt + (100 - alpha) * resp->rtt_avg) / 100;
++ rtt_avg = div_s64(alpha * rtt + (100 - alpha) * resp->rtt_avg, 100);
+
+ IWL_DEBUG_INFO(mvm,
+ "%pM: prev rtt_avg=%lld, new rtt_avg=%lld, rtt=%lld\n",
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+index 74404c96063bc..6d439ae7b50b1 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+@@ -1489,8 +1489,10 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
+ while (!sband && i < NUM_NL80211_BANDS)
+ sband = mvm->hw->wiphy->bands[i++];
+
+- if (WARN_ON_ONCE(!sband))
++ if (WARN_ON_ONCE(!sband)) {
++ ret = -ENODEV;
+ goto error;
++ }
+
+ chan = &sband->channels[0];
+
+@@ -1572,7 +1574,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
+ ret = iwl_mvm_sar_init(mvm);
+ if (ret == 0)
+ ret = iwl_mvm_sar_geo_init(mvm);
+- else if (ret < 0)
++ if (ret < 0)
+ goto error;
+
+ iwl_mvm_tas_init(mvm);
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+index 3a4585222d6d4..56c7a68a6491c 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+@@ -16,6 +16,7 @@
+ #include <net/ieee80211_radiotap.h>
+ #include <net/tcp.h>
+
++#include "iwl-drv.h"
+ #include "iwl-op-mode.h"
+ #include "iwl-io.h"
+ #include "mvm.h"
+@@ -294,7 +295,6 @@ static const u8 he_if_types_ext_capa_sta[] = {
+ [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+ [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
+ [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF,
+- [9] = WLAN_EXT_CAPA10_TWT_REQUESTER_SUPPORT,
+ };
+
+ static const struct wiphy_iftype_ext_capab he_iftypes_ext_capa[] = {
+@@ -1116,9 +1116,30 @@ static int iwl_mvm_mac_start(struct ieee80211_hw *hw)
+ {
+ struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+ int ret;
++ int retry, max_retry = 0;
+
+ mutex_lock(&mvm->mutex);
+- ret = __iwl_mvm_mac_start(mvm);
++
++ /* we are starting the mac not in error flow, and restart is enabled */
++ if (!test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status) &&
++ iwlwifi_mod_params.fw_restart) {
++ max_retry = IWL_MAX_INIT_RETRY;
++ /*
++ * This will prevent mac80211 recovery flows to trigger during
++ * init failures
++ */
++ set_bit(IWL_MVM_STATUS_STARTING, &mvm->status);
++ }
++
++ for (retry = 0; retry <= max_retry; retry++) {
++ ret = __iwl_mvm_mac_start(mvm);
++ if (!ret)
++ break;
++
++ IWL_ERR(mvm, "mac start retry %d\n", retry);
++ }
++ clear_bit(IWL_MVM_STATUS_STARTING, &mvm->status);
++
+ mutex_unlock(&mvm->mutex);
+
+ return ret;
+@@ -1665,6 +1686,7 @@ static void iwl_mvm_recalc_multicast(struct iwl_mvm *mvm)
+ struct iwl_mvm_mc_iter_data iter_data = {
+ .mvm = mvm,
+ };
++ int ret;
+
+ lockdep_assert_held(&mvm->mutex);
+
+@@ -1674,6 +1696,22 @@ static void iwl_mvm_recalc_multicast(struct iwl_mvm *mvm)
+ ieee80211_iterate_active_interfaces_atomic(
+ mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
+ iwl_mvm_mc_iface_iterator, &iter_data);
++
++ /*
++ * Send a (synchronous) ech command so that we wait for the
++ * multiple asynchronous MCAST_FILTER_CMD commands sent by
++ * the interface iterator. Otherwise, we might get here over
++ * and over again (by userspace just sending a lot of these)
++ * and the CPU can send them faster than the firmware can
++ * process them.
++ * Note that the CPU is still faster - but with this we'll
++ * actually send fewer commands overall because the CPU will
++ * not schedule the work in mac80211 as frequently if it's
++ * still running when rescheduled (possibly multiple times).
++ */
++ ret = iwl_mvm_send_cmd_pdu(mvm, ECHO_CMD, 0, 0, NULL);
++ if (ret)
++ IWL_ERR(mvm, "Failed to synchronize multicast groups update\n");
+ }
+
+ static u64 iwl_mvm_prepare_multicast(struct ieee80211_hw *hw,
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+index f877d86b038e3..46af8dd2dc930 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+@@ -1121,6 +1121,8 @@ struct iwl_mvm {
+ * @IWL_MVM_STATUS_FIRMWARE_RUNNING: firmware is running
+ * @IWL_MVM_STATUS_NEED_FLUSH_P2P: need to flush P2P bcast STA
+ * @IWL_MVM_STATUS_IN_D3: in D3 (or at least about to go into it)
++ * @IWL_MVM_STATUS_STARTING: starting mac,
++ * used to disable restart flow while in STARTING state
+ */
+ enum iwl_mvm_status {
+ IWL_MVM_STATUS_HW_RFKILL,
+@@ -1132,6 +1134,7 @@ enum iwl_mvm_status {
+ IWL_MVM_STATUS_FIRMWARE_RUNNING,
+ IWL_MVM_STATUS_NEED_FLUSH_P2P,
+ IWL_MVM_STATUS_IN_D3,
++ IWL_MVM_STATUS_STARTING,
+ };
+
+ /* Keep track of completed init configuration */
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+index 77ea2d0a30916..c77d98c888116 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+@@ -238,7 +238,8 @@ static void iwl_mvm_rx_thermal_dual_chain_req(struct iwl_mvm *mvm,
+ */
+ mvm->fw_static_smps_request =
+ req->event == cpu_to_le32(THERMAL_DUAL_CHAIN_REQ_DISABLE);
+- ieee80211_iterate_interfaces(mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
++ ieee80211_iterate_interfaces(mvm->hw,
++ IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER,
+ iwl_mvm_intf_dual_chain_req, NULL);
+ }
+
+@@ -687,6 +688,7 @@ static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm)
+ int ret;
+
+ rtnl_lock();
++ wiphy_lock(mvm->hw->wiphy);
+ mutex_lock(&mvm->mutex);
+
+ ret = iwl_run_init_mvm_ucode(mvm);
+@@ -702,6 +704,7 @@ static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm)
+ iwl_mvm_stop_device(mvm);
+
+ mutex_unlock(&mvm->mutex);
++ wiphy_unlock(mvm->hw->wiphy);
+ rtnl_unlock();
+
+ if (ret < 0)
+@@ -1424,6 +1427,9 @@ void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
+ */
+ if (!mvm->fw_restart && fw_error) {
+ iwl_fw_error_collect(&mvm->fwrt, false);
++ } else if (test_bit(IWL_MVM_STATUS_STARTING,
++ &mvm->status)) {
++ IWL_ERR(mvm, "Starting mac, retry will be triggered anyway\n");
+ } else if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+ struct iwl_mvm_reprobe *reprobe;
+
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
+index 035336a9e755e..6d82725cb87d0 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
+@@ -1,6 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+ /*
+- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
++ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
+ * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
+ * Copyright (C) 2017 Intel Deutschland GmbH
+ */
+@@ -295,18 +295,31 @@ void iwl_mvm_phy_ctxt_unref(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt)
+ * otherwise we might not be able to reuse this phy.
+ */
+ if (ctxt->ref == 0) {
+- struct ieee80211_channel *chan;
++ struct ieee80211_channel *chan = NULL;
+ struct cfg80211_chan_def chandef;
+- struct ieee80211_supported_band *sband = NULL;
+- enum nl80211_band band = NL80211_BAND_2GHZ;
++ struct ieee80211_supported_band *sband;
++ enum nl80211_band band;
++ int channel;
+
+- while (!sband && band < NUM_NL80211_BANDS)
+- sband = mvm->hw->wiphy->bands[band++];
++ for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) {
++ sband = mvm->hw->wiphy->bands[band];
+
+- if (WARN_ON(!sband))
+- return;
++ if (!sband)
++ continue;
++
++ for (channel = 0; channel < sband->n_channels; channel++)
++ if (!(sband->channels[channel].flags &
++ IEEE80211_CHAN_DISABLED)) {
++ chan = &sband->channels[channel];
++ break;
++ }
+
+- chan = &sband->channels[0];
++ if (chan)
++ break;
++ }
++
++ if (WARN_ON(!chan))
++ return;
+
+ cfg80211_chandef_create(&chandef, chan, NL80211_CHAN_NO_HT);
+ iwl_mvm_phy_ctxt_changed(mvm, ctxt, &chandef, 1, 1);
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+index c12f303cf652c..efccdd3f33773 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+@@ -121,12 +121,39 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
+ struct iwl_rx_mpdu_desc *desc = (void *)pkt->data;
+ unsigned int headlen, fraglen, pad_len = 0;
+ unsigned int hdrlen = ieee80211_hdrlen(hdr->frame_control);
++ u8 mic_crc_len = u8_get_bits(desc->mac_flags1,
++ IWL_RX_MPDU_MFLG1_MIC_CRC_LEN_MASK) << 1;
+
+ if (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
+ len -= 2;
+ pad_len = 2;
+ }
+
++ /*
++ * For non monitor interface strip the bytes the RADA might not have
++ * removed. As monitor interface cannot exist with other interfaces
++ * this removal is safe.
++ */
++ if (mic_crc_len && !ieee80211_hw_check(mvm->hw, RX_INCLUDES_FCS)) {
++ u32 pkt_flags = le32_to_cpu(pkt->len_n_flags);
++
++ /*
++ * If RADA was not enabled then decryption was not performed so
++ * the MIC cannot be removed.
++ */
++ if (!(pkt_flags & FH_RSCSR_RADA_EN)) {
++ if (WARN_ON(crypt_len > mic_crc_len))
++ return -EINVAL;
++
++ mic_crc_len -= crypt_len;
++ }
++
++ if (WARN_ON(mic_crc_len > len))
++ return -EINVAL;
++
++ len -= mic_crc_len;
++ }
++
+ /* If frame is small enough to fit in skb->head, pull it completely.
+ * If not, only pull ieee80211_hdr (including crypto if present, and
+ * an additional 8 bytes for SNAP/ethertype, see below) so that
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+index d78e436fa8b53..65e382756de68 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+@@ -1890,7 +1890,10 @@ static u8 iwl_mvm_scan_umac_chan_flags_v2(struct iwl_mvm *mvm,
+ IWL_SCAN_CHANNEL_FLAG_CACHE_ADD;
+
+ /* set fragmented ebs for fragmented scan on HB channels */
+- if (iwl_mvm_is_scan_fragmented(params->hb_type))
++ if ((!iwl_mvm_is_cdb_supported(mvm) &&
++ iwl_mvm_is_scan_fragmented(params->type)) ||
++ (iwl_mvm_is_cdb_supported(mvm) &&
++ iwl_mvm_is_scan_fragmented(params->hb_type)))
+ flags |= IWL_SCAN_CHANNEL_FLAG_EBS_FRAG;
+
+ return flags;
+@@ -1924,22 +1927,19 @@ static void iwl_mvm_scan_6ghz_passive_scan(struct iwl_mvm *mvm,
+ }
+
+ /*
+- * 6GHz passive scan is allowed while associated in a defined time
+- * interval following HW reset or resume flow
++ * 6GHz passive scan is allowed in a defined time interval following HW
++ * reset or resume flow, or while not associated and a large interval
++ * has passed since the last 6GHz passive scan.
+ */
+- if (vif->bss_conf.assoc &&
++ if ((vif->bss_conf.assoc ||
++ time_after(mvm->last_6ghz_passive_scan_jiffies +
++ (IWL_MVM_6GHZ_PASSIVE_SCAN_TIMEOUT * HZ), jiffies)) &&
+ (time_before(mvm->last_reset_or_resume_time_jiffies +
+ (IWL_MVM_6GHZ_PASSIVE_SCAN_ASSOC_TIMEOUT * HZ),
+ jiffies))) {
+- IWL_DEBUG_SCAN(mvm, "6GHz passive scan: associated\n");
+- return;
+- }
+-
+- /* No need for 6GHz passive scan if not enough time elapsed */
+- if (time_after(mvm->last_6ghz_passive_scan_jiffies +
+- (IWL_MVM_6GHZ_PASSIVE_SCAN_TIMEOUT * HZ), jiffies)) {
+- IWL_DEBUG_SCAN(mvm,
+- "6GHz passive scan: timeout did not expire\n");
++ IWL_DEBUG_SCAN(mvm, "6GHz passive scan: %s\n",
++ vif->bss_conf.assoc ? "associated" :
++ "timeout did not expire");
+ return;
+ }
+
+@@ -2490,7 +2490,7 @@ static int iwl_mvm_check_running_scans(struct iwl_mvm *mvm, int type)
+ return -EIO;
+ }
+
+-#define SCAN_TIMEOUT 20000
++#define SCAN_TIMEOUT 30000
+
+ void iwl_mvm_scan_timeout_wk(struct work_struct *work)
+ {
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+index e91f8e889df70..ab06dcda1462a 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+@@ -49,14 +49,13 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
+ struct iwl_mvm *mvm = container_of(wk, struct iwl_mvm, roc_done_wk);
+
+ /*
+- * Clear the ROC_RUNNING /ROC_AUX_RUNNING status bit.
++ * Clear the ROC_RUNNING status bit.
+ * This will cause the TX path to drop offchannel transmissions.
+ * That would also be done by mac80211, but it is racy, in particular
+ * in the case that the time event actually completed in the firmware
+ * (which is handled in iwl_mvm_te_handle_notif).
+ */
+ clear_bit(IWL_MVM_STATUS_ROC_RUNNING, &mvm->status);
+- clear_bit(IWL_MVM_STATUS_ROC_AUX_RUNNING, &mvm->status);
+
+ synchronize_net();
+
+@@ -82,9 +81,19 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
+ mvmvif = iwl_mvm_vif_from_mac80211(mvm->p2p_device_vif);
+ iwl_mvm_flush_sta(mvm, &mvmvif->bcast_sta, true);
+ }
+- } else {
++ }
++
++ /*
++ * Clear the ROC_AUX_RUNNING status bit.
++ * This will cause the TX path to drop offchannel transmissions.
++ * That would also be done by mac80211, but it is racy, in particular
++ * in the case that the time event actually completed in the firmware
++ * (which is handled in iwl_mvm_te_handle_notif).
++ */
++ if (test_and_clear_bit(IWL_MVM_STATUS_ROC_AUX_RUNNING, &mvm->status)) {
+ /* do the same in case of hot spot 2.0 */
+ iwl_mvm_flush_sta(mvm, &mvm->aux_sta, true);
++
+ /* In newer version of this command an aux station is added only
+ * in cases of dedicated tx queue and need to be removed in end
+ * of use */
+@@ -687,11 +696,14 @@ static bool __iwl_mvm_remove_time_event(struct iwl_mvm *mvm,
+ iwl_mvm_te_clear_data(mvm, te_data);
+ spin_unlock_bh(&mvm->time_event_lock);
+
+- /* When session protection is supported, the te_data->id field
++ /* When session protection is used, the te_data->id field
+ * is reused to save session protection's configuration.
++ * For AUX ROC, HOT_SPOT_CMD is used and the te_data->id field is set
++ * to HOT_SPOT_CMD.
+ */
+ if (fw_has_capa(&mvm->fw->ucode_capa,
+- IWL_UCODE_TLV_CAPA_SESSION_PROT_CMD)) {
++ IWL_UCODE_TLV_CAPA_SESSION_PROT_CMD) &&
++ id != HOT_SPOT_CMD) {
+ if (mvmvif && id < SESSION_PROTECT_CONF_MAX_ID) {
+ /* Session protection is still ongoing. Cancel it */
+ iwl_mvm_cancel_session_protection(mvm, mvmvif, id);
+@@ -1027,7 +1039,7 @@ void iwl_mvm_stop_roc(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+ iwl_mvm_p2p_roc_finished(mvm);
+ } else {
+ iwl_mvm_remove_aux_roc_te(mvm, mvmvif,
+- &mvmvif->time_event_data);
++ &mvmvif->hs_time_event_data);
+ iwl_mvm_roc_finished(mvm);
+ }
+
+@@ -1158,15 +1170,10 @@ void iwl_mvm_schedule_session_protection(struct iwl_mvm *mvm,
+ cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
+ mvmvif->color)),
+ .action = cpu_to_le32(FW_CTXT_ACTION_ADD),
++ .conf_id = cpu_to_le32(SESSION_PROTECT_CONF_ASSOC),
+ .duration_tu = cpu_to_le32(MSEC_TO_TU(duration)),
+ };
+
+- /* The time_event_data.id field is reused to save session
+- * protection's configuration.
+- */
+- mvmvif->time_event_data.id = SESSION_PROTECT_CONF_ASSOC;
+- cmd.conf_id = cpu_to_le32(mvmvif->time_event_data.id);
+-
+ lockdep_assert_held(&mvm->mutex);
+
+ spin_lock_bh(&mvm->time_event_lock);
+@@ -1180,6 +1187,11 @@ void iwl_mvm_schedule_session_protection(struct iwl_mvm *mvm,
+ }
+
+ iwl_mvm_te_clear_data(mvm, te_data);
++ /*
++ * The time_event_data.id field is reused to save session
++ * protection's configuration.
++ */
++ te_data->id = le32_to_cpu(cmd.conf_id);
+ te_data->duration = le32_to_cpu(cmd.duration_tu);
+ te_data->vif = vif;
+ spin_unlock_bh(&mvm->time_event_lock);
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+index 0a13c2bda2eed..b5368cb57ca8c 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+@@ -268,7 +268,6 @@ static u32 iwl_mvm_get_tx_rate(struct iwl_mvm *mvm,
+ int rate_idx = -1;
+ u8 rate_plcp;
+ u32 rate_flags = 0;
+- struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+
+ /* info->control is only relevant for non HW rate control */
+ if (!ieee80211_hw_check(mvm->hw, HAS_RATE_CONTROL)) {
+@@ -278,7 +277,8 @@ static u32 iwl_mvm_get_tx_rate(struct iwl_mvm *mvm,
+ "Got a HT rate (flags:0x%x/mcs:%d/fc:0x%x/state:%d) for a non data frame\n",
+ info->control.rates[0].flags,
+ info->control.rates[0].idx,
+- le16_to_cpu(fc), mvmsta->sta_state);
++ le16_to_cpu(fc),
++ sta ? iwl_mvm_sta_from_mac80211(sta)->sta_state : -1);
+
+ rate_idx = info->control.rates[0].idx;
+ }
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+index 4a3d2971a98b7..ec8a223f90e85 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+@@ -405,6 +405,9 @@ bool iwl_mvm_rx_diversity_allowed(struct iwl_mvm *mvm,
+
+ lockdep_assert_held(&mvm->mutex);
+
++ if (iwlmvm_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
++ return false;
++
+ if (num_of_ant(iwl_mvm_get_valid_rx_ant(mvm)) == 1)
+ return false;
+
+diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+index e3996ff99bad5..3b974388d834d 100644
+--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
++++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+@@ -931,9 +931,9 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
+ IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
+ iwl_qu_b0_hr1_b0, iwl_ax101_name),
+ _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+- IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
++ IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
+ IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+- IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
++ IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
+ iwl_qu_b0_hr_b0, iwl_ax203_name),
+
+ /* Qu C step */
+@@ -945,7 +945,7 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
+ _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+ IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
+ IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+- IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB,
++ IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB,
+ iwl_qu_c0_hr_b0, iwl_ax203_name),
+
+ /* QuZ */
+diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+index 8e45eb38304b2..fea89330f692c 100644
+--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
++++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+@@ -2261,7 +2261,12 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
+ }
+ }
+
+- if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP) {
++ /*
++ * In some rare cases when the HW is in a bad state, we may
++ * get this interrupt too early, when prph_info is still NULL.
++ * So make sure that it's not NULL to prevent crashing.
++ */
++ if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) {
+ u32 sleep_notif =
+ le32_to_cpu(trans_pcie->prph_info->sleep_notif);
+ if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND ||
+diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+index bf0c32a74ca47..a9c19be29e92e 100644
+--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
++++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+@@ -408,8 +408,7 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
+ /* This may fail if AMT took ownership of the device */
+ if (iwl_pcie_prepare_card_hw(trans)) {
+ IWL_WARN(trans, "Exit HW not ready\n");
+- ret = -EIO;
+- goto out;
++ return -EIO;
+ }
+
+ iwl_enable_rfkill_int(trans);
+diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+index f252680f18e88..02da9cc8646cf 100644
+--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
++++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+@@ -1273,8 +1273,7 @@ static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
+ /* This may fail if AMT took ownership of the device */
+ if (iwl_pcie_prepare_card_hw(trans)) {
+ IWL_WARN(trans, "Exit HW not ready\n");
+- ret = -EIO;
+- goto out;
++ return -EIO;
+ }
+
+ iwl_enable_rfkill_int(trans);
+diff --git a/drivers/net/wireless/intel/iwlwifi/queue/tx.c b/drivers/net/wireless/intel/iwlwifi/queue/tx.c
+index 451b060693501..0f3526b0c5b00 100644
+--- a/drivers/net/wireless/intel/iwlwifi/queue/tx.c
++++ b/drivers/net/wireless/intel/iwlwifi/queue/tx.c
+@@ -1072,6 +1072,7 @@ int iwl_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num,
+ return 0;
+ err_free_tfds:
+ dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr);
++ txq->tfds = NULL;
+ error:
+ if (txq->entries && cmd_queue)
+ for (i = 0; i < slots_num; i++)
+diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
+index 0adae76eb8df1..c3189e2c7c93a 100644
+--- a/drivers/net/wireless/mac80211_hwsim.c
++++ b/drivers/net/wireless/mac80211_hwsim.c
+@@ -2336,9 +2336,21 @@ static void hw_scan_work(struct work_struct *work)
+ if (req->ie_len)
+ skb_put_data(probe, req->ie, req->ie_len);
+
++ rcu_read_lock();
++ if (!ieee80211_tx_prepare_skb(hwsim->hw,
++ hwsim->hw_scan_vif,
++ probe,
++ hwsim->tmp_chan->band,
++ NULL)) {
++ rcu_read_unlock();
++ kfree_skb(probe);
++ continue;
++ }
++
+ local_bh_disable();
+ mac80211_hwsim_tx_frame(hwsim->hw, probe,
+ hwsim->tmp_chan);
++ rcu_read_unlock();
+ local_bh_enable();
+ }
+ }
+@@ -3641,6 +3653,10 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
+ }
+ txi->flags |= IEEE80211_TX_STAT_ACK;
+ }
++
++ if (hwsim_flags & HWSIM_TX_CTL_NO_ACK)
++ txi->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
++
+ ieee80211_tx_status_irqsafe(data2->hw, skb);
+ return 0;
+ out:
+diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c
+index 20436a289d5cd..5d6dc1dd050d4 100644
+--- a/drivers/net/wireless/marvell/libertas/if_usb.c
++++ b/drivers/net/wireless/marvell/libertas/if_usb.c
+@@ -292,6 +292,7 @@ err_add_card:
+ if_usb_reset_device(cardp);
+ dealloc:
+ if_usb_free(cardp);
++ kfree(cardp);
+
+ error:
+ return r;
+@@ -316,6 +317,7 @@ static void if_usb_disconnect(struct usb_interface *intf)
+
+ /* Unlink and free urb */
+ if_usb_free(cardp);
++ kfree(cardp);
+
+ usb_set_intfdata(intf, NULL);
+ usb_put_dev(interface_to_usbdev(intf));
+diff --git a/drivers/net/wireless/marvell/libertas_tf/if_usb.c b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+index fe0a69e804d8c..75b5319d033f3 100644
+--- a/drivers/net/wireless/marvell/libertas_tf/if_usb.c
++++ b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+@@ -230,6 +230,7 @@ static int if_usb_probe(struct usb_interface *intf,
+
+ dealloc:
+ if_usb_free(cardp);
++ kfree(cardp);
+ error:
+ lbtf_deb_leave(LBTF_DEB_MAIN);
+ return -ENOMEM;
+@@ -254,6 +255,7 @@ static void if_usb_disconnect(struct usb_interface *intf)
+
+ /* Unlink and free urb */
+ if_usb_free(cardp);
++ kfree(cardp);
+
+ usb_set_intfdata(intf, NULL);
+ usb_put_dev(interface_to_usbdev(intf));
+diff --git a/drivers/net/wireless/marvell/mwifiex/11n.c b/drivers/net/wireless/marvell/mwifiex/11n.c
+index 6696bce561786..cf08a4af84d6d 100644
+--- a/drivers/net/wireless/marvell/mwifiex/11n.c
++++ b/drivers/net/wireless/marvell/mwifiex/11n.c
+@@ -657,14 +657,15 @@ int mwifiex_send_delba(struct mwifiex_private *priv, int tid, u8 *peer_mac,
+ uint16_t del_ba_param_set;
+
+ memset(&delba, 0, sizeof(delba));
+- delba.del_ba_param_set = cpu_to_le16(tid << DELBA_TID_POS);
+
+- del_ba_param_set = le16_to_cpu(delba.del_ba_param_set);
++ del_ba_param_set = tid << DELBA_TID_POS;
++
+ if (initiator)
+ del_ba_param_set |= IEEE80211_DELBA_PARAM_INITIATOR_MASK;
+ else
+ del_ba_param_set &= ~IEEE80211_DELBA_PARAM_INITIATOR_MASK;
+
++ delba.del_ba_param_set = cpu_to_le16(del_ba_param_set);
+ memcpy(&delba.peer_mac_addr, peer_mac, ETH_ALEN);
+
+ /* We don't wait for the response of this command */
+diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+index 0961f4a5e415c..97f0f39364d67 100644
+--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
++++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+@@ -908,16 +908,20 @@ mwifiex_init_new_priv_params(struct mwifiex_private *priv,
+ switch (type) {
+ case NL80211_IFTYPE_STATION:
+ case NL80211_IFTYPE_ADHOC:
+- priv->bss_role = MWIFIEX_BSS_ROLE_STA;
++ priv->bss_role = MWIFIEX_BSS_ROLE_STA;
++ priv->bss_type = MWIFIEX_BSS_TYPE_STA;
+ break;
+ case NL80211_IFTYPE_P2P_CLIENT:
+- priv->bss_role = MWIFIEX_BSS_ROLE_STA;
++ priv->bss_role = MWIFIEX_BSS_ROLE_STA;
++ priv->bss_type = MWIFIEX_BSS_TYPE_P2P;
+ break;
+ case NL80211_IFTYPE_P2P_GO:
+- priv->bss_role = MWIFIEX_BSS_ROLE_UAP;
++ priv->bss_role = MWIFIEX_BSS_ROLE_UAP;
++ priv->bss_type = MWIFIEX_BSS_TYPE_P2P;
+ break;
+ case NL80211_IFTYPE_AP:
+ priv->bss_role = MWIFIEX_BSS_ROLE_UAP;
++ priv->bss_type = MWIFIEX_BSS_TYPE_UAP;
+ break;
+ default:
+ mwifiex_dbg(adapter, ERROR,
+@@ -1229,29 +1233,15 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
+ break;
+ case NL80211_IFTYPE_P2P_CLIENT:
+ case NL80211_IFTYPE_P2P_GO:
++ if (mwifiex_cfg80211_deinit_p2p(priv))
++ return -EFAULT;
++
+ switch (type) {
+- case NL80211_IFTYPE_STATION:
+- if (mwifiex_cfg80211_deinit_p2p(priv))
+- return -EFAULT;
+- priv->adapter->curr_iface_comb.p2p_intf--;
+- priv->adapter->curr_iface_comb.sta_intf++;
+- dev->ieee80211_ptr->iftype = type;
+- if (mwifiex_deinit_priv_params(priv))
+- return -1;
+- if (mwifiex_init_new_priv_params(priv, dev, type))
+- return -1;
+- if (mwifiex_sta_init_cmd(priv, false, false))
+- return -1;
+- break;
+ case NL80211_IFTYPE_ADHOC:
+- if (mwifiex_cfg80211_deinit_p2p(priv))
+- return -EFAULT;
++ case NL80211_IFTYPE_STATION:
+ return mwifiex_change_vif_to_sta_adhoc(dev, curr_iftype,
+ type, params);
+- break;
+ case NL80211_IFTYPE_AP:
+- if (mwifiex_cfg80211_deinit_p2p(priv))
+- return -EFAULT;
+ return mwifiex_change_vif_to_ap(dev, curr_iftype, type,
+ params);
+ case NL80211_IFTYPE_UNSPECIFIED:
+diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
+index c6ccce426b496..c3f5583ea70df 100644
+--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
++++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
+@@ -17,6 +17,7 @@
+ * this warranty disclaimer.
+ */
+
++#include <linux/iopoll.h>
+ #include <linux/firmware.h>
+
+ #include "decl.h"
+@@ -647,11 +648,15 @@ static void mwifiex_delay_for_sleep_cookie(struct mwifiex_adapter *adapter,
+ "max count reached while accessing sleep cookie\n");
+ }
+
++#define N_WAKEUP_TRIES_SHORT_INTERVAL 15
++#define N_WAKEUP_TRIES_LONG_INTERVAL 35
++
+ /* This function wakes up the card by reading fw_status register. */
+ static int mwifiex_pm_wakeup_card(struct mwifiex_adapter *adapter)
+ {
+ struct pcie_service_card *card = adapter->card;
+ const struct mwifiex_pcie_card_reg *reg = card->pcie.reg;
++ int retval;
+
+ mwifiex_dbg(adapter, EVENT,
+ "event: Wakeup device...\n");
+@@ -659,11 +664,24 @@ static int mwifiex_pm_wakeup_card(struct mwifiex_adapter *adapter)
+ if (reg->sleep_cookie)
+ mwifiex_pcie_dev_wakeup_delay(adapter);
+
+- /* Accessing fw_status register will wakeup device */
+- if (mwifiex_write_reg(adapter, reg->fw_status, FIRMWARE_READY_PCIE)) {
+- mwifiex_dbg(adapter, ERROR,
+- "Writing fw_status register failed\n");
+- return -1;
++ /* The 88W8897 PCIe+USB firmware (latest version 15.68.19.p21) sometimes
++ * appears to ignore or miss our wakeup request, so we continue trying
++ * until we receive an interrupt from the card.
++ */
++ if (read_poll_timeout(mwifiex_write_reg, retval,
++ READ_ONCE(adapter->int_status) != 0,
++ 500, 500 * N_WAKEUP_TRIES_SHORT_INTERVAL,
++ false,
++ adapter, reg->fw_status, FIRMWARE_READY_PCIE)) {
++ if (read_poll_timeout(mwifiex_write_reg, retval,
++ READ_ONCE(adapter->int_status) != 0,
++ 10000, 10000 * N_WAKEUP_TRIES_LONG_INTERVAL,
++ false,
++ adapter, reg->fw_status, FIRMWARE_READY_PCIE)) {
++ mwifiex_dbg(adapter, ERROR,
++ "Firmware didn't wake up\n");
++ return -EIO;
++ }
+ }
+
+ if (reg->sleep_cookie) {
+@@ -1490,6 +1508,14 @@ mwifiex_pcie_send_data(struct mwifiex_adapter *adapter, struct sk_buff *skb,
+ ret = -1;
+ goto done_unmap;
+ }
++
++ /* The firmware (latest version 15.68.19.p21) of the 88W8897 PCIe+USB card
++ * seems to crash randomly after setting the TX ring write pointer when
++ * ASPM powersaving is enabled. A workaround seems to be keeping the bus
++ * busy by reading a random register afterwards.
++ */
++ mwifiex_read_reg(adapter, PCI_VENDOR_ID, &rx_val);
++
+ if ((mwifiex_pcie_txbd_not_full(card)) &&
+ tx_param->next_pkt_len) {
+ /* have more packets and TxBD still can hold more */
+diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
+index 68c63268e2e6b..2b2e6e0166e14 100644
+--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
++++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
+@@ -365,10 +365,12 @@ static void mwifiex_process_uap_tx_pause(struct mwifiex_private *priv,
+ sta_ptr = mwifiex_get_sta_entry(priv, tp->peermac);
+ if (sta_ptr && sta_ptr->tx_pause != tp->tx_pause) {
+ sta_ptr->tx_pause = tp->tx_pause;
++ spin_unlock_bh(&priv->sta_list_spinlock);
+ mwifiex_update_ralist_tx_pause(priv, tp->peermac,
+ tp->tx_pause);
++ } else {
++ spin_unlock_bh(&priv->sta_list_spinlock);
+ }
+- spin_unlock_bh(&priv->sta_list_spinlock);
+ }
+ }
+
+@@ -400,11 +402,13 @@ static void mwifiex_process_sta_tx_pause(struct mwifiex_private *priv,
+ sta_ptr = mwifiex_get_sta_entry(priv, tp->peermac);
+ if (sta_ptr && sta_ptr->tx_pause != tp->tx_pause) {
+ sta_ptr->tx_pause = tp->tx_pause;
++ spin_unlock_bh(&priv->sta_list_spinlock);
+ mwifiex_update_ralist_tx_pause(priv,
+ tp->peermac,
+ tp->tx_pause);
++ } else {
++ spin_unlock_bh(&priv->sta_list_spinlock);
+ }
+- spin_unlock_bh(&priv->sta_list_spinlock);
+ }
+ }
+ }
+diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
+index 426e39d4ccf0f..8f01fcbe93961 100644
+--- a/drivers/net/wireless/marvell/mwifiex/usb.c
++++ b/drivers/net/wireless/marvell/mwifiex/usb.c
+@@ -130,7 +130,8 @@ static int mwifiex_usb_recv(struct mwifiex_adapter *adapter,
+ default:
+ mwifiex_dbg(adapter, ERROR,
+ "unknown recv_type %#x\n", recv_type);
+- return -1;
++ ret = -1;
++ goto exit_restore_skb;
+ }
+ break;
+ case MWIFIEX_USB_EP_DATA:
+@@ -505,6 +506,22 @@ static int mwifiex_usb_probe(struct usb_interface *intf,
+ }
+ }
+
++ switch (card->usb_boot_state) {
++ case USB8XXX_FW_DNLD:
++ /* Reject broken descriptors. */
++ if (!card->rx_cmd_ep || !card->tx_cmd_ep)
++ return -ENODEV;
++ if (card->bulk_out_maxpktsize == 0)
++ return -ENODEV;
++ break;
++ case USB8XXX_FW_READY:
++ /* Assume the driver can handle missing endpoints for now. */
++ break;
++ default:
++ WARN_ON(1);
++ return -ENODEV;
++ }
++
+ usb_set_intfdata(intf, card);
+
+ ret = mwifiex_add_card(card, &card->fw_done, &usb_ops,
+diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c
+index 3bf6571f41490..529e325498cdb 100644
+--- a/drivers/net/wireless/marvell/mwl8k.c
++++ b/drivers/net/wireless/marvell/mwl8k.c
+@@ -5800,8 +5800,8 @@ static void mwl8k_fw_state_machine(const struct firmware *fw, void *context)
+ fail:
+ priv->fw_state = FW_STATE_ERROR;
+ complete(&priv->firmware_loading_complete);
+- device_release_driver(&priv->pdev->dev);
+ mwl8k_release_firmware(priv);
++ device_release_driver(&priv->pdev->dev);
+ }
+
+ #define MAX_RESTART_ATTEMPTS 1
+diff --git a/drivers/net/wireless/mediatek/mt76/debugfs.c b/drivers/net/wireless/mediatek/mt76/debugfs.c
+index fa48cc3a7a8f7..ad97308c78534 100644
+--- a/drivers/net/wireless/mediatek/mt76/debugfs.c
++++ b/drivers/net/wireless/mediatek/mt76/debugfs.c
+@@ -116,8 +116,11 @@ static int mt76_read_rate_txpower(struct seq_file *s, void *data)
+ return 0;
+ }
+
+-struct dentry *mt76_register_debugfs(struct mt76_dev *dev)
++struct dentry *
++mt76_register_debugfs_fops(struct mt76_dev *dev,
++ const struct file_operations *ops)
+ {
++ const struct file_operations *fops = ops ? ops : &fops_regval;
+ struct dentry *dir;
+
+ dir = debugfs_create_dir("mt76", dev->hw->wiphy->debugfsdir);
+@@ -126,8 +129,7 @@ struct dentry *mt76_register_debugfs(struct mt76_dev *dev)
+
+ debugfs_create_u8("led_pin", 0600, dir, &dev->led_pin);
+ debugfs_create_u32("regidx", 0600, dir, &dev->debugfs_reg);
+- debugfs_create_file_unsafe("regval", 0600, dir, dev,
+- &fops_regval);
++ debugfs_create_file_unsafe("regval", 0600, dir, dev, fops);
+ debugfs_create_file_unsafe("napi_threaded", 0600, dir, dev,
+ &fops_napi_threaded);
+ debugfs_create_blob("eeprom", 0400, dir, &dev->eeprom);
+@@ -140,4 +142,4 @@ struct dentry *mt76_register_debugfs(struct mt76_dev *dev)
+
+ return dir;
+ }
+-EXPORT_SYMBOL_GPL(mt76_register_debugfs);
++EXPORT_SYMBOL_GPL(mt76_register_debugfs_fops);
+diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c
+index 5e1c1506a4c65..7aecde35cb9a3 100644
+--- a/drivers/net/wireless/mediatek/mt76/dma.c
++++ b/drivers/net/wireless/mediatek/mt76/dma.c
+@@ -465,6 +465,7 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
+
+ qbuf.addr = addr + offset;
+ qbuf.len = len - offset;
++ qbuf.skip_unmap = false;
+ mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
+ frames++;
+ }
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h
+index 25c5ceef52577..6e4d697159279 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76.h
++++ b/drivers/net/wireless/mediatek/mt76/mt76.h
+@@ -19,7 +19,7 @@
+
+ #define MT_MCU_RING_SIZE 32
+ #define MT_RX_BUF_SIZE 2048
+-#define MT_SKB_HEAD_LEN 128
++#define MT_SKB_HEAD_LEN 256
+
+ #define MT_MAX_NON_AQL_PKT 16
+ #define MT_TXQ_FREE_THR 32
+@@ -869,7 +869,13 @@ struct mt76_phy *mt76_alloc_phy(struct mt76_dev *dev, unsigned int size,
+ int mt76_register_phy(struct mt76_phy *phy, bool vht,
+ struct ieee80211_rate *rates, int n_rates);
+
+-struct dentry *mt76_register_debugfs(struct mt76_dev *dev);
++struct dentry *mt76_register_debugfs_fops(struct mt76_dev *dev,
++ const struct file_operations *ops);
++static inline struct dentry *mt76_register_debugfs(struct mt76_dev *dev)
++{
++ return mt76_register_debugfs_fops(dev, NULL);
++}
++
+ int mt76_queues_read(struct seq_file *s, void *data);
+ void mt76_seq_puts_array(struct seq_file *file, const char *str,
+ s8 *val, int len);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
+index 3972c56136a20..65f1f2bb80835 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
+@@ -525,6 +525,10 @@ mt7603_mac_fill_rx(struct mt7603_dev *dev, struct sk_buff *skb)
+ if (rxd2 & MT_RXD2_NORMAL_TKIP_MIC_ERR)
+ status->flag |= RX_FLAG_MMIC_ERROR;
+
++ /* ICV error or CCMP/BIP/WPI MIC error */
++ if (rxd2 & MT_RXD2_NORMAL_ICV_ERR)
++ status->flag |= RX_FLAG_ONLY_MONITOR;
++
+ if (FIELD_GET(MT_RXD2_NORMAL_SEC_MODE, rxd2) != 0 &&
+ !(rxd2 & (MT_RXD2_NORMAL_CLM | MT_RXD2_NORMAL_CM))) {
+ status->flag |= RX_FLAG_DECRYPTED;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/main.c b/drivers/net/wireless/mediatek/mt76/mt7603/main.c
+index 8edea1e7a602f..7f52a4a11ceaa 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7603/main.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7603/main.c
+@@ -620,6 +620,9 @@ mt7603_sta_rate_tbl_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+ struct ieee80211_sta_rates *sta_rates = rcu_dereference(sta->rates);
+ int i;
+
++ if (!sta_rates)
++ return;
++
+ spin_lock_bh(&dev->mt76.lock);
+ for (i = 0; i < ARRAY_SIZE(msta->rates); i++) {
+ msta->rates[i].idx = sta_rates->rate[i].idx;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c
+index cb4659771fd97..bda22ca0bd714 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c
+@@ -2,6 +2,33 @@
+
+ #include "mt7615.h"
+
++static int
++mt7615_reg_set(void *data, u64 val)
++{
++ struct mt7615_dev *dev = data;
++
++ mt7615_mutex_acquire(dev);
++ mt76_wr(dev, dev->mt76.debugfs_reg, val);
++ mt7615_mutex_release(dev);
++
++ return 0;
++}
++
++static int
++mt7615_reg_get(void *data, u64 *val)
++{
++ struct mt7615_dev *dev = data;
++
++ mt7615_mutex_acquire(dev);
++ *val = mt76_rr(dev, dev->mt76.debugfs_reg);
++ mt7615_mutex_release(dev);
++
++ return 0;
++}
++
++DEFINE_DEBUGFS_ATTRIBUTE(fops_regval, mt7615_reg_get, mt7615_reg_set,
++ "0x%08llx\n");
++
+ static int
+ mt7615_radar_pattern_set(void *data, u64 val)
+ {
+@@ -506,7 +533,7 @@ int mt7615_init_debugfs(struct mt7615_dev *dev)
+ {
+ struct dentry *dir;
+
+- dir = mt76_register_debugfs(&dev->mt76);
++ dir = mt76_register_debugfs_fops(&dev->mt76, &fops_regval);
+ if (!dir)
+ return -ENOMEM;
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/init.c b/drivers/net/wireless/mediatek/mt76/mt7615/init.c
+index 2f1ac644e018e..47f23ac905a3c 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/init.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/init.c
+@@ -49,12 +49,14 @@ int mt7615_thermal_init(struct mt7615_dev *dev)
+ {
+ struct wiphy *wiphy = mt76_hw(dev)->wiphy;
+ struct device *hwmon;
++ const char *name;
+
+ if (!IS_REACHABLE(CONFIG_HWMON))
+ return 0;
+
+- hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev,
+- wiphy_name(wiphy), dev,
++ name = devm_kasprintf(&wiphy->dev, GFP_KERNEL, "mt7615_%s",
++ wiphy_name(wiphy));
++ hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev, name, dev,
+ mt7615_hwmon_groups);
+ if (IS_ERR(hwmon))
+ return PTR_ERR(hwmon);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
+index ff3f85e4087c9..8f4a5d4929e09 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
+@@ -286,9 +286,16 @@ static int mt7615_mac_fill_rx(struct mt7615_dev *dev, struct sk_buff *skb)
+ if (rxd2 & MT_RXD2_NORMAL_AMSDU_ERR)
+ return -EINVAL;
+
++ hdr_trans = rxd1 & MT_RXD1_NORMAL_HDR_TRANS;
++ if (hdr_trans && (rxd2 & MT_RXD2_NORMAL_CM))
++ return -EINVAL;
++
++ /* ICV error or CCMP/BIP/WPI MIC error */
++ if (rxd2 & MT_RXD2_NORMAL_ICV_ERR)
++ status->flag |= RX_FLAG_ONLY_MONITOR;
++
+ unicast = (rxd1 & MT_RXD1_NORMAL_ADDR_TYPE) == MT_RXD1_NORMAL_U2M;
+ idx = FIELD_GET(MT_RXD2_NORMAL_WLAN_IDX, rxd2);
+- hdr_trans = rxd1 & MT_RXD1_NORMAL_HDR_TRANS;
+ status->wcid = mt7615_rx_get_wcid(dev, idx, unicast);
+
+ if (status->wcid) {
+@@ -755,12 +762,15 @@ int mt7615_mac_write_txwi(struct mt7615_dev *dev, __le32 *txwi,
+ if (info->flags & IEEE80211_TX_CTL_NO_ACK)
+ txwi[3] |= cpu_to_le32(MT_TXD3_NO_ACK);
+
+- txwi[7] = FIELD_PREP(MT_TXD7_TYPE, fc_type) |
+- FIELD_PREP(MT_TXD7_SUB_TYPE, fc_stype) |
+- FIELD_PREP(MT_TXD7_SPE_IDX, 0x18);
+- if (!is_mmio)
+- txwi[8] = FIELD_PREP(MT_TXD8_L_TYPE, fc_type) |
+- FIELD_PREP(MT_TXD8_L_SUB_TYPE, fc_stype);
++ val = FIELD_PREP(MT_TXD7_TYPE, fc_type) |
++ FIELD_PREP(MT_TXD7_SUB_TYPE, fc_stype) |
++ FIELD_PREP(MT_TXD7_SPE_IDX, 0x18);
++ txwi[7] = cpu_to_le32(val);
++ if (!is_mmio) {
++ val = FIELD_PREP(MT_TXD8_L_TYPE, fc_type) |
++ FIELD_PREP(MT_TXD8_L_SUB_TYPE, fc_stype);
++ txwi[8] = cpu_to_le32(val);
++ }
+
+ return 0;
+ }
+@@ -1494,32 +1504,41 @@ out:
+ }
+
+ static void
+-mt7615_mac_tx_free_token(struct mt7615_dev *dev, u16 token)
++mt7615_txwi_free(struct mt7615_dev *dev, struct mt76_txwi_cache *txwi)
+ {
+ struct mt76_dev *mdev = &dev->mt76;
+- struct mt76_txwi_cache *txwi;
+ __le32 *txwi_data;
+ u32 val;
+ u8 wcid;
+
+- trace_mac_tx_free(dev, token);
+- txwi = mt76_token_put(mdev, token);
+- if (!txwi)
+- return;
++ mt7615_txp_skb_unmap(mdev, txwi);
++ if (!txwi->skb)
++ goto out;
+
+ txwi_data = (__le32 *)mt76_get_txwi_ptr(mdev, txwi);
+ val = le32_to_cpu(txwi_data[1]);
+ wcid = FIELD_GET(MT_TXD1_WLAN_IDX, val);
++ mt76_tx_complete_skb(mdev, wcid, txwi->skb);
+
+- mt7615_txp_skb_unmap(mdev, txwi);
+- if (txwi->skb) {
+- mt76_tx_complete_skb(mdev, wcid, txwi->skb);
+- txwi->skb = NULL;
+- }
+-
++out:
++ txwi->skb = NULL;
+ mt76_put_txwi(mdev, txwi);
+ }
+
++static void
++mt7615_mac_tx_free_token(struct mt7615_dev *dev, u16 token)
++{
++ struct mt76_dev *mdev = &dev->mt76;
++ struct mt76_txwi_cache *txwi;
++
++ trace_mac_tx_free(dev, token);
++ txwi = mt76_token_put(mdev, token);
++ if (!txwi)
++ return;
++
++ mt7615_txwi_free(dev, txwi);
++}
++
+ static void mt7615_mac_tx_free(struct mt7615_dev *dev, struct sk_buff *skb)
+ {
+ struct mt7615_tx_free *free = (struct mt7615_tx_free *)skb->data;
+@@ -1713,7 +1732,7 @@ mt7615_mac_adjust_sensitivity(struct mt7615_phy *phy,
+ struct mt7615_dev *dev = phy->dev;
+ int false_cca = ofdm ? phy->false_cca_ofdm : phy->false_cca_cck;
+ bool ext_phy = phy != &dev->phy;
+- u16 def_th = ofdm ? -98 : -110;
++ s16 def_th = ofdm ? -98 : -110;
+ bool update = false;
+ s8 *sensitivity;
+ int signal;
+@@ -1981,6 +2000,14 @@ void mt7615_pm_power_save_work(struct work_struct *work)
+ test_bit(MT76_HW_SCHED_SCANNING, &dev->mphy.state))
+ goto out;
+
++ if (mutex_is_locked(&dev->mt76.mutex))
++ /* if mt76 mutex is held we should not put the device
++ * to sleep since we are currently accessing device
++ * register map. We need to wait for the next power_save
++ * trigger.
++ */
++ goto out;
++
+ if (time_is_after_jiffies(dev->pm.last_activity + delta)) {
+ delta = dev->pm.last_activity + delta - jiffies;
+ goto out;
+@@ -2026,16 +2053,8 @@ void mt7615_tx_token_put(struct mt7615_dev *dev)
+ int id;
+
+ spin_lock_bh(&dev->mt76.token_lock);
+- idr_for_each_entry(&dev->mt76.token, txwi, id) {
+- mt7615_txp_skb_unmap(&dev->mt76, txwi);
+- if (txwi->skb) {
+- struct ieee80211_hw *hw;
+-
+- hw = mt76_tx_status_get_hw(&dev->mt76, txwi->skb);
+- ieee80211_free_txskb(hw, txwi->skb);
+- }
+- mt76_put_txwi(&dev->mt76, txwi);
+- }
++ idr_for_each_entry(&dev->mt76.token, txwi, id)
++ mt7615_txwi_free(dev, txwi);
+ spin_unlock_bh(&dev->mt76.token_lock);
+ idr_destroy(&dev->mt76.token);
+ }
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c
+index dada43d6d879e..60a41d0829611 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c
+@@ -135,8 +135,6 @@ static int get_omac_idx(enum nl80211_iftype type, u64 mask)
+ int i;
+
+ switch (type) {
+- case NL80211_IFTYPE_MESH_POINT:
+- case NL80211_IFTYPE_ADHOC:
+ case NL80211_IFTYPE_STATION:
+ /* prefer hw bssid slot 1-3 */
+ i = get_free_idx(mask, HW_BSSID_1, HW_BSSID_3);
+@@ -160,6 +158,8 @@ static int get_omac_idx(enum nl80211_iftype type, u64 mask)
+ return HW_BSSID_0;
+
+ break;
++ case NL80211_IFTYPE_ADHOC:
++ case NL80211_IFTYPE_MESH_POINT:
+ case NL80211_IFTYPE_MONITOR:
+ case NL80211_IFTYPE_AP:
+ /* ap uses hw bssid 0 and ext bssid */
+@@ -211,11 +211,9 @@ static int mt7615_add_interface(struct ieee80211_hw *hw,
+ mvif->mt76.omac_idx = idx;
+
+ mvif->mt76.band_idx = ext_phy;
+- if (mt7615_ext_phy(dev))
+- mvif->mt76.wmm_idx = ext_phy * (MT7615_MAX_WMM_SETS / 2) +
+- mvif->mt76.idx % (MT7615_MAX_WMM_SETS / 2);
+- else
+- mvif->mt76.wmm_idx = mvif->mt76.idx % MT7615_MAX_WMM_SETS;
++ mvif->mt76.wmm_idx = vif->type != NL80211_IFTYPE_AP;
++ if (ext_phy)
++ mvif->mt76.wmm_idx += 2;
+
+ dev->mt76.vif_mask |= BIT(mvif->mt76.idx);
+ dev->omac_mask |= BIT_ULL(mvif->mt76.omac_idx);
+@@ -684,6 +682,9 @@ static void mt7615_sta_rate_tbl_update(struct ieee80211_hw *hw,
+ struct ieee80211_sta_rates *sta_rates = rcu_dereference(sta->rates);
+ int i;
+
++ if (!sta_rates)
++ return;
++
+ spin_lock_bh(&dev->mt76.lock);
+ for (i = 0; i < ARRAY_SIZE(msta->rates); i++) {
+ msta->rates[i].idx = sta_rates->rate[i].idx;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
+index f8a09692d3e4c..4fed3afad67cc 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
+@@ -808,7 +808,8 @@ mt7615_mcu_ctrl_pm_state(struct mt7615_dev *dev, int band, int state)
+
+ static int
+ mt7615_mcu_bss_basic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+- struct ieee80211_sta *sta, bool enable)
++ struct ieee80211_sta *sta, struct mt7615_phy *phy,
++ bool enable)
+ {
+ struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv;
+ u32 type = vif->p2p ? NETWORK_P2P : NETWORK_INFRA;
+@@ -821,6 +822,7 @@ mt7615_mcu_bss_basic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ switch (vif->type) {
+ case NL80211_IFTYPE_MESH_POINT:
+ case NL80211_IFTYPE_AP:
++ case NL80211_IFTYPE_MONITOR:
+ break;
+ case NL80211_IFTYPE_STATION:
+ /* TODO: enable BSS_INFO_UAPSD & BSS_INFO_PM */
+@@ -840,14 +842,19 @@ mt7615_mcu_bss_basic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ }
+
+ bss = (struct bss_info_basic *)tlv;
+- memcpy(bss->bssid, vif->bss_conf.bssid, ETH_ALEN);
+- bss->bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int);
+ bss->network_type = cpu_to_le32(type);
+- bss->dtim_period = vif->bss_conf.dtim_period;
+ bss->bmc_tx_wlan_idx = wlan_idx;
+ bss->wmm_idx = mvif->mt76.wmm_idx;
+ bss->active = enable;
+
++ if (vif->type != NL80211_IFTYPE_MONITOR) {
++ memcpy(bss->bssid, vif->bss_conf.bssid, ETH_ALEN);
++ bss->bcn_interval = cpu_to_le16(vif->bss_conf.beacon_int);
++ bss->dtim_period = vif->bss_conf.dtim_period;
++ } else {
++ memcpy(bss->bssid, phy->mt76->macaddr, ETH_ALEN);
++ }
++
+ return 0;
+ }
+
+@@ -863,6 +870,7 @@ mt7615_mcu_bss_omac_tlv(struct sk_buff *skb, struct ieee80211_vif *vif)
+ tlv = mt76_connac_mcu_add_tlv(skb, BSS_INFO_OMAC, sizeof(*omac));
+
+ switch (vif->type) {
++ case NL80211_IFTYPE_MONITOR:
+ case NL80211_IFTYPE_MESH_POINT:
+ case NL80211_IFTYPE_AP:
+ if (vif->p2p)
+@@ -929,7 +937,7 @@ mt7615_mcu_add_bss(struct mt7615_phy *phy, struct ieee80211_vif *vif,
+ if (enable)
+ mt7615_mcu_bss_omac_tlv(skb, vif);
+
+- mt7615_mcu_bss_basic_tlv(skb, vif, sta, enable);
++ mt7615_mcu_bss_basic_tlv(skb, vif, sta, phy, enable);
+
+ if (enable && mvif->mt76.omac_idx >= EXT_BSSID_START &&
+ mvif->mt76.omac_idx < REPEATER_BSSID_START)
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
+index a2465b49ecd0c..87b4aa52ee0f9 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
+@@ -28,8 +28,6 @@ static void mt7615_pci_init_work(struct work_struct *work)
+ return;
+
+ mt7615_init_work(dev);
+- if (dev->dbdc_support)
+- mt7615_register_ext_phy(dev);
+ }
+
+ static int mt7615_init_hardware(struct mt7615_dev *dev)
+@@ -160,6 +158,12 @@ int mt7615_register_device(struct mt7615_dev *dev)
+ mt7615_init_txpower(dev, &dev->mphy.sband_2g.sband);
+ mt7615_init_txpower(dev, &dev->mphy.sband_5g.sband);
+
++ if (dev->dbdc_support) {
++ ret = mt7615_register_ext_phy(dev);
++ if (ret)
++ return ret;
++ }
++
+ return mt7615_init_debugfs(dev);
+ }
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+index 5c3a81e5f559d..017bd59c4ea80 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+@@ -295,7 +295,7 @@ mt76_connac_mcu_alloc_wtbl_req(struct mt76_dev *dev, struct mt76_wcid *wcid,
+ }
+
+ if (sta_hdr)
+- sta_hdr->len = cpu_to_le16(sizeof(hdr));
++ le16_add_cpu(&sta_hdr->len, sizeof(hdr));
+
+ return skb_put_data(nskb, &hdr, sizeof(hdr));
+ }
+@@ -689,7 +689,7 @@ mt76_connac_get_phy_mode_v2(struct mt76_phy *mphy, struct ieee80211_vif *vif,
+ if (ht_cap->ht_supported)
+ mode |= PHY_TYPE_BIT_HT;
+
+- if (he_cap->has_he)
++ if (he_cap && he_cap->has_he)
+ mode |= PHY_TYPE_BIT_HE;
+ } else if (band == NL80211_BAND_5GHZ) {
+ mode |= PHY_TYPE_BIT_OFDM;
+@@ -700,7 +700,7 @@ mt76_connac_get_phy_mode_v2(struct mt76_phy *mphy, struct ieee80211_vif *vif,
+ if (vht_cap->vht_supported)
+ mode |= PHY_TYPE_BIT_VHT;
+
+- if (he_cap->has_he)
++ if (he_cap && he_cap->has_he)
+ mode |= PHY_TYPE_BIT_HE;
+ }
+
+@@ -719,6 +719,7 @@ void mt76_connac_mcu_sta_tlv(struct mt76_phy *mphy, struct sk_buff *skb,
+ struct sta_rec_state *state;
+ struct sta_rec_phy *phy;
+ struct tlv *tlv;
++ u16 supp_rates;
+
+ /* starec ht */
+ if (sta->ht_cap.ht_supported) {
+@@ -767,7 +768,15 @@ void mt76_connac_mcu_sta_tlv(struct mt76_phy *mphy, struct sk_buff *skb,
+
+ tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA, sizeof(*ra_info));
+ ra_info = (struct sta_rec_ra_info *)tlv;
+- ra_info->legacy = cpu_to_le16((u16)sta->supp_rates[band]);
++
++ supp_rates = sta->supp_rates[band];
++ if (band == NL80211_BAND_2GHZ)
++ supp_rates = FIELD_PREP(RA_LEGACY_OFDM, supp_rates >> 4) |
++ FIELD_PREP(RA_LEGACY_CCK, supp_rates & 0xf);
++ else
++ supp_rates = FIELD_PREP(RA_LEGACY_OFDM, supp_rates);
++
++ ra_info->legacy = cpu_to_le16(supp_rates);
+
+ if (sta->ht_cap.ht_supported)
+ memcpy(ra_info->rx_mcs_bitmask, sta->ht_cap.mcs.rx_mask,
+@@ -1929,19 +1938,22 @@ mt76_connac_mcu_key_iter(struct ieee80211_hw *hw,
+ key->cipher != WLAN_CIPHER_SUITE_TKIP)
+ return;
+
+- if (key->cipher == WLAN_CIPHER_SUITE_TKIP) {
+- gtk_tlv->proto = cpu_to_le32(NL80211_WPA_VERSION_1);
++ if (key->cipher == WLAN_CIPHER_SUITE_TKIP)
+ cipher = BIT(3);
+- } else {
+- gtk_tlv->proto = cpu_to_le32(NL80211_WPA_VERSION_2);
++ else
+ cipher = BIT(4);
+- }
+
+ /* we are assuming here to have a single pairwise key */
+ if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) {
++ if (key->cipher == WLAN_CIPHER_SUITE_TKIP)
++ gtk_tlv->proto = cpu_to_le32(NL80211_WPA_VERSION_1);
++ else
++ gtk_tlv->proto = cpu_to_le32(NL80211_WPA_VERSION_2);
++
+ gtk_tlv->pairwise_cipher = cpu_to_le32(cipher);
+- gtk_tlv->group_cipher = cpu_to_le32(cipher);
+ gtk_tlv->keyid = key->keyidx;
++ } else {
++ gtk_tlv->group_cipher = cpu_to_le32(cipher);
+ }
+ }
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
+index 1c73beb226771..77d4435e4581e 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
++++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
+@@ -124,6 +124,8 @@ struct sta_rec_state {
+ u8 rsv[1];
+ } __packed;
+
++#define RA_LEGACY_OFDM GENMASK(13, 6)
++#define RA_LEGACY_CCK GENMASK(3, 0)
+ #define HT_MCS_MASK_NUM 10
+ struct sta_rec_ra_info {
+ __le16 tag;
+@@ -844,14 +846,14 @@ struct mt76_connac_gtk_rekey_tlv {
+ * 2: rekey update
+ */
+ u8 keyid;
+- u8 pad[2];
++ u8 option; /* 1: rekey data update without enabling offload */
++ u8 pad[1];
+ __le32 proto; /* WPA-RSN-WAPI-OPSN */
+ __le32 pairwise_cipher;
+ __le32 group_cipher;
+ __le32 key_mgmt; /* NONE-PSK-IEEE802.1X */
+ __le32 mgmt_group_cipher;
+- u8 option; /* 1: rekey data update without enabling offload */
+- u8 reserverd[3];
++ u8 reserverd[4];
+ } __packed;
+
+ #define MT76_CONNAC_WOW_MASK_MAX_LEN 16
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
+index c32e6dc687739..07b21b2085823 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
+@@ -176,7 +176,7 @@ void mt76x02_mac_wcid_set_drop(struct mt76x02_dev *dev, u8 idx, bool drop)
+ mt76_wr(dev, MT_WCID_DROP(idx), (val & ~bit) | (bit * drop));
+ }
+
+-static __le16
++static u16
+ mt76x02_mac_tx_rate_val(struct mt76x02_dev *dev,
+ const struct ieee80211_tx_rate *rate, u8 *nss_val)
+ {
+@@ -222,14 +222,14 @@ mt76x02_mac_tx_rate_val(struct mt76x02_dev *dev,
+ rateval |= MT_RXWI_RATE_SGI;
+
+ *nss_val = nss;
+- return cpu_to_le16(rateval);
++ return rateval;
+ }
+
+ void mt76x02_mac_wcid_set_rate(struct mt76x02_dev *dev, struct mt76_wcid *wcid,
+ const struct ieee80211_tx_rate *rate)
+ {
+ s8 max_txpwr_adj = mt76x02_tx_get_max_txpwr_adj(dev, rate);
+- __le16 rateval;
++ u16 rateval;
+ u32 tx_info;
+ s8 nss;
+
+@@ -342,7 +342,7 @@ void mt76x02_mac_write_txwi(struct mt76x02_dev *dev, struct mt76x02_txwi *txwi,
+ struct ieee80211_key_conf *key = info->control.hw_key;
+ u32 wcid_tx_info;
+ u16 rate_ht_mask = FIELD_PREP(MT_RXWI_RATE_PHY, BIT(1) | BIT(2));
+- u16 txwi_flags = 0;
++ u16 txwi_flags = 0, rateval;
+ u8 nss;
+ s8 txpwr_adj, max_txpwr_adj;
+ u8 ccmp_pn[8], nstreams = dev->mphy.chainmask & 0xf;
+@@ -380,14 +380,15 @@ void mt76x02_mac_write_txwi(struct mt76x02_dev *dev, struct mt76x02_txwi *txwi,
+
+ if (wcid && (rate->idx < 0 || !rate->count)) {
+ wcid_tx_info = wcid->tx_info;
+- txwi->rate = FIELD_GET(MT_WCID_TX_INFO_RATE, wcid_tx_info);
++ rateval = FIELD_GET(MT_WCID_TX_INFO_RATE, wcid_tx_info);
+ max_txpwr_adj = FIELD_GET(MT_WCID_TX_INFO_TXPWR_ADJ,
+ wcid_tx_info);
+ nss = FIELD_GET(MT_WCID_TX_INFO_NSS, wcid_tx_info);
+ } else {
+- txwi->rate = mt76x02_mac_tx_rate_val(dev, rate, &nss);
++ rateval = mt76x02_mac_tx_rate_val(dev, rate, &nss);
+ max_txpwr_adj = mt76x02_tx_get_max_txpwr_adj(dev, rate);
+ }
++ txwi->rate = cpu_to_le16(rateval);
+
+ txpwr_adj = mt76x02_tx_get_txpwr_adj(dev, dev->txpower_conf,
+ max_txpwr_adj);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c b/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c
+index adf288e50e212..5cd0379d86de8 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c
++++ b/drivers/net/wireless/mediatek/mt76/mt76x2/pci.c
+@@ -80,7 +80,7 @@ mt76x2e_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ mt76_rmw_field(dev, 0x15a10, 0x1f << 16, 0x9);
+
+ /* RG_SSUSB_G1_CDR_BIC_LTR = 0xf */
+- mt76_rmw_field(dev, 0x15a0c, 0xf << 28, 0xf);
++ mt76_rmw_field(dev, 0x15a0c, 0xfU << 28, 0xf);
+
+ /* RG_SSUSB_CDR_BR_PE1D = 0x3 */
+ mt76_rmw_field(dev, 0x15c58, 0x3 << 6, 0x3);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c
+index 4798d6344305d..b171027e0cfa8 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c
+@@ -130,9 +130,12 @@ static int mt7915_thermal_init(struct mt7915_phy *phy)
+ struct wiphy *wiphy = phy->mt76->hw->wiphy;
+ struct thermal_cooling_device *cdev;
+ struct device *hwmon;
++ const char *name;
+
+- cdev = thermal_cooling_device_register(wiphy_name(wiphy), phy,
+- &mt7915_thermal_ops);
++ name = devm_kasprintf(&wiphy->dev, GFP_KERNEL, "mt7915_%s",
++ wiphy_name(wiphy));
++
++ cdev = thermal_cooling_device_register(name, phy, &mt7915_thermal_ops);
+ if (!IS_ERR(cdev)) {
+ if (sysfs_create_link(&wiphy->dev.kobj, &cdev->device.kobj,
+ "cooling_device") < 0)
+@@ -144,8 +147,7 @@ static int mt7915_thermal_init(struct mt7915_phy *phy)
+ if (!IS_REACHABLE(CONFIG_HWMON))
+ return 0;
+
+- hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev,
+- wiphy_name(wiphy), phy,
++ hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev, name, phy,
+ mt7915_hwmon_groups);
+ if (IS_ERR(hwmon))
+ return PTR_ERR(hwmon);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
+index 2462704094b0a..7691292526e05 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
+@@ -349,9 +349,16 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb)
+ if (rxd2 & MT_RXD2_NORMAL_AMSDU_ERR)
+ return -EINVAL;
+
++ hdr_trans = rxd2 & MT_RXD2_NORMAL_HDR_TRANS;
++ if (hdr_trans && (rxd1 & MT_RXD1_NORMAL_CM))
++ return -EINVAL;
++
++ /* ICV error or CCMP/BIP/WPI MIC error */
++ if (rxd1 & MT_RXD1_NORMAL_ICV_ERR)
++ status->flag |= RX_FLAG_ONLY_MONITOR;
++
+ unicast = FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) == MT_RXD3_NORMAL_U2M;
+ idx = FIELD_GET(MT_RXD1_NORMAL_WLAN_IDX, rxd1);
+- hdr_trans = rxd2 & MT_RXD2_NORMAL_HDR_TRANS;
+ status->wcid = mt7915_rx_get_wcid(dev, idx, unicast);
+
+ if (status->wcid) {
+@@ -892,6 +899,7 @@ mt7915_mac_write_txwi_80211(struct mt7915_dev *dev, __le32 *txwi,
+ val = MT_TXD3_SN_VALID |
+ FIELD_PREP(MT_TXD3_SEQ, IEEE80211_SEQ_TO_SN(seqno));
+ txwi[3] |= cpu_to_le32(val);
++ txwi[7] &= ~cpu_to_le32(MT_TXD7_HW_AMSDU);
+ }
+
+ val = FIELD_PREP(MT_TXD7_TYPE, fc_type) |
+@@ -1232,7 +1240,7 @@ mt7915_mac_add_txs_skb(struct mt7915_dev *dev, struct mt76_wcid *wcid, int pid,
+ goto out;
+
+ info = IEEE80211_SKB_CB(skb);
+- if (!(txs_data[0] & le32_to_cpu(MT_TXS0_ACK_ERROR_MASK)))
++ if (!(txs_data[0] & cpu_to_le32(MT_TXS0_ACK_ERROR_MASK)))
+ info->flags |= IEEE80211_TX_STAT_ACK;
+
+ info->status.ampdu_len = 1;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.h b/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
+index eb1885f4bd8eb..fee7741b5d421 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
++++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
+@@ -272,7 +272,8 @@ enum tx_mcu_port_q_idx {
+ #define MT_TX_RATE_MODE GENMASK(9, 6)
+ #define MT_TX_RATE_SU_EXT_TONE BIT(5)
+ #define MT_TX_RATE_DCM BIT(4)
+-#define MT_TX_RATE_IDX GENMASK(3, 0)
++/* VHT/HE only use bits 0-3 */
++#define MT_TX_RATE_IDX GENMASK(5, 0)
+
+ #define MT_TXP_MAX_BUF_NUM 6
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+index 43960770a9af2..e9d854e3293e4 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+@@ -176,7 +176,7 @@ mt7915_get_phy_mode(struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+ if (ht_cap->ht_supported)
+ mode |= PHY_MODE_GN;
+
+- if (he_cap->has_he)
++ if (he_cap && he_cap->has_he)
+ mode |= PHY_MODE_AX_24G;
+ } else if (band == NL80211_BAND_5GHZ) {
+ mode |= PHY_MODE_A;
+@@ -187,7 +187,7 @@ mt7915_get_phy_mode(struct ieee80211_vif *vif, struct ieee80211_sta *sta)
+ if (vht_cap->vht_supported)
+ mode |= PHY_MODE_AC;
+
+- if (he_cap->has_he)
++ if (he_cap && he_cap->has_he)
+ mode |= PHY_MODE_AX_5G;
+ }
+
+@@ -721,7 +721,7 @@ mt7915_mcu_alloc_sta_req(struct mt7915_dev *dev, struct mt7915_vif *mvif,
+ .bss_idx = mvif->idx,
+ .wlan_idx_lo = msta ? to_wcid_lo(msta->wcid.idx) : 0,
+ .wlan_idx_hi = msta ? to_wcid_hi(msta->wcid.idx) : 0,
+- .muar_idx = msta ? mvif->omac_idx : 0,
++ .muar_idx = msta && msta->wcid.sta ? mvif->omac_idx : 0xe,
+ .is_tlv_append = 1,
+ };
+ struct sk_buff *skb;
+@@ -757,7 +757,7 @@ mt7915_mcu_alloc_wtbl_req(struct mt7915_dev *dev, struct mt7915_sta *msta,
+ }
+
+ if (sta_hdr)
+- sta_hdr->len = cpu_to_le16(sizeof(hdr));
++ le16_add_cpu(&sta_hdr->len, sizeof(hdr));
+
+ return skb_put_data(nskb, &hdr, sizeof(hdr));
+ }
+@@ -925,7 +925,7 @@ static void mt7915_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
+
+ elem = ieee80211_bss_get_elem(bss, WLAN_EID_EXT_CAPABILITY);
+
+- if (!elem || elem->datalen < 10 ||
++ if (!elem || elem->datalen <= 10 ||
+ !(elem->data[10] &
+ WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT))
+ data->tolerated = false;
+@@ -1201,7 +1201,7 @@ mt7915_mcu_sta_key_tlv(struct mt7915_sta *msta, struct sk_buff *skb,
+ u8 cipher;
+
+ cipher = mt7915_mcu_get_cipher(key->cipher);
+- if (cipher == MT_CIPHER_NONE)
++ if (cipher == MCU_CIPHER_NONE)
+ return -EOPNOTSUPP;
+
+ sec_key = &sec->key[0];
+@@ -1396,8 +1396,11 @@ mt7915_mcu_wtbl_generic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ generic = (struct wtbl_generic *)tlv;
+
+ if (sta) {
++ if (vif->type == NL80211_IFTYPE_STATION)
++ generic->partial_aid = cpu_to_le16(vif->bss_conf.aid);
++ else
++ generic->partial_aid = cpu_to_le16(sta->aid);
+ memcpy(generic->peer_addr, sta->addr, ETH_ALEN);
+- generic->partial_aid = cpu_to_le16(sta->aid);
+ generic->muar_idx = mvif->omac_idx;
+ generic->qos = sta->wme;
+ } else {
+@@ -1451,12 +1454,15 @@ mt7915_mcu_sta_basic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ case NL80211_IFTYPE_MESH_POINT:
+ case NL80211_IFTYPE_AP:
+ basic->conn_type = cpu_to_le32(CONNECTION_INFRA_STA);
++ basic->aid = cpu_to_le16(sta->aid);
+ break;
+ case NL80211_IFTYPE_STATION:
+ basic->conn_type = cpu_to_le32(CONNECTION_INFRA_AP);
++ basic->aid = cpu_to_le16(vif->bss_conf.aid);
+ break;
+ case NL80211_IFTYPE_ADHOC:
+ basic->conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC);
++ basic->aid = cpu_to_le16(sta->aid);
+ break;
+ default:
+ WARN_ON(1);
+@@ -1464,7 +1470,6 @@ mt7915_mcu_sta_basic_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ }
+
+ memcpy(basic->peer_addr, sta->addr, ETH_ALEN);
+- basic->aid = cpu_to_le16(sta->aid);
+ basic->qos = sta->wme;
+ }
+
+@@ -2790,7 +2795,7 @@ out:
+ default:
+ ret = -EAGAIN;
+ dev_err(dev->mt76.dev, "Failed to release patch semaphore\n");
+- goto out;
++ break;
+ }
+ release_firmware(fw);
+
+@@ -3391,20 +3396,20 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd)
+
+ static int mt7915_mcu_set_eeprom_flash(struct mt7915_dev *dev)
+ {
+-#define TOTAL_PAGE_MASK GENMASK(7, 5)
++#define MAX_PAGE_IDX_MASK GENMASK(7, 5)
+ #define PAGE_IDX_MASK GENMASK(4, 2)
+ #define PER_PAGE_SIZE 0x400
+ struct mt7915_mcu_eeprom req = { .buffer_mode = EE_MODE_BUFFER };
+- u8 total = MT7915_EEPROM_SIZE / PER_PAGE_SIZE;
++ u8 total = DIV_ROUND_UP(MT7915_EEPROM_SIZE, PER_PAGE_SIZE);
+ u8 *eep = (u8 *)dev->mt76.eeprom.data;
+ int eep_len;
+ int i;
+
+- for (i = 0; i <= total; i++, eep += eep_len) {
++ for (i = 0; i < total; i++, eep += eep_len) {
+ struct sk_buff *skb;
+ int ret;
+
+- if (i == total)
++ if (i == total - 1 && !!(MT7915_EEPROM_SIZE % PER_PAGE_SIZE))
+ eep_len = MT7915_EEPROM_SIZE % PER_PAGE_SIZE;
+ else
+ eep_len = PER_PAGE_SIZE;
+@@ -3414,7 +3419,7 @@ static int mt7915_mcu_set_eeprom_flash(struct mt7915_dev *dev)
+ if (!skb)
+ return -ENOMEM;
+
+- req.format = FIELD_PREP(TOTAL_PAGE_MASK, total) |
++ req.format = FIELD_PREP(MAX_PAGE_IDX_MASK, total - 1) |
+ FIELD_PREP(PAGE_IDX_MASK, i) | EE_FORMAT_WHOLE;
+ req.len = cpu_to_le16(eep_len);
+
+@@ -3481,7 +3486,7 @@ static int mt7915_mcu_set_pre_cal(struct mt7915_dev *dev, u8 idx,
+ u8 idx;
+ u8 rsv[4];
+ __le32 len;
+- } req;
++ } req = {};
+ struct sk_buff *skb;
+
+ skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, sizeof(req) + len);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
+index 77468bdae460b..8d5e261cd10f6 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
+@@ -4,6 +4,32 @@
+ #include "mt7921.h"
+ #include "eeprom.h"
+
++static int
++mt7921_reg_set(void *data, u64 val)
++{
++ struct mt7921_dev *dev = data;
++
++ mt7921_mutex_acquire(dev);
++ mt76_wr(dev, dev->mt76.debugfs_reg, val);
++ mt7921_mutex_release(dev);
++
++ return 0;
++}
++
++static int
++mt7921_reg_get(void *data, u64 *val)
++{
++ struct mt7921_dev *dev = data;
++
++ mt7921_mutex_acquire(dev);
++ *val = mt76_rr(dev, dev->mt76.debugfs_reg);
++ mt7921_mutex_release(dev);
++
++ return 0;
++}
++
++DEFINE_DEBUGFS_ATTRIBUTE(fops_regval, mt7921_reg_get, mt7921_reg_set,
++ "0x%08llx\n");
+ static int
+ mt7921_fw_debug_set(void *data, u64 val)
+ {
+@@ -69,6 +95,8 @@ mt7921_tx_stats_show(struct seq_file *file, void *data)
+ struct mt7921_dev *dev = file->private;
+ int stat[8], i, n;
+
++ mt7921_mutex_acquire(dev);
++
+ mt7921_ampdu_stat_read_phy(&dev->phy, file);
+
+ /* Tx amsdu info */
+@@ -78,6 +106,8 @@ mt7921_tx_stats_show(struct seq_file *file, void *data)
+ n += stat[i];
+ }
+
++ mt7921_mutex_release(dev);
++
+ for (i = 0; i < ARRAY_SIZE(stat); i++) {
+ seq_printf(file, "AMSDU pack count of %d MSDU in TXD: 0x%x ",
+ i + 1, stat[i]);
+@@ -98,25 +128,28 @@ mt7921_queues_acq(struct seq_file *s, void *data)
+ struct mt7921_dev *dev = dev_get_drvdata(s->private);
+ int i;
+
+- for (i = 0; i < 16; i++) {
+- int j, acs = i / 4, index = i % 4;
++ mt7921_mutex_acquire(dev);
++
++ for (i = 0; i < 4; i++) {
+ u32 ctrl, val, qlen = 0;
++ int j;
+
+- val = mt76_rr(dev, MT_PLE_AC_QEMPTY(acs, index));
+- ctrl = BIT(31) | BIT(15) | (acs << 8);
++ val = mt76_rr(dev, MT_PLE_AC_QEMPTY(i));
++ ctrl = BIT(31) | BIT(11) | (i << 24);
+
+ for (j = 0; j < 32; j++) {
+ if (val & BIT(j))
+ continue;
+
+- mt76_wr(dev, MT_PLE_FL_Q0_CTRL,
+- ctrl | (j + (index << 5)));
++ mt76_wr(dev, MT_PLE_FL_Q0_CTRL, ctrl | j);
+ qlen += mt76_get_field(dev, MT_PLE_FL_Q3_CTRL,
+ GENMASK(11, 0));
+ }
+- seq_printf(s, "AC%d%d: queued=%d\n", acs, index, qlen);
++ seq_printf(s, "AC%d: queued=%d\n", i, qlen);
+ }
+
++ mt7921_mutex_release(dev);
++
+ return 0;
+ }
+
+@@ -373,7 +406,7 @@ int mt7921_init_debugfs(struct mt7921_dev *dev)
+ {
+ struct dentry *dir;
+
+- dir = mt76_register_debugfs(&dev->mt76);
++ dir = mt76_register_debugfs_fops(&dev->mt76, &fops_regval);
+ if (!dir)
+ return -ENOMEM;
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/dma.c b/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
+index 7d7d43a5422f8..93d0cc1827d26 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
+@@ -118,109 +118,6 @@ static void mt7921_dma_prefetch(struct mt7921_dev *dev)
+ mt76_wr(dev, MT_WFDMA0_TX_RING17_EXT_CTRL, PREFETCH(0x380, 0x4));
+ }
+
+-static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr)
+-{
+- static const struct {
+- u32 phys;
+- u32 mapped;
+- u32 size;
+- } fixed_map[] = {
+- { 0x00400000, 0x80000, 0x10000}, /* WF_MCU_SYSRAM */
+- { 0x00410000, 0x90000, 0x10000}, /* WF_MCU_SYSRAM (configure register) */
+- { 0x40000000, 0x70000, 0x10000}, /* WF_UMAC_SYSRAM */
+- { 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */
+- { 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */
+- { 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
+- { 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */
+- { 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */
+- { 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */
+- { 0x7c060000, 0xe0000, 0x10000}, /* CONN_INFRA, conn_host_csr_top */
+- { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */
+- { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */
+- { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
+- { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
+- { 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */
+- { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
+- { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
+- { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
+- { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
+- { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
+- { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
+- { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
+- { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+- { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+- { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
+- { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+- { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+- { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
+- { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
+- { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
+- { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
+- { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
+- { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
+- { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
+- { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+- { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+- { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
+- { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+- { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+- { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
+- { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
+- { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
+- };
+- int i;
+-
+- if (addr < 0x100000)
+- return addr;
+-
+- for (i = 0; i < ARRAY_SIZE(fixed_map); i++) {
+- u32 ofs;
+-
+- if (addr < fixed_map[i].phys)
+- continue;
+-
+- ofs = addr - fixed_map[i].phys;
+- if (ofs > fixed_map[i].size)
+- continue;
+-
+- return fixed_map[i].mapped + ofs;
+- }
+-
+- if ((addr >= 0x18000000 && addr < 0x18c00000) ||
+- (addr >= 0x70000000 && addr < 0x78000000) ||
+- (addr >= 0x7c000000 && addr < 0x7c400000))
+- return mt7921_reg_map_l1(dev, addr);
+-
+- dev_err(dev->mt76.dev, "Access currently unsupported address %08x\n",
+- addr);
+-
+- return 0;
+-}
+-
+-static u32 mt7921_rr(struct mt76_dev *mdev, u32 offset)
+-{
+- struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
+- u32 addr = __mt7921_reg_addr(dev, offset);
+-
+- return dev->bus_ops->rr(mdev, addr);
+-}
+-
+-static void mt7921_wr(struct mt76_dev *mdev, u32 offset, u32 val)
+-{
+- struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
+- u32 addr = __mt7921_reg_addr(dev, offset);
+-
+- dev->bus_ops->wr(mdev, addr, val);
+-}
+-
+-static u32 mt7921_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val)
+-{
+- struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
+- u32 addr = __mt7921_reg_addr(dev, offset);
+-
+- return dev->bus_ops->rmw(mdev, addr, mask, val);
+-}
+-
+ static int mt7921_dma_disable(struct mt7921_dev *dev, bool force)
+ {
+ if (force) {
+@@ -380,20 +277,8 @@ int mt7921_wpdma_reinit_cond(struct mt7921_dev *dev)
+
+ int mt7921_dma_init(struct mt7921_dev *dev)
+ {
+- struct mt76_bus_ops *bus_ops;
+ int ret;
+
+- dev->bus_ops = dev->mt76.bus;
+- bus_ops = devm_kmemdup(dev->mt76.dev, dev->bus_ops, sizeof(*bus_ops),
+- GFP_KERNEL);
+- if (!bus_ops)
+- return -ENOMEM;
+-
+- bus_ops->rr = mt7921_rr;
+- bus_ops->wr = mt7921_wr;
+- bus_ops->rmw = mt7921_rmw;
+- dev->mt76.bus = bus_ops;
+-
+ mt76_dma_attach(&dev->mt76);
+
+ ret = mt7921_dma_disable(dev, true);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
+index a9ce10b988273..78a00028137bd 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
+@@ -106,6 +106,10 @@ mt7921_mac_init_band(struct mt7921_dev *dev, u8 band)
+ mt76_set(dev, MT_WF_RMAC_MIB_TIME0(band), MT_WF_RMAC_MIB_RXTIME_EN);
+ mt76_set(dev, MT_WF_RMAC_MIB_AIRTIME0(band), MT_WF_RMAC_MIB_RXTIME_EN);
+
++ /* enable MIB tx-rx time reporting */
++ mt76_set(dev, MT_MIB_SCR1(band), MT_MIB_TXDUR_EN);
++ mt76_set(dev, MT_MIB_SCR1(band), MT_MIB_RXDUR_EN);
++
+ mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_MAX_RX_LEN, 1536);
+ /* disable rx rate report by default due to hw issues */
+ mt76_clear(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN);
+@@ -247,8 +251,17 @@ int mt7921_register_device(struct mt7921_dev *dev)
+
+ void mt7921_unregister_device(struct mt7921_dev *dev)
+ {
++ int i;
++ struct mt76_connac_pm *pm = &dev->pm;
++
+ mt76_unregister_device(&dev->mt76);
++ mt76_for_each_q_rx(&dev->mt76, i)
++ napi_disable(&dev->mt76.napi[i]);
++ cancel_delayed_work_sync(&pm->ps_work);
++ cancel_work_sync(&pm->wake_work);
++
+ mt7921_tx_token_put(dev);
++ mt7921_mcu_drv_pmctrl(dev);
+ mt7921_dma_cleanup(dev);
+ mt7921_mcu_exit(dev);
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+index 7fe2e3a50428f..c093920a597d1 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+@@ -180,12 +180,56 @@ mt7921_mac_decode_he_radiotap_ru(struct mt76_rx_status *status,
+ IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET);
+ }
+
++static void
++mt7921_mac_decode_he_mu_radiotap(struct sk_buff *skb,
++ struct mt76_rx_status *status,
++ __le32 *rxv)
++{
++ static const struct ieee80211_radiotap_he_mu mu_known = {
++ .flags1 = HE_BITS(MU_FLAGS1_SIG_B_MCS_KNOWN) |
++ HE_BITS(MU_FLAGS1_SIG_B_DCM_KNOWN) |
++ HE_BITS(MU_FLAGS1_CH1_RU_KNOWN) |
++ HE_BITS(MU_FLAGS1_SIG_B_SYMS_USERS_KNOWN) |
++ HE_BITS(MU_FLAGS1_SIG_B_COMP_KNOWN),
++ .flags2 = HE_BITS(MU_FLAGS2_BW_FROM_SIG_A_BW_KNOWN) |
++ HE_BITS(MU_FLAGS2_PUNC_FROM_SIG_A_BW_KNOWN),
++ };
++ struct ieee80211_radiotap_he_mu *he_mu = NULL;
++
++ he_mu = skb_push(skb, sizeof(mu_known));
++ memcpy(he_mu, &mu_known, sizeof(mu_known));
++
++#define MU_PREP(f, v) le16_encode_bits(v, IEEE80211_RADIOTAP_HE_MU_##f)
++
++ he_mu->flags1 |= MU_PREP(FLAGS1_SIG_B_MCS, status->rate_idx);
++ if (status->he_dcm)
++ he_mu->flags1 |= MU_PREP(FLAGS1_SIG_B_DCM, status->he_dcm);
++
++ he_mu->flags2 |= MU_PREP(FLAGS2_BW_FROM_SIG_A_BW, status->bw) |
++ MU_PREP(FLAGS2_SIG_B_SYMS_USERS,
++ le32_get_bits(rxv[2], MT_CRXV_HE_NUM_USER));
++
++ he_mu->ru_ch1[0] = FIELD_GET(MT_CRXV_HE_RU0, cpu_to_le32(rxv[3]));
++
++ if (status->bw >= RATE_INFO_BW_40) {
++ he_mu->flags1 |= HE_BITS(MU_FLAGS1_CH2_RU_KNOWN);
++ he_mu->ru_ch2[0] =
++ FIELD_GET(MT_CRXV_HE_RU1, cpu_to_le32(rxv[3]));
++ }
++
++ if (status->bw >= RATE_INFO_BW_80) {
++ he_mu->ru_ch1[1] =
++ FIELD_GET(MT_CRXV_HE_RU2, cpu_to_le32(rxv[3]));
++ he_mu->ru_ch2[1] =
++ FIELD_GET(MT_CRXV_HE_RU3, cpu_to_le32(rxv[3]));
++ }
++}
++
+ static void
+ mt7921_mac_decode_he_radiotap(struct sk_buff *skb,
+ struct mt76_rx_status *status,
+ __le32 *rxv, u32 phy)
+ {
+- /* TODO: struct ieee80211_radiotap_he_mu */
+ static const struct ieee80211_radiotap_he known = {
+ .data1 = HE_BITS(DATA1_DATA_MCS_KNOWN) |
+ HE_BITS(DATA1_DATA_DCM_KNOWN) |
+@@ -193,6 +237,7 @@ mt7921_mac_decode_he_radiotap(struct sk_buff *skb,
+ HE_BITS(DATA1_CODING_KNOWN) |
+ HE_BITS(DATA1_LDPC_XSYMSEG_KNOWN) |
+ HE_BITS(DATA1_DOPPLER_KNOWN) |
++ HE_BITS(DATA1_SPTL_REUSE_KNOWN) |
+ HE_BITS(DATA1_BSS_COLOR_KNOWN),
+ .data2 = HE_BITS(DATA2_GI_KNOWN) |
+ HE_BITS(DATA2_TXBF_KNOWN) |
+@@ -207,9 +252,12 @@ mt7921_mac_decode_he_radiotap(struct sk_buff *skb,
+
+ he->data3 = HE_PREP(DATA3_BSS_COLOR, BSS_COLOR, rxv[14]) |
+ HE_PREP(DATA3_LDPC_XSYMSEG, LDPC_EXT_SYM, rxv[2]);
++ he->data4 = HE_PREP(DATA4_SU_MU_SPTL_REUSE, SR_MASK, rxv[11]);
+ he->data5 = HE_PREP(DATA5_PE_DISAMBIG, PE_DISAMBIG, rxv[2]) |
+ le16_encode_bits(ltf_size,
+ IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE);
++ if (cpu_to_le32(rxv[0]) & MT_PRXV_TXBF)
++ he->data5 |= HE_BITS(DATA5_TXBF);
+ he->data6 = HE_PREP(DATA6_TXOP, TXOP_DUR, rxv[14]) |
+ HE_PREP(DATA6_DOPPLER, DOPPLER, rxv[14]);
+
+@@ -217,8 +265,7 @@ mt7921_mac_decode_he_radiotap(struct sk_buff *skb,
+ case MT_PHY_TYPE_HE_SU:
+ he->data1 |= HE_BITS(DATA1_FORMAT_SU) |
+ HE_BITS(DATA1_UL_DL_KNOWN) |
+- HE_BITS(DATA1_BEAM_CHANGE_KNOWN) |
+- HE_BITS(DATA1_SPTL_REUSE_KNOWN);
++ HE_BITS(DATA1_BEAM_CHANGE_KNOWN);
+
+ he->data3 |= HE_PREP(DATA3_BEAM_CHANGE, BEAM_CHNG, rxv[14]) |
+ HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]);
+@@ -232,17 +279,15 @@ mt7921_mac_decode_he_radiotap(struct sk_buff *skb,
+ break;
+ case MT_PHY_TYPE_HE_MU:
+ he->data1 |= HE_BITS(DATA1_FORMAT_MU) |
+- HE_BITS(DATA1_UL_DL_KNOWN) |
+- HE_BITS(DATA1_SPTL_REUSE_KNOWN);
++ HE_BITS(DATA1_UL_DL_KNOWN);
+
+ he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]);
+- he->data4 |= HE_PREP(DATA4_SU_MU_SPTL_REUSE, SR_MASK, rxv[11]);
++ he->data4 |= HE_PREP(DATA4_MU_STA_ID, MU_AID, rxv[7]);
+
+ mt7921_mac_decode_he_radiotap_ru(status, he, rxv);
+ break;
+ case MT_PHY_TYPE_HE_TB:
+ he->data1 |= HE_BITS(DATA1_FORMAT_TRIG) |
+- HE_BITS(DATA1_SPTL_REUSE_KNOWN) |
+ HE_BITS(DATA1_SPTL_REUSE2_KNOWN) |
+ HE_BITS(DATA1_SPTL_REUSE3_KNOWN) |
+ HE_BITS(DATA1_SPTL_REUSE4_KNOWN);
+@@ -338,10 +383,17 @@ int mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb)
+ if (rxd2 & MT_RXD2_NORMAL_AMSDU_ERR)
+ return -EINVAL;
+
++ hdr_trans = rxd2 & MT_RXD2_NORMAL_HDR_TRANS;
++ if (hdr_trans && (rxd1 & MT_RXD1_NORMAL_CM))
++ return -EINVAL;
++
++ /* ICV error or CCMP/BIP/WPI MIC error */
++ if (rxd1 & MT_RXD1_NORMAL_ICV_ERR)
++ status->flag |= RX_FLAG_ONLY_MONITOR;
++
+ chfreq = FIELD_GET(MT_RXD3_NORMAL_CH_FREQ, rxd3);
+ unicast = FIELD_GET(MT_RXD3_NORMAL_ADDR_TYPE, rxd3) == MT_RXD3_NORMAL_U2M;
+ idx = FIELD_GET(MT_RXD1_NORMAL_WLAN_IDX, rxd1);
+- hdr_trans = rxd2 & MT_RXD2_NORMAL_HDR_TRANS;
+ status->wcid = mt7921_rx_get_wcid(dev, idx, unicast);
+
+ if (status->wcid) {
+@@ -606,9 +658,13 @@ int mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb)
+
+ mt7921_mac_assoc_rssi(dev, skb);
+
+- if (rxv && status->flag & RX_FLAG_RADIOTAP_HE)
++ if (rxv && status->flag & RX_FLAG_RADIOTAP_HE) {
+ mt7921_mac_decode_he_radiotap(skb, status, rxv, mode);
+
++ if (status->flag & RX_FLAG_RADIOTAP_HE_MU)
++ mt7921_mac_decode_he_mu_radiotap(skb, status, rxv);
++ }
++
+ if (!status->wcid || !ieee80211_is_data_qos(fc))
+ return 0;
+
+@@ -735,8 +791,9 @@ mt7921_mac_write_txwi_80211(struct mt7921_dev *dev, __le32 *txwi,
+ static void mt7921_update_txs(struct mt76_wcid *wcid, __le32 *txwi)
+ {
+ struct mt7921_sta *msta = container_of(wcid, struct mt7921_sta, wcid);
+- u32 pid, frame_type = FIELD_GET(MT_TXD2_FRAME_TYPE, txwi[2]);
++ u32 pid, frame_type;
+
++ frame_type = FIELD_GET(MT_TXD2_FRAME_TYPE, le32_to_cpu(txwi[2]));
+ if (!(frame_type & (IEEE80211_FTYPE_DATA >> 2)))
+ return;
+
+@@ -1493,6 +1550,14 @@ void mt7921_pm_power_save_work(struct work_struct *work)
+ test_bit(MT76_HW_SCHED_SCANNING, &mphy->state))
+ goto out;
+
++ if (mutex_is_locked(&dev->mt76.mutex))
++ /* if mt76 mutex is held we should not put the device
++ * to sleep since we are currently accessing device
++ * register map. We need to wait for the next power_save
++ * trigger.
++ */
++ goto out;
++
+ if (time_is_after_jiffies(dev->pm.last_activity + delta)) {
+ delta = dev->pm.last_activity + delta - jiffies;
+ goto out;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h b/drivers/net/wireless/mediatek/mt76/mt7921/mac.h
+index 3af67fac213df..f0194c8780372 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.h
+@@ -116,6 +116,7 @@ enum rx_pkt_type {
+ #define MT_PRXV_TX_DCM BIT(4)
+ #define MT_PRXV_TX_ER_SU_106T BIT(5)
+ #define MT_PRXV_NSTS GENMASK(9, 7)
++#define MT_PRXV_TXBF BIT(10)
+ #define MT_PRXV_HT_AD_CODE BIT(11)
+ #define MT_PRXV_FRAME_MODE GENMASK(14, 12)
+ #define MT_PRXV_SGI GENMASK(16, 15)
+@@ -138,8 +139,15 @@ enum rx_pkt_type {
+ #define MT_CRXV_HE_LTF_SIZE GENMASK(18, 17)
+ #define MT_CRXV_HE_LDPC_EXT_SYM BIT(20)
+ #define MT_CRXV_HE_PE_DISAMBIG BIT(23)
++#define MT_CRXV_HE_NUM_USER GENMASK(30, 24)
+ #define MT_CRXV_HE_UPLINK BIT(31)
+
++#define MT_CRXV_HE_RU0 GENMASK(7, 0)
++#define MT_CRXV_HE_RU1 GENMASK(15, 8)
++#define MT_CRXV_HE_RU2 GENMASK(23, 16)
++#define MT_CRXV_HE_RU3 GENMASK(31, 24)
++#define MT_CRXV_HE_MU_AID GENMASK(30, 20)
++
+ #define MT_CRXV_HE_SR_MASK GENMASK(11, 8)
+ #define MT_CRXV_HE_SR1_MASK GENMASK(16, 12)
+ #define MT_CRXV_HE_SR2_MASK GENMASK(20, 17)
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
+index 63ec140c9c372..30252f408ddc4 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
+@@ -224,6 +224,7 @@ static void mt7921_stop(struct ieee80211_hw *hw)
+
+ cancel_delayed_work_sync(&dev->pm.ps_work);
+ cancel_work_sync(&dev->pm.wake_work);
++ cancel_work_sync(&dev->reset_work);
+ mt76_connac_free_pending_tx_skbs(&dev->pm, NULL);
+
+ mt7921_mutex_acquire(dev);
+@@ -285,12 +286,6 @@ static int mt7921_add_interface(struct ieee80211_hw *hw,
+ mtxq->wcid = &mvif->sta.wcid;
+ }
+
+- if (vif->type != NL80211_IFTYPE_AP &&
+- (!mvif->mt76.omac_idx || mvif->mt76.omac_idx > 3))
+- vif->offload_flags = 0;
+-
+- vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
+-
+ out:
+ mt7921_mutex_release(dev);
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
+index 9fbaacc67cfad..4119f8efd896b 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
+@@ -157,6 +157,7 @@ mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
+ struct sk_buff *skb, int seq)
+ {
+ struct mt7921_mcu_rxd *rxd;
++ int mcu_cmd = cmd & MCU_CMD_MASK;
+ int ret = 0;
+
+ if (!skb) {
+@@ -194,6 +195,9 @@ mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
+ skb_pull(skb, sizeof(*rxd));
+ event = (struct mt7921_mcu_uni_event *)skb->data;
+ ret = le32_to_cpu(event->status);
++ /* skip invalid event */
++ if (mcu_cmd != event->cid)
++ ret = -EAGAIN;
+ break;
+ }
+ case MCU_CMD_REG_READ: {
+@@ -316,11 +320,13 @@ mt7921_mcu_tx_rate_parse(struct mt76_phy *mphy,
+ struct rate_info *rate, u16 r)
+ {
+ struct ieee80211_supported_band *sband;
+- u16 flags = 0;
++ u16 flags = 0, rate_idx;
+ u8 txmode = FIELD_GET(MT_WTBL_RATE_TX_MODE, r);
+ u8 gi = 0;
+ u8 bw = 0;
++ bool cck = false;
+
++ memset(rate, 0, sizeof(*rate));
+ rate->mcs = FIELD_GET(MT_WTBL_RATE_MCS, r);
+ rate->nss = FIELD_GET(MT_WTBL_RATE_NSS, r) + 1;
+
+@@ -345,13 +351,18 @@ mt7921_mcu_tx_rate_parse(struct mt76_phy *mphy,
+
+ switch (txmode) {
+ case MT_PHY_TYPE_CCK:
++ cck = true;
++ fallthrough;
+ case MT_PHY_TYPE_OFDM:
+ if (mphy->chandef.chan->band == NL80211_BAND_5GHZ)
+ sband = &mphy->sband_5g.sband;
+ else
+ sband = &mphy->sband_2g.sband;
+
+- rate->legacy = sband->bitrates[rate->mcs].bitrate;
++ rate_idx = FIELD_GET(MT_TX_RATE_IDX, r);
++ rate_idx = mt76_get_rate(mphy->dev, sband, rate_idx,
++ cck);
++ rate->legacy = sband->bitrates[rate_idx].bitrate;
+ break;
+ case MT_PHY_TYPE_HT:
+ case MT_PHY_TYPE_HT_GF:
+@@ -532,7 +543,8 @@ mt7921_mcu_tx_done_event(struct mt7921_dev *dev, struct sk_buff *skb)
+ peer.g8 = !!(sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80);
+ peer.g16 = !!(sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_160);
+ mt7921_mcu_tx_rate_parse(mphy->mt76, &peer,
+- &msta->stats.tx_rate, event->tx_rate);
++ &msta->stats.tx_rate,
++ le16_to_cpu(event->tx_rate));
+
+ spin_lock_bh(&dev->sta_poll_lock);
+ break;
+@@ -619,7 +631,7 @@ mt7921_mcu_sta_key_tlv(struct mt7921_sta *msta, struct sk_buff *skb,
+ u8 cipher;
+
+ cipher = mt7921_mcu_get_cipher(key->cipher);
+- if (cipher == MT_CIPHER_NONE)
++ if (cipher == MCU_CIPHER_NONE)
+ return -EOPNOTSUPP;
+
+ sec_key = &sec->key[0];
+@@ -815,7 +827,7 @@ out:
+ default:
+ ret = -EAGAIN;
+ dev_err(dev->mt76.dev, "Failed to release patch semaphore\n");
+- goto out;
++ break;
+ }
+ release_firmware(fw);
+
+@@ -1294,8 +1306,6 @@ int mt7921_mcu_sta_update(struct mt7921_dev *dev, struct ieee80211_sta *sta,
+
+ int __mt7921_mcu_drv_pmctrl(struct mt7921_dev *dev)
+ {
+- struct mt76_phy *mphy = &dev->mt76.phy;
+- struct mt76_connac_pm *pm = &dev->pm;
+ int i, err = 0;
+
+ for (i = 0; i < MT7921_DRV_OWN_RETRY_COUNT; i++) {
+@@ -1308,16 +1318,8 @@ int __mt7921_mcu_drv_pmctrl(struct mt7921_dev *dev)
+ if (i == MT7921_DRV_OWN_RETRY_COUNT) {
+ dev_err(dev->mt76.dev, "driver own failed\n");
+ err = -EIO;
+- goto out;
+ }
+
+- mt7921_wpdma_reinit_cond(dev);
+- clear_bit(MT76_STATE_PM, &mphy->state);
+-
+- pm->stats.last_wake_event = jiffies;
+- pm->stats.doze_time += pm->stats.last_wake_event -
+- pm->stats.last_doze_event;
+-out:
+ return err;
+ }
+
+@@ -1333,6 +1335,16 @@ int mt7921_mcu_drv_pmctrl(struct mt7921_dev *dev)
+ goto out;
+
+ err = __mt7921_mcu_drv_pmctrl(dev);
++ if (err < 0)
++ goto out;
++
++ mt7921_wpdma_reinit_cond(dev);
++ clear_bit(MT76_STATE_PM, &mphy->state);
++
++ pm->stats.last_wake_event = jiffies;
++ pm->stats.doze_time += pm->stats.last_wake_event -
++ pm->stats.last_doze_event;
++
+ out:
+ mutex_unlock(&pm->mutex);
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
+index de3c091f67368..42e7271848956 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
+@@ -296,11 +296,11 @@ struct mt7921_txpwr_event {
+ struct mt7921_mcu_tx_done_event {
+ u8 pid;
+ u8 status;
+- u16 seq;
++ __le16 seq;
+
+ u8 wlan_idx;
+ u8 tx_cnt;
+- u16 tx_rate;
++ __le16 tx_rate;
+
+ u8 flag;
+ u8 tid;
+@@ -312,9 +312,9 @@ struct mt7921_mcu_tx_done_event {
+ u8 reason;
+ u8 rsv0[1];
+
+- u32 delay;
+- u32 timestamp;
+- u32 applied_flag;
++ __le32 delay;
++ __le32 timestamp;
++ __le32 applied_flag;
+
+ u8 txs[28];
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
+index c3905bcab3604..7d9b23a002388 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
+@@ -88,6 +88,110 @@ static void mt7921_irq_tasklet(unsigned long data)
+ napi_schedule(&dev->mt76.napi[MT_RXQ_MAIN]);
+ }
+
++static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr)
++{
++ static const struct {
++ u32 phys;
++ u32 mapped;
++ u32 size;
++ } fixed_map[] = {
++ { 0x00400000, 0x80000, 0x10000}, /* WF_MCU_SYSRAM */
++ { 0x00410000, 0x90000, 0x10000}, /* WF_MCU_SYSRAM (configure register) */
++ { 0x40000000, 0x70000, 0x10000}, /* WF_UMAC_SYSRAM */
++ { 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */
++ { 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */
++ { 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
++ { 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */
++ { 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */
++ { 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */
++ { 0x7c060000, 0xe0000, 0x10000}, /* CONN_INFRA, conn_host_csr_top */
++ { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */
++ { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */
++ { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
++ { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
++ { 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */
++ { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
++ { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
++ { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
++ { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
++ { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
++ { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
++ { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
++ { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
++ { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
++ { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
++ { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
++ { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
++ { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
++ { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
++ { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
++ { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
++ { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
++ { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
++ { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
++ { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
++ { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
++ { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
++ { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
++ { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
++ { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
++ { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
++ { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
++ };
++ int i;
++
++ if (addr < 0x100000)
++ return addr;
++
++ for (i = 0; i < ARRAY_SIZE(fixed_map); i++) {
++ u32 ofs;
++
++ if (addr < fixed_map[i].phys)
++ continue;
++
++ ofs = addr - fixed_map[i].phys;
++ if (ofs > fixed_map[i].size)
++ continue;
++
++ return fixed_map[i].mapped + ofs;
++ }
++
++ if ((addr >= 0x18000000 && addr < 0x18c00000) ||
++ (addr >= 0x70000000 && addr < 0x78000000) ||
++ (addr >= 0x7c000000 && addr < 0x7c400000))
++ return mt7921_reg_map_l1(dev, addr);
++
++ dev_err(dev->mt76.dev, "Access currently unsupported address %08x\n",
++ addr);
++
++ return 0;
++}
++
++static u32 mt7921_rr(struct mt76_dev *mdev, u32 offset)
++{
++ struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
++ u32 addr = __mt7921_reg_addr(dev, offset);
++
++ return dev->bus_ops->rr(mdev, addr);
++}
++
++static void mt7921_wr(struct mt76_dev *mdev, u32 offset, u32 val)
++{
++ struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
++ u32 addr = __mt7921_reg_addr(dev, offset);
++
++ dev->bus_ops->wr(mdev, addr, val);
++}
++
++static u32 mt7921_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val)
++{
++ struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
++ u32 addr = __mt7921_reg_addr(dev, offset);
++
++ return dev->bus_ops->rmw(mdev, addr, mask, val);
++}
++
++
+ static int mt7921_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+ {
+@@ -110,6 +214,7 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
+ .sta_remove = mt7921_mac_sta_remove,
+ .update_survey = mt7921_update_channel,
+ };
++ struct mt76_bus_ops *bus_ops;
+ struct mt7921_dev *dev;
+ struct mt76_dev *mdev;
+ int ret;
+@@ -145,6 +250,22 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
+
+ mt76_mmio_init(&dev->mt76, pcim_iomap_table(pdev)[0]);
+ tasklet_init(&dev->irq_tasklet, mt7921_irq_tasklet, (unsigned long)dev);
++
++ dev->bus_ops = dev->mt76.bus;
++ bus_ops = devm_kmemdup(dev->mt76.dev, dev->bus_ops, sizeof(*bus_ops),
++ GFP_KERNEL);
++ if (!bus_ops)
++ return -ENOMEM;
++
++ bus_ops->rr = mt7921_rr;
++ bus_ops->wr = mt7921_wr;
++ bus_ops->rmw = mt7921_rmw;
++ dev->mt76.bus = bus_ops;
++
++ ret = __mt7921_mcu_drv_pmctrl(dev);
++ if (ret)
++ return ret;
++
+ mdev->rev = (mt7921_l1_rr(dev, MT_HW_CHIPID) << 16) |
+ (mt7921_l1_rr(dev, MT_HW_REV) & 0xff);
+ dev_err(mdev->dev, "ASIC revision: %04x\n", mdev->rev);
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
+index b6944c867a573..41c2855e7a3d3 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
+@@ -17,13 +17,12 @@
+ #define MT_PLE_BASE 0x8000
+ #define MT_PLE(ofs) (MT_PLE_BASE + (ofs))
+
+-#define MT_PLE_FL_Q0_CTRL MT_PLE(0x1b0)
+-#define MT_PLE_FL_Q1_CTRL MT_PLE(0x1b4)
+-#define MT_PLE_FL_Q2_CTRL MT_PLE(0x1b8)
+-#define MT_PLE_FL_Q3_CTRL MT_PLE(0x1bc)
++#define MT_PLE_FL_Q0_CTRL MT_PLE(0x3e0)
++#define MT_PLE_FL_Q1_CTRL MT_PLE(0x3e4)
++#define MT_PLE_FL_Q2_CTRL MT_PLE(0x3e8)
++#define MT_PLE_FL_Q3_CTRL MT_PLE(0x3ec)
+
+-#define MT_PLE_AC_QEMPTY(ac, n) MT_PLE(0x300 + 0x10 * (ac) + \
+- ((n) << 2))
++#define MT_PLE_AC_QEMPTY(_n) MT_PLE(0x500 + 0x40 * (_n))
+ #define MT_PLE_AMSDU_PACK_MSDU_CNT(n) MT_PLE(0x10e0 + ((n) << 2))
+
+ #define MT_MDP_BASE 0xf000
+@@ -96,6 +95,10 @@
+ #define MT_WF_MIB_BASE(_band) ((_band) ? 0xa4800 : 0x24800)
+ #define MT_WF_MIB(_band, ofs) (MT_WF_MIB_BASE(_band) + (ofs))
+
++#define MT_MIB_SCR1(_band) MT_WF_MIB(_band, 0x004)
++#define MT_MIB_TXDUR_EN BIT(8)
++#define MT_MIB_RXDUR_EN BIT(9)
++
+ #define MT_MIB_SDR3(_band) MT_WF_MIB(_band, 0x698)
+ #define MT_MIB_SDR3_FCS_ERR_MASK GENMASK(31, 16)
+
+@@ -108,9 +111,9 @@
+ #define MT_MIB_SDR34(_band) MT_WF_MIB(_band, 0x090)
+ #define MT_MIB_MU_BF_TX_CNT GENMASK(15, 0)
+
+-#define MT_MIB_SDR36(_band) MT_WF_MIB(_band, 0x098)
++#define MT_MIB_SDR36(_band) MT_WF_MIB(_band, 0x054)
+ #define MT_MIB_SDR36_TXTIME_MASK GENMASK(23, 0)
+-#define MT_MIB_SDR37(_band) MT_WF_MIB(_band, 0x09c)
++#define MT_MIB_SDR37(_band) MT_WF_MIB(_band, 0x058)
+ #define MT_MIB_SDR37_RXTIME_MASK GENMASK(23, 0)
+
+ #define MT_MIB_DR8(_band) MT_WF_MIB(_band, 0x0c0)
+diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+index 96973ec7bd9ac..87c14969c75fa 100644
+--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
++++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+@@ -129,8 +129,7 @@ static void cfg_scan_result(enum scan_event scan_event,
+ info->frame_len,
+ (s32)info->rssi * 100,
+ GFP_KERNEL);
+- if (!bss)
+- cfg80211_put_bss(wiphy, bss);
++ cfg80211_put_bss(wiphy, bss);
+ } else if (scan_event == SCAN_EVENT_DONE) {
+ mutex_lock(&priv->scan_req_lock);
+
+diff --git a/drivers/net/wireless/microchip/wilc1000/netdev.c b/drivers/net/wireless/microchip/wilc1000/netdev.c
+index 7e4d9235251cb..9dfb1a285e6a4 100644
+--- a/drivers/net/wireless/microchip/wilc1000/netdev.c
++++ b/drivers/net/wireless/microchip/wilc1000/netdev.c
+@@ -901,7 +901,6 @@ void wilc_netdev_cleanup(struct wilc *wilc)
+
+ wilc_wlan_cfg_deinit(wilc);
+ wlan_deinit_locks(wilc);
+- kfree(wilc->bus_data);
+ wiphy_unregister(wilc->wiphy);
+ wiphy_free(wilc->wiphy);
+ }
+diff --git a/drivers/net/wireless/microchip/wilc1000/sdio.c b/drivers/net/wireless/microchip/wilc1000/sdio.c
+index 42e03a701ae16..8b3b735231085 100644
+--- a/drivers/net/wireless/microchip/wilc1000/sdio.c
++++ b/drivers/net/wireless/microchip/wilc1000/sdio.c
+@@ -167,9 +167,11 @@ free:
+ static void wilc_sdio_remove(struct sdio_func *func)
+ {
+ struct wilc *wilc = sdio_get_drvdata(func);
++ struct wilc_sdio *sdio_priv = wilc->bus_data;
+
+ clk_disable_unprepare(wilc->rtc_clk);
+ wilc_netdev_cleanup(wilc);
++ kfree(sdio_priv);
+ }
+
+ static int wilc_sdio_reset(struct wilc *wilc)
+diff --git a/drivers/net/wireless/microchip/wilc1000/spi.c b/drivers/net/wireless/microchip/wilc1000/spi.c
+index dd481dc0b5ce0..c98c0999a6b67 100644
+--- a/drivers/net/wireless/microchip/wilc1000/spi.c
++++ b/drivers/net/wireless/microchip/wilc1000/spi.c
+@@ -182,9 +182,11 @@ free:
+ static int wilc_bus_remove(struct spi_device *spi)
+ {
+ struct wilc *wilc = spi_get_drvdata(spi);
++ struct wilc_spi *spi_priv = wilc->bus_data;
+
+ clk_disable_unprepare(wilc->rtc_clk);
+ wilc_netdev_cleanup(wilc);
++ kfree(spi_priv);
+
+ return 0;
+ }
+diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+index e4473a5512415..74c3d8cb31002 100644
+--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
++++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+@@ -25,6 +25,9 @@ static bool rt2x00usb_check_usb_error(struct rt2x00_dev *rt2x00dev, int status)
+ if (status == -ENODEV || status == -ENOENT)
+ return true;
+
++ if (!test_bit(DEVICE_STATE_STARTED, &rt2x00dev->flags))
++ return false;
++
+ if (status == -EPROTO || status == -ETIMEDOUT)
+ rt2x00dev->num_proto_errs++;
+ else
+diff --git a/drivers/net/wireless/ray_cs.c b/drivers/net/wireless/ray_cs.c
+index 0f5009c47cd0a..f8409e93fe33e 100644
+--- a/drivers/net/wireless/ray_cs.c
++++ b/drivers/net/wireless/ray_cs.c
+@@ -382,6 +382,8 @@ static int ray_config(struct pcmcia_device *link)
+ goto failed;
+ local->sram = ioremap(link->resource[2]->start,
+ resource_size(link->resource[2]));
++ if (!local->sram)
++ goto failed;
+
+ /*** Set up 16k window for shared memory (receive buffer) ***************/
+ link->resource[3]->flags |=
+@@ -396,6 +398,8 @@ static int ray_config(struct pcmcia_device *link)
+ goto failed;
+ local->rmem = ioremap(link->resource[3]->start,
+ resource_size(link->resource[3]));
++ if (!local->rmem)
++ goto failed;
+
+ /*** Set up window for attribute memory ***********************************/
+ link->resource[4]->flags |=
+@@ -410,6 +414,8 @@ static int ray_config(struct pcmcia_device *link)
+ goto failed;
+ local->amem = ioremap(link->resource[4]->start,
+ resource_size(link->resource[4]));
++ if (!local->amem)
++ goto failed;
+
+ dev_dbg(&link->dev, "ray_config sram=%p\n", local->sram);
+ dev_dbg(&link->dev, "ray_config rmem=%p\n", local->rmem);
+diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
+index 585784258c665..4efab907a3ac6 100644
+--- a/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
++++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
+@@ -28,7 +28,7 @@ u8 rtl818x_ioread8_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_rcvctrlpipe(priv->udev, 0),
+ RTL8187_REQ_GET_REG, RTL8187_REQT_READ,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits8, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits8, sizeof(val), 500);
+
+ val = priv->io_dmabuf->bits8;
+ mutex_unlock(&priv->io_mutex);
+@@ -45,7 +45,7 @@ u16 rtl818x_ioread16_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_rcvctrlpipe(priv->udev, 0),
+ RTL8187_REQ_GET_REG, RTL8187_REQT_READ,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits16, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits16, sizeof(val), 500);
+
+ val = priv->io_dmabuf->bits16;
+ mutex_unlock(&priv->io_mutex);
+@@ -62,7 +62,7 @@ u32 rtl818x_ioread32_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_rcvctrlpipe(priv->udev, 0),
+ RTL8187_REQ_GET_REG, RTL8187_REQT_READ,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits32, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits32, sizeof(val), 500);
+
+ val = priv->io_dmabuf->bits32;
+ mutex_unlock(&priv->io_mutex);
+@@ -79,7 +79,7 @@ void rtl818x_iowrite8_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_sndctrlpipe(priv->udev, 0),
+ RTL8187_REQ_SET_REG, RTL8187_REQT_WRITE,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits8, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits8, sizeof(val), 500);
+
+ mutex_unlock(&priv->io_mutex);
+ }
+@@ -93,7 +93,7 @@ void rtl818x_iowrite16_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_sndctrlpipe(priv->udev, 0),
+ RTL8187_REQ_SET_REG, RTL8187_REQT_WRITE,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits16, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits16, sizeof(val), 500);
+
+ mutex_unlock(&priv->io_mutex);
+ }
+@@ -107,7 +107,7 @@ void rtl818x_iowrite32_idx(struct rtl8187_priv *priv,
+ usb_control_msg(priv->udev, usb_sndctrlpipe(priv->udev, 0),
+ RTL8187_REQ_SET_REG, RTL8187_REQT_WRITE,
+ (unsigned long)addr, idx & 0x03,
+- &priv->io_dmabuf->bits32, sizeof(val), HZ / 2);
++ &priv->io_dmabuf->bits32, sizeof(val), 500);
+
+ mutex_unlock(&priv->io_mutex);
+ }
+@@ -183,7 +183,7 @@ static void rtl8225_write_8051(struct ieee80211_hw *dev, u8 addr, __le16 data)
+ usb_control_msg(priv->udev, usb_sndctrlpipe(priv->udev, 0),
+ RTL8187_REQ_SET_REG, RTL8187_REQT_WRITE,
+ addr, 0x8225, &priv->io_dmabuf->bits16, sizeof(data),
+- HZ / 2);
++ 500);
+
+ mutex_unlock(&priv->io_mutex);
+
+diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
+index 6312fddd9c00a..eaba661133280 100644
+--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
+@@ -1000,6 +1000,7 @@ int rtl92cu_hw_init(struct ieee80211_hw *hw)
+ _initpabias(hw);
+ rtl92c_dm_init(hw);
+ exit:
++ local_irq_disable();
+ local_irq_restore(flags);
+ return err;
+ }
+diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c
+index e6399519584bd..a384fc3a4f2b0 100644
+--- a/drivers/net/wireless/realtek/rtw88/fw.c
++++ b/drivers/net/wireless/realtek/rtw88/fw.c
+@@ -1556,12 +1556,10 @@ static void rtw_fw_read_fifo_page(struct rtw_dev *rtwdev, u32 offset, u32 size,
+ u32 i;
+ u16 idx = 0;
+ u16 ctl;
+- u8 rcr;
+
+- rcr = rtw_read8(rtwdev, REG_RCR + 2);
+ ctl = rtw_read16(rtwdev, REG_PKTBUF_DBG_CTRL) & 0xf000;
+ /* disable rx clock gate */
+- rtw_write8(rtwdev, REG_RCR, rcr | BIT(3));
++ rtw_write32_set(rtwdev, REG_RCR, BIT_DISGCLK);
+
+ do {
+ rtw_write16(rtwdev, REG_PKTBUF_DBG_CTRL, start_pg | ctl);
+@@ -1580,7 +1578,8 @@ static void rtw_fw_read_fifo_page(struct rtw_dev *rtwdev, u32 offset, u32 size,
+
+ out:
+ rtw_write16(rtwdev, REG_PKTBUF_DBG_CTRL, ctl);
+- rtw_write8(rtwdev, REG_RCR + 2, rcr);
++ /* restore rx clock gate */
++ rtw_write32_clr(rtwdev, REG_RCR, BIT_DISGCLK);
+ }
+
+ static void rtw_fw_read_fifo(struct rtw_dev *rtwdev, enum rtw_fw_fifo_sel sel,
+diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c
+index 6bb55e663fc36..69512856bb462 100644
+--- a/drivers/net/wireless/realtek/rtw88/main.c
++++ b/drivers/net/wireless/realtek/rtw88/main.c
+@@ -1859,7 +1859,7 @@ int rtw_core_init(struct rtw_dev *rtwdev)
+
+ /* default rx filter setting */
+ rtwdev->hal.rcr = BIT_APP_FCS | BIT_APP_MIC | BIT_APP_ICV |
+- BIT_HTC_LOC_CTRL | BIT_APP_PHYSTS |
++ BIT_PKTCTL_DLEN | BIT_HTC_LOC_CTRL | BIT_APP_PHYSTS |
+ BIT_AB | BIT_AM | BIT_APM;
+
+ ret = rtw_load_firmware(rtwdev, RTW_NORMAL_FW);
+diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c
+index a7a6ebfaa203c..08cf66141889b 100644
+--- a/drivers/net/wireless/realtek/rtw88/pci.c
++++ b/drivers/net/wireless/realtek/rtw88/pci.c
+@@ -2,7 +2,6 @@
+ /* Copyright(c) 2018-2019 Realtek Corporation
+ */
+
+-#include <linux/dmi.h>
+ #include <linux/module.h>
+ #include <linux/pci.h>
+ #include "main.h"
+@@ -1409,7 +1408,11 @@ static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
+ * throughput. This is probably because the ASPM behavior slightly
+ * varies from different SOC.
+ */
+- if (rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
++ if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))
++ return;
++
++ if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||
++ (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))
+ rtw_pci_aspm_set(rtwdev, enter);
+ }
+
+@@ -1658,6 +1661,9 @@ static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
+ priv);
+ int work_done = 0;
+
++ if (rtwpci->rx_no_aspm)
++ rtw_pci_link_ps(rtwdev, false);
++
+ while (work_done < budget) {
+ u32 work_done_once;
+
+@@ -1681,6 +1687,8 @@ static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
+ if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))
+ napi_schedule(napi);
+ }
++ if (rtwpci->rx_no_aspm)
++ rtw_pci_link_ps(rtwdev, true);
+
+ return work_done;
+ }
+@@ -1702,50 +1710,13 @@ static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
+ netif_napi_del(&rtwpci->napi);
+ }
+
+-enum rtw88_quirk_dis_pci_caps {
+- QUIRK_DIS_PCI_CAP_MSI,
+- QUIRK_DIS_PCI_CAP_ASPM,
+-};
+-
+-static int disable_pci_caps(const struct dmi_system_id *dmi)
+-{
+- uintptr_t dis_caps = (uintptr_t)dmi->driver_data;
+-
+- if (dis_caps & BIT(QUIRK_DIS_PCI_CAP_MSI))
+- rtw_disable_msi = true;
+- if (dis_caps & BIT(QUIRK_DIS_PCI_CAP_ASPM))
+- rtw_pci_disable_aspm = true;
+-
+- return 1;
+-}
+-
+-static const struct dmi_system_id rtw88_pci_quirks[] = {
+- {
+- .callback = disable_pci_caps,
+- .ident = "Protempo Ltd L116HTN6SPW",
+- .matches = {
+- DMI_MATCH(DMI_SYS_VENDOR, "Protempo Ltd"),
+- DMI_MATCH(DMI_PRODUCT_NAME, "L116HTN6SPW"),
+- },
+- .driver_data = (void *)BIT(QUIRK_DIS_PCI_CAP_ASPM),
+- },
+- {
+- .callback = disable_pci_caps,
+- .ident = "HP HP Pavilion Laptop 14-ce0xxx",
+- .matches = {
+- DMI_MATCH(DMI_SYS_VENDOR, "HP"),
+- DMI_MATCH(DMI_PRODUCT_NAME, "HP Pavilion Laptop 14-ce0xxx"),
+- },
+- .driver_data = (void *)BIT(QUIRK_DIS_PCI_CAP_ASPM),
+- },
+- {}
+-};
+-
+ int rtw_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+ {
++ struct pci_dev *bridge = pci_upstream_bridge(pdev);
+ struct ieee80211_hw *hw;
+ struct rtw_dev *rtwdev;
++ struct rtw_pci *rtwpci;
+ int drv_data_size;
+ int ret;
+
+@@ -1763,6 +1734,9 @@ int rtw_pci_probe(struct pci_dev *pdev,
+ rtwdev->hci.ops = &rtw_pci_ops;
+ rtwdev->hci.type = RTW_HCI_TYPE_PCIE;
+
++ rtwpci = (struct rtw_pci *)rtwdev->priv;
++ atomic_set(&rtwpci->link_usage, 1);
++
+ ret = rtw_core_init(rtwdev);
+ if (ret)
+ goto err_release_hw;
+@@ -1791,7 +1765,10 @@ int rtw_pci_probe(struct pci_dev *pdev,
+ goto err_destroy_pci;
+ }
+
+- dmi_check_system(rtw88_pci_quirks);
++ /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */
++ if (pdev->device == 0xc821 && bridge->vendor == PCI_VENDOR_ID_INTEL)
++ rtwpci->rx_no_aspm = true;
++
+ rtw_pci_phy_cfg(rtwdev);
+
+ ret = rtw_register_hw(rtwdev, hw);
+diff --git a/drivers/net/wireless/realtek/rtw88/pci.h b/drivers/net/wireless/realtek/rtw88/pci.h
+index 66f78eb7757c5..0c37efd8c66fa 100644
+--- a/drivers/net/wireless/realtek/rtw88/pci.h
++++ b/drivers/net/wireless/realtek/rtw88/pci.h
+@@ -223,6 +223,8 @@ struct rtw_pci {
+ struct rtw_pci_tx_ring tx_rings[RTK_MAX_TX_QUEUE_NUM];
+ struct rtw_pci_rx_ring rx_rings[RTK_MAX_RX_QUEUE_NUM];
+ u16 link_ctrl;
++ atomic_t link_usage;
++ bool rx_no_aspm;
+ DECLARE_BITMAP(flags, NUM_OF_RTW_PCI_FLAGS);
+
+ void __iomem *mmap;
+diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h
+index f5ce75095e904..c0fb1e446245f 100644
+--- a/drivers/net/wireless/realtek/rtw88/reg.h
++++ b/drivers/net/wireless/realtek/rtw88/reg.h
+@@ -406,6 +406,7 @@
+ #define BIT_MFBEN BIT(22)
+ #define BIT_DISCHKPPDLLEN BIT(21)
+ #define BIT_PKTCTL_DLEN BIT(20)
++#define BIT_DISGCLK BIT(19)
+ #define BIT_TIM_PARSER_EN BIT(18)
+ #define BIT_BC_MD_EN BIT(17)
+ #define BIT_UC_MD_EN BIT(16)
+diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.h b/drivers/net/wireless/realtek/rtw88/rtw8821c.h
+index 112faa60f653e..d9fbddd7b0f35 100644
+--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.h
++++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.h
+@@ -131,7 +131,7 @@ _rtw_write32s_mask(struct rtw_dev *rtwdev, u32 addr, u32 mask, u32 data)
+ #define WLAN_TX_FUNC_CFG2 0x30
+ #define WLAN_MAC_OPT_NORM_FUNC1 0x98
+ #define WLAN_MAC_OPT_LB_FUNC1 0x80
+-#define WLAN_MAC_OPT_FUNC2 0x30810041
++#define WLAN_MAC_OPT_FUNC2 0xb0810041
+
+ #define WLAN_SIFS_CFG (WLAN_SIFS_CCK_CONT_TX | \
+ (WLAN_SIFS_OFDM_CONT_TX << BIT_SHIFT_SIFS_OFDM_CTX) | \
+diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
+index f1789155e9016..247f26e3e8192 100644
+--- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c
++++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
+@@ -204,7 +204,7 @@ static void rtw8822b_phy_set_param(struct rtw_dev *rtwdev)
+ #define WLAN_TX_FUNC_CFG2 0x30
+ #define WLAN_MAC_OPT_NORM_FUNC1 0x98
+ #define WLAN_MAC_OPT_LB_FUNC1 0x80
+-#define WLAN_MAC_OPT_FUNC2 0x30810041
++#define WLAN_MAC_OPT_FUNC2 0xb0810041
+
+ #define WLAN_SIFS_CFG (WLAN_SIFS_CCK_CONT_TX | \
+ (WLAN_SIFS_OFDM_CONT_TX << BIT_SHIFT_SIFS_OFDM_CTX) | \
+diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
+index f3ad079967a68..bc87e3cb9cdce 100644
+--- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c
++++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
+@@ -1962,7 +1962,7 @@ static void rtw8822c_phy_set_param(struct rtw_dev *rtwdev)
+ #define WLAN_TX_FUNC_CFG2 0x30
+ #define WLAN_MAC_OPT_NORM_FUNC1 0x98
+ #define WLAN_MAC_OPT_LB_FUNC1 0x80
+-#define WLAN_MAC_OPT_FUNC2 0x30810041
++#define WLAN_MAC_OPT_FUNC2 0xb0810041
+ #define WLAN_MAC_INT_MIG_CFG 0x33330000
+
+ #define WLAN_SIFS_CFG (WLAN_SIFS_CCK_CONT_TX | \
+diff --git a/drivers/net/wireless/rsi/rsi_91x_core.c b/drivers/net/wireless/rsi/rsi_91x_core.c
+index a48e616e0fb91..6bfaab48b507d 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_core.c
++++ b/drivers/net/wireless/rsi/rsi_91x_core.c
+@@ -399,6 +399,8 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
+
+ info = IEEE80211_SKB_CB(skb);
+ tx_params = (struct skb_info *)info->driver_data;
++ /* info->driver_data and info->control part of union so make copy */
++ tx_params->have_key = !!info->control.hw_key;
+ wh = (struct ieee80211_hdr *)&skb->data[0];
+ tx_params->sta_id = 0;
+
+diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c
+index f4a26f16f00f4..dca81a4bbdd7f 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_hal.c
++++ b/drivers/net/wireless/rsi/rsi_91x_hal.c
+@@ -203,7 +203,7 @@ int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
+ wh->frame_control |= cpu_to_le16(RSI_SET_PS_ENABLE);
+
+ if ((!(info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT)) &&
+- info->control.hw_key) {
++ tx_params->have_key) {
+ if (rsi_is_cipher_wep(common))
+ ieee80211_size += 4;
+ else
+@@ -214,15 +214,17 @@ int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
+ RSI_WIFI_DATA_Q);
+ data_desc->header_len = ieee80211_size;
+
+- if (common->min_rate != RSI_RATE_AUTO) {
++ if (common->rate_config[common->band].fixed_enabled) {
+ /* Send fixed rate */
++ u16 fixed_rate = common->rate_config[common->band].fixed_hw_rate;
++
+ data_desc->frame_info = cpu_to_le16(RATE_INFO_ENABLE);
+- data_desc->rate_info = cpu_to_le16(common->min_rate);
++ data_desc->rate_info = cpu_to_le16(fixed_rate);
+
+ if (conf_is_ht40(&common->priv->hw->conf))
+ data_desc->bbp_info = cpu_to_le16(FULL40M_ENABLE);
+
+- if ((common->vif_info[0].sgi) && (common->min_rate & 0x100)) {
++ if (common->vif_info[0].sgi && (fixed_rate & 0x100)) {
+ /* Only MCS rates */
+ data_desc->rate_info |=
+ cpu_to_le16(ENABLE_SHORTGI_RATE);
+diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+index b66975f545675..e70c1c7fdf595 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c
++++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+@@ -510,7 +510,6 @@ static int rsi_mac80211_add_interface(struct ieee80211_hw *hw,
+ if ((vif->type == NL80211_IFTYPE_AP) ||
+ (vif->type == NL80211_IFTYPE_P2P_GO)) {
+ rsi_send_rx_filter_frame(common, DISALLOW_BEACONS);
+- common->min_rate = RSI_RATE_AUTO;
+ for (i = 0; i < common->max_stations; i++)
+ common->stations[i].sta = NULL;
+ }
+@@ -1228,20 +1227,32 @@ static int rsi_mac80211_set_rate_mask(struct ieee80211_hw *hw,
+ struct ieee80211_vif *vif,
+ const struct cfg80211_bitrate_mask *mask)
+ {
++ const unsigned int mcs_offset = ARRAY_SIZE(rsi_rates);
+ struct rsi_hw *adapter = hw->priv;
+ struct rsi_common *common = adapter->priv;
+- enum nl80211_band band = hw->conf.chandef.chan->band;
++ int i;
+
+ mutex_lock(&common->mutex);
+- common->fixedrate_mask[band] = 0;
+
+- if (mask->control[band].legacy == 0xfff) {
+- common->fixedrate_mask[band] =
+- (mask->control[band].ht_mcs[0] << 12);
+- } else {
+- common->fixedrate_mask[band] =
+- mask->control[band].legacy;
++ for (i = 0; i < ARRAY_SIZE(common->rate_config); i++) {
++ struct rsi_rate_config *cfg = &common->rate_config[i];
++ u32 bm;
++
++ bm = mask->control[i].legacy | (mask->control[i].ht_mcs[0] << mcs_offset);
++ if (hweight32(bm) == 1) { /* single rate */
++ int rate_index = ffs(bm) - 1;
++
++ if (rate_index < mcs_offset)
++ cfg->fixed_hw_rate = rsi_rates[rate_index].hw_value;
++ else
++ cfg->fixed_hw_rate = rsi_mcsrates[rate_index - mcs_offset];
++ cfg->fixed_enabled = true;
++ } else {
++ cfg->configured_mask = bm;
++ cfg->fixed_enabled = false;
++ }
+ }
++
+ mutex_unlock(&common->mutex);
+
+ return 0;
+@@ -1378,46 +1389,6 @@ void rsi_indicate_pkt_to_os(struct rsi_common *common,
+ ieee80211_rx_irqsafe(hw, skb);
+ }
+
+-static void rsi_set_min_rate(struct ieee80211_hw *hw,
+- struct ieee80211_sta *sta,
+- struct rsi_common *common)
+-{
+- u8 band = hw->conf.chandef.chan->band;
+- u8 ii;
+- u32 rate_bitmap;
+- bool matched = false;
+-
+- common->bitrate_mask[band] = sta->supp_rates[band];
+-
+- rate_bitmap = (common->fixedrate_mask[band] & sta->supp_rates[band]);
+-
+- if (rate_bitmap & 0xfff) {
+- /* Find out the min rate */
+- for (ii = 0; ii < ARRAY_SIZE(rsi_rates); ii++) {
+- if (rate_bitmap & BIT(ii)) {
+- common->min_rate = rsi_rates[ii].hw_value;
+- matched = true;
+- break;
+- }
+- }
+- }
+-
+- common->vif_info[0].is_ht = sta->ht_cap.ht_supported;
+-
+- if ((common->vif_info[0].is_ht) && (rate_bitmap >> 12)) {
+- for (ii = 0; ii < ARRAY_SIZE(rsi_mcsrates); ii++) {
+- if ((rate_bitmap >> 12) & BIT(ii)) {
+- common->min_rate = rsi_mcsrates[ii];
+- matched = true;
+- break;
+- }
+- }
+- }
+-
+- if (!matched)
+- common->min_rate = 0xffff;
+-}
+-
+ /**
+ * rsi_mac80211_sta_add() - This function notifies driver about a peer getting
+ * connected.
+@@ -1516,9 +1487,9 @@ static int rsi_mac80211_sta_add(struct ieee80211_hw *hw,
+
+ if ((vif->type == NL80211_IFTYPE_STATION) ||
+ (vif->type == NL80211_IFTYPE_P2P_CLIENT)) {
+- rsi_set_min_rate(hw, sta, common);
++ common->bitrate_mask[common->band] = sta->supp_rates[common->band];
++ common->vif_info[0].is_ht = sta->ht_cap.ht_supported;
+ if (sta->ht_cap.ht_supported) {
+- common->vif_info[0].is_ht = true;
+ common->bitrate_mask[NL80211_BAND_2GHZ] =
+ sta->supp_rates[NL80211_BAND_2GHZ];
+ if ((sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20) ||
+@@ -1592,7 +1563,6 @@ static int rsi_mac80211_sta_remove(struct ieee80211_hw *hw,
+ bss->qos = sta->wme;
+ common->bitrate_mask[NL80211_BAND_2GHZ] = 0;
+ common->bitrate_mask[NL80211_BAND_5GHZ] = 0;
+- common->min_rate = 0xffff;
+ common->vif_info[0].is_ht = false;
+ common->vif_info[0].sgi = false;
+ common->vif_info[0].seq_start = 0;
+diff --git a/drivers/net/wireless/rsi/rsi_91x_main.c b/drivers/net/wireless/rsi/rsi_91x_main.c
+index d98483298555c..5d1490fc32db4 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_main.c
++++ b/drivers/net/wireless/rsi/rsi_91x_main.c
+@@ -23,6 +23,7 @@
+ #include "rsi_common.h"
+ #include "rsi_coex.h"
+ #include "rsi_hal.h"
++#include "rsi_usb.h"
+
+ u32 rsi_zone_enabled = /* INFO_ZONE |
+ INIT_ZONE |
+@@ -168,6 +169,9 @@ int rsi_read_pkt(struct rsi_common *common, u8 *rx_pkt, s32 rcv_pkt_len)
+ frame_desc = &rx_pkt[index];
+ actual_length = *(u16 *)&frame_desc[0];
+ offset = *(u16 *)&frame_desc[2];
++ if (!rcv_pkt_len && offset >
++ RSI_MAX_RX_USB_PKT_SIZE - FRAME_DESC_SZ)
++ goto fail;
+
+ queueno = rsi_get_queueno(frame_desc, offset);
+ length = rsi_get_length(frame_desc, offset);
+@@ -211,9 +215,10 @@ int rsi_read_pkt(struct rsi_common *common, u8 *rx_pkt, s32 rcv_pkt_len)
+ bt_pkt_type = frame_desc[offset + BT_RX_PKT_TYPE_OFST];
+ if (bt_pkt_type == BT_CARD_READY_IND) {
+ rsi_dbg(INFO_ZONE, "BT Card ready recvd\n");
+- if (rsi_bt_ops.attach(common, &g_proto_ops))
+- rsi_dbg(ERR_ZONE,
+- "Failed to attach BT module\n");
++ if (common->fsm_state == FSM_MAC_INIT_DONE)
++ rsi_attach_bt(common);
++ else
++ common->bt_defer_attach = true;
+ } else {
+ if (common->bt_adapter)
+ rsi_bt_ops.recv_pkt(common->bt_adapter,
+@@ -278,6 +283,15 @@ void rsi_set_bt_context(void *priv, void *bt_context)
+ }
+ #endif
+
++void rsi_attach_bt(struct rsi_common *common)
++{
++#ifdef CONFIG_RSI_COEX
++ if (rsi_bt_ops.attach(common, &g_proto_ops))
++ rsi_dbg(ERR_ZONE,
++ "Failed to attach BT module\n");
++#endif
++}
++
+ /**
+ * rsi_91x_init() - This function initializes os interface operations.
+ * @oper_mode: One of DEV_OPMODE_*.
+@@ -359,6 +373,7 @@ struct rsi_hw *rsi_91x_init(u16 oper_mode)
+ if (common->coex_mode > 1) {
+ if (rsi_coex_attach(common)) {
+ rsi_dbg(ERR_ZONE, "Failed to init coex module\n");
++ rsi_kill_thread(&common->tx_thread);
+ goto err;
+ }
+ }
+diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+index 891fd5f0fa765..0848f7a7e76c6 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
++++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+@@ -276,7 +276,7 @@ static void rsi_set_default_parameters(struct rsi_common *common)
+ common->channel_width = BW_20MHZ;
+ common->rts_threshold = IEEE80211_MAX_RTS_THRESHOLD;
+ common->channel = 1;
+- common->min_rate = 0xffff;
++ memset(&common->rate_config, 0, sizeof(common->rate_config));
+ common->fsm_state = FSM_CARD_NOT_READY;
+ common->iface_down = true;
+ common->endpoint = EP_2GHZ_20MHZ;
+@@ -1314,7 +1314,7 @@ static int rsi_send_auto_rate_request(struct rsi_common *common,
+ u8 band = hw->conf.chandef.chan->band;
+ u8 num_supported_rates = 0;
+ u8 rate_table_offset, rate_offset = 0;
+- u32 rate_bitmap;
++ u32 rate_bitmap, configured_rates;
+ u16 *selected_rates, min_rate;
+ bool is_ht = false, is_sgi = false;
+ u16 frame_len = sizeof(struct rsi_auto_rate);
+@@ -1364,6 +1364,10 @@ static int rsi_send_auto_rate_request(struct rsi_common *common,
+ is_sgi = true;
+ }
+
++ /* Limit to any rates administratively configured by cfg80211 */
++ configured_rates = common->rate_config[band].configured_mask ?: 0xffffffff;
++ rate_bitmap &= configured_rates;
++
+ if (band == NL80211_BAND_2GHZ) {
+ if ((rate_bitmap == 0) && (is_ht))
+ min_rate = RSI_RATE_MCS0;
+@@ -1389,10 +1393,13 @@ static int rsi_send_auto_rate_request(struct rsi_common *common,
+ num_supported_rates = jj;
+
+ if (is_ht) {
+- for (ii = 0; ii < ARRAY_SIZE(mcs); ii++)
+- selected_rates[jj++] = mcs[ii];
+- num_supported_rates += ARRAY_SIZE(mcs);
+- rate_offset += ARRAY_SIZE(mcs);
++ for (ii = 0; ii < ARRAY_SIZE(mcs); ii++) {
++ if (configured_rates & BIT(ii + ARRAY_SIZE(rsi_rates))) {
++ selected_rates[jj++] = mcs[ii];
++ num_supported_rates++;
++ rate_offset++;
++ }
++ }
+ }
+
+ sort(selected_rates, jj, sizeof(u16), &rsi_compare, NULL);
+@@ -1482,7 +1489,7 @@ void rsi_inform_bss_status(struct rsi_common *common,
+ qos_enable,
+ aid, sta_id,
+ vif);
+- if (common->min_rate == 0xffff)
++ if (!common->rate_config[common->band].fixed_enabled)
+ rsi_send_auto_rate_request(common, sta, sta_id, vif);
+ if (opmode == RSI_OPMODE_STA &&
+ !(assoc_cap & WLAN_CAPABILITY_PRIVACY) &&
+@@ -2071,6 +2078,9 @@ static int rsi_handle_ta_confirm_type(struct rsi_common *common,
+ if (common->reinit_hw) {
+ complete(&common->wlan_init_completion);
+ } else {
++ if (common->bt_defer_attach)
++ rsi_attach_bt(common);
++
+ return rsi_mac80211_attach(common);
+ }
+ }
+diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio.c b/drivers/net/wireless/rsi/rsi_91x_sdio.c
+index e0c502bc42707..9f16128e4ffab 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_sdio.c
++++ b/drivers/net/wireless/rsi/rsi_91x_sdio.c
+@@ -24,10 +24,7 @@
+ /* Default operating mode is wlan STA + BT */
+ static u16 dev_oper_mode = DEV_OPMODE_STA_BT_DUAL;
+ module_param(dev_oper_mode, ushort, 0444);
+-MODULE_PARM_DESC(dev_oper_mode,
+- "1[Wi-Fi], 4[BT], 8[BT LE], 5[Wi-Fi STA + BT classic]\n"
+- "9[Wi-Fi STA + BT LE], 13[Wi-Fi STA + BT classic + BT LE]\n"
+- "6[AP + BT classic], 14[AP + BT classic + BT LE]");
++MODULE_PARM_DESC(dev_oper_mode, DEV_OPMODE_PARAM_DESC);
+
+ /**
+ * rsi_sdio_set_cmd52_arg() - This function prepares cmd 52 read/write arg.
+diff --git a/drivers/net/wireless/rsi/rsi_91x_usb.c b/drivers/net/wireless/rsi/rsi_91x_usb.c
+index 416976f098882..66fe386ec9cc6 100644
+--- a/drivers/net/wireless/rsi/rsi_91x_usb.c
++++ b/drivers/net/wireless/rsi/rsi_91x_usb.c
+@@ -25,10 +25,7 @@
+ /* Default operating mode is wlan STA + BT */
+ static u16 dev_oper_mode = DEV_OPMODE_STA_BT_DUAL;
+ module_param(dev_oper_mode, ushort, 0444);
+-MODULE_PARM_DESC(dev_oper_mode,
+- "1[Wi-Fi], 4[BT], 8[BT LE], 5[Wi-Fi STA + BT classic]\n"
+- "9[Wi-Fi STA + BT LE], 13[Wi-Fi STA + BT classic + BT LE]\n"
+- "6[AP + BT classic], 14[AP + BT classic + BT LE]");
++MODULE_PARM_DESC(dev_oper_mode, DEV_OPMODE_PARAM_DESC);
+
+ static int rsi_rx_urb_submit(struct rsi_hw *adapter, u8 ep_num, gfp_t flags);
+
+@@ -61,7 +58,7 @@ static int rsi_usb_card_write(struct rsi_hw *adapter,
+ (void *)seg,
+ (int)len,
+ &transfer,
+- HZ * 5);
++ USB_CTRL_SET_TIMEOUT);
+
+ if (status < 0) {
+ rsi_dbg(ERR_ZONE,
+@@ -272,8 +269,12 @@ static void rsi_rx_done_handler(struct urb *urb)
+ struct rsi_91x_usbdev *dev = (struct rsi_91x_usbdev *)rx_cb->data;
+ int status = -EINVAL;
+
++ if (!rx_cb->rx_skb)
++ return;
++
+ if (urb->status) {
+ dev_kfree_skb(rx_cb->rx_skb);
++ rx_cb->rx_skb = NULL;
+ return;
+ }
+
+@@ -297,8 +298,10 @@ out:
+ if (rsi_rx_urb_submit(dev->priv, rx_cb->ep_num, GFP_ATOMIC))
+ rsi_dbg(ERR_ZONE, "%s: Failed in urb submission", __func__);
+
+- if (status)
++ if (status) {
+ dev_kfree_skb(rx_cb->rx_skb);
++ rx_cb->rx_skb = NULL;
++ }
+ }
+
+ static void rsi_rx_urb_kill(struct rsi_hw *adapter, u8 ep_num)
+@@ -327,7 +330,6 @@ static int rsi_rx_urb_submit(struct rsi_hw *adapter, u8 ep_num, gfp_t mem_flags)
+ struct sk_buff *skb;
+ u8 dword_align_bytes = 0;
+
+-#define RSI_MAX_RX_USB_PKT_SIZE 3000
+ skb = dev_alloc_skb(RSI_MAX_RX_USB_PKT_SIZE);
+ if (!skb)
+ return -ENOMEM;
+diff --git a/drivers/net/wireless/rsi/rsi_hal.h b/drivers/net/wireless/rsi/rsi_hal.h
+index d044a440fa080..5b07262a97408 100644
+--- a/drivers/net/wireless/rsi/rsi_hal.h
++++ b/drivers/net/wireless/rsi/rsi_hal.h
+@@ -28,6 +28,17 @@
+ #define DEV_OPMODE_AP_BT 6
+ #define DEV_OPMODE_AP_BT_DUAL 14
+
++#define DEV_OPMODE_PARAM_DESC \
++ __stringify(DEV_OPMODE_WIFI_ALONE) "[Wi-Fi alone], " \
++ __stringify(DEV_OPMODE_BT_ALONE) "[BT classic alone], " \
++ __stringify(DEV_OPMODE_BT_LE_ALONE) "[BT LE alone], " \
++ __stringify(DEV_OPMODE_BT_DUAL) "[BT classic + BT LE alone], " \
++ __stringify(DEV_OPMODE_STA_BT) "[Wi-Fi STA + BT classic], " \
++ __stringify(DEV_OPMODE_STA_BT_LE) "[Wi-Fi STA + BT LE], " \
++ __stringify(DEV_OPMODE_STA_BT_DUAL) "[Wi-Fi STA + BT classic + BT LE], " \
++ __stringify(DEV_OPMODE_AP_BT) "[Wi-Fi AP + BT classic], " \
++ __stringify(DEV_OPMODE_AP_BT_DUAL) "[Wi-Fi AP + BT classic + BT LE]"
++
+ #define FLASH_WRITE_CHUNK_SIZE (4 * 1024)
+ #define FLASH_SECTOR_SIZE (4 * 1024)
+
+diff --git a/drivers/net/wireless/rsi/rsi_main.h b/drivers/net/wireless/rsi/rsi_main.h
+index 0f535850a3836..dcf8fb40698b7 100644
+--- a/drivers/net/wireless/rsi/rsi_main.h
++++ b/drivers/net/wireless/rsi/rsi_main.h
+@@ -61,6 +61,7 @@ enum RSI_FSM_STATES {
+ extern u32 rsi_zone_enabled;
+ extern __printf(2, 3) void rsi_dbg(u32 zone, const char *fmt, ...);
+
++#define RSI_MAX_BANDS 2
+ #define RSI_MAX_VIFS 3
+ #define NUM_EDCA_QUEUES 4
+ #define IEEE80211_ADDR_LEN 6
+@@ -139,6 +140,7 @@ struct skb_info {
+ u8 internal_hdr_size;
+ struct ieee80211_vif *vif;
+ u8 vap_id;
++ bool have_key;
+ };
+
+ enum edca_queue {
+@@ -229,6 +231,12 @@ struct rsi_9116_features {
+ u32 ps_options;
+ };
+
++struct rsi_rate_config {
++ u32 configured_mask; /* configured by mac80211 bits 0-11=legacy 12+ mcs */
++ u16 fixed_hw_rate;
++ bool fixed_enabled;
++};
++
+ struct rsi_common {
+ struct rsi_hw *priv;
+ struct vif_priv vif_info[RSI_MAX_VIFS];
+@@ -254,8 +262,8 @@ struct rsi_common {
+ u8 channel_width;
+
+ u16 rts_threshold;
+- u16 bitrate_mask[2];
+- u32 fixedrate_mask[2];
++ u32 bitrate_mask[RSI_MAX_BANDS];
++ struct rsi_rate_config rate_config[RSI_MAX_BANDS];
+
+ u8 rf_reset;
+ struct transmit_q_stats tx_stats;
+@@ -276,7 +284,6 @@ struct rsi_common {
+ u8 mac_id;
+ u8 radio_id;
+ u16 rate_pwr[20];
+- u16 min_rate;
+
+ /* WMM algo related */
+ u8 selected_qnum;
+@@ -320,6 +327,7 @@ struct rsi_common {
+ struct ieee80211_vif *roc_vif;
+
+ bool eapol4_confirm;
++ bool bt_defer_attach;
+ void *bt_adapter;
+
+ struct cfg80211_scan_request *hwscan;
+@@ -401,5 +409,6 @@ struct rsi_host_intf_ops {
+
+ enum rsi_host_intf rsi_get_host_intf(void *priv);
+ void rsi_set_bt_context(void *priv, void *bt_context);
++void rsi_attach_bt(struct rsi_common *common);
+
+ #endif
+diff --git a/drivers/net/wireless/rsi/rsi_usb.h b/drivers/net/wireless/rsi/rsi_usb.h
+index 254d19b664123..961851748bc4c 100644
+--- a/drivers/net/wireless/rsi/rsi_usb.h
++++ b/drivers/net/wireless/rsi/rsi_usb.h
+@@ -44,6 +44,8 @@
+ #define RSI_USB_BUF_SIZE 4096
+ #define RSI_USB_CTRL_BUF_SIZE 0x04
+
++#define RSI_MAX_RX_USB_PKT_SIZE 3000
++
+ struct rx_usb_ctrl_block {
+ u8 *data;
+ struct urb *rx_urb;
+diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c
+index 71bf9b4f769f5..6872782e8dd89 100644
+--- a/drivers/net/wwan/mhi_wwan_mbim.c
++++ b/drivers/net/wwan/mhi_wwan_mbim.c
+@@ -385,13 +385,13 @@ static void mhi_net_rx_refill_work(struct work_struct *work)
+ int err;
+
+ while (!mhi_queue_is_full(mdev, DMA_FROM_DEVICE)) {
+- struct sk_buff *skb = alloc_skb(MHI_DEFAULT_MRU, GFP_KERNEL);
++ struct sk_buff *skb = alloc_skb(mbim->mru, GFP_KERNEL);
+
+ if (unlikely(!skb))
+ break;
+
+ err = mhi_queue_skb(mdev, DMA_FROM_DEVICE, skb,
+- MHI_DEFAULT_MRU, MHI_EOT);
++ mbim->mru, MHI_EOT);
+ if (unlikely(err)) {
+ kfree_skb(skb);
+ break;
+diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
+index 4a16d6e33c093..d9dea4829c86e 100644
+--- a/drivers/net/xen-netback/common.h
++++ b/drivers/net/xen-netback/common.h
+@@ -203,6 +203,7 @@ struct xenvif_queue { /* Per-queue data for xenvif */
+ unsigned int rx_queue_max;
+ unsigned int rx_queue_len;
+ unsigned long last_rx_time;
++ unsigned int rx_slots_needed;
+ bool stalled;
+
+ struct xenvif_copy_state rx_copy;
+diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c
+index accc991d153f7..dbac4c03d21a1 100644
+--- a/drivers/net/xen-netback/rx.c
++++ b/drivers/net/xen-netback/rx.c
+@@ -33,28 +33,36 @@
+ #include <xen/xen.h>
+ #include <xen/events.h>
+
+-static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
++/*
++ * Update the needed ring page slots for the first SKB queued.
++ * Note that any call sequence outside the RX thread calling this function
++ * needs to wake up the RX thread via a call of xenvif_kick_thread()
++ * afterwards in order to avoid a race with putting the thread to sleep.
++ */
++static void xenvif_update_needed_slots(struct xenvif_queue *queue,
++ const struct sk_buff *skb)
+ {
+- RING_IDX prod, cons;
+- struct sk_buff *skb;
+- int needed;
+- unsigned long flags;
+-
+- spin_lock_irqsave(&queue->rx_queue.lock, flags);
++ unsigned int needed = 0;
+
+- skb = skb_peek(&queue->rx_queue);
+- if (!skb) {
+- spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
+- return false;
++ if (skb) {
++ needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
++ if (skb_is_gso(skb))
++ needed++;
++ if (skb->sw_hash)
++ needed++;
+ }
+
+- needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
+- if (skb_is_gso(skb))
+- needed++;
+- if (skb->sw_hash)
+- needed++;
++ WRITE_ONCE(queue->rx_slots_needed, needed);
++}
+
+- spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
++static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
++{
++ RING_IDX prod, cons;
++ unsigned int needed;
++
++ needed = READ_ONCE(queue->rx_slots_needed);
++ if (!needed)
++ return false;
+
+ do {
+ prod = queue->rx.sring->req_prod;
+@@ -80,13 +88,19 @@ void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)
+
+ spin_lock_irqsave(&queue->rx_queue.lock, flags);
+
+- __skb_queue_tail(&queue->rx_queue, skb);
+-
+- queue->rx_queue_len += skb->len;
+- if (queue->rx_queue_len > queue->rx_queue_max) {
++ if (queue->rx_queue_len >= queue->rx_queue_max) {
+ struct net_device *dev = queue->vif->dev;
+
+ netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
++ kfree_skb(skb);
++ queue->vif->dev->stats.rx_dropped++;
++ } else {
++ if (skb_queue_empty(&queue->rx_queue))
++ xenvif_update_needed_slots(queue, skb);
++
++ __skb_queue_tail(&queue->rx_queue, skb);
++
++ queue->rx_queue_len += skb->len;
+ }
+
+ spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
+@@ -100,6 +114,8 @@ static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)
+
+ skb = __skb_dequeue(&queue->rx_queue);
+ if (skb) {
++ xenvif_update_needed_slots(queue, skb_peek(&queue->rx_queue));
++
+ queue->rx_queue_len -= skb->len;
+ if (queue->rx_queue_len < queue->rx_queue_max) {
+ struct netdev_queue *txq;
+@@ -134,6 +150,7 @@ static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
+ break;
+ xenvif_rx_dequeue(queue);
+ kfree_skb(skb);
++ queue->vif->dev->stats.rx_dropped++;
+ }
+ }
+
+@@ -487,27 +504,31 @@ void xenvif_rx_action(struct xenvif_queue *queue)
+ xenvif_rx_copy_flush(queue);
+ }
+
+-static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
++static RING_IDX xenvif_rx_queue_slots(const struct xenvif_queue *queue)
+ {
+ RING_IDX prod, cons;
+
+ prod = queue->rx.sring->req_prod;
+ cons = queue->rx.req_cons;
+
++ return prod - cons;
++}
++
++static bool xenvif_rx_queue_stalled(const struct xenvif_queue *queue)
++{
++ unsigned int needed = READ_ONCE(queue->rx_slots_needed);
++
+ return !queue->stalled &&
+- prod - cons < 1 &&
++ xenvif_rx_queue_slots(queue) < needed &&
+ time_after(jiffies,
+ queue->last_rx_time + queue->vif->stall_timeout);
+ }
+
+ static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
+ {
+- RING_IDX prod, cons;
+-
+- prod = queue->rx.sring->req_prod;
+- cons = queue->rx.req_cons;
++ unsigned int needed = READ_ONCE(queue->rx_slots_needed);
+
+- return queue->stalled && prod - cons >= 1;
++ return queue->stalled && xenvif_rx_queue_slots(queue) >= needed;
+ }
+
+ bool xenvif_have_rx_work(struct xenvif_queue *queue, bool test_kthread)
+diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c
+index d24b7a7993aa0..990360d75cb64 100644
+--- a/drivers/net/xen-netback/xenbus.c
++++ b/drivers/net/xen-netback/xenbus.c
+@@ -256,6 +256,7 @@ static void backend_disconnect(struct backend_info *be)
+ unsigned int queue_index;
+
+ xen_unregister_watchers(vif);
++ xenbus_rm(XBT_NIL, be->dev->nodename, "hotplug-status");
+ #ifdef CONFIG_DEBUG_FS
+ xenvif_debugfs_delif(vif);
+ #endif /* CONFIG_DEBUG_FS */
+@@ -675,7 +676,6 @@ static void hotplug_status_changed(struct xenbus_watch *watch,
+
+ /* Not interested in this watch anymore. */
+ unregister_hotplug_status_watch(be);
+- xenbus_rm(XBT_NIL, be->dev->nodename, "hotplug-status");
+ }
+ kfree(str);
+ }
+@@ -824,15 +824,11 @@ static void connect(struct backend_info *be)
+ xenvif_carrier_on(be->vif);
+
+ unregister_hotplug_status_watch(be);
+- if (xenbus_exists(XBT_NIL, dev->nodename, "hotplug-status")) {
+- err = xenbus_watch_pathfmt(dev, &be->hotplug_status_watch,
+- NULL, hotplug_status_changed,
+- "%s/%s", dev->nodename,
+- "hotplug-status");
+- if (err)
+- goto err;
++ err = xenbus_watch_pathfmt(dev, &be->hotplug_status_watch, NULL,
++ hotplug_status_changed,
++ "%s/%s", dev->nodename, "hotplug-status");
++ if (!err)
+ be->have_hotplug_status_watch = 1;
+- }
+
+ netif_tx_wake_all_queues(be->vif->dev);
+
+diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
+index fc41ba95f81d0..2492a27467b46 100644
+--- a/drivers/net/xen-netfront.c
++++ b/drivers/net/xen-netfront.c
+@@ -148,6 +148,9 @@ struct netfront_queue {
+ grant_ref_t gref_rx_head;
+ grant_ref_t grant_rx_ref[NET_RX_RING_SIZE];
+
++ unsigned int rx_rsp_unconsumed;
++ spinlock_t rx_cons_lock;
++
+ struct page_pool *page_pool;
+ struct xdp_rxq_info xdp_rxq;
+ };
+@@ -376,12 +379,13 @@ static int xennet_open(struct net_device *dev)
+ return 0;
+ }
+
+-static void xennet_tx_buf_gc(struct netfront_queue *queue)
++static bool xennet_tx_buf_gc(struct netfront_queue *queue)
+ {
+ RING_IDX cons, prod;
+ unsigned short id;
+ struct sk_buff *skb;
+ bool more_to_do;
++ bool work_done = false;
+ const struct device *dev = &queue->info->netdev->dev;
+
+ BUG_ON(!netif_carrier_ok(queue->info->netdev));
+@@ -398,6 +402,8 @@ static void xennet_tx_buf_gc(struct netfront_queue *queue)
+ for (cons = queue->tx.rsp_cons; cons != prod; cons++) {
+ struct xen_netif_tx_response txrsp;
+
++ work_done = true;
++
+ RING_COPY_RESPONSE(&queue->tx, cons, &txrsp);
+ if (txrsp.status == XEN_NETIF_RSP_NULL)
+ continue;
+@@ -418,14 +424,12 @@ static void xennet_tx_buf_gc(struct netfront_queue *queue)
+ queue->tx_link[id] = TX_LINK_NONE;
+ skb = queue->tx_skbs[id];
+ queue->tx_skbs[id] = NULL;
+- if (unlikely(gnttab_query_foreign_access(
+- queue->grant_tx_ref[id]) != 0)) {
++ if (unlikely(!gnttab_end_foreign_access_ref(
++ queue->grant_tx_ref[id], GNTMAP_readonly))) {
+ dev_alert(dev,
+ "Grant still in use by backend domain\n");
+ goto err;
+ }
+- gnttab_end_foreign_access_ref(
+- queue->grant_tx_ref[id], GNTMAP_readonly);
+ gnttab_release_grant_reference(
+ &queue->gref_tx_head, queue->grant_tx_ref[id]);
+ queue->grant_tx_ref[id] = GRANT_INVALID_REF;
+@@ -441,11 +445,13 @@ static void xennet_tx_buf_gc(struct netfront_queue *queue)
+
+ xennet_maybe_wake_tx(queue);
+
+- return;
++ return work_done;
+
+ err:
+ queue->info->broken = true;
+ dev_alert(dev, "Disabled for further use\n");
++
++ return work_done;
+ }
+
+ struct xennet_gnttab_make_txreq {
+@@ -834,6 +840,38 @@ static int xennet_close(struct net_device *dev)
+ return 0;
+ }
+
++static void xennet_destroy_queues(struct netfront_info *info)
++{
++ unsigned int i;
++
++ for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
++ struct netfront_queue *queue = &info->queues[i];
++
++ if (netif_running(info->netdev))
++ napi_disable(&queue->napi);
++ netif_napi_del(&queue->napi);
++ }
++
++ kfree(info->queues);
++ info->queues = NULL;
++}
++
++static void xennet_uninit(struct net_device *dev)
++{
++ struct netfront_info *np = netdev_priv(dev);
++ xennet_destroy_queues(np);
++}
++
++static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val)
++{
++ unsigned long flags;
++
++ spin_lock_irqsave(&queue->rx_cons_lock, flags);
++ queue->rx.rsp_cons = val;
++ queue->rx_rsp_unconsumed = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx);
++ spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
++}
++
+ static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb,
+ grant_ref_t ref)
+ {
+@@ -885,7 +923,7 @@ static int xennet_get_extras(struct netfront_queue *queue,
+ xennet_move_rx_slot(queue, skb, ref);
+ } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
+
+- queue->rx.rsp_cons = cons;
++ xennet_set_rx_rsp_cons(queue, cons);
+ return err;
+ }
+
+@@ -950,7 +988,6 @@ static int xennet_get_responses(struct netfront_queue *queue,
+ struct device *dev = &queue->info->netdev->dev;
+ struct bpf_prog *xdp_prog;
+ struct xdp_buff xdp;
+- unsigned long ret;
+ int slots = 1;
+ int err = 0;
+ u32 verdict;
+@@ -992,8 +1029,13 @@ static int xennet_get_responses(struct netfront_queue *queue,
+ goto next;
+ }
+
+- ret = gnttab_end_foreign_access_ref(ref, 0);
+- BUG_ON(!ret);
++ if (!gnttab_end_foreign_access_ref(ref, 0)) {
++ dev_alert(dev,
++ "Grant still in use by backend domain\n");
++ queue->info->broken = true;
++ dev_alert(dev, "Disabled for further use\n");
++ return -EINVAL;
++ }
+
+ gnttab_release_grant_reference(&queue->gref_rx_head, ref);
+
+@@ -1039,7 +1081,7 @@ next:
+ }
+
+ if (unlikely(err))
+- queue->rx.rsp_cons = cons + slots;
++ xennet_set_rx_rsp_cons(queue, cons + slots);
+
+ return err;
+ }
+@@ -1093,7 +1135,8 @@ static int xennet_fill_frags(struct netfront_queue *queue,
+ __pskb_pull_tail(skb, pull_to - skb_headlen(skb));
+ }
+ if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) {
+- queue->rx.rsp_cons = ++cons + skb_queue_len(list);
++ xennet_set_rx_rsp_cons(queue,
++ ++cons + skb_queue_len(list));
+ kfree_skb(nskb);
+ return -ENOENT;
+ }
+@@ -1106,7 +1149,7 @@ static int xennet_fill_frags(struct netfront_queue *queue,
+ kfree_skb(nskb);
+ }
+
+- queue->rx.rsp_cons = cons;
++ xennet_set_rx_rsp_cons(queue, cons);
+
+ return 0;
+ }
+@@ -1213,6 +1256,10 @@ static int xennet_poll(struct napi_struct *napi, int budget)
+ &need_xdp_flush);
+
+ if (unlikely(err)) {
++ if (queue->info->broken) {
++ spin_unlock(&queue->rx_lock);
++ return 0;
++ }
+ err:
+ while ((skb = __skb_dequeue(&tmpq)))
+ __skb_queue_tail(&errq, skb);
+@@ -1229,7 +1276,9 @@ err:
+
+ if (unlikely(xennet_set_skb_gso(skb, gso))) {
+ __skb_queue_head(&tmpq, skb);
+- queue->rx.rsp_cons += skb_queue_len(&tmpq);
++ xennet_set_rx_rsp_cons(queue,
++ queue->rx.rsp_cons +
++ skb_queue_len(&tmpq));
+ goto err;
+ }
+ }
+@@ -1253,7 +1302,8 @@ err:
+
+ __skb_queue_tail(&rxq, skb);
+
+- i = ++queue->rx.rsp_cons;
++ i = queue->rx.rsp_cons + 1;
++ xennet_set_rx_rsp_cons(queue, i);
+ work_done++;
+ }
+ if (need_xdp_flush)
+@@ -1417,40 +1467,79 @@ static int xennet_set_features(struct net_device *dev,
+ return 0;
+ }
+
+-static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
++static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi)
+ {
+- struct netfront_queue *queue = dev_id;
+ unsigned long flags;
+
+- if (queue->info->broken)
+- return IRQ_HANDLED;
++ if (unlikely(queue->info->broken))
++ return false;
+
+ spin_lock_irqsave(&queue->tx_lock, flags);
+- xennet_tx_buf_gc(queue);
++ if (xennet_tx_buf_gc(queue))
++ *eoi = 0;
+ spin_unlock_irqrestore(&queue->tx_lock, flags);
+
++ return true;
++}
++
++static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
++{
++ unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
++
++ if (likely(xennet_handle_tx(dev_id, &eoiflag)))
++ xen_irq_lateeoi(irq, eoiflag);
++
+ return IRQ_HANDLED;
+ }
+
+-static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
++static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi)
+ {
+- struct netfront_queue *queue = dev_id;
+- struct net_device *dev = queue->info->netdev;
++ unsigned int work_queued;
++ unsigned long flags;
++
++ if (unlikely(queue->info->broken))
++ return false;
+
+- if (queue->info->broken)
+- return IRQ_HANDLED;
++ spin_lock_irqsave(&queue->rx_cons_lock, flags);
++ work_queued = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx);
++ if (work_queued > queue->rx_rsp_unconsumed) {
++ queue->rx_rsp_unconsumed = work_queued;
++ *eoi = 0;
++ } else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) {
++ const struct device *dev = &queue->info->netdev->dev;
++
++ spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
++ dev_alert(dev, "RX producer index going backwards\n");
++ dev_alert(dev, "Disabled for further use\n");
++ queue->info->broken = true;
++ return false;
++ }
++ spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
+
+- if (likely(netif_carrier_ok(dev) &&
+- RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
++ if (likely(netif_carrier_ok(queue->info->netdev) && work_queued))
+ napi_schedule(&queue->napi);
+
++ return true;
++}
++
++static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
++{
++ unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
++
++ if (likely(xennet_handle_rx(dev_id, &eoiflag)))
++ xen_irq_lateeoi(irq, eoiflag);
++
+ return IRQ_HANDLED;
+ }
+
+ static irqreturn_t xennet_interrupt(int irq, void *dev_id)
+ {
+- xennet_tx_interrupt(irq, dev_id);
+- xennet_rx_interrupt(irq, dev_id);
++ unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
++
++ if (xennet_handle_tx(dev_id, &eoiflag) &&
++ xennet_handle_rx(dev_id, &eoiflag))
++ xen_irq_lateeoi(irq, eoiflag);
++
+ return IRQ_HANDLED;
+ }
+
+@@ -1550,6 +1639,7 @@ static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+ }
+
+ static const struct net_device_ops xennet_netdev_ops = {
++ .ndo_uninit = xennet_uninit,
+ .ndo_open = xennet_open,
+ .ndo_stop = xennet_close,
+ .ndo_start_xmit = xennet_start_xmit,
+@@ -1768,9 +1858,10 @@ static int setup_netfront_single(struct netfront_queue *queue)
+ if (err < 0)
+ goto fail;
+
+- err = bind_evtchn_to_irqhandler(queue->tx_evtchn,
+- xennet_interrupt,
+- 0, queue->info->netdev->name, queue);
++ err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
++ xennet_interrupt, 0,
++ queue->info->netdev->name,
++ queue);
+ if (err < 0)
+ goto bind_fail;
+ queue->rx_evtchn = queue->tx_evtchn;
+@@ -1798,18 +1889,18 @@ static int setup_netfront_split(struct netfront_queue *queue)
+
+ snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
+ "%s-tx", queue->name);
+- err = bind_evtchn_to_irqhandler(queue->tx_evtchn,
+- xennet_tx_interrupt,
+- 0, queue->tx_irq_name, queue);
++ err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
++ xennet_tx_interrupt, 0,
++ queue->tx_irq_name, queue);
+ if (err < 0)
+ goto bind_tx_fail;
+ queue->tx_irq = err;
+
+ snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
+ "%s-rx", queue->name);
+- err = bind_evtchn_to_irqhandler(queue->rx_evtchn,
+- xennet_rx_interrupt,
+- 0, queue->rx_irq_name, queue);
++ err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn,
++ xennet_rx_interrupt, 0,
++ queue->rx_irq_name, queue);
+ if (err < 0)
+ goto bind_rx_fail;
+ queue->rx_irq = err;
+@@ -1833,7 +1924,7 @@ static int setup_netfront(struct xenbus_device *dev,
+ struct netfront_queue *queue, unsigned int feature_split_evtchn)
+ {
+ struct xen_netif_tx_sring *txs;
+- struct xen_netif_rx_sring *rxs;
++ struct xen_netif_rx_sring *rxs = NULL;
+ grant_ref_t gref;
+ int err;
+
+@@ -1853,21 +1944,21 @@ static int setup_netfront(struct xenbus_device *dev,
+
+ err = xenbus_grant_ring(dev, txs, 1, &gref);
+ if (err < 0)
+- goto grant_tx_ring_fail;
++ goto fail;
+ queue->tx_ring_ref = gref;
+
+ rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH);
+ if (!rxs) {
+ err = -ENOMEM;
+ xenbus_dev_fatal(dev, err, "allocating rx ring page");
+- goto alloc_rx_ring_fail;
++ goto fail;
+ }
+ SHARED_RING_INIT(rxs);
+ FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE);
+
+ err = xenbus_grant_ring(dev, rxs, 1, &gref);
+ if (err < 0)
+- goto grant_rx_ring_fail;
++ goto fail;
+ queue->rx_ring_ref = gref;
+
+ if (feature_split_evtchn)
+@@ -1880,22 +1971,28 @@ static int setup_netfront(struct xenbus_device *dev,
+ err = setup_netfront_single(queue);
+
+ if (err)
+- goto alloc_evtchn_fail;
++ goto fail;
+
+ return 0;
+
+ /* If we fail to setup netfront, it is safe to just revoke access to
+ * granted pages because backend is not accessing it at this point.
+ */
+-alloc_evtchn_fail:
+- gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0);
+-grant_rx_ring_fail:
+- free_page((unsigned long)rxs);
+-alloc_rx_ring_fail:
+- gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0);
+-grant_tx_ring_fail:
+- free_page((unsigned long)txs);
+-fail:
++ fail:
++ if (queue->rx_ring_ref != GRANT_INVALID_REF) {
++ gnttab_end_foreign_access(queue->rx_ring_ref, 0,
++ (unsigned long)rxs);
++ queue->rx_ring_ref = GRANT_INVALID_REF;
++ } else {
++ free_page((unsigned long)rxs);
++ }
++ if (queue->tx_ring_ref != GRANT_INVALID_REF) {
++ gnttab_end_foreign_access(queue->tx_ring_ref, 0,
++ (unsigned long)txs);
++ queue->tx_ring_ref = GRANT_INVALID_REF;
++ } else {
++ free_page((unsigned long)txs);
++ }
+ return err;
+ }
+
+@@ -1911,6 +2008,7 @@ static int xennet_init_queue(struct netfront_queue *queue)
+
+ spin_lock_init(&queue->tx_lock);
+ spin_lock_init(&queue->rx_lock);
++ spin_lock_init(&queue->rx_cons_lock);
+
+ timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0);
+
+@@ -2040,22 +2138,6 @@ error:
+ return err;
+ }
+
+-static void xennet_destroy_queues(struct netfront_info *info)
+-{
+- unsigned int i;
+-
+- for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
+- struct netfront_queue *queue = &info->queues[i];
+-
+- if (netif_running(info->netdev))
+- napi_disable(&queue->napi);
+- netif_napi_del(&queue->napi);
+- }
+-
+- kfree(info->queues);
+- info->queues = NULL;
+-}
+-
+
+
+ static int xennet_create_page_pool(struct netfront_queue *queue)
+diff --git a/drivers/nfc/nfcmrvl/main.c b/drivers/nfc/nfcmrvl/main.c
+index 2fcf545012b16..1a5284de4341b 100644
+--- a/drivers/nfc/nfcmrvl/main.c
++++ b/drivers/nfc/nfcmrvl/main.c
+@@ -183,6 +183,7 @@ void nfcmrvl_nci_unregister_dev(struct nfcmrvl_private *priv)
+ {
+ struct nci_dev *ndev = priv->ndev;
+
++ nci_unregister_device(ndev);
+ if (priv->ndev->nfc_dev->fw_download_in_progress)
+ nfcmrvl_fw_dnld_abort(priv);
+
+@@ -191,7 +192,6 @@ void nfcmrvl_nci_unregister_dev(struct nfcmrvl_private *priv)
+ if (gpio_is_valid(priv->config.reset_n_io))
+ gpio_free(priv->config.reset_n_io);
+
+- nci_unregister_device(ndev);
+ nci_free_device(ndev);
+ kfree(priv);
+ }
+diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c
+index 2f3f3fe9a0baa..d32aec0c334fe 100644
+--- a/drivers/nfc/pn533/pn533.c
++++ b/drivers/nfc/pn533/pn533.c
+@@ -2218,7 +2218,7 @@ static int pn533_fill_fragment_skbs(struct pn533 *dev, struct sk_buff *skb)
+ frag = pn533_alloc_skb(dev, frag_size);
+ if (!frag) {
+ skb_queue_purge(&dev->fragment_skb);
+- break;
++ return -ENOMEM;
+ }
+
+ if (!dev->tgt_mode) {
+@@ -2287,7 +2287,7 @@ static int pn533_transceive(struct nfc_dev *nfc_dev,
+ /* jumbo frame ? */
+ if (skb->len > PN533_CMD_DATAEXCH_DATA_MAXLEN) {
+ rc = pn533_fill_fragment_skbs(dev, skb);
+- if (rc <= 0)
++ if (rc < 0)
+ goto error;
+
+ skb = skb_dequeue(&dev->fragment_skb);
+@@ -2355,7 +2355,7 @@ static int pn533_tm_send(struct nfc_dev *nfc_dev, struct sk_buff *skb)
+ /* let's split in multiple chunks if size's too big */
+ if (skb->len > PN533_CMD_DATAEXCH_DATA_MAXLEN) {
+ rc = pn533_fill_fragment_skbs(dev, skb);
+- if (rc <= 0)
++ if (rc < 0)
+ goto error;
+
+ /* get the first skb */
+diff --git a/drivers/nfc/port100.c b/drivers/nfc/port100.c
+index 16ceb763594fc..90e30e2f15125 100644
+--- a/drivers/nfc/port100.c
++++ b/drivers/nfc/port100.c
+@@ -1612,7 +1612,9 @@ free_nfc_dev:
+ nfc_digital_free_device(dev->nfc_digital_dev);
+
+ error:
++ usb_kill_urb(dev->in_urb);
+ usb_free_urb(dev->in_urb);
++ usb_kill_urb(dev->out_urb);
+ usb_free_urb(dev->out_urb);
+ usb_put_dev(dev->udev);
+
+diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c
+index 279d88128b2e4..d56bc24709b5c 100644
+--- a/drivers/nfc/st21nfca/i2c.c
++++ b/drivers/nfc/st21nfca/i2c.c
+@@ -528,7 +528,8 @@ static int st21nfca_hci_i2c_probe(struct i2c_client *client,
+ phy->gpiod_ena = devm_gpiod_get(dev, "enable", GPIOD_OUT_LOW);
+ if (IS_ERR(phy->gpiod_ena)) {
+ nfc_err(dev, "Unable to get ENABLE GPIO\n");
+- return PTR_ERR(phy->gpiod_ena);
++ r = PTR_ERR(phy->gpiod_ena);
++ goto out_free;
+ }
+
+ phy->se_status.is_ese_present =
+@@ -539,7 +540,7 @@ static int st21nfca_hci_i2c_probe(struct i2c_client *client,
+ r = st21nfca_hci_platform_init(phy);
+ if (r < 0) {
+ nfc_err(&client->dev, "Unable to reboot st21nfca\n");
+- return r;
++ goto out_free;
+ }
+
+ r = devm_request_threaded_irq(&client->dev, client->irq, NULL,
+@@ -548,15 +549,23 @@ static int st21nfca_hci_i2c_probe(struct i2c_client *client,
+ ST21NFCA_HCI_DRIVER_NAME, phy);
+ if (r < 0) {
+ nfc_err(&client->dev, "Unable to register IRQ handler\n");
+- return r;
++ goto out_free;
+ }
+
+- return st21nfca_hci_probe(phy, &i2c_phy_ops, LLC_SHDLC_NAME,
+- ST21NFCA_FRAME_HEADROOM,
+- ST21NFCA_FRAME_TAILROOM,
+- ST21NFCA_HCI_LLC_MAX_PAYLOAD,
+- &phy->hdev,
+- &phy->se_status);
++ r = st21nfca_hci_probe(phy, &i2c_phy_ops, LLC_SHDLC_NAME,
++ ST21NFCA_FRAME_HEADROOM,
++ ST21NFCA_FRAME_TAILROOM,
++ ST21NFCA_HCI_LLC_MAX_PAYLOAD,
++ &phy->hdev,
++ &phy->se_status);
++ if (r)
++ goto out_free;
++
++ return 0;
++
++out_free:
++ kfree_skb(phy->pending_skb);
++ return r;
+ }
+
+ static int st21nfca_hci_i2c_remove(struct i2c_client *client)
+@@ -567,6 +576,8 @@ static int st21nfca_hci_i2c_remove(struct i2c_client *client)
+
+ if (phy->powered)
+ st21nfca_hci_i2c_disable(phy);
++ if (phy->pending_skb)
++ kfree_skb(phy->pending_skb);
+
+ return 0;
+ }
+diff --git a/drivers/nfc/st21nfca/se.c b/drivers/nfc/st21nfca/se.c
+index c8bdf078d1115..0841e0e370a03 100644
+--- a/drivers/nfc/st21nfca/se.c
++++ b/drivers/nfc/st21nfca/se.c
+@@ -320,6 +320,11 @@ int st21nfca_connectivity_event_received(struct nfc_hci_dev *hdev, u8 host,
+ return -ENOMEM;
+
+ transaction->aid_len = skb->data[1];
++
++ /* Checking if the length of the AID is valid */
++ if (transaction->aid_len > sizeof(transaction->aid))
++ return -EINVAL;
++
+ memcpy(transaction->aid, &skb->data[2],
+ transaction->aid_len);
+
+@@ -329,6 +334,11 @@ int st21nfca_connectivity_event_received(struct nfc_hci_dev *hdev, u8 host,
+ return -EPROTO;
+
+ transaction->params_len = skb->data[transaction->aid_len + 3];
++
++ /* Total size is allocated (skb->len - 2) minus fixed array members */
++ if (transaction->params_len > ((skb->len - 2) - sizeof(struct nfc_evt_transaction)))
++ return -EINVAL;
++
+ memcpy(transaction->params, skb->data +
+ transaction->aid_len + 4, transaction->params_len);
+
+diff --git a/drivers/ntb/hw/intel/ntb_hw_gen4.c b/drivers/ntb/hw/intel/ntb_hw_gen4.c
+index fede05151f698..4081fc538ff45 100644
+--- a/drivers/ntb/hw/intel/ntb_hw_gen4.c
++++ b/drivers/ntb/hw/intel/ntb_hw_gen4.c
+@@ -168,6 +168,18 @@ static enum ntb_topo gen4_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
+ return NTB_TOPO_NONE;
+ }
+
++static enum ntb_topo spr_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
++{
++ switch (ppd & SPR_PPD_TOPO_MASK) {
++ case SPR_PPD_TOPO_B2B_USD:
++ return NTB_TOPO_B2B_USD;
++ case SPR_PPD_TOPO_B2B_DSD:
++ return NTB_TOPO_B2B_DSD;
++ }
++
++ return NTB_TOPO_NONE;
++}
++
+ int gen4_init_dev(struct intel_ntb_dev *ndev)
+ {
+ struct pci_dev *pdev = ndev->ntb.pdev;
+@@ -183,7 +195,10 @@ int gen4_init_dev(struct intel_ntb_dev *ndev)
+ }
+
+ ppd1 = ioread32(ndev->self_mmio + GEN4_PPD1_OFFSET);
+- ndev->ntb.topo = gen4_ppd_topo(ndev, ppd1);
++ if (pdev_is_ICX(pdev))
++ ndev->ntb.topo = gen4_ppd_topo(ndev, ppd1);
++ else if (pdev_is_SPR(pdev))
++ ndev->ntb.topo = spr_ppd_topo(ndev, ppd1);
+ dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd1,
+ ntb_topo_string(ndev->ntb.topo));
+ if (ndev->ntb.topo == NTB_TOPO_NONE)
+diff --git a/drivers/ntb/hw/intel/ntb_hw_gen4.h b/drivers/ntb/hw/intel/ntb_hw_gen4.h
+index 3fcd3fdce9edf..f91323eaf5ce4 100644
+--- a/drivers/ntb/hw/intel/ntb_hw_gen4.h
++++ b/drivers/ntb/hw/intel/ntb_hw_gen4.h
+@@ -49,10 +49,14 @@
+ #define GEN4_PPD_CLEAR_TRN 0x0001
+ #define GEN4_PPD_LINKTRN 0x0008
+ #define GEN4_PPD_CONN_MASK 0x0300
++#define SPR_PPD_CONN_MASK 0x0700
+ #define GEN4_PPD_CONN_B2B 0x0200
+ #define GEN4_PPD_DEV_MASK 0x1000
+ #define GEN4_PPD_DEV_DSD 0x1000
+ #define GEN4_PPD_DEV_USD 0x0000
++#define SPR_PPD_DEV_MASK 0x4000
++#define SPR_PPD_DEV_DSD 0x4000
++#define SPR_PPD_DEV_USD 0x0000
+ #define GEN4_LINK_CTRL_LINK_DISABLE 0x0010
+
+ #define GEN4_SLOTSTS 0xb05a
+@@ -62,6 +66,10 @@
+ #define GEN4_PPD_TOPO_B2B_USD (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_USD)
+ #define GEN4_PPD_TOPO_B2B_DSD (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_DSD)
+
++#define SPR_PPD_TOPO_MASK (SPR_PPD_CONN_MASK | SPR_PPD_DEV_MASK)
++#define SPR_PPD_TOPO_B2B_USD (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_USD)
++#define SPR_PPD_TOPO_B2B_DSD (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_DSD)
++
+ #define GEN4_DB_COUNT 32
+ #define GEN4_DB_LINK 32
+ #define GEN4_DB_LINK_BIT BIT_ULL(GEN4_DB_LINK)
+@@ -112,4 +120,12 @@ static inline int pdev_is_ICX(struct pci_dev *pdev)
+ return 0;
+ }
+
++static inline int pdev_is_SPR(struct pci_dev *pdev)
++{
++ if (pdev_is_gen4(pdev) &&
++ pdev->revision > PCI_DEVICE_REVISION_ICX_MAX)
++ return 1;
++ return 0;
++}
++
+ #endif
+diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+index 4c6eb61a6ac62..ec9cb6c81edae 100644
+--- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
++++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+@@ -419,8 +419,8 @@ static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
+ enum ntb_width *width)
+ {
+ struct switchtec_dev *stdev = sndev->stdev;
+-
+- u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id);
++ u32 pff =
++ ioread32(&stdev->mmio_part_cfg_all[partition].vep_pff_inst_id);
+ u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
+
+ if (speed)
+@@ -840,7 +840,6 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
+ u64 tpart_vec;
+ int self;
+ u64 part_map;
+- int bit;
+
+ sndev->ntb.pdev = sndev->stdev->pdev;
+ sndev->ntb.topo = NTB_TOPO_SWITCH;
+@@ -861,29 +860,28 @@ static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
+ part_map = ioread64(&sndev->mmio_ntb->ep_map);
+ part_map &= ~(1 << sndev->self_partition);
+
+- if (!ffs(tpart_vec)) {
++ if (!tpart_vec) {
+ if (sndev->stdev->partition_count != 2) {
+ dev_err(&sndev->stdev->dev,
+ "ntb target partition not defined\n");
+ return -ENODEV;
+ }
+
+- bit = ffs(part_map);
+- if (!bit) {
++ if (!part_map) {
+ dev_err(&sndev->stdev->dev,
+ "peer partition is not NT partition\n");
+ return -ENODEV;
+ }
+
+- sndev->peer_partition = bit - 1;
++ sndev->peer_partition = __ffs64(part_map);
+ } else {
+- if (ffs(tpart_vec) != fls(tpart_vec)) {
++ if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) {
+ dev_err(&sndev->stdev->dev,
+ "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
+ return -ENODEV;
+ }
+
+- sndev->peer_partition = ffs(tpart_vec) - 1;
++ sndev->peer_partition = __ffs64(tpart_vec);
+ if (!(part_map & (1ULL << sndev->peer_partition))) {
+ dev_err(&sndev->stdev->dev,
+ "ntb target partition is not NT partition\n");
+diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
+index 92dec49522972..3fd1bdb9fc05b 100644
+--- a/drivers/nvdimm/btt.c
++++ b/drivers/nvdimm/btt.c
+@@ -1538,7 +1538,6 @@ static int btt_blk_init(struct btt *btt)
+ int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));
+
+ if (rc) {
+- del_gendisk(btt->btt_disk);
+ blk_cleanup_disk(btt->btt_disk);
+ return rc;
+ }
+diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
+index 054154c22899a..2721dd2ead0a7 100644
+--- a/drivers/nvdimm/pmem.c
++++ b/drivers/nvdimm/pmem.c
+@@ -429,8 +429,10 @@ static int pmem_attach_disk(struct device *dev,
+ bb_range.end = res->end;
+ }
+
+- if (IS_ERR(addr))
+- return PTR_ERR(addr);
++ if (IS_ERR(addr)) {
++ rc = PTR_ERR(addr);
++ goto out;
++ }
+ pmem->virt_addr = addr;
+
+ blk_queue_write_cache(q, true, fua);
+@@ -455,7 +457,8 @@ static int pmem_attach_disk(struct device *dev,
+ flags = DAXDEV_F_SYNC;
+ dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags);
+ if (IS_ERR(dax_dev)) {
+- return PTR_ERR(dax_dev);
++ rc = PTR_ERR(dax_dev);
++ goto out;
+ }
+ dax_write_cache(dax_dev, nvdimm_has_cache(nd_region));
+ pmem->dax_dev = dax_dev;
+@@ -470,8 +473,10 @@ static int pmem_attach_disk(struct device *dev,
+ "badblocks");
+ if (!pmem->bb_state)
+ dev_warn(dev, "'badblocks' notification disabled\n");
+-
+ return 0;
++out:
++ blk_cleanup_disk(pmem->disk);
++ return rc;
+ }
+
+ static int nd_pmem_probe(struct device *dev)
+diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
+index 9ccf3d6087993..70ad891a76bae 100644
+--- a/drivers/nvdimm/region_devs.c
++++ b/drivers/nvdimm/region_devs.c
+@@ -1025,6 +1025,9 @@ static unsigned long default_align(struct nd_region *nd_region)
+ }
+ }
+
++ if (nd_region->ndr_size < MEMREMAP_COMPAT_ALIGN_MAX)
++ align = PAGE_SIZE;
++
+ mappings = max_t(u16, 1, nd_region->ndr_mappings);
+ div_u64_rem(align, mappings, &remainder);
+ if (remainder)
+diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
+index f8dd664b2eda5..87877397d1add 100644
+--- a/drivers/nvme/host/core.c
++++ b/drivers/nvme/host/core.c
+@@ -131,7 +131,7 @@ static void nvme_set_queue_dying(struct nvme_ns *ns)
+ if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
+ return;
+
+- blk_set_queue_dying(ns->queue);
++ blk_mark_disk_dead(ns->disk);
+ blk_mq_unquiesce_queue(ns->queue);
+
+ set_capacity_and_notify(ns->disk, 0);
+@@ -1354,6 +1354,8 @@ static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
+ warn_str, cur->nidl);
+ return -1;
+ }
++ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
++ return NVME_NIDT_EUI64_LEN;
+ memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
+ return NVME_NIDT_EUI64_LEN;
+ case NVME_NIDT_NGUID:
+@@ -1362,6 +1364,8 @@ static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
+ warn_str, cur->nidl);
+ return -1;
+ }
++ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
++ return NVME_NIDT_NGUID_LEN;
+ memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
+ return NVME_NIDT_NGUID_LEN;
+ case NVME_NIDT_UUID:
+@@ -1370,6 +1374,8 @@ static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
+ warn_str, cur->nidl);
+ return -1;
+ }
++ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
++ return NVME_NIDT_UUID_LEN;
+ uuid_copy(&ids->uuid, data + sizeof(*cur));
+ return NVME_NIDT_UUID_LEN;
+ case NVME_NIDT_CSI:
+@@ -1466,12 +1472,18 @@ static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
+ if ((*id)->ncap == 0) /* namespace not allocated or attached */
+ goto out_free_id;
+
+- if (ctrl->vs >= NVME_VS(1, 1, 0) &&
+- !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
+- memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
+- if (ctrl->vs >= NVME_VS(1, 2, 0) &&
+- !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
+- memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
++
++ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
++ dev_info(ctrl->device,
++ "Ignoring bogus Namespace Identifiers\n");
++ } else {
++ if (ctrl->vs >= NVME_VS(1, 1, 0) &&
++ !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
++ memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
++ if (ctrl->vs >= NVME_VS(1, 2, 0) &&
++ !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
++ memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
++ }
+
+ return 0;
+
+@@ -1674,13 +1686,6 @@ static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
+ blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
+ }
+
+-static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
+-{
+- return !uuid_is_null(&ids->uuid) ||
+- memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
+- memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
+-}
+-
+ static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
+ {
+ return uuid_equal(&a->uuid, &b->uuid) &&
+@@ -1845,9 +1850,6 @@ static void nvme_update_disk_info(struct gendisk *disk,
+ nvme_config_discard(disk, ns);
+ blk_queue_max_write_zeroes_sectors(disk->queue,
+ ns->ctrl->max_zeroes_sectors);
+-
+- set_disk_ro(disk, (id->nsattr & NVME_NS_ATTR_RO) ||
+- test_bit(NVME_NS_FORCE_RO, &ns->flags));
+ }
+
+ static inline bool nvme_first_scan(struct gendisk *disk)
+@@ -1908,18 +1910,23 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
+ goto out_unfreeze;
+ }
+
++ set_disk_ro(ns->disk, (id->nsattr & NVME_NS_ATTR_RO) ||
++ test_bit(NVME_NS_FORCE_RO, &ns->flags));
+ set_bit(NVME_NS_READY, &ns->flags);
+ blk_mq_unfreeze_queue(ns->disk->queue);
+
+ if (blk_queue_is_zoned(ns->queue)) {
+ ret = nvme_revalidate_zones(ns);
+ if (ret && !nvme_first_scan(ns->disk))
+- goto out;
++ return ret;
+ }
+
+ if (nvme_ns_head_multipath(ns->head)) {
+ blk_mq_freeze_queue(ns->head->disk->queue);
+ nvme_update_disk_info(ns->head->disk, ns, id);
++ set_disk_ro(ns->head->disk,
++ (id->nsattr & NVME_NS_ATTR_RO) ||
++ test_bit(NVME_NS_FORCE_RO, &ns->flags));
+ nvme_mpath_revalidate_paths(ns);
+ blk_stack_limits(&ns->head->disk->queue->limits,
+ &ns->queue->limits, 0);
+@@ -1929,16 +1936,16 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
+ return 0;
+
+ out_unfreeze:
+- blk_mq_unfreeze_queue(ns->disk->queue);
+-out:
+ /*
+ * If probing fails due an unsupported feature, hide the block device,
+ * but still allow other access.
+ */
+ if (ret == -ENODEV) {
+ ns->disk->flags |= GENHD_FL_HIDDEN;
++ set_bit(NVME_NS_READY, &ns->flags);
+ ret = 0;
+ }
++ blk_mq_unfreeze_queue(ns->disk->queue);
+ return ret;
+ }
+
+@@ -3517,15 +3524,20 @@ static const struct attribute_group *nvme_dev_attr_groups[] = {
+ NULL,
+ };
+
+-static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
++static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
+ unsigned nsid)
+ {
+ struct nvme_ns_head *h;
+
+- lockdep_assert_held(&subsys->lock);
++ lockdep_assert_held(&ctrl->subsys->lock);
+
+- list_for_each_entry(h, &subsys->nsheads, entry) {
+- if (h->ns_id != nsid)
++ list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
++ /*
++ * Private namespaces can share NSIDs under some conditions.
++ * In that case we can't use the same ns_head for namespaces
++ * with the same NSID.
++ */
++ if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
+ continue;
+ if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
+ return h;
+@@ -3534,16 +3546,24 @@ static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
+ return NULL;
+ }
+
+-static int __nvme_check_ids(struct nvme_subsystem *subsys,
+- struct nvme_ns_head *new)
++static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
++ struct nvme_ns_ids *ids)
+ {
++ bool has_uuid = !uuid_is_null(&ids->uuid);
++ bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
++ bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
+ struct nvme_ns_head *h;
+
+ lockdep_assert_held(&subsys->lock);
+
+ list_for_each_entry(h, &subsys->nsheads, entry) {
+- if (nvme_ns_ids_valid(&new->ids) &&
+- nvme_ns_ids_equal(&new->ids, &h->ids))
++ if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
++ return -EINVAL;
++ if (has_nguid &&
++ memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
++ return -EINVAL;
++ if (has_eui64 &&
++ memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
+ return -EINVAL;
+ }
+
+@@ -3642,7 +3662,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
+ head->ids = *ids;
+ kref_init(&head->ref);
+
+- ret = __nvme_check_ids(ctrl->subsys, head);
++ ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &head->ids);
+ if (ret) {
+ dev_err(ctrl->device,
+ "duplicate IDs for nsid %d\n", nsid);
+@@ -3685,7 +3705,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
+ int ret = 0;
+
+ mutex_lock(&ctrl->subsys->lock);
+- head = nvme_find_ns_head(ctrl->subsys, nsid);
++ head = nvme_find_ns_head(ctrl, nsid);
+ if (!head) {
+ head = nvme_alloc_ns_head(ctrl, nsid, ids);
+ if (IS_ERR(head)) {
+@@ -4187,7 +4207,14 @@ static void nvme_async_event_work(struct work_struct *work)
+ container_of(work, struct nvme_ctrl, async_event_work);
+
+ nvme_aen_uevent(ctrl);
+- ctrl->ops->submit_async_event(ctrl);
++
++ /*
++ * The transport drivers must guarantee AER submission here is safe by
++ * flushing ctrl async_event_work after changing the controller state
++ * from LIVE and before freeing the admin queue.
++ */
++ if (ctrl->state == NVME_CTRL_LIVE)
++ ctrl->ops->submit_async_event(ctrl);
+ }
+
+ static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
+@@ -4331,6 +4358,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)
+ if (ctrl->queue_count > 1) {
+ nvme_queue_scan(ctrl);
+ nvme_start_queues(ctrl);
++ nvme_mpath_update(ctrl);
+ }
+ }
+ EXPORT_SYMBOL_GPL(nvme_start_ctrl);
+diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
+index a146cb903869c..561c2abd3892b 100644
+--- a/drivers/nvme/host/fabrics.h
++++ b/drivers/nvme/host/fabrics.h
+@@ -169,6 +169,7 @@ nvmf_ctlr_matches_baseopts(struct nvme_ctrl *ctrl,
+ struct nvmf_ctrl_options *opts)
+ {
+ if (ctrl->state == NVME_CTRL_DELETING ||
++ ctrl->state == NVME_CTRL_DELETING_NOIO ||
+ ctrl->state == NVME_CTRL_DEAD ||
+ strcmp(opts->subsysnqn, ctrl->opts->subsysnqn) ||
+ strcmp(opts->host->nqn, ctrl->opts->host->nqn) ||
+diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
+index fba06618c6c23..064acad505d38 100644
+--- a/drivers/nvme/host/multipath.c
++++ b/drivers/nvme/host/multipath.c
+@@ -138,13 +138,12 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
+ {
+ struct nvme_ns *ns;
+
+- mutex_lock(&ctrl->scan_lock);
+ down_read(&ctrl->namespaces_rwsem);
+- list_for_each_entry(ns, &ctrl->namespaces, list)
+- if (nvme_mpath_clear_current_path(ns))
+- kblockd_schedule_work(&ns->head->requeue_work);
++ list_for_each_entry(ns, &ctrl->namespaces, list) {
++ nvme_mpath_clear_current_path(ns);
++ kblockd_schedule_work(&ns->head->requeue_work);
++ }
+ up_read(&ctrl->namespaces_rwsem);
+- mutex_unlock(&ctrl->scan_lock);
+ }
+
+ void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
+@@ -463,10 +462,11 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
+
+ /*
+ * Add a multipath node if the subsystems supports multiple controllers.
+- * We also do this for private namespaces as the namespace sharing data could
+- * change after a rescan.
++ * We also do this for private namespaces as the namespace sharing flag
++ * could change after a rescan.
+ */
+- if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || !multipath)
++ if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
++ !nvme_is_unique_nsid(ctrl, head) || !multipath)
+ return 0;
+
+ head->disk = blk_alloc_disk(ctrl->numa_node);
+@@ -574,8 +574,17 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
+ ns->ana_grpid = le32_to_cpu(desc->grpid);
+ ns->ana_state = desc->state;
+ clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
+-
+- if (nvme_state_is_live(ns->ana_state))
++ /*
++ * nvme_mpath_set_live() will trigger I/O to the multipath path device
++ * and in turn to this path device. However we cannot accept this I/O
++ * if the controller is not live. This may deadlock if called from
++ * nvme_mpath_init_identify() and the ctrl will never complete
++ * initialization, preventing I/O from completing. For this case we
++ * will reprocess the ANA log page in nvme_mpath_update() once the
++ * controller is ready.
++ */
++ if (nvme_state_is_live(ns->ana_state) &&
++ ns->ctrl->state == NVME_CTRL_LIVE)
+ nvme_mpath_set_live(ns);
+ }
+
+@@ -662,6 +671,18 @@ static void nvme_ana_work(struct work_struct *work)
+ nvme_read_ana_log(ctrl);
+ }
+
++void nvme_mpath_update(struct nvme_ctrl *ctrl)
++{
++ u32 nr_change_groups = 0;
++
++ if (!ctrl->ana_log_buf)
++ return;
++
++ mutex_lock(&ctrl->ana_lock);
++ nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);
++ mutex_unlock(&ctrl->ana_lock);
++}
++
+ static void nvme_anatt_timeout(struct timer_list *t)
+ {
+ struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
+@@ -793,7 +814,7 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
+ {
+ if (!head->disk)
+ return;
+- blk_set_queue_dying(head->disk->queue);
++ blk_mark_disk_dead(head->disk);
+ /* make sure all pending bios are cleaned up */
+ kblockd_schedule_work(&head->requeue_work);
+ flush_work(&head->requeue_work);
+diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
+index ed79a6c7e8043..72bcd7e5716e5 100644
+--- a/drivers/nvme/host/nvme.h
++++ b/drivers/nvme/host/nvme.h
+@@ -144,6 +144,11 @@ enum nvme_quirks {
+ * encoding the generation sequence number.
+ */
+ NVME_QUIRK_SKIP_CID_GEN = (1 << 17),
++
++ /*
++ * Reports garbage in the namespace identifiers (eui64, nguid, uuid).
++ */
++ NVME_QUIRK_BOGUS_NID = (1 << 18),
+ };
+
+ /*
+@@ -693,6 +698,25 @@ static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
+ return true;
+ return __nvme_check_ready(ctrl, rq, queue_live);
+ }
++
++/*
++ * NSID shall be unique for all shared namespaces, or if at least one of the
++ * following conditions is met:
++ * 1. Namespace Management is supported by the controller
++ * 2. ANA is supported by the controller
++ * 3. NVM Set are supported by the controller
++ *
++ * In other case, private namespace are not required to report a unique NSID.
++ */
++static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl,
++ struct nvme_ns_head *head)
++{
++ return head->shared ||
++ (ctrl->oacs & NVME_CTRL_OACS_NS_MNGT_SUPP) ||
++ (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) ||
++ (ctrl->ctratt & NVME_CTRL_CTRATT_NVM_SETS);
++}
++
+ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
+ void *buf, unsigned bufflen);
+ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
+@@ -752,6 +776,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);
+ void nvme_mpath_remove_disk(struct nvme_ns_head *head);
+ int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
+ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
++void nvme_mpath_update(struct nvme_ctrl *ctrl);
+ void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
+ void nvme_mpath_stop(struct nvme_ctrl *ctrl);
+ bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
+@@ -826,6 +851,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,
+ "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
+ return 0;
+ }
++static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)
++{
++}
+ static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
+ {
+ }
+diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
+index 149ecf73df384..e6f55cf6e494a 100644
+--- a/drivers/nvme/host/pci.c
++++ b/drivers/nvme/host/pci.c
+@@ -3300,7 +3300,8 @@ static const struct pci_device_id nvme_id_table[] = {
+ NVME_QUIRK_DEALLOCATE_ZEROES, },
+ { PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */
+ .driver_data = NVME_QUIRK_STRIPE_SIZE |
+- NVME_QUIRK_DEALLOCATE_ZEROES, },
++ NVME_QUIRK_DEALLOCATE_ZEROES |
++ NVME_QUIRK_IGNORE_DEV_SUBNQN, },
+ { PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */
+ .driver_data = NVME_QUIRK_STRIPE_SIZE |
+ NVME_QUIRK_DEALLOCATE_ZEROES, },
+@@ -3313,7 +3314,10 @@ static const struct pci_device_id nvme_id_table[] = {
+ .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
+ { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */
+ .driver_data = NVME_QUIRK_IDENTIFY_CNS |
+- NVME_QUIRK_DISABLE_WRITE_ZEROES, },
++ NVME_QUIRK_DISABLE_WRITE_ZEROES |
++ NVME_QUIRK_BOGUS_NID, },
++ { PCI_VDEVICE(REDHAT, 0x0010), /* Qemu emulated controller */
++ .driver_data = NVME_QUIRK_BOGUS_NID, },
+ { PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */
+ .driver_data = NVME_QUIRK_NO_NS_DESC_LIST, },
+ { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */
+@@ -3351,6 +3355,10 @@ static const struct pci_device_id nvme_id_table[] = {
+ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
+ { PCI_DEVICE(0x2646, 0x2263), /* KINGSTON A2000 NVMe SSD */
+ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
++ { PCI_DEVICE(0x1e4B, 0x1002), /* MAXIO MAP1002 */
++ .driver_data = NVME_QUIRK_BOGUS_NID, },
++ { PCI_DEVICE(0x1e4B, 0x1202), /* MAXIO MAP1202 */
++ .driver_data = NVME_QUIRK_BOGUS_NID, },
+ { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
+ .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
+ { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
+@@ -3371,7 +3379,10 @@ static const struct pci_device_id nvme_id_table[] = {
+ NVME_QUIRK_128_BYTES_SQES |
+ NVME_QUIRK_SHARED_TAGS |
+ NVME_QUIRK_SKIP_CID_GEN },
+-
++ { PCI_DEVICE(0x144d, 0xa808), /* Samsung X5 */
++ .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY|
++ NVME_QUIRK_NO_DEEPEST_PS |
++ NVME_QUIRK_IGNORE_DEV_SUBNQN, },
+ { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
+ { 0, }
+ };
+diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
+index 042c594bc57e2..d51f52e296f50 100644
+--- a/drivers/nvme/host/rdma.c
++++ b/drivers/nvme/host/rdma.c
+@@ -1095,11 +1095,13 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
+ return ret;
+
+ if (ctrl->ctrl.icdoff) {
++ ret = -EOPNOTSUPP;
+ dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
+ goto destroy_admin;
+ }
+
+ if (!(ctrl->ctrl.sgls & (1 << 2))) {
++ ret = -EOPNOTSUPP;
+ dev_err(ctrl->ctrl.device,
+ "Mandatory keyed sgls are not supported!\n");
+ goto destroy_admin;
+@@ -1190,6 +1192,7 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
+ struct nvme_rdma_ctrl, err_work);
+
+ nvme_stop_keep_alive(&ctrl->ctrl);
++ flush_work(&ctrl->ctrl.async_event_work);
+ nvme_rdma_teardown_io_queues(ctrl, false);
+ nvme_start_queues(&ctrl->ctrl);
+ nvme_rdma_teardown_admin_queue(ctrl, false);
+diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
+index 4ae562d30d2b9..10882d3d554c2 100644
+--- a/drivers/nvme/host/tcp.c
++++ b/drivers/nvme/host/tcp.c
+@@ -30,6 +30,44 @@ static int so_priority;
+ module_param(so_priority, int, 0644);
+ MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
+
++#ifdef CONFIG_DEBUG_LOCK_ALLOC
++/* lockdep can detect a circular dependency of the form
++ * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
++ * because dependencies are tracked for both nvme-tcp and user contexts. Using
++ * a separate class prevents lockdep from conflating nvme-tcp socket use with
++ * user-space socket API use.
++ */
++static struct lock_class_key nvme_tcp_sk_key[2];
++static struct lock_class_key nvme_tcp_slock_key[2];
++
++static void nvme_tcp_reclassify_socket(struct socket *sock)
++{
++ struct sock *sk = sock->sk;
++
++ if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
++ return;
++
++ switch (sk->sk_family) {
++ case AF_INET:
++ sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
++ &nvme_tcp_slock_key[0],
++ "sk_lock-AF_INET-NVME",
++ &nvme_tcp_sk_key[0]);
++ break;
++ case AF_INET6:
++ sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
++ &nvme_tcp_slock_key[1],
++ "sk_lock-AF_INET6-NVME",
++ &nvme_tcp_sk_key[1]);
++ break;
++ default:
++ WARN_ON_ONCE(1);
++ }
++}
++#else
++static void nvme_tcp_reclassify_socket(struct socket *sock) { }
++#endif
++
+ enum nvme_tcp_send_state {
+ NVME_TCP_SEND_CMD_PDU = 0,
+ NVME_TCP_SEND_H2C_PDU,
+@@ -920,7 +958,15 @@ static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
+
+ static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
+ {
+- nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_HOST_PATH_ERROR);
++ if (nvme_tcp_async_req(req)) {
++ union nvme_result res = {};
++
++ nvme_complete_async_event(&req->queue->ctrl->ctrl,
++ cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
++ } else {
++ nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
++ NVME_SC_HOST_PATH_ERROR);
++ }
+ }
+
+ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
+@@ -1428,6 +1474,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
+ goto err_destroy_mutex;
+ }
+
++ nvme_tcp_reclassify_socket(queue->sock);
++
+ /* Single syn retry */
+ tcp_sock_set_syncnt(queue->sock->sk, 1);
+
+@@ -2097,6 +2145,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work)
+ struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
+
+ nvme_stop_keep_alive(ctrl);
++ flush_work(&ctrl->async_event_work);
+ nvme_tcp_teardown_io_queues(ctrl, false);
+ /* unquiesce to fail fast pending requests */
+ nvme_start_queues(ctrl);
+diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
+index aa6d84d8848e7..52bb262d267ac 100644
+--- a/drivers/nvme/target/admin-cmd.c
++++ b/drivers/nvme/target/admin-cmd.c
+@@ -978,7 +978,7 @@ void nvmet_execute_async_event(struct nvmet_req *req)
+ ctrl->async_event_cmds[ctrl->nr_async_event_cmds++] = req;
+ mutex_unlock(&ctrl->lock);
+
+- schedule_work(&ctrl->async_event_work);
++ queue_work(nvmet_wq, &ctrl->async_event_work);
+ }
+
+ void nvmet_execute_keep_alive(struct nvmet_req *req)
+diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
+index be5d82421e3a4..cea30e4f50533 100644
+--- a/drivers/nvme/target/configfs.c
++++ b/drivers/nvme/target/configfs.c
+@@ -1553,6 +1553,8 @@ static void nvmet_port_release(struct config_item *item)
+ {
+ struct nvmet_port *port = to_nvmet_port(item);
+
++ /* Let inflight controllers teardown complete */
++ flush_workqueue(nvmet_wq);
+ list_del(&port->global_entry);
+
+ kfree(port->ana_state);
+diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
+index b8425fa34300f..a8dafe8670f20 100644
+--- a/drivers/nvme/target/core.c
++++ b/drivers/nvme/target/core.c
+@@ -20,6 +20,9 @@ struct workqueue_struct *zbd_wq;
+ static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
+ static DEFINE_IDA(cntlid_ida);
+
++struct workqueue_struct *nvmet_wq;
++EXPORT_SYMBOL_GPL(nvmet_wq);
++
+ /*
+ * This read/write semaphore is used to synchronize access to configuration
+ * information on a target system that will result in discovery log page
+@@ -205,7 +208,7 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
+ list_add_tail(&aen->entry, &ctrl->async_events);
+ mutex_unlock(&ctrl->lock);
+
+- schedule_work(&ctrl->async_event_work);
++ queue_work(nvmet_wq, &ctrl->async_event_work);
+ }
+
+ static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid)
+@@ -385,7 +388,7 @@ static void nvmet_keep_alive_timer(struct work_struct *work)
+ if (reset_tbkas) {
+ pr_debug("ctrl %d reschedule traffic based keep-alive timer\n",
+ ctrl->cntlid);
+- schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
++ queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ);
+ return;
+ }
+
+@@ -403,7 +406,7 @@ void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)
+ pr_debug("ctrl %d start keep-alive timer for %d secs\n",
+ ctrl->cntlid, ctrl->kato);
+
+- schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
++ queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ);
+ }
+
+ void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)
+@@ -1477,7 +1480,7 @@ void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)
+ mutex_lock(&ctrl->lock);
+ if (!(ctrl->csts & NVME_CSTS_CFS)) {
+ ctrl->csts |= NVME_CSTS_CFS;
+- schedule_work(&ctrl->fatal_err_work);
++ queue_work(nvmet_wq, &ctrl->fatal_err_work);
+ }
+ mutex_unlock(&ctrl->lock);
+ }
+@@ -1617,9 +1620,15 @@ static int __init nvmet_init(void)
+ goto out_free_zbd_work_queue;
+ }
+
++ nvmet_wq = alloc_workqueue("nvmet-wq", WQ_MEM_RECLAIM, 0);
++ if (!nvmet_wq) {
++ error = -ENOMEM;
++ goto out_free_buffered_work_queue;
++ }
++
+ error = nvmet_init_discovery();
+ if (error)
+- goto out_free_work_queue;
++ goto out_free_nvmet_work_queue;
+
+ error = nvmet_init_configfs();
+ if (error)
+@@ -1628,7 +1637,9 @@ static int __init nvmet_init(void)
+
+ out_exit_discovery:
+ nvmet_exit_discovery();
+-out_free_work_queue:
++out_free_nvmet_work_queue:
++ destroy_workqueue(nvmet_wq);
++out_free_buffered_work_queue:
+ destroy_workqueue(buffered_io_wq);
+ out_free_zbd_work_queue:
+ destroy_workqueue(zbd_wq);
+@@ -1640,6 +1651,7 @@ static void __exit nvmet_exit(void)
+ nvmet_exit_configfs();
+ nvmet_exit_discovery();
+ ida_destroy(&cntlid_ida);
++ destroy_workqueue(nvmet_wq);
+ destroy_workqueue(buffered_io_wq);
+ destroy_workqueue(zbd_wq);
+
+diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
+index 22b5108168a6a..c43bc5e1c7a28 100644
+--- a/drivers/nvme/target/fc.c
++++ b/drivers/nvme/target/fc.c
+@@ -1491,7 +1491,7 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
+ list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
+ if (!nvmet_fc_tgt_a_get(assoc))
+ continue;
+- if (!schedule_work(&assoc->del_work))
++ if (!queue_work(nvmet_wq, &assoc->del_work))
+ /* already deleting - release local reference */
+ nvmet_fc_tgt_a_put(assoc);
+ }
+@@ -1546,7 +1546,7 @@ nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,
+ continue;
+ assoc->hostport->invalid = 1;
+ noassoc = false;
+- if (!schedule_work(&assoc->del_work))
++ if (!queue_work(nvmet_wq, &assoc->del_work))
+ /* already deleting - release local reference */
+ nvmet_fc_tgt_a_put(assoc);
+ }
+@@ -1592,7 +1592,7 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
+ nvmet_fc_tgtport_put(tgtport);
+
+ if (found_ctrl) {
+- if (!schedule_work(&assoc->del_work))
++ if (!queue_work(nvmet_wq, &assoc->del_work))
+ /* already deleting - release local reference */
+ nvmet_fc_tgt_a_put(assoc);
+ return;
+@@ -2060,7 +2060,7 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
+ iod->rqstdatalen = lsreqbuf_len;
+ iod->hosthandle = hosthandle;
+
+- schedule_work(&iod->work);
++ queue_work(nvmet_wq, &iod->work);
+
+ return 0;
+ }
+diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c
+index 54606f1872b4a..5c16372f3b533 100644
+--- a/drivers/nvme/target/fcloop.c
++++ b/drivers/nvme/target/fcloop.c
+@@ -360,7 +360,7 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport,
+ spin_lock(&rport->lock);
+ list_add_tail(&rport->ls_list, &tls_req->ls_list);
+ spin_unlock(&rport->lock);
+- schedule_work(&rport->ls_work);
++ queue_work(nvmet_wq, &rport->ls_work);
+ return ret;
+ }
+
+@@ -393,7 +393,7 @@ fcloop_h2t_xmt_ls_rsp(struct nvmet_fc_target_port *targetport,
+ spin_lock(&rport->lock);
+ list_add_tail(&rport->ls_list, &tls_req->ls_list);
+ spin_unlock(&rport->lock);
+- schedule_work(&rport->ls_work);
++ queue_work(nvmet_wq, &rport->ls_work);
+ }
+
+ return 0;
+@@ -448,7 +448,7 @@ fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle,
+ spin_lock(&tport->lock);
+ list_add_tail(&tport->ls_list, &tls_req->ls_list);
+ spin_unlock(&tport->lock);
+- schedule_work(&tport->ls_work);
++ queue_work(nvmet_wq, &tport->ls_work);
+ return ret;
+ }
+
+@@ -480,7 +480,7 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport,
+ spin_lock(&tport->lock);
+ list_add_tail(&tport->ls_list, &tls_req->ls_list);
+ spin_unlock(&tport->lock);
+- schedule_work(&tport->ls_work);
++ queue_work(nvmet_wq, &tport->ls_work);
+ }
+
+ return 0;
+@@ -520,7 +520,7 @@ fcloop_tgt_discovery_evt(struct nvmet_fc_target_port *tgtport)
+ tgt_rscn->tport = tgtport->private;
+ INIT_WORK(&tgt_rscn->work, fcloop_tgt_rscn_work);
+
+- schedule_work(&tgt_rscn->work);
++ queue_work(nvmet_wq, &tgt_rscn->work);
+ }
+
+ static void
+@@ -739,7 +739,7 @@ fcloop_fcp_req(struct nvme_fc_local_port *localport,
+ INIT_WORK(&tfcp_req->tio_done_work, fcloop_tgt_fcprqst_done_work);
+ kref_init(&tfcp_req->ref);
+
+- schedule_work(&tfcp_req->fcp_rcv_work);
++ queue_work(nvmet_wq, &tfcp_req->fcp_rcv_work);
+
+ return 0;
+ }
+@@ -921,7 +921,7 @@ fcloop_fcp_req_release(struct nvmet_fc_target_port *tgtport,
+ {
+ struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
+
+- schedule_work(&tfcp_req->tio_done_work);
++ queue_work(nvmet_wq, &tfcp_req->tio_done_work);
+ }
+
+ static void
+@@ -976,7 +976,7 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport,
+
+ if (abortio)
+ /* leave the reference while the work item is scheduled */
+- WARN_ON(!schedule_work(&tfcp_req->abort_rcv_work));
++ WARN_ON(!queue_work(nvmet_wq, &tfcp_req->abort_rcv_work));
+ else {
+ /*
+ * as the io has already had the done callback made,
+diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c
+index 1dd1a0fe2e819..228871d48106b 100644
+--- a/drivers/nvme/target/io-cmd-file.c
++++ b/drivers/nvme/target/io-cmd-file.c
+@@ -8,6 +8,7 @@
+ #include <linux/uio.h>
+ #include <linux/falloc.h>
+ #include <linux/file.h>
++#include <linux/fs.h>
+ #include "nvmet.h"
+
+ #define NVMET_MAX_MPOOL_BVEC 16
+@@ -266,7 +267,8 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)
+
+ if (req->ns->buffered_io) {
+ if (likely(!req->f.mpool_alloc) &&
+- nvmet_file_execute_io(req, IOCB_NOWAIT))
++ (req->ns->file->f_mode & FMODE_NOWAIT) &&
++ nvmet_file_execute_io(req, IOCB_NOWAIT))
+ return;
+ nvmet_file_submit_buffered_io(req);
+ } else
+@@ -290,7 +292,7 @@ static void nvmet_file_execute_flush(struct nvmet_req *req)
+ if (!nvmet_check_transfer_len(req, 0))
+ return;
+ INIT_WORK(&req->f.work, nvmet_file_flush_work);
+- schedule_work(&req->f.work);
++ queue_work(nvmet_wq, &req->f.work);
+ }
+
+ static void nvmet_file_execute_discard(struct nvmet_req *req)
+@@ -350,7 +352,7 @@ static void nvmet_file_execute_dsm(struct nvmet_req *req)
+ if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req)))
+ return;
+ INIT_WORK(&req->f.work, nvmet_file_dsm_work);
+- schedule_work(&req->f.work);
++ queue_work(nvmet_wq, &req->f.work);
+ }
+
+ static void nvmet_file_write_zeroes_work(struct work_struct *w)
+@@ -380,7 +382,7 @@ static void nvmet_file_execute_write_zeroes(struct nvmet_req *req)
+ if (!nvmet_check_transfer_len(req, 0))
+ return;
+ INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work);
+- schedule_work(&req->f.work);
++ queue_work(nvmet_wq, &req->f.work);
+ }
+
+ u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
+diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
+index 0285ccc7541f6..2553f487c9f24 100644
+--- a/drivers/nvme/target/loop.c
++++ b/drivers/nvme/target/loop.c
+@@ -166,7 +166,7 @@ static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
+ iod->req.transfer_len = blk_rq_payload_bytes(req);
+ }
+
+- schedule_work(&iod->work);
++ queue_work(nvmet_wq, &iod->work);
+ return BLK_STS_OK;
+ }
+
+@@ -187,7 +187,7 @@ static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
+ return;
+ }
+
+- schedule_work(&iod->work);
++ queue_work(nvmet_wq, &iod->work);
+ }
+
+ static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
+diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
+index 7143c7fa74641..dbeb0b8c11947 100644
+--- a/drivers/nvme/target/nvmet.h
++++ b/drivers/nvme/target/nvmet.h
+@@ -365,6 +365,7 @@ struct nvmet_req {
+
+ extern struct workqueue_struct *buffered_io_wq;
+ extern struct workqueue_struct *zbd_wq;
++extern struct workqueue_struct *nvmet_wq;
+
+ static inline void nvmet_set_result(struct nvmet_req *req, u32 result)
+ {
+diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c
+index f0efb35379898..6220e1dd961ad 100644
+--- a/drivers/nvme/target/passthru.c
++++ b/drivers/nvme/target/passthru.c
+@@ -281,7 +281,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
+ if (req->p.use_workqueue || effects) {
+ INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work);
+ req->p.rq = rq;
+- schedule_work(&req->p.work);
++ queue_work(nvmet_wq, &req->p.work);
+ } else {
+ rq->end_io_data = req;
+ blk_execute_rq_nowait(ns ? ns->disk : NULL, rq, 0,
+diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
+index 891174ccd44bb..18e082091c821 100644
+--- a/drivers/nvme/target/rdma.c
++++ b/drivers/nvme/target/rdma.c
+@@ -1583,7 +1583,7 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id,
+
+ if (queue->host_qid == 0) {
+ /* Let inflight controller teardown complete */
+- flush_scheduled_work();
++ flush_workqueue(nvmet_wq);
+ }
+
+ ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn);
+@@ -1668,7 +1668,7 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
+
+ if (disconnect) {
+ rdma_disconnect(queue->cm_id);
+- schedule_work(&queue->release_work);
++ queue_work(nvmet_wq, &queue->release_work);
+ }
+ }
+
+@@ -1698,7 +1698,7 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
+ mutex_unlock(&nvmet_rdma_queue_mutex);
+
+ pr_err("failed to connect queue %d\n", queue->idx);
+- schedule_work(&queue->release_work);
++ queue_work(nvmet_wq, &queue->release_work);
+ }
+
+ /**
+@@ -1772,7 +1772,7 @@ static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
+ if (!queue) {
+ struct nvmet_rdma_port *port = cm_id->context;
+
+- schedule_delayed_work(&port->repair_work, 0);
++ queue_delayed_work(nvmet_wq, &port->repair_work, 0);
+ break;
+ }
+ fallthrough;
+@@ -1818,12 +1818,36 @@ restart:
+ mutex_unlock(&nvmet_rdma_queue_mutex);
+ }
+
++static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port)
++{
++ struct nvmet_rdma_queue *queue, *tmp;
++ struct nvmet_port *nport = port->nport;
++
++ mutex_lock(&nvmet_rdma_queue_mutex);
++ list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
++ queue_list) {
++ if (queue->port != nport)
++ continue;
++
++ list_del_init(&queue->queue_list);
++ __nvmet_rdma_queue_disconnect(queue);
++ }
++ mutex_unlock(&nvmet_rdma_queue_mutex);
++}
++
+ static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port)
+ {
+ struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL);
+
+ if (cm_id)
+ rdma_destroy_id(cm_id);
++
++ /*
++ * Destroy the remaining queues, which are not belong to any
++ * controller yet. Do it here after the RDMA-CM was destroyed
++ * guarantees that no new queue will be created.
++ */
++ nvmet_rdma_destroy_port_queues(port);
+ }
+
+ static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port)
+@@ -1878,7 +1902,7 @@ static void nvmet_rdma_repair_port_work(struct work_struct *w)
+ nvmet_rdma_disable_port(port);
+ ret = nvmet_rdma_enable_port(port);
+ if (ret)
+- schedule_delayed_work(&port->repair_work, 5 * HZ);
++ queue_delayed_work(nvmet_wq, &port->repair_work, 5 * HZ);
+ }
+
+ static int nvmet_rdma_add_port(struct nvmet_port *nport)
+@@ -2022,7 +2046,7 @@ static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data
+ }
+ mutex_unlock(&nvmet_rdma_queue_mutex);
+
+- flush_scheduled_work();
++ flush_workqueue(nvmet_wq);
+ }
+
+ static struct ib_client nvmet_rdma_ib_client = {
+diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
+index 46c3b3be7e033..f592e5f7f5f3d 100644
+--- a/drivers/nvme/target/tcp.c
++++ b/drivers/nvme/target/tcp.c
+@@ -700,10 +700,11 @@ static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
+ static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
+ {
+ struct nvmet_tcp_queue *queue = cmd->queue;
++ int left = NVME_TCP_DIGEST_LENGTH - cmd->offset;
+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
+ struct kvec iov = {
+ .iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset,
+- .iov_len = NVME_TCP_DIGEST_LENGTH - cmd->offset
++ .iov_len = left
+ };
+ int ret;
+
+@@ -717,6 +718,10 @@ static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
+ return ret;
+
+ cmd->offset += ret;
++ left -= ret;
++
++ if (left)
++ return -EAGAIN;
+
+ if (queue->nvme_sq.sqhd_disabled) {
+ cmd->queue->snd_cmd = NULL;
+@@ -1246,7 +1251,7 @@ static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
+ spin_lock(&queue->state_lock);
+ if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
+ queue->state = NVMET_TCP_Q_DISCONNECTING;
+- schedule_work(&queue->release_work);
++ queue_work(nvmet_wq, &queue->release_work);
+ }
+ spin_unlock(&queue->state_lock);
+ }
+@@ -1657,7 +1662,7 @@ static void nvmet_tcp_listen_data_ready(struct sock *sk)
+ goto out;
+
+ if (sk->sk_state == TCP_LISTEN)
+- schedule_work(&port->accept_work);
++ queue_work(nvmet_wq, &port->accept_work);
+ out:
+ read_unlock_bh(&sk->sk_callback_lock);
+ }
+@@ -1740,6 +1745,17 @@ err_port:
+ return ret;
+ }
+
++static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port)
++{
++ struct nvmet_tcp_queue *queue;
++
++ mutex_lock(&nvmet_tcp_queue_mutex);
++ list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
++ if (queue->port == port)
++ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
++ mutex_unlock(&nvmet_tcp_queue_mutex);
++}
++
+ static void nvmet_tcp_remove_port(struct nvmet_port *nport)
+ {
+ struct nvmet_tcp_port *port = nport->priv;
+@@ -1749,6 +1765,11 @@ static void nvmet_tcp_remove_port(struct nvmet_port *nport)
+ port->sock->sk->sk_user_data = NULL;
+ write_unlock_bh(&port->sock->sk->sk_callback_lock);
+ cancel_work_sync(&port->accept_work);
++ /*
++ * Destroy the remaining queues, which are not belong to any
++ * controller yet.
++ */
++ nvmet_tcp_destroy_port_queues(port);
+
+ sock_release(port->sock);
+ kfree(port);
+@@ -1772,7 +1793,7 @@ static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq)
+
+ if (sq->qid == 0) {
+ /* Let inflight controller teardown complete */
+- flush_scheduled_work();
++ flush_workqueue(nvmet_wq);
+ }
+
+ queue->nr_cmds = sq->size * 2;
+@@ -1833,12 +1854,12 @@ static void __exit nvmet_tcp_exit(void)
+
+ nvmet_unregister_transport(&nvmet_tcp_ops);
+
+- flush_scheduled_work();
++ flush_workqueue(nvmet_wq);
+ mutex_lock(&nvmet_tcp_queue_mutex);
+ list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
+ kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ mutex_unlock(&nvmet_tcp_queue_mutex);
+- flush_scheduled_work();
++ flush_workqueue(nvmet_wq);
+
+ destroy_workqueue(nvmet_tcp_wq);
+ }
+diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
+index 8976da38b375a..fb7840c73765a 100644
+--- a/drivers/nvmem/core.c
++++ b/drivers/nvmem/core.c
+@@ -307,6 +307,8 @@ static umode_t nvmem_bin_attr_is_visible(struct kobject *kobj,
+ struct device *dev = kobj_to_dev(kobj);
+ struct nvmem_device *nvmem = to_nvmem_device(dev);
+
++ attr->size = nvmem->size;
++
+ return nvmem_bin_attr_get_umode(nvmem);
+ }
+
+@@ -766,7 +768,7 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
+
+ if (config->wp_gpio)
+ nvmem->wp_gpio = config->wp_gpio;
+- else
++ else if (!config->ignore_wp)
+ nvmem->wp_gpio = gpiod_get_optional(config->dev, "wp",
+ GPIOD_OUT_HIGH);
+ if (IS_ERR(nvmem->wp_gpio)) {
+diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig
+index 3dfeae8912dfc..80b5fd44ab1c7 100644
+--- a/drivers/of/Kconfig
++++ b/drivers/of/Kconfig
+@@ -70,10 +70,6 @@ config OF_IRQ
+ def_bool y
+ depends on !SPARC && IRQ_DOMAIN
+
+-config OF_NET
+- depends on NETDEVICES
+- def_bool y
+-
+ config OF_RESERVED_MEM
+ def_bool OF_EARLY_FLATTREE
+
+diff --git a/drivers/of/Makefile b/drivers/of/Makefile
+index c13b982084a3a..e0360a44306e2 100644
+--- a/drivers/of/Makefile
++++ b/drivers/of/Makefile
+@@ -7,7 +7,6 @@ obj-$(CONFIG_OF_EARLY_FLATTREE) += fdt_address.o
+ obj-$(CONFIG_OF_PROMTREE) += pdt.o
+ obj-$(CONFIG_OF_ADDRESS) += address.o
+ obj-$(CONFIG_OF_IRQ) += irq.o
+-obj-$(CONFIG_OF_NET) += of_net.o
+ obj-$(CONFIG_OF_UNITTEST) += unittest.o
+ obj-$(CONFIG_OF_RESERVED_MEM) += of_reserved_mem.o
+ obj-$(CONFIG_OF_RESOLVE) += resolver.o
+diff --git a/drivers/of/base.c b/drivers/of/base.c
+index 0ac17256258d5..54719f8156ed1 100644
+--- a/drivers/of/base.c
++++ b/drivers/of/base.c
+@@ -1327,9 +1327,14 @@ int of_phandle_iterator_next(struct of_phandle_iterator *it)
+ * property data length
+ */
+ if (it->cur + count > it->list_end) {
+- pr_err("%pOF: %s = %d found %d\n",
+- it->parent, it->cells_name,
+- count, it->cell_count);
++ if (it->cells_name)
++ pr_err("%pOF: %s = %d found %td\n",
++ it->parent, it->cells_name,
++ count, it->list_end - it->cur);
++ else
++ pr_err("%pOF: phandle %s needs %d, found %td\n",
++ it->parent, of_node_full_name(it->node),
++ count, it->list_end - it->cur);
+ goto err;
+ }
+ }
+diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
+index 4546572af24bb..59a7a9ee58ef7 100644
+--- a/drivers/of/fdt.c
++++ b/drivers/of/fdt.c
+@@ -482,9 +482,11 @@ static int __init early_init_dt_reserve_memory_arch(phys_addr_t base,
+ if (nomap) {
+ /*
+ * If the memory is already reserved (by another region), we
+- * should not allow it to be marked nomap.
++ * should not allow it to be marked nomap, but don't worry
++ * if the region isn't memory as it won't be mapped.
+ */
+- if (memblock_is_region_reserved(base, size))
++ if (memblock_overlaps_region(&memblock.memory, base, size) &&
++ memblock_is_region_reserved(base, size))
+ return -EBUSY;
+
+ return memblock_mark_nomap(base, size);
+@@ -969,18 +971,22 @@ static void __init early_init_dt_check_for_elfcorehdr(unsigned long node)
+ elfcorehdr_addr, elfcorehdr_size);
+ }
+
+-static phys_addr_t cap_mem_addr;
+-static phys_addr_t cap_mem_size;
++static unsigned long chosen_node_offset = -FDT_ERR_NOTFOUND;
+
+ /**
+ * early_init_dt_check_for_usable_mem_range - Decode usable memory range
+ * location from flat tree
+- * @node: reference to node containing usable memory range location ('chosen')
+ */
+-static void __init early_init_dt_check_for_usable_mem_range(unsigned long node)
++void __init early_init_dt_check_for_usable_mem_range(void)
+ {
+ const __be32 *prop;
+ int len;
++ phys_addr_t cap_mem_addr;
++ phys_addr_t cap_mem_size;
++ unsigned long node = chosen_node_offset;
++
++ if ((long)node < 0)
++ return;
+
+ pr_debug("Looking for usable-memory-range property... ");
+
+@@ -993,6 +999,8 @@ static void __init early_init_dt_check_for_usable_mem_range(unsigned long node)
+
+ pr_debug("cap_mem_start=%pa cap_mem_size=%pa\n", &cap_mem_addr,
+ &cap_mem_size);
++
++ memblock_cap_memory_range(cap_mem_addr, cap_mem_size);
+ }
+
+ #ifdef CONFIG_SERIAL_EARLYCON
+@@ -1141,9 +1149,10 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname,
+ (strcmp(uname, "chosen") != 0 && strcmp(uname, "chosen@0") != 0))
+ return 0;
+
++ chosen_node_offset = node;
++
+ early_init_dt_check_for_initrd(node);
+ early_init_dt_check_for_elfcorehdr(node);
+- early_init_dt_check_for_usable_mem_range(node);
+
+ /* Retrieve command line */
+ p = of_get_flat_dt_prop(node, "bootargs", &l);
+@@ -1279,7 +1288,7 @@ void __init early_init_dt_scan_nodes(void)
+ of_scan_flat_dt(early_init_dt_scan_memory, NULL);
+
+ /* Handle linux,usable-memory-range property */
+- memblock_cap_memory_range(cap_mem_addr, cap_mem_size);
++ early_init_dt_check_for_usable_mem_range();
+ }
+
+ bool __init early_init_dt_scan(void *params)
+diff --git a/drivers/of/of_net.c b/drivers/of/of_net.c
+deleted file mode 100644
+index dbac3a172a11e..0000000000000
+--- a/drivers/of/of_net.c
++++ /dev/null
+@@ -1,145 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0-only
+-/*
+- * OF helpers for network devices.
+- *
+- * Initially copied out of arch/powerpc/kernel/prom_parse.c
+- */
+-#include <linux/etherdevice.h>
+-#include <linux/kernel.h>
+-#include <linux/of_net.h>
+-#include <linux/of_platform.h>
+-#include <linux/phy.h>
+-#include <linux/export.h>
+-#include <linux/device.h>
+-#include <linux/nvmem-consumer.h>
+-
+-/**
+- * of_get_phy_mode - Get phy mode for given device_node
+- * @np: Pointer to the given device_node
+- * @interface: Pointer to the result
+- *
+- * The function gets phy interface string from property 'phy-mode' or
+- * 'phy-connection-type'. The index in phy_modes table is set in
+- * interface and 0 returned. In case of error interface is set to
+- * PHY_INTERFACE_MODE_NA and an errno is returned, e.g. -ENODEV.
+- */
+-int of_get_phy_mode(struct device_node *np, phy_interface_t *interface)
+-{
+- const char *pm;
+- int err, i;
+-
+- *interface = PHY_INTERFACE_MODE_NA;
+-
+- err = of_property_read_string(np, "phy-mode", &pm);
+- if (err < 0)
+- err = of_property_read_string(np, "phy-connection-type", &pm);
+- if (err < 0)
+- return err;
+-
+- for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++)
+- if (!strcasecmp(pm, phy_modes(i))) {
+- *interface = i;
+- return 0;
+- }
+-
+- return -ENODEV;
+-}
+-EXPORT_SYMBOL_GPL(of_get_phy_mode);
+-
+-static int of_get_mac_addr(struct device_node *np, const char *name, u8 *addr)
+-{
+- struct property *pp = of_find_property(np, name, NULL);
+-
+- if (pp && pp->length == ETH_ALEN && is_valid_ether_addr(pp->value)) {
+- memcpy(addr, pp->value, ETH_ALEN);
+- return 0;
+- }
+- return -ENODEV;
+-}
+-
+-static int of_get_mac_addr_nvmem(struct device_node *np, u8 *addr)
+-{
+- struct platform_device *pdev = of_find_device_by_node(np);
+- struct nvmem_cell *cell;
+- const void *mac;
+- size_t len;
+- int ret;
+-
+- /* Try lookup by device first, there might be a nvmem_cell_lookup
+- * associated with a given device.
+- */
+- if (pdev) {
+- ret = nvmem_get_mac_address(&pdev->dev, addr);
+- put_device(&pdev->dev);
+- return ret;
+- }
+-
+- cell = of_nvmem_cell_get(np, "mac-address");
+- if (IS_ERR(cell))
+- return PTR_ERR(cell);
+-
+- mac = nvmem_cell_read(cell, &len);
+- nvmem_cell_put(cell);
+-
+- if (IS_ERR(mac))
+- return PTR_ERR(mac);
+-
+- if (len != ETH_ALEN || !is_valid_ether_addr(mac)) {
+- kfree(mac);
+- return -EINVAL;
+- }
+-
+- memcpy(addr, mac, ETH_ALEN);
+- kfree(mac);
+-
+- return 0;
+-}
+-
+-/**
+- * of_get_mac_address()
+- * @np: Caller's Device Node
+- * @addr: Pointer to a six-byte array for the result
+- *
+- * Search the device tree for the best MAC address to use. 'mac-address' is
+- * checked first, because that is supposed to contain to "most recent" MAC
+- * address. If that isn't set, then 'local-mac-address' is checked next,
+- * because that is the default address. If that isn't set, then the obsolete
+- * 'address' is checked, just in case we're using an old device tree. If any
+- * of the above isn't set, then try to get MAC address from nvmem cell named
+- * 'mac-address'.
+- *
+- * Note that the 'address' property is supposed to contain a virtual address of
+- * the register set, but some DTS files have redefined that property to be the
+- * MAC address.
+- *
+- * All-zero MAC addresses are rejected, because those could be properties that
+- * exist in the device tree, but were not set by U-Boot. For example, the
+- * DTS could define 'mac-address' and 'local-mac-address', with zero MAC
+- * addresses. Some older U-Boots only initialized 'local-mac-address'. In
+- * this case, the real MAC is in 'local-mac-address', and 'mac-address' exists
+- * but is all zeros.
+- *
+- * Return: 0 on success and errno in case of error.
+-*/
+-int of_get_mac_address(struct device_node *np, u8 *addr)
+-{
+- int ret;
+-
+- if (!np)
+- return -ENODEV;
+-
+- ret = of_get_mac_addr(np, "mac-address", addr);
+- if (!ret)
+- return 0;
+-
+- ret = of_get_mac_addr(np, "local-mac-address", addr);
+- if (!ret)
+- return 0;
+-
+- ret = of_get_mac_addr(np, "address", addr);
+- if (!ret)
+- return 0;
+-
+- return of_get_mac_addr_nvmem(np, addr);
+-}
+-EXPORT_SYMBOL(of_get_mac_address);
+diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
+index 8c056972a6ddc..2bee1d992408f 100644
+--- a/drivers/of/unittest.c
++++ b/drivers/of/unittest.c
+@@ -911,11 +911,18 @@ static void __init of_unittest_dma_ranges_one(const char *path,
+ if (!rc) {
+ phys_addr_t paddr;
+ dma_addr_t dma_addr;
+- struct device dev_bogus;
++ struct device *dev_bogus;
+
+- dev_bogus.dma_range_map = map;
+- paddr = dma_to_phys(&dev_bogus, expect_dma_addr);
+- dma_addr = phys_to_dma(&dev_bogus, expect_paddr);
++ dev_bogus = kzalloc(sizeof(struct device), GFP_KERNEL);
++ if (!dev_bogus) {
++ unittest(0, "kzalloc() failed\n");
++ kfree(map);
++ return;
++ }
++
++ dev_bogus->dma_range_map = map;
++ paddr = dma_to_phys(dev_bogus, expect_dma_addr);
++ dma_addr = phys_to_dma(dev_bogus, expect_paddr);
+
+ unittest(paddr == expect_paddr,
+ "of_dma_get_range: wrong phys addr %pap (expecting %llx) on node %pOF\n",
+@@ -925,6 +932,7 @@ static void __init of_unittest_dma_ranges_one(const char *path,
+ &dma_addr, expect_dma_addr, np);
+
+ kfree(map);
++ kfree(dev_bogus);
+ }
+ of_node_put(np);
+ #endif
+@@ -934,8 +942,9 @@ static void __init of_unittest_parse_dma_ranges(void)
+ {
+ of_unittest_dma_ranges_one("/testcase-data/address-tests/device@70000000",
+ 0x0, 0x20000000);
+- of_unittest_dma_ranges_one("/testcase-data/address-tests/bus@80000000/device@1000",
+- 0x100000000, 0x20000000);
++ if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
++ of_unittest_dma_ranges_one("/testcase-data/address-tests/bus@80000000/device@1000",
++ 0x100000000, 0x20000000);
+ of_unittest_dma_ranges_one("/testcase-data/address-tests/pci@90000000",
+ 0x80000000, 0x20000000);
+ }
+@@ -1688,19 +1697,19 @@ static void __init of_unittest_overlay_gpio(void)
+ */
+
+ EXPECT_BEGIN(KERN_INFO,
+- "GPIO line <<int>> (line-B-input) hogged as input\n");
++ "gpio-<<int>> (line-B-input): hogged as input\n");
+
+ EXPECT_BEGIN(KERN_INFO,
+- "GPIO line <<int>> (line-A-input) hogged as input\n");
++ "gpio-<<int>> (line-A-input): hogged as input\n");
+
+ ret = platform_driver_register(&unittest_gpio_driver);
+ if (unittest(ret == 0, "could not register unittest gpio driver\n"))
+ return;
+
+ EXPECT_END(KERN_INFO,
+- "GPIO line <<int>> (line-A-input) hogged as input\n");
++ "gpio-<<int>> (line-A-input): hogged as input\n");
+ EXPECT_END(KERN_INFO,
+- "GPIO line <<int>> (line-B-input) hogged as input\n");
++ "gpio-<<int>> (line-B-input): hogged as input\n");
+
+ unittest(probe_pass_count + 2 == unittest_gpio_probe_pass_count,
+ "unittest_gpio_probe() failed or not called\n");
+@@ -1727,7 +1736,7 @@ static void __init of_unittest_overlay_gpio(void)
+ chip_request_count = unittest_gpio_chip_request_count;
+
+ EXPECT_BEGIN(KERN_INFO,
+- "GPIO line <<int>> (line-D-input) hogged as input\n");
++ "gpio-<<int>> (line-D-input): hogged as input\n");
+
+ /* overlay_gpio_03 contains gpio node and child gpio hog node */
+
+@@ -1735,7 +1744,7 @@ static void __init of_unittest_overlay_gpio(void)
+ "Adding overlay 'overlay_gpio_03' failed\n");
+
+ EXPECT_END(KERN_INFO,
+- "GPIO line <<int>> (line-D-input) hogged as input\n");
++ "gpio-<<int>> (line-D-input): hogged as input\n");
+
+ unittest(probe_pass_count + 1 == unittest_gpio_probe_pass_count,
+ "unittest_gpio_probe() failed or not called\n");
+@@ -1774,7 +1783,7 @@ static void __init of_unittest_overlay_gpio(void)
+ */
+
+ EXPECT_BEGIN(KERN_INFO,
+- "GPIO line <<int>> (line-C-input) hogged as input\n");
++ "gpio-<<int>> (line-C-input): hogged as input\n");
+
+ /* overlay_gpio_04b contains child gpio hog node */
+
+@@ -1782,7 +1791,7 @@ static void __init of_unittest_overlay_gpio(void)
+ "Adding overlay 'overlay_gpio_04b' failed\n");
+
+ EXPECT_END(KERN_INFO,
+- "GPIO line <<int>> (line-C-input) hogged as input\n");
++ "gpio-<<int>> (line-C-input): hogged as input\n");
+
+ unittest(chip_request_count + 1 == unittest_gpio_chip_request_count,
+ "unittest_gpio_chip_request() called %d times (expected 1 time)\n",
+diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c
+index 596c185b5dda4..b5f2f9f393926 100644
+--- a/drivers/opp/debugfs.c
++++ b/drivers/opp/debugfs.c
+@@ -10,6 +10,7 @@
+ #include <linux/debugfs.h>
+ #include <linux/device.h>
+ #include <linux/err.h>
++#include <linux/of.h>
+ #include <linux/init.h>
+ #include <linux/limits.h>
+ #include <linux/slab.h>
+@@ -131,9 +132,13 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
+ debugfs_create_bool("suspend", S_IRUGO, d, &opp->suspend);
+ debugfs_create_u32("performance_state", S_IRUGO, d, &opp->pstate);
+ debugfs_create_ulong("rate_hz", S_IRUGO, d, &opp->rate);
++ debugfs_create_u32("level", S_IRUGO, d, &opp->level);
+ debugfs_create_ulong("clock_latency_ns", S_IRUGO, d,
+ &opp->clock_latency_ns);
+
++ opp->of_name = of_node_full_name(opp->np);
++ debugfs_create_str("of_name", S_IRUGO, d, (char **)&opp->of_name);
++
+ opp_debug_create_supplies(opp, opp_table, d);
+ opp_debug_create_bw(opp, opp_table, d);
+
+diff --git a/drivers/opp/of.c b/drivers/opp/of.c
+index 2a97c6535c4c6..c32ae7497392b 100644
+--- a/drivers/opp/of.c
++++ b/drivers/opp/of.c
+@@ -921,7 +921,7 @@ free_required_opps:
+ free_opp:
+ _opp_free(new_opp);
+
+- return ERR_PTR(ret);
++ return ret ? ERR_PTR(ret) : NULL;
+ }
+
+ /* Initializes OPP tables based on new bindings */
+diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h
+index 407c3bfe51d96..45e3a55239a13 100644
+--- a/drivers/opp/opp.h
++++ b/drivers/opp/opp.h
+@@ -96,6 +96,7 @@ struct dev_pm_opp {
+
+ #ifdef CONFIG_DEBUG_FS
+ struct dentry *dentry;
++ const char *of_name;
+ #endif
+ };
+
+diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c
+index 059566f544291..9be007c9420f9 100644
+--- a/drivers/parisc/ccio-dma.c
++++ b/drivers/parisc/ccio-dma.c
+@@ -1003,7 +1003,7 @@ ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
+ ioc->usg_calls++;
+ #endif
+
+- while(sg_dma_len(sglist) && nents--) {
++ while (nents && sg_dma_len(sglist)) {
+
+ #ifdef CCIO_COLLECT_STATS
+ ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT;
+@@ -1011,6 +1011,7 @@ ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
+ ccio_unmap_page(dev, sg_dma_address(sglist),
+ sg_dma_len(sglist), direction, 0);
+ ++sglist;
++ nents--;
+ }
+
+ DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
+diff --git a/drivers/parisc/dino.c b/drivers/parisc/dino.c
+index 952a92504df69..e33036281327d 100644
+--- a/drivers/parisc/dino.c
++++ b/drivers/parisc/dino.c
+@@ -142,9 +142,8 @@ struct dino_device
+ {
+ struct pci_hba_data hba; /* 'C' inheritance - must be first */
+ spinlock_t dinosaur_pen;
+- unsigned long txn_addr; /* EIR addr to generate interrupt */
+- u32 txn_data; /* EIR data assign to each dino */
+ u32 imr; /* IRQ's which are enabled */
++ struct gsc_irq gsc_irq;
+ int global_irq[DINO_LOCAL_IRQS]; /* map IMR bit to global irq */
+ #ifdef DINO_DEBUG
+ unsigned int dino_irr0; /* save most recent IRQ line stat */
+@@ -339,14 +338,43 @@ static void dino_unmask_irq(struct irq_data *d)
+ if (tmp & DINO_MASK_IRQ(local_irq)) {
+ DBG(KERN_WARNING "%s(): IRQ asserted! (ILR 0x%x)\n",
+ __func__, tmp);
+- gsc_writel(dino_dev->txn_data, dino_dev->txn_addr);
++ gsc_writel(dino_dev->gsc_irq.txn_data, dino_dev->gsc_irq.txn_addr);
+ }
+ }
+
++#ifdef CONFIG_SMP
++static int dino_set_affinity_irq(struct irq_data *d, const struct cpumask *dest,
++ bool force)
++{
++ struct dino_device *dino_dev = irq_data_get_irq_chip_data(d);
++ struct cpumask tmask;
++ int cpu_irq;
++ u32 eim;
++
++ if (!cpumask_and(&tmask, dest, cpu_online_mask))
++ return -EINVAL;
++
++ cpu_irq = cpu_check_affinity(d, &tmask);
++ if (cpu_irq < 0)
++ return cpu_irq;
++
++ dino_dev->gsc_irq.txn_addr = txn_affinity_addr(d->irq, cpu_irq);
++ eim = ((u32) dino_dev->gsc_irq.txn_addr) | dino_dev->gsc_irq.txn_data;
++ __raw_writel(eim, dino_dev->hba.base_addr+DINO_IAR0);
++
++ irq_data_update_effective_affinity(d, &tmask);
++
++ return IRQ_SET_MASK_OK;
++}
++#endif
++
+ static struct irq_chip dino_interrupt_type = {
+ .name = "GSC-PCI",
+ .irq_unmask = dino_unmask_irq,
+ .irq_mask = dino_mask_irq,
++#ifdef CONFIG_SMP
++ .irq_set_affinity = dino_set_affinity_irq,
++#endif
+ };
+
+
+@@ -806,7 +834,6 @@ static int __init dino_common_init(struct parisc_device *dev,
+ {
+ int status;
+ u32 eim;
+- struct gsc_irq gsc_irq;
+ struct resource *res;
+
+ pcibios_register_hba(&dino_dev->hba);
+@@ -821,10 +848,8 @@ static int __init dino_common_init(struct parisc_device *dev,
+ ** still only has 11 IRQ input lines - just map some of them
+ ** to a different processor.
+ */
+- dev->irq = gsc_alloc_irq(&gsc_irq);
+- dino_dev->txn_addr = gsc_irq.txn_addr;
+- dino_dev->txn_data = gsc_irq.txn_data;
+- eim = ((u32) gsc_irq.txn_addr) | gsc_irq.txn_data;
++ dev->irq = gsc_alloc_irq(&dino_dev->gsc_irq);
++ eim = ((u32) dino_dev->gsc_irq.txn_addr) | dino_dev->gsc_irq.txn_data;
+
+ /*
+ ** Dino needs a PA "IRQ" to get a processor's attention.
+diff --git a/drivers/parisc/gsc.c b/drivers/parisc/gsc.c
+index ed9371acf37eb..ec175ae998733 100644
+--- a/drivers/parisc/gsc.c
++++ b/drivers/parisc/gsc.c
+@@ -135,10 +135,41 @@ static void gsc_asic_unmask_irq(struct irq_data *d)
+ */
+ }
+
++#ifdef CONFIG_SMP
++static int gsc_set_affinity_irq(struct irq_data *d, const struct cpumask *dest,
++ bool force)
++{
++ struct gsc_asic *gsc_dev = irq_data_get_irq_chip_data(d);
++ struct cpumask tmask;
++ int cpu_irq;
++
++ if (!cpumask_and(&tmask, dest, cpu_online_mask))
++ return -EINVAL;
++
++ cpu_irq = cpu_check_affinity(d, &tmask);
++ if (cpu_irq < 0)
++ return cpu_irq;
++
++ gsc_dev->gsc_irq.txn_addr = txn_affinity_addr(d->irq, cpu_irq);
++ gsc_dev->eim = ((u32) gsc_dev->gsc_irq.txn_addr) | gsc_dev->gsc_irq.txn_data;
++
++ /* switch IRQ's for devices below LASI/WAX to other CPU */
++ gsc_writel(gsc_dev->eim, gsc_dev->hpa + OFFSET_IAR);
++
++ irq_data_update_effective_affinity(d, &tmask);
++
++ return IRQ_SET_MASK_OK;
++}
++#endif
++
++
+ static struct irq_chip gsc_asic_interrupt_type = {
+ .name = "GSC-ASIC",
+ .irq_unmask = gsc_asic_unmask_irq,
+ .irq_mask = gsc_asic_mask_irq,
++#ifdef CONFIG_SMP
++ .irq_set_affinity = gsc_set_affinity_irq,
++#endif
+ };
+
+ int gsc_assign_irq(struct irq_chip *type, void *data)
+diff --git a/drivers/parisc/gsc.h b/drivers/parisc/gsc.h
+index 86abad3fa2150..73cbd0bb1975a 100644
+--- a/drivers/parisc/gsc.h
++++ b/drivers/parisc/gsc.h
+@@ -31,6 +31,7 @@ struct gsc_asic {
+ int version;
+ int type;
+ int eim;
++ struct gsc_irq gsc_irq;
+ int global_irq[32];
+ };
+
+diff --git a/drivers/parisc/lasi.c b/drivers/parisc/lasi.c
+index 4e4fd12c2112e..6ef621adb63a8 100644
+--- a/drivers/parisc/lasi.c
++++ b/drivers/parisc/lasi.c
+@@ -163,7 +163,6 @@ static int __init lasi_init_chip(struct parisc_device *dev)
+ {
+ extern void (*chassis_power_off)(void);
+ struct gsc_asic *lasi;
+- struct gsc_irq gsc_irq;
+ int ret;
+
+ lasi = kzalloc(sizeof(*lasi), GFP_KERNEL);
+@@ -185,7 +184,7 @@ static int __init lasi_init_chip(struct parisc_device *dev)
+ lasi_init_irq(lasi);
+
+ /* the IRQ lasi should use */
+- dev->irq = gsc_alloc_irq(&gsc_irq);
++ dev->irq = gsc_alloc_irq(&lasi->gsc_irq);
+ if (dev->irq < 0) {
+ printk(KERN_ERR "%s(): cannot get GSC irq\n",
+ __func__);
+@@ -193,9 +192,9 @@ static int __init lasi_init_chip(struct parisc_device *dev)
+ return -EBUSY;
+ }
+
+- lasi->eim = ((u32) gsc_irq.txn_addr) | gsc_irq.txn_data;
++ lasi->eim = ((u32) lasi->gsc_irq.txn_addr) | lasi->gsc_irq.txn_data;
+
+- ret = request_irq(gsc_irq.irq, gsc_asic_intr, 0, "lasi", lasi);
++ ret = request_irq(lasi->gsc_irq.irq, gsc_asic_intr, 0, "lasi", lasi);
+ if (ret < 0) {
+ kfree(lasi);
+ return ret;
+diff --git a/drivers/parisc/pdc_stable.c b/drivers/parisc/pdc_stable.c
+index e090978518f1a..4760f82def6ec 100644
+--- a/drivers/parisc/pdc_stable.c
++++ b/drivers/parisc/pdc_stable.c
+@@ -979,8 +979,10 @@ pdcs_register_pathentries(void)
+ entry->kobj.kset = paths_kset;
+ err = kobject_init_and_add(&entry->kobj, &ktype_pdcspath, NULL,
+ "%s", entry->name);
+- if (err)
++ if (err) {
++ kobject_put(&entry->kobj);
+ return err;
++ }
+
+ /* kobject is now registered */
+ write_lock(&entry->rw_lock);
+diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
+index e60690d38d677..374b9199878d4 100644
+--- a/drivers/parisc/sba_iommu.c
++++ b/drivers/parisc/sba_iommu.c
+@@ -1047,7 +1047,7 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
+ spin_unlock_irqrestore(&ioc->res_lock, flags);
+ #endif
+
+- while (sg_dma_len(sglist) && nents--) {
++ while (nents && sg_dma_len(sglist)) {
+
+ sba_unmap_page(dev, sg_dma_address(sglist), sg_dma_len(sglist),
+ direction, 0);
+@@ -1056,6 +1056,7 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
+ ioc->usingle_calls--; /* kluge since call is unmap_sg() */
+ #endif
+ ++sglist;
++ nents--;
+ }
+
+ DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
+diff --git a/drivers/parisc/wax.c b/drivers/parisc/wax.c
+index 5b6df15162354..73a2b01f8d9ca 100644
+--- a/drivers/parisc/wax.c
++++ b/drivers/parisc/wax.c
+@@ -68,7 +68,6 @@ static int __init wax_init_chip(struct parisc_device *dev)
+ {
+ struct gsc_asic *wax;
+ struct parisc_device *parent;
+- struct gsc_irq gsc_irq;
+ int ret;
+
+ wax = kzalloc(sizeof(*wax), GFP_KERNEL);
+@@ -85,7 +84,7 @@ static int __init wax_init_chip(struct parisc_device *dev)
+ wax_init_irq(wax);
+
+ /* the IRQ wax should use */
+- dev->irq = gsc_claim_irq(&gsc_irq, WAX_GSC_IRQ);
++ dev->irq = gsc_claim_irq(&wax->gsc_irq, WAX_GSC_IRQ);
+ if (dev->irq < 0) {
+ printk(KERN_ERR "%s(): cannot get GSC irq\n",
+ __func__);
+@@ -93,9 +92,9 @@ static int __init wax_init_chip(struct parisc_device *dev)
+ return -EBUSY;
+ }
+
+- wax->eim = ((u32) gsc_irq.txn_addr) | gsc_irq.txn_data;
++ wax->eim = ((u32) wax->gsc_irq.txn_addr) | wax->gsc_irq.txn_data;
+
+- ret = request_irq(gsc_irq.irq, gsc_asic_intr, 0, "wax", wax);
++ ret = request_irq(wax->gsc_irq.irq, gsc_asic_intr, 0, "wax", wax);
+ if (ret < 0) {
+ kfree(wax);
+ return ret;
+diff --git a/drivers/pci/access.c b/drivers/pci/access.c
+index 46935695cfb90..8d0d1f61c650d 100644
+--- a/drivers/pci/access.c
++++ b/drivers/pci/access.c
+@@ -160,9 +160,12 @@ int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
+ * write happen to have any RW1C (write-one-to-clear) bits set, we
+ * just inadvertently cleared something we shouldn't have.
+ */
+- dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
+- size, pci_domain_nr(bus), bus->number,
+- PCI_SLOT(devfn), PCI_FUNC(devfn), where);
++ if (!bus->unsafe_warn) {
++ dev_warn(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
++ size, pci_domain_nr(bus), bus->number,
++ PCI_SLOT(devfn), PCI_FUNC(devfn), where);
++ bus->unsafe_warn = 1;
++ }
+
+ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
+ tmp = readl(addr) & mask;
+diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c
+index ffb176d288cd9..918e11082e6a7 100644
+--- a/drivers/pci/controller/cadence/pci-j721e.c
++++ b/drivers/pci/controller/cadence/pci-j721e.c
+@@ -474,7 +474,7 @@ static int j721e_pcie_probe(struct platform_device *pdev)
+ ret = clk_prepare_enable(clk);
+ if (ret) {
+ dev_err(dev, "failed to enable pcie_refclk\n");
+- goto err_get_sync;
++ goto err_pcie_setup;
+ }
+ pcie->refclk = clk;
+
+diff --git a/drivers/pci/controller/cadence/pcie-cadence-plat.c b/drivers/pci/controller/cadence/pcie-cadence-plat.c
+index 5fee0f89ab594..a224afadbcc00 100644
+--- a/drivers/pci/controller/cadence/pcie-cadence-plat.c
++++ b/drivers/pci/controller/cadence/pcie-cadence-plat.c
+@@ -127,6 +127,8 @@ static int cdns_plat_pcie_probe(struct platform_device *pdev)
+ goto err_init;
+ }
+
++ return 0;
++
+ err_init:
+ err_get_sync:
+ pm_runtime_put_sync(dev);
+diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
+index 80fc98acf097f..2c05f2f7d1c0b 100644
+--- a/drivers/pci/controller/dwc/pci-imx6.c
++++ b/drivers/pci/controller/dwc/pci-imx6.c
+@@ -779,9 +779,7 @@ static int imx6_pcie_start_link(struct dw_pcie *pci)
+ /* Start LTSSM. */
+ imx6_pcie_ltssm_enable(dev);
+
+- ret = dw_pcie_wait_for_link(pci);
+- if (ret)
+- goto err_reset_phy;
++ dw_pcie_wait_for_link(pci);
+
+ if (pci->link_gen == 2) {
+ /* Allow Gen2 mode after the link is up. */
+@@ -817,11 +815,7 @@ static int imx6_pcie_start_link(struct dw_pcie *pci)
+ }
+
+ /* Make sure link training is finished as well! */
+- ret = dw_pcie_wait_for_link(pci);
+- if (ret) {
+- dev_err(dev, "Failed to bring link up!\n");
+- goto err_reset_phy;
+- }
++ dw_pcie_wait_for_link(pci);
+ } else {
+ dev_info(dev, "Link: Gen2 disabled\n");
+ }
+diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
+index a945f0c0e73dc..3254f60d1713f 100644
+--- a/drivers/pci/controller/dwc/pcie-designware.c
++++ b/drivers/pci/controller/dwc/pcie-designware.c
+@@ -671,10 +671,11 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci)
+ if (!pci->atu_base) {
+ struct resource *res =
+ platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu");
+- if (res)
++ if (res) {
+ pci->atu_size = resource_size(res);
+- pci->atu_base = devm_ioremap_resource(dev, res);
+- if (IS_ERR(pci->atu_base))
++ pci->atu_base = devm_ioremap_resource(dev, res);
++ }
++ if (!pci->atu_base || IS_ERR(pci->atu_base))
+ pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET;
+ }
+
+diff --git a/drivers/pci/controller/dwc/pcie-fu740.c b/drivers/pci/controller/dwc/pcie-fu740.c
+index 00cde9a248b5a..78d002be4f821 100644
+--- a/drivers/pci/controller/dwc/pcie-fu740.c
++++ b/drivers/pci/controller/dwc/pcie-fu740.c
+@@ -181,10 +181,59 @@ static int fu740_pcie_start_link(struct dw_pcie *pci)
+ {
+ struct device *dev = pci->dev;
+ struct fu740_pcie *afp = dev_get_drvdata(dev);
++ u8 cap_exp = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
++ int ret;
++ u32 orig, tmp;
++
++ /*
++ * Force 2.5GT/s when starting the link, due to some devices not
++ * probing at higher speeds. This happens with the PCIe switch
++ * on the Unmatched board when U-Boot has not initialised the PCIe.
++ * The fix in U-Boot is to force 2.5GT/s, which then gets cleared
++ * by the soft reset done by this driver.
++ */
++ dev_dbg(dev, "cap_exp at %x\n", cap_exp);
++ dw_pcie_dbi_ro_wr_en(pci);
++
++ tmp = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCAP);
++ orig = tmp & PCI_EXP_LNKCAP_SLS;
++ tmp &= ~PCI_EXP_LNKCAP_SLS;
++ tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;
++ dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCAP, tmp);
+
+ /* Enable LTSSM */
+ writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_APP_LTSSM_ENABLE);
+- return 0;
++
++ ret = dw_pcie_wait_for_link(pci);
++ if (ret) {
++ dev_err(dev, "error: link did not start\n");
++ goto err;
++ }
++
++ tmp = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCAP);
++ if ((tmp & PCI_EXP_LNKCAP_SLS) != orig) {
++ dev_dbg(dev, "changing speed back to original\n");
++
++ tmp &= ~PCI_EXP_LNKCAP_SLS;
++ tmp |= orig;
++ dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCAP, tmp);
++
++ tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
++ tmp |= PORT_LOGIC_SPEED_CHANGE;
++ dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
++
++ ret = dw_pcie_wait_for_link(pci);
++ if (ret) {
++ dev_err(dev, "error: link did not start at new speed\n");
++ goto err;
++ }
++ }
++
++ ret = 0;
++err:
++ WARN_ON(ret); /* we assume that errors will be very rare */
++ dw_pcie_dbi_ro_wr_dis(pci);
++ return ret;
+ }
+
+ static int fu740_pcie_host_init(struct pcie_port *pp)
+diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c
+index d842fd0181299..d05be942956e2 100644
+--- a/drivers/pci/controller/dwc/pcie-uniphier.c
++++ b/drivers/pci/controller/dwc/pcie-uniphier.c
+@@ -168,30 +168,21 @@ static void uniphier_pcie_irq_enable(struct uniphier_pcie_priv *priv)
+ writel(PCL_RCV_INTX_ALL_ENABLE, priv->base + PCL_RCV_INTX);
+ }
+
+-static void uniphier_pcie_irq_ack(struct irq_data *d)
+-{
+- struct pcie_port *pp = irq_data_get_irq_chip_data(d);
+- struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+- struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci);
+- u32 val;
+-
+- val = readl(priv->base + PCL_RCV_INTX);
+- val &= ~PCL_RCV_INTX_ALL_STATUS;
+- val |= BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_STATUS_SHIFT);
+- writel(val, priv->base + PCL_RCV_INTX);
+-}
+-
+ static void uniphier_pcie_irq_mask(struct irq_data *d)
+ {
+ struct pcie_port *pp = irq_data_get_irq_chip_data(d);
+ struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+ struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci);
++ unsigned long flags;
+ u32 val;
+
++ raw_spin_lock_irqsave(&pp->lock, flags);
++
+ val = readl(priv->base + PCL_RCV_INTX);
+- val &= ~PCL_RCV_INTX_ALL_MASK;
+ val |= BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT);
+ writel(val, priv->base + PCL_RCV_INTX);
++
++ raw_spin_unlock_irqrestore(&pp->lock, flags);
+ }
+
+ static void uniphier_pcie_irq_unmask(struct irq_data *d)
+@@ -199,17 +190,20 @@ static void uniphier_pcie_irq_unmask(struct irq_data *d)
+ struct pcie_port *pp = irq_data_get_irq_chip_data(d);
+ struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+ struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci);
++ unsigned long flags;
+ u32 val;
+
++ raw_spin_lock_irqsave(&pp->lock, flags);
++
+ val = readl(priv->base + PCL_RCV_INTX);
+- val &= ~PCL_RCV_INTX_ALL_MASK;
+ val &= ~BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT);
+ writel(val, priv->base + PCL_RCV_INTX);
++
++ raw_spin_unlock_irqrestore(&pp->lock, flags);
+ }
+
+ static struct irq_chip uniphier_pcie_irq_chip = {
+ .name = "PCI",
+- .irq_ack = uniphier_pcie_irq_ack,
+ .irq_mask = uniphier_pcie_irq_mask,
+ .irq_unmask = uniphier_pcie_irq_unmask,
+ };
+diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c
+index 596ebcfcc82dc..7cc2c54daad03 100644
+--- a/drivers/pci/controller/pci-aardvark.c
++++ b/drivers/pci/controller/pci-aardvark.c
+@@ -31,9 +31,6 @@
+ /* PCIe core registers */
+ #define PCIE_CORE_DEV_ID_REG 0x0
+ #define PCIE_CORE_CMD_STATUS_REG 0x4
+-#define PCIE_CORE_CMD_IO_ACCESS_EN BIT(0)
+-#define PCIE_CORE_CMD_MEM_ACCESS_EN BIT(1)
+-#define PCIE_CORE_CMD_MEM_IO_REQ_EN BIT(2)
+ #define PCIE_CORE_DEV_REV_REG 0x8
+ #define PCIE_CORE_PCIEXP_CAP 0xc0
+ #define PCIE_CORE_ERR_CAPCTL_REG 0x118
+@@ -41,10 +38,6 @@
+ #define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN BIT(6)
+ #define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK BIT(7)
+ #define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV BIT(8)
+-#define PCIE_CORE_INT_A_ASSERT_ENABLE 1
+-#define PCIE_CORE_INT_B_ASSERT_ENABLE 2
+-#define PCIE_CORE_INT_C_ASSERT_ENABLE 3
+-#define PCIE_CORE_INT_D_ASSERT_ENABLE 4
+ /* PIO registers base address and register offsets */
+ #define PIO_BASE_ADDR 0x4000
+ #define PIO_CTRL (PIO_BASE_ADDR + 0x0)
+@@ -99,25 +92,32 @@
+ #define PCIE_CORE_CTRL2_MSI_ENABLE BIT(10)
+ #define PCIE_CORE_REF_CLK_REG (CONTROL_BASE_ADDR + 0x14)
+ #define PCIE_CORE_REF_CLK_TX_ENABLE BIT(1)
++#define PCIE_CORE_REF_CLK_RX_ENABLE BIT(2)
+ #define PCIE_MSG_LOG_REG (CONTROL_BASE_ADDR + 0x30)
+ #define PCIE_ISR0_REG (CONTROL_BASE_ADDR + 0x40)
+ #define PCIE_MSG_PM_PME_MASK BIT(7)
+ #define PCIE_ISR0_MASK_REG (CONTROL_BASE_ADDR + 0x44)
+ #define PCIE_ISR0_MSI_INT_PENDING BIT(24)
++#define PCIE_ISR0_CORR_ERR BIT(11)
++#define PCIE_ISR0_NFAT_ERR BIT(12)
++#define PCIE_ISR0_FAT_ERR BIT(13)
++#define PCIE_ISR0_ERR_MASK GENMASK(13, 11)
+ #define PCIE_ISR0_INTX_ASSERT(val) BIT(16 + (val))
+ #define PCIE_ISR0_INTX_DEASSERT(val) BIT(20 + (val))
+-#define PCIE_ISR0_ALL_MASK GENMASK(26, 0)
++#define PCIE_ISR0_ALL_MASK GENMASK(31, 0)
+ #define PCIE_ISR1_REG (CONTROL_BASE_ADDR + 0x48)
+ #define PCIE_ISR1_MASK_REG (CONTROL_BASE_ADDR + 0x4C)
+ #define PCIE_ISR1_POWER_STATE_CHANGE BIT(4)
+ #define PCIE_ISR1_FLUSH BIT(5)
+ #define PCIE_ISR1_INTX_ASSERT(val) BIT(8 + (val))
+-#define PCIE_ISR1_ALL_MASK GENMASK(11, 4)
++#define PCIE_ISR1_ALL_MASK GENMASK(31, 0)
+ #define PCIE_MSI_ADDR_LOW_REG (CONTROL_BASE_ADDR + 0x50)
+ #define PCIE_MSI_ADDR_HIGH_REG (CONTROL_BASE_ADDR + 0x54)
+ #define PCIE_MSI_STATUS_REG (CONTROL_BASE_ADDR + 0x58)
+ #define PCIE_MSI_MASK_REG (CONTROL_BASE_ADDR + 0x5C)
++#define PCIE_MSI_ALL_MASK GENMASK(31, 0)
+ #define PCIE_MSI_PAYLOAD_REG (CONTROL_BASE_ADDR + 0x9C)
++#define PCIE_MSI_DATA_MASK GENMASK(15, 0)
+
+ /* PCIe window configuration */
+ #define OB_WIN_BASE_ADDR 0x4c00
+@@ -164,8 +164,50 @@
+ #define CFG_REG (LMI_BASE_ADDR + 0x0)
+ #define LTSSM_SHIFT 24
+ #define LTSSM_MASK 0x3f
+-#define LTSSM_L0 0x10
+ #define RC_BAR_CONFIG 0x300
++
++/* LTSSM values in CFG_REG */
++enum {
++ LTSSM_DETECT_QUIET = 0x0,
++ LTSSM_DETECT_ACTIVE = 0x1,
++ LTSSM_POLLING_ACTIVE = 0x2,
++ LTSSM_POLLING_COMPLIANCE = 0x3,
++ LTSSM_POLLING_CONFIGURATION = 0x4,
++ LTSSM_CONFIG_LINKWIDTH_START = 0x5,
++ LTSSM_CONFIG_LINKWIDTH_ACCEPT = 0x6,
++ LTSSM_CONFIG_LANENUM_ACCEPT = 0x7,
++ LTSSM_CONFIG_LANENUM_WAIT = 0x8,
++ LTSSM_CONFIG_COMPLETE = 0x9,
++ LTSSM_CONFIG_IDLE = 0xa,
++ LTSSM_RECOVERY_RCVR_LOCK = 0xb,
++ LTSSM_RECOVERY_SPEED = 0xc,
++ LTSSM_RECOVERY_RCVR_CFG = 0xd,
++ LTSSM_RECOVERY_IDLE = 0xe,
++ LTSSM_L0 = 0x10,
++ LTSSM_RX_L0S_ENTRY = 0x11,
++ LTSSM_RX_L0S_IDLE = 0x12,
++ LTSSM_RX_L0S_FTS = 0x13,
++ LTSSM_TX_L0S_ENTRY = 0x14,
++ LTSSM_TX_L0S_IDLE = 0x15,
++ LTSSM_TX_L0S_FTS = 0x16,
++ LTSSM_L1_ENTRY = 0x17,
++ LTSSM_L1_IDLE = 0x18,
++ LTSSM_L2_IDLE = 0x19,
++ LTSSM_L2_TRANSMIT_WAKE = 0x1a,
++ LTSSM_DISABLED = 0x20,
++ LTSSM_LOOPBACK_ENTRY_MASTER = 0x21,
++ LTSSM_LOOPBACK_ACTIVE_MASTER = 0x22,
++ LTSSM_LOOPBACK_EXIT_MASTER = 0x23,
++ LTSSM_LOOPBACK_ENTRY_SLAVE = 0x24,
++ LTSSM_LOOPBACK_ACTIVE_SLAVE = 0x25,
++ LTSSM_LOOPBACK_EXIT_SLAVE = 0x26,
++ LTSSM_HOT_RESET = 0x27,
++ LTSSM_RECOVERY_EQUALIZATION_PHASE0 = 0x28,
++ LTSSM_RECOVERY_EQUALIZATION_PHASE1 = 0x29,
++ LTSSM_RECOVERY_EQUALIZATION_PHASE2 = 0x2a,
++ LTSSM_RECOVERY_EQUALIZATION_PHASE3 = 0x2b,
++};
++
+ #define VENDOR_ID_REG (LMI_BASE_ADDR + 0x44)
+
+ /* PCIe core controller registers */
+@@ -198,7 +240,7 @@
+ #define PCIE_IRQ_MSI_INT2_DET BIT(21)
+ #define PCIE_IRQ_RC_DBELL_DET BIT(22)
+ #define PCIE_IRQ_EP_STATUS BIT(23)
+-#define PCIE_IRQ_ALL_MASK 0xfff0fb
++#define PCIE_IRQ_ALL_MASK GENMASK(31, 0)
+ #define PCIE_IRQ_ENABLE_INTS_MASK PCIE_IRQ_CORE_INT
+
+ /* Transaction types */
+@@ -230,17 +272,15 @@ struct advk_pcie {
+ u32 actions;
+ } wins[OB_WIN_COUNT];
+ u8 wins_count;
++ struct irq_domain *rp_irq_domain;
+ struct irq_domain *irq_domain;
+ struct irq_chip irq_chip;
+ raw_spinlock_t irq_lock;
+ struct irq_domain *msi_domain;
+ struct irq_domain *msi_inner_domain;
+- struct irq_chip msi_bottom_irq_chip;
+- struct irq_chip msi_irq_chip;
+- struct msi_domain_info msi_domain_info;
++ raw_spinlock_t msi_irq_lock;
+ DECLARE_BITMAP(msi_used, MSI_IRQ_NUM);
+ struct mutex msi_used_lock;
+- u16 msi_msg;
+ int link_gen;
+ struct pci_bridge_emul bridge;
+ struct gpio_desc *reset_gpio;
+@@ -257,18 +297,49 @@ static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg)
+ return readl(pcie->base + reg);
+ }
+
+-static inline u16 advk_read16(struct advk_pcie *pcie, u64 reg)
++static u8 advk_pcie_ltssm_state(struct advk_pcie *pcie)
+ {
+- return advk_readl(pcie, (reg & ~0x3)) >> ((reg & 0x3) * 8);
++ u32 val;
++ u8 ltssm_state;
++
++ val = advk_readl(pcie, CFG_REG);
++ ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK;
++ return ltssm_state;
+ }
+
+-static int advk_pcie_link_up(struct advk_pcie *pcie)
++static inline bool advk_pcie_link_up(struct advk_pcie *pcie)
+ {
+- u32 val, ltssm_state;
++ /* check if LTSSM is in normal operation - some L* state */
++ u8 ltssm_state = advk_pcie_ltssm_state(pcie);
++ return ltssm_state >= LTSSM_L0 && ltssm_state < LTSSM_DISABLED;
++}
+
+- val = advk_readl(pcie, CFG_REG);
+- ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK;
+- return ltssm_state >= LTSSM_L0;
++static inline bool advk_pcie_link_active(struct advk_pcie *pcie)
++{
++ /*
++ * According to PCIe Base specification 3.0, Table 4-14: Link
++ * Status Mapped to the LTSSM, and 4.2.6.3.6 Configuration.Idle
++ * is Link Up mapped to LTSSM Configuration.Idle, Recovery, L0,
++ * L0s, L1 and L2 states. And according to 3.2.1. Data Link
++ * Control and Management State Machine Rules is DL Up status
++ * reported in DL Active state.
++ */
++ u8 ltssm_state = advk_pcie_ltssm_state(pcie);
++ return ltssm_state >= LTSSM_CONFIG_IDLE && ltssm_state < LTSSM_DISABLED;
++}
++
++static inline bool advk_pcie_link_training(struct advk_pcie *pcie)
++{
++ /*
++ * According to PCIe Base specification 3.0, Table 4-14: Link
++ * Status Mapped to the LTSSM is Link Training mapped to LTSSM
++ * Configuration and Recovery states.
++ */
++ u8 ltssm_state = advk_pcie_ltssm_state(pcie);
++ return ((ltssm_state >= LTSSM_CONFIG_LINKWIDTH_START &&
++ ltssm_state < LTSSM_L0) ||
++ (ltssm_state >= LTSSM_RECOVERY_EQUALIZATION_PHASE0 &&
++ ltssm_state <= LTSSM_RECOVERY_EQUALIZATION_PHASE3));
+ }
+
+ static int advk_pcie_wait_for_link(struct advk_pcie *pcie)
+@@ -291,7 +362,7 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie)
+ size_t retries;
+
+ for (retries = 0; retries < RETRAIN_WAIT_MAX_RETRIES; ++retries) {
+- if (!advk_pcie_link_up(pcie))
++ if (advk_pcie_link_training(pcie))
+ break;
+ udelay(RETRAIN_WAIT_USLEEP_US);
+ }
+@@ -299,23 +370,9 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie)
+
+ static void advk_pcie_issue_perst(struct advk_pcie *pcie)
+ {
+- u32 reg;
+-
+ if (!pcie->reset_gpio)
+ return;
+
+- /*
+- * As required by PCI Express spec (PCI Express Base Specification, REV.
+- * 4.0 PCI Express, February 19 2014, 6.6.1 Conventional Reset) a delay
+- * for at least 100ms after de-asserting PERST# signal is needed before
+- * link training is enabled. So ensure that link training is disabled
+- * prior de-asserting PERST# signal to fulfill that PCI Express spec
+- * requirement.
+- */
+- reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+- reg &= ~LINK_TRAINING_EN;
+- advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+-
+ /* 10ms delay is needed for some cards */
+ dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n");
+ gpiod_set_value_cansleep(pcie->reset_gpio, 1);
+@@ -323,53 +380,46 @@ static void advk_pcie_issue_perst(struct advk_pcie *pcie)
+ gpiod_set_value_cansleep(pcie->reset_gpio, 0);
+ }
+
+-static int advk_pcie_train_at_gen(struct advk_pcie *pcie, int gen)
++static void advk_pcie_train_link(struct advk_pcie *pcie)
+ {
+- int ret, neg_gen;
++ struct device *dev = &pcie->pdev->dev;
+ u32 reg;
++ int ret;
+
+- /* Setup link speed */
++ /*
++ * Setup PCIe rev / gen compliance based on device tree property
++ * 'max-link-speed' which also forces maximal link speed.
++ */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg &= ~PCIE_GEN_SEL_MSK;
+- if (gen == 3)
++ if (pcie->link_gen == 3)
+ reg |= SPEED_GEN_3;
+- else if (gen == 2)
++ else if (pcie->link_gen == 2)
+ reg |= SPEED_GEN_2;
+ else
+ reg |= SPEED_GEN_1;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+ /*
+- * Enable link training. This is not needed in every call to this
+- * function, just once suffices, but it does not break anything either.
++ * Set maximal link speed value also into PCIe Link Control 2 register.
++ * Armada 3700 Functional Specification says that default value is based
++ * on SPEED_GEN but tests showed that default value is always 8.0 GT/s.
+ */
++ reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
++ reg &= ~PCI_EXP_LNKCTL2_TLS;
++ if (pcie->link_gen == 3)
++ reg |= PCI_EXP_LNKCTL2_TLS_8_0GT;
++ else if (pcie->link_gen == 2)
++ reg |= PCI_EXP_LNKCTL2_TLS_5_0GT;
++ else
++ reg |= PCI_EXP_LNKCTL2_TLS_2_5GT;
++ advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
++
++ /* Enable link training after selecting PCIe generation */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg |= LINK_TRAINING_EN;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+- /*
+- * Start link training immediately after enabling it.
+- * This solves problems for some buggy cards.
+- */
+- reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL);
+- reg |= PCI_EXP_LNKCTL_RL;
+- advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL);
+-
+- ret = advk_pcie_wait_for_link(pcie);
+- if (ret)
+- return ret;
+-
+- reg = advk_read16(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKSTA);
+- neg_gen = reg & PCI_EXP_LNKSTA_CLS;
+-
+- return neg_gen;
+-}
+-
+-static void advk_pcie_train_link(struct advk_pcie *pcie)
+-{
+- struct device *dev = &pcie->pdev->dev;
+- int neg_gen = -1, gen;
+-
+ /*
+ * Reset PCIe card via PERST# signal. Some cards are not detected
+ * during link training when they are in some non-initial state.
+@@ -380,41 +430,18 @@ static void advk_pcie_train_link(struct advk_pcie *pcie)
+ * PERST# signal could have been asserted by pinctrl subsystem before
+ * probe() callback has been called or issued explicitly by reset gpio
+ * function advk_pcie_issue_perst(), making the endpoint going into
+- * fundamental reset. As required by PCI Express spec a delay for at
+- * least 100ms after such a reset before link training is needed.
+- */
+- msleep(PCI_PM_D3COLD_WAIT);
+-
+- /*
+- * Try link training at link gen specified by device tree property
+- * 'max-link-speed'. If this fails, iteratively train at lower gen.
++ * fundamental reset. As required by PCI Express spec (PCI Express
++ * Base Specification, REV. 4.0 PCI Express, February 19 2014, 6.6.1
++ * Conventional Reset) a delay for at least 100ms after such a reset
++ * before sending a Configuration Request to the device is needed.
++ * So wait until PCIe link is up. Function advk_pcie_wait_for_link()
++ * waits for link at least 900ms.
+ */
+- for (gen = pcie->link_gen; gen > 0; --gen) {
+- neg_gen = advk_pcie_train_at_gen(pcie, gen);
+- if (neg_gen > 0)
+- break;
+- }
+-
+- if (neg_gen < 0)
+- goto err;
+-
+- /*
+- * After successful training if negotiated gen is lower than requested,
+- * train again on negotiated gen. This solves some stability issues for
+- * some buggy gen1 cards.
+- */
+- if (neg_gen < gen) {
+- gen = neg_gen;
+- neg_gen = advk_pcie_train_at_gen(pcie, gen);
+- }
+-
+- if (neg_gen == gen) {
+- dev_info(dev, "link up at gen %i\n", gen);
+- return;
+- }
+-
+-err:
+- dev_err(dev, "link never came up\n");
++ ret = advk_pcie_wait_for_link(pcie);
++ if (ret < 0)
++ dev_err(dev, "link never came up\n");
++ else
++ dev_info(dev, "link up\n");
+ }
+
+ /*
+@@ -448,12 +475,19 @@ static void advk_pcie_disable_ob_win(struct advk_pcie *pcie, u8 win_num)
+
+ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+ {
++ phys_addr_t msi_addr;
+ u32 reg;
+ int i;
+
+- /* Enable TX */
++ /*
++ * Configure PCIe Reference clock. Direction is from the PCIe
++ * controller to the endpoint card, so enable transmitting of
++ * Reference clock differential signal off-chip and disable
++ * receiving off-chip differential signal.
++ */
+ reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG);
+ reg |= PCIE_CORE_REF_CLK_TX_ENABLE;
++ reg &= ~PCIE_CORE_REF_CLK_RX_ENABLE;
+ advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG);
+
+ /* Set to Direct mode */
+@@ -477,6 +511,31 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+ reg = (PCI_VENDOR_ID_MARVELL << 16) | PCI_VENDOR_ID_MARVELL;
+ advk_writel(pcie, reg, VENDOR_ID_REG);
+
++ /*
++ * Change Class Code of PCI Bridge device to PCI Bridge (0x600400),
++ * because the default value is Mass storage controller (0x010400).
++ *
++ * Note that this Aardvark PCI Bridge does not have compliant Type 1
++ * Configuration Space and it even cannot be accessed via Aardvark's
++ * PCI config space access method. Something like config space is
++ * available in internal Aardvark registers starting at offset 0x0
++ * and is reported as Type 0. In range 0x10 - 0x34 it has totally
++ * different registers.
++ *
++ * Therefore driver uses emulation of PCI Bridge which emulates
++ * access to configuration space via internal Aardvark registers or
++ * emulated configuration buffer.
++ */
++ reg = advk_readl(pcie, PCIE_CORE_DEV_REV_REG);
++ reg &= ~0xffffff00;
++ reg |= (PCI_CLASS_BRIDGE_PCI << 8) << 8;
++ advk_writel(pcie, reg, PCIE_CORE_DEV_REV_REG);
++
++ /* Disable Root Bridge I/O space, memory space and bus mastering */
++ reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
++ reg &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
++ advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG);
++
+ /* Set Advanced Error Capabilities and Control PF0 register */
+ reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX |
+ PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN |
+@@ -488,8 +547,9 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+ reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
+ reg &= ~PCI_EXP_DEVCTL_RELAX_EN;
+ reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
++ reg &= ~PCI_EXP_DEVCTL_PAYLOAD;
+ reg &= ~PCI_EXP_DEVCTL_READRQ;
+- reg |= PCI_EXP_DEVCTL_PAYLOAD; /* Set max payload size */
++ reg |= PCI_EXP_DEVCTL_PAYLOAD_512B;
+ reg |= PCI_EXP_DEVCTL_READRQ_512B;
+ advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
+
+@@ -504,25 +564,36 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+ reg |= LANE_COUNT_1;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
++ /* Set MSI address */
++ msi_addr = virt_to_phys(pcie);
++ advk_writel(pcie, lower_32_bits(msi_addr), PCIE_MSI_ADDR_LOW_REG);
++ advk_writel(pcie, upper_32_bits(msi_addr), PCIE_MSI_ADDR_HIGH_REG);
++
+ /* Enable MSI */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
+ reg |= PCIE_CORE_CTRL2_MSI_ENABLE;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
+
+ /* Clear all interrupts */
++ advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
+ advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
+ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
+ advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
+
+- /* Disable All ISR0/1 Sources */
+- reg = PCIE_ISR0_ALL_MASK;
++ /* Disable All ISR0/1 and MSI Sources */
++ advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
++ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
++ advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
++
++ /* Unmask summary MSI interrupt */
++ reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
+ reg &= ~PCIE_ISR0_MSI_INT_PENDING;
+ advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
+
+- advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
+-
+- /* Unmask all MSIs */
+- advk_writel(pcie, 0, PCIE_MSI_MASK_REG);
++ /* Unmask PME interrupt for processing of PME requester */
++ reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
++ reg &= ~PCIE_MSG_PM_PME_MASK;
++ advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
+
+ /* Enable summary interrupt for GIC SPI source */
+ reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK);
+@@ -574,19 +645,6 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+ advk_pcie_disable_ob_win(pcie, i);
+
+ advk_pcie_train_link(pcie);
+-
+- /*
+- * FIXME: The following register update is suspicious. This register is
+- * applicable only when the PCI controller is configured for Endpoint
+- * mode, not as a Root Complex. But apparently when this code is
+- * removed, some cards stop working. This should be investigated and
+- * a comment explaining this should be put here.
+- */
+- reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
+- reg |= PCIE_CORE_CMD_MEM_ACCESS_EN |
+- PCIE_CORE_CMD_IO_ACCESS_EN |
+- PCIE_CORE_CMD_MEM_IO_REQ_EN;
+- advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG);
+ }
+
+ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u32 *val)
+@@ -595,6 +653,7 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3
+ u32 reg;
+ unsigned int status;
+ char *strcomp_status, *str_posted;
++ int ret;
+
+ reg = advk_readl(pcie, PIO_STAT);
+ status = (reg & PIO_COMPLETION_STATUS_MASK) >>
+@@ -619,6 +678,7 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3
+ case PIO_COMPLETION_STATUS_OK:
+ if (reg & PIO_ERR_STATUS) {
+ strcomp_status = "COMP_ERR";
++ ret = -EFAULT;
+ break;
+ }
+ /* Get the read result */
+@@ -626,9 +686,11 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3
+ *val = advk_readl(pcie, PIO_RD_DATA);
+ /* No error */
+ strcomp_status = NULL;
++ ret = 0;
+ break;
+ case PIO_COMPLETION_STATUS_UR:
+ strcomp_status = "UR";
++ ret = -EOPNOTSUPP;
+ break;
+ case PIO_COMPLETION_STATUS_CRS:
+ if (allow_crs && val) {
+@@ -646,6 +708,7 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3
+ */
+ *val = CFG_RD_CRS_VAL;
+ strcomp_status = NULL;
++ ret = 0;
+ break;
+ }
+ /* PCIe r4.0, sec 2.3.2, says:
+@@ -661,31 +724,34 @@ static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u3
+ * Request and taking appropriate action, e.g., complete the
+ * Request to the host as a failed transaction.
+ *
+- * To simplify implementation do not re-issue the Configuration
+- * Request and complete the Request as a failed transaction.
++ * So return -EAGAIN and caller (pci-aardvark.c driver) will
++ * re-issue request again up to the PIO_RETRY_CNT retries.
+ */
+ strcomp_status = "CRS";
++ ret = -EAGAIN;
+ break;
+ case PIO_COMPLETION_STATUS_CA:
+ strcomp_status = "CA";
++ ret = -ECANCELED;
+ break;
+ default:
+ strcomp_status = "Unknown";
++ ret = -EINVAL;
+ break;
+ }
+
+ if (!strcomp_status)
+- return 0;
++ return ret;
+
+ if (reg & PIO_NON_POSTED_REQ)
+ str_posted = "Non-posted";
+ else
+ str_posted = "Posted";
+
+- dev_err(dev, "%s PIO Response Status: %s, %#x @ %#x\n",
++ dev_dbg(dev, "%s PIO Response Status: %s, %#x @ %#x\n",
+ str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS));
+
+- return -EFAULT;
++ return ret;
+ }
+
+ static int advk_pcie_wait_pio(struct advk_pcie *pcie)
+@@ -693,13 +759,13 @@ static int advk_pcie_wait_pio(struct advk_pcie *pcie)
+ struct device *dev = &pcie->pdev->dev;
+ int i;
+
+- for (i = 0; i < PIO_RETRY_CNT; i++) {
++ for (i = 1; i <= PIO_RETRY_CNT; i++) {
+ u32 start, isr;
+
+ start = advk_readl(pcie, PIO_START);
+ isr = advk_readl(pcie, PIO_ISR);
+ if (!start && isr)
+- return 0;
++ return i;
+ udelay(PIO_RETRY_DELAY);
+ }
+
+@@ -707,6 +773,81 @@ static int advk_pcie_wait_pio(struct advk_pcie *pcie)
+ return -ETIMEDOUT;
+ }
+
++static pci_bridge_emul_read_status_t
++advk_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge,
++ int reg, u32 *value)
++{
++ struct advk_pcie *pcie = bridge->data;
++
++ switch (reg) {
++ case PCI_COMMAND:
++ *value = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
++ return PCI_BRIDGE_EMUL_HANDLED;
++
++ case PCI_INTERRUPT_LINE: {
++ /*
++ * From the whole 32bit register we support reading from HW only
++ * two bits: PCI_BRIDGE_CTL_BUS_RESET and PCI_BRIDGE_CTL_SERR.
++ * Other bits are retrieved only from emulated config buffer.
++ */
++ __le32 *cfgspace = (__le32 *)&bridge->conf;
++ u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]);
++ if (advk_readl(pcie, PCIE_ISR0_MASK_REG) & PCIE_ISR0_ERR_MASK)
++ val &= ~(PCI_BRIDGE_CTL_SERR << 16);
++ else
++ val |= PCI_BRIDGE_CTL_SERR << 16;
++ if (advk_readl(pcie, PCIE_CORE_CTRL1_REG) & HOT_RESET_GEN)
++ val |= PCI_BRIDGE_CTL_BUS_RESET << 16;
++ else
++ val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16);
++ *value = val;
++ return PCI_BRIDGE_EMUL_HANDLED;
++ }
++
++ default:
++ return PCI_BRIDGE_EMUL_NOT_HANDLED;
++ }
++}
++
++static void
++advk_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge,
++ int reg, u32 old, u32 new, u32 mask)
++{
++ struct advk_pcie *pcie = bridge->data;
++
++ switch (reg) {
++ case PCI_COMMAND:
++ advk_writel(pcie, new, PCIE_CORE_CMD_STATUS_REG);
++ break;
++
++ case PCI_INTERRUPT_LINE:
++ /*
++ * According to Figure 6-3: Pseudo Logic Diagram for Error
++ * Message Controls in PCIe base specification, SERR# Enable bit
++ * in Bridge Control register enable receiving of ERR_* messages
++ */
++ if (mask & (PCI_BRIDGE_CTL_SERR << 16)) {
++ u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG);
++ if (new & (PCI_BRIDGE_CTL_SERR << 16))
++ val &= ~PCIE_ISR0_ERR_MASK;
++ else
++ val |= PCIE_ISR0_ERR_MASK;
++ advk_writel(pcie, val, PCIE_ISR0_MASK_REG);
++ }
++ if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) {
++ u32 val = advk_readl(pcie, PCIE_CORE_CTRL1_REG);
++ if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16))
++ val |= HOT_RESET_GEN;
++ else
++ val &= ~HOT_RESET_GEN;
++ advk_writel(pcie, val, PCIE_CORE_CTRL1_REG);
++ }
++ break;
++
++ default:
++ break;
++ }
++}
+
+ static pci_bridge_emul_read_status_t
+ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+@@ -720,17 +861,21 @@ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+ *value = PCI_EXP_SLTSTA_PDS << 16;
+ return PCI_BRIDGE_EMUL_HANDLED;
+
+- case PCI_EXP_RTCTL: {
+- u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG);
+- *value = (val & PCIE_MSG_PM_PME_MASK) ? 0 : PCI_EXP_RTCTL_PMEIE;
+- *value |= PCI_EXP_RTCAP_CRSVIS << 16;
+- return PCI_BRIDGE_EMUL_HANDLED;
+- }
++ /*
++ * PCI_EXP_RTCTL and PCI_EXP_RTSTA are also supported, but do not need
++ * to be handled here, because their values are stored in emulated
++ * config space buffer, and we read them from there when needed.
++ */
+
+- case PCI_EXP_RTSTA: {
+- u32 isr0 = advk_readl(pcie, PCIE_ISR0_REG);
+- u32 msglog = advk_readl(pcie, PCIE_MSG_LOG_REG);
+- *value = (isr0 & PCIE_MSG_PM_PME_MASK) << 16 | (msglog >> 16);
++ case PCI_EXP_LNKCAP: {
++ u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
++ /*
++ * PCI_EXP_LNKCAP_DLLLARC bit is hardwired in aardvark HW to 0.
++ * But support for PCI_EXP_LNKSTA_DLLLA is emulated via ltssm
++ * state so explicitly enable PCI_EXP_LNKCAP_DLLLARC flag.
++ */
++ val |= PCI_EXP_LNKCAP_DLLLARC;
++ *value = val;
+ return PCI_BRIDGE_EMUL_HANDLED;
+ }
+
+@@ -738,18 +883,23 @@ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+ /* u32 contains both PCI_EXP_LNKCTL and PCI_EXP_LNKSTA */
+ u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg) &
+ ~(PCI_EXP_LNKSTA_LT << 16);
+- if (!advk_pcie_link_up(pcie))
++ if (advk_pcie_link_training(pcie))
+ val |= (PCI_EXP_LNKSTA_LT << 16);
++ if (advk_pcie_link_active(pcie))
++ val |= (PCI_EXP_LNKSTA_DLLLA << 16);
+ *value = val;
+ return PCI_BRIDGE_EMUL_HANDLED;
+ }
+
+- case PCI_CAP_LIST_ID:
+ case PCI_EXP_DEVCAP:
+ case PCI_EXP_DEVCTL:
+- case PCI_EXP_LNKCAP:
++ case PCI_EXP_DEVCAP2:
++ case PCI_EXP_DEVCTL2:
++ case PCI_EXP_LNKCAP2:
++ case PCI_EXP_LNKCTL2:
+ *value = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
+ return PCI_BRIDGE_EMUL_HANDLED;
++
+ default:
+ return PCI_BRIDGE_EMUL_NOT_HANDLED;
+ }
+@@ -763,10 +913,6 @@ advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
+ struct advk_pcie *pcie = bridge->data;
+
+ switch (reg) {
+- case PCI_EXP_DEVCTL:
+- advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
+- break;
+-
+ case PCI_EXP_LNKCTL:
+ advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
+ if (new & PCI_EXP_LNKCTL_RL)
+@@ -774,18 +920,23 @@ advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
+ break;
+
+ case PCI_EXP_RTCTL: {
+- /* Only mask/unmask PME interrupt */
+- u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG) &
+- ~PCIE_MSG_PM_PME_MASK;
+- if ((new & PCI_EXP_RTCTL_PMEIE) == 0)
+- val |= PCIE_MSG_PM_PME_MASK;
+- advk_writel(pcie, val, PCIE_ISR0_MASK_REG);
++ u16 rootctl = le16_to_cpu(bridge->pcie_conf.rootctl);
++ /* Only emulation of PMEIE and CRSSVE bits is provided */
++ rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_CRSSVE;
++ bridge->pcie_conf.rootctl = cpu_to_le16(rootctl);
+ break;
+ }
+
+- case PCI_EXP_RTSTA:
+- new = (new & PCI_EXP_RTSTA_PME) >> 9;
+- advk_writel(pcie, new, PCIE_ISR0_REG);
++ /*
++ * PCI_EXP_RTSTA is also supported, but does not need to be handled
++ * here, because its value is stored in emulated config space buffer,
++ * and we write it there when needed.
++ */
++
++ case PCI_EXP_DEVCTL:
++ case PCI_EXP_DEVCTL2:
++ case PCI_EXP_LNKCTL2:
++ advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
+ break;
+
+ default:
+@@ -794,6 +945,8 @@ advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
+ }
+
+ static struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = {
++ .read_base = advk_pci_bridge_emul_base_conf_read,
++ .write_base = advk_pci_bridge_emul_base_conf_write,
+ .read_pcie = advk_pci_bridge_emul_pcie_conf_read,
+ .write_pcie = advk_pci_bridge_emul_pcie_conf_write,
+ };
+@@ -805,7 +958,6 @@ static struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = {
+ static int advk_sw_pci_bridge_init(struct advk_pcie *pcie)
+ {
+ struct pci_bridge_emul *bridge = &pcie->bridge;
+- int ret;
+
+ bridge->conf.vendor =
+ cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) & 0xffff);
+@@ -823,21 +975,19 @@ static int advk_sw_pci_bridge_init(struct advk_pcie *pcie)
+ bridge->conf.pref_mem_limit = cpu_to_le16(PCI_PREF_RANGE_TYPE_64);
+
+ /* Support interrupt A for MSI feature */
+- bridge->conf.intpin = PCIE_CORE_INT_A_ASSERT_ENABLE;
++ bridge->conf.intpin = PCI_INTERRUPT_INTA;
+
+- bridge->has_pcie = true;
+- bridge->data = pcie;
+- bridge->ops = &advk_pci_bridge_emul_ops;
+-
+- /* PCIe config space can be initialized after pci_bridge_emul_init() */
+- ret = pci_bridge_emul_init(bridge, 0);
+- if (ret < 0)
+- return ret;
++ /* Aardvark HW provides PCIe Capability structure in version 2 */
++ bridge->pcie_conf.cap = cpu_to_le16(2);
+
+ /* Indicates supports for Completion Retry Status */
+ bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS);
+
+- return 0;
++ bridge->has_pcie = true;
++ bridge->data = pcie;
++ bridge->ops = &advk_pci_bridge_emul_ops;
++
++ return pci_bridge_emul_init(bridge, 0);
+ }
+
+ static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus,
+@@ -847,8 +997,12 @@ static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus,
+ return false;
+
+ /*
+- * If the link goes down after we check for link-up, nothing bad
+- * happens but the config access times out.
++ * If the link goes down after we check for link-up, we have a problem:
++ * if a PIO request is executed while link-down, the whole controller
++ * gets stuck in a non-functional state, and even after link comes up
++ * again, PIO requests won't work anymore, and a reset of the whole PCIe
++ * controller is needed. Therefore we need to prevent sending PIO
++ * requests while the link is down.
+ */
+ if (!pci_is_root_bus(bus) && !advk_pcie_link_up(pcie))
+ return false;
+@@ -889,6 +1043,7 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 *val)
+ {
+ struct advk_pcie *pcie = bus->sysdata;
++ int retry_count;
+ bool allow_crs;
+ u32 reg;
+ int ret;
+@@ -911,18 +1066,8 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
+ (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) &
+ PCI_EXP_RTCTL_CRSSVE);
+
+- if (advk_pcie_pio_is_running(pcie)) {
+- /*
+- * If it is possible return Completion Retry Status so caller
+- * tries to issue the request again instead of failing.
+- */
+- if (allow_crs) {
+- *val = CFG_RD_CRS_VAL;
+- return PCIBIOS_SUCCESSFUL;
+- }
+- *val = 0xffffffff;
+- return PCIBIOS_SET_FAILED;
+- }
++ if (advk_pcie_pio_is_running(pcie))
++ goto try_crs;
+
+ /* Program the control register */
+ reg = advk_readl(pcie, PIO_CTRL);
+@@ -941,30 +1086,24 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
+ /* Program the data strobe */
+ advk_writel(pcie, 0xf, PIO_WR_DATA_STRB);
+
+- /* Clear PIO DONE ISR and start the transfer */
+- advk_writel(pcie, 1, PIO_ISR);
+- advk_writel(pcie, 1, PIO_START);
++ retry_count = 0;
++ do {
++ /* Clear PIO DONE ISR and start the transfer */
++ advk_writel(pcie, 1, PIO_ISR);
++ advk_writel(pcie, 1, PIO_START);
+
+- ret = advk_pcie_wait_pio(pcie);
+- if (ret < 0) {
+- /*
+- * If it is possible return Completion Retry Status so caller
+- * tries to issue the request again instead of failing.
+- */
+- if (allow_crs) {
+- *val = CFG_RD_CRS_VAL;
+- return PCIBIOS_SUCCESSFUL;
+- }
+- *val = 0xffffffff;
+- return PCIBIOS_SET_FAILED;
+- }
++ ret = advk_pcie_wait_pio(pcie);
++ if (ret < 0)
++ goto try_crs;
+
+- /* Check PIO status and get the read result */
+- ret = advk_pcie_check_pio_status(pcie, allow_crs, val);
+- if (ret < 0) {
+- *val = 0xffffffff;
+- return PCIBIOS_SET_FAILED;
+- }
++ retry_count += ret;
++
++ /* Check PIO status and get the read result */
++ ret = advk_pcie_check_pio_status(pcie, allow_crs, val);
++ } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
++
++ if (ret < 0)
++ goto fail;
+
+ if (size == 1)
+ *val = (*val >> (8 * (where & 3))) & 0xff;
+@@ -972,6 +1111,20 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
+ *val = (*val >> (8 * (where & 3))) & 0xffff;
+
+ return PCIBIOS_SUCCESSFUL;
++
++try_crs:
++ /*
++ * If it is possible, return Completion Retry Status so that caller
++ * tries to issue the request again instead of failing.
++ */
++ if (allow_crs) {
++ *val = CFG_RD_CRS_VAL;
++ return PCIBIOS_SUCCESSFUL;
++ }
++
++fail:
++ *val = 0xffffffff;
++ return PCIBIOS_SET_FAILED;
+ }
+
+ static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+@@ -980,6 +1133,7 @@ static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+ struct advk_pcie *pcie = bus->sysdata;
+ u32 reg;
+ u32 data_strobe = 0x0;
++ int retry_count;
+ int offset;
+ int ret;
+
+@@ -1021,19 +1175,22 @@ static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+ /* Program the data strobe */
+ advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB);
+
+- /* Clear PIO DONE ISR and start the transfer */
+- advk_writel(pcie, 1, PIO_ISR);
+- advk_writel(pcie, 1, PIO_START);
++ retry_count = 0;
++ do {
++ /* Clear PIO DONE ISR and start the transfer */
++ advk_writel(pcie, 1, PIO_ISR);
++ advk_writel(pcie, 1, PIO_START);
+
+- ret = advk_pcie_wait_pio(pcie);
+- if (ret < 0)
+- return PCIBIOS_SET_FAILED;
++ ret = advk_pcie_wait_pio(pcie);
++ if (ret < 0)
++ return PCIBIOS_SET_FAILED;
+
+- ret = advk_pcie_check_pio_status(pcie, false, NULL);
+- if (ret < 0)
+- return PCIBIOS_SET_FAILED;
++ retry_count += ret;
+
+- return PCIBIOS_SUCCESSFUL;
++ ret = advk_pcie_check_pio_status(pcie, false, NULL);
++ } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
++
++ return ret < 0 ? PCIBIOS_SET_FAILED : PCIBIOS_SUCCESSFUL;
+ }
+
+ static struct pci_ops advk_pcie_ops = {
+@@ -1045,11 +1202,11 @@ static void advk_msi_irq_compose_msi_msg(struct irq_data *data,
+ struct msi_msg *msg)
+ {
+ struct advk_pcie *pcie = irq_data_get_irq_chip_data(data);
+- phys_addr_t msi_msg = virt_to_phys(&pcie->msi_msg);
++ phys_addr_t msi_addr = virt_to_phys(pcie);
+
+- msg->address_lo = lower_32_bits(msi_msg);
+- msg->address_hi = upper_32_bits(msi_msg);
+- msg->data = data->irq;
++ msg->address_lo = lower_32_bits(msi_addr);
++ msg->address_hi = upper_32_bits(msi_addr);
++ msg->data = data->hwirq;
+ }
+
+ static int advk_msi_set_affinity(struct irq_data *irq_data,
+@@ -1058,6 +1215,54 @@ static int advk_msi_set_affinity(struct irq_data *irq_data,
+ return -EINVAL;
+ }
+
++static void advk_msi_irq_mask(struct irq_data *d)
++{
++ struct advk_pcie *pcie = d->domain->host_data;
++ irq_hw_number_t hwirq = irqd_to_hwirq(d);
++ unsigned long flags;
++ u32 mask;
++
++ raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
++ mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
++ mask |= BIT(hwirq);
++ advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
++ raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
++}
++
++static void advk_msi_irq_unmask(struct irq_data *d)
++{
++ struct advk_pcie *pcie = d->domain->host_data;
++ irq_hw_number_t hwirq = irqd_to_hwirq(d);
++ unsigned long flags;
++ u32 mask;
++
++ raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
++ mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
++ mask &= ~BIT(hwirq);
++ advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
++ raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
++}
++
++static void advk_msi_top_irq_mask(struct irq_data *d)
++{
++ pci_msi_mask_irq(d);
++ irq_chip_mask_parent(d);
++}
++
++static void advk_msi_top_irq_unmask(struct irq_data *d)
++{
++ pci_msi_unmask_irq(d);
++ irq_chip_unmask_parent(d);
++}
++
++static struct irq_chip advk_msi_bottom_irq_chip = {
++ .name = "MSI",
++ .irq_compose_msi_msg = advk_msi_irq_compose_msi_msg,
++ .irq_set_affinity = advk_msi_set_affinity,
++ .irq_mask = advk_msi_irq_mask,
++ .irq_unmask = advk_msi_irq_unmask,
++};
++
+ static int advk_msi_irq_domain_alloc(struct irq_domain *domain,
+ unsigned int virq,
+ unsigned int nr_irqs, void *args)
+@@ -1066,23 +1271,19 @@ static int advk_msi_irq_domain_alloc(struct irq_domain *domain,
+ int hwirq, i;
+
+ mutex_lock(&pcie->msi_used_lock);
+- hwirq = bitmap_find_next_zero_area(pcie->msi_used, MSI_IRQ_NUM,
+- 0, nr_irqs, 0);
+- if (hwirq >= MSI_IRQ_NUM) {
+- mutex_unlock(&pcie->msi_used_lock);
+- return -ENOSPC;
+- }
+-
+- bitmap_set(pcie->msi_used, hwirq, nr_irqs);
++ hwirq = bitmap_find_free_region(pcie->msi_used, MSI_IRQ_NUM,
++ order_base_2(nr_irqs));
+ mutex_unlock(&pcie->msi_used_lock);
++ if (hwirq < 0)
++ return -ENOSPC;
+
+ for (i = 0; i < nr_irqs; i++)
+ irq_domain_set_info(domain, virq + i, hwirq + i,
+- &pcie->msi_bottom_irq_chip,
++ &advk_msi_bottom_irq_chip,
+ domain->host_data, handle_simple_irq,
+ NULL, NULL);
+
+- return hwirq;
++ return 0;
+ }
+
+ static void advk_msi_irq_domain_free(struct irq_domain *domain,
+@@ -1092,7 +1293,7 @@ static void advk_msi_irq_domain_free(struct irq_domain *domain,
+ struct advk_pcie *pcie = domain->host_data;
+
+ mutex_lock(&pcie->msi_used_lock);
+- bitmap_clear(pcie->msi_used, d->hwirq, nr_irqs);
++ bitmap_release_region(pcie->msi_used, d->hwirq, order_base_2(nr_irqs));
+ mutex_unlock(&pcie->msi_used_lock);
+ }
+
+@@ -1134,7 +1335,6 @@ static int advk_pcie_irq_map(struct irq_domain *h,
+ {
+ struct advk_pcie *pcie = h->host_data;
+
+- advk_pcie_irq_mask(irq_get_irq_data(virq));
+ irq_set_status_flags(virq, IRQ_LEVEL);
+ irq_set_chip_and_handler(virq, &pcie->irq_chip,
+ handle_level_irq);
+@@ -1148,37 +1348,25 @@ static const struct irq_domain_ops advk_pcie_irq_domain_ops = {
+ .xlate = irq_domain_xlate_onecell,
+ };
+
++static struct irq_chip advk_msi_irq_chip = {
++ .name = "advk-MSI",
++ .irq_mask = advk_msi_top_irq_mask,
++ .irq_unmask = advk_msi_top_irq_unmask,
++};
++
++static struct msi_domain_info advk_msi_domain_info = {
++ .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
++ MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX,
++ .chip = &advk_msi_irq_chip,
++};
++
+ static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
+ {
+ struct device *dev = &pcie->pdev->dev;
+- struct device_node *node = dev->of_node;
+- struct irq_chip *bottom_ic, *msi_ic;
+- struct msi_domain_info *msi_di;
+- phys_addr_t msi_msg_phys;
+
++ raw_spin_lock_init(&pcie->msi_irq_lock);
+ mutex_init(&pcie->msi_used_lock);
+
+- bottom_ic = &pcie->msi_bottom_irq_chip;
+-
+- bottom_ic->name = "MSI";
+- bottom_ic->irq_compose_msi_msg = advk_msi_irq_compose_msi_msg;
+- bottom_ic->irq_set_affinity = advk_msi_set_affinity;
+-
+- msi_ic = &pcie->msi_irq_chip;
+- msi_ic->name = "advk-MSI";
+-
+- msi_di = &pcie->msi_domain_info;
+- msi_di->flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+- MSI_FLAG_MULTI_PCI_MSI;
+- msi_di->chip = msi_ic;
+-
+- msi_msg_phys = virt_to_phys(&pcie->msi_msg);
+-
+- advk_writel(pcie, lower_32_bits(msi_msg_phys),
+- PCIE_MSI_ADDR_LOW_REG);
+- advk_writel(pcie, upper_32_bits(msi_msg_phys),
+- PCIE_MSI_ADDR_HIGH_REG);
+-
+ pcie->msi_inner_domain =
+ irq_domain_add_linear(NULL, MSI_IRQ_NUM,
+ &advk_msi_domain_ops, pcie);
+@@ -1186,8 +1374,9 @@ static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
+ return -ENOMEM;
+
+ pcie->msi_domain =
+- pci_msi_create_irq_domain(of_node_to_fwnode(node),
+- msi_di, pcie->msi_inner_domain);
++ pci_msi_create_irq_domain(dev_fwnode(dev),
++ &advk_msi_domain_info,
++ pcie->msi_inner_domain);
+ if (!pcie->msi_domain) {
+ irq_domain_remove(pcie->msi_inner_domain);
+ return -ENOMEM;
+@@ -1228,7 +1417,6 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie)
+ }
+
+ irq_chip->irq_mask = advk_pcie_irq_mask;
+- irq_chip->irq_mask_ack = advk_pcie_irq_mask;
+ irq_chip->irq_unmask = advk_pcie_irq_unmask;
+
+ pcie->irq_domain =
+@@ -1250,22 +1438,85 @@ static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie)
+ irq_domain_remove(pcie->irq_domain);
+ }
+
++static struct irq_chip advk_rp_irq_chip = {
++ .name = "advk-RP",
++};
++
++static int advk_pcie_rp_irq_map(struct irq_domain *h,
++ unsigned int virq, irq_hw_number_t hwirq)
++{
++ struct advk_pcie *pcie = h->host_data;
++
++ irq_set_chip_and_handler(virq, &advk_rp_irq_chip, handle_simple_irq);
++ irq_set_chip_data(virq, pcie);
++
++ return 0;
++}
++
++static const struct irq_domain_ops advk_pcie_rp_irq_domain_ops = {
++ .map = advk_pcie_rp_irq_map,
++ .xlate = irq_domain_xlate_onecell,
++};
++
++static int advk_pcie_init_rp_irq_domain(struct advk_pcie *pcie)
++{
++ pcie->rp_irq_domain = irq_domain_add_linear(NULL, 1,
++ &advk_pcie_rp_irq_domain_ops,
++ pcie);
++ if (!pcie->rp_irq_domain) {
++ dev_err(&pcie->pdev->dev, "Failed to add Root Port IRQ domain\n");
++ return -ENOMEM;
++ }
++
++ return 0;
++}
++
++static void advk_pcie_remove_rp_irq_domain(struct advk_pcie *pcie)
++{
++ irq_domain_remove(pcie->rp_irq_domain);
++}
++
++static void advk_pcie_handle_pme(struct advk_pcie *pcie)
++{
++ u32 requester = advk_readl(pcie, PCIE_MSG_LOG_REG) >> 16;
++
++ advk_writel(pcie, PCIE_MSG_PM_PME_MASK, PCIE_ISR0_REG);
++
++ /*
++ * PCIE_MSG_LOG_REG contains the last inbound message, so store
++ * the requester ID only when PME was not asserted yet.
++ * Also do not trigger PME interrupt when PME is still asserted.
++ */
++ if (!(le32_to_cpu(pcie->bridge.pcie_conf.rootsta) & PCI_EXP_RTSTA_PME)) {
++ pcie->bridge.pcie_conf.rootsta = cpu_to_le32(requester | PCI_EXP_RTSTA_PME);
++
++ /*
++ * Trigger PME interrupt only if PMEIE bit in Root Control is set.
++ * Aardvark HW returns zero for PCI_EXP_FLAGS_IRQ, so use PCIe interrupt 0.
++ */
++ if (!(le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & PCI_EXP_RTCTL_PMEIE))
++ return;
++
++ if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
++ dev_err_ratelimited(&pcie->pdev->dev, "unhandled PME IRQ\n");
++ }
++}
++
+ static void advk_pcie_handle_msi(struct advk_pcie *pcie)
+ {
+ u32 msi_val, msi_mask, msi_status, msi_idx;
+- u16 msi_data;
+
+ msi_mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
+ msi_val = advk_readl(pcie, PCIE_MSI_STATUS_REG);
+- msi_status = msi_val & ~msi_mask;
++ msi_status = msi_val & ((~msi_mask) & PCIE_MSI_ALL_MASK);
+
+ for (msi_idx = 0; msi_idx < MSI_IRQ_NUM; msi_idx++) {
+ if (!(BIT(msi_idx) & msi_status))
+ continue;
+
+ advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG);
+- msi_data = advk_readl(pcie, PCIE_MSI_PAYLOAD_REG) & 0xFF;
+- generic_handle_irq(msi_data);
++ if (generic_handle_domain_irq(pcie->msi_inner_domain, msi_idx) == -EINVAL)
++ dev_err_ratelimited(&pcie->pdev->dev, "unexpected MSI 0x%02x\n", msi_idx);
+ }
+
+ advk_writel(pcie, PCIE_ISR0_MSI_INT_PENDING,
+@@ -1286,10 +1537,20 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie)
+ isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
+ isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK);
+
+- if (!isr0_status && !isr1_status) {
+- advk_writel(pcie, isr0_val, PCIE_ISR0_REG);
+- advk_writel(pcie, isr1_val, PCIE_ISR1_REG);
+- return;
++ /* Process PME interrupt as the first one to do not miss PME requester id */
++ if (isr0_status & PCIE_MSG_PM_PME_MASK)
++ advk_pcie_handle_pme(pcie);
++
++ /* Process ERR interrupt */
++ if (isr0_status & PCIE_ISR0_ERR_MASK) {
++ advk_writel(pcie, PCIE_ISR0_ERR_MASK, PCIE_ISR0_REG);
++
++ /*
++ * Aardvark HW returns zero for PCI_ERR_ROOT_AER_IRQ, so use
++ * PCIe interrupt 0
++ */
++ if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
++ dev_err_ratelimited(&pcie->pdev->dev, "unhandled ERR IRQ\n");
+ }
+
+ /* Process MSI interrupts */
+@@ -1304,7 +1565,9 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie)
+ advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i),
+ PCIE_ISR1_REG);
+
+- generic_handle_domain_irq(pcie->irq_domain, i);
++ if (generic_handle_domain_irq(pcie->irq_domain, i) == -EINVAL)
++ dev_err_ratelimited(&pcie->pdev->dev, "unexpected INT%c IRQ\n",
++ (char)i + 'A');
+ }
+ }
+
+@@ -1325,7 +1588,22 @@ static irqreturn_t advk_pcie_irq_handler(int irq, void *arg)
+ return IRQ_HANDLED;
+ }
+
+-static void __maybe_unused advk_pcie_disable_phy(struct advk_pcie *pcie)
++static int advk_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
++{
++ struct advk_pcie *pcie = dev->bus->sysdata;
++
++ /*
++ * Emulated root bridge has its own emulated irq chip and irq domain.
++ * Argument pin is the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) and
++ * hwirq for irq_create_mapping() is indexed from zero.
++ */
++ if (pci_is_root_bus(dev->bus))
++ return irq_create_mapping(pcie->rp_irq_domain, pin - 1);
++ else
++ return of_irq_parse_and_map_pci(dev, slot, pin);
++}
++
++static void advk_pcie_disable_phy(struct advk_pcie *pcie)
+ {
+ phy_power_off(pcie->phy);
+ phy_exit(pcie->phy);
+@@ -1411,8 +1689,7 @@ static int advk_pcie_probe(struct platform_device *pdev)
+ * only PIO for issuing configuration transfers which does
+ * not use PCIe window configuration.
+ */
+- if (type != IORESOURCE_MEM && type != IORESOURCE_MEM_64 &&
+- type != IORESOURCE_IO)
++ if (type != IORESOURCE_MEM && type != IORESOURCE_IO)
+ continue;
+
+ /*
+@@ -1420,8 +1697,7 @@ static int advk_pcie_probe(struct platform_device *pdev)
+ * configuration is set to transparent memory access so it
+ * does not need window configuration.
+ */
+- if ((type == IORESOURCE_MEM || type == IORESOURCE_MEM_64) &&
+- entry->offset == 0)
++ if (type == IORESOURCE_MEM && entry->offset == 0)
+ continue;
+
+ /*
+@@ -1536,11 +1812,21 @@ static int advk_pcie_probe(struct platform_device *pdev)
+ return ret;
+ }
+
++ ret = advk_pcie_init_rp_irq_domain(pcie);
++ if (ret) {
++ dev_err(dev, "Failed to initialize irq\n");
++ advk_pcie_remove_msi_irq_domain(pcie);
++ advk_pcie_remove_irq_domain(pcie);
++ return ret;
++ }
++
+ bridge->sysdata = pcie;
+ bridge->ops = &advk_pcie_ops;
++ bridge->map_irq = advk_pcie_map_irq;
+
+ ret = pci_host_probe(bridge);
+ if (ret < 0) {
++ advk_pcie_remove_rp_irq_domain(pcie);
+ advk_pcie_remove_msi_irq_domain(pcie);
+ advk_pcie_remove_irq_domain(pcie);
+ return ret;
+@@ -1553,20 +1839,65 @@ static int advk_pcie_remove(struct platform_device *pdev)
+ {
+ struct advk_pcie *pcie = platform_get_drvdata(pdev);
+ struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
++ u32 val;
+ int i;
+
++ /* Remove PCI bus with all devices */
+ pci_lock_rescan_remove();
+ pci_stop_root_bus(bridge->bus);
+ pci_remove_root_bus(bridge->bus);
+ pci_unlock_rescan_remove();
+
++ /* Disable Root Bridge I/O space, memory space and bus mastering */
++ val = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
++ val &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
++ advk_writel(pcie, val, PCIE_CORE_CMD_STATUS_REG);
++
++ /* Disable MSI */
++ val = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
++ val &= ~PCIE_CORE_CTRL2_MSI_ENABLE;
++ advk_writel(pcie, val, PCIE_CORE_CTRL2_REG);
++
++ /* Clear MSI address */
++ advk_writel(pcie, 0, PCIE_MSI_ADDR_LOW_REG);
++ advk_writel(pcie, 0, PCIE_MSI_ADDR_HIGH_REG);
++
++ /* Mask all interrupts */
++ advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
++ advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
++ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
++ advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_MASK_REG);
++
++ /* Clear all interrupts */
++ advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
++ advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
++ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
++ advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
++
++ /* Remove IRQ domains */
++ advk_pcie_remove_rp_irq_domain(pcie);
+ advk_pcie_remove_msi_irq_domain(pcie);
+ advk_pcie_remove_irq_domain(pcie);
+
++ /* Free config space for emulated root bridge */
++ pci_bridge_emul_cleanup(&pcie->bridge);
++
++ /* Assert PERST# signal which prepares PCIe card for power down */
++ if (pcie->reset_gpio)
++ gpiod_set_value_cansleep(pcie->reset_gpio, 1);
++
++ /* Disable link training */
++ val = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
++ val &= ~LINK_TRAINING_EN;
++ advk_writel(pcie, val, PCIE_CORE_CTRL0_REG);
++
+ /* Disable outbound address windows mapping */
+ for (i = 0; i < OB_WIN_COUNT; i++)
+ advk_pcie_disable_ob_win(pcie, i);
+
++ /* Disable phy */
++ advk_pcie_disable_phy(pcie);
++
+ return 0;
+ }
+
+diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
+index 67c46e52c0dc3..5b156c563e3a4 100644
+--- a/drivers/pci/controller/pci-hyperv.c
++++ b/drivers/pci/controller/pci-hyperv.c
+@@ -1899,8 +1899,17 @@ static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus)
+ if (!hv_dev)
+ continue;
+
+- if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY)
+- set_dev_node(&dev->dev, hv_dev->desc.virtual_numa_node);
++ if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY &&
++ hv_dev->desc.virtual_numa_node < num_possible_nodes())
++ /*
++ * The kernel may boot with some NUMA nodes offline
++ * (e.g. in a KDUMP kernel) or with NUMA disabled via
++ * "numa=off". In those cases, adjust the host provided
++ * NUMA node to a valid NUMA node used by the kernel.
++ */
++ set_dev_node(&dev->dev,
++ numa_map_to_online_node(
++ hv_dev->desc.virtual_numa_node));
+
+ put_pcichild(hv_dev);
+ }
+@@ -3139,6 +3148,15 @@ static int hv_pci_probe(struct hv_device *hdev,
+ hbus->bridge->domain_nr = dom;
+ #ifdef CONFIG_X86
+ hbus->sysdata.domain = dom;
++#elif defined(CONFIG_ARM64)
++ /*
++ * Set the PCI bus parent to be the corresponding VMbus
++ * device. Then the VMbus device will be assigned as the
++ * ACPI companion in pcibios_root_bridge_prepare() and
++ * pci_dma_configure() will propagate device coherence
++ * information to devices created on the bus.
++ */
++ hbus->sysdata.parent = hdev->device.parent;
+ #endif
+
+ hbus->hdev = hdev;
+diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c
+index ed13e81cd691d..2a3bf82aa4e26 100644
+--- a/drivers/pci/controller/pci-mvebu.c
++++ b/drivers/pci/controller/pci-mvebu.c
+@@ -51,10 +51,14 @@
+ PCIE_CONF_FUNC(PCI_FUNC(devfn)) | PCIE_CONF_REG(where) | \
+ PCIE_CONF_ADDR_EN)
+ #define PCIE_CONF_DATA_OFF 0x18fc
++#define PCIE_INT_CAUSE_OFF 0x1900
++#define PCIE_INT_PM_PME BIT(28)
+ #define PCIE_MASK_OFF 0x1910
+ #define PCIE_MASK_ENABLE_INTS 0x0f000000
+ #define PCIE_CTRL_OFF 0x1a00
+ #define PCIE_CTRL_X1_MODE 0x0001
++#define PCIE_CTRL_RC_MODE BIT(1)
++#define PCIE_CTRL_MASTER_HOT_RESET BIT(24)
+ #define PCIE_STAT_OFF 0x1a04
+ #define PCIE_STAT_BUS 0xff00
+ #define PCIE_STAT_DEV 0x1f0000
+@@ -125,6 +129,11 @@ static bool mvebu_pcie_link_up(struct mvebu_pcie_port *port)
+ return !(mvebu_readl(port, PCIE_STAT_OFF) & PCIE_STAT_LINK_DOWN);
+ }
+
++static u8 mvebu_pcie_get_local_bus_nr(struct mvebu_pcie_port *port)
++{
++ return (mvebu_readl(port, PCIE_STAT_OFF) & PCIE_STAT_BUS) >> 8;
++}
++
+ static void mvebu_pcie_set_local_bus_nr(struct mvebu_pcie_port *port, int nr)
+ {
+ u32 stat;
+@@ -213,18 +222,21 @@ static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port)
+
+ static void mvebu_pcie_setup_hw(struct mvebu_pcie_port *port)
+ {
+- u32 cmd, mask;
++ u32 ctrl, cmd, mask;
+
+- /* Point PCIe unit MBUS decode windows to DRAM space. */
+- mvebu_pcie_setup_wins(port);
++ /* Setup PCIe controller to Root Complex mode. */
++ ctrl = mvebu_readl(port, PCIE_CTRL_OFF);
++ ctrl |= PCIE_CTRL_RC_MODE;
++ mvebu_writel(port, ctrl, PCIE_CTRL_OFF);
+
+- /* Master + slave enable. */
++ /* Disable Root Bridge I/O space, memory space and bus mastering. */
+ cmd = mvebu_readl(port, PCIE_CMD_OFF);
+- cmd |= PCI_COMMAND_IO;
+- cmd |= PCI_COMMAND_MEMORY;
+- cmd |= PCI_COMMAND_MASTER;
++ cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
+ mvebu_writel(port, cmd, PCIE_CMD_OFF);
+
++ /* Point PCIe unit MBUS decode windows to DRAM space. */
++ mvebu_pcie_setup_wins(port);
++
+ /* Enable interrupt lines A-D. */
+ mask = mvebu_readl(port, PCIE_MASK_OFF);
+ mask |= PCIE_MASK_ENABLE_INTS;
+@@ -371,8 +383,7 @@ static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port)
+
+ /* Are the new iobase/iolimit values invalid? */
+ if (conf->iolimit < conf->iobase ||
+- conf->iolimitupper < conf->iobaseupper ||
+- !(conf->command & PCI_COMMAND_IO)) {
++ conf->iolimitupper < conf->iobaseupper) {
+ mvebu_pcie_set_window(port, port->io_target, port->io_attr,
+ &desired, &port->iowin);
+ return;
+@@ -409,8 +420,7 @@ static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port)
+ struct pci_bridge_emul_conf *conf = &port->bridge.conf;
+
+ /* Are the new membase/memlimit values invalid? */
+- if (conf->memlimit < conf->membase ||
+- !(conf->command & PCI_COMMAND_MEMORY)) {
++ if (conf->memlimit < conf->membase) {
+ mvebu_pcie_set_window(port, port->mem_target, port->mem_attr,
+ &desired, &port->memwin);
+ return;
+@@ -430,6 +440,54 @@ static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port)
+ &port->memwin);
+ }
+
++static pci_bridge_emul_read_status_t
++mvebu_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge,
++ int reg, u32 *value)
++{
++ struct mvebu_pcie_port *port = bridge->data;
++
++ switch (reg) {
++ case PCI_COMMAND:
++ *value = mvebu_readl(port, PCIE_CMD_OFF);
++ break;
++
++ case PCI_PRIMARY_BUS: {
++ /*
++ * From the whole 32bit register we support reading from HW only
++ * secondary bus number which is mvebu local bus number.
++ * Other bits are retrieved only from emulated config buffer.
++ */
++ __le32 *cfgspace = (__le32 *)&bridge->conf;
++ u32 val = le32_to_cpu(cfgspace[PCI_PRIMARY_BUS / 4]);
++ val &= ~0xff00;
++ val |= mvebu_pcie_get_local_bus_nr(port) << 8;
++ *value = val;
++ break;
++ }
++
++ case PCI_INTERRUPT_LINE: {
++ /*
++ * From the whole 32bit register we support reading from HW only
++ * one bit: PCI_BRIDGE_CTL_BUS_RESET.
++ * Other bits are retrieved only from emulated config buffer.
++ */
++ __le32 *cfgspace = (__le32 *)&bridge->conf;
++ u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]);
++ if (mvebu_readl(port, PCIE_CTRL_OFF) & PCIE_CTRL_MASTER_HOT_RESET)
++ val |= PCI_BRIDGE_CTL_BUS_RESET << 16;
++ else
++ val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16);
++ *value = val;
++ break;
++ }
++
++ default:
++ return PCI_BRIDGE_EMUL_NOT_HANDLED;
++ }
++
++ return PCI_BRIDGE_EMUL_HANDLED;
++}
++
+ static pci_bridge_emul_read_status_t
+ mvebu_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+ int reg, u32 *value)
+@@ -442,9 +500,7 @@ mvebu_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+ break;
+
+ case PCI_EXP_DEVCTL:
+- *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL) &
+- ~(PCI_EXP_DEVCTL_URRE | PCI_EXP_DEVCTL_FERE |
+- PCI_EXP_DEVCTL_NFERE | PCI_EXP_DEVCTL_CERE);
++ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL);
+ break;
+
+ case PCI_EXP_LNKCAP:
+@@ -468,6 +524,18 @@ mvebu_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
+ *value = mvebu_readl(port, PCIE_RC_RTSTA);
+ break;
+
++ case PCI_EXP_DEVCAP2:
++ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCAP2);
++ break;
++
++ case PCI_EXP_DEVCTL2:
++ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL2);
++ break;
++
++ case PCI_EXP_LNKCTL2:
++ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL2);
++ break;
++
+ default:
+ return PCI_BRIDGE_EMUL_NOT_HANDLED;
+ }
+@@ -484,26 +552,16 @@ mvebu_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge,
+
+ switch (reg) {
+ case PCI_COMMAND:
+- {
+- if (!mvebu_has_ioport(port))
+- conf->command &= ~PCI_COMMAND_IO;
+-
+- if ((old ^ new) & PCI_COMMAND_IO)
+- mvebu_pcie_handle_iobase_change(port);
+- if ((old ^ new) & PCI_COMMAND_MEMORY)
+- mvebu_pcie_handle_membase_change(port);
++ if (!mvebu_has_ioport(port)) {
++ conf->command = cpu_to_le16(
++ le16_to_cpu(conf->command) & ~PCI_COMMAND_IO);
++ new &= ~PCI_COMMAND_IO;
++ }
+
++ mvebu_writel(port, new, PCIE_CMD_OFF);
+ break;
+- }
+
+ case PCI_IO_BASE:
+- /*
+- * We keep bit 1 set, it is a read-only bit that
+- * indicates we support 32 bits addressing for the
+- * I/O
+- */
+- conf->iobase |= PCI_IO_RANGE_TYPE_32;
+- conf->iolimit |= PCI_IO_RANGE_TYPE_32;
+ mvebu_pcie_handle_iobase_change(port);
+ break;
+
+@@ -516,7 +574,19 @@ mvebu_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge,
+ break;
+
+ case PCI_PRIMARY_BUS:
+- mvebu_pcie_set_local_bus_nr(port, conf->secondary_bus);
++ if (mask & 0xff00)
++ mvebu_pcie_set_local_bus_nr(port, conf->secondary_bus);
++ break;
++
++ case PCI_INTERRUPT_LINE:
++ if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) {
++ u32 ctrl = mvebu_readl(port, PCIE_CTRL_OFF);
++ if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16))
++ ctrl |= PCIE_CTRL_MASTER_HOT_RESET;
++ else
++ ctrl &= ~PCIE_CTRL_MASTER_HOT_RESET;
++ mvebu_writel(port, ctrl, PCIE_CTRL_OFF);
++ }
+ break;
+
+ default:
+@@ -532,13 +602,6 @@ mvebu_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
+
+ switch (reg) {
+ case PCI_EXP_DEVCTL:
+- /*
+- * Armada370 data says these bits must always
+- * be zero when in root complex mode.
+- */
+- new &= ~(PCI_EXP_DEVCTL_URRE | PCI_EXP_DEVCTL_FERE |
+- PCI_EXP_DEVCTL_NFERE | PCI_EXP_DEVCTL_CERE);
+-
+ mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL);
+ break;
+
+@@ -555,12 +618,31 @@ mvebu_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
+ break;
+
+ case PCI_EXP_RTSTA:
+- mvebu_writel(port, new, PCIE_RC_RTSTA);
++ /*
++ * PME Status bit in Root Status Register (PCIE_RC_RTSTA)
++ * is read-only and can be cleared only by writing 0b to the
++ * Interrupt Cause RW0C register (PCIE_INT_CAUSE_OFF). So
++ * clear PME via Interrupt Cause.
++ */
++ if (new & PCI_EXP_RTSTA_PME)
++ mvebu_writel(port, ~PCIE_INT_PM_PME, PCIE_INT_CAUSE_OFF);
++ break;
++
++ case PCI_EXP_DEVCTL2:
++ mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL2);
++ break;
++
++ case PCI_EXP_LNKCTL2:
++ mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL2);
++ break;
++
++ default:
+ break;
+ }
+ }
+
+ static struct pci_bridge_emul_ops mvebu_pci_bridge_emul_ops = {
++ .read_base = mvebu_pci_bridge_emul_base_conf_read,
+ .write_base = mvebu_pci_bridge_emul_base_conf_write,
+ .read_pcie = mvebu_pci_bridge_emul_pcie_conf_read,
+ .write_pcie = mvebu_pci_bridge_emul_pcie_conf_write,
+@@ -570,9 +652,11 @@ static struct pci_bridge_emul_ops mvebu_pci_bridge_emul_ops = {
+ * Initialize the configuration space of the PCI-to-PCI bridge
+ * associated with the given PCIe interface.
+ */
+-static void mvebu_pci_bridge_emul_init(struct mvebu_pcie_port *port)
++static int mvebu_pci_bridge_emul_init(struct mvebu_pcie_port *port)
+ {
+ struct pci_bridge_emul *bridge = &port->bridge;
++ u32 pcie_cap = mvebu_readl(port, PCIE_CAP_PCIEXP);
++ u8 pcie_cap_ver = ((pcie_cap >> 16) & PCI_EXP_FLAGS_VERS);
+
+ bridge->conf.vendor = PCI_VENDOR_ID_MARVELL;
+ bridge->conf.device = mvebu_readl(port, PCIE_DEV_ID_OFF) >> 16;
+@@ -585,11 +669,17 @@ static void mvebu_pci_bridge_emul_init(struct mvebu_pcie_port *port)
+ bridge->conf.iolimit = PCI_IO_RANGE_TYPE_32;
+ }
+
++ /*
++ * Older mvebu hardware provides PCIe Capability structure only in
++ * version 1. New hardware provides it in version 2.
++ */
++ bridge->pcie_conf.cap = cpu_to_le16(pcie_cap_ver);
++
+ bridge->has_pcie = true;
+ bridge->data = port;
+ bridge->ops = &mvebu_pci_bridge_emul_ops;
+
+- pci_bridge_emul_init(bridge, PCI_BRIDGE_EMUL_NO_PREFETCHABLE_BAR);
++ return pci_bridge_emul_init(bridge, PCI_BRIDGE_EMUL_NO_PREFETCHABLE_BAR);
+ }
+
+ static inline struct mvebu_pcie *sys_to_pcie(struct pci_sys_data *sys)
+@@ -1112,9 +1202,94 @@ static int mvebu_pcie_probe(struct platform_device *pdev)
+ continue;
+ }
+
++ ret = mvebu_pci_bridge_emul_init(port);
++ if (ret < 0) {
++ dev_err(dev, "%s: cannot init emulated bridge\n",
++ port->name);
++ devm_iounmap(dev, port->base);
++ port->base = NULL;
++ mvebu_pcie_powerdown(port);
++ continue;
++ }
++
++ /*
++ * PCIe topology exported by mvebu hw is quite complicated. In
++ * reality has something like N fully independent host bridges
++ * where each host bridge has one PCIe Root Port (which acts as
++ * PCI Bridge device). Each host bridge has its own independent
++ * internal registers, independent access to PCI config space,
++ * independent interrupt lines, independent window and memory
++ * access configuration. But additionally there is some kind of
++ * peer-to-peer support between PCIe devices behind different
++ * host bridges limited just to forwarding of memory and I/O
++ * transactions (forwarding of error messages and config cycles
++ * is not supported). So we could say there are N independent
++ * PCIe Root Complexes.
++ *
++ * For this kind of setup DT should have been structured into
++ * N independent PCIe controllers / host bridges. But instead
++ * structure in past was defined to put PCIe Root Ports of all
++ * host bridges into one bus zero, like in classic multi-port
++ * Root Complex setup with just one host bridge.
++ *
++ * This means that pci-mvebu.c driver provides "virtual" bus 0
++ * on which registers all PCIe Root Ports (PCI Bridge devices)
++ * specified in DT by their BDF addresses and virtually routes
++ * PCI config access of each PCI bridge device to specific PCIe
++ * host bridge.
++ *
++ * Normally PCI Bridge should choose between Type 0 and Type 1
++ * config requests based on primary and secondary bus numbers
++ * configured on the bridge itself. But because mvebu PCI Bridge
++ * does not have registers for primary and secondary bus numbers
++ * in its config space, it determinates type of config requests
++ * via its own custom way.
++ *
++ * There are two options how mvebu determinate type of config
++ * request.
++ *
++ * 1. If Secondary Bus Number Enable bit is not set or is not
++ * available (applies for pre-XP PCIe controllers) then Type 0
++ * is used if target bus number equals Local Bus Number (bits
++ * [15:8] in register 0x1a04) and target device number differs
++ * from Local Device Number (bits [20:16] in register 0x1a04).
++ * Type 1 is used if target bus number differs from Local Bus
++ * Number. And when target bus number equals Local Bus Number
++ * and target device equals Local Device Number then request is
++ * routed to Local PCI Bridge (PCIe Root Port).
++ *
++ * 2. If Secondary Bus Number Enable bit is set (bit 7 in
++ * register 0x1a2c) then mvebu hw determinate type of config
++ * request like compliant PCI Bridge based on primary bus number
++ * which is configured via Local Bus Number (bits [15:8] in
++ * register 0x1a04) and secondary bus number which is configured
++ * via Secondary Bus Number (bits [7:0] in register 0x1a2c).
++ * Local PCI Bridge (PCIe Root Port) is available on primary bus
++ * as device with Local Device Number (bits [20:16] in register
++ * 0x1a04).
++ *
++ * Secondary Bus Number Enable bit is disabled by default and
++ * option 2. is not available on pre-XP PCIe controllers. Hence
++ * this driver always use option 1.
++ *
++ * Basically it means that primary and secondary buses shares
++ * one virtual number configured via Local Bus Number bits and
++ * Local Device Number bits determinates if accessing primary
++ * or secondary bus. Set Local Device Number to 1 and redirect
++ * all writes of PCI Bridge Secondary Bus Number register to
++ * Local Bus Number (bits [15:8] in register 0x1a04).
++ *
++ * So when accessing devices on buses behind secondary bus
++ * number it would work correctly. And also when accessing
++ * device 0 at secondary bus number via config space would be
++ * correctly routed to secondary bus. Due to issues described
++ * in mvebu_pcie_setup_hw(), PCI Bridges at primary bus (zero)
++ * are not accessed directly via PCI config space but rarher
++ * indirectly via kernel emulated PCI bridge driver.
++ */
+ mvebu_pcie_setup_hw(port);
+ mvebu_pcie_set_local_dev_nr(port, 1);
+- mvebu_pci_bridge_emul_init(port);
++ mvebu_pcie_set_local_bus_nr(port, 0);
+ }
+
+ pcie->nports = i;
+diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c
+index e64536047b651..4641e57487cfd 100644
+--- a/drivers/pci/controller/pci-xgene.c
++++ b/drivers/pci/controller/pci-xgene.c
+@@ -480,28 +480,27 @@ static int xgene_pcie_select_ib_reg(u8 *ib_reg_mask, u64 size)
+ }
+
+ static void xgene_pcie_setup_ib_reg(struct xgene_pcie_port *port,
+- struct resource_entry *entry,
+- u8 *ib_reg_mask)
++ struct of_pci_range *range, u8 *ib_reg_mask)
+ {
+ void __iomem *cfg_base = port->cfg_base;
+ struct device *dev = port->dev;
+ void __iomem *bar_addr;
+ u32 pim_reg;
+- u64 cpu_addr = entry->res->start;
+- u64 pci_addr = cpu_addr - entry->offset;
+- u64 size = resource_size(entry->res);
++ u64 cpu_addr = range->cpu_addr;
++ u64 pci_addr = range->pci_addr;
++ u64 size = range->size;
+ u64 mask = ~(size - 1) | EN_REG;
+ u32 flags = PCI_BASE_ADDRESS_MEM_TYPE_64;
+ u32 bar_low;
+ int region;
+
+- region = xgene_pcie_select_ib_reg(ib_reg_mask, size);
++ region = xgene_pcie_select_ib_reg(ib_reg_mask, range->size);
+ if (region < 0) {
+ dev_warn(dev, "invalid pcie dma-range config\n");
+ return;
+ }
+
+- if (entry->res->flags & IORESOURCE_PREFETCH)
++ if (range->flags & IORESOURCE_PREFETCH)
+ flags |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+
+ bar_low = pcie_bar_low_val((u32)cpu_addr, flags);
+@@ -532,13 +531,25 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie_port *port,
+
+ static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie_port *port)
+ {
+- struct pci_host_bridge *bridge = pci_host_bridge_from_priv(port);
+- struct resource_entry *entry;
++ struct device_node *np = port->node;
++ struct of_pci_range range;
++ struct of_pci_range_parser parser;
++ struct device *dev = port->dev;
+ u8 ib_reg_mask = 0;
+
+- resource_list_for_each_entry(entry, &bridge->dma_ranges)
+- xgene_pcie_setup_ib_reg(port, entry, &ib_reg_mask);
++ if (of_pci_dma_range_parser_init(&parser, np)) {
++ dev_err(dev, "missing dma-ranges property\n");
++ return -EINVAL;
++ }
++
++ /* Get the dma-ranges from DT */
++ for_each_of_pci_range(&parser, &range) {
++ u64 end = range.cpu_addr + range.size - 1;
+
++ dev_dbg(dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
++ range.flags, range.cpu_addr, end, range.pci_addr);
++ xgene_pcie_setup_ib_reg(port, &range, &ib_reg_mask);
++ }
+ return 0;
+ }
+
+diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c
+index 17c59b0d6978b..21207df680ccf 100644
+--- a/drivers/pci/controller/pcie-mediatek-gen3.c
++++ b/drivers/pci/controller/pcie-mediatek-gen3.c
+@@ -79,6 +79,9 @@
+ #define PCIE_ICMD_PM_REG 0x198
+ #define PCIE_TURN_OFF_LINK BIT(4)
+
++#define PCIE_MISC_CTRL_REG 0x348
++#define PCIE_DISABLE_DVFSRC_VLT_REQ BIT(1)
++
+ #define PCIE_TRANS_TABLE_BASE_REG 0x800
+ #define PCIE_ATR_SRC_ADDR_MSB_OFFSET 0x4
+ #define PCIE_ATR_TRSL_ADDR_LSB_OFFSET 0x8
+@@ -297,6 +300,11 @@ static int mtk_pcie_startup_port(struct mtk_pcie_port *port)
+ val &= ~PCIE_INTX_ENABLE;
+ writel_relaxed(val, port->base + PCIE_INT_ENABLE_REG);
+
++ /* Disable DVFSRC voltage request */
++ val = readl_relaxed(port->base + PCIE_MISC_CTRL_REG);
++ val |= PCIE_DISABLE_DVFSRC_VLT_REQ;
++ writel_relaxed(val, port->base + PCIE_MISC_CTRL_REG);
++
+ /* Assert all reset signals */
+ val = readl_relaxed(port->base + PCIE_RST_CTRL_REG);
+ val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB | PCIE_PE_RSTB;
+diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c
+index 8f3131844e777..bfb13f358d073 100644
+--- a/drivers/pci/controller/pcie-rcar-host.c
++++ b/drivers/pci/controller/pcie-rcar-host.c
+@@ -52,10 +52,10 @@ struct rcar_msi {
+ */
+ static void __iomem *pcie_base;
+ /*
+- * Static copy of bus clock pointer, so we can check whether the clock
+- * is enabled or not.
++ * Static copy of PCIe device pointer, so we can check whether the
++ * device is runtime suspended or not.
+ */
+-static struct clk *pcie_bus_clk;
++static struct device *pcie_dev;
+ #endif
+
+ /* Structure representing the PCIe interface */
+@@ -794,7 +794,7 @@ static int rcar_pcie_get_resources(struct rcar_pcie_host *host)
+ #ifdef CONFIG_ARM
+ /* Cache static copy for L1 link state fixup hook on aarch32 */
+ pcie_base = pcie->base;
+- pcie_bus_clk = host->bus_clk;
++ pcie_dev = pcie->dev;
+ #endif
+
+ return 0;
+@@ -1064,7 +1064,7 @@ static int rcar_pcie_aarch32_abort_handler(unsigned long addr,
+
+ spin_lock_irqsave(&pmsr_lock, flags);
+
+- if (!pcie_base || !__clk_is_enabled(pcie_bus_clk)) {
++ if (!pcie_base || pm_runtime_suspended(pcie_dev)) {
+ ret = 1;
+ goto unlock_exit;
+ }
+diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
+index 90d84d3bc868f..5b833f00e9800 100644
+--- a/drivers/pci/endpoint/functions/pci-epf-test.c
++++ b/drivers/pci/endpoint/functions/pci-epf-test.c
+@@ -285,7 +285,17 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test)
+ if (ret)
+ dev_err(dev, "Data transfer failed\n");
+ } else {
+- memcpy(dst_addr, src_addr, reg->size);
++ void *buf;
++
++ buf = kzalloc(reg->size, GFP_KERNEL);
++ if (!buf) {
++ ret = -ENOMEM;
++ goto err_map_addr;
++ }
++
++ memcpy_fromio(buf, src_addr, reg->size);
++ memcpy_toio(dst_addr, buf, reg->size);
++ kfree(buf);
+ }
+ ktime_get_ts64(&end);
+ pci_epf_test_print_rate("COPY", reg->size, &start, &end, use_dma);
+@@ -441,7 +451,7 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test)
+ if (!epf_test->dma_supported) {
+ dev_err(dev, "Cannot transfer data using DMA\n");
+ ret = -EINVAL;
+- goto err_map_addr;
++ goto err_dma_map;
+ }
+
+ src_phys_addr = dma_map_single(dma_dev, buf, reg->size,
+diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
+index 69fd401691be6..10d7e7e1b5530 100644
+--- a/drivers/pci/hotplug/pciehp.h
++++ b/drivers/pci/hotplug/pciehp.h
+@@ -75,6 +75,8 @@ extern int pciehp_poll_time;
+ * @reset_lock: prevents access to the Data Link Layer Link Active bit in the
+ * Link Status register and to the Presence Detect State bit in the Slot
+ * Status register during a slot reset which may cause them to flap
++ * @depth: Number of additional hotplug ports in the path to the root bus,
++ * used as lock subclass for @reset_lock
+ * @ist_running: flag to keep user request waiting while IRQ thread is running
+ * @request_result: result of last user request submitted to the IRQ thread
+ * @requester: wait queue to wake up on completion of user request,
+@@ -106,6 +108,7 @@ struct controller {
+
+ struct hotplug_slot hotplug_slot; /* hotplug core interface */
+ struct rw_semaphore reset_lock;
++ unsigned int depth;
+ unsigned int ist_running;
+ int request_result;
+ wait_queue_head_t requester;
+diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
+index ad3393930ecb4..e7fe4b42f0394 100644
+--- a/drivers/pci/hotplug/pciehp_core.c
++++ b/drivers/pci/hotplug/pciehp_core.c
+@@ -166,7 +166,7 @@ static void pciehp_check_presence(struct controller *ctrl)
+ {
+ int occupied;
+
+- down_read(&ctrl->reset_lock);
++ down_read_nested(&ctrl->reset_lock, ctrl->depth);
+ mutex_lock(&ctrl->state_lock);
+
+ occupied = pciehp_card_present_or_link_active(ctrl);
+diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
+index 3024d7e85e6a7..8bedbc77fe95f 100644
+--- a/drivers/pci/hotplug/pciehp_hpc.c
++++ b/drivers/pci/hotplug/pciehp_hpc.c
+@@ -98,6 +98,8 @@ static int pcie_poll_cmd(struct controller *ctrl, int timeout)
+ if (slot_status & PCI_EXP_SLTSTA_CC) {
+ pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
+ PCI_EXP_SLTSTA_CC);
++ ctrl->cmd_busy = 0;
++ smp_mb();
+ return 1;
+ }
+ msleep(10);
+@@ -583,7 +585,7 @@ static void pciehp_ignore_dpc_link_change(struct controller *ctrl,
+ * the corresponding link change may have been ignored above.
+ * Synthesize it to ensure that it is acted on.
+ */
+- down_read(&ctrl->reset_lock);
++ down_read_nested(&ctrl->reset_lock, ctrl->depth);
+ if (!pciehp_check_link_active(ctrl))
+ pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC);
+ up_read(&ctrl->reset_lock);
+@@ -642,6 +644,8 @@ read_status:
+ */
+ if (ctrl->power_fault_detected)
+ status &= ~PCI_EXP_SLTSTA_PFD;
++ else if (status & PCI_EXP_SLTSTA_PFD)
++ ctrl->power_fault_detected = true;
+
+ events |= status;
+ if (!events) {
+@@ -651,7 +655,7 @@ read_status:
+ }
+
+ if (status) {
+- pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, events);
++ pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, status);
+
+ /*
+ * In MSI mode, all event bits must be zero before the port
+@@ -725,8 +729,7 @@ static irqreturn_t pciehp_ist(int irq, void *dev_id)
+ }
+
+ /* Check Power Fault Detected */
+- if ((events & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) {
+- ctrl->power_fault_detected = 1;
++ if (events & PCI_EXP_SLTSTA_PFD) {
+ ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));
+ pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
+ PCI_EXP_SLTCTL_ATTN_IND_ON);
+@@ -746,7 +749,7 @@ static irqreturn_t pciehp_ist(int irq, void *dev_id)
+ * Disable requests have higher priority than Presence Detect Changed
+ * or Data Link Layer State Changed events.
+ */
+- down_read(&ctrl->reset_lock);
++ down_read_nested(&ctrl->reset_lock, ctrl->depth);
+ if (events & DISABLE_SLOT)
+ pciehp_handle_disable_request(ctrl);
+ else if (events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC))
+@@ -880,7 +883,7 @@ int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe)
+ if (probe)
+ return 0;
+
+- down_write(&ctrl->reset_lock);
++ down_write_nested(&ctrl->reset_lock, ctrl->depth);
+
+ if (!ATTN_BUTTN(ctrl)) {
+ ctrl_mask |= PCI_EXP_SLTCTL_PDCE;
+@@ -936,6 +939,20 @@ static inline void dbg_ctrl(struct controller *ctrl)
+
+ #define FLAG(x, y) (((x) & (y)) ? '+' : '-')
+
++static inline int pcie_hotplug_depth(struct pci_dev *dev)
++{
++ struct pci_bus *bus = dev->bus;
++ int depth = 0;
++
++ while (bus->parent) {
++ bus = bus->parent;
++ if (bus->self && bus->self->is_hotplug_bridge)
++ depth++;
++ }
++
++ return depth;
++}
++
+ struct controller *pcie_init(struct pcie_device *dev)
+ {
+ struct controller *ctrl;
+@@ -949,6 +966,7 @@ struct controller *pcie_init(struct pcie_device *dev)
+ return NULL;
+
+ ctrl->pcie = dev;
++ ctrl->depth = pcie_hotplug_depth(dev->port);
+ pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &slot_cap);
+
+ if (pdev->hotplug_user_indicators)
+@@ -1042,6 +1060,8 @@ static void quirk_cmd_compl(struct pci_dev *pdev)
+ }
+ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
+ PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
++DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0110,
++ PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
+ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0400,
+ PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
+ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0401,
+diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
+index 4b4792940e869..cc4c2b8a5efd7 100644
+--- a/drivers/pci/msi.c
++++ b/drivers/pci/msi.c
+@@ -148,6 +148,9 @@ static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 s
+ raw_spinlock_t *lock = &desc->dev->msi_lock;
+ unsigned long flags;
+
++ if (!desc->msi_attrib.can_mask)
++ return;
++
+ raw_spin_lock_irqsave(lock, flags);
+ desc->msi_mask &= ~clear;
+ desc->msi_mask |= set;
+@@ -181,7 +184,8 @@ static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl)
+ {
+ void __iomem *desc_addr = pci_msix_desc_addr(desc);
+
+- writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
++ if (desc->msi_attrib.can_mask)
++ writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+ }
+
+ static inline void pci_msix_mask(struct msi_desc *desc)
+@@ -200,23 +204,17 @@ static inline void pci_msix_unmask(struct msi_desc *desc)
+
+ static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask)
+ {
+- if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+- return;
+-
+ if (desc->msi_attrib.is_msix)
+ pci_msix_mask(desc);
+- else if (desc->msi_attrib.maskbit)
++ else
+ pci_msi_mask(desc, mask);
+ }
+
+ static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask)
+ {
+- if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+- return;
+-
+ if (desc->msi_attrib.is_msix)
+ pci_msix_unmask(desc);
+- else if (desc->msi_attrib.maskbit)
++ else
+ pci_msi_unmask(desc, mask);
+ }
+
+@@ -370,6 +368,11 @@ static void free_msi_irqs(struct pci_dev *dev)
+ for (i = 0; i < entry->nvec_used; i++)
+ BUG_ON(irq_has_action(entry->irq + i));
+
++ if (dev->msi_irq_groups) {
++ msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups);
++ dev->msi_irq_groups = NULL;
++ }
++
+ pci_msi_teardown_msi_irqs(dev);
+
+ list_for_each_entry_safe(entry, tmp, msi_list, list) {
+@@ -381,11 +384,6 @@ static void free_msi_irqs(struct pci_dev *dev)
+ list_del(&entry->list);
+ free_msi_entry(entry);
+ }
+-
+- if (dev->msi_irq_groups) {
+- msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups);
+- dev->msi_irq_groups = NULL;
+- }
+ }
+
+ static void pci_intx_for_msi(struct pci_dev *dev, int enable)
+@@ -479,12 +477,16 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
+ goto out;
+
+ pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
++ /* Lies, damned lies, and MSIs */
++ if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING)
++ control |= PCI_MSI_FLAGS_MASKBIT;
+
+ entry->msi_attrib.is_msix = 0;
+ entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
+ entry->msi_attrib.is_virtual = 0;
+ entry->msi_attrib.entry_nr = 0;
+- entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
++ entry->msi_attrib.can_mask = !pci_msi_ignore_mask &&
++ !!(control & PCI_MSI_FLAGS_MASKBIT);
+ entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
+ entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
+ entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
+@@ -495,7 +497,7 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
+ entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32;
+
+ /* Save the initial mask status */
+- if (entry->msi_attrib.maskbit)
++ if (entry->msi_attrib.can_mask)
+ pci_read_config_dword(dev, entry->mask_pos, &entry->msi_mask);
+
+ out:
+@@ -638,10 +640,13 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
+ entry->msi_attrib.is_virtual =
+ entry->msi_attrib.entry_nr >= vec_count;
+
++ entry->msi_attrib.can_mask = !pci_msi_ignore_mask &&
++ !entry->msi_attrib.is_virtual;
++
+ entry->msi_attrib.default_irq = dev->irq;
+ entry->mask_base = base;
+
+- if (!entry->msi_attrib.is_virtual) {
++ if (entry->msi_attrib.can_mask) {
+ addr = pci_msix_desc_addr(entry);
+ entry->msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
+ }
+@@ -716,9 +721,6 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
+ goto out_disable;
+ }
+
+- /* Ensure that all table entries are masked. */
+- msix_mask_all(base, tsize);
+-
+ ret = msix_setup_entries(dev, base, entries, nvec, affd);
+ if (ret)
+ goto out_disable;
+@@ -745,6 +747,16 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
+ /* Set MSI-X enabled bits and unmask the function */
+ pci_intx_for_msi(dev, 0);
+ dev->msix_enabled = 1;
++
++ /*
++ * Ensure that all table entries are masked to prevent
++ * stale entries from firing in a crash kernel.
++ *
++ * Done late to deal with a broken Marvell NVME device
++ * which takes the MSI-X mask bits into account even
++ * when MSI-X is disabled, which prevents MSI delivery.
++ */
++ msix_mask_all(base, tsize);
+ pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
+
+ pcibios_free_irq(dev);
+@@ -771,7 +783,7 @@ out_free:
+ free_msi_irqs(dev);
+
+ out_disable:
+- pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
++ pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0);
+
+ return ret;
+ }
+@@ -1181,19 +1193,24 @@ EXPORT_SYMBOL(pci_free_irq_vectors);
+
+ /**
+ * pci_irq_vector - return Linux IRQ number of a device vector
+- * @dev: PCI device to operate on
+- * @nr: device-relative interrupt vector index (0-based).
++ * @dev: PCI device to operate on
++ * @nr: Interrupt vector index (0-based)
++ *
++ * @nr has the following meanings depending on the interrupt mode:
++ * MSI-X: The index in the MSI-X vector table
++ * MSI: The index of the enabled MSI vectors
++ * INTx: Must be 0
++ *
++ * Return: The Linux interrupt number or -EINVAl if @nr is out of range.
+ */
+ int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
+ {
+ if (dev->msix_enabled) {
+ struct msi_desc *entry;
+- int i = 0;
+
+ for_each_pci_msi_entry(entry, dev) {
+- if (i == nr)
++ if (entry->msi_attrib.entry_nr == nr)
+ return entry->irq;
+- i++;
+ }
+ WARN_ON_ONCE(1);
+ return -EINVAL;
+@@ -1217,17 +1234,22 @@ EXPORT_SYMBOL(pci_irq_vector);
+ * pci_irq_get_affinity - return the affinity of a particular MSI vector
+ * @dev: PCI device to operate on
+ * @nr: device-relative interrupt vector index (0-based).
++ *
++ * @nr has the following meanings depending on the interrupt mode:
++ * MSI-X: The index in the MSI-X vector table
++ * MSI: The index of the enabled MSI vectors
++ * INTx: Must be 0
++ *
++ * Return: A cpumask pointer or NULL if @nr is out of range
+ */
+ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
+ {
+ if (dev->msix_enabled) {
+ struct msi_desc *entry;
+- int i = 0;
+
+ for_each_pci_msi_entry(entry, dev) {
+- if (i == nr)
++ if (entry->msi_attrib.entry_nr == nr)
+ return &entry->affinity->mask;
+- i++;
+ }
+ WARN_ON_ONCE(1);
+ return NULL;
+diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c
+index fdaf86a888b73..c994ebec23603 100644
+--- a/drivers/pci/pci-bridge-emul.c
++++ b/drivers/pci/pci-bridge-emul.c
+@@ -139,8 +139,13 @@ struct pci_bridge_reg_behavior pci_regs_behavior[PCI_STD_HEADER_SIZEOF / 4] = {
+ .ro = GENMASK(7, 0),
+ },
+
++ /*
++ * If expansion ROM is unsupported then ROM Base Address register must
++ * be implemented as read-only register that return 0 when read, same
++ * as for unused Base Address registers.
++ */
+ [PCI_ROM_ADDRESS1 / 4] = {
+- .rw = GENMASK(31, 11) | BIT(0),
++ .ro = ~0,
+ },
+
+ /*
+@@ -171,41 +176,55 @@ struct pci_bridge_reg_behavior pcie_cap_regs_behavior[PCI_CAP_PCIE_SIZEOF / 4] =
+ [PCI_CAP_LIST_ID / 4] = {
+ /*
+ * Capability ID, Next Capability Pointer and
+- * Capabilities register are all read-only.
++ * bits [14:0] of Capabilities register are all read-only.
++ * Bit 15 of Capabilities register is reserved.
+ */
+- .ro = ~0,
++ .ro = GENMASK(30, 0),
+ },
+
+ [PCI_EXP_DEVCAP / 4] = {
+- .ro = ~0,
++ /*
++ * Bits [31:29] and [17:16] are reserved.
++ * Bits [27:18] are reserved for non-upstream ports.
++ * Bits 28 and [14:6] are reserved for non-endpoint devices.
++ * Other bits are read-only.
++ */
++ .ro = BIT(15) | GENMASK(5, 0),
+ },
+
+ [PCI_EXP_DEVCTL / 4] = {
+- /* Device control register is RW */
+- .rw = GENMASK(15, 0),
++ /*
++ * Device control register is RW, except bit 15 which is
++ * reserved for non-endpoints or non-PCIe-to-PCI/X bridges.
++ */
++ .rw = GENMASK(14, 0),
+
+ /*
+ * Device status register has bits 6 and [3:0] W1C, [5:4] RO,
+- * the rest is reserved
++ * the rest is reserved. Also bit 6 is reserved for non-upstream
++ * ports.
+ */
+- .w1c = (BIT(6) | GENMASK(3, 0)) << 16,
++ .w1c = GENMASK(3, 0) << 16,
+ .ro = GENMASK(5, 4) << 16,
+ },
+
+ [PCI_EXP_LNKCAP / 4] = {
+- /* All bits are RO, except bit 23 which is reserved */
+- .ro = lower_32_bits(~BIT(23)),
++ /*
++ * All bits are RO, except bit 23 which is reserved and
++ * bit 18 which is reserved for non-upstream ports.
++ */
++ .ro = lower_32_bits(~(BIT(23) | PCI_EXP_LNKCAP_CLKPM)),
+ },
+
+ [PCI_EXP_LNKCTL / 4] = {
+ /*
+ * Link control has bits [15:14], [11:3] and [1:0] RW, the
+- * rest is reserved.
++ * rest is reserved. Bit 8 is reserved for non-upstream ports.
+ *
+ * Link status has bits [13:0] RO, and bits [15:14]
+ * W1C.
+ */
+- .rw = GENMASK(15, 14) | GENMASK(11, 3) | GENMASK(1, 0),
++ .rw = GENMASK(15, 14) | GENMASK(11, 9) | GENMASK(7, 3) | GENMASK(1, 0),
+ .ro = GENMASK(13, 0) << 16,
+ .w1c = GENMASK(15, 14) << 16,
+ },
+@@ -251,6 +270,49 @@ struct pci_bridge_reg_behavior pcie_cap_regs_behavior[PCI_CAP_PCIE_SIZEOF / 4] =
+ .ro = GENMASK(15, 0) | PCI_EXP_RTSTA_PENDING,
+ .w1c = PCI_EXP_RTSTA_PME,
+ },
++
++ [PCI_EXP_DEVCAP2 / 4] = {
++ /*
++ * Device capabilities 2 register has reserved bits [30:27].
++ * Also bits [26:24] are reserved for non-upstream ports.
++ */
++ .ro = BIT(31) | GENMASK(23, 0),
++ },
++
++ [PCI_EXP_DEVCTL2 / 4] = {
++ /*
++ * Device control 2 register is RW. Bit 11 is reserved for
++ * non-upstream ports.
++ *
++ * Device status 2 register is reserved.
++ */
++ .rw = GENMASK(15, 12) | GENMASK(10, 0),
++ },
++
++ [PCI_EXP_LNKCAP2 / 4] = {
++ /* Link capabilities 2 register has reserved bits [30:25] and 0. */
++ .ro = BIT(31) | GENMASK(24, 1),
++ },
++
++ [PCI_EXP_LNKCTL2 / 4] = {
++ /*
++ * Link control 2 register is RW.
++ *
++ * Link status 2 register has bits 5, 15 W1C;
++ * bits 10, 11 reserved and others are RO.
++ */
++ .rw = GENMASK(15, 0),
++ .w1c = (BIT(15) | BIT(5)) << 16,
++ .ro = (GENMASK(14, 12) | GENMASK(9, 6) | GENMASK(4, 0)) << 16,
++ },
++
++ [PCI_EXP_SLTCAP2 / 4] = {
++ /* Slot capabilities 2 register is reserved. */
++ },
++
++ [PCI_EXP_SLTCTL2 / 4] = {
++ /* Both Slot control 2 and Slot status 2 registers are reserved. */
++ },
+ };
+
+ /*
+@@ -265,7 +327,11 @@ int pci_bridge_emul_init(struct pci_bridge_emul *bridge,
+ {
+ BUILD_BUG_ON(sizeof(bridge->conf) != PCI_BRIDGE_CONF_END);
+
+- bridge->conf.class_revision |= cpu_to_le32(PCI_CLASS_BRIDGE_PCI << 16);
++ /*
++ * class_revision: Class is high 24 bits and revision is low 8 bit of this member,
++ * while class for PCI Bridge Normal Decode has the 24-bit value: PCI_CLASS_BRIDGE_PCI << 8
++ */
++ bridge->conf.class_revision |= cpu_to_le32((PCI_CLASS_BRIDGE_PCI << 8) << 8);
+ bridge->conf.header_type = PCI_HEADER_TYPE_BRIDGE;
+ bridge->conf.cache_line_size = 0x10;
+ bridge->conf.status = cpu_to_le16(PCI_STATUS_CAP_LIST);
+@@ -277,11 +343,9 @@ int pci_bridge_emul_init(struct pci_bridge_emul *bridge,
+
+ if (bridge->has_pcie) {
+ bridge->conf.capabilities_pointer = PCI_CAP_PCIE_START;
++ bridge->conf.status |= cpu_to_le16(PCI_STATUS_CAP_LIST);
+ bridge->pcie_conf.cap_id = PCI_CAP_ID_EXP;
+- /* Set PCIe v2, root port, slot support */
+- bridge->pcie_conf.cap =
+- cpu_to_le16(PCI_EXP_TYPE_ROOT_PORT << 4 | 2 |
+- PCI_EXP_FLAGS_SLOT);
++ bridge->pcie_conf.cap |= cpu_to_le16(PCI_EXP_TYPE_ROOT_PORT << 4);
+ bridge->pcie_cap_regs_behavior =
+ kmemdup(pcie_cap_regs_behavior,
+ sizeof(pcie_cap_regs_behavior),
+@@ -290,6 +354,27 @@ int pci_bridge_emul_init(struct pci_bridge_emul *bridge,
+ kfree(bridge->pci_regs_behavior);
+ return -ENOMEM;
+ }
++ /* These bits are applicable only for PCI and reserved on PCIe */
++ bridge->pci_regs_behavior[PCI_CACHE_LINE_SIZE / 4].ro &=
++ ~GENMASK(15, 8);
++ bridge->pci_regs_behavior[PCI_COMMAND / 4].ro &=
++ ~((PCI_COMMAND_SPECIAL | PCI_COMMAND_INVALIDATE |
++ PCI_COMMAND_VGA_PALETTE | PCI_COMMAND_WAIT |
++ PCI_COMMAND_FAST_BACK) |
++ (PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK |
++ PCI_STATUS_DEVSEL_MASK) << 16);
++ bridge->pci_regs_behavior[PCI_PRIMARY_BUS / 4].ro &=
++ ~GENMASK(31, 24);
++ bridge->pci_regs_behavior[PCI_IO_BASE / 4].ro &=
++ ~((PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK |
++ PCI_STATUS_DEVSEL_MASK) << 16);
++ bridge->pci_regs_behavior[PCI_INTERRUPT_LINE / 4].rw &=
++ ~((PCI_BRIDGE_CTL_MASTER_ABORT |
++ BIT(8) | BIT(9) | BIT(11)) << 16);
++ bridge->pci_regs_behavior[PCI_INTERRUPT_LINE / 4].ro &=
++ ~((PCI_BRIDGE_CTL_FAST_BACK) << 16);
++ bridge->pci_regs_behavior[PCI_INTERRUPT_LINE / 4].w1c &=
++ ~(BIT(10) << 16);
+ }
+
+ if (flags & PCI_BRIDGE_EMUL_NO_PREFETCHABLE_BAR) {
+@@ -431,8 +516,21 @@ int pci_bridge_emul_conf_write(struct pci_bridge_emul *bridge, int where,
+ /* Clear the W1C bits */
+ new &= ~((value << shift) & (behavior[reg / 4].w1c & mask));
+
++ /* Save the new value with the cleared W1C bits into the cfgspace */
+ cfgspace[reg / 4] = cpu_to_le32(new);
+
++ /*
++ * Clear the W1C bits not specified by the write mask, so that the
++ * write_op() does not clear them.
++ */
++ new &= ~(behavior[reg / 4].w1c & ~mask);
++
++ /*
++ * Set the W1C bits specified by the write mask, so that write_op()
++ * knows about that they are to be cleared.
++ */
++ new |= (value << shift) & (behavior[reg / 4].w1c & mask);
++
+ if (write_op)
+ write_op(bridge, reg, old, new, mask);
+
+diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
+index ce2ab62b64cfa..0380543d10fdc 100644
+--- a/drivers/pci/pci.c
++++ b/drivers/pci/pci.c
+@@ -2888,6 +2888,16 @@ static const struct dmi_system_id bridge_d3_blacklist[] = {
+ DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
+ DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
+ },
++ /*
++ * Downstream device is not accessible after putting a root port
++ * into D3cold and back into D0 on Elo i2.
++ */
++ .ident = "Elo i2",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Elo Touch Solutions"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "Elo i2"),
++ DMI_MATCH(DMI_PRODUCT_VERSION, "RevB"),
++ },
+ },
+ #endif
+ { }
+@@ -3719,6 +3729,14 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
+ struct pci_dev *bridge;
+ u32 cap, ctl2;
+
++ /*
++ * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
++ * in Device Control 2 is reserved in VFs and the PF value applies
++ * to all associated VFs.
++ */
++ if (dev->is_virtfn)
++ return -EINVAL;
++
+ if (!pci_is_pcie(dev))
+ return -EINVAL;
+
+diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
+index 4537d1ea14fdc..4893b1e824031 100644
+--- a/drivers/pci/quirks.c
++++ b/drivers/pci/quirks.c
+@@ -1811,6 +1811,18 @@ static void quirk_alder_ioapic(struct pci_dev *pdev)
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_EESSC, quirk_alder_ioapic);
+ #endif
+
++static void quirk_no_msi(struct pci_dev *dev)
++{
++ pci_info(dev, "avoiding MSI to work around a hardware defect\n");
++ dev->no_msi = 1;
++}
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x4386, quirk_no_msi);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x4387, quirk_no_msi);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x4388, quirk_no_msi);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x4389, quirk_no_msi);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x438a, quirk_no_msi);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x438b, quirk_no_msi);
++
+ static void quirk_pcie_mch(struct pci_dev *pdev)
+ {
+ pdev->no_msi = 1;
+@@ -3612,6 +3624,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0032, quirk_no_bus_reset);
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003c, quirk_no_bus_reset);
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0033, quirk_no_bus_reset);
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0034, quirk_no_bus_reset);
++DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003e, quirk_no_bus_reset);
+
+ /*
+ * Root port on some Cavium CN8xxx chips do not successfully complete a bus
+@@ -4102,6 +4115,9 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL_EXT, 0x9120,
+ quirk_dma_func1_alias);
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL_EXT, 0x9123,
+ quirk_dma_func1_alias);
++/* https://bugzilla.kernel.org/show_bug.cgi?id=42679#c136 */
++DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL_EXT, 0x9125,
++ quirk_dma_func1_alias);
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MARVELL_EXT, 0x9128,
+ quirk_dma_func1_alias);
+ /* https://bugzilla.kernel.org/show_bug.cgi?id=42679#c14 */
+@@ -5340,11 +5356,6 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0422, quirk_no_ext_tags);
+ */
+ static void quirk_amd_harvest_no_ats(struct pci_dev *pdev)
+ {
+- if ((pdev->device == 0x7312 && pdev->revision != 0x00) ||
+- (pdev->device == 0x7340 && pdev->revision != 0xc5) ||
+- (pdev->device == 0x7341 && pdev->revision != 0x00))
+- return;
+-
+ if (pdev->device == 0x15d8) {
+ if (pdev->revision == 0xcf &&
+ pdev->subsystem_vendor == 0xea50 &&
+@@ -5366,10 +5377,19 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x98e4, quirk_amd_harvest_no_ats);
+ /* AMD Iceland dGPU */
+ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x6900, quirk_amd_harvest_no_ats);
+ /* AMD Navi10 dGPU */
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7310, quirk_amd_harvest_no_ats);
+ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7312, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7318, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7319, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x731a, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x731b, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x731e, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x731f, quirk_amd_harvest_no_ats);
+ /* AMD Navi14 dGPU */
+ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7340, quirk_amd_harvest_no_ats);
+ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7341, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x7347, quirk_amd_harvest_no_ats);
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x734f, quirk_amd_harvest_no_ats);
+ /* AMD Raven platform iGPU */
+ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x15d8, quirk_amd_harvest_no_ats);
+ #endif /* CONFIG_PCI_ATS */
+@@ -5795,3 +5815,9 @@ static void apex_pci_fixup_class(struct pci_dev *pdev)
+ }
+ DECLARE_PCI_FIXUP_CLASS_HEADER(0x1ac1, 0x089a,
+ PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class);
++
++static void nvidia_ion_ahci_fixup(struct pci_dev *pdev)
++{
++ pdev->dev_flags |= PCI_DEV_FLAGS_HAS_MSI_MASKING;
++}
++DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0ab8, nvidia_ion_ahci_fixup);
+diff --git a/drivers/pcmcia/cs.c b/drivers/pcmcia/cs.c
+index e211e2619680c..f70197154a362 100644
+--- a/drivers/pcmcia/cs.c
++++ b/drivers/pcmcia/cs.c
+@@ -666,18 +666,16 @@ static int pccardd(void *__skt)
+ if (events || sysfs_events)
+ continue;
+
++ set_current_state(TASK_INTERRUPTIBLE);
+ if (kthread_should_stop())
+ break;
+
+- set_current_state(TASK_INTERRUPTIBLE);
+-
+ schedule();
+
+- /* make sure we are running */
+- __set_current_state(TASK_RUNNING);
+-
+ try_to_freeze();
+ }
++ /* make sure we are running before we exit */
++ __set_current_state(TASK_RUNNING);
+
+ /* shut down socket, if a device is still present */
+ if (skt->state & SOCKET_PRESENT) {
+diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c
+index bb15a8bdbaab5..1cac528707111 100644
+--- a/drivers/pcmcia/rsrc_nonstatic.c
++++ b/drivers/pcmcia/rsrc_nonstatic.c
+@@ -690,6 +690,9 @@ static struct resource *__nonstatic_find_io_region(struct pcmcia_socket *s,
+ unsigned long min = base;
+ int ret;
+
++ if (!res)
++ return NULL;
++
+ data.mask = align - 1;
+ data.offset = base & data.mask;
+ data.map = &s_data->io_db;
+@@ -809,6 +812,9 @@ static struct resource *nonstatic_find_mem_region(u_long base, u_long num,
+ unsigned long min, max;
+ int ret, i, j;
+
++ if (!res)
++ return NULL;
++
+ low = low || !(s->features & SS_CAP_PAGE_REGS);
+
+ data.mask = align - 1;
+diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c
+index bc3cba5f8c5dc..400eb7f579dce 100644
+--- a/drivers/perf/arm-cmn.c
++++ b/drivers/perf/arm-cmn.c
+@@ -1561,7 +1561,8 @@ static int arm_cmn_probe(struct platform_device *pdev)
+
+ err = perf_pmu_register(&cmn->pmu, name, -1);
+ if (err)
+- cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
++ cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);
++
+ return err;
+ }
+
+@@ -1572,7 +1573,7 @@ static int arm_cmn_remove(struct platform_device *pdev)
+ writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);
+
+ perf_pmu_unregister(&cmn->pmu);
+- cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
++ cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);
+ return 0;
+ }
+
+diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
+index 295cc7952d0ed..57d20cf3da7a3 100644
+--- a/drivers/perf/arm_pmu.c
++++ b/drivers/perf/arm_pmu.c
+@@ -398,6 +398,9 @@ validate_group(struct perf_event *event)
+ if (!validate_event(event->pmu, &fake_pmu, leader))
+ return -EINVAL;
+
++ if (event == leader)
++ return 0;
++
+ for_each_sibling_event(sibling, leader) {
+ if (!validate_event(event->pmu, &fake_pmu, sibling))
+ return -EINVAL;
+@@ -487,12 +490,7 @@ __hw_perf_event_init(struct perf_event *event)
+ local64_set(&hwc->period_left, hwc->sample_period);
+ }
+
+- if (event->group_leader != event) {
+- if (validate_group(event) != 0)
+- return -EINVAL;
+- }
+-
+- return 0;
++ return validate_group(event);
+ }
+
+ static int armpmu_event_init(struct perf_event *event)
+diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c
+index 94ebc1ecace7c..b1b2a55de77fc 100644
+--- a/drivers/perf/fsl_imx8_ddr_perf.c
++++ b/drivers/perf/fsl_imx8_ddr_perf.c
+@@ -29,7 +29,7 @@
+ #define CNTL_OVER_MASK 0xFFFFFFFE
+
+ #define CNTL_CSV_SHIFT 24
+-#define CNTL_CSV_MASK (0xFF << CNTL_CSV_SHIFT)
++#define CNTL_CSV_MASK (0xFFU << CNTL_CSV_SHIFT)
+
+ #define EVENT_CYCLES_ID 0
+ #define EVENT_CYCLES_COUNTER 0
+diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c
+index 5b093badd0f65..f60e79fac2021 100644
+--- a/drivers/perf/qcom_l2_pmu.c
++++ b/drivers/perf/qcom_l2_pmu.c
+@@ -736,7 +736,7 @@ static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
+ {
+ u64 mpidr;
+ int cpu_cluster_id;
+- struct cluster_pmu *cluster = NULL;
++ struct cluster_pmu *cluster;
+
+ /*
+ * This assumes that the cluster_id is in MPIDR[aff1] for
+@@ -758,10 +758,10 @@ static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
+ cluster->cluster_id);
+ cpumask_set_cpu(cpu, &cluster->cluster_cpus);
+ *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
+- break;
++ return cluster;
+ }
+
+- return cluster;
++ return NULL;
+ }
+
+ static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
+diff --git a/drivers/phy/amlogic/phy-meson-axg-mipi-dphy.c b/drivers/phy/amlogic/phy-meson-axg-mipi-dphy.c
+index cd2332bf0e31a..fdbd64c03e12b 100644
+--- a/drivers/phy/amlogic/phy-meson-axg-mipi-dphy.c
++++ b/drivers/phy/amlogic/phy-meson-axg-mipi-dphy.c
+@@ -9,6 +9,7 @@
+
+ #include <linux/bitfield.h>
+ #include <linux/bitops.h>
++#include <linux/bits.h>
+ #include <linux/clk.h>
+ #include <linux/delay.h>
+ #include <linux/io.h>
+@@ -250,7 +251,7 @@ static int phy_meson_axg_mipi_dphy_power_on(struct phy *phy)
+ (DIV_ROUND_UP(priv->config.clk_zero, temp) << 16) |
+ (DIV_ROUND_UP(priv->config.clk_prepare, temp) << 24));
+ regmap_write(priv->regmap, MIPI_DSI_CLK_TIM1,
+- DIV_ROUND_UP(priv->config.clk_pre, temp));
++ DIV_ROUND_UP(priv->config.clk_pre, BITS_PER_BYTE));
+
+ regmap_write(priv->regmap, MIPI_DSI_HS_TIM,
+ DIV_ROUND_UP(priv->config.hs_exit, temp) |
+diff --git a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c
+index 5b471ab80fe28..54d65a6f0fccf 100644
+--- a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c
++++ b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c
+@@ -414,19 +414,19 @@ static int phy_g12a_usb3_pcie_probe(struct platform_device *pdev)
+
+ ret = clk_prepare_enable(priv->clk_ref);
+ if (ret)
+- goto err_disable_clk_ref;
++ return ret;
+
+ priv->reset = devm_reset_control_array_get_exclusive(dev);
+- if (IS_ERR(priv->reset))
+- return PTR_ERR(priv->reset);
++ if (IS_ERR(priv->reset)) {
++ ret = PTR_ERR(priv->reset);
++ goto err_disable_clk_ref;
++ }
+
+ priv->phy = devm_phy_create(dev, np, &phy_g12a_usb3_pcie_ops);
+ if (IS_ERR(priv->phy)) {
+ ret = PTR_ERR(priv->phy);
+- if (ret != -EPROBE_DEFER)
+- dev_err(dev, "failed to create PHY\n");
+-
+- return ret;
++ dev_err_probe(dev, ret, "failed to create PHY\n");
++ goto err_disable_clk_ref;
+ }
+
+ phy_set_drvdata(priv->phy, priv);
+@@ -434,8 +434,12 @@ static int phy_g12a_usb3_pcie_probe(struct platform_device *pdev)
+
+ phy_provider = devm_of_phy_provider_register(dev,
+ phy_g12a_usb3_pcie_xlate);
++ if (IS_ERR(phy_provider)) {
++ ret = PTR_ERR(phy_provider);
++ goto err_disable_clk_ref;
++ }
+
+- return PTR_ERR_OR_ZERO(phy_provider);
++ return 0;
+
+ err_disable_clk_ref:
+ clk_disable_unprepare(priv->clk_ref);
+diff --git a/drivers/phy/amlogic/phy-meson-gxl-usb2.c b/drivers/phy/amlogic/phy-meson-gxl-usb2.c
+index 2b3c0d730f20f..db17c3448bfed 100644
+--- a/drivers/phy/amlogic/phy-meson-gxl-usb2.c
++++ b/drivers/phy/amlogic/phy-meson-gxl-usb2.c
+@@ -114,8 +114,10 @@ static int phy_meson_gxl_usb2_init(struct phy *phy)
+ return ret;
+
+ ret = clk_prepare_enable(priv->clk);
+- if (ret)
++ if (ret) {
++ reset_control_rearm(priv->reset);
+ return ret;
++ }
+
+ return 0;
+ }
+@@ -125,6 +127,7 @@ static int phy_meson_gxl_usb2_exit(struct phy *phy)
+ struct phy_meson_gxl_usb2_priv *priv = phy_get_drvdata(phy);
+
+ clk_disable_unprepare(priv->clk);
++ reset_control_rearm(priv->reset);
+
+ return 0;
+ }
+diff --git a/drivers/phy/amlogic/phy-meson8b-usb2.c b/drivers/phy/amlogic/phy-meson8b-usb2.c
+index cf10bed40528a..dd96763911b8b 100644
+--- a/drivers/phy/amlogic/phy-meson8b-usb2.c
++++ b/drivers/phy/amlogic/phy-meson8b-usb2.c
+@@ -154,6 +154,7 @@ static int phy_meson8b_usb2_power_on(struct phy *phy)
+ ret = clk_prepare_enable(priv->clk_usb_general);
+ if (ret) {
+ dev_err(&phy->dev, "Failed to enable USB general clock\n");
++ reset_control_rearm(priv->reset);
+ return ret;
+ }
+
+@@ -161,6 +162,7 @@ static int phy_meson8b_usb2_power_on(struct phy *phy)
+ if (ret) {
+ dev_err(&phy->dev, "Failed to enable USB DDR clock\n");
+ clk_disable_unprepare(priv->clk_usb_general);
++ reset_control_rearm(priv->reset);
+ return ret;
+ }
+
+@@ -199,6 +201,7 @@ static int phy_meson8b_usb2_power_on(struct phy *phy)
+ dev_warn(&phy->dev, "USB ID detect failed!\n");
+ clk_disable_unprepare(priv->clk_usb);
+ clk_disable_unprepare(priv->clk_usb_general);
++ reset_control_rearm(priv->reset);
+ return -EINVAL;
+ }
+ }
+@@ -218,6 +221,7 @@ static int phy_meson8b_usb2_power_off(struct phy *phy)
+
+ clk_disable_unprepare(priv->clk_usb);
+ clk_disable_unprepare(priv->clk_usb_general);
++ reset_control_rearm(priv->reset);
+
+ /* power off the PHY by putting it into reset mode */
+ regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_POWER_ON_RESET,
+@@ -265,8 +269,9 @@ static int phy_meson8b_usb2_probe(struct platform_device *pdev)
+ return PTR_ERR(priv->clk_usb);
+
+ priv->reset = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
+- if (PTR_ERR(priv->reset) == -EPROBE_DEFER)
+- return PTR_ERR(priv->reset);
++ if (IS_ERR(priv->reset))
++ return dev_err_probe(&pdev->dev, PTR_ERR(priv->reset),
++ "Failed to get the reset line");
+
+ priv->dr_mode = of_usb_get_dr_mode_by_phy(pdev->dev.of_node, -1);
+ if (priv->dr_mode == USB_DR_MODE_UNKNOWN) {
+diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig
+index fd92b73b71095..1dcfa3bd1442d 100644
+--- a/drivers/phy/broadcom/Kconfig
++++ b/drivers/phy/broadcom/Kconfig
+@@ -95,8 +95,7 @@ config PHY_BRCM_USB
+ depends on OF
+ select GENERIC_PHY
+ select SOC_BRCMSTB if ARCH_BRCMSTB
+- default ARCH_BCM4908
+- default ARCH_BRCMSTB
++ default ARCH_BCM4908 || ARCH_BRCMSTB
+ help
+ Enable this to support the Broadcom STB USB PHY.
+ This driver is required by the USB XHCI, EHCI and OHCI
+diff --git a/drivers/phy/broadcom/phy-brcm-usb-init.c b/drivers/phy/broadcom/phy-brcm-usb-init.c
+index 9391ab42a12b3..dd0f66288fbdd 100644
+--- a/drivers/phy/broadcom/phy-brcm-usb-init.c
++++ b/drivers/phy/broadcom/phy-brcm-usb-init.c
+@@ -79,6 +79,7 @@
+
+ enum brcm_family_type {
+ BRCM_FAMILY_3390A0,
++ BRCM_FAMILY_4908,
+ BRCM_FAMILY_7250B0,
+ BRCM_FAMILY_7271A0,
+ BRCM_FAMILY_7364A0,
+@@ -96,6 +97,7 @@ enum brcm_family_type {
+
+ static const char *family_names[BRCM_FAMILY_COUNT] = {
+ USB_BRCM_FAMILY(3390A0),
++ USB_BRCM_FAMILY(4908),
+ USB_BRCM_FAMILY(7250B0),
+ USB_BRCM_FAMILY(7271A0),
+ USB_BRCM_FAMILY(7364A0),
+@@ -203,6 +205,27 @@ usb_reg_bits_map_table[BRCM_FAMILY_COUNT][USB_CTRL_SELECTOR_COUNT] = {
+ USB_CTRL_USB_PM_USB20_HC_RESETB_VAR_MASK,
+ ENDIAN_SETTINGS, /* USB_CTRL_SETUP ENDIAN bits */
+ },
++ /* 4908 */
++ [BRCM_FAMILY_4908] = {
++ 0, /* USB_CTRL_SETUP_SCB1_EN_MASK */
++ 0, /* USB_CTRL_SETUP_SCB2_EN_MASK */
++ 0, /* USB_CTRL_SETUP_SS_EHCI64BIT_EN_MASK */
++ 0, /* USB_CTRL_SETUP_STRAP_IPP_SEL_MASK */
++ 0, /* USB_CTRL_SETUP_OC3_DISABLE_MASK */
++ 0, /* USB_CTRL_PLL_CTL_PLL_IDDQ_PWRDN_MASK */
++ 0, /* USB_CTRL_USB_PM_BDC_SOFT_RESETB_MASK */
++ USB_CTRL_USB_PM_XHC_SOFT_RESETB_MASK,
++ USB_CTRL_USB_PM_USB_PWRDN_MASK,
++ 0, /* USB_CTRL_USB30_CTL1_XHC_SOFT_RESETB_MASK */
++ 0, /* USB_CTRL_USB30_CTL1_USB3_IOC_MASK */
++ 0, /* USB_CTRL_USB30_CTL1_USB3_IPP_MASK */
++ 0, /* USB_CTRL_USB_DEVICE_CTL1_PORT_MODE_MASK */
++ 0, /* USB_CTRL_USB_PM_SOFT_RESET_MASK */
++ 0, /* USB_CTRL_SETUP_CC_DRD_MODE_ENABLE_MASK */
++ 0, /* USB_CTRL_SETUP_STRAP_CC_DRD_MODE_ENABLE_SEL_MASK */
++ 0, /* USB_CTRL_USB_PM_USB20_HC_RESETB_VAR_MASK */
++ 0, /* USB_CTRL_SETUP ENDIAN bits */
++ },
+ /* 7250b0 */
+ [BRCM_FAMILY_7250B0] = {
+ USB_CTRL_SETUP_SCB1_EN_MASK,
+@@ -559,6 +582,7 @@ static void brcmusb_usb3_pll_54mhz(struct brcm_usb_init_params *params)
+ */
+ switch (params->selected_family) {
+ case BRCM_FAMILY_3390A0:
++ case BRCM_FAMILY_4908:
+ case BRCM_FAMILY_7250B0:
+ case BRCM_FAMILY_7366C0:
+ case BRCM_FAMILY_74371A0:
+@@ -1004,6 +1028,18 @@ static const struct brcm_usb_init_ops bcm7445_ops = {
+ .set_dual_select = usb_set_dual_select,
+ };
+
++void brcm_usb_dvr_init_4908(struct brcm_usb_init_params *params)
++{
++ int fam;
++
++ fam = BRCM_FAMILY_4908;
++ params->selected_family = fam;
++ params->usb_reg_bits_map =
++ &usb_reg_bits_map_table[fam][0];
++ params->family_name = family_names[fam];
++ params->ops = &bcm7445_ops;
++}
++
+ void brcm_usb_dvr_init_7445(struct brcm_usb_init_params *params)
+ {
+ int fam;
+diff --git a/drivers/phy/broadcom/phy-brcm-usb-init.h b/drivers/phy/broadcom/phy-brcm-usb-init.h
+index a39f30fa2e991..1ccb5ddab865c 100644
+--- a/drivers/phy/broadcom/phy-brcm-usb-init.h
++++ b/drivers/phy/broadcom/phy-brcm-usb-init.h
+@@ -64,6 +64,7 @@ struct brcm_usb_init_params {
+ bool suspend_with_clocks;
+ };
+
++void brcm_usb_dvr_init_4908(struct brcm_usb_init_params *params);
+ void brcm_usb_dvr_init_7445(struct brcm_usb_init_params *params);
+ void brcm_usb_dvr_init_7216(struct brcm_usb_init_params *params);
+ void brcm_usb_dvr_init_7211b0(struct brcm_usb_init_params *params);
+diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c
+index 116fb23aebd99..2cb3779fcdf82 100644
+--- a/drivers/phy/broadcom/phy-brcm-usb.c
++++ b/drivers/phy/broadcom/phy-brcm-usb.c
+@@ -18,6 +18,7 @@
+ #include <linux/soc/brcmstb/brcmstb.h>
+ #include <dt-bindings/phy/phy.h>
+ #include <linux/mfd/syscon.h>
++#include <linux/suspend.h>
+
+ #include "phy-brcm-usb-init.h"
+
+@@ -70,12 +71,35 @@ struct brcm_usb_phy_data {
+ int init_count;
+ int wake_irq;
+ struct brcm_usb_phy phys[BRCM_USB_PHY_ID_MAX];
++ struct notifier_block pm_notifier;
++ bool pm_active;
+ };
+
+ static s8 *node_reg_names[BRCM_REGS_MAX] = {
+ "crtl", "xhci_ec", "xhci_gbl", "usb_phy", "usb_mdio", "bdc_ec"
+ };
+
++static int brcm_pm_notifier(struct notifier_block *notifier,
++ unsigned long pm_event,
++ void *unused)
++{
++ struct brcm_usb_phy_data *priv =
++ container_of(notifier, struct brcm_usb_phy_data, pm_notifier);
++
++ switch (pm_event) {
++ case PM_HIBERNATION_PREPARE:
++ case PM_SUSPEND_PREPARE:
++ priv->pm_active = true;
++ break;
++ case PM_POST_RESTORE:
++ case PM_POST_HIBERNATION:
++ case PM_POST_SUSPEND:
++ priv->pm_active = false;
++ break;
++ }
++ return NOTIFY_DONE;
++}
++
+ static irqreturn_t brcm_usb_phy_wake_isr(int irq, void *dev_id)
+ {
+ struct phy *gphy = dev_id;
+@@ -91,6 +115,9 @@ static int brcm_usb_phy_init(struct phy *gphy)
+ struct brcm_usb_phy_data *priv =
+ container_of(phy, struct brcm_usb_phy_data, phys[phy->id]);
+
++ if (priv->pm_active)
++ return 0;
++
+ /*
+ * Use a lock to make sure a second caller waits until
+ * the base phy is inited before using it.
+@@ -120,6 +147,9 @@ static int brcm_usb_phy_exit(struct phy *gphy)
+ struct brcm_usb_phy_data *priv =
+ container_of(phy, struct brcm_usb_phy_data, phys[phy->id]);
+
++ if (priv->pm_active)
++ return 0;
++
+ dev_dbg(&gphy->dev, "EXIT\n");
+ if (phy->id == BRCM_USB_PHY_2_0)
+ brcm_usb_uninit_eohci(&priv->ini);
+@@ -253,6 +283,15 @@ static const struct attribute_group brcm_usb_phy_group = {
+ .attrs = brcm_usb_phy_attrs,
+ };
+
++static const struct match_chip_info chip_info_4908 = {
++ .init_func = &brcm_usb_dvr_init_4908,
++ .required_regs = {
++ BRCM_REGS_CTRL,
++ BRCM_REGS_XHCI_EC,
++ -1,
++ },
++};
++
+ static const struct match_chip_info chip_info_7216 = {
+ .init_func = &brcm_usb_dvr_init_7216,
+ .required_regs = {
+@@ -288,7 +327,7 @@ static const struct match_chip_info chip_info_7445 = {
+ static const struct of_device_id brcm_usb_dt_ids[] = {
+ {
+ .compatible = "brcm,bcm4908-usb-phy",
+- .data = &chip_info_7445,
++ .data = &chip_info_4908,
+ },
+ {
+ .compatible = "brcm,bcm7216-usb-phy",
+@@ -488,6 +527,9 @@ static int brcm_usb_phy_probe(struct platform_device *pdev)
+ if (err)
+ return err;
+
++ priv->pm_notifier.notifier_call = brcm_pm_notifier;
++ register_pm_notifier(&priv->pm_notifier);
++
+ mutex_init(&priv->mutex);
+
+ /* make sure invert settings are correct */
+@@ -528,7 +570,10 @@ static int brcm_usb_phy_probe(struct platform_device *pdev)
+
+ static int brcm_usb_phy_remove(struct platform_device *pdev)
+ {
++ struct brcm_usb_phy_data *priv = dev_get_drvdata(&pdev->dev);
++
+ sysfs_remove_group(&pdev->dev.kobj, &brcm_usb_phy_group);
++ unregister_pm_notifier(&priv->pm_notifier);
+
+ return 0;
+ }
+@@ -539,6 +584,7 @@ static int brcm_usb_phy_suspend(struct device *dev)
+ struct brcm_usb_phy_data *priv = dev_get_drvdata(dev);
+
+ if (priv->init_count) {
++ dev_dbg(dev, "SUSPEND\n");
+ priv->ini.wake_enabled = device_may_wakeup(dev);
+ if (priv->phys[BRCM_USB_PHY_3_0].inited)
+ brcm_usb_uninit_xhci(&priv->ini);
+@@ -578,6 +624,7 @@ static int brcm_usb_phy_resume(struct device *dev)
+ * Uninitialize anything that wasn't previously initialized.
+ */
+ if (priv->init_count) {
++ dev_dbg(dev, "RESUME\n");
+ if (priv->wake_irq >= 0)
+ disable_irq_wake(priv->wake_irq);
+ brcm_usb_init_common(&priv->ini);
+diff --git a/drivers/phy/cadence/phy-cadence-sierra.c b/drivers/phy/cadence/phy-cadence-sierra.c
+index e93818e3991fd..3e2d096d54fd7 100644
+--- a/drivers/phy/cadence/phy-cadence-sierra.c
++++ b/drivers/phy/cadence/phy-cadence-sierra.c
+@@ -215,7 +215,10 @@ static const int pll_mux_parent_index[][SIERRA_NUM_CMN_PLLC_PARENTS] = {
+ [CMN_PLLLC1] = { PLL1_REFCLK, PLL0_REFCLK },
+ };
+
+-static u32 cdns_sierra_pll_mux_table[] = { 0, 1 };
++static u32 cdns_sierra_pll_mux_table[][SIERRA_NUM_CMN_PLLC_PARENTS] = {
++ [CMN_PLLLC] = { 0, 1 },
++ [CMN_PLLLC1] = { 1, 0 },
++};
+
+ struct cdns_sierra_inst {
+ struct phy *phy;
+@@ -436,11 +439,25 @@ static const struct phy_ops ops = {
+ static u8 cdns_sierra_pll_mux_get_parent(struct clk_hw *hw)
+ {
+ struct cdns_sierra_pll_mux *mux = to_cdns_sierra_pll_mux(hw);
++ struct regmap_field *plllc1en_field = mux->plllc1en_field;
++ struct regmap_field *termen_field = mux->termen_field;
+ struct regmap_field *field = mux->pfdclk_sel_preg;
+ unsigned int val;
++ int index;
+
+ regmap_field_read(field, &val);
+- return clk_mux_val_to_index(hw, cdns_sierra_pll_mux_table, 0, val);
++
++ if (strstr(clk_hw_get_name(hw), clk_names[CDNS_SIERRA_PLL_CMNLC1])) {
++ index = clk_mux_val_to_index(hw, cdns_sierra_pll_mux_table[CMN_PLLLC1], 0, val);
++ if (index == 1) {
++ regmap_field_write(plllc1en_field, 1);
++ regmap_field_write(termen_field, 1);
++ }
++ } else {
++ index = clk_mux_val_to_index(hw, cdns_sierra_pll_mux_table[CMN_PLLLC], 0, val);
++ }
++
++ return index;
+ }
+
+ static int cdns_sierra_pll_mux_set_parent(struct clk_hw *hw, u8 index)
+@@ -458,7 +475,11 @@ static int cdns_sierra_pll_mux_set_parent(struct clk_hw *hw, u8 index)
+ ret |= regmap_field_write(termen_field, 1);
+ }
+
+- val = cdns_sierra_pll_mux_table[index];
++ if (strstr(clk_hw_get_name(hw), clk_names[CDNS_SIERRA_PLL_CMNLC1]))
++ val = cdns_sierra_pll_mux_table[CMN_PLLLC1][index];
++ else
++ val = cdns_sierra_pll_mux_table[CMN_PLLLC][index];
++
+ ret |= regmap_field_write(field, val);
+
+ return ret;
+@@ -496,8 +517,8 @@ static int cdns_sierra_pll_mux_register(struct cdns_sierra_phy *sp,
+ for (i = 0; i < num_parents; i++) {
+ clk = sp->input_clks[pll_mux_parent_index[clk_index][i]];
+ if (IS_ERR_OR_NULL(clk)) {
+- dev_err(dev, "No parent clock for derived_refclk\n");
+- return PTR_ERR(clk);
++ dev_err(dev, "No parent clock for PLL mux clocks\n");
++ return IS_ERR(clk) ? PTR_ERR(clk) : -ENOENT;
+ }
+ parent_names[i] = __clk_get_name(clk);
+ }
+diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c
+index 28ad9403c4414..67b005d5b9e35 100644
+--- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c
++++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c
+@@ -146,6 +146,8 @@ static int mtk_mipi_tx_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ mipi_tx->driver_data = of_device_get_match_data(dev);
++ if (!mipi_tx->driver_data)
++ return -ENODEV;
+
+ mipi_tx->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(mipi_tx->regs))
+diff --git a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c
+index cdcef865fe9e5..db39b0c4649a2 100644
+--- a/drivers/phy/mediatek/phy-mtk-tphy.c
++++ b/drivers/phy/mediatek/phy-mtk-tphy.c
+@@ -12,6 +12,7 @@
+ #include <linux/iopoll.h>
+ #include <linux/mfd/syscon.h>
+ #include <linux/module.h>
++#include <linux/nvmem-consumer.h>
+ #include <linux/of_address.h>
+ #include <linux/of_device.h>
+ #include <linux/phy/phy.h>
+@@ -41,6 +42,9 @@
+ #define SSUSB_SIFSLV_V2_U3PHYD 0x200
+ #define SSUSB_SIFSLV_V2_U3PHYA 0x400
+
++#define U3P_MISC_REG1 0x04
++#define MR1_EFUSE_AUTO_LOAD_DIS BIT(6)
++
+ #define U3P_USBPHYACR0 0x000
+ #define PA0_RG_U2PLL_FORCE_ON BIT(15)
+ #define PA0_USB20_PLL_PREDIV GENMASK(7, 6)
+@@ -133,6 +137,8 @@
+ #define P3C_RG_SWRST_U3_PHYD_FORCE_EN BIT(24)
+
+ #define U3P_U3_PHYA_REG0 0x000
++#define P3A_RG_IEXT_INTR GENMASK(15, 10)
++#define P3A_RG_IEXT_INTR_VAL(x) ((0x3f & (x)) << 10)
+ #define P3A_RG_CLKDRV_OFF GENMASK(3, 2)
+ #define P3A_RG_CLKDRV_OFF_VAL(x) ((0x3 & (x)) << 2)
+
+@@ -187,6 +193,19 @@
+ #define P3D_RG_FWAKE_TH GENMASK(21, 16)
+ #define P3D_RG_FWAKE_TH_VAL(x) ((0x3f & (x)) << 16)
+
++#define U3P_U3_PHYD_IMPCAL0 0x010
++#define P3D_RG_FORCE_TX_IMPEL BIT(31)
++#define P3D_RG_TX_IMPEL GENMASK(28, 24)
++#define P3D_RG_TX_IMPEL_VAL(x) ((0x1f & (x)) << 24)
++
++#define U3P_U3_PHYD_IMPCAL1 0x014
++#define P3D_RG_FORCE_RX_IMPEL BIT(31)
++#define P3D_RG_RX_IMPEL GENMASK(28, 24)
++#define P3D_RG_RX_IMPEL_VAL(x) ((0x1f & (x)) << 24)
++
++#define U3P_U3_PHYD_RSV 0x054
++#define P3D_RG_EFUSE_AUTO_LOAD_DIS BIT(12)
++
+ #define U3P_U3_PHYD_CDR1 0x05c
+ #define P3D_RG_CDR_BIR_LTD1 GENMASK(28, 24)
+ #define P3D_RG_CDR_BIR_LTD1_VAL(x) ((0x1f & (x)) << 24)
+@@ -307,6 +326,11 @@ struct mtk_phy_pdata {
+ * 48M PLL, fix it by switching PLL to 26M from default 48M
+ */
+ bool sw_pll_48m_to_26m;
++ /*
++ * Some SoCs (e.g. mt8195) drop a bit when use auto load efuse,
++ * support sw way, also support it for v2/v3 optionally.
++ */
++ bool sw_efuse_supported;
+ enum mtk_phy_version version;
+ };
+
+@@ -336,6 +360,10 @@ struct mtk_phy_instance {
+ struct regmap *type_sw;
+ u32 type_sw_reg;
+ u32 type_sw_index;
++ u32 efuse_sw_en;
++ u32 efuse_intr;
++ u32 efuse_tx_imp;
++ u32 efuse_rx_imp;
+ int eye_src;
+ int eye_vrt;
+ int eye_term;
+@@ -1040,6 +1068,130 @@ static int phy_type_set(struct mtk_phy_instance *instance)
+ return 0;
+ }
+
++static int phy_efuse_get(struct mtk_tphy *tphy, struct mtk_phy_instance *instance)
++{
++ struct device *dev = &instance->phy->dev;
++ int ret = 0;
++
++ /* tphy v1 doesn't support sw efuse, skip it */
++ if (!tphy->pdata->sw_efuse_supported) {
++ instance->efuse_sw_en = 0;
++ return 0;
++ }
++
++ /* software efuse is optional */
++ instance->efuse_sw_en = device_property_read_bool(dev, "nvmem-cells");
++ if (!instance->efuse_sw_en)
++ return 0;
++
++ switch (instance->type) {
++ case PHY_TYPE_USB2:
++ ret = nvmem_cell_read_variable_le_u32(dev, "intr", &instance->efuse_intr);
++ if (ret) {
++ dev_err(dev, "fail to get u2 intr efuse, %d\n", ret);
++ break;
++ }
++
++ /* no efuse, ignore it */
++ if (!instance->efuse_intr) {
++ dev_warn(dev, "no u2 intr efuse, but dts enable it\n");
++ instance->efuse_sw_en = 0;
++ break;
++ }
++
++ dev_dbg(dev, "u2 efuse - intr %x\n", instance->efuse_intr);
++ break;
++
++ case PHY_TYPE_USB3:
++ case PHY_TYPE_PCIE:
++ ret = nvmem_cell_read_variable_le_u32(dev, "intr", &instance->efuse_intr);
++ if (ret) {
++ dev_err(dev, "fail to get u3 intr efuse, %d\n", ret);
++ break;
++ }
++
++ ret = nvmem_cell_read_variable_le_u32(dev, "rx_imp", &instance->efuse_rx_imp);
++ if (ret) {
++ dev_err(dev, "fail to get u3 rx_imp efuse, %d\n", ret);
++ break;
++ }
++
++ ret = nvmem_cell_read_variable_le_u32(dev, "tx_imp", &instance->efuse_tx_imp);
++ if (ret) {
++ dev_err(dev, "fail to get u3 tx_imp efuse, %d\n", ret);
++ break;
++ }
++
++ /* no efuse, ignore it */
++ if (!instance->efuse_intr &&
++ !instance->efuse_rx_imp &&
++ !instance->efuse_tx_imp) {
++ dev_warn(dev, "no u3 intr efuse, but dts enable it\n");
++ instance->efuse_sw_en = 0;
++ break;
++ }
++
++ dev_dbg(dev, "u3 efuse - intr %x, rx_imp %x, tx_imp %x\n",
++ instance->efuse_intr, instance->efuse_rx_imp,instance->efuse_tx_imp);
++ break;
++ default:
++ dev_err(dev, "no sw efuse for type %d\n", instance->type);
++ ret = -EINVAL;
++ }
++
++ return ret;
++}
++
++static void phy_efuse_set(struct mtk_phy_instance *instance)
++{
++ struct device *dev = &instance->phy->dev;
++ struct u2phy_banks *u2_banks = &instance->u2_banks;
++ struct u3phy_banks *u3_banks = &instance->u3_banks;
++ u32 tmp;
++
++ if (!instance->efuse_sw_en)
++ return;
++
++ switch (instance->type) {
++ case PHY_TYPE_USB2:
++ tmp = readl(u2_banks->misc + U3P_MISC_REG1);
++ tmp |= MR1_EFUSE_AUTO_LOAD_DIS;
++ writel(tmp, u2_banks->misc + U3P_MISC_REG1);
++
++ tmp = readl(u2_banks->com + U3P_USBPHYACR1);
++ tmp &= ~PA1_RG_INTR_CAL;
++ tmp |= PA1_RG_INTR_CAL_VAL(instance->efuse_intr);
++ writel(tmp, u2_banks->com + U3P_USBPHYACR1);
++ break;
++ case PHY_TYPE_USB3:
++ case PHY_TYPE_PCIE:
++ tmp = readl(u3_banks->phyd + U3P_U3_PHYD_RSV);
++ tmp |= P3D_RG_EFUSE_AUTO_LOAD_DIS;
++ writel(tmp, u3_banks->phyd + U3P_U3_PHYD_RSV);
++
++ tmp = readl(u3_banks->phyd + U3P_U3_PHYD_IMPCAL0);
++ tmp &= ~P3D_RG_TX_IMPEL;
++ tmp |= P3D_RG_TX_IMPEL_VAL(instance->efuse_tx_imp);
++ tmp |= P3D_RG_FORCE_TX_IMPEL;
++ writel(tmp, u3_banks->phyd + U3P_U3_PHYD_IMPCAL0);
++
++ tmp = readl(u3_banks->phyd + U3P_U3_PHYD_IMPCAL1);
++ tmp &= ~P3D_RG_RX_IMPEL;
++ tmp |= P3D_RG_RX_IMPEL_VAL(instance->efuse_rx_imp);
++ tmp |= P3D_RG_FORCE_RX_IMPEL;
++ writel(tmp, u3_banks->phyd + U3P_U3_PHYD_IMPCAL1);
++
++ tmp = readl(u3_banks->phya + U3P_U3_PHYA_REG0);
++ tmp &= ~P3A_RG_IEXT_INTR;
++ tmp |= P3A_RG_IEXT_INTR_VAL(instance->efuse_intr);
++ writel(tmp, u3_banks->phya + U3P_U3_PHYA_REG0);
++ break;
++ default:
++ dev_warn(dev, "no sw efuse for type %d\n", instance->type);
++ break;
++ }
++}
++
+ static int mtk_phy_init(struct phy *phy)
+ {
+ struct mtk_phy_instance *instance = phy_get_drvdata(phy);
+@@ -1050,6 +1202,8 @@ static int mtk_phy_init(struct phy *phy)
+ if (ret)
+ return ret;
+
++ phy_efuse_set(instance);
++
+ switch (instance->type) {
+ case PHY_TYPE_USB2:
+ u2_phy_instance_init(tphy, instance);
+@@ -1134,6 +1288,7 @@ static struct phy *mtk_phy_xlate(struct device *dev,
+ struct mtk_phy_instance *instance = NULL;
+ struct device_node *phy_np = args->np;
+ int index;
++ int ret;
+
+ if (args->args_count != 1) {
+ dev_err(dev, "invalid number of cells in 'phy' property\n");
+@@ -1174,6 +1329,10 @@ static struct phy *mtk_phy_xlate(struct device *dev,
+ return ERR_PTR(-EINVAL);
+ }
+
++ ret = phy_efuse_get(tphy, instance);
++ if (ret)
++ return ERR_PTR(ret);
++
+ phy_parse_property(tphy, instance);
+ phy_type_set(instance);
+
+@@ -1196,10 +1355,12 @@ static const struct mtk_phy_pdata tphy_v1_pdata = {
+
+ static const struct mtk_phy_pdata tphy_v2_pdata = {
+ .avoid_rx_sen_degradation = false,
++ .sw_efuse_supported = true,
+ .version = MTK_PHY_V2,
+ };
+
+ static const struct mtk_phy_pdata tphy_v3_pdata = {
++ .sw_efuse_supported = true,
+ .version = MTK_PHY_V3,
+ };
+
+@@ -1210,6 +1371,7 @@ static const struct mtk_phy_pdata mt8173_pdata = {
+
+ static const struct mtk_phy_pdata mt8195_pdata = {
+ .sw_pll_48m_to_26m = true,
++ .sw_efuse_supported = true,
+ .version = MTK_PHY_V3,
+ };
+
+diff --git a/drivers/phy/microchip/sparx5_serdes.c b/drivers/phy/microchip/sparx5_serdes.c
+index 4076580fc2cd9..ab1b0986aa671 100644
+--- a/drivers/phy/microchip/sparx5_serdes.c
++++ b/drivers/phy/microchip/sparx5_serdes.c
+@@ -2475,10 +2475,10 @@ static int sparx5_serdes_probe(struct platform_device *pdev)
+ return -EINVAL;
+ }
+ iomem = devm_ioremap(priv->dev, iores->start, resource_size(iores));
+- if (IS_ERR(iomem)) {
++ if (!iomem) {
+ dev_err(priv->dev, "Unable to get serdes registers: %s\n",
+ iores->name);
+- return PTR_ERR(iomem);
++ return -ENOMEM;
+ }
+ for (idx = 0; idx < ARRAY_SIZE(sparx5_serdes_iomap); idx++) {
+ struct sparx5_serdes_io_resource *iomap = &sparx5_serdes_iomap[idx];
+diff --git a/drivers/phy/motorola/phy-mapphone-mdm6600.c b/drivers/phy/motorola/phy-mapphone-mdm6600.c
+index 5172971f4c360..3cd4d51c247c3 100644
+--- a/drivers/phy/motorola/phy-mapphone-mdm6600.c
++++ b/drivers/phy/motorola/phy-mapphone-mdm6600.c
+@@ -629,7 +629,8 @@ idle:
+ cleanup:
+ if (error < 0)
+ phy_mdm6600_device_power_off(ddata);
+-
++ pm_runtime_disable(ddata->dev);
++ pm_runtime_dont_use_autosuspend(ddata->dev);
+ return error;
+ }
+
+diff --git a/drivers/phy/phy-core-mipi-dphy.c b/drivers/phy/phy-core-mipi-dphy.c
+index 288c9c67aa748..929e86d6558e0 100644
+--- a/drivers/phy/phy-core-mipi-dphy.c
++++ b/drivers/phy/phy-core-mipi-dphy.c
+@@ -36,7 +36,7 @@ int phy_mipi_dphy_get_default_config(unsigned long pixel_clock,
+
+ cfg->clk_miss = 0;
+ cfg->clk_post = 60000 + 52 * ui;
+- cfg->clk_pre = 8000;
++ cfg->clk_pre = 8;
+ cfg->clk_prepare = 38000;
+ cfg->clk_settle = 95000;
+ cfg->clk_term_en = 0;
+@@ -64,10 +64,10 @@ int phy_mipi_dphy_get_default_config(unsigned long pixel_clock,
+ cfg->hs_trail = max(4 * 8 * ui, 60000 + 4 * 4 * ui);
+
+ cfg->init = 100;
+- cfg->lpx = 60000;
++ cfg->lpx = 50000;
+ cfg->ta_get = 5 * cfg->lpx;
+ cfg->ta_go = 4 * cfg->lpx;
+- cfg->ta_sure = 2 * cfg->lpx;
++ cfg->ta_sure = cfg->lpx;
+ cfg->wakeup = 1000;
+
+ cfg->hs_clk_rate = hs_clk_rate;
+@@ -97,7 +97,7 @@ int phy_mipi_dphy_config_validate(struct phy_configure_opts_mipi_dphy *cfg)
+ if (cfg->clk_post < (60000 + 52 * ui))
+ return -EINVAL;
+
+- if (cfg->clk_pre < 8000)
++ if (cfg->clk_pre < 8)
+ return -EINVAL;
+
+ if (cfg->clk_prepare < 38000 || cfg->clk_prepare > 95000)
+diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
+index f14032170b1c1..06b04606dd7ea 100644
+--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
++++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
+@@ -3632,7 +3632,7 @@ static const struct qmp_phy_cfg sc8180x_pciephy_cfg = {
+ .nlanes = 1,
+
+ .serdes_tbl = sc8180x_qmp_pcie_serdes_tbl,
+- .serdes_tbl_num = ARRAY_SIZE(sm8250_qmp_pcie_serdes_tbl),
++ .serdes_tbl_num = ARRAY_SIZE(sc8180x_qmp_pcie_serdes_tbl),
+ .tx_tbl = sc8180x_qmp_pcie_tx_tbl,
+ .tx_tbl_num = ARRAY_SIZE(sc8180x_qmp_pcie_tx_tbl),
+ .rx_tbl = sc8180x_qmp_pcie_rx_tbl,
+diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c
+index 3c1d3b71c825b..f1d97fbd13318 100644
+--- a/drivers/phy/qualcomm/phy-qcom-qusb2.c
++++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c
+@@ -561,7 +561,7 @@ static void qusb2_phy_set_tune2_param(struct qusb2_phy *qphy)
+ {
+ struct device *dev = &qphy->phy->dev;
+ const struct qusb2_phy_cfg *cfg = qphy->cfg;
+- u8 *val;
++ u8 *val, hstx_trim;
+
+ /* efuse register is optional */
+ if (!qphy->cell)
+@@ -575,7 +575,13 @@ static void qusb2_phy_set_tune2_param(struct qusb2_phy *qphy)
+ * set while configuring the phy.
+ */
+ val = nvmem_cell_read(qphy->cell, NULL);
+- if (IS_ERR(val) || !val[0]) {
++ if (IS_ERR(val)) {
++ dev_dbg(dev, "failed to read a valid hs-tx trim value\n");
++ return;
++ }
++ hstx_trim = val[0];
++ kfree(val);
++ if (!hstx_trim) {
+ dev_dbg(dev, "failed to read a valid hs-tx trim value\n");
+ return;
+ }
+@@ -583,12 +589,10 @@ static void qusb2_phy_set_tune2_param(struct qusb2_phy *qphy)
+ /* Fused TUNE1/2 value is the higher nibble only */
+ if (cfg->update_tune1_with_efuse)
+ qusb2_write_mask(qphy->base, cfg->regs[QUSB2PHY_PORT_TUNE1],
+- val[0] << HSTX_TRIM_SHIFT,
+- HSTX_TRIM_MASK);
++ hstx_trim << HSTX_TRIM_SHIFT, HSTX_TRIM_MASK);
+ else
+ qusb2_write_mask(qphy->base, cfg->regs[QUSB2PHY_PORT_TUNE2],
+- val[0] << HSTX_TRIM_SHIFT,
+- HSTX_TRIM_MASK);
++ hstx_trim << HSTX_TRIM_SHIFT, HSTX_TRIM_MASK);
+ }
+
+ static int qusb2_phy_set_mode(struct phy *phy,
+diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c
+index ae4bac024c7b1..7e61202aa234e 100644
+--- a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c
++++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c
+@@ -33,7 +33,7 @@
+
+ #define USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON0 (0x54)
+ #define RETENABLEN BIT(3)
+-#define FSEL_MASK GENMASK(7, 5)
++#define FSEL_MASK GENMASK(6, 4)
+ #define FSEL_DEFAULT (0x3 << 4)
+
+ #define USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON1 (0x58)
+diff --git a/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c
+index 347dc79a18c18..630e01b5c19b9 100644
+--- a/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c
++++ b/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c
+@@ -5,6 +5,7 @@
+ * Author: Wyon Bi <bivvy.bi@rock-chips.com>
+ */
+
++#include <linux/bits.h>
+ #include <linux/kernel.h>
+ #include <linux/clk.h>
+ #include <linux/iopoll.h>
+@@ -364,7 +365,7 @@ static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno)
+ * The value of counter for HS Tclk-pre
+ * Tclk-pre = Tpin_txbyteclkhs * value
+ */
+- clk_pre = DIV_ROUND_UP(cfg->clk_pre, t_txbyteclkhs);
++ clk_pre = DIV_ROUND_UP(cfg->clk_pre, BITS_PER_BYTE);
+
+ /*
+ * The value of counter for HS Tlpx Time
+diff --git a/drivers/phy/samsung/phy-exynos5250-sata.c b/drivers/phy/samsung/phy-exynos5250-sata.c
+index 9ec234243f7c6..595adba5fb8f1 100644
+--- a/drivers/phy/samsung/phy-exynos5250-sata.c
++++ b/drivers/phy/samsung/phy-exynos5250-sata.c
+@@ -187,6 +187,7 @@ static int exynos_sata_phy_probe(struct platform_device *pdev)
+ return -EINVAL;
+
+ sata_phy->client = of_find_i2c_device_by_node(node);
++ of_node_put(node);
+ if (!sata_phy->client)
+ return -EPROBE_DEFER;
+
+@@ -195,20 +196,21 @@ static int exynos_sata_phy_probe(struct platform_device *pdev)
+ sata_phy->phyclk = devm_clk_get(dev, "sata_phyctrl");
+ if (IS_ERR(sata_phy->phyclk)) {
+ dev_err(dev, "failed to get clk for PHY\n");
+- return PTR_ERR(sata_phy->phyclk);
++ ret = PTR_ERR(sata_phy->phyclk);
++ goto put_dev;
+ }
+
+ ret = clk_prepare_enable(sata_phy->phyclk);
+ if (ret < 0) {
+ dev_err(dev, "failed to enable source clk\n");
+- return ret;
++ goto put_dev;
+ }
+
+ sata_phy->phy = devm_phy_create(dev, NULL, &exynos_sata_phy_ops);
+ if (IS_ERR(sata_phy->phy)) {
+- clk_disable_unprepare(sata_phy->phyclk);
+ dev_err(dev, "failed to create PHY\n");
+- return PTR_ERR(sata_phy->phy);
++ ret = PTR_ERR(sata_phy->phy);
++ goto clk_disable;
+ }
+
+ phy_set_drvdata(sata_phy->phy, sata_phy);
+@@ -216,11 +218,18 @@ static int exynos_sata_phy_probe(struct platform_device *pdev)
+ phy_provider = devm_of_phy_provider_register(dev,
+ of_phy_simple_xlate);
+ if (IS_ERR(phy_provider)) {
+- clk_disable_unprepare(sata_phy->phyclk);
+- return PTR_ERR(phy_provider);
++ ret = PTR_ERR(phy_provider);
++ goto clk_disable;
+ }
+
+ return 0;
++
++clk_disable:
++ clk_disable_unprepare(sata_phy->phyclk);
++put_dev:
++ put_device(&sata_phy->client->dev);
++
++ return ret;
+ }
+
+ static const struct of_device_id exynos_sata_phy_of_match[] = {
+diff --git a/drivers/phy/socionext/phy-uniphier-usb3ss.c b/drivers/phy/socionext/phy-uniphier-usb3ss.c
+index 6700645bcbe6b..3b5ffc16a6947 100644
+--- a/drivers/phy/socionext/phy-uniphier-usb3ss.c
++++ b/drivers/phy/socionext/phy-uniphier-usb3ss.c
+@@ -22,11 +22,13 @@
+ #include <linux/reset.h>
+
+ #define SSPHY_TESTI 0x0
+-#define SSPHY_TESTO 0x4
+ #define TESTI_DAT_MASK GENMASK(13, 6)
+ #define TESTI_ADR_MASK GENMASK(5, 1)
+ #define TESTI_WR_EN BIT(0)
+
++#define SSPHY_TESTO 0x4
++#define TESTO_DAT_MASK GENMASK(7, 0)
++
+ #define PHY_F(regno, msb, lsb) { (regno), (msb), (lsb) }
+
+ #define CDR_CPD_TRIM PHY_F(7, 3, 0) /* RxPLL charge pump current */
+@@ -84,12 +86,12 @@ static void uniphier_u3ssphy_set_param(struct uniphier_u3ssphy_priv *priv,
+ val = FIELD_PREP(TESTI_DAT_MASK, 1);
+ val |= FIELD_PREP(TESTI_ADR_MASK, p->field.reg_no);
+ uniphier_u3ssphy_testio_write(priv, val);
+- val = readl(priv->base + SSPHY_TESTO);
++ val = readl(priv->base + SSPHY_TESTO) & TESTO_DAT_MASK;
+
+ /* update value */
+- val &= ~FIELD_PREP(TESTI_DAT_MASK, field_mask);
++ val &= ~field_mask;
+ data = field_mask & (p->value << p->field.lsb);
+- val = FIELD_PREP(TESTI_DAT_MASK, data);
++ val = FIELD_PREP(TESTI_DAT_MASK, data | val);
+ val |= FIELD_PREP(TESTI_ADR_MASK, p->field.reg_no);
+ uniphier_u3ssphy_testio_write(priv, val);
+ uniphier_u3ssphy_testio_write(priv, val | TESTI_WR_EN);
+diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c
+index 937a14fa7448a..da05642d3bd4a 100644
+--- a/drivers/phy/st/phy-stm32-usbphyc.c
++++ b/drivers/phy/st/phy-stm32-usbphyc.c
+@@ -225,7 +225,7 @@ static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc)
+
+ ret = __stm32_usbphyc_pll_disable(usbphyc);
+ if (ret)
+- return ret;
++ goto dec_n_pll_cons;
+ }
+
+ ret = stm32_usbphyc_regulators_enable(usbphyc);
+diff --git a/drivers/phy/ti/phy-am654-serdes.c b/drivers/phy/ti/phy-am654-serdes.c
+index 2ff56ce77b307..21c0088f5ca9e 100644
+--- a/drivers/phy/ti/phy-am654-serdes.c
++++ b/drivers/phy/ti/phy-am654-serdes.c
+@@ -838,7 +838,7 @@ static int serdes_am654_probe(struct platform_device *pdev)
+
+ clk_err:
+ of_clk_del_provider(node);
+-
++ pm_runtime_disable(dev);
+ return ret;
+ }
+
+diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c
+index 5fd2e8a08bfcf..d0ab69750c6b4 100644
+--- a/drivers/phy/ti/phy-gmii-sel.c
++++ b/drivers/phy/ti/phy-gmii-sel.c
+@@ -320,6 +320,8 @@ static int phy_gmii_sel_init_ports(struct phy_gmii_sel_priv *priv)
+ u64 size;
+
+ offset = of_get_address(dev->of_node, 0, &size, NULL);
++ if (!offset)
++ return -EINVAL;
+ priv->num_ports = size / sizeof(u32);
+ if (!priv->num_ports)
+ return -EINVAL;
+diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c
+index 126f5b8735cc1..8963fbf7aa73b 100644
+--- a/drivers/phy/ti/phy-j721e-wiz.c
++++ b/drivers/phy/ti/phy-j721e-wiz.c
+@@ -233,6 +233,7 @@ static const struct clk_div_table clk_div_table[] = {
+ { .val = 1, .div = 2, },
+ { .val = 2, .div = 4, },
+ { .val = 3, .div = 8, },
++ { /* sentinel */ },
+ };
+
+ static const struct wiz_clk_div_sel clk_div_sel[] = {
+diff --git a/drivers/phy/ti/phy-omap-usb2.c b/drivers/phy/ti/phy-omap-usb2.c
+index ebceb1520ce88..ca8532a3f1931 100644
+--- a/drivers/phy/ti/phy-omap-usb2.c
++++ b/drivers/phy/ti/phy-omap-usb2.c
+@@ -215,7 +215,7 @@ static int omap_usb2_enable_clocks(struct omap_usb *phy)
+ return 0;
+
+ err1:
+- clk_disable(phy->wkupclk);
++ clk_disable_unprepare(phy->wkupclk);
+
+ err0:
+ return ret;
+diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c
+index f478d8a17115b..9be9535ad7ab7 100644
+--- a/drivers/phy/xilinx/phy-zynqmp.c
++++ b/drivers/phy/xilinx/phy-zynqmp.c
+@@ -134,7 +134,8 @@
+ #define PROT_BUS_WIDTH_10 0x0
+ #define PROT_BUS_WIDTH_20 0x1
+ #define PROT_BUS_WIDTH_40 0x2
+-#define PROT_BUS_WIDTH_SHIFT 2
++#define PROT_BUS_WIDTH_SHIFT(n) ((n) * 2)
++#define PROT_BUS_WIDTH_MASK(n) GENMASK((n) * 2 + 1, (n) * 2)
+
+ /* Number of GT lanes */
+ #define NUM_LANES 4
+@@ -445,12 +446,12 @@ static void xpsgtr_phy_init_sata(struct xpsgtr_phy *gtr_phy)
+ static void xpsgtr_phy_init_sgmii(struct xpsgtr_phy *gtr_phy)
+ {
+ struct xpsgtr_dev *gtr_dev = gtr_phy->dev;
++ u32 mask = PROT_BUS_WIDTH_MASK(gtr_phy->lane);
++ u32 val = PROT_BUS_WIDTH_10 << PROT_BUS_WIDTH_SHIFT(gtr_phy->lane);
+
+ /* Set SGMII protocol TX and RX bus width to 10 bits. */
+- xpsgtr_write(gtr_dev, TX_PROT_BUS_WIDTH,
+- PROT_BUS_WIDTH_10 << (gtr_phy->lane * PROT_BUS_WIDTH_SHIFT));
+- xpsgtr_write(gtr_dev, RX_PROT_BUS_WIDTH,
+- PROT_BUS_WIDTH_10 << (gtr_phy->lane * PROT_BUS_WIDTH_SHIFT));
++ xpsgtr_clr_set(gtr_dev, TX_PROT_BUS_WIDTH, mask, val);
++ xpsgtr_clr_set(gtr_dev, RX_PROT_BUS_WIDTH, mask, val);
+
+ xpsgtr_bypass_scrambler_8b10b(gtr_phy);
+ }
+diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g6.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g6.c
+index a3fa03bcd9a30..54064714d73fb 100644
+--- a/drivers/pinctrl/aspeed/pinctrl-aspeed-g6.c
++++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g6.c
+@@ -1236,18 +1236,12 @@ FUNC_GROUP_DECL(SALT8, AA12);
+ FUNC_GROUP_DECL(WDTRST4, AA12);
+
+ #define AE12 196
+-SIG_EXPR_LIST_DECL_SEMG(AE12, FWSPIDQ2, FWQSPID, FWSPID,
+- SIG_DESC_SET(SCU438, 4));
+ SIG_EXPR_LIST_DECL_SESG(AE12, GPIOY4, GPIOY4);
+-PIN_DECL_(AE12, SIG_EXPR_LIST_PTR(AE12, FWSPIDQ2),
+- SIG_EXPR_LIST_PTR(AE12, GPIOY4));
++PIN_DECL_(AE12, SIG_EXPR_LIST_PTR(AE12, GPIOY4));
+
+ #define AF12 197
+-SIG_EXPR_LIST_DECL_SEMG(AF12, FWSPIDQ3, FWQSPID, FWSPID,
+- SIG_DESC_SET(SCU438, 5));
+ SIG_EXPR_LIST_DECL_SESG(AF12, GPIOY5, GPIOY5);
+-PIN_DECL_(AF12, SIG_EXPR_LIST_PTR(AF12, FWSPIDQ3),
+- SIG_EXPR_LIST_PTR(AF12, GPIOY5));
++PIN_DECL_(AF12, SIG_EXPR_LIST_PTR(AF12, GPIOY5));
+
+ #define AC12 198
+ SSSF_PIN_DECL(AC12, GPIOY6, FWSPIABR, SIG_DESC_SET(SCU438, 6));
+@@ -1520,9 +1514,8 @@ SIG_EXPR_LIST_DECL_SEMG(Y4, EMMCDAT7, EMMCG8, EMMC, SIG_DESC_SET(SCU404, 3));
+ PIN_DECL_3(Y4, GPIO18E3, FWSPIDMISO, VBMISO, EMMCDAT7);
+
+ GROUP_DECL(FWSPID, Y1, Y2, Y3, Y4);
+-GROUP_DECL(FWQSPID, Y1, Y2, Y3, Y4, AE12, AF12);
+ GROUP_DECL(EMMCG8, AB4, AA4, AC4, AA5, Y5, AB5, AB6, AC5, Y1, Y2, Y3, Y4);
+-FUNC_DECL_2(FWSPID, FWSPID, FWQSPID);
++FUNC_DECL_1(FWSPID, FWSPID);
+ FUNC_GROUP_DECL(VB, Y1, Y2, Y3, Y4);
+ FUNC_DECL_3(EMMC, EMMCG1, EMMCG4, EMMCG8);
+ /*
+@@ -1918,7 +1911,6 @@ static const struct aspeed_pin_group aspeed_g6_groups[] = {
+ ASPEED_PINCTRL_GROUP(FSI2),
+ ASPEED_PINCTRL_GROUP(FWSPIABR),
+ ASPEED_PINCTRL_GROUP(FWSPID),
+- ASPEED_PINCTRL_GROUP(FWQSPID),
+ ASPEED_PINCTRL_GROUP(FWSPIWP),
+ ASPEED_PINCTRL_GROUP(GPIT0),
+ ASPEED_PINCTRL_GROUP(GPIT1),
+diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig
+index c9c5efc927311..5973a279e6b8c 100644
+--- a/drivers/pinctrl/bcm/Kconfig
++++ b/drivers/pinctrl/bcm/Kconfig
+@@ -35,6 +35,7 @@ config PINCTRL_BCM63XX
+ select PINCONF
+ select GENERIC_PINCONF
+ select GPIOLIB
++ select REGMAP
+ select GPIO_REGMAP
+
+ config PINCTRL_BCM6318
+diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+index 6e6fefeb21ead..cb339299adf92 100644
+--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
++++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+@@ -1243,6 +1243,18 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
+ raw_spin_lock_init(&pc->irq_lock[i]);
+ }
+
++ pc->pctl_desc = *pdata->pctl_desc;
++ pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc);
++ if (IS_ERR(pc->pctl_dev)) {
++ gpiochip_remove(&pc->gpio_chip);
++ return PTR_ERR(pc->pctl_dev);
++ }
++
++ pc->gpio_range = *pdata->gpio_range;
++ pc->gpio_range.base = pc->gpio_chip.base;
++ pc->gpio_range.gc = &pc->gpio_chip;
++ pinctrl_add_gpio_range(pc->pctl_dev, &pc->gpio_range);
++
+ girq = &pc->gpio_chip.irq;
+ girq->chip = &bcm2835_gpio_irq_chip;
+ girq->parent_handler = bcm2835_gpio_irq_handler;
+@@ -1250,15 +1262,19 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
+ girq->parents = devm_kcalloc(dev, BCM2835_NUM_IRQS,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+- if (!girq->parents)
+- return -ENOMEM;
++ if (!girq->parents) {
++ err = -ENOMEM;
++ goto out_remove;
++ }
+
+ if (is_7211) {
+ pc->wake_irq = devm_kcalloc(dev, BCM2835_NUM_IRQS,
+ sizeof(*pc->wake_irq),
+ GFP_KERNEL);
+- if (!pc->wake_irq)
+- return -ENOMEM;
++ if (!pc->wake_irq) {
++ err = -ENOMEM;
++ goto out_remove;
++ }
+ }
+
+ /*
+@@ -1286,8 +1302,10 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
+
+ len = strlen(dev_name(pc->dev)) + 16;
+ name = devm_kzalloc(pc->dev, len, GFP_KERNEL);
+- if (!name)
+- return -ENOMEM;
++ if (!name) {
++ err = -ENOMEM;
++ goto out_remove;
++ }
+
+ snprintf(name, len, "%s:bank%d", dev_name(pc->dev), i);
+
+@@ -1306,22 +1324,14 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
+ err = gpiochip_add_data(&pc->gpio_chip, pc);
+ if (err) {
+ dev_err(dev, "could not add GPIO chip\n");
+- return err;
+- }
+-
+- pc->pctl_desc = *pdata->pctl_desc;
+- pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc);
+- if (IS_ERR(pc->pctl_dev)) {
+- gpiochip_remove(&pc->gpio_chip);
+- return PTR_ERR(pc->pctl_dev);
++ goto out_remove;
+ }
+
+- pc->gpio_range = *pdata->gpio_range;
+- pc->gpio_range.base = pc->gpio_chip.base;
+- pc->gpio_range.gc = &pc->gpio_chip;
+- pinctrl_add_gpio_range(pc->pctl_dev, &pc->gpio_range);
+-
+ return 0;
++
++out_remove:
++ pinctrl_remove_gpio_range(pc->pctl_dev, &pc->gpio_range);
++ return err;
+ }
+
+ static struct platform_driver bcm2835_pinctrl_driver = {
+diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
+index 5082102d7d0d9..ffe39336fcaca 100644
+--- a/drivers/pinctrl/core.c
++++ b/drivers/pinctrl/core.c
+@@ -2100,6 +2100,8 @@ int pinctrl_enable(struct pinctrl_dev *pctldev)
+ if (error) {
+ dev_err(pctldev->dev, "could not claim hogs: %i\n",
+ error);
++ pinctrl_free_pindescs(pctldev, pctldev->desc->pins,
++ pctldev->desc->npins);
+ mutex_destroy(&pctldev->mutex);
+ kfree(pctldev);
+
+diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
+index 85750974d1825..826d494f3cc66 100644
+--- a/drivers/pinctrl/intel/pinctrl-intel.c
++++ b/drivers/pinctrl/intel/pinctrl-intel.c
+@@ -451,8 +451,8 @@ static void intel_gpio_set_gpio_mode(void __iomem *padcfg0)
+ value &= ~PADCFG0_PMODE_MASK;
+ value |= PADCFG0_PMODE_GPIO;
+
+- /* Disable input and output buffers */
+- value |= PADCFG0_GPIORXDIS;
++ /* Disable TX buffer and enable RX (this will be input) */
++ value &= ~PADCFG0_GPIORXDIS;
+ value |= PADCFG0_GPIOTXDIS;
+
+ /* Disable SCI/SMI/NMI generation */
+@@ -497,9 +497,6 @@ static int intel_gpio_request_enable(struct pinctrl_dev *pctldev,
+
+ intel_gpio_set_gpio_mode(padcfg0);
+
+- /* Disable TX buffer and enable RX (this will be input) */
+- __intel_gpio_set_direction(padcfg0, true);
+-
+ raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+ return 0;
+@@ -1115,9 +1112,6 @@ static int intel_gpio_irq_type(struct irq_data *d, unsigned int type)
+
+ intel_gpio_set_gpio_mode(reg);
+
+- /* Disable TX buffer and enable RX (this will be input) */
+- __intel_gpio_set_direction(reg, true);
+-
+ value = readl(reg);
+
+ value &= ~(PADCFG0_RXEVCFG_MASK | PADCFG0_RXINV);
+@@ -1216,6 +1210,39 @@ static irqreturn_t intel_gpio_irq(int irq, void *data)
+ return IRQ_RETVAL(ret);
+ }
+
++static void intel_gpio_irq_init(struct intel_pinctrl *pctrl)
++{
++ int i;
++
++ for (i = 0; i < pctrl->ncommunities; i++) {
++ const struct intel_community *community;
++ void __iomem *base;
++ unsigned int gpp;
++
++ community = &pctrl->communities[i];
++ base = community->regs;
++
++ for (gpp = 0; gpp < community->ngpps; gpp++) {
++ /* Mask and clear all interrupts */
++ writel(0, base + community->ie_offset + gpp * 4);
++ writel(0xffff, base + community->is_offset + gpp * 4);
++ }
++ }
++}
++
++static int intel_gpio_irq_init_hw(struct gpio_chip *gc)
++{
++ struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
++
++ /*
++ * Make sure the interrupt lines are in a proper state before
++ * further configuration.
++ */
++ intel_gpio_irq_init(pctrl);
++
++ return 0;
++}
++
+ static int intel_gpio_add_community_ranges(struct intel_pinctrl *pctrl,
+ const struct intel_community *community)
+ {
+@@ -1320,6 +1347,7 @@ static int intel_gpio_probe(struct intel_pinctrl *pctrl, int irq)
+ girq->num_parents = 0;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
++ girq->init_hw = intel_gpio_irq_init_hw;
+
+ ret = devm_gpiochip_add_data(pctrl->dev, &pctrl->chip, pctrl);
+ if (ret) {
+@@ -1695,26 +1723,6 @@ int intel_pinctrl_suspend_noirq(struct device *dev)
+ }
+ EXPORT_SYMBOL_GPL(intel_pinctrl_suspend_noirq);
+
+-static void intel_gpio_irq_init(struct intel_pinctrl *pctrl)
+-{
+- size_t i;
+-
+- for (i = 0; i < pctrl->ncommunities; i++) {
+- const struct intel_community *community;
+- void __iomem *base;
+- unsigned int gpp;
+-
+- community = &pctrl->communities[i];
+- base = community->regs;
+-
+- for (gpp = 0; gpp < community->ngpps; gpp++) {
+- /* Mask and clear all interrupts */
+- writel(0, base + community->ie_offset + gpp * 4);
+- writel(0xffff, base + community->is_offset + gpp * 4);
+- }
+- }
+-}
+-
+ static bool intel_gpio_update_reg(void __iomem *reg, u32 mask, u32 value)
+ {
+ u32 curr, updated;
+diff --git a/drivers/pinctrl/intel/pinctrl-tigerlake.c b/drivers/pinctrl/intel/pinctrl-tigerlake.c
+index 0bcd19597e4ad..3ddaeffc04150 100644
+--- a/drivers/pinctrl/intel/pinctrl-tigerlake.c
++++ b/drivers/pinctrl/intel/pinctrl-tigerlake.c
+@@ -749,7 +749,6 @@ static const struct acpi_device_id tgl_pinctrl_acpi_match[] = {
+ { "INT34C5", (kernel_ulong_t)&tgllp_soc_data },
+ { "INT34C6", (kernel_ulong_t)&tglh_soc_data },
+ { "INTC1055", (kernel_ulong_t)&tgllp_soc_data },
+- { "INTC1057", (kernel_ulong_t)&tgllp_soc_data },
+ { }
+ };
+ MODULE_DEVICE_TABLE(acpi, tgl_pinctrl_acpi_match);
+diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig
+index 7040a7a7bd5d1..246b0e951e1ce 100644
+--- a/drivers/pinctrl/mediatek/Kconfig
++++ b/drivers/pinctrl/mediatek/Kconfig
+@@ -30,6 +30,7 @@ config PINCTRL_MTK_MOORE
+ select GENERIC_PINMUX_FUNCTIONS
+ select GPIOLIB
+ select OF_GPIO
++ select EINT_MTK
+ select PINCTRL_MTK_V2
+
+ config PINCTRL_MTK_PARIS
+diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8365.c b/drivers/pinctrl/mediatek/pinctrl-mt8365.c
+index 79b1fee5a1eba..ddee0db72d264 100644
+--- a/drivers/pinctrl/mediatek/pinctrl-mt8365.c
++++ b/drivers/pinctrl/mediatek/pinctrl-mt8365.c
+@@ -259,7 +259,7 @@ static const struct mtk_pin_ies_smt_set mt8365_ies_set[] = {
+ MTK_PIN_IES_SMT_SPEC(104, 104, 0x420, 13),
+ MTK_PIN_IES_SMT_SPEC(105, 109, 0x420, 14),
+ MTK_PIN_IES_SMT_SPEC(110, 113, 0x420, 15),
+- MTK_PIN_IES_SMT_SPEC(114, 112, 0x420, 16),
++ MTK_PIN_IES_SMT_SPEC(114, 116, 0x420, 16),
+ MTK_PIN_IES_SMT_SPEC(117, 119, 0x420, 17),
+ MTK_PIN_IES_SMT_SPEC(120, 122, 0x420, 18),
+ MTK_PIN_IES_SMT_SPEC(123, 125, 0x420, 19),
+diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
+index 45ebdeba985ae..12163d3c4bcb0 100644
+--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
++++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
+@@ -285,8 +285,12 @@ static int mtk_xt_get_gpio_n(void *data, unsigned long eint_n,
+ desc = (const struct mtk_pin_desc *)hw->soc->pins;
+ *gpio_chip = &hw->chip;
+
+- /* Be greedy to guess first gpio_n is equal to eint_n */
+- if (desc[eint_n].eint.eint_n == eint_n)
++ /*
++ * Be greedy to guess first gpio_n is equal to eint_n.
++ * Only eint virtual eint number is greater than gpio number.
++ */
++ if (hw->soc->npins > eint_n &&
++ desc[eint_n].eint.eint_n == eint_n)
+ *gpio_n = eint_n;
+ else
+ *gpio_n = mtk_xt_find_eint_num(hw, eint_n);
+diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+index 5f7c421ab6e76..334cb85855a93 100644
+--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
++++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+@@ -1038,6 +1038,7 @@ int mtk_pctrl_init(struct platform_device *pdev,
+ node = of_parse_phandle(np, "mediatek,pctl-regmap", 0);
+ if (node) {
+ pctl->regmap1 = syscon_node_to_regmap(node);
++ of_node_put(node);
+ if (IS_ERR(pctl->regmap1))
+ return PTR_ERR(pctl->regmap1);
+ } else if (regmap) {
+@@ -1051,6 +1052,7 @@ int mtk_pctrl_init(struct platform_device *pdev,
+ node = of_parse_phandle(np, "mediatek,pctl-regmap", 1);
+ if (node) {
+ pctl->regmap2 = syscon_node_to_regmap(node);
++ of_node_put(node);
+ if (IS_ERR(pctl->regmap2))
+ return PTR_ERR(pctl->regmap2);
+ }
+diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c
+index 85db2e4377f0c..02e2a259edd39 100644
+--- a/drivers/pinctrl/mediatek/pinctrl-paris.c
++++ b/drivers/pinctrl/mediatek/pinctrl-paris.c
+@@ -96,20 +96,16 @@ static int mtk_pinconf_get(struct pinctrl_dev *pctldev,
+ err = hw->soc->bias_get_combo(hw, desc, &pullup, &ret);
+ if (err)
+ goto out;
++ if (ret == MTK_PUPD_SET_R1R0_00)
++ ret = MTK_DISABLE;
+ if (param == PIN_CONFIG_BIAS_DISABLE) {
+- if (ret == MTK_PUPD_SET_R1R0_00)
+- ret = MTK_DISABLE;
++ if (ret != MTK_DISABLE)
++ err = -EINVAL;
+ } else if (param == PIN_CONFIG_BIAS_PULL_UP) {
+- /* When desire to get pull-up value, return
+- * error if current setting is pull-down
+- */
+- if (!pullup)
++ if (!pullup || ret == MTK_DISABLE)
+ err = -EINVAL;
+ } else if (param == PIN_CONFIG_BIAS_PULL_DOWN) {
+- /* When desire to get pull-down value, return
+- * error if current setting is pull-up
+- */
+- if (pullup)
++ if (pullup || ret == MTK_DISABLE)
+ err = -EINVAL;
+ }
+ } else {
+@@ -188,8 +184,7 @@ out:
+ }
+
+ static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
+- enum pin_config_param param,
+- enum pin_config_param arg)
++ enum pin_config_param param, u32 arg)
+ {
+ struct mtk_pinctrl *hw = pinctrl_dev_get_drvdata(pctldev);
+ const struct mtk_pin_desc *desc;
+@@ -585,6 +580,9 @@ ssize_t mtk_pctrl_show_one_pin(struct mtk_pinctrl *hw,
+ if (gpio >= hw->soc->npins)
+ return -EINVAL;
+
++ if (mtk_is_virt_gpio(hw, gpio))
++ return -EINVAL;
++
+ desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio];
+ pinmux = mtk_pctrl_get_pinmux(hw, gpio);
+ if (pinmux >= hw->soc->nfuncs)
+@@ -719,10 +717,10 @@ static int mtk_pconf_group_get(struct pinctrl_dev *pctldev, unsigned group,
+ unsigned long *config)
+ {
+ struct mtk_pinctrl *hw = pinctrl_dev_get_drvdata(pctldev);
++ struct mtk_pinctrl_group *grp = &hw->groups[group];
+
+- *config = hw->groups[group].config;
+-
+- return 0;
++ /* One pin per group only */
++ return mtk_pinconf_get(pctldev, grp->pin, config);
+ }
+
+ static int mtk_pconf_group_set(struct pinctrl_dev *pctldev, unsigned group,
+@@ -738,8 +736,6 @@ static int mtk_pconf_group_set(struct pinctrl_dev *pctldev, unsigned group,
+ pinconf_to_config_argument(configs[i]));
+ if (ret < 0)
+ return ret;
+-
+- grp->config = configs[i];
+ }
+
+ return 0;
+diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+index 39828e9c3120a..4757bf964d3cd 100644
+--- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c
++++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+@@ -1883,8 +1883,10 @@ static int nmk_pinctrl_probe(struct platform_device *pdev)
+ }
+
+ prcm_np = of_parse_phandle(np, "prcm", 0);
+- if (prcm_np)
++ if (prcm_np) {
+ npct->prcm_base = of_iomap(prcm_np, 0);
++ of_node_put(prcm_np);
++ }
+ if (!npct->prcm_base) {
+ if (version == PINCTRL_NMK_STN8815) {
+ dev_info(&pdev->dev,
+diff --git a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
+index 4d81908d6725d..41136f63014a4 100644
+--- a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
++++ b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
+@@ -78,7 +78,6 @@ struct npcm7xx_gpio {
+ struct gpio_chip gc;
+ int irqbase;
+ int irq;
+- void *priv;
+ struct irq_chip irq_chip;
+ u32 pinctrl_id;
+ int (*direction_input)(struct gpio_chip *chip, unsigned offset);
+@@ -226,7 +225,7 @@ static void npcmgpio_irq_handler(struct irq_desc *desc)
+ chained_irq_enter(chip, desc);
+ sts = ioread32(bank->base + NPCM7XX_GP_N_EVST);
+ en = ioread32(bank->base + NPCM7XX_GP_N_EVEN);
+- dev_dbg(chip->parent_device, "==> got irq sts %.8x %.8x\n", sts,
++ dev_dbg(bank->gc.parent, "==> got irq sts %.8x %.8x\n", sts,
+ en);
+
+ sts &= en;
+@@ -241,33 +240,33 @@ static int npcmgpio_set_irq_type(struct irq_data *d, unsigned int type)
+ gpiochip_get_data(irq_data_get_irq_chip_data(d));
+ unsigned int gpio = BIT(d->hwirq);
+
+- dev_dbg(d->chip->parent_device, "setirqtype: %u.%u = %u\n", gpio,
++ dev_dbg(bank->gc.parent, "setirqtype: %u.%u = %u\n", gpio,
+ d->irq, type);
+ switch (type) {
+ case IRQ_TYPE_EDGE_RISING:
+- dev_dbg(d->chip->parent_device, "edge.rising\n");
++ dev_dbg(bank->gc.parent, "edge.rising\n");
+ npcm_gpio_clr(&bank->gc, bank->base + NPCM7XX_GP_N_EVBE, gpio);
+ npcm_gpio_clr(&bank->gc, bank->base + NPCM7XX_GP_N_POL, gpio);
+ break;
+ case IRQ_TYPE_EDGE_FALLING:
+- dev_dbg(d->chip->parent_device, "edge.falling\n");
++ dev_dbg(bank->gc.parent, "edge.falling\n");
+ npcm_gpio_clr(&bank->gc, bank->base + NPCM7XX_GP_N_EVBE, gpio);
+ npcm_gpio_set(&bank->gc, bank->base + NPCM7XX_GP_N_POL, gpio);
+ break;
+ case IRQ_TYPE_EDGE_BOTH:
+- dev_dbg(d->chip->parent_device, "edge.both\n");
++ dev_dbg(bank->gc.parent, "edge.both\n");
+ npcm_gpio_set(&bank->gc, bank->base + NPCM7XX_GP_N_EVBE, gpio);
+ break;
+ case IRQ_TYPE_LEVEL_LOW:
+- dev_dbg(d->chip->parent_device, "level.low\n");
++ dev_dbg(bank->gc.parent, "level.low\n");
+ npcm_gpio_set(&bank->gc, bank->base + NPCM7XX_GP_N_POL, gpio);
+ break;
+ case IRQ_TYPE_LEVEL_HIGH:
+- dev_dbg(d->chip->parent_device, "level.high\n");
++ dev_dbg(bank->gc.parent, "level.high\n");
+ npcm_gpio_clr(&bank->gc, bank->base + NPCM7XX_GP_N_POL, gpio);
+ break;
+ default:
+- dev_dbg(d->chip->parent_device, "invalid irq type\n");
++ dev_dbg(bank->gc.parent, "invalid irq type\n");
+ return -EINVAL;
+ }
+
+@@ -289,7 +288,7 @@ static void npcmgpio_irq_ack(struct irq_data *d)
+ gpiochip_get_data(irq_data_get_irq_chip_data(d));
+ unsigned int gpio = d->hwirq;
+
+- dev_dbg(d->chip->parent_device, "irq_ack: %u.%u\n", gpio, d->irq);
++ dev_dbg(bank->gc.parent, "irq_ack: %u.%u\n", gpio, d->irq);
+ iowrite32(BIT(gpio), bank->base + NPCM7XX_GP_N_EVST);
+ }
+
+@@ -301,7 +300,7 @@ static void npcmgpio_irq_mask(struct irq_data *d)
+ unsigned int gpio = d->hwirq;
+
+ /* Clear events */
+- dev_dbg(d->chip->parent_device, "irq_mask: %u.%u\n", gpio, d->irq);
++ dev_dbg(bank->gc.parent, "irq_mask: %u.%u\n", gpio, d->irq);
+ iowrite32(BIT(gpio), bank->base + NPCM7XX_GP_N_EVENC);
+ }
+
+@@ -313,7 +312,7 @@ static void npcmgpio_irq_unmask(struct irq_data *d)
+ unsigned int gpio = d->hwirq;
+
+ /* Enable events */
+- dev_dbg(d->chip->parent_device, "irq_unmask: %u.%u\n", gpio, d->irq);
++ dev_dbg(bank->gc.parent, "irq_unmask: %u.%u\n", gpio, d->irq);
+ iowrite32(BIT(gpio), bank->base + NPCM7XX_GP_N_EVENS);
+ }
+
+@@ -323,7 +322,7 @@ static unsigned int npcmgpio_irq_startup(struct irq_data *d)
+ unsigned int gpio = d->hwirq;
+
+ /* active-high, input, clear interrupt, enable interrupt */
+- dev_dbg(d->chip->parent_device, "startup: %u.%u\n", gpio, d->irq);
++ dev_dbg(gc->parent, "startup: %u.%u\n", gpio, d->irq);
+ npcmgpio_direction_input(gc, gpio);
+ npcmgpio_irq_ack(d);
+ npcmgpio_irq_unmask(d);
+@@ -905,7 +904,7 @@ static struct npcm7xx_func npcm7xx_funcs[] = {
+ #define DRIVE_STRENGTH_HI_SHIFT 12
+ #define DRIVE_STRENGTH_MASK 0x0000FF00
+
+-#define DS(lo, hi) (((lo) << DRIVE_STRENGTH_LO_SHIFT) | \
++#define DSTR(lo, hi) (((lo) << DRIVE_STRENGTH_LO_SHIFT) | \
+ ((hi) << DRIVE_STRENGTH_HI_SHIFT))
+ #define DSLO(x) (((x) >> DRIVE_STRENGTH_LO_SHIFT) & 0xF)
+ #define DSHI(x) (((x) >> DRIVE_STRENGTH_HI_SHIFT) & 0xF)
+@@ -925,31 +924,31 @@ struct npcm7xx_pincfg {
+ static const struct npcm7xx_pincfg pincfg[] = {
+ /* PIN FUNCTION 1 FUNCTION 2 FUNCTION 3 FLAGS */
+ NPCM7XX_PINCFG(0, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(1, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(2, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(1, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(2, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(3, iox1, MFSEL1, 30, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(4, iox2, MFSEL3, 14, smb1d, I2CSEGSEL, 7, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(5, iox2, MFSEL3, 14, smb1d, I2CSEGSEL, 7, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(6, iox2, MFSEL3, 14, smb2d, I2CSEGSEL, 10, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(7, iox2, MFSEL3, 14, smb2d, I2CSEGSEL, 10, none, NONE, 0, SLEW),
+- NPCM7XX_PINCFG(8, lkgpo1, FLOCKR1, 4, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(9, lkgpo2, FLOCKR1, 8, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(10, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(11, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(8, lkgpo1, FLOCKR1, 4, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(9, lkgpo2, FLOCKR1, 8, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(10, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(11, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(12, gspi, MFSEL1, 24, smb5b, I2CSEGSEL, 19, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(13, gspi, MFSEL1, 24, smb5b, I2CSEGSEL, 19, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(14, gspi, MFSEL1, 24, smb5c, I2CSEGSEL, 20, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(15, gspi, MFSEL1, 24, smb5c, I2CSEGSEL, 20, none, NONE, 0, SLEW),
+- NPCM7XX_PINCFG(16, lkgpo0, FLOCKR1, 0, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(17, pspi2, MFSEL3, 13, smb4den, I2CSEGSEL, 23, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(18, pspi2, MFSEL3, 13, smb4b, I2CSEGSEL, 14, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(19, pspi2, MFSEL3, 13, smb4b, I2CSEGSEL, 14, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(16, lkgpo0, FLOCKR1, 0, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(17, pspi2, MFSEL3, 13, smb4den, I2CSEGSEL, 23, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(18, pspi2, MFSEL3, 13, smb4b, I2CSEGSEL, 14, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(19, pspi2, MFSEL3, 13, smb4b, I2CSEGSEL, 14, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(20, smb4c, I2CSEGSEL, 15, smb15, MFSEL3, 8, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(21, smb4c, I2CSEGSEL, 15, smb15, MFSEL3, 8, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(22, smb4d, I2CSEGSEL, 16, smb14, MFSEL3, 7, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(23, smb4d, I2CSEGSEL, 16, smb14, MFSEL3, 7, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(24, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(25, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(24, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(25, ioxh, MFSEL3, 18, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(26, smb5, MFSEL1, 2, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(27, smb5, MFSEL1, 2, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(28, smb4, MFSEL1, 1, none, NONE, 0, none, NONE, 0, 0),
+@@ -965,12 +964,12 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(39, smb3b, I2CSEGSEL, 11, none, NONE, 0, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(40, smb3b, I2CSEGSEL, 11, none, NONE, 0, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(41, bmcuart0a, MFSEL1, 9, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(42, bmcuart0a, MFSEL1, 9, none, NONE, 0, none, NONE, 0, DS(2, 4) | GPO),
++ NPCM7XX_PINCFG(42, bmcuart0a, MFSEL1, 9, none, NONE, 0, none, NONE, 0, DSTR(2, 4) | GPO),
+ NPCM7XX_PINCFG(43, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, bmcuart1, MFSEL3, 24, 0),
+ NPCM7XX_PINCFG(44, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, bmcuart1, MFSEL3, 24, 0),
+ NPCM7XX_PINCFG(45, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(46, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, none, NONE, 0, DS(2, 8)),
+- NPCM7XX_PINCFG(47, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, none, NONE, 0, DS(2, 8)),
++ NPCM7XX_PINCFG(46, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, none, NONE, 0, DSTR(2, 8)),
++ NPCM7XX_PINCFG(47, uart1, MFSEL1, 10, jtag2, MFSEL4, 0, none, NONE, 0, DSTR(2, 8)),
+ NPCM7XX_PINCFG(48, uart2, MFSEL1, 11, bmcuart0b, MFSEL4, 1, none, NONE, 0, GPO),
+ NPCM7XX_PINCFG(49, uart2, MFSEL1, 11, bmcuart0b, MFSEL4, 1, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(50, uart2, MFSEL1, 11, none, NONE, 0, none, NONE, 0, 0),
+@@ -980,8 +979,8 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(54, uart2, MFSEL1, 11, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(55, uart2, MFSEL1, 11, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(56, r1err, MFSEL1, 12, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(57, r1md, MFSEL1, 13, none, NONE, 0, none, NONE, 0, DS(2, 4)),
+- NPCM7XX_PINCFG(58, r1md, MFSEL1, 13, none, NONE, 0, none, NONE, 0, DS(2, 4)),
++ NPCM7XX_PINCFG(57, r1md, MFSEL1, 13, none, NONE, 0, none, NONE, 0, DSTR(2, 4)),
++ NPCM7XX_PINCFG(58, r1md, MFSEL1, 13, none, NONE, 0, none, NONE, 0, DSTR(2, 4)),
+ NPCM7XX_PINCFG(59, smb3d, I2CSEGSEL, 13, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(60, smb3d, I2CSEGSEL, 13, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(61, uart1, MFSEL1, 10, none, NONE, 0, none, NONE, 0, GPO),
+@@ -1004,19 +1003,19 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(77, fanin13, MFSEL2, 13, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(78, fanin14, MFSEL2, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(79, fanin15, MFSEL2, 15, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(80, pwm0, MFSEL2, 16, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(81, pwm1, MFSEL2, 17, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(82, pwm2, MFSEL2, 18, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(83, pwm3, MFSEL2, 19, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(84, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(85, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(86, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(80, pwm0, MFSEL2, 16, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(81, pwm1, MFSEL2, 17, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(82, pwm2, MFSEL2, 18, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(83, pwm3, MFSEL2, 19, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(84, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(85, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(86, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(87, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(88, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(89, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(90, r2err, MFSEL1, 15, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(91, r2md, MFSEL1, 16, none, NONE, 0, none, NONE, 0, DS(2, 4)),
+- NPCM7XX_PINCFG(92, r2md, MFSEL1, 16, none, NONE, 0, none, NONE, 0, DS(2, 4)),
++ NPCM7XX_PINCFG(91, r2md, MFSEL1, 16, none, NONE, 0, none, NONE, 0, DSTR(2, 4)),
++ NPCM7XX_PINCFG(92, r2md, MFSEL1, 16, none, NONE, 0, none, NONE, 0, DSTR(2, 4)),
+ NPCM7XX_PINCFG(93, ga20kbc, MFSEL1, 17, smb5d, I2CSEGSEL, 21, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(94, ga20kbc, MFSEL1, 17, smb5d, I2CSEGSEL, 21, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(95, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, 0),
+@@ -1062,34 +1061,34 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(133, smb10, MFSEL4, 13, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(134, smb11, MFSEL4, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(135, smb11, MFSEL4, 14, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(136, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(137, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(138, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(139, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(140, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(136, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(137, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(138, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(139, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(140, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(141, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(142, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(142, sd1, MFSEL3, 12, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(143, sd1, MFSEL3, 12, sd1pwr, MFSEL4, 5, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(144, pwm4, MFSEL2, 20, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(145, pwm5, MFSEL2, 21, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(146, pwm6, MFSEL2, 22, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(147, pwm7, MFSEL2, 23, none, NONE, 0, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(148, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(149, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(150, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(151, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(152, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(144, pwm4, MFSEL2, 20, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(145, pwm5, MFSEL2, 21, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(146, pwm6, MFSEL2, 22, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(147, pwm7, MFSEL2, 23, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(148, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(149, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(150, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(151, mmc8, MFSEL3, 11, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(152, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(153, mmcwp, FLOCKR1, 24, none, NONE, 0, none, NONE, 0, 0), /* Z1/A1 */
+- NPCM7XX_PINCFG(154, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(154, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(155, mmccd, MFSEL3, 25, mmcrst, MFSEL4, 6, none, NONE, 0, 0), /* Z1/A1 */
+- NPCM7XX_PINCFG(156, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(157, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(158, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(159, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+-
+- NPCM7XX_PINCFG(160, clkout, MFSEL1, 21, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(161, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, DS(8, 12)),
+- NPCM7XX_PINCFG(162, serirq, NONE, 0, gpio, MFSEL1, 31, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(156, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(157, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(158, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(159, mmc, MFSEL3, 10, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++
++ NPCM7XX_PINCFG(160, clkout, MFSEL1, 21, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(161, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, DSTR(8, 12)),
++ NPCM7XX_PINCFG(162, serirq, NONE, 0, gpio, MFSEL1, 31, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(163, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, 0),
+ NPCM7XX_PINCFG(164, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, SLEWLPC),
+ NPCM7XX_PINCFG(165, lpc, NONE, 0, espi, MFSEL4, 8, gpio, MFSEL1, 26, SLEWLPC),
+@@ -1102,25 +1101,25 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(172, smb6, MFSEL3, 1, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(173, smb7, MFSEL3, 2, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(174, smb7, MFSEL3, 2, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(175, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(176, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(177, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(178, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(179, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(180, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
++ NPCM7XX_PINCFG(175, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(176, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(177, pspi1, MFSEL3, 4, faninx, MFSEL3, 3, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(178, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(179, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(180, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
+ NPCM7XX_PINCFG(181, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(182, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(183, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(184, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW | GPO),
+- NPCM7XX_PINCFG(185, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW | GPO),
+- NPCM7XX_PINCFG(186, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(187, spi3cs1, MFSEL4, 17, none, NONE, 0, none, NONE, 0, DS(8, 12)),
+- NPCM7XX_PINCFG(188, spi3quad, MFSEL4, 20, spi3cs2, MFSEL4, 18, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(189, spi3quad, MFSEL4, 20, spi3cs3, MFSEL4, 19, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(190, gpio, FLOCKR1, 20, nprd_smi, NONE, 0, none, NONE, 0, DS(2, 4)),
+- NPCM7XX_PINCFG(191, none, NONE, 0, none, NONE, 0, none, NONE, 0, DS(8, 12)), /* XX */
+-
+- NPCM7XX_PINCFG(192, none, NONE, 0, none, NONE, 0, none, NONE, 0, DS(8, 12)), /* XX */
++ NPCM7XX_PINCFG(183, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(184, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW | GPO),
++ NPCM7XX_PINCFG(185, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW | GPO),
++ NPCM7XX_PINCFG(186, spi3, MFSEL4, 16, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(187, spi3cs1, MFSEL4, 17, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
++ NPCM7XX_PINCFG(188, spi3quad, MFSEL4, 20, spi3cs2, MFSEL4, 18, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(189, spi3quad, MFSEL4, 20, spi3cs3, MFSEL4, 19, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(190, gpio, FLOCKR1, 20, nprd_smi, NONE, 0, none, NONE, 0, DSTR(2, 4)),
++ NPCM7XX_PINCFG(191, none, NONE, 0, none, NONE, 0, none, NONE, 0, DSTR(8, 12)), /* XX */
++
++ NPCM7XX_PINCFG(192, none, NONE, 0, none, NONE, 0, none, NONE, 0, DSTR(8, 12)), /* XX */
+ NPCM7XX_PINCFG(193, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(194, smb0b, I2CSEGSEL, 0, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(195, smb0b, I2CSEGSEL, 0, none, NONE, 0, none, NONE, 0, 0),
+@@ -1131,11 +1130,11 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(200, r2, MFSEL1, 14, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(201, r1, MFSEL3, 9, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(202, smb0c, I2CSEGSEL, 1, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(203, faninx, MFSEL3, 3, none, NONE, 0, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(203, faninx, MFSEL3, 3, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(204, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, SLEW),
+ NPCM7XX_PINCFG(205, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, SLEW),
+- NPCM7XX_PINCFG(206, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, DS(4, 8)),
+- NPCM7XX_PINCFG(207, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, DS(4, 8)),
++ NPCM7XX_PINCFG(206, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, DSTR(4, 8)),
++ NPCM7XX_PINCFG(207, ddc, NONE, 0, gpio, MFSEL3, 22, none, NONE, 0, DSTR(4, 8)),
+ NPCM7XX_PINCFG(208, rg2, MFSEL4, 24, ddr, MFSEL3, 26, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(209, rg2, MFSEL4, 24, ddr, MFSEL3, 26, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(210, rg2, MFSEL4, 24, ddr, MFSEL3, 26, none, NONE, 0, 0),
+@@ -1147,20 +1146,20 @@ static const struct npcm7xx_pincfg pincfg[] = {
+ NPCM7XX_PINCFG(216, rg2mdio, MFSEL4, 23, ddr, MFSEL3, 26, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(217, rg2mdio, MFSEL4, 23, ddr, MFSEL3, 26, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(218, wdog1, MFSEL3, 19, none, NONE, 0, none, NONE, 0, 0),
+- NPCM7XX_PINCFG(219, wdog2, MFSEL3, 20, none, NONE, 0, none, NONE, 0, DS(4, 8)),
++ NPCM7XX_PINCFG(219, wdog2, MFSEL3, 20, none, NONE, 0, none, NONE, 0, DSTR(4, 8)),
+ NPCM7XX_PINCFG(220, smb12, MFSEL3, 5, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(221, smb12, MFSEL3, 5, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(222, smb13, MFSEL3, 6, none, NONE, 0, none, NONE, 0, 0),
+ NPCM7XX_PINCFG(223, smb13, MFSEL3, 6, none, NONE, 0, none, NONE, 0, 0),
+
+ NPCM7XX_PINCFG(224, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, SLEW),
+- NPCM7XX_PINCFG(225, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW | GPO),
+- NPCM7XX_PINCFG(226, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW | GPO),
+- NPCM7XX_PINCFG(227, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(228, spixcs1, MFSEL4, 28, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(229, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(230, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DS(8, 12) | SLEW),
+- NPCM7XX_PINCFG(231, clkreq, MFSEL4, 9, none, NONE, 0, none, NONE, 0, DS(8, 12)),
++ NPCM7XX_PINCFG(225, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW | GPO),
++ NPCM7XX_PINCFG(226, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW | GPO),
++ NPCM7XX_PINCFG(227, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(228, spixcs1, MFSEL4, 28, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(229, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(230, spix, MFSEL4, 27, none, NONE, 0, none, NONE, 0, DSTR(8, 12) | SLEW),
++ NPCM7XX_PINCFG(231, clkreq, MFSEL4, 9, none, NONE, 0, none, NONE, 0, DSTR(8, 12)),
+ NPCM7XX_PINCFG(253, none, NONE, 0, none, NONE, 0, none, NONE, 0, GPI), /* SDHC1 power */
+ NPCM7XX_PINCFG(254, none, NONE, 0, none, NONE, 0, none, NONE, 0, GPI), /* SDHC2 power */
+ NPCM7XX_PINCFG(255, none, NONE, 0, none, NONE, 0, none, NONE, 0, GPI), /* DACOSEL */
+@@ -1561,7 +1560,7 @@ static int npcm7xx_get_groups_count(struct pinctrl_dev *pctldev)
+ {
+ struct npcm7xx_pinctrl *npcm = pinctrl_dev_get_drvdata(pctldev);
+
+- dev_dbg(npcm->dev, "group size: %d\n", ARRAY_SIZE(npcm7xx_groups));
++ dev_dbg(npcm->dev, "group size: %zu\n", ARRAY_SIZE(npcm7xx_groups));
+ return ARRAY_SIZE(npcm7xx_groups);
+ }
+
+diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c
+index 22e8d4c4040e1..b1db28007986e 100644
+--- a/drivers/pinctrl/pinconf-generic.c
++++ b/drivers/pinctrl/pinconf-generic.c
+@@ -30,10 +30,10 @@ static const struct pin_config_item conf_items[] = {
+ PCONFDUMP(PIN_CONFIG_BIAS_BUS_HOLD, "input bias bus hold", NULL, false),
+ PCONFDUMP(PIN_CONFIG_BIAS_DISABLE, "input bias disabled", NULL, false),
+ PCONFDUMP(PIN_CONFIG_BIAS_HIGH_IMPEDANCE, "input bias high impedance", NULL, false),
+- PCONFDUMP(PIN_CONFIG_BIAS_PULL_DOWN, "input bias pull down", NULL, false),
++ PCONFDUMP(PIN_CONFIG_BIAS_PULL_DOWN, "input bias pull down", "ohms", true),
+ PCONFDUMP(PIN_CONFIG_BIAS_PULL_PIN_DEFAULT,
+- "input bias pull to pin specific state", NULL, false),
+- PCONFDUMP(PIN_CONFIG_BIAS_PULL_UP, "input bias pull up", NULL, false),
++ "input bias pull to pin specific state", "ohms", true),
++ PCONFDUMP(PIN_CONFIG_BIAS_PULL_UP, "input bias pull up", "ohms", true),
+ PCONFDUMP(PIN_CONFIG_DRIVE_OPEN_DRAIN, "output drive open drain", NULL, false),
+ PCONFDUMP(PIN_CONFIG_DRIVE_OPEN_SOURCE, "output drive open source", NULL, false),
+ PCONFDUMP(PIN_CONFIG_DRIVE_PUSH_PULL, "output drive push pull", NULL, false),
+diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
+index bae9d429b813e..ecab9064a8458 100644
+--- a/drivers/pinctrl/pinctrl-amd.c
++++ b/drivers/pinctrl/pinctrl-amd.c
+@@ -598,14 +598,14 @@ static struct irq_chip amd_gpio_irqchip = {
+
+ #define PIN_IRQ_PENDING (BIT(INTERRUPT_STS_OFF) | BIT(WAKE_STS_OFF))
+
+-static irqreturn_t amd_gpio_irq_handler(int irq, void *dev_id)
++static bool do_amd_gpio_irq_handler(int irq, void *dev_id)
+ {
+ struct amd_gpio *gpio_dev = dev_id;
+ struct gpio_chip *gc = &gpio_dev->gc;
+- irqreturn_t ret = IRQ_NONE;
+ unsigned int i, irqnr;
+ unsigned long flags;
+ u32 __iomem *regs;
++ bool ret = false;
+ u32 regval;
+ u64 status, mask;
+
+@@ -627,6 +627,14 @@ static irqreturn_t amd_gpio_irq_handler(int irq, void *dev_id)
+ /* Each status bit covers four pins */
+ for (i = 0; i < 4; i++) {
+ regval = readl(regs + i);
++ /* caused wake on resume context for shared IRQ */
++ if (irq < 0 && (regval & BIT(WAKE_STS_OFF))) {
++ dev_dbg(&gpio_dev->pdev->dev,
++ "Waking due to GPIO %d: 0x%x",
++ irqnr + i, regval);
++ return true;
++ }
++
+ if (!(regval & PIN_IRQ_PENDING) ||
+ !(regval & BIT(INTERRUPT_MASK_OFF)))
+ continue;
+@@ -650,9 +658,12 @@ static irqreturn_t amd_gpio_irq_handler(int irq, void *dev_id)
+ }
+ writel(regval, regs + i);
+ raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
+- ret = IRQ_HANDLED;
++ ret = true;
+ }
+ }
++ /* did not cause wake on resume context for shared IRQ */
++ if (irq < 0)
++ return false;
+
+ /* Signal EOI to the GPIO unit */
+ raw_spin_lock_irqsave(&gpio_dev->lock, flags);
+@@ -664,6 +675,16 @@ static irqreturn_t amd_gpio_irq_handler(int irq, void *dev_id)
+ return ret;
+ }
+
++static irqreturn_t amd_gpio_irq_handler(int irq, void *dev_id)
++{
++ return IRQ_RETVAL(do_amd_gpio_irq_handler(irq, dev_id));
++}
++
++static bool __maybe_unused amd_gpio_check_wake(void *dev_id)
++{
++ return do_amd_gpio_irq_handler(-1, dev_id);
++}
++
+ static int amd_get_groups_count(struct pinctrl_dev *pctldev)
+ {
+ struct amd_gpio *gpio_dev = pinctrl_dev_get_drvdata(pctldev);
+@@ -1033,6 +1054,7 @@ static int amd_gpio_probe(struct platform_device *pdev)
+ goto out2;
+
+ platform_set_drvdata(pdev, gpio_dev);
++ acpi_register_wakeup_handler(gpio_dev->irq, amd_gpio_check_wake, gpio_dev);
+
+ dev_dbg(&pdev->dev, "amd gpio driver loaded\n");
+ return ret;
+@@ -1050,6 +1072,7 @@ static int amd_gpio_remove(struct platform_device *pdev)
+ gpio_dev = platform_get_drvdata(pdev);
+
+ gpiochip_remove(&gpio_dev->gc);
++ acpi_unregister_wakeup_handler(amd_gpio_check_wake, gpio_dev);
+
+ return 0;
+ }
+diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c
+index fb713f9c53d0e..3f0143087cc77 100644
+--- a/drivers/pinctrl/pinctrl-equilibrium.c
++++ b/drivers/pinctrl/pinctrl-equilibrium.c
+@@ -675,6 +675,11 @@ static int eqbr_build_functions(struct eqbr_pinctrl_drv_data *drvdata)
+ return ret;
+
+ for (i = 0; i < nr_funcs; i++) {
++
++ /* Ignore the same function with multiple groups */
++ if (funcs[i].name == NULL)
++ continue;
++
+ ret = pinmux_generic_add_function(drvdata->pctl_dev,
+ funcs[i].name,
+ funcs[i].groups,
+@@ -815,7 +820,7 @@ static int pinctrl_reg(struct eqbr_pinctrl_drv_data *drvdata)
+
+ ret = eqbr_build_functions(drvdata);
+ if (ret) {
+- dev_err(dev, "Failed to build groups\n");
++ dev_err(dev, "Failed to build functions\n");
+ return ret;
+ }
+
+diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c
+index 2712f51eb2381..fa6becca17889 100644
+--- a/drivers/pinctrl/pinctrl-ingenic.c
++++ b/drivers/pinctrl/pinctrl-ingenic.c
+@@ -119,6 +119,8 @@ struct ingenic_chip_info {
+ unsigned int num_functions;
+
+ const u32 *pull_ups, *pull_downs;
++
++ const struct regmap_access_table *access_table;
+ };
+
+ struct ingenic_pinctrl {
+@@ -2179,6 +2181,17 @@ static const struct function_desc x1000_functions[] = {
+ { "mac", x1000_mac_groups, ARRAY_SIZE(x1000_mac_groups), },
+ };
+
++static const struct regmap_range x1000_access_ranges[] = {
++ regmap_reg_range(0x000, 0x400 - 4),
++ regmap_reg_range(0x700, 0x800 - 4),
++};
++
++/* shared with X1500 */
++static const struct regmap_access_table x1000_access_table = {
++ .yes_ranges = x1000_access_ranges,
++ .n_yes_ranges = ARRAY_SIZE(x1000_access_ranges),
++};
++
+ static const struct ingenic_chip_info x1000_chip_info = {
+ .num_chips = 4,
+ .reg_offset = 0x100,
+@@ -2189,6 +2202,7 @@ static const struct ingenic_chip_info x1000_chip_info = {
+ .num_functions = ARRAY_SIZE(x1000_functions),
+ .pull_ups = x1000_pull_ups,
+ .pull_downs = x1000_pull_downs,
++ .access_table = &x1000_access_table,
+ };
+
+ static int x1500_uart0_data_pins[] = { 0x4a, 0x4b, };
+@@ -2300,6 +2314,7 @@ static const struct ingenic_chip_info x1500_chip_info = {
+ .num_functions = ARRAY_SIZE(x1500_functions),
+ .pull_ups = x1000_pull_ups,
+ .pull_downs = x1000_pull_downs,
++ .access_table = &x1000_access_table,
+ };
+
+ static const u32 x1830_pull_ups[4] = {
+@@ -2506,6 +2521,16 @@ static const struct function_desc x1830_functions[] = {
+ { "mac", x1830_mac_groups, ARRAY_SIZE(x1830_mac_groups), },
+ };
+
++static const struct regmap_range x1830_access_ranges[] = {
++ regmap_reg_range(0x0000, 0x4000 - 4),
++ regmap_reg_range(0x7000, 0x8000 - 4),
++};
++
++static const struct regmap_access_table x1830_access_table = {
++ .yes_ranges = x1830_access_ranges,
++ .n_yes_ranges = ARRAY_SIZE(x1830_access_ranges),
++};
++
+ static const struct ingenic_chip_info x1830_chip_info = {
+ .num_chips = 4,
+ .reg_offset = 0x1000,
+@@ -2516,6 +2541,7 @@ static const struct ingenic_chip_info x1830_chip_info = {
+ .num_functions = ARRAY_SIZE(x1830_functions),
+ .pull_ups = x1830_pull_ups,
+ .pull_downs = x1830_pull_downs,
++ .access_table = &x1830_access_table,
+ };
+
+ static const u32 x2000_pull_ups[5] = {
+@@ -2969,6 +2995,17 @@ static const struct function_desc x2000_functions[] = {
+ { "otg", x2000_otg_groups, ARRAY_SIZE(x2000_otg_groups), },
+ };
+
++static const struct regmap_range x2000_access_ranges[] = {
++ regmap_reg_range(0x000, 0x500 - 4),
++ regmap_reg_range(0x700, 0x800 - 4),
++};
++
++/* shared with X2100 */
++static const struct regmap_access_table x2000_access_table = {
++ .yes_ranges = x2000_access_ranges,
++ .n_yes_ranges = ARRAY_SIZE(x2000_access_ranges),
++};
++
+ static const struct ingenic_chip_info x2000_chip_info = {
+ .num_chips = 5,
+ .reg_offset = 0x100,
+@@ -2979,6 +3016,7 @@ static const struct ingenic_chip_info x2000_chip_info = {
+ .num_functions = ARRAY_SIZE(x2000_functions),
+ .pull_ups = x2000_pull_ups,
+ .pull_downs = x2000_pull_downs,
++ .access_table = &x2000_access_table,
+ };
+
+ static const u32 x2100_pull_ups[5] = {
+@@ -3189,6 +3227,7 @@ static const struct ingenic_chip_info x2100_chip_info = {
+ .num_functions = ARRAY_SIZE(x2100_functions),
+ .pull_ups = x2100_pull_ups,
+ .pull_downs = x2100_pull_downs,
++ .access_table = &x2000_access_table,
+ };
+
+ static u32 ingenic_gpio_read_reg(struct ingenic_gpio_chip *jzgc, u8 reg)
+@@ -4168,7 +4207,12 @@ static int __init ingenic_pinctrl_probe(struct platform_device *pdev)
+ return PTR_ERR(base);
+
+ regmap_config = ingenic_pinctrl_regmap_config;
+- regmap_config.max_register = chip_info->num_chips * chip_info->reg_offset;
++ if (chip_info->access_table) {
++ regmap_config.rd_table = chip_info->access_table;
++ regmap_config.wr_table = chip_info->access_table;
++ } else {
++ regmap_config.max_register = chip_info->num_chips * chip_info->reg_offset - 4;
++ }
+
+ jzpc->map = devm_regmap_init_mmio(dev, base, &regmap_config);
+ if (IS_ERR(jzpc->map)) {
+diff --git a/drivers/pinctrl/pinctrl-k210.c b/drivers/pinctrl/pinctrl-k210.c
+index 49e32684dbb25..ecab6bf63dc6d 100644
+--- a/drivers/pinctrl/pinctrl-k210.c
++++ b/drivers/pinctrl/pinctrl-k210.c
+@@ -482,7 +482,7 @@ static int k210_pinconf_get_drive(unsigned int max_strength_ua)
+ {
+ int i;
+
+- for (i = K210_PC_DRIVE_MAX; i; i--) {
++ for (i = K210_PC_DRIVE_MAX; i >= 0; i--) {
+ if (k210_pinconf_drive_strength[i] <= max_strength_ua)
+ return i;
+ }
+@@ -527,7 +527,7 @@ static int k210_pinconf_set_param(struct pinctrl_dev *pctldev,
+ case PIN_CONFIG_BIAS_PULL_UP:
+ if (!arg)
+ return -EINVAL;
+- val |= K210_PC_PD;
++ val |= K210_PC_PU;
+ break;
+ case PIN_CONFIG_DRIVE_STRENGTH:
+ arg *= 1000;
+diff --git a/drivers/pinctrl/pinctrl-microchip-sgpio.c b/drivers/pinctrl/pinctrl-microchip-sgpio.c
+index 072bccdea2a5d..dfa374195694d 100644
+--- a/drivers/pinctrl/pinctrl-microchip-sgpio.c
++++ b/drivers/pinctrl/pinctrl-microchip-sgpio.c
+@@ -17,6 +17,8 @@
+ #include <linux/pinctrl/pinmux.h>
+ #include <linux/platform_device.h>
+ #include <linux/property.h>
++#include <linux/reset.h>
++#include <linux/spinlock.h>
+
+ #include "core.h"
+ #include "pinconf.h"
+@@ -114,6 +116,7 @@ struct sgpio_priv {
+ u32 clock;
+ u32 __iomem *regs;
+ const struct sgpio_properties *properties;
++ spinlock_t lock;
+ };
+
+ struct sgpio_port_addr {
+@@ -215,6 +218,7 @@ static void sgpio_output_set(struct sgpio_priv *priv,
+ int value)
+ {
+ unsigned int bit = SGPIO_SRC_BITS * addr->bit;
++ unsigned long flags;
+ u32 clr, set;
+
+ switch (priv->properties->arch) {
+@@ -233,7 +237,10 @@ static void sgpio_output_set(struct sgpio_priv *priv,
+ default:
+ return;
+ }
++
++ spin_lock_irqsave(&priv->lock, flags);
+ sgpio_clrsetbits(priv, REG_PORT_CONFIG, addr->port, clr, set);
++ spin_unlock_irqrestore(&priv->lock, flags);
+ }
+
+ static int sgpio_output_get(struct sgpio_priv *priv,
+@@ -561,10 +568,13 @@ static void microchip_sgpio_irq_settype(struct irq_data *data,
+ struct sgpio_bank *bank = gpiochip_get_data(chip);
+ unsigned int gpio = irqd_to_hwirq(data);
+ struct sgpio_port_addr addr;
++ unsigned long flags;
+ u32 ena;
+
+ sgpio_pin_to_addr(bank->priv, gpio, &addr);
+
++ spin_lock_irqsave(&bank->priv->lock, flags);
++
+ /* Disable interrupt while changing type */
+ ena = sgpio_readl(bank->priv, REG_INT_ENABLE, addr.bit);
+ sgpio_writel(bank->priv, ena & ~BIT(addr.port), REG_INT_ENABLE, addr.bit);
+@@ -581,6 +591,8 @@ static void microchip_sgpio_irq_settype(struct irq_data *data,
+
+ /* Possibly re-enable interrupts */
+ sgpio_writel(bank->priv, ena, REG_INT_ENABLE, addr.bit);
++
++ spin_unlock_irqrestore(&bank->priv->lock, flags);
+ }
+
+ static void microchip_sgpio_irq_setreg(struct irq_data *data,
+@@ -591,13 +603,16 @@ static void microchip_sgpio_irq_setreg(struct irq_data *data,
+ struct sgpio_bank *bank = gpiochip_get_data(chip);
+ unsigned int gpio = irqd_to_hwirq(data);
+ struct sgpio_port_addr addr;
++ unsigned long flags;
+
+ sgpio_pin_to_addr(bank->priv, gpio, &addr);
+
++ spin_lock_irqsave(&bank->priv->lock, flags);
+ if (clear)
+ sgpio_clrsetbits(bank->priv, reg, addr.bit, BIT(addr.port), 0);
+ else
+ sgpio_clrsetbits(bank->priv, reg, addr.bit, 0, BIT(addr.port));
++ spin_unlock_irqrestore(&bank->priv->lock, flags);
+ }
+
+ static void microchip_sgpio_irq_mask(struct irq_data *data)
+@@ -803,6 +818,7 @@ static int microchip_sgpio_probe(struct platform_device *pdev)
+ int div_clock = 0, ret, port, i, nbanks;
+ struct device *dev = &pdev->dev;
+ struct fwnode_handle *fwnode;
++ struct reset_control *reset;
+ struct sgpio_priv *priv;
+ struct clk *clk;
+ u32 val;
+@@ -812,6 +828,12 @@ static int microchip_sgpio_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ priv->dev = dev;
++ spin_lock_init(&priv->lock);
++
++ reset = devm_reset_control_get_optional_shared(&pdev->dev, "switch");
++ if (IS_ERR(reset))
++ return dev_err_probe(dev, PTR_ERR(reset), "Failed to get reset\n");
++ reset_control_reset(reset);
+
+ clk = devm_clk_get(dev, NULL);
+ if (IS_ERR(clk))
+diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c
+index 8d271c6b0ca41..5de691c630b4f 100644
+--- a/drivers/pinctrl/pinctrl-pistachio.c
++++ b/drivers/pinctrl/pinctrl-pistachio.c
+@@ -1374,10 +1374,10 @@ static int pistachio_gpio_register(struct pistachio_pinctrl *pctl)
+ }
+
+ irq = irq_of_parse_and_map(child, 0);
+- if (irq < 0) {
+- dev_err(pctl->dev, "No IRQ for bank %u: %d\n", i, irq);
++ if (!irq) {
++ dev_err(pctl->dev, "No IRQ for bank %u\n", i);
+ of_node_put(child);
+- ret = irq;
++ ret = -EINVAL;
+ goto err;
+ }
+
+diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
+index 5ce260f152ce5..543a4991cf700 100644
+--- a/drivers/pinctrl/pinctrl-rockchip.c
++++ b/drivers/pinctrl/pinctrl-rockchip.c
+@@ -455,95 +455,110 @@ static struct rockchip_mux_recalced_data rk3128_mux_recalced_data[] = {
+
+ static struct rockchip_mux_recalced_data rk3308_mux_recalced_data[] = {
+ {
++ /* gpio1b6_sel */
+ .num = 1,
+ .pin = 14,
+ .reg = 0x28,
+ .bit = 12,
+ .mask = 0xf
+ }, {
++ /* gpio1b7_sel */
+ .num = 1,
+ .pin = 15,
+ .reg = 0x2c,
+ .bit = 0,
+ .mask = 0x3
+ }, {
++ /* gpio1c2_sel */
+ .num = 1,
+ .pin = 18,
+ .reg = 0x30,
+ .bit = 4,
+ .mask = 0xf
+ }, {
++ /* gpio1c3_sel */
+ .num = 1,
+ .pin = 19,
+ .reg = 0x30,
+ .bit = 8,
+ .mask = 0xf
+ }, {
++ /* gpio1c4_sel */
+ .num = 1,
+ .pin = 20,
+ .reg = 0x30,
+ .bit = 12,
+ .mask = 0xf
+ }, {
++ /* gpio1c5_sel */
+ .num = 1,
+ .pin = 21,
+ .reg = 0x34,
+ .bit = 0,
+ .mask = 0xf
+ }, {
++ /* gpio1c6_sel */
+ .num = 1,
+ .pin = 22,
+ .reg = 0x34,
+ .bit = 4,
+ .mask = 0xf
+ }, {
++ /* gpio1c7_sel */
+ .num = 1,
+ .pin = 23,
+ .reg = 0x34,
+ .bit = 8,
+ .mask = 0xf
+ }, {
++ /* gpio3b4_sel */
+ .num = 3,
+ .pin = 12,
+ .reg = 0x68,
+ .bit = 8,
+ .mask = 0xf
+ }, {
++ /* gpio3b5_sel */
+ .num = 3,
+ .pin = 13,
+ .reg = 0x68,
+ .bit = 12,
+ .mask = 0xf
+ }, {
++ /* gpio2a2_sel */
+ .num = 2,
+ .pin = 2,
+- .reg = 0x608,
+- .bit = 0,
+- .mask = 0x7
++ .reg = 0x40,
++ .bit = 4,
++ .mask = 0x3
+ }, {
++ /* gpio2a3_sel */
+ .num = 2,
+ .pin = 3,
+- .reg = 0x608,
+- .bit = 4,
+- .mask = 0x7
++ .reg = 0x40,
++ .bit = 6,
++ .mask = 0x3
+ }, {
++ /* gpio2c0_sel */
+ .num = 2,
+ .pin = 16,
+- .reg = 0x610,
+- .bit = 8,
+- .mask = 0x7
++ .reg = 0x50,
++ .bit = 0,
++ .mask = 0x3
+ }, {
++ /* gpio3b2_sel */
+ .num = 3,
+ .pin = 10,
+- .reg = 0x610,
+- .bit = 0,
+- .mask = 0x7
++ .reg = 0x68,
++ .bit = 4,
++ .mask = 0x3
+ }, {
++ /* gpio3b3_sel */
+ .num = 3,
+ .pin = 11,
+- .reg = 0x610,
+- .bit = 4,
+- .mask = 0x7
++ .reg = 0x68,
++ .bit = 6,
++ .mask = 0x3
+ },
+ };
+
+@@ -2702,6 +2717,7 @@ static int rockchip_pinctrl_probe(struct platform_device *pdev)
+ node = of_parse_phandle(np, "rockchip,grf", 0);
+ if (node) {
+ info->regmap_base = syscon_node_to_regmap(node);
++ of_node_put(node);
+ if (IS_ERR(info->regmap_base))
+ return PTR_ERR(info->regmap_base);
+ } else {
+@@ -2738,6 +2754,7 @@ static int rockchip_pinctrl_probe(struct platform_device *pdev)
+ node = of_parse_phandle(np, "rockchip,pmu", 0);
+ if (node) {
+ info->regmap_pmu = syscon_node_to_regmap(node);
++ of_node_put(node);
+ if (IS_ERR(info->regmap_pmu))
+ return PTR_ERR(info->regmap_pmu);
+ }
+@@ -2748,7 +2765,7 @@ static int rockchip_pinctrl_probe(struct platform_device *pdev)
+
+ platform_set_drvdata(pdev, info);
+
+- ret = of_platform_populate(np, rockchip_bank_match, NULL, NULL);
++ ret = of_platform_populate(np, NULL, NULL, &pdev->dev);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register gpio device\n");
+ return ret;
+diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
+index 5ff4207df66e1..f1b5176a5085b 100644
+--- a/drivers/pinctrl/qcom/Kconfig
++++ b/drivers/pinctrl/qcom/Kconfig
+@@ -189,6 +189,7 @@ config PINCTRL_QCOM_SPMI_PMIC
+ select PINMUX
+ select PINCONF
+ select GENERIC_PINCONF
++ select GPIOLIB
+ select GPIOLIB_IRQCHIP
+ select IRQ_DOMAIN_HIERARCHY
+ help
+@@ -203,6 +204,7 @@ config PINCTRL_QCOM_SSBI_PMIC
+ select PINMUX
+ select PINCONF
+ select GENERIC_PINCONF
++ select GPIOLIB
+ select GPIOLIB_IRQCHIP
+ select IRQ_DOMAIN_HIERARCHY
+ help
+diff --git a/drivers/pinctrl/qcom/pinctrl-sdm845.c b/drivers/pinctrl/qcom/pinctrl-sdm845.c
+index c51793f6546f1..fdfd7b8f3a76d 100644
+--- a/drivers/pinctrl/qcom/pinctrl-sdm845.c
++++ b/drivers/pinctrl/qcom/pinctrl-sdm845.c
+@@ -1310,6 +1310,7 @@ static const struct msm_pinctrl_soc_data sdm845_pinctrl = {
+ .ngpios = 151,
+ .wakeirq_map = sdm845_pdc_map,
+ .nwakeirq_map = ARRAY_SIZE(sdm845_pdc_map),
++ .wakeirq_dual_edge_errata = true,
+ };
+
+ static const struct msm_pinctrl_soc_data sdm845_acpi_pinctrl = {
+diff --git a/drivers/pinctrl/qcom/pinctrl-sm8350.c b/drivers/pinctrl/qcom/pinctrl-sm8350.c
+index 4d8f8636c2b39..1c042d39380c6 100644
+--- a/drivers/pinctrl/qcom/pinctrl-sm8350.c
++++ b/drivers/pinctrl/qcom/pinctrl-sm8350.c
+@@ -1597,10 +1597,10 @@ static const struct msm_pingroup sm8350_groups[] = {
+ [200] = PINGROUP(200, qdss_gpio, _, _, _, _, _, _, _, _),
+ [201] = PINGROUP(201, _, _, _, _, _, _, _, _, _),
+ [202] = PINGROUP(202, _, _, _, _, _, _, _, _, _),
+- [203] = UFS_RESET(ufs_reset, 0x1d8000),
+- [204] = SDC_PINGROUP(sdc2_clk, 0x1cf000, 14, 6),
+- [205] = SDC_PINGROUP(sdc2_cmd, 0x1cf000, 11, 3),
+- [206] = SDC_PINGROUP(sdc2_data, 0x1cf000, 9, 0),
++ [203] = UFS_RESET(ufs_reset, 0xd8000),
++ [204] = SDC_PINGROUP(sdc2_clk, 0xcf000, 14, 6),
++ [205] = SDC_PINGROUP(sdc2_cmd, 0xcf000, 11, 3),
++ [206] = SDC_PINGROUP(sdc2_data, 0xcf000, 9, 0),
+ };
+
+ static const struct msm_gpio_wakeirq_map sm8350_pdc_map[] = {
+diff --git a/drivers/pinctrl/ralink/pinctrl-mt7620.c b/drivers/pinctrl/ralink/pinctrl-mt7620.c
+index 425d55a2ee19f..6853b5b8b0fe7 100644
+--- a/drivers/pinctrl/ralink/pinctrl-mt7620.c
++++ b/drivers/pinctrl/ralink/pinctrl-mt7620.c
+@@ -1,5 +1,6 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+
++#include <asm/mach-ralink/ralink_regs.h>
+ #include <asm/mach-ralink/mt7620.h>
+ #include <linux/module.h>
+ #include <linux/platform_device.h>
+diff --git a/drivers/pinctrl/renesas/core.c b/drivers/pinctrl/renesas/core.c
+index f2ab02225837e..bc17f3131de5c 100644
+--- a/drivers/pinctrl/renesas/core.c
++++ b/drivers/pinctrl/renesas/core.c
+@@ -739,7 +739,7 @@ static int sh_pfc_suspend_init(struct sh_pfc *pfc) { return 0; }
+
+ #ifdef DEBUG
+ #define SH_PFC_MAX_REGS 300
+-#define SH_PFC_MAX_ENUMS 3000
++#define SH_PFC_MAX_ENUMS 5000
+
+ static unsigned int sh_pfc_errors __initdata = 0;
+ static unsigned int sh_pfc_warnings __initdata = 0;
+@@ -851,7 +851,8 @@ static void __init sh_pfc_check_cfg_reg(const char *drvname,
+ sh_pfc_check_reg(drvname, cfg_reg->reg);
+
+ if (cfg_reg->field_width) {
+- n = cfg_reg->reg_width / cfg_reg->field_width;
++ fw = cfg_reg->field_width;
++ n = (cfg_reg->reg_width / fw) << fw;
+ /* Skip field checks (done at build time) */
+ goto check_enum_ids;
+ }
+@@ -890,7 +891,7 @@ static void __init sh_pfc_check_drive_reg(const struct sh_pfc_soc_info *info,
+ if (!field->pin && !field->offset && !field->size)
+ continue;
+
+- mask = GENMASK(field->offset + field->size, field->offset);
++ mask = GENMASK(field->offset + field->size - 1, field->offset);
+ if (mask & seen)
+ sh_pfc_err("drive_reg 0x%x: field %u overlap\n",
+ drive->reg, i);
+diff --git a/drivers/pinctrl/renesas/pfc-r8a77470.c b/drivers/pinctrl/renesas/pfc-r8a77470.c
+index e6e5487691c16..cf7153d06a953 100644
+--- a/drivers/pinctrl/renesas/pfc-r8a77470.c
++++ b/drivers/pinctrl/renesas/pfc-r8a77470.c
+@@ -2140,7 +2140,7 @@ static const unsigned int vin0_clk_mux[] = {
+ VI0_CLK_MARK,
+ };
+ /* - VIN1 ------------------------------------------------------------------- */
+-static const union vin_data vin1_data_pins = {
++static const union vin_data12 vin1_data_pins = {
+ .data12 = {
+ RCAR_GP_PIN(3, 1), RCAR_GP_PIN(3, 2),
+ RCAR_GP_PIN(3, 3), RCAR_GP_PIN(3, 4),
+@@ -2150,7 +2150,7 @@ static const union vin_data vin1_data_pins = {
+ RCAR_GP_PIN(3, 15), RCAR_GP_PIN(3, 16),
+ },
+ };
+-static const union vin_data vin1_data_mux = {
++static const union vin_data12 vin1_data_mux = {
+ .data12 = {
+ VI1_DATA0_MARK, VI1_DATA1_MARK,
+ VI1_DATA2_MARK, VI1_DATA3_MARK,
+diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+index dbf2f521bb272..20b2af889ca96 100644
+--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
++++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+@@ -852,7 +852,7 @@ static const u32 rzg2l_gpio_configs[] = {
+ RZG2L_GPIO_PORT_PACK(2, 0x1e, RZG2L_MPXED_PIN_FUNCS),
+ RZG2L_GPIO_PORT_PACK(2, 0x1f, RZG2L_MPXED_PIN_FUNCS),
+ RZG2L_GPIO_PORT_PACK(2, 0x20, RZG2L_MPXED_PIN_FUNCS),
+- RZG2L_GPIO_PORT_PACK(3, 0x22, RZG2L_MPXED_PIN_FUNCS),
++ RZG2L_GPIO_PORT_PACK(3, 0x21, RZG2L_MPXED_PIN_FUNCS),
+ RZG2L_GPIO_PORT_PACK(2, 0x22, RZG2L_MPXED_PIN_FUNCS),
+ RZG2L_GPIO_PORT_PACK(2, 0x23, RZG2L_MPXED_PIN_FUNCS),
+ RZG2L_GPIO_PORT_PACK(3, 0x24, RZG2L_MPXED_ETH_PIN_FUNCS(PIN_CFG_IOLH_ETH0)),
+diff --git a/drivers/pinctrl/samsung/Kconfig b/drivers/pinctrl/samsung/Kconfig
+index dfd805e768624..7b0576f71376e 100644
+--- a/drivers/pinctrl/samsung/Kconfig
++++ b/drivers/pinctrl/samsung/Kconfig
+@@ -4,14 +4,13 @@
+ #
+ config PINCTRL_SAMSUNG
+ bool
+- depends on OF_GPIO
++ select GPIOLIB
+ select PINMUX
+ select PINCONF
+
+ config PINCTRL_EXYNOS
+ bool "Pinctrl common driver part for Samsung Exynos SoCs"
+- depends on OF_GPIO
+- depends on ARCH_EXYNOS || ARCH_S5PV210 || COMPILE_TEST
++ depends on ARCH_EXYNOS || ARCH_S5PV210 || (COMPILE_TEST && OF)
+ select PINCTRL_SAMSUNG
+ select PINCTRL_EXYNOS_ARM if ARM && (ARCH_EXYNOS || ARCH_S5PV210)
+ select PINCTRL_EXYNOS_ARM64 if ARM64 && ARCH_EXYNOS
+@@ -26,12 +25,10 @@ config PINCTRL_EXYNOS_ARM64
+
+ config PINCTRL_S3C24XX
+ bool "Samsung S3C24XX SoC pinctrl driver"
+- depends on OF_GPIO
+- depends on ARCH_S3C24XX || COMPILE_TEST
++ depends on ARCH_S3C24XX || (COMPILE_TEST && OF)
+ select PINCTRL_SAMSUNG
+
+ config PINCTRL_S3C64XX
+ bool "Samsung S3C64XX SoC pinctrl driver"
+- depends on OF_GPIO
+- depends on ARCH_S3C64XX || COMPILE_TEST
++ depends on ARCH_S3C64XX || (COMPILE_TEST && OF)
+ select PINCTRL_SAMSUNG
+diff --git a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
+index fe5f6046fbd52..cc66f852ef7bc 100644
+--- a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
++++ b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
+@@ -504,13 +504,11 @@ static const struct samsung_pin_ctrl exynos850_pin_ctrl[] __initconst = {
+ /* pin-controller instance 0 ALIVE data */
+ .pin_banks = exynos850_pin_banks0,
+ .nr_banks = ARRAY_SIZE(exynos850_pin_banks0),
+- .eint_gpio_init = exynos_eint_gpio_init,
+ .eint_wkup_init = exynos_eint_wkup_init,
+ }, {
+ /* pin-controller instance 1 CMGP data */
+ .pin_banks = exynos850_pin_banks1,
+ .nr_banks = ARRAY_SIZE(exynos850_pin_banks1),
+- .eint_gpio_init = exynos_eint_gpio_init,
+ .eint_wkup_init = exynos_eint_wkup_init,
+ }, {
+ /* pin-controller instance 2 AUD data */
+diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
+index 2a0fc63516f12..463b9e578237e 100644
+--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
++++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
+@@ -1002,6 +1002,16 @@ samsung_pinctrl_get_soc_data_for_of_alias(struct platform_device *pdev)
+ return &(of_data->ctrl[id]);
+ }
+
++static void samsung_banks_of_node_put(struct samsung_pinctrl_drv_data *d)
++{
++ struct samsung_pin_bank *bank;
++ unsigned int i;
++
++ bank = d->pin_banks;
++ for (i = 0; i < d->nr_banks; ++i, ++bank)
++ of_node_put(bank->of_node);
++}
++
+ /* retrieve the soc specific data */
+ static const struct samsung_pin_ctrl *
+ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
+@@ -1116,19 +1126,19 @@ static int samsung_pinctrl_probe(struct platform_device *pdev)
+ if (ctrl->retention_data) {
+ drvdata->retention_ctrl = ctrl->retention_data->init(drvdata,
+ ctrl->retention_data);
+- if (IS_ERR(drvdata->retention_ctrl))
+- return PTR_ERR(drvdata->retention_ctrl);
++ if (IS_ERR(drvdata->retention_ctrl)) {
++ ret = PTR_ERR(drvdata->retention_ctrl);
++ goto err_put_banks;
++ }
+ }
+
+ ret = samsung_pinctrl_register(pdev, drvdata);
+ if (ret)
+- return ret;
++ goto err_put_banks;
+
+ ret = samsung_gpiolib_register(pdev, drvdata);
+- if (ret) {
+- samsung_pinctrl_unregister(pdev, drvdata);
+- return ret;
+- }
++ if (ret)
++ goto err_unregister;
+
+ if (ctrl->eint_gpio_init)
+ ctrl->eint_gpio_init(drvdata);
+@@ -1138,6 +1148,12 @@ static int samsung_pinctrl_probe(struct platform_device *pdev)
+ platform_set_drvdata(pdev, drvdata);
+
+ return 0;
++
++err_unregister:
++ samsung_pinctrl_unregister(pdev, drvdata);
++err_put_banks:
++ samsung_banks_of_node_put(drvdata);
++ return ret;
+ }
+
+ /*
+diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c
+index dfd8888a222a4..97a4fb5a93280 100644
+--- a/drivers/pinctrl/stm32/pinctrl-stm32.c
++++ b/drivers/pinctrl/stm32/pinctrl-stm32.c
+@@ -225,6 +225,13 @@ static void stm32_gpio_free(struct gpio_chip *chip, unsigned offset)
+ pinctrl_gpio_free(chip->base + offset);
+ }
+
++static int stm32_gpio_get_noclk(struct gpio_chip *chip, unsigned int offset)
++{
++ struct stm32_gpio_bank *bank = gpiochip_get_data(chip);
++
++ return !!(readl_relaxed(bank->base + STM32_GPIO_IDR) & BIT(offset));
++}
++
+ static int stm32_gpio_get(struct gpio_chip *chip, unsigned offset)
+ {
+ struct stm32_gpio_bank *bank = gpiochip_get_data(chip);
+@@ -232,7 +239,7 @@ static int stm32_gpio_get(struct gpio_chip *chip, unsigned offset)
+
+ clk_enable(bank->clk);
+
+- ret = !!(readl_relaxed(bank->base + STM32_GPIO_IDR) & BIT(offset));
++ ret = stm32_gpio_get_noclk(chip, offset);
+
+ clk_disable(bank->clk);
+
+@@ -311,8 +318,12 @@ static void stm32_gpio_irq_trigger(struct irq_data *d)
+ struct stm32_gpio_bank *bank = d->domain->host_data;
+ int level;
+
++ /* Do not access the GPIO if this is not LEVEL triggered IRQ. */
++ if (!(bank->irq_type[d->hwirq] & IRQ_TYPE_LEVEL_MASK))
++ return;
++
+ /* If level interrupt type then retrig */
+- level = stm32_gpio_get(&bank->gpio_chip, d->hwirq);
++ level = stm32_gpio_get_noclk(&bank->gpio_chip, d->hwirq);
+ if ((level == 0 && bank->irq_type[d->hwirq] == IRQ_TYPE_LEVEL_LOW) ||
+ (level == 1 && bank->irq_type[d->hwirq] == IRQ_TYPE_LEVEL_HIGH))
+ irq_chip_retrigger_hierarchy(d);
+@@ -354,6 +365,7 @@ static int stm32_gpio_irq_request_resources(struct irq_data *irq_data)
+ {
+ struct stm32_gpio_bank *bank = irq_data->domain->host_data;
+ struct stm32_pinctrl *pctl = dev_get_drvdata(bank->gpio_chip.parent);
++ unsigned long flags;
+ int ret;
+
+ ret = stm32_gpio_direction_input(&bank->gpio_chip, irq_data->hwirq);
+@@ -367,6 +379,10 @@ static int stm32_gpio_irq_request_resources(struct irq_data *irq_data)
+ return ret;
+ }
+
++ flags = irqd_get_trigger_type(irq_data);
++ if (flags & IRQ_TYPE_LEVEL_MASK)
++ clk_enable(bank->clk);
++
+ return 0;
+ }
+
+@@ -374,6 +390,9 @@ static void stm32_gpio_irq_release_resources(struct irq_data *irq_data)
+ {
+ struct stm32_gpio_bank *bank = irq_data->domain->host_data;
+
++ if (bank->irq_type[irq_data->hwirq] & IRQ_TYPE_LEVEL_MASK)
++ clk_disable(bank->clk);
++
+ gpiochip_unlock_as_irq(&bank->gpio_chip, irq_data->hwirq);
+ }
+
+@@ -1251,10 +1270,10 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl,
+ bank_nr = args.args[1] / STM32_GPIO_PINS_PER_BANK;
+ bank->gpio_chip.base = args.args[1];
+
+- npins = args.args[2];
+- while (!of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3,
+- ++i, &args))
+- npins += args.args[2];
++ /* get the last defined gpio line (offset + nb of pins) */
++ npins = args.args[0] + args.args[2];
++ while (!of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, ++i, &args))
++ npins = max(npins, (int)(args.args[0] + args.args[2]));
+ } else {
+ bank_nr = pctl->nbanks;
+ bank->gpio_chip.base = bank_nr * STM32_GPIO_PINS_PER_BANK;
+diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-h616.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-h616.c
+index ce1917e230f41..152b71226a807 100644
+--- a/drivers/pinctrl/sunxi/pinctrl-sun50i-h616.c
++++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-h616.c
+@@ -363,16 +363,16 @@ static const struct sunxi_desc_pin h616_pins[] = {
+ SUNXI_FUNCTION(0x0, "gpio_in"),
+ SUNXI_FUNCTION(0x1, "gpio_out"),
+ SUNXI_FUNCTION(0x2, "uart2"), /* CTS */
+- SUNXI_FUNCTION(0x3, "i2s3"), /* DO0 */
++ SUNXI_FUNCTION(0x3, "i2s3_dout0"), /* DO0 */
+ SUNXI_FUNCTION(0x4, "spi1"), /* MISO */
+- SUNXI_FUNCTION(0x5, "i2s3"), /* DI1 */
++ SUNXI_FUNCTION(0x5, "i2s3_din1"), /* DI1 */
+ SUNXI_FUNCTION_IRQ_BANK(0x6, 6, 8)), /* PH_EINT8 */
+ SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 9),
+ SUNXI_FUNCTION(0x0, "gpio_in"),
+ SUNXI_FUNCTION(0x1, "gpio_out"),
+- SUNXI_FUNCTION(0x3, "i2s3"), /* DI0 */
++ SUNXI_FUNCTION(0x3, "i2s3_din0"), /* DI0 */
+ SUNXI_FUNCTION(0x4, "spi1"), /* CS1 */
+- SUNXI_FUNCTION(0x3, "i2s3"), /* DO1 */
++ SUNXI_FUNCTION(0x5, "i2s3_dout1"), /* DO1 */
+ SUNXI_FUNCTION_IRQ_BANK(0x6, 6, 9)), /* PH_EINT9 */
+ SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 10),
+ SUNXI_FUNCTION(0x0, "gpio_in"),
+diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+index 862c84efb718f..ce3f9ea415117 100644
+--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
++++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+@@ -36,6 +36,13 @@
+ #include "../core.h"
+ #include "pinctrl-sunxi.h"
+
++/*
++ * These lock classes tell lockdep that GPIO IRQs are in a different
++ * category than their parents, so it won't report false recursion.
++ */
++static struct lock_class_key sunxi_pinctrl_irq_lock_class;
++static struct lock_class_key sunxi_pinctrl_irq_request_class;
++
+ static struct irq_chip sunxi_pinctrl_edge_irq_chip;
+ static struct irq_chip sunxi_pinctrl_level_irq_chip;
+
+@@ -1551,6 +1558,8 @@ int sunxi_pinctrl_init_with_variant(struct platform_device *pdev,
+ for (i = 0; i < (pctl->desc->irq_banks * IRQ_PER_BANK); i++) {
+ int irqno = irq_create_mapping(pctl->domain, i);
+
++ irq_set_lockdep_class(irqno, &sunxi_pinctrl_irq_lock_class,
++ &sunxi_pinctrl_irq_request_class);
+ irq_set_chip_and_handler(irqno, &sunxi_pinctrl_edge_irq_chip,
+ handle_edge_irq);
+ irq_set_chip_data(irqno, pctl);
+diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile
+index f901d2e43166c..88cbc434c06b2 100644
+--- a/drivers/platform/chrome/Makefile
++++ b/drivers/platform/chrome/Makefile
+@@ -2,6 +2,7 @@
+
+ # tell define_trace.h where to find the cros ec trace header
+ CFLAGS_cros_ec_trace.o:= -I$(src)
++CFLAGS_cros_ec_sensorhub_ring.o:= -I$(src)
+
+ obj-$(CONFIG_CHROMEOS_LAPTOP) += chromeos_laptop.o
+ obj-$(CONFIG_CHROMEOS_PSTORE) += chromeos_pstore.o
+@@ -20,7 +21,7 @@ obj-$(CONFIG_CROS_EC_CHARDEV) += cros_ec_chardev.o
+ obj-$(CONFIG_CROS_EC_LIGHTBAR) += cros_ec_lightbar.o
+ obj-$(CONFIG_CROS_EC_VBC) += cros_ec_vbc.o
+ obj-$(CONFIG_CROS_EC_DEBUGFS) += cros_ec_debugfs.o
+-cros-ec-sensorhub-objs := cros_ec_sensorhub.o cros_ec_sensorhub_ring.o cros_ec_trace.o
++cros-ec-sensorhub-objs := cros_ec_sensorhub.o cros_ec_sensorhub_ring.o
+ obj-$(CONFIG_CROS_EC_SENSORHUB) += cros-ec-sensorhub.o
+ obj-$(CONFIG_CROS_EC_SYSFS) += cros_ec_sysfs.o
+ obj-$(CONFIG_CROS_USBPD_LOGGER) += cros_usbpd_logger.o
+diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c
+index 272c89837d745..0dbceee87a4b1 100644
+--- a/drivers/platform/chrome/cros_ec_debugfs.c
++++ b/drivers/platform/chrome/cros_ec_debugfs.c
+@@ -25,6 +25,9 @@
+
+ #define CIRC_ADD(idx, size, value) (((idx) + (value)) & ((size) - 1))
+
++/* waitqueue for log readers */
++static DECLARE_WAIT_QUEUE_HEAD(cros_ec_debugfs_log_wq);
++
+ /**
+ * struct cros_ec_debugfs - EC debugging information.
+ *
+@@ -33,7 +36,6 @@
+ * @log_buffer: circular buffer for console log information
+ * @read_msg: preallocated EC command and buffer to read console log
+ * @log_mutex: mutex to protect circular buffer
+- * @log_wq: waitqueue for log readers
+ * @log_poll_work: recurring task to poll EC for new console log data
+ * @panicinfo_blob: panicinfo debugfs blob
+ */
+@@ -44,7 +46,6 @@ struct cros_ec_debugfs {
+ struct circ_buf log_buffer;
+ struct cros_ec_command *read_msg;
+ struct mutex log_mutex;
+- wait_queue_head_t log_wq;
+ struct delayed_work log_poll_work;
+ /* EC panicinfo */
+ struct debugfs_blob_wrapper panicinfo_blob;
+@@ -107,7 +108,7 @@ static void cros_ec_console_log_work(struct work_struct *__work)
+ buf_space--;
+ }
+
+- wake_up(&debug_info->log_wq);
++ wake_up(&cros_ec_debugfs_log_wq);
+ }
+
+ mutex_unlock(&debug_info->log_mutex);
+@@ -141,7 +142,7 @@ static ssize_t cros_ec_console_log_read(struct file *file, char __user *buf,
+
+ mutex_unlock(&debug_info->log_mutex);
+
+- ret = wait_event_interruptible(debug_info->log_wq,
++ ret = wait_event_interruptible(cros_ec_debugfs_log_wq,
+ CIRC_CNT(cb->head, cb->tail, LOG_SIZE));
+ if (ret < 0)
+ return ret;
+@@ -173,7 +174,7 @@ static __poll_t cros_ec_console_log_poll(struct file *file,
+ struct cros_ec_debugfs *debug_info = file->private_data;
+ __poll_t mask = 0;
+
+- poll_wait(file, &debug_info->log_wq, wait);
++ poll_wait(file, &cros_ec_debugfs_log_wq, wait);
+
+ mutex_lock(&debug_info->log_mutex);
+ if (CIRC_CNT(debug_info->log_buffer.head,
+@@ -377,7 +378,6 @@ static int cros_ec_create_console_log(struct cros_ec_debugfs *debug_info)
+ debug_info->log_buffer.tail = 0;
+
+ mutex_init(&debug_info->log_mutex);
+- init_waitqueue_head(&debug_info->log_wq);
+
+ debugfs_create_file("console_log", S_IFREG | 0444, debug_info->dir,
+ debug_info, &cros_ec_console_log_fops);
+diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
+index 98e37080f7609..71948dade0e2a 100644
+--- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c
++++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
+@@ -17,7 +17,8 @@
+ #include <linux/sort.h>
+ #include <linux/slab.h>
+
+-#include "cros_ec_trace.h"
++#define CREATE_TRACE_POINTS
++#include "cros_ec_sensorhub_trace.h"
+
+ /* Precision of fixed point for the m values from the filter */
+ #define M_PRECISION BIT(23)
+diff --git a/drivers/platform/chrome/cros_ec_sensorhub_trace.h b/drivers/platform/chrome/cros_ec_sensorhub_trace.h
+new file mode 100644
+index 0000000000000..57d9b47859692
+--- /dev/null
++++ b/drivers/platform/chrome/cros_ec_sensorhub_trace.h
+@@ -0,0 +1,123 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Trace events for the ChromeOS Sensorhub kernel module
++ *
++ * Copyright 2021 Google LLC.
++ */
++
++#undef TRACE_SYSTEM
++#define TRACE_SYSTEM cros_ec
++
++#if !defined(_CROS_EC_SENSORHUB_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
++#define _CROS_EC_SENSORHUB_TRACE_H_
++
++#include <linux/types.h>
++#include <linux/platform_data/cros_ec_sensorhub.h>
++
++#include <linux/tracepoint.h>
++
++TRACE_EVENT(cros_ec_sensorhub_timestamp,
++ TP_PROTO(u32 ec_sample_timestamp, u32 ec_fifo_timestamp, s64 fifo_timestamp,
++ s64 current_timestamp, s64 current_time),
++ TP_ARGS(ec_sample_timestamp, ec_fifo_timestamp, fifo_timestamp, current_timestamp,
++ current_time),
++ TP_STRUCT__entry(
++ __field(u32, ec_sample_timestamp)
++ __field(u32, ec_fifo_timestamp)
++ __field(s64, fifo_timestamp)
++ __field(s64, current_timestamp)
++ __field(s64, current_time)
++ __field(s64, delta)
++ ),
++ TP_fast_assign(
++ __entry->ec_sample_timestamp = ec_sample_timestamp;
++ __entry->ec_fifo_timestamp = ec_fifo_timestamp;
++ __entry->fifo_timestamp = fifo_timestamp;
++ __entry->current_timestamp = current_timestamp;
++ __entry->current_time = current_time;
++ __entry->delta = current_timestamp - current_time;
++ ),
++ TP_printk("ec_ts: %9u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld",
++ __entry->ec_sample_timestamp,
++ __entry->ec_fifo_timestamp,
++ __entry->fifo_timestamp,
++ __entry->current_timestamp,
++ __entry->current_time,
++ __entry->delta
++ )
++);
++
++TRACE_EVENT(cros_ec_sensorhub_data,
++ TP_PROTO(u32 ec_sensor_num, u32 ec_fifo_timestamp, s64 fifo_timestamp,
++ s64 current_timestamp, s64 current_time),
++ TP_ARGS(ec_sensor_num, ec_fifo_timestamp, fifo_timestamp, current_timestamp, current_time),
++ TP_STRUCT__entry(
++ __field(u32, ec_sensor_num)
++ __field(u32, ec_fifo_timestamp)
++ __field(s64, fifo_timestamp)
++ __field(s64, current_timestamp)
++ __field(s64, current_time)
++ __field(s64, delta)
++ ),
++ TP_fast_assign(
++ __entry->ec_sensor_num = ec_sensor_num;
++ __entry->ec_fifo_timestamp = ec_fifo_timestamp;
++ __entry->fifo_timestamp = fifo_timestamp;
++ __entry->current_timestamp = current_timestamp;
++ __entry->current_time = current_time;
++ __entry->delta = current_timestamp - current_time;
++ ),
++ TP_printk("ec_num: %4u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld",
++ __entry->ec_sensor_num,
++ __entry->ec_fifo_timestamp,
++ __entry->fifo_timestamp,
++ __entry->current_timestamp,
++ __entry->current_time,
++ __entry->delta
++ )
++);
++
++TRACE_EVENT(cros_ec_sensorhub_filter,
++ TP_PROTO(struct cros_ec_sensors_ts_filter_state *state, s64 dx, s64 dy),
++ TP_ARGS(state, dx, dy),
++ TP_STRUCT__entry(
++ __field(s64, dx)
++ __field(s64, dy)
++ __field(s64, median_m)
++ __field(s64, median_error)
++ __field(s64, history_len)
++ __field(s64, x)
++ __field(s64, y)
++ ),
++ TP_fast_assign(
++ __entry->dx = dx;
++ __entry->dy = dy;
++ __entry->median_m = state->median_m;
++ __entry->median_error = state->median_error;
++ __entry->history_len = state->history_len;
++ __entry->x = state->x_offset;
++ __entry->y = state->y_offset;
++ ),
++ TP_printk("dx: %12lld. dy: %12lld median_m: %12lld median_error: %12lld len: %lld x: %12lld y: %12lld",
++ __entry->dx,
++ __entry->dy,
++ __entry->median_m,
++ __entry->median_error,
++ __entry->history_len,
++ __entry->x,
++ __entry->y
++ )
++);
++
++
++#endif /* _CROS_EC_SENSORHUB_TRACE_H_ */
++
++/* this part must be outside header guard */
++
++#undef TRACE_INCLUDE_PATH
++#define TRACE_INCLUDE_PATH .
++
++#undef TRACE_INCLUDE_FILE
++#define TRACE_INCLUDE_FILE cros_ec_sensorhub_trace
++
++#include <trace/define_trace.h>
+diff --git a/drivers/platform/chrome/cros_ec_trace.h b/drivers/platform/chrome/cros_ec_trace.h
+index 7e7cfc98657a4..9bb5cd2c98b8b 100644
+--- a/drivers/platform/chrome/cros_ec_trace.h
++++ b/drivers/platform/chrome/cros_ec_trace.h
+@@ -15,7 +15,6 @@
+ #include <linux/types.h>
+ #include <linux/platform_data/cros_ec_commands.h>
+ #include <linux/platform_data/cros_ec_proto.h>
+-#include <linux/platform_data/cros_ec_sensorhub.h>
+
+ #include <linux/tracepoint.h>
+
+@@ -71,100 +70,6 @@ TRACE_EVENT(cros_ec_request_done,
+ __entry->retval)
+ );
+
+-TRACE_EVENT(cros_ec_sensorhub_timestamp,
+- TP_PROTO(u32 ec_sample_timestamp, u32 ec_fifo_timestamp, s64 fifo_timestamp,
+- s64 current_timestamp, s64 current_time),
+- TP_ARGS(ec_sample_timestamp, ec_fifo_timestamp, fifo_timestamp, current_timestamp,
+- current_time),
+- TP_STRUCT__entry(
+- __field(u32, ec_sample_timestamp)
+- __field(u32, ec_fifo_timestamp)
+- __field(s64, fifo_timestamp)
+- __field(s64, current_timestamp)
+- __field(s64, current_time)
+- __field(s64, delta)
+- ),
+- TP_fast_assign(
+- __entry->ec_sample_timestamp = ec_sample_timestamp;
+- __entry->ec_fifo_timestamp = ec_fifo_timestamp;
+- __entry->fifo_timestamp = fifo_timestamp;
+- __entry->current_timestamp = current_timestamp;
+- __entry->current_time = current_time;
+- __entry->delta = current_timestamp - current_time;
+- ),
+- TP_printk("ec_ts: %9u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld",
+- __entry->ec_sample_timestamp,
+- __entry->ec_fifo_timestamp,
+- __entry->fifo_timestamp,
+- __entry->current_timestamp,
+- __entry->current_time,
+- __entry->delta
+- )
+-);
+-
+-TRACE_EVENT(cros_ec_sensorhub_data,
+- TP_PROTO(u32 ec_sensor_num, u32 ec_fifo_timestamp, s64 fifo_timestamp,
+- s64 current_timestamp, s64 current_time),
+- TP_ARGS(ec_sensor_num, ec_fifo_timestamp, fifo_timestamp, current_timestamp, current_time),
+- TP_STRUCT__entry(
+- __field(u32, ec_sensor_num)
+- __field(u32, ec_fifo_timestamp)
+- __field(s64, fifo_timestamp)
+- __field(s64, current_timestamp)
+- __field(s64, current_time)
+- __field(s64, delta)
+- ),
+- TP_fast_assign(
+- __entry->ec_sensor_num = ec_sensor_num;
+- __entry->ec_fifo_timestamp = ec_fifo_timestamp;
+- __entry->fifo_timestamp = fifo_timestamp;
+- __entry->current_timestamp = current_timestamp;
+- __entry->current_time = current_time;
+- __entry->delta = current_timestamp - current_time;
+- ),
+- TP_printk("ec_num: %4u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld",
+- __entry->ec_sensor_num,
+- __entry->ec_fifo_timestamp,
+- __entry->fifo_timestamp,
+- __entry->current_timestamp,
+- __entry->current_time,
+- __entry->delta
+- )
+-);
+-
+-TRACE_EVENT(cros_ec_sensorhub_filter,
+- TP_PROTO(struct cros_ec_sensors_ts_filter_state *state, s64 dx, s64 dy),
+- TP_ARGS(state, dx, dy),
+- TP_STRUCT__entry(
+- __field(s64, dx)
+- __field(s64, dy)
+- __field(s64, median_m)
+- __field(s64, median_error)
+- __field(s64, history_len)
+- __field(s64, x)
+- __field(s64, y)
+- ),
+- TP_fast_assign(
+- __entry->dx = dx;
+- __entry->dy = dy;
+- __entry->median_m = state->median_m;
+- __entry->median_error = state->median_error;
+- __entry->history_len = state->history_len;
+- __entry->x = state->x_offset;
+- __entry->y = state->y_offset;
+- ),
+- TP_printk("dx: %12lld. dy: %12lld median_m: %12lld median_error: %12lld len: %lld x: %12lld y: %12lld",
+- __entry->dx,
+- __entry->dy,
+- __entry->median_m,
+- __entry->median_error,
+- __entry->history_len,
+- __entry->x,
+- __entry->y
+- )
+-);
+-
+-
+ #endif /* _CROS_EC_TRACE_H_ */
+
+ /* this part must be outside header guard */
+diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c
+index 262a891eded34..4027c3ef90d7c 100644
+--- a/drivers/platform/chrome/cros_ec_typec.c
++++ b/drivers/platform/chrome/cros_ec_typec.c
+@@ -1106,7 +1106,13 @@ static int cros_typec_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ typec->dev = dev;
++
+ typec->ec = dev_get_drvdata(pdev->dev.parent);
++ if (!typec->ec) {
++ dev_err(dev, "couldn't find parent EC device\n");
++ return -ENODEV;
++ }
++
+ platform_set_drvdata(pdev, typec);
+
+ ret = cros_typec_get_cmd_version(typec);
+diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c
+index 04bc3b50aa7a4..65b4a819f1bdf 100644
+--- a/drivers/platform/mellanox/mlxbf-pmc.c
++++ b/drivers/platform/mellanox/mlxbf-pmc.c
+@@ -1374,8 +1374,8 @@ static int mlxbf_pmc_map_counters(struct device *dev)
+ pmc->block[i].counters = info[2];
+ pmc->block[i].type = info[3];
+
+- if (IS_ERR(pmc->block[i].mmio_base))
+- return PTR_ERR(pmc->block[i].mmio_base);
++ if (!pmc->block[i].mmio_base)
++ return -ENOMEM;
+
+ ret = mlxbf_pmc_create_groups(dev, i);
+ if (ret)
+diff --git a/drivers/platform/surface/aggregator/core.c b/drivers/platform/surface/aggregator/core.c
+index c61bbeeec2dfd..54f86df77a37b 100644
+--- a/drivers/platform/surface/aggregator/core.c
++++ b/drivers/platform/surface/aggregator/core.c
+@@ -816,7 +816,7 @@ err_cpkg:
+ err_bus:
+ return status;
+ }
+-module_init(ssam_core_init);
++subsys_initcall(ssam_core_init);
+
+ static void __exit ssam_core_exit(void)
+ {
+diff --git a/drivers/platform/surface/surface3_power.c b/drivers/platform/surface/surface3_power.c
+index 90c1568ea4e09..3cc004c68bdba 100644
+--- a/drivers/platform/surface/surface3_power.c
++++ b/drivers/platform/surface/surface3_power.c
+@@ -233,14 +233,21 @@ static int mshw0011_bix(struct mshw0011_data *cdata, struct bix *bix)
+ }
+ bix->last_full_charg_capacity = ret;
+
+- /* get serial number */
++ /*
++ * Get serial number, on some devices (with unofficial replacement
++ * battery?) reading any of the serial number range addresses gets
++ * nacked in this case just leave the serial number empty.
++ */
+ ret = i2c_smbus_read_i2c_block_data(client, MSHW0011_BAT0_REG_SERIAL_NO,
+ sizeof(buf), buf);
+- if (ret != sizeof(buf)) {
++ if (ret == -EREMOTEIO) {
++ /* no serial number available */
++ } else if (ret != sizeof(buf)) {
+ dev_err(&client->dev, "Error reading serial no: %d\n", ret);
+ return ret;
++ } else {
++ snprintf(bix->serial, ARRAY_SIZE(bix->serial), "%3pE%6pE", buf + 7, buf);
+ }
+- snprintf(bix->serial, ARRAY_SIZE(bix->serial), "%3pE%6pE", buf + 7, buf);
+
+ /* get cycle count */
+ ret = i2c_smbus_read_word_data(client, MSHW0011_BAT0_REG_CYCLE_CNT);
+diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c
+index 4428c4330229a..1679811eff502 100644
+--- a/drivers/platform/surface/surface_aggregator_registry.c
++++ b/drivers/platform/surface/surface_aggregator_registry.c
+@@ -77,6 +77,42 @@ static const struct software_node ssam_node_bas_dtx = {
+ .parent = &ssam_node_root,
+ };
+
++/* HID keyboard (TID1). */
++static const struct software_node ssam_node_hid_tid1_keyboard = {
++ .name = "ssam:01:15:01:01:00",
++ .parent = &ssam_node_root,
++};
++
++/* HID pen stash (TID1; pen taken / stashed away evens). */
++static const struct software_node ssam_node_hid_tid1_penstash = {
++ .name = "ssam:01:15:01:02:00",
++ .parent = &ssam_node_root,
++};
++
++/* HID touchpad (TID1). */
++static const struct software_node ssam_node_hid_tid1_touchpad = {
++ .name = "ssam:01:15:01:03:00",
++ .parent = &ssam_node_root,
++};
++
++/* HID device instance 6 (TID1, unknown HID device). */
++static const struct software_node ssam_node_hid_tid1_iid6 = {
++ .name = "ssam:01:15:01:06:00",
++ .parent = &ssam_node_root,
++};
++
++/* HID device instance 7 (TID1, unknown HID device). */
++static const struct software_node ssam_node_hid_tid1_iid7 = {
++ .name = "ssam:01:15:01:07:00",
++ .parent = &ssam_node_root,
++};
++
++/* HID system controls (TID1). */
++static const struct software_node ssam_node_hid_tid1_sysctrl = {
++ .name = "ssam:01:15:01:08:00",
++ .parent = &ssam_node_root,
++};
++
+ /* HID keyboard. */
+ static const struct software_node ssam_node_hid_main_keyboard = {
+ .name = "ssam:01:15:02:01:00",
+@@ -159,6 +195,21 @@ static const struct software_node *ssam_node_group_sl3[] = {
+ NULL,
+ };
+
++/* Devices for Surface Laptop Studio. */
++static const struct software_node *ssam_node_group_sls[] = {
++ &ssam_node_root,
++ &ssam_node_bat_ac,
++ &ssam_node_bat_main,
++ &ssam_node_tmp_pprof,
++ &ssam_node_hid_tid1_keyboard,
++ &ssam_node_hid_tid1_penstash,
++ &ssam_node_hid_tid1_touchpad,
++ &ssam_node_hid_tid1_iid6,
++ &ssam_node_hid_tid1_iid7,
++ &ssam_node_hid_tid1_sysctrl,
++ NULL,
++};
++
+ /* Devices for Surface Laptop Go. */
+ static const struct software_node *ssam_node_group_slg1[] = {
+ &ssam_node_root,
+@@ -507,6 +558,9 @@ static const struct acpi_device_id ssam_platform_hub_match[] = {
+ /* Surface Laptop Go 1 */
+ { "MSHW0118", (unsigned long)ssam_node_group_slg1 },
+
++ /* Surface Laptop Studio */
++ { "MSHW0123", (unsigned long)ssam_node_group_sls },
++
+ { },
+ };
+ MODULE_DEVICE_TABLE(acpi, ssam_platform_hub_match);
+diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
+index 69690e26bb6d4..2734a771d1f00 100644
+--- a/drivers/platform/x86/Makefile
++++ b/drivers/platform/x86/Makefile
+@@ -67,7 +67,7 @@ obj-$(CONFIG_THINKPAD_ACPI) += thinkpad_acpi.o
+ obj-$(CONFIG_THINKPAD_LMI) += think-lmi.o
+
+ # Intel
+-obj-$(CONFIG_X86_PLATFORM_DRIVERS_INTEL) += intel/
++obj-y += intel/
+
+ # MSI
+ obj-$(CONFIG_MSI_LAPTOP) += msi-laptop.o
+diff --git a/drivers/platform/x86/amd-pmc.c b/drivers/platform/x86/amd-pmc.c
+index fc95620101e85..d3efd514614c0 100644
+--- a/drivers/platform/x86/amd-pmc.c
++++ b/drivers/platform/x86/amd-pmc.c
+@@ -70,7 +70,7 @@
+ #define AMD_CPU_ID_CZN AMD_CPU_ID_RN
+ #define AMD_CPU_ID_YC 0x14B5
+
+-#define PMC_MSG_DELAY_MIN_US 100
++#define PMC_MSG_DELAY_MIN_US 50
+ #define RESPONSE_REGISTER_LOOP_MAX 20000
+
+ #define SOC_SUBSYSTEM_IP_MAX 12
+@@ -375,7 +375,8 @@ static int __maybe_unused amd_pmc_resume(struct device *dev)
+ }
+
+ static const struct dev_pm_ops amd_pmc_pm_ops = {
+- SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(amd_pmc_suspend, amd_pmc_resume)
++ .suspend_noirq = amd_pmc_suspend,
++ .resume_noirq = amd_pmc_resume,
+ };
+
+ static const struct pci_device_id pmc_pci_ids[] = {
+diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
+index 9aae45a452002..57553f9b4d1dc 100644
+--- a/drivers/platform/x86/apple-gmux.c
++++ b/drivers/platform/x86/apple-gmux.c
+@@ -625,7 +625,7 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
+ }
+
+ gmux_data->iostart = res->start;
+- gmux_data->iolen = res->end - res->start;
++ gmux_data->iolen = resource_size(res);
+
+ if (gmux_data->iolen < GMUX_MIN_IO_LEN) {
+ pr_err("gmux I/O region too small (%lu < %u)\n",
+diff --git a/drivers/platform/x86/dell/Kconfig b/drivers/platform/x86/dell/Kconfig
+index 2fffa57e596e4..fe224a54f24c0 100644
+--- a/drivers/platform/x86/dell/Kconfig
++++ b/drivers/platform/x86/dell/Kconfig
+@@ -187,7 +187,7 @@ config DELL_WMI_AIO
+
+ config DELL_WMI_DESCRIPTOR
+ tristate
+- default m
++ default n
+ depends on ACPI_WMI
+
+ config DELL_WMI_LED
+diff --git a/drivers/platform/x86/hp_accel.c b/drivers/platform/x86/hp_accel.c
+index cc53f725c0419..ef24f53753c6e 100644
+--- a/drivers/platform/x86/hp_accel.c
++++ b/drivers/platform/x86/hp_accel.c
+@@ -331,9 +331,11 @@ static int lis3lv02d_probe(struct platform_device *device)
+ INIT_WORK(&hpled_led.work, delayed_set_status_worker);
+ ret = led_classdev_register(NULL, &hpled_led.led_classdev);
+ if (ret) {
++ i8042_remove_filter(hp_accel_i8042_filter);
+ lis3lv02d_joystick_disable(&lis3_dev);
+ lis3lv02d_poweroff(&lis3_dev);
+ flush_work(&hpled_led.work);
++ lis3lv02d_remove_fs(&lis3_dev);
+ return ret;
+ }
+
+diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c
+index a2d846c4a7eef..eac3e6b4ea113 100644
+--- a/drivers/platform/x86/huawei-wmi.c
++++ b/drivers/platform/x86/huawei-wmi.c
+@@ -470,10 +470,17 @@ static DEVICE_ATTR_RW(charge_control_thresholds);
+
+ static int huawei_wmi_battery_add(struct power_supply *battery)
+ {
+- device_create_file(&battery->dev, &dev_attr_charge_control_start_threshold);
+- device_create_file(&battery->dev, &dev_attr_charge_control_end_threshold);
++ int err = 0;
+
+- return 0;
++ err = device_create_file(&battery->dev, &dev_attr_charge_control_start_threshold);
++ if (err)
++ return err;
++
++ err = device_create_file(&battery->dev, &dev_attr_charge_control_end_threshold);
++ if (err)
++ device_remove_file(&battery->dev, &dev_attr_charge_control_start_threshold);
++
++ return err;
+ }
+
+ static int huawei_wmi_battery_remove(struct power_supply *battery)
+diff --git a/drivers/platform/x86/intel/Kconfig b/drivers/platform/x86/intel/Kconfig
+index 0b21468e1bd01..02e4481b384e4 100644
+--- a/drivers/platform/x86/intel/Kconfig
++++ b/drivers/platform/x86/intel/Kconfig
+@@ -3,19 +3,6 @@
+ # Intel x86 Platform Specific Drivers
+ #
+
+-menuconfig X86_PLATFORM_DRIVERS_INTEL
+- bool "Intel x86 Platform Specific Device Drivers"
+- default y
+- help
+- Say Y here to get to see options for device drivers for
+- various Intel x86 platforms, including vendor-specific
+- drivers. This option alone does not add any kernel code.
+-
+- If you say N, all options in this submenu will be skipped
+- and disabled.
+-
+-if X86_PLATFORM_DRIVERS_INTEL
+-
+ source "drivers/platform/x86/intel/atomisp2/Kconfig"
+ source "drivers/platform/x86/intel/int1092/Kconfig"
+ source "drivers/platform/x86/intel/int33fe/Kconfig"
+@@ -167,5 +154,3 @@ config INTEL_UNCORE_FREQ_CONTROL
+
+ To compile this driver as a module, choose M here: the module
+ will be called intel-uncore-frequency.
+-
+-endif # X86_PLATFORM_DRIVERS_INTEL
+diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
+index 08598942a6d78..41a2a026f1568 100644
+--- a/drivers/platform/x86/intel/hid.c
++++ b/drivers/platform/x86/intel/hid.c
+@@ -99,6 +99,20 @@ static const struct dmi_system_id button_array_table[] = {
+ DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Tablet Gen 2"),
+ },
+ },
++ {
++ .ident = "Microsoft Surface Go 3",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "Surface Go 3"),
++ },
++ },
++ {
++ .ident = "Microsoft Surface Go 3",
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "Surface Go 3"),
++ },
++ },
+ { }
+ };
+
+diff --git a/drivers/platform/x86/intel/pmc/pltdrv.c b/drivers/platform/x86/intel/pmc/pltdrv.c
+index 73797680b895c..15ca8afdd973d 100644
+--- a/drivers/platform/x86/intel/pmc/pltdrv.c
++++ b/drivers/platform/x86/intel/pmc/pltdrv.c
+@@ -65,7 +65,7 @@ static int __init pmc_core_platform_init(void)
+
+ retval = platform_device_register(pmc_core_device);
+ if (retval)
+- kfree(pmc_core_device);
++ platform_device_put(pmc_core_device);
+
+ return retval;
+ }
+diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
+index c9a85eb2e8600..e8424e70d81d2 100644
+--- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
++++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
+@@ -596,7 +596,10 @@ static long isst_if_def_ioctl(struct file *file, unsigned int cmd,
+ return ret;
+ }
+
+-static DEFINE_MUTEX(punit_misc_dev_lock);
++/* Lock to prevent module registration when already opened by user space */
++static DEFINE_MUTEX(punit_misc_dev_open_lock);
++/* Lock to allow one share misc device for all ISST interace */
++static DEFINE_MUTEX(punit_misc_dev_reg_lock);
+ static int misc_usage_count;
+ static int misc_device_ret;
+ static int misc_device_open;
+@@ -606,7 +609,7 @@ static int isst_if_open(struct inode *inode, struct file *file)
+ int i, ret = 0;
+
+ /* Fail open, if a module is going away */
+- mutex_lock(&punit_misc_dev_lock);
++ mutex_lock(&punit_misc_dev_open_lock);
+ for (i = 0; i < ISST_IF_DEV_MAX; ++i) {
+ struct isst_if_cmd_cb *cb = &punit_callbacks[i];
+
+@@ -628,7 +631,7 @@ static int isst_if_open(struct inode *inode, struct file *file)
+ } else {
+ misc_device_open++;
+ }
+- mutex_unlock(&punit_misc_dev_lock);
++ mutex_unlock(&punit_misc_dev_open_lock);
+
+ return ret;
+ }
+@@ -637,7 +640,7 @@ static int isst_if_relase(struct inode *inode, struct file *f)
+ {
+ int i;
+
+- mutex_lock(&punit_misc_dev_lock);
++ mutex_lock(&punit_misc_dev_open_lock);
+ misc_device_open--;
+ for (i = 0; i < ISST_IF_DEV_MAX; ++i) {
+ struct isst_if_cmd_cb *cb = &punit_callbacks[i];
+@@ -645,7 +648,7 @@ static int isst_if_relase(struct inode *inode, struct file *f)
+ if (cb->registered)
+ module_put(cb->owner);
+ }
+- mutex_unlock(&punit_misc_dev_lock);
++ mutex_unlock(&punit_misc_dev_open_lock);
+
+ return 0;
+ }
+@@ -662,6 +665,43 @@ static struct miscdevice isst_if_char_driver = {
+ .fops = &isst_if_char_driver_ops,
+ };
+
++static int isst_misc_reg(void)
++{
++ mutex_lock(&punit_misc_dev_reg_lock);
++ if (misc_device_ret)
++ goto unlock_exit;
++
++ if (!misc_usage_count) {
++ misc_device_ret = isst_if_cpu_info_init();
++ if (misc_device_ret)
++ goto unlock_exit;
++
++ misc_device_ret = misc_register(&isst_if_char_driver);
++ if (misc_device_ret) {
++ isst_if_cpu_info_exit();
++ goto unlock_exit;
++ }
++ }
++ misc_usage_count++;
++
++unlock_exit:
++ mutex_unlock(&punit_misc_dev_reg_lock);
++
++ return misc_device_ret;
++}
++
++static void isst_misc_unreg(void)
++{
++ mutex_lock(&punit_misc_dev_reg_lock);
++ if (misc_usage_count)
++ misc_usage_count--;
++ if (!misc_usage_count && !misc_device_ret) {
++ misc_deregister(&isst_if_char_driver);
++ isst_if_cpu_info_exit();
++ }
++ mutex_unlock(&punit_misc_dev_reg_lock);
++}
++
+ /**
+ * isst_if_cdev_register() - Register callback for IOCTL
+ * @device_type: The device type this callback handling.
+@@ -679,38 +719,31 @@ static struct miscdevice isst_if_char_driver = {
+ */
+ int isst_if_cdev_register(int device_type, struct isst_if_cmd_cb *cb)
+ {
+- if (misc_device_ret)
+- return misc_device_ret;
++ int ret;
+
+ if (device_type >= ISST_IF_DEV_MAX)
+ return -EINVAL;
+
+- mutex_lock(&punit_misc_dev_lock);
++ mutex_lock(&punit_misc_dev_open_lock);
++ /* Device is already open, we don't want to add new callbacks */
+ if (misc_device_open) {
+- mutex_unlock(&punit_misc_dev_lock);
++ mutex_unlock(&punit_misc_dev_open_lock);
+ return -EAGAIN;
+ }
+- if (!misc_usage_count) {
+- int ret;
+-
+- misc_device_ret = misc_register(&isst_if_char_driver);
+- if (misc_device_ret)
+- goto unlock_exit;
+-
+- ret = isst_if_cpu_info_init();
+- if (ret) {
+- misc_deregister(&isst_if_char_driver);
+- misc_device_ret = ret;
+- goto unlock_exit;
+- }
+- }
+ memcpy(&punit_callbacks[device_type], cb, sizeof(*cb));
+ punit_callbacks[device_type].registered = 1;
+- misc_usage_count++;
+-unlock_exit:
+- mutex_unlock(&punit_misc_dev_lock);
++ mutex_unlock(&punit_misc_dev_open_lock);
+
+- return misc_device_ret;
++ ret = isst_misc_reg();
++ if (ret) {
++ /*
++ * No need of mutex as the misc device register failed
++ * as no one can open device yet. Hence no contention.
++ */
++ punit_callbacks[device_type].registered = 0;
++ return ret;
++ }
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(isst_if_cdev_register);
+
+@@ -725,16 +758,12 @@ EXPORT_SYMBOL_GPL(isst_if_cdev_register);
+ */
+ void isst_if_cdev_unregister(int device_type)
+ {
+- mutex_lock(&punit_misc_dev_lock);
+- misc_usage_count--;
++ isst_misc_unreg();
++ mutex_lock(&punit_misc_dev_open_lock);
+ punit_callbacks[device_type].registered = 0;
+ if (device_type == ISST_IF_DEV_MBOX)
+ isst_delete_hash();
+- if (!misc_usage_count && !misc_device_ret) {
+- misc_deregister(&isst_if_char_driver);
+- isst_if_cpu_info_exit();
+- }
+- mutex_unlock(&punit_misc_dev_lock);
++ mutex_unlock(&punit_misc_dev_open_lock);
+ }
+ EXPORT_SYMBOL_GPL(isst_if_cdev_unregister);
+
+diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c
+index 7ee010aa740aa..404bdb4cbfae4 100644
+--- a/drivers/platform/x86/samsung-laptop.c
++++ b/drivers/platform/x86/samsung-laptop.c
+@@ -1121,8 +1121,6 @@ static void kbd_led_set(struct led_classdev *led_cdev,
+
+ if (value > samsung->kbd_led.max_brightness)
+ value = samsung->kbd_led.max_brightness;
+- else if (value < 0)
+- value = 0;
+
+ samsung->kbd_led_wk = value;
+ queue_work(samsung->led_workqueue, &samsung->kbd_led_work);
+diff --git a/drivers/platform/x86/think-lmi.c b/drivers/platform/x86/think-lmi.c
+index 9472aae72df29..c4d9c45350f7c 100644
+--- a/drivers/platform/x86/think-lmi.c
++++ b/drivers/platform/x86/think-lmi.c
+@@ -888,8 +888,10 @@ static int tlmi_analyze(void)
+ break;
+ if (!item)
+ break;
+- if (!*item)
++ if (!*item) {
++ kfree(item);
+ continue;
++ }
+
+ /* It is not allowed to have '/' for file name. Convert it into '\'. */
+ strreplace(item, '/', '\\');
+@@ -902,6 +904,7 @@ static int tlmi_analyze(void)
+ setting = kzalloc(sizeof(*setting), GFP_KERNEL);
+ if (!setting) {
+ ret = -ENOMEM;
++ kfree(item);
+ goto fail_clear_attr;
+ }
+ setting->index = i;
+@@ -916,7 +919,6 @@ static int tlmi_analyze(void)
+ }
+ kobject_init(&setting->kobj, &tlmi_attr_setting_ktype);
+ tlmi_priv.setting[i] = setting;
+- tlmi_priv.settings_count++;
+ kfree(item);
+ }
+
+@@ -983,7 +985,12 @@ static void tlmi_remove(struct wmi_device *wdev)
+
+ static int tlmi_probe(struct wmi_device *wdev, const void *context)
+ {
+- tlmi_analyze();
++ int ret;
++
++ ret = tlmi_analyze();
++ if (ret)
++ return ret;
++
+ return tlmi_sysfs_init();
+ }
+
+diff --git a/drivers/platform/x86/think-lmi.h b/drivers/platform/x86/think-lmi.h
+index f8e26823075fd..2ce5086a5af27 100644
+--- a/drivers/platform/x86/think-lmi.h
++++ b/drivers/platform/x86/think-lmi.h
+@@ -55,7 +55,6 @@ struct tlmi_attr_setting {
+ struct think_lmi {
+ struct wmi_device *wmi_device;
+
+- int settings_count;
+ bool can_set_bios_settings;
+ bool can_get_bios_selections;
+ bool can_set_bios_password;
+diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
+index 50ff04c84650c..3dc055ce6e61b 100644
+--- a/drivers/platform/x86/thinkpad_acpi.c
++++ b/drivers/platform/x86/thinkpad_acpi.c
+@@ -1178,15 +1178,6 @@ static int tpacpi_rfk_update_swstate(const struct tpacpi_rfk *tp_rfk)
+ return status;
+ }
+
+-/* Query FW and update rfkill sw state for all rfkill switches */
+-static void tpacpi_rfk_update_swstate_all(void)
+-{
+- unsigned int i;
+-
+- for (i = 0; i < TPACPI_RFK_SW_MAX; i++)
+- tpacpi_rfk_update_swstate(tpacpi_rfkill_switches[i]);
+-}
+-
+ /*
+ * Sync the HW-blocking state of all rfkill switches,
+ * do notice it causes the rfkill core to schedule uevents
+@@ -3129,9 +3120,6 @@ static void tpacpi_send_radiosw_update(void)
+ if (wlsw == TPACPI_RFK_RADIO_OFF)
+ tpacpi_rfk_update_hwblock_state(true);
+
+- /* Sync sw blocking state */
+- tpacpi_rfk_update_swstate_all();
+-
+ /* Sync hw blocking state last if it is hw-unblocked */
+ if (wlsw == TPACPI_RFK_RADIO_ON)
+ tpacpi_rfk_update_hwblock_state(false);
+@@ -8853,6 +8841,7 @@ static const struct tpacpi_quirk fan_quirk_table[] __initconst = {
+ TPACPI_Q_LNV3('N', '2', 'E', TPACPI_FAN_2CTL), /* P1 / X1 Extreme (1st gen) */
+ TPACPI_Q_LNV3('N', '2', 'O', TPACPI_FAN_2CTL), /* P1 / X1 Extreme (2nd gen) */
+ TPACPI_Q_LNV3('N', '2', 'V', TPACPI_FAN_2CTL), /* P1 / X1 Extreme (3nd gen) */
++ TPACPI_Q_LNV3('N', '4', '0', TPACPI_FAN_2CTL), /* P1 / X1 Extreme (4nd gen) */
+ TPACPI_Q_LNV3('N', '3', '0', TPACPI_FAN_2CTL), /* P15 (1st gen) / P15v (1st gen) */
+ TPACPI_Q_LNV3('N', '3', '2', TPACPI_FAN_2CTL), /* X1 Carbon (9th gen) */
+ };
+@@ -9145,7 +9134,7 @@ static int fan_write_cmd_level(const char *cmd, int *rc)
+
+ if (strlencmp(cmd, "level auto") == 0)
+ level = TP_EC_FAN_AUTO;
+- else if ((strlencmp(cmd, "level disengaged") == 0) |
++ else if ((strlencmp(cmd, "level disengaged") == 0) ||
+ (strlencmp(cmd, "level full-speed") == 0))
+ level = TP_EC_FAN_FULLSPEED;
+ else if (sscanf(cmd, "level %d", &level) != 1)
+diff --git a/drivers/platform/x86/touchscreen_dmi.c b/drivers/platform/x86/touchscreen_dmi.c
+index 033f797861d8a..c608078538a79 100644
+--- a/drivers/platform/x86/touchscreen_dmi.c
++++ b/drivers/platform/x86/touchscreen_dmi.c
+@@ -773,6 +773,21 @@ static const struct ts_dmi_data predia_basic_data = {
+ .properties = predia_basic_props,
+ };
+
++static const struct property_entry rwc_nanote_p8_props[] = {
++ PROPERTY_ENTRY_U32("touchscreen-min-y", 46),
++ PROPERTY_ENTRY_U32("touchscreen-size-x", 1728),
++ PROPERTY_ENTRY_U32("touchscreen-size-y", 1140),
++ PROPERTY_ENTRY_BOOL("touchscreen-inverted-y"),
++ PROPERTY_ENTRY_STRING("firmware-name", "gsl1680-rwc-nanote-p8.fw"),
++ PROPERTY_ENTRY_U32("silead,max-fingers", 10),
++ { }
++};
++
++static const struct ts_dmi_data rwc_nanote_p8_data = {
++ .acpi_name = "MSSL1680:00",
++ .properties = rwc_nanote_p8_props,
++};
++
+ static const struct property_entry schneider_sct101ctm_props[] = {
+ PROPERTY_ENTRY_U32("touchscreen-size-x", 1715),
+ PROPERTY_ENTRY_U32("touchscreen-size-y", 1140),
+@@ -1379,6 +1394,15 @@ const struct dmi_system_id touchscreen_dmi_table[] = {
+ DMI_EXACT_MATCH(DMI_BOARD_NAME, "0E57"),
+ },
+ },
++ {
++ /* RWC NANOTE P8 */
++ .driver_data = (void *)&rwc_nanote_p8_data,
++ .matches = {
++ DMI_MATCH(DMI_BOARD_VENDOR, "Default string"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "AY07J"),
++ DMI_MATCH(DMI_PRODUCT_SKU, "0001")
++ },
++ },
+ {
+ /* Schneider SCT101CTM */
+ .driver_data = (void *)&schneider_sct101ctm_data,
+diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c
+index a76313006bdc4..1b65bb61ce888 100644
+--- a/drivers/platform/x86/wmi.c
++++ b/drivers/platform/x86/wmi.c
+@@ -353,7 +353,14 @@ static acpi_status __query_block(struct wmi_block *wblock, u8 instance,
+ * the WQxx method failed - we should disable collection anyway.
+ */
+ if ((block->flags & ACPI_WMI_EXPENSIVE) && ACPI_SUCCESS(wc_status)) {
+- status = acpi_execute_simple_method(handle, wc_method, 0);
++ /*
++ * Ignore whether this WCxx call succeeds or not since
++ * the previously executed WQxx method call might have
++ * succeeded, and returning the failing status code
++ * of this call would throw away the result of the WQxx
++ * call, potentially leaking memory.
++ */
++ acpi_execute_simple_method(handle, wc_method, 0);
+ }
+
+ return status;
+diff --git a/drivers/power/reset/at91-reset.c b/drivers/power/reset/at91-reset.c
+index 026649409135c..64def79d557a8 100644
+--- a/drivers/power/reset/at91-reset.c
++++ b/drivers/power/reset/at91-reset.c
+@@ -193,7 +193,7 @@ static int __init at91_reset_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ reset->rstc_base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL);
+- if (!reset->rstc_base) {
++ if (IS_ERR(reset->rstc_base)) {
+ dev_err(&pdev->dev, "Could not map reset controller address\n");
+ return -ENODEV;
+ }
+@@ -203,7 +203,7 @@ static int __init at91_reset_probe(struct platform_device *pdev)
+ for_each_matching_node_and_match(np, at91_ramc_of_match, &match) {
+ reset->ramc_lpr = (u32)match->data;
+ reset->ramc_base[idx] = devm_of_iomap(&pdev->dev, np, 0, NULL);
+- if (!reset->ramc_base[idx]) {
++ if (IS_ERR(reset->ramc_base[idx])) {
+ dev_err(&pdev->dev, "Could not map ram controller address\n");
+ of_node_put(np);
+ return -ENODEV;
+diff --git a/drivers/power/reset/gemini-poweroff.c b/drivers/power/reset/gemini-poweroff.c
+index 90e35c07240ae..b7f7a8225f22e 100644
+--- a/drivers/power/reset/gemini-poweroff.c
++++ b/drivers/power/reset/gemini-poweroff.c
+@@ -107,8 +107,8 @@ static int gemini_poweroff_probe(struct platform_device *pdev)
+ return PTR_ERR(gpw->base);
+
+ irq = platform_get_irq(pdev, 0);
+- if (!irq)
+- return -EINVAL;
++ if (irq < 0)
++ return irq;
+
+ gpw->dev = dev;
+
+diff --git a/drivers/power/reset/ltc2952-poweroff.c b/drivers/power/reset/ltc2952-poweroff.c
+index 8688c8ba8894c..81be33c041d6b 100644
+--- a/drivers/power/reset/ltc2952-poweroff.c
++++ b/drivers/power/reset/ltc2952-poweroff.c
+@@ -161,8 +161,8 @@ static void ltc2952_poweroff_kill(void)
+
+ static void ltc2952_poweroff_default(struct ltc2952_poweroff *data)
+ {
+- data->wde_interval = 300L * 1E6L;
+- data->trigger_delay = ktime_set(2, 500L*1E6L);
++ data->wde_interval = 300L * NSEC_PER_MSEC;
++ data->trigger_delay = ktime_set(2, 500L * NSEC_PER_MSEC);
+
+ hrtimer_init(&data->timer_trigger, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ data->timer_trigger.function = ltc2952_poweroff_timer_trigger;
+diff --git a/drivers/power/reset/mt6323-poweroff.c b/drivers/power/reset/mt6323-poweroff.c
+index 0532803e6cbc4..d90e76fcb9383 100644
+--- a/drivers/power/reset/mt6323-poweroff.c
++++ b/drivers/power/reset/mt6323-poweroff.c
+@@ -57,6 +57,9 @@ static int mt6323_pwrc_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++ if (!res)
++ return -EINVAL;
++
+ pwrc->base = res->start;
+ pwrc->regmap = mt6397_chip->regmap;
+ pwrc->dev = &pdev->dev;
+diff --git a/drivers/power/supply/ab8500_chargalg.c b/drivers/power/supply/ab8500_chargalg.c
+index ff4b26b1cecae..b809fa5abbbaf 100644
+--- a/drivers/power/supply/ab8500_chargalg.c
++++ b/drivers/power/supply/ab8500_chargalg.c
+@@ -2019,11 +2019,11 @@ static int ab8500_chargalg_probe(struct platform_device *pdev)
+ psy_cfg.drv_data = di;
+
+ /* Initilialize safety timer */
+- hrtimer_init(&di->safety_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
++ hrtimer_init(&di->safety_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ di->safety_timer.function = ab8500_chargalg_safety_timer_expired;
+
+ /* Initilialize maintenance timer */
+- hrtimer_init(&di->maintenance_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
++ hrtimer_init(&di->maintenance_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ di->maintenance_timer.function =
+ ab8500_chargalg_maintenance_timer_expired;
+
+diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
+index 05fe9724ba508..57799a8079d44 100644
+--- a/drivers/power/supply/ab8500_fg.c
++++ b/drivers/power/supply/ab8500_fg.c
+@@ -2545,8 +2545,10 @@ static int ab8500_fg_sysfs_init(struct ab8500_fg *di)
+ ret = kobject_init_and_add(&di->fg_kobject,
+ &ab8500_fg_ktype,
+ NULL, "battery");
+- if (ret < 0)
++ if (ret < 0) {
++ kobject_put(&di->fg_kobject);
+ dev_err(di->dev, "failed to create sysfs entry\n");
++ }
+
+ return ret;
+ }
+diff --git a/drivers/power/supply/axp20x_battery.c b/drivers/power/supply/axp20x_battery.c
+index 18a9db0df4b1f..335e12cc5e2f9 100644
+--- a/drivers/power/supply/axp20x_battery.c
++++ b/drivers/power/supply/axp20x_battery.c
+@@ -186,7 +186,6 @@ static int axp20x_battery_get_prop(struct power_supply *psy,
+ union power_supply_propval *val)
+ {
+ struct axp20x_batt_ps *axp20x_batt = power_supply_get_drvdata(psy);
+- struct iio_channel *chan;
+ int ret = 0, reg, val1;
+
+ switch (psp) {
+@@ -266,12 +265,12 @@ static int axp20x_battery_get_prop(struct power_supply *psy,
+ if (ret)
+ return ret;
+
+- if (reg & AXP20X_PWR_STATUS_BAT_CHARGING)
+- chan = axp20x_batt->batt_chrg_i;
+- else
+- chan = axp20x_batt->batt_dischrg_i;
+-
+- ret = iio_read_channel_processed(chan, &val->intval);
++ if (reg & AXP20X_PWR_STATUS_BAT_CHARGING) {
++ ret = iio_read_channel_processed(axp20x_batt->batt_chrg_i, &val->intval);
++ } else {
++ ret = iio_read_channel_processed(axp20x_batt->batt_dischrg_i, &val1);
++ val->intval = -val1;
++ }
+ if (ret)
+ return ret;
+
+diff --git a/drivers/power/supply/axp288_charger.c b/drivers/power/supply/axp288_charger.c
+index b9553be9bed56..fb9db7f438951 100644
+--- a/drivers/power/supply/axp288_charger.c
++++ b/drivers/power/supply/axp288_charger.c
+@@ -41,11 +41,11 @@
+ #define VBUS_ISPOUT_CUR_LIM_1500MA 0x1 /* 1500mA */
+ #define VBUS_ISPOUT_CUR_LIM_2000MA 0x2 /* 2000mA */
+ #define VBUS_ISPOUT_CUR_NO_LIM 0x3 /* 2500mA */
+-#define VBUS_ISPOUT_VHOLD_SET_MASK 0x31
++#define VBUS_ISPOUT_VHOLD_SET_MASK 0x38
+ #define VBUS_ISPOUT_VHOLD_SET_BIT_POS 0x3
+ #define VBUS_ISPOUT_VHOLD_SET_OFFSET 4000 /* 4000mV */
+ #define VBUS_ISPOUT_VHOLD_SET_LSB_RES 100 /* 100mV */
+-#define VBUS_ISPOUT_VHOLD_SET_4300MV 0x3 /* 4300mV */
++#define VBUS_ISPOUT_VHOLD_SET_4400MV 0x4 /* 4400mV */
+ #define VBUS_ISPOUT_VBUS_PATH_DIS BIT(7)
+
+ #define CHRG_CCCV_CC_MASK 0xf /* 4 bits */
+@@ -744,6 +744,16 @@ static int charger_init_hw_regs(struct axp288_chrg_info *info)
+ ret = axp288_charger_vbus_path_select(info, true);
+ if (ret < 0)
+ return ret;
++ } else {
++ /* Set Vhold to the factory default / recommended 4.4V */
++ val = VBUS_ISPOUT_VHOLD_SET_4400MV << VBUS_ISPOUT_VHOLD_SET_BIT_POS;
++ ret = regmap_update_bits(info->regmap, AXP20X_VBUS_IPSOUT_MGMT,
++ VBUS_ISPOUT_VHOLD_SET_MASK, val);
++ if (ret < 0) {
++ dev_err(&info->pdev->dev, "register(%x) write error(%d)\n",
++ AXP20X_VBUS_IPSOUT_MGMT, ret);
++ return ret;
++ }
+ }
+
+ /* Read current charge voltage and current limit */
+diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
+index 35ff0c8fe96f5..16c4876fe5afb 100644
+--- a/drivers/power/supply/bq24190_charger.c
++++ b/drivers/power/supply/bq24190_charger.c
+@@ -39,6 +39,7 @@
+ #define BQ24190_REG_POC_CHG_CONFIG_DISABLE 0x0
+ #define BQ24190_REG_POC_CHG_CONFIG_CHARGE 0x1
+ #define BQ24190_REG_POC_CHG_CONFIG_OTG 0x2
++#define BQ24190_REG_POC_CHG_CONFIG_OTG_ALT 0x3
+ #define BQ24190_REG_POC_SYS_MIN_MASK (BIT(3) | BIT(2) | BIT(1))
+ #define BQ24190_REG_POC_SYS_MIN_SHIFT 1
+ #define BQ24190_REG_POC_SYS_MIN_MIN 3000
+@@ -550,7 +551,11 @@ static int bq24190_vbus_is_enabled(struct regulator_dev *dev)
+ pm_runtime_mark_last_busy(bdi->dev);
+ pm_runtime_put_autosuspend(bdi->dev);
+
+- return ret ? ret : val == BQ24190_REG_POC_CHG_CONFIG_OTG;
++ if (ret)
++ return ret;
++
++ return (val == BQ24190_REG_POC_CHG_CONFIG_OTG ||
++ val == BQ24190_REG_POC_CHG_CONFIG_OTG_ALT);
+ }
+
+ static const struct regulator_ops bq24190_vbus_ops = {
+diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c
+index 945c3257ca931..fe814805c68b5 100644
+--- a/drivers/power/supply/bq25890_charger.c
++++ b/drivers/power/supply/bq25890_charger.c
+@@ -581,12 +581,12 @@ static irqreturn_t __bq25890_handle_irq(struct bq25890_device *bq)
+
+ if (!new_state.online && bq->state.online) { /* power removed */
+ /* disable ADC */
+- ret = bq25890_field_write(bq, F_CONV_START, 0);
++ ret = bq25890_field_write(bq, F_CONV_RATE, 0);
+ if (ret < 0)
+ goto error;
+ } else if (new_state.online && !bq->state.online) { /* power inserted */
+ /* enable ADC, to have control of charge current/voltage */
+- ret = bq25890_field_write(bq, F_CONV_START, 1);
++ ret = bq25890_field_write(bq, F_CONV_RATE, 1);
+ if (ret < 0)
+ goto error;
+ }
+diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
+index 46f078350fd3f..cf38cbfe13e9d 100644
+--- a/drivers/power/supply/bq27xxx_battery_i2c.c
++++ b/drivers/power/supply/bq27xxx_battery_i2c.c
+@@ -187,7 +187,8 @@ static int bq27xxx_battery_i2c_probe(struct i2c_client *client,
+ dev_err(&client->dev,
+ "Unable to register IRQ %d error %d\n",
+ client->irq, ret);
+- return ret;
++ bq27xxx_battery_teardown(di);
++ goto err_failed;
+ }
+ }
+
+diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c
+index 3cea92e28dc3e..a9aef1e8b186e 100644
+--- a/drivers/power/supply/max17040_battery.c
++++ b/drivers/power/supply/max17040_battery.c
+@@ -449,6 +449,8 @@ static int max17040_probe(struct i2c_client *client,
+
+ chip->client = client;
+ chip->regmap = devm_regmap_init_i2c(client, &max17040_regmap);
++ if (IS_ERR(chip->regmap))
++ return PTR_ERR(chip->regmap);
+ chip_id = (enum chip_id) id->driver_data;
+ if (client->dev.of_node) {
+ ret = max17040_get_of_data(chip);
+diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c
+index 8dffae76b6a31..aaf6f1bd3d298 100644
+--- a/drivers/power/supply/max17042_battery.c
++++ b/drivers/power/supply/max17042_battery.c
+@@ -313,7 +313,10 @@ static int max17042_get_property(struct power_supply *psy,
+ val->intval = data * 625 / 8;
+ break;
+ case POWER_SUPPLY_PROP_CAPACITY:
+- ret = regmap_read(map, MAX17042_RepSOC, &data);
++ if (chip->pdata->enable_current_sense)
++ ret = regmap_read(map, MAX17042_RepSOC, &data);
++ else
++ ret = regmap_read(map, MAX17042_VFSOC, &data);
+ if (ret < 0)
+ return ret;
+
+@@ -857,7 +860,8 @@ static void max17042_set_soc_threshold(struct max17042_chip *chip, u16 off)
+ regmap_read(map, MAX17042_RepSOC, &soc);
+ soc >>= 8;
+ soc_tr = (soc + off) << 8;
+- soc_tr |= (soc - off);
++ if (off < soc)
++ soc_tr |= soc - off;
+ regmap_write(map, MAX17042_SALRT_Th, soc_tr);
+ }
+
+@@ -876,6 +880,10 @@ static irqreturn_t max17042_thread_handler(int id, void *dev)
+ max17042_set_soc_threshold(chip, 1);
+ }
+
++ /* we implicitly handle all alerts via power_supply_changed */
++ regmap_clear_bits(chip->regmap, MAX17042_STATUS,
++ 0xFFFF & ~(STATUS_POR_BIT | STATUS_BST_BIT));
++
+ power_supply_changed(chip->battery);
+ return IRQ_HANDLED;
+ }
+diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
+index 0c2132c7f5d40..a6e9afa5a1cff 100644
+--- a/drivers/power/supply/power_supply_core.c
++++ b/drivers/power/supply/power_supply_core.c
+@@ -853,6 +853,10 @@ power_supply_find_ocv2cap_table(struct power_supply_battery_info *info,
+ return NULL;
+
+ for (i = 0; i < POWER_SUPPLY_OCV_TEMP_MAX; i++) {
++ /* Out of capacity tables */
++ if (!info->ocv_table[i])
++ break;
++
+ temp_diff = abs(info->ocv_temp[i] - temp);
+
+ if (temp_diff < best_temp_diff) {
+diff --git a/drivers/power/supply/rt5033_battery.c b/drivers/power/supply/rt5033_battery.c
+index 9ad0afe83d1b7..7a23c70f48791 100644
+--- a/drivers/power/supply/rt5033_battery.c
++++ b/drivers/power/supply/rt5033_battery.c
+@@ -60,7 +60,7 @@ static int rt5033_battery_get_watt_prop(struct i2c_client *client,
+ regmap_read(battery->regmap, regh, &msb);
+ regmap_read(battery->regmap, regl, &lsb);
+
+- ret = ((msb << 4) + (lsb >> 4)) * 1250 / 1000;
++ ret = ((msb << 4) + (lsb >> 4)) * 1250;
+
+ return ret;
+ }
+diff --git a/drivers/power/supply/sbs-charger.c b/drivers/power/supply/sbs-charger.c
+index 6fa65d118ec12..b08f7d0c41815 100644
+--- a/drivers/power/supply/sbs-charger.c
++++ b/drivers/power/supply/sbs-charger.c
+@@ -18,6 +18,7 @@
+ #include <linux/interrupt.h>
+ #include <linux/regmap.h>
+ #include <linux/bitops.h>
++#include <linux/devm-helpers.h>
+
+ #define SBS_CHARGER_REG_SPEC_INFO 0x11
+ #define SBS_CHARGER_REG_STATUS 0x13
+@@ -209,7 +210,12 @@ static int sbs_probe(struct i2c_client *client,
+ if (ret)
+ return dev_err_probe(&client->dev, ret, "Failed to request irq\n");
+ } else {
+- INIT_DELAYED_WORK(&chip->work, sbs_delayed_work);
++ ret = devm_delayed_work_autocancel(&client->dev, &chip->work,
++ sbs_delayed_work);
++ if (ret)
++ return dev_err_probe(&client->dev, ret,
++ "Failed to init work for polling\n");
++
+ schedule_delayed_work(&chip->work,
+ msecs_to_jiffies(SBS_CHARGER_POLL_TIME));
+ }
+@@ -220,15 +226,6 @@ static int sbs_probe(struct i2c_client *client,
+ return 0;
+ }
+
+-static int sbs_remove(struct i2c_client *client)
+-{
+- struct sbs_info *chip = i2c_get_clientdata(client);
+-
+- cancel_delayed_work_sync(&chip->work);
+-
+- return 0;
+-}
+-
+ #ifdef CONFIG_OF
+ static const struct of_device_id sbs_dt_ids[] = {
+ { .compatible = "sbs,sbs-charger" },
+@@ -245,7 +242,6 @@ MODULE_DEVICE_TABLE(i2c, sbs_id);
+
+ static struct i2c_driver sbs_driver = {
+ .probe = sbs_probe,
+- .remove = sbs_remove,
+ .id_table = sbs_id,
+ .driver = {
+ .name = "sbs-charger",
+diff --git a/drivers/power/supply/wm8350_power.c b/drivers/power/supply/wm8350_power.c
+index e05cee457471b..908cfd45d2624 100644
+--- a/drivers/power/supply/wm8350_power.c
++++ b/drivers/power/supply/wm8350_power.c
+@@ -408,44 +408,112 @@ static const struct power_supply_desc wm8350_usb_desc = {
+ * Initialisation
+ *********************************************************************/
+
+-static void wm8350_init_charger(struct wm8350 *wm8350)
++static int wm8350_init_charger(struct wm8350 *wm8350)
+ {
++ int ret;
++
+ /* register our interest in charger events */
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_HOT,
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_HOT,
+ wm8350_charger_handler, 0, "Battery hot", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_COLD,
++ if (ret)
++ goto err;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_COLD,
+ wm8350_charger_handler, 0, "Battery cold", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_FAIL,
++ if (ret)
++ goto free_chg_bat_hot;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_BAT_FAIL,
+ wm8350_charger_handler, 0, "Battery fail", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_TO,
++ if (ret)
++ goto free_chg_bat_cold;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_TO,
+ wm8350_charger_handler, 0,
+ "Charger timeout", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_END,
++ if (ret)
++ goto free_chg_bat_fail;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_END,
+ wm8350_charger_handler, 0,
+ "Charge end", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_START,
++ if (ret)
++ goto free_chg_to;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_START,
+ wm8350_charger_handler, 0,
+ "Charge start", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_FAST_RDY,
++ if (ret)
++ goto free_chg_end;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_FAST_RDY,
+ wm8350_charger_handler, 0,
+ "Fast charge ready", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P9,
++ if (ret)
++ goto free_chg_start;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P9,
+ wm8350_charger_handler, 0,
+ "Battery <3.9V", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P1,
++ if (ret)
++ goto free_chg_fast_rdy;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P1,
+ wm8350_charger_handler, 0,
+ "Battery <3.1V", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_2P85,
++ if (ret)
++ goto free_chg_vbatt_lt_3p9;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_2P85,
+ wm8350_charger_handler, 0,
+ "Battery <2.85V", wm8350);
++ if (ret)
++ goto free_chg_vbatt_lt_3p1;
+
+ /* and supply change events */
+- wm8350_register_irq(wm8350, WM8350_IRQ_EXT_USB_FB,
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_EXT_USB_FB,
+ wm8350_charger_handler, 0, "USB", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_EXT_WALL_FB,
++ if (ret)
++ goto free_chg_vbatt_lt_2p85;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_EXT_WALL_FB,
+ wm8350_charger_handler, 0, "Wall", wm8350);
+- wm8350_register_irq(wm8350, WM8350_IRQ_EXT_BAT_FB,
++ if (ret)
++ goto free_ext_usb_fb;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_EXT_BAT_FB,
+ wm8350_charger_handler, 0, "Battery", wm8350);
++ if (ret)
++ goto free_ext_wall_fb;
++
++ return 0;
++
++free_ext_wall_fb:
++ wm8350_free_irq(wm8350, WM8350_IRQ_EXT_WALL_FB, wm8350);
++free_ext_usb_fb:
++ wm8350_free_irq(wm8350, WM8350_IRQ_EXT_USB_FB, wm8350);
++free_chg_vbatt_lt_2p85:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_2P85, wm8350);
++free_chg_vbatt_lt_3p1:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P1, wm8350);
++free_chg_vbatt_lt_3p9:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P9, wm8350);
++free_chg_fast_rdy:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_FAST_RDY, wm8350);
++free_chg_start:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_START, wm8350);
++free_chg_end:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_END, wm8350);
++free_chg_to:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_TO, wm8350);
++free_chg_bat_fail:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_BAT_FAIL, wm8350);
++free_chg_bat_cold:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_BAT_COLD, wm8350);
++free_chg_bat_hot:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_BAT_HOT, wm8350);
++err:
++ return ret;
+ }
+
+ static void free_charger_irq(struct wm8350 *wm8350)
+@@ -456,6 +524,7 @@ static void free_charger_irq(struct wm8350 *wm8350)
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_TO, wm8350);
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_END, wm8350);
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_START, wm8350);
++ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_FAST_RDY, wm8350);
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P9, wm8350);
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_3P1, wm8350);
+ wm8350_free_irq(wm8350, WM8350_IRQ_CHG_VBATT_LT_2P85, wm8350);
+diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c
+index 35799e6401c99..2f4b11b4dfcd9 100644
+--- a/drivers/pps/clients/pps-gpio.c
++++ b/drivers/pps/clients/pps-gpio.c
+@@ -169,7 +169,7 @@ static int pps_gpio_probe(struct platform_device *pdev)
+ /* GPIO setup */
+ ret = pps_gpio_setup(dev);
+ if (ret)
+- return -EINVAL;
++ return ret;
+
+ /* IRQ setup */
+ ret = gpiod_to_irq(data->gpio_pin);
+diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c
+index f9b2d66b04433..8a652a367625b 100644
+--- a/drivers/ptp/ptp_clock.c
++++ b/drivers/ptp/ptp_clock.c
+@@ -317,11 +317,18 @@ no_memory:
+ }
+ EXPORT_SYMBOL(ptp_clock_register);
+
++static int unregister_vclock(struct device *dev, void *data)
++{
++ struct ptp_clock *ptp = dev_get_drvdata(dev);
++
++ ptp_vclock_unregister(info_to_vclock(ptp->info));
++ return 0;
++}
++
+ int ptp_clock_unregister(struct ptp_clock *ptp)
+ {
+ if (ptp_vclock_in_use(ptp)) {
+- pr_err("ptp: virtual clock in use\n");
+- return -EBUSY;
++ device_for_each_child(&ptp->dev, NULL, unregister_vclock);
+ }
+
+ ptp->defunct = 1;
+diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
+index caf9b37c5eb1e..e238ae8e94709 100644
+--- a/drivers/ptp/ptp_ocp.c
++++ b/drivers/ptp/ptp_ocp.c
+@@ -1049,10 +1049,11 @@ ptp_ocp_register_ext(struct ptp_ocp *bp, struct ocp_resource *r)
+ if (!ext)
+ return -ENOMEM;
+
+- err = -EINVAL;
+ ext->mem = ptp_ocp_get_mem(bp, r);
+- if (!ext->mem)
++ if (IS_ERR(ext->mem)) {
++ err = PTR_ERR(ext->mem);
+ goto out;
++ }
+
+ ext->bp = bp;
+ ext->info = r->extra;
+@@ -1122,8 +1123,8 @@ ptp_ocp_register_mem(struct ptp_ocp *bp, struct ocp_resource *r)
+ void __iomem *mem;
+
+ mem = ptp_ocp_get_mem(bp, r);
+- if (!mem)
+- return -EINVAL;
++ if (IS_ERR(mem))
++ return PTR_ERR(mem);
+
+ bp_assign_entry(bp, r, mem);
+
+diff --git a/drivers/ptp/ptp_sysfs.c b/drivers/ptp/ptp_sysfs.c
+index 41b92dc2f011a..9233bfedeb174 100644
+--- a/drivers/ptp/ptp_sysfs.c
++++ b/drivers/ptp/ptp_sysfs.c
+@@ -14,7 +14,7 @@ static ssize_t clock_name_show(struct device *dev,
+ struct device_attribute *attr, char *page)
+ {
+ struct ptp_clock *ptp = dev_get_drvdata(dev);
+- return snprintf(page, PAGE_SIZE-1, "%s\n", ptp->info->name);
++ return sysfs_emit(page, "%s\n", ptp->info->name);
+ }
+ static DEVICE_ATTR_RO(clock_name);
+
+@@ -387,7 +387,7 @@ static ssize_t ptp_pin_show(struct device *dev, struct device_attribute *attr,
+
+ mutex_unlock(&ptp->pincfg_mux);
+
+- return snprintf(page, PAGE_SIZE, "%u %u\n", func, chan);
++ return sysfs_emit(page, "%u %u\n", func, chan);
+ }
+
+ static ssize_t ptp_pin_store(struct device *dev, struct device_attribute *attr,
+diff --git a/drivers/ptp/ptp_vclock.c b/drivers/ptp/ptp_vclock.c
+index baee0379482bc..ab1d233173e13 100644
+--- a/drivers/ptp/ptp_vclock.c
++++ b/drivers/ptp/ptp_vclock.c
+@@ -185,8 +185,8 @@ out:
+ }
+ EXPORT_SYMBOL(ptp_get_vclocks_index);
+
+-void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps,
+- int vclock_index)
++ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps,
++ int vclock_index)
+ {
+ char name[PTP_CLOCK_NAME_LEN] = "";
+ struct ptp_vclock *vclock;
+@@ -198,12 +198,12 @@ void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps,
+ snprintf(name, PTP_CLOCK_NAME_LEN, "ptp%d", vclock_index);
+ dev = class_find_device_by_name(ptp_class, name);
+ if (!dev)
+- return;
++ return 0;
+
+ ptp = dev_get_drvdata(dev);
+ if (!ptp->is_virtual_clock) {
+ put_device(dev);
+- return;
++ return 0;
+ }
+
+ vclock = info_to_vclock(ptp->info);
+@@ -215,7 +215,7 @@ void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps,
+ spin_unlock_irqrestore(&vclock->lock, flags);
+
+ put_device(dev);
+- hwtstamps->hwtstamp = ns_to_ktime(ns);
++ return ns_to_ktime(ns);
+ }
+ EXPORT_SYMBOL(ptp_convert_timestamp);
+ #endif
+diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c
+index 8e461f3baa05a..8cc8ae16553cf 100644
+--- a/drivers/pwm/pwm-lpc18xx-sct.c
++++ b/drivers/pwm/pwm-lpc18xx-sct.c
+@@ -395,12 +395,6 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev)
+ lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_LIMIT,
+ BIT(lpc18xx_pwm->period_event));
+
+- ret = pwmchip_add(&lpc18xx_pwm->chip);
+- if (ret < 0) {
+- dev_err(&pdev->dev, "pwmchip_add failed: %d\n", ret);
+- goto disable_pwmclk;
+- }
+-
+ for (i = 0; i < lpc18xx_pwm->chip.npwm; i++) {
+ struct lpc18xx_pwm_data *data;
+
+@@ -410,14 +404,12 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev)
+ GFP_KERNEL);
+ if (!data) {
+ ret = -ENOMEM;
+- goto remove_pwmchip;
++ goto disable_pwmclk;
+ }
+
+ pwm_set_chip_data(pwm, data);
+ }
+
+- platform_set_drvdata(pdev, lpc18xx_pwm);
+-
+ val = lpc18xx_pwm_readl(lpc18xx_pwm, LPC18XX_PWM_CTRL);
+ val &= ~LPC18XX_PWM_BIDIR;
+ val &= ~LPC18XX_PWM_CTRL_HALT;
+@@ -425,10 +417,16 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev)
+ val |= LPC18XX_PWM_PRE(0);
+ lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CTRL, val);
+
++ ret = pwmchip_add(&lpc18xx_pwm->chip);
++ if (ret < 0) {
++ dev_err(&pdev->dev, "pwmchip_add failed: %d\n", ret);
++ goto disable_pwmclk;
++ }
++
++ platform_set_drvdata(pdev, lpc18xx_pwm);
++
+ return 0;
+
+-remove_pwmchip:
+- pwmchip_remove(&lpc18xx_pwm->chip);
+ disable_pwmclk:
+ clk_disable_unprepare(lpc18xx_pwm->pwm_clk);
+ return ret;
+diff --git a/drivers/regulator/atc260x-regulator.c b/drivers/regulator/atc260x-regulator.c
+index 05147d2c38428..485e58b264c04 100644
+--- a/drivers/regulator/atc260x-regulator.c
++++ b/drivers/regulator/atc260x-regulator.c
+@@ -292,6 +292,7 @@ enum atc2603c_reg_ids {
+ .bypass_mask = BIT(5), \
+ .active_discharge_reg = ATC2603C_PMU_SWITCH_CTL, \
+ .active_discharge_mask = BIT(1), \
++ .active_discharge_on = BIT(1), \
+ .owner = THIS_MODULE, \
+ }
+
+diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
+index ca6caba8a191a..46e76b5b21efe 100644
+--- a/drivers/regulator/core.c
++++ b/drivers/regulator/core.c
+@@ -6010,9 +6010,8 @@ core_initcall(regulator_init);
+ static int regulator_late_cleanup(struct device *dev, void *data)
+ {
+ struct regulator_dev *rdev = dev_to_rdev(dev);
+- const struct regulator_ops *ops = rdev->desc->ops;
+ struct regulation_constraints *c = rdev->constraints;
+- int enabled, ret;
++ int ret;
+
+ if (c && c->always_on)
+ return 0;
+@@ -6025,14 +6024,8 @@ static int regulator_late_cleanup(struct device *dev, void *data)
+ if (rdev->use_count)
+ goto unlock;
+
+- /* If we can't read the status assume it's always on. */
+- if (ops->is_enabled)
+- enabled = ops->is_enabled(rdev);
+- else
+- enabled = 1;
+-
+- /* But if reading the status failed, assume that it's off. */
+- if (enabled <= 0)
++ /* If reading the status failed, assume that it's off. */
++ if (_regulator_is_enabled(rdev) <= 0)
+ goto unlock;
+
+ if (have_full_constraints()) {
+diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c
+index e669250902580..0a4fd449c27d1 100644
+--- a/drivers/regulator/da9121-regulator.c
++++ b/drivers/regulator/da9121-regulator.c
+@@ -253,6 +253,11 @@ static int da9121_set_current_limit(struct regulator_dev *rdev,
+ goto error;
+ }
+
++ if (rdev->desc->ops->is_enabled(rdev)) {
++ ret = -EBUSY;
++ goto error;
++ }
++
+ ret = da9121_ceiling_selector(rdev, min_ua, max_ua, &sel);
+ if (ret < 0)
+ goto error;
+diff --git a/drivers/regulator/qcom-labibb-regulator.c b/drivers/regulator/qcom-labibb-regulator.c
+index b3da0dc58782f..639b71eb41ffe 100644
+--- a/drivers/regulator/qcom-labibb-regulator.c
++++ b/drivers/regulator/qcom-labibb-regulator.c
+@@ -260,7 +260,7 @@ static irqreturn_t qcom_labibb_ocp_isr(int irq, void *chip)
+
+ /* If the regulator is not enabled, this is a fake event */
+ if (!ops->is_enabled(vreg->rdev))
+- return 0;
++ return IRQ_HANDLED;
+
+ /* If we tried to recover for too many times it's not getting better */
+ if (vreg->ocp_irq_count > LABIBB_MAX_OCP_COUNT)
+diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
+index 198fcc6551f6d..b6287f7e78f49 100644
+--- a/drivers/regulator/qcom_smd-regulator.c
++++ b/drivers/regulator/qcom_smd-regulator.c
+@@ -9,6 +9,7 @@
+ #include <linux/of_device.h>
+ #include <linux/platform_device.h>
+ #include <linux/regulator/driver.h>
++#include <linux/regulator/of_regulator.h>
+ #include <linux/soc/qcom/smd-rpm.h>
+
+ struct qcom_rpm_reg {
+@@ -1190,52 +1191,93 @@ static const struct of_device_id rpm_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, rpm_of_match);
+
+-static int rpm_reg_probe(struct platform_device *pdev)
++/**
++ * rpm_regulator_init_vreg() - initialize all attributes of a qcom_smd-regulator
++ * @vreg: Pointer to the individual qcom_smd-regulator resource
++ * @dev: Pointer to the top level qcom_smd-regulator PMIC device
++ * @node: Pointer to the individual qcom_smd-regulator resource
++ * device node
++ * @rpm: Pointer to the rpm bus node
++ * @pmic_rpm_data: Pointer to a null-terminated array of qcom_smd-regulator
++ * resources defined for the top level PMIC device
++ *
++ * Return: 0 on success, errno on failure
++ */
++static int rpm_regulator_init_vreg(struct qcom_rpm_reg *vreg, struct device *dev,
++ struct device_node *node, struct qcom_smd_rpm *rpm,
++ const struct rpm_regulator_data *pmic_rpm_data)
+ {
+- const struct rpm_regulator_data *reg;
+- const struct of_device_id *match;
+- struct regulator_config config = { };
++ struct regulator_config config = {};
++ const struct rpm_regulator_data *rpm_data;
+ struct regulator_dev *rdev;
++ int ret;
++
++ for (rpm_data = pmic_rpm_data; rpm_data->name; rpm_data++)
++ if (of_node_name_eq(node, rpm_data->name))
++ break;
++
++ if (!rpm_data->name) {
++ dev_err(dev, "Unknown regulator %pOFn\n", node);
++ return -EINVAL;
++ }
++
++ vreg->dev = dev;
++ vreg->rpm = rpm;
++ vreg->type = rpm_data->type;
++ vreg->id = rpm_data->id;
++
++ memcpy(&vreg->desc, rpm_data->desc, sizeof(vreg->desc));
++ vreg->desc.name = rpm_data->name;
++ vreg->desc.supply_name = rpm_data->supply;
++ vreg->desc.owner = THIS_MODULE;
++ vreg->desc.type = REGULATOR_VOLTAGE;
++ vreg->desc.of_match = rpm_data->name;
++
++ config.dev = dev;
++ config.of_node = node;
++ config.driver_data = vreg;
++
++ rdev = devm_regulator_register(dev, &vreg->desc, &config);
++ if (IS_ERR(rdev)) {
++ ret = PTR_ERR(rdev);
++ dev_err(dev, "%pOFn: devm_regulator_register() failed, ret=%d\n", node, ret);
++ return ret;
++ }
++
++ return 0;
++}
++
++static int rpm_reg_probe(struct platform_device *pdev)
++{
++ struct device *dev = &pdev->dev;
++ const struct rpm_regulator_data *vreg_data;
++ struct device_node *node;
+ struct qcom_rpm_reg *vreg;
+ struct qcom_smd_rpm *rpm;
++ int ret;
+
+ rpm = dev_get_drvdata(pdev->dev.parent);
+ if (!rpm) {
+- dev_err(&pdev->dev, "unable to retrieve handle to rpm\n");
++ dev_err(&pdev->dev, "Unable to retrieve handle to rpm\n");
+ return -ENODEV;
+ }
+
+- match = of_match_device(rpm_of_match, &pdev->dev);
+- if (!match) {
+- dev_err(&pdev->dev, "failed to match device\n");
++ vreg_data = of_device_get_match_data(dev);
++ if (!vreg_data)
+ return -ENODEV;
+- }
+
+- for (reg = match->data; reg->name; reg++) {
++ for_each_available_child_of_node(dev->of_node, node) {
+ vreg = devm_kzalloc(&pdev->dev, sizeof(*vreg), GFP_KERNEL);
+- if (!vreg)
++ if (!vreg) {
++ of_node_put(node);
+ return -ENOMEM;
++ }
++
++ ret = rpm_regulator_init_vreg(vreg, dev, node, rpm, vreg_data);
+
+- vreg->dev = &pdev->dev;
+- vreg->type = reg->type;
+- vreg->id = reg->id;
+- vreg->rpm = rpm;
+-
+- memcpy(&vreg->desc, reg->desc, sizeof(vreg->desc));
+-
+- vreg->desc.id = -1;
+- vreg->desc.owner = THIS_MODULE;
+- vreg->desc.type = REGULATOR_VOLTAGE;
+- vreg->desc.name = reg->name;
+- vreg->desc.supply_name = reg->supply;
+- vreg->desc.of_match = reg->name;
+-
+- config.dev = &pdev->dev;
+- config.driver_data = vreg;
+- rdev = devm_regulator_register(&pdev->dev, &vreg->desc, &config);
+- if (IS_ERR(rdev)) {
+- dev_err(&pdev->dev, "failed to register %s\n", reg->name);
+- return PTR_ERR(rdev);
++ if (ret < 0) {
++ of_node_put(node);
++ return ret;
+ }
+ }
+
+diff --git a/drivers/regulator/rpi-panel-attiny-regulator.c b/drivers/regulator/rpi-panel-attiny-regulator.c
+index ee46bfbf5eee7..991b4730d7687 100644
+--- a/drivers/regulator/rpi-panel-attiny-regulator.c
++++ b/drivers/regulator/rpi-panel-attiny-regulator.c
+@@ -37,11 +37,24 @@ static const struct regmap_config attiny_regmap_config = {
+ static int attiny_lcd_power_enable(struct regulator_dev *rdev)
+ {
+ unsigned int data;
++ int ret, i;
+
+ regmap_write(rdev->regmap, REG_POWERON, 1);
++ msleep(80);
++
+ /* Wait for nPWRDWN to go low to indicate poweron is done. */
+- regmap_read_poll_timeout(rdev->regmap, REG_PORTB, data,
+- data & BIT(0), 10, 1000000);
++ for (i = 0; i < 20; i++) {
++ ret = regmap_read(rdev->regmap, REG_PORTB, &data);
++ if (!ret) {
++ if (data & BIT(0))
++ break;
++ }
++ usleep_range(10000, 12000);
++ }
++ usleep_range(10000, 12000);
++
++ if (ret)
++ pr_err("%s: regmap_read_poll_timeout failed %d\n", __func__, ret);
+
+ /* Default to the same orientation as the closed source
+ * firmware used for the panel. Runtime rotation
+@@ -57,23 +70,34 @@ static int attiny_lcd_power_disable(struct regulator_dev *rdev)
+ {
+ regmap_write(rdev->regmap, REG_PWM, 0);
+ regmap_write(rdev->regmap, REG_POWERON, 0);
+- udelay(1);
++ msleep(30);
+ return 0;
+ }
+
+ static int attiny_lcd_power_is_enabled(struct regulator_dev *rdev)
+ {
+ unsigned int data;
+- int ret;
++ int ret, i;
+
+- ret = regmap_read(rdev->regmap, REG_POWERON, &data);
++ for (i = 0; i < 10; i++) {
++ ret = regmap_read(rdev->regmap, REG_POWERON, &data);
++ if (!ret)
++ break;
++ usleep_range(10000, 12000);
++ }
+ if (ret < 0)
+ return ret;
+
+ if (!(data & BIT(0)))
+ return 0;
+
+- ret = regmap_read(rdev->regmap, REG_PORTB, &data);
++ for (i = 0; i < 10; i++) {
++ ret = regmap_read(rdev->regmap, REG_PORTB, &data);
++ if (!ret)
++ break;
++ usleep_range(10000, 12000);
++ }
++
+ if (ret < 0)
+ return ret;
+
+@@ -103,20 +127,32 @@ static int attiny_update_status(struct backlight_device *bl)
+ {
+ struct regmap *regmap = bl_get_data(bl);
+ int brightness = bl->props.brightness;
++ int ret, i;
+
+ if (bl->props.power != FB_BLANK_UNBLANK ||
+ bl->props.fb_blank != FB_BLANK_UNBLANK)
+ brightness = 0;
+
+- return regmap_write(regmap, REG_PWM, brightness);
++ for (i = 0; i < 10; i++) {
++ ret = regmap_write(regmap, REG_PWM, brightness);
++ if (!ret)
++ break;
++ }
++
++ return ret;
+ }
+
+ static int attiny_get_brightness(struct backlight_device *bl)
+ {
+ struct regmap *regmap = bl_get_data(bl);
+- int ret, brightness;
++ int ret, brightness, i;
++
++ for (i = 0; i < 10; i++) {
++ ret = regmap_read(regmap, REG_PWM, &brightness);
++ if (!ret)
++ break;
++ }
+
+- ret = regmap_read(regmap, REG_PWM, &brightness);
+ if (ret)
+ return ret;
+
+@@ -166,7 +202,7 @@ static int attiny_i2c_probe(struct i2c_client *i2c,
+ }
+
+ regmap_write(regmap, REG_POWERON, 0);
+- mdelay(1);
++ msleep(30);
+
+ config.dev = &i2c->dev;
+ config.regmap = regmap;
+diff --git a/drivers/regulator/rtq2134-regulator.c b/drivers/regulator/rtq2134-regulator.c
+index f21e3f8b21f23..8e13dea354a21 100644
+--- a/drivers/regulator/rtq2134-regulator.c
++++ b/drivers/regulator/rtq2134-regulator.c
+@@ -285,6 +285,7 @@ static const unsigned int rtq2134_buck_ramp_delay_table[] = {
+ .enable_mask = RTQ2134_VOUTEN_MASK, \
+ .active_discharge_reg = RTQ2134_REG_BUCK##_id##_CFG0, \
+ .active_discharge_mask = RTQ2134_ACTDISCHG_MASK, \
++ .active_discharge_on = RTQ2134_ACTDISCHG_MASK, \
+ .ramp_reg = RTQ2134_REG_BUCK##_id##_RSPCFG, \
+ .ramp_mask = RTQ2134_RSPUP_MASK, \
+ .ramp_delay_table = rtq2134_buck_ramp_delay_table, \
+diff --git a/drivers/regulator/s5m8767.c b/drivers/regulator/s5m8767.c
+index 7c111bbdc2afa..35269f9982105 100644
+--- a/drivers/regulator/s5m8767.c
++++ b/drivers/regulator/s5m8767.c
+@@ -850,18 +850,15 @@ static int s5m8767_pmic_probe(struct platform_device *pdev)
+ /* DS4 GPIO */
+ gpio_direction_output(pdata->buck_ds[2], 0x0);
+
+- if (pdata->buck2_gpiodvs || pdata->buck3_gpiodvs ||
+- pdata->buck4_gpiodvs) {
+- regmap_update_bits(s5m8767->iodev->regmap_pmic,
+- S5M8767_REG_BUCK2CTRL, 1 << 1,
+- (pdata->buck2_gpiodvs) ? (1 << 1) : (0 << 1));
+- regmap_update_bits(s5m8767->iodev->regmap_pmic,
+- S5M8767_REG_BUCK3CTRL, 1 << 1,
+- (pdata->buck3_gpiodvs) ? (1 << 1) : (0 << 1));
+- regmap_update_bits(s5m8767->iodev->regmap_pmic,
+- S5M8767_REG_BUCK4CTRL, 1 << 1,
+- (pdata->buck4_gpiodvs) ? (1 << 1) : (0 << 1));
+- }
++ regmap_update_bits(s5m8767->iodev->regmap_pmic,
++ S5M8767_REG_BUCK2CTRL, 1 << 1,
++ (pdata->buck2_gpiodvs) ? (1 << 1) : (0 << 1));
++ regmap_update_bits(s5m8767->iodev->regmap_pmic,
++ S5M8767_REG_BUCK3CTRL, 1 << 1,
++ (pdata->buck3_gpiodvs) ? (1 << 1) : (0 << 1));
++ regmap_update_bits(s5m8767->iodev->regmap_pmic,
++ S5M8767_REG_BUCK4CTRL, 1 << 1,
++ (pdata->buck4_gpiodvs) ? (1 << 1) : (0 << 1));
+
+ /* Initialize GPIO DVS registers */
+ for (i = 0; i < 8; i++) {
+diff --git a/drivers/regulator/wm8994-regulator.c b/drivers/regulator/wm8994-regulator.c
+index cadea0344486f..40befdd9dfa92 100644
+--- a/drivers/regulator/wm8994-regulator.c
++++ b/drivers/regulator/wm8994-regulator.c
+@@ -71,6 +71,35 @@ static const struct regulator_ops wm8994_ldo2_ops = {
+ };
+
+ static const struct regulator_desc wm8994_ldo_desc[] = {
++ {
++ .name = "LDO1",
++ .id = 1,
++ .type = REGULATOR_VOLTAGE,
++ .n_voltages = WM8994_LDO1_MAX_SELECTOR + 1,
++ .vsel_reg = WM8994_LDO_1,
++ .vsel_mask = WM8994_LDO1_VSEL_MASK,
++ .ops = &wm8994_ldo1_ops,
++ .min_uV = 2400000,
++ .uV_step = 100000,
++ .enable_time = 3000,
++ .off_on_delay = 36000,
++ .owner = THIS_MODULE,
++ },
++ {
++ .name = "LDO2",
++ .id = 2,
++ .type = REGULATOR_VOLTAGE,
++ .n_voltages = WM8994_LDO2_MAX_SELECTOR + 1,
++ .vsel_reg = WM8994_LDO_2,
++ .vsel_mask = WM8994_LDO2_VSEL_MASK,
++ .ops = &wm8994_ldo2_ops,
++ .enable_time = 3000,
++ .off_on_delay = 36000,
++ .owner = THIS_MODULE,
++ },
++};
++
++static const struct regulator_desc wm8958_ldo_desc[] = {
+ {
+ .name = "LDO1",
+ .id = 1,
+@@ -172,9 +201,16 @@ static int wm8994_ldo_probe(struct platform_device *pdev)
+ * regulator core and we need not worry about it on the
+ * error path.
+ */
+- ldo->regulator = devm_regulator_register(&pdev->dev,
+- &wm8994_ldo_desc[id],
+- &config);
++ if (ldo->wm8994->type == WM8994) {
++ ldo->regulator = devm_regulator_register(&pdev->dev,
++ &wm8994_ldo_desc[id],
++ &config);
++ } else {
++ ldo->regulator = devm_regulator_register(&pdev->dev,
++ &wm8958_ldo_desc[id],
++ &config);
++ }
++
+ if (IS_ERR(ldo->regulator)) {
+ ret = PTR_ERR(ldo->regulator);
+ dev_err(wm8994->dev, "Failed to register LDO%d: %d\n",
+diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
+index d88f76f5305eb..05c39e1c56b49 100644
+--- a/drivers/remoteproc/imx_rproc.c
++++ b/drivers/remoteproc/imx_rproc.c
+@@ -71,6 +71,7 @@ struct imx_rproc_mem {
+ /* att flags */
+ /* M4 own area. Can be mapped at probe */
+ #define ATT_OWN BIT(1)
++#define ATT_IOMEM BIT(2)
+
+ /* address translation table */
+ struct imx_rproc_att {
+@@ -117,7 +118,7 @@ struct imx_rproc {
+ static const struct imx_rproc_att imx_rproc_att_imx8mn[] = {
+ /* dev addr , sys addr , size , flags */
+ /* ITCM */
+- { 0x00000000, 0x007E0000, 0x00020000, ATT_OWN },
++ { 0x00000000, 0x007E0000, 0x00020000, ATT_OWN | ATT_IOMEM },
+ /* OCRAM_S */
+ { 0x00180000, 0x00180000, 0x00009000, 0 },
+ /* OCRAM */
+@@ -131,7 +132,7 @@ static const struct imx_rproc_att imx_rproc_att_imx8mn[] = {
+ /* DDR (Code) - alias */
+ { 0x10000000, 0x40000000, 0x0FFE0000, 0 },
+ /* DTCM */
+- { 0x20000000, 0x00800000, 0x00020000, ATT_OWN },
++ { 0x20000000, 0x00800000, 0x00020000, ATT_OWN | ATT_IOMEM },
+ /* OCRAM_S - alias */
+ { 0x20180000, 0x00180000, 0x00008000, ATT_OWN },
+ /* OCRAM */
+@@ -147,7 +148,7 @@ static const struct imx_rproc_att imx_rproc_att_imx8mn[] = {
+ static const struct imx_rproc_att imx_rproc_att_imx8mq[] = {
+ /* dev addr , sys addr , size , flags */
+ /* TCML - alias */
+- { 0x00000000, 0x007e0000, 0x00020000, 0 },
++ { 0x00000000, 0x007e0000, 0x00020000, ATT_IOMEM},
+ /* OCRAM_S */
+ { 0x00180000, 0x00180000, 0x00008000, 0 },
+ /* OCRAM */
+@@ -159,9 +160,9 @@ static const struct imx_rproc_att imx_rproc_att_imx8mq[] = {
+ /* DDR (Code) - alias */
+ { 0x10000000, 0x80000000, 0x0FFE0000, 0 },
+ /* TCML */
+- { 0x1FFE0000, 0x007E0000, 0x00020000, ATT_OWN },
++ { 0x1FFE0000, 0x007E0000, 0x00020000, ATT_OWN | ATT_IOMEM},
+ /* TCMU */
+- { 0x20000000, 0x00800000, 0x00020000, ATT_OWN },
++ { 0x20000000, 0x00800000, 0x00020000, ATT_OWN | ATT_IOMEM},
+ /* OCRAM_S */
+ { 0x20180000, 0x00180000, 0x00008000, ATT_OWN },
+ /* OCRAM */
+@@ -199,12 +200,12 @@ static const struct imx_rproc_att imx_rproc_att_imx7d[] = {
+ /* OCRAM_PXP (Code) - alias */
+ { 0x00940000, 0x00940000, 0x00008000, 0 },
+ /* TCML (Code) */
+- { 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN },
++ { 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN | ATT_IOMEM },
+ /* DDR (Code) - alias, first part of DDR (Data) */
+ { 0x10000000, 0x80000000, 0x0FFF0000, 0 },
+
+ /* TCMU (Data) */
+- { 0x20000000, 0x00800000, 0x00008000, ATT_OWN },
++ { 0x20000000, 0x00800000, 0x00008000, ATT_OWN | ATT_IOMEM },
+ /* OCRAM (Data) */
+ { 0x20200000, 0x00900000, 0x00020000, 0 },
+ /* OCRAM_EPDC (Data) */
+@@ -218,18 +219,18 @@ static const struct imx_rproc_att imx_rproc_att_imx7d[] = {
+ static const struct imx_rproc_att imx_rproc_att_imx6sx[] = {
+ /* dev addr , sys addr , size , flags */
+ /* TCML (M4 Boot Code) - alias */
+- { 0x00000000, 0x007F8000, 0x00008000, 0 },
++ { 0x00000000, 0x007F8000, 0x00008000, ATT_IOMEM },
+ /* OCRAM_S (Code) */
+ { 0x00180000, 0x008F8000, 0x00004000, 0 },
+ /* OCRAM_S (Code) - alias */
+ { 0x00180000, 0x008FC000, 0x00004000, 0 },
+ /* TCML (Code) */
+- { 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN },
++ { 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN | ATT_IOMEM },
+ /* DDR (Code) - alias, first part of DDR (Data) */
+ { 0x10000000, 0x80000000, 0x0FFF8000, 0 },
+
+ /* TCMU (Data) */
+- { 0x20000000, 0x00800000, 0x00008000, ATT_OWN },
++ { 0x20000000, 0x00800000, 0x00008000, ATT_OWN | ATT_IOMEM },
+ /* OCRAM_S (Data) - alias? */
+ { 0x208F8000, 0x008F8000, 0x00004000, 0 },
+ /* DDR (Data) */
+@@ -341,7 +342,7 @@ static int imx_rproc_stop(struct rproc *rproc)
+ }
+
+ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
+- size_t len, u64 *sys)
++ size_t len, u64 *sys, bool *is_iomem)
+ {
+ const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+ int i;
+@@ -354,6 +355,8 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
+ unsigned int offset = da - att->da;
+
+ *sys = att->sa + offset;
++ if (is_iomem)
++ *is_iomem = att->flags & ATT_IOMEM;
+ return 0;
+ }
+ }
+@@ -377,7 +380,7 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *i
+ * On device side we have many aliases, so we need to convert device
+ * address (M4) to system bus address first.
+ */
+- if (imx_rproc_da_to_sys(priv, da, len, &sys))
++ if (imx_rproc_da_to_sys(priv, da, len, &sys, is_iomem))
+ return NULL;
+
+ for (i = 0; i < IMX_RPROC_MEM_MAX; i++) {
+@@ -553,8 +556,12 @@ static int imx_rproc_addr_init(struct imx_rproc *priv,
+ if (b >= IMX_RPROC_MEM_MAX)
+ break;
+
+- priv->mem[b].cpu_addr = devm_ioremap(&pdev->dev,
+- att->sa, att->size);
++ if (att->flags & ATT_IOMEM)
++ priv->mem[b].cpu_addr = devm_ioremap(&pdev->dev,
++ att->sa, att->size);
++ else
++ priv->mem[b].cpu_addr = devm_ioremap_wc(&pdev->dev,
++ att->sa, att->size);
+ if (!priv->mem[b].cpu_addr) {
+ dev_err(dev, "failed to remap %#x bytes from %#x\n", att->size, att->sa);
+ return -ENOMEM;
+@@ -575,8 +582,8 @@ static int imx_rproc_addr_init(struct imx_rproc *priv,
+ struct resource res;
+
+ node = of_parse_phandle(np, "memory-region", a);
+- /* Not map vdev region */
+- if (!strcmp(node->name, "vdev"))
++ /* Not map vdevbuffer, vdevring region */
++ if (!strncmp(node->name, "vdev", strlen("vdev")))
+ continue;
+ err = of_address_to_resource(node, 0, &res);
+ if (err) {
+@@ -597,7 +604,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv,
+ }
+ priv->mem[b].sys_addr = res.start;
+ priv->mem[b].size = resource_size(&res);
+- if (!strcmp(node->name, "rsc_table"))
++ if (!strcmp(node->name, "rsc-table"))
+ priv->rsc_table = priv->mem[b].cpu_addr;
+ b++;
+ }
+@@ -823,6 +830,7 @@ static int imx_rproc_remove(struct platform_device *pdev)
+ clk_disable_unprepare(priv->clk);
+ rproc_del(rproc);
+ imx_rproc_free_mbox(rproc);
++ destroy_workqueue(priv->workqueue);
+ rproc_free(rproc);
+
+ return 0;
+diff --git a/drivers/remoteproc/qcom_pil_info.c b/drivers/remoteproc/qcom_pil_info.c
+index 7c007dd7b2000..aca21560e20b8 100644
+--- a/drivers/remoteproc/qcom_pil_info.c
++++ b/drivers/remoteproc/qcom_pil_info.c
+@@ -104,7 +104,7 @@ int qcom_pil_info_store(const char *image, phys_addr_t base, size_t size)
+ return -ENOMEM;
+
+ found_unused:
+- memcpy_toio(entry, image, PIL_RELOC_NAME_LEN);
++ memcpy_toio(entry, image, strnlen(image, PIL_RELOC_NAME_LEN));
+ found_existing:
+ /* Use two writel() as base is only aligned to 4 bytes on odd entries */
+ writel(base, entry + PIL_RELOC_NAME_LEN);
+diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c
+index 8b0d8bbacd2e4..c10d452c97032 100644
+--- a/drivers/remoteproc/qcom_q6v5_adsp.c
++++ b/drivers/remoteproc/qcom_q6v5_adsp.c
+@@ -406,6 +406,7 @@ static int adsp_alloc_memory_region(struct qcom_adsp *adsp)
+ }
+
+ ret = of_address_to_resource(node, 0, &r);
++ of_node_put(node);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
+index 423b31dfa5741..ca1c7387776b5 100644
+--- a/drivers/remoteproc/qcom_q6v5_mss.c
++++ b/drivers/remoteproc/qcom_q6v5_mss.c
+@@ -1624,18 +1624,20 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc)
+ * reserved memory regions from device's memory-region property.
+ */
+ child = of_get_child_by_name(qproc->dev->of_node, "mba");
+- if (!child)
++ if (!child) {
+ node = of_parse_phandle(qproc->dev->of_node,
+ "memory-region", 0);
+- else
++ } else {
+ node = of_parse_phandle(child, "memory-region", 0);
++ of_node_put(child);
++ }
+
+ ret = of_address_to_resource(node, 0, &r);
++ of_node_put(node);
+ if (ret) {
+ dev_err(qproc->dev, "unable to resolve mba region\n");
+ return ret;
+ }
+- of_node_put(node);
+
+ qproc->mba_phys = r.start;
+ qproc->mba_size = resource_size(&r);
+@@ -1646,14 +1648,15 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc)
+ } else {
+ child = of_get_child_by_name(qproc->dev->of_node, "mpss");
+ node = of_parse_phandle(child, "memory-region", 0);
++ of_node_put(child);
+ }
+
+ ret = of_address_to_resource(node, 0, &r);
++ of_node_put(node);
+ if (ret) {
+ dev_err(qproc->dev, "unable to resolve mpss region\n");
+ return ret;
+ }
+- of_node_put(node);
+
+ qproc->mpss_phys = qproc->mpss_reloc = r.start;
+ qproc->mpss_size = resource_size(&r);
+diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c
+index 401b1ec907852..699eaac5b760e 100644
+--- a/drivers/remoteproc/qcom_q6v5_pas.c
++++ b/drivers/remoteproc/qcom_q6v5_pas.c
+@@ -661,6 +661,7 @@ static const struct adsp_data sm8350_cdsp_resource = {
+ },
+ .proxy_pd_names = (char*[]){
+ "cx",
++ "mxc",
+ NULL
+ },
+ .ssr_name = "cdsp",
+diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
+index ebadc6c08e116..b17742eac9ffd 100644
+--- a/drivers/remoteproc/qcom_wcnss.c
++++ b/drivers/remoteproc/qcom_wcnss.c
+@@ -501,6 +501,7 @@ static int wcnss_alloc_memory_region(struct qcom_wcnss *wcnss)
+ }
+
+ ret = of_address_to_resource(node, 0, &r);
++ of_node_put(node);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
+index 502b6604b757b..775df165eb450 100644
+--- a/drivers/remoteproc/remoteproc_core.c
++++ b/drivers/remoteproc/remoteproc_core.c
+@@ -556,9 +556,6 @@ static int rproc_handle_vdev(struct rproc *rproc, void *ptr,
+ /* Initialise vdev subdevice */
+ snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index);
+ rvdev->dev.parent = &rproc->dev;
+- ret = copy_dma_range_map(&rvdev->dev, rproc->dev.parent);
+- if (ret)
+- return ret;
+ rvdev->dev.release = rproc_rvdev_release;
+ dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev->dev.parent), name);
+ dev_set_drvdata(&rvdev->dev, rvdev);
+@@ -568,6 +565,11 @@ static int rproc_handle_vdev(struct rproc *rproc, void *ptr,
+ put_device(&rvdev->dev);
+ return ret;
+ }
++
++ ret = copy_dma_range_map(&rvdev->dev, rproc->dev.parent);
++ if (ret)
++ goto free_rvdev;
++
+ /* Make device dma capable by inheriting from parent's capabilities */
+ set_dma_ops(&rvdev->dev, get_dma_ops(rproc->dev.parent));
+
+diff --git a/drivers/remoteproc/remoteproc_coredump.c b/drivers/remoteproc/remoteproc_coredump.c
+index aee657cc08c6a..c892f433a323e 100644
+--- a/drivers/remoteproc/remoteproc_coredump.c
++++ b/drivers/remoteproc/remoteproc_coredump.c
+@@ -152,8 +152,8 @@ static void rproc_copy_segment(struct rproc *rproc, void *dest,
+ struct rproc_dump_segment *segment,
+ size_t offset, size_t size)
+ {
++ bool is_iomem = false;
+ void *ptr;
+- bool is_iomem;
+
+ if (segment->dump) {
+ segment->dump(rproc, segment, dest, offset, size);
+diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
+index b5a1e3b697d9f..581930483ef84 100644
+--- a/drivers/remoteproc/remoteproc_debugfs.c
++++ b/drivers/remoteproc/remoteproc_debugfs.c
+@@ -76,7 +76,7 @@ static ssize_t rproc_coredump_write(struct file *filp,
+ int ret, err = 0;
+ char buf[20];
+
+- if (count > sizeof(buf))
++ if (count < 1 || count > sizeof(buf))
+ return -EINVAL;
+
+ ret = copy_from_user(buf, user_buf, count);
+diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c
+index 469c52e62faff..d635d19a5aa8a 100644
+--- a/drivers/remoteproc/remoteproc_elf_loader.c
++++ b/drivers/remoteproc/remoteproc_elf_loader.c
+@@ -178,8 +178,8 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
+ u64 filesz = elf_phdr_get_p_filesz(class, phdr);
+ u64 offset = elf_phdr_get_p_offset(class, phdr);
+ u32 type = elf_phdr_get_p_type(class, phdr);
++ bool is_iomem = false;
+ void *ptr;
+- bool is_iomem;
+
+ if (type != PT_LOAD)
+ continue;
+@@ -220,7 +220,7 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
+ /* put the segment where the remote processor expects it */
+ if (filesz) {
+ if (is_iomem)
+- memcpy_fromio(ptr, (void __iomem *)(elf_data + offset), filesz);
++ memcpy_toio((void __iomem *)ptr, elf_data + offset, filesz);
+ else
+ memcpy(ptr, elf_data + offset, filesz);
+ }
+diff --git a/drivers/reset/reset-rzg2l-usbphy-ctrl.c b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
+index e0704fd2b5336..a8dde46063602 100644
+--- a/drivers/reset/reset-rzg2l-usbphy-ctrl.c
++++ b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
+@@ -121,7 +121,9 @@ static int rzg2l_usbphy_ctrl_probe(struct platform_device *pdev)
+ return dev_err_probe(dev, PTR_ERR(priv->rstc),
+ "failed to get reset\n");
+
+- reset_control_deassert(priv->rstc);
++ error = reset_control_deassert(priv->rstc);
++ if (error)
++ return error;
+
+ priv->rcdev.ops = &rzg2l_usbphy_ctrl_reset_ops;
+ priv->rcdev.of_reset_n_cells = 1;
+@@ -137,7 +139,12 @@ static int rzg2l_usbphy_ctrl_probe(struct platform_device *pdev)
+ dev_set_drvdata(dev, priv);
+
+ pm_runtime_enable(&pdev->dev);
+- pm_runtime_resume_and_get(&pdev->dev);
++ error = pm_runtime_resume_and_get(&pdev->dev);
++ if (error < 0) {
++ pm_runtime_disable(&pdev->dev);
++ reset_control_assert(priv->rstc);
++ return dev_err_probe(&pdev->dev, error, "pm_runtime_resume_and_get failed");
++ }
+
+ /* put pll and phy into reset state */
+ spin_lock_irqsave(&priv->lock, flags);
+diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c
+index 2bebc9b2d1634..49dd5a2009982 100644
+--- a/drivers/rpmsg/rpmsg_char.c
++++ b/drivers/rpmsg/rpmsg_char.c
+@@ -92,7 +92,7 @@ static int rpmsg_eptdev_destroy(struct device *dev, void *data)
+ /* wake up any blocked readers */
+ wake_up_interruptible(&eptdev->readq);
+
+- device_del(&eptdev->dev);
++ cdev_device_del(&eptdev->cdev, &eptdev->dev);
+ put_device(&eptdev->dev);
+
+ return 0;
+@@ -335,7 +335,6 @@ static void rpmsg_eptdev_release_device(struct device *dev)
+
+ ida_simple_remove(&rpmsg_ept_ida, dev->id);
+ ida_simple_remove(&rpmsg_minor_ida, MINOR(eptdev->dev.devt));
+- cdev_del(&eptdev->cdev);
+ kfree(eptdev);
+ }
+
+@@ -380,19 +379,13 @@ static int rpmsg_eptdev_create(struct rpmsg_ctrldev *ctrldev,
+ dev->id = ret;
+ dev_set_name(dev, "rpmsg%d", ret);
+
+- ret = cdev_add(&eptdev->cdev, dev->devt, 1);
++ ret = cdev_device_add(&eptdev->cdev, &eptdev->dev);
+ if (ret)
+ goto free_ept_ida;
+
+ /* We can now rely on the release function for cleanup */
+ dev->release = rpmsg_eptdev_release_device;
+
+- ret = device_add(dev);
+- if (ret) {
+- dev_err(dev, "device_add failed: %d\n", ret);
+- put_device(dev);
+- }
+-
+ return ret;
+
+ free_ept_ida:
+@@ -461,7 +454,6 @@ static void rpmsg_ctrldev_release_device(struct device *dev)
+
+ ida_simple_remove(&rpmsg_ctrl_ida, dev->id);
+ ida_simple_remove(&rpmsg_minor_ida, MINOR(dev->devt));
+- cdev_del(&ctrldev->cdev);
+ kfree(ctrldev);
+ }
+
+@@ -496,19 +488,13 @@ static int rpmsg_chrdev_probe(struct rpmsg_device *rpdev)
+ dev->id = ret;
+ dev_set_name(&ctrldev->dev, "rpmsg_ctrl%d", ret);
+
+- ret = cdev_add(&ctrldev->cdev, dev->devt, 1);
++ ret = cdev_device_add(&ctrldev->cdev, &ctrldev->dev);
+ if (ret)
+ goto free_ctrl_ida;
+
+ /* We can now rely on the release function for cleanup */
+ dev->release = rpmsg_ctrldev_release_device;
+
+- ret = device_add(dev);
+- if (ret) {
+- dev_err(&rpdev->dev, "device_add failed: %d\n", ret);
+- put_device(dev);
+- }
+-
+ dev_set_drvdata(&rpdev->dev, ctrldev);
+
+ return ret;
+@@ -534,7 +520,7 @@ static void rpmsg_chrdev_remove(struct rpmsg_device *rpdev)
+ if (ret)
+ dev_warn(&rpdev->dev, "failed to nuke endpoints: %d\n", ret);
+
+- device_del(&ctrldev->dev);
++ cdev_device_del(&ctrldev->cdev, &ctrldev->dev);
+ put_device(&ctrldev->dev);
+ }
+
+diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
+index 9151836190ce3..a71de08acc7b9 100644
+--- a/drivers/rpmsg/rpmsg_core.c
++++ b/drivers/rpmsg/rpmsg_core.c
+@@ -519,13 +519,25 @@ static int rpmsg_dev_probe(struct device *dev)
+ err = rpdrv->probe(rpdev);
+ if (err) {
+ dev_err(dev, "%s: failed: %d\n", __func__, err);
+- if (ept)
+- rpmsg_destroy_ept(ept);
+- goto out;
++ goto destroy_ept;
+ }
+
+- if (ept && rpdev->ops->announce_create)
++ if (ept && rpdev->ops->announce_create) {
+ err = rpdev->ops->announce_create(rpdev);
++ if (err) {
++ dev_err(dev, "failed to announce creation\n");
++ goto remove_rpdev;
++ }
++ }
++
++ return 0;
++
++remove_rpdev:
++ if (rpdrv->remove)
++ rpdrv->remove(rpdev);
++destroy_ept:
++ if (ept)
++ rpmsg_destroy_ept(ept);
+ out:
+ return err;
+ }
+diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c
+index f77bc089eb6b7..0aef7df2ea704 100644
+--- a/drivers/rtc/class.c
++++ b/drivers/rtc/class.c
+@@ -26,6 +26,15 @@ struct class *rtc_class;
+ static void rtc_device_release(struct device *dev)
+ {
+ struct rtc_device *rtc = to_rtc_device(dev);
++ struct timerqueue_head *head = &rtc->timerqueue;
++ struct timerqueue_node *node;
++
++ mutex_lock(&rtc->ops_lock);
++ while ((node = timerqueue_getnext(head)))
++ timerqueue_del(head, node);
++ mutex_unlock(&rtc->ops_lock);
++
++ cancel_work_sync(&rtc->irqwork);
+
+ ida_simple_remove(&rtc_ida, rtc->id);
+ mutex_destroy(&rtc->ops_lock);
+diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c
+index 9a2bd4947007c..3ee17c4d72987 100644
+--- a/drivers/rtc/interface.c
++++ b/drivers/rtc/interface.c
+@@ -793,9 +793,13 @@ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer)
+ struct timerqueue_node *next = timerqueue_getnext(&rtc->timerqueue);
+ struct rtc_time tm;
+ ktime_t now;
++ int err;
++
++ err = __rtc_read_time(rtc, &tm);
++ if (err)
++ return err;
+
+ timer->enabled = 1;
+- __rtc_read_time(rtc, &tm);
+ now = rtc_tm_to_ktime(tm);
+
+ /* Skip over expired timers */
+@@ -809,7 +813,6 @@ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer)
+ trace_rtc_timer_enqueue(timer);
+ if (!next || ktime_before(timer->node.expires, next->expires)) {
+ struct rtc_wkalrm alarm;
+- int err;
+
+ alarm.time = rtc_ktime_to_tm(timer->node.expires);
+ alarm.enabled = 1;
+diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c
+index 4eb53412b8085..b90a603d6b12f 100644
+--- a/drivers/rtc/rtc-cmos.c
++++ b/drivers/rtc/rtc-cmos.c
+@@ -222,6 +222,8 @@ static inline void cmos_write_bank2(unsigned char val, unsigned char addr)
+
+ static int cmos_read_time(struct device *dev, struct rtc_time *t)
+ {
++ int ret;
++
+ /*
+ * If pm_trace abused the RTC for storage, set the timespec to 0,
+ * which tells the caller that this RTC value is unusable.
+@@ -229,7 +231,12 @@ static int cmos_read_time(struct device *dev, struct rtc_time *t)
+ if (!pm_trace_rtc_valid())
+ return -EIO;
+
+- mc146818_get_time(t);
++ ret = mc146818_get_time(t);
++ if (ret < 0) {
++ dev_err_ratelimited(dev, "unable to read current time\n");
++ return ret;
++ }
++
+ return 0;
+ }
+
+@@ -457,7 +464,10 @@ static int cmos_set_alarm(struct device *dev, struct rtc_wkalrm *t)
+ min = t->time.tm_min;
+ sec = t->time.tm_sec;
+
++ spin_lock_irq(&rtc_lock);
+ rtc_control = CMOS_READ(RTC_CONTROL);
++ spin_unlock_irq(&rtc_lock);
++
+ if (!(rtc_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) {
+ /* Writing 0xff means "don't care" or "match all". */
+ mon = (mon <= 12) ? bin2bcd(mon) : 0xff;
+@@ -790,16 +800,14 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq)
+
+ rename_region(ports, dev_name(&cmos_rtc.rtc->dev));
+
+- spin_lock_irq(&rtc_lock);
+-
+- /* Ensure that the RTC is accessible. Bit 6 must be 0! */
+- if ((CMOS_READ(RTC_VALID) & 0x40) != 0) {
+- spin_unlock_irq(&rtc_lock);
+- dev_warn(dev, "not accessible\n");
++ if (!mc146818_does_rtc_work()) {
++ dev_warn(dev, "broken or not accessible\n");
+ retval = -ENXIO;
+ goto cleanup1;
+ }
+
++ spin_lock_irq(&rtc_lock);
++
+ if (!(flags & CMOS_RTC_FLAGS_NOFREQ)) {
+ /* force periodic irq to CMOS reset default of 1024Hz;
+ *
+diff --git a/drivers/rtc/rtc-ds1302.c b/drivers/rtc/rtc-ds1302.c
+index b3de6d2e680a4..2f83adef966eb 100644
+--- a/drivers/rtc/rtc-ds1302.c
++++ b/drivers/rtc/rtc-ds1302.c
+@@ -199,11 +199,18 @@ static const struct of_device_id ds1302_dt_ids[] = {
+ MODULE_DEVICE_TABLE(of, ds1302_dt_ids);
+ #endif
+
++static const struct spi_device_id ds1302_spi_ids[] = {
++ { .name = "ds1302", },
++ { /* sentinel */ }
++};
++MODULE_DEVICE_TABLE(spi, ds1302_spi_ids);
++
+ static struct spi_driver ds1302_driver = {
+ .driver.name = "rtc-ds1302",
+ .driver.of_match_table = of_match_ptr(ds1302_dt_ids),
+ .probe = ds1302_probe,
+ .remove = ds1302_remove,
++ .id_table = ds1302_spi_ids,
+ };
+
+ module_spi_driver(ds1302_driver);
+diff --git a/drivers/rtc/rtc-ds1390.c b/drivers/rtc/rtc-ds1390.c
+index 66fc8617d07ee..93ce72b9ae59e 100644
+--- a/drivers/rtc/rtc-ds1390.c
++++ b/drivers/rtc/rtc-ds1390.c
+@@ -219,12 +219,19 @@ static const struct of_device_id ds1390_of_match[] = {
+ };
+ MODULE_DEVICE_TABLE(of, ds1390_of_match);
+
++static const struct spi_device_id ds1390_spi_ids[] = {
++ { .name = "ds1390" },
++ {}
++};
++MODULE_DEVICE_TABLE(spi, ds1390_spi_ids);
++
+ static struct spi_driver ds1390_driver = {
+ .driver = {
+ .name = "rtc-ds1390",
+ .of_match_table = of_match_ptr(ds1390_of_match),
+ },
+ .probe = ds1390_probe,
++ .id_table = ds1390_spi_ids,
+ };
+
+ module_spi_driver(ds1390_driver);
+diff --git a/drivers/rtc/rtc-mc146818-lib.c b/drivers/rtc/rtc-mc146818-lib.c
+index dcfaf09946ee3..f3f5a87fe376e 100644
+--- a/drivers/rtc/rtc-mc146818-lib.c
++++ b/drivers/rtc/rtc-mc146818-lib.c
+@@ -8,10 +8,36 @@
+ #include <linux/acpi.h>
+ #endif
+
+-unsigned int mc146818_get_time(struct rtc_time *time)
++/*
++ * If the UIP (Update-in-progress) bit of the RTC is set for more then
++ * 10ms, the RTC is apparently broken or not present.
++ */
++bool mc146818_does_rtc_work(void)
++{
++ int i;
++ unsigned char val;
++ unsigned long flags;
++
++ for (i = 0; i < 10; i++) {
++ spin_lock_irqsave(&rtc_lock, flags);
++ val = CMOS_READ(RTC_FREQ_SELECT);
++ spin_unlock_irqrestore(&rtc_lock, flags);
++
++ if ((val & RTC_UIP) == 0)
++ return true;
++
++ mdelay(1);
++ }
++
++ return false;
++}
++EXPORT_SYMBOL_GPL(mc146818_does_rtc_work);
++
++int mc146818_get_time(struct rtc_time *time)
+ {
+ unsigned char ctrl;
+ unsigned long flags;
++ unsigned int iter_count = 0;
+ unsigned char century = 0;
+ bool retry;
+
+@@ -20,13 +46,13 @@ unsigned int mc146818_get_time(struct rtc_time *time)
+ #endif
+
+ again:
+- spin_lock_irqsave(&rtc_lock, flags);
+- /* Ensure that the RTC is accessible. Bit 6 must be 0! */
+- if (WARN_ON_ONCE((CMOS_READ(RTC_VALID) & 0x40) != 0)) {
+- spin_unlock_irqrestore(&rtc_lock, flags);
+- memset(time, 0xff, sizeof(*time));
+- return 0;
++ if (iter_count > 10) {
++ memset(time, 0, sizeof(*time));
++ return -EIO;
+ }
++ iter_count++;
++
++ spin_lock_irqsave(&rtc_lock, flags);
+
+ /*
+ * Check whether there is an update in progress during which the
+@@ -104,7 +130,7 @@ again:
+ time->tm_year += real_year - 72;
+ #endif
+
+- if (century > 20)
++ if (century > 19)
+ time->tm_year += (century - 19) * 100;
+
+ /*
+@@ -116,10 +142,21 @@ again:
+
+ time->tm_mon--;
+
+- return RTC_24H;
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(mc146818_get_time);
+
++/* AMD systems don't allow access to AltCentury with DV1 */
++static bool apply_amd_register_a_behavior(void)
++{
++#ifdef CONFIG_X86
++ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
++ boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
++ return true;
++#endif
++ return false;
++}
++
+ /* Set the current date and time in the real time clock. */
+ int mc146818_set_time(struct rtc_time *time)
+ {
+@@ -176,8 +213,10 @@ int mc146818_set_time(struct rtc_time *time)
+ if (yrs >= 100)
+ yrs -= 100;
+
+- if (!(CMOS_READ(RTC_CONTROL) & RTC_DM_BINARY)
+- || RTC_ALWAYS_BCD) {
++ spin_lock_irqsave(&rtc_lock, flags);
++ save_control = CMOS_READ(RTC_CONTROL);
++ spin_unlock_irqrestore(&rtc_lock, flags);
++ if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) {
+ sec = bin2bcd(sec);
+ min = bin2bcd(min);
+ hrs = bin2bcd(hrs);
+@@ -191,7 +230,10 @@ int mc146818_set_time(struct rtc_time *time)
+ save_control = CMOS_READ(RTC_CONTROL);
+ CMOS_WRITE((save_control|RTC_SET), RTC_CONTROL);
+ save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
+- CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT);
++ if (apply_amd_register_a_behavior())
++ CMOS_WRITE((save_freq_select & ~RTC_AMD_BANK_SELECT), RTC_FREQ_SELECT);
++ else
++ CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT);
+
+ #ifdef CONFIG_MACH_DECSTATION
+ CMOS_WRITE(real_yrs, RTC_DEC_YEAR);
+diff --git a/drivers/rtc/rtc-mcp795.c b/drivers/rtc/rtc-mcp795.c
+index bad7792b6ca58..0d515b3df5710 100644
+--- a/drivers/rtc/rtc-mcp795.c
++++ b/drivers/rtc/rtc-mcp795.c
+@@ -430,12 +430,19 @@ static const struct of_device_id mcp795_of_match[] = {
+ MODULE_DEVICE_TABLE(of, mcp795_of_match);
+ #endif
+
++static const struct spi_device_id mcp795_spi_ids[] = {
++ { .name = "mcp795" },
++ { }
++};
++MODULE_DEVICE_TABLE(spi, mcp795_spi_ids);
++
+ static struct spi_driver mcp795_driver = {
+ .driver = {
+ .name = "rtc-mcp795",
+ .of_match_table = of_match_ptr(mcp795_of_match),
+ },
+ .probe = mcp795_probe,
++ .id_table = mcp795_spi_ids,
+ };
+
+ module_spi_driver(mcp795_driver);
+diff --git a/drivers/rtc/rtc-pcf2123.c b/drivers/rtc/rtc-pcf2123.c
+index 0f58cac81d8c0..7473e6c8a183b 100644
+--- a/drivers/rtc/rtc-pcf2123.c
++++ b/drivers/rtc/rtc-pcf2123.c
+@@ -451,12 +451,21 @@ static const struct of_device_id pcf2123_dt_ids[] = {
+ MODULE_DEVICE_TABLE(of, pcf2123_dt_ids);
+ #endif
+
++static const struct spi_device_id pcf2123_spi_ids[] = {
++ { .name = "pcf2123", },
++ { .name = "rv2123", },
++ { .name = "rtc-pcf2123", },
++ { /* sentinel */ }
++};
++MODULE_DEVICE_TABLE(spi, pcf2123_spi_ids);
++
+ static struct spi_driver pcf2123_driver = {
+ .driver = {
+ .name = "rtc-pcf2123",
+ .of_match_table = of_match_ptr(pcf2123_dt_ids),
+ },
+ .probe = pcf2123_probe,
++ .id_table = pcf2123_spi_ids,
+ };
+
+ module_spi_driver(pcf2123_driver);
+diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c
+index 56c58b055dfff..43f8011070952 100644
+--- a/drivers/rtc/rtc-pcf2127.c
++++ b/drivers/rtc/rtc-pcf2127.c
+@@ -374,7 +374,8 @@ static int pcf2127_watchdog_init(struct device *dev, struct pcf2127 *pcf2127)
+ static int pcf2127_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+ {
+ struct pcf2127 *pcf2127 = dev_get_drvdata(dev);
+- unsigned int buf[5], ctrl2;
++ u8 buf[5];
++ unsigned int ctrl2;
+ int ret;
+
+ ret = regmap_read(pcf2127->regmap, PCF2127_REG_CTRL2, &ctrl2);
+diff --git a/drivers/rtc/rtc-pl031.c b/drivers/rtc/rtc-pl031.c
+index e38ee88483855..bad6a5d9c6839 100644
+--- a/drivers/rtc/rtc-pl031.c
++++ b/drivers/rtc/rtc-pl031.c
+@@ -350,9 +350,6 @@ static int pl031_probe(struct amba_device *adev, const struct amba_id *id)
+ }
+ }
+
+- if (!adev->irq[0])
+- clear_bit(RTC_FEATURE_ALARM, ldata->rtc->features);
+-
+ device_init_wakeup(&adev->dev, true);
+ ldata->rtc = devm_rtc_allocate_device(&adev->dev);
+ if (IS_ERR(ldata->rtc)) {
+@@ -360,6 +357,9 @@ static int pl031_probe(struct amba_device *adev, const struct amba_id *id)
+ goto out;
+ }
+
++ if (!adev->irq[0])
++ clear_bit(RTC_FEATURE_ALARM, ldata->rtc->features);
++
+ ldata->rtc->ops = ops;
+ ldata->rtc->range_min = vendor->range_min;
+ ldata->rtc->range_max = vendor->range_max;
+diff --git a/drivers/rtc/rtc-pxa.c b/drivers/rtc/rtc-pxa.c
+index d2f1d8f754bf3..cf8119b6d3204 100644
+--- a/drivers/rtc/rtc-pxa.c
++++ b/drivers/rtc/rtc-pxa.c
+@@ -330,6 +330,10 @@ static int __init pxa_rtc_probe(struct platform_device *pdev)
+ if (sa1100_rtc->irq_alarm < 0)
+ return -ENXIO;
+
++ sa1100_rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
++ if (IS_ERR(sa1100_rtc->rtc))
++ return PTR_ERR(sa1100_rtc->rtc);
++
+ pxa_rtc->base = devm_ioremap(dev, pxa_rtc->ress->start,
+ resource_size(pxa_rtc->ress));
+ if (!pxa_rtc->base) {
+diff --git a/drivers/rtc/rtc-rv3032.c b/drivers/rtc/rtc-rv3032.c
+index d63102d5cb1e4..1b62ed2f14594 100644
+--- a/drivers/rtc/rtc-rv3032.c
++++ b/drivers/rtc/rtc-rv3032.c
+@@ -617,11 +617,11 @@ static int rv3032_clkout_set_rate(struct clk_hw *hw, unsigned long rate,
+
+ ret = rv3032_enter_eerd(rv3032, &eerd);
+ if (ret)
+- goto exit_eerd;
++ return ret;
+
+ ret = regmap_write(rv3032->regmap, RV3032_CLKOUT1, hfd & 0xff);
+ if (ret)
+- return ret;
++ goto exit_eerd;
+
+ ret = regmap_write(rv3032->regmap, RV3032_CLKOUT2, RV3032_CLKOUT2_OS |
+ FIELD_PREP(RV3032_CLKOUT2_HFD_MSK, hfd >> 8));
+diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c
+index adec1b14a8deb..c551ebf0ac00f 100644
+--- a/drivers/rtc/rtc-sun6i.c
++++ b/drivers/rtc/rtc-sun6i.c
+@@ -138,7 +138,7 @@ struct sun6i_rtc_dev {
+ const struct sun6i_rtc_clk_data *data;
+ void __iomem *base;
+ int irq;
+- unsigned long alarm;
++ time64_t alarm;
+
+ struct clk_hw hw;
+ struct clk_hw *int_osc;
+@@ -510,10 +510,8 @@ static int sun6i_rtc_setalarm(struct device *dev, struct rtc_wkalrm *wkalrm)
+ struct sun6i_rtc_dev *chip = dev_get_drvdata(dev);
+ struct rtc_time *alrm_tm = &wkalrm->time;
+ struct rtc_time tm_now;
+- unsigned long time_now = 0;
+- unsigned long time_set = 0;
+- unsigned long time_gap = 0;
+- int ret = 0;
++ time64_t time_now, time_set;
++ int ret;
+
+ ret = sun6i_rtc_gettime(dev, &tm_now);
+ if (ret < 0) {
+@@ -528,9 +526,7 @@ static int sun6i_rtc_setalarm(struct device *dev, struct rtc_wkalrm *wkalrm)
+ return -EINVAL;
+ }
+
+- time_gap = time_set - time_now;
+-
+- if (time_gap > U32_MAX) {
++ if ((time_set - time_now) > U32_MAX) {
+ dev_err(dev, "Date too far in the future\n");
+ return -EINVAL;
+ }
+@@ -539,7 +535,7 @@ static int sun6i_rtc_setalarm(struct device *dev, struct rtc_wkalrm *wkalrm)
+ writel(0, chip->base + SUN6I_ALRM_COUNTER);
+ usleep_range(100, 300);
+
+- writel(time_gap, chip->base + SUN6I_ALRM_COUNTER);
++ writel(time_set - time_now, chip->base + SUN6I_ALRM_COUNTER);
+ chip->alarm = time_set;
+
+ sun6i_rtc_setaie(wkalrm->enabled, chip);
+diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c
+index 2018614f258f6..6eaa9321c0741 100644
+--- a/drivers/rtc/rtc-wm8350.c
++++ b/drivers/rtc/rtc-wm8350.c
+@@ -432,14 +432,21 @@ static int wm8350_rtc_probe(struct platform_device *pdev)
+ return ret;
+ }
+
+- wm8350_register_irq(wm8350, WM8350_IRQ_RTC_SEC,
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_RTC_SEC,
+ wm8350_rtc_update_handler, 0,
+ "RTC Seconds", wm8350);
++ if (ret)
++ return ret;
++
+ wm8350_mask_irq(wm8350, WM8350_IRQ_RTC_SEC);
+
+- wm8350_register_irq(wm8350, WM8350_IRQ_RTC_ALM,
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_RTC_ALM,
+ wm8350_rtc_alarm_handler, 0,
+ "RTC Alarm", wm8350);
++ if (ret) {
++ wm8350_free_irq(wm8350, WM8350_IRQ_RTC_SEC, wm8350);
++ return ret;
++ }
+
+ return 0;
+ }
+diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
+index e34c6cc61983b..8240b804ef847 100644
+--- a/drivers/s390/block/dasd.c
++++ b/drivers/s390/block/dasd.c
+@@ -1422,6 +1422,13 @@ int dasd_start_IO(struct dasd_ccw_req *cqr)
+ if (!cqr->lpm)
+ cqr->lpm = dasd_path_get_opm(device);
+ }
++ /*
++ * remember the amount of formatted tracks to prevent double format on
++ * ESE devices
++ */
++ if (cqr->block)
++ cqr->trkcount = atomic_read(&cqr->block->trkcount);
++
+ if (cqr->cpmode == 1) {
+ rc = ccw_device_tm_start(device->cdev, cqr->cpaddr,
+ (long) cqr, cqr->lpm);
+@@ -1639,6 +1646,7 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
+ unsigned long now;
+ int nrf_suppressed = 0;
+ int fp_suppressed = 0;
++ struct request *req;
+ u8 *sense = NULL;
+ int expires;
+
+@@ -1739,7 +1747,12 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
+ }
+
+ if (dasd_ese_needs_format(cqr->block, irb)) {
+- if (rq_data_dir((struct request *)cqr->callback_data) == READ) {
++ req = dasd_get_callback_data(cqr);
++ if (!req) {
++ cqr->status = DASD_CQR_ERROR;
++ return;
++ }
++ if (rq_data_dir(req) == READ) {
+ device->discipline->ese_read(cqr, irb);
+ cqr->status = DASD_CQR_SUCCESS;
+ cqr->stopclk = now;
+@@ -2762,8 +2775,7 @@ static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr)
+ * complete a request partially.
+ */
+ if (proc_bytes) {
+- blk_update_request(req, BLK_STS_OK,
+- blk_rq_bytes(req) - proc_bytes);
++ blk_update_request(req, BLK_STS_OK, proc_bytes);
+ blk_mq_requeue_request(req, true);
+ } else if (likely(!blk_should_fake_timeout(req->q))) {
+ blk_mq_complete_request(req);
+diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
+index 460e0f1cca533..ff7b7d470e96f 100644
+--- a/drivers/s390/block/dasd_eckd.c
++++ b/drivers/s390/block/dasd_eckd.c
+@@ -3095,13 +3095,24 @@ static int dasd_eckd_format_device(struct dasd_device *base,
+ }
+
+ static bool test_and_set_format_track(struct dasd_format_entry *to_format,
+- struct dasd_block *block)
++ struct dasd_ccw_req *cqr)
+ {
++ struct dasd_block *block = cqr->block;
+ struct dasd_format_entry *format;
+ unsigned long flags;
+ bool rc = false;
+
+ spin_lock_irqsave(&block->format_lock, flags);
++ if (cqr->trkcount != atomic_read(&block->trkcount)) {
++ /*
++ * The number of formatted tracks has changed after request
++ * start and we can not tell if the current track was involved.
++ * To avoid data corruption treat it as if the current track is
++ * involved
++ */
++ rc = true;
++ goto out;
++ }
+ list_for_each_entry(format, &block->format_list, list) {
+ if (format->track == to_format->track) {
+ rc = true;
+@@ -3121,6 +3132,7 @@ static void clear_format_track(struct dasd_format_entry *format,
+ unsigned long flags;
+
+ spin_lock_irqsave(&block->format_lock, flags);
++ atomic_inc(&block->trkcount);
+ list_del_init(&format->list);
+ spin_unlock_irqrestore(&block->format_lock, flags);
+ }
+@@ -3157,7 +3169,7 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr,
+ sector_t curr_trk;
+ int rc;
+
+- req = cqr->callback_data;
++ req = dasd_get_callback_data(cqr);
+ block = cqr->block;
+ base = block->base;
+ private = base->private;
+@@ -3182,8 +3194,11 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr,
+ }
+ format->track = curr_trk;
+ /* test if track is already in formatting by another thread */
+- if (test_and_set_format_track(format, block))
++ if (test_and_set_format_track(format, cqr)) {
++ /* this is no real error so do not count down retries */
++ cqr->retries++;
+ return ERR_PTR(-EEXIST);
++ }
+
+ fdata.start_unit = curr_trk;
+ fdata.stop_unit = curr_trk;
+@@ -3282,12 +3297,11 @@ static int dasd_eckd_ese_read(struct dasd_ccw_req *cqr, struct irb *irb)
+ cqr->proc_bytes = blk_count * blksize;
+ return 0;
+ }
+- if (dst && !skip_block) {
+- dst += off;
++ if (dst && !skip_block)
+ memset(dst, 0, blksize);
+- } else {
++ else
+ skip_block--;
+- }
++ dst += blksize;
+ blk_count++;
+ }
+ }
+diff --git a/drivers/s390/block/dasd_genhd.c b/drivers/s390/block/dasd_genhd.c
+index fa966e0db6ca9..3a6f3af240fa7 100644
+--- a/drivers/s390/block/dasd_genhd.c
++++ b/drivers/s390/block/dasd_genhd.c
+@@ -14,6 +14,7 @@
+ #define KMSG_COMPONENT "dasd"
+
+ #include <linux/interrupt.h>
++#include <linux/major.h>
+ #include <linux/fs.h>
+ #include <linux/blkpg.h>
+
+diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
+index 155428bfed8ac..d94ae067f085e 100644
+--- a/drivers/s390/block/dasd_int.h
++++ b/drivers/s390/block/dasd_int.h
+@@ -188,6 +188,7 @@ struct dasd_ccw_req {
+ void (*callback)(struct dasd_ccw_req *, void *data);
+ void *callback_data;
+ unsigned int proc_bytes; /* bytes for partial completion */
++ unsigned int trkcount; /* count formatted tracks */
+ };
+
+ /*
+@@ -611,6 +612,7 @@ struct dasd_block {
+
+ struct list_head format_list;
+ spinlock_t format_lock;
++ atomic_t trkcount;
+ };
+
+ struct dasd_attention_data {
+@@ -757,6 +759,18 @@ dasd_check_blocksize(int bsize)
+ return 0;
+ }
+
++/*
++ * return the callback data of the original request in case there are
++ * ERP requests build on top of it
++ */
++static inline void *dasd_get_callback_data(struct dasd_ccw_req *cqr)
++{
++ while (cqr->refers)
++ cqr = cqr->refers;
++
++ return cqr->callback_data;
++}
++
+ /* externals in dasd.c */
+ #define DASD_PROFILE_OFF 0
+ #define DASD_PROFILE_ON 1
+diff --git a/drivers/s390/char/tape_std.c b/drivers/s390/char/tape_std.c
+index 1f5fab617b679..f7e75d9fedf61 100644
+--- a/drivers/s390/char/tape_std.c
++++ b/drivers/s390/char/tape_std.c
+@@ -53,7 +53,6 @@ int
+ tape_std_assign(struct tape_device *device)
+ {
+ int rc;
+- struct timer_list timeout;
+ struct tape_request *request;
+
+ request = tape_alloc_request(2, 11);
+@@ -70,7 +69,7 @@ tape_std_assign(struct tape_device *device)
+ * So we set up a timeout for this call.
+ */
+ timer_setup(&request->timer, tape_std_assign_timeout, 0);
+- mod_timer(&timeout, jiffies + 2 * HZ);
++ mod_timer(&request->timer, jiffies + msecs_to_jiffies(2000));
+
+ rc = tape_do_io_interruptible(device, request);
+
+diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
+index 44461928aab8a..c278097926093 100644
+--- a/drivers/s390/cio/css.c
++++ b/drivers/s390/cio/css.c
+@@ -437,8 +437,8 @@ static ssize_t dev_busid_show(struct device *dev,
+ struct subchannel *sch = to_subchannel(dev);
+ struct pmcw *pmcw = &sch->schib.pmcw;
+
+- if ((pmcw->st == SUBCHANNEL_TYPE_IO ||
+- pmcw->st == SUBCHANNEL_TYPE_MSG) && pmcw->dnv)
++ if ((pmcw->st == SUBCHANNEL_TYPE_IO && pmcw->dnv) ||
++ (pmcw->st == SUBCHANNEL_TYPE_MSG && pmcw->w))
+ return sysfs_emit(buf, "0.%x.%04x\n", sch->schid.ssid,
+ pmcw->dev);
+ else
+diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
+index 8d14569823d73..61cde02b23fec 100644
+--- a/drivers/s390/cio/device.c
++++ b/drivers/s390/cio/device.c
+@@ -1194,7 +1194,7 @@ static int io_subchannel_chp_event(struct subchannel *sch,
+ else
+ path_event[chpid] = PE_NONE;
+ }
+- if (cdev)
++ if (cdev && cdev->drv && cdev->drv->path_event)
+ cdev->drv->path_event(cdev, path_event);
+ break;
+ }
+diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
+index 0fe7b2f2e7f52..c533d1dadc6bb 100644
+--- a/drivers/s390/cio/device_ops.c
++++ b/drivers/s390/cio/device_ops.c
+@@ -825,13 +825,23 @@ EXPORT_SYMBOL_GPL(ccw_device_get_chid);
+ */
+ void *ccw_device_dma_zalloc(struct ccw_device *cdev, size_t size)
+ {
+- return cio_gp_dma_zalloc(cdev->private->dma_pool, &cdev->dev, size);
++ void *addr;
++
++ if (!get_device(&cdev->dev))
++ return NULL;
++ addr = cio_gp_dma_zalloc(cdev->private->dma_pool, &cdev->dev, size);
++ if (IS_ERR_OR_NULL(addr))
++ put_device(&cdev->dev);
++ return addr;
+ }
+ EXPORT_SYMBOL(ccw_device_dma_zalloc);
+
+ void ccw_device_dma_free(struct ccw_device *cdev, void *cpu_addr, size_t size)
+ {
++ if (!cpu_addr)
++ return;
+ cio_gp_dma_free(cdev->private->dma_pool, cpu_addr, size);
++ put_device(&cdev->dev);
+ }
+ EXPORT_SYMBOL(ccw_device_dma_free);
+
+diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
+index 9ea48bf0ee40d..032bf7b282bab 100644
+--- a/drivers/s390/crypto/ap_queue.c
++++ b/drivers/s390/crypto/ap_queue.c
+@@ -157,6 +157,8 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq)
+ switch (status.response_code) {
+ case AP_RESPONSE_NORMAL:
+ aq->queue_count = max_t(int, 0, aq->queue_count - 1);
++ if (!status.queue_empty && !aq->queue_count)
++ aq->queue_count++;
+ if (aq->queue_count > 0)
+ mod_timer(&aq->timeout,
+ jiffies + aq->request_timeout);
+diff --git a/drivers/s390/net/ctcm_mpc.c b/drivers/s390/net/ctcm_mpc.c
+index f0436f555c62a..be03cb123ef48 100644
+--- a/drivers/s390/net/ctcm_mpc.c
++++ b/drivers/s390/net/ctcm_mpc.c
+@@ -626,8 +626,6 @@ static void mpc_rcvd_sweep_resp(struct mpcg_info *mpcginfo)
+ ctcm_clear_busy_do(dev);
+ }
+
+- kfree(mpcginfo);
+-
+ return;
+
+ }
+@@ -1192,10 +1190,10 @@ static void ctcmpc_unpack_skb(struct channel *ch, struct sk_buff *pskb)
+ CTCM_FUNTAIL, dev->name);
+ priv->stats.rx_dropped++;
+ /* mpcginfo only used for non-data transfers */
+- kfree(mpcginfo);
+ if (do_debug_data)
+ ctcmpc_dump_skb(pskb, -8);
+ }
++ kfree(mpcginfo);
+ }
+ done:
+
+@@ -1977,7 +1975,6 @@ static void mpc_action_rcvd_xid0(fsm_instance *fsm, int event, void *arg)
+ }
+ break;
+ }
+- kfree(mpcginfo);
+
+ CTCM_PR_DEBUG("ctcmpc:%s() %s xid2:%i xid7:%i xidt_p2:%i \n",
+ __func__, ch->id, grp->outstanding_xid2,
+@@ -2038,7 +2035,6 @@ static void mpc_action_rcvd_xid7(fsm_instance *fsm, int event, void *arg)
+ mpc_validate_xid(mpcginfo);
+ break;
+ }
+- kfree(mpcginfo);
+ return;
+ }
+
+diff --git a/drivers/s390/net/ctcm_sysfs.c b/drivers/s390/net/ctcm_sysfs.c
+index ded1930a00b2d..e3813a7aa5e68 100644
+--- a/drivers/s390/net/ctcm_sysfs.c
++++ b/drivers/s390/net/ctcm_sysfs.c
+@@ -39,11 +39,12 @@ static ssize_t ctcm_buffer_write(struct device *dev,
+ struct ctcm_priv *priv = dev_get_drvdata(dev);
+ int rc;
+
+- ndev = priv->channel[CTCM_READ]->netdev;
+- if (!(priv && priv->channel[CTCM_READ] && ndev)) {
++ if (!(priv && priv->channel[CTCM_READ] &&
++ priv->channel[CTCM_READ]->netdev)) {
+ CTCM_DBF_TEXT(SETUP, CTC_DBF_ERROR, "bfnondev");
+ return -ENODEV;
+ }
++ ndev = priv->channel[CTCM_READ]->netdev;
+
+ rc = kstrtouint(buf, 0, &bs1);
+ if (rc)
+diff --git a/drivers/s390/net/lcs.c b/drivers/s390/net/lcs.c
+index 440219bcaa2be..06a322bdced6d 100644
+--- a/drivers/s390/net/lcs.c
++++ b/drivers/s390/net/lcs.c
+@@ -1735,10 +1735,11 @@ lcs_get_control(struct lcs_card *card, struct lcs_cmd *cmd)
+ lcs_schedule_recovery(card);
+ break;
+ case LCS_CMD_STOPLAN:
+- pr_warn("Stoplan for %s initiated by LGW\n",
+- card->dev->name);
+- if (card->dev)
++ if (card->dev) {
++ pr_warn("Stoplan for %s initiated by LGW\n",
++ card->dev->name);
+ netif_carrier_off(card->dev);
++ }
+ break;
+ default:
+ LCS_DBF_TEXT(5, trace, "noLGWcmd");
+diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
+index d24cafe02708f..511bf8e0a436c 100644
+--- a/drivers/s390/scsi/zfcp_fc.c
++++ b/drivers/s390/scsi/zfcp_fc.c
+@@ -521,6 +521,8 @@ static void zfcp_fc_adisc_handler(void *data)
+ goto out;
+ }
+
++ /* re-init to undo drop from zfcp_fc_adisc() */
++ port->d_id = ntoh24(adisc_resp->adisc_port_id);
+ /* port is good, unblock rport without going through erp */
+ zfcp_scsi_schedule_rport_register(port);
+ out:
+@@ -534,6 +536,7 @@ static int zfcp_fc_adisc(struct zfcp_port *port)
+ struct zfcp_fc_req *fc_req;
+ struct zfcp_adapter *adapter = port->adapter;
+ struct Scsi_Host *shost = adapter->scsi_host;
++ u32 d_id;
+ int ret;
+
+ fc_req = kmem_cache_zalloc(zfcp_fc_req_cache, GFP_ATOMIC);
+@@ -558,7 +561,15 @@ static int zfcp_fc_adisc(struct zfcp_port *port)
+ fc_req->u.adisc.req.adisc_cmd = ELS_ADISC;
+ hton24(fc_req->u.adisc.req.adisc_port_id, fc_host_port_id(shost));
+
+- ret = zfcp_fsf_send_els(adapter, port->d_id, &fc_req->ct_els,
++ d_id = port->d_id; /* remember as destination for send els below */
++ /*
++ * Force fresh GID_PN lookup on next port recovery.
++ * Must happen after request setup and before sending request,
++ * to prevent race with port->d_id re-init in zfcp_fc_adisc_handler().
++ */
++ port->d_id = 0;
++
++ ret = zfcp_fsf_send_els(adapter, d_id, &fc_req->ct_els,
+ ZFCP_FC_CTELS_TMO);
+ if (ret)
+ kmem_cache_free(zfcp_fc_req_cache, fc_req);
+diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
+index ffb3919675734..97be0765cb9bd 100644
+--- a/drivers/scsi/advansys.c
++++ b/drivers/scsi/advansys.c
+@@ -3308,8 +3308,8 @@ static void asc_prt_adv_board_info(struct seq_file *m, struct Scsi_Host *shost)
+ shost->host_no);
+
+ seq_printf(m,
+- " iop_base 0x%lx, cable_detect: %X, err_code %u\n",
+- (unsigned long)v->iop_base,
++ " iop_base 0x%p, cable_detect: %X, err_code %u\n",
++ v->iop_base,
+ AdvReadWordRegister(iop_base,IOPW_SCSI_CFG1) & CABLE_DETECT,
+ v->err_code);
+
+diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
+index b13b5c85f3de0..75a5a4765f425 100644
+--- a/drivers/scsi/aha152x.c
++++ b/drivers/scsi/aha152x.c
+@@ -3370,13 +3370,11 @@ static int __init aha152x_setup(char *str)
+ setup[setup_count].synchronous = ints[0] >= 6 ? ints[6] : 1;
+ setup[setup_count].delay = ints[0] >= 7 ? ints[7] : DELAY_DEFAULT;
+ setup[setup_count].ext_trans = ints[0] >= 8 ? ints[8] : 0;
+- if (ints[0] > 8) { /*}*/
++ if (ints[0] > 8)
+ printk(KERN_NOTICE "aha152x: usage: aha152x=<IOBASE>[,<IRQ>[,<SCSI ID>"
+ "[,<RECONNECT>[,<PARITY>[,<SYNCHRONOUS>[,<DELAY>[,<EXT_TRANS>]]]]]]]\n");
+- } else {
++ else
+ setup_count++;
+- return 0;
+- }
+
+ return 1;
+ }
+diff --git a/drivers/scsi/bfa/bfad_attr.c b/drivers/scsi/bfa/bfad_attr.c
+index 5ae1e3f789101..e049cdb3c286c 100644
+--- a/drivers/scsi/bfa/bfad_attr.c
++++ b/drivers/scsi/bfa/bfad_attr.c
+@@ -711,7 +711,7 @@ bfad_im_serial_num_show(struct device *dev, struct device_attribute *attr,
+ char serial_num[BFA_ADAPTER_SERIAL_NUM_LEN];
+
+ bfa_get_adapter_serial_num(&bfad->bfa, serial_num);
+- return snprintf(buf, PAGE_SIZE, "%s\n", serial_num);
++ return sysfs_emit(buf, "%s\n", serial_num);
+ }
+
+ static ssize_t
+@@ -725,7 +725,7 @@ bfad_im_model_show(struct device *dev, struct device_attribute *attr,
+ char model[BFA_ADAPTER_MODEL_NAME_LEN];
+
+ bfa_get_adapter_model(&bfad->bfa, model);
+- return snprintf(buf, PAGE_SIZE, "%s\n", model);
++ return sysfs_emit(buf, "%s\n", model);
+ }
+
+ static ssize_t
+@@ -805,7 +805,7 @@ bfad_im_model_desc_show(struct device *dev, struct device_attribute *attr,
+ snprintf(model_descr, BFA_ADAPTER_MODEL_DESCR_LEN,
+ "Invalid Model");
+
+- return snprintf(buf, PAGE_SIZE, "%s\n", model_descr);
++ return sysfs_emit(buf, "%s\n", model_descr);
+ }
+
+ static ssize_t
+@@ -819,7 +819,7 @@ bfad_im_node_name_show(struct device *dev, struct device_attribute *attr,
+ u64 nwwn;
+
+ nwwn = bfa_fcs_lport_get_nwwn(port->fcs_port);
+- return snprintf(buf, PAGE_SIZE, "0x%llx\n", cpu_to_be64(nwwn));
++ return sysfs_emit(buf, "0x%llx\n", cpu_to_be64(nwwn));
+ }
+
+ static ssize_t
+@@ -836,7 +836,7 @@ bfad_im_symbolic_name_show(struct device *dev, struct device_attribute *attr,
+ bfa_fcs_lport_get_attr(&bfad->bfa_fcs.fabric.bport, &port_attr);
+ strlcpy(symname, port_attr.port_cfg.sym_name.symname,
+ BFA_SYMNAME_MAXLEN);
+- return snprintf(buf, PAGE_SIZE, "%s\n", symname);
++ return sysfs_emit(buf, "%s\n", symname);
+ }
+
+ static ssize_t
+@@ -850,14 +850,14 @@ bfad_im_hw_version_show(struct device *dev, struct device_attribute *attr,
+ char hw_ver[BFA_VERSION_LEN];
+
+ bfa_get_pci_chip_rev(&bfad->bfa, hw_ver);
+- return snprintf(buf, PAGE_SIZE, "%s\n", hw_ver);
++ return sysfs_emit(buf, "%s\n", hw_ver);
+ }
+
+ static ssize_t
+ bfad_im_drv_version_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+ {
+- return snprintf(buf, PAGE_SIZE, "%s\n", BFAD_DRIVER_VERSION);
++ return sysfs_emit(buf, "%s\n", BFAD_DRIVER_VERSION);
+ }
+
+ static ssize_t
+@@ -871,7 +871,7 @@ bfad_im_optionrom_version_show(struct device *dev,
+ char optrom_ver[BFA_VERSION_LEN];
+
+ bfa_get_adapter_optrom_ver(&bfad->bfa, optrom_ver);
+- return snprintf(buf, PAGE_SIZE, "%s\n", optrom_ver);
++ return sysfs_emit(buf, "%s\n", optrom_ver);
+ }
+
+ static ssize_t
+@@ -885,7 +885,7 @@ bfad_im_fw_version_show(struct device *dev, struct device_attribute *attr,
+ char fw_ver[BFA_VERSION_LEN];
+
+ bfa_get_adapter_fw_ver(&bfad->bfa, fw_ver);
+- return snprintf(buf, PAGE_SIZE, "%s\n", fw_ver);
++ return sysfs_emit(buf, "%s\n", fw_ver);
+ }
+
+ static ssize_t
+@@ -897,7 +897,7 @@ bfad_im_num_of_ports_show(struct device *dev, struct device_attribute *attr,
+ (struct bfad_im_port_s *) shost->hostdata[0];
+ struct bfad_s *bfad = im_port->bfad;
+
+- return snprintf(buf, PAGE_SIZE, "%d\n",
++ return sysfs_emit(buf, "%d\n",
+ bfa_get_nports(&bfad->bfa));
+ }
+
+@@ -905,7 +905,7 @@ static ssize_t
+ bfad_im_drv_name_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+ {
+- return snprintf(buf, PAGE_SIZE, "%s\n", BFAD_DRIVER_NAME);
++ return sysfs_emit(buf, "%s\n", BFAD_DRIVER_NAME);
+ }
+
+ static ssize_t
+@@ -924,14 +924,14 @@ bfad_im_num_of_discovered_ports_show(struct device *dev,
+ rports = kcalloc(nrports, sizeof(struct bfa_rport_qualifier_s),
+ GFP_ATOMIC);
+ if (rports == NULL)
+- return snprintf(buf, PAGE_SIZE, "Failed\n");
++ return sysfs_emit(buf, "Failed\n");
+
+ spin_lock_irqsave(&bfad->bfad_lock, flags);
+ bfa_fcs_lport_get_rport_quals(port->fcs_port, rports, &nrports);
+ spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+ kfree(rports);
+
+- return snprintf(buf, PAGE_SIZE, "%d\n", nrports);
++ return sysfs_emit(buf, "%d\n", nrports);
+ }
+
+ static DEVICE_ATTR(serial_number, S_IRUGO,
+diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+index 8863a74e6c57d..e2586472ecad4 100644
+--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
++++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+@@ -82,7 +82,7 @@ static int bnx2fc_bind_pcidev(struct bnx2fc_hba *hba);
+ static void bnx2fc_unbind_pcidev(struct bnx2fc_hba *hba);
+ static struct fc_lport *bnx2fc_if_create(struct bnx2fc_interface *interface,
+ struct device *parent, int npiv);
+-static void bnx2fc_destroy_work(struct work_struct *work);
++static void bnx2fc_port_destroy(struct fcoe_port *port);
+
+ static struct bnx2fc_hba *bnx2fc_hba_lookup(struct net_device *phys_dev);
+ static struct bnx2fc_interface *bnx2fc_interface_lookup(struct net_device
+@@ -508,7 +508,8 @@ static int bnx2fc_l2_rcv_thread(void *arg)
+
+ static void bnx2fc_recv_frame(struct sk_buff *skb)
+ {
+- u32 fr_len;
++ u64 crc_err;
++ u32 fr_len, fr_crc;
+ struct fc_lport *lport;
+ struct fcoe_rcv_info *fr;
+ struct fc_stats *stats;
+@@ -542,6 +543,11 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
+ skb_pull(skb, sizeof(struct fcoe_hdr));
+ fr_len = skb->len - sizeof(struct fcoe_crc_eof);
+
++ stats = per_cpu_ptr(lport->stats, get_cpu());
++ stats->RxFrames++;
++ stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
++ put_cpu();
++
+ fp = (struct fc_frame *)skb;
+ fc_frame_init(fp);
+ fr_dev(fp) = lport;
+@@ -624,16 +630,15 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
+ return;
+ }
+
+- stats = per_cpu_ptr(lport->stats, smp_processor_id());
+- stats->RxFrames++;
+- stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
++ fr_crc = le32_to_cpu(fr_crc(fp));
+
+- if (le32_to_cpu(fr_crc(fp)) !=
+- ~crc32(~0, skb->data, fr_len)) {
+- if (stats->InvalidCRCCount < 5)
++ if (unlikely(fr_crc != ~crc32(~0, skb->data, fr_len))) {
++ stats = per_cpu_ptr(lport->stats, get_cpu());
++ crc_err = (stats->InvalidCRCCount++);
++ put_cpu();
++ if (crc_err < 5)
+ printk(KERN_WARNING PFX "dropping frame with "
+ "CRC error\n");
+- stats->InvalidCRCCount++;
+ kfree_skb(skb);
+ return;
+ }
+@@ -907,9 +912,6 @@ static void bnx2fc_indicate_netevent(void *context, unsigned long event,
+ __bnx2fc_destroy(interface);
+ }
+ mutex_unlock(&bnx2fc_dev_lock);
+-
+- /* Ensure ALL destroy work has been completed before return */
+- flush_workqueue(bnx2fc_wq);
+ return;
+
+ default:
+@@ -1215,8 +1217,8 @@ static int bnx2fc_vport_destroy(struct fc_vport *vport)
+ mutex_unlock(&n_port->lp_mutex);
+ bnx2fc_free_vport(interface->hba, port->lport);
+ bnx2fc_port_shutdown(port->lport);
++ bnx2fc_port_destroy(port);
+ bnx2fc_interface_put(interface);
+- queue_work(bnx2fc_wq, &port->destroy_work);
+ return 0;
+ }
+
+@@ -1525,7 +1527,6 @@ static struct fc_lport *bnx2fc_if_create(struct bnx2fc_interface *interface,
+ port->lport = lport;
+ port->priv = interface;
+ port->get_netdev = bnx2fc_netdev;
+- INIT_WORK(&port->destroy_work, bnx2fc_destroy_work);
+
+ /* Configure fcoe_port */
+ rc = bnx2fc_lport_config(lport);
+@@ -1653,8 +1654,8 @@ static void __bnx2fc_destroy(struct bnx2fc_interface *interface)
+ bnx2fc_interface_cleanup(interface);
+ bnx2fc_stop(interface);
+ list_del(&interface->list);
++ bnx2fc_port_destroy(port);
+ bnx2fc_interface_put(interface);
+- queue_work(bnx2fc_wq, &port->destroy_work);
+ }
+
+ /**
+@@ -1694,15 +1695,12 @@ netdev_err:
+ return rc;
+ }
+
+-static void bnx2fc_destroy_work(struct work_struct *work)
++static void bnx2fc_port_destroy(struct fcoe_port *port)
+ {
+- struct fcoe_port *port;
+ struct fc_lport *lport;
+
+- port = container_of(work, struct fcoe_port, destroy_work);
+ lport = port->lport;
+-
+- BNX2FC_HBA_DBG(lport, "Entered bnx2fc_destroy_work\n");
++ BNX2FC_HBA_DBG(lport, "Entered %s, destroying lport %p\n", __func__, lport);
+
+ bnx2fc_if_destroy(lport);
+ }
+@@ -2556,9 +2554,6 @@ static void bnx2fc_ulp_exit(struct cnic_dev *dev)
+ __bnx2fc_destroy(interface);
+ mutex_unlock(&bnx2fc_dev_lock);
+
+- /* Ensure ALL destroy work has been completed before return */
+- flush_workqueue(bnx2fc_wq);
+-
+ bnx2fc_ulp_stop(hba);
+ /* unregister cnic device */
+ if (test_and_clear_bit(BNX2FC_CNIC_REGISTERED, &hba->reg_with_cnic))
+diff --git a/drivers/scsi/bnx2i/bnx2i_hwi.c b/drivers/scsi/bnx2i/bnx2i_hwi.c
+index 5521469ce678b..e16327a4b4c96 100644
+--- a/drivers/scsi/bnx2i/bnx2i_hwi.c
++++ b/drivers/scsi/bnx2i/bnx2i_hwi.c
+@@ -1977,7 +1977,7 @@ static int bnx2i_process_new_cqes(struct bnx2i_conn *bnx2i_conn)
+ if (nopin->cq_req_sn != qp->cqe_exp_seq_sn)
+ break;
+
+- if (unlikely(test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx))) {
++ if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) {
+ if (nopin->op_code == ISCSI_OP_NOOP_IN &&
+ nopin->itt == (u16) RESERVED_ITT) {
+ printk(KERN_ALERT "bnx2i: Unsolicited "
+diff --git a/drivers/scsi/bnx2i/bnx2i_iscsi.c b/drivers/scsi/bnx2i/bnx2i_iscsi.c
+index 1b5f3e143f071..2e5241d12dc3a 100644
+--- a/drivers/scsi/bnx2i/bnx2i_iscsi.c
++++ b/drivers/scsi/bnx2i/bnx2i_iscsi.c
+@@ -1721,7 +1721,7 @@ static int bnx2i_tear_down_conn(struct bnx2i_hba *hba,
+ struct iscsi_conn *conn = ep->conn->cls_conn->dd_data;
+
+ /* Must suspend all rx queue activity for this ep */
+- set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
++ set_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags);
+ }
+ /* CONN_DISCONNECT timeout may or may not be an issue depending
+ * on what transcribed in TCP layer, different targets behave
+diff --git a/drivers/scsi/csiostor/csio_lnode.c b/drivers/scsi/csiostor/csio_lnode.c
+index dc98f51f466fb..d5ac938970232 100644
+--- a/drivers/scsi/csiostor/csio_lnode.c
++++ b/drivers/scsi/csiostor/csio_lnode.c
+@@ -619,7 +619,7 @@ csio_ln_vnp_read_cbfn(struct csio_hw *hw, struct csio_mb *mbp)
+ struct fc_els_csp *csp;
+ struct fc_els_cssp *clsp;
+ enum fw_retval retval;
+- __be32 nport_id;
++ __be32 nport_id = 0;
+
+ retval = FW_CMD_RETVAL_G(ntohl(rsp->alloc_to_len16));
+ if (retval != FW_SUCCESS) {
+diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
+index 8c7d4dda4cf29..4365d52c6430e 100644
+--- a/drivers/scsi/cxgbi/libcxgbi.c
++++ b/drivers/scsi/cxgbi/libcxgbi.c
+@@ -1634,11 +1634,11 @@ void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, conn 0x%p.\n", csk, conn);
+
+- if (unlikely(!conn || conn->suspend_rx)) {
++ if (unlikely(!conn || test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) {
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+- "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
++ "csk 0x%p, conn 0x%p, id %d, conn flags 0x%lx!\n",
+ csk, conn, conn ? conn->id : 0xFF,
+- conn ? conn->suspend_rx : 0xFF);
++ conn ? conn->flags : 0xFF);
+ return;
+ }
+
+diff --git a/drivers/scsi/dc395x.c b/drivers/scsi/dc395x.c
+index 24c7cefb0b78a..1c79e6c271630 100644
+--- a/drivers/scsi/dc395x.c
++++ b/drivers/scsi/dc395x.c
+@@ -4618,6 +4618,7 @@ static int dc395x_init_one(struct pci_dev *dev, const struct pci_device_id *id)
+ /* initialise the adapter and everything we need */
+ if (adapter_init(acb, io_port_base, io_port_len, irq)) {
+ dprintkl(KERN_INFO, "adapter init failed\n");
++ acb = NULL;
+ goto fail;
+ }
+
+diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
+index 37d06f993b761..1d9be771f3ee0 100644
+--- a/drivers/scsi/device_handler/scsi_dh_alua.c
++++ b/drivers/scsi/device_handler/scsi_dh_alua.c
+@@ -1172,9 +1172,8 @@ static blk_status_t alua_prep_fn(struct scsi_device *sdev, struct request *req)
+ case SCSI_ACCESS_STATE_OPTIMAL:
+ case SCSI_ACCESS_STATE_ACTIVE:
+ case SCSI_ACCESS_STATE_LBA:
+- return BLK_STS_OK;
+ case SCSI_ACCESS_STATE_TRANSITIONING:
+- return BLK_STS_AGAIN;
++ return BLK_STS_OK;
+ default:
+ req->rq_flags |= RQF_QUIET;
+ return BLK_STS_IOERR;
+diff --git a/drivers/scsi/elx/libefc/efc_els.c b/drivers/scsi/elx/libefc/efc_els.c
+index 24db0accb256e..5f690378fe9a9 100644
+--- a/drivers/scsi/elx/libefc/efc_els.c
++++ b/drivers/scsi/elx/libefc/efc_els.c
+@@ -46,18 +46,14 @@ efc_els_io_alloc_size(struct efc_node *node, u32 reqlen, u32 rsplen)
+
+ efc = node->efc;
+
+- spin_lock_irqsave(&node->els_ios_lock, flags);
+-
+ if (!node->els_io_enabled) {
+ efc_log_err(efc, "els io alloc disabled\n");
+- spin_unlock_irqrestore(&node->els_ios_lock, flags);
+ return NULL;
+ }
+
+ els = mempool_alloc(efc->els_io_pool, GFP_ATOMIC);
+ if (!els) {
+ atomic_add_return(1, &efc->els_io_alloc_failed_count);
+- spin_unlock_irqrestore(&node->els_ios_lock, flags);
+ return NULL;
+ }
+
+@@ -74,7 +70,6 @@ efc_els_io_alloc_size(struct efc_node *node, u32 reqlen, u32 rsplen)
+ &els->io.req.phys, GFP_DMA);
+ if (!els->io.req.virt) {
+ mempool_free(els, efc->els_io_pool);
+- spin_unlock_irqrestore(&node->els_ios_lock, flags);
+ return NULL;
+ }
+
+@@ -94,10 +89,11 @@ efc_els_io_alloc_size(struct efc_node *node, u32 reqlen, u32 rsplen)
+
+ /* add els structure to ELS IO list */
+ INIT_LIST_HEAD(&els->list_entry);
++ spin_lock_irqsave(&node->els_ios_lock, flags);
+ list_add_tail(&els->list_entry, &node->els_ios_list);
++ spin_unlock_irqrestore(&node->els_ios_lock, flags);
+ }
+
+- spin_unlock_irqrestore(&node->els_ios_lock, flags);
+ return els;
+ }
+
+diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
+index f8afbfb468dca..d084a7db3925e 100644
+--- a/drivers/scsi/fnic/fnic_scsi.c
++++ b/drivers/scsi/fnic/fnic_scsi.c
+@@ -604,7 +604,7 @@ out:
+
+ FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no,
+ tag, sc, io_req, sg_count, cmd_trace,
+- (((u64)CMD_FLAGS(sc) >> 32) | CMD_STATE(sc)));
++ (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc)));
+
+ /* if only we issued IO, will we have the io lock */
+ if (io_lock_acquired)
+diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+index 3ab669dc806f6..1f5e0688c0c84 100644
+--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
++++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+@@ -527,7 +527,7 @@ MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)");
+
+ /* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
+ static int prot_mask;
+-module_param(prot_mask, int, 0);
++module_param(prot_mask, int, 0444);
+ MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 ");
+
+ static void debugfs_work_handler_v3_hw(struct work_struct *work);
+@@ -2392,17 +2392,25 @@ static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
+ return IRQ_WAKE_THREAD;
+ }
+
++static void hisi_sas_v3_free_vectors(void *data)
++{
++ struct pci_dev *pdev = data;
++
++ pci_free_irq_vectors(pdev);
++}
++
+ static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
+ {
+ int vectors;
+ int max_msi = HISI_SAS_MSI_COUNT_V3_HW, min_msi;
+ struct Scsi_Host *shost = hisi_hba->shost;
++ struct pci_dev *pdev = hisi_hba->pci_dev;
+ struct irq_affinity desc = {
+ .pre_vectors = BASE_VECTORS_V3_HW,
+ };
+
+ min_msi = MIN_AFFINE_VECTORS_V3_HW;
+- vectors = pci_alloc_irq_vectors_affinity(hisi_hba->pci_dev,
++ vectors = pci_alloc_irq_vectors_affinity(pdev,
+ min_msi, max_msi,
+ PCI_IRQ_MSI |
+ PCI_IRQ_AFFINITY,
+@@ -2414,6 +2422,7 @@ static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
+ hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW;
+ shost->nr_hw_queues = hisi_hba->cq_nvecs;
+
++ devm_add_action(&pdev->dev, hisi_sas_v3_free_vectors, pdev);
+ return 0;
+ }
+
+@@ -3959,6 +3968,54 @@ static const struct file_operations debugfs_bist_phy_v3_hw_fops = {
+ .owner = THIS_MODULE,
+ };
+
++static ssize_t debugfs_bist_cnt_v3_hw_write(struct file *filp,
++ const char __user *buf,
++ size_t count, loff_t *ppos)
++{
++ struct seq_file *m = filp->private_data;
++ struct hisi_hba *hisi_hba = m->private;
++ unsigned int cnt;
++ int val;
++
++ if (hisi_hba->debugfs_bist_enable)
++ return -EPERM;
++
++ val = kstrtouint_from_user(buf, count, 0, &cnt);
++ if (val)
++ return val;
++
++ if (cnt)
++ return -EINVAL;
++
++ hisi_hba->debugfs_bist_cnt = 0;
++ return count;
++}
++
++static int debugfs_bist_cnt_v3_hw_show(struct seq_file *s, void *p)
++{
++ struct hisi_hba *hisi_hba = s->private;
++
++ seq_printf(s, "%u\n", hisi_hba->debugfs_bist_cnt);
++
++ return 0;
++}
++
++static int debugfs_bist_cnt_v3_hw_open(struct inode *inode,
++ struct file *filp)
++{
++ return single_open(filp, debugfs_bist_cnt_v3_hw_show,
++ inode->i_private);
++}
++
++static const struct file_operations debugfs_bist_cnt_v3_hw_ops = {
++ .open = debugfs_bist_cnt_v3_hw_open,
++ .read = seq_read,
++ .write = debugfs_bist_cnt_v3_hw_write,
++ .llseek = seq_lseek,
++ .release = single_release,
++ .owner = THIS_MODULE,
++};
++
+ static const struct {
+ int value;
+ char *name;
+@@ -4596,8 +4653,8 @@ static void debugfs_bist_init_v3_hw(struct hisi_hba *hisi_hba)
+ debugfs_create_file("phy_id", 0600, hisi_hba->debugfs_bist_dentry,
+ hisi_hba, &debugfs_bist_phy_v3_hw_fops);
+
+- debugfs_create_u32("cnt", 0600, hisi_hba->debugfs_bist_dentry,
+- &hisi_hba->debugfs_bist_cnt);
++ debugfs_create_file("cnt", 0600, hisi_hba->debugfs_bist_dentry,
++ hisi_hba, &debugfs_bist_cnt_v3_hw_ops);
+
+ debugfs_create_file("loopback_mode", 0600,
+ hisi_hba->debugfs_bist_dentry,
+@@ -4763,7 +4820,7 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ dev_err(dev, "%d hw queues\n", shost->nr_hw_queues);
+ rc = scsi_add_host(shost, dev);
+ if (rc)
+- goto err_out_free_irq_vectors;
++ goto err_out_debugfs;
+
+ rc = sas_register_ha(sha);
+ if (rc)
+@@ -4792,8 +4849,6 @@ err_out_hw_init:
+ sas_unregister_ha(sha);
+ err_out_register_ha:
+ scsi_remove_host(shost);
+-err_out_free_irq_vectors:
+- pci_free_irq_vectors(pdev);
+ err_out_debugfs:
+ debugfs_exit_v3_hw(hisi_hba);
+ err_out_ha:
+@@ -4821,7 +4876,6 @@ hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
+
+ devm_free_irq(&pdev->dev, pci_irq_vector(pdev, nr), cq);
+ }
+- pci_free_irq_vectors(pdev);
+ }
+
+ static void hisi_sas_v3_remove(struct pci_dev *pdev)
+diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
+index 24b72ee4246fb..0165dad803001 100644
+--- a/drivers/scsi/hosts.c
++++ b/drivers/scsi/hosts.c
+@@ -388,6 +388,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
+ shost->shost_state = SHOST_CREATED;
+ INIT_LIST_HEAD(&shost->__devices);
+ INIT_LIST_HEAD(&shost->__targets);
++ INIT_LIST_HEAD(&shost->eh_abort_list);
+ INIT_LIST_HEAD(&shost->eh_cmd_q);
+ INIT_LIST_HEAD(&shost->starved_list);
+ init_waitqueue_head(&shost->host_wait);
+diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+index 10b6c6daaacda..d43bb18f58fd5 100644
+--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
++++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+@@ -36,7 +36,7 @@
+
+ #define IBMVSCSIS_VERSION "v0.2"
+
+-#define INITIAL_SRP_LIMIT 800
++#define INITIAL_SRP_LIMIT 1024
+ #define DEFAULT_MAX_SECTORS 256
+ #define MAX_TXU 1024 * 1024
+
+diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c
+index 841000445b9a1..aa223db4cf53c 100644
+--- a/drivers/scsi/libfc/fc_exch.c
++++ b/drivers/scsi/libfc/fc_exch.c
+@@ -1701,6 +1701,7 @@ static void fc_exch_abts_resp(struct fc_exch *ep, struct fc_frame *fp)
+ if (cancel_delayed_work_sync(&ep->timeout_work)) {
+ FC_EXCH_DBG(ep, "Exchange timer canceled due to ABTS response\n");
+ fc_exch_release(ep); /* release from pending timer hold */
++ return;
+ }
+
+ spin_lock_bh(&ep->ex_lock);
+diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
+index 5bc91d34df634..0f2c7098f9d6e 100644
+--- a/drivers/scsi/libiscsi.c
++++ b/drivers/scsi/libiscsi.c
+@@ -678,7 +678,8 @@ __iscsi_conn_send_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
+ struct iscsi_task *task;
+ itt_t itt;
+
+- if (session->state == ISCSI_STATE_TERMINATE)
++ if (session->state == ISCSI_STATE_TERMINATE ||
++ !test_bit(ISCSI_CONN_FLAG_BOUND, &conn->flags))
+ return NULL;
+
+ if (opcode == ISCSI_OP_LOGIN || opcode == ISCSI_OP_TEXT) {
+@@ -1392,8 +1393,8 @@ static bool iscsi_set_conn_failed(struct iscsi_conn *conn)
+ if (conn->stop_stage == 0)
+ session->state = ISCSI_STATE_FAILED;
+
+- set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
+- set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
++ set_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags);
++ set_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags);
+ return true;
+ }
+
+@@ -1454,7 +1455,7 @@ static int iscsi_xmit_task(struct iscsi_conn *conn, struct iscsi_task *task,
+ * Do this after dropping the extra ref because if this was a requeue
+ * it's removed from that list and cleanup_queued_task would miss it.
+ */
+- if (test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx)) {
++ if (test_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags)) {
+ /*
+ * Save the task and ref in case we weren't cleaning up this
+ * task and get woken up again.
+@@ -1532,7 +1533,7 @@ static int iscsi_data_xmit(struct iscsi_conn *conn)
+ int rc = 0;
+
+ spin_lock_bh(&conn->session->frwd_lock);
+- if (test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx)) {
++ if (test_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags)) {
+ ISCSI_DBG_SESSION(conn->session, "Tx suspended!\n");
+ spin_unlock_bh(&conn->session->frwd_lock);
+ return -ENODATA;
+@@ -1746,7 +1747,7 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
+ goto fault;
+ }
+
+- if (test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx)) {
++ if (test_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags)) {
+ reason = FAILURE_SESSION_IN_RECOVERY;
+ sc->result = DID_REQUEUE << 16;
+ goto fault;
+@@ -1935,7 +1936,7 @@ static void fail_scsi_tasks(struct iscsi_conn *conn, u64 lun, int error)
+ void iscsi_suspend_queue(struct iscsi_conn *conn)
+ {
+ spin_lock_bh(&conn->session->frwd_lock);
+- set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
++ set_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags);
+ spin_unlock_bh(&conn->session->frwd_lock);
+ }
+ EXPORT_SYMBOL_GPL(iscsi_suspend_queue);
+@@ -1953,7 +1954,7 @@ void iscsi_suspend_tx(struct iscsi_conn *conn)
+ struct Scsi_Host *shost = conn->session->host;
+ struct iscsi_host *ihost = shost_priv(shost);
+
+- set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
++ set_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags);
+ if (ihost->workq)
+ flush_workqueue(ihost->workq);
+ }
+@@ -1961,7 +1962,7 @@ EXPORT_SYMBOL_GPL(iscsi_suspend_tx);
+
+ static void iscsi_start_tx(struct iscsi_conn *conn)
+ {
+- clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
++ clear_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags);
+ iscsi_conn_queue_work(conn);
+ }
+
+@@ -2214,6 +2215,8 @@ void iscsi_conn_unbind(struct iscsi_cls_conn *cls_conn, bool is_active)
+ iscsi_suspend_tx(conn);
+
+ spin_lock_bh(&session->frwd_lock);
++ clear_bit(ISCSI_CONN_FLAG_BOUND, &conn->flags);
++
+ if (!is_active) {
+ /*
+ * if logout timed out before userspace could even send a PDU
+@@ -3101,6 +3104,8 @@ void iscsi_conn_teardown(struct iscsi_cls_conn *cls_conn)
+ {
+ struct iscsi_conn *conn = cls_conn->dd_data;
+ struct iscsi_session *session = conn->session;
++ char *tmp_persistent_address = conn->persistent_address;
++ char *tmp_local_ipaddr = conn->local_ipaddr;
+
+ del_timer_sync(&conn->transport_timer);
+
+@@ -3122,8 +3127,6 @@ void iscsi_conn_teardown(struct iscsi_cls_conn *cls_conn)
+ spin_lock_bh(&session->frwd_lock);
+ free_pages((unsigned long) conn->data,
+ get_order(ISCSI_DEF_MAX_RECV_SEG_LEN));
+- kfree(conn->persistent_address);
+- kfree(conn->local_ipaddr);
+ /* regular RX path uses back_lock */
+ spin_lock_bh(&session->back_lock);
+ kfifo_in(&session->cmdpool.queue, (void*)&conn->login_task,
+@@ -3135,6 +3138,8 @@ void iscsi_conn_teardown(struct iscsi_cls_conn *cls_conn)
+ mutex_unlock(&session->eh_mutex);
+
+ iscsi_destroy_conn(cls_conn);
++ kfree(tmp_persistent_address);
++ kfree(tmp_local_ipaddr);
+ }
+ EXPORT_SYMBOL_GPL(iscsi_conn_teardown);
+
+@@ -3310,6 +3315,8 @@ int iscsi_conn_bind(struct iscsi_cls_session *cls_session,
+ spin_lock_bh(&session->frwd_lock);
+ if (is_leading)
+ session->leadconn = conn;
++
++ set_bit(ISCSI_CONN_FLAG_BOUND, &conn->flags);
+ spin_unlock_bh(&session->frwd_lock);
+
+ /*
+@@ -3322,8 +3329,8 @@ int iscsi_conn_bind(struct iscsi_cls_session *cls_session,
+ /*
+ * Unblock xmitworker(), Login Phase will pass through.
+ */
+- clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
+- clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
++ clear_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags);
++ clear_bit(ISCSI_CONN_FLAG_SUSPEND_TX, &conn->flags);
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(iscsi_conn_bind);
+diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c
+index 2e9ffe3d1a55e..883005757ddb8 100644
+--- a/drivers/scsi/libiscsi_tcp.c
++++ b/drivers/scsi/libiscsi_tcp.c
+@@ -927,7 +927,7 @@ int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,
+ */
+ conn->last_recv = jiffies;
+
+- if (unlikely(conn->suspend_rx)) {
++ if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) {
+ ISCSI_DBG_TCP(conn, "Rx suspended!\n");
+ *status = ISCSI_TCP_SUSPENDED;
+ return 0;
+diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
+index a315715b36227..7e0cde710fc3c 100644
+--- a/drivers/scsi/libsas/sas_ata.c
++++ b/drivers/scsi/libsas/sas_ata.c
+@@ -197,7 +197,7 @@ static unsigned int sas_ata_qc_issue(struct ata_queued_cmd *qc)
+ task->total_xfer_len = qc->nbytes;
+ task->num_scatter = qc->n_elem;
+ task->data_dir = qc->dma_dir;
+- } else if (qc->tf.protocol == ATA_PROT_NODATA) {
++ } else if (!ata_is_data(qc->tf.protocol)) {
+ task->data_dir = DMA_NONE;
+ } else {
+ for_each_sg(qc->sg, sg, qc->n_elem, si)
+diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
+index befeb7c342903..1044832b60549 100644
+--- a/drivers/scsi/lpfc/lpfc.h
++++ b/drivers/scsi/lpfc/lpfc.h
+@@ -593,6 +593,7 @@ struct lpfc_vport {
+ #define FC_VPORT_LOGO_RCVD 0x200 /* LOGO received on vport */
+ #define FC_RSCN_DISCOVERY 0x400 /* Auth all devices after RSCN */
+ #define FC_LOGO_RCVD_DID_CHNG 0x800 /* FDISC on phys port detect DID chng*/
++#define FC_PT2PT_NO_NVME 0x1000 /* Don't send NVME PRLI */
+ #define FC_SCSI_SCAN_TMO 0x4000 /* scsi scan timer running */
+ #define FC_ABORT_DISCOVERY 0x8000 /* we want to abort discovery */
+ #define FC_NDISC_ACTIVE 0x10000 /* NPort discovery active */
+@@ -1022,7 +1023,6 @@ struct lpfc_hba {
+ #define HBA_DEVLOSS_TMO 0x2000 /* HBA in devloss timeout */
+ #define HBA_RRQ_ACTIVE 0x4000 /* process the rrq active list */
+ #define HBA_IOQ_FLUSH 0x8000 /* FCP/NVME I/O queues being flushed */
+-#define HBA_FW_DUMP_OP 0x10000 /* Skips fn reset before FW dump */
+ #define HBA_RECOVERABLE_UE 0x20000 /* Firmware supports recoverable UE */
+ #define HBA_FORCED_LINK_SPEED 0x40000 /*
+ * Firmware supports Forced Link Speed
+@@ -1038,6 +1038,7 @@ struct lpfc_hba {
+ #define HBA_HBEAT_TMO 0x8000000 /* HBEAT initiated after timeout */
+ #define HBA_FLOGI_OUTSTANDING 0x10000000 /* FLOGI is outstanding */
+
++ struct completion *fw_dump_cmpl; /* cmpl event tracker for fw_dump */
+ uint32_t fcp_ring_in_use; /* When polling test if intr-hndlr active*/
+ struct lpfc_dmabuf slim2p;
+
+@@ -1163,6 +1164,16 @@ struct lpfc_hba {
+ uint32_t cfg_hostmem_hgp;
+ uint32_t cfg_log_verbose;
+ uint32_t cfg_enable_fc4_type;
++#define LPFC_ENABLE_FCP 1
++#define LPFC_ENABLE_NVME 2
++#define LPFC_ENABLE_BOTH 3
++#if (IS_ENABLED(CONFIG_NVME_FC))
++#define LPFC_MAX_ENBL_FC4_TYPE LPFC_ENABLE_BOTH
++#define LPFC_DEF_ENBL_FC4_TYPE LPFC_ENABLE_BOTH
++#else
++#define LPFC_MAX_ENBL_FC4_TYPE LPFC_ENABLE_FCP
++#define LPFC_DEF_ENBL_FC4_TYPE LPFC_ENABLE_FCP
++#endif
+ uint32_t cfg_aer_support;
+ uint32_t cfg_sriov_nr_virtfn;
+ uint32_t cfg_request_firmware_upgrade;
+@@ -1184,9 +1195,6 @@ struct lpfc_hba {
+ uint32_t cfg_ras_fwlog_func;
+ uint32_t cfg_enable_bbcr; /* Enable BB Credit Recovery */
+ uint32_t cfg_enable_dpp; /* Enable Direct Packet Push */
+-#define LPFC_ENABLE_FCP 1
+-#define LPFC_ENABLE_NVME 2
+-#define LPFC_ENABLE_BOTH 3
+ uint32_t cfg_enable_pbde;
+ uint32_t cfg_enable_mi;
+ struct nvmet_fc_target_port *targetport;
+diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
+index ebe417921dac0..9f3f7805f1f95 100644
+--- a/drivers/scsi/lpfc/lpfc_attr.c
++++ b/drivers/scsi/lpfc/lpfc_attr.c
+@@ -1315,6 +1315,9 @@ lpfc_issue_lip(struct Scsi_Host *shost)
+ pmboxq->u.mb.mbxCommand = MBX_DOWN_LINK;
+ pmboxq->u.mb.mbxOwner = OWN_HOST;
+
++ if ((vport->fc_flag & FC_PT2PT) && (vport->fc_flag & FC_PT2PT_NO_NVME))
++ vport->fc_flag &= ~FC_PT2PT_NO_NVME;
++
+ mbxstatus = lpfc_sli_issue_mbox_wait(phba, pmboxq, LPFC_MBOX_TMO * 2);
+
+ if ((mbxstatus == MBX_SUCCESS) &&
+@@ -1709,25 +1712,25 @@ lpfc_sli4_pdev_reg_request(struct lpfc_hba *phba, uint32_t opcode)
+ before_fc_flag = phba->pport->fc_flag;
+ sriov_nr_virtfn = phba->cfg_sriov_nr_virtfn;
+
+- /* Disable SR-IOV virtual functions if enabled */
+- if (phba->cfg_sriov_nr_virtfn) {
+- pci_disable_sriov(pdev);
+- phba->cfg_sriov_nr_virtfn = 0;
+- }
++ if (opcode == LPFC_FW_DUMP) {
++ init_completion(&online_compl);
++ phba->fw_dump_cmpl = &online_compl;
++ } else {
++ /* Disable SR-IOV virtual functions if enabled */
++ if (phba->cfg_sriov_nr_virtfn) {
++ pci_disable_sriov(pdev);
++ phba->cfg_sriov_nr_virtfn = 0;
++ }
+
+- if (opcode == LPFC_FW_DUMP)
+- phba->hba_flag |= HBA_FW_DUMP_OP;
++ status = lpfc_do_offline(phba, LPFC_EVT_OFFLINE);
+
+- status = lpfc_do_offline(phba, LPFC_EVT_OFFLINE);
++ if (status != 0)
++ return status;
+
+- if (status != 0) {
+- phba->hba_flag &= ~HBA_FW_DUMP_OP;
+- return status;
++ /* wait for the device to be quiesced before firmware reset */
++ msleep(100);
+ }
+
+- /* wait for the device to be quiesced before firmware reset */
+- msleep(100);
+-
+ reg_val = readl(phba->sli4_hba.conf_regs_memmap_p +
+ LPFC_CTL_PDEV_CTL_OFFSET);
+
+@@ -1756,24 +1759,42 @@ lpfc_sli4_pdev_reg_request(struct lpfc_hba *phba, uint32_t opcode)
+ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "3153 Fail to perform the requested "
+ "access: x%x\n", reg_val);
++ if (phba->fw_dump_cmpl)
++ phba->fw_dump_cmpl = NULL;
+ return rc;
+ }
+
+ /* keep the original port state */
+- if (before_fc_flag & FC_OFFLINE_MODE)
+- goto out;
+-
+- init_completion(&online_compl);
+- job_posted = lpfc_workq_post_event(phba, &status, &online_compl,
+- LPFC_EVT_ONLINE);
+- if (!job_posted)
++ if (before_fc_flag & FC_OFFLINE_MODE) {
++ if (phba->fw_dump_cmpl)
++ phba->fw_dump_cmpl = NULL;
+ goto out;
++ }
+
+- wait_for_completion(&online_compl);
++ /* Firmware dump will trigger an HA_ERATT event, and
++ * lpfc_handle_eratt_s4 routine already handles bringing the port back
++ * online.
++ */
++ if (opcode == LPFC_FW_DUMP) {
++ wait_for_completion(phba->fw_dump_cmpl);
++ } else {
++ init_completion(&online_compl);
++ job_posted = lpfc_workq_post_event(phba, &status, &online_compl,
++ LPFC_EVT_ONLINE);
++ if (!job_posted)
++ goto out;
+
++ wait_for_completion(&online_compl);
++ }
+ out:
+ /* in any case, restore the virtual functions enabled as before */
+ if (sriov_nr_virtfn) {
++ /* If fw_dump was performed, first disable to clean up */
++ if (opcode == LPFC_FW_DUMP) {
++ pci_disable_sriov(pdev);
++ phba->cfg_sriov_nr_virtfn = 0;
++ }
++
+ sriov_err =
+ lpfc_sli_probe_sriov_nr_virtfn(phba, sriov_nr_virtfn);
+ if (!sriov_err)
+@@ -3960,8 +3981,8 @@ LPFC_ATTR_R(nvmet_mrq_post,
+ * 3 - register both FCP and NVME
+ * Supported values are [1,3]. Default value is 3
+ */
+-LPFC_ATTR_R(enable_fc4_type, LPFC_ENABLE_BOTH,
+- LPFC_ENABLE_FCP, LPFC_ENABLE_BOTH,
++LPFC_ATTR_R(enable_fc4_type, LPFC_DEF_ENBL_FC4_TYPE,
++ LPFC_ENABLE_FCP, LPFC_MAX_ENBL_FC4_TYPE,
+ "Enable FC4 Protocol support - FCP / NVME");
+
+ /*
+diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
+index c512f41991429..c9770b1d2366a 100644
+--- a/drivers/scsi/lpfc/lpfc_crtn.h
++++ b/drivers/scsi/lpfc/lpfc_crtn.h
+@@ -119,6 +119,8 @@ int lpfc_check_sli_ndlp(struct lpfc_hba *, struct lpfc_sli_ring *,
+ struct lpfc_nodelist *lpfc_nlp_init(struct lpfc_vport *vport, uint32_t did);
+ struct lpfc_nodelist *lpfc_nlp_get(struct lpfc_nodelist *);
+ int lpfc_nlp_put(struct lpfc_nodelist *);
++void lpfc_check_nlp_post_devloss(struct lpfc_vport *vport,
++ struct lpfc_nodelist *ndlp);
+ void lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ struct lpfc_iocbq *rspiocb);
+ int lpfc_nlp_not_used(struct lpfc_nodelist *ndlp);
+diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
+index bd6d459afce54..08b2e85dcd7d8 100644
+--- a/drivers/scsi/lpfc/lpfc_debugfs.c
++++ b/drivers/scsi/lpfc/lpfc_debugfs.c
+@@ -2954,8 +2954,8 @@ lpfc_debugfs_nvmeio_trc_write(struct file *file, const char __user *buf,
+ char mybuf[64];
+ char *pbuf;
+
+- if (nbytes > 64)
+- nbytes = 64;
++ if (nbytes > 63)
++ nbytes = 63;
+
+ memset(mybuf, 0, sizeof(mybuf));
+
+diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
+index 871b665bd72e3..37a4b79010bfc 100644
+--- a/drivers/scsi/lpfc/lpfc_disc.h
++++ b/drivers/scsi/lpfc/lpfc_disc.h
+@@ -85,6 +85,13 @@ enum lpfc_fc4_xpt_flags {
+ NLP_XPT_HAS_HH = 0x10
+ };
+
++enum lpfc_nlp_save_flags {
++ /* devloss occurred during recovery */
++ NLP_IN_RECOV_POST_DEV_LOSS = 0x1,
++ /* wait for outstanding LOGO to cmpl */
++ NLP_WAIT_FOR_LOGO = 0x2,
++};
++
+ struct lpfc_nodelist {
+ struct list_head nlp_listp;
+ struct serv_parm fc_sparam; /* buffer for service params */
+@@ -144,8 +151,9 @@ struct lpfc_nodelist {
+ unsigned long *active_rrqs_xri_bitmap;
+ struct lpfc_scsicmd_bkt *lat_data; /* Latency data */
+ uint32_t fc4_prli_sent;
+- u32 upcall_flags;
+-#define NLP_WAIT_FOR_LOGO 0x2
++
++ /* flags to keep ndlp alive until special conditions are met */
++ enum lpfc_nlp_save_flags save_flags;
+
+ enum lpfc_fc4_xpt_flags fc4_xpt_flags;
+
+diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
+index 052c0e5b11195..886006ad12a29 100644
+--- a/drivers/scsi/lpfc/lpfc_els.c
++++ b/drivers/scsi/lpfc/lpfc_els.c
+@@ -1059,9 +1059,10 @@ stop_rr_fcf_flogi:
+
+ lpfc_printf_vlog(vport, KERN_WARNING, LOG_TRACE_EVENT,
+ "0150 FLOGI failure Status:x%x/x%x "
+- "xri x%x TMO:x%x\n",
++ "xri x%x TMO:x%x refcnt %d\n",
+ irsp->ulpStatus, irsp->un.ulpWord[4],
+- cmdiocb->sli4_xritag, irsp->ulpTimeout);
++ cmdiocb->sli4_xritag, irsp->ulpTimeout,
++ kref_read(&ndlp->kref));
+
+ /* If this is not a loop open failure, bail out */
+ if (!(irsp->ulpStatus == IOSTAT_LOCAL_REJECT &&
+@@ -1071,7 +1072,8 @@ stop_rr_fcf_flogi:
+
+ /* FLOGI failed, so there is no fabric */
+ spin_lock_irq(shost->host_lock);
+- vport->fc_flag &= ~(FC_FABRIC | FC_PUBLIC_LOOP);
++ vport->fc_flag &= ~(FC_FABRIC | FC_PUBLIC_LOOP |
++ FC_PT2PT_NO_NVME);
+ spin_unlock_irq(shost->host_lock);
+
+ /* If private loop, then allow max outstanding els to be
+@@ -1122,12 +1124,12 @@ stop_rr_fcf_flogi:
+ /* FLOGI completes successfully */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+ "0101 FLOGI completes successfully, I/O tag:x%x, "
+- "xri x%x Data: x%x x%x x%x x%x x%x x%x x%x\n",
++ "xri x%x Data: x%x x%x x%x x%x x%x x%x x%x %d\n",
+ cmdiocb->iotag, cmdiocb->sli4_xritag,
+ irsp->un.ulpWord[4], sp->cmn.e_d_tov,
+ sp->cmn.w2.r_a_tov, sp->cmn.edtovResolution,
+ vport->port_state, vport->fc_flag,
+- sp->cmn.priority_tagging);
++ sp->cmn.priority_tagging, kref_read(&ndlp->kref));
+
+ if (sp->cmn.priority_tagging)
+ vport->vmid_flag |= LPFC_VMID_ISSUE_QFPA;
+@@ -1205,8 +1207,6 @@ flogifail:
+ phba->fcf.fcf_flag &= ~FCF_DISCOVERY;
+ spin_unlock_irq(&phba->hbalock);
+
+- if (!(ndlp->fc4_xpt_flags & (SCSI_XPT_REGD | NVME_XPT_REGD)))
+- lpfc_nlp_put(ndlp);
+ if (!lpfc_error_lost_link(irsp)) {
+ /* FLOGI failed, so just use loop map to make discovery list */
+ lpfc_disc_list_loopmap(vport);
+@@ -2899,9 +2899,9 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ irsp = &(rspiocb->iocb);
+ spin_lock_irq(&ndlp->lock);
+ ndlp->nlp_flag &= ~NLP_LOGO_SND;
+- if (ndlp->upcall_flags & NLP_WAIT_FOR_LOGO) {
++ if (ndlp->save_flags & NLP_WAIT_FOR_LOGO) {
+ wake_up_waiter = 1;
+- ndlp->upcall_flags &= ~NLP_WAIT_FOR_LOGO;
++ ndlp->save_flags &= ~NLP_WAIT_FOR_LOGO;
+ }
+ spin_unlock_irq(&ndlp->lock);
+
+@@ -3532,11 +3532,6 @@ lpfc_issue_els_rscn(struct lpfc_vport *vport, uint8_t retry)
+ return 1;
+ }
+
+- /* This will cause the callback-function lpfc_cmpl_els_cmd to
+- * trigger the release of node.
+- */
+- if (!(vport->fc_flag & FC_PT2PT))
+- lpfc_nlp_put(ndlp);
+ return 0;
+ }
+
+@@ -4593,6 +4588,23 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ /* Added for Vendor specifc support
+ * Just keep retrying for these Rsn / Exp codes
+ */
++ if ((vport->fc_flag & FC_PT2PT) &&
++ cmd == ELS_CMD_NVMEPRLI) {
++ switch (stat.un.b.lsRjtRsnCode) {
++ case LSRJT_UNABLE_TPC:
++ case LSRJT_INVALID_CMD:
++ case LSRJT_LOGICAL_ERR:
++ case LSRJT_CMD_UNSUPPORTED:
++ lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
++ "0168 NVME PRLI LS_RJT "
++ "reason %x port doesn't "
++ "support NVME, disabling NVME\n",
++ stat.un.b.lsRjtRsnCode);
++ retry = 0;
++ vport->fc_flag |= FC_PT2PT_NO_NVME;
++ goto out_retry;
++ }
++ }
+ switch (stat.un.b.lsRjtRsnCode) {
+ case LSRJT_UNABLE_TPC:
+ /* The driver has a VALID PLOGI but the rport has
+@@ -5076,14 +5088,9 @@ lpfc_cmpl_els_logo_acc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ /* NPort Recovery mode or node is just allocated */
+ if (!lpfc_nlp_not_used(ndlp)) {
+ /* A LOGO is completing and the node is in NPR state.
+- * If this a fabric node that cleared its transport
+- * registration, release the rpi.
++ * Just unregister the RPI because the node is still
++ * required.
+ */
+- spin_lock_irq(&ndlp->lock);
+- ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;
+- if (phba->sli_rev == LPFC_SLI_REV4)
+- ndlp->nlp_flag |= NLP_RELEASE_RPI;
+- spin_unlock_irq(&ndlp->lock);
+ lpfc_unreg_rpi(vport, ndlp);
+ } else {
+ /* Indicate the node has already released, should
+@@ -6216,6 +6223,7 @@ lpfc_els_disc_adisc(struct lpfc_vport *vport)
+ * from backend
+ */
+ lpfc_nlp_unreg_node(vport, ndlp);
++ lpfc_unreg_rpi(vport, ndlp);
+ continue;
+ }
+
+@@ -6882,6 +6890,7 @@ static int
+ lpfc_get_rdp_info(struct lpfc_hba *phba, struct lpfc_rdp_context *rdp_context)
+ {
+ LPFC_MBOXQ_t *mbox = NULL;
++ struct lpfc_dmabuf *mp;
+ int rc;
+
+ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+@@ -6897,8 +6906,11 @@ lpfc_get_rdp_info(struct lpfc_hba *phba, struct lpfc_rdp_context *rdp_context)
+ mbox->mbox_cmpl = lpfc_mbx_cmpl_rdp_page_a0;
+ mbox->ctx_ndlp = (struct lpfc_rdp_context *)rdp_context;
+ rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
+- if (rc == MBX_NOT_FINISHED)
++ if (rc == MBX_NOT_FINISHED) {
++ mp = (struct lpfc_dmabuf *)mbox->ctx_buf;
++ lpfc_mbuf_free(phba, mp->virt, mp->phys);
+ goto issue_mbox_fail;
++ }
+
+ return 0;
+
+@@ -10713,6 +10725,9 @@ lpfc_cmpl_els_fdisc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ irsp->ulpStatus, irsp->un.ulpWord[4]);
+ goto fdisc_failed;
+ }
++
++ lpfc_check_nlp_post_devloss(vport, ndlp);
++
+ spin_lock_irq(shost->host_lock);
+ vport->fc_flag &= ~FC_VPORT_CVL_RCVD;
+ vport->fc_flag &= ~FC_VPORT_LOGO_RCVD;
+diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
+index 7195ca0275f93..3bb7c2aa949f7 100644
+--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
++++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
+@@ -209,7 +209,12 @@ lpfc_dev_loss_tmo_callbk(struct fc_rport *rport)
+
+ spin_lock_irqsave(&ndlp->lock, iflags);
+ ndlp->nlp_flag |= NLP_IN_DEV_LOSS;
+- ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;
++
++ /* If there is a PLOGI in progress, and we are in a
++ * NLP_NPR_2B_DISC state, don't turn off the flag.
++ */
++ if (ndlp->nlp_state != NLP_STE_PLOGI_ISSUE)
++ ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;
+
+ /*
+ * The backend does not expect any more calls associated with this
+@@ -340,6 +345,37 @@ static void lpfc_check_inactive_vmid(struct lpfc_hba *phba)
+ lpfc_destroy_vport_work_array(phba, vports);
+ }
+
++/**
++ * lpfc_check_nlp_post_devloss - Check to restore ndlp refcnt after devloss
++ * @vport: Pointer to vport object.
++ * @ndlp: Pointer to remote node object.
++ *
++ * If NLP_IN_RECOV_POST_DEV_LOSS flag was set due to outstanding recovery of
++ * node during dev_loss_tmo processing, then this function restores the nlp_put
++ * kref decrement from lpfc_dev_loss_tmo_handler.
++ **/
++void
++lpfc_check_nlp_post_devloss(struct lpfc_vport *vport,
++ struct lpfc_nodelist *ndlp)
++{
++ unsigned long iflags;
++
++ spin_lock_irqsave(&ndlp->lock, iflags);
++ if (ndlp->save_flags & NLP_IN_RECOV_POST_DEV_LOSS) {
++ ndlp->save_flags &= ~NLP_IN_RECOV_POST_DEV_LOSS;
++ spin_unlock_irqrestore(&ndlp->lock, iflags);
++ lpfc_nlp_get(ndlp);
++ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY | LOG_NODE,
++ "8438 Devloss timeout reversed on DID x%x "
++ "refcnt %d ndlp %p flag x%x "
++ "port_state = x%x\n",
++ ndlp->nlp_DID, kref_read(&ndlp->kref), ndlp,
++ ndlp->nlp_flag, vport->port_state);
++ spin_lock_irqsave(&ndlp->lock, iflags);
++ }
++ spin_unlock_irqrestore(&ndlp->lock, iflags);
++}
++
+ /**
+ * lpfc_dev_loss_tmo_handler - Remote node devloss timeout handler
+ * @ndlp: Pointer to remote node object.
+@@ -358,6 +394,8 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp)
+ uint8_t *name;
+ int warn_on = 0;
+ int fcf_inuse = 0;
++ bool recovering = false;
++ struct fc_vport *fc_vport = NULL;
+ unsigned long iflags;
+
+ vport = ndlp->vport;
+@@ -394,6 +432,64 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp)
+
+ /* Fabric nodes are done. */
+ if (ndlp->nlp_type & NLP_FABRIC) {
++ spin_lock_irqsave(&ndlp->lock, iflags);
++ /* In massive vport configuration settings, it's possible
++ * dev_loss_tmo fired during node recovery. So, check if
++ * fabric nodes are in discovery states outstanding.
++ */
++ switch (ndlp->nlp_DID) {
++ case Fabric_DID:
++ fc_vport = vport->fc_vport;
++ if (fc_vport &&
++ fc_vport->vport_state == FC_VPORT_INITIALIZING)
++ recovering = true;
++ break;
++ case Fabric_Cntl_DID:
++ if (ndlp->nlp_flag & NLP_REG_LOGIN_SEND)
++ recovering = true;
++ break;
++ case FDMI_DID:
++ fallthrough;
++ case NameServer_DID:
++ if (ndlp->nlp_state >= NLP_STE_PLOGI_ISSUE &&
++ ndlp->nlp_state <= NLP_STE_REG_LOGIN_ISSUE)
++ recovering = true;
++ break;
++ }
++ spin_unlock_irqrestore(&ndlp->lock, iflags);
++
++ /* Mark an NLP_IN_RECOV_POST_DEV_LOSS flag to know if reversing
++ * the following lpfc_nlp_put is necessary after fabric node is
++ * recovered.
++ */
++ if (recovering) {
++ lpfc_printf_vlog(vport, KERN_INFO,
++ LOG_DISCOVERY | LOG_NODE,
++ "8436 Devloss timeout marked on "
++ "DID x%x refcnt %d ndlp %p "
++ "flag x%x port_state = x%x\n",
++ ndlp->nlp_DID, kref_read(&ndlp->kref),
++ ndlp, ndlp->nlp_flag,
++ vport->port_state);
++ spin_lock_irqsave(&ndlp->lock, iflags);
++ ndlp->save_flags |= NLP_IN_RECOV_POST_DEV_LOSS;
++ spin_unlock_irqrestore(&ndlp->lock, iflags);
++ } else if (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) {
++ /* Fabric node fully recovered before this dev_loss_tmo
++ * queue work is processed. Thus, ignore the
++ * dev_loss_tmo event.
++ */
++ lpfc_printf_vlog(vport, KERN_INFO,
++ LOG_DISCOVERY | LOG_NODE,
++ "8437 Devloss timeout ignored on "
++ "DID x%x refcnt %d ndlp %p "
++ "flag x%x port_state = x%x\n",
++ ndlp->nlp_DID, kref_read(&ndlp->kref),
++ ndlp, ndlp->nlp_flag,
++ vport->port_state);
++ return fcf_inuse;
++ }
++
+ lpfc_nlp_put(ndlp);
+ return fcf_inuse;
+ }
+@@ -423,6 +519,14 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp)
+ ndlp->nlp_state, ndlp->nlp_rpi);
+ }
+
++ /* If we are devloss, but we are in the process of rediscovering the
++ * ndlp, don't issue a NLP_EVT_DEVICE_RM event.
++ */
++ if (ndlp->nlp_state >= NLP_STE_PLOGI_ISSUE &&
++ ndlp->nlp_state <= NLP_STE_PRLI_ISSUE) {
++ return fcf_inuse;
++ }
++
+ if (!(ndlp->fc4_xpt_flags & NVME_XPT_REGD))
+ lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
+
+@@ -765,10 +869,16 @@ lpfc_work_done(struct lpfc_hba *phba)
+ if (phba->pci_dev_grp == LPFC_PCI_DEV_OC)
+ lpfc_sli4_post_async_mbox(phba);
+
+- if (ha_copy & HA_ERATT)
++ if (ha_copy & HA_ERATT) {
+ /* Handle the error attention event */
+ lpfc_handle_eratt(phba);
+
++ if (phba->fw_dump_cmpl) {
++ complete(phba->fw_dump_cmpl);
++ phba->fw_dump_cmpl = NULL;
++ }
++ }
++
+ if (ha_copy & HA_MBATT)
+ lpfc_sli_handle_mb_event(phba);
+
+@@ -4351,6 +4461,8 @@ lpfc_mbx_cmpl_fc_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
+ goto out;
+ }
+
++ lpfc_check_nlp_post_devloss(vport, ndlp);
++
+ if (phba->sli_rev < LPFC_SLI_REV4)
+ ndlp->nlp_rpi = mb->un.varWords[0];
+
+@@ -4360,6 +4472,7 @@ lpfc_mbx_cmpl_fc_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
+ ndlp->nlp_state);
+
+ ndlp->nlp_flag |= NLP_RPI_REGISTERED;
++ ndlp->nlp_flag &= ~NLP_REG_LOGIN_SEND;
+ ndlp->nlp_type |= NLP_FABRIC;
+ lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
+
+@@ -4449,8 +4562,9 @@ lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
+ fc_remote_port_rolechg(rport, rport_ids.roles);
+
+ lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+- "3183 %s rport x%px DID x%x, role x%x\n",
+- __func__, rport, rport->port_id, rport->roles);
++ "3183 %s rport x%px DID x%x, role x%x refcnt %d\n",
++ __func__, rport, rport->port_id, rport->roles,
++ kref_read(&ndlp->kref));
+
+ if ((rport->scsi_target_id != -1) &&
+ (rport->scsi_target_id < LPFC_MAX_TARGET)) {
+@@ -4475,8 +4589,9 @@ lpfc_unregister_remote_port(struct lpfc_nodelist *ndlp)
+
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,
+ "3184 rport unregister x%06x, rport x%px "
+- "xptflg x%x\n",
+- ndlp->nlp_DID, rport, ndlp->fc4_xpt_flags);
++ "xptflg x%x refcnt %d\n",
++ ndlp->nlp_DID, rport, ndlp->fc4_xpt_flags,
++ kref_read(&ndlp->kref));
+
+ fc_remote_port_delete(rport);
+ lpfc_nlp_put(ndlp);
+@@ -4525,9 +4640,10 @@ lpfc_nlp_counters(struct lpfc_vport *vport, int state, int count)
+ void
+ lpfc_nlp_reg_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
+ {
+-
+ unsigned long iflags;
+
++ lpfc_check_nlp_post_devloss(vport, ndlp);
++
+ spin_lock_irqsave(&ndlp->lock, iflags);
+ if (ndlp->fc4_xpt_flags & NLP_XPT_REGD) {
+ /* Already registered with backend, trigger rescan */
+diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
+index 195169badb372..16246526e4c1e 100644
+--- a/drivers/scsi/lpfc/lpfc_init.c
++++ b/drivers/scsi/lpfc/lpfc_init.c
+@@ -2055,7 +2055,7 @@ lpfc_handle_eratt_s4(struct lpfc_hba *phba)
+ }
+ if (reg_err1 == SLIPORT_ERR1_REG_ERR_CODE_2 &&
+ reg_err2 == SLIPORT_ERR2_REG_FW_RESTART) {
+- lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
++ lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+ "3143 Port Down: Firmware Update "
+ "Detected\n");
+ en_rn_msg = false;
+@@ -3694,12 +3694,16 @@ lpfc_offline_prep(struct lpfc_hba *phba, int mbx_action)
+ lpfc_disc_state_machine(vports[i], ndlp,
+ NULL, NLP_EVT_DEVICE_RECOVERY);
+
+- /* Don't remove the node unless the
++ /* Don't remove the node unless the node
+ * has been unregistered with the
+- * transport. If so, let dev_loss
+- * take care of the node.
++ * transport, and we're not in recovery
++ * before dev_loss_tmo triggered.
++ * Otherwise, let dev_loss take care of
++ * the node.
+ */
+- if (!(ndlp->fc4_xpt_flags &
++ if (!(ndlp->save_flags &
++ NLP_IN_RECOV_POST_DEV_LOSS) &&
++ !(ndlp->fc4_xpt_flags &
+ (NVME_XPT_REGD | SCSI_XPT_REGD)))
+ lpfc_disc_state_machine
+ (vports[i], ndlp,
+@@ -5310,8 +5314,10 @@ lpfc_sli4_async_link_evt(struct lpfc_hba *phba,
+ */
+ if (!(phba->hba_flag & HBA_FCOE_MODE)) {
+ rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
+- if (rc == MBX_NOT_FINISHED)
++ if (rc == MBX_NOT_FINISHED) {
++ lpfc_mbuf_free(phba, mp->virt, mp->phys);
+ goto out_free_dmabuf;
++ }
+ return;
+ }
+ /*
+@@ -6262,8 +6268,10 @@ lpfc_sli4_async_fc_evt(struct lpfc_hba *phba, struct lpfc_acqe_fc_la *acqe_fc)
+ }
+
+ rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
+- if (rc == MBX_NOT_FINISHED)
++ if (rc == MBX_NOT_FINISHED) {
++ lpfc_mbuf_free(phba, mp->virt, mp->phys);
+ goto out_free_dmabuf;
++ }
+ return;
+
+ out_free_dmabuf:
+@@ -15097,6 +15105,8 @@ lpfc_io_slot_reset_s4(struct pci_dev *pdev)
+ psli->sli_flag &= ~LPFC_SLI_ACTIVE;
+ spin_unlock_irq(&phba->hbalock);
+
++ /* Init cpu_map array */
++ lpfc_cpu_map_array_init(phba);
+ /* Configure and enable interrupt */
+ intr_mode = lpfc_sli4_enable_intr(phba, phba->intr_mode);
+ if (intr_mode == LPFC_INTR_ERROR) {
+diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
+index 27263f02ab9f6..fdf5e777bf113 100644
+--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
++++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
+@@ -322,6 +322,7 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+ {
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_dmabuf *pcmd;
++ struct lpfc_dmabuf *mp;
+ uint64_t nlp_portwwn = 0;
+ uint32_t *lp;
+ IOCB_t *icmd;
+@@ -571,6 +572,11 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+ * a default RPI.
+ */
+ if (phba->sli_rev == LPFC_SLI_REV4) {
++ mp = (struct lpfc_dmabuf *)login_mbox->ctx_buf;
++ if (mp) {
++ lpfc_mbuf_free(phba, mp->virt, mp->phys);
++ kfree(mp);
++ }
+ mempool_free(login_mbox, phba->mbox_mem_pool);
+ login_mbox = NULL;
+ } else {
+@@ -1955,8 +1961,9 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport,
+ * is configured try it.
+ */
+ ndlp->nlp_fc4_type |= NLP_FC4_FCP;
+- if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
+- (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
++ if ((!(vport->fc_flag & FC_PT2PT_NO_NVME)) &&
++ (vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH ||
++ vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) {
+ ndlp->nlp_fc4_type |= NLP_FC4_NVME;
+ /* We need to update the localport also */
+ lpfc_nvme_update_localport(vport);
+diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
+index 479b3eed62085..66cb66aea2cf9 100644
+--- a/drivers/scsi/lpfc/lpfc_nvme.c
++++ b/drivers/scsi/lpfc/lpfc_nvme.c
+@@ -209,8 +209,9 @@ lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
+ * calling state machine to remove the node.
+ */
+ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
+- "6146 remoteport delete of remoteport x%px\n",
+- remoteport);
++ "6146 remoteport delete of remoteport x%px, ndlp x%px "
++ "DID x%x xflags x%x\n",
++ remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
+ spin_lock_irq(&ndlp->lock);
+
+ /* The register rebind might have occurred before the delete
+diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
+index befdf864c43bd..3d9175f1b6787 100644
+--- a/drivers/scsi/lpfc/lpfc_scsi.c
++++ b/drivers/scsi/lpfc/lpfc_scsi.c
+@@ -6455,28 +6455,28 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
+
+ /* Issue LOGO, if no LOGO is outstanding */
+ spin_lock_irqsave(&pnode->lock, flags);
+- if (!(pnode->upcall_flags & NLP_WAIT_FOR_LOGO) &&
++ if (!(pnode->save_flags & NLP_WAIT_FOR_LOGO) &&
+ !pnode->logo_waitq) {
+ pnode->logo_waitq = &waitq;
+ pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
+ pnode->nlp_flag |= NLP_ISSUE_LOGO;
+- pnode->upcall_flags |= NLP_WAIT_FOR_LOGO;
++ pnode->save_flags |= NLP_WAIT_FOR_LOGO;
+ spin_unlock_irqrestore(&pnode->lock, flags);
+ lpfc_unreg_rpi(vport, pnode);
+ wait_event_timeout(waitq,
+- (!(pnode->upcall_flags &
++ (!(pnode->save_flags &
+ NLP_WAIT_FOR_LOGO)),
+ msecs_to_jiffies(dev_loss_tmo *
+ 1000));
+
+- if (pnode->upcall_flags & NLP_WAIT_FOR_LOGO) {
++ if (pnode->save_flags & NLP_WAIT_FOR_LOGO) {
+ lpfc_printf_vlog(vport, KERN_ERR, logit,
+ "0725 SCSI layer TGTRST "
+ "failed & LOGO TMO (%d, %llu) "
+ "return x%x\n",
+ tgt_id, lun_id, status);
+ spin_lock_irqsave(&pnode->lock, flags);
+- pnode->upcall_flags &= ~NLP_WAIT_FOR_LOGO;
++ pnode->save_flags &= ~NLP_WAIT_FOR_LOGO;
+ } else {
+ spin_lock_irqsave(&pnode->lock, flags);
+ }
+@@ -6628,6 +6628,13 @@ lpfc_host_reset_handler(struct scsi_cmnd *cmnd)
+ if (rc)
+ goto error;
+
++ /* Wait for successful restart of adapter */
++ if (phba->sli_rev < LPFC_SLI_REV4) {
++ rc = lpfc_sli_chipset_init(phba);
++ if (rc)
++ goto error;
++ }
++
+ rc = lpfc_online(phba);
+ if (rc)
+ goto error;
+diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
+index 026a1196a54d5..68d8e55c1205c 100644
+--- a/drivers/scsi/lpfc/lpfc_sli.c
++++ b/drivers/scsi/lpfc/lpfc_sli.c
+@@ -5043,12 +5043,6 @@ lpfc_sli4_brdreset(struct lpfc_hba *phba)
+ phba->fcf.fcf_flag = 0;
+ spin_unlock_irq(&phba->hbalock);
+
+- /* SLI4 INTF 2: if FW dump is being taken skip INIT_PORT */
+- if (phba->hba_flag & HBA_FW_DUMP_OP) {
+- phba->hba_flag &= ~HBA_FW_DUMP_OP;
+- return rc;
+- }
+-
+ /* Now physically reset the device */
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "0389 Performing PCI function reset!\n");
+@@ -8153,6 +8147,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
+ struct lpfc_vport *vport = phba->pport;
+ struct lpfc_dmabuf *mp;
+ struct lpfc_rqb *rqbp;
++ u32 flg;
+
+ /* Perform a PCI function reset to start from clean */
+ rc = lpfc_pci_function_reset(phba);
+@@ -8166,7 +8161,17 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
+ else {
+ spin_lock_irq(&phba->hbalock);
+ phba->sli.sli_flag |= LPFC_SLI_ACTIVE;
++ flg = phba->sli.sli_flag;
+ spin_unlock_irq(&phba->hbalock);
++ /* Allow a little time after setting SLI_ACTIVE for any polled
++ * MBX commands to complete via BSG.
++ */
++ for (i = 0; i < 50 && (flg & LPFC_SLI_MBOX_ACTIVE); i++) {
++ msleep(20);
++ spin_lock_irq(&phba->hbalock);
++ flg = phba->sli.sli_flag;
++ spin_unlock_irq(&phba->hbalock);
++ }
+ }
+
+ lpfc_sli4_dip(phba);
+@@ -9750,7 +9755,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
+ "(%d):2541 Mailbox command x%x "
+ "(x%x/x%x) failure: "
+ "mqe_sta: x%x mcqe_sta: x%x/x%x "
+- "Data: x%x x%x\n,",
++ "Data: x%x x%x\n",
+ mboxq->vport ? mboxq->vport->vpi : 0,
+ mboxq->u.mb.mbxCommand,
+ lpfc_sli_config_mbox_subsys_get(phba,
+@@ -9784,7 +9789,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
+ "(%d):2597 Sync Mailbox command "
+ "x%x (x%x/x%x) failure: "
+ "mqe_sta: x%x mcqe_sta: x%x/x%x "
+- "Data: x%x x%x\n,",
++ "Data: x%x x%x\n",
+ mboxq->vport ? mboxq->vport->vpi : 0,
+ mboxq->u.mb.mbxCommand,
+ lpfc_sli_config_mbox_subsys_get(phba,
+@@ -12404,17 +12409,17 @@ lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
+
+ /* ABTS WQE must go to the same WQ as the WQE to be aborted */
+ abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx;
+- if (cmdiocb->iocb_flag & LPFC_IO_FCP) {
+- abtsiocbp->iocb_flag |= LPFC_IO_FCP;
+- abtsiocbp->iocb_flag |= LPFC_USE_FCPWQIDX;
+- }
++ if (cmdiocb->iocb_flag & LPFC_IO_FCP)
++ abtsiocbp->iocb_flag |= (LPFC_IO_FCP | LPFC_USE_FCPWQIDX);
+ if (cmdiocb->iocb_flag & LPFC_IO_FOF)
+ abtsiocbp->iocb_flag |= LPFC_IO_FOF;
+
+- if (phba->link_state >= LPFC_LINK_UP)
+- iabt->ulpCommand = CMD_ABORT_XRI_CN;
+- else
++ if (phba->link_state < LPFC_LINK_UP ||
++ (phba->sli_rev == LPFC_SLI_REV4 &&
++ phba->sli4_hba.link_state.status == LPFC_FC_LA_TYPE_LINK_DOWN))
+ iabt->ulpCommand = CMD_CLOSE_XRI_CN;
++ else
++ iabt->ulpCommand = CMD_ABORT_XRI_CN;
+
+ if (cmpl)
+ abtsiocbp->iocb_cmpl = cmpl;
+@@ -12488,15 +12493,54 @@ lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba)
+ }
+
+ /**
+- * lpfc_sli_validate_fcp_iocb - find commands associated with a vport or LUN
++ * lpfc_sli_validate_fcp_iocb_for_abort - filter iocbs appropriate for FCP aborts
++ * @iocbq: Pointer to iocb object.
++ * @vport: Pointer to driver virtual port object.
++ *
++ * This function acts as an iocb filter for functions which abort FCP iocbs.
++ *
++ * Return values
++ * -ENODEV, if a null iocb or vport ptr is encountered
++ * -EINVAL, if the iocb is not an FCP I/O, not on the TX cmpl queue, premarked as
++ * driver already started the abort process, or is an abort iocb itself
++ * 0, passes criteria for aborting the FCP I/O iocb
++ **/
++static int
++lpfc_sli_validate_fcp_iocb_for_abort(struct lpfc_iocbq *iocbq,
++ struct lpfc_vport *vport)
++{
++ IOCB_t *icmd = NULL;
++
++ /* No null ptr vports */
++ if (!iocbq || iocbq->vport != vport)
++ return -ENODEV;
++
++ /* iocb must be for FCP IO, already exists on the TX cmpl queue,
++ * can't be premarked as driver aborted, nor be an ABORT iocb itself
++ */
++ icmd = &iocbq->iocb;
++ if (!(iocbq->iocb_flag & LPFC_IO_FCP) ||
++ !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ) ||
++ (iocbq->iocb_flag & LPFC_DRIVER_ABORTED) ||
++ (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
++ icmd->ulpCommand == CMD_CLOSE_XRI_CN))
++ return -EINVAL;
++
++ return 0;
++}
++
++/**
++ * lpfc_sli_validate_fcp_iocb - validate commands associated with a SCSI target
+ * @iocbq: Pointer to driver iocb object.
+ * @vport: Pointer to driver virtual port object.
+ * @tgt_id: SCSI ID of the target.
+ * @lun_id: LUN ID of the scsi device.
+ * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST
+ *
+- * This function acts as an iocb filter for functions which abort or count
+- * all FCP iocbs pending on a lun/SCSI target/SCSI host. It will return
++ * This function acts as an iocb filter for validating a lun/SCSI target/SCSI
++ * host.
++ *
++ * It will return
+ * 0 if the filtering criteria is met for the given iocb and will return
+ * 1 if the filtering criteria is not met.
+ * If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the
+@@ -12515,22 +12559,8 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport,
+ lpfc_ctx_cmd ctx_cmd)
+ {
+ struct lpfc_io_buf *lpfc_cmd;
+- IOCB_t *icmd = NULL;
+ int rc = 1;
+
+- if (!iocbq || iocbq->vport != vport)
+- return rc;
+-
+- if (!(iocbq->iocb_flag & LPFC_IO_FCP) ||
+- !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ) ||
+- iocbq->iocb_flag & LPFC_DRIVER_ABORTED)
+- return rc;
+-
+- icmd = &iocbq->iocb;
+- if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
+- icmd->ulpCommand == CMD_CLOSE_XRI_CN)
+- return rc;
+-
+ lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
+
+ if (lpfc_cmd->pCmd == NULL)
+@@ -12585,17 +12615,33 @@ lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id,
+ {
+ struct lpfc_hba *phba = vport->phba;
+ struct lpfc_iocbq *iocbq;
++ IOCB_t *icmd = NULL;
+ int sum, i;
++ unsigned long iflags;
+
+- spin_lock_irq(&phba->hbalock);
++ spin_lock_irqsave(&phba->hbalock, iflags);
+ for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) {
+ iocbq = phba->sli.iocbq_lookup[i];
+
+- if (lpfc_sli_validate_fcp_iocb (iocbq, vport, tgt_id, lun_id,
+- ctx_cmd) == 0)
++ if (!iocbq || iocbq->vport != vport)
++ continue;
++ if (!(iocbq->iocb_flag & LPFC_IO_FCP) ||
++ !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ))
++ continue;
++
++ /* Include counting outstanding aborts */
++ icmd = &iocbq->iocb;
++ if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
++ icmd->ulpCommand == CMD_CLOSE_XRI_CN) {
++ sum++;
++ continue;
++ }
++
++ if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
++ ctx_cmd) == 0)
+ sum++;
+ }
+- spin_unlock_irq(&phba->hbalock);
++ spin_unlock_irqrestore(&phba->hbalock, iflags);
+
+ return sum;
+ }
+@@ -12662,7 +12708,11 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
+ *
+ * This function sends an abort command for every SCSI command
+ * associated with the given virtual port pending on the ring
+- * filtered by lpfc_sli_validate_fcp_iocb function.
++ * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then
++ * lpfc_sli_validate_fcp_iocb function. The ordering for validation before
++ * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort
++ * followed by lpfc_sli_validate_fcp_iocb.
++ *
+ * When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the
+ * FCP iocbs associated with lun specified by tgt_id and lun_id
+ * parameters
+@@ -12694,6 +12744,9 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id,
+ for (i = 1; i <= phba->sli.last_iotag; i++) {
+ iocbq = phba->sli.iocbq_lookup[i];
+
++ if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport))
++ continue;
++
+ if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
+ abort_cmd) != 0)
+ continue;
+@@ -12726,7 +12779,11 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id,
+ *
+ * This function sends an abort command for every SCSI command
+ * associated with the given virtual port pending on the ring
+- * filtered by lpfc_sli_validate_fcp_iocb function.
++ * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then
++ * lpfc_sli_validate_fcp_iocb function. The ordering for validation before
++ * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort
++ * followed by lpfc_sli_validate_fcp_iocb.
++ *
+ * When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the
+ * FCP iocbs associated with lun specified by tgt_id and lun_id
+ * parameters
+@@ -12764,6 +12821,9 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
+ for (i = 1; i <= phba->sli.last_iotag; i++) {
+ iocbq = phba->sli.iocbq_lookup[i];
+
++ if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport))
++ continue;
++
+ if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
+ cmd) != 0)
+ continue;
+@@ -13317,6 +13377,7 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba)
+ uint32_t uerr_sta_hi, uerr_sta_lo;
+ uint32_t if_type, portsmphr;
+ struct lpfc_register portstat_reg;
++ u32 logmask;
+
+ /*
+ * For now, use the SLI4 device internal unrecoverable error
+@@ -13367,7 +13428,12 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba)
+ readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
+ phba->work_status[1] =
+ readl(phba->sli4_hba.u.if_type2.ERR2regaddr);
+- lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
++ logmask = LOG_TRACE_EVENT;
++ if (phba->work_status[0] ==
++ SLIPORT_ERR1_REG_ERR_CODE_2 &&
++ phba->work_status[1] == SLIPORT_ERR2_REG_FW_RESTART)
++ logmask = LOG_SLI;
++ lpfc_printf_log(phba, KERN_ERR, logmask,
+ "2885 Port Status Event: "
+ "port status reg 0x%x, "
+ "port smphr reg 0x%x, "
+@@ -21107,6 +21173,7 @@ lpfc_drain_txq(struct lpfc_hba *phba)
+ fail_msg,
+ piocbq->iotag, piocbq->sli4_xritag);
+ list_add_tail(&piocbq->list, &completions);
++ fail_msg = NULL;
+ }
+ spin_unlock_irqrestore(&pring->ring_lock, iflags);
+ }
+diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
+index 7af2c23652b01..650210d2abb4d 100644
+--- a/drivers/scsi/megaraid/megaraid_sas.h
++++ b/drivers/scsi/megaraid/megaraid_sas.h
+@@ -2558,6 +2558,9 @@ struct megasas_instance_template {
+ #define MEGASAS_IS_LOGICAL(sdev) \
+ ((sdev->channel < MEGASAS_MAX_PD_CHANNELS) ? 0 : 1)
+
++#define MEGASAS_IS_LUN_VALID(sdev) \
++ (((sdev)->lun == 0) ? 1 : 0)
++
+ #define MEGASAS_DEV_INDEX(scp) \
+ (((scp->device->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL) + \
+ scp->device->id)
+diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
+index 39d8754e63acf..bb3f78013a13b 100644
+--- a/drivers/scsi/megaraid/megaraid_sas_base.c
++++ b/drivers/scsi/megaraid/megaraid_sas_base.c
+@@ -2126,6 +2126,9 @@ static int megasas_slave_alloc(struct scsi_device *sdev)
+ goto scan_target;
+ }
+ return -ENXIO;
++ } else if (!MEGASAS_IS_LUN_VALID(sdev)) {
++ sdev_printk(KERN_INFO, sdev, "%s: invalid LUN\n", __func__);
++ return -ENXIO;
+ }
+
+ scan_target:
+@@ -2156,6 +2159,10 @@ static void megasas_slave_destroy(struct scsi_device *sdev)
+ instance = megasas_lookup_instance(sdev->host->host_no);
+
+ if (MEGASAS_IS_LOGICAL(sdev)) {
++ if (!MEGASAS_IS_LUN_VALID(sdev)) {
++ sdev_printk(KERN_INFO, sdev, "%s: invalid LUN\n", __func__);
++ return;
++ }
+ ld_tgt_id = MEGASAS_TARGET_ID(sdev);
+ instance->ld_tgtid_status[ld_tgt_id] = LD_TARGET_ID_DELETED;
+ if (megasas_dbg_lvl & LD_PD_DEBUG)
+diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
+index 26d0cf9353dd6..eb5ceb75a15ec 100644
+--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
++++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
+@@ -3530,6 +3530,9 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
+ if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
+ return IRQ_HANDLED;
+
++ if (irq_context && !atomic_add_unless(&irq_context->in_used, 1, 1))
++ return 0;
++
+ desc = fusion->reply_frames_desc[MSIxIndex] +
+ fusion->last_reply_idx[MSIxIndex];
+
+@@ -3540,11 +3543,11 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
+ reply_descript_type = reply_desc->ReplyFlags &
+ MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
+
+- if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
++ if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
++ if (irq_context)
++ atomic_dec(&irq_context->in_used);
+ return IRQ_NONE;
+-
+- if (irq_context && !atomic_add_unless(&irq_context->in_used, 1, 1))
+- return 0;
++ }
+
+ num_completed = 0;
+
+diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
+index 9787b53a2b598..2cc42432bd0c0 100644
+--- a/drivers/scsi/mpi3mr/mpi3mr.h
++++ b/drivers/scsi/mpi3mr/mpi3mr.h
+@@ -79,7 +79,8 @@ extern int prot_mask;
+
+ /* Operational queue management definitions */
+ #define MPI3MR_OP_REQ_Q_QD 512
+-#define MPI3MR_OP_REP_Q_QD 4096
++#define MPI3MR_OP_REP_Q_QD 1024
++#define MPI3MR_OP_REP_Q_QD4K 4096
+ #define MPI3MR_OP_REQ_Q_SEG_SIZE 4096
+ #define MPI3MR_OP_REP_Q_SEG_SIZE 4096
+ #define MPI3MR_MAX_SEG_LIST_SIZE 4096
+diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
+index 4a8316c6bd41a..3ef6b6edef46d 100644
+--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
++++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
+@@ -1275,10 +1275,10 @@ static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
+ MPI3MR_MAX_SEG_LIST_SIZE,
+ mrioc->req_qinfo[q_idx].q_segment_list,
+ mrioc->req_qinfo[q_idx].q_segment_list_dma);
+- mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
++ mrioc->req_qinfo[q_idx].q_segment_list = NULL;
+ }
+ } else
+- size = mrioc->req_qinfo[q_idx].num_requests *
++ size = mrioc->req_qinfo[q_idx].segment_qd *
+ mrioc->facts.op_req_sz;
+
+ for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
+@@ -1565,6 +1565,8 @@ static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
+
+ reply_qid = qidx + 1;
+ op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
++ if (!mrioc->pdev->revision)
++ op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
+ op_reply_q->ci = 0;
+ op_reply_q->ephase = 1;
+ atomic_set(&op_reply_q->pend_ios, 0);
+diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c
+index 3cae8803383b6..b2c650542bac5 100644
+--- a/drivers/scsi/mpi3mr/mpi3mr_os.c
++++ b/drivers/scsi/mpi3mr/mpi3mr_os.c
+@@ -2204,6 +2204,8 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
+ scmd->result = DID_OK << 16;
+ goto out_success;
+ }
++
++ scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_count);
+ if (ioc_status == MPI3_IOCSTATUS_SCSI_DATA_UNDERRUN &&
+ xfer_count == 0 && (scsi_status == MPI3_SCSI_STATUS_BUSY ||
+ scsi_status == MPI3_SCSI_STATUS_RESERVATION_CONFLICT ||
+diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
+index 27eb652b564f5..c38e689432054 100644
+--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
++++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
+@@ -639,8 +639,8 @@ static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc)
+ mpi_request->IOCParameter = MPI26_SET_IOC_PARAMETER_SYNC_TIMESTAMP;
+ current_time = ktime_get_real();
+ TimeStamp = ktime_to_ms(current_time);
+- mpi_request->Reserved7 = cpu_to_le32(TimeStamp & 0xFFFFFFFF);
+- mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp >> 32);
++ mpi_request->Reserved7 = cpu_to_le32(TimeStamp >> 32);
++ mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp & 0xFFFFFFFF);
+ init_completion(&ioc->scsih_cmds.done);
+ ioc->put_smid_default(ioc, smid);
+ dinitprintk(ioc, ioc_info(ioc,
+@@ -2011,9 +2011,10 @@ mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll)
+ enable_irq(reply_q->os_irq);
+ }
+ }
++
++ if (poll)
++ _base_process_reply_queue(reply_q);
+ }
+- if (poll)
+- _base_process_reply_queue(reply_q);
+ }
+
+ /**
+@@ -5736,14 +5737,13 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
+ */
+
+ static int
+-mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz)
++mpt3sas_check_same_4gb_region(dma_addr_t start_address, u32 pool_sz)
+ {
+- long reply_pool_end_address;
++ dma_addr_t end_address;
+
+- reply_pool_end_address = reply_pool_start_address + pool_sz;
++ end_address = start_address + pool_sz - 1;
+
+- if (upper_32_bits(reply_pool_start_address) ==
+- upper_32_bits(reply_pool_end_address))
++ if (upper_32_bits(start_address) == upper_32_bits(end_address))
+ return 1;
+ else
+ return 0;
+@@ -5804,7 +5804,7 @@ _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
+ }
+
+ if (!mpt3sas_check_same_4gb_region(
+- (long)ioc->pcie_sg_lookup[i].pcie_sgl, sz)) {
++ ioc->pcie_sg_lookup[i].pcie_sgl_dma, sz)) {
+ ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n",
+ ioc->pcie_sg_lookup[i].pcie_sgl,
+ (unsigned long long)
+@@ -5859,8 +5859,8 @@ _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
+ GFP_KERNEL, &ctr->chain_buffer_dma);
+ if (!ctr->chain_buffer)
+ return -EAGAIN;
+- if (!mpt3sas_check_same_4gb_region((long)
+- ctr->chain_buffer, ioc->chain_segment_sz)) {
++ if (!mpt3sas_check_same_4gb_region(
++ ctr->chain_buffer_dma, ioc->chain_segment_sz)) {
+ ioc_err(ioc,
+ "Chain buffers are not in same 4G !!! Chain buff (0x%p) dma = (0x%llx)\n",
+ ctr->chain_buffer,
+@@ -5896,7 +5896,7 @@ _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
+ GFP_KERNEL, &ioc->sense_dma);
+ if (!ioc->sense)
+ return -EAGAIN;
+- if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) {
++ if (!mpt3sas_check_same_4gb_region(ioc->sense_dma, sz)) {
+ dinitprintk(ioc, pr_err(
+ "Bad Sense Pool! sense (0x%p) sense_dma = (0x%llx)\n",
+ ioc->sense, (unsigned long long) ioc->sense_dma));
+@@ -5929,7 +5929,7 @@ _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
+ &ioc->reply_dma);
+ if (!ioc->reply)
+ return -EAGAIN;
+- if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) {
++ if (!mpt3sas_check_same_4gb_region(ioc->reply_dma, sz)) {
+ dinitprintk(ioc, pr_err(
+ "Bad Reply Pool! Reply (0x%p) Reply dma = (0x%llx)\n",
+ ioc->reply, (unsigned long long) ioc->reply_dma));
+@@ -5964,7 +5964,7 @@ _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
+ GFP_KERNEL, &ioc->reply_free_dma);
+ if (!ioc->reply_free)
+ return -EAGAIN;
+- if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) {
++ if (!mpt3sas_check_same_4gb_region(ioc->reply_free_dma, sz)) {
+ dinitprintk(ioc,
+ pr_err("Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n",
+ ioc->reply_free, (unsigned long long) ioc->reply_free_dma));
+@@ -6003,7 +6003,7 @@ _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc,
+ GFP_KERNEL, &ioc->reply_post_free_array_dma);
+ if (!ioc->reply_post_free_array)
+ return -EAGAIN;
+- if (!mpt3sas_check_same_4gb_region((long)ioc->reply_post_free_array,
++ if (!mpt3sas_check_same_4gb_region(ioc->reply_post_free_array_dma,
+ reply_post_free_array_sz)) {
+ dinitprintk(ioc, pr_err(
+ "Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n",
+@@ -6068,7 +6068,7 @@ base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)
+ * resources and set DMA mask to 32 and allocate.
+ */
+ if (!mpt3sas_check_same_4gb_region(
+- (long)ioc->reply_post[i].reply_post_free, sz)) {
++ ioc->reply_post[i].reply_post_free_dma, sz)) {
+ dinitprintk(ioc,
+ ioc_err(ioc, "bad Replypost free pool(0x%p)"
+ "reply_post_free_dma = (0x%llx)\n",
+diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
+index f87c0911f66ad..1b3a44ce65aae 100644
+--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
++++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
+@@ -142,6 +142,8 @@
+
+ #define MPT_MAX_CALLBACKS 32
+
++#define MPT_MAX_HBA_NUM_PHYS 32
++
+ #define INTERNAL_CMDS_COUNT 10 /* reserved cmds */
+ /* reserved for issuing internally framed scsi io cmds */
+ #define INTERNAL_SCSIIO_CMDS_COUNT 3
+@@ -798,6 +800,7 @@ struct _sas_phy {
+ * @enclosure_handle: handle for this a member of an enclosure
+ * @device_info: bitwise defining capabilities of this sas_host/expander
+ * @responding: used in _scsih_expander_device_mark_responding
++ * @nr_phys_allocated: Allocated memory for this many count phys
+ * @phy: a list of phys that make up this sas_host/expander
+ * @sas_port_list: list of ports attached to this sas_host/expander
+ * @port: hba port entry containing node's port number info
+@@ -813,6 +816,7 @@ struct _sas_node {
+ u16 enclosure_handle;
+ u64 enclosure_logical_id;
+ u8 responding;
++ u8 nr_phys_allocated;
+ struct hba_port *port;
+ struct _sas_phy *phy;
+ struct list_head sas_port_list;
+diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
+index 0563078227de6..a8dd14c91efdb 100644
+--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
++++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
+@@ -394,10 +394,13 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t
+ retry_count++;
+ if (ioc->config_cmds.smid == smid)
+ mpt3sas_base_free_smid(ioc, smid);
+- if ((ioc->shost_recovery) || (ioc->config_cmds.status &
+- MPT3_CMD_RESET) || ioc->pci_error_recovery)
++ if (ioc->config_cmds.status & MPT3_CMD_RESET)
+ goto retry_config;
+- issue_host_reset = 1;
++ if (ioc->shost_recovery || ioc->pci_error_recovery) {
++ issue_host_reset = 0;
++ r = -EFAULT;
++ } else
++ issue_host_reset = 1;
+ goto free_mem;
+ }
+
+diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+index ad1b6c2b37a74..af275ac427954 100644
+--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
++++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+@@ -3869,7 +3869,7 @@ _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,
+
+ shost_for_each_device(sdev, ioc->shost) {
+ sas_device_priv_data = sdev->hostdata;
+- if (!sas_device_priv_data)
++ if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
+ continue;
+ if (sas_device_priv_data->sas_target->sas_address
+ != sas_address)
+@@ -6406,11 +6406,26 @@ _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)
+ int i, j, count = 0, lcount = 0;
+ int ret;
+ u64 sas_addr;
++ u8 num_phys;
+
+ drsprintk(ioc, ioc_info(ioc,
+ "updating ports for sas_host(0x%016llx)\n",
+ (unsigned long long)ioc->sas_hba.sas_address));
+
++ mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
++ if (!num_phys) {
++ ioc_err(ioc, "failure at %s:%d/%s()!\n",
++ __FILE__, __LINE__, __func__);
++ return;
++ }
++
++ if (num_phys > ioc->sas_hba.nr_phys_allocated) {
++ ioc_err(ioc, "failure at %s:%d/%s()!\n",
++ __FILE__, __LINE__, __func__);
++ return;
++ }
++ ioc->sas_hba.num_phys = num_phys;
++
+ port_table = kcalloc(ioc->sas_hba.num_phys,
+ sizeof(struct hba_port), GFP_KERNEL);
+ if (!port_table)
+@@ -6611,6 +6626,30 @@ _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
+ ioc->sas_hba.phy[i].hba_vphy = 1;
+ }
+
++ /*
++ * Add new HBA phys to STL if these new phys got added as part
++ * of HBA Firmware upgrade/downgrade operation.
++ */
++ if (!ioc->sas_hba.phy[i].phy) {
++ if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,
++ &phy_pg0, i))) {
++ ioc_err(ioc, "failure at %s:%d/%s()!\n",
++ __FILE__, __LINE__, __func__);
++ continue;
++ }
++ ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
++ MPI2_IOCSTATUS_MASK;
++ if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
++ ioc_err(ioc, "failure at %s:%d/%s()!\n",
++ __FILE__, __LINE__, __func__);
++ continue;
++ }
++ ioc->sas_hba.phy[i].phy_id = i;
++ mpt3sas_transport_add_host_phy(ioc,
++ &ioc->sas_hba.phy[i], phy_pg0,
++ ioc->sas_hba.parent_dev);
++ continue;
++ }
+ ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
+ attached_handle = le16_to_cpu(sas_iounit_pg0->PhyData[i].
+ AttachedDevHandle);
+@@ -6622,6 +6661,19 @@ _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
+ attached_handle, i, link_rate,
+ ioc->sas_hba.phy[i].port);
+ }
++ /*
++ * Clear the phy details if this phy got disabled as part of
++ * HBA Firmware upgrade/downgrade operation.
++ */
++ for (i = ioc->sas_hba.num_phys;
++ i < ioc->sas_hba.nr_phys_allocated; i++) {
++ if (ioc->sas_hba.phy[i].phy &&
++ ioc->sas_hba.phy[i].phy->negotiated_linkrate >=
++ SAS_LINK_RATE_1_5_GBPS)
++ mpt3sas_transport_update_links(ioc,
++ ioc->sas_hba.sas_address, 0, i,
++ MPI2_SAS_NEG_LINK_RATE_PHY_DISABLED, NULL);
++ }
+ out:
+ kfree(sas_iounit_pg0);
+ }
+@@ -6654,7 +6706,10 @@ _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
+ __FILE__, __LINE__, __func__);
+ return;
+ }
+- ioc->sas_hba.phy = kcalloc(num_phys,
++
++ ioc->sas_hba.nr_phys_allocated = max_t(u8,
++ MPT_MAX_HBA_NUM_PHYS, num_phys);
++ ioc->sas_hba.phy = kcalloc(ioc->sas_hba.nr_phys_allocated,
+ sizeof(struct _sas_phy), GFP_KERNEL);
+ if (!ioc->sas_hba.phy) {
+ ioc_err(ioc, "failure at %s:%d/%s()!\n",
+@@ -10980,6 +11035,7 @@ _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
+ {
+ struct _sas_port *mpt3sas_port, *next;
+ unsigned long flags;
++ int port_id;
+
+ /* remove sibling ports attached to this expander */
+ list_for_each_entry_safe(mpt3sas_port, next,
+@@ -11000,6 +11056,8 @@ _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
+ mpt3sas_port->hba_port);
+ }
+
++ port_id = sas_expander->port->port_id;
++
+ mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,
+ sas_expander->sas_address_parent, sas_expander->port);
+
+@@ -11007,7 +11065,7 @@ _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
+ "expander_remove: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
+ sas_expander->handle, (unsigned long long)
+ sas_expander->sas_address,
+- sas_expander->port->port_id);
++ port_id);
+
+ spin_lock_irqsave(&ioc->sas_node_lock, flags);
+ list_del(&sas_expander->list);
+diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c
+index f18dd97035955..f6f8ca3c8c7f5 100644
+--- a/drivers/scsi/mvsas/mv_init.c
++++ b/drivers/scsi/mvsas/mv_init.c
+@@ -646,6 +646,7 @@ static struct pci_device_id mvs_pci_table[] = {
+ { PCI_VDEVICE(ARECA, PCI_DEVICE_ID_ARECA_1300), chip_1300 },
+ { PCI_VDEVICE(ARECA, PCI_DEVICE_ID_ARECA_1320), chip_1320 },
+ { PCI_VDEVICE(ADAPTEC2, 0x0450), chip_6440 },
++ { PCI_VDEVICE(TTI, 0x2640), chip_6440 },
+ { PCI_VDEVICE(TTI, 0x2710), chip_9480 },
+ { PCI_VDEVICE(TTI, 0x2720), chip_9480 },
+ { PCI_VDEVICE(TTI, 0x2721), chip_9480 },
+@@ -696,7 +697,7 @@ static struct pci_driver mvs_pci_driver = {
+ static ssize_t driver_version_show(struct device *cdev,
+ struct device_attribute *attr, char *buffer)
+ {
+- return snprintf(buffer, PAGE_SIZE, "%s\n", DRV_VERSION);
++ return sysfs_emit(buffer, "%s\n", DRV_VERSION);
+ }
+
+ static DEVICE_ATTR_RO(driver_version);
+@@ -744,7 +745,7 @@ static ssize_t interrupt_coalescing_store(struct device *cdev,
+ static ssize_t interrupt_coalescing_show(struct device *cdev,
+ struct device_attribute *attr, char *buffer)
+ {
+- return snprintf(buffer, PAGE_SIZE, "%d\n", interrupt_coalescing);
++ return sysfs_emit(buffer, "%d\n", interrupt_coalescing);
+ }
+
+ static DEVICE_ATTR_RW(interrupt_coalescing);
+diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c
+index 07f274afd7e5e..a4d244ee4548c 100644
+--- a/drivers/scsi/myrs.c
++++ b/drivers/scsi/myrs.c
+@@ -2265,7 +2265,8 @@ static void myrs_cleanup(struct myrs_hba *cs)
+ myrs_unmap(cs);
+
+ if (cs->mmio_base) {
+- cs->disable_intr(cs);
++ if (cs->disable_intr)
++ cs->disable_intr(cs);
+ iounmap(cs->mmio_base);
+ cs->mmio_base = NULL;
+ }
+diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
+index 63690508313b7..32fc450bf84b4 100644
+--- a/drivers/scsi/pm8001/pm8001_hwi.c
++++ b/drivers/scsi/pm8001/pm8001_hwi.c
+@@ -1325,7 +1325,9 @@ int pm8001_mpi_build_cmd(struct pm8001_hba_info *pm8001_ha,
+ int q_index = circularQ - pm8001_ha->inbnd_q_tbl;
+ int rv;
+
+- WARN_ON(q_index >= PM8001_MAX_INB_NUM);
++ if (WARN_ON(q_index >= pm8001_ha->max_q_num))
++ return -EINVAL;
++
+ spin_lock_irqsave(&circularQ->iq_lock, flags);
+ rv = pm8001_mpi_msg_free_get(circularQ, pm8001_ha->iomb_size,
+ &pMessage);
+@@ -1765,7 +1767,6 @@ static void pm8001_send_abort_all(struct pm8001_hba_info *pm8001_ha,
+ }
+
+ task = sas_alloc_slow_task(GFP_ATOMIC);
+-
+ if (!task) {
+ pm8001_dbg(pm8001_ha, FAIL, "cannot allocate task\n");
+ return;
+@@ -1774,13 +1775,16 @@ static void pm8001_send_abort_all(struct pm8001_hba_info *pm8001_ha,
+ task->task_done = pm8001_task_done;
+
+ res = pm8001_tag_alloc(pm8001_ha, &ccb_tag);
+- if (res)
++ if (res) {
++ sas_free_task(task);
+ return;
++ }
+
+ ccb = &pm8001_ha->ccb_info[ccb_tag];
+ ccb->device = pm8001_ha_dev;
+ ccb->ccb_tag = ccb_tag;
+ ccb->task = task;
++ ccb->n_elem = 0;
+
+ circularQ = &pm8001_ha->inbnd_q_tbl[0];
+
+@@ -1791,8 +1795,10 @@ static void pm8001_send_abort_all(struct pm8001_hba_info *pm8001_ha,
+
+ ret = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &task_abort,
+ sizeof(task_abort), 0);
+- if (ret)
++ if (ret) {
++ sas_free_task(task);
+ pm8001_tag_free(pm8001_ha, ccb_tag);
++ }
+
+ }
+
+@@ -1842,6 +1848,7 @@ static void pm8001_send_read_log(struct pm8001_hba_info *pm8001_ha,
+ ccb->device = pm8001_ha_dev;
+ ccb->ccb_tag = ccb_tag;
+ ccb->task = task;
++ ccb->n_elem = 0;
+ pm8001_ha_dev->id |= NCQ_READ_LOG_FLAG;
+ pm8001_ha_dev->id |= NCQ_2ND_RLE_FLAG;
+
+@@ -1858,7 +1865,7 @@ static void pm8001_send_read_log(struct pm8001_hba_info *pm8001_ha,
+
+ sata_cmd.tag = cpu_to_le32(ccb_tag);
+ sata_cmd.device_id = cpu_to_le32(pm8001_ha_dev->device_id);
+- sata_cmd.ncqtag_atap_dir_m |= ((0x1 << 7) | (0x5 << 9));
++ sata_cmd.ncqtag_atap_dir_m = cpu_to_le32((0x1 << 7) | (0x5 << 9));
+ memcpy(&sata_cmd.sata_fis, &fis, sizeof(struct host_to_dev_fis));
+
+ res = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &sata_cmd,
+@@ -2419,7 +2426,8 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ len = sizeof(struct pio_setup_fis);
+ pm8001_dbg(pm8001_ha, IO,
+ "PIO read len = %d\n", len);
+- } else if (t->ata_task.use_ncq) {
++ } else if (t->ata_task.use_ncq &&
++ t->data_dir != DMA_NONE) {
+ len = sizeof(struct set_dev_bits_fis);
+ pm8001_dbg(pm8001_ha, IO, "FPDMA len = %d\n",
+ len);
+@@ -2693,7 +2701,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ u32 tag = le32_to_cpu(psataPayload->tag);
+ u32 port_id = le32_to_cpu(psataPayload->port_id);
+ u32 dev_id = le32_to_cpu(psataPayload->device_id);
+- unsigned long flags;
+
+ ccb = &pm8001_ha->ccb_info[tag];
+
+@@ -2733,8 +2740,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ ts->resp = SAS_TASK_COMPLETE;
+ ts->stat = SAS_DATA_OVERRUN;
+ ts->residual = 0;
+- if (pm8001_dev)
+- atomic_dec(&pm8001_dev->running_req);
+ break;
+ case IO_XFER_ERROR_BREAK:
+ pm8001_dbg(pm8001_ha, IO, "IO_XFER_ERROR_BREAK\n");
+@@ -2776,7 +2781,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_OPEN_CNX_ERROR_IT_NEXUS_LOSS);
+ ts->resp = SAS_TASK_COMPLETE;
+ ts->stat = SAS_QUEUE_FULL;
+- pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
+ return;
+ }
+ break;
+@@ -2862,20 +2866,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ ts->stat = SAS_OPEN_TO;
+ break;
+ }
+- spin_lock_irqsave(&t->task_state_lock, flags);
+- t->task_state_flags &= ~SAS_TASK_STATE_PENDING;
+- t->task_state_flags &= ~SAS_TASK_AT_INITIATOR;
+- t->task_state_flags |= SAS_TASK_STATE_DONE;
+- if (unlikely((t->task_state_flags & SAS_TASK_STATE_ABORTED))) {
+- spin_unlock_irqrestore(&t->task_state_lock, flags);
+- pm8001_dbg(pm8001_ha, FAIL,
+- "task 0x%p done with io_status 0x%x resp 0x%x stat 0x%x but aborted by upper layer!\n",
+- t, event, ts->resp, ts->stat);
+- pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+- } else {
+- spin_unlock_irqrestore(&t->task_state_lock, flags);
+- pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
+- }
+ }
+
+ /*See the comments for mpi_ssp_completion */
+@@ -3169,7 +3159,7 @@ pm8001_mpi_get_nvmd_resp(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ * fw_control_context->usrAddr
+ */
+ complete(pm8001_ha->nvmd_completion);
+- pm8001_dbg(pm8001_ha, MSG, "Set nvm data complete!\n");
++ pm8001_dbg(pm8001_ha, MSG, "Get nvmd data complete!\n");
+ ccb->task = NULL;
+ ccb->ccb_tag = 0xFFFFFFFF;
+ pm8001_tag_free(pm8001_ha, tag);
+@@ -3726,12 +3716,11 @@ int pm8001_mpi_task_abort_resp(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ mb();
+
+ if (pm8001_dev->id & NCQ_ABORT_ALL_FLAG) {
+- pm8001_tag_free(pm8001_ha, tag);
+ sas_free_task(t);
+- /* clear the flag */
+- pm8001_dev->id &= 0xBFFFFFFF;
+- } else
++ pm8001_dev->id &= ~NCQ_ABORT_ALL_FLAG;
++ } else {
+ t->task_done(t);
++ }
+
+ return 0;
+ }
+@@ -4294,22 +4283,22 @@ static int pm8001_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ u32 opc = OPC_INB_SATA_HOST_OPSTART;
+ memset(&sata_cmd, 0, sizeof(sata_cmd));
+ circularQ = &pm8001_ha->inbnd_q_tbl[0];
+- if (task->data_dir == DMA_NONE) {
++
++ if (task->data_dir == DMA_NONE && !task->ata_task.use_ncq) {
+ ATAP = 0x04; /* no data*/
+ pm8001_dbg(pm8001_ha, IO, "no data\n");
+ } else if (likely(!task->ata_task.device_control_reg_update)) {
+- if (task->ata_task.dma_xfer) {
++ if (task->ata_task.use_ncq &&
++ dev->sata_dev.class != ATA_DEV_ATAPI) {
++ ATAP = 0x07; /* FPDMA */
++ pm8001_dbg(pm8001_ha, IO, "FPDMA\n");
++ } else if (task->ata_task.dma_xfer) {
+ ATAP = 0x06; /* DMA */
+ pm8001_dbg(pm8001_ha, IO, "DMA\n");
+ } else {
+ ATAP = 0x05; /* PIO*/
+ pm8001_dbg(pm8001_ha, IO, "PIO\n");
+ }
+- if (task->ata_task.use_ncq &&
+- dev->sata_dev.class != ATA_DEV_ATAPI) {
+- ATAP = 0x07; /* FPDMA */
+- pm8001_dbg(pm8001_ha, IO, "FPDMA\n");
+- }
+ }
+ if (task->ata_task.use_ncq && pm8001_get_ncq_tag(task, &hdr_tag)) {
+ task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
+@@ -4499,6 +4488,9 @@ static int pm8001_chip_reg_dev_req(struct pm8001_hba_info *pm8001_ha,
+ SAS_ADDR_SIZE);
+ rc = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload,
+ sizeof(payload), 0);
++ if (rc)
++ pm8001_tag_free(pm8001_ha, tag);
++
+ return rc;
+ }
+
+@@ -4649,7 +4641,7 @@ int pm8001_chip_ssp_tm_req(struct pm8001_hba_info *pm8001_ha,
+ memcpy(sspTMCmd.lun, task->ssp_task.LUN, 8);
+ sspTMCmd.tag = cpu_to_le32(ccb->ccb_tag);
+ if (pm8001_ha->chip_id != chip_8001)
+- sspTMCmd.ds_ads_m = 0x08;
++ sspTMCmd.ds_ads_m = cpu_to_le32(0x08);
+ circularQ = &pm8001_ha->inbnd_q_tbl[0];
+ ret = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &sspTMCmd,
+ sizeof(sspTMCmd), 0);
+@@ -4911,6 +4903,11 @@ pm8001_chip_fw_flash_update_req(struct pm8001_hba_info *pm8001_ha,
+ ccb->ccb_tag = tag;
+ rc = pm8001_chip_fw_flash_update_build(pm8001_ha, &flash_update_info,
+ tag);
++ if (rc) {
++ kfree(fw_control_context);
++ pm8001_tag_free(pm8001_ha, tag);
++ }
++
+ return rc;
+ }
+
+@@ -5015,6 +5012,9 @@ pm8001_chip_set_dev_state_req(struct pm8001_hba_info *pm8001_ha,
+ payload.nds = cpu_to_le32(state);
+ rc = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload,
+ sizeof(payload), 0);
++ if (rc)
++ pm8001_tag_free(pm8001_ha, tag);
++
+ return rc;
+
+ }
+diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
+index 47db7e0beae6f..a25a34535b7a4 100644
+--- a/drivers/scsi/pm8001/pm8001_init.c
++++ b/drivers/scsi/pm8001/pm8001_init.c
+@@ -281,12 +281,12 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha,
+ if (rc) {
+ pm8001_dbg(pm8001_ha, FAIL,
+ "pm8001_setup_irq failed [ret: %d]\n", rc);
+- goto err_out_shost;
++ goto err_out;
+ }
+ /* Request Interrupt */
+ rc = pm8001_request_irq(pm8001_ha);
+ if (rc)
+- goto err_out_shost;
++ goto err_out;
+
+ count = pm8001_ha->max_q_num;
+ /* Queues are chosen based on the number of cores/msix availability */
+@@ -422,8 +422,6 @@ static int pm8001_alloc(struct pm8001_hba_info *pm8001_ha,
+ pm8001_tag_init(pm8001_ha);
+ return 0;
+
+-err_out_shost:
+- scsi_remove_host(pm8001_ha->shost);
+ err_out_nodev:
+ for (i = 0; i < pm8001_ha->max_memcnt; i++) {
+ if (pm8001_ha->memoryMap.region[i].virt_ptr != NULL) {
+@@ -1198,6 +1196,7 @@ pm8001_init_ccb_tag(struct pm8001_hba_info *pm8001_ha, struct Scsi_Host *shost,
+ goto err_out;
+
+ /* Memory region for ccb_info*/
++ pm8001_ha->ccb_count = ccb_count;
+ pm8001_ha->ccb_info =
+ kcalloc(ccb_count, sizeof(struct pm8001_ccb_info), GFP_KERNEL);
+ if (!pm8001_ha->ccb_info) {
+@@ -1259,6 +1258,16 @@ static void pm8001_pci_remove(struct pci_dev *pdev)
+ tasklet_kill(&pm8001_ha->tasklet[j]);
+ #endif
+ scsi_host_put(pm8001_ha->shost);
++
++ for (i = 0; i < pm8001_ha->ccb_count; i++) {
++ dma_free_coherent(&pm8001_ha->pdev->dev,
++ sizeof(struct pm8001_prd) * PM8001_MAX_DMA_SG,
++ pm8001_ha->ccb_info[i].buf_prd,
++ pm8001_ha->ccb_info[i].ccb_dma_handle);
++ }
++ kfree(pm8001_ha->ccb_info);
++ kfree(pm8001_ha->devices);
++
+ pm8001_free(pm8001_ha);
+ kfree(sha->sas_phy);
+ kfree(sha->sas_port);
+diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c
+index 32e60f0c3b148..5fb08acbc0e5e 100644
+--- a/drivers/scsi/pm8001/pm8001_sas.c
++++ b/drivers/scsi/pm8001/pm8001_sas.c
+@@ -753,8 +753,13 @@ static int pm8001_exec_internal_tmf_task(struct domain_device *dev,
+ res = -TMF_RESP_FUNC_FAILED;
+ /* Even TMF timed out, return direct. */
+ if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
++ struct pm8001_ccb_info *ccb = task->lldd_task;
++
+ pm8001_dbg(pm8001_ha, FAIL, "TMF task[%x]timeout.\n",
+ tmf->tmf);
++
++ if (ccb)
++ ccb->task = NULL;
+ goto ex_err;
+ }
+
+@@ -826,10 +831,10 @@ pm8001_exec_internal_task_abort(struct pm8001_hba_info *pm8001_ha,
+
+ res = PM8001_CHIP_DISP->task_abort(pm8001_ha,
+ pm8001_dev, flag, task_tag, ccb_tag);
+-
+ if (res) {
+ del_timer(&task->slow_task->timer);
+ pm8001_dbg(pm8001_ha, FAIL, "Executing internal task failed\n");
++ pm8001_tag_free(pm8001_ha, ccb_tag);
+ goto ex_err;
+ }
+ wait_for_completion(&task->slow_task->completion);
+diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h
+index 62d08b535a4b6..f40a41f450d9b 100644
+--- a/drivers/scsi/pm8001/pm8001_sas.h
++++ b/drivers/scsi/pm8001/pm8001_sas.h
+@@ -457,6 +457,7 @@ struct outbound_queue_table {
+ __le32 producer_index;
+ u32 consumer_idx;
+ spinlock_t oq_lock;
++ unsigned long lock_flags;
+ };
+ struct pm8001_hba_memspace {
+ void __iomem *memvirtaddr;
+@@ -516,6 +517,7 @@ struct pm8001_hba_info {
+ u32 iomb_size; /* SPC and SPCV IOMB size */
+ struct pm8001_device *devices;
+ struct pm8001_ccb_info *ccb_info;
++ u32 ccb_count;
+ #ifdef PM8001_USE_MSIX
+ int number_of_intr;/*will be used in remove()*/
+ char intr_drvname[PM8001_MAX_MSIX_VEC]
+@@ -738,9 +740,7 @@ pm8001_ccb_task_free_done(struct pm8001_hba_info *pm8001_ha,
+ {
+ pm8001_ccb_task_free(pm8001_ha, task, ccb, ccb_idx);
+ smp_mb(); /*in order to force CPU ordering*/
+- spin_unlock(&pm8001_ha->lock);
+ task->task_done(task);
+- spin_lock(&pm8001_ha->lock);
+ }
+
+ #endif
+diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
+index 6ffe17b849ae8..04746df26c6c9 100644
+--- a/drivers/scsi/pm8001/pm80xx_hwi.c
++++ b/drivers/scsi/pm8001/pm80xx_hwi.c
+@@ -66,18 +66,16 @@ int pm80xx_bar4_shift(struct pm8001_hba_info *pm8001_ha, u32 shift_value)
+ }
+
+ static void pm80xx_pci_mem_copy(struct pm8001_hba_info *pm8001_ha, u32 soffset,
+- const void *destination,
++ __le32 *destination,
+ u32 dw_count, u32 bus_base_number)
+ {
+ u32 index, value, offset;
+- u32 *destination1;
+- destination1 = (u32 *)destination;
+
+- for (index = 0; index < dw_count; index += 4, destination1++) {
++ for (index = 0; index < dw_count; index += 4, destination++) {
+ offset = (soffset + index);
+ if (offset < (64 * 1024)) {
+ value = pm8001_cr32(pm8001_ha, bus_base_number, offset);
+- *destination1 = cpu_to_le32(value);
++ *destination = cpu_to_le32(value);
+ }
+ }
+ return;
+@@ -767,6 +765,10 @@ static void init_default_table_values(struct pm8001_hba_info *pm8001_ha)
+ pm8001_ha->main_cfg_tbl.pm80xx_tbl.pcs_event_log_severity = 0x01;
+ pm8001_ha->main_cfg_tbl.pm80xx_tbl.fatal_err_interrupt = 0x01;
+
++ /* Enable higher IQs and OQs, 32 to 63, bit 16 */
++ if (pm8001_ha->max_q_num > 32)
++ pm8001_ha->main_cfg_tbl.pm80xx_tbl.fatal_err_interrupt |=
++ 1 << 16;
+ /* Disable end to end CRC checking */
+ pm8001_ha->main_cfg_tbl.pm80xx_tbl.crc_core_dump = (0x1 << 16);
+
+@@ -1028,6 +1030,13 @@ static int mpi_init_check(struct pm8001_hba_info *pm8001_ha)
+ if (0x0000 != gst_len_mpistate)
+ return -EBUSY;
+
++ /*
++ * As per controller datasheet, after successful MPI
++ * initialization minimum 500ms delay is required before
++ * issuing commands.
++ */
++ msleep(500);
++
+ return 0;
+ }
+
+@@ -1202,9 +1211,11 @@ pm80xx_set_thermal_config(struct pm8001_hba_info *pm8001_ha)
+ else
+ page_code = THERMAL_PAGE_CODE_8H;
+
+- payload.cfg_pg[0] = (THERMAL_LOG_ENABLE << 9) |
+- (THERMAL_ENABLE << 8) | page_code;
+- payload.cfg_pg[1] = (LTEMPHIL << 24) | (RTEMPHIL << 8);
++ payload.cfg_pg[0] =
++ cpu_to_le32((THERMAL_LOG_ENABLE << 9) |
++ (THERMAL_ENABLE << 8) | page_code);
++ payload.cfg_pg[1] =
++ cpu_to_le32((LTEMPHIL << 24) | (RTEMPHIL << 8));
+
+ pm8001_dbg(pm8001_ha, DEV,
+ "Setting up thermal config. cfg_pg 0 0x%x cfg_pg 1 0x%x\n",
+@@ -1244,43 +1255,41 @@ pm80xx_set_sas_protocol_timer_config(struct pm8001_hba_info *pm8001_ha)
+ circularQ = &pm8001_ha->inbnd_q_tbl[0];
+ payload.tag = cpu_to_le32(tag);
+
+- SASConfigPage.pageCode = SAS_PROTOCOL_TIMER_CONFIG_PAGE;
+- SASConfigPage.MST_MSI = 3 << 15;
+- SASConfigPage.STP_SSP_MCT_TMO = (STP_MCT_TMO << 16) | SSP_MCT_TMO;
+- SASConfigPage.STP_FRM_TMO = (SAS_MAX_OPEN_TIME << 24) |
+- (SMP_MAX_CONN_TIMER << 16) | STP_FRM_TIMER;
+- SASConfigPage.STP_IDLE_TMO = STP_IDLE_TIME;
+-
+- if (SASConfigPage.STP_IDLE_TMO > 0x3FFFFFF)
+- SASConfigPage.STP_IDLE_TMO = 0x3FFFFFF;
+-
+-
+- SASConfigPage.OPNRJT_RTRY_INTVL = (SAS_MFD << 16) |
+- SAS_OPNRJT_RTRY_INTVL;
+- SASConfigPage.Data_Cmd_OPNRJT_RTRY_TMO = (SAS_DOPNRJT_RTRY_TMO << 16)
+- | SAS_COPNRJT_RTRY_TMO;
+- SASConfigPage.Data_Cmd_OPNRJT_RTRY_THR = (SAS_DOPNRJT_RTRY_THR << 16)
+- | SAS_COPNRJT_RTRY_THR;
+- SASConfigPage.MAX_AIP = SAS_MAX_AIP;
++ SASConfigPage.pageCode = cpu_to_le32(SAS_PROTOCOL_TIMER_CONFIG_PAGE);
++ SASConfigPage.MST_MSI = cpu_to_le32(3 << 15);
++ SASConfigPage.STP_SSP_MCT_TMO =
++ cpu_to_le32((STP_MCT_TMO << 16) | SSP_MCT_TMO);
++ SASConfigPage.STP_FRM_TMO =
++ cpu_to_le32((SAS_MAX_OPEN_TIME << 24) |
++ (SMP_MAX_CONN_TIMER << 16) | STP_FRM_TIMER);
++ SASConfigPage.STP_IDLE_TMO = cpu_to_le32(STP_IDLE_TIME);
++
++ SASConfigPage.OPNRJT_RTRY_INTVL =
++ cpu_to_le32((SAS_MFD << 16) | SAS_OPNRJT_RTRY_INTVL);
++ SASConfigPage.Data_Cmd_OPNRJT_RTRY_TMO =
++ cpu_to_le32((SAS_DOPNRJT_RTRY_TMO << 16) | SAS_COPNRJT_RTRY_TMO);
++ SASConfigPage.Data_Cmd_OPNRJT_RTRY_THR =
++ cpu_to_le32((SAS_DOPNRJT_RTRY_THR << 16) | SAS_COPNRJT_RTRY_THR);
++ SASConfigPage.MAX_AIP = cpu_to_le32(SAS_MAX_AIP);
+
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.pageCode 0x%08x\n",
+- SASConfigPage.pageCode);
++ le32_to_cpu(SASConfigPage.pageCode));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.MST_MSI 0x%08x\n",
+- SASConfigPage.MST_MSI);
++ le32_to_cpu(SASConfigPage.MST_MSI));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.STP_SSP_MCT_TMO 0x%08x\n",
+- SASConfigPage.STP_SSP_MCT_TMO);
++ le32_to_cpu(SASConfigPage.STP_SSP_MCT_TMO));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.STP_FRM_TMO 0x%08x\n",
+- SASConfigPage.STP_FRM_TMO);
++ le32_to_cpu(SASConfigPage.STP_FRM_TMO));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.STP_IDLE_TMO 0x%08x\n",
+- SASConfigPage.STP_IDLE_TMO);
++ le32_to_cpu(SASConfigPage.STP_IDLE_TMO));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.OPNRJT_RTRY_INTVL 0x%08x\n",
+- SASConfigPage.OPNRJT_RTRY_INTVL);
++ le32_to_cpu(SASConfigPage.OPNRJT_RTRY_INTVL));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.Data_Cmd_OPNRJT_RTRY_TMO 0x%08x\n",
+- SASConfigPage.Data_Cmd_OPNRJT_RTRY_TMO);
++ le32_to_cpu(SASConfigPage.Data_Cmd_OPNRJT_RTRY_TMO));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.Data_Cmd_OPNRJT_RTRY_THR 0x%08x\n",
+- SASConfigPage.Data_Cmd_OPNRJT_RTRY_THR);
++ le32_to_cpu(SASConfigPage.Data_Cmd_OPNRJT_RTRY_THR));
+ pm8001_dbg(pm8001_ha, INIT, "SASConfigPage.MAX_AIP 0x%08x\n",
+- SASConfigPage.MAX_AIP);
++ le32_to_cpu(SASConfigPage.MAX_AIP));
+
+ memcpy(&payload.cfg_pg, &SASConfigPage,
+ sizeof(SASProtocolTimerConfig_t));
+@@ -1406,12 +1415,13 @@ static int pm80xx_encrypt_update(struct pm8001_hba_info *pm8001_ha)
+ /* Currently only one key is used. New KEK index is 1.
+ * Current KEK index is 1. Store KEK to NVRAM is 1.
+ */
+- payload.new_curidx_ksop = ((1 << 24) | (1 << 16) | (1 << 8) |
+- KEK_MGMT_SUBOP_KEYCARDUPDATE);
++ payload.new_curidx_ksop =
++ cpu_to_le32(((1 << 24) | (1 << 16) | (1 << 8) |
++ KEK_MGMT_SUBOP_KEYCARDUPDATE));
+
+ pm8001_dbg(pm8001_ha, DEV,
+ "Saving Encryption info to flash. payload 0x%x\n",
+- payload.new_curidx_ksop);
++ le32_to_cpu(payload.new_curidx_ksop));
+
+ rc = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload,
+ sizeof(payload), 0);
+@@ -1734,10 +1744,11 @@ static void
+ pm80xx_chip_interrupt_enable(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ {
+ #ifdef PM8001_USE_MSIX
+- u32 mask;
+- mask = (u32)(1 << vec);
+-
+- pm8001_cw32(pm8001_ha, 0, MSGU_ODMR_CLR, (u32)(mask & 0xFFFFFFFF));
++ if (vec < 32)
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR_CLR, 1U << vec);
++ else
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR_CLR_U,
++ 1U << (vec - 32));
+ return;
+ #endif
+ pm80xx_chip_intx_interrupt_enable(pm8001_ha);
+@@ -1753,12 +1764,15 @@ static void
+ pm80xx_chip_interrupt_disable(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ {
+ #ifdef PM8001_USE_MSIX
+- u32 mask;
+- if (vec == 0xFF)
+- mask = 0xFFFFFFFF;
++ if (vec == 0xFF) {
++ /* disable all vectors 0-31, 32-63 */
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR, 0xFFFFFFFF);
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR_U, 0xFFFFFFFF);
++ } else if (vec < 32)
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR, 1U << vec);
+ else
+- mask = (u32)(1 << vec);
+- pm8001_cw32(pm8001_ha, 0, MSGU_ODMR, (u32)(mask & 0xFFFFFFFF));
++ pm8001_cw32(pm8001_ha, 0, MSGU_ODMR_U,
++ 1U << (vec - 32));
+ return;
+ #endif
+ pm80xx_chip_intx_interrupt_disable(pm8001_ha);
+@@ -1800,6 +1814,7 @@ static void pm80xx_send_abort_all(struct pm8001_hba_info *pm8001_ha,
+ ccb->device = pm8001_ha_dev;
+ ccb->ccb_tag = ccb_tag;
+ ccb->task = task;
++ ccb->n_elem = 0;
+
+ circularQ = &pm8001_ha->inbnd_q_tbl[0];
+
+@@ -1881,7 +1896,7 @@ static void pm80xx_send_read_log(struct pm8001_hba_info *pm8001_ha,
+
+ sata_cmd.tag = cpu_to_le32(ccb_tag);
+ sata_cmd.device_id = cpu_to_le32(pm8001_ha_dev->device_id);
+- sata_cmd.ncqtag_atap_dir_m_dad |= ((0x1 << 7) | (0x5 << 9));
++ sata_cmd.ncqtag_atap_dir_m_dad = cpu_to_le32(((0x1 << 7) | (0x5 << 9)));
+ memcpy(&sata_cmd.sata_fis, &fis, sizeof(struct host_to_dev_fis));
+
+ res = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &sata_cmd,
+@@ -2184,9 +2199,9 @@ mpi_ssp_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ pm8001_dbg(pm8001_ha, FAIL,
+ "task 0x%p done with io_status 0x%x resp 0x%x stat 0x%x but aborted by upper layer!\n",
+ t, status, ts->resp, ts->stat);
++ pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+ if (t->slow_task)
+ complete(&t->slow_task->completion);
+- pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+ } else {
+ spin_unlock_irqrestore(&t->task_state_lock, flags);
+ pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+@@ -2379,7 +2394,8 @@ static void mpi_ssp_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+
+ /*See the comments for mpi_ssp_completion */
+ static void
+-mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
++mpi_sata_completion(struct pm8001_hba_info *pm8001_ha,
++ struct outbound_queue_table *circularQ, void *piomb)
+ {
+ struct sas_task *t;
+ struct pm8001_ccb_info *ccb;
+@@ -2516,7 +2532,8 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ len = sizeof(struct pio_setup_fis);
+ pm8001_dbg(pm8001_ha, IO,
+ "PIO read len = %d\n", len);
+- } else if (t->ata_task.use_ncq) {
++ } else if (t->ata_task.use_ncq &&
++ t->data_dir != DMA_NONE) {
+ len = sizeof(struct set_dev_bits_fis);
+ pm8001_dbg(pm8001_ha, IO, "FPDMA len = %d\n",
+ len);
+@@ -2616,7 +2633,11 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_OPEN_CNX_ERROR_IT_NEXUS_LOSS);
+ ts->resp = SAS_TASK_UNDELIVERED;
+ ts->stat = SAS_QUEUE_FULL;
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ return;
+ }
+ break;
+@@ -2632,7 +2653,11 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_OPEN_CNX_ERROR_IT_NEXUS_LOSS);
+ ts->resp = SAS_TASK_UNDELIVERED;
+ ts->stat = SAS_QUEUE_FULL;
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ return;
+ }
+ break;
+@@ -2656,7 +2681,11 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_OPEN_CNX_ERROR_STP_RESOURCES_BUSY);
+ ts->resp = SAS_TASK_UNDELIVERED;
+ ts->stat = SAS_QUEUE_FULL;
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ return;
+ }
+ break;
+@@ -2727,7 +2756,11 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_DS_NON_OPERATIONAL);
+ ts->resp = SAS_TASK_UNDELIVERED;
+ ts->stat = SAS_QUEUE_FULL;
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ return;
+ }
+ break;
+@@ -2747,7 +2780,11 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_DS_IN_ERROR);
+ ts->resp = SAS_TASK_UNDELIVERED;
+ ts->stat = SAS_QUEUE_FULL;
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ return;
+ }
+ break;
+@@ -2780,17 +2817,22 @@ mpi_sata_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ pm8001_dbg(pm8001_ha, FAIL,
+ "task 0x%p done with io_status 0x%x resp 0x%x stat 0x%x but aborted by upper layer!\n",
+ t, status, ts->resp, ts->stat);
++ pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+ if (t->slow_task)
+ complete(&t->slow_task->completion);
+- pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+ } else {
+ spin_unlock_irqrestore(&t->task_state_lock, flags);
++ spin_unlock_irqrestore(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
++ spin_lock_irqsave(&circularQ->oq_lock,
++ circularQ->lock_flags);
+ }
+ }
+
+ /*See the comments for mpi_ssp_completion */
+-static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
++static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha,
++ struct outbound_queue_table *circularQ, void *piomb)
+ {
+ struct sas_task *t;
+ struct task_status_struct *ts;
+@@ -2802,7 +2844,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ u32 tag = le32_to_cpu(psataPayload->tag);
+ u32 port_id = le32_to_cpu(psataPayload->port_id);
+ u32 dev_id = le32_to_cpu(psataPayload->device_id);
+- unsigned long flags;
+
+ ccb = &pm8001_ha->ccb_info[tag];
+
+@@ -2840,8 +2881,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ ts->resp = SAS_TASK_COMPLETE;
+ ts->stat = SAS_DATA_OVERRUN;
+ ts->residual = 0;
+- if (pm8001_dev)
+- atomic_dec(&pm8001_dev->running_req);
+ break;
+ case IO_XFER_ERROR_BREAK:
+ pm8001_dbg(pm8001_ha, IO, "IO_XFER_ERROR_BREAK\n");
+@@ -2890,7 +2929,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ IO_OPEN_CNX_ERROR_IT_NEXUS_LOSS);
+ ts->resp = SAS_TASK_COMPLETE;
+ ts->stat = SAS_QUEUE_FULL;
+- pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
+ return;
+ }
+ break;
+@@ -2990,20 +3028,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ ts->stat = SAS_OPEN_TO;
+ break;
+ }
+- spin_lock_irqsave(&t->task_state_lock, flags);
+- t->task_state_flags &= ~SAS_TASK_STATE_PENDING;
+- t->task_state_flags &= ~SAS_TASK_AT_INITIATOR;
+- t->task_state_flags |= SAS_TASK_STATE_DONE;
+- if (unlikely((t->task_state_flags & SAS_TASK_STATE_ABORTED))) {
+- spin_unlock_irqrestore(&t->task_state_lock, flags);
+- pm8001_dbg(pm8001_ha, FAIL,
+- "task 0x%p done with io_status 0x%x resp 0x%x stat 0x%x but aborted by upper layer!\n",
+- t, event, ts->resp, ts->stat);
+- pm8001_ccb_task_free(pm8001_ha, t, ccb, tag);
+- } else {
+- spin_unlock_irqrestore(&t->task_state_lock, flags);
+- pm8001_ccb_task_free_done(pm8001_ha, t, ccb, tag);
+- }
+ }
+
+ /*See the comments for mpi_ssp_completion */
+@@ -3902,7 +3926,8 @@ static int ssp_coalesced_comp_resp(struct pm8001_hba_info *pm8001_ha,
+ * @pm8001_ha: our hba card information
+ * @piomb: IO message buffer
+ */
+-static void process_one_iomb(struct pm8001_hba_info *pm8001_ha, void *piomb)
++static void process_one_iomb(struct pm8001_hba_info *pm8001_ha,
++ struct outbound_queue_table *circularQ, void *piomb)
+ {
+ __le32 pHeader = *(__le32 *)piomb;
+ u32 opc = (u32)((le32_to_cpu(pHeader)) & 0xFFF);
+@@ -3944,11 +3969,11 @@ static void process_one_iomb(struct pm8001_hba_info *pm8001_ha, void *piomb)
+ break;
+ case OPC_OUB_SATA_COMP:
+ pm8001_dbg(pm8001_ha, MSG, "OPC_OUB_SATA_COMP\n");
+- mpi_sata_completion(pm8001_ha, piomb);
++ mpi_sata_completion(pm8001_ha, circularQ, piomb);
+ break;
+ case OPC_OUB_SATA_EVENT:
+ pm8001_dbg(pm8001_ha, MSG, "OPC_OUB_SATA_EVENT\n");
+- mpi_sata_event(pm8001_ha, piomb);
++ mpi_sata_event(pm8001_ha, circularQ, piomb);
+ break;
+ case OPC_OUB_SSP_EVENT:
+ pm8001_dbg(pm8001_ha, MSG, "OPC_OUB_SSP_EVENT\n");
+@@ -4117,13 +4142,24 @@ static int process_oq(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ void *pMsg1 = NULL;
+ u8 bc;
+ u32 ret = MPI_IO_STATUS_FAIL;
+- unsigned long flags;
+ u32 regval;
+
++ /*
++ * Fatal errors are programmed to be signalled in irq vector
++ * pm8001_ha->max_q_num - 1 through pm8001_ha->main_cfg_tbl.pm80xx_tbl.
++ * fatal_err_interrupt
++ */
+ if (vec == (pm8001_ha->max_q_num - 1)) {
++ u32 mipsall_ready;
++
++ if (pm8001_ha->chip_id == chip_8008 ||
++ pm8001_ha->chip_id == chip_8009)
++ mipsall_ready = SCRATCH_PAD_MIPSALL_READY_8PORT;
++ else
++ mipsall_ready = SCRATCH_PAD_MIPSALL_READY_16PORT;
++
+ regval = pm8001_cr32(pm8001_ha, 0, MSGU_SCRATCH_PAD_1);
+- if ((regval & SCRATCH_PAD_MIPSALL_READY) !=
+- SCRATCH_PAD_MIPSALL_READY) {
++ if ((regval & mipsall_ready) != mipsall_ready) {
+ pm8001_ha->controller_fatal_error = true;
+ pm8001_dbg(pm8001_ha, FAIL,
+ "Firmware Fatal error! Regval:0x%x\n",
+@@ -4134,7 +4170,7 @@ static int process_oq(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ }
+ }
+ circularQ = &pm8001_ha->outbnd_q_tbl[vec];
+- spin_lock_irqsave(&circularQ->oq_lock, flags);
++ spin_lock_irqsave(&circularQ->oq_lock, circularQ->lock_flags);
+ do {
+ /* spurious interrupt during setup if kexec-ing and
+ * driver doing a doorbell access w/ the pre-kexec oq
+@@ -4145,7 +4181,8 @@ static int process_oq(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ ret = pm8001_mpi_msg_consume(pm8001_ha, circularQ, &pMsg1, &bc);
+ if (MPI_IO_STATUS_SUCCESS == ret) {
+ /* process the outbound message */
+- process_one_iomb(pm8001_ha, (void *)(pMsg1 - 4));
++ process_one_iomb(pm8001_ha, circularQ,
++ (void *)(pMsg1 - 4));
+ /* free the message from the outbound circular buffer */
+ pm8001_mpi_msg_free_set(pm8001_ha, pMsg1,
+ circularQ, bc);
+@@ -4160,7 +4197,7 @@ static int process_oq(struct pm8001_hba_info *pm8001_ha, u8 vec)
+ break;
+ }
+ } while (1);
+- spin_unlock_irqrestore(&circularQ->oq_lock, flags);
++ spin_unlock_irqrestore(&circularQ->oq_lock, circularQ->lock_flags);
+ return ret;
+ }
+
+@@ -4359,13 +4396,15 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ struct ssp_ini_io_start_req ssp_cmd;
+ u32 tag = ccb->ccb_tag;
+ int ret;
+- u64 phys_addr, start_addr, end_addr;
++ u64 phys_addr, end_addr;
+ u32 end_addr_high, end_addr_low;
+ struct inbound_queue_table *circularQ;
+ u32 q_index, cpu_id;
+ u32 opc = OPC_INB_SSPINIIOSTART;
++
+ memset(&ssp_cmd, 0, sizeof(ssp_cmd));
+ memcpy(ssp_cmd.ssp_iu.lun, task->ssp_task.LUN, 8);
++
+ /* data address domain added for spcv; set to 0 by host,
+ * used internally by controller
+ * 0 for SAS 1.1 and SAS 2.0 compatible TLR
+@@ -4376,7 +4415,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ ssp_cmd.device_id = cpu_to_le32(pm8001_dev->device_id);
+ ssp_cmd.tag = cpu_to_le32(tag);
+ if (task->ssp_task.enable_first_burst)
+- ssp_cmd.ssp_iu.efb_prio_attr |= 0x80;
++ ssp_cmd.ssp_iu.efb_prio_attr = 0x80;
+ ssp_cmd.ssp_iu.efb_prio_attr |= (task->ssp_task.task_prio << 3);
+ ssp_cmd.ssp_iu.efb_prio_attr |= (task->ssp_task.task_attr & 7);
+ memcpy(ssp_cmd.ssp_iu.cdb, task->ssp_task.cmd->cmnd,
+@@ -4408,21 +4447,24 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ ssp_cmd.enc_esgl = cpu_to_le32(1<<31);
+ } else if (task->num_scatter == 1) {
+ u64 dma_addr = sg_dma_address(task->scatter);
++
+ ssp_cmd.enc_addr_low =
+ cpu_to_le32(lower_32_bits(dma_addr));
+ ssp_cmd.enc_addr_high =
+ cpu_to_le32(upper_32_bits(dma_addr));
+ ssp_cmd.enc_len = cpu_to_le32(task->total_xfer_len);
+ ssp_cmd.enc_esgl = 0;
++
+ /* Check 4G Boundary */
+- start_addr = cpu_to_le64(dma_addr);
+- end_addr = (start_addr + ssp_cmd.enc_len) - 1;
+- end_addr_low = cpu_to_le32(lower_32_bits(end_addr));
+- end_addr_high = cpu_to_le32(upper_32_bits(end_addr));
+- if (end_addr_high != ssp_cmd.enc_addr_high) {
++ end_addr = dma_addr + le32_to_cpu(ssp_cmd.enc_len) - 1;
++ end_addr_low = lower_32_bits(end_addr);
++ end_addr_high = upper_32_bits(end_addr);
++
++ if (end_addr_high != le32_to_cpu(ssp_cmd.enc_addr_high)) {
+ pm8001_dbg(pm8001_ha, FAIL,
+ "The sg list address start_addr=0x%016llx data_len=0x%x end_addr_high=0x%08x end_addr_low=0x%08x has crossed 4G boundary\n",
+- start_addr, ssp_cmd.enc_len,
++ dma_addr,
++ le32_to_cpu(ssp_cmd.enc_len),
+ end_addr_high, end_addr_low);
+ pm8001_chip_make_sg(task->scatter, 1,
+ ccb->buf_prd);
+@@ -4431,7 +4473,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ cpu_to_le32(lower_32_bits(phys_addr));
+ ssp_cmd.enc_addr_high =
+ cpu_to_le32(upper_32_bits(phys_addr));
+- ssp_cmd.enc_esgl = cpu_to_le32(1<<31);
++ ssp_cmd.enc_esgl = cpu_to_le32(1U<<31);
+ }
+ } else if (task->num_scatter == 0) {
+ ssp_cmd.enc_addr_low = 0;
+@@ -4439,8 +4481,10 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ ssp_cmd.enc_len = cpu_to_le32(task->total_xfer_len);
+ ssp_cmd.enc_esgl = 0;
+ }
++
+ /* XTS mode. All other fields are 0 */
+- ssp_cmd.key_cmode = 0x6 << 4;
++ ssp_cmd.key_cmode = cpu_to_le32(0x6 << 4);
++
+ /* set tweak values. Should be the start lba */
+ ssp_cmd.twk_val0 = cpu_to_le32((task->ssp_task.cmd->cmnd[2] << 24) |
+ (task->ssp_task.cmd->cmnd[3] << 16) |
+@@ -4462,20 +4506,22 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
+ ssp_cmd.esgl = cpu_to_le32(1<<31);
+ } else if (task->num_scatter == 1) {
+ u64 dma_addr = sg_dma_address(task->scatter);
++
+ ssp_cmd.addr_low = cpu_to_le32(lower_32_bits(dma_addr));
+ ssp_cmd.addr_high =
+ cpu_to_le32(upper_32_bits(dma_addr));
+ ssp_cmd.len = cpu_to_le32(task->total_xfer_len);
+ ssp_cmd.esgl = 0;
++
+ /* Check 4G Boundary */
+- start_addr = cpu_to_le64(dma_addr);
+- end_addr = (start_addr + ssp_cmd.len) - 1;
+- end_addr_low = cpu_to_le32(lower_32_bits(end_addr));
+- end_addr_high = cpu_to_le32(upper_32_bits(end_addr));
+- if (end_addr_high != ssp_cmd.addr_high) {
++ end_addr = dma_addr + le32_to_cpu(ssp_cmd.len) - 1;
++ end_addr_low = lower_32_bits(end_addr);
++ end_addr_high = upper_32_bits(end_addr);
++ if (end_addr_high != le32_to_cpu(ssp_cmd.addr_high)) {
+ pm8001_dbg(pm8001_ha, FAIL,
+ "The sg list address start_addr=0x%016llx data_len=0x%x end_addr_high=0x%08x end_addr_low=0x%08x has crossed 4G boundary\n",
+- start_addr, ssp_cmd.len,
++ dma_addr,
++ le32_to_cpu(ssp_cmd.len),
+ end_addr_high, end_addr_low);
+ pm8001_chip_make_sg(task->scatter, 1,
+ ccb->buf_prd);
+@@ -4509,7 +4555,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ u32 q_index, cpu_id;
+ struct sata_start_req sata_cmd;
+ u32 hdr_tag, ncg_tag = 0;
+- u64 phys_addr, start_addr, end_addr;
++ u64 phys_addr, end_addr;
+ u32 end_addr_high, end_addr_low;
+ u32 ATAP = 0x0;
+ u32 dir;
+@@ -4521,22 +4567,21 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ q_index = (u32) (cpu_id) % (pm8001_ha->max_q_num);
+ circularQ = &pm8001_ha->inbnd_q_tbl[q_index];
+
+- if (task->data_dir == DMA_NONE) {
++ if (task->data_dir == DMA_NONE && !task->ata_task.use_ncq) {
+ ATAP = 0x04; /* no data*/
+ pm8001_dbg(pm8001_ha, IO, "no data\n");
+ } else if (likely(!task->ata_task.device_control_reg_update)) {
+- if (task->ata_task.dma_xfer) {
++ if (task->ata_task.use_ncq &&
++ dev->sata_dev.class != ATA_DEV_ATAPI) {
++ ATAP = 0x07; /* FPDMA */
++ pm8001_dbg(pm8001_ha, IO, "FPDMA\n");
++ } else if (task->ata_task.dma_xfer) {
+ ATAP = 0x06; /* DMA */
+ pm8001_dbg(pm8001_ha, IO, "DMA\n");
+ } else {
+ ATAP = 0x05; /* PIO*/
+ pm8001_dbg(pm8001_ha, IO, "PIO\n");
+ }
+- if (task->ata_task.use_ncq &&
+- dev->sata_dev.class != ATA_DEV_ATAPI) {
+- ATAP = 0x07; /* FPDMA */
+- pm8001_dbg(pm8001_ha, IO, "FPDMA\n");
+- }
+ }
+ if (task->ata_task.use_ncq && pm8001_get_ncq_tag(task, &hdr_tag)) {
+ task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
+@@ -4570,32 +4615,38 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ pm8001_chip_make_sg(task->scatter,
+ ccb->n_elem, ccb->buf_prd);
+ phys_addr = ccb->ccb_dma_handle;
+- sata_cmd.enc_addr_low = lower_32_bits(phys_addr);
+- sata_cmd.enc_addr_high = upper_32_bits(phys_addr);
++ sata_cmd.enc_addr_low =
++ cpu_to_le32(lower_32_bits(phys_addr));
++ sata_cmd.enc_addr_high =
++ cpu_to_le32(upper_32_bits(phys_addr));
+ sata_cmd.enc_esgl = cpu_to_le32(1 << 31);
+ } else if (task->num_scatter == 1) {
+ u64 dma_addr = sg_dma_address(task->scatter);
+- sata_cmd.enc_addr_low = lower_32_bits(dma_addr);
+- sata_cmd.enc_addr_high = upper_32_bits(dma_addr);
++
++ sata_cmd.enc_addr_low =
++ cpu_to_le32(lower_32_bits(dma_addr));
++ sata_cmd.enc_addr_high =
++ cpu_to_le32(upper_32_bits(dma_addr));
+ sata_cmd.enc_len = cpu_to_le32(task->total_xfer_len);
+ sata_cmd.enc_esgl = 0;
++
+ /* Check 4G Boundary */
+- start_addr = cpu_to_le64(dma_addr);
+- end_addr = (start_addr + sata_cmd.enc_len) - 1;
+- end_addr_low = cpu_to_le32(lower_32_bits(end_addr));
+- end_addr_high = cpu_to_le32(upper_32_bits(end_addr));
+- if (end_addr_high != sata_cmd.enc_addr_high) {
++ end_addr = dma_addr + le32_to_cpu(sata_cmd.enc_len) - 1;
++ end_addr_low = lower_32_bits(end_addr);
++ end_addr_high = upper_32_bits(end_addr);
++ if (end_addr_high != le32_to_cpu(sata_cmd.enc_addr_high)) {
+ pm8001_dbg(pm8001_ha, FAIL,
+ "The sg list address start_addr=0x%016llx data_len=0x%x end_addr_high=0x%08x end_addr_low=0x%08x has crossed 4G boundary\n",
+- start_addr, sata_cmd.enc_len,
++ dma_addr,
++ le32_to_cpu(sata_cmd.enc_len),
+ end_addr_high, end_addr_low);
+ pm8001_chip_make_sg(task->scatter, 1,
+ ccb->buf_prd);
+ phys_addr = ccb->ccb_dma_handle;
+ sata_cmd.enc_addr_low =
+- lower_32_bits(phys_addr);
++ cpu_to_le32(lower_32_bits(phys_addr));
+ sata_cmd.enc_addr_high =
+- upper_32_bits(phys_addr);
++ cpu_to_le32(upper_32_bits(phys_addr));
+ sata_cmd.enc_esgl =
+ cpu_to_le32(1 << 31);
+ }
+@@ -4606,7 +4657,8 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ sata_cmd.enc_esgl = 0;
+ }
+ /* XTS mode. All other fields are 0 */
+- sata_cmd.key_index_mode = 0x6 << 4;
++ sata_cmd.key_index_mode = cpu_to_le32(0x6 << 4);
++
+ /* set tweak values. Should be the start lba */
+ sata_cmd.twk_val0 =
+ cpu_to_le32((sata_cmd.sata_fis.lbal_exp << 24) |
+@@ -4632,31 +4684,31 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ phys_addr = ccb->ccb_dma_handle;
+ sata_cmd.addr_low = lower_32_bits(phys_addr);
+ sata_cmd.addr_high = upper_32_bits(phys_addr);
+- sata_cmd.esgl = cpu_to_le32(1 << 31);
++ sata_cmd.esgl = cpu_to_le32(1U << 31);
+ } else if (task->num_scatter == 1) {
+ u64 dma_addr = sg_dma_address(task->scatter);
++
+ sata_cmd.addr_low = lower_32_bits(dma_addr);
+ sata_cmd.addr_high = upper_32_bits(dma_addr);
+ sata_cmd.len = cpu_to_le32(task->total_xfer_len);
+ sata_cmd.esgl = 0;
++
+ /* Check 4G Boundary */
+- start_addr = cpu_to_le64(dma_addr);
+- end_addr = (start_addr + sata_cmd.len) - 1;
+- end_addr_low = cpu_to_le32(lower_32_bits(end_addr));
+- end_addr_high = cpu_to_le32(upper_32_bits(end_addr));
++ end_addr = dma_addr + le32_to_cpu(sata_cmd.len) - 1;
++ end_addr_low = lower_32_bits(end_addr);
++ end_addr_high = upper_32_bits(end_addr);
+ if (end_addr_high != sata_cmd.addr_high) {
+ pm8001_dbg(pm8001_ha, FAIL,
+ "The sg list address start_addr=0x%016llx data_len=0x%xend_addr_high=0x%08x end_addr_low=0x%08x has crossed 4G boundary\n",
+- start_addr, sata_cmd.len,
++ dma_addr,
++ le32_to_cpu(sata_cmd.len),
+ end_addr_high, end_addr_low);
+ pm8001_chip_make_sg(task->scatter, 1,
+ ccb->buf_prd);
+ phys_addr = ccb->ccb_dma_handle;
+- sata_cmd.addr_low =
+- lower_32_bits(phys_addr);
+- sata_cmd.addr_high =
+- upper_32_bits(phys_addr);
+- sata_cmd.esgl = cpu_to_le32(1 << 31);
++ sata_cmd.addr_low = lower_32_bits(phys_addr);
++ sata_cmd.addr_high = upper_32_bits(phys_addr);
++ sata_cmd.esgl = cpu_to_le32(1U << 31);
+ }
+ } else if (task->num_scatter == 0) {
+ sata_cmd.addr_low = 0;
+@@ -4664,27 +4716,28 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
+ sata_cmd.len = cpu_to_le32(task->total_xfer_len);
+ sata_cmd.esgl = 0;
+ }
++
+ /* scsi cdb */
+ sata_cmd.atapi_scsi_cdb[0] =
+ cpu_to_le32(((task->ata_task.atapi_packet[0]) |
+- (task->ata_task.atapi_packet[1] << 8) |
+- (task->ata_task.atapi_packet[2] << 16) |
+- (task->ata_task.atapi_packet[3] << 24)));
++ (task->ata_task.atapi_packet[1] << 8) |
++ (task->ata_task.atapi_packet[2] << 16) |
++ (task->ata_task.atapi_packet[3] << 24)));
+ sata_cmd.atapi_scsi_cdb[1] =
+ cpu_to_le32(((task->ata_task.atapi_packet[4]) |
+- (task->ata_task.atapi_packet[5] << 8) |
+- (task->ata_task.atapi_packet[6] << 16) |
+- (task->ata_task.atapi_packet[7] << 24)));
++ (task->ata_task.atapi_packet[5] << 8) |
++ (task->ata_task.atapi_packet[6] << 16) |
++ (task->ata_task.atapi_packet[7] << 24)));
+ sata_cmd.atapi_scsi_cdb[2] =
+ cpu_to_le32(((task->ata_task.atapi_packet[8]) |
+- (task->ata_task.atapi_packet[9] << 8) |
+- (task->ata_task.atapi_packet[10] << 16) |
+- (task->ata_task.atapi_packet[11] << 24)));
++ (task->ata_task.atapi_packet[9] << 8) |
++ (task->ata_task.atapi_packet[10] << 16) |
++ (task->ata_task.atapi_packet[11] << 24)));
+ sata_cmd.atapi_scsi_cdb[3] =
+ cpu_to_le32(((task->ata_task.atapi_packet[12]) |
+- (task->ata_task.atapi_packet[13] << 8) |
+- (task->ata_task.atapi_packet[14] << 16) |
+- (task->ata_task.atapi_packet[15] << 24)));
++ (task->ata_task.atapi_packet[13] << 8) |
++ (task->ata_task.atapi_packet[14] << 16) |
++ (task->ata_task.atapi_packet[15] << 24)));
+ }
+
+ /* Check for read log for failed drive and return */
+@@ -4882,8 +4935,13 @@ static int pm80xx_chip_phy_ctl_req(struct pm8001_hba_info *pm8001_ha,
+ payload.tag = cpu_to_le32(tag);
+ payload.phyop_phyid =
+ cpu_to_le32(((phy_op & 0xFF) << 8) | (phyId & 0xFF));
+- return pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload,
+- sizeof(payload), 0);
++
++ rc = pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload,
++ sizeof(payload), 0);
++ if (rc)
++ pm8001_tag_free(pm8001_ha, tag);
++
++ return rc;
+ }
+
+ static u32 pm80xx_chip_is_our_interrupt(struct pm8001_hba_info *pm8001_ha)
+diff --git a/drivers/scsi/pm8001/pm80xx_hwi.h b/drivers/scsi/pm8001/pm80xx_hwi.h
+index c7e5d93bea924..c41ed039c92ac 100644
+--- a/drivers/scsi/pm8001/pm80xx_hwi.h
++++ b/drivers/scsi/pm8001/pm80xx_hwi.h
+@@ -1405,8 +1405,12 @@ typedef struct SASProtocolTimerConfig SASProtocolTimerConfig_t;
+ #define SCRATCH_PAD_BOOT_LOAD_SUCCESS 0x0
+ #define SCRATCH_PAD_IOP0_READY 0xC00
+ #define SCRATCH_PAD_IOP1_READY 0x3000
+-#define SCRATCH_PAD_MIPSALL_READY (SCRATCH_PAD_IOP1_READY | \
++#define SCRATCH_PAD_MIPSALL_READY_16PORT (SCRATCH_PAD_IOP1_READY | \
+ SCRATCH_PAD_IOP0_READY | \
++ SCRATCH_PAD_ILA_READY | \
++ SCRATCH_PAD_RAAE_READY)
++#define SCRATCH_PAD_MIPSALL_READY_8PORT (SCRATCH_PAD_IOP0_READY | \
++ SCRATCH_PAD_ILA_READY | \
+ SCRATCH_PAD_RAAE_READY)
+
+ /* boot loader state */
+diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
+index 3404782988d58..bb5761ed3f511 100644
+--- a/drivers/scsi/qedf/qedf_io.c
++++ b/drivers/scsi/qedf/qedf_io.c
+@@ -2257,6 +2257,7 @@ process_els:
+ io_req->tm_flags == FCP_TMF_TGT_RESET) {
+ clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+ io_req->sc_cmd = NULL;
++ kref_put(&io_req->refcount, qedf_release_cmd);
+ complete(&io_req->tm_done);
+ }
+
+diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
+index 42d0d941dba5c..544401f76c079 100644
+--- a/drivers/scsi/qedf/qedf_main.c
++++ b/drivers/scsi/qedf/qedf_main.c
+@@ -911,7 +911,7 @@ void qedf_ctx_soft_reset(struct fc_lport *lport)
+ struct qed_link_output if_link;
+
+ if (lport->vport) {
+- QEDF_ERR(NULL, "Cannot issue host reset on NPIV port.\n");
++ printk_ratelimited("Cannot issue host reset on NPIV port.\n");
+ return;
+ }
+
+@@ -1862,6 +1862,7 @@ static int qedf_vport_create(struct fc_vport *vport, bool disabled)
+ vport_qedf->cmd_mgr = base_qedf->cmd_mgr;
+ init_completion(&vport_qedf->flogi_compl);
+ INIT_LIST_HEAD(&vport_qedf->fcports);
++ INIT_DELAYED_WORK(&vport_qedf->stag_work, qedf_stag_change_work);
+
+ rc = qedf_vport_libfc_config(vport, vn_port);
+ if (rc) {
+@@ -3416,7 +3417,9 @@ retry_probe:
+ qedf->devlink = qed_ops->common->devlink_register(qedf->cdev);
+ if (IS_ERR(qedf->devlink)) {
+ QEDF_ERR(&qedf->dbg_ctx, "Cannot register devlink\n");
++ rc = PTR_ERR(qedf->devlink);
+ qedf->devlink = NULL;
++ goto err2;
+ }
+ }
+
+@@ -3976,7 +3979,9 @@ void qedf_stag_change_work(struct work_struct *work)
+ struct qedf_ctx *qedf =
+ container_of(work, struct qedf_ctx, stag_work.work);
+
+- QEDF_ERR(&qedf->dbg_ctx, "Performing software context reset.\n");
++ printk_ratelimited("[%s]:[%s:%d]:%d: Performing software context reset.",
++ dev_name(&qedf->pdev->dev), __func__, __LINE__,
++ qedf->dbg_ctx.host_no);
+ qedf_ctx_soft_reset(qedf->lport);
+ }
+
+diff --git a/drivers/scsi/qedi/qedi_fw.c b/drivers/scsi/qedi/qedi_fw.c
+index d01cd829ef975..df9ce6ed52bf9 100644
+--- a/drivers/scsi/qedi/qedi_fw.c
++++ b/drivers/scsi/qedi/qedi_fw.c
+@@ -772,11 +772,10 @@ static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi,
+ qedi_cmd->list_tmf_work = NULL;
+ }
+ }
++ spin_unlock_bh(&qedi_conn->tmf_work_lock);
+
+- if (!found) {
+- spin_unlock_bh(&qedi_conn->tmf_work_lock);
++ if (!found)
+ goto check_cleanup_reqs;
+- }
+
+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
+ "TMF work, cqe->tid=0x%x, tmf flags=0x%x, cid=0x%x\n",
+@@ -807,7 +806,6 @@ static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi,
+ qedi_cmd->state = CLEANUP_RECV;
+ unlock:
+ spin_unlock_bh(&conn->session->back_lock);
+- spin_unlock_bh(&qedi_conn->tmf_work_lock);
+ wake_up_interruptible(&qedi_conn->wait_queue);
+ return;
+
+diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
+index c5260429c637e..04b40a6c1afff 100644
+--- a/drivers/scsi/qedi/qedi_iscsi.c
++++ b/drivers/scsi/qedi/qedi_iscsi.c
+@@ -859,6 +859,37 @@ static int qedi_task_xmit(struct iscsi_task *task)
+ return qedi_iscsi_send_ioreq(task);
+ }
+
++static void qedi_offload_work(struct work_struct *work)
++{
++ struct qedi_endpoint *qedi_ep =
++ container_of(work, struct qedi_endpoint, offload_work);
++ struct qedi_ctx *qedi;
++ int wait_delay = 5 * HZ;
++ int ret;
++
++ qedi = qedi_ep->qedi;
++
++ ret = qedi_iscsi_offload_conn(qedi_ep);
++ if (ret) {
++ QEDI_ERR(&qedi->dbg_ctx,
++ "offload error: iscsi_cid=%u, qedi_ep=%p, ret=%d\n",
++ qedi_ep->iscsi_cid, qedi_ep, ret);
++ qedi_ep->state = EP_STATE_OFLDCONN_FAILED;
++ return;
++ }
++
++ ret = wait_event_interruptible_timeout(qedi_ep->tcp_ofld_wait,
++ (qedi_ep->state ==
++ EP_STATE_OFLDCONN_COMPL),
++ wait_delay);
++ if (ret <= 0 || qedi_ep->state != EP_STATE_OFLDCONN_COMPL) {
++ qedi_ep->state = EP_STATE_OFLDCONN_FAILED;
++ QEDI_ERR(&qedi->dbg_ctx,
++ "Offload conn TIMEOUT iscsi_cid=%u, qedi_ep=%p\n",
++ qedi_ep->iscsi_cid, qedi_ep);
++ }
++}
++
+ static struct iscsi_endpoint *
+ qedi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
+ int non_blocking)
+@@ -907,6 +938,7 @@ qedi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
+ }
+ qedi_ep = ep->dd_data;
+ memset(qedi_ep, 0, sizeof(struct qedi_endpoint));
++ INIT_WORK(&qedi_ep->offload_work, qedi_offload_work);
+ qedi_ep->state = EP_STATE_IDLE;
+ qedi_ep->iscsi_cid = (u32)-1;
+ qedi_ep->qedi = qedi;
+@@ -1055,12 +1087,11 @@ static void qedi_ep_disconnect(struct iscsi_endpoint *ep)
+ qedi_ep = ep->dd_data;
+ qedi = qedi_ep->qedi;
+
++ flush_work(&qedi_ep->offload_work);
++
+ if (qedi_ep->state == EP_STATE_OFLDCONN_START)
+ goto ep_exit_recover;
+
+- if (qedi_ep->state != EP_STATE_OFLDCONN_NONE)
+- flush_work(&qedi_ep->offload_work);
+-
+ if (qedi_ep->conn) {
+ qedi_conn = qedi_ep->conn;
+ abrt_conn = qedi_conn->abrt_conn;
+@@ -1234,37 +1265,6 @@ static int qedi_data_avail(struct qedi_ctx *qedi, u16 vlanid)
+ return rc;
+ }
+
+-static void qedi_offload_work(struct work_struct *work)
+-{
+- struct qedi_endpoint *qedi_ep =
+- container_of(work, struct qedi_endpoint, offload_work);
+- struct qedi_ctx *qedi;
+- int wait_delay = 5 * HZ;
+- int ret;
+-
+- qedi = qedi_ep->qedi;
+-
+- ret = qedi_iscsi_offload_conn(qedi_ep);
+- if (ret) {
+- QEDI_ERR(&qedi->dbg_ctx,
+- "offload error: iscsi_cid=%u, qedi_ep=%p, ret=%d\n",
+- qedi_ep->iscsi_cid, qedi_ep, ret);
+- qedi_ep->state = EP_STATE_OFLDCONN_FAILED;
+- return;
+- }
+-
+- ret = wait_event_interruptible_timeout(qedi_ep->tcp_ofld_wait,
+- (qedi_ep->state ==
+- EP_STATE_OFLDCONN_COMPL),
+- wait_delay);
+- if ((ret <= 0) || (qedi_ep->state != EP_STATE_OFLDCONN_COMPL)) {
+- qedi_ep->state = EP_STATE_OFLDCONN_FAILED;
+- QEDI_ERR(&qedi->dbg_ctx,
+- "Offload conn TIMEOUT iscsi_cid=%u, qedi_ep=%p\n",
+- qedi_ep->iscsi_cid, qedi_ep);
+- }
+-}
+-
+ static int qedi_set_path(struct Scsi_Host *shost, struct iscsi_path *path_data)
+ {
+ struct qedi_ctx *qedi;
+@@ -1380,7 +1380,6 @@ static int qedi_set_path(struct Scsi_Host *shost, struct iscsi_path *path_data)
+ qedi_ep->dst_addr, qedi_ep->dst_port);
+ }
+
+- INIT_WORK(&qedi_ep->offload_work, qedi_offload_work);
+ queue_work(qedi->offload_thread, &qedi_ep->offload_work);
+
+ ret = 0;
+diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
+index d09776b77af2e..d3534e7f2d214 100644
+--- a/drivers/scsi/qla2xxx/qla_attr.c
++++ b/drivers/scsi/qla2xxx/qla_attr.c
+@@ -555,7 +555,7 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj,
+ if (!capable(CAP_SYS_ADMIN))
+ return -EINVAL;
+
+- if (IS_NOCACHE_VPD_TYPE(ha))
++ if (!IS_NOCACHE_VPD_TYPE(ha))
+ goto skip;
+
+ faddr = ha->flt_region_vpd << 2;
+@@ -745,7 +745,7 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
+ ql_log(ql_log_info, vha, 0x706f,
+ "Issuing MPI reset.\n");
+
+- if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
++ if (IS_QLA83XX(ha)) {
+ uint32_t idc_control;
+
+ qla83xx_idc_lock(vha, 0);
+@@ -1056,9 +1056,6 @@ qla2x00_free_sysfs_attr(scsi_qla_host_t *vha, bool stop_beacon)
+ continue;
+ if (iter->type == 3 && !(IS_CNA_CAPABLE(ha)))
+ continue;
+- if (iter->type == 0x27 &&
+- (!IS_QLA27XX(ha) || !IS_QLA28XX(ha)))
+- continue;
+
+ sysfs_remove_bin_file(&host->shost_gendev.kobj,
+ iter->attr);
+@@ -1868,6 +1865,18 @@ qla2x00_port_speed_store(struct device *dev, struct device_attribute *attr,
+ return strlen(buf);
+ }
+
++static const struct {
++ u16 rate;
++ char *str;
++} port_speed_str[] = {
++ { PORT_SPEED_4GB, "4" },
++ { PORT_SPEED_8GB, "8" },
++ { PORT_SPEED_16GB, "16" },
++ { PORT_SPEED_32GB, "32" },
++ { PORT_SPEED_64GB, "64" },
++ { PORT_SPEED_10GB, "10" },
++};
++
+ static ssize_t
+ qla2x00_port_speed_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+@@ -1875,7 +1884,8 @@ qla2x00_port_speed_show(struct device *dev, struct device_attribute *attr,
+ struct scsi_qla_host *vha = shost_priv(dev_to_shost(dev));
+ struct qla_hw_data *ha = vha->hw;
+ ssize_t rval;
+- char *spd[7] = {"0", "0", "0", "4", "8", "16", "32"};
++ u16 i;
++ char *speed = "Unknown";
+
+ rval = qla2x00_get_data_rate(vha);
+ if (rval != QLA_SUCCESS) {
+@@ -1884,7 +1894,14 @@ qla2x00_port_speed_show(struct device *dev, struct device_attribute *attr,
+ return -EINVAL;
+ }
+
+- return scnprintf(buf, PAGE_SIZE, "%s\n", spd[ha->link_data_rate]);
++ for (i = 0; i < ARRAY_SIZE(port_speed_str); i++) {
++ if (port_speed_str[i].rate != ha->link_data_rate)
++ continue;
++ speed = port_speed_str[i].str;
++ break;
++ }
++
++ return scnprintf(buf, PAGE_SIZE, "%s\n", speed);
+ }
+
+ static ssize_t
+diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
+index 655cf5de604b8..c636165be52be 100644
+--- a/drivers/scsi/qla2xxx/qla_bsg.c
++++ b/drivers/scsi/qla2xxx/qla_bsg.c
+@@ -29,7 +29,8 @@ void qla2x00_bsg_job_done(srb_t *sp, int res)
+ "%s: sp hdl %x, result=%x bsg ptr %p\n",
+ __func__, sp->handle, res, bsg_job);
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+
+ bsg_reply->result = res;
+ bsg_job_done(bsg_job, bsg_reply->result,
+@@ -3010,6 +3011,7 @@ qla24xx_bsg_timeout(struct bsg_job *bsg_job)
+
+ done:
+ spin_unlock_irqrestore(&ha->hardware_lock, flags);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return 0;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
+index 25549a8a2d72d..7cf1f78cbaeee 100644
+--- a/drivers/scsi/qla2xxx/qla_dbg.c
++++ b/drivers/scsi/qla2xxx/qla_dbg.c
+@@ -2491,6 +2491,9 @@ ql_dbg(uint level, scsi_qla_host_t *vha, uint id, const char *fmt, ...)
+ struct va_format vaf;
+ char pbuf[64];
+
++ if (!ql_mask_match(level) && !trace_ql_dbg_log_enabled())
++ return;
++
+ va_start(va, fmt);
+
+ vaf.fmt = fmt;
+diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
+index be2eb75ee1a37..2ea35e47ea441 100644
+--- a/drivers/scsi/qla2xxx/qla_def.h
++++ b/drivers/scsi/qla2xxx/qla_def.h
+@@ -726,6 +726,11 @@ typedef struct srb {
+ * code.
+ */
+ void (*put_fn)(struct kref *kref);
++
++ /*
++ * Report completion for asynchronous commands.
++ */
++ void (*async_done)(struct srb *sp, int res);
+ } srb_t;
+
+ #define GET_CMD_SP(sp) (sp->u.scmd.cmd)
+@@ -2886,7 +2891,11 @@ struct ct_fdmi2_hba_attributes {
+ #define FDMI_PORT_SPEED_8GB 0x10
+ #define FDMI_PORT_SPEED_16GB 0x20
+ #define FDMI_PORT_SPEED_32GB 0x40
+-#define FDMI_PORT_SPEED_64GB 0x80
++#define FDMI_PORT_SPEED_20GB 0x80
++#define FDMI_PORT_SPEED_40GB 0x100
++#define FDMI_PORT_SPEED_128GB 0x200
++#define FDMI_PORT_SPEED_64GB 0x400
++#define FDMI_PORT_SPEED_256GB 0x800
+ #define FDMI_PORT_SPEED_UNKNOWN 0x8000
+
+ #define FC_CLASS_2 0x04
+@@ -4261,8 +4270,10 @@ struct qla_hw_data {
+ #define QLA_ABTS_WAIT_ENABLED(_sp) \
+ (QLA_NVME_IOS(_sp) && QLA_ABTS_FW_ENABLED(_sp->fcport->vha->hw))
+
+-#define IS_PI_UNINIT_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+-#define IS_PI_IPGUARD_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha))
++#define IS_PI_UNINIT_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
++ IS_QLA28XX(ha))
++#define IS_PI_IPGUARD_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
++ IS_QLA28XX(ha))
+ #define IS_PI_DIFB_DIX0_CAPABLE(ha) (0)
+ #define IS_PI_SPLIT_DET_CAPABLE_HBA(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
+ IS_QLA28XX(ha))
+@@ -5427,4 +5438,8 @@ struct ql_vnd_tgt_stats_resp {
+ #include "qla_gbl.h"
+ #include "qla_dbg.h"
+ #include "qla_inline.h"
++
++#define IS_SESSION_DELETED(_fcport) (_fcport->disc_state == DSC_DELETE_PEND || \
++ _fcport->disc_state == DSC_DELETED)
++
+ #endif
+diff --git a/drivers/scsi/qla2xxx/qla_edif.c b/drivers/scsi/qla2xxx/qla_edif.c
+index ad746c62f0d44..a00fe88c60218 100644
+--- a/drivers/scsi/qla2xxx/qla_edif.c
++++ b/drivers/scsi/qla2xxx/qla_edif.c
+@@ -290,63 +290,6 @@ qla_edif_app_check(scsi_qla_host_t *vha, struct app_id appid)
+ return false;
+ }
+
+-static void qla_edif_reset_auth_wait(struct fc_port *fcport, int state,
+- int waitonly)
+-{
+- int cnt, max_cnt = 200;
+- bool traced = false;
+-
+- fcport->keep_nport_handle = 1;
+-
+- if (!waitonly) {
+- qla2x00_set_fcport_disc_state(fcport, state);
+- qlt_schedule_sess_for_deletion(fcport);
+- } else {
+- qla2x00_set_fcport_disc_state(fcport, state);
+- }
+-
+- ql_dbg(ql_dbg_edif, fcport->vha, 0xf086,
+- "%s: waiting for session, max_cnt=%u\n",
+- __func__, max_cnt);
+-
+- cnt = 0;
+-
+- if (waitonly) {
+- /* Marker wait min 10 msecs. */
+- msleep(50);
+- cnt += 50;
+- }
+- while (1) {
+- if (!traced) {
+- ql_dbg(ql_dbg_edif, fcport->vha, 0xf086,
+- "%s: session sleep.\n",
+- __func__);
+- traced = true;
+- }
+- msleep(20);
+- cnt++;
+- if (waitonly && (fcport->disc_state == state ||
+- fcport->disc_state == DSC_LOGIN_COMPLETE))
+- break;
+- if (fcport->disc_state == DSC_LOGIN_AUTH_PEND)
+- break;
+- if (cnt > max_cnt)
+- break;
+- }
+-
+- if (!waitonly) {
+- ql_dbg(ql_dbg_edif, fcport->vha, 0xf086,
+- "%s: waited for session - %8phC, loopid=%x portid=%06x fcport=%p state=%u, cnt=%u\n",
+- __func__, fcport->port_name, fcport->loop_id,
+- fcport->d_id.b24, fcport, fcport->disc_state, cnt);
+- } else {
+- ql_dbg(ql_dbg_edif, fcport->vha, 0xf086,
+- "%s: waited ONLY for session - %8phC, loopid=%x portid=%06x fcport=%p state=%u, cnt=%u\n",
+- __func__, fcport->port_name, fcport->loop_id,
+- fcport->d_id.b24, fcport, fcport->disc_state, cnt);
+- }
+-}
+-
+ static void
+ qla_edif_free_sa_ctl(fc_port_t *fcport, struct edif_sa_ctl *sa_ctl,
+ int index)
+@@ -529,7 +472,8 @@ qla_edif_app_start(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ struct app_start_reply appreply;
+ struct fc_port *fcport, *tf;
+
+- ql_dbg(ql_dbg_edif, vha, 0x911d, "%s app start\n", __func__);
++ ql_log(ql_log_info, vha, 0x1313,
++ "EDIF application registration with driver, FC device connections will be re-established.\n");
+
+ sg_copy_to_buffer(bsg_job->request_payload.sg_list,
+ bsg_job->request_payload.sg_cnt, &appstart,
+@@ -554,37 +498,36 @@ qla_edif_app_start(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ qla2xxx_wake_dpc(vha);
+ } else {
+ list_for_each_entry_safe(fcport, tf, &vha->vp_fcports, list) {
++ ql_dbg(ql_dbg_edif, vha, 0x2058,
++ "FCSP - nn %8phN pn %8phN portid=%06x.\n",
++ fcport->node_name, fcport->port_name,
++ fcport->d_id.b24);
+ ql_dbg(ql_dbg_edif, vha, 0xf084,
+- "%s: sess %p %8phC lid %#04x s_id %06x logout %d\n",
+- __func__, fcport, fcport->port_name,
+- fcport->loop_id, fcport->d_id.b24,
+- fcport->logout_on_delete);
+-
+- ql_dbg(ql_dbg_edif, vha, 0xf084,
+- "keep %d els_logo %d disc state %d auth state %d stop state %d\n",
+- fcport->keep_nport_handle,
+- fcport->send_els_logo, fcport->disc_state,
+- fcport->edif.auth_state, fcport->edif.app_stop);
++ "%s: se_sess %p / sess %p from port %8phC "
++ "loop_id %#04x s_id %06x logout %d "
++ "keep %d els_logo %d disc state %d auth state %d"
++ "stop state %d\n",
++ __func__, fcport->se_sess, fcport,
++ fcport->port_name, fcport->loop_id,
++ fcport->d_id.b24, fcport->logout_on_delete,
++ fcport->keep_nport_handle, fcport->send_els_logo,
++ fcport->disc_state, fcport->edif.auth_state,
++ fcport->edif.app_stop);
+
+ if (atomic_read(&vha->loop_state) == LOOP_DOWN)
+ break;
+- if (!(fcport->flags & FCF_FCSP_DEVICE))
+- continue;
+
+ fcport->edif.app_started = 1;
+- if (fcport->edif.app_stop ||
+- (fcport->disc_state != DSC_LOGIN_COMPLETE &&
+- fcport->disc_state != DSC_LOGIN_PEND &&
+- fcport->disc_state != DSC_DELETED)) {
+- /* no activity */
+- fcport->edif.app_stop = 0;
+-
+- ql_dbg(ql_dbg_edif, vha, 0x911e,
+- "%s wwpn %8phC calling qla_edif_reset_auth_wait\n",
+- __func__, fcport->port_name);
+- fcport->edif.app_sess_online = 1;
+- qla_edif_reset_auth_wait(fcport, DSC_LOGIN_PEND, 0);
+- }
++ fcport->login_retry = vha->hw->login_retry_count;
++
++ /* no activity */
++ fcport->edif.app_stop = 0;
++
++ ql_dbg(ql_dbg_edif, vha, 0x911e,
++ "%s wwpn %8phC calling qla_edif_reset_auth_wait\n",
++ __func__, fcport->port_name);
++ fcport->edif.app_sess_online = 0;
++ qlt_schedule_sess_for_deletion(fcport);
+ qla_edif_sa_ctl_init(vha, fcport);
+ }
+ }
+@@ -601,14 +544,14 @@ qla_edif_app_start(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ appreply.edif_enode_active = vha->pur_cinfo.enode_flags;
+ appreply.edif_edb_active = vha->e_dbell.db_flags;
+
+- bsg_job->reply_len = sizeof(struct fc_bsg_reply) +
+- sizeof(struct app_start_reply);
++ bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+
+ SET_DID_STATUS(bsg_reply->result, DID_OK);
+
+- sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+- bsg_job->reply_payload.sg_cnt, &appreply,
+- sizeof(struct app_start_reply));
++ bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
++ bsg_job->reply_payload.sg_cnt,
++ &appreply,
++ sizeof(struct app_start_reply));
+
+ ql_dbg(ql_dbg_edif, vha, 0x911d,
+ "%s app start completed with 0x%x\n",
+@@ -725,6 +668,11 @@ qla_edif_app_authok(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ bsg_job->request_payload.sg_cnt, &appplogiok,
+ sizeof(struct auth_complete_cmd));
+
++ /* silent unaligned access warning */
++ portid.b.domain = appplogiok.u.d_id.b.domain;
++ portid.b.area = appplogiok.u.d_id.b.area;
++ portid.b.al_pa = appplogiok.u.d_id.b.al_pa;
++
+ switch (appplogiok.type) {
+ case PL_TYPE_WWPN:
+ fcport = qla2x00_find_fcport_by_wwpn(vha,
+@@ -735,7 +683,7 @@ qla_edif_app_authok(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ __func__, appplogiok.u.wwpn);
+ break;
+ case PL_TYPE_DID:
+- fcport = qla2x00_find_fcport_by_pid(vha, &appplogiok.u.d_id);
++ fcport = qla2x00_find_fcport_by_pid(vha, &portid);
+ if (!fcport)
+ ql_dbg(ql_dbg_edif, vha, 0x911d,
+ "%s d_id lookup failed: %x\n", __func__,
+@@ -800,15 +748,15 @@ qla_edif_app_authok(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ ql_dbg(ql_dbg_edif, vha, 0x911e,
+ "%s AUTH complete - RESUME with prli for wwpn %8phC\n",
+ __func__, fcport->port_name);
+- qla_edif_reset_auth_wait(fcport, DSC_LOGIN_PEND, 1);
+ qla24xx_post_prli_work(vha, fcport);
+ }
+
+ errstate_exit:
+ bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+- sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+- bsg_job->reply_payload.sg_cnt, &appplogireply,
+- sizeof(struct app_plogi_reply));
++ bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
++ bsg_job->reply_payload.sg_cnt,
++ &appplogireply,
++ sizeof(struct app_plogi_reply));
+
+ return rval;
+ }
+@@ -834,6 +782,11 @@ qla_edif_app_authfail(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ bsg_job->request_payload.sg_cnt, &appplogifail,
+ sizeof(struct auth_complete_cmd));
+
++ /* silent unaligned access warning */
++ portid.b.domain = appplogifail.u.d_id.b.domain;
++ portid.b.area = appplogifail.u.d_id.b.area;
++ portid.b.al_pa = appplogifail.u.d_id.b.al_pa;
++
+ /*
+ * TODO: edif: app has failed this plogi. Inform driver to
+ * take any action (if any).
+@@ -845,7 +798,7 @@ qla_edif_app_authfail(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ SET_DID_STATUS(bsg_reply->result, DID_OK);
+ break;
+ case PL_TYPE_DID:
+- fcport = qla2x00_find_fcport_by_pid(vha, &appplogifail.u.d_id);
++ fcport = qla2x00_find_fcport_by_pid(vha, &portid);
+ if (!fcport)
+ ql_dbg(ql_dbg_edif, vha, 0x911d,
+ "%s d_id lookup failed: %x\n", __func__,
+@@ -873,7 +826,7 @@ qla_edif_app_authfail(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+
+ if (qla_ini_mode_enabled(fcport->vha)) {
+ fcport->send_els_logo = 1;
+- qla_edif_reset_auth_wait(fcport, DSC_LOGIN_PEND, 0);
++ qlt_schedule_sess_for_deletion(fcport);
+ }
+ }
+
+@@ -891,7 +844,7 @@ static int
+ qla_edif_app_getfcinfo(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ {
+ int32_t rval = 0;
+- int32_t num_cnt;
++ int32_t pcnt = 0;
+ struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+ struct app_pinfo_req app_req;
+ struct app_pinfo_reply *app_reply;
+@@ -903,16 +856,14 @@ qla_edif_app_getfcinfo(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ bsg_job->request_payload.sg_cnt, &app_req,
+ sizeof(struct app_pinfo_req));
+
+- num_cnt = app_req.num_ports; /* num of ports alloc'd by app */
+-
+ app_reply = kzalloc((sizeof(struct app_pinfo_reply) +
+- sizeof(struct app_pinfo) * num_cnt), GFP_KERNEL);
++ sizeof(struct app_pinfo) * app_req.num_ports), GFP_KERNEL);
++
+ if (!app_reply) {
+ SET_DID_STATUS(bsg_reply->result, DID_ERROR);
+ rval = -1;
+ } else {
+ struct fc_port *fcport = NULL, *tf;
+- uint32_t pcnt = 0;
+
+ list_for_each_entry_safe(fcport, tf, &vha->vp_fcports, list) {
+ if (!(fcport->flags & FCF_FCSP_DEVICE))
+@@ -924,7 +875,7 @@ qla_edif_app_getfcinfo(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ "APP request entry - portid=%06x.\n", tdid.b24);
+
+ /* Ran out of space */
+- if (pcnt > app_req.num_ports)
++ if (pcnt >= app_req.num_ports)
+ break;
+
+ if (tdid.b24 != 0 && tdid.b24 != fcport->d_id.b24)
+@@ -981,9 +932,11 @@ qla_edif_app_getfcinfo(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ SET_DID_STATUS(bsg_reply->result, DID_OK);
+ }
+
+- sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+- bsg_job->reply_payload.sg_cnt, app_reply,
+- sizeof(struct app_pinfo_reply) + sizeof(struct app_pinfo) * num_cnt);
++ bsg_job->reply_len = sizeof(struct fc_bsg_reply);
++ bsg_reply->reply_payload_rcv_len = sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
++ bsg_job->reply_payload.sg_cnt,
++ app_reply,
++ sizeof(struct app_pinfo_reply) + sizeof(struct app_pinfo) * pcnt);
+
+ kfree(app_reply);
+
+@@ -1000,10 +953,11 @@ qla_edif_app_getstats(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ {
+ int32_t rval = 0;
+ struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+- uint32_t ret_size, size;
++ uint32_t size;
+
+ struct app_sinfo_req app_req;
+ struct app_stats_reply *app_reply;
++ uint32_t pcnt = 0;
+
+ sg_copy_to_buffer(bsg_job->request_payload.sg_list,
+ bsg_job->request_payload.sg_cnt, &app_req,
+@@ -1019,18 +973,12 @@ qla_edif_app_getstats(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ size = sizeof(struct app_stats_reply) +
+ (sizeof(struct app_sinfo) * app_req.num_ports);
+
+- if (size > bsg_job->reply_payload.payload_len)
+- ret_size = bsg_job->reply_payload.payload_len;
+- else
+- ret_size = size;
+-
+ app_reply = kzalloc(size, GFP_KERNEL);
+ if (!app_reply) {
+ SET_DID_STATUS(bsg_reply->result, DID_ERROR);
+ rval = -1;
+ } else {
+ struct fc_port *fcport = NULL, *tf;
+- uint32_t pcnt = 0;
+
+ list_for_each_entry_safe(fcport, tf, &vha->vp_fcports, list) {
+ if (fcport->edif.enable) {
+@@ -1054,9 +1002,11 @@ qla_edif_app_getstats(scsi_qla_host_t *vha, struct bsg_job *bsg_job)
+ SET_DID_STATUS(bsg_reply->result, DID_OK);
+ }
+
++ bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+ bsg_reply->reply_payload_rcv_len =
+ sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+- bsg_job->reply_payload.sg_cnt, app_reply, ret_size);
++ bsg_job->reply_payload.sg_cnt, app_reply,
++ sizeof(struct app_stats_reply) + (sizeof(struct app_sinfo) * pcnt));
+
+ kfree(app_reply);
+
+@@ -1130,8 +1080,7 @@ qla_edif_app_mgmt(struct bsg_job *bsg_job)
+ __func__,
+ bsg_request->rqst_data.h_vendor.vendor_cmd[1]);
+ rval = EXT_STATUS_INVALID_PARAM;
+- bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+- SET_DID_STATUS(bsg_reply->result, DID_ERROR);
++ done = false;
+ break;
+ }
+
+@@ -1314,6 +1263,7 @@ qla24xx_sadb_update(struct bsg_job *bsg_job)
+ int result = 0;
+ struct qla_sa_update_frame sa_frame;
+ struct srb_iocb *iocb_cmd;
++ port_id_t portid;
+
+ ql_dbg(ql_dbg_edif + ql_dbg_verbose, vha, 0x911d,
+ "%s entered, vha: 0x%p\n", __func__, vha);
+@@ -1337,7 +1287,12 @@ qla24xx_sadb_update(struct bsg_job *bsg_job)
+ goto done;
+ }
+
+- fcport = qla2x00_find_fcport_by_pid(vha, &sa_frame.port_id);
++ /* silent unaligned access warning */
++ portid.b.domain = sa_frame.port_id.b.domain;
++ portid.b.area = sa_frame.port_id.b.area;
++ portid.b.al_pa = sa_frame.port_id.b.al_pa;
++
++ fcport = qla2x00_find_fcport_by_pid(vha, &portid);
+ if (fcport) {
+ found = 1;
+ if (sa_frame.flags == QLA_SA_UPDATE_FLAGS_TX_KEY)
+@@ -1651,6 +1606,40 @@ qla_enode_stop(scsi_qla_host_t *vha)
+ spin_unlock_irqrestore(&vha->pur_cinfo.pur_lock, flags);
+ }
+
++static void qla_enode_clear(scsi_qla_host_t *vha, port_id_t portid)
++{
++ unsigned long flags;
++ struct enode *e, *tmp;
++ struct purexevent *purex;
++ LIST_HEAD(enode_list);
++
++ if (vha->pur_cinfo.enode_flags != ENODE_ACTIVE) {
++ ql_dbg(ql_dbg_edif, vha, 0x09102,
++ "%s enode not active\n", __func__);
++ return;
++ }
++ spin_lock_irqsave(&vha->pur_cinfo.pur_lock, flags);
++ list_for_each_entry_safe(e, tmp, &vha->pur_cinfo.head, list) {
++ purex = &e->u.purexinfo;
++ if (purex->pur_info.pur_sid.b24 == portid.b24) {
++ ql_dbg(ql_dbg_edif, vha, 0x911d,
++ "%s free ELS sid=%06x. xchg %x, nb=%xh\n",
++ __func__, portid.b24,
++ purex->pur_info.pur_rx_xchg_address,
++ purex->pur_info.pur_bytes_rcvd);
++
++ list_del_init(&e->list);
++ list_add_tail(&e->list, &enode_list);
++ }
++ }
++ spin_unlock_irqrestore(&vha->pur_cinfo.pur_lock, flags);
++
++ list_for_each_entry_safe(e, tmp, &enode_list, list) {
++ list_del_init(&e->list);
++ qla_enode_free(vha, e);
++ }
++}
++
+ /*
+ * allocate enode struct and populate buffer
+ * returns: enode pointer with buffers
+@@ -1850,6 +1839,57 @@ qla_edb_node_free(scsi_qla_host_t *vha, struct edb_node *node)
+ node->ntype = N_UNDEF;
+ }
+
++static void qla_edb_clear(scsi_qla_host_t *vha, port_id_t portid)
++{
++ unsigned long flags;
++ struct edb_node *e, *tmp;
++ port_id_t sid;
++ LIST_HEAD(edb_list);
++
++ if (vha->e_dbell.db_flags != EDB_ACTIVE) {
++ /* doorbell list not enabled */
++ ql_dbg(ql_dbg_edif, vha, 0x09102,
++ "%s doorbell not enabled\n", __func__);
++ return;
++ }
++
++ /* grab lock so list doesn't move */
++ spin_lock_irqsave(&vha->e_dbell.db_lock, flags);
++ list_for_each_entry_safe(e, tmp, &vha->e_dbell.head, list) {
++ switch (e->ntype) {
++ case VND_CMD_AUTH_STATE_NEEDED:
++ case VND_CMD_AUTH_STATE_SESSION_SHUTDOWN:
++ sid = e->u.plogi_did;
++ break;
++ case VND_CMD_AUTH_STATE_ELS_RCVD:
++ sid = e->u.els_sid;
++ break;
++ case VND_CMD_AUTH_STATE_SAUPDATE_COMPL:
++ /* app wants to see this */
++ continue;
++ default:
++ ql_log(ql_log_warn, vha, 0x09102,
++ "%s unknown node type: %x\n", __func__, e->ntype);
++ sid.b24 = 0;
++ break;
++ }
++ if (sid.b24 == portid.b24) {
++ ql_dbg(ql_dbg_edif, vha, 0x910f,
++ "%s free doorbell event : node type = %x %p\n",
++ __func__, e->ntype, e);
++ list_del_init(&e->list);
++ list_add_tail(&e->list, &edb_list);
++ }
++ }
++ spin_unlock_irqrestore(&vha->e_dbell.db_lock, flags);
++
++ list_for_each_entry_safe(e, tmp, &edb_list, list) {
++ qla_edb_node_free(vha, e);
++ list_del_init(&e->list);
++ kfree(e);
++ }
++}
++
+ /* function called when app is stopping */
+
+ void
+@@ -2137,7 +2177,8 @@ edif_doorbell_show(struct device *dev, struct device_attribute *attr,
+
+ static void qla_noop_sp_done(srb_t *sp, int res)
+ {
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ /*
+@@ -2357,7 +2398,7 @@ void qla24xx_auth_els(scsi_qla_host_t *vha, void **pkt, struct rsp_que **rsp)
+ return;
+ }
+
+- if (totlen > MAX_PAYLOAD) {
++ if (totlen > ELS_MAX_PAYLOAD) {
+ ql_dbg(ql_dbg_edif, vha, 0x0910d,
+ "%s WARNING: verbose ELS frame received (totlen=%x)\n",
+ __func__, totlen);
+@@ -2436,7 +2477,7 @@ void qla24xx_auth_els(scsi_qla_host_t *vha, void **pkt, struct rsp_que **rsp)
+ ql_dbg(ql_dbg_edif, host, 0x0910c,
+ "%s COMPLETE purex->pur_info.pur_bytes_rcvd =%xh s:%06x -> d:%06x xchg=%xh\n",
+ __func__, purex->pur_info.pur_bytes_rcvd, purex->pur_info.pur_sid.b24,
+- purex->pur_info.pur_did.b24, p->rx_xchg_addr);
++ purex->pur_info.pur_did.b24, purex->pur_info.pur_rx_xchg_address);
+
+ qla_edb_eventcreate(host, VND_CMD_AUTH_STATE_ELS_RCVD, sid, 0, NULL);
+ }
+@@ -3459,3 +3500,12 @@ void qla_edif_sess_down(struct scsi_qla_host *vha, struct fc_port *sess)
+ qla2x00_post_aen_work(vha, FCH_EVT_PORT_OFFLINE, sess->d_id.b24);
+ }
+ }
++
++void qla_edif_clear_appdata(struct scsi_qla_host *vha, struct fc_port *fcport)
++{
++ if (!(fcport->flags & FCF_FCSP_DEVICE))
++ return;
++
++ qla_edb_clear(vha, fcport->d_id);
++ qla_enode_clear(vha, fcport->d_id);
++}
+diff --git a/drivers/scsi/qla2xxx/qla_edif.h b/drivers/scsi/qla2xxx/qla_edif.h
+index 9e8f28d0caa1b..45cf87e337780 100644
+--- a/drivers/scsi/qla2xxx/qla_edif.h
++++ b/drivers/scsi/qla2xxx/qla_edif.h
+@@ -93,7 +93,6 @@ struct sa_update_28xx {
+ };
+
+ #define NUM_ENTRIES 256
+-#define MAX_PAYLOAD 1024
+ #define PUR_GET 1
+
+ struct dinfo {
+@@ -128,6 +127,8 @@ struct enode {
+ } u;
+ };
+
++#define RX_ELS_SIZE (roundup(sizeof(struct enode) + ELS_MAX_PAYLOAD, SMP_CACHE_BYTES))
++
+ #define EDIF_SESSION_DOWN(_s) \
+ (qla_ini_mode_enabled(_s->vha) && (_s->disc_state == DSC_DELETE_PEND || \
+ _s->disc_state == DSC_DELETED || \
+diff --git a/drivers/scsi/qla2xxx/qla_edif_bsg.h b/drivers/scsi/qla2xxx/qla_edif_bsg.h
+index 58b718d35d19a..53026d82ebffe 100644
+--- a/drivers/scsi/qla2xxx/qla_edif_bsg.h
++++ b/drivers/scsi/qla2xxx/qla_edif_bsg.h
+@@ -8,7 +8,7 @@
+ #define __QLA_EDIF_BSG_H
+
+ /* BSG Vendor specific commands */
+-#define ELS_MAX_PAYLOAD 1024
++#define ELS_MAX_PAYLOAD 2112
+ #ifndef WWN_SIZE
+ #define WWN_SIZE 8
+ #endif
+diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
+index 1c3f055d41b8e..83912787fa2e0 100644
+--- a/drivers/scsi/qla2xxx/qla_gbl.h
++++ b/drivers/scsi/qla2xxx/qla_gbl.h
+@@ -142,6 +142,8 @@ void qlt_chk_edif_rx_sa_delete_pending(scsi_qla_host_t *vha, fc_port_t *fcport,
+ void qla2x00_release_all_sadb(struct scsi_qla_host *vha, struct fc_port *fcport);
+ int qla_edif_process_els(scsi_qla_host_t *vha, struct bsg_job *bsgjob);
+ void qla_edif_sess_down(struct scsi_qla_host *vha, struct fc_port *sess);
++void qla_edif_clear_appdata(struct scsi_qla_host *vha,
++ struct fc_port *fcport);
+ const char *sc_to_str(uint16_t cmd);
+
+ /*
+@@ -171,7 +173,6 @@ extern int ql2xasynctmfenable;
+ extern int ql2xgffidenable;
+ extern int ql2xenabledif;
+ extern int ql2xenablehba_err_chk;
+-extern int ql2xtargetreset;
+ extern int ql2xdontresethba;
+ extern uint64_t ql2xmaxlun;
+ extern int ql2xmdcapmask;
+@@ -315,7 +316,8 @@ extern int qla2x00_start_sp(srb_t *);
+ extern int qla24xx_dif_start_scsi(srb_t *);
+ extern int qla2x00_start_bidir(srb_t *, struct scsi_qla_host *, uint32_t);
+ extern int qla2xxx_dif_start_scsi_mq(srb_t *);
+-extern void qla2x00_init_timer(srb_t *sp, unsigned long tmo);
++extern void qla2x00_init_async_sp(srb_t *sp, unsigned long tmo,
++ void (*done)(struct srb *, int));
+ extern unsigned long qla2x00_get_async_timeout(struct scsi_qla_host *);
+
+ extern void *qla2x00_alloc_iocbs(struct scsi_qla_host *, srb_t *);
+@@ -331,6 +333,7 @@ extern int qla24xx_get_one_block_sg(uint32_t, struct qla2_sgx *, uint32_t *);
+ extern int qla24xx_configure_prot_mode(srb_t *, uint16_t *);
+ extern int qla24xx_issue_sa_replace_iocb(scsi_qla_host_t *vha,
+ struct qla_work_evt *e);
++void qla2x00_sp_release(struct kref *kref);
+
+ /*
+ * Global Function Prototypes in qla_mbx.c source file.
+@@ -816,7 +819,6 @@ extern void qlafx00_abort_iocb(srb_t *, struct abort_iocb_entry_fx00 *);
+ extern void qlafx00_fxdisc_iocb(srb_t *, struct fxdisc_entry_fx00 *);
+ extern void qlafx00_timer_routine(scsi_qla_host_t *);
+ extern int qlafx00_rescan_isp(scsi_qla_host_t *);
+-extern int qlafx00_loop_reset(scsi_qla_host_t *vha);
+
+ /* qla82xx related functions */
+
+diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
+index ebc8fdb0b43d3..da7080c4bd008 100644
+--- a/drivers/scsi/qla2xxx/qla_gs.c
++++ b/drivers/scsi/qla2xxx/qla_gs.c
+@@ -529,7 +529,6 @@ static void qla2x00_async_sns_sp_done(srb_t *sp, int rc)
+ if (!e)
+ goto err2;
+
+- del_timer(&sp->u.iocb_cmd.timer);
+ e->u.iosb.sp = sp;
+ qla2x00_post_work(vha, e);
+ return;
+@@ -556,8 +555,8 @@ err2:
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+
+- sp->free(sp);
+-
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return;
+ }
+
+@@ -592,13 +591,15 @@ static int qla_async_rftid(scsi_qla_host_t *vha, port_id_t *d_id)
+ if (!vha->flags.online)
+ goto done;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "rft_id";
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_sns_sp_done);
+
+ sp->u.iocb_cmd.u.ctarg.req = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.req_dma,
+@@ -638,8 +639,6 @@ static int qla_async_rftid(scsi_qla_host_t *vha, port_id_t *d_id)
+ sp->u.iocb_cmd.u.ctarg.req_size = RFT_ID_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.rsp_size = RFT_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla2x00_async_sns_sp_done;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - hdl=%x portid %06x.\n",
+@@ -653,7 +652,8 @@ static int qla_async_rftid(scsi_qla_host_t *vha, port_id_t *d_id)
+ }
+ return rval;
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -676,8 +676,7 @@ qla2x00_rff_id(scsi_qla_host_t *vha, u8 type)
+ return (QLA_SUCCESS);
+ }
+
+- return qla_async_rffid(vha, &vha->d_id, qlt_rff_id(vha),
+- FC4_TYPE_FCP_SCSI);
++ return qla_async_rffid(vha, &vha->d_id, qlt_rff_id(vha), type);
+ }
+
+ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id,
+@@ -688,13 +687,15 @@ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id,
+ srb_t *sp;
+ struct ct_sns_pkt *ct_sns;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "rff_id";
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_sns_sp_done);
+
+ sp->u.iocb_cmd.u.ctarg.req = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.req_dma,
+@@ -727,13 +728,11 @@ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id,
+ /* Prepare CT arguments -- port_id, FC-4 feature, FC-4 type */
+ ct_req->req.rff_id.port_id = port_id_to_be_id(*d_id);
+ ct_req->req.rff_id.fc4_feature = fc4feature;
+- ct_req->req.rff_id.fc4_type = fc4type; /* SCSI - FCP */
++ ct_req->req.rff_id.fc4_type = fc4type; /* SCSI-FCP or FC-NVMe */
+
+ sp->u.iocb_cmd.u.ctarg.req_size = RFF_ID_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.rsp_size = RFF_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla2x00_async_sns_sp_done;
+
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - hdl=%x portid %06x feature %x type %x.\n",
+@@ -749,7 +748,8 @@ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id,
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -779,13 +779,15 @@ static int qla_async_rnnid(scsi_qla_host_t *vha, port_id_t *d_id,
+ srb_t *sp;
+ struct ct_sns_pkt *ct_sns;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "rnid";
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_sns_sp_done);
+
+ sp->u.iocb_cmd.u.ctarg.req = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.req_dma,
+@@ -823,9 +825,6 @@ static int qla_async_rnnid(scsi_qla_host_t *vha, port_id_t *d_id,
+ sp->u.iocb_cmd.u.ctarg.rsp_size = RNN_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla2x00_async_sns_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - hdl=%x portid %06x\n",
+ sp->name, sp->handle, d_id->b24);
+@@ -840,7 +839,8 @@ static int qla_async_rnnid(scsi_qla_host_t *vha, port_id_t *d_id,
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -886,13 +886,15 @@ static int qla_async_rsnn_nn(scsi_qla_host_t *vha)
+ srb_t *sp;
+ struct ct_sns_pkt *ct_sns;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_CT_PTHRU_CMD;
+ sp->name = "rsnn_nn";
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_sns_sp_done);
+
+ sp->u.iocb_cmd.u.ctarg.req = dma_alloc_coherent(&vha->hw->pdev->dev,
+ sizeof(struct ct_sns_pkt), &sp->u.iocb_cmd.u.ctarg.req_dma,
+@@ -936,9 +938,6 @@ static int qla_async_rsnn_nn(scsi_qla_host_t *vha)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = RSNN_NN_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla2x00_async_sns_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - hdl=%x.\n",
+ sp->name, sp->handle);
+@@ -953,7 +952,8 @@ static int qla_async_rsnn_nn(scsi_qla_host_t *vha)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -2892,7 +2892,8 @@ static void qla24xx_async_gpsc_sp_done(srb_t *sp, int res)
+ qla24xx_handle_gpsc_event(vha, &ea);
+
+ done:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+@@ -2904,6 +2905,7 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+ if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
+ return rval;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+@@ -2912,8 +2914,8 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->name = "gpsc";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla24xx_async_gpsc_sp_done);
+
+ /* CT_IU preamble */
+ ct_req = qla24xx_prep_ct_fm_req(fcport->ct_desc.ct_sns, GPSC_CMD,
+@@ -2931,9 +2933,6 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GPSC_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = vha->mgmt_svr_loop_id;
+
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla24xx_async_gpsc_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0x205e,
+ "Async-%s %8phC hdl=%x loopid=%x portid=%02x%02x%02x.\n",
+ sp->name, fcport->port_name, sp->handle,
+@@ -2946,7 +2945,8 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -2995,7 +2995,8 @@ void qla24xx_sp_unmap(scsi_qla_host_t *vha, srb_t *sp)
+ break;
+ }
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ void qla24xx_handle_gpnid_event(scsi_qla_host_t *vha, struct event_arg *ea)
+@@ -3134,13 +3135,15 @@ static void qla2x00_async_gpnid_sp_done(srb_t *sp, int res)
+ if (res) {
+ if (res == QLA_FUNCTION_TIMEOUT) {
+ qla24xx_post_gpnid_work(sp->vha, &ea.id);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return;
+ }
+ } else if (sp->gen1) {
+ /* There was another RSCN for this Nport ID */
+ qla24xx_post_gpnid_work(sp->vha, &ea.id);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return;
+ }
+
+@@ -3161,7 +3164,8 @@ static void qla2x00_async_gpnid_sp_done(srb_t *sp, int res)
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return;
+ }
+
+@@ -3181,6 +3185,7 @@ int qla24xx_async_gpnid(scsi_qla_host_t *vha, port_id_t *id)
+ if (!vha->flags.online)
+ goto done;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+@@ -3189,14 +3194,16 @@ int qla24xx_async_gpnid(scsi_qla_host_t *vha, port_id_t *id)
+ sp->name = "gpnid";
+ sp->u.iocb_cmd.u.ctarg.id = *id;
+ sp->gen1 = 0;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_gpnid_sp_done);
+
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+ list_for_each_entry(tsp, &vha->gpnid_list, elem) {
+ if (tsp->u.iocb_cmd.u.ctarg.id.b24 == id->b24) {
+ tsp->gen1++;
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ goto done;
+ }
+ }
+@@ -3237,9 +3244,6 @@ int qla24xx_async_gpnid(scsi_qla_host_t *vha, port_id_t *id)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GPN_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- sp->done = qla2x00_async_gpnid_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0x2067,
+ "Async-%s hdl=%x ID %3phC.\n", sp->name,
+ sp->handle, &ct_req->req.port_id.port_id);
+@@ -3269,8 +3273,8 @@ done_free_sp:
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+-
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -3325,7 +3329,8 @@ void qla24xx_async_gffid_sp_done(srb_t *sp, int res)
+ ea.rc = res;
+
+ qla24xx_handle_gffid_event(vha, &ea);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ /* Get FC4 Feature with Nport ID. */
+@@ -3338,6 +3343,7 @@ int qla24xx_async_gffid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
+ return rval;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ return rval;
+@@ -3347,9 +3353,8 @@ int qla24xx_async_gffid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->name = "gffid";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla24xx_async_gffid_sp_done);
+
+ /* CT_IU preamble */
+ ct_req = qla2x00_prep_ct_req(fcport->ct_desc.ct_sns, GFF_ID_CMD,
+@@ -3367,8 +3372,6 @@ int qla24xx_async_gffid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GFF_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->done = qla24xx_async_gffid_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0x2132,
+ "Async-%s hdl=%x %8phC.\n", sp->name,
+ sp->handle, fcport->port_name);
+@@ -3379,7 +3382,8 @@ int qla24xx_async_gffid(scsi_qla_host_t *vha, fc_port_t *fcport)
+
+ return rval;
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+ }
+@@ -3766,7 +3770,6 @@ static void qla2x00_async_gpnft_gnnft_sp_done(srb_t *sp, int res)
+ "Async done-%s res %x FC4Type %x\n",
+ sp->name, res, sp->gen2);
+
+- del_timer(&sp->u.iocb_cmd.timer);
+ sp->rc = res;
+ if (res) {
+ unsigned long flags;
+@@ -3891,9 +3894,8 @@ static int qla24xx_async_gnnft(scsi_qla_host_t *vha, struct srb *sp,
+ sp->name = "gnnft";
+ sp->gen1 = vha->hw->base_qpair->chip_reset;
+ sp->gen2 = fc4_type;
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_gpnft_gnnft_sp_done);
+
+ memset(sp->u.iocb_cmd.u.ctarg.rsp, 0, sp->u.iocb_cmd.u.ctarg.rsp_size);
+ memset(sp->u.iocb_cmd.u.ctarg.req, 0, sp->u.iocb_cmd.u.ctarg.req_size);
+@@ -3909,8 +3911,6 @@ static int qla24xx_async_gnnft(scsi_qla_host_t *vha, struct srb *sp,
+ sp->u.iocb_cmd.u.ctarg.req_size = GNN_FT_REQ_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->done = qla2x00_async_gpnft_gnnft_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s hdl=%x FC4Type %x.\n", sp->name,
+ sp->handle, ct_req->req.gpn_ft.port_type);
+@@ -3937,8 +3937,8 @@ done_free_sp:
+ sp->u.iocb_cmd.u.ctarg.rsp_dma);
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+-
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+
+ spin_lock_irqsave(&vha->work_lock, flags);
+ vha->scan.scan_flags &= ~SF_SCANNING;
+@@ -3990,9 +3990,12 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp)
+ ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0xffff,
+ "%s: Performing FCP Scan\n", __func__);
+
+- if (sp)
+- sp->free(sp); /* should not happen */
++ if (sp) {
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
++ }
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp) {
+ spin_lock_irqsave(&vha->work_lock, flags);
+@@ -4037,6 +4040,7 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp)
+ sp->u.iocb_cmd.u.ctarg.req,
+ sp->u.iocb_cmd.u.ctarg.req_dma);
+ sp->u.iocb_cmd.u.ctarg.req = NULL;
++ /* ref: INIT */
+ qla2x00_rel_sp(sp);
+ return rval;
+ }
+@@ -4056,9 +4060,8 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp)
+ sp->name = "gpnft";
+ sp->gen1 = vha->hw->base_qpair->chip_reset;
+ sp->gen2 = fc4_type;
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_gpnft_gnnft_sp_done);
+
+ rspsz = sp->u.iocb_cmd.u.ctarg.rsp_size;
+ memset(sp->u.iocb_cmd.u.ctarg.rsp, 0, sp->u.iocb_cmd.u.ctarg.rsp_size);
+@@ -4073,8 +4076,6 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp)
+
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->done = qla2x00_async_gpnft_gnnft_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s hdl=%x FC4Type %x.\n", sp->name,
+ sp->handle, ct_req->req.gpn_ft.port_type);
+@@ -4102,7 +4103,8 @@ done_free_sp:
+ sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+ }
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+
+ spin_lock_irqsave(&vha->work_lock, flags);
+ vha->scan.scan_flags &= ~SF_SCANNING;
+@@ -4166,7 +4168,8 @@ static void qla2x00_async_gnnid_sp_done(srb_t *sp, int res)
+
+ qla24xx_handle_gnnid_event(vha, &ea);
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+@@ -4179,6 +4182,7 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ return rval;
+
+ qla2x00_set_fcport_disc_state(fcport, DSC_GNN_ID);
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
+ if (!sp)
+ goto done;
+@@ -4188,9 +4192,8 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->name = "gnnid";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_gnnid_sp_done);
+
+ /* CT_IU preamble */
+ ct_req = qla2x00_prep_ct_req(fcport->ct_desc.ct_sns, GNN_ID_CMD,
+@@ -4209,8 +4212,6 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GNN_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->done = qla2x00_async_gnnid_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - %8phC hdl=%x loopid=%x portid %06x.\n",
+ sp->name, fcport->port_name,
+@@ -4222,7 +4223,8 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ done:
+ return rval;
+@@ -4296,7 +4298,8 @@ static void qla2x00_async_gfpnid_sp_done(srb_t *sp, int res)
+
+ qla24xx_handle_gfpnid_event(vha, &ea);
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+@@ -4308,6 +4311,7 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
+ return rval;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
+ if (!sp)
+ goto done;
+@@ -4316,9 +4320,8 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->name = "gfpnid";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_gfpnid_sp_done);
+
+ /* CT_IU preamble */
+ ct_req = qla2x00_prep_ct_req(fcport->ct_desc.ct_sns, GFPN_ID_CMD,
+@@ -4337,8 +4340,6 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ sp->u.iocb_cmd.u.ctarg.rsp_size = GFPN_ID_RSP_SIZE;
+ sp->u.iocb_cmd.u.ctarg.nport_handle = NPH_SNS;
+
+- sp->done = qla2x00_async_gfpnid_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0xffff,
+ "Async-%s - %8phC hdl=%x loopid=%x portid %06x.\n",
+ sp->name, fcport->port_name,
+@@ -4351,7 +4352,8 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
+index 5fc7697f0af4c..af8df5a800c61 100644
+--- a/drivers/scsi/qla2xxx/qla_init.c
++++ b/drivers/scsi/qla2xxx/qla_init.c
+@@ -51,6 +51,9 @@ qla2x00_sp_timeout(struct timer_list *t)
+ WARN_ON(irqs_disabled());
+ iocb = &sp->u.iocb_cmd;
+ iocb->timeout(sp);
++
++ /* ref: TMR */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ void qla2x00_sp_free(srb_t *sp)
+@@ -125,8 +128,13 @@ static void qla24xx_abort_iocb_timeout(void *data)
+ }
+ spin_unlock_irqrestore(qpair->qp_lock_ptr, flags);
+
+- if (sp->cmd_sp)
++ if (sp->cmd_sp) {
++ /*
++ * This done function should take care of
++ * original command ref: INIT
++ */
+ sp->cmd_sp->done(sp->cmd_sp, QLA_OS_TIMER_EXPIRED);
++ }
+
+ abt->u.abt.comp_status = cpu_to_le16(CS_TIMEOUT);
+ sp->done(sp, QLA_OS_TIMER_EXPIRED);
+@@ -140,11 +148,11 @@ static void qla24xx_abort_sp_done(srb_t *sp, int res)
+ if (orig_sp)
+ qla_wait_nvme_release_cmd_kref(orig_sp);
+
+- del_timer(&sp->u.iocb_cmd.timer);
+ if (sp->flags & SRB_WAKEUP_ON_COMP)
+ complete(&abt->u.abt.comp);
+ else
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
+@@ -154,6 +162,7 @@ int qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
+ srb_t *sp;
+ int rval = QLA_FUNCTION_FAILED;
+
++ /* ref: INIT for ABTS command */
+ sp = qla2xxx_get_qpair_sp(cmd_sp->vha, cmd_sp->qpair, cmd_sp->fcport,
+ GFP_ATOMIC);
+ if (!sp)
+@@ -167,23 +176,22 @@ int qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
+ if (wait)
+ sp->flags = SRB_WAKEUP_ON_COMP;
+
+- abt_iocb->timeout = qla24xx_abort_iocb_timeout;
+ init_completion(&abt_iocb->u.abt.comp);
+ /* FW can send 2 x ABTS's timeout/20s */
+- qla2x00_init_timer(sp, 42);
++ qla2x00_init_async_sp(sp, 42, qla24xx_abort_sp_done);
++ sp->u.iocb_cmd.timeout = qla24xx_abort_iocb_timeout;
+
+ abt_iocb->u.abt.cmd_hndl = cmd_sp->handle;
+ abt_iocb->u.abt.req_que_no = cpu_to_le16(cmd_sp->qpair->req->id);
+
+- sp->done = qla24xx_abort_sp_done;
+-
+ ql_dbg(ql_dbg_async, vha, 0x507c,
+ "Abort command issued - hdl=%x, type=%x\n", cmd_sp->handle,
+ cmd_sp->type);
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS) {
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return rval;
+ }
+
+@@ -191,7 +199,8 @@ int qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
+ wait_for_completion(&abt_iocb->u.abt.comp);
+ rval = abt_iocb->u.abt.comp_status == CS_COMPLETE ?
+ QLA_SUCCESS : QLA_ERR_FROM_FW;
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ return rval;
+@@ -286,10 +295,13 @@ static void qla2x00_async_login_sp_done(srb_t *sp, int res)
+ ea.iop[0] = lio->u.logio.iop[0];
+ ea.iop[1] = lio->u.logio.iop[1];
+ ea.sp = sp;
++ if (res)
++ ea.data[0] = MBS_COMMAND_ERROR;
+ qla24xx_handle_plogi_done_event(vha, &ea);
+ }
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int
+@@ -308,6 +320,7 @@ qla2x00_async_login(struct scsi_qla_host *vha, fc_port_t *fcport,
+ return rval;
+ }
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+@@ -320,12 +333,10 @@ qla2x00_async_login(struct scsi_qla_host *vha, fc_port_t *fcport,
+ sp->name = "login";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_login_sp_done);
+
+ lio = &sp->u.iocb_cmd;
+- lio->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+-
+- sp->done = qla2x00_async_login_sp_done;
+ if (N2N_TOPO(fcport->vha->hw) && fcport_is_bigger(fcport)) {
+ lio->u.logio.flags |= SRB_LOGIN_PRLI_ONLY;
+ } else {
+@@ -359,7 +370,8 @@ qla2x00_async_login(struct scsi_qla_host *vha, fc_port_t *fcport,
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ done:
+ fcport->flags &= ~FCF_ASYNC_ACTIVE;
+@@ -371,29 +383,26 @@ static void qla2x00_async_logout_sp_done(srb_t *sp, int res)
+ sp->fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+ sp->fcport->login_gen++;
+ qlt_logo_completion_handler(sp->fcport, sp->u.iocb_cmd.u.logio.data[0]);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int
+ qla2x00_async_logout(struct scsi_qla_host *vha, fc_port_t *fcport)
+ {
+ srb_t *sp;
+- struct srb_iocb *lio;
+ int rval = QLA_FUNCTION_FAILED;
+
+ fcport->flags |= FCF_ASYNC_SENT;
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_LOGOUT_CMD;
+ sp->name = "logout";
+-
+- lio = &sp->u.iocb_cmd;
+- lio->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+-
+- sp->done = qla2x00_async_logout_sp_done;
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_logout_sp_done),
+
+ ql_dbg(ql_dbg_disc, vha, 0x2070,
+ "Async-logout - hdl=%x loop-id=%x portid=%02x%02x%02x %8phC explicit %d.\n",
+@@ -407,7 +416,8 @@ qla2x00_async_logout(struct scsi_qla_host *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+ return rval;
+@@ -433,29 +443,26 @@ static void qla2x00_async_prlo_sp_done(srb_t *sp, int res)
+ if (!test_bit(UNLOADING, &vha->dpc_flags))
+ qla2x00_post_async_prlo_done_work(sp->fcport->vha, sp->fcport,
+ lio->u.logio.data);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int
+ qla2x00_async_prlo(struct scsi_qla_host *vha, fc_port_t *fcport)
+ {
+ srb_t *sp;
+- struct srb_iocb *lio;
+ int rval;
+
+ rval = QLA_FUNCTION_FAILED;
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_PRLO_CMD;
+ sp->name = "prlo";
+-
+- lio = &sp->u.iocb_cmd;
+- lio->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+-
+- sp->done = qla2x00_async_prlo_sp_done;
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_prlo_sp_done);
+
+ ql_dbg(ql_dbg_disc, vha, 0x2070,
+ "Async-prlo - hdl=%x loop-id=%x portid=%02x%02x%02x.\n",
+@@ -469,7 +476,8 @@ qla2x00_async_prlo(struct scsi_qla_host *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ fcport->flags &= ~FCF_ASYNC_ACTIVE;
+ return rval;
+@@ -552,10 +560,12 @@ static void qla2x00_async_adisc_sp_done(srb_t *sp, int res)
+ ea.iop[1] = lio->u.logio.iop[1];
+ ea.fcport = sp->fcport;
+ ea.sp = sp;
++ if (res)
++ ea.data[0] = MBS_COMMAND_ERROR;
+
+ qla24xx_handle_adisc_event(vha, &ea);
+-
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int
+@@ -566,26 +576,34 @@ qla2x00_async_adisc(struct scsi_qla_host *vha, fc_port_t *fcport,
+ struct srb_iocb *lio;
+ int rval = QLA_FUNCTION_FAILED;
+
++ if (IS_SESSION_DELETED(fcport)) {
++ ql_log(ql_log_warn, vha, 0xffff,
++ "%s: %8phC is being delete - not sending command.\n",
++ __func__, fcport->port_name);
++ fcport->flags &= ~FCF_ASYNC_ACTIVE;
++ return rval;
++ }
++
+ if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
+ return rval;
+
+ fcport->flags |= FCF_ASYNC_SENT;
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_ADISC_CMD;
+ sp->name = "adisc";
+-
+- lio = &sp->u.iocb_cmd;
+- lio->timeout = qla2x00_async_iocb_timeout;
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_adisc_sp_done);
+
+- sp->done = qla2x00_async_adisc_sp_done;
+- if (data[1] & QLA_LOGIO_LOGIN_RETRIED)
++ if (data[1] & QLA_LOGIO_LOGIN_RETRIED) {
++ lio = &sp->u.iocb_cmd;
+ lio->u.logio.flags |= SRB_LOGIN_RETRIED;
++ }
+
+ ql_dbg(ql_dbg_disc, vha, 0x206f,
+ "Async-adisc - hdl=%x loopid=%x portid=%06x %8phC.\n",
+@@ -598,7 +616,8 @@ qla2x00_async_adisc(struct scsi_qla_host *vha, fc_port_t *fcport,
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+ qla2x00_post_async_adisc_work(vha, fcport, data);
+@@ -964,6 +983,9 @@ static void qla24xx_handle_gnl_done_event(scsi_qla_host_t *vha,
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ }
+ break;
++ case ISP_CFG_NL:
++ qla24xx_fcport_handle_login(vha, fcport);
++ break;
+ default:
+ break;
+ }
+@@ -987,8 +1009,6 @@ static void qla24xx_async_gnl_sp_done(srb_t *sp, int res)
+ sp->name, res, sp->u.iocb_cmd.u.mbx.in_mb[1],
+ sp->u.iocb_cmd.u.mbx.in_mb[2]);
+
+- if (res == QLA_FUNCTION_TIMEOUT)
+- return;
+
+ sp->fcport->flags &= ~(FCF_ASYNC_SENT|FCF_ASYNC_ACTIVE);
+ memset(&ea, 0, sizeof(ea));
+@@ -1026,8 +1046,8 @@ static void qla24xx_async_gnl_sp_done(srb_t *sp, int res)
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+ list_for_each_entry_safe(fcport, tf, &h, gnl_entry) {
+- list_del_init(&fcport->gnl_entry);
+ spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
++ list_del_init(&fcport->gnl_entry);
+ fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+ ea.fcport = fcport;
+@@ -1081,13 +1101,13 @@ static void qla24xx_async_gnl_sp_done(srb_t *sp, int res)
+ }
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+ {
+ srb_t *sp;
+- struct srb_iocb *mbx;
+ int rval = QLA_FUNCTION_FAILED;
+ unsigned long flags;
+ u16 *mb;
+@@ -1112,6 +1132,7 @@ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+ vha->gnl.sent = 1;
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+@@ -1120,10 +1141,8 @@ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+ sp->name = "gnlist";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- mbx = &sp->u.iocb_cmd;
+- mbx->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha)+2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla24xx_async_gnl_sp_done);
+
+ mb = sp->u.iocb_cmd.u.mbx.out_mb;
+ mb[0] = MBC_PORT_NODE_NAME_LIST;
+@@ -1135,8 +1154,6 @@ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+ mb[8] = vha->gnl.size;
+ mb[9] = vha->vp_idx;
+
+- sp->done = qla24xx_async_gnl_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0x20da,
+ "Async-%s - OUT WWPN %8phC hndl %x\n",
+ sp->name, fcport->port_name, sp->handle);
+@@ -1148,7 +1165,8 @@ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ fcport->flags &= ~(FCF_ASYNC_ACTIVE | FCF_ASYNC_SENT);
+ return rval;
+@@ -1194,7 +1212,7 @@ done:
+ dma_pool_free(ha->s_dma_pool, sp->u.iocb_cmd.u.mbx.in,
+ sp->u.iocb_cmd.u.mbx.in_dma);
+
+- sp->free(sp);
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_post_prli_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+@@ -1235,11 +1253,13 @@ static void qla2x00_async_prli_sp_done(srb_t *sp, int res)
+ ea.sp = sp;
+ if (res == QLA_OS_TIMER_EXPIRED)
+ ea.data[0] = QLA_OS_TIMER_EXPIRED;
++ else if (res)
++ ea.data[0] = MBS_COMMAND_ERROR;
+
+ qla24xx_handle_prli_done_event(vha, &ea);
+ }
+
+- sp->free(sp);
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int
+@@ -1272,12 +1292,10 @@ qla24xx_async_prli(struct scsi_qla_host *vha, fc_port_t *fcport)
+
+ sp->type = SRB_PRLI_CMD;
+ sp->name = "prli";
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_prli_sp_done);
+
+ lio = &sp->u.iocb_cmd;
+- lio->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+-
+- sp->done = qla2x00_async_prli_sp_done;
+ lio->u.logio.flags = 0;
+
+ if (NVME_TARGET(vha->hw, fcport))
+@@ -1299,7 +1317,8 @@ qla24xx_async_prli(struct scsi_qla_host *vha, fc_port_t *fcport)
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+ }
+@@ -1328,14 +1347,21 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
+ struct port_database_24xx *pd;
+ struct qla_hw_data *ha = vha->hw;
+
+- if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT) ||
+- fcport->loop_id == FC_NO_LOOP_ID) {
++ if (IS_SESSION_DELETED(fcport)) {
+ ql_log(ql_log_warn, vha, 0xffff,
+- "%s: %8phC - not sending command.\n",
+- __func__, fcport->port_name);
++ "%s: %8phC is being delete - not sending command.\n",
++ __func__, fcport->port_name);
++ fcport->flags &= ~FCF_ASYNC_ACTIVE;
+ return rval;
+ }
+
++ if (!vha->flags.online || fcport->flags & FCF_ASYNC_SENT) {
++ ql_log(ql_log_warn, vha, 0xffff,
++ "%s: %8phC online %d flags %x - not sending command.\n",
++ __func__, fcport->port_name, vha->flags.online, fcport->flags);
++ goto done;
++ }
++
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+@@ -1347,10 +1373,8 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
+ sp->name = "gpdb";
+ sp->gen1 = fcport->rscn_gen;
+ sp->gen2 = fcport->login_gen;
+-
+- mbx = &sp->u.iocb_cmd;
+- mbx->timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla24xx_async_gpdb_sp_done);
+
+ pd = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &pd_dma);
+ if (pd == NULL) {
+@@ -1369,11 +1393,10 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
+ mb[9] = vha->vp_idx;
+ mb[10] = opt;
+
+- mbx->u.mbx.in = pd;
++ mbx = &sp->u.iocb_cmd;
++ mbx->u.mbx.in = (void *)pd;
+ mbx->u.mbx.in_dma = pd_dma;
+
+- sp->done = qla24xx_async_gpdb_sp_done;
+-
+ ql_dbg(ql_dbg_disc, vha, 0x20dc,
+ "Async-%s %8phC hndl %x opt %x\n",
+ sp->name, fcport->port_name, sp->handle, opt);
+@@ -1387,7 +1410,7 @@ done_free_sp:
+ if (pd)
+ dma_pool_free(ha->s_dma_pool, pd, pd_dma);
+
+- sp->free(sp);
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ done:
+ fcport->flags &= ~FCF_ASYNC_ACTIVE;
+@@ -1559,6 +1582,11 @@ static void qla_chk_n2n_b4_login(struct scsi_qla_host *vha, fc_port_t *fcport)
+ u8 login = 0;
+ int rc;
+
++ ql_dbg(ql_dbg_disc, vha, 0x307b,
++ "%s %8phC DS %d LS %d lid %d retries=%d\n",
++ __func__, fcport->port_name, fcport->disc_state,
++ fcport->fw_login_state, fcport->loop_id, fcport->login_retry);
++
+ if (qla_tgt_mode_enabled(vha))
+ return;
+
+@@ -1617,7 +1645,8 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
+ fcport->login_gen, fcport->loop_id, fcport->scan_state,
+ fcport->fc4_type);
+
+- if (fcport->scan_state != QLA_FCPORT_FOUND)
++ if (fcport->scan_state != QLA_FCPORT_FOUND ||
++ fcport->disc_state == DSC_DELETE_PEND)
+ return 0;
+
+ if ((fcport->loop_id != FC_NO_LOOP_ID) &&
+@@ -1638,7 +1667,7 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
+ if (vha->host->active_mode == MODE_TARGET && !N2N_TOPO(vha->hw))
+ return 0;
+
+- if (fcport->flags & FCF_ASYNC_SENT) {
++ if (fcport->flags & (FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE)) {
+ set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+ return 0;
+ }
+@@ -1786,16 +1815,52 @@ void qla2x00_handle_rscn(scsi_qla_host_t *vha, struct event_arg *ea)
+ fc_port_t *fcport;
+ unsigned long flags;
+
+- fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1);
+- if (fcport) {
+- if (fcport->flags & FCF_FCP2_DEVICE) {
+- ql_dbg(ql_dbg_disc, vha, 0x2115,
+- "Delaying session delete for FCP2 portid=%06x %8phC ",
+- fcport->d_id.b24, fcport->port_name);
+- return;
++ switch (ea->id.b.rsvd_1) {
++ case RSCN_PORT_ADDR:
++ fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1);
++ if (fcport) {
++ if (fcport->flags & FCF_FCP2_DEVICE) {
++ ql_dbg(ql_dbg_disc, vha, 0x2115,
++ "Delaying session delete for FCP2 portid=%06x %8phC ",
++ fcport->d_id.b24, fcport->port_name);
++ return;
++ }
++ fcport->scan_needed = 1;
++ fcport->rscn_gen++;
+ }
+- fcport->scan_needed = 1;
+- fcport->rscn_gen++;
++ break;
++ case RSCN_AREA_ADDR:
++ list_for_each_entry(fcport, &vha->vp_fcports, list) {
++ if (fcport->flags & FCF_FCP2_DEVICE)
++ continue;
++
++ if ((ea->id.b24 & 0xffff00) == (fcport->d_id.b24 & 0xffff00)) {
++ fcport->scan_needed = 1;
++ fcport->rscn_gen++;
++ }
++ }
++ break;
++ case RSCN_DOM_ADDR:
++ list_for_each_entry(fcport, &vha->vp_fcports, list) {
++ if (fcport->flags & FCF_FCP2_DEVICE)
++ continue;
++
++ if ((ea->id.b24 & 0xff0000) == (fcport->d_id.b24 & 0xff0000)) {
++ fcport->scan_needed = 1;
++ fcport->rscn_gen++;
++ }
++ }
++ break;
++ case RSCN_FAB_ADDR:
++ default:
++ list_for_each_entry(fcport, &vha->vp_fcports, list) {
++ if (fcport->flags & FCF_FCP2_DEVICE)
++ continue;
++
++ fcport->scan_needed = 1;
++ fcport->rscn_gen++;
++ }
++ break;
+ }
+
+ spin_lock_irqsave(&vha->work_lock, flags);
+@@ -1917,22 +1982,21 @@ qla2x00_async_tm_cmd(fc_port_t *fcport, uint32_t flags, uint32_t lun,
+ srb_t *sp;
+ int rval = QLA_FUNCTION_FAILED;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+- tm_iocb = &sp->u.iocb_cmd;
+ sp->type = SRB_TM_CMD;
+ sp->name = "tmf";
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha),
++ qla2x00_tmf_sp_done);
++ sp->u.iocb_cmd.timeout = qla2x00_tmf_iocb_timeout;
+
+- tm_iocb->timeout = qla2x00_tmf_iocb_timeout;
++ tm_iocb = &sp->u.iocb_cmd;
+ init_completion(&tm_iocb->u.tmf.comp);
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha));
+-
+ tm_iocb->u.tmf.flags = flags;
+ tm_iocb->u.tmf.lun = lun;
+- tm_iocb->u.tmf.data = tag;
+- sp->done = qla2x00_tmf_sp_done;
+
+ ql_dbg(ql_dbg_taskm, vha, 0x802f,
+ "Async-tmf hdl=%x loop-id=%x portid=%02x%02x%02x.\n",
+@@ -1962,7 +2026,8 @@ qla2x00_async_tm_cmd(fc_port_t *fcport, uint32_t flags, uint32_t lun,
+ }
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ done:
+ return rval;
+@@ -2021,13 +2086,6 @@ qla24xx_handle_prli_done_event(struct scsi_qla_host *vha, struct event_arg *ea)
+ qla24xx_post_gpdb_work(vha, ea->fcport, 0);
+ break;
+ default:
+- if ((ea->iop[0] == LSC_SCODE_ELS_REJECT) &&
+- (ea->iop[1] == 0x50000)) { /* reson 5=busy expl:0x0 */
+- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+- ea->fcport->fw_login_state = DSC_LS_PLOGI_COMP;
+- break;
+- }
+-
+ sp = ea->sp;
+ ql_dbg(ql_dbg_disc, vha, 0x2118,
+ "%s %d %8phC priority %s, fc4type %x prev try %s\n",
+@@ -2171,12 +2229,7 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea)
+ ql_dbg(ql_dbg_disc, vha, 0x20eb, "%s %d %8phC cmd error %x\n",
+ __func__, __LINE__, ea->fcport->port_name, ea->data[1]);
+
+- ea->fcport->flags &= ~FCF_ASYNC_SENT;
+- qla2x00_set_fcport_disc_state(ea->fcport, DSC_LOGIN_FAILED);
+- if (ea->data[1] & QLA_LOGIO_LOGIN_RETRIED)
+- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
+- else
+- qla2x00_mark_device_lost(vha, ea->fcport, 1);
++ qlt_schedule_sess_for_deletion(ea->fcport);
+ break;
+ case MBS_LOOP_ID_USED:
+ /* data[1] = IO PARAM 1 = nport ID */
+@@ -3419,6 +3472,14 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
+ struct rsp_que *rsp = ha->rsp_q_map[0];
+ struct qla2xxx_fw_dump *fw_dump;
+
++ if (ha->fw_dump) {
++ ql_dbg(ql_dbg_init, vha, 0x00bd,
++ "Firmware dump already allocated.\n");
++ return;
++ }
++
++ ha->fw_dumped = 0;
++ ha->fw_dump_cap_flags = 0;
+ dump_size = fixed_size = mem_size = eft_size = fce_size = mq_size = 0;
+ req_q_size = rsp_q_size = 0;
+
+@@ -3429,7 +3490,7 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
+ mem_size = (ha->fw_memory_size - 0x11000 + 1) *
+ sizeof(uint16_t);
+ } else if (IS_FWI2_CAPABLE(ha)) {
+- if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
++ if (IS_QLA83XX(ha))
+ fixed_size = offsetof(struct qla83xx_fw_dump, ext_mem);
+ else if (IS_QLA81XX(ha))
+ fixed_size = offsetof(struct qla81xx_fw_dump, ext_mem);
+@@ -3441,8 +3502,7 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
+ mem_size = (ha->fw_memory_size - 0x100000 + 1) *
+ sizeof(uint32_t);
+ if (ha->mqenable) {
+- if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) &&
+- !IS_QLA28XX(ha))
++ if (!IS_QLA83XX(ha))
+ mq_size = sizeof(struct qla2xxx_mq_chain);
+ /*
+ * Allocate maximum buffer size for all queues - Q0.
+@@ -4003,8 +4063,7 @@ enable_82xx_npiv:
+ ha->fw_major_version, ha->fw_minor_version,
+ ha->fw_subminor_version);
+
+- if (IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+- IS_QLA28XX(ha)) {
++ if (IS_QLA83XX(ha)) {
+ ha->flags.fac_supported = 0;
+ rval = QLA_SUCCESS;
+ }
+@@ -4433,6 +4492,10 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
+ (ha->flags.fawwpn_enabled) ? "enabled" : "disabled");
+ }
+
++ /* ELS pass through payload is limit by frame size. */
++ if (ha->flags.edif_enabled)
++ mid_init_cb->init_cb.frame_payload_size = cpu_to_le16(ELS_MAX_PAYLOAD);
++
+ rval = qla2x00_init_firmware(vha, ha->init_cb_size);
+ next_check:
+ if (rval) {
+@@ -5335,15 +5398,14 @@ qla2x00_configure_loop(scsi_qla_host_t *vha)
+ "LOOP READY.\n");
+ ha->flags.fw_init_done = 1;
+
++ /*
++ * use link up to wake up app to get ready for
++ * authentication.
++ */
+ if (ha->flags.edif_enabled &&
+- !(vha->e_dbell.db_flags & EDB_ACTIVE) &&
+- N2N_TOPO(vha->hw)) {
+- /*
+- * use port online to wake up app to get ready
+- * for authentication
+- */
+- qla2x00_post_aen_work(vha, FCH_EVT_PORT_ONLINE, 0);
+- }
++ !(vha->e_dbell.db_flags & EDB_ACTIVE))
++ qla2x00_post_aen_work(vha, FCH_EVT_LINKUP,
++ ha->link_data_rate);
+
+ /*
+ * Process any ATIO queue entries that came in
+@@ -5547,6 +5609,13 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
+ memcpy(fcport->node_name, new_fcport->node_name,
+ WWN_SIZE);
+ fcport->scan_state = QLA_FCPORT_FOUND;
++ if (fcport->login_retry == 0) {
++ fcport->login_retry = vha->hw->login_retry_count;
++ ql_dbg(ql_dbg_disc, vha, 0x2135,
++ "Port login retry %8phN, lid 0x%04x retry cnt=%d.\n",
++ fcport->port_name, fcport->loop_id,
++ fcport->login_retry);
++ }
+ found++;
+ break;
+ }
+@@ -5680,6 +5749,8 @@ qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
+ if (atomic_read(&fcport->state) == FCS_ONLINE)
+ return;
+
++ qla2x00_set_fcport_state(fcport, FCS_ONLINE);
++
+ rport_ids.node_name = wwn_to_u64(fcport->node_name);
+ rport_ids.port_name = wwn_to_u64(fcport->port_name);
+ rport_ids.port_id = fcport->d_id.b.domain << 16 |
+@@ -5787,6 +5858,7 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
+ qla2x00_reg_remote_port(vha, fcport);
+ break;
+ case MODE_TARGET:
++ qla2x00_set_fcport_state(fcport, FCS_ONLINE);
+ if (!vha->vha_tgt.qla_tgt->tgt_stop &&
+ !vha->vha_tgt.qla_tgt->tgt_stopped)
+ qlt_fc_port_added(vha, fcport);
+@@ -5801,8 +5873,6 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
+ break;
+ }
+
+- qla2x00_set_fcport_state(fcport, FCS_ONLINE);
+-
+ if (IS_IIDMA_CAPABLE(vha->hw) && vha->hw->flags.gpsc_supported) {
+ if (fcport->id_changed) {
+ fcport->id_changed = 0;
+@@ -9333,7 +9403,7 @@ struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *vha, int qos,
+ qpair->rsp->req = qpair->req;
+ qpair->rsp->qpair = qpair;
+ /* init qpair to this cpu. Will adjust at run time. */
+- qla_cpu_update(qpair, smp_processor_id());
++ qla_cpu_update(qpair, raw_smp_processor_id());
+
+ if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif) {
+ if (ha->fw_attributes & BIT_4)
+diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
+index 5f3b7995cc8f3..db17f7f410cdd 100644
+--- a/drivers/scsi/qla2xxx/qla_inline.h
++++ b/drivers/scsi/qla2xxx/qla_inline.h
+@@ -184,6 +184,8 @@ static void qla2xxx_init_sp(srb_t *sp, scsi_qla_host_t *vha,
+ sp->vha = vha;
+ sp->qpair = qpair;
+ sp->cmd_type = TYPE_SRB;
++ /* ref : INIT - normal flow */
++ kref_init(&sp->cmd_kref);
+ INIT_LIST_HEAD(&sp->elem);
+ }
+
+diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
+index 9d4ad1d2b00a2..606228f4a8b53 100644
+--- a/drivers/scsi/qla2xxx/qla_iocb.c
++++ b/drivers/scsi/qla2xxx/qla_iocb.c
+@@ -2560,11 +2560,38 @@ qla24xx_tm_iocb(srb_t *sp, struct tsk_mgmt_entry *tsk)
+ }
+ }
+
+-void qla2x00_init_timer(srb_t *sp, unsigned long tmo)
++static void
++qla2x00_async_done(struct srb *sp, int res)
++{
++ if (del_timer(&sp->u.iocb_cmd.timer)) {
++ /*
++ * Successfully cancelled the timeout handler
++ * ref: TMR
++ */
++ if (kref_put(&sp->cmd_kref, qla2x00_sp_release))
++ return;
++ }
++ sp->async_done(sp, res);
++}
++
++void
++qla2x00_sp_release(struct kref *kref)
++{
++ struct srb *sp = container_of(kref, struct srb, cmd_kref);
++
++ sp->free(sp);
++}
++
++void
++qla2x00_init_async_sp(srb_t *sp, unsigned long tmo,
++ void (*done)(struct srb *sp, int res))
+ {
+ timer_setup(&sp->u.iocb_cmd.timer, qla2x00_sp_timeout, 0);
+- sp->u.iocb_cmd.timer.expires = jiffies + tmo * HZ;
++ sp->done = qla2x00_async_done;
++ sp->async_done = done;
+ sp->free = qla2x00_sp_free;
++ sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
++ sp->u.iocb_cmd.timer.expires = jiffies + tmo * HZ;
+ if (IS_QLAFX00(sp->vha->hw) && sp->type == SRB_FXIOCB_DCMD)
+ init_completion(&sp->u.iocb_cmd.u.fxiocb.fxiocb_comp);
+ sp->start_timer = 1;
+@@ -2651,7 +2678,9 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode,
+ return -ENOMEM;
+ }
+
+- /* Alloc SRB structure */
++ /* Alloc SRB structure
++ * ref: INIT
++ */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp) {
+ kfree(fcport);
+@@ -2672,18 +2701,19 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode,
+ sp->type = SRB_ELS_DCMD;
+ sp->name = "ELS_DCMD";
+ sp->fcport = fcport;
+- elsio->timeout = qla2x00_els_dcmd_iocb_timeout;
+- qla2x00_init_timer(sp, ELS_DCMD_TIMEOUT);
+- init_completion(&sp->u.iocb_cmd.u.els_logo.comp);
+- sp->done = qla2x00_els_dcmd_sp_done;
++ qla2x00_init_async_sp(sp, ELS_DCMD_TIMEOUT,
++ qla2x00_els_dcmd_sp_done);
+ sp->free = qla2x00_els_dcmd_sp_free;
++ sp->u.iocb_cmd.timeout = qla2x00_els_dcmd_iocb_timeout;
++ init_completion(&sp->u.iocb_cmd.u.els_logo.comp);
+
+ elsio->u.els_logo.els_logo_pyld = dma_alloc_coherent(&ha->pdev->dev,
+ DMA_POOL_SIZE, &elsio->u.els_logo.els_logo_pyld_dma,
+ GFP_KERNEL);
+
+ if (!elsio->u.els_logo.els_logo_pyld) {
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return QLA_FUNCTION_FAILED;
+ }
+
+@@ -2706,7 +2736,8 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode,
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS) {
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return QLA_FUNCTION_FAILED;
+ }
+
+@@ -2717,7 +2748,8 @@ qla24xx_els_dcmd_iocb(scsi_qla_host_t *vha, int els_opcode,
+
+ wait_for_completion(&elsio->u.els_logo.comp);
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return rval;
+ }
+
+@@ -2850,7 +2882,6 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
+ sp->name, res, sp->handle, fcport->d_id.b24, fcport->port_name);
+
+ fcport->flags &= ~(FCF_ASYNC_SENT|FCF_ASYNC_ACTIVE);
+- del_timer(&sp->u.iocb_cmd.timer);
+
+ if (sp->flags & SRB_WAKEUP_ON_COMP)
+ complete(&lio->u.els_plogi.comp);
+@@ -2927,6 +2958,7 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
+ set_bit(ISP_ABORT_NEEDED,
+ &vha->dpc_flags);
+ qla2xxx_wake_dpc(vha);
++ break;
+ }
+ fallthrough;
+ default:
+@@ -2936,9 +2968,7 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
+ fw_status[0], fw_status[1], fw_status[2]);
+
+ fcport->flags &= ~FCF_ASYNC_SENT;
+- qla2x00_set_fcport_disc_state(fcport,
+- DSC_LOGIN_FAILED);
+- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
++ qlt_schedule_sess_for_deletion(fcport);
+ break;
+ }
+ break;
+@@ -2950,8 +2980,7 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
+ fw_status[0], fw_status[1], fw_status[2]);
+
+ sp->fcport->flags &= ~FCF_ASYNC_SENT;
+- qla2x00_set_fcport_disc_state(fcport, DSC_LOGIN_FAILED);
+- set_bit(RELOGIN_NEEDED, &vha->dpc_flags);
++ qlt_schedule_sess_for_deletion(fcport);
+ break;
+ }
+
+@@ -2960,7 +2989,8 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
+ struct srb_iocb *elsio = &sp->u.iocb_cmd;
+
+ qla2x00_els_dcmd2_free(vha, &elsio->u.els_plogi);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return;
+ }
+ e->u.iosb.sp = sp;
+@@ -2978,7 +3008,9 @@ qla24xx_els_dcmd2_iocb(scsi_qla_host_t *vha, int els_opcode,
+ int rval = QLA_SUCCESS;
+ void *ptr, *resp_ptr;
+
+- /* Alloc SRB structure */
++ /* Alloc SRB structure
++ * ref: INIT
++ */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp) {
+ ql_log(ql_log_info, vha, 0x70e6,
+@@ -2993,17 +3025,16 @@ qla24xx_els_dcmd2_iocb(scsi_qla_host_t *vha, int els_opcode,
+ ql_dbg(ql_dbg_io, vha, 0x3073,
+ "%s Enter: PLOGI portid=%06x\n", __func__, fcport->d_id.b24);
+
+- sp->type = SRB_ELS_DCMD;
+- sp->name = "ELS_DCMD";
+- sp->fcport = fcport;
+-
+- elsio->timeout = qla2x00_els_dcmd2_iocb_timeout;
+ if (wait)
+ sp->flags = SRB_WAKEUP_ON_COMP;
+
+- qla2x00_init_timer(sp, ELS_DCMD_TIMEOUT + 2);
++ sp->type = SRB_ELS_DCMD;
++ sp->name = "ELS_DCMD";
++ sp->fcport = fcport;
++ qla2x00_init_async_sp(sp, ELS_DCMD_TIMEOUT + 2,
++ qla2x00_els_dcmd2_sp_done);
++ sp->u.iocb_cmd.timeout = qla2x00_els_dcmd2_iocb_timeout;
+
+- sp->done = qla2x00_els_dcmd2_sp_done;
+ elsio->u.els_plogi.tx_size = elsio->u.els_plogi.rx_size = DMA_POOL_SIZE;
+
+ ptr = elsio->u.els_plogi.els_plogi_pyld =
+@@ -3069,7 +3100,8 @@ qla24xx_els_dcmd2_iocb(scsi_qla_host_t *vha, int els_opcode,
+ out:
+ fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+ qla2x00_els_dcmd2_free(vha, &elsio->u.els_plogi);
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+@@ -3880,8 +3912,15 @@ qla2x00_start_sp(srb_t *sp)
+ break;
+ }
+
+- if (sp->start_timer)
++ if (sp->start_timer) {
++ /* ref: TMR timer ref
++ * this code should be just before start_iocbs function
++ * This will make sure that caller function don't to do
++ * kref_put even on failure
++ */
++ kref_get(&sp->cmd_kref);
+ add_timer(&sp->u.iocb_cmd.timer);
++ }
+
+ wmb();
+ qla2x00_start_iocbs(vha, qp->req);
+diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
+index b26f2699adb27..62d2f14848e7b 100644
+--- a/drivers/scsi/qla2xxx/qla_isr.c
++++ b/drivers/scsi/qla2xxx/qla_isr.c
+@@ -2494,6 +2494,7 @@ qla24xx_tm_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
+ iocb->u.tmf.data = QLA_FUNCTION_FAILED;
+ } else if ((le16_to_cpu(sts->scsi_status) &
+ SS_RESPONSE_INFO_LEN_VALID)) {
++ host_to_fcp_swap(sts->data, sizeof(sts->data));
+ if (le32_to_cpu(sts->rsp_data_len) < 4) {
+ ql_log(ql_log_warn, fcport->vha, 0x503b,
+ "Async-%s error - hdl=%x not enough response(%d).\n",
+diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
+index 7811c4952035b..950c5903e7992 100644
+--- a/drivers/scsi/qla2xxx/qla_mbx.c
++++ b/drivers/scsi/qla2xxx/qla_mbx.c
+@@ -9,6 +9,12 @@
+ #include <linux/delay.h>
+ #include <linux/gfp.h>
+
++#ifdef CONFIG_PPC
++#define IS_PPCARCH true
++#else
++#define IS_PPCARCH false
++#endif
++
+ static struct mb_cmd_name {
+ uint16_t cmd;
+ const char *str;
+@@ -728,6 +734,9 @@ again:
+ vha->min_supported_speed =
+ nv->min_supported_speed;
+ }
++
++ if (IS_PPCARCH)
++ mcp->mb[11] |= BIT_4;
+ }
+
+ if (ha->flags.exlogins_enabled)
+@@ -1695,10 +1704,8 @@ qla2x00_get_adapter_id(scsi_qla_host_t *vha, uint16_t *id, uint8_t *al_pa,
+ mcp->in_mb |= MBX_13|MBX_12|MBX_11|MBX_10;
+ if (IS_FWI2_CAPABLE(vha->hw))
+ mcp->in_mb |= MBX_19|MBX_18|MBX_17|MBX_16;
+- if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw)) {
+- mcp->in_mb |= MBX_15;
+- mcp->out_mb |= MBX_7|MBX_21|MBX_22|MBX_23;
+- }
++ if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw))
++ mcp->in_mb |= MBX_15|MBX_21|MBX_22|MBX_23;
+
+ mcp->tov = MBX_TOV_SECONDS;
+ mcp->flags = 0;
+@@ -3031,8 +3038,7 @@ qla2x00_get_resource_cnts(scsi_qla_host_t *vha)
+ ha->orig_fw_iocb_count = mcp->mb[10];
+ if (ha->flags.npiv_supported)
+ ha->max_npiv_vports = mcp->mb[11];
+- if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+- IS_QLA28XX(ha))
++ if (IS_QLA81XX(ha) || IS_QLA83XX(ha))
+ ha->fw_max_fcf_count = mcp->mb[12];
+ }
+
+@@ -5623,7 +5629,7 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha)
+ mcp->out_mb = MBX_1|MBX_0;
+ mcp->in_mb = MBX_2|MBX_1|MBX_0;
+ if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
+- mcp->in_mb |= MBX_3;
++ mcp->in_mb |= MBX_4|MBX_3;
+ mcp->tov = MBX_TOV_SECONDS;
+ mcp->flags = 0;
+ rval = qla2x00_mailbox_command(vha, mcp);
+@@ -6481,23 +6487,21 @@ int qla24xx_send_mb_cmd(struct scsi_qla_host *vha, mbx_cmd_t *mcp)
+ if (!vha->hw->flags.fw_started)
+ goto done;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, NULL, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+- sp->type = SRB_MB_IOCB;
+- sp->name = mb_to_str(mcp->mb[0]);
+-
+ c = &sp->u.iocb_cmd;
+- c->timeout = qla2x00_async_iocb_timeout;
+ init_completion(&c->u.mbx.comp);
+
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ sp->type = SRB_MB_IOCB;
++ sp->name = mb_to_str(mcp->mb[0]);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_mb_sp_done);
+
+ memcpy(sp->u.iocb_cmd.u.mbx.out_mb, mcp->mb, SIZEOF_IOCB_MB_REG);
+
+- sp->done = qla2x00_async_mb_sp_done;
+-
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS) {
+ ql_dbg(ql_dbg_mbx, vha, 0x1018,
+@@ -6529,7 +6533,8 @@ int qla24xx_send_mb_cmd(struct scsi_qla_host *vha, mbx_cmd_t *mcp)
+ }
+
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
+index 1c024055f8c50..e6b5c4ccce97b 100644
+--- a/drivers/scsi/qla2xxx/qla_mid.c
++++ b/drivers/scsi/qla2xxx/qla_mid.c
+@@ -965,6 +965,7 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
+ if (vp_index == 0 || vp_index >= ha->max_npiv_vports)
+ return QLA_PARAMETER_ERROR;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(base_vha, NULL, GFP_KERNEL);
+ if (!sp)
+ return rval;
+@@ -972,9 +973,8 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
+ sp->type = SRB_CTRL_VP;
+ sp->name = "ctrl_vp";
+ sp->comp = &comp;
+- sp->done = qla_ctrlvp_sp_done;
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla_ctrlvp_sp_done);
+ sp->u.iocb_cmd.u.ctrlvp.cmd = cmd;
+ sp->u.iocb_cmd.u.ctrlvp.vp_index = vp_index;
+
+@@ -1008,6 +1008,7 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
+ break;
+ }
+ done:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ return rval;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
+index 6e920da64863e..f726eb8449c5e 100644
+--- a/drivers/scsi/qla2xxx/qla_mr.c
++++ b/drivers/scsi/qla2xxx/qla_mr.c
+@@ -738,29 +738,6 @@ qlafx00_lun_reset(fc_port_t *fcport, uint64_t l, int tag)
+ return qla2x00_async_tm_cmd(fcport, TCF_LUN_RESET, l, tag);
+ }
+
+-int
+-qlafx00_loop_reset(scsi_qla_host_t *vha)
+-{
+- int ret;
+- struct fc_port *fcport;
+- struct qla_hw_data *ha = vha->hw;
+-
+- if (ql2xtargetreset) {
+- list_for_each_entry(fcport, &vha->vp_fcports, list) {
+- if (fcport->port_type != FCT_TARGET)
+- continue;
+-
+- ret = ha->isp_ops->target_reset(fcport, 0, 0);
+- if (ret != QLA_SUCCESS) {
+- ql_dbg(ql_dbg_taskm, vha, 0x803d,
+- "Bus Reset failed: Reset=%d "
+- "d_id=%x.\n", ret, fcport->d_id.b24);
+- }
+- }
+- }
+- return QLA_SUCCESS;
+-}
+-
+ int
+ qlafx00_iospace_config(struct qla_hw_data *ha)
+ {
+@@ -1810,17 +1787,18 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
+ struct register_host_info *preg_hsi;
+ struct new_utsname *p_sysid = NULL;
+
++ /* ref: INIT */
+ sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL);
+ if (!sp)
+ goto done;
+
+ sp->type = SRB_FXIOCB_DCMD;
+ sp->name = "fxdisc";
++ qla2x00_init_async_sp(sp, FXDISC_TIMEOUT,
++ qla2x00_fxdisc_sp_done);
++ sp->u.iocb_cmd.timeout = qla2x00_fxdisc_iocb_timeout;
+
+ fdisc = &sp->u.iocb_cmd;
+- fdisc->timeout = qla2x00_fxdisc_iocb_timeout;
+- qla2x00_init_timer(sp, FXDISC_TIMEOUT);
+-
+ switch (fx_type) {
+ case FXDISC_GET_CONFIG_INFO:
+ fdisc->u.fxiocb.flags =
+@@ -1921,7 +1899,6 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
+ }
+
+ fdisc->u.fxiocb.req_func_type = cpu_to_le16(fx_type);
+- sp->done = qla2x00_fxdisc_sp_done;
+
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS)
+@@ -1997,7 +1974,8 @@ done_unmap_req:
+ dma_free_coherent(&ha->pdev->dev, fdisc->u.fxiocb.req_len,
+ fdisc->u.fxiocb.req_addr, fdisc->u.fxiocb.req_dma_handle);
+ done_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ return rval;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
+index 1c5da2dbd6f97..42b29f4fd9371 100644
+--- a/drivers/scsi/qla2xxx/qla_nvme.c
++++ b/drivers/scsi/qla2xxx/qla_nvme.c
+@@ -35,6 +35,11 @@ int qla_nvme_register_remote(struct scsi_qla_host *vha, struct fc_port *fcport)
+ (fcport->nvme_flag & NVME_FLAG_REGISTERED))
+ return 0;
+
++ if (atomic_read(&fcport->state) == FCS_ONLINE)
++ return 0;
++
++ qla2x00_set_fcport_state(fcport, FCS_ONLINE);
++
+ fcport->nvme_flag &= ~NVME_FLAG_RESETTING;
+
+ memset(&req, 0, sizeof(struct nvme_fc_port_info));
+@@ -165,6 +170,18 @@ out:
+ qla2xxx_rel_qpair_sp(sp->qpair, sp);
+ }
+
++static void qla_nvme_ls_unmap(struct srb *sp, struct nvmefc_ls_req *fd)
++{
++ if (sp->flags & SRB_DMA_VALID) {
++ struct srb_iocb *nvme = &sp->u.iocb_cmd;
++ struct qla_hw_data *ha = sp->fcport->vha->hw;
++
++ dma_unmap_single(&ha->pdev->dev, nvme->u.nvme.cmd_dma,
++ fd->rqstlen, DMA_TO_DEVICE);
++ sp->flags &= ~SRB_DMA_VALID;
++ }
++}
++
+ static void qla_nvme_release_ls_cmd_kref(struct kref *kref)
+ {
+ struct srb *sp = container_of(kref, struct srb, cmd_kref);
+@@ -181,6 +198,8 @@ static void qla_nvme_release_ls_cmd_kref(struct kref *kref)
+ spin_unlock_irqrestore(&priv->cmd_lock, flags);
+
+ fd = priv->fd;
++
++ qla_nvme_ls_unmap(sp, fd);
+ fd->done(fd, priv->comp_status);
+ out:
+ qla2x00_rel_sp(sp);
+@@ -228,6 +247,8 @@ static void qla_nvme_abort_work(struct work_struct *work)
+ fc_port_t *fcport = sp->fcport;
+ struct qla_hw_data *ha = fcport->vha->hw;
+ int rval, abts_done_called = 1;
++ bool io_wait_for_abort_done;
++ uint32_t handle;
+
+ ql_dbg(ql_dbg_io, fcport->vha, 0xffff,
+ "%s called for sp=%p, hndl=%x on fcport=%p desc=%p deleted=%d\n",
+@@ -244,12 +265,20 @@ static void qla_nvme_abort_work(struct work_struct *work)
+ goto out;
+ }
+
++ /*
++ * sp may not be valid after abort_command if return code is either
++ * SUCCESS or ERR_FROM_FW codes, so cache the value here.
++ */
++ io_wait_for_abort_done = ql2xabts_wait_nvme &&
++ QLA_ABTS_WAIT_ENABLED(sp);
++ handle = sp->handle;
++
+ rval = ha->isp_ops->abort_command(sp);
+
+ ql_dbg(ql_dbg_io, fcport->vha, 0x212b,
+ "%s: %s command for sp=%p, handle=%x on fcport=%p rval=%x\n",
+ __func__, (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
+- sp, sp->handle, fcport, rval);
++ sp, handle, fcport, rval);
+
+ /*
+ * If async tmf is enabled, the abort callback is called only on
+@@ -264,7 +293,7 @@ static void qla_nvme_abort_work(struct work_struct *work)
+ * are waited until ABTS complete. This kref is decreased
+ * at qla24xx_abort_sp_done function.
+ */
+- if (abts_done_called && ql2xabts_wait_nvme && QLA_ABTS_WAIT_ENABLED(sp))
++ if (abts_done_called && io_wait_for_abort_done)
+ return;
+ out:
+ /* kref_get was done before work was schedule. */
+@@ -341,6 +370,8 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
+ dma_sync_single_for_device(&ha->pdev->dev, nvme->u.nvme.cmd_dma,
+ fd->rqstlen, DMA_TO_DEVICE);
+
++ sp->flags |= SRB_DMA_VALID;
++
+ rval = qla2x00_start_sp(sp);
+ if (rval != QLA_SUCCESS) {
+ ql_log(ql_log_warn, vha, 0x700e,
+@@ -348,6 +379,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
+ wake_up(&sp->nvme_ls_waitq);
+ sp->priv = NULL;
+ priv->sp = NULL;
++ qla_nvme_ls_unmap(sp, fd);
+ qla2x00_rel_sp(sp);
+ return rval;
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
+index 836fedcea241b..77c0bf06f162b 100644
+--- a/drivers/scsi/qla2xxx/qla_os.c
++++ b/drivers/scsi/qla2xxx/qla_os.c
+@@ -202,12 +202,6 @@ MODULE_PARM_DESC(ql2xdbwr,
+ " 0 -- Regular doorbell.\n"
+ " 1 -- CAMRAM doorbell (faster).\n");
+
+-int ql2xtargetreset = 1;
+-module_param(ql2xtargetreset, int, S_IRUGO);
+-MODULE_PARM_DESC(ql2xtargetreset,
+- "Enable target reset."
+- "Default is 1 - use hw defaults.");
+-
+ int ql2xgffidenable;
+ module_param(ql2xgffidenable, int, S_IRUGO);
+ MODULE_PARM_DESC(ql2xgffidenable,
+@@ -734,7 +728,8 @@ void qla2x00_sp_compl(srb_t *sp, int res)
+ struct scsi_cmnd *cmd = GET_CMD_SP(sp);
+ struct completion *comp = sp->comp;
+
+- sp->free(sp);
++ /* kref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ cmd->result = res;
+ CMD_SP(cmd) = NULL;
+ cmd->scsi_done(cmd);
+@@ -825,7 +820,8 @@ void qla2xxx_qpair_sp_compl(srb_t *sp, int res)
+ struct scsi_cmnd *cmd = GET_CMD_SP(sp);
+ struct completion *comp = sp->comp;
+
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ cmd->result = res;
+ CMD_SP(cmd) = NULL;
+ cmd->scsi_done(cmd);
+@@ -925,6 +921,7 @@ qla2xxx_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
+ goto qc24_target_busy;
+
+ sp = scsi_cmd_priv(cmd);
++ /* ref: INIT */
+ qla2xxx_init_sp(sp, vha, vha->hw->base_qpair, fcport);
+
+ sp->u.scmd.cmd = cmd;
+@@ -944,7 +941,8 @@ qla2xxx_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
+ return 0;
+
+ qc24_host_busy_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+
+ qc24_target_busy:
+ return SCSI_MLQUEUE_TARGET_BUSY;
+@@ -1014,6 +1012,7 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd,
+ goto qc24_target_busy;
+
+ sp = scsi_cmd_priv(cmd);
++ /* ref: INIT */
+ qla2xxx_init_sp(sp, vha, qpair, fcport);
+
+ sp->u.scmd.cmd = cmd;
+@@ -1032,7 +1031,8 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd,
+ return 0;
+
+ qc24_host_busy_free_sp:
+- sp->free(sp);
++ /* ref: INIT */
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+
+ qc24_target_busy:
+ return SCSI_MLQUEUE_TARGET_BUSY;
+@@ -1258,6 +1258,7 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
+ uint32_t ratov_j;
+ struct qla_qpair *qpair;
+ unsigned long flags;
++ int fast_fail_status = SUCCESS;
+
+ if (qla2x00_isp_reg_stat(ha)) {
+ ql_log(ql_log_info, vha, 0x8042,
+@@ -1266,9 +1267,10 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
+ return FAILED;
+ }
+
++ /* Save any FAST_IO_FAIL value to return later if abort succeeds */
+ ret = fc_block_scsi_eh(cmd);
+ if (ret != 0)
+- return ret;
++ fast_fail_status = ret;
+
+ sp = scsi_cmd_priv(cmd);
+ qpair = sp->qpair;
+@@ -1276,7 +1278,7 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
+ vha->cmd_timeout_cnt++;
+
+ if ((sp->fcport && sp->fcport->deleted) || !qpair)
+- return SUCCESS;
++ return fast_fail_status != SUCCESS ? fast_fail_status : FAILED;
+
+ spin_lock_irqsave(qpair->qp_lock_ptr, flags);
+ sp->comp = &comp;
+@@ -1311,7 +1313,7 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
+ __func__, ha->r_a_tov/10);
+ ret = FAILED;
+ } else {
+- ret = SUCCESS;
++ ret = fast_fail_status;
+ }
+ break;
+ default:
+@@ -1693,27 +1695,10 @@ int
+ qla2x00_loop_reset(scsi_qla_host_t *vha)
+ {
+ int ret;
+- struct fc_port *fcport;
+ struct qla_hw_data *ha = vha->hw;
+
+- if (IS_QLAFX00(ha)) {
+- return qlafx00_loop_reset(vha);
+- }
+-
+- if (ql2xtargetreset == 1 && ha->flags.enable_target_reset) {
+- list_for_each_entry(fcport, &vha->vp_fcports, list) {
+- if (fcport->port_type != FCT_TARGET)
+- continue;
+-
+- ret = ha->isp_ops->target_reset(fcport, 0, 0);
+- if (ret != QLA_SUCCESS) {
+- ql_dbg(ql_dbg_taskm, vha, 0x802c,
+- "Bus Reset failed: Reset=%d "
+- "d_id=%x.\n", ret, fcport->d_id.b24);
+- }
+- }
+- }
+-
++ if (IS_QLAFX00(ha))
++ return QLA_SUCCESS;
+
+ if (ha->flags.enable_lip_full_login && !IS_CNA_CAPABLE(ha)) {
+ atomic_set(&vha->loop_state, LOOP_DOWN);
+@@ -3754,8 +3739,7 @@ qla2x00_unmap_iobases(struct qla_hw_data *ha)
+ if (ha->mqiobase)
+ iounmap(ha->mqiobase);
+
+- if ((IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&
+- ha->msixbase)
++ if (ha->msixbase)
+ iounmap(ha->msixbase);
+ }
+ }
+@@ -4358,7 +4342,7 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len,
+
+ /* allocate the purex dma pool */
+ ha->purex_dma_pool = dma_pool_create(name, &ha->pdev->dev,
+- MAX_PAYLOAD, 8, 0);
++ ELS_MAX_PAYLOAD, 8, 0);
+
+ if (!ha->purex_dma_pool) {
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0x011b,
+@@ -5532,6 +5516,11 @@ void qla2x00_relogin(struct scsi_qla_host *vha)
+ memset(&ea, 0, sizeof(ea));
+ ea.fcport = fcport;
+ qla24xx_handle_relogin_event(vha, &ea);
++ } else if (vha->hw->current_topology ==
++ ISP_CFG_NL &&
++ IS_QLA2XXX_MIDTYPE(vha->hw)) {
++ (void)qla24xx_fcport_handle_login(vha,
++ fcport);
+ } else if (vha->hw->current_topology ==
+ ISP_CFG_NL) {
+ fcport->login_retry--;
+@@ -7656,7 +7645,7 @@ qla2xxx_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
+
+ switch (state) {
+ case pci_channel_io_normal:
+- ha->flags.eeh_busy = 0;
++ qla_pci_set_eeh_busy(vha);
+ if (ql2xmqsupport || ql2xnvmeenable) {
+ set_bit(QPAIR_ONLINE_CHECK_NEEDED, &vha->dpc_flags);
+ qla2xxx_wake_dpc(vha);
+@@ -7697,9 +7686,16 @@ qla2xxx_pci_mmio_enabled(struct pci_dev *pdev)
+ "mmio enabled\n");
+
+ ha->pci_error_state = QLA_PCI_MMIO_ENABLED;
++
+ if (IS_QLA82XX(ha))
+ return PCI_ERS_RESULT_RECOVERED;
+
++ if (qla2x00_isp_reg_stat(ha)) {
++ ql_log(ql_log_info, base_vha, 0x803f,
++ "During mmio enabled, PCI/Register disconnect still detected.\n");
++ goto out;
++ }
++
+ spin_lock_irqsave(&ha->hardware_lock, flags);
+ if (IS_QLA2100(ha) || IS_QLA2200(ha)){
+ stat = rd_reg_word(&reg->hccr);
+@@ -7721,6 +7717,7 @@ qla2xxx_pci_mmio_enabled(struct pci_dev *pdev)
+ "RISC paused -- mmio_enabled, Dumping firmware.\n");
+ qla2xxx_dump_fw(base_vha);
+ }
++out:
+ /* set PCI_ERS_RESULT_NEED_RESET to trigger call to qla2xxx_pci_slot_reset */
+ ql_dbg(ql_dbg_aer, base_vha, 0x600d,
+ "mmio enabled returning.\n");
+diff --git a/drivers/scsi/qla2xxx/qla_sup.c b/drivers/scsi/qla2xxx/qla_sup.c
+index a0aeba69513d4..c092a6b1ced4f 100644
+--- a/drivers/scsi/qla2xxx/qla_sup.c
++++ b/drivers/scsi/qla2xxx/qla_sup.c
+@@ -844,7 +844,7 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
+ ha->flt_region_nvram = start;
+ break;
+ case FLT_REG_IMG_PRI_27XX:
+- if (IS_QLA27XX(ha) && !IS_QLA28XX(ha))
++ if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+ ha->flt_region_img_status_pri = start;
+ break;
+ case FLT_REG_IMG_SEC_27XX:
+@@ -1356,7 +1356,7 @@ next:
+ flash_data_addr(ha, faddr), le32_to_cpu(*dwptr));
+ if (ret) {
+ ql_dbg(ql_dbg_user, vha, 0x7006,
+- "Failed slopw write %x (%x)\n", faddr, *dwptr);
++ "Failed slow write %x (%x)\n", faddr, *dwptr);
+ break;
+ }
+ }
+diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
+index 7d8242c120fc7..ae5eaa4a9283a 100644
+--- a/drivers/scsi/qla2xxx/qla_target.c
++++ b/drivers/scsi/qla2xxx/qla_target.c
+@@ -620,7 +620,7 @@ static void qla2x00_async_nack_sp_done(srb_t *sp, int res)
+ }
+ spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+- sp->free(sp);
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ }
+
+ int qla24xx_async_notify_ack(scsi_qla_host_t *vha, fc_port_t *fcport,
+@@ -656,12 +656,10 @@ int qla24xx_async_notify_ack(scsi_qla_host_t *vha, fc_port_t *fcport,
+
+ sp->type = type;
+ sp->name = "nack";
+-
+- sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
+- qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha)+2);
++ qla2x00_init_async_sp(sp, qla2x00_get_async_timeout(vha) + 2,
++ qla2x00_async_nack_sp_done);
+
+ sp->u.iocb_cmd.u.nack.ntfy = ntfy;
+- sp->done = qla2x00_async_nack_sp_done;
+
+ ql_dbg(ql_dbg_disc, vha, 0x20f4,
+ "Async-%s %8phC hndl %x %s\n",
+@@ -674,7 +672,7 @@ int qla24xx_async_notify_ack(scsi_qla_host_t *vha, fc_port_t *fcport,
+ return rval;
+
+ done_free_sp:
+- sp->free(sp);
++ kref_put(&sp->cmd_kref, qla2x00_sp_release);
+ done:
+ fcport->flags &= ~FCF_ASYNC_SENT;
+ return rval;
+@@ -1003,6 +1001,7 @@ void qlt_free_session_done(struct work_struct *work)
+ "%s bypassing release_all_sadb\n",
+ __func__);
+ }
++ qla_edif_clear_appdata(vha, sess);
+ qla_edif_sess_down(vha, sess);
+ }
+ qla2x00_mark_device_lost(vha, sess, 0);
+@@ -3319,6 +3318,7 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type,
+ "RESET-RSP online/active/old-count/new-count = %d/%d/%d/%d.\n",
+ vha->flags.online, qla2x00_reset_active(vha),
+ cmd->reset_count, qpair->chip_reset);
++ res = 0;
+ goto out_unmap_unlock;
+ }
+
+@@ -3837,6 +3837,9 @@ int qlt_abort_cmd(struct qla_tgt_cmd *cmd)
+
+ spin_lock_irqsave(&cmd->cmd_lock, flags);
+ if (cmd->aborted) {
++ if (cmd->sg_mapped)
++ qlt_unmap_sg(vha, cmd);
++
+ spin_unlock_irqrestore(&cmd->cmd_lock, flags);
+ /*
+ * It's normal to see 2 calls in this path:
+@@ -7220,8 +7223,7 @@ qlt_probe_one_stage1(struct scsi_qla_host *base_vha, struct qla_hw_data *ha)
+ if (!QLA_TGT_MODE_ENABLED())
+ return;
+
+- if ((ql2xenablemsix == 0) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+- IS_QLA28XX(ha)) {
++ if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+ ISP_ATIO_Q_IN(base_vha) = &ha->mqiobase->isp25mq.atio_q_in;
+ ISP_ATIO_Q_OUT(base_vha) = &ha->mqiobase->isp25mq.atio_q_out;
+ } else {
+diff --git a/drivers/scsi/qla2xxx/qla_tmpl.c b/drivers/scsi/qla2xxx/qla_tmpl.c
+index 26c13a953b975..b0a74b036cf4b 100644
+--- a/drivers/scsi/qla2xxx/qla_tmpl.c
++++ b/drivers/scsi/qla2xxx/qla_tmpl.c
+@@ -435,8 +435,13 @@ qla27xx_fwdt_entry_t266(struct scsi_qla_host *vha,
+ {
+ ql_dbg(ql_dbg_misc, vha, 0xd20a,
+ "%s: reset risc [%lx]\n", __func__, *len);
+- if (buf)
+- WARN_ON_ONCE(qla24xx_soft_reset(vha->hw) != QLA_SUCCESS);
++ if (buf) {
++ if (qla24xx_soft_reset(vha->hw) != QLA_SUCCESS) {
++ ql_dbg(ql_dbg_async, vha, 0x5001,
++ "%s: unable to soft reset\n", __func__);
++ return INVALID_ENTRY;
++ }
++ }
+
+ return qla27xx_next_entry(ent);
+ }
+diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
+index 291ecc33b1fe6..4fc9466d820a7 100644
+--- a/drivers/scsi/scsi.c
++++ b/drivers/scsi/scsi.c
+@@ -209,11 +209,11 @@ void scsi_finish_command(struct scsi_cmnd *cmd)
+
+
+ /*
+- * 1024 is big enough for saturating the fast scsi LUN now
++ * 1024 is big enough for saturating fast SCSI LUNs.
+ */
+ int scsi_device_max_queue_depth(struct scsi_device *sdev)
+ {
+- return max_t(int, sdev->host->can_queue, 1024);
++ return min_t(int, sdev->host->can_queue, 1024);
+ }
+
+ /**
+diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
+index 66f507469a31a..cfeadd5f61f18 100644
+--- a/drivers/scsi/scsi_debug.c
++++ b/drivers/scsi/scsi_debug.c
+@@ -1189,7 +1189,7 @@ static int p_fill_from_dev_buffer(struct scsi_cmnd *scp, const void *arr,
+ __func__, off_dst, scsi_bufflen(scp), act_len,
+ scsi_get_resid(scp));
+ n = scsi_bufflen(scp) - (off_dst + act_len);
+- scsi_set_resid(scp, min_t(int, scsi_get_resid(scp), n));
++ scsi_set_resid(scp, min_t(u32, scsi_get_resid(scp), n));
+ return 0;
+ }
+
+@@ -1562,7 +1562,8 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+ unsigned char pq_pdt;
+ unsigned char *arr;
+ unsigned char *cmd = scp->cmnd;
+- int alloc_len, n, ret;
++ u32 alloc_len, n;
++ int ret;
+ bool have_wlun, is_disk, is_zbc, is_disk_zbc;
+
+ alloc_len = get_unaligned_be16(cmd + 3);
+@@ -1585,7 +1586,8 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+ kfree(arr);
+ return check_condition_result;
+ } else if (0x1 & cmd[1]) { /* EVPD bit set */
+- int lu_id_num, port_group_id, target_dev_id, len;
++ int lu_id_num, port_group_id, target_dev_id;
++ u32 len;
+ char lu_id_str[6];
+ int host_no = devip->sdbg_host->shost->host_no;
+
+@@ -1676,9 +1678,9 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+ kfree(arr);
+ return check_condition_result;
+ }
+- len = min(get_unaligned_be16(arr + 2) + 4, alloc_len);
++ len = min_t(u32, get_unaligned_be16(arr + 2) + 4, alloc_len);
+ ret = fill_from_dev_buffer(scp, arr,
+- min(len, SDEBUG_MAX_INQ_ARR_SZ));
++ min_t(u32, len, SDEBUG_MAX_INQ_ARR_SZ));
+ kfree(arr);
+ return ret;
+ }
+@@ -1714,7 +1716,7 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+ }
+ put_unaligned_be16(0x2100, arr + n); /* SPL-4 no version claimed */
+ ret = fill_from_dev_buffer(scp, arr,
+- min_t(int, alloc_len, SDEBUG_LONG_INQ_SZ));
++ min_t(u32, alloc_len, SDEBUG_LONG_INQ_SZ));
+ kfree(arr);
+ return ret;
+ }
+@@ -1729,8 +1731,8 @@ static int resp_requests(struct scsi_cmnd *scp,
+ unsigned char *cmd = scp->cmnd;
+ unsigned char arr[SCSI_SENSE_BUFFERSIZE]; /* assume >= 18 bytes */
+ bool dsense = !!(cmd[1] & 1);
+- int alloc_len = cmd[4];
+- int len = 18;
++ u32 alloc_len = cmd[4];
++ u32 len = 18;
+ int stopped_state = atomic_read(&devip->stopped);
+
+ memset(arr, 0, sizeof(arr));
+@@ -1774,7 +1776,7 @@ static int resp_requests(struct scsi_cmnd *scp,
+ arr[7] = 0xa;
+ }
+ }
+- return fill_from_dev_buffer(scp, arr, min_t(int, len, alloc_len));
++ return fill_from_dev_buffer(scp, arr, min_t(u32, len, alloc_len));
+ }
+
+ static int resp_start_stop(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+@@ -1856,7 +1858,7 @@ static int resp_readcap16(struct scsi_cmnd *scp,
+ {
+ unsigned char *cmd = scp->cmnd;
+ unsigned char arr[SDEBUG_READCAP16_ARR_SZ];
+- int alloc_len;
++ u32 alloc_len;
+
+ alloc_len = get_unaligned_be32(cmd + 10);
+ /* following just in case virtual_gb changed */
+@@ -1885,7 +1887,7 @@ static int resp_readcap16(struct scsi_cmnd *scp,
+ }
+
+ return fill_from_dev_buffer(scp, arr,
+- min_t(int, alloc_len, SDEBUG_READCAP16_ARR_SZ));
++ min_t(u32, alloc_len, SDEBUG_READCAP16_ARR_SZ));
+ }
+
+ #define SDEBUG_MAX_TGTPGS_ARR_SZ 1412
+@@ -1896,8 +1898,9 @@ static int resp_report_tgtpgs(struct scsi_cmnd *scp,
+ unsigned char *cmd = scp->cmnd;
+ unsigned char *arr;
+ int host_no = devip->sdbg_host->shost->host_no;
+- int n, ret, alen, rlen;
+ int port_group_a, port_group_b, port_a, port_b;
++ u32 alen, n, rlen;
++ int ret;
+
+ alen = get_unaligned_be32(cmd + 6);
+ arr = kzalloc(SDEBUG_MAX_TGTPGS_ARR_SZ, GFP_ATOMIC);
+@@ -1959,9 +1962,9 @@ static int resp_report_tgtpgs(struct scsi_cmnd *scp,
+ * - The constructed command length
+ * - The maximum array size
+ */
+- rlen = min_t(int, alen, n);
++ rlen = min(alen, n);
+ ret = fill_from_dev_buffer(scp, arr,
+- min_t(int, rlen, SDEBUG_MAX_TGTPGS_ARR_SZ));
++ min_t(u32, rlen, SDEBUG_MAX_TGTPGS_ARR_SZ));
+ kfree(arr);
+ return ret;
+ }
+@@ -2311,7 +2314,8 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
+ {
+ int pcontrol, pcode, subpcode, bd_len;
+ unsigned char dev_spec;
+- int alloc_len, offset, len, target_dev_id;
++ u32 alloc_len, offset, len;
++ int target_dev_id;
+ int target = scp->device->id;
+ unsigned char *ap;
+ unsigned char arr[SDEBUG_MAX_MSENSE_SZ];
+@@ -2467,7 +2471,7 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
+ arr[0] = offset - 1;
+ else
+ put_unaligned_be16((offset - 2), arr + 0);
+- return fill_from_dev_buffer(scp, arr, min_t(int, alloc_len, offset));
++ return fill_from_dev_buffer(scp, arr, min_t(u32, alloc_len, offset));
+ }
+
+ #define SDEBUG_MAX_MSELECT_SZ 512
+@@ -2498,11 +2502,11 @@ static int resp_mode_select(struct scsi_cmnd *scp,
+ __func__, param_len, res);
+ md_len = mselect6 ? (arr[0] + 1) : (get_unaligned_be16(arr + 0) + 2);
+ bd_len = mselect6 ? arr[3] : get_unaligned_be16(arr + 6);
+- if (md_len > 2) {
++ off = bd_len + (mselect6 ? 4 : 8);
++ if (md_len > 2 || off >= res) {
+ mk_sense_invalid_fld(scp, SDEB_IN_DATA, 0, -1);
+ return check_condition_result;
+ }
+- off = bd_len + (mselect6 ? 4 : 8);
+ mpage = arr[off] & 0x3f;
+ ps = !!(arr[off] & 0x80);
+ if (ps) {
+@@ -2582,7 +2586,8 @@ static int resp_ie_l_pg(unsigned char *arr)
+ static int resp_log_sense(struct scsi_cmnd *scp,
+ struct sdebug_dev_info *devip)
+ {
+- int ppc, sp, pcode, subpcode, alloc_len, len, n;
++ int ppc, sp, pcode, subpcode;
++ u32 alloc_len, len, n;
+ unsigned char arr[SDEBUG_MAX_LSENSE_SZ];
+ unsigned char *cmd = scp->cmnd;
+
+@@ -2652,9 +2657,9 @@ static int resp_log_sense(struct scsi_cmnd *scp,
+ mk_sense_invalid_fld(scp, SDEB_IN_CDB, 3, -1);
+ return check_condition_result;
+ }
+- len = min_t(int, get_unaligned_be16(arr + 2) + 4, alloc_len);
++ len = min_t(u32, get_unaligned_be16(arr + 2) + 4, alloc_len);
+ return fill_from_dev_buffer(scp, arr,
+- min_t(int, len, SDEBUG_MAX_INQ_ARR_SZ));
++ min_t(u32, len, SDEBUG_MAX_INQ_ARR_SZ));
+ }
+
+ static inline bool sdebug_dev_is_zoned(struct sdebug_dev_info *devip)
+@@ -4258,6 +4263,8 @@ static int resp_verify(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+ mk_sense_invalid_opcode(scp);
+ return check_condition_result;
+ }
++ if (vnum == 0)
++ return 0; /* not an error */
+ a_num = is_bytchk3 ? 1 : vnum;
+ /* Treat following check like one for read (i.e. no write) access */
+ ret = check_device_access_params(scp, lba, a_num, false);
+@@ -4321,6 +4328,8 @@ static int resp_report_zones(struct scsi_cmnd *scp,
+ }
+ zs_lba = get_unaligned_be64(cmd + 2);
+ alloc_len = get_unaligned_be32(cmd + 10);
++ if (alloc_len == 0)
++ return 0; /* not an error */
+ rep_opts = cmd[14] & 0x3f;
+ partial = cmd[14] & 0x80;
+
+@@ -4333,7 +4342,7 @@ static int resp_report_zones(struct scsi_cmnd *scp,
+ rep_max_zones = min((alloc_len - 64) >> ilog2(RZONES_DESC_HD),
+ max_zones);
+
+- arr = kcalloc(RZONES_DESC_HD, alloc_len, GFP_ATOMIC);
++ arr = kzalloc(alloc_len, GFP_ATOMIC);
+ if (!arr) {
+ mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
+ INSUFF_RES_ASCQ);
+@@ -4425,7 +4434,7 @@ static int resp_report_zones(struct scsi_cmnd *scp,
+ put_unaligned_be64(sdebug_capacity - 1, arr + 8);
+
+ rep_len = (unsigned long)desc - (unsigned long)arr;
+- ret = fill_from_dev_buffer(scp, arr, min_t(int, alloc_len, rep_len));
++ ret = fill_from_dev_buffer(scp, arr, min_t(u32, alloc_len, rep_len));
+
+ fini:
+ read_unlock(macc_lckp);
+@@ -4648,6 +4657,7 @@ static void zbc_rwp_zone(struct sdebug_dev_info *devip,
+ struct sdeb_zone_state *zsp)
+ {
+ enum sdebug_z_cond zc;
++ struct sdeb_store_info *sip = devip2sip(devip, false);
+
+ if (zbc_zone_is_conv(zsp))
+ return;
+@@ -4659,6 +4669,10 @@ static void zbc_rwp_zone(struct sdebug_dev_info *devip,
+ if (zsp->z_cond == ZC4_CLOSED)
+ devip->nr_closed--;
+
++ if (zsp->z_wp > zsp->z_start)
++ memset(sip->storep + zsp->z_start * sdebug_sector_size, 0,
++ (zsp->z_wp - zsp->z_start) * sdebug_sector_size);
++
+ zsp->z_non_seq_resource = false;
+ zsp->z_wp = zsp->z_start;
+ zsp->z_cond = ZC1_EMPTY;
+diff --git a/drivers/scsi/scsi_debugfs.c b/drivers/scsi/scsi_debugfs.c
+index d9109771f274d..db8517f1a485a 100644
+--- a/drivers/scsi/scsi_debugfs.c
++++ b/drivers/scsi/scsi_debugfs.c
+@@ -9,6 +9,7 @@
+ static const char *const scsi_cmd_flags[] = {
+ SCSI_CMD_FLAG_NAME(TAGGED),
+ SCSI_CMD_FLAG_NAME(INITIALIZED),
++ SCSI_CMD_FLAG_NAME(LAST),
+ };
+ #undef SCSI_CMD_FLAG_NAME
+
+diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
+index b6c86cce57bfa..bb5a6e0fa49ab 100644
+--- a/drivers/scsi/scsi_error.c
++++ b/drivers/scsi/scsi_error.c
+@@ -135,6 +135,23 @@ static bool scsi_eh_should_retry_cmd(struct scsi_cmnd *cmd)
+ return true;
+ }
+
++static void scsi_eh_complete_abort(struct scsi_cmnd *scmd, struct Scsi_Host *shost)
++{
++ unsigned long flags;
++
++ spin_lock_irqsave(shost->host_lock, flags);
++ list_del_init(&scmd->eh_entry);
++ /*
++ * If the abort succeeds, and there is no further
++ * EH action, clear the ->last_reset time.
++ */
++ if (list_empty(&shost->eh_abort_list) &&
++ list_empty(&shost->eh_cmd_q))
++ if (shost->eh_deadline != -1)
++ shost->last_reset = 0;
++ spin_unlock_irqrestore(shost->host_lock, flags);
++}
++
+ /**
+ * scmd_eh_abort_handler - Handle command aborts
+ * @work: command to be aborted.
+@@ -152,6 +169,7 @@ scmd_eh_abort_handler(struct work_struct *work)
+ container_of(work, struct scsi_cmnd, abort_work.work);
+ struct scsi_device *sdev = scmd->device;
+ enum scsi_disposition rtn;
++ unsigned long flags;
+
+ if (scsi_host_eh_past_deadline(sdev->host)) {
+ SCSI_LOG_ERROR_RECOVERY(3,
+@@ -175,12 +193,14 @@ scmd_eh_abort_handler(struct work_struct *work)
+ SCSI_LOG_ERROR_RECOVERY(3,
+ scmd_printk(KERN_WARNING, scmd,
+ "retry aborted command\n"));
++ scsi_eh_complete_abort(scmd, sdev->host);
+ scsi_queue_insert(scmd, SCSI_MLQUEUE_EH_RETRY);
+ return;
+ } else {
+ SCSI_LOG_ERROR_RECOVERY(3,
+ scmd_printk(KERN_WARNING, scmd,
+ "finish aborted command\n"));
++ scsi_eh_complete_abort(scmd, sdev->host);
+ scsi_finish_command(scmd);
+ return;
+ }
+@@ -193,6 +213,9 @@ scmd_eh_abort_handler(struct work_struct *work)
+ }
+ }
+
++ spin_lock_irqsave(sdev->host->host_lock, flags);
++ list_del_init(&scmd->eh_entry);
++ spin_unlock_irqrestore(sdev->host->host_lock, flags);
+ scsi_eh_scmd_add(scmd);
+ }
+
+@@ -223,6 +246,8 @@ scsi_abort_command(struct scsi_cmnd *scmd)
+ spin_lock_irqsave(shost->host_lock, flags);
+ if (shost->eh_deadline != -1 && !shost->last_reset)
+ shost->last_reset = jiffies;
++ BUG_ON(!list_empty(&scmd->eh_entry));
++ list_add_tail(&scmd->eh_entry, &shost->eh_abort_list);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ scmd->eh_eflags |= SCSI_EH_ABORT_SCHEDULED;
+@@ -460,8 +485,13 @@ static void scsi_report_sense(struct scsi_device *sdev,
+
+ if (sshdr->asc == 0x29) {
+ evt_type = SDEV_EVT_POWER_ON_RESET_OCCURRED;
+- sdev_printk(KERN_WARNING, sdev,
+- "Power-on or device reset occurred\n");
++ /*
++ * Do not print message if it is an expected side-effect
++ * of runtime PM.
++ */
++ if (!sdev->silence_suspend)
++ sdev_printk(KERN_WARNING, sdev,
++ "Power-on or device reset occurred\n");
+ }
+
+ if (sshdr->asc == 0x2a && sshdr->ascq == 0x01) {
+diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c
+index 6ff2207bd45a0..a06c61f22742e 100644
+--- a/drivers/scsi/scsi_ioctl.c
++++ b/drivers/scsi/scsi_ioctl.c
+@@ -347,6 +347,8 @@ static int scsi_fill_sghdr_rq(struct scsi_device *sdev, struct request *rq,
+ {
+ struct scsi_request *req = scsi_req(rq);
+
++ if (hdr->cmd_len < 6)
++ return -EMSGSIZE;
+ if (copy_from_user(req->cmd, hdr->cmdp, hdr->cmd_len))
+ return -EFAULT;
+ if (!scsi_cmd_allowed(req->cmd, mode))
+diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
+index 572673873ddf8..ef4361b2d1423 100644
+--- a/drivers/scsi/scsi_lib.c
++++ b/drivers/scsi/scsi_lib.c
+@@ -1143,6 +1143,7 @@ void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
+ cmd->sense_buffer = buf;
+ cmd->prot_sdb = prot;
+ cmd->flags = flags;
++ INIT_LIST_HEAD(&cmd->eh_entry);
+ INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
+ cmd->jiffies_at_alloc = jiffies_at_alloc;
+ cmd->retries = retries;
+@@ -1174,8 +1175,6 @@ static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,
+ }
+
+ cmd->cmd_len = scsi_req(req)->cmd_len;
+- if (cmd->cmd_len == 0)
+- cmd->cmd_len = scsi_command_size(cmd->cmnd);
+ cmd->cmnd = scsi_req(req)->cmd;
+ cmd->transfersize = blk_rq_bytes(req);
+ cmd->allowed = scsi_req(req)->retries;
+@@ -2075,7 +2074,7 @@ EXPORT_SYMBOL_GPL(scsi_mode_select);
+ /**
+ * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
+ * @sdev: SCSI device to be queried
+- * @dbd: set if mode sense will allow block descriptors to be returned
++ * @dbd: set to prevent mode sense from returning block descriptors
+ * @modepage: mode page being requested
+ * @buffer: request buffer (may not be smaller than eight bytes)
+ * @len: length of request buffer.
+@@ -2110,18 +2109,18 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
+ sshdr = &my_sshdr;
+
+ retry:
+- use_10_for_ms = sdev->use_10_for_ms;
++ use_10_for_ms = sdev->use_10_for_ms || len > 255;
+
+ if (use_10_for_ms) {
+- if (len < 8)
+- len = 8;
++ if (len < 8 || len > 65535)
++ return -EINVAL;
+
+ cmd[0] = MODE_SENSE_10;
+- cmd[8] = len;
++ put_unaligned_be16(len, &cmd[7]);
+ header_length = 8;
+ } else {
+ if (len < 4)
+- len = 4;
++ return -EINVAL;
+
+ cmd[0] = MODE_SENSE;
+ cmd[4] = len;
+@@ -2145,9 +2144,15 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
+ if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
+ (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
+ /*
+- * Invalid command operation code
++ * Invalid command operation code: retry using
++ * MODE SENSE(6) if this was a MODE SENSE(10)
++ * request, except if the request mode page is
++ * too large for MODE SENSE single byte
++ * allocation length field.
+ */
+ if (use_10_for_ms) {
++ if (len > 255)
++ return -EIO;
+ sdev->use_10_for_ms = 0;
+ goto retry;
+ }
+@@ -2171,12 +2176,11 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
+ data->longlba = 0;
+ data->block_descriptor_length = 0;
+ } else if (use_10_for_ms) {
+- data->length = buffer[0]*256 + buffer[1] + 2;
++ data->length = get_unaligned_be16(&buffer[0]) + 2;
+ data->medium_type = buffer[2];
+ data->device_specific = buffer[3];
+ data->longlba = buffer[4] & 0x01;
+- data->block_descriptor_length = buffer[6]*256
+- + buffer[7];
++ data->block_descriptor_length = get_unaligned_be16(&buffer[6]);
+ } else {
+ data->length = buffer[0] + 1;
+ data->medium_type = buffer[1];
+diff --git a/drivers/scsi/scsi_pm.c b/drivers/scsi/scsi_pm.c
+index 3717eea37ecb3..e91a0a5bc7a3e 100644
+--- a/drivers/scsi/scsi_pm.c
++++ b/drivers/scsi/scsi_pm.c
+@@ -262,7 +262,7 @@ static int sdev_runtime_resume(struct device *dev)
+ blk_pre_runtime_resume(sdev->request_queue);
+ if (pm && pm->runtime_resume)
+ err = pm->runtime_resume(dev);
+- blk_post_runtime_resume(sdev->request_queue, err);
++ blk_post_runtime_resume(sdev->request_queue);
+
+ return err;
+ }
+diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
+index fe22191522a3b..9466474ff01b6 100644
+--- a/drivers/scsi/scsi_scan.c
++++ b/drivers/scsi/scsi_scan.c
+@@ -198,6 +198,53 @@ static void scsi_unlock_floptical(struct scsi_device *sdev,
+ SCSI_TIMEOUT, 3, NULL);
+ }
+
++static int scsi_realloc_sdev_budget_map(struct scsi_device *sdev,
++ unsigned int depth)
++{
++ int new_shift = sbitmap_calculate_shift(depth);
++ bool need_alloc = !sdev->budget_map.map;
++ bool need_free = false;
++ int ret;
++ struct sbitmap sb_backup;
++
++ depth = min_t(unsigned int, depth, scsi_device_max_queue_depth(sdev));
++
++ /*
++ * realloc if new shift is calculated, which is caused by setting
++ * up one new default queue depth after calling ->slave_configure
++ */
++ if (!need_alloc && new_shift != sdev->budget_map.shift)
++ need_alloc = need_free = true;
++
++ if (!need_alloc)
++ return 0;
++
++ /*
++ * Request queue has to be frozen for reallocating budget map,
++ * and here disk isn't added yet, so freezing is pretty fast
++ */
++ if (need_free) {
++ blk_mq_freeze_queue(sdev->request_queue);
++ sb_backup = sdev->budget_map;
++ }
++ ret = sbitmap_init_node(&sdev->budget_map,
++ scsi_device_max_queue_depth(sdev),
++ new_shift, GFP_KERNEL,
++ sdev->request_queue->node, false, true);
++ if (!ret)
++ sbitmap_resize(&sdev->budget_map, depth);
++
++ if (need_free) {
++ if (ret)
++ sdev->budget_map = sb_backup;
++ else
++ sbitmap_free(&sb_backup);
++ ret = 0;
++ blk_mq_unfreeze_queue(sdev->request_queue);
++ }
++ return ret;
++}
++
+ /**
+ * scsi_alloc_sdev - allocate and setup a scsi_Device
+ * @starget: which target to allocate a &scsi_device for
+@@ -291,11 +338,7 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget,
+ * default device queue depth to figure out sbitmap shift
+ * since we use this queue depth most of times.
+ */
+- if (sbitmap_init_node(&sdev->budget_map,
+- scsi_device_max_queue_depth(sdev),
+- sbitmap_calculate_shift(depth),
+- GFP_KERNEL, sdev->request_queue->node,
+- false, true)) {
++ if (scsi_realloc_sdev_budget_map(sdev, depth)) {
+ put_device(&starget->dev);
+ kfree(sdev);
+ goto out;
+@@ -1001,6 +1044,13 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result,
+ }
+ return SCSI_SCAN_NO_RESPONSE;
+ }
++
++ /*
++ * The queue_depth is often changed in ->slave_configure.
++ * Set up budget map again since memory consumption of
++ * the map depends on actual queue depth.
++ */
++ scsi_realloc_sdev_budget_map(sdev, sdev->queue_depth);
+ }
+
+ if (sdev->scsi_level >= SCSI_3)
+diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
+index a35841b34bfd9..920aae661c5b2 100644
+--- a/drivers/scsi/scsi_sysfs.c
++++ b/drivers/scsi/scsi_sysfs.c
+@@ -797,6 +797,7 @@ store_state_field(struct device *dev, struct device_attribute *attr,
+ int i, ret;
+ struct scsi_device *sdev = to_scsi_device(dev);
+ enum scsi_device_state state = 0;
++ bool rescan_dev = false;
+
+ for (i = 0; i < ARRAY_SIZE(sdev_states); i++) {
+ const int len = strlen(sdev_states[i].name);
+@@ -815,20 +816,27 @@ store_state_field(struct device *dev, struct device_attribute *attr,
+ }
+
+ mutex_lock(&sdev->state_mutex);
+- ret = scsi_device_set_state(sdev, state);
+- /*
+- * If the device state changes to SDEV_RUNNING, we need to
+- * run the queue to avoid I/O hang, and rescan the device
+- * to revalidate it. Running the queue first is necessary
+- * because another thread may be waiting inside
+- * blk_mq_freeze_queue_wait() and because that call may be
+- * waiting for pending I/O to finish.
+- */
+- if (ret == 0 && state == SDEV_RUNNING) {
++ if (sdev->sdev_state == SDEV_RUNNING && state == SDEV_RUNNING) {
++ ret = 0;
++ } else {
++ ret = scsi_device_set_state(sdev, state);
++ if (ret == 0 && state == SDEV_RUNNING)
++ rescan_dev = true;
++ }
++ mutex_unlock(&sdev->state_mutex);
++
++ if (rescan_dev) {
++ /*
++ * If the device state changes to SDEV_RUNNING, we need to
++ * run the queue to avoid I/O hang, and rescan the device
++ * to revalidate it. Running the queue first is necessary
++ * because another thread may be waiting inside
++ * blk_mq_freeze_queue_wait() and because that call may be
++ * waiting for pending I/O to finish.
++ */
+ blk_mq_run_hw_queues(sdev->request_queue, true);
+ scsi_rescan_device(dev);
+ }
+- mutex_unlock(&sdev->state_mutex);
+
+ return ret == 0 ? count : -EINVAL;
+ }
+@@ -1388,6 +1396,7 @@ int scsi_sysfs_add_sdev(struct scsi_device *sdev)
+ * We're treating error on bsg register as non-fatal, so
+ * pretend nothing went wrong.
+ */
++ error = PTR_ERR(sdev->bsg_dev);
+ sdev_printk(KERN_INFO, sdev,
+ "Failed to register bsg queue, errno=%d\n",
+ error);
+diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
+index 60e406bcf42a9..a2524106206db 100644
+--- a/drivers/scsi/scsi_transport_fc.c
++++ b/drivers/scsi/scsi_transport_fc.c
+@@ -34,7 +34,7 @@ static int fc_bsg_hostadd(struct Scsi_Host *, struct fc_host_attrs *);
+ static int fc_bsg_rportadd(struct Scsi_Host *, struct fc_rport *);
+ static void fc_bsg_remove(struct request_queue *);
+ static void fc_bsg_goose_queue(struct fc_rport *);
+-static void fc_li_stats_update(struct fc_fn_li_desc *li_desc,
++static void fc_li_stats_update(u16 event_type,
+ struct fc_fpin_stats *stats);
+ static void fc_delivery_stats_update(u32 reason_code,
+ struct fc_fpin_stats *stats);
+@@ -670,42 +670,34 @@ fc_find_rport_by_wwpn(struct Scsi_Host *shost, u64 wwpn)
+ EXPORT_SYMBOL(fc_find_rport_by_wwpn);
+
+ static void
+-fc_li_stats_update(struct fc_fn_li_desc *li_desc,
++fc_li_stats_update(u16 event_type,
+ struct fc_fpin_stats *stats)
+ {
+- stats->li += be32_to_cpu(li_desc->event_count);
+- switch (be16_to_cpu(li_desc->event_type)) {
++ stats->li++;
++ switch (event_type) {
+ case FPIN_LI_UNKNOWN:
+- stats->li_failure_unknown +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_failure_unknown++;
+ break;
+ case FPIN_LI_LINK_FAILURE:
+- stats->li_link_failure_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_link_failure_count++;
+ break;
+ case FPIN_LI_LOSS_OF_SYNC:
+- stats->li_loss_of_sync_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_loss_of_sync_count++;
+ break;
+ case FPIN_LI_LOSS_OF_SIG:
+- stats->li_loss_of_signals_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_loss_of_signals_count++;
+ break;
+ case FPIN_LI_PRIM_SEQ_ERR:
+- stats->li_prim_seq_err_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_prim_seq_err_count++;
+ break;
+ case FPIN_LI_INVALID_TX_WD:
+- stats->li_invalid_tx_word_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_invalid_tx_word_count++;
+ break;
+ case FPIN_LI_INVALID_CRC:
+- stats->li_invalid_crc_count +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_invalid_crc_count++;
+ break;
+ case FPIN_LI_DEVICE_SPEC:
+- stats->li_device_specific +=
+- be32_to_cpu(li_desc->event_count);
++ stats->li_device_specific++;
+ break;
+ }
+ }
+@@ -767,6 +759,7 @@ fc_fpin_li_stats_update(struct Scsi_Host *shost, struct fc_tlv_desc *tlv)
+ struct fc_rport *attach_rport = NULL;
+ struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
+ struct fc_fn_li_desc *li_desc = (struct fc_fn_li_desc *)tlv;
++ u16 event_type = be16_to_cpu(li_desc->event_type);
+ u64 wwpn;
+
+ rport = fc_find_rport_by_wwpn(shost,
+@@ -775,7 +768,7 @@ fc_fpin_li_stats_update(struct Scsi_Host *shost, struct fc_tlv_desc *tlv)
+ (rport->roles & FC_PORT_ROLE_FCP_TARGET ||
+ rport->roles & FC_PORT_ROLE_NVME_TARGET)) {
+ attach_rport = rport;
+- fc_li_stats_update(li_desc, &attach_rport->fpin_stats);
++ fc_li_stats_update(event_type, &attach_rport->fpin_stats);
+ }
+
+ if (be32_to_cpu(li_desc->pname_count) > 0) {
+@@ -789,14 +782,14 @@ fc_fpin_li_stats_update(struct Scsi_Host *shost, struct fc_tlv_desc *tlv)
+ rport->roles & FC_PORT_ROLE_NVME_TARGET)) {
+ if (rport == attach_rport)
+ continue;
+- fc_li_stats_update(li_desc,
++ fc_li_stats_update(event_type,
+ &rport->fpin_stats);
+ }
+ }
+ }
+
+ if (fc_host->port_name == be64_to_cpu(li_desc->attached_wwpn))
+- fc_li_stats_update(li_desc, &fc_host->fpin_stats);
++ fc_li_stats_update(event_type, &fc_host->fpin_stats);
+ }
+
+ /*
+diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
+index 78343d3f93857..bcdfcb25349ad 100644
+--- a/drivers/scsi/scsi_transport_iscsi.c
++++ b/drivers/scsi/scsi_transport_iscsi.c
+@@ -86,6 +86,9 @@ struct iscsi_internal {
+ struct transport_container session_cont;
+ };
+
++static DEFINE_IDR(iscsi_ep_idr);
++static DEFINE_MUTEX(iscsi_ep_idr_mutex);
++
+ static atomic_t iscsi_session_nr; /* sysfs session id for next new session */
+ static struct workqueue_struct *iscsi_eh_timer_workq;
+
+@@ -169,6 +172,11 @@ struct device_attribute dev_attr_##_prefix##_##_name = \
+ static void iscsi_endpoint_release(struct device *dev)
+ {
+ struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev);
++
++ mutex_lock(&iscsi_ep_idr_mutex);
++ idr_remove(&iscsi_ep_idr, ep->id);
++ mutex_unlock(&iscsi_ep_idr_mutex);
++
+ kfree(ep);
+ }
+
+@@ -181,7 +189,7 @@ static ssize_t
+ show_ep_handle(struct device *dev, struct device_attribute *attr, char *buf)
+ {
+ struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev);
+- return sysfs_emit(buf, "%llu\n", (unsigned long long) ep->id);
++ return sysfs_emit(buf, "%d\n", ep->id);
+ }
+ static ISCSI_ATTR(ep, handle, S_IRUGO, show_ep_handle, NULL);
+
+@@ -194,48 +202,32 @@ static struct attribute_group iscsi_endpoint_group = {
+ .attrs = iscsi_endpoint_attrs,
+ };
+
+-#define ISCSI_MAX_EPID -1
+-
+-static int iscsi_match_epid(struct device *dev, const void *data)
+-{
+- struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev);
+- const uint64_t *epid = data;
+-
+- return *epid == ep->id;
+-}
+-
+ struct iscsi_endpoint *
+ iscsi_create_endpoint(int dd_size)
+ {
+- struct device *dev;
+ struct iscsi_endpoint *ep;
+- uint64_t id;
+- int err;
+-
+- for (id = 1; id < ISCSI_MAX_EPID; id++) {
+- dev = class_find_device(&iscsi_endpoint_class, NULL, &id,
+- iscsi_match_epid);
+- if (!dev)
+- break;
+- else
+- put_device(dev);
+- }
+- if (id == ISCSI_MAX_EPID) {
+- printk(KERN_ERR "Too many connections. Max supported %u\n",
+- ISCSI_MAX_EPID - 1);
+- return NULL;
+- }
++ int err, id;
+
+ ep = kzalloc(sizeof(*ep) + dd_size, GFP_KERNEL);
+ if (!ep)
+ return NULL;
+
++ mutex_lock(&iscsi_ep_idr_mutex);
++ id = idr_alloc(&iscsi_ep_idr, ep, 0, -1, GFP_NOIO);
++ if (id < 0) {
++ mutex_unlock(&iscsi_ep_idr_mutex);
++ printk(KERN_ERR "Could not allocate endpoint ID. Error %d.\n",
++ id);
++ goto free_ep;
++ }
++ mutex_unlock(&iscsi_ep_idr_mutex);
++
+ ep->id = id;
+ ep->dev.class = &iscsi_endpoint_class;
+- dev_set_name(&ep->dev, "ep-%llu", (unsigned long long) id);
++ dev_set_name(&ep->dev, "ep-%d", id);
+ err = device_register(&ep->dev);
+ if (err)
+- goto free_ep;
++ goto free_id;
+
+ err = sysfs_create_group(&ep->dev.kobj, &iscsi_endpoint_group);
+ if (err)
+@@ -249,6 +241,10 @@ unregister_dev:
+ device_unregister(&ep->dev);
+ return NULL;
+
++free_id:
++ mutex_lock(&iscsi_ep_idr_mutex);
++ idr_remove(&iscsi_ep_idr, id);
++ mutex_unlock(&iscsi_ep_idr_mutex);
+ free_ep:
+ kfree(ep);
+ return NULL;
+@@ -276,14 +272,17 @@ EXPORT_SYMBOL_GPL(iscsi_put_endpoint);
+ */
+ struct iscsi_endpoint *iscsi_lookup_endpoint(u64 handle)
+ {
+- struct device *dev;
++ struct iscsi_endpoint *ep;
+
+- dev = class_find_device(&iscsi_endpoint_class, NULL, &handle,
+- iscsi_match_epid);
+- if (!dev)
+- return NULL;
++ mutex_lock(&iscsi_ep_idr_mutex);
++ ep = idr_find(&iscsi_ep_idr, handle);
++ if (!ep)
++ goto unlock;
+
+- return iscsi_dev_to_endpoint(dev);
++ get_device(&ep->dev);
++unlock:
++ mutex_unlock(&iscsi_ep_idr_mutex);
++ return ep;
+ }
+ EXPORT_SYMBOL_GPL(iscsi_lookup_endpoint);
+
+@@ -1899,12 +1898,12 @@ static void session_recovery_timedout(struct work_struct *work)
+ }
+ spin_unlock_irqrestore(&session->lock, flags);
+
+- if (session->transport->session_recovery_timedout)
+- session->transport->session_recovery_timedout(session);
+-
+ ISCSI_DBG_TRANS_SESSION(session, "Unblocking SCSI target\n");
+ scsi_target_unblock(&session->dev, SDEV_TRANSPORT_OFFLINE);
+ ISCSI_DBG_TRANS_SESSION(session, "Completed unblocking SCSI target\n");
++
++ if (session->transport->session_recovery_timedout)
++ session->transport->session_recovery_timedout(session);
+ }
+
+ static void __iscsi_unblock_session(struct work_struct *work)
+@@ -2221,10 +2220,10 @@ static void iscsi_stop_conn(struct iscsi_cls_conn *conn, int flag)
+
+ switch (flag) {
+ case STOP_CONN_RECOVER:
+- conn->state = ISCSI_CONN_FAILED;
++ WRITE_ONCE(conn->state, ISCSI_CONN_FAILED);
+ break;
+ case STOP_CONN_TERM:
+- conn->state = ISCSI_CONN_DOWN;
++ WRITE_ONCE(conn->state, ISCSI_CONN_DOWN);
+ break;
+ default:
+ iscsi_cls_conn_printk(KERN_ERR, conn, "invalid stop flag %d\n",
+@@ -2236,6 +2235,49 @@ static void iscsi_stop_conn(struct iscsi_cls_conn *conn, int flag)
+ ISCSI_DBG_TRANS_CONN(conn, "Stopping conn done.\n");
+ }
+
++static void iscsi_ep_disconnect(struct iscsi_cls_conn *conn, bool is_active)
++{
++ struct iscsi_cls_session *session = iscsi_conn_to_session(conn);
++ struct iscsi_endpoint *ep;
++
++ ISCSI_DBG_TRANS_CONN(conn, "disconnect ep.\n");
++ WRITE_ONCE(conn->state, ISCSI_CONN_FAILED);
++
++ if (!conn->ep || !session->transport->ep_disconnect)
++ return;
++
++ ep = conn->ep;
++ conn->ep = NULL;
++
++ session->transport->unbind_conn(conn, is_active);
++ session->transport->ep_disconnect(ep);
++ ISCSI_DBG_TRANS_CONN(conn, "disconnect ep done.\n");
++}
++
++static void iscsi_if_disconnect_bound_ep(struct iscsi_cls_conn *conn,
++ struct iscsi_endpoint *ep,
++ bool is_active)
++{
++ /* Check if this was a conn error and the kernel took ownership */
++ spin_lock_irq(&conn->lock);
++ if (!test_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
++ spin_unlock_irq(&conn->lock);
++ iscsi_ep_disconnect(conn, is_active);
++ } else {
++ spin_unlock_irq(&conn->lock);
++ ISCSI_DBG_TRANS_CONN(conn, "flush kernel conn cleanup.\n");
++ mutex_unlock(&conn->ep_mutex);
++
++ flush_work(&conn->cleanup_work);
++ /*
++ * Userspace is now done with the EP so we can release the ref
++ * iscsi_cleanup_conn_work_fn took.
++ */
++ iscsi_put_endpoint(ep);
++ mutex_lock(&conn->ep_mutex);
++ }
++}
++
+ static int iscsi_if_stop_conn(struct iscsi_transport *transport,
+ struct iscsi_uevent *ev)
+ {
+@@ -2256,12 +2298,25 @@ static int iscsi_if_stop_conn(struct iscsi_transport *transport,
+ cancel_work_sync(&conn->cleanup_work);
+ iscsi_stop_conn(conn, flag);
+ } else {
++ /*
++ * For offload, when iscsid is restarted it won't know about
++ * existing endpoints so it can't do a ep_disconnect. We clean
++ * it up here for userspace.
++ */
++ mutex_lock(&conn->ep_mutex);
++ if (conn->ep)
++ iscsi_if_disconnect_bound_ep(conn, conn->ep, true);
++ mutex_unlock(&conn->ep_mutex);
++
+ /*
+ * Figure out if it was the kernel or userspace initiating this.
+ */
++ spin_lock_irq(&conn->lock);
+ if (!test_and_set_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
++ spin_unlock_irq(&conn->lock);
+ iscsi_stop_conn(conn, flag);
+ } else {
++ spin_unlock_irq(&conn->lock);
+ ISCSI_DBG_TRANS_CONN(conn,
+ "flush kernel conn cleanup.\n");
+ flush_work(&conn->cleanup_work);
+@@ -2270,31 +2325,14 @@ static int iscsi_if_stop_conn(struct iscsi_transport *transport,
+ * Only clear for recovery to avoid extra cleanup runs during
+ * termination.
+ */
++ spin_lock_irq(&conn->lock);
+ clear_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags);
++ spin_unlock_irq(&conn->lock);
+ }
+ ISCSI_DBG_TRANS_CONN(conn, "iscsi if conn stop done.\n");
+ return 0;
+ }
+
+-static void iscsi_ep_disconnect(struct iscsi_cls_conn *conn, bool is_active)
+-{
+- struct iscsi_cls_session *session = iscsi_conn_to_session(conn);
+- struct iscsi_endpoint *ep;
+-
+- ISCSI_DBG_TRANS_CONN(conn, "disconnect ep.\n");
+- conn->state = ISCSI_CONN_FAILED;
+-
+- if (!conn->ep || !session->transport->ep_disconnect)
+- return;
+-
+- ep = conn->ep;
+- conn->ep = NULL;
+-
+- session->transport->unbind_conn(conn, is_active);
+- session->transport->ep_disconnect(ep);
+- ISCSI_DBG_TRANS_CONN(conn, "disconnect ep done.\n");
+-}
+-
+ static void iscsi_cleanup_conn_work_fn(struct work_struct *work)
+ {
+ struct iscsi_cls_conn *conn = container_of(work, struct iscsi_cls_conn,
+@@ -2303,18 +2341,11 @@ static void iscsi_cleanup_conn_work_fn(struct work_struct *work)
+
+ mutex_lock(&conn->ep_mutex);
+ /*
+- * If we are not at least bound there is nothing for us to do. Userspace
+- * will do a ep_disconnect call if offload is used, but will not be
+- * doing a stop since there is nothing to clean up, so we have to clear
+- * the cleanup bit here.
++ * Get a ref to the ep, so we don't release its ID until after
++ * userspace is done referencing it in iscsi_if_disconnect_bound_ep.
+ */
+- if (conn->state != ISCSI_CONN_BOUND && conn->state != ISCSI_CONN_UP) {
+- ISCSI_DBG_TRANS_CONN(conn, "Got error while conn is already failed. Ignoring.\n");
+- clear_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags);
+- mutex_unlock(&conn->ep_mutex);
+- return;
+- }
+-
++ if (conn->ep)
++ get_device(&conn->ep->dev);
+ iscsi_ep_disconnect(conn, false);
+
+ if (system_state != SYSTEM_RUNNING) {
+@@ -2370,11 +2401,12 @@ iscsi_create_conn(struct iscsi_cls_session *session, int dd_size, uint32_t cid)
+ conn->dd_data = &conn[1];
+
+ mutex_init(&conn->ep_mutex);
++ spin_lock_init(&conn->lock);
+ INIT_LIST_HEAD(&conn->conn_list);
+ INIT_WORK(&conn->cleanup_work, iscsi_cleanup_conn_work_fn);
+ conn->transport = transport;
+ conn->cid = cid;
+- conn->state = ISCSI_CONN_DOWN;
++ WRITE_ONCE(conn->state, ISCSI_CONN_DOWN);
+
+ /* this is released in the dev's release function */
+ if (!get_device(&session->dev))
+@@ -2561,9 +2593,32 @@ void iscsi_conn_error_event(struct iscsi_cls_conn *conn, enum iscsi_err error)
+ struct iscsi_uevent *ev;
+ struct iscsi_internal *priv;
+ int len = nlmsg_total_size(sizeof(*ev));
++ unsigned long flags;
++ int state;
+
+- if (!test_and_set_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags))
+- queue_work(iscsi_conn_cleanup_workq, &conn->cleanup_work);
++ spin_lock_irqsave(&conn->lock, flags);
++ /*
++ * Userspace will only do a stop call if we are at least bound. And, we
++ * only need to do the in kernel cleanup if in the UP state so cmds can
++ * be released to upper layers. If in other states just wait for
++ * userspace to avoid races that can leave the cleanup_work queued.
++ */
++ state = READ_ONCE(conn->state);
++ switch (state) {
++ case ISCSI_CONN_BOUND:
++ case ISCSI_CONN_UP:
++ if (!test_and_set_bit(ISCSI_CLS_CONN_BIT_CLEANUP,
++ &conn->flags)) {
++ queue_work(iscsi_conn_cleanup_workq,
++ &conn->cleanup_work);
++ }
++ break;
++ default:
++ ISCSI_DBG_TRANS_CONN(conn, "Got conn error in state %d\n",
++ state);
++ break;
++ }
++ spin_unlock_irqrestore(&conn->lock, flags);
+
+ priv = iscsi_if_transport_lookup(conn->transport);
+ if (!priv)
+@@ -2913,7 +2968,7 @@ iscsi_set_param(struct iscsi_transport *transport, struct iscsi_uevent *ev)
+ char *data = (char*)ev + sizeof(*ev);
+ struct iscsi_cls_conn *conn;
+ struct iscsi_cls_session *session;
+- int err = 0, value = 0;
++ int err = 0, value = 0, state;
+
+ if (ev->u.set_param.len > PAGE_SIZE)
+ return -EINVAL;
+@@ -2930,8 +2985,8 @@ iscsi_set_param(struct iscsi_transport *transport, struct iscsi_uevent *ev)
+ session->recovery_tmo = value;
+ break;
+ default:
+- if ((conn->state == ISCSI_CONN_BOUND) ||
+- (conn->state == ISCSI_CONN_UP)) {
++ state = READ_ONCE(conn->state);
++ if (state == ISCSI_CONN_BOUND || state == ISCSI_CONN_UP) {
+ err = transport->set_param(conn, ev->u.set_param.param,
+ data, ev->u.set_param.len);
+ } else {
+@@ -3003,16 +3058,7 @@ static int iscsi_if_ep_disconnect(struct iscsi_transport *transport,
+ }
+
+ mutex_lock(&conn->ep_mutex);
+- /* Check if this was a conn error and the kernel took ownership */
+- if (test_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
+- ISCSI_DBG_TRANS_CONN(conn, "flush kernel conn cleanup.\n");
+- mutex_unlock(&conn->ep_mutex);
+-
+- flush_work(&conn->cleanup_work);
+- goto put_ep;
+- }
+-
+- iscsi_ep_disconnect(conn, false);
++ iscsi_if_disconnect_bound_ep(conn, ep, false);
+ mutex_unlock(&conn->ep_mutex);
+ put_ep:
+ iscsi_put_endpoint(ep);
+@@ -3715,24 +3761,17 @@ static int iscsi_if_transport_conn(struct iscsi_transport *transport,
+ return -EINVAL;
+
+ mutex_lock(&conn->ep_mutex);
++ spin_lock_irq(&conn->lock);
+ if (test_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
++ spin_unlock_irq(&conn->lock);
+ mutex_unlock(&conn->ep_mutex);
+ ev->r.retcode = -ENOTCONN;
+ return 0;
+ }
++ spin_unlock_irq(&conn->lock);
+
+ switch (nlh->nlmsg_type) {
+ case ISCSI_UEVENT_BIND_CONN:
+- if (conn->ep) {
+- /*
+- * For offload boot support where iscsid is restarted
+- * during the pivot root stage, the ep will be intact
+- * here when the new iscsid instance starts up and
+- * reconnects.
+- */
+- iscsi_ep_disconnect(conn, true);
+- }
+-
+ session = iscsi_session_lookup(ev->u.b_conn.sid);
+ if (!session) {
+ err = -EINVAL;
+@@ -3743,7 +3782,7 @@ static int iscsi_if_transport_conn(struct iscsi_transport *transport,
+ ev->u.b_conn.transport_eph,
+ ev->u.b_conn.is_leading);
+ if (!ev->r.retcode)
+- conn->state = ISCSI_CONN_BOUND;
++ WRITE_ONCE(conn->state, ISCSI_CONN_BOUND);
+
+ if (ev->r.retcode || !transport->ep_connect)
+ break;
+@@ -3762,7 +3801,8 @@ static int iscsi_if_transport_conn(struct iscsi_transport *transport,
+ case ISCSI_UEVENT_START_CONN:
+ ev->r.retcode = transport->start_conn(conn);
+ if (!ev->r.retcode)
+- conn->state = ISCSI_CONN_UP;
++ WRITE_ONCE(conn->state, ISCSI_CONN_UP);
++
+ break;
+ case ISCSI_UEVENT_SEND_PDU:
+ pdu_len = nlh->nlmsg_len - sizeof(*nlh) - sizeof(*ev);
+@@ -4070,10 +4110,11 @@ static ssize_t show_conn_state(struct device *dev,
+ {
+ struct iscsi_cls_conn *conn = iscsi_dev_to_conn(dev->parent);
+ const char *state = "unknown";
++ int conn_state = READ_ONCE(conn->state);
+
+- if (conn->state >= 0 &&
+- conn->state < ARRAY_SIZE(connection_state_names))
+- state = connection_state_names[conn->state];
++ if (conn_state >= 0 &&
++ conn_state < ARRAY_SIZE(connection_state_names))
++ state = connection_state_names[conn_state];
+
+ return sysfs_emit(buf, "%s\n", state);
+ }
+diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
+index fce63335084ed..a713babaee0f5 100644
+--- a/drivers/scsi/sd.c
++++ b/drivers/scsi/sd.c
+@@ -48,6 +48,7 @@
+ #include <linux/blkpg.h>
+ #include <linux/blk-pm.h>
+ #include <linux/delay.h>
++#include <linux/major.h>
+ #include <linux/mutex.h>
+ #include <linux/string_helpers.h>
+ #include <linux/async.h>
+@@ -2607,6 +2608,13 @@ sd_do_mode_sense(struct scsi_disk *sdkp, int dbd, int modepage,
+ unsigned char *buffer, int len, struct scsi_mode_data *data,
+ struct scsi_sense_hdr *sshdr)
+ {
++ /*
++ * If we must use MODE SENSE(10), make sure that the buffer length
++ * is at least 8 bytes so that the mode sense header fits.
++ */
++ if (sdkp->device->use_10_for_ms && len < 8)
++ len = 8;
++
+ return scsi_mode_sense(sdkp->device, dbd, modepage, buffer, len,
+ SD_TIMEOUT, sdkp->max_retries, data,
+ sshdr);
+@@ -3620,7 +3628,8 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors)
+ return 0;
+
+ if (sdkp->WCE && sdkp->media_present) {
+- sd_printk(KERN_NOTICE, sdkp, "Synchronizing SCSI cache\n");
++ if (!sdkp->device->silence_suspend)
++ sd_printk(KERN_NOTICE, sdkp, "Synchronizing SCSI cache\n");
+ ret = sd_sync_cache(sdkp, &sshdr);
+
+ if (ret) {
+@@ -3642,7 +3651,8 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors)
+ }
+
+ if (sdkp->device->manage_start_stop) {
+- sd_printk(KERN_NOTICE, sdkp, "Stopping disk\n");
++ if (!sdkp->device->silence_suspend)
++ sd_printk(KERN_NOTICE, sdkp, "Stopping disk\n");
+ /* an error is not worth aborting a system sleep */
+ ret = sd_start_stop_device(sdkp, 0);
+ if (ignore_stop_errors)
+diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
+index 8f05248920e8e..3c98f08dc25d9 100644
+--- a/drivers/scsi/sg.c
++++ b/drivers/scsi/sg.c
+@@ -31,6 +31,7 @@ static int sg_version_num = 30536; /* 2 digits for each component */
+ #include <linux/errno.h>
+ #include <linux/mtio.h>
+ #include <linux/ioctl.h>
++#include <linux/major.h>
+ #include <linux/slab.h>
+ #include <linux/fcntl.h>
+ #include <linux/init.h>
+diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
+index ecb2af3f43ca3..2e690d8a34440 100644
+--- a/drivers/scsi/smartpqi/smartpqi_init.c
++++ b/drivers/scsi/smartpqi/smartpqi_init.c
+@@ -234,15 +234,46 @@ static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
+ return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
+ }
+
++#define PQI_DRIVER_SCRATCH_PQI_MODE 0x1
++#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED 0x2
++
+ static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
+ {
+- return sis_read_driver_scratch(ctrl_info);
++ return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
+ }
+
+ static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
+ enum pqi_ctrl_mode mode)
+ {
+- sis_write_driver_scratch(ctrl_info, mode);
++ u32 driver_scratch;
++
++ driver_scratch = sis_read_driver_scratch(ctrl_info);
++
++ if (mode == PQI_MODE)
++ driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
++ else
++ driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
++
++ sis_write_driver_scratch(ctrl_info, driver_scratch);
++}
++
++static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
++{
++ return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
++}
++
++static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
++{
++ u32 driver_scratch;
++
++ driver_scratch = sis_read_driver_scratch(ctrl_info);
++
++ if (is_supported)
++ driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
++ else
++ driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
++
++ sis_write_driver_scratch(ctrl_info, driver_scratch);
+ }
+
+ static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
+@@ -7301,6 +7332,7 @@ static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
+ ctrl_info->unique_wwid_in_report_phys_lun_supported =
+ firmware_feature->enabled;
+ break;
++ pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
+ }
+
+ pqi_firmware_feature_status(ctrl_info, firmware_feature);
+@@ -7621,12 +7653,32 @@ static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
+ return pqi_revert_to_sis_mode(ctrl_info);
+ }
+
++static void pqi_perform_lockup_action(void)
++{
++ switch (pqi_lockup_action) {
++ case PANIC:
++ panic("FATAL: Smart Family Controller lockup detected");
++ break;
++ case REBOOT:
++ emergency_restart();
++ break;
++ case NONE:
++ default:
++ break;
++ }
++}
++
+ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
+ {
+ int rc;
+ u32 product_id;
+
+ if (reset_devices) {
++ if (pqi_is_fw_triage_supported(ctrl_info)) {
++ rc = sis_wait_for_fw_triage_completion(ctrl_info);
++ if (rc)
++ return rc;
++ }
+ sis_soft_reset(ctrl_info);
+ msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
+ } else {
+@@ -7640,8 +7692,15 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
+ * commands.
+ */
+ rc = sis_wait_for_ctrl_ready(ctrl_info);
+- if (rc)
++ if (rc) {
++ if (reset_devices) {
++ dev_err(&ctrl_info->pci_dev->dev,
++ "kdump init failed with error %d\n", rc);
++ pqi_lockup_action = REBOOT;
++ pqi_perform_lockup_action();
++ }
+ return rc;
++ }
+
+ /*
+ * Get the controller properties. This allows us to determine
+@@ -8365,21 +8424,6 @@ static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int de
+ return pqi_ctrl_init_resume(ctrl_info);
+ }
+
+-static void pqi_perform_lockup_action(void)
+-{
+- switch (pqi_lockup_action) {
+- case PANIC:
+- panic("FATAL: Smart Family Controller lockup detected");
+- break;
+- case REBOOT:
+- emergency_restart();
+- break;
+- case NONE:
+- default:
+- break;
+- }
+-}
+-
+ static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
+ .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
+ .status = SAM_STAT_CHECK_CONDITION,
+diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
+index d63c46a8e38bb..8acd3a80f5822 100644
+--- a/drivers/scsi/smartpqi/smartpqi_sis.c
++++ b/drivers/scsi/smartpqi/smartpqi_sis.c
+@@ -51,12 +51,20 @@
+ #define SIS_BASE_STRUCT_REVISION 9
+ #define SIS_BASE_STRUCT_ALIGNMENT 16
+
++#define SIS_CTRL_KERNEL_FW_TRIAGE 0x3
+ #define SIS_CTRL_KERNEL_UP 0x80
+ #define SIS_CTRL_KERNEL_PANIC 0x100
+ #define SIS_CTRL_READY_TIMEOUT_SECS 180
+ #define SIS_CTRL_READY_RESUME_TIMEOUT_SECS 90
+ #define SIS_CTRL_READY_POLL_INTERVAL_MSECS 10
+
++enum sis_fw_triage_status {
++ FW_TRIAGE_NOT_STARTED = 0,
++ FW_TRIAGE_STARTED,
++ FW_TRIAGE_COND_INVALID,
++ FW_TRIAGE_COMPLETED
++};
++
+ #pragma pack(1)
+
+ /* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
+@@ -419,12 +427,55 @@ u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
+ return readl(&ctrl_info->registers->sis_driver_scratch);
+ }
+
++static inline enum sis_fw_triage_status
++ sis_read_firmware_triage_status(struct pqi_ctrl_info *ctrl_info)
++{
++ return ((enum sis_fw_triage_status)(readl(&ctrl_info->registers->sis_firmware_status) &
++ SIS_CTRL_KERNEL_FW_TRIAGE));
++}
++
+ void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
+ {
+ writel(SIS_SOFT_RESET,
+ &ctrl_info->registers->sis_host_to_ctrl_doorbell);
+ }
+
++#define SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS 300
++#define SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS 1
++
++int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info)
++{
++ int rc;
++ enum sis_fw_triage_status status;
++ unsigned long timeout;
++
++ timeout = (SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
++ while (1) {
++ status = sis_read_firmware_triage_status(ctrl_info);
++ if (status == FW_TRIAGE_COND_INVALID) {
++ dev_err(&ctrl_info->pci_dev->dev,
++ "firmware triage condition invalid\n");
++ rc = -EINVAL;
++ break;
++ } else if (status == FW_TRIAGE_NOT_STARTED ||
++ status == FW_TRIAGE_COMPLETED) {
++ rc = 0;
++ break;
++ }
++
++ if (time_after(jiffies, timeout)) {
++ dev_err(&ctrl_info->pci_dev->dev,
++ "timed out waiting for firmware triage status\n");
++ rc = -ETIMEDOUT;
++ break;
++ }
++
++ ssleep(SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS);
++ }
++
++ return rc;
++}
++
+ static void __attribute__((unused)) verify_structures(void)
+ {
+ BUILD_BUG_ON(offsetof(struct sis_base_struct,
+diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
+index d29c1352a826a..c1db93054c863 100644
+--- a/drivers/scsi/smartpqi/smartpqi_sis.h
++++ b/drivers/scsi/smartpqi/smartpqi_sis.h
+@@ -28,5 +28,6 @@ void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
+ u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
+ void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
+ u32 sis_get_product_id(struct pqi_ctrl_info *ctrl_info);
++int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info);
+
+ #endif /* _SMARTPQI_SIS_H */
+diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
+index 8b17b35283aa5..652cd81d77753 100644
+--- a/drivers/scsi/sr.c
++++ b/drivers/scsi/sr.c
+@@ -44,6 +44,7 @@
+ #include <linux/cdrom.h>
+ #include <linux/interrupt.h>
+ #include <linux/init.h>
++#include <linux/major.h>
+ #include <linux/blkdev.h>
+ #include <linux/blk-pm.h>
+ #include <linux/mutex.h>
+@@ -578,7 +579,7 @@ static int sr_block_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd,
+
+ scsi_autopm_get_device(sdev);
+
+- if (ret != CDROMCLOSETRAY && ret != CDROMEJECT) {
++ if (cmd != CDROMCLOSETRAY && cmd != CDROMEJECT) {
+ ret = cdrom_ioctl(&cd->cdi, bdev, mode, cmd, arg);
+ if (ret != -ENOSYS)
+ goto put;
+@@ -851,7 +852,7 @@ static void get_capabilities(struct scsi_cd *cd)
+
+
+ /* allocate transfer buffer */
+- buffer = kmalloc(512, GFP_KERNEL | GFP_DMA);
++ buffer = kmalloc(512, GFP_KERNEL);
+ if (!buffer) {
+ sr_printk(KERN_ERR, cd, "out of memory.\n");
+ return;
+diff --git a/drivers/scsi/sr_ioctl.c b/drivers/scsi/sr_ioctl.c
+index ddd00efc48825..fbdb5124d7f7d 100644
+--- a/drivers/scsi/sr_ioctl.c
++++ b/drivers/scsi/sr_ioctl.c
+@@ -41,7 +41,7 @@ static int sr_read_tochdr(struct cdrom_device_info *cdi,
+ int result;
+ unsigned char *buffer;
+
+- buffer = kmalloc(32, GFP_KERNEL);
++ buffer = kzalloc(32, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+@@ -55,10 +55,13 @@ static int sr_read_tochdr(struct cdrom_device_info *cdi,
+ cgc.data_direction = DMA_FROM_DEVICE;
+
+ result = sr_do_ioctl(cd, &cgc);
++ if (result)
++ goto err;
+
+ tochdr->cdth_trk0 = buffer[2];
+ tochdr->cdth_trk1 = buffer[3];
+
++err:
+ kfree(buffer);
+ return result;
+ }
+@@ -71,7 +74,7 @@ static int sr_read_tocentry(struct cdrom_device_info *cdi,
+ int result;
+ unsigned char *buffer;
+
+- buffer = kmalloc(32, GFP_KERNEL);
++ buffer = kzalloc(32, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+@@ -86,6 +89,8 @@ static int sr_read_tocentry(struct cdrom_device_info *cdi,
+ cgc.data_direction = DMA_FROM_DEVICE;
+
+ result = sr_do_ioctl(cd, &cgc);
++ if (result)
++ goto err;
+
+ tocentry->cdte_ctrl = buffer[5] & 0xf;
+ tocentry->cdte_adr = buffer[5] >> 4;
+@@ -98,6 +103,7 @@ static int sr_read_tocentry(struct cdrom_device_info *cdi,
+ tocentry->cdte_addr.lba = (((((buffer[8] << 8) + buffer[9]) << 8)
+ + buffer[10]) << 8) + buffer[11];
+
++err:
+ kfree(buffer);
+ return result;
+ }
+@@ -384,7 +390,7 @@ int sr_get_mcn(struct cdrom_device_info *cdi, struct cdrom_mcn *mcn)
+ {
+ Scsi_CD *cd = cdi->handle;
+ struct packet_command cgc;
+- char *buffer = kmalloc(32, GFP_KERNEL);
++ char *buffer = kzalloc(32, GFP_KERNEL);
+ int result;
+
+ if (!buffer)
+@@ -400,10 +406,13 @@ int sr_get_mcn(struct cdrom_device_info *cdi, struct cdrom_mcn *mcn)
+ cgc.data_direction = DMA_FROM_DEVICE;
+ cgc.timeout = IOCTL_TIMEOUT;
+ result = sr_do_ioctl(cd, &cgc);
++ if (result)
++ goto err;
+
+ memcpy(mcn->medium_catalog_number, buffer + 9, 13);
+ mcn->medium_catalog_number[13] = 0;
+
++err:
+ kfree(buffer);
+ return result;
+ }
+diff --git a/drivers/scsi/sr_vendor.c b/drivers/scsi/sr_vendor.c
+index 1f988a1b9166f..a61635326ae0a 100644
+--- a/drivers/scsi/sr_vendor.c
++++ b/drivers/scsi/sr_vendor.c
+@@ -131,7 +131,7 @@ int sr_set_blocklength(Scsi_CD *cd, int blocklength)
+ if (cd->vendor == VENDOR_TOSHIBA)
+ density = (blocklength > 2048) ? 0x81 : 0x83;
+
+- buffer = kmalloc(512, GFP_KERNEL | GFP_DMA);
++ buffer = kmalloc(512, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+@@ -179,7 +179,7 @@ int sr_cd_check(struct cdrom_device_info *cdi)
+ if (cd->cdi.mask & CDC_MULTI_SESSION)
+ return 0;
+
+- buffer = kmalloc(512, GFP_KERNEL | GFP_DMA);
++ buffer = kmalloc(512, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
+index ae8636d3780b6..9933722acfd96 100644
+--- a/drivers/scsi/st.c
++++ b/drivers/scsi/st.c
+@@ -32,6 +32,7 @@ static const char *verstr = "20160209";
+ #include <linux/slab.h>
+ #include <linux/errno.h>
+ #include <linux/mtio.h>
++#include <linux/major.h>
+ #include <linux/cdrom.h>
+ #include <linux/ioctl.h>
+ #include <linux/fcntl.h>
+diff --git a/drivers/scsi/ufs/tc-dwc-g210-pci.c b/drivers/scsi/ufs/tc-dwc-g210-pci.c
+index 679289e1a78e6..7b08e2e07cc5f 100644
+--- a/drivers/scsi/ufs/tc-dwc-g210-pci.c
++++ b/drivers/scsi/ufs/tc-dwc-g210-pci.c
+@@ -110,7 +110,6 @@ tc_dwc_g210_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ return err;
+ }
+
+- pci_set_drvdata(pdev, hba);
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_allow(&pdev->dev);
+
+diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c
+index 80b3545dd17d6..4e53857605de8 100644
+--- a/drivers/scsi/ufs/ufs-mediatek.c
++++ b/drivers/scsi/ufs/ufs-mediatek.c
+@@ -501,7 +501,7 @@ static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba)
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+ host->reg_va09 = regulator_get(hba->dev, "va09");
+- if (!host->reg_va09)
++ if (IS_ERR(host->reg_va09))
+ dev_info(hba->dev, "failed to get va09");
+ else
+ host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL;
+diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c
+index 51424557810da..e892b9feffb11 100644
+--- a/drivers/scsi/ufs/ufshcd-pci.c
++++ b/drivers/scsi/ufs/ufshcd-pci.c
+@@ -421,6 +421,19 @@ static int ufs_intel_lkf_init(struct ufs_hba *hba)
+ return err;
+ }
+
++static int ufs_intel_adl_init(struct ufs_hba *hba)
++{
++ hba->nop_out_timeout = 200;
++ hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8;
++ return ufs_intel_common_init(hba);
++}
++
++static int ufs_intel_mtl_init(struct ufs_hba *hba)
++{
++ hba->caps |= UFSHCD_CAP_CRYPTO | UFSHCD_CAP_WB_EN;
++ return ufs_intel_common_init(hba);
++}
++
+ static struct ufs_hba_variant_ops ufs_intel_cnl_hba_vops = {
+ .name = "intel-pci",
+ .init = ufs_intel_common_init,
+@@ -449,6 +462,25 @@ static struct ufs_hba_variant_ops ufs_intel_lkf_hba_vops = {
+ .device_reset = ufs_intel_device_reset,
+ };
+
++static struct ufs_hba_variant_ops ufs_intel_adl_hba_vops = {
++ .name = "intel-pci",
++ .init = ufs_intel_adl_init,
++ .exit = ufs_intel_common_exit,
++ .link_startup_notify = ufs_intel_link_startup_notify,
++ .resume = ufs_intel_resume,
++ .device_reset = ufs_intel_device_reset,
++};
++
++static struct ufs_hba_variant_ops ufs_intel_mtl_hba_vops = {
++ .name = "intel-pci",
++ .init = ufs_intel_mtl_init,
++ .exit = ufs_intel_common_exit,
++ .hce_enable_notify = ufs_intel_hce_enable_notify,
++ .link_startup_notify = ufs_intel_link_startup_notify,
++ .resume = ufs_intel_resume,
++ .device_reset = ufs_intel_device_reset,
++};
++
+ #ifdef CONFIG_PM_SLEEP
+ static int ufshcd_pci_restore(struct device *dev)
+ {
+@@ -522,8 +554,6 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ return err;
+ }
+
+- pci_set_drvdata(pdev, hba);
+-
+ hba->vops = (struct ufs_hba_variant_ops *)id->driver_data;
+
+ err = ufshcd_init(hba, mmio_base, pdev->irq);
+@@ -563,6 +593,9 @@ static const struct pci_device_id ufshcd_pci_tbl[] = {
+ { PCI_VDEVICE(INTEL, 0x4B41), (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
+ { PCI_VDEVICE(INTEL, 0x4B43), (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
+ { PCI_VDEVICE(INTEL, 0x98FA), (kernel_ulong_t)&ufs_intel_lkf_hba_vops },
++ { PCI_VDEVICE(INTEL, 0x51FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
++ { PCI_VDEVICE(INTEL, 0x54FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
++ { PCI_VDEVICE(INTEL, 0x7E47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+ { } /* terminate list */
+ };
+
+diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c
+index 8859c13f4e091..87975d1a21c8b 100644
+--- a/drivers/scsi/ufs/ufshcd-pltfrm.c
++++ b/drivers/scsi/ufs/ufshcd-pltfrm.c
+@@ -91,7 +91,12 @@ static int ufshcd_parse_clock_info(struct ufs_hba *hba)
+
+ clki->min_freq = clkfreq[i];
+ clki->max_freq = clkfreq[i+1];
+- clki->name = kstrdup(name, GFP_KERNEL);
++ clki->name = devm_kstrdup(dev, name, GFP_KERNEL);
++ if (!clki->name) {
++ ret = -ENOMEM;
++ goto out;
++ }
++
+ if (!strcmp(name, "ref_clk"))
+ clki->keep_link_active = true;
+ dev_dbg(dev, "%s: min %u max %u name %s\n", "freq-table-hz",
+@@ -126,7 +131,9 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name,
+ if (!vreg)
+ return -ENOMEM;
+
+- vreg->name = kstrdup(name, GFP_KERNEL);
++ vreg->name = devm_kstrdup(dev, name, GFP_KERNEL);
++ if (!vreg->name)
++ return -ENOMEM;
+
+ snprintf(prop_name, MAX_PROP_SIZE, "%s-max-microamp", name);
+ if (of_property_read_u32(np, prop_name, &vreg->max_uA)) {
+@@ -361,8 +368,6 @@ int ufshcd_pltfrm_init(struct platform_device *pdev,
+ goto dealloc_host;
+ }
+
+- platform_set_drvdata(pdev, hba);
+-
+ pm_runtime_set_active(&pdev->dev);
+ pm_runtime_enable(&pdev->dev);
+
+diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
+index 41f2ff35f82b2..b55e0a07363f2 100644
+--- a/drivers/scsi/ufs/ufshcd.c
++++ b/drivers/scsi/ufs/ufshcd.c
+@@ -125,8 +125,9 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
+ enum {
+ UFSHCD_MAX_CHANNEL = 0,
+ UFSHCD_MAX_ID = 1,
+- UFSHCD_CMD_PER_LUN = 32,
+- UFSHCD_CAN_QUEUE = 32,
++ UFSHCD_NUM_RESERVED = 1,
++ UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED,
++ UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED,
+ };
+
+ /* UFSHCD error handling flags */
+@@ -222,8 +223,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba);
+ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
+ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
+ static void ufshcd_hba_exit(struct ufs_hba *hba);
+-static int ufshcd_clear_ua_wluns(struct ufs_hba *hba);
+-static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
++static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
+ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
+ static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
+ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
+@@ -358,7 +358,7 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
+ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
+ enum ufs_trace_str_t str_t)
+ {
+- u64 lba;
++ u64 lba = 0;
+ u8 opcode = 0, group_id = 0;
+ u32 intr, doorbell;
+ struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+@@ -375,7 +375,6 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
+ return;
+
+ opcode = cmd->cmnd[0];
+- lba = scsi_get_lba(cmd);
+
+ if (opcode == READ_10 || opcode == WRITE_10) {
+ /*
+@@ -383,6 +382,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
+ */
+ transfer_len =
+ be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
++ lba = scsi_get_lba(cmd);
+ if (opcode == WRITE_10)
+ group_id = lrbp->cmd->cmnd[6];
+ } else if (opcode == UNMAP) {
+@@ -390,6 +390,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
+ * The number of Bytes to be unmapped beginning with the lba.
+ */
+ transfer_len = blk_rq_bytes(rq);
++ lba = scsi_get_lba(cmd);
+ }
+
+ intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+@@ -576,7 +577,12 @@ static void ufshcd_print_pwr_info(struct ufs_hba *hba)
+ "INVALID MODE",
+ };
+
+- dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
++ /*
++ * Using dev_dbg to avoid messages during runtime PM to avoid
++ * never-ending cycles of messages written back to storage by user space
++ * causing runtime resume, causing more messages and so on.
++ */
++ dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
+ __func__,
+ hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
+ hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
+@@ -1658,7 +1664,8 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)
+ bool flush_result;
+ unsigned long flags;
+
+- if (!ufshcd_is_clkgating_allowed(hba))
++ if (!ufshcd_is_clkgating_allowed(hba) ||
++ !hba->clk_gating.is_initialized)
+ goto out;
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ hba->clk_gating.active_reqs++;
+@@ -1818,7 +1825,7 @@ static void __ufshcd_release(struct ufs_hba *hba)
+
+ if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
+ hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
+- hba->outstanding_tasks ||
++ hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
+ hba->active_uic_cmd || hba->uic_async_done ||
+ hba->clk_gating.state == CLKS_OFF)
+ return;
+@@ -1953,11 +1960,15 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
+ {
+ if (!hba->clk_gating.is_initialized)
+ return;
++
+ ufshcd_remove_clk_gating_sysfs(hba);
+- cancel_work_sync(&hba->clk_gating.ungate_work);
+- cancel_delayed_work_sync(&hba->clk_gating.gate_work);
+- destroy_workqueue(hba->clk_gating.clk_gating_workq);
++
++ /* Ungate the clock if necessary. */
++ ufshcd_hold(hba, false);
+ hba->clk_gating.is_initialized = false;
++ ufshcd_release(hba);
++
++ destroy_workqueue(hba->clk_gating.clk_gating_workq);
+ }
+
+ /* Must be called with host lock acquired */
+@@ -2181,6 +2192,7 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
+ hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
+ hba->nutmrs =
+ ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
++ hba->reserved_slot = hba->nutrs - 1;
+
+ /* Read crypto capabilities */
+ err = ufshcd_hba_init_crypto_capabilities(hba);
+@@ -2906,30 +2918,15 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
+ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
+ enum dev_cmd_type cmd_type, int timeout)
+ {
+- struct request_queue *q = hba->cmd_queue;
+ DECLARE_COMPLETION_ONSTACK(wait);
+- struct request *req;
++ const u32 tag = hba->reserved_slot;
+ struct ufshcd_lrb *lrbp;
+ int err;
+- int tag;
+
+- down_read(&hba->clk_scaling_lock);
++ /* Protects use of hba->reserved_slot. */
++ lockdep_assert_held(&hba->dev_cmd.lock);
+
+- /*
+- * Get free slot, sleep if slots are unavailable.
+- * Even though we use wait_event() which sleeps indefinitely,
+- * the maximum wait time is bounded by SCSI request timeout.
+- */
+- req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+- if (IS_ERR(req)) {
+- err = PTR_ERR(req);
+- goto out_unlock;
+- }
+- tag = req->tag;
+- WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
+- /* Set the timeout such that the SCSI error handler is not activated. */
+- req->timeout = msecs_to_jiffies(2 * timeout);
+- blk_mq_start_request(req);
++ down_read(&hba->clk_scaling_lock);
+
+ lrbp = &hba->lrb[tag];
+ WARN_ON(lrbp->cmd);
+@@ -2947,8 +2944,6 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
+ (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
+
+ out:
+- blk_put_request(req);
+-out_unlock:
+ up_read(&hba->clk_scaling_lock);
+ return err;
+ }
+@@ -4073,8 +4068,6 @@ int ufshcd_link_recovery(struct ufs_hba *hba)
+ if (ret)
+ dev_err(hba->dev, "%s: link recovery failed, err %d",
+ __func__, ret);
+- else
+- ufshcd_clear_ua_wluns(hba);
+
+ return ret;
+ }
+@@ -4980,6 +4973,12 @@ static int ufshcd_slave_configure(struct scsi_device *sdev)
+ pm_runtime_get_noresume(&sdev->sdev_gendev);
+ else if (ufshcd_is_rpm_autosuspend_allowed(hba))
+ sdev->rpm_autosuspend = 1;
++ /*
++ * Do not print messages during runtime PM to avoid never-ending cycles
++ * of messages written back to storage by user space causing runtime
++ * resume, causing more messages and so on.
++ */
++ sdev->silence_suspend = 1;
+
+ ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
+
+@@ -5959,7 +5958,6 @@ static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
+ ufshcd_release(hba);
+ if (ufshcd_is_clkscaling_supported(hba))
+ ufshcd_clk_scaling_suspend(hba, false);
+- ufshcd_clear_ua_wluns(hba);
+ ufshcd_rpm_put(hba);
+ }
+
+@@ -6386,9 +6384,8 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
+ irqreturn_t ret = IRQ_NONE;
+ int tag;
+
+- pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
+-
+ spin_lock_irqsave(hba->host->host_lock, flags);
++ pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
+ issued = hba->outstanding_tasks & ~pending;
+ for_each_set_bit(tag, &issued, hba->nutmrs) {
+ struct request *req = hba->tmf_rqs[tag];
+@@ -6545,11 +6542,6 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
+ err = wait_for_completion_io_timeout(&wait,
+ msecs_to_jiffies(TM_CMD_TIMEOUT));
+ if (!err) {
+- /*
+- * Make sure that ufshcd_compl_tm() does not trigger a
+- * use-after-free.
+- */
+- req->end_io_data = NULL;
+ ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
+ dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
+ __func__, tm_function);
+@@ -6645,28 +6637,16 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
+ enum dev_cmd_type cmd_type,
+ enum query_opcode desc_op)
+ {
+- struct request_queue *q = hba->cmd_queue;
+ DECLARE_COMPLETION_ONSTACK(wait);
+- struct request *req;
++ const u32 tag = hba->reserved_slot;
+ struct ufshcd_lrb *lrbp;
+ int err = 0;
+- int tag;
+ u8 upiu_flags;
+
+- down_read(&hba->clk_scaling_lock);
+-
+- req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+- if (IS_ERR(req)) {
+- err = PTR_ERR(req);
+- goto out_unlock;
+- }
+- tag = req->tag;
+- WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
++ /* Protects use of hba->reserved_slot. */
++ lockdep_assert_held(&hba->dev_cmd.lock);
+
+- if (unlikely(test_bit(tag, &hba->outstanding_reqs))) {
+- err = -EBUSY;
+- goto out;
+- }
++ down_read(&hba->clk_scaling_lock);
+
+ lrbp = &hba->lrb[tag];
+ WARN_ON(lrbp->cmd);
+@@ -6735,9 +6715,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
+ ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
+ (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
+
+-out:
+- blk_put_request(req);
+-out_unlock:
+ up_read(&hba->clk_scaling_lock);
+ return err;
+ }
+@@ -7044,6 +7021,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
+ goto release;
+ }
+
++ lrbp->cmd = NULL;
+ err = SUCCESS;
+
+ release:
+@@ -7233,7 +7211,13 @@ static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
+
+ if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
+ !hba->vreg_info.vccq2) {
+- dev_err(hba->dev,
++ /*
++ * Using dev_dbg to avoid messages during runtime PM to avoid
++ * never-ending cycles of messages written back to storage by
++ * user space causing runtime resume, causing more messages and
++ * so on.
++ */
++ dev_dbg(hba->dev,
+ "%s: Regulator capability was not set, actvIccLevel=%d",
+ __func__, icc_level);
+ goto out;
+@@ -7875,8 +7859,6 @@ static int ufshcd_add_lus(struct ufs_hba *hba)
+ if (ret)
+ goto out;
+
+- ufshcd_clear_ua_wluns(hba);
+-
+ /* Initialize devfreq after UFS device is detected */
+ if (ufshcd_is_clkscaling_supported(hba)) {
+ memcpy(&hba->clk_scaling.saved_pwr_info.info,
+@@ -7902,116 +7884,6 @@ out:
+ return ret;
+ }
+
+-static void ufshcd_request_sense_done(struct request *rq, blk_status_t error)
+-{
+- if (error != BLK_STS_OK)
+- pr_err("%s: REQUEST SENSE failed (%d)\n", __func__, error);
+- kfree(rq->end_io_data);
+- blk_put_request(rq);
+-}
+-
+-static int
+-ufshcd_request_sense_async(struct ufs_hba *hba, struct scsi_device *sdev)
+-{
+- /*
+- * Some UFS devices clear unit attention condition only if the sense
+- * size used (UFS_SENSE_SIZE in this case) is non-zero.
+- */
+- static const u8 cmd[6] = {REQUEST_SENSE, 0, 0, 0, UFS_SENSE_SIZE, 0};
+- struct scsi_request *rq;
+- struct request *req;
+- char *buffer;
+- int ret;
+-
+- buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
+- if (!buffer)
+- return -ENOMEM;
+-
+- req = blk_get_request(sdev->request_queue, REQ_OP_DRV_IN,
+- /*flags=*/BLK_MQ_REQ_PM);
+- if (IS_ERR(req)) {
+- ret = PTR_ERR(req);
+- goto out_free;
+- }
+-
+- ret = blk_rq_map_kern(sdev->request_queue, req,
+- buffer, UFS_SENSE_SIZE, GFP_NOIO);
+- if (ret)
+- goto out_put;
+-
+- rq = scsi_req(req);
+- rq->cmd_len = ARRAY_SIZE(cmd);
+- memcpy(rq->cmd, cmd, rq->cmd_len);
+- rq->retries = 3;
+- req->timeout = 1 * HZ;
+- req->rq_flags |= RQF_PM | RQF_QUIET;
+- req->end_io_data = buffer;
+-
+- blk_execute_rq_nowait(/*bd_disk=*/NULL, req, /*at_head=*/true,
+- ufshcd_request_sense_done);
+- return 0;
+-
+-out_put:
+- blk_put_request(req);
+-out_free:
+- kfree(buffer);
+- return ret;
+-}
+-
+-static int ufshcd_clear_ua_wlun(struct ufs_hba *hba, u8 wlun)
+-{
+- struct scsi_device *sdp;
+- unsigned long flags;
+- int ret = 0;
+-
+- spin_lock_irqsave(hba->host->host_lock, flags);
+- if (wlun == UFS_UPIU_UFS_DEVICE_WLUN)
+- sdp = hba->sdev_ufs_device;
+- else if (wlun == UFS_UPIU_RPMB_WLUN)
+- sdp = hba->sdev_rpmb;
+- else
+- BUG();
+- if (sdp) {
+- ret = scsi_device_get(sdp);
+- if (!ret && !scsi_device_online(sdp)) {
+- ret = -ENODEV;
+- scsi_device_put(sdp);
+- }
+- } else {
+- ret = -ENODEV;
+- }
+- spin_unlock_irqrestore(hba->host->host_lock, flags);
+- if (ret)
+- goto out_err;
+-
+- ret = ufshcd_request_sense_async(hba, sdp);
+- scsi_device_put(sdp);
+-out_err:
+- if (ret)
+- dev_err(hba->dev, "%s: UAC clear LU=%x ret = %d\n",
+- __func__, wlun, ret);
+- return ret;
+-}
+-
+-static int ufshcd_clear_ua_wluns(struct ufs_hba *hba)
+-{
+- int ret = 0;
+-
+- if (!hba->wlun_dev_clr_ua)
+- goto out;
+-
+- ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
+- if (!ret)
+- ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
+- if (!ret)
+- hba->wlun_dev_clr_ua = false;
+-out:
+- if (ret)
+- dev_err(hba->dev, "%s: Failed to clear UAC WLUNS ret = %d\n",
+- __func__, ret);
+- return ret;
+-}
+-
+ /**
+ * ufshcd_probe_hba - probe hba to detect device and initialize it
+ * @hba: per-adapter instance
+@@ -8062,8 +7934,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
+ /* UFS device is also active now */
+ ufshcd_set_ufs_dev_active(hba);
+ ufshcd_force_reset_auto_bkops(hba);
+- hba->wlun_dev_clr_ua = true;
+- hba->wlun_rpmb_clr_ua = true;
+
+ /* Gear up to HS gear if supported */
+ if (hba->max_pwr_info.is_valid) {
+@@ -8591,7 +8461,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba)
+ * @pwr_mode: device power mode to set
+ *
+ * Returns 0 if requested power mode is set successfully
+- * Returns non-zero if failed to set the requested power mode
++ * Returns < 0 if failed to set the requested power mode
+ */
+ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
+ enum ufs_dev_pwr_mode pwr_mode)
+@@ -8600,7 +8470,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
+ struct scsi_sense_hdr sshdr;
+ struct scsi_device *sdp;
+ unsigned long flags;
+- int ret;
++ int ret, retries;
+
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ sdp = hba->sdev_ufs_device;
+@@ -8625,8 +8495,6 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
+ * handling context.
+ */
+ hba->host->eh_noresume = 1;
+- if (hba->wlun_dev_clr_ua)
+- ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
+
+ cmd[4] = pwr_mode << 4;
+
+@@ -8635,14 +8503,23 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
+ * callbacks hence set the RQF_PM flag so that it doesn't resume the
+ * already suspended childs.
+ */
+- ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+- START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
++ for (retries = 3; retries > 0; --retries) {
++ ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
++ START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
++ if (!scsi_status_is_check_condition(ret) ||
++ !scsi_sense_valid(&sshdr) ||
++ sshdr.sense_key != UNIT_ATTENTION)
++ break;
++ }
+ if (ret) {
+ sdev_printk(KERN_WARNING, sdp,
+ "START_STOP failed for power mode: %d, result %x\n",
+ pwr_mode, ret);
+- if (ret > 0 && scsi_sense_valid(&sshdr))
+- scsi_print_sense_hdr(sdp, NULL, &sshdr);
++ if (ret > 0) {
++ if (scsi_sense_valid(&sshdr))
++ scsi_print_sense_hdr(sdp, NULL, &sshdr);
++ ret = -EIO;
++ }
+ }
+
+ if (!ret)
+@@ -9485,6 +9362,13 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
+ struct device *dev = hba->dev;
+ char eh_wq_name[sizeof("ufs_eh_wq_00")];
+
++ /*
++ * dev_set_drvdata() must be called before any callbacks are registered
++ * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
++ * sysfs).
++ */
++ dev_set_drvdata(dev, hba);
++
+ if (!mmio_base) {
+ dev_err(hba->dev,
+ "Invalid memory reference for mmio_base is NULL\n");
+@@ -9527,8 +9411,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
+ /* Configure LRB */
+ ufshcd_host_memory_configure(hba);
+
+- host->can_queue = hba->nutrs;
+- host->cmd_per_lun = hba->nutrs;
++ host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
++ host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
+ host->max_id = UFSHCD_MAX_ID;
+ host->max_lun = UFS_MAX_LUNS;
+ host->max_channel = UFSHCD_MAX_CHANNEL;
+@@ -9699,10 +9583,6 @@ void ufshcd_resume_complete(struct device *dev)
+ ufshcd_rpm_put(hba);
+ hba->complete_put = false;
+ }
+- if (hba->rpmb_complete_put) {
+- ufshcd_rpmb_rpm_put(hba);
+- hba->rpmb_complete_put = false;
+- }
+ }
+ EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
+
+@@ -9725,10 +9605,6 @@ int ufshcd_suspend_prepare(struct device *dev)
+ }
+ hba->complete_put = true;
+ }
+- if (hba->sdev_rpmb) {
+- ufshcd_rpmb_rpm_get_sync(hba);
+- hba->rpmb_complete_put = true;
+- }
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
+@@ -9797,49 +9673,6 @@ static struct scsi_driver ufs_dev_wlun_template = {
+ },
+ };
+
+-static int ufshcd_rpmb_probe(struct device *dev)
+-{
+- return is_rpmb_wlun(to_scsi_device(dev)) ? 0 : -ENODEV;
+-}
+-
+-static inline int ufshcd_clear_rpmb_uac(struct ufs_hba *hba)
+-{
+- int ret = 0;
+-
+- if (!hba->wlun_rpmb_clr_ua)
+- return 0;
+- ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
+- if (!ret)
+- hba->wlun_rpmb_clr_ua = 0;
+- return ret;
+-}
+-
+-#ifdef CONFIG_PM
+-static int ufshcd_rpmb_resume(struct device *dev)
+-{
+- struct ufs_hba *hba = wlun_dev_to_hba(dev);
+-
+- if (hba->sdev_rpmb)
+- ufshcd_clear_rpmb_uac(hba);
+- return 0;
+-}
+-#endif
+-
+-static const struct dev_pm_ops ufs_rpmb_pm_ops = {
+- SET_RUNTIME_PM_OPS(NULL, ufshcd_rpmb_resume, NULL)
+- SET_SYSTEM_SLEEP_PM_OPS(NULL, ufshcd_rpmb_resume)
+-};
+-
+-/* ufs_rpmb_wlun_template - Describes UFS RPMB WLUN. Used only to send UAC. */
+-static struct scsi_driver ufs_rpmb_wlun_template = {
+- .gendrv = {
+- .name = "ufs_rpmb_wlun",
+- .owner = THIS_MODULE,
+- .probe = ufshcd_rpmb_probe,
+- .pm = &ufs_rpmb_pm_ops,
+- },
+-};
+-
+ static int __init ufshcd_core_init(void)
+ {
+ int ret;
+@@ -9848,24 +9681,13 @@ static int __init ufshcd_core_init(void)
+
+ ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
+ if (ret)
+- goto debugfs_exit;
+-
+- ret = scsi_register_driver(&ufs_rpmb_wlun_template.gendrv);
+- if (ret)
+- goto unregister;
+-
+- return ret;
+-unregister:
+- scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
+-debugfs_exit:
+- ufs_debugfs_exit();
++ ufs_debugfs_exit();
+ return ret;
+ }
+
+ static void __exit ufshcd_core_exit(void)
+ {
+ ufs_debugfs_exit();
+- scsi_unregister_driver(&ufs_rpmb_wlun_template.gendrv);
+ scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
+ }
+
+diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
+index 41f6e06f91856..d470a52ff24c3 100644
+--- a/drivers/scsi/ufs/ufshcd.h
++++ b/drivers/scsi/ufs/ufshcd.h
+@@ -725,6 +725,7 @@ struct ufs_hba_monitor {
+ * @capabilities: UFS Controller Capabilities
+ * @nutrs: Transfer Request Queue depth supported by controller
+ * @nutmrs: Task Management Queue depth supported by controller
++ * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock.
+ * @ufs_version: UFS Version to which controller complies
+ * @vops: pointer to variant specific operations
+ * @priv: pointer to variant specific private data
+@@ -813,6 +814,7 @@ struct ufs_hba {
+ u32 capabilities;
+ int nutrs;
+ int nutmrs;
++ u32 reserved_slot;
+ u32 ufs_version;
+ const struct ufs_hba_variant_ops *vops;
+ struct ufs_hba_variant_params *vps;
+@@ -871,9 +873,6 @@ struct ufs_hba {
+ struct ufs_vreg_info vreg_info;
+ struct list_head clk_list_head;
+
+- bool wlun_dev_clr_ua;
+- bool wlun_rpmb_clr_ua;
+-
+ /* Number of requests aborts */
+ int req_abort_count;
+
+@@ -920,7 +919,6 @@ struct ufs_hba {
+ #endif
+ u32 luns_avail;
+ bool complete_put;
+- bool rpmb_complete_put;
+ };
+
+ /* Returns true if clocks can be gated. Otherwise false */
+@@ -1393,14 +1391,4 @@ static inline int ufshcd_rpm_put(struct ufs_hba *hba)
+ return pm_runtime_put(&hba->sdev_ufs_device->sdev_gendev);
+ }
+
+-static inline int ufshcd_rpmb_rpm_get_sync(struct ufs_hba *hba)
+-{
+- return pm_runtime_get_sync(&hba->sdev_rpmb->sdev_gendev);
+-}
+-
+-static inline int ufshcd_rpmb_rpm_put(struct ufs_hba *hba)
+-{
+- return pm_runtime_put(&hba->sdev_rpmb->sdev_gendev);
+-}
+-
+ #endif /* End of Header */
+diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h
+index de95be5d11d4e..3ed60068c4ea6 100644
+--- a/drivers/scsi/ufs/ufshci.h
++++ b/drivers/scsi/ufs/ufshci.h
+@@ -142,7 +142,8 @@ static inline u32 ufshci_version(u32 major, u32 minor)
+ #define INT_FATAL_ERRORS (DEVICE_FATAL_ERROR |\
+ CONTROLLER_FATAL_ERROR |\
+ SYSTEM_BUS_FATAL_ERROR |\
+- CRYPTO_ENGINE_FATAL_ERROR)
++ CRYPTO_ENGINE_FATAL_ERROR |\
++ UIC_LINK_LOST)
+
+ /* HCS - Host Controller Status 30h */
+ #define DEVICE_PRESENT 0x1
+diff --git a/drivers/scsi/ufs/ufshpb.c b/drivers/scsi/ufs/ufshpb.c
+index 026a133149dce..14300896c57fe 100644
+--- a/drivers/scsi/ufs/ufshpb.c
++++ b/drivers/scsi/ufs/ufshpb.c
+@@ -394,8 +394,6 @@ int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+ if (!ufshpb_is_supported_chunk(hpb, transfer_len))
+ return 0;
+
+- WARN_ON_ONCE(transfer_len > HPB_MULTI_CHUNK_HIGH);
+-
+ if (hpb->is_hcm) {
+ /*
+ * in host control mode, reads are the main source for
+@@ -872,12 +870,6 @@ static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb)
+ struct ufshpb_region *rgn, *victim_rgn = NULL;
+
+ list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) {
+- if (!rgn) {
+- dev_err(&hpb->sdev_ufs_lu->sdev_dev,
+- "%s: no region allocated\n",
+- __func__);
+- return NULL;
+- }
+ if (ufshpb_check_srgns_issue_state(hpb, rgn))
+ continue;
+
+@@ -893,6 +885,11 @@ static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb)
+ break;
+ }
+
++ if (!victim_rgn)
++ dev_err(&hpb->sdev_ufs_lu->sdev_dev,
++ "%s: no region allocated\n",
++ __func__);
++
+ return victim_rgn;
+ }
+
+@@ -1260,6 +1257,13 @@ void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+ struct utp_hpb_rsp *rsp_field = &lrbp->ucd_rsp_ptr->hr;
+ int data_seg_len;
+
++ data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2)
++ & MASK_RSP_UPIU_DATA_SEG_LEN;
++
++ /* If data segment length is zero, rsp_field is not valid */
++ if (!data_seg_len)
++ return;
++
+ if (unlikely(lrbp->lun != rsp_field->lun)) {
+ struct scsi_device *sdev;
+ bool found = false;
+@@ -1294,18 +1298,6 @@ void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+ return;
+ }
+
+- data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2)
+- & MASK_RSP_UPIU_DATA_SEG_LEN;
+-
+- /* To flush remained rsp_list, we queue the map_work task */
+- if (!data_seg_len) {
+- if (!ufshpb_is_general_lun(hpb->lun))
+- return;
+-
+- ufshpb_kick_map_work(hpb);
+- return;
+- }
+-
+ BUILD_BUG_ON(sizeof(struct utp_hpb_rsp) != UTP_HPB_RSP_SIZE);
+
+ if (!ufshpb_is_hpb_rsp_valid(hba, lrbp, rsp_field))
+@@ -1572,7 +1564,7 @@ static void ufshpb_lu_parameter_init(struct ufs_hba *hba,
+ if (ufshpb_is_legacy(hba))
+ hpb->pre_req_max_tr_len = HPB_LEGACY_CHUNK_HIGH;
+ else
+- hpb->pre_req_max_tr_len = HPB_MULTI_CHUNK_HIGH;
++ hpb->pre_req_max_tr_len = hpb_dev_info->max_hpb_single_cmd;
+
+ hpb->lu_pinned_start = hpb_lu_info->pinned_start;
+ hpb->lu_pinned_end = hpb_lu_info->num_pinned ?
+@@ -2371,11 +2363,11 @@ static int ufshpb_get_lu_info(struct ufs_hba *hba, int lun,
+
+ ufshcd_map_desc_id_to_length(hba, QUERY_DESC_IDN_UNIT, &size);
+
+- pm_runtime_get_sync(hba->dev);
++ ufshcd_rpm_get_sync(hba);
+ ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
+ QUERY_DESC_IDN_UNIT, lun, 0,
+ desc_buf, &size);
+- pm_runtime_put_sync(hba->dev);
++ ufshcd_rpm_put_sync(hba);
+
+ if (ret) {
+ dev_err(hba->dev,
+@@ -2582,7 +2574,7 @@ void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf)
+ {
+ struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev;
+ int version, ret;
+- u32 max_hpb_single_cmd = HPB_MULTI_CHUNK_LOW;
++ int max_single_cmd;
+
+ hpb_dev_info->control_mode = desc_buf[DEVICE_DESC_PARAM_HPB_CONTROL];
+
+@@ -2598,21 +2590,22 @@ void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf)
+ if (version == HPB_SUPPORT_LEGACY_VERSION)
+ hpb_dev_info->is_legacy = true;
+
+- pm_runtime_get_sync(hba->dev);
+- ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+- QUERY_ATTR_IDN_MAX_HPB_SINGLE_CMD, 0, 0, &max_hpb_single_cmd);
+- pm_runtime_put_sync(hba->dev);
+-
+- if (ret)
+- dev_err(hba->dev, "%s: idn: read max size of single hpb cmd query request failed",
+- __func__);
+- hpb_dev_info->max_hpb_single_cmd = max_hpb_single_cmd;
+-
+ /*
+ * Get the number of user logical unit to check whether all
+ * scsi_device finish initialization
+ */
+ hpb_dev_info->num_lu = desc_buf[DEVICE_DESC_PARAM_NUM_LU];
++
++ if (hpb_dev_info->is_legacy)
++ return;
++
++ ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
++ QUERY_ATTR_IDN_MAX_HPB_SINGLE_CMD, 0, 0, &max_single_cmd);
++
++ if (ret)
++ hpb_dev_info->max_hpb_single_cmd = HPB_LEGACY_CHUNK_HIGH;
++ else
++ hpb_dev_info->max_hpb_single_cmd = min(max_single_cmd + 1, HPB_MULTI_CHUNK_HIGH);
+ }
+
+ void ufshpb_init(struct ufs_hba *hba)
+diff --git a/drivers/scsi/ufs/ufshpb.h b/drivers/scsi/ufs/ufshpb.h
+index f15d8fdbce2ef..b475dbd789883 100644
+--- a/drivers/scsi/ufs/ufshpb.h
++++ b/drivers/scsi/ufs/ufshpb.h
+@@ -31,7 +31,6 @@
+
+ /* hpb support chunk size */
+ #define HPB_LEGACY_CHUNK_HIGH 1
+-#define HPB_MULTI_CHUNK_LOW 7
+ #define HPB_MULTI_CHUNK_HIGH 255
+
+ /* hpb vender defined opcode */
+diff --git a/drivers/scsi/vmw_pvscsi.c b/drivers/scsi/vmw_pvscsi.c
+index ce1ba1b936298..9419d6d1d8d26 100644
+--- a/drivers/scsi/vmw_pvscsi.c
++++ b/drivers/scsi/vmw_pvscsi.c
+@@ -586,9 +586,12 @@ static void pvscsi_complete_request(struct pvscsi_adapter *adapter,
+ * Commands like INQUIRY may transfer less data than
+ * requested by the initiator via bufflen. Set residual
+ * count to make upper layer aware of the actual amount
+- * of data returned.
++ * of data returned. There are cases when controller
++ * returns zero dataLen with non zero data - do not set
++ * residual count in that case.
+ */
+- scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
++ if (e->dataLen && (e->dataLen < scsi_bufflen(cmd)))
++ scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
+ cmd->result = (DID_OK << 16);
+ break;
+
+diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c
+index 0204e314b4825..17b8c88840873 100644
+--- a/drivers/scsi/xen-scsifront.c
++++ b/drivers/scsi/xen-scsifront.c
+@@ -233,12 +233,11 @@ static void scsifront_gnttab_done(struct vscsifrnt_info *info,
+ return;
+
+ for (i = 0; i < shadow->nr_grants; i++) {
+- if (unlikely(gnttab_query_foreign_access(shadow->gref[i]))) {
++ if (unlikely(!gnttab_try_end_foreign_access(shadow->gref[i]))) {
+ shost_printk(KERN_ALERT, info->host, KBUILD_MODNAME
+ "grant still in use by backend\n");
+ BUG();
+ }
+- gnttab_end_foreign_access(shadow->gref[i], 0, 0UL);
+ }
+
+ kfree(shadow->sg);
+diff --git a/drivers/scsi/zorro7xx.c b/drivers/scsi/zorro7xx.c
+index 27b9e2baab1a6..7acf9193a9e80 100644
+--- a/drivers/scsi/zorro7xx.c
++++ b/drivers/scsi/zorro7xx.c
+@@ -159,6 +159,8 @@ static void zorro7xx_remove_one(struct zorro_dev *z)
+ scsi_remove_host(host);
+
+ NCR_700_release(host);
++ if (host->base > 0x01000000)
++ iounmap(hostdata->base);
+ kfree(hostdata);
+ free_irq(host->irq, host);
+ zorro_release_device(z);
+diff --git a/drivers/sh/maple/maple.c b/drivers/sh/maple/maple.c
+index bd0fbcdbdefe9..e24e220e56eea 100644
+--- a/drivers/sh/maple/maple.c
++++ b/drivers/sh/maple/maple.c
+@@ -834,8 +834,10 @@ static int __init maple_bus_init(void)
+
+ maple_queue_cache = KMEM_CACHE(maple_buffer, SLAB_HWCACHE_ALIGN);
+
+- if (!maple_queue_cache)
++ if (!maple_queue_cache) {
++ retval = -ENOMEM;
+ goto cleanup_bothirqs;
++ }
+
+ INIT_LIST_HEAD(&maple_waitq);
+ INIT_LIST_HEAD(&maple_sentq);
+@@ -848,6 +850,7 @@ static int __init maple_bus_init(void)
+ if (!mdev[i]) {
+ while (i-- > 0)
+ maple_free_dev(mdev[i]);
++ retval = -ENOMEM;
+ goto cleanup_cache;
+ }
+ baseunits[i] = mdev[i];
+diff --git a/drivers/slimbus/qcom-ctrl.c b/drivers/slimbus/qcom-ctrl.c
+index f04b961b96cd4..ec58091fc948a 100644
+--- a/drivers/slimbus/qcom-ctrl.c
++++ b/drivers/slimbus/qcom-ctrl.c
+@@ -510,9 +510,9 @@ static int qcom_slim_probe(struct platform_device *pdev)
+ }
+
+ ctrl->irq = platform_get_irq(pdev, 0);
+- if (!ctrl->irq) {
++ if (ctrl->irq < 0) {
+ dev_err(&pdev->dev, "no slimbus IRQ\n");
+- return -ENODEV;
++ return ctrl->irq;
+ }
+
+ sctrl = &ctrl->ctrl;
+diff --git a/drivers/soc/aspeed/aspeed-lpc-ctrl.c b/drivers/soc/aspeed/aspeed-lpc-ctrl.c
+index 72771e018c42e..258894ed234b3 100644
+--- a/drivers/soc/aspeed/aspeed-lpc-ctrl.c
++++ b/drivers/soc/aspeed/aspeed-lpc-ctrl.c
+@@ -306,10 +306,9 @@ static int aspeed_lpc_ctrl_probe(struct platform_device *pdev)
+ }
+
+ lpc_ctrl->clk = devm_clk_get(dev, NULL);
+- if (IS_ERR(lpc_ctrl->clk)) {
+- dev_err(dev, "couldn't get clock\n");
+- return PTR_ERR(lpc_ctrl->clk);
+- }
++ if (IS_ERR(lpc_ctrl->clk))
++ return dev_err_probe(dev, PTR_ERR(lpc_ctrl->clk),
++ "couldn't get clock\n");
+ rc = clk_prepare_enable(lpc_ctrl->clk);
+ if (rc) {
+ dev_err(dev, "couldn't enable clock\n");
+diff --git a/drivers/soc/fsl/dpaa2-console.c b/drivers/soc/fsl/dpaa2-console.c
+index 27243f706f376..53917410f2bdb 100644
+--- a/drivers/soc/fsl/dpaa2-console.c
++++ b/drivers/soc/fsl/dpaa2-console.c
+@@ -231,6 +231,7 @@ static ssize_t dpaa2_console_read(struct file *fp, char __user *buf,
+ cd->cur_ptr += bytes;
+ written += bytes;
+
++ kfree(kbuf);
+ return written;
+
+ err_free_buf:
+diff --git a/drivers/soc/fsl/dpio/dpio-service.c b/drivers/soc/fsl/dpio/dpio-service.c
+index 7351f30305506..779c319a4b820 100644
+--- a/drivers/soc/fsl/dpio/dpio-service.c
++++ b/drivers/soc/fsl/dpio/dpio-service.c
+@@ -59,7 +59,7 @@ static inline struct dpaa2_io *service_select_by_cpu(struct dpaa2_io *d,
+ * potentially being migrated away.
+ */
+ if (cpu < 0)
+- cpu = smp_processor_id();
++ cpu = raw_smp_processor_id();
+
+ /* If a specific cpu was requested, pick it up immediately */
+ return dpio_by_cpu[cpu];
+diff --git a/drivers/soc/fsl/dpio/qbman-portal.c b/drivers/soc/fsl/dpio/qbman-portal.c
+index f13da4d7d1c52..3ec8ab08b9889 100644
+--- a/drivers/soc/fsl/dpio/qbman-portal.c
++++ b/drivers/soc/fsl/dpio/qbman-portal.c
+@@ -732,8 +732,7 @@ int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+ int i, num_enqueued = 0;
+ unsigned long irq_flags;
+
+- spin_lock(&s->access_spinlock);
+- local_irq_save(irq_flags);
++ spin_lock_irqsave(&s->access_spinlock, irq_flags);
+
+ half_mask = (s->eqcr.pi_ci_mask>>1);
+ full_mask = s->eqcr.pi_ci_mask;
+@@ -744,8 +743,7 @@ int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+ s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+ eqcr_ci, s->eqcr.ci);
+ if (!s->eqcr.available) {
+- local_irq_restore(irq_flags);
+- spin_unlock(&s->access_spinlock);
++ spin_unlock_irqrestore(&s->access_spinlock, irq_flags);
+ return 0;
+ }
+ }
+@@ -784,8 +782,7 @@ int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+ dma_wmb();
+ qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
+ (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+- local_irq_restore(irq_flags);
+- spin_unlock(&s->access_spinlock);
++ spin_unlock_irqrestore(&s->access_spinlock, irq_flags);
+
+ return num_enqueued;
+ }
+diff --git a/drivers/soc/fsl/guts.c b/drivers/soc/fsl/guts.c
+index d5e9a5f2c0874..75eabfb916cb5 100644
+--- a/drivers/soc/fsl/guts.c
++++ b/drivers/soc/fsl/guts.c
+@@ -28,7 +28,6 @@ struct fsl_soc_die_attr {
+ static struct guts *guts;
+ static struct soc_device_attribute soc_dev_attr;
+ static struct soc_device *soc_dev;
+-static struct device_node *root;
+
+
+ /* SoC die attribute definition for QorIQ platform */
+@@ -138,7 +137,7 @@ static u32 fsl_guts_get_svr(void)
+
+ static int fsl_guts_probe(struct platform_device *pdev)
+ {
+- struct device_node *np = pdev->dev.of_node;
++ struct device_node *root, *np = pdev->dev.of_node;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+ const struct fsl_soc_die_attr *soc_die;
+@@ -161,8 +160,14 @@ static int fsl_guts_probe(struct platform_device *pdev)
+ root = of_find_node_by_path("/");
+ if (of_property_read_string(root, "model", &machine))
+ of_property_read_string_index(root, "compatible", 0, &machine);
+- if (machine)
+- soc_dev_attr.machine = machine;
++ if (machine) {
++ soc_dev_attr.machine = devm_kstrdup(dev, machine, GFP_KERNEL);
++ if (!soc_dev_attr.machine) {
++ of_node_put(root);
++ return -ENOMEM;
++ }
++ }
++ of_node_put(root);
+
+ svr = fsl_guts_get_svr();
+ soc_die = fsl_soc_die_match(svr, fsl_soc_die);
+@@ -197,7 +202,6 @@ static int fsl_guts_probe(struct platform_device *pdev)
+ static int fsl_guts_remove(struct platform_device *dev)
+ {
+ soc_device_unregister(soc_dev);
+- of_node_put(root);
+ return 0;
+ }
+
+diff --git a/drivers/soc/fsl/qe/qe_io.c b/drivers/soc/fsl/qe/qe_io.c
+index e277c827bdf33..a5e2d0e5ab511 100644
+--- a/drivers/soc/fsl/qe/qe_io.c
++++ b/drivers/soc/fsl/qe/qe_io.c
+@@ -35,6 +35,8 @@ int par_io_init(struct device_node *np)
+ if (ret)
+ return ret;
+ par_io = ioremap(res.start, resource_size(&res));
++ if (!par_io)
++ return -ENOMEM;
+
+ if (!of_property_read_u32(np, "num-ports", &num_ports))
+ num_par_io_ports = num_ports;
+diff --git a/drivers/soc/imx/gpcv2.c b/drivers/soc/imx/gpcv2.c
+index 34a9ac1f2b9b1..b4aa28420f2a8 100644
+--- a/drivers/soc/imx/gpcv2.c
++++ b/drivers/soc/imx/gpcv2.c
+@@ -244,6 +244,8 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)
+ goto out_regulator_disable;
+ }
+
++ reset_control_assert(domain->reset);
++
+ if (domain->bits.pxx) {
+ /* request the domain to power up */
+ regmap_update_bits(domain->regmap, GPC_PU_PGC_SW_PUP_REQ,
+@@ -266,8 +268,6 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)
+ GPC_PGC_CTRL_PCR);
+ }
+
+- reset_control_assert(domain->reset);
+-
+ /* delay for reset to propagate */
+ udelay(5);
+
+@@ -369,7 +369,7 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd)
+ }
+ }
+
+- pm_runtime_put(domain->dev);
++ pm_runtime_put_sync_suspend(domain->dev);
+
+ return 0;
+
+diff --git a/drivers/soc/imx/soc-imx.c b/drivers/soc/imx/soc-imx.c
+index ac6d856ba228d..77bc12039c3d4 100644
+--- a/drivers/soc/imx/soc-imx.c
++++ b/drivers/soc/imx/soc-imx.c
+@@ -36,6 +36,10 @@ static int __init imx_soc_device_init(void)
+ int ret;
+ int i;
+
++ /* Return early if this is running on devices with different SoCs */
++ if (!__mxc_cpu_type)
++ return 0;
++
+ if (of_machine_is_compatible("fsl,ls1021a"))
+ return 0;
+
+diff --git a/drivers/soc/mediatek/mtk-pm-domains.c b/drivers/soc/mediatek/mtk-pm-domains.c
+index b762bc40f56bd..afd2fd74802d2 100644
+--- a/drivers/soc/mediatek/mtk-pm-domains.c
++++ b/drivers/soc/mediatek/mtk-pm-domains.c
+@@ -443,6 +443,9 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
+ pd->genpd.power_off = scpsys_power_off;
+ pd->genpd.power_on = scpsys_power_on;
+
++ if (MTK_SCPD_CAPS(pd, MTK_SCPD_ACTIVE_WAKEUP))
++ pd->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP;
++
+ if (MTK_SCPD_CAPS(pd, MTK_SCPD_KEEP_DEFAULT_OFF))
+ pm_genpd_init(&pd->genpd, NULL, true);
+ else
+diff --git a/drivers/soc/qcom/apr.c b/drivers/soc/qcom/apr.c
+index 475a57b435b24..2e455d9e3d94a 100644
+--- a/drivers/soc/qcom/apr.c
++++ b/drivers/soc/qcom/apr.c
+@@ -321,12 +321,14 @@ static int of_apr_add_pd_lookups(struct device *dev)
+ 1, &service_path);
+ if (ret < 0) {
+ dev_err(dev, "pdr service path missing: %d\n", ret);
++ of_node_put(node);
+ return ret;
+ }
+
+ pds = pdr_add_lookup(apr->pdr, service_name, service_path);
+ if (IS_ERR(pds) && PTR_ERR(pds) != -EALREADY) {
+ dev_err(dev, "pdr add lookup failed: %ld\n", PTR_ERR(pds));
++ of_node_put(node);
+ return PTR_ERR(pds);
+ }
+ }
+diff --git a/drivers/soc/qcom/cpr.c b/drivers/soc/qcom/cpr.c
+index 4ce8e816154f9..84dd93472a252 100644
+--- a/drivers/soc/qcom/cpr.c
++++ b/drivers/soc/qcom/cpr.c
+@@ -1010,7 +1010,7 @@ static int cpr_interpolate(const struct corner *corner, int step_volt,
+ return corner->uV;
+
+ temp = f_diff * (uV_high - uV_low);
+- do_div(temp, f_high - f_low);
++ temp = div64_ul(temp, f_high - f_low);
+
+ /*
+ * max_volt_scale has units of uV/MHz while freq values
+diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
+index 15a36dcab990e..e53109a5c3da9 100644
+--- a/drivers/soc/qcom/llcc-qcom.c
++++ b/drivers/soc/qcom/llcc-qcom.c
+@@ -115,7 +115,7 @@ static const struct llcc_slice_config sc7280_data[] = {
+ { LLCC_CMPT, 10, 768, 1, 1, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+ { LLCC_GPUHTW, 11, 256, 1, 1, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+ { LLCC_GPU, 12, 512, 1, 0, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+- { LLCC_MMUHWT, 13, 256, 1, 1, 0x3f, 0x0, 0, 0, 0, 1, 1, 0},
++ { LLCC_MMUHWT, 13, 256, 1, 1, 0x3f, 0x0, 0, 0, 0, 0, 1, 0},
+ { LLCC_MDMPNG, 21, 768, 0, 1, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+ { LLCC_WLHW, 24, 256, 1, 1, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+ { LLCC_MODPE, 29, 64, 1, 1, 0x3f, 0x0, 0, 0, 0, 1, 0, 0},
+diff --git a/drivers/soc/qcom/ocmem.c b/drivers/soc/qcom/ocmem.c
+index f1875dc31ae2c..85f82e195ef8b 100644
+--- a/drivers/soc/qcom/ocmem.c
++++ b/drivers/soc/qcom/ocmem.c
+@@ -206,6 +206,7 @@ struct ocmem *of_get_ocmem(struct device *dev)
+ ocmem = platform_get_drvdata(pdev);
+ if (!ocmem) {
+ dev_err(dev, "Cannot get ocmem\n");
++ put_device(&pdev->dev);
+ return ERR_PTR(-ENODEV);
+ }
+ return ocmem;
+diff --git a/drivers/soc/qcom/qcom_aoss.c b/drivers/soc/qcom/qcom_aoss.c
+index 536c3e4114fb9..8583c1e558aeb 100644
+--- a/drivers/soc/qcom/qcom_aoss.c
++++ b/drivers/soc/qcom/qcom_aoss.c
+@@ -8,10 +8,12 @@
+ #include <linux/io.h>
+ #include <linux/mailbox_client.h>
+ #include <linux/module.h>
++#include <linux/of_platform.h>
+ #include <linux/platform_device.h>
+ #include <linux/pm_domain.h>
+ #include <linux/thermal.h>
+ #include <linux/slab.h>
++#include <linux/soc/qcom/qcom_aoss.h>
+
+ #define QMP_DESC_MAGIC 0x0
+ #define QMP_DESC_VERSION 0x4
+@@ -223,11 +225,14 @@ static bool qmp_message_empty(struct qmp *qmp)
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+-static int qmp_send(struct qmp *qmp, const void *data, size_t len)
++int qmp_send(struct qmp *qmp, const void *data, size_t len)
+ {
+ long time_left;
+ int ret;
+
++ if (WARN_ON(IS_ERR_OR_NULL(qmp) || !data))
++ return -EINVAL;
++
+ if (WARN_ON(len + sizeof(u32) > qmp->size))
+ return -EINVAL;
+
+@@ -261,6 +266,7 @@ static int qmp_send(struct qmp *qmp, const void *data, size_t len)
+
+ return ret;
+ }
++EXPORT_SYMBOL(qmp_send);
+
+ static int qmp_qdss_clk_prepare(struct clk_hw *hw)
+ {
+@@ -519,6 +525,55 @@ static void qmp_cooling_devices_remove(struct qmp *qmp)
+ thermal_cooling_device_unregister(qmp->cooling_devs[i].cdev);
+ }
+
++/**
++ * qmp_get() - get a qmp handle from a device
++ * @dev: client device pointer
++ *
++ * Return: handle to qmp device on success, ERR_PTR() on failure
++ */
++struct qmp *qmp_get(struct device *dev)
++{
++ struct platform_device *pdev;
++ struct device_node *np;
++ struct qmp *qmp;
++
++ if (!dev || !dev->of_node)
++ return ERR_PTR(-EINVAL);
++
++ np = of_parse_phandle(dev->of_node, "qcom,qmp", 0);
++ if (!np)
++ return ERR_PTR(-ENODEV);
++
++ pdev = of_find_device_by_node(np);
++ of_node_put(np);
++ if (!pdev)
++ return ERR_PTR(-EINVAL);
++
++ qmp = platform_get_drvdata(pdev);
++
++ if (!qmp) {
++ put_device(&pdev->dev);
++ return ERR_PTR(-EPROBE_DEFER);
++ }
++ return qmp;
++}
++EXPORT_SYMBOL(qmp_get);
++
++/**
++ * qmp_put() - release a qmp handle
++ * @qmp: qmp handle obtained from qmp_get()
++ */
++void qmp_put(struct qmp *qmp)
++{
++ /*
++ * Match get_device() inside of_find_device_by_node() in
++ * qmp_get()
++ */
++ if (!IS_ERR_OR_NULL(qmp))
++ put_device(qmp->dev);
++}
++EXPORT_SYMBOL(qmp_put);
++
+ static int qmp_probe(struct platform_device *pdev)
+ {
+ struct resource *res;
+@@ -548,7 +603,7 @@ static int qmp_probe(struct platform_device *pdev)
+ }
+
+ irq = platform_get_irq(pdev, 0);
+- ret = devm_request_irq(&pdev->dev, irq, qmp_intr, IRQF_ONESHOT,
++ ret = devm_request_irq(&pdev->dev, irq, qmp_intr, 0,
+ "aoss-qmp", qmp);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "failed to request interrupt\n");
+@@ -615,6 +670,7 @@ static struct platform_driver qmp_driver = {
+ .driver = {
+ .name = "qcom_aoss_qmp",
+ .of_match_table = qmp_dt_match,
++ .suppress_bind_attrs = true,
+ },
+ .probe = qmp_probe,
+ .remove = qmp_remove,
+diff --git a/drivers/soc/qcom/rpmhpd.c b/drivers/soc/qcom/rpmhpd.c
+index fa209b479ab35..d98cc8c2e5d5c 100644
+--- a/drivers/soc/qcom/rpmhpd.c
++++ b/drivers/soc/qcom/rpmhpd.c
+@@ -30,6 +30,7 @@
+ * @active_only: True if it represents an Active only peer
+ * @corner: current corner
+ * @active_corner: current active corner
++ * @enable_corner: lowest non-zero corner
+ * @level: An array of level (vlvl) to corner (hlvl) mappings
+ * derived from cmd-db
+ * @level_count: Number of levels supported by the power domain. max
+@@ -47,6 +48,7 @@ struct rpmhpd {
+ const bool active_only;
+ unsigned int corner;
+ unsigned int active_corner;
++ unsigned int enable_corner;
+ u32 level[RPMH_ARC_MAX_LEVELS];
+ size_t level_count;
+ bool enabled;
+@@ -204,7 +206,7 @@ static const struct rpmhpd_desc sm8250_desc = {
+ static struct rpmhpd sm8350_mxc_ao;
+ static struct rpmhpd sm8350_mxc = {
+ .pd = { .name = "mxc", },
+- .peer = &sm8150_mmcx_ao,
++ .peer = &sm8350_mxc_ao,
+ .res_name = "mxc.lvl",
+ };
+
+@@ -385,13 +387,13 @@ static int rpmhpd_aggregate_corner(struct rpmhpd *pd, unsigned int corner)
+ static int rpmhpd_power_on(struct generic_pm_domain *domain)
+ {
+ struct rpmhpd *pd = domain_to_rpmhpd(domain);
+- int ret = 0;
++ unsigned int corner;
++ int ret;
+
+ mutex_lock(&rpmhpd_lock);
+
+- if (pd->corner)
+- ret = rpmhpd_aggregate_corner(pd, pd->corner);
+-
++ corner = max(pd->corner, pd->enable_corner);
++ ret = rpmhpd_aggregate_corner(pd, corner);
+ if (!ret)
+ pd->enabled = true;
+
+@@ -436,6 +438,10 @@ static int rpmhpd_set_performance_state(struct generic_pm_domain *domain,
+ i--;
+
+ if (pd->enabled) {
++ /* Ensure that the domain isn't turn off */
++ if (i < pd->enable_corner)
++ i = pd->enable_corner;
++
+ ret = rpmhpd_aggregate_corner(pd, i);
+ if (ret)
+ goto out;
+@@ -472,6 +478,10 @@ static int rpmhpd_update_level_mapping(struct rpmhpd *rpmhpd)
+ for (i = 0; i < rpmhpd->level_count; i++) {
+ rpmhpd->level[i] = buf[i];
+
++ /* Remember the first corner with non-zero level */
++ if (!rpmhpd->level[rpmhpd->enable_corner] && rpmhpd->level[i])
++ rpmhpd->enable_corner = i;
++
+ /*
+ * The AUX data may be zero padded. These 0 valued entries at
+ * the end of the map must be ignored.
+diff --git a/drivers/soc/qcom/rpmpd.c b/drivers/soc/qcom/rpmpd.c
+index dbf494e925743..9f07274b0d281 100644
+--- a/drivers/soc/qcom/rpmpd.c
++++ b/drivers/soc/qcom/rpmpd.c
+@@ -546,6 +546,9 @@ static int rpmpd_probe(struct platform_device *pdev)
+
+ data->domains = devm_kcalloc(&pdev->dev, num, sizeof(*data->domains),
+ GFP_KERNEL);
++ if (!data->domains)
++ return -ENOMEM;
++
+ data->num_domains = num;
+
+ for (i = 0; i < num; i++) {
+diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
+index 52e5811671155..5beb452f24013 100644
+--- a/drivers/soc/qcom/socinfo.c
++++ b/drivers/soc/qcom/socinfo.c
+@@ -87,8 +87,8 @@ static const char *const pmic_models[] = {
+ [15] = "PM8901",
+ [16] = "PM8950/PM8027",
+ [17] = "PMI8950/ISL9519",
+- [18] = "PM8921",
+- [19] = "PM8018",
++ [18] = "PMK8001/PM8921",
++ [19] = "PMI8996/PM8018",
+ [20] = "PM8998/PM8015",
+ [21] = "PMI8998/PM8014",
+ [22] = "PM8821",
+diff --git a/drivers/soc/samsung/Kconfig b/drivers/soc/samsung/Kconfig
+index 5745d7e5908e9..1f643c0f5c93f 100644
+--- a/drivers/soc/samsung/Kconfig
++++ b/drivers/soc/samsung/Kconfig
+@@ -25,6 +25,7 @@ config EXYNOS_PMU
+ bool "Exynos PMU controller driver" if COMPILE_TEST
+ depends on ARCH_EXYNOS || ((ARM || ARM64) && COMPILE_TEST)
+ select EXYNOS_PMU_ARM_DRIVERS if ARM && ARCH_EXYNOS
++ select MFD_CORE
+
+ # There is no need to enable these drivers for ARMv8
+ config EXYNOS_PMU_ARM_DRIVERS
+diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
+index f2151815db585..e714ed3b61bc3 100644
+--- a/drivers/soc/tegra/fuse/fuse-tegra.c
++++ b/drivers/soc/tegra/fuse/fuse-tegra.c
+@@ -320,7 +320,7 @@ static struct platform_driver tegra_fuse_driver = {
+ };
+ builtin_platform_driver(tegra_fuse_driver);
+
+-bool __init tegra_fuse_read_spare(unsigned int spare)
++u32 __init tegra_fuse_read_spare(unsigned int spare)
+ {
+ unsigned int offset = fuse->soc->info->spare + spare * 4;
+
+diff --git a/drivers/soc/tegra/fuse/fuse.h b/drivers/soc/tegra/fuse/fuse.h
+index de58feba04350..ecff0c08e9595 100644
+--- a/drivers/soc/tegra/fuse/fuse.h
++++ b/drivers/soc/tegra/fuse/fuse.h
+@@ -65,7 +65,7 @@ struct tegra_fuse {
+ void tegra_init_revision(void);
+ void tegra_init_apbmisc(void);
+
+-bool __init tegra_fuse_read_spare(unsigned int spare);
++u32 __init tegra_fuse_read_spare(unsigned int spare);
+ u32 __init tegra_fuse_read_early(unsigned int offset);
+
+ u8 tegra_get_major_rev(void);
+diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
+index 50091c4ec9481..a60e142ade344 100644
+--- a/drivers/soc/tegra/pmc.c
++++ b/drivers/soc/tegra/pmc.c
+@@ -782,7 +782,7 @@ static int tegra_powergate_power_up(struct tegra_powergate *pg,
+
+ err = reset_control_deassert(pg->reset);
+ if (err)
+- goto powergate_off;
++ goto disable_clks;
+
+ usleep_range(10, 20);
+
+diff --git a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c
+index 49da387d77494..b36779309e49b 100644
+--- a/drivers/soc/ti/pruss.c
++++ b/drivers/soc/ti/pruss.c
+@@ -129,7 +129,7 @@ static int pruss_clk_init(struct pruss *pruss, struct device_node *cfg_node)
+
+ clks_np = of_get_child_by_name(cfg_node, "clocks");
+ if (!clks_np) {
+- dev_err(dev, "%pOF is missing its 'clocks' node\n", clks_np);
++ dev_err(dev, "%pOF is missing its 'clocks' node\n", cfg_node);
+ return -ENODEV;
+ }
+
+diff --git a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c
+index 09abd17065ba5..8b3ff44fd9010 100644
+--- a/drivers/soc/ti/wkup_m3_ipc.c
++++ b/drivers/soc/ti/wkup_m3_ipc.c
+@@ -449,9 +449,9 @@ static int wkup_m3_ipc_probe(struct platform_device *pdev)
+ return PTR_ERR(m3_ipc->ipc_mem_base);
+
+ irq = platform_get_irq(pdev, 0);
+- if (!irq) {
++ if (irq < 0) {
+ dev_err(&pdev->dev, "no irq resource\n");
+- return -ENXIO;
++ return irq;
+ }
+
+ ret = devm_request_irq(dev, irq, wkup_m3_txev_handler,
+diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
+index 1b115734a8f6b..67369e941d0d6 100644
+--- a/drivers/soundwire/bus.c
++++ b/drivers/soundwire/bus.c
+@@ -1110,7 +1110,7 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus)
+ if (!simple_clk_stop) {
+ ret = sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM);
+ if (ret < 0)
+- dev_warn(&slave->dev, "clock stop deprepare wait failed:%d\n", ret);
++ dev_warn(bus->dev, "clock stop deprepare wait failed:%d\n", ret);
+ }
+
+ list_for_each_entry(slave, &bus->slaves, node) {
+diff --git a/drivers/soundwire/debugfs.c b/drivers/soundwire/debugfs.c
+index b6cad0d59b7b9..49900cd207bc7 100644
+--- a/drivers/soundwire/debugfs.c
++++ b/drivers/soundwire/debugfs.c
+@@ -19,7 +19,7 @@ void sdw_bus_debugfs_init(struct sdw_bus *bus)
+ return;
+
+ /* create the debugfs master-N */
+- snprintf(name, sizeof(name), "master-%d", bus->link_id);
++ snprintf(name, sizeof(name), "master-%d-%d", bus->id, bus->link_id);
+ bus->debugfs = debugfs_create_dir(name, sdw_debugfs_root);
+ }
+
+diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c
+index 0ca2a3e3a02e2..747983743a14b 100644
+--- a/drivers/soundwire/dmi-quirks.c
++++ b/drivers/soundwire/dmi-quirks.c
+@@ -59,7 +59,7 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "HP"),
+- DMI_MATCH(DMI_PRODUCT_NAME, "HP Spectre x360 Convertible"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "HP Spectre x360 Conv"),
+ },
+ .driver_data = (void *)intel_tgl_bios,
+ },
+diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c
+index 78037ffdb09ba..f72d36654ac2a 100644
+--- a/drivers/soundwire/intel.c
++++ b/drivers/soundwire/intel.c
+@@ -448,8 +448,8 @@ static void intel_shim_wake(struct sdw_intel *sdw, bool wake_enable)
+
+ /* Clear wake status */
+ wake_sts = intel_readw(shim, SDW_SHIM_WAKESTS);
+- wake_sts |= (SDW_SHIM_WAKEEN_ENABLE << link_id);
+- intel_writew(shim, SDW_SHIM_WAKESTS_STATUS, wake_sts);
++ wake_sts |= (SDW_SHIM_WAKESTS_STATUS << link_id);
++ intel_writew(shim, SDW_SHIM_WAKESTS, wake_sts);
+ }
+ mutex_unlock(sdw->link_res->shim_lock);
+ }
+diff --git a/drivers/spi/atmel-quadspi.c b/drivers/spi/atmel-quadspi.c
+index 95d4fa32c2995..938017a60c8ed 100644
+--- a/drivers/spi/atmel-quadspi.c
++++ b/drivers/spi/atmel-quadspi.c
+@@ -277,6 +277,9 @@ static int atmel_qspi_find_mode(const struct spi_mem_op *op)
+ static bool atmel_qspi_supports_op(struct spi_mem *mem,
+ const struct spi_mem_op *op)
+ {
++ if (!spi_mem_default_supports_op(mem, op))
++ return false;
++
+ if (atmel_qspi_find_mode(op) < 0)
+ return false;
+
+@@ -310,7 +313,7 @@ static int atmel_qspi_set_cfg(struct atmel_qspi *aq,
+ return mode;
+ ifr |= atmel_qspi_modes[mode].config;
+
+- if (op->dummy.buswidth && op->dummy.nbytes)
++ if (op->dummy.nbytes)
+ dummy_cycles = op->dummy.nbytes * 8 / op->dummy.buswidth;
+
+ /*
+diff --git a/drivers/spi/spi-armada-3700.c b/drivers/spi/spi-armada-3700.c
+index 46feafe4e201c..d8cc4b270644a 100644
+--- a/drivers/spi/spi-armada-3700.c
++++ b/drivers/spi/spi-armada-3700.c
+@@ -901,7 +901,7 @@ static int a3700_spi_probe(struct platform_device *pdev)
+ return 0;
+
+ error_clk:
+- clk_disable_unprepare(spi->clk);
++ clk_unprepare(spi->clk);
+ error:
+ spi_master_put(master);
+ out:
+diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
+index 3043677ba2226..bd7c7fc739610 100644
+--- a/drivers/spi/spi-bcm-qspi.c
++++ b/drivers/spi/spi-bcm-qspi.c
+@@ -395,7 +395,8 @@ static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,
+ if (addrlen == BSPI_ADDRLEN_4BYTES)
+ bpp = BSPI_BPP_ADDR_SELECT_MASK;
+
+- bpp |= (op->dummy.nbytes * 8) / op->dummy.buswidth;
++ if (op->dummy.nbytes)
++ bpp |= (op->dummy.nbytes * 8) / op->dummy.buswidth;
+
+ switch (width) {
+ case SPI_NBITS_SINGLE:
+@@ -551,7 +552,7 @@ static void bcm_qspi_chip_select(struct bcm_qspi *qspi, int cs)
+ u32 rd = 0;
+ u32 wr = 0;
+
+- if (qspi->base[CHIP_SELECT]) {
++ if (cs >= 0 && qspi->base[CHIP_SELECT]) {
+ rd = bcm_qspi_read(qspi, CHIP_SELECT, 0);
+ wr = (rd & ~0xff) | (1 << cs);
+ if (rd == wr)
+@@ -1032,7 +1033,7 @@ static int bcm_qspi_exec_mem_op(struct spi_mem *mem,
+ addr = op->addr.val;
+ len = op->data.nbytes;
+
+- if (bcm_qspi_bspi_ver_three(qspi) == true) {
++ if (has_bspi(qspi) && bcm_qspi_bspi_ver_three(qspi) == true) {
+ /*
+ * The address coming into this function is a raw flash offset.
+ * But for BSPI <= V3, we need to convert it to a remapped BSPI
+@@ -1051,7 +1052,7 @@ static int bcm_qspi_exec_mem_op(struct spi_mem *mem,
+ len < 4)
+ mspi_read = true;
+
+- if (mspi_read)
++ if (!has_bspi(qspi) || mspi_read)
+ return bcm_qspi_mspi_exec_mem_op(spi, op);
+
+ ret = bcm_qspi_bspi_set_mode(qspi, op, 0);
+@@ -1460,7 +1461,7 @@ int bcm_qspi_probe(struct platform_device *pdev,
+ &qspi->dev_ids[val]);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "IRQ %s not found\n", name);
+- goto qspi_probe_err;
++ goto qspi_unprepare_err;
+ }
+
+ qspi->dev_ids[val].dev = qspi;
+@@ -1475,7 +1476,7 @@ int bcm_qspi_probe(struct platform_device *pdev,
+ if (!num_ints) {
+ dev_err(&pdev->dev, "no IRQs registered, cannot init driver\n");
+ ret = -EINVAL;
+- goto qspi_probe_err;
++ goto qspi_unprepare_err;
+ }
+
+ bcm_qspi_hw_init(qspi);
+@@ -1499,6 +1500,7 @@ int bcm_qspi_probe(struct platform_device *pdev,
+
+ qspi_reg_err:
+ bcm_qspi_hw_uninit(qspi);
++qspi_unprepare_err:
+ clk_disable_unprepare(qspi->clk);
+ qspi_probe_err:
+ kfree(qspi->dev_ids);
+diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
+index 101cc71bffa75..2714ba02b176b 100644
+--- a/drivers/spi/spi-cadence-quadspi.c
++++ b/drivers/spi/spi-cadence-quadspi.c
+@@ -18,6 +18,7 @@
+ #include <linux/iopoll.h>
+ #include <linux/jiffies.h>
+ #include <linux/kernel.h>
++#include <linux/log2.h>
+ #include <linux/module.h>
+ #include <linux/of_device.h>
+ #include <linux/of.h>
+@@ -35,6 +36,7 @@
+ /* Quirks */
+ #define CQSPI_NEEDS_WR_DELAY BIT(0)
+ #define CQSPI_DISABLE_DAC_MODE BIT(1)
++#define CQSPI_NO_SUPPORT_WR_COMPLETION BIT(3)
+
+ /* Capabilities */
+ #define CQSPI_SUPPORTS_OCTAL BIT(0)
+@@ -82,6 +84,7 @@ struct cqspi_st {
+ u32 wr_delay;
+ bool use_direct_mode;
+ struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT];
++ bool wr_completion;
+ };
+
+ struct cqspi_driver_platdata {
+@@ -93,12 +96,6 @@ struct cqspi_driver_platdata {
+ #define CQSPI_TIMEOUT_MS 500
+ #define CQSPI_READ_TIMEOUT_MS 10
+
+-/* Instruction type */
+-#define CQSPI_INST_TYPE_SINGLE 0
+-#define CQSPI_INST_TYPE_DUAL 1
+-#define CQSPI_INST_TYPE_QUAD 2
+-#define CQSPI_INST_TYPE_OCTAL 3
+-
+ #define CQSPI_DUMMY_CLKS_PER_BYTE 8
+ #define CQSPI_DUMMY_BYTES_MAX 4
+ #define CQSPI_DUMMY_CLKS_MAX 31
+@@ -322,10 +319,6 @@ static unsigned int cqspi_calc_dummy(const struct spi_mem_op *op, bool dtr)
+ static int cqspi_set_protocol(struct cqspi_flash_pdata *f_pdata,
+ const struct spi_mem_op *op)
+ {
+- f_pdata->inst_width = CQSPI_INST_TYPE_SINGLE;
+- f_pdata->addr_width = CQSPI_INST_TYPE_SINGLE;
+- f_pdata->data_width = CQSPI_INST_TYPE_SINGLE;
+-
+ /*
+ * For an op to be DTR, cmd phase along with every other non-empty
+ * phase should have dtr field set to 1. If an op phase has zero
+@@ -335,32 +328,23 @@ static int cqspi_set_protocol(struct cqspi_flash_pdata *f_pdata,
+ (!op->addr.nbytes || op->addr.dtr) &&
+ (!op->data.nbytes || op->data.dtr);
+
+- switch (op->data.buswidth) {
+- case 0:
+- break;
+- case 1:
+- f_pdata->data_width = CQSPI_INST_TYPE_SINGLE;
+- break;
+- case 2:
+- f_pdata->data_width = CQSPI_INST_TYPE_DUAL;
+- break;
+- case 4:
+- f_pdata->data_width = CQSPI_INST_TYPE_QUAD;
+- break;
+- case 8:
+- f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
+- break;
+- default:
+- return -EINVAL;
+- }
++ f_pdata->inst_width = 0;
++ if (op->cmd.buswidth)
++ f_pdata->inst_width = ilog2(op->cmd.buswidth);
++
++ f_pdata->addr_width = 0;
++ if (op->addr.buswidth)
++ f_pdata->addr_width = ilog2(op->addr.buswidth);
++
++ f_pdata->data_width = 0;
++ if (op->data.buswidth)
++ f_pdata->data_width = ilog2(op->data.buswidth);
+
+ /* Right now we only support 8-8-8 DTR mode. */
+ if (f_pdata->dtr) {
+ switch (op->cmd.buswidth) {
+ case 0:
+- break;
+ case 8:
+- f_pdata->inst_width = CQSPI_INST_TYPE_OCTAL;
+ break;
+ default:
+ return -EINVAL;
+@@ -368,9 +352,7 @@ static int cqspi_set_protocol(struct cqspi_flash_pdata *f_pdata,
+
+ switch (op->addr.buswidth) {
+ case 0:
+- break;
+ case 8:
+- f_pdata->addr_width = CQSPI_INST_TYPE_OCTAL;
+ break;
+ default:
+ return -EINVAL;
+@@ -378,9 +360,7 @@ static int cqspi_set_protocol(struct cqspi_flash_pdata *f_pdata,
+
+ switch (op->data.buswidth) {
+ case 0:
+- break;
+ case 8:
+- f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
+ break;
+ default:
+ return -EINVAL;
+@@ -819,9 +799,11 @@ static int cqspi_write_setup(struct cqspi_flash_pdata *f_pdata,
+ * polling on the controller's side. spinand and spi-nor will take
+ * care of polling the status register.
+ */
+- reg = readl(reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
+- reg |= CQSPI_REG_WR_DISABLE_AUTO_POLL;
+- writel(reg, reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
++ if (cqspi->wr_completion) {
++ reg = readl(reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
++ reg |= CQSPI_REG_WR_DISABLE_AUTO_POLL;
++ writel(reg, reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
++ }
+
+ reg = readl(reg_base + CQSPI_REG_SIZE);
+ reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
+@@ -1248,9 +1230,24 @@ static bool cqspi_supports_mem_op(struct spi_mem *mem,
+ all_false = !op->cmd.dtr && !op->addr.dtr && !op->dummy.dtr &&
+ !op->data.dtr;
+
+- /* Mixed DTR modes not supported. */
+- if (!(all_true || all_false))
++ if (all_true) {
++ /* Right now we only support 8-8-8 DTR mode. */
++ if (op->cmd.nbytes && op->cmd.buswidth != 8)
++ return false;
++ if (op->addr.nbytes && op->addr.buswidth != 8)
++ return false;
++ if (op->data.nbytes && op->data.buswidth != 8)
++ return false;
++ } else if (all_false) {
++ /* Only 1-1-X ops are supported without DTR */
++ if (op->cmd.nbytes && op->cmd.buswidth > 1)
++ return false;
++ if (op->addr.nbytes && op->addr.buswidth > 1)
++ return false;
++ } else {
++ /* Mixed DTR modes are not supported. */
+ return false;
++ }
+
+ if (all_true)
+ return spi_mem_dtr_supports_op(mem, op);
+@@ -1539,6 +1536,10 @@ static int cqspi_probe(struct platform_device *pdev)
+
+ cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk);
+ master->max_speed_hz = cqspi->master_ref_clk_hz;
++
++ /* write completion is supported by default */
++ cqspi->wr_completion = true;
++
+ ddata = of_device_get_match_data(dev);
+ if (ddata) {
+ if (ddata->quirks & CQSPI_NEEDS_WR_DELAY)
+@@ -1548,6 +1549,8 @@ static int cqspi_probe(struct platform_device *pdev)
+ master->mode_bits |= SPI_RX_OCTAL | SPI_TX_OCTAL;
+ if (!(ddata->quirks & CQSPI_DISABLE_DAC_MODE))
+ cqspi->use_direct_mode = true;
++ if (ddata->quirks & CQSPI_NO_SUPPORT_WR_COMPLETION)
++ cqspi->wr_completion = false;
+ }
+
+ ret = devm_request_irq(dev, irq, cqspi_irq_handler, 0,
+@@ -1656,6 +1659,10 @@ static const struct cqspi_driver_platdata intel_lgm_qspi = {
+ .quirks = CQSPI_DISABLE_DAC_MODE,
+ };
+
++static const struct cqspi_driver_platdata socfpga_qspi = {
++ .quirks = CQSPI_NO_SUPPORT_WR_COMPLETION,
++};
++
+ static const struct of_device_id cqspi_dt_ids[] = {
+ {
+ .compatible = "cdns,qspi-nor",
+@@ -1673,6 +1680,10 @@ static const struct of_device_id cqspi_dt_ids[] = {
+ .compatible = "intel,lgm-qspi",
+ .data = &intel_lgm_qspi,
+ },
++ {
++ .compatible = "intel,socfpga-qspi",
++ .data = (void *)&socfpga_qspi,
++ },
+ { /* end of table */ }
+ };
+
+diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c
+index 58b823a16fc4d..525cc0143a305 100644
+--- a/drivers/spi/spi-hisi-kunpeng.c
++++ b/drivers/spi/spi-hisi-kunpeng.c
+@@ -127,7 +127,6 @@ struct hisi_spi {
+ void __iomem *regs;
+ int irq;
+ u32 fifo_len; /* depth of the FIFO buffer */
+- u16 bus_num;
+
+ /* Current message transfer state info */
+ const void *tx;
+@@ -165,7 +164,10 @@ static int hisi_spi_debugfs_init(struct hisi_spi *hs)
+ {
+ char name[32];
+
+- snprintf(name, 32, "hisi_spi%d", hs->bus_num);
++ struct spi_controller *master;
++
++ master = container_of(hs->dev, struct spi_controller, dev);
++ snprintf(name, 32, "hisi_spi%d", master->bus_num);
+ hs->debugfs = debugfs_create_dir(name, NULL);
+ if (!hs->debugfs)
+ return -ENOMEM;
+@@ -467,7 +469,6 @@ static int hisi_spi_probe(struct platform_device *pdev)
+ hs = spi_controller_get_devdata(master);
+ hs->dev = dev;
+ hs->irq = irq;
+- hs->bus_num = pdev->id;
+
+ hs->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(hs->regs))
+@@ -490,7 +491,7 @@ static int hisi_spi_probe(struct platform_device *pdev)
+ master->use_gpio_descriptors = true;
+ master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LOOP;
+ master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
+- master->bus_num = hs->bus_num;
++ master->bus_num = pdev->id;
+ master->setup = hisi_spi_setup;
+ master->cleanup = hisi_spi_cleanup;
+ master->transfer_one = hisi_spi_transfer_one;
+@@ -506,15 +507,15 @@ static int hisi_spi_probe(struct platform_device *pdev)
+ return ret;
+ }
+
+- if (hisi_spi_debugfs_init(hs))
+- dev_info(dev, "failed to create debugfs dir\n");
+-
+ ret = spi_register_controller(master);
+ if (ret) {
+ dev_err(dev, "failed to register spi master, ret=%d\n", ret);
+ return ret;
+ }
+
++ if (hisi_spi_debugfs_init(hs))
++ dev_info(dev, "failed to create debugfs dir\n");
++
+ dev_info(dev, "hw version:0x%x max-freq:%u kHz\n",
+ readl(hs->regs + HISI_SPI_VERSION),
+ master->max_speed_hz / 1000);
+diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c
+index c208efeadd184..0bc7daa7afc83 100644
+--- a/drivers/spi/spi-meson-spicc.c
++++ b/drivers/spi/spi-meson-spicc.c
+@@ -693,6 +693,11 @@ static int meson_spicc_probe(struct platform_device *pdev)
+ writel_relaxed(0, spicc->base + SPICC_INTREG);
+
+ irq = platform_get_irq(pdev, 0);
++ if (irq < 0) {
++ ret = irq;
++ goto out_master;
++ }
++
+ ret = devm_request_irq(&pdev->dev, irq, meson_spicc_irq,
+ 0, NULL, spicc);
+ if (ret) {
+diff --git a/drivers/spi/spi-meson-spifc.c b/drivers/spi/spi-meson-spifc.c
+index 8eca6f24cb799..c8ed7815c4ba6 100644
+--- a/drivers/spi/spi-meson-spifc.c
++++ b/drivers/spi/spi-meson-spifc.c
+@@ -349,6 +349,7 @@ static int meson_spifc_probe(struct platform_device *pdev)
+ return 0;
+ out_clk:
+ clk_disable_unprepare(spifc->clk);
++ pm_runtime_disable(spifc->dev);
+ out_err:
+ spi_master_put(master);
+ return ret;
+diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c
+index a15de10ee286a..2ca19b01948a2 100644
+--- a/drivers/spi/spi-mt65xx.c
++++ b/drivers/spi/spi-mt65xx.c
+@@ -43,8 +43,11 @@
+ #define SPI_CFG1_PACKET_LOOP_OFFSET 8
+ #define SPI_CFG1_PACKET_LENGTH_OFFSET 16
+ #define SPI_CFG1_GET_TICK_DLY_OFFSET 29
++#define SPI_CFG1_GET_TICK_DLY_OFFSET_V1 30
+
+ #define SPI_CFG1_GET_TICK_DLY_MASK 0xe0000000
++#define SPI_CFG1_GET_TICK_DLY_MASK_V1 0xc0000000
++
+ #define SPI_CFG1_CS_IDLE_MASK 0xff
+ #define SPI_CFG1_PACKET_LOOP_MASK 0xff00
+ #define SPI_CFG1_PACKET_LENGTH_MASK 0x3ff0000
+@@ -346,9 +349,15 @@ static int mtk_spi_prepare_message(struct spi_master *master,
+
+ /* tick delay */
+ reg_val = readl(mdata->base + SPI_CFG1_REG);
+- reg_val &= ~SPI_CFG1_GET_TICK_DLY_MASK;
+- reg_val |= ((chip_config->tick_delay & 0x7)
+- << SPI_CFG1_GET_TICK_DLY_OFFSET);
++ if (mdata->dev_comp->enhance_timing) {
++ reg_val &= ~SPI_CFG1_GET_TICK_DLY_MASK;
++ reg_val |= ((chip_config->tick_delay & 0x7)
++ << SPI_CFG1_GET_TICK_DLY_OFFSET);
++ } else {
++ reg_val &= ~SPI_CFG1_GET_TICK_DLY_MASK_V1;
++ reg_val |= ((chip_config->tick_delay & 0x3)
++ << SPI_CFG1_GET_TICK_DLY_OFFSET_V1);
++ }
+ writel(reg_val, mdata->base + SPI_CFG1_REG);
+
+ /* set hw cs timing */
+@@ -624,7 +633,7 @@ static irqreturn_t mtk_spi_interrupt(int irq, void *dev_id)
+ else
+ mdata->state = MTK_SPI_IDLE;
+
+- if (!master->can_dma(master, master->cur_msg->spi, trans)) {
++ if (!master->can_dma(master, NULL, trans)) {
+ if (trans->rx_buf) {
+ cnt = mdata->xfer_len / 4;
+ ioread32_rep(mdata->base + SPI_RX_DATA_REG,
+diff --git a/drivers/spi/spi-mtk-nor.c b/drivers/spi/spi-mtk-nor.c
+index 41e7b341d2616..6d203477c04b1 100644
+--- a/drivers/spi/spi-mtk-nor.c
++++ b/drivers/spi/spi-mtk-nor.c
+@@ -160,7 +160,7 @@ static bool mtk_nor_match_read(const struct spi_mem_op *op)
+ {
+ int dummy = 0;
+
+- if (op->dummy.buswidth)
++ if (op->dummy.nbytes)
+ dummy = op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth;
+
+ if ((op->data.buswidth == 2) || (op->data.buswidth == 4)) {
+@@ -909,7 +909,17 @@ static int __maybe_unused mtk_nor_suspend(struct device *dev)
+
+ static int __maybe_unused mtk_nor_resume(struct device *dev)
+ {
+- return pm_runtime_force_resume(dev);
++ struct spi_controller *ctlr = dev_get_drvdata(dev);
++ struct mtk_nor *sp = spi_controller_get_devdata(ctlr);
++ int ret;
++
++ ret = pm_runtime_force_resume(dev);
++ if (ret)
++ return ret;
++
++ mtk_nor_init(sp);
++
++ return 0;
+ }
+
+ static const struct dev_pm_ops mtk_nor_pm_ops = {
+diff --git a/drivers/spi/spi-mxic.c b/drivers/spi/spi-mxic.c
+index 45889947afed8..03fce4493aa79 100644
+--- a/drivers/spi/spi-mxic.c
++++ b/drivers/spi/spi-mxic.c
+@@ -304,25 +304,21 @@ static int mxic_spi_data_xfer(struct mxic_spi *mxic, const void *txbuf,
+
+ writel(data, mxic->regs + TXD(nbytes % 4));
+
++ ret = readl_poll_timeout(mxic->regs + INT_STS, sts,
++ sts & INT_TX_EMPTY, 0, USEC_PER_SEC);
++ if (ret)
++ return ret;
++
++ ret = readl_poll_timeout(mxic->regs + INT_STS, sts,
++ sts & INT_RX_NOT_EMPTY, 0,
++ USEC_PER_SEC);
++ if (ret)
++ return ret;
++
++ data = readl(mxic->regs + RXD);
+ if (rxbuf) {
+- ret = readl_poll_timeout(mxic->regs + INT_STS, sts,
+- sts & INT_TX_EMPTY, 0,
+- USEC_PER_SEC);
+- if (ret)
+- return ret;
+-
+- ret = readl_poll_timeout(mxic->regs + INT_STS, sts,
+- sts & INT_RX_NOT_EMPTY, 0,
+- USEC_PER_SEC);
+- if (ret)
+- return ret;
+-
+- data = readl(mxic->regs + RXD);
+ data >>= (8 * (4 - nbytes));
+ memcpy(rxbuf + pos, &data, nbytes);
+- WARN_ON(readl(mxic->regs + INT_STS) & INT_RX_NOT_EMPTY);
+- } else {
+- readl(mxic->regs + RXD);
+ }
+ WARN_ON(readl(mxic->regs + INT_STS) & INT_RX_NOT_EMPTY);
+
+diff --git a/drivers/spi/spi-pxa2xx-pci.c b/drivers/spi/spi-pxa2xx-pci.c
+index 2e134eb4bd2c9..6502fda6243e0 100644
+--- a/drivers/spi/spi-pxa2xx-pci.c
++++ b/drivers/spi/spi-pxa2xx-pci.c
+@@ -76,14 +76,23 @@ static bool lpss_dma_filter(struct dma_chan *chan, void *param)
+ return true;
+ }
+
++static void lpss_dma_put_device(void *dma_dev)
++{
++ pci_dev_put(dma_dev);
++}
++
+ static int lpss_spi_setup(struct pci_dev *dev, struct pxa_spi_info *c)
+ {
+ struct pci_dev *dma_dev;
++ int ret;
+
+ c->num_chipselect = 1;
+ c->max_clk_rate = 50000000;
+
+ dma_dev = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
++ ret = devm_add_action_or_reset(&dev->dev, lpss_dma_put_device, dma_dev);
++ if (ret)
++ return ret;
+
+ if (c->tx_param) {
+ struct dw_dma_slave *slave = c->tx_param;
+@@ -107,8 +116,9 @@ static int lpss_spi_setup(struct pci_dev *dev, struct pxa_spi_info *c)
+
+ static int mrfld_spi_setup(struct pci_dev *dev, struct pxa_spi_info *c)
+ {
+- struct pci_dev *dma_dev = pci_get_slot(dev->bus, PCI_DEVFN(21, 0));
+ struct dw_dma_slave *tx, *rx;
++ struct pci_dev *dma_dev;
++ int ret;
+
+ switch (PCI_FUNC(dev->devfn)) {
+ case 0:
+@@ -133,6 +143,11 @@ static int mrfld_spi_setup(struct pci_dev *dev, struct pxa_spi_info *c)
+ return -ENODEV;
+ }
+
++ dma_dev = pci_get_slot(dev->bus, PCI_DEVFN(21, 0));
++ ret = devm_add_action_or_reset(&dev->dev, lpss_dma_put_device, dma_dev);
++ if (ret)
++ return ret;
++
+ tx = c->tx_param;
+ tx->dma_dev = &dma_dev->dev;
+
+diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c
+index 553b6b9d02222..c6a1bb09be056 100644
+--- a/drivers/spi/spi-rockchip.c
++++ b/drivers/spi/spi-rockchip.c
+@@ -585,6 +585,12 @@ static int rockchip_spi_slave_abort(struct spi_controller *ctlr)
+ {
+ struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);
+
++ if (atomic_read(&rs->state) & RXDMA)
++ dmaengine_terminate_sync(ctlr->dma_rx);
++ if (atomic_read(&rs->state) & TXDMA)
++ dmaengine_terminate_sync(ctlr->dma_tx);
++ atomic_set(&rs->state, 0);
++ spi_enable_chip(rs, false);
+ rs->slave_abort = true;
+ spi_finalize_current_transfer(ctlr);
+
+@@ -654,7 +660,7 @@ static int rockchip_spi_probe(struct platform_device *pdev)
+ struct spi_controller *ctlr;
+ struct resource *mem;
+ struct device_node *np = pdev->dev.of_node;
+- u32 rsd_nsecs;
++ u32 rsd_nsecs, num_cs;
+ bool slave_mode;
+
+ slave_mode = of_property_read_bool(np, "spi-slave");
+@@ -764,8 +770,9 @@ static int rockchip_spi_probe(struct platform_device *pdev)
+ * rk spi0 has two native cs, spi1..5 one cs only
+ * if num-cs is missing in the dts, default to 1
+ */
+- if (of_property_read_u16(np, "num-cs", &ctlr->num_chipselect))
+- ctlr->num_chipselect = 1;
++ if (of_property_read_u32(np, "num-cs", &num_cs))
++ num_cs = 1;
++ ctlr->num_chipselect = num_cs;
+ ctlr->use_gpio_descriptors = true;
+ }
+ ctlr->dev.of_node = pdev->dev.of_node;
+diff --git a/drivers/spi/spi-rpc-if.c b/drivers/spi/spi-rpc-if.c
+index c53138ce00309..83796a4ead34a 100644
+--- a/drivers/spi/spi-rpc-if.c
++++ b/drivers/spi/spi-rpc-if.c
+@@ -139,7 +139,9 @@ static int rpcif_spi_probe(struct platform_device *pdev)
+ return -ENOMEM;
+
+ rpc = spi_controller_get_devdata(ctlr);
+- rpcif_sw_init(rpc, parent);
++ error = rpcif_sw_init(rpc, parent);
++ if (error)
++ return error;
+
+ platform_set_drvdata(pdev, ctlr);
+
+diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c
+index 27f35aa2d746d..ffdc55f87e821 100644
+--- a/drivers/spi/spi-stm32-qspi.c
++++ b/drivers/spi/spi-stm32-qspi.c
+@@ -397,7 +397,7 @@ static int stm32_qspi_send(struct spi_mem *mem, const struct spi_mem_op *op)
+ ccr |= FIELD_PREP(CCR_ADSIZE_MASK, op->addr.nbytes - 1);
+ }
+
+- if (op->dummy.buswidth && op->dummy.nbytes)
++ if (op->dummy.nbytes)
+ ccr |= FIELD_PREP(CCR_DCYC_MASK,
+ op->dummy.nbytes * 8 / op->dummy.buswidth);
+
+@@ -688,7 +688,7 @@ static int stm32_qspi_probe(struct platform_device *pdev)
+ struct resource *res;
+ int ret, irq;
+
+- ctrl = spi_alloc_master(dev, sizeof(*qspi));
++ ctrl = devm_spi_alloc_master(dev, sizeof(*qspi));
+ if (!ctrl)
+ return -ENOMEM;
+
+@@ -697,58 +697,46 @@ static int stm32_qspi_probe(struct platform_device *pdev)
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi");
+ qspi->io_base = devm_ioremap_resource(dev, res);
+- if (IS_ERR(qspi->io_base)) {
+- ret = PTR_ERR(qspi->io_base);
+- goto err_master_put;
+- }
++ if (IS_ERR(qspi->io_base))
++ return PTR_ERR(qspi->io_base);
+
+ qspi->phys_base = res->start;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi_mm");
+ qspi->mm_base = devm_ioremap_resource(dev, res);
+- if (IS_ERR(qspi->mm_base)) {
+- ret = PTR_ERR(qspi->mm_base);
+- goto err_master_put;
+- }
++ if (IS_ERR(qspi->mm_base))
++ return PTR_ERR(qspi->mm_base);
+
+ qspi->mm_size = resource_size(res);
+- if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ) {
+- ret = -EINVAL;
+- goto err_master_put;
+- }
++ if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ)
++ return -EINVAL;
+
+ irq = platform_get_irq(pdev, 0);
+- if (irq < 0) {
+- ret = irq;
+- goto err_master_put;
+- }
++ if (irq < 0)
++ return irq;
+
+ ret = devm_request_irq(dev, irq, stm32_qspi_irq, 0,
+ dev_name(dev), qspi);
+ if (ret) {
+ dev_err(dev, "failed to request irq\n");
+- goto err_master_put;
++ return ret;
+ }
+
+ init_completion(&qspi->data_completion);
+ init_completion(&qspi->match_completion);
+
+ qspi->clk = devm_clk_get(dev, NULL);
+- if (IS_ERR(qspi->clk)) {
+- ret = PTR_ERR(qspi->clk);
+- goto err_master_put;
+- }
++ if (IS_ERR(qspi->clk))
++ return PTR_ERR(qspi->clk);
+
+ qspi->clk_rate = clk_get_rate(qspi->clk);
+- if (!qspi->clk_rate) {
+- ret = -EINVAL;
+- goto err_master_put;
+- }
++ if (!qspi->clk_rate)
++ return -EINVAL;
+
+ ret = clk_prepare_enable(qspi->clk);
+ if (ret) {
+ dev_err(dev, "can not enable the clock\n");
+- goto err_master_put;
++ return ret;
+ }
+
+ rstc = devm_reset_control_get_exclusive(dev, NULL);
+@@ -784,7 +772,7 @@ static int stm32_qspi_probe(struct platform_device *pdev)
+ pm_runtime_enable(dev);
+ pm_runtime_get_noresume(dev);
+
+- ret = devm_spi_register_master(dev, ctrl);
++ ret = spi_register_master(ctrl);
+ if (ret)
+ goto err_pm_runtime_free;
+
+@@ -806,8 +794,6 @@ err_dma_free:
+ stm32_qspi_dma_free(qspi);
+ err_clk_disable:
+ clk_disable_unprepare(qspi->clk);
+-err_master_put:
+- spi_master_put(qspi->ctrl);
+
+ return ret;
+ }
+@@ -817,6 +803,7 @@ static int stm32_qspi_remove(struct platform_device *pdev)
+ struct stm32_qspi *qspi = platform_get_drvdata(pdev);
+
+ pm_runtime_get_sync(qspi->dev);
++ spi_unregister_master(qspi->ctrl);
+ /* disable qspi */
+ writel_relaxed(0, qspi->io_base + QSPI_CR);
+ stm32_qspi_dma_free(qspi);
+diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c
+index e9de1d958bbd2..8f345247a8c32 100644
+--- a/drivers/spi/spi-tegra114.c
++++ b/drivers/spi/spi-tegra114.c
+@@ -1352,6 +1352,10 @@ static int tegra_spi_probe(struct platform_device *pdev)
+ tspi->phys = r->start;
+
+ spi_irq = platform_get_irq(pdev, 0);
++ if (spi_irq < 0) {
++ ret = spi_irq;
++ goto exit_free_master;
++ }
+ tspi->irq = spi_irq;
+
+ tspi->clk = devm_clk_get(&pdev->dev, "spi");
+diff --git a/drivers/spi/spi-tegra20-slink.c b/drivers/spi/spi-tegra20-slink.c
+index 3226c4e1c7c0d..3b44ca455049d 100644
+--- a/drivers/spi/spi-tegra20-slink.c
++++ b/drivers/spi/spi-tegra20-slink.c
+@@ -1003,14 +1003,8 @@ static int tegra_slink_probe(struct platform_device *pdev)
+ struct resource *r;
+ int ret, spi_irq;
+ const struct tegra_slink_chip_data *cdata = NULL;
+- const struct of_device_id *match;
+
+- match = of_match_device(tegra_slink_of_match, &pdev->dev);
+- if (!match) {
+- dev_err(&pdev->dev, "Error: No device match found\n");
+- return -ENODEV;
+- }
+- cdata = match->data;
++ cdata = of_device_get_match_data(&pdev->dev);
+
+ master = spi_alloc_master(&pdev->dev, sizeof(*tspi));
+ if (!master) {
+diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c
+index 2354ca1e38581..7967073c13545 100644
+--- a/drivers/spi/spi-tegra210-quad.c
++++ b/drivers/spi/spi-tegra210-quad.c
+@@ -1249,6 +1249,8 @@ static int tegra_qspi_probe(struct platform_device *pdev)
+
+ tqspi->phys = r->start;
+ qspi_irq = platform_get_irq(pdev, 0);
++ if (qspi_irq < 0)
++ return qspi_irq;
+ tqspi->irq = qspi_irq;
+
+ tqspi->clk = devm_clk_get(&pdev->dev, "qspi");
+diff --git a/drivers/spi/spi-uniphier.c b/drivers/spi/spi-uniphier.c
+index 8900e51e1a1cc..cc0da48222311 100644
+--- a/drivers/spi/spi-uniphier.c
++++ b/drivers/spi/spi-uniphier.c
+@@ -726,7 +726,7 @@ static int uniphier_spi_probe(struct platform_device *pdev)
+ if (ret) {
+ dev_err(&pdev->dev, "failed to get TX DMA capacities: %d\n",
+ ret);
+- goto out_disable_clk;
++ goto out_release_dma;
+ }
+ dma_tx_burst = caps.max_burst;
+ }
+@@ -735,7 +735,7 @@ static int uniphier_spi_probe(struct platform_device *pdev)
+ if (IS_ERR_OR_NULL(master->dma_rx)) {
+ if (PTR_ERR(master->dma_rx) == -EPROBE_DEFER) {
+ ret = -EPROBE_DEFER;
+- goto out_disable_clk;
++ goto out_release_dma;
+ }
+ master->dma_rx = NULL;
+ dma_rx_burst = INT_MAX;
+@@ -744,7 +744,7 @@ static int uniphier_spi_probe(struct platform_device *pdev)
+ if (ret) {
+ dev_err(&pdev->dev, "failed to get RX DMA capacities: %d\n",
+ ret);
+- goto out_disable_clk;
++ goto out_release_dma;
+ }
+ dma_rx_burst = caps.max_burst;
+ }
+@@ -753,10 +753,20 @@ static int uniphier_spi_probe(struct platform_device *pdev)
+
+ ret = devm_spi_register_master(&pdev->dev, master);
+ if (ret)
+- goto out_disable_clk;
++ goto out_release_dma;
+
+ return 0;
+
++out_release_dma:
++ if (!IS_ERR_OR_NULL(master->dma_rx)) {
++ dma_release_channel(master->dma_rx);
++ master->dma_rx = NULL;
++ }
++ if (!IS_ERR_OR_NULL(master->dma_tx)) {
++ dma_release_channel(master->dma_tx);
++ master->dma_tx = NULL;
++ }
++
+ out_disable_clk:
+ clk_disable_unprepare(priv->clk);
+
+@@ -767,12 +777,13 @@ out_master_put:
+
+ static int uniphier_spi_remove(struct platform_device *pdev)
+ {
+- struct uniphier_spi_priv *priv = platform_get_drvdata(pdev);
++ struct spi_master *master = platform_get_drvdata(pdev);
++ struct uniphier_spi_priv *priv = spi_master_get_devdata(master);
+
+- if (priv->master->dma_tx)
+- dma_release_channel(priv->master->dma_tx);
+- if (priv->master->dma_rx)
+- dma_release_channel(priv->master->dma_rx);
++ if (master->dma_tx)
++ dma_release_channel(master->dma_tx);
++ if (master->dma_rx)
++ dma_release_channel(master->dma_rx);
+
+ clk_disable_unprepare(priv->clk);
+
+diff --git a/drivers/spi/spi-zynq-qspi.c b/drivers/spi/spi-zynq-qspi.c
+index cfa222c9bd5e7..78f31b61a2aac 100644
+--- a/drivers/spi/spi-zynq-qspi.c
++++ b/drivers/spi/spi-zynq-qspi.c
+@@ -570,6 +570,9 @@ static int zynq_qspi_exec_mem_op(struct spi_mem *mem,
+
+ if (op->dummy.nbytes) {
+ tmpbuf = kzalloc(op->dummy.nbytes, GFP_KERNEL);
++ if (!tmpbuf)
++ return -ENOMEM;
++
+ memset(tmpbuf, 0xff, op->dummy.nbytes);
+ reinit_completion(&xqspi->data_completion);
+ xqspi->txbuf = tmpbuf;
+diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c
+index 328b6559bb19a..2b5afae8ff7fc 100644
+--- a/drivers/spi/spi-zynqmp-gqspi.c
++++ b/drivers/spi/spi-zynqmp-gqspi.c
+@@ -1172,7 +1172,10 @@ static int zynqmp_qspi_probe(struct platform_device *pdev)
+ goto clk_dis_all;
+ }
+
+- dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
++ ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
++ if (ret)
++ goto clk_dis_all;
++
+ ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
+ ctlr->num_chipselect = GQSPI_DEFAULT_NUM_CS;
+ ctlr->mem_ops = &zynqmp_qspi_mem_ops;
+diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
+index 926b68aa45d3e..d0bbf8f9414d4 100644
+--- a/drivers/spi/spi.c
++++ b/drivers/spi/spi.c
+@@ -451,6 +451,47 @@ int __spi_register_driver(struct module *owner, struct spi_driver *sdrv)
+ {
+ sdrv->driver.owner = owner;
+ sdrv->driver.bus = &spi_bus_type;
++
++ /*
++ * For Really Good Reasons we use spi: modaliases not of:
++ * modaliases for DT so module autoloading won't work if we
++ * don't have a spi_device_id as well as a compatible string.
++ */
++ if (sdrv->driver.of_match_table) {
++ const struct of_device_id *of_id;
++
++ for (of_id = sdrv->driver.of_match_table; of_id->compatible[0];
++ of_id++) {
++ const char *of_name;
++
++ /* Strip off any vendor prefix */
++ of_name = strnchr(of_id->compatible,
++ sizeof(of_id->compatible), ',');
++ if (of_name)
++ of_name++;
++ else
++ of_name = of_id->compatible;
++
++ if (sdrv->id_table) {
++ const struct spi_device_id *spi_id;
++
++ for (spi_id = sdrv->id_table; spi_id->name[0];
++ spi_id++)
++ if (strcmp(spi_id->name, of_name) == 0)
++ break;
++
++ if (spi_id->name[0])
++ continue;
++ } else {
++ if (strcmp(sdrv->driver.name, of_name) == 0)
++ continue;
++ }
++
++ pr_warn("SPI driver %s has no spi_device_id for %s\n",
++ sdrv->driver.name, of_id->compatible);
++ }
++ }
++
+ return driver_register(&sdrv->driver);
+ }
+ EXPORT_SYMBOL_GPL(__spi_register_driver);
+@@ -827,12 +868,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force)
+ spi->controller->last_cs_enable = enable;
+ spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH;
+
+- if (spi->cs_gpiod || gpio_is_valid(spi->cs_gpio) ||
+- !spi->controller->set_cs_timing) {
+- if (activate)
+- spi_delay_exec(&spi->cs_setup, NULL);
+- else
+- spi_delay_exec(&spi->cs_hold, NULL);
++ if ((spi->cs_gpiod || gpio_is_valid(spi->cs_gpio) ||
++ !spi->controller->set_cs_timing) && !activate) {
++ spi_delay_exec(&spi->cs_hold, NULL);
+ }
+
+ if (spi->mode & SPI_CS_HIGH)
+@@ -874,7 +912,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force)
+
+ if (spi->cs_gpiod || gpio_is_valid(spi->cs_gpio) ||
+ !spi->controller->set_cs_timing) {
+- if (!activate)
++ if (activate)
++ spi_delay_exec(&spi->cs_setup, NULL);
++ else
+ spi_delay_exec(&spi->cs_inactive, NULL);
+ }
+ }
+@@ -902,10 +942,10 @@ int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
+ int i, ret;
+
+ if (vmalloced_buf || kmap_buf) {
+- desc_len = min_t(int, max_seg_size, PAGE_SIZE);
++ desc_len = min_t(unsigned long, max_seg_size, PAGE_SIZE);
+ sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);
+ } else if (virt_addr_valid(buf)) {
+- desc_len = min_t(int, max_seg_size, ctlr->max_dma_len);
++ desc_len = min_t(size_t, max_seg_size, ctlr->max_dma_len);
+ sgs = DIV_ROUND_UP(len, desc_len);
+ } else {
+ return -EINVAL;
+@@ -1032,11 +1072,15 @@ static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)
+
+ if (ctlr->dma_tx)
+ tx_dev = ctlr->dma_tx->device->dev;
++ else if (ctlr->dma_map_dev)
++ tx_dev = ctlr->dma_map_dev;
+ else
+ tx_dev = ctlr->dev.parent;
+
+ if (ctlr->dma_rx)
+ rx_dev = ctlr->dma_rx->device->dev;
++ else if (ctlr->dma_map_dev)
++ rx_dev = ctlr->dma_map_dev;
+ else
+ rx_dev = ctlr->dev.parent;
+
+@@ -2979,12 +3023,6 @@ void spi_unregister_controller(struct spi_controller *ctlr)
+
+ device_del(&ctlr->dev);
+
+- /* Release the last reference on the controller if its driver
+- * has not yet been converted to devm_spi_alloc_master/slave().
+- */
+- if (!ctlr->devm_allocated)
+- put_device(&ctlr->dev);
+-
+ /* free bus id */
+ mutex_lock(&board_lock);
+ if (found == ctlr)
+@@ -2993,6 +3031,12 @@ void spi_unregister_controller(struct spi_controller *ctlr)
+
+ if (IS_ENABLED(CONFIG_SPI_DYNAMIC))
+ mutex_unlock(&ctlr->add_lock);
++
++ /* Release the last reference on the controller if its driver
++ * has not yet been converted to devm_spi_alloc_master/slave().
++ */
++ if (!ctlr->devm_allocated)
++ put_device(&ctlr->dev);
+ }
+ EXPORT_SYMBOL_GPL(spi_unregister_controller);
+
+diff --git a/drivers/staging/fbtft/fb_ssd1351.c b/drivers/staging/fbtft/fb_ssd1351.c
+index cf263a58a1489..6fd549a424d53 100644
+--- a/drivers/staging/fbtft/fb_ssd1351.c
++++ b/drivers/staging/fbtft/fb_ssd1351.c
+@@ -187,7 +187,6 @@ static struct fbtft_display display = {
+ },
+ };
+
+-#ifdef CONFIG_FB_BACKLIGHT
+ static int update_onboard_backlight(struct backlight_device *bd)
+ {
+ struct fbtft_par *par = bl_get_data(bd);
+@@ -231,9 +230,6 @@ static void register_onboard_backlight(struct fbtft_par *par)
+ if (!par->fbtftops.unregister_backlight)
+ par->fbtftops.unregister_backlight = fbtft_unregister_backlight;
+ }
+-#else
+-static void register_onboard_backlight(struct fbtft_par *par) { };
+-#endif
+
+ FBTFT_REGISTER_DRIVER(DRVNAME, "solomon,ssd1351", &display);
+
+diff --git a/drivers/staging/fbtft/fb_st7789v.c b/drivers/staging/fbtft/fb_st7789v.c
+index abe9395a0aefd..861a154144e66 100644
+--- a/drivers/staging/fbtft/fb_st7789v.c
++++ b/drivers/staging/fbtft/fb_st7789v.c
+@@ -144,6 +144,8 @@ static int init_display(struct fbtft_par *par)
+ {
+ int rc;
+
++ par->fbtftops.reset(par);
++
+ rc = init_tearing_effect_line(par);
+ if (rc)
+ return rc;
+diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
+index ed992ca605ebe..1690358b8f018 100644
+--- a/drivers/staging/fbtft/fbtft-core.c
++++ b/drivers/staging/fbtft/fbtft-core.c
+@@ -128,7 +128,6 @@ static int fbtft_request_gpios(struct fbtft_par *par)
+ return 0;
+ }
+
+-#ifdef CONFIG_FB_BACKLIGHT
+ static int fbtft_backlight_update_status(struct backlight_device *bd)
+ {
+ struct fbtft_par *par = bl_get_data(bd);
+@@ -161,6 +160,7 @@ void fbtft_unregister_backlight(struct fbtft_par *par)
+ par->info->bl_dev = NULL;
+ }
+ }
++EXPORT_SYMBOL(fbtft_unregister_backlight);
+
+ static const struct backlight_ops fbtft_bl_ops = {
+ .get_brightness = fbtft_backlight_get_brightness,
+@@ -198,12 +198,7 @@ void fbtft_register_backlight(struct fbtft_par *par)
+ if (!par->fbtftops.unregister_backlight)
+ par->fbtftops.unregister_backlight = fbtft_unregister_backlight;
+ }
+-#else
+-void fbtft_register_backlight(struct fbtft_par *par) { };
+-void fbtft_unregister_backlight(struct fbtft_par *par) { };
+-#endif
+ EXPORT_SYMBOL(fbtft_register_backlight);
+-EXPORT_SYMBOL(fbtft_unregister_backlight);
+
+ static void fbtft_set_addr_win(struct fbtft_par *par, int xs, int ys, int xe,
+ int ye)
+@@ -853,13 +848,11 @@ int fbtft_register_framebuffer(struct fb_info *fb_info)
+ fb_info->fix.smem_len >> 10, text1,
+ HZ / fb_info->fbdefio->delay, text2);
+
+-#ifdef CONFIG_FB_BACKLIGHT
+ /* Turn on backlight if available */
+ if (fb_info->bl_dev) {
+ fb_info->bl_dev->props.power = FB_BLANK_UNBLANK;
+ fb_info->bl_dev->ops->update_status(fb_info->bl_dev);
+ }
+-#endif
+
+ return 0;
+
+diff --git a/drivers/staging/fbtft/fbtft.h b/drivers/staging/fbtft/fbtft.h
+index 76f8c090a8370..06afaa9d505ba 100644
+--- a/drivers/staging/fbtft/fbtft.h
++++ b/drivers/staging/fbtft/fbtft.h
+@@ -332,7 +332,10 @@ static int __init fbtft_driver_module_init(void) \
+ ret = spi_register_driver(&fbtft_driver_spi_driver); \
+ if (ret < 0) \
+ return ret; \
+- return platform_driver_register(&fbtft_driver_platform_driver); \
++ ret = platform_driver_register(&fbtft_driver_platform_driver); \
++ if (ret < 0) \
++ spi_unregister_driver(&fbtft_driver_spi_driver); \
++ return ret; \
+ } \
+ \
+ static void __exit fbtft_driver_module_exit(void) \
+diff --git a/drivers/staging/gdm724x/gdm_lte.c b/drivers/staging/gdm724x/gdm_lte.c
+index e390c924ec1c8..3c680ed4429c1 100644
+--- a/drivers/staging/gdm724x/gdm_lte.c
++++ b/drivers/staging/gdm724x/gdm_lte.c
+@@ -76,14 +76,15 @@ static void tx_complete(void *arg)
+
+ static int gdm_lte_rx(struct sk_buff *skb, struct nic *nic, int nic_type)
+ {
+- int ret;
++ int ret, len;
+
++ len = skb->len + ETH_HLEN;
+ ret = netif_rx_ni(skb);
+ if (ret == NET_RX_DROP) {
+ nic->stats.rx_dropped++;
+ } else {
+ nic->stats.rx_packets++;
+- nic->stats.rx_bytes += skb->len + ETH_HLEN;
++ nic->stats.rx_bytes += len;
+ }
+
+ return 0;
+diff --git a/drivers/staging/greybus/audio_helper.c b/drivers/staging/greybus/audio_helper.c
+index 1ed4772d27715..843760675876a 100644
+--- a/drivers/staging/greybus/audio_helper.c
++++ b/drivers/staging/greybus/audio_helper.c
+@@ -192,7 +192,11 @@ int gbaudio_remove_component_controls(struct snd_soc_component *component,
+ unsigned int num_controls)
+ {
+ struct snd_card *card = component->card->snd_card;
++ int err;
+
+- return gbaudio_remove_controls(card, component->dev, controls,
+- num_controls, component->name_prefix);
++ down_write(&card->controls_rwsem);
++ err = gbaudio_remove_controls(card, component->dev, controls,
++ num_controls, component->name_prefix);
++ up_write(&card->controls_rwsem);
++ return err;
+ }
+diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c
+index 1e613d42d8237..62d7674852bec 100644
+--- a/drivers/staging/greybus/audio_topology.c
++++ b/drivers/staging/greybus/audio_topology.c
+@@ -147,6 +147,9 @@ static const char **gb_generate_enum_strings(struct gbaudio_module_info *gb,
+
+ items = le32_to_cpu(gbenum->items);
+ strings = devm_kcalloc(gb->dev, items, sizeof(char *), GFP_KERNEL);
++ if (!strings)
++ return NULL;
++
+ data = gbenum->names;
+
+ for (i = 0; i < items; i++) {
+@@ -655,6 +658,8 @@ static int gbaudio_tplg_create_enum_kctl(struct gbaudio_module_info *gb,
+ /* since count=1, and reg is dummy */
+ gbe->items = le32_to_cpu(gb_enum->items);
+ gbe->texts = gb_generate_enum_strings(gb, gb_enum);
++ if (!gbe->texts)
++ return -ENOMEM;
+
+ /* debug enum info */
+ dev_dbg(gb->dev, "Max:%d, name_length:%d\n", gbe->items,
+@@ -862,6 +867,8 @@ static int gbaudio_tplg_create_enum_ctl(struct gbaudio_module_info *gb,
+ /* since count=1, and reg is dummy */
+ gbe->items = le32_to_cpu(gb_enum->items);
+ gbe->texts = gb_generate_enum_strings(gb, gb_enum);
++ if (!gbe->texts)
++ return -ENOMEM;
+
+ /* debug enum info */
+ dev_dbg(gb->dev, "Max:%d, name_length:%d\n", gbe->items,
+@@ -974,6 +981,44 @@ static int gbaudio_widget_event(struct snd_soc_dapm_widget *w,
+ return ret;
+ }
+
++static const struct snd_soc_dapm_widget gbaudio_widgets[] = {
++ [snd_soc_dapm_spk] = SND_SOC_DAPM_SPK(NULL, gbcodec_event_spk),
++ [snd_soc_dapm_hp] = SND_SOC_DAPM_HP(NULL, gbcodec_event_hp),
++ [snd_soc_dapm_mic] = SND_SOC_DAPM_MIC(NULL, gbcodec_event_int_mic),
++ [snd_soc_dapm_output] = SND_SOC_DAPM_OUTPUT(NULL),
++ [snd_soc_dapm_input] = SND_SOC_DAPM_INPUT(NULL),
++ [snd_soc_dapm_switch] = SND_SOC_DAPM_SWITCH_E(NULL, SND_SOC_NOPM,
++ 0, 0, NULL,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++ [snd_soc_dapm_pga] = SND_SOC_DAPM_PGA_E(NULL, SND_SOC_NOPM,
++ 0, 0, NULL, 0,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++ [snd_soc_dapm_mixer] = SND_SOC_DAPM_MIXER_E(NULL, SND_SOC_NOPM,
++ 0, 0, NULL, 0,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++ [snd_soc_dapm_mux] = SND_SOC_DAPM_MUX_E(NULL, SND_SOC_NOPM,
++ 0, 0, NULL,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++ [snd_soc_dapm_aif_in] = SND_SOC_DAPM_AIF_IN_E(NULL, NULL, 0,
++ SND_SOC_NOPM, 0, 0,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++ [snd_soc_dapm_aif_out] = SND_SOC_DAPM_AIF_OUT_E(NULL, NULL, 0,
++ SND_SOC_NOPM, 0, 0,
++ gbaudio_widget_event,
++ SND_SOC_DAPM_PRE_PMU |
++ SND_SOC_DAPM_POST_PMD),
++};
++
+ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
+ struct snd_soc_dapm_widget *dw,
+ struct gb_audio_widget *w, int *w_size)
+@@ -1034,6 +1079,10 @@ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
+ csize += le16_to_cpu(gbenum->names_length);
+ control->texts = (const char * const *)
+ gb_generate_enum_strings(module, gbenum);
++ if (!control->texts) {
++ ret = -ENOMEM;
++ goto error;
++ }
+ control->items = le32_to_cpu(gbenum->items);
+ } else {
+ csize = sizeof(struct gb_audio_control);
+@@ -1052,77 +1101,37 @@ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
+
+ switch (w->type) {
+ case snd_soc_dapm_spk:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_SPK(w->name, gbcodec_event_spk);
++ *dw = gbaudio_widgets[w->type];
+ module->op_devices |= GBAUDIO_DEVICE_OUT_SPEAKER;
+ break;
+ case snd_soc_dapm_hp:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_HP(w->name, gbcodec_event_hp);
++ *dw = gbaudio_widgets[w->type];
+ module->op_devices |= (GBAUDIO_DEVICE_OUT_WIRED_HEADSET
+ | GBAUDIO_DEVICE_OUT_WIRED_HEADPHONE);
+ module->ip_devices |= GBAUDIO_DEVICE_IN_WIRED_HEADSET;
+ break;
+ case snd_soc_dapm_mic:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_MIC(w->name, gbcodec_event_int_mic);
++ *dw = gbaudio_widgets[w->type];
+ module->ip_devices |= GBAUDIO_DEVICE_IN_BUILTIN_MIC;
+ break;
+ case snd_soc_dapm_output:
+- *dw = (struct snd_soc_dapm_widget)SND_SOC_DAPM_OUTPUT(w->name);
+- break;
+ case snd_soc_dapm_input:
+- *dw = (struct snd_soc_dapm_widget)SND_SOC_DAPM_INPUT(w->name);
+- break;
+ case snd_soc_dapm_switch:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_SWITCH_E(w->name, SND_SOC_NOPM, 0, 0,
+- widget_kctls,
+- gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
+- break;
+ case snd_soc_dapm_pga:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_PGA_E(w->name, SND_SOC_NOPM, 0, 0, NULL, 0,
+- gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
+- break;
+ case snd_soc_dapm_mixer:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_MIXER_E(w->name, SND_SOC_NOPM, 0, 0, NULL,
+- 0, gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
+- break;
+ case snd_soc_dapm_mux:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_MUX_E(w->name, SND_SOC_NOPM, 0, 0,
+- widget_kctls, gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
++ *dw = gbaudio_widgets[w->type];
+ break;
+ case snd_soc_dapm_aif_in:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_AIF_IN_E(w->name, w->sname, 0,
+- SND_SOC_NOPM,
+- 0, 0, gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
+- break;
+ case snd_soc_dapm_aif_out:
+- *dw = (struct snd_soc_dapm_widget)
+- SND_SOC_DAPM_AIF_OUT_E(w->name, w->sname, 0,
+- SND_SOC_NOPM,
+- 0, 0, gbaudio_widget_event,
+- SND_SOC_DAPM_PRE_PMU |
+- SND_SOC_DAPM_POST_PMD);
++ *dw = gbaudio_widgets[w->type];
++ dw->sname = w->sname;
+ break;
+ default:
+ ret = -EINVAL;
+ goto error;
+ }
++ dw->name = w->name;
+
+ dev_dbg(module->dev, "%s: widget of type %d created\n", dw->name,
+ dw->id);
+@@ -1183,6 +1192,10 @@ static int gbaudio_tplg_process_kcontrols(struct gbaudio_module_info *module,
+ csize += le16_to_cpu(gbenum->names_length);
+ control->texts = (const char * const *)
+ gb_generate_enum_strings(module, gbenum);
++ if (!control->texts) {
++ ret = -ENOMEM;
++ goto error;
++ }
+ control->items = le32_to_cpu(gbenum->items);
+ } else {
+ csize = sizeof(struct gb_audio_control);
+diff --git a/drivers/staging/iio/adc/ad7280a.c b/drivers/staging/iio/adc/ad7280a.c
+index fef0055b89909..20183b2ea1279 100644
+--- a/drivers/staging/iio/adc/ad7280a.c
++++ b/drivers/staging/iio/adc/ad7280a.c
+@@ -107,9 +107,9 @@
+ static unsigned int ad7280a_devaddr(unsigned int addr)
+ {
+ return ((addr & 0x1) << 4) |
+- ((addr & 0x2) << 3) |
++ ((addr & 0x2) << 2) |
+ (addr & 0x4) |
+- ((addr & 0x8) >> 3) |
++ ((addr & 0x8) >> 2) |
+ ((addr & 0x10) >> 4);
+ }
+
+diff --git a/drivers/staging/ks7010/Kconfig b/drivers/staging/ks7010/Kconfig
+index 0987fdc2f70db..8ea6c09286798 100644
+--- a/drivers/staging/ks7010/Kconfig
++++ b/drivers/staging/ks7010/Kconfig
+@@ -5,6 +5,9 @@ config KS7010
+ select WIRELESS_EXT
+ select WEXT_PRIV
+ select FW_LOADER
++ select CRYPTO
++ select CRYPTO_HASH
++ select CRYPTO_MICHAEL_MIC
+ help
+ This is a driver for KeyStream KS7010 based SDIO WIFI cards. It is
+ found on at least later Spectec SDW-821 (FCC-ID "S2Y-WLAN-11G-K" only,
+diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+index 362ed44b4effa..e046489cd253b 100644
+--- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
++++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+@@ -835,7 +835,6 @@ static int lm3554_probe(struct i2c_client *client)
+ int err = 0;
+ struct lm3554 *flash;
+ unsigned int i;
+- int ret;
+
+ flash = kzalloc(sizeof(*flash), GFP_KERNEL);
+ if (!flash)
+@@ -844,7 +843,7 @@ static int lm3554_probe(struct i2c_client *client)
+ flash->pdata = lm3554_platform_data_func(client);
+ if (IS_ERR(flash->pdata)) {
+ err = PTR_ERR(flash->pdata);
+- goto fail1;
++ goto free_flash;
+ }
+
+ v4l2_i2c_subdev_init(&flash->sd, client, &lm3554_ops);
+@@ -852,12 +851,12 @@ static int lm3554_probe(struct i2c_client *client)
+ flash->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+ flash->mode = ATOMISP_FLASH_MODE_OFF;
+ flash->timeout = LM3554_MAX_TIMEOUT / LM3554_TIMEOUT_STEPSIZE - 1;
+- ret =
++ err =
+ v4l2_ctrl_handler_init(&flash->ctrl_handler,
+ ARRAY_SIZE(lm3554_controls));
+- if (ret) {
++ if (err) {
+ dev_err(&client->dev, "error initialize a ctrl_handler.\n");
+- goto fail3;
++ goto unregister_subdev;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(lm3554_controls); i++)
+@@ -866,14 +865,15 @@ static int lm3554_probe(struct i2c_client *client)
+
+ if (flash->ctrl_handler.error) {
+ dev_err(&client->dev, "ctrl_handler error.\n");
+- goto fail3;
++ err = flash->ctrl_handler.error;
++ goto free_handler;
+ }
+
+ flash->sd.ctrl_handler = &flash->ctrl_handler;
+ err = media_entity_pads_init(&flash->sd.entity, 0, NULL);
+ if (err) {
+ dev_err(&client->dev, "error initialize a media entity.\n");
+- goto fail2;
++ goto free_handler;
+ }
+
+ flash->sd.entity.function = MEDIA_ENT_F_FLASH;
+@@ -884,16 +884,27 @@ static int lm3554_probe(struct i2c_client *client)
+
+ err = lm3554_gpio_init(client);
+ if (err) {
+- dev_err(&client->dev, "gpio request/direction_output fail");
+- goto fail3;
++ dev_err(&client->dev, "gpio request/direction_output fail.\n");
++ goto cleanup_media;
++ }
++
++ err = atomisp_register_i2c_module(&flash->sd, NULL, LED_FLASH);
++ if (err) {
++ dev_err(&client->dev, "fail to register atomisp i2c module.\n");
++ goto uninit_gpio;
+ }
+- return atomisp_register_i2c_module(&flash->sd, NULL, LED_FLASH);
+-fail3:
++
++ return 0;
++
++uninit_gpio:
++ lm3554_gpio_uninit(client);
++cleanup_media:
+ media_entity_cleanup(&flash->sd.entity);
++free_handler:
+ v4l2_ctrl_handler_free(&flash->ctrl_handler);
+-fail2:
++unregister_subdev:
+ v4l2_device_unregister_subdev(&flash->sd);
+-fail1:
++free_flash:
+ kfree(flash);
+
+ return err;
+diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h
+index 874115f35fcad..798b28e134b64 100644
+--- a/drivers/staging/media/atomisp/i2c/ov2680.h
++++ b/drivers/staging/media/atomisp/i2c/ov2680.h
+@@ -289,8 +289,6 @@ static struct ov2680_reg const ov2680_global_setting[] = {
+ */
+ static struct ov2680_reg const ov2680_QCIF_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x24},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -334,8 +332,6 @@ static struct ov2680_reg const ov2680_QCIF_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_CIF_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x24},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -377,8 +373,6 @@ static struct ov2680_reg const ov2680_CIF_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_QVGA_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x24},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -420,8 +414,6 @@ static struct ov2680_reg const ov2680_QVGA_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_656x496_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x24},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -463,8 +455,6 @@ static struct ov2680_reg const ov2680_656x496_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_720x592_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x26},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0x00}, // X_ADDR_START;
+ {0x3802, 0x00},
+@@ -508,8 +498,6 @@ static struct ov2680_reg const ov2680_720x592_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_800x600_30fps[] = {
+ {0x3086, 0x01},
+- {0x3501, 0x26},
+- {0x3502, 0x40},
+ {0x370a, 0x23},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+@@ -551,8 +539,6 @@ static struct ov2680_reg const ov2680_800x600_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_720p_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -594,8 +580,6 @@ static struct ov2680_reg const ov2680_720p_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_1296x976_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0xa0},
+ {0x3802, 0x00},
+@@ -637,8 +621,6 @@ static struct ov2680_reg const ov2680_1296x976_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_1456x1096_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0x90},
+ {0x3802, 0x00},
+@@ -682,8 +664,6 @@ static struct ov2680_reg const ov2680_1456x1096_30fps[] = {
+
+ static struct ov2680_reg const ov2680_1616x916_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+@@ -726,8 +706,6 @@ static struct ov2680_reg const ov2680_1616x916_30fps[] = {
+ #if 0
+ static struct ov2680_reg const ov2680_1616x1082_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+@@ -769,8 +747,6 @@ static struct ov2680_reg const ov2680_1616x1082_30fps[] = {
+ */
+ static struct ov2680_reg const ov2680_1616x1216_30fps[] = {
+ {0x3086, 0x00},
+- {0x3501, 0x48},
+- {0x3502, 0xe0},
+ {0x370a, 0x21},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.c b/drivers/staging/media/atomisp/pci/atomisp_acc.c
+index 9a1751895ab03..28cb271663c47 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_acc.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_acc.c
+@@ -439,6 +439,18 @@ int atomisp_acc_s_mapped_arg(struct atomisp_sub_device *asd,
+ return 0;
+ }
+
++static void atomisp_acc_unload_some_extensions(struct atomisp_sub_device *asd,
++ int i,
++ struct atomisp_acc_fw *acc_fw)
++{
++ while (--i >= 0) {
++ if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
++ atomisp_css_unload_acc_extension(asd, acc_fw->fw,
++ acc_flag_to_pipe[i].pipe_id);
++ }
++ }
++}
++
+ /*
+ * Appends the loaded acceleration binary extensions to the
+ * current ISP mode. Must be called just before sh_css_start().
+@@ -479,16 +491,20 @@ int atomisp_acc_load_extensions(struct atomisp_sub_device *asd)
+ acc_fw->fw,
+ acc_flag_to_pipe[i].pipe_id,
+ acc_fw->type);
+- if (ret)
++ if (ret) {
++ atomisp_acc_unload_some_extensions(asd, i, acc_fw);
+ goto error;
++ }
+
+ ext_loaded = true;
+ }
+ }
+
+ ret = atomisp_css_set_acc_parameters(acc_fw);
+- if (ret < 0)
++ if (ret < 0) {
++ atomisp_acc_unload_some_extensions(asd, i, acc_fw);
+ goto error;
++ }
+ }
+
+ if (!ext_loaded)
+@@ -497,6 +513,7 @@ int atomisp_acc_load_extensions(struct atomisp_sub_device *asd)
+ ret = atomisp_css_update_stream(asd);
+ if (ret) {
+ dev_err(isp->dev, "%s: update stream failed.\n", __func__);
++ atomisp_acc_unload_extensions(asd);
+ goto error;
+ }
+
+@@ -504,13 +521,6 @@ int atomisp_acc_load_extensions(struct atomisp_sub_device *asd)
+ return 0;
+
+ error:
+- while (--i >= 0) {
+- if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
+- atomisp_css_unload_acc_extension(asd, acc_fw->fw,
+- acc_flag_to_pipe[i].pipe_id);
+- }
+- }
+-
+ list_for_each_entry_continue_reverse(acc_fw, &asd->acc.fw, list) {
+ if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT &&
+ acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER)
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+index 366161cff5602..ef0b0963cf930 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+@@ -1715,6 +1715,12 @@ void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe,
+ {
+ unsigned long next;
+
++ if (!pipe->asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, pipe->vdev.name);
++ return;
++ }
++
+ if (delay != ATOMISP_WDT_KEEP_CURRENT_DELAY)
+ pipe->wdt_duration = delay;
+
+@@ -1777,6 +1783,12 @@ void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay)
+ /* ISP2401 */
+ void atomisp_wdt_stop_pipe(struct atomisp_video_pipe *pipe, bool sync)
+ {
++ if (!pipe->asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, pipe->vdev.name);
++ return;
++ }
++
+ if (!atomisp_is_wdt_running(pipe))
+ return;
+
+@@ -4109,6 +4121,12 @@ void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe)
+ unsigned long irqflags;
+ bool need_to_enqueue_buffer = false;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, pipe->vdev.name);
++ return;
++ }
++
+ if (atomisp_is_vf_pipe(pipe))
+ return;
+
+@@ -4196,6 +4214,12 @@ int atomisp_set_parameters(struct video_device *vdev,
+ struct atomisp_css_params *css_param = &asd->params.css_param;
+ int ret;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
+ dev_err(asd->isp->dev, "%s: internal error!\n", __func__);
+ return -EINVAL;
+@@ -4857,6 +4881,12 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f,
+ int source_pad = atomisp_subdev_source_pad(vdev);
+ int ret;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (!isp->inputs[asd->input_curr].camera)
+ return -EINVAL;
+
+@@ -5194,10 +5224,17 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
+ int (*configure_pp_input)(struct atomisp_sub_device *asd,
+ unsigned int width, unsigned int height) =
+ configure_pp_input_nop;
+- u16 stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
++ u16 stream_index;
+ const struct atomisp_in_fmt_conv *fc;
+ int ret, i;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++ stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
++
+ v4l2_fh_init(&fh.vfh, vdev);
+
+ isp_sink_crop = atomisp_subdev_get_rect(
+@@ -5493,7 +5530,8 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev,
+ unsigned int padding_w, unsigned int padding_h,
+ unsigned int dvs_env_w, unsigned int dvs_env_h)
+ {
+- struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
++ struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
++ struct atomisp_sub_device *asd = pipe->asd;
+ const struct atomisp_format_bridge *format;
+ struct v4l2_subdev_pad_config pad_cfg;
+ struct v4l2_subdev_state pad_state = {
+@@ -5504,7 +5542,7 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev,
+ };
+ struct v4l2_mbus_framefmt *ffmt = &vformat.format;
+ struct v4l2_mbus_framefmt *req_ffmt;
+- struct atomisp_device *isp = asd->isp;
++ struct atomisp_device *isp;
+ struct atomisp_input_stream_info *stream_info =
+ (struct atomisp_input_stream_info *)ffmt->reserved;
+ u16 stream_index = ATOMISP_INPUT_STREAM_GENERAL;
+@@ -5512,6 +5550,14 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev,
+ struct v4l2_subdev_fh fh;
+ int ret;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
++ isp = asd->isp;
++
+ v4l2_fh_init(&fh.vfh, vdev);
+
+ stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
+@@ -5602,6 +5648,12 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
+ struct v4l2_subdev_fh fh;
+ int ret;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (source_pad >= ATOMISP_SUBDEV_PADS_NUM)
+ return -EINVAL;
+
+@@ -6034,6 +6086,12 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f)
+ struct v4l2_subdev_fh fh;
+ int ret;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ v4l2_fh_init(&fh.vfh, vdev);
+
+ dev_dbg(isp->dev, "setting fmt %ux%u 0x%x for file inject\n",
+@@ -6359,6 +6417,12 @@ bool atomisp_is_vf_pipe(struct atomisp_video_pipe *pipe)
+ {
+ struct atomisp_sub_device *asd = pipe->asd;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, pipe->vdev.name);
++ return false;
++ }
++
+ if (pipe == &asd->video_out_vf)
+ return true;
+
+@@ -6572,6 +6636,12 @@ static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe)
+ {
+ struct atomisp_sub_device *asd = pipe->asd;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, pipe->vdev.name);
++ return -EINVAL;
++ }
++
+ if (ATOMISP_USE_YUVPP(asd)) {
+ return IA_CSS_PIPE_ID_YUVPP;
+ } else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
+@@ -6609,6 +6679,12 @@ int atomisp_get_invalid_frame_num(struct video_device *vdev,
+ struct ia_css_pipe_info p_info;
+ int ret;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (asd->isp->inputs[asd->input_curr].camera_caps->
+ sensor[asd->sensor_curr].stream_num > 1) {
+ /* External ISP */
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c
+index f82bf082aa796..18fff47bd25d2 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_fops.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c
+@@ -877,6 +877,11 @@ done:
+ else
+ pipe->users++;
+ rt_mutex_unlock(&isp->mutex);
++
++ /* Ensure that a mode is set */
++ if (asd)
++ v4l2_ctrl_s_ctrl(asd->run_mode, pipe->default_run_mode);
++
+ return 0;
+
+ css_error:
+@@ -1171,6 +1176,12 @@ static int atomisp_mmap(struct file *file, struct vm_area_struct *vma)
+ u32 origin_size, new_size;
+ int ret;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (!(vma->vm_flags & (VM_WRITE | VM_READ)))
+ return -EACCES;
+
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
+index d8c9e31314b2e..cd0a771454da4 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
+@@ -481,7 +481,7 @@ fail:
+
+ static u8 gmin_get_pmic_id_and_addr(struct device *dev)
+ {
+- struct i2c_client *power;
++ struct i2c_client *power = NULL;
+ static u8 pmic_i2c_addr;
+
+ if (pmic_id)
+@@ -729,6 +729,21 @@ static int axp_regulator_set(struct device *dev, struct gmin_subdev *gs,
+ return 0;
+ }
+
++/*
++ * Some boards contain a hw-bug where turning eldo2 back on after having turned
++ * it off causes the CPLM3218 ambient-light-sensor on the image-sensor's I2C bus
++ * to crash, hanging the bus. Do not turn eldo2 off on these systems.
++ */
++static const struct dmi_system_id axp_leave_eldo2_on_ids[] = {
++ {
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "TrekStor"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "SurfTab duo W1 10.1 (VT4)"),
++ },
++ },
++ { }
++};
++
+ static int axp_v1p8_on(struct device *dev, struct gmin_subdev *gs)
+ {
+ int ret;
+@@ -763,6 +778,9 @@ static int axp_v1p8_off(struct device *dev, struct gmin_subdev *gs)
+ if (ret)
+ return ret;
+
++ if (dmi_check_system(axp_leave_eldo2_on_ids))
++ return 0;
++
+ ret = axp_regulator_set(dev, gs, gs->eldo2_sel_reg, gs->eldo2_1p8v,
+ ELDO_CTRL_REG, gs->eldo2_ctrl_shift, false);
+ return ret;
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
+index c8a625667e81e..b7dda4b96d49c 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
+@@ -646,6 +646,12 @@ static int atomisp_g_input(struct file *file, void *fh, unsigned int *input)
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+ struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ rt_mutex_lock(&isp->mutex);
+ *input = asd->input_curr;
+ rt_mutex_unlock(&isp->mutex);
+@@ -665,6 +671,12 @@ static int atomisp_s_input(struct file *file, void *fh, unsigned int input)
+ struct v4l2_subdev *motor;
+ int ret;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ rt_mutex_lock(&isp->mutex);
+ if (input >= ATOM_ISP_MAX_INPUTS || input >= isp->input_cnt) {
+ dev_dbg(isp->dev, "input_cnt: %d\n", isp->input_cnt);
+@@ -761,18 +773,33 @@ static int atomisp_enum_fmt_cap(struct file *file, void *fh,
+ struct video_device *vdev = video_devdata(file);
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+ struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
+- struct v4l2_subdev_mbus_code_enum code = { 0 };
++ struct v4l2_subdev_mbus_code_enum code = {
++ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
++ };
++ struct v4l2_subdev *camera;
+ unsigned int i, fi = 0;
+ int rval;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
++ camera = isp->inputs[asd->input_curr].camera;
++ if(!camera) {
++ dev_err(isp->dev, "%s(): camera is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ rt_mutex_lock(&isp->mutex);
+- rval = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, pad,
+- enum_mbus_code, NULL, &code);
++
++ rval = v4l2_subdev_call(camera, pad, enum_mbus_code, NULL, &code);
+ if (rval == -ENOIOCTLCMD) {
+ dev_warn(isp->dev,
+- "enum_mbus_code pad op not supported. Please fix your sensor driver!\n");
+- // rval = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
+- // video, enum_mbus_fmt, 0, &code.code);
++ "enum_mbus_code pad op not supported by %s. Please fix your sensor driver!\n",
++ camera->name);
+ }
+ rt_mutex_unlock(&isp->mutex);
+
+@@ -802,6 +829,8 @@ static int atomisp_enum_fmt_cap(struct file *file, void *fh,
+ f->pixelformat = format->pixelformat;
+ return 0;
+ }
++ dev_err(isp->dev, "%s(): format for code %x not found.\n",
++ __func__, code.code);
+
+ return -EINVAL;
+ }
+@@ -834,6 +863,72 @@ static int atomisp_g_fmt_file(struct file *file, void *fh,
+ return 0;
+ }
+
++static int atomisp_adjust_fmt(struct v4l2_format *f)
++{
++ const struct atomisp_format_bridge *format_bridge;
++ u32 padded_width;
++
++ format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
++
++ padded_width = f->fmt.pix.width + pad_w;
++
++ if (format_bridge->planar) {
++ f->fmt.pix.bytesperline = padded_width;
++ f->fmt.pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height *
++ DIV_ROUND_UP(format_bridge->depth *
++ padded_width, 8));
++ } else {
++ f->fmt.pix.bytesperline = DIV_ROUND_UP(format_bridge->depth *
++ padded_width, 8);
++ f->fmt.pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height * f->fmt.pix.bytesperline);
++ }
++
++ if (f->fmt.pix.field == V4L2_FIELD_ANY)
++ f->fmt.pix.field = V4L2_FIELD_NONE;
++
++ format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
++ if (!format_bridge)
++ return -EINVAL;
++
++ /* Currently, raw formats are broken!!! */
++ if (format_bridge->sh_fmt == IA_CSS_FRAME_FORMAT_RAW) {
++ f->fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420;
++
++ format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
++ if (!format_bridge)
++ return -EINVAL;
++ }
++
++ padded_width = f->fmt.pix.width + pad_w;
++
++ if (format_bridge->planar) {
++ f->fmt.pix.bytesperline = padded_width;
++ f->fmt.pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height *
++ DIV_ROUND_UP(format_bridge->depth *
++ padded_width, 8));
++ } else {
++ f->fmt.pix.bytesperline = DIV_ROUND_UP(format_bridge->depth *
++ padded_width, 8);
++ f->fmt.pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height * f->fmt.pix.bytesperline);
++ }
++
++ if (f->fmt.pix.field == V4L2_FIELD_ANY)
++ f->fmt.pix.field = V4L2_FIELD_NONE;
++
++ /*
++ * FIXME: do we need to setup this differently, depending on the
++ * sensor or the pipeline?
++ */
++ f->fmt.pix.colorspace = V4L2_COLORSPACE_REC709;
++ f->fmt.pix.ycbcr_enc = V4L2_YCBCR_ENC_709;
++ f->fmt.pix.xfer_func = V4L2_XFER_FUNC_709;
++
++ f->fmt.pix.width -= pad_w;
++ f->fmt.pix.height -= pad_h;
++
++ return 0;
++}
++
+ /* This function looks up the closest available resolution. */
+ static int atomisp_try_fmt_cap(struct file *file, void *fh,
+ struct v4l2_format *f)
+@@ -845,7 +940,11 @@ static int atomisp_try_fmt_cap(struct file *file, void *fh,
+ rt_mutex_lock(&isp->mutex);
+ ret = atomisp_try_fmt(vdev, &f->fmt.pix, NULL);
+ rt_mutex_unlock(&isp->mutex);
+- return ret;
++
++ if (ret)
++ return ret;
++
++ return atomisp_adjust_fmt(f);
+ }
+
+ static int atomisp_s_fmt_cap(struct file *file, void *fh,
+@@ -1024,9 +1123,16 @@ int __atomisp_reqbufs(struct file *file, void *fh,
+ struct ia_css_frame *frame;
+ struct videobuf_vmalloc_memory *vm_mem;
+ u16 source_pad = atomisp_subdev_source_pad(vdev);
+- u16 stream_id = atomisp_source_pad_to_stream_id(asd, source_pad);
++ u16 stream_id;
+ int ret = 0, i = 0;
+
++ if (!asd) {
++ dev_err(pipe->isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++ stream_id = atomisp_source_pad_to_stream_id(asd, source_pad);
++
+ if (req->count == 0) {
+ mutex_lock(&pipe->capq.vb_lock);
+ if (!list_empty(&pipe->capq.stream))
+@@ -1154,6 +1260,12 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
+ u32 pgnr;
+ int ret = 0;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ rt_mutex_lock(&isp->mutex);
+ if (isp->isp_fatal_error) {
+ ret = -EIO;
+@@ -1389,6 +1501,12 @@ static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+ int ret = 0;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ rt_mutex_lock(&isp->mutex);
+
+ if (isp->isp_fatal_error) {
+@@ -1640,6 +1758,12 @@ static int atomisp_streamon(struct file *file, void *fh,
+ int ret = 0;
+ unsigned long irqflags;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ dev_dbg(isp->dev, "Start stream on pad %d for asd%d\n",
+ atomisp_subdev_source_pad(vdev), asd->index);
+
+@@ -1901,6 +2025,12 @@ int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type)
+ unsigned long flags;
+ bool first_streamoff = false;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ dev_dbg(isp->dev, "Stop stream on pad %d for asd%d\n",
+ atomisp_subdev_source_pad(vdev), asd->index);
+
+@@ -2150,6 +2280,12 @@ static int atomisp_g_ctrl(struct file *file, void *fh,
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+ int i, ret = -EINVAL;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ for (i = 0; i < ctrls_num; i++) {
+ if (ci_v4l2_controls[i].id == control->id) {
+ ret = 0;
+@@ -2229,6 +2365,12 @@ static int atomisp_s_ctrl(struct file *file, void *fh,
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+ int i, ret = -EINVAL;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ for (i = 0; i < ctrls_num; i++) {
+ if (ci_v4l2_controls[i].id == control->id) {
+ ret = 0;
+@@ -2310,6 +2452,12 @@ static int atomisp_queryctl(struct file *file, void *fh,
+ struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ switch (qc->id) {
+ case V4L2_CID_FOCUS_ABSOLUTE:
+ case V4L2_CID_FOCUS_RELATIVE:
+@@ -2355,6 +2503,12 @@ static int atomisp_camera_g_ext_ctrls(struct file *file, void *fh,
+ int i;
+ int ret = 0;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (!IS_ISP2401)
+ motor = isp->inputs[asd->input_curr].motor;
+ else
+@@ -2466,6 +2620,12 @@ static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh,
+ int i;
+ int ret = 0;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (!IS_ISP2401)
+ motor = isp->inputs[asd->input_curr].motor;
+ else
+@@ -2591,6 +2751,12 @@ static int atomisp_g_parm(struct file *file, void *fh,
+ struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
+ struct atomisp_device *isp = video_get_drvdata(vdev);
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+ dev_err(isp->dev, "unsupported v4l2 buf type\n");
+ return -EINVAL;
+@@ -2613,6 +2779,12 @@ static int atomisp_s_parm(struct file *file, void *fh,
+ int rval;
+ int fps;
+
++ if (!asd) {
++ dev_err(isp->dev, "%s(): asd is NULL, device is %s\n",
++ __func__, vdev->name);
++ return -EINVAL;
++ }
++
+ if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+ dev_err(isp->dev, "unsupported v4l2 buf type\n");
+ return -EINVAL;
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+index 12f22ad007c73..ffaf11e0b0ad8 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+@@ -1164,23 +1164,28 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
+
+ atomisp_init_acc_pipe(asd, &asd->video_acc);
+
+- ret = atomisp_video_init(&asd->video_in, "MEMORY");
++ ret = atomisp_video_init(&asd->video_in, "MEMORY",
++ ATOMISP_RUN_MODE_SDV);
+ if (ret < 0)
+ return ret;
+
+- ret = atomisp_video_init(&asd->video_out_capture, "CAPTURE");
++ ret = atomisp_video_init(&asd->video_out_capture, "CAPTURE",
++ ATOMISP_RUN_MODE_STILL_CAPTURE);
+ if (ret < 0)
+ return ret;
+
+- ret = atomisp_video_init(&asd->video_out_vf, "VIEWFINDER");
++ ret = atomisp_video_init(&asd->video_out_vf, "VIEWFINDER",
++ ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE);
+ if (ret < 0)
+ return ret;
+
+- ret = atomisp_video_init(&asd->video_out_preview, "PREVIEW");
++ ret = atomisp_video_init(&asd->video_out_preview, "PREVIEW",
++ ATOMISP_RUN_MODE_PREVIEW);
+ if (ret < 0)
+ return ret;
+
+- ret = atomisp_video_init(&asd->video_out_video_capture, "VIDEO");
++ ret = atomisp_video_init(&asd->video_out_video_capture, "VIDEO",
++ ATOMISP_RUN_MODE_VIDEO);
+ if (ret < 0)
+ return ret;
+
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
+index d6fcfab6352d7..a8d210ea5f8be 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h
++++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
+@@ -81,6 +81,9 @@ struct atomisp_video_pipe {
+ /* the link list to store per_frame parameters */
+ struct list_head per_frame_params;
+
++ /* Store here the initial run mode */
++ unsigned int default_run_mode;
++
+ unsigned int buffers_in_css;
+
+ /* irq_lock is used to protect video buffer state change operations and
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+index 1e324f1f656e5..14c39b8987c95 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
++++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+@@ -447,7 +447,8 @@ const struct atomisp_dfs_config dfs_config_cht_soc = {
+ .dfs_table_size = ARRAY_SIZE(dfs_rules_cht_soc),
+ };
+
+-int atomisp_video_init(struct atomisp_video_pipe *video, const char *name)
++int atomisp_video_init(struct atomisp_video_pipe *video, const char *name,
++ unsigned int run_mode)
+ {
+ int ret;
+ const char *direction;
+@@ -478,6 +479,7 @@ int atomisp_video_init(struct atomisp_video_pipe *video, const char *name)
+ "ATOMISP ISP %s %s", name, direction);
+ video->vdev.release = video_device_release_empty;
+ video_set_drvdata(&video->vdev, video->isp);
++ video->default_run_mode = run_mode;
+
+ return 0;
+ }
+@@ -711,15 +713,15 @@ static int atomisp_mrfld_power(struct atomisp_device *isp, bool enable)
+
+ dev_dbg(isp->dev, "IUNIT power-%s.\n", enable ? "on" : "off");
+
+- /*WA:Enable DVFS*/
++ /* WA for P-Unit, if DVFS enabled, ISP timeout observed */
+ if (IS_CHT && enable)
+- punit_ddr_dvfs_enable(true);
++ punit_ddr_dvfs_enable(false);
+
+ /*
+ * FIXME:WA for ECS28A, with this sleep, CTS
+ * android.hardware.camera2.cts.CameraDeviceTest#testCameraDeviceAbort
+ * PASS, no impact on other platforms
+- */
++ */
+ if (IS_BYT && enable)
+ msleep(10);
+
+@@ -727,7 +729,7 @@ static int atomisp_mrfld_power(struct atomisp_device *isp, bool enable)
+ iosf_mbi_modify(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSPM0,
+ val, MRFLD_ISPSSPM0_ISPSSC_MASK);
+
+- /*WA:Enable DVFS*/
++ /* WA:Enable DVFS */
+ if (IS_CHT && !enable)
+ punit_ddr_dvfs_enable(true);
+
+@@ -1182,6 +1184,7 @@ static void atomisp_unregister_entities(struct atomisp_device *isp)
+
+ v4l2_device_unregister(&isp->v4l2_dev);
+ media_device_unregister(&isp->media_dev);
++ media_device_cleanup(&isp->media_dev);
+ }
+
+ static int atomisp_register_entities(struct atomisp_device *isp)
+diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
+index 81bb356b81720..72611b8286a4a 100644
+--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
++++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
+@@ -27,7 +27,8 @@ struct v4l2_device;
+ struct atomisp_device;
+ struct firmware;
+
+-int atomisp_video_init(struct atomisp_video_pipe *video, const char *name);
++int atomisp_video_init(struct atomisp_video_pipe *video, const char *name,
++ unsigned int run_mode);
+ void atomisp_acc_init(struct atomisp_acc_pipe *video, const char *name);
+ void atomisp_video_unregister(struct atomisp_video_pipe *video);
+ void atomisp_acc_unregister(struct atomisp_acc_pipe *video);
+diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm.c b/drivers/staging/media/atomisp/pci/hmm/hmm.c
+index 6a5ee46070898..c1cda16f2dc01 100644
+--- a/drivers/staging/media/atomisp/pci/hmm/hmm.c
++++ b/drivers/staging/media/atomisp/pci/hmm/hmm.c
+@@ -39,7 +39,7 @@
+ struct hmm_bo_device bo_device;
+ struct hmm_pool dynamic_pool;
+ struct hmm_pool reserved_pool;
+-static ia_css_ptr dummy_ptr;
++static ia_css_ptr dummy_ptr = mmgr_EXCEPTION;
+ static bool hmm_initialized;
+ struct _hmm_mem_stat hmm_mem_stat;
+
+@@ -209,7 +209,7 @@ int hmm_init(void)
+
+ void hmm_cleanup(void)
+ {
+- if (!dummy_ptr)
++ if (dummy_ptr == mmgr_EXCEPTION)
+ return;
+ sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
+
+@@ -288,7 +288,8 @@ void hmm_free(ia_css_ptr virt)
+
+ dev_dbg(atomisp_dev, "%s: free 0x%08x\n", __func__, virt);
+
+- WARN_ON(!virt);
++ if (WARN_ON(virt == mmgr_EXCEPTION))
++ return;
+
+ bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt);
+
+diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c
+index c4b35cbab3737..ba25d0da8b811 100644
+--- a/drivers/staging/media/atomisp/pci/sh_css.c
++++ b/drivers/staging/media/atomisp/pci/sh_css.c
+@@ -522,6 +522,7 @@ ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream)
+ return bpp;
+ }
+
++/* TODO: move define to proper file in tools */
+ #define GP_ISEL_TPG_MODE 0x90058
+
+ #if !defined(ISP2401)
+@@ -573,12 +574,8 @@ sh_css_config_input_network(struct ia_css_stream *stream)
+ vblank_cycles = vblank_lines * (width + hblank_cycles);
+ sh_css_sp_configure_sync_gen(width, height, hblank_cycles,
+ vblank_cycles);
+- if (!IS_ISP2401) {
+- if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG) {
+- /* TODO: move define to proper file in tools */
+- ia_css_device_store_uint32(GP_ISEL_TPG_MODE, 0);
+- }
+- }
++ if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG)
++ ia_css_device_store_uint32(GP_ISEL_TPG_MODE, 0);
+ }
+ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
+ "sh_css_config_input_network() leave:\n");
+@@ -1009,16 +1006,14 @@ static bool sh_css_translate_stream_cfg_to_isys_stream_descr(
+ * ia_css_isys_stream_capture_indication() instead of
+ * ia_css_pipeline_sp_wait_for_isys_stream_N() as isp processing of
+ * capture takes longer than getting an ISYS frame
+- *
+- * Only 2401 relevant ??
+ */
+-#if 0 // FIXME: NOT USED on Yocto Aero
+- isys_stream_descr->polling_mode
+- = early_polling ? INPUT_SYSTEM_POLL_ON_CAPTURE_REQUEST
+- : INPUT_SYSTEM_POLL_ON_WAIT_FOR_FRAME;
+- ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
+- "sh_css_translate_stream_cfg_to_isys_stream_descr() leave:\n");
+-#endif
++ if (IS_ISP2401) {
++ isys_stream_descr->polling_mode
++ = early_polling ? INPUT_SYSTEM_POLL_ON_CAPTURE_REQUEST
++ : INPUT_SYSTEM_POLL_ON_WAIT_FOR_FRAME;
++ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
++ "sh_css_translate_stream_cfg_to_isys_stream_descr() leave:\n");
++ }
+
+ return rc;
+ }
+@@ -1433,7 +1428,7 @@ static void start_pipe(
+
+ assert(me); /* all callers are in this file and call with non null argument */
+
+- if (!IS_ISP2401) {
++ if (IS_ISP2401) {
+ coord = &me->config.internal_frame_origin_bqs_on_sctbl;
+ params = me->stream->isp_params_configs;
+ }
+diff --git a/drivers/staging/media/atomisp/pci/sh_css_mipi.c b/drivers/staging/media/atomisp/pci/sh_css_mipi.c
+index 75489f7d75eec..c1f2f6151c5f8 100644
+--- a/drivers/staging/media/atomisp/pci/sh_css_mipi.c
++++ b/drivers/staging/media/atomisp/pci/sh_css_mipi.c
+@@ -374,17 +374,17 @@ static bool buffers_needed(struct ia_css_pipe *pipe)
+ {
+ if (!IS_ISP2401) {
+ if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
+- return false;
+- else
+ return true;
++ else
++ return false;
+ }
+
+ if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR ||
+ pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
+ pipe->stream->config.mode == IA_CSS_INPUT_MODE_PRBS)
+- return false;
++ return true;
+
+- return true;
++ return false;
+ }
+
+ int
+@@ -423,14 +423,17 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
+ return 0; /* AM TODO: Check */
+ }
+
+- if (!IS_ISP2401)
++ if (!IS_ISP2401) {
+ port = (unsigned int)pipe->stream->config.source.port.port;
+- else
+- err = ia_css_mipi_is_source_port_valid(pipe, &port);
++ } else {
++ /* Returns true if port is valid. So, invert it */
++ err = !ia_css_mipi_is_source_port_valid(pipe, &port);
++ }
+
+ assert(port < N_CSI_PORTS);
+
+- if (port >= N_CSI_PORTS || err) {
++ if ((!IS_ISP2401 && port >= N_CSI_PORTS) ||
++ (IS_ISP2401 && err)) {
+ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
+ "allocate_mipi_frames(%p) exit: error: port is not correct (port=%d).\n",
+ pipe, port);
+@@ -552,14 +555,17 @@ free_mipi_frames(struct ia_css_pipe *pipe)
+ return err;
+ }
+
+- if (!IS_ISP2401)
++ if (!IS_ISP2401) {
+ port = (unsigned int)pipe->stream->config.source.port.port;
+- else
+- err = ia_css_mipi_is_source_port_valid(pipe, &port);
++ } else {
++ /* Returns true if port is valid. So, invert it */
++ err = !ia_css_mipi_is_source_port_valid(pipe, &port);
++ }
+
+ assert(port < N_CSI_PORTS);
+
+- if (port >= N_CSI_PORTS || err) {
++ if ((!IS_ISP2401 && port >= N_CSI_PORTS) ||
++ (IS_ISP2401 && err)) {
+ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
+ "free_mipi_frames(%p, %d) exit: error: pipe port is not correct.\n",
+ pipe, port);
+@@ -663,14 +669,17 @@ send_mipi_frames(struct ia_css_pipe *pipe)
+ /* TODO: AM: maybe this should be returning an error. */
+ }
+
+- if (!IS_ISP2401)
++ if (!IS_ISP2401) {
+ port = (unsigned int)pipe->stream->config.source.port.port;
+- else
+- err = ia_css_mipi_is_source_port_valid(pipe, &port);
++ } else {
++ /* Returns true if port is valid. So, invert it */
++ err = !ia_css_mipi_is_source_port_valid(pipe, &port);
++ }
+
+ assert(port < N_CSI_PORTS);
+
+- if (port >= N_CSI_PORTS || err) {
++ if ((!IS_ISP2401 && port >= N_CSI_PORTS) ||
++ (IS_ISP2401 && err)) {
+ IA_CSS_ERROR("send_mipi_frames(%p) exit: invalid port specified (port=%d).\n",
+ pipe, port);
+ return err;
+diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.c b/drivers/staging/media/atomisp/pci/sh_css_params.c
+index dbd3bfe3d343c..ccc0078795648 100644
+--- a/drivers/staging/media/atomisp/pci/sh_css_params.c
++++ b/drivers/staging/media/atomisp/pci/sh_css_params.c
+@@ -2431,7 +2431,7 @@ sh_css_create_isp_params(struct ia_css_stream *stream,
+ unsigned int i;
+ struct sh_css_ddr_address_map *ddr_ptrs;
+ struct sh_css_ddr_address_map_size *ddr_ptrs_size;
+- int err = 0;
++ int err;
+ size_t params_size;
+ struct ia_css_isp_parameters *params =
+ kvmalloc(sizeof(struct ia_css_isp_parameters), GFP_KERNEL);
+@@ -2473,7 +2473,11 @@ sh_css_create_isp_params(struct ia_css_stream *stream,
+ succ &= (ddr_ptrs->macc_tbl != mmgr_NULL);
+
+ *isp_params_out = params;
+- return err;
++
++ if (!succ)
++ return -ENOMEM;
++
++ return 0;
+ }
+
+ static bool
+diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
+index 20e5081588719..281aa585e3375 100644
+--- a/drivers/staging/media/hantro/hantro_drv.c
++++ b/drivers/staging/media/hantro/hantro_drv.c
+@@ -958,7 +958,7 @@ static int hantro_probe(struct platform_device *pdev)
+ ret = clk_bulk_prepare(vpu->variant->num_clocks, vpu->clocks);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to prepare clocks\n");
+- return ret;
++ goto err_pm_disable;
+ }
+
+ ret = v4l2_device_register(&pdev->dev, &vpu->v4l2_dev);
+@@ -1014,6 +1014,7 @@ err_v4l2_unreg:
+ v4l2_device_unregister(&vpu->v4l2_dev);
+ err_clk_unprepare:
+ clk_bulk_unprepare(vpu->variant->num_clocks, vpu->clocks);
++err_pm_disable:
+ pm_runtime_dont_use_autosuspend(vpu->dev);
+ pm_runtime_disable(vpu->dev);
+ return ret;
+diff --git a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c
+index 56cf261a8e958..686d813f5c626 100644
+--- a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c
++++ b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c
+@@ -23,7 +23,7 @@ static void hantro_h1_set_src_img_ctrl(struct hantro_dev *vpu,
+
+ reg = H1_REG_IN_IMG_CTRL_ROW_LEN(pix_fmt->width)
+ | H1_REG_IN_IMG_CTRL_OVRFLR_D4(0)
+- | H1_REG_IN_IMG_CTRL_OVRFLB_D4(0)
++ | H1_REG_IN_IMG_CTRL_OVRFLB(0)
+ | H1_REG_IN_IMG_CTRL_FMT(ctx->vpu_src_fmt->enc_fmt);
+ vepu_write_relaxed(vpu, reg, H1_REG_IN_IMG_CTRL);
+ }
+@@ -140,7 +140,7 @@ int hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx)
+ return 0;
+ }
+
+-void hantro_jpeg_enc_done(struct hantro_ctx *ctx)
++void hantro_h1_jpeg_enc_done(struct hantro_ctx *ctx)
+ {
+ struct hantro_dev *vpu = ctx->dev;
+ u32 bytesused = vepu_read(vpu, H1_REG_STR_BUF_LIMIT) / 8;
+diff --git a/drivers/staging/media/hantro/hantro_h1_regs.h b/drivers/staging/media/hantro/hantro_h1_regs.h
+index d6e9825bb5c7b..30e7e7b920b55 100644
+--- a/drivers/staging/media/hantro/hantro_h1_regs.h
++++ b/drivers/staging/media/hantro/hantro_h1_regs.h
+@@ -47,7 +47,7 @@
+ #define H1_REG_IN_IMG_CTRL 0x03c
+ #define H1_REG_IN_IMG_CTRL_ROW_LEN(x) ((x) << 12)
+ #define H1_REG_IN_IMG_CTRL_OVRFLR_D4(x) ((x) << 10)
+-#define H1_REG_IN_IMG_CTRL_OVRFLB_D4(x) ((x) << 6)
++#define H1_REG_IN_IMG_CTRL_OVRFLB(x) ((x) << 6)
+ #define H1_REG_IN_IMG_CTRL_FMT(x) ((x) << 2)
+ #define H1_REG_ENC_CTRL0 0x040
+ #define H1_REG_ENC_CTRL0_INIT_QP(x) ((x) << 26)
+diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
+index df7b5e3a57b9b..fd738653a5735 100644
+--- a/drivers/staging/media/hantro/hantro_hw.h
++++ b/drivers/staging/media/hantro/hantro_hw.h
+@@ -235,7 +235,8 @@ int hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx);
+ int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx *ctx);
+ int hantro_jpeg_enc_init(struct hantro_ctx *ctx);
+ void hantro_jpeg_enc_exit(struct hantro_ctx *ctx);
+-void hantro_jpeg_enc_done(struct hantro_ctx *ctx);
++void hantro_h1_jpeg_enc_done(struct hantro_ctx *ctx);
++void rockchip_vpu2_jpeg_enc_done(struct hantro_ctx *ctx);
+
+ dma_addr_t hantro_h264_get_ref_buf(struct hantro_ctx *ctx,
+ unsigned int dpb_idx);
+diff --git a/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c b/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c
+index 991213ce16108..5d9ff420f0b5f 100644
+--- a/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c
++++ b/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c
+@@ -171,3 +171,20 @@ int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx *ctx)
+
+ return 0;
+ }
++
++void rockchip_vpu2_jpeg_enc_done(struct hantro_ctx *ctx)
++{
++ struct hantro_dev *vpu = ctx->dev;
++ u32 bytesused = vepu_read(vpu, VEPU_REG_STR_BUF_LIMIT) / 8;
++ struct vb2_v4l2_buffer *dst_buf = hantro_get_dst_buf(ctx);
++
++ /*
++ * TODO: Rework the JPEG encoder to eliminate the need
++ * for a bounce buffer.
++ */
++ memcpy(vb2_plane_vaddr(&dst_buf->vb2_buf, 0) +
++ ctx->vpu_dst_fmt->header_size,
++ ctx->jpeg_enc.bounce_buffer.cpu, bytesused);
++ vb2_set_plane_payload(&dst_buf->vb2_buf, 0,
++ ctx->vpu_dst_fmt->header_size + bytesused);
++}
+diff --git a/drivers/staging/media/hantro/rockchip_vpu_hw.c b/drivers/staging/media/hantro/rockchip_vpu_hw.c
+index d4f52957cc534..0c22039162a00 100644
+--- a/drivers/staging/media/hantro/rockchip_vpu_hw.c
++++ b/drivers/staging/media/hantro/rockchip_vpu_hw.c
+@@ -343,7 +343,7 @@ static const struct hantro_codec_ops rk3066_vpu_codec_ops[] = {
+ .run = hantro_h1_jpeg_enc_run,
+ .reset = rockchip_vpu1_enc_reset,
+ .init = hantro_jpeg_enc_init,
+- .done = hantro_jpeg_enc_done,
++ .done = hantro_h1_jpeg_enc_done,
+ .exit = hantro_jpeg_enc_exit,
+ },
+ [HANTRO_MODE_H264_DEC] = {
+@@ -371,7 +371,7 @@ static const struct hantro_codec_ops rk3288_vpu_codec_ops[] = {
+ .run = hantro_h1_jpeg_enc_run,
+ .reset = rockchip_vpu1_enc_reset,
+ .init = hantro_jpeg_enc_init,
+- .done = hantro_jpeg_enc_done,
++ .done = hantro_h1_jpeg_enc_done,
+ .exit = hantro_jpeg_enc_exit,
+ },
+ [HANTRO_MODE_H264_DEC] = {
+@@ -399,6 +399,7 @@ static const struct hantro_codec_ops rk3399_vpu_codec_ops[] = {
+ .run = rockchip_vpu2_jpeg_enc_run,
+ .reset = rockchip_vpu2_enc_reset,
+ .init = hantro_jpeg_enc_init,
++ .done = rockchip_vpu2_jpeg_enc_done,
+ .exit = hantro_jpeg_enc_exit,
+ },
+ [HANTRO_MODE_H264_DEC] = {
+diff --git a/drivers/staging/media/imx/imx-media-dev-common.c b/drivers/staging/media/imx/imx-media-dev-common.c
+index d186179388d03..4d873726a461b 100644
+--- a/drivers/staging/media/imx/imx-media-dev-common.c
++++ b/drivers/staging/media/imx/imx-media-dev-common.c
+@@ -367,6 +367,8 @@ struct imx_media_dev *imx_media_dev_init(struct device *dev,
+ imxmd->v4l2_dev.notify = imx_media_notify;
+ strscpy(imxmd->v4l2_dev.name, "imx-media",
+ sizeof(imxmd->v4l2_dev.name));
++ snprintf(imxmd->md.bus_info, sizeof(imxmd->md.bus_info),
++ "platform:%s", dev_name(imxmd->md.dev));
+
+ media_device_init(&imxmd->md);
+
+diff --git a/drivers/staging/media/imx/imx7-mipi-csis.c b/drivers/staging/media/imx/imx7-mipi-csis.c
+index 41e33535de552..d35e523741168 100644
+--- a/drivers/staging/media/imx/imx7-mipi-csis.c
++++ b/drivers/staging/media/imx/imx7-mipi-csis.c
+@@ -32,7 +32,6 @@
+ #include <media/v4l2-subdev.h>
+
+ #define CSIS_DRIVER_NAME "imx7-mipi-csis"
+-#define CSIS_SUBDEV_NAME CSIS_DRIVER_NAME
+
+ #define CSIS_PAD_SINK 0
+ #define CSIS_PAD_SOURCE 1
+@@ -311,7 +310,6 @@ struct csi_state {
+ struct reset_control *mrst;
+ struct regulator *mipi_phy_regulator;
+ const struct mipi_csis_info *info;
+- u8 index;
+
+ struct v4l2_subdev sd;
+ struct media_pad pads[CSIS_PADS_NUM];
+@@ -1303,8 +1301,8 @@ static int mipi_csis_subdev_init(struct csi_state *state)
+
+ v4l2_subdev_init(sd, &mipi_csis_subdev_ops);
+ sd->owner = THIS_MODULE;
+- snprintf(sd->name, sizeof(sd->name), "%s.%d",
+- CSIS_SUBDEV_NAME, state->index);
++ snprintf(sd->name, sizeof(sd->name), "csis-%s",
++ dev_name(state->dev));
+
+ sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+ sd->ctrl_handler = NULL;
+diff --git a/drivers/staging/media/imx/imx8mq-mipi-csi2.c b/drivers/staging/media/imx/imx8mq-mipi-csi2.c
+index a6f562009b9a0..1d28313dbed7d 100644
+--- a/drivers/staging/media/imx/imx8mq-mipi-csi2.c
++++ b/drivers/staging/media/imx/imx8mq-mipi-csi2.c
+@@ -398,9 +398,6 @@ static int imx8mq_mipi_csi_s_stream(struct v4l2_subdev *sd, int enable)
+ struct csi_state *state = mipi_sd_to_csi2_state(sd);
+ int ret = 0;
+
+- imx8mq_mipi_csi_write(state, CSI2RX_IRQ_MASK,
+- CSI2RX_IRQ_MASK_ULPS_STATUS_CHANGE);
+-
+ if (enable) {
+ ret = pm_runtime_resume_and_get(state->dev);
+ if (ret < 0)
+@@ -696,7 +693,7 @@ err_parse:
+ * Suspend/resume
+ */
+
+-static int imx8mq_mipi_csi_pm_suspend(struct device *dev, bool runtime)
++static int imx8mq_mipi_csi_pm_suspend(struct device *dev)
+ {
+ struct v4l2_subdev *sd = dev_get_drvdata(dev);
+ struct csi_state *state = mipi_sd_to_csi2_state(sd);
+@@ -708,36 +705,21 @@ static int imx8mq_mipi_csi_pm_suspend(struct device *dev, bool runtime)
+ imx8mq_mipi_csi_stop_stream(state);
+ imx8mq_mipi_csi_clk_disable(state);
+ state->state &= ~ST_POWERED;
+- if (!runtime)
+- state->state |= ST_SUSPENDED;
+ }
+
+ mutex_unlock(&state->lock);
+
+- ret = icc_set_bw(state->icc_path, 0, 0);
+- if (ret)
+- dev_err(dev, "icc_set_bw failed with %d\n", ret);
+-
+ return ret ? -EAGAIN : 0;
+ }
+
+-static int imx8mq_mipi_csi_pm_resume(struct device *dev, bool runtime)
++static int imx8mq_mipi_csi_pm_resume(struct device *dev)
+ {
+ struct v4l2_subdev *sd = dev_get_drvdata(dev);
+ struct csi_state *state = mipi_sd_to_csi2_state(sd);
+ int ret = 0;
+
+- ret = icc_set_bw(state->icc_path, 0, state->icc_path_bw);
+- if (ret) {
+- dev_err(dev, "icc_set_bw failed with %d\n", ret);
+- return ret;
+- }
+-
+ mutex_lock(&state->lock);
+
+- if (!runtime && !(state->state & ST_SUSPENDED))
+- goto unlock;
+-
+ if (!(state->state & ST_POWERED)) {
+ state->state |= ST_POWERED;
+ ret = imx8mq_mipi_csi_clk_enable(state);
+@@ -758,22 +740,60 @@ unlock:
+
+ static int __maybe_unused imx8mq_mipi_csi_suspend(struct device *dev)
+ {
+- return imx8mq_mipi_csi_pm_suspend(dev, false);
++ struct v4l2_subdev *sd = dev_get_drvdata(dev);
++ struct csi_state *state = mipi_sd_to_csi2_state(sd);
++ int ret;
++
++ ret = imx8mq_mipi_csi_pm_suspend(dev);
++ if (ret)
++ return ret;
++
++ state->state |= ST_SUSPENDED;
++
++ return ret;
+ }
+
+ static int __maybe_unused imx8mq_mipi_csi_resume(struct device *dev)
+ {
+- return imx8mq_mipi_csi_pm_resume(dev, false);
++ struct v4l2_subdev *sd = dev_get_drvdata(dev);
++ struct csi_state *state = mipi_sd_to_csi2_state(sd);
++
++ if (!(state->state & ST_SUSPENDED))
++ return 0;
++
++ return imx8mq_mipi_csi_pm_resume(dev);
+ }
+
+ static int __maybe_unused imx8mq_mipi_csi_runtime_suspend(struct device *dev)
+ {
+- return imx8mq_mipi_csi_pm_suspend(dev, true);
++ struct v4l2_subdev *sd = dev_get_drvdata(dev);
++ struct csi_state *state = mipi_sd_to_csi2_state(sd);
++ int ret;
++
++ ret = imx8mq_mipi_csi_pm_suspend(dev);
++ if (ret)
++ return ret;
++
++ ret = icc_set_bw(state->icc_path, 0, 0);
++ if (ret)
++ dev_err(dev, "icc_set_bw failed with %d\n", ret);
++
++ return ret;
+ }
+
+ static int __maybe_unused imx8mq_mipi_csi_runtime_resume(struct device *dev)
+ {
+- return imx8mq_mipi_csi_pm_resume(dev, true);
++ struct v4l2_subdev *sd = dev_get_drvdata(dev);
++ struct csi_state *state = mipi_sd_to_csi2_state(sd);
++ int ret;
++
++ ret = icc_set_bw(state->icc_path, 0, state->icc_path_bw);
++ if (ret) {
++ dev_err(dev, "icc_set_bw failed with %d\n", ret);
++ return ret;
++ }
++
++ return imx8mq_mipi_csi_pm_resume(dev);
+ }
+
+ static const struct dev_pm_ops imx8mq_mipi_csi_pm_ops = {
+@@ -921,7 +941,7 @@ static int imx8mq_mipi_csi_probe(struct platform_device *pdev)
+ /* Enable runtime PM. */
+ pm_runtime_enable(dev);
+ if (!pm_runtime_enabled(dev)) {
+- ret = imx8mq_mipi_csi_pm_resume(dev, true);
++ ret = imx8mq_mipi_csi_runtime_resume(dev);
+ if (ret < 0)
+ goto icc;
+ }
+@@ -934,7 +954,7 @@ static int imx8mq_mipi_csi_probe(struct platform_device *pdev)
+
+ cleanup:
+ pm_runtime_disable(&pdev->dev);
+- imx8mq_mipi_csi_pm_suspend(&pdev->dev, true);
++ imx8mq_mipi_csi_runtime_suspend(&pdev->dev);
+
+ media_entity_cleanup(&state->sd.entity);
+ v4l2_async_notifier_unregister(&state->notifier);
+@@ -958,7 +978,7 @@ static int imx8mq_mipi_csi_remove(struct platform_device *pdev)
+ v4l2_async_unregister_subdev(&state->sd);
+
+ pm_runtime_disable(&pdev->dev);
+- imx8mq_mipi_csi_pm_suspend(&pdev->dev, true);
++ imx8mq_mipi_csi_runtime_suspend(&pdev->dev);
+ media_entity_cleanup(&state->sd.entity);
+ mutex_destroy(&state->lock);
+ pm_runtime_set_suspended(&pdev->dev);
+diff --git a/drivers/staging/media/ipu3/ipu3-css-fw.c b/drivers/staging/media/ipu3/ipu3-css-fw.c
+index 45aff76198e2c..981693eed8155 100644
+--- a/drivers/staging/media/ipu3/ipu3-css-fw.c
++++ b/drivers/staging/media/ipu3/ipu3-css-fw.c
+@@ -124,12 +124,11 @@ int imgu_css_fw_init(struct imgu_css *css)
+ /* Check and display fw header info */
+
+ css->fwp = (struct imgu_fw_header *)css->fw->data;
+- if (css->fw->size < sizeof(struct imgu_fw_header *) ||
++ if (css->fw->size < struct_size(css->fwp, binary_header, 1) ||
+ css->fwp->file_header.h_size != sizeof(struct imgu_fw_bi_file_h))
+ goto bad_fw;
+- if (sizeof(struct imgu_fw_bi_file_h) +
+- css->fwp->file_header.binary_nr * sizeof(struct imgu_fw_info) >
+- css->fw->size)
++ if (struct_size(css->fwp, binary_header,
++ css->fwp->file_header.binary_nr) > css->fw->size)
+ goto bad_fw;
+
+ dev_info(dev, "loaded firmware version %.64s, %u binaries, %zu bytes\n",
+diff --git a/drivers/staging/media/ipu3/ipu3-css-fw.h b/drivers/staging/media/ipu3/ipu3-css-fw.h
+index 3c078f15a2959..c0bc57fd678a7 100644
+--- a/drivers/staging/media/ipu3/ipu3-css-fw.h
++++ b/drivers/staging/media/ipu3/ipu3-css-fw.h
+@@ -171,7 +171,7 @@ struct imgu_fw_bi_file_h {
+
+ struct imgu_fw_header {
+ struct imgu_fw_bi_file_h file_header;
+- struct imgu_fw_info binary_header[1]; /* binary_nr items */
++ struct imgu_fw_info binary_header[]; /* binary_nr items */
+ };
+
+ /******************* Firmware functions *******************/
+diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c
+index 38a2407645096..90c86ba5040e3 100644
+--- a/drivers/staging/media/ipu3/ipu3-v4l2.c
++++ b/drivers/staging/media/ipu3/ipu3-v4l2.c
+@@ -592,11 +592,12 @@ static const struct imgu_fmt *find_format(struct v4l2_format *f, u32 type)
+ static int imgu_vidioc_querycap(struct file *file, void *fh,
+ struct v4l2_capability *cap)
+ {
+- struct imgu_video_device *node = file_to_intel_imgu_node(file);
++ struct imgu_device *imgu = video_drvdata(file);
+
+ strscpy(cap->driver, IMGU_NAME, sizeof(cap->driver));
+ strscpy(cap->card, IMGU_NAME, sizeof(cap->card));
+- snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s", node->name);
++ snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s",
++ pci_name(imgu->pci_dev));
+
+ return 0;
+ }
+@@ -696,7 +697,7 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node,
+
+ /* CSS expects some format on OUT queue */
+ if (i != IPU3_CSS_QUEUE_OUT &&
+- !imgu_pipe->nodes[inode].enabled) {
++ !imgu_pipe->nodes[inode].enabled && !try) {
+ fmts[i] = NULL;
+ continue;
+ }
+diff --git a/drivers/staging/media/meson/vdec/esparser.c b/drivers/staging/media/meson/vdec/esparser.c
+index db7022707ff8d..86ccc8937afca 100644
+--- a/drivers/staging/media/meson/vdec/esparser.c
++++ b/drivers/staging/media/meson/vdec/esparser.c
+@@ -328,7 +328,12 @@ esparser_queue(struct amvdec_session *sess, struct vb2_v4l2_buffer *vbuf)
+
+ offset = esparser_get_offset(sess);
+
+- amvdec_add_ts(sess, vb->timestamp, vbuf->timecode, offset, vbuf->flags);
++ ret = amvdec_add_ts(sess, vb->timestamp, vbuf->timecode, offset, vbuf->flags);
++ if (ret) {
++ v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
++ return ret;
++ }
++
+ dev_dbg(core->dev, "esparser: ts = %llu pld_size = %u offset = %08X flags = %08X\n",
+ vb->timestamp, payload_size, offset, vbuf->flags);
+
+diff --git a/drivers/staging/media/meson/vdec/vdec_helpers.c b/drivers/staging/media/meson/vdec/vdec_helpers.c
+index b9125c295d1d3..06fd66539797a 100644
+--- a/drivers/staging/media/meson/vdec/vdec_helpers.c
++++ b/drivers/staging/media/meson/vdec/vdec_helpers.c
+@@ -227,13 +227,16 @@ int amvdec_set_canvases(struct amvdec_session *sess,
+ }
+ EXPORT_SYMBOL_GPL(amvdec_set_canvases);
+
+-void amvdec_add_ts(struct amvdec_session *sess, u64 ts,
+- struct v4l2_timecode tc, u32 offset, u32 vbuf_flags)
++int amvdec_add_ts(struct amvdec_session *sess, u64 ts,
++ struct v4l2_timecode tc, u32 offset, u32 vbuf_flags)
+ {
+ struct amvdec_timestamp *new_ts;
+ unsigned long flags;
+
+ new_ts = kzalloc(sizeof(*new_ts), GFP_KERNEL);
++ if (!new_ts)
++ return -ENOMEM;
++
+ new_ts->ts = ts;
+ new_ts->tc = tc;
+ new_ts->offset = offset;
+@@ -242,6 +245,7 @@ void amvdec_add_ts(struct amvdec_session *sess, u64 ts,
+ spin_lock_irqsave(&sess->ts_spinlock, flags);
+ list_add_tail(&new_ts->list, &sess->timestamps);
+ spin_unlock_irqrestore(&sess->ts_spinlock, flags);
++ return 0;
+ }
+ EXPORT_SYMBOL_GPL(amvdec_add_ts);
+
+diff --git a/drivers/staging/media/meson/vdec/vdec_helpers.h b/drivers/staging/media/meson/vdec/vdec_helpers.h
+index cfaed52ab5265..798e5a8a9b3f1 100644
+--- a/drivers/staging/media/meson/vdec/vdec_helpers.h
++++ b/drivers/staging/media/meson/vdec/vdec_helpers.h
+@@ -55,8 +55,8 @@ void amvdec_dst_buf_done_offset(struct amvdec_session *sess,
+ * @offset: offset in the VIFIFO where the associated packet was written
+ * @flags the vb2_v4l2_buffer flags
+ */
+-void amvdec_add_ts(struct amvdec_session *sess, u64 ts,
+- struct v4l2_timecode tc, u32 offset, u32 flags);
++int amvdec_add_ts(struct amvdec_session *sess, u64 ts,
++ struct v4l2_timecode tc, u32 offset, u32 flags);
+ void amvdec_remove_ts(struct amvdec_session *sess, u64 ts);
+
+ /**
+diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c
+index 76e97cbe25123..951e19231da21 100644
+--- a/drivers/staging/media/rkvdec/rkvdec-h264.c
++++ b/drivers/staging/media/rkvdec/rkvdec-h264.c
+@@ -1015,8 +1015,9 @@ static int rkvdec_h264_adjust_fmt(struct rkvdec_ctx *ctx,
+ struct v4l2_pix_format_mplane *fmt = &f->fmt.pix_mp;
+
+ fmt->num_planes = 1;
+- fmt->plane_fmt[0].sizeimage = fmt->width * fmt->height *
+- RKVDEC_H264_MAX_DEPTH_IN_BYTES;
++ if (!fmt->plane_fmt[0].sizeimage)
++ fmt->plane_fmt[0].sizeimage = fmt->width * fmt->height *
++ RKVDEC_H264_MAX_DEPTH_IN_BYTES;
+ return 0;
+ }
+
+diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c
+index 7131156c1f2cf..3f3f96488d741 100644
+--- a/drivers/staging/media/rkvdec/rkvdec.c
++++ b/drivers/staging/media/rkvdec/rkvdec.c
+@@ -280,31 +280,20 @@ static int rkvdec_try_output_fmt(struct file *file, void *priv,
+ return 0;
+ }
+
+-static int rkvdec_s_fmt(struct file *file, void *priv,
+- struct v4l2_format *f,
+- int (*try_fmt)(struct file *, void *,
+- struct v4l2_format *))
++static int rkvdec_s_capture_fmt(struct file *file, void *priv,
++ struct v4l2_format *f)
+ {
+ struct rkvdec_ctx *ctx = fh_to_rkvdec_ctx(priv);
+ struct vb2_queue *vq;
++ int ret;
+
+- if (!try_fmt)
+- return -EINVAL;
+-
+- vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
++ /* Change not allowed if queue is busy */
++ vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
++ V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
+ if (vb2_is_busy(vq))
+ return -EBUSY;
+
+- return try_fmt(file, priv, f);
+-}
+-
+-static int rkvdec_s_capture_fmt(struct file *file, void *priv,
+- struct v4l2_format *f)
+-{
+- struct rkvdec_ctx *ctx = fh_to_rkvdec_ctx(priv);
+- int ret;
+-
+- ret = rkvdec_s_fmt(file, priv, f, rkvdec_try_capture_fmt);
++ ret = rkvdec_try_capture_fmt(file, priv, f);
+ if (ret)
+ return ret;
+
+@@ -319,9 +308,20 @@ static int rkvdec_s_output_fmt(struct file *file, void *priv,
+ struct v4l2_m2m_ctx *m2m_ctx = ctx->fh.m2m_ctx;
+ const struct rkvdec_coded_fmt_desc *desc;
+ struct v4l2_format *cap_fmt;
+- struct vb2_queue *peer_vq;
++ struct vb2_queue *peer_vq, *vq;
+ int ret;
+
++ /*
++ * In order to support dynamic resolution change, the decoder admits
++ * a resolution change, as long as the pixelformat remains. Can't be
++ * done if streaming.
++ */
++ vq = v4l2_m2m_get_vq(m2m_ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
++ if (vb2_is_streaming(vq) ||
++ (vb2_is_busy(vq) &&
++ f->fmt.pix_mp.pixelformat != ctx->coded_fmt.fmt.pix_mp.pixelformat))
++ return -EBUSY;
++
+ /*
+ * Since format change on the OUTPUT queue will reset the CAPTURE
+ * queue, we can't allow doing so when the CAPTURE queue has buffers
+@@ -331,7 +331,7 @@ static int rkvdec_s_output_fmt(struct file *file, void *priv,
+ if (vb2_is_busy(peer_vq))
+ return -EBUSY;
+
+- ret = rkvdec_s_fmt(file, priv, f, rkvdec_try_output_fmt);
++ ret = rkvdec_try_output_fmt(file, priv, f);
+ if (ret)
+ return ret;
+
+diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+index de7442d4834dc..d3e26bfe6c90b 100644
+--- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
++++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+@@ -38,7 +38,7 @@ struct cedrus_h264_sram_ref_pic {
+
+ #define CEDRUS_H264_FRAME_NUM 18
+
+-#define CEDRUS_NEIGHBOR_INFO_BUF_SIZE (16 * SZ_1K)
++#define CEDRUS_NEIGHBOR_INFO_BUF_SIZE (32 * SZ_1K)
+ #define CEDRUS_MIN_PIC_INFO_BUF_SIZE (130 * SZ_1K)
+
+ static void cedrus_h264_write_sram(struct cedrus_dev *dev,
+diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
+index ef0311a16d019..754942ecf064c 100644
+--- a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
++++ b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
+@@ -23,7 +23,7 @@
+ * Subsequent BSP implementations seem to double the neighbor info buffer size
+ * for the H6 SoC, which may be related to 10 bit H265 support.
+ */
+-#define CEDRUS_H265_NEIGHBOR_INFO_BUF_SIZE (397 * SZ_1K)
++#define CEDRUS_H265_NEIGHBOR_INFO_BUF_SIZE (794 * SZ_1K)
+ #define CEDRUS_H265_ENTRY_POINTS_BUF_SIZE (4 * SZ_1K)
+ #define CEDRUS_H265_MV_COL_BUF_UNIT_CTB_SIZE 160
+
+diff --git a/drivers/staging/media/zoran/zoran.h b/drivers/staging/media/zoran/zoran.h
+index b1ad2a2b914cd..50d5a7acfab6c 100644
+--- a/drivers/staging/media/zoran/zoran.h
++++ b/drivers/staging/media/zoran/zoran.h
+@@ -313,6 +313,6 @@ static inline struct zoran *to_zoran(struct v4l2_device *v4l2_dev)
+
+ #endif
+
+-int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq);
++int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq, int dir);
+ void zoran_queue_exit(struct zoran *zr);
+ int zr_set_buf(struct zoran *zr);
+diff --git a/drivers/staging/media/zoran/zoran_card.c b/drivers/staging/media/zoran/zoran_card.c
+index f259585b06897..11d415c0c05d2 100644
+--- a/drivers/staging/media/zoran/zoran_card.c
++++ b/drivers/staging/media/zoran/zoran_card.c
+@@ -803,6 +803,52 @@ int zoran_check_jpg_settings(struct zoran *zr,
+ return 0;
+ }
+
++static int zoran_init_video_device(struct zoran *zr, struct video_device *video_dev, int dir)
++{
++ int err;
++
++ /* Now add the template and register the device unit. */
++ *video_dev = zoran_template;
++ video_dev->v4l2_dev = &zr->v4l2_dev;
++ video_dev->lock = &zr->lock;
++ video_dev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE | dir;
++
++ strscpy(video_dev->name, ZR_DEVNAME(zr), sizeof(video_dev->name));
++ /*
++ * It's not a mem2mem device, but you can both capture and output from one and the same
++ * device. This should really be split up into two device nodes, but that's a job for
++ * another day.
++ */
++ video_dev->vfl_dir = VFL_DIR_M2M;
++ zoran_queue_init(zr, &zr->vq, V4L2_BUF_TYPE_VIDEO_CAPTURE);
++
++ err = video_register_device(video_dev, VFL_TYPE_VIDEO, video_nr[zr->id]);
++ if (err < 0)
++ return err;
++ video_set_drvdata(video_dev, zr);
++ return 0;
++}
++
++static void zoran_exit_video_devices(struct zoran *zr)
++{
++ video_unregister_device(zr->video_dev);
++ kfree(zr->video_dev);
++}
++
++static int zoran_init_video_devices(struct zoran *zr)
++{
++ int err;
++
++ zr->video_dev = video_device_alloc();
++ if (!zr->video_dev)
++ return -ENOMEM;
++
++ err = zoran_init_video_device(zr, zr->video_dev, V4L2_CAP_VIDEO_CAPTURE);
++ if (err)
++ kfree(zr->video_dev);
++ return err;
++}
++
+ void zoran_open_init_params(struct zoran *zr)
+ {
+ int i;
+@@ -874,17 +920,11 @@ static int zr36057_init(struct zoran *zr)
+ zoran_open_init_params(zr);
+
+ /* allocate memory *before* doing anything to the hardware in case allocation fails */
+- zr->video_dev = video_device_alloc();
+- if (!zr->video_dev) {
+- err = -ENOMEM;
+- goto exit;
+- }
+ zr->stat_com = dma_alloc_coherent(&zr->pci_dev->dev,
+ BUZ_NUM_STAT_COM * sizeof(u32),
+ &zr->p_sc, GFP_KERNEL);
+ if (!zr->stat_com) {
+- err = -ENOMEM;
+- goto exit_video;
++ return -ENOMEM;
+ }
+ for (j = 0; j < BUZ_NUM_STAT_COM; j++)
+ zr->stat_com[j] = cpu_to_le32(1); /* mark as unavailable to zr36057 */
+@@ -897,26 +937,9 @@ static int zr36057_init(struct zoran *zr)
+ goto exit_statcom;
+ }
+
+- /* Now add the template and register the device unit. */
+- *zr->video_dev = zoran_template;
+- zr->video_dev->v4l2_dev = &zr->v4l2_dev;
+- zr->video_dev->lock = &zr->lock;
+- zr->video_dev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+-
+- strscpy(zr->video_dev->name, ZR_DEVNAME(zr), sizeof(zr->video_dev->name));
+- /*
+- * It's not a mem2mem device, but you can both capture and output from one and the same
+- * device. This should really be split up into two device nodes, but that's a job for
+- * another day.
+- */
+- zr->video_dev->vfl_dir = VFL_DIR_M2M;
+-
+- zoran_queue_init(zr, &zr->vq);
+-
+- err = video_register_device(zr->video_dev, VFL_TYPE_VIDEO, video_nr[zr->id]);
+- if (err < 0)
++ err = zoran_init_video_devices(zr);
++ if (err)
+ goto exit_statcomb;
+- video_set_drvdata(zr->video_dev, zr);
+
+ zoran_init_hardware(zr);
+ if (!pass_through) {
+@@ -931,9 +954,6 @@ exit_statcomb:
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb);
+ exit_statcom:
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), zr->stat_com, zr->p_sc);
+-exit_video:
+- kfree(zr->video_dev);
+-exit:
+ return err;
+ }
+
+@@ -965,7 +985,7 @@ static void zoran_remove(struct pci_dev *pdev)
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb);
+ pci_release_regions(pdev);
+ pci_disable_device(zr->pci_dev);
+- video_unregister_device(zr->video_dev);
++ zoran_exit_video_devices(zr);
+ exit_free:
+ v4l2_ctrl_handler_free(&zr->hdl);
+ v4l2_device_unregister(&zr->v4l2_dev);
+@@ -1069,8 +1089,10 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+
+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+ if (err)
+- return -ENODEV;
+- vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
++ return err;
++ err = vb2_dma_contig_set_max_seg_size(&pdev->dev, U32_MAX);
++ if (err)
++ return err;
+
+ nr = zoran_num++;
+ if (nr >= BUZ_MAX) {
+diff --git a/drivers/staging/media/zoran/zoran_device.c b/drivers/staging/media/zoran/zoran_device.c
+index 5b12a730a2290..fb1f0465ca87f 100644
+--- a/drivers/staging/media/zoran/zoran_device.c
++++ b/drivers/staging/media/zoran/zoran_device.c
+@@ -814,7 +814,7 @@ static void zoran_reap_stat_com(struct zoran *zr)
+ if (zr->jpg_settings.tmp_dcm == 1)
+ i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
+ else
+- i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2 + 1;
++ i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2;
+
+ stat_com = le32_to_cpu(zr->stat_com[i]);
+ if ((stat_com & 1) == 0) {
+@@ -826,6 +826,11 @@ static void zoran_reap_stat_com(struct zoran *zr)
+ size = (stat_com & GENMASK(22, 1)) >> 1;
+
+ buf = zr->inuse[i];
++ if (!buf) {
++ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
++ pci_err(zr->pci_dev, "No buffer at slot %d\n", i);
++ return;
++ }
+ buf->vbuf.vb2_buf.timestamp = ktime_get_ns();
+
+ if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
+diff --git a/drivers/staging/media/zoran/zoran_driver.c b/drivers/staging/media/zoran/zoran_driver.c
+index 46382e43f1bf7..84665637ebb79 100644
+--- a/drivers/staging/media/zoran/zoran_driver.c
++++ b/drivers/staging/media/zoran/zoran_driver.c
+@@ -255,8 +255,6 @@ static int zoran_querycap(struct file *file, void *__fh, struct v4l2_capability
+ strscpy(cap->card, ZR_DEVNAME(zr), sizeof(cap->card));
+ strscpy(cap->driver, "zoran", sizeof(cap->driver));
+ snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s", pci_name(zr->pci_dev));
+- cap->device_caps = zr->video_dev->device_caps;
+- cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+ return 0;
+ }
+
+@@ -582,6 +580,9 @@ static int zoran_s_std(struct file *file, void *__fh, v4l2_std_id std)
+ struct zoran *zr = video_drvdata(file);
+ int res = 0;
+
++ if (zr->norm == std)
++ return 0;
++
+ if (zr->running != ZORAN_MAP_MODE_NONE)
+ return -EBUSY;
+
+@@ -739,6 +740,7 @@ static int zoran_g_parm(struct file *file, void *priv, struct v4l2_streamparm *p
+ if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+
++ parm->parm.capture.readbuffers = 9;
+ return 0;
+ }
+
+@@ -869,6 +871,10 @@ int zr_set_buf(struct zoran *zr)
+ vbuf = &buf->vbuf;
+
+ buf->vbuf.field = V4L2_FIELD_INTERLACED;
++ if (BUZ_MAX_HEIGHT < (zr->v4l_settings.height * 2))
++ buf->vbuf.field = V4L2_FIELD_INTERLACED;
++ else
++ buf->vbuf.field = V4L2_FIELD_TOP;
+ vb2_set_plane_payload(&buf->vbuf.vb2_buf, 0, zr->buffer_size);
+ vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_DONE);
+ zr->inuse[0] = NULL;
+@@ -928,6 +934,7 @@ static int zr_vb2_start_streaming(struct vb2_queue *vq, unsigned int count)
+ zr->stat_com[j] = cpu_to_le32(1);
+ zr->inuse[j] = NULL;
+ }
++ zr->vbseq = 0;
+
+ if (zr->map_mode != ZORAN_MAP_MODE_RAW) {
+ pci_info(zr->pci_dev, "START JPG\n");
+@@ -1008,7 +1015,7 @@ static const struct vb2_ops zr_video_qops = {
+ .wait_finish = vb2_ops_wait_finish,
+ };
+
+-int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq)
++int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq, int dir)
+ {
+ int err;
+
+@@ -1016,8 +1023,9 @@ int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq)
+ INIT_LIST_HEAD(&zr->queued_bufs);
+
+ vq->dev = &zr->pci_dev->dev;
+- vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+- vq->io_modes = VB2_USERPTR | VB2_DMABUF | VB2_MMAP | VB2_READ | VB2_WRITE;
++ vq->type = dir;
++
++ vq->io_modes = VB2_DMABUF | VB2_MMAP | VB2_READ | VB2_WRITE;
+ vq->drv_priv = zr;
+ vq->buf_struct_size = sizeof(struct zr_buffer);
+ vq->ops = &zr_video_qops;
+diff --git a/drivers/staging/most/dim2/Makefile b/drivers/staging/most/dim2/Makefile
+index 861adacf6c729..5f9612af3fa3c 100644
+--- a/drivers/staging/most/dim2/Makefile
++++ b/drivers/staging/most/dim2/Makefile
+@@ -1,4 +1,4 @@
+ # SPDX-License-Identifier: GPL-2.0
+ obj-$(CONFIG_MOST_DIM2) += most_dim2.o
+
+-most_dim2-objs := dim2.o hal.o sysfs.o
++most_dim2-objs := dim2.o hal.o
+diff --git a/drivers/staging/most/dim2/dim2.c b/drivers/staging/most/dim2/dim2.c
+index 093ef9a2b2919..81e062009d271 100644
+--- a/drivers/staging/most/dim2/dim2.c
++++ b/drivers/staging/most/dim2/dim2.c
+@@ -117,7 +117,8 @@ struct dim2_platform_data {
+ (((p)[1] == 0x18) && ((p)[2] == 0x05) && ((p)[3] == 0x0C) && \
+ ((p)[13] == 0x3C) && ((p)[14] == 0x00) && ((p)[15] == 0x0A))
+
+-bool dim2_sysfs_get_state_cb(void)
++static ssize_t state_show(struct device *dev, struct device_attribute *attr,
++ char *buf)
+ {
+ bool state;
+ unsigned long flags;
+@@ -126,9 +127,18 @@ bool dim2_sysfs_get_state_cb(void)
+ state = dim_get_lock_state();
+ spin_unlock_irqrestore(&dim_lock, flags);
+
+- return state;
++ return sysfs_emit(buf, "%s\n", state ? "locked" : "");
+ }
+
++static DEVICE_ATTR_RO(state);
++
++static struct attribute *dim2_attrs[] = {
++ &dev_attr_state.attr,
++ NULL,
++};
++
++ATTRIBUTE_GROUPS(dim2);
++
+ /**
+ * dimcb_on_error - callback from HAL to report miscommunication between
+ * HDM and HAL
+@@ -716,6 +726,23 @@ static int get_dim2_clk_speed(const char *clock_speed, u8 *val)
+ return -EINVAL;
+ }
+
++static void dim2_release(struct device *d)
++{
++ struct dim2_hdm *dev = container_of(d, struct dim2_hdm, dev);
++ unsigned long flags;
++
++ kthread_stop(dev->netinfo_task);
++
++ spin_lock_irqsave(&dim_lock, flags);
++ dim_shutdown();
++ spin_unlock_irqrestore(&dim_lock, flags);
++
++ if (dev->disable_platform)
++ dev->disable_platform(to_platform_device(d->parent));
++
++ kfree(dev);
++}
++
+ /*
+ * dim2_probe - dim2 probe handler
+ * @pdev: platform device structure
+@@ -736,7 +763,7 @@ static int dim2_probe(struct platform_device *pdev)
+
+ enum { MLB_INT_IDX, AHB0_INT_IDX };
+
+- dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
++ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return -ENOMEM;
+
+@@ -748,25 +775,27 @@ static int dim2_probe(struct platform_device *pdev)
+ "microchip,clock-speed", &clock_speed);
+ if (ret) {
+ dev_err(&pdev->dev, "missing dt property clock-speed\n");
+- return ret;
++ goto err_free_dev;
+ }
+
+ ret = get_dim2_clk_speed(clock_speed, &dev->clk_speed);
+ if (ret) {
+ dev_err(&pdev->dev, "bad dt property clock-speed\n");
+- return ret;
++ goto err_free_dev;
+ }
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ dev->io_base = devm_ioremap_resource(&pdev->dev, res);
+- if (IS_ERR(dev->io_base))
+- return PTR_ERR(dev->io_base);
++ if (IS_ERR(dev->io_base)) {
++ ret = PTR_ERR(dev->io_base);
++ goto err_free_dev;
++ }
+
+ of_id = of_match_node(dim2_of_match, pdev->dev.of_node);
+ pdata = of_id->data;
+ ret = pdata && pdata->enable ? pdata->enable(pdev) : 0;
+ if (ret)
+- return ret;
++ goto err_free_dev;
+
+ dev->disable_platform = pdata ? pdata->disable : NULL;
+
+@@ -857,32 +886,19 @@ static int dim2_probe(struct platform_device *pdev)
+ dev->most_iface.request_netinfo = request_netinfo;
+ dev->most_iface.driver_dev = &pdev->dev;
+ dev->most_iface.dev = &dev->dev;
+- dev->dev.init_name = "dim2_state";
++ dev->dev.init_name = dev->name;
+ dev->dev.parent = &pdev->dev;
++ dev->dev.release = dim2_release;
+
+- ret = most_register_interface(&dev->most_iface);
+- if (ret) {
+- dev_err(&pdev->dev, "failed to register MOST interface\n");
+- goto err_stop_thread;
+- }
+-
+- ret = dim2_sysfs_probe(&dev->dev);
+- if (ret) {
+- dev_err(&pdev->dev, "failed to create sysfs attribute\n");
+- goto err_unreg_iface;
+- }
+-
+- return 0;
++ return most_register_interface(&dev->most_iface);
+
+-err_unreg_iface:
+- most_deregister_interface(&dev->most_iface);
+-err_stop_thread:
+- kthread_stop(dev->netinfo_task);
+ err_shutdown_dim:
+ dim_shutdown();
+ err_disable_platform:
+ if (dev->disable_platform)
+ dev->disable_platform(pdev);
++err_free_dev:
++ kfree(dev);
+
+ return ret;
+ }
+@@ -896,18 +912,8 @@ err_disable_platform:
+ static int dim2_remove(struct platform_device *pdev)
+ {
+ struct dim2_hdm *dev = platform_get_drvdata(pdev);
+- unsigned long flags;
+
+- dim2_sysfs_destroy(&dev->dev);
+ most_deregister_interface(&dev->most_iface);
+- kthread_stop(dev->netinfo_task);
+-
+- spin_lock_irqsave(&dim_lock, flags);
+- dim_shutdown();
+- spin_unlock_irqrestore(&dim_lock, flags);
+-
+- if (dev->disable_platform)
+- dev->disable_platform(pdev);
+
+ return 0;
+ }
+@@ -1082,6 +1088,7 @@ static struct platform_driver dim2_driver = {
+ .driver = {
+ .name = "hdm_dim2",
+ .of_match_table = dim2_of_match,
++ .dev_groups = dim2_groups,
+ },
+ };
+
+diff --git a/drivers/staging/most/dim2/sysfs.c b/drivers/staging/most/dim2/sysfs.c
+deleted file mode 100644
+index c85b2cdcdca3d..0000000000000
+--- a/drivers/staging/most/dim2/sysfs.c
++++ /dev/null
+@@ -1,49 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0
+-/*
+- * sysfs.c - MediaLB sysfs information
+- *
+- * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
+- */
+-
+-/* Author: Andrey Shvetsov <andrey.shvetsov@k2l.de> */
+-
+-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+-
+-#include <linux/kernel.h>
+-#include "sysfs.h"
+-#include <linux/device.h>
+-
+-static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+- char *buf)
+-{
+- bool state = dim2_sysfs_get_state_cb();
+-
+- return sprintf(buf, "%s\n", state ? "locked" : "");
+-}
+-
+-static DEVICE_ATTR_RO(state);
+-
+-static struct attribute *dev_attrs[] = {
+- &dev_attr_state.attr,
+- NULL,
+-};
+-
+-static struct attribute_group dev_attr_group = {
+- .attrs = dev_attrs,
+-};
+-
+-static const struct attribute_group *dev_attr_groups[] = {
+- &dev_attr_group,
+- NULL,
+-};
+-
+-int dim2_sysfs_probe(struct device *dev)
+-{
+- dev->groups = dev_attr_groups;
+- return device_register(dev);
+-}
+-
+-void dim2_sysfs_destroy(struct device *dev)
+-{
+- device_unregister(dev);
+-}
+diff --git a/drivers/staging/most/dim2/sysfs.h b/drivers/staging/most/dim2/sysfs.h
+index 24277a17cff3d..09115cf4ed00e 100644
+--- a/drivers/staging/most/dim2/sysfs.h
++++ b/drivers/staging/most/dim2/sysfs.h
+@@ -16,15 +16,4 @@ struct medialb_bus {
+ struct kobject kobj_group;
+ };
+
+-struct device;
+-
+-int dim2_sysfs_probe(struct device *dev);
+-void dim2_sysfs_destroy(struct device *dev);
+-
+-/*
+- * callback,
+- * must deliver MediaLB state as true if locked or false if unlocked
+- */
+-bool dim2_sysfs_get_state_cb(void);
+-
+ #endif /* DIM2_SYSFS_H */
+diff --git a/drivers/staging/mt7621-dts/gbpc1.dts b/drivers/staging/mt7621-dts/gbpc1.dts
+index b65d716868146..02fd9be5e1734 100644
+--- a/drivers/staging/mt7621-dts/gbpc1.dts
++++ b/drivers/staging/mt7621-dts/gbpc1.dts
+@@ -11,7 +11,8 @@
+
+ memory@0 {
+ device_type = "memory";
+- reg = <0x0 0x1c000000>, <0x20000000 0x4000000>;
++ reg = <0x00000000 0x1c000000>,
++ <0x20000000 0x04000000>;
+ };
+
+ chosen {
+@@ -37,24 +38,16 @@
+ gpio-leds {
+ compatible = "gpio-leds";
+
+- system {
+- label = "gb-pc1:green:system";
++ power {
++ label = "green:power";
+ gpios = <&gpio 6 GPIO_ACTIVE_LOW>;
++ linux,default-trigger = "default-on";
+ };
+
+- status {
+- label = "gb-pc1:green:status";
++ system {
++ label = "green:system";
+ gpios = <&gpio 8 GPIO_ACTIVE_LOW>;
+- };
+-
+- lan1 {
+- label = "gb-pc1:green:lan1";
+- gpios = <&gpio 24 GPIO_ACTIVE_LOW>;
+- };
+-
+- lan2 {
+- label = "gb-pc1:green:lan2";
+- gpios = <&gpio 25 GPIO_ACTIVE_LOW>;
++ linux,default-trigger = "disk-activity";
+ };
+ };
+ };
+@@ -94,9 +87,8 @@
+
+ partition@50000 {
+ label = "firmware";
+- reg = <0x50000 0x1FB0000>;
++ reg = <0x50000 0x1fb0000>;
+ };
+-
+ };
+ };
+
+@@ -105,9 +97,12 @@
+ };
+
+ &pinctrl {
+- state_default: pinctrl0 {
+- default_gpio: gpio {
+- groups = "wdt", "rgmii2", "uart3";
++ pinctrl-names = "default";
++ pinctrl-0 = <&state_default>;
++
++ state_default: state-default {
++ gpio-pinmux {
++ groups = "rgmii2", "uart3", "wdt";
+ function = "gpio";
+ };
+ };
+@@ -116,12 +111,13 @@
+ &switch0 {
+ ports {
+ port@0 {
++ status = "okay";
+ label = "ethblack";
+- status = "ok";
+ };
++
+ port@4 {
++ status = "okay";
+ label = "ethblue";
+- status = "ok";
+ };
+ };
+ };
+diff --git a/drivers/staging/mt7621-dts/gbpc2.dts b/drivers/staging/mt7621-dts/gbpc2.dts
+index 52760e7351f6c..6f6fed071dda0 100644
+--- a/drivers/staging/mt7621-dts/gbpc2.dts
++++ b/drivers/staging/mt7621-dts/gbpc2.dts
+@@ -1,21 +1,121 @@
+ /dts-v1/;
+
+-#include "gbpc1.dts"
++#include "mt7621.dtsi"
++
++#include <dt-bindings/gpio/gpio.h>
++#include <dt-bindings/input/input.h>
+
+ / {
+ compatible = "gnubee,gb-pc2", "mediatek,mt7621-soc";
+ model = "GB-PC2";
++
++ memory@0 {
++ device_type = "memory";
++ reg = <0x00000000 0x1c000000>,
++ <0x20000000 0x04000000>;
++ };
++
++ chosen {
++ bootargs = "console=ttyS0,57600";
++ };
++
++ palmbus: palmbus@1e000000 {
++ i2c@900 {
++ status = "okay";
++ };
++ };
++
++ gpio-keys {
++ compatible = "gpio-keys";
++
++ reset {
++ label = "reset";
++ gpios = <&gpio 18 GPIO_ACTIVE_HIGH>;
++ linux,code = <KEY_RESTART>;
++ };
++ };
++};
++
++&sdhci {
++ status = "okay";
++};
++
++&spi0 {
++ status = "okay";
++
++ m25p80@0 {
++ #address-cells = <1>;
++ #size-cells = <1>;
++ compatible = "jedec,spi-nor";
++ reg = <0>;
++ spi-max-frequency = <50000000>;
++ broken-flash-reset;
++
++ partition@0 {
++ label = "u-boot";
++ reg = <0x0 0x30000>;
++ read-only;
++ };
++
++ partition@30000 {
++ label = "u-boot-env";
++ reg = <0x30000 0x10000>;
++ read-only;
++ };
++
++ factory: partition@40000 {
++ label = "factory";
++ reg = <0x40000 0x10000>;
++ read-only;
++ };
++
++ partition@50000 {
++ label = "firmware";
++ reg = <0x50000 0x1fb0000>;
++ };
++ };
+ };
+
+-&default_gpio {
+- groups = "wdt", "uart3";
+- function = "gpio";
++&pcie {
++ status = "okay";
+ };
+
+-&gmac1 {
+- status = "ok";
++&pinctrl {
++ pinctrl-names = "default";
++ pinctrl-0 = <&state_default>;
++
++ state_default: state-default {
++ gpio-pinmux {
++ groups = "wdt";
++ function = "gpio";
++ };
++ };
+ };
+
+-&phy_external {
+- status = "ok";
++&ethernet {
++ gmac1: mac@1 {
++ status = "okay";
++ phy-handle = <&ethphy7>;
++ };
++
++ mdio-bus {
++ ethphy7: ethernet-phy@7 {
++ reg = <7>;
++ phy-mode = "rgmii-rxid";
++ };
++ };
++};
++
++&switch0 {
++ ports {
++ port@0 {
++ status = "okay";
++ label = "ethblack";
++ };
++
++ port@4 {
++ status = "okay";
++ label = "ethblue";
++ };
++ };
+ };
+diff --git a/drivers/staging/mt7621-dts/mt7621.dtsi b/drivers/staging/mt7621-dts/mt7621.dtsi
+index eeabe9c0f4fb8..59a9ce282a3b3 100644
+--- a/drivers/staging/mt7621-dts/mt7621.dtsi
++++ b/drivers/staging/mt7621-dts/mt7621.dtsi
+@@ -36,9 +36,9 @@
+ regulator-max-microvolt = <3300000>;
+ enable-active-high;
+ regulator-always-on;
+- };
++ };
+
+- mmc_fixed_1v8_io: fixedregulator@1 {
++ mmc_fixed_1v8_io: fixedregulator@1 {
+ compatible = "regulator-fixed";
+ regulator-name = "mmc_io";
+ regulator-min-microvolt = <1800000>;
+@@ -391,37 +391,32 @@
+
+ mediatek,ethsys = <&sysc>;
+
++ pinctrl-names = "default";
++ pinctrl-0 = <&mdio_pins>, <&rgmii1_pins>, <&rgmii2_pins>;
+
+ gmac0: mac@0 {
+ compatible = "mediatek,eth-mac";
+ reg = <0>;
+ phy-mode = "rgmii";
++
+ fixed-link {
+ speed = <1000>;
+ full-duplex;
+ pause;
+ };
+ };
++
+ gmac1: mac@1 {
+ compatible = "mediatek,eth-mac";
+ reg = <1>;
+ status = "off";
+ phy-mode = "rgmii-rxid";
+- phy-handle = <&phy_external>;
+ };
++
+ mdio-bus {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+- phy_external: ethernet-phy@5 {
+- status = "off";
+- reg = <5>;
+- phy-mode = "rgmii-rxid";
+-
+- pinctrl-names = "default";
+- pinctrl-0 = <&rgmii2_pins>;
+- };
+-
+ switch0: switch0@0 {
+ compatible = "mediatek,mt7621";
+ #address-cells = <1>;
+@@ -439,36 +434,43 @@
+ #address-cells = <1>;
+ #size-cells = <0>;
+ reg = <0>;
++
+ port@0 {
+ status = "off";
+ reg = <0>;
+ label = "lan0";
+ };
++
+ port@1 {
+ status = "off";
+ reg = <1>;
+ label = "lan1";
+ };
++
+ port@2 {
+ status = "off";
+ reg = <2>;
+ label = "lan2";
+ };
++
+ port@3 {
+ status = "off";
+ reg = <3>;
+ label = "lan3";
+ };
++
+ port@4 {
+ status = "off";
+ reg = <4>;
+ label = "lan4";
+ };
++
+ port@6 {
+ reg = <6>;
+ label = "cpu";
+ ethernet = <&gmac0>;
+ phy-mode = "trgmii";
++
+ fixed-link {
+ speed = <1000>;
+ full-duplex;
+diff --git a/drivers/staging/r8188eu/core/rtw_led.c b/drivers/staging/r8188eu/core/rtw_led.c
+index b33e34cce12e4..f9a8cdd9a1689 100644
+--- a/drivers/staging/r8188eu/core/rtw_led.c
++++ b/drivers/staging/r8188eu/core/rtw_led.c
+@@ -74,6 +74,7 @@ void DeInitLed871x(struct LED_871x *pLed)
+ _cancel_workitem_sync(&pLed->BlinkWorkItem);
+ _cancel_timer_ex(&pLed->BlinkTimer);
+ ResetLedStatus(pLed);
++ SwLedOff(pLed->padapter, pLed);
+ }
+
+ /* */
+diff --git a/drivers/staging/r8188eu/core/rtw_mlme.c b/drivers/staging/r8188eu/core/rtw_mlme.c
+index 1115ff5d865ad..bd991d7ed8090 100644
+--- a/drivers/staging/r8188eu/core/rtw_mlme.c
++++ b/drivers/staging/r8188eu/core/rtw_mlme.c
+@@ -1722,6 +1722,8 @@ int rtw_set_key(struct adapter *adapter, struct security_priv *psecuritypriv, in
+ psetkeyparm->grpkey = 1;
+ break;
+ default:
++ kfree(psetkeyparm);
++ kfree(pcmd);
+ res = _FAIL;
+ goto exit;
+ }
+diff --git a/drivers/staging/r8188eu/core/rtw_mlme_ext.c b/drivers/staging/r8188eu/core/rtw_mlme_ext.c
+index 5a472a4954b0f..63d312d01171e 100644
+--- a/drivers/staging/r8188eu/core/rtw_mlme_ext.c
++++ b/drivers/staging/r8188eu/core/rtw_mlme_ext.c
+@@ -104,6 +104,7 @@ static struct rt_channel_plan_map RTW_ChannelPlanMap[RT_CHANNEL_DOMAIN_MAX] = {
+ {0x01}, /* 0x10, RT_CHANNEL_DOMAIN_JAPAN */
+ {0x02}, /* 0x11, RT_CHANNEL_DOMAIN_FCC_NO_DFS */
+ {0x01}, /* 0x12, RT_CHANNEL_DOMAIN_JAPAN_NO_DFS */
++ {0x00}, /* 0x13 */
+ {0x02}, /* 0x14, RT_CHANNEL_DOMAIN_TAIWAN_NO_DFS */
+ {0x00}, /* 0x15, RT_CHANNEL_DOMAIN_ETSI_NO_DFS */
+ {0x00}, /* 0x16, RT_CHANNEL_DOMAIN_KOREA_NO_DFS */
+@@ -115,6 +116,7 @@ static struct rt_channel_plan_map RTW_ChannelPlanMap[RT_CHANNEL_DOMAIN_MAX] = {
+ {0x00}, /* 0x1C, */
+ {0x00}, /* 0x1D, */
+ {0x00}, /* 0x1E, */
++ {0x00}, /* 0x1F, */
+ /* 0x20 ~ 0x7F , New Define ===== */
+ {0x00}, /* 0x20, RT_CHANNEL_DOMAIN_WORLD_NULL */
+ {0x01}, /* 0x21, RT_CHANNEL_DOMAIN_ETSI1_NULL */
+@@ -7080,12 +7082,12 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
+ struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+ struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
+
+- pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
++ pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC);
+ if (!pcmd_obj)
+ return;
+
+ cmdsz = (sizeof(struct stadel_event) + sizeof(struct C2HEvent_Header));
+- pevtcmd = kzalloc(cmdsz, GFP_KERNEL);
++ pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
+ if (!pevtcmd) {
+ kfree(pcmd_obj);
+ return;
+diff --git a/drivers/staging/r8188eu/core/rtw_recv.c b/drivers/staging/r8188eu/core/rtw_recv.c
+index e082edfbaad82..30ca9f1e03639 100644
+--- a/drivers/staging/r8188eu/core/rtw_recv.c
++++ b/drivers/staging/r8188eu/core/rtw_recv.c
+@@ -1942,8 +1942,7 @@ static int recv_func(struct adapter *padapter, struct recv_frame *rframe)
+ struct recv_frame *pending_frame;
+ int cnt = 0;
+
+- pending_frame = rtw_alloc_recvframe(&padapter->recvpriv.uc_swdec_pending_queue);
+- while (pending_frame) {
++ while ((pending_frame = rtw_alloc_recvframe(&padapter->recvpriv.uc_swdec_pending_queue))) {
+ cnt++;
+ recv_func_posthandle(padapter, pending_frame);
+ }
+diff --git a/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c b/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c
+index 14758361960cc..9f2b86f9b6604 100644
+--- a/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c
++++ b/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c
+@@ -572,10 +572,10 @@ static int load_firmware(struct rt_firmware *pFirmware, struct device *device)
+ }
+ memcpy(pFirmware->szFwBuffer, fw->data, fw->size);
+ pFirmware->ulFwLength = fw->size;
+- release_firmware(fw);
+- DBG_88E_LEVEL(_drv_info_, "+%s: !bUsedWoWLANFw, FmrmwareLen:%d+\n", __func__, pFirmware->ulFwLength);
++ dev_dbg(device, "!bUsedWoWLANFw, FmrmwareLen:%d+\n", pFirmware->ulFwLength);
+
+ Exit:
++ release_firmware(fw);
+ return rtStatus;
+ }
+
+diff --git a/drivers/staging/r8188eu/os_dep/ioctl_linux.c b/drivers/staging/r8188eu/os_dep/ioctl_linux.c
+index 1fd3750760018..0eccce57c63a6 100644
+--- a/drivers/staging/r8188eu/os_dep/ioctl_linux.c
++++ b/drivers/staging/r8188eu/os_dep/ioctl_linux.c
+@@ -1978,7 +1978,7 @@ static int rtw_wx_set_enc_ext(struct net_device *dev,
+ struct ieee_param *param = NULL;
+ struct iw_point *pencoding = &wrqu->encoding;
+ struct iw_encode_ext *pext = (struct iw_encode_ext *)extra;
+- int ret = 0;
++ int ret = -1;
+
+ param_len = sizeof(struct ieee_param) + pext->key_len;
+ param = kzalloc(param_len, GFP_KERNEL);
+@@ -2004,7 +2004,7 @@ static int rtw_wx_set_enc_ext(struct net_device *dev,
+ alg_name = "CCMP";
+ break;
+ default:
+- return -1;
++ goto out;
+ }
+
+ strncpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN);
+@@ -2031,6 +2031,7 @@ static int rtw_wx_set_enc_ext(struct net_device *dev,
+
+ ret = wpa_set_encryption(dev, param, param_len);
+
++out:
+ kfree(param);
+ return ret;
+ }
+@@ -2060,6 +2061,7 @@ static int rtw_wx_read32(struct net_device *dev,
+ u32 data32;
+ u32 bytes;
+ u8 *ptmp;
++ int ret;
+
+ padapter = (struct adapter *)rtw_netdev_priv(dev);
+ p = &wrqu->data;
+@@ -2092,12 +2094,17 @@ static int rtw_wx_read32(struct net_device *dev,
+ break;
+ default:
+ DBG_88E(KERN_INFO "%s: usage> read [bytes],[address(hex)]\n", __func__);
+- return -EINVAL;
++ ret = -EINVAL;
++ goto err_free_ptmp;
+ }
+ DBG_88E(KERN_INFO "%s: addr = 0x%08X data =%s\n", __func__, addr, extra);
+
+ kfree(ptmp);
+ return 0;
++
++err_free_ptmp:
++ kfree(ptmp);
++ return ret;
+ }
+
+ static int rtw_wx_write32(struct net_device *dev,
+diff --git a/drivers/staging/r8188eu/os_dep/mlme_linux.c b/drivers/staging/r8188eu/os_dep/mlme_linux.c
+index e3ee9dc7ab900..b0d1e20edc4c2 100644
+--- a/drivers/staging/r8188eu/os_dep/mlme_linux.c
++++ b/drivers/staging/r8188eu/os_dep/mlme_linux.c
+@@ -114,7 +114,7 @@ void rtw_report_sec_ie(struct adapter *adapter, u8 authmode, u8 *sec_ie)
+
+ buff = NULL;
+ if (authmode == _WPA_IE_ID_) {
+- buff = kzalloc(IW_CUSTOM_MAX, GFP_KERNEL);
++ buff = kzalloc(IW_CUSTOM_MAX, GFP_ATOMIC);
+ if (!buff)
+ return;
+ p = buff;
+diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+index a7dd1578b2c6a..616ab3c8fde4f 100644
+--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
++++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+@@ -2549,13 +2549,14 @@ static void _rtl92e_pci_disconnect(struct pci_dev *pdev)
+ free_irq(dev->irq, dev);
+ priv->irq = 0;
+ }
+- free_rtllib(dev);
+
+ if (dev->mem_start != 0) {
+ iounmap((void __iomem *)dev->mem_start);
+ release_mem_region(pci_resource_start(pdev, 1),
+ pci_resource_len(pdev, 1));
+ }
++
++ free_rtllib(dev);
+ }
+
+ pci_disable_device(pdev);
+diff --git a/drivers/staging/rtl8192e/rtllib.h b/drivers/staging/rtl8192e/rtllib.h
+index c6f8b772335c1..c985e4ebc545a 100644
+--- a/drivers/staging/rtl8192e/rtllib.h
++++ b/drivers/staging/rtl8192e/rtllib.h
+@@ -1980,7 +1980,7 @@ void SendDisassociation(struct rtllib_device *ieee, bool deauth, u16 asRsn);
+ void rtllib_softmac_xmit(struct rtllib_txb *txb, struct rtllib_device *ieee);
+
+ void rtllib_start_ibss(struct rtllib_device *ieee);
+-void rtllib_softmac_init(struct rtllib_device *ieee);
++int rtllib_softmac_init(struct rtllib_device *ieee);
+ void rtllib_softmac_free(struct rtllib_device *ieee);
+ void rtllib_disassociate(struct rtllib_device *ieee);
+ void rtllib_stop_scan(struct rtllib_device *ieee);
+diff --git a/drivers/staging/rtl8192e/rtllib_module.c b/drivers/staging/rtl8192e/rtllib_module.c
+index 64d9feee1f392..f00ac94b2639b 100644
+--- a/drivers/staging/rtl8192e/rtllib_module.c
++++ b/drivers/staging/rtl8192e/rtllib_module.c
+@@ -88,7 +88,7 @@ struct net_device *alloc_rtllib(int sizeof_priv)
+ err = rtllib_networks_allocate(ieee);
+ if (err) {
+ pr_err("Unable to allocate beacon storage: %d\n", err);
+- goto failed;
++ goto free_netdev;
+ }
+ rtllib_networks_initialize(ieee);
+
+@@ -121,11 +121,13 @@ struct net_device *alloc_rtllib(int sizeof_priv)
+ ieee->hwsec_active = 0;
+
+ memset(ieee->swcamtable, 0, sizeof(struct sw_cam_table) * 32);
+- rtllib_softmac_init(ieee);
++ err = rtllib_softmac_init(ieee);
++ if (err)
++ goto free_crypt_info;
+
+ ieee->pHTInfo = kzalloc(sizeof(struct rt_hi_throughput), GFP_KERNEL);
+ if (!ieee->pHTInfo)
+- return NULL;
++ goto free_softmac;
+
+ HTUpdateDefaultSetting(ieee);
+ HTInitializeHTInfo(ieee);
+@@ -141,8 +143,14 @@ struct net_device *alloc_rtllib(int sizeof_priv)
+
+ return dev;
+
+- failed:
++free_softmac:
++ rtllib_softmac_free(ieee);
++free_crypt_info:
++ lib80211_crypt_info_free(&ieee->crypt_info);
++ rtllib_networks_free(ieee);
++free_netdev:
+ free_netdev(dev);
++
+ return NULL;
+ }
+ EXPORT_SYMBOL(alloc_rtllib);
+diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c
+index d2726d01c7573..503d33be71d99 100644
+--- a/drivers/staging/rtl8192e/rtllib_softmac.c
++++ b/drivers/staging/rtl8192e/rtllib_softmac.c
+@@ -2952,7 +2952,7 @@ void rtllib_start_protocol(struct rtllib_device *ieee)
+ }
+ }
+
+-void rtllib_softmac_init(struct rtllib_device *ieee)
++int rtllib_softmac_init(struct rtllib_device *ieee)
+ {
+ int i;
+
+@@ -2963,7 +2963,8 @@ void rtllib_softmac_init(struct rtllib_device *ieee)
+ ieee->seq_ctrl[i] = 0;
+ ieee->dot11d_info = kzalloc(sizeof(struct rt_dot11d_info), GFP_ATOMIC);
+ if (!ieee->dot11d_info)
+- netdev_err(ieee->dev, "Can't alloc memory for DOT11D\n");
++ return -ENOMEM;
++
+ ieee->LinkDetectInfo.SlotIndex = 0;
+ ieee->LinkDetectInfo.SlotNum = 2;
+ ieee->LinkDetectInfo.NumRecvBcnInPeriod = 0;
+@@ -3029,6 +3030,7 @@ void rtllib_softmac_init(struct rtllib_device *ieee)
+
+ tasklet_setup(&ieee->ps_task, rtllib_sta_ps);
+
++ return 0;
+ }
+
+ void rtllib_softmac_free(struct rtllib_device *ieee)
+diff --git a/drivers/staging/rtl8192u/r8192U_core.c b/drivers/staging/rtl8192u/r8192U_core.c
+index b6698656fc014..cf5cfee2936fd 100644
+--- a/drivers/staging/rtl8192u/r8192U_core.c
++++ b/drivers/staging/rtl8192u/r8192U_core.c
+@@ -229,7 +229,7 @@ int write_nic_byte_E(struct net_device *dev, int indx, u8 data)
+
+ status = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+ RTL8187_REQ_SET_REGS, RTL8187_REQT_WRITE,
+- indx | 0xfe00, 0, usbdata, 1, HZ / 2);
++ indx | 0xfe00, 0, usbdata, 1, 500);
+ kfree(usbdata);
+
+ if (status < 0) {
+@@ -251,7 +251,7 @@ int read_nic_byte_E(struct net_device *dev, int indx, u8 *data)
+
+ status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ RTL8187_REQ_GET_REGS, RTL8187_REQT_READ,
+- indx | 0xfe00, 0, usbdata, 1, HZ / 2);
++ indx | 0xfe00, 0, usbdata, 1, 500);
+ *data = *usbdata;
+ kfree(usbdata);
+
+@@ -279,7 +279,7 @@ int write_nic_byte(struct net_device *dev, int indx, u8 data)
+ status = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+ RTL8187_REQ_SET_REGS, RTL8187_REQT_WRITE,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 1, HZ / 2);
++ usbdata, 1, 500);
+ kfree(usbdata);
+
+ if (status < 0) {
+@@ -305,7 +305,7 @@ int write_nic_word(struct net_device *dev, int indx, u16 data)
+ status = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+ RTL8187_REQ_SET_REGS, RTL8187_REQT_WRITE,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 2, HZ / 2);
++ usbdata, 2, 500);
+ kfree(usbdata);
+
+ if (status < 0) {
+@@ -331,7 +331,7 @@ int write_nic_dword(struct net_device *dev, int indx, u32 data)
+ status = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+ RTL8187_REQ_SET_REGS, RTL8187_REQT_WRITE,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 4, HZ / 2);
++ usbdata, 4, 500);
+ kfree(usbdata);
+
+ if (status < 0) {
+@@ -355,7 +355,7 @@ int read_nic_byte(struct net_device *dev, int indx, u8 *data)
+ status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ RTL8187_REQ_GET_REGS, RTL8187_REQT_READ,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 1, HZ / 2);
++ usbdata, 1, 500);
+ *data = *usbdata;
+ kfree(usbdata);
+
+@@ -380,7 +380,7 @@ int read_nic_word(struct net_device *dev, int indx, u16 *data)
+ status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ RTL8187_REQ_GET_REGS, RTL8187_REQT_READ,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 2, HZ / 2);
++ usbdata, 2, 500);
+ *data = *usbdata;
+ kfree(usbdata);
+
+@@ -404,7 +404,7 @@ static int read_nic_word_E(struct net_device *dev, int indx, u16 *data)
+
+ status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ RTL8187_REQ_GET_REGS, RTL8187_REQT_READ,
+- indx | 0xfe00, 0, usbdata, 2, HZ / 2);
++ indx | 0xfe00, 0, usbdata, 2, 500);
+ *data = *usbdata;
+ kfree(usbdata);
+
+@@ -430,7 +430,7 @@ int read_nic_dword(struct net_device *dev, int indx, u32 *data)
+ status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+ RTL8187_REQ_GET_REGS, RTL8187_REQT_READ,
+ (indx & 0xff) | 0xff00, (indx >> 8) & 0x0f,
+- usbdata, 4, HZ / 2);
++ usbdata, 4, 500);
+ *data = *usbdata;
+ kfree(usbdata);
+
+diff --git a/drivers/staging/rtl8712/usb_intf.c b/drivers/staging/rtl8712/usb_intf.c
+index 505ebeb643dc2..cae04272deffe 100644
+--- a/drivers/staging/rtl8712/usb_intf.c
++++ b/drivers/staging/rtl8712/usb_intf.c
+@@ -595,12 +595,12 @@ static void r871xu_dev_remove(struct usb_interface *pusb_intf)
+
+ /* never exit with a firmware callback pending */
+ wait_for_completion(&padapter->rtl8712_fw_ready);
++ if (pnetdev->reg_state != NETREG_UNINITIALIZED)
++ unregister_netdev(pnetdev); /* will call netdev_close() */
+ usb_set_intfdata(pusb_intf, NULL);
+ release_firmware(padapter->fw);
+ if (drvpriv.drv_registered)
+ padapter->surprise_removed = true;
+- if (pnetdev->reg_state != NETREG_UNINITIALIZED)
+- unregister_netdev(pnetdev); /* will call netdev_close() */
+ r8712_flush_rwctrl_works(padapter);
+ r8712_flush_led_works(padapter);
+ udelay(1);
+diff --git a/drivers/staging/rtl8712/usb_ops_linux.c b/drivers/staging/rtl8712/usb_ops_linux.c
+index 655497cead122..f984a5ab2c6ff 100644
+--- a/drivers/staging/rtl8712/usb_ops_linux.c
++++ b/drivers/staging/rtl8712/usb_ops_linux.c
+@@ -494,7 +494,7 @@ int r8712_usbctrl_vendorreq(struct intf_priv *pintfpriv, u8 request, u16 value,
+ memcpy(pIo_buf, pdata, len);
+ }
+ status = usb_control_msg(udev, pipe, request, reqtype, value, index,
+- pIo_buf, len, HZ / 2);
++ pIo_buf, len, 500);
+ if (status > 0) { /* Success this control transfer. */
+ if (requesttype == 0x01) {
+ /* For Control read transfer, we have to copy the read
+diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
+index ab6a24d70cc96..cf79bec916c51 100644
+--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
++++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
+@@ -897,7 +897,6 @@ void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue)
+ {
+ struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+ struct wlan_network *tgt_network = &pmlmepriv->cur_network;
+- struct sta_priv *pstapriv = &adapter->stapriv;
+ struct dvobj_priv *psdpriv = adapter->dvobj;
+ struct debug_priv *pdbgpriv = &psdpriv->drv_dbg;
+
+@@ -905,11 +904,7 @@ void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue)
+ struct sta_info *psta;
+
+ psta = rtw_get_stainfo(&adapter->stapriv, tgt_network->network.mac_address);
+- spin_lock_bh(&(pstapriv->sta_hash_lock));
+ rtw_free_stainfo(adapter, psta);
+-
+- spin_unlock_bh(&(pstapriv->sta_hash_lock));
+-
+ }
+
+ if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE|WIFI_AP_STATE)) {
+@@ -1239,16 +1234,13 @@ void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
+ rtw_indicate_connect(adapter);
+ }
+
++ spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
++
+ /* s5. Cancel assoc_timer */
+ del_timer_sync(&pmlmepriv->assoc_timer);
+-
+ } else {
+ spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
+- goto ignore_joinbss_callback;
+ }
+-
+- spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
+-
+ } else if (pnetwork->join_res == -4) {
+ rtw_reset_securitypriv(adapter);
+ _set_timer(&pmlmepriv->assoc_timer, 1);
+diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+index 375d2a742dd2d..1a4b4c75c4bf5 100644
+--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
++++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+@@ -1489,9 +1489,7 @@ unsigned int OnDeAuth(struct adapter *padapter, union recv_frame *precv_frame)
+ struct sta_info *psta;
+ struct sta_priv *pstapriv = &padapter->stapriv;
+
+- /* spin_lock_bh(&(pstapriv->sta_hash_lock)); */
+ /* rtw_free_stainfo(padapter, psta); */
+- /* spin_unlock_bh(&(pstapriv->sta_hash_lock)); */
+
+ netdev_dbg(padapter->pnetdev,
+ "ap recv deauth reason code(%d) sta:%pM\n", reason,
+@@ -1565,9 +1563,7 @@ unsigned int OnDisassoc(struct adapter *padapter, union recv_frame *precv_frame)
+ struct sta_info *psta;
+ struct sta_priv *pstapriv = &padapter->stapriv;
+
+- /* spin_lock_bh(&(pstapriv->sta_hash_lock)); */
+ /* rtw_free_stainfo(padapter, psta); */
+- /* spin_unlock_bh(&(pstapriv->sta_hash_lock)); */
+
+ netdev_dbg(padapter->pnetdev,
+ "ap recv disassoc reason code(%d) sta:%pM\n",
+diff --git a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+index 67ca219f95bf8..5eae3ccb1ff59 100644
+--- a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
++++ b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+@@ -263,7 +263,6 @@ exit:
+ return psta;
+ }
+
+-/* using pstapriv->sta_hash_lock to protect */
+ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
+ {
+ int i;
+@@ -332,8 +331,10 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
+
+ spin_unlock_bh(&pxmitpriv->lock);
+
++ spin_lock_bh(&pstapriv->sta_hash_lock);
+ list_del_init(&psta->hash_list);
+ pstapriv->asoc_sta_count--;
++ spin_unlock_bh(&pstapriv->sta_hash_lock);
+
+ /* re-init sta_info; 20061114 will be init in alloc_stainfo */
+ /* _rtw_init_sta_xmit_priv(&psta->sta_xmitpriv); */
+@@ -428,6 +429,7 @@ void rtw_free_all_stainfo(struct adapter *padapter)
+ struct sta_info *psta = NULL;
+ struct sta_priv *pstapriv = &padapter->stapriv;
+ struct sta_info *pbcmc_stainfo = rtw_get_bcmc_stainfo(padapter);
++ LIST_HEAD(stainfo_free_list);
+
+ if (pstapriv->asoc_sta_count == 1)
+ return;
+@@ -440,11 +442,16 @@ void rtw_free_all_stainfo(struct adapter *padapter)
+ psta = list_entry(plist, struct sta_info, hash_list);
+
+ if (pbcmc_stainfo != psta)
+- rtw_free_stainfo(padapter, psta);
++ list_move(&psta->hash_list, &stainfo_free_list);
+ }
+ }
+
+ spin_unlock_bh(&pstapriv->sta_hash_lock);
++
++ list_for_each_safe(plist, tmp, &stainfo_free_list) {
++ psta = list_entry(plist, struct sta_info, hash_list);
++ rtw_free_stainfo(padapter, psta);
++ }
+ }
+
+ /* any station allocated can be searched by hash list */
+diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
+index 9d4a233a861e3..295121c268bd4 100644
+--- a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
++++ b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
+@@ -835,9 +835,7 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
+ psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+ if (psta)
+ {
+- spin_lock_bh(&(pstapriv->sta_hash_lock));
+ rtw_free_stainfo(padapter, psta);
+- spin_unlock_bh(&(pstapriv->sta_hash_lock));
+
+ psta = NULL;
+ }
+diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
+index 967f10b9582a8..099359fc01152 100644
+--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
++++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
+@@ -1033,15 +1033,27 @@ service_callback(enum vchiq_reason reason, struct vchiq_header *header,
+
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+
++ rcu_read_lock();
+ service = handle_to_service(handle);
+- if (WARN_ON(!service))
++ if (WARN_ON(!service)) {
++ rcu_read_unlock();
+ return VCHIQ_SUCCESS;
++ }
+
+ user_service = (struct user_service *)service->base.userdata;
+ instance = user_service->instance;
+
+- if (!instance || instance->closing)
++ if (!instance || instance->closing) {
++ rcu_read_unlock();
+ return VCHIQ_SUCCESS;
++ }
++
++ /*
++ * As hopping around different synchronization mechanism,
++ * taking an extra reference results in simpler implementation.
++ */
++ vchiq_service_get(service);
++ rcu_read_unlock();
+
+ vchiq_log_trace(vchiq_arm_log_level,
+ "%s - service %lx(%d,%p), reason %d, header %lx, instance %lx, bulk_userdata %lx",
+@@ -1074,6 +1086,7 @@ service_callback(enum vchiq_reason reason, struct vchiq_header *header,
+ NULL, user_service, bulk_userdata);
+ if (status != VCHIQ_SUCCESS) {
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
++ vchiq_service_put(service);
+ return status;
+ }
+ }
+@@ -1084,11 +1097,13 @@ service_callback(enum vchiq_reason reason, struct vchiq_header *header,
+ vchiq_log_info(vchiq_arm_log_level,
+ "%s interrupted", __func__);
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
++ vchiq_service_put(service);
+ return VCHIQ_RETRY;
+ } else if (instance->closing) {
+ vchiq_log_info(vchiq_arm_log_level,
+ "%s closing", __func__);
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
++ vchiq_service_put(service);
+ return VCHIQ_ERROR;
+ }
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+@@ -1117,6 +1132,7 @@ service_callback(enum vchiq_reason reason, struct vchiq_header *header,
+ header = NULL;
+ }
+ DEBUG_TRACE(SERVICE_CALLBACK_LINE);
++ vchiq_service_put(service);
+
+ if (skip_completion)
+ return VCHIQ_SUCCESS;
+@@ -1173,6 +1189,9 @@ int vchiq_dump_platform_instances(void *dump_context)
+ int len;
+ int i;
+
++ if (!state)
++ return -ENOTCONN;
++
+ /*
+ * There is no list of instances, so instead scan all services,
+ * marking those that have been dumped.
+diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
+index 9429b8a642fbb..630ed0dc24c39 100644
+--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
++++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
+@@ -2421,6 +2421,9 @@ void vchiq_msg_queue_push(unsigned int handle, struct vchiq_header *header)
+ struct vchiq_service *service = find_service_by_handle(handle);
+ int pos;
+
++ if (!service)
++ return;
++
+ while (service->msg_queue_write == service->msg_queue_read +
+ VCHIQ_MAX_SLOTS) {
+ if (wait_for_completion_interruptible(&service->msg_queue_pop))
+@@ -2441,6 +2444,9 @@ struct vchiq_header *vchiq_msg_hold(unsigned int handle)
+ struct vchiq_header *header;
+ int pos;
+
++ if (!service)
++ return NULL;
++
+ if (service->msg_queue_write == service->msg_queue_read)
+ return NULL;
+
+diff --git a/drivers/staging/wfx/bus_sdio.c b/drivers/staging/wfx/bus_sdio.c
+index e06d7e1ebe9c3..61b8cc05f2935 100644
+--- a/drivers/staging/wfx/bus_sdio.c
++++ b/drivers/staging/wfx/bus_sdio.c
+@@ -120,19 +120,22 @@ static int wfx_sdio_irq_subscribe(void *priv)
+ return ret;
+ }
+
++ flags = irq_get_trigger_type(bus->of_irq);
++ if (!flags)
++ flags = IRQF_TRIGGER_HIGH;
++ flags |= IRQF_ONESHOT;
++ ret = devm_request_threaded_irq(&bus->func->dev, bus->of_irq, NULL,
++ wfx_sdio_irq_handler_ext, flags,
++ "wfx", bus);
++ if (ret)
++ return ret;
+ sdio_claim_host(bus->func);
+ cccr = sdio_f0_readb(bus->func, SDIO_CCCR_IENx, NULL);
+ cccr |= BIT(0);
+ cccr |= BIT(bus->func->num);
+ sdio_f0_writeb(bus->func, cccr, SDIO_CCCR_IENx, NULL);
+ sdio_release_host(bus->func);
+- flags = irq_get_trigger_type(bus->of_irq);
+- if (!flags)
+- flags = IRQF_TRIGGER_HIGH;
+- flags |= IRQF_ONESHOT;
+- return devm_request_threaded_irq(&bus->func->dev, bus->of_irq, NULL,
+- wfx_sdio_irq_handler_ext, flags,
+- "wfx", bus);
++ return 0;
+ }
+
+ static int wfx_sdio_irq_unsubscribe(void *priv)
+diff --git a/drivers/staging/wfx/main.c b/drivers/staging/wfx/main.c
+index 4b9fdf99981b1..9ff69c5e0ae97 100644
+--- a/drivers/staging/wfx/main.c
++++ b/drivers/staging/wfx/main.c
+@@ -309,7 +309,8 @@ struct wfx_dev *wfx_init_common(struct device *dev,
+ wdev->pdata.gpio_wakeup = devm_gpiod_get_optional(dev, "wakeup",
+ GPIOD_OUT_LOW);
+ if (IS_ERR(wdev->pdata.gpio_wakeup))
+- return NULL;
++ goto err;
++
+ if (wdev->pdata.gpio_wakeup)
+ gpiod_set_consumer_name(wdev->pdata.gpio_wakeup, "wfx wakeup");
+
+@@ -328,6 +329,10 @@ struct wfx_dev *wfx_init_common(struct device *dev,
+ return NULL;
+
+ return wdev;
++
++err:
++ ieee80211_free_hw(hw);
++ return NULL;
+ }
+
+ int wfx_probe(struct wfx_dev *wdev)
+diff --git a/drivers/staging/wlan-ng/hfa384x_usb.c b/drivers/staging/wlan-ng/hfa384x_usb.c
+index 8c8524679ba38..0d869b5e309c0 100644
+--- a/drivers/staging/wlan-ng/hfa384x_usb.c
++++ b/drivers/staging/wlan-ng/hfa384x_usb.c
+@@ -3778,18 +3778,18 @@ static void hfa384x_usb_throttlefn(struct timer_list *t)
+
+ spin_lock_irqsave(&hw->ctlxq.lock, flags);
+
+- /*
+- * We need to check BOTH the RX and the TX throttle controls,
+- * so we use the bitwise OR instead of the logical OR.
+- */
+ pr_debug("flags=0x%lx\n", hw->usb_flags);
+- if (!hw->wlandev->hwremoved &&
+- ((test_and_clear_bit(THROTTLE_RX, &hw->usb_flags) &&
+- !test_and_set_bit(WORK_RX_RESUME, &hw->usb_flags)) |
+- (test_and_clear_bit(THROTTLE_TX, &hw->usb_flags) &&
+- !test_and_set_bit(WORK_TX_RESUME, &hw->usb_flags))
+- )) {
+- schedule_work(&hw->usb_work);
++ if (!hw->wlandev->hwremoved) {
++ bool rx_throttle = test_and_clear_bit(THROTTLE_RX, &hw->usb_flags) &&
++ !test_and_set_bit(WORK_RX_RESUME, &hw->usb_flags);
++ bool tx_throttle = test_and_clear_bit(THROTTLE_TX, &hw->usb_flags) &&
++ !test_and_set_bit(WORK_TX_RESUME, &hw->usb_flags);
++ /*
++ * We need to check BOTH the RX and the TX throttle controls,
++ * so we use the bitwise OR instead of the logical OR.
++ */
++ if (rx_throttle | tx_throttle)
++ schedule_work(&hw->usb_work);
+ }
+
+ spin_unlock_irqrestore(&hw->ctlxq.lock, flags);
+diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
+index 8075f60fd02c3..2d5cf1714ae05 100644
+--- a/drivers/target/iscsi/iscsi_target_tpg.c
++++ b/drivers/target/iscsi/iscsi_target_tpg.c
+@@ -443,6 +443,9 @@ static bool iscsit_tpg_check_network_portal(
+ break;
+ }
+ spin_unlock(&tpg->tpg_np_lock);
++
++ if (match)
++ break;
+ }
+ spin_unlock(&tiqn->tiqn_tpg_lock);
+
+diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
+index cb1de1ecaaa61..bd0f2ce011dd7 100644
+--- a/drivers/target/target_core_alua.c
++++ b/drivers/target/target_core_alua.c
+@@ -1674,7 +1674,6 @@ int core_alua_set_tg_pt_gp_id(
+ pr_err("Maximum ALUA alua_tg_pt_gps_count:"
+ " 0x0000ffff reached\n");
+ spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
+- kmem_cache_free(t10_alua_tg_pt_gp_cache, tg_pt_gp);
+ return -ENOSPC;
+ }
+ again:
+diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
+index 8cb1fa0c05857..44bb380e7390c 100644
+--- a/drivers/target/target_core_device.c
++++ b/drivers/target/target_core_device.c
+@@ -772,6 +772,8 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
+ INIT_LIST_HEAD(&dev->t10_alua.lba_map_list);
+ spin_lock_init(&dev->t10_alua.lba_map_lock);
+
++ INIT_WORK(&dev->delayed_cmd_work, target_do_delayed_work);
++
+ dev->t10_wwn.t10_dev = dev;
+ /*
+ * Use OpenFabrics IEEE Company ID: 00 14 05
+diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h
+index a343bcfa2180f..a889a6237d9c1 100644
+--- a/drivers/target/target_core_internal.h
++++ b/drivers/target/target_core_internal.h
+@@ -151,6 +151,7 @@ int transport_dump_vpd_ident(struct t10_vpd *, unsigned char *, int);
+ void transport_clear_lun_ref(struct se_lun *);
+ sense_reason_t target_cmd_size_check(struct se_cmd *cmd, unsigned int size);
+ void target_qf_do_work(struct work_struct *work);
++void target_do_delayed_work(struct work_struct *work);
+ bool target_check_wce(struct se_device *dev);
+ bool target_check_fua(struct se_device *dev);
+ void __target_execute_cmd(struct se_cmd *, bool);
+diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
+index e7fcbc09f9dbc..bac111456fa1d 100644
+--- a/drivers/target/target_core_tmr.c
++++ b/drivers/target/target_core_tmr.c
+@@ -50,15 +50,6 @@ EXPORT_SYMBOL(core_tmr_alloc_req);
+
+ void core_tmr_release_req(struct se_tmr_req *tmr)
+ {
+- struct se_device *dev = tmr->tmr_dev;
+- unsigned long flags;
+-
+- if (dev) {
+- spin_lock_irqsave(&dev->se_tmr_lock, flags);
+- list_del_init(&tmr->tmr_list);
+- spin_unlock_irqrestore(&dev->se_tmr_lock, flags);
+- }
+-
+ kfree(tmr);
+ }
+
+@@ -156,13 +147,6 @@ void core_tmr_abort_task(
+ se_cmd->state_active = false;
+ spin_unlock_irqrestore(&dev->queues[i].lock, flags);
+
+- /*
+- * Ensure that this ABORT request is visible to the LU
+- * RESET code.
+- */
+- if (!tmr->tmr_dev)
+- WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd) < 0);
+-
+ if (dev->transport->tmr_notify)
+ dev->transport->tmr_notify(dev, TMR_ABORT_TASK,
+ &aborted_list);
+@@ -234,6 +218,7 @@ static void core_tmr_drain_tmr_list(
+ }
+
+ list_move_tail(&tmr_p->tmr_list, &drain_tmr_list);
++ tmr_p->tmr_dev = NULL;
+ }
+ spin_unlock_irqrestore(&dev->se_tmr_lock, flags);
+
+diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
+index 14c6f2bb1b01d..72edf5bd75ee6 100644
+--- a/drivers/target/target_core_transport.c
++++ b/drivers/target/target_core_transport.c
+@@ -676,6 +676,21 @@ static void target_remove_from_state_list(struct se_cmd *cmd)
+ spin_unlock_irqrestore(&dev->queues[cmd->cpuid].lock, flags);
+ }
+
++static void target_remove_from_tmr_list(struct se_cmd *cmd)
++{
++ struct se_device *dev = NULL;
++ unsigned long flags;
++
++ if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
++ dev = cmd->se_tmr_req->tmr_dev;
++
++ if (dev) {
++ spin_lock_irqsave(&dev->se_tmr_lock, flags);
++ if (cmd->se_tmr_req->tmr_dev)
++ list_del_init(&cmd->se_tmr_req->tmr_list);
++ spin_unlock_irqrestore(&dev->se_tmr_lock, flags);
++ }
++}
+ /*
+ * This function is called by the target core after the target core has
+ * finished processing a SCSI command or SCSI TMF. Both the regular command
+@@ -687,13 +702,6 @@ static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
+ {
+ unsigned long flags;
+
+- target_remove_from_state_list(cmd);
+-
+- /*
+- * Clear struct se_cmd->se_lun before the handoff to FE.
+- */
+- cmd->se_lun = NULL;
+-
+ spin_lock_irqsave(&cmd->t_state_lock, flags);
+ /*
+ * Determine if frontend context caller is requesting the stopping of
+@@ -728,8 +736,16 @@ static void transport_lun_remove_cmd(struct se_cmd *cmd)
+ if (!lun)
+ return;
+
++ target_remove_from_state_list(cmd);
++ target_remove_from_tmr_list(cmd);
++
+ if (cmpxchg(&cmd->lun_ref_active, true, false))
+ percpu_ref_put(&lun->lun_ref);
++
++ /*
++ * Clear struct se_cmd->se_lun before the handoff to FE.
++ */
++ cmd->se_lun = NULL;
+ }
+
+ static void target_complete_failure_work(struct work_struct *work)
+@@ -2173,32 +2189,35 @@ static bool target_handle_task_attr(struct se_cmd *cmd)
+ */
+ switch (cmd->sam_task_attr) {
+ case TCM_HEAD_TAG:
++ atomic_inc_mb(&dev->non_ordered);
+ pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
+ cmd->t_task_cdb[0]);
+ return false;
+ case TCM_ORDERED_TAG:
+- atomic_inc_mb(&dev->dev_ordered_sync);
++ atomic_inc_mb(&dev->delayed_cmd_count);
+
+ pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
+ cmd->t_task_cdb[0]);
+-
+- /*
+- * Execute an ORDERED command if no other older commands
+- * exist that need to be completed first.
+- */
+- if (!atomic_read(&dev->simple_cmds))
+- return false;
+ break;
+ default:
+ /*
+ * For SIMPLE and UNTAGGED Task Attribute commands
+ */
+- atomic_inc_mb(&dev->simple_cmds);
++ atomic_inc_mb(&dev->non_ordered);
++
++ if (atomic_read(&dev->delayed_cmd_count) == 0)
++ return false;
+ break;
+ }
+
+- if (atomic_read(&dev->dev_ordered_sync) == 0)
+- return false;
++ if (cmd->sam_task_attr != TCM_ORDERED_TAG) {
++ atomic_inc_mb(&dev->delayed_cmd_count);
++ /*
++ * We will account for this when we dequeue from the delayed
++ * list.
++ */
++ atomic_dec_mb(&dev->non_ordered);
++ }
+
+ spin_lock(&dev->delayed_cmd_lock);
+ list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
+@@ -2206,6 +2225,12 @@ static bool target_handle_task_attr(struct se_cmd *cmd)
+
+ pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
+ cmd->t_task_cdb[0], cmd->sam_task_attr);
++ /*
++ * We may have no non ordered cmds when this function started or we
++ * could have raced with the last simple/head cmd completing, so kick
++ * the delayed handler here.
++ */
++ schedule_work(&dev->delayed_cmd_work);
+ return true;
+ }
+
+@@ -2243,29 +2268,48 @@ EXPORT_SYMBOL(target_execute_cmd);
+ * Process all commands up to the last received ORDERED task attribute which
+ * requires another blocking boundary
+ */
+-static void target_restart_delayed_cmds(struct se_device *dev)
++void target_do_delayed_work(struct work_struct *work)
+ {
+- for (;;) {
++ struct se_device *dev = container_of(work, struct se_device,
++ delayed_cmd_work);
++
++ spin_lock(&dev->delayed_cmd_lock);
++ while (!dev->ordered_sync_in_progress) {
+ struct se_cmd *cmd;
+
+- spin_lock(&dev->delayed_cmd_lock);
+- if (list_empty(&dev->delayed_cmd_list)) {
+- spin_unlock(&dev->delayed_cmd_lock);
++ if (list_empty(&dev->delayed_cmd_list))
+ break;
+- }
+
+ cmd = list_entry(dev->delayed_cmd_list.next,
+ struct se_cmd, se_delayed_node);
++
++ if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
++ /*
++ * Check if we started with:
++ * [ordered] [simple] [ordered]
++ * and we are now at the last ordered so we have to wait
++ * for the simple cmd.
++ */
++ if (atomic_read(&dev->non_ordered) > 0)
++ break;
++
++ dev->ordered_sync_in_progress = true;
++ }
++
+ list_del(&cmd->se_delayed_node);
++ atomic_dec_mb(&dev->delayed_cmd_count);
+ spin_unlock(&dev->delayed_cmd_lock);
+
++ if (cmd->sam_task_attr != TCM_ORDERED_TAG)
++ atomic_inc_mb(&dev->non_ordered);
++
+ cmd->transport_state |= CMD_T_SENT;
+
+ __target_execute_cmd(cmd, true);
+
+- if (cmd->sam_task_attr == TCM_ORDERED_TAG)
+- break;
++ spin_lock(&dev->delayed_cmd_lock);
+ }
++ spin_unlock(&dev->delayed_cmd_lock);
+ }
+
+ /*
+@@ -2283,14 +2327,17 @@ static void transport_complete_task_attr(struct se_cmd *cmd)
+ goto restart;
+
+ if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
+- atomic_dec_mb(&dev->simple_cmds);
++ atomic_dec_mb(&dev->non_ordered);
+ dev->dev_cur_ordered_id++;
+ } else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
++ atomic_dec_mb(&dev->non_ordered);
+ dev->dev_cur_ordered_id++;
+ pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
+ dev->dev_cur_ordered_id);
+ } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
+- atomic_dec_mb(&dev->dev_ordered_sync);
++ spin_lock(&dev->delayed_cmd_lock);
++ dev->ordered_sync_in_progress = false;
++ spin_unlock(&dev->delayed_cmd_lock);
+
+ dev->dev_cur_ordered_id++;
+ pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
+@@ -2299,7 +2346,8 @@ static void transport_complete_task_attr(struct se_cmd *cmd)
+ cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET;
+
+ restart:
+- target_restart_delayed_cmds(dev);
++ if (atomic_read(&dev->delayed_cmd_count) > 0)
++ schedule_work(&dev->delayed_cmd_work);
+ }
+
+ static void transport_complete_qf(struct se_cmd *cmd)
+diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
+index 9f552f48084cd..0ca5ec14d3dbe 100644
+--- a/drivers/target/target_core_user.c
++++ b/drivers/target/target_core_user.c
+@@ -1821,6 +1821,7 @@ static struct page *tcmu_try_get_data_page(struct tcmu_dev *udev, uint32_t dpi)
+ mutex_lock(&udev->cmdr_lock);
+ page = xa_load(&udev->data_pages, dpi);
+ if (likely(page)) {
++ get_page(page);
+ mutex_unlock(&udev->cmdr_lock);
+ return page;
+ }
+@@ -1877,6 +1878,7 @@ static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
+ /* For the vmalloc()ed cmd area pages */
+ addr = (void *)(unsigned long)info->mem[mi].addr + offset;
+ page = vmalloc_to_page(addr);
++ get_page(page);
+ } else {
+ uint32_t dpi;
+
+@@ -1887,7 +1889,6 @@ static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
+ return VM_FAULT_SIGBUS;
+ }
+
+- get_page(page);
+ vmf->page = page;
+ return 0;
+ }
+diff --git a/drivers/tee/amdtee/core.c b/drivers/tee/amdtee/core.c
+index da6b88e80dc07..297dc62bca298 100644
+--- a/drivers/tee/amdtee/core.c
++++ b/drivers/tee/amdtee/core.c
+@@ -203,9 +203,8 @@ static int copy_ta_binary(struct tee_context *ctx, void *ptr, void **ta,
+
+ *ta_size = roundup(fw->size, PAGE_SIZE);
+ *ta = (void *)__get_free_pages(GFP_KERNEL, get_order(*ta_size));
+- if (IS_ERR(*ta)) {
+- pr_err("%s: get_free_pages failed 0x%llx\n", __func__,
+- (u64)*ta);
++ if (!*ta) {
++ pr_err("%s: get_free_pages failed\n", __func__);
+ rc = -ENOMEM;
+ goto rel_fw;
+ }
+diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c
+index 5363ebebfc357..50c0d839fe751 100644
+--- a/drivers/tee/optee/core.c
++++ b/drivers/tee/optee/core.c
+@@ -588,6 +588,7 @@ static int optee_remove(struct platform_device *pdev)
+ /* Unregister OP-TEE specific client devices on TEE bus */
+ optee_unregister_devices();
+
++ teedev_close_context(optee->ctx);
+ /*
+ * Ask OP-TEE to free all cached shared memory objects to decrease
+ * reference counters and also avoid wild pointers in secure world
+@@ -633,6 +634,7 @@ static int optee_probe(struct platform_device *pdev)
+ struct optee *optee = NULL;
+ void *memremaped_shm = NULL;
+ struct tee_device *teedev;
++ struct tee_context *ctx;
+ u32 sec_caps;
+ int rc;
+
+@@ -719,6 +721,12 @@ static int optee_probe(struct platform_device *pdev)
+ optee_supp_init(&optee->supp);
+ optee->memremaped_shm = memremaped_shm;
+ optee->pool = pool;
++ ctx = teedev_open(optee->teedev);
++ if (IS_ERR(ctx)) {
++ rc = PTR_ERR(ctx);
++ goto err;
++ }
++ optee->ctx = ctx;
+
+ /*
+ * Ensure that there are no pre-existing shm objects before enabling
+diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
+index f6bb4a763ba94..ea09533e30cde 100644
+--- a/drivers/tee/optee/optee_private.h
++++ b/drivers/tee/optee/optee_private.h
+@@ -70,6 +70,7 @@ struct optee_supp {
+ * struct optee - main service struct
+ * @supp_teedev: supplicant device
+ * @teedev: client device
++ * @ctx: driver internal TEE context
+ * @invoke_fn: function to issue smc or hvc
+ * @call_queue: queue of threads waiting to call @invoke_fn
+ * @wait_queue: queue of threads from secure world waiting for a
+@@ -87,6 +88,7 @@ struct optee {
+ struct tee_device *supp_teedev;
+ struct tee_device *teedev;
+ optee_invoke_fn *invoke_fn;
++ struct tee_context *ctx;
+ struct optee_call_queue call_queue;
+ struct optee_wait_queue wait_queue;
+ struct optee_supp supp;
+diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c
+index efbaff7ad7e59..456833d820078 100644
+--- a/drivers/tee/optee/rpc.c
++++ b/drivers/tee/optee/rpc.c
+@@ -285,6 +285,7 @@ static struct tee_shm *cmd_alloc_suppl(struct tee_context *ctx, size_t sz)
+ }
+
+ static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
++ struct optee *optee,
+ struct optee_msg_arg *arg,
+ struct optee_call_ctx *call_ctx)
+ {
+@@ -314,7 +315,8 @@ static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
+ shm = cmd_alloc_suppl(ctx, sz);
+ break;
+ case OPTEE_RPC_SHM_TYPE_KERNEL:
+- shm = tee_shm_alloc(ctx, sz, TEE_SHM_MAPPED | TEE_SHM_PRIV);
++ shm = tee_shm_alloc(optee->ctx, sz,
++ TEE_SHM_MAPPED | TEE_SHM_PRIV);
+ break;
+ default:
+ arg->ret = TEEC_ERROR_BAD_PARAMETERS;
+@@ -471,7 +473,7 @@ static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
+ break;
+ case OPTEE_RPC_CMD_SHM_ALLOC:
+ free_pages_list(call_ctx);
+- handle_rpc_func_cmd_shm_alloc(ctx, arg, call_ctx);
++ handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
+ break;
+ case OPTEE_RPC_CMD_SHM_FREE:
+ handle_rpc_func_cmd_shm_free(ctx, arg);
+@@ -502,7 +504,7 @@ void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param,
+
+ switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
+ case OPTEE_SMC_RPC_FUNC_ALLOC:
+- shm = tee_shm_alloc(ctx, param->a1,
++ shm = tee_shm_alloc(optee->ctx, param->a1,
+ TEE_SHM_MAPPED | TEE_SHM_PRIV);
+ if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
+ reg_pair_from_64(&param->a1, &param->a2, pa);
+diff --git a/drivers/tee/optee/shm_pool.c b/drivers/tee/optee/shm_pool.c
+index d167039af519e..1aa843f2ecc7c 100644
+--- a/drivers/tee/optee/shm_pool.c
++++ b/drivers/tee/optee/shm_pool.c
+@@ -41,10 +41,8 @@ static int pool_op_alloc(struct tee_shm_pool_mgr *poolm,
+ goto err;
+ }
+
+- for (i = 0; i < nr_pages; i++) {
+- pages[i] = page;
+- page++;
+- }
++ for (i = 0; i < nr_pages; i++)
++ pages[i] = page + i;
+
+ shm->flags |= TEE_SHM_REGISTER;
+ rc = optee_shm_register(shm->ctx, shm, pages, nr_pages,
+diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
+index 2b37bc408fc3d..3fc426dad2df3 100644
+--- a/drivers/tee/tee_core.c
++++ b/drivers/tee/tee_core.c
+@@ -43,7 +43,7 @@ static DEFINE_SPINLOCK(driver_lock);
+ static struct class *tee_class;
+ static dev_t tee_devt;
+
+-static struct tee_context *teedev_open(struct tee_device *teedev)
++struct tee_context *teedev_open(struct tee_device *teedev)
+ {
+ int rc;
+ struct tee_context *ctx;
+@@ -70,6 +70,7 @@ err:
+ return ERR_PTR(rc);
+
+ }
++EXPORT_SYMBOL_GPL(teedev_open);
+
+ void teedev_ctx_get(struct tee_context *ctx)
+ {
+@@ -96,11 +97,14 @@ void teedev_ctx_put(struct tee_context *ctx)
+ kref_put(&ctx->refcount, teedev_ctx_release);
+ }
+
+-static void teedev_close_context(struct tee_context *ctx)
++void teedev_close_context(struct tee_context *ctx)
+ {
+- tee_device_put(ctx->teedev);
++ struct tee_device *teedev = ctx->teedev;
++
+ teedev_ctx_put(ctx);
++ tee_device_put(teedev);
+ }
++EXPORT_SYMBOL_GPL(teedev_close_context);
+
+ static int tee_open(struct inode *inode, struct file *filp)
+ {
+diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
+index 8a9384a64f3e2..499fccba3d74b 100644
+--- a/drivers/tee/tee_shm.c
++++ b/drivers/tee/tee_shm.c
+@@ -1,11 +1,11 @@
+ // SPDX-License-Identifier: GPL-2.0-only
+ /*
+- * Copyright (c) 2015-2016, Linaro Limited
++ * Copyright (c) 2015-2017, 2019-2021 Linaro Limited
+ */
++#include <linux/anon_inodes.h>
+ #include <linux/device.h>
+-#include <linux/dma-buf.h>
+-#include <linux/fdtable.h>
+ #include <linux/idr.h>
++#include <linux/mm.h>
+ #include <linux/sched.h>
+ #include <linux/slab.h>
+ #include <linux/tee_drv.h>
+@@ -28,16 +28,8 @@ static void release_registered_pages(struct tee_shm *shm)
+ }
+ }
+
+-static void tee_shm_release(struct tee_shm *shm)
++static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)
+ {
+- struct tee_device *teedev = shm->ctx->teedev;
+-
+- if (shm->flags & TEE_SHM_DMA_BUF) {
+- mutex_lock(&teedev->mutex);
+- idr_remove(&teedev->idr, shm->id);
+- mutex_unlock(&teedev->mutex);
+- }
+-
+ if (shm->flags & TEE_SHM_POOL) {
+ struct tee_shm_pool_mgr *poolm;
+
+@@ -64,45 +56,6 @@ static void tee_shm_release(struct tee_shm *shm)
+ tee_device_put(teedev);
+ }
+
+-static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
+- *attach, enum dma_data_direction dir)
+-{
+- return NULL;
+-}
+-
+-static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
+- struct sg_table *table,
+- enum dma_data_direction dir)
+-{
+-}
+-
+-static void tee_shm_op_release(struct dma_buf *dmabuf)
+-{
+- struct tee_shm *shm = dmabuf->priv;
+-
+- tee_shm_release(shm);
+-}
+-
+-static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+-{
+- struct tee_shm *shm = dmabuf->priv;
+- size_t size = vma->vm_end - vma->vm_start;
+-
+- /* Refuse sharing shared memory provided by application */
+- if (shm->flags & TEE_SHM_USER_MAPPED)
+- return -EINVAL;
+-
+- return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
+- size, vma->vm_page_prot);
+-}
+-
+-static const struct dma_buf_ops tee_shm_dma_buf_ops = {
+- .map_dma_buf = tee_shm_op_map_dma_buf,
+- .unmap_dma_buf = tee_shm_op_unmap_dma_buf,
+- .release = tee_shm_op_release,
+- .mmap = tee_shm_op_mmap,
+-};
+-
+ struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
+ {
+ struct tee_device *teedev = ctx->teedev;
+@@ -137,6 +90,7 @@ struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
+ goto err_dev_put;
+ }
+
++ refcount_set(&shm->refcount, 1);
+ shm->flags = flags | TEE_SHM_POOL;
+ shm->ctx = ctx;
+ if (flags & TEE_SHM_DMA_BUF)
+@@ -150,10 +104,7 @@ struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
+ goto err_kfree;
+ }
+
+-
+ if (flags & TEE_SHM_DMA_BUF) {
+- DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+-
+ mutex_lock(&teedev->mutex);
+ shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
+ mutex_unlock(&teedev->mutex);
+@@ -161,28 +112,11 @@ struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
+ ret = ERR_PTR(shm->id);
+ goto err_pool_free;
+ }
+-
+- exp_info.ops = &tee_shm_dma_buf_ops;
+- exp_info.size = shm->size;
+- exp_info.flags = O_RDWR;
+- exp_info.priv = shm;
+-
+- shm->dmabuf = dma_buf_export(&exp_info);
+- if (IS_ERR(shm->dmabuf)) {
+- ret = ERR_CAST(shm->dmabuf);
+- goto err_rem;
+- }
+ }
+
+ teedev_ctx_get(ctx);
+
+ return shm;
+-err_rem:
+- if (flags & TEE_SHM_DMA_BUF) {
+- mutex_lock(&teedev->mutex);
+- idr_remove(&teedev->idr, shm->id);
+- mutex_unlock(&teedev->mutex);
+- }
+ err_pool_free:
+ poolm->ops->free(poolm, shm);
+ err_kfree:
+@@ -243,6 +177,7 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
+ goto err;
+ }
+
++ refcount_set(&shm->refcount, 1);
+ shm->flags = flags | TEE_SHM_REGISTER;
+ shm->ctx = ctx;
+ shm->id = -1;
+@@ -303,22 +238,6 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
+ goto err;
+ }
+
+- if (flags & TEE_SHM_DMA_BUF) {
+- DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+-
+- exp_info.ops = &tee_shm_dma_buf_ops;
+- exp_info.size = shm->size;
+- exp_info.flags = O_RDWR;
+- exp_info.priv = shm;
+-
+- shm->dmabuf = dma_buf_export(&exp_info);
+- if (IS_ERR(shm->dmabuf)) {
+- ret = ERR_CAST(shm->dmabuf);
+- teedev->desc->ops->shm_unregister(ctx, shm);
+- goto err;
+- }
+- }
+-
+ return shm;
+ err:
+ if (shm) {
+@@ -336,6 +255,35 @@ err:
+ }
+ EXPORT_SYMBOL_GPL(tee_shm_register);
+
++static int tee_shm_fop_release(struct inode *inode, struct file *filp)
++{
++ tee_shm_put(filp->private_data);
++ return 0;
++}
++
++static int tee_shm_fop_mmap(struct file *filp, struct vm_area_struct *vma)
++{
++ struct tee_shm *shm = filp->private_data;
++ size_t size = vma->vm_end - vma->vm_start;
++
++ /* Refuse sharing shared memory provided by application */
++ if (shm->flags & TEE_SHM_USER_MAPPED)
++ return -EINVAL;
++
++ /* check for overflowing the buffer's size */
++ if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)
++ return -EINVAL;
++
++ return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
++ size, vma->vm_page_prot);
++}
++
++static const struct file_operations tee_shm_fops = {
++ .owner = THIS_MODULE,
++ .release = tee_shm_fop_release,
++ .mmap = tee_shm_fop_mmap,
++};
++
+ /**
+ * tee_shm_get_fd() - Increase reference count and return file descriptor
+ * @shm: Shared memory handle
+@@ -348,10 +296,11 @@ int tee_shm_get_fd(struct tee_shm *shm)
+ if (!(shm->flags & TEE_SHM_DMA_BUF))
+ return -EINVAL;
+
+- get_dma_buf(shm->dmabuf);
+- fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
++ /* matched by tee_shm_put() in tee_shm_op_release() */
++ refcount_inc(&shm->refcount);
++ fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);
+ if (fd < 0)
+- dma_buf_put(shm->dmabuf);
++ tee_shm_put(shm);
+ return fd;
+ }
+
+@@ -361,17 +310,7 @@ int tee_shm_get_fd(struct tee_shm *shm)
+ */
+ void tee_shm_free(struct tee_shm *shm)
+ {
+- /*
+- * dma_buf_put() decreases the dmabuf reference counter and will
+- * call tee_shm_release() when the last reference is gone.
+- *
+- * In the case of driver private memory we call tee_shm_release
+- * directly instead as it doesn't have a reference counter.
+- */
+- if (shm->flags & TEE_SHM_DMA_BUF)
+- dma_buf_put(shm->dmabuf);
+- else
+- tee_shm_release(shm);
++ tee_shm_put(shm);
+ }
+ EXPORT_SYMBOL_GPL(tee_shm_free);
+
+@@ -478,10 +417,15 @@ struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
+ teedev = ctx->teedev;
+ mutex_lock(&teedev->mutex);
+ shm = idr_find(&teedev->idr, id);
++ /*
++ * If the tee_shm was found in the IDR it must have a refcount
++ * larger than 0 due to the guarantee in tee_shm_put() below. So
++ * it's safe to use refcount_inc().
++ */
+ if (!shm || shm->ctx != ctx)
+ shm = ERR_PTR(-EINVAL);
+- else if (shm->flags & TEE_SHM_DMA_BUF)
+- get_dma_buf(shm->dmabuf);
++ else
++ refcount_inc(&shm->refcount);
+ mutex_unlock(&teedev->mutex);
+ return shm;
+ }
+@@ -493,7 +437,24 @@ EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
+ */
+ void tee_shm_put(struct tee_shm *shm)
+ {
+- if (shm->flags & TEE_SHM_DMA_BUF)
+- dma_buf_put(shm->dmabuf);
++ struct tee_device *teedev = shm->ctx->teedev;
++ bool do_release = false;
++
++ mutex_lock(&teedev->mutex);
++ if (refcount_dec_and_test(&shm->refcount)) {
++ /*
++ * refcount has reached 0, we must now remove it from the
++ * IDR before releasing the mutex. This will guarantee that
++ * the refcount_inc() in tee_shm_get_from_id() never starts
++ * from 0.
++ */
++ if (shm->flags & TEE_SHM_DMA_BUF)
++ idr_remove(&teedev->idr, shm->id);
++ do_release = true;
++ }
++ mutex_unlock(&teedev->mutex);
++
++ if (do_release)
++ tee_shm_release(teedev, shm);
+ }
+ EXPORT_SYMBOL_GPL(tee_shm_put);
+diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c
+index 7442e013738f8..af666bd9e8d4d 100644
+--- a/drivers/thermal/imx8mm_thermal.c
++++ b/drivers/thermal/imx8mm_thermal.c
+@@ -21,6 +21,7 @@
+ #define TPS 0x4
+ #define TRITSR 0x20 /* TMU immediate temp */
+
++#define TER_ADC_PD BIT(30)
+ #define TER_EN BIT(31)
+ #define TRITSR_TEMP0_VAL_MASK 0xff
+ #define TRITSR_TEMP1_VAL_MASK 0xff0000
+@@ -113,6 +114,8 @@ static void imx8mm_tmu_enable(struct imx8mm_tmu *tmu, bool enable)
+
+ val = readl_relaxed(tmu->base + TER);
+ val = enable ? (val | TER_EN) : (val & ~TER_EN);
++ if (tmu->socdata->version == TMU_VER2)
++ val = enable ? (val & ~TER_ADC_PD) : (val | TER_ADC_PD);
+ writel_relaxed(val, tmu->base + TER);
+ }
+
+diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c
+index 2c7473d86a59b..16663373b6829 100644
+--- a/drivers/thermal/imx_thermal.c
++++ b/drivers/thermal/imx_thermal.c
+@@ -15,6 +15,7 @@
+ #include <linux/regmap.h>
+ #include <linux/thermal.h>
+ #include <linux/nvmem-consumer.h>
++#include <linux/pm_runtime.h>
+
+ #define REG_SET 0x4
+ #define REG_CLR 0x8
+@@ -194,6 +195,7 @@ static struct thermal_soc_data thermal_imx7d_data = {
+ };
+
+ struct imx_thermal_data {
++ struct device *dev;
+ struct cpufreq_policy *policy;
+ struct thermal_zone_device *tz;
+ struct thermal_cooling_device *cdev;
+@@ -252,44 +254,15 @@ static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
+ const struct thermal_soc_data *soc_data = data->socdata;
+ struct regmap *map = data->tempmon;
+ unsigned int n_meas;
+- bool wait, run_measurement;
+ u32 val;
++ int ret;
+
+- run_measurement = !data->irq_enabled;
+- if (!run_measurement) {
+- /* Check if a measurement is currently in progress */
+- regmap_read(map, soc_data->temp_data, &val);
+- wait = !(val & soc_data->temp_valid_mask);
+- } else {
+- /*
+- * Every time we measure the temperature, we will power on the
+- * temperature sensor, enable measurements, take a reading,
+- * disable measurements, power off the temperature sensor.
+- */
+- regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
+- soc_data->power_down_mask);
+- regmap_write(map, soc_data->sensor_ctrl + REG_SET,
+- soc_data->measure_temp_mask);
+-
+- wait = true;
+- }
+-
+- /*
+- * According to the temp sensor designers, it may require up to ~17us
+- * to complete a measurement.
+- */
+- if (wait)
+- usleep_range(20, 50);
++ ret = pm_runtime_resume_and_get(data->dev);
++ if (ret < 0)
++ return ret;
+
+ regmap_read(map, soc_data->temp_data, &val);
+
+- if (run_measurement) {
+- regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
+- soc_data->measure_temp_mask);
+- regmap_write(map, soc_data->sensor_ctrl + REG_SET,
+- soc_data->power_down_mask);
+- }
+-
+ if ((val & soc_data->temp_valid_mask) == 0) {
+ dev_dbg(&tz->device, "temp measurement never finished\n");
+ return -EAGAIN;
+@@ -328,6 +301,8 @@ static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
+ enable_irq(data->irq);
+ }
+
++ pm_runtime_put(data->dev);
++
+ return 0;
+ }
+
+@@ -335,24 +310,16 @@ static int imx_change_mode(struct thermal_zone_device *tz,
+ enum thermal_device_mode mode)
+ {
+ struct imx_thermal_data *data = tz->devdata;
+- struct regmap *map = data->tempmon;
+- const struct thermal_soc_data *soc_data = data->socdata;
+
+ if (mode == THERMAL_DEVICE_ENABLED) {
+- regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
+- soc_data->power_down_mask);
+- regmap_write(map, soc_data->sensor_ctrl + REG_SET,
+- soc_data->measure_temp_mask);
++ pm_runtime_get(data->dev);
+
+ if (!data->irq_enabled) {
+ data->irq_enabled = true;
+ enable_irq(data->irq);
+ }
+ } else {
+- regmap_write(map, soc_data->sensor_ctrl + REG_CLR,
+- soc_data->measure_temp_mask);
+- regmap_write(map, soc_data->sensor_ctrl + REG_SET,
+- soc_data->power_down_mask);
++ pm_runtime_put(data->dev);
+
+ if (data->irq_enabled) {
+ disable_irq(data->irq);
+@@ -393,6 +360,11 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip,
+ int temp)
+ {
+ struct imx_thermal_data *data = tz->devdata;
++ int ret;
++
++ ret = pm_runtime_resume_and_get(data->dev);
++ if (ret < 0)
++ return ret;
+
+ /* do not allow changing critical threshold */
+ if (trip == IMX_TRIP_CRITICAL)
+@@ -406,6 +378,8 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip,
+
+ imx_set_alarm_temp(data, temp);
+
++ pm_runtime_put(data->dev);
++
+ return 0;
+ }
+
+@@ -681,6 +655,8 @@ static int imx_thermal_probe(struct platform_device *pdev)
+ if (!data)
+ return -ENOMEM;
+
++ data->dev = &pdev->dev;
++
+ map = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "fsl,tempmon");
+ if (IS_ERR(map)) {
+ ret = PTR_ERR(map);
+@@ -800,6 +776,16 @@ static int imx_thermal_probe(struct platform_device *pdev)
+ data->socdata->power_down_mask);
+ regmap_write(map, data->socdata->sensor_ctrl + REG_SET,
+ data->socdata->measure_temp_mask);
++ /* After power up, we need a delay before first access can be done. */
++ usleep_range(20, 50);
++
++ /* the core was configured and enabled just before */
++ pm_runtime_set_active(&pdev->dev);
++ pm_runtime_enable(data->dev);
++
++ ret = pm_runtime_resume_and_get(data->dev);
++ if (ret < 0)
++ goto disable_runtime_pm;
+
+ data->irq_enabled = true;
+ ret = thermal_zone_device_enable(data->tz);
+@@ -814,10 +800,15 @@ static int imx_thermal_probe(struct platform_device *pdev)
+ goto thermal_zone_unregister;
+ }
+
++ pm_runtime_put(data->dev);
++
+ return 0;
+
+ thermal_zone_unregister:
+ thermal_zone_device_unregister(data->tz);
++disable_runtime_pm:
++ pm_runtime_put_noidle(data->dev);
++ pm_runtime_disable(data->dev);
+ clk_disable:
+ clk_disable_unprepare(data->thermal_clk);
+ legacy_cleanup:
+@@ -829,13 +820,9 @@ legacy_cleanup:
+ static int imx_thermal_remove(struct platform_device *pdev)
+ {
+ struct imx_thermal_data *data = platform_get_drvdata(pdev);
+- struct regmap *map = data->tempmon;
+
+- /* Disable measurements */
+- regmap_write(map, data->socdata->sensor_ctrl + REG_SET,
+- data->socdata->power_down_mask);
+- if (!IS_ERR(data->thermal_clk))
+- clk_disable_unprepare(data->thermal_clk);
++ pm_runtime_put_noidle(data->dev);
++ pm_runtime_disable(data->dev);
+
+ thermal_zone_device_unregister(data->tz);
+ imx_thermal_unregister_legacy_cooling(data);
+@@ -858,29 +845,79 @@ static int __maybe_unused imx_thermal_suspend(struct device *dev)
+ ret = thermal_zone_device_disable(data->tz);
+ if (ret)
+ return ret;
++
++ return pm_runtime_force_suspend(data->dev);
++}
++
++static int __maybe_unused imx_thermal_resume(struct device *dev)
++{
++ struct imx_thermal_data *data = dev_get_drvdata(dev);
++ int ret;
++
++ ret = pm_runtime_force_resume(data->dev);
++ if (ret)
++ return ret;
++ /* Enabled thermal sensor after resume */
++ return thermal_zone_device_enable(data->tz);
++}
++
++static int __maybe_unused imx_thermal_runtime_suspend(struct device *dev)
++{
++ struct imx_thermal_data *data = dev_get_drvdata(dev);
++ const struct thermal_soc_data *socdata = data->socdata;
++ struct regmap *map = data->tempmon;
++ int ret;
++
++ ret = regmap_write(map, socdata->sensor_ctrl + REG_CLR,
++ socdata->measure_temp_mask);
++ if (ret)
++ return ret;
++
++ ret = regmap_write(map, socdata->sensor_ctrl + REG_SET,
++ socdata->power_down_mask);
++ if (ret)
++ return ret;
++
+ clk_disable_unprepare(data->thermal_clk);
+
+ return 0;
+ }
+
+-static int __maybe_unused imx_thermal_resume(struct device *dev)
++static int __maybe_unused imx_thermal_runtime_resume(struct device *dev)
+ {
+ struct imx_thermal_data *data = dev_get_drvdata(dev);
++ const struct thermal_soc_data *socdata = data->socdata;
++ struct regmap *map = data->tempmon;
+ int ret;
+
+ ret = clk_prepare_enable(data->thermal_clk);
+ if (ret)
+ return ret;
+- /* Enabled thermal sensor after resume */
+- ret = thermal_zone_device_enable(data->tz);
++
++ ret = regmap_write(map, socdata->sensor_ctrl + REG_CLR,
++ socdata->power_down_mask);
++ if (ret)
++ return ret;
++
++ ret = regmap_write(map, socdata->sensor_ctrl + REG_SET,
++ socdata->measure_temp_mask);
+ if (ret)
+ return ret;
+
++ /*
++ * According to the temp sensor designers, it may require up to ~17us
++ * to complete a measurement.
++ */
++ usleep_range(20, 50);
++
+ return 0;
+ }
+
+-static SIMPLE_DEV_PM_OPS(imx_thermal_pm_ops,
+- imx_thermal_suspend, imx_thermal_resume);
++static const struct dev_pm_ops imx_thermal_pm_ops = {
++ SET_SYSTEM_SLEEP_PM_OPS(imx_thermal_suspend, imx_thermal_resume)
++ SET_RUNTIME_PM_OPS(imx_thermal_runtime_suspend,
++ imx_thermal_runtime_resume, NULL)
++};
+
+ static struct platform_driver imx_thermal = {
+ .driver = {
+diff --git a/drivers/thermal/intel/int340x_thermal/Kconfig b/drivers/thermal/intel/int340x_thermal/Kconfig
+index 45c31f3d6054b..5d046de96a5d0 100644
+--- a/drivers/thermal/intel/int340x_thermal/Kconfig
++++ b/drivers/thermal/intel/int340x_thermal/Kconfig
+@@ -5,12 +5,12 @@
+
+ config INT340X_THERMAL
+ tristate "ACPI INT340X thermal drivers"
+- depends on X86 && ACPI && PCI
++ depends on X86_64 && ACPI && PCI
+ select THERMAL_GOV_USER_SPACE
+ select ACPI_THERMAL_REL
+ select ACPI_FAN
+ select INTEL_SOC_DTS_IOSF_CORE
+- select PROC_THERMAL_MMIO_RAPL if X86_64 && POWERCAP
++ select PROC_THERMAL_MMIO_RAPL if POWERCAP
+ help
+ Newer laptops and tablets that use ACPI may have thermal sensors and
+ other devices with thermal control capabilities outside the core
+diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+index 19926beeb3b71..6aa5fe9736138 100644
+--- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
++++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+@@ -53,7 +53,7 @@ struct int3400_thermal_priv {
+ struct art *arts;
+ int trt_count;
+ struct trt *trts;
+- u8 uuid_bitmap;
++ u32 uuid_bitmap;
+ int rel_misc_dev_res;
+ int current_uuid_index;
+ char *data_vault;
+@@ -67,7 +67,7 @@ static int evaluate_odvp(struct int3400_thermal_priv *priv);
+ struct odvp_attr {
+ int odvp;
+ struct int3400_thermal_priv *priv;
+- struct kobj_attribute attr;
++ struct device_attribute attr;
+ };
+
+ static ssize_t data_vault_read(struct file *file, struct kobject *kobj,
+@@ -272,7 +272,7 @@ static int int3400_thermal_run_osc(acpi_handle handle,
+ return result;
+ }
+
+-static ssize_t odvp_show(struct kobject *kobj, struct kobj_attribute *attr,
++static ssize_t odvp_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+ {
+ struct odvp_attr *odvp_attr;
+@@ -405,6 +405,10 @@ static void int3400_notify(acpi_handle handle,
+ thermal_prop[3] = kasprintf(GFP_KERNEL, "EVENT=%d", therm_event);
+ thermal_prop[4] = NULL;
+ kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE, thermal_prop);
++ kfree(thermal_prop[0]);
++ kfree(thermal_prop[1]);
++ kfree(thermal_prop[2]);
++ kfree(thermal_prop[3]);
+ }
+
+ static int int3400_thermal_get_temp(struct thermal_zone_device *thermal,
+@@ -465,6 +469,11 @@ static void int3400_setup_gddv(struct int3400_thermal_priv *priv)
+ priv->data_vault = kmemdup(obj->package.elements[0].buffer.pointer,
+ obj->package.elements[0].buffer.length,
+ GFP_KERNEL);
++ if (!priv->data_vault) {
++ kfree(buffer.pointer);
++ return;
++ }
++
+ bin_attr_data_vault.private = priv->data_vault;
+ bin_attr_data_vault.size = obj->package.elements[0].buffer.length;
+ kfree(buffer.pointer);
+diff --git a/drivers/thermal/intel/int340x_thermal/int3401_thermal.c b/drivers/thermal/intel/int340x_thermal/int3401_thermal.c
+index acebc8ba94e29..217786fba185c 100644
+--- a/drivers/thermal/intel/int340x_thermal/int3401_thermal.c
++++ b/drivers/thermal/intel/int340x_thermal/int3401_thermal.c
+@@ -44,15 +44,21 @@ static int int3401_remove(struct platform_device *pdev)
+ }
+
+ #ifdef CONFIG_PM_SLEEP
++static int int3401_thermal_suspend(struct device *dev)
++{
++ return proc_thermal_suspend(dev);
++}
+ static int int3401_thermal_resume(struct device *dev)
+ {
+ return proc_thermal_resume(dev);
+ }
+ #else
++#define int3401_thermal_suspend NULL
+ #define int3401_thermal_resume NULL
+ #endif
+
+-static SIMPLE_DEV_PM_OPS(int3401_proc_thermal_pm, NULL, int3401_thermal_resume);
++static SIMPLE_DEV_PM_OPS(int3401_proc_thermal_pm, int3401_thermal_suspend,
++ int3401_thermal_resume);
+
+ static struct platform_driver int3401_driver = {
+ .probe = int3401_add,
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
+index fb64acfd5e07d..a8d98f1bd6c67 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
+@@ -68,8 +68,7 @@ static const struct attribute_group power_limit_attribute_group = {
+ .name = "power_limits"
+ };
+
+-static ssize_t tcc_offset_degree_celsius_show(struct device *dev,
+- struct device_attribute *attr, char *buf)
++static int tcc_get_offset(void)
+ {
+ u64 val;
+ int err;
+@@ -78,8 +77,20 @@ static ssize_t tcc_offset_degree_celsius_show(struct device *dev,
+ if (err)
+ return err;
+
+- val = (val >> 24) & 0x3f;
+- return sprintf(buf, "%d\n", (int)val);
++ return (val >> 24) & 0x3f;
++}
++
++static ssize_t tcc_offset_degree_celsius_show(struct device *dev,
++ struct device_attribute *attr,
++ char *buf)
++{
++ int tcc;
++
++ tcc = tcc_get_offset();
++ if (tcc < 0)
++ return tcc;
++
++ return sprintf(buf, "%d\n", tcc);
+ }
+
+ static int tcc_offset_update(unsigned int tcc)
+@@ -107,8 +118,6 @@ static int tcc_offset_update(unsigned int tcc)
+ return 0;
+ }
+
+-static int tcc_offset_save = -1;
+-
+ static ssize_t tcc_offset_degree_celsius_store(struct device *dev,
+ struct device_attribute *attr, const char *buf,
+ size_t count)
+@@ -131,8 +140,6 @@ static ssize_t tcc_offset_degree_celsius_store(struct device *dev,
+ if (err)
+ return err;
+
+- tcc_offset_save = tcc;
+-
+ return count;
+ }
+
+@@ -345,6 +352,18 @@ void proc_thermal_remove(struct proc_thermal_device *proc_priv)
+ }
+ EXPORT_SYMBOL_GPL(proc_thermal_remove);
+
++static int tcc_offset_save = -1;
++
++int proc_thermal_suspend(struct device *dev)
++{
++ tcc_offset_save = tcc_get_offset();
++ if (tcc_offset_save < 0)
++ dev_warn(dev, "failed to save offset (%d)\n", tcc_offset_save);
++
++ return 0;
++}
++EXPORT_SYMBOL_GPL(proc_thermal_suspend);
++
+ int proc_thermal_resume(struct device *dev)
+ {
+ struct proc_thermal_device *proc_dev;
+@@ -352,6 +371,7 @@ int proc_thermal_resume(struct device *dev)
+ proc_dev = dev_get_drvdata(dev);
+ proc_thermal_read_ppcc(proc_dev);
+
++ /* Do not update if saving failed */
+ if (tcc_offset_save >= 0)
+ tcc_offset_update(tcc_offset_save);
+
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
+index 5a1cfe4864f16..9b2a64ef55d02 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
+@@ -80,9 +80,11 @@ void proc_thermal_rfim_remove(struct pci_dev *pdev);
+ int proc_thermal_mbox_add(struct pci_dev *pdev, struct proc_thermal_device *proc_priv);
+ void proc_thermal_mbox_remove(struct pci_dev *pdev);
+
+-int processor_thermal_send_mbox_cmd(struct pci_dev *pdev, u16 cmd_id, u32 cmd_data, u32 *cmd_resp);
++int processor_thermal_send_mbox_read_cmd(struct pci_dev *pdev, u16 id, u64 *resp);
++int processor_thermal_send_mbox_write_cmd(struct pci_dev *pdev, u16 id, u32 data);
+ int proc_thermal_add(struct device *dev, struct proc_thermal_device *priv);
+ void proc_thermal_remove(struct proc_thermal_device *proc_priv);
++int proc_thermal_suspend(struct device *dev);
+ int proc_thermal_resume(struct device *dev);
+ int proc_thermal_mmio_add(struct pci_dev *pdev,
+ struct proc_thermal_device *proc_priv,
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
+index 11dd2e825f4ff..b4bcd3fe9eb2f 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
+@@ -314,6 +314,20 @@ static void proc_thermal_pci_remove(struct pci_dev *pdev)
+ }
+
+ #ifdef CONFIG_PM_SLEEP
++static int proc_thermal_pci_suspend(struct device *dev)
++{
++ struct pci_dev *pdev = to_pci_dev(dev);
++ struct proc_thermal_device *proc_priv;
++ struct proc_thermal_pci *pci_info;
++
++ proc_priv = pci_get_drvdata(pdev);
++ pci_info = proc_priv->priv_data;
++
++ if (!pci_info->no_legacy)
++ return proc_thermal_suspend(dev);
++
++ return 0;
++}
+ static int proc_thermal_pci_resume(struct device *dev)
+ {
+ struct pci_dev *pdev = to_pci_dev(dev);
+@@ -335,10 +349,12 @@ static int proc_thermal_pci_resume(struct device *dev)
+ return 0;
+ }
+ #else
++#define proc_thermal_pci_suspend NULL
+ #define proc_thermal_pci_resume NULL
+ #endif
+
+-static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, NULL, proc_thermal_pci_resume);
++static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, proc_thermal_pci_suspend,
++ proc_thermal_pci_resume);
+
+ static const struct pci_device_id proc_thermal_pci_ids[] = {
+ { PCI_DEVICE_DATA(INTEL, ADL_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_MBOX) },
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
+index f5fc1791b11ef..4571a1a53b841 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
+@@ -107,15 +107,21 @@ static void proc_thermal_pci_remove(struct pci_dev *pdev)
+ }
+
+ #ifdef CONFIG_PM_SLEEP
++static int proc_thermal_pci_suspend(struct device *dev)
++{
++ return proc_thermal_suspend(dev);
++}
+ static int proc_thermal_pci_resume(struct device *dev)
+ {
+ return proc_thermal_resume(dev);
+ }
+ #else
++#define proc_thermal_pci_suspend NULL
+ #define proc_thermal_pci_resume NULL
+ #endif
+
+-static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, NULL, proc_thermal_pci_resume);
++static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, proc_thermal_pci_suspend,
++ proc_thermal_pci_resume);
+
+ static const struct pci_device_id proc_thermal_pci_ids[] = {
+ { PCI_DEVICE_DATA(INTEL, BDW_THERMAL, 0) },
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.c
+index 59e93b04f0a9e..0b89a4340ff4e 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.c
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.c
+@@ -7,6 +7,7 @@
+ #include <linux/kernel.h>
+ #include <linux/module.h>
+ #include <linux/pci.h>
++#include <linux/io-64-nonatomic-lo-hi.h>
+ #include "processor_thermal_device.h"
+
+ #define MBOX_CMD_WORKLOAD_TYPE_READ 0x0E
+@@ -23,19 +24,15 @@
+
+ static DEFINE_MUTEX(mbox_lock);
+
+-static int send_mbox_cmd(struct pci_dev *pdev, u16 cmd_id, u32 cmd_data, u32 *cmd_resp)
++static int wait_for_mbox_ready(struct proc_thermal_device *proc_priv)
+ {
+- struct proc_thermal_device *proc_priv;
+ u32 retries, data;
+ int ret;
+
+- mutex_lock(&mbox_lock);
+- proc_priv = pci_get_drvdata(pdev);
+-
+ /* Poll for rb bit == 0 */
+ retries = MBOX_RETRY_COUNT;
+ do {
+- data = readl((void __iomem *) (proc_priv->mmio_base + MBOX_OFFSET_INTERFACE));
++ data = readl(proc_priv->mmio_base + MBOX_OFFSET_INTERFACE);
+ if (data & BIT_ULL(MBOX_BUSY_BIT)) {
+ ret = -EBUSY;
+ continue;
+@@ -44,49 +41,78 @@ static int send_mbox_cmd(struct pci_dev *pdev, u16 cmd_id, u32 cmd_data, u32 *cm
+ break;
+ } while (--retries);
+
++ return ret;
++}
++
++static int send_mbox_write_cmd(struct pci_dev *pdev, u16 id, u32 data)
++{
++ struct proc_thermal_device *proc_priv;
++ u32 reg_data;
++ int ret;
++
++ proc_priv = pci_get_drvdata(pdev);
++
++ mutex_lock(&mbox_lock);
++
++ ret = wait_for_mbox_ready(proc_priv);
+ if (ret)
+ goto unlock_mbox;
+
+- if (cmd_id == MBOX_CMD_WORKLOAD_TYPE_WRITE)
+- writel(cmd_data, (void __iomem *) ((proc_priv->mmio_base + MBOX_OFFSET_DATA)));
+-
++ writel(data, (proc_priv->mmio_base + MBOX_OFFSET_DATA));
+ /* Write command register */
+- data = BIT_ULL(MBOX_BUSY_BIT) | cmd_id;
+- writel(data, (void __iomem *) ((proc_priv->mmio_base + MBOX_OFFSET_INTERFACE)));
++ reg_data = BIT_ULL(MBOX_BUSY_BIT) | id;
++ writel(reg_data, (proc_priv->mmio_base + MBOX_OFFSET_INTERFACE));
+
+- /* Poll for rb bit == 0 */
+- retries = MBOX_RETRY_COUNT;
+- do {
+- data = readl((void __iomem *) (proc_priv->mmio_base + MBOX_OFFSET_INTERFACE));
+- if (data & BIT_ULL(MBOX_BUSY_BIT)) {
+- ret = -EBUSY;
+- continue;
+- }
++ ret = wait_for_mbox_ready(proc_priv);
+
+- if (data) {
+- ret = -ENXIO;
+- goto unlock_mbox;
+- }
++unlock_mbox:
++ mutex_unlock(&mbox_lock);
++ return ret;
++}
+
+- if (cmd_id == MBOX_CMD_WORKLOAD_TYPE_READ) {
+- data = readl((void __iomem *) (proc_priv->mmio_base + MBOX_OFFSET_DATA));
+- *cmd_resp = data & 0xff;
+- }
++static int send_mbox_read_cmd(struct pci_dev *pdev, u16 id, u64 *resp)
++{
++ struct proc_thermal_device *proc_priv;
++ u32 reg_data;
++ int ret;
+
+- ret = 0;
+- break;
+- } while (--retries);
++ proc_priv = pci_get_drvdata(pdev);
++
++ mutex_lock(&mbox_lock);
++
++ ret = wait_for_mbox_ready(proc_priv);
++ if (ret)
++ goto unlock_mbox;
++
++ /* Write command register */
++ reg_data = BIT_ULL(MBOX_BUSY_BIT) | id;
++ writel(reg_data, (proc_priv->mmio_base + MBOX_OFFSET_INTERFACE));
++
++ ret = wait_for_mbox_ready(proc_priv);
++ if (ret)
++ goto unlock_mbox;
++
++ if (id == MBOX_CMD_WORKLOAD_TYPE_READ)
++ *resp = readl(proc_priv->mmio_base + MBOX_OFFSET_DATA);
++ else
++ *resp = readq(proc_priv->mmio_base + MBOX_OFFSET_DATA);
+
+ unlock_mbox:
+ mutex_unlock(&mbox_lock);
+ return ret;
+ }
+
+-int processor_thermal_send_mbox_cmd(struct pci_dev *pdev, u16 cmd_id, u32 cmd_data, u32 *cmd_resp)
++int processor_thermal_send_mbox_read_cmd(struct pci_dev *pdev, u16 id, u64 *resp)
+ {
+- return send_mbox_cmd(pdev, cmd_id, cmd_data, cmd_resp);
++ return send_mbox_read_cmd(pdev, id, resp);
+ }
+-EXPORT_SYMBOL_GPL(processor_thermal_send_mbox_cmd);
++EXPORT_SYMBOL_NS_GPL(processor_thermal_send_mbox_read_cmd, INT340X_THERMAL);
++
++int processor_thermal_send_mbox_write_cmd(struct pci_dev *pdev, u16 id, u32 data)
++{
++ return send_mbox_write_cmd(pdev, id, data);
++}
++EXPORT_SYMBOL_NS_GPL(processor_thermal_send_mbox_write_cmd, INT340X_THERMAL);
+
+ /* List of workload types */
+ static const char * const workload_types[] = {
+@@ -99,7 +125,6 @@ static const char * const workload_types[] = {
+ NULL
+ };
+
+-
+ static ssize_t workload_available_types_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+@@ -141,7 +166,7 @@ static ssize_t workload_type_store(struct device *dev,
+
+ data |= ret;
+
+- ret = send_mbox_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_WRITE, data, NULL);
++ ret = send_mbox_write_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_WRITE, data);
+ if (ret)
+ return false;
+
+@@ -153,10 +178,10 @@ static ssize_t workload_type_show(struct device *dev,
+ char *buf)
+ {
+ struct pci_dev *pdev = to_pci_dev(dev);
+- u32 cmd_resp;
++ u64 cmd_resp;
+ int ret;
+
+- ret = send_mbox_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_READ, 0, &cmd_resp);
++ ret = send_mbox_read_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_READ, &cmd_resp);
+ if (ret)
+ return false;
+
+@@ -181,17 +206,15 @@ static const struct attribute_group workload_req_attribute_group = {
+ .name = "workload_request"
+ };
+
+-
+-
+ static bool workload_req_created;
+
+ int proc_thermal_mbox_add(struct pci_dev *pdev, struct proc_thermal_device *proc_priv)
+ {
+- u32 cmd_resp;
++ u64 cmd_resp;
+ int ret;
+
+ /* Check if there is a mailbox support, if fails return success */
+- ret = send_mbox_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_READ, 0, &cmd_resp);
++ ret = send_mbox_read_cmd(pdev, MBOX_CMD_WORKLOAD_TYPE_READ, &cmd_resp);
+ if (ret)
+ return 0;
+
+diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
+index 2b8a3235d518b..8c42e76620333 100644
+--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
++++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
+@@ -9,6 +9,8 @@
+ #include <linux/pci.h>
+ #include "processor_thermal_device.h"
+
++MODULE_IMPORT_NS(INT340X_THERMAL);
++
+ struct mmio_reg {
+ int read_only;
+ u32 offset;
+@@ -29,7 +31,7 @@ static const char * const fivr_strings[] = {
+ };
+
+ static const struct mmio_reg tgl_fivr_mmio_regs[] = {
+- { 0, 0x5A18, 3, 0x7, 12}, /* vco_ref_code_lo */
++ { 0, 0x5A18, 3, 0x7, 11}, /* vco_ref_code_lo */
+ { 0, 0x5A18, 8, 0xFF, 16}, /* vco_ref_code_hi */
+ { 0, 0x5A08, 8, 0xFF, 0}, /* spread_spectrum_pct */
+ { 0, 0x5A08, 1, 0x1, 8}, /* spread_spectrum_clk_enable */
+@@ -194,8 +196,7 @@ static ssize_t rfi_restriction_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+ {
+- u16 cmd_id = 0x0008;
+- u32 cmd_resp;
++ u16 id = 0x0008;
+ u32 input;
+ int ret;
+
+@@ -203,7 +204,7 @@ static ssize_t rfi_restriction_store(struct device *dev,
+ if (ret)
+ return ret;
+
+- ret = processor_thermal_send_mbox_cmd(to_pci_dev(dev), cmd_id, input, &cmd_resp);
++ ret = processor_thermal_send_mbox_write_cmd(to_pci_dev(dev), id, input);
+ if (ret)
+ return ret;
+
+@@ -214,30 +215,30 @@ static ssize_t rfi_restriction_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+ {
+- u16 cmd_id = 0x0007;
+- u32 cmd_resp;
++ u16 id = 0x0007;
++ u64 resp;
+ int ret;
+
+- ret = processor_thermal_send_mbox_cmd(to_pci_dev(dev), cmd_id, 0, &cmd_resp);
++ ret = processor_thermal_send_mbox_read_cmd(to_pci_dev(dev), id, &resp);
+ if (ret)
+ return ret;
+
+- return sprintf(buf, "%u\n", cmd_resp);
++ return sprintf(buf, "%llu\n", resp);
+ }
+
+ static ssize_t ddr_data_rate_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+ {
+- u16 cmd_id = 0x0107;
+- u32 cmd_resp;
++ u16 id = 0x0107;
++ u64 resp;
+ int ret;
+
+- ret = processor_thermal_send_mbox_cmd(to_pci_dev(dev), cmd_id, 0, &cmd_resp);
++ ret = processor_thermal_send_mbox_read_cmd(to_pci_dev(dev), id, &resp);
+ if (ret)
+ return ret;
+
+- return sprintf(buf, "%u\n", cmd_resp);
++ return sprintf(buf, "%llu\n", resp);
+ }
+
+ static DEVICE_ATTR_RW(rfi_restriction);
+diff --git a/drivers/thermal/qcom/Kconfig b/drivers/thermal/qcom/Kconfig
+index 7d942f71e5328..bfd889422dd32 100644
+--- a/drivers/thermal/qcom/Kconfig
++++ b/drivers/thermal/qcom/Kconfig
+@@ -34,7 +34,7 @@ config QCOM_SPMI_TEMP_ALARM
+
+ config QCOM_LMH
+ tristate "Qualcomm Limits Management Hardware"
+- depends on ARCH_QCOM
++ depends on ARCH_QCOM && QCOM_SCM
+ help
+ This enables initialization of Qualcomm limits management
+ hardware(LMh). LMh allows for hardware-enforced mitigation for cpus based on
+diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c
+index b1162e566a707..99a8d9f3e03ca 100644
+--- a/drivers/thermal/qcom/tsens.c
++++ b/drivers/thermal/qcom/tsens.c
+@@ -603,22 +603,21 @@ int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp)
+ int ret;
+
+ /* VER_0 doesn't have VALID bit */
+- if (tsens_version(priv) >= VER_0_1) {
+- ret = regmap_field_read(priv->rf[valid_idx], &valid);
+- if (ret)
+- return ret;
+- while (!valid) {
+- /* Valid bit is 0 for 6 AHB clock cycles.
+- * At 19.2MHz, 1 AHB clock is ~60ns.
+- * We should enter this loop very, very rarely.
+- */
+- ndelay(400);
+- ret = regmap_field_read(priv->rf[valid_idx], &valid);
+- if (ret)
+- return ret;
+- }
+- }
++ if (tsens_version(priv) == VER_0)
++ goto get_temp;
++
++ /* Valid bit is 0 for 6 AHB clock cycles.
++ * At 19.2MHz, 1 AHB clock is ~60ns.
++ * We should enter this loop very, very rarely.
++ * Wait 1 us since it's the min of poll_timeout macro.
++ * Old value was 400 ns.
++ */
++ ret = regmap_field_read_poll_timeout(priv->rf[valid_idx], valid,
++ valid, 1, 20 * USEC_PER_MSEC);
++ if (ret)
++ return ret;
+
++get_temp:
+ /* Valid bit is set, OK to read the temperature */
+ *temp = tsens_hw_to_mC(s, temp_idx);
+
+diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
+index 51374f4e1ccaf..13891745a9719 100644
+--- a/drivers/thermal/thermal_core.c
++++ b/drivers/thermal/thermal_core.c
+@@ -419,6 +419,8 @@ static void thermal_zone_device_init(struct thermal_zone_device *tz)
+ {
+ struct thermal_instance *pos;
+ tz->temperature = THERMAL_TEMP_INVALID;
++ tz->prev_low_trip = -INT_MAX;
++ tz->prev_high_trip = INT_MAX;
+ list_for_each_entry(pos, &tz->thermal_instances, tz_node)
+ pos->initialized = false;
+ }
+@@ -887,7 +889,7 @@ __thermal_cooling_device_register(struct device_node *np,
+ {
+ struct thermal_cooling_device *cdev;
+ struct thermal_zone_device *pos = NULL;
+- int ret;
++ int id, ret;
+
+ if (!ops || !ops->get_max_state || !ops->get_cur_state ||
+ !ops->set_cur_state)
+@@ -901,6 +903,11 @@ __thermal_cooling_device_register(struct device_node *np,
+ if (ret < 0)
+ goto out_kfree_cdev;
+ cdev->id = ret;
++ id = ret;
++
++ ret = dev_set_name(&cdev->device, "cooling_device%d", cdev->id);
++ if (ret)
++ goto out_ida_remove;
+
+ cdev->type = kstrdup(type ? type : "", GFP_KERNEL);
+ if (!cdev->type) {
+@@ -916,7 +923,6 @@ __thermal_cooling_device_register(struct device_node *np,
+ cdev->device.class = &thermal_class;
+ cdev->devdata = devdata;
+ thermal_cooling_device_setup_sysfs(cdev);
+- dev_set_name(&cdev->device, "cooling_device%d", cdev->id);
+ ret = device_register(&cdev->device);
+ if (ret)
+ goto out_kfree_type;
+@@ -941,8 +947,9 @@ __thermal_cooling_device_register(struct device_node *np,
+ out_kfree_type:
+ kfree(cdev->type);
+ put_device(&cdev->device);
++ cdev = NULL;
+ out_ida_remove:
+- ida_simple_remove(&thermal_cdev_ida, cdev->id);
++ ida_simple_remove(&thermal_cdev_ida, id);
+ out_kfree_cdev:
+ kfree(cdev);
+ return ERR_PTR(ret);
+@@ -1227,6 +1234,10 @@ thermal_zone_device_register(const char *type, int trips, int mask,
+ tz->id = id;
+ strlcpy(tz->type, type, sizeof(tz->type));
+
++ result = dev_set_name(&tz->device, "thermal_zone%d", tz->id);
++ if (result)
++ goto remove_id;
++
+ if (!ops->critical)
+ ops->critical = thermal_zone_device_critical;
+
+@@ -1248,7 +1259,6 @@ thermal_zone_device_register(const char *type, int trips, int mask,
+ /* A new thermal zone needs to be updated anyway. */
+ atomic_set(&tz->need_update, 1);
+
+- dev_set_name(&tz->device, "thermal_zone%d", tz->id);
+ result = device_register(&tz->device);
+ if (result)
+ goto release_device;
+diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c
+index 1234dbe958951..41c8d47805c4e 100644
+--- a/drivers/thermal/thermal_netlink.c
++++ b/drivers/thermal/thermal_netlink.c
+@@ -418,11 +418,12 @@ static int thermal_genl_cmd_tz_get_trip(struct param *p)
+ for (i = 0; i < tz->trips; i++) {
+
+ enum thermal_trip_type type;
+- int temp, hyst;
++ int temp, hyst = 0;
+
+ tz->ops->get_trip_type(tz, i, &type);
+ tz->ops->get_trip_temp(tz, i, &temp);
+- tz->ops->get_trip_hyst(tz, i, &hyst);
++ if (tz->ops->get_trip_hyst)
++ tz->ops->get_trip_hyst(tz, i, &hyst);
+
+ if (nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_ID, i) ||
+ nla_put_u32(msg, THERMAL_GENL_ATTR_TZ_TRIP_TYPE, type) ||
+diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
+index 6379f26a335f6..9233f7e744544 100644
+--- a/drivers/thermal/thermal_of.c
++++ b/drivers/thermal/thermal_of.c
+@@ -89,7 +89,7 @@ static int of_thermal_get_temp(struct thermal_zone_device *tz,
+ {
+ struct __thermal_zone *data = tz->devdata;
+
+- if (!data->ops->get_temp)
++ if (!data->ops || !data->ops->get_temp)
+ return -EINVAL;
+
+ return data->ops->get_temp(data->sensor_data, temp);
+@@ -186,6 +186,9 @@ static int of_thermal_set_emul_temp(struct thermal_zone_device *tz,
+ {
+ struct __thermal_zone *data = tz->devdata;
+
++ if (!data->ops || !data->ops->set_emul_temp)
++ return -EINVAL;
++
+ return data->ops->set_emul_temp(data->sensor_data, temp);
+ }
+
+@@ -194,7 +197,7 @@ static int of_thermal_get_trend(struct thermal_zone_device *tz, int trip,
+ {
+ struct __thermal_zone *data = tz->devdata;
+
+- if (!data->ops->get_trend)
++ if (!data->ops || !data->ops->get_trend)
+ return -EINVAL;
+
+ return data->ops->get_trend(data->sensor_data, trip, trend);
+@@ -301,7 +304,7 @@ static int of_thermal_set_trip_temp(struct thermal_zone_device *tz, int trip,
+ if (trip >= data->ntrips || trip < 0)
+ return -EDOM;
+
+- if (data->ops->set_trip_temp) {
++ if (data->ops && data->ops->set_trip_temp) {
+ int ret;
+
+ ret = data->ops->set_trip_temp(data->sensor_data, trip, temp);
+diff --git a/drivers/thunderbolt/acpi.c b/drivers/thunderbolt/acpi.c
+index b67e72d5644b3..7c9597a339295 100644
+--- a/drivers/thunderbolt/acpi.c
++++ b/drivers/thunderbolt/acpi.c
+@@ -7,6 +7,7 @@
+ */
+
+ #include <linux/acpi.h>
++#include <linux/pm_runtime.h>
+
+ #include "tb.h"
+
+@@ -74,8 +75,18 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
+ pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM))) {
+ const struct device_link *link;
+
++ /*
++ * Make them both active first to make sure the NHI does
++ * not runtime suspend before the consumer. The
++ * pm_runtime_put() below then allows the consumer to
++ * runtime suspend again (which then allows NHI runtime
++ * suspend too now that the device link is established).
++ */
++ pm_runtime_get_sync(&pdev->dev);
++
+ link = device_link_add(&pdev->dev, &nhi->pdev->dev,
+ DL_FLAG_AUTOREMOVE_SUPPLIER |
++ DL_FLAG_RPM_ACTIVE |
+ DL_FLAG_PM_RUNTIME);
+ if (link) {
+ dev_dbg(&nhi->pdev->dev, "created link from %s\n",
+@@ -84,6 +95,8 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
+ dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n",
+ dev_name(&pdev->dev));
+ }
++
++ pm_runtime_put(&pdev->dev);
+ }
+
+ out_put:
+diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c
+index 82a76cac94deb..32366caca6623 100644
+--- a/drivers/tty/hvc/hvc_iucv.c
++++ b/drivers/tty/hvc/hvc_iucv.c
+@@ -1417,7 +1417,9 @@ out_error:
+ */
+ static int __init hvc_iucv_config(char *val)
+ {
+- return kstrtoul(val, 10, &hvc_iucv_devices);
++ if (kstrtoul(val, 10, &hvc_iucv_devices))
++ pr_warn("hvc_iucv= invalid parameter value '%s'\n", val);
++ return 1;
+ }
+
+
+diff --git a/drivers/tty/hvc/hvc_xen.c b/drivers/tty/hvc/hvc_xen.c
+index f0bf01ea069ae..8ee7ce1206925 100644
+--- a/drivers/tty/hvc/hvc_xen.c
++++ b/drivers/tty/hvc/hvc_xen.c
+@@ -37,6 +37,8 @@ struct xencons_info {
+ struct xenbus_device *xbdev;
+ struct xencons_interface *intf;
+ unsigned int evtchn;
++ XENCONS_RING_IDX out_cons;
++ unsigned int out_cons_same;
+ struct hvc_struct *hvc;
+ int irq;
+ int vtermno;
+@@ -138,6 +140,8 @@ static int domU_read_console(uint32_t vtermno, char *buf, int len)
+ XENCONS_RING_IDX cons, prod;
+ int recv = 0;
+ struct xencons_info *xencons = vtermno_to_xencons(vtermno);
++ unsigned int eoiflag = 0;
++
+ if (xencons == NULL)
+ return -EINVAL;
+ intf = xencons->intf;
+@@ -157,7 +161,27 @@ static int domU_read_console(uint32_t vtermno, char *buf, int len)
+ mb(); /* read ring before consuming */
+ intf->in_cons = cons;
+
+- notify_daemon(xencons);
++ /*
++ * When to mark interrupt having been spurious:
++ * - there was no new data to be read, and
++ * - the backend did not consume some output bytes, and
++ * - the previous round with no read data didn't see consumed bytes
++ * (we might have a race with an interrupt being in flight while
++ * updating xencons->out_cons, so account for that by allowing one
++ * round without any visible reason)
++ */
++ if (intf->out_cons != xencons->out_cons) {
++ xencons->out_cons = intf->out_cons;
++ xencons->out_cons_same = 0;
++ }
++ if (recv) {
++ notify_daemon(xencons);
++ } else if (xencons->out_cons_same++ > 1) {
++ eoiflag = XEN_EOI_FLAG_SPURIOUS;
++ }
++
++ xen_irq_lateeoi(xencons->irq, eoiflag);
++
+ return recv;
+ }
+
+@@ -386,7 +410,7 @@ static int xencons_connect_backend(struct xenbus_device *dev,
+ if (ret)
+ return ret;
+ info->evtchn = evtchn;
+- irq = bind_evtchn_to_irq(evtchn);
++ irq = bind_interdomain_evtchn_to_irq_lateeoi(dev, evtchn);
+ if (irq < 0)
+ return irq;
+ info->irq = irq;
+@@ -550,7 +574,7 @@ static int __init xen_hvc_init(void)
+ return r;
+
+ info = vtermno_to_xencons(HVC_COOKIE);
+- info->irq = bind_evtchn_to_irq(info->evtchn);
++ info->irq = bind_evtchn_to_irq_lateeoi(info->evtchn);
+ }
+ if (info->irq < 0)
+ info->irq = 0; /* NO_IRQ */
+diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c
+index 1216f3985e18e..3b3e169c1f699 100644
+--- a/drivers/tty/mxser.c
++++ b/drivers/tty/mxser.c
+@@ -261,7 +261,6 @@ struct mxser_port {
+ unsigned int xmit_head;
+ unsigned int xmit_tail;
+ unsigned int xmit_cnt;
+- int closing;
+
+ spinlock_t slock;
+ };
+@@ -712,6 +711,7 @@ static int mxser_activate(struct tty_port *port, struct tty_struct *tty)
+ struct mxser_port *info = container_of(port, struct mxser_port, port);
+ unsigned long page;
+ unsigned long flags;
++ int ret;
+
+ page = __get_free_page(GFP_KERNEL);
+ if (!page)
+@@ -721,9 +721,9 @@ static int mxser_activate(struct tty_port *port, struct tty_struct *tty)
+
+ if (!info->type) {
+ set_bit(TTY_IO_ERROR, &tty->flags);
+- free_page(page);
+ spin_unlock_irqrestore(&info->slock, flags);
+- return 0;
++ ret = 0;
++ goto err_free_xmit;
+ }
+ info->port.xmit_buf = (unsigned char *) page;
+
+@@ -749,8 +749,10 @@ static int mxser_activate(struct tty_port *port, struct tty_struct *tty)
+ if (capable(CAP_SYS_ADMIN)) {
+ set_bit(TTY_IO_ERROR, &tty->flags);
+ return 0;
+- } else
+- return -ENODEV;
++ }
++
++ ret = -ENODEV;
++ goto err_free_xmit;
+ }
+
+ /*
+@@ -795,6 +797,10 @@ static int mxser_activate(struct tty_port *port, struct tty_struct *tty)
+ spin_unlock_irqrestore(&info->slock, flags);
+
+ return 0;
++err_free_xmit:
++ free_page(page);
++ info->port.xmit_buf = NULL;
++ return ret;
+ }
+
+ /*
+@@ -923,7 +929,6 @@ static void mxser_close(struct tty_struct *tty, struct file *filp)
+ return;
+ if (tty_port_close_start(port, tty, filp) == 0)
+ return;
+- info->closing = 1;
+ mutex_lock(&port->mutex);
+ mxser_close_port(port);
+ mxser_flush_buffer(tty);
+@@ -932,7 +937,6 @@ static void mxser_close(struct tty_struct *tty, struct file *filp)
+ mxser_shutdown_port(port);
+ tty_port_set_initialized(port, 0);
+ mutex_unlock(&port->mutex);
+- info->closing = 0;
+ /* Right now the tty_port set is done outside of the close_end helper
+ as we don't yet have everyone using refcounts */
+ tty_port_close_end(port, tty);
+@@ -1693,7 +1697,7 @@ static bool mxser_port_isr(struct mxser_port *port)
+
+ iir &= MOXA_MUST_IIR_MASK;
+ tty = tty_port_tty_get(&port->port);
+- if (!tty || port->closing || !tty_port_initialized(&port->port)) {
++ if (!tty) {
+ status = inb(port->ioaddr + UART_LSR);
+ outb(MOXA_MUST_FCR_GDA_MODE_ENABLE | UART_FCR_ENABLE_FIFO |
+ UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT,
+diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c
+index 1d92d2a848894..1767503de7446 100644
+--- a/drivers/tty/n_gsm.c
++++ b/drivers/tty/n_gsm.c
+@@ -73,6 +73,8 @@ module_param(debug, int, 0600);
+ */
+ #define MAX_MRU 1500
+ #define MAX_MTU 1500
++/* SOF, ADDR, CTRL, LEN1, LEN2, ..., FCS, EOF */
++#define PROT_OVERHEAD 7
+ #define GSM_NET_TX_TIMEOUT (HZ*10)
+
+ /*
+@@ -231,6 +233,7 @@ struct gsm_mux {
+ int initiator; /* Did we initiate connection */
+ bool dead; /* Has the mux been shut down */
+ struct gsm_dlci *dlci[NUM_DLCI];
++ int old_c_iflag; /* termios c_iflag value before attach */
+ bool constipated; /* Asked by remote to shut up */
+
+ spinlock_t tx_lock;
+@@ -318,6 +321,7 @@ static struct tty_driver *gsm_tty_driver;
+ #define GSM1_ESCAPE_BITS 0x20
+ #define XON 0x11
+ #define XOFF 0x13
++#define ISO_IEC_646_MASK 0x7F
+
+ static const struct tty_port_operations gsm_port_ops;
+
+@@ -434,7 +438,7 @@ static u8 gsm_encode_modem(const struct gsm_dlci *dlci)
+ modembits |= MDM_RTR;
+ if (dlci->modem_tx & TIOCM_RI)
+ modembits |= MDM_IC;
+- if (dlci->modem_tx & TIOCM_CD)
++ if (dlci->modem_tx & TIOCM_CD || dlci->gsm->initiator)
+ modembits |= MDM_DV;
+ return modembits;
+ }
+@@ -527,7 +531,8 @@ static int gsm_stuff_frame(const u8 *input, u8 *output, int len)
+ int olen = 0;
+ while (len--) {
+ if (*input == GSM1_SOF || *input == GSM1_ESCAPE
+- || *input == XON || *input == XOFF) {
++ || (*input & ISO_IEC_646_MASK) == XON
++ || (*input & ISO_IEC_646_MASK) == XOFF) {
+ *output++ = GSM1_ESCAPE;
+ *output++ = *input++ ^ GSM1_ESCAPE_BITS;
+ olen++;
+@@ -818,7 +823,7 @@ static int gsm_dlci_data_output(struct gsm_mux *gsm, struct gsm_dlci *dlci)
+ break;
+ case 2: /* Unstructed with modem bits.
+ Always one byte as we never send inline break data */
+- *dp++ = gsm_encode_modem(dlci);
++ *dp++ = (gsm_encode_modem(dlci) << 1) | EA;
+ break;
+ }
+ WARN_ON(kfifo_out_locked(&dlci->fifo, dp , len, &dlci->lock) != len);
+@@ -1007,25 +1012,25 @@ static void gsm_control_reply(struct gsm_mux *gsm, int cmd, const u8 *data,
+ * @tty: virtual tty bound to the DLCI
+ * @dlci: DLCI to affect
+ * @modem: modem bits (full EA)
+- * @clen: command length
++ * @slen: number of signal octets
+ *
+ * Used when a modem control message or line state inline in adaption
+ * layer 2 is processed. Sort out the local modem state and throttles
+ */
+
+ static void gsm_process_modem(struct tty_struct *tty, struct gsm_dlci *dlci,
+- u32 modem, int clen)
++ u32 modem, int slen)
+ {
+ int mlines = 0;
+ u8 brk = 0;
+ int fc;
+
+- /* The modem status command can either contain one octet (v.24 signals)
+- or two octets (v.24 signals + break signals). The length field will
+- either be 2 or 3 respectively. This is specified in section
+- 5.4.6.3.7 of the 27.010 mux spec. */
++ /* The modem status command can either contain one octet (V.24 signals)
++ * or two octets (V.24 signals + break signals). This is specified in
++ * section 5.4.6.3.7 of the 07.10 mux spec.
++ */
+
+- if (clen == 2)
++ if (slen == 1)
+ modem = modem & 0x7f;
+ else {
+ brk = modem & 0x7f;
+@@ -1079,9 +1084,9 @@ static void gsm_control_modem(struct gsm_mux *gsm, const u8 *data, int clen)
+ {
+ unsigned int addr = 0;
+ unsigned int modem = 0;
+- unsigned int brk = 0;
+ struct gsm_dlci *dlci;
+ int len = clen;
++ int slen;
+ const u8 *dp = data;
+ struct tty_struct *tty;
+
+@@ -1101,23 +1106,15 @@ static void gsm_control_modem(struct gsm_mux *gsm, const u8 *data, int clen)
+ return;
+ dlci = gsm->dlci[addr];
+
++ slen = len;
+ while (gsm_read_ea(&modem, *dp++) == 0) {
+ len--;
+ if (len == 0)
+ return;
+ }
+ len--;
+- if (len > 0) {
+- while (gsm_read_ea(&brk, *dp++) == 0) {
+- len--;
+- if (len == 0)
+- return;
+- }
+- modem <<= 7;
+- modem |= (brk & 0x7f);
+- }
+ tty = tty_port_tty_get(&dlci->port);
+- gsm_process_modem(tty, dlci, modem, clen);
++ gsm_process_modem(tty, dlci, modem, slen - len);
+ if (tty) {
+ tty_wakeup(tty);
+ tty_kref_put(tty);
+@@ -1296,11 +1293,12 @@ static void gsm_control_response(struct gsm_mux *gsm, unsigned int command,
+
+ static void gsm_control_transmit(struct gsm_mux *gsm, struct gsm_control *ctrl)
+ {
+- struct gsm_msg *msg = gsm_data_alloc(gsm, 0, ctrl->len + 1, gsm->ftype);
++ struct gsm_msg *msg = gsm_data_alloc(gsm, 0, ctrl->len + 2, gsm->ftype);
+ if (msg == NULL)
+ return;
+- msg->data[0] = (ctrl->cmd << 1) | 2 | EA; /* command */
+- memcpy(msg->data + 1, ctrl->data, ctrl->len);
++ msg->data[0] = (ctrl->cmd << 1) | CR | EA; /* command */
++ msg->data[1] = (ctrl->len << 1) | EA;
++ memcpy(msg->data + 2, ctrl->data, ctrl->len);
+ gsm_data_queue(gsm->dlci[0], msg);
+ }
+
+@@ -1323,7 +1321,6 @@ static void gsm_control_retransmit(struct timer_list *t)
+ spin_lock_irqsave(&gsm->control_lock, flags);
+ ctrl = gsm->pending_cmd;
+ if (ctrl) {
+- gsm->cretries--;
+ if (gsm->cretries == 0) {
+ gsm->pending_cmd = NULL;
+ ctrl->error = -ETIMEDOUT;
+@@ -1332,6 +1329,7 @@ static void gsm_control_retransmit(struct timer_list *t)
+ wake_up(&gsm->event);
+ return;
+ }
++ gsm->cretries--;
+ gsm_control_transmit(gsm, ctrl);
+ mod_timer(&gsm->t2_timer, jiffies + gsm->t2 * HZ / 100);
+ }
+@@ -1372,7 +1370,7 @@ retry:
+
+ /* If DLCI0 is in ADM mode skip retries, it won't respond */
+ if (gsm->dlci[0]->mode == DLCI_MODE_ADM)
+- gsm->cretries = 1;
++ gsm->cretries = 0;
+ else
+ gsm->cretries = gsm->n2;
+
+@@ -1420,13 +1418,20 @@ static int gsm_control_wait(struct gsm_mux *gsm, struct gsm_control *control)
+
+ static void gsm_dlci_close(struct gsm_dlci *dlci)
+ {
++ unsigned long flags;
++
+ del_timer(&dlci->t1);
+ if (debug & 8)
+ pr_debug("DLCI %d goes closed.\n", dlci->addr);
+ dlci->state = DLCI_CLOSED;
+ if (dlci->addr != 0) {
+ tty_port_tty_hangup(&dlci->port, false);
++ spin_lock_irqsave(&dlci->lock, flags);
+ kfifo_reset(&dlci->fifo);
++ spin_unlock_irqrestore(&dlci->lock, flags);
++ /* Ensure that gsmtty_open() can return. */
++ tty_port_set_initialized(&dlci->port, 0);
++ wake_up_interruptible(&dlci->port.open_wait);
+ } else
+ dlci->gsm->dead = true;
+ wake_up(&dlci->gsm->event);
+@@ -1486,7 +1491,7 @@ static void gsm_dlci_t1(struct timer_list *t)
+ dlci->mode = DLCI_MODE_ADM;
+ gsm_dlci_open(dlci);
+ } else {
+- gsm_dlci_close(dlci);
++ gsm_dlci_begin_close(dlci); /* prevent half open link */
+ }
+
+ break;
+@@ -1565,6 +1570,7 @@ static void gsm_dlci_data(struct gsm_dlci *dlci, const u8 *data, int clen)
+ struct tty_struct *tty;
+ unsigned int modem = 0;
+ int len = clen;
++ int slen = 0;
+
+ if (debug & 16)
+ pr_debug("%d bytes for tty\n", len);
+@@ -1577,12 +1583,16 @@ static void gsm_dlci_data(struct gsm_dlci *dlci, const u8 *data, int clen)
+ case 2: /* Asynchronous serial with line state in each frame */
+ while (gsm_read_ea(&modem, *data++) == 0) {
+ len--;
++ slen++;
+ if (len == 0)
+ return;
+ }
++ len--;
++ slen++;
+ tty = tty_port_tty_get(port);
+ if (tty) {
+- gsm_process_modem(tty, dlci, modem, clen);
++ gsm_process_modem(tty, dlci, modem, slen);
++ tty_wakeup(tty);
+ tty_kref_put(tty);
+ }
+ fallthrough;
+@@ -1720,7 +1730,12 @@ static void gsm_dlci_release(struct gsm_dlci *dlci)
+ gsm_destroy_network(dlci);
+ mutex_unlock(&dlci->mutex);
+
+- tty_hangup(tty);
++ /* We cannot use tty_hangup() because in tty_kref_put() the tty
++ * driver assumes that the hangup queue is free and reuses it to
++ * queue release_one_tty() -> NULL pointer panic in
++ * process_one_work().
++ */
++ tty_vhangup(tty);
+
+ tty_port_tty_set(&dlci->port, NULL);
+ tty_kref_put(tty);
+@@ -1803,7 +1818,6 @@ static void gsm_queue(struct gsm_mux *gsm)
+ gsm_response(gsm, address, UA);
+ gsm_dlci_close(dlci);
+ break;
+- case UA:
+ case UA|PF:
+ if (cr == 0 || dlci == NULL)
+ break;
+@@ -1947,6 +1961,16 @@ static void gsm0_receive(struct gsm_mux *gsm, unsigned char c)
+
+ static void gsm1_receive(struct gsm_mux *gsm, unsigned char c)
+ {
++ /* handle XON/XOFF */
++ if ((c & ISO_IEC_646_MASK) == XON) {
++ gsm->constipated = true;
++ return;
++ } else if ((c & ISO_IEC_646_MASK) == XOFF) {
++ gsm->constipated = false;
++ /* Kick the link in case it is idling */
++ gsm_data_kick(gsm, NULL);
++ return;
++ }
+ if (c == GSM1_SOF) {
+ /* EOF is only valid in frame if we have got to the data state
+ and received at least one byte (the FCS) */
+@@ -1961,7 +1985,8 @@ static void gsm1_receive(struct gsm_mux *gsm, unsigned char c)
+ }
+ /* Any partial frame was a runt so go back to start */
+ if (gsm->state != GSM_START) {
+- gsm->malformed++;
++ if (gsm->state != GSM_SEARCH)
++ gsm->malformed++;
+ gsm->state = GSM_START;
+ }
+ /* A SOF in GSM_START means we are still reading idling or
+@@ -2033,74 +2058,43 @@ static void gsm_error(struct gsm_mux *gsm,
+ gsm->io_error++;
+ }
+
+-static int gsm_disconnect(struct gsm_mux *gsm)
+-{
+- struct gsm_dlci *dlci = gsm->dlci[0];
+- struct gsm_control *gc;
+-
+- if (!dlci)
+- return 0;
+-
+- /* In theory disconnecting DLCI 0 is sufficient but for some
+- modems this is apparently not the case. */
+- gc = gsm_control_send(gsm, CMD_CLD, NULL, 0);
+- if (gc)
+- gsm_control_wait(gsm, gc);
+-
+- del_timer_sync(&gsm->t2_timer);
+- /* Now we are sure T2 has stopped */
+-
+- gsm_dlci_begin_close(dlci);
+- wait_event_interruptible(gsm->event,
+- dlci->state == DLCI_CLOSED);
+-
+- if (signal_pending(current))
+- return -EINTR;
+-
+- return 0;
+-}
+-
+ /**
+ * gsm_cleanup_mux - generic GSM protocol cleanup
+ * @gsm: our mux
++ * @disc: disconnect link?
+ *
+ * Clean up the bits of the mux which are the same for all framing
+ * protocols. Remove the mux from the mux table, stop all the timers
+ * and then shut down each device hanging up the channels as we go.
+ */
+
+-static void gsm_cleanup_mux(struct gsm_mux *gsm)
++static void gsm_cleanup_mux(struct gsm_mux *gsm, bool disc)
+ {
+ int i;
+ struct gsm_dlci *dlci = gsm->dlci[0];
+ struct gsm_msg *txq, *ntxq;
+
+ gsm->dead = true;
++ mutex_lock(&gsm->mutex);
+
+- spin_lock(&gsm_mux_lock);
+- for (i = 0; i < MAX_MUX; i++) {
+- if (gsm_mux[i] == gsm) {
+- gsm_mux[i] = NULL;
+- break;
++ if (dlci) {
++ if (disc && dlci->state != DLCI_CLOSED) {
++ gsm_dlci_begin_close(dlci);
++ wait_event(gsm->event, dlci->state == DLCI_CLOSED);
+ }
++ dlci->dead = true;
+ }
+- spin_unlock(&gsm_mux_lock);
+- /* open failed before registering => nothing to do */
+- if (i == MAX_MUX)
+- return;
+
++ /* Finish outstanding timers, making sure they are done */
+ del_timer_sync(&gsm->t2_timer);
+- /* Now we are sure T2 has stopped */
+- if (dlci)
+- dlci->dead = true;
+
+- /* Free up any link layer users */
+- mutex_lock(&gsm->mutex);
+- for (i = 0; i < NUM_DLCI; i++)
++ /* Free up any link layer users and finally the control channel */
++ for (i = NUM_DLCI - 1; i >= 0; i--)
+ if (gsm->dlci[i])
+ gsm_dlci_release(gsm->dlci[i]);
+ mutex_unlock(&gsm->mutex);
+ /* Now wipe the queues */
++ tty_ldisc_flush(gsm->tty);
+ list_for_each_entry_safe(txq, ntxq, &gsm->tx_list, list)
+ kfree(txq);
+ INIT_LIST_HEAD(&gsm->tx_list);
+@@ -2118,7 +2112,6 @@ static void gsm_cleanup_mux(struct gsm_mux *gsm)
+ static int gsm_activate_mux(struct gsm_mux *gsm)
+ {
+ struct gsm_dlci *dlci;
+- int i = 0;
+
+ timer_setup(&gsm->t2_timer, gsm_control_retransmit, 0);
+ init_waitqueue_head(&gsm->event);
+@@ -2130,18 +2123,6 @@ static int gsm_activate_mux(struct gsm_mux *gsm)
+ else
+ gsm->receive = gsm1_receive;
+
+- spin_lock(&gsm_mux_lock);
+- for (i = 0; i < MAX_MUX; i++) {
+- if (gsm_mux[i] == NULL) {
+- gsm->num = i;
+- gsm_mux[i] = gsm;
+- break;
+- }
+- }
+- spin_unlock(&gsm_mux_lock);
+- if (i == MAX_MUX)
+- return -EBUSY;
+-
+ dlci = gsm_dlci_alloc(gsm, 0);
+ if (dlci == NULL)
+ return -ENOMEM;
+@@ -2157,6 +2138,15 @@ static int gsm_activate_mux(struct gsm_mux *gsm)
+ */
+ static void gsm_free_mux(struct gsm_mux *gsm)
+ {
++ int i;
++
++ for (i = 0; i < MAX_MUX; i++) {
++ if (gsm == gsm_mux[i]) {
++ gsm_mux[i] = NULL;
++ break;
++ }
++ }
++ mutex_destroy(&gsm->mutex);
+ kfree(gsm->txframe);
+ kfree(gsm->buf);
+ kfree(gsm);
+@@ -2176,12 +2166,20 @@ static void gsm_free_muxr(struct kref *ref)
+
+ static inline void mux_get(struct gsm_mux *gsm)
+ {
++ unsigned long flags;
++
++ spin_lock_irqsave(&gsm_mux_lock, flags);
+ kref_get(&gsm->ref);
++ spin_unlock_irqrestore(&gsm_mux_lock, flags);
+ }
+
+ static inline void mux_put(struct gsm_mux *gsm)
+ {
++ unsigned long flags;
++
++ spin_lock_irqsave(&gsm_mux_lock, flags);
+ kref_put(&gsm->ref, gsm_free_muxr);
++ spin_unlock_irqrestore(&gsm_mux_lock, flags);
+ }
+
+ static inline unsigned int mux_num_to_base(struct gsm_mux *gsm)
+@@ -2202,6 +2200,7 @@ static inline unsigned int mux_line_to_num(unsigned int line)
+
+ static struct gsm_mux *gsm_alloc_mux(void)
+ {
++ int i;
+ struct gsm_mux *gsm = kzalloc(sizeof(struct gsm_mux), GFP_KERNEL);
+ if (gsm == NULL)
+ return NULL;
+@@ -2210,7 +2209,7 @@ static struct gsm_mux *gsm_alloc_mux(void)
+ kfree(gsm);
+ return NULL;
+ }
+- gsm->txframe = kmalloc(2 * MAX_MRU + 2, GFP_KERNEL);
++ gsm->txframe = kmalloc(2 * (MAX_MTU + PROT_OVERHEAD - 1), GFP_KERNEL);
+ if (gsm->txframe == NULL) {
+ kfree(gsm->buf);
+ kfree(gsm);
+@@ -2231,6 +2230,26 @@ static struct gsm_mux *gsm_alloc_mux(void)
+ gsm->mtu = 64;
+ gsm->dead = true; /* Avoid early tty opens */
+
++ /* Store the instance to the mux array or abort if no space is
++ * available.
++ */
++ spin_lock(&gsm_mux_lock);
++ for (i = 0; i < MAX_MUX; i++) {
++ if (!gsm_mux[i]) {
++ gsm_mux[i] = gsm;
++ gsm->num = i;
++ break;
++ }
++ }
++ spin_unlock(&gsm_mux_lock);
++ if (i == MAX_MUX) {
++ mutex_destroy(&gsm->mutex);
++ kfree(gsm->txframe);
++ kfree(gsm->buf);
++ kfree(gsm);
++ return NULL;
++ }
++
+ return gsm;
+ }
+
+@@ -2257,6 +2276,7 @@ static void gsm_copy_config_values(struct gsm_mux *gsm,
+
+ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c)
+ {
++ int ret = 0;
+ int need_close = 0;
+ int need_restart = 0;
+
+@@ -2266,7 +2286,7 @@ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c)
+ /* Check the MRU/MTU range looks sane */
+ if (c->mru > MAX_MRU || c->mtu > MAX_MTU || c->mru < 8 || c->mtu < 8)
+ return -EINVAL;
+- if (c->n2 < 3)
++ if (c->n2 > 255)
+ return -EINVAL;
+ if (c->encapsulation > 1) /* Basic, advanced, no I */
+ return -EINVAL;
+@@ -2297,19 +2317,11 @@ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c)
+
+ /*
+ * Close down what is needed, restart and initiate the new
+- * configuration
++ * configuration. On the first time there is no DLCI[0]
++ * and closing or cleaning up is not necessary.
+ */
+-
+- if (need_close || need_restart) {
+- int ret;
+-
+- ret = gsm_disconnect(gsm);
+-
+- if (ret)
+- return ret;
+- }
+- if (need_restart)
+- gsm_cleanup_mux(gsm);
++ if (need_close || need_restart)
++ gsm_cleanup_mux(gsm, true);
+
+ gsm->initiator = c->initiator;
+ gsm->mru = c->mru;
+@@ -2332,10 +2344,13 @@ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c)
+ * FIXME: We need to separate activation/deactivation from adding
+ * and removing from the mux array
+ */
+- if (need_restart)
+- gsm_activate_mux(gsm);
+- if (gsm->initiator && need_close)
+- gsm_dlci_begin_open(gsm->dlci[0]);
++ if (gsm->dead) {
++ ret = gsm_activate_mux(gsm);
++ if (ret)
++ return ret;
++ if (gsm->initiator)
++ gsm_dlci_begin_open(gsm->dlci[0]);
++ }
+ return 0;
+ }
+
+@@ -2378,6 +2393,9 @@ static int gsmld_attach_gsm(struct tty_struct *tty, struct gsm_mux *gsm)
+ int ret, i;
+
+ gsm->tty = tty_kref_get(tty);
++ /* Turn off tty XON/XOFF handling to handle it explicitly. */
++ gsm->old_c_iflag = tty->termios.c_iflag;
++ tty->termios.c_iflag &= (IXON | IXOFF);
+ ret = gsm_activate_mux(gsm);
+ if (ret != 0)
+ tty_kref_put(gsm->tty);
+@@ -2418,7 +2436,8 @@ static void gsmld_detach_gsm(struct tty_struct *tty, struct gsm_mux *gsm)
+ WARN_ON(tty != gsm->tty);
+ for (i = 1; i < NUM_DLCI; i++)
+ tty_unregister_device(gsm_tty_driver, base + i);
+- gsm_cleanup_mux(gsm);
++ /* Restore tty XON/XOFF handling. */
++ gsm->tty->termios.c_iflag = gsm->old_c_iflag;
+ tty_kref_put(gsm->tty);
+ gsm->tty = NULL;
+ }
+@@ -2483,6 +2502,12 @@ static void gsmld_close(struct tty_struct *tty)
+ {
+ struct gsm_mux *gsm = tty->disc_data;
+
++ /* The ldisc locks and closes the port before calling our close. This
++ * means we have no way to do a proper disconnect. We will not bother
++ * to do one.
++ */
++ gsm_cleanup_mux(gsm, false);
++
+ gsmld_detach_gsm(tty, gsm);
+
+ gsmld_flush_buffer(tty);
+@@ -2521,7 +2546,7 @@ static int gsmld_open(struct tty_struct *tty)
+
+ ret = gsmld_attach_gsm(tty, gsm);
+ if (ret != 0) {
+- gsm_cleanup_mux(gsm);
++ gsm_cleanup_mux(gsm, false);
+ mux_put(gsm);
+ }
+ return ret;
+@@ -2880,19 +2905,19 @@ static struct tty_ldisc_ops tty_ldisc_packet = {
+
+ static int gsmtty_modem_update(struct gsm_dlci *dlci, u8 brk)
+ {
+- u8 modembits[5];
++ u8 modembits[3];
+ struct gsm_control *ctrl;
+ int len = 2;
+
+- if (brk)
++ modembits[0] = (dlci->addr << 2) | 2 | EA; /* DLCI, Valid, EA */
++ if (!brk) {
++ modembits[1] = (gsm_encode_modem(dlci) << 1) | EA;
++ } else {
++ modembits[1] = gsm_encode_modem(dlci) << 1;
++ modembits[2] = (brk << 4) | 2 | EA; /* Length, Break, EA */
+ len++;
+-
+- modembits[0] = len << 1 | EA; /* Data bytes */
+- modembits[1] = dlci->addr << 2 | 3; /* DLCI, EA, 1 */
+- modembits[2] = gsm_encode_modem(dlci) << 1 | EA;
+- if (brk)
+- modembits[3] = brk << 4 | 2 | EA; /* Valid, EA */
+- ctrl = gsm_control_send(dlci->gsm, CMD_MSC, modembits, len + 1);
++ }
++ ctrl = gsm_control_send(dlci->gsm, CMD_MSC, modembits, len);
+ if (ctrl == NULL)
+ return -ENOMEM;
+ return gsm_control_wait(dlci->gsm, ctrl);
+@@ -3077,13 +3102,17 @@ static unsigned int gsmtty_chars_in_buffer(struct tty_struct *tty)
+ static void gsmtty_flush_buffer(struct tty_struct *tty)
+ {
+ struct gsm_dlci *dlci = tty->driver_data;
++ unsigned long flags;
++
+ if (dlci->state == DLCI_CLOSED)
+ return;
+ /* Caution needed: If we implement reliable transport classes
+ then the data being transmitted can't simply be junked once
+ it has first hit the stack. Until then we can just blow it
+ away */
++ spin_lock_irqsave(&dlci->lock, flags);
+ kfifo_reset(&dlci->fifo);
++ spin_unlock_irqrestore(&dlci->lock, flags);
+ /* Need to unhook this DLCI from the transmit queue logic */
+ }
+
+@@ -3173,9 +3202,9 @@ static void gsmtty_throttle(struct tty_struct *tty)
+ if (dlci->state == DLCI_CLOSED)
+ return;
+ if (C_CRTSCTS(tty))
+- dlci->modem_tx &= ~TIOCM_DTR;
++ dlci->modem_tx &= ~TIOCM_RTS;
+ dlci->throttled = true;
+- /* Send an MSC with DTR cleared */
++ /* Send an MSC with RTS cleared */
+ gsmtty_modem_update(dlci, 0);
+ }
+
+@@ -3185,9 +3214,9 @@ static void gsmtty_unthrottle(struct tty_struct *tty)
+ if (dlci->state == DLCI_CLOSED)
+ return;
+ if (C_CRTSCTS(tty))
+- dlci->modem_tx |= TIOCM_DTR;
++ dlci->modem_tx |= TIOCM_RTS;
+ dlci->throttled = false;
+- /* Send an MSC with DTR set */
++ /* Send an MSC with RTS set */
+ gsmtty_modem_update(dlci, 0);
+ }
+
+diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c
+index 580a37b3fe1b9..166f651fde33a 100644
+--- a/drivers/tty/n_hdlc.c
++++ b/drivers/tty/n_hdlc.c
+@@ -140,6 +140,8 @@ struct n_hdlc {
+ struct n_hdlc_buf_list rx_buf_list;
+ struct n_hdlc_buf_list tx_free_buf_list;
+ struct n_hdlc_buf_list rx_free_buf_list;
++ struct work_struct write_work;
++ struct tty_struct *tty_for_write_work;
+ };
+
+ /*
+@@ -154,6 +156,7 @@ static struct n_hdlc_buf *n_hdlc_buf_get(struct n_hdlc_buf_list *list);
+ /* Local functions */
+
+ static struct n_hdlc *n_hdlc_alloc(void);
++static void n_hdlc_tty_write_work(struct work_struct *work);
+
+ /* max frame size for memory allocations */
+ static int maxframe = 4096;
+@@ -210,6 +213,8 @@ static void n_hdlc_tty_close(struct tty_struct *tty)
+ wake_up_interruptible(&tty->read_wait);
+ wake_up_interruptible(&tty->write_wait);
+
++ cancel_work_sync(&n_hdlc->write_work);
++
+ n_hdlc_free_buf_list(&n_hdlc->rx_free_buf_list);
+ n_hdlc_free_buf_list(&n_hdlc->tx_free_buf_list);
+ n_hdlc_free_buf_list(&n_hdlc->rx_buf_list);
+@@ -241,6 +246,8 @@ static int n_hdlc_tty_open(struct tty_struct *tty)
+ return -ENFILE;
+ }
+
++ INIT_WORK(&n_hdlc->write_work, n_hdlc_tty_write_work);
++ n_hdlc->tty_for_write_work = tty;
+ tty->disc_data = n_hdlc;
+ tty->receive_room = 65536;
+
+@@ -334,6 +341,20 @@ check_again:
+ goto check_again;
+ } /* end of n_hdlc_send_frames() */
+
++/**
++ * n_hdlc_tty_write_work - Asynchronous callback for transmit wakeup
++ * @work: pointer to work_struct
++ *
++ * Called when low level device driver can accept more send data.
++ */
++static void n_hdlc_tty_write_work(struct work_struct *work)
++{
++ struct n_hdlc *n_hdlc = container_of(work, struct n_hdlc, write_work);
++ struct tty_struct *tty = n_hdlc->tty_for_write_work;
++
++ n_hdlc_send_frames(n_hdlc, tty);
++} /* end of n_hdlc_tty_write_work() */
++
+ /**
+ * n_hdlc_tty_wakeup - Callback for transmit wakeup
+ * @tty: pointer to associated tty instance data
+@@ -344,7 +365,7 @@ static void n_hdlc_tty_wakeup(struct tty_struct *tty)
+ {
+ struct n_hdlc *n_hdlc = tty->disc_data;
+
+- n_hdlc_send_frames(n_hdlc, tty);
++ schedule_work(&n_hdlc->write_work);
+ } /* end of n_hdlc_tty_wakeup() */
+
+ /**
+diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c
+index 0ec93f1a61f5d..de5b45de50402 100644
+--- a/drivers/tty/n_tty.c
++++ b/drivers/tty/n_tty.c
+@@ -1369,7 +1369,7 @@ handle_newline:
+ put_tty_queue(c, ldata);
+ smp_store_release(&ldata->canon_head, ldata->read_head);
+ kill_fasync(&tty->fasync, SIGIO, POLL_IN);
+- wake_up_interruptible_poll(&tty->read_wait, EPOLLIN);
++ wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM);
+ return;
+ }
+ }
+@@ -1589,7 +1589,7 @@ static void __receive_buf(struct tty_struct *tty, const unsigned char *cp,
+
+ if (read_cnt(ldata)) {
+ kill_fasync(&tty->fasync, SIGIO, POLL_IN);
+- wake_up_interruptible_poll(&tty->read_wait, EPOLLIN);
++ wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM);
+ }
+ }
+
+@@ -1963,7 +1963,7 @@ static bool canon_copy_from_read_buf(struct tty_struct *tty,
+ return false;
+
+ canon_head = smp_load_acquire(&ldata->canon_head);
+- n = min(*nr + 1, canon_head - ldata->read_tail);
++ n = min(*nr, canon_head - ldata->read_tail);
+
+ tail = ldata->read_tail & (N_TTY_BUF_SIZE - 1);
+ size = min_t(size_t, tail + n, N_TTY_BUF_SIZE);
+@@ -1985,10 +1985,8 @@ static bool canon_copy_from_read_buf(struct tty_struct *tty,
+ n += N_TTY_BUF_SIZE;
+ c = n + found;
+
+- if (!found || read_buf(ldata, eol) != __DISABLED_CHAR) {
+- c = min(*nr, c);
++ if (!found || read_buf(ldata, eol) != __DISABLED_CHAR)
+ n = c;
+- }
+
+ n_tty_trace("%s: eol:%zu found:%d n:%zu c:%zu tail:%zu more:%zu\n",
+ __func__, eol, found, n, c, tail, more);
+diff --git a/drivers/tty/serial/8250/8250_aspeed_vuart.c b/drivers/tty/serial/8250/8250_aspeed_vuart.c
+index 2350fb3bb5e4c..c2cecc6f47db4 100644
+--- a/drivers/tty/serial/8250/8250_aspeed_vuart.c
++++ b/drivers/tty/serial/8250/8250_aspeed_vuart.c
+@@ -487,7 +487,7 @@ static int aspeed_vuart_probe(struct platform_device *pdev)
+ port.port.irq = irq_of_parse_and_map(np, 0);
+ port.port.handle_irq = aspeed_vuart_handle_irq;
+ port.port.iotype = UPIO_MEM;
+- port.port.type = PORT_16550A;
++ port.port.type = PORT_ASPEED_VUART;
+ port.port.uartclk = clk;
+ port.port.flags = UPF_SHARE_IRQ | UPF_BOOT_AUTOCONF | UPF_IOREMAP
+ | UPF_FIXED_PORT | UPF_FIXED_TYPE | UPF_NO_THRE_TEST;
+diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c
+index 7f656fac503fe..0877cf24f7de0 100644
+--- a/drivers/tty/serial/8250/8250_bcm7271.c
++++ b/drivers/tty/serial/8250/8250_bcm7271.c
+@@ -237,6 +237,7 @@ struct brcmuart_priv {
+ u32 rx_err;
+ u32 rx_timeout;
+ u32 rx_abort;
++ u32 saved_mctrl;
+ };
+
+ static struct dentry *brcmuart_debugfs_root;
+@@ -1075,14 +1076,18 @@ static int brcmuart_probe(struct platform_device *pdev)
+ priv->rx_bufs = dma_alloc_coherent(dev,
+ priv->rx_size,
+ &priv->rx_addr, GFP_KERNEL);
+- if (!priv->rx_bufs)
++ if (!priv->rx_bufs) {
++ ret = -EINVAL;
+ goto err;
++ }
+ priv->tx_size = UART_XMIT_SIZE;
+ priv->tx_buf = dma_alloc_coherent(dev,
+ priv->tx_size,
+ &priv->tx_addr, GFP_KERNEL);
+- if (!priv->tx_buf)
++ if (!priv->tx_buf) {
++ ret = -EINVAL;
+ goto err;
++ }
+ }
+
+ ret = serial8250_register_8250_port(&up);
+@@ -1096,6 +1101,7 @@ static int brcmuart_probe(struct platform_device *pdev)
+ if (priv->dma_enabled) {
+ dma_irq = platform_get_irq_byname(pdev, "dma");
+ if (dma_irq < 0) {
++ ret = dma_irq;
+ dev_err(dev, "no IRQ resource info\n");
+ goto err1;
+ }
+@@ -1115,7 +1121,7 @@ err1:
+ err:
+ brcmuart_free_bufs(dev, priv);
+ brcmuart_arbitration(priv, 0);
+- return -ENODEV;
++ return ret;
+ }
+
+ static int brcmuart_remove(struct platform_device *pdev)
+@@ -1133,16 +1139,27 @@ static int brcmuart_remove(struct platform_device *pdev)
+ static int __maybe_unused brcmuart_suspend(struct device *dev)
+ {
+ struct brcmuart_priv *priv = dev_get_drvdata(dev);
++ struct uart_8250_port *up = serial8250_get_port(priv->line);
++ struct uart_port *port = &up->port;
+
+ serial8250_suspend_port(priv->line);
+ clk_disable_unprepare(priv->baud_mux_clk);
+
++ /*
++ * This will prevent resume from enabling RTS before the
++ * baud rate has been resored.
++ */
++ priv->saved_mctrl = port->mctrl;
++ port->mctrl = 0;
++
+ return 0;
+ }
+
+ static int __maybe_unused brcmuart_resume(struct device *dev)
+ {
+ struct brcmuart_priv *priv = dev_get_drvdata(dev);
++ struct uart_8250_port *up = serial8250_get_port(priv->line);
++ struct uart_port *port = &up->port;
+ int ret;
+
+ ret = clk_prepare_enable(priv->baud_mux_clk);
+@@ -1165,6 +1182,7 @@ static int __maybe_unused brcmuart_resume(struct device *dev)
+ start_rx_dma(serial8250_get_port(priv->line));
+ }
+ serial8250_resume_port(priv->line);
++ port->mctrl = priv->saved_mctrl;
+ return 0;
+ }
+
+diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c
+index 890fa7ddaa7f3..b3c3f7e5851ab 100644
+--- a/drivers/tty/serial/8250/8250_dma.c
++++ b/drivers/tty/serial/8250/8250_dma.c
+@@ -64,10 +64,19 @@ int serial8250_tx_dma(struct uart_8250_port *p)
+ struct uart_8250_dma *dma = p->dma;
+ struct circ_buf *xmit = &p->port.state->xmit;
+ struct dma_async_tx_descriptor *desc;
++ struct uart_port *up = &p->port;
+ int ret;
+
+- if (dma->tx_running)
++ if (dma->tx_running) {
++ if (up->x_char) {
++ dmaengine_pause(dma->txchan);
++ uart_xchar_out(up, UART_TX);
++ dmaengine_resume(dma->txchan);
++ }
+ return 0;
++ } else if (up->x_char) {
++ uart_xchar_out(up, UART_TX);
++ }
+
+ if (uart_tx_stopped(&p->port) || uart_circ_empty(xmit)) {
+ /* We have been called from __dma_tx_complete() */
+diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
+index a3a0154da567d..49559731bbcf1 100644
+--- a/drivers/tty/serial/8250/8250_dw.c
++++ b/drivers/tty/serial/8250/8250_dw.c
+@@ -726,7 +726,7 @@ static struct platform_driver dw8250_platform_driver = {
+ .name = "dw-apb-uart",
+ .pm = &dw8250_pm_ops,
+ .of_match_table = dw8250_of_match,
+- .acpi_match_table = ACPI_PTR(dw8250_acpi_match),
++ .acpi_match_table = dw8250_acpi_match,
+ },
+ .probe = dw8250_probe,
+ .remove = dw8250_remove,
+diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c
+index 31c9e83ea3cb2..251f0018ae8ca 100644
+--- a/drivers/tty/serial/8250/8250_fintek.c
++++ b/drivers/tty/serial/8250/8250_fintek.c
+@@ -290,25 +290,6 @@ static void fintek_8250_set_max_fifo(struct fintek_8250 *pdata)
+ }
+ }
+
+-static void fintek_8250_goto_highspeed(struct uart_8250_port *uart,
+- struct fintek_8250 *pdata)
+-{
+- sio_write_reg(pdata, LDN, pdata->index);
+-
+- switch (pdata->pid) {
+- case CHIP_ID_F81966:
+- case CHIP_ID_F81866: /* set uart clock for high speed serial mode */
+- sio_write_mask_reg(pdata, F81866_UART_CLK,
+- F81866_UART_CLK_MASK,
+- F81866_UART_CLK_14_769MHZ);
+-
+- uart->port.uartclk = 921600 * 16;
+- break;
+- default: /* leave clock speed untouched */
+- break;
+- }
+-}
+-
+ static void fintek_8250_set_termios(struct uart_port *port,
+ struct ktermios *termios,
+ struct ktermios *old)
+@@ -430,7 +411,6 @@ static int probe_setup_port(struct fintek_8250 *pdata,
+
+ fintek_8250_set_irq_mode(pdata, level_mode);
+ fintek_8250_set_max_fifo(pdata);
+- fintek_8250_goto_highspeed(uart, pdata);
+
+ fintek_8250_exit_key(addr[i]);
+
+diff --git a/drivers/tty/serial/8250/8250_gsc.c b/drivers/tty/serial/8250/8250_gsc.c
+index 673cda3d011d0..948d0a1c6ae8e 100644
+--- a/drivers/tty/serial/8250/8250_gsc.c
++++ b/drivers/tty/serial/8250/8250_gsc.c
+@@ -26,7 +26,7 @@ static int __init serial_init_chip(struct parisc_device *dev)
+ unsigned long address;
+ int err;
+
+-#ifdef CONFIG_64BIT
++#if defined(CONFIG_64BIT) && defined(CONFIG_IOSAPIC)
+ if (!dev->irq && (dev->id.sversion == 0xad))
+ dev->irq = iosapic_serial_irq(dev);
+ #endif
+diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c
+index 848d81e3838c2..49ae73f4d3a04 100644
+--- a/drivers/tty/serial/8250/8250_lpss.c
++++ b/drivers/tty/serial/8250/8250_lpss.c
+@@ -121,8 +121,7 @@ static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+ {
+ struct dw_dma_slave *param = &lpss->dma_param;
+ struct pci_dev *pdev = to_pci_dev(port->dev);
+- unsigned int dma_devfn = PCI_DEVFN(PCI_SLOT(pdev->devfn), 0);
+- struct pci_dev *dma_dev = pci_get_slot(pdev->bus, dma_devfn);
++ struct pci_dev *dma_dev;
+
+ switch (pdev->device) {
+ case PCI_DEVICE_ID_INTEL_BYT_UART1:
+@@ -141,6 +140,8 @@ static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+ return -EINVAL;
+ }
+
++ dma_dev = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
++
+ param->dma_dev = &dma_dev->dev;
+ param->m_master = 0;
+ param->p_master = 1;
+@@ -156,6 +157,14 @@ static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+ return 0;
+ }
+
++static void byt_serial_exit(struct lpss8250 *lpss)
++{
++ struct dw_dma_slave *param = &lpss->dma_param;
++
++ /* Paired with pci_get_slot() in the byt_serial_setup() above */
++ put_device(param->dma_dev);
++}
++
+ static int ehl_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+ {
+ struct uart_8250_dma *dma = &lpss->data.dma;
+@@ -171,6 +180,13 @@ static int ehl_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+ return 0;
+ }
+
++static void ehl_serial_exit(struct lpss8250 *lpss)
++{
++ struct uart_8250_port *up = serial8250_get_port(lpss->data.line);
++
++ up->dma = NULL;
++}
++
+ #ifdef CONFIG_SERIAL_8250_DMA
+ static const struct dw_dma_platform_data qrk_serial_dma_pdata = {
+ .nr_channels = 2,
+@@ -345,8 +361,7 @@ static int lpss8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ return 0;
+
+ err_exit:
+- if (lpss->board->exit)
+- lpss->board->exit(lpss);
++ lpss->board->exit(lpss);
+ pci_free_irq_vectors(pdev);
+ return ret;
+ }
+@@ -357,8 +372,7 @@ static void lpss8250_remove(struct pci_dev *pdev)
+
+ serial8250_unregister_port(lpss->data.line);
+
+- if (lpss->board->exit)
+- lpss->board->exit(lpss);
++ lpss->board->exit(lpss);
+ pci_free_irq_vectors(pdev);
+ }
+
+@@ -366,12 +380,14 @@ static const struct lpss8250_board byt_board = {
+ .freq = 100000000,
+ .base_baud = 2764800,
+ .setup = byt_serial_setup,
++ .exit = byt_serial_exit,
+ };
+
+ static const struct lpss8250_board ehl_board = {
+ .freq = 200000000,
+ .base_baud = 12500000,
+ .setup = ehl_serial_setup,
++ .exit = ehl_serial_exit,
+ };
+
+ static const struct lpss8250_board qrk_board = {
+diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
+index efa0515139f8e..e6c1791609ddf 100644
+--- a/drivers/tty/serial/8250/8250_mid.c
++++ b/drivers/tty/serial/8250/8250_mid.c
+@@ -73,6 +73,11 @@ static int pnw_setup(struct mid8250 *mid, struct uart_port *p)
+ return 0;
+ }
+
++static void pnw_exit(struct mid8250 *mid)
++{
++ pci_dev_put(mid->dma_dev);
++}
++
+ static int tng_handle_irq(struct uart_port *p)
+ {
+ struct mid8250 *mid = p->private_data;
+@@ -124,6 +129,11 @@ static int tng_setup(struct mid8250 *mid, struct uart_port *p)
+ return 0;
+ }
+
++static void tng_exit(struct mid8250 *mid)
++{
++ pci_dev_put(mid->dma_dev);
++}
++
+ static int dnv_handle_irq(struct uart_port *p)
+ {
+ struct mid8250 *mid = p->private_data;
+@@ -330,9 +340,9 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+
+ pci_set_drvdata(pdev, mid);
+ return 0;
++
+ err:
+- if (mid->board->exit)
+- mid->board->exit(mid);
++ mid->board->exit(mid);
+ return ret;
+ }
+
+@@ -342,8 +352,7 @@ static void mid8250_remove(struct pci_dev *pdev)
+
+ serial8250_unregister_port(mid->line);
+
+- if (mid->board->exit)
+- mid->board->exit(mid);
++ mid->board->exit(mid);
+ }
+
+ static const struct mid8250_board pnw_board = {
+@@ -351,6 +360,7 @@ static const struct mid8250_board pnw_board = {
+ .freq = 50000000,
+ .base_baud = 115200,
+ .setup = pnw_setup,
++ .exit = pnw_exit,
+ };
+
+ static const struct mid8250_board tng_board = {
+@@ -358,6 +368,7 @@ static const struct mid8250_board tng_board = {
+ .freq = 38400000,
+ .base_baud = 1843200,
+ .setup = tng_setup,
++ .exit = tng_exit,
+ };
+
+ static const struct mid8250_board dnv_board = {
+diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c
+index fb65dc601b237..de48a58460f47 100644
+--- a/drivers/tty/serial/8250/8250_mtk.c
++++ b/drivers/tty/serial/8250/8250_mtk.c
+@@ -37,6 +37,7 @@
+ #define MTK_UART_IER_RTSI 0x40 /* Enable RTS Modem status interrupt */
+ #define MTK_UART_IER_CTSI 0x80 /* Enable CTS Modem status interrupt */
+
++#define MTK_UART_EFR 38 /* I/O: Extended Features Register */
+ #define MTK_UART_EFR_EN 0x10 /* Enable enhancement feature */
+ #define MTK_UART_EFR_RTS 0x40 /* Enable hardware rx flow control */
+ #define MTK_UART_EFR_CTS 0x80 /* Enable hardware tx flow control */
+@@ -53,6 +54,9 @@
+ #define MTK_UART_TX_TRIGGER 1
+ #define MTK_UART_RX_TRIGGER MTK_UART_RX_SIZE
+
++#define MTK_UART_XON1 40 /* I/O: Xon character 1 */
++#define MTK_UART_XOFF1 42 /* I/O: Xoff character 1 */
++
+ #ifdef CONFIG_SERIAL_8250_DMA
+ enum dma_rx_status {
+ DMA_RX_START = 0,
+@@ -169,7 +173,7 @@ static void mtk8250_dma_enable(struct uart_8250_port *up)
+ MTK_UART_DMA_EN_RX | MTK_UART_DMA_EN_TX);
+
+ serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+- serial_out(up, UART_EFR, UART_EFR_ECB);
++ serial_out(up, MTK_UART_EFR, UART_EFR_ECB);
+ serial_out(up, UART_LCR, lcr);
+
+ if (dmaengine_slave_config(dma->rxchan, &dma->rxconf) != 0)
+@@ -232,7 +236,7 @@ static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode)
+ int lcr = serial_in(up, UART_LCR);
+
+ serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+- serial_out(up, UART_EFR, UART_EFR_ECB);
++ serial_out(up, MTK_UART_EFR, UART_EFR_ECB);
+ serial_out(up, UART_LCR, lcr);
+ lcr = serial_in(up, UART_LCR);
+
+@@ -241,7 +245,7 @@ static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode)
+ serial_out(up, MTK_UART_ESCAPE_DAT, MTK_UART_ESCAPE_CHAR);
+ serial_out(up, MTK_UART_ESCAPE_EN, 0x00);
+ serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+- serial_out(up, UART_EFR, serial_in(up, UART_EFR) &
++ serial_out(up, MTK_UART_EFR, serial_in(up, MTK_UART_EFR) &
+ (~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK)));
+ serial_out(up, UART_LCR, lcr);
+ mtk8250_disable_intrs(up, MTK_UART_IER_XOFFI |
+@@ -255,8 +259,8 @@ static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode)
+ serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+
+ /*enable hw flow control*/
+- serial_out(up, UART_EFR, MTK_UART_EFR_HW_FC |
+- (serial_in(up, UART_EFR) &
++ serial_out(up, MTK_UART_EFR, MTK_UART_EFR_HW_FC |
++ (serial_in(up, MTK_UART_EFR) &
+ (~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK))));
+
+ serial_out(up, UART_LCR, lcr);
+@@ -270,12 +274,12 @@ static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode)
+ serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+
+ /*enable sw flow control */
+- serial_out(up, UART_EFR, MTK_UART_EFR_XON1_XOFF1 |
+- (serial_in(up, UART_EFR) &
++ serial_out(up, MTK_UART_EFR, MTK_UART_EFR_XON1_XOFF1 |
++ (serial_in(up, MTK_UART_EFR) &
+ (~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK))));
+
+- serial_out(up, UART_XON1, START_CHAR(port->state->port.tty));
+- serial_out(up, UART_XOFF1, STOP_CHAR(port->state->port.tty));
++ serial_out(up, MTK_UART_XON1, START_CHAR(port->state->port.tty));
++ serial_out(up, MTK_UART_XOFF1, STOP_CHAR(port->state->port.tty));
+ serial_out(up, UART_LCR, lcr);
+ mtk8250_disable_intrs(up, MTK_UART_IER_CTSI|MTK_UART_IER_RTSI);
+ mtk8250_enable_intrs(up, MTK_UART_IER_XOFFI);
+diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c
+index bce28729dd7bd..be8626234627e 100644
+--- a/drivers/tty/serial/8250/8250_of.c
++++ b/drivers/tty/serial/8250/8250_of.c
+@@ -83,8 +83,17 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
+ port->mapsize = resource_size(&resource);
+
+ /* Check for shifted address mapping */
+- if (of_property_read_u32(np, "reg-offset", &prop) == 0)
++ if (of_property_read_u32(np, "reg-offset", &prop) == 0) {
++ if (prop >= port->mapsize) {
++ dev_warn(&ofdev->dev, "reg-offset %u exceeds region size %pa\n",
++ prop, &port->mapsize);
++ ret = -EINVAL;
++ goto err_unprepare;
++ }
++
+ port->mapbase += prop;
++ port->mapsize -= prop;
++ }
+
+ port->iotype = UPIO_MEM;
+ if (of_property_read_u32(np, "reg-io-width", &prop) == 0) {
+diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
+index 726912b16a559..e7b9805903f4c 100644
+--- a/drivers/tty/serial/8250/8250_pci.c
++++ b/drivers/tty/serial/8250/8250_pci.c
+@@ -1349,29 +1349,33 @@ pericom_do_set_divisor(struct uart_port *port, unsigned int baud,
+ {
+ int scr;
+ int lcr;
+- int actual_baud;
+- int tolerance;
+
+- for (scr = 5 ; scr <= 15 ; scr++) {
+- actual_baud = 921600 * 16 / scr;
+- tolerance = actual_baud / 50;
++ for (scr = 16; scr > 4; scr--) {
++ unsigned int maxrate = port->uartclk / scr;
++ unsigned int divisor = max(maxrate / baud, 1U);
++ int delta = maxrate / divisor - baud;
+
+- if ((baud < actual_baud + tolerance) &&
+- (baud > actual_baud - tolerance)) {
++ if (baud > maxrate + baud / 50)
++ continue;
+
++ if (delta > baud / 50)
++ divisor++;
++
++ if (divisor > 0xffff)
++ continue;
++
++ /* Update delta due to possible divisor change */
++ delta = maxrate / divisor - baud;
++ if (abs(delta) < baud / 50) {
+ lcr = serial_port_in(port, UART_LCR);
+ serial_port_out(port, UART_LCR, lcr | 0x80);
+-
+- serial_port_out(port, UART_DLL, 1);
+- serial_port_out(port, UART_DLM, 0);
++ serial_port_out(port, UART_DLL, divisor & 0xff);
++ serial_port_out(port, UART_DLM, divisor >> 8 & 0xff);
+ serial_port_out(port, 2, 16 - scr);
+ serial_port_out(port, UART_LCR, lcr);
+ return;
+- } else if (baud > actual_baud) {
+- break;
+ }
+ }
+- serial8250_do_set_divisor(port, baud, quot, quot_frac);
+ }
+ static int pci_pericom_setup(struct serial_private *priv,
+ const struct pciserial_board *board,
+@@ -2317,12 +2321,19 @@ static struct pci_serial_quirk pci_serial_quirks[] = {
+ .setup = pci_pericom_setup_four_at_eight,
+ },
+ {
+- .vendor = PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4S,
++ .vendor = PCI_VENDOR_ID_ACCESIO,
+ .device = PCI_DEVICE_ID_ACCESIO_PCIE_ICM232_4,
+ .subvendor = PCI_ANY_ID,
+ .subdevice = PCI_ANY_ID,
+ .setup = pci_pericom_setup_four_at_eight,
+ },
++ {
++ .vendor = PCI_VENDOR_ID_ACCESIO,
++ .device = PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4S,
++ .subvendor = PCI_ANY_ID,
++ .subdevice = PCI_ANY_ID,
++ .setup = pci_pericom_setup_four_at_eight,
++ },
+ {
+ .vendor = PCI_VENDOR_ID_ACCESIO,
+ .device = PCI_DEVICE_ID_ACCESIO_MPCIE_ICM232_4,
+@@ -2929,7 +2940,7 @@ enum pci_board_num_t {
+ pbn_panacom2,
+ pbn_panacom4,
+ pbn_plx_romulus,
+- pbn_endrun_2_4000000,
++ pbn_endrun_2_3906250,
+ pbn_oxsemi,
+ pbn_oxsemi_1_3906250,
+ pbn_oxsemi_2_3906250,
+@@ -3461,10 +3472,10 @@ static struct pciserial_board pci_boards[] = {
+ * signal now many ports are available
+ * 2 port 952 Uart support
+ */
+- [pbn_endrun_2_4000000] = {
++ [pbn_endrun_2_3906250] = {
+ .flags = FL_BASE0,
+ .num_ports = 2,
+- .base_baud = 4000000,
++ .base_baud = 3906250,
+ .uart_offset = 0x200,
+ .first_offset = 0x1000,
+ },
+@@ -4407,7 +4418,7 @@ static const struct pci_device_id serial_pci_tbl[] = {
+ */
+ { PCI_VENDOR_ID_ENDRUN, PCI_DEVICE_ID_ENDRUN_1588,
+ PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+- pbn_endrun_2_4000000 },
++ pbn_endrun_2_3906250 },
+ /*
+ * Quatech cards. These actually have configurable clocks but for
+ * now we just use the default.
+@@ -5192,8 +5203,30 @@ static const struct pci_device_id serial_pci_tbl[] = {
+ { PCI_VENDOR_ID_INTASHIELD, PCI_DEVICE_ID_INTASHIELD_IS400,
+ PCI_ANY_ID, PCI_ANY_ID, 0, 0, /* 135a.0dc0 */
+ pbn_b2_4_115200 },
++ /* Brainboxes Devices */
++ /*
++ * Brainboxes UC-101
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0BA1,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
+ /*
+- * BrainBoxes UC-260
++ * Brainboxes UC-235/246
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0AA1,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_1_115200 },
++ /*
++ * Brainboxes UC-257
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0861,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ /*
++ * Brainboxes UC-260/271/701/756
+ */
+ { PCI_VENDOR_ID_INTASHIELD, 0x0D21,
+ PCI_ANY_ID, PCI_ANY_ID,
+@@ -5201,7 +5234,81 @@ static const struct pci_device_id serial_pci_tbl[] = {
+ pbn_b2_4_115200 },
+ { PCI_VENDOR_ID_INTASHIELD, 0x0E34,
+ PCI_ANY_ID, PCI_ANY_ID,
+- PCI_CLASS_COMMUNICATION_MULTISERIAL << 8, 0xffff00,
++ PCI_CLASS_COMMUNICATION_MULTISERIAL << 8, 0xffff00,
++ pbn_b2_4_115200 },
++ /*
++ * Brainboxes UC-268
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0841,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_4_115200 },
++ /*
++ * Brainboxes UC-275/279
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0881,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_8_115200 },
++ /*
++ * Brainboxes UC-302
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x08E1,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ /*
++ * Brainboxes UC-310
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x08C1,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ /*
++ * Brainboxes UC-313
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x08A3,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ /*
++ * Brainboxes UC-320/324
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0A61,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_1_115200 },
++ /*
++ * Brainboxes UC-346
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0B02,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_4_115200 },
++ /*
++ * Brainboxes UC-357
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0A81,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ { PCI_VENDOR_ID_INTASHIELD, 0x0A83,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_2_115200 },
++ /*
++ * Brainboxes UC-368
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0C41,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
++ pbn_b2_4_115200 },
++ /*
++ * Brainboxes UC-420/431
++ */
++ { PCI_VENDOR_ID_INTASHIELD, 0x0921,
++ PCI_ANY_ID, PCI_ANY_ID,
++ 0, 0,
+ pbn_b2_4_115200 },
+ /*
+ * Perle PCI-RAS cards
+diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
+index 66374704747ec..2285ef9477556 100644
+--- a/drivers/tty/serial/8250/8250_port.c
++++ b/drivers/tty/serial/8250/8250_port.c
+@@ -307,6 +307,14 @@ static const struct serial8250_config uart_config[] = {
+ .rxtrig_bytes = {1, 32, 64, 112},
+ .flags = UART_CAP_FIFO | UART_CAP_SLEEP,
+ },
++ [PORT_ASPEED_VUART] = {
++ .name = "ASPEED VUART",
++ .fifo_size = 16,
++ .tx_loadsz = 16,
++ .fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_00,
++ .rxtrig_bytes = {1, 4, 8, 14},
++ .flags = UART_CAP_FIFO,
++ },
+ };
+
+ /* Uart divisor latch read */
+@@ -1615,6 +1623,18 @@ static inline void start_tx_rs485(struct uart_port *port)
+ struct uart_8250_port *up = up_to_u8250p(port);
+ struct uart_8250_em485 *em485 = up->em485;
+
++ /*
++ * While serial8250_em485_handle_stop_tx() is a noop if
++ * em485->active_timer != &em485->stop_tx_timer, it might happen that
++ * the timer is still armed and triggers only after the current bunch of
++ * chars is send and em485->active_timer == &em485->stop_tx_timer again.
++ * So cancel the timer. There is still a theoretical race condition if
++ * the timer is already running and only comes around to check for
++ * em485->active_timer when &em485->stop_tx_timer is armed again.
++ */
++ if (em485->active_timer == &em485->stop_tx_timer)
++ hrtimer_try_to_cancel(&em485->stop_tx_timer);
++
+ em485->active_timer = NULL;
+
+ if (em485->tx_stopped) {
+@@ -1799,9 +1819,7 @@ void serial8250_tx_chars(struct uart_8250_port *up)
+ int count;
+
+ if (port->x_char) {
+- serial_out(up, UART_TX, port->x_char);
+- port->icount.tx++;
+- port->x_char = 0;
++ uart_xchar_out(port, UART_TX);
+ return;
+ }
+ if (uart_tx_stopped(port)) {
+@@ -2024,13 +2042,6 @@ void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl)
+ struct uart_8250_port *up = up_to_u8250p(port);
+ unsigned char mcr;
+
+- if (port->rs485.flags & SER_RS485_ENABLED) {
+- if (serial8250_in_MCR(up) & UART_MCR_RTS)
+- mctrl |= TIOCM_RTS;
+- else
+- mctrl &= ~TIOCM_RTS;
+- }
+-
+ mcr = serial8250_TIOCM_to_MCR(mctrl);
+
+ mcr = (mcr & up->mcr_mask) | up->mcr_force | up->mcr;
+@@ -2696,21 +2707,32 @@ static unsigned int serial8250_get_baud_rate(struct uart_port *port,
+ void serial8250_update_uartclk(struct uart_port *port, unsigned int uartclk)
+ {
+ struct uart_8250_port *up = up_to_u8250p(port);
++ struct tty_port *tport = &port->state->port;
+ unsigned int baud, quot, frac = 0;
+ struct ktermios *termios;
++ struct tty_struct *tty;
+ unsigned long flags;
+
+- mutex_lock(&port->state->port.mutex);
++ tty = tty_port_tty_get(tport);
++ if (!tty) {
++ mutex_lock(&tport->mutex);
++ port->uartclk = uartclk;
++ mutex_unlock(&tport->mutex);
++ return;
++ }
++
++ down_write(&tty->termios_rwsem);
++ mutex_lock(&tport->mutex);
+
+ if (port->uartclk == uartclk)
+ goto out_lock;
+
+ port->uartclk = uartclk;
+
+- if (!tty_port_initialized(&port->state->port))
++ if (!tty_port_initialized(tport))
+ goto out_lock;
+
+- termios = &port->state->port.tty->termios;
++ termios = &tty->termios;
+
+ baud = serial8250_get_baud_rate(port, termios, NULL);
+ quot = serial8250_get_divisor(port, baud, &frac);
+@@ -2727,7 +2749,9 @@ void serial8250_update_uartclk(struct uart_port *port, unsigned int uartclk)
+ serial8250_rpm_put(up);
+
+ out_lock:
+- mutex_unlock(&port->state->port.mutex);
++ mutex_unlock(&tport->mutex);
++ up_write(&tty->termios_rwsem);
++ tty_kref_put(tty);
+ }
+ EXPORT_SYMBOL_GPL(serial8250_update_uartclk);
+
+@@ -3316,7 +3340,7 @@ static void serial8250_console_restore(struct uart_8250_port *up)
+
+ serial8250_set_divisor(port, baud, quot, frac);
+ serial_port_out(port, UART_LCR, up->lcr);
+- serial8250_out_MCR(up, UART_MCR_DTR | UART_MCR_RTS);
++ serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS);
+ }
+
+ /*
+diff --git a/drivers/tty/serial/amba-pl010.c b/drivers/tty/serial/amba-pl010.c
+index e744b953ca346..47654073123d6 100644
+--- a/drivers/tty/serial/amba-pl010.c
++++ b/drivers/tty/serial/amba-pl010.c
+@@ -446,14 +446,11 @@ pl010_set_termios(struct uart_port *port, struct ktermios *termios,
+ if ((termios->c_cflag & CREAD) == 0)
+ uap->port.ignore_status_mask |= UART_DUMMY_RSR_RX;
+
+- /* first, disable everything */
+ old_cr = readb(uap->port.membase + UART010_CR) & ~UART010_CR_MSIE;
+
+ if (UART_ENABLE_MS(port, termios->c_cflag))
+ old_cr |= UART010_CR_MSIE;
+
+- writel(0, uap->port.membase + UART010_CR);
+-
+ /* Set baud rate */
+ quot -= 1;
+ writel((quot & 0xf00) >> 8, uap->port.membase + UART010_LCRM);
+diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
+index d361cd84ff8cf..3d40306971b81 100644
+--- a/drivers/tty/serial/amba-pl011.c
++++ b/drivers/tty/serial/amba-pl011.c
+@@ -1288,13 +1288,18 @@ static inline bool pl011_dma_rx_running(struct uart_amba_port *uap)
+
+ static void pl011_rs485_tx_stop(struct uart_amba_port *uap)
+ {
++ /*
++ * To be on the safe side only time out after twice as many iterations
++ * as fifo size.
++ */
++ const int MAX_TX_DRAIN_ITERS = uap->port.fifosize * 2;
+ struct uart_port *port = &uap->port;
+ int i = 0;
+ u32 cr;
+
+ /* Wait until hardware tx queue is empty */
+ while (!pl011_tx_empty(port)) {
+- if (i == port->fifosize) {
++ if (i > MAX_TX_DRAIN_ITERS) {
+ dev_warn(port->dev,
+ "timeout while draining hardware tx queue\n");
+ break;
+@@ -1615,8 +1620,12 @@ static void pl011_set_mctrl(struct uart_port *port, unsigned int mctrl)
+ container_of(port, struct uart_amba_port, port);
+ unsigned int cr;
+
+- if (port->rs485.flags & SER_RS485_ENABLED)
+- mctrl &= ~TIOCM_RTS;
++ if (port->rs485.flags & SER_RS485_ENABLED) {
++ if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)
++ mctrl &= ~TIOCM_RTS;
++ else
++ mctrl |= TIOCM_RTS;
++ }
+
+ cr = pl011_read(uap, REG_CR);
+
+@@ -2095,7 +2104,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
+ * with the given baud rate. We use this as the poll interval when we
+ * wait for the tx queue to empty.
+ */
+- uap->rs485_tx_drain_interval = (bits * 1000 * 1000) / baud;
++ uap->rs485_tx_drain_interval = DIV_ROUND_UP(bits * 1000 * 1000, baud);
+
+ pl011_setup_status_masks(port, termios);
+
+@@ -2105,9 +2114,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
+ if (port->rs485.flags & SER_RS485_ENABLED)
+ termios->c_cflag &= ~CRTSCTS;
+
+- /* first, disable everything */
+ old_cr = pl011_read(uap, REG_CR);
+- pl011_write(0, uap, REG_CR);
+
+ if (termios->c_cflag & CRTSCTS) {
+ if (old_cr & UART011_CR_RTS)
+@@ -2183,32 +2190,13 @@ static const char *pl011_type(struct uart_port *port)
+ return uap->port.type == PORT_AMBA ? uap->type : NULL;
+ }
+
+-/*
+- * Release the memory region(s) being used by 'port'
+- */
+-static void pl011_release_port(struct uart_port *port)
+-{
+- release_mem_region(port->mapbase, SZ_4K);
+-}
+-
+-/*
+- * Request the memory region(s) being used by 'port'
+- */
+-static int pl011_request_port(struct uart_port *port)
+-{
+- return request_mem_region(port->mapbase, SZ_4K, "uart-pl011")
+- != NULL ? 0 : -EBUSY;
+-}
+-
+ /*
+ * Configure/autoconfigure the port.
+ */
+ static void pl011_config_port(struct uart_port *port, int flags)
+ {
+- if (flags & UART_CONFIG_TYPE) {
++ if (flags & UART_CONFIG_TYPE)
+ port->type = PORT_AMBA;
+- pl011_request_port(port);
+- }
+ }
+
+ /*
+@@ -2223,6 +2211,8 @@ static int pl011_verify_port(struct uart_port *port, struct serial_struct *ser)
+ ret = -EINVAL;
+ if (ser->baud_base < 9600)
+ ret = -EINVAL;
++ if (port->mapbase != (unsigned long) ser->iomem_base)
++ ret = -EINVAL;
+ return ret;
+ }
+
+@@ -2275,8 +2265,6 @@ static const struct uart_ops amba_pl011_pops = {
+ .flush_buffer = pl011_dma_flush_buffer,
+ .set_termios = pl011_set_termios,
+ .type = pl011_type,
+- .release_port = pl011_release_port,
+- .request_port = pl011_request_port,
+ .config_port = pl011_config_port,
+ .verify_port = pl011_verify_port,
+ #ifdef CONFIG_CONSOLE_POLL
+@@ -2306,8 +2294,6 @@ static const struct uart_ops sbsa_uart_pops = {
+ .shutdown = sbsa_uart_shutdown,
+ .set_termios = sbsa_uart_set_termios,
+ .type = pl011_type,
+- .release_port = pl011_release_port,
+- .request_port = pl011_request_port,
+ .config_port = pl011_config_port,
+ .verify_port = pl011_verify_port,
+ #ifdef CONFIG_CONSOLE_POLL
+@@ -2947,6 +2933,7 @@ MODULE_DEVICE_TABLE(of, sbsa_uart_of_match);
+
+ static const struct acpi_device_id __maybe_unused sbsa_uart_acpi_match[] = {
+ { "ARMH0011", 0 },
++ { "ARMHB000", 0 },
+ {},
+ };
+ MODULE_DEVICE_TABLE(acpi, sbsa_uart_acpi_match);
+diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c
+index 249ea35088d27..dd350c5908804 100644
+--- a/drivers/tty/serial/atmel_serial.c
++++ b/drivers/tty/serial/atmel_serial.c
+@@ -1004,6 +1004,13 @@ static void atmel_tx_dma(struct uart_port *port)
+ desc->callback = atmel_complete_tx_dma;
+ desc->callback_param = atmel_port;
+ atmel_port->cookie_tx = dmaengine_submit(desc);
++ if (dma_submit_error(atmel_port->cookie_tx)) {
++ dev_err(port->dev, "dma_submit_error %d\n",
++ atmel_port->cookie_tx);
++ return;
++ }
++
++ dma_async_issue_pending(chan);
+ }
+
+ if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+@@ -1258,6 +1265,13 @@ static int atmel_prepare_rx_dma(struct uart_port *port)
+ desc->callback_param = port;
+ atmel_port->desc_rx = desc;
+ atmel_port->cookie_rx = dmaengine_submit(desc);
++ if (dma_submit_error(atmel_port->cookie_rx)) {
++ dev_err(port->dev, "dma_submit_error %d\n",
++ atmel_port->cookie_rx);
++ goto chan_err;
++ }
++
++ dma_async_issue_pending(atmel_port->chan_rx);
+
+ return 0;
+
+diff --git a/drivers/tty/serial/cpm_uart/cpm_uart_core.c b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
+index c719aa2b18328..d6d3db9c3b1f8 100644
+--- a/drivers/tty/serial/cpm_uart/cpm_uart_core.c
++++ b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
+@@ -1090,6 +1090,7 @@ static void cpm_put_poll_char(struct uart_port *port,
+ cpm_uart_early_write(pinfo, ch, 1, false);
+ }
+
++#ifdef CONFIG_SERIAL_CPM_CONSOLE
+ static struct uart_port *udbg_port;
+
+ static void udbg_cpm_putc(char c)
+@@ -1114,6 +1115,7 @@ static int udbg_cpm_getc(void)
+ cpu_relax();
+ return c;
+ }
++#endif /* CONFIG_SERIAL_CPM_CONSOLE */
+
+ #endif /* CONFIG_CONSOLE_POLL */
+
+diff --git a/drivers/tty/serial/digicolor-usart.c b/drivers/tty/serial/digicolor-usart.c
+index 13ac36e2da4f0..c7f81aa1ce912 100644
+--- a/drivers/tty/serial/digicolor-usart.c
++++ b/drivers/tty/serial/digicolor-usart.c
+@@ -471,11 +471,10 @@ static int digicolor_uart_probe(struct platform_device *pdev)
+ if (IS_ERR(uart_clk))
+ return PTR_ERR(uart_clk);
+
+- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+- dp->port.mapbase = res->start;
+- dp->port.membase = devm_ioremap_resource(&pdev->dev, res);
++ dp->port.membase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
+ if (IS_ERR(dp->port.membase))
+ return PTR_ERR(dp->port.membase);
++ dp->port.mapbase = res->start;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0)
+diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
+index b1e7190ae4836..33e5eba6ff043 100644
+--- a/drivers/tty/serial/fsl_lpuart.c
++++ b/drivers/tty/serial/fsl_lpuart.c
+@@ -2625,6 +2625,7 @@ OF_EARLYCON_DECLARE(lpuart, "fsl,vf610-lpuart", lpuart_early_console_setup);
+ OF_EARLYCON_DECLARE(lpuart32, "fsl,ls1021a-lpuart", lpuart32_early_console_setup);
+ OF_EARLYCON_DECLARE(lpuart32, "fsl,ls1028a-lpuart", ls1028a_early_console_setup);
+ OF_EARLYCON_DECLARE(lpuart32, "fsl,imx7ulp-lpuart", lpuart32_imx_early_console_setup);
++OF_EARLYCON_DECLARE(lpuart32, "fsl,imx8qxp-lpuart", lpuart32_imx_early_console_setup);
+ EARLYCON_DECLARE(lpuart, lpuart_early_console_setup);
+ EARLYCON_DECLARE(lpuart32, lpuart32_early_console_setup);
+
+@@ -2649,6 +2650,7 @@ static int lpuart_probe(struct platform_device *pdev)
+ struct device_node *np = pdev->dev.of_node;
+ struct lpuart_port *sport;
+ struct resource *res;
++ irq_handler_t handler;
+ int ret;
+
+ sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);
+@@ -2726,17 +2728,11 @@ static int lpuart_probe(struct platform_device *pdev)
+
+ if (lpuart_is_32(sport)) {
+ lpuart_reg.cons = LPUART32_CONSOLE;
+- ret = devm_request_irq(&pdev->dev, sport->port.irq, lpuart32_int, 0,
+- DRIVER_NAME, sport);
++ handler = lpuart32_int;
+ } else {
+ lpuart_reg.cons = LPUART_CONSOLE;
+- ret = devm_request_irq(&pdev->dev, sport->port.irq, lpuart_int, 0,
+- DRIVER_NAME, sport);
++ handler = lpuart_int;
+ }
+-
+- if (ret)
+- goto failed_irq_request;
+-
+ ret = uart_add_one_port(&lpuart_reg, &sport->port);
+ if (ret)
+ goto failed_attach_port;
+@@ -2758,13 +2754,18 @@ static int lpuart_probe(struct platform_device *pdev)
+
+ sport->port.rs485_config(&sport->port, &sport->port.rs485);
+
++ ret = devm_request_irq(&pdev->dev, sport->port.irq, handler, 0,
++ DRIVER_NAME, sport);
++ if (ret)
++ goto failed_irq_request;
++
+ return 0;
+
++failed_irq_request:
+ failed_get_rs485:
+ failed_reset:
+ uart_remove_one_port(&lpuart_reg, &sport->port);
+ failed_attach_port:
+-failed_irq_request:
+ lpuart_disable_clks(sport);
+ failed_clock_enable:
+ failed_out_of_range:
+diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
+index 8b121cd869e94..b7ef075a4005b 100644
+--- a/drivers/tty/serial/imx.c
++++ b/drivers/tty/serial/imx.c
+@@ -486,18 +486,21 @@ static void imx_uart_stop_tx(struct uart_port *port)
+ static void imx_uart_stop_rx(struct uart_port *port)
+ {
+ struct imx_port *sport = (struct imx_port *)port;
+- u32 ucr1, ucr2;
++ u32 ucr1, ucr2, ucr4;
+
+ ucr1 = imx_uart_readl(sport, UCR1);
+ ucr2 = imx_uart_readl(sport, UCR2);
++ ucr4 = imx_uart_readl(sport, UCR4);
+
+ if (sport->dma_is_enabled) {
+ ucr1 &= ~(UCR1_RXDMAEN | UCR1_ATDMAEN);
+ } else {
+ ucr1 &= ~UCR1_RRDYEN;
+ ucr2 &= ~UCR2_ATEN;
++ ucr4 &= ~UCR4_OREN;
+ }
+ imx_uart_writel(sport, ucr1, UCR1);
++ imx_uart_writel(sport, ucr4, UCR4);
+
+ ucr2 &= ~UCR2_RXEN;
+ imx_uart_writel(sport, ucr2, UCR2);
+@@ -1435,7 +1438,7 @@ static int imx_uart_startup(struct uart_port *port)
+ imx_uart_writel(sport, ucr1, UCR1);
+
+ ucr4 = imx_uart_readl(sport, UCR4) & ~(UCR4_OREN | UCR4_INVR);
+- if (!sport->dma_is_enabled)
++ if (!dma_is_inited)
+ ucr4 |= UCR4_OREN;
+ if (sport->inverted_rx)
+ ucr4 |= UCR4_INVR;
+@@ -1544,7 +1547,7 @@ static void imx_uart_shutdown(struct uart_port *port)
+ imx_uart_writel(sport, ucr1, UCR1);
+
+ ucr4 = imx_uart_readl(sport, UCR4);
+- ucr4 &= ~(UCR4_OREN | UCR4_TCEN);
++ ucr4 &= ~UCR4_TCEN;
+ imx_uart_writel(sport, ucr4, UCR4);
+
+ spin_unlock_irqrestore(&sport->port.lock, flags);
+@@ -2017,7 +2020,7 @@ imx_uart_console_write(struct console *co, const char *s, unsigned int count)
+ * If the port was already initialised (eg, by a boot loader),
+ * try to determine the current setup.
+ */
+-static void __init
++static void
+ imx_uart_console_get_options(struct imx_port *sport, int *baud,
+ int *parity, int *bits)
+ {
+@@ -2076,7 +2079,7 @@ imx_uart_console_get_options(struct imx_port *sport, int *baud,
+ }
+ }
+
+-static int __init
++static int
+ imx_uart_console_setup(struct console *co, char *options)
+ {
+ struct imx_port *sport;
+diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c
+index 49d0c7f2b29b8..79b7db8580e05 100644
+--- a/drivers/tty/serial/kgdboc.c
++++ b/drivers/tty/serial/kgdboc.c
+@@ -403,16 +403,16 @@ static int kgdboc_option_setup(char *opt)
+ {
+ if (!opt) {
+ pr_err("config string not provided\n");
+- return -EINVAL;
++ return 1;
+ }
+
+ if (strlen(opt) >= MAX_CONFIG_LEN) {
+ pr_err("config string too long\n");
+- return -ENOSPC;
++ return 1;
+ }
+ strcpy(config, opt);
+
+- return 0;
++ return 1;
+ }
+
+ __setup("kgdboc=", kgdboc_option_setup);
+diff --git a/drivers/tty/serial/liteuart.c b/drivers/tty/serial/liteuart.c
+index dbc0559a91575..7f74bf7bdcff8 100644
+--- a/drivers/tty/serial/liteuart.c
++++ b/drivers/tty/serial/liteuart.c
+@@ -270,8 +270,10 @@ static int liteuart_probe(struct platform_device *pdev)
+
+ /* get membase */
+ port->membase = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
+- if (IS_ERR(port->membase))
+- return PTR_ERR(port->membase);
++ if (IS_ERR(port->membase)) {
++ ret = PTR_ERR(port->membase);
++ goto err_erase_id;
++ }
+
+ /* values not from device tree */
+ port->dev = &pdev->dev;
+@@ -285,7 +287,18 @@ static int liteuart_probe(struct platform_device *pdev)
+ port->line = dev_id;
+ spin_lock_init(&port->lock);
+
+- return uart_add_one_port(&liteuart_driver, &uart->port);
++ platform_set_drvdata(pdev, port);
++
++ ret = uart_add_one_port(&liteuart_driver, &uart->port);
++ if (ret)
++ goto err_erase_id;
++
++ return 0;
++
++err_erase_id:
++ xa_erase(&liteuart_array, uart->id);
++
++ return ret;
+ }
+
+ static int liteuart_remove(struct platform_device *pdev)
+@@ -293,6 +306,7 @@ static int liteuart_remove(struct platform_device *pdev)
+ struct uart_port *port = platform_get_drvdata(pdev);
+ struct liteuart_port *uart = to_liteuart_port(port);
+
++ uart_remove_one_port(&liteuart_driver, port);
+ xa_erase(&liteuart_array, uart->id);
+
+ return 0;
+@@ -422,4 +436,4 @@ module_exit(liteuart_exit);
+ MODULE_AUTHOR("Antmicro <www.antmicro.com>");
+ MODULE_DESCRIPTION("LiteUART serial driver");
+ MODULE_LICENSE("GPL v2");
+-MODULE_ALIAS("platform: liteuart");
++MODULE_ALIAS("platform:liteuart");
+diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c
+index fcef7a961430b..489d19274f9ad 100644
+--- a/drivers/tty/serial/msm_serial.c
++++ b/drivers/tty/serial/msm_serial.c
+@@ -598,6 +598,9 @@ static void msm_start_rx_dma(struct msm_port *msm_port)
+ u32 val;
+ int ret;
+
++ if (IS_ENABLED(CONFIG_CONSOLE_POLL))
++ return;
++
+ if (!dma->chan)
+ return;
+
+diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c
+index e2f49863e9c2d..319533b3c32a5 100644
+--- a/drivers/tty/serial/samsung_tty.c
++++ b/drivers/tty/serial/samsung_tty.c
+@@ -922,11 +922,8 @@ static void s3c24xx_serial_tx_chars(struct s3c24xx_uart_port *ourport)
+ return;
+ }
+
+- if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) {
+- spin_unlock(&port->lock);
++ if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+ uart_write_wakeup(port);
+- spin_lock(&port->lock);
+- }
+
+ if (uart_circ_empty(xmit))
+ s3c24xx_serial_stop_tx(port);
+diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
+index acbb615dd28fd..0ab788058fa2a 100644
+--- a/drivers/tty/serial/sc16is7xx.c
++++ b/drivers/tty/serial/sc16is7xx.c
+@@ -734,12 +734,15 @@ static irqreturn_t sc16is7xx_irq(int irq, void *dev_id)
+ static void sc16is7xx_tx_proc(struct kthread_work *ws)
+ {
+ struct uart_port *port = &(to_sc16is7xx_one(ws, tx_work)->port);
++ struct sc16is7xx_port *s = dev_get_drvdata(port->dev);
+
+ if ((port->rs485.flags & SER_RS485_ENABLED) &&
+ (port->rs485.delay_rts_before_send > 0))
+ msleep(port->rs485.delay_rts_before_send);
+
++ mutex_lock(&s->efr_lock);
+ sc16is7xx_handle_tx(port);
++ mutex_unlock(&s->efr_lock);
+ }
+
+ static void sc16is7xx_reconf_rs485(struct uart_port *port)
+diff --git a/drivers/tty/serial/serial-tegra.c b/drivers/tty/serial/serial-tegra.c
+index 45e2e4109acd0..b6223fab0687d 100644
+--- a/drivers/tty/serial/serial-tegra.c
++++ b/drivers/tty/serial/serial-tegra.c
+@@ -1506,7 +1506,7 @@ static struct tegra_uart_chip_data tegra20_uart_chip_data = {
+ .fifo_mode_enable_status = false,
+ .uart_max_port = 5,
+ .max_dma_burst_bytes = 4,
+- .error_tolerance_low_range = 0,
++ .error_tolerance_low_range = -4,
+ .error_tolerance_high_range = 4,
+ };
+
+@@ -1517,7 +1517,7 @@ static struct tegra_uart_chip_data tegra30_uart_chip_data = {
+ .fifo_mode_enable_status = false,
+ .uart_max_port = 5,
+ .max_dma_burst_bytes = 4,
+- .error_tolerance_low_range = 0,
++ .error_tolerance_low_range = -4,
+ .error_tolerance_high_range = 4,
+ };
+
+diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
+index 0e2e35ab64c79..eb15423f935a3 100644
+--- a/drivers/tty/serial/serial_core.c
++++ b/drivers/tty/serial/serial_core.c
+@@ -162,7 +162,7 @@ static void uart_port_dtr_rts(struct uart_port *uport, int raise)
+ int RTS_after_send = !!(uport->rs485.flags & SER_RS485_RTS_AFTER_SEND);
+
+ if (raise) {
+- if (rs485_on && !RTS_after_send) {
++ if (rs485_on && RTS_after_send) {
+ uart_set_mctrl(uport, TIOCM_DTR);
+ uart_clear_mctrl(uport, TIOCM_RTS);
+ } else {
+@@ -171,7 +171,7 @@ static void uart_port_dtr_rts(struct uart_port *uport, int raise)
+ } else {
+ unsigned int clear = TIOCM_DTR;
+
+- clear |= (!rs485_on || !RTS_after_send) ? TIOCM_RTS : 0;
++ clear |= (!rs485_on || RTS_after_send) ? TIOCM_RTS : 0;
+ uart_clear_mctrl(uport, clear);
+ }
+ }
+@@ -222,7 +222,11 @@ static int uart_port_startup(struct tty_struct *tty, struct uart_state *state,
+ if (retval == 0) {
+ if (uart_console(uport) && uport->cons->cflag) {
+ tty->termios.c_cflag = uport->cons->cflag;
++ tty->termios.c_ispeed = uport->cons->ispeed;
++ tty->termios.c_ospeed = uport->cons->ospeed;
+ uport->cons->cflag = 0;
++ uport->cons->ispeed = 0;
++ uport->cons->ospeed = 0;
+ }
+ /*
+ * Initialise the hardware port settings.
+@@ -290,8 +294,11 @@ static void uart_shutdown(struct tty_struct *tty, struct uart_state *state)
+ /*
+ * Turn off DTR and RTS early.
+ */
+- if (uport && uart_console(uport) && tty)
++ if (uport && uart_console(uport) && tty) {
+ uport->cons->cflag = tty->termios.c_cflag;
++ uport->cons->ispeed = tty->termios.c_ispeed;
++ uport->cons->ospeed = tty->termios.c_ospeed;
++ }
+
+ if (!tty || C_HUPCL(tty))
+ uart_port_dtr_rts(uport, 0);
+@@ -645,6 +652,20 @@ static void uart_flush_buffer(struct tty_struct *tty)
+ tty_port_tty_wakeup(&state->port);
+ }
+
++/*
++ * This function performs low-level write of high-priority XON/XOFF
++ * character and accounting for it.
++ *
++ * Requires uart_port to implement .serial_out().
++ */
++void uart_xchar_out(struct uart_port *uport, int offset)
++{
++ serial_port_out(uport, offset, uport->x_char);
++ uport->icount.tx++;
++ uport->x_char = 0;
++}
++EXPORT_SYMBOL_GPL(uart_xchar_out);
++
+ /*
+ * This function is used to send a high-priority XON/XOFF character to
+ * the device
+@@ -1068,6 +1089,11 @@ uart_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear)
+ goto out;
+
+ if (!tty_io_error(tty)) {
++ if (uport->rs485.flags & SER_RS485_ENABLED) {
++ set &= ~TIOCM_RTS;
++ clear &= ~TIOCM_RTS;
++ }
++
+ uart_update_mctrl(uport, set, clear);
+ ret = 0;
+ }
+@@ -1542,6 +1568,7 @@ static void uart_tty_port_shutdown(struct tty_port *port)
+ {
+ struct uart_state *state = container_of(port, struct uart_state, port);
+ struct uart_port *uport = uart_port_check(state);
++ char *buf;
+
+ /*
+ * At this point, we stop accepting input. To do this, we
+@@ -1563,8 +1590,18 @@ static void uart_tty_port_shutdown(struct tty_port *port)
+ */
+ tty_port_set_suspended(port, 0);
+
+- uart_change_pm(state, UART_PM_STATE_OFF);
++ /*
++ * Free the transmit buffer.
++ */
++ spin_lock_irq(&uport->lock);
++ buf = state->xmit.buf;
++ state->xmit.buf = NULL;
++ spin_unlock_irq(&uport->lock);
+
++ if (buf)
++ free_page((unsigned long)buf);
++
++ uart_change_pm(state, UART_PM_STATE_OFF);
+ }
+
+ static void uart_wait_until_sent(struct tty_struct *tty, int timeout)
+@@ -2094,8 +2131,11 @@ uart_set_options(struct uart_port *port, struct console *co,
+ * Allow the setting of the UART parameters with a NULL console
+ * too:
+ */
+- if (co)
++ if (co) {
+ co->cflag = termios.c_cflag;
++ co->ispeed = termios.c_ispeed;
++ co->ospeed = termios.c_ospeed;
++ }
+
+ return 0;
+ }
+@@ -2229,6 +2269,8 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport)
+ */
+ memset(&termios, 0, sizeof(struct ktermios));
+ termios.c_cflag = uport->cons->cflag;
++ termios.c_ispeed = uport->cons->ispeed;
++ termios.c_ospeed = uport->cons->ospeed;
+
+ /*
+ * If that's unset, use the tty termios setting.
+@@ -2365,7 +2407,8 @@ uart_configure_port(struct uart_driver *drv, struct uart_state *state,
+ * We probably don't need a spinlock around this, but
+ */
+ spin_lock_irqsave(&port->lock, flags);
+- port->ops->set_mctrl(port, port->mctrl & TIOCM_DTR);
++ port->mctrl &= TIOCM_DTR;
++ port->ops->set_mctrl(port, port->mctrl);
+ spin_unlock_irqrestore(&port->lock, flags);
+
+ /*
+diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c
+index 8f032e77b954a..810a1b0b65203 100644
+--- a/drivers/tty/serial/stm32-usart.c
++++ b/drivers/tty/serial/stm32-usart.c
+@@ -421,10 +421,22 @@ static void stm32_usart_transmit_chars(struct uart_port *port)
+ struct stm32_port *stm32_port = to_stm32_port(port);
+ const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+ struct circ_buf *xmit = &port->state->xmit;
++ u32 isr;
++ int ret;
+
+ if (port->x_char) {
+ if (stm32_port->tx_dma_busy)
+ stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
++
++ /* Check that TDR is empty before filling FIFO */
++ ret =
++ readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
++ isr,
++ (isr & USART_SR_TXE),
++ 10, 1000);
++ if (ret)
++ dev_warn(port->dev, "1 character may be erased\n");
++
+ writel_relaxed(port->x_char, port->membase + ofs->tdr);
+ port->x_char = 0;
+ port->icount.tx++;
+@@ -575,7 +587,7 @@ static void stm32_usart_start_tx(struct uart_port *port)
+ struct serial_rs485 *rs485conf = &port->rs485;
+ struct circ_buf *xmit = &port->state->xmit;
+
+- if (uart_circ_empty(xmit))
++ if (uart_circ_empty(xmit) && !port->x_char)
+ return;
+
+ if (rs485conf->flags & SER_RS485_ENABLED) {
+@@ -691,6 +703,11 @@ static void stm32_usart_shutdown(struct uart_port *port)
+ u32 val, isr;
+ int ret;
+
++ if (stm32_port->tx_dma_busy) {
++ dmaengine_terminate_async(stm32_port->tx_ch);
++ stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
++ }
++
+ /* Disable modem control interrupts */
+ stm32_usart_disable_ms(port);
+
+@@ -1385,7 +1402,6 @@ static int stm32_usart_serial_remove(struct platform_device *pdev)
+ stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
+
+ if (stm32_port->tx_ch) {
+- dmaengine_terminate_async(stm32_port->tx_ch);
+ stm32_usart_of_dma_tx_remove(stm32_port, pdev);
+ dma_release_channel(stm32_port->tx_ch);
+ }
+diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c
+index dfc1ba4e15724..36871cebd6a0f 100644
+--- a/drivers/tty/serial/uartlite.c
++++ b/drivers/tty/serial/uartlite.c
+@@ -612,7 +612,7 @@ static struct uart_driver ulite_uart_driver = {
+ *
+ * Returns: 0 on success, <0 otherwise
+ */
+-static int ulite_assign(struct device *dev, int id, u32 base, int irq,
++static int ulite_assign(struct device *dev, int id, phys_addr_t base, int irq,
+ struct uartlite_data *pdata)
+ {
+ struct uart_port *port;
+diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
+index 962e522ccc45c..d5e243908d9fd 100644
+--- a/drivers/tty/serial/xilinx_uartps.c
++++ b/drivers/tty/serial/xilinx_uartps.c
+@@ -601,9 +601,10 @@ static void cdns_uart_start_tx(struct uart_port *port)
+ if (uart_circ_empty(&port->state->xmit))
+ return;
+
++ writel(CDNS_UART_IXR_TXEMPTY, port->membase + CDNS_UART_ISR);
++
+ cdns_uart_handle_tx(port);
+
+- writel(CDNS_UART_IXR_TXEMPTY, port->membase + CDNS_UART_ISR);
+ /* Enable the TX Empty interrupt */
+ writel(CDNS_UART_IXR_TXEMPTY, port->membase + CDNS_UART_IER);
+ }
+diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c
+index 635d0af229b72..6c7e65b1d9a1c 100644
+--- a/drivers/tty/tty_buffer.c
++++ b/drivers/tty/tty_buffer.c
+@@ -544,6 +544,9 @@ static void flush_to_ldisc(struct work_struct *work)
+ if (!count)
+ break;
+ head->read += count;
++
++ if (need_resched())
++ cond_resched();
+ }
+
+ mutex_unlock(&buf->lock);
+diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c
+index 3639bb6dc372e..58013698635f0 100644
+--- a/drivers/tty/vt/vt_ioctl.c
++++ b/drivers/tty/vt/vt_ioctl.c
+@@ -599,8 +599,8 @@ static int vt_setactivate(struct vt_setactivate __user *sa)
+ if (vsa.console == 0 || vsa.console > MAX_NR_CONSOLES)
+ return -ENXIO;
+
+- vsa.console = array_index_nospec(vsa.console, MAX_NR_CONSOLES + 1);
+ vsa.console--;
++ vsa.console = array_index_nospec(vsa.console, MAX_NR_CONSOLES);
+ console_lock();
+ ret = vc_allocate(vsa.console);
+ if (ret) {
+@@ -845,6 +845,7 @@ int vt_ioctl(struct tty_struct *tty,
+ return -ENXIO;
+
+ arg--;
++ arg = array_index_nospec(arg, MAX_NR_CONSOLES);
+ console_lock();
+ ret = vc_allocate(arg);
+ console_unlock();
+diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c
+index 1f3b4a1422126..d6d515d598dc0 100644
+--- a/drivers/usb/cdns3/cdns3-gadget.c
++++ b/drivers/usb/cdns3/cdns3-gadget.c
+@@ -337,19 +337,6 @@ static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
+ cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
+ }
+
+-static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
+-{
+- struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+- int current_trb = priv_req->start_trb;
+-
+- while (current_trb != priv_req->end_trb) {
+- cdns3_ep_inc_deq(priv_ep);
+- current_trb = priv_ep->dequeue;
+- }
+-
+- cdns3_ep_inc_deq(priv_ep);
+-}
+-
+ /**
+ * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
+ * @priv_dev: Extended gadget object
+@@ -1517,10 +1504,11 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
+
+ trb = priv_ep->trb_pool + priv_ep->dequeue;
+
+- /* Request was dequeued and TRB was changed to TRB_LINK. */
+- if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
++ /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
++ while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
+ trace_cdns3_complete_trb(priv_ep, trb);
+- cdns3_move_deq_to_next_trb(priv_req);
++ cdns3_ep_inc_deq(priv_ep);
++ trb = priv_ep->trb_pool + priv_ep->dequeue;
+ }
+
+ if (!request->stream_id) {
+@@ -2696,6 +2684,7 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
+ struct usb_request *request;
+ struct cdns3_request *priv_req;
+ struct cdns3_trb *trb = NULL;
++ struct cdns3_trb trb_tmp;
+ int ret;
+ int val;
+
+@@ -2705,8 +2694,10 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
+ if (request) {
+ priv_req = to_cdns3_request(request);
+ trb = priv_req->trb;
+- if (trb)
++ if (trb) {
++ trb_tmp = *trb;
+ trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
++ }
+ }
+
+ writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+@@ -2721,7 +2712,7 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
+
+ if (request) {
+ if (trb)
+- trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
++ *trb = trb_tmp;
+
+ cdns3_rearm_transfer(priv_ep, 1);
+ }
+diff --git a/drivers/usb/cdns3/cdnsp-debug.h b/drivers/usb/cdns3/cdnsp-debug.h
+index a8776df2d4e0c..f0ca865cce2a0 100644
+--- a/drivers/usb/cdns3/cdnsp-debug.h
++++ b/drivers/usb/cdns3/cdnsp-debug.h
+@@ -182,208 +182,211 @@ static inline const char *cdnsp_decode_trb(char *str, size_t size, u32 field0,
+ int ep_id = TRB_TO_EP_INDEX(field3) - 1;
+ int type = TRB_FIELD_TO_TYPE(field3);
+ unsigned int ep_num;
+- int ret = 0;
++ int ret;
+ u32 temp;
+
+ ep_num = DIV_ROUND_UP(ep_id, 2);
+
+ switch (type) {
+ case TRB_LINK:
+- ret += snprintf(str, size,
+- "LINK %08x%08x intr %ld type '%s' flags %c:%c:%c:%c",
+- field1, field0, GET_INTR_TARGET(field2),
+- cdnsp_trb_type_string(type),
+- field3 & TRB_IOC ? 'I' : 'i',
+- field3 & TRB_CHAIN ? 'C' : 'c',
+- field3 & TRB_TC ? 'T' : 't',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "LINK %08x%08x intr %ld type '%s' flags %c:%c:%c:%c",
++ field1, field0, GET_INTR_TARGET(field2),
++ cdnsp_trb_type_string(type),
++ field3 & TRB_IOC ? 'I' : 'i',
++ field3 & TRB_CHAIN ? 'C' : 'c',
++ field3 & TRB_TC ? 'T' : 't',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_TRANSFER:
+ case TRB_COMPLETION:
+ case TRB_PORT_STATUS:
+ case TRB_HC_EVENT:
+- ret += snprintf(str, size,
+- "ep%d%s(%d) type '%s' TRB %08x%08x status '%s'"
+- " len %ld slot %ld flags %c:%c",
+- ep_num, ep_id % 2 ? "out" : "in",
+- TRB_TO_EP_INDEX(field3),
+- cdnsp_trb_type_string(type), field1, field0,
+- cdnsp_trb_comp_code_string(GET_COMP_CODE(field2)),
+- EVENT_TRB_LEN(field2), TRB_TO_SLOT_ID(field3),
+- field3 & EVENT_DATA ? 'E' : 'e',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "ep%d%s(%d) type '%s' TRB %08x%08x status '%s'"
++ " len %ld slot %ld flags %c:%c",
++ ep_num, ep_id % 2 ? "out" : "in",
++ TRB_TO_EP_INDEX(field3),
++ cdnsp_trb_type_string(type), field1, field0,
++ cdnsp_trb_comp_code_string(GET_COMP_CODE(field2)),
++ EVENT_TRB_LEN(field2), TRB_TO_SLOT_ID(field3),
++ field3 & EVENT_DATA ? 'E' : 'e',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_MFINDEX_WRAP:
+- ret += snprintf(str, size, "%s: flags %c",
+- cdnsp_trb_type_string(type),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size, "%s: flags %c",
++ cdnsp_trb_type_string(type),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_SETUP:
+- ret += snprintf(str, size,
+- "type '%s' bRequestType %02x bRequest %02x "
+- "wValue %02x%02x wIndex %02x%02x wLength %d "
+- "length %ld TD size %ld intr %ld Setup ID %ld "
+- "flags %c:%c:%c",
+- cdnsp_trb_type_string(type),
+- field0 & 0xff,
+- (field0 & 0xff00) >> 8,
+- (field0 & 0xff000000) >> 24,
+- (field0 & 0xff0000) >> 16,
+- (field1 & 0xff00) >> 8,
+- field1 & 0xff,
+- (field1 & 0xff000000) >> 16 |
+- (field1 & 0xff0000) >> 16,
+- TRB_LEN(field2), GET_TD_SIZE(field2),
+- GET_INTR_TARGET(field2),
+- TRB_SETUPID_TO_TYPE(field3),
+- field3 & TRB_IDT ? 'D' : 'd',
+- field3 & TRB_IOC ? 'I' : 'i',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "type '%s' bRequestType %02x bRequest %02x "
++ "wValue %02x%02x wIndex %02x%02x wLength %d "
++ "length %ld TD size %ld intr %ld Setup ID %ld "
++ "flags %c:%c:%c",
++ cdnsp_trb_type_string(type),
++ field0 & 0xff,
++ (field0 & 0xff00) >> 8,
++ (field0 & 0xff000000) >> 24,
++ (field0 & 0xff0000) >> 16,
++ (field1 & 0xff00) >> 8,
++ field1 & 0xff,
++ (field1 & 0xff000000) >> 16 |
++ (field1 & 0xff0000) >> 16,
++ TRB_LEN(field2), GET_TD_SIZE(field2),
++ GET_INTR_TARGET(field2),
++ TRB_SETUPID_TO_TYPE(field3),
++ field3 & TRB_IDT ? 'D' : 'd',
++ field3 & TRB_IOC ? 'I' : 'i',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_DATA:
+- ret += snprintf(str, size,
+- "type '%s' Buffer %08x%08x length %ld TD size %ld "
+- "intr %ld flags %c:%c:%c:%c:%c:%c:%c",
+- cdnsp_trb_type_string(type),
+- field1, field0, TRB_LEN(field2),
+- GET_TD_SIZE(field2),
+- GET_INTR_TARGET(field2),
+- field3 & TRB_IDT ? 'D' : 'i',
+- field3 & TRB_IOC ? 'I' : 'i',
+- field3 & TRB_CHAIN ? 'C' : 'c',
+- field3 & TRB_NO_SNOOP ? 'S' : 's',
+- field3 & TRB_ISP ? 'I' : 'i',
+- field3 & TRB_ENT ? 'E' : 'e',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "type '%s' Buffer %08x%08x length %ld TD size %ld "
++ "intr %ld flags %c:%c:%c:%c:%c:%c:%c",
++ cdnsp_trb_type_string(type),
++ field1, field0, TRB_LEN(field2),
++ GET_TD_SIZE(field2),
++ GET_INTR_TARGET(field2),
++ field3 & TRB_IDT ? 'D' : 'i',
++ field3 & TRB_IOC ? 'I' : 'i',
++ field3 & TRB_CHAIN ? 'C' : 'c',
++ field3 & TRB_NO_SNOOP ? 'S' : 's',
++ field3 & TRB_ISP ? 'I' : 'i',
++ field3 & TRB_ENT ? 'E' : 'e',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_STATUS:
+- ret += snprintf(str, size,
+- "Buffer %08x%08x length %ld TD size %ld intr"
+- "%ld type '%s' flags %c:%c:%c:%c",
+- field1, field0, TRB_LEN(field2),
+- GET_TD_SIZE(field2),
+- GET_INTR_TARGET(field2),
+- cdnsp_trb_type_string(type),
+- field3 & TRB_IOC ? 'I' : 'i',
+- field3 & TRB_CHAIN ? 'C' : 'c',
+- field3 & TRB_ENT ? 'E' : 'e',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "Buffer %08x%08x length %ld TD size %ld intr"
++ "%ld type '%s' flags %c:%c:%c:%c",
++ field1, field0, TRB_LEN(field2),
++ GET_TD_SIZE(field2),
++ GET_INTR_TARGET(field2),
++ cdnsp_trb_type_string(type),
++ field3 & TRB_IOC ? 'I' : 'i',
++ field3 & TRB_CHAIN ? 'C' : 'c',
++ field3 & TRB_ENT ? 'E' : 'e',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_NORMAL:
+ case TRB_ISOC:
+ case TRB_EVENT_DATA:
+ case TRB_TR_NOOP:
+- ret += snprintf(str, size,
+- "type '%s' Buffer %08x%08x length %ld "
+- "TD size %ld intr %ld "
+- "flags %c:%c:%c:%c:%c:%c:%c:%c:%c",
+- cdnsp_trb_type_string(type),
+- field1, field0, TRB_LEN(field2),
+- GET_TD_SIZE(field2),
+- GET_INTR_TARGET(field2),
+- field3 & TRB_BEI ? 'B' : 'b',
+- field3 & TRB_IDT ? 'T' : 't',
+- field3 & TRB_IOC ? 'I' : 'i',
+- field3 & TRB_CHAIN ? 'C' : 'c',
+- field3 & TRB_NO_SNOOP ? 'S' : 's',
+- field3 & TRB_ISP ? 'I' : 'i',
+- field3 & TRB_ENT ? 'E' : 'e',
+- field3 & TRB_CYCLE ? 'C' : 'c',
+- !(field3 & TRB_EVENT_INVALIDATE) ? 'V' : 'v');
++ ret = snprintf(str, size,
++ "type '%s' Buffer %08x%08x length %ld "
++ "TD size %ld intr %ld "
++ "flags %c:%c:%c:%c:%c:%c:%c:%c:%c",
++ cdnsp_trb_type_string(type),
++ field1, field0, TRB_LEN(field2),
++ GET_TD_SIZE(field2),
++ GET_INTR_TARGET(field2),
++ field3 & TRB_BEI ? 'B' : 'b',
++ field3 & TRB_IDT ? 'T' : 't',
++ field3 & TRB_IOC ? 'I' : 'i',
++ field3 & TRB_CHAIN ? 'C' : 'c',
++ field3 & TRB_NO_SNOOP ? 'S' : 's',
++ field3 & TRB_ISP ? 'I' : 'i',
++ field3 & TRB_ENT ? 'E' : 'e',
++ field3 & TRB_CYCLE ? 'C' : 'c',
++ !(field3 & TRB_EVENT_INVALIDATE) ? 'V' : 'v');
+ break;
+ case TRB_CMD_NOOP:
+ case TRB_ENABLE_SLOT:
+- ret += snprintf(str, size, "%s: flags %c",
+- cdnsp_trb_type_string(type),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size, "%s: flags %c",
++ cdnsp_trb_type_string(type),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_DISABLE_SLOT:
+- ret += snprintf(str, size, "%s: slot %ld flags %c",
+- cdnsp_trb_type_string(type),
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size, "%s: slot %ld flags %c",
++ cdnsp_trb_type_string(type),
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_ADDR_DEV:
+- ret += snprintf(str, size,
+- "%s: ctx %08x%08x slot %ld flags %c:%c",
+- cdnsp_trb_type_string(type), field1, field0,
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_BSR ? 'B' : 'b',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ctx %08x%08x slot %ld flags %c:%c",
++ cdnsp_trb_type_string(type), field1, field0,
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_BSR ? 'B' : 'b',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_CONFIG_EP:
+- ret += snprintf(str, size,
+- "%s: ctx %08x%08x slot %ld flags %c:%c",
+- cdnsp_trb_type_string(type), field1, field0,
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_DC ? 'D' : 'd',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ctx %08x%08x slot %ld flags %c:%c",
++ cdnsp_trb_type_string(type), field1, field0,
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_DC ? 'D' : 'd',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_EVAL_CONTEXT:
+- ret += snprintf(str, size,
+- "%s: ctx %08x%08x slot %ld flags %c",
+- cdnsp_trb_type_string(type), field1, field0,
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ctx %08x%08x slot %ld flags %c",
++ cdnsp_trb_type_string(type), field1, field0,
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_RESET_EP:
+ case TRB_HALT_ENDPOINT:
+ case TRB_FLUSH_ENDPOINT:
+- ret += snprintf(str, size,
+- "%s: ep%d%s(%d) ctx %08x%08x slot %ld flags %c",
+- cdnsp_trb_type_string(type),
+- ep_num, ep_id % 2 ? "out" : "in",
+- TRB_TO_EP_INDEX(field3), field1, field0,
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ep%d%s(%d) ctx %08x%08x slot %ld flags %c",
++ cdnsp_trb_type_string(type),
++ ep_num, ep_id % 2 ? "out" : "in",
++ TRB_TO_EP_INDEX(field3), field1, field0,
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_STOP_RING:
+- ret += snprintf(str, size,
+- "%s: ep%d%s(%d) slot %ld sp %d flags %c",
+- cdnsp_trb_type_string(type),
+- ep_num, ep_id % 2 ? "out" : "in",
+- TRB_TO_EP_INDEX(field3),
+- TRB_TO_SLOT_ID(field3),
+- TRB_TO_SUSPEND_PORT(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ep%d%s(%d) slot %ld sp %d flags %c",
++ cdnsp_trb_type_string(type),
++ ep_num, ep_id % 2 ? "out" : "in",
++ TRB_TO_EP_INDEX(field3),
++ TRB_TO_SLOT_ID(field3),
++ TRB_TO_SUSPEND_PORT(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_SET_DEQ:
+- ret += snprintf(str, size,
+- "%s: ep%d%s(%d) deq %08x%08x stream %ld slot %ld flags %c",
+- cdnsp_trb_type_string(type),
+- ep_num, ep_id % 2 ? "out" : "in",
+- TRB_TO_EP_INDEX(field3), field1, field0,
+- TRB_TO_STREAM_ID(field2),
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size,
++ "%s: ep%d%s(%d) deq %08x%08x stream %ld slot %ld flags %c",
++ cdnsp_trb_type_string(type),
++ ep_num, ep_id % 2 ? "out" : "in",
++ TRB_TO_EP_INDEX(field3), field1, field0,
++ TRB_TO_STREAM_ID(field2),
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_RESET_DEV:
+- ret += snprintf(str, size, "%s: slot %ld flags %c",
+- cdnsp_trb_type_string(type),
+- TRB_TO_SLOT_ID(field3),
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ ret = snprintf(str, size, "%s: slot %ld flags %c",
++ cdnsp_trb_type_string(type),
++ TRB_TO_SLOT_ID(field3),
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ case TRB_ENDPOINT_NRDY:
+- temp = TRB_TO_HOST_STREAM(field2);
+-
+- ret += snprintf(str, size,
+- "%s: ep%d%s(%d) H_SID %x%s%s D_SID %lx flags %c:%c",
+- cdnsp_trb_type_string(type),
+- ep_num, ep_id % 2 ? "out" : "in",
+- TRB_TO_EP_INDEX(field3), temp,
+- temp == STREAM_PRIME_ACK ? "(PRIME)" : "",
+- temp == STREAM_REJECTED ? "(REJECTED)" : "",
+- TRB_TO_DEV_STREAM(field0),
+- field3 & TRB_STAT ? 'S' : 's',
+- field3 & TRB_CYCLE ? 'C' : 'c');
++ temp = TRB_TO_HOST_STREAM(field2);
++
++ ret = snprintf(str, size,
++ "%s: ep%d%s(%d) H_SID %x%s%s D_SID %lx flags %c:%c",
++ cdnsp_trb_type_string(type),
++ ep_num, ep_id % 2 ? "out" : "in",
++ TRB_TO_EP_INDEX(field3), temp,
++ temp == STREAM_PRIME_ACK ? "(PRIME)" : "",
++ temp == STREAM_REJECTED ? "(REJECTED)" : "",
++ TRB_TO_DEV_STREAM(field0),
++ field3 & TRB_STAT ? 'S' : 's',
++ field3 & TRB_CYCLE ? 'C' : 'c');
+ break;
+ default:
+- ret += snprintf(str, size,
+- "type '%s' -> raw %08x %08x %08x %08x",
+- cdnsp_trb_type_string(type),
+- field0, field1, field2, field3);
++ ret = snprintf(str, size,
++ "type '%s' -> raw %08x %08x %08x %08x",
++ cdnsp_trb_type_string(type),
++ field0, field1, field2, field3);
+ }
+
++ if (ret >= size)
++ pr_info("CDNSP: buffer overflowed.\n");
++
+ return str;
+ }
+
+diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c
+index 27df0c6978978..e85bf768c66da 100644
+--- a/drivers/usb/cdns3/cdnsp-gadget.c
++++ b/drivers/usb/cdns3/cdnsp-gadget.c
+@@ -1541,15 +1541,27 @@ static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
+ {
+ struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
+ struct cdns *cdns = dev_get_drvdata(pdev->dev);
++ unsigned long flags;
+
+ trace_cdnsp_pullup(is_on);
+
++ /*
++ * Disable events handling while controller is being
++ * enabled/disabled.
++ */
++ disable_irq(cdns->dev_irq);
++ spin_lock_irqsave(&pdev->lock, flags);
++
+ if (!is_on) {
+ cdnsp_reset_device(pdev);
+ cdns_clear_vbus(cdns);
+ } else {
+ cdns_set_vbus(cdns);
+ }
++
++ spin_unlock_irqrestore(&pdev->lock, flags);
++ enable_irq(cdns->dev_irq);
++
+ return 0;
+ }
+
+diff --git a/drivers/usb/cdns3/cdnsp-mem.c b/drivers/usb/cdns3/cdnsp-mem.c
+index ad9aee3f1e398..97866bfb2da9d 100644
+--- a/drivers/usb/cdns3/cdnsp-mem.c
++++ b/drivers/usb/cdns3/cdnsp-mem.c
+@@ -987,6 +987,9 @@ int cdnsp_endpoint_init(struct cdnsp_device *pdev,
+
+ /* Set up the endpoint ring. */
+ pep->ring = cdnsp_ring_alloc(pdev, 2, ring_type, max_packet, mem_flags);
++ if (!pep->ring)
++ return -ENOMEM;
++
+ pep->skip = false;
+
+ /* Fill the endpoint context */
+diff --git a/drivers/usb/cdns3/cdnsp-ring.c b/drivers/usb/cdns3/cdnsp-ring.c
+index 1b1438457fb04..e45c3d6e1536c 100644
+--- a/drivers/usb/cdns3/cdnsp-ring.c
++++ b/drivers/usb/cdns3/cdnsp-ring.c
+@@ -1029,6 +1029,8 @@ static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev,
+ return;
+ }
+
++ *status = 0;
++
+ cdnsp_finish_td(pdev, td, event, pep, status);
+ }
+
+@@ -1523,7 +1525,14 @@ irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) {
+- cdnsp_died(pdev);
++ /*
++ * While removing or stopping driver there may still be deferred
++ * not handled interrupt which should not be treated as error.
++ * Driver should simply ignore it.
++ */
++ if (pdev->gadget_driver)
++ cdnsp_died(pdev);
++
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return IRQ_HANDLED;
+ }
+diff --git a/drivers/usb/cdns3/cdnsp-trace.h b/drivers/usb/cdns3/cdnsp-trace.h
+index 6a2571c6aa9ed..5983dfb996537 100644
+--- a/drivers/usb/cdns3/cdnsp-trace.h
++++ b/drivers/usb/cdns3/cdnsp-trace.h
+@@ -57,9 +57,9 @@ DECLARE_EVENT_CLASS(cdnsp_log_ep,
+ __entry->first_prime_det = pep->stream_info.first_prime_det;
+ __entry->drbls_count = pep->stream_info.drbls_count;
+ ),
+- TP_printk("%s: SID: %08x ep state: %x stream: enabled: %d num %d "
++ TP_printk("%s: SID: %08x, ep state: %x, stream: enabled: %d num %d "
+ "tds %d, first prime: %d drbls %d",
+- __get_str(name), __entry->state, __entry->stream_id,
++ __get_str(name), __entry->stream_id, __entry->state,
+ __entry->enabled, __entry->num_streams, __entry->td_count,
+ __entry->first_prime_det, __entry->drbls_count)
+ );
+diff --git a/drivers/usb/cdns3/drd.c b/drivers/usb/cdns3/drd.c
+index 55c73b1d87047..d00ff98dffabf 100644
+--- a/drivers/usb/cdns3/drd.c
++++ b/drivers/usb/cdns3/drd.c
+@@ -483,11 +483,11 @@ int cdns_drd_exit(struct cdns *cdns)
+ /* Indicate the cdns3 core was power lost before */
+ bool cdns_power_is_lost(struct cdns *cdns)
+ {
+- if (cdns->version == CDNS3_CONTROLLER_V1) {
+- if (!(readl(&cdns->otg_v1_regs->simulate) & BIT(0)))
++ if (cdns->version == CDNS3_CONTROLLER_V0) {
++ if (!(readl(&cdns->otg_v0_regs->simulate) & BIT(0)))
+ return true;
+ } else {
+- if (!(readl(&cdns->otg_v0_regs->simulate) & BIT(0)))
++ if (!(readl(&cdns->otg_v1_regs->simulate) & BIT(0)))
+ return true;
+ }
+ return false;
+diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
+index f1d100671ee6a..097142ffb1842 100644
+--- a/drivers/usb/chipidea/ci_hdrc_imx.c
++++ b/drivers/usb/chipidea/ci_hdrc_imx.c
+@@ -420,15 +420,15 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
+ data->phy = devm_usb_get_phy_by_phandle(dev, "fsl,usbphy", 0);
+ if (IS_ERR(data->phy)) {
+ ret = PTR_ERR(data->phy);
+- if (ret == -ENODEV) {
+- data->phy = devm_usb_get_phy_by_phandle(dev, "phys", 0);
+- if (IS_ERR(data->phy)) {
+- ret = PTR_ERR(data->phy);
+- if (ret == -ENODEV)
+- data->phy = NULL;
+- else
+- goto err_clk;
+- }
++ if (ret != -ENODEV)
++ goto err_clk;
++ data->phy = devm_usb_get_phy_by_phandle(dev, "phys", 0);
++ if (IS_ERR(data->phy)) {
++ ret = PTR_ERR(data->phy);
++ if (ret == -ENODEV)
++ data->phy = NULL;
++ else
++ goto err_clk;
+ }
+ }
+
+diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
+index 2b18f5088ae4a..a56f06368d142 100644
+--- a/drivers/usb/chipidea/core.c
++++ b/drivers/usb/chipidea/core.c
+@@ -514,7 +514,7 @@ int hw_device_reset(struct ci_hdrc *ci)
+ return 0;
+ }
+
+-static irqreturn_t ci_irq(int irq, void *data)
++static irqreturn_t ci_irq_handler(int irq, void *data)
+ {
+ struct ci_hdrc *ci = data;
+ irqreturn_t ret = IRQ_NONE;
+@@ -567,6 +567,15 @@ static irqreturn_t ci_irq(int irq, void *data)
+ return ret;
+ }
+
++static void ci_irq(struct ci_hdrc *ci)
++{
++ unsigned long flags;
++
++ local_irq_save(flags);
++ ci_irq_handler(ci->irq, ci);
++ local_irq_restore(flags);
++}
++
+ static int ci_cable_notifier(struct notifier_block *nb, unsigned long event,
+ void *ptr)
+ {
+@@ -576,7 +585,7 @@ static int ci_cable_notifier(struct notifier_block *nb, unsigned long event,
+ cbl->connected = event;
+ cbl->changed = true;
+
+- ci_irq(ci->irq, ci);
++ ci_irq(ci);
+ return NOTIFY_DONE;
+ }
+
+@@ -617,7 +626,7 @@ static int ci_usb_role_switch_set(struct usb_role_switch *sw,
+ if (cable) {
+ cable->changed = true;
+ cable->connected = false;
+- ci_irq(ci->irq, ci);
++ ci_irq(ci);
+ spin_unlock_irqrestore(&ci->lock, flags);
+ if (ci->wq && role != USB_ROLE_NONE)
+ flush_workqueue(ci->wq);
+@@ -635,7 +644,7 @@ static int ci_usb_role_switch_set(struct usb_role_switch *sw,
+ if (cable) {
+ cable->changed = true;
+ cable->connected = true;
+- ci_irq(ci->irq, ci);
++ ci_irq(ci);
+ }
+ spin_unlock_irqrestore(&ci->lock, flags);
+ pm_runtime_put_sync(ci->dev);
+@@ -1174,7 +1183,7 @@ static int ci_hdrc_probe(struct platform_device *pdev)
+ }
+ }
+
+- ret = devm_request_irq(dev, ci->irq, ci_irq, IRQF_SHARED,
++ ret = devm_request_irq(dev, ci->irq, ci_irq_handler, IRQF_SHARED,
+ ci->platdata->name, ci);
+ if (ret)
+ goto stop;
+@@ -1295,11 +1304,11 @@ static void ci_extcon_wakeup_int(struct ci_hdrc *ci)
+
+ if (!IS_ERR(cable_id->edev) && ci->is_otg &&
+ (otgsc & OTGSC_IDIE) && (otgsc & OTGSC_IDIS))
+- ci_irq(ci->irq, ci);
++ ci_irq(ci);
+
+ if (!IS_ERR(cable_vbus->edev) && ci->is_otg &&
+ (otgsc & OTGSC_BSVIE) && (otgsc & OTGSC_BSVIS))
+- ci_irq(ci->irq, ci);
++ ci_irq(ci);
+ }
+
+ static int ci_controller_resume(struct device *dev)
+diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c
+index fdf79bcf7eb09..0d99ba64ea528 100644
+--- a/drivers/usb/class/cdc-wdm.c
++++ b/drivers/usb/class/cdc-wdm.c
+@@ -774,6 +774,7 @@ static int wdm_release(struct inode *inode, struct file *file)
+ poison_urbs(desc);
+ spin_lock_irq(&desc->iuspin);
+ desc->resp_count = 0;
++ clear_bit(WDM_RESPONDING, &desc->flags);
+ spin_unlock_irq(&desc->iuspin);
+ desc->manage_power(desc->intf, 0);
+ unpoison_urbs(desc);
+diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
+index 73f419adce610..4bb6d304eb4b2 100644
+--- a/drivers/usb/class/usbtmc.c
++++ b/drivers/usb/class/usbtmc.c
+@@ -1919,6 +1919,7 @@ static int usbtmc_ioctl_request(struct usbtmc_device_data *data,
+ struct usbtmc_ctrlrequest request;
+ u8 *buffer = NULL;
+ int rv;
++ unsigned int is_in, pipe;
+ unsigned long res;
+
+ res = copy_from_user(&request, arg, sizeof(struct usbtmc_ctrlrequest));
+@@ -1928,12 +1929,14 @@ static int usbtmc_ioctl_request(struct usbtmc_device_data *data,
+ if (request.req.wLength > USBTMC_BUFSIZE)
+ return -EMSGSIZE;
+
++ is_in = request.req.bRequestType & USB_DIR_IN;
++
+ if (request.req.wLength) {
+ buffer = kmalloc(request.req.wLength, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+- if ((request.req.bRequestType & USB_DIR_IN) == 0) {
++ if (!is_in) {
+ /* Send control data to device */
+ res = copy_from_user(buffer, request.data,
+ request.req.wLength);
+@@ -1944,8 +1947,12 @@ static int usbtmc_ioctl_request(struct usbtmc_device_data *data,
+ }
+ }
+
++ if (is_in)
++ pipe = usb_rcvctrlpipe(data->usb_dev, 0);
++ else
++ pipe = usb_sndctrlpipe(data->usb_dev, 0);
+ rv = usb_control_msg(data->usb_dev,
+- usb_rcvctrlpipe(data->usb_dev, 0),
++ pipe,
+ request.req.bRequest,
+ request.req.bRequestType,
+ request.req.wValue,
+@@ -1957,7 +1964,7 @@ static int usbtmc_ioctl_request(struct usbtmc_device_data *data,
+ goto exit;
+ }
+
+- if (rv && (request.req.bRequestType & USB_DIR_IN)) {
++ if (rv && is_in) {
+ /* Read control data from device */
+ res = copy_to_user(request.data, buffer, rv);
+ if (res)
+diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
+index 4169cf40a03b5..5509d3847af4b 100644
+--- a/drivers/usb/common/ulpi.c
++++ b/drivers/usb/common/ulpi.c
+@@ -39,8 +39,11 @@ static int ulpi_match(struct device *dev, struct device_driver *driver)
+ struct ulpi *ulpi = to_ulpi_dev(dev);
+ const struct ulpi_device_id *id;
+
+- /* Some ULPI devices don't have a vendor id so rely on OF match */
+- if (ulpi->id.vendor == 0)
++ /*
++ * Some ULPI devices don't have a vendor id
++ * or provide an id_table so rely on OF match.
++ */
++ if (ulpi->id.vendor == 0 || !drv->id_table)
+ return of_driver_match_device(dev, driver);
+
+ for (id = drv->id_table; id->vendor; id++)
+@@ -127,6 +130,7 @@ static const struct attribute_group *ulpi_dev_attr_groups[] = {
+
+ static void ulpi_dev_release(struct device *dev)
+ {
++ of_node_put(dev->of_node);
+ kfree(to_ulpi_dev(dev));
+ }
+
+@@ -244,12 +248,16 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi)
+ return ret;
+
+ ret = ulpi_read_id(ulpi);
+- if (ret)
++ if (ret) {
++ of_node_put(ulpi->dev.of_node);
+ return ret;
++ }
+
+ ret = device_register(&ulpi->dev);
+- if (ret)
++ if (ret) {
++ put_device(&ulpi->dev);
+ return ret;
++ }
+
+ dev_dbg(&ulpi->dev, "registered ULPI PHY: vendor %04x, product %04x\n",
+ ulpi->id.vendor, ulpi->id.product);
+@@ -296,7 +304,6 @@ EXPORT_SYMBOL_GPL(ulpi_register_interface);
+ */
+ void ulpi_unregister_interface(struct ulpi *ulpi)
+ {
+- of_node_put(ulpi->dev.of_node);
+ device_unregister(&ulpi->dev);
+ }
+ EXPORT_SYMBOL_GPL(ulpi_unregister_interface);
+diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
+index b199eb65f3780..00e28456e4cc2 100644
+--- a/drivers/usb/core/config.c
++++ b/drivers/usb/core/config.c
+@@ -406,7 +406,7 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno,
+ * the USB-2 spec requires such endpoints to have wMaxPacketSize = 0
+ * (see the end of section 5.6.3), so don't warn about them.
+ */
+- maxp = usb_endpoint_maxp(&endpoint->desc);
++ maxp = le16_to_cpu(endpoint->desc.wMaxPacketSize);
+ if (maxp == 0 && !(usb_endpoint_xfer_isoc(d) && asnum == 0)) {
+ dev_warn(ddev, "config %d interface %d altsetting %d endpoint 0x%X has invalid wMaxPacketSize 0\n",
+ cfgno, inum, asnum, d->bEndpointAddress);
+@@ -422,9 +422,9 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno,
+ maxpacket_maxes = full_speed_maxpacket_maxes;
+ break;
+ case USB_SPEED_HIGH:
+- /* Bits 12..11 are allowed only for HS periodic endpoints */
++ /* Multiple-transactions bits are allowed only for HS periodic endpoints */
+ if (usb_endpoint_xfer_int(d) || usb_endpoint_xfer_isoc(d)) {
+- i = maxp & (BIT(12) | BIT(11));
++ i = maxp & USB_EP_MAXP_MULT_MASK;
+ maxp &= ~i;
+ }
+ fallthrough;
+diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
+index 9618ba622a2d0..656ba91c32831 100644
+--- a/drivers/usb/core/devio.c
++++ b/drivers/usb/core/devio.c
+@@ -32,6 +32,7 @@
+ #include <linux/usb.h>
+ #include <linux/usbdevice_fs.h>
+ #include <linux/usb/hcd.h> /* for usbcore internals */
++#include <linux/usb/quirks.h>
+ #include <linux/cdev.h>
+ #include <linux/notifier.h>
+ #include <linux/security.h>
+@@ -1102,14 +1103,55 @@ static int usbdev_release(struct inode *inode, struct file *file)
+ return 0;
+ }
+
++static void usbfs_blocking_completion(struct urb *urb)
++{
++ complete((struct completion *) urb->context);
++}
++
++/*
++ * Much like usb_start_wait_urb, but returns status separately from
++ * actual_length and uses a killable wait.
++ */
++static int usbfs_start_wait_urb(struct urb *urb, int timeout,
++ unsigned int *actlen)
++{
++ DECLARE_COMPLETION_ONSTACK(ctx);
++ unsigned long expire;
++ int rc;
++
++ urb->context = &ctx;
++ urb->complete = usbfs_blocking_completion;
++ *actlen = 0;
++ rc = usb_submit_urb(urb, GFP_KERNEL);
++ if (unlikely(rc))
++ return rc;
++
++ expire = (timeout ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT);
++ rc = wait_for_completion_killable_timeout(&ctx, expire);
++ if (rc <= 0) {
++ usb_kill_urb(urb);
++ *actlen = urb->actual_length;
++ if (urb->status != -ENOENT)
++ ; /* Completed before it was killed */
++ else if (rc < 0)
++ return -EINTR;
++ else
++ return -ETIMEDOUT;
++ }
++ *actlen = urb->actual_length;
++ return urb->status;
++}
++
+ static int do_proc_control(struct usb_dev_state *ps,
+ struct usbdevfs_ctrltransfer *ctrl)
+ {
+ struct usb_device *dev = ps->dev;
+ unsigned int tmo;
+ unsigned char *tbuf;
+- unsigned wLength;
++ unsigned int wLength, actlen;
+ int i, pipe, ret;
++ struct urb *urb = NULL;
++ struct usb_ctrlrequest *dr = NULL;
+
+ ret = check_ctrlrecip(ps, ctrl->bRequestType, ctrl->bRequest,
+ ctrl->wIndex);
+@@ -1122,51 +1164,71 @@ static int do_proc_control(struct usb_dev_state *ps,
+ sizeof(struct usb_ctrlrequest));
+ if (ret)
+ return ret;
++
++ ret = -ENOMEM;
+ tbuf = (unsigned char *)__get_free_page(GFP_KERNEL);
+- if (!tbuf) {
+- ret = -ENOMEM;
++ if (!tbuf)
+ goto done;
+- }
++ urb = usb_alloc_urb(0, GFP_NOIO);
++ if (!urb)
++ goto done;
++ dr = kmalloc(sizeof(struct usb_ctrlrequest), GFP_NOIO);
++ if (!dr)
++ goto done;
++
++ dr->bRequestType = ctrl->bRequestType;
++ dr->bRequest = ctrl->bRequest;
++ dr->wValue = cpu_to_le16(ctrl->wValue);
++ dr->wIndex = cpu_to_le16(ctrl->wIndex);
++ dr->wLength = cpu_to_le16(ctrl->wLength);
++
+ tmo = ctrl->timeout;
+ snoop(&dev->dev, "control urb: bRequestType=%02x "
+ "bRequest=%02x wValue=%04x "
+ "wIndex=%04x wLength=%04x\n",
+ ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
+ ctrl->wIndex, ctrl->wLength);
+- if ((ctrl->bRequestType & USB_DIR_IN) && ctrl->wLength) {
++
++ if ((ctrl->bRequestType & USB_DIR_IN) && wLength) {
+ pipe = usb_rcvctrlpipe(dev, 0);
+- snoop_urb(dev, NULL, pipe, ctrl->wLength, tmo, SUBMIT, NULL, 0);
++ usb_fill_control_urb(urb, dev, pipe, (unsigned char *) dr, tbuf,
++ wLength, NULL, NULL);
++ snoop_urb(dev, NULL, pipe, wLength, tmo, SUBMIT, NULL, 0);
+
+ usb_unlock_device(dev);
+- i = usb_control_msg(dev, pipe, ctrl->bRequest,
+- ctrl->bRequestType, ctrl->wValue, ctrl->wIndex,
+- tbuf, ctrl->wLength, tmo);
++ i = usbfs_start_wait_urb(urb, tmo, &actlen);
++
++ /* Linger a bit, prior to the next control message. */
++ if (dev->quirks & USB_QUIRK_DELAY_CTRL_MSG)
++ msleep(200);
+ usb_lock_device(dev);
+- snoop_urb(dev, NULL, pipe, max(i, 0), min(i, 0), COMPLETE,
+- tbuf, max(i, 0));
+- if ((i > 0) && ctrl->wLength) {
+- if (copy_to_user(ctrl->data, tbuf, i)) {
++ snoop_urb(dev, NULL, pipe, actlen, i, COMPLETE, tbuf, actlen);
++ if (!i && actlen) {
++ if (copy_to_user(ctrl->data, tbuf, actlen)) {
+ ret = -EFAULT;
+ goto done;
+ }
+ }
+ } else {
+- if (ctrl->wLength) {
+- if (copy_from_user(tbuf, ctrl->data, ctrl->wLength)) {
++ if (wLength) {
++ if (copy_from_user(tbuf, ctrl->data, wLength)) {
+ ret = -EFAULT;
+ goto done;
+ }
+ }
+ pipe = usb_sndctrlpipe(dev, 0);
+- snoop_urb(dev, NULL, pipe, ctrl->wLength, tmo, SUBMIT,
+- tbuf, ctrl->wLength);
++ usb_fill_control_urb(urb, dev, pipe, (unsigned char *) dr, tbuf,
++ wLength, NULL, NULL);
++ snoop_urb(dev, NULL, pipe, wLength, tmo, SUBMIT, tbuf, wLength);
+
+ usb_unlock_device(dev);
+- i = usb_control_msg(dev, pipe, ctrl->bRequest,
+- ctrl->bRequestType, ctrl->wValue, ctrl->wIndex,
+- tbuf, ctrl->wLength, tmo);
++ i = usbfs_start_wait_urb(urb, tmo, &actlen);
++
++ /* Linger a bit, prior to the next control message. */
++ if (dev->quirks & USB_QUIRK_DELAY_CTRL_MSG)
++ msleep(200);
+ usb_lock_device(dev);
+- snoop_urb(dev, NULL, pipe, max(i, 0), min(i, 0), COMPLETE, NULL, 0);
++ snoop_urb(dev, NULL, pipe, actlen, i, COMPLETE, NULL, 0);
+ }
+ if (i < 0 && i != -EPIPE) {
+ dev_printk(KERN_DEBUG, &dev->dev, "usbfs: USBDEVFS_CONTROL "
+@@ -1174,8 +1236,11 @@ static int do_proc_control(struct usb_dev_state *ps,
+ current->comm, ctrl->bRequestType, ctrl->bRequest,
+ ctrl->wLength, i);
+ }
+- ret = i;
++ ret = (i < 0 ? i : actlen);
++
+ done:
++ kfree(dr);
++ usb_free_urb(urb);
+ free_page((unsigned long) tbuf);
+ usbfs_decrease_memory_usage(PAGE_SIZE + sizeof(struct urb) +
+ sizeof(struct usb_ctrlrequest));
+@@ -1195,10 +1260,11 @@ static int do_proc_bulk(struct usb_dev_state *ps,
+ struct usbdevfs_bulktransfer *bulk)
+ {
+ struct usb_device *dev = ps->dev;
+- unsigned int tmo, len1, pipe;
+- int len2;
++ unsigned int tmo, len1, len2, pipe;
+ unsigned char *tbuf;
+ int i, ret;
++ struct urb *urb = NULL;
++ struct usb_host_endpoint *ep;
+
+ ret = findintfep(ps->dev, bulk->ep);
+ if (ret < 0)
+@@ -1206,14 +1272,17 @@ static int do_proc_bulk(struct usb_dev_state *ps,
+ ret = checkintf(ps, ret);
+ if (ret)
+ return ret;
++
++ len1 = bulk->len;
++ if (len1 < 0 || len1 >= (INT_MAX - sizeof(struct urb)))
++ return -EINVAL;
++
+ if (bulk->ep & USB_DIR_IN)
+ pipe = usb_rcvbulkpipe(dev, bulk->ep & 0x7f);
+ else
+ pipe = usb_sndbulkpipe(dev, bulk->ep & 0x7f);
+- if (!usb_maxpacket(dev, pipe, !(bulk->ep & USB_DIR_IN)))
+- return -EINVAL;
+- len1 = bulk->len;
+- if (len1 >= (INT_MAX - sizeof(struct urb)))
++ ep = usb_pipe_endpoint(dev, pipe);
++ if (!ep || !usb_endpoint_maxp(&ep->desc))
+ return -EINVAL;
+ ret = usbfs_increase_memory_usage(len1 + sizeof(struct urb));
+ if (ret)
+@@ -1223,17 +1292,29 @@ static int do_proc_bulk(struct usb_dev_state *ps,
+ * len1 can be almost arbitrarily large. Don't WARN if it's
+ * too big, just fail the request.
+ */
++ ret = -ENOMEM;
+ tbuf = kmalloc(len1, GFP_KERNEL | __GFP_NOWARN);
+- if (!tbuf) {
+- ret = -ENOMEM;
++ if (!tbuf)
++ goto done;
++ urb = usb_alloc_urb(0, GFP_KERNEL);
++ if (!urb)
+ goto done;
++
++ if ((ep->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
++ USB_ENDPOINT_XFER_INT) {
++ pipe = (pipe & ~(3 << 30)) | (PIPE_INTERRUPT << 30);
++ usb_fill_int_urb(urb, dev, pipe, tbuf, len1,
++ NULL, NULL, ep->desc.bInterval);
++ } else {
++ usb_fill_bulk_urb(urb, dev, pipe, tbuf, len1, NULL, NULL);
+ }
++
+ tmo = bulk->timeout;
+ if (bulk->ep & 0x80) {
+ snoop_urb(dev, NULL, pipe, len1, tmo, SUBMIT, NULL, 0);
+
+ usb_unlock_device(dev);
+- i = usb_bulk_msg(dev, pipe, tbuf, len1, &len2, tmo);
++ i = usbfs_start_wait_urb(urb, tmo, &len2);
+ usb_lock_device(dev);
+ snoop_urb(dev, NULL, pipe, len2, i, COMPLETE, tbuf, len2);
+
+@@ -1253,12 +1334,13 @@ static int do_proc_bulk(struct usb_dev_state *ps,
+ snoop_urb(dev, NULL, pipe, len1, tmo, SUBMIT, tbuf, len1);
+
+ usb_unlock_device(dev);
+- i = usb_bulk_msg(dev, pipe, tbuf, len1, &len2, tmo);
++ i = usbfs_start_wait_urb(urb, tmo, &len2);
+ usb_lock_device(dev);
+ snoop_urb(dev, NULL, pipe, len2, i, COMPLETE, NULL, 0);
+ }
+ ret = (i < 0 ? i : len2);
+ done:
++ usb_free_urb(urb);
+ kfree(tbuf);
+ usbfs_decrease_memory_usage(len1 + sizeof(struct urb));
+ return ret;
+diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
+index 7ee6e4cc0d89e..dd3c288fa952f 100644
+--- a/drivers/usb/core/hcd.c
++++ b/drivers/usb/core/hcd.c
+@@ -753,6 +753,7 @@ void usb_hcd_poll_rh_status(struct usb_hcd *hcd)
+ {
+ struct urb *urb;
+ int length;
++ int status;
+ unsigned long flags;
+ char buffer[6]; /* Any root hubs with > 31 ports? */
+
+@@ -770,11 +771,17 @@ void usb_hcd_poll_rh_status(struct usb_hcd *hcd)
+ if (urb) {
+ clear_bit(HCD_FLAG_POLL_PENDING, &hcd->flags);
+ hcd->status_urb = NULL;
++ if (urb->transfer_buffer_length >= length) {
++ status = 0;
++ } else {
++ status = -EOVERFLOW;
++ length = urb->transfer_buffer_length;
++ }
+ urb->actual_length = length;
+ memcpy(urb->transfer_buffer, buffer, length);
+
+ usb_hcd_unlink_urb_from_ep(hcd, urb);
+- usb_hcd_giveback_urb(hcd, urb, 0);
++ usb_hcd_giveback_urb(hcd, urb, status);
+ } else {
+ length = 0;
+ set_bit(HCD_FLAG_POLL_PENDING, &hcd->flags);
+@@ -1556,6 +1563,13 @@ int usb_hcd_submit_urb (struct urb *urb, gfp_t mem_flags)
+ urb->hcpriv = NULL;
+ INIT_LIST_HEAD(&urb->urb_list);
+ atomic_dec(&urb->use_count);
++ /*
++ * Order the write of urb->use_count above before the read
++ * of urb->reject below. Pairs with the memory barriers in
++ * usb_kill_urb() and usb_poison_urb().
++ */
++ smp_mb__after_atomic();
++
+ atomic_dec(&urb->dev->urbnum);
+ if (atomic_read(&urb->reject))
+ wake_up(&usb_kill_urb_queue);
+@@ -1658,6 +1672,13 @@ static void __usb_hcd_giveback_urb(struct urb *urb)
+
+ usb_anchor_resume_wakeups(anchor);
+ atomic_dec(&urb->use_count);
++ /*
++ * Order the write of urb->use_count above before the read
++ * of urb->reject below. Pairs with the memory barriers in
++ * usb_kill_urb() and usb_poison_urb().
++ */
++ smp_mb__after_atomic();
++
+ if (unlikely(atomic_read(&urb->reject)))
+ wake_up(&usb_kill_urb_queue);
+ usb_put_urb(urb);
+@@ -2795,7 +2816,6 @@ int usb_add_hcd(struct usb_hcd *hcd,
+ {
+ int retval;
+ struct usb_device *rhdev;
+- struct usb_hcd *shared_hcd;
+
+ if (!hcd->skip_phy_initialization && usb_hcd_is_primary_hcd(hcd)) {
+ hcd->phy_roothub = usb_phy_roothub_alloc(hcd->self.sysdev);
+@@ -2956,26 +2976,13 @@ int usb_add_hcd(struct usb_hcd *hcd,
+ goto err_hcd_driver_start;
+ }
+
+- /* starting here, usbcore will pay attention to the shared HCD roothub */
+- shared_hcd = hcd->shared_hcd;
+- if (!usb_hcd_is_primary_hcd(hcd) && shared_hcd && HCD_DEFER_RH_REGISTER(shared_hcd)) {
+- retval = register_root_hub(shared_hcd);
+- if (retval != 0)
+- goto err_register_root_hub;
+-
+- if (shared_hcd->uses_new_polling && HCD_POLL_RH(shared_hcd))
+- usb_hcd_poll_rh_status(shared_hcd);
+- }
+-
+ /* starting here, usbcore will pay attention to this root hub */
+- if (!HCD_DEFER_RH_REGISTER(hcd)) {
+- retval = register_root_hub(hcd);
+- if (retval != 0)
+- goto err_register_root_hub;
++ retval = register_root_hub(hcd);
++ if (retval != 0)
++ goto err_register_root_hub;
+
+- if (hcd->uses_new_polling && HCD_POLL_RH(hcd))
+- usb_hcd_poll_rh_status(hcd);
+- }
++ if (hcd->uses_new_polling && HCD_POLL_RH(hcd))
++ usb_hcd_poll_rh_status(hcd);
+
+ return retval;
+
+@@ -3013,7 +3020,6 @@ EXPORT_SYMBOL_GPL(usb_add_hcd);
+ void usb_remove_hcd(struct usb_hcd *hcd)
+ {
+ struct usb_device *rhdev = hcd->self.root_hub;
+- bool rh_registered;
+
+ dev_info(hcd->self.controller, "remove, state %x\n", hcd->state);
+
+@@ -3024,7 +3030,6 @@ void usb_remove_hcd(struct usb_hcd *hcd)
+
+ dev_dbg(hcd->self.controller, "roothub graceful disconnect\n");
+ spin_lock_irq (&hcd_root_hub_lock);
+- rh_registered = hcd->rh_registered;
+ hcd->rh_registered = 0;
+ spin_unlock_irq (&hcd_root_hub_lock);
+
+@@ -3034,8 +3039,7 @@ void usb_remove_hcd(struct usb_hcd *hcd)
+ cancel_work_sync(&hcd->died_work);
+
+ mutex_lock(&usb_bus_idr_lock);
+- if (rh_registered)
+- usb_disconnect(&rhdev); /* Sets rhdev to NULL */
++ usb_disconnect(&rhdev); /* Sets rhdev to NULL */
+ mutex_unlock(&usb_bus_idr_lock);
+
+ /*
+diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
+index 86658a81d2844..ac6c5ccfe1cb7 100644
+--- a/drivers/usb/core/hub.c
++++ b/drivers/usb/core/hub.c
+@@ -1110,7 +1110,10 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
+ } else {
+ hub_power_on(hub, true);
+ }
+- }
++ /* Give some time on remote wakeup to let links to transit to U0 */
++ } else if (hub_is_superspeed(hub->hdev))
++ msleep(20);
++
+ init2:
+
+ /*
+@@ -1225,7 +1228,7 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
+ */
+ if (portchange || (hub_is_superspeed(hub->hdev) &&
+ port_resumed))
+- set_bit(port1, hub->change_bits);
++ set_bit(port1, hub->event_bits);
+
+ } else if (udev->persist_enabled) {
+ #ifdef CONFIG_PM
+@@ -4700,8 +4703,6 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1,
+ if (oldspeed == USB_SPEED_LOW)
+ delay = HUB_LONG_RESET_TIME;
+
+- mutex_lock(hcd->address0_mutex);
+-
+ /* Reset the device; full speed may morph to high speed */
+ /* FIXME a USB 2.0 device may morph into SuperSpeed on reset. */
+ retval = hub_port_reset(hub, port1, udev, delay, false);
+@@ -5016,7 +5017,6 @@ fail:
+ hub_port_disable(hub, port1, 0);
+ update_devnum(udev, devnum); /* for disconnect processing */
+ }
+- mutex_unlock(hcd->address0_mutex);
+ return retval;
+ }
+
+@@ -5191,6 +5191,7 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus,
+ struct usb_port *port_dev = hub->ports[port1 - 1];
+ struct usb_device *udev = port_dev->child;
+ static int unreliable_port = -1;
++ bool retry_locked;
+
+ /* Disconnect any existing devices under this port */
+ if (udev) {
+@@ -5246,8 +5247,11 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus,
+ unit_load = 100;
+
+ status = 0;
+- for (i = 0; i < PORT_INIT_TRIES; i++) {
+
++ for (i = 0; i < PORT_INIT_TRIES; i++) {
++ usb_lock_port(port_dev);
++ mutex_lock(hcd->address0_mutex);
++ retry_locked = true;
+ /* reallocate for each attempt, since references
+ * to the previous one can escape in various ways
+ */
+@@ -5255,6 +5259,8 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus,
+ if (!udev) {
+ dev_err(&port_dev->dev,
+ "couldn't allocate usb_device\n");
++ mutex_unlock(hcd->address0_mutex);
++ usb_unlock_port(port_dev);
+ goto done;
+ }
+
+@@ -5276,12 +5282,14 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus,
+ }
+
+ /* reset (non-USB 3.0 devices) and get descriptor */
+- usb_lock_port(port_dev);
+ status = hub_port_init(hub, udev, port1, i);
+- usb_unlock_port(port_dev);
+ if (status < 0)
+ goto loop;
+
++ mutex_unlock(hcd->address0_mutex);
++ usb_unlock_port(port_dev);
++ retry_locked = false;
++
+ if (udev->quirks & USB_QUIRK_DELAY_INIT)
+ msleep(2000);
+
+@@ -5374,6 +5382,10 @@ loop:
+ usb_ep0_reinit(udev);
+ release_devnum(udev);
+ hub_free_dev(udev);
++ if (retry_locked) {
++ mutex_unlock(hcd->address0_mutex);
++ usb_unlock_port(port_dev);
++ }
+ usb_put_dev(udev);
+ if ((status == -ENOTCONN) || (status == -ENOTSUPP))
+ break;
+@@ -5915,6 +5927,8 @@ static int usb_reset_and_verify_device(struct usb_device *udev)
+ bos = udev->bos;
+ udev->bos = NULL;
+
++ mutex_lock(hcd->address0_mutex);
++
+ for (i = 0; i < PORT_INIT_TRIES; ++i) {
+
+ /* ep0 maxpacket size may change; let the HCD know about it.
+@@ -5924,6 +5938,7 @@ static int usb_reset_and_verify_device(struct usb_device *udev)
+ if (ret >= 0 || ret == -ENOTCONN || ret == -ENODEV)
+ break;
+ }
++ mutex_unlock(hcd->address0_mutex);
+
+ if (ret < 0)
+ goto re_enumerate;
+diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c
+index 8239fe7129dd7..97b44a68668a5 100644
+--- a/drivers/usb/core/quirks.c
++++ b/drivers/usb/core/quirks.c
+@@ -404,6 +404,9 @@ static const struct usb_device_id usb_quirk_list[] = {
+ { USB_DEVICE(0x0b05, 0x17e0), .driver_info =
+ USB_QUIRK_IGNORE_REMOTE_WAKEUP },
+
++ /* Realtek Semiconductor Corp. Mass Storage Device (Multicard Reader)*/
++ { USB_DEVICE(0x0bda, 0x0151), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS },
++
+ /* Realtek hub in Dell WD19 (Type-C) */
+ { USB_DEVICE(0x0bda, 0x0487), .driver_info = USB_QUIRK_NO_LPM },
+
+@@ -434,6 +437,12 @@ static const struct usb_device_id usb_quirk_list[] = {
+ { USB_DEVICE(0x1532, 0x0116), .driver_info =
+ USB_QUIRK_LINEAR_UFRAME_INTR_BINTERVAL },
+
++ /* Lenovo USB-C to Ethernet Adapter RTL8153-04 */
++ { USB_DEVICE(0x17ef, 0x720c), .driver_info = USB_QUIRK_NO_LPM },
++
++ /* Lenovo Powered USB-C Travel Hub (4X90S92381, RTL8153 GigE) */
++ { USB_DEVICE(0x17ef, 0x721e), .driver_info = USB_QUIRK_NO_LPM },
++
+ /* Lenovo ThinkCenter A630Z TI024Gen3 usb-audio */
+ { USB_DEVICE(0x17ef, 0xa012), .driver_info =
+ USB_QUIRK_DISCONNECT_SUSPEND },
+@@ -501,6 +510,9 @@ static const struct usb_device_id usb_quirk_list[] = {
+ /* DJI CineSSD */
+ { USB_DEVICE(0x2ca3, 0x0031), .driver_info = USB_QUIRK_NO_LPM },
+
++ /* VCOM device */
++ { USB_DEVICE(0x4296, 0x7570), .driver_info = USB_QUIRK_CONFIG_INTF_STRINGS },
++
+ /* INTEL VALUE SSD */
+ { USB_DEVICE(0x8086, 0xf1a5), .driver_info = USB_QUIRK_RESET_RESUME },
+
+diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c
+index 30727729a44cc..33d62d7e3929f 100644
+--- a/drivers/usb/core/urb.c
++++ b/drivers/usb/core/urb.c
+@@ -715,6 +715,12 @@ void usb_kill_urb(struct urb *urb)
+ if (!(urb && urb->dev && urb->ep))
+ return;
+ atomic_inc(&urb->reject);
++ /*
++ * Order the write of urb->reject above before the read
++ * of urb->use_count below. Pairs with the barriers in
++ * __usb_hcd_giveback_urb() and usb_hcd_submit_urb().
++ */
++ smp_mb__after_atomic();
+
+ usb_hcd_unlink_urb(urb, -ENOENT);
+ wait_event(usb_kill_urb_queue, atomic_read(&urb->use_count) == 0);
+@@ -756,6 +762,12 @@ void usb_poison_urb(struct urb *urb)
+ if (!urb)
+ return;
+ atomic_inc(&urb->reject);
++ /*
++ * Order the write of urb->reject above before the read
++ * of urb->use_count below. Pairs with the barriers in
++ * __usb_hcd_giveback_urb() and usb_hcd_submit_urb().
++ */
++ smp_mb__after_atomic();
+
+ if (!urb->dev || !urb->ep)
+ return;
+diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
+index cb9059a8444b5..71e62b3081dbb 100644
+--- a/drivers/usb/dwc2/core.h
++++ b/drivers/usb/dwc2/core.h
+@@ -1417,6 +1417,7 @@ void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg);
+ void dwc2_hsotg_disconnect(struct dwc2_hsotg *dwc2);
+ int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode);
+ #define dwc2_is_device_connected(hsotg) (hsotg->connected)
++#define dwc2_is_device_enabled(hsotg) (hsotg->enabled)
+ int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg);
+ int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg, int remote_wakeup);
+ int dwc2_gadget_enter_hibernation(struct dwc2_hsotg *hsotg);
+@@ -1453,6 +1454,7 @@ static inline int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg,
+ int testmode)
+ { return 0; }
+ #define dwc2_is_device_connected(hsotg) (0)
++#define dwc2_is_device_enabled(hsotg) (0)
+ static inline int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg)
+ { return 0; }
+ static inline int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg,
+diff --git a/drivers/usb/dwc2/drd.c b/drivers/usb/dwc2/drd.c
+index 2d4176f5788eb..36f2c38416e5e 100644
+--- a/drivers/usb/dwc2/drd.c
++++ b/drivers/usb/dwc2/drd.c
+@@ -7,6 +7,7 @@
+ * Author(s): Amelie Delaunay <amelie.delaunay@st.com>
+ */
+
++#include <linux/clk.h>
+ #include <linux/iopoll.h>
+ #include <linux/platform_device.h>
+ #include <linux/usb/role.h>
+@@ -25,9 +26,9 @@ static void dwc2_ovr_init(struct dwc2_hsotg *hsotg)
+ gotgctl &= ~(GOTGCTL_BVALOVAL | GOTGCTL_AVALOVAL | GOTGCTL_VBVALOVAL);
+ dwc2_writel(hsotg, gotgctl, GOTGCTL);
+
+- dwc2_force_mode(hsotg, false);
+-
+ spin_unlock_irqrestore(&hsotg->lock, flags);
++
++ dwc2_force_mode(hsotg, (hsotg->dr_mode == USB_DR_MODE_HOST));
+ }
+
+ static int dwc2_ovr_avalid(struct dwc2_hsotg *hsotg, bool valid)
+@@ -39,6 +40,7 @@ static int dwc2_ovr_avalid(struct dwc2_hsotg *hsotg, bool valid)
+ (!valid && !(gotgctl & GOTGCTL_ASESVLD)))
+ return -EALREADY;
+
++ gotgctl &= ~GOTGCTL_BVALOVAL;
+ if (valid)
+ gotgctl |= GOTGCTL_AVALOVAL | GOTGCTL_VBVALOVAL;
+ else
+@@ -57,6 +59,7 @@ static int dwc2_ovr_bvalid(struct dwc2_hsotg *hsotg, bool valid)
+ (!valid && !(gotgctl & GOTGCTL_BSESVLD)))
+ return -EALREADY;
+
++ gotgctl &= ~GOTGCTL_AVALOVAL;
+ if (valid)
+ gotgctl |= GOTGCTL_BVALOVAL | GOTGCTL_VBVALOVAL;
+ else
+@@ -86,14 +89,30 @@ static int dwc2_drd_role_sw_set(struct usb_role_switch *sw, enum usb_role role)
+ }
+ #endif
+
++ /*
++ * In case of USB_DR_MODE_PERIPHERAL, clock is disabled at the end of
++ * the probe and enabled on udc_start.
++ * If role-switch set is called before the udc_start, we need to enable
++ * the clock to read/write GOTGCTL and GUSBCFG registers to override
++ * mode and sessions. It is the case if cable is plugged at boot.
++ */
++ if (!hsotg->ll_hw_enabled && hsotg->clk) {
++ int ret = clk_prepare_enable(hsotg->clk);
++
++ if (ret)
++ return ret;
++ }
++
+ spin_lock_irqsave(&hsotg->lock, flags);
+
+ if (role == USB_ROLE_HOST) {
+ already = dwc2_ovr_avalid(hsotg, true);
+ } else if (role == USB_ROLE_DEVICE) {
+ already = dwc2_ovr_bvalid(hsotg, true);
+- /* This clear DCTL.SFTDISCON bit */
+- dwc2_hsotg_core_connect(hsotg);
++ if (dwc2_is_device_enabled(hsotg)) {
++ /* This clear DCTL.SFTDISCON bit */
++ dwc2_hsotg_core_connect(hsotg);
++ }
+ } else {
+ if (dwc2_is_device_mode(hsotg)) {
+ if (!dwc2_ovr_bvalid(hsotg, false))
+@@ -110,6 +129,9 @@ static int dwc2_drd_role_sw_set(struct usb_role_switch *sw, enum usb_role role)
+ /* This will raise a Connector ID Status Change Interrupt */
+ dwc2_force_mode(hsotg, role == USB_ROLE_HOST);
+
++ if (!hsotg->ll_hw_enabled && hsotg->clk)
++ clk_disable_unprepare(hsotg->clk);
++
+ dev_dbg(hsotg->dev, "%s-session valid\n",
+ role == USB_ROLE_NONE ? "No" :
+ role == USB_ROLE_HOST ? "A" : "B");
+diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
+index 11d85a6e0b0dc..0909b088a284b 100644
+--- a/drivers/usb/dwc2/gadget.c
++++ b/drivers/usb/dwc2/gadget.c
+@@ -1198,6 +1198,8 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
+ }
+ ctrl |= DXEPCTL_CNAK;
+ } else {
++ hs_req->req.frame_number = hs_ep->target_frame;
++ hs_req->req.actual = 0;
+ dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
+ return;
+ }
+@@ -2857,9 +2859,12 @@ static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
+
+ do {
+ hs_req = get_ep_head(hs_ep);
+- if (hs_req)
++ if (hs_req) {
++ hs_req->req.frame_number = hs_ep->target_frame;
++ hs_req->req.actual = 0;
+ dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
+ -ENODATA);
++ }
+ dwc2_gadget_incr_frame_num(hs_ep);
+ /* Update current frame number value. */
+ hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
+@@ -2912,8 +2917,11 @@ static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
+
+ while (dwc2_gadget_target_frame_elapsed(ep)) {
+ hs_req = get_ep_head(ep);
+- if (hs_req)
++ if (hs_req) {
++ hs_req->req.frame_number = ep->target_frame;
++ hs_req->req.actual = 0;
+ dwc2_hsotg_complete_request(hsotg, ep, hs_req, -ENODATA);
++ }
+
+ dwc2_gadget_incr_frame_num(ep);
+ /* Update current frame number value. */
+@@ -3002,8 +3010,11 @@ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
+
+ while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
+ hs_req = get_ep_head(hs_ep);
+- if (hs_req)
++ if (hs_req) {
++ hs_req->req.frame_number = hs_ep->target_frame;
++ hs_req->req.actual = 0;
+ dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
++ }
+
+ dwc2_gadget_incr_frame_num(hs_ep);
+ /* Update current frame number value. */
+@@ -4963,7 +4974,18 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg)
+ hsotg->params.g_np_tx_fifo_size);
+ dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
+
+- hsotg->gadget.max_speed = USB_SPEED_HIGH;
++ switch (hsotg->params.speed) {
++ case DWC2_SPEED_PARAM_LOW:
++ hsotg->gadget.max_speed = USB_SPEED_LOW;
++ break;
++ case DWC2_SPEED_PARAM_FULL:
++ hsotg->gadget.max_speed = USB_SPEED_FULL;
++ break;
++ default:
++ hsotg->gadget.max_speed = USB_SPEED_HIGH;
++ break;
++ }
++
+ hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
+ hsotg->gadget.name = dev_name(dev);
+ hsotg->remote_wakeup_allowed = 0;
+@@ -5074,7 +5096,7 @@ int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg)
+ hsotg->gadget.speed = USB_SPEED_UNKNOWN;
+ spin_unlock_irqrestore(&hsotg->lock, flags);
+
+- for (ep = 0; ep < hsotg->num_of_eps; ep++) {
++ for (ep = 1; ep < hsotg->num_of_eps; ep++) {
+ if (hsotg->eps_in[ep])
+ dwc2_hsotg_ep_disable_lock(&hsotg->eps_in[ep]->ep);
+ if (hsotg->eps_out[ep])
+diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
+index a215ec9e172e6..657dbd50faf11 100644
+--- a/drivers/usb/dwc2/hcd.c
++++ b/drivers/usb/dwc2/hcd.c
+@@ -4403,11 +4403,12 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
+ * If not hibernation nor partial power down are supported,
+ * clock gating is used to save power.
+ */
+- if (!hsotg->params.no_clock_gating)
++ if (!hsotg->params.no_clock_gating) {
+ dwc2_host_enter_clock_gating(hsotg);
+
+- /* After entering suspend, hardware is not accessible */
+- clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
++ /* After entering suspend, hardware is not accessible */
++ clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
++ }
+ break;
+ default:
+ goto skip_power_saving;
+diff --git a/drivers/usb/dwc2/hcd_queue.c b/drivers/usb/dwc2/hcd_queue.c
+index 89a788326c562..24beff610cf2c 100644
+--- a/drivers/usb/dwc2/hcd_queue.c
++++ b/drivers/usb/dwc2/hcd_queue.c
+@@ -59,7 +59,7 @@
+ #define DWC2_UNRESERVE_DELAY (msecs_to_jiffies(5))
+
+ /* If we get a NAK, wait this long before retrying */
+-#define DWC2_RETRY_WAIT_DELAY (1 * 1E6L)
++#define DWC2_RETRY_WAIT_DELAY (1 * NSEC_PER_MSEC)
+
+ /**
+ * dwc2_periodic_channel_available() - Checks that a channel is available for a
+diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
+index c8f18f3ba9e35..c331a5128c2c0 100644
+--- a/drivers/usb/dwc2/platform.c
++++ b/drivers/usb/dwc2/platform.c
+@@ -575,6 +575,9 @@ static int dwc2_driver_probe(struct platform_device *dev)
+ ggpio |= GGPIO_STM32_OTG_GCCFG_IDEN;
+ ggpio |= GGPIO_STM32_OTG_GCCFG_VBDEN;
+ dwc2_writel(hsotg, ggpio, GGPIO);
++
++ /* ID/VBUS detection startup time */
++ usleep_range(5000, 7000);
+ }
+
+ retval = dwc2_drd_init(hsotg);
+diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
+index 0104a80b185e1..5cb1350ec66d1 100644
+--- a/drivers/usb/dwc3/core.c
++++ b/drivers/usb/dwc3/core.c
+@@ -275,7 +275,8 @@ static int dwc3_core_soft_reset(struct dwc3 *dwc)
+
+ reg = dwc3_readl(dwc->regs, DWC3_DCTL);
+ reg |= DWC3_DCTL_CSFTRST;
+- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
++ reg &= ~DWC3_DCTL_RUN_STOP;
++ dwc3_gadget_dctl_write_safe(dwc, reg);
+
+ /*
+ * For DWC_usb31 controller 1.90a and later, the DCTL.CSFRST bit
+@@ -1268,10 +1269,10 @@ static void dwc3_get_properties(struct dwc3 *dwc)
+ u8 lpm_nyet_threshold;
+ u8 tx_de_emphasis;
+ u8 hird_threshold;
+- u8 rx_thr_num_pkt_prd;
+- u8 rx_max_burst_prd;
+- u8 tx_thr_num_pkt_prd;
+- u8 tx_max_burst_prd;
++ u8 rx_thr_num_pkt_prd = 0;
++ u8 rx_max_burst_prd = 0;
++ u8 tx_thr_num_pkt_prd = 0;
++ u8 tx_max_burst_prd = 0;
+ u8 tx_fifo_resize_max_num;
+ const char *usb_psy_name;
+ int ret;
+@@ -1565,9 +1566,11 @@ static int dwc3_probe(struct platform_device *pdev)
+
+ dwc3_get_properties(dwc);
+
+- ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64));
+- if (ret)
+- return ret;
++ if (!dwc->sysdev_is_parent) {
++ ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64));
++ if (ret)
++ return ret;
++ }
+
+ dwc->reset = devm_reset_control_array_get_optional_shared(dev);
+ if (IS_ERR(dwc->reset))
+diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
+index 5612bfdf37da9..fd5d42ec53501 100644
+--- a/drivers/usb/dwc3/core.h
++++ b/drivers/usb/dwc3/core.h
+@@ -143,7 +143,7 @@
+ #define DWC3_GHWPARAMS8 0xc600
+ #define DWC3_GUCTL3 0xc60c
+ #define DWC3_GFLADJ 0xc630
+-#define DWC3_GHWPARAMS9 0xc680
++#define DWC3_GHWPARAMS9 0xc6e0
+
+ /* Device Registers */
+ #define DWC3_DCFG 0xc700
+@@ -723,6 +723,7 @@ struct dwc3_ep {
+ #define DWC3_EP_FORCE_RESTART_STREAM BIT(9)
+ #define DWC3_EP_FIRST_STREAM_PRIMED BIT(10)
+ #define DWC3_EP_PENDING_CLEAR_STALL BIT(11)
++#define DWC3_EP_TXFIFO_RESIZED BIT(12)
+
+ /* This last one is specific to EP0 */
+ #define DWC3_EP0_DIR_IN BIT(31)
+diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c
+index d7f76835137fa..f148b0370f829 100644
+--- a/drivers/usb/dwc3/drd.c
++++ b/drivers/usb/dwc3/drd.c
+@@ -571,16 +571,15 @@ int dwc3_drd_init(struct dwc3 *dwc)
+ {
+ int ret, irq;
+
++ if (ROLE_SWITCH &&
++ device_property_read_bool(dwc->dev, "usb-role-switch"))
++ return dwc3_setup_role_switch(dwc);
++
+ dwc->edev = dwc3_get_extcon(dwc);
+ if (IS_ERR(dwc->edev))
+ return PTR_ERR(dwc->edev);
+
+- if (ROLE_SWITCH &&
+- device_property_read_bool(dwc->dev, "usb-role-switch")) {
+- ret = dwc3_setup_role_switch(dwc);
+- if (ret < 0)
+- return ret;
+- } else if (dwc->edev) {
++ if (dwc->edev) {
+ dwc->edev_nb.notifier_call = dwc3_drd_notifier;
+ ret = extcon_register_notifier(dwc->edev, EXTCON_USB_HOST,
+ &dwc->edev_nb);
+diff --git a/drivers/usb/dwc3/dwc3-meson-g12a.c b/drivers/usb/dwc3/dwc3-meson-g12a.c
+index d0f9b7c296b0d..bd814df3bf8b8 100644
+--- a/drivers/usb/dwc3/dwc3-meson-g12a.c
++++ b/drivers/usb/dwc3/dwc3-meson-g12a.c
+@@ -755,16 +755,16 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
+
+ ret = dwc3_meson_g12a_get_phys(priv);
+ if (ret)
+- goto err_disable_clks;
++ goto err_rearm;
+
+ ret = priv->drvdata->setup_regmaps(priv, base);
+ if (ret)
+- goto err_disable_clks;
++ goto err_rearm;
+
+ if (priv->vbus) {
+ ret = regulator_enable(priv->vbus);
+ if (ret)
+- goto err_disable_clks;
++ goto err_rearm;
+ }
+
+ /* Get dr_mode */
+@@ -825,6 +825,9 @@ err_disable_regulator:
+ if (priv->vbus)
+ regulator_disable(priv->vbus);
+
++err_rearm:
++ reset_control_rearm(priv->reset);
++
+ err_disable_clks:
+ clk_bulk_disable_unprepare(priv->drvdata->num_clks,
+ priv->drvdata->clks);
+@@ -852,6 +855,8 @@ static int dwc3_meson_g12a_remove(struct platform_device *pdev)
+ pm_runtime_put_noidle(dev);
+ pm_runtime_set_suspended(dev);
+
++ reset_control_rearm(priv->reset);
++
+ clk_bulk_disable_unprepare(priv->drvdata->num_clks,
+ priv->drvdata->clks);
+
+@@ -892,7 +897,7 @@ static int __maybe_unused dwc3_meson_g12a_suspend(struct device *dev)
+ phy_exit(priv->phys[i]);
+ }
+
+- reset_control_assert(priv->reset);
++ reset_control_rearm(priv->reset);
+
+ return 0;
+ }
+@@ -902,7 +907,9 @@ static int __maybe_unused dwc3_meson_g12a_resume(struct device *dev)
+ struct dwc3_meson_g12a *priv = dev_get_drvdata(dev);
+ int i, ret;
+
+- reset_control_deassert(priv->reset);
++ ret = reset_control_reset(priv->reset);
++ if (ret)
++ return ret;
+
+ ret = priv->drvdata->usb_init(priv);
+ if (ret)
+diff --git a/drivers/usb/dwc3/dwc3-omap.c b/drivers/usb/dwc3/dwc3-omap.c
+index e196673f5c647..efaf0db595f46 100644
+--- a/drivers/usb/dwc3/dwc3-omap.c
++++ b/drivers/usb/dwc3/dwc3-omap.c
+@@ -242,7 +242,7 @@ static void dwc3_omap_set_mailbox(struct dwc3_omap *omap,
+ break;
+
+ case OMAP_DWC3_ID_FLOAT:
+- if (omap->vbus_reg)
++ if (omap->vbus_reg && regulator_is_enabled(omap->vbus_reg))
+ regulator_disable(omap->vbus_reg);
+ val = dwc3_omap_read_utmi_ctrl(omap);
+ val |= USBOTGSS_UTMI_OTG_CTRL_IDDIG;
+diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c
+index 7ff8fc8f79a9b..f08b2178fd32d 100644
+--- a/drivers/usb/dwc3/dwc3-pci.c
++++ b/drivers/usb/dwc3/dwc3-pci.c
+@@ -43,6 +43,9 @@
+ #define PCI_DEVICE_ID_INTEL_ADLP 0x51ee
+ #define PCI_DEVICE_ID_INTEL_ADLM 0x54ee
+ #define PCI_DEVICE_ID_INTEL_ADLS 0x7ae1
++#define PCI_DEVICE_ID_INTEL_RPLS 0x7a61
++#define PCI_DEVICE_ID_INTEL_MTLP 0x7ec1
++#define PCI_DEVICE_ID_INTEL_MTL 0x7e7e
+ #define PCI_DEVICE_ID_INTEL_TGL 0x9a15
+ #define PCI_DEVICE_ID_AMD_MR 0x163a
+
+@@ -85,8 +88,8 @@ static const struct acpi_gpio_mapping acpi_dwc3_byt_gpios[] = {
+ static struct gpiod_lookup_table platform_bytcr_gpios = {
+ .dev_id = "0000:00:16.0",
+ .table = {
+- GPIO_LOOKUP("INT33FC:00", 54, "reset", GPIO_ACTIVE_HIGH),
+- GPIO_LOOKUP("INT33FC:02", 14, "cs", GPIO_ACTIVE_HIGH),
++ GPIO_LOOKUP("INT33FC:00", 54, "cs", GPIO_ACTIVE_HIGH),
++ GPIO_LOOKUP("INT33FC:02", 14, "reset", GPIO_ACTIVE_HIGH),
+ {}
+ },
+ };
+@@ -119,6 +122,13 @@ static const struct property_entry dwc3_pci_intel_properties[] = {
+ {}
+ };
+
++static const struct property_entry dwc3_pci_intel_byt_properties[] = {
++ PROPERTY_ENTRY_STRING("dr_mode", "peripheral"),
++ PROPERTY_ENTRY_BOOL("snps,dis_u2_susphy_quirk"),
++ PROPERTY_ENTRY_BOOL("linux,sysdev_is_parent"),
++ {}
++};
++
+ static const struct property_entry dwc3_pci_mrfld_properties[] = {
+ PROPERTY_ENTRY_STRING("dr_mode", "otg"),
+ PROPERTY_ENTRY_STRING("linux,extcon-name", "mrfld_bcove_pwrsrc"),
+@@ -161,6 +171,10 @@ static const struct software_node dwc3_pci_intel_swnode = {
+ .properties = dwc3_pci_intel_properties,
+ };
+
++static const struct software_node dwc3_pci_intel_byt_swnode = {
++ .properties = dwc3_pci_intel_byt_properties,
++};
++
+ static const struct software_node dwc3_pci_intel_mrfld_swnode = {
+ .properties = dwc3_pci_mrfld_properties,
+ };
+@@ -173,7 +187,8 @@ static const struct software_node dwc3_pci_amd_mr_swnode = {
+ .properties = dwc3_pci_mr_properties,
+ };
+
+-static int dwc3_pci_quirks(struct dwc3_pci *dwc)
++static int dwc3_pci_quirks(struct dwc3_pci *dwc,
++ const struct software_node *swnode)
+ {
+ struct pci_dev *pdev = dwc->pci;
+
+@@ -230,7 +245,7 @@ static int dwc3_pci_quirks(struct dwc3_pci *dwc)
+ }
+ }
+
+- return 0;
++ return device_add_software_node(&dwc->dwc3->dev, swnode);
+ }
+
+ #ifdef CONFIG_PM
+@@ -295,11 +310,7 @@ static int dwc3_pci_probe(struct pci_dev *pci, const struct pci_device_id *id)
+ dwc->dwc3->dev.parent = dev;
+ ACPI_COMPANION_SET(&dwc->dwc3->dev, ACPI_COMPANION(dev));
+
+- ret = device_add_software_node(&dwc->dwc3->dev, (void *)id->driver_data);
+- if (ret < 0)
+- goto err;
+-
+- ret = dwc3_pci_quirks(dwc);
++ ret = dwc3_pci_quirks(dwc, (void *)id->driver_data);
+ if (ret)
+ goto err;
+
+@@ -344,7 +355,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = {
+ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
+
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BYT),
+- (kernel_ulong_t) &dwc3_pci_intel_swnode, },
++ (kernel_ulong_t) &dwc3_pci_intel_byt_swnode, },
+
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MRFLD),
+ (kernel_ulong_t) &dwc3_pci_intel_mrfld_swnode, },
+@@ -409,6 +420,15 @@ static const struct pci_device_id dwc3_pci_id_table[] = {
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLS),
+ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
+
++ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPLS),
++ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
++
++ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTLP),
++ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
++
++ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL),
++ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
++
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL),
+ (kernel_ulong_t) &dwc3_pci_intel_swnode, },
+
+diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c
+index 9abbd01028c5f..b81a9e1c13153 100644
+--- a/drivers/usb/dwc3/dwc3-qcom.c
++++ b/drivers/usb/dwc3/dwc3-qcom.c
+@@ -649,7 +649,6 @@ static int dwc3_qcom_of_register_core(struct platform_device *pdev)
+ struct dwc3_qcom *qcom = platform_get_drvdata(pdev);
+ struct device_node *np = pdev->dev.of_node, *dwc3_np;
+ struct device *dev = &pdev->dev;
+- struct property *prop;
+ int ret;
+
+ dwc3_np = of_get_compatible_child(np, "snps,dwc3");
+@@ -658,20 +657,6 @@ static int dwc3_qcom_of_register_core(struct platform_device *pdev)
+ return -ENODEV;
+ }
+
+- prop = devm_kzalloc(dev, sizeof(*prop), GFP_KERNEL);
+- if (!prop) {
+- ret = -ENOMEM;
+- dev_err(dev, "unable to allocate memory for property\n");
+- goto node_put;
+- }
+-
+- prop->name = "tx-fifo-resize";
+- ret = of_add_property(dwc3_np, prop);
+- if (ret) {
+- dev_err(dev, "unable to add property\n");
+- goto node_put;
+- }
+-
+ ret = of_platform_populate(np, NULL, NULL, dev);
+ if (ret) {
+ dev_err(dev, "failed to register dwc3 core - %d\n", ret);
+@@ -784,9 +769,12 @@ static int dwc3_qcom_probe(struct platform_device *pdev)
+
+ if (qcom->acpi_pdata->is_urs) {
+ qcom->urs_usb = dwc3_qcom_create_urs_usb_platdev(dev);
+- if (!qcom->urs_usb) {
++ if (IS_ERR_OR_NULL(qcom->urs_usb)) {
+ dev_err(dev, "failed to create URS USB platdev\n");
+- return -ENODEV;
++ if (!qcom->urs_usb)
++ return -ENODEV;
++ else
++ return PTR_ERR(qcom->urs_usb);
+ }
+ }
+ }
+diff --git a/drivers/usb/dwc3/dwc3-xilinx.c b/drivers/usb/dwc3/dwc3-xilinx.c
+index 9cc3ad701a295..a6f3a9b38789e 100644
+--- a/drivers/usb/dwc3/dwc3-xilinx.c
++++ b/drivers/usb/dwc3/dwc3-xilinx.c
+@@ -99,17 +99,29 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
+ struct device *dev = priv_data->dev;
+ struct reset_control *crst, *hibrst, *apbrst;
+ struct phy *usb3_phy;
+- int ret;
++ int ret = 0;
+ u32 reg;
+
+- usb3_phy = devm_phy_get(dev, "usb3-phy");
+- if (PTR_ERR(usb3_phy) == -EPROBE_DEFER) {
+- ret = -EPROBE_DEFER;
++ usb3_phy = devm_phy_optional_get(dev, "usb3-phy");
++ if (IS_ERR(usb3_phy)) {
++ ret = PTR_ERR(usb3_phy);
++ dev_err_probe(dev, ret,
++ "failed to get USB3 PHY\n");
+ goto err;
+- } else if (IS_ERR(usb3_phy)) {
+- usb3_phy = NULL;
+ }
+
++ /*
++ * The following core resets are not required unless a USB3 PHY
++ * is used, and the subsequent register settings are not required
++ * unless a core reset is performed (they should be set properly
++ * by the first-stage boot loader, but may be reverted by a core
++ * reset). They may also break the configuration if USB3 is actually
++ * in use but the usb3-phy entry is missing from the device tree.
++ * Therefore, skip these operations in this case.
++ */
++ if (!usb3_phy)
++ goto skip_usb3_phy;
++
+ crst = devm_reset_control_get_exclusive(dev, "usb_crst");
+ if (IS_ERR(crst)) {
+ ret = PTR_ERR(crst);
+@@ -188,6 +200,7 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
+ goto err;
+ }
+
++skip_usb3_phy:
+ /*
+ * This routes the USB DMA traffic to go through FPD path instead
+ * of reaching DDR directly. This traffic routing is needed to
+diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
+index 4519d06c9ca2b..c32f3116d1a0f 100644
+--- a/drivers/usb/dwc3/gadget.c
++++ b/drivers/usb/dwc3/gadget.c
+@@ -310,13 +310,24 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
+ if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
+ int link_state;
+
++ /*
++ * Initiate remote wakeup if the link state is in U3 when
++ * operating in SS/SSP or L1/L2 when operating in HS/FS. If the
++ * link state is in U1/U2, no remote wakeup is needed. The Start
++ * Transfer command will initiate the link recovery.
++ */
+ link_state = dwc3_gadget_get_link_state(dwc);
+- if (link_state == DWC3_LINK_STATE_U1 ||
+- link_state == DWC3_LINK_STATE_U2 ||
+- link_state == DWC3_LINK_STATE_U3) {
++ switch (link_state) {
++ case DWC3_LINK_STATE_U2:
++ if (dwc->gadget->speed >= USB_SPEED_SUPER)
++ break;
++
++ fallthrough;
++ case DWC3_LINK_STATE_U3:
+ ret = __dwc3_gadget_wakeup(dwc);
+ dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
+ ret);
++ break;
+ }
+ }
+
+@@ -702,6 +713,7 @@ void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
+ DWC31_GTXFIFOSIZ_TXFRAMNUM;
+
+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
++ dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
+ }
+ dwc->num_ep_resized = 0;
+ }
+@@ -747,6 +759,10 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
+ if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
+ return 0;
+
++ /* bail if already resized */
++ if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
++ return 0;
++
+ ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
+
+ if ((dep->endpoint.maxburst > 1 &&
+@@ -807,6 +823,7 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
+ }
+
+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
++ dep->flags |= DWC3_EP_TXFIFO_RESIZED;
+ dwc->num_ep_resized++;
+
+ return 0;
+@@ -995,7 +1012,7 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
+
+ dep->stream_capable = false;
+ dep->type = 0;
+- dep->flags = 0;
++ dep->flags &= DWC3_EP_TXFIFO_RESIZED;
+
+ return 0;
+ }
+@@ -1254,6 +1271,19 @@ static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
+ if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
+ trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
+
++ /*
++ * As per data book 4.2.3.2TRB Control Bit Rules section
++ *
++ * The controller autonomously checks the HWO field of a TRB to determine if the
++ * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
++ * is valid before setting the HWO field to '1'. In most systems, this means that
++ * software must update the fourth DWORD of a TRB last.
++ *
++ * However there is a possibility of CPU re-ordering here which can cause
++ * controller to observe the HWO bit set prematurely.
++ * Add a write memory barrier to prevent CPU re-ordering.
++ */
++ wmb();
+ trb->ctrl |= DWC3_TRB_CTRL_HWO;
+
+ dwc3_ep_inc_enq(dep);
+@@ -3169,6 +3199,7 @@ static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
+ const struct dwc3_event_depevt *event,
+ struct dwc3_request *req, int status)
+ {
++ int request_status;
+ int ret;
+
+ if (req->request.num_mapped_sgs)
+@@ -3189,7 +3220,35 @@ static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
+ req->needs_extra_trb = false;
+ }
+
+- dwc3_gadget_giveback(dep, req, status);
++ /*
++ * The event status only reflects the status of the TRB with IOC set.
++ * For the requests that don't set interrupt on completion, the driver
++ * needs to check and return the status of the completed TRBs associated
++ * with the request. Use the status of the last TRB of the request.
++ */
++ if (req->request.no_interrupt) {
++ struct dwc3_trb *trb;
++
++ trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
++ switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
++ case DWC3_TRBSTS_MISSED_ISOC:
++ /* Isoc endpoint only */
++ request_status = -EXDEV;
++ break;
++ case DWC3_TRB_STS_XFER_IN_PROG:
++ /* Applicable when End Transfer with ForceRM=0 */
++ case DWC3_TRBSTS_SETUP_PENDING:
++ /* Control endpoint only */
++ case DWC3_TRBSTS_OK:
++ default:
++ request_status = 0;
++ break;
++ }
++ } else {
++ request_status = status;
++ }
++
++ dwc3_gadget_giveback(dep, req, request_status);
+
+ out:
+ return ret;
+@@ -3246,6 +3305,9 @@ static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
+ struct dwc3 *dwc = dep->dwc;
+ bool no_started_trb = true;
+
++ if (!dep->endpoint.desc)
++ return no_started_trb;
++
+ dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
+
+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
+@@ -3293,6 +3355,9 @@ static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
+ {
+ int status = 0;
+
++ if (!dep->endpoint.desc)
++ return;
++
+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
+ dwc3_gadget_endpoint_frame_from_event(dep, event);
+
+@@ -3346,6 +3411,14 @@ static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
+ if (cmd != DWC3_DEPCMD_ENDTRANSFER)
+ return;
+
++ /*
++ * The END_TRANSFER command will cause the controller to generate a
++ * NoStream Event, and it's not due to the host DP NoStream rejection.
++ * Ignore the next NoStream event.
++ */
++ if (dep->stream_capable)
++ dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
++
+ dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
+ dwc3_gadget_ep_cleanup_cancelled_requests(dep);
+@@ -3568,14 +3641,6 @@ static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
+ WARN_ON_ONCE(ret);
+ dep->resource_index = 0;
+
+- /*
+- * The END_TRANSFER command will cause the controller to generate a
+- * NoStream Event, and it's not due to the host DP NoStream rejection.
+- * Ignore the next NoStream event.
+- */
+- if (dep->stream_capable)
+- dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
+-
+ if (!interrupt)
+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
+ else
+@@ -4095,9 +4160,11 @@ static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
+ unsigned long flags;
+ irqreturn_t ret = IRQ_NONE;
+
++ local_bh_disable();
+ spin_lock_irqsave(&dwc->lock, flags);
+ ret = dwc3_process_event_buf(evt);
+ spin_unlock_irqrestore(&dwc->lock, flags);
++ local_bh_enable();
+
+ return ret;
+ }
+diff --git a/drivers/usb/early/xhci-dbc.c b/drivers/usb/early/xhci-dbc.c
+index be4ecbabdd586..6c0434100e38c 100644
+--- a/drivers/usb/early/xhci-dbc.c
++++ b/drivers/usb/early/xhci-dbc.c
+@@ -14,7 +14,6 @@
+ #include <linux/pci_ids.h>
+ #include <linux/memblock.h>
+ #include <linux/io.h>
+-#include <linux/iopoll.h>
+ #include <asm/pci-direct.h>
+ #include <asm/fixmap.h>
+ #include <linux/bcd.h>
+@@ -136,9 +135,17 @@ static int handshake(void __iomem *ptr, u32 mask, u32 done, int wait, int delay)
+ {
+ u32 result;
+
+- return readl_poll_timeout_atomic(ptr, result,
+- ((result & mask) == done),
+- delay, wait);
++ /* Can not use readl_poll_timeout_atomic() for early boot things */
++ do {
++ result = readl(ptr);
++ result &= mask;
++ if (result == done)
++ return 0;
++ udelay(delay);
++ wait -= delay;
++ } while (wait > 0);
++
++ return -ETIMEDOUT;
+ }
+
+ static void __init xdbc_bios_handoff(void)
+diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
+index 504c1cbc255d1..553382ce38378 100644
+--- a/drivers/usb/gadget/composite.c
++++ b/drivers/usb/gadget/composite.c
+@@ -1679,6 +1679,18 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
+ struct usb_function *f = NULL;
+ u8 endp;
+
++ if (w_length > USB_COMP_EP0_BUFSIZ) {
++ if (ctrl->bRequestType & USB_DIR_IN) {
++ /* Cast away the const, we are going to overwrite on purpose. */
++ __le16 *temp = (__le16 *)&ctrl->wLength;
++
++ *temp = cpu_to_le16(USB_COMP_EP0_BUFSIZ);
++ w_length = USB_COMP_EP0_BUFSIZ;
++ } else {
++ goto done;
++ }
++ }
++
+ /* partial re-init of the response message; the function or the
+ * gadget might need to intercept e.g. a control-OUT completion
+ * when we delegate to it.
+@@ -1963,6 +1975,9 @@ unknown:
+ if (w_index != 0x5 || (w_value >> 8))
+ break;
+ interface = w_value & 0xFF;
++ if (interface >= MAX_CONFIG_INTERFACES ||
++ !os_desc_cfg->interface[interface])
++ break;
+ buf[6] = w_index;
+ count = count_ext_prop(os_desc_cfg,
+ interface);
+@@ -2209,7 +2224,7 @@ int composite_dev_prepare(struct usb_composite_driver *composite,
+ if (!cdev->req)
+ return -ENOMEM;
+
+- cdev->req->buf = kmalloc(USB_COMP_EP0_BUFSIZ, GFP_KERNEL);
++ cdev->req->buf = kzalloc(USB_COMP_EP0_BUFSIZ, GFP_KERNEL);
+ if (!cdev->req->buf)
+ goto fail;
+
+diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
+index 477e72a1d11e7..5ade844db4046 100644
+--- a/drivers/usb/gadget/configfs.c
++++ b/drivers/usb/gadget/configfs.c
+@@ -1447,6 +1447,8 @@ static void configfs_composite_unbind(struct usb_gadget *gadget)
+ usb_ep_autoconfig_reset(cdev->gadget);
+ spin_lock_irqsave(&gi->spinlock, flags);
+ cdev->gadget = NULL;
++ cdev->deactivations = 0;
++ gadget->deactivated = false;
+ set_gadget_data(gadget, NULL);
+ spin_unlock_irqrestore(&gi->spinlock, flags);
+ }
+diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
+index 8260f38025b72..02f70c5c65fc2 100644
+--- a/drivers/usb/gadget/function/f_fs.c
++++ b/drivers/usb/gadget/function/f_fs.c
+@@ -614,7 +614,7 @@ static int ffs_ep0_open(struct inode *inode, struct file *file)
+ file->private_data = ffs;
+ ffs_data_opened(ffs);
+
+- return 0;
++ return stream_open(inode, file);
+ }
+
+ static int ffs_ep0_release(struct inode *inode, struct file *file)
+@@ -1154,7 +1154,7 @@ ffs_epfile_open(struct inode *inode, struct file *file)
+ file->private_data = epfile;
+ ffs_data_opened(epfile->ffs);
+
+- return 0;
++ return stream_open(inode, file);
+ }
+
+ static int ffs_aio_cancel(struct kiocb *kiocb)
+@@ -1711,16 +1711,24 @@ static void ffs_data_put(struct ffs_data *ffs)
+
+ static void ffs_data_closed(struct ffs_data *ffs)
+ {
++ struct ffs_epfile *epfiles;
++ unsigned long flags;
++
+ ENTER();
+
+ if (atomic_dec_and_test(&ffs->opened)) {
+ if (ffs->no_disconnect) {
+ ffs->state = FFS_DEACTIVATED;
+- if (ffs->epfiles) {
+- ffs_epfiles_destroy(ffs->epfiles,
+- ffs->eps_count);
+- ffs->epfiles = NULL;
+- }
++ spin_lock_irqsave(&ffs->eps_lock, flags);
++ epfiles = ffs->epfiles;
++ ffs->epfiles = NULL;
++ spin_unlock_irqrestore(&ffs->eps_lock,
++ flags);
++
++ if (epfiles)
++ ffs_epfiles_destroy(epfiles,
++ ffs->eps_count);
++
+ if (ffs->setup_state == FFS_SETUP_PENDING)
+ __ffs_ep0_stall(ffs);
+ } else {
+@@ -1767,17 +1775,34 @@ static struct ffs_data *ffs_data_new(const char *dev_name)
+
+ static void ffs_data_clear(struct ffs_data *ffs)
+ {
++ struct ffs_epfile *epfiles;
++ unsigned long flags;
++
+ ENTER();
+
+ ffs_closed(ffs);
+
+ BUG_ON(ffs->gadget);
+
+- if (ffs->epfiles)
+- ffs_epfiles_destroy(ffs->epfiles, ffs->eps_count);
++ spin_lock_irqsave(&ffs->eps_lock, flags);
++ epfiles = ffs->epfiles;
++ ffs->epfiles = NULL;
++ spin_unlock_irqrestore(&ffs->eps_lock, flags);
+
+- if (ffs->ffs_eventfd)
++ /*
++ * potential race possible between ffs_func_eps_disable
++ * & ffs_epfile_release therefore maintaining a local
++ * copy of epfile will save us from use-after-free.
++ */
++ if (epfiles) {
++ ffs_epfiles_destroy(epfiles, ffs->eps_count);
++ ffs->epfiles = NULL;
++ }
++
++ if (ffs->ffs_eventfd) {
+ eventfd_ctx_put(ffs->ffs_eventfd);
++ ffs->ffs_eventfd = NULL;
++ }
+
+ kfree(ffs->raw_descs_data);
+ kfree(ffs->raw_strings);
+@@ -1790,7 +1815,6 @@ static void ffs_data_reset(struct ffs_data *ffs)
+
+ ffs_data_clear(ffs);
+
+- ffs->epfiles = NULL;
+ ffs->raw_descs_data = NULL;
+ ffs->raw_descs = NULL;
+ ffs->raw_strings = NULL;
+@@ -1919,12 +1943,15 @@ static void ffs_epfiles_destroy(struct ffs_epfile *epfiles, unsigned count)
+
+ static void ffs_func_eps_disable(struct ffs_function *func)
+ {
+- struct ffs_ep *ep = func->eps;
+- struct ffs_epfile *epfile = func->ffs->epfiles;
+- unsigned count = func->ffs->eps_count;
++ struct ffs_ep *ep;
++ struct ffs_epfile *epfile;
++ unsigned short count;
+ unsigned long flags;
+
+ spin_lock_irqsave(&func->ffs->eps_lock, flags);
++ count = func->ffs->eps_count;
++ epfile = func->ffs->epfiles;
++ ep = func->eps;
+ while (count--) {
+ /* pending requests get nuked */
+ if (ep->ep)
+@@ -1942,14 +1969,18 @@ static void ffs_func_eps_disable(struct ffs_function *func)
+
+ static int ffs_func_eps_enable(struct ffs_function *func)
+ {
+- struct ffs_data *ffs = func->ffs;
+- struct ffs_ep *ep = func->eps;
+- struct ffs_epfile *epfile = ffs->epfiles;
+- unsigned count = ffs->eps_count;
++ struct ffs_data *ffs;
++ struct ffs_ep *ep;
++ struct ffs_epfile *epfile;
++ unsigned short count;
+ unsigned long flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&func->ffs->eps_lock, flags);
++ ffs = func->ffs;
++ ep = func->eps;
++ epfile = ffs->epfiles;
++ count = ffs->eps_count;
+ while(count--) {
+ ep->ep->driver_data = ep;
+
+diff --git a/drivers/usb/gadget/function/f_sourcesink.c b/drivers/usb/gadget/function/f_sourcesink.c
+index 1abf08e5164af..6803cd60cc6dc 100644
+--- a/drivers/usb/gadget/function/f_sourcesink.c
++++ b/drivers/usb/gadget/function/f_sourcesink.c
+@@ -584,6 +584,7 @@ static int source_sink_start_ep(struct f_sourcesink *ss, bool is_in,
+
+ if (is_iso) {
+ switch (speed) {
++ case USB_SPEED_SUPER_PLUS:
+ case USB_SPEED_SUPER:
+ size = ss->isoc_maxpacket *
+ (ss->isoc_mult + 1) *
+diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c
+index ef55b8bb5870a..5226a47b68fda 100644
+--- a/drivers/usb/gadget/function/f_uac2.c
++++ b/drivers/usb/gadget/function/f_uac2.c
+@@ -202,7 +202,7 @@ static struct uac2_input_terminal_descriptor io_in_it_desc = {
+
+ .bDescriptorSubtype = UAC_INPUT_TERMINAL,
+ /* .bTerminalID = DYNAMIC */
+- .wTerminalType = cpu_to_le16(UAC_INPUT_TERMINAL_UNDEFINED),
++ .wTerminalType = cpu_to_le16(UAC_INPUT_TERMINAL_MICROPHONE),
+ .bAssocTerminal = 0,
+ /* .bCSourceID = DYNAMIC */
+ .iChannelNames = 0,
+@@ -230,7 +230,7 @@ static struct uac2_output_terminal_descriptor io_out_ot_desc = {
+
+ .bDescriptorSubtype = UAC_OUTPUT_TERMINAL,
+ /* .bTerminalID = DYNAMIC */
+- .wTerminalType = cpu_to_le16(UAC_OUTPUT_TERMINAL_UNDEFINED),
++ .wTerminalType = cpu_to_le16(UAC_OUTPUT_TERMINAL_SPEAKER),
+ .bAssocTerminal = 0,
+ /* .bSourceID = DYNAMIC */
+ /* .bCSourceID = DYNAMIC */
+diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c
+index 9d87c0fb8f92e..bf0a3fc2d7767 100644
+--- a/drivers/usb/gadget/function/f_uvc.c
++++ b/drivers/usb/gadget/function/f_uvc.c
+@@ -884,17 +884,42 @@ static void uvc_free(struct usb_function *f)
+ kfree(uvc);
+ }
+
+-static void uvc_unbind(struct usb_configuration *c, struct usb_function *f)
++static void uvc_function_unbind(struct usb_configuration *c,
++ struct usb_function *f)
+ {
+ struct usb_composite_dev *cdev = c->cdev;
+ struct uvc_device *uvc = to_uvc(f);
++ long wait_ret = 1;
+
+- uvcg_info(f, "%s\n", __func__);
++ uvcg_info(f, "%s()\n", __func__);
++
++ /* If we know we're connected via v4l2, then there should be a cleanup
++ * of the device from userspace either via UVC_EVENT_DISCONNECT or
++ * though the video device removal uevent. Allow some time for the
++ * application to close out before things get deleted.
++ */
++ if (uvc->func_connected) {
++ uvcg_dbg(f, "waiting for clean disconnect\n");
++ wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue,
++ uvc->func_connected == false, msecs_to_jiffies(500));
++ uvcg_dbg(f, "done waiting with ret: %ld\n", wait_ret);
++ }
+
+ device_remove_file(&uvc->vdev.dev, &dev_attr_function_name);
+ video_unregister_device(&uvc->vdev);
+ v4l2_device_unregister(&uvc->v4l2_dev);
+
++ if (uvc->func_connected) {
++ /* Wait for the release to occur to ensure there are no longer any
++ * pending operations that may cause panics when resources are cleaned
++ * up.
++ */
++ uvcg_warn(f, "%s no clean disconnect, wait for release\n", __func__);
++ wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue,
++ uvc->func_connected == false, msecs_to_jiffies(1000));
++ uvcg_dbg(f, "done waiting for release with ret: %ld\n", wait_ret);
++ }
++
+ usb_ep_free_request(cdev->gadget->ep0, uvc->control_req);
+ kfree(uvc->control_buf);
+
+@@ -913,6 +938,7 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi)
+
+ mutex_init(&uvc->video.mutex);
+ uvc->state = UVC_STATE_DISCONNECTED;
++ init_waitqueue_head(&uvc->func_connected_queue);
+ opts = fi_to_f_uvc_opts(fi);
+
+ mutex_lock(&opts->lock);
+@@ -943,7 +969,7 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi)
+ /* Register the function. */
+ uvc->func.name = "uvc";
+ uvc->func.bind = uvc_function_bind;
+- uvc->func.unbind = uvc_unbind;
++ uvc->func.unbind = uvc_function_unbind;
+ uvc->func.get_alt = uvc_function_get_alt;
+ uvc->func.set_alt = uvc_function_set_alt;
+ uvc->func.disable = uvc_function_disable;
+diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c
+index 64de9f1b874c5..4150de96b937a 100644
+--- a/drivers/usb/gadget/function/rndis.c
++++ b/drivers/usb/gadget/function/rndis.c
+@@ -637,14 +637,18 @@ static int rndis_set_response(struct rndis_params *params,
+ rndis_set_cmplt_type *resp;
+ rndis_resp_t *r;
+
++ BufLength = le32_to_cpu(buf->InformationBufferLength);
++ BufOffset = le32_to_cpu(buf->InformationBufferOffset);
++ if ((BufLength > RNDIS_MAX_TOTAL_SIZE) ||
++ (BufOffset > RNDIS_MAX_TOTAL_SIZE) ||
++ (BufOffset + 8 >= RNDIS_MAX_TOTAL_SIZE))
++ return -EINVAL;
++
+ r = rndis_add_response(params, sizeof(rndis_set_cmplt_type));
+ if (!r)
+ return -ENOMEM;
+ resp = (rndis_set_cmplt_type *)r->buf;
+
+- BufLength = le32_to_cpu(buf->InformationBufferLength);
+- BufOffset = le32_to_cpu(buf->InformationBufferOffset);
+-
+ #ifdef VERBOSE_DEBUG
+ pr_debug("%s: Length: %d\n", __func__, BufLength);
+ pr_debug("%s: Offset: %d\n", __func__, BufOffset);
+@@ -919,6 +923,7 @@ struct rndis_params *rndis_register(void (*resp_avail)(void *v), void *v)
+ params->resp_avail = resp_avail;
+ params->v = v;
+ INIT_LIST_HEAD(&params->resp_queue);
++ spin_lock_init(&params->resp_lock);
+ pr_debug("%s: configNr = %d\n", __func__, i);
+
+ return params;
+@@ -1012,12 +1017,14 @@ void rndis_free_response(struct rndis_params *params, u8 *buf)
+ {
+ rndis_resp_t *r, *n;
+
++ spin_lock(&params->resp_lock);
+ list_for_each_entry_safe(r, n, &params->resp_queue, list) {
+ if (r->buf == buf) {
+ list_del(&r->list);
+ kfree(r);
+ }
+ }
++ spin_unlock(&params->resp_lock);
+ }
+ EXPORT_SYMBOL_GPL(rndis_free_response);
+
+@@ -1027,14 +1034,17 @@ u8 *rndis_get_next_response(struct rndis_params *params, u32 *length)
+
+ if (!length) return NULL;
+
++ spin_lock(&params->resp_lock);
+ list_for_each_entry_safe(r, n, &params->resp_queue, list) {
+ if (!r->send) {
+ r->send = 1;
+ *length = r->length;
++ spin_unlock(&params->resp_lock);
+ return r->buf;
+ }
+ }
+
++ spin_unlock(&params->resp_lock);
+ return NULL;
+ }
+ EXPORT_SYMBOL_GPL(rndis_get_next_response);
+@@ -1051,7 +1061,9 @@ static rndis_resp_t *rndis_add_response(struct rndis_params *params, u32 length)
+ r->length = length;
+ r->send = 0;
+
++ spin_lock(&params->resp_lock);
+ list_add_tail(&r->list, &params->resp_queue);
++ spin_unlock(&params->resp_lock);
+ return r;
+ }
+
+diff --git a/drivers/usb/gadget/function/rndis.h b/drivers/usb/gadget/function/rndis.h
+index f6167f7fea82b..6206b8b7490f6 100644
+--- a/drivers/usb/gadget/function/rndis.h
++++ b/drivers/usb/gadget/function/rndis.h
+@@ -174,6 +174,7 @@ typedef struct rndis_params {
+ void (*resp_avail)(void *v);
+ void *v;
+ struct list_head resp_queue;
++ spinlock_t resp_lock;
+ } rndis_params;
+
+ /* RNDIS Message parser and other useless functions */
+diff --git a/drivers/usb/gadget/function/u_audio.c b/drivers/usb/gadget/function/u_audio.c
+index ad16163b5ff80..d22ac23c94b0f 100644
+--- a/drivers/usb/gadget/function/u_audio.c
++++ b/drivers/usb/gadget/function/u_audio.c
+@@ -1097,7 +1097,7 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name,
+ }
+
+ kctl->id.device = pcm->device;
+- kctl->id.subdevice = i;
++ kctl->id.subdevice = 0;
+
+ err = snd_ctl_add(card, kctl);
+ if (err < 0)
+@@ -1120,7 +1120,7 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name,
+ }
+
+ kctl->id.device = pcm->device;
+- kctl->id.subdevice = i;
++ kctl->id.subdevice = 0;
+
+
+ kctl->tlv.c = u_audio_volume_tlv;
+diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
+index 85a3f6d4b5af3..d15a54f6c24b9 100644
+--- a/drivers/usb/gadget/function/u_ether.c
++++ b/drivers/usb/gadget/function/u_ether.c
+@@ -17,6 +17,7 @@
+ #include <linux/etherdevice.h>
+ #include <linux/ethtool.h>
+ #include <linux/if_vlan.h>
++#include <linux/etherdevice.h>
+
+ #include "u_ether.h"
+
+@@ -861,19 +862,23 @@ int gether_register_netdev(struct net_device *net)
+ {
+ struct eth_dev *dev;
+ struct usb_gadget *g;
+- struct sockaddr sa;
+ int status;
+
+ if (!net->dev.parent)
+ return -EINVAL;
+ dev = netdev_priv(net);
+ g = dev->gadget;
++
++ net->addr_assign_type = NET_ADDR_RANDOM;
++ eth_hw_addr_set(net, dev->dev_mac);
++
+ status = register_netdev(net);
+ if (status < 0) {
+ dev_dbg(&g->dev, "register_netdev failed, %d\n", status);
+ return status;
+ } else {
+ INFO(dev, "HOST MAC %pM\n", dev->host_mac);
++ INFO(dev, "MAC %pM\n", dev->dev_mac);
+
+ /* two kinds of host-initiated state changes:
+ * - iff DATA transfer is active, carrier is "on"
+@@ -881,15 +886,6 @@ int gether_register_netdev(struct net_device *net)
+ */
+ netif_carrier_off(net);
+ }
+- sa.sa_family = net->type;
+- memcpy(sa.sa_data, dev->dev_mac, ETH_ALEN);
+- rtnl_lock();
+- status = dev_set_mac_address(net, &sa, NULL);
+- rtnl_unlock();
+- if (status)
+- pr_warn("cannot set self ethernet address: %d\n", status);
+- else
+- INFO(dev, "MAC %pM\n", dev->dev_mac);
+
+ return status;
+ }
+diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h
+index 255a61bd6a6a8..0966c5aa24921 100644
+--- a/drivers/usb/gadget/function/uvc.h
++++ b/drivers/usb/gadget/function/uvc.h
+@@ -14,6 +14,7 @@
+ #include <linux/spinlock.h>
+ #include <linux/usb/composite.h>
+ #include <linux/videodev2.h>
++#include <linux/wait.h>
+
+ #include <media/v4l2-device.h>
+ #include <media/v4l2-dev.h>
+@@ -126,6 +127,8 @@ struct uvc_device {
+ enum uvc_state state;
+ struct usb_function func;
+ struct uvc_video video;
++ bool func_connected;
++ wait_queue_head_t func_connected_queue;
+
+ /* Descriptors */
+ struct {
+@@ -156,6 +159,7 @@ static inline struct uvc_device *to_uvc(struct usb_function *f)
+ struct uvc_file_handle {
+ struct v4l2_fh vfh;
+ struct uvc_video *device;
++ bool is_uvc_app_handle;
+ };
+
+ #define to_uvc_file_handle(handle) \
+diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c
+index 7d00ad7c154c2..99dc9adf56efa 100644
+--- a/drivers/usb/gadget/function/uvc_queue.c
++++ b/drivers/usb/gadget/function/uvc_queue.c
+@@ -264,6 +264,8 @@ void uvcg_queue_cancel(struct uvc_video_queue *queue, int disconnect)
+ buf->state = UVC_BUF_STATE_ERROR;
+ vb2_buffer_done(&buf->buf.vb2_buf, VB2_BUF_STATE_ERROR);
+ }
++ queue->buf_used = 0;
++
+ /* This must be protected by the irqlock spinlock to avoid race
+ * conditions between uvc_queue_buffer and the disconnection event that
+ * could result in an interruptible wait in uvc_dequeue_buffer. Do not
+diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c
+index 4ca89eab61590..65abd55ce2348 100644
+--- a/drivers/usb/gadget/function/uvc_v4l2.c
++++ b/drivers/usb/gadget/function/uvc_v4l2.c
+@@ -227,17 +227,56 @@ static int
+ uvc_v4l2_subscribe_event(struct v4l2_fh *fh,
+ const struct v4l2_event_subscription *sub)
+ {
++ struct uvc_device *uvc = video_get_drvdata(fh->vdev);
++ struct uvc_file_handle *handle = to_uvc_file_handle(fh);
++ int ret;
++
+ if (sub->type < UVC_EVENT_FIRST || sub->type > UVC_EVENT_LAST)
+ return -EINVAL;
+
+- return v4l2_event_subscribe(fh, sub, 2, NULL);
++ if (sub->type == UVC_EVENT_SETUP && uvc->func_connected)
++ return -EBUSY;
++
++ ret = v4l2_event_subscribe(fh, sub, 2, NULL);
++ if (ret < 0)
++ return ret;
++
++ if (sub->type == UVC_EVENT_SETUP) {
++ uvc->func_connected = true;
++ handle->is_uvc_app_handle = true;
++ uvc_function_connect(uvc);
++ }
++
++ return 0;
++}
++
++static void uvc_v4l2_disable(struct uvc_device *uvc)
++{
++ uvc_function_disconnect(uvc);
++ uvcg_video_enable(&uvc->video, 0);
++ uvcg_free_buffers(&uvc->video.queue);
++ uvc->func_connected = false;
++ wake_up_interruptible(&uvc->func_connected_queue);
+ }
+
+ static int
+ uvc_v4l2_unsubscribe_event(struct v4l2_fh *fh,
+ const struct v4l2_event_subscription *sub)
+ {
+- return v4l2_event_unsubscribe(fh, sub);
++ struct uvc_device *uvc = video_get_drvdata(fh->vdev);
++ struct uvc_file_handle *handle = to_uvc_file_handle(fh);
++ int ret;
++
++ ret = v4l2_event_unsubscribe(fh, sub);
++ if (ret < 0)
++ return ret;
++
++ if (sub->type == UVC_EVENT_SETUP && handle->is_uvc_app_handle) {
++ uvc_v4l2_disable(uvc);
++ handle->is_uvc_app_handle = false;
++ }
++
++ return 0;
+ }
+
+ static long
+@@ -292,7 +331,6 @@ uvc_v4l2_open(struct file *file)
+ handle->device = &uvc->video;
+ file->private_data = &handle->vfh;
+
+- uvc_function_connect(uvc);
+ return 0;
+ }
+
+@@ -304,11 +342,9 @@ uvc_v4l2_release(struct file *file)
+ struct uvc_file_handle *handle = to_uvc_file_handle(file->private_data);
+ struct uvc_video *video = handle->device;
+
+- uvc_function_disconnect(uvc);
+-
+ mutex_lock(&video->mutex);
+- uvcg_video_enable(video, 0);
+- uvcg_free_buffers(&video->queue);
++ if (handle->is_uvc_app_handle)
++ uvc_v4l2_disable(uvc);
+ mutex_unlock(&video->mutex);
+
+ file->private_data = NULL;
+diff --git a/drivers/usb/gadget/legacy/dbgp.c b/drivers/usb/gadget/legacy/dbgp.c
+index e1d566c9918ae..6bcbad3825802 100644
+--- a/drivers/usb/gadget/legacy/dbgp.c
++++ b/drivers/usb/gadget/legacy/dbgp.c
+@@ -137,7 +137,7 @@ static int dbgp_enable_ep_req(struct usb_ep *ep)
+ goto fail_1;
+ }
+
+- req->buf = kmalloc(DBGP_REQ_LEN, GFP_KERNEL);
++ req->buf = kzalloc(DBGP_REQ_LEN, GFP_KERNEL);
+ if (!req->buf) {
+ err = -ENOMEM;
+ stp = 2;
+@@ -345,6 +345,19 @@ static int dbgp_setup(struct usb_gadget *gadget,
+ void *data = NULL;
+ u16 len = 0;
+
++ if (length > DBGP_REQ_LEN) {
++ if (ctrl->bRequestType & USB_DIR_IN) {
++ /* Cast away the const, we are going to overwrite on purpose. */
++ __le16 *temp = (__le16 *)&ctrl->wLength;
++
++ *temp = cpu_to_le16(DBGP_REQ_LEN);
++ length = DBGP_REQ_LEN;
++ } else {
++ return err;
++ }
++ }
++
++
+ if (request == USB_REQ_GET_DESCRIPTOR) {
+ switch (value>>8) {
+ case USB_DT_DEVICE:
+diff --git a/drivers/usb/gadget/legacy/hid.c b/drivers/usb/gadget/legacy/hid.c
+index 5b27d289443fe..3912cc805f3af 100644
+--- a/drivers/usb/gadget/legacy/hid.c
++++ b/drivers/usb/gadget/legacy/hid.c
+@@ -99,8 +99,10 @@ static int do_config(struct usb_configuration *c)
+
+ list_for_each_entry(e, &hidg_func_list, node) {
+ e->f = usb_get_function(e->fi);
+- if (IS_ERR(e->f))
++ if (IS_ERR(e->f)) {
++ status = PTR_ERR(e->f);
+ goto put;
++ }
+ status = usb_add_function(c, e->f);
+ if (status < 0) {
+ usb_put_function(e->f);
+diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
+index 539220d7f5b62..3279b4767424c 100644
+--- a/drivers/usb/gadget/legacy/inode.c
++++ b/drivers/usb/gadget/legacy/inode.c
+@@ -110,6 +110,8 @@ enum ep0_state {
+ /* enough for the whole queue: most events invalidate others */
+ #define N_EVENT 5
+
++#define RBUF_SIZE 256
++
+ struct dev_data {
+ spinlock_t lock;
+ refcount_t count;
+@@ -144,7 +146,7 @@ struct dev_data {
+ struct dentry *dentry;
+
+ /* except this scratch i/o buffer for ep0 */
+- u8 rbuf [256];
++ u8 rbuf[RBUF_SIZE];
+ };
+
+ static inline void get_dev (struct dev_data *data)
+@@ -1334,6 +1336,18 @@ gadgetfs_setup (struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
+ u16 w_value = le16_to_cpu(ctrl->wValue);
+ u16 w_length = le16_to_cpu(ctrl->wLength);
+
++ if (w_length > RBUF_SIZE) {
++ if (ctrl->bRequestType & USB_DIR_IN) {
++ /* Cast away the const, we are going to overwrite on purpose. */
++ __le16 *temp = (__le16 *)&ctrl->wLength;
++
++ *temp = cpu_to_le16(RBUF_SIZE);
++ w_length = RBUF_SIZE;
++ } else {
++ return value;
++ }
++ }
++
+ spin_lock (&dev->lock);
+ dev->setup_abort = 0;
+ if (dev->state == STATE_DEV_UNCONNECTED) {
+@@ -1815,8 +1829,9 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr)
+ spin_lock_irq (&dev->lock);
+ value = -EINVAL;
+ if (dev->buf) {
++ spin_unlock_irq(&dev->lock);
+ kfree(kbuf);
+- goto fail;
++ return value;
+ }
+ dev->buf = kbuf;
+
+@@ -1863,8 +1878,8 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr)
+
+ value = usb_gadget_probe_driver(&gadgetfs_driver);
+ if (value != 0) {
+- kfree (dev->buf);
+- dev->buf = NULL;
++ spin_lock_irq(&dev->lock);
++ goto fail;
+ } else {
+ /* at this point "good" hardware has for the first time
+ * let the USB the host see us. alternatively, if users
+@@ -1881,6 +1896,9 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr)
+ return value;
+
+ fail:
++ dev->config = NULL;
++ dev->hs_config = NULL;
++ dev->dev = NULL;
+ spin_unlock_irq (&dev->lock);
+ pr_debug ("%s: %s fail %zd, %p\n", shortname, __func__, value, dev);
+ kfree (dev->buf);
+diff --git a/drivers/usb/gadget/legacy/raw_gadget.c b/drivers/usb/gadget/legacy/raw_gadget.c
+index c5a2c734234a5..3427ce37a5c5b 100644
+--- a/drivers/usb/gadget/legacy/raw_gadget.c
++++ b/drivers/usb/gadget/legacy/raw_gadget.c
+@@ -145,6 +145,7 @@ enum dev_state {
+ STATE_DEV_INVALID = 0,
+ STATE_DEV_OPENED,
+ STATE_DEV_INITIALIZED,
++ STATE_DEV_REGISTERING,
+ STATE_DEV_RUNNING,
+ STATE_DEV_CLOSED,
+ STATE_DEV_FAILED
+@@ -508,6 +509,7 @@ static int raw_ioctl_run(struct raw_dev *dev, unsigned long value)
+ ret = -EINVAL;
+ goto out_unlock;
+ }
++ dev->state = STATE_DEV_REGISTERING;
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ ret = usb_gadget_probe_driver(&dev->driver);
+@@ -1004,7 +1006,7 @@ static int raw_process_ep_io(struct raw_dev *dev, struct usb_raw_ep_io *io,
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+- if ((in && !ep->ep->caps.dir_in) || (!in && ep->ep->caps.dir_in)) {
++ if (in != usb_endpoint_dir_in(ep->ep->desc)) {
+ dev_dbg(&dev->gadget->dev, "fail, wrong direction\n");
+ ret = -EINVAL;
+ goto out_unlock;
+diff --git a/drivers/usb/gadget/udc/Kconfig b/drivers/usb/gadget/udc/Kconfig
+index 8c614bb86c665..69394dc1cdfb6 100644
+--- a/drivers/usb/gadget/udc/Kconfig
++++ b/drivers/usb/gadget/udc/Kconfig
+@@ -330,6 +330,7 @@ config USB_AMD5536UDC
+ config USB_FSL_QE
+ tristate "Freescale QE/CPM USB Device Controller"
+ depends on FSL_SOC && (QUICC_ENGINE || CPM)
++ depends on !64BIT || BROKEN
+ help
+ Some of Freescale PowerPC processors have a Full Speed
+ QE/CPM2 USB controller, which support device mode with 4
+diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
+index 14fdf918ecfeb..61099f2d057dc 100644
+--- a/drivers/usb/gadget/udc/core.c
++++ b/drivers/usb/gadget/udc/core.c
+@@ -1434,7 +1434,6 @@ static void usb_gadget_remove_driver(struct usb_udc *udc)
+ usb_gadget_udc_stop(udc);
+
+ udc->driver = NULL;
+- udc->dev.driver = NULL;
+ udc->gadget->dev.driver = NULL;
+ }
+
+@@ -1496,7 +1495,6 @@ static int udc_bind_to_driver(struct usb_udc *udc, struct usb_gadget_driver *dri
+ driver->function);
+
+ udc->driver = driver;
+- udc->dev.driver = &driver->driver;
+ udc->gadget->dev.driver = &driver->driver;
+
+ usb_gadget_udc_set_speed(udc, driver->max_speed);
+@@ -1519,7 +1517,6 @@ err1:
+ dev_err(&udc->dev, "failed to start %s: %d\n",
+ udc->driver->function, ret);
+ udc->driver = NULL;
+- udc->dev.driver = NULL;
+ udc->gadget->dev.driver = NULL;
+ return ret;
+ }
+diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c
+index 57d417a7c3e0a..601829a6b4bad 100644
+--- a/drivers/usb/gadget/udc/renesas_usb3.c
++++ b/drivers/usb/gadget/udc/renesas_usb3.c
+@@ -2378,6 +2378,8 @@ static void handle_ext_role_switch_states(struct device *dev,
+ switch (role) {
+ case USB_ROLE_NONE:
+ usb3->connection_state = USB_ROLE_NONE;
++ if (cur_role == USB_ROLE_HOST)
++ device_release_driver(host);
+ if (usb3->driver)
+ usb3_disconnect(usb3);
+ usb3_vbus_out(usb3, false);
+diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c
+index 43f1b0d461c1e..be76f891b9c52 100644
+--- a/drivers/usb/gadget/udc/tegra-xudc.c
++++ b/drivers/usb/gadget/udc/tegra-xudc.c
+@@ -32,9 +32,6 @@
+ #include <linux/workqueue.h>
+
+ /* XUSB_DEV registers */
+-#define SPARAM 0x000
+-#define SPARAM_ERSTMAX_MASK GENMASK(20, 16)
+-#define SPARAM_ERSTMAX(x) (((x) << 16) & SPARAM_ERSTMAX_MASK)
+ #define DB 0x004
+ #define DB_TARGET_MASK GENMASK(15, 8)
+ #define DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
+@@ -275,8 +272,10 @@ BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
+ BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
+ BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
+ BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
+-BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 24, 0xff)
++BUILD_EP_CONTEXT_RW(rsvd, rsvd[0], 24, 0x1)
+ BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
++BUILD_EP_CONTEXT_RW(splitxstate, rsvd[0], 26, 0x1)
++BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 27, 0x1f)
+ BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
+ BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
+ BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
+@@ -1557,6 +1556,9 @@ static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
+ ep_reload(xudc, ep->index);
+
+ ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
++ ep_ctx_write_rsvd(ep->context, 0);
++ ep_ctx_write_partial_td(ep->context, 0);
++ ep_ctx_write_splitxstate(ep->context, 0);
+ ep_ctx_write_seq_num(ep->context, 0);
+
+ ep_reload(xudc, ep->index);
+@@ -2812,7 +2814,10 @@ static void tegra_xudc_reset(struct tegra_xudc *xudc)
+ xudc->setup_seq_num = 0;
+ xudc->queued_setup_packet = false;
+
+- ep_ctx_write_seq_num(ep0->context, xudc->setup_seq_num);
++ ep_ctx_write_rsvd(ep0->context, 0);
++ ep_ctx_write_partial_td(ep0->context, 0);
++ ep_ctx_write_splitxstate(ep0->context, 0);
++ ep_ctx_write_seq_num(ep0->context, 0);
+
+ deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
+
+@@ -3295,11 +3300,6 @@ static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
+ unsigned int i;
+ u32 val;
+
+- val = xudc_readl(xudc, SPARAM);
+- val &= ~(SPARAM_ERSTMAX_MASK);
+- val |= SPARAM_ERSTMAX(XUDC_NR_EVENT_RINGS);
+- xudc_writel(xudc, val, SPARAM);
+-
+ for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
+ memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
+ sizeof(*xudc->event_ring[i]));
+diff --git a/drivers/usb/gadget/udc/udc-xilinx.c b/drivers/usb/gadget/udc/udc-xilinx.c
+index fb4ffedd6f0dd..9cf43731bcd18 100644
+--- a/drivers/usb/gadget/udc/udc-xilinx.c
++++ b/drivers/usb/gadget/udc/udc-xilinx.c
+@@ -1612,6 +1612,8 @@ static void xudc_getstatus(struct xusb_udc *udc)
+ break;
+ case USB_RECIP_ENDPOINT:
+ epnum = udc->setup.wIndex & USB_ENDPOINT_NUMBER_MASK;
++ if (epnum >= XUSB_MAX_ENDPOINTS)
++ goto stall;
+ target_ep = &udc->ep[epnum];
+ epcfgreg = udc->read_fn(udc->addr + target_ep->offset);
+ halt = epcfgreg & XUSB_EP_CFG_STALL_MASK;
+@@ -1679,6 +1681,10 @@ static void xudc_set_clear_feature(struct xusb_udc *udc)
+ case USB_RECIP_ENDPOINT:
+ if (!udc->setup.wValue) {
+ endpoint = udc->setup.wIndex & USB_ENDPOINT_NUMBER_MASK;
++ if (endpoint >= XUSB_MAX_ENDPOINTS) {
++ xudc_ep0_stall(udc);
++ return;
++ }
+ target_ep = &udc->ep[endpoint];
+ outinbit = udc->setup.wIndex & USB_ENDPOINT_DIR_MASK;
+ outinbit = outinbit >> 7;
+diff --git a/drivers/usb/host/ehci-brcm.c b/drivers/usb/host/ehci-brcm.c
+index d3626bfa966b4..6a0f64c9e5e88 100644
+--- a/drivers/usb/host/ehci-brcm.c
++++ b/drivers/usb/host/ehci-brcm.c
+@@ -62,8 +62,12 @@ static int ehci_brcm_hub_control(
+ u32 __iomem *status_reg;
+ unsigned long flags;
+ int retval, irq_disabled = 0;
++ u32 temp;
+
+- status_reg = &ehci->regs->port_status[(wIndex & 0xff) - 1];
++ temp = (wIndex & 0xff) - 1;
++ if (temp >= HCS_N_PORTS_MAX) /* Avoid index-out-of-bounds warning */
++ temp = 0;
++ status_reg = &ehci->regs->port_status[temp];
+
+ /*
+ * RESUME is cleared when GetPortStatus() is called 20ms after start
+diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c
+index 1776c05d0a486..1440803216297 100644
+--- a/drivers/usb/host/ehci-hcd.c
++++ b/drivers/usb/host/ehci-hcd.c
+@@ -635,7 +635,16 @@ static int ehci_run (struct usb_hcd *hcd)
+ /* Wait until HC become operational */
+ ehci_readl(ehci, &ehci->regs->command); /* unblock posted writes */
+ msleep(5);
+- rc = ehci_handshake(ehci, &ehci->regs->status, STS_HALT, 0, 100 * 1000);
++
++ /* For Aspeed, STS_HALT also depends on ASS/PSS status.
++ * Check CMD_RUN instead.
++ */
++ if (ehci->is_aspeed)
++ rc = ehci_handshake(ehci, &ehci->regs->command, CMD_RUN,
++ 1, 100 * 1000);
++ else
++ rc = ehci_handshake(ehci, &ehci->regs->status, STS_HALT,
++ 0, 100 * 1000);
+
+ up_write(&ehci_cf_port_reset_rwsem);
+
+diff --git a/drivers/usb/host/ehci-pci.c b/drivers/usb/host/ehci-pci.c
+index e87cf3a00fa4b..638f03b897394 100644
+--- a/drivers/usb/host/ehci-pci.c
++++ b/drivers/usb/host/ehci-pci.c
+@@ -21,6 +21,9 @@ static const char hcd_name[] = "ehci-pci";
+ /* defined here to avoid adding to pci_ids.h for single instance use */
+ #define PCI_DEVICE_ID_INTEL_CE4100_USB 0x2e70
+
++#define PCI_VENDOR_ID_ASPEED 0x1a03
++#define PCI_DEVICE_ID_ASPEED_EHCI 0x2603
++
+ /*-------------------------------------------------------------------------*/
+ #define PCI_DEVICE_ID_INTEL_QUARK_X1000_SOC 0x0939
+ static inline bool is_intel_quark_x1000(struct pci_dev *pdev)
+@@ -222,6 +225,12 @@ static int ehci_pci_setup(struct usb_hcd *hcd)
+ ehci->has_synopsys_hc_bug = 1;
+ }
+ break;
++ case PCI_VENDOR_ID_ASPEED:
++ if (pdev->device == PCI_DEVICE_ID_ASPEED_EHCI) {
++ ehci_info(ehci, "applying Aspeed HC workaround\n");
++ ehci->is_aspeed = 1;
++ }
++ break;
+ }
+
+ /* optional debug port, normally in the first BAR */
+diff --git a/drivers/usb/host/ehci-platform.c b/drivers/usb/host/ehci-platform.c
+index c70f2d0b4aaf0..c3dc906274d93 100644
+--- a/drivers/usb/host/ehci-platform.c
++++ b/drivers/usb/host/ehci-platform.c
+@@ -297,6 +297,12 @@ static int ehci_platform_probe(struct platform_device *dev)
+ "has-transaction-translator"))
+ hcd->has_tt = 1;
+
++ if (of_device_is_compatible(dev->dev.of_node,
++ "aspeed,ast2500-ehci") ||
++ of_device_is_compatible(dev->dev.of_node,
++ "aspeed,ast2600-ehci"))
++ ehci->is_aspeed = 1;
++
+ if (soc_device_match(quirk_poll_match))
+ priv->quirk_poll = true;
+
+diff --git a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h
+index 80bb823aa9fe8..fdd073cc053b8 100644
+--- a/drivers/usb/host/ehci.h
++++ b/drivers/usb/host/ehci.h
+@@ -219,6 +219,7 @@ struct ehci_hcd { /* one per controller */
+ unsigned need_oc_pp_cycle:1; /* MPC834X port power */
+ unsigned imx28_write_fix:1; /* For Freescale i.MX28 */
+ unsigned spurious_oc:1;
++ unsigned is_aspeed:1;
+
+ /* required for usb32 quirk */
+ #define OHCI_CTRL_HCFS (3 << 6)
+diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c
+index 59cc1bc7f12f5..30de85a707fef 100644
+--- a/drivers/usb/host/max3421-hcd.c
++++ b/drivers/usb/host/max3421-hcd.c
+@@ -125,8 +125,6 @@ struct max3421_hcd {
+
+ struct task_struct *spi_thread;
+
+- struct max3421_hcd *next;
+-
+ enum max3421_rh_state rh_state;
+ /* lower 16 bits contain port status, upper 16 bits the change mask: */
+ u32 port_status;
+@@ -174,8 +172,6 @@ struct max3421_ep {
+ u8 retransmit; /* packet needs retransmission */
+ };
+
+-static struct max3421_hcd *max3421_hcd_list;
+-
+ #define MAX3421_FIFO_SIZE 64
+
+ #define MAX3421_SPI_DIR_RD 0 /* read register from MAX3421 */
+@@ -1882,9 +1878,8 @@ max3421_probe(struct spi_device *spi)
+ }
+ set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
+ max3421_hcd = hcd_to_max3421(hcd);
+- max3421_hcd->next = max3421_hcd_list;
+- max3421_hcd_list = max3421_hcd;
+ INIT_LIST_HEAD(&max3421_hcd->ep_list);
++ spi_set_drvdata(spi, max3421_hcd);
+
+ max3421_hcd->tx = kmalloc(sizeof(*max3421_hcd->tx), GFP_KERNEL);
+ if (!max3421_hcd->tx)
+@@ -1934,28 +1929,18 @@ error:
+ static int
+ max3421_remove(struct spi_device *spi)
+ {
+- struct max3421_hcd *max3421_hcd = NULL, **prev;
+- struct usb_hcd *hcd = NULL;
++ struct max3421_hcd *max3421_hcd;
++ struct usb_hcd *hcd;
+ unsigned long flags;
+
+- for (prev = &max3421_hcd_list; *prev; prev = &(*prev)->next) {
+- max3421_hcd = *prev;
+- hcd = max3421_to_hcd(max3421_hcd);
+- if (hcd->self.controller == &spi->dev)
+- break;
+- }
+- if (!max3421_hcd) {
+- dev_err(&spi->dev, "no MAX3421 HCD found for SPI device %p\n",
+- spi);
+- return -ENODEV;
+- }
++ max3421_hcd = spi_get_drvdata(spi);
++ hcd = max3421_to_hcd(max3421_hcd);
+
+ usb_remove_hcd(hcd);
+
+ spin_lock_irqsave(&max3421_hcd->lock, flags);
+
+ kthread_stop(max3421_hcd->spi_thread);
+- *prev = max3421_hcd->next;
+
+ spin_unlock_irqrestore(&max3421_hcd->lock, flags);
+
+diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c
+index 08ec2ab0d95a5..3f3d62dc06746 100644
+--- a/drivers/usb/host/ohci-tmio.c
++++ b/drivers/usb/host/ohci-tmio.c
+@@ -199,7 +199,7 @@ static int ohci_hcd_tmio_drv_probe(struct platform_device *dev)
+ if (usb_disabled())
+ return -ENODEV;
+
+- if (!cell)
++ if (!cell || !regs || !config || !sram)
+ return -EINVAL;
+
+ if (irq < 0)
+diff --git a/drivers/usb/host/uhci-platform.c b/drivers/usb/host/uhci-platform.c
+index 70dbd95c3f063..be9e9db7cad10 100644
+--- a/drivers/usb/host/uhci-platform.c
++++ b/drivers/usb/host/uhci-platform.c
+@@ -113,7 +113,8 @@ static int uhci_hcd_platform_probe(struct platform_device *pdev)
+ num_ports);
+ }
+ if (of_device_is_compatible(np, "aspeed,ast2400-uhci") ||
+- of_device_is_compatible(np, "aspeed,ast2500-uhci")) {
++ of_device_is_compatible(np, "aspeed,ast2500-uhci") ||
++ of_device_is_compatible(np, "aspeed,ast2600-uhci")) {
+ uhci->is_aspeed = 1;
+ dev_info(&pdev->dev,
+ "Enabled Aspeed implementation workarounds\n");
+diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c
+index a3f875eea7519..f65f1ba2b5929 100644
+--- a/drivers/usb/host/xhci-hub.c
++++ b/drivers/usb/host/xhci-hub.c
+@@ -257,7 +257,6 @@ static void xhci_common_hub_descriptor(struct xhci_hcd *xhci,
+ {
+ u16 temp;
+
+- desc->bPwrOn2PwrGood = 10; /* xhci section 5.4.9 says 20ms max */
+ desc->bHubContrCurrent = 0;
+
+ desc->bNbrPorts = ports;
+@@ -292,6 +291,7 @@ static void xhci_usb2_hub_descriptor(struct usb_hcd *hcd, struct xhci_hcd *xhci,
+ desc->bDescriptorType = USB_DT_HUB;
+ temp = 1 + (ports / 8);
+ desc->bDescLength = USB_DT_HUB_NONVAR_SIZE + 2 * temp;
++ desc->bPwrOn2PwrGood = 10; /* xhci section 5.4.8 says 20ms */
+
+ /* The Device Removable bits are reported on a byte granularity.
+ * If the port doesn't exist within that byte, the bit is set to 0.
+@@ -344,6 +344,7 @@ static void xhci_usb3_hub_descriptor(struct usb_hcd *hcd, struct xhci_hcd *xhci,
+ xhci_common_hub_descriptor(xhci, desc, ports);
+ desc->bDescriptorType = USB_DT_SS_HUB;
+ desc->bDescLength = USB_DT_SS_HUB_SIZE;
++ desc->bPwrOn2PwrGood = 50; /* usb 3.1 may fail if less than 100ms */
+
+ /* header decode latency should be zero for roothubs,
+ * see section 4.23.5.2.
+@@ -716,6 +717,7 @@ static int xhci_enter_test_mode(struct xhci_hcd *xhci,
+ continue;
+
+ retval = xhci_disable_slot(xhci, i);
++ xhci_free_virt_device(xhci, i);
+ if (retval)
+ xhci_err(xhci, "Failed to disable slot %d, %d. Enter test mode anyway\n",
+ i, retval);
+@@ -760,7 +762,7 @@ static int xhci_exit_test_mode(struct xhci_hcd *xhci)
+ }
+ pm_runtime_allow(xhci_to_hcd(xhci)->self.controller);
+ xhci->test_mode = 0;
+- return xhci_reset(xhci);
++ return xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
+ }
+
+ void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port,
+@@ -1086,6 +1088,9 @@ static void xhci_get_usb2_port_status(struct xhci_port *port, u32 *status,
+ if (link_state == XDEV_U2)
+ *status |= USB_PORT_STAT_L1;
+ if (link_state == XDEV_U0) {
++ if (bus_state->resume_done[portnum])
++ usb_hcd_end_port_resume(&port->rhub->hcd->self,
++ portnum);
+ bus_state->resume_done[portnum] = 0;
+ clear_bit(portnum, &bus_state->resuming_ports);
+ if (bus_state->suspended_ports & (1 << portnum)) {
+@@ -1429,7 +1434,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
+ }
+ spin_unlock_irqrestore(&xhci->lock, flags);
+ if (!wait_for_completion_timeout(&bus_state->u3exit_done[wIndex],
+- msecs_to_jiffies(100)))
++ msecs_to_jiffies(500)))
+ xhci_dbg(xhci, "missing U0 port change event for port %d-%d\n",
+ hcd->self.busnum, wIndex + 1);
+ spin_lock_irqsave(&xhci->lock, flags);
+diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
+index 0e312066c5c63..b398d3fdabf61 100644
+--- a/drivers/usb/host/xhci-mem.c
++++ b/drivers/usb/host/xhci-mem.c
+@@ -2583,7 +2583,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
+
+ fail:
+ xhci_halt(xhci);
+- xhci_reset(xhci);
++ xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
+ xhci_mem_cleanup(xhci);
+ return -ENOMEM;
+ }
+diff --git a/drivers/usb/host/xhci-mtk-sch.c b/drivers/usb/host/xhci-mtk-sch.c
+index 134f4789bd897..f91a304320563 100644
+--- a/drivers/usb/host/xhci-mtk-sch.c
++++ b/drivers/usb/host/xhci-mtk-sch.c
+@@ -465,7 +465,7 @@ static int check_fs_bus_bw(struct mu3h_sch_ep_info *sch_ep, int offset)
+ */
+ for (j = 0; j < sch_ep->num_budget_microframes; j++) {
+ k = XHCI_MTK_BW_INDEX(base + j);
+- tmp = tt->fs_bus_bw[k] + sch_ep->bw_budget_table[j];
++ tmp = tt->fs_bus_bw[k] + sch_ep->bw_cost_per_microframe;
+ if (tmp > FS_PAYLOAD_MAX)
+ return -ESCH_BW_OVERFLOW;
+ }
+@@ -539,19 +539,17 @@ static int check_sch_tt(struct mu3h_sch_ep_info *sch_ep, u32 offset)
+ static void update_sch_tt(struct mu3h_sch_ep_info *sch_ep, bool used)
+ {
+ struct mu3h_sch_tt *tt = sch_ep->sch_tt;
++ int bw_updated;
+ u32 base;
+- int i, j, k;
++ int i, j;
++
++ bw_updated = sch_ep->bw_cost_per_microframe * (used ? 1 : -1);
+
+ for (i = 0; i < sch_ep->num_esit; i++) {
+ base = sch_ep->offset + i * sch_ep->esit;
+
+- for (j = 0; j < sch_ep->num_budget_microframes; j++) {
+- k = XHCI_MTK_BW_INDEX(base + j);
+- if (used)
+- tt->fs_bus_bw[k] += sch_ep->bw_budget_table[j];
+- else
+- tt->fs_bus_bw[k] -= sch_ep->bw_budget_table[j];
+- }
++ for (j = 0; j < sch_ep->num_budget_microframes; j++)
++ tt->fs_bus_bw[XHCI_MTK_BW_INDEX(base + j)] += bw_updated;
+ }
+
+ if (used)
+@@ -781,7 +779,7 @@ int xhci_mtk_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
+
+ ret = xhci_check_bandwidth(hcd, udev);
+ if (!ret)
+- INIT_LIST_HEAD(&mtk->bw_ep_chk_list);
++ list_del_init(&mtk->bw_ep_chk_list);
+
+ return ret;
+ }
+diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
+index 2484a9d38ce2b..cb8b481a94990 100644
+--- a/drivers/usb/host/xhci-pci.c
++++ b/drivers/usb/host/xhci-pci.c
+@@ -59,12 +59,22 @@
+ #define PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI 0x9a13
+ #define PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI 0x1138
+ #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI 0x461e
++#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI 0x51ed
+
+ #define PCI_DEVICE_ID_AMD_RENOIR_XHCI 0x1639
+ #define PCI_DEVICE_ID_AMD_PROMONTORYA_4 0x43b9
+ #define PCI_DEVICE_ID_AMD_PROMONTORYA_3 0x43ba
+ #define PCI_DEVICE_ID_AMD_PROMONTORYA_2 0x43bb
+ #define PCI_DEVICE_ID_AMD_PROMONTORYA_1 0x43bc
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_1 0x161a
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_2 0x161b
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_3 0x161d
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_4 0x161e
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_5 0x15d6
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_6 0x15d7
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_7 0x161c
++#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_8 0x161f
++
+ #define PCI_DEVICE_ID_ASMEDIA_1042_XHCI 0x1042
+ #define PCI_DEVICE_ID_ASMEDIA_1042A_XHCI 0x1142
+ #define PCI_DEVICE_ID_ASMEDIA_1142_XHCI 0x1242
+@@ -114,7 +124,6 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
+ /* Look for vendor-specific quirks */
+ if (pdev->vendor == PCI_VENDOR_ID_FRESCO_LOGIC &&
+ (pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_PDK ||
+- pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_FL1100 ||
+ pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_FL1400)) {
+ if (pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_PDK &&
+ pdev->revision == 0x0) {
+@@ -149,6 +158,10 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
+ pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_FL1009)
+ xhci->quirks |= XHCI_BROKEN_STREAMS;
+
++ if (pdev->vendor == PCI_VENDOR_ID_FRESCO_LOGIC &&
++ pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_FL1100)
++ xhci->quirks |= XHCI_TRUST_TX_LENGTH;
++
+ if (pdev->vendor == PCI_VENDOR_ID_NEC)
+ xhci->quirks |= XHCI_NEC_HOST;
+
+@@ -254,7 +267,8 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
+ pdev->device == PCI_DEVICE_ID_INTEL_ICE_LAKE_XHCI ||
+ pdev->device == PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI ||
+ pdev->device == PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI ||
+- pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI))
++ pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI ||
++ pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI))
+ xhci->quirks |= XHCI_DEFAULT_PM_RUNTIME_ALLOW;
+
+ if (pdev->vendor == PCI_VENDOR_ID_ETRON &&
+@@ -317,6 +331,17 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
+ pdev->device == PCI_DEVICE_ID_AMD_PROMONTORYA_4))
+ xhci->quirks |= XHCI_NO_SOFT_RETRY;
+
++ if (pdev->vendor == PCI_VENDOR_ID_AMD &&
++ (pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_1 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_2 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_3 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_4 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_5 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_6 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_7 ||
++ pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_8))
++ xhci->quirks |= XHCI_DEFAULT_PM_RUNTIME_ALLOW;
++
+ if (xhci->quirks & XHCI_RESET_ON_RESUME)
+ xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
+ "QUIRK: Resetting on resume");
+diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
+index c1edcc9b13cec..dc570ce4e8319 100644
+--- a/drivers/usb/host/xhci-plat.c
++++ b/drivers/usb/host/xhci-plat.c
+@@ -437,6 +437,9 @@ static int __maybe_unused xhci_plat_suspend(struct device *dev)
+ struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+ int ret;
+
++ if (pm_runtime_suspended(dev))
++ pm_runtime_resume(dev);
++
+ ret = xhci_priv_suspend_quirk(hcd);
+ if (ret)
+ return ret;
+diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
+index 311597bba80e2..f9707997969d4 100644
+--- a/drivers/usb/host/xhci-ring.c
++++ b/drivers/usb/host/xhci-ring.c
+@@ -366,7 +366,9 @@ static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
+ /* Must be called with xhci->lock held, releases and aquires lock back */
+ static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
+ {
+- u32 temp_32;
++ struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg;
++ union xhci_trb *new_deq = xhci->cmd_ring->dequeue;
++ u64 crcr;
+ int ret;
+
+ xhci_dbg(xhci, "Abort command ring\n");
+@@ -375,13 +377,18 @@ static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
+
+ /*
+ * The control bits like command stop, abort are located in lower
+- * dword of the command ring control register. Limit the write
+- * to the lower dword to avoid corrupting the command ring pointer
+- * in case if the command ring is stopped by the time upper dword
+- * is written.
++ * dword of the command ring control register.
++ * Some controllers require all 64 bits to be written to abort the ring.
++ * Make sure the upper dword is valid, pointing to the next command,
++ * avoiding corrupting the command ring pointer in case the command ring
++ * is stopped by the time the upper dword is written.
+ */
+- temp_32 = readl(&xhci->op_regs->cmd_ring);
+- writel(temp_32 | CMD_RING_ABORT, &xhci->op_regs->cmd_ring);
++ next_trb(xhci, NULL, &new_seg, &new_deq);
++ if (trb_is_link(new_deq))
++ next_trb(xhci, NULL, &new_seg, &new_deq);
++
++ crcr = xhci_trb_virt_to_dma(new_seg, new_deq);
++ xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring);
+
+ /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the
+ * completion of the Command Abort operation. If CRR is not negated in 5
+@@ -1518,7 +1525,6 @@ static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
+ if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
+ /* Delete default control endpoint resources */
+ xhci_free_device_endpoint_resources(xhci, virt_dev, true);
+- xhci_free_virt_device(xhci, slot_id);
+ }
+
+ static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
+@@ -3135,6 +3141,7 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd)
+ if (event_loop++ < TRBS_PER_SEGMENT / 2)
+ continue;
+ xhci_update_erst_dequeue(xhci, event_ring_deq);
++ event_ring_deq = xhci->event_ring->dequeue;
+
+ /* ring is half-full, force isoc trbs to interrupt more often */
+ if (xhci->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN)
+diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
+index 1bf494b649bd2..996958a6565c3 100644
+--- a/drivers/usb/host/xhci-tegra.c
++++ b/drivers/usb/host/xhci-tegra.c
+@@ -1034,13 +1034,13 @@ static int tegra_xusb_unpowergate_partitions(struct tegra_xusb *tegra)
+ int rc;
+
+ if (tegra->use_genpd) {
+- rc = pm_runtime_get_sync(tegra->genpd_dev_ss);
++ rc = pm_runtime_resume_and_get(tegra->genpd_dev_ss);
+ if (rc < 0) {
+ dev_err(dev, "failed to enable XUSB SS partition\n");
+ return rc;
+ }
+
+- rc = pm_runtime_get_sync(tegra->genpd_dev_host);
++ rc = pm_runtime_resume_and_get(tegra->genpd_dev_host);
+ if (rc < 0) {
+ dev_err(dev, "failed to enable XUSB Host partition\n");
+ pm_runtime_put_sync(tegra->genpd_dev_ss);
+@@ -1400,6 +1400,7 @@ static void tegra_xusb_deinit_usb_phy(struct tegra_xusb *tegra)
+
+ static int tegra_xusb_probe(struct platform_device *pdev)
+ {
++ struct of_phandle_args args;
+ struct tegra_xusb *tegra;
+ struct device_node *np;
+ struct resource *regs;
+@@ -1454,10 +1455,17 @@ static int tegra_xusb_probe(struct platform_device *pdev)
+ goto put_padctl;
+ }
+
+- tegra->padctl_irq = of_irq_get(np, 0);
+- if (tegra->padctl_irq <= 0) {
+- err = (tegra->padctl_irq == 0) ? -ENODEV : tegra->padctl_irq;
+- goto put_padctl;
++ /* Older device-trees don't have padctrl interrupt */
++ err = of_irq_parse_one(np, 0, &args);
++ if (!err) {
++ tegra->padctl_irq = of_irq_get(np, 0);
++ if (tegra->padctl_irq <= 0) {
++ err = (tegra->padctl_irq == 0) ? -ENODEV : tegra->padctl_irq;
++ goto put_padctl;
++ }
++ } else {
++ dev_dbg(&pdev->dev,
++ "%pOF is missing an interrupt, disabling PM support\n", np);
+ }
+
+ tegra->host_clk = devm_clk_get(&pdev->dev, "xusb_host");
+@@ -1696,11 +1704,15 @@ static int tegra_xusb_probe(struct platform_device *pdev)
+ goto remove_usb3;
+ }
+
+- err = devm_request_threaded_irq(&pdev->dev, tegra->padctl_irq, NULL, tegra_xusb_padctl_irq,
+- IRQF_ONESHOT, dev_name(&pdev->dev), tegra);
+- if (err < 0) {
+- dev_err(&pdev->dev, "failed to request padctl IRQ: %d\n", err);
+- goto remove_usb3;
++ if (tegra->padctl_irq) {
++ err = devm_request_threaded_irq(&pdev->dev, tegra->padctl_irq,
++ NULL, tegra_xusb_padctl_irq,
++ IRQF_ONESHOT, dev_name(&pdev->dev),
++ tegra);
++ if (err < 0) {
++ dev_err(&pdev->dev, "failed to request padctl IRQ: %d\n", err);
++ goto remove_usb3;
++ }
+ }
+
+ err = tegra_xusb_enable_firmware_messages(tegra);
+@@ -1718,13 +1730,16 @@ static int tegra_xusb_probe(struct platform_device *pdev)
+ /* Enable wake for both USB 2.0 and USB 3.0 roothubs */
+ device_init_wakeup(&tegra->hcd->self.root_hub->dev, true);
+ device_init_wakeup(&xhci->shared_hcd->self.root_hub->dev, true);
+- device_init_wakeup(tegra->dev, true);
+
+ pm_runtime_use_autosuspend(tegra->dev);
+ pm_runtime_set_autosuspend_delay(tegra->dev, 2000);
+ pm_runtime_mark_last_busy(tegra->dev);
+ pm_runtime_set_active(tegra->dev);
+- pm_runtime_enable(tegra->dev);
++
++ if (tegra->padctl_irq) {
++ device_init_wakeup(tegra->dev, true);
++ pm_runtime_enable(tegra->dev);
++ }
+
+ return 0;
+
+@@ -1772,7 +1787,9 @@ static int tegra_xusb_remove(struct platform_device *pdev)
+ dma_free_coherent(&pdev->dev, tegra->fw.size, tegra->fw.virt,
+ tegra->fw.phys);
+
+- pm_runtime_disable(&pdev->dev);
++ if (tegra->padctl_irq)
++ pm_runtime_disable(&pdev->dev);
++
+ pm_runtime_put(&pdev->dev);
+
+ tegra_xusb_powergate_partitions(tegra);
+diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
+index 541fe4dcc43a2..90f5a3ce7c348 100644
+--- a/drivers/usb/host/xhci.c
++++ b/drivers/usb/host/xhci.c
+@@ -65,7 +65,7 @@ static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
+ * handshake done). There are two failure modes: "usec" have passed (major
+ * hardware flakeout), or the register reads as all-ones (hardware removed).
+ */
+-int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
++int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us)
+ {
+ u32 result;
+ int ret;
+@@ -73,7 +73,7 @@ int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
+ ret = readl_poll_timeout_atomic(ptr, result,
+ (result & mask) == done ||
+ result == U32_MAX,
+- 1, usec);
++ 1, timeout_us);
+ if (result == U32_MAX) /* card removed */
+ return -ENODEV;
+
+@@ -162,7 +162,7 @@ int xhci_start(struct xhci_hcd *xhci)
+ * Transactions will be terminated immediately, and operational registers
+ * will be set to their defaults.
+ */
+-int xhci_reset(struct xhci_hcd *xhci)
++int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us)
+ {
+ u32 command;
+ u32 state;
+@@ -195,8 +195,7 @@ int xhci_reset(struct xhci_hcd *xhci)
+ if (xhci->quirks & XHCI_INTEL_HOST)
+ udelay(1000);
+
+- ret = xhci_handshake(&xhci->op_regs->command,
+- CMD_RESET, 0, 10 * 1000 * 1000);
++ ret = xhci_handshake(&xhci->op_regs->command, CMD_RESET, 0, timeout_us);
+ if (ret)
+ return ret;
+
+@@ -209,8 +208,7 @@ int xhci_reset(struct xhci_hcd *xhci)
+ * xHCI cannot write to any doorbells or operational registers other
+ * than status until the "Controller Not Ready" flag is cleared.
+ */
+- ret = xhci_handshake(&xhci->op_regs->status,
+- STS_CNR, 0, 10 * 1000 * 1000);
++ ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us);
+
+ xhci->usb2_rhub.bus_state.port_c_suspend = 0;
+ xhci->usb2_rhub.bus_state.suspended_ports = 0;
+@@ -692,7 +690,6 @@ int xhci_run(struct usb_hcd *hcd)
+ if (ret)
+ xhci_free_command(xhci, command);
+ }
+- set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags);
+ xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+ "Finished xhci_run for USB2 roothub");
+
+@@ -732,7 +729,7 @@ static void xhci_stop(struct usb_hcd *hcd)
+ xhci->xhc_state |= XHCI_STATE_HALTED;
+ xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
+ xhci_halt(xhci);
+- xhci_reset(xhci);
++ xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
+ spin_unlock_irq(&xhci->lock);
+
+ xhci_cleanup_msix(xhci);
+@@ -781,11 +778,22 @@ void xhci_shutdown(struct usb_hcd *hcd)
+ if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
+ usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
+
++ /* Don't poll the roothubs after shutdown. */
++ xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
++ __func__, hcd->self.busnum);
++ clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
++ del_timer_sync(&hcd->rh_timer);
++
++ if (xhci->shared_hcd) {
++ clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
++ del_timer_sync(&xhci->shared_hcd->rh_timer);
++ }
++
+ spin_lock_irq(&xhci->lock);
+ xhci_halt(xhci);
+ /* Workaround for spurious wakeups at shutdown with HSW */
+ if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
+- xhci_reset(xhci);
++ xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
+ spin_unlock_irq(&xhci->lock);
+
+ xhci_cleanup_msix(xhci);
+@@ -1092,6 +1100,7 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
+ int retval = 0;
+ bool comp_timer_running = false;
+ bool pending_portevent = false;
++ bool reinit_xhc = false;
+
+ if (!hcd->state)
+ return 0;
+@@ -1108,10 +1117,11 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
+ set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
+
+ spin_lock_irq(&xhci->lock);
+- if ((xhci->quirks & XHCI_RESET_ON_RESUME) || xhci->broken_suspend)
+- hibernated = true;
+
+- if (!hibernated) {
++ if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend)
++ reinit_xhc = true;
++
++ if (!reinit_xhc) {
+ /*
+ * Some controllers might lose power during suspend, so wait
+ * for controller not ready bit to clear, just as in xHC init.
+@@ -1144,12 +1154,17 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
+ spin_unlock_irq(&xhci->lock);
+ return -ETIMEDOUT;
+ }
+- temp = readl(&xhci->op_regs->status);
+ }
+
+- /* If restore operation fails, re-initialize the HC during resume */
+- if ((temp & STS_SRE) || hibernated) {
++ temp = readl(&xhci->op_regs->status);
+
++ /* re-initialize the HC on Restore Error, or Host Controller Error */
++ if (temp & (STS_SRE | STS_HCE)) {
++ reinit_xhc = true;
++ xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp);
++ }
++
++ if (reinit_xhc) {
+ if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
+ !(xhci_all_ports_seen_u0(xhci))) {
+ del_timer_sync(&xhci->comp_mode_recovery_timer);
+@@ -1164,7 +1179,7 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
+ xhci_dbg(xhci, "Stop HCD\n");
+ xhci_halt(xhci);
+ xhci_zero_64b_regs(xhci);
+- retval = xhci_reset(xhci);
++ retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
+ spin_unlock_irq(&xhci->lock);
+ if (retval)
+ return retval;
+@@ -1605,9 +1620,12 @@ static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flag
+ struct urb_priv *urb_priv;
+ int num_tds;
+
+- if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
+- true, true, __func__) <= 0)
++ if (!urb)
+ return -EINVAL;
++ ret = xhci_check_args(hcd, urb->dev, urb->ep,
++ true, true, __func__);
++ if (ret <= 0)
++ return ret ? ret : -EINVAL;
+
+ slot_id = urb->dev->slot_id;
+ ep_index = xhci_get_endpoint_index(&urb->ep->desc);
+@@ -3324,7 +3342,7 @@ static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
+ return -EINVAL;
+ ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
+ if (ret <= 0)
+- return -EINVAL;
++ return ret ? ret : -EINVAL;
+ if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
+ xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
+ " descriptor for ep 0x%x does not support streams\n",
+@@ -3935,7 +3953,6 @@ static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
+ struct xhci_slot_ctx *slot_ctx;
+ int i, ret;
+
+-#ifndef CONFIG_USB_DEFAULT_PERSIST
+ /*
+ * We called pm_runtime_get_noresume when the device was attached.
+ * Decrement the counter here to allow controller to runtime suspend
+@@ -3943,7 +3960,6 @@ static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
+ */
+ if (xhci->quirks & XHCI_RESET_ON_RESUME)
+ pm_runtime_put_noidle(hcd->self.controller);
+-#endif
+
+ ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
+ /* If the host is halted due to driver unload, we still need to free the
+@@ -3962,9 +3978,8 @@ static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
+ del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
+ }
+ virt_dev->udev = NULL;
+- ret = xhci_disable_slot(xhci, udev->slot_id);
+- if (ret)
+- xhci_free_virt_device(xhci, udev->slot_id);
++ xhci_disable_slot(xhci, udev->slot_id);
++ xhci_free_virt_device(xhci, udev->slot_id);
+ }
+
+ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
+@@ -3974,7 +3989,7 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
+ u32 state;
+ int ret = 0;
+
+- command = xhci_alloc_command(xhci, false, GFP_KERNEL);
++ command = xhci_alloc_command(xhci, true, GFP_KERNEL);
+ if (!command)
+ return -ENOMEM;
+
+@@ -3999,6 +4014,15 @@ int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
+ }
+ xhci_ring_cmd_db(xhci);
+ spin_unlock_irqrestore(&xhci->lock, flags);
++
++ wait_for_completion(command->completion);
++
++ if (command->status != COMP_SUCCESS)
++ xhci_warn(xhci, "Unsuccessful disable slot %u command, status %d\n",
++ slot_id, command->status);
++
++ xhci_free_command(xhci, command);
++
+ return ret;
+ }
+
+@@ -4095,23 +4119,20 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
+
+ xhci_debugfs_create_slot(xhci, slot_id);
+
+-#ifndef CONFIG_USB_DEFAULT_PERSIST
+ /*
+ * If resetting upon resume, we can't put the controller into runtime
+ * suspend if there is a device attached.
+ */
+ if (xhci->quirks & XHCI_RESET_ON_RESUME)
+ pm_runtime_get_noresume(hcd->self.controller);
+-#endif
+
+ /* Is this a LS or FS device under a HS hub? */
+ /* Hub or peripherial? */
+ return 1;
+
+ disable_slot:
+- ret = xhci_disable_slot(xhci, udev->slot_id);
+- if (ret)
+- xhci_free_virt_device(xhci, udev->slot_id);
++ xhci_disable_slot(xhci, udev->slot_id);
++ xhci_free_virt_device(xhci, udev->slot_id);
+
+ return 0;
+ }
+@@ -4241,6 +4262,7 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
+
+ mutex_unlock(&xhci->mutex);
+ ret = xhci_disable_slot(xhci, udev->slot_id);
++ xhci_free_virt_device(xhci, udev->slot_id);
+ if (!ret)
+ xhci_alloc_dev(hcd, udev);
+ kfree(command->completion);
+@@ -5305,7 +5327,7 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
+
+ xhci_dbg(xhci, "Resetting HCD\n");
+ /* Reset the internal HC memory state and registers. */
+- retval = xhci_reset(xhci);
++ retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
+ if (retval)
+ return retval;
+ xhci_dbg(xhci, "Reset complete\n");
+diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
+index 5a75fe5631238..bc0789229527f 100644
+--- a/drivers/usb/host/xhci.h
++++ b/drivers/usb/host/xhci.h
+@@ -229,6 +229,9 @@ struct xhci_op_regs {
+ #define CMD_ETE (1 << 14)
+ /* bits 15:31 are reserved (and should be preserved on writes). */
+
++#define XHCI_RESET_LONG_USEC (10 * 1000 * 1000)
++#define XHCI_RESET_SHORT_USEC (250 * 1000)
++
+ /* IMAN - Interrupt Management Register */
+ #define IMAN_IE (1 << 1)
+ #define IMAN_IP (1 << 0)
+@@ -2083,11 +2086,11 @@ void xhci_free_container_ctx(struct xhci_hcd *xhci,
+
+ /* xHCI host controller glue */
+ typedef void (*xhci_get_quirks_t)(struct device *, struct xhci_hcd *);
+-int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec);
++int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us);
+ void xhci_quiesce(struct xhci_hcd *xhci);
+ int xhci_halt(struct xhci_hcd *xhci);
+ int xhci_start(struct xhci_hcd *xhci);
+-int xhci_reset(struct xhci_hcd *xhci);
++int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us);
+ int xhci_run(struct usb_hcd *hcd);
+ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks);
+ void xhci_shutdown(struct usb_hcd *hcd);
+@@ -2467,6 +2470,8 @@ static inline const char *xhci_decode_ctrl_ctx(char *str,
+ unsigned int bit;
+ int ret = 0;
+
++ str[0] = '\0';
++
+ if (drop) {
+ ret = sprintf(str, "Drop:");
+ for_each_set_bit(bit, &drop, 32)
+@@ -2624,8 +2629,11 @@ static inline const char *xhci_decode_usbsts(char *str, u32 usbsts)
+ {
+ int ret = 0;
+
++ ret = sprintf(str, " 0x%08x", usbsts);
++
+ if (usbsts == ~(u32)0)
+- return " 0xffffffff";
++ return str;
++
+ if (usbsts & STS_HALT)
+ ret += sprintf(str + ret, " HCHalted");
+ if (usbsts & STS_FATAL)
+diff --git a/drivers/usb/misc/ftdi-elan.c b/drivers/usb/misc/ftdi-elan.c
+index e5a8fcdbb78e7..6c38c62d29b26 100644
+--- a/drivers/usb/misc/ftdi-elan.c
++++ b/drivers/usb/misc/ftdi-elan.c
+@@ -202,6 +202,7 @@ static void ftdi_elan_delete(struct kref *kref)
+ mutex_unlock(&ftdi_module_lock);
+ kfree(ftdi->bulk_in_buffer);
+ ftdi->bulk_in_buffer = NULL;
++ kfree(ftdi);
+ }
+
+ static void ftdi_elan_put_kref(struct usb_ftdi *ftdi)
+diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c
+index efbd317f2f252..988a8c02e7e24 100644
+--- a/drivers/usb/misc/iowarrior.c
++++ b/drivers/usb/misc/iowarrior.c
+@@ -99,10 +99,6 @@ struct iowarrior {
+ /* globals */
+ /*--------------*/
+
+-/*
+- * USB spec identifies 5 second timeouts.
+- */
+-#define GET_TIMEOUT 5
+ #define USB_REQ_GET_REPORT 0x01
+ //#if 0
+ static int usb_get_report(struct usb_device *dev,
+@@ -114,7 +110,7 @@ static int usb_get_report(struct usb_device *dev,
+ USB_DIR_IN | USB_TYPE_CLASS |
+ USB_RECIP_INTERFACE, (type << 8) + id,
+ inter->desc.bInterfaceNumber, buf, size,
+- GET_TIMEOUT*HZ);
++ USB_CTRL_GET_TIMEOUT);
+ }
+ //#endif
+
+@@ -129,7 +125,7 @@ static int usb_set_report(struct usb_interface *intf, unsigned char type,
+ USB_TYPE_CLASS | USB_RECIP_INTERFACE,
+ (type << 8) + id,
+ intf->cur_altsetting->desc.bInterfaceNumber, buf,
+- size, HZ);
++ size, 1000);
+ }
+
+ /*---------------------*/
+diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c
+index 748139d262633..0be8efcda15d5 100644
+--- a/drivers/usb/misc/uss720.c
++++ b/drivers/usb/misc/uss720.c
+@@ -71,6 +71,7 @@ static void destroy_priv(struct kref *kref)
+
+ dev_dbg(&priv->usbdev->dev, "destroying priv datastructure\n");
+ usb_put_dev(priv->usbdev);
++ priv->usbdev = NULL;
+ kfree(priv);
+ }
+
+@@ -736,7 +737,6 @@ static int uss720_probe(struct usb_interface *intf,
+ parport_announce_port(pp);
+
+ usb_set_intfdata(intf, pp);
+- usb_put_dev(usbdev);
+ return 0;
+
+ probe_abort:
+@@ -754,7 +754,6 @@ static void uss720_disconnect(struct usb_interface *intf)
+ usb_set_intfdata(intf, NULL);
+ if (pp) {
+ priv = pp->private_data;
+- priv->usbdev = NULL;
+ priv->pp = NULL;
+ dev_dbg(&intf->dev, "parport_remove_port\n");
+ parport_remove_port(pp);
+diff --git a/drivers/usb/mtu3/mtu3_dr.c b/drivers/usb/mtu3/mtu3_dr.c
+index a6b04831b20bf..9b8aded3d95e9 100644
+--- a/drivers/usb/mtu3/mtu3_dr.c
++++ b/drivers/usb/mtu3/mtu3_dr.c
+@@ -21,10 +21,8 @@ static inline struct ssusb_mtk *otg_sx_to_ssusb(struct otg_switch_mtk *otg_sx)
+
+ static void toggle_opstate(struct ssusb_mtk *ssusb)
+ {
+- if (!ssusb->otg_switch.is_u3_drd) {
+- mtu3_setbits(ssusb->mac_base, U3D_DEVICE_CONTROL, DC_SESSION);
+- mtu3_setbits(ssusb->mac_base, U3D_POWER_MANAGEMENT, SOFT_CONN);
+- }
++ mtu3_setbits(ssusb->mac_base, U3D_DEVICE_CONTROL, DC_SESSION);
++ mtu3_setbits(ssusb->mac_base, U3D_POWER_MANAGEMENT, SOFT_CONN);
+ }
+
+ /* only port0 supports dual-role mode */
+diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c
+index a9a65b4bbfede..9977600616d7e 100644
+--- a/drivers/usb/mtu3/mtu3_gadget.c
++++ b/drivers/usb/mtu3/mtu3_gadget.c
+@@ -77,7 +77,7 @@ static int mtu3_ep_enable(struct mtu3_ep *mep)
+ if (usb_endpoint_xfer_int(desc) ||
+ usb_endpoint_xfer_isoc(desc)) {
+ interval = desc->bInterval;
+- interval = clamp_val(interval, 1, 16) - 1;
++ interval = clamp_val(interval, 1, 16);
+ if (usb_endpoint_xfer_isoc(desc) && comp_desc)
+ mult = comp_desc->bmAttributes;
+ }
+@@ -89,9 +89,16 @@ static int mtu3_ep_enable(struct mtu3_ep *mep)
+ if (usb_endpoint_xfer_isoc(desc) ||
+ usb_endpoint_xfer_int(desc)) {
+ interval = desc->bInterval;
+- interval = clamp_val(interval, 1, 16) - 1;
++ interval = clamp_val(interval, 1, 16);
+ mult = usb_endpoint_maxp_mult(desc) - 1;
+ }
++ break;
++ case USB_SPEED_FULL:
++ if (usb_endpoint_xfer_isoc(desc))
++ interval = clamp_val(desc->bInterval, 1, 16);
++ else if (usb_endpoint_xfer_int(desc))
++ interval = clamp_val(desc->bInterval, 1, 255);
++
+ break;
+ default:
+ break; /*others are ignored */
+@@ -235,6 +242,7 @@ struct usb_request *mtu3_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
+ mreq->request.dma = DMA_ADDR_INVALID;
+ mreq->epnum = mep->epnum;
+ mreq->mep = mep;
++ INIT_LIST_HEAD(&mreq->list);
+ trace_mtu3_alloc_request(mreq);
+
+ return &mreq->request;
+diff --git a/drivers/usb/mtu3/mtu3_qmu.c b/drivers/usb/mtu3/mtu3_qmu.c
+index 3f414f91b5899..2ea3157ddb6e2 100644
+--- a/drivers/usb/mtu3/mtu3_qmu.c
++++ b/drivers/usb/mtu3/mtu3_qmu.c
+@@ -273,6 +273,8 @@ static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
+ gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP);
+ }
+
++ /* prevent reorder, make sure GPD's HWO is set last */
++ mb();
+ gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
+
+ mreq->gpd = gpd;
+@@ -306,6 +308,8 @@ static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
+ gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
+ ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
+ gpd->dw3_info = cpu_to_le32(ext_addr);
++ /* prevent reorder, make sure GPD's HWO is set last */
++ mb();
+ gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
+
+ mreq->gpd = gpd;
+@@ -445,7 +449,8 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
+ return;
+ }
+ mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
+-
++ /* prevent reorder, make sure GPD's HWO is set last */
++ mb();
+ /* by pass the current GDP */
+ gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
+
+diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig
+index 8de143807c1ae..4d61df6a9b5c8 100644
+--- a/drivers/usb/musb/Kconfig
++++ b/drivers/usb/musb/Kconfig
+@@ -120,7 +120,7 @@ config USB_MUSB_MEDIATEK
+ tristate "MediaTek platforms"
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ depends on NOP_USB_XCEIV
+- depends on GENERIC_PHY
++ select GENERIC_PHY
+ select USB_ROLE_SWITCH
+
+ comment "MUSB DMA mode"
+diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c
+index 98c0f4c1bffd9..51274b87f46c9 100644
+--- a/drivers/usb/musb/musb_gadget.c
++++ b/drivers/usb/musb/musb_gadget.c
+@@ -1247,9 +1247,11 @@ static int musb_gadget_queue(struct usb_ep *ep, struct usb_request *req,
+ status = musb_queue_resume_work(musb,
+ musb_ep_restart_resume_work,
+ request);
+- if (status < 0)
++ if (status < 0) {
+ dev_err(musb->controller, "%s resume work: %i\n",
+ __func__, status);
++ list_del(&request->list);
++ }
+ }
+
+ unlock:
+diff --git a/drivers/usb/musb/tusb6010.c b/drivers/usb/musb/tusb6010.c
+index c968ecda42aa8..7ed4cc348d993 100644
+--- a/drivers/usb/musb/tusb6010.c
++++ b/drivers/usb/musb/tusb6010.c
+@@ -1104,6 +1104,11 @@ static int tusb_musb_init(struct musb *musb)
+
+ /* dma address for async dma */
+ mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++ if (!mem) {
++ pr_debug("no async dma resource?\n");
++ ret = -ENODEV;
++ goto done;
++ }
+ musb->async = mem->start;
+
+ /* dma address for sync dma */
+diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c
+index 661a229c105dd..34b9f81401871 100644
+--- a/drivers/usb/phy/phy-generic.c
++++ b/drivers/usb/phy/phy-generic.c
+@@ -268,6 +268,13 @@ int usb_phy_gen_create_phy(struct device *dev, struct usb_phy_generic *nop)
+ return -EPROBE_DEFER;
+ }
+
++ nop->vbus_draw = devm_regulator_get_exclusive(dev, "vbus");
++ if (PTR_ERR(nop->vbus_draw) == -ENODEV)
++ nop->vbus_draw = NULL;
++ if (IS_ERR(nop->vbus_draw))
++ return dev_err_probe(dev, PTR_ERR(nop->vbus_draw),
++ "could not get vbus regulator\n");
++
+ nop->dev = dev;
+ nop->phy.dev = nop->dev;
+ nop->phy.label = "nop-xceiv";
+diff --git a/drivers/usb/serial/Kconfig b/drivers/usb/serial/Kconfig
+index de5c012570603..ef8d1c73c7545 100644
+--- a/drivers/usb/serial/Kconfig
++++ b/drivers/usb/serial/Kconfig
+@@ -66,6 +66,7 @@ config USB_SERIAL_SIMPLE
+ - Libtransistor USB console
+ - a number of Motorola phones
+ - Motorola Tetra devices
++ - Nokia mobile phones
+ - Novatel Wireless GPS receivers
+ - Siemens USB/MPI adapter.
+ - ViVOtech ViVOpay USB device.
+diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c
+index 2db917eab7995..b5a1864e9cfdc 100644
+--- a/drivers/usb/serial/ch341.c
++++ b/drivers/usb/serial/ch341.c
+@@ -81,10 +81,10 @@
+ #define CH341_QUIRK_SIMULATE_BREAK BIT(1)
+
+ static const struct usb_device_id id_table[] = {
+- { USB_DEVICE(0x1a86, 0x5512) },
+ { USB_DEVICE(0x1a86, 0x5523) },
+ { USB_DEVICE(0x1a86, 0x7522) },
+ { USB_DEVICE(0x1a86, 0x7523) },
++ { USB_DEVICE(0x2184, 0x0057) },
+ { USB_DEVICE(0x4348, 0x5523) },
+ { USB_DEVICE(0x9986, 0x7523) },
+ { },
+diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c
+index 189279869a8b0..bd006e1712ccb 100644
+--- a/drivers/usb/serial/cp210x.c
++++ b/drivers/usb/serial/cp210x.c
+@@ -51,6 +51,7 @@ static void cp210x_enable_event_mode(struct usb_serial_port *port);
+ static void cp210x_disable_event_mode(struct usb_serial_port *port);
+
+ static const struct usb_device_id id_table[] = {
++ { USB_DEVICE(0x0404, 0x034C) }, /* NCR Retail IO Box */
+ { USB_DEVICE(0x045B, 0x0053) }, /* Renesas RX610 RX-Stick */
+ { USB_DEVICE(0x0471, 0x066A) }, /* AKTAKOM ACE-1001 cable */
+ { USB_DEVICE(0x0489, 0xE000) }, /* Pirelli Broadband S.p.A, DP-L10 SIP/GSM Mobile */
+@@ -68,6 +69,7 @@ static const struct usb_device_id id_table[] = {
+ { USB_DEVICE(0x0FCF, 0x1004) }, /* Dynastream ANT2USB */
+ { USB_DEVICE(0x0FCF, 0x1006) }, /* Dynastream ANT development board */
+ { USB_DEVICE(0x0FDE, 0xCA05) }, /* OWL Wireless Electricity Monitor CM-160 */
++ { USB_DEVICE(0x106F, 0x0003) }, /* CPI / Money Controls Bulk Coin Recycler */
+ { USB_DEVICE(0x10A6, 0xAA26) }, /* Knock-off DCU-11 cable */
+ { USB_DEVICE(0x10AB, 0x10C5) }, /* Siemens MC60 Cable */
+ { USB_DEVICE(0x10B5, 0xAC70) }, /* Nokia CA-42 USB */
+@@ -192,6 +194,8 @@ static const struct usb_device_id id_table[] = {
+ { USB_DEVICE(0x16DC, 0x0015) }, /* W-IE-NE-R Plein & Baus GmbH CML Control, Monitoring and Data Logger */
+ { USB_DEVICE(0x17A8, 0x0001) }, /* Kamstrup Optical Eye/3-wire */
+ { USB_DEVICE(0x17A8, 0x0005) }, /* Kamstrup M-Bus Master MultiPort 250D */
++ { USB_DEVICE(0x17A8, 0x0101) }, /* Kamstrup 868 MHz wM-Bus C-Mode Meter Reader (Int Ant) */
++ { USB_DEVICE(0x17A8, 0x0102) }, /* Kamstrup 868 MHz wM-Bus C-Mode Meter Reader (Ext Ant) */
+ { USB_DEVICE(0x17F4, 0xAAAA) }, /* Wavesense Jazz blood glucose meter */
+ { USB_DEVICE(0x1843, 0x0200) }, /* Vaisala USB Instrument Cable */
+ { USB_DEVICE(0x18EF, 0xE00F) }, /* ELV USB-I2C-Interface */
+@@ -1682,6 +1686,8 @@ static int cp2105_gpioconf_init(struct usb_serial *serial)
+
+ /* 2 banks of GPIO - One for the pins taken from each serial port */
+ if (intf_num == 0) {
++ priv->gc.ngpio = 2;
++
+ if (mode.eci == CP210X_PIN_MODE_MODEM) {
+ /* mark all GPIOs of this interface as reserved */
+ priv->gpio_altfunc = 0xff;
+@@ -1692,8 +1698,9 @@ static int cp2105_gpioconf_init(struct usb_serial *serial)
+ priv->gpio_pushpull = (u8)((le16_to_cpu(config.gpio_mode) &
+ CP210X_ECI_GPIO_MODE_MASK) >>
+ CP210X_ECI_GPIO_MODE_OFFSET);
+- priv->gc.ngpio = 2;
+ } else if (intf_num == 1) {
++ priv->gc.ngpio = 3;
++
+ if (mode.sci == CP210X_PIN_MODE_MODEM) {
+ /* mark all GPIOs of this interface as reserved */
+ priv->gpio_altfunc = 0xff;
+@@ -1704,7 +1711,6 @@ static int cp2105_gpioconf_init(struct usb_serial *serial)
+ priv->gpio_pushpull = (u8)((le16_to_cpu(config.gpio_mode) &
+ CP210X_SCI_GPIO_MODE_MASK) >>
+ CP210X_SCI_GPIO_MODE_OFFSET);
+- priv->gc.ngpio = 3;
+ } else {
+ return -ENODEV;
+ }
+diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
+index 99d19828dae6d..7e852561d55ca 100644
+--- a/drivers/usb/serial/ftdi_sio.c
++++ b/drivers/usb/serial/ftdi_sio.c
+@@ -969,6 +969,7 @@ static const struct usb_device_id id_table_combined[] = {
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_VX_023_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_VX_034_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_101_PID) },
++ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_159_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_1_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_2_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_3_PID) },
+@@ -977,12 +978,14 @@ static const struct usb_device_id id_table_combined[] = {
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_6_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_7_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_160_8_PID) },
++ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_235_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_257_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_279_1_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_279_2_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_279_3_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_279_4_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_313_PID) },
++ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_320_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_324_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_346_1_PID) },
+ { USB_DEVICE(BRAINBOXES_VID, BRAINBOXES_US_346_2_PID) },
+diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h
+index 755858ca20bac..d1a9564697a4b 100644
+--- a/drivers/usb/serial/ftdi_sio_ids.h
++++ b/drivers/usb/serial/ftdi_sio_ids.h
+@@ -1506,6 +1506,9 @@
+ #define BRAINBOXES_VX_023_PID 0x1003 /* VX-023 ExpressCard 1 Port RS422/485 */
+ #define BRAINBOXES_VX_034_PID 0x1004 /* VX-034 ExpressCard 2 Port RS422/485 */
+ #define BRAINBOXES_US_101_PID 0x1011 /* US-101 1xRS232 */
++#define BRAINBOXES_US_159_PID 0x1021 /* US-159 1xRS232 */
++#define BRAINBOXES_US_235_PID 0x1017 /* US-235 1xRS232 */
++#define BRAINBOXES_US_320_PID 0x1019 /* US-320 1xRS422/485 */
+ #define BRAINBOXES_US_324_PID 0x1013 /* US-324 1xRS422/485 1Mbaud */
+ #define BRAINBOXES_US_606_1_PID 0x2001 /* US-606 6 Port RS232 Serial Port 1 and 2 */
+ #define BRAINBOXES_US_606_2_PID 0x2002 /* US-606 6 Port RS232 Serial Port 3 and 4 */
+diff --git a/drivers/usb/serial/keyspan.c b/drivers/usb/serial/keyspan.c
+index 87b89c99d5177..1cfcd805f2868 100644
+--- a/drivers/usb/serial/keyspan.c
++++ b/drivers/usb/serial/keyspan.c
+@@ -2890,22 +2890,22 @@ static int keyspan_port_probe(struct usb_serial_port *port)
+ for (i = 0; i < ARRAY_SIZE(p_priv->in_buffer); ++i) {
+ p_priv->in_buffer[i] = kzalloc(IN_BUFLEN, GFP_KERNEL);
+ if (!p_priv->in_buffer[i])
+- goto err_in_buffer;
++ goto err_free_in_buffer;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(p_priv->out_buffer); ++i) {
+ p_priv->out_buffer[i] = kzalloc(OUT_BUFLEN, GFP_KERNEL);
+ if (!p_priv->out_buffer[i])
+- goto err_out_buffer;
++ goto err_free_out_buffer;
+ }
+
+ p_priv->inack_buffer = kzalloc(INACK_BUFLEN, GFP_KERNEL);
+ if (!p_priv->inack_buffer)
+- goto err_inack_buffer;
++ goto err_free_out_buffer;
+
+ p_priv->outcont_buffer = kzalloc(OUTCONT_BUFLEN, GFP_KERNEL);
+ if (!p_priv->outcont_buffer)
+- goto err_outcont_buffer;
++ goto err_free_inack_buffer;
+
+ p_priv->device_details = d_details;
+
+@@ -2951,15 +2951,14 @@ static int keyspan_port_probe(struct usb_serial_port *port)
+
+ return 0;
+
+-err_outcont_buffer:
++err_free_inack_buffer:
+ kfree(p_priv->inack_buffer);
+-err_inack_buffer:
++err_free_out_buffer:
+ for (i = 0; i < ARRAY_SIZE(p_priv->out_buffer); ++i)
+ kfree(p_priv->out_buffer[i]);
+-err_out_buffer:
++err_free_in_buffer:
+ for (i = 0; i < ARRAY_SIZE(p_priv->in_buffer); ++i)
+ kfree(p_priv->in_buffer[i]);
+-err_in_buffer:
+ kfree(p_priv);
+
+ return -ENOMEM;
+diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
+index a484ff5e4ebf8..152ad882657d7 100644
+--- a/drivers/usb/serial/option.c
++++ b/drivers/usb/serial/option.c
+@@ -198,6 +198,8 @@ static void option_instat_callback(struct urb *urb);
+
+ #define DELL_PRODUCT_5821E 0x81d7
+ #define DELL_PRODUCT_5821E_ESIM 0x81e0
++#define DELL_PRODUCT_5829E_ESIM 0x81e4
++#define DELL_PRODUCT_5829E 0x81e6
+
+ #define KYOCERA_VENDOR_ID 0x0c88
+ #define KYOCERA_PRODUCT_KPC650 0x17da
+@@ -430,6 +432,8 @@ static void option_instat_callback(struct urb *urb);
+ #define CINTERION_PRODUCT_CLS8 0x00b0
+ #define CINTERION_PRODUCT_MV31_MBIM 0x00b3
+ #define CINTERION_PRODUCT_MV31_RMNET 0x00b7
++#define CINTERION_PRODUCT_MV32_WA 0x00f1
++#define CINTERION_PRODUCT_MV32_WB 0x00f2
+
+ /* Olivetti products */
+ #define OLIVETTI_VENDOR_ID 0x0b3c
+@@ -1063,6 +1067,10 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = RSVD(0) | RSVD(1) | RSVD(6) },
+ { USB_DEVICE(DELL_VENDOR_ID, DELL_PRODUCT_5821E_ESIM),
+ .driver_info = RSVD(0) | RSVD(1) | RSVD(6) },
++ { USB_DEVICE(DELL_VENDOR_ID, DELL_PRODUCT_5829E),
++ .driver_info = RSVD(0) | RSVD(6) },
++ { USB_DEVICE(DELL_VENDOR_ID, DELL_PRODUCT_5829E_ESIM),
++ .driver_info = RSVD(0) | RSVD(6) },
+ { USB_DEVICE(ANYDATA_VENDOR_ID, ANYDATA_PRODUCT_ADU_E100A) }, /* ADU-E100, ADU-310 */
+ { USB_DEVICE(ANYDATA_VENDOR_ID, ANYDATA_PRODUCT_ADU_500A) },
+ { USB_DEVICE(ANYDATA_VENDOR_ID, ANYDATA_PRODUCT_ADU_620UW) },
+@@ -1211,6 +1219,10 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = NCTRL(0) | RSVD(1) },
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1056, 0xff), /* Telit FD980 */
+ .driver_info = NCTRL(2) | RSVD(3) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1057, 0xff), /* Telit FN980 */
++ .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1058, 0xff), /* Telit FN980 (PCIe) */
++ .driver_info = NCTRL(0) | RSVD(1) },
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1060, 0xff), /* Telit LN920 (rmnet) */
+ .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) },
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1061, 0xff), /* Telit LN920 (MBIM) */
+@@ -1219,6 +1231,16 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = NCTRL(2) | RSVD(3) },
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1063, 0xff), /* Telit LN920 (ECM) */
+ .driver_info = NCTRL(0) | RSVD(1) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1070, 0xff), /* Telit FN990 (rmnet) */
++ .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1071, 0xff), /* Telit FN990 (MBIM) */
++ .driver_info = NCTRL(0) | RSVD(1) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1072, 0xff), /* Telit FN990 (RNDIS) */
++ .driver_info = NCTRL(2) | RSVD(3) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1073, 0xff), /* Telit FN990 (ECM) */
++ .driver_info = NCTRL(0) | RSVD(1) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1075, 0xff), /* Telit FN990 (PCIe) */
++ .driver_info = RSVD(0) },
+ { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_ME910),
+ .driver_info = NCTRL(0) | RSVD(1) | RSVD(3) },
+ { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_ME910_DUAL_MODEM),
+@@ -1265,8 +1287,16 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = NCTRL(2) },
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x7011, 0xff), /* Telit LE910-S1 (ECM) */
+ .driver_info = NCTRL(2) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x701a, 0xff), /* Telit LE910R1 (RNDIS) */
++ .driver_info = NCTRL(2) },
++ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x701b, 0xff), /* Telit LE910R1 (ECM) */
++ .driver_info = NCTRL(2) },
+ { USB_DEVICE(TELIT_VENDOR_ID, 0x9010), /* Telit SBL FN980 flashing device */
+ .driver_info = NCTRL(0) | ZLP },
++ { USB_DEVICE(TELIT_VENDOR_ID, 0x9200), /* Telit LE910S1 flashing device */
++ .driver_info = NCTRL(0) | ZLP },
++ { USB_DEVICE(TELIT_VENDOR_ID, 0x9201), /* Telit LE910R1 flashing device */
++ .driver_info = NCTRL(0) | ZLP },
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, ZTE_PRODUCT_MF622, 0xff, 0xff, 0xff) }, /* ZTE WCDMA products */
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0002, 0xff, 0xff, 0xff),
+ .driver_info = RSVD(1) },
+@@ -1639,6 +1669,8 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = RSVD(2) },
+ { USB_DEVICE_INTERFACE_CLASS(ZTE_VENDOR_ID, 0x1476, 0xff) }, /* GosunCn ZTE WeLink ME3630 (ECM/NCM mode) */
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x1481, 0xff, 0x00, 0x00) }, /* ZTE MF871A */
++ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x1485, 0xff, 0xff, 0xff), /* ZTE MF286D */
++ .driver_info = RSVD(5) },
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x1533, 0xff, 0xff, 0xff) },
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x1534, 0xff, 0xff, 0xff) },
+ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x1535, 0xff, 0xff, 0xff) },
+@@ -1945,6 +1977,10 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = RSVD(3)},
+ { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV31_RMNET, 0xff),
+ .driver_info = RSVD(0)},
++ { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WA, 0xff),
++ .driver_info = RSVD(3)},
++ { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WB, 0xff),
++ .driver_info = RSVD(3)},
+ { USB_DEVICE(OLIVETTI_VENDOR_ID, OLIVETTI_PRODUCT_OLICARD100),
+ .driver_info = RSVD(4) },
+ { USB_DEVICE(OLIVETTI_VENDOR_ID, OLIVETTI_PRODUCT_OLICARD120),
+@@ -2087,13 +2123,20 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = RSVD(3) },
+ { USB_DEVICE(0x1508, 0x1001), /* Fibocom NL668 (IOT version) */
+ .driver_info = RSVD(4) | RSVD(5) | RSVD(6) },
++ { USB_DEVICE(0x1782, 0x4d10) }, /* Fibocom L610 (AT mode) */
++ { USB_DEVICE_INTERFACE_CLASS(0x1782, 0x4d11, 0xff) }, /* Fibocom L610 (ECM/RNDIS mode) */
+ { USB_DEVICE(0x2cb7, 0x0104), /* Fibocom NL678 series */
+ .driver_info = RSVD(4) | RSVD(5) },
+ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x0105, 0xff), /* Fibocom NL678 series */
+ .driver_info = RSVD(6) },
++ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x0106, 0xff) }, /* Fibocom MA510 (ECM mode w/ diag intf.) */
++ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x010a, 0xff) }, /* Fibocom MA510 (ECM mode) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2cb7, 0x010b, 0xff, 0xff, 0x30) }, /* Fibocom FG150 Diag */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2cb7, 0x010b, 0xff, 0, 0) }, /* Fibocom FG150 AT */
+ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x01a0, 0xff) }, /* Fibocom NL668-AM/NL652-EU (laptop MBIM) */
++ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x01a2, 0xff) }, /* Fibocom FM101-GL (laptop MBIM) */
++ { USB_DEVICE_INTERFACE_CLASS(0x2cb7, 0x01a4, 0xff), /* Fibocom FM101-GL (laptop MBIM) */
++ .driver_info = RSVD(4) },
+ { USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) }, /* LongSung M5710 */
+ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */
+ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */
+diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c
+index f45ca7ddf78ea..1d878d05a6584 100644
+--- a/drivers/usb/serial/pl2303.c
++++ b/drivers/usb/serial/pl2303.c
+@@ -106,6 +106,7 @@ static const struct usb_device_id id_table[] = {
+ { USB_DEVICE(HP_VENDOR_ID, HP_LCM220_PRODUCT_ID) },
+ { USB_DEVICE(HP_VENDOR_ID, HP_LCM960_PRODUCT_ID) },
+ { USB_DEVICE(HP_VENDOR_ID, HP_LM920_PRODUCT_ID) },
++ { USB_DEVICE(HP_VENDOR_ID, HP_LM930_PRODUCT_ID) },
+ { USB_DEVICE(HP_VENDOR_ID, HP_LM940_PRODUCT_ID) },
+ { USB_DEVICE(HP_VENDOR_ID, HP_TD620_PRODUCT_ID) },
+ { USB_DEVICE(CRESSI_VENDOR_ID, CRESSI_EDY_PRODUCT_ID) },
+@@ -116,6 +117,7 @@ static const struct usb_device_id id_table[] = {
+ { USB_DEVICE(ADLINK_VENDOR_ID, ADLINK_ND6530GC_PRODUCT_ID) },
+ { USB_DEVICE(SMART_VENDOR_ID, SMART_PRODUCT_ID) },
+ { USB_DEVICE(AT_VENDOR_ID, AT_VTKIT3_PRODUCT_ID) },
++ { USB_DEVICE(IBM_VENDOR_ID, IBM_PRODUCT_ID) },
+ { } /* Terminating entry */
+ };
+
+@@ -432,8 +434,10 @@ static int pl2303_detect_type(struct usb_serial *serial)
+ case 0x200:
+ switch (bcdDevice) {
+ case 0x100:
++ case 0x105:
+ case 0x305:
+ case 0x405:
++ case 0x605:
+ /*
+ * Assume it's an HXN-type if the device doesn't
+ * support the old read request value.
+diff --git a/drivers/usb/serial/pl2303.h b/drivers/usb/serial/pl2303.h
+index 6097ee8fccb25..732f9b13ad5d5 100644
+--- a/drivers/usb/serial/pl2303.h
++++ b/drivers/usb/serial/pl2303.h
+@@ -35,6 +35,9 @@
+ #define ATEN_PRODUCT_UC232B 0x2022
+ #define ATEN_PRODUCT_ID2 0x2118
+
++#define IBM_VENDOR_ID 0x04b3
++#define IBM_PRODUCT_ID 0x4016
++
+ #define IODATA_VENDOR_ID 0x04bb
+ #define IODATA_PRODUCT_ID 0x0a03
+ #define IODATA_PRODUCT_ID_RSAQ5 0x0a0e
+@@ -132,6 +135,7 @@
+ #define HP_TD620_PRODUCT_ID 0x0956
+ #define HP_LD960_PRODUCT_ID 0x0b39
+ #define HP_LD381_PRODUCT_ID 0x0f7f
++#define HP_LM930_PRODUCT_ID 0x0f9b
+ #define HP_LCM220_PRODUCT_ID 0x3139
+ #define HP_LCM960_PRODUCT_ID 0x3239
+ #define HP_LD220_PRODUCT_ID 0x3524
+diff --git a/drivers/usb/serial/qcserial.c b/drivers/usb/serial/qcserial.c
+index c18bf8164bc2e..586ef5551e76e 100644
+--- a/drivers/usb/serial/qcserial.c
++++ b/drivers/usb/serial/qcserial.c
+@@ -166,6 +166,8 @@ static const struct usb_device_id id_table[] = {
+ {DEVICE_SWI(0x1199, 0x9090)}, /* Sierra Wireless EM7565 QDL */
+ {DEVICE_SWI(0x1199, 0x9091)}, /* Sierra Wireless EM7565 */
+ {DEVICE_SWI(0x1199, 0x90d2)}, /* Sierra Wireless EM9191 QDL */
++ {DEVICE_SWI(0x1199, 0xc080)}, /* Sierra Wireless EM7590 QDL */
++ {DEVICE_SWI(0x1199, 0xc081)}, /* Sierra Wireless EM7590 */
+ {DEVICE_SWI(0x413c, 0x81a2)}, /* Dell Wireless 5806 Gobi(TM) 4G LTE Mobile Broadband Card */
+ {DEVICE_SWI(0x413c, 0x81a3)}, /* Dell Wireless 5570 HSPA+ (42Mbps) Mobile Broadband Card */
+ {DEVICE_SWI(0x413c, 0x81a4)}, /* Dell Wireless 5570e HSPA+ (42Mbps) Mobile Broadband Card */
+diff --git a/drivers/usb/serial/usb-serial-simple.c b/drivers/usb/serial/usb-serial-simple.c
+index bd23a7cb1be2b..4c6747889a194 100644
+--- a/drivers/usb/serial/usb-serial-simple.c
++++ b/drivers/usb/serial/usb-serial-simple.c
+@@ -91,6 +91,11 @@ DEVICE(moto_modem, MOTO_IDS);
+ { USB_DEVICE(0x0cad, 0x9016) } /* TPG2200 */
+ DEVICE(motorola_tetra, MOTOROLA_TETRA_IDS);
+
++/* Nokia mobile phone driver */
++#define NOKIA_IDS() \
++ { USB_DEVICE(0x0421, 0x069a) } /* Nokia 130 (RM-1035) */
++DEVICE(nokia, NOKIA_IDS);
++
+ /* Novatel Wireless GPS driver */
+ #define NOVATEL_IDS() \
+ { USB_DEVICE(0x09d7, 0x0100) } /* NovAtel FlexPack GPS */
+@@ -123,6 +128,7 @@ static struct usb_serial_driver * const serial_drivers[] = {
+ &vivopay_device,
+ &moto_modem_device,
+ &motorola_tetra_device,
++ &nokia_device,
+ &novatel_gps_device,
+ &hp4x_device,
+ &suunto_device,
+@@ -140,6 +146,7 @@ static const struct usb_device_id id_table[] = {
+ VIVOPAY_IDS(),
+ MOTO_IDS(),
+ MOTOROLA_TETRA_IDS(),
++ NOKIA_IDS(),
+ NOVATEL_IDS(),
+ HP4X_IDS(),
+ SUUNTO_IDS(),
+diff --git a/drivers/usb/serial/whiteheat.c b/drivers/usb/serial/whiteheat.c
+index da65d14c9ed5e..06aad0d727ddc 100644
+--- a/drivers/usb/serial/whiteheat.c
++++ b/drivers/usb/serial/whiteheat.c
+@@ -584,9 +584,8 @@ static int firm_send_command(struct usb_serial_port *port, __u8 command,
+ switch (command) {
+ case WHITEHEAT_GET_DTR_RTS:
+ info = usb_get_serial_port_data(port);
+- memcpy(&info->mcr, command_info->result_buffer,
+- sizeof(struct whiteheat_dr_info));
+- break;
++ info->mcr = command_info->result_buffer[0];
++ break;
+ }
+ }
+ exit:
+diff --git a/drivers/usb/storage/ene_ub6250.c b/drivers/usb/storage/ene_ub6250.c
+index 5f7d678502be4..6012603f3630e 100644
+--- a/drivers/usb/storage/ene_ub6250.c
++++ b/drivers/usb/storage/ene_ub6250.c
+@@ -237,36 +237,33 @@ static struct us_unusual_dev ene_ub6250_unusual_dev_list[] = {
+ #define memstick_logaddr(logadr1, logadr0) ((((u16)(logadr1)) << 8) | (logadr0))
+
+
+-struct SD_STATUS {
+- u8 Insert:1;
+- u8 Ready:1;
+- u8 MediaChange:1;
+- u8 IsMMC:1;
+- u8 HiCapacity:1;
+- u8 HiSpeed:1;
+- u8 WtP:1;
+- u8 Reserved:1;
+-};
+-
+-struct MS_STATUS {
+- u8 Insert:1;
+- u8 Ready:1;
+- u8 MediaChange:1;
+- u8 IsMSPro:1;
+- u8 IsMSPHG:1;
+- u8 Reserved1:1;
+- u8 WtP:1;
+- u8 Reserved2:1;
+-};
+-
+-struct SM_STATUS {
+- u8 Insert:1;
+- u8 Ready:1;
+- u8 MediaChange:1;
+- u8 Reserved:3;
+- u8 WtP:1;
+- u8 IsMS:1;
+-};
++/* SD_STATUS bits */
++#define SD_Insert BIT(0)
++#define SD_Ready BIT(1)
++#define SD_MediaChange BIT(2)
++#define SD_IsMMC BIT(3)
++#define SD_HiCapacity BIT(4)
++#define SD_HiSpeed BIT(5)
++#define SD_WtP BIT(6)
++ /* Bit 7 reserved */
++
++/* MS_STATUS bits */
++#define MS_Insert BIT(0)
++#define MS_Ready BIT(1)
++#define MS_MediaChange BIT(2)
++#define MS_IsMSPro BIT(3)
++#define MS_IsMSPHG BIT(4)
++ /* Bit 5 reserved */
++#define MS_WtP BIT(6)
++ /* Bit 7 reserved */
++
++/* SM_STATUS bits */
++#define SM_Insert BIT(0)
++#define SM_Ready BIT(1)
++#define SM_MediaChange BIT(2)
++ /* Bits 3-5 reserved */
++#define SM_WtP BIT(6)
++#define SM_IsMS BIT(7)
+
+ struct ms_bootblock_cis {
+ u8 bCistplDEVICE[6]; /* 0 */
+@@ -437,9 +434,9 @@ struct ene_ub6250_info {
+ u8 *bbuf;
+
+ /* for 6250 code */
+- struct SD_STATUS SD_Status;
+- struct MS_STATUS MS_Status;
+- struct SM_STATUS SM_Status;
++ u8 SD_Status;
++ u8 MS_Status;
++ u8 SM_Status;
+
+ /* ----- SD Control Data ---------------- */
+ /*SD_REGISTER SD_Regs; */
+@@ -602,7 +599,7 @@ static int sd_scsi_test_unit_ready(struct us_data *us, struct scsi_cmnd *srb)
+ {
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *) us->extra;
+
+- if (info->SD_Status.Insert && info->SD_Status.Ready)
++ if ((info->SD_Status & SD_Insert) && (info->SD_Status & SD_Ready))
+ return USB_STOR_TRANSPORT_GOOD;
+ else {
+ ene_sd_init(us);
+@@ -622,7 +619,7 @@ static int sd_scsi_mode_sense(struct us_data *us, struct scsi_cmnd *srb)
+ 0x0b, 0x00, 0x80, 0x08, 0x00, 0x00,
+ 0x71, 0xc0, 0x00, 0x00, 0x02, 0x00 };
+
+- if (info->SD_Status.WtP)
++ if (info->SD_Status & SD_WtP)
+ usb_stor_set_xfer_buf(mediaWP, 12, srb);
+ else
+ usb_stor_set_xfer_buf(mediaNoWP, 12, srb);
+@@ -641,9 +638,9 @@ static int sd_scsi_read_capacity(struct us_data *us, struct scsi_cmnd *srb)
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *) us->extra;
+
+ usb_stor_dbg(us, "sd_scsi_read_capacity\n");
+- if (info->SD_Status.HiCapacity) {
++ if (info->SD_Status & SD_HiCapacity) {
+ bl_len = 0x200;
+- if (info->SD_Status.IsMMC)
++ if (info->SD_Status & SD_IsMMC)
+ bl_num = info->HC_C_SIZE-1;
+ else
+ bl_num = (info->HC_C_SIZE + 1) * 1024 - 1;
+@@ -693,7 +690,7 @@ static int sd_scsi_read(struct us_data *us, struct scsi_cmnd *srb)
+ return USB_STOR_TRANSPORT_ERROR;
+ }
+
+- if (info->SD_Status.HiCapacity)
++ if (info->SD_Status & SD_HiCapacity)
+ bnByte = bn;
+
+ /* set up the command wrapper */
+@@ -733,7 +730,7 @@ static int sd_scsi_write(struct us_data *us, struct scsi_cmnd *srb)
+ return USB_STOR_TRANSPORT_ERROR;
+ }
+
+- if (info->SD_Status.HiCapacity)
++ if (info->SD_Status & SD_HiCapacity)
+ bnByte = bn;
+
+ /* set up the command wrapper */
+@@ -1456,7 +1453,7 @@ static int ms_scsi_test_unit_ready(struct us_data *us, struct scsi_cmnd *srb)
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *)(us->extra);
+
+ /* pr_info("MS_SCSI_Test_Unit_Ready\n"); */
+- if (info->MS_Status.Insert && info->MS_Status.Ready) {
++ if ((info->MS_Status & MS_Insert) && (info->MS_Status & MS_Ready)) {
+ return USB_STOR_TRANSPORT_GOOD;
+ } else {
+ ene_ms_init(us);
+@@ -1476,7 +1473,7 @@ static int ms_scsi_mode_sense(struct us_data *us, struct scsi_cmnd *srb)
+ 0x0b, 0x00, 0x80, 0x08, 0x00, 0x00,
+ 0x71, 0xc0, 0x00, 0x00, 0x02, 0x00 };
+
+- if (info->MS_Status.WtP)
++ if (info->MS_Status & MS_WtP)
+ usb_stor_set_xfer_buf(mediaWP, 12, srb);
+ else
+ usb_stor_set_xfer_buf(mediaNoWP, 12, srb);
+@@ -1495,7 +1492,7 @@ static int ms_scsi_read_capacity(struct us_data *us, struct scsi_cmnd *srb)
+
+ usb_stor_dbg(us, "ms_scsi_read_capacity\n");
+ bl_len = 0x200;
+- if (info->MS_Status.IsMSPro)
++ if (info->MS_Status & MS_IsMSPro)
+ bl_num = info->MSP_TotalBlock - 1;
+ else
+ bl_num = info->MS_Lib.NumberOfLogBlock * info->MS_Lib.blockSize * 2 - 1;
+@@ -1650,7 +1647,7 @@ static int ms_scsi_read(struct us_data *us, struct scsi_cmnd *srb)
+ if (bn > info->bl_num)
+ return USB_STOR_TRANSPORT_ERROR;
+
+- if (info->MS_Status.IsMSPro) {
++ if (info->MS_Status & MS_IsMSPro) {
+ result = ene_load_bincode(us, MSP_RW_PATTERN);
+ if (result != USB_STOR_XFER_GOOD) {
+ usb_stor_dbg(us, "Load MPS RW pattern Fail !!\n");
+@@ -1751,7 +1748,7 @@ static int ms_scsi_write(struct us_data *us, struct scsi_cmnd *srb)
+ if (bn > info->bl_num)
+ return USB_STOR_TRANSPORT_ERROR;
+
+- if (info->MS_Status.IsMSPro) {
++ if (info->MS_Status & MS_IsMSPro) {
+ result = ene_load_bincode(us, MSP_RW_PATTERN);
+ if (result != USB_STOR_XFER_GOOD) {
+ pr_info("Load MSP RW pattern Fail !!\n");
+@@ -1859,12 +1856,12 @@ static int ene_get_card_status(struct us_data *us, u8 *buf)
+
+ tmpreg = (u16) reg4b;
+ reg4b = *(u32 *)(&buf[0x14]);
+- if (info->SD_Status.HiCapacity && !info->SD_Status.IsMMC)
++ if ((info->SD_Status & SD_HiCapacity) && !(info->SD_Status & SD_IsMMC))
+ info->HC_C_SIZE = (reg4b >> 8) & 0x3fffff;
+
+ info->SD_C_SIZE = ((tmpreg & 0x03) << 10) | (u16)(reg4b >> 22);
+ info->SD_C_SIZE_MULT = (u8)(reg4b >> 7) & 0x07;
+- if (info->SD_Status.HiCapacity && info->SD_Status.IsMMC)
++ if ((info->SD_Status & SD_HiCapacity) && (info->SD_Status & SD_IsMMC))
+ info->HC_C_SIZE = *(u32 *)(&buf[0x100]);
+
+ if (info->SD_READ_BL_LEN > SD_BLOCK_LEN) {
+@@ -2076,6 +2073,7 @@ static int ene_ms_init(struct us_data *us)
+ u16 MSP_BlockSize, MSP_UserAreaBlocks;
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *) us->extra;
+ u8 *bbuf = info->bbuf;
++ unsigned int s;
+
+ printk(KERN_INFO "transport --- ENE_MSInit\n");
+
+@@ -2100,15 +2098,16 @@ static int ene_ms_init(struct us_data *us)
+ return USB_STOR_TRANSPORT_ERROR;
+ }
+ /* the same part to test ENE */
+- info->MS_Status = *(struct MS_STATUS *) bbuf;
+-
+- if (info->MS_Status.Insert && info->MS_Status.Ready) {
+- printk(KERN_INFO "Insert = %x\n", info->MS_Status.Insert);
+- printk(KERN_INFO "Ready = %x\n", info->MS_Status.Ready);
+- printk(KERN_INFO "IsMSPro = %x\n", info->MS_Status.IsMSPro);
+- printk(KERN_INFO "IsMSPHG = %x\n", info->MS_Status.IsMSPHG);
+- printk(KERN_INFO "WtP= %x\n", info->MS_Status.WtP);
+- if (info->MS_Status.IsMSPro) {
++ info->MS_Status = bbuf[0];
++
++ s = info->MS_Status;
++ if ((s & MS_Insert) && (s & MS_Ready)) {
++ printk(KERN_INFO "Insert = %x\n", !!(s & MS_Insert));
++ printk(KERN_INFO "Ready = %x\n", !!(s & MS_Ready));
++ printk(KERN_INFO "IsMSPro = %x\n", !!(s & MS_IsMSPro));
++ printk(KERN_INFO "IsMSPHG = %x\n", !!(s & MS_IsMSPHG));
++ printk(KERN_INFO "WtP= %x\n", !!(s & MS_WtP));
++ if (s & MS_IsMSPro) {
+ MSP_BlockSize = (bbuf[6] << 8) | bbuf[7];
+ MSP_UserAreaBlocks = (bbuf[10] << 8) | bbuf[11];
+ info->MSP_TotalBlock = MSP_BlockSize * MSP_UserAreaBlocks;
+@@ -2169,17 +2168,17 @@ static int ene_sd_init(struct us_data *us)
+ return USB_STOR_TRANSPORT_ERROR;
+ }
+
+- info->SD_Status = *(struct SD_STATUS *) bbuf;
+- if (info->SD_Status.Insert && info->SD_Status.Ready) {
+- struct SD_STATUS *s = &info->SD_Status;
++ info->SD_Status = bbuf[0];
++ if ((info->SD_Status & SD_Insert) && (info->SD_Status & SD_Ready)) {
++ unsigned int s = info->SD_Status;
+
+ ene_get_card_status(us, bbuf);
+- usb_stor_dbg(us, "Insert = %x\n", s->Insert);
+- usb_stor_dbg(us, "Ready = %x\n", s->Ready);
+- usb_stor_dbg(us, "IsMMC = %x\n", s->IsMMC);
+- usb_stor_dbg(us, "HiCapacity = %x\n", s->HiCapacity);
+- usb_stor_dbg(us, "HiSpeed = %x\n", s->HiSpeed);
+- usb_stor_dbg(us, "WtP = %x\n", s->WtP);
++ usb_stor_dbg(us, "Insert = %x\n", !!(s & SD_Insert));
++ usb_stor_dbg(us, "Ready = %x\n", !!(s & SD_Ready));
++ usb_stor_dbg(us, "IsMMC = %x\n", !!(s & SD_IsMMC));
++ usb_stor_dbg(us, "HiCapacity = %x\n", !!(s & SD_HiCapacity));
++ usb_stor_dbg(us, "HiSpeed = %x\n", !!(s & SD_HiSpeed));
++ usb_stor_dbg(us, "WtP = %x\n", !!(s & SD_WtP));
+ } else {
+ usb_stor_dbg(us, "SD Card Not Ready --- %x\n", bbuf[0]);
+ return USB_STOR_TRANSPORT_ERROR;
+@@ -2201,14 +2200,14 @@ static int ene_init(struct us_data *us)
+
+ misc_reg03 = bbuf[0];
+ if (misc_reg03 & 0x01) {
+- if (!info->SD_Status.Ready) {
++ if (!(info->SD_Status & SD_Ready)) {
+ result = ene_sd_init(us);
+ if (result != USB_STOR_XFER_GOOD)
+ return USB_STOR_TRANSPORT_ERROR;
+ }
+ }
+ if (misc_reg03 & 0x02) {
+- if (!info->MS_Status.Ready) {
++ if (!(info->MS_Status & MS_Ready)) {
+ result = ene_ms_init(us);
+ if (result != USB_STOR_XFER_GOOD)
+ return USB_STOR_TRANSPORT_ERROR;
+@@ -2307,14 +2306,14 @@ static int ene_transport(struct scsi_cmnd *srb, struct us_data *us)
+
+ /*US_DEBUG(usb_stor_show_command(us, srb)); */
+ scsi_set_resid(srb, 0);
+- if (unlikely(!(info->SD_Status.Ready || info->MS_Status.Ready)))
++ if (unlikely(!(info->SD_Status & SD_Ready) || (info->MS_Status & MS_Ready)))
+ result = ene_init(us);
+ if (result == USB_STOR_XFER_GOOD) {
+ result = USB_STOR_TRANSPORT_ERROR;
+- if (info->SD_Status.Ready)
++ if (info->SD_Status & SD_Ready)
+ result = sd_scsi_irp(us, srb);
+
+- if (info->MS_Status.Ready)
++ if (info->MS_Status & MS_Ready)
+ result = ms_scsi_irp(us, srb);
+ }
+ return result;
+@@ -2378,7 +2377,6 @@ static int ene_ub6250_probe(struct usb_interface *intf,
+
+ static int ene_ub6250_resume(struct usb_interface *iface)
+ {
+- u8 tmp = 0;
+ struct us_data *us = usb_get_intfdata(iface);
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *)(us->extra);
+
+@@ -2390,17 +2388,16 @@ static int ene_ub6250_resume(struct usb_interface *iface)
+ mutex_unlock(&us->dev_mutex);
+
+ info->Power_IsResum = true;
+- /*info->SD_Status.Ready = 0; */
+- info->SD_Status = *(struct SD_STATUS *)&tmp;
+- info->MS_Status = *(struct MS_STATUS *)&tmp;
+- info->SM_Status = *(struct SM_STATUS *)&tmp;
++ /* info->SD_Status &= ~SD_Ready; */
++ info->SD_Status = 0;
++ info->MS_Status = 0;
++ info->SM_Status = 0;
+
+ return 0;
+ }
+
+ static int ene_ub6250_reset_resume(struct usb_interface *iface)
+ {
+- u8 tmp = 0;
+ struct us_data *us = usb_get_intfdata(iface);
+ struct ene_ub6250_info *info = (struct ene_ub6250_info *)(us->extra);
+
+@@ -2412,10 +2409,10 @@ static int ene_ub6250_reset_resume(struct usb_interface *iface)
+ * the device
+ */
+ info->Power_IsResum = true;
+- /*info->SD_Status.Ready = 0; */
+- info->SD_Status = *(struct SD_STATUS *)&tmp;
+- info->MS_Status = *(struct MS_STATUS *)&tmp;
+- info->SM_Status = *(struct SM_STATUS *)&tmp;
++ /* info->SD_Status &= ~SD_Ready; */
++ info->SD_Status = 0;
++ info->MS_Status = 0;
++ info->SM_Status = 0;
+
+ return 0;
+ }
+diff --git a/drivers/usb/storage/realtek_cr.c b/drivers/usb/storage/realtek_cr.c
+index 3789698d9d3c6..0c423916d7bfa 100644
+--- a/drivers/usb/storage/realtek_cr.c
++++ b/drivers/usb/storage/realtek_cr.c
+@@ -365,7 +365,7 @@ static int rts51x_read_mem(struct us_data *us, u16 addr, u8 *data, u16 len)
+
+ buf = kmalloc(len, GFP_NOIO);
+ if (buf == NULL)
+- return USB_STOR_TRANSPORT_ERROR;
++ return -ENOMEM;
+
+ usb_stor_dbg(us, "addr = 0x%x, len = %d\n", addr, len);
+
+diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h
+index c6b3fcf901805..1a05e3dcfec8a 100644
+--- a/drivers/usb/storage/unusual_devs.h
++++ b/drivers/usb/storage/unusual_devs.h
+@@ -406,6 +406,16 @@ UNUSUAL_DEV( 0x04b8, 0x0602, 0x0110, 0x0110,
+ "785EPX Storage",
+ USB_SC_SCSI, USB_PR_BULK, NULL, US_FL_SINGLE_LUN),
+
++/*
++ * Reported by James Buren <braewoods+lkml@braewoods.net>
++ * Virtual ISOs cannot be remounted if ejected while the device is locked
++ * Disable locking to mimic Windows behavior that bypasses the issue
++ */
++UNUSUAL_DEV( 0x04c5, 0x2028, 0x0001, 0x0001,
++ "iODD",
++ "2531/2541",
++ USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NOT_LOCKABLE),
++
+ /*
+ * Not sure who reported this originally but
+ * Pavel Machek <pavel@ucw.cz> reported that the extra US_FL_SINGLE_LUN
+@@ -2291,6 +2301,16 @@ UNUSUAL_DEV( 0x2027, 0xa001, 0x0000, 0x9999,
+ USB_SC_DEVICE, USB_PR_DEVICE, usb_stor_euscsi_init,
+ US_FL_SCM_MULT_TARG ),
+
++/*
++ * Reported by DocMAX <mail@vacharakis.de>
++ * and Thomas Weißschuh <linux@weissschuh.net>
++ */
++UNUSUAL_DEV( 0x2109, 0x0715, 0x9999, 0x9999,
++ "VIA Labs, Inc.",
++ "VL817 SATA Bridge",
++ USB_SC_DEVICE, USB_PR_DEVICE, NULL,
++ US_FL_IGNORE_UAS),
++
+ UNUSUAL_DEV( 0x2116, 0x0320, 0x0001, 0x0001,
+ "ST",
+ "2A",
+diff --git a/drivers/usb/typec/Kconfig b/drivers/usb/typec/Kconfig
+index a0418f23b4aae..ab480f38523aa 100644
+--- a/drivers/usb/typec/Kconfig
++++ b/drivers/usb/typec/Kconfig
+@@ -65,9 +65,9 @@ config TYPEC_HD3SS3220
+
+ config TYPEC_STUSB160X
+ tristate "STMicroelectronics STUSB160x Type-C controller driver"
+- depends on I2C
+- depends on REGMAP_I2C
+ depends on USB_ROLE_SWITCH || !USB_ROLE_SWITCH
++ depends on I2C
++ select REGMAP_I2C
+ help
+ Say Y or M here if your system has STMicroelectronics STUSB160x
+ Type-C port controller.
+diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
+index 7a2a17866a823..72f9001b07921 100644
+--- a/drivers/usb/typec/tcpm/fusb302.c
++++ b/drivers/usb/typec/tcpm/fusb302.c
+@@ -669,25 +669,27 @@ static int tcpm_set_cc(struct tcpc_dev *dev, enum typec_cc_status cc)
+ ret = fusb302_i2c_mask_write(chip, FUSB_REG_MASK,
+ FUSB_REG_MASK_BC_LVL |
+ FUSB_REG_MASK_COMP_CHNG,
+- FUSB_REG_MASK_COMP_CHNG);
++ FUSB_REG_MASK_BC_LVL);
+ if (ret < 0) {
+ fusb302_log(chip, "cannot set SRC interrupt, ret=%d",
+ ret);
+ goto done;
+ }
+ chip->intr_comp_chng = true;
++ chip->intr_bc_lvl = false;
+ break;
+ case TYPEC_CC_RD:
+ ret = fusb302_i2c_mask_write(chip, FUSB_REG_MASK,
+ FUSB_REG_MASK_BC_LVL |
+ FUSB_REG_MASK_COMP_CHNG,
+- FUSB_REG_MASK_BC_LVL);
++ FUSB_REG_MASK_COMP_CHNG);
+ if (ret < 0) {
+ fusb302_log(chip, "cannot set SRC interrupt, ret=%d",
+ ret);
+ goto done;
+ }
+ chip->intr_bc_lvl = true;
++ chip->intr_comp_chng = false;
+ break;
+ default:
+ break;
+diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c
+index c15eec9cc460a..64e248117c41a 100644
+--- a/drivers/usb/typec/tcpm/tcpci.c
++++ b/drivers/usb/typec/tcpm/tcpci.c
+@@ -75,9 +75,25 @@ static int tcpci_write16(struct tcpci *tcpci, unsigned int reg, u16 val)
+ static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc)
+ {
+ struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
++ bool vconn_pres;
++ enum typec_cc_polarity polarity = TYPEC_POLARITY_CC1;
+ unsigned int reg;
+ int ret;
+
++ ret = regmap_read(tcpci->regmap, TCPC_POWER_STATUS, &reg);
++ if (ret < 0)
++ return ret;
++
++ vconn_pres = !!(reg & TCPC_POWER_STATUS_VCONN_PRES);
++ if (vconn_pres) {
++ ret = regmap_read(tcpci->regmap, TCPC_TCPC_CTRL, &reg);
++ if (ret < 0)
++ return ret;
++
++ if (reg & TCPC_TCPC_CTRL_ORIENTATION)
++ polarity = TYPEC_POLARITY_CC2;
++ }
++
+ switch (cc) {
+ case TYPEC_CC_RA:
+ reg = (TCPC_ROLE_CTRL_CC_RA << TCPC_ROLE_CTRL_CC1_SHIFT) |
+@@ -112,6 +128,16 @@ static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc)
+ break;
+ }
+
++ if (vconn_pres) {
++ if (polarity == TYPEC_POLARITY_CC2) {
++ reg &= ~(TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT);
++ reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC1_SHIFT);
++ } else {
++ reg &= ~(TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT);
++ reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC2_SHIFT);
++ }
++ }
++
+ ret = regmap_write(tcpci->regmap, TCPC_ROLE_CTRL, reg);
+ if (ret < 0)
+ return ret;
+@@ -851,7 +877,7 @@ static int tcpci_remove(struct i2c_client *client)
+ /* Disable chip interrupts before unregistering port */
+ err = tcpci_write16(chip->tcpci, TCPC_ALERT_MASK, 0);
+ if (err < 0)
+- return err;
++ dev_warn(&client->dev, "Failed to disable irqs (%pe)\n", ERR_PTR(err));
+
+ tcpci_unregister_port(chip->tcpci);
+
+diff --git a/drivers/usb/typec/tcpm/tcpci.h b/drivers/usb/typec/tcpm/tcpci.h
+index 2be7a77d400ef..b2edd45f13c68 100644
+--- a/drivers/usb/typec/tcpm/tcpci.h
++++ b/drivers/usb/typec/tcpm/tcpci.h
+@@ -98,6 +98,7 @@
+ #define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4)
+ #define TCPC_POWER_STATUS_VBUS_DET BIT(3)
+ #define TCPC_POWER_STATUS_VBUS_PRES BIT(2)
++#define TCPC_POWER_STATUS_VCONN_PRES BIT(1)
+ #define TCPC_POWER_STATUS_SINKING_VBUS BIT(0)
+
+ #define TCPC_FAULT_STATUS 0x1f
+diff --git a/drivers/usb/typec/tcpm/tcpci_mt6360.c b/drivers/usb/typec/tcpm/tcpci_mt6360.c
+index f1bd9e09bc87f..8a952eaf90163 100644
+--- a/drivers/usb/typec/tcpm/tcpci_mt6360.c
++++ b/drivers/usb/typec/tcpm/tcpci_mt6360.c
+@@ -15,6 +15,9 @@
+
+ #include "tcpci.h"
+
++#define MT6360_REG_PHYCTRL1 0x80
++#define MT6360_REG_PHYCTRL3 0x82
++#define MT6360_REG_PHYCTRL7 0x86
+ #define MT6360_REG_VCONNCTRL1 0x8C
+ #define MT6360_REG_MODECTRL2 0x8F
+ #define MT6360_REG_SWRESET 0xA0
+@@ -22,6 +25,8 @@
+ #define MT6360_REG_DRPCTRL1 0xA2
+ #define MT6360_REG_DRPCTRL2 0xA3
+ #define MT6360_REG_I2CTORST 0xBF
++#define MT6360_REG_PHYCTRL11 0xCA
++#define MT6360_REG_RXCTRL1 0xCE
+ #define MT6360_REG_RXCTRL2 0xCF
+ #define MT6360_REG_CTDCTRL2 0xEC
+
+@@ -106,6 +111,27 @@ static int mt6360_tcpc_init(struct tcpci *tcpci, struct tcpci_data *tdata)
+ if (ret)
+ return ret;
+
++ /* BMC PHY */
++ ret = mt6360_tcpc_write16(regmap, MT6360_REG_PHYCTRL1, 0x3A70);
++ if (ret)
++ return ret;
++
++ ret = regmap_write(regmap, MT6360_REG_PHYCTRL3, 0x82);
++ if (ret)
++ return ret;
++
++ ret = regmap_write(regmap, MT6360_REG_PHYCTRL7, 0x36);
++ if (ret)
++ return ret;
++
++ ret = mt6360_tcpc_write16(regmap, MT6360_REG_PHYCTRL11, 0x3C60);
++ if (ret)
++ return ret;
++
++ ret = regmap_write(regmap, MT6360_REG_RXCTRL1, 0xE8);
++ if (ret)
++ return ret;
++
+ /* Set shipping mode off, AUTOIDLE on */
+ return regmap_write(regmap, MT6360_REG_MODECTRL2, 0x7A);
+ }
+diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
+index 7f2f3ff1b3911..5fce795b69c7f 100644
+--- a/drivers/usb/typec/tcpm/tcpm.c
++++ b/drivers/usb/typec/tcpm/tcpm.c
+@@ -324,6 +324,7 @@ struct tcpm_port {
+
+ bool attached;
+ bool connected;
++ bool registered;
+ bool pd_supported;
+ enum typec_port_type port_type;
+
+@@ -4110,11 +4111,7 @@ static void run_state_machine(struct tcpm_port *port)
+ tcpm_try_src(port) ? SRC_TRY
+ : SNK_ATTACHED,
+ 0);
+- else
+- /* Wait for VBUS, but not forever */
+- tcpm_set_state(port, PORT_RESET, PD_T_PS_SOURCE_ON);
+ break;
+-
+ case SRC_TRY:
+ port->try_src_count++;
+ tcpm_set_cc(port, tcpm_rp_cc(port));
+@@ -5159,7 +5156,8 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port)
+ case SNK_TRYWAIT_DEBOUNCE:
+ break;
+ case SNK_ATTACH_WAIT:
+- tcpm_set_state(port, SNK_UNATTACHED, 0);
++ case SNK_DEBOUNCED:
++ /* Do nothing, as TCPM is still waiting for vbus to reaach VSAFE5V to connect */
+ break;
+
+ case SNK_NEGOTIATE_CAPABILITIES:
+@@ -5266,6 +5264,10 @@ static void _tcpm_pd_vbus_vsafe0v(struct tcpm_port *port)
+ case PR_SWAP_SNK_SRC_SOURCE_ON:
+ /* Do nothing, vsafe0v is expected during transition */
+ break;
++ case SNK_ATTACH_WAIT:
++ case SNK_DEBOUNCED:
++ /*Do nothing, still waiting for VSAFE5V for connect */
++ break;
+ default:
+ if (port->pwr_role == TYPEC_SINK && port->auto_vbus_discharge_enabled)
+ tcpm_set_state(port, SNK_UNATTACHED, 0);
+@@ -6295,7 +6297,8 @@ static enum hrtimer_restart state_machine_timer_handler(struct hrtimer *timer)
+ {
+ struct tcpm_port *port = container_of(timer, struct tcpm_port, state_machine_timer);
+
+- kthread_queue_work(port->wq, &port->state_machine);
++ if (port->registered)
++ kthread_queue_work(port->wq, &port->state_machine);
+ return HRTIMER_NORESTART;
+ }
+
+@@ -6303,7 +6306,8 @@ static enum hrtimer_restart vdm_state_machine_timer_handler(struct hrtimer *time
+ {
+ struct tcpm_port *port = container_of(timer, struct tcpm_port, vdm_state_machine_timer);
+
+- kthread_queue_work(port->wq, &port->vdm_state_machine);
++ if (port->registered)
++ kthread_queue_work(port->wq, &port->vdm_state_machine);
+ return HRTIMER_NORESTART;
+ }
+
+@@ -6311,7 +6315,8 @@ static enum hrtimer_restart enable_frs_timer_handler(struct hrtimer *timer)
+ {
+ struct tcpm_port *port = container_of(timer, struct tcpm_port, enable_frs_timer);
+
+- kthread_queue_work(port->wq, &port->enable_frs);
++ if (port->registered)
++ kthread_queue_work(port->wq, &port->enable_frs);
+ return HRTIMER_NORESTART;
+ }
+
+@@ -6319,7 +6324,8 @@ static enum hrtimer_restart send_discover_timer_handler(struct hrtimer *timer)
+ {
+ struct tcpm_port *port = container_of(timer, struct tcpm_port, send_discover_timer);
+
+- kthread_queue_work(port->wq, &port->send_discover_work);
++ if (port->registered)
++ kthread_queue_work(port->wq, &port->send_discover_work);
+ return HRTIMER_NORESTART;
+ }
+
+@@ -6407,6 +6413,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
+ typec_port_register_altmodes(port->typec_port,
+ &tcpm_altmode_ops, port,
+ port->port_altmode, ALTMODE_DISCOVERY_MAX);
++ port->registered = true;
+
+ mutex_lock(&port->lock);
+ tcpm_init(port);
+@@ -6428,6 +6435,9 @@ void tcpm_unregister_port(struct tcpm_port *port)
+ {
+ int i;
+
++ port->registered = false;
++ kthread_destroy_worker(port->wq);
++
+ hrtimer_cancel(&port->send_discover_timer);
+ hrtimer_cancel(&port->enable_frs_timer);
+ hrtimer_cancel(&port->vdm_state_machine_timer);
+@@ -6439,7 +6449,6 @@ void tcpm_unregister_port(struct tcpm_port *port)
+ typec_unregister_port(port->typec_port);
+ usb_role_switch_put(port->role_sw);
+ tcpm_debugfs_exit(port);
+- kthread_destroy_worker(port->wq);
+ }
+ EXPORT_SYMBOL_GPL(tcpm_unregister_port);
+
+diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
+index ea4cc0a6e40cc..23a8b9b0b1fef 100644
+--- a/drivers/usb/typec/tipd/core.c
++++ b/drivers/usb/typec/tipd/core.c
+@@ -117,7 +117,7 @@ tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len)
+ u8 data[TPS_MAX_LEN + 1];
+ int ret;
+
+- if (WARN_ON(len + 1 > sizeof(data)))
++ if (len + 1 > sizeof(data))
+ return -EINVAL;
+
+ if (!tps->i2c_protocol)
+@@ -246,6 +246,10 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status)
+ typec_set_pwr_opmode(tps->port, mode);
+ typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status));
+ typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status));
++ if (TPS_STATUS_TO_UPSIDE_DOWN(status))
++ typec_set_orientation(tps->port, TYPEC_ORIENTATION_REVERSE);
++ else
++ typec_set_orientation(tps->port, TYPEC_ORIENTATION_NORMAL);
+ tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true);
+
+ tps->partner = typec_register_partner(tps->port, &desc);
+@@ -268,6 +272,7 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status)
+ typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB);
+ typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status));
+ typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status));
++ typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE);
+ tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false);
+
+ power_supply_changed(tps->psy);
+@@ -618,12 +623,12 @@ static int tps6598x_probe(struct i2c_client *client)
+
+ ret = tps6598x_read32(tps, TPS_REG_STATUS, &status);
+ if (ret < 0)
+- return ret;
++ goto err_clear_mask;
+ trace_tps6598x_status(status);
+
+ ret = tps6598x_read32(tps, TPS_REG_SYSTEM_CONF, &conf);
+ if (ret < 0)
+- return ret;
++ goto err_clear_mask;
+
+ /*
+ * This fwnode has a "compatible" property, but is never populated as a
+@@ -712,7 +717,8 @@ err_role_put:
+ usb_role_switch_put(tps->role_sw);
+ err_fwnode_put:
+ fwnode_handle_put(fwnode);
+-
++err_clear_mask:
++ tps6598x_write64(tps, TPS_REG_INT_MASK1, 0);
+ return ret;
+ }
+
+diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h
+index 003a577be2164..1f59b9fa3fad2 100644
+--- a/drivers/usb/typec/tipd/tps6598x.h
++++ b/drivers/usb/typec/tipd/tps6598x.h
+@@ -17,6 +17,7 @@
+ /* TPS_REG_STATUS bits */
+ #define TPS_STATUS_PLUG_PRESENT BIT(0)
+ #define TPS_STATUS_PLUG_UPSIDE_DOWN BIT(4)
++#define TPS_STATUS_TO_UPSIDE_DOWN(s) (!!((s) & TPS_STATUS_PLUG_UPSIDE_DOWN))
+ #define TPS_STATUS_PORTROLE BIT(5)
+ #define TPS_STATUS_TO_TYPEC_PORTROLE(s) (!!((s) & TPS_STATUS_PORTROLE))
+ #define TPS_STATUS_DATAROLE BIT(6)
+diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
+index 5ef5bd0e87cf2..8a7e2dd52ad5a 100644
+--- a/drivers/usb/typec/ucsi/ucsi.c
++++ b/drivers/usb/typec/ucsi/ucsi.c
+@@ -955,6 +955,8 @@ static int ucsi_dr_swap(struct typec_port *port, enum typec_data_role role)
+ role == TYPEC_HOST))
+ goto out_unlock;
+
++ reinit_completion(&con->complete);
++
+ command = UCSI_SET_UOR | UCSI_CONNECTOR_NUMBER(con->num);
+ command |= UCSI_SET_UOR_ROLE(role);
+ command |= UCSI_SET_UOR_ACCEPT_ROLE_SWAPS;
+@@ -962,14 +964,18 @@ static int ucsi_dr_swap(struct typec_port *port, enum typec_data_role role)
+ if (ret < 0)
+ goto out_unlock;
+
++ mutex_unlock(&con->lock);
++
+ if (!wait_for_completion_timeout(&con->complete,
+- msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS)))
+- ret = -ETIMEDOUT;
++ msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS)))
++ return -ETIMEDOUT;
++
++ return 0;
+
+ out_unlock:
+ mutex_unlock(&con->lock);
+
+- return ret < 0 ? ret : 0;
++ return ret;
+ }
+
+ static int ucsi_pr_swap(struct typec_port *port, enum typec_role role)
+@@ -991,6 +997,8 @@ static int ucsi_pr_swap(struct typec_port *port, enum typec_role role)
+ if (cur_role == role)
+ goto out_unlock;
+
++ reinit_completion(&con->complete);
++
+ command = UCSI_SET_PDR | UCSI_CONNECTOR_NUMBER(con->num);
+ command |= UCSI_SET_PDR_ROLE(role);
+ command |= UCSI_SET_PDR_ACCEPT_ROLE_SWAPS;
+@@ -998,11 +1006,13 @@ static int ucsi_pr_swap(struct typec_port *port, enum typec_role role)
+ if (ret < 0)
+ goto out_unlock;
+
++ mutex_unlock(&con->lock);
++
+ if (!wait_for_completion_timeout(&con->complete,
+- msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS))) {
+- ret = -ETIMEDOUT;
+- goto out_unlock;
+- }
++ msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS)))
++ return -ETIMEDOUT;
++
++ mutex_lock(&con->lock);
+
+ /* Something has gone wrong while swapping the role */
+ if (UCSI_CONSTAT_PWR_OPMODE(con->status.flags) !=
+diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
+index bff96d64dddff..6db7c8ddd51cd 100644
+--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
++++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
+@@ -325,7 +325,7 @@ static int ucsi_ccg_init(struct ucsi_ccg *uc)
+ if (status < 0)
+ return status;
+
+- if (!data)
++ if (!(data & DEV_INT))
+ return 0;
+
+ status = ccg_write(uc, CCGX_RAB_INTR_REG, &data, sizeof(data));
+diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
+index 2808f1ba9f7b8..7d41dfe48adee 100644
+--- a/drivers/vdpa/ifcvf/ifcvf_base.c
++++ b/drivers/vdpa/ifcvf/ifcvf_base.c
+@@ -143,8 +143,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
+ IFCVF_DBG(pdev, "hw->isr = %p\n", hw->isr);
+ break;
+ case VIRTIO_PCI_CAP_DEVICE_CFG:
+- hw->net_cfg = get_cap_addr(hw, &cap);
+- IFCVF_DBG(pdev, "hw->net_cfg = %p\n", hw->net_cfg);
++ hw->dev_cfg = get_cap_addr(hw, &cap);
++ IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg);
+ break;
+ }
+
+@@ -153,7 +153,7 @@ next:
+ }
+
+ if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+- hw->isr == NULL || hw->net_cfg == NULL) {
++ hw->isr == NULL || hw->dev_cfg == NULL) {
+ IFCVF_ERR(pdev, "Incomplete PCI capabilities\n");
+ return -EIO;
+ }
+@@ -174,7 +174,7 @@ next:
+ IFCVF_DBG(pdev,
+ "PCI capability mapping: common cfg: %p, notify base: %p\n, isr cfg: %p, device cfg: %p, multiplier: %u\n",
+ hw->common_cfg, hw->notify_base, hw->isr,
+- hw->net_cfg, hw->notify_off_multiplier);
++ hw->dev_cfg, hw->notify_off_multiplier);
+
+ return 0;
+ }
+@@ -242,33 +242,54 @@ int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features)
+ return 0;
+ }
+
+-void ifcvf_read_net_config(struct ifcvf_hw *hw, u64 offset,
++u32 ifcvf_get_config_size(struct ifcvf_hw *hw)
++{
++ struct ifcvf_adapter *adapter;
++ u32 config_size;
++
++ adapter = vf_to_adapter(hw);
++ switch (hw->dev_type) {
++ case VIRTIO_ID_NET:
++ config_size = sizeof(struct virtio_net_config);
++ break;
++ case VIRTIO_ID_BLOCK:
++ config_size = sizeof(struct virtio_blk_config);
++ break;
++ default:
++ config_size = 0;
++ IFCVF_ERR(adapter->pdev, "VIRTIO ID %u not supported\n", hw->dev_type);
++ }
++
++ return config_size;
++}
++
++void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+ void *dst, int length)
+ {
+ u8 old_gen, new_gen, *p;
+ int i;
+
+- WARN_ON(offset + length > sizeof(struct virtio_net_config));
++ WARN_ON(offset + length > hw->config_size);
+ do {
+ old_gen = ifc_ioread8(&hw->common_cfg->config_generation);
+ p = dst;
+ for (i = 0; i < length; i++)
+- *p++ = ifc_ioread8(hw->net_cfg + offset + i);
++ *p++ = ifc_ioread8(hw->dev_cfg + offset + i);
+
+ new_gen = ifc_ioread8(&hw->common_cfg->config_generation);
+ } while (old_gen != new_gen);
+ }
+
+-void ifcvf_write_net_config(struct ifcvf_hw *hw, u64 offset,
++void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset,
+ const void *src, int length)
+ {
+ const u8 *p;
+ int i;
+
+ p = src;
+- WARN_ON(offset + length > sizeof(struct virtio_net_config));
++ WARN_ON(offset + length > hw->config_size);
+ for (i = 0; i < length; i++)
+- ifc_iowrite8(*p++, hw->net_cfg + offset + i);
++ ifc_iowrite8(*p++, hw->dev_cfg + offset + i);
+ }
+
+ static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
+index 09918af3ecf82..c486873f370a8 100644
+--- a/drivers/vdpa/ifcvf/ifcvf_base.h
++++ b/drivers/vdpa/ifcvf/ifcvf_base.h
+@@ -71,12 +71,14 @@ struct ifcvf_hw {
+ u64 hw_features;
+ u32 dev_type;
+ struct virtio_pci_common_cfg __iomem *common_cfg;
+- void __iomem *net_cfg;
++ void __iomem *dev_cfg;
+ struct vring_info vring[IFCVF_MAX_QUEUES];
+ void __iomem * const *base;
+ char config_msix_name[256];
+ struct vdpa_callback config_cb;
+ unsigned int config_irq;
++ /* virtio-net or virtio-blk device config size */
++ u32 config_size;
+ };
+
+ struct ifcvf_adapter {
+@@ -105,9 +107,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+ int ifcvf_start_hw(struct ifcvf_hw *hw);
+ void ifcvf_stop_hw(struct ifcvf_hw *hw);
+ void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+-void ifcvf_read_net_config(struct ifcvf_hw *hw, u64 offset,
++void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+ void *dst, int length);
+-void ifcvf_write_net_config(struct ifcvf_hw *hw, u64 offset,
++void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset,
+ const void *src, int length);
+ u8 ifcvf_get_status(struct ifcvf_hw *hw);
+ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status);
+@@ -120,4 +122,5 @@ u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid);
+ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num);
+ struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw);
+ int ifcvf_probed_virtio_net(struct ifcvf_hw *hw);
++u32 ifcvf_get_config_size(struct ifcvf_hw *hw);
+ #endif /* _IFCVF_H_ */
+diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
+index dcd648e1f7e7e..003530b19b4ee 100644
+--- a/drivers/vdpa/ifcvf/ifcvf_main.c
++++ b/drivers/vdpa/ifcvf/ifcvf_main.c
+@@ -366,24 +366,9 @@ static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
+
+ static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
+ {
+- struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+ struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+- struct pci_dev *pdev = adapter->pdev;
+- size_t size;
+-
+- switch (vf->dev_type) {
+- case VIRTIO_ID_NET:
+- size = sizeof(struct virtio_net_config);
+- break;
+- case VIRTIO_ID_BLOCK:
+- size = sizeof(struct virtio_blk_config);
+- break;
+- default:
+- size = 0;
+- IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type);
+- }
+
+- return size;
++ return vf->config_size;
+ }
+
+ static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
+@@ -392,8 +377,7 @@ static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
+ {
+ struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+- WARN_ON(offset + len > sizeof(struct virtio_net_config));
+- ifcvf_read_net_config(vf, offset, buf, len);
++ ifcvf_read_dev_config(vf, offset, buf, len);
+ }
+
+ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
+@@ -402,8 +386,7 @@ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
+ {
+ struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+- WARN_ON(offset + len > sizeof(struct virtio_net_config));
+- ifcvf_write_net_config(vf, offset, buf, len);
++ ifcvf_write_dev_config(vf, offset, buf, len);
+ }
+
+ static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
+@@ -541,6 +524,7 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
+ vf->vring[i].irq = -EINVAL;
+
+ vf->hw_features = ifcvf_get_hw_features(vf);
++ vf->config_size = ifcvf_get_config_size(vf);
+
+ adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
+ ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);
+diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
+index 01a848adf5903..81dc3d88d3ddd 100644
+--- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
++++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
+@@ -63,7 +63,7 @@ struct mlx5_control_vq {
+ unsigned short head;
+ };
+
+-struct mlx5_ctrl_wq_ent {
++struct mlx5_vdpa_wq_ent {
+ struct work_struct work;
+ struct mlx5_vdpa_dev *mvdev;
+ };
+diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
+index bd56de7484dcb..e4258f40dcd74 100644
+--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
++++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
+@@ -161,6 +161,9 @@ struct mlx5_vdpa_net {
+ bool setup;
+ u16 mtu;
+ u32 cur_num_vqs;
++ struct notifier_block nb;
++ struct vdpa_callback config_cb;
++ struct mlx5_vdpa_wq_ent cvq_ent;
+ };
+
+ static void free_resources(struct mlx5_vdpa_net *ndev);
+@@ -873,8 +876,6 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
+ MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id);
+ MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size);
+ MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn);
+- if (MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, eth_frame_offload_type))
+- MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 1);
+
+ err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
+ if (err)
+@@ -1512,9 +1513,11 @@ static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps)
+ return 0;
+
+ clean_added:
+- for (--i; i >= cur_qps; --i)
++ for (--i; i >= 2 * cur_qps; --i)
+ teardown_vq(ndev, &ndev->vqs[i]);
+
++ ndev->cur_num_vqs = 2 * cur_qps;
++
+ return err;
+ }
+
+@@ -1529,11 +1532,27 @@ static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd)
+
+ switch (cmd) {
+ case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET:
++ /* This mq feature check aligns with pre-existing userspace
++ * implementation.
++ *
++ * Without it, an untrusted driver could fake a multiqueue config
++ * request down to a non-mq device that may cause kernel to
++ * panic due to uninitialized resources for extra vqs. Even with
++ * a well behaving guest driver, it is not expected to allow
++ * changing the number of vqs on a non-mq device.
++ */
++ if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ))
++ break;
++
+ read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq));
+ if (read != sizeof(mq))
+ break;
+
+ newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs);
++ if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
++ newqps > mlx5_vdpa_max_qps(mvdev->max_vqs))
++ break;
++
+ if (ndev->cur_num_vqs == 2 * newqps) {
+ status = VIRTIO_NET_OK;
+ break;
+@@ -1557,22 +1576,22 @@ static void mlx5_cvq_kick_handler(struct work_struct *work)
+ {
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+ struct virtio_net_ctrl_hdr ctrl;
+- struct mlx5_ctrl_wq_ent *wqent;
++ struct mlx5_vdpa_wq_ent *wqent;
+ struct mlx5_vdpa_dev *mvdev;
+ struct mlx5_control_vq *cvq;
+ struct mlx5_vdpa_net *ndev;
+ size_t read, write;
+ int err;
+
+- wqent = container_of(work, struct mlx5_ctrl_wq_ent, work);
++ wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
+ mvdev = wqent->mvdev;
+ ndev = to_mlx5_vdpa_ndev(mvdev);
+ cvq = &mvdev->cvq;
+ if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
+- goto out;
++ return;
+
+ if (!cvq->ready)
+- goto out;
++ return;
+
+ while (true) {
+ err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head,
+@@ -1606,9 +1625,10 @@ static void mlx5_cvq_kick_handler(struct work_struct *work)
+
+ if (vringh_need_notify_iotlb(&cvq->vring))
+ vringh_notify(&cvq->vring);
++
++ queue_work(mvdev->wq, &wqent->work);
++ break;
+ }
+-out:
+- kfree(wqent);
+ }
+
+ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
+@@ -1616,7 +1636,6 @@ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ struct mlx5_vdpa_virtqueue *mvq;
+- struct mlx5_ctrl_wq_ent *wqent;
+
+ if (!is_index_valid(mvdev, idx))
+ return;
+@@ -1625,13 +1644,7 @@ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
+ if (!mvdev->cvq.ready)
+ return;
+
+- wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
+- if (!wqent)
+- return;
+-
+- wqent->mvdev = mvdev;
+- INIT_WORK(&wqent->work, mlx5_cvq_kick_handler);
+- queue_work(mvdev->wq, &wqent->work);
++ queue_work(mvdev->wq, &ndev->cvq_ent.work);
+ return;
+ }
+
+@@ -1852,16 +1865,31 @@ static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev)
+ ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ);
+ ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR);
+ ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MQ);
++ ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_STATUS);
+
+ print_features(mvdev, ndev->mvdev.mlx_features, false);
+ return ndev->mvdev.mlx_features;
+ }
+
+-static int verify_min_features(struct mlx5_vdpa_dev *mvdev, u64 features)
++static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features)
+ {
++ /* Minimum features to expect */
+ if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)))
+ return -EOPNOTSUPP;
+
++ /* Double check features combination sent down by the driver.
++ * Fail invalid features due to absence of the depended feature.
++ *
++ * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit
++ * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ".
++ * By failing the invalid features sent down by untrusted drivers,
++ * we're assured the assumption made upon is_index_valid() and
++ * is_ctrl_vq_idx() will not be compromised.
++ */
++ if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) ==
++ BIT_ULL(VIRTIO_NET_F_MQ))
++ return -EINVAL;
++
+ return 0;
+ }
+
+@@ -1937,7 +1965,7 @@ static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features)
+
+ print_features(mvdev, features, true);
+
+- err = verify_min_features(mvdev, features);
++ err = verify_driver_features(mvdev, features);
+ if (err)
+ return err;
+
+@@ -1950,8 +1978,10 @@ static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features)
+
+ static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
+ {
+- /* not implemented */
+- mlx5_vdpa_warn(to_mvdev(vdev), "set config callback not supported\n");
++ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
++ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
++
++ ndev->config_cb = *cb;
+ }
+
+ #define MLX5_VDPA_MAX_VQ_ENTRIES 256
+@@ -2192,7 +2222,6 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
+ clear_vqs_ready(ndev);
+ mlx5_vdpa_destroy_mr(&ndev->mvdev);
+ ndev->mvdev.status = 0;
+- ndev->mvdev.mlx_features = 0;
+ memset(ndev->event_cbs, 0, sizeof(ndev->event_cbs));
+ ndev->mvdev.actual_features = 0;
+ ++mvdev->generation;
+@@ -2404,6 +2433,82 @@ struct mlx5_vdpa_mgmtdev {
+ struct mlx5_vdpa_net *ndev;
+ };
+
++static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
++{
++ u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
++ u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
++ int err;
++
++ MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
++ MLX5_SET(query_vport_state_in, in, op_mod, opmod);
++ MLX5_SET(query_vport_state_in, in, vport_number, vport);
++ if (vport)
++ MLX5_SET(query_vport_state_in, in, other_vport, 1);
++
++ err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
++ if (err)
++ return 0;
++
++ return MLX5_GET(query_vport_state_out, out, state);
++}
++
++static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
++{
++ if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
++ VPORT_STATE_UP)
++ return true;
++
++ return false;
++}
++
++static void update_carrier(struct work_struct *work)
++{
++ struct mlx5_vdpa_wq_ent *wqent;
++ struct mlx5_vdpa_dev *mvdev;
++ struct mlx5_vdpa_net *ndev;
++
++ wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
++ mvdev = wqent->mvdev;
++ ndev = to_mlx5_vdpa_ndev(mvdev);
++ if (get_link_state(mvdev))
++ ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
++ else
++ ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
++
++ if (ndev->config_cb.callback)
++ ndev->config_cb.callback(ndev->config_cb.private);
++
++ kfree(wqent);
++}
++
++static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
++{
++ struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
++ struct mlx5_eqe *eqe = param;
++ int ret = NOTIFY_DONE;
++ struct mlx5_vdpa_wq_ent *wqent;
++
++ if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
++ switch (eqe->sub_type) {
++ case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
++ case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
++ wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
++ if (!wqent)
++ return NOTIFY_DONE;
++
++ wqent->mvdev = &ndev->mvdev;
++ INIT_WORK(&wqent->work, update_carrier);
++ queue_work(ndev->mvdev.wq, &wqent->work);
++ ret = NOTIFY_OK;
++ break;
++ default:
++ return NOTIFY_DONE;
++ }
++ return ret;
++ }
++ return ret;
++}
++
+ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name)
+ {
+ struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
+@@ -2448,6 +2553,11 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name)
+ if (err)
+ goto err_mtu;
+
++ if (get_link_state(mvdev))
++ ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
++ else
++ ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
++
+ if (!is_zero_ether_addr(config->mac)) {
+ pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev));
+ err = mlx5_mpfs_add_mac(pfmdev, config->mac);
+@@ -2473,12 +2583,16 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name)
+ if (err)
+ goto err_mr;
+
+- mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_ctrl_wq");
++ ndev->cvq_ent.mvdev = mvdev;
++ INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler);
++ mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq");
+ if (!mvdev->wq) {
+ err = -ENOMEM;
+ goto err_res2;
+ }
+
++ ndev->nb.notifier_call = event_handler;
++ mlx5_notifier_register(mdev, &ndev->nb);
+ ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
+ mvdev->vdev.mdev = &mgtdev->mgtdev;
+ err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+@@ -2509,7 +2623,9 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *
+ {
+ struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
+ struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);
++ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+
++ mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
+ destroy_workqueue(mvdev->wq);
+ _vdpa_unregister_device(dev);
+ mgtdev->ndev = NULL;
+diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c
+index 1dc121a07a934..12bf3d16a40ff 100644
+--- a/drivers/vdpa/vdpa.c
++++ b/drivers/vdpa/vdpa.c
+@@ -353,7 +353,8 @@ static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *m
+ goto msg_err;
+
+ while (mdev->id_table[i].device) {
+- supported_classes |= BIT(mdev->id_table[i].device);
++ if (mdev->id_table[i].device <= 63)
++ supported_classes |= BIT_ULL(mdev->id_table[i].device);
+ i++;
+ }
+
+diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
+index 5f484fff8dbec..41b0cd17fcbac 100644
+--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
++++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
+@@ -591,8 +591,11 @@ static void vdpasim_free(struct vdpa_device *vdpa)
+ vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);
+ }
+
+- put_iova_domain(&vdpasim->iova);
+- iova_cache_put();
++ if (vdpa_get_dma_dev(vdpa)) {
++ put_iova_domain(&vdpasim->iova);
++ iova_cache_put();
++ }
++
+ kvfree(vdpasim->buffer);
+ if (vdpasim->iommu)
+ vhost_iotlb_free(vdpasim->iommu);
+diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
+index 1daae26088609..0678c25141973 100644
+--- a/drivers/vdpa/vdpa_user/iova_domain.c
++++ b/drivers/vdpa/vdpa_user/iova_domain.c
+@@ -302,7 +302,7 @@ vduse_domain_alloc_iova(struct iova_domain *iovad,
+ iova_len = roundup_pow_of_two(iova_len);
+ iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
+
+- return iova_pfn << shift;
++ return (dma_addr_t)iova_pfn << shift;
+ }
+
+ static void vduse_domain_free_iova(struct iova_domain *iovad,
+diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
+index 841667a896dd0..8c3de5a76282a 100644
+--- a/drivers/vdpa/vdpa_user/vduse_dev.c
++++ b/drivers/vdpa/vdpa_user/vduse_dev.c
+@@ -655,7 +655,8 @@ static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
+ {
+ struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+
+- if (len > dev->config_size - offset)
++ if (offset > dev->config_size ||
++ len > dev->config_size - offset)
+ return;
+
+ memcpy(buf, dev->config + offset, len);
+@@ -975,7 +976,8 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
+ break;
+
+ ret = -EINVAL;
+- if (config.length == 0 ||
++ if (config.offset > dev->config_size ||
++ config.length == 0 ||
+ config.length > dev->config_size - config.offset)
+ break;
+
+diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
+index 5bcd00246d2e8..dead832b4571f 100644
+--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
++++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
+@@ -513,8 +513,8 @@ static void vp_vdpa_remove(struct pci_dev *pdev)
+ {
+ struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev);
+
+- vdpa_unregister_device(&vp_vdpa->vdpa);
+ vp_modern_remove(&vp_vdpa->mdev);
++ vdpa_unregister_device(&vp_vdpa->vdpa);
+ }
+
+ static struct pci_driver vp_vdpa_driver = {
+diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
+index a03b5a99c2dac..f3916e6b16b9d 100644
+--- a/drivers/vfio/pci/vfio_pci_core.c
++++ b/drivers/vfio/pci/vfio_pci_core.c
+@@ -36,6 +36,10 @@ static bool nointxmask;
+ static bool disable_vga;
+ static bool disable_idle_d3;
+
++/* List of PF's that vfio_pci_core_sriov_configure() has been called on */
++static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);
++static LIST_HEAD(vfio_pci_sriov_pfs);
++
+ static inline bool vfio_vga_disabled(void)
+ {
+ #ifdef CONFIG_VFIO_PCI_VGA
+@@ -228,6 +232,19 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat
+ if (!ret) {
+ /* D3 might be unsupported via quirk, skip unless in D3 */
+ if (needs_save && pdev->current_state >= PCI_D3hot) {
++ /*
++ * The current PCI state will be saved locally in
++ * 'pm_save' during the D3hot transition. When the
++ * device state is changed to D0 again with the current
++ * function, then pci_store_saved_state() will restore
++ * the state and will free the memory pointed by
++ * 'pm_save'. There are few cases where the PCI power
++ * state can be changed to D0 without the involvement
++ * of the driver. For these cases, free the earlier
++ * allocated memory first before overwriting 'pm_save'
++ * to prevent the memory leak.
++ */
++ kfree(vdev->pm_save);
+ vdev->pm_save = pci_store_saved_state(pdev);
+ } else if (needs_restore) {
+ pci_load_and_free_saved_state(pdev, &vdev->pm_save);
+@@ -322,6 +339,17 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
+ /* For needs_reset */
+ lockdep_assert_held(&vdev->vdev.dev_set->lock);
+
++ /*
++ * This function can be invoked while the power state is non-D0.
++ * This function calls __pci_reset_function_locked() which internally
++ * can use pci_pm_reset() for the function reset. pci_pm_reset() will
++ * fail if the power state is non-D0. Also, for the devices which
++ * have NoSoftRst-, the reset function can cause the PCI config space
++ * reset without restoring the original state (saved locally in
++ * 'vdev->pm_save').
++ */
++ vfio_pci_set_power_state(vdev, PCI_D0);
++
+ /* Stop the device from further DMA */
+ pci_clear_master(pdev);
+
+@@ -410,47 +438,17 @@ out:
+ }
+ EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
+
+-static struct vfio_pci_core_device *get_pf_vdev(struct vfio_pci_core_device *vdev)
+-{
+- struct pci_dev *physfn = pci_physfn(vdev->pdev);
+- struct vfio_device *pf_dev;
+-
+- if (!vdev->pdev->is_virtfn)
+- return NULL;
+-
+- pf_dev = vfio_device_get_from_dev(&physfn->dev);
+- if (!pf_dev)
+- return NULL;
+-
+- if (pci_dev_driver(physfn) != pci_dev_driver(vdev->pdev)) {
+- vfio_device_put(pf_dev);
+- return NULL;
+- }
+-
+- return container_of(pf_dev, struct vfio_pci_core_device, vdev);
+-}
+-
+-static void vfio_pci_vf_token_user_add(struct vfio_pci_core_device *vdev, int val)
+-{
+- struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
+-
+- if (!pf_vdev)
+- return;
+-
+- mutex_lock(&pf_vdev->vf_token->lock);
+- pf_vdev->vf_token->users += val;
+- WARN_ON(pf_vdev->vf_token->users < 0);
+- mutex_unlock(&pf_vdev->vf_token->lock);
+-
+- vfio_device_put(&pf_vdev->vdev);
+-}
+-
+ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
+ {
+ struct vfio_pci_core_device *vdev =
+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
+
+- vfio_pci_vf_token_user_add(vdev, -1);
++ if (vdev->sriov_pf_core_dev) {
++ mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
++ WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);
++ vdev->sriov_pf_core_dev->vf_token->users--;
++ mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
++ }
+ vfio_spapr_pci_eeh_release(vdev->pdev);
+ vfio_pci_core_disable(vdev);
+
+@@ -471,7 +469,12 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
+ {
+ vfio_pci_probe_mmaps(vdev);
+ vfio_spapr_pci_eeh_open(vdev->pdev);
+- vfio_pci_vf_token_user_add(vdev, 1);
++
++ if (vdev->sriov_pf_core_dev) {
++ mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
++ vdev->sriov_pf_core_dev->vf_token->users++;
++ mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
++ }
+ }
+ EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
+
+@@ -921,6 +924,19 @@ long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
+ return -EINVAL;
+
+ vfio_pci_zap_and_down_write_memory_lock(vdev);
++
++ /*
++ * This function can be invoked while the power state is non-D0.
++ * If pci_try_reset_function() has been called while the power
++ * state is non-D0, then pci_try_reset_function() will
++ * internally set the power state to D0 without vfio driver
++ * involvement. For the devices which have NoSoftRst-, the
++ * reset function can cause the PCI config space reset without
++ * restoring the original state (saved locally in
++ * 'vdev->pm_save').
++ */
++ vfio_pci_set_power_state(vdev, PCI_D0);
++
+ ret = pci_try_reset_function(vdev->pdev);
+ up_write(&vdev->memory_lock);
+
+@@ -1566,11 +1582,8 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
+ *
+ * If the VF token is provided but unused, an error is generated.
+ */
+- if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
+- return 0; /* No VF token provided or required */
+-
+ if (vdev->pdev->is_virtfn) {
+- struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
++ struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev;
+ bool match;
+
+ if (!pf_vdev) {
+@@ -1583,7 +1596,6 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
+ }
+
+ if (!vf_token) {
+- vfio_device_put(&pf_vdev->vdev);
+ pci_info_ratelimited(vdev->pdev,
+ "VF token required to access device\n");
+ return -EACCES;
+@@ -1593,8 +1605,6 @@ static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
+ match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
+ mutex_unlock(&pf_vdev->vf_token->lock);
+
+- vfio_device_put(&pf_vdev->vdev);
+-
+ if (!match) {
+ pci_info_ratelimited(vdev->pdev,
+ "Incorrect VF token provided for device\n");
+@@ -1715,8 +1725,30 @@ static int vfio_pci_bus_notifier(struct notifier_block *nb,
+ static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
+ {
+ struct pci_dev *pdev = vdev->pdev;
++ struct vfio_pci_core_device *cur;
++ struct pci_dev *physfn;
+ int ret;
+
++ if (pdev->is_virtfn) {
++ /*
++ * If this VF was created by our vfio_pci_core_sriov_configure()
++ * then we can find the PF vfio_pci_core_device now, and due to
++ * the locking in pci_disable_sriov() it cannot change until
++ * this VF device driver is removed.
++ */
++ physfn = pci_physfn(vdev->pdev);
++ mutex_lock(&vfio_pci_sriov_pfs_mutex);
++ list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) {
++ if (cur->pdev == physfn) {
++ vdev->sriov_pf_core_dev = cur;
++ break;
++ }
++ }
++ mutex_unlock(&vfio_pci_sriov_pfs_mutex);
++ return 0;
++ }
++
++ /* Not a SRIOV PF */
+ if (!pdev->is_physfn)
+ return 0;
+
+@@ -1788,6 +1820,7 @@ void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
+ INIT_LIST_HEAD(&vdev->ioeventfds_list);
+ mutex_init(&vdev->vma_lock);
+ INIT_LIST_HEAD(&vdev->vma_list);
++ INIT_LIST_HEAD(&vdev->sriov_pfs_item);
+ init_rwsem(&vdev->memory_lock);
+ }
+ EXPORT_SYMBOL_GPL(vfio_pci_core_init_device);
+@@ -1886,7 +1919,7 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
+ {
+ struct pci_dev *pdev = vdev->pdev;
+
+- pci_disable_sriov(pdev);
++ vfio_pci_core_sriov_configure(pdev, 0);
+
+ vfio_unregister_group_dev(&vdev->vdev);
+
+@@ -1926,21 +1959,49 @@ static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
+
+ int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
+ {
++ struct vfio_pci_core_device *vdev;
+ struct vfio_device *device;
+ int ret = 0;
+
++ device_lock_assert(&pdev->dev);
++
+ device = vfio_device_get_from_dev(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
+- if (nr_virtfn == 0)
+- pci_disable_sriov(pdev);
+- else
++ vdev = container_of(device, struct vfio_pci_core_device, vdev);
++
++ if (nr_virtfn) {
++ mutex_lock(&vfio_pci_sriov_pfs_mutex);
++ /*
++ * The thread that adds the vdev to the list is the only thread
++ * that gets to call pci_enable_sriov() and we will only allow
++ * it to be called once without going through
++ * pci_disable_sriov()
++ */
++ if (!list_empty(&vdev->sriov_pfs_item)) {
++ ret = -EINVAL;
++ goto out_unlock;
++ }
++ list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs);
++ mutex_unlock(&vfio_pci_sriov_pfs_mutex);
+ ret = pci_enable_sriov(pdev, nr_virtfn);
++ if (ret)
++ goto out_del;
++ ret = nr_virtfn;
++ goto out_put;
++ }
+
+- vfio_device_put(device);
++ pci_disable_sriov(pdev);
+
+- return ret < 0 ? ret : nr_virtfn;
++out_del:
++ mutex_lock(&vfio_pci_sriov_pfs_mutex);
++ list_del_init(&vdev->sriov_pfs_item);
++out_unlock:
++ mutex_unlock(&vfio_pci_sriov_pfs_mutex);
++out_put:
++ vfio_device_put(device);
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
+
+@@ -2064,6 +2125,18 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
+ }
+ cur_mem = NULL;
+
++ /*
++ * The pci_reset_bus() will reset all the devices in the bus.
++ * The power state can be non-D0 for some of the devices in the bus.
++ * For these devices, the pci_reset_bus() will internally set
++ * the power state to D0 without vfio driver involvement.
++ * For the devices which have NoSoftRst-, the reset function can
++ * cause the PCI config space reset without restoring the original
++ * state (saved locally in 'vdev->pm_save').
++ */
++ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
++ vfio_pci_set_power_state(cur, PCI_D0);
++
+ ret = pci_reset_bus(pdev);
+
+ err_undo:
+@@ -2117,6 +2190,18 @@ static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
+ if (!pdev)
+ return false;
+
++ /*
++ * The pci_reset_bus() will reset all the devices in the bus.
++ * The power state can be non-D0 for some of the devices in the bus.
++ * For these devices, the pci_reset_bus() will internally set
++ * the power state to D0 without vfio driver involvement.
++ * For the devices which have NoSoftRst-, the reset function can
++ * cause the PCI config space reset without restoring the original
++ * state (saved locally in 'vdev->pm_save').
++ */
++ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
++ vfio_pci_set_power_state(cur, PCI_D0);
++
+ ret = pci_reset_bus(pdev);
+ if (ret)
+ return false;
+diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
+index 57d3b2cbbd8e5..82ac1569deb05 100644
+--- a/drivers/vfio/pci/vfio_pci_rdwr.c
++++ b/drivers/vfio/pci/vfio_pci_rdwr.c
+@@ -288,6 +288,7 @@ out:
+ return done;
+ }
+
++#ifdef CONFIG_VFIO_PCI_VGA
+ ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite)
+ {
+@@ -355,6 +356,7 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+
+ return done;
+ }
++#endif
+
+ static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
+ bool test_mem)
+diff --git a/drivers/vhost/iotlb.c b/drivers/vhost/iotlb.c
+index 670d56c879e50..5829cf2d0552d 100644
+--- a/drivers/vhost/iotlb.c
++++ b/drivers/vhost/iotlb.c
+@@ -57,6 +57,21 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
+ if (last < start)
+ return -EFAULT;
+
++ /* If the range being mapped is [0, ULONG_MAX], split it into two entries
++ * otherwise its size would overflow u64.
++ */
++ if (start == 0 && last == ULONG_MAX) {
++ u64 mid = last / 2;
++ int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
++ perm, opaque);
++
++ if (err)
++ return err;
++
++ addr += mid + 1;
++ start = mid + 1;
++ }
++
+ if (iotlb->limit &&
+ iotlb->nmaps == iotlb->limit &&
+ iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
+diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
+index 28ef323882fb2..297b5db474545 100644
+--- a/drivers/vhost/net.c
++++ b/drivers/vhost/net.c
+@@ -473,6 +473,7 @@ static void vhost_tx_batch(struct vhost_net *net,
+ goto signal_used;
+
+ msghdr->msg_control = &ctl;
++ msghdr->msg_controllen = sizeof(ctl);
+ err = sock->ops->sendmsg(sock, msghdr, 0);
+ if (unlikely(err < 0)) {
+ vq_err(&nvq->vq, "Fail to batch sending packets\n");
+@@ -1449,13 +1450,9 @@ err:
+ return ERR_PTR(r);
+ }
+
+-static struct ptr_ring *get_tap_ptr_ring(int fd)
++static struct ptr_ring *get_tap_ptr_ring(struct file *file)
+ {
+ struct ptr_ring *ring;
+- struct file *file = fget(fd);
+-
+- if (!file)
+- return NULL;
+ ring = tun_get_tx_ring(file);
+ if (!IS_ERR(ring))
+ goto out;
+@@ -1464,7 +1461,6 @@ static struct ptr_ring *get_tap_ptr_ring(int fd)
+ goto out;
+ ring = NULL;
+ out:
+- fput(file);
+ return ring;
+ }
+
+@@ -1551,8 +1547,12 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
+ r = vhost_net_enable_vq(n, vq);
+ if (r)
+ goto err_used;
+- if (index == VHOST_NET_VQ_RX)
+- nvq->rx_ring = get_tap_ptr_ring(fd);
++ if (index == VHOST_NET_VQ_RX) {
++ if (sock)
++ nvq->rx_ring = get_tap_ptr_ring(sock->file);
++ else
++ nvq->rx_ring = NULL;
++ }
+
+ oldubufs = nvq->ubufs;
+ nvq->ubufs = ubufs;
+diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
+index 39039e0461175..299a995326185 100644
+--- a/drivers/vhost/vdpa.c
++++ b/drivers/vhost/vdpa.c
+@@ -97,8 +97,11 @@ static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
+ return;
+
+ irq = ops->get_vq_irq(vdpa, qid);
++ if (irq < 0)
++ return;
++
+ irq_bypass_unregister_producer(&vq->call_ctx.producer);
+- if (!vq->call_ctx.ctx || irq < 0)
++ if (!vq->call_ctx.ctx)
+ return;
+
+ vq->call_ctx.producer.token = vq->call_ctx.ctx;
+@@ -197,7 +200,7 @@ static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
+ struct vdpa_device *vdpa = v->vdpa;
+ long size = vdpa->config->get_config_size(vdpa);
+
+- if (c->len == 0)
++ if (c->len == 0 || c->off > size)
+ return -EINVAL;
+
+ if (c->len > size - c->off)
+diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
+index 59edb5a1ffe28..6942472cffb0f 100644
+--- a/drivers/vhost/vhost.c
++++ b/drivers/vhost/vhost.c
+@@ -1170,6 +1170,13 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
+ goto done;
+ }
+
++ if ((msg.type == VHOST_IOTLB_UPDATE ||
++ msg.type == VHOST_IOTLB_INVALIDATE) &&
++ msg.size == 0) {
++ ret = -EINVAL;
++ goto done;
++ }
++
+ if (dev->msg_handler)
+ ret = dev->msg_handler(dev, &msg);
+ else
+diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
+index 938aefbc75ecc..dcb1585819a1d 100644
+--- a/drivers/vhost/vsock.c
++++ b/drivers/vhost/vsock.c
+@@ -554,7 +554,7 @@ static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
+ virtio_transport_free_pkt(pkt);
+
+ len += sizeof(pkt->hdr);
+- vhost_add_used(vq, head, len);
++ vhost_add_used(vq, head, 0);
+ total_len += len;
+ added = true;
+ } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
+@@ -633,16 +633,18 @@ err:
+ return ret;
+ }
+
+-static int vhost_vsock_stop(struct vhost_vsock *vsock)
++static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
+ {
+ size_t i;
+- int ret;
++ int ret = 0;
+
+ mutex_lock(&vsock->dev.mutex);
+
+- ret = vhost_dev_check_owner(&vsock->dev);
+- if (ret)
+- goto err;
++ if (check_owner) {
++ ret = vhost_dev_check_owner(&vsock->dev);
++ if (ret)
++ goto err;
++ }
+
+ for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
+ struct vhost_virtqueue *vq = &vsock->vqs[i];
+@@ -755,9 +757,15 @@ static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
+
+ /* Iterating over all connections for all CIDs to find orphans is
+ * inefficient. Room for improvement here. */
+- vsock_for_each_connected_socket(vhost_vsock_reset_orphans);
++ vsock_for_each_connected_socket(&vhost_transport.transport,
++ vhost_vsock_reset_orphans);
+
+- vhost_vsock_stop(vsock);
++ /* Don't check the owner, because we are in the release path, so we
++ * need to stop the vsock device in any case.
++ * vhost_vsock_stop() can not fail in this case, so we don't need to
++ * check the return code.
++ */
++ vhost_vsock_stop(vsock, false);
+ vhost_vsock_flush(vsock);
+ vhost_dev_stop(&vsock->dev);
+
+@@ -872,7 +880,7 @@ static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
+ if (start)
+ return vhost_vsock_start(vsock);
+ else
+- return vhost_vsock_stop(vsock);
++ return vhost_vsock_stop(vsock, true);
+ case VHOST_GET_FEATURES:
+ features = VHOST_VSOCK_FEATURES;
+ if (copy_to_user(argp, &features, sizeof(features)))
+diff --git a/drivers/video/backlight/backlight.c b/drivers/video/backlight/backlight.c
+index 537fe1b376ad7..fc990e576340b 100644
+--- a/drivers/video/backlight/backlight.c
++++ b/drivers/video/backlight/backlight.c
+@@ -688,12 +688,6 @@ static struct backlight_device *of_find_backlight(struct device *dev)
+ of_node_put(np);
+ if (!bd)
+ return ERR_PTR(-EPROBE_DEFER);
+- /*
+- * Note: gpio_backlight uses brightness as
+- * power state during probe
+- */
+- if (!bd->props.brightness)
+- bd->props.brightness = bd->props.max_brightness;
+ }
+ }
+
+diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c
+index d094299c2a485..f12c76d6e61de 100644
+--- a/drivers/video/backlight/qcom-wled.c
++++ b/drivers/video/backlight/qcom-wled.c
+@@ -231,14 +231,14 @@ struct wled {
+ static int wled3_set_brightness(struct wled *wled, u16 brightness)
+ {
+ int rc, i;
+- u8 v[2];
++ __le16 v;
+
+- v[0] = brightness & 0xff;
+- v[1] = (brightness >> 8) & 0xf;
++ v = cpu_to_le16(brightness & WLED3_SINK_REG_BRIGHT_MAX);
+
+ for (i = 0; i < wled->cfg.num_strings; ++i) {
+ rc = regmap_bulk_write(wled->regmap, wled->ctrl_addr +
+- WLED3_SINK_REG_BRIGHT(i), v, 2);
++ WLED3_SINK_REG_BRIGHT(wled->cfg.enabled_strings[i]),
++ &v, sizeof(v));
+ if (rc < 0)
+ return rc;
+ }
+@@ -250,18 +250,18 @@ static int wled4_set_brightness(struct wled *wled, u16 brightness)
+ {
+ int rc, i;
+ u16 low_limit = wled->max_brightness * 4 / 1000;
+- u8 v[2];
++ __le16 v;
+
+ /* WLED4's lower limit of operation is 0.4% */
+ if (brightness > 0 && brightness < low_limit)
+ brightness = low_limit;
+
+- v[0] = brightness & 0xff;
+- v[1] = (brightness >> 8) & 0xf;
++ v = cpu_to_le16(brightness & WLED3_SINK_REG_BRIGHT_MAX);
+
+ for (i = 0; i < wled->cfg.num_strings; ++i) {
+ rc = regmap_bulk_write(wled->regmap, wled->sink_addr +
+- WLED4_SINK_REG_BRIGHT(i), v, 2);
++ WLED4_SINK_REG_BRIGHT(wled->cfg.enabled_strings[i]),
++ &v, sizeof(v));
+ if (rc < 0)
+ return rc;
+ }
+@@ -273,21 +273,20 @@ static int wled5_set_brightness(struct wled *wled, u16 brightness)
+ {
+ int rc, offset;
+ u16 low_limit = wled->max_brightness * 1 / 1000;
+- u8 v[2];
++ __le16 v;
+
+ /* WLED5's lower limit is 0.1% */
+ if (brightness < low_limit)
+ brightness = low_limit;
+
+- v[0] = brightness & 0xff;
+- v[1] = (brightness >> 8) & 0x7f;
++ v = cpu_to_le16(brightness & WLED5_SINK_REG_BRIGHT_MAX_15B);
+
+ offset = (wled->cfg.mod_sel == MOD_A) ?
+ WLED5_SINK_REG_MOD_A_BRIGHTNESS_LSB :
+ WLED5_SINK_REG_MOD_B_BRIGHTNESS_LSB;
+
+ rc = regmap_bulk_write(wled->regmap, wled->sink_addr + offset,
+- v, 2);
++ &v, sizeof(v));
+ return rc;
+ }
+
+@@ -572,7 +571,7 @@ unlock_mutex:
+
+ static void wled_auto_string_detection(struct wled *wled)
+ {
+- int rc = 0, i, delay_time_us;
++ int rc = 0, i, j, delay_time_us;
+ u32 sink_config = 0;
+ u8 sink_test = 0, sink_valid = 0, val;
+ bool fault_set;
+@@ -619,14 +618,15 @@ static void wled_auto_string_detection(struct wled *wled)
+
+ /* Iterate through the strings one by one */
+ for (i = 0; i < wled->cfg.num_strings; i++) {
+- sink_test = BIT((WLED4_SINK_REG_CURR_SINK_SHFT + i));
++ j = wled->cfg.enabled_strings[i];
++ sink_test = BIT((WLED4_SINK_REG_CURR_SINK_SHFT + j));
+
+ /* Enable feedback control */
+ rc = regmap_write(wled->regmap, wled->ctrl_addr +
+- WLED3_CTRL_REG_FEEDBACK_CONTROL, i + 1);
++ WLED3_CTRL_REG_FEEDBACK_CONTROL, j + 1);
+ if (rc < 0) {
+ dev_err(wled->dev, "Failed to enable feedback for SINK %d rc = %d\n",
+- i + 1, rc);
++ j + 1, rc);
+ goto failed_detect;
+ }
+
+@@ -635,7 +635,7 @@ static void wled_auto_string_detection(struct wled *wled)
+ WLED4_SINK_REG_CURR_SINK, sink_test);
+ if (rc < 0) {
+ dev_err(wled->dev, "Failed to configure SINK %d rc=%d\n",
+- i + 1, rc);
++ j + 1, rc);
+ goto failed_detect;
+ }
+
+@@ -662,7 +662,7 @@ static void wled_auto_string_detection(struct wled *wled)
+
+ if (fault_set)
+ dev_dbg(wled->dev, "WLED OVP fault detected with SINK %d\n",
+- i + 1);
++ j + 1);
+ else
+ sink_valid |= sink_test;
+
+@@ -702,15 +702,16 @@ static void wled_auto_string_detection(struct wled *wled)
+ /* Enable valid sinks */
+ if (wled->version == 4) {
+ for (i = 0; i < wled->cfg.num_strings; i++) {
++ j = wled->cfg.enabled_strings[i];
+ if (sink_config &
+- BIT(WLED4_SINK_REG_CURR_SINK_SHFT + i))
++ BIT(WLED4_SINK_REG_CURR_SINK_SHFT + j))
+ val = WLED4_SINK_REG_STR_MOD_MASK;
+ else
+ /* Disable modulator_en for unused sink */
+ val = 0;
+
+ rc = regmap_write(wled->regmap, wled->sink_addr +
+- WLED4_SINK_REG_STR_MOD_EN(i), val);
++ WLED4_SINK_REG_STR_MOD_EN(j), val);
+ if (rc < 0) {
+ dev_err(wled->dev, "Failed to configure MODULATOR_EN rc=%d\n",
+ rc);
+@@ -1256,21 +1257,6 @@ static const struct wled_var_cfg wled5_ovp_cfg = {
+ .size = 16,
+ };
+
+-static u32 wled3_num_strings_values_fn(u32 idx)
+-{
+- return idx + 1;
+-}
+-
+-static const struct wled_var_cfg wled3_num_strings_cfg = {
+- .fn = wled3_num_strings_values_fn,
+- .size = 3,
+-};
+-
+-static const struct wled_var_cfg wled4_num_strings_cfg = {
+- .fn = wled3_num_strings_values_fn,
+- .size = 4,
+-};
+-
+ static u32 wled3_switch_freq_values_fn(u32 idx)
+ {
+ return 19200 / (2 * (1 + idx));
+@@ -1344,11 +1330,6 @@ static int wled_configure(struct wled *wled)
+ .val_ptr = &cfg->switch_freq,
+ .cfg = &wled3_switch_freq_cfg,
+ },
+- {
+- .name = "qcom,num-strings",
+- .val_ptr = &cfg->num_strings,
+- .cfg = &wled3_num_strings_cfg,
+- },
+ };
+
+ const struct wled_u32_opts wled4_opts[] = {
+@@ -1372,11 +1353,6 @@ static int wled_configure(struct wled *wled)
+ .val_ptr = &cfg->switch_freq,
+ .cfg = &wled3_switch_freq_cfg,
+ },
+- {
+- .name = "qcom,num-strings",
+- .val_ptr = &cfg->num_strings,
+- .cfg = &wled4_num_strings_cfg,
+- },
+ };
+
+ const struct wled_u32_opts wled5_opts[] = {
+@@ -1400,11 +1376,6 @@ static int wled_configure(struct wled *wled)
+ .val_ptr = &cfg->switch_freq,
+ .cfg = &wled3_switch_freq_cfg,
+ },
+- {
+- .name = "qcom,num-strings",
+- .val_ptr = &cfg->num_strings,
+- .cfg = &wled4_num_strings_cfg,
+- },
+ {
+ .name = "qcom,modulator-sel",
+ .val_ptr = &cfg->mod_sel,
+@@ -1523,16 +1494,57 @@ static int wled_configure(struct wled *wled)
+ *bool_opts[i].val_ptr = true;
+ }
+
+- cfg->num_strings = cfg->num_strings + 1;
+-
+ string_len = of_property_count_elems_of_size(dev->of_node,
+ "qcom,enabled-strings",
+ sizeof(u32));
+- if (string_len > 0)
+- of_property_read_u32_array(dev->of_node,
++ if (string_len > 0) {
++ if (string_len > wled->max_string_count) {
++ dev_err(dev, "Cannot have more than %d strings\n",
++ wled->max_string_count);
++ return -EINVAL;
++ }
++
++ rc = of_property_read_u32_array(dev->of_node,
+ "qcom,enabled-strings",
+ wled->cfg.enabled_strings,
+- sizeof(u32));
++ string_len);
++ if (rc) {
++ dev_err(dev, "Failed to read %d elements from qcom,enabled-strings: %d\n",
++ string_len, rc);
++ return rc;
++ }
++
++ for (i = 0; i < string_len; ++i) {
++ if (wled->cfg.enabled_strings[i] >= wled->max_string_count) {
++ dev_err(dev,
++ "qcom,enabled-strings index %d at %d is out of bounds\n",
++ wled->cfg.enabled_strings[i], i);
++ return -EINVAL;
++ }
++ }
++
++ cfg->num_strings = string_len;
++ }
++
++ rc = of_property_read_u32(dev->of_node, "qcom,num-strings", &val);
++ if (!rc) {
++ if (val < 1 || val > wled->max_string_count) {
++ dev_err(dev, "qcom,num-strings must be between 1 and %d\n",
++ wled->max_string_count);
++ return -EINVAL;
++ }
++
++ if (string_len > 0) {
++ dev_warn(dev, "Only one of qcom,num-strings or qcom,enabled-strings"
++ " should be set\n");
++ if (val > string_len) {
++ dev_err(dev, "qcom,num-strings exceeds qcom,enabled-strings\n");
++ return -EINVAL;
++ }
++ }
++
++ cfg->num_strings = val;
++ }
+
+ return 0;
+ }
+diff --git a/drivers/video/console/Kconfig b/drivers/video/console/Kconfig
+index 840d9813b0bc6..fcc46380e7c91 100644
+--- a/drivers/video/console/Kconfig
++++ b/drivers/video/console/Kconfig
+@@ -78,6 +78,26 @@ config FRAMEBUFFER_CONSOLE
+ help
+ Low-level framebuffer-based console driver.
+
++config FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
++ bool "Enable legacy fbcon hardware acceleration code"
++ depends on FRAMEBUFFER_CONSOLE
++ default y if PARISC
++ default n
++ help
++ This option enables the fbcon (framebuffer text-based) hardware
++ acceleration for graphics drivers which were written for the fbdev
++ graphics interface.
++
++ On modern machines, on mainstream machines (like x86-64) or when
++ using a modern Linux distribution those fbdev drivers usually aren't used.
++ So enabling this option wouldn't have any effect, which is why you want
++ to disable this option on such newer machines.
++
++ If you compile this kernel for older machines which still require the
++ fbdev drivers, you may want to say Y.
++
++ If unsure, select n.
++
+ config FRAMEBUFFER_CONSOLE_DETECT_PRIMARY
+ bool "Map the console to the primary display device"
+ depends on FRAMEBUFFER_CONSOLE
+diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c
+index 1b451165311c9..40496e9e9b438 100644
+--- a/drivers/video/console/sticon.c
++++ b/drivers/video/console/sticon.c
+@@ -332,13 +332,13 @@ static u8 sticon_build_attr(struct vc_data *conp, u8 color,
+ bool blink, bool underline, bool reverse,
+ bool italic)
+ {
+- u8 attr = ((color & 0x70) >> 1) | ((color & 7));
++ u8 fg = color & 7;
++ u8 bg = (color & 0x70) >> 4;
+
+- if (reverse) {
+- color = ((color >> 3) & 0x7) | ((color & 0x7) << 3);
+- }
+-
+- return attr;
++ if (reverse)
++ return (fg << 3) | bg;
++ else
++ return (bg << 3) | fg;
+ }
+
+ static void sticon_invert_region(struct vc_data *conp, u16 *p, int count)
+diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c
+index ef9c57ce09066..9a49ea6b5112f 100644
+--- a/drivers/video/console/vgacon.c
++++ b/drivers/video/console/vgacon.c
+@@ -366,11 +366,17 @@ static void vgacon_init(struct vc_data *c, int init)
+ struct uni_pagedir *p;
+
+ /*
+- * We cannot be loaded as a module, therefore init is always 1,
+- * but vgacon_init can be called more than once, and init will
+- * not be 1.
++ * We cannot be loaded as a module, therefore init will be 1
++ * if we are the default console, however if we are a fallback
++ * console, for example if fbcon has failed registration, then
++ * init will be 0, so we need to make sure our boot parameters
++ * have been copied to the console structure for vgacon_resize
++ * ultimately called by vc_resize. Any subsequent calls to
++ * vgacon_init init will have init set to 0 too.
+ */
+ c->vc_can_do_color = vga_can_do_color;
++ c->vc_scan_lines = vga_scan_lines;
++ c->vc_font.height = c->vc_cell_height = vga_video_font_height;
+
+ /* set dimensions manually if init != 0 since vc_resize() will fail */
+ if (init) {
+@@ -379,8 +385,6 @@ static void vgacon_init(struct vc_data *c, int init)
+ } else
+ vc_resize(c, vga_video_num_columns, vga_video_num_lines);
+
+- c->vc_scan_lines = vga_scan_lines;
+- c->vc_font.height = c->vc_cell_height = vga_video_font_height;
+ c->vc_complement_mask = 0x7700;
+ if (vga_512_chars)
+ c->vc_hi_font_mask = 0x0800;
+diff --git a/drivers/video/fbdev/atafb.c b/drivers/video/fbdev/atafb.c
+index e3812a8ff55a4..29e650ecfceb1 100644
+--- a/drivers/video/fbdev/atafb.c
++++ b/drivers/video/fbdev/atafb.c
+@@ -1683,9 +1683,9 @@ static int falcon_setcolreg(unsigned int regno, unsigned int red,
+ ((blue & 0xfc00) >> 8));
+ if (regno < 16) {
+ shifter_tt.color_reg[regno] =
+- (((red & 0xe000) >> 13) | ((red & 0x1000) >> 12) << 8) |
+- (((green & 0xe000) >> 13) | ((green & 0x1000) >> 12) << 4) |
+- ((blue & 0xe000) >> 13) | ((blue & 0x1000) >> 12);
++ ((((red & 0xe000) >> 13) | ((red & 0x1000) >> 12)) << 8) |
++ ((((green & 0xe000) >> 13) | ((green & 0x1000) >> 12)) << 4) |
++ ((blue & 0xe000) >> 13) | ((blue & 0x1000) >> 12);
+ ((u32 *)info->pseudo_palette)[regno] = ((red & 0xf800) |
+ ((green & 0xfc00) >> 5) |
+ ((blue & 0xf800) >> 11));
+@@ -1971,9 +1971,9 @@ static int stste_setcolreg(unsigned int regno, unsigned int red,
+ green >>= 12;
+ if (ATARIHW_PRESENT(EXTD_SHIFTER))
+ shifter_tt.color_reg[regno] =
+- (((red & 0xe) >> 1) | ((red & 1) << 3) << 8) |
+- (((green & 0xe) >> 1) | ((green & 1) << 3) << 4) |
+- ((blue & 0xe) >> 1) | ((blue & 1) << 3);
++ ((((red & 0xe) >> 1) | ((red & 1) << 3)) << 8) |
++ ((((green & 0xe) >> 1) | ((green & 1) << 3)) << 4) |
++ ((blue & 0xe) >> 1) | ((blue & 1) << 3);
+ else
+ shifter_tt.color_reg[regno] =
+ ((red & 0xe) << 7) |
+diff --git a/drivers/video/fbdev/atmel_lcdfb.c b/drivers/video/fbdev/atmel_lcdfb.c
+index 355b6120dc4f0..1fc8de4ecbebf 100644
+--- a/drivers/video/fbdev/atmel_lcdfb.c
++++ b/drivers/video/fbdev/atmel_lcdfb.c
+@@ -1062,15 +1062,16 @@ static int __init atmel_lcdfb_probe(struct platform_device *pdev)
+
+ INIT_LIST_HEAD(&info->modelist);
+
+- if (pdev->dev.of_node) {
+- ret = atmel_lcdfb_of_init(sinfo);
+- if (ret)
+- goto free_info;
+- } else {
++ if (!pdev->dev.of_node) {
+ dev_err(dev, "cannot get default configuration\n");
+ goto free_info;
+ }
+
++ ret = atmel_lcdfb_of_init(sinfo);
++ if (ret)
++ goto free_info;
++
++ ret = -ENODEV;
+ if (!sinfo->config)
+ goto free_info;
+
+diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c
+index 998067b701fa0..393894af26f84 100644
+--- a/drivers/video/fbdev/chipsfb.c
++++ b/drivers/video/fbdev/chipsfb.c
+@@ -331,7 +331,7 @@ static const struct fb_var_screeninfo chipsfb_var = {
+
+ static void init_chips(struct fb_info *p, unsigned long addr)
+ {
+- memset(p->screen_base, 0, 0x100000);
++ fb_memset(p->screen_base, 0, 0x100000);
+
+ p->fix = chipsfb_fix;
+ p->fix.smem_start = addr;
+diff --git a/drivers/video/fbdev/cirrusfb.c b/drivers/video/fbdev/cirrusfb.c
+index 93802abbbc72a..3d47c347b8970 100644
+--- a/drivers/video/fbdev/cirrusfb.c
++++ b/drivers/video/fbdev/cirrusfb.c
+@@ -469,7 +469,7 @@ static int cirrusfb_check_mclk(struct fb_info *info, long freq)
+ return 0;
+ }
+
+-static int cirrusfb_check_pixclock(const struct fb_var_screeninfo *var,
++static int cirrusfb_check_pixclock(struct fb_var_screeninfo *var,
+ struct fb_info *info)
+ {
+ long freq;
+@@ -478,9 +478,7 @@ static int cirrusfb_check_pixclock(const struct fb_var_screeninfo *var,
+ unsigned maxclockidx = var->bits_per_pixel >> 3;
+
+ /* convert from ps to kHz */
+- freq = PICOS2KHZ(var->pixclock);
+-
+- dev_dbg(info->device, "desired pixclock: %ld kHz\n", freq);
++ freq = PICOS2KHZ(var->pixclock ? : 1);
+
+ maxclock = cirrusfb_board_info[cinfo->btype].maxclock[maxclockidx];
+ cinfo->multiplexing = 0;
+@@ -488,11 +486,13 @@ static int cirrusfb_check_pixclock(const struct fb_var_screeninfo *var,
+ /* If the frequency is greater than we can support, we might be able
+ * to use multiplexing for the video mode */
+ if (freq > maxclock) {
+- dev_err(info->device,
+- "Frequency greater than maxclock (%ld kHz)\n",
+- maxclock);
+- return -EINVAL;
++ var->pixclock = KHZ2PICOS(maxclock);
++
++ while ((freq = PICOS2KHZ(var->pixclock)) > maxclock)
++ var->pixclock++;
+ }
++ dev_dbg(info->device, "desired pixclock: %ld kHz\n", freq);
++
+ /*
+ * Additional constraint: 8bpp uses DAC clock doubling to allow maximum
+ * pixel clock
+diff --git a/drivers/video/fbdev/controlfb.c b/drivers/video/fbdev/controlfb.c
+index 509311471d515..bd59e7b11ed53 100644
+--- a/drivers/video/fbdev/controlfb.c
++++ b/drivers/video/fbdev/controlfb.c
+@@ -67,7 +67,9 @@
+ #define out_8(addr, val) (void)(val)
+ #define in_le32(addr) 0
+ #define out_le32(addr, val) (void)(val)
++#ifndef pgprot_cached_wthru
+ #define pgprot_cached_wthru(prot) (prot)
++#endif
+ #else
+ static void invalid_vram_cache(void __force *addr)
+ {
+diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
+index 22bb3892f6bd1..a53c1f6906f09 100644
+--- a/drivers/video/fbdev/core/fbcon.c
++++ b/drivers/video/fbdev/core/fbcon.c
+@@ -1135,13 +1135,13 @@ static void fbcon_init(struct vc_data *vc, int init)
+
+ ops->graphics = 0;
+
+- /*
+- * No more hw acceleration for fbcon.
+- *
+- * FIXME: Garbage collect all the now dead code after sufficient time
+- * has passed.
+- */
+- p->scrollmode = SCROLL_REDRAW;
++#ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
++ if ((info->flags & FBINFO_HWACCEL_COPYAREA) &&
++ !(info->flags & FBINFO_HWACCEL_DISABLED))
++ p->scrollmode = SCROLL_MOVE;
++ else /* default to something safe */
++ p->scrollmode = SCROLL_REDRAW;
++#endif
+
+ /*
+ * ++guenther: console.c:vc_allocate() relies on initializing
+@@ -1706,7 +1706,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
+ count = vc->vc_rows;
+ if (logo_shown >= 0)
+ goto redraw_up;
+- switch (p->scrollmode) {
++ switch (fb_scrollmode(p)) {
+ case SCROLL_MOVE:
+ fbcon_redraw_blit(vc, info, p, t, b - t - count,
+ count);
+@@ -1796,7 +1796,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
+ count = vc->vc_rows;
+ if (logo_shown >= 0)
+ goto redraw_down;
+- switch (p->scrollmode) {
++ switch (fb_scrollmode(p)) {
+ case SCROLL_MOVE:
+ fbcon_redraw_blit(vc, info, p, b - 1, b - t - count,
+ -count);
+@@ -1947,6 +1947,48 @@ static void fbcon_bmove_rec(struct vc_data *vc, struct fbcon_display *p, int sy,
+ height, width);
+ }
+
++static void updatescrollmode_accel(struct fbcon_display *p,
++ struct fb_info *info,
++ struct vc_data *vc)
++{
++#ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
++ struct fbcon_ops *ops = info->fbcon_par;
++ int cap = info->flags;
++ u16 t = 0;
++ int ypan = FBCON_SWAP(ops->rotate, info->fix.ypanstep,
++ info->fix.xpanstep);
++ int ywrap = FBCON_SWAP(ops->rotate, info->fix.ywrapstep, t);
++ int yres = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ int vyres = FBCON_SWAP(ops->rotate, info->var.yres_virtual,
++ info->var.xres_virtual);
++ int good_pan = (cap & FBINFO_HWACCEL_YPAN) &&
++ divides(ypan, vc->vc_font.height) && vyres > yres;
++ int good_wrap = (cap & FBINFO_HWACCEL_YWRAP) &&
++ divides(ywrap, vc->vc_font.height) &&
++ divides(vc->vc_font.height, vyres) &&
++ divides(vc->vc_font.height, yres);
++ int reading_fast = cap & FBINFO_READS_FAST;
++ int fast_copyarea = (cap & FBINFO_HWACCEL_COPYAREA) &&
++ !(cap & FBINFO_HWACCEL_DISABLED);
++ int fast_imageblit = (cap & FBINFO_HWACCEL_IMAGEBLIT) &&
++ !(cap & FBINFO_HWACCEL_DISABLED);
++
++ if (good_wrap || good_pan) {
++ if (reading_fast || fast_copyarea)
++ p->scrollmode = good_wrap ?
++ SCROLL_WRAP_MOVE : SCROLL_PAN_MOVE;
++ else
++ p->scrollmode = good_wrap ? SCROLL_REDRAW :
++ SCROLL_PAN_REDRAW;
++ } else {
++ if (reading_fast || (fast_copyarea && !fast_imageblit))
++ p->scrollmode = SCROLL_MOVE;
++ else
++ p->scrollmode = SCROLL_REDRAW;
++ }
++#endif
++}
++
+ static void updatescrollmode(struct fbcon_display *p,
+ struct fb_info *info,
+ struct vc_data *vc)
+@@ -1962,6 +2004,9 @@ static void updatescrollmode(struct fbcon_display *p,
+ p->vrows -= (yres - (fh * vc->vc_rows)) / fh;
+ if ((yres % fh) && (vyres % fh < yres % fh))
+ p->vrows--;
++
++ /* update scrollmode in case hardware acceleration is used */
++ updatescrollmode_accel(p, info, vc);
+ }
+
+ #define PITCH(w) (((w) + 7) >> 3)
+@@ -2119,7 +2164,7 @@ static int fbcon_switch(struct vc_data *vc)
+
+ updatescrollmode(p, info, vc);
+
+- switch (p->scrollmode) {
++ switch (fb_scrollmode(p)) {
+ case SCROLL_WRAP_MOVE:
+ scrollback_phys_max = p->vrows - vc->vc_rows;
+ break;
+diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h
+index 9315b360c8981..0f16cbc99e6a4 100644
+--- a/drivers/video/fbdev/core/fbcon.h
++++ b/drivers/video/fbdev/core/fbcon.h
+@@ -29,7 +29,9 @@ struct fbcon_display {
+ /* Filled in by the low-level console driver */
+ const u_char *fontdata;
+ int userfont; /* != 0 if fontdata kmalloc()ed */
+- u_short scrollmode; /* Scroll Method */
++#ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
++ u_short scrollmode; /* Scroll Method, use fb_scrollmode() */
++#endif
+ u_short inverse; /* != 0 text black on white as default */
+ short yscroll; /* Hardware scrolling */
+ int vrows; /* number of virtual rows */
+@@ -208,6 +210,17 @@ static inline int attr_col_ec(int shift, struct vc_data *vc,
+ #define SCROLL_REDRAW 0x004
+ #define SCROLL_PAN_REDRAW 0x005
+
++static inline u_short fb_scrollmode(struct fbcon_display *fb)
++{
++#ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
++ return fb->scrollmode;
++#else
++ /* hardcoded to SCROLL_REDRAW if acceleration was disabled. */
++ return SCROLL_REDRAW;
++#endif
++}
++
++
+ #ifdef CONFIG_FB_TILEBLITTING
+ extern void fbcon_set_tileops(struct vc_data *vc, struct fb_info *info);
+ #endif
+diff --git a/drivers/video/fbdev/core/fbcon_ccw.c b/drivers/video/fbdev/core/fbcon_ccw.c
+index 9cd2c4b05c328..2789ace796342 100644
+--- a/drivers/video/fbdev/core/fbcon_ccw.c
++++ b/drivers/video/fbdev/core/fbcon_ccw.c
+@@ -65,7 +65,7 @@ static void ccw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
+
+ area.sx = sy * vc->vc_font.height;
+ area.sy = vyres - ((sx + width) * vc->vc_font.width);
+@@ -83,7 +83,7 @@ static void ccw_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_fillrect region;
+ int bgshift = (vc->vc_hi_font_mask) ? 13 : 12;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
+
+ region.color = attr_bgcol_ec(bgshift,vc,info);
+ region.dx = sy * vc->vc_font.height;
+@@ -140,7 +140,7 @@ static void ccw_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -229,7 +229,7 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, int mode,
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -387,7 +387,7 @@ static int ccw_update_start(struct fb_info *info)
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+ u32 yoffset;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
+ int err;
+
+ yoffset = (vyres - info->var.yres) - ops->var.xoffset;
+diff --git a/drivers/video/fbdev/core/fbcon_cw.c b/drivers/video/fbdev/core/fbcon_cw.c
+index 88d89fad3f05e..86a254c1b2b7b 100644
+--- a/drivers/video/fbdev/core/fbcon_cw.c
++++ b/drivers/video/fbdev/core/fbcon_cw.c
+@@ -50,7 +50,7 @@ static void cw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ area.sx = vxres - ((sy + height) * vc->vc_font.height);
+ area.sy = sx * vc->vc_font.width;
+@@ -68,7 +68,7 @@ static void cw_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_fillrect region;
+ int bgshift = (vc->vc_hi_font_mask) ? 13 : 12;
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ region.color = attr_bgcol_ec(bgshift,vc,info);
+ region.dx = vxres - ((sy + height) * vc->vc_font.height);
+@@ -125,7 +125,7 @@ static void cw_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -212,7 +212,7 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, int mode,
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -369,7 +369,7 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, int mode,
+ static int cw_update_start(struct fb_info *info)
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vxres = GETVXRES(ops->p, info);
+ u32 xoffset;
+ int err;
+
+diff --git a/drivers/video/fbdev/core/fbcon_rotate.h b/drivers/video/fbdev/core/fbcon_rotate.h
+index e233444cda664..01cbe303b8a29 100644
+--- a/drivers/video/fbdev/core/fbcon_rotate.h
++++ b/drivers/video/fbdev/core/fbcon_rotate.h
+@@ -12,11 +12,11 @@
+ #define _FBCON_ROTATE_H
+
+ #define GETVYRES(s,i) ({ \
+- (s == SCROLL_REDRAW || s == SCROLL_MOVE) ? \
++ (fb_scrollmode(s) == SCROLL_REDRAW || fb_scrollmode(s) == SCROLL_MOVE) ? \
+ (i)->var.yres : (i)->var.yres_virtual; })
+
+ #define GETVXRES(s,i) ({ \
+- (s == SCROLL_REDRAW || s == SCROLL_MOVE || !(i)->fix.xpanstep) ? \
++ (fb_scrollmode(s) == SCROLL_REDRAW || fb_scrollmode(s) == SCROLL_MOVE || !(i)->fix.xpanstep) ? \
+ (i)->var.xres : (i)->var.xres_virtual; })
+
+
+diff --git a/drivers/video/fbdev/core/fbcon_ud.c b/drivers/video/fbdev/core/fbcon_ud.c
+index 8d5e66b1bdfbb..23bc045769d08 100644
+--- a/drivers/video/fbdev/core/fbcon_ud.c
++++ b/drivers/video/fbdev/core/fbcon_ud.c
+@@ -50,8 +50,8 @@ static void ud_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ area.sy = vyres - ((sy + height) * vc->vc_font.height);
+ area.sx = vxres - ((sx + width) * vc->vc_font.width);
+@@ -69,8 +69,8 @@ static void ud_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ struct fbcon_ops *ops = info->fbcon_par;
+ struct fb_fillrect region;
+ int bgshift = (vc->vc_hi_font_mask) ? 13 : 12;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ region.color = attr_bgcol_ec(bgshift,vc,info);
+ region.dy = vyres - ((sy + height) * vc->vc_font.height);
+@@ -162,8 +162,8 @@ static void ud_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 mod = vc->vc_font.width % 8, cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -259,8 +259,8 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, int mode,
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
++ u32 vxres = GETVXRES(ops->p, info);
+
+ if (!ops->fontbuffer)
+ return;
+@@ -410,8 +410,8 @@ static int ud_update_start(struct fb_info *info)
+ {
+ struct fbcon_ops *ops = info->fbcon_par;
+ int xoffset, yoffset;
+- u32 vyres = GETVYRES(ops->p->scrollmode, info);
+- u32 vxres = GETVXRES(ops->p->scrollmode, info);
++ u32 vyres = GETVYRES(ops->p, info);
++ u32 vxres = GETVXRES(ops->p, info);
+ int err;
+
+ xoffset = vxres - info->var.xres - ops->var.xoffset;
+diff --git a/drivers/video/fbdev/core/fbcvt.c b/drivers/video/fbdev/core/fbcvt.c
+index 55d2bd0ce5c02..64843464c6613 100644
+--- a/drivers/video/fbdev/core/fbcvt.c
++++ b/drivers/video/fbdev/core/fbcvt.c
+@@ -214,9 +214,11 @@ static u32 fb_cvt_aspect_ratio(struct fb_cvt_data *cvt)
+ static void fb_cvt_print_name(struct fb_cvt_data *cvt)
+ {
+ u32 pixcount, pixcount_mod;
+- int cnt = 255, offset = 0, read = 0;
+- u8 *buf = kzalloc(256, GFP_KERNEL);
++ int size = 256;
++ int off = 0;
++ u8 *buf;
+
++ buf = kzalloc(size, GFP_KERNEL);
+ if (!buf)
+ return;
+
+@@ -224,43 +226,30 @@ static void fb_cvt_print_name(struct fb_cvt_data *cvt)
+ pixcount_mod = (cvt->xres * (cvt->yres/cvt->interlace)) % 1000000;
+ pixcount_mod /= 1000;
+
+- read = snprintf(buf+offset, cnt, "fbcvt: %dx%d@%d: CVT Name - ",
+- cvt->xres, cvt->yres, cvt->refresh);
+- offset += read;
+- cnt -= read;
++ off += scnprintf(buf + off, size - off, "fbcvt: %dx%d@%d: CVT Name - ",
++ cvt->xres, cvt->yres, cvt->refresh);
+
+- if (cvt->status)
+- snprintf(buf+offset, cnt, "Not a CVT standard - %d.%03d Mega "
+- "Pixel Image\n", pixcount, pixcount_mod);
+- else {
+- if (pixcount) {
+- read = snprintf(buf+offset, cnt, "%d", pixcount);
+- cnt -= read;
+- offset += read;
+- }
++ if (cvt->status) {
++ off += scnprintf(buf + off, size - off,
++ "Not a CVT standard - %d.%03d Mega Pixel Image\n",
++ pixcount, pixcount_mod);
++ } else {
++ if (pixcount)
++ off += scnprintf(buf + off, size - off, "%d", pixcount);
+
+- read = snprintf(buf+offset, cnt, ".%03dM", pixcount_mod);
+- cnt -= read;
+- offset += read;
++ off += scnprintf(buf + off, size - off, ".%03dM", pixcount_mod);
+
+ if (cvt->aspect_ratio == 0)
+- read = snprintf(buf+offset, cnt, "3");
++ off += scnprintf(buf + off, size - off, "3");
+ else if (cvt->aspect_ratio == 3)
+- read = snprintf(buf+offset, cnt, "4");
++ off += scnprintf(buf + off, size - off, "4");
+ else if (cvt->aspect_ratio == 1 || cvt->aspect_ratio == 4)
+- read = snprintf(buf+offset, cnt, "9");
++ off += scnprintf(buf + off, size - off, "9");
+ else if (cvt->aspect_ratio == 2)
+- read = snprintf(buf+offset, cnt, "A");
+- else
+- read = 0;
+- cnt -= read;
+- offset += read;
+-
+- if (cvt->flags & FB_CVT_FLAG_REDUCED_BLANK) {
+- read = snprintf(buf+offset, cnt, "-R");
+- cnt -= read;
+- offset += read;
+- }
++ off += scnprintf(buf + off, size - off, "A");
++
++ if (cvt->flags & FB_CVT_FLAG_REDUCED_BLANK)
++ off += scnprintf(buf + off, size - off, "-R");
+ }
+
+ printk(KERN_INFO "%s\n", buf);
+diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
+index 7420d2c16e47e..0371ad233fdf2 100644
+--- a/drivers/video/fbdev/core/fbmem.c
++++ b/drivers/video/fbdev/core/fbmem.c
+@@ -25,6 +25,7 @@
+ #include <linux/init.h>
+ #include <linux/linux_logo.h>
+ #include <linux/proc_fs.h>
++#include <linux/platform_device.h>
+ #include <linux/seq_file.h>
+ #include <linux/console.h>
+ #include <linux/kmod.h>
+@@ -1557,18 +1558,43 @@ static void do_remove_conflicting_framebuffers(struct apertures_struct *a,
+ /* check all firmware fbs and kick off if the base addr overlaps */
+ for_each_registered_fb(i) {
+ struct apertures_struct *gen_aper;
++ struct device *device;
+
+ if (!(registered_fb[i]->flags & FBINFO_MISC_FIRMWARE))
+ continue;
+
+ gen_aper = registered_fb[i]->apertures;
++ device = registered_fb[i]->device;
+ if (fb_do_apertures_overlap(gen_aper, a) ||
+ (primary && gen_aper && gen_aper->count &&
+ gen_aper->ranges[0].base == VGA_FB_PHYS)) {
+
+ printk(KERN_INFO "fb%d: switching to %s from %s\n",
+ i, name, registered_fb[i]->fix.id);
+- do_unregister_framebuffer(registered_fb[i]);
++
++ /*
++ * If we kick-out a firmware driver, we also want to remove
++ * the underlying platform device, such as simple-framebuffer,
++ * VESA, EFI, etc. A native driver will then be able to
++ * allocate the memory range.
++ *
++ * If it's not a platform device, at least print a warning. A
++ * fix would add code to remove the device from the system.
++ */
++ if (!device) {
++ /* TODO: Represent each OF framebuffer as its own
++ * device in the device hierarchy. For now, offb
++ * doesn't have such a device, so unregister the
++ * framebuffer as before without warning.
++ */
++ do_unregister_framebuffer(registered_fb[i]);
++ } else if (dev_is_platform(device)) {
++ registered_fb[i]->forced_out = true;
++ platform_device_unregister(to_platform_device(device));
++ } else {
++ pr_warn("fb%d: cannot remove device\n", i);
++ do_unregister_framebuffer(registered_fb[i]);
++ }
+ }
+ }
+ }
+@@ -1759,6 +1785,53 @@ int remove_conflicting_framebuffers(struct apertures_struct *a,
+ }
+ EXPORT_SYMBOL(remove_conflicting_framebuffers);
+
++/**
++ * is_firmware_framebuffer - detect if firmware-configured framebuffer matches
++ * @a: memory range, users of which are to be checked
++ *
++ * This function checks framebuffer devices (initialized by firmware/bootloader)
++ * which use memory range described by @a. If @a matchesm the function returns
++ * true, otherwise false.
++ */
++bool is_firmware_framebuffer(struct apertures_struct *a)
++{
++ bool do_free = false;
++ bool found = false;
++ int i;
++
++ if (!a) {
++ a = alloc_apertures(1);
++ if (!a)
++ return false;
++
++ a->ranges[0].base = 0;
++ a->ranges[0].size = ~0;
++ do_free = true;
++ }
++
++ mutex_lock(&registration_lock);
++ /* check all firmware fbs and kick off if the base addr overlaps */
++ for_each_registered_fb(i) {
++ struct apertures_struct *gen_aper;
++
++ if (!(registered_fb[i]->flags & FBINFO_MISC_FIRMWARE))
++ continue;
++
++ gen_aper = registered_fb[i]->apertures;
++ if (fb_do_apertures_overlap(gen_aper, a)) {
++ found = true;
++ break;
++ }
++ }
++ mutex_unlock(&registration_lock);
++
++ if (do_free)
++ kfree(a);
++
++ return found;
++}
++EXPORT_SYMBOL(is_firmware_framebuffer);
++
+ /**
+ * remove_conflicting_pci_framebuffers - remove firmware-configured framebuffers for PCI devices
+ * @pdev: PCI device
+@@ -1848,9 +1921,13 @@ EXPORT_SYMBOL(register_framebuffer);
+ void
+ unregister_framebuffer(struct fb_info *fb_info)
+ {
+- mutex_lock(&registration_lock);
++ bool forced_out = fb_info->forced_out;
++
++ if (!forced_out)
++ mutex_lock(&registration_lock);
+ do_unregister_framebuffer(fb_info);
+- mutex_unlock(&registration_lock);
++ if (!forced_out)
++ mutex_unlock(&registration_lock);
+ }
+ EXPORT_SYMBOL(unregister_framebuffer);
+
+diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c
+index 65dae05fff8e6..ce699396d6bad 100644
+--- a/drivers/video/fbdev/core/fbsysfs.c
++++ b/drivers/video/fbdev/core/fbsysfs.c
+@@ -80,6 +80,10 @@ void framebuffer_release(struct fb_info *info)
+ {
+ if (!info)
+ return;
++
++ if (WARN_ON(refcount_read(&info->count)))
++ return;
++
+ kfree(info->apertures);
+ kfree(info);
+ }
+diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
+index 8ea8f079cde26..b3d5f884c5445 100644
+--- a/drivers/video/fbdev/efifb.c
++++ b/drivers/video/fbdev/efifb.c
+@@ -47,6 +47,8 @@ static bool use_bgrt = true;
+ static bool request_mem_succeeded = false;
+ static u64 mem_flags = EFI_MEMORY_WC | EFI_MEMORY_UC;
+
++static struct pci_dev *efifb_pci_dev; /* dev with BAR covering the efifb */
++
+ static struct fb_var_screeninfo efifb_defined = {
+ .activate = FB_ACTIVATE_NOW,
+ .height = -1,
+@@ -241,18 +243,28 @@ error:
+ static inline void efifb_show_boot_graphics(struct fb_info *info) {}
+ #endif
+
++/*
++ * fb_ops.fb_destroy is called by the last put_fb_info() call at the end
++ * of unregister_framebuffer() or fb_release(). Do any cleanup here.
++ */
+ static void efifb_destroy(struct fb_info *info)
+ {
++ if (efifb_pci_dev)
++ pm_runtime_put(&efifb_pci_dev->dev);
++
+ if (info->screen_base) {
+ if (mem_flags & (EFI_MEMORY_UC | EFI_MEMORY_WC))
+ iounmap(info->screen_base);
+ else
+ memunmap(info->screen_base);
+ }
++
+ if (request_mem_succeeded)
+ release_mem_region(info->apertures->ranges[0].base,
+ info->apertures->ranges[0].size);
+ fb_dealloc_cmap(&info->cmap);
++
++ framebuffer_release(info);
+ }
+
+ static const struct fb_ops efifb_ops = {
+@@ -333,7 +345,6 @@ ATTRIBUTE_GROUPS(efifb);
+
+ static bool pci_dev_disabled; /* FB base matches BAR of a disabled device */
+
+-static struct pci_dev *efifb_pci_dev; /* dev with BAR covering the efifb */
+ static struct resource *bar_resource;
+ static u64 bar_offset;
+
+@@ -347,6 +358,17 @@ static int efifb_probe(struct platform_device *dev)
+ char *option = NULL;
+ efi_memory_desc_t md;
+
++ /*
++ * Generic drivers must not be registered if a framebuffer exists.
++ * If a native driver was probed, the display hardware was already
++ * taken and attempting to use the system framebuffer is dangerous.
++ */
++ if (num_registered_fb > 0) {
++ dev_err(&dev->dev,
++ "efifb: a framebuffer is already registered\n");
++ return -EINVAL;
++ }
++
+ if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI || pci_dev_disabled)
+ return -ENODEV;
+
+@@ -569,17 +591,22 @@ static int efifb_probe(struct platform_device *dev)
+ pr_err("efifb: cannot allocate colormap\n");
+ goto err_groups;
+ }
++
++ if (efifb_pci_dev)
++ WARN_ON(pm_runtime_get_sync(&efifb_pci_dev->dev) < 0);
++
+ err = register_framebuffer(info);
+ if (err < 0) {
+ pr_err("efifb: cannot register framebuffer\n");
+- goto err_fb_dealoc;
++ goto err_put_rpm_ref;
+ }
+ fb_info(info, "%s frame buffer device\n", info->fix.id);
+- if (efifb_pci_dev)
+- pm_runtime_get_sync(&efifb_pci_dev->dev);
+ return 0;
+
+-err_fb_dealoc:
++err_put_rpm_ref:
++ if (efifb_pci_dev)
++ pm_runtime_put(&efifb_pci_dev->dev);
++
+ fb_dealloc_cmap(&info->cmap);
+ err_groups:
+ sysfs_remove_groups(&dev->dev.kobj, efifb_groups);
+@@ -600,11 +627,9 @@ static int efifb_remove(struct platform_device *pdev)
+ {
+ struct fb_info *info = platform_get_drvdata(pdev);
+
++ /* efifb_destroy takes care of info cleanup */
+ unregister_framebuffer(info);
+ sysfs_remove_groups(&pdev->dev.kobj, efifb_groups);
+- framebuffer_release(info);
+- if (efifb_pci_dev)
+- pm_runtime_put(&efifb_pci_dev->dev);
+
+ return 0;
+ }
+diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c
+index 23999df527393..c8e0ea27caf1d 100644
+--- a/drivers/video/fbdev/hyperv_fb.c
++++ b/drivers/video/fbdev/hyperv_fb.c
+@@ -287,8 +287,6 @@ struct hvfb_par {
+
+ static uint screen_width = HVFB_WIDTH;
+ static uint screen_height = HVFB_HEIGHT;
+-static uint screen_width_max = HVFB_WIDTH;
+-static uint screen_height_max = HVFB_HEIGHT;
+ static uint screen_depth;
+ static uint screen_fb_size;
+ static uint dio_fb_size; /* FB size for deferred IO */
+@@ -582,7 +580,6 @@ static int synthvid_get_supported_resolution(struct hv_device *hdev)
+ int ret = 0;
+ unsigned long t;
+ u8 index;
+- int i;
+
+ memset(msg, 0, sizeof(struct synthvid_msg));
+ msg->vid_hdr.type = SYNTHVID_RESOLUTION_REQUEST;
+@@ -613,13 +610,6 @@ static int synthvid_get_supported_resolution(struct hv_device *hdev)
+ goto out;
+ }
+
+- for (i = 0; i < msg->resolution_resp.resolution_count; i++) {
+- screen_width_max = max_t(unsigned int, screen_width_max,
+- msg->resolution_resp.supported_resolution[i].width);
+- screen_height_max = max_t(unsigned int, screen_height_max,
+- msg->resolution_resp.supported_resolution[i].height);
+- }
+-
+ screen_width =
+ msg->resolution_resp.supported_resolution[index].width;
+ screen_height =
+@@ -941,7 +931,7 @@ static void hvfb_get_option(struct fb_info *info)
+
+ if (x < HVFB_WIDTH_MIN || y < HVFB_HEIGHT_MIN ||
+ (synthvid_ver_ge(par->synthvid_version, SYNTHVID_VERSION_WIN10) &&
+- (x > screen_width_max || y > screen_height_max)) ||
++ (x * y * screen_depth / 8 > screen_fb_size)) ||
+ (par->synthvid_version == SYNTHVID_VERSION_WIN8 &&
+ x * y * screen_depth / 8 > SYNTHVID_FB_SIZE_WIN8) ||
+ (par->synthvid_version == SYNTHVID_VERSION_WIN7 &&
+@@ -1194,8 +1184,8 @@ static int hvfb_probe(struct hv_device *hdev,
+ }
+
+ hvfb_get_option(info);
+- pr_info("Screen resolution: %dx%d, Color depth: %d\n",
+- screen_width, screen_height, screen_depth);
++ pr_info("Screen resolution: %dx%d, Color depth: %d, Frame buffer size: %d\n",
++ screen_width, screen_height, screen_depth, screen_fb_size);
+
+ ret = hvfb_getmem(hdev, info);
+ if (ret) {
+diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c
+index 5c82611e93d99..236521b19daf7 100644
+--- a/drivers/video/fbdev/matrox/matroxfb_base.c
++++ b/drivers/video/fbdev/matrox/matroxfb_base.c
+@@ -1377,7 +1377,7 @@ static struct video_board vbG200 = {
+ .lowlevel = &matrox_G100
+ };
+ static struct video_board vbG200eW = {
+- .maxvram = 0x800000,
++ .maxvram = 0x100000,
+ .maxdisplayable = 0x800000,
+ .accelID = FB_ACCEL_MATROX_MGAG200,
+ .lowlevel = &matrox_G100
+diff --git a/drivers/video/fbdev/nvidia/nv_i2c.c b/drivers/video/fbdev/nvidia/nv_i2c.c
+index d7994a1732459..0b48965a6420c 100644
+--- a/drivers/video/fbdev/nvidia/nv_i2c.c
++++ b/drivers/video/fbdev/nvidia/nv_i2c.c
+@@ -86,7 +86,7 @@ static int nvidia_setup_i2c_bus(struct nvidia_i2c_chan *chan, const char *name,
+ {
+ int rc;
+
+- strcpy(chan->adapter.name, name);
++ strscpy(chan->adapter.name, name, sizeof(chan->adapter.name));
+ chan->adapter.owner = THIS_MODULE;
+ chan->adapter.class = i2c_class;
+ chan->adapter.algo_data = &chan->algo;
+diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
+index 2fa436475b406..c8ad3ef42bd31 100644
+--- a/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
++++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
+@@ -246,6 +246,7 @@ static int dvic_probe_of(struct platform_device *pdev)
+ adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
+ if (adapter_node) {
+ adapter = of_get_i2c_adapter_by_node(adapter_node);
++ of_node_put(adapter_node);
+ if (adapter == NULL) {
+ dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
+ omap_dss_put_device(ddata->in);
+diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
+index 4b0793abdd84b..a2c7c5cb15234 100644
+--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
++++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
+@@ -409,7 +409,7 @@ static ssize_t dsicm_num_errors_show(struct device *dev,
+ if (r)
+ return r;
+
+- return snprintf(buf, PAGE_SIZE, "%d\n", errors);
++ return sysfs_emit(buf, "%d\n", errors);
+ }
+
+ static ssize_t dsicm_hw_revision_show(struct device *dev,
+@@ -439,7 +439,7 @@ static ssize_t dsicm_hw_revision_show(struct device *dev,
+ if (r)
+ return r;
+
+- return snprintf(buf, PAGE_SIZE, "%02x.%02x.%02x\n", id1, id2, id3);
++ return sysfs_emit(buf, "%02x.%02x.%02x\n", id1, id2, id3);
+ }
+
+ static ssize_t dsicm_store_ulps(struct device *dev,
+@@ -487,7 +487,7 @@ static ssize_t dsicm_show_ulps(struct device *dev,
+ t = ddata->ulps_enabled;
+ mutex_unlock(&ddata->lock);
+
+- return snprintf(buf, PAGE_SIZE, "%u\n", t);
++ return sysfs_emit(buf, "%u\n", t);
+ }
+
+ static ssize_t dsicm_store_ulps_timeout(struct device *dev,
+@@ -532,7 +532,7 @@ static ssize_t dsicm_show_ulps_timeout(struct device *dev,
+ t = ddata->ulps_timeout;
+ mutex_unlock(&ddata->lock);
+
+- return snprintf(buf, PAGE_SIZE, "%u\n", t);
++ return sysfs_emit(buf, "%u\n", t);
+ }
+
+ static DEVICE_ATTR(num_dsi_errors, S_IRUGO, dsicm_num_errors_show, NULL);
+diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
+index 8d8b5ff7d43c8..3696eb09b69b4 100644
+--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
++++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
+@@ -476,7 +476,7 @@ static ssize_t show_cabc_available_modes(struct device *dev,
+ int i;
+
+ if (!ddata->has_cabc)
+- return snprintf(buf, PAGE_SIZE, "%s\n", cabc_modes[0]);
++ return sysfs_emit(buf, "%s\n", cabc_modes[0]);
+
+ for (i = 0, len = 0;
+ len < PAGE_SIZE && i < ARRAY_SIZE(cabc_modes); i++)
+diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
+index afac1d9445aa2..57b7d1f490962 100644
+--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
++++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
+@@ -169,7 +169,7 @@ static ssize_t tpo_td043_vmirror_show(struct device *dev,
+ {
+ struct panel_drv_data *ddata = dev_get_drvdata(dev);
+
+- return snprintf(buf, PAGE_SIZE, "%d\n", ddata->vmirror);
++ return sysfs_emit(buf, "%d\n", ddata->vmirror);
+ }
+
+ static ssize_t tpo_td043_vmirror_store(struct device *dev,
+@@ -199,7 +199,7 @@ static ssize_t tpo_td043_mode_show(struct device *dev,
+ {
+ struct panel_drv_data *ddata = dev_get_drvdata(dev);
+
+- return snprintf(buf, PAGE_SIZE, "%d\n", ddata->mode);
++ return sysfs_emit(buf, "%d\n", ddata->mode);
+ }
+
+ static ssize_t tpo_td043_mode_store(struct device *dev,
+diff --git a/drivers/video/fbdev/simplefb.c b/drivers/video/fbdev/simplefb.c
+index 62f0ded706815..a2e3a46900252 100644
+--- a/drivers/video/fbdev/simplefb.c
++++ b/drivers/video/fbdev/simplefb.c
+@@ -70,12 +70,18 @@ struct simplefb_par;
+ static void simplefb_clocks_destroy(struct simplefb_par *par);
+ static void simplefb_regulators_destroy(struct simplefb_par *par);
+
++/*
++ * fb_ops.fb_destroy is called by the last put_fb_info() call at the end
++ * of unregister_framebuffer() or fb_release(). Do any cleanup here.
++ */
+ static void simplefb_destroy(struct fb_info *info)
+ {
+ simplefb_regulators_destroy(info->par);
+ simplefb_clocks_destroy(info->par);
+ if (info->screen_base)
+ iounmap(info->screen_base);
++
++ framebuffer_release(info);
+ }
+
+ static const struct fb_ops simplefb_ops = {
+@@ -407,6 +413,17 @@ static int simplefb_probe(struct platform_device *pdev)
+ struct simplefb_par *par;
+ struct resource *mem;
+
++ /*
++ * Generic drivers must not be registered if a framebuffer exists.
++ * If a native driver was probed, the display hardware was already
++ * taken and attempting to use the system framebuffer is dangerous.
++ */
++ if (num_registered_fb > 0) {
++ dev_err(&pdev->dev,
++ "simplefb: a framebuffer is already registered\n");
++ return -EINVAL;
++ }
++
+ if (fb_get_options("simplefb", NULL))
+ return -ENODEV;
+
+@@ -509,8 +526,8 @@ static int simplefb_remove(struct platform_device *pdev)
+ {
+ struct fb_info *info = platform_get_drvdata(pdev);
+
++ /* simplefb_destroy takes care of info cleanup */
+ unregister_framebuffer(info);
+- framebuffer_release(info);
+
+ return 0;
+ }
+diff --git a/drivers/video/fbdev/sm712fb.c b/drivers/video/fbdev/sm712fb.c
+index 0dbc6bf8268ac..092a1caa1208e 100644
+--- a/drivers/video/fbdev/sm712fb.c
++++ b/drivers/video/fbdev/sm712fb.c
+@@ -1047,7 +1047,7 @@ static ssize_t smtcfb_read(struct fb_info *info, char __user *buf,
+ if (count + p > total_size)
+ count = total_size - p;
+
+- buffer = kmalloc((count > PAGE_SIZE) ? PAGE_SIZE : count, GFP_KERNEL);
++ buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+@@ -1059,25 +1059,14 @@ static ssize_t smtcfb_read(struct fb_info *info, char __user *buf,
+ while (count) {
+ c = (count > PAGE_SIZE) ? PAGE_SIZE : count;
+ dst = buffer;
+- for (i = c >> 2; i--;) {
+- *dst = fb_readl(src++);
+- *dst = big_swap(*dst);
++ for (i = (c + 3) >> 2; i--;) {
++ u32 val;
++
++ val = fb_readl(src);
++ *dst = big_swap(val);
++ src++;
+ dst++;
+ }
+- if (c & 3) {
+- u8 *dst8 = (u8 *)dst;
+- u8 __iomem *src8 = (u8 __iomem *)src;
+-
+- for (i = c & 3; i--;) {
+- if (i & 1) {
+- *dst8++ = fb_readb(++src8);
+- } else {
+- *dst8++ = fb_readb(--src8);
+- src8 += 2;
+- }
+- }
+- src = (u32 __iomem *)src8;
+- }
+
+ if (copy_to_user(buf, buffer, c)) {
+ err = -EFAULT;
+@@ -1130,7 +1119,7 @@ static ssize_t smtcfb_write(struct fb_info *info, const char __user *buf,
+ count = total_size - p;
+ }
+
+- buffer = kmalloc((count > PAGE_SIZE) ? PAGE_SIZE : count, GFP_KERNEL);
++ buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+@@ -1148,24 +1137,11 @@ static ssize_t smtcfb_write(struct fb_info *info, const char __user *buf,
+ break;
+ }
+
+- for (i = c >> 2; i--;) {
+- fb_writel(big_swap(*src), dst++);
++ for (i = (c + 3) >> 2; i--;) {
++ fb_writel(big_swap(*src), dst);
++ dst++;
+ src++;
+ }
+- if (c & 3) {
+- u8 *src8 = (u8 *)src;
+- u8 __iomem *dst8 = (u8 __iomem *)dst;
+-
+- for (i = c & 3; i--;) {
+- if (i & 1) {
+- fb_writeb(*src8++, ++dst8);
+- } else {
+- fb_writeb(*src8++, --dst8);
+- dst8 += 2;
+- }
+- }
+- dst = (u32 __iomem *)dst8;
+- }
+
+ *ppos += c;
+ buf += c;
+diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c
+index bfac3ee4a6422..28768c272b73d 100644
+--- a/drivers/video/fbdev/smscufx.c
++++ b/drivers/video/fbdev/smscufx.c
+@@ -1656,6 +1656,7 @@ static int ufx_usb_probe(struct usb_interface *interface,
+ info->par = dev;
+ info->pseudo_palette = dev->pseudo_palette;
+ info->fbops = &ufx_ops;
++ INIT_LIST_HEAD(&info->modelist);
+
+ retval = fb_alloc_cmap(&info->cmap, 256, 0);
+ if (retval < 0) {
+@@ -1666,8 +1667,6 @@ static int ufx_usb_probe(struct usb_interface *interface,
+ INIT_DELAYED_WORK(&dev->free_framebuffer_work,
+ ufx_free_framebuffer_work);
+
+- INIT_LIST_HEAD(&info->modelist);
+-
+ retval = ufx_reg_read(dev, 0x3000, &id_rev);
+ check_warn_goto_error(retval, "error %d reading 0x3000 register from device", retval);
+ dev_dbg(dev->gdev, "ID_REV register value 0x%08x", id_rev);
+diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c
+index b9cdd02c10009..d9eec1b60e665 100644
+--- a/drivers/video/fbdev/udlfb.c
++++ b/drivers/video/fbdev/udlfb.c
+@@ -1426,7 +1426,7 @@ static ssize_t metrics_bytes_rendered_show(struct device *fbdev,
+ struct device_attribute *a, char *buf) {
+ struct fb_info *fb_info = dev_get_drvdata(fbdev);
+ struct dlfb_data *dlfb = fb_info->par;
+- return snprintf(buf, PAGE_SIZE, "%u\n",
++ return sysfs_emit(buf, "%u\n",
+ atomic_read(&dlfb->bytes_rendered));
+ }
+
+@@ -1434,7 +1434,7 @@ static ssize_t metrics_bytes_identical_show(struct device *fbdev,
+ struct device_attribute *a, char *buf) {
+ struct fb_info *fb_info = dev_get_drvdata(fbdev);
+ struct dlfb_data *dlfb = fb_info->par;
+- return snprintf(buf, PAGE_SIZE, "%u\n",
++ return sysfs_emit(buf, "%u\n",
+ atomic_read(&dlfb->bytes_identical));
+ }
+
+@@ -1442,7 +1442,7 @@ static ssize_t metrics_bytes_sent_show(struct device *fbdev,
+ struct device_attribute *a, char *buf) {
+ struct fb_info *fb_info = dev_get_drvdata(fbdev);
+ struct dlfb_data *dlfb = fb_info->par;
+- return snprintf(buf, PAGE_SIZE, "%u\n",
++ return sysfs_emit(buf, "%u\n",
+ atomic_read(&dlfb->bytes_sent));
+ }
+
+@@ -1450,7 +1450,7 @@ static ssize_t metrics_cpu_kcycles_used_show(struct device *fbdev,
+ struct device_attribute *a, char *buf) {
+ struct fb_info *fb_info = dev_get_drvdata(fbdev);
+ struct dlfb_data *dlfb = fb_info->par;
+- return snprintf(buf, PAGE_SIZE, "%u\n",
++ return sysfs_emit(buf, "%u\n",
+ atomic_read(&dlfb->cpu_kcycles_used));
+ }
+
+@@ -1649,8 +1649,9 @@ static int dlfb_usb_probe(struct usb_interface *intf,
+ const struct device_attribute *attr;
+ struct dlfb_data *dlfb;
+ struct fb_info *info;
+- int retval = -ENOMEM;
++ int retval;
+ struct usb_device *usbdev = interface_to_usbdev(intf);
++ struct usb_endpoint_descriptor *out;
+
+ /* usb initialization */
+ dlfb = kzalloc(sizeof(*dlfb), GFP_KERNEL);
+@@ -1664,6 +1665,12 @@ static int dlfb_usb_probe(struct usb_interface *intf,
+ dlfb->udev = usb_get_dev(usbdev);
+ usb_set_intfdata(intf, dlfb);
+
++ retval = usb_find_common_endpoints(intf->cur_altsetting, NULL, &out, NULL, NULL);
++ if (retval) {
++ dev_err(&intf->dev, "Device should have at lease 1 bulk endpoint!\n");
++ goto error;
++ }
++
+ dev_dbg(&intf->dev, "console enable=%d\n", console);
+ dev_dbg(&intf->dev, "fb_defio enable=%d\n", fb_defio);
+ dev_dbg(&intf->dev, "shadow enable=%d\n", shadow);
+@@ -1673,6 +1680,7 @@ static int dlfb_usb_probe(struct usb_interface *intf,
+ if (!dlfb_parse_vendor_descriptor(dlfb, intf)) {
+ dev_err(&intf->dev,
+ "firmware not recognized, incompatible device?\n");
++ retval = -ENODEV;
+ goto error;
+ }
+
+@@ -1686,8 +1694,10 @@ static int dlfb_usb_probe(struct usb_interface *intf,
+
+ /* allocates framebuffer driver structure, not framebuffer memory */
+ info = framebuffer_alloc(0, &dlfb->udev->dev);
+- if (!info)
++ if (!info) {
++ retval = -ENOMEM;
+ goto error;
++ }
+
+ dlfb->info = info;
+ info->par = dlfb;
+diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c
+index df6de5a9dd4cd..e25e8de5ff672 100644
+--- a/drivers/video/fbdev/vesafb.c
++++ b/drivers/video/fbdev/vesafb.c
+@@ -179,6 +179,10 @@ static int vesafb_setcolreg(unsigned regno, unsigned red, unsigned green,
+ return err;
+ }
+
++/*
++ * fb_ops.fb_destroy is called by the last put_fb_info() call at the end
++ * of unregister_framebuffer() or fb_release(). Do any cleanup here.
++ */
+ static void vesafb_destroy(struct fb_info *info)
+ {
+ struct vesafb_par *par = info->par;
+@@ -188,6 +192,8 @@ static void vesafb_destroy(struct fb_info *info)
+ if (info->screen_base)
+ iounmap(info->screen_base);
+ release_mem_region(info->apertures->ranges[0].base, info->apertures->ranges[0].size);
++
++ framebuffer_release(info);
+ }
+
+ static struct fb_ops vesafb_ops = {
+@@ -484,10 +490,10 @@ static int vesafb_remove(struct platform_device *pdev)
+ {
+ struct fb_info *info = platform_get_drvdata(pdev);
+
++ /* vesafb_destroy takes care of info cleanup */
+ unregister_framebuffer(info);
+ if (((struct vesafb_par *)(info->par))->region)
+ release_region(0x3c0, 32);
+- framebuffer_release(info);
+
+ return 0;
+ }
+diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c
+index e2757ff1c23d2..96e312a3eac75 100644
+--- a/drivers/video/fbdev/vga16fb.c
++++ b/drivers/video/fbdev/vga16fb.c
+@@ -184,6 +184,25 @@ static inline void setindex(int index)
+ vga_io_w(VGA_GFX_I, index);
+ }
+
++/* Check if the video mode is supported by the driver */
++static inline int check_mode_supported(void)
++{
++ /* non-x86 architectures treat orig_video_isVGA as a boolean flag */
++#if defined(CONFIG_X86)
++ /* only EGA and VGA in 16 color graphic mode are supported */
++ if (screen_info.orig_video_isVGA != VIDEO_TYPE_EGAC &&
++ screen_info.orig_video_isVGA != VIDEO_TYPE_VGAC)
++ return -ENODEV;
++
++ if (screen_info.orig_video_mode != 0x0D && /* 320x200/4 (EGA) */
++ screen_info.orig_video_mode != 0x0E && /* 640x200/4 (EGA) */
++ screen_info.orig_video_mode != 0x10 && /* 640x350/4 (EGA) */
++ screen_info.orig_video_mode != 0x12) /* 640x480/4 (VGA) */
++ return -ENODEV;
++#endif
++ return 0;
++}
++
+ static void vga16fb_pan_var(struct fb_info *info,
+ struct fb_var_screeninfo *var)
+ {
+@@ -1422,6 +1441,11 @@ static int __init vga16fb_init(void)
+
+ vga16fb_setup(option);
+ #endif
++
++ ret = check_mode_supported();
++ if (ret)
++ return ret;
++
+ ret = platform_driver_register(&vga16fb_driver);
+
+ if (!ret) {
+diff --git a/drivers/video/fbdev/w100fb.c b/drivers/video/fbdev/w100fb.c
+index d96ab28f8ce4a..4e641a780726e 100644
+--- a/drivers/video/fbdev/w100fb.c
++++ b/drivers/video/fbdev/w100fb.c
+@@ -770,12 +770,18 @@ out:
+ fb_dealloc_cmap(&info->cmap);
+ kfree(info->pseudo_palette);
+ }
+- if (remapped_fbuf != NULL)
++ if (remapped_fbuf != NULL) {
+ iounmap(remapped_fbuf);
+- if (remapped_regs != NULL)
++ remapped_fbuf = NULL;
++ }
++ if (remapped_regs != NULL) {
+ iounmap(remapped_regs);
+- if (remapped_base != NULL)
++ remapped_regs = NULL;
++ }
++ if (remapped_base != NULL) {
+ iounmap(remapped_base);
++ remapped_base = NULL;
++ }
+ if (info)
+ framebuffer_release(info);
+ return err;
+@@ -795,8 +801,11 @@ static int w100fb_remove(struct platform_device *pdev)
+ fb_dealloc_cmap(&info->cmap);
+
+ iounmap(remapped_base);
++ remapped_base = NULL;
+ iounmap(remapped_regs);
++ remapped_regs = NULL;
+ iounmap(remapped_fbuf);
++ remapped_fbuf = NULL;
+
+ framebuffer_release(info);
+
+diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c
+index 130e12b8652a6..af889cee66805 100644
+--- a/drivers/virt/acrn/hsm.c
++++ b/drivers/virt/acrn/hsm.c
+@@ -134,8 +134,10 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd,
+ if (IS_ERR(vm_param))
+ return PTR_ERR(vm_param);
+
+- if ((vm_param->reserved0 | vm_param->reserved1) != 0)
++ if ((vm_param->reserved0 | vm_param->reserved1) != 0) {
++ kfree(vm_param);
+ return -EINVAL;
++ }
+
+ vm = acrn_vm_create(vm, vm_param);
+ if (!vm) {
+@@ -180,21 +182,29 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd,
+ return PTR_ERR(cpu_regs);
+
+ for (i = 0; i < ARRAY_SIZE(cpu_regs->reserved); i++)
+- if (cpu_regs->reserved[i])
++ if (cpu_regs->reserved[i]) {
++ kfree(cpu_regs);
+ return -EINVAL;
++ }
+
+ for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_32); i++)
+- if (cpu_regs->vcpu_regs.reserved_32[i])
++ if (cpu_regs->vcpu_regs.reserved_32[i]) {
++ kfree(cpu_regs);
+ return -EINVAL;
++ }
+
+ for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_64); i++)
+- if (cpu_regs->vcpu_regs.reserved_64[i])
++ if (cpu_regs->vcpu_regs.reserved_64[i]) {
++ kfree(cpu_regs);
+ return -EINVAL;
++ }
+
+ for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.gdt.reserved); i++)
+ if (cpu_regs->vcpu_regs.gdt.reserved[i] |
+- cpu_regs->vcpu_regs.idt.reserved[i])
++ cpu_regs->vcpu_regs.idt.reserved[i]) {
++ kfree(cpu_regs);
+ return -EINVAL;
++ }
+
+ ret = hcall_set_vcpu_regs(vm->vmid, virt_to_phys(cpu_regs));
+ if (ret < 0)
+diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c
+index c4f2e15c8a2ba..3b1b1e7a844b4 100644
+--- a/drivers/virt/acrn/mm.c
++++ b/drivers/virt/acrn/mm.c
+@@ -162,10 +162,34 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap)
+ void *remap_vaddr;
+ int ret, pinned;
+ u64 user_vm_pa;
++ unsigned long pfn;
++ struct vm_area_struct *vma;
+
+ if (!vm || !memmap)
+ return -EINVAL;
+
++ mmap_read_lock(current->mm);
++ vma = vma_lookup(current->mm, memmap->vma_base);
++ if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) {
++ if ((memmap->vma_base + memmap->len) > vma->vm_end) {
++ mmap_read_unlock(current->mm);
++ return -EINVAL;
++ }
++
++ ret = follow_pfn(vma, memmap->vma_base, &pfn);
++ mmap_read_unlock(current->mm);
++ if (ret < 0) {
++ dev_dbg(acrn_dev.this_device,
++ "Failed to lookup PFN at VMA:%pK.\n", (void *)memmap->vma_base);
++ return ret;
++ }
++
++ return acrn_mm_region_add(vm, memmap->user_vm_pa,
++ PFN_PHYS(pfn), memmap->len,
++ ACRN_MEM_TYPE_WB, memmap->attr);
++ }
++ mmap_read_unlock(current->mm);
++
+ /* Get the page number of the map region */
+ nr_pages = memmap->len >> PAGE_SHIFT;
+ pages = vzalloc(nr_pages * sizeof(struct page *));
+diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
+index e21e1e86ad15f..fe7a8e4034097 100644
+--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
++++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
+@@ -886,8 +886,9 @@ static int ne_set_user_memory_region_ioctl(struct ne_enclave *ne_enclave,
+ goto put_pages;
+ }
+
+- gup_rc = get_user_pages(mem_region.userspace_addr + memory_size, 1, FOLL_GET,
+- ne_mem_region->pages + i, NULL);
++ gup_rc = get_user_pages_unlocked(mem_region.userspace_addr + memory_size, 1,
++ ne_mem_region->pages + i, FOLL_GET);
++
+ if (gup_rc < 0) {
+ rc = gup_rc;
+
+diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
+index 236081afe9a2a..c2b733ef95b0d 100644
+--- a/drivers/virtio/virtio.c
++++ b/drivers/virtio/virtio.c
+@@ -166,14 +166,13 @@ void virtio_add_status(struct virtio_device *dev, unsigned int status)
+ }
+ EXPORT_SYMBOL_GPL(virtio_add_status);
+
+-int virtio_finalize_features(struct virtio_device *dev)
++/* Do some validation, then set FEATURES_OK */
++static int virtio_features_ok(struct virtio_device *dev)
+ {
+- int ret = dev->config->finalize_features(dev);
+ unsigned status;
++ int ret;
+
+ might_sleep();
+- if (ret)
+- return ret;
+
+ ret = arch_has_restricted_virtio_memory_access();
+ if (ret) {
+@@ -202,7 +201,6 @@ int virtio_finalize_features(struct virtio_device *dev)
+ }
+ return 0;
+ }
+-EXPORT_SYMBOL_GPL(virtio_finalize_features);
+
+ static int virtio_dev_probe(struct device *_d)
+ {
+@@ -239,17 +237,6 @@ static int virtio_dev_probe(struct device *_d)
+ driver_features_legacy = driver_features;
+ }
+
+- /*
+- * Some devices detect legacy solely via F_VERSION_1. Write
+- * F_VERSION_1 to force LE config space accesses before FEATURES_OK for
+- * these when needed.
+- */
+- if (drv->validate && !virtio_legacy_is_little_endian()
+- && device_features & BIT_ULL(VIRTIO_F_VERSION_1)) {
+- dev->features = BIT_ULL(VIRTIO_F_VERSION_1);
+- dev->config->finalize_features(dev);
+- }
+-
+ if (device_features & (1ULL << VIRTIO_F_VERSION_1))
+ dev->features = driver_features & device_features;
+ else
+@@ -260,13 +247,26 @@ static int virtio_dev_probe(struct device *_d)
+ if (device_features & (1ULL << i))
+ __virtio_set_bit(dev, i);
+
++ err = dev->config->finalize_features(dev);
++ if (err)
++ goto err;
++
+ if (drv->validate) {
++ u64 features = dev->features;
++
+ err = drv->validate(dev);
+ if (err)
+ goto err;
++
++ /* Did validation change any features? Then write them again. */
++ if (features != dev->features) {
++ err = dev->config->finalize_features(dev);
++ if (err)
++ goto err;
++ }
+ }
+
+- err = virtio_finalize_features(dev);
++ err = virtio_features_ok(dev);
+ if (err)
+ goto err;
+
+@@ -490,7 +490,11 @@ int virtio_device_restore(struct virtio_device *dev)
+ /* We have a driver! */
+ virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
+
+- ret = virtio_finalize_features(dev);
++ ret = dev->config->finalize_features(dev);
++ if (ret)
++ goto err;
++
++ ret = virtio_features_ok(dev);
+ if (ret)
+ goto err;
+
+diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c
+index bef8ad6bf4661..4624a2c3d0553 100644
+--- a/drivers/virtio/virtio_mem.c
++++ b/drivers/virtio/virtio_mem.c
+@@ -577,7 +577,7 @@ static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm)
+ return -ENOMEM;
+
+ mutex_lock(&vm->hotplug_mutex);
+- if (new_bitmap)
++ if (vm->sbm.sb_states)
+ memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE);
+
+ old_bitmap = vm->sbm.sb_states;
+diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
+index 3035bb6f54585..603a6f4345efd 100644
+--- a/drivers/virtio/virtio_ring.c
++++ b/drivers/virtio/virtio_ring.c
+@@ -268,7 +268,7 @@ size_t virtio_max_dma_size(struct virtio_device *vdev)
+ size_t max_segment_size = SIZE_MAX;
+
+ if (vring_use_dma_api(vdev))
+- max_segment_size = dma_max_mapping_size(&vdev->dev);
++ max_segment_size = dma_max_mapping_size(vdev->dev.parent);
+
+ return max_segment_size;
+ }
+@@ -1065,6 +1065,8 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
+
+ head = vq->packed.next_avail_idx;
+ desc = alloc_indirect_packed(total_sg, gfp);
++ if (!desc)
++ return -ENOMEM;
+
+ if (unlikely(vq->vq.num_free < 1)) {
+ pr_debug("Can't add buf len 1 - avail = 0\n");
+@@ -1176,6 +1178,7 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
+ unsigned int i, n, c, descs_used, err_idx;
+ __le16 head_flags, flags;
+ u16 head, id, prev, curr, avail_used_flags;
++ int err;
+
+ START_USE(vq);
+
+@@ -1191,9 +1194,16 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
+
+ BUG_ON(total_sg == 0);
+
+- if (virtqueue_use_indirect(_vq, total_sg))
+- return virtqueue_add_indirect_packed(vq, sgs, total_sg,
+- out_sgs, in_sgs, data, gfp);
++ if (virtqueue_use_indirect(_vq, total_sg)) {
++ err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
++ in_sgs, data, gfp);
++ if (err != -ENOMEM) {
++ END_USE(vq);
++ return err;
++ }
++
++ /* fall back on direct */
++ }
+
+ head = vq->packed.next_avail_idx;
+ avail_used_flags = vq->packed.avail_used_flags;
+diff --git a/drivers/w1/slaves/w1_ds28e04.c b/drivers/w1/slaves/w1_ds28e04.c
+index e4f336111edc6..6cef6e2edb892 100644
+--- a/drivers/w1/slaves/w1_ds28e04.c
++++ b/drivers/w1/slaves/w1_ds28e04.c
+@@ -32,7 +32,7 @@ static int w1_strong_pullup = 1;
+ module_param_named(strong_pullup, w1_strong_pullup, int, 0);
+
+ /* enable/disable CRC checking on DS28E04-100 memory accesses */
+-static char w1_enable_crccheck = 1;
++static bool w1_enable_crccheck = true;
+
+ #define W1_EEPROM_SIZE 512
+ #define W1_PAGE_COUNT 16
+@@ -339,32 +339,18 @@ static BIN_ATTR_RW(pio, 1);
+ static ssize_t crccheck_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+ {
+- if (put_user(w1_enable_crccheck + 0x30, buf))
+- return -EFAULT;
+-
+- return sizeof(w1_enable_crccheck);
++ return sysfs_emit(buf, "%d\n", w1_enable_crccheck);
+ }
+
+ static ssize_t crccheck_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+ {
+- char val;
+-
+- if (count != 1 || !buf)
+- return -EINVAL;
++ int err = kstrtobool(buf, &w1_enable_crccheck);
+
+- if (get_user(val, buf))
+- return -EFAULT;
++ if (err)
++ return err;
+
+- /* convert to decimal */
+- val = val - 0x30;
+- if (val != 0 && val != 1)
+- return -EINVAL;
+-
+- /* set the new value */
+- w1_enable_crccheck = val;
+-
+- return sizeof(w1_enable_crccheck);
++ return count;
+ }
+
+ static DEVICE_ATTR_RW(crccheck);
+diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
+index ca70c5f032060..9cbeeb4923ecf 100644
+--- a/drivers/w1/slaves/w1_therm.c
++++ b/drivers/w1/slaves/w1_therm.c
+@@ -2090,16 +2090,20 @@ static ssize_t w1_seq_show(struct device *device,
+ if (sl->reg_num.id == reg_num->id)
+ seq = i;
+
++ if (w1_reset_bus(sl->master))
++ goto error;
++
++ /* Put the device into chain DONE state */
++ w1_write_8(sl->master, W1_MATCH_ROM);
++ w1_write_block(sl->master, (u8 *)&rn, 8);
+ w1_write_8(sl->master, W1_42_CHAIN);
+ w1_write_8(sl->master, W1_42_CHAIN_DONE);
+ w1_write_8(sl->master, W1_42_CHAIN_DONE_INV);
+- w1_read_block(sl->master, &ack, sizeof(ack));
+
+ /* check for acknowledgment */
+ ack = w1_read_8(sl->master);
+ if (ack != W1_42_SUCCESS_CONFIRM_BYTE)
+ goto error;
+-
+ }
+
+ /* Exit from CHAIN state */
+diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
+index bf59faeb3de1b..d937f957f8df8 100644
+--- a/drivers/watchdog/Kconfig
++++ b/drivers/watchdog/Kconfig
+@@ -1679,7 +1679,7 @@ config SIBYTE_WDOG
+
+ config AR7_WDT
+ tristate "TI AR7 Watchdog Timer"
+- depends on AR7 || (MIPS && COMPILE_TEST)
++ depends on AR7 || (MIPS && 32BIT && COMPILE_TEST)
+ help
+ Hardware driver for the TI AR7 Watchdog Timer.
+
+diff --git a/drivers/watchdog/f71808e_wdt.c b/drivers/watchdog/f71808e_wdt.c
+index f60beec1bbaea..f7d82d2619133 100644
+--- a/drivers/watchdog/f71808e_wdt.c
++++ b/drivers/watchdog/f71808e_wdt.c
+@@ -228,15 +228,17 @@ static int watchdog_set_timeout(int timeout)
+
+ mutex_lock(&watchdog.lock);
+
+- watchdog.timeout = timeout;
+ if (timeout > 0xff) {
+ watchdog.timer_val = DIV_ROUND_UP(timeout, 60);
+ watchdog.minutes_mode = true;
++ timeout = watchdog.timer_val * 60;
+ } else {
+ watchdog.timer_val = timeout;
+ watchdog.minutes_mode = false;
+ }
+
++ watchdog.timeout = timeout;
++
+ mutex_unlock(&watchdog.lock);
+
+ return 0;
+diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c
+index 359302f71f7ef..ae7f9357bb871 100644
+--- a/drivers/watchdog/rti_wdt.c
++++ b/drivers/watchdog/rti_wdt.c
+@@ -229,6 +229,7 @@ static int rti_wdt_probe(struct platform_device *pdev)
+ ret = pm_runtime_get_sync(dev);
+ if (ret) {
+ pm_runtime_put_noidle(dev);
++ pm_runtime_disable(&pdev->dev);
+ return dev_err_probe(dev, ret, "runtime pm failed\n");
+ }
+
+diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c
+index a730ecbf78cd5..4820af929a826 100644
+--- a/drivers/watchdog/sp5100_tco.c
++++ b/drivers/watchdog/sp5100_tco.c
+@@ -48,7 +48,7 @@
+ /* internal variables */
+
+ enum tco_reg_layout {
+- sp5100, sb800, efch
++ sp5100, sb800, efch, efch_mmio
+ };
+
+ struct sp5100_tco {
+@@ -85,6 +85,10 @@ static enum tco_reg_layout tco_reg_layout(struct pci_dev *dev)
+ dev->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
+ dev->revision < 0x40) {
+ return sp5100;
++ } else if (dev->vendor == PCI_VENDOR_ID_AMD &&
++ sp5100_tco_pci->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS &&
++ sp5100_tco_pci->revision >= AMD_ZEN_SMBUS_PCI_REV) {
++ return efch_mmio;
+ } else if (dev->vendor == PCI_VENDOR_ID_AMD &&
+ ((dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS &&
+ dev->revision >= 0x41) ||
+@@ -201,6 +205,8 @@ static void tco_timer_enable(struct sp5100_tco *tco)
+ ~EFCH_PM_WATCHDOG_DISABLE,
+ EFCH_PM_DECODEEN_SECOND_RES);
+ break;
++ default:
++ break;
+ }
+ }
+
+@@ -215,14 +221,195 @@ static u32 sp5100_tco_read_pm_reg32(u8 index)
+ return val;
+ }
+
++static u32 sp5100_tco_request_region(struct device *dev,
++ u32 mmio_addr,
++ const char *dev_name)
++{
++ if (!devm_request_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE,
++ dev_name)) {
++ dev_dbg(dev, "MMIO address 0x%08x already in use\n", mmio_addr);
++ return 0;
++ }
++
++ return mmio_addr;
++}
++
++static u32 sp5100_tco_prepare_base(struct sp5100_tco *tco,
++ u32 mmio_addr,
++ u32 alt_mmio_addr,
++ const char *dev_name)
++{
++ struct device *dev = tco->wdd.parent;
++
++ dev_dbg(dev, "Got 0x%08x from SBResource_MMIO register\n", mmio_addr);
++
++ if (!mmio_addr && !alt_mmio_addr)
++ return -ENODEV;
++
++ /* Check for MMIO address and alternate MMIO address conflicts */
++ if (mmio_addr)
++ mmio_addr = sp5100_tco_request_region(dev, mmio_addr, dev_name);
++
++ if (!mmio_addr && alt_mmio_addr)
++ mmio_addr = sp5100_tco_request_region(dev, alt_mmio_addr, dev_name);
++
++ if (!mmio_addr) {
++ dev_err(dev, "Failed to reserve MMIO or alternate MMIO region\n");
++ return -EBUSY;
++ }
++
++ tco->tcobase = devm_ioremap(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE);
++ if (!tco->tcobase) {
++ dev_err(dev, "MMIO address 0x%08x failed mapping\n", mmio_addr);
++ devm_release_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE);
++ return -ENOMEM;
++ }
++
++ dev_info(dev, "Using 0x%08x for watchdog MMIO address\n", mmio_addr);
++
++ return 0;
++}
++
++static int sp5100_tco_timer_init(struct sp5100_tco *tco)
++{
++ struct watchdog_device *wdd = &tco->wdd;
++ struct device *dev = wdd->parent;
++ u32 val;
++
++ val = readl(SP5100_WDT_CONTROL(tco->tcobase));
++ if (val & SP5100_WDT_DISABLED) {
++ dev_err(dev, "Watchdog hardware is disabled\n");
++ return -ENODEV;
++ }
++
++ /*
++ * Save WatchDogFired status, because WatchDogFired flag is
++ * cleared here.
++ */
++ if (val & SP5100_WDT_FIRED)
++ wdd->bootstatus = WDIOF_CARDRESET;
++
++ /* Set watchdog action to reset the system */
++ val &= ~SP5100_WDT_ACTION_RESET;
++ writel(val, SP5100_WDT_CONTROL(tco->tcobase));
++
++ /* Set a reasonable heartbeat before we stop the timer */
++ tco_timer_set_timeout(wdd, wdd->timeout);
++
++ /*
++ * Stop the TCO before we change anything so we don't race with
++ * a zeroed timer.
++ */
++ tco_timer_stop(wdd);
++
++ return 0;
++}
++
++static u8 efch_read_pm_reg8(void __iomem *addr, u8 index)
++{
++ return readb(addr + index);
++}
++
++static void efch_update_pm_reg8(void __iomem *addr, u8 index, u8 reset, u8 set)
++{
++ u8 val;
++
++ val = readb(addr + index);
++ val &= reset;
++ val |= set;
++ writeb(val, addr + index);
++}
++
++static void tco_timer_enable_mmio(void __iomem *addr)
++{
++ efch_update_pm_reg8(addr, EFCH_PM_DECODEEN3,
++ ~EFCH_PM_WATCHDOG_DISABLE,
++ EFCH_PM_DECODEEN_SECOND_RES);
++}
++
++static int sp5100_tco_setupdevice_mmio(struct device *dev,
++ struct watchdog_device *wdd)
++{
++ struct sp5100_tco *tco = watchdog_get_drvdata(wdd);
++ const char *dev_name = SB800_DEVNAME;
++ u32 mmio_addr = 0, alt_mmio_addr = 0;
++ struct resource *res;
++ void __iomem *addr;
++ int ret;
++ u32 val;
++
++ res = request_mem_region_muxed(EFCH_PM_ACPI_MMIO_PM_ADDR,
++ EFCH_PM_ACPI_MMIO_PM_SIZE,
++ "sp5100_tco");
++
++ if (!res) {
++ dev_err(dev,
++ "Memory region 0x%08x already in use\n",
++ EFCH_PM_ACPI_MMIO_PM_ADDR);
++ return -EBUSY;
++ }
++
++ addr = ioremap(EFCH_PM_ACPI_MMIO_PM_ADDR, EFCH_PM_ACPI_MMIO_PM_SIZE);
++ if (!addr) {
++ dev_err(dev, "Address mapping failed\n");
++ ret = -ENOMEM;
++ goto out;
++ }
++
++ /*
++ * EFCH_PM_DECODEEN_WDT_TMREN is dual purpose. This bitfield
++ * enables sp5100_tco register MMIO space decoding. The bitfield
++ * also starts the timer operation. Enable if not already enabled.
++ */
++ val = efch_read_pm_reg8(addr, EFCH_PM_DECODEEN);
++ if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) {
++ efch_update_pm_reg8(addr, EFCH_PM_DECODEEN, 0xff,
++ EFCH_PM_DECODEEN_WDT_TMREN);
++ }
++
++ /* Error if the timer could not be enabled */
++ val = efch_read_pm_reg8(addr, EFCH_PM_DECODEEN);
++ if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) {
++ dev_err(dev, "Failed to enable the timer\n");
++ ret = -EFAULT;
++ goto out;
++ }
++
++ mmio_addr = EFCH_PM_WDT_ADDR;
++
++ /* Determine alternate MMIO base address */
++ val = efch_read_pm_reg8(addr, EFCH_PM_ISACONTROL);
++ if (val & EFCH_PM_ISACONTROL_MMIOEN)
++ alt_mmio_addr = EFCH_PM_ACPI_MMIO_ADDR +
++ EFCH_PM_ACPI_MMIO_WDT_OFFSET;
++
++ ret = sp5100_tco_prepare_base(tco, mmio_addr, alt_mmio_addr, dev_name);
++ if (!ret) {
++ tco_timer_enable_mmio(addr);
++ ret = sp5100_tco_timer_init(tco);
++ }
++
++out:
++ if (addr)
++ iounmap(addr);
++
++ release_resource(res);
++
++ return ret;
++}
++
+ static int sp5100_tco_setupdevice(struct device *dev,
+ struct watchdog_device *wdd)
+ {
+ struct sp5100_tco *tco = watchdog_get_drvdata(wdd);
+ const char *dev_name;
+ u32 mmio_addr = 0, val;
++ u32 alt_mmio_addr = 0;
+ int ret;
+
++ if (tco->tco_reg_layout == efch_mmio)
++ return sp5100_tco_setupdevice_mmio(dev, wdd);
++
+ /* Request the IO ports used by this driver */
+ if (!request_muxed_region(SP5100_IO_PM_INDEX_REG,
+ SP5100_PM_IOPORTS_SIZE, "sp5100_tco")) {
+@@ -239,138 +426,55 @@ static int sp5100_tco_setupdevice(struct device *dev,
+ dev_name = SP5100_DEVNAME;
+ mmio_addr = sp5100_tco_read_pm_reg32(SP5100_PM_WATCHDOG_BASE) &
+ 0xfffffff8;
++
++ /*
++ * Secondly, find the watchdog timer MMIO address
++ * from SBResource_MMIO register.
++ */
++
++ /* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */
++ pci_read_config_dword(sp5100_tco_pci,
++ SP5100_SB_RESOURCE_MMIO_BASE,
++ &val);
++
++ /* Verify MMIO is enabled and using bar0 */
++ if ((val & SB800_ACPI_MMIO_MASK) == SB800_ACPI_MMIO_DECODE_EN)
++ alt_mmio_addr = (val & ~0xfff) + SB800_PM_WDT_MMIO_OFFSET;
+ break;
+ case sb800:
+ dev_name = SB800_DEVNAME;
+ mmio_addr = sp5100_tco_read_pm_reg32(SB800_PM_WATCHDOG_BASE) &
+ 0xfffffff8;
++
++ /* Read SBResource_MMIO from AcpiMmioEn(PM_Reg: 24h) */
++ val = sp5100_tco_read_pm_reg32(SB800_PM_ACPI_MMIO_EN);
++
++ /* Verify MMIO is enabled and using bar0 */
++ if ((val & SB800_ACPI_MMIO_MASK) == SB800_ACPI_MMIO_DECODE_EN)
++ alt_mmio_addr = (val & ~0xfff) + SB800_PM_WDT_MMIO_OFFSET;
+ break;
+ case efch:
+ dev_name = SB800_DEVNAME;
+- /*
+- * On Family 17h devices, the EFCH_PM_DECODEEN_WDT_TMREN bit of
+- * EFCH_PM_DECODEEN not only enables the EFCH_PM_WDT_ADDR memory
+- * region, it also enables the watchdog itself.
+- */
+- if (boot_cpu_data.x86 == 0x17) {
+- val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN);
+- if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) {
+- sp5100_tco_update_pm_reg8(EFCH_PM_DECODEEN, 0xff,
+- EFCH_PM_DECODEEN_WDT_TMREN);
+- }
+- }
+ val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN);
+ if (val & EFCH_PM_DECODEEN_WDT_TMREN)
+ mmio_addr = EFCH_PM_WDT_ADDR;
++
++ val = sp5100_tco_read_pm_reg8(EFCH_PM_ISACONTROL);
++ if (val & EFCH_PM_ISACONTROL_MMIOEN)
++ alt_mmio_addr = EFCH_PM_ACPI_MMIO_ADDR +
++ EFCH_PM_ACPI_MMIO_WDT_OFFSET;
+ break;
+ default:
+ return -ENODEV;
+ }
+
+- /* Check MMIO address conflict */
+- if (!mmio_addr ||
+- !devm_request_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE,
+- dev_name)) {
+- if (mmio_addr)
+- dev_dbg(dev, "MMIO address 0x%08x already in use\n",
+- mmio_addr);
+- switch (tco->tco_reg_layout) {
+- case sp5100:
+- /*
+- * Secondly, Find the watchdog timer MMIO address
+- * from SBResource_MMIO register.
+- */
+- /* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */
+- pci_read_config_dword(sp5100_tco_pci,
+- SP5100_SB_RESOURCE_MMIO_BASE,
+- &mmio_addr);
+- if ((mmio_addr & (SB800_ACPI_MMIO_DECODE_EN |
+- SB800_ACPI_MMIO_SEL)) !=
+- SB800_ACPI_MMIO_DECODE_EN) {
+- ret = -ENODEV;
+- goto unreg_region;
+- }
+- mmio_addr &= ~0xFFF;
+- mmio_addr += SB800_PM_WDT_MMIO_OFFSET;
+- break;
+- case sb800:
+- /* Read SBResource_MMIO from AcpiMmioEn(PM_Reg: 24h) */
+- mmio_addr =
+- sp5100_tco_read_pm_reg32(SB800_PM_ACPI_MMIO_EN);
+- if ((mmio_addr & (SB800_ACPI_MMIO_DECODE_EN |
+- SB800_ACPI_MMIO_SEL)) !=
+- SB800_ACPI_MMIO_DECODE_EN) {
+- ret = -ENODEV;
+- goto unreg_region;
+- }
+- mmio_addr &= ~0xFFF;
+- mmio_addr += SB800_PM_WDT_MMIO_OFFSET;
+- break;
+- case efch:
+- val = sp5100_tco_read_pm_reg8(EFCH_PM_ISACONTROL);
+- if (!(val & EFCH_PM_ISACONTROL_MMIOEN)) {
+- ret = -ENODEV;
+- goto unreg_region;
+- }
+- mmio_addr = EFCH_PM_ACPI_MMIO_ADDR +
+- EFCH_PM_ACPI_MMIO_WDT_OFFSET;
+- break;
+- }
+- dev_dbg(dev, "Got 0x%08x from SBResource_MMIO register\n",
+- mmio_addr);
+- if (!devm_request_mem_region(dev, mmio_addr,
+- SP5100_WDT_MEM_MAP_SIZE,
+- dev_name)) {
+- dev_dbg(dev, "MMIO address 0x%08x already in use\n",
+- mmio_addr);
+- ret = -EBUSY;
+- goto unreg_region;
+- }
+- }
+-
+- tco->tcobase = devm_ioremap(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE);
+- if (!tco->tcobase) {
+- dev_err(dev, "failed to get tcobase address\n");
+- ret = -ENOMEM;
+- goto unreg_region;
+- }
+-
+- dev_info(dev, "Using 0x%08x for watchdog MMIO address\n", mmio_addr);
+-
+- /* Setup the watchdog timer */
+- tco_timer_enable(tco);
+-
+- val = readl(SP5100_WDT_CONTROL(tco->tcobase));
+- if (val & SP5100_WDT_DISABLED) {
+- dev_err(dev, "Watchdog hardware is disabled\n");
+- ret = -ENODEV;
+- goto unreg_region;
++ ret = sp5100_tco_prepare_base(tco, mmio_addr, alt_mmio_addr, dev_name);
++ if (!ret) {
++ /* Setup the watchdog timer */
++ tco_timer_enable(tco);
++ ret = sp5100_tco_timer_init(tco);
+ }
+
+- /*
+- * Save WatchDogFired status, because WatchDogFired flag is
+- * cleared here.
+- */
+- if (val & SP5100_WDT_FIRED)
+- wdd->bootstatus = WDIOF_CARDRESET;
+- /* Set watchdog action to reset the system */
+- val &= ~SP5100_WDT_ACTION_RESET;
+- writel(val, SP5100_WDT_CONTROL(tco->tcobase));
+-
+- /* Set a reasonable heartbeat before we stop the timer */
+- tco_timer_set_timeout(wdd, wdd->timeout);
+-
+- /*
+- * Stop the TCO before we change anything so we don't race with
+- * a zeroed timer.
+- */
+- tco_timer_stop(wdd);
+-
+- release_region(SP5100_IO_PM_INDEX_REG, SP5100_PM_IOPORTS_SIZE);
+-
+- return 0;
+-
+-unreg_region:
+ release_region(SP5100_IO_PM_INDEX_REG, SP5100_PM_IOPORTS_SIZE);
+ return ret;
+ }
+diff --git a/drivers/watchdog/sp5100_tco.h b/drivers/watchdog/sp5100_tco.h
+index adf015aa4126f..6a0986d2c94b7 100644
+--- a/drivers/watchdog/sp5100_tco.h
++++ b/drivers/watchdog/sp5100_tco.h
+@@ -58,6 +58,7 @@
+ #define SB800_PM_WATCHDOG_SECOND_RES GENMASK(1, 0)
+ #define SB800_ACPI_MMIO_DECODE_EN BIT(0)
+ #define SB800_ACPI_MMIO_SEL BIT(1)
++#define SB800_ACPI_MMIO_MASK GENMASK(1, 0)
+
+ #define SB800_PM_WDT_MMIO_OFFSET 0xB00
+
+@@ -82,4 +83,10 @@
+ #define EFCH_PM_ISACONTROL_MMIOEN BIT(1)
+
+ #define EFCH_PM_ACPI_MMIO_ADDR 0xfed80000
++#define EFCH_PM_ACPI_MMIO_PM_OFFSET 0x00000300
+ #define EFCH_PM_ACPI_MMIO_WDT_OFFSET 0x00000b00
++
++#define EFCH_PM_ACPI_MMIO_PM_ADDR (EFCH_PM_ACPI_MMIO_ADDR + \
++ EFCH_PM_ACPI_MMIO_PM_OFFSET)
++#define EFCH_PM_ACPI_MMIO_PM_SIZE 8
++#define AMD_ZEN_SMBUS_PCI_REV 0x51
+diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
+index 3a50f097ed3ed..8db96b5e72536 100644
+--- a/drivers/xen/balloon.c
++++ b/drivers/xen/balloon.c
+@@ -58,6 +58,7 @@
+ #include <linux/percpu-defs.h>
+ #include <linux/slab.h>
+ #include <linux/sysctl.h>
++#include <linux/moduleparam.h>
+
+ #include <asm/page.h>
+ #include <asm/tlb.h>
+@@ -73,6 +74,12 @@
+ #include <xen/page.h>
+ #include <xen/mem-reservation.h>
+
++#undef MODULE_PARAM_PREFIX
++#define MODULE_PARAM_PREFIX "xen."
++
++static uint __read_mostly balloon_boot_timeout = 180;
++module_param(balloon_boot_timeout, uint, 0444);
++
+ static int xen_hotplug_unpopulated;
+
+ #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
+@@ -125,12 +132,12 @@ static struct ctl_table xen_root[] = {
+ * BP_ECANCELED: error, balloon operation canceled.
+ */
+
+-enum bp_state {
++static enum bp_state {
+ BP_DONE,
+ BP_WAIT,
+ BP_EAGAIN,
+ BP_ECANCELED
+-};
++} balloon_state = BP_DONE;
+
+ /* Main waiting point for xen-balloon thread. */
+ static DECLARE_WAIT_QUEUE_HEAD(balloon_thread_wq);
+@@ -199,18 +206,15 @@ static struct page *balloon_next_page(struct page *page)
+ return list_entry(next, struct page, lru);
+ }
+
+-static enum bp_state update_schedule(enum bp_state state)
++static void update_schedule(void)
+ {
+- if (state == BP_WAIT)
+- return BP_WAIT;
+-
+- if (state == BP_ECANCELED)
+- return BP_ECANCELED;
++ if (balloon_state == BP_WAIT || balloon_state == BP_ECANCELED)
++ return;
+
+- if (state == BP_DONE) {
++ if (balloon_state == BP_DONE) {
+ balloon_stats.schedule_delay = 1;
+ balloon_stats.retry_count = 1;
+- return BP_DONE;
++ return;
+ }
+
+ ++balloon_stats.retry_count;
+@@ -219,7 +223,8 @@ static enum bp_state update_schedule(enum bp_state state)
+ balloon_stats.retry_count > balloon_stats.max_retry_count) {
+ balloon_stats.schedule_delay = 1;
+ balloon_stats.retry_count = 1;
+- return BP_ECANCELED;
++ balloon_state = BP_ECANCELED;
++ return;
+ }
+
+ balloon_stats.schedule_delay <<= 1;
+@@ -227,7 +232,7 @@ static enum bp_state update_schedule(enum bp_state state)
+ if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay)
+ balloon_stats.schedule_delay = balloon_stats.max_schedule_delay;
+
+- return BP_EAGAIN;
++ balloon_state = BP_EAGAIN;
+ }
+
+ #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
+@@ -494,9 +499,9 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
+ * Stop waiting if either state is BP_DONE and ballooning action is
+ * needed, or if the credit has changed while state is not BP_DONE.
+ */
+-static bool balloon_thread_cond(enum bp_state state, long credit)
++static bool balloon_thread_cond(long credit)
+ {
+- if (state == BP_DONE)
++ if (balloon_state == BP_DONE)
+ credit = 0;
+
+ return current_credit() != credit || kthread_should_stop();
+@@ -510,13 +515,12 @@ static bool balloon_thread_cond(enum bp_state state, long credit)
+ */
+ static int balloon_thread(void *unused)
+ {
+- enum bp_state state = BP_DONE;
+ long credit;
+ unsigned long timeout;
+
+ set_freezable();
+ for (;;) {
+- switch (state) {
++ switch (balloon_state) {
+ case BP_DONE:
+ case BP_ECANCELED:
+ timeout = 3600 * HZ;
+@@ -532,7 +536,7 @@ static int balloon_thread(void *unused)
+ credit = current_credit();
+
+ wait_event_freezable_timeout(balloon_thread_wq,
+- balloon_thread_cond(state, credit), timeout);
++ balloon_thread_cond(credit), timeout);
+
+ if (kthread_should_stop())
+ return 0;
+@@ -543,22 +547,23 @@ static int balloon_thread(void *unused)
+
+ if (credit > 0) {
+ if (balloon_is_inflated())
+- state = increase_reservation(credit);
++ balloon_state = increase_reservation(credit);
+ else
+- state = reserve_additional_memory();
++ balloon_state = reserve_additional_memory();
+ }
+
+ if (credit < 0) {
+ long n_pages;
+
+ n_pages = min(-credit, si_mem_available());
+- state = decrease_reservation(n_pages, GFP_BALLOON);
+- if (state == BP_DONE && n_pages != -credit &&
++ balloon_state = decrease_reservation(n_pages,
++ GFP_BALLOON);
++ if (balloon_state == BP_DONE && n_pages != -credit &&
+ n_pages < totalreserve_pages)
+- state = BP_EAGAIN;
++ balloon_state = BP_EAGAIN;
+ }
+
+- state = update_schedule(state);
++ update_schedule();
+
+ mutex_unlock(&balloon_mutex);
+
+@@ -765,3 +770,38 @@ static int __init balloon_init(void)
+ return 0;
+ }
+ subsys_initcall(balloon_init);
++
++static int __init balloon_wait_finish(void)
++{
++ long credit, last_credit = 0;
++ unsigned long last_changed = 0;
++
++ if (!xen_domain())
++ return -ENODEV;
++
++ /* PV guests don't need to wait. */
++ if (xen_pv_domain() || !current_credit())
++ return 0;
++
++ pr_notice("Waiting for initial ballooning down having finished.\n");
++
++ while ((credit = current_credit()) < 0) {
++ if (credit != last_credit) {
++ last_changed = jiffies;
++ last_credit = credit;
++ }
++ if (balloon_state == BP_ECANCELED) {
++ pr_warn_once("Initial ballooning failed, %ld pages need to be freed.\n",
++ -credit);
++ if (jiffies - last_changed >= HZ * balloon_boot_timeout)
++ panic("Initial ballooning failed!\n");
++ }
++
++ schedule_timeout_interruptible(HZ / 10);
++ }
++
++ pr_notice("Initial ballooning down finished.\n");
++
++ return 0;
++}
++late_initcall_sync(balloon_wait_finish);
+diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
+index a78704ae36186..46d9295d9a6e4 100644
+--- a/drivers/xen/events/events_base.c
++++ b/drivers/xen/events/events_base.c
+@@ -1251,6 +1251,12 @@ int bind_evtchn_to_irq(evtchn_port_t evtchn)
+ }
+ EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
+
++int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn)
++{
++ return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip, NULL);
++}
++EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi);
++
+ static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
+ {
+ struct evtchn_bind_ipi bind_ipi;
+diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c
+index 3fa40c723e8e9..edb0acd0b8323 100644
+--- a/drivers/xen/gntalloc.c
++++ b/drivers/xen/gntalloc.c
+@@ -169,20 +169,14 @@ undo:
+ __del_gref(gref);
+ }
+
+- /* It's possible for the target domain to map the just-allocated grant
+- * references by blindly guessing their IDs; if this is done, then
+- * __del_gref will leave them in the queue_gref list. They need to be
+- * added to the global list so that we can free them when they are no
+- * longer referenced.
+- */
+- if (unlikely(!list_empty(&queue_gref)))
+- list_splice_tail(&queue_gref, &gref_list);
+ mutex_unlock(&gref_mutex);
+ return rc;
+ }
+
+ static void __del_gref(struct gntalloc_gref *gref)
+ {
++ unsigned long addr;
++
+ if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
+ uint8_t *tmp = kmap(gref->page);
+ tmp[gref->notify.pgoff] = 0;
+@@ -196,21 +190,16 @@ static void __del_gref(struct gntalloc_gref *gref)
+ gref->notify.flags = 0;
+
+ if (gref->gref_id) {
+- if (gnttab_query_foreign_access(gref->gref_id))
+- return;
+-
+- if (!gnttab_end_foreign_access_ref(gref->gref_id, 0))
+- return;
+-
+- gnttab_free_grant_reference(gref->gref_id);
++ if (gref->page) {
++ addr = (unsigned long)page_to_virt(gref->page);
++ gnttab_end_foreign_access(gref->gref_id, 0, addr);
++ } else
++ gnttab_free_grant_reference(gref->gref_id);
+ }
+
+ gref_size--;
+ list_del(&gref->next_gref);
+
+- if (gref->page)
+- __free_page(gref->page);
+-
+ kfree(gref);
+ }
+
+diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
+index fec1b65371665..59ffea8000791 100644
+--- a/drivers/xen/gntdev.c
++++ b/drivers/xen/gntdev.c
+@@ -250,13 +250,13 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)
+ if (!refcount_dec_and_test(&map->users))
+ return;
+
++ if (map->pages && !use_ptemod)
++ unmap_grant_pages(map, 0, map->count);
++
+ if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {
+ notify_remote_via_evtchn(map->notify.event);
+ evtchn_put(map->notify.event);
+ }
+-
+- if (map->pages && !use_ptemod)
+- unmap_grant_pages(map, 0, map->count);
+ gntdev_free_map(map);
+ }
+
+diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
+index 3729bea0c9895..5c83d41766c85 100644
+--- a/drivers/xen/grant-table.c
++++ b/drivers/xen/grant-table.c
+@@ -134,12 +134,9 @@ struct gnttab_ops {
+ */
+ unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
+ /*
+- * Query the status of a grant entry. Ref parameter is reference of
+- * queried grant entry, return value is the status of queried entry.
+- * Detailed status(writing/reading) can be gotten from the return value
+- * by bit operations.
++ * Read the frame number related to a given grant reference.
+ */
+- int (*query_foreign_access)(grant_ref_t ref);
++ unsigned long (*read_frame)(grant_ref_t ref);
+ };
+
+ struct unmap_refs_callback_data {
+@@ -284,22 +281,6 @@ int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
+ }
+ EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
+
+-static int gnttab_query_foreign_access_v1(grant_ref_t ref)
+-{
+- return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
+-}
+-
+-static int gnttab_query_foreign_access_v2(grant_ref_t ref)
+-{
+- return grstatus[ref] & (GTF_reading|GTF_writing);
+-}
+-
+-int gnttab_query_foreign_access(grant_ref_t ref)
+-{
+- return gnttab_interface->query_foreign_access(ref);
+-}
+-EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
+-
+ static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
+ {
+ u16 flags, nflags;
+@@ -353,6 +334,16 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
+ }
+ EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
+
++static unsigned long gnttab_read_frame_v1(grant_ref_t ref)
++{
++ return gnttab_shared.v1[ref].frame;
++}
++
++static unsigned long gnttab_read_frame_v2(grant_ref_t ref)
++{
++ return gnttab_shared.v2[ref].full_page.frame;
++}
++
+ struct deferred_entry {
+ struct list_head list;
+ grant_ref_t ref;
+@@ -382,12 +373,9 @@ static void gnttab_handle_deferred(struct timer_list *unused)
+ spin_unlock_irqrestore(&gnttab_list_lock, flags);
+ if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
+ put_free_entry(entry->ref);
+- if (entry->page) {
+- pr_debug("freeing g.e. %#x (pfn %#lx)\n",
+- entry->ref, page_to_pfn(entry->page));
+- put_page(entry->page);
+- } else
+- pr_info("freeing g.e. %#x\n", entry->ref);
++ pr_debug("freeing g.e. %#x (pfn %#lx)\n",
++ entry->ref, page_to_pfn(entry->page));
++ put_page(entry->page);
+ kfree(entry);
+ entry = NULL;
+ } else {
+@@ -412,9 +400,18 @@ static void gnttab_handle_deferred(struct timer_list *unused)
+ static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
+ struct page *page)
+ {
+- struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
++ struct deferred_entry *entry;
++ gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;
+ const char *what = KERN_WARNING "leaking";
+
++ entry = kmalloc(sizeof(*entry), gfp);
++ if (!page) {
++ unsigned long gfn = gnttab_interface->read_frame(ref);
++
++ page = pfn_to_page(gfn_to_pfn(gfn));
++ get_page(page);
++ }
++
+ if (entry) {
+ unsigned long flags;
+
+@@ -435,11 +432,21 @@ static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
+ what, ref, page ? page_to_pfn(page) : -1);
+ }
+
++int gnttab_try_end_foreign_access(grant_ref_t ref)
++{
++ int ret = _gnttab_end_foreign_access_ref(ref, 0);
++
++ if (ret)
++ put_free_entry(ref);
++
++ return ret;
++}
++EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);
++
+ void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
+ unsigned long page)
+ {
+- if (gnttab_end_foreign_access_ref(ref, readonly)) {
+- put_free_entry(ref);
++ if (gnttab_try_end_foreign_access(ref)) {
+ if (page != 0)
+ put_page(virt_to_page(page));
+ } else
+@@ -1417,7 +1424,7 @@ static const struct gnttab_ops gnttab_v1_ops = {
+ .update_entry = gnttab_update_entry_v1,
+ .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
+ .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
+- .query_foreign_access = gnttab_query_foreign_access_v1,
++ .read_frame = gnttab_read_frame_v1,
+ };
+
+ static const struct gnttab_ops gnttab_v2_ops = {
+@@ -1429,7 +1436,7 @@ static const struct gnttab_ops gnttab_v2_ops = {
+ .update_entry = gnttab_update_entry_v2,
+ .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
+ .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
+- .query_foreign_access = gnttab_query_foreign_access_v2,
++ .read_frame = gnttab_read_frame_v2,
+ };
+
+ static bool gnttab_need_v2(void)
+diff --git a/drivers/xen/pvcalls-front.c b/drivers/xen/pvcalls-front.c
+index 7984645b59563..bbe337dc296e3 100644
+--- a/drivers/xen/pvcalls-front.c
++++ b/drivers/xen/pvcalls-front.c
+@@ -337,8 +337,8 @@ static void free_active_ring(struct sock_mapping *map)
+ if (!map->active.ring)
+ return;
+
+- free_pages((unsigned long)map->active.data.in,
+- map->active.ring->ring_order);
++ free_pages_exact(map->active.data.in,
++ PAGE_SIZE << map->active.ring->ring_order);
+ free_page((unsigned long)map->active.ring);
+ }
+
+@@ -352,8 +352,8 @@ static int alloc_active_ring(struct sock_mapping *map)
+ goto out;
+
+ map->active.ring->ring_order = PVCALLS_RING_ORDER;
+- bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+- PVCALLS_RING_ORDER);
++ bytes = alloc_pages_exact(PAGE_SIZE << PVCALLS_RING_ORDER,
++ GFP_KERNEL | __GFP_ZERO);
+ if (!bytes)
+ goto out;
+
+diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
+index e56a5faac395c..cbdff89799807 100644
+--- a/drivers/xen/swiotlb-xen.c
++++ b/drivers/xen/swiotlb-xen.c
+@@ -380,7 +380,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
+ */
+ trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
+
+- map = swiotlb_tbl_map_single(dev, phys, size, size, dir, attrs);
++ map = swiotlb_tbl_map_single(dev, phys, size, size, 0, dir, attrs);
+ if (map == (phys_addr_t)DMA_MAPPING_ERROR)
+ return DMA_MAPPING_ERROR;
+
+diff --git a/drivers/xen/xen-pciback/conf_space_capability.c b/drivers/xen/xen-pciback/conf_space_capability.c
+index 22f13abbe9130..5e53b4817f167 100644
+--- a/drivers/xen/xen-pciback/conf_space_capability.c
++++ b/drivers/xen/xen-pciback/conf_space_capability.c
+@@ -160,7 +160,7 @@ static void *pm_ctrl_init(struct pci_dev *dev, int offset)
+ }
+
+ out:
+- return ERR_PTR(err);
++ return err ? ERR_PTR(err) : NULL;
+ }
+
+ static const struct config_field caplist_pm[] = {
+diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c
+index e8bed1cb76ba2..df68906812315 100644
+--- a/drivers/xen/xenbus/xenbus_client.c
++++ b/drivers/xen/xenbus/xenbus_client.c
+@@ -379,7 +379,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr,
+ unsigned int nr_pages, grant_ref_t *grefs)
+ {
+ int err;
+- int i, j;
++ unsigned int i;
++ grant_ref_t gref_head;
++
++ err = gnttab_alloc_grant_references(nr_pages, &gref_head);
++ if (err) {
++ xenbus_dev_fatal(dev, err, "granting access to ring page");
++ return err;
++ }
+
+ for (i = 0; i < nr_pages; i++) {
+ unsigned long gfn;
+@@ -389,23 +396,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr,
+ else
+ gfn = virt_to_gfn(vaddr);
+
+- err = gnttab_grant_foreign_access(dev->otherend_id, gfn, 0);
+- if (err < 0) {
+- xenbus_dev_fatal(dev, err,
+- "granting access to ring page");
+- goto fail;
+- }
+- grefs[i] = err;
++ grefs[i] = gnttab_claim_grant_reference(&gref_head);
++ gnttab_grant_foreign_access_ref(grefs[i], dev->otherend_id,
++ gfn, 0);
+
+ vaddr = vaddr + XEN_PAGE_SIZE;
+ }
+
+ return 0;
+-
+-fail:
+- for (j = 0; j < i; j++)
+- gnttab_end_foreign_access_ref(grefs[j], 0);
+- return err;
+ }
+ EXPORT_SYMBOL_GPL(xenbus_grant_ring);
+
+diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
+index bd003ca8acbe9..fe360c33ce717 100644
+--- a/drivers/xen/xenbus/xenbus_probe.c
++++ b/drivers/xen/xenbus/xenbus_probe.c
+@@ -909,7 +909,7 @@ static struct notifier_block xenbus_resume_nb = {
+
+ static int __init xenbus_init(void)
+ {
+- int err = 0;
++ int err;
+ uint64_t v = 0;
+ xen_store_domain_type = XS_UNKNOWN;
+
+@@ -949,6 +949,29 @@ static int __init xenbus_init(void)
+ err = hvm_get_parameter(HVM_PARAM_STORE_PFN, &v);
+ if (err)
+ goto out_error;
++ /*
++ * Uninitialized hvm_params are zero and return no error.
++ * Although it is theoretically possible to have
++ * HVM_PARAM_STORE_PFN set to zero on purpose, in reality it is
++ * not zero when valid. If zero, it means that Xenstore hasn't
++ * been properly initialized. Instead of attempting to map a
++ * wrong guest physical address return error.
++ *
++ * Also recognize all bits set as an invalid value.
++ */
++ if (!v || !~v) {
++ err = -ENOENT;
++ goto out_error;
++ }
++ /* Avoid truncation on 32-bit. */
++#if BITS_PER_LONG == 32
++ if (v > ULONG_MAX) {
++ pr_err("%s: cannot handle HVM_PARAM_STORE_PFN=%llx > ULONG_MAX\n",
++ __func__, v);
++ err = -EINVAL;
++ goto out_error;
++ }
++#endif
+ xen_store_gfn = (unsigned long)v;
+ xen_store_interface =
+ xen_remap(xen_store_gfn << XEN_PAGE_SHIFT,
+@@ -983,8 +1006,10 @@ static int __init xenbus_init(void)
+ */
+ proc_create_mount_point("xen");
+ #endif
++ return 0;
+
+ out_error:
++ xen_store_domain_type = XS_UNKNOWN;
+ return err;
+ }
+
+diff --git a/fs/9p/fid.c b/fs/9p/fid.c
+index b8863dd0de5cc..e8a3b891b0366 100644
+--- a/fs/9p/fid.c
++++ b/fs/9p/fid.c
+@@ -96,12 +96,8 @@ static struct p9_fid *v9fs_fid_find(struct dentry *dentry, kuid_t uid, int any)
+ dentry, dentry, from_kuid(&init_user_ns, uid),
+ any);
+ ret = NULL;
+-
+- if (d_inode(dentry))
+- ret = v9fs_fid_find_inode(d_inode(dentry), uid);
+-
+ /* we'll recheck under lock if there's anything to look in */
+- if (!ret && dentry->d_fsdata) {
++ if (dentry->d_fsdata) {
+ struct hlist_head *h = (struct hlist_head *)&dentry->d_fsdata;
+ spin_lock(&dentry->d_lock);
+ hlist_for_each_entry(fid, h, dlist) {
+@@ -112,6 +108,9 @@ static struct p9_fid *v9fs_fid_find(struct dentry *dentry, kuid_t uid, int any)
+ }
+ }
+ spin_unlock(&dentry->d_lock);
++ } else {
++ if (dentry->d_inode)
++ ret = v9fs_fid_find_inode(dentry->d_inode, uid);
+ }
+
+ return ret;
+diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
+index 01b9e1281a297..a61df2e0ae521 100644
+--- a/fs/9p/vfs_inode_dotl.c
++++ b/fs/9p/vfs_inode_dotl.c
+@@ -553,7 +553,10 @@ int v9fs_vfs_setattr_dotl(struct user_namespace *mnt_userns,
+ {
+ int retval, use_dentry = 0;
+ struct p9_fid *fid = NULL;
+- struct p9_iattr_dotl p9attr;
++ struct p9_iattr_dotl p9attr = {
++ .uid = INVALID_UID,
++ .gid = INVALID_GID,
++ };
+ struct inode *inode = d_inode(dentry);
+
+ p9_debug(P9_DEBUG_VFS, "\n");
+@@ -563,14 +566,22 @@ int v9fs_vfs_setattr_dotl(struct user_namespace *mnt_userns,
+ return retval;
+
+ p9attr.valid = v9fs_mapped_iattr_valid(iattr->ia_valid);
+- p9attr.mode = iattr->ia_mode;
+- p9attr.uid = iattr->ia_uid;
+- p9attr.gid = iattr->ia_gid;
+- p9attr.size = iattr->ia_size;
+- p9attr.atime_sec = iattr->ia_atime.tv_sec;
+- p9attr.atime_nsec = iattr->ia_atime.tv_nsec;
+- p9attr.mtime_sec = iattr->ia_mtime.tv_sec;
+- p9attr.mtime_nsec = iattr->ia_mtime.tv_nsec;
++ if (iattr->ia_valid & ATTR_MODE)
++ p9attr.mode = iattr->ia_mode;
++ if (iattr->ia_valid & ATTR_UID)
++ p9attr.uid = iattr->ia_uid;
++ if (iattr->ia_valid & ATTR_GID)
++ p9attr.gid = iattr->ia_gid;
++ if (iattr->ia_valid & ATTR_SIZE)
++ p9attr.size = iattr->ia_size;
++ if (iattr->ia_valid & ATTR_ATIME_SET) {
++ p9attr.atime_sec = iattr->ia_atime.tv_sec;
++ p9attr.atime_nsec = iattr->ia_atime.tv_nsec;
++ }
++ if (iattr->ia_valid & ATTR_MTIME_SET) {
++ p9attr.mtime_sec = iattr->ia_mtime.tv_sec;
++ p9attr.mtime_nsec = iattr->ia_mtime.tv_nsec;
++ }
+
+ if (iattr->ia_valid & ATTR_FILE) {
+ fid = iattr->ia_file->private_data;
+diff --git a/fs/afs/file.c b/fs/afs/file.c
+index e6c447ae91f38..b165377179c3c 100644
+--- a/fs/afs/file.c
++++ b/fs/afs/file.c
+@@ -502,8 +502,9 @@ static void afs_add_open_mmap(struct afs_vnode *vnode)
+ if (atomic_inc_return(&vnode->cb_nr_mmap) == 1) {
+ down_write(&vnode->volume->cell->fs_open_mmaps_lock);
+
+- list_add_tail(&vnode->cb_mmap_link,
+- &vnode->volume->cell->fs_open_mmaps);
++ if (list_empty(&vnode->cb_mmap_link))
++ list_add_tail(&vnode->cb_mmap_link,
++ &vnode->volume->cell->fs_open_mmaps);
+
+ up_write(&vnode->volume->cell->fs_open_mmaps_lock);
+ }
+diff --git a/fs/afs/inode.c b/fs/afs/inode.c
+index 8fcffea2daf50..a47666ba48f56 100644
+--- a/fs/afs/inode.c
++++ b/fs/afs/inode.c
+@@ -728,10 +728,22 @@ int afs_getattr(struct user_namespace *mnt_userns, const struct path *path,
+ {
+ struct inode *inode = d_inode(path->dentry);
+ struct afs_vnode *vnode = AFS_FS_I(inode);
+- int seq = 0;
++ struct key *key;
++ int ret, seq = 0;
+
+ _enter("{ ino=%lu v=%u }", inode->i_ino, inode->i_generation);
+
++ if (!(query_flags & AT_STATX_DONT_SYNC) &&
++ !test_bit(AFS_VNODE_CB_PROMISED, &vnode->flags)) {
++ key = afs_request_key(vnode->volume->cell);
++ if (IS_ERR(key))
++ return PTR_ERR(key);
++ ret = afs_validate(vnode, key);
++ key_put(key);
++ if (ret < 0)
++ return ret;
++ }
++
+ do {
+ read_seqbegin_or_lock(&vnode->cb_lock, &seq);
+ generic_fillattr(&init_user_ns, inode, stat);
+diff --git a/fs/afs/super.c b/fs/afs/super.c
+index d110def8aa8eb..34c68724c98be 100644
+--- a/fs/afs/super.c
++++ b/fs/afs/super.c
+@@ -667,6 +667,7 @@ static void afs_i_init_once(void *_vnode)
+ INIT_LIST_HEAD(&vnode->pending_locks);
+ INIT_LIST_HEAD(&vnode->granted_locks);
+ INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work);
++ INIT_LIST_HEAD(&vnode->cb_mmap_link);
+ seqlock_init(&vnode->cb_lock);
+ }
+
+diff --git a/fs/aio.c b/fs/aio.c
+index 51b08ab01dffc..1a78979663dca 100644
+--- a/fs/aio.c
++++ b/fs/aio.c
+@@ -181,8 +181,9 @@ struct poll_iocb {
+ struct file *file;
+ struct wait_queue_head *head;
+ __poll_t events;
+- bool done;
+ bool cancelled;
++ bool work_scheduled;
++ bool work_need_resched;
+ struct wait_queue_entry wait;
+ struct work_struct work;
+ };
+@@ -1620,6 +1621,51 @@ static void aio_poll_put_work(struct work_struct *work)
+ iocb_put(iocb);
+ }
+
++/*
++ * Safely lock the waitqueue which the request is on, synchronizing with the
++ * case where the ->poll() provider decides to free its waitqueue early.
++ *
++ * Returns true on success, meaning that req->head->lock was locked, req->wait
++ * is on req->head, and an RCU read lock was taken. Returns false if the
++ * request was already removed from its waitqueue (which might no longer exist).
++ */
++static bool poll_iocb_lock_wq(struct poll_iocb *req)
++{
++ wait_queue_head_t *head;
++
++ /*
++ * While we hold the waitqueue lock and the waitqueue is nonempty,
++ * wake_up_pollfree() will wait for us. However, taking the waitqueue
++ * lock in the first place can race with the waitqueue being freed.
++ *
++ * We solve this as eventpoll does: by taking advantage of the fact that
++ * all users of wake_up_pollfree() will RCU-delay the actual free. If
++ * we enter rcu_read_lock() and see that the pointer to the queue is
++ * non-NULL, we can then lock it without the memory being freed out from
++ * under us, then check whether the request is still on the queue.
++ *
++ * Keep holding rcu_read_lock() as long as we hold the queue lock, in
++ * case the caller deletes the entry from the queue, leaving it empty.
++ * In that case, only RCU prevents the queue memory from being freed.
++ */
++ rcu_read_lock();
++ head = smp_load_acquire(&req->head);
++ if (head) {
++ spin_lock(&head->lock);
++ if (!list_empty(&req->wait.entry))
++ return true;
++ spin_unlock(&head->lock);
++ }
++ rcu_read_unlock();
++ return false;
++}
++
++static void poll_iocb_unlock_wq(struct poll_iocb *req)
++{
++ spin_unlock(&req->head->lock);
++ rcu_read_unlock();
++}
++
+ static void aio_poll_complete_work(struct work_struct *work)
+ {
+ struct poll_iocb *req = container_of(work, struct poll_iocb, work);
+@@ -1639,14 +1685,27 @@ static void aio_poll_complete_work(struct work_struct *work)
+ * avoid further branches in the fast path.
+ */
+ spin_lock_irq(&ctx->ctx_lock);
+- if (!mask && !READ_ONCE(req->cancelled)) {
+- add_wait_queue(req->head, &req->wait);
+- spin_unlock_irq(&ctx->ctx_lock);
+- return;
+- }
++ if (poll_iocb_lock_wq(req)) {
++ if (!mask && !READ_ONCE(req->cancelled)) {
++ /*
++ * The request isn't actually ready to be completed yet.
++ * Reschedule completion if another wakeup came in.
++ */
++ if (req->work_need_resched) {
++ schedule_work(&req->work);
++ req->work_need_resched = false;
++ } else {
++ req->work_scheduled = false;
++ }
++ poll_iocb_unlock_wq(req);
++ spin_unlock_irq(&ctx->ctx_lock);
++ return;
++ }
++ list_del_init(&req->wait.entry);
++ poll_iocb_unlock_wq(req);
++ } /* else, POLLFREE has freed the waitqueue, so we must complete */
+ list_del_init(&iocb->ki_list);
+ iocb->ki_res.res = mangle_poll(mask);
+- req->done = true;
+ spin_unlock_irq(&ctx->ctx_lock);
+
+ iocb_put(iocb);
+@@ -1658,13 +1717,14 @@ static int aio_poll_cancel(struct kiocb *iocb)
+ struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);
+ struct poll_iocb *req = &aiocb->poll;
+
+- spin_lock(&req->head->lock);
+- WRITE_ONCE(req->cancelled, true);
+- if (!list_empty(&req->wait.entry)) {
+- list_del_init(&req->wait.entry);
+- schedule_work(&aiocb->poll.work);
+- }
+- spin_unlock(&req->head->lock);
++ if (poll_iocb_lock_wq(req)) {
++ WRITE_ONCE(req->cancelled, true);
++ if (!req->work_scheduled) {
++ schedule_work(&aiocb->poll.work);
++ req->work_scheduled = true;
++ }
++ poll_iocb_unlock_wq(req);
++ } /* else, the request was force-cancelled by POLLFREE already */
+
+ return 0;
+ }
+@@ -1681,21 +1741,27 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
+ if (mask && !(mask & req->events))
+ return 0;
+
+- list_del_init(&req->wait.entry);
+-
+- if (mask && spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) {
++ /*
++ * Complete the request inline if possible. This requires that three
++ * conditions be met:
++ * 1. An event mask must have been passed. If a plain wakeup was done
++ * instead, then mask == 0 and we have to call vfs_poll() to get
++ * the events, so inline completion isn't possible.
++ * 2. The completion work must not have already been scheduled.
++ * 3. ctx_lock must not be busy. We have to use trylock because we
++ * already hold the waitqueue lock, so this inverts the normal
++ * locking order. Use irqsave/irqrestore because not all
++ * filesystems (e.g. fuse) call this function with IRQs disabled,
++ * yet IRQs have to be disabled before ctx_lock is obtained.
++ */
++ if (mask && !req->work_scheduled &&
++ spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) {
+ struct kioctx *ctx = iocb->ki_ctx;
+
+- /*
+- * Try to complete the iocb inline if we can. Use
+- * irqsave/irqrestore because not all filesystems (e.g. fuse)
+- * call this function with IRQs disabled and because IRQs
+- * have to be disabled before ctx_lock is obtained.
+- */
++ list_del_init(&req->wait.entry);
+ list_del(&iocb->ki_list);
+ iocb->ki_res.res = mangle_poll(mask);
+- req->done = true;
+- if (iocb->ki_eventfd && eventfd_signal_allowed()) {
++ if (iocb->ki_eventfd && !eventfd_signal_allowed()) {
+ iocb = NULL;
+ INIT_WORK(&req->work, aio_poll_put_work);
+ schedule_work(&req->work);
+@@ -1704,7 +1770,43 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
+ if (iocb)
+ iocb_put(iocb);
+ } else {
+- schedule_work(&req->work);
++ /*
++ * Schedule the completion work if needed. If it was already
++ * scheduled, record that another wakeup came in.
++ *
++ * Don't remove the request from the waitqueue here, as it might
++ * not actually be complete yet (we won't know until vfs_poll()
++ * is called), and we must not miss any wakeups. POLLFREE is an
++ * exception to this; see below.
++ */
++ if (req->work_scheduled) {
++ req->work_need_resched = true;
++ } else {
++ schedule_work(&req->work);
++ req->work_scheduled = true;
++ }
++
++ /*
++ * If the waitqueue is being freed early but we can't complete
++ * the request inline, we have to tear down the request as best
++ * we can. That means immediately removing the request from its
++ * waitqueue and preventing all further accesses to the
++ * waitqueue via the request. We also need to schedule the
++ * completion work (done above). Also mark the request as
++ * cancelled, to potentially skip an unneeded call to ->poll().
++ */
++ if (mask & POLLFREE) {
++ WRITE_ONCE(req->cancelled, true);
++ list_del_init(&req->wait.entry);
++
++ /*
++ * Careful: this *must* be the last step, since as soon
++ * as req->head is NULL'ed out, the request can be
++ * completed and freed, since aio_poll_complete_work()
++ * will no longer need to take the waitqueue lock.
++ */
++ smp_store_release(&req->head, NULL);
++ }
+ }
+ return 1;
+ }
+@@ -1712,6 +1814,7 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
+ struct aio_poll_table {
+ struct poll_table_struct pt;
+ struct aio_kiocb *iocb;
++ bool queued;
+ int error;
+ };
+
+@@ -1722,11 +1825,12 @@ aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,
+ struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt);
+
+ /* multiple wait queues per file are not supported */
+- if (unlikely(pt->iocb->poll.head)) {
++ if (unlikely(pt->queued)) {
+ pt->error = -EINVAL;
+ return;
+ }
+
++ pt->queued = true;
+ pt->error = 0;
+ pt->iocb->poll.head = head;
+ add_wait_queue(head, &pt->iocb->poll.wait);
+@@ -1751,12 +1855,14 @@ static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
+ req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
+
+ req->head = NULL;
+- req->done = false;
+ req->cancelled = false;
++ req->work_scheduled = false;
++ req->work_need_resched = false;
+
+ apt.pt._qproc = aio_poll_queue_proc;
+ apt.pt._key = req->events;
+ apt.iocb = aiocb;
++ apt.queued = false;
+ apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
+
+ /* initialized the list so that we can do list_empty checks */
+@@ -1765,23 +1871,35 @@ static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
+
+ mask = vfs_poll(req->file, &apt.pt) & req->events;
+ spin_lock_irq(&ctx->ctx_lock);
+- if (likely(req->head)) {
+- spin_lock(&req->head->lock);
+- if (unlikely(list_empty(&req->wait.entry))) {
+- if (apt.error)
++ if (likely(apt.queued)) {
++ bool on_queue = poll_iocb_lock_wq(req);
++
++ if (!on_queue || req->work_scheduled) {
++ /*
++ * aio_poll_wake() already either scheduled the async
++ * completion work, or completed the request inline.
++ */
++ if (apt.error) /* unsupported case: multiple queues */
+ cancel = true;
+ apt.error = 0;
+ mask = 0;
+ }
+ if (mask || apt.error) {
++ /* Steal to complete synchronously. */
+ list_del_init(&req->wait.entry);
+ } else if (cancel) {
++ /* Cancel if possible (may be too late though). */
+ WRITE_ONCE(req->cancelled, true);
+- } else if (!req->done) { /* actually waiting for an event */
++ } else if (on_queue) {
++ /*
++ * Actually waiting for an event, so add the request to
++ * active_reqs so that it can be cancelled if needed.
++ */
+ list_add_tail(&aiocb->ki_list, &ctx->active_reqs);
+ aiocb->ki_cancel = aio_poll_cancel;
+ }
+- spin_unlock(&req->head->lock);
++ if (on_queue)
++ poll_iocb_unlock_wq(req);
+ }
+ if (mask) { /* no async, we'd stolen it */
+ aiocb->ki_res.res = mangle_poll(mask);
+diff --git a/fs/attr.c b/fs/attr.c
+index 473d21b3a86de..66899b6e9bd86 100644
+--- a/fs/attr.c
++++ b/fs/attr.c
+@@ -35,7 +35,7 @@ static bool chown_ok(struct user_namespace *mnt_userns,
+ kuid_t uid)
+ {
+ kuid_t kuid = i_uid_into_mnt(mnt_userns, inode);
+- if (uid_eq(current_fsuid(), kuid) && uid_eq(uid, kuid))
++ if (uid_eq(current_fsuid(), kuid) && uid_eq(uid, inode->i_uid))
+ return true;
+ if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_CHOWN))
+ return true;
+@@ -62,7 +62,7 @@ static bool chgrp_ok(struct user_namespace *mnt_userns,
+ {
+ kgid_t kgid = i_gid_into_mnt(mnt_userns, inode);
+ if (uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode)) &&
+- (in_group_p(gid) || gid_eq(gid, kgid)))
++ (in_group_p(gid) || gid_eq(gid, inode->i_gid)))
+ return true;
+ if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_CHOWN))
+ return true;
+diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
+index a813b70f594e6..c93150f36a52a 100644
+--- a/fs/binfmt_elf.c
++++ b/fs/binfmt_elf.c
+@@ -170,8 +170,8 @@ static int padzero(unsigned long elf_bss)
+
+ static int
+ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
+- unsigned long load_addr, unsigned long interp_load_addr,
+- unsigned long e_entry)
++ unsigned long interp_load_addr,
++ unsigned long e_entry, unsigned long phdr_addr)
+ {
+ struct mm_struct *mm = current->mm;
+ unsigned long p = bprm->p;
+@@ -257,7 +257,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
+ NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP);
+ NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE);
+ NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC);
+- NEW_AUX_ENT(AT_PHDR, load_addr + exec->e_phoff);
++ NEW_AUX_ENT(AT_PHDR, phdr_addr);
+ NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr));
+ NEW_AUX_ENT(AT_PHNUM, exec->e_phnum);
+ NEW_AUX_ENT(AT_BASE, interp_load_addr);
+@@ -823,7 +823,7 @@ static int parse_elf_properties(struct file *f, const struct elf_phdr *phdr,
+ static int load_elf_binary(struct linux_binprm *bprm)
+ {
+ struct file *interpreter = NULL; /* to shut gcc up */
+- unsigned long load_addr = 0, load_bias = 0;
++ unsigned long load_addr, load_bias = 0, phdr_addr = 0;
+ int load_addr_set = 0;
+ unsigned long error;
+ struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL;
+@@ -1156,6 +1156,17 @@ out_free_interp:
+ reloc_func_desc = load_bias;
+ }
+ }
++
++ /*
++ * Figure out which segment in the file contains the Program
++ * Header table, and map to the associated memory address.
++ */
++ if (elf_ppnt->p_offset <= elf_ex->e_phoff &&
++ elf_ex->e_phoff < elf_ppnt->p_offset + elf_ppnt->p_filesz) {
++ phdr_addr = elf_ex->e_phoff - elf_ppnt->p_offset +
++ elf_ppnt->p_vaddr;
++ }
++
+ k = elf_ppnt->p_vaddr;
+ if ((elf_ppnt->p_flags & PF_X) && k < start_code)
+ start_code = k;
+@@ -1191,6 +1202,7 @@ out_free_interp:
+ }
+
+ e_entry = elf_ex->e_entry + load_bias;
++ phdr_addr += load_bias;
+ elf_bss += load_bias;
+ elf_brk += load_bias;
+ start_code += load_bias;
+@@ -1254,8 +1266,8 @@ out_free_interp:
+ goto out;
+ #endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */
+
+- retval = create_elf_tables(bprm, elf_ex,
+- load_addr, interp_load_addr, e_entry);
++ retval = create_elf_tables(bprm, elf_ex, interp_load_addr,
++ e_entry, phdr_addr);
+ if (retval < 0)
+ goto out;
+
+@@ -1606,17 +1618,16 @@ static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata,
+ * long file_ofs
+ * followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL...
+ */
+-static int fill_files_note(struct memelfnote *note)
++static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm)
+ {
+- struct mm_struct *mm = current->mm;
+- struct vm_area_struct *vma;
+ unsigned count, size, names_ofs, remaining, n;
+ user_long_t *data;
+ user_long_t *start_end_ofs;
+ char *name_base, *name_curpos;
++ int i;
+
+ /* *Estimated* file count and total data size needed */
+- count = mm->map_count;
++ count = cprm->vma_count;
+ if (count > UINT_MAX / 64)
+ return -EINVAL;
+ size = count * 64;
+@@ -1638,11 +1649,12 @@ static int fill_files_note(struct memelfnote *note)
+ name_base = name_curpos = ((char *)data) + names_ofs;
+ remaining = size - names_ofs;
+ count = 0;
+- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct core_vma_metadata *m = &cprm->vma_meta[i];
+ struct file *file;
+ const char *filename;
+
+- file = vma->vm_file;
++ file = m->file;
+ if (!file)
+ continue;
+ filename = file_path(file, name_curpos, remaining);
+@@ -1662,9 +1674,9 @@ static int fill_files_note(struct memelfnote *note)
+ memmove(name_curpos, filename, n);
+ name_curpos += n;
+
+- *start_end_ofs++ = vma->vm_start;
+- *start_end_ofs++ = vma->vm_end;
+- *start_end_ofs++ = vma->vm_pgoff;
++ *start_end_ofs++ = m->start;
++ *start_end_ofs++ = m->end;
++ *start_end_ofs++ = m->pgoff;
+ count++;
+ }
+
+@@ -1675,7 +1687,7 @@ static int fill_files_note(struct memelfnote *note)
+ * Count usually is less than mm->map_count,
+ * we need to move filenames down.
+ */
+- n = mm->map_count - count;
++ n = cprm->vma_count - count;
+ if (n != 0) {
+ unsigned shift_bytes = n * 3 * sizeof(data[0]);
+ memmove(name_base - shift_bytes, name_base,
+@@ -1787,7 +1799,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t,
+
+ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ struct elf_note_info *info,
+- const kernel_siginfo_t *siginfo, struct pt_regs *regs)
++ struct coredump_params *cprm)
+ {
+ struct task_struct *dump_task = current;
+ const struct user_regset_view *view = task_user_regset_view(dump_task);
+@@ -1859,7 +1871,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ * Now fill in each thread's information.
+ */
+ for (t = info->thread; t != NULL; t = t->next)
+- if (!fill_thread_core_info(t, view, siginfo->si_signo, &info->size))
++ if (!fill_thread_core_info(t, view, cprm->siginfo->si_signo, &info->size))
+ return 0;
+
+ /*
+@@ -1868,13 +1880,13 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ fill_psinfo(psinfo, dump_task->group_leader, dump_task->mm);
+ info->size += notesize(&info->psinfo);
+
+- fill_siginfo_note(&info->signote, &info->csigdata, siginfo);
++ fill_siginfo_note(&info->signote, &info->csigdata, cprm->siginfo);
+ info->size += notesize(&info->signote);
+
+ fill_auxv_note(&info->auxv, current->mm);
+ info->size += notesize(&info->auxv);
+
+- if (fill_files_note(&info->files) == 0)
++ if (fill_files_note(&info->files, cprm) == 0)
+ info->size += notesize(&info->files);
+
+ return 1;
+@@ -2016,7 +2028,7 @@ static int elf_note_info_init(struct elf_note_info *info)
+
+ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ struct elf_note_info *info,
+- const kernel_siginfo_t *siginfo, struct pt_regs *regs)
++ struct coredump_params *cprm)
+ {
+ struct core_thread *ct;
+ struct elf_thread_status *ets;
+@@ -2037,13 +2049,13 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ list_for_each_entry(ets, &info->thread_list, list) {
+ int sz;
+
+- sz = elf_dump_thread_status(siginfo->si_signo, ets);
++ sz = elf_dump_thread_status(cprm->siginfo->si_signo, ets);
+ info->thread_status_size += sz;
+ }
+ /* now collect the dump for the current */
+ memset(info->prstatus, 0, sizeof(*info->prstatus));
+- fill_prstatus(&info->prstatus->common, current, siginfo->si_signo);
+- elf_core_copy_regs(&info->prstatus->pr_reg, regs);
++ fill_prstatus(&info->prstatus->common, current, cprm->siginfo->si_signo);
++ elf_core_copy_regs(&info->prstatus->pr_reg, cprm->regs);
+
+ /* Set up header */
+ fill_elf_header(elf, phdrs, ELF_ARCH, ELF_CORE_EFLAGS);
+@@ -2059,18 +2071,18 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
+ fill_note(info->notes + 1, "CORE", NT_PRPSINFO,
+ sizeof(*info->psinfo), info->psinfo);
+
+- fill_siginfo_note(info->notes + 2, &info->csigdata, siginfo);
++ fill_siginfo_note(info->notes + 2, &info->csigdata, cprm->siginfo);
+ fill_auxv_note(info->notes + 3, current->mm);
+ info->numnote = 4;
+
+- if (fill_files_note(info->notes + info->numnote) == 0) {
++ if (fill_files_note(info->notes + info->numnote, cprm) == 0) {
+ info->notes_files = info->notes + info->numnote;
+ info->numnote++;
+ }
+
+ /* Try to dump the FPU. */
+- info->prstatus->pr_fpvalid = elf_core_copy_task_fpregs(current, regs,
+- info->fpu);
++ info->prstatus->pr_fpvalid =
++ elf_core_copy_task_fpregs(current, cprm->regs, info->fpu);
+ if (info->prstatus->pr_fpvalid)
+ fill_note(info->notes + info->numnote++,
+ "CORE", NT_PRFPREG, sizeof(*info->fpu), info->fpu);
+@@ -2156,8 +2168,7 @@ static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum,
+ static int elf_core_dump(struct coredump_params *cprm)
+ {
+ int has_dumped = 0;
+- int vma_count, segs, i;
+- size_t vma_data_size;
++ int segs, i;
+ struct elfhdr elf;
+ loff_t offset = 0, dataoff;
+ struct elf_note_info info = { };
+@@ -2165,16 +2176,12 @@ static int elf_core_dump(struct coredump_params *cprm)
+ struct elf_shdr *shdr4extnum = NULL;
+ Elf_Half e_phnum;
+ elf_addr_t e_shoff;
+- struct core_vma_metadata *vma_meta;
+-
+- if (dump_vma_snapshot(cprm, &vma_count, &vma_meta, &vma_data_size))
+- return 0;
+
+ /*
+ * The number of segs are recored into ELF header as 16bit value.
+ * Please check DEFAULT_MAX_MAP_COUNT definition when you modify here.
+ */
+- segs = vma_count + elf_core_extra_phdrs();
++ segs = cprm->vma_count + elf_core_extra_phdrs();
+
+ /* for notes section */
+ segs++;
+@@ -2188,7 +2195,7 @@ static int elf_core_dump(struct coredump_params *cprm)
+ * Collect all the non-memory information about the process for the
+ * notes. This also sets up the file header.
+ */
+- if (!fill_note_info(&elf, e_phnum, &info, cprm->siginfo, cprm->regs))
++ if (!fill_note_info(&elf, e_phnum, &info, cprm))
+ goto end_coredump;
+
+ has_dumped = 1;
+@@ -2213,7 +2220,7 @@ static int elf_core_dump(struct coredump_params *cprm)
+
+ dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE);
+
+- offset += vma_data_size;
++ offset += cprm->vma_data_size;
+ offset += elf_core_extra_data_size();
+ e_shoff = offset;
+
+@@ -2233,8 +2240,8 @@ static int elf_core_dump(struct coredump_params *cprm)
+ goto end_coredump;
+
+ /* Write program headers for segments dump */
+- for (i = 0; i < vma_count; i++) {
+- struct core_vma_metadata *meta = vma_meta + i;
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct core_vma_metadata *meta = cprm->vma_meta + i;
+ struct elf_phdr phdr;
+
+ phdr.p_type = PT_LOAD;
+@@ -2271,8 +2278,8 @@ static int elf_core_dump(struct coredump_params *cprm)
+ /* Align to page */
+ dump_skip_to(cprm, dataoff);
+
+- for (i = 0; i < vma_count; i++) {
+- struct core_vma_metadata *meta = vma_meta + i;
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct core_vma_metadata *meta = cprm->vma_meta + i;
+
+ if (!dump_user_range(cprm, meta->start, meta->dump_size))
+ goto end_coredump;
+@@ -2289,7 +2296,6 @@ static int elf_core_dump(struct coredump_params *cprm)
+ end_coredump:
+ free_note_info(&info);
+ kfree(shdr4extnum);
+- kvfree(vma_meta);
+ kfree(phdr4note);
+ return has_dumped;
+ }
+diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
+index 6d8fd6030cbb5..830a6a876ffea 100644
+--- a/fs/binfmt_elf_fdpic.c
++++ b/fs/binfmt_elf_fdpic.c
+@@ -1465,7 +1465,7 @@ static bool elf_fdpic_dump_segments(struct coredump_params *cprm,
+ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ {
+ int has_dumped = 0;
+- int vma_count, segs;
++ int segs;
+ int i;
+ struct elfhdr *elf = NULL;
+ loff_t offset = 0, dataoff;
+@@ -1480,8 +1480,6 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ elf_addr_t e_shoff;
+ struct core_thread *ct;
+ struct elf_thread_status *tmp;
+- struct core_vma_metadata *vma_meta = NULL;
+- size_t vma_data_size;
+
+ /* alloc memory for large data structures: too large to be on stack */
+ elf = kmalloc(sizeof(*elf), GFP_KERNEL);
+@@ -1491,9 +1489,6 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ if (!psinfo)
+ goto end_coredump;
+
+- if (dump_vma_snapshot(cprm, &vma_count, &vma_meta, &vma_data_size))
+- goto end_coredump;
+-
+ for (ct = current->mm->core_state->dumper.next;
+ ct; ct = ct->next) {
+ tmp = elf_dump_thread_status(cprm->siginfo->si_signo,
+@@ -1513,7 +1508,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ tmp->next = thread_list;
+ thread_list = tmp;
+
+- segs = vma_count + elf_core_extra_phdrs();
++ segs = cprm->vma_count + elf_core_extra_phdrs();
+
+ /* for notes section */
+ segs++;
+@@ -1558,7 +1553,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ /* Page-align dumped data */
+ dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE);
+
+- offset += vma_data_size;
++ offset += cprm->vma_data_size;
+ offset += elf_core_extra_data_size();
+ e_shoff = offset;
+
+@@ -1578,8 +1573,8 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+ goto end_coredump;
+
+ /* write program headers for segments dump */
+- for (i = 0; i < vma_count; i++) {
+- struct core_vma_metadata *meta = vma_meta + i;
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct core_vma_metadata *meta = cprm->vma_meta + i;
+ struct elf_phdr phdr;
+ size_t sz;
+
+@@ -1628,7 +1623,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
+
+ dump_skip_to(cprm, dataoff);
+
+- if (!elf_fdpic_dump_segments(cprm, vma_meta, vma_count))
++ if (!elf_fdpic_dump_segments(cprm, cprm->vma_meta, cprm->vma_count))
+ goto end_coredump;
+
+ if (!elf_core_write_extra_data(cprm))
+@@ -1652,7 +1647,6 @@ end_coredump:
+ thread_list = thread_list->next;
+ kfree(tmp);
+ }
+- kvfree(vma_meta);
+ kfree(phdr4note);
+ kfree(elf);
+ kfree(psinfo);
+diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c
+index 309516e6a9682..43c89952b7d25 100644
+--- a/fs/btrfs/async-thread.c
++++ b/fs/btrfs/async-thread.c
+@@ -234,6 +234,13 @@ static void run_ordered_work(struct __btrfs_workqueue *wq,
+ ordered_list);
+ if (!test_bit(WORK_DONE_BIT, &work->flags))
+ break;
++ /*
++ * Orders all subsequent loads after reading WORK_DONE_BIT,
++ * paired with the smp_mb__before_atomic in btrfs_work_helper
++ * this guarantees that the ordered function will see all
++ * updates from ordinary work function.
++ */
++ smp_rmb();
+
+ /*
+ * we are going to call the ordered done function, but
+@@ -317,6 +324,13 @@ static void btrfs_work_helper(struct work_struct *normal_work)
+ thresh_exec_hook(wq);
+ work->func(work);
+ if (need_order) {
++ /*
++ * Ensures all memory accesses done in the work function are
++ * ordered before setting the WORK_DONE_BIT. Ensuring the thread
++ * which is going to executed the ordered work sees them.
++ * Pairs with the smp_rmb in run_ordered_work.
++ */
++ smp_mb__before_atomic();
+ set_bit(WORK_DONE_BIT, &work->flags);
+ run_ordered_work(wq, work);
+ } else {
+diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c
+index f735b8798ba12..8b090c40daf77 100644
+--- a/fs/btrfs/backref.c
++++ b/fs/btrfs/backref.c
+@@ -1214,7 +1214,12 @@ again:
+ ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0);
+ if (ret < 0)
+ goto out;
+- BUG_ON(ret == 0);
++ if (ret == 0) {
++ /* This shouldn't happen, indicates a bug or fs corruption. */
++ ASSERT(ret != 0);
++ ret = -EUCLEAN;
++ goto out;
++ }
+
+ #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
+ if (trans && likely(trans->type != __TRANS_DUMMY) &&
+@@ -1360,10 +1365,18 @@ again:
+ goto out;
+ if (!ret && extent_item_pos) {
+ /*
+- * we've recorded that parent, so we must extend
+- * its inode list here
++ * We've recorded that parent, so we must extend
++ * its inode list here.
++ *
++ * However if there was corruption we may not
++ * have found an eie, return an error in this
++ * case.
+ */
+- BUG_ON(!eie);
++ ASSERT(eie);
++ if (!eie) {
++ ret = -EUCLEAN;
++ goto out;
++ }
+ while (eie->next)
+ eie = eie->next;
+ eie->next = ref->inode_list;
+diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c
+index a3b830b8410a8..37418b183b078 100644
+--- a/fs/btrfs/block-group.c
++++ b/fs/btrfs/block-group.c
+@@ -123,7 +123,16 @@ void btrfs_put_block_group(struct btrfs_block_group *cache)
+ {
+ if (refcount_dec_and_test(&cache->refs)) {
+ WARN_ON(cache->pinned > 0);
+- WARN_ON(cache->reserved > 0);
++ /*
++ * If there was a failure to cleanup a log tree, very likely due
++ * to an IO failure on a writeback attempt of one or more of its
++ * extent buffers, we could not do proper (and cheap) unaccounting
++ * of their reserved space, so don't warn on reserved > 0 in that
++ * case.
++ */
++ if (!(cache->flags & BTRFS_BLOCK_GROUP_METADATA) ||
++ !BTRFS_FS_LOG_CLEANUP_ERROR(cache->fs_info))
++ WARN_ON(cache->reserved > 0);
+
+ /*
+ * A block_group shouldn't be on the discard_list anymore.
+@@ -902,6 +911,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
+ spin_unlock(&cluster->refill_lock);
+
+ btrfs_clear_treelog_bg(block_group);
++ btrfs_clear_data_reloc_bg(block_group);
+
+ path = btrfs_alloc_path();
+ if (!path) {
+@@ -1490,13 +1500,16 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
+ container_of(work, struct btrfs_fs_info, reclaim_bgs_work);
+ struct btrfs_block_group *bg;
+ struct btrfs_space_info *space_info;
+- LIST_HEAD(again_list);
+
+ if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
+ return;
+
+- if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE))
++ sb_start_write(fs_info->sb);
++
++ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
++ sb_end_write(fs_info->sb);
+ return;
++ }
+
+ /*
+ * Long running balances can keep us blocked here for eternity, so
+@@ -1504,6 +1517,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
+ */
+ if (!mutex_trylock(&fs_info->reclaim_bgs_lock)) {
+ btrfs_exclop_finish(fs_info);
++ sb_end_write(fs_info->sb);
+ return;
+ }
+
+@@ -1561,21 +1575,18 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
+ div64_u64(zone_unusable * 100, bg->length));
+ trace_btrfs_reclaim_block_group(bg);
+ ret = btrfs_relocate_chunk(fs_info, bg->start);
+- if (ret && ret != -EAGAIN)
++ if (ret)
+ btrfs_err(fs_info, "error relocating chunk %llu",
+ bg->start);
+
+ next:
++ btrfs_put_block_group(bg);
+ spin_lock(&fs_info->unused_bgs_lock);
+- if (ret == -EAGAIN && list_empty(&bg->bg_list))
+- list_add_tail(&bg->bg_list, &again_list);
+- else
+- btrfs_put_block_group(bg);
+ }
+- list_splice_tail(&again_list, &fs_info->reclaim_bgs);
+ spin_unlock(&fs_info->unused_bgs_lock);
+ mutex_unlock(&fs_info->reclaim_bgs_lock);
+ btrfs_exclop_finish(fs_info);
++ sb_end_write(fs_info->sb);
+ }
+
+ void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info)
+@@ -2510,6 +2521,19 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
+ int ret;
+ bool dirty_bg_running;
+
++ /*
++ * This can only happen when we are doing read-only scrub on read-only
++ * mount.
++ * In that case we should not start a new transaction on read-only fs.
++ * Thus here we skip all chunk allocations.
++ */
++ if (sb_rdonly(fs_info->sb)) {
++ mutex_lock(&fs_info->ro_block_group_mutex);
++ ret = inc_block_group_ro(cache, 0);
++ mutex_unlock(&fs_info->ro_block_group_mutex);
++ return ret;
++ }
++
+ do {
+ trans = btrfs_join_transaction(fs_info->extent_root);
+ if (IS_ERR(trans))
+@@ -2858,7 +2882,6 @@ int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
+ struct btrfs_path *path = NULL;
+ LIST_HEAD(dirty);
+ struct list_head *io = &cur_trans->io_bgs;
+- int num_started = 0;
+ int loops = 0;
+
+ spin_lock(&cur_trans->dirty_bgs_lock);
+@@ -2924,7 +2947,6 @@ again:
+ cache->io_ctl.inode = NULL;
+ ret = btrfs_write_out_cache(trans, cache, path);
+ if (ret == 0 && cache->io_ctl.inode) {
+- num_started++;
+ should_put = 0;
+
+ /*
+@@ -3025,7 +3047,6 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
+ int should_put;
+ struct btrfs_path *path;
+ struct list_head *io = &cur_trans->io_bgs;
+- int num_started = 0;
+
+ path = btrfs_alloc_path();
+ if (!path)
+@@ -3083,7 +3104,6 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
+ cache->io_ctl.inode = NULL;
+ ret = btrfs_write_out_cache(trans, cache, path);
+ if (ret == 0 && cache->io_ctl.inode) {
+- num_started++;
+ should_put = 0;
+ list_add_tail(&cache->io_list, io);
+ } else {
+@@ -3879,9 +3899,22 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info)
+ * important and indicates a real bug if this happens.
+ */
+ if (WARN_ON(space_info->bytes_pinned > 0 ||
+- space_info->bytes_reserved > 0 ||
+ space_info->bytes_may_use > 0))
+ btrfs_dump_space_info(info, space_info, 0, 0);
++
++ /*
++ * If there was a failure to cleanup a log tree, very likely due
++ * to an IO failure on a writeback attempt of one or more of its
++ * extent buffers, we could not do proper (and cheap) unaccounting
++ * of their reserved space, so don't warn on bytes_reserved > 0 in
++ * that case.
++ */
++ if (!(space_info->flags & BTRFS_BLOCK_GROUP_METADATA) ||
++ !BTRFS_FS_LOG_CLEANUP_ERROR(info)) {
++ if (WARN_ON(space_info->bytes_reserved > 0))
++ btrfs_dump_space_info(info, space_info, 0, 0);
++ }
++
+ WARN_ON(space_info->reclaim_size > 0);
+ list_del(&space_info->list);
+ btrfs_sysfs_remove_space_info(space_info);
+diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
+index 0913ee50e6c34..701fbd1b56766 100644
+--- a/fs/btrfs/compression.c
++++ b/fs/btrfs/compression.c
+@@ -550,7 +550,6 @@ static noinline int add_ra_bio_pages(struct inode *inode,
+ u64 isize = i_size_read(inode);
+ int ret;
+ struct page *page;
+- unsigned long nr_pages = 0;
+ struct extent_map *em;
+ struct address_space *mapping = inode->i_mapping;
+ struct extent_map_tree *em_tree;
+@@ -646,7 +645,6 @@ static noinline int add_ra_bio_pages(struct inode *inode,
+ PAGE_SIZE, 0);
+
+ if (ret == PAGE_SIZE) {
+- nr_pages++;
+ put_page(page);
+ } else {
+ unlock_extent(tree, last_offset, end);
+diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
+index 84627cbd5b5b5..899f854459252 100644
+--- a/fs/btrfs/ctree.c
++++ b/fs/btrfs/ctree.c
+@@ -1566,35 +1566,13 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
+ struct btrfs_path *p,
+ int write_lock_level)
+ {
+- struct btrfs_fs_info *fs_info = root->fs_info;
+ struct extent_buffer *b;
+- int root_lock;
++ int root_lock = 0;
+ int level = 0;
+
+- /* We try very hard to do read locks on the root */
+- root_lock = BTRFS_READ_LOCK;
+-
+ if (p->search_commit_root) {
+- /*
+- * The commit roots are read only so we always do read locks,
+- * and we always must hold the commit_root_sem when doing
+- * searches on them, the only exception is send where we don't
+- * want to block transaction commits for a long time, so
+- * we need to clone the commit root in order to avoid races
+- * with transaction commits that create a snapshot of one of
+- * the roots used by a send operation.
+- */
+- if (p->need_commit_sem) {
+- down_read(&fs_info->commit_root_sem);
+- b = btrfs_clone_extent_buffer(root->commit_root);
+- up_read(&fs_info->commit_root_sem);
+- if (!b)
+- return ERR_PTR(-ENOMEM);
+-
+- } else {
+- b = root->commit_root;
+- atomic_inc(&b->refs);
+- }
++ b = root->commit_root;
++ atomic_inc(&b->refs);
+ level = btrfs_header_level(b);
+ /*
+ * Ensure that all callers have set skip_locking when
+@@ -1611,6 +1589,9 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
+ goto out;
+ }
+
++ /* We try very hard to do read locks on the root */
++ root_lock = BTRFS_READ_LOCK;
++
+ /*
+ * If the level is set to maximum, we can skip trying to get the read
+ * lock.
+@@ -1637,6 +1618,17 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
+ level = btrfs_header_level(b);
+
+ out:
++ /*
++ * The root may have failed to write out at some point, and thus is no
++ * longer valid, return an error in this case.
++ */
++ if (!extent_buffer_uptodate(b)) {
++ if (root_lock)
++ btrfs_tree_unlock_rw(b, root_lock);
++ free_extent_buffer(b);
++ return ERR_PTR(-EIO);
++ }
++
+ p->nodes[level] = b;
+ if (!p->skip_locking)
+ p->locks[level] = root_lock;
+@@ -1646,6 +1638,42 @@ out:
+ return b;
+ }
+
++/*
++ * Replace the extent buffer at the lowest level of the path with a cloned
++ * version. The purpose is to be able to use it safely, after releasing the
++ * commit root semaphore, even if relocation is happening in parallel, the
++ * transaction used for relocation is committed and the extent buffer is
++ * reallocated in the next transaction.
++ *
++ * This is used in a context where the caller does not prevent transaction
++ * commits from happening, either by holding a transaction handle or holding
++ * some lock, while it's doing searches through a commit root.
++ * At the moment it's only used for send operations.
++ */
++static int finish_need_commit_sem_search(struct btrfs_path *path)
++{
++ const int i = path->lowest_level;
++ const int slot = path->slots[i];
++ struct extent_buffer *lowest = path->nodes[i];
++ struct extent_buffer *clone;
++
++ ASSERT(path->need_commit_sem);
++
++ if (!lowest)
++ return 0;
++
++ lockdep_assert_held_read(&lowest->fs_info->commit_root_sem);
++
++ clone = btrfs_clone_extent_buffer(lowest);
++ if (!clone)
++ return -ENOMEM;
++
++ btrfs_release_path(path);
++ path->nodes[i] = clone;
++ path->slots[i] = slot;
++
++ return 0;
++}
+
+ /*
+ * btrfs_search_slot - look for a key in a tree and perform necessary
+@@ -1682,6 +1710,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
+ const struct btrfs_key *key, struct btrfs_path *p,
+ int ins_len, int cow)
+ {
++ struct btrfs_fs_info *fs_info = root->fs_info;
+ struct extent_buffer *b;
+ int slot;
+ int ret;
+@@ -1723,6 +1752,11 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
+
+ min_write_lock_level = write_lock_level;
+
++ if (p->need_commit_sem) {
++ ASSERT(p->search_commit_root);
++ down_read(&fs_info->commit_root_sem);
++ }
++
+ again:
+ prev_cmp = -1;
+ b = btrfs_search_slot_get_root(root, p, write_lock_level);
+@@ -1917,6 +1951,16 @@ cow_done:
+ done:
+ if (ret < 0 && !p->skip_release_on_error)
+ btrfs_release_path(p);
++
++ if (p->need_commit_sem) {
++ int ret2;
++
++ ret2 = finish_need_commit_sem_search(p);
++ up_read(&fs_info->commit_root_sem);
++ if (ret2)
++ ret = ret2;
++ }
++
+ return ret;
+ }
+ ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO);
+@@ -4385,7 +4429,9 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
+ int level;
+ struct extent_buffer *c;
+ struct extent_buffer *next;
++ struct btrfs_fs_info *fs_info = root->fs_info;
+ struct btrfs_key key;
++ bool need_commit_sem = false;
+ u32 nritems;
+ int ret;
+ int i;
+@@ -4402,14 +4448,20 @@ again:
+
+ path->keep_locks = 1;
+
+- if (time_seq)
++ if (time_seq) {
+ ret = btrfs_search_old_slot(root, &key, path, time_seq);
+- else
++ } else {
++ if (path->need_commit_sem) {
++ path->need_commit_sem = 0;
++ need_commit_sem = true;
++ down_read(&fs_info->commit_root_sem);
++ }
+ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
++ }
+ path->keep_locks = 0;
+
+ if (ret < 0)
+- return ret;
++ goto done;
+
+ nritems = btrfs_header_nritems(path->nodes[0]);
+ /*
+@@ -4532,6 +4584,15 @@ again:
+ ret = 0;
+ done:
+ unlock_up(path, 0, 1, 0, NULL);
++ if (need_commit_sem) {
++ int ret2;
++
++ path->need_commit_sem = 1;
++ ret2 = finish_need_commit_sem_search(path);
++ up_read(&fs_info->commit_root_sem);
++ if (ret2)
++ ret = ret2;
++ }
+
+ return ret;
+ }
+diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
+index c0cebcf745cef..21c44846b0022 100644
+--- a/fs/btrfs/ctree.h
++++ b/fs/btrfs/ctree.h
+@@ -142,6 +142,9 @@ enum {
+ BTRFS_FS_STATE_DEV_REPLACING,
+ /* The btrfs_fs_info created for self-tests */
+ BTRFS_FS_STATE_DUMMY_FS_INFO,
++
++ /* Indicates there was an error cleaning up a log tree. */
++ BTRFS_FS_STATE_LOG_CLEANUP_ERROR,
+ };
+
+ #define BTRFS_BACKREF_REV_MAX 256
+@@ -568,7 +571,6 @@ enum {
+ /*
+ * Indicate that relocation of a chunk has started, it's set per chunk
+ * and is toggled between chunks.
+- * Set, tested and cleared while holding fs_info::send_reloc_lock.
+ */
+ BTRFS_FS_RELOC_RUNNING,
+
+@@ -593,6 +595,9 @@ enum {
+ /* Indicate whether there are any tree modification log users */
+ BTRFS_FS_TREE_MOD_LOG_USERS,
+
++ /* Indicate we have half completed snapshot deletions pending. */
++ BTRFS_FS_UNFINISHED_DROPS,
++
+ #if BITS_PER_LONG == 32
+ /* Indicate if we have error/warn message printed on 32bit systems */
+ BTRFS_FS_32BIT_ERROR,
+@@ -665,6 +670,12 @@ struct btrfs_fs_info {
+
+ u64 generation;
+ u64 last_trans_committed;
++ /*
++ * Generation of the last transaction used for block group relocation
++ * since the filesystem was last mounted (or 0 if none happened yet).
++ * Must be written and read while holding btrfs_fs_info::commit_root_sem.
++ */
++ u64 last_reloc_trans;
+ u64 avg_delayed_ref_runtime;
+
+ /*
+@@ -994,13 +1005,6 @@ struct btrfs_fs_info {
+
+ struct crypto_shash *csum_shash;
+
+- spinlock_t send_reloc_lock;
+- /*
+- * Number of send operations in progress.
+- * Updated while holding fs_info::send_reloc_lock.
+- */
+- int send_in_progress;
+-
+ /* Type of exclusive operation running, protected by super_lock */
+ enum btrfs_exclusive_operation exclusive_operation;
+
+@@ -1017,6 +1021,13 @@ struct btrfs_fs_info {
+ spinlock_t treelog_bg_lock;
+ u64 treelog_bg;
+
++ /*
++ * Start of the dedicated data relocation block group, protected by
++ * relocation_bg_lock.
++ */
++ spinlock_t relocation_bg_lock;
++ u64 data_reloc_bg;
++
+ #ifdef CONFIG_BTRFS_FS_REF_VERIFY
+ spinlock_t ref_verify_lock;
+ struct rb_root block_tree;
+@@ -1091,8 +1102,15 @@ enum {
+ BTRFS_ROOT_HAS_LOG_TREE,
+ /* Qgroup flushing is in progress */
+ BTRFS_ROOT_QGROUP_FLUSHING,
++ /* This root has a drop operation that was started previously. */
++ BTRFS_ROOT_UNFINISHED_DROP,
+ };
+
++static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info)
++{
++ clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags);
++}
++
+ /*
+ * Record swapped tree blocks of a subvolume tree for delayed subtree trace
+ * code. For detail check comment in fs/btrfs/qgroup.c.
+@@ -3563,6 +3581,10 @@ do { \
+ (errno), fmt, ##args); \
+ } while (0)
+
++#define BTRFS_FS_LOG_CLEANUP_ERROR(fs_info) \
++ (unlikely(test_bit(BTRFS_FS_STATE_LOG_CLEANUP_ERROR, \
++ &(fs_info)->fs_state)))
++
+ __printf(5, 6)
+ __cold
+ void __btrfs_panic(struct btrfs_fs_info *fs_info, const char *function,
+@@ -3842,6 +3864,11 @@ static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info)
+ return fs_info->zoned != 0;
+ }
+
++static inline bool btrfs_is_data_reloc_root(const struct btrfs_root *root)
++{
++ return root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID;
++}
++
+ /*
+ * We use page status Private2 to indicate there is an ordered extent with
+ * unfinished IO.
+diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c
+index 2059d1504149a..40c4d6ba3fb9a 100644
+--- a/fs/btrfs/delalloc-space.c
++++ b/fs/btrfs/delalloc-space.c
+@@ -143,10 +143,13 @@ int btrfs_check_data_free_space(struct btrfs_inode *inode,
+
+ /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */
+ ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
+- if (ret < 0)
++ if (ret < 0) {
+ btrfs_free_reserved_data_space_noquota(fs_info, len);
+- else
++ extent_changeset_free(*reserved);
++ *reserved = NULL;
++ } else {
+ ret = 0;
++ }
+ return ret;
+ }
+
+@@ -452,8 +455,11 @@ int btrfs_delalloc_reserve_space(struct btrfs_inode *inode,
+ if (ret < 0)
+ return ret;
+ ret = btrfs_delalloc_reserve_metadata(inode, len);
+- if (ret < 0)
++ if (ret < 0) {
+ btrfs_free_reserved_data_space(inode, *reserved, start, len);
++ extent_changeset_free(*reserved);
++ *reserved = NULL;
++ }
+ return ret;
+ }
+
+diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
+index d029be40ea6f0..bdbc310a8f8c5 100644
+--- a/fs/btrfs/dev-replace.c
++++ b/fs/btrfs/dev-replace.c
+@@ -325,7 +325,7 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
+ set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
+ device->fs_devices = fs_info->fs_devices;
+
+- ret = btrfs_get_dev_zone_info(device);
++ ret = btrfs_get_dev_zone_info(device, false);
+ if (ret)
+ goto error;
+
+diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
+index 355ea88d5c5f7..8cbed2f08d1bd 100644
+--- a/fs/btrfs/disk-io.c
++++ b/fs/btrfs/disk-io.c
+@@ -441,17 +441,31 @@ static int csum_one_extent_buffer(struct extent_buffer *eb)
+ else
+ ret = btrfs_check_leaf_full(eb);
+
+- if (ret < 0) {
+- btrfs_print_tree(eb, 0);
++ if (ret < 0)
++ goto error;
++
++ /*
++ * Also check the generation, the eb reached here must be newer than
++ * last committed. Or something seriously wrong happened.
++ */
++ if (unlikely(btrfs_header_generation(eb) <= fs_info->last_trans_committed)) {
++ ret = -EUCLEAN;
+ btrfs_err(fs_info,
+- "block=%llu write time tree block corruption detected",
+- eb->start);
+- WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
+- return ret;
++ "block=%llu bad generation, have %llu expect > %llu",
++ eb->start, btrfs_header_generation(eb),
++ fs_info->last_trans_committed);
++ goto error;
+ }
+ write_extent_buffer(eb, result, 0, fs_info->csum_size);
+
+ return 0;
++
++error:
++ btrfs_print_tree(eb, 0);
++ btrfs_err(fs_info, "block=%llu write time tree block corruption detected",
++ eb->start);
++ WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
++ return ret;
+ }
+
+ /* Checksum all dirty extent buffers in one bio_vec */
+@@ -1500,7 +1514,7 @@ static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
+ goto fail;
+
+ if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
+- root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID) {
++ !btrfs_is_data_reloc_root(root)) {
+ set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
+ btrfs_check_and_init_root_item(&root->root_item);
+ }
+@@ -1724,13 +1738,22 @@ again:
+
+ ret = btrfs_insert_fs_root(fs_info, root);
+ if (ret) {
+- btrfs_put_root(root);
+- if (ret == -EEXIST)
++ if (ret == -EEXIST) {
++ btrfs_put_root(root);
+ goto again;
++ }
+ goto fail;
+ }
+ return root;
+ fail:
++ /*
++ * If our caller provided us an anonymous device, then it's his
++ * responsability to free it in case we fail. So we have to set our
++ * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
++ * and once again by our caller.
++ */
++ if (anon_dev)
++ root->anon_dev = 0;
+ btrfs_put_root(root);
+ return ERR_PTR(ret);
+ }
+@@ -2851,6 +2874,7 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
+ /* All successful */
+ fs_info->generation = generation;
+ fs_info->last_trans_committed = generation;
++ fs_info->last_reloc_trans = 0;
+
+ /* Always begin writing backup roots after the one being used */
+ if (backup_index < 0) {
+@@ -2883,6 +2907,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
+ spin_lock_init(&fs_info->buffer_lock);
+ spin_lock_init(&fs_info->unused_bgs_lock);
+ spin_lock_init(&fs_info->treelog_bg_lock);
++ spin_lock_init(&fs_info->relocation_bg_lock);
+ rwlock_init(&fs_info->tree_mod_log_lock);
+ mutex_init(&fs_info->unused_bg_unpin_mutex);
+ mutex_init(&fs_info->reclaim_bgs_lock);
+@@ -2983,9 +3008,6 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
+ spin_lock_init(&fs_info->swapfile_pins_lock);
+ fs_info->swapfile_pins = RB_ROOT;
+
+- spin_lock_init(&fs_info->send_reloc_lock);
+- fs_info->send_in_progress = 0;
+-
+ fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH;
+ INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work);
+ }
+@@ -3228,12 +3250,12 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
+ mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
+ btrfs_init_btree_inode(fs_info);
+
+- invalidate_bdev(fs_devices->latest_bdev);
++ invalidate_bdev(fs_devices->latest_dev->bdev);
+
+ /*
+ * Read super block and check the signature bytes only
+ */
+- disk_super = btrfs_read_dev_super(fs_devices->latest_bdev);
++ disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
+ if (IS_ERR(disk_super)) {
+ err = PTR_ERR(disk_super);
+ goto fail_alloc;
+@@ -3393,6 +3415,17 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
+ }
+
+ if (sectorsize != PAGE_SIZE) {
++ /*
++ * V1 space cache has some hardcoded PAGE_SIZE usage, and is
++ * going to be deprecated.
++ *
++ * Force to use v2 cache for subpage case.
++ */
++ btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
++ btrfs_set_and_info(fs_info, FREE_SPACE_TREE,
++ "forcing free space tree for sector size %u with page size %lu",
++ sectorsize, PAGE_SIZE);
++
+ btrfs_warn(fs_info,
+ "read-write for sector size %u with page size %lu is experimental",
+ sectorsize, PAGE_SIZE);
+@@ -3465,7 +3498,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
+ * below in btrfs_init_dev_replace().
+ */
+ btrfs_free_extra_devids(fs_devices);
+- if (!fs_devices->latest_bdev) {
++ if (!fs_devices->latest_dev->bdev) {
+ btrfs_err(fs_info, "failed to read devices");
+ goto fail_tree_roots;
+ }
+@@ -3556,7 +3589,10 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
+ goto fail_sysfs;
+ }
+
+- if (!sb_rdonly(sb) && !btrfs_check_rw_degradable(fs_info, NULL)) {
++ btrfs_free_zone_cache(fs_info);
++
++ if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices &&
++ !btrfs_check_rw_degradable(fs_info, NULL)) {
+ btrfs_warn(fs_info,
+ "writable mount is not allowed due to too many missing devices");
+ goto fail_sysfs;
+@@ -3647,6 +3683,10 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
+
+ set_bit(BTRFS_FS_OPEN, &fs_info->flags);
+
++ /* Kick the cleaner thread so it'll start deleting snapshots. */
++ if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags))
++ wake_up_process(fs_info->cleaner_kthread);
++
+ clear_oneshot:
+ btrfs_clear_oneshot_options(fs_info);
+ return 0;
+@@ -3968,11 +4008,23 @@ static void btrfs_end_empty_barrier(struct bio *bio)
+ */
+ static void write_dev_flush(struct btrfs_device *device)
+ {
+- struct request_queue *q = bdev_get_queue(device->bdev);
+ struct bio *bio = device->flush_bio;
+
++#ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
++ /*
++ * When a disk has write caching disabled, we skip submission of a bio
++ * with flush and sync requests before writing the superblock, since
++ * it's not needed. However when the integrity checker is enabled, this
++ * results in reports that there are metadata blocks referred by a
++ * superblock that were not properly flushed. So don't skip the bio
++ * submission only when the integrity checker is enabled for the sake
++ * of simplicity, since this is a debug tool and not meant for use in
++ * non-debug builds.
++ */
++ struct request_queue *q = bdev_get_queue(device->bdev);
+ if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags))
+ return;
++#endif
+
+ bio_reset(bio);
+ bio->bi_end_io = btrfs_end_empty_barrier;
+@@ -4316,6 +4368,12 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info)
+ */
+ kthread_park(fs_info->cleaner_kthread);
+
++ /*
++ * If we had UNFINISHED_DROPS we could still be processing them, so
++ * clear that bit and wake up relocation so it can stop.
++ */
++ btrfs_wake_unfinished_drop(fs_info);
++
+ /* wait for the qgroup rescan worker to stop */
+ btrfs_qgroup_wait_for_completion(fs_info, false);
+
+diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
+index 0ab456cb4bf80..514adc83577f0 100644
+--- a/fs/btrfs/extent-tree.c
++++ b/fs/btrfs/extent-tree.c
+@@ -2376,7 +2376,7 @@ int btrfs_cross_ref_exist(struct btrfs_root *root, u64 objectid, u64 offset,
+
+ out:
+ btrfs_free_path(path);
+- if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
++ if (btrfs_is_data_reloc_root(root))
+ WARN_ON(ret > 0);
+ return ret;
+ }
+@@ -3495,6 +3495,9 @@ struct find_free_extent_ctl {
+ /* Allocation is called for tree-log */
+ bool for_treelog;
+
++ /* Allocation is called for data relocation */
++ bool for_data_reloc;
++
+ /* RAID index, converted from flags */
+ int index;
+
+@@ -3756,6 +3759,7 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
+ u64 avail;
+ u64 bytenr = block_group->start;
+ u64 log_bytenr;
++ u64 data_reloc_bytenr;
+ int ret = 0;
+ bool skip;
+
+@@ -3773,13 +3777,31 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
+ if (skip)
+ return 1;
+
++ /*
++ * Do not allow non-relocation blocks in the dedicated relocation block
++ * group, and vice versa.
++ */
++ spin_lock(&fs_info->relocation_bg_lock);
++ data_reloc_bytenr = fs_info->data_reloc_bg;
++ if (data_reloc_bytenr &&
++ ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) ||
++ (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr)))
++ skip = true;
++ spin_unlock(&fs_info->relocation_bg_lock);
++ if (skip)
++ return 1;
++
+ spin_lock(&space_info->lock);
+ spin_lock(&block_group->lock);
+ spin_lock(&fs_info->treelog_bg_lock);
++ spin_lock(&fs_info->relocation_bg_lock);
+
+ ASSERT(!ffe_ctl->for_treelog ||
+ block_group->start == fs_info->treelog_bg ||
+ fs_info->treelog_bg == 0);
++ ASSERT(!ffe_ctl->for_data_reloc ||
++ block_group->start == fs_info->data_reloc_bg ||
++ fs_info->data_reloc_bg == 0);
+
+ if (block_group->ro) {
+ ret = 1;
+@@ -3796,6 +3818,16 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
+ goto out;
+ }
+
++ /*
++ * Do not allow currently used block group to be the data relocation
++ * dedicated block group.
++ */
++ if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg &&
++ (block_group->used || block_group->reserved)) {
++ ret = 1;
++ goto out;
++ }
++
+ avail = block_group->length - block_group->alloc_offset;
+ if (avail < num_bytes) {
+ if (ffe_ctl->max_extent_size < avail) {
+@@ -3813,6 +3845,9 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
+ if (ffe_ctl->for_treelog && !fs_info->treelog_bg)
+ fs_info->treelog_bg = block_group->start;
+
++ if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg)
++ fs_info->data_reloc_bg = block_group->start;
++
+ ffe_ctl->found_offset = start + block_group->alloc_offset;
+ block_group->alloc_offset += num_bytes;
+ spin_lock(&ctl->tree_lock);
+@@ -3829,6 +3864,9 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
+ out:
+ if (ret && ffe_ctl->for_treelog)
+ fs_info->treelog_bg = 0;
++ if (ret && ffe_ctl->for_data_reloc)
++ fs_info->data_reloc_bg = 0;
++ spin_unlock(&fs_info->relocation_bg_lock);
+ spin_unlock(&fs_info->treelog_bg_lock);
+ spin_unlock(&block_group->lock);
+ spin_unlock(&space_info->lock);
+@@ -4085,6 +4123,12 @@ static int prepare_allocation(struct btrfs_fs_info *fs_info,
+ ffe_ctl->hint_byte = fs_info->treelog_bg;
+ spin_unlock(&fs_info->treelog_bg_lock);
+ }
++ if (ffe_ctl->for_data_reloc) {
++ spin_lock(&fs_info->relocation_bg_lock);
++ if (fs_info->data_reloc_bg)
++ ffe_ctl->hint_byte = fs_info->data_reloc_bg;
++ spin_unlock(&fs_info->relocation_bg_lock);
++ }
+ return 0;
+ default:
+ BUG();
+@@ -4129,6 +4173,8 @@ static noinline int find_free_extent(struct btrfs_root *root,
+ struct btrfs_space_info *space_info;
+ bool full_search = false;
+ bool for_treelog = (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
++ bool for_data_reloc = (btrfs_is_data_reloc_root(root) &&
++ flags & BTRFS_BLOCK_GROUP_DATA);
+
+ WARN_ON(num_bytes < fs_info->sectorsize);
+
+@@ -4143,6 +4189,7 @@ static noinline int find_free_extent(struct btrfs_root *root,
+ ffe_ctl.found_offset = 0;
+ ffe_ctl.hint_byte = hint_byte_orig;
+ ffe_ctl.for_treelog = for_treelog;
++ ffe_ctl.for_data_reloc = for_data_reloc;
+ ffe_ctl.policy = BTRFS_EXTENT_ALLOC_CLUSTERED;
+
+ /* For clustered allocation */
+@@ -4220,6 +4267,8 @@ search:
+ if (unlikely(block_group->ro)) {
+ if (for_treelog)
+ btrfs_clear_treelog_bg(block_group);
++ if (ffe_ctl.for_data_reloc)
++ btrfs_clear_data_reloc_bg(block_group);
+ continue;
+ }
+
+@@ -4408,6 +4457,7 @@ int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes,
+ u64 flags;
+ int ret;
+ bool for_treelog = (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
++ bool for_data_reloc = (btrfs_is_data_reloc_root(root) && is_data);
+
+ flags = get_alloc_profile_by_root(root, is_data);
+ again:
+@@ -4431,8 +4481,8 @@ again:
+
+ sinfo = btrfs_find_space_info(fs_info, flags);
+ btrfs_err(fs_info,
+- "allocation failed flags %llu, wanted %llu tree-log %d",
+- flags, num_bytes, for_treelog);
++ "allocation failed flags %llu, wanted %llu tree-log %d, relocation: %d",
++ flags, num_bytes, for_treelog, for_data_reloc);
+ if (sinfo)
+ btrfs_dump_space_info(fs_info, sinfo,
+ num_bytes, 1);
+@@ -5491,6 +5541,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc)
+ int ret;
+ int level;
+ bool root_dropped = false;
++ bool unfinished_drop = false;
+
+ btrfs_debug(fs_info, "Drop subvolume %llu", root->root_key.objectid);
+
+@@ -5533,6 +5584,8 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc)
+ * already dropped.
+ */
+ set_bit(BTRFS_ROOT_DELETING, &root->state);
++ unfinished_drop = test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state);
++
+ if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
+ level = btrfs_header_level(root->node);
+ path->nodes[level] = btrfs_lock_root_node(root);
+@@ -5707,6 +5760,13 @@ out_free:
+ kfree(wc);
+ btrfs_free_path(path);
+ out:
++ /*
++ * We were an unfinished drop root, check to see if there are any
++ * pending, and if not clear and wake up any waiters.
++ */
++ if (!err && unfinished_drop)
++ btrfs_maybe_wake_unfinished_drop(fs_info);
++
+ /*
+ * So if we need to stop dropping the snapshot for whatever reason we
+ * need to make sure to add it back to the dead root list so that we
+diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
+index aaddd72253481..96aeb16bd65c9 100644
+--- a/fs/btrfs/extent_io.c
++++ b/fs/btrfs/extent_io.c
+@@ -3327,7 +3327,7 @@ static int alloc_new_bio(struct btrfs_inode *inode,
+ if (wbc) {
+ struct block_device *bdev;
+
+- bdev = fs_info->fs_devices->latest_bdev;
++ bdev = fs_info->fs_devices->latest_dev->bdev;
+ bio_set_dev(bio, bdev);
+ wbc_init_bio(wbc, bio);
+ }
+@@ -3561,7 +3561,6 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
+ u64 cur_end;
+ struct extent_map *em;
+ int ret = 0;
+- int nr = 0;
+ size_t pg_offset = 0;
+ size_t iosize;
+ size_t blocksize = inode->i_sb->s_blocksize;
+@@ -3727,9 +3726,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
+ end_bio_extent_readpage, 0,
+ this_bio_flag,
+ force_bio_submit);
+- if (!ret) {
+- nr++;
+- } else {
++ if (ret) {
+ unlock_extent(tree, cur, cur + iosize - 1);
+ end_page_read(page, false, cur, iosize);
+ goto out;
+@@ -4284,6 +4281,12 @@ static void set_btree_ioerr(struct page *page, struct extent_buffer *eb)
+ if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
+ return;
+
++ /*
++ * A read may stumble upon this buffer later, make sure that it gets an
++ * error and knows there was an error.
++ */
++ clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
++
+ /*
+ * If we error out, we should add back the dirty_metadata_bytes
+ * to make it consistent.
+@@ -5120,6 +5123,9 @@ int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
+ int extent_writepages(struct address_space *mapping,
+ struct writeback_control *wbc)
+ {
++ struct inode *inode = mapping->host;
++ const bool data_reloc = btrfs_is_data_reloc_root(BTRFS_I(inode)->root);
++ const bool zoned = btrfs_is_zoned(BTRFS_I(inode)->root->fs_info);
+ int ret = 0;
+ struct extent_page_data epd = {
+ .bio_ctrl = { 0 },
+@@ -5127,7 +5133,15 @@ int extent_writepages(struct address_space *mapping,
+ .sync_io = wbc->sync_mode == WB_SYNC_ALL,
+ };
+
++ /*
++ * Allow only a single thread to do the reloc work in zoned mode to
++ * protect the write pointer updates.
++ */
++ if (data_reloc && zoned)
++ btrfs_inode_lock(inode, 0);
+ ret = extent_write_cache_pages(mapping, wbc, &epd);
++ if (data_reloc && zoned)
++ btrfs_inode_unlock(inode, 0);
+ ASSERT(ret <= 0);
+ if (ret < 0) {
+ end_write_bio(&epd, ret);
+@@ -6530,6 +6544,14 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num)
+ if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
+ return 0;
+
++ /*
++ * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
++ * operation, which could potentially still be in flight. In this case
++ * we simply want to return an error.
++ */
++ if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
++ return -EIO;
++
+ if (eb->fs_info->sectorsize < PAGE_SIZE)
+ return read_extent_buffer_subpage(eb, wait, mirror_num);
+
+@@ -6776,14 +6798,24 @@ static void assert_eb_page_uptodate(const struct extent_buffer *eb,
+ {
+ struct btrfs_fs_info *fs_info = eb->fs_info;
+
++ /*
++ * If we are using the commit root we could potentially clear a page
++ * Uptodate while we're using the extent buffer that we've previously
++ * looked up. We don't want to complain in this case, as the page was
++ * valid before, we just didn't write it out. Instead we want to catch
++ * the case where we didn't actually read the block properly, which
++ * would have !PageUptodate && !PageError, as we clear PageError before
++ * reading.
++ */
+ if (fs_info->sectorsize < PAGE_SIZE) {
+- bool uptodate;
++ bool uptodate, error;
+
+ uptodate = btrfs_subpage_test_uptodate(fs_info, page,
+ eb->start, eb->len);
+- WARN_ON(!uptodate);
++ error = btrfs_subpage_test_error(fs_info, page, eb->start, eb->len);
++ WARN_ON(!uptodate && !error);
+ } else {
+- WARN_ON(!PageUptodate(page));
++ WARN_ON(!PageUptodate(page) && !PageError(page));
+ }
+ }
+
+diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
+index 53abdc280451b..f7ab6ba8238e1 100644
+--- a/fs/btrfs/extent_io.h
++++ b/fs/btrfs/extent_io.h
+@@ -117,7 +117,7 @@ struct btrfs_bio_ctrl {
+ */
+ struct extent_changeset {
+ /* How many bytes are set/cleared in this operation */
+- unsigned int bytes_changed;
++ u64 bytes_changed;
+
+ /* Changed ranges */
+ struct ulist range_changed;
+diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
+index 0b9401a5afd33..161a69d7e117e 100644
+--- a/fs/btrfs/file-item.c
++++ b/fs/btrfs/file-item.c
+@@ -303,7 +303,7 @@ found:
+ read_extent_buffer(path->nodes[0], dst, (unsigned long)item,
+ ret * csum_size);
+ out:
+- if (ret == -ENOENT)
++ if (ret == -ENOENT || ret == -EFBIG)
+ ret = 0;
+ return ret;
+ }
+diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
+index a1762363f61fa..ff578c934bbcf 100644
+--- a/fs/btrfs/file.c
++++ b/fs/btrfs/file.c
+@@ -1709,7 +1709,7 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
+ * Fault pages before locking them in prepare_pages
+ * to avoid recursive lock
+ */
+- if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
++ if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) {
+ ret = -EFAULT;
+ break;
+ }
+@@ -1903,16 +1903,17 @@ static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
+
+ static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
+ {
++ const bool is_sync_write = (iocb->ki_flags & IOCB_DSYNC);
+ struct file *file = iocb->ki_filp;
+ struct inode *inode = file_inode(file);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ loff_t pos;
+ ssize_t written = 0;
+ ssize_t written_buffered;
++ size_t prev_left = 0;
+ loff_t endbyte;
+ ssize_t err;
+ unsigned int ilock_flags = 0;
+- struct iomap_dio *dio = NULL;
+
+ if (iocb->ki_flags & IOCB_NOWAIT)
+ ilock_flags |= BTRFS_ILOCK_TRY;
+@@ -1955,23 +1956,80 @@ relock:
+ goto buffered;
+ }
+
+- dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
+- 0);
++ /*
++ * We remove IOCB_DSYNC so that we don't deadlock when iomap_dio_rw()
++ * calls generic_write_sync() (through iomap_dio_complete()), because
++ * that results in calling fsync (btrfs_sync_file()) which will try to
++ * lock the inode in exclusive/write mode.
++ */
++ if (is_sync_write)
++ iocb->ki_flags &= ~IOCB_DSYNC;
+
+- btrfs_inode_unlock(inode, ilock_flags);
++ /*
++ * The iov_iter can be mapped to the same file range we are writing to.
++ * If that's the case, then we will deadlock in the iomap code, because
++ * it first calls our callback btrfs_dio_iomap_begin(), which will create
++ * an ordered extent, and after that it will fault in the pages that the
++ * iov_iter refers to. During the fault in we end up in the readahead
++ * pages code (starting at btrfs_readahead()), which will lock the range,
++ * find that ordered extent and then wait for it to complete (at
++ * btrfs_lock_and_flush_ordered_range()), resulting in a deadlock since
++ * obviously the ordered extent can never complete as we didn't submit
++ * yet the respective bio(s). This always happens when the buffer is
++ * memory mapped to the same file range, since the iomap DIO code always
++ * invalidates pages in the target file range (after starting and waiting
++ * for any writeback).
++ *
++ * So here we disable page faults in the iov_iter and then retry if we
++ * got -EFAULT, faulting in the pages before the retry.
++ */
++again:
++ from->nofault = true;
++ err = iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
++ IOMAP_DIO_PARTIAL, written);
++ from->nofault = false;
+
+- if (IS_ERR_OR_NULL(dio)) {
+- err = PTR_ERR_OR_ZERO(dio);
+- if (err < 0 && err != -ENOTBLK)
+- goto out;
+- } else {
+- written = iomap_dio_complete(dio);
++ /* No increment (+=) because iomap returns a cumulative value. */
++ if (err > 0)
++ written = err;
++
++ if (iov_iter_count(from) > 0 && (err == -EFAULT || err > 0)) {
++ const size_t left = iov_iter_count(from);
++ /*
++ * We have more data left to write. Try to fault in as many as
++ * possible of the remainder pages and retry. We do this without
++ * releasing and locking again the inode, to prevent races with
++ * truncate.
++ *
++ * Also, in case the iov refers to pages in the file range of the
++ * file we want to write to (due to a mmap), we could enter an
++ * infinite loop if we retry after faulting the pages in, since
++ * iomap will invalidate any pages in the range early on, before
++ * it tries to fault in the pages of the iov. So we keep track of
++ * how much was left of iov in the previous EFAULT and fallback
++ * to buffered IO in case we haven't made any progress.
++ */
++ if (left == prev_left) {
++ err = -ENOTBLK;
++ } else {
++ fault_in_iov_iter_readable(from, left);
++ prev_left = left;
++ goto again;
++ }
+ }
+
+- if (written < 0 || !iov_iter_count(from)) {
+- err = written;
++ btrfs_inode_unlock(inode, ilock_flags);
++
++ /*
++ * Add back IOCB_DSYNC. Our caller, btrfs_file_write_iter(), will do
++ * the fsync (call generic_write_sync()).
++ */
++ if (is_sync_write)
++ iocb->ki_flags |= IOCB_DSYNC;
++
++ /* If 'err' is -ENOTBLK then it means we must fallback to buffered IO. */
++ if ((err < 0 && err != -ENOTBLK) || !iov_iter_count(from))
+ goto out;
+- }
+
+ buffered:
+ pos = iocb->ki_pos;
+@@ -1996,7 +2054,7 @@ buffered:
+ invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
+ endbyte >> PAGE_SHIFT);
+ out:
+- return written ? written : err;
++ return err < 0 ? err : written;
+ }
+
+ static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
+@@ -2878,8 +2936,9 @@ out:
+ return ret;
+ }
+
+-static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
++static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len)
+ {
++ struct inode *inode = file_inode(file);
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct extent_state *cached_state = NULL;
+@@ -2911,6 +2970,10 @@ static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
+ goto out_only_mutex;
+ }
+
++ ret = file_modified(file);
++ if (ret)
++ goto out_only_mutex;
++
+ lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode)));
+ lockend = round_down(offset + len,
+ btrfs_inode_sectorsize(BTRFS_I(inode))) - 1;
+@@ -3351,7 +3414,7 @@ static long btrfs_fallocate(struct file *file, int mode,
+ return -EOPNOTSUPP;
+
+ if (mode & FALLOC_FL_PUNCH_HOLE)
+- return btrfs_punch_hole(inode, offset, len);
++ return btrfs_punch_hole(file, offset, len);
+
+ /*
+ * Only trigger disk allocation, don't trigger qgroup reserve
+@@ -3373,6 +3436,10 @@ static long btrfs_fallocate(struct file *file, int mode,
+ goto out;
+ }
+
++ ret = file_modified(file);
++ if (ret)
++ goto out;
++
+ /*
+ * TODO: Move these two operations after we have checked
+ * accurate reserved space, or fallocate can still fail but
+@@ -3650,6 +3717,8 @@ static int check_direct_read(struct btrfs_fs_info *fs_info,
+ static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
+ {
+ struct inode *inode = file_inode(iocb->ki_filp);
++ size_t prev_left = 0;
++ ssize_t read = 0;
+ ssize_t ret;
+
+ if (fsverity_active(inode))
+@@ -3659,9 +3728,57 @@ static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
+ return 0;
+
+ btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
+- ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops, 0);
++again:
++ /*
++ * This is similar to what we do for direct IO writes, see the comment
++ * at btrfs_direct_write(), but we also disable page faults in addition
++ * to disabling them only at the iov_iter level. This is because when
++ * reading from a hole or prealloc extent, iomap calls iov_iter_zero(),
++ * which can still trigger page fault ins despite having set ->nofault
++ * to true of our 'to' iov_iter.
++ *
++ * The difference to direct IO writes is that we deadlock when trying
++ * to lock the extent range in the inode's tree during he page reads
++ * triggered by the fault in (while for writes it is due to waiting for
++ * our own ordered extent). This is because for direct IO reads,
++ * btrfs_dio_iomap_begin() returns with the extent range locked, which
++ * is only unlocked in the endio callback (end_bio_extent_readpage()).
++ */
++ pagefault_disable();
++ to->nofault = true;
++ ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
++ IOMAP_DIO_PARTIAL, read);
++ to->nofault = false;
++ pagefault_enable();
++
++ /* No increment (+=) because iomap returns a cumulative value. */
++ if (ret > 0)
++ read = ret;
++
++ if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) {
++ const size_t left = iov_iter_count(to);
++
++ if (left == prev_left) {
++ /*
++ * We didn't make any progress since the last attempt,
++ * fallback to a buffered read for the remainder of the
++ * range. This is just to avoid any possibility of looping
++ * for too long.
++ */
++ ret = read;
++ } else {
++ /*
++ * We made some progress since the last retry or this is
++ * the first time we are retrying. Fault in as many pages
++ * as possible and retry.
++ */
++ fault_in_iov_iter_writeable(to, left);
++ prev_left = left;
++ goto again;
++ }
++ }
+ btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
+- return ret;
++ return ret < 0 ? ret : read;
+ }
+
+ static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
+index 7c096ab9bb5eb..044d584c3467c 100644
+--- a/fs/btrfs/inode.c
++++ b/fs/btrfs/inode.c
+@@ -60,8 +60,6 @@ struct btrfs_iget_args {
+ };
+
+ struct btrfs_dio_data {
+- u64 reserve;
+- loff_t length;
+ ssize_t submitted;
+ struct extent_changeset *data_reserved;
+ };
+@@ -1077,7 +1075,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
+ int ret = 0;
+
+ if (btrfs_is_free_space_inode(inode)) {
+- WARN_ON_ONCE(1);
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+@@ -1151,7 +1148,7 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
+ * fails during the stage where it updates the bytenr of file extent
+ * items.
+ */
+- if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
++ if (btrfs_is_data_reloc_root(root))
+ min_alloc_size = num_bytes;
+ else
+ min_alloc_size = fs_info->sectorsize;
+@@ -1187,8 +1184,7 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
+ if (ret)
+ goto out_drop_extent_cache;
+
+- if (root->root_key.objectid ==
+- BTRFS_DATA_RELOC_TREE_OBJECTID) {
++ if (btrfs_is_data_reloc_root(root)) {
+ ret = btrfs_reloc_clone_csums(inode, start,
+ cur_alloc_size);
+ /*
+@@ -1504,8 +1500,7 @@ static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page,
+ int *page_started, unsigned long *nr_written)
+ {
+ const bool is_space_ino = btrfs_is_free_space_inode(inode);
+- const bool is_reloc_ino = (inode->root->root_key.objectid ==
+- BTRFS_DATA_RELOC_TREE_OBJECTID);
++ const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root);
+ const u64 range_bytes = end + 1 - start;
+ struct extent_io_tree *io_tree = &inode->io_tree;
+ u64 range_start = start;
+@@ -1867,8 +1862,7 @@ out_check:
+ btrfs_dec_nocow_writers(fs_info, disk_bytenr);
+ nocow = false;
+
+- if (root->root_key.objectid ==
+- BTRFS_DATA_RELOC_TREE_OBJECTID)
++ if (btrfs_is_data_reloc_root(root))
+ /*
+ * Error handled later, as we must prevent
+ * extent_clear_unlock_delalloc() in error handler
+@@ -1948,7 +1942,15 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page
+ const bool zoned = btrfs_is_zoned(inode->root->fs_info);
+
+ if (should_nocow(inode, start, end)) {
+- ASSERT(!zoned);
++ /*
++ * Normally on a zoned device we're only doing COW writes, but
++ * in case of relocation on a zoned filesystem we have taken
++ * precaution, that we're only writing sequentially. It's safe
++ * to use run_delalloc_nocow() here, like for regular
++ * preallocated inodes.
++ */
++ ASSERT(!zoned ||
++ (zoned && btrfs_is_data_reloc_root(inode->root)));
+ ret = run_delalloc_nocow(inode, locked_page, start, end,
+ page_started, nr_written);
+ } else if (!inode_can_compress(inode) ||
+@@ -2207,7 +2209,7 @@ void btrfs_clear_delalloc_extent(struct inode *vfs_inode,
+ if (btrfs_is_testing(fs_info))
+ return;
+
+- if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
++ if (!btrfs_is_data_reloc_root(root) &&
+ do_list && !(state->state & EXTENT_NORESERVE) &&
+ (*bits & EXTENT_CLEAR_DATA_RESV))
+ btrfs_free_reserved_data_space_noquota(fs_info, len);
+@@ -2532,7 +2534,7 @@ blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio,
+ goto mapit;
+ } else if (async && !skip_sum) {
+ /* csum items have already been cloned */
+- if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
++ if (btrfs_is_data_reloc_root(root))
+ goto mapit;
+ /* we're doing a write, do the async checksumming */
+ ret = btrfs_wq_submit_bio(inode, bio, mirror_num, bio_flags,
+@@ -3304,7 +3306,7 @@ unsigned int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset,
+ u64 file_offset = pg_off + page_offset(page);
+ int ret;
+
+- if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
++ if (btrfs_is_data_reloc_root(root) &&
+ test_range_bit(io_tree, file_offset,
+ file_offset + sectorsize - 1,
+ EXTENT_NODATASUM, 1, NULL)) {
+@@ -4005,7 +4007,7 @@ noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
+ * without delay
+ */
+ if (!btrfs_is_free_space_inode(inode)
+- && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
++ && !btrfs_is_data_reloc_root(root)
+ && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
+ btrfs_update_root_times(trans, root);
+
+@@ -4447,6 +4449,13 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry)
+ dest->root_key.objectid);
+ return -EPERM;
+ }
++ if (atomic_read(&dest->nr_swapfiles)) {
++ spin_unlock(&dest->root_item_lock);
++ btrfs_warn(fs_info,
++ "attempt to delete subvolume %llu with active swapfile",
++ root->root_key.objectid);
++ return -EPERM;
++ }
+ root_flags = btrfs_root_flags(&dest->root_item);
+ btrfs_set_root_flags(&dest->root_item,
+ root_flags | BTRFS_ROOT_SUBVOL_DEAD);
+@@ -7758,6 +7767,11 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,
+ {
+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct extent_map *em = *map;
++ int type;
++ u64 block_start, orig_start, orig_block_len, ram_bytes;
++ bool can_nocow = false;
++ bool space_reserved = false;
++ u64 prev_len;
+ int ret = 0;
+
+ /*
+@@ -7772,9 +7786,6 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,
+ if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
+ ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
+ em->block_start != EXTENT_MAP_HOLE)) {
+- int type;
+- u64 block_start, orig_start, orig_block_len, ram_bytes;
+-
+ if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
+ type = BTRFS_ORDERED_PREALLOC;
+ else
+@@ -7784,53 +7795,91 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,
+
+ if (can_nocow_extent(inode, start, &len, &orig_start,
+ &orig_block_len, &ram_bytes, false) == 1 &&
+- btrfs_inc_nocow_writers(fs_info, block_start)) {
+- struct extent_map *em2;
++ btrfs_inc_nocow_writers(fs_info, block_start))
++ can_nocow = true;
++ }
++
++ prev_len = len;
++ if (can_nocow) {
++ struct extent_map *em2;
+
+- em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len,
+- orig_start, block_start,
+- len, orig_block_len,
+- ram_bytes, type);
++ /* We can NOCOW, so only need to reserve metadata space. */
++ ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len);
++ if (ret < 0) {
++ /* Our caller expects us to free the input extent map. */
++ free_extent_map(em);
++ *map = NULL;
+ btrfs_dec_nocow_writers(fs_info, block_start);
+- if (type == BTRFS_ORDERED_PREALLOC) {
+- free_extent_map(em);
+- *map = em = em2;
+- }
++ goto out;
++ }
++ space_reserved = true;
+
+- if (em2 && IS_ERR(em2)) {
+- ret = PTR_ERR(em2);
+- goto out;
+- }
+- /*
+- * For inode marked NODATACOW or extent marked PREALLOC,
+- * use the existing or preallocated extent, so does not
+- * need to adjust btrfs_space_info's bytes_may_use.
+- */
+- btrfs_free_reserved_data_space_noquota(fs_info, len);
+- goto skip_cow;
++ em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len,
++ orig_start, block_start,
++ len, orig_block_len,
++ ram_bytes, type);
++ btrfs_dec_nocow_writers(fs_info, block_start);
++ if (type == BTRFS_ORDERED_PREALLOC) {
++ free_extent_map(em);
++ *map = em = em2;
+ }
+- }
+
+- /* this will cow the extent */
+- free_extent_map(em);
+- *map = em = btrfs_new_extent_direct(BTRFS_I(inode), start, len);
+- if (IS_ERR(em)) {
+- ret = PTR_ERR(em);
+- goto out;
++ if (IS_ERR(em2)) {
++ ret = PTR_ERR(em2);
++ goto out;
++ }
++ } else {
++ /* Our caller expects us to free the input extent map. */
++ free_extent_map(em);
++ *map = NULL;
++
++ /* We have to COW, so need to reserve metadata and data space. */
++ ret = btrfs_delalloc_reserve_space(BTRFS_I(inode),
++ &dio_data->data_reserved,
++ start, len);
++ if (ret < 0)
++ goto out;
++ space_reserved = true;
++
++ em = btrfs_new_extent_direct(BTRFS_I(inode), start, len);
++ if (IS_ERR(em)) {
++ ret = PTR_ERR(em);
++ goto out;
++ }
++ *map = em;
++ len = min(len, em->len - (start - em->start));
++ if (len < prev_len)
++ btrfs_delalloc_release_space(BTRFS_I(inode),
++ dio_data->data_reserved,
++ start + len, prev_len - len,
++ true);
+ }
+
+- len = min(len, em->len - (start - em->start));
++ /*
++ * We have created our ordered extent, so we can now release our reservation
++ * for an outstanding extent.
++ */
++ btrfs_delalloc_release_extents(BTRFS_I(inode), prev_len);
+
+-skip_cow:
+ /*
+ * Need to update the i_size under the extent lock so buffered
+ * readers will get the updated i_size when we unlock.
+ */
+ if (start + len > i_size_read(inode))
+ i_size_write(inode, start + len);
+-
+- dio_data->reserve -= len;
+ out:
++ if (ret && space_reserved) {
++ btrfs_delalloc_release_extents(BTRFS_I(inode), len);
++ if (can_nocow) {
++ btrfs_delalloc_release_metadata(BTRFS_I(inode), len, true);
++ } else {
++ btrfs_delalloc_release_space(BTRFS_I(inode),
++ dio_data->data_reserved,
++ start, len, true);
++ extent_changeset_free(dio_data->data_reserved);
++ dio_data->data_reserved = NULL;
++ }
++ }
+ return ret;
+ }
+
+@@ -7872,18 +7921,6 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
+ if (!dio_data)
+ return -ENOMEM;
+
+- dio_data->length = length;
+- if (write) {
+- dio_data->reserve = round_up(length, fs_info->sectorsize);
+- ret = btrfs_delalloc_reserve_space(BTRFS_I(inode),
+- &dio_data->data_reserved,
+- start, dio_data->reserve);
+- if (ret) {
+- extent_changeset_free(dio_data->data_reserved);
+- kfree(dio_data);
+- return ret;
+- }
+- }
+ iomap->private = dio_data;
+
+
+@@ -7924,6 +7961,34 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
+ }
+
+ len = min(len, em->len - (start - em->start));
++
++ /*
++ * If we have a NOWAIT request and the range contains multiple extents
++ * (or a mix of extents and holes), then we return -EAGAIN to make the
++ * caller fallback to a context where it can do a blocking (without
++ * NOWAIT) request. This way we avoid doing partial IO and returning
++ * success to the caller, which is not optimal for writes and for reads
++ * it can result in unexpected behaviour for an application.
++ *
++ * When doing a read, because we use IOMAP_DIO_PARTIAL when calling
++ * iomap_dio_rw(), we can end up returning less data then what the caller
++ * asked for, resulting in an unexpected, and incorrect, short read.
++ * That is, the caller asked to read N bytes and we return less than that,
++ * which is wrong unless we are crossing EOF. This happens if we get a
++ * page fault error when trying to fault in pages for the buffer that is
++ * associated to the struct iov_iter passed to iomap_dio_rw(), and we
++ * have previously submitted bios for other extents in the range, in
++ * which case iomap_dio_rw() may return us EIOCBQUEUED if not all of
++ * those bios have completed by the time we get the page fault error,
++ * which we return back to our caller - we should only return EIOCBQUEUED
++ * after we have submitted bios for all the extents in the range.
++ */
++ if ((flags & IOMAP_NOWAIT) && len < length) {
++ free_extent_map(em);
++ ret = -EAGAIN;
++ goto unlock_err;
++ }
++
+ if (write) {
+ ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
+ start, len);
+@@ -7962,7 +8027,7 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
+ iomap->type = IOMAP_MAPPED;
+ }
+ iomap->offset = start;
+- iomap->bdev = fs_info->fs_devices->latest_bdev;
++ iomap->bdev = fs_info->fs_devices->latest_dev->bdev;
+ iomap->length = len;
+
+ if (write && btrfs_use_zone_append(BTRFS_I(inode), em->block_start))
+@@ -7976,14 +8041,8 @@ unlock_err:
+ unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
+ &cached_state);
+ err:
+- if (dio_data) {
+- btrfs_delalloc_release_space(BTRFS_I(inode),
+- dio_data->data_reserved, start,
+- dio_data->reserve, true);
+- btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve);
+- extent_changeset_free(dio_data->data_reserved);
+- kfree(dio_data);
+- }
++ kfree(dio_data);
++
+ return ret;
+ }
+
+@@ -8013,14 +8072,8 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+ ret = -ENOTBLK;
+ }
+
+- if (write) {
+- if (dio_data->reserve)
+- btrfs_delalloc_release_space(BTRFS_I(inode),
+- dio_data->data_reserved, pos,
+- dio_data->reserve, true);
+- btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length);
++ if (write)
+ extent_changeset_free(dio_data->data_reserved);
+- }
+ out:
+ kfree(dio_data);
+ iomap->private = NULL;
+@@ -10581,9 +10634,19 @@ static int btrfs_add_swap_extent(struct swap_info_struct *sis,
+ struct btrfs_swap_info *bsi)
+ {
+ unsigned long nr_pages;
++ unsigned long max_pages;
+ u64 first_ppage, first_ppage_reported, next_ppage;
+ int ret;
+
++ /*
++ * Our swapfile may have had its size extended after the swap header was
++ * written. In that case activating the swapfile should not go beyond
++ * the max size set in the swap header.
++ */
++ if (bsi->nr_pages >= sis->max)
++ return 0;
++
++ max_pages = sis->max - bsi->nr_pages;
+ first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT;
+ next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len,
+ PAGE_SIZE) >> PAGE_SHIFT;
+@@ -10591,6 +10654,7 @@ static int btrfs_add_swap_extent(struct swap_info_struct *sis,
+ if (first_ppage >= next_ppage)
+ return 0;
+ nr_pages = next_ppage - first_ppage;
++ nr_pages = min(nr_pages, max_pages);
+
+ first_ppage_reported = first_ppage;
+ if (bsi->start == 0)
+@@ -10691,8 +10755,23 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
+ * set. We use this counter to prevent snapshots. We must increment it
+ * before walking the extents because we don't want a concurrent
+ * snapshot to run after we've already checked the extents.
++ *
++ * It is possible that subvolume is marked for deletion but still not
++ * removed yet. To prevent this race, we check the root status before
++ * activating the swapfile.
+ */
++ spin_lock(&root->root_item_lock);
++ if (btrfs_root_dead(root)) {
++ spin_unlock(&root->root_item_lock);
++
++ btrfs_exclop_finish(fs_info);
++ btrfs_warn(fs_info,
++ "cannot activate swapfile because subvolume %llu is being deleted",
++ root->root_key.objectid);
++ return -EPERM;
++ }
+ atomic_inc(&root->nr_swapfiles);
++ spin_unlock(&root->root_item_lock);
+
+ isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
+
+diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
+index cc61813213d83..bf53af8694f8e 100644
+--- a/fs/btrfs/ioctl.c
++++ b/fs/btrfs/ioctl.c
+@@ -775,10 +775,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir,
+ goto fail;
+ }
+
+- spin_lock(&fs_info->trans_lock);
+- list_add(&pending_snapshot->list,
+- &trans->transaction->pending_snapshots);
+- spin_unlock(&fs_info->trans_lock);
++ trans->pending_snapshot = pending_snapshot;
+
+ ret = btrfs_commit_transaction(trans);
+ if (ret)
+@@ -2261,9 +2258,8 @@ static noinline int search_ioctl(struct inode *inode,
+ key.offset = sk->min_offset;
+
+ while (1) {
+- ret = fault_in_pages_writeable(ubuf + sk_offset,
+- *buf_size - sk_offset);
+- if (ret)
++ ret = -EFAULT;
++ if (fault_in_writeable(ubuf + sk_offset, *buf_size - sk_offset))
+ break;
+
+ ret = btrfs_search_forward(root, &key, path, sk->min_transid);
+@@ -3098,10 +3094,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file,
+ btrfs_inode_lock(inode, 0);
+ err = btrfs_delete_subvolume(dir, dentry);
+ btrfs_inode_unlock(inode, 0);
+- if (!err) {
+- fsnotify_rmdir(dir, dentry);
+- d_delete(dentry);
+- }
++ if (!err)
++ d_delete_notify(dir, dentry);
+
+ out_dput:
+ dput(dentry);
+diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c
+index 3dbe6eb5fda75..fcd7eb496478c 100644
+--- a/fs/btrfs/lzo.c
++++ b/fs/btrfs/lzo.c
+@@ -357,11 +357,23 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
+ ASSERT(cur_in / sectorsize ==
+ (cur_in + LZO_LEN - 1) / sectorsize);
+ cur_page = cb->compressed_pages[cur_in / PAGE_SIZE];
+- kaddr = kmap(cur_page);
+ ASSERT(cur_page);
++ kaddr = kmap(cur_page);
+ seg_len = read_compress_length(kaddr + offset_in_page(cur_in));
++ kunmap(cur_page);
+ cur_in += LZO_LEN;
+
++ if (seg_len > lzo1x_worst_compress(PAGE_SIZE)) {
++ /*
++ * seg_len shouldn't be larger than we have allocated
++ * for workspace->cbuf
++ */
++ btrfs_err(fs_info, "unexpectedly large lzo segment len %u",
++ seg_len);
++ ret = -EIO;
++ goto out;
++ }
++
+ /* Copy the compressed segment payload into workspace */
+ copy_compressed_segment(cb, workspace->cbuf, seg_len, &cur_in);
+
+diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
+index db680f5be745a..2c803108ea944 100644
+--- a/fs/btrfs/qgroup.c
++++ b/fs/btrfs/qgroup.c
+@@ -940,6 +940,14 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)
+ int ret = 0;
+ int slot;
+
++ /*
++ * We need to have subvol_sem write locked, to prevent races between
++ * concurrent tasks trying to enable quotas, because we will unlock
++ * and relock qgroup_ioctl_lock before setting fs_info->quota_root
++ * and before setting BTRFS_FS_QUOTA_ENABLED.
++ */
++ lockdep_assert_held_write(&fs_info->subvol_sem);
++
+ mutex_lock(&fs_info->qgroup_ioctl_lock);
+ if (fs_info->quota_root)
+ goto out;
+@@ -1117,8 +1125,19 @@ out_add_root:
+ goto out_free_path;
+ }
+
++ mutex_unlock(&fs_info->qgroup_ioctl_lock);
++ /*
++ * Commit the transaction while not holding qgroup_ioctl_lock, to avoid
++ * a deadlock with tasks concurrently doing other qgroup operations, such
++ * adding/removing qgroups or adding/deleting qgroup relations for example,
++ * because all qgroup operations first start or join a transaction and then
++ * lock the qgroup_ioctl_lock mutex.
++ * We are safe from a concurrent task trying to enable quotas, by calling
++ * this function, since we are serialized by fs_info->subvol_sem.
++ */
+ ret = btrfs_commit_transaction(trans);
+ trans = NULL;
++ mutex_lock(&fs_info->qgroup_ioctl_lock);
+ if (ret)
+ goto out_free_path;
+
+@@ -1166,11 +1185,33 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info)
+ struct btrfs_trans_handle *trans = NULL;
+ int ret = 0;
+
++ /*
++ * We need to have subvol_sem write locked, to prevent races between
++ * concurrent tasks trying to disable quotas, because we will unlock
++ * and relock qgroup_ioctl_lock across BTRFS_FS_QUOTA_ENABLED changes.
++ */
++ lockdep_assert_held_write(&fs_info->subvol_sem);
++
+ mutex_lock(&fs_info->qgroup_ioctl_lock);
+ if (!fs_info->quota_root)
+ goto out;
++
++ /*
++ * Unlock the qgroup_ioctl_lock mutex before waiting for the rescan worker to
++ * complete. Otherwise we can deadlock because btrfs_remove_qgroup() needs
++ * to lock that mutex while holding a transaction handle and the rescan
++ * worker needs to commit a transaction.
++ */
+ mutex_unlock(&fs_info->qgroup_ioctl_lock);
+
++ /*
++ * Request qgroup rescan worker to complete and wait for it. This wait
++ * must be done before transaction start for quota disable since it may
++ * deadlock with transaction by the qgroup rescan worker.
++ */
++ clear_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
++ btrfs_qgroup_wait_for_completion(fs_info, false);
++
+ /*
+ * 1 For the root item
+ *
+@@ -1186,14 +1227,13 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info)
+ if (IS_ERR(trans)) {
+ ret = PTR_ERR(trans);
+ trans = NULL;
++ set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
+ goto out;
+ }
+
+ if (!fs_info->quota_root)
+ goto out;
+
+- clear_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
+- btrfs_qgroup_wait_for_completion(fs_info, false);
+ spin_lock(&fs_info->qgroup_lock);
+ quota_root = fs_info->quota_root;
+ fs_info->quota_root = NULL;
+@@ -3360,6 +3400,9 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
+ btrfs_warn(fs_info,
+ "qgroup rescan init failed, qgroup is not enabled");
+ ret = -EINVAL;
++ } else if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) {
++ /* Quota disable is in progress */
++ ret = -EBUSY;
+ }
+
+ if (ret) {
+diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c
+index 9b0814318e726..fa60af00ebca2 100644
+--- a/fs/btrfs/reflink.c
++++ b/fs/btrfs/reflink.c
+@@ -505,8 +505,11 @@ process_slot:
+ */
+ ASSERT(key.offset == 0);
+ ASSERT(datal <= fs_info->sectorsize);
+- if (key.offset != 0 || datal > fs_info->sectorsize)
+- return -EUCLEAN;
++ if (WARN_ON(key.offset != 0) ||
++ WARN_ON(datal > fs_info->sectorsize)) {
++ ret = -EUCLEAN;
++ goto out;
++ }
+
+ ret = clone_copy_inline_extent(inode, path, &new_key,
+ drop_start, datal, size,
+@@ -649,7 +652,7 @@ static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len,
+ static int btrfs_extent_same(struct inode *src, u64 loff, u64 olen,
+ struct inode *dst, u64 dst_loff)
+ {
+- int ret;
++ int ret = 0;
+ u64 i, tail_len, chunk_count;
+ struct btrfs_root *root_dst = BTRFS_I(dst)->root;
+
+diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
+index 914d403b4415d..a6661f2ad2c00 100644
+--- a/fs/btrfs/relocation.c
++++ b/fs/btrfs/relocation.c
+@@ -2852,31 +2852,6 @@ static noinline_for_stack int prealloc_file_extent_cluster(
+ if (ret)
+ return ret;
+
+- /*
+- * On a zoned filesystem, we cannot preallocate the file region.
+- * Instead, we dirty and fiemap_write the region.
+- */
+- if (btrfs_is_zoned(inode->root->fs_info)) {
+- struct btrfs_root *root = inode->root;
+- struct btrfs_trans_handle *trans;
+-
+- end = cluster->end - offset + 1;
+- trans = btrfs_start_transaction(root, 1);
+- if (IS_ERR(trans))
+- return PTR_ERR(trans);
+-
+- inode->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
+- i_size_write(&inode->vfs_inode, end);
+- ret = btrfs_update_inode(trans, root, inode);
+- if (ret) {
+- btrfs_abort_transaction(trans, ret);
+- btrfs_end_transaction(trans);
+- return ret;
+- }
+-
+- return btrfs_end_transaction(trans);
+- }
+-
+ btrfs_inode_lock(&inode->vfs_inode, 0);
+ for (nr = 0; nr < cluster->nr; nr++) {
+ start = cluster->boundary[nr] - offset;
+@@ -3084,7 +3059,6 @@ release_page:
+ static int relocate_file_extent_cluster(struct inode *inode,
+ struct file_extent_cluster *cluster)
+ {
+- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ u64 offset = BTRFS_I(inode)->index_cnt;
+ unsigned long index;
+ unsigned long last_index;
+@@ -3114,8 +3088,6 @@ static int relocate_file_extent_cluster(struct inode *inode,
+ for (index = (cluster->start - offset) >> PAGE_SHIFT;
+ index <= last_index && !ret; index++)
+ ret = relocate_one_page(inode, ra, cluster, &cluster_nr, index);
+- if (btrfs_is_zoned(fs_info) && !ret)
+- ret = btrfs_wait_ordered_range(inode, 0, (u64)-1);
+ if (ret == 0)
+ WARN_ON(cluster_nr != cluster->nr);
+ out:
+@@ -3770,12 +3742,8 @@ static int __insert_orphan_inode(struct btrfs_trans_handle *trans,
+ struct btrfs_path *path;
+ struct btrfs_inode_item *item;
+ struct extent_buffer *leaf;
+- u64 flags = BTRFS_INODE_NOCOMPRESS | BTRFS_INODE_PREALLOC;
+ int ret;
+
+- if (btrfs_is_zoned(trans->fs_info))
+- flags &= ~BTRFS_INODE_PREALLOC;
+-
+ path = btrfs_alloc_path();
+ if (!path)
+ return -ENOMEM;
+@@ -3790,7 +3758,8 @@ static int __insert_orphan_inode(struct btrfs_trans_handle *trans,
+ btrfs_set_inode_generation(leaf, item, 1);
+ btrfs_set_inode_size(leaf, item, 0);
+ btrfs_set_inode_mode(leaf, item, S_IFREG | 0600);
+- btrfs_set_inode_flags(leaf, item, flags);
++ btrfs_set_inode_flags(leaf, item, BTRFS_INODE_NOCOMPRESS |
++ BTRFS_INODE_PREALLOC);
+ btrfs_mark_buffer_dirty(leaf);
+ out:
+ btrfs_free_path(path);
+@@ -3885,25 +3854,14 @@ out:
+ * 0 success
+ * -EINPROGRESS operation is already in progress, that's probably a bug
+ * -ECANCELED cancellation request was set before the operation started
+- * -EAGAIN can not start because there are ongoing send operations
+ */
+ static int reloc_chunk_start(struct btrfs_fs_info *fs_info)
+ {
+- spin_lock(&fs_info->send_reloc_lock);
+- if (fs_info->send_in_progress) {
+- btrfs_warn_rl(fs_info,
+-"cannot run relocation while send operations are in progress (%d in progress)",
+- fs_info->send_in_progress);
+- spin_unlock(&fs_info->send_reloc_lock);
+- return -EAGAIN;
+- }
+ if (test_and_set_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) {
+ /* This should not happen */
+- spin_unlock(&fs_info->send_reloc_lock);
+ btrfs_err(fs_info, "reloc already running, cannot start");
+ return -EINPROGRESS;
+ }
+- spin_unlock(&fs_info->send_reloc_lock);
+
+ if (atomic_read(&fs_info->reloc_cancel_req) > 0) {
+ btrfs_info(fs_info, "chunk relocation canceled on start");
+@@ -3925,9 +3883,7 @@ static void reloc_chunk_end(struct btrfs_fs_info *fs_info)
+ /* Requested after start, clear bit first so any waiters can continue */
+ if (atomic_read(&fs_info->reloc_cancel_req) > 0)
+ btrfs_info(fs_info, "chunk relocation canceled during operation");
+- spin_lock(&fs_info->send_reloc_lock);
+ clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags);
+- spin_unlock(&fs_info->send_reloc_lock);
+ atomic_set(&fs_info->reloc_cancel_req, 0);
+ }
+
+@@ -3998,6 +3954,19 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start)
+ int rw = 0;
+ int err = 0;
+
++ /*
++ * This only gets set if we had a half-deleted snapshot on mount. We
++ * cannot allow relocation to start while we're still trying to clean up
++ * these pending deletions.
++ */
++ ret = wait_on_bit(&fs_info->flags, BTRFS_FS_UNFINISHED_DROPS, TASK_INTERRUPTIBLE);
++ if (ret)
++ return ret;
++
++ /* We may have been woken up by close_ctree, so bail if we're closing. */
++ if (btrfs_fs_closing(fs_info))
++ return -EINTR;
++
+ bg = btrfs_lookup_block_group(fs_info, group_start);
+ if (!bg)
+ return -ENOENT;
+@@ -4386,8 +4355,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans,
+ if (!rc)
+ return 0;
+
+- BUG_ON(rc->stage == UPDATE_DATA_PTRS &&
+- root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID);
++ BUG_ON(rc->stage == UPDATE_DATA_PTRS && btrfs_is_data_reloc_root(root));
+
+ level = btrfs_header_level(buf);
+ if (btrfs_header_generation(buf) <=
+diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c
+index 702dc5441f039..1fa0e5e2e3505 100644
+--- a/fs/btrfs/root-tree.c
++++ b/fs/btrfs/root-tree.c
+@@ -280,6 +280,21 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info)
+
+ WARN_ON(!test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state));
+ if (btrfs_root_refs(&root->root_item) == 0) {
++ struct btrfs_key drop_key;
++
++ btrfs_disk_key_to_cpu(&drop_key, &root->root_item.drop_progress);
++ /*
++ * If we have a non-zero drop_progress then we know we
++ * made it partly through deleting this snapshot, and
++ * thus we need to make sure we block any balance from
++ * happening until this snapshot is completely dropped.
++ */
++ if (drop_key.objectid != 0 || drop_key.type != 0 ||
++ drop_key.offset != 0) {
++ set_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags);
++ set_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state);
++ }
++
+ set_bit(BTRFS_ROOT_DEAD_TREE, &root->state);
+ btrfs_add_dead_root(root);
+ }
+@@ -336,7 +351,8 @@ int btrfs_del_root_ref(struct btrfs_trans_handle *trans, u64 root_id,
+ key.offset = ref_id;
+ again:
+ ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
+- BUG_ON(ret < 0);
++ if (ret < 0)
++ goto out;
+ if (ret == 0) {
+ leaf = path->nodes[0];
+ ref = btrfs_item_ptr(leaf, path->slots[0],
+diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
+index 088641ba7a8e6..62f4bafbe54bb 100644
+--- a/fs/btrfs/scrub.c
++++ b/fs/btrfs/scrub.c
+@@ -73,8 +73,8 @@ struct scrub_page {
+ u64 physical_for_dev_replace;
+ atomic_t refs;
+ u8 mirror_num;
+- int have_csum:1;
+- int io_error:1;
++ unsigned int have_csum:1;
++ unsigned int io_error:1;
+ u8 csum[BTRFS_CSUM_SIZE];
+
+ struct scrub_recover *recover;
+diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
+index 72f9b865e8479..4d2c6ce29fe58 100644
+--- a/fs/btrfs/send.c
++++ b/fs/btrfs/send.c
+@@ -24,6 +24,7 @@
+ #include "transaction.h"
+ #include "compression.h"
+ #include "xattr.h"
++#include "print-tree.h"
+
+ /*
+ * Maximum number of references an extent can have in order for us to attempt to
+@@ -95,6 +96,15 @@ struct send_ctx {
+ struct btrfs_path *right_path;
+ struct btrfs_key *cmp_key;
+
++ /*
++ * Keep track of the generation of the last transaction that was used
++ * for relocating a block group. This is periodically checked in order
++ * to detect if a relocation happened since the last check, so that we
++ * don't operate on stale extent buffers for nodes (level >= 1) or on
++ * stale disk_bytenr values of file extent items.
++ */
++ u64 last_reloc_trans;
++
+ /*
+ * infos of the currently processed inode. In case of deleted inodes,
+ * these are the values from the deleted inode.
+@@ -1415,6 +1425,26 @@ static int find_extent_clone(struct send_ctx *sctx,
+ if (ret < 0)
+ goto out;
+
++ down_read(&fs_info->commit_root_sem);
++ if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
++ /*
++ * A transaction commit for a transaction in which block group
++ * relocation was done just happened.
++ * The disk_bytenr of the file extent item we processed is
++ * possibly stale, referring to the extent's location before
++ * relocation. So act as if we haven't found any clone sources
++ * and fallback to write commands, which will read the correct
++ * data from the new extent location. Otherwise we will fail
++ * below because we haven't found our own back reference or we
++ * could be getting incorrect sources in case the old extent
++ * was already reallocated after the relocation.
++ */
++ up_read(&fs_info->commit_root_sem);
++ ret = -ENOENT;
++ goto out;
++ }
++ up_read(&fs_info->commit_root_sem);
++
+ if (!backref_ctx.found_itself) {
+ /* found a bug in backref code? */
+ ret = -EIO;
+@@ -4978,6 +5008,10 @@ static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
+ lock_page(page);
+ if (!PageUptodate(page)) {
+ unlock_page(page);
++ btrfs_err(fs_info,
++ "send: IO error at offset %llu for inode %llu root %llu",
++ page_offset(page), sctx->cur_ino,
++ sctx->send_root->root_key.objectid);
+ put_page(page);
+ ret = -EIO;
+ break;
+@@ -6592,6 +6626,50 @@ static int changed_cb(struct btrfs_path *left_path,
+ {
+ int ret = 0;
+
++ /*
++ * We can not hold the commit root semaphore here. This is because in
++ * the case of sending and receiving to the same filesystem, using a
++ * pipe, could result in a deadlock:
++ *
++ * 1) The task running send blocks on the pipe because it's full;
++ *
++ * 2) The task running receive, which is the only consumer of the pipe,
++ * is waiting for a transaction commit (for example due to a space
++ * reservation when doing a write or triggering a transaction commit
++ * when creating a subvolume);
++ *
++ * 3) The transaction is waiting to write lock the commit root semaphore,
++ * but can not acquire it since it's being held at 1).
++ *
++ * Down this call chain we write to the pipe through kernel_write().
++ * The same type of problem can also happen when sending to a file that
++ * is stored in the same filesystem - when reserving space for a write
++ * into the file, we can trigger a transaction commit.
++ *
++ * Our caller has supplied us with clones of leaves from the send and
++ * parent roots, so we're safe here from a concurrent relocation and
++ * further reallocation of metadata extents while we are here. Below we
++ * also assert that the leaves are clones.
++ */
++ lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
++
++ /*
++ * We always have a send root, so left_path is never NULL. We will not
++ * have a leaf when we have reached the end of the send root but have
++ * not yet reached the end of the parent root.
++ */
++ if (left_path->nodes[0])
++ ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
++ &left_path->nodes[0]->bflags));
++ /*
++ * When doing a full send we don't have a parent root, so right_path is
++ * NULL. When doing an incremental send, we may have reached the end of
++ * the parent root already, so we don't have a leaf at right_path.
++ */
++ if (right_path && right_path->nodes[0])
++ ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
++ &right_path->nodes[0]->bflags));
++
+ if (result == BTRFS_COMPARE_TREE_SAME) {
+ if (key->type == BTRFS_INODE_REF_KEY ||
+ key->type == BTRFS_INODE_EXTREF_KEY) {
+@@ -6638,14 +6716,46 @@ out:
+ return ret;
+ }
+
++static int search_key_again(const struct send_ctx *sctx,
++ struct btrfs_root *root,
++ struct btrfs_path *path,
++ const struct btrfs_key *key)
++{
++ int ret;
++
++ if (!path->need_commit_sem)
++ lockdep_assert_held_read(&root->fs_info->commit_root_sem);
++
++ /*
++ * Roots used for send operations are readonly and no one can add,
++ * update or remove keys from them, so we should be able to find our
++ * key again. The only exception is deduplication, which can operate on
++ * readonly roots and add, update or remove keys to/from them - but at
++ * the moment we don't allow it to run in parallel with send.
++ */
++ ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
++ ASSERT(ret <= 0);
++ if (ret > 0) {
++ btrfs_print_tree(path->nodes[path->lowest_level], false);
++ btrfs_err(root->fs_info,
++"send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
++ key->objectid, key->type, key->offset,
++ (root == sctx->parent_root ? "parent" : "send"),
++ root->root_key.objectid, path->lowest_level,
++ path->slots[path->lowest_level]);
++ return -EUCLEAN;
++ }
++
++ return ret;
++}
++
+ static int full_send_tree(struct send_ctx *sctx)
+ {
+ int ret;
+ struct btrfs_root *send_root = sctx->send_root;
+ struct btrfs_key key;
++ struct btrfs_fs_info *fs_info = send_root->fs_info;
+ struct btrfs_path *path;
+- struct extent_buffer *eb;
+- int slot;
+
+ path = alloc_path_for_send();
+ if (!path)
+@@ -6656,6 +6766,10 @@ static int full_send_tree(struct send_ctx *sctx)
+ key.type = BTRFS_INODE_ITEM_KEY;
+ key.offset = 0;
+
++ down_read(&fs_info->commit_root_sem);
++ sctx->last_reloc_trans = fs_info->last_reloc_trans;
++ up_read(&fs_info->commit_root_sem);
++
+ ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
+ if (ret < 0)
+ goto out;
+@@ -6663,15 +6777,35 @@ static int full_send_tree(struct send_ctx *sctx)
+ goto out_finish;
+
+ while (1) {
+- eb = path->nodes[0];
+- slot = path->slots[0];
+- btrfs_item_key_to_cpu(eb, &key, slot);
++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+
+ ret = changed_cb(path, NULL, &key,
+ BTRFS_COMPARE_TREE_NEW, sctx);
+ if (ret < 0)
+ goto out;
+
++ down_read(&fs_info->commit_root_sem);
++ if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
++ sctx->last_reloc_trans = fs_info->last_reloc_trans;
++ up_read(&fs_info->commit_root_sem);
++ /*
++ * A transaction used for relocating a block group was
++ * committed or is about to finish its commit. Release
++ * our path (leaf) and restart the search, so that we
++ * avoid operating on any file extent items that are
++ * stale, with a disk_bytenr that reflects a pre
++ * relocation value. This way we avoid as much as
++ * possible to fallback to regular writes when checking
++ * if we can clone file ranges.
++ */
++ btrfs_release_path(path);
++ ret = search_key_again(sctx, send_root, path, &key);
++ if (ret < 0)
++ goto out;
++ } else {
++ up_read(&fs_info->commit_root_sem);
++ }
++
+ ret = btrfs_next_item(send_root, path);
+ if (ret < 0)
+ goto out;
+@@ -6689,6 +6823,20 @@ out:
+ return ret;
+ }
+
++static int replace_node_with_clone(struct btrfs_path *path, int level)
++{
++ struct extent_buffer *clone;
++
++ clone = btrfs_clone_extent_buffer(path->nodes[level]);
++ if (!clone)
++ return -ENOMEM;
++
++ free_extent_buffer(path->nodes[level]);
++ path->nodes[level] = clone;
++
++ return 0;
++}
++
+ static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
+ {
+ struct extent_buffer *eb;
+@@ -6698,6 +6846,8 @@ static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen
+ u64 reada_max;
+ u64 reada_done = 0;
+
++ lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
++
+ BUG_ON(*level == 0);
+ eb = btrfs_read_node_slot(parent, slot);
+ if (IS_ERR(eb))
+@@ -6721,6 +6871,10 @@ static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen
+ path->nodes[*level - 1] = eb;
+ path->slots[*level - 1] = 0;
+ (*level)--;
++
++ if (*level == 0)
++ return replace_node_with_clone(path, 0);
++
+ return 0;
+ }
+
+@@ -6734,8 +6888,10 @@ static int tree_move_next_or_upnext(struct btrfs_path *path,
+ path->slots[*level]++;
+
+ while (path->slots[*level] >= nritems) {
+- if (*level == root_level)
++ if (*level == root_level) {
++ path->slots[*level] = nritems - 1;
+ return -1;
++ }
+
+ /* move upnext */
+ path->slots[*level] = 0;
+@@ -6767,14 +6923,20 @@ static int tree_advance(struct btrfs_path *path,
+ } else {
+ ret = tree_move_down(path, level, reada_min_gen);
+ }
+- if (ret >= 0) {
+- if (*level == 0)
+- btrfs_item_key_to_cpu(path->nodes[*level], key,
+- path->slots[*level]);
+- else
+- btrfs_node_key_to_cpu(path->nodes[*level], key,
+- path->slots[*level]);
+- }
++
++ /*
++ * Even if we have reached the end of a tree, ret is -1, update the key
++ * anyway, so that in case we need to restart due to a block group
++ * relocation, we can assert that the last key of the root node still
++ * exists in the tree.
++ */
++ if (*level == 0)
++ btrfs_item_key_to_cpu(path->nodes[*level], key,
++ path->slots[*level]);
++ else
++ btrfs_node_key_to_cpu(path->nodes[*level], key,
++ path->slots[*level]);
++
+ return ret;
+ }
+
+@@ -6803,6 +6965,97 @@ static int tree_compare_item(struct btrfs_path *left_path,
+ return 0;
+ }
+
++/*
++ * A transaction used for relocating a block group was committed or is about to
++ * finish its commit. Release our paths and restart the search, so that we are
++ * not using stale extent buffers:
++ *
++ * 1) For levels > 0, we are only holding references of extent buffers, without
++ * any locks on them, which does not prevent them from having been relocated
++ * and reallocated after the last time we released the commit root semaphore.
++ * The exception are the root nodes, for which we always have a clone, see
++ * the comment at btrfs_compare_trees();
++ *
++ * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
++ * we are safe from the concurrent relocation and reallocation. However they
++ * can have file extent items with a pre relocation disk_bytenr value, so we
++ * restart the start from the current commit roots and clone the new leaves so
++ * that we get the post relocation disk_bytenr values. Not doing so, could
++ * make us clone the wrong data in case there are new extents using the old
++ * disk_bytenr that happen to be shared.
++ */
++static int restart_after_relocation(struct btrfs_path *left_path,
++ struct btrfs_path *right_path,
++ const struct btrfs_key *left_key,
++ const struct btrfs_key *right_key,
++ int left_level,
++ int right_level,
++ const struct send_ctx *sctx)
++{
++ int root_level;
++ int ret;
++
++ lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
++
++ btrfs_release_path(left_path);
++ btrfs_release_path(right_path);
++
++ /*
++ * Since keys can not be added or removed to/from our roots because they
++ * are readonly and we do not allow deduplication to run in parallel
++ * (which can add, remove or change keys), the layout of the trees should
++ * not change.
++ */
++ left_path->lowest_level = left_level;
++ ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
++ if (ret < 0)
++ return ret;
++
++ right_path->lowest_level = right_level;
++ ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
++ if (ret < 0)
++ return ret;
++
++ /*
++ * If the lowest level nodes are leaves, clone them so that they can be
++ * safely used by changed_cb() while not under the protection of the
++ * commit root semaphore, even if relocation and reallocation happens in
++ * parallel.
++ */
++ if (left_level == 0) {
++ ret = replace_node_with_clone(left_path, 0);
++ if (ret < 0)
++ return ret;
++ }
++
++ if (right_level == 0) {
++ ret = replace_node_with_clone(right_path, 0);
++ if (ret < 0)
++ return ret;
++ }
++
++ /*
++ * Now clone the root nodes (unless they happen to be the leaves we have
++ * already cloned). This is to protect against concurrent snapshotting of
++ * the send and parent roots (see the comment at btrfs_compare_trees()).
++ */
++ root_level = btrfs_header_level(sctx->send_root->commit_root);
++ if (root_level > 0) {
++ ret = replace_node_with_clone(left_path, root_level);
++ if (ret < 0)
++ return ret;
++ }
++
++ root_level = btrfs_header_level(sctx->parent_root->commit_root);
++ if (root_level > 0) {
++ ret = replace_node_with_clone(right_path, root_level);
++ if (ret < 0)
++ return ret;
++ }
++
++ return 0;
++}
++
+ /*
+ * This function compares two trees and calls the provided callback for
+ * every changed/new/deleted item it finds.
+@@ -6831,10 +7084,10 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ int right_root_level;
+ int left_level;
+ int right_level;
+- int left_end_reached;
+- int right_end_reached;
+- int advance_left;
+- int advance_right;
++ int left_end_reached = 0;
++ int right_end_reached = 0;
++ int advance_left = 0;
++ int advance_right = 0;
+ u64 left_blockptr;
+ u64 right_blockptr;
+ u64 left_gen;
+@@ -6902,12 +7155,18 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ down_read(&fs_info->commit_root_sem);
+ left_level = btrfs_header_level(left_root->commit_root);
+ left_root_level = left_level;
++ /*
++ * We clone the root node of the send and parent roots to prevent races
++ * with snapshot creation of these roots. Snapshot creation COWs the
++ * root node of a tree, so after the transaction is committed the old
++ * extent can be reallocated while this send operation is still ongoing.
++ * So we clone them, under the commit root semaphore, to be race free.
++ */
+ left_path->nodes[left_level] =
+ btrfs_clone_extent_buffer(left_root->commit_root);
+ if (!left_path->nodes[left_level]) {
+- up_read(&fs_info->commit_root_sem);
+ ret = -ENOMEM;
+- goto out;
++ goto out_unlock;
+ }
+
+ right_level = btrfs_header_level(right_root->commit_root);
+@@ -6915,9 +7174,8 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ right_path->nodes[right_level] =
+ btrfs_clone_extent_buffer(right_root->commit_root);
+ if (!right_path->nodes[right_level]) {
+- up_read(&fs_info->commit_root_sem);
+ ret = -ENOMEM;
+- goto out;
++ goto out_unlock;
+ }
+ /*
+ * Our right root is the parent root, while the left root is the "send"
+@@ -6927,7 +7185,6 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ * will need to read them at some point.
+ */
+ reada_min_gen = btrfs_header_generation(right_root->commit_root);
+- up_read(&fs_info->commit_root_sem);
+
+ if (left_level == 0)
+ btrfs_item_key_to_cpu(left_path->nodes[left_level],
+@@ -6942,11 +7199,26 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ btrfs_node_key_to_cpu(right_path->nodes[right_level],
+ &right_key, right_path->slots[right_level]);
+
+- left_end_reached = right_end_reached = 0;
+- advance_left = advance_right = 0;
++ sctx->last_reloc_trans = fs_info->last_reloc_trans;
+
+ while (1) {
+- cond_resched();
++ if (need_resched() ||
++ rwsem_is_contended(&fs_info->commit_root_sem)) {
++ up_read(&fs_info->commit_root_sem);
++ cond_resched();
++ down_read(&fs_info->commit_root_sem);
++ }
++
++ if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
++ ret = restart_after_relocation(left_path, right_path,
++ &left_key, &right_key,
++ left_level, right_level,
++ sctx);
++ if (ret < 0)
++ goto out_unlock;
++ sctx->last_reloc_trans = fs_info->last_reloc_trans;
++ }
++
+ if (advance_left && !left_end_reached) {
+ ret = tree_advance(left_path, &left_level,
+ left_root_level,
+@@ -6955,7 +7227,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ if (ret == -1)
+ left_end_reached = ADVANCE;
+ else if (ret < 0)
+- goto out;
++ goto out_unlock;
+ advance_left = 0;
+ }
+ if (advance_right && !right_end_reached) {
+@@ -6966,54 +7238,55 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ if (ret == -1)
+ right_end_reached = ADVANCE;
+ else if (ret < 0)
+- goto out;
++ goto out_unlock;
+ advance_right = 0;
+ }
+
+ if (left_end_reached && right_end_reached) {
+ ret = 0;
+- goto out;
++ goto out_unlock;
+ } else if (left_end_reached) {
+ if (right_level == 0) {
++ up_read(&fs_info->commit_root_sem);
+ ret = changed_cb(left_path, right_path,
+ &right_key,
+ BTRFS_COMPARE_TREE_DELETED,
+ sctx);
+ if (ret < 0)
+ goto out;
++ down_read(&fs_info->commit_root_sem);
+ }
+ advance_right = ADVANCE;
+ continue;
+ } else if (right_end_reached) {
+ if (left_level == 0) {
++ up_read(&fs_info->commit_root_sem);
+ ret = changed_cb(left_path, right_path,
+ &left_key,
+ BTRFS_COMPARE_TREE_NEW,
+ sctx);
+ if (ret < 0)
+ goto out;
++ down_read(&fs_info->commit_root_sem);
+ }
+ advance_left = ADVANCE;
+ continue;
+ }
+
+ if (left_level == 0 && right_level == 0) {
++ up_read(&fs_info->commit_root_sem);
+ cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
+ if (cmp < 0) {
+ ret = changed_cb(left_path, right_path,
+ &left_key,
+ BTRFS_COMPARE_TREE_NEW,
+ sctx);
+- if (ret < 0)
+- goto out;
+ advance_left = ADVANCE;
+ } else if (cmp > 0) {
+ ret = changed_cb(left_path, right_path,
+ &right_key,
+ BTRFS_COMPARE_TREE_DELETED,
+ sctx);
+- if (ret < 0)
+- goto out;
+ advance_right = ADVANCE;
+ } else {
+ enum btrfs_compare_tree_result result;
+@@ -7027,11 +7300,13 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ result = BTRFS_COMPARE_TREE_SAME;
+ ret = changed_cb(left_path, right_path,
+ &left_key, result, sctx);
+- if (ret < 0)
+- goto out;
+ advance_left = ADVANCE;
+ advance_right = ADVANCE;
+ }
++
++ if (ret < 0)
++ goto out;
++ down_read(&fs_info->commit_root_sem);
+ } else if (left_level == right_level) {
+ cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
+ if (cmp < 0) {
+@@ -7071,6 +7346,8 @@ static int btrfs_compare_trees(struct btrfs_root *left_root,
+ }
+ }
+
++out_unlock:
++ up_read(&fs_info->commit_root_sem);
+ out:
+ btrfs_free_path(left_path);
+ btrfs_free_path(right_path);
+@@ -7409,21 +7686,7 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg)
+ if (ret)
+ goto out;
+
+- spin_lock(&fs_info->send_reloc_lock);
+- if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) {
+- spin_unlock(&fs_info->send_reloc_lock);
+- btrfs_warn_rl(fs_info,
+- "cannot run send because a relocation operation is in progress");
+- ret = -EAGAIN;
+- goto out;
+- }
+- fs_info->send_in_progress++;
+- spin_unlock(&fs_info->send_reloc_lock);
+-
+ ret = send_subvol(sctx);
+- spin_lock(&fs_info->send_reloc_lock);
+- fs_info->send_in_progress--;
+- spin_unlock(&fs_info->send_reloc_lock);
+ if (ret < 0)
+ goto out;
+
+diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
+index aa5be0b24987a..5ed66a794e577 100644
+--- a/fs/btrfs/space-info.c
++++ b/fs/btrfs/space-info.c
+@@ -1054,7 +1054,6 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work)
+ trans_rsv->reserved;
+ if (block_rsv_size < space_info->bytes_may_use)
+ delalloc_size = space_info->bytes_may_use - block_rsv_size;
+- spin_unlock(&space_info->lock);
+
+ /*
+ * We don't want to include the global_rsv in our calculation,
+@@ -1085,6 +1084,8 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work)
+ flush = FLUSH_DELAYED_REFS_NR;
+ }
+
++ spin_unlock(&space_info->lock);
++
+ /*
+ * We don't want to reclaim everything, just a portion, so scale
+ * down the to_reclaim by 1/4. If it takes us down to 0,
+diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
+index 537d90bf5d844..7f91d62c2225a 100644
+--- a/fs/btrfs/super.c
++++ b/fs/btrfs/super.c
+@@ -1705,7 +1705,7 @@ static struct dentry *btrfs_mount_root(struct file_system_type *fs_type,
+ goto error_close_devices;
+ }
+
+- bdev = fs_devices->latest_bdev;
++ bdev = fs_devices->latest_dev->bdev;
+ s = sget(fs_type, btrfs_test_super, btrfs_set_super, flags | SB_NOSEC,
+ fs_info);
+ if (IS_ERR(s)) {
+@@ -2463,30 +2463,16 @@ static int btrfs_unfreeze(struct super_block *sb)
+ static int btrfs_show_devname(struct seq_file *m, struct dentry *root)
+ {
+ struct btrfs_fs_info *fs_info = btrfs_sb(root->d_sb);
+- struct btrfs_device *dev, *first_dev = NULL;
+
+ /*
+- * Lightweight locking of the devices. We should not need
+- * device_list_mutex here as we only read the device data and the list
+- * is protected by RCU. Even if a device is deleted during the list
+- * traversals, we'll get valid data, the freeing callback will wait at
+- * least until the rcu_read_unlock.
++ * There should be always a valid pointer in latest_dev, it may be stale
++ * for a short moment in case it's being deleted but still valid until
++ * the end of RCU grace period.
+ */
+ rcu_read_lock();
+- list_for_each_entry_rcu(dev, &fs_info->fs_devices->devices, dev_list) {
+- if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
+- continue;
+- if (!dev->name)
+- continue;
+- if (!first_dev || dev->devid < first_dev->devid)
+- first_dev = dev;
+- }
+-
+- if (first_dev)
+- seq_escape(m, rcu_str_deref(first_dev->name), " \t\n\\");
+- else
+- WARN_ON(1);
++ seq_escape(m, rcu_str_deref(fs_info->fs_devices->latest_dev->name), " \t\n\\");
+ rcu_read_unlock();
++
+ return 0;
+ }
+
+diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
+index 14b9fdc8aaa9a..642cd2b55fa08 100644
+--- a/fs/btrfs/transaction.c
++++ b/fs/btrfs/transaction.c
+@@ -163,6 +163,10 @@ static noinline void switch_commit_roots(struct btrfs_trans_handle *trans)
+ struct btrfs_caching_control *caching_ctl, *next;
+
+ down_write(&fs_info->commit_root_sem);
++
++ if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags))
++ fs_info->last_reloc_trans = trans->transid;
++
+ list_for_each_entry_safe(root, tmp, &cur_trans->switch_commits,
+ dirty_list) {
+ list_del_init(&root->dirty_list);
+@@ -846,7 +850,37 @@ btrfs_attach_transaction_barrier(struct btrfs_root *root)
+ static noinline void wait_for_commit(struct btrfs_transaction *commit,
+ const enum btrfs_trans_state min_state)
+ {
+- wait_event(commit->commit_wait, commit->state >= min_state);
++ struct btrfs_fs_info *fs_info = commit->fs_info;
++ u64 transid = commit->transid;
++ bool put = false;
++
++ while (1) {
++ wait_event(commit->commit_wait, commit->state >= min_state);
++ if (put)
++ btrfs_put_transaction(commit);
++
++ if (min_state < TRANS_STATE_COMPLETED)
++ break;
++
++ /*
++ * A transaction isn't really completed until all of the
++ * previous transactions are completed, but with fsync we can
++ * end up with SUPER_COMMITTED transactions before a COMPLETED
++ * transaction. Wait for those.
++ */
++
++ spin_lock(&fs_info->trans_lock);
++ commit = list_first_entry_or_null(&fs_info->trans_list,
++ struct btrfs_transaction,
++ list);
++ if (!commit || commit->transid > transid) {
++ spin_unlock(&fs_info->trans_lock);
++ break;
++ }
++ refcount_inc(&commit->use_count);
++ put = true;
++ spin_unlock(&fs_info->trans_lock);
++ }
+ }
+
+ int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid)
+@@ -1310,6 +1344,32 @@ again:
+ return 0;
+ }
+
++/*
++ * If we had a pending drop we need to see if there are any others left in our
++ * dead roots list, and if not clear our bit and wake any waiters.
++ */
++void btrfs_maybe_wake_unfinished_drop(struct btrfs_fs_info *fs_info)
++{
++ /*
++ * We put the drop in progress roots at the front of the list, so if the
++ * first entry doesn't have UNFINISHED_DROP set we can wake everybody
++ * up.
++ */
++ spin_lock(&fs_info->trans_lock);
++ if (!list_empty(&fs_info->dead_roots)) {
++ struct btrfs_root *root = list_first_entry(&fs_info->dead_roots,
++ struct btrfs_root,
++ root_list);
++ if (test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state)) {
++ spin_unlock(&fs_info->trans_lock);
++ return;
++ }
++ }
++ spin_unlock(&fs_info->trans_lock);
++
++ btrfs_wake_unfinished_drop(fs_info);
++}
++
+ /*
+ * dead roots are old snapshots that need to be deleted. This allocates
+ * a dirty root struct and adds it into the list of dead roots that need to
+@@ -1322,7 +1382,12 @@ void btrfs_add_dead_root(struct btrfs_root *root)
+ spin_lock(&fs_info->trans_lock);
+ if (list_empty(&root->root_list)) {
+ btrfs_grab_root(root);
+- list_add_tail(&root->root_list, &fs_info->dead_roots);
++
++ /* We want to process the partially complete drops first. */
++ if (test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state))
++ list_add(&root->root_list, &fs_info->dead_roots);
++ else
++ list_add_tail(&root->root_list, &fs_info->dead_roots);
+ }
+ spin_unlock(&fs_info->trans_lock);
+ }
+@@ -2014,16 +2079,24 @@ static void btrfs_cleanup_pending_block_groups(struct btrfs_trans_handle *trans)
+ static inline int btrfs_start_delalloc_flush(struct btrfs_fs_info *fs_info)
+ {
+ /*
+- * We use writeback_inodes_sb here because if we used
++ * We use try_to_writeback_inodes_sb() here because if we used
+ * btrfs_start_delalloc_roots we would deadlock with fs freeze.
+ * Currently are holding the fs freeze lock, if we do an async flush
+ * we'll do btrfs_join_transaction() and deadlock because we need to
+ * wait for the fs freeze lock. Using the direct flushing we benefit
+ * from already being in a transaction and our join_transaction doesn't
+ * have to re-take the fs freeze lock.
++ *
++ * Note that try_to_writeback_inodes_sb() will only trigger writeback
++ * if it can read lock sb->s_umount. It will always be able to lock it,
++ * except when the filesystem is being unmounted or being frozen, but in
++ * those cases sync_filesystem() is called, which results in calling
++ * writeback_inodes_sb() while holding a write lock on sb->s_umount.
++ * Note that we don't call writeback_inodes_sb() directly, because it
++ * will emit a warning if sb->s_umount is not locked.
+ */
+ if (btrfs_test_opt(fs_info, FLUSHONCOMMIT))
+- writeback_inodes_sb(fs_info->sb, WB_REASON_SYNC);
++ try_to_writeback_inodes_sb(fs_info->sb, WB_REASON_SYNC);
+ return 0;
+ }
+
+@@ -2033,6 +2106,27 @@ static inline void btrfs_wait_delalloc_flush(struct btrfs_fs_info *fs_info)
+ btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
+ }
+
++/*
++ * Add a pending snapshot associated with the given transaction handle to the
++ * respective handle. This must be called after the transaction commit started
++ * and while holding fs_info->trans_lock.
++ * This serves to guarantee a caller of btrfs_commit_transaction() that it can
++ * safely free the pending snapshot pointer in case btrfs_commit_transaction()
++ * returns an error.
++ */
++static void add_pending_snapshot(struct btrfs_trans_handle *trans)
++{
++ struct btrfs_transaction *cur_trans = trans->transaction;
++
++ if (!trans->pending_snapshot)
++ return;
++
++ lockdep_assert_held(&trans->fs_info->trans_lock);
++ ASSERT(cur_trans->state >= TRANS_STATE_COMMIT_START);
++
++ list_add(&trans->pending_snapshot->list, &cur_trans->pending_snapshots);
++}
++
+ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
+ {
+ struct btrfs_fs_info *fs_info = trans->fs_info;
+@@ -2106,6 +2200,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
+ if (cur_trans->state >= TRANS_STATE_COMMIT_START) {
+ enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED;
+
++ add_pending_snapshot(trans);
++
+ spin_unlock(&fs_info->trans_lock);
+ refcount_inc(&cur_trans->use_count);
+
+@@ -2196,6 +2292,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
+ * COMMIT_DOING so make sure to wait for num_writers to == 1 again.
+ */
+ spin_lock(&fs_info->trans_lock);
++ add_pending_snapshot(trans);
+ cur_trans->state = TRANS_STATE_COMMIT_DOING;
+ spin_unlock(&fs_info->trans_lock);
+ wait_event(cur_trans->writer_wait,
+diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h
+index ba45065f94511..0ded32bbd001e 100644
+--- a/fs/btrfs/transaction.h
++++ b/fs/btrfs/transaction.h
+@@ -123,6 +123,8 @@ struct btrfs_trans_handle {
+ struct btrfs_transaction *transaction;
+ struct btrfs_block_rsv *block_rsv;
+ struct btrfs_block_rsv *orig_rsv;
++ /* Set by a task that wants to create a snapshot. */
++ struct btrfs_pending_snapshot *pending_snapshot;
+ refcount_t use_count;
+ unsigned int type;
+ /*
+@@ -215,6 +217,7 @@ int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid);
+
+ void btrfs_add_dead_root(struct btrfs_root *root);
+ int btrfs_defrag_root(struct btrfs_root *root);
++void btrfs_maybe_wake_unfinished_drop(struct btrfs_fs_info *fs_info);
+ int btrfs_clean_one_deleted_snapshot(struct btrfs_root *root);
+ int btrfs_commit_transaction(struct btrfs_trans_handle *trans);
+ int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans);
+diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
+index 7733e8ac0a698..51382d2be3d44 100644
+--- a/fs/btrfs/tree-checker.c
++++ b/fs/btrfs/tree-checker.c
+@@ -965,6 +965,7 @@ static int check_dev_item(struct extent_buffer *leaf,
+ struct btrfs_key *key, int slot)
+ {
+ struct btrfs_dev_item *ditem;
++ const u32 item_size = btrfs_item_size_nr(leaf, slot);
+
+ if (unlikely(key->objectid != BTRFS_DEV_ITEMS_OBJECTID)) {
+ dev_item_err(leaf, slot,
+@@ -972,6 +973,13 @@ static int check_dev_item(struct extent_buffer *leaf,
+ key->objectid, BTRFS_DEV_ITEMS_OBJECTID);
+ return -EUCLEAN;
+ }
++
++ if (unlikely(item_size != sizeof(*ditem))) {
++ dev_item_err(leaf, slot, "invalid item size: has %u expect %zu",
++ item_size, sizeof(*ditem));
++ return -EUCLEAN;
++ }
++
+ ditem = btrfs_item_ptr(leaf, slot, struct btrfs_dev_item);
+ if (unlikely(btrfs_device_id(leaf, ditem) != key->offset)) {
+ dev_item_err(leaf, slot,
+@@ -1007,6 +1015,7 @@ static int check_inode_item(struct extent_buffer *leaf,
+ struct btrfs_inode_item *iitem;
+ u64 super_gen = btrfs_super_generation(fs_info->super_copy);
+ u32 valid_mask = (S_IFMT | S_ISUID | S_ISGID | S_ISVTX | 0777);
++ const u32 item_size = btrfs_item_size_nr(leaf, slot);
+ u32 mode;
+ int ret;
+ u32 flags;
+@@ -1016,6 +1025,12 @@ static int check_inode_item(struct extent_buffer *leaf,
+ if (unlikely(ret < 0))
+ return ret;
+
++ if (unlikely(item_size != sizeof(*iitem))) {
++ generic_err(leaf, slot, "invalid item size: has %u expect %zu",
++ item_size, sizeof(*iitem));
++ return -EUCLEAN;
++ }
++
+ iitem = btrfs_item_ptr(leaf, slot, struct btrfs_inode_item);
+
+ /* Here we use super block generation + 1 to handle log tree */
+diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
+index b415c5ec03ea0..1221d8483d63b 100644
+--- a/fs/btrfs/tree-log.c
++++ b/fs/btrfs/tree-log.c
+@@ -1153,6 +1153,7 @@ again:
+ parent_objectid, victim_name,
+ victim_name_len);
+ if (ret < 0) {
++ kfree(victim_name);
+ return ret;
+ } else if (!ret) {
+ ret = -ENOENT;
+@@ -1328,6 +1329,15 @@ again:
+ inode, name, namelen);
+ kfree(name);
+ iput(dir);
++ /*
++ * Whenever we need to check if a name exists or not, we
++ * check the subvolume tree. So after an unlink we must
++ * run delayed items, so that future checks for a name
++ * during log replay see that the name does not exists
++ * anymore.
++ */
++ if (!ret)
++ ret = btrfs_run_delayed_items(trans);
+ if (ret)
+ goto out;
+ goto again;
+@@ -1579,6 +1589,15 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans,
+ */
+ if (!ret && inode->i_nlink == 0)
+ inc_nlink(inode);
++ /*
++ * Whenever we need to check if a name exists or
++ * not, we check the subvolume tree. So after an
++ * unlink we must run delayed items, so that future
++ * checks for a name during log replay see that the
++ * name does not exists anymore.
++ */
++ if (!ret)
++ ret = btrfs_run_delayed_items(trans);
+ }
+ if (ret < 0)
+ goto out;
+@@ -2500,7 +2519,9 @@ again:
+ else {
+ ret = find_dir_range(log, path, dirid, key_type,
+ &range_start, &range_end);
+- if (ret != 0)
++ if (ret < 0)
++ goto out;
++ else if (ret > 0)
+ break;
+ }
+
+@@ -2877,6 +2898,8 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans,
+ path->nodes[*level]->len);
+ if (ret)
+ return ret;
++ btrfs_redirty_list_add(trans->transaction,
++ next);
+ } else {
+ if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags))
+ clear_extent_buffer_dirty(next);
+@@ -2957,6 +2980,7 @@ static int walk_log_tree(struct btrfs_trans_handle *trans,
+ next->start, next->len);
+ if (ret)
+ goto out;
++ btrfs_redirty_list_add(trans->transaction, next);
+ } else {
+ if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags))
+ clear_extent_buffer_dirty(next);
+@@ -3192,6 +3216,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans,
+ ret = btrfs_alloc_log_tree_node(trans, log_root_tree);
+ if (ret) {
+ mutex_unlock(&fs_info->tree_root->log_mutex);
++ blk_finish_plug(&plug);
+ goto out;
+ }
+ }
+@@ -3399,6 +3424,29 @@ static void free_log_tree(struct btrfs_trans_handle *trans,
+ if (log->node) {
+ ret = walk_log_tree(trans, log, &wc);
+ if (ret) {
++ /*
++ * We weren't able to traverse the entire log tree, the
++ * typical scenario is getting an -EIO when reading an
++ * extent buffer of the tree, due to a previous writeback
++ * failure of it.
++ */
++ set_bit(BTRFS_FS_STATE_LOG_CLEANUP_ERROR,
++ &log->fs_info->fs_state);
++
++ /*
++ * Some extent buffers of the log tree may still be dirty
++ * and not yet written back to storage, because we may
++ * have updates to a log tree without syncing a log tree,
++ * such as during rename and link operations. So flush
++ * them out and wait for their writeback to complete, so
++ * that we properly cleanup their state and pages.
++ */
++ btrfs_write_marked_extents(log->fs_info,
++ &log->dirty_log_pages,
++ EXTENT_DIRTY | EXTENT_NEW);
++ btrfs_wait_tree_log_extents(log,
++ EXTENT_DIRTY | EXTENT_NEW);
++
+ if (trans)
+ btrfs_abort_transaction(trans, ret);
+ else
+@@ -3410,8 +3458,6 @@ static void free_log_tree(struct btrfs_trans_handle *trans,
+ EXTENT_DIRTY | EXTENT_NEW | EXTENT_NEED_WAIT);
+ extent_io_tree_release(&log->log_csum_range);
+
+- if (trans && log->node)
+- btrfs_redirty_list_add(trans->transaction, log->node);
+ btrfs_put_root(log);
+ }
+
+@@ -4419,7 +4465,7 @@ static int log_one_extent(struct btrfs_trans_handle *trans,
+
+ /*
+ * Log all prealloc extents beyond the inode's i_size to make sure we do not
+- * lose them after doing a fast fsync and replaying the log. We scan the
++ * lose them after doing a full/fast fsync and replaying the log. We scan the
+ * subvolume's root instead of iterating the inode's extent map tree because
+ * otherwise we can log incorrect extent items based on extent map conversion.
+ * That can happen due to the fact that extent maps are merged when they
+@@ -5204,6 +5250,7 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans,
+ struct btrfs_log_ctx *ctx,
+ bool *need_log_inode_item)
+ {
++ const u64 i_size = i_size_read(&inode->vfs_inode);
+ struct btrfs_root *root = inode->root;
+ int ins_start_slot = 0;
+ int ins_nr = 0;
+@@ -5224,13 +5271,21 @@ again:
+ if (min_key->type > max_key->type)
+ break;
+
+- if (min_key->type == BTRFS_INODE_ITEM_KEY)
++ if (min_key->type == BTRFS_INODE_ITEM_KEY) {
+ *need_log_inode_item = false;
+-
+- if ((min_key->type == BTRFS_INODE_REF_KEY ||
+- min_key->type == BTRFS_INODE_EXTREF_KEY) &&
+- inode->generation == trans->transid &&
+- !recursive_logging) {
++ } else if (min_key->type == BTRFS_EXTENT_DATA_KEY &&
++ min_key->offset >= i_size) {
++ /*
++ * Extents at and beyond eof are logged with
++ * btrfs_log_prealloc_extents().
++ * Only regular files have BTRFS_EXTENT_DATA_KEY keys,
++ * and no keys greater than that, so bail out.
++ */
++ break;
++ } else if ((min_key->type == BTRFS_INODE_REF_KEY ||
++ min_key->type == BTRFS_INODE_EXTREF_KEY) &&
++ inode->generation == trans->transid &&
++ !recursive_logging) {
+ u64 other_ino = 0;
+ u64 other_parent = 0;
+
+@@ -5261,10 +5316,8 @@ again:
+ btrfs_release_path(path);
+ goto next_key;
+ }
+- }
+-
+- /* Skip xattrs, we log them later with btrfs_log_all_xattrs() */
+- if (min_key->type == BTRFS_XATTR_ITEM_KEY) {
++ } else if (min_key->type == BTRFS_XATTR_ITEM_KEY) {
++ /* Skip xattrs, logged later with btrfs_log_all_xattrs() */
+ if (ins_nr == 0)
+ goto next_slot;
+ ret = copy_items(trans, inode, dst_path, path,
+@@ -5317,9 +5370,21 @@ next_key:
+ break;
+ }
+ }
+- if (ins_nr)
++ if (ins_nr) {
+ ret = copy_items(trans, inode, dst_path, path, ins_start_slot,
+ ins_nr, inode_only, logged_isize);
++ if (ret)
++ return ret;
++ }
++
++ if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) {
++ /*
++ * Release the path because otherwise we might attempt to double
++ * lock the same leaf with btrfs_log_prealloc_extents() below.
++ */
++ btrfs_release_path(path);
++ ret = btrfs_log_prealloc_extents(trans, inode, dst_path);
++ }
+
+ return ret;
+ }
+@@ -5418,6 +5483,18 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans,
+ mutex_lock(&inode->log_mutex);
+ }
+
++ /*
++ * For symlinks, we must always log their content, which is stored in an
++ * inline extent, otherwise we could end up with an empty symlink after
++ * log replay, which is invalid on linux (symlink(2) returns -ENOENT if
++ * one attempts to create an empty symlink).
++ * We don't need to worry about flushing delalloc, because when we create
++ * the inline extent when the symlink is created (we never have delalloc
++ * for symlinks).
++ */
++ if (S_ISLNK(inode->vfs_inode.i_mode))
++ inode_only = LOG_INODE_ALL;
++
+ /*
+ * This is for cases where logging a directory could result in losing a
+ * a file after replaying the log. For example, if we move a file from a
+@@ -5788,7 +5865,7 @@ process_leaf:
+ }
+
+ ctx->log_new_dentries = false;
+- if (type == BTRFS_FT_DIR || type == BTRFS_FT_SYMLINK)
++ if (type == BTRFS_FT_DIR)
+ log_mode = LOG_INODE_ALL;
+ ret = btrfs_log_inode(trans, root, BTRFS_I(di_inode),
+ log_mode, ctx);
+diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
+index 2ec3b8ac8fa35..471cc4706a074 100644
+--- a/fs/btrfs/volumes.c
++++ b/fs/btrfs/volumes.c
+@@ -14,6 +14,7 @@
+ #include <linux/semaphore.h>
+ #include <linux/uuid.h>
+ #include <linux/list_sort.h>
++#include <linux/namei.h>
+ #include "misc.h"
+ #include "ctree.h"
+ #include "extent_map.h"
+@@ -529,15 +530,48 @@ error:
+ return ret;
+ }
+
+-static bool device_path_matched(const char *path, struct btrfs_device *device)
++/*
++ * Check if the device in the path matches the device in the given struct device.
++ *
++ * Returns:
++ * true If it is the same device.
++ * false If it is not the same device or on error.
++ */
++static bool device_matched(const struct btrfs_device *device, const char *path)
+ {
+- int found;
++ char *device_name;
++ dev_t dev_old;
++ dev_t dev_new;
++ int ret;
++
++ /*
++ * If we are looking for a device with the matching dev_t, then skip
++ * device without a name (a missing device).
++ */
++ if (!device->name)
++ return false;
++
++ device_name = kzalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
++ if (!device_name)
++ return false;
+
+ rcu_read_lock();
+- found = strcmp(rcu_str_deref(device->name), path);
++ scnprintf(device_name, BTRFS_PATH_NAME_MAX, "%s", rcu_str_deref(device->name));
+ rcu_read_unlock();
+
+- return found == 0;
++ ret = lookup_bdev(device_name, &dev_old);
++ kfree(device_name);
++ if (ret)
++ return false;
++
++ ret = lookup_bdev(path, &dev_new);
++ if (ret)
++ return false;
++
++ if (dev_old == dev_new)
++ return true;
++
++ return false;
+ }
+
+ /*
+@@ -570,9 +604,7 @@ static int btrfs_free_stale_devices(const char *path,
+ &fs_devices->devices, dev_list) {
+ if (skip_device && skip_device == device)
+ continue;
+- if (path && !device->name)
+- continue;
+- if (path && !device_path_matched(path, device))
++ if (path && !device_matched(device, path))
+ continue;
+ if (fs_devices->opened) {
+ /* for an already deleted device return 0 */
+@@ -1091,7 +1123,7 @@ void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
+ list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
+ __btrfs_free_extra_devids(seed_dev, &latest_dev);
+
+- fs_devices->latest_bdev = latest_dev->bdev;
++ fs_devices->latest_dev = latest_dev;
+
+ mutex_unlock(&uuid_mutex);
+ }
+@@ -1122,8 +1154,10 @@ static void btrfs_close_one_device(struct btrfs_device *device)
+ if (device->devid == BTRFS_DEV_REPLACE_DEVID)
+ clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
+
+- if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
++ if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
++ clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
+ fs_devices->missing_devices--;
++ }
+
+ btrfs_close_bdev(device);
+ if (device->bdev) {
+@@ -1222,7 +1256,7 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
+ return -EINVAL;
+
+ fs_devices->opened = 1;
+- fs_devices->latest_bdev = latest_dev->bdev;
++ fs_devices->latest_dev = latest_dev;
+ fs_devices->total_rw_bytes = 0;
+ fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
+ fs_devices->read_policy = BTRFS_READ_POLICY_PID;
+@@ -1363,8 +1397,10 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
+
+ bytenr_orig = btrfs_sb_offset(0);
+ ret = btrfs_sb_log_location_bdev(bdev, 0, READ, &bytenr);
+- if (ret)
+- return ERR_PTR(ret);
++ if (ret) {
++ device = ERR_PTR(ret);
++ goto error_bdev_put;
++ }
+
+ disk_super = btrfs_read_disk_super(bdev, bytenr, bytenr_orig);
+ if (IS_ERR(disk_super)) {
+@@ -1882,18 +1918,22 @@ out:
+ /*
+ * Function to update ctime/mtime for a given device path.
+ * Mainly used for ctime/mtime based probe like libblkid.
++ *
++ * We don't care about errors here, this is just to be kind to userspace.
+ */
+-static void update_dev_time(struct block_device *bdev)
++static void update_dev_time(const char *device_path)
+ {
+- struct inode *inode = bdev->bd_inode;
++ struct path path;
+ struct timespec64 now;
++ int ret;
+
+- /* Shouldn't happen but just in case. */
+- if (!inode)
++ ret = kern_path(device_path, LOOKUP_FOLLOW, &path);
++ if (ret)
+ return;
+
+- now = current_time(inode);
+- generic_update_time(inode, &now, S_MTIME | S_CTIME);
++ now = current_time(d_inode(path.dentry));
++ inode_update_time(d_inode(path.dentry), &now, S_MTIME | S_CTIME);
++ path_put(&path);
+ }
+
+ static int btrfs_rm_dev_item(struct btrfs_device *device)
+@@ -1986,7 +2026,7 @@ static struct btrfs_device * btrfs_find_next_active_device(
+ }
+
+ /*
+- * Helper function to check if the given device is part of s_bdev / latest_bdev
++ * Helper function to check if the given device is part of s_bdev / latest_dev
+ * and replace it with the provided or the next active device, in the context
+ * where this function called, there should be always be another device (or
+ * this_dev) which is active.
+@@ -2005,8 +2045,8 @@ void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
+ (fs_info->sb->s_bdev == device->bdev))
+ fs_info->sb->s_bdev = next_device->bdev;
+
+- if (fs_info->fs_devices->latest_bdev == device->bdev)
+- fs_info->fs_devices->latest_bdev = next_device->bdev;
++ if (fs_info->fs_devices->latest_dev->bdev == device->bdev)
++ fs_info->fs_devices->latest_dev = next_device;
+ }
+
+ /*
+@@ -2069,7 +2109,7 @@ void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
+ btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
+
+ /* Update ctime/mtime for device path for libblkid */
+- update_dev_time(bdev);
++ update_dev_time(device_path);
+ }
+
+ int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
+@@ -2081,8 +2121,11 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
+ u64 num_devices;
+ int ret = 0;
+
+- mutex_lock(&uuid_mutex);
+-
++ /*
++ * The device list in fs_devices is accessed without locks (neither
++ * uuid_mutex nor device_list_mutex) as it won't change on a mounted
++ * filesystem and another device rm cannot run.
++ */
+ num_devices = btrfs_num_devices(fs_info);
+
+ ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
+@@ -2126,11 +2169,9 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
+ mutex_unlock(&fs_info->chunk_mutex);
+ }
+
+- mutex_unlock(&uuid_mutex);
+ ret = btrfs_shrink_device(device, 0);
+ if (!ret)
+ btrfs_reada_remove_dev(device);
+- mutex_lock(&uuid_mutex);
+ if (ret)
+ goto error_undo;
+
+@@ -2217,7 +2258,6 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
+ }
+
+ out:
+- mutex_unlock(&uuid_mutex);
+ return ret;
+
+ error_undo:
+@@ -2305,13 +2345,6 @@ void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
+
+ mutex_unlock(&fs_devices->device_list_mutex);
+
+- /*
+- * The update_dev_time() with in btrfs_scratch_superblocks()
+- * may lead to a call to btrfs_show_devname() which will try
+- * to hold device_list_mutex. And here this device
+- * is already out of device list, so we don't have to hold
+- * the device_list_mutex lock.
+- */
+ btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev->bdev,
+ tgtdev->name->str);
+
+@@ -2594,7 +2627,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
+ device->fs_info = fs_info;
+ device->bdev = bdev;
+
+- ret = btrfs_get_dev_zone_info(device);
++ ret = btrfs_get_dev_zone_info(device, false);
+ if (ret)
+ goto error_free_device;
+
+@@ -2627,6 +2660,8 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
+ btrfs_abort_transaction(trans, ret);
+ goto error_trans;
+ }
++ btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev,
++ device);
+ }
+
+ device->fs_devices = fs_devices;
+@@ -2733,7 +2768,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
+ btrfs_forget_devices(device_path);
+
+ /* Update ctime/mtime for blkid or udev */
+- update_dev_time(bdev);
++ update_dev_time(device_path);
+
+ return ret;
+
+@@ -4354,10 +4389,12 @@ static int balance_kthread(void *data)
+ struct btrfs_fs_info *fs_info = data;
+ int ret = 0;
+
++ sb_start_write(fs_info->sb);
+ mutex_lock(&fs_info->balance_mutex);
+ if (fs_info->balance_ctl)
+ ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL);
+ mutex_unlock(&fs_info->balance_mutex);
++ sb_end_write(fs_info->sb);
+
+ return ret;
+ }
+@@ -7481,6 +7518,19 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
+ */
+ fs_info->fs_devices->total_rw_bytes = 0;
+
++ /*
++ * Lockdep complains about possible circular locking dependency between
++ * a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores
++ * used for freeze procection of a fs (struct super_block.s_writers),
++ * which we take when starting a transaction, and extent buffers of the
++ * chunk tree if we call read_one_dev() while holding a lock on an
++ * extent buffer of the chunk tree. Since we are mounting the filesystem
++ * and at this point there can't be any concurrent task modifying the
++ * chunk tree, to keep it simple, just skip locking on the chunk tree.
++ */
++ ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
++ path->skip_locking = 1;
++
+ /*
+ * Read all device items, and then all the chunk items. All
+ * device items are found before any chunk item (their object id
+@@ -7506,10 +7556,6 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
+ goto error;
+ break;
+ }
+- /*
+- * The nodes on level 1 are not locked but we don't need to do
+- * that during mount time as nothing else can access the tree
+- */
+ node = path->nodes[1];
+ if (node) {
+ if (last_ra_node != node->start) {
+@@ -7537,7 +7583,6 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
+ * requirement for chunk allocation, see the comment on
+ * top of btrfs_chunk_alloc() for details.
+ */
+- ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
+ chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
+ ret = read_one_chunk(&found_key, leaf, chunk);
+ if (ret)
+@@ -8173,10 +8218,12 @@ static int relocating_repair_kthread(void *data)
+ target = cache->start;
+ btrfs_put_block_group(cache);
+
++ sb_start_write(fs_info->sb);
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
+ btrfs_info(fs_info,
+ "zoned: skip relocating block group %llu to repair: EBUSY",
+ target);
++ sb_end_write(fs_info->sb);
+ return -EBUSY;
+ }
+
+@@ -8204,6 +8251,7 @@ out:
+ btrfs_put_block_group(cache);
+ mutex_unlock(&fs_info->reclaim_bgs_lock);
+ btrfs_exclop_finish(fs_info);
++ sb_end_write(fs_info->sb);
+
+ return ret;
+ }
+diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
+index 2183361db614d..4db10d071d67f 100644
+--- a/fs/btrfs/volumes.h
++++ b/fs/btrfs/volumes.h
+@@ -246,7 +246,11 @@ struct btrfs_fs_devices {
+ /* Highest generation number of seen devices */
+ u64 latest_generation;
+
+- struct block_device *latest_bdev;
++ /*
++ * The mount device or a device with highest generation after removal
++ * or replace.
++ */
++ struct btrfs_device *latest_dev;
+
+ /* all of the devices in the FS, protected by a mutex
+ * so we can safely walk it to write out the supers without
+diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c
+index 8a4514283a4b8..c5c5b97c2a852 100644
+--- a/fs/btrfs/xattr.c
++++ b/fs/btrfs/xattr.c
+@@ -264,7 +264,8 @@ int btrfs_setxattr_trans(struct inode *inode, const char *name,
+ inode_inc_iversion(inode);
+ inode->i_ctime = current_time(inode);
+ ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
+- BUG_ON(ret);
++ if (ret)
++ btrfs_abort_transaction(trans, ret);
+ out:
+ if (start_trans)
+ btrfs_end_transaction(trans);
+@@ -418,7 +419,8 @@ static int btrfs_xattr_handler_set_prop(const struct xattr_handler *handler,
+ inode_inc_iversion(inode);
+ inode->i_ctime = current_time(inode);
+ ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
+- BUG_ON(ret);
++ if (ret)
++ btrfs_abort_transaction(trans, ret);
+ }
+
+ btrfs_end_transaction(trans);
+diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c
+index 47af1ab3bf120..596b2148807d4 100644
+--- a/fs/btrfs/zoned.c
++++ b/fs/btrfs/zoned.c
+@@ -4,6 +4,7 @@
+ #include <linux/slab.h>
+ #include <linux/blkdev.h>
+ #include <linux/sched/mm.h>
++#include <linux/vmalloc.h>
+ #include "ctree.h"
+ #include "volumes.h"
+ #include "zoned.h"
+@@ -195,6 +196,8 @@ static int emulate_report_zones(struct btrfs_device *device, u64 pos,
+ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos,
+ struct blk_zone *zones, unsigned int *nr_zones)
+ {
++ struct btrfs_zoned_device_info *zinfo = device->zone_info;
++ u32 zno;
+ int ret;
+
+ if (!*nr_zones)
+@@ -206,6 +209,34 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos,
+ return 0;
+ }
+
++ /* Check cache */
++ if (zinfo->zone_cache) {
++ unsigned int i;
++
++ ASSERT(IS_ALIGNED(pos, zinfo->zone_size));
++ zno = pos >> zinfo->zone_size_shift;
++ /*
++ * We cannot report zones beyond the zone end. So, it is OK to
++ * cap *nr_zones to at the end.
++ */
++ *nr_zones = min_t(u32, *nr_zones, zinfo->nr_zones - zno);
++
++ for (i = 0; i < *nr_zones; i++) {
++ struct blk_zone *zone_info;
++
++ zone_info = &zinfo->zone_cache[zno + i];
++ if (!zone_info->len)
++ break;
++ }
++
++ if (i == *nr_zones) {
++ /* Cache hit on all the zones */
++ memcpy(zones, zinfo->zone_cache + zno,
++ sizeof(*zinfo->zone_cache) * *nr_zones);
++ return 0;
++ }
++ }
++
+ ret = blkdev_report_zones(device->bdev, pos >> SECTOR_SHIFT, *nr_zones,
+ copy_zone_info_cb, zones);
+ if (ret < 0) {
+@@ -219,6 +250,11 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos,
+ if (!ret)
+ return -EIO;
+
++ /* Populate cache */
++ if (zinfo->zone_cache)
++ memcpy(zinfo->zone_cache + zno, zones,
++ sizeof(*zinfo->zone_cache) * *nr_zones);
++
+ return 0;
+ }
+
+@@ -282,7 +318,7 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
+ if (!device->bdev)
+ continue;
+
+- ret = btrfs_get_dev_zone_info(device);
++ ret = btrfs_get_dev_zone_info(device, true);
+ if (ret)
+ break;
+ }
+@@ -291,7 +327,7 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
+ return ret;
+ }
+
+-int btrfs_get_dev_zone_info(struct btrfs_device *device)
++int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache)
+ {
+ struct btrfs_fs_info *fs_info = device->fs_info;
+ struct btrfs_zoned_device_info *zone_info = NULL;
+@@ -318,6 +354,8 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device)
+ if (!zone_info)
+ return -ENOMEM;
+
++ device->zone_info = zone_info;
++
+ if (!bdev_is_zoned(bdev)) {
+ if (!fs_info->zone_size) {
+ ret = calculate_emulated_zone_size(fs_info);
+@@ -369,6 +407,23 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device)
+ goto out;
+ }
+
++ /*
++ * Enable zone cache only for a zoned device. On a non-zoned device, we
++ * fill the zone info with emulated CONVENTIONAL zones, so no need to
++ * use the cache.
++ */
++ if (populate_cache && bdev_is_zoned(device->bdev)) {
++ zone_info->zone_cache = vzalloc(sizeof(struct blk_zone) *
++ zone_info->nr_zones);
++ if (!zone_info->zone_cache) {
++ btrfs_err_in_rcu(device->fs_info,
++ "zoned: failed to allocate zone cache for %s",
++ rcu_str_deref(device->name));
++ ret = -ENOMEM;
++ goto out;
++ }
++ }
++
+ /* Get zones type */
+ while (sector < nr_sectors) {
+ nr_zones = BTRFS_REPORT_NR_ZONES;
+@@ -444,8 +499,6 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device)
+
+ kfree(zones);
+
+- device->zone_info = zone_info;
+-
+ switch (bdev_zoned_model(bdev)) {
+ case BLK_ZONED_HM:
+ model = "host-managed zoned";
+@@ -478,10 +531,7 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device)
+ out:
+ kfree(zones);
+ out_free_zone_info:
+- bitmap_free(zone_info->empty_zones);
+- bitmap_free(zone_info->seq_zones);
+- kfree(zone_info);
+- device->zone_info = NULL;
++ btrfs_destroy_dev_zone_info(device);
+
+ return ret;
+ }
+@@ -495,6 +545,7 @@ void btrfs_destroy_dev_zone_info(struct btrfs_device *device)
+
+ bitmap_free(zone_info->seq_zones);
+ bitmap_free(zone_info->empty_zones);
++ vfree(zone_info->zone_cache);
+ kfree(zone_info);
+ device->zone_info = NULL;
+ }
+@@ -1304,6 +1355,17 @@ bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start)
+ if (!is_data_inode(&inode->vfs_inode))
+ return false;
+
++ /*
++ * Using REQ_OP_ZONE_APPNED for relocation can break assumptions on the
++ * extent layout the relocation code has.
++ * Furthermore we have set aside own block-group from which only the
++ * relocation "process" can allocate and make sure only one process at a
++ * time can add pages to an extent that gets relocated, so it's safe to
++ * use regular REQ_OP_WRITE for this special case.
++ */
++ if (btrfs_is_data_reloc_root(inode->root))
++ return false;
++
+ cache = btrfs_lookup_block_group(fs_info, start);
+ ASSERT(cache);
+ if (!cache)
+@@ -1530,3 +1592,31 @@ struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info,
+
+ return device;
+ }
++
++void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg)
++{
++ struct btrfs_fs_info *fs_info = bg->fs_info;
++
++ spin_lock(&fs_info->relocation_bg_lock);
++ if (fs_info->data_reloc_bg == bg->start)
++ fs_info->data_reloc_bg = 0;
++ spin_unlock(&fs_info->relocation_bg_lock);
++}
++
++void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info)
++{
++ struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
++ struct btrfs_device *device;
++
++ if (!btrfs_is_zoned(fs_info))
++ return;
++
++ mutex_lock(&fs_devices->device_list_mutex);
++ list_for_each_entry(device, &fs_devices->devices, dev_list) {
++ if (device->zone_info) {
++ vfree(device->zone_info->zone_cache);
++ device->zone_info->zone_cache = NULL;
++ }
++ }
++ mutex_unlock(&fs_devices->device_list_mutex);
++}
+diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h
+index 4b299705bb12b..813aa3cddc11f 100644
+--- a/fs/btrfs/zoned.h
++++ b/fs/btrfs/zoned.h
+@@ -25,6 +25,7 @@ struct btrfs_zoned_device_info {
+ u32 nr_zones;
+ unsigned long *seq_zones;
+ unsigned long *empty_zones;
++ struct blk_zone *zone_cache;
+ struct blk_zone sb_zones[2 * BTRFS_SUPER_MIRROR_MAX];
+ };
+
+@@ -32,7 +33,7 @@ struct btrfs_zoned_device_info {
+ int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos,
+ struct blk_zone *zone);
+ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info);
+-int btrfs_get_dev_zone_info(struct btrfs_device *device);
++int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache);
+ void btrfs_destroy_dev_zone_info(struct btrfs_device *device);
+ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info);
+ int btrfs_check_mountopts_zoned(struct btrfs_fs_info *info);
+@@ -66,6 +67,8 @@ int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical,
+ u64 physical_start, u64 physical_pos);
+ struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info,
+ u64 logical, u64 length);
++void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg);
++void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info);
+ #else /* CONFIG_BLK_DEV_ZONED */
+ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos,
+ struct blk_zone *zone)
+@@ -78,7 +81,8 @@ static inline int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_i
+ return 0;
+ }
+
+-static inline int btrfs_get_dev_zone_info(struct btrfs_device *device)
++static inline int btrfs_get_dev_zone_info(struct btrfs_device *device,
++ bool populate_cache)
+ {
+ return 0;
+ }
+@@ -199,6 +203,9 @@ static inline struct btrfs_device *btrfs_zoned_get_device(
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
++static inline void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) { }
++
++static inline void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) { }
+ #endif
+
+ static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos)
+diff --git a/fs/buffer.c b/fs/buffer.c
+index c615387aedcae..f6d2835794918 100644
+--- a/fs/buffer.c
++++ b/fs/buffer.c
+@@ -1235,16 +1235,18 @@ static void bh_lru_install(struct buffer_head *bh)
+ int i;
+
+ check_irqs_on();
++ bh_lru_lock();
++
+ /*
+ * the refcount of buffer_head in bh_lru prevents dropping the
+ * attached page(i.e., try_to_free_buffers) so it could cause
+ * failing page migration.
+ * Skip putting upcoming bh into bh_lru until migration is done.
+ */
+- if (lru_cache_disabled())
++ if (lru_cache_disabled()) {
++ bh_lru_unlock();
+ return;
+-
+- bh_lru_lock();
++ }
+
+ b = this_cpu_ptr(&bh_lrus);
+ for (i = 0; i < BH_LRU_SIZE; i++) {
+diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
+index 8f537f1d9d1d3..d1faa9d2f1e8a 100644
+--- a/fs/ceph/caps.c
++++ b/fs/ceph/caps.c
+@@ -2217,6 +2217,7 @@ static int unsafe_request_wait(struct inode *inode)
+ struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
+ struct ceph_inode_info *ci = ceph_inode(inode);
+ struct ceph_mds_request *req1 = NULL, *req2 = NULL;
++ unsigned int max_sessions;
+ int ret, err = 0;
+
+ spin_lock(&ci->i_unsafe_lock);
+@@ -2234,37 +2235,47 @@ static int unsafe_request_wait(struct inode *inode)
+ }
+ spin_unlock(&ci->i_unsafe_lock);
+
++ /*
++ * The mdsc->max_sessions is unlikely to be changed
++ * mostly, here we will retry it by reallocating the
++ * sessions array memory to get rid of the mdsc->mutex
++ * lock.
++ */
++retry:
++ max_sessions = mdsc->max_sessions;
++
+ /*
+ * Trigger to flush the journal logs in all the relevant MDSes
+ * manually, or in the worst case we must wait at most 5 seconds
+ * to wait the journal logs to be flushed by the MDSes periodically.
+ */
+- if (req1 || req2) {
++ if ((req1 || req2) && likely(max_sessions)) {
+ struct ceph_mds_session **sessions = NULL;
+ struct ceph_mds_session *s;
+ struct ceph_mds_request *req;
+- unsigned int max;
+ int i;
+
+- /*
+- * The mdsc->max_sessions is unlikely to be changed
+- * mostly, here we will retry it by reallocating the
+- * sessions arrary memory to get rid of the mdsc->mutex
+- * lock.
+- */
+-retry:
+- max = mdsc->max_sessions;
+- sessions = krealloc(sessions, max * sizeof(s), __GFP_ZERO);
+- if (!sessions)
+- return -ENOMEM;
++ sessions = kzalloc(max_sessions * sizeof(s), GFP_KERNEL);
++ if (!sessions) {
++ err = -ENOMEM;
++ goto out;
++ }
+
+ spin_lock(&ci->i_unsafe_lock);
+ if (req1) {
+ list_for_each_entry(req, &ci->i_unsafe_dirops,
+ r_unsafe_dir_item) {
+ s = req->r_session;
+- if (unlikely(s->s_mds >= max)) {
++ if (!s)
++ continue;
++ if (unlikely(s->s_mds >= max_sessions)) {
+ spin_unlock(&ci->i_unsafe_lock);
++ for (i = 0; i < max_sessions; i++) {
++ s = sessions[i];
++ if (s)
++ ceph_put_mds_session(s);
++ }
++ kfree(sessions);
+ goto retry;
+ }
+ if (!sessions[s->s_mds]) {
+@@ -2277,8 +2288,16 @@ retry:
+ list_for_each_entry(req, &ci->i_unsafe_iops,
+ r_unsafe_target_item) {
+ s = req->r_session;
+- if (unlikely(s->s_mds >= max)) {
++ if (!s)
++ continue;
++ if (unlikely(s->s_mds >= max_sessions)) {
+ spin_unlock(&ci->i_unsafe_lock);
++ for (i = 0; i < max_sessions; i++) {
++ s = sessions[i];
++ if (s)
++ ceph_put_mds_session(s);
++ }
++ kfree(sessions);
+ goto retry;
+ }
+ if (!sessions[s->s_mds]) {
+@@ -2299,7 +2318,7 @@ retry:
+ spin_unlock(&ci->i_ceph_lock);
+
+ /* send flush mdlog request to MDSes */
+- for (i = 0; i < max; i++) {
++ for (i = 0; i < max_sessions; i++) {
+ s = sessions[i];
+ if (s) {
+ send_flush_mdlog(s);
+@@ -2316,15 +2335,19 @@ retry:
+ ceph_timeout_jiffies(req1->r_timeout));
+ if (ret)
+ err = -EIO;
+- ceph_mdsc_put_request(req1);
+ }
+ if (req2) {
+ ret = !wait_for_completion_timeout(&req2->r_safe_completion,
+ ceph_timeout_jiffies(req2->r_timeout));
+ if (ret)
+ err = -EIO;
+- ceph_mdsc_put_request(req2);
+ }
++
++out:
++ if (req1)
++ ceph_mdsc_put_request(req1);
++ if (req2)
++ ceph_mdsc_put_request(req2);
+ return err;
+ }
+
+@@ -4349,7 +4372,7 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
+ {
+ struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->vfs_inode.i_sb);
+ int bits = (fmode << 1) | 1;
+- bool is_opened = false;
++ bool already_opened = false;
+ int i;
+
+ if (count == 1)
+@@ -4357,19 +4380,19 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
+
+ spin_lock(&ci->i_ceph_lock);
+ for (i = 0; i < CEPH_FILE_MODE_BITS; i++) {
+- if (bits & (1 << i))
+- ci->i_nr_by_mode[i] += count;
+-
+ /*
+- * If any of the mode ref is larger than 1,
++ * If any of the mode ref is larger than 0,
+ * that means it has been already opened by
+ * others. Just skip checking the PIN ref.
+ */
+- if (i && ci->i_nr_by_mode[i] > 1)
+- is_opened = true;
++ if (i && ci->i_nr_by_mode[i])
++ already_opened = true;
++
++ if (bits & (1 << i))
++ ci->i_nr_by_mode[i] += count;
+ }
+
+- if (!is_opened)
++ if (!already_opened)
+ percpu_counter_inc(&mdsc->metric.opened_inodes);
+ spin_unlock(&ci->i_ceph_lock);
+ }
+diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c
+index 133dbd9338e73..d91fa53e12b33 100644
+--- a/fs/ceph/dir.c
++++ b/fs/ceph/dir.c
+@@ -478,8 +478,11 @@ more:
+ 2 : (fpos_off(rde->offset) + 1);
+ err = note_last_dentry(dfi, rde->name, rde->name_len,
+ next_offset);
+- if (err)
++ if (err) {
++ ceph_mdsc_put_request(dfi->last_readdir);
++ dfi->last_readdir = NULL;
+ return err;
++ }
+ } else if (req->r_reply_info.dir_end) {
+ dfi->next_offset = 2;
+ /* keep last name */
+@@ -520,6 +523,12 @@ more:
+ if (!dir_emit(ctx, rde->name, rde->name_len,
+ ceph_present_ino(inode->i_sb, le64_to_cpu(rde->inode.in->ino)),
+ le32_to_cpu(rde->inode.in->mode) >> 12)) {
++ /*
++ * NOTE: Here no need to put the 'dfi->last_readdir',
++ * because when dir_emit stops us it's most likely
++ * doesn't have enough memory, etc. So for next readdir
++ * it will continue.
++ */
+ dout("filldir stopping us...\n");
+ return 0;
+ }
+diff --git a/fs/ceph/file.c b/fs/ceph/file.c
+index e61018d9764ee..e34d52df4a13c 100644
+--- a/fs/ceph/file.c
++++ b/fs/ceph/file.c
+@@ -577,6 +577,7 @@ static int ceph_finish_async_create(struct inode *dir, struct dentry *dentry,
+ struct ceph_inode_info *ci = ceph_inode(dir);
+ struct inode *inode;
+ struct timespec64 now;
++ struct ceph_string *pool_ns;
+ struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
+ struct ceph_vino vino = { .ino = req->r_deleg_ino,
+ .snap = CEPH_NOSNAP };
+@@ -591,9 +592,15 @@ static int ceph_finish_async_create(struct inode *dir, struct dentry *dentry,
+ iinfo.change_attr = 1;
+ ceph_encode_timespec64(&iinfo.btime, &now);
+
+- iinfo.xattr_len = ARRAY_SIZE(xattr_buf);
+- iinfo.xattr_data = xattr_buf;
+- memset(iinfo.xattr_data, 0, iinfo.xattr_len);
++ if (req->r_pagelist) {
++ iinfo.xattr_len = req->r_pagelist->length;
++ iinfo.xattr_data = req->r_pagelist->mapped_tail;
++ } else {
++ /* fake it */
++ iinfo.xattr_len = ARRAY_SIZE(xattr_buf);
++ iinfo.xattr_data = xattr_buf;
++ memset(iinfo.xattr_data, 0, iinfo.xattr_len);
++ }
+
+ in.ino = cpu_to_le64(vino.ino);
+ in.snapid = cpu_to_le64(CEPH_NOSNAP);
+@@ -603,17 +610,35 @@ static int ceph_finish_async_create(struct inode *dir, struct dentry *dentry,
+ in.cap.realm = cpu_to_le64(ci->i_snap_realm->ino);
+ in.cap.flags = CEPH_CAP_FLAG_AUTH;
+ in.ctime = in.mtime = in.atime = iinfo.btime;
+- in.mode = cpu_to_le32((u32)mode);
+ in.truncate_seq = cpu_to_le32(1);
+ in.truncate_size = cpu_to_le64(-1ULL);
+ in.xattr_version = cpu_to_le64(1);
+ in.uid = cpu_to_le32(from_kuid(&init_user_ns, current_fsuid()));
+- in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_mode & S_ISGID ?
+- dir->i_gid : current_fsgid()));
++ if (dir->i_mode & S_ISGID) {
++ in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_gid));
++
++ /* Directories always inherit the setgid bit. */
++ if (S_ISDIR(mode))
++ mode |= S_ISGID;
++ else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
++ !in_group_p(dir->i_gid) &&
++ !capable_wrt_inode_uidgid(&init_user_ns, dir, CAP_FSETID))
++ mode &= ~S_ISGID;
++ } else {
++ in.gid = cpu_to_le32(from_kgid(&init_user_ns, current_fsgid()));
++ }
++ in.mode = cpu_to_le32((u32)mode);
++
+ in.nlink = cpu_to_le32(1);
+ in.max_size = cpu_to_le64(lo->stripe_unit);
+
+ ceph_file_layout_to_legacy(lo, &in.layout);
++ /* lo is private, so pool_ns can't change */
++ pool_ns = rcu_dereference_raw(lo->pool_ns);
++ if (pool_ns) {
++ iinfo.pool_ns_len = pool_ns->len;
++ iinfo.pool_ns_data = pool_ns->str;
++ }
+
+ down_read(&mdsc->snap_rwsem);
+ ret = ceph_fill_inode(inode, NULL, &iinfo, NULL, req->r_session,
+@@ -687,6 +712,10 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry,
+ err = ceph_security_init_secctx(dentry, mode, &as_ctx);
+ if (err < 0)
+ goto out_ctx;
++ /* Async create can't handle more than a page of xattrs */
++ if (as_ctx.pagelist &&
++ !list_is_singular(&as_ctx.pagelist->head))
++ try_async = false;
+ } else if (!d_in_lookup(dentry)) {
+ /* If it's not being looked up, it's negative */
+ return -ENOENT;
+@@ -732,8 +761,10 @@ retry:
+ restore_deleg_ino(dir, req->r_deleg_ino);
+ ceph_mdsc_put_request(req);
+ try_async = false;
++ ceph_put_string(rcu_dereference_raw(lo.pool_ns));
+ goto retry;
+ }
++ ceph_put_string(rcu_dereference_raw(lo.pool_ns));
+ goto out_req;
+ }
+ }
+diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c
+index 1c7574105478f..42e449d3f18b8 100644
+--- a/fs/ceph/inode.c
++++ b/fs/ceph/inode.c
+@@ -87,13 +87,13 @@ struct inode *ceph_get_snapdir(struct inode *parent)
+ if (!S_ISDIR(parent->i_mode)) {
+ pr_warn_once("bad snapdir parent type (mode=0%o)\n",
+ parent->i_mode);
+- return ERR_PTR(-ENOTDIR);
++ goto err;
+ }
+
+ if (!(inode->i_state & I_NEW) && !S_ISDIR(inode->i_mode)) {
+ pr_warn_once("bad snapdir inode type (mode=0%o)\n",
+ inode->i_mode);
+- return ERR_PTR(-ENOTDIR);
++ goto err;
+ }
+
+ inode->i_mode = parent->i_mode;
+@@ -113,6 +113,12 @@ struct inode *ceph_get_snapdir(struct inode *parent)
+ }
+
+ return inode;
++err:
++ if ((inode->i_state & I_NEW))
++ discard_new_inode(inode);
++ else
++ iput(inode);
++ return ERR_PTR(-ENOTDIR);
+ }
+
+ const struct inode_operations ceph_file_iops = {
+diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
+index d64413adc0fd2..e9409c460acd0 100644
+--- a/fs/ceph/mds_client.c
++++ b/fs/ceph/mds_client.c
+@@ -3772,7 +3772,7 @@ static int reconnect_caps_cb(struct inode *inode, struct ceph_cap *cap,
+ struct ceph_pagelist *pagelist = recon_state->pagelist;
+ struct dentry *dentry;
+ char *path;
+- int pathlen, err;
++ int pathlen = 0, err;
+ u64 pathbase;
+ u64 snap_follows;
+
+@@ -3792,7 +3792,6 @@ static int reconnect_caps_cb(struct inode *inode, struct ceph_cap *cap,
+ }
+ } else {
+ path = NULL;
+- pathlen = 0;
+ pathbase = 0;
+ }
+
+diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c
+index 61d67cbcb3671..30387733765d5 100644
+--- a/fs/ceph/mdsmap.c
++++ b/fs/ceph/mdsmap.c
+@@ -263,10 +263,6 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end, bool msgr2)
+ goto nomem;
+ for (j = 0; j < num_export_targets; j++) {
+ target = ceph_decode_32(&pexport_targets);
+- if (target >= m->possible_max_rank) {
+- err = -EIO;
+- goto corrupt;
+- }
+ info->export_targets[j] = target;
+ }
+ } else {
+diff --git a/fs/ceph/super.c b/fs/ceph/super.c
+index fd8742bae8471..202ddde3d62ad 100644
+--- a/fs/ceph/super.c
++++ b/fs/ceph/super.c
+@@ -52,8 +52,7 @@ static int ceph_statfs(struct dentry *dentry, struct kstatfs *buf)
+ struct ceph_fs_client *fsc = ceph_inode_to_client(d_inode(dentry));
+ struct ceph_mon_client *monc = &fsc->client->monc;
+ struct ceph_statfs st;
+- u64 fsid;
+- int err;
++ int i, err;
+ u64 data_pool;
+
+ if (fsc->mdsc->mdsmap->m_num_data_pg_pools == 1) {
+@@ -99,12 +98,14 @@ static int ceph_statfs(struct dentry *dentry, struct kstatfs *buf)
+ buf->f_namelen = NAME_MAX;
+
+ /* Must convert the fsid, for consistent values across arches */
++ buf->f_fsid.val[0] = 0;
+ mutex_lock(&monc->mutex);
+- fsid = le64_to_cpu(*(__le64 *)(&monc->monmap->fsid)) ^
+- le64_to_cpu(*((__le64 *)&monc->monmap->fsid + 1));
++ for (i = 0 ; i < sizeof(monc->monmap->fsid) / sizeof(__le32) ; ++i)
++ buf->f_fsid.val[0] ^= le32_to_cpu(((__le32 *)&monc->monmap->fsid)[i]);
+ mutex_unlock(&monc->mutex);
+
+- buf->f_fsid = u64_to_fsid(fsid);
++ /* fold the fs_cluster_id into the upper bits */
++ buf->f_fsid.val[1] = monc->fs_cluster_id;
+
+ return 0;
+ }
+diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c
+index de2c12bcfa4bc..248a8f973cf9c 100644
+--- a/fs/cifs/cifs_debug.c
++++ b/fs/cifs/cifs_debug.c
+@@ -358,6 +358,8 @@ skip_rdma:
+ seq_printf(m, " signed");
+ if (server->posix_ext_supported)
+ seq_printf(m, " posix");
++ if (server->nosharesock)
++ seq_printf(m, " nosharesock");
+
+ if (server->rdma)
+ seq_printf(m, "\nRDMA ");
+@@ -412,12 +414,14 @@ skip_rdma:
+ from_kuid(&init_user_ns, ses->linux_uid),
+ from_kuid(&init_user_ns, ses->cred_uid));
+
++ spin_lock(&ses->chan_lock);
+ if (ses->chan_count > 1) {
+ seq_printf(m, "\n\n\tExtra Channels: %zu ",
+ ses->chan_count-1);
+ for (j = 1; j < ses->chan_count; j++)
+ cifs_dump_channel(m, j, &ses->chans[j]);
+ }
++ spin_unlock(&ses->chan_lock);
+
+ seq_puts(m, "\n\n\tShares: ");
+ j = 0;
+diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c
+index ee3aab3dd4ac6..bf861fef2f0c3 100644
+--- a/fs/cifs/cifsacl.c
++++ b/fs/cifs/cifsacl.c
+@@ -949,6 +949,9 @@ static void populate_new_aces(char *nacl_base,
+ pnntace = (struct cifs_ace *) (nacl_base + nsize);
+ nsize += setup_special_mode_ACE(pnntace, nmode);
+ num_aces++;
++ pnntace = (struct cifs_ace *) (nacl_base + nsize);
++ nsize += setup_authusers_ACE(pnntace);
++ num_aces++;
+ goto set_size;
+ }
+
+@@ -1297,7 +1300,7 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd,
+
+ if (uid_valid(uid)) { /* chown */
+ uid_t id;
+- nowner_sid_ptr = kmalloc(sizeof(struct cifs_sid),
++ nowner_sid_ptr = kzalloc(sizeof(struct cifs_sid),
+ GFP_KERNEL);
+ if (!nowner_sid_ptr) {
+ rc = -ENOMEM;
+@@ -1326,7 +1329,7 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd,
+ }
+ if (gid_valid(gid)) { /* chgrp */
+ gid_t id;
+- ngroup_sid_ptr = kmalloc(sizeof(struct cifs_sid),
++ ngroup_sid_ptr = kzalloc(sizeof(struct cifs_sid),
+ GFP_KERNEL);
+ if (!ngroup_sid_ptr) {
+ rc = -ENOMEM;
+@@ -1613,7 +1616,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode,
+ nsecdesclen = secdesclen;
+ if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */
+ if (mode_from_sid)
+- nsecdesclen += sizeof(struct cifs_ace);
++ nsecdesclen += 2 * sizeof(struct cifs_ace);
+ else /* cifsacl */
+ nsecdesclen += 5 * sizeof(struct cifs_ace);
+ } else { /* chown */
+diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c
+index 9fa930dfd78d6..8f8d281e31510 100644
+--- a/fs/cifs/cifsfs.c
++++ b/fs/cifs/cifsfs.c
+@@ -210,6 +210,9 @@ cifs_read_super(struct super_block *sb)
+ if (rc)
+ goto out_no_root;
+ /* tune readahead according to rsize if readahead size not set on mount */
++ if (cifs_sb->ctx->rsize == 0)
++ cifs_sb->ctx->rsize =
++ tcon->ses->server->ops->negotiate_rsize(tcon, cifs_sb->ctx);
+ if (cifs_sb->ctx->rasize)
+ sb->s_bdi->ra_pages = cifs_sb->ctx->rasize / PAGE_SIZE;
+ else
+@@ -254,26 +257,33 @@ static void cifs_kill_sb(struct super_block *sb)
+ struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
+ struct cifs_tcon *tcon;
+ struct cached_fid *cfid;
++ struct rb_root *root = &cifs_sb->tlink_tree;
++ struct rb_node *node;
++ struct tcon_link *tlink;
+
+ /*
+ * We ned to release all dentries for the cached directories
+ * before we kill the sb.
+ */
+ if (cifs_sb->root) {
++ for (node = rb_first(root); node; node = rb_next(node)) {
++ tlink = rb_entry(node, struct tcon_link, tl_rbnode);
++ tcon = tlink_tcon(tlink);
++ if (IS_ERR(tcon))
++ continue;
++ cfid = &tcon->crfid;
++ mutex_lock(&cfid->fid_mutex);
++ if (cfid->dentry) {
++ dput(cfid->dentry);
++ cfid->dentry = NULL;
++ }
++ mutex_unlock(&cfid->fid_mutex);
++ }
++
++ /* finally release root dentry */
+ dput(cifs_sb->root);
+ cifs_sb->root = NULL;
+ }
+- tcon = cifs_sb_master_tcon(cifs_sb);
+- if (tcon) {
+- cfid = &tcon->crfid;
+- mutex_lock(&cfid->fid_mutex);
+- if (cfid->dentry) {
+-
+- dput(cfid->dentry);
+- cfid->dentry = NULL;
+- }
+- mutex_unlock(&cfid->fid_mutex);
+- }
+
+ kill_anon_super(sb);
+ cifs_umount(cifs_sb);
+@@ -909,6 +919,7 @@ cifs_smb3_do_mount(struct file_system_type *fs_type,
+
+ out_super:
+ deactivate_locked_super(sb);
++ return root;
+ out:
+ if (cifs_sb) {
+ kfree(cifs_sb->prepath);
+@@ -925,7 +936,7 @@ cifs_loose_read_iter(struct kiocb *iocb, struct iov_iter *iter)
+ ssize_t rc;
+ struct inode *inode = file_inode(iocb->ki_filp);
+
+- if (iocb->ki_filp->f_flags & O_DIRECT)
++ if (iocb->ki_flags & IOCB_DIRECT)
+ return cifs_user_readv(iocb, iter);
+
+ rc = cifs_revalidate_mapping(inode);
+diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h
+index e916470468ea9..b33835b2943e9 100644
+--- a/fs/cifs/cifsglob.h
++++ b/fs/cifs/cifsglob.h
+@@ -74,7 +74,8 @@
+ #define SMB_ECHO_INTERVAL_MAX 600
+ #define SMB_ECHO_INTERVAL_DEFAULT 60
+
+-/* dns resolution interval in seconds */
++/* dns resolution intervals in seconds */
++#define SMB_DNS_RESOLVE_INTERVAL_MIN 120
+ #define SMB_DNS_RESOLVE_INTERVAL_DEFAULT 600
+
+ /* maximum number of PDUs in one compound */
+@@ -591,6 +592,7 @@ struct TCP_Server_Info {
+ struct list_head pending_mid_q;
+ bool noblocksnd; /* use blocking sendmsg */
+ bool noautotune; /* do not autotune send buf sizes */
++ bool nosharesock;
+ bool tcp_nodelay;
+ unsigned int credits; /* send no more requests at once */
+ unsigned int max_credits; /* can override large 32000 default at mnt */
+@@ -932,16 +934,21 @@ struct cifs_ses {
+ * iface_lock should be taken when accessing any of these fields
+ */
+ spinlock_t iface_lock;
++ /* ========= begin: protected by iface_lock ======== */
+ struct cifs_server_iface *iface_list;
+ size_t iface_count;
+ unsigned long iface_last_update; /* jiffies */
++ /* ========= end: protected by iface_lock ======== */
+
++ spinlock_t chan_lock;
++ /* ========= begin: protected by chan_lock ======== */
+ #define CIFS_MAX_CHANNELS 16
+ struct cifs_chan chans[CIFS_MAX_CHANNELS];
+ struct cifs_chan *binding_chan;
+ size_t chan_count;
+ size_t chan_max;
+ atomic_t chan_seq; /* round robin state */
++ /* ========= end: protected by chan_lock ======== */
+ };
+
+ /*
+diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c
+index c3b94c1e45913..70da1d27be3db 100644
+--- a/fs/cifs/connect.c
++++ b/fs/cifs/connect.c
+@@ -115,7 +115,7 @@ static int reconn_set_ipaddr_from_hostname(struct TCP_Server_Info *server)
+ * To make sure we don't use the cached entry, retry 1s
+ * after expiry.
+ */
+- ttl = (expiry - now + 1);
++ ttl = max_t(unsigned long, expiry - now, SMB_DNS_RESOLVE_INTERVAL_MIN) + 1;
+ }
+ rc = !rc ? -1 : 0;
+
+@@ -794,7 +794,6 @@ static void clean_demultiplex_info(struct TCP_Server_Info *server)
+ */
+ }
+
+- kfree(server->hostname);
+ kfree(server);
+
+ length = atomic_dec_return(&tcpSesAllocCount);
+@@ -1221,6 +1220,10 @@ static int match_server(struct TCP_Server_Info *server, struct smb3_fs_context *
+ if (ctx->nosharesock)
+ return 0;
+
++ /* this server does not share socket */
++ if (server->nosharesock)
++ return 0;
++
+ /* If multidialect negotiation see if existing sessions match one */
+ if (strcmp(ctx->vals->version_string, SMB3ANY_VERSION_STRING) == 0) {
+ if (server->vals->protocol_id < SMB30_PROT_ID)
+@@ -1235,6 +1238,9 @@ static int match_server(struct TCP_Server_Info *server, struct smb3_fs_context *
+ if (!net_eq(cifs_net_ns(server), current->nsproxy->net_ns))
+ return 0;
+
++ if (strcasecmp(server->hostname, ctx->server_hostname))
++ return 0;
++
+ if (!match_address(server, addr,
+ (struct sockaddr *)&ctx->srcaddr))
+ return 0;
+@@ -1336,6 +1342,7 @@ cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect)
+ kfree(server->session_key.response);
+ server->session_key.response = NULL;
+ server->session_key.len = 0;
++ kfree(server->hostname);
+
+ task = xchg(&server->tsk, NULL);
+ if (task)
+@@ -1361,14 +1368,18 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx)
+ goto out_err;
+ }
+
++ tcp_ses->hostname = kstrdup(ctx->server_hostname, GFP_KERNEL);
++ if (!tcp_ses->hostname) {
++ rc = -ENOMEM;
++ goto out_err;
++ }
++
++ if (ctx->nosharesock)
++ tcp_ses->nosharesock = true;
++
+ tcp_ses->ops = ctx->ops;
+ tcp_ses->vals = ctx->vals;
+ cifs_set_net_ns(tcp_ses, get_net(current->nsproxy->net_ns));
+- tcp_ses->hostname = extract_hostname(ctx->UNC);
+- if (IS_ERR(tcp_ses->hostname)) {
+- rc = PTR_ERR(tcp_ses->hostname);
+- goto out_err_crypto_release;
+- }
+
+ tcp_ses->conn_id = atomic_inc_return(&tcpSesNextId);
+ tcp_ses->noblockcnt = ctx->rootfs;
+@@ -1497,8 +1508,7 @@ out_err_crypto_release:
+
+ out_err:
+ if (tcp_ses) {
+- if (!IS_ERR(tcp_ses->hostname))
+- kfree(tcp_ses->hostname);
++ kfree(tcp_ses->hostname);
+ if (tcp_ses->ssocket)
+ sock_release(tcp_ses->ssocket);
+ kfree(tcp_ses);
+@@ -1516,8 +1526,12 @@ static int match_session(struct cifs_ses *ses, struct smb3_fs_context *ctx)
+ * If an existing session is limited to less channels than
+ * requested, it should not be reused
+ */
+- if (ses->chan_max < ctx->max_channels)
++ spin_lock(&ses->chan_lock);
++ if (ses->chan_max < ctx->max_channels) {
++ spin_unlock(&ses->chan_lock);
+ return 0;
++ }
++ spin_unlock(&ses->chan_lock);
+
+ switch (ses->sectype) {
+ case Kerberos:
+@@ -1652,6 +1666,7 @@ cifs_find_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx)
+ void cifs_put_smb_ses(struct cifs_ses *ses)
+ {
+ unsigned int rc, xid;
++ unsigned int chan_count;
+ struct TCP_Server_Info *server = ses->server;
+ cifs_dbg(FYI, "%s: ses_count=%d\n", __func__, ses->ses_count);
+
+@@ -1693,12 +1708,24 @@ void cifs_put_smb_ses(struct cifs_ses *ses)
+ list_del_init(&ses->smb_ses_list);
+ spin_unlock(&cifs_tcp_ses_lock);
+
++ spin_lock(&ses->chan_lock);
++ chan_count = ses->chan_count;
++ spin_unlock(&ses->chan_lock);
++
+ /* close any extra channels */
+- if (ses->chan_count > 1) {
++ if (chan_count > 1) {
+ int i;
+
+- for (i = 1; i < ses->chan_count; i++)
++ for (i = 1; i < chan_count; i++) {
++ /*
++ * note: for now, we're okay accessing ses->chans
++ * without chan_lock. But when chans can go away, we'll
++ * need to introduce ref counting to make sure that chan
++ * is not freed from under us.
++ */
+ cifs_put_tcp_session(ses->chans[i].server, 0);
++ ses->chans[i].server = NULL;
++ }
+ }
+
+ sesInfoFree(ses);
+@@ -1949,9 +1976,11 @@ cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx)
+ mutex_lock(&ses->session_mutex);
+
+ /* add server as first channel */
++ spin_lock(&ses->chan_lock);
+ ses->chans[0].server = server;
+ ses->chan_count = 1;
+ ses->chan_max = ctx->multichannel ? ctx->max_channels:1;
++ spin_unlock(&ses->chan_lock);
+
+ rc = cifs_negotiate_protocol(xid, ses);
+ if (!rc)
+diff --git a/fs/cifs/file.c b/fs/cifs/file.c
+index 13f3182cf7969..b23f6b489bb9d 100644
+--- a/fs/cifs/file.c
++++ b/fs/cifs/file.c
+@@ -2692,12 +2692,23 @@ int cifs_strict_fsync(struct file *file, loff_t start, loff_t end,
+ tcon = tlink_tcon(smbfile->tlink);
+ if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOSSYNC)) {
+ server = tcon->ses->server;
+- if (server->ops->flush)
+- rc = server->ops->flush(xid, tcon, &smbfile->fid);
+- else
++ if (server->ops->flush == NULL) {
+ rc = -ENOSYS;
++ goto strict_fsync_exit;
++ }
++
++ if ((OPEN_FMODE(smbfile->f_flags) & FMODE_WRITE) == 0) {
++ smbfile = find_writable_file(CIFS_I(inode), FIND_WR_ANY);
++ if (smbfile) {
++ rc = server->ops->flush(xid, tcon, &smbfile->fid);
++ cifsFileInfo_put(smbfile);
++ } else
++ cifs_dbg(FYI, "ignore fsync for file not open for write\n");
++ } else
++ rc = server->ops->flush(xid, tcon, &smbfile->fid);
+ }
+
++strict_fsync_exit:
+ free_xid(xid);
+ return rc;
+ }
+@@ -2709,6 +2720,7 @@ int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
+ struct cifs_tcon *tcon;
+ struct TCP_Server_Info *server;
+ struct cifsFileInfo *smbfile = file->private_data;
++ struct inode *inode = file_inode(file);
+ struct cifs_sb_info *cifs_sb = CIFS_FILE_SB(file);
+
+ rc = file_write_and_wait_range(file, start, end);
+@@ -2725,12 +2737,23 @@ int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
+ tcon = tlink_tcon(smbfile->tlink);
+ if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOSSYNC)) {
+ server = tcon->ses->server;
+- if (server->ops->flush)
+- rc = server->ops->flush(xid, tcon, &smbfile->fid);
+- else
++ if (server->ops->flush == NULL) {
+ rc = -ENOSYS;
++ goto fsync_exit;
++ }
++
++ if ((OPEN_FMODE(smbfile->f_flags) & FMODE_WRITE) == 0) {
++ smbfile = find_writable_file(CIFS_I(inode), FIND_WR_ANY);
++ if (smbfile) {
++ rc = server->ops->flush(xid, tcon, &smbfile->fid);
++ cifsFileInfo_put(smbfile);
++ } else
++ cifs_dbg(FYI, "ignore fsync for file not open for write\n");
++ } else
++ rc = server->ops->flush(xid, tcon, &smbfile->fid);
+ }
+
++fsync_exit:
+ free_xid(xid);
+ return rc;
+ }
+@@ -3711,6 +3734,11 @@ cifs_send_async_read(loff_t offset, size_t len, struct cifsFileInfo *open_file,
+ break;
+ }
+
++ if (cifs_sb->ctx->rsize == 0)
++ cifs_sb->ctx->rsize =
++ server->ops->negotiate_rsize(tlink_tcon(open_file->tlink),
++ cifs_sb->ctx);
++
+ rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,
+ &rsize, credits);
+ if (rc)
+@@ -4489,6 +4517,11 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
+ break;
+ }
+
++ if (cifs_sb->ctx->rsize == 0)
++ cifs_sb->ctx->rsize =
++ server->ops->negotiate_rsize(tlink_tcon(open_file->tlink),
++ cifs_sb->ctx);
++
+ rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,
+ &rsize, credits);
+ if (rc)
+diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c
+index 3109def8e1998..3b8ed36b37113 100644
+--- a/fs/cifs/fs_context.c
++++ b/fs/cifs/fs_context.c
+@@ -146,7 +146,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = {
+ fsparam_u32("echo_interval", Opt_echo_interval),
+ fsparam_u32("max_credits", Opt_max_credits),
+ fsparam_u32("handletimeout", Opt_handletimeout),
+- fsparam_u32("snapshot", Opt_snapshot),
++ fsparam_u64("snapshot", Opt_snapshot),
+ fsparam_u32("max_channels", Opt_max_channels),
+
+ /* Mount options which take string value */
+@@ -307,6 +307,7 @@ smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx
+ new_ctx->nodename = NULL;
+ new_ctx->username = NULL;
+ new_ctx->password = NULL;
++ new_ctx->server_hostname = NULL;
+ new_ctx->domainname = NULL;
+ new_ctx->UNC = NULL;
+ new_ctx->source = NULL;
+@@ -318,6 +319,7 @@ smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx
+ DUP_CTX_STR(mount_options);
+ DUP_CTX_STR(username);
+ DUP_CTX_STR(password);
++ DUP_CTX_STR(server_hostname);
+ DUP_CTX_STR(UNC);
+ DUP_CTX_STR(source);
+ DUP_CTX_STR(domainname);
+@@ -429,6 +431,42 @@ out:
+ return rc;
+ }
+
++/*
++ * Remove duplicate path delimiters. Windows is supposed to do that
++ * but there are some bugs that prevent rename from working if there are
++ * multiple delimiters.
++ *
++ * Returns a sanitized duplicate of @path. The caller is responsible for
++ * cleaning up the original.
++ */
++#define IS_DELIM(c) ((c) == '/' || (c) == '\\')
++static char *sanitize_path(char *path)
++{
++ char *cursor1 = path, *cursor2 = path;
++
++ /* skip all prepended delimiters */
++ while (IS_DELIM(*cursor1))
++ cursor1++;
++
++ /* copy the first letter */
++ *cursor2 = *cursor1;
++
++ /* copy the remainder... */
++ while (*(cursor1++)) {
++ /* ... skipping all duplicated delimiters */
++ if (IS_DELIM(*cursor1) && IS_DELIM(*cursor2))
++ continue;
++ *(++cursor2) = *cursor1;
++ }
++
++ /* if the last character is a delimiter, skip it */
++ if (IS_DELIM(*(cursor2 - 1)))
++ cursor2--;
++
++ *(cursor2) = '\0';
++ return kstrdup(path, GFP_KERNEL);
++}
++
+ /*
+ * Parse a devname into substrings and populate the ctx->UNC and ctx->prepath
+ * fields with the result. Returns 0 on success and an error otherwise
+@@ -456,6 +494,12 @@ smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx)
+ if (!pos)
+ return -EINVAL;
+
++ /* record the server hostname */
++ kfree(ctx->server_hostname);
++ ctx->server_hostname = kstrndup(devname + 2, pos - devname - 2, GFP_KERNEL);
++ if (!ctx->server_hostname)
++ return -ENOMEM;
++
+ /* skip past delimiter */
+ ++pos;
+
+@@ -482,7 +526,7 @@ smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx)
+ if (!*pos)
+ return 0;
+
+- ctx->prepath = kstrdup(pos, GFP_KERNEL);
++ ctx->prepath = sanitize_path(pos);
+ if (!ctx->prepath)
+ return -ENOMEM;
+
+@@ -1018,7 +1062,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc,
+ ctx->echo_interval = result.uint_32;
+ break;
+ case Opt_snapshot:
+- ctx->snapshot_time = result.uint_32;
++ ctx->snapshot_time = result.uint_64;
+ break;
+ case Opt_max_credits:
+ if (result.uint_32 < 20 || result.uint_32 > 60000) {
+@@ -1496,6 +1540,8 @@ smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx)
+ ctx->username = NULL;
+ kfree_sensitive(ctx->password);
+ ctx->password = NULL;
++ kfree(ctx->server_hostname);
++ ctx->server_hostname = NULL;
+ kfree(ctx->UNC);
+ ctx->UNC = NULL;
+ kfree(ctx->source);
+diff --git a/fs/cifs/fs_context.h b/fs/cifs/fs_context.h
+index a42ba71d7a81f..29601a4eb4116 100644
+--- a/fs/cifs/fs_context.h
++++ b/fs/cifs/fs_context.h
+@@ -166,6 +166,7 @@ struct smb3_fs_context {
+ char *password;
+ char *domainname;
+ char *source;
++ char *server_hostname;
+ char *UNC;
+ char *nodename;
+ char *iocharset; /* local code page for mapping to and from Unicode */
+diff --git a/fs/cifs/link.c b/fs/cifs/link.c
+index 852e54ee82c28..bbdf3281559c8 100644
+--- a/fs/cifs/link.c
++++ b/fs/cifs/link.c
+@@ -85,6 +85,9 @@ parse_mf_symlink(const u8 *buf, unsigned int buf_len, unsigned int *_link_len,
+ if (rc != 1)
+ return -EINVAL;
+
++ if (link_len > CIFS_MF_SYMLINK_LINK_MAXLEN)
++ return -EINVAL;
++
+ rc = symlink_hash(link_len, link_str, md5_hash);
+ if (rc) {
+ cifs_dbg(FYI, "%s: MD5 hash failure: %d\n", __func__, rc);
+diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c
+index bb1185fff8cc4..0a0d0724c4294 100644
+--- a/fs/cifs/misc.c
++++ b/fs/cifs/misc.c
+@@ -75,6 +75,7 @@ sesInfoAlloc(void)
+ INIT_LIST_HEAD(&ret_buf->tcon_list);
+ mutex_init(&ret_buf->session_mutex);
+ spin_lock_init(&ret_buf->iface_lock);
++ spin_lock_init(&ret_buf->chan_lock);
+ }
+ return ret_buf;
+ }
+diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c
+index 23e02db7923f6..5500ea7837845 100644
+--- a/fs/cifs/sess.c
++++ b/fs/cifs/sess.c
+@@ -54,32 +54,43 @@ bool is_ses_using_iface(struct cifs_ses *ses, struct cifs_server_iface *iface)
+ {
+ int i;
+
++ spin_lock(&ses->chan_lock);
+ for (i = 0; i < ses->chan_count; i++) {
+- if (is_server_using_iface(ses->chans[i].server, iface))
++ if (is_server_using_iface(ses->chans[i].server, iface)) {
++ spin_unlock(&ses->chan_lock);
+ return true;
++ }
+ }
++ spin_unlock(&ses->chan_lock);
+ return false;
+ }
+
+ /* returns number of channels added */
+ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses)
+ {
+- int old_chan_count = ses->chan_count;
+- int left = ses->chan_max - ses->chan_count;
++ int old_chan_count, new_chan_count;
++ int left;
+ int i = 0;
+ int rc = 0;
+ int tries = 0;
+ struct cifs_server_iface *ifaces = NULL;
+ size_t iface_count;
+
++ spin_lock(&ses->chan_lock);
++
++ new_chan_count = old_chan_count = ses->chan_count;
++ left = ses->chan_max - ses->chan_count;
++
+ if (left <= 0) {
+ cifs_dbg(FYI,
+ "ses already at max_channels (%zu), nothing to open\n",
+ ses->chan_max);
++ spin_unlock(&ses->chan_lock);
+ return 0;
+ }
+
+ if (ses->server->dialect < SMB30_PROT_ID) {
++ spin_unlock(&ses->chan_lock);
+ cifs_dbg(VFS, "multichannel is not supported on this protocol version, use 3.0 or above\n");
+ return 0;
+ }
+@@ -87,8 +98,10 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses)
+ if (!(ses->server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) {
+ cifs_dbg(VFS, "server %s does not support multichannel\n", ses->server->hostname);
+ ses->chan_max = 1;
++ spin_unlock(&ses->chan_lock);
+ return 0;
+ }
++ spin_unlock(&ses->chan_lock);
+
+ /*
+ * Make a copy of the iface list at the time and use that
+@@ -142,10 +155,11 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses)
+ cifs_dbg(FYI, "successfully opened new channel on iface#%d\n",
+ i);
+ left--;
++ new_chan_count++;
+ }
+
+ kfree(ifaces);
+- return ses->chan_count - old_chan_count;
++ return new_chan_count - old_chan_count;
+ }
+
+ /*
+@@ -157,10 +171,14 @@ cifs_ses_find_chan(struct cifs_ses *ses, struct TCP_Server_Info *server)
+ {
+ int i;
+
++ spin_lock(&ses->chan_lock);
+ for (i = 0; i < ses->chan_count; i++) {
+- if (ses->chans[i].server == server)
++ if (ses->chans[i].server == server) {
++ spin_unlock(&ses->chan_lock);
+ return &ses->chans[i];
++ }
+ }
++ spin_unlock(&ses->chan_lock);
+ return NULL;
+ }
+
+@@ -168,6 +186,7 @@ static int
+ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses,
+ struct cifs_server_iface *iface)
+ {
++ struct TCP_Server_Info *chan_server;
+ struct cifs_chan *chan;
+ struct smb3_fs_context ctx = {NULL};
+ static const char unc_fmt[] = "\\%s\\foo";
+@@ -240,15 +259,20 @@ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses,
+ SMB2_CLIENT_GUID_SIZE);
+ ctx.use_client_guid = true;
+
+- mutex_lock(&ses->session_mutex);
++ chan_server = cifs_get_tcp_session(&ctx);
+
++ mutex_lock(&ses->session_mutex);
++ spin_lock(&ses->chan_lock);
+ chan = ses->binding_chan = &ses->chans[ses->chan_count];
+- chan->server = cifs_get_tcp_session(&ctx);
++ chan->server = chan_server;
+ if (IS_ERR(chan->server)) {
+ rc = PTR_ERR(chan->server);
+ chan->server = NULL;
++ spin_unlock(&ses->chan_lock);
+ goto out;
+ }
++ spin_unlock(&ses->chan_lock);
++
+ spin_lock(&cifs_tcp_ses_lock);
+ chan->server->is_channel = true;
+ spin_unlock(&cifs_tcp_ses_lock);
+@@ -283,8 +307,11 @@ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses,
+ * ses to the new server.
+ */
+
++ spin_lock(&ses->chan_lock);
+ ses->chan_count++;
+ atomic_set(&ses->chan_seq, 0);
++ spin_unlock(&ses->chan_lock);
++
+ out:
+ ses->binding = false;
+ ses->binding_chan = NULL;
+diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c
+index bda606dc72b1f..0c1af2dd9069e 100644
+--- a/fs/cifs/smb2ops.c
++++ b/fs/cifs/smb2ops.c
+@@ -1631,6 +1631,7 @@ smb2_ioctl_query_info(const unsigned int xid,
+ unsigned int size[2];
+ void *data[2];
+ int create_options = is_dir ? CREATE_NOT_FILE : CREATE_NOT_DIR;
++ void (*free_req1_func)(struct smb_rqst *r);
+
+ vars = kzalloc(sizeof(*vars), GFP_ATOMIC);
+ if (vars == NULL)
+@@ -1640,27 +1641,29 @@ smb2_ioctl_query_info(const unsigned int xid,
+
+ resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER;
+
+- if (copy_from_user(&qi, arg, sizeof(struct smb_query_info)))
+- goto e_fault;
+-
++ if (copy_from_user(&qi, arg, sizeof(struct smb_query_info))) {
++ rc = -EFAULT;
++ goto free_vars;
++ }
+ if (qi.output_buffer_length > 1024) {
+- kfree(vars);
+- return -EINVAL;
++ rc = -EINVAL;
++ goto free_vars;
+ }
+
+ if (!ses || !server) {
+- kfree(vars);
+- return -EIO;
++ rc = -EIO;
++ goto free_vars;
+ }
+
+ if (smb3_encryption_required(tcon))
+ flags |= CIFS_TRANSFORM_REQ;
+
+- buffer = memdup_user(arg + sizeof(struct smb_query_info),
+- qi.output_buffer_length);
+- if (IS_ERR(buffer)) {
+- kfree(vars);
+- return PTR_ERR(buffer);
++ if (qi.output_buffer_length) {
++ buffer = memdup_user(arg + sizeof(struct smb_query_info), qi.output_buffer_length);
++ if (IS_ERR(buffer)) {
++ rc = PTR_ERR(buffer);
++ goto free_vars;
++ }
+ }
+
+ /* Open */
+@@ -1698,45 +1701,45 @@ smb2_ioctl_query_info(const unsigned int xid,
+ rc = SMB2_open_init(tcon, server,
+ &rqst[0], &oplock, &oparms, path);
+ if (rc)
+- goto iqinf_exit;
++ goto free_output_buffer;
+ smb2_set_next_command(tcon, &rqst[0]);
+
+ /* Query */
+ if (qi.flags & PASSTHRU_FSCTL) {
+ /* Can eventually relax perm check since server enforces too */
+- if (!capable(CAP_SYS_ADMIN))
++ if (!capable(CAP_SYS_ADMIN)) {
+ rc = -EPERM;
+- else {
+- rqst[1].rq_iov = &vars->io_iov[0];
+- rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE;
+-
+- rc = SMB2_ioctl_init(tcon, server,
+- &rqst[1],
+- COMPOUND_FID, COMPOUND_FID,
+- qi.info_type, true, buffer,
+- qi.output_buffer_length,
+- CIFSMaxBufSize -
+- MAX_SMB2_CREATE_RESPONSE_SIZE -
+- MAX_SMB2_CLOSE_RESPONSE_SIZE);
++ goto free_open_req;
+ }
++ rqst[1].rq_iov = &vars->io_iov[0];
++ rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE;
++
++ rc = SMB2_ioctl_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID,
++ qi.info_type, true, buffer, qi.output_buffer_length,
++ CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE -
++ MAX_SMB2_CLOSE_RESPONSE_SIZE);
++ free_req1_func = SMB2_ioctl_free;
+ } else if (qi.flags == PASSTHRU_SET_INFO) {
+ /* Can eventually relax perm check since server enforces too */
+- if (!capable(CAP_SYS_ADMIN))
++ if (!capable(CAP_SYS_ADMIN)) {
+ rc = -EPERM;
+- else {
+- rqst[1].rq_iov = &vars->si_iov[0];
+- rqst[1].rq_nvec = 1;
+-
+- size[0] = 8;
+- data[0] = buffer;
+-
+- rc = SMB2_set_info_init(tcon, server,
+- &rqst[1],
+- COMPOUND_FID, COMPOUND_FID,
+- current->tgid,
+- FILE_END_OF_FILE_INFORMATION,
+- SMB2_O_INFO_FILE, 0, data, size);
++ goto free_open_req;
++ }
++ if (qi.output_buffer_length < 8) {
++ rc = -EINVAL;
++ goto free_open_req;
+ }
++ rqst[1].rq_iov = &vars->si_iov[0];
++ rqst[1].rq_nvec = 1;
++
++ /* MS-FSCC 2.4.13 FileEndOfFileInformation */
++ size[0] = 8;
++ data[0] = buffer;
++
++ rc = SMB2_set_info_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID,
++ current->tgid, FILE_END_OF_FILE_INFORMATION,
++ SMB2_O_INFO_FILE, 0, data, size);
++ free_req1_func = SMB2_set_info_free;
+ } else if (qi.flags == PASSTHRU_QUERY_INFO) {
+ rqst[1].rq_iov = &vars->qi_iov[0];
+ rqst[1].rq_nvec = 1;
+@@ -1747,6 +1750,7 @@ smb2_ioctl_query_info(const unsigned int xid,
+ qi.info_type, qi.additional_information,
+ qi.input_buffer_length,
+ qi.output_buffer_length, buffer);
++ free_req1_func = SMB2_query_info_free;
+ } else { /* unknown flags */
+ cifs_tcon_dbg(VFS, "Invalid passthru query flags: 0x%x\n",
+ qi.flags);
+@@ -1754,7 +1758,7 @@ smb2_ioctl_query_info(const unsigned int xid,
+ }
+
+ if (rc)
+- goto iqinf_exit;
++ goto free_open_req;
+ smb2_set_next_command(tcon, &rqst[1]);
+ smb2_set_related(&rqst[1]);
+
+@@ -1765,14 +1769,14 @@ smb2_ioctl_query_info(const unsigned int xid,
+ rc = SMB2_close_init(tcon, server,
+ &rqst[2], COMPOUND_FID, COMPOUND_FID, false);
+ if (rc)
+- goto iqinf_exit;
++ goto free_req_1;
+ smb2_set_related(&rqst[2]);
+
+ rc = compound_send_recv(xid, ses, server,
+ flags, 3, rqst,
+ resp_buftype, rsp_iov);
+ if (rc)
+- goto iqinf_exit;
++ goto out;
+
+ /* No need to bump num_remote_opens since handle immediately closed */
+ if (qi.flags & PASSTHRU_FSCTL) {
+@@ -1782,18 +1786,22 @@ smb2_ioctl_query_info(const unsigned int xid,
+ qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount);
+ if (qi.input_buffer_length > 0 &&
+ le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length
+- > rsp_iov[1].iov_len)
+- goto e_fault;
++ > rsp_iov[1].iov_len) {
++ rc = -EFAULT;
++ goto out;
++ }
+
+ if (copy_to_user(&pqi->input_buffer_length,
+ &qi.input_buffer_length,
+- sizeof(qi.input_buffer_length)))
+- goto e_fault;
++ sizeof(qi.input_buffer_length))) {
++ rc = -EFAULT;
++ goto out;
++ }
+
+ if (copy_to_user((void __user *)pqi + sizeof(struct smb_query_info),
+ (const void *)io_rsp + le32_to_cpu(io_rsp->OutputOffset),
+ qi.input_buffer_length))
+- goto e_fault;
++ rc = -EFAULT;
+ } else {
+ pqi = (struct smb_query_info __user *)arg;
+ qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base;
+@@ -1801,28 +1809,30 @@ smb2_ioctl_query_info(const unsigned int xid,
+ qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength);
+ if (copy_to_user(&pqi->input_buffer_length,
+ &qi.input_buffer_length,
+- sizeof(qi.input_buffer_length)))
+- goto e_fault;
++ sizeof(qi.input_buffer_length))) {
++ rc = -EFAULT;
++ goto out;
++ }
+
+ if (copy_to_user(pqi + 1, qi_rsp->Buffer,
+ qi.input_buffer_length))
+- goto e_fault;
++ rc = -EFAULT;
+ }
+
+- iqinf_exit:
+- cifs_small_buf_release(rqst[0].rq_iov[0].iov_base);
+- cifs_small_buf_release(rqst[1].rq_iov[0].iov_base);
+- cifs_small_buf_release(rqst[2].rq_iov[0].iov_base);
++out:
+ free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
+ free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
+ free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
+- kfree(vars);
++ SMB2_close_free(&rqst[2]);
++free_req_1:
++ free_req1_func(&rqst[1]);
++free_open_req:
++ SMB2_open_free(&rqst[0]);
++free_output_buffer:
+ kfree(buffer);
++free_vars:
++ kfree(vars);
+ return rc;
+-
+-e_fault:
+- rc = -EFAULT;
+- goto iqinf_exit;
+ }
+
+ static ssize_t
+@@ -1839,9 +1849,17 @@ smb2_copychunk_range(const unsigned int xid,
+ int chunks_copied = 0;
+ bool chunk_sizes_updated = false;
+ ssize_t bytes_written, total_bytes_written = 0;
++ struct inode *inode;
+
+ pcchunk = kmalloc(sizeof(struct copychunk_ioctl), GFP_KERNEL);
+
++ /*
++ * We need to flush all unwritten data before we can send the
++ * copychunk ioctl to the server.
++ */
++ inode = d_inode(trgtfile->dentry);
++ filemap_write_and_wait(inode->i_mapping);
++
+ if (pcchunk == NULL)
+ return -ENOMEM;
+
+diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c
+index b7379329b741c..61ea3d3f95b4a 100644
+--- a/fs/cifs/transport.c
++++ b/fs/cifs/transport.c
+@@ -1044,14 +1044,17 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses)
+ if (!ses)
+ return NULL;
+
++ spin_lock(&ses->chan_lock);
+ if (!ses->binding) {
+ /* round robin */
+ if (ses->chan_count > 1) {
+ index = (uint)atomic_inc_return(&ses->chan_seq);
+ index %= ses->chan_count;
+ }
++ spin_unlock(&ses->chan_lock);
+ return ses->chans[index].server;
+ } else {
++ spin_unlock(&ses->chan_lock);
+ return cifs_ses_server(ses);
+ }
+ }
+diff --git a/fs/cifs/xattr.c b/fs/cifs/xattr.c
+index 7d8b72d67c803..9d486fbbfbbde 100644
+--- a/fs/cifs/xattr.c
++++ b/fs/cifs/xattr.c
+@@ -175,11 +175,13 @@ static int cifs_xattr_set(const struct xattr_handler *handler,
+ switch (handler->flags) {
+ case XATTR_CIFS_NTSD_FULL:
+ aclflags = (CIFS_ACL_OWNER |
++ CIFS_ACL_GROUP |
+ CIFS_ACL_DACL |
+ CIFS_ACL_SACL);
+ break;
+ case XATTR_CIFS_NTSD:
+ aclflags = (CIFS_ACL_OWNER |
++ CIFS_ACL_GROUP |
+ CIFS_ACL_DACL);
+ break;
+ case XATTR_CIFS_ACL:
+diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c
+index 1466b5d01cbb9..d1f9d26322027 100644
+--- a/fs/configfs/dir.c
++++ b/fs/configfs/dir.c
+@@ -34,6 +34,14 @@
+ */
+ DEFINE_SPINLOCK(configfs_dirent_lock);
+
++/*
++ * All of link_obj/unlink_obj/link_group/unlink_group require that
++ * subsys->su_mutex is held.
++ * But parent configfs_subsystem is NULL when config_item is root.
++ * Use this mutex when config_item is root.
++ */
++static DEFINE_MUTEX(configfs_subsystem_mutex);
++
+ static void configfs_d_iput(struct dentry * dentry,
+ struct inode * inode)
+ {
+@@ -1780,8 +1788,8 @@ void configfs_unregister_group(struct config_group *group)
+ configfs_detach_group(&group->cg_item);
+ d_inode(dentry)->i_flags |= S_DEAD;
+ dont_mount(dentry);
++ d_drop(dentry);
+ fsnotify_rmdir(d_inode(parent), dentry);
+- d_delete(dentry);
+ inode_unlock(d_inode(parent));
+
+ dput(dentry);
+@@ -1859,7 +1867,9 @@ int configfs_register_subsystem(struct configfs_subsystem *subsys)
+ group->cg_item.ci_name = group->cg_item.ci_namebuf;
+
+ sd = root->d_fsdata;
++ mutex_lock(&configfs_subsystem_mutex);
+ link_group(to_config_group(sd->s_element), group);
++ mutex_unlock(&configfs_subsystem_mutex);
+
+ inode_lock_nested(d_inode(root), I_MUTEX_PARENT);
+
+@@ -1884,7 +1894,9 @@ int configfs_register_subsystem(struct configfs_subsystem *subsys)
+ inode_unlock(d_inode(root));
+
+ if (err) {
++ mutex_lock(&configfs_subsystem_mutex);
+ unlink_group(group);
++ mutex_unlock(&configfs_subsystem_mutex);
+ configfs_release_fs();
+ }
+ put_fragment(frag);
+@@ -1922,16 +1934,18 @@ void configfs_unregister_subsystem(struct configfs_subsystem *subsys)
+ configfs_detach_group(&group->cg_item);
+ d_inode(dentry)->i_flags |= S_DEAD;
+ dont_mount(dentry);
+- fsnotify_rmdir(d_inode(root), dentry);
+ inode_unlock(d_inode(dentry));
+
+- d_delete(dentry);
++ d_drop(dentry);
++ fsnotify_rmdir(d_inode(root), dentry);
+
+ inode_unlock(d_inode(root));
+
+ dput(dentry);
+
++ mutex_lock(&configfs_subsystem_mutex);
+ unlink_group(group);
++ mutex_unlock(&configfs_subsystem_mutex);
+ configfs_release_fs();
+ }
+
+diff --git a/fs/coredump.c b/fs/coredump.c
+index 3224dee44d30e..26eb5a095832f 100644
+--- a/fs/coredump.c
++++ b/fs/coredump.c
+@@ -41,6 +41,7 @@
+ #include <linux/fs.h>
+ #include <linux/path.h>
+ #include <linux/timekeeping.h>
++#include <linux/elf.h>
+
+ #include <linux/uaccess.h>
+ #include <asm/mmu_context.h>
+@@ -52,6 +53,9 @@
+
+ #include <trace/events/sched.h>
+
++static bool dump_vma_snapshot(struct coredump_params *cprm);
++static void free_vma_snapshot(struct coredump_params *cprm);
++
+ int core_uses_pid;
+ unsigned int core_pipe_limit;
+ char core_pattern[CORENAME_MAX_SIZE] = "core";
+@@ -600,6 +604,7 @@ void do_coredump(const kernel_siginfo_t *siginfo)
+ * by any locks.
+ */
+ .mm_flags = mm->flags,
++ .vma_meta = NULL,
+ };
+
+ audit_core_dumps(siginfo->si_signo);
+@@ -814,6 +819,9 @@ void do_coredump(const kernel_siginfo_t *siginfo)
+ pr_info("Core dump to |%s disabled\n", cn.corename);
+ goto close_fail;
+ }
++ if (!dump_vma_snapshot(&cprm))
++ goto close_fail;
++
+ file_start_write(cprm.file);
+ core_dumped = binfmt->core_dump(&cprm);
+ /*
+@@ -827,6 +835,7 @@ void do_coredump(const kernel_siginfo_t *siginfo)
+ dump_emit(&cprm, "", 1);
+ }
+ file_end_write(cprm.file);
++ free_vma_snapshot(&cprm);
+ }
+ if (ispipe && core_pipe_limit)
+ wait_for_dump_helpers(cprm.file);
+@@ -992,6 +1001,8 @@ static bool always_dump_vma(struct vm_area_struct *vma)
+ return false;
+ }
+
++#define DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER 1
++
+ /*
+ * Decide how much of @vma's contents should be included in a core dump.
+ */
+@@ -1051,9 +1062,20 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma,
+ * dump the first page to aid in determining what was mapped here.
+ */
+ if (FILTER(ELF_HEADERS) &&
+- vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ) &&
+- (READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0)
+- return PAGE_SIZE;
++ vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) {
++ if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0)
++ return PAGE_SIZE;
++
++ /*
++ * ELF libraries aren't always executable.
++ * We'll want to check whether the mapping starts with the ELF
++ * magic, but not now - we're holding the mmap lock,
++ * so copy_from_user() doesn't work here.
++ * Use a placeholder instead, and fix it up later in
++ * dump_vma_snapshot().
++ */
++ return DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER;
++ }
+
+ #undef FILTER
+
+@@ -1090,18 +1112,29 @@ static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma,
+ return gate_vma;
+ }
+
++static void free_vma_snapshot(struct coredump_params *cprm)
++{
++ if (cprm->vma_meta) {
++ int i;
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct file *file = cprm->vma_meta[i].file;
++ if (file)
++ fput(file);
++ }
++ kvfree(cprm->vma_meta);
++ cprm->vma_meta = NULL;
++ }
++}
++
+ /*
+ * Under the mmap_lock, take a snapshot of relevant information about the task's
+ * VMAs.
+ */
+-int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count,
+- struct core_vma_metadata **vma_meta,
+- size_t *vma_data_size_ptr)
++static bool dump_vma_snapshot(struct coredump_params *cprm)
+ {
+ struct vm_area_struct *vma, *gate_vma;
+ struct mm_struct *mm = current->mm;
+ int i;
+- size_t vma_data_size = 0;
+
+ /*
+ * Once the stack expansion code is fixed to not change VMA bounds
+@@ -1109,36 +1142,51 @@ int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count,
+ * mmap_lock in read mode.
+ */
+ if (mmap_write_lock_killable(mm))
+- return -EINTR;
++ return false;
+
++ cprm->vma_data_size = 0;
+ gate_vma = get_gate_vma(mm);
+- *vma_count = mm->map_count + (gate_vma ? 1 : 0);
++ cprm->vma_count = mm->map_count + (gate_vma ? 1 : 0);
+
+- *vma_meta = kvmalloc_array(*vma_count, sizeof(**vma_meta), GFP_KERNEL);
+- if (!*vma_meta) {
++ cprm->vma_meta = kvmalloc_array(cprm->vma_count, sizeof(*cprm->vma_meta), GFP_KERNEL);
++ if (!cprm->vma_meta) {
+ mmap_write_unlock(mm);
+- return -ENOMEM;
++ return false;
+ }
+
+ for (i = 0, vma = first_vma(current, gate_vma); vma != NULL;
+ vma = next_vma(vma, gate_vma), i++) {
+- struct core_vma_metadata *m = (*vma_meta) + i;
++ struct core_vma_metadata *m = cprm->vma_meta + i;
+
+ m->start = vma->vm_start;
+ m->end = vma->vm_end;
+ m->flags = vma->vm_flags;
+ m->dump_size = vma_dump_size(vma, cprm->mm_flags);
++ m->pgoff = vma->vm_pgoff;
+
+- vma_data_size += m->dump_size;
++ m->file = vma->vm_file;
++ if (m->file)
++ get_file(m->file);
+ }
+
+ mmap_write_unlock(mm);
+
+- if (WARN_ON(i != *vma_count)) {
+- kvfree(*vma_meta);
+- return -EFAULT;
++ for (i = 0; i < cprm->vma_count; i++) {
++ struct core_vma_metadata *m = cprm->vma_meta + i;
++
++ if (m->dump_size == DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER) {
++ char elfmag[SELFMAG];
++
++ if (copy_from_user(elfmag, (void __user *)m->start, SELFMAG) ||
++ memcmp(elfmag, ELFMAG, SELFMAG) != 0) {
++ m->dump_size = 0;
++ } else {
++ m->dump_size = PAGE_SIZE;
++ }
++ }
++
++ cprm->vma_data_size += m->dump_size;
+ }
+
+- *vma_data_size_ptr = vma_data_size;
+- return 0;
++ return true;
+ }
+diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
+index 3fa965eb3336d..cb25ef0cdf1f3 100644
+--- a/fs/crypto/fscrypt_private.h
++++ b/fs/crypto/fscrypt_private.h
+@@ -549,8 +549,9 @@ int __init fscrypt_init_keyring(void);
+ struct fscrypt_mode {
+ const char *friendly_name;
+ const char *cipher_str;
+- int keysize;
+- int ivsize;
++ int keysize; /* key size in bytes */
++ int security_strength; /* security strength in bytes */
++ int ivsize; /* IV size in bytes */
+ int logged_impl_name;
+ enum blk_crypto_mode_num blk_crypto_mode;
+ };
+diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c
+index e0ec210555053..7607d18b35fc0 100644
+--- a/fs/crypto/hkdf.c
++++ b/fs/crypto/hkdf.c
+@@ -16,9 +16,14 @@
+
+ /*
+ * HKDF supports any unkeyed cryptographic hash algorithm, but fscrypt uses
+- * SHA-512 because it is reasonably secure and efficient; and since it produces
+- * a 64-byte digest, deriving an AES-256-XTS key preserves all 64 bytes of
+- * entropy from the master key and requires only one iteration of HKDF-Expand.
++ * SHA-512 because it is well-established, secure, and reasonably efficient.
++ *
++ * HKDF-SHA256 was also considered, as its 256-bit security strength would be
++ * sufficient here. A 512-bit security strength is "nice to have", though.
++ * Also, on 64-bit CPUs, SHA-512 is usually just as fast as SHA-256. In the
++ * common case of deriving an AES-256-XTS key (512 bits), that can result in
++ * HKDF-SHA512 being much faster than HKDF-SHA256, as the longer digest size of
++ * SHA-512 causes HKDF-Expand to only need to do one iteration rather than two.
+ */
+ #define HKDF_HMAC_ALG "hmac(sha512)"
+ #define HKDF_HASHLEN SHA512_DIGEST_SIZE
+diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
+index bca9c6658a7c5..89cd533a88bff 100644
+--- a/fs/crypto/keysetup.c
++++ b/fs/crypto/keysetup.c
+@@ -19,6 +19,7 @@ struct fscrypt_mode fscrypt_modes[] = {
+ .friendly_name = "AES-256-XTS",
+ .cipher_str = "xts(aes)",
+ .keysize = 64,
++ .security_strength = 32,
+ .ivsize = 16,
+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_256_XTS,
+ },
+@@ -26,12 +27,14 @@ struct fscrypt_mode fscrypt_modes[] = {
+ .friendly_name = "AES-256-CTS-CBC",
+ .cipher_str = "cts(cbc(aes))",
+ .keysize = 32,
++ .security_strength = 32,
+ .ivsize = 16,
+ },
+ [FSCRYPT_MODE_AES_128_CBC] = {
+ .friendly_name = "AES-128-CBC-ESSIV",
+ .cipher_str = "essiv(cbc(aes),sha256)",
+ .keysize = 16,
++ .security_strength = 16,
+ .ivsize = 16,
+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV,
+ },
+@@ -39,12 +42,14 @@ struct fscrypt_mode fscrypt_modes[] = {
+ .friendly_name = "AES-128-CTS-CBC",
+ .cipher_str = "cts(cbc(aes))",
+ .keysize = 16,
++ .security_strength = 16,
+ .ivsize = 16,
+ },
+ [FSCRYPT_MODE_ADIANTUM] = {
+ .friendly_name = "Adiantum",
+ .cipher_str = "adiantum(xchacha12,aes)",
+ .keysize = 32,
++ .security_strength = 32,
+ .ivsize = 32,
+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM,
+ },
+@@ -357,6 +362,45 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci,
+ return 0;
+ }
+
++/*
++ * Check whether the size of the given master key (@mk) is appropriate for the
++ * encryption settings which a particular file will use (@ci).
++ *
++ * If the file uses a v1 encryption policy, then the master key must be at least
++ * as long as the derived key, as this is a requirement of the v1 KDF.
++ *
++ * Otherwise, the KDF can accept any size key, so we enforce a slightly looser
++ * requirement: we require that the size of the master key be at least the
++ * maximum security strength of any algorithm whose key will be derived from it
++ * (but in practice we only need to consider @ci->ci_mode, since any other
++ * possible subkeys such as DIRHASH and INODE_HASH will never increase the
++ * required key size over @ci->ci_mode). This allows AES-256-XTS keys to be
++ * derived from a 256-bit master key, which is cryptographically sufficient,
++ * rather than requiring a 512-bit master key which is unnecessarily long. (We
++ * still allow 512-bit master keys if the user chooses to use them, though.)
++ */
++static bool fscrypt_valid_master_key_size(const struct fscrypt_master_key *mk,
++ const struct fscrypt_info *ci)
++{
++ unsigned int min_keysize;
++
++ if (ci->ci_policy.version == FSCRYPT_POLICY_V1)
++ min_keysize = ci->ci_mode->keysize;
++ else
++ min_keysize = ci->ci_mode->security_strength;
++
++ if (mk->mk_secret.size < min_keysize) {
++ fscrypt_warn(NULL,
++ "key with %s %*phN is too short (got %u bytes, need %u+ bytes)",
++ master_key_spec_type(&mk->mk_spec),
++ master_key_spec_len(&mk->mk_spec),
++ (u8 *)&mk->mk_spec.u,
++ mk->mk_secret.size, min_keysize);
++ return false;
++ }
++ return true;
++}
++
+ /*
+ * Find the master key, then set up the inode's actual encryption key.
+ *
+@@ -422,18 +466,7 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
+ goto out_release_key;
+ }
+
+- /*
+- * Require that the master key be at least as long as the derived key.
+- * Otherwise, the derived key cannot possibly contain as much entropy as
+- * that required by the encryption mode it will be used for. For v1
+- * policies it's also required for the KDF to work at all.
+- */
+- if (mk->mk_secret.size < ci->ci_mode->keysize) {
+- fscrypt_warn(NULL,
+- "key with %s %*phN is too short (got %u bytes, need %u+ bytes)",
+- master_key_spec_type(&mk_spec),
+- master_key_spec_len(&mk_spec), (u8 *)&mk_spec.u,
+- mk->mk_secret.size, ci->ci_mode->keysize);
++ if (!fscrypt_valid_master_key_size(mk, ci)) {
+ err = -ENOKEY;
+ goto out_release_key;
+ }
+diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c
+index 7d162b0efbf03..950c63fa4d0b2 100644
+--- a/fs/debugfs/file.c
++++ b/fs/debugfs/file.c
+@@ -147,7 +147,7 @@ static int debugfs_locked_down(struct inode *inode,
+ struct file *filp,
+ const struct file_operations *real_fops)
+ {
+- if ((inode->i_mode & 07777) == 0444 &&
++ if ((inode->i_mode & 07777 & ~0444) == 0 &&
+ !(filp->f_mode & FMODE_WRITE) &&
+ !real_fops->unlocked_ioctl &&
+ !real_fops->compat_ioctl &&
+diff --git a/fs/devpts/inode.c b/fs/devpts/inode.c
+index 42e5a766d33c7..4f25015aa5342 100644
+--- a/fs/devpts/inode.c
++++ b/fs/devpts/inode.c
+@@ -621,8 +621,8 @@ void devpts_pty_kill(struct dentry *dentry)
+
+ dentry->d_fsdata = NULL;
+ drop_nlink(dentry->d_inode);
+- fsnotify_unlink(d_inode(dentry->d_parent), dentry);
+ d_drop(dentry);
++ fsnotify_unlink(d_inode(dentry->d_parent), dentry);
+ dput(dentry); /* d_alloc_name() in devpts_pty_new() */
+ }
+
+diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c
+index c502c065d0075..28d1f35b11a4d 100644
+--- a/fs/dlm/lock.c
++++ b/fs/dlm/lock.c
+@@ -3973,6 +3973,14 @@ static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms)
+ int from = ms->m_header.h_nodeid;
+ int error = 0;
+
++ /* currently mixing of user/kernel locks are not supported */
++ if (ms->m_flags & DLM_IFL_USER && ~lkb->lkb_flags & DLM_IFL_USER) {
++ log_error(lkb->lkb_resource->res_ls,
++ "got user dlm message for a kernel lock");
++ error = -EINVAL;
++ goto out;
++ }
++
+ switch (ms->m_type) {
+ case DLM_MSG_CONVERT:
+ case DLM_MSG_UNLOCK:
+@@ -4001,6 +4009,7 @@ static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms)
+ error = -EINVAL;
+ }
+
++out:
+ if (error)
+ log_error(lkb->lkb_resource->res_ls,
+ "ignore invalid message %d from %d %x %x %x %d",
+diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
+index 8f715c620e1f8..7a8efce1c343e 100644
+--- a/fs/dlm/lowcomms.c
++++ b/fs/dlm/lowcomms.c
+@@ -592,8 +592,8 @@ int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark)
+ static void lowcomms_error_report(struct sock *sk)
+ {
+ struct connection *con;
+- struct sockaddr_storage saddr;
+ void (*orig_report)(struct sock *) = NULL;
++ struct inet_sock *inet;
+
+ read_lock_bh(&sk->sk_callback_lock);
+ con = sock2con(sk);
+@@ -601,33 +601,33 @@ static void lowcomms_error_report(struct sock *sk)
+ goto out;
+
+ orig_report = listen_sock.sk_error_report;
+- if (kernel_getpeername(sk->sk_socket, (struct sockaddr *)&saddr) < 0) {
+- printk_ratelimited(KERN_ERR "dlm: node %d: socket error "
+- "sending to node %d, port %d, "
+- "sk_err=%d/%d\n", dlm_our_nodeid(),
+- con->nodeid, dlm_config.ci_tcp_port,
+- sk->sk_err, sk->sk_err_soft);
+- } else if (saddr.ss_family == AF_INET) {
+- struct sockaddr_in *sin4 = (struct sockaddr_in *)&saddr;
+
++ inet = inet_sk(sk);
++ switch (sk->sk_family) {
++ case AF_INET:
+ printk_ratelimited(KERN_ERR "dlm: node %d: socket error "
+- "sending to node %d at %pI4, port %d, "
++ "sending to node %d at %pI4, dport %d, "
+ "sk_err=%d/%d\n", dlm_our_nodeid(),
+- con->nodeid, &sin4->sin_addr.s_addr,
+- dlm_config.ci_tcp_port, sk->sk_err,
++ con->nodeid, &inet->inet_daddr,
++ ntohs(inet->inet_dport), sk->sk_err,
+ sk->sk_err_soft);
+- } else {
+- struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&saddr;
+-
++ break;
++#if IS_ENABLED(CONFIG_IPV6)
++ case AF_INET6:
+ printk_ratelimited(KERN_ERR "dlm: node %d: socket error "
+- "sending to node %d at %u.%u.%u.%u, "
+- "port %d, sk_err=%d/%d\n", dlm_our_nodeid(),
+- con->nodeid, sin6->sin6_addr.s6_addr32[0],
+- sin6->sin6_addr.s6_addr32[1],
+- sin6->sin6_addr.s6_addr32[2],
+- sin6->sin6_addr.s6_addr32[3],
+- dlm_config.ci_tcp_port, sk->sk_err,
++ "sending to node %d at %pI6c, "
++ "dport %d, sk_err=%d/%d\n", dlm_our_nodeid(),
++ con->nodeid, &sk->sk_v6_daddr,
++ ntohs(inet->inet_dport), sk->sk_err,
+ sk->sk_err_soft);
++ break;
++#endif
++ default:
++ printk_ratelimited(KERN_ERR "dlm: node %d: socket error "
++ "invalid socket family %d set, "
++ "sk_err=%d/%d\n", dlm_our_nodeid(),
++ sk->sk_family, sk->sk_err, sk->sk_err_soft);
++ goto out;
+ }
+
+ /* below sendcon only handling */
+diff --git a/fs/erofs/data.c b/fs/erofs/data.c
+index 9db8297156527..16a41d0db55a3 100644
+--- a/fs/erofs/data.c
++++ b/fs/erofs/data.c
+@@ -287,7 +287,7 @@ static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+
+ if (!err)
+ return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
+- NULL, 0);
++ NULL, 0, 0);
+ if (err < 0)
+ return err;
+ }
+diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
+index a5bc4b1b7813e..ad3f31380e6b2 100644
+--- a/fs/erofs/decompressor.c
++++ b/fs/erofs/decompressor.c
+@@ -233,7 +233,6 @@ static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)
+ erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
+ ret, rq->inputsize, inputmargin, rq->outputsize);
+
+- WARN_ON(1);
+ print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET,
+ 16, 1, src + inputmargin, rq->inputsize, true);
+ print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET,
+diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c
+index bd86067a63f7f..3ca703cd5b24a 100644
+--- a/fs/erofs/utils.c
++++ b/fs/erofs/utils.c
+@@ -141,7 +141,7 @@ static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
+ * however in order to avoid some race conditions, add a
+ * DBG_BUGON to observe this in advance.
+ */
+- DBG_BUGON(xa_erase(&sbi->managed_pslots, grp->index) != grp);
++ DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp);
+
+ /* last refcount should be connected with its managed pslot. */
+ erofs_workgroup_unfreeze(grp, 0);
+@@ -156,15 +156,19 @@ static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi,
+ unsigned int freed = 0;
+ unsigned long index;
+
++ xa_lock(&sbi->managed_pslots);
+ xa_for_each(&sbi->managed_pslots, index, grp) {
+ /* try to shrink each valid workgroup */
+ if (!erofs_try_to_release_workgroup(sbi, grp))
+ continue;
++ xa_unlock(&sbi->managed_pslots);
+
+ ++freed;
+ if (!--nr_shrink)
+- break;
++ return freed;
++ xa_lock(&sbi->managed_pslots);
+ }
++ xa_unlock(&sbi->managed_pslots);
+ return freed;
+ }
+
+diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
+index 11c7a1aaebade..eb51df4a9f770 100644
+--- a/fs/erofs/zdata.c
++++ b/fs/erofs/zdata.c
+@@ -373,8 +373,8 @@ static bool z_erofs_try_inplace_io(struct z_erofs_collector *clt,
+
+ /* callers must be with collection lock held */
+ static int z_erofs_attach_page(struct z_erofs_collector *clt,
+- struct page *page,
+- enum z_erofs_page_type type)
++ struct page *page, enum z_erofs_page_type type,
++ bool pvec_safereuse)
+ {
+ int ret;
+
+@@ -384,9 +384,9 @@ static int z_erofs_attach_page(struct z_erofs_collector *clt,
+ z_erofs_try_inplace_io(clt, page))
+ return 0;
+
+- ret = z_erofs_pagevec_enqueue(&clt->vector, page, type);
++ ret = z_erofs_pagevec_enqueue(&clt->vector, page, type,
++ pvec_safereuse);
+ clt->cl->vcnt += (unsigned int)ret;
+-
+ return ret ? 0 : -EAGAIN;
+ }
+
+@@ -729,7 +729,8 @@ hitted:
+ tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);
+
+ retry:
+- err = z_erofs_attach_page(clt, page, page_type);
++ err = z_erofs_attach_page(clt, page, page_type,
++ clt->mode >= COLLECT_PRIMARY_FOLLOWED);
+ /* should allocate an additional short-lived page for pagevec */
+ if (err == -EAGAIN) {
+ struct page *const newpage =
+@@ -737,7 +738,7 @@ retry:
+
+ set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE);
+ err = z_erofs_attach_page(clt, newpage,
+- Z_EROFS_PAGE_TYPE_EXCLUSIVE);
++ Z_EROFS_PAGE_TYPE_EXCLUSIVE, true);
+ if (!err)
+ goto retry;
+ }
+diff --git a/fs/erofs/zpvec.h b/fs/erofs/zpvec.h
+index dfd7fe0503bb1..b05464f4a8083 100644
+--- a/fs/erofs/zpvec.h
++++ b/fs/erofs/zpvec.h
+@@ -106,11 +106,18 @@ static inline void z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor *ctor,
+
+ static inline bool z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor *ctor,
+ struct page *page,
+- enum z_erofs_page_type type)
++ enum z_erofs_page_type type,
++ bool pvec_safereuse)
+ {
+- if (!ctor->next && type)
+- if (ctor->index + 1 == ctor->nr)
++ if (!ctor->next) {
++ /* some pages cannot be reused as pvec safely without I/O */
++ if (type == Z_EROFS_PAGE_TYPE_EXCLUSIVE && !pvec_safereuse)
++ type = Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED;
++
++ if (type != Z_EROFS_PAGE_TYPE_EXCLUSIVE &&
++ ctor->index + 1 == ctor->nr)
+ return false;
++ }
+
+ if (ctor->index >= ctor->nr)
+ z_erofs_pagevec_ctor_pagedown(ctor, false);
+diff --git a/fs/exec.c b/fs/exec.c
+index a098c133d8d74..29e865c59854a 100644
+--- a/fs/exec.c
++++ b/fs/exec.c
+@@ -494,8 +494,14 @@ static int bprm_stack_limits(struct linux_binprm *bprm)
+ * the stack. They aren't stored until much later when we can't
+ * signal to the parent that the child has run out of stack space.
+ * Instead, calculate it here so it's possible to fail gracefully.
++ *
++ * In the case of argc = 0, make sure there is space for adding a
++ * empty string (which will bump argc to 1), to ensure confused
++ * userspace programs don't start processing from argv[1], thinking
++ * argc can never be 0, to keep them from walking envp by accident.
++ * See do_execveat_common().
+ */
+- ptr_size = (bprm->argc + bprm->envc) * sizeof(void *);
++ ptr_size = (max(bprm->argc, 1) + bprm->envc) * sizeof(void *);
+ if (limit <= ptr_size)
+ return -E2BIG;
+ limit -= ptr_size;
+@@ -1852,7 +1858,7 @@ out:
+ * SIGSEGV.
+ */
+ if (bprm->point_of_no_return && !fatal_signal_pending(current))
+- force_sigsegv(SIGSEGV);
++ force_fatal_sig(SIGSEGV);
+
+ out_unmark:
+ current->fs->in_exec = 0;
+@@ -1895,6 +1901,9 @@ static int do_execveat_common(int fd, struct filename *filename,
+ }
+
+ retval = count(argv, MAX_ARG_STRINGS);
++ if (retval == 0)
++ pr_warn_once("process '%s' launched '%s' with NULL argv: empty string added\n",
++ current->comm, bprm->filename);
+ if (retval < 0)
+ goto out_free;
+ bprm->argc = retval;
+@@ -1921,6 +1930,19 @@ static int do_execveat_common(int fd, struct filename *filename,
+ if (retval < 0)
+ goto out_free;
+
++ /*
++ * When argv is empty, add an empty string ("") as argv[0] to
++ * ensure confused userspace programs that start processing
++ * from argv[1] won't end up walking envp. See also
++ * bprm_stack_limits().
++ */
++ if (bprm->argc == 0) {
++ retval = copy_string_kernel("", bprm);
++ if (retval < 0)
++ goto out_free;
++ bprm->argc = 1;
++ }
++
+ retval = bprm_execve(bprm, fd, filename, flags);
+ out_free:
+ free_bprm(bprm);
+@@ -1949,6 +1971,8 @@ int kernel_execve(const char *kernel_filename,
+ }
+
+ retval = count_strings_kernel(argv);
++ if (WARN_ON_ONCE(retval == 0))
++ retval = -EINVAL;
+ if (retval < 0)
+ goto out_free;
+ bprm->argc = retval;
+diff --git a/fs/exfat/file.c b/fs/exfat/file.c
+index 6af0191b648f1..d890fd34bb2d0 100644
+--- a/fs/exfat/file.c
++++ b/fs/exfat/file.c
+@@ -110,8 +110,7 @@ int __exfat_truncate(struct inode *inode, loff_t new_size)
+ exfat_set_volume_dirty(sb);
+
+ num_clusters_new = EXFAT_B_TO_CLU_ROUND_UP(i_size_read(inode), sbi);
+- num_clusters_phys =
+- EXFAT_B_TO_CLU_ROUND_UP(EXFAT_I(inode)->i_size_ondisk, sbi);
++ num_clusters_phys = EXFAT_B_TO_CLU_ROUND_UP(ei->i_size_ondisk, sbi);
+
+ exfat_chain_set(&clu, ei->start_clu, num_clusters_phys, ei->flags);
+
+@@ -228,12 +227,13 @@ void exfat_truncate(struct inode *inode, loff_t size)
+ {
+ struct super_block *sb = inode->i_sb;
+ struct exfat_sb_info *sbi = EXFAT_SB(sb);
++ struct exfat_inode_info *ei = EXFAT_I(inode);
+ unsigned int blocksize = i_blocksize(inode);
+ loff_t aligned_size;
+ int err;
+
+ mutex_lock(&sbi->s_lock);
+- if (EXFAT_I(inode)->start_clu == 0) {
++ if (ei->start_clu == 0) {
+ /*
+ * Empty start_clu != ~0 (not allocated)
+ */
+@@ -251,8 +251,8 @@ void exfat_truncate(struct inode *inode, loff_t size)
+ else
+ mark_inode_dirty(inode);
+
+- inode->i_blocks = ((i_size_read(inode) + (sbi->cluster_size - 1)) &
+- ~(sbi->cluster_size - 1)) >> inode->i_blkbits;
++ inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >>
++ inode->i_blkbits;
+ write_size:
+ aligned_size = i_size_read(inode);
+ if (aligned_size & (blocksize - 1)) {
+@@ -260,11 +260,11 @@ write_size:
+ aligned_size++;
+ }
+
+- if (EXFAT_I(inode)->i_size_ondisk > i_size_read(inode))
+- EXFAT_I(inode)->i_size_ondisk = aligned_size;
++ if (ei->i_size_ondisk > i_size_read(inode))
++ ei->i_size_ondisk = aligned_size;
+
+- if (EXFAT_I(inode)->i_size_aligned > i_size_read(inode))
+- EXFAT_I(inode)->i_size_aligned = aligned_size;
++ if (ei->i_size_aligned > i_size_read(inode))
++ ei->i_size_aligned = aligned_size;
+ mutex_unlock(&sbi->s_lock);
+ }
+
+diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c
+index ca37d43443612..72a0ccfb616c3 100644
+--- a/fs/exfat/inode.c
++++ b/fs/exfat/inode.c
+@@ -114,10 +114,9 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset,
+ unsigned int local_clu_offset = clu_offset;
+ unsigned int num_to_be_allocated = 0, num_clusters = 0;
+
+- if (EXFAT_I(inode)->i_size_ondisk > 0)
++ if (ei->i_size_ondisk > 0)
+ num_clusters =
+- EXFAT_B_TO_CLU_ROUND_UP(EXFAT_I(inode)->i_size_ondisk,
+- sbi);
++ EXFAT_B_TO_CLU_ROUND_UP(ei->i_size_ondisk, sbi);
+
+ if (clu_offset >= num_clusters)
+ num_to_be_allocated = clu_offset - num_clusters + 1;
+@@ -416,10 +415,10 @@ static int exfat_write_end(struct file *file, struct address_space *mapping,
+
+ err = generic_write_end(file, mapping, pos, len, copied, pagep, fsdata);
+
+- if (EXFAT_I(inode)->i_size_aligned < i_size_read(inode)) {
++ if (ei->i_size_aligned < i_size_read(inode)) {
+ exfat_fs_error(inode->i_sb,
+ "invalid size(size(%llu) > aligned(%llu)\n",
+- i_size_read(inode), EXFAT_I(inode)->i_size_aligned);
++ i_size_read(inode), ei->i_size_aligned);
+ return -EIO;
+ }
+
+@@ -603,8 +602,8 @@ static int exfat_fill_inode(struct inode *inode, struct exfat_dir_entry *info)
+
+ exfat_save_attr(inode, info->attr);
+
+- inode->i_blocks = ((i_size_read(inode) + (sbi->cluster_size - 1)) &
+- ~(sbi->cluster_size - 1)) >> inode->i_blkbits;
++ inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >>
++ inode->i_blkbits;
+ inode->i_mtime = info->mtime;
+ inode->i_ctime = info->mtime;
+ ei->i_crtime = info->crtime;
+diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
+index 24b41103d1cc0..9d8ada781250b 100644
+--- a/fs/exfat/namei.c
++++ b/fs/exfat/namei.c
+@@ -395,9 +395,9 @@ static int exfat_find_empty_entry(struct inode *inode,
+
+ /* directory inode should be updated in here */
+ i_size_write(inode, size);
+- EXFAT_I(inode)->i_size_ondisk += sbi->cluster_size;
+- EXFAT_I(inode)->i_size_aligned += sbi->cluster_size;
+- EXFAT_I(inode)->flags = p_dir->flags;
++ ei->i_size_ondisk += sbi->cluster_size;
++ ei->i_size_aligned += sbi->cluster_size;
++ ei->flags = p_dir->flags;
+ inode->i_blocks += 1 << sbi->sect_per_clus_bits;
+ }
+
+diff --git a/fs/exfat/super.c b/fs/exfat/super.c
+index 5539ffc20d164..4b5d02b1df585 100644
+--- a/fs/exfat/super.c
++++ b/fs/exfat/super.c
+@@ -364,11 +364,11 @@ static int exfat_read_root(struct inode *inode)
+ inode->i_op = &exfat_dir_inode_operations;
+ inode->i_fop = &exfat_dir_operations;
+
+- inode->i_blocks = ((i_size_read(inode) + (sbi->cluster_size - 1))
+- & ~(sbi->cluster_size - 1)) >> inode->i_blkbits;
+- EXFAT_I(inode)->i_pos = ((loff_t)sbi->root_dir << 32) | 0xffffffff;
+- EXFAT_I(inode)->i_size_aligned = i_size_read(inode);
+- EXFAT_I(inode)->i_size_ondisk = i_size_read(inode);
++ inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >>
++ inode->i_blkbits;
++ ei->i_pos = ((loff_t)sbi->root_dir << 32) | 0xffffffff;
++ ei->i_size_aligned = i_size_read(inode);
++ ei->i_size_ondisk = i_size_read(inode);
+
+ exfat_save_attr(inode, ATTR_SUBDIR);
+ inode->i_mtime = inode->i_atime = inode->i_ctime = ei->i_crtime =
+diff --git a/fs/ext2/super.c b/fs/ext2/super.c
+index d8d580b609baa..3d21279fe2cb5 100644
+--- a/fs/ext2/super.c
++++ b/fs/ext2/super.c
+@@ -753,8 +753,12 @@ static loff_t ext2_max_size(int bits)
+ res += 1LL << (bits-2);
+ res += 1LL << (2*(bits-2));
+ res += 1LL << (3*(bits-2));
++ /* Compute how many metadata blocks are needed */
++ meta_blocks = 1;
++ meta_blocks += 1 + ppb;
++ meta_blocks += 1 + ppb + ppb * ppb;
+ /* Does block tree limit file size? */
+- if (res < upper_limit)
++ if (res + meta_blocks <= upper_limit)
+ goto check_lfs;
+
+ res = upper_limit;
+diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
+index 3825195539d74..a0a9878578949 100644
+--- a/fs/ext4/ext4.h
++++ b/fs/ext4/ext4.h
+@@ -1724,9 +1724,9 @@ struct ext4_sb_info {
+ */
+ struct work_struct s_error_work;
+
+- /* Ext4 fast commit stuff */
++ /* Ext4 fast commit sub transaction ID */
+ atomic_t s_fc_subtid;
+- atomic_t s_fc_ineligible_updates;
++
+ /*
+ * After commit starts, the main queue gets locked, and the further
+ * updates get added in the staging queue.
+@@ -1746,7 +1746,7 @@ struct ext4_sb_info {
+ spinlock_t s_fc_lock;
+ struct buffer_head *s_fc_bh;
+ struct ext4_fc_stats s_fc_stats;
+- u64 s_fc_avg_commit_time;
++ tid_t s_fc_ineligible_tid;
+ #ifdef CONFIG_EXT4_DEBUG
+ int s_fc_debug_max_replay;
+ #endif
+@@ -1792,10 +1792,7 @@ static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino)
+ enum {
+ EXT4_MF_MNTDIR_SAMPLED,
+ EXT4_MF_FS_ABORTED, /* Fatal error detected */
+- EXT4_MF_FC_INELIGIBLE, /* Fast commit ineligible */
+- EXT4_MF_FC_COMMITTING /* File system underoing a fast
+- * commit.
+- */
++ EXT4_MF_FC_INELIGIBLE /* Fast commit ineligible */
+ };
+
+ static inline void ext4_set_mount_flag(struct super_block *sb, int bit)
+@@ -2270,6 +2267,10 @@ static inline int ext4_forced_shutdown(struct ext4_sb_info *sbi)
+ * Structure of a directory entry
+ */
+ #define EXT4_NAME_LEN 255
++/*
++ * Base length of the ext4 directory entry excluding the name length
++ */
++#define EXT4_BASE_DIR_LEN (sizeof(struct ext4_dir_entry_2) - EXT4_NAME_LEN)
+
+ struct ext4_dir_entry {
+ __le32 inode; /* Inode number */
+@@ -2924,9 +2925,7 @@ void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
+ struct dentry *dentry);
+ void ext4_fc_track_create(handle_t *handle, struct dentry *dentry);
+ void ext4_fc_track_inode(handle_t *handle, struct inode *inode);
+-void ext4_fc_mark_ineligible(struct super_block *sb, int reason);
+-void ext4_fc_start_ineligible(struct super_block *sb, int reason);
+-void ext4_fc_stop_ineligible(struct super_block *sb);
++void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle);
+ void ext4_fc_start_update(struct inode *inode);
+ void ext4_fc_stop_update(struct inode *inode);
+ void ext4_fc_del(struct inode *inode);
+@@ -2934,6 +2933,10 @@ bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t block);
+ void ext4_fc_replay_cleanup(struct super_block *sb);
+ int ext4_fc_commit(journal_t *journal, tid_t commit_tid);
+ int __init ext4_fc_init_dentry_cache(void);
++void ext4_fc_destroy_dentry_cache(void);
++int ext4_fc_record_regions(struct super_block *sb, int ino,
++ ext4_lblk_t lblk, ext4_fsblk_t pblk,
++ int len, int replay);
+
+ /* mballoc.c */
+ extern const struct seq_operations ext4_mb_seq_groups_ops;
+@@ -3028,7 +3031,7 @@ extern int ext4_inode_attach_jinode(struct inode *inode);
+ extern int ext4_can_truncate(struct inode *inode);
+ extern int ext4_truncate(struct inode *);
+ extern int ext4_break_layouts(struct inode *);
+-extern int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length);
++extern int ext4_punch_hole(struct file *file, loff_t offset, loff_t length);
+ extern void ext4_set_inode_flags(struct inode *, bool init);
+ extern int ext4_alloc_da_blocks(struct inode *inode);
+ extern void ext4_set_aops(struct inode *inode);
+diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c
+index 6def7339056db..3477a16d08aee 100644
+--- a/fs/ext4/ext4_jbd2.c
++++ b/fs/ext4/ext4_jbd2.c
+@@ -162,6 +162,8 @@ int __ext4_journal_ensure_credits(handle_t *handle, int check_cred,
+ {
+ if (!ext4_handle_valid(handle))
+ return 0;
++ if (is_handle_aborted(handle))
++ return -EROFS;
+ if (jbd2_handle_buffer_credits(handle) >= check_cred &&
+ handle->h_revoke_credits >= revoke_cred)
+ return 0;
+diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
+index 0e02571f2f828..44d00951e6092 100644
+--- a/fs/ext4/extents.c
++++ b/fs/ext4/extents.c
+@@ -136,15 +136,25 @@ int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode,
+ static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
+ struct ext4_ext_path *path)
+ {
++ int err = 0;
++
+ if (path->p_bh) {
+ /* path points to block */
+ BUFFER_TRACE(path->p_bh, "get_write_access");
+- return ext4_journal_get_write_access(handle, inode->i_sb,
+- path->p_bh, EXT4_JTR_NONE);
++ err = ext4_journal_get_write_access(handle, inode->i_sb,
++ path->p_bh, EXT4_JTR_NONE);
++ /*
++ * The extent buffer's verified bit will be set again in
++ * __ext4_ext_dirty(). We could leave an inconsistent
++ * buffer if the extents updating procudure break off du
++ * to some error happens, force to check it again.
++ */
++ if (!err)
++ clear_buffer_verified(path->p_bh);
+ }
+ /* path points to leaf/index in inode body */
+ /* we use in-core data, no need to protect them */
+- return 0;
++ return err;
+ }
+
+ /*
+@@ -165,6 +175,9 @@ static int __ext4_ext_dirty(const char *where, unsigned int line,
+ /* path points to block */
+ err = __ext4_handle_dirty_metadata(where, line, handle,
+ inode, path->p_bh);
++ /* Extents updating done, re-set verified flag */
++ if (!err)
++ set_buffer_verified(path->p_bh);
+ } else {
+ /* path points to leaf/index in inode body */
+ err = ext4_mark_inode_dirty(handle, inode);
+@@ -354,9 +367,13 @@ static int ext4_valid_extent_idx(struct inode *inode,
+
+ static int ext4_valid_extent_entries(struct inode *inode,
+ struct ext4_extent_header *eh,
+- ext4_fsblk_t *pblk, int depth)
++ ext4_lblk_t lblk, ext4_fsblk_t *pblk,
++ int depth)
+ {
+ unsigned short entries;
++ ext4_lblk_t lblock = 0;
++ ext4_lblk_t prev = 0;
++
+ if (eh->eh_entries == 0)
+ return 1;
+
+@@ -365,31 +382,51 @@ static int ext4_valid_extent_entries(struct inode *inode,
+ if (depth == 0) {
+ /* leaf entries */
+ struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);
+- ext4_lblk_t lblock = 0;
+- ext4_lblk_t prev = 0;
+- int len = 0;
++
++ /*
++ * The logical block in the first entry should equal to
++ * the number in the index block.
++ */
++ if (depth != ext_depth(inode) &&
++ lblk != le32_to_cpu(ext->ee_block))
++ return 0;
+ while (entries) {
+ if (!ext4_valid_extent(inode, ext))
+ return 0;
+
+ /* Check for overlapping extents */
+ lblock = le32_to_cpu(ext->ee_block);
+- len = ext4_ext_get_actual_len(ext);
+ if ((lblock <= prev) && prev) {
+ *pblk = ext4_ext_pblock(ext);
+ return 0;
+ }
++ prev = lblock + ext4_ext_get_actual_len(ext) - 1;
+ ext++;
+ entries--;
+- prev = lblock + len - 1;
+ }
+ } else {
+ struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);
++
++ /*
++ * The logical block in the first entry should equal to
++ * the number in the parent index block.
++ */
++ if (depth != ext_depth(inode) &&
++ lblk != le32_to_cpu(ext_idx->ei_block))
++ return 0;
+ while (entries) {
+ if (!ext4_valid_extent_idx(inode, ext_idx))
+ return 0;
++
++ /* Check for overlapping index extents */
++ lblock = le32_to_cpu(ext_idx->ei_block);
++ if ((lblock <= prev) && prev) {
++ *pblk = ext4_idx_pblock(ext_idx);
++ return 0;
++ }
+ ext_idx++;
+ entries--;
++ prev = lblock;
+ }
+ }
+ return 1;
+@@ -397,7 +434,7 @@ static int ext4_valid_extent_entries(struct inode *inode,
+
+ static int __ext4_ext_check(const char *function, unsigned int line,
+ struct inode *inode, struct ext4_extent_header *eh,
+- int depth, ext4_fsblk_t pblk)
++ int depth, ext4_fsblk_t pblk, ext4_lblk_t lblk)
+ {
+ const char *error_msg;
+ int max = 0, err = -EFSCORRUPTED;
+@@ -423,7 +460,7 @@ static int __ext4_ext_check(const char *function, unsigned int line,
+ error_msg = "invalid eh_entries";
+ goto corrupted;
+ }
+- if (!ext4_valid_extent_entries(inode, eh, &pblk, depth)) {
++ if (!ext4_valid_extent_entries(inode, eh, lblk, &pblk, depth)) {
+ error_msg = "invalid extent entries";
+ goto corrupted;
+ }
+@@ -453,7 +490,7 @@ corrupted:
+ }
+
+ #define ext4_ext_check(inode, eh, depth, pblk) \
+- __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk))
++ __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk), 0)
+
+ int ext4_ext_check_inode(struct inode *inode)
+ {
+@@ -486,16 +523,18 @@ static void ext4_cache_extents(struct inode *inode,
+
+ static struct buffer_head *
+ __read_extent_tree_block(const char *function, unsigned int line,
+- struct inode *inode, ext4_fsblk_t pblk, int depth,
+- int flags)
++ struct inode *inode, struct ext4_extent_idx *idx,
++ int depth, int flags)
+ {
+ struct buffer_head *bh;
+ int err;
+ gfp_t gfp_flags = __GFP_MOVABLE | GFP_NOFS;
++ ext4_fsblk_t pblk;
+
+ if (flags & EXT4_EX_NOFAIL)
+ gfp_flags |= __GFP_NOFAIL;
+
++ pblk = ext4_idx_pblock(idx);
+ bh = sb_getblk_gfp(inode->i_sb, pblk, gfp_flags);
+ if (unlikely(!bh))
+ return ERR_PTR(-ENOMEM);
+@@ -508,8 +547,8 @@ __read_extent_tree_block(const char *function, unsigned int line,
+ }
+ if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
+ return bh;
+- err = __ext4_ext_check(function, line, inode,
+- ext_block_hdr(bh), depth, pblk);
++ err = __ext4_ext_check(function, line, inode, ext_block_hdr(bh),
++ depth, pblk, le32_to_cpu(idx->ei_block));
+ if (err)
+ goto errout;
+ set_buffer_verified(bh);
+@@ -527,8 +566,8 @@ errout:
+
+ }
+
+-#define read_extent_tree_block(inode, pblk, depth, flags) \
+- __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \
++#define read_extent_tree_block(inode, idx, depth, flags) \
++ __read_extent_tree_block(__func__, __LINE__, (inode), (idx), \
+ (depth), (flags))
+
+ /*
+@@ -578,8 +617,7 @@ int ext4_ext_precache(struct inode *inode)
+ i--;
+ continue;
+ }
+- bh = read_extent_tree_block(inode,
+- ext4_idx_pblock(path[i].p_idx++),
++ bh = read_extent_tree_block(inode, path[i].p_idx++,
+ depth - i - 1,
+ EXT4_EX_FORCE_CACHE);
+ if (IS_ERR(bh)) {
+@@ -884,8 +922,7 @@ ext4_find_extent(struct inode *inode, ext4_lblk_t block,
+ path[ppos].p_depth = i;
+ path[ppos].p_ext = NULL;
+
+- bh = read_extent_tree_block(inode, path[ppos].p_block, --i,
+- flags);
++ bh = read_extent_tree_block(inode, path[ppos].p_idx, --i, flags);
+ if (IS_ERR(bh)) {
+ ret = PTR_ERR(bh);
+ goto err;
+@@ -1494,7 +1531,6 @@ static int ext4_ext_search_right(struct inode *inode,
+ struct ext4_extent_header *eh;
+ struct ext4_extent_idx *ix;
+ struct ext4_extent *ex;
+- ext4_fsblk_t block;
+ int depth; /* Note, NOT eh_depth; depth from top of tree */
+ int ee_len;
+
+@@ -1561,20 +1597,17 @@ got_index:
+ * follow it and find the closest allocated
+ * block to the right */
+ ix++;
+- block = ext4_idx_pblock(ix);
+ while (++depth < path->p_depth) {
+ /* subtract from p_depth to get proper eh_depth */
+- bh = read_extent_tree_block(inode, block,
+- path->p_depth - depth, 0);
++ bh = read_extent_tree_block(inode, ix, path->p_depth - depth, 0);
+ if (IS_ERR(bh))
+ return PTR_ERR(bh);
+ eh = ext_block_hdr(bh);
+ ix = EXT_FIRST_INDEX(eh);
+- block = ext4_idx_pblock(ix);
+ put_bh(bh);
+ }
+
+- bh = read_extent_tree_block(inode, block, path->p_depth - depth, 0);
++ bh = read_extent_tree_block(inode, ix, path->p_depth - depth, 0);
+ if (IS_ERR(bh))
+ return PTR_ERR(bh);
+ eh = ext_block_hdr(bh);
+@@ -2953,9 +2986,9 @@ again:
+ ext_debug(inode, "move to level %d (block %llu)\n",
+ i + 1, ext4_idx_pblock(path[i].p_idx));
+ memset(path + i + 1, 0, sizeof(*path));
+- bh = read_extent_tree_block(inode,
+- ext4_idx_pblock(path[i].p_idx), depth - i - 1,
+- EXT4_EX_NOCACHE);
++ bh = read_extent_tree_block(inode, path[i].p_idx,
++ depth - i - 1,
++ EXT4_EX_NOCACHE);
+ if (IS_ERR(bh)) {
+ /* should we reset i_size? */
+ err = PTR_ERR(bh);
+@@ -4471,9 +4504,9 @@ retry:
+ return ret > 0 ? ret2 : ret;
+ }
+
+-static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len);
++static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len);
+
+-static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len);
++static int ext4_insert_range(struct file *file, loff_t offset, loff_t len);
+
+ static long ext4_zero_range(struct file *file, loff_t offset,
+ loff_t len, int mode)
+@@ -4545,6 +4578,10 @@ static long ext4_zero_range(struct file *file, loff_t offset,
+ /* Wait all existing dio workers, newcomers will block on i_mutex */
+ inode_dio_wait(inode);
+
++ ret = file_modified(file);
++ if (ret)
++ goto out_mutex;
++
+ /* Preallocate the range including the unaligned edges */
+ if (partial_begin || partial_end) {
+ ret = ext4_alloc_file_blocks(file,
+@@ -4612,8 +4649,6 @@ static long ext4_zero_range(struct file *file, loff_t offset,
+ ret = ext4_mark_inode_dirty(handle, inode);
+ if (unlikely(ret))
+ goto out_handle;
+- ext4_fc_track_range(handle, inode, offset >> inode->i_sb->s_blocksize_bits,
+- (offset + len - 1) >> inode->i_sb->s_blocksize_bits);
+ /* Zero out partial block at the edges of the range */
+ ret = ext4_zero_partial_blocks(handle, inode, offset, len);
+ if (ret >= 0)
+@@ -4665,7 +4700,7 @@ long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
+ ext4_fc_start_update(inode);
+
+ if (mode & FALLOC_FL_PUNCH_HOLE) {
+- ret = ext4_punch_hole(inode, offset, len);
++ ret = ext4_punch_hole(file, offset, len);
+ goto exit;
+ }
+
+@@ -4674,12 +4709,12 @@ long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
+ goto exit;
+
+ if (mode & FALLOC_FL_COLLAPSE_RANGE) {
+- ret = ext4_collapse_range(inode, offset, len);
++ ret = ext4_collapse_range(file, offset, len);
+ goto exit;
+ }
+
+ if (mode & FALLOC_FL_INSERT_RANGE) {
+- ret = ext4_insert_range(inode, offset, len);
++ ret = ext4_insert_range(file, offset, len);
+ goto exit;
+ }
+
+@@ -4715,6 +4750,10 @@ long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
+ /* Wait all existing dio workers, newcomers will block on i_mutex */
+ inode_dio_wait(inode);
+
++ ret = file_modified(file);
++ if (ret)
++ goto out;
++
+ ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, flags);
+ if (ret)
+ goto out;
+@@ -4977,36 +5016,6 @@ int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo,
+ return ext4_fill_es_cache_info(inode, start_blk, len_blks, fieinfo);
+ }
+
+-/*
+- * ext4_access_path:
+- * Function to access the path buffer for marking it dirty.
+- * It also checks if there are sufficient credits left in the journal handle
+- * to update path.
+- */
+-static int
+-ext4_access_path(handle_t *handle, struct inode *inode,
+- struct ext4_ext_path *path)
+-{
+- int credits, err;
+-
+- if (!ext4_handle_valid(handle))
+- return 0;
+-
+- /*
+- * Check if need to extend journal credits
+- * 3 for leaf, sb, and inode plus 2 (bmap and group
+- * descriptor) for each block group; assume two block
+- * groups
+- */
+- credits = ext4_writepage_trans_blocks(inode);
+- err = ext4_datasem_ensure_credits(handle, inode, 7, credits, 0);
+- if (err < 0)
+- return err;
+-
+- err = ext4_ext_get_access(handle, inode, path);
+- return err;
+-}
+-
+ /*
+ * ext4_ext_shift_path_extents:
+ * Shift the extents of a path structure lying between path[depth].p_ext
+@@ -5021,6 +5030,7 @@ ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
+ int depth, err = 0;
+ struct ext4_extent *ex_start, *ex_last;
+ bool update = false;
++ int credits, restart_credits;
+ depth = path->p_depth;
+
+ while (depth >= 0) {
+@@ -5030,13 +5040,26 @@ ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
+ return -EFSCORRUPTED;
+
+ ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
++ /* leaf + sb + inode */
++ credits = 3;
++ if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr)) {
++ update = true;
++ /* extent tree + sb + inode */
++ credits = depth + 2;
++ }
+
+- err = ext4_access_path(handle, inode, path + depth);
+- if (err)
++ restart_credits = ext4_writepage_trans_blocks(inode);
++ err = ext4_datasem_ensure_credits(handle, inode, credits,
++ restart_credits, 0);
++ if (err) {
++ if (err > 0)
++ err = -EAGAIN;
+ goto out;
++ }
+
+- if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr))
+- update = true;
++ err = ext4_ext_get_access(handle, inode, path + depth);
++ if (err)
++ goto out;
+
+ while (ex_start <= ex_last) {
+ if (SHIFT == SHIFT_LEFT) {
+@@ -5067,7 +5090,7 @@ ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
+ }
+
+ /* Update index too */
+- err = ext4_access_path(handle, inode, path + depth);
++ err = ext4_ext_get_access(handle, inode, path + depth);
+ if (err)
+ goto out;
+
+@@ -5106,6 +5129,7 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
+ int ret = 0, depth;
+ struct ext4_extent *extent;
+ ext4_lblk_t stop, *iterator, ex_start, ex_end;
++ ext4_lblk_t tmp = EXT_MAX_BLOCKS;
+
+ /* Let path point to the last extent */
+ path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
+@@ -5159,11 +5183,15 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
+ * till we reach stop. In case of right shift, iterator points to stop
+ * and it is decreased till we reach start.
+ */
++again:
+ if (SHIFT == SHIFT_LEFT)
+ iterator = &start;
+ else
+ iterator = &stop;
+
++ if (tmp != EXT_MAX_BLOCKS)
++ *iterator = tmp;
++
+ /*
+ * Its safe to start updating extents. Start and stop are unsigned, so
+ * in case of right shift if extent with 0 block is reached, iterator
+@@ -5192,6 +5220,7 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
+ }
+ }
+
++ tmp = *iterator;
+ if (SHIFT == SHIFT_LEFT) {
+ extent = EXT_LAST_EXTENT(path[depth].p_hdr);
+ *iterator = le32_to_cpu(extent->ee_block) +
+@@ -5210,6 +5239,9 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
+ }
+ ret = ext4_ext_shift_path_extents(path, shift, inode,
+ handle, SHIFT);
++ /* iterator can be NULL which means we should break */
++ if (ret == -EAGAIN)
++ goto again;
+ if (ret)
+ break;
+ }
+@@ -5224,8 +5256,9 @@ out:
+ * This implements the fallocate's collapse range functionality for ext4
+ * Returns: 0 and non-zero on error.
+ */
+-static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
++static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len)
+ {
++ struct inode *inode = file_inode(file);
+ struct super_block *sb = inode->i_sb;
+ struct address_space *mapping = inode->i_mapping;
+ ext4_lblk_t punch_start, punch_stop;
+@@ -5277,6 +5310,10 @@ static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
+ /* Wait for existing dio to complete */
+ inode_dio_wait(inode);
+
++ ret = file_modified(file);
++ if (ret)
++ goto out_mutex;
++
+ /*
+ * Prevent page faults from reinstantiating pages we have released from
+ * page cache.
+@@ -5316,7 +5353,7 @@ static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
+ ret = PTR_ERR(handle);
+ goto out_mmap;
+ }
+- ext4_fc_start_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE);
++ ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle);
+
+ down_write(&EXT4_I(inode)->i_data_sem);
+ ext4_discard_preallocations(inode, 0);
+@@ -5355,7 +5392,6 @@ static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
+
+ out_stop:
+ ext4_journal_stop(handle);
+- ext4_fc_stop_ineligible(sb);
+ out_mmap:
+ filemap_invalidate_unlock(mapping);
+ out_mutex:
+@@ -5371,8 +5407,9 @@ out_mutex:
+ * by len bytes.
+ * Returns 0 on success, error otherwise.
+ */
+-static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
++static int ext4_insert_range(struct file *file, loff_t offset, loff_t len)
+ {
++ struct inode *inode = file_inode(file);
+ struct super_block *sb = inode->i_sb;
+ struct address_space *mapping = inode->i_mapping;
+ handle_t *handle;
+@@ -5429,6 +5466,10 @@ static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
+ /* Wait for existing dio to complete */
+ inode_dio_wait(inode);
+
++ ret = file_modified(file);
++ if (ret)
++ goto out_mutex;
++
+ /*
+ * Prevent page faults from reinstantiating pages we have released from
+ * page cache.
+@@ -5457,7 +5498,7 @@ static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
+ ret = PTR_ERR(handle);
+ goto out_mmap;
+ }
+- ext4_fc_start_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE);
++ ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle);
+
+ /* Expand file to avoid data loss if there is error while shifting */
+ inode->i_size += len;
+@@ -5532,7 +5573,6 @@ static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
+
+ out_stop:
+ ext4_journal_stop(handle);
+- ext4_fc_stop_ineligible(sb);
+ out_mmap:
+ filemap_invalidate_unlock(mapping);
+ out_mutex:
+@@ -6072,11 +6112,15 @@ int ext4_ext_clear_bb(struct inode *inode)
+
+ ext4_mb_mark_bb(inode->i_sb,
+ path[j].p_block, 1, 0);
++ ext4_fc_record_regions(inode->i_sb, inode->i_ino,
++ 0, path[j].p_block, 1, 1);
+ }
+ ext4_ext_drop_refs(path);
+ kfree(path);
+ }
+ ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
++ ext4_fc_record_regions(inode->i_sb, inode->i_ino,
++ map.m_lblk, map.m_pblk, map.m_len, 1);
+ }
+ cur = cur + map.m_len;
+ }
+diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c
+index 8ea5a81e65548..285c91b0166c0 100644
+--- a/fs/ext4/fast_commit.c
++++ b/fs/ext4/fast_commit.c
+@@ -65,21 +65,11 @@
+ *
+ * Fast Commit Ineligibility
+ * -------------------------
+- * Not all operations are supported by fast commits today (e.g extended
+- * attributes). Fast commit ineligibility is marked by calling one of the
+- * two following functions:
+- *
+- * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall
+- * back to full commit. This is useful in case of transient errors.
+ *
+- * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all
+- * the fast commits happening between ext4_fc_start_ineligible() and
+- * ext4_fc_stop_ineligible() and one fast commit after the call to
+- * ext4_fc_stop_ineligible() to fall back to full commits. It is important to
+- * make one more fast commit to fall back to full commit after stop call so
+- * that it guaranteed that the fast commit ineligible operation contained
+- * within ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() is
+- * followed by at least 1 full commit.
++ * Not all operations are supported by fast commits today (e.g extended
++ * attributes). Fast commit ineligibility is marked by calling
++ * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
++ * to full commit.
+ *
+ * Atomicity of commits
+ * --------------------
+@@ -312,60 +302,36 @@ restart:
+ }
+
+ /*
+- * Mark file system as fast commit ineligible. This means that next commit
+- * operation would result in a full jbd2 commit.
++ * Mark file system as fast commit ineligible, and record latest
++ * ineligible transaction tid. This means until the recorded
++ * transaction, commit operation would result in a full jbd2 commit.
+ */
+-void ext4_fc_mark_ineligible(struct super_block *sb, int reason)
++void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
+ {
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
++ tid_t tid;
+
+ if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
+ (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
+ return;
+
+ ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
++ if (handle && !IS_ERR(handle))
++ tid = handle->h_transaction->t_tid;
++ else {
++ read_lock(&sbi->s_journal->j_state_lock);
++ tid = sbi->s_journal->j_running_transaction ?
++ sbi->s_journal->j_running_transaction->t_tid : 0;
++ read_unlock(&sbi->s_journal->j_state_lock);
++ }
++ spin_lock(&sbi->s_fc_lock);
++ if (sbi->s_fc_ineligible_tid < tid)
++ sbi->s_fc_ineligible_tid = tid;
++ spin_unlock(&sbi->s_fc_lock);
+ WARN_ON(reason >= EXT4_FC_REASON_MAX);
+ sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
+ }
+
+-/*
+- * Start a fast commit ineligible update. Any commits that happen while
+- * such an operation is in progress fall back to full commits.
+- */
+-void ext4_fc_start_ineligible(struct super_block *sb, int reason)
+-{
+- struct ext4_sb_info *sbi = EXT4_SB(sb);
+-
+- if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
+- (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
+- return;
+-
+- WARN_ON(reason >= EXT4_FC_REASON_MAX);
+- sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
+- atomic_inc(&sbi->s_fc_ineligible_updates);
+-}
+-
+-/*
+- * Stop a fast commit ineligible update. We set EXT4_MF_FC_INELIGIBLE flag here
+- * to ensure that after stopping the ineligible update, at least one full
+- * commit takes place.
+- */
+-void ext4_fc_stop_ineligible(struct super_block *sb)
+-{
+- if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
+- (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
+- return;
+-
+- ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
+- atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates);
+-}
+-
+-static inline int ext4_fc_is_ineligible(struct super_block *sb)
+-{
+- return (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE) ||
+- atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates));
+-}
+-
+ /*
+ * Generic fast commit tracking function. If this is the first time this we are
+ * called after a full commit, we initialize fast commit fields and then call
+@@ -391,7 +357,7 @@ static int ext4_fc_track_template(
+ (sbi->s_mount_state & EXT4_FC_REPLAY))
+ return -EOPNOTSUPP;
+
+- if (ext4_fc_is_ineligible(inode->i_sb))
++ if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
+ return -EINVAL;
+
+ tid = handle->h_transaction->t_tid;
+@@ -411,7 +377,8 @@ static int ext4_fc_track_template(
+ spin_lock(&sbi->s_fc_lock);
+ if (list_empty(&EXT4_I(inode)->i_fc_list))
+ list_add_tail(&EXT4_I(inode)->i_fc_list,
+- (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) ?
++ (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
++ sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
+ &sbi->s_fc_q[FC_Q_STAGING] :
+ &sbi->s_fc_q[FC_Q_MAIN]);
+ spin_unlock(&sbi->s_fc_lock);
+@@ -437,7 +404,7 @@ static int __track_dentry_update(struct inode *inode, void *arg, bool update)
+ mutex_unlock(&ei->i_fc_lock);
+ node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
+ if (!node) {
+- ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM);
++ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
+ mutex_lock(&ei->i_fc_lock);
+ return -ENOMEM;
+ }
+@@ -450,7 +417,7 @@ static int __track_dentry_update(struct inode *inode, void *arg, bool update)
+ if (!node->fcd_name.name) {
+ kmem_cache_free(ext4_fc_dentry_cachep, node);
+ ext4_fc_mark_ineligible(inode->i_sb,
+- EXT4_FC_REASON_NOMEM);
++ EXT4_FC_REASON_NOMEM, NULL);
+ mutex_lock(&ei->i_fc_lock);
+ return -ENOMEM;
+ }
+@@ -464,7 +431,8 @@ static int __track_dentry_update(struct inode *inode, void *arg, bool update)
+ node->fcd_name.len = dentry->d_name.len;
+
+ spin_lock(&sbi->s_fc_lock);
+- if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING))
++ if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
++ sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
+ list_add_tail(&node->fcd_list,
+ &sbi->s_fc_dentry_q[FC_Q_STAGING]);
+ else
+@@ -552,7 +520,7 @@ void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
+
+ if (ext4_should_journal_data(inode)) {
+ ext4_fc_mark_ineligible(inode->i_sb,
+- EXT4_FC_REASON_INODE_JOURNAL_DATA);
++ EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
+ return;
+ }
+
+@@ -928,7 +896,6 @@ static int ext4_fc_submit_inode_data_all(journal_t *journal)
+ int ret = 0;
+
+ spin_lock(&sbi->s_fc_lock);
+- ext4_set_mount_flag(sb, EXT4_MF_FC_COMMITTING);
+ list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
+ ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
+ while (atomic_read(&ei->i_fc_updates)) {
+@@ -1121,6 +1088,32 @@ out:
+ return ret;
+ }
+
++static void ext4_fc_update_stats(struct super_block *sb, int status,
++ u64 commit_time, int nblks)
++{
++ struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
++
++ jbd_debug(1, "Fast commit ended with status = %d", status);
++ if (status == EXT4_FC_STATUS_OK) {
++ stats->fc_num_commits++;
++ stats->fc_numblks += nblks;
++ if (likely(stats->s_fc_avg_commit_time))
++ stats->s_fc_avg_commit_time =
++ (commit_time +
++ stats->s_fc_avg_commit_time * 3) / 4;
++ else
++ stats->s_fc_avg_commit_time = commit_time;
++ } else if (status == EXT4_FC_STATUS_FAILED ||
++ status == EXT4_FC_STATUS_INELIGIBLE) {
++ if (status == EXT4_FC_STATUS_FAILED)
++ stats->fc_failed_commits++;
++ stats->fc_ineligible_commits++;
++ } else {
++ stats->fc_skipped_commits++;
++ }
++ trace_ext4_fc_commit_stop(sb, nblks, status);
++}
++
+ /*
+ * The main commit entry point. Performs a fast commit for transaction
+ * commit_tid if needed. If it's not possible to perform a fast commit
+@@ -1133,18 +1126,15 @@ int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
+ int nblks = 0, ret, bsize = journal->j_blocksize;
+ int subtid = atomic_read(&sbi->s_fc_subtid);
+- int reason = EXT4_FC_REASON_OK, fc_bufs_before = 0;
++ int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
+ ktime_t start_time, commit_time;
+
+ trace_ext4_fc_commit_start(sb);
+
+ start_time = ktime_get();
+
+- if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
+- (ext4_fc_is_ineligible(sb))) {
+- reason = EXT4_FC_REASON_INELIGIBLE;
+- goto out;
+- }
++ if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
++ return jbd2_complete_transaction(journal, commit_tid);
+
+ restart_fc:
+ ret = jbd2_fc_begin_commit(journal, commit_tid);
+@@ -1153,74 +1143,59 @@ restart_fc:
+ if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
+ commit_tid > journal->j_commit_sequence)
+ goto restart_fc;
+- reason = EXT4_FC_REASON_ALREADY_COMMITTED;
+- goto out;
++ ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0);
++ return 0;
+ } else if (ret) {
+- sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
+- reason = EXT4_FC_REASON_FC_START_FAILED;
+- goto out;
++ /*
++ * Commit couldn't start. Just update stats and perform a
++ * full commit.
++ */
++ ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0);
++ return jbd2_complete_transaction(journal, commit_tid);
++ }
++
++ /*
++ * After establishing journal barrier via jbd2_fc_begin_commit(), check
++ * if we are fast commit ineligible.
++ */
++ if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) {
++ status = EXT4_FC_STATUS_INELIGIBLE;
++ goto fallback;
+ }
+
+ fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
+ ret = ext4_fc_perform_commit(journal);
+ if (ret < 0) {
+- sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
+- reason = EXT4_FC_REASON_FC_FAILED;
+- goto out;
++ status = EXT4_FC_STATUS_FAILED;
++ goto fallback;
+ }
+ nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
+ ret = jbd2_fc_wait_bufs(journal, nblks);
+ if (ret < 0) {
+- sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
+- reason = EXT4_FC_REASON_FC_FAILED;
+- goto out;
++ status = EXT4_FC_STATUS_FAILED;
++ goto fallback;
+ }
+ atomic_inc(&sbi->s_fc_subtid);
+- jbd2_fc_end_commit(journal);
+-out:
+- /* Has any ineligible update happened since we started? */
+- if (reason == EXT4_FC_REASON_OK && ext4_fc_is_ineligible(sb)) {
+- sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
+- reason = EXT4_FC_REASON_INELIGIBLE;
+- }
+-
+- spin_lock(&sbi->s_fc_lock);
+- if (reason != EXT4_FC_REASON_OK &&
+- reason != EXT4_FC_REASON_ALREADY_COMMITTED) {
+- sbi->s_fc_stats.fc_ineligible_commits++;
+- } else {
+- sbi->s_fc_stats.fc_num_commits++;
+- sbi->s_fc_stats.fc_numblks += nblks;
+- }
+- spin_unlock(&sbi->s_fc_lock);
+- nblks = (reason == EXT4_FC_REASON_OK) ? nblks : 0;
+- trace_ext4_fc_commit_stop(sb, nblks, reason);
+- commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
++ ret = jbd2_fc_end_commit(journal);
+ /*
+- * weight the commit time higher than the average time so we don't
+- * react too strongly to vast changes in the commit time
++ * weight the commit time higher than the average time so we
++ * don't react too strongly to vast changes in the commit time
+ */
+- if (likely(sbi->s_fc_avg_commit_time))
+- sbi->s_fc_avg_commit_time = (commit_time +
+- sbi->s_fc_avg_commit_time * 3) / 4;
+- else
+- sbi->s_fc_avg_commit_time = commit_time;
+- jbd_debug(1,
+- "Fast commit ended with blks = %d, reason = %d, subtid - %d",
+- nblks, reason, subtid);
+- if (reason == EXT4_FC_REASON_FC_FAILED)
+- return jbd2_fc_end_commit_fallback(journal);
+- if (reason == EXT4_FC_REASON_FC_START_FAILED ||
+- reason == EXT4_FC_REASON_INELIGIBLE)
+- return jbd2_complete_transaction(journal, commit_tid);
+- return 0;
++ commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
++ ext4_fc_update_stats(sb, status, commit_time, nblks);
++ return ret;
++
++fallback:
++ ret = jbd2_fc_end_commit_fallback(journal);
++ ext4_fc_update_stats(sb, status, 0, 0);
++ return ret;
+ }
+
+ /*
+ * Fast commit cleanup routine. This is called after every fast commit and
+ * full commit. full is true if we are called after a full commit.
+ */
+-static void ext4_fc_cleanup(journal_t *journal, int full)
++static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
+ {
+ struct super_block *sb = journal->j_private;
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
+@@ -1238,7 +1213,8 @@ static void ext4_fc_cleanup(journal_t *journal, int full)
+ list_del_init(&iter->i_fc_list);
+ ext4_clear_inode_state(&iter->vfs_inode,
+ EXT4_STATE_FC_COMMITTING);
+- ext4_fc_reset_inode(&iter->vfs_inode);
++ if (iter->i_sync_tid <= tid)
++ ext4_fc_reset_inode(&iter->vfs_inode);
+ /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
+ smp_mb();
+ #if (BITS_PER_LONG < 64)
+@@ -1267,8 +1243,10 @@ static void ext4_fc_cleanup(journal_t *journal, int full)
+ list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
+ &sbi->s_fc_q[FC_Q_MAIN]);
+
+- ext4_clear_mount_flag(sb, EXT4_MF_FC_COMMITTING);
+- ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
++ if (tid >= sbi->s_fc_ineligible_tid) {
++ sbi->s_fc_ineligible_tid = 0;
++ ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
++ }
+
+ if (full)
+ sbi->s_fc_bytes = 0;
+@@ -1433,14 +1411,15 @@ static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
+ if (state->fc_modified_inodes[i] == ino)
+ return 0;
+ if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
+- state->fc_modified_inodes_size +=
+- EXT4_FC_REPLAY_REALLOC_INCREMENT;
+ state->fc_modified_inodes = krealloc(
+- state->fc_modified_inodes, sizeof(int) *
+- state->fc_modified_inodes_size,
+- GFP_KERNEL);
++ state->fc_modified_inodes,
++ sizeof(int) * (state->fc_modified_inodes_size +
++ EXT4_FC_REPLAY_REALLOC_INCREMENT),
++ GFP_KERNEL);
+ if (!state->fc_modified_inodes)
+ return -ENOMEM;
++ state->fc_modified_inodes_size +=
++ EXT4_FC_REPLAY_REALLOC_INCREMENT;
+ }
+ state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
+ return 0;
+@@ -1472,7 +1451,9 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,
+ }
+ inode = NULL;
+
+- ext4_fc_record_modified_inode(sb, ino);
++ ret = ext4_fc_record_modified_inode(sb, ino);
++ if (ret)
++ goto out;
+
+ raw_fc_inode = (struct ext4_inode *)
+ (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
+@@ -1603,16 +1584,23 @@ out:
+ }
+
+ /*
+- * Record physical disk regions which are in use as per fast commit area. Our
+- * simple replay phase allocator excludes these regions from allocation.
++ * Record physical disk regions which are in use as per fast commit area,
++ * and used by inodes during replay phase. Our simple replay phase
++ * allocator excludes these regions from allocation.
+ */
+-static int ext4_fc_record_regions(struct super_block *sb, int ino,
+- ext4_lblk_t lblk, ext4_fsblk_t pblk, int len)
++int ext4_fc_record_regions(struct super_block *sb, int ino,
++ ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
+ {
+ struct ext4_fc_replay_state *state;
+ struct ext4_fc_alloc_region *region;
+
+ state = &EXT4_SB(sb)->s_fc_replay_state;
++ /*
++ * during replay phase, the fc_regions_valid may not same as
++ * fc_regions_used, update it when do new additions.
++ */
++ if (replay && state->fc_regions_used != state->fc_regions_valid)
++ state->fc_regions_used = state->fc_regions_valid;
+ if (state->fc_regions_used == state->fc_regions_size) {
+ state->fc_regions_size +=
+ EXT4_FC_REPLAY_REALLOC_INCREMENT;
+@@ -1630,6 +1618,9 @@ static int ext4_fc_record_regions(struct super_block *sb, int ino,
+ region->pblk = pblk;
+ region->len = len;
+
++ if (replay)
++ state->fc_regions_valid++;
++
+ return 0;
+ }
+
+@@ -1661,6 +1652,8 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
+ }
+
+ ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
++ if (ret)
++ goto out;
+
+ start = le32_to_cpu(ex->ee_block);
+ start_pblk = ext4_ext_pblock(ex);
+@@ -1678,18 +1671,14 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
+ map.m_pblk = 0;
+ ret = ext4_map_blocks(NULL, inode, &map, 0);
+
+- if (ret < 0) {
+- iput(inode);
+- return 0;
+- }
++ if (ret < 0)
++ goto out;
+
+ if (ret == 0) {
+ /* Range is not mapped */
+ path = ext4_find_extent(inode, cur, NULL, 0);
+- if (IS_ERR(path)) {
+- iput(inode);
+- return 0;
+- }
++ if (IS_ERR(path))
++ goto out;
+ memset(&newex, 0, sizeof(newex));
+ newex.ee_block = cpu_to_le32(cur);
+ ext4_ext_store_pblock(
+@@ -1703,10 +1692,8 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
+ up_write((&EXT4_I(inode)->i_data_sem));
+ ext4_ext_drop_refs(path);
+ kfree(path);
+- if (ret) {
+- iput(inode);
+- return 0;
+- }
++ if (ret)
++ goto out;
+ goto next;
+ }
+
+@@ -1719,10 +1706,8 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
+ ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
+ ext4_ext_is_unwritten(ex),
+ start_pblk + cur - start);
+- if (ret) {
+- iput(inode);
+- return 0;
+- }
++ if (ret)
++ goto out;
+ /*
+ * Mark the old blocks as free since they aren't used
+ * anymore. We maintain an array of all the modified
+@@ -1742,10 +1727,8 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
+ ext4_ext_is_unwritten(ex), map.m_pblk);
+ ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
+ ext4_ext_is_unwritten(ex), map.m_pblk);
+- if (ret) {
+- iput(inode);
+- return 0;
+- }
++ if (ret)
++ goto out;
+ /*
+ * We may have split the extent tree while toggling the state.
+ * Try to shrink the extent tree now.
+@@ -1757,6 +1740,7 @@ next:
+ }
+ ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
+ sb->s_blocksize_bits);
++out:
+ iput(inode);
+ return 0;
+ }
+@@ -1786,6 +1770,8 @@ ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
+ }
+
+ ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
++ if (ret)
++ goto out;
+
+ jbd_debug(1, "DEL_RANGE, inode %ld, lblk %d, len %d\n",
+ inode->i_ino, le32_to_cpu(lrange.fc_lblk),
+@@ -1795,10 +1781,8 @@ ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
+ map.m_len = remaining;
+
+ ret = ext4_map_blocks(NULL, inode, &map, 0);
+- if (ret < 0) {
+- iput(inode);
+- return 0;
+- }
++ if (ret < 0)
++ goto out;
+ if (ret > 0) {
+ remaining -= ret;
+ cur += ret;
+@@ -1809,16 +1793,18 @@ ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
+ }
+ }
+
+- ret = ext4_punch_hole(inode,
+- le32_to_cpu(lrange.fc_lblk) << sb->s_blocksize_bits,
+- le32_to_cpu(lrange.fc_len) << sb->s_blocksize_bits);
++ down_write(&EXT4_I(inode)->i_data_sem);
++ ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
++ le32_to_cpu(lrange.fc_lblk) +
++ le32_to_cpu(lrange.fc_len) - 1);
++ up_write(&EXT4_I(inode)->i_data_sem);
+ if (ret)
+- jbd_debug(1, "ext4_punch_hole returned %d", ret);
++ goto out;
+ ext4_ext_replay_shrink_inode(inode,
+ i_size_read(inode) >> sb->s_blocksize_bits);
+ ext4_mark_inode_dirty(NULL, inode);
++out:
+ iput(inode);
+-
+ return 0;
+ }
+
+@@ -1970,7 +1956,7 @@ static int ext4_fc_replay_scan(journal_t *journal,
+ ret = ext4_fc_record_regions(sb,
+ le32_to_cpu(ext.fc_ino),
+ le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
+- ext4_ext_get_actual_len(ex));
++ ext4_ext_get_actual_len(ex), 0);
+ if (ret < 0)
+ break;
+ ret = JBD2_FC_REPLAY_CONTINUE;
+@@ -2166,7 +2152,7 @@ int ext4_fc_info_show(struct seq_file *seq, void *v)
+ "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
+ stats->fc_num_commits, stats->fc_ineligible_commits,
+ stats->fc_numblks,
+- div_u64(sbi->s_fc_avg_commit_time, 1000));
++ div_u64(stats->s_fc_avg_commit_time, 1000));
+ seq_puts(seq, "Ineligible reasons:\n");
+ for (i = 0; i < EXT4_FC_REASON_MAX; i++)
+ seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
+@@ -2185,3 +2171,8 @@ int __init ext4_fc_init_dentry_cache(void)
+
+ return 0;
+ }
++
++void ext4_fc_destroy_dentry_cache(void)
++{
++ kmem_cache_destroy(ext4_fc_dentry_cachep);
++}
+diff --git a/fs/ext4/fast_commit.h b/fs/ext4/fast_commit.h
+index 937c381b4c85e..083ad1cb705a7 100644
+--- a/fs/ext4/fast_commit.h
++++ b/fs/ext4/fast_commit.h
+@@ -71,21 +71,19 @@ struct ext4_fc_tail {
+ };
+
+ /*
+- * Fast commit reason codes
++ * Fast commit status codes
++ */
++enum {
++ EXT4_FC_STATUS_OK = 0,
++ EXT4_FC_STATUS_INELIGIBLE,
++ EXT4_FC_STATUS_SKIPPED,
++ EXT4_FC_STATUS_FAILED,
++};
++
++/*
++ * Fast commit ineligiblity reasons:
+ */
+ enum {
+- /*
+- * Commit status codes:
+- */
+- EXT4_FC_REASON_OK = 0,
+- EXT4_FC_REASON_INELIGIBLE,
+- EXT4_FC_REASON_ALREADY_COMMITTED,
+- EXT4_FC_REASON_FC_START_FAILED,
+- EXT4_FC_REASON_FC_FAILED,
+-
+- /*
+- * Fast commit ineligiblity reasons:
+- */
+ EXT4_FC_REASON_XATTR = 0,
+ EXT4_FC_REASON_CROSS_RENAME,
+ EXT4_FC_REASON_JOURNAL_FLAG_CHANGE,
+@@ -117,7 +115,10 @@ struct ext4_fc_stats {
+ unsigned int fc_ineligible_reason_count[EXT4_FC_REASON_MAX];
+ unsigned long fc_num_commits;
+ unsigned long fc_ineligible_commits;
++ unsigned long fc_failed_commits;
++ unsigned long fc_skipped_commits;
+ unsigned long fc_numblks;
++ u64 s_fc_avg_commit_time;
+ };
+
+ #define EXT4_FC_REPLAY_REALLOC_INCREMENT 4
+diff --git a/fs/ext4/file.c b/fs/ext4/file.c
+index ac0e11bbb4450..b25c1f8f7c4f1 100644
+--- a/fs/ext4/file.c
++++ b/fs/ext4/file.c
+@@ -74,7 +74,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
+ return generic_file_read_iter(iocb, to);
+ }
+
+- ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0);
++ ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, 0);
+ inode_unlock_shared(inode);
+
+ file_accessed(iocb->ki_filp);
+@@ -566,7 +566,8 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ if (ilock_shared)
+ iomap_ops = &ext4_iomap_overwrite_ops;
+ ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,
+- (unaligned_io || extend) ? IOMAP_DIO_FORCE_WAIT : 0);
++ (unaligned_io || extend) ? IOMAP_DIO_FORCE_WAIT : 0,
++ 0);
+ if (ret == -ENOTBLK)
+ ret = 0;
+
+diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
+index 39a1ab129fdc9..46fdb40c3962c 100644
+--- a/fs/ext4/inline.c
++++ b/fs/ext4/inline.c
+@@ -1133,7 +1133,15 @@ static void ext4_restore_inline_data(handle_t *handle, struct inode *inode,
+ struct ext4_iloc *iloc,
+ void *buf, int inline_size)
+ {
+- ext4_create_inline_data(handle, inode, inline_size);
++ int ret;
++
++ ret = ext4_create_inline_data(handle, inode, inline_size);
++ if (ret) {
++ ext4_msg(inode->i_sb, KERN_EMERG,
++ "error restoring inline_data for inode -- potential data loss! (inode %lu, error %d)",
++ inode->i_ino, ret);
++ return;
++ }
+ ext4_write_inline_data(inode, iloc, buf, 0, inline_size);
+ ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
+ }
+@@ -1780,19 +1788,20 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
+ void *inline_pos;
+ unsigned int offset;
+ struct ext4_dir_entry_2 *de;
+- bool ret = true;
++ bool ret = false;
+
+ err = ext4_get_inode_loc(dir, &iloc);
+ if (err) {
+ EXT4_ERROR_INODE_ERR(dir, -err,
+ "error %d getting inode %lu block",
+ err, dir->i_ino);
+- return true;
++ return false;
+ }
+
+ down_read(&EXT4_I(dir)->xattr_sem);
+ if (!ext4_has_inline_data(dir)) {
+ *has_inline_data = 0;
++ ret = true;
+ goto out;
+ }
+
+@@ -1801,7 +1810,6 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
+ ext4_warning(dir->i_sb,
+ "bad inline directory (dir #%lu) - no `..'",
+ dir->i_ino);
+- ret = true;
+ goto out;
+ }
+
+@@ -1820,16 +1828,15 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
+ dir->i_ino, le32_to_cpu(de->inode),
+ le16_to_cpu(de->rec_len), de->name_len,
+ inline_size);
+- ret = true;
+ goto out;
+ }
+ if (le32_to_cpu(de->inode)) {
+- ret = false;
+ goto out;
+ }
+ offset += ext4_rec_len_from_disk(de->rec_len, inline_size);
+ }
+
++ ret = true;
+ out:
+ up_read(&EXT4_I(dir)->xattr_sem);
+ brelse(iloc.bh);
+diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
+index 0f06305167d5a..db73b49bd9795 100644
+--- a/fs/ext4/inode.c
++++ b/fs/ext4/inode.c
+@@ -337,7 +337,7 @@ stop_handle:
+ return;
+ no_delete:
+ if (!list_empty(&EXT4_I(inode)->i_fc_list))
+- ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM);
++ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
+ ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
+ }
+
+@@ -741,10 +741,11 @@ out_sem:
+ if (ret)
+ return ret;
+ }
+- ext4_fc_track_range(handle, inode, map->m_lblk,
+- map->m_lblk + map->m_len - 1);
+ }
+-
++ if (retval > 0 && (map->m_flags & EXT4_MAP_UNWRITTEN ||
++ map->m_flags & EXT4_MAP_MAPPED))
++ ext4_fc_track_range(handle, inode, map->m_lblk,
++ map->m_lblk + map->m_len - 1);
+ if (retval < 0)
+ ext_debug(inode, "failed with err %d\n", retval);
+ return retval;
+@@ -1711,16 +1712,13 @@ static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
+ }
+
+ /*
+- * the buffer head associated with a delayed and not unwritten
+- * block found in the extent status cache must contain an
+- * invalid block number and have its BH_New and BH_Delay bits
+- * set, reflecting the state assigned when the block was
+- * initially delayed allocated
++ * Delayed extent could be allocated by fallocate.
++ * So we need to check it.
+ */
+- if (ext4_es_is_delonly(&es)) {
+- BUG_ON(bh->b_blocknr != invalid_block);
+- BUG_ON(!buffer_new(bh));
+- BUG_ON(!buffer_delay(bh));
++ if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
++ map_bh(bh, inode->i_sb, invalid_block);
++ set_buffer_new(bh);
++ set_buffer_delay(bh);
+ return 0;
+ }
+
+@@ -1847,30 +1845,16 @@ int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
+ return 0;
+ }
+
+-static int bget_one(handle_t *handle, struct inode *inode,
+- struct buffer_head *bh)
+-{
+- get_bh(bh);
+- return 0;
+-}
+-
+-static int bput_one(handle_t *handle, struct inode *inode,
+- struct buffer_head *bh)
+-{
+- put_bh(bh);
+- return 0;
+-}
+-
+ static int __ext4_journalled_writepage(struct page *page,
+ unsigned int len)
+ {
+ struct address_space *mapping = page->mapping;
+ struct inode *inode = mapping->host;
+- struct buffer_head *page_bufs = NULL;
+ handle_t *handle = NULL;
+ int ret = 0, err = 0;
+ int inline_data = ext4_has_inline_data(inode);
+ struct buffer_head *inode_bh = NULL;
++ loff_t size;
+
+ ClearPageChecked(page);
+
+@@ -1880,14 +1864,6 @@ static int __ext4_journalled_writepage(struct page *page,
+ inode_bh = ext4_journalled_write_inline_data(inode, len, page);
+ if (inode_bh == NULL)
+ goto out;
+- } else {
+- page_bufs = page_buffers(page);
+- if (!page_bufs) {
+- BUG();
+- goto out;
+- }
+- ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
+- NULL, bget_one);
+ }
+ /*
+ * We need to release the page lock before we start the
+@@ -1908,7 +1884,8 @@ static int __ext4_journalled_writepage(struct page *page,
+
+ lock_page(page);
+ put_page(page);
+- if (page->mapping != mapping) {
++ size = i_size_read(inode);
++ if (page->mapping != mapping || page_offset(page) > size) {
+ /* The page got truncated from under us */
+ ext4_journal_stop(handle);
+ ret = 0;
+@@ -1918,6 +1895,13 @@ static int __ext4_journalled_writepage(struct page *page,
+ if (inline_data) {
+ ret = ext4_mark_inode_dirty(handle, inode);
+ } else {
++ struct buffer_head *page_bufs = page_buffers(page);
++
++ if (page->index == size >> PAGE_SHIFT)
++ len = size & ~PAGE_MASK;
++ else
++ len = PAGE_SIZE;
++
+ ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
+ NULL, do_journal_get_write_access);
+
+@@ -1938,9 +1922,6 @@ static int __ext4_journalled_writepage(struct page *page,
+ out:
+ unlock_page(page);
+ out_no_pagelock:
+- if (!inline_data && page_bufs)
+- ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len,
+- NULL, bput_one);
+ brelse(inode_bh);
+ return ret;
+ }
+@@ -2011,6 +1992,15 @@ static int ext4_writepage(struct page *page,
+ else
+ len = PAGE_SIZE;
+
++ /* Should never happen but for bugs in other kernel subsystems */
++ if (!page_has_buffers(page)) {
++ ext4_warning_inode(inode,
++ "page %lu does not have buffers attached", page->index);
++ ClearPageDirty(page);
++ unlock_page(page);
++ return 0;
++ }
++
+ page_bufs = page_buffers(page);
+ /*
+ * We cannot do block allocation or other extent handling in this
+@@ -2614,6 +2604,22 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
+ wait_on_page_writeback(page);
+ BUG_ON(PageWriteback(page));
+
++ /*
++ * Should never happen but for buggy code in
++ * other subsystems that call
++ * set_page_dirty() without properly warning
++ * the file system first. See [1] for more
++ * information.
++ *
++ * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz
++ */
++ if (!page_has_buffers(page)) {
++ ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index);
++ ClearPageDirty(page);
++ unlock_page(page);
++ continue;
++ }
++
+ if (mpd->map.m_len == 0)
+ mpd->first_page = page->index;
+ mpd->next_page = page->index + 1;
+@@ -3933,12 +3939,14 @@ int ext4_break_layouts(struct inode *inode)
+ * Returns: 0 on success or negative on failure
+ */
+
+-int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
++int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
+ {
++ struct inode *inode = file_inode(file);
+ struct super_block *sb = inode->i_sb;
+ ext4_lblk_t first_block, stop_block;
+ struct address_space *mapping = inode->i_mapping;
+- loff_t first_block_offset, last_block_offset;
++ loff_t first_block_offset, last_block_offset, max_length;
++ struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
+ handle_t *handle;
+ unsigned int credits;
+ int ret = 0, ret2 = 0;
+@@ -3981,6 +3989,14 @@ int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
+ offset;
+ }
+
++ /*
++ * For punch hole the length + offset needs to be within one block
++ * before last range. Adjust the length if it goes beyond that limit.
++ */
++ max_length = sbi->s_bitmap_maxbytes - inode->i_sb->s_blocksize;
++ if (offset + length > max_length)
++ length = max_length - offset;
++
+ if (offset & (sb->s_blocksize - 1) ||
+ (offset + length) & (sb->s_blocksize - 1)) {
+ /*
+@@ -3996,6 +4012,10 @@ int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
+ /* Wait all existing dio workers, newcomers will block on i_mutex */
+ inode_dio_wait(inode);
+
++ ret = file_modified(file);
++ if (ret)
++ goto out_mutex;
++
+ /*
+ * Prevent page faults from reinstantiating pages we have released from
+ * page cache.
+@@ -4374,7 +4394,7 @@ has_buffer:
+ static int __ext4_get_inode_loc_noinmem(struct inode *inode,
+ struct ext4_iloc *iloc)
+ {
+- ext4_fsblk_t err_blk;
++ ext4_fsblk_t err_blk = 0;
+ int ret;
+
+ ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc, 0,
+@@ -4389,7 +4409,7 @@ static int __ext4_get_inode_loc_noinmem(struct inode *inode,
+
+ int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
+ {
+- ext4_fsblk_t err_blk;
++ ext4_fsblk_t err_blk = 0;
+ int ret;
+
+ /* We have all inode data except xattrs in memory here. */
+@@ -5416,8 +5436,7 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
+ ext4_fc_track_range(handle, inode,
+ (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
+ inode->i_sb->s_blocksize_bits,
+- (oldsize > 0 ? oldsize - 1 : 0) >>
+- inode->i_sb->s_blocksize_bits);
++ EXT_MAX_BLOCKS - 1);
+ else
+ ext4_fc_track_range(
+ handle, inode,
+@@ -5989,7 +6008,7 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val)
+ return PTR_ERR(handle);
+
+ ext4_fc_mark_ineligible(inode->i_sb,
+- EXT4_FC_REASON_JOURNAL_FLAG_CHANGE);
++ EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
+ err = ext4_mark_inode_dirty(handle, inode);
+ ext4_handle_sync(handle);
+ ext4_journal_stop(handle);
+diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
+index 606dee9e08a32..f61b59045c6d3 100644
+--- a/fs/ext4/ioctl.c
++++ b/fs/ext4/ioctl.c
+@@ -169,7 +169,7 @@ static long swap_inode_boot_loader(struct super_block *sb,
+ err = -EINVAL;
+ goto err_out;
+ }
+- ext4_fc_start_ineligible(sb, EXT4_FC_REASON_SWAP_BOOT);
++ ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_SWAP_BOOT, handle);
+
+ /* Protect extent tree against block allocations via delalloc */
+ ext4_double_down_write_data_sem(inode, inode_bl);
+@@ -252,7 +252,6 @@ revert:
+
+ err_out1:
+ ext4_journal_stop(handle);
+- ext4_fc_stop_ineligible(sb);
+ ext4_double_up_write_data_sem(inode, inode_bl);
+
+ err_out:
+@@ -1076,7 +1075,7 @@ mext_out:
+
+ err = ext4_resize_fs(sb, n_blocks_count);
+ if (EXT4_SB(sb)->s_journal) {
+- ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE);
++ ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL);
+ jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal);
+ err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0);
+ jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal);
+@@ -1117,8 +1116,6 @@ resizefs_out:
+ sizeof(range)))
+ return -EFAULT;
+
+- range.minlen = max((unsigned int)range.minlen,
+- q->limits.discard_granularity);
+ ret = ext4_trim_fs(sb, &range);
+ if (ret < 0)
+ return ret;
+diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
+index 72bfac2d6dce9..46608723b6ed7 100644
+--- a/fs/ext4/mballoc.c
++++ b/fs/ext4/mballoc.c
+@@ -1000,7 +1000,7 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac)
+ return 0;
+ if (ac->ac_criteria >= 2)
+ return 0;
+- if (ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))
++ if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))
+ return 0;
+ return 1;
+ }
+@@ -3899,69 +3899,95 @@ void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block,
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
+ ext4_group_t group;
+ ext4_grpblk_t blkoff;
+- int i, clen, err;
++ int i, err;
+ int already;
++ unsigned int clen, clen_changed, thisgrp_len;
+
+- clen = EXT4_B2C(sbi, len);
++ while (len > 0) {
++ ext4_get_group_no_and_offset(sb, block, &group, &blkoff);
+
+- ext4_get_group_no_and_offset(sb, block, &group, &blkoff);
+- bitmap_bh = ext4_read_block_bitmap(sb, group);
+- if (IS_ERR(bitmap_bh)) {
+- err = PTR_ERR(bitmap_bh);
+- bitmap_bh = NULL;
+- goto out_err;
+- }
++ /*
++ * Check to see if we are freeing blocks across a group
++ * boundary.
++ * In case of flex_bg, this can happen that (block, len) may
++ * span across more than one group. In that case we need to
++ * get the corresponding group metadata to work with.
++ * For this we have goto again loop.
++ */
++ thisgrp_len = min_t(unsigned int, (unsigned int)len,
++ EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));
++ clen = EXT4_NUM_B2C(sbi, thisgrp_len);
+
+- err = -EIO;
+- gdp = ext4_get_group_desc(sb, group, &gdp_bh);
+- if (!gdp)
+- goto out_err;
++ bitmap_bh = ext4_read_block_bitmap(sb, group);
++ if (IS_ERR(bitmap_bh)) {
++ err = PTR_ERR(bitmap_bh);
++ bitmap_bh = NULL;
++ break;
++ }
+
+- ext4_lock_group(sb, group);
+- already = 0;
+- for (i = 0; i < clen; i++)
+- if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) == !state)
+- already++;
++ err = -EIO;
++ gdp = ext4_get_group_desc(sb, group, &gdp_bh);
++ if (!gdp)
++ break;
+
+- if (state)
+- ext4_set_bits(bitmap_bh->b_data, blkoff, clen);
+- else
+- mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen);
+- if (ext4_has_group_desc_csum(sb) &&
+- (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {
+- gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
+- ext4_free_group_clusters_set(sb, gdp,
+- ext4_free_clusters_after_init(sb,
+- group, gdp));
+- }
+- if (state)
+- clen = ext4_free_group_clusters(sb, gdp) - clen + already;
+- else
+- clen = ext4_free_group_clusters(sb, gdp) + clen - already;
++ ext4_lock_group(sb, group);
++ already = 0;
++ for (i = 0; i < clen; i++)
++ if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) ==
++ !state)
++ already++;
++
++ clen_changed = clen - already;
++ if (state)
++ ext4_set_bits(bitmap_bh->b_data, blkoff, clen);
++ else
++ mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen);
++ if (ext4_has_group_desc_csum(sb) &&
++ (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {
++ gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
++ ext4_free_group_clusters_set(sb, gdp,
++ ext4_free_clusters_after_init(sb, group, gdp));
++ }
++ if (state)
++ clen = ext4_free_group_clusters(sb, gdp) - clen_changed;
++ else
++ clen = ext4_free_group_clusters(sb, gdp) + clen_changed;
+
+- ext4_free_group_clusters_set(sb, gdp, clen);
+- ext4_block_bitmap_csum_set(sb, group, gdp, bitmap_bh);
+- ext4_group_desc_csum_set(sb, group, gdp);
++ ext4_free_group_clusters_set(sb, gdp, clen);
++ ext4_block_bitmap_csum_set(sb, group, gdp, bitmap_bh);
++ ext4_group_desc_csum_set(sb, group, gdp);
+
+- ext4_unlock_group(sb, group);
++ ext4_unlock_group(sb, group);
+
+- if (sbi->s_log_groups_per_flex) {
+- ext4_group_t flex_group = ext4_flex_group(sbi, group);
++ if (sbi->s_log_groups_per_flex) {
++ ext4_group_t flex_group = ext4_flex_group(sbi, group);
++ struct flex_groups *fg = sbi_array_rcu_deref(sbi,
++ s_flex_groups, flex_group);
+
+- atomic64_sub(len,
+- &sbi_array_rcu_deref(sbi, s_flex_groups,
+- flex_group)->free_clusters);
++ if (state)
++ atomic64_sub(clen_changed, &fg->free_clusters);
++ else
++ atomic64_add(clen_changed, &fg->free_clusters);
++
++ }
++
++ err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh);
++ if (err)
++ break;
++ sync_dirty_buffer(bitmap_bh);
++ err = ext4_handle_dirty_metadata(NULL, NULL, gdp_bh);
++ sync_dirty_buffer(gdp_bh);
++ if (err)
++ break;
++
++ block += thisgrp_len;
++ len -= thisgrp_len;
++ brelse(bitmap_bh);
++ BUG_ON(len < 0);
+ }
+
+- err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh);
+ if (err)
+- goto out_err;
+- sync_dirty_buffer(bitmap_bh);
+- err = ext4_handle_dirty_metadata(NULL, NULL, gdp_bh);
+- sync_dirty_buffer(gdp_bh);
+-
+-out_err:
+- brelse(bitmap_bh);
++ brelse(bitmap_bh);
+ }
+
+ /*
+@@ -4814,7 +4840,7 @@ ext4_mb_release_group_pa(struct ext4_buddy *e4b,
+ */
+ static noinline_for_stack int
+ ext4_mb_discard_group_preallocations(struct super_block *sb,
+- ext4_group_t group, int needed)
++ ext4_group_t group, int *busy)
+ {
+ struct ext4_group_info *grp = ext4_get_group_info(sb, group);
+ struct buffer_head *bitmap_bh = NULL;
+@@ -4822,8 +4848,7 @@ ext4_mb_discard_group_preallocations(struct super_block *sb,
+ struct list_head list;
+ struct ext4_buddy e4b;
+ int err;
+- int busy = 0;
+- int free, free_total = 0;
++ int free = 0;
+
+ mb_debug(sb, "discard preallocation for group %u\n", group);
+ if (list_empty(&grp->bb_prealloc_list))
+@@ -4846,19 +4871,14 @@ ext4_mb_discard_group_preallocations(struct super_block *sb,
+ goto out_dbg;
+ }
+
+- if (needed == 0)
+- needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1;
+-
+ INIT_LIST_HEAD(&list);
+-repeat:
+- free = 0;
+ ext4_lock_group(sb, group);
+ list_for_each_entry_safe(pa, tmp,
+ &grp->bb_prealloc_list, pa_group_list) {
+ spin_lock(&pa->pa_lock);
+ if (atomic_read(&pa->pa_count)) {
+ spin_unlock(&pa->pa_lock);
+- busy = 1;
++ *busy = 1;
+ continue;
+ }
+ if (pa->pa_deleted) {
+@@ -4898,22 +4918,13 @@ repeat:
+ call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);
+ }
+
+- free_total += free;
+-
+- /* if we still need more blocks and some PAs were used, try again */
+- if (free_total < needed && busy) {
+- ext4_unlock_group(sb, group);
+- cond_resched();
+- busy = 0;
+- goto repeat;
+- }
+ ext4_unlock_group(sb, group);
+ ext4_mb_unload_buddy(&e4b);
+ put_bh(bitmap_bh);
+ out_dbg:
+ mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n",
+- free_total, group, grp->bb_free);
+- return free_total;
++ free, group, grp->bb_free);
++ return free;
+ }
+
+ /*
+@@ -5455,13 +5466,24 @@ static int ext4_mb_discard_preallocations(struct super_block *sb, int needed)
+ {
+ ext4_group_t i, ngroups = ext4_get_groups_count(sb);
+ int ret;
+- int freed = 0;
++ int freed = 0, busy = 0;
++ int retry = 0;
+
+ trace_ext4_mb_discard_preallocations(sb, needed);
++
++ if (needed == 0)
++ needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1;
++ repeat:
+ for (i = 0; i < ngroups && needed > 0; i++) {
+- ret = ext4_mb_discard_group_preallocations(sb, i, needed);
++ ret = ext4_mb_discard_group_preallocations(sb, i, &busy);
+ freed += ret;
+ needed -= ret;
++ cond_resched();
++ }
++
++ if (needed > 0 && busy && ++retry < 3) {
++ busy = 0;
++ goto repeat;
+ }
+
+ return freed;
+@@ -5757,7 +5779,8 @@ static ext4_fsblk_t ext4_mb_new_blocks_simple(handle_t *handle,
+ struct super_block *sb = ar->inode->i_sb;
+ ext4_group_t group;
+ ext4_grpblk_t blkoff;
+- int i = sb->s_blocksize;
++ ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb);
++ ext4_grpblk_t i = 0;
+ ext4_fsblk_t goal, block;
+ struct ext4_super_block *es = EXT4_SB(sb)->s_es;
+
+@@ -5779,19 +5802,26 @@ static ext4_fsblk_t ext4_mb_new_blocks_simple(handle_t *handle,
+ ext4_get_group_no_and_offset(sb,
+ max(ext4_group_first_block_no(sb, group), goal),
+ NULL, &blkoff);
+- i = mb_find_next_zero_bit(bitmap_bh->b_data, sb->s_blocksize,
++ while (1) {
++ i = mb_find_next_zero_bit(bitmap_bh->b_data, max,
+ blkoff);
++ if (i >= max)
++ break;
++ if (ext4_fc_replay_check_excluded(sb,
++ ext4_group_first_block_no(sb, group) + i)) {
++ blkoff = i + 1;
++ } else
++ break;
++ }
+ brelse(bitmap_bh);
+- if (i >= sb->s_blocksize)
+- continue;
+- if (ext4_fc_replay_check_excluded(sb,
+- ext4_group_first_block_no(sb, group) + i))
+- continue;
+- break;
++ if (i < max)
++ break;
+ }
+
+- if (group >= ext4_get_groups_count(sb) && i >= sb->s_blocksize)
++ if (group >= ext4_get_groups_count(sb) || i >= max) {
++ *errp = -ENOSPC;
+ return 0;
++ }
+
+ block = ext4_group_first_block_no(sb, group) + i;
+ ext4_mb_mark_bb(sb, block, 1, 1);
+@@ -6405,6 +6435,7 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group,
+ */
+ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
+ {
++ struct request_queue *q = bdev_get_queue(sb->s_bdev);
+ struct ext4_group_info *grp;
+ ext4_group_t group, first_group, last_group;
+ ext4_grpblk_t cnt = 0, first_cluster, last_cluster;
+@@ -6423,6 +6454,13 @@ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
+ start >= max_blks ||
+ range->len < sb->s_blocksize)
+ return -EINVAL;
++ /* No point to try to trim less than discard granularity */
++ if (range->minlen < q->limits.discard_granularity) {
++ minlen = EXT4_NUM_B2C(EXT4_SB(sb),
++ q->limits.discard_granularity >> sb->s_blocksize_bits);
++ if (minlen > EXT4_CLUSTERS_PER_GROUP(sb))
++ goto out;
++ }
+ if (end >= max_blks)
+ end = max_blks - 1;
+ if (end <= first_data_blk)
+diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c
+index 7e0b4f81c6c06..ff8916e1d38e9 100644
+--- a/fs/ext4/migrate.c
++++ b/fs/ext4/migrate.c
+@@ -437,12 +437,12 @@ int ext4_ext_migrate(struct inode *inode)
+ percpu_down_write(&sbi->s_writepages_rwsem);
+
+ /*
+- * Worst case we can touch the allocation bitmaps, a bgd
+- * block, and a block to link in the orphan list. We do need
+- * need to worry about credits for modifying the quota inode.
++ * Worst case we can touch the allocation bitmaps and a block
++ * group descriptor block. We do need need to worry about
++ * credits for modifying the quota inode.
+ */
+ handle = ext4_journal_start(inode, EXT4_HT_MIGRATE,
+- 4 + EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb));
++ 3 + EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb));
+
+ if (IS_ERR(handle)) {
+ retval = PTR_ERR(handle);
+@@ -459,6 +459,13 @@ int ext4_ext_migrate(struct inode *inode)
+ ext4_journal_stop(handle);
+ goto out_unlock;
+ }
++ /*
++ * Use the correct seed for checksum (i.e. the seed from 'inode'). This
++ * is so that the metadata blocks will have the correct checksum after
++ * the migration.
++ */
++ ei = EXT4_I(inode);
++ EXT4_I(tmp_inode)->i_csum_seed = ei->i_csum_seed;
+ i_size_write(tmp_inode, i_size_read(inode));
+ /*
+ * Set the i_nlink to zero so it will be deleted later
+@@ -467,7 +474,6 @@ int ext4_ext_migrate(struct inode *inode)
+ clear_nlink(tmp_inode);
+
+ ext4_ext_tree_init(handle, tmp_inode);
+- ext4_orphan_add(handle, tmp_inode);
+ ext4_journal_stop(handle);
+
+ /*
+@@ -492,17 +498,10 @@ int ext4_ext_migrate(struct inode *inode)
+
+ handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 1);
+ if (IS_ERR(handle)) {
+- /*
+- * It is impossible to update on-disk structures without
+- * a handle, so just rollback in-core changes and live other
+- * work to orphan_list_cleanup()
+- */
+- ext4_orphan_del(NULL, tmp_inode);
+ retval = PTR_ERR(handle);
+ goto out_tmp_inode;
+ }
+
+- ei = EXT4_I(inode);
+ i_data = ei->i_data;
+ memset(&lb, 0, sizeof(lb));
+
+diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
+index da7698341d7d3..19c620118e628 100644
+--- a/fs/ext4/namei.c
++++ b/fs/ext4/namei.c
+@@ -1466,10 +1466,10 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size,
+
+ de = (struct ext4_dir_entry_2 *)search_buf;
+ dlimit = search_buf + buf_size;
+- while ((char *) de < dlimit) {
++ while ((char *) de < dlimit - EXT4_BASE_DIR_LEN) {
+ /* this code is executed quadratically often */
+ /* do minimal checking `by hand' */
+- if ((char *) de + de->name_len <= dlimit &&
++ if (de->name + de->name_len <= dlimit &&
+ ext4_match(dir, fname, de)) {
+ /* found a match - just to be sure, do
+ * a full check */
+@@ -2997,14 +2997,14 @@ bool ext4_empty_dir(struct inode *inode)
+ if (inode->i_size < ext4_dir_rec_len(1, NULL) +
+ ext4_dir_rec_len(2, NULL)) {
+ EXT4_ERROR_INODE(inode, "invalid size");
+- return true;
++ return false;
+ }
+ /* The first directory block must not be a hole,
+ * so treat it as DIRENT_HTREE
+ */
+ bh = ext4_read_dirblock(inode, 0, DIRENT_HTREE);
+ if (IS_ERR(bh))
+- return true;
++ return false;
+
+ de = (struct ext4_dir_entry_2 *) bh->b_data;
+ if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size,
+@@ -3012,7 +3012,7 @@ bool ext4_empty_dir(struct inode *inode)
+ le32_to_cpu(de->inode) != inode->i_ino || strcmp(".", de->name)) {
+ ext4_warning_inode(inode, "directory missing '.'");
+ brelse(bh);
+- return true;
++ return false;
+ }
+ offset = ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
+ de = ext4_next_entry(de, sb->s_blocksize);
+@@ -3021,7 +3021,7 @@ bool ext4_empty_dir(struct inode *inode)
+ le32_to_cpu(de->inode) == 0 || strcmp("..", de->name)) {
+ ext4_warning_inode(inode, "directory missing '..'");
+ brelse(bh);
+- return true;
++ return false;
+ }
+ offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
+ while (offset < inode->i_size) {
+@@ -3035,7 +3035,7 @@ bool ext4_empty_dir(struct inode *inode)
+ continue;
+ }
+ if (IS_ERR(bh))
+- return true;
++ return false;
+ }
+ de = (struct ext4_dir_entry_2 *) (bh->b_data +
+ (offset & (sb->s_blocksize - 1)));
+@@ -3889,7 +3889,7 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir,
+ * dirents in directories.
+ */
+ ext4_fc_mark_ineligible(old.inode->i_sb,
+- EXT4_FC_REASON_RENAME_DIR);
++ EXT4_FC_REASON_RENAME_DIR, handle);
+ } else {
+ if (new.inode)
+ ext4_fc_track_unlink(handle, new.dentry);
+@@ -4049,7 +4049,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
+ if (unlikely(retval))
+ goto end_rename;
+ ext4_fc_mark_ineligible(new.inode->i_sb,
+- EXT4_FC_REASON_CROSS_RENAME);
++ EXT4_FC_REASON_CROSS_RENAME, handle);
+ if (old.dir_bh) {
+ retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
+ if (retval)
+diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
+index f038d578d8d8f..18977ff8e4939 100644
+--- a/fs/ext4/page-io.c
++++ b/fs/ext4/page-io.c
+@@ -134,8 +134,10 @@ static void ext4_finish_bio(struct bio *bio)
+ continue;
+ }
+ clear_buffer_async_write(bh);
+- if (bio->bi_status)
++ if (bio->bi_status) {
++ set_buffer_write_io_error(bh);
+ buffer_io_error(bh);
++ }
+ } while ((bh = bh->b_this_page) != head);
+ spin_unlock_irqrestore(&head->b_uptodate_lock, flags);
+ if (!under_io) {
+diff --git a/fs/ext4/super.c b/fs/ext4/super.c
+index 88d5d274a8684..d12f11c6fbf25 100644
+--- a/fs/ext4/super.c
++++ b/fs/ext4/super.c
+@@ -1167,20 +1167,25 @@ static void ext4_put_super(struct super_block *sb)
+ int aborted = 0;
+ int i, err;
+
+- ext4_unregister_li_request(sb);
+- ext4_quota_off_umount(sb);
+-
+- flush_work(&sbi->s_error_work);
+- destroy_workqueue(sbi->rsv_conversion_wq);
+- ext4_release_orphan_info(sb);
+-
+ /*
+ * Unregister sysfs before destroying jbd2 journal.
+ * Since we could still access attr_journal_task attribute via sysfs
+ * path which could have sbi->s_journal->j_task as NULL
++ * Unregister sysfs before flush sbi->s_error_work.
++ * Since user may read /proc/fs/ext4/xx/mb_groups during umount, If
++ * read metadata verify failed then will queue error work.
++ * flush_stashed_error_work will call start_this_handle may trigger
++ * BUG_ON.
+ */
+ ext4_unregister_sysfs(sb);
+
++ ext4_unregister_li_request(sb);
++ ext4_quota_off_umount(sb);
++
++ flush_work(&sbi->s_error_work);
++ destroy_workqueue(sbi->rsv_conversion_wq);
++ ext4_release_orphan_info(sb);
++
+ if (sbi->s_journal) {
+ aborted = is_journal_aborted(sbi->s_journal);
+ err = jbd2_journal_destroy(sbi->s_journal);
+@@ -3263,9 +3268,9 @@ static int ext4_run_li_request(struct ext4_li_request *elr)
+ struct super_block *sb = elr->lr_super;
+ ext4_group_t ngroups = EXT4_SB(sb)->s_groups_count;
+ ext4_group_t group = elr->lr_next_group;
+- unsigned long timeout = 0;
+ unsigned int prefetch_ios = 0;
+ int ret = 0;
++ u64 start_time;
+
+ if (elr->lr_mode == EXT4_LI_MODE_PREFETCH_BBITMAP) {
+ elr->lr_next_group = ext4_mb_prefetch(sb, group,
+@@ -3302,14 +3307,13 @@ static int ext4_run_li_request(struct ext4_li_request *elr)
+ ret = 1;
+
+ if (!ret) {
+- timeout = jiffies;
++ start_time = ktime_get_real_ns();
+ ret = ext4_init_inode_table(sb, group,
+ elr->lr_timeout ? 0 : 1);
+ trace_ext4_lazy_itable_init(sb, group);
+ if (elr->lr_timeout == 0) {
+- timeout = (jiffies - timeout) *
+- EXT4_SB(elr->lr_super)->s_li_wait_mult;
+- elr->lr_timeout = timeout;
++ elr->lr_timeout = nsecs_to_jiffies((ktime_get_real_ns() - start_time) *
++ EXT4_SB(elr->lr_super)->s_li_wait_mult);
+ }
+ elr->lr_next_sched = jiffies + elr->lr_timeout;
+ elr->lr_next_group = group + 1;
+@@ -3698,9 +3702,11 @@ static int count_overhead(struct super_block *sb, ext4_group_t grp,
+ ext4_fsblk_t first_block, last_block, b;
+ ext4_group_t i, ngroups = ext4_get_groups_count(sb);
+ int s, j, count = 0;
++ int has_super = ext4_bg_has_super(sb, grp);
+
+ if (!ext4_has_feature_bigalloc(sb))
+- return (ext4_bg_has_super(sb, grp) + ext4_bg_num_gdb(sb, grp) +
++ return (has_super + ext4_bg_num_gdb(sb, grp) +
++ (has_super ? le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) : 0) +
+ sbi->s_itb_per_group + 2);
+
+ first_block = le32_to_cpu(sbi->s_es->s_first_data_block) +
+@@ -4614,14 +4620,13 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
+
+ /* Initialize fast commit stuff */
+ atomic_set(&sbi->s_fc_subtid, 0);
+- atomic_set(&sbi->s_fc_ineligible_updates, 0);
+ INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]);
+ INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]);
+ INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]);
+ INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]);
+ sbi->s_fc_bytes = 0;
+ ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
+- ext4_clear_mount_flag(sb, EXT4_MF_FC_COMMITTING);
++ sbi->s_fc_ineligible_tid = 0;
+ spin_lock_init(&sbi->s_fc_lock);
+ memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats));
+ sbi->s_fc_replay_state.fc_regions = NULL;
+@@ -4788,9 +4793,18 @@ no_journal:
+ * Get the # of file system overhead blocks from the
+ * superblock if present.
+ */
+- if (es->s_overhead_clusters)
+- sbi->s_overhead = le32_to_cpu(es->s_overhead_clusters);
+- else {
++ sbi->s_overhead = le32_to_cpu(es->s_overhead_clusters);
++ /* ignore the precalculated value if it is ridiculous */
++ if (sbi->s_overhead > ext4_blocks_count(es))
++ sbi->s_overhead = 0;
++ /*
++ * If the bigalloc feature is not enabled recalculating the
++ * overhead doesn't take long, so we might as well just redo
++ * it to make sure we are using the correct value.
++ */
++ if (!ext4_has_feature_bigalloc(sb))
++ sbi->s_overhead = 0;
++ if (sbi->s_overhead == 0) {
+ err = ext4_calculate_overhead(sb);
+ if (err)
+ goto failed_mount_wq;
+@@ -6267,10 +6281,7 @@ static int ext4_quota_on(struct super_block *sb, int type, int format_id,
+
+ lockdep_set_quota_inode(path->dentry->d_inode, I_DATA_SEM_QUOTA);
+ err = dquot_quota_on(sb, type, format_id, path);
+- if (err) {
+- lockdep_set_quota_inode(path->dentry->d_inode,
+- I_DATA_SEM_NORMAL);
+- } else {
++ if (!err) {
+ struct inode *inode = d_inode(path->dentry);
+ handle_t *handle;
+
+@@ -6290,7 +6301,12 @@ static int ext4_quota_on(struct super_block *sb, int type, int format_id,
+ ext4_journal_stop(handle);
+ unlock_inode:
+ inode_unlock(inode);
++ if (err)
++ dquot_quota_off(sb, type);
+ }
++ if (err)
++ lockdep_set_quota_inode(path->dentry->d_inode,
++ I_DATA_SEM_NORMAL);
+ return err;
+ }
+
+@@ -6353,8 +6369,19 @@ int ext4_enable_quotas(struct super_block *sb)
+ "Failed to enable quota tracking "
+ "(type=%d, err=%d). Please run "
+ "e2fsck to fix.", type, err);
+- for (type--; type >= 0; type--)
++ for (type--; type >= 0; type--) {
++ struct inode *inode;
++
++ inode = sb_dqopt(sb)->files[type];
++ if (inode)
++ inode = igrab(inode);
+ dquot_quota_off(sb, type);
++ if (inode) {
++ lockdep_set_quota_inode(inode,
++ I_DATA_SEM_NORMAL);
++ iput(inode);
++ }
++ }
+
+ return err;
+ }
+@@ -6458,7 +6485,7 @@ static ssize_t ext4_quota_write(struct super_block *sb, int type,
+ struct buffer_head *bh;
+ handle_t *handle = journal_current_handle();
+
+- if (EXT4_SB(sb)->s_journal && !handle) {
++ if (!handle) {
+ ext4_msg(sb, KERN_WARNING, "Quota write (off=%llu, len=%llu)"
+ " cancelled because transaction is not started",
+ (unsigned long long)off, (unsigned long long)len);
+@@ -6641,6 +6668,7 @@ static int __init ext4_init_fs(void)
+ out:
+ unregister_as_ext2();
+ unregister_as_ext3();
++ ext4_fc_destroy_dentry_cache();
+ out05:
+ destroy_inodecache();
+ out1:
+@@ -6667,6 +6695,7 @@ static void __exit ext4_exit_fs(void)
+ unregister_as_ext2();
+ unregister_as_ext3();
+ unregister_filesystem(&ext4_fs_type);
++ ext4_fc_destroy_dentry_cache();
+ destroy_inodecache();
+ ext4_exit_mballoc();
+ ext4_exit_sysfs();
+diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c
+index 1e0fc1ed845bf..0423253490986 100644
+--- a/fs/ext4/xattr.c
++++ b/fs/ext4/xattr.c
+@@ -2408,7 +2408,7 @@ retry_inode:
+ if (IS_SYNC(inode))
+ ext4_handle_sync(handle);
+ }
+- ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR);
++ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle);
+
+ cleanup:
+ brelse(is.iloc.bh);
+@@ -2486,7 +2486,7 @@ retry:
+ if (error == 0)
+ error = error2;
+ }
+- ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR);
++ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, NULL);
+
+ return error;
+ }
+@@ -2920,7 +2920,7 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
+ error);
+ goto cleanup;
+ }
+- ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR);
++ ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle);
+ }
+ error = 0;
+ cleanup:
+diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
+index 83e9bc0f91ffd..1ff32926e1999 100644
+--- a/fs/f2fs/checkpoint.c
++++ b/fs/f2fs/checkpoint.c
+@@ -867,6 +867,7 @@ static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
+ struct page *cp_page_1 = NULL, *cp_page_2 = NULL;
+ struct f2fs_checkpoint *cp_block = NULL;
+ unsigned long long cur_version = 0, pre_version = 0;
++ unsigned int cp_blocks;
+ int err;
+
+ err = get_checkpoint_version(sbi, cp_addr, &cp_block,
+@@ -874,15 +875,16 @@ static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
+ if (err)
+ return NULL;
+
+- if (le32_to_cpu(cp_block->cp_pack_total_block_count) >
+- sbi->blocks_per_seg) {
++ cp_blocks = le32_to_cpu(cp_block->cp_pack_total_block_count);
++
++ if (cp_blocks > sbi->blocks_per_seg || cp_blocks <= F2FS_CP_PACKS) {
+ f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u",
+ le32_to_cpu(cp_block->cp_pack_total_block_count));
+ goto invalid_cp;
+ }
+ pre_version = *version;
+
+- cp_addr += le32_to_cpu(cp_block->cp_pack_total_block_count) - 1;
++ cp_addr += cp_blocks - 1;
+ err = get_checkpoint_version(sbi, cp_addr, &cp_block,
+ &cp_page_2, version);
+ if (err)
+@@ -1162,7 +1164,8 @@ static bool __need_flush_quota(struct f2fs_sb_info *sbi)
+ if (!is_journalled_quota(sbi))
+ return false;
+
+- down_write(&sbi->quota_sem);
++ if (!down_write_trylock(&sbi->quota_sem))
++ return true;
+ if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
+ ret = false;
+ } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
+@@ -1304,8 +1307,8 @@ static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
+ unsigned long flags;
+
+ if (cpc->reason & CP_UMOUNT) {
+- if (le32_to_cpu(ckpt->cp_pack_total_block_count) >
+- sbi->blocks_per_seg - NM_I(sbi)->nat_bits_blocks) {
++ if (le32_to_cpu(ckpt->cp_pack_total_block_count) +
++ NM_I(sbi)->nat_bits_blocks > sbi->blocks_per_seg) {
+ clear_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
+ f2fs_notice(sbi, "Disable nat_bits due to no space");
+ } else if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG) &&
+diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
+index c1bf9ad4c2207..6adf047259546 100644
+--- a/fs/f2fs/compress.c
++++ b/fs/f2fs/compress.c
+@@ -312,10 +312,9 @@ static int lz4_decompress_pages(struct decompress_io_ctx *dic)
+ }
+
+ if (ret != PAGE_SIZE << dic->log_cluster_size) {
+- printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
++ printk_ratelimited("%sF2FS-fs (%s): lz4 invalid ret:%d, "
+ "expected:%lu\n", KERN_ERR,
+- F2FS_I_SB(dic->inode)->sb->s_id,
+- dic->rlen,
++ F2FS_I_SB(dic->inode)->sb->s_id, ret,
+ PAGE_SIZE << dic->log_cluster_size);
+ return -EIO;
+ }
+@@ -1448,25 +1447,38 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc,
+ enum iostat_type io_type)
+ {
+ struct address_space *mapping = cc->inode->i_mapping;
+- int _submitted, compr_blocks, ret;
+- int i = -1, err = 0;
++ int _submitted, compr_blocks, ret, i;
+
+ compr_blocks = f2fs_compressed_blocks(cc);
+- if (compr_blocks < 0) {
+- err = compr_blocks;
+- goto out_err;
++
++ for (i = 0; i < cc->cluster_size; i++) {
++ if (!cc->rpages[i])
++ continue;
++
++ redirty_page_for_writepage(wbc, cc->rpages[i]);
++ unlock_page(cc->rpages[i]);
+ }
+
++ if (compr_blocks < 0)
++ return compr_blocks;
++
+ for (i = 0; i < cc->cluster_size; i++) {
+ if (!cc->rpages[i])
+ continue;
+ retry_write:
++ lock_page(cc->rpages[i]);
++
+ if (cc->rpages[i]->mapping != mapping) {
++continue_unlock:
+ unlock_page(cc->rpages[i]);
+ continue;
+ }
+
+- BUG_ON(!PageLocked(cc->rpages[i]));
++ if (!PageDirty(cc->rpages[i]))
++ goto continue_unlock;
++
++ if (!clear_page_dirty_for_io(cc->rpages[i]))
++ goto continue_unlock;
+
+ ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
+ NULL, NULL, wbc, io_type,
+@@ -1481,26 +1493,15 @@ retry_write:
+ * avoid deadlock caused by cluster update race
+ * from foreground operation.
+ */
+- if (IS_NOQUOTA(cc->inode)) {
+- err = 0;
+- goto out_err;
+- }
++ if (IS_NOQUOTA(cc->inode))
++ return 0;
+ ret = 0;
+ cond_resched();
+ congestion_wait(BLK_RW_ASYNC,
+ DEFAULT_IO_TIMEOUT);
+- lock_page(cc->rpages[i]);
+-
+- if (!PageDirty(cc->rpages[i])) {
+- unlock_page(cc->rpages[i]);
+- continue;
+- }
+-
+- clear_page_dirty_for_io(cc->rpages[i]);
+ goto retry_write;
+ }
+- err = ret;
+- goto out_err;
++ return ret;
+ }
+
+ *submitted += _submitted;
+@@ -1509,14 +1510,6 @@ retry_write:
+ f2fs_balance_fs(F2FS_M_SB(mapping), true);
+
+ return 0;
+-out_err:
+- for (++i; i < cc->cluster_size; i++) {
+- if (!cc->rpages[i])
+- continue;
+- redirty_page_for_writepage(wbc, cc->rpages[i]);
+- unlock_page(cc->rpages[i]);
+- }
+- return err;
+ }
+
+ int f2fs_write_multi_pages(struct compress_ctx *cc,
+@@ -1530,6 +1523,7 @@ int f2fs_write_multi_pages(struct compress_ctx *cc,
+ if (cluster_may_compress(cc)) {
+ err = f2fs_compress_pages(cc);
+ if (err == -EAGAIN) {
++ add_compr_block_stat(cc->inode, cc->cluster_size);
+ goto write;
+ } else if (err) {
+ f2fs_put_rpages_wbc(cc, wbc, true, 1);
+diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
+index f4fd6c246c9a9..4cf522120cb1a 100644
+--- a/fs/f2fs/data.c
++++ b/fs/f2fs/data.c
+@@ -2564,6 +2564,11 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
+ {
+ struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+
++ /* The below cases were checked when setting it. */
++ if (f2fs_is_pinned_file(inode))
++ return false;
++ if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK))
++ return true;
+ if (f2fs_lfs_mode(sbi))
+ return true;
+ if (S_ISDIR(inode->i_mode))
+@@ -2572,8 +2577,6 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
+ return true;
+ if (f2fs_is_atomic_file(inode))
+ return true;
+- if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
+- return true;
+
+ /* swap file is migrating in aligned write mode */
+ if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
+@@ -3211,8 +3214,12 @@ static int __f2fs_write_data_pages(struct address_space *mapping,
+ /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
+ if (wbc->sync_mode == WB_SYNC_ALL)
+ atomic_inc(&sbi->wb_sync_req[DATA]);
+- else if (atomic_read(&sbi->wb_sync_req[DATA]))
++ else if (atomic_read(&sbi->wb_sync_req[DATA])) {
++ /* to avoid potential deadlock */
++ if (current->plug)
++ blk_finish_plug(current->plug);
+ goto skip_write;
++ }
+
+ if (__should_serialize_io(inode, wbc)) {
+ mutex_lock(&sbi->writepages);
+@@ -3403,7 +3410,7 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping,
+
+ *fsdata = NULL;
+
+- if (len == PAGE_SIZE)
++ if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode)))
+ goto repeat;
+
+ ret = f2fs_prepare_compress_overwrite(inode, pagep,
+diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
+index 8c50518475a99..b449c7a372a4b 100644
+--- a/fs/f2fs/debug.c
++++ b/fs/f2fs/debug.c
+@@ -21,7 +21,7 @@
+ #include "gc.h"
+
+ static LIST_HEAD(f2fs_stat_list);
+-static DEFINE_MUTEX(f2fs_stat_mutex);
++static DEFINE_RAW_SPINLOCK(f2fs_stat_lock);
+ #ifdef CONFIG_DEBUG_FS
+ static struct dentry *f2fs_debugfs_root;
+ #endif
+@@ -338,14 +338,16 @@ static char *s_flag[] = {
+ [SBI_QUOTA_SKIP_FLUSH] = " quota_skip_flush",
+ [SBI_QUOTA_NEED_REPAIR] = " quota_need_repair",
+ [SBI_IS_RESIZEFS] = " resizefs",
++ [SBI_IS_FREEZING] = " freezefs",
+ };
+
+ static int stat_show(struct seq_file *s, void *v)
+ {
+ struct f2fs_stat_info *si;
+ int i = 0, j = 0;
++ unsigned long flags;
+
+- mutex_lock(&f2fs_stat_mutex);
++ raw_spin_lock_irqsave(&f2fs_stat_lock, flags);
+ list_for_each_entry(si, &f2fs_stat_list, stat_list) {
+ update_general_status(si->sbi);
+
+@@ -573,7 +575,7 @@ static int stat_show(struct seq_file *s, void *v)
+ seq_printf(s, " - paged : %llu KB\n",
+ si->page_mem >> 10);
+ }
+- mutex_unlock(&f2fs_stat_mutex);
++ raw_spin_unlock_irqrestore(&f2fs_stat_lock, flags);
+ return 0;
+ }
+
+@@ -584,6 +586,7 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi)
+ {
+ struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
+ struct f2fs_stat_info *si;
++ unsigned long flags;
+ int i;
+
+ si = f2fs_kzalloc(sbi, sizeof(struct f2fs_stat_info), GFP_KERNEL);
+@@ -619,9 +622,9 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi)
+ atomic_set(&sbi->max_aw_cnt, 0);
+ atomic_set(&sbi->max_vw_cnt, 0);
+
+- mutex_lock(&f2fs_stat_mutex);
++ raw_spin_lock_irqsave(&f2fs_stat_lock, flags);
+ list_add_tail(&si->stat_list, &f2fs_stat_list);
+- mutex_unlock(&f2fs_stat_mutex);
++ raw_spin_unlock_irqrestore(&f2fs_stat_lock, flags);
+
+ return 0;
+ }
+@@ -629,10 +632,11 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi)
+ void f2fs_destroy_stats(struct f2fs_sb_info *sbi)
+ {
+ struct f2fs_stat_info *si = F2FS_STAT(sbi);
++ unsigned long flags;
+
+- mutex_lock(&f2fs_stat_mutex);
++ raw_spin_lock_irqsave(&f2fs_stat_lock, flags);
+ list_del(&si->stat_list);
+- mutex_unlock(&f2fs_stat_mutex);
++ raw_spin_unlock_irqrestore(&f2fs_stat_lock, flags);
+
+ kfree(si);
+ }
+diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
+index b339ae89c1ad1..0a0fa1a64d06e 100644
+--- a/fs/f2fs/f2fs.h
++++ b/fs/f2fs/f2fs.h
+@@ -1012,6 +1012,7 @@ struct f2fs_sm_info {
+ unsigned int segment_count; /* total # of segments */
+ unsigned int main_segments; /* # of segments in main area */
+ unsigned int reserved_segments; /* # of reserved segments */
++ unsigned int additional_reserved_segments;/* reserved segs for IO align feature */
+ unsigned int ovp_segments; /* # of overprovision segments */
+
+ /* a threshold to reclaim prefree segments */
+@@ -1252,6 +1253,7 @@ enum {
+ SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */
+ SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */
+ SBI_IS_RESIZEFS, /* resizefs is in process */
++ SBI_IS_FREEZING, /* freezefs is in process */
+ };
+
+ enum {
+@@ -2184,6 +2186,11 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
+
+ if (!__allow_reserved_blocks(sbi, inode, true))
+ avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
++
++ if (F2FS_IO_ALIGNED(sbi))
++ avail_user_block_count -= sbi->blocks_per_seg *
++ SM_I(sbi)->additional_reserved_segments;
++
+ if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
+ if (avail_user_block_count > sbi->unusable_block_count)
+ avail_user_block_count -= sbi->unusable_block_count;
+@@ -2430,6 +2437,11 @@ static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
+
+ if (!__allow_reserved_blocks(sbi, inode, false))
+ valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
++
++ if (F2FS_IO_ALIGNED(sbi))
++ valid_block_count += sbi->blocks_per_seg *
++ SM_I(sbi)->additional_reserved_segments;
++
+ user_block_count = sbi->user_block_count;
+ if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
+ user_block_count -= sbi->unusable_block_count;
+@@ -4152,8 +4164,7 @@ static inline bool f2fs_disable_compressed_file(struct inode *inode)
+
+ if (!f2fs_compressed_file(inode))
+ return true;
+- if (S_ISREG(inode->i_mode) &&
+- (get_dirty_pages(inode) || atomic_read(&fi->i_compr_blocks)))
++ if (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))
+ return false;
+
+ fi->i_flags &= ~F2FS_COMPR_FL;
+diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
+index 9c8ef33bd8d32..8ef92719c6799 100644
+--- a/fs/f2fs/file.c
++++ b/fs/f2fs/file.c
+@@ -2002,7 +2002,10 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)
+
+ inode_lock(inode);
+
+- f2fs_disable_compressed_file(inode);
++ if (!f2fs_disable_compressed_file(inode)) {
++ ret = -EINVAL;
++ goto out;
++ }
+
+ if (f2fs_is_atomic_file(inode)) {
+ if (is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST))
+@@ -3143,17 +3146,17 @@ static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg)
+
+ inode_lock(inode);
+
+- if (f2fs_should_update_outplace(inode, NULL)) {
+- ret = -EINVAL;
+- goto out;
+- }
+-
+ if (!pin) {
+ clear_inode_flag(inode, FI_PIN_FILE);
+ f2fs_i_gc_failures_write(inode, 0);
+ goto done;
+ }
+
++ if (f2fs_should_update_outplace(inode, NULL)) {
++ ret = -EINVAL;
++ goto out;
++ }
++
+ if (f2fs_pin_file_control(inode, false)) {
+ ret = -EAGAIN;
+ goto out;
+@@ -4276,7 +4279,7 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ size_t target_size = 0;
+ int err;
+
+- if (iov_iter_fault_in_readable(from, iov_iter_count(from)))
++ if (fault_in_iov_iter_readable(from, iov_iter_count(from)))
+ set_inode_flag(inode, FI_NO_PREALLOC);
+
+ if ((iocb->ki_flags & IOCB_NOWAIT)) {
+diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
+index 77391e3b7d68f..ceb5fc6b40398 100644
+--- a/fs/f2fs/gc.c
++++ b/fs/f2fs/gc.c
+@@ -1023,6 +1023,11 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
+ set_sbi_flag(sbi, SBI_NEED_FSCK);
+ }
+
++ if (f2fs_check_nid_range(sbi, dni->ino)) {
++ f2fs_put_page(node_page, 1);
++ return false;
++ }
++
+ *nofs = ofs_of_node(node_page);
+ source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node);
+ f2fs_put_page(node_page, 1);
+@@ -1036,7 +1041,7 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
+ if (!test_and_set_bit(segno, SIT_I(sbi)->invalid_segmap)) {
+ f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u",
+ blkaddr, source_blkaddr, segno);
+- f2fs_bug_on(sbi, 1);
++ set_sbi_flag(sbi, SBI_NEED_FSCK);
+ }
+ }
+ #endif
+@@ -1454,7 +1459,8 @@ next_step:
+
+ if (phase == 3) {
+ inode = f2fs_iget(sb, dni.ino);
+- if (IS_ERR(inode) || is_bad_inode(inode))
++ if (IS_ERR(inode) || is_bad_inode(inode) ||
++ special_file(inode->i_mode))
+ continue;
+
+ if (!down_write_trylock(
+diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
+index 9141147b5bb00..2272000fb10b5 100644
+--- a/fs/f2fs/inode.c
++++ b/fs/f2fs/inode.c
+@@ -516,6 +516,11 @@ make_now:
+ } else if (ino == F2FS_COMPRESS_INO(sbi)) {
+ #ifdef CONFIG_F2FS_FS_COMPRESSION
+ inode->i_mapping->a_ops = &f2fs_compress_aops;
++ /*
++ * generic_error_remove_page only truncates pages of regular
++ * inode
++ */
++ inode->i_mode |= S_IFREG;
+ #endif
+ mapping_set_gfp_mask(inode->i_mapping,
+ GFP_NOFS | __GFP_HIGHMEM | __GFP_MOVABLE);
+@@ -527,7 +532,7 @@ make_now:
+ inode->i_op = &f2fs_dir_inode_operations;
+ inode->i_fop = &f2fs_dir_operations;
+ inode->i_mapping->a_ops = &f2fs_dblock_aops;
+- inode_nohighmem(inode);
++ mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
+ } else if (S_ISLNK(inode->i_mode)) {
+ if (file_is_encrypt(inode))
+ inode->i_op = &f2fs_encrypted_symlink_inode_operations;
+@@ -764,7 +769,8 @@ void f2fs_evict_inode(struct inode *inode)
+ f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
+ f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
+
+- sb_start_intwrite(inode->i_sb);
++ if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING))
++ sb_start_intwrite(inode->i_sb);
+ set_inode_flag(inode, FI_NO_ALLOC);
+ i_size_write(inode, 0);
+ retry:
+@@ -795,7 +801,8 @@ retry:
+ if (dquot_initialize_needed(inode))
+ set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
+ }
+- sb_end_intwrite(inode->i_sb);
++ if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING))
++ sb_end_intwrite(inode->i_sb);
+ no_delete:
+ dquot_drop(inode);
+
+@@ -871,6 +878,7 @@ void f2fs_handle_failed_inode(struct inode *inode)
+ err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
+ if (err) {
+ set_sbi_flag(sbi, SBI_NEED_FSCK);
++ set_inode_flag(inode, FI_FREE_NID);
+ f2fs_warn(sbi, "May loss orphan inode, run fsck to fix.");
+ goto out;
+ }
+diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
+index 9c528e583c9d5..ae0838001480a 100644
+--- a/fs/f2fs/namei.c
++++ b/fs/f2fs/namei.c
+@@ -757,7 +757,7 @@ static int f2fs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
+ inode->i_op = &f2fs_dir_inode_operations;
+ inode->i_fop = &f2fs_dir_operations;
+ inode->i_mapping->a_ops = &f2fs_dblock_aops;
+- inode_nohighmem(inode);
++ mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
+
+ set_inode_flag(inode, FI_INC_LINK);
+ f2fs_lock_op(sbi);
+diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
+index e863136081b47..69c6bcaf5aae8 100644
+--- a/fs/f2fs/node.c
++++ b/fs/f2fs/node.c
+@@ -1443,6 +1443,7 @@ page_hit:
+ nid, nid_of_node(page), ino_of_node(page),
+ ofs_of_node(page), cpver_of_node(page),
+ next_blkaddr_of_node(page));
++ set_sbi_flag(sbi, SBI_NEED_FSCK);
+ err = -EINVAL;
+ out_err:
+ ClearPageUptodate(page);
+@@ -2105,8 +2106,12 @@ static int f2fs_write_node_pages(struct address_space *mapping,
+
+ if (wbc->sync_mode == WB_SYNC_ALL)
+ atomic_inc(&sbi->wb_sync_req[NODE]);
+- else if (atomic_read(&sbi->wb_sync_req[NODE]))
++ else if (atomic_read(&sbi->wb_sync_req[NODE])) {
++ /* to avoid potential deadlock */
++ if (current->plug)
++ blk_finish_plug(current->plug);
+ goto skip_write;
++ }
+
+ trace_f2fs_writepages(mapping->host, wbc, NODE);
+
+diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
+index a135d22474154..338a57360bb8c 100644
+--- a/fs/f2fs/segment.c
++++ b/fs/f2fs/segment.c
+@@ -561,7 +561,7 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
+ goto do_sync;
+
+ /* checkpoint is the only way to shrink partial cached entries */
+- if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
++ if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
+ f2fs_available_free_memory(sbi, INO_ENTRIES))
+ return;
+
+@@ -4747,6 +4747,13 @@ static int sanity_check_curseg(struct f2fs_sb_info *sbi)
+
+ sanity_check_seg_type(sbi, curseg->seg_type);
+
++ if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
++ f2fs_err(sbi,
++ "Current segment has invalid alloc_type:%d",
++ curseg->alloc_type);
++ return -EFSCORRUPTED;
++ }
++
+ if (f2fs_test_bit(blkofs, se->cur_valid_map))
+ goto out;
+
+diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
+index 89fff258727d1..b2d82c9f5a16d 100644
+--- a/fs/f2fs/segment.h
++++ b/fs/f2fs/segment.h
+@@ -537,7 +537,8 @@ static inline unsigned int free_segments(struct f2fs_sb_info *sbi)
+
+ static inline unsigned int reserved_segments(struct f2fs_sb_info *sbi)
+ {
+- return SM_I(sbi)->reserved_segments;
++ return SM_I(sbi)->reserved_segments +
++ SM_I(sbi)->additional_reserved_segments;
+ }
+
+ static inline unsigned int free_sections(struct f2fs_sb_info *sbi)
+diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
+index 78ebc306ee2b5..6dc66b7bc1f58 100644
+--- a/fs/f2fs/super.c
++++ b/fs/f2fs/super.c
+@@ -327,6 +327,46 @@ static inline void limit_reserve_root(struct f2fs_sb_info *sbi)
+ F2FS_OPTION(sbi).s_resgid));
+ }
+
++static inline int adjust_reserved_segment(struct f2fs_sb_info *sbi)
++{
++ unsigned int sec_blks = sbi->blocks_per_seg * sbi->segs_per_sec;
++ unsigned int avg_vblocks;
++ unsigned int wanted_reserved_segments;
++ block_t avail_user_block_count;
++
++ if (!F2FS_IO_ALIGNED(sbi))
++ return 0;
++
++ /* average valid block count in section in worst case */
++ avg_vblocks = sec_blks / F2FS_IO_SIZE(sbi);
++
++ /*
++ * we need enough free space when migrating one section in worst case
++ */
++ wanted_reserved_segments = (F2FS_IO_SIZE(sbi) / avg_vblocks) *
++ reserved_segments(sbi);
++ wanted_reserved_segments -= reserved_segments(sbi);
++
++ avail_user_block_count = sbi->user_block_count -
++ sbi->current_reserved_blocks -
++ F2FS_OPTION(sbi).root_reserved_blocks;
++
++ if (wanted_reserved_segments * sbi->blocks_per_seg >
++ avail_user_block_count) {
++ f2fs_err(sbi, "IO align feature can't grab additional reserved segment: %u, available segments: %u",
++ wanted_reserved_segments,
++ avail_user_block_count >> sbi->log_blocks_per_seg);
++ return -ENOSPC;
++ }
++
++ SM_I(sbi)->additional_reserved_segments = wanted_reserved_segments;
++
++ f2fs_info(sbi, "IO align feature needs additional reserved segment: %u",
++ wanted_reserved_segments);
++
++ return 0;
++}
++
+ static inline void adjust_unusable_cap_perc(struct f2fs_sb_info *sbi)
+ {
+ if (!F2FS_OPTION(sbi).unusable_cap_perc)
+@@ -1292,7 +1332,7 @@ default_check:
+ /* Not pass down write hints if the number of active logs is lesser
+ * than NR_CURSEG_PERSIST_TYPE.
+ */
+- if (F2FS_OPTION(sbi).active_logs != NR_CURSEG_TYPE)
++ if (F2FS_OPTION(sbi).active_logs != NR_CURSEG_PERSIST_TYPE)
+ F2FS_OPTION(sbi).whint_mode = WHINT_MODE_OFF;
+
+ if (f2fs_sb_has_readonly(sbi) && !f2fs_readonly(sbi->sb)) {
+@@ -1623,11 +1663,15 @@ static int f2fs_freeze(struct super_block *sb)
+ /* ensure no checkpoint required */
+ if (!llist_empty(&F2FS_SB(sb)->cprc_info.issue_list))
+ return -EINVAL;
++
++ /* to avoid deadlock on f2fs_evict_inode->SB_FREEZE_FS */
++ set_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING);
+ return 0;
+ }
+
+ static int f2fs_unfreeze(struct super_block *sb)
+ {
++ clear_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING);
+ return 0;
+ }
+
+@@ -2636,7 +2680,7 @@ int f2fs_quota_sync(struct super_block *sb, int type)
+ struct f2fs_sb_info *sbi = F2FS_SB(sb);
+ struct quota_info *dqopt = sb_dqopt(sb);
+ int cnt;
+- int ret;
++ int ret = 0;
+
+ /*
+ * Now when everything is written we can discard the pagecache so
+@@ -2647,8 +2691,8 @@ int f2fs_quota_sync(struct super_block *sb, int type)
+ if (type != -1 && cnt != type)
+ continue;
+
+- if (!sb_has_quota_active(sb, type))
+- return 0;
++ if (!sb_has_quota_active(sb, cnt))
++ continue;
+
+ inode_lock(dqopt->files[cnt]);
+
+@@ -3487,7 +3531,7 @@ skip_cross:
+ NR_CURSEG_PERSIST_TYPE + nat_bits_blocks >= blocks_per_seg)) {
+ f2fs_warn(sbi, "Insane cp_payload: %u, nat_bits_blocks: %u)",
+ cp_payload, nat_bits_blocks);
+- return -EFSCORRUPTED;
++ return 1;
+ }
+
+ if (unlikely(f2fs_cp_error(sbi))) {
+@@ -4148,6 +4192,10 @@ try_onemore:
+ goto free_nm;
+ }
+
++ err = adjust_reserved_segment(sbi);
++ if (err)
++ goto free_nm;
++
+ /* For write statistics */
+ sbi->sectors_written_start = f2fs_get_sectors_written(sbi);
+
+@@ -4352,6 +4400,8 @@ free_node_inode:
+ free_stats:
+ f2fs_destroy_stats(sbi);
+ free_nm:
++ /* stop discard thread before destroying node manager */
++ f2fs_stop_discard_thread(sbi);
+ f2fs_destroy_node_manager(sbi);
+ free_sm:
+ f2fs_destroy_segment_manager(sbi);
+diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c
+index a32fe31c33b8e..8b36e61fe7edb 100644
+--- a/fs/f2fs/sysfs.c
++++ b/fs/f2fs/sysfs.c
+@@ -415,7 +415,9 @@ out:
+ if (a->struct_type == RESERVED_BLOCKS) {
+ spin_lock(&sbi->stat_lock);
+ if (t > (unsigned long)(sbi->user_block_count -
+- F2FS_OPTION(sbi).root_reserved_blocks)) {
++ F2FS_OPTION(sbi).root_reserved_blocks -
++ sbi->blocks_per_seg *
++ SM_I(sbi)->additional_reserved_segments)) {
+ spin_unlock(&sbi->stat_lock);
+ return -EINVAL;
+ }
+@@ -471,7 +473,7 @@ out:
+ } else if (t == GC_IDLE_AT) {
+ if (!sbi->am.atgc_enabled)
+ return -EINVAL;
+- sbi->gc_mode = GC_AT;
++ sbi->gc_mode = GC_IDLE_AT;
+ } else {
+ sbi->gc_mode = GC_NORMAL;
+ }
+diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
+index 1d2d29dcd41ce..6a3b3bec7989d 100644
+--- a/fs/f2fs/xattr.c
++++ b/fs/f2fs/xattr.c
+@@ -684,8 +684,17 @@ static int __f2fs_setxattr(struct inode *inode, int index,
+ }
+
+ last = here;
+- while (!IS_XATTR_LAST_ENTRY(last))
++ while (!IS_XATTR_LAST_ENTRY(last)) {
++ if ((void *)(last) + sizeof(__u32) > last_base_addr ||
++ (void *)XATTR_NEXT_ENTRY(last) > last_base_addr) {
++ f2fs_err(F2FS_I_SB(inode), "inode (%lu) has invalid last xattr entry, entry_size: %zu",
++ inode->i_ino, ENTRY_SIZE(last));
++ set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
++ error = -EFSCORRUPTED;
++ goto exit;
++ }
+ last = XATTR_NEXT_ENTRY(last);
++ }
+
+ newsize = XATTR_ALIGN(sizeof(struct f2fs_xattr_entry) + len + size);
+
+diff --git a/fs/file.c b/fs/file.c
+index 8627dacfc4246..ee93173467025 100644
+--- a/fs/file.c
++++ b/fs/file.c
+@@ -87,6 +87,21 @@ static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
+ copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
+ }
+
++/*
++ * Note how the fdtable bitmap allocations very much have to be a multiple of
++ * BITS_PER_LONG. This is not only because we walk those things in chunks of
++ * 'unsigned long' in some places, but simply because that is how the Linux
++ * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
++ * they are very much "bits in an array of unsigned long".
++ *
++ * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
++ * by that "1024/sizeof(ptr)" before, we already know there are sufficient
++ * clear low bits. Clang seems to realize that, gcc ends up being confused.
++ *
++ * On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
++ * let's consider it documentation (and maybe a test-case for gcc to improve
++ * its code generation ;)
++ */
+ static struct fdtable * alloc_fdtable(unsigned int nr)
+ {
+ struct fdtable *fdt;
+@@ -102,6 +117,7 @@ static struct fdtable * alloc_fdtable(unsigned int nr)
+ nr /= (1024 / sizeof(struct file *));
+ nr = roundup_pow_of_two(nr + 1);
+ nr *= (1024 / sizeof(struct file *));
++ nr = ALIGN(nr, BITS_PER_LONG);
+ /*
+ * Note that this can drive nr *below* what we had passed if sysctl_nr_open
+ * had been set lower between the check in expand_files() and here. Deal
+@@ -269,6 +285,19 @@ static unsigned int count_open_files(struct fdtable *fdt)
+ return i;
+ }
+
++/*
++ * Note that a sane fdtable size always has to be a multiple of
++ * BITS_PER_LONG, since we have bitmaps that are sized by this.
++ *
++ * 'max_fds' will normally already be properly aligned, but it
++ * turns out that in the close_range() -> __close_range() ->
++ * unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
++ * up having a 'max_fds' value that isn't already aligned.
++ *
++ * Rather than make close_range() have to worry about this,
++ * just make that BITS_PER_LONG alignment be part of a sane
++ * fdtable size. Becuase that's really what it is.
++ */
+ static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
+ {
+ unsigned int count;
+@@ -276,7 +305,7 @@ static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
+ count = count_open_files(fdt);
+ if (max_fds < NR_OPEN_DEFAULT)
+ max_fds = NR_OPEN_DEFAULT;
+- return min(count, max_fds);
++ return ALIGN(min(count, max_fds), BITS_PER_LONG);
+ }
+
+ /*
+@@ -841,24 +870,68 @@ void do_close_on_exec(struct files_struct *files)
+ spin_unlock(&files->file_lock);
+ }
+
++static inline struct file *__fget_files_rcu(struct files_struct *files,
++ unsigned int fd, fmode_t mask, unsigned int refs)
++{
++ for (;;) {
++ struct file *file;
++ struct fdtable *fdt = rcu_dereference_raw(files->fdt);
++ struct file __rcu **fdentry;
++
++ if (unlikely(fd >= fdt->max_fds))
++ return NULL;
++
++ fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds);
++ file = rcu_dereference_raw(*fdentry);
++ if (unlikely(!file))
++ return NULL;
++
++ if (unlikely(file->f_mode & mask))
++ return NULL;
++
++ /*
++ * Ok, we have a file pointer. However, because we do
++ * this all locklessly under RCU, we may be racing with
++ * that file being closed.
++ *
++ * Such a race can take two forms:
++ *
++ * (a) the file ref already went down to zero,
++ * and get_file_rcu_many() fails. Just try
++ * again:
++ */
++ if (unlikely(!get_file_rcu_many(file, refs)))
++ continue;
++
++ /*
++ * (b) the file table entry has changed under us.
++ * Note that we don't need to re-check the 'fdt->fd'
++ * pointer having changed, because it always goes
++ * hand-in-hand with 'fdt'.
++ *
++ * If so, we need to put our refs and try again.
++ */
++ if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||
++ unlikely(rcu_dereference_raw(*fdentry) != file)) {
++ fput_many(file, refs);
++ continue;
++ }
++
++ /*
++ * Ok, we have a ref to the file, and checked that it
++ * still exists.
++ */
++ return file;
++ }
++}
++
+ static struct file *__fget_files(struct files_struct *files, unsigned int fd,
+ fmode_t mask, unsigned int refs)
+ {
+ struct file *file;
+
+ rcu_read_lock();
+-loop:
+- file = files_lookup_fd_rcu(files, fd);
+- if (file) {
+- /* File object ref couldn't be taken.
+- * dup2() atomicity guarantee is the reason
+- * we loop to catch the new file (or NULL pointer)
+- */
+- if (file->f_mode & mask)
+- file = NULL;
+- else if (!get_file_rcu_many(file, refs))
+- goto loop;
+- }
++ file = __fget_files_rcu(files, fd, mask, refs);
+ rcu_read_unlock();
+
+ return file;
+diff --git a/fs/file_table.c b/fs/file_table.c
+index 45437f8e1003e..e8c9016703ad6 100644
+--- a/fs/file_table.c
++++ b/fs/file_table.c
+@@ -375,6 +375,7 @@ void __fput_sync(struct file *file)
+ }
+
+ EXPORT_SYMBOL(fput);
++EXPORT_SYMBOL(__fput_sync);
+
+ void __init files_init(void)
+ {
+diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
+index 81ec192ce0673..85b4259f104a2 100644
+--- a/fs/fs-writeback.c
++++ b/fs/fs-writeback.c
+@@ -1739,6 +1739,10 @@ static int writeback_single_inode(struct inode *inode,
+ */
+ if (!(inode->i_state & I_DIRTY_ALL))
+ inode_cgwb_move_to_attached(inode, wb);
++ else if (!(inode->i_state & I_SYNC_QUEUED) &&
++ (inode->i_state & I_DIRTY))
++ redirty_tail_locked(inode, wb);
++
+ spin_unlock(&wb->list_lock);
+ inode_sync_complete(inode);
+ out:
+diff --git a/fs/fs_context.c b/fs/fs_context.c
+index b7e43a780a625..24ce12f0db32e 100644
+--- a/fs/fs_context.c
++++ b/fs/fs_context.c
+@@ -548,7 +548,7 @@ static int legacy_parse_param(struct fs_context *fc, struct fs_parameter *param)
+ param->key);
+ }
+
+- if (len > PAGE_SIZE - 2 - size)
++ if (size + len + 2 > PAGE_SIZE)
+ return invalf(fc, "VFS: Legacy: Cumulative options too large");
+ if (strchr(param->key, ',') ||
+ (param->type == fs_value_is_string &&
+diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
+index dde341a6388a1..d6b5339c56e2c 100644
+--- a/fs/fuse/dev.c
++++ b/fs/fuse/dev.c
+@@ -852,6 +852,12 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
+ if (!(buf->flags & PIPE_BUF_FLAG_LRU))
+ lru_cache_add(newpage);
+
++ /*
++ * Release while we have extra ref on stolen page. Otherwise
++ * anon_pipe_buf_release() might think the page can be reused.
++ */
++ pipe_buf_release(cs->pipe, buf);
++
+ err = 0;
+ spin_lock(&cs->req->waitq.lock);
+ if (test_bit(FR_ABORTED, &cs->req->flags))
+@@ -935,7 +941,17 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
+
+ while (count) {
+ if (cs->write && cs->pipebufs && page) {
+- return fuse_ref_page(cs, page, offset, count);
++ /*
++ * Can't control lifetime of pipe buffers, so always
++ * copy user pages.
++ */
++ if (cs->req->args->user_pages) {
++ err = fuse_copy_fill(cs);
++ if (err)
++ return err;
++ } else {
++ return fuse_ref_page(cs, page, offset, count);
++ }
+ } else if (!cs->len) {
+ if (cs->move_pages && page &&
+ offset == 0 && count == PAGE_SIZE) {
+@@ -2031,8 +2047,12 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
+
+ pipe_lock(pipe);
+ out_free:
+- for (idx = 0; idx < nbuf; idx++)
+- pipe_buf_release(pipe, &bufs[idx]);
++ for (idx = 0; idx < nbuf; idx++) {
++ struct pipe_buffer *buf = &bufs[idx];
++
++ if (buf->ops)
++ pipe_buf_release(pipe, buf);
++ }
+ pipe_unlock(pipe);
+
+ kvfree(bufs);
+diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
+index d9b977c0f38dc..80a2181b402b2 100644
+--- a/fs/fuse/dir.c
++++ b/fs/fuse/dir.c
+@@ -738,11 +738,19 @@ static int fuse_symlink(struct user_namespace *mnt_userns, struct inode *dir,
+ return create_new_entry(fm, &args, dir, entry, S_IFLNK);
+ }
+
++void fuse_flush_time_update(struct inode *inode)
++{
++ int err = sync_inode_metadata(inode, 1);
++
++ mapping_set_error(inode->i_mapping, err);
++}
++
+ void fuse_update_ctime(struct inode *inode)
+ {
+ if (!IS_NOCMTIME(inode)) {
+ inode->i_ctime = current_time(inode);
+ mark_inode_dirty_sync(inode);
++ fuse_flush_time_update(inode);
+ }
+ }
+
+@@ -1071,7 +1079,7 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid,
+ if (!parent)
+ return -ENOENT;
+
+- inode_lock(parent);
++ inode_lock_nested(parent, I_MUTEX_PARENT);
+ if (!S_ISDIR(parent->i_mode))
+ goto unlock;
+
+diff --git a/fs/fuse/file.c b/fs/fuse/file.c
+index 11404f8c21c75..71e9e301e569d 100644
+--- a/fs/fuse/file.c
++++ b/fs/fuse/file.c
+@@ -1164,7 +1164,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
+
+ again:
+ err = -EFAULT;
+- if (iov_iter_fault_in_readable(ii, bytes))
++ if (fault_in_iov_iter_readable(ii, bytes))
+ break;
+
+ err = -ENOMEM;
+@@ -1417,6 +1417,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
+ (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
+ }
+
++ ap->args.user_pages = true;
+ if (write)
+ ap->args.in_pages = true;
+ else
+@@ -1848,6 +1849,17 @@ int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
+ struct fuse_file *ff;
+ int err;
+
++ /*
++ * Inode is always written before the last reference is dropped and
++ * hence this should not be reached from reclaim.
++ *
++ * Writing back the inode from reclaim can deadlock if the request
++ * processing itself needs an allocation. Allocations triggering
++ * reclaim while serving a request can't be prevented, because it can
++ * involve any number of unrelated userspace processes.
++ */
++ WARN_ON(wbc->for_reclaim);
++
+ ff = __fuse_write_file_get(fi);
+ err = fuse_flush_times(inode, ff);
+ if (ff)
+@@ -2902,7 +2914,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+
+ static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
+ {
+- int err = filemap_write_and_wait_range(inode->i_mapping, start, -1);
++ int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
+
+ if (!err)
+ fuse_sync_writes(inode);
+@@ -3002,6 +3014,8 @@ out:
+ if (lock_inode)
+ inode_unlock(inode);
+
++ fuse_flush_time_update(inode);
++
+ return err;
+ }
+
+@@ -3111,6 +3125,8 @@ out:
+ inode_unlock(inode_out);
+ file_accessed(file_in);
+
++ fuse_flush_time_update(inode_out);
++
+ return err;
+ }
+
+diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
+index f55f9f94b1a4f..c3a87586a15fa 100644
+--- a/fs/fuse/fuse_i.h
++++ b/fs/fuse/fuse_i.h
+@@ -256,6 +256,7 @@ struct fuse_args {
+ bool nocreds:1;
+ bool in_pages:1;
+ bool out_pages:1;
++ bool user_pages:1;
+ bool out_argvar:1;
+ bool page_zeroing:1;
+ bool page_replace:1;
+@@ -1148,6 +1149,7 @@ int fuse_allow_current_process(struct fuse_conn *fc);
+
+ u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id);
+
++void fuse_flush_time_update(struct inode *inode);
+ void fuse_update_ctime(struct inode *inode);
+
+ int fuse_update_attributes(struct inode *inode, struct file *file);
+diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
+index 12d49a1914e84..2f999d38c9b4a 100644
+--- a/fs/fuse/inode.c
++++ b/fs/fuse/inode.c
+@@ -118,6 +118,9 @@ static void fuse_evict_inode(struct inode *inode)
+ {
+ struct fuse_inode *fi = get_fuse_inode(inode);
+
++ /* Will write inode on close/munmap and in all other dirtiers */
++ WARN_ON(inode->i_state & I_DIRTY_INODE);
++
+ truncate_inode_pages_final(&inode->i_data);
+ clear_inode(inode);
+ if (inode->i_sb->s_flags & SB_ACTIVE) {
+diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c
+index 546ea3d58fb47..fc69e1797a331 100644
+--- a/fs/fuse/ioctl.c
++++ b/fs/fuse/ioctl.c
+@@ -394,9 +394,12 @@ static int fuse_priv_ioctl(struct inode *inode, struct fuse_file *ff,
+ args.out_args[1].value = ptr;
+
+ err = fuse_simple_request(fm, &args);
+- if (!err && outarg.flags & FUSE_IOCTL_RETRY)
+- err = -EIO;
+-
++ if (!err) {
++ if (outarg.result < 0)
++ err = outarg.result;
++ else if (outarg.flags & FUSE_IOCTL_RETRY)
++ err = -EIO;
++ }
+ return err;
+ }
+
+diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c
+index 5414c2c335809..f785af2aa23cf 100644
+--- a/fs/gfs2/bmap.c
++++ b/fs/gfs2/bmap.c
+@@ -940,7 +940,7 @@ do_alloc:
+ else if (height == ip->i_height)
+ ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
+ else
+- iomap->length = size - pos;
++ iomap->length = size - iomap->offset;
+ } else if (flags & IOMAP_WRITE) {
+ u64 alloc_size;
+
+@@ -961,46 +961,6 @@ hole_found:
+ goto out;
+ }
+
+-static int gfs2_write_lock(struct inode *inode)
+-{
+- struct gfs2_inode *ip = GFS2_I(inode);
+- struct gfs2_sbd *sdp = GFS2_SB(inode);
+- int error;
+-
+- gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &ip->i_gh);
+- error = gfs2_glock_nq(&ip->i_gh);
+- if (error)
+- goto out_uninit;
+- if (&ip->i_inode == sdp->sd_rindex) {
+- struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
+-
+- error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
+- GL_NOCACHE, &m_ip->i_gh);
+- if (error)
+- goto out_unlock;
+- }
+- return 0;
+-
+-out_unlock:
+- gfs2_glock_dq(&ip->i_gh);
+-out_uninit:
+- gfs2_holder_uninit(&ip->i_gh);
+- return error;
+-}
+-
+-static void gfs2_write_unlock(struct inode *inode)
+-{
+- struct gfs2_inode *ip = GFS2_I(inode);
+- struct gfs2_sbd *sdp = GFS2_SB(inode);
+-
+- if (&ip->i_inode == sdp->sd_rindex) {
+- struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
+-
+- gfs2_glock_dq_uninit(&m_ip->i_gh);
+- }
+- gfs2_glock_dq_uninit(&ip->i_gh);
+-}
+-
+ static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
+ unsigned len)
+ {
+@@ -1118,11 +1078,6 @@ out_qunlock:
+ return ret;
+ }
+
+-static inline bool gfs2_iomap_need_write_lock(unsigned flags)
+-{
+- return (flags & IOMAP_WRITE) && !(flags & IOMAP_DIRECT);
+-}
+-
+ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
+ unsigned flags, struct iomap *iomap,
+ struct iomap *srcmap)
+@@ -1135,12 +1090,6 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
+ iomap->flags |= IOMAP_F_BUFFER_HEAD;
+
+ trace_gfs2_iomap_start(ip, pos, length, flags);
+- if (gfs2_iomap_need_write_lock(flags)) {
+- ret = gfs2_write_lock(inode);
+- if (ret)
+- goto out;
+- }
+-
+ ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
+ if (ret)
+ goto out_unlock;
+@@ -1168,10 +1117,7 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
+ ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
+
+ out_unlock:
+- if (ret && gfs2_iomap_need_write_lock(flags))
+- gfs2_write_unlock(inode);
+ release_metapath(&mp);
+-out:
+ trace_gfs2_iomap_end(ip, iomap, ret);
+ return ret;
+ }
+@@ -1208,26 +1154,21 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+
+ if (length != written && (iomap->flags & IOMAP_F_NEW)) {
+ /* Deallocate blocks that were just allocated. */
+- loff_t blockmask = i_blocksize(inode) - 1;
+- loff_t end = (pos + length) & ~blockmask;
++ loff_t hstart = round_up(pos + written, i_blocksize(inode));
++ loff_t hend = iomap->offset + iomap->length;
+
+- pos = (pos + written + blockmask) & ~blockmask;
+- if (pos < end) {
+- truncate_pagecache_range(inode, pos, end - 1);
+- punch_hole(ip, pos, end - pos);
++ if (hstart < hend) {
++ truncate_pagecache_range(inode, hstart, hend - 1);
++ punch_hole(ip, hstart, hend - hstart);
+ }
+ }
+
+ if (unlikely(!written))
+- goto out_unlock;
++ return 0;
+
+ if (iomap->flags & IOMAP_F_SIZE_CHANGED)
+ mark_inode_dirty(inode);
+ set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
+-
+-out_unlock:
+- if (gfs2_iomap_need_write_lock(flags))
+- gfs2_write_unlock(inode);
+ return 0;
+ }
+
+@@ -2204,7 +2145,7 @@ int gfs2_setattr_size(struct inode *inode, u64 newsize)
+
+ ret = do_shrink(inode, newsize);
+ out:
+- gfs2_rs_delete(ip, NULL);
++ gfs2_rs_delete(ip);
+ gfs2_qa_put(ip);
+ return ret;
+ }
+diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c
+index c559827cb6f91..60390f9dc31f4 100644
+--- a/fs/gfs2/file.c
++++ b/fs/gfs2/file.c
+@@ -711,10 +711,11 @@ static int gfs2_release(struct inode *inode, struct file *file)
+ kfree(file->private_data);
+ file->private_data = NULL;
+
+- if (gfs2_rs_active(&ip->i_res))
+- gfs2_rs_delete(ip, &inode->i_writecount);
+- if (file->f_mode & FMODE_WRITE)
++ if (file->f_mode & FMODE_WRITE) {
++ if (gfs2_rs_active(&ip->i_res))
++ gfs2_rs_delete(ip);
+ gfs2_qa_put(ip);
++ }
+ return 0;
+ }
+
+@@ -776,27 +777,99 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end,
+ return ret ? ret : ret1;
+ }
+
++static inline bool should_fault_in_pages(ssize_t ret, struct iov_iter *i,
++ size_t *prev_count,
++ size_t *window_size)
++{
++ char __user *p = i->iov[0].iov_base + i->iov_offset;
++ size_t count = iov_iter_count(i);
++ int pages = 1;
++
++ if (likely(!count))
++ return false;
++ if (ret <= 0 && ret != -EFAULT)
++ return false;
++ if (!iter_is_iovec(i))
++ return false;
++
++ if (*prev_count != count || !*window_size) {
++ int pages, nr_dirtied;
++
++ pages = min_t(int, BIO_MAX_VECS,
++ DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE));
++ nr_dirtied = max(current->nr_dirtied_pause -
++ current->nr_dirtied, 1);
++ pages = min(pages, nr_dirtied);
++ }
++
++ *prev_count = count;
++ *window_size = (size_t)PAGE_SIZE * pages - offset_in_page(p);
++ return true;
++}
++
+ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to,
+ struct gfs2_holder *gh)
+ {
+ struct file *file = iocb->ki_filp;
+ struct gfs2_inode *ip = GFS2_I(file->f_mapping->host);
+- size_t count = iov_iter_count(to);
++ size_t prev_count = 0, window_size = 0;
++ size_t written = 0;
+ ssize_t ret;
+
+- if (!count)
++ /*
++ * In this function, we disable page faults when we're holding the
++ * inode glock while doing I/O. If a page fault occurs, we indicate
++ * that the inode glock may be dropped, fault in the pages manually,
++ * and retry.
++ *
++ * Unlike generic_file_read_iter, for reads, iomap_dio_rw can trigger
++ * physical as well as manual page faults, and we need to disable both
++ * kinds.
++ *
++ * For direct I/O, gfs2 takes the inode glock in deferred mode. This
++ * locking mode is compatible with other deferred holders, so multiple
++ * processes and nodes can do direct I/O to a file at the same time.
++ * There's no guarantee that reads or writes will be atomic. Any
++ * coordination among readers and writers needs to happen externally.
++ */
++
++ if (!iov_iter_count(to))
+ return 0; /* skip atime */
+
+ gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh);
++retry:
+ ret = gfs2_glock_nq(gh);
+ if (ret)
+ goto out_uninit;
++retry_under_glock:
++ pagefault_disable();
++ to->nofault = true;
++ ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL,
++ IOMAP_DIO_PARTIAL, written);
++ to->nofault = false;
++ pagefault_enable();
++ if (ret > 0)
++ written = ret;
+
+- ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, 0);
+- gfs2_glock_dq(gh);
++ if (should_fault_in_pages(ret, to, &prev_count, &window_size)) {
++ size_t leftover;
++
++ gfs2_holder_allow_demote(gh);
++ leftover = fault_in_iov_iter_writeable(to, window_size);
++ gfs2_holder_disallow_demote(gh);
++ if (leftover != window_size) {
++ if (gfs2_holder_queued(gh))
++ goto retry_under_glock;
++ goto retry;
++ }
++ }
++ if (gfs2_holder_queued(gh))
++ gfs2_glock_dq(gh);
+ out_uninit:
+ gfs2_holder_uninit(gh);
+- return ret;
++ if (ret < 0)
++ return ret;
++ return written;
+ }
+
+ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
+@@ -805,10 +878,20 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
+ struct file *file = iocb->ki_filp;
+ struct inode *inode = file->f_mapping->host;
+ struct gfs2_inode *ip = GFS2_I(inode);
+- size_t len = iov_iter_count(from);
+- loff_t offset = iocb->ki_pos;
++ size_t prev_count = 0, window_size = 0;
++ size_t read = 0;
+ ssize_t ret;
+
++ /*
++ * In this function, we disable page faults when we're holding the
++ * inode glock while doing I/O. If a page fault occurs, we indicate
++ * that the inode glock may be dropped, fault in the pages manually,
++ * and retry.
++ *
++ * For writes, iomap_dio_rw only triggers manual page faults, so we
++ * don't need to disable physical ones.
++ */
++
+ /*
+ * Deferred lock, even if its a write, since we do no allocation on
+ * this path. All we need to change is the atime, and this lock mode
+@@ -818,45 +901,78 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
+ * VFS does.
+ */
+ gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh);
++retry:
+ ret = gfs2_glock_nq(gh);
+ if (ret)
+ goto out_uninit;
+-
++retry_under_glock:
+ /* Silently fall back to buffered I/O when writing beyond EOF */
+- if (offset + len > i_size_read(&ip->i_inode))
++ if (iocb->ki_pos + iov_iter_count(from) > i_size_read(&ip->i_inode))
+ goto out;
+
+- ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, 0);
++ from->nofault = true;
++ ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL,
++ IOMAP_DIO_PARTIAL, read);
++ from->nofault = false;
++
+ if (ret == -ENOTBLK)
+ ret = 0;
++ if (ret > 0)
++ read = ret;
++
++ if (should_fault_in_pages(ret, from, &prev_count, &window_size)) {
++ size_t leftover;
++
++ gfs2_holder_allow_demote(gh);
++ leftover = fault_in_iov_iter_readable(from, window_size);
++ gfs2_holder_disallow_demote(gh);
++ if (leftover != window_size) {
++ if (gfs2_holder_queued(gh))
++ goto retry_under_glock;
++ goto retry;
++ }
++ }
+ out:
+- gfs2_glock_dq(gh);
++ if (gfs2_holder_queued(gh))
++ gfs2_glock_dq(gh);
+ out_uninit:
+ gfs2_holder_uninit(gh);
+- return ret;
++ if (ret < 0)
++ return ret;
++ return read;
+ }
+
+ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+ {
+ struct gfs2_inode *ip;
+ struct gfs2_holder gh;
++ size_t prev_count = 0, window_size = 0;
+ size_t written = 0;
+ ssize_t ret;
+
++ /*
++ * In this function, we disable page faults when we're holding the
++ * inode glock while doing I/O. If a page fault occurs, we indicate
++ * that the inode glock may be dropped, fault in the pages manually,
++ * and retry.
++ */
++
+ if (iocb->ki_flags & IOCB_DIRECT) {
+ ret = gfs2_file_direct_read(iocb, to, &gh);
+ if (likely(ret != -ENOTBLK))
+ return ret;
+ iocb->ki_flags &= ~IOCB_DIRECT;
+ }
++ pagefault_disable();
+ iocb->ki_flags |= IOCB_NOIO;
+ ret = generic_file_read_iter(iocb, to);
+ iocb->ki_flags &= ~IOCB_NOIO;
++ pagefault_enable();
+ if (ret >= 0) {
+ if (!iov_iter_count(to))
+ return ret;
+ written = ret;
+- } else {
++ } else if (ret != -EFAULT) {
+ if (ret != -EAGAIN)
+ return ret;
+ if (iocb->ki_flags & IOCB_NOWAIT)
+@@ -864,18 +980,115 @@ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+ }
+ ip = GFS2_I(iocb->ki_filp->f_mapping->host);
+ gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
++retry:
+ ret = gfs2_glock_nq(&gh);
+ if (ret)
+ goto out_uninit;
++retry_under_glock:
++ pagefault_disable();
+ ret = generic_file_read_iter(iocb, to);
++ pagefault_enable();
+ if (ret > 0)
+ written += ret;
+- gfs2_glock_dq(&gh);
++
++ if (should_fault_in_pages(ret, to, &prev_count, &window_size)) {
++ size_t leftover;
++
++ gfs2_holder_allow_demote(&gh);
++ leftover = fault_in_iov_iter_writeable(to, window_size);
++ gfs2_holder_disallow_demote(&gh);
++ if (leftover != window_size) {
++ if (gfs2_holder_queued(&gh))
++ goto retry_under_glock;
++ goto retry;
++ }
++ }
++ if (gfs2_holder_queued(&gh))
++ gfs2_glock_dq(&gh);
+ out_uninit:
+ gfs2_holder_uninit(&gh);
+ return written ? written : ret;
+ }
+
++static ssize_t gfs2_file_buffered_write(struct kiocb *iocb,
++ struct iov_iter *from,
++ struct gfs2_holder *gh)
++{
++ struct file *file = iocb->ki_filp;
++ struct inode *inode = file_inode(file);
++ struct gfs2_inode *ip = GFS2_I(inode);
++ struct gfs2_sbd *sdp = GFS2_SB(inode);
++ struct gfs2_holder *statfs_gh = NULL;
++ size_t prev_count = 0, window_size = 0;
++ size_t orig_count = iov_iter_count(from);
++ size_t read = 0;
++ ssize_t ret;
++
++ /*
++ * In this function, we disable page faults when we're holding the
++ * inode glock while doing I/O. If a page fault occurs, we indicate
++ * that the inode glock may be dropped, fault in the pages manually,
++ * and retry.
++ */
++
++ if (inode == sdp->sd_rindex) {
++ statfs_gh = kmalloc(sizeof(*statfs_gh), GFP_NOFS);
++ if (!statfs_gh)
++ return -ENOMEM;
++ }
++
++ gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, gh);
++retry:
++ ret = gfs2_glock_nq(gh);
++ if (ret)
++ goto out_uninit;
++retry_under_glock:
++ if (inode == sdp->sd_rindex) {
++ struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
++
++ ret = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
++ GL_NOCACHE, statfs_gh);
++ if (ret)
++ goto out_unlock;
++ }
++
++ current->backing_dev_info = inode_to_bdi(inode);
++ pagefault_disable();
++ ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops);
++ pagefault_enable();
++ current->backing_dev_info = NULL;
++ if (ret > 0) {
++ iocb->ki_pos += ret;
++ read += ret;
++ }
++
++ if (inode == sdp->sd_rindex)
++ gfs2_glock_dq_uninit(statfs_gh);
++
++ from->count = orig_count - read;
++ if (should_fault_in_pages(ret, from, &prev_count, &window_size)) {
++ size_t leftover;
++
++ gfs2_holder_allow_demote(gh);
++ leftover = fault_in_iov_iter_readable(from, window_size);
++ gfs2_holder_disallow_demote(gh);
++ if (leftover != window_size) {
++ from->count = min(from->count, window_size - leftover);
++ if (gfs2_holder_queued(gh))
++ goto retry_under_glock;
++ goto retry;
++ }
++ }
++out_unlock:
++ if (gfs2_holder_queued(gh))
++ gfs2_glock_dq(gh);
++out_uninit:
++ gfs2_holder_uninit(gh);
++ if (statfs_gh)
++ kfree(statfs_gh);
++ return read ? read : ret;
++}
++
+ /**
+ * gfs2_file_write_iter - Perform a write to a file
+ * @iocb: The io context
+@@ -927,9 +1140,7 @@ static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ goto out_unlock;
+
+ iocb->ki_flags |= IOCB_DSYNC;
+- current->backing_dev_info = inode_to_bdi(inode);
+- buffered = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops);
+- current->backing_dev_info = NULL;
++ buffered = gfs2_file_buffered_write(iocb, from, &gh);
+ if (unlikely(buffered <= 0)) {
+ if (!ret)
+ ret = buffered;
+@@ -943,7 +1154,6 @@ static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ * the direct I/O range as we don't know if the buffered pages
+ * made it to disk.
+ */
+- iocb->ki_pos += buffered;
+ ret2 = generic_write_sync(iocb, buffered);
+ invalidate_mapping_pages(mapping,
+ (iocb->ki_pos - buffered) >> PAGE_SHIFT,
+@@ -951,13 +1161,9 @@ static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ if (!ret || ret2 > 0)
+ ret += ret2;
+ } else {
+- current->backing_dev_info = inode_to_bdi(inode);
+- ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops);
+- current->backing_dev_info = NULL;
+- if (likely(ret > 0)) {
+- iocb->ki_pos += ret;
++ ret = gfs2_file_buffered_write(iocb, from, &gh);
++ if (likely(ret > 0))
+ ret = generic_write_sync(iocb, ret);
+- }
+ }
+
+ out_unlock:
+diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
+index e0eaa9cf9fb6f..e85ef6b14777d 100644
+--- a/fs/gfs2/glock.c
++++ b/fs/gfs2/glock.c
+@@ -58,6 +58,7 @@ struct gfs2_glock_iter {
+ typedef void (*glock_examiner) (struct gfs2_glock * gl);
+
+ static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
++static void __gfs2_glock_dq(struct gfs2_holder *gh);
+
+ static struct dentry *gfs2_root;
+ static struct workqueue_struct *glock_workqueue;
+@@ -197,6 +198,12 @@ static int demote_ok(const struct gfs2_glock *gl)
+
+ if (gl->gl_state == LM_ST_UNLOCKED)
+ return 0;
++ /*
++ * Note that demote_ok is used for the lru process of disposing of
++ * glocks. For this purpose, we don't care if the glock's holders
++ * have the HIF_MAY_DEMOTE flag set or not. If someone is using
++ * them, don't demote.
++ */
+ if (!list_empty(&gl->gl_holders))
+ return 0;
+ if (glops->go_demote_ok)
+@@ -301,46 +308,59 @@ void gfs2_glock_put(struct gfs2_glock *gl)
+ }
+
+ /**
+- * may_grant - check if its ok to grant a new lock
++ * may_grant - check if it's ok to grant a new lock
+ * @gl: The glock
++ * @current_gh: One of the current holders of @gl
+ * @gh: The lock request which we wish to grant
+ *
+- * Returns: true if its ok to grant the lock
++ * With our current compatibility rules, if a glock has one or more active
++ * holders (HIF_HOLDER flag set), any of those holders can be passed in as
++ * @current_gh; they are all the same as far as compatibility with the new @gh
++ * goes.
++ *
++ * Returns true if it's ok to grant the lock.
+ */
+
+-static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)
+-{
+- const struct gfs2_holder *gh_head = list_first_entry(&gl->gl_holders, const struct gfs2_holder, gh_list);
++static inline bool may_grant(struct gfs2_glock *gl,
++ struct gfs2_holder *current_gh,
++ struct gfs2_holder *gh)
++{
++ if (current_gh) {
++ GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, &current_gh->gh_iflags));
++
++ switch(current_gh->gh_state) {
++ case LM_ST_EXCLUSIVE:
++ /*
++ * Here we make a special exception to grant holders
++ * who agree to share the EX lock with other holders
++ * who also have the bit set. If the original holder
++ * has the LM_FLAG_NODE_SCOPE bit set, we grant more
++ * holders with the bit set.
++ */
++ return gh->gh_state == LM_ST_EXCLUSIVE &&
++ (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
++ (gh->gh_flags & LM_FLAG_NODE_SCOPE);
+
+- if (gh != gh_head) {
+- /**
+- * Here we make a special exception to grant holders who agree
+- * to share the EX lock with other holders who also have the
+- * bit set. If the original holder has the LM_FLAG_NODE_SCOPE bit
+- * is set, we grant more holders with the bit set.
+- */
+- if (gh_head->gh_state == LM_ST_EXCLUSIVE &&
+- (gh_head->gh_flags & LM_FLAG_NODE_SCOPE) &&
+- gh->gh_state == LM_ST_EXCLUSIVE &&
+- (gh->gh_flags & LM_FLAG_NODE_SCOPE))
+- return 1;
+- if ((gh->gh_state == LM_ST_EXCLUSIVE ||
+- gh_head->gh_state == LM_ST_EXCLUSIVE))
+- return 0;
++ case LM_ST_SHARED:
++ case LM_ST_DEFERRED:
++ return gh->gh_state == current_gh->gh_state;
++
++ default:
++ return false;
++ }
+ }
++
+ if (gl->gl_state == gh->gh_state)
+- return 1;
++ return true;
+ if (gh->gh_flags & GL_EXACT)
+- return 0;
++ return false;
+ if (gl->gl_state == LM_ST_EXCLUSIVE) {
+- if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
+- return 1;
+- if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
+- return 1;
++ return gh->gh_state == LM_ST_SHARED ||
++ gh->gh_state == LM_ST_DEFERRED;
+ }
+- if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))
+- return 1;
+- return 0;
++ if (gh->gh_flags & LM_FLAG_ANY)
++ return gl->gl_state != LM_ST_UNLOCKED;
++ return false;
+ }
+
+ static void gfs2_holder_wake(struct gfs2_holder *gh)
+@@ -366,7 +386,7 @@ static void do_error(struct gfs2_glock *gl, const int ret)
+ struct gfs2_holder *gh, *tmp;
+
+ list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
+- if (test_bit(HIF_HOLDER, &gh->gh_iflags))
++ if (!test_bit(HIF_WAIT, &gh->gh_iflags))
+ continue;
+ if (ret & LM_OUT_ERROR)
+ gh->gh_error = -EIO;
+@@ -380,6 +400,78 @@ static void do_error(struct gfs2_glock *gl, const int ret)
+ }
+ }
+
++/**
++ * demote_incompat_holders - demote incompatible demoteable holders
++ * @gl: the glock we want to promote
++ * @new_gh: the new holder to be promoted
++ */
++static void demote_incompat_holders(struct gfs2_glock *gl,
++ struct gfs2_holder *new_gh)
++{
++ struct gfs2_holder *gh;
++
++ /*
++ * Demote incompatible holders before we make ourselves eligible.
++ * (This holder may or may not allow auto-demoting, but we don't want
++ * to demote the new holder before it's even granted.)
++ */
++ list_for_each_entry(gh, &gl->gl_holders, gh_list) {
++ /*
++ * Since holders are at the front of the list, we stop when we
++ * find the first non-holder.
++ */
++ if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
++ return;
++ if (test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags) &&
++ !may_grant(gl, new_gh, gh)) {
++ /*
++ * We should not recurse into do_promote because
++ * __gfs2_glock_dq only calls handle_callback,
++ * gfs2_glock_add_to_lru and __gfs2_glock_queue_work.
++ */
++ __gfs2_glock_dq(gh);
++ }
++ }
++}
++
++/**
++ * find_first_holder - find the first "holder" gh
++ * @gl: the glock
++ */
++
++static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
++{
++ struct gfs2_holder *gh;
++
++ if (!list_empty(&gl->gl_holders)) {
++ gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
++ gh_list);
++ if (test_bit(HIF_HOLDER, &gh->gh_iflags))
++ return gh;
++ }
++ return NULL;
++}
++
++/**
++ * find_first_strong_holder - find the first non-demoteable holder
++ * @gl: the glock
++ *
++ * Find the first holder that doesn't have the HIF_MAY_DEMOTE flag set.
++ */
++static inline struct gfs2_holder *
++find_first_strong_holder(struct gfs2_glock *gl)
++{
++ struct gfs2_holder *gh;
++
++ list_for_each_entry(gh, &gl->gl_holders, gh_list) {
++ if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
++ return NULL;
++ if (!test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags))
++ return gh;
++ }
++ return NULL;
++}
++
+ /**
+ * do_promote - promote as many requests as possible on the current queue
+ * @gl: The glock
+@@ -393,14 +485,21 @@ __releases(&gl->gl_lockref.lock)
+ __acquires(&gl->gl_lockref.lock)
+ {
+ const struct gfs2_glock_operations *glops = gl->gl_ops;
+- struct gfs2_holder *gh, *tmp;
++ struct gfs2_holder *gh, *tmp, *first_gh;
++ bool incompat_holders_demoted = false;
+ int ret;
+
+ restart:
++ first_gh = find_first_strong_holder(gl);
+ list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
+- if (test_bit(HIF_HOLDER, &gh->gh_iflags))
++ if (!test_bit(HIF_WAIT, &gh->gh_iflags))
+ continue;
+- if (may_grant(gl, gh)) {
++ if (may_grant(gl, first_gh, gh)) {
++ if (!incompat_holders_demoted) {
++ demote_incompat_holders(gl, first_gh);
++ incompat_holders_demoted = true;
++ first_gh = gh;
++ }
+ if (gh->gh_list.prev == &gl->gl_holders &&
+ glops->go_lock) {
+ spin_unlock(&gl->gl_lockref.lock);
+@@ -426,6 +525,11 @@ restart:
+ gfs2_holder_wake(gh);
+ continue;
+ }
++ /*
++ * If we get here, it means we may not grant this holder for
++ * some reason. If this holder is the head of the list, it
++ * means we have a blocked holder at the head, so return 1.
++ */
+ if (gh->gh_list.prev == &gl->gl_holders)
+ return 1;
+ do_error(gl, 0);
+@@ -722,23 +826,6 @@ out:
+ spin_lock(&gl->gl_lockref.lock);
+ }
+
+-/**
+- * find_first_holder - find the first "holder" gh
+- * @gl: the glock
+- */
+-
+-static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
+-{
+- struct gfs2_holder *gh;
+-
+- if (!list_empty(&gl->gl_holders)) {
+- gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
+- if (test_bit(HIF_HOLDER, &gh->gh_iflags))
+- return gh;
+- }
+- return NULL;
+-}
+-
+ /**
+ * run_queue - do all outstanding tasks related to a glock
+ * @gl: The glock in question
+@@ -1354,15 +1441,20 @@ __acquires(&gl->gl_lockref.lock)
+ GLOCK_BUG_ON(gl, true);
+
+ if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
+- if (test_bit(GLF_LOCK, &gl->gl_flags))
+- try_futile = !may_grant(gl, gh);
++ if (test_bit(GLF_LOCK, &gl->gl_flags)) {
++ struct gfs2_holder *first_gh;
++
++ first_gh = find_first_strong_holder(gl);
++ try_futile = !may_grant(gl, first_gh, gh);
++ }
+ if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
+ goto fail;
+ }
+
+ list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
+ if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
+- (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))
++ (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK) &&
++ !test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags)))
+ goto trap_recursive;
+ if (try_futile &&
+ !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
+@@ -1458,51 +1550,83 @@ int gfs2_glock_poll(struct gfs2_holder *gh)
+ return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
+ }
+
+-/**
+- * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
+- * @gh: the glock holder
+- *
+- */
++static inline bool needs_demote(struct gfs2_glock *gl)
++{
++ return (test_bit(GLF_DEMOTE, &gl->gl_flags) ||
++ test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags));
++}
+
+-void gfs2_glock_dq(struct gfs2_holder *gh)
++static void __gfs2_glock_dq(struct gfs2_holder *gh)
+ {
+ struct gfs2_glock *gl = gh->gh_gl;
+ struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+ unsigned delay = 0;
+ int fast_path = 0;
+
+- spin_lock(&gl->gl_lockref.lock);
+ /*
+- * If we're in the process of file system withdraw, we cannot just
+- * dequeue any glocks until our journal is recovered, lest we
+- * introduce file system corruption. We need two exceptions to this
+- * rule: We need to allow unlocking of nondisk glocks and the glock
+- * for our own journal that needs recovery.
++ * This while loop is similar to function demote_incompat_holders:
++ * If the glock is due to be demoted (which may be from another node
++ * or even if this holder is GL_NOCACHE), the weak holders are
++ * demoted as well, allowing the glock to be demoted.
+ */
+- if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
+- glock_blocked_by_withdraw(gl) &&
+- gh->gh_gl != sdp->sd_jinode_gl) {
+- sdp->sd_glock_dqs_held++;
+- spin_unlock(&gl->gl_lockref.lock);
+- might_sleep();
+- wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
+- TASK_UNINTERRUPTIBLE);
+- spin_lock(&gl->gl_lockref.lock);
+- }
+- if (gh->gh_flags & GL_NOCACHE)
+- handle_callback(gl, LM_ST_UNLOCKED, 0, false);
++ while (gh) {
++ /*
++ * If we're in the process of file system withdraw, we cannot
++ * just dequeue any glocks until our journal is recovered, lest
++ * we introduce file system corruption. We need two exceptions
++ * to this rule: We need to allow unlocking of nondisk glocks
++ * and the glock for our own journal that needs recovery.
++ */
++ if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
++ glock_blocked_by_withdraw(gl) &&
++ gh->gh_gl != sdp->sd_jinode_gl) {
++ sdp->sd_glock_dqs_held++;
++ spin_unlock(&gl->gl_lockref.lock);
++ might_sleep();
++ wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
++ TASK_UNINTERRUPTIBLE);
++ spin_lock(&gl->gl_lockref.lock);
++ }
+
+- list_del_init(&gh->gh_list);
+- clear_bit(HIF_HOLDER, &gh->gh_iflags);
+- if (list_empty(&gl->gl_holders) &&
+- !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
+- !test_bit(GLF_DEMOTE, &gl->gl_flags))
+- fast_path = 1;
++ /*
++ * This holder should not be cached, so mark it for demote.
++ * Note: this should be done before the check for needs_demote
++ * below.
++ */
++ if (gh->gh_flags & GL_NOCACHE)
++ handle_callback(gl, LM_ST_UNLOCKED, 0, false);
++
++ list_del_init(&gh->gh_list);
++ clear_bit(HIF_HOLDER, &gh->gh_iflags);
++ trace_gfs2_glock_queue(gh, 0);
++
++ /*
++ * If there hasn't been a demote request we are done.
++ * (Let the remaining holders, if any, keep holding it.)
++ */
++ if (!needs_demote(gl)) {
++ if (list_empty(&gl->gl_holders))
++ fast_path = 1;
++ break;
++ }
++ /*
++ * If we have another strong holder (we cannot auto-demote)
++ * we are done. It keeps holding it until it is done.
++ */
++ if (find_first_strong_holder(gl))
++ break;
++
++ /*
++ * If we have a weak holder at the head of the list, it
++ * (and all others like it) must be auto-demoted. If there
++ * are no more weak holders, we exit the while loop.
++ */
++ gh = find_first_holder(gl);
++ }
+
+ if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
+ gfs2_glock_add_to_lru(gl);
+
+- trace_gfs2_glock_queue(gh, 0);
+ if (unlikely(!fast_path)) {
+ gl->gl_lockref.count++;
+ if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
+@@ -1511,6 +1635,19 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
+ delay = gl->gl_hold_time;
+ __gfs2_glock_queue_work(gl, delay);
+ }
++}
++
++/**
++ * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
++ * @gh: the glock holder
++ *
++ */
++void gfs2_glock_dq(struct gfs2_holder *gh)
++{
++ struct gfs2_glock *gl = gh->gh_gl;
++
++ spin_lock(&gl->gl_lockref.lock);
++ __gfs2_glock_dq(gh);
+ spin_unlock(&gl->gl_lockref.lock);
+ }
+
+@@ -1673,6 +1810,7 @@ void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
+
+ void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
+ {
++ struct gfs2_holder mock_gh = { .gh_gl = gl, .gh_state = state, };
+ unsigned long delay = 0;
+ unsigned long holdtime;
+ unsigned long now = jiffies;
+@@ -1687,6 +1825,28 @@ void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
+ if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
+ delay = gl->gl_hold_time;
+ }
++ /*
++ * Note 1: We cannot call demote_incompat_holders from handle_callback
++ * or gfs2_set_demote due to recursion problems like: gfs2_glock_dq ->
++ * handle_callback -> demote_incompat_holders -> gfs2_glock_dq
++ * Plus, we only want to demote the holders if the request comes from
++ * a remote cluster node because local holder conflicts are resolved
++ * elsewhere.
++ *
++ * Note 2: if a remote node wants this glock in EX mode, lock_dlm will
++ * request that we set our state to UNLOCKED. Here we mock up a holder
++ * to make it look like someone wants the lock EX locally. Any SH
++ * and DF requests should be able to share the lock without demoting.
++ *
++ * Note 3: We only want to demote the demoteable holders when there
++ * are no more strong holders. The demoteable holders might as well
++ * keep the glock until the last strong holder is done with it.
++ */
++ if (!find_first_strong_holder(gl)) {
++ if (state == LM_ST_UNLOCKED)
++ mock_gh.gh_state = LM_ST_EXCLUSIVE;
++ demote_incompat_holders(gl, &mock_gh);
++ }
+ handle_callback(gl, state, delay, true);
+ __gfs2_glock_queue_work(gl, delay);
+ spin_unlock(&gl->gl_lockref.lock);
+@@ -1893,10 +2053,10 @@ static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
+ do {
+ rhashtable_walk_start(&iter);
+
+- while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl))
+- if (gl->gl_name.ln_sbd == sdp &&
+- lockref_get_not_dead(&gl->gl_lockref))
++ while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
++ if (gl->gl_name.ln_sbd == sdp)
+ examiner(gl);
++ }
+
+ rhashtable_walk_stop(&iter);
+ } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
+@@ -1919,7 +2079,7 @@ bool gfs2_queue_delete_work(struct gfs2_glock *gl, unsigned long delay)
+
+ void gfs2_cancel_delete_work(struct gfs2_glock *gl)
+ {
+- if (cancel_delayed_work_sync(&gl->gl_delete)) {
++ if (cancel_delayed_work(&gl->gl_delete)) {
+ clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
+ gfs2_glock_put(gl);
+ }
+@@ -1938,7 +2098,6 @@ static void flush_delete_work(struct gfs2_glock *gl)
+ &gl->gl_delete, 0);
+ }
+ }
+- gfs2_glock_queue_work(gl, 0);
+ }
+
+ void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
+@@ -1955,10 +2114,10 @@ void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
+
+ static void thaw_glock(struct gfs2_glock *gl)
+ {
+- if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) {
+- gfs2_glock_put(gl);
++ if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))
++ return;
++ if (!lockref_get_not_dead(&gl->gl_lockref))
+ return;
+- }
+ set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
+ gfs2_glock_queue_work(gl, 0);
+ }
+@@ -1974,9 +2133,12 @@ static void clear_glock(struct gfs2_glock *gl)
+ gfs2_glock_remove_from_lru(gl);
+
+ spin_lock(&gl->gl_lockref.lock);
+- if (gl->gl_state != LM_ST_UNLOCKED)
+- handle_callback(gl, LM_ST_UNLOCKED, 0, false);
+- __gfs2_glock_queue_work(gl, 0);
++ if (!__lockref_is_dead(&gl->gl_lockref)) {
++ gl->gl_lockref.count++;
++ if (gl->gl_state != LM_ST_UNLOCKED)
++ handle_callback(gl, LM_ST_UNLOCKED, 0, false);
++ __gfs2_glock_queue_work(gl, 0);
++ }
+ spin_unlock(&gl->gl_lockref.lock);
+ }
+
+@@ -2076,6 +2238,8 @@ static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
+ *p++ = 'H';
+ if (test_bit(HIF_WAIT, &iflags))
+ *p++ = 'W';
++ if (test_bit(HIF_MAY_DEMOTE, &iflags))
++ *p++ = 'D';
+ *p = 0;
+ return buf;
+ }
+diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
+index 31a8f2f649b52..9012487da4c69 100644
+--- a/fs/gfs2/glock.h
++++ b/fs/gfs2/glock.h
+@@ -150,6 +150,8 @@ static inline struct gfs2_holder *gfs2_glock_is_locked_by_me(struct gfs2_glock *
+ list_for_each_entry(gh, &gl->gl_holders, gh_list) {
+ if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
+ break;
++ if (test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags))
++ continue;
+ if (gh->gh_owner_pid == pid)
+ goto out;
+ }
+@@ -325,6 +327,24 @@ static inline void glock_clear_object(struct gfs2_glock *gl, void *object)
+ spin_unlock(&gl->gl_lockref.lock);
+ }
+
++static inline void gfs2_holder_allow_demote(struct gfs2_holder *gh)
++{
++ struct gfs2_glock *gl = gh->gh_gl;
++
++ spin_lock(&gl->gl_lockref.lock);
++ set_bit(HIF_MAY_DEMOTE, &gh->gh_iflags);
++ spin_unlock(&gl->gl_lockref.lock);
++}
++
++static inline void gfs2_holder_disallow_demote(struct gfs2_holder *gh)
++{
++ struct gfs2_glock *gl = gh->gh_gl;
++
++ spin_lock(&gl->gl_lockref.lock);
++ clear_bit(HIF_MAY_DEMOTE, &gh->gh_iflags);
++ spin_unlock(&gl->gl_lockref.lock);
++}
++
+ extern void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation);
+ extern bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation);
+
+diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
+index 0fe49770166ea..ca42d310fd4d6 100644
+--- a/fs/gfs2/incore.h
++++ b/fs/gfs2/incore.h
+@@ -252,6 +252,7 @@ struct gfs2_lkstats {
+
+ enum {
+ /* States */
++ HIF_MAY_DEMOTE = 1,
+ HIF_HOLDER = 6, /* Set for gh that "holds" the glock */
+ HIF_WAIT = 10,
+ };
+@@ -386,9 +387,8 @@ struct gfs2_inode {
+ u64 i_generation;
+ u64 i_eattr;
+ unsigned long i_flags; /* GIF_... */
+- struct gfs2_glock *i_gl; /* Move into i_gh? */
++ struct gfs2_glock *i_gl;
+ struct gfs2_holder i_iopen_gh;
+- struct gfs2_holder i_gh; /* for prepare/commit_write only */
+ struct gfs2_qadata *i_qadata; /* quota allocation data */
+ struct gfs2_holder i_rgd_gh;
+ struct gfs2_blkreserv i_res; /* rgrp multi-block reservation */
+diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
+index 3130f85d2b3f4..97ee17843b4d0 100644
+--- a/fs/gfs2/inode.c
++++ b/fs/gfs2/inode.c
+@@ -811,7 +811,7 @@ fail_free_inode:
+ if (free_vfs_inode) /* else evict will do the put for us */
+ gfs2_glock_put(ip->i_gl);
+ }
+- gfs2_rs_delete(ip, NULL);
++ gfs2_rs_deltree(&ip->i_res);
+ gfs2_qa_put(ip);
+ fail_free_acls:
+ posix_acl_release(default_acl);
+diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c
+index c3b00ba92ed2e..6901cd85f1df7 100644
+--- a/fs/gfs2/rgrp.c
++++ b/fs/gfs2/rgrp.c
+@@ -680,13 +680,14 @@ void gfs2_rs_deltree(struct gfs2_blkreserv *rs)
+ /**
+ * gfs2_rs_delete - delete a multi-block reservation
+ * @ip: The inode for this reservation
+- * @wcount: The inode's write count, or NULL
+ *
+ */
+-void gfs2_rs_delete(struct gfs2_inode *ip, atomic_t *wcount)
++void gfs2_rs_delete(struct gfs2_inode *ip)
+ {
++ struct inode *inode = &ip->i_inode;
++
+ down_write(&ip->i_rw_mutex);
+- if ((wcount == NULL) || (atomic_read(wcount) <= 1))
++ if (atomic_read(&inode->i_writecount) <= 1)
+ gfs2_rs_deltree(&ip->i_res);
+ up_write(&ip->i_rw_mutex);
+ }
+@@ -922,15 +923,15 @@ static int read_rindex_entry(struct gfs2_inode *ip)
+ spin_lock_init(&rgd->rd_rsspin);
+ mutex_init(&rgd->rd_mutex);
+
+- error = compute_bitstructs(rgd);
+- if (error)
+- goto fail;
+-
+ error = gfs2_glock_get(sdp, rgd->rd_addr,
+ &gfs2_rgrp_glops, CREATE, &rgd->rd_gl);
+ if (error)
+ goto fail;
+
++ error = compute_bitstructs(rgd);
++ if (error)
++ goto fail_glock;
++
+ rgd->rd_rgl = (struct gfs2_rgrp_lvb *)rgd->rd_gl->gl_lksb.sb_lvbptr;
+ rgd->rd_flags &= ~(GFS2_RDF_UPTODATE | GFS2_RDF_PREFERRED);
+ if (rgd->rd_data > sdp->sd_max_rg_data)
+@@ -944,6 +945,7 @@ static int read_rindex_entry(struct gfs2_inode *ip)
+ }
+
+ error = 0; /* someone else read in the rgrp; free it and ignore it */
++fail_glock:
+ gfs2_glock_put(rgd->rd_gl);
+
+ fail:
+@@ -1427,7 +1429,8 @@ int gfs2_fitrim(struct file *filp, void __user *argp)
+
+ start = r.start >> bs_shift;
+ end = start + (r.len >> bs_shift);
+- minlen = max_t(u64, r.minlen,
++ minlen = max_t(u64, r.minlen, sdp->sd_sb.sb_bsize);
++ minlen = max_t(u64, minlen,
+ q->limits.discard_granularity) >> bs_shift;
+
+ if (end <= start || minlen > sdp->sd_max_rg_data)
+diff --git a/fs/gfs2/rgrp.h b/fs/gfs2/rgrp.h
+index a6855fd796e03..2f80f3bbf8767 100644
+--- a/fs/gfs2/rgrp.h
++++ b/fs/gfs2/rgrp.h
+@@ -45,7 +45,7 @@ extern int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *n,
+ bool dinode, u64 *generation);
+
+ extern void gfs2_rs_deltree(struct gfs2_blkreserv *rs);
+-extern void gfs2_rs_delete(struct gfs2_inode *ip, atomic_t *wcount);
++extern void gfs2_rs_delete(struct gfs2_inode *ip);
+ extern void __gfs2_free_blocks(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd,
+ u64 bstart, u32 blen, int meta);
+ extern void gfs2_free_meta(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd,
+diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
+index 6e00d15ef0a82..0f2e0530dd433 100644
+--- a/fs/gfs2/super.c
++++ b/fs/gfs2/super.c
+@@ -1398,17 +1398,10 @@ out:
+ truncate_inode_pages_final(&inode->i_data);
+ if (ip->i_qadata)
+ gfs2_assert_warn(sdp, ip->i_qadata->qa_ref == 0);
+- gfs2_rs_delete(ip, NULL);
++ gfs2_rs_deltree(&ip->i_res);
+ gfs2_ordered_del_inode(ip);
+ clear_inode(inode);
+ gfs2_dir_hash_inval(ip);
+- if (ip->i_gl) {
+- glock_clear_object(ip->i_gl, ip);
+- wait_on_bit_io(&ip->i_flags, GIF_GLOP_PENDING, TASK_UNINTERRUPTIBLE);
+- gfs2_glock_add_to_lru(ip->i_gl);
+- gfs2_glock_put_eventually(ip->i_gl);
+- ip->i_gl = NULL;
+- }
+ if (gfs2_holder_initialized(&ip->i_iopen_gh)) {
+ struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl;
+
+@@ -1421,6 +1414,13 @@ out:
+ gfs2_holder_uninit(&ip->i_iopen_gh);
+ gfs2_glock_put_eventually(gl);
+ }
++ if (ip->i_gl) {
++ glock_clear_object(ip->i_gl, ip);
++ wait_on_bit_io(&ip->i_flags, GIF_GLOP_PENDING, TASK_UNINTERRUPTIBLE);
++ gfs2_glock_add_to_lru(ip->i_gl);
++ gfs2_glock_put_eventually(ip->i_gl);
++ ip->i_gl = NULL;
++ }
+ }
+
+ static struct inode *gfs2_alloc_inode(struct super_block *sb)
+diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
+index cdfb1ae78a3f8..bb0651a4a1287 100644
+--- a/fs/hugetlbfs/inode.c
++++ b/fs/hugetlbfs/inode.c
+@@ -206,7 +206,7 @@ hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr,
+ info.flags = 0;
+ info.length = len;
+ info.low_limit = current->mm->mmap_base;
+- info.high_limit = TASK_SIZE;
++ info.high_limit = arch_get_mmap_end(addr);
+ info.align_mask = PAGE_MASK & ~huge_page_mask(h);
+ info.align_offset = 0;
+ return vm_unmapped_area(&info);
+@@ -222,7 +222,7 @@ hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr,
+ info.flags = VM_UNMAPPED_AREA_TOPDOWN;
+ info.length = len;
+ info.low_limit = max(PAGE_SIZE, mmap_min_addr);
+- info.high_limit = current->mm->mmap_base;
++ info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base);
+ info.align_mask = PAGE_MASK & ~huge_page_mask(h);
+ info.align_offset = 0;
+ addr = vm_unmapped_area(&info);
+@@ -237,7 +237,7 @@ hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr,
+ VM_BUG_ON(addr != -ENOMEM);
+ info.flags = 0;
+ info.low_limit = current->mm->mmap_base;
+- info.high_limit = TASK_SIZE;
++ info.high_limit = arch_get_mmap_end(addr);
+ addr = vm_unmapped_area(&info);
+ }
+
+@@ -251,6 +251,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
+ struct mm_struct *mm = current->mm;
+ struct vm_area_struct *vma;
+ struct hstate *h = hstate_file(file);
++ const unsigned long mmap_end = arch_get_mmap_end(addr);
+
+ if (len & ~huge_page_mask(h))
+ return -EINVAL;
+@@ -266,7 +267,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
+ if (addr) {
+ addr = ALIGN(addr, huge_page_size(h));
+ vma = find_vma(mm, addr);
+- if (TASK_SIZE - len >= addr &&
++ if (mmap_end - len >= addr &&
+ (!vma || addr + len <= vm_start_gap(vma)))
+ return addr;
+ }
+@@ -409,10 +410,11 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
+ struct vm_area_struct *vma;
+
+ /*
+- * end == 0 indicates that the entire range after
+- * start should be unmapped.
++ * end == 0 indicates that the entire range after start should be
++ * unmapped. Note, end is exclusive, whereas the interval tree takes
++ * an inclusive "last".
+ */
+- vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
++ vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
+ unsigned long v_offset;
+ unsigned long v_end;
+
+diff --git a/fs/inode.c b/fs/inode.c
+index ed0cab8a32db1..9abc88d7959cb 100644
+--- a/fs/inode.c
++++ b/fs/inode.c
+@@ -1782,12 +1782,13 @@ EXPORT_SYMBOL(generic_update_time);
+ * This does the actual work of updating an inodes time or version. Must have
+ * had called mnt_want_write() before calling this.
+ */
+-static int update_time(struct inode *inode, struct timespec64 *time, int flags)
++int inode_update_time(struct inode *inode, struct timespec64 *time, int flags)
+ {
+ if (inode->i_op->update_time)
+ return inode->i_op->update_time(inode, time, flags);
+ return generic_update_time(inode, time, flags);
+ }
++EXPORT_SYMBOL(inode_update_time);
+
+ /**
+ * atime_needs_update - update the access time
+@@ -1857,7 +1858,7 @@ void touch_atime(const struct path *path)
+ * of the fs read only, e.g. subvolumes in Btrfs.
+ */
+ now = current_time(inode);
+- update_time(inode, &now, S_ATIME);
++ inode_update_time(inode, &now, S_ATIME);
+ __mnt_drop_write(mnt);
+ skip_update:
+ sb_end_write(inode->i_sb);
+@@ -2002,7 +2003,7 @@ int file_update_time(struct file *file)
+ if (__mnt_want_write_file(file))
+ return 0;
+
+- ret = update_time(inode, &now, sync_it);
++ ret = inode_update_time(inode, &now, sync_it);
+ __mnt_drop_write_file(file);
+
+ return ret;
+diff --git a/fs/internal.h b/fs/internal.h
+index 3cd065c8a66b4..cdd83d4899bb3 100644
+--- a/fs/internal.h
++++ b/fs/internal.h
+@@ -23,22 +23,11 @@ struct pipe_inode_info;
+ #ifdef CONFIG_BLOCK
+ extern void __init bdev_cache_init(void);
+
+-extern int __sync_blockdev(struct block_device *bdev, int wait);
+-void iterate_bdevs(void (*)(struct block_device *, void *), void *);
+ void emergency_thaw_bdev(struct super_block *sb);
+ #else
+ static inline void bdev_cache_init(void)
+ {
+ }
+-
+-static inline int __sync_blockdev(struct block_device *bdev, int wait)
+-{
+- return 0;
+-}
+-static inline void iterate_bdevs(void (*f)(struct block_device *, void *),
+- void *arg)
+-{
+-}
+ static inline int emergency_thaw_bdev(struct super_block *sb)
+ {
+ return 0;
+diff --git a/fs/io-wq.c b/fs/io-wq.c
+index 422a7ed6a9bdb..01d4e3ff3170d 100644
+--- a/fs/io-wq.c
++++ b/fs/io-wq.c
+@@ -140,6 +140,8 @@ static void io_wqe_dec_running(struct io_worker *worker);
+ static bool io_acct_cancel_pending_work(struct io_wqe *wqe,
+ struct io_wqe_acct *acct,
+ struct io_cb_cancel_data *match);
++static void create_worker_cb(struct callback_head *cb);
++static void io_wq_cancel_tw_create(struct io_wq *wq);
+
+ static bool io_worker_get(struct io_worker *worker)
+ {
+@@ -174,9 +176,44 @@ static void io_worker_ref_put(struct io_wq *wq)
+ complete(&wq->worker_done);
+ }
+
++static void io_worker_cancel_cb(struct io_worker *worker)
++{
++ struct io_wqe_acct *acct = io_wqe_get_acct(worker);
++ struct io_wqe *wqe = worker->wqe;
++ struct io_wq *wq = wqe->wq;
++
++ atomic_dec(&acct->nr_running);
++ raw_spin_lock(&worker->wqe->lock);
++ acct->nr_workers--;
++ raw_spin_unlock(&worker->wqe->lock);
++ io_worker_ref_put(wq);
++ clear_bit_unlock(0, &worker->create_state);
++ io_worker_release(worker);
++}
++
++static bool io_task_worker_match(struct callback_head *cb, void *data)
++{
++ struct io_worker *worker;
++
++ if (cb->func != create_worker_cb)
++ return false;
++ worker = container_of(cb, struct io_worker, create_work);
++ return worker == data;
++}
++
+ static void io_worker_exit(struct io_worker *worker)
+ {
+ struct io_wqe *wqe = worker->wqe;
++ struct io_wq *wq = wqe->wq;
++
++ while (1) {
++ struct callback_head *cb = task_work_cancel_match(wq->task,
++ io_task_worker_match, worker);
++
++ if (!cb)
++ break;
++ io_worker_cancel_cb(worker);
++ }
+
+ if (refcount_dec_and_test(&worker->ref))
+ complete(&worker->ref_done);
+@@ -321,10 +358,22 @@ static bool io_queue_worker_create(struct io_worker *worker,
+ test_and_set_bit_lock(0, &worker->create_state))
+ goto fail_release;
+
++ atomic_inc(&wq->worker_refs);
+ init_task_work(&worker->create_work, func);
+ worker->create_index = acct->index;
+- if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL))
++ if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) {
++ /*
++ * EXIT may have been set after checking it above, check after
++ * adding the task_work and remove any creation item if it is
++ * now set. wq exit does that too, but we can have added this
++ * work item after we canceled in io_wq_exit_workers().
++ */
++ if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
++ io_wq_cancel_tw_create(wq);
++ io_worker_ref_put(wq);
+ return true;
++ }
++ io_worker_ref_put(wq);
+ clear_bit_unlock(0, &worker->create_state);
+ fail_release:
+ io_worker_release(worker);
+@@ -346,7 +395,9 @@ static void io_wqe_dec_running(struct io_worker *worker)
+ if (atomic_dec_and_test(&acct->nr_running) && io_acct_run_queue(acct)) {
+ atomic_inc(&acct->nr_running);
+ atomic_inc(&wqe->wq->worker_refs);
++ raw_spin_unlock(&wqe->lock);
+ io_queue_worker_create(worker, acct, create_worker_cb);
++ raw_spin_lock(&wqe->lock);
+ }
+ }
+
+@@ -385,9 +436,10 @@ static inline unsigned int io_get_work_hash(struct io_wq_work *work)
+ return work->flags >> IO_WQ_HASH_SHIFT;
+ }
+
+-static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)
++static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)
+ {
+ struct io_wq *wq = wqe->wq;
++ bool ret = false;
+
+ spin_lock_irq(&wq->hash->wait.lock);
+ if (list_empty(&wqe->wait.entry)) {
+@@ -395,9 +447,11 @@ static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)
+ if (!test_bit(hash, &wq->hash->map)) {
+ __set_current_state(TASK_RUNNING);
+ list_del_init(&wqe->wait.entry);
++ ret = true;
+ }
+ }
+ spin_unlock_irq(&wq->hash->wait.lock);
++ return ret;
+ }
+
+ static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,
+@@ -437,14 +491,21 @@ static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,
+ }
+
+ if (stall_hash != -1U) {
++ bool unstalled;
++
+ /*
+ * Set this before dropping the lock to avoid racing with new
+ * work being added and clearing the stalled bit.
+ */
+ set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
+ raw_spin_unlock(&wqe->lock);
+- io_wait_on_hash(wqe, stall_hash);
++ unstalled = io_wait_on_hash(wqe, stall_hash);
+ raw_spin_lock(&wqe->lock);
++ if (unstalled) {
++ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
++ if (wq_has_sleeper(&wqe->wq->hash->wait))
++ wake_up(&wqe->wq->hash->wait);
++ }
+ }
+
+ return NULL;
+@@ -526,8 +587,11 @@ get_next:
+ io_wqe_enqueue(wqe, linked);
+
+ if (hash != -1U && !next_hashed) {
++ /* serialize hash clear with wake_up() */
++ spin_lock_irq(&wq->hash->wait.lock);
+ clear_bit(hash, &wq->hash->map);
+ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
++ spin_unlock_irq(&wq->hash->wait.lock);
+ if (wq_has_sleeper(&wq->hash->wait))
+ wake_up(&wq->hash->wait);
+ raw_spin_lock(&wqe->lock);
+@@ -660,6 +724,13 @@ static bool io_wq_work_match_all(struct io_wq_work *work, void *data)
+
+ static inline bool io_should_retry_thread(long err)
+ {
++ /*
++ * Prevent perpetual task_work retry, if the task (or its group) is
++ * exiting.
++ */
++ if (fatal_signal_pending(current))
++ return false;
++
+ switch (err) {
+ case -EAGAIN:
+ case -ERESTARTSYS:
+@@ -716,11 +787,8 @@ static void io_workqueue_create(struct work_struct *work)
+ struct io_worker *worker = container_of(work, struct io_worker, work);
+ struct io_wqe_acct *acct = io_wqe_get_acct(worker);
+
+- if (!io_queue_worker_create(worker, acct, create_worker_cont)) {
+- clear_bit_unlock(0, &worker->create_state);
+- io_worker_release(worker);
++ if (!io_queue_worker_create(worker, acct, create_worker_cont))
+ kfree(worker);
+- }
+ }
+
+ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index)
+@@ -1140,28 +1208,26 @@ void io_wq_exit_start(struct io_wq *wq)
+ set_bit(IO_WQ_BIT_EXIT, &wq->state);
+ }
+
+-static void io_wq_exit_workers(struct io_wq *wq)
++static void io_wq_cancel_tw_create(struct io_wq *wq)
+ {
+ struct callback_head *cb;
+- int node;
+-
+- if (!wq->task)
+- return;
+
+ while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) {
+ struct io_worker *worker;
+- struct io_wqe_acct *acct;
+
+ worker = container_of(cb, struct io_worker, create_work);
+- acct = io_wqe_get_acct(worker);
+- atomic_dec(&acct->nr_running);
+- raw_spin_lock(&worker->wqe->lock);
+- acct->nr_workers--;
+- raw_spin_unlock(&worker->wqe->lock);
+- io_worker_ref_put(wq);
+- clear_bit_unlock(0, &worker->create_state);
+- io_worker_release(worker);
++ io_worker_cancel_cb(worker);
+ }
++}
++
++static void io_wq_exit_workers(struct io_wq *wq)
++{
++ int node;
++
++ if (!wq->task)
++ return;
++
++ io_wq_cancel_tw_create(wq);
+
+ rcu_read_lock();
+ for_each_node(node) {
+@@ -1278,7 +1344,9 @@ int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask)
+ */
+ int io_wq_max_workers(struct io_wq *wq, int *new_count)
+ {
+- int i, node, prev = 0;
++ int prev[IO_WQ_ACCT_NR];
++ bool first_node = true;
++ int i, node;
+
+ BUILD_BUG_ON((int) IO_WQ_ACCT_BOUND != (int) IO_WQ_BOUND);
+ BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND);
+@@ -1289,6 +1357,9 @@ int io_wq_max_workers(struct io_wq *wq, int *new_count)
+ new_count[i] = task_rlimit(current, RLIMIT_NPROC);
+ }
+
++ for (i = 0; i < IO_WQ_ACCT_NR; i++)
++ prev[i] = 0;
++
+ rcu_read_lock();
+ for_each_node(node) {
+ struct io_wqe *wqe = wq->wqes[node];
+@@ -1297,14 +1368,19 @@ int io_wq_max_workers(struct io_wq *wq, int *new_count)
+ raw_spin_lock(&wqe->lock);
+ for (i = 0; i < IO_WQ_ACCT_NR; i++) {
+ acct = &wqe->acct[i];
+- prev = max_t(int, acct->max_workers, prev);
++ if (first_node)
++ prev[i] = max_t(int, acct->max_workers, prev[i]);
+ if (new_count[i])
+ acct->max_workers = new_count[i];
+- new_count[i] = prev;
+ }
+ raw_spin_unlock(&wqe->lock);
++ first_node = false;
+ }
+ rcu_read_unlock();
++
++ for (i = 0; i < IO_WQ_ACCT_NR; i++)
++ new_count[i] = prev[i];
++
+ return 0;
+ }
+
+diff --git a/fs/io_uring.c b/fs/io_uring.c
+index bc18af5e0a934..b8e6398d94307 100644
+--- a/fs/io_uring.c
++++ b/fs/io_uring.c
+@@ -623,10 +623,10 @@ struct io_epoll {
+
+ struct io_splice {
+ struct file *file_out;
+- struct file *file_in;
+ loff_t off_out;
+ loff_t off_in;
+ u64 len;
++ int splice_fd_in;
+ unsigned int flags;
+ };
+
+@@ -1204,6 +1204,7 @@ static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
+
+ static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
+ bool cancel_all)
++ __must_hold(&req->ctx->timeout_lock)
+ {
+ struct io_kiocb *req;
+
+@@ -1219,6 +1220,44 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
+ return false;
+ }
+
++static bool io_match_linked(struct io_kiocb *head)
++{
++ struct io_kiocb *req;
++
++ io_for_each_link(req, head) {
++ if (req->flags & REQ_F_INFLIGHT)
++ return true;
++ }
++ return false;
++}
++
++/*
++ * As io_match_task() but protected against racing with linked timeouts.
++ * User must not hold timeout_lock.
++ */
++static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
++ bool cancel_all)
++{
++ bool matched;
++
++ if (task && head->task != task)
++ return false;
++ if (cancel_all)
++ return true;
++
++ if (head->flags & REQ_F_LINK_TIMEOUT) {
++ struct io_ring_ctx *ctx = head->ctx;
++
++ /* protect against races with linked timeouts */
++ spin_lock_irq(&ctx->timeout_lock);
++ matched = io_match_linked(head);
++ spin_unlock_irq(&ctx->timeout_lock);
++ } else {
++ matched = io_match_linked(head);
++ }
++ return matched;
++}
++
+ static inline void req_set_fail(struct io_kiocb *req)
+ {
+ req->flags |= REQ_F_FAIL;
+@@ -1413,14 +1452,6 @@ static void io_prep_async_work(struct io_kiocb *req)
+ if (def->unbound_nonreg_file)
+ req->work.flags |= IO_WQ_WORK_UNBOUND;
+ }
+-
+- switch (req->opcode) {
+- case IORING_OP_SPLICE:
+- case IORING_OP_TEE:
+- if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
+- req->work.flags |= IO_WQ_WORK_UNBOUND;
+- break;
+- }
+ }
+
+ static void io_prep_async_link(struct io_kiocb *req)
+@@ -1430,10 +1461,10 @@ static void io_prep_async_link(struct io_kiocb *req)
+ if (req->flags & REQ_F_LINK_TIMEOUT) {
+ struct io_ring_ctx *ctx = req->ctx;
+
+- spin_lock(&ctx->completion_lock);
++ spin_lock_irq(&ctx->timeout_lock);
+ io_for_each_link(cur, req)
+ io_prep_async_work(cur);
+- spin_unlock(&ctx->completion_lock);
++ spin_unlock_irq(&ctx->timeout_lock);
+ } else {
+ io_for_each_link(cur, req)
+ io_prep_async_work(cur);
+@@ -1507,12 +1538,11 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx)
+ __must_hold(&ctx->completion_lock)
+ {
+ u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
++ struct io_kiocb *req, *tmp;
+
+ spin_lock_irq(&ctx->timeout_lock);
+- while (!list_empty(&ctx->timeout_list)) {
++ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
+ u32 events_needed, events_got;
+- struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
+- struct io_kiocb, timeout.list);
+
+ if (io_is_timeout_noseq(req))
+ break;
+@@ -1529,7 +1559,6 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx)
+ if (events_got < events_needed)
+ break;
+
+- list_del_init(&req->timeout.list);
+ io_kill_timeout(req, 0);
+ }
+ ctx->cq_last_tm_flush = seq;
+@@ -1721,6 +1750,18 @@ static inline void io_get_task_refs(int nr)
+ io_task_refs_refill(tctx);
+ }
+
++static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
++{
++ struct io_uring_task *tctx = task->io_uring;
++ unsigned int refs = tctx->cached_refs;
++
++ if (refs) {
++ tctx->cached_refs = 0;
++ percpu_counter_sub(&tctx->inflight, refs);
++ put_task_struct_many(task, refs);
++ }
++}
++
+ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
+ long res, unsigned int cflags)
+ {
+@@ -2161,6 +2202,10 @@ static void tctx_task_work(struct callback_head *cb)
+ }
+
+ ctx_flush_and_put(ctx, &locked);
++
++ /* relaxed read is enough as only the task itself sets ->in_idle */
++ if (unlikely(atomic_read(&tctx->in_idle)))
++ io_uring_drop_tctx_refs(current);
+ }
+
+ static void io_req_task_work_add(struct io_kiocb *req)
+@@ -2840,9 +2885,13 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ req->flags |= REQ_F_ISREG;
+
+ kiocb->ki_pos = READ_ONCE(sqe->off);
+- if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
+- req->flags |= REQ_F_CUR_POS;
+- kiocb->ki_pos = file->f_pos;
++ if (kiocb->ki_pos == -1) {
++ if (!(file->f_mode & FMODE_STREAM)) {
++ req->flags |= REQ_F_CUR_POS;
++ kiocb->ki_pos = file->f_pos;
++ } else {
++ kiocb->ki_pos = 0;
++ }
+ }
+ kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+ kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+@@ -3260,13 +3309,15 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
+ ret = nr;
+ break;
+ }
++ ret += nr;
+ if (!iov_iter_is_bvec(iter)) {
+ iov_iter_advance(iter, nr);
+ } else {
+- req->rw.len -= nr;
+ req->rw.addr += nr;
++ req->rw.len -= nr;
++ if (!req->rw.len)
++ break;
+ }
+- ret += nr;
+ if (nr != iovec.iov_len)
+ break;
+ }
+@@ -3966,18 +4017,11 @@ static int __io_splice_prep(struct io_kiocb *req,
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+
+- sp->file_in = NULL;
+ sp->len = READ_ONCE(sqe->len);
+ sp->flags = READ_ONCE(sqe->splice_flags);
+-
+ if (unlikely(sp->flags & ~valid_flags))
+ return -EINVAL;
+-
+- sp->file_in = io_file_get(req->ctx, req, READ_ONCE(sqe->splice_fd_in),
+- (sp->flags & SPLICE_F_FD_IN_FIXED));
+- if (!sp->file_in)
+- return -EBADF;
+- req->flags |= REQ_F_NEED_CLEANUP;
++ sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
+ return 0;
+ }
+
+@@ -3992,20 +4036,27 @@ static int io_tee_prep(struct io_kiocb *req,
+ static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
+ {
+ struct io_splice *sp = &req->splice;
+- struct file *in = sp->file_in;
+ struct file *out = sp->file_out;
+ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
++ struct file *in;
+ long ret = 0;
+
+ if (issue_flags & IO_URING_F_NONBLOCK)
+ return -EAGAIN;
++
++ in = io_file_get(req->ctx, req, sp->splice_fd_in,
++ (sp->flags & SPLICE_F_FD_IN_FIXED));
++ if (!in) {
++ ret = -EBADF;
++ goto done;
++ }
++
+ if (sp->len)
+ ret = do_tee(in, out, sp->len, flags);
+
+ if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
+ io_put_file(in);
+- req->flags &= ~REQ_F_NEED_CLEANUP;
+-
++done:
+ if (ret != sp->len)
+ req_set_fail(req);
+ io_req_complete(req, ret);
+@@ -4024,15 +4075,22 @@ static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
+ {
+ struct io_splice *sp = &req->splice;
+- struct file *in = sp->file_in;
+ struct file *out = sp->file_out;
+ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
+ loff_t *poff_in, *poff_out;
++ struct file *in;
+ long ret = 0;
+
+ if (issue_flags & IO_URING_F_NONBLOCK)
+ return -EAGAIN;
+
++ in = io_file_get(req->ctx, req, sp->splice_fd_in,
++ (sp->flags & SPLICE_F_FD_IN_FIXED));
++ if (!in) {
++ ret = -EBADF;
++ goto done;
++ }
++
+ poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
+ poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
+
+@@ -4041,8 +4099,7 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
+
+ if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
+ io_put_file(in);
+- req->flags &= ~REQ_F_NEED_CLEANUP;
+-
++done:
+ if (ret != sp->len)
+ req_set_fail(req);
+ io_req_complete(req, ret);
+@@ -4067,9 +4124,6 @@ static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ struct io_ring_ctx *ctx = req->ctx;
+
+- if (!req->file)
+- return -EBADF;
+-
+ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
+@@ -4304,6 +4358,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
+ kfree(nxt);
+ if (++i == nbufs)
+ return i;
++ cond_resched();
+ }
+ i++;
+ kfree(buf);
+@@ -4394,6 +4449,7 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
+ } else {
+ list_add_tail(&buf->list, &(*head)->list);
+ }
++ cond_resched();
+ }
+
+ return i ? i : -ENOMEM;
+@@ -4730,6 +4786,8 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
++ if (unlikely(sqe->addr2 || sqe->file_index))
++ return -EINVAL;
+
+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
+ sr->len = READ_ONCE(sqe->len);
+@@ -4951,6 +5009,8 @@ static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
++ if (unlikely(sqe->addr2 || sqe->file_index))
++ return -EINVAL;
+
+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
+ sr->len = READ_ONCE(sqe->len);
+@@ -5093,8 +5153,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ accept->nofile = rlimit(RLIMIT_NOFILE);
+
+ accept->file_slot = READ_ONCE(sqe->file_index);
+- if (accept->file_slot && ((req->open.how.flags & O_CLOEXEC) ||
+- (accept->flags & SOCK_CLOEXEC)))
++ if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
+ return -EINVAL;
+ if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
+ return -EINVAL;
+@@ -5603,7 +5662,6 @@ static int io_arm_poll_handler(struct io_kiocb *req)
+ struct async_poll *apoll;
+ struct io_poll_table ipt;
+ __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI;
+- int rw;
+
+ if (!req->file || !file_can_poll(req->file))
+ return IO_APOLL_ABORTED;
+@@ -5613,7 +5671,6 @@ static int io_arm_poll_handler(struct io_kiocb *req)
+ return IO_APOLL_ABORTED;
+
+ if (def->pollin) {
+- rw = READ;
+ mask |= POLLIN | POLLRDNORM;
+
+ /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
+@@ -5621,14 +5678,9 @@ static int io_arm_poll_handler(struct io_kiocb *req)
+ (req->sr_msg.msg_flags & MSG_ERRQUEUE))
+ mask &= ~POLLIN;
+ } else {
+- rw = WRITE;
+ mask |= POLLOUT | POLLWRNORM;
+ }
+
+- /* if we can't nonblock try, then no point in arming a poll handler */
+- if (!io_file_supports_nowait(req, rw))
+- return IO_APOLL_ABORTED;
+-
+ apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
+ if (unlikely(!apoll))
+ return IO_APOLL_ABORTED;
+@@ -5702,7 +5754,7 @@ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
+
+ list = &ctx->cancel_hash[i];
+ hlist_for_each_entry_safe(req, tmp, list, hash_node) {
+- if (io_match_task(req, tsk, cancel_all))
++ if (io_match_task_safe(req, tsk, cancel_all))
+ posted += io_poll_remove_one(req);
+ }
+ }
+@@ -5881,6 +5933,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
+ * update those. For multishot, if we're racing with completion, just
+ * let completion re-add it.
+ */
++ io_poll_remove_double(preq);
+ completing = !__io_poll_remove_one(preq, &preq->poll, false);
+ if (completing && (preq->poll.events & EPOLLONESHOT)) {
+ ret = -EALREADY;
+@@ -6147,6 +6200,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
+ return -EFAULT;
+
++ INIT_LIST_HEAD(&req->timeout.list);
+ data->mode = io_translate_timeout_mode(flags);
+ hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
+
+@@ -6346,6 +6400,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
+ up.nr = 0;
+ up.tags = 0;
+ up.resv = 0;
++ up.resv2 = 0;
+
+ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
+ ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
+@@ -6511,11 +6566,14 @@ static bool io_drain_req(struct io_kiocb *req)
+ }
+
+ /* Still need defer if there is pending req in defer list. */
++ spin_lock(&ctx->completion_lock);
+ if (likely(list_empty_careful(&ctx->defer_list) &&
+ !(req->flags & REQ_F_IO_DRAIN))) {
++ spin_unlock(&ctx->completion_lock);
+ ctx->drain_active = false;
+ return false;
+ }
++ spin_unlock(&ctx->completion_lock);
+
+ seq = io_get_sequence(req);
+ /* Still a chance to pass the sequence check */
+@@ -6586,11 +6644,6 @@ static void io_clean_op(struct io_kiocb *req)
+ kfree(io->free_iov);
+ break;
+ }
+- case IORING_OP_SPLICE:
+- case IORING_OP_TEE:
+- if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
+- io_put_file(req->splice.file_in);
+- break;
+ case IORING_OP_OPENAT:
+ case IORING_OP_OPENAT2:
+ if (req->open.filename)
+@@ -6884,10 +6937,11 @@ static inline struct file *io_file_get(struct io_ring_ctx *ctx,
+ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
+ {
+ struct io_kiocb *prev = req->timeout.prev;
+- int ret;
++ int ret = -ENOENT;
+
+ if (prev) {
+- ret = io_try_cancel_userdata(req, prev->user_data);
++ if (!(req->task->flags & PF_EXITING))
++ ret = io_try_cancel_userdata(req, prev->user_data);
+ io_req_complete_post(req, ret ?: -ETIME, 0);
+ io_put_req(prev);
+ } else {
+@@ -7528,7 +7582,7 @@ static int io_run_task_work_sig(void)
+ /* when returns >0, the caller should retry */
+ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+ struct io_wait_queue *iowq,
+- signed long *timeout)
++ ktime_t timeout)
+ {
+ int ret;
+
+@@ -7540,8 +7594,9 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+ if (test_bit(0, &ctx->check_cq_overflow))
+ return 1;
+
+- *timeout = schedule_timeout(*timeout);
+- return !*timeout ? -ETIME : 1;
++ if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))
++ return -ETIME;
++ return 1;
+ }
+
+ /*
+@@ -7554,7 +7609,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
+ {
+ struct io_wait_queue iowq;
+ struct io_rings *rings = ctx->rings;
+- signed long timeout = MAX_SCHEDULE_TIMEOUT;
++ ktime_t timeout = KTIME_MAX;
+ int ret;
+
+ do {
+@@ -7570,7 +7625,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
+
+ if (get_timespec64(&ts, uts))
+ return -EFAULT;
+- timeout = timespec64_to_jiffies(&ts);
++ timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
+ }
+
+ if (sig) {
+@@ -7602,7 +7657,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
+ }
+ prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
+ TASK_INTERRUPTIBLE);
+- ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
++ ret = io_cqring_wait_schedule(ctx, &iowq, timeout);
+ finish_wait(&ctx->cq_wait, &iowq.wq);
+ cond_resched();
+ } while (ret > 0);
+@@ -7656,10 +7711,15 @@ static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
+ struct io_ring_ctx *ctx = node->rsrc_data->ctx;
+ unsigned long flags;
+ bool first_add = false;
++ unsigned long delay = HZ;
+
+ spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
+ node->done = true;
+
++ /* if we are mid-quiesce then do not delay */
++ if (node->rsrc_data->quiesce)
++ delay = 0;
++
+ while (!list_empty(&ctx->rsrc_ref_list)) {
+ node = list_first_entry(&ctx->rsrc_ref_list,
+ struct io_rsrc_node, node);
+@@ -7672,7 +7732,7 @@ static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
+ spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
+
+ if (first_add)
+- mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
++ mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
+ }
+
+ static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
+@@ -7750,7 +7810,15 @@ static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ct
+ ret = wait_for_completion_interruptible(&data->done);
+ if (!ret) {
+ mutex_lock(&ctx->uring_lock);
+- break;
++ if (atomic_read(&data->refs) > 0) {
++ /*
++ * it has been revived by another thread while
++ * we were unlocked
++ */
++ mutex_unlock(&ctx->uring_lock);
++ } else {
++ break;
++ }
+ }
+
+ atomic_inc(&data->refs);
+@@ -8045,10 +8113,15 @@ static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
+ refcount_add(skb->truesize, &sk->sk_wmem_alloc);
+ skb_queue_head(&sk->sk_receive_queue, skb);
+
+- for (i = 0; i < nr_files; i++)
+- fput(fpl->fp[i]);
++ for (i = 0; i < nr; i++) {
++ struct file *file = io_file_from_index(ctx, i + offset);
++
++ if (file)
++ fput(file);
++ }
+ } else {
+ kfree_skb(skb);
++ free_uid(fpl->user);
+ kfree(fpl);
+ }
+
+@@ -8337,13 +8410,15 @@ static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
+ static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
+ struct io_rsrc_node *node, void *rsrc)
+ {
++ u64 *tag_slot = io_get_tag_slot(data, idx);
+ struct io_rsrc_put *prsrc;
+
+ prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
+ if (!prsrc)
+ return -ENOMEM;
+
+- prsrc->tag = *io_get_tag_slot(data, idx);
++ prsrc->tag = *tag_slot;
++ *tag_slot = 0;
+ prsrc->rsrc = rsrc;
+ list_add(&prsrc->list, &node->rsrc_list);
+ return 0;
+@@ -8411,7 +8486,7 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
+ struct io_ring_ctx *ctx = req->ctx;
+ struct io_fixed_file *file_slot;
+ struct file *file;
+- int ret, i;
++ int ret;
+
+ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
+ ret = -ENXIO;
+@@ -8424,8 +8499,8 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
+ if (ret)
+ goto out;
+
+- i = array_index_nospec(offset, ctx->nr_user_files);
+- file_slot = io_fixed_file_slot(&ctx->file_table, i);
++ offset = array_index_nospec(offset, ctx->nr_user_files);
++ file_slot = io_fixed_file_slot(&ctx->file_table, offset);
+ ret = -EBADF;
+ if (!file_slot->file_ptr)
+ goto out;
+@@ -8481,8 +8556,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
+
+ if (file_slot->file_ptr) {
+ file = (struct file *)(file_slot->file_ptr & FFS_MASK);
+- err = io_queue_rsrc_removal(data, up->offset + done,
+- ctx->rsrc_node, file);
++ err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
+ if (err)
+ break;
+ file_slot->file_ptr = 0;
+@@ -8507,7 +8581,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
+ err = -EBADF;
+ break;
+ }
+- *io_get_tag_slot(data, up->offset + done) = tag;
++ *io_get_tag_slot(data, i) = tag;
+ io_fixed_file_set(file_slot, file);
+ err = io_sqe_file_register(ctx, file, i);
+ if (err) {
+@@ -8753,10 +8827,9 @@ static void io_mem_free(void *ptr)
+
+ static void *io_mem_alloc(size_t size)
+ {
+- gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
+- __GFP_NORETRY | __GFP_ACCOUNT;
++ gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
+
+- return (void *) __get_free_pages(gfp_flags, get_order(size));
++ return (void *) __get_free_pages(gfp, get_order(size));
+ }
+
+ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
+@@ -9152,7 +9225,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
+
+ i = array_index_nospec(offset, ctx->nr_user_bufs);
+ if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
+- err = io_queue_rsrc_removal(ctx->buf_data, offset,
++ err = io_queue_rsrc_removal(ctx->buf_data, i,
+ ctx->rsrc_node, ctx->user_bufs[i]);
+ if (unlikely(err)) {
+ io_buffer_unmap(ctx, &imu);
+@@ -9209,10 +9282,8 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx)
+ struct io_buffer *buf;
+ unsigned long index;
+
+- xa_for_each(&ctx->io_buffers, index, buf) {
++ xa_for_each(&ctx->io_buffers, index, buf)
+ __io_remove_buffers(ctx, buf, index, -1U);
+- cond_resched();
+- }
+ }
+
+ static void io_req_cache_free(struct list_head *list)
+@@ -9517,19 +9588,8 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
+ {
+ struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+ struct io_task_cancel *cancel = data;
+- bool ret;
+-
+- if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
+- struct io_ring_ctx *ctx = req->ctx;
+
+- /* protect against races with linked timeouts */
+- spin_lock(&ctx->completion_lock);
+- ret = io_match_task(req, cancel->task, cancel->all);
+- spin_unlock(&ctx->completion_lock);
+- } else {
+- ret = io_match_task(req, cancel->task, cancel->all);
+- }
+- return ret;
++ return io_match_task_safe(req, cancel->task, cancel->all);
+ }
+
+ static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
+@@ -9540,7 +9600,7 @@ static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
+
+ spin_lock(&ctx->completion_lock);
+ list_for_each_entry_reverse(de, &ctx->defer_list, list) {
+- if (io_match_task(de->req, task, cancel_all)) {
++ if (io_match_task_safe(de->req, task, cancel_all)) {
+ list_cut_position(&list, &ctx->defer_list, &de->list);
+ break;
+ }
+@@ -9733,21 +9793,9 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
+ return percpu_counter_sum(&tctx->inflight);
+ }
+
+-static void io_uring_drop_tctx_refs(struct task_struct *task)
+-{
+- struct io_uring_task *tctx = task->io_uring;
+- unsigned int refs = tctx->cached_refs;
+-
+- if (refs) {
+- tctx->cached_refs = 0;
+- percpu_counter_sub(&tctx->inflight, refs);
+- put_task_struct_many(task, refs);
+- }
+-}
+-
+ /*
+ * Find any io_uring ctx that this task has registered or done IO on, and cancel
+- * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation.
++ * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
+ */
+ static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
+ {
+@@ -9788,8 +9836,10 @@ static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
+ cancel_all);
+ }
+
+- prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
++ prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
++ io_run_task_work();
+ io_uring_drop_tctx_refs(current);
++
+ /*
+ * If we've seen completions, retry without waiting. This
+ * avoids a race where a completion comes in before we did
+@@ -9799,10 +9849,14 @@ static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
+ schedule();
+ finish_wait(&tctx->wait, &wait);
+ } while (1);
+- atomic_dec(&tctx->in_idle);
+
+ io_uring_clean_tctx(tctx);
+ if (cancel_all) {
++ /*
++ * We shouldn't run task_works after cancel, so just leave
++ * ->in_idle set for normal exit.
++ */
++ atomic_dec(&tctx->in_idle);
+ /* for exec all current's requests should be gone, kill tctx */
+ __io_uring_free(current);
+ }
+@@ -9925,6 +9979,8 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz
+ return -EINVAL;
+ if (copy_from_user(&arg, argp, sizeof(arg)))
+ return -EFAULT;
++ if (arg.pad)
++ return -EINVAL;
+ *sig = u64_to_user_ptr(arg.sigmask);
+ *argsz = arg.sigmask_sz;
+ *ts = u64_to_user_ptr(arg.ts);
+@@ -10540,8 +10596,6 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
+ __u32 tmp;
+ int err;
+
+- if (up->resv)
+- return -EINVAL;
+ if (check_add_overflow(up->offset, nr_args, &tmp))
+ return -EOVERFLOW;
+ err = io_rsrc_node_switch_start(ctx);
+@@ -10567,6 +10621,8 @@ static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
+ memset(&up, 0, sizeof(up));
+ if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
+ return -EFAULT;
++ if (up.resv || up.resv2)
++ return -EINVAL;
+ return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
+ }
+
+@@ -10579,7 +10635,7 @@ static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
+ return -EINVAL;
+ if (copy_from_user(&up, arg, sizeof(up)))
+ return -EFAULT;
+- if (!up.nr || up.resv)
++ if (!up.nr || up.resv || up.resv2)
+ return -EINVAL;
+ return __io_register_rsrc_update(ctx, type, &up, up.nr);
+ }
+@@ -10627,7 +10683,15 @@ static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
+ if (len > cpumask_size())
+ len = cpumask_size();
+
+- if (copy_from_user(new_mask, arg, len)) {
++ if (in_compat_syscall()) {
++ ret = compat_get_bitmap(cpumask_bits(new_mask),
++ (const compat_ulong_t __user *)arg,
++ len * 8 /* CHAR_BIT */);
++ } else {
++ ret = copy_from_user(new_mask, arg, len);
++ }
++
++ if (ret) {
+ free_cpumask_var(new_mask);
+ return -EFAULT;
+ }
+@@ -10684,7 +10748,9 @@ static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
+
+ BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
+
+- memcpy(ctx->iowq_limits, new_count, sizeof(new_count));
++ for (i = 0; i < ARRAY_SIZE(new_count); i++)
++ if (new_count[i])
++ ctx->iowq_limits[i] = new_count[i];
+ ctx->iowq_limits_set = true;
+
+ ret = -EINVAL;
+diff --git a/fs/ioctl.c b/fs/ioctl.c
+index 504e695781124..e0a3455f9a0f6 100644
+--- a/fs/ioctl.c
++++ b/fs/ioctl.c
+@@ -173,7 +173,7 @@ int fiemap_prep(struct inode *inode, struct fiemap_extent_info *fieinfo,
+
+ if (*len == 0)
+ return -EINVAL;
+- if (start > maxbytes)
++ if (start >= maxbytes)
+ return -EFBIG;
+
+ /*
+diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
+index 9cc5798423d12..fe10d8a30f6bd 100644
+--- a/fs/iomap/buffered-io.c
++++ b/fs/iomap/buffered-io.c
+@@ -256,8 +256,13 @@ static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
+ unsigned poff, plen;
+ sector_t sector;
+
+- if (iomap->type == IOMAP_INLINE)
+- return min(iomap_read_inline_data(iter, page), length);
++ if (iomap->type == IOMAP_INLINE) {
++ loff_t ret = iomap_read_inline_data(iter, page);
++
++ if (ret < 0)
++ return ret;
++ return 0;
++ }
+
+ /* zero post-eof blocks as the page may be mapped */
+ iop = iomap_page_create(iter->inode, page);
+@@ -370,6 +375,8 @@ static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
+ ctx->cur_page_in_bio = false;
+ }
+ ret = iomap_readpage_iter(iter, ctx, done);
++ if (ret <= 0)
++ return ret;
+ }
+
+ return done;
+@@ -750,7 +757,7 @@ again:
+ * same page as we're writing to, without it being marked
+ * up-to-date.
+ */
+- if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
++ if (unlikely(fault_in_iov_iter_readable(i, bytes))) {
+ status = -EFAULT;
+ break;
+ }
+diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
+index 4ecd255e0511c..468dcbba45bcb 100644
+--- a/fs/iomap/direct-io.c
++++ b/fs/iomap/direct-io.c
+@@ -31,6 +31,7 @@ struct iomap_dio {
+ atomic_t ref;
+ unsigned flags;
+ int error;
++ size_t done_before;
+ bool wait_for_completion;
+
+ union {
+@@ -124,6 +125,9 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio)
+ if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
+ ret = generic_write_sync(iocb, ret);
+
++ if (ret > 0)
++ ret += dio->done_before;
++
+ kfree(dio);
+
+ return ret;
+@@ -371,6 +375,8 @@ static loff_t iomap_dio_hole_iter(const struct iomap_iter *iter,
+ loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);
+
+ dio->size += length;
++ if (!length)
++ return -EFAULT;
+ return length;
+ }
+
+@@ -402,6 +408,8 @@ static loff_t iomap_dio_inline_iter(const struct iomap_iter *iomi,
+ copied = copy_to_iter(inline_data, length, iter);
+ }
+ dio->size += copied;
++ if (!copied)
++ return -EFAULT;
+ return copied;
+ }
+
+@@ -446,13 +454,21 @@ static loff_t iomap_dio_iter(const struct iomap_iter *iter,
+ * may be pure data writes. In that case, we still need to do a full data sync
+ * completion.
+ *
++ * When page faults are disabled and @dio_flags includes IOMAP_DIO_PARTIAL,
++ * __iomap_dio_rw can return a partial result if it encounters a non-resident
++ * page in @iter after preparing a transfer. In that case, the non-resident
++ * pages can be faulted in and the request resumed with @done_before set to the
++ * number of bytes previously transferred. The request will then complete with
++ * the correct total number of bytes transferred; this is essential for
++ * completing partial requests asynchronously.
++ *
+ * Returns -ENOTBLK In case of a page invalidation invalidation failure for
+ * writes. The callers needs to fall back to buffered I/O in this case.
+ */
+ struct iomap_dio *
+ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+- unsigned int dio_flags)
++ unsigned int dio_flags, size_t done_before)
+ {
+ struct address_space *mapping = iocb->ki_filp->f_mapping;
+ struct inode *inode = file_inode(iocb->ki_filp);
+@@ -482,6 +498,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ dio->dops = dops;
+ dio->error = 0;
+ dio->flags = 0;
++ dio->done_before = done_before;
+
+ dio->submit.iter = iter;
+ dio->submit.waiter = current;
+@@ -577,6 +594,12 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)
+ iov_iter_revert(iter, iomi.pos - dio->i_size);
+
++ if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {
++ if (!(iocb->ki_flags & IOCB_NOWAIT))
++ wait_for_completion = true;
++ ret = 0;
++ }
++
+ /* magic error code to fall back to buffered I/O */
+ if (ret == -ENOTBLK) {
+ wait_for_completion = true;
+@@ -642,11 +665,11 @@ EXPORT_SYMBOL_GPL(__iomap_dio_rw);
+ ssize_t
+ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+- unsigned int dio_flags)
++ unsigned int dio_flags, size_t done_before)
+ {
+ struct iomap_dio *dio;
+
+- dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags);
++ dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, done_before);
+ if (IS_ERR_OR_NULL(dio))
+ return PTR_ERR_OR_ZERO(dio);
+ return iomap_dio_complete(dio);
+diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c
+index 678e2c51b855c..0c6eacfcbeef1 100644
+--- a/fs/isofs/inode.c
++++ b/fs/isofs/inode.c
+@@ -1322,6 +1322,8 @@ static int isofs_read_inode(struct inode *inode, int relocated)
+
+ de = (struct iso_directory_record *) (bh->b_data + offset);
+ de_len = *(unsigned char *) de;
++ if (de_len < sizeof(struct iso_directory_record))
++ goto fail;
+
+ if (offset + de_len > bufsize) {
+ int frag1 = bufsize - offset;
+diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
+index 3cc4ab2ba7f4f..34b1406c06fde 100644
+--- a/fs/jbd2/commit.c
++++ b/fs/jbd2/commit.c
+@@ -501,7 +501,6 @@ void jbd2_journal_commit_transaction(journal_t *journal)
+ }
+ spin_unlock(&commit_transaction->t_handle_lock);
+ commit_transaction->t_state = T_SWITCH;
+- write_unlock(&journal->j_state_lock);
+
+ J_ASSERT (atomic_read(&commit_transaction->t_outstanding_credits) <=
+ journal->j_max_transaction_buffers);
+@@ -521,6 +520,8 @@ void jbd2_journal_commit_transaction(journal_t *journal)
+ * has reserved. This is consistent with the existing behaviour
+ * that multiple jbd2_journal_get_write_access() calls to the same
+ * buffer are perfectly permissible.
++ * We use journal->j_state_lock here to serialize processing of
++ * t_reserved_list with eviction of buffers from journal_unmap_buffer().
+ */
+ while (commit_transaction->t_reserved_list) {
+ jh = commit_transaction->t_reserved_list;
+@@ -540,6 +541,7 @@ void jbd2_journal_commit_transaction(journal_t *journal)
+ jbd2_journal_refile_buffer(journal, jh);
+ }
+
++ write_unlock(&journal->j_state_lock);
+ /*
+ * Now try to drop any written-back buffers from the journal's
+ * checkpoint lists. We do this *before* commit because it potentially
+@@ -1170,7 +1172,7 @@ restart_loop:
+ if (journal->j_commit_callback)
+ journal->j_commit_callback(journal, commit_transaction);
+ if (journal->j_fc_cleanup_callback)
+- journal->j_fc_cleanup_callback(journal, 1);
++ journal->j_fc_cleanup_callback(journal, 1, commit_transaction->t_tid);
+
+ trace_jbd2_end_commit(journal, commit_transaction);
+ jbd_debug(1, "JBD2: commit %d complete, head %d\n",
+diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c
+index 35302bc192eb9..1f8493ef181d6 100644
+--- a/fs/jbd2/journal.c
++++ b/fs/jbd2/journal.c
+@@ -769,7 +769,7 @@ EXPORT_SYMBOL(jbd2_fc_begin_commit);
+ static int __jbd2_fc_end_commit(journal_t *journal, tid_t tid, bool fallback)
+ {
+ if (journal->j_fc_cleanup_callback)
+- journal->j_fc_cleanup_callback(journal, 0);
++ journal->j_fc_cleanup_callback(journal, 0, tid);
+ write_lock(&journal->j_state_lock);
+ journal->j_flags &= ~JBD2_FAST_COMMIT_ONGOING;
+ if (fallback)
+@@ -2970,6 +2970,7 @@ struct journal_head *jbd2_journal_grab_journal_head(struct buffer_head *bh)
+ jbd_unlock_bh_journal_head(bh);
+ return jh;
+ }
++EXPORT_SYMBOL(jbd2_journal_grab_journal_head);
+
+ static void __journal_remove_journal_head(struct buffer_head *bh)
+ {
+@@ -3022,6 +3023,7 @@ void jbd2_journal_put_journal_head(struct journal_head *jh)
+ jbd_unlock_bh_journal_head(bh);
+ }
+ }
++EXPORT_SYMBOL(jbd2_journal_put_journal_head);
+
+ /*
+ * Initialize jbd inode head
+diff --git a/fs/jffs2/build.c b/fs/jffs2/build.c
+index b288c8ae1236b..837cd55fd4c5e 100644
+--- a/fs/jffs2/build.c
++++ b/fs/jffs2/build.c
+@@ -415,13 +415,15 @@ int jffs2_do_mount_fs(struct jffs2_sb_info *c)
+ jffs2_free_ino_caches(c);
+ jffs2_free_raw_node_refs(c);
+ ret = -EIO;
+- goto out_free;
++ goto out_sum_exit;
+ }
+
+ jffs2_calc_trigger_levels(c);
+
+ return 0;
+
++ out_sum_exit:
++ jffs2_sum_exit(c);
+ out_free:
+ kvfree(c->blocks);
+
+diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c
+index 4fc8cd698d1a4..bd7d58d27bfc6 100644
+--- a/fs/jffs2/file.c
++++ b/fs/jffs2/file.c
+@@ -136,20 +136,15 @@ static int jffs2_write_begin(struct file *filp, struct address_space *mapping,
+ struct page *pg;
+ struct inode *inode = mapping->host;
+ struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode);
++ struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb);
+ pgoff_t index = pos >> PAGE_SHIFT;
+ uint32_t pageofs = index << PAGE_SHIFT;
+ int ret = 0;
+
+- pg = grab_cache_page_write_begin(mapping, index, flags);
+- if (!pg)
+- return -ENOMEM;
+- *pagep = pg;
+-
+ jffs2_dbg(1, "%s()\n", __func__);
+
+ if (pageofs > inode->i_size) {
+ /* Make new hole frag from old EOF to new page */
+- struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb);
+ struct jffs2_raw_inode ri;
+ struct jffs2_full_dnode *fn;
+ uint32_t alloc_len;
+@@ -160,7 +155,7 @@ static int jffs2_write_begin(struct file *filp, struct address_space *mapping,
+ ret = jffs2_reserve_space(c, sizeof(ri), &alloc_len,
+ ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE);
+ if (ret)
+- goto out_page;
++ goto out_err;
+
+ mutex_lock(&f->sem);
+ memset(&ri, 0, sizeof(ri));
+@@ -190,7 +185,7 @@ static int jffs2_write_begin(struct file *filp, struct address_space *mapping,
+ ret = PTR_ERR(fn);
+ jffs2_complete_reservation(c);
+ mutex_unlock(&f->sem);
+- goto out_page;
++ goto out_err;
+ }
+ ret = jffs2_add_full_dnode_to_inode(c, f, fn);
+ if (f->metadata) {
+@@ -205,13 +200,26 @@ static int jffs2_write_begin(struct file *filp, struct address_space *mapping,
+ jffs2_free_full_dnode(fn);
+ jffs2_complete_reservation(c);
+ mutex_unlock(&f->sem);
+- goto out_page;
++ goto out_err;
+ }
+ jffs2_complete_reservation(c);
+ inode->i_size = pageofs;
+ mutex_unlock(&f->sem);
+ }
+
++ /*
++ * While getting a page and reading data in, lock c->alloc_sem until
++ * the page is Uptodate. Otherwise GC task may attempt to read the same
++ * page in read_cache_page(), which causes a deadlock.
++ */
++ mutex_lock(&c->alloc_sem);
++ pg = grab_cache_page_write_begin(mapping, index, flags);
++ if (!pg) {
++ ret = -ENOMEM;
++ goto release_sem;
++ }
++ *pagep = pg;
++
+ /*
+ * Read in the page if it wasn't already present. Cannot optimize away
+ * the whole page write case until jffs2_write_end can handle the
+@@ -221,15 +229,17 @@ static int jffs2_write_begin(struct file *filp, struct address_space *mapping,
+ mutex_lock(&f->sem);
+ ret = jffs2_do_readpage_nolock(inode, pg);
+ mutex_unlock(&f->sem);
+- if (ret)
+- goto out_page;
++ if (ret) {
++ unlock_page(pg);
++ put_page(pg);
++ goto release_sem;
++ }
+ }
+ jffs2_dbg(1, "end write_begin(). pg->flags %lx\n", pg->flags);
+- return ret;
+
+-out_page:
+- unlock_page(pg);
+- put_page(pg);
++release_sem:
++ mutex_unlock(&c->alloc_sem);
++out_err:
+ return ret;
+ }
+
+diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c
+index 2ac410477c4f4..71f03a5d36ed2 100644
+--- a/fs/jffs2/fs.c
++++ b/fs/jffs2/fs.c
+@@ -603,8 +603,8 @@ out_root:
+ jffs2_free_ino_caches(c);
+ jffs2_free_raw_node_refs(c);
+ kvfree(c->blocks);
+- out_inohash:
+ jffs2_clear_xattr_subsystem(c);
++ out_inohash:
+ kfree(c->inocache_list);
+ out_wbuf:
+ jffs2_flash_cleanup(c);
+diff --git a/fs/jffs2/scan.c b/fs/jffs2/scan.c
+index b676056826beb..29671e33a1714 100644
+--- a/fs/jffs2/scan.c
++++ b/fs/jffs2/scan.c
+@@ -136,7 +136,7 @@ int jffs2_scan_medium(struct jffs2_sb_info *c)
+ if (!s) {
+ JFFS2_WARNING("Can't allocate memory for summary\n");
+ ret = -ENOMEM;
+- goto out;
++ goto out_buf;
+ }
+ }
+
+@@ -275,13 +275,15 @@ int jffs2_scan_medium(struct jffs2_sb_info *c)
+ }
+ ret = 0;
+ out:
++ jffs2_sum_reset_collected(s);
++ kfree(s);
++ out_buf:
+ if (buf_size)
+ kfree(flashbuf);
+ #ifndef __ECOS
+ else
+ mtd_unpoint(c->mtd, 0, c->mtd->size);
+ #endif
+- kfree(s);
+ return ret;
+ }
+
+diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c
+index 57ab424c05ff0..072821b50ab91 100644
+--- a/fs/jfs/inode.c
++++ b/fs/jfs/inode.c
+@@ -146,12 +146,13 @@ void jfs_evict_inode(struct inode *inode)
+ dquot_initialize(inode);
+
+ if (JFS_IP(inode)->fileset == FILESYSTEM_I) {
++ struct inode *ipimap = JFS_SBI(inode->i_sb)->ipimap;
+ truncate_inode_pages_final(&inode->i_data);
+
+ if (test_cflag(COMMIT_Freewmap, inode))
+ jfs_free_zero_link(inode);
+
+- if (JFS_SBI(inode->i_sb)->ipimap)
++ if (ipimap && JFS_IP(ipimap)->i_imap)
+ diFree(inode);
+
+ /*
+diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c
+index 91f4ec93dab1f..d8502f4989d9d 100644
+--- a/fs/jfs/jfs_dmap.c
++++ b/fs/jfs/jfs_dmap.c
+@@ -148,6 +148,7 @@ static const s8 budtab[256] = {
+ * 0 - success
+ * -ENOMEM - insufficient memory
+ * -EIO - i/o error
++ * -EINVAL - wrong bmap data
+ */
+ int dbMount(struct inode *ipbmap)
+ {
+@@ -179,6 +180,12 @@ int dbMount(struct inode *ipbmap)
+ bmp->db_nfree = le64_to_cpu(dbmp_le->dn_nfree);
+ bmp->db_l2nbperpage = le32_to_cpu(dbmp_le->dn_l2nbperpage);
+ bmp->db_numag = le32_to_cpu(dbmp_le->dn_numag);
++ if (!bmp->db_numag) {
++ release_metapage(mp);
++ kfree(bmp);
++ return -EINVAL;
++ }
++
+ bmp->db_maxlevel = le32_to_cpu(dbmp_le->dn_maxlevel);
+ bmp->db_maxag = le32_to_cpu(dbmp_le->dn_maxag);
+ bmp->db_agpref = le32_to_cpu(dbmp_le->dn_agpref);
+diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c
+index 5d7d7170c03c0..aa4ff7bcaff23 100644
+--- a/fs/jfs/jfs_mount.c
++++ b/fs/jfs/jfs_mount.c
+@@ -81,14 +81,14 @@ int jfs_mount(struct super_block *sb)
+ * (initialize mount inode from the superblock)
+ */
+ if ((rc = chkSuper(sb))) {
+- goto errout20;
++ goto out;
+ }
+
+ ipaimap = diReadSpecial(sb, AGGREGATE_I, 0);
+ if (ipaimap == NULL) {
+ jfs_err("jfs_mount: Failed to read AGGREGATE_I");
+ rc = -EIO;
+- goto errout20;
++ goto out;
+ }
+ sbi->ipaimap = ipaimap;
+
+@@ -99,7 +99,7 @@ int jfs_mount(struct super_block *sb)
+ */
+ if ((rc = diMount(ipaimap))) {
+ jfs_err("jfs_mount: diMount(ipaimap) failed w/rc = %d", rc);
+- goto errout21;
++ goto err_ipaimap;
+ }
+
+ /*
+@@ -108,7 +108,7 @@ int jfs_mount(struct super_block *sb)
+ ipbmap = diReadSpecial(sb, BMAP_I, 0);
+ if (ipbmap == NULL) {
+ rc = -EIO;
+- goto errout22;
++ goto err_umount_ipaimap;
+ }
+
+ jfs_info("jfs_mount: ipbmap:0x%p", ipbmap);
+@@ -120,7 +120,7 @@ int jfs_mount(struct super_block *sb)
+ */
+ if ((rc = dbMount(ipbmap))) {
+ jfs_err("jfs_mount: dbMount failed w/rc = %d", rc);
+- goto errout22;
++ goto err_ipbmap;
+ }
+
+ /*
+@@ -139,7 +139,7 @@ int jfs_mount(struct super_block *sb)
+ if (!ipaimap2) {
+ jfs_err("jfs_mount: Failed to read AGGREGATE_I");
+ rc = -EIO;
+- goto errout35;
++ goto err_umount_ipbmap;
+ }
+ sbi->ipaimap2 = ipaimap2;
+
+@@ -151,7 +151,7 @@ int jfs_mount(struct super_block *sb)
+ if ((rc = diMount(ipaimap2))) {
+ jfs_err("jfs_mount: diMount(ipaimap2) failed, rc = %d",
+ rc);
+- goto errout35;
++ goto err_ipaimap2;
+ }
+ } else
+ /* Secondary aggregate inode table is not valid */
+@@ -168,7 +168,7 @@ int jfs_mount(struct super_block *sb)
+ jfs_err("jfs_mount: Failed to read FILESYSTEM_I");
+ /* open fileset secondary inode allocation map */
+ rc = -EIO;
+- goto errout40;
++ goto err_umount_ipaimap2;
+ }
+ jfs_info("jfs_mount: ipimap:0x%p", ipimap);
+
+@@ -178,41 +178,34 @@ int jfs_mount(struct super_block *sb)
+ /* initialize fileset inode allocation map */
+ if ((rc = diMount(ipimap))) {
+ jfs_err("jfs_mount: diMount failed w/rc = %d", rc);
+- goto errout41;
++ goto err_ipimap;
+ }
+
+- goto out;
++ return rc;
+
+ /*
+ * unwind on error
+ */
+- errout41: /* close fileset inode allocation map inode */
++err_ipimap:
++ /* close fileset inode allocation map inode */
+ diFreeSpecial(ipimap);
+-
+- errout40: /* fileset closed */
+-
++err_umount_ipaimap2:
+ /* close secondary aggregate inode allocation map */
+- if (ipaimap2) {
++ if (ipaimap2)
+ diUnmount(ipaimap2, 1);
++err_ipaimap2:
++ /* close aggregate inodes */
++ if (ipaimap2)
+ diFreeSpecial(ipaimap2);
+- }
+-
+- errout35:
+-
+- /* close aggregate block allocation map */
++err_umount_ipbmap: /* close aggregate block allocation map */
+ dbUnmount(ipbmap, 1);
++err_ipbmap: /* close aggregate inodes */
+ diFreeSpecial(ipbmap);
+-
+- errout22: /* close aggregate inode allocation map */
+-
++err_umount_ipaimap: /* close aggregate inode allocation map */
+ diUnmount(ipaimap, 1);
+-
+- errout21: /* close aggregate inodes */
++err_ipaimap: /* close aggregate inodes */
+ diFreeSpecial(ipaimap);
+- errout20: /* aggregate closed */
+-
+- out:
+-
++out:
+ if (rc)
+ jfs_err("Mount JFS Failure: %d", rc);
+
+diff --git a/fs/ksmbd/Kconfig b/fs/ksmbd/Kconfig
+index b83cbd756ae50..6af339cfdc041 100644
+--- a/fs/ksmbd/Kconfig
++++ b/fs/ksmbd/Kconfig
+@@ -19,6 +19,7 @@ config SMB_SERVER
+ select CRYPTO_GCM
+ select ASN1
+ select OID_REGISTRY
++ select CRC32
+ default n
+ help
+ Choose Y here if you want to allow SMB3 compliant clients
+diff --git a/fs/ksmbd/connection.c b/fs/ksmbd/connection.c
+index b57a0d8a392ff..f7d5e8b7bef77 100644
+--- a/fs/ksmbd/connection.c
++++ b/fs/ksmbd/connection.c
+@@ -62,6 +62,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void)
+ atomic_set(&conn->req_running, 0);
+ atomic_set(&conn->r_count, 0);
+ conn->total_credits = 1;
++ conn->outstanding_credits = 1;
+
+ init_waitqueue_head(&conn->req_running_q);
+ INIT_LIST_HEAD(&conn->conns_list);
+diff --git a/fs/ksmbd/connection.h b/fs/ksmbd/connection.h
+index e5403c587a58c..8694aef482c1a 100644
+--- a/fs/ksmbd/connection.h
++++ b/fs/ksmbd/connection.h
+@@ -61,8 +61,8 @@ struct ksmbd_conn {
+ atomic_t req_running;
+ /* References which are made for this Server object*/
+ atomic_t r_count;
+- unsigned short total_credits;
+- unsigned short max_credits;
++ unsigned int total_credits;
++ unsigned int outstanding_credits;
+ spinlock_t credits_lock;
+ wait_queue_head_t req_running_q;
+ /* Lock to protect requests list*/
+diff --git a/fs/ksmbd/ksmbd_netlink.h b/fs/ksmbd/ksmbd_netlink.h
+index c6718a05d347f..71bfb7de44725 100644
+--- a/fs/ksmbd/ksmbd_netlink.h
++++ b/fs/ksmbd/ksmbd_netlink.h
+@@ -103,6 +103,8 @@ struct ksmbd_startup_request {
+ * we set the SPARSE_FILES bit (0x40).
+ */
+ __u32 sub_auth[3]; /* Subauth value for Security ID */
++ __u32 smb2_max_credits; /* MAX credits */
++ __u32 reserved[128]; /* Reserved room */
+ __u32 ifc_list_sz; /* interfaces list size */
+ __s8 ____payload[];
+ };
+@@ -113,7 +115,7 @@ struct ksmbd_startup_request {
+ * IPC request to shutdown ksmbd server.
+ */
+ struct ksmbd_shutdown_request {
+- __s32 reserved;
++ __s32 reserved[16];
+ };
+
+ /*
+@@ -122,6 +124,7 @@ struct ksmbd_shutdown_request {
+ struct ksmbd_login_request {
+ __u32 handle;
+ __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; /* user account name */
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -135,6 +138,7 @@ struct ksmbd_login_response {
+ __u16 status;
+ __u16 hash_sz; /* hash size */
+ __s8 hash[KSMBD_REQ_MAX_HASH_SZ]; /* password hash */
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -143,6 +147,7 @@ struct ksmbd_login_response {
+ struct ksmbd_share_config_request {
+ __u32 handle;
+ __s8 share_name[KSMBD_REQ_MAX_SHARE_NAME]; /* share name */
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -157,6 +162,7 @@ struct ksmbd_share_config_response {
+ __u16 force_directory_mode;
+ __u16 force_uid;
+ __u16 force_gid;
++ __u32 reserved[128]; /* Reserved room */
+ __u32 veto_list_sz;
+ __s8 ____payload[];
+ };
+@@ -187,6 +193,7 @@ struct ksmbd_tree_connect_request {
+ __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ];
+ __s8 share[KSMBD_REQ_MAX_SHARE_NAME];
+ __s8 peer_addr[64];
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -196,6 +203,7 @@ struct ksmbd_tree_connect_response {
+ __u32 handle;
+ __u16 status;
+ __u16 connection_flags;
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -204,6 +212,7 @@ struct ksmbd_tree_connect_response {
+ struct ksmbd_tree_disconnect_request {
+ __u64 session_id; /* session id */
+ __u64 connect_id; /* tree connection id */
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+@@ -212,6 +221,7 @@ struct ksmbd_tree_disconnect_request {
+ struct ksmbd_logout_request {
+ __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; /* user account name */
+ __u32 account_flags;
++ __u32 reserved[16]; /* Reserved room */
+ };
+
+ /*
+diff --git a/fs/ksmbd/ndr.c b/fs/ksmbd/ndr.c
+index 8317f7ca402b4..5052be9261d91 100644
+--- a/fs/ksmbd/ndr.c
++++ b/fs/ksmbd/ndr.c
+@@ -148,7 +148,7 @@ static int ndr_read_int16(struct ndr *n, __u16 *value)
+ static int ndr_read_int32(struct ndr *n, __u32 *value)
+ {
+ if (n->offset + sizeof(__u32) > n->length)
+- return 0;
++ return -EINVAL;
+
+ if (value)
+ *value = le32_to_cpu(*(__le32 *)ndr_get_field(n));
+diff --git a/fs/ksmbd/server.c b/fs/ksmbd/server.c
+index 2a2b2135bfded..36d368e59a640 100644
+--- a/fs/ksmbd/server.c
++++ b/fs/ksmbd/server.c
+@@ -632,5 +632,6 @@ MODULE_SOFTDEP("pre: sha512");
+ MODULE_SOFTDEP("pre: aead2");
+ MODULE_SOFTDEP("pre: ccm");
+ MODULE_SOFTDEP("pre: gcm");
++MODULE_SOFTDEP("pre: crc32");
+ module_init(ksmbd_server_init)
+ module_exit(ksmbd_server_exit)
+diff --git a/fs/ksmbd/smb2misc.c b/fs/ksmbd/smb2misc.c
+index 030ca57c37849..cc1c38686ecd7 100644
+--- a/fs/ksmbd/smb2misc.c
++++ b/fs/ksmbd/smb2misc.c
+@@ -290,7 +290,7 @@ static int smb2_validate_credit_charge(struct ksmbd_conn *conn,
+ unsigned int req_len = 0, expect_resp_len = 0, calc_credit_num, max_len;
+ unsigned short credit_charge = le16_to_cpu(hdr->CreditCharge);
+ void *__hdr = hdr;
+- int ret;
++ int ret = 0;
+
+ switch (hdr->Command) {
+ case SMB2_QUERY_INFO:
+@@ -327,21 +327,27 @@ static int smb2_validate_credit_charge(struct ksmbd_conn *conn,
+ ksmbd_debug(SMB, "Insufficient credit charge, given: %d, needed: %d\n",
+ credit_charge, calc_credit_num);
+ return 1;
+- } else if (credit_charge > conn->max_credits) {
++ } else if (credit_charge > conn->vals->max_credits) {
+ ksmbd_debug(SMB, "Too large credit charge: %d\n", credit_charge);
+ return 1;
+ }
+
+ spin_lock(&conn->credits_lock);
+- if (credit_charge <= conn->total_credits) {
+- conn->total_credits -= credit_charge;
+- ret = 0;
+- } else {
++ if (credit_charge > conn->total_credits) {
+ ksmbd_debug(SMB, "Insufficient credits granted, given: %u, granted: %u\n",
+ credit_charge, conn->total_credits);
+ ret = 1;
+ }
++
++ if ((u64)conn->outstanding_credits + credit_charge > conn->vals->max_credits) {
++ ksmbd_debug(SMB, "Limits exceeding the maximum allowable outstanding requests, given : %u, pending : %u\n",
++ credit_charge, conn->outstanding_credits);
++ ret = 1;
++ } else
++ conn->outstanding_credits += credit_charge;
++
+ spin_unlock(&conn->credits_lock);
++
+ return ret;
+ }
+
+@@ -358,12 +364,10 @@ int ksmbd_smb2_check_message(struct ksmbd_work *work)
+ hdr = &pdu->hdr;
+ }
+
+- if (le32_to_cpu(hdr->NextCommand) > 0) {
++ if (le32_to_cpu(hdr->NextCommand) > 0)
+ len = le32_to_cpu(hdr->NextCommand);
+- } else if (work->next_smb2_rcv_hdr_off) {
++ else if (work->next_smb2_rcv_hdr_off)
+ len -= work->next_smb2_rcv_hdr_off;
+- len = round_up(len, 8);
+- }
+
+ if (check_smb2_hdr(hdr))
+ return 1;
+diff --git a/fs/ksmbd/smb2ops.c b/fs/ksmbd/smb2ops.c
+index fb6a65d231391..f0a5b704f301c 100644
+--- a/fs/ksmbd/smb2ops.c
++++ b/fs/ksmbd/smb2ops.c
+@@ -20,6 +20,7 @@ static struct smb_version_values smb21_server_values = {
+ .max_read_size = SMB21_DEFAULT_IOSIZE,
+ .max_write_size = SMB21_DEFAULT_IOSIZE,
+ .max_trans_size = SMB21_DEFAULT_IOSIZE,
++ .max_credits = SMB2_MAX_CREDITS,
+ .large_lock_type = 0,
+ .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE,
+ .shared_lock_type = SMB2_LOCKFLAG_SHARED,
+@@ -45,6 +46,7 @@ static struct smb_version_values smb30_server_values = {
+ .max_read_size = SMB3_DEFAULT_IOSIZE,
+ .max_write_size = SMB3_DEFAULT_IOSIZE,
+ .max_trans_size = SMB3_DEFAULT_TRANS_SIZE,
++ .max_credits = SMB2_MAX_CREDITS,
+ .large_lock_type = 0,
+ .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE,
+ .shared_lock_type = SMB2_LOCKFLAG_SHARED,
+@@ -71,6 +73,7 @@ static struct smb_version_values smb302_server_values = {
+ .max_read_size = SMB3_DEFAULT_IOSIZE,
+ .max_write_size = SMB3_DEFAULT_IOSIZE,
+ .max_trans_size = SMB3_DEFAULT_TRANS_SIZE,
++ .max_credits = SMB2_MAX_CREDITS,
+ .large_lock_type = 0,
+ .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE,
+ .shared_lock_type = SMB2_LOCKFLAG_SHARED,
+@@ -97,6 +100,7 @@ static struct smb_version_values smb311_server_values = {
+ .max_read_size = SMB3_DEFAULT_IOSIZE,
+ .max_write_size = SMB3_DEFAULT_IOSIZE,
+ .max_trans_size = SMB3_DEFAULT_TRANS_SIZE,
++ .max_credits = SMB2_MAX_CREDITS,
+ .large_lock_type = 0,
+ .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE,
+ .shared_lock_type = SMB2_LOCKFLAG_SHARED,
+@@ -198,7 +202,6 @@ void init_smb2_1_server(struct ksmbd_conn *conn)
+ conn->ops = &smb2_0_server_ops;
+ conn->cmds = smb2_0_server_cmds;
+ conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds);
+- conn->max_credits = SMB2_MAX_CREDITS;
+ conn->signing_algorithm = SIGNING_ALG_HMAC_SHA256;
+
+ if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES)
+@@ -216,7 +219,6 @@ void init_smb3_0_server(struct ksmbd_conn *conn)
+ conn->ops = &smb3_0_server_ops;
+ conn->cmds = smb2_0_server_cmds;
+ conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds);
+- conn->max_credits = SMB2_MAX_CREDITS;
+ conn->signing_algorithm = SIGNING_ALG_AES_CMAC;
+
+ if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES)
+@@ -241,7 +243,6 @@ void init_smb3_02_server(struct ksmbd_conn *conn)
+ conn->ops = &smb3_0_server_ops;
+ conn->cmds = smb2_0_server_cmds;
+ conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds);
+- conn->max_credits = SMB2_MAX_CREDITS;
+ conn->signing_algorithm = SIGNING_ALG_AES_CMAC;
+
+ if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES)
+@@ -266,15 +267,11 @@ int init_smb3_11_server(struct ksmbd_conn *conn)
+ conn->ops = &smb3_11_server_ops;
+ conn->cmds = smb2_0_server_cmds;
+ conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds);
+- conn->max_credits = SMB2_MAX_CREDITS;
+ conn->signing_algorithm = SIGNING_ALG_AES_CMAC;
+
+ if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES)
+ conn->vals->capabilities |= SMB2_GLOBAL_CAP_LEASING;
+
+- if (conn->cipher_type)
+- conn->vals->capabilities |= SMB2_GLOBAL_CAP_ENCRYPTION;
+-
+ if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL)
+ conn->vals->capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL;
+
+@@ -308,3 +305,11 @@ void init_smb2_max_trans_size(unsigned int sz)
+ smb302_server_values.max_trans_size = sz;
+ smb311_server_values.max_trans_size = sz;
+ }
++
++void init_smb2_max_credits(unsigned int sz)
++{
++ smb21_server_values.max_credits = sz;
++ smb30_server_values.max_credits = sz;
++ smb302_server_values.max_credits = sz;
++ smb311_server_values.max_credits = sz;
++}
+diff --git a/fs/ksmbd/smb2pdu.c b/fs/ksmbd/smb2pdu.c
+index 7e448df3f8474..1ed3046dd5b3f 100644
+--- a/fs/ksmbd/smb2pdu.c
++++ b/fs/ksmbd/smb2pdu.c
+@@ -11,6 +11,7 @@
+ #include <linux/statfs.h>
+ #include <linux/ethtool.h>
+ #include <linux/falloc.h>
++#include <linux/mount.h>
+
+ #include "glob.h"
+ #include "smb2pdu.h"
+@@ -301,16 +302,15 @@ int smb2_set_rsp_credits(struct ksmbd_work *work)
+ struct smb2_hdr *req_hdr = ksmbd_req_buf_next(work);
+ struct smb2_hdr *hdr = ksmbd_resp_buf_next(work);
+ struct ksmbd_conn *conn = work->conn;
+- unsigned short credits_requested;
++ unsigned short credits_requested, aux_max;
+ unsigned short credit_charge, credits_granted = 0;
+- unsigned short aux_max, aux_credits;
+
+ if (work->send_no_response)
+ return 0;
+
+ hdr->CreditCharge = req_hdr->CreditCharge;
+
+- if (conn->total_credits > conn->max_credits) {
++ if (conn->total_credits > conn->vals->max_credits) {
+ hdr->CreditRequest = 0;
+ pr_err("Total credits overflow: %d\n", conn->total_credits);
+ return -EINVAL;
+@@ -318,6 +318,14 @@ int smb2_set_rsp_credits(struct ksmbd_work *work)
+
+ credit_charge = max_t(unsigned short,
+ le16_to_cpu(req_hdr->CreditCharge), 1);
++ if (credit_charge > conn->total_credits) {
++ ksmbd_debug(SMB, "Insufficient credits granted, given: %u, granted: %u\n",
++ credit_charge, conn->total_credits);
++ return -EINVAL;
++ }
++
++ conn->total_credits -= credit_charge;
++ conn->outstanding_credits -= credit_charge;
+ credits_requested = max_t(unsigned short,
+ le16_to_cpu(req_hdr->CreditRequest), 1);
+
+@@ -327,16 +335,14 @@ int smb2_set_rsp_credits(struct ksmbd_work *work)
+ * TODO: Need to adjuct CreditRequest value according to
+ * current cpu load
+ */
+- aux_credits = credits_requested - 1;
+ if (hdr->Command == SMB2_NEGOTIATE)
+- aux_max = 0;
++ aux_max = 1;
+ else
+- aux_max = conn->max_credits - credit_charge;
+- aux_credits = min_t(unsigned short, aux_credits, aux_max);
+- credits_granted = credit_charge + aux_credits;
++ aux_max = conn->vals->max_credits - credit_charge;
++ credits_granted = min_t(unsigned short, credits_requested, aux_max);
+
+- if (conn->max_credits - conn->total_credits < credits_granted)
+- credits_granted = conn->max_credits -
++ if (conn->vals->max_credits - conn->total_credits < credits_granted)
++ credits_granted = conn->vals->max_credits -
+ conn->total_credits;
+
+ conn->total_credits += credits_granted;
+@@ -917,6 +923,25 @@ static void decode_encrypt_ctxt(struct ksmbd_conn *conn,
+ }
+ }
+
++/**
++ * smb3_encryption_negotiated() - checks if server and client agreed on enabling encryption
++ * @conn: smb connection
++ *
++ * Return: true if connection should be encrypted, else false
++ */
++static bool smb3_encryption_negotiated(struct ksmbd_conn *conn)
++{
++ if (!conn->ops->generate_encryptionkey)
++ return false;
++
++ /*
++ * SMB 3.0 and 3.0.2 dialects use the SMB2_GLOBAL_CAP_ENCRYPTION flag.
++ * SMB 3.1.1 uses the cipher_type field.
++ */
++ return (conn->vals->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION) ||
++ conn->cipher_type;
++}
++
+ static void decode_compress_ctxt(struct ksmbd_conn *conn,
+ struct smb2_compression_ctx *pneg_ctxt)
+ {
+@@ -1438,11 +1463,6 @@ static int ntlm_authenticate(struct ksmbd_work *work)
+
+ sess->user = user;
+ if (user_guest(sess->user)) {
+- if (conn->sign) {
+- ksmbd_debug(SMB, "Guest login not allowed when signing enabled\n");
+- return -EPERM;
+- }
+-
+ rsp->SessionFlags = SMB2_SESSION_FLAG_IS_GUEST_LE;
+ } else {
+ struct authenticate_message *authblob;
+@@ -1455,39 +1475,39 @@ static int ntlm_authenticate(struct ksmbd_work *work)
+ ksmbd_debug(SMB, "authentication failed\n");
+ return -EPERM;
+ }
++ }
+
+- /*
+- * If session state is SMB2_SESSION_VALID, We can assume
+- * that it is reauthentication. And the user/password
+- * has been verified, so return it here.
+- */
+- if (sess->state == SMB2_SESSION_VALID) {
+- if (conn->binding)
+- goto binding_session;
+- return 0;
+- }
++ /*
++ * If session state is SMB2_SESSION_VALID, We can assume
++ * that it is reauthentication. And the user/password
++ * has been verified, so return it here.
++ */
++ if (sess->state == SMB2_SESSION_VALID) {
++ if (conn->binding)
++ goto binding_session;
++ return 0;
++ }
+
+- if ((conn->sign || server_conf.enforced_signing) ||
+- (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED))
+- sess->sign = true;
++ if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE &&
++ (conn->sign || server_conf.enforced_signing)) ||
++ (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED))
++ sess->sign = true;
+
+- if (conn->vals->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION &&
+- conn->ops->generate_encryptionkey &&
+- !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) {
+- rc = conn->ops->generate_encryptionkey(sess);
+- if (rc) {
+- ksmbd_debug(SMB,
+- "SMB3 encryption key generation failed\n");
+- return -EINVAL;
+- }
+- sess->enc = true;
+- rsp->SessionFlags = SMB2_SESSION_FLAG_ENCRYPT_DATA_LE;
+- /*
+- * signing is disable if encryption is enable
+- * on this session
+- */
+- sess->sign = false;
++ if (smb3_encryption_negotiated(conn) &&
++ !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) {
++ rc = conn->ops->generate_encryptionkey(sess);
++ if (rc) {
++ ksmbd_debug(SMB,
++ "SMB3 encryption key generation failed\n");
++ return -EINVAL;
+ }
++ sess->enc = true;
++ rsp->SessionFlags = SMB2_SESSION_FLAG_ENCRYPT_DATA_LE;
++ /*
++ * signing is disable if encryption is enable
++ * on this session
++ */
++ sess->sign = false;
+ }
+
+ binding_session:
+@@ -1562,8 +1582,7 @@ static int krb5_authenticate(struct ksmbd_work *work)
+ (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED))
+ sess->sign = true;
+
+- if ((conn->vals->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION) &&
+- conn->ops->generate_encryptionkey) {
++ if (smb3_encryption_negotiated(conn)) {
+ retval = conn->ops->generate_encryptionkey(sess);
+ if (retval) {
+ ksmbd_debug(SMB,
+@@ -1700,8 +1719,10 @@ int smb2_sess_setup(struct ksmbd_work *work)
+ negblob_off = le16_to_cpu(req->SecurityBufferOffset);
+ negblob_len = le16_to_cpu(req->SecurityBufferLength);
+ if (negblob_off < (offsetof(struct smb2_sess_setup_req, Buffer) - 4) ||
+- negblob_len < offsetof(struct negotiate_message, NegotiateFlags))
+- return -EINVAL;
++ negblob_len < offsetof(struct negotiate_message, NegotiateFlags)) {
++ rc = -EINVAL;
++ goto out_err;
++ }
+
+ negblob = (struct negotiate_message *)((char *)&req->hdr.ProtocolId +
+ negblob_off);
+@@ -2670,7 +2691,7 @@ int smb2_open(struct ksmbd_work *work)
+ (struct create_posix *)context;
+ if (le16_to_cpu(context->DataOffset) +
+ le32_to_cpu(context->DataLength) <
+- sizeof(struct create_posix)) {
++ sizeof(struct create_posix) - 4) {
+ rc = -EINVAL;
+ goto err_out1;
+ }
+@@ -2962,6 +2983,10 @@ int smb2_open(struct ksmbd_work *work)
+ &pntsd_size, &fattr);
+ posix_acl_release(fattr.cf_acls);
+ posix_acl_release(fattr.cf_dacls);
++ if (rc) {
++ kfree(pntsd);
++ goto err_out;
++ }
+
+ rc = ksmbd_vfs_set_sd_xattr(conn,
+ user_ns,
+@@ -3398,9 +3423,9 @@ static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level,
+ goto free_conv_name;
+ }
+
+- struct_sz = readdir_info_level_struct_sz(info_level);
+- next_entry_offset = ALIGN(struct_sz - 1 + conv_len,
+- KSMBD_DIR_INFO_ALIGNMENT);
++ struct_sz = readdir_info_level_struct_sz(info_level) - 1 + conv_len;
++ next_entry_offset = ALIGN(struct_sz, KSMBD_DIR_INFO_ALIGNMENT);
++ d_info->last_entry_off_align = next_entry_offset - struct_sz;
+
+ if (next_entry_offset > d_info->out_buf_len) {
+ d_info->out_buf_len = 0;
+@@ -3952,6 +3977,7 @@ int smb2_query_dir(struct ksmbd_work *work)
+ ((struct file_directory_info *)
+ ((char *)rsp->Buffer + d_info.last_entry_offset))
+ ->NextEntryOffset = 0;
++ d_info.data_count -= d_info.last_entry_off_align;
+
+ rsp->StructureSize = cpu_to_le16(9);
+ rsp->OutputBufferOffset = cpu_to_le16(72);
+@@ -4450,6 +4476,12 @@ static void get_file_stream_info(struct ksmbd_work *work,
+ &stat);
+ file_info = (struct smb2_file_stream_info *)rsp->Buffer;
+
++ buf_free_len =
++ smb2_calc_max_out_buf_len(work, 8,
++ le32_to_cpu(req->OutputBufferLength));
++ if (buf_free_len < 0)
++ goto out;
++
+ xattr_list_len = ksmbd_vfs_listxattr(path->dentry, &xattr_list);
+ if (xattr_list_len < 0) {
+ goto out;
+@@ -4458,12 +4490,6 @@ static void get_file_stream_info(struct ksmbd_work *work,
+ goto out;
+ }
+
+- buf_free_len =
+- smb2_calc_max_out_buf_len(work, 8,
+- le32_to_cpu(req->OutputBufferLength));
+- if (buf_free_len < 0)
+- goto out;
+-
+ while (idx < xattr_list_len) {
+ stream_name = xattr_list + idx;
+ streamlen = strlen(stream_name);
+@@ -4489,8 +4515,10 @@ static void get_file_stream_info(struct ksmbd_work *work,
+ ":%s", &stream_name[XATTR_NAME_STREAM_LEN]);
+
+ next = sizeof(struct smb2_file_stream_info) + streamlen * 2;
+- if (next > buf_free_len)
++ if (next > buf_free_len) {
++ kfree(stream_buf);
+ break;
++ }
+
+ file_info = (struct smb2_file_stream_info *)&rsp->Buffer[nbytes];
+ streamlen = smbConvertToUTF16((__le16 *)file_info->StreamName,
+@@ -4507,6 +4535,7 @@ static void get_file_stream_info(struct ksmbd_work *work,
+ file_info->NextEntryOffset = cpu_to_le32(next);
+ }
+
++out:
+ if (!S_ISDIR(stat.mode) &&
+ buf_free_len >= sizeof(struct smb2_file_stream_info) + 7 * 2) {
+ file_info = (struct smb2_file_stream_info *)
+@@ -4515,14 +4544,13 @@ static void get_file_stream_info(struct ksmbd_work *work,
+ "::$DATA", 7, conn->local_nls, 0);
+ streamlen *= 2;
+ file_info->StreamNameLength = cpu_to_le32(streamlen);
+- file_info->StreamSize = 0;
+- file_info->StreamAllocationSize = 0;
++ file_info->StreamSize = cpu_to_le64(stat.size);
++ file_info->StreamAllocationSize = cpu_to_le64(stat.blocks << 9);
+ nbytes += sizeof(struct smb2_file_stream_info) + streamlen;
+ }
+
+ /* last entry offset should be 0 */
+ file_info->NextEntryOffset = 0;
+-out:
+ kvfree(xattr_list);
+
+ rsp->OutputBufferLength = cpu_to_le32(nbytes);
+@@ -4891,11 +4919,18 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
+ {
+ struct filesystem_vol_info *info;
+ size_t sz;
++ unsigned int serial_crc = 0;
+
+ info = (struct filesystem_vol_info *)(rsp->Buffer);
+ info->VolumeCreationTime = 0;
++ serial_crc = crc32_le(serial_crc, share->name,
++ strlen(share->name));
++ serial_crc = crc32_le(serial_crc, share->path,
++ strlen(share->path));
++ serial_crc = crc32_le(serial_crc, ksmbd_netbios_name(),
++ strlen(ksmbd_netbios_name()));
+ /* Taking dummy value of serial number*/
+- info->SerialNumber = cpu_to_le32(0xbc3ac512);
++ info->SerialNumber = cpu_to_le32(serial_crc);
+ len = smbConvertToUTF16((__le16 *)info->VolumeLabel,
+ share->name, PATH_MAX,
+ conn->local_nls, 0);
+@@ -4963,15 +4998,17 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
+ case FS_SECTOR_SIZE_INFORMATION:
+ {
+ struct smb3_fs_ss_info *info;
++ unsigned int sector_size =
++ min_t(unsigned int, path.mnt->mnt_sb->s_blocksize, 4096);
+
+ info = (struct smb3_fs_ss_info *)(rsp->Buffer);
+
+- info->LogicalBytesPerSector = cpu_to_le32(stfs.f_bsize);
++ info->LogicalBytesPerSector = cpu_to_le32(sector_size);
+ info->PhysicalBytesPerSectorForAtomicity =
+- cpu_to_le32(stfs.f_bsize);
+- info->PhysicalBytesPerSectorForPerf = cpu_to_le32(stfs.f_bsize);
++ cpu_to_le32(sector_size);
++ info->PhysicalBytesPerSectorForPerf = cpu_to_le32(sector_size);
+ info->FSEffPhysicalBytesPerSectorForAtomicity =
+- cpu_to_le32(stfs.f_bsize);
++ cpu_to_le32(sector_size);
+ info->Flags = cpu_to_le32(SSINFO_FLAGS_ALIGNED_DEVICE |
+ SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE);
+ info->ByteOffsetForSectorAlignment = 0;
+@@ -5053,7 +5090,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work,
+ if (addition_info & ~(OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO |
+ PROTECTED_DACL_SECINFO |
+ UNPROTECTED_DACL_SECINFO)) {
+- pr_err("Unsupported addition info: 0x%x)\n",
++ ksmbd_debug(SMB, "Unsupported addition info: 0x%x)\n",
+ addition_info);
+
+ pntsd->revision = cpu_to_le16(1);
+@@ -5734,8 +5771,10 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp,
+ if (parent_fp) {
+ if (parent_fp->daccess & FILE_DELETE_LE) {
+ pr_err("parent dir is opened with delete access\n");
++ ksmbd_fd_put(work, parent_fp);
+ return -ESHARE;
+ }
++ ksmbd_fd_put(work, parent_fp);
+ }
+ next:
+ return smb2_rename(work, fp, user_ns, rename_info,
+@@ -7312,7 +7351,7 @@ static int fsctl_validate_negotiate_info(struct ksmbd_conn *conn,
+ int ret = 0;
+ int dialect;
+
+- if (in_buf_len < sizeof(struct validate_negotiate_info_req) +
++ if (in_buf_len < offsetof(struct validate_negotiate_info_req, Dialects) +
+ le16_to_cpu(neg_req->DialectCount) * sizeof(__le16))
+ return -EINVAL;
+
+diff --git a/fs/ksmbd/smb2pdu.h b/fs/ksmbd/smb2pdu.h
+index ff5a2f01d34ae..4f8574944ac19 100644
+--- a/fs/ksmbd/smb2pdu.h
++++ b/fs/ksmbd/smb2pdu.h
+@@ -1647,6 +1647,7 @@ int init_smb3_11_server(struct ksmbd_conn *conn);
+ void init_smb2_max_read_size(unsigned int sz);
+ void init_smb2_max_write_size(unsigned int sz);
+ void init_smb2_max_trans_size(unsigned int sz);
++void init_smb2_max_credits(unsigned int sz);
+
+ bool is_smb2_neg_cmd(struct ksmbd_work *work);
+ bool is_smb2_rsp(struct ksmbd_work *work);
+diff --git a/fs/ksmbd/smb_common.c b/fs/ksmbd/smb_common.c
+index 707490ab1f4c4..f2e7e3a654b34 100644
+--- a/fs/ksmbd/smb_common.c
++++ b/fs/ksmbd/smb_common.c
+@@ -308,14 +308,17 @@ int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, int info_level,
+ for (i = 0; i < 2; i++) {
+ struct kstat kstat;
+ struct ksmbd_kstat ksmbd_kstat;
++ struct dentry *dentry;
+
+ if (!dir->dot_dotdot[i]) { /* fill dot entry info */
+ if (i == 0) {
+ d_info->name = ".";
+ d_info->name_len = 1;
++ dentry = dir->filp->f_path.dentry;
+ } else {
+ d_info->name = "..";
+ d_info->name_len = 2;
++ dentry = dir->filp->f_path.dentry->d_parent;
+ }
+
+ if (!match_pattern(d_info->name, d_info->name_len,
+@@ -327,7 +330,7 @@ int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, int info_level,
+ ksmbd_kstat.kstat = &kstat;
+ ksmbd_vfs_fill_dentry_attrs(work,
+ user_ns,
+- dir->filp->f_path.dentry->d_parent,
++ dentry,
+ &ksmbd_kstat);
+ rc = fn(conn, info_level, d_info, &ksmbd_kstat);
+ if (rc)
+diff --git a/fs/ksmbd/smb_common.h b/fs/ksmbd/smb_common.h
+index 6e79e7577f6b7..1eba8dabaf317 100644
+--- a/fs/ksmbd/smb_common.h
++++ b/fs/ksmbd/smb_common.h
+@@ -412,6 +412,7 @@ struct smb_version_values {
+ __u32 max_read_size;
+ __u32 max_write_size;
+ __u32 max_trans_size;
++ __u32 max_credits;
+ __u32 large_lock_type;
+ __u32 exclusive_lock_type;
+ __u32 shared_lock_type;
+diff --git a/fs/ksmbd/transport_ipc.c b/fs/ksmbd/transport_ipc.c
+index 1acf1892a466c..3ad6881e0f7ed 100644
+--- a/fs/ksmbd/transport_ipc.c
++++ b/fs/ksmbd/transport_ipc.c
+@@ -301,6 +301,8 @@ static int ipc_server_config_on_startup(struct ksmbd_startup_request *req)
+ init_smb2_max_write_size(req->smb2_max_write);
+ if (req->smb2_max_trans)
+ init_smb2_max_trans_size(req->smb2_max_trans);
++ if (req->smb2_max_credits)
++ init_smb2_max_credits(req->smb2_max_credits);
+
+ ret = ksmbd_set_netbios_name(req->netbios_name);
+ ret |= ksmbd_set_server_string(req->server_string);
+diff --git a/fs/ksmbd/transport_tcp.c b/fs/ksmbd/transport_tcp.c
+index c14320e03b698..82a1429bbe127 100644
+--- a/fs/ksmbd/transport_tcp.c
++++ b/fs/ksmbd/transport_tcp.c
+@@ -404,7 +404,7 @@ static int create_socket(struct interface *iface)
+ &ksmbd_socket);
+ if (ret) {
+ pr_err("Can't create socket for ipv4: %d\n", ret);
+- goto out_error;
++ goto out_clear;
+ }
+
+ sin.sin_family = PF_INET;
+@@ -462,6 +462,7 @@ static int create_socket(struct interface *iface)
+
+ out_error:
+ tcp_destroy_socket(ksmbd_socket);
++out_clear:
+ iface->ksmbd_socket = NULL;
+ return ret;
+ }
+diff --git a/fs/ksmbd/vfs.h b/fs/ksmbd/vfs.h
+index b0d5b8feb4a36..432c947731779 100644
+--- a/fs/ksmbd/vfs.h
++++ b/fs/ksmbd/vfs.h
+@@ -86,6 +86,7 @@ struct ksmbd_dir_info {
+ int last_entry_offset;
+ bool hide_dot_file;
+ int flags;
++ int last_entry_off_align;
+ };
+
+ struct ksmbd_readdir_data {
+diff --git a/fs/ksmbd/vfs_cache.c b/fs/ksmbd/vfs_cache.c
+index 29c1db66bd0f7..8b873d92d7854 100644
+--- a/fs/ksmbd/vfs_cache.c
++++ b/fs/ksmbd/vfs_cache.c
+@@ -497,6 +497,7 @@ struct ksmbd_file *ksmbd_lookup_fd_inode(struct inode *inode)
+ list_for_each_entry(lfp, &ci->m_fp_list, node) {
+ if (inode == file_inode(lfp->filp)) {
+ atomic_dec(&ci->m_count);
++ lfp = ksmbd_fp_get(lfp);
+ read_unlock(&ci->m_lock);
+ return lfp;
+ }
+diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c
+index cb3a7512c33ec..0a22a2faf5522 100644
+--- a/fs/lockd/svcsubs.c
++++ b/fs/lockd/svcsubs.c
+@@ -179,19 +179,21 @@ nlm_delete_file(struct nlm_file *file)
+ static int nlm_unlock_files(struct nlm_file *file)
+ {
+ struct file_lock lock;
+- struct file *f;
+
++ locks_init_lock(&lock);
+ lock.fl_type = F_UNLCK;
+ lock.fl_start = 0;
+ lock.fl_end = OFFSET_MAX;
+- for (f = file->f_file[0]; f <= file->f_file[1]; f++) {
+- if (f && vfs_lock_file(f, F_SETLK, &lock, NULL) < 0) {
+- pr_warn("lockd: unlock failure in %s:%d\n",
+- __FILE__, __LINE__);
+- return 1;
+- }
+- }
++ if (file->f_file[O_RDONLY] &&
++ vfs_lock_file(file->f_file[O_RDONLY], F_SETLK, &lock, NULL))
++ goto out_err;
++ if (file->f_file[O_WRONLY] &&
++ vfs_lock_file(file->f_file[O_WRONLY], F_SETLK, &lock, NULL))
++ goto out_err;
+ return 0;
++out_err:
++ pr_warn("lockd: unlock failure in %s:%d\n", __FILE__, __LINE__);
++ return 1;
+ }
+
+ /*
+diff --git a/fs/minix/inode.c b/fs/minix/inode.c
+index a71f1cf894b9f..d4bd94234ef73 100644
+--- a/fs/minix/inode.c
++++ b/fs/minix/inode.c
+@@ -447,7 +447,8 @@ static const struct address_space_operations minix_aops = {
+ .writepage = minix_writepage,
+ .write_begin = minix_write_begin,
+ .write_end = generic_write_end,
+- .bmap = minix_bmap
++ .bmap = minix_bmap,
++ .direct_IO = noop_direct_IO
+ };
+
+ static const struct inode_operations minix_symlink_inode_operations = {
+diff --git a/fs/namei.c b/fs/namei.c
+index 1946d96677908..8882a70dc1192 100644
+--- a/fs/namei.c
++++ b/fs/namei.c
+@@ -3625,18 +3625,14 @@ static struct dentry *filename_create(int dfd, struct filename *name,
+ {
+ struct dentry *dentry = ERR_PTR(-EEXIST);
+ struct qstr last;
++ bool want_dir = lookup_flags & LOOKUP_DIRECTORY;
++ unsigned int reval_flag = lookup_flags & LOOKUP_REVAL;
++ unsigned int create_flags = LOOKUP_CREATE | LOOKUP_EXCL;
+ int type;
+ int err2;
+ int error;
+- bool is_dir = (lookup_flags & LOOKUP_DIRECTORY);
+
+- /*
+- * Note that only LOOKUP_REVAL and LOOKUP_DIRECTORY matter here. Any
+- * other flags passed in are ignored!
+- */
+- lookup_flags &= LOOKUP_REVAL;
+-
+- error = filename_parentat(dfd, name, lookup_flags, path, &last, &type);
++ error = filename_parentat(dfd, name, reval_flag, path, &last, &type);
+ if (error)
+ return ERR_PTR(error);
+
+@@ -3650,11 +3646,13 @@ static struct dentry *filename_create(int dfd, struct filename *name,
+ /* don't fail immediately if it's r/o, at least try to report other errors */
+ err2 = mnt_want_write(path->mnt);
+ /*
+- * Do the final lookup.
++ * Do the final lookup. Suppress 'create' if there is a trailing
++ * '/', and a directory wasn't requested.
+ */
+- lookup_flags |= LOOKUP_CREATE | LOOKUP_EXCL;
++ if (last.name[last.len] && !want_dir)
++ create_flags = 0;
+ inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
+- dentry = __lookup_hash(&last, path->dentry, lookup_flags);
++ dentry = __lookup_hash(&last, path->dentry, reval_flag | create_flags);
+ if (IS_ERR(dentry))
+ goto unlock;
+
+@@ -3668,7 +3666,7 @@ static struct dentry *filename_create(int dfd, struct filename *name,
+ * all is fine. Let's be bastards - you had / on the end, you've
+ * been asking for (non-existent) directory. -ENOENT for you.
+ */
+- if (unlikely(!is_dir && last.name[last.len])) {
++ if (unlikely(!create_flags)) {
+ error = -ENOENT;
+ goto fail;
+ }
+@@ -3975,13 +3973,12 @@ int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir,
+ dentry->d_inode->i_flags |= S_DEAD;
+ dont_mount(dentry);
+ detach_mounts(dentry);
+- fsnotify_rmdir(dir, dentry);
+
+ out:
+ inode_unlock(dentry->d_inode);
+ dput(dentry);
+ if (!error)
+- d_delete(dentry);
++ d_delete_notify(dir, dentry);
+ return error;
+ }
+ EXPORT_SYMBOL(vfs_rmdir);
+@@ -4103,7 +4100,6 @@ int vfs_unlink(struct user_namespace *mnt_userns, struct inode *dir,
+ if (!error) {
+ dont_mount(dentry);
+ detach_mounts(dentry);
+- fsnotify_unlink(dir, dentry);
+ }
+ }
+ }
+@@ -4111,9 +4107,11 @@ out:
+ inode_unlock(target);
+
+ /* We don't d_delete() NFS sillyrenamed files--they still exist. */
+- if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) {
++ if (!error && dentry->d_flags & DCACHE_NFSFS_RENAMED) {
++ fsnotify_unlink(dir, dentry);
++ } else if (!error) {
+ fsnotify_link_count(target);
+- d_delete(dentry);
++ d_delete_notify(dir, dentry);
+ }
+
+ return error;
+diff --git a/fs/namespace.c b/fs/namespace.c
+index 659a8f39c61af..b696543adab84 100644
+--- a/fs/namespace.c
++++ b/fs/namespace.c
+@@ -4263,12 +4263,11 @@ SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path,
+ return err;
+
+ err = user_path_at(dfd, path, kattr.lookup_flags, &target);
+- if (err)
+- return err;
+-
+- err = do_mount_setattr(&target, &kattr);
++ if (!err) {
++ err = do_mount_setattr(&target, &kattr);
++ path_put(&target);
++ }
+ finish_mount_kattr(&kattr);
+- path_put(&target);
+ return err;
+ }
+
+diff --git a/fs/netfs/read_helper.c b/fs/netfs/read_helper.c
+index 994ec22d40402..242f8bcb34a4c 100644
+--- a/fs/netfs/read_helper.c
++++ b/fs/netfs/read_helper.c
+@@ -354,16 +354,11 @@ static void netfs_rreq_write_to_cache_work(struct work_struct *work)
+ netfs_rreq_do_write_to_cache(rreq);
+ }
+
+-static void netfs_rreq_write_to_cache(struct netfs_read_request *rreq,
+- bool was_async)
++static void netfs_rreq_write_to_cache(struct netfs_read_request *rreq)
+ {
+- if (was_async) {
+- rreq->work.func = netfs_rreq_write_to_cache_work;
+- if (!queue_work(system_unbound_wq, &rreq->work))
+- BUG();
+- } else {
+- netfs_rreq_do_write_to_cache(rreq);
+- }
++ rreq->work.func = netfs_rreq_write_to_cache_work;
++ if (!queue_work(system_unbound_wq, &rreq->work))
++ BUG();
+ }
+
+ /*
+@@ -560,7 +555,7 @@ again:
+ wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
+
+ if (test_bit(NETFS_RREQ_WRITE_TO_CACHE, &rreq->flags))
+- return netfs_rreq_write_to_cache(rreq, was_async);
++ return netfs_rreq_write_to_cache(rreq);
+
+ netfs_rreq_completed(rreq, was_async);
+ }
+@@ -963,7 +958,7 @@ int netfs_readpage(struct file *file,
+ rreq = netfs_alloc_read_request(ops, netfs_priv, file);
+ if (!rreq) {
+ if (netfs_priv)
+- ops->cleanup(netfs_priv, page_file_mapping(page));
++ ops->cleanup(page_file_mapping(page), netfs_priv);
+ unlock_page(page);
+ return -ENOMEM;
+ }
+@@ -1190,7 +1185,7 @@ have_page:
+ goto error;
+ have_page_no_wait:
+ if (netfs_priv)
+- ops->cleanup(netfs_priv, mapping);
++ ops->cleanup(mapping, netfs_priv);
+ *_page = page;
+ _leave(" = 0");
+ return 0;
+@@ -1201,7 +1196,7 @@ error:
+ unlock_page(page);
+ put_page(page);
+ if (netfs_priv)
+- ops->cleanup(netfs_priv, mapping);
++ ops->cleanup(mapping, netfs_priv);
+ _leave(" = %d", ret);
+ return ret;
+ }
+diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
+index 6a2033131c068..ccd4f245cae24 100644
+--- a/fs/nfs/callback.h
++++ b/fs/nfs/callback.h
+@@ -170,7 +170,7 @@ struct cb_devicenotifyitem {
+ };
+
+ struct cb_devicenotifyargs {
+- int ndevs;
++ uint32_t ndevs;
+ struct cb_devicenotifyitem *devs;
+ };
+
+diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
+index ed9d580826f5a..a30dd35ec1c26 100644
+--- a/fs/nfs/callback_proc.c
++++ b/fs/nfs/callback_proc.c
+@@ -358,12 +358,11 @@ __be32 nfs4_callback_devicenotify(void *argp, void *resp,
+ struct cb_process_state *cps)
+ {
+ struct cb_devicenotifyargs *args = argp;
+- int i;
++ const struct pnfs_layoutdriver_type *ld = NULL;
++ uint32_t i;
+ __be32 res = 0;
+- struct nfs_client *clp = cps->clp;
+- struct nfs_server *server = NULL;
+
+- if (!clp) {
++ if (!cps->clp) {
+ res = cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION);
+ goto out;
+ }
+@@ -371,23 +370,15 @@ __be32 nfs4_callback_devicenotify(void *argp, void *resp,
+ for (i = 0; i < args->ndevs; i++) {
+ struct cb_devicenotifyitem *dev = &args->devs[i];
+
+- if (!server ||
+- server->pnfs_curr_ld->id != dev->cbd_layout_type) {
+- rcu_read_lock();
+- list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link)
+- if (server->pnfs_curr_ld &&
+- server->pnfs_curr_ld->id == dev->cbd_layout_type) {
+- rcu_read_unlock();
+- goto found;
+- }
+- rcu_read_unlock();
+- continue;
++ if (!ld || ld->id != dev->cbd_layout_type) {
++ pnfs_put_layoutdriver(ld);
++ ld = pnfs_find_layoutdriver(dev->cbd_layout_type);
++ if (!ld)
++ continue;
+ }
+-
+- found:
+- nfs4_delete_deviceid(server->pnfs_curr_ld, clp, &dev->cbd_dev_id);
++ nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id);
+ }
+-
++ pnfs_put_layoutdriver(ld);
+ out:
+ kfree(args->devs);
+ return res;
+diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
+index 4c48d85f65170..ea17085ef884b 100644
+--- a/fs/nfs/callback_xdr.c
++++ b/fs/nfs/callback_xdr.c
+@@ -258,11 +258,9 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp,
+ void *argp)
+ {
+ struct cb_devicenotifyargs *args = argp;
++ uint32_t tmp, n, i;
+ __be32 *p;
+ __be32 status = 0;
+- u32 tmp;
+- int n, i;
+- args->ndevs = 0;
+
+ /* Num of device notifications */
+ p = xdr_inline_decode(xdr, sizeof(uint32_t));
+@@ -271,12 +269,8 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp,
+ goto out;
+ }
+ n = ntohl(*p++);
+- if (n <= 0)
+- goto out;
+- if (n > ULONG_MAX / sizeof(*args->devs)) {
+- status = htonl(NFS4ERR_BADXDR);
++ if (n == 0)
+ goto out;
+- }
+
+ args->devs = kmalloc_array(n, sizeof(*args->devs), GFP_KERNEL);
+ if (!args->devs) {
+@@ -330,19 +324,21 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp,
+ dev->cbd_immediate = 0;
+ }
+
+- args->ndevs++;
+-
+ dprintk("%s: type %d layout 0x%x immediate %d\n",
+ __func__, dev->cbd_notify_type, dev->cbd_layout_type,
+ dev->cbd_immediate);
+ }
++ args->ndevs = n;
++ dprintk("%s: ndevs %d\n", __func__, args->ndevs);
++ return 0;
++err:
++ kfree(args->devs);
+ out:
++ args->devs = NULL;
++ args->ndevs = 0;
+ dprintk("%s: status %d ndevs %d\n",
+ __func__, ntohl(status), args->ndevs);
+ return status;
+-err:
+- kfree(args->devs);
+- goto out;
+ }
+
+ static __be32 decode_sessionid(struct xdr_stream *xdr,
+diff --git a/fs/nfs/client.c b/fs/nfs/client.c
+index 23e165d5ec9ca..090b16890e3d6 100644
+--- a/fs/nfs/client.c
++++ b/fs/nfs/client.c
+@@ -177,6 +177,7 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init)
+ INIT_LIST_HEAD(&clp->cl_superblocks);
+ clp->cl_rpcclient = ERR_PTR(-EINVAL);
+
++ clp->cl_flags = cl_init->init_flags;
+ clp->cl_proto = cl_init->proto;
+ clp->cl_nconnect = cl_init->nconnect;
+ clp->cl_max_connect = cl_init->max_connect ? cl_init->max_connect : 1;
+@@ -427,7 +428,6 @@ struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init)
+ list_add_tail(&new->cl_share_link,
+ &nn->nfs_client_list);
+ spin_unlock(&nn->nfs_client_lock);
+- new->cl_flags = cl_init->init_flags;
+ return rpc_ops->init_client(new, cl_init);
+ }
+
+@@ -860,6 +860,13 @@ int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, struct nfs
+ server->namelen = pathinfo.max_namelen;
+ }
+
++ if (clp->rpc_ops->discover_trunking != NULL &&
++ (server->caps & NFS_CAP_FS_LOCATIONS)) {
++ error = clp->rpc_ops->discover_trunking(server, mntfh);
++ if (error < 0)
++ return error;
++ }
++
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(nfs_probe_fsinfo);
+diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
+index 1a6d2867fba4f..78219396788b4 100644
+--- a/fs/nfs/dir.c
++++ b/fs/nfs/dir.c
+@@ -870,7 +870,8 @@ static int nfs_readdir_xdr_to_array(struct nfs_readdir_descriptor *desc,
+
+ status = nfs_readdir_page_filler(desc, entry, pages, pglen,
+ arrays, narrays);
+- } while (!status && nfs_readdir_page_needs_filling(page));
++ } while (!status && nfs_readdir_page_needs_filling(page) &&
++ page_mapping(page));
+
+ nfs_readdir_free_pages(pages, array_size);
+ out_release_label:
+@@ -1048,6 +1049,7 @@ static int uncached_readdir(struct nfs_readdir_descriptor *desc)
+ goto out;
+
+ desc->page_index = 0;
++ desc->cache_entry_index = 0;
+ desc->last_cookie = desc->dir_cookie;
+ desc->duped = 0;
+
+@@ -1269,13 +1271,12 @@ static bool nfs_verifier_is_delegated(struct dentry *dentry)
+ static void nfs_set_verifier_locked(struct dentry *dentry, unsigned long verf)
+ {
+ struct inode *inode = d_inode(dentry);
++ struct inode *dir = d_inode(dentry->d_parent);
+
+- if (!nfs_verifier_is_delegated(dentry) &&
+- !nfs_verify_change_attribute(d_inode(dentry->d_parent), verf))
+- goto out;
++ if (!nfs_verify_change_attribute(dir, verf))
++ return;
+ if (inode && NFS_PROTO(inode)->have_delegation(inode, FMODE_READ))
+ nfs_set_verifier_delegated(&verf);
+-out:
+ dentry->d_time = verf;
+ }
+
+@@ -1413,7 +1414,7 @@ out_force:
+ static void nfs_mark_dir_for_revalidate(struct inode *inode)
+ {
+ spin_lock(&inode->i_lock);
+- nfs_set_cache_invalid(inode, NFS_INO_REVAL_PAGECACHE);
++ nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE);
+ spin_unlock(&inode->i_lock);
+ }
+
+@@ -1834,16 +1835,6 @@ const struct dentry_operations nfs4_dentry_operations = {
+ };
+ EXPORT_SYMBOL_GPL(nfs4_dentry_operations);
+
+-static fmode_t flags_to_mode(int flags)
+-{
+- fmode_t res = (__force fmode_t)flags & FMODE_EXEC;
+- if ((flags & O_ACCMODE) != O_WRONLY)
+- res |= FMODE_READ;
+- if ((flags & O_ACCMODE) != O_RDONLY)
+- res |= FMODE_WRITE;
+- return res;
+-}
+-
+ static struct nfs_open_context *create_nfs_open_context(struct dentry *dentry, int open_flags, struct file *filp)
+ {
+ return alloc_nfs_open_context(dentry, flags_to_mode(open_flags), filp);
+@@ -1983,6 +1974,24 @@ out:
+
+ no_open:
+ res = nfs_lookup(dir, dentry, lookup_flags);
++ if (!res) {
++ inode = d_inode(dentry);
++ if ((lookup_flags & LOOKUP_DIRECTORY) && inode &&
++ !(S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)))
++ res = ERR_PTR(-ENOTDIR);
++ else if (inode && S_ISREG(inode->i_mode))
++ res = ERR_PTR(-EOPENSTALE);
++ } else if (!IS_ERR(res)) {
++ inode = d_inode(res);
++ if ((lookup_flags & LOOKUP_DIRECTORY) && inode &&
++ !(S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))) {
++ dput(res);
++ res = ERR_PTR(-ENOTDIR);
++ } else if (inode && S_ISREG(inode->i_mode)) {
++ dput(res);
++ res = ERR_PTR(-EOPENSTALE);
++ }
++ }
+ if (switched) {
+ d_lookup_done(dentry);
+ if (!res)
+@@ -2383,6 +2392,8 @@ nfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
+
+ trace_nfs_link_enter(inode, dir, dentry);
+ d_drop(dentry);
++ if (S_ISREG(inode->i_mode))
++ nfs_sync_inode(inode);
+ error = NFS_PROTO(dir)->link(inode, dir, &dentry->d_name);
+ if (error == 0) {
+ ihold(inode);
+@@ -2471,6 +2482,8 @@ int nfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir,
+ }
+ }
+
++ if (S_ISREG(old_inode->i_mode))
++ nfs_sync_inode(old_inode);
+ task = nfs_async_rename(old_dir, new_dir, old_dentry, new_dentry, NULL);
+ if (IS_ERR(task)) {
+ error = PTR_ERR(task);
+@@ -2676,7 +2689,7 @@ static struct nfs_access_entry *nfs_access_search_rbtree(struct inode *inode, co
+ return NULL;
+ }
+
+-static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res, bool may_block)
++static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *cred, u32 *mask, bool may_block)
+ {
+ struct nfs_inode *nfsi = NFS_I(inode);
+ struct nfs_access_entry *cache;
+@@ -2706,8 +2719,7 @@ static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *
+ spin_lock(&inode->i_lock);
+ retry = false;
+ }
+- res->cred = cache->cred;
+- res->mask = cache->mask;
++ *mask = cache->mask;
+ list_move_tail(&cache->lru, &nfsi->access_cache_entry_lru);
+ err = 0;
+ out:
+@@ -2719,7 +2731,7 @@ out_zap:
+ return -ENOENT;
+ }
+
+-static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res)
++static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cred, u32 *mask)
+ {
+ /* Only check the most recently returned cache entry,
+ * but do it without locking.
+@@ -2741,22 +2753,21 @@ static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cre
+ goto out;
+ if (nfs_check_cache_invalid(inode, NFS_INO_INVALID_ACCESS))
+ goto out;
+- res->cred = cache->cred;
+- res->mask = cache->mask;
++ *mask = cache->mask;
+ err = 0;
+ out:
+ rcu_read_unlock();
+ return err;
+ }
+
+-int nfs_access_get_cached(struct inode *inode, const struct cred *cred, struct
+-nfs_access_entry *res, bool may_block)
++int nfs_access_get_cached(struct inode *inode, const struct cred *cred,
++ u32 *mask, bool may_block)
+ {
+ int status;
+
+- status = nfs_access_get_cached_rcu(inode, cred, res);
++ status = nfs_access_get_cached_rcu(inode, cred, mask);
+ if (status != 0)
+- status = nfs_access_get_cached_locked(inode, cred, res,
++ status = nfs_access_get_cached_locked(inode, cred, mask,
+ may_block);
+
+ return status;
+@@ -2877,7 +2888,7 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask)
+
+ trace_nfs_access_enter(inode);
+
+- status = nfs_access_get_cached(inode, cred, &cache, may_block);
++ status = nfs_access_get_cached(inode, cred, &cache.mask, may_block);
+ if (status == 0)
+ goto out_cached;
+
+diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
+index 2e894fec036b0..c220810c61d14 100644
+--- a/fs/nfs/direct.c
++++ b/fs/nfs/direct.c
+@@ -172,8 +172,8 @@ ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+ VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
+
+ if (iov_iter_rw(iter) == READ)
+- return nfs_file_direct_read(iocb, iter);
+- return nfs_file_direct_write(iocb, iter);
++ return nfs_file_direct_read(iocb, iter, true);
++ return nfs_file_direct_write(iocb, iter, true);
+ }
+
+ static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
+@@ -424,6 +424,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
+ * nfs_file_direct_read - file direct read operation for NFS files
+ * @iocb: target I/O control block
+ * @iter: vector of user buffers into which to read data
++ * @swap: flag indicating this is swap IO, not O_DIRECT IO
+ *
+ * We use this function for direct reads instead of calling
+ * generic_file_aio_read() in order to avoid gfar's check to see if
+@@ -439,7 +440,8 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
+ * client must read the updated atime from the server back into its
+ * cache.
+ */
+-ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
++ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
++ bool swap)
+ {
+ struct file *file = iocb->ki_filp;
+ struct address_space *mapping = file->f_mapping;
+@@ -481,12 +483,14 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
+ if (iter_is_iovec(iter))
+ dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
+
+- nfs_start_io_direct(inode);
++ if (!swap)
++ nfs_start_io_direct(inode);
+
+ NFS_I(inode)->read_io += count;
+ requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
+
+- nfs_end_io_direct(inode);
++ if (!swap)
++ nfs_end_io_direct(inode);
+
+ if (requested > 0) {
+ result = nfs_direct_wait(dreq);
+@@ -620,7 +624,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
+ nfs_unlock_and_release_request(req);
+ }
+
+- if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
++ if (nfs_commit_end(cinfo.mds))
+ nfs_direct_write_complete(dreq);
+ }
+
+@@ -789,7 +793,7 @@ static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
+ */
+ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
+ struct iov_iter *iter,
+- loff_t pos)
++ loff_t pos, int ioflags)
+ {
+ struct nfs_pageio_descriptor desc;
+ struct inode *inode = dreq->inode;
+@@ -797,7 +801,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
+ size_t requested_bytes = 0;
+ size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
+
+- nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
++ nfs_pageio_init_write(&desc, inode, ioflags, false,
+ &nfs_direct_write_completion_ops);
+ desc.pg_dreq = dreq;
+ get_dreq(dreq);
+@@ -875,6 +879,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
+ * nfs_file_direct_write - file direct write operation for NFS files
+ * @iocb: target I/O control block
+ * @iter: vector of user buffers from which to write data
++ * @swap: flag indicating this is swap IO, not O_DIRECT IO
+ *
+ * We use this function for direct writes instead of calling
+ * generic_file_aio_write() in order to avoid taking the inode
+@@ -891,7 +896,8 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
+ * Note that O_APPEND is not supported for NFS direct writes, as there
+ * is no atomic O_APPEND write facility in the NFS protocol.
+ */
+-ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
++ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
++ bool swap)
+ {
+ ssize_t result, requested;
+ size_t count;
+@@ -905,7 +911,11 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
+ dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
+ file, iov_iter_count(iter), (long long) iocb->ki_pos);
+
+- result = generic_write_checks(iocb, iter);
++ if (swap)
++ /* bypass generic checks */
++ result = iov_iter_count(iter);
++ else
++ result = generic_write_checks(iocb, iter);
+ if (result <= 0)
+ return result;
+ count = result;
+@@ -936,16 +946,22 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
+ dreq->iocb = iocb;
+ pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
+
+- nfs_start_io_direct(inode);
++ if (swap) {
++ requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
++ FLUSH_STABLE);
++ } else {
++ nfs_start_io_direct(inode);
+
+- requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);
++ requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
++ FLUSH_COND_STABLE);
+
+- if (mapping->nrpages) {
+- invalidate_inode_pages2_range(mapping,
+- pos >> PAGE_SHIFT, end);
+- }
++ if (mapping->nrpages) {
++ invalidate_inode_pages2_range(mapping,
++ pos >> PAGE_SHIFT, end);
++ }
+
+- nfs_end_io_direct(inode);
++ nfs_end_io_direct(inode);
++ }
+
+ if (requested > 0) {
+ result = nfs_direct_wait(dreq);
+diff --git a/fs/nfs/file.c b/fs/nfs/file.c
+index aa353fd582404..42a16993913a8 100644
+--- a/fs/nfs/file.c
++++ b/fs/nfs/file.c
+@@ -161,7 +161,7 @@ nfs_file_read(struct kiocb *iocb, struct iov_iter *to)
+ ssize_t result;
+
+ if (iocb->ki_flags & IOCB_DIRECT)
+- return nfs_file_direct_read(iocb, to);
++ return nfs_file_direct_read(iocb, to, false);
+
+ dprintk("NFS: read(%pD2, %zu@%lu)\n",
+ iocb->ki_filp,
+@@ -616,7 +616,7 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from)
+ return result;
+
+ if (iocb->ki_flags & IOCB_DIRECT)
+- return nfs_file_direct_write(iocb, from);
++ return nfs_file_direct_write(iocb, from, false);
+
+ dprintk("NFS: write(%pD2, %zu@%Ld)\n",
+ file, iov_iter_count(from), (long long) iocb->ki_pos);
+diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+index c9b61b818ec11..bfa7202ca7be1 100644
+--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
++++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+@@ -378,10 +378,10 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
+ goto noconnect;
+
+ ds = mirror->mirror_ds->ds;
++ if (READ_ONCE(ds->ds_clp))
++ goto out;
+ /* matching smp_wmb() in _nfs4_pnfs_v3/4_ds_connect */
+ smp_rmb();
+- if (ds->ds_clp)
+- goto out;
+
+ /* FIXME: For now we assume the server sent only one version of NFS
+ * to use for the DS.
+diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c
+index 0d444a90f513a..fb3cad38b1497 100644
+--- a/fs/nfs/fs_context.c
++++ b/fs/nfs/fs_context.c
+@@ -514,7 +514,7 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
+ if (result.negated)
+ ctx->flags &= ~NFS_MOUNT_SOFTREVAL;
+ else
+- ctx->flags &= NFS_MOUNT_SOFTREVAL;
++ ctx->flags |= NFS_MOUNT_SOFTREVAL;
+ break;
+ case Opt_posix:
+ if (result.negated)
+diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
+index 853213b3a2095..f4f75db7a8258 100644
+--- a/fs/nfs/inode.c
++++ b/fs/nfs/inode.c
+@@ -210,10 +210,15 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags)
+ flags &= ~NFS_INO_INVALID_XATTR;
+ if (flags & NFS_INO_INVALID_DATA)
+ nfs_fscache_invalidate(inode);
+- if (inode->i_mapping->nrpages == 0)
+- flags &= ~(NFS_INO_INVALID_DATA|NFS_INO_DATA_INVAL_DEFER);
+ flags &= ~(NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED);
++
+ nfsi->cache_validity |= flags;
++
++ if (inode->i_mapping->nrpages == 0)
++ nfsi->cache_validity &= ~(NFS_INO_INVALID_DATA |
++ NFS_INO_DATA_INVAL_DEFER);
++ else if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
++ nfsi->cache_validity &= ~NFS_INO_DATA_INVAL_DEFER;
+ }
+ EXPORT_SYMBOL_GPL(nfs_set_cache_invalid);
+
+@@ -835,12 +840,9 @@ int nfs_getattr(struct user_namespace *mnt_userns, const struct path *path,
+ }
+
+ /* Flush out writes to the server in order to update c/mtime. */
+- if ((request_mask & (STATX_CTIME|STATX_MTIME)) &&
+- S_ISREG(inode->i_mode)) {
+- err = filemap_write_and_wait(inode->i_mapping);
+- if (err)
+- goto out;
+- }
++ if ((request_mask & (STATX_CTIME | STATX_MTIME)) &&
++ S_ISREG(inode->i_mode))
++ filemap_write_and_wait(inode->i_mapping);
+
+ /*
+ * We may force a getattr if the user cares about atime.
+@@ -1165,7 +1167,6 @@ int nfs_open(struct inode *inode, struct file *filp)
+ nfs_fscache_open_file(inode, filp);
+ return 0;
+ }
+-EXPORT_SYMBOL_GPL(nfs_open);
+
+ /*
+ * This function is called whenever some part of NFS notices that
+@@ -1777,8 +1778,10 @@ static int nfs_inode_finish_partial_attr_update(const struct nfs_fattr *fattr,
+ NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_OTHER |
+ NFS_INO_INVALID_NLINK;
+ unsigned long cache_validity = NFS_I(inode)->cache_validity;
++ enum nfs4_change_attr_type ctype = NFS_SERVER(inode)->change_attr_type;
+
+- if (!(cache_validity & NFS_INO_INVALID_CHANGE) &&
++ if (ctype != NFS4_CHANGE_TYPE_IS_UNDEFINED &&
++ !(cache_validity & NFS_INO_INVALID_CHANGE) &&
+ (cache_validity & check_valid) != 0 &&
+ (fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 &&
+ nfs_inode_attrs_cmp_monotonic(fattr, inode) == 0)
+diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
+index 66fc936834f23..c8845242d4225 100644
+--- a/fs/nfs/internal.h
++++ b/fs/nfs/internal.h
+@@ -42,6 +42,16 @@ static inline bool nfs_lookup_is_soft_revalidate(const struct dentry *dentry)
+ return true;
+ }
+
++static inline fmode_t flags_to_mode(int flags)
++{
++ fmode_t res = (__force fmode_t)flags & FMODE_EXEC;
++ if ((flags & O_ACCMODE) != O_WRONLY)
++ res |= FMODE_READ;
++ if ((flags & O_ACCMODE) != O_RDONLY)
++ res |= FMODE_WRITE;
++ return res;
++}
++
+ /*
+ * Note: RFC 1813 doesn't limit the number of auth flavors that
+ * a server can return, so make something up.
+@@ -580,6 +590,13 @@ nfs_write_match_verf(const struct nfs_writeverf *verf,
+ !nfs_write_verifier_cmp(&req->wb_verf, &verf->verifier);
+ }
+
++static inline gfp_t nfs_io_gfp_mask(void)
++{
++ if (current->flags & PF_WQ_WORKER)
++ return GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
++ return GFP_KERNEL;
++}
++
+ /* unlink.c */
+ extern struct rpc_task *
+ nfs_async_rename(struct inode *old_dir, struct inode *new_dir,
+diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c
+index 7fba7711e6b3a..3d5ba43f44bb6 100644
+--- a/fs/nfs/nfs2xdr.c
++++ b/fs/nfs/nfs2xdr.c
+@@ -949,7 +949,7 @@ int nfs2_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+
+ error = decode_filename_inline(xdr, &entry->name, &entry->len);
+ if (unlikely(error))
+- return error;
++ return -EAGAIN;
+
+ /*
+ * The type (size and byte order) of nfscookie isn't defined in
+diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c
+index e6eca1d7481b8..7ab60ad98776f 100644
+--- a/fs/nfs/nfs3xdr.c
++++ b/fs/nfs/nfs3xdr.c
+@@ -1967,7 +1967,6 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+ bool plus)
+ {
+ struct user_namespace *userns = rpc_userns(entry->server->client);
+- struct nfs_entry old = *entry;
+ __be32 *p;
+ int error;
+ u64 new_cookie;
+@@ -1987,15 +1986,15 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+
+ error = decode_fileid3(xdr, &entry->ino);
+ if (unlikely(error))
+- return error;
++ return -EAGAIN;
+
+ error = decode_inline_filename3(xdr, &entry->name, &entry->len);
+ if (unlikely(error))
+- return error;
++ return -EAGAIN;
+
+ error = decode_cookie3(xdr, &new_cookie);
+ if (unlikely(error))
+- return error;
++ return -EAGAIN;
+
+ entry->d_type = DT_UNKNOWN;
+
+@@ -2003,7 +2002,7 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+ entry->fattr->valid = 0;
+ error = decode_post_op_attr(xdr, entry->fattr, userns);
+ if (unlikely(error))
+- return error;
++ return -EAGAIN;
+ if (entry->fattr->valid & NFS_ATTR_FATTR_V3)
+ entry->d_type = nfs_umode_to_dtype(entry->fattr->mode);
+
+@@ -2018,11 +2017,8 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+ return -EAGAIN;
+ if (*p != xdr_zero) {
+ error = decode_nfs_fh3(xdr, entry->fh);
+- if (unlikely(error)) {
+- if (error == -E2BIG)
+- goto out_truncated;
+- return error;
+- }
++ if (unlikely(error))
++ return -EAGAIN;
+ } else
+ zero_nfs_fh3(entry->fh);
+ }
+@@ -2031,11 +2027,6 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
+ entry->cookie = new_cookie;
+
+ return 0;
+-
+-out_truncated:
+- dprintk("NFS: directory entry contains invalid file handle\n");
+- *entry = old;
+- return -EAGAIN;
+ }
+
+ /*
+@@ -2227,7 +2218,8 @@ static int decode_fsinfo3resok(struct xdr_stream *xdr,
+
+ /* ignore properties */
+ result->lease_time = 0;
+- result->change_attr_type = NFS4_CHANGE_TYPE_IS_TIME_METADATA;
++ result->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED;
++ result->xattr_support = 0;
+ return 0;
+ }
+
+diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
+index a24349512ffe9..93f4d8257525b 100644
+--- a/fs/nfs/nfs42proc.c
++++ b/fs/nfs/nfs42proc.c
+@@ -285,7 +285,9 @@ static void nfs42_copy_dest_done(struct inode *inode, loff_t pos, loff_t len)
+ loff_t newsize = pos + len;
+ loff_t end = newsize - 1;
+
+- truncate_pagecache_range(inode, pos, end);
++ WARN_ON_ONCE(invalidate_inode_pages2_range(inode->i_mapping,
++ pos >> PAGE_SHIFT, end >> PAGE_SHIFT));
++
+ spin_lock(&inode->i_lock);
+ if (newsize > i_size_read(inode))
+ i_size_write(inode, newsize);
+@@ -584,8 +586,10 @@ static int _nfs42_proc_copy_notify(struct file *src, struct file *dst,
+
+ ctx = get_nfs_open_context(nfs_file_open_context(src));
+ l_ctx = nfs_get_lock_context(ctx);
+- if (IS_ERR(l_ctx))
+- return PTR_ERR(l_ctx);
++ if (IS_ERR(l_ctx)) {
++ status = PTR_ERR(l_ctx);
++ goto out;
++ }
+
+ status = nfs4_set_rw_stateid(&args->cna_src_stateid, ctx, l_ctx,
+ FMODE_READ);
+@@ -593,7 +597,7 @@ static int _nfs42_proc_copy_notify(struct file *src, struct file *dst,
+ if (status) {
+ if (status == -EAGAIN)
+ status = -NFS4ERR_BAD_STATEID;
+- return status;
++ goto out;
+ }
+
+ status = nfs4_call_sync(src_server->client, src_server, &msg,
+@@ -601,6 +605,7 @@ static int _nfs42_proc_copy_notify(struct file *src, struct file *dst,
+ if (status == -ENOTSUPP)
+ src_server->caps &= ~NFS_CAP_COPY_NOTIFY;
+
++out:
+ put_nfs_open_context(nfs_file_open_context(src));
+ return status;
+ }
+diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c
+index c8bad735e4c19..271e5f92ed019 100644
+--- a/fs/nfs/nfs42xdr.c
++++ b/fs/nfs/nfs42xdr.c
+@@ -1434,8 +1434,7 @@ static int nfs4_xdr_dec_clone(struct rpc_rqst *rqstp,
+ status = decode_clone(xdr);
+ if (status)
+ goto out;
+- status = decode_getfattr(xdr, res->dst_fattr, res->server);
+-
++ decode_getfattr(xdr, res->dst_fattr, res->server);
+ out:
+ res->rpc_status = status;
+ return status;
+diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
+index ba78df4b13d94..f8672a34fd635 100644
+--- a/fs/nfs/nfs4_fs.h
++++ b/fs/nfs/nfs4_fs.h
+@@ -261,8 +261,8 @@ struct nfs4_state_maintenance_ops {
+ };
+
+ struct nfs4_mig_recovery_ops {
+- int (*get_locations)(struct inode *, struct nfs4_fs_locations *,
+- struct page *, const struct cred *);
++ int (*get_locations)(struct nfs_server *, struct nfs_fh *,
++ struct nfs4_fs_locations *, struct page *, const struct cred *);
+ int (*fsid_present)(struct inode *, const struct cred *);
+ };
+
+@@ -281,7 +281,8 @@ struct rpc_clnt *nfs4_negotiate_security(struct rpc_clnt *, struct inode *,
+ int nfs4_submount(struct fs_context *, struct nfs_server *);
+ int nfs4_replace_transport(struct nfs_server *server,
+ const struct nfs4_fs_locations *locations);
+-
++size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr *sa,
++ size_t salen, struct net *net, int port);
+ /* nfs4proc.c */
+ extern int nfs4_handle_exception(struct nfs_server *, int, struct nfs4_exception *);
+ extern int nfs4_async_handle_error(struct rpc_task *task,
+@@ -303,8 +304,9 @@ extern int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait);
+ extern int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle);
+ extern int nfs4_proc_fs_locations(struct rpc_clnt *, struct inode *, const struct qstr *,
+ struct nfs4_fs_locations *, struct page *);
+-extern int nfs4_proc_get_locations(struct inode *, struct nfs4_fs_locations *,
+- struct page *page, const struct cred *);
++extern int nfs4_proc_get_locations(struct nfs_server *, struct nfs_fh *,
++ struct nfs4_fs_locations *,
++ struct page *page, const struct cred *);
+ extern int nfs4_proc_fsid_present(struct inode *, const struct cred *);
+ extern struct rpc_clnt *nfs4_proc_lookup_mountpoint(struct inode *,
+ struct dentry *,
+diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
+index af57332503bed..ed06b68b2b4e9 100644
+--- a/fs/nfs/nfs4client.c
++++ b/fs/nfs/nfs4client.c
+@@ -1368,8 +1368,11 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname,
+ }
+ nfs_put_client(clp);
+
+- if (server->nfs_client->cl_hostname == NULL)
++ if (server->nfs_client->cl_hostname == NULL) {
+ server->nfs_client->cl_hostname = kstrdup(hostname, GFP_KERNEL);
++ if (server->nfs_client->cl_hostname == NULL)
++ return -ENOMEM;
++ }
+ nfs_server_insert_lists(server);
+
+ return nfs_probe_destination(server);
+diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
+index c91565227ea2a..4120e1cb3feef 100644
+--- a/fs/nfs/nfs4file.c
++++ b/fs/nfs/nfs4file.c
+@@ -32,6 +32,7 @@ nfs4_file_open(struct inode *inode, struct file *filp)
+ struct dentry *parent = NULL;
+ struct inode *dir;
+ unsigned openflags = filp->f_flags;
++ fmode_t f_mode;
+ struct iattr attr;
+ int err;
+
+@@ -50,8 +51,9 @@ nfs4_file_open(struct inode *inode, struct file *filp)
+ if (err)
+ return err;
+
++ f_mode = filp->f_mode;
+ if ((openflags & O_ACCMODE) == 3)
+- return nfs_open(inode, filp);
++ f_mode |= flags_to_mode(openflags);
+
+ /* We can't create new files here */
+ openflags &= ~(O_CREAT|O_EXCL);
+@@ -59,7 +61,7 @@ nfs4_file_open(struct inode *inode, struct file *filp)
+ parent = dget_parent(dentry);
+ dir = d_inode(parent);
+
+- ctx = alloc_nfs_open_context(file_dentry(filp), filp->f_mode, filp);
++ ctx = alloc_nfs_open_context(file_dentry(filp), f_mode, filp);
+ err = PTR_ERR(ctx);
+ if (IS_ERR(ctx))
+ goto out;
+diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c
+index 8d8aba305ecca..f331866dd4182 100644
+--- a/fs/nfs/nfs4idmap.c
++++ b/fs/nfs/nfs4idmap.c
+@@ -487,7 +487,7 @@ nfs_idmap_new(struct nfs_client *clp)
+ err_destroy_pipe:
+ rpc_destroy_pipe_data(idmap->idmap_pipe);
+ err:
+- get_user_ns(idmap->user_ns);
++ put_user_ns(idmap->user_ns);
+ kfree(idmap);
+ return error;
+ }
+diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c
+index 873342308dc0d..3680c8da510c9 100644
+--- a/fs/nfs/nfs4namespace.c
++++ b/fs/nfs/nfs4namespace.c
+@@ -164,16 +164,21 @@ static int nfs4_validate_fspath(struct dentry *dentry,
+ return 0;
+ }
+
+-static size_t nfs_parse_server_name(char *string, size_t len,
+- struct sockaddr *sa, size_t salen, struct net *net)
++size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr *sa,
++ size_t salen, struct net *net, int port)
+ {
+ ssize_t ret;
+
+ ret = rpc_pton(net, string, len, sa, salen);
+ if (ret == 0) {
+- ret = nfs_dns_resolve_name(net, string, len, sa, salen);
+- if (ret < 0)
+- ret = 0;
++ ret = rpc_uaddr2sockaddr(net, string, len, sa, salen);
++ if (ret == 0) {
++ ret = nfs_dns_resolve_name(net, string, len, sa, salen);
++ if (ret < 0)
++ ret = 0;
++ }
++ } else if (port) {
++ rpc_set_port(sa, port);
+ }
+ return ret;
+ }
+@@ -328,7 +333,7 @@ static int try_location(struct fs_context *fc,
+ nfs_parse_server_name(buf->data, buf->len,
+ &ctx->nfs_server.address,
+ sizeof(ctx->nfs_server._address),
+- fc->net_ns);
++ fc->net_ns, 0);
+ if (ctx->nfs_server.addrlen == 0)
+ continue;
+
+@@ -496,7 +501,7 @@ static int nfs4_try_replacing_one_location(struct nfs_server *server,
+ continue;
+
+ salen = nfs_parse_server_name(buf->data, buf->len,
+- sap, addr_bufsize, net);
++ sap, addr_bufsize, net, 0);
+ if (salen == 0)
+ continue;
+ rpc_set_port(sap, NFS_PORT);
+diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
+index e1214bb6b7ee5..ae6b3600ed11b 100644
+--- a/fs/nfs/nfs4proc.c
++++ b/fs/nfs/nfs4proc.c
+@@ -366,6 +366,14 @@ static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dent
+ kunmap_atomic(start);
+ }
+
++static void nfs4_fattr_set_prechange(struct nfs_fattr *fattr, u64 version)
++{
++ if (!(fattr->valid & NFS_ATTR_FATTR_PRECHANGE)) {
++ fattr->pre_change_attr = version;
++ fattr->valid |= NFS_ATTR_FATTR_PRECHANGE;
++ }
++}
++
+ static void nfs4_test_and_free_stateid(struct nfs_server *server,
+ nfs4_stateid *stateid,
+ const struct cred *cred)
+@@ -1232,8 +1240,7 @@ nfs4_update_changeattr_locked(struct inode *inode,
+ NFS_INO_INVALID_ACCESS | NFS_INO_INVALID_ACL |
+ NFS_INO_INVALID_SIZE | NFS_INO_INVALID_OTHER |
+ NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK |
+- NFS_INO_INVALID_MODE | NFS_INO_INVALID_XATTR |
+- NFS_INO_REVAL_PAGECACHE;
++ NFS_INO_INVALID_MODE | NFS_INO_INVALID_XATTR;
+ nfsi->attrtimeo = NFS_MINATTRTIMEO(inode);
+ }
+ nfsi->attrtimeo_timestamp = jiffies;
+@@ -1609,15 +1616,16 @@ static bool nfs_stateid_is_sequential(struct nfs4_state *state,
+ {
+ if (test_bit(NFS_OPEN_STATE, &state->flags)) {
+ /* The common case - we're updating to a new sequence number */
+- if (nfs4_stateid_match_other(stateid, &state->open_stateid) &&
+- nfs4_stateid_is_next(&state->open_stateid, stateid)) {
+- return true;
++ if (nfs4_stateid_match_other(stateid, &state->open_stateid)) {
++ if (nfs4_stateid_is_next(&state->open_stateid, stateid))
++ return true;
++ return false;
+ }
+- } else {
+- /* This is the first OPEN in this generation */
+- if (stateid->seqid == cpu_to_be32(1))
+- return true;
++ /* The server returned a new stateid */
+ }
++ /* This is the first OPEN in this generation */
++ if (stateid->seqid == cpu_to_be32(1))
++ return true;
+ return false;
+ }
+
+@@ -3893,6 +3901,8 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f
+ if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL)
+ server->caps |= NFS_CAP_SECURITY_LABEL;
+ #endif
++ if (res.attr_bitmask[0] & FATTR4_WORD0_FS_LOCATIONS)
++ server->caps |= NFS_CAP_FS_LOCATIONS;
+ if (!(res.attr_bitmask[0] & FATTR4_WORD0_FILEID))
+ server->fattr_valid &= ~NFS_ATTR_FATTR_FILEID;
+ if (!(res.attr_bitmask[1] & FATTR4_WORD1_MODE))
+@@ -3949,6 +3959,60 @@ int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle)
+ return err;
+ }
+
++static int _nfs4_discover_trunking(struct nfs_server *server,
++ struct nfs_fh *fhandle)
++{
++ struct nfs4_fs_locations *locations = NULL;
++ struct page *page;
++ const struct cred *cred;
++ struct nfs_client *clp = server->nfs_client;
++ const struct nfs4_state_maintenance_ops *ops =
++ clp->cl_mvops->state_renewal_ops;
++ int status = -ENOMEM;
++
++ cred = ops->get_state_renewal_cred(clp);
++ if (cred == NULL) {
++ cred = nfs4_get_clid_cred(clp);
++ if (cred == NULL)
++ return -ENOKEY;
++ }
++
++ page = alloc_page(GFP_KERNEL);
++ locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
++ if (page == NULL || locations == NULL)
++ goto out;
++
++ status = nfs4_proc_get_locations(server, fhandle, locations, page,
++ cred);
++ if (status)
++ goto out;
++out:
++ if (page)
++ __free_page(page);
++ kfree(locations);
++ return status;
++}
++
++static int nfs4_discover_trunking(struct nfs_server *server,
++ struct nfs_fh *fhandle)
++{
++ struct nfs4_exception exception = {
++ .interruptible = true,
++ };
++ struct nfs_client *clp = server->nfs_client;
++ int err = 0;
++
++ if (!nfs4_has_session(clp))
++ goto out;
++ do {
++ err = nfs4_handle_exception(server,
++ _nfs4_discover_trunking(server, fhandle),
++ &exception);
++ } while (exception.retry);
++out:
++ return err;
++}
++
+ static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle,
+ struct nfs_fsinfo *info)
+ {
+@@ -6502,7 +6566,9 @@ static void nfs4_delegreturn_release(void *calldata)
+ pnfs_roc_release(&data->lr.arg, &data->lr.res,
+ data->res.lr_ret);
+ if (inode) {
+- nfs_post_op_update_inode_force_wcc(inode, &data->fattr);
++ nfs4_fattr_set_prechange(&data->fattr,
++ inode_peek_iversion_raw(inode));
++ nfs_refresh_inode(inode, &data->fattr);
+ nfs_iput_and_deactive(inode);
+ }
+ kfree(calldata);
+@@ -7676,7 +7742,7 @@ static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler,
+ const char *key, const void *buf,
+ size_t buflen, int flags)
+ {
+- struct nfs_access_entry cache;
++ u32 mask;
+ int ret;
+
+ if (!nfs_server_capable(inode, NFS_CAP_XATTR))
+@@ -7691,8 +7757,8 @@ static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler,
+ * do a cached access check for the XA* flags to possibly avoid
+ * doing an RPC and getting EACCES back.
+ */
+- if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) {
+- if (!(cache.mask & NFS_ACCESS_XAWRITE))
++ if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) {
++ if (!(mask & NFS_ACCESS_XAWRITE))
+ return -EACCES;
+ }
+
+@@ -7713,14 +7779,14 @@ static int nfs4_xattr_get_nfs4_user(const struct xattr_handler *handler,
+ struct dentry *unused, struct inode *inode,
+ const char *key, void *buf, size_t buflen)
+ {
+- struct nfs_access_entry cache;
++ u32 mask;
+ ssize_t ret;
+
+ if (!nfs_server_capable(inode, NFS_CAP_XATTR))
+ return -EOPNOTSUPP;
+
+- if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) {
+- if (!(cache.mask & NFS_ACCESS_XAREAD))
++ if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) {
++ if (!(mask & NFS_ACCESS_XAREAD))
+ return -EACCES;
+ }
+
+@@ -7745,13 +7811,13 @@ nfs4_listxattr_nfs4_user(struct inode *inode, char *list, size_t list_len)
+ ssize_t ret, size;
+ char *buf;
+ size_t buflen;
+- struct nfs_access_entry cache;
++ u32 mask;
+
+ if (!nfs_server_capable(inode, NFS_CAP_XATTR))
+ return 0;
+
+- if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) {
+- if (!(cache.mask & NFS_ACCESS_XALIST))
++ if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) {
++ if (!(mask & NFS_ACCESS_XALIST))
+ return 0;
+ }
+
+@@ -7883,18 +7949,18 @@ int nfs4_proc_fs_locations(struct rpc_clnt *client, struct inode *dir,
+ * appended to this compound to identify the client ID which is
+ * performing recovery.
+ */
+-static int _nfs40_proc_get_locations(struct inode *inode,
++static int _nfs40_proc_get_locations(struct nfs_server *server,
++ struct nfs_fh *fhandle,
+ struct nfs4_fs_locations *locations,
+ struct page *page, const struct cred *cred)
+ {
+- struct nfs_server *server = NFS_SERVER(inode);
+ struct rpc_clnt *clnt = server->client;
+ u32 bitmask[2] = {
+ [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS,
+ };
+ struct nfs4_fs_locations_arg args = {
+ .clientid = server->nfs_client->cl_clientid,
+- .fh = NFS_FH(inode),
++ .fh = fhandle,
+ .page = page,
+ .bitmask = bitmask,
+ .migration = 1, /* skip LOOKUP */
+@@ -7940,17 +8006,17 @@ static int _nfs40_proc_get_locations(struct inode *inode,
+ * When the client supports GETATTR(fs_locations_info), it can
+ * be plumbed in here.
+ */
+-static int _nfs41_proc_get_locations(struct inode *inode,
++static int _nfs41_proc_get_locations(struct nfs_server *server,
++ struct nfs_fh *fhandle,
+ struct nfs4_fs_locations *locations,
+ struct page *page, const struct cred *cred)
+ {
+- struct nfs_server *server = NFS_SERVER(inode);
+ struct rpc_clnt *clnt = server->client;
+ u32 bitmask[2] = {
+ [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS,
+ };
+ struct nfs4_fs_locations_arg args = {
+- .fh = NFS_FH(inode),
++ .fh = fhandle,
+ .page = page,
+ .bitmask = bitmask,
+ .migration = 1, /* skip LOOKUP */
+@@ -7999,11 +8065,11 @@ static int _nfs41_proc_get_locations(struct inode *inode,
+ * -NFS4ERR_LEASE_MOVED is returned if the server still has leases
+ * from this client that require migration recovery.
+ */
+-int nfs4_proc_get_locations(struct inode *inode,
++int nfs4_proc_get_locations(struct nfs_server *server,
++ struct nfs_fh *fhandle,
+ struct nfs4_fs_locations *locations,
+ struct page *page, const struct cred *cred)
+ {
+- struct nfs_server *server = NFS_SERVER(inode);
+ struct nfs_client *clp = server->nfs_client;
+ const struct nfs4_mig_recovery_ops *ops =
+ clp->cl_mvops->mig_recovery_ops;
+@@ -8016,10 +8082,11 @@ int nfs4_proc_get_locations(struct inode *inode,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor,
+ clp->cl_hostname);
+- nfs_display_fhandle(NFS_FH(inode), __func__);
++ nfs_display_fhandle(fhandle, __func__);
+
+ do {
+- status = ops->get_locations(inode, locations, page, cred);
++ status = ops->get_locations(server, fhandle, locations, page,
++ cred);
+ if (status != -NFS4ERR_DELAY)
+ break;
+ nfs4_handle_exception(server, status, &exception);
+@@ -8284,6 +8351,7 @@ nfs4_bind_one_conn_to_session_done(struct rpc_task *task, void *calldata)
+ case -NFS4ERR_DEADSESSION:
+ nfs4_schedule_session_recovery(clp->cl_session,
+ task->tk_status);
++ return;
+ }
+ if (args->dir == NFS4_CDFC4_FORE_OR_BOTH &&
+ res->dir != NFS4_CDFS4_BOTH) {
+@@ -10513,6 +10581,7 @@ const struct nfs_rpc_ops nfs_v4_clientops = {
+ .free_client = nfs4_free_client,
+ .create_server = nfs4_create_server,
+ .clone_server = nfs_clone_server,
++ .discover_trunking = nfs4_discover_trunking,
+ };
+
+ static const struct xattr_handler nfs4_xattr_nfs4_acl_handler = {
+diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
+index f22818a80c2c7..57ea63e2cdb41 100644
+--- a/fs/nfs/nfs4state.c
++++ b/fs/nfs/nfs4state.c
+@@ -49,6 +49,7 @@
+ #include <linux/workqueue.h>
+ #include <linux/bitops.h>
+ #include <linux/jiffies.h>
++#include <linux/sched/mm.h>
+
+ #include <linux/sunrpc/clnt.h>
+
+@@ -2097,7 +2098,8 @@ static int nfs4_try_migration(struct nfs_server *server, const struct cred *cred
+ }
+
+ inode = d_inode(server->super->s_root);
+- result = nfs4_proc_get_locations(inode, locations, page, cred);
++ result = nfs4_proc_get_locations(server, NFS_FH(inode), locations,
++ page, cred);
+ if (result) {
+ dprintk("<-- %s: failed to retrieve fs_locations: %d\n",
+ __func__, result);
+@@ -2105,6 +2107,9 @@ static int nfs4_try_migration(struct nfs_server *server, const struct cred *cred
+ }
+
+ result = -NFS4ERR_NXIO;
++ if (!locations->nlocations)
++ goto out;
++
+ if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
+ dprintk("<-- %s: No fs_locations data, migration skipped\n",
+ __func__);
+@@ -2555,9 +2560,17 @@ static void nfs4_layoutreturn_any_run(struct nfs_client *clp)
+
+ static void nfs4_state_manager(struct nfs_client *clp)
+ {
++ unsigned int memflags;
+ int status = 0;
+ const char *section = "", *section_sep = "";
+
++ /*
++ * State recovery can deadlock if the direct reclaim code tries
++ * start NFS writeback. So ensure memory allocations are all
++ * GFP_NOFS.
++ */
++ memflags = memalloc_nofs_save();
++
+ /* Ensure exclusive access to NFSv4 state */
+ do {
+ trace_nfs4_state_mgr(clp);
+@@ -2652,6 +2665,7 @@ static void nfs4_state_manager(struct nfs_client *clp)
+ clear_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state);
+ }
+
++ memalloc_nofs_restore(memflags);
+ nfs4_end_drain_session(clp);
+ nfs4_clear_state_manager_bit(clp);
+
+@@ -2669,6 +2683,7 @@ static void nfs4_state_manager(struct nfs_client *clp)
+ return;
+ if (test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) != 0)
+ return;
++ memflags = memalloc_nofs_save();
+ } while (refcount_read(&clp->cl_count) > 1 && !signalled());
+ goto out_drain;
+
+@@ -2681,6 +2696,7 @@ out_error:
+ clp->cl_hostname, -status);
+ ssleep(1);
+ out_drain:
++ memalloc_nofs_restore(memflags);
+ nfs4_end_drain_session(clp);
+ nfs4_clear_state_manager_bit(clp);
+ }
+diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
+index a8cff19c6f00c..2a1bf0a72d5bf 100644
+--- a/fs/nfs/nfs4xdr.c
++++ b/fs/nfs/nfs4xdr.c
+@@ -3693,8 +3693,6 @@ static int decode_attr_fs_locations(struct xdr_stream *xdr, uint32_t *bitmap, st
+ if (unlikely(!p))
+ goto out_eio;
+ n = be32_to_cpup(p);
+- if (n <= 0)
+- goto out_eio;
+ for (res->nlocations = 0; res->nlocations < n; res->nlocations++) {
+ u32 m;
+ struct nfs4_fs_location *loc;
+@@ -4197,10 +4195,11 @@ static int decode_attr_security_label(struct xdr_stream *xdr, uint32_t *bitmap,
+ } else
+ printk(KERN_WARNING "%s: label too long (%u)!\n",
+ __func__, len);
++ if (label && label->label)
++ dprintk("%s: label=%.*s, len=%d, PI=%d, LFS=%d\n",
++ __func__, label->len, (char *)label->label,
++ label->len, label->pi, label->lfs);
+ }
+- if (label && label->label)
+- dprintk("%s: label=%s, len=%d, PI=%d, LFS=%d\n", __func__,
+- (char *)label->label, label->len, label->pi, label->lfs);
+ return status;
+ }
+
+diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c
+index cc232d1f16f2f..f2fe23e6c51f4 100644
+--- a/fs/nfs/pagelist.c
++++ b/fs/nfs/pagelist.c
+@@ -90,10 +90,10 @@ void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
+ }
+ }
+
+-static inline struct nfs_page *
+-nfs_page_alloc(void)
++static inline struct nfs_page *nfs_page_alloc(void)
+ {
+- struct nfs_page *p = kmem_cache_zalloc(nfs_page_cachep, GFP_KERNEL);
++ struct nfs_page *p =
++ kmem_cache_zalloc(nfs_page_cachep, nfs_io_gfp_mask());
+ if (p)
+ INIT_LIST_HEAD(&p->wb_list);
+ return p;
+@@ -901,7 +901,7 @@ int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
+ struct nfs_commit_info cinfo;
+ struct nfs_page_array *pg_array = &hdr->page_array;
+ unsigned int pagecount, pageused;
+- gfp_t gfp_flags = GFP_KERNEL;
++ gfp_t gfp_flags = nfs_io_gfp_mask();
+
+ pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
+ pg_array->npages = pagecount;
+@@ -988,7 +988,7 @@ nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc,
+ desc->pg_mirrors_dynamic = NULL;
+ if (mirror_count == 1)
+ return desc->pg_mirrors_static;
+- ret = kmalloc_array(mirror_count, sizeof(*ret), GFP_KERNEL);
++ ret = kmalloc_array(mirror_count, sizeof(*ret), nfs_io_gfp_mask());
+ if (ret != NULL) {
+ for (i = 0; i < mirror_count; i++)
+ nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);
+@@ -1227,6 +1227,7 @@ static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
+
+ do {
+ list_splice_init(&mirror->pg_list, &head);
++ mirror->pg_recoalesce = 0;
+
+ while (!list_empty(&head)) {
+ struct nfs_page *req;
+diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
+index 7c9090a28e5c3..7ddd003ab8b1a 100644
+--- a/fs/nfs/pnfs.c
++++ b/fs/nfs/pnfs.c
+@@ -92,6 +92,17 @@ find_pnfs_driver(u32 id)
+ return local;
+ }
+
++const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id)
++{
++ return find_pnfs_driver(id);
++}
++
++void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld)
++{
++ if (ld)
++ module_put(ld->owner);
++}
++
+ void
+ unset_pnfs_layoutdriver(struct nfs_server *nfss)
+ {
+diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
+index d810ae674f4e8..5a54cf8ac6f3d 100644
+--- a/fs/nfs/pnfs.h
++++ b/fs/nfs/pnfs.h
+@@ -238,6 +238,8 @@ struct pnfs_devicelist {
+
+ extern int pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *);
+ extern void pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *);
++extern const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id);
++extern void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld);
+
+ /* nfs4proc.c */
+ extern size_t max_response_pages(struct nfs_server *server);
+@@ -517,7 +519,7 @@ pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
+ {
+ struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
+
+- if (!lseg || !fl_cinfo->ops->mark_request_commit)
++ if (!lseg || !fl_cinfo->ops || !fl_cinfo->ops->mark_request_commit)
+ return false;
+ fl_cinfo->ops->mark_request_commit(req, lseg, cinfo, ds_commit_idx);
+ return true;
+diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
+index cf19914fec817..657c242a18ff1 100644
+--- a/fs/nfs/pnfs_nfs.c
++++ b/fs/nfs/pnfs_nfs.c
+@@ -419,7 +419,7 @@ static struct nfs_commit_data *
+ pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket *bucket,
+ struct nfs_commit_info *cinfo)
+ {
+- struct nfs_commit_data *data = nfs_commitdata_alloc(false);
++ struct nfs_commit_data *data = nfs_commitdata_alloc();
+
+ if (!data)
+ return NULL;
+@@ -468,7 +468,6 @@ pnfs_bucket_alloc_ds_commits(struct list_head *list,
+ goto out_error;
+ data->ds_commit_index = i;
+ list_add_tail(&data->list, list);
+- atomic_inc(&cinfo->mds->rpcs_out);
+ nreq++;
+ }
+ mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
+@@ -516,11 +515,14 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
+ unsigned int nreq = 0;
+
+ if (!list_empty(mds_pages)) {
+- data = nfs_commitdata_alloc(true);
++ data = nfs_commitdata_alloc();
++ if (!data) {
++ nfs_retry_commit(mds_pages, NULL, cinfo, -1);
++ return -ENOMEM;
++ }
+ data->ds_commit_index = -1;
+ list_splice_init(mds_pages, &data->pages);
+ list_add_tail(&data->list, &list);
+- atomic_inc(&cinfo->mds->rpcs_out);
+ nreq++;
+ }
+
+@@ -895,7 +897,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
+ }
+
+ smp_wmb();
+- ds->ds_clp = clp;
++ WRITE_ONCE(ds->ds_clp, clp);
+ dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
+ out:
+ return status;
+@@ -973,7 +975,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
+ }
+
+ smp_wmb();
+- ds->ds_clp = clp;
++ WRITE_ONCE(ds->ds_clp, clp);
+ dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
+ out:
+ return status;
+diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c
+index ea19dbf123014..a5b0bdcb53963 100644
+--- a/fs/nfs/proc.c
++++ b/fs/nfs/proc.c
+@@ -91,7 +91,8 @@ nfs_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle,
+ info->dtpref = fsinfo.tsize;
+ info->maxfilesize = 0x7FFFFFFF;
+ info->lease_time = 0;
+- info->change_attr_type = NFS4_CHANGE_TYPE_IS_TIME_METADATA;
++ info->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED;
++ info->xattr_support = 0;
+ return 0;
+ }
+
+diff --git a/fs/nfs/write.c b/fs/nfs/write.c
+index eae9bf1140417..d21b255114993 100644
+--- a/fs/nfs/write.c
++++ b/fs/nfs/write.c
+@@ -70,27 +70,17 @@ static mempool_t *nfs_wdata_mempool;
+ static struct kmem_cache *nfs_cdata_cachep;
+ static mempool_t *nfs_commit_mempool;
+
+-struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
++struct nfs_commit_data *nfs_commitdata_alloc(void)
+ {
+ struct nfs_commit_data *p;
+
+- if (never_fail)
+- p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
+- else {
+- /* It is OK to do some reclaim, not no safe to wait
+- * for anything to be returned to the pool.
+- * mempool_alloc() cannot handle that particular combination,
+- * so we need two separate attempts.
+- */
++ p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask());
++ if (!p) {
+ p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
+- if (!p)
+- p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
+- __GFP_NOWARN | __GFP_NORETRY);
+ if (!p)
+ return NULL;
++ memset(p, 0, sizeof(*p));
+ }
+-
+- memset(p, 0, sizeof(*p));
+ INIT_LIST_HEAD(&p->pages);
+ return p;
+ }
+@@ -104,9 +94,15 @@ EXPORT_SYMBOL_GPL(nfs_commit_free);
+
+ static struct nfs_pgio_header *nfs_writehdr_alloc(void)
+ {
+- struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_KERNEL);
++ struct nfs_pgio_header *p;
+
+- memset(p, 0, sizeof(*p));
++ p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask());
++ if (!p) {
++ p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT);
++ if (!p)
++ return NULL;
++ memset(p, 0, sizeof(*p));
++ }
+ p->rw_mode = FMODE_WRITE;
+ return p;
+ }
+@@ -314,7 +310,10 @@ static void nfs_mapping_set_error(struct page *page, int error)
+ struct address_space *mapping = page_file_mapping(page);
+
+ SetPageError(page);
+- mapping_set_error(mapping, error);
++ filemap_set_wb_err(mapping, error);
++ if (mapping->host)
++ errseq_set(&mapping->host->i_sb->s_wb_err,
++ error == -ENOSPC ? -ENOSPC : -EIO);
+ nfs_set_pageerror(mapping);
+ }
+
+@@ -1038,25 +1037,11 @@ nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
+ struct nfs_page *req, *tmp;
+ int ret = 0;
+
+-restart:
+ list_for_each_entry_safe(req, tmp, src, wb_list) {
+ kref_get(&req->wb_kref);
+ if (!nfs_lock_request(req)) {
+- int status;
+-
+- /* Prevent deadlock with nfs_lock_and_join_requests */
+- if (!list_empty(dst)) {
+- nfs_release_request(req);
+- continue;
+- }
+- /* Ensure we make progress to prevent livelock */
+- mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
+- status = nfs_wait_on_request(req);
+ nfs_release_request(req);
+- mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
+- if (status < 0)
+- break;
+- goto restart;
++ continue;
+ }
+ nfs_request_remove_commit_list(req, cinfo);
+ clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
+@@ -1671,10 +1656,13 @@ static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
+ atomic_inc(&cinfo->rpcs_out);
+ }
+
+-static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
++bool nfs_commit_end(struct nfs_mds_commit_info *cinfo)
+ {
+- if (atomic_dec_and_test(&cinfo->rpcs_out))
++ if (atomic_dec_and_test(&cinfo->rpcs_out)) {
+ wake_up_var(&cinfo->rpcs_out);
++ return true;
++ }
++ return false;
+ }
+
+ void nfs_commitdata_release(struct nfs_commit_data *data)
+@@ -1774,6 +1762,7 @@ void nfs_init_commit(struct nfs_commit_data *data,
+ data->res.fattr = &data->fattr;
+ data->res.verf = &data->verf;
+ nfs_fattr_init(&data->fattr);
++ nfs_commit_begin(cinfo->mds);
+ }
+ EXPORT_SYMBOL_GPL(nfs_init_commit);
+
+@@ -1816,11 +1805,14 @@ nfs_commit_list(struct inode *inode, struct list_head *head, int how,
+ if (list_empty(head))
+ return 0;
+
+- data = nfs_commitdata_alloc(true);
++ data = nfs_commitdata_alloc();
++ if (!data) {
++ nfs_retry_commit(head, NULL, cinfo, -1);
++ return -ENOMEM;
++ }
+
+ /* Set up the argument struct */
+ nfs_init_commit(data, head, NULL, cinfo);
+- atomic_inc(&cinfo->mds->rpcs_out);
+ if (NFS_SERVER(inode)->nfs_client->cl_minorversion)
+ task_flags = RPC_TASK_MOVEABLE;
+ return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
+@@ -1936,6 +1928,7 @@ static int __nfs_commit_inode(struct inode *inode, int how,
+ int may_wait = how & FLUSH_SYNC;
+ int ret, nscan;
+
++ how &= ~FLUSH_SYNC;
+ nfs_init_cinfo_from_inode(&cinfo, inode);
+ nfs_commit_begin(cinfo.mds);
+ for (;;) {
+diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
+index be3c1aad50ea3..7e23c588f4842 100644
+--- a/fs/nfsd/filecache.c
++++ b/fs/nfsd/filecache.c
+@@ -641,7 +641,7 @@ nfsd_file_cache_init(void)
+ if (!nfsd_filecache_wq)
+ goto out;
+
+- nfsd_file_hashtbl = kcalloc(NFSD_FILE_HASH_SIZE,
++ nfsd_file_hashtbl = kvcalloc(NFSD_FILE_HASH_SIZE,
+ sizeof(*nfsd_file_hashtbl), GFP_KERNEL);
+ if (!nfsd_file_hashtbl) {
+ pr_err("nfsd: unable to allocate nfsd_file_hashtbl\n");
+@@ -709,7 +709,7 @@ out_err:
+ nfsd_file_slab = NULL;
+ kmem_cache_destroy(nfsd_file_mark_slab);
+ nfsd_file_mark_slab = NULL;
+- kfree(nfsd_file_hashtbl);
++ kvfree(nfsd_file_hashtbl);
+ nfsd_file_hashtbl = NULL;
+ destroy_workqueue(nfsd_filecache_wq);
+ nfsd_filecache_wq = NULL;
+@@ -855,7 +855,7 @@ nfsd_file_cache_shutdown(void)
+ fsnotify_wait_marks_destroyed();
+ kmem_cache_destroy(nfsd_file_mark_slab);
+ nfsd_file_mark_slab = NULL;
+- kfree(nfsd_file_hashtbl);
++ kvfree(nfsd_file_hashtbl);
+ nfsd_file_hashtbl = NULL;
+ destroy_workqueue(nfsd_filecache_wq);
+ nfsd_filecache_wq = NULL;
+diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
+index 17715a6c7a409..b540489ea240d 100644
+--- a/fs/nfsd/nfs3proc.c
++++ b/fs/nfsd/nfs3proc.c
+@@ -150,13 +150,17 @@ nfsd3_proc_read(struct svc_rqst *rqstp)
+ unsigned int len;
+ int v;
+
+- argp->count = min_t(u32, argp->count, max_blocksize);
+-
+ dprintk("nfsd: READ(3) %s %lu bytes at %Lu\n",
+ SVCFH_fmt(&argp->fh),
+ (unsigned long) argp->count,
+ (unsigned long long) argp->offset);
+
++ argp->count = min_t(u32, argp->count, max_blocksize);
++ if (argp->offset > (u64)OFFSET_MAX)
++ argp->offset = (u64)OFFSET_MAX;
++ if (argp->offset + argp->count > (u64)OFFSET_MAX)
++ argp->count = (u64)OFFSET_MAX - argp->offset;
++
+ v = 0;
+ len = argp->count;
+ resp->pages = rqstp->rq_next_page;
+@@ -199,19 +203,19 @@ nfsd3_proc_write(struct svc_rqst *rqstp)
+ (unsigned long long) argp->offset,
+ argp->stable? " stable" : "");
+
++ resp->status = nfserr_fbig;
++ if (argp->offset > (u64)OFFSET_MAX ||
++ argp->offset + argp->len > (u64)OFFSET_MAX)
++ return rpc_success;
++
+ fh_copy(&resp->fh, &argp->fh);
+ resp->committed = argp->stable;
+- nvecs = svc_fill_write_vector(rqstp, rqstp->rq_arg.pages,
+- &argp->first, cnt);
+- if (!nvecs) {
+- resp->status = nfserr_io;
+- goto out;
+- }
++ nvecs = svc_fill_write_vector(rqstp, &argp->payload);
++
+ resp->status = nfsd_write(rqstp, &resp->fh, argp->offset,
+ rqstp->rq_vec, nvecs, &cnt,
+ resp->committed, resp->verf);
+ resp->count = cnt;
+-out:
+ return rpc_success;
+ }
+
+@@ -439,22 +443,19 @@ nfsd3_proc_link(struct svc_rqst *rqstp)
+
+ static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp,
+ struct nfsd3_readdirres *resp,
+- int count)
++ u32 count)
+ {
+ struct xdr_buf *buf = &resp->dirlist;
+ struct xdr_stream *xdr = &resp->xdr;
+
+- count = min_t(u32, count, svc_max_payload(rqstp));
++ count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp));
+
+ memset(buf, 0, sizeof(*buf));
+
+ /* Reserve room for the NULL ptr & eof flag (-2 words) */
+ buf->buflen = count - XDR_UNIT * 2;
+ buf->pages = rqstp->rq_next_page;
+- while (count > 0) {
+- rqstp->rq_next_page++;
+- count -= PAGE_SIZE;
+- }
++ rqstp->rq_next_page += (buf->buflen + PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+ /* This is xdr_init_encode(), but it assumes that
+ * the head kvec has already been consumed. */
+@@ -463,7 +464,7 @@ static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp,
+ xdr->page_ptr = buf->pages;
+ xdr->iov = NULL;
+ xdr->p = page_address(*buf->pages);
+- xdr->end = xdr->p + (PAGE_SIZE >> 2);
++ xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
+ xdr->rqst = NULL;
+ }
+
+diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
+index 0a5ebc52e6a9c..48d4f99b7f901 100644
+--- a/fs/nfsd/nfs3xdr.c
++++ b/fs/nfsd/nfs3xdr.c
+@@ -254,7 +254,7 @@ svcxdr_decode_sattr3(struct svc_rqst *rqstp, struct xdr_stream *xdr,
+ if (xdr_stream_decode_u64(xdr, &newsize) < 0)
+ return false;
+ iap->ia_valid |= ATTR_SIZE;
+- iap->ia_size = min_t(u64, newsize, NFS_OFFSET_MAX);
++ iap->ia_size = newsize;
+ }
+ if (xdr_stream_decode_u32(xdr, &set_it) < 0)
+ return false;
+@@ -487,11 +487,6 @@ neither:
+ return true;
+ }
+
+-static bool fs_supports_change_attribute(struct super_block *sb)
+-{
+- return sb->s_flags & SB_I_VERSION || sb->s_export_op->fetch_iversion;
+-}
+-
+ /*
+ * Fill in the pre_op attr for the wcc data
+ */
+@@ -500,26 +495,24 @@ void fill_pre_wcc(struct svc_fh *fhp)
+ struct inode *inode;
+ struct kstat stat;
+ bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE);
++ __be32 err;
+
+ if (fhp->fh_no_wcc || fhp->fh_pre_saved)
+ return;
+ inode = d_inode(fhp->fh_dentry);
+- if (fs_supports_change_attribute(inode->i_sb) || !v4) {
+- __be32 err = fh_getattr(fhp, &stat);
+-
+- if (err) {
+- /* Grab the times from inode anyway */
+- stat.mtime = inode->i_mtime;
+- stat.ctime = inode->i_ctime;
+- stat.size = inode->i_size;
+- }
+- fhp->fh_pre_mtime = stat.mtime;
+- fhp->fh_pre_ctime = stat.ctime;
+- fhp->fh_pre_size = stat.size;
++ err = fh_getattr(fhp, &stat);
++ if (err) {
++ /* Grab the times from inode anyway */
++ stat.mtime = inode->i_mtime;
++ stat.ctime = inode->i_ctime;
++ stat.size = inode->i_size;
+ }
+ if (v4)
+ fhp->fh_pre_change = nfsd4_change_attribute(&stat, inode);
+
++ fhp->fh_pre_mtime = stat.mtime;
++ fhp->fh_pre_ctime = stat.ctime;
++ fhp->fh_pre_size = stat.size;
+ fhp->fh_pre_saved = true;
+ }
+
+@@ -530,6 +523,7 @@ void fill_post_wcc(struct svc_fh *fhp)
+ {
+ bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE);
+ struct inode *inode = d_inode(fhp->fh_dentry);
++ __be32 err;
+
+ if (fhp->fh_no_wcc)
+ return;
+@@ -537,16 +531,12 @@ void fill_post_wcc(struct svc_fh *fhp)
+ if (fhp->fh_post_saved)
+ printk("nfsd: inode locked twice during operation.\n");
+
+- fhp->fh_post_saved = true;
+-
+- if (fs_supports_change_attribute(inode->i_sb) || !v4) {
+- __be32 err = fh_getattr(fhp, &fhp->fh_post_attr);
+-
+- if (err) {
+- fhp->fh_post_saved = false;
+- fhp->fh_post_attr.ctime = inode->i_ctime;
+- }
+- }
++ err = fh_getattr(fhp, &fhp->fh_post_attr);
++ if (err) {
++ fhp->fh_post_saved = false;
++ fhp->fh_post_attr.ctime = inode->i_ctime;
++ } else
++ fhp->fh_post_saved = true;
+ if (v4)
+ fhp->fh_post_change =
+ nfsd4_change_attribute(&fhp->fh_post_attr, inode);
+@@ -621,9 +611,6 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p)
+ struct xdr_stream *xdr = &rqstp->rq_arg_stream;
+ struct nfsd3_writeargs *args = rqstp->rq_argp;
+ u32 max_blocksize = svc_max_payload(rqstp);
+- struct kvec *head = rqstp->rq_arg.head;
+- struct kvec *tail = rqstp->rq_arg.tail;
+- size_t remaining;
+
+ if (!svcxdr_decode_nfs_fh3(xdr, &args->fh))
+ return 0;
+@@ -641,17 +628,12 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p)
+ /* request sanity */
+ if (args->count != args->len)
+ return 0;
+- remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len;
+- remaining -= xdr_stream_pos(xdr);
+- if (remaining < xdr_align_size(args->len))
+- return 0;
+ if (args->count > max_blocksize) {
+ args->count = max_blocksize;
+ args->len = max_blocksize;
+ }
+-
+- args->first.iov_base = xdr->p;
+- args->first.iov_len = head->iov_len - xdr_stream_pos(xdr);
++ if (!xdr_stream_subsegment(xdr, &args->payload, args->count))
++ return 0;
+
+ return 1;
+ }
+diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
+index 486c5dba4b650..65200910107f3 100644
+--- a/fs/nfsd/nfs4proc.c
++++ b/fs/nfsd/nfs4proc.c
+@@ -782,12 +782,16 @@ nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+ __be32 status;
+
+ read->rd_nf = NULL;
+- if (read->rd_offset >= OFFSET_MAX)
+- return nfserr_inval;
+
+ trace_nfsd_read_start(rqstp, &cstate->current_fh,
+ read->rd_offset, read->rd_length);
+
++ read->rd_length = min_t(u32, read->rd_length, svc_max_payload(rqstp));
++ if (read->rd_offset > (u64)OFFSET_MAX)
++ read->rd_offset = (u64)OFFSET_MAX;
++ if (read->rd_offset + read->rd_length > (u64)OFFSET_MAX)
++ read->rd_length = (u64)OFFSET_MAX - read->rd_offset;
++
+ /*
+ * If we do a zero copy read, then a client will see read data
+ * that reflects the state of the file *after* performing the
+@@ -1018,8 +1022,9 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+ unsigned long cnt;
+ int nvecs;
+
+- if (write->wr_offset >= OFFSET_MAX)
+- return nfserr_inval;
++ if (write->wr_offset > (u64)OFFSET_MAX ||
++ write->wr_offset + write->wr_buflen > (u64)OFFSET_MAX)
++ return nfserr_fbig;
+
+ cnt = write->wr_buflen;
+ trace_nfsd_write_start(rqstp, &cstate->current_fh,
+@@ -1033,8 +1038,7 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+
+ write->wr_how_written = write->wr_stable_how;
+
+- nvecs = svc_fill_write_vector(rqstp, write->wr_payload.pages,
+- write->wr_payload.head, write->wr_buflen);
++ nvecs = svc_fill_write_vector(rqstp, &write->wr_payload);
+ WARN_ON_ONCE(nvecs > ARRAY_SIZE(rqstp->rq_vec));
+
+ status = nfsd_vfs_write(rqstp, &cstate->current_fh, nf,
+diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
+index 6fedc49726bf7..c634483d85d2a 100644
+--- a/fs/nfsd/nfs4recover.c
++++ b/fs/nfsd/nfs4recover.c
+@@ -2156,6 +2156,7 @@ static struct notifier_block nfsd4_cld_block = {
+ int
+ register_cld_notifier(void)
+ {
++ WARN_ON(!nfsd_net_id);
+ return rpc_pipefs_notifier_register(&nfsd4_cld_block);
+ }
+
+diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
+index 3f4027a5de883..181bc3d9f5663 100644
+--- a/fs/nfsd/nfs4state.c
++++ b/fs/nfsd/nfs4state.c
+@@ -1207,6 +1207,11 @@ hash_delegation_locked(struct nfs4_delegation *dp, struct nfs4_file *fp)
+ return 0;
+ }
+
++static bool delegation_hashed(struct nfs4_delegation *dp)
++{
++ return !(list_empty(&dp->dl_perfile));
++}
++
+ static bool
+ unhash_delegation_locked(struct nfs4_delegation *dp)
+ {
+@@ -1214,7 +1219,7 @@ unhash_delegation_locked(struct nfs4_delegation *dp)
+
+ lockdep_assert_held(&state_lock);
+
+- if (list_empty(&dp->dl_perfile))
++ if (!delegation_hashed(dp))
+ return false;
+
+ dp->dl_stid.sc_type = NFS4_CLOSED_DELEG_STID;
+@@ -4107,8 +4112,10 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
+ status = nfserr_clid_inuse;
+ if (client_has_state(old)
+ && !same_creds(&unconf->cl_cred,
+- &old->cl_cred))
++ &old->cl_cred)) {
++ old = NULL;
+ goto out;
++ }
+ status = mark_client_expired_locked(old);
+ if (status) {
+ old = NULL;
+@@ -4598,7 +4605,7 @@ static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
+ * queued for a lease break. Don't queue it again.
+ */
+ spin_lock(&state_lock);
+- if (dp->dl_time == 0) {
++ if (delegation_hashed(dp) && dp->dl_time == 0) {
+ dp->dl_time = ktime_get_boottime_seconds();
+ list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru);
+ }
+@@ -4686,6 +4693,14 @@ nfsd_break_deleg_cb(struct file_lock *fl)
+ return ret;
+ }
+
++/**
++ * nfsd_breaker_owns_lease - Check if lease conflict was resolved
++ * @fl: Lock state to check
++ *
++ * Return values:
++ * %true: Lease conflict was resolved
++ * %false: Lease conflict was not resolved.
++ */
+ static bool nfsd_breaker_owns_lease(struct file_lock *fl)
+ {
+ struct nfs4_delegation *dl = fl->fl_owner;
+@@ -4693,11 +4708,11 @@ static bool nfsd_breaker_owns_lease(struct file_lock *fl)
+ struct nfs4_client *clp;
+
+ if (!i_am_nfsd())
+- return NULL;
++ return false;
+ rqst = kthread_data(current);
+ /* Note rq_prog == NFS_ACL_PROGRAM is also possible: */
+ if (rqst->rq_prog != NFS_PROGRAM || rqst->rq_vers < 4)
+- return NULL;
++ return false;
+ clp = *(rqst->rq_lease_breaker);
+ return dl->dl_stid.sc_client == clp;
+ }
+@@ -6035,7 +6050,11 @@ nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
+ *nfp = NULL;
+
+ if (ZERO_STATEID(stateid) || ONE_STATEID(stateid)) {
+- status = check_special_stateids(net, fhp, stateid, flags);
++ if (cstid)
++ status = nfserr_bad_stateid;
++ else
++ status = check_special_stateids(net, fhp, stateid,
++ flags);
+ goto done;
+ }
+
+diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
+index cf030ebe28275..f6d385e0efece 100644
+--- a/fs/nfsd/nfs4xdr.c
++++ b/fs/nfsd/nfs4xdr.c
+@@ -288,11 +288,8 @@ nfsd4_decode_bitmap4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen)
+ p = xdr_inline_decode(argp->xdr, count << 2);
+ if (!p)
+ return nfserr_bad_xdr;
+- i = 0;
+- while (i < count)
+- bmval[i++] = be32_to_cpup(p++);
+- while (i < bmlen)
+- bmval[i++] = 0;
++ for (i = 0; i < bmlen; i++)
++ bmval[i] = (i < count) ? be32_to_cpup(p++) : 0;
+
+ return nfs_ok;
+ }
+@@ -4000,10 +3997,8 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr,
+ }
+ xdr_commit_encode(xdr);
+
+- maxcount = svc_max_payload(resp->rqstp);
+- maxcount = min_t(unsigned long, maxcount,
++ maxcount = min_t(unsigned long, read->rd_length,
+ (xdr->buf->buflen - xdr->buf->len));
+- maxcount = min_t(unsigned long, maxcount, read->rd_length);
+
+ if (file->f_op->splice_read &&
+ test_bit(RQ_SPLICE_OK, &resp->rqstp->rq_flags))
+@@ -4840,10 +4835,8 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr,
+ return nfserr_resource;
+ xdr_commit_encode(xdr);
+
+- maxcount = svc_max_payload(resp->rqstp);
+- maxcount = min_t(unsigned long, maxcount,
++ maxcount = min_t(unsigned long, read->rd_length,
+ (xdr->buf->buflen - xdr->buf->len));
+- maxcount = min_t(unsigned long, maxcount, read->rd_length);
+ count = maxcount;
+
+ eof = read->rd_offset >= i_size_read(file_inode(file));
+diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
+index 070e5dd03e26f..cb73c12925629 100644
+--- a/fs/nfsd/nfsctl.c
++++ b/fs/nfsd/nfsctl.c
+@@ -1249,7 +1249,8 @@ static void nfsdfs_remove_file(struct inode *dir, struct dentry *dentry)
+ clear_ncl(d_inode(dentry));
+ dget(dentry);
+ ret = simple_unlink(dir, dentry);
+- d_delete(dentry);
++ d_drop(dentry);
++ fsnotify_unlink(dir, dentry);
+ dput(dentry);
+ WARN_ON_ONCE(ret);
+ }
+@@ -1340,8 +1341,8 @@ void nfsd_client_rmdir(struct dentry *dentry)
+ dget(dentry);
+ ret = simple_rmdir(dir, dentry);
+ WARN_ON_ONCE(ret);
++ d_drop(dentry);
+ fsnotify_rmdir(dir, dentry);
+- d_delete(dentry);
+ dput(dentry);
+ inode_unlock(dir);
+ }
+@@ -1521,12 +1522,9 @@ static int __init init_nfsd(void)
+ int retval;
+ printk(KERN_INFO "Installing knfsd (copyright (C) 1996 okir@monad.swb.de).\n");
+
+- retval = register_cld_notifier();
+- if (retval)
+- return retval;
+ retval = nfsd4_init_slabs();
+ if (retval)
+- goto out_unregister_notifier;
++ return retval;
+ retval = nfsd4_init_pnfs();
+ if (retval)
+ goto out_free_slabs;
+@@ -1545,9 +1543,14 @@ static int __init init_nfsd(void)
+ goto out_free_exports;
+ retval = register_pernet_subsys(&nfsd_net_ops);
+ if (retval < 0)
++ goto out_free_filesystem;
++ retval = register_cld_notifier();
++ if (retval)
+ goto out_free_all;
+ return 0;
+ out_free_all:
++ unregister_pernet_subsys(&nfsd_net_ops);
++out_free_filesystem:
+ unregister_filesystem(&nfsd_fs_type);
+ out_free_exports:
+ remove_proc_entry("fs/nfs/exports", NULL);
+@@ -1561,13 +1564,12 @@ out_free_pnfs:
+ nfsd4_exit_pnfs();
+ out_free_slabs:
+ nfsd4_free_slabs();
+-out_unregister_notifier:
+- unregister_cld_notifier();
+ return retval;
+ }
+
+ static void __exit exit_nfsd(void)
+ {
++ unregister_cld_notifier();
+ unregister_pernet_subsys(&nfsd_net_ops);
+ nfsd_drc_slab_free();
+ remove_proc_entry("fs/nfs/exports", NULL);
+@@ -1577,7 +1579,6 @@ static void __exit exit_nfsd(void)
+ nfsd4_free_slabs();
+ nfsd4_exit_pnfs();
+ unregister_filesystem(&nfsd_fs_type);
+- unregister_cld_notifier();
+ }
+
+ MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>");
+diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
+index 90fcd6178823b..9700ad433b498 100644
+--- a/fs/nfsd/nfsproc.c
++++ b/fs/nfsd/nfsproc.c
+@@ -230,16 +230,11 @@ nfsd_proc_write(struct svc_rqst *rqstp)
+ unsigned long cnt = argp->len;
+ unsigned int nvecs;
+
+- dprintk("nfsd: WRITE %s %d bytes at %d\n",
++ dprintk("nfsd: WRITE %s %u bytes at %d\n",
+ SVCFH_fmt(&argp->fh),
+ argp->len, argp->offset);
+
+- nvecs = svc_fill_write_vector(rqstp, rqstp->rq_arg.pages,
+- &argp->first, cnt);
+- if (!nvecs) {
+- resp->status = nfserr_io;
+- goto out;
+- }
++ nvecs = svc_fill_write_vector(rqstp, &argp->payload);
+
+ resp->status = nfsd_write(rqstp, fh_copy(&resp->fh, &argp->fh),
+ argp->offset, rqstp->rq_vec, nvecs,
+@@ -248,7 +243,6 @@ nfsd_proc_write(struct svc_rqst *rqstp)
+ resp->status = fh_getattr(&resp->fh, &resp->stat);
+ else if (resp->status == nfserr_jukebox)
+ return rpc_drop_reply;
+-out:
+ return rpc_success;
+ }
+
+@@ -557,17 +551,17 @@ nfsd_proc_rmdir(struct svc_rqst *rqstp)
+
+ static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp,
+ struct nfsd_readdirres *resp,
+- int count)
++ u32 count)
+ {
+ struct xdr_buf *buf = &resp->dirlist;
+ struct xdr_stream *xdr = &resp->xdr;
+
+- count = min_t(u32, count, PAGE_SIZE);
++ count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp));
+
+ memset(buf, 0, sizeof(*buf));
+
+ /* Reserve room for the NULL ptr & eof flag (-2 words) */
+- buf->buflen = count - sizeof(__be32) * 2;
++ buf->buflen = count - XDR_UNIT * 2;
+ buf->pages = rqstp->rq_next_page;
+ rqstp->rq_next_page++;
+
+@@ -578,7 +572,7 @@ static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp,
+ xdr->page_ptr = buf->pages;
+ xdr->iov = NULL;
+ xdr->p = page_address(*buf->pages);
+- xdr->end = xdr->p + (PAGE_SIZE >> 2);
++ xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
+ xdr->rqst = NULL;
+ }
+
+diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c
+index a06c05fe3b421..26a42f87c2409 100644
+--- a/fs/nfsd/nfsxdr.c
++++ b/fs/nfsd/nfsxdr.c
+@@ -325,10 +325,7 @@ nfssvc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p)
+ {
+ struct xdr_stream *xdr = &rqstp->rq_arg_stream;
+ struct nfsd_writeargs *args = rqstp->rq_argp;
+- struct kvec *head = rqstp->rq_arg.head;
+- struct kvec *tail = rqstp->rq_arg.tail;
+ u32 beginoffset, totalcount;
+- size_t remaining;
+
+ if (!svcxdr_decode_fhandle(xdr, &args->fh))
+ return 0;
+@@ -346,12 +343,8 @@ nfssvc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p)
+ return 0;
+ if (args->len > NFSSVC_MAXBLKSIZE_V2)
+ return 0;
+- remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len;
+- remaining -= xdr_stream_pos(xdr);
+- if (remaining < xdr_align_size(args->len))
++ if (!xdr_stream_subsegment(xdr, &args->payload, args->len))
+ return 0;
+- args->first.iov_base = xdr->p;
+- args->first.iov_len = head->iov_len - xdr_stream_pos(xdr);
+
+ return 1;
+ }
+diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
+index 538520957a815..b302836c7fdf9 100644
+--- a/fs/nfsd/trace.h
++++ b/fs/nfsd/trace.h
+@@ -319,14 +319,14 @@ TRACE_EVENT(nfsd_export_update,
+ DECLARE_EVENT_CLASS(nfsd_io_class,
+ TP_PROTO(struct svc_rqst *rqstp,
+ struct svc_fh *fhp,
+- loff_t offset,
+- unsigned long len),
++ u64 offset,
++ u32 len),
+ TP_ARGS(rqstp, fhp, offset, len),
+ TP_STRUCT__entry(
+ __field(u32, xid)
+ __field(u32, fh_hash)
+- __field(loff_t, offset)
+- __field(unsigned long, len)
++ __field(u64, offset)
++ __field(u32, len)
+ ),
+ TP_fast_assign(
+ __entry->xid = be32_to_cpu(rqstp->rq_xid);
+@@ -334,7 +334,7 @@ DECLARE_EVENT_CLASS(nfsd_io_class,
+ __entry->offset = offset;
+ __entry->len = len;
+ ),
+- TP_printk("xid=0x%08x fh_hash=0x%08x offset=%lld len=%lu",
++ TP_printk("xid=0x%08x fh_hash=0x%08x offset=%llu len=%u",
+ __entry->xid, __entry->fh_hash,
+ __entry->offset, __entry->len)
+ )
+@@ -343,8 +343,8 @@ DECLARE_EVENT_CLASS(nfsd_io_class,
+ DEFINE_EVENT(nfsd_io_class, nfsd_##name, \
+ TP_PROTO(struct svc_rqst *rqstp, \
+ struct svc_fh *fhp, \
+- loff_t offset, \
+- unsigned long len), \
++ u64 offset, \
++ u32 len), \
+ TP_ARGS(rqstp, fhp, offset, len))
+
+ DEFINE_NFSD_IO_EVENT(read_start);
+diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
+index 738d564ca4ce3..271f7c15d6e52 100644
+--- a/fs/nfsd/vfs.c
++++ b/fs/nfsd/vfs.c
+@@ -433,6 +433,10 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, struct iattr *iap,
+ .ia_size = iap->ia_size,
+ };
+
++ host_err = -EFBIG;
++ if (iap->ia_size < 0)
++ goto out_unlock;
++
+ host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL);
+ if (host_err)
+ goto out_unlock;
+@@ -989,6 +993,10 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf,
+ iov_iter_kvec(&iter, WRITE, vec, vlen, *cnt);
+ if (flags & RWF_SYNC) {
+ down_write(&nf->nf_rwsem);
++ if (verf)
++ nfsd_copy_boot_verifier(verf,
++ net_generic(SVC_NET(rqstp),
++ nfsd_net_id));
+ host_err = vfs_iter_write(file, &iter, &pos, flags);
+ if (host_err < 0)
+ nfsd_reset_boot_verifier(net_generic(SVC_NET(rqstp),
+diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h
+index f45b4bc93f527..863a35f24910a 100644
+--- a/fs/nfsd/xdr.h
++++ b/fs/nfsd/xdr.h
+@@ -32,8 +32,8 @@ struct nfsd_readargs {
+ struct nfsd_writeargs {
+ svc_fh fh;
+ __u32 offset;
+- int len;
+- struct kvec first;
++ __u32 len;
++ struct xdr_buf payload;
+ };
+
+ struct nfsd_createargs {
+diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h
+index 933008382bbeb..712c117300cb7 100644
+--- a/fs/nfsd/xdr3.h
++++ b/fs/nfsd/xdr3.h
+@@ -40,7 +40,7 @@ struct nfsd3_writeargs {
+ __u32 count;
+ int stable;
+ __u32 len;
+- struct kvec first;
++ struct xdr_buf payload;
+ };
+
+ struct nfsd3_createargs {
+diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c
+index 4391fd3abd8f8..e00e184b12615 100644
+--- a/fs/nilfs2/btnode.c
++++ b/fs/nilfs2/btnode.c
+@@ -20,6 +20,23 @@
+ #include "page.h"
+ #include "btnode.h"
+
++
++/**
++ * nilfs_init_btnc_inode - initialize B-tree node cache inode
++ * @btnc_inode: inode to be initialized
++ *
++ * nilfs_init_btnc_inode() sets up an inode for B-tree node cache.
++ */
++void nilfs_init_btnc_inode(struct inode *btnc_inode)
++{
++ struct nilfs_inode_info *ii = NILFS_I(btnc_inode);
++
++ btnc_inode->i_mode = S_IFREG;
++ ii->i_flags = 0;
++ memset(&ii->i_bmap_data, 0, sizeof(struct nilfs_bmap));
++ mapping_set_gfp_mask(btnc_inode->i_mapping, GFP_NOFS);
++}
++
+ void nilfs_btnode_cache_clear(struct address_space *btnc)
+ {
+ invalidate_mapping_pages(btnc, 0, -1);
+@@ -29,7 +46,7 @@ void nilfs_btnode_cache_clear(struct address_space *btnc)
+ struct buffer_head *
+ nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr)
+ {
+- struct inode *inode = NILFS_BTNC_I(btnc);
++ struct inode *inode = btnc->host;
+ struct buffer_head *bh;
+
+ bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node));
+@@ -57,7 +74,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
+ struct buffer_head **pbh, sector_t *submit_ptr)
+ {
+ struct buffer_head *bh;
+- struct inode *inode = NILFS_BTNC_I(btnc);
++ struct inode *inode = btnc->host;
+ struct page *page;
+ int err;
+
+@@ -157,7 +174,7 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc,
+ struct nilfs_btnode_chkey_ctxt *ctxt)
+ {
+ struct buffer_head *obh, *nbh;
+- struct inode *inode = NILFS_BTNC_I(btnc);
++ struct inode *inode = btnc->host;
+ __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey;
+ int err;
+
+diff --git a/fs/nilfs2/btnode.h b/fs/nilfs2/btnode.h
+index 0f88dbc9bcb3e..05ab64d354dc9 100644
+--- a/fs/nilfs2/btnode.h
++++ b/fs/nilfs2/btnode.h
+@@ -30,6 +30,7 @@ struct nilfs_btnode_chkey_ctxt {
+ struct buffer_head *newbh;
+ };
+
++void nilfs_init_btnc_inode(struct inode *btnc_inode);
+ void nilfs_btnode_cache_clear(struct address_space *);
+ struct buffer_head *nilfs_btnode_create_block(struct address_space *btnc,
+ __u64 blocknr);
+diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c
+index ab9ec073330f1..2301b57ca17ff 100644
+--- a/fs/nilfs2/btree.c
++++ b/fs/nilfs2/btree.c
+@@ -58,7 +58,8 @@ static void nilfs_btree_free_path(struct nilfs_btree_path *path)
+ static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree,
+ __u64 ptr, struct buffer_head **bhp)
+ {
+- struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache;
++ struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode;
++ struct address_space *btnc = btnc_inode->i_mapping;
+ struct buffer_head *bh;
+
+ bh = nilfs_btnode_create_block(btnc, ptr);
+@@ -470,7 +471,8 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr,
+ struct buffer_head **bhp,
+ const struct nilfs_btree_readahead_info *ra)
+ {
+- struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache;
++ struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode;
++ struct address_space *btnc = btnc_inode->i_mapping;
+ struct buffer_head *bh, *ra_bh;
+ sector_t submit_ptr = 0;
+ int ret;
+@@ -1741,6 +1743,10 @@ nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key,
+ dat = nilfs_bmap_get_dat(btree);
+ }
+
++ ret = nilfs_attach_btree_node_cache(&NILFS_BMAP_I(btree)->vfs_inode);
++ if (ret < 0)
++ return ret;
++
+ ret = nilfs_bmap_prepare_alloc_ptr(btree, dreq, dat);
+ if (ret < 0)
+ return ret;
+@@ -1913,7 +1919,7 @@ static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree,
+ path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr;
+ path[level].bp_ctxt.bh = path[level].bp_bh;
+ ret = nilfs_btnode_prepare_change_key(
+- &NILFS_BMAP_I(btree)->i_btnode_cache,
++ NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping,
+ &path[level].bp_ctxt);
+ if (ret < 0) {
+ nilfs_dat_abort_update(dat,
+@@ -1939,7 +1945,7 @@ static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree,
+
+ if (buffer_nilfs_node(path[level].bp_bh)) {
+ nilfs_btnode_commit_change_key(
+- &NILFS_BMAP_I(btree)->i_btnode_cache,
++ NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping,
+ &path[level].bp_ctxt);
+ path[level].bp_bh = path[level].bp_ctxt.bh;
+ }
+@@ -1958,7 +1964,7 @@ static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree,
+ &path[level].bp_newreq.bpr_req);
+ if (buffer_nilfs_node(path[level].bp_bh))
+ nilfs_btnode_abort_change_key(
+- &NILFS_BMAP_I(btree)->i_btnode_cache,
++ NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping,
+ &path[level].bp_ctxt);
+ }
+
+@@ -2134,7 +2140,8 @@ static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree,
+ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree,
+ struct list_head *listp)
+ {
+- struct address_space *btcache = &NILFS_BMAP_I(btree)->i_btnode_cache;
++ struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode;
++ struct address_space *btcache = btnc_inode->i_mapping;
+ struct list_head lists[NILFS_BTREE_LEVEL_MAX];
+ struct pagevec pvec;
+ struct buffer_head *bh, *head;
+@@ -2188,12 +2195,12 @@ static int nilfs_btree_assign_p(struct nilfs_bmap *btree,
+ path[level].bp_ctxt.newkey = blocknr;
+ path[level].bp_ctxt.bh = *bh;
+ ret = nilfs_btnode_prepare_change_key(
+- &NILFS_BMAP_I(btree)->i_btnode_cache,
++ NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping,
+ &path[level].bp_ctxt);
+ if (ret < 0)
+ return ret;
+ nilfs_btnode_commit_change_key(
+- &NILFS_BMAP_I(btree)->i_btnode_cache,
++ NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping,
+ &path[level].bp_ctxt);
+ *bh = path[level].bp_ctxt.bh;
+ }
+@@ -2398,6 +2405,10 @@ int nilfs_btree_init(struct nilfs_bmap *bmap)
+
+ if (nilfs_btree_root_broken(nilfs_btree_get_root(bmap), bmap->b_inode))
+ ret = -EIO;
++ else
++ ret = nilfs_attach_btree_node_cache(
++ &NILFS_BMAP_I(bmap)->vfs_inode);
++
+ return ret;
+ }
+
+diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c
+index 8bccdf1158fce..1a3d183027b9e 100644
+--- a/fs/nilfs2/dat.c
++++ b/fs/nilfs2/dat.c
+@@ -497,7 +497,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size,
+ di = NILFS_DAT_I(dat);
+ lockdep_set_class(&di->mi.mi_sem, &dat_lock_key);
+ nilfs_palloc_setup_cache(dat, &di->palloc_cache);
+- nilfs_mdt_setup_shadow_map(dat, &di->shadow);
++ err = nilfs_mdt_setup_shadow_map(dat, &di->shadow);
++ if (err)
++ goto failed;
+
+ err = nilfs_read_inode_common(dat, raw_inode);
+ if (err)
+diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c
+index 4483204968568..aadea660c66c9 100644
+--- a/fs/nilfs2/gcinode.c
++++ b/fs/nilfs2/gcinode.c
+@@ -126,9 +126,10 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff,
+ int nilfs_gccache_submit_read_node(struct inode *inode, sector_t pbn,
+ __u64 vbn, struct buffer_head **out_bh)
+ {
++ struct inode *btnc_inode = NILFS_I(inode)->i_assoc_inode;
+ int ret;
+
+- ret = nilfs_btnode_submit_block(&NILFS_I(inode)->i_btnode_cache,
++ ret = nilfs_btnode_submit_block(btnc_inode->i_mapping,
+ vbn ? : pbn, pbn, REQ_OP_READ, 0,
+ out_bh, &pbn);
+ if (ret == -EEXIST) /* internal code (cache hit) */
+@@ -170,7 +171,7 @@ int nilfs_init_gcinode(struct inode *inode)
+ ii->i_flags = 0;
+ nilfs_bmap_init_gc(ii->i_bmap);
+
+- return 0;
++ return nilfs_attach_btree_node_cache(inode);
+ }
+
+ /**
+@@ -185,7 +186,7 @@ void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs)
+ ii = list_first_entry(head, struct nilfs_inode_info, i_dirty);
+ list_del_init(&ii->i_dirty);
+ truncate_inode_pages(&ii->vfs_inode.i_data, 0);
+- nilfs_btnode_cache_clear(&ii->i_btnode_cache);
++ nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping);
+ iput(&ii->vfs_inode);
+ }
+ }
+diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c
+index 2e8eb263cf0f6..2466f8b8be95c 100644
+--- a/fs/nilfs2/inode.c
++++ b/fs/nilfs2/inode.c
+@@ -29,12 +29,16 @@
+ * @cno: checkpoint number
+ * @root: pointer on NILFS root object (mounted checkpoint)
+ * @for_gc: inode for GC flag
++ * @for_btnc: inode for B-tree node cache flag
++ * @for_shadow: inode for shadowed page cache flag
+ */
+ struct nilfs_iget_args {
+ u64 ino;
+ __u64 cno;
+ struct nilfs_root *root;
+- int for_gc;
++ bool for_gc;
++ bool for_btnc;
++ bool for_shadow;
+ };
+
+ static int nilfs_iget_test(struct inode *inode, void *opaque);
+@@ -314,7 +318,8 @@ static int nilfs_insert_inode_locked(struct inode *inode,
+ unsigned long ino)
+ {
+ struct nilfs_iget_args args = {
+- .ino = ino, .root = root, .cno = 0, .for_gc = 0
++ .ino = ino, .root = root, .cno = 0, .for_gc = false,
++ .for_btnc = false, .for_shadow = false
+ };
+
+ return insert_inode_locked4(inode, ino, nilfs_iget_test, &args);
+@@ -527,6 +532,19 @@ static int nilfs_iget_test(struct inode *inode, void *opaque)
+ return 0;
+
+ ii = NILFS_I(inode);
++ if (test_bit(NILFS_I_BTNC, &ii->i_state)) {
++ if (!args->for_btnc)
++ return 0;
++ } else if (args->for_btnc) {
++ return 0;
++ }
++ if (test_bit(NILFS_I_SHADOW, &ii->i_state)) {
++ if (!args->for_shadow)
++ return 0;
++ } else if (args->for_shadow) {
++ return 0;
++ }
++
+ if (!test_bit(NILFS_I_GCINODE, &ii->i_state))
+ return !args->for_gc;
+
+@@ -538,15 +556,17 @@ static int nilfs_iget_set(struct inode *inode, void *opaque)
+ struct nilfs_iget_args *args = opaque;
+
+ inode->i_ino = args->ino;
+- if (args->for_gc) {
++ NILFS_I(inode)->i_cno = args->cno;
++ NILFS_I(inode)->i_root = args->root;
++ if (args->root && args->ino == NILFS_ROOT_INO)
++ nilfs_get_root(args->root);
++
++ if (args->for_gc)
+ NILFS_I(inode)->i_state = BIT(NILFS_I_GCINODE);
+- NILFS_I(inode)->i_cno = args->cno;
+- NILFS_I(inode)->i_root = NULL;
+- } else {
+- if (args->root && args->ino == NILFS_ROOT_INO)
+- nilfs_get_root(args->root);
+- NILFS_I(inode)->i_root = args->root;
+- }
++ if (args->for_btnc)
++ NILFS_I(inode)->i_state |= BIT(NILFS_I_BTNC);
++ if (args->for_shadow)
++ NILFS_I(inode)->i_state |= BIT(NILFS_I_SHADOW);
+ return 0;
+ }
+
+@@ -554,7 +574,8 @@ struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root,
+ unsigned long ino)
+ {
+ struct nilfs_iget_args args = {
+- .ino = ino, .root = root, .cno = 0, .for_gc = 0
++ .ino = ino, .root = root, .cno = 0, .for_gc = false,
++ .for_btnc = false, .for_shadow = false
+ };
+
+ return ilookup5(sb, ino, nilfs_iget_test, &args);
+@@ -564,7 +585,8 @@ struct inode *nilfs_iget_locked(struct super_block *sb, struct nilfs_root *root,
+ unsigned long ino)
+ {
+ struct nilfs_iget_args args = {
+- .ino = ino, .root = root, .cno = 0, .for_gc = 0
++ .ino = ino, .root = root, .cno = 0, .for_gc = false,
++ .for_btnc = false, .for_shadow = false
+ };
+
+ return iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args);
+@@ -595,7 +617,8 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino,
+ __u64 cno)
+ {
+ struct nilfs_iget_args args = {
+- .ino = ino, .root = NULL, .cno = cno, .for_gc = 1
++ .ino = ino, .root = NULL, .cno = cno, .for_gc = true,
++ .for_btnc = false, .for_shadow = false
+ };
+ struct inode *inode;
+ int err;
+@@ -615,6 +638,113 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino,
+ return inode;
+ }
+
++/**
++ * nilfs_attach_btree_node_cache - attach a B-tree node cache to the inode
++ * @inode: inode object
++ *
++ * nilfs_attach_btree_node_cache() attaches a B-tree node cache to @inode,
++ * or does nothing if the inode already has it. This function allocates
++ * an additional inode to maintain page cache of B-tree nodes one-on-one.
++ *
++ * Return Value: On success, 0 is returned. On errors, one of the following
++ * negative error code is returned.
++ *
++ * %-ENOMEM - Insufficient memory available.
++ */
++int nilfs_attach_btree_node_cache(struct inode *inode)
++{
++ struct nilfs_inode_info *ii = NILFS_I(inode);
++ struct inode *btnc_inode;
++ struct nilfs_iget_args args;
++
++ if (ii->i_assoc_inode)
++ return 0;
++
++ args.ino = inode->i_ino;
++ args.root = ii->i_root;
++ args.cno = ii->i_cno;
++ args.for_gc = test_bit(NILFS_I_GCINODE, &ii->i_state) != 0;
++ args.for_btnc = true;
++ args.for_shadow = test_bit(NILFS_I_SHADOW, &ii->i_state) != 0;
++
++ btnc_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test,
++ nilfs_iget_set, &args);
++ if (unlikely(!btnc_inode))
++ return -ENOMEM;
++ if (btnc_inode->i_state & I_NEW) {
++ nilfs_init_btnc_inode(btnc_inode);
++ unlock_new_inode(btnc_inode);
++ }
++ NILFS_I(btnc_inode)->i_assoc_inode = inode;
++ NILFS_I(btnc_inode)->i_bmap = ii->i_bmap;
++ ii->i_assoc_inode = btnc_inode;
++
++ return 0;
++}
++
++/**
++ * nilfs_detach_btree_node_cache - detach the B-tree node cache from the inode
++ * @inode: inode object
++ *
++ * nilfs_detach_btree_node_cache() detaches the B-tree node cache and its
++ * holder inode bound to @inode, or does nothing if @inode doesn't have it.
++ */
++void nilfs_detach_btree_node_cache(struct inode *inode)
++{
++ struct nilfs_inode_info *ii = NILFS_I(inode);
++ struct inode *btnc_inode = ii->i_assoc_inode;
++
++ if (btnc_inode) {
++ NILFS_I(btnc_inode)->i_assoc_inode = NULL;
++ ii->i_assoc_inode = NULL;
++ iput(btnc_inode);
++ }
++}
++
++/**
++ * nilfs_iget_for_shadow - obtain inode for shadow mapping
++ * @inode: inode object that uses shadow mapping
++ *
++ * nilfs_iget_for_shadow() allocates a pair of inodes that holds page
++ * caches for shadow mapping. The page cache for data pages is set up
++ * in one inode and the one for b-tree node pages is set up in the
++ * other inode, which is attached to the former inode.
++ *
++ * Return Value: On success, a pointer to the inode for data pages is
++ * returned. On errors, one of the following negative error code is returned
++ * in a pointer type.
++ *
++ * %-ENOMEM - Insufficient memory available.
++ */
++struct inode *nilfs_iget_for_shadow(struct inode *inode)
++{
++ struct nilfs_iget_args args = {
++ .ino = inode->i_ino, .root = NULL, .cno = 0, .for_gc = false,
++ .for_btnc = false, .for_shadow = true
++ };
++ struct inode *s_inode;
++ int err;
++
++ s_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test,
++ nilfs_iget_set, &args);
++ if (unlikely(!s_inode))
++ return ERR_PTR(-ENOMEM);
++ if (!(s_inode->i_state & I_NEW))
++ return inode;
++
++ NILFS_I(s_inode)->i_flags = 0;
++ memset(NILFS_I(s_inode)->i_bmap, 0, sizeof(struct nilfs_bmap));
++ mapping_set_gfp_mask(s_inode->i_mapping, GFP_NOFS);
++
++ err = nilfs_attach_btree_node_cache(s_inode);
++ if (unlikely(err)) {
++ iget_failed(s_inode);
++ return ERR_PTR(err);
++ }
++ unlock_new_inode(s_inode);
++ return s_inode;
++}
++
+ void nilfs_write_inode_common(struct inode *inode,
+ struct nilfs_inode *raw_inode, int has_bmap)
+ {
+@@ -762,7 +892,8 @@ static void nilfs_clear_inode(struct inode *inode)
+ if (test_bit(NILFS_I_BMAP, &ii->i_state))
+ nilfs_bmap_clear(ii->i_bmap);
+
+- nilfs_btnode_cache_clear(&ii->i_btnode_cache);
++ if (!test_bit(NILFS_I_BTNC, &ii->i_state))
++ nilfs_detach_btree_node_cache(inode);
+
+ if (ii->i_root && inode->i_ino == NILFS_ROOT_INO)
+ nilfs_put_root(ii->i_root);
+diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c
+index 97769fe4d5885..131b5add32eeb 100644
+--- a/fs/nilfs2/mdt.c
++++ b/fs/nilfs2/mdt.c
+@@ -470,9 +470,18 @@ int nilfs_mdt_init(struct inode *inode, gfp_t gfp_mask, size_t objsz)
+ void nilfs_mdt_clear(struct inode *inode)
+ {
+ struct nilfs_mdt_info *mdi = NILFS_MDT(inode);
++ struct nilfs_shadow_map *shadow = mdi->mi_shadow;
+
+ if (mdi->mi_palloc_cache)
+ nilfs_palloc_destroy_cache(inode);
++
++ if (shadow) {
++ struct inode *s_inode = shadow->inode;
++
++ shadow->inode = NULL;
++ iput(s_inode);
++ mdi->mi_shadow = NULL;
++ }
+ }
+
+ /**
+@@ -506,12 +515,15 @@ int nilfs_mdt_setup_shadow_map(struct inode *inode,
+ struct nilfs_shadow_map *shadow)
+ {
+ struct nilfs_mdt_info *mi = NILFS_MDT(inode);
++ struct inode *s_inode;
+
+ INIT_LIST_HEAD(&shadow->frozen_buffers);
+- address_space_init_once(&shadow->frozen_data);
+- nilfs_mapping_init(&shadow->frozen_data, inode);
+- address_space_init_once(&shadow->frozen_btnodes);
+- nilfs_mapping_init(&shadow->frozen_btnodes, inode);
++
++ s_inode = nilfs_iget_for_shadow(inode);
++ if (IS_ERR(s_inode))
++ return PTR_ERR(s_inode);
++
++ shadow->inode = s_inode;
+ mi->mi_shadow = shadow;
+ return 0;
+ }
+@@ -525,14 +537,15 @@ int nilfs_mdt_save_to_shadow_map(struct inode *inode)
+ struct nilfs_mdt_info *mi = NILFS_MDT(inode);
+ struct nilfs_inode_info *ii = NILFS_I(inode);
+ struct nilfs_shadow_map *shadow = mi->mi_shadow;
++ struct inode *s_inode = shadow->inode;
+ int ret;
+
+- ret = nilfs_copy_dirty_pages(&shadow->frozen_data, inode->i_mapping);
++ ret = nilfs_copy_dirty_pages(s_inode->i_mapping, inode->i_mapping);
+ if (ret)
+ goto out;
+
+- ret = nilfs_copy_dirty_pages(&shadow->frozen_btnodes,
+- &ii->i_btnode_cache);
++ ret = nilfs_copy_dirty_pages(NILFS_I(s_inode)->i_assoc_inode->i_mapping,
++ ii->i_assoc_inode->i_mapping);
+ if (ret)
+ goto out;
+
+@@ -548,7 +561,7 @@ int nilfs_mdt_freeze_buffer(struct inode *inode, struct buffer_head *bh)
+ struct page *page;
+ int blkbits = inode->i_blkbits;
+
+- page = grab_cache_page(&shadow->frozen_data, bh->b_page->index);
++ page = grab_cache_page(shadow->inode->i_mapping, bh->b_page->index);
+ if (!page)
+ return -ENOMEM;
+
+@@ -580,7 +593,7 @@ nilfs_mdt_get_frozen_buffer(struct inode *inode, struct buffer_head *bh)
+ struct page *page;
+ int n;
+
+- page = find_lock_page(&shadow->frozen_data, bh->b_page->index);
++ page = find_lock_page(shadow->inode->i_mapping, bh->b_page->index);
+ if (page) {
+ if (page_has_buffers(page)) {
+ n = bh_offset(bh) >> inode->i_blkbits;
+@@ -621,10 +634,11 @@ void nilfs_mdt_restore_from_shadow_map(struct inode *inode)
+ nilfs_palloc_clear_cache(inode);
+
+ nilfs_clear_dirty_pages(inode->i_mapping, true);
+- nilfs_copy_back_pages(inode->i_mapping, &shadow->frozen_data);
++ nilfs_copy_back_pages(inode->i_mapping, shadow->inode->i_mapping);
+
+- nilfs_clear_dirty_pages(&ii->i_btnode_cache, true);
+- nilfs_copy_back_pages(&ii->i_btnode_cache, &shadow->frozen_btnodes);
++ nilfs_clear_dirty_pages(ii->i_assoc_inode->i_mapping, true);
++ nilfs_copy_back_pages(ii->i_assoc_inode->i_mapping,
++ NILFS_I(shadow->inode)->i_assoc_inode->i_mapping);
+
+ nilfs_bmap_restore(ii->i_bmap, &shadow->bmap_store);
+
+@@ -639,10 +653,11 @@ void nilfs_mdt_clear_shadow_map(struct inode *inode)
+ {
+ struct nilfs_mdt_info *mi = NILFS_MDT(inode);
+ struct nilfs_shadow_map *shadow = mi->mi_shadow;
++ struct inode *shadow_btnc_inode = NILFS_I(shadow->inode)->i_assoc_inode;
+
+ down_write(&mi->mi_sem);
+ nilfs_release_frozen_buffers(shadow);
+- truncate_inode_pages(&shadow->frozen_data, 0);
+- truncate_inode_pages(&shadow->frozen_btnodes, 0);
++ truncate_inode_pages(shadow->inode->i_mapping, 0);
++ truncate_inode_pages(shadow_btnc_inode->i_mapping, 0);
+ up_write(&mi->mi_sem);
+ }
+diff --git a/fs/nilfs2/mdt.h b/fs/nilfs2/mdt.h
+index e77aea4bb921c..9d8ac0d27c16e 100644
+--- a/fs/nilfs2/mdt.h
++++ b/fs/nilfs2/mdt.h
+@@ -18,14 +18,12 @@
+ /**
+ * struct nilfs_shadow_map - shadow mapping of meta data file
+ * @bmap_store: shadow copy of bmap state
+- * @frozen_data: shadowed dirty data pages
+- * @frozen_btnodes: shadowed dirty b-tree nodes' pages
++ * @inode: holder of page caches used in shadow mapping
+ * @frozen_buffers: list of frozen buffers
+ */
+ struct nilfs_shadow_map {
+ struct nilfs_bmap_store bmap_store;
+- struct address_space frozen_data;
+- struct address_space frozen_btnodes;
++ struct inode *inode;
+ struct list_head frozen_buffers;
+ };
+
+diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h
+index 60b21b6eeac06..7dcb77d387593 100644
+--- a/fs/nilfs2/nilfs.h
++++ b/fs/nilfs2/nilfs.h
+@@ -28,7 +28,7 @@
+ * @i_xattr: <TODO>
+ * @i_dir_start_lookup: page index of last successful search
+ * @i_cno: checkpoint number for GC inode
+- * @i_btnode_cache: cached pages of b-tree nodes
++ * @i_assoc_inode: associated inode (B-tree node cache holder or back pointer)
+ * @i_dirty: list for connecting dirty files
+ * @xattr_sem: semaphore for extended attributes processing
+ * @i_bh: buffer contains disk inode
+@@ -43,7 +43,7 @@ struct nilfs_inode_info {
+ __u64 i_xattr; /* sector_t ??? */
+ __u32 i_dir_start_lookup;
+ __u64 i_cno; /* check point number for GC inode */
+- struct address_space i_btnode_cache;
++ struct inode *i_assoc_inode;
+ struct list_head i_dirty; /* List for connecting dirty files */
+
+ #ifdef CONFIG_NILFS_XATTR
+@@ -75,13 +75,6 @@ NILFS_BMAP_I(const struct nilfs_bmap *bmap)
+ return container_of(bmap, struct nilfs_inode_info, i_bmap_data);
+ }
+
+-static inline struct inode *NILFS_BTNC_I(struct address_space *btnc)
+-{
+- struct nilfs_inode_info *ii =
+- container_of(btnc, struct nilfs_inode_info, i_btnode_cache);
+- return &ii->vfs_inode;
+-}
+-
+ /*
+ * Dynamic state flags of NILFS on-memory inode (i_state)
+ */
+@@ -98,6 +91,8 @@ enum {
+ NILFS_I_INODE_SYNC, /* dsync is not allowed for inode */
+ NILFS_I_BMAP, /* has bmap and btnode_cache */
+ NILFS_I_GCINODE, /* inode for GC, on memory only */
++ NILFS_I_BTNC, /* inode for btree node cache */
++ NILFS_I_SHADOW, /* inode for shadowed page cache */
+ };
+
+ /*
+@@ -267,6 +262,9 @@ struct inode *nilfs_iget(struct super_block *sb, struct nilfs_root *root,
+ unsigned long ino);
+ extern struct inode *nilfs_iget_for_gc(struct super_block *sb,
+ unsigned long ino, __u64 cno);
++int nilfs_attach_btree_node_cache(struct inode *inode);
++void nilfs_detach_btree_node_cache(struct inode *inode);
++struct inode *nilfs_iget_for_shadow(struct inode *inode);
+ extern void nilfs_update_inode(struct inode *, struct buffer_head *, int);
+ extern void nilfs_truncate(struct inode *);
+ extern void nilfs_evict_inode(struct inode *);
+diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c
+index 171fb5cd427fd..d1a148f0cae33 100644
+--- a/fs/nilfs2/page.c
++++ b/fs/nilfs2/page.c
+@@ -448,10 +448,9 @@ void nilfs_mapping_init(struct address_space *mapping, struct inode *inode)
+ /*
+ * NILFS2 needs clear_page_dirty() in the following two cases:
+ *
+- * 1) For B-tree node pages and data pages of the dat/gcdat, NILFS2 clears
+- * page dirty flags when it copies back pages from the shadow cache
+- * (gcdat->{i_mapping,i_btnode_cache}) to its original cache
+- * (dat->{i_mapping,i_btnode_cache}).
++ * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty
++ * flag of pages when it copies back pages from shadow cache to the
++ * original cache.
+ *
+ * 2) Some B-tree operations like insertion or deletion may dispose buffers
+ * in dirty state, and this needs to cancel the dirty state of their pages.
+diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c
+index 686c8ee7b29ce..314a23a166893 100644
+--- a/fs/nilfs2/segment.c
++++ b/fs/nilfs2/segment.c
+@@ -733,15 +733,18 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode,
+ struct list_head *listp)
+ {
+ struct nilfs_inode_info *ii = NILFS_I(inode);
+- struct address_space *mapping = &ii->i_btnode_cache;
++ struct inode *btnc_inode = ii->i_assoc_inode;
+ struct pagevec pvec;
+ struct buffer_head *bh, *head;
+ unsigned int i;
+ pgoff_t index = 0;
+
++ if (!btnc_inode)
++ return;
++
+ pagevec_init(&pvec);
+
+- while (pagevec_lookup_tag(&pvec, mapping, &index,
++ while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index,
+ PAGECACHE_TAG_DIRTY)) {
+ for (i = 0; i < pagevec_count(&pvec); i++) {
+ bh = head = page_buffers(pvec.pages[i]);
+@@ -2410,7 +2413,7 @@ nilfs_remove_written_gcinodes(struct the_nilfs *nilfs, struct list_head *head)
+ continue;
+ list_del_init(&ii->i_dirty);
+ truncate_inode_pages(&ii->vfs_inode.i_data, 0);
+- nilfs_btnode_cache_clear(&ii->i_btnode_cache);
++ nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping);
+ iput(&ii->vfs_inode);
+ }
+ }
+diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c
+index f6b2d280aab5a..2883ab625f61f 100644
+--- a/fs/nilfs2/super.c
++++ b/fs/nilfs2/super.c
+@@ -157,7 +157,8 @@ struct inode *nilfs_alloc_inode(struct super_block *sb)
+ ii->i_bh = NULL;
+ ii->i_state = 0;
+ ii->i_cno = 0;
+- nilfs_mapping_init(&ii->i_btnode_cache, &ii->vfs_inode);
++ ii->i_assoc_inode = NULL;
++ ii->i_bmap = &ii->i_bmap_data;
+ return &ii->vfs_inode;
+ }
+
+@@ -1377,8 +1378,6 @@ static void nilfs_inode_init_once(void *obj)
+ #ifdef CONFIG_NILFS_XATTR
+ init_rwsem(&ii->xattr_sem);
+ #endif
+- address_space_init_once(&ii->i_btnode_cache);
+- ii->i_bmap = &ii->i_bmap_data;
+ inode_init_once(&ii->vfs_inode);
+ }
+
+diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
+index 6facdf476255d..84ec851211d91 100644
+--- a/fs/notify/fanotify/fanotify_user.c
++++ b/fs/notify/fanotify/fanotify_user.c
+@@ -611,9 +611,6 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group,
+ if (fanotify_is_perm_event(event->mask))
+ FANOTIFY_PERM(event)->fd = fd;
+
+- if (f)
+- fd_install(fd, f);
+-
+ if (info_mode) {
+ ret = copy_info_records_to_user(event, info, info_mode, pidfd,
+ buf, count);
+@@ -621,6 +618,9 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group,
+ goto out_close_fd;
+ }
+
++ if (f)
++ fd_install(fd, f);
++
+ return metadata.event_len;
+
+ out_close_fd:
+diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
+index ab4f3362466d0..a43adeacd930c 100644
+--- a/fs/ntfs/file.c
++++ b/fs/ntfs/file.c
+@@ -1829,7 +1829,7 @@ again:
+ * pages being swapped out between us bringing them into memory
+ * and doing the actual copying.
+ */
+- if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
++ if (unlikely(fault_in_iov_iter_readable(i, bytes))) {
+ status = -EFAULT;
+ break;
+ }
+diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c
+index 4474adb393ca8..517b71c73aa96 100644
+--- a/fs/ntfs/inode.c
++++ b/fs/ntfs/inode.c
+@@ -1881,6 +1881,10 @@ int ntfs_read_inode_mount(struct inode *vi)
+ }
+ /* Now allocate memory for the attribute list. */
+ ni->attr_list_size = (u32)ntfs_attr_size(a);
++ if (!ni->attr_list_size) {
++ ntfs_error(sb, "Attr_list_size is zero");
++ goto put_err_out;
++ }
+ ni->attr_list = ntfs_malloc_nofs(ni->attr_list_size);
+ if (!ni->attr_list) {
+ ntfs_error(sb, "Not enough memory to allocate buffer "
+diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c
+index 43b1451bff539..54b9599640ef4 100644
+--- a/fs/ntfs3/file.c
++++ b/fs/ntfs3/file.c
+@@ -989,7 +989,7 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from)
+ frame_vbo = pos & ~(frame_size - 1);
+ index = frame_vbo >> PAGE_SHIFT;
+
+- if (unlikely(iov_iter_fault_in_readable(from, bytes))) {
++ if (unlikely(fault_in_iov_iter_readable(from, bytes))) {
+ err = -EFAULT;
+ goto out;
+ }
+diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c
+index 54d7843c02114..fc5f780fa2355 100644
+--- a/fs/ocfs2/file.c
++++ b/fs/ocfs2/file.c
+@@ -476,10 +476,11 @@ int ocfs2_truncate_file(struct inode *inode,
+ * greater than page size, so we have to truncate them
+ * anyway.
+ */
+- unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1);
+- truncate_inode_pages(inode->i_mapping, new_i_size);
+
+ if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) {
++ unmap_mapping_range(inode->i_mapping,
++ new_i_size + PAGE_SIZE - 1, 0, 1);
++ truncate_inode_pages(inode->i_mapping, new_i_size);
+ status = ocfs2_truncate_inline(inode, di_bh, new_i_size,
+ i_size_read(inode), 1);
+ if (status)
+@@ -498,6 +499,9 @@ int ocfs2_truncate_file(struct inode *inode,
+ goto bail_unlock_sem;
+ }
+
++ unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1);
++ truncate_inode_pages(inode->i_mapping, new_i_size);
++
+ status = ocfs2_commit_truncate(osb, inode, di_bh);
+ if (status < 0) {
+ mlog_errno(status);
+diff --git a/fs/ocfs2/quota_global.c b/fs/ocfs2/quota_global.c
+index f033de733adb3..effe92c7d6937 100644
+--- a/fs/ocfs2/quota_global.c
++++ b/fs/ocfs2/quota_global.c
+@@ -337,7 +337,6 @@ void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex)
+ /* Read information header from global quota file */
+ int ocfs2_global_read_info(struct super_block *sb, int type)
+ {
+- struct inode *gqinode = NULL;
+ unsigned int ino[OCFS2_MAXQUOTAS] = { USER_QUOTA_SYSTEM_INODE,
+ GROUP_QUOTA_SYSTEM_INODE };
+ struct ocfs2_global_disk_dqinfo dinfo;
+@@ -346,29 +345,31 @@ int ocfs2_global_read_info(struct super_block *sb, int type)
+ u64 pcount;
+ int status;
+
++ oinfo->dqi_gi.dqi_sb = sb;
++ oinfo->dqi_gi.dqi_type = type;
++ ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo);
++ oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk);
++ oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops;
++ oinfo->dqi_gqi_bh = NULL;
++ oinfo->dqi_gqi_count = 0;
++
+ /* Read global header */
+- gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type],
++ oinfo->dqi_gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type],
+ OCFS2_INVALID_SLOT);
+- if (!gqinode) {
++ if (!oinfo->dqi_gqinode) {
+ mlog(ML_ERROR, "failed to get global quota inode (type=%d)\n",
+ type);
+ status = -EINVAL;
+ goto out_err;
+ }
+- oinfo->dqi_gi.dqi_sb = sb;
+- oinfo->dqi_gi.dqi_type = type;
+- oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk);
+- oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops;
+- oinfo->dqi_gqi_bh = NULL;
+- oinfo->dqi_gqi_count = 0;
+- oinfo->dqi_gqinode = gqinode;
++
+ status = ocfs2_lock_global_qf(oinfo, 0);
+ if (status < 0) {
+ mlog_errno(status);
+ goto out_err;
+ }
+
+- status = ocfs2_extent_map_get_blocks(gqinode, 0, &oinfo->dqi_giblk,
++ status = ocfs2_extent_map_get_blocks(oinfo->dqi_gqinode, 0, &oinfo->dqi_giblk,
+ &pcount, NULL);
+ if (status < 0)
+ goto out_unlock;
+diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c
+index 0e4b16d4c037f..b1a8b046f4c22 100644
+--- a/fs/ocfs2/quota_local.c
++++ b/fs/ocfs2/quota_local.c
+@@ -702,8 +702,6 @@ static int ocfs2_local_read_info(struct super_block *sb, int type)
+ info->dqi_priv = oinfo;
+ oinfo->dqi_type = type;
+ INIT_LIST_HEAD(&oinfo->dqi_chunk);
+- oinfo->dqi_gqinode = NULL;
+- ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo);
+ oinfo->dqi_rec = NULL;
+ oinfo->dqi_lqi_bh = NULL;
+ oinfo->dqi_libh = NULL;
+diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c
+index 481017e1dac5a..166c8918c825a 100644
+--- a/fs/ocfs2/suballoc.c
++++ b/fs/ocfs2/suballoc.c
+@@ -1251,26 +1251,23 @@ static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh,
+ {
+ struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data;
+ struct journal_head *jh;
+- int ret = 1;
++ int ret;
+
+ if (ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap))
+ return 0;
+
+- if (!buffer_jbd(bg_bh))
++ jh = jbd2_journal_grab_journal_head(bg_bh);
++ if (!jh)
+ return 1;
+
+- jbd_lock_bh_journal_head(bg_bh);
+- if (buffer_jbd(bg_bh)) {
+- jh = bh2jh(bg_bh);
+- spin_lock(&jh->b_state_lock);
+- bg = (struct ocfs2_group_desc *) jh->b_committed_data;
+- if (bg)
+- ret = !ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap);
+- else
+- ret = 1;
+- spin_unlock(&jh->b_state_lock);
+- }
+- jbd_unlock_bh_journal_head(bg_bh);
++ spin_lock(&jh->b_state_lock);
++ bg = (struct ocfs2_group_desc *) jh->b_committed_data;
++ if (bg)
++ ret = !ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap);
++ else
++ ret = 1;
++ spin_unlock(&jh->b_state_lock);
++ jbd2_journal_put_journal_head(jh);
+
+ return ret;
+ }
+diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
+index 5c914ce9b3ac9..7ba3dabe16f09 100644
+--- a/fs/ocfs2/super.c
++++ b/fs/ocfs2/super.c
+@@ -1106,17 +1106,6 @@ static int ocfs2_fill_super(struct super_block *sb, void *data, int silent)
+ goto read_super_error;
+ }
+
+- root = d_make_root(inode);
+- if (!root) {
+- status = -ENOMEM;
+- mlog_errno(status);
+- goto read_super_error;
+- }
+-
+- sb->s_root = root;
+-
+- ocfs2_complete_mount_recovery(osb);
+-
+ osb->osb_dev_kset = kset_create_and_add(sb->s_id, NULL,
+ &ocfs2_kset->kobj);
+ if (!osb->osb_dev_kset) {
+@@ -1134,6 +1123,17 @@ static int ocfs2_fill_super(struct super_block *sb, void *data, int silent)
+ goto read_super_error;
+ }
+
++ root = d_make_root(inode);
++ if (!root) {
++ status = -ENOMEM;
++ mlog_errno(status);
++ goto read_super_error;
++ }
++
++ sb->s_root = root;
++
++ ocfs2_complete_mount_recovery(osb);
++
+ if (ocfs2_mount_local(osb))
+ snprintf(nodestr, sizeof(nodestr), "local");
+ else
+diff --git a/fs/open.c b/fs/open.c
+index daa324606a41f..e0df1536eb69f 100644
+--- a/fs/open.c
++++ b/fs/open.c
+@@ -856,8 +856,20 @@ static int do_dentry_open(struct file *f,
+ * of THPs into the page cache will fail.
+ */
+ smp_mb();
+- if (filemap_nr_thps(inode->i_mapping))
+- truncate_pagecache(inode, 0);
++ if (filemap_nr_thps(inode->i_mapping)) {
++ struct address_space *mapping = inode->i_mapping;
++
++ filemap_invalidate_lock(inode->i_mapping);
++ /*
++ * unmap_mapping_range just need to be called once
++ * here, because the private pages is not need to be
++ * unmapped mapping (e.g. data segment of dynamic
++ * shared libraries here).
++ */
++ unmap_mapping_range(mapping, 0, 0, 0);
++ truncate_inode_pages(mapping, 0);
++ filemap_invalidate_unlock(inode->i_mapping);
++ }
+ }
+
+ return 0;
+diff --git a/fs/orangefs/dcache.c b/fs/orangefs/dcache.c
+index fe484cf93e5cd..8bbe9486e3a62 100644
+--- a/fs/orangefs/dcache.c
++++ b/fs/orangefs/dcache.c
+@@ -26,8 +26,10 @@ static int orangefs_revalidate_lookup(struct dentry *dentry)
+ gossip_debug(GOSSIP_DCACHE_DEBUG, "%s: attempting lookup.\n", __func__);
+
+ new_op = op_alloc(ORANGEFS_VFS_OP_LOOKUP);
+- if (!new_op)
++ if (!new_op) {
++ ret = -ENOMEM;
+ goto out_put_parent;
++ }
+
+ new_op->upcall.req.lookup.sym_follow = ORANGEFS_LOOKUP_LINK_NO_FOLLOW;
+ new_op->upcall.req.lookup.parent_refn = parent->refn;
+diff --git a/fs/orangefs/orangefs-bufmap.c b/fs/orangefs/orangefs-bufmap.c
+index 538e839590ef5..b501dc07f9222 100644
+--- a/fs/orangefs/orangefs-bufmap.c
++++ b/fs/orangefs/orangefs-bufmap.c
+@@ -176,7 +176,7 @@ orangefs_bufmap_free(struct orangefs_bufmap *bufmap)
+ {
+ kfree(bufmap->page_array);
+ kfree(bufmap->desc_array);
+- kfree(bufmap->buffer_index_array);
++ bitmap_free(bufmap->buffer_index_array);
+ kfree(bufmap);
+ }
+
+@@ -226,8 +226,7 @@ orangefs_bufmap_alloc(struct ORANGEFS_dev_map_desc *user_desc)
+ bufmap->desc_size = user_desc->size;
+ bufmap->desc_shift = ilog2(bufmap->desc_size);
+
+- bufmap->buffer_index_array =
+- kzalloc(DIV_ROUND_UP(bufmap->desc_count, BITS_PER_LONG), GFP_KERNEL);
++ bufmap->buffer_index_array = bitmap_zalloc(bufmap->desc_count, GFP_KERNEL);
+ if (!bufmap->buffer_index_array)
+ goto out_free_bufmap;
+
+@@ -250,7 +249,7 @@ orangefs_bufmap_alloc(struct ORANGEFS_dev_map_desc *user_desc)
+ out_free_desc_array:
+ kfree(bufmap->desc_array);
+ out_free_index_array:
+- kfree(bufmap->buffer_index_array);
++ bitmap_free(bufmap->buffer_index_array);
+ out_free_bufmap:
+ kfree(bufmap);
+ out:
+diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c
+index 4e7d5bfa2949f..e040970408d4f 100644
+--- a/fs/overlayfs/copy_up.c
++++ b/fs/overlayfs/copy_up.c
+@@ -140,12 +140,14 @@ static int ovl_copy_fileattr(struct inode *inode, struct path *old,
+ int err;
+
+ err = ovl_real_fileattr_get(old, &oldfa);
+- if (err)
+- return err;
+-
+- err = ovl_real_fileattr_get(new, &newfa);
+- if (err)
++ if (err) {
++ /* Ntfs-3g returns -EINVAL for "no fileattr support" */
++ if (err == -ENOTTY || err == -EINVAL)
++ return 0;
++ pr_warn("failed to retrieve lower fileattr (%pd2, err=%i)\n",
++ old->dentry, err);
+ return err;
++ }
+
+ /*
+ * We cannot set immutable and append-only flags on upper inode,
+@@ -155,10 +157,31 @@ static int ovl_copy_fileattr(struct inode *inode, struct path *old,
+ */
+ if (oldfa.flags & OVL_PROT_FS_FLAGS_MASK) {
+ err = ovl_set_protattr(inode, new->dentry, &oldfa);
+- if (err)
++ if (err == -EPERM)
++ pr_warn_once("copying fileattr: no xattr on upper\n");
++ else if (err)
+ return err;
+ }
+
++ /* Don't bother copying flags if none are set */
++ if (!(oldfa.flags & OVL_COPY_FS_FLAGS_MASK))
++ return 0;
++
++ err = ovl_real_fileattr_get(new, &newfa);
++ if (err) {
++ /*
++ * Returning an error if upper doesn't support fileattr will
++ * result in a regression, so revert to the old behavior.
++ */
++ if (err == -ENOTTY || err == -EINVAL) {
++ pr_warn_once("copying fileattr: no support on upper\n");
++ return 0;
++ }
++ pr_warn("failed to retrieve upper fileattr (%pd2, err=%i)\n",
++ new->dentry, err);
++ return err;
++ }
++
+ BUILD_BUG_ON(OVL_COPY_FS_FLAGS_MASK & ~FS_COMMON_FL);
+ newfa.flags &= ~OVL_COPY_FS_FLAGS_MASK;
+ newfa.flags |= (oldfa.flags & OVL_COPY_FS_FLAGS_MASK);
+diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
+index 93c7c267de934..f18490813170a 100644
+--- a/fs/overlayfs/dir.c
++++ b/fs/overlayfs/dir.c
+@@ -137,8 +137,7 @@ kill_whiteout:
+ goto out;
+ }
+
+-static int ovl_mkdir_real(struct inode *dir, struct dentry **newdentry,
+- umode_t mode)
++int ovl_mkdir_real(struct inode *dir, struct dentry **newdentry, umode_t mode)
+ {
+ int err;
+ struct dentry *d, *dentry = *newdentry;
+diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
+index c88ac571593dc..44fea16751f1d 100644
+--- a/fs/overlayfs/file.c
++++ b/fs/overlayfs/file.c
+@@ -17,6 +17,7 @@
+
+ struct ovl_aio_req {
+ struct kiocb iocb;
++ refcount_t ref;
+ struct kiocb *orig_iocb;
+ struct fd fd;
+ };
+@@ -252,6 +253,14 @@ static rwf_t ovl_iocb_to_rwf(int ifl)
+ return flags;
+ }
+
++static inline void ovl_aio_put(struct ovl_aio_req *aio_req)
++{
++ if (refcount_dec_and_test(&aio_req->ref)) {
++ fdput(aio_req->fd);
++ kmem_cache_free(ovl_aio_request_cachep, aio_req);
++ }
++}
++
+ static void ovl_aio_cleanup_handler(struct ovl_aio_req *aio_req)
+ {
+ struct kiocb *iocb = &aio_req->iocb;
+@@ -268,8 +277,7 @@ static void ovl_aio_cleanup_handler(struct ovl_aio_req *aio_req)
+ }
+
+ orig_iocb->ki_pos = iocb->ki_pos;
+- fdput(aio_req->fd);
+- kmem_cache_free(ovl_aio_request_cachep, aio_req);
++ ovl_aio_put(aio_req);
+ }
+
+ static void ovl_aio_rw_complete(struct kiocb *iocb, long res, long res2)
+@@ -319,7 +327,9 @@ static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter)
+ aio_req->orig_iocb = iocb;
+ kiocb_clone(&aio_req->iocb, iocb, real.file);
+ aio_req->iocb.ki_complete = ovl_aio_rw_complete;
++ refcount_set(&aio_req->ref, 2);
+ ret = vfs_iocb_iter_read(real.file, &aio_req->iocb, iter);
++ ovl_aio_put(aio_req);
+ if (ret != -EIOCBQUEUED)
+ ovl_aio_cleanup_handler(aio_req);
+ }
+@@ -390,7 +400,9 @@ static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter)
+ kiocb_clone(&aio_req->iocb, iocb, real.file);
+ aio_req->iocb.ki_flags = ifl;
+ aio_req->iocb.ki_complete = ovl_aio_rw_complete;
++ refcount_set(&aio_req->ref, 2);
+ ret = vfs_iocb_iter_write(real.file, &aio_req->iocb, iter);
++ ovl_aio_put(aio_req);
+ if (ret != -EIOCBQUEUED)
+ ovl_aio_cleanup_handler(aio_req);
+ }
+diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
+index 832b17589733a..1f36158c7dbe2 100644
+--- a/fs/overlayfs/inode.c
++++ b/fs/overlayfs/inode.c
+@@ -610,7 +610,10 @@ int ovl_real_fileattr_get(struct path *realpath, struct fileattr *fa)
+ if (err)
+ return err;
+
+- return vfs_fileattr_get(realpath->dentry, fa);
++ err = vfs_fileattr_get(realpath->dentry, fa);
++ if (err == -ENOIOCTLCMD)
++ err = -ENOTTY;
++ return err;
+ }
+
+ int ovl_fileattr_get(struct dentry *dentry, struct fileattr *fa)
+diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
+index 3894f33479552..2cd5741c873b6 100644
+--- a/fs/overlayfs/overlayfs.h
++++ b/fs/overlayfs/overlayfs.h
+@@ -570,6 +570,7 @@ struct ovl_cattr {
+
+ #define OVL_CATTR(m) (&(struct ovl_cattr) { .mode = (m) })
+
++int ovl_mkdir_real(struct inode *dir, struct dentry **newdentry, umode_t mode);
+ struct dentry *ovl_create_real(struct inode *dir, struct dentry *newdentry,
+ struct ovl_cattr *attr);
+ int ovl_cleanup(struct inode *dir, struct dentry *dentry);
+diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
+index 178daa5e82c9d..265181c110ae2 100644
+--- a/fs/overlayfs/super.c
++++ b/fs/overlayfs/super.c
+@@ -787,10 +787,14 @@ retry:
+ goto retry;
+ }
+
+- work = ovl_create_real(dir, work, OVL_CATTR(attr.ia_mode));
+- err = PTR_ERR(work);
+- if (IS_ERR(work))
+- goto out_err;
++ err = ovl_mkdir_real(dir, &work, attr.ia_mode);
++ if (err)
++ goto out_dput;
++
++ /* Weird filesystem returning with hashed negative (kernfs)? */
++ err = -EINVAL;
++ if (d_really_is_negative(work))
++ goto out_dput;
+
+ /*
+ * Try to remove POSIX ACL xattrs from workdir. We are good if:
+diff --git a/fs/pipe.c b/fs/pipe.c
+index 6d4342bad9f15..751d5b36c84bb 100644
+--- a/fs/pipe.c
++++ b/fs/pipe.c
+@@ -252,7 +252,8 @@ pipe_read(struct kiocb *iocb, struct iov_iter *to)
+ */
+ was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
+ for (;;) {
+- unsigned int head = pipe->head;
++ /* Read ->head with a barrier vs post_one_notification() */
++ unsigned int head = smp_load_acquire(&pipe->head);
+ unsigned int tail = pipe->tail;
+ unsigned int mask = pipe->ring_size - 1;
+
+@@ -830,10 +831,8 @@ void free_pipe_info(struct pipe_inode_info *pipe)
+ int i;
+
+ #ifdef CONFIG_WATCH_QUEUE
+- if (pipe->watch_queue) {
++ if (pipe->watch_queue)
+ watch_queue_clear(pipe->watch_queue);
+- put_watch_queue(pipe->watch_queue);
+- }
+ #endif
+
+ (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
+@@ -843,6 +842,10 @@ void free_pipe_info(struct pipe_inode_info *pipe)
+ if (buf->ops)
+ pipe_buf_release(pipe, buf);
+ }
++#ifdef CONFIG_WATCH_QUEUE
++ if (pipe->watch_queue)
++ put_watch_queue(pipe->watch_queue);
++#endif
+ if (pipe->tmp_page)
+ __free_page(pipe->tmp_page);
+ kfree(pipe->bufs);
+diff --git a/fs/proc/base.c b/fs/proc/base.c
+index 533d5836eb9a4..1f394095eb880 100644
+--- a/fs/proc/base.c
++++ b/fs/proc/base.c
+@@ -67,6 +67,7 @@
+ #include <linux/mm.h>
+ #include <linux/swap.h>
+ #include <linux/rcupdate.h>
++#include <linux/kallsyms.h>
+ #include <linux/stacktrace.h>
+ #include <linux/resource.h>
+ #include <linux/module.h>
+@@ -386,17 +387,19 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns,
+ struct pid *pid, struct task_struct *task)
+ {
+ unsigned long wchan;
++ char symname[KSYM_NAME_LEN];
+
+- if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
+- wchan = get_wchan(task);
+- else
+- wchan = 0;
++ if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
++ goto print0;
+
+- if (wchan)
+- seq_printf(m, "%ps", (void *) wchan);
+- else
+- seq_putc(m, '0');
++ wchan = get_wchan(task);
++ if (wchan && !lookup_symbol_name(wchan, symname)) {
++ seq_puts(m, symname);
++ return 0;
++ }
+
++print0:
++ seq_putc(m, '0');
+ return 0;
+ }
+ #endif /* CONFIG_KALLSYMS */
+diff --git a/fs/proc/bootconfig.c b/fs/proc/bootconfig.c
+index 6d8d4bf208377..2e244ada1f970 100644
+--- a/fs/proc/bootconfig.c
++++ b/fs/proc/bootconfig.c
+@@ -32,6 +32,8 @@ static int __init copy_xbc_key_value_list(char *dst, size_t size)
+ int ret = 0;
+
+ key = kzalloc(XBC_KEYLEN_MAX, GFP_KERNEL);
++ if (!key)
++ return -ENOMEM;
+
+ xbc_for_each_key_value(leaf, val) {
+ ret = xbc_node_compose_key(leaf, key, XBC_KEYLEN_MAX);
+diff --git a/fs/proc/fd.c b/fs/proc/fd.c
+index 172c86270b312..913bef0d2a36c 100644
+--- a/fs/proc/fd.c
++++ b/fs/proc/fd.c
+@@ -72,7 +72,7 @@ out:
+ return 0;
+ }
+
+-static int seq_fdinfo_open(struct inode *inode, struct file *file)
++static int proc_fdinfo_access_allowed(struct inode *inode)
+ {
+ bool allowed = false;
+ struct task_struct *task = get_proc_task(inode);
+@@ -86,6 +86,16 @@ static int seq_fdinfo_open(struct inode *inode, struct file *file)
+ if (!allowed)
+ return -EACCES;
+
++ return 0;
++}
++
++static int seq_fdinfo_open(struct inode *inode, struct file *file)
++{
++ int ret = proc_fdinfo_access_allowed(inode);
++
++ if (ret)
++ return ret;
++
+ return single_open(file, seq_show, inode);
+ }
+
+@@ -348,12 +358,23 @@ static int proc_readfdinfo(struct file *file, struct dir_context *ctx)
+ proc_fdinfo_instantiate);
+ }
+
++static int proc_open_fdinfo(struct inode *inode, struct file *file)
++{
++ int ret = proc_fdinfo_access_allowed(inode);
++
++ if (ret)
++ return ret;
++
++ return 0;
++}
++
+ const struct inode_operations proc_fdinfo_inode_operations = {
+ .lookup = proc_lookupfdinfo,
+ .setattr = proc_setattr,
+ };
+
+ const struct file_operations proc_fdinfo_operations = {
++ .open = proc_open_fdinfo,
+ .read = generic_read_dir,
+ .iterate_shared = proc_readfdinfo,
+ .llseek = generic_file_llseek,
+diff --git a/fs/proc/stat.c b/fs/proc/stat.c
+index 6561a06ef9059..4fb8729a68d4e 100644
+--- a/fs/proc/stat.c
++++ b/fs/proc/stat.c
+@@ -24,7 +24,7 @@
+
+ #ifdef arch_idle_time
+
+-static u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
++u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
+ {
+ u64 idle;
+
+@@ -46,7 +46,7 @@ static u64 get_iowait_time(struct kernel_cpustat *kcs, int cpu)
+
+ #else
+
+-static u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
++u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
+ {
+ u64 idle, idle_usecs = -1ULL;
+
+diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
+index cf25be3e03212..79ca4d69dfd6b 100644
+--- a/fs/proc/task_mmu.c
++++ b/fs/proc/task_mmu.c
+@@ -430,7 +430,8 @@ static void smaps_page_accumulate(struct mem_size_stats *mss,
+ }
+
+ static void smaps_account(struct mem_size_stats *mss, struct page *page,
+- bool compound, bool young, bool dirty, bool locked)
++ bool compound, bool young, bool dirty, bool locked,
++ bool migration)
+ {
+ int i, nr = compound ? compound_nr(page) : 1;
+ unsigned long size = nr * PAGE_SIZE;
+@@ -457,8 +458,15 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
+ * page_count(page) == 1 guarantees the page is mapped exactly once.
+ * If any subpage of the compound page mapped with PTE it would elevate
+ * page_count().
++ *
++ * The page_mapcount() is called to get a snapshot of the mapcount.
++ * Without holding the page lock this snapshot can be slightly wrong as
++ * we cannot always read the mapcount atomically. It is not safe to
++ * call page_mapcount() even with PTL held if the page is not mapped,
++ * especially for migration entries. Treat regular migration entries
++ * as mapcount == 1.
+ */
+- if (page_count(page) == 1) {
++ if ((page_count(page) == 1) || migration) {
+ smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
+ locked, true);
+ return;
+@@ -495,6 +503,7 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr,
+ struct vm_area_struct *vma = walk->vma;
+ bool locked = !!(vma->vm_flags & VM_LOCKED);
+ struct page *page = NULL;
++ bool migration = false;
+
+ if (pte_present(*pte)) {
+ page = vm_normal_page(vma, addr, *pte);
+@@ -514,8 +523,11 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr,
+ } else {
+ mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
+ }
+- } else if (is_pfn_swap_entry(swpent))
++ } else if (is_pfn_swap_entry(swpent)) {
++ if (is_migration_entry(swpent))
++ migration = true;
+ page = pfn_swap_entry_to_page(swpent);
++ }
+ } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
+ && pte_none(*pte))) {
+ page = xa_load(&vma->vm_file->f_mapping->i_pages,
+@@ -528,7 +540,8 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr,
+ if (!page)
+ return;
+
+- smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), locked);
++ smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte),
++ locked, migration);
+ }
+
+ #ifdef CONFIG_TRANSPARENT_HUGEPAGE
+@@ -539,6 +552,7 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
+ struct vm_area_struct *vma = walk->vma;
+ bool locked = !!(vma->vm_flags & VM_LOCKED);
+ struct page *page = NULL;
++ bool migration = false;
+
+ if (pmd_present(*pmd)) {
+ /* FOLL_DUMP will return -EFAULT on huge zero page */
+@@ -546,8 +560,10 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
+ } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
+ swp_entry_t entry = pmd_to_swp_entry(*pmd);
+
+- if (is_migration_entry(entry))
++ if (is_migration_entry(entry)) {
++ migration = true;
+ page = pfn_swap_entry_to_page(entry);
++ }
+ }
+ if (IS_ERR_OR_NULL(page))
+ return;
+@@ -559,7 +575,9 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
+ /* pass */;
+ else
+ mss->file_thp += HPAGE_PMD_SIZE;
+- smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), locked);
++
++ smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
++ locked, migration);
+ }
+ #else
+ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
+@@ -1363,6 +1381,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
+ {
+ u64 frame = 0, flags = 0;
+ struct page *page = NULL;
++ bool migration = false;
+
+ if (pte_present(pte)) {
+ if (pm->show_pfn)
+@@ -1384,13 +1403,14 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
+ frame = swp_type(entry) |
+ (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
+ flags |= PM_SWAP;
++ migration = is_migration_entry(entry);
+ if (is_pfn_swap_entry(entry))
+ page = pfn_swap_entry_to_page(entry);
+ }
+
+ if (page && !PageAnon(page))
+ flags |= PM_FILE;
+- if (page && page_mapcount(page) == 1)
++ if (page && !migration && page_mapcount(page) == 1)
+ flags |= PM_MMAP_EXCLUSIVE;
+ if (vma->vm_flags & VM_SOFTDIRTY)
+ flags |= PM_SOFT_DIRTY;
+@@ -1406,8 +1426,9 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
+ spinlock_t *ptl;
+ pte_t *pte, *orig_pte;
+ int err = 0;
+-
+ #ifdef CONFIG_TRANSPARENT_HUGEPAGE
++ bool migration = false;
++
+ ptl = pmd_trans_huge_lock(pmdp, vma);
+ if (ptl) {
+ u64 flags = 0, frame = 0;
+@@ -1446,11 +1467,12 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
+ if (pmd_swp_uffd_wp(pmd))
+ flags |= PM_UFFD_WP;
+ VM_BUG_ON(!is_pmd_migration_entry(pmd));
++ migration = is_migration_entry(entry);
+ page = pfn_swap_entry_to_page(entry);
+ }
+ #endif
+
+- if (page && page_mapcount(page) == 1)
++ if (page && !migration && page_mapcount(page) == 1)
+ flags |= PM_MMAP_EXCLUSIVE;
+
+ for (; addr != end; addr += PAGE_SIZE) {
+@@ -1560,7 +1582,8 @@ static const struct mm_walk_ops pagemap_ops = {
+ * Bits 5-54 swap offset if swapped
+ * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
+ * Bit 56 page exclusively mapped
+- * Bits 57-60 zero
++ * Bit 57 pte is uffd-wp write-protected
++ * Bits 58-60 zero
+ * Bit 61 page is file-page or shared-anon
+ * Bit 62 page swapped
+ * Bit 63 page present
+diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c
+index 5a1b228964fb7..deb99bc9b7e6b 100644
+--- a/fs/proc/uptime.c
++++ b/fs/proc/uptime.c
+@@ -12,18 +12,22 @@ static int uptime_proc_show(struct seq_file *m, void *v)
+ {
+ struct timespec64 uptime;
+ struct timespec64 idle;
+- u64 nsec;
++ u64 idle_nsec;
+ u32 rem;
+ int i;
+
+- nsec = 0;
+- for_each_possible_cpu(i)
+- nsec += (__force u64) kcpustat_cpu(i).cpustat[CPUTIME_IDLE];
++ idle_nsec = 0;
++ for_each_possible_cpu(i) {
++ struct kernel_cpustat kcs;
++
++ kcpustat_cpu_fetch(&kcs, i);
++ idle_nsec += get_idle_time(&kcs, i);
++ }
+
+ ktime_get_boottime_ts64(&uptime);
+ timens_add_boottime(&uptime);
+
+- idle.tv_sec = div_u64_rem(nsec, NSEC_PER_SEC, &rem);
++ idle.tv_sec = div_u64_rem(idle_nsec, NSEC_PER_SEC, &rem);
+ idle.tv_nsec = rem;
+ seq_printf(m, "%lu.%02lu %lu.%02lu\n",
+ (unsigned long) uptime.tv_sec,
+diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c
+index 9a15334da2086..e5730986758fa 100644
+--- a/fs/proc/vmcore.c
++++ b/fs/proc/vmcore.c
+@@ -124,9 +124,13 @@ ssize_t read_from_oldmem(char *buf, size_t count,
+ nr_bytes = count;
+
+ /* If pfn is not ram, return zeros for sparse dump files */
+- if (pfn_is_ram(pfn) == 0)
+- memset(buf, 0, nr_bytes);
+- else {
++ if (pfn_is_ram(pfn) == 0) {
++ tmp = 0;
++ if (!userbuf)
++ memset(buf, 0, nr_bytes);
++ else if (clear_user(buf, nr_bytes))
++ tmp = -EFAULT;
++ } else {
+ if (encrypted)
+ tmp = copy_oldmem_page_encrypted(pfn, buf,
+ nr_bytes,
+@@ -135,10 +139,10 @@ ssize_t read_from_oldmem(char *buf, size_t count,
+ else
+ tmp = copy_oldmem_page(pfn, buf, nr_bytes,
+ offset, userbuf);
+-
+- if (tmp < 0)
+- return tmp;
+ }
++ if (tmp < 0)
++ return tmp;
++
+ *ppos += nr_bytes;
+ count -= nr_bytes;
+ buf += nr_bytes;
+diff --git a/fs/pstore/Kconfig b/fs/pstore/Kconfig
+index 328da35da3908..8adabde685f13 100644
+--- a/fs/pstore/Kconfig
++++ b/fs/pstore/Kconfig
+@@ -173,7 +173,6 @@ config PSTORE_BLK
+ tristate "Log panic/oops to a block device"
+ depends on PSTORE
+ depends on BLOCK
+- depends on BROKEN
+ select PSTORE_ZONE
+ default n
+ help
+diff --git a/fs/pstore/blk.c b/fs/pstore/blk.c
+index 04ce58c939a0b..6093088de49fd 100644
+--- a/fs/pstore/blk.c
++++ b/fs/pstore/blk.c
+@@ -311,7 +311,7 @@ static int __init __best_effort_init(void)
+ if (ret)
+ kfree(best_effort_dev);
+ else
+- pr_info("attached %s (%zu) (no dedicated panic_write!)\n",
++ pr_info("attached %s (%lu) (no dedicated panic_write!)\n",
+ blkdev, best_effort_dev->zone.total_size);
+
+ return ret;
+diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
+index b9614db48b1de..ad96ba97d8f97 100644
+--- a/fs/pstore/platform.c
++++ b/fs/pstore/platform.c
+@@ -143,21 +143,22 @@ static void pstore_timer_kick(void)
+ mod_timer(&pstore_timer, jiffies + msecs_to_jiffies(pstore_update_ms));
+ }
+
+-/*
+- * Should pstore_dump() wait for a concurrent pstore_dump()? If
+- * not, the current pstore_dump() will report a failure to dump
+- * and return.
+- */
+-static bool pstore_cannot_wait(enum kmsg_dump_reason reason)
++static bool pstore_cannot_block_path(enum kmsg_dump_reason reason)
+ {
+- /* In NMI path, pstore shouldn't block regardless of reason. */
++ /*
++ * In case of NMI path, pstore shouldn't be blocked
++ * regardless of reason.
++ */
+ if (in_nmi())
+ return true;
+
+ switch (reason) {
+ /* In panic case, other cpus are stopped by smp_send_stop(). */
+ case KMSG_DUMP_PANIC:
+- /* Emergency restart shouldn't be blocked. */
++ /*
++ * Emergency restart shouldn't be blocked by spinning on
++ * pstore_info::buf_lock.
++ */
+ case KMSG_DUMP_EMERG:
+ return true;
+ default:
+@@ -389,21 +390,19 @@ static void pstore_dump(struct kmsg_dumper *dumper,
+ unsigned long total = 0;
+ const char *why;
+ unsigned int part = 1;
++ unsigned long flags = 0;
+ int ret;
+
+ why = kmsg_dump_reason_str(reason);
+
+- if (down_trylock(&psinfo->buf_lock)) {
+- /* Failed to acquire lock: give up if we cannot wait. */
+- if (pstore_cannot_wait(reason)) {
+- pr_err("dump skipped in %s path: may corrupt error record\n",
+- in_nmi() ? "NMI" : why);
+- return;
+- }
+- if (down_interruptible(&psinfo->buf_lock)) {
+- pr_err("could not grab semaphore?!\n");
++ if (pstore_cannot_block_path(reason)) {
++ if (!spin_trylock_irqsave(&psinfo->buf_lock, flags)) {
++ pr_err("dump skipped in %s path because of concurrent dump\n",
++ in_nmi() ? "NMI" : why);
+ return;
+ }
++ } else {
++ spin_lock_irqsave(&psinfo->buf_lock, flags);
+ }
+
+ kmsg_dump_rewind(&iter);
+@@ -467,8 +466,7 @@ static void pstore_dump(struct kmsg_dumper *dumper,
+ total += record.size;
+ part++;
+ }
+-
+- up(&psinfo->buf_lock);
++ spin_unlock_irqrestore(&psinfo->buf_lock, flags);
+ }
+
+ static struct kmsg_dumper pstore_dumper = {
+@@ -594,7 +592,7 @@ int pstore_register(struct pstore_info *psi)
+ psi->write_user = pstore_write_user_compat;
+ psinfo = psi;
+ mutex_init(&psinfo->read_mutex);
+- sema_init(&psinfo->buf_lock, 1);
++ spin_lock_init(&psinfo->buf_lock);
+
+ if (psi->flags & PSTORE_FLAGS_DMESG)
+ allocate_buf_for_compression();
+diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
+index 22d904bde6ab9..a74aef99bd3d6 100644
+--- a/fs/quota/dquot.c
++++ b/fs/quota/dquot.c
+@@ -690,9 +690,14 @@ int dquot_quota_sync(struct super_block *sb, int type)
+ /* This is not very clever (and fast) but currently I don't know about
+ * any other simple way of getting quota data to disk and we must get
+ * them there for userspace to be visible... */
+- if (sb->s_op->sync_fs)
+- sb->s_op->sync_fs(sb, 1);
+- sync_blockdev(sb->s_bdev);
++ if (sb->s_op->sync_fs) {
++ ret = sb->s_op->sync_fs(sb, 1);
++ if (ret)
++ return ret;
++ }
++ ret = sync_blockdev(sb->s_bdev);
++ if (ret)
++ return ret;
+
+ /*
+ * Now when everything is written we can discard the pagecache so
+diff --git a/fs/quota/quota_tree.c b/fs/quota/quota_tree.c
+index d3e995e1046fb..5f2405994280a 100644
+--- a/fs/quota/quota_tree.c
++++ b/fs/quota/quota_tree.c
+@@ -414,6 +414,7 @@ static int free_dqentry(struct qtree_mem_dqinfo *info, struct dquot *dquot,
+ quota_error(dquot->dq_sb, "Quota structure has offset to "
+ "other block (%u) than it should (%u)", blk,
+ (uint)(dquot->dq_off >> info->dqi_blocksize_bits));
++ ret = -EIO;
+ goto out_buf;
+ }
+ ret = read_blk(info, blk, buf);
+@@ -479,6 +480,13 @@ static int remove_tree(struct qtree_mem_dqinfo *info, struct dquot *dquot,
+ goto out_buf;
+ }
+ newblk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]);
++ if (newblk < QT_TREEOFF || newblk >= info->dqi_blocks) {
++ quota_error(dquot->dq_sb, "Getting block too big (%u >= %u)",
++ newblk, info->dqi_blocks);
++ ret = -EUCLEAN;
++ goto out_buf;
++ }
++
+ if (depth == info->dqi_qtree_depth - 1) {
+ ret = free_dqentry(info, dquot, newblk);
+ newblk = 0;
+@@ -578,6 +586,13 @@ static loff_t find_tree_dqentry(struct qtree_mem_dqinfo *info,
+ blk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]);
+ if (!blk) /* No reference? */
+ goto out_buf;
++ if (blk < QT_TREEOFF || blk >= info->dqi_blocks) {
++ quota_error(dquot->dq_sb, "Getting block too big (%u >= %u)",
++ blk, info->dqi_blocks);
++ ret = -EUCLEAN;
++ goto out_buf;
++ }
++
+ if (depth < info->dqi_qtree_depth - 1)
+ ret = find_tree_dqentry(info, dquot, blk, depth+1);
+ else
+diff --git a/fs/select.c b/fs/select.c
+index 945896d0ac9e7..5edffee1162c2 100644
+--- a/fs/select.c
++++ b/fs/select.c
+@@ -458,9 +458,11 @@ get_max:
+ return max;
+ }
+
+-#define POLLIN_SET (EPOLLRDNORM | EPOLLRDBAND | EPOLLIN | EPOLLHUP | EPOLLERR)
+-#define POLLOUT_SET (EPOLLWRBAND | EPOLLWRNORM | EPOLLOUT | EPOLLERR)
+-#define POLLEX_SET (EPOLLPRI)
++#define POLLIN_SET (EPOLLRDNORM | EPOLLRDBAND | EPOLLIN | EPOLLHUP | EPOLLERR |\
++ EPOLLNVAL)
++#define POLLOUT_SET (EPOLLWRBAND | EPOLLWRNORM | EPOLLOUT | EPOLLERR |\
++ EPOLLNVAL)
++#define POLLEX_SET (EPOLLPRI | EPOLLNVAL)
+
+ static inline void wait_key_set(poll_table *wait, unsigned long in,
+ unsigned long out, unsigned long bit,
+@@ -527,6 +529,7 @@ static int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time)
+ break;
+ if (!(bit & all_bits))
+ continue;
++ mask = EPOLLNVAL;
+ f = fdget(i);
+ if (f.file) {
+ wait_key_set(wait, in, out, bit,
+@@ -534,34 +537,34 @@ static int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time)
+ mask = vfs_poll(f.file, wait);
+
+ fdput(f);
+- if ((mask & POLLIN_SET) && (in & bit)) {
+- res_in |= bit;
+- retval++;
+- wait->_qproc = NULL;
+- }
+- if ((mask & POLLOUT_SET) && (out & bit)) {
+- res_out |= bit;
+- retval++;
+- wait->_qproc = NULL;
+- }
+- if ((mask & POLLEX_SET) && (ex & bit)) {
+- res_ex |= bit;
+- retval++;
+- wait->_qproc = NULL;
+- }
+- /* got something, stop busy polling */
+- if (retval) {
+- can_busy_loop = false;
+- busy_flag = 0;
+-
+- /*
+- * only remember a returned
+- * POLL_BUSY_LOOP if we asked for it
+- */
+- } else if (busy_flag & mask)
+- can_busy_loop = true;
+-
+ }
++ if ((mask & POLLIN_SET) && (in & bit)) {
++ res_in |= bit;
++ retval++;
++ wait->_qproc = NULL;
++ }
++ if ((mask & POLLOUT_SET) && (out & bit)) {
++ res_out |= bit;
++ retval++;
++ wait->_qproc = NULL;
++ }
++ if ((mask & POLLEX_SET) && (ex & bit)) {
++ res_ex |= bit;
++ retval++;
++ wait->_qproc = NULL;
++ }
++ /* got something, stop busy polling */
++ if (retval) {
++ can_busy_loop = false;
++ busy_flag = 0;
++
++ /*
++ * only remember a returned
++ * POLL_BUSY_LOOP if we asked for it
++ */
++ } else if (busy_flag & mask)
++ can_busy_loop = true;
++
+ }
+ if (res_in)
+ *rinp = res_in;
+diff --git a/fs/signalfd.c b/fs/signalfd.c
+index 040e1cf905282..65ce0e72e7b95 100644
+--- a/fs/signalfd.c
++++ b/fs/signalfd.c
+@@ -35,17 +35,7 @@
+
+ void signalfd_cleanup(struct sighand_struct *sighand)
+ {
+- wait_queue_head_t *wqh = &sighand->signalfd_wqh;
+- /*
+- * The lockless check can race with remove_wait_queue() in progress,
+- * but in this case its caller should run under rcu_read_lock() and
+- * sighand_cachep is SLAB_TYPESAFE_BY_RCU, we can safely return.
+- */
+- if (likely(!waitqueue_active(wqh)))
+- return;
+-
+- /* wait_queue_entry_t->func(POLLFREE) should do remove_wait_queue() */
+- wake_up_poll(wqh, EPOLLHUP | POLLFREE);
++ wake_up_pollfree(&sighand->signalfd_wqh);
+ }
+
+ struct signalfd_ctx {
+diff --git a/fs/smbfs_common/cifs_arc4.c b/fs/smbfs_common/cifs_arc4.c
+index 85ba15a60b13b..043e4cb839fa2 100644
+--- a/fs/smbfs_common/cifs_arc4.c
++++ b/fs/smbfs_common/cifs_arc4.c
+@@ -72,16 +72,3 @@ void cifs_arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int l
+ ctx->y = y;
+ }
+ EXPORT_SYMBOL_GPL(cifs_arc4_crypt);
+-
+-static int __init
+-init_smbfs_common(void)
+-{
+- return 0;
+-}
+-static void __init
+-exit_smbfs_common(void)
+-{
+-}
+-
+-module_init(init_smbfs_common)
+-module_exit(exit_smbfs_common)
+diff --git a/fs/stat.c b/fs/stat.c
+index 28d2020ba1f42..246d138ec0669 100644
+--- a/fs/stat.c
++++ b/fs/stat.c
+@@ -334,9 +334,6 @@ SYSCALL_DEFINE2(fstat, unsigned int, fd, struct __old_kernel_stat __user *, stat
+ # define choose_32_64(a,b) b
+ #endif
+
+-#define valid_dev(x) choose_32_64(old_valid_dev(x),true)
+-#define encode_dev(x) choose_32_64(old_encode_dev,new_encode_dev)(x)
+-
+ #ifndef INIT_STRUCT_STAT_PADDING
+ # define INIT_STRUCT_STAT_PADDING(st) memset(&st, 0, sizeof(st))
+ #endif
+@@ -345,7 +342,9 @@ static int cp_new_stat(struct kstat *stat, struct stat __user *statbuf)
+ {
+ struct stat tmp;
+
+- if (!valid_dev(stat->dev) || !valid_dev(stat->rdev))
++ if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev))
++ return -EOVERFLOW;
++ if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev))
+ return -EOVERFLOW;
+ #if BITS_PER_LONG == 32
+ if (stat->size > MAX_NON_LFS)
+@@ -353,7 +352,7 @@ static int cp_new_stat(struct kstat *stat, struct stat __user *statbuf)
+ #endif
+
+ INIT_STRUCT_STAT_PADDING(tmp);
+- tmp.st_dev = encode_dev(stat->dev);
++ tmp.st_dev = new_encode_dev(stat->dev);
+ tmp.st_ino = stat->ino;
+ if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
+ return -EOVERFLOW;
+@@ -363,7 +362,7 @@ static int cp_new_stat(struct kstat *stat, struct stat __user *statbuf)
+ return -EOVERFLOW;
+ SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid));
+ SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid));
+- tmp.st_rdev = encode_dev(stat->rdev);
++ tmp.st_rdev = new_encode_dev(stat->rdev);
+ tmp.st_size = stat->size;
+ tmp.st_atime = stat->atime.tv_sec;
+ tmp.st_mtime = stat->mtime.tv_sec;
+@@ -644,11 +643,13 @@ static int cp_compat_stat(struct kstat *stat, struct compat_stat __user *ubuf)
+ {
+ struct compat_stat tmp;
+
+- if (!old_valid_dev(stat->dev) || !old_valid_dev(stat->rdev))
++ if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev))
++ return -EOVERFLOW;
++ if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev))
+ return -EOVERFLOW;
+
+ memset(&tmp, 0, sizeof(tmp));
+- tmp.st_dev = old_encode_dev(stat->dev);
++ tmp.st_dev = new_encode_dev(stat->dev);
+ tmp.st_ino = stat->ino;
+ if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
+ return -EOVERFLOW;
+@@ -658,7 +659,7 @@ static int cp_compat_stat(struct kstat *stat, struct compat_stat __user *ubuf)
+ return -EOVERFLOW;
+ SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid));
+ SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid));
+- tmp.st_rdev = old_encode_dev(stat->rdev);
++ tmp.st_rdev = new_encode_dev(stat->rdev);
+ if ((u64) stat->size > MAX_NON_LFS)
+ return -EOVERFLOW;
+ tmp.st_size = stat->size;
+diff --git a/fs/super.c b/fs/super.c
+index bcef3a6f4c4b5..87379bb1f7a30 100644
+--- a/fs/super.c
++++ b/fs/super.c
+@@ -1421,8 +1421,8 @@ struct dentry *mount_nodev(struct file_system_type *fs_type,
+ }
+ EXPORT_SYMBOL(mount_nodev);
+
+-static int reconfigure_single(struct super_block *s,
+- int flags, void *data)
++int reconfigure_single(struct super_block *s,
++ int flags, void *data)
+ {
+ struct fs_context *fc;
+ int ret;
+@@ -1616,11 +1616,9 @@ static void lockdep_sb_freeze_acquire(struct super_block *sb)
+ percpu_rwsem_acquire(sb->s_writers.rw_sem + level, 0, _THIS_IP_);
+ }
+
+-static void sb_freeze_unlock(struct super_block *sb)
++static void sb_freeze_unlock(struct super_block *sb, int level)
+ {
+- int level;
+-
+- for (level = SB_FREEZE_LEVELS - 1; level >= 0; level--)
++ for (level--; level >= 0; level--)
+ percpu_up_write(sb->s_writers.rw_sem + level);
+ }
+
+@@ -1691,7 +1689,14 @@ int freeze_super(struct super_block *sb)
+ sb_wait_write(sb, SB_FREEZE_PAGEFAULT);
+
+ /* All writers are done so after syncing there won't be dirty data */
+- sync_filesystem(sb);
++ ret = sync_filesystem(sb);
++ if (ret) {
++ sb->s_writers.frozen = SB_UNFROZEN;
++ sb_freeze_unlock(sb, SB_FREEZE_PAGEFAULT);
++ wake_up(&sb->s_writers.wait_unfrozen);
++ deactivate_locked_super(sb);
++ return ret;
++ }
+
+ /* Now wait for internal filesystem counter */
+ sb->s_writers.frozen = SB_FREEZE_FS;
+@@ -1703,7 +1708,7 @@ int freeze_super(struct super_block *sb)
+ printk(KERN_ERR
+ "VFS:Filesystem freeze failed\n");
+ sb->s_writers.frozen = SB_UNFROZEN;
+- sb_freeze_unlock(sb);
++ sb_freeze_unlock(sb, SB_FREEZE_FS);
+ wake_up(&sb->s_writers.wait_unfrozen);
+ deactivate_locked_super(sb);
+ return ret;
+@@ -1748,7 +1753,7 @@ static int thaw_super_locked(struct super_block *sb)
+ }
+
+ sb->s_writers.frozen = SB_UNFROZEN;
+- sb_freeze_unlock(sb);
++ sb_freeze_unlock(sb, SB_FREEZE_FS);
+ out:
+ wake_up(&sb->s_writers.wait_unfrozen);
+ deactivate_locked_super(sb);
+diff --git a/fs/sync.c b/fs/sync.c
+index 1373a610dc784..c7690016453e4 100644
+--- a/fs/sync.c
++++ b/fs/sync.c
+@@ -3,6 +3,7 @@
+ * High-level sync()-related operations
+ */
+
++#include <linux/blkdev.h>
+ #include <linux/kernel.h>
+ #include <linux/file.h>
+ #include <linux/fs.h>
+@@ -21,25 +22,6 @@
+ #define VALID_FLAGS (SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| \
+ SYNC_FILE_RANGE_WAIT_AFTER)
+
+-/*
+- * Do the filesystem syncing work. For simple filesystems
+- * writeback_inodes_sb(sb) just dirties buffers with inodes so we have to
+- * submit IO for these buffers via __sync_blockdev(). This also speeds up the
+- * wait == 1 case since in that case write_inode() functions do
+- * sync_dirty_buffer() and thus effectively write one block at a time.
+- */
+-static int __sync_filesystem(struct super_block *sb, int wait)
+-{
+- if (wait)
+- sync_inodes_sb(sb);
+- else
+- writeback_inodes_sb(sb, WB_REASON_SYNC);
+-
+- if (sb->s_op->sync_fs)
+- sb->s_op->sync_fs(sb, wait);
+- return __sync_blockdev(sb->s_bdev, wait);
+-}
+-
+ /*
+ * Write out and wait upon all dirty data associated with this
+ * superblock. Filesystem data as well as the underlying block
+@@ -47,7 +29,7 @@ static int __sync_filesystem(struct super_block *sb, int wait)
+ */
+ int sync_filesystem(struct super_block *sb)
+ {
+- int ret;
++ int ret = 0;
+
+ /*
+ * We need to be protected against the filesystem going from
+@@ -61,10 +43,31 @@ int sync_filesystem(struct super_block *sb)
+ if (sb_rdonly(sb))
+ return 0;
+
+- ret = __sync_filesystem(sb, 0);
+- if (ret < 0)
++ /*
++ * Do the filesystem syncing work. For simple filesystems
++ * writeback_inodes_sb(sb) just dirties buffers with inodes so we have
++ * to submit I/O for these buffers via sync_blockdev(). This also
++ * speeds up the wait == 1 case since in that case write_inode()
++ * methods call sync_dirty_buffer() and thus effectively write one block
++ * at a time.
++ */
++ writeback_inodes_sb(sb, WB_REASON_SYNC);
++ if (sb->s_op->sync_fs) {
++ ret = sb->s_op->sync_fs(sb, 0);
++ if (ret)
++ return ret;
++ }
++ ret = sync_blockdev_nowait(sb->s_bdev);
++ if (ret)
+ return ret;
+- return __sync_filesystem(sb, 1);
++
++ sync_inodes_sb(sb);
++ if (sb->s_op->sync_fs) {
++ ret = sb->s_op->sync_fs(sb, 1);
++ if (ret)
++ return ret;
++ }
++ return sync_blockdev(sb->s_bdev);
+ }
+ EXPORT_SYMBOL(sync_filesystem);
+
+@@ -81,21 +84,6 @@ static void sync_fs_one_sb(struct super_block *sb, void *arg)
+ sb->s_op->sync_fs(sb, *(int *)arg);
+ }
+
+-static void fdatawrite_one_bdev(struct block_device *bdev, void *arg)
+-{
+- filemap_fdatawrite(bdev->bd_inode->i_mapping);
+-}
+-
+-static void fdatawait_one_bdev(struct block_device *bdev, void *arg)
+-{
+- /*
+- * We keep the error status of individual mapping so that
+- * applications can catch the writeback error using fsync(2).
+- * See filemap_fdatawait_keep_errors() for details.
+- */
+- filemap_fdatawait_keep_errors(bdev->bd_inode->i_mapping);
+-}
+-
+ /*
+ * Sync everything. We start by waking flusher threads so that most of
+ * writeback runs on all devices in parallel. Then we sync all inodes reliably
+@@ -114,8 +102,8 @@ void ksys_sync(void)
+ iterate_supers(sync_inodes_one_sb, NULL);
+ iterate_supers(sync_fs_one_sb, &nowait);
+ iterate_supers(sync_fs_one_sb, &wait);
+- iterate_bdevs(fdatawrite_one_bdev, NULL);
+- iterate_bdevs(fdatawait_one_bdev, NULL);
++ sync_bdevs(false);
++ sync_bdevs(true);
+ if (unlikely(laptop_mode))
+ laptop_sync_completion();
+ }
+@@ -136,10 +124,10 @@ static void do_sync_work(struct work_struct *work)
+ */
+ iterate_supers(sync_inodes_one_sb, &nowait);
+ iterate_supers(sync_fs_one_sb, &nowait);
+- iterate_bdevs(fdatawrite_one_bdev, NULL);
++ sync_bdevs(false);
+ iterate_supers(sync_inodes_one_sb, &nowait);
+ iterate_supers(sync_fs_one_sb, &nowait);
+- iterate_bdevs(fdatawrite_one_bdev, NULL);
++ sync_bdevs(false);
+ printk("Emergency Sync complete\n");
+ kfree(work);
+ }
+diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c
+index 1261e8b41edb4..f2625a372a3ae 100644
+--- a/fs/tracefs/inode.c
++++ b/fs/tracefs/inode.c
+@@ -161,6 +161,77 @@ struct tracefs_fs_info {
+ struct tracefs_mount_opts mount_opts;
+ };
+
++static void change_gid(struct dentry *dentry, kgid_t gid)
++{
++ if (!dentry->d_inode)
++ return;
++ dentry->d_inode->i_gid = gid;
++}
++
++/*
++ * Taken from d_walk, but without he need for handling renames.
++ * Nothing can be renamed while walking the list, as tracefs
++ * does not support renames. This is only called when mounting
++ * or remounting the file system, to set all the files to
++ * the given gid.
++ */
++static void set_gid(struct dentry *parent, kgid_t gid)
++{
++ struct dentry *this_parent;
++ struct list_head *next;
++
++ this_parent = parent;
++ spin_lock(&this_parent->d_lock);
++
++ change_gid(this_parent, gid);
++repeat:
++ next = this_parent->d_subdirs.next;
++resume:
++ while (next != &this_parent->d_subdirs) {
++ struct list_head *tmp = next;
++ struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
++ next = tmp->next;
++
++ spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
++
++ change_gid(dentry, gid);
++
++ if (!list_empty(&dentry->d_subdirs)) {
++ spin_unlock(&this_parent->d_lock);
++ spin_release(&dentry->d_lock.dep_map, _RET_IP_);
++ this_parent = dentry;
++ spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
++ goto repeat;
++ }
++ spin_unlock(&dentry->d_lock);
++ }
++ /*
++ * All done at this level ... ascend and resume the search.
++ */
++ rcu_read_lock();
++ascend:
++ if (this_parent != parent) {
++ struct dentry *child = this_parent;
++ this_parent = child->d_parent;
++
++ spin_unlock(&child->d_lock);
++ spin_lock(&this_parent->d_lock);
++
++ /* go into the first sibling still alive */
++ do {
++ next = child->d_child.next;
++ if (next == &this_parent->d_subdirs)
++ goto ascend;
++ child = list_entry(next, struct dentry, d_child);
++ } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
++ rcu_read_unlock();
++ goto resume;
++ }
++ rcu_read_unlock();
++ spin_unlock(&this_parent->d_lock);
++ return;
++}
++
+ static int tracefs_parse_options(char *data, struct tracefs_mount_opts *opts)
+ {
+ substring_t args[MAX_OPT_ARGS];
+@@ -219,7 +290,9 @@ static int tracefs_apply_options(struct super_block *sb)
+ inode->i_mode |= opts->mode;
+
+ inode->i_uid = opts->uid;
+- inode->i_gid = opts->gid;
++
++ /* Set all the group ids to the mount option */
++ set_gid(sb->s_root, opts->gid);
+
+ return 0;
+ }
+@@ -414,6 +487,8 @@ struct dentry *tracefs_create_file(const char *name, umode_t mode,
+ inode->i_mode = mode;
+ inode->i_fop = fops ? fops : &tracefs_file_operations;
+ inode->i_private = data;
++ inode->i_uid = d_inode(dentry->d_parent)->i_uid;
++ inode->i_gid = d_inode(dentry->d_parent)->i_gid;
+ d_instantiate(dentry, inode);
+ fsnotify_create(dentry->d_parent->d_inode, dentry);
+ return end_creating(dentry);
+@@ -432,9 +507,12 @@ static struct dentry *__create_dir(const char *name, struct dentry *parent,
+ if (unlikely(!inode))
+ return failed_creating(dentry);
+
+- inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO;
++ /* Do not set bits for OTH */
++ inode->i_mode = S_IFDIR | S_IRWXU | S_IRUSR| S_IRGRP | S_IXUSR | S_IXGRP;
+ inode->i_op = ops;
+ inode->i_fop = &simple_dir_operations;
++ inode->i_uid = d_inode(dentry->d_parent)->i_uid;
++ inode->i_gid = d_inode(dentry->d_parent)->i_gid;
+
+ /* directory inodes start off with i_nlink == 2 (for "." entry) */
+ inc_nlink(inode);
+diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
+index 7c61d0ec0159e..79e371bc15e1e 100644
+--- a/fs/ubifs/dir.c
++++ b/fs/ubifs/dir.c
+@@ -349,20 +349,97 @@ out_budg:
+ return err;
+ }
+
+-static int do_tmpfile(struct inode *dir, struct dentry *dentry,
+- umode_t mode, struct inode **whiteout)
++static struct inode *create_whiteout(struct inode *dir, struct dentry *dentry)
++{
++ int err;
++ umode_t mode = S_IFCHR | WHITEOUT_MODE;
++ struct inode *inode;
++ struct ubifs_info *c = dir->i_sb->s_fs_info;
++ struct fscrypt_name nm;
++
++ /*
++ * Create an inode('nlink = 1') for whiteout without updating journal,
++ * let ubifs_jnl_rename() store it on flash to complete rename whiteout
++ * atomically.
++ */
++
++ dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
++ dentry, mode, dir->i_ino);
++
++ err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
++ if (err)
++ return ERR_PTR(err);
++
++ inode = ubifs_new_inode(c, dir, mode);
++ if (IS_ERR(inode)) {
++ err = PTR_ERR(inode);
++ goto out_free;
++ }
++
++ init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);
++ ubifs_assert(c, inode->i_op == &ubifs_file_inode_operations);
++
++ err = ubifs_init_security(dir, inode, &dentry->d_name);
++ if (err)
++ goto out_inode;
++
++ /* The dir size is updated by do_rename. */
++ insert_inode_hash(inode);
++
++ return inode;
++
++out_inode:
++ make_bad_inode(inode);
++ iput(inode);
++out_free:
++ fscrypt_free_filename(&nm);
++ ubifs_err(c, "cannot create whiteout file, error %d", err);
++ return ERR_PTR(err);
++}
++
++/**
++ * lock_2_inodes - a wrapper for locking two UBIFS inodes.
++ * @inode1: first inode
++ * @inode2: second inode
++ *
++ * We do not implement any tricks to guarantee strict lock ordering, because
++ * VFS has already done it for us on the @i_mutex. So this is just a simple
++ * wrapper function.
++ */
++static void lock_2_inodes(struct inode *inode1, struct inode *inode2)
++{
++ mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1);
++ mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2);
++}
++
++/**
++ * unlock_2_inodes - a wrapper for unlocking two UBIFS inodes.
++ * @inode1: first inode
++ * @inode2: second inode
++ */
++static void unlock_2_inodes(struct inode *inode1, struct inode *inode2)
++{
++ mutex_unlock(&ubifs_inode(inode2)->ui_mutex);
++ mutex_unlock(&ubifs_inode(inode1)->ui_mutex);
++}
++
++static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
++ struct dentry *dentry, umode_t mode)
+ {
+ struct inode *inode;
+ struct ubifs_info *c = dir->i_sb->s_fs_info;
+- struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1};
++ struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
++ .dirtied_ino = 1};
+ struct ubifs_budget_req ino_req = { .dirtied_ino = 1 };
+- struct ubifs_inode *ui, *dir_ui = ubifs_inode(dir);
++ struct ubifs_inode *ui;
+ int err, instantiated = 0;
+ struct fscrypt_name nm;
+
+ /*
+- * Budget request settings: new dirty inode, new direntry,
+- * budget for dirtied inode will be released via writeback.
++ * Budget request settings: new inode, new direntry, changing the
++ * parent directory inode.
++ * Allocate budget separately for new dirtied inode, the budget will
++ * be released via writeback.
+ */
+
+ dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
+@@ -392,42 +469,30 @@ static int do_tmpfile(struct inode *dir, struct dentry *dentry,
+ }
+ ui = ubifs_inode(inode);
+
+- if (whiteout) {
+- init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);
+- ubifs_assert(c, inode->i_op == &ubifs_file_inode_operations);
+- }
+-
+ err = ubifs_init_security(dir, inode, &dentry->d_name);
+ if (err)
+ goto out_inode;
+
+ mutex_lock(&ui->ui_mutex);
+ insert_inode_hash(inode);
+-
+- if (whiteout) {
+- mark_inode_dirty(inode);
+- drop_nlink(inode);
+- *whiteout = inode;
+- } else {
+- d_tmpfile(dentry, inode);
+- }
++ d_tmpfile(dentry, inode);
+ ubifs_assert(c, ui->dirty);
+
+ instantiated = 1;
+ mutex_unlock(&ui->ui_mutex);
+
+- mutex_lock(&dir_ui->ui_mutex);
++ lock_2_inodes(dir, inode);
+ err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0);
+ if (err)
+ goto out_cancel;
+- mutex_unlock(&dir_ui->ui_mutex);
++ unlock_2_inodes(dir, inode);
+
+ ubifs_release_budget(c, &req);
+
+ return 0;
+
+ out_cancel:
+- mutex_unlock(&dir_ui->ui_mutex);
++ unlock_2_inodes(dir, inode);
+ out_inode:
+ make_bad_inode(inode);
+ if (!instantiated)
+@@ -441,12 +506,6 @@ out_budg:
+ return err;
+ }
+
+-static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
+- struct dentry *dentry, umode_t mode)
+-{
+- return do_tmpfile(dir, dentry, mode, NULL);
+-}
+-
+ /**
+ * vfs_dent_type - get VFS directory entry type.
+ * @type: UBIFS directory entry type
+@@ -660,32 +719,6 @@ static int ubifs_dir_release(struct inode *dir, struct file *file)
+ return 0;
+ }
+
+-/**
+- * lock_2_inodes - a wrapper for locking two UBIFS inodes.
+- * @inode1: first inode
+- * @inode2: second inode
+- *
+- * We do not implement any tricks to guarantee strict lock ordering, because
+- * VFS has already done it for us on the @i_mutex. So this is just a simple
+- * wrapper function.
+- */
+-static void lock_2_inodes(struct inode *inode1, struct inode *inode2)
+-{
+- mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1);
+- mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2);
+-}
+-
+-/**
+- * unlock_2_inodes - a wrapper for unlocking two UBIFS inodes.
+- * @inode1: first inode
+- * @inode2: second inode
+- */
+-static void unlock_2_inodes(struct inode *inode1, struct inode *inode2)
+-{
+- mutex_unlock(&ubifs_inode(inode2)->ui_mutex);
+- mutex_unlock(&ubifs_inode(inode1)->ui_mutex);
+-}
+-
+ static int ubifs_link(struct dentry *old_dentry, struct inode *dir,
+ struct dentry *dentry)
+ {
+@@ -949,7 +982,8 @@ static int ubifs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
+ struct ubifs_inode *dir_ui = ubifs_inode(dir);
+ struct ubifs_info *c = dir->i_sb->s_fs_info;
+ int err, sz_change;
+- struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1 };
++ struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
++ .dirtied_ino = 1};
+ struct fscrypt_name nm;
+
+ /*
+@@ -1264,17 +1298,19 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
+ .dirtied_ino = 3 };
+ struct ubifs_budget_req ino_req = { .dirtied_ino = 1,
+ .dirtied_ino_d = ALIGN(old_inode_ui->data_len, 8) };
++ struct ubifs_budget_req wht_req;
+ struct timespec64 time;
+ unsigned int saved_nlink;
+ struct fscrypt_name old_nm, new_nm;
+
+ /*
+- * Budget request settings: deletion direntry, new direntry, removing
+- * the old inode, and changing old and new parent directory inodes.
++ * Budget request settings:
++ * req: deletion direntry, new direntry, removing the old inode,
++ * and changing old and new parent directory inodes.
++ *
++ * wht_req: new whiteout inode for RENAME_WHITEOUT.
+ *
+- * However, this operation also marks the target inode as dirty and
+- * does not write it, so we allocate budget for the target inode
+- * separately.
++ * ino_req: marks the target inode as dirty and does not write it.
+ */
+
+ dbg_gen("dent '%pd' ino %lu in dir ino %lu to dent '%pd' in dir ino %lu flags 0x%x",
+@@ -1331,20 +1367,44 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
+ goto out_release;
+ }
+
+- err = do_tmpfile(old_dir, old_dentry, S_IFCHR | WHITEOUT_MODE, &whiteout);
+- if (err) {
++ /*
++ * The whiteout inode without dentry is pinned in memory,
++ * umount won't happen during rename process because we
++ * got parent dentry.
++ */
++ whiteout = create_whiteout(old_dir, old_dentry);
++ if (IS_ERR(whiteout)) {
++ err = PTR_ERR(whiteout);
+ kfree(dev);
+ goto out_release;
+ }
+
+- spin_lock(&whiteout->i_lock);
+- whiteout->i_state |= I_LINKABLE;
+- spin_unlock(&whiteout->i_lock);
+-
+ whiteout_ui = ubifs_inode(whiteout);
+ whiteout_ui->data = dev;
+ whiteout_ui->data_len = ubifs_encode_dev(dev, MKDEV(0, 0));
+ ubifs_assert(c, !whiteout_ui->dirty);
++
++ memset(&wht_req, 0, sizeof(struct ubifs_budget_req));
++ wht_req.new_ino = 1;
++ wht_req.new_ino_d = ALIGN(whiteout_ui->data_len, 8);
++ /*
++ * To avoid deadlock between space budget (holds ui_mutex and
++ * waits wb work) and writeback work(waits ui_mutex), do space
++ * budget before ubifs inodes locked.
++ */
++ err = ubifs_budget_space(c, &wht_req);
++ if (err) {
++ /*
++ * Whiteout inode can not be written on flash by
++ * ubifs_jnl_write_inode(), because it's neither
++ * dirty nor zero-nlink.
++ */
++ iput(whiteout);
++ goto out_release;
++ }
++
++ /* Add the old_dentry size to the old_dir size. */
++ old_sz -= CALC_DENT_SIZE(fname_len(&old_nm));
+ }
+
+ lock_4_inodes(old_dir, new_dir, new_inode, whiteout);
+@@ -1416,29 +1476,11 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
+ sync = IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir);
+ if (unlink && IS_SYNC(new_inode))
+ sync = 1;
+- }
+-
+- if (whiteout) {
+- struct ubifs_budget_req wht_req = { .dirtied_ino = 1,
+- .dirtied_ino_d = \
+- ALIGN(ubifs_inode(whiteout)->data_len, 8) };
+-
+- err = ubifs_budget_space(c, &wht_req);
+- if (err) {
+- kfree(whiteout_ui->data);
+- whiteout_ui->data_len = 0;
+- iput(whiteout);
+- goto out_release;
+- }
+-
+- inc_nlink(whiteout);
+- mark_inode_dirty(whiteout);
+-
+- spin_lock(&whiteout->i_lock);
+- whiteout->i_state &= ~I_LINKABLE;
+- spin_unlock(&whiteout->i_lock);
+-
+- iput(whiteout);
++ /*
++ * S_SYNC flag of whiteout inherits from the old_dir, and we
++ * have already checked the old dir inode. So there is no need
++ * to check whiteout.
++ */
+ }
+
+ err = ubifs_jnl_rename(c, old_dir, old_inode, &old_nm, new_dir,
+@@ -1449,6 +1491,11 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
+ unlock_4_inodes(old_dir, new_dir, new_inode, whiteout);
+ ubifs_release_budget(c, &req);
+
++ if (whiteout) {
++ ubifs_release_budget(c, &wht_req);
++ iput(whiteout);
++ }
++
+ mutex_lock(&old_inode_ui->ui_mutex);
+ release = old_inode_ui->dirty;
+ mark_inode_dirty_sync(old_inode);
+@@ -1457,11 +1504,16 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
+ if (release)
+ ubifs_release_budget(c, &ino_req);
+ if (IS_SYNC(old_inode))
+- err = old_inode->i_sb->s_op->write_inode(old_inode, NULL);
++ /*
++ * Rename finished here. Although old inode cannot be updated
++ * on flash, old ctime is not a big problem, don't return err
++ * code to userspace.
++ */
++ old_inode->i_sb->s_op->write_inode(old_inode, NULL);
+
+ fscrypt_free_filename(&old_nm);
+ fscrypt_free_filename(&new_nm);
+- return err;
++ return 0;
+
+ out_cancel:
+ if (unlink) {
+@@ -1482,11 +1534,11 @@ out_cancel:
+ inc_nlink(old_dir);
+ }
+ }
++ unlock_4_inodes(old_dir, new_dir, new_inode, whiteout);
+ if (whiteout) {
+- drop_nlink(whiteout);
++ ubifs_release_budget(c, &wht_req);
+ iput(whiteout);
+ }
+- unlock_4_inodes(old_dir, new_dir, new_inode, whiteout);
+ out_release:
+ ubifs_release_budget(c, &ino_req);
+ ubifs_release_budget(c, &req);
+diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
+index 5cfa28cd00cdc..6b45a037a0471 100644
+--- a/fs/ubifs/file.c
++++ b/fs/ubifs/file.c
+@@ -570,7 +570,7 @@ static int ubifs_write_end(struct file *file, struct address_space *mapping,
+ }
+
+ if (!PagePrivate(page)) {
+- SetPagePrivate(page);
++ attach_page_private(page, (void *)1);
+ atomic_long_inc(&c->dirty_pg_cnt);
+ __set_page_dirty_nobuffers(page);
+ }
+@@ -947,7 +947,7 @@ static int do_writepage(struct page *page, int len)
+ release_existing_page_budget(c);
+
+ atomic_long_dec(&c->dirty_pg_cnt);
+- ClearPagePrivate(page);
++ detach_page_private(page);
+ ClearPageChecked(page);
+
+ kunmap(page);
+@@ -1304,7 +1304,7 @@ static void ubifs_invalidatepage(struct page *page, unsigned int offset,
+ release_existing_page_budget(c);
+
+ atomic_long_dec(&c->dirty_pg_cnt);
+- ClearPagePrivate(page);
++ detach_page_private(page);
+ ClearPageChecked(page);
+ }
+
+@@ -1471,8 +1471,8 @@ static int ubifs_migrate_page(struct address_space *mapping,
+ return rc;
+
+ if (PagePrivate(page)) {
+- ClearPagePrivate(page);
+- SetPagePrivate(newpage);
++ detach_page_private(page);
++ attach_page_private(newpage, (void *)1);
+ }
+
+ if (mode != MIGRATE_SYNC_NO_COPY)
+@@ -1496,7 +1496,7 @@ static int ubifs_releasepage(struct page *page, gfp_t unused_gfp_flags)
+ return 0;
+ ubifs_assert(c, PagePrivate(page));
+ ubifs_assert(c, 0);
+- ClearPagePrivate(page);
++ detach_page_private(page);
+ ClearPageChecked(page);
+ return 1;
+ }
+@@ -1567,7 +1567,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf)
+ else {
+ if (!PageChecked(page))
+ ubifs_convert_page_budget(c);
+- SetPagePrivate(page);
++ attach_page_private(page, (void *)1);
+ atomic_long_inc(&c->dirty_pg_cnt);
+ __set_page_dirty_nobuffers(page);
+ }
+diff --git a/fs/ubifs/io.c b/fs/ubifs/io.c
+index 00b61dba62b70..b019dd6f7fa06 100644
+--- a/fs/ubifs/io.c
++++ b/fs/ubifs/io.c
+@@ -833,16 +833,42 @@ int ubifs_wbuf_write_nolock(struct ubifs_wbuf *wbuf, void *buf, int len)
+ */
+ n = aligned_len >> c->max_write_shift;
+ if (n) {
+- n <<= c->max_write_shift;
++ int m = n - 1;
++
+ dbg_io("write %d bytes to LEB %d:%d", n, wbuf->lnum,
+ wbuf->offs);
+- err = ubifs_leb_write(c, wbuf->lnum, buf + written,
+- wbuf->offs, n);
++
++ if (m) {
++ /* '(n-1)<<c->max_write_shift < len' is always true. */
++ m <<= c->max_write_shift;
++ err = ubifs_leb_write(c, wbuf->lnum, buf + written,
++ wbuf->offs, m);
++ if (err)
++ goto out;
++ wbuf->offs += m;
++ aligned_len -= m;
++ len -= m;
++ written += m;
++ }
++
++ /*
++ * The non-written len of buf may be less than 'n' because
++ * parameter 'len' is not 8 bytes aligned, so here we read
++ * min(len, n) bytes from buf.
++ */
++ n = 1 << c->max_write_shift;
++ memcpy(wbuf->buf, buf + written, min(len, n));
++ if (n > len) {
++ ubifs_assert(c, n - len < 8);
++ ubifs_pad(c, wbuf->buf + len, n - len);
++ }
++
++ err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, wbuf->offs, n);
+ if (err)
+ goto out;
+ wbuf->offs += n;
+ aligned_len -= n;
+- len -= n;
++ len -= min(len, n);
+ written += n;
+ }
+
+diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c
+index c6a8634877803..71bcebe45f9c5 100644
+--- a/fs/ubifs/ioctl.c
++++ b/fs/ubifs/ioctl.c
+@@ -108,7 +108,7 @@ static int setflags(struct inode *inode, int flags)
+ struct ubifs_inode *ui = ubifs_inode(inode);
+ struct ubifs_info *c = inode->i_sb->s_fs_info;
+ struct ubifs_budget_req req = { .dirtied_ino = 1,
+- .dirtied_ino_d = ui->data_len };
++ .dirtied_ino_d = ALIGN(ui->data_len, 8) };
+
+ err = ubifs_budget_space(c, &req);
+ if (err)
+diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c
+index 8ea680dba61e3..75dab0ae3939d 100644
+--- a/fs/ubifs/journal.c
++++ b/fs/ubifs/journal.c
+@@ -1207,9 +1207,9 @@ out_free:
+ * @sync: non-zero if the write-buffer has to be synchronized
+ *
+ * This function implements the re-name operation which may involve writing up
+- * to 4 inodes and 2 directory entries. It marks the written inodes as clean
+- * and returns zero on success. In case of failure, a negative error code is
+- * returned.
++ * to 4 inodes(new inode, whiteout inode, old and new parent directory inodes)
++ * and 2 directory entries. It marks the written inodes as clean and returns
++ * zero on success. In case of failure, a negative error code is returned.
+ */
+ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ const struct inode *old_inode,
+@@ -1222,14 +1222,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ void *p;
+ union ubifs_key key;
+ struct ubifs_dent_node *dent, *dent2;
+- int err, dlen1, dlen2, ilen, lnum, offs, len, orphan_added = 0;
++ int err, dlen1, dlen2, ilen, wlen, lnum, offs, len, orphan_added = 0;
+ int aligned_dlen1, aligned_dlen2, plen = UBIFS_INO_NODE_SZ;
+ int last_reference = !!(new_inode && new_inode->i_nlink == 0);
+ int move = (old_dir != new_dir);
+- struct ubifs_inode *new_ui;
++ struct ubifs_inode *new_ui, *whiteout_ui;
+ u8 hash_old_dir[UBIFS_HASH_ARR_SZ];
+ u8 hash_new_dir[UBIFS_HASH_ARR_SZ];
+ u8 hash_new_inode[UBIFS_HASH_ARR_SZ];
++ u8 hash_whiteout_inode[UBIFS_HASH_ARR_SZ];
+ u8 hash_dent1[UBIFS_HASH_ARR_SZ];
+ u8 hash_dent2[UBIFS_HASH_ARR_SZ];
+
+@@ -1249,9 +1250,20 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ } else
+ ilen = 0;
+
++ if (whiteout) {
++ whiteout_ui = ubifs_inode(whiteout);
++ ubifs_assert(c, mutex_is_locked(&whiteout_ui->ui_mutex));
++ ubifs_assert(c, whiteout->i_nlink == 1);
++ ubifs_assert(c, !whiteout_ui->dirty);
++ wlen = UBIFS_INO_NODE_SZ;
++ wlen += whiteout_ui->data_len;
++ } else
++ wlen = 0;
++
+ aligned_dlen1 = ALIGN(dlen1, 8);
+ aligned_dlen2 = ALIGN(dlen2, 8);
+- len = aligned_dlen1 + aligned_dlen2 + ALIGN(ilen, 8) + ALIGN(plen, 8);
++ len = aligned_dlen1 + aligned_dlen2 + ALIGN(ilen, 8) +
++ ALIGN(wlen, 8) + ALIGN(plen, 8);
+ if (move)
+ len += plen;
+
+@@ -1313,6 +1325,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ p += ALIGN(ilen, 8);
+ }
+
++ if (whiteout) {
++ pack_inode(c, p, whiteout, 0);
++ err = ubifs_node_calc_hash(c, p, hash_whiteout_inode);
++ if (err)
++ goto out_release;
++
++ p += ALIGN(wlen, 8);
++ }
++
+ if (!move) {
+ pack_inode(c, p, old_dir, 1);
+ err = ubifs_node_calc_hash(c, p, hash_old_dir);
+@@ -1352,6 +1373,9 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ if (new_inode)
+ ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf,
+ new_inode->i_ino);
++ if (whiteout)
++ ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf,
++ whiteout->i_ino);
+ }
+ release_head(c, BASEHD);
+
+@@ -1368,8 +1392,6 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, hash_dent2, old_nm);
+ if (err)
+ goto out_ro;
+-
+- ubifs_delete_orphan(c, whiteout->i_ino);
+ } else {
+ err = ubifs_add_dirt(c, lnum, dlen2);
+ if (err)
+@@ -1390,6 +1412,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ offs += ALIGN(ilen, 8);
+ }
+
++ if (whiteout) {
++ ino_key_init(c, &key, whiteout->i_ino);
++ err = ubifs_tnc_add(c, &key, lnum, offs, wlen,
++ hash_whiteout_inode);
++ if (err)
++ goto out_ro;
++ offs += ALIGN(wlen, 8);
++ }
++
+ ino_key_init(c, &key, old_dir->i_ino);
+ err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_old_dir);
+ if (err)
+@@ -1410,6 +1441,11 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
+ new_ui->synced_i_size = new_ui->ui_size;
+ spin_unlock(&new_ui->ui_lock);
+ }
++ /*
++ * No need to mark whiteout inode clean.
++ * Whiteout doesn't have non-zero size, no need to update
++ * synced_i_size for whiteout_ui.
++ */
+ mark_inode_clean(c, ubifs_inode(old_dir));
+ if (move)
+ mark_inode_clean(c, ubifs_inode(new_dir));
+diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
+index f0fb25727d961..eb05038b71911 100644
+--- a/fs/ubifs/super.c
++++ b/fs/ubifs/super.c
+@@ -1853,7 +1853,6 @@ out:
+ kthread_stop(c->bgt);
+ c->bgt = NULL;
+ }
+- free_wbufs(c);
+ kfree(c->write_reserve_buf);
+ c->write_reserve_buf = NULL;
+ vfree(c->ileb_buf);
+diff --git a/fs/udf/dir.c b/fs/udf/dir.c
+index 70abdfad2df17..42e3e551fa4c3 100644
+--- a/fs/udf/dir.c
++++ b/fs/udf/dir.c
+@@ -31,6 +31,7 @@
+ #include <linux/mm.h>
+ #include <linux/slab.h>
+ #include <linux/bio.h>
++#include <linux/iversion.h>
+
+ #include "udf_i.h"
+ #include "udf_sb.h"
+@@ -43,7 +44,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx)
+ struct fileIdentDesc *fi = NULL;
+ struct fileIdentDesc cfi;
+ udf_pblk_t block, iblock;
+- loff_t nf_pos;
++ loff_t nf_pos, emit_pos = 0;
+ int flen;
+ unsigned char *fname = NULL, *copy_name = NULL;
+ unsigned char *nameptr;
+@@ -57,6 +58,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx)
+ int i, num, ret = 0;
+ struct extent_position epos = { NULL, 0, {0, 0} };
+ struct super_block *sb = dir->i_sb;
++ bool pos_valid = false;
+
+ if (ctx->pos == 0) {
+ if (!dir_emit_dot(file, ctx))
+@@ -67,6 +69,21 @@ static int udf_readdir(struct file *file, struct dir_context *ctx)
+ if (nf_pos >= size)
+ goto out;
+
++ /*
++ * Something changed since last readdir (either lseek was called or dir
++ * changed)? We need to verify the position correctly points at the
++ * beginning of some dir entry so that the directory parsing code does
++ * not get confused. Since UDF does not have any reliable way of
++ * identifying beginning of dir entry (names are under user control),
++ * we need to scan the directory from the beginning.
++ */
++ if (!inode_eq_iversion(dir, file->f_version)) {
++ emit_pos = nf_pos;
++ nf_pos = 0;
++ } else {
++ pos_valid = true;
++ }
++
+ fname = kmalloc(UDF_NAME_LEN, GFP_NOFS);
+ if (!fname) {
+ ret = -ENOMEM;
+@@ -122,13 +139,21 @@ static int udf_readdir(struct file *file, struct dir_context *ctx)
+
+ while (nf_pos < size) {
+ struct kernel_lb_addr tloc;
++ loff_t cur_pos = nf_pos;
+
+- ctx->pos = (nf_pos >> 2) + 1;
++ /* Update file position only if we got past the current one */
++ if (nf_pos >= emit_pos) {
++ ctx->pos = (nf_pos >> 2) + 1;
++ pos_valid = true;
++ }
+
+ fi = udf_fileident_read(dir, &nf_pos, &fibh, &cfi, &epos, &eloc,
+ &elen, &offset);
+ if (!fi)
+ goto out;
++ /* Still not at offset where user asked us to read from? */
++ if (cur_pos < emit_pos)
++ continue;
+
+ liu = le16_to_cpu(cfi.lengthOfImpUse);
+ lfi = cfi.lengthFileIdent;
+@@ -186,8 +211,11 @@ static int udf_readdir(struct file *file, struct dir_context *ctx)
+ } /* end while */
+
+ ctx->pos = (nf_pos >> 2) + 1;
++ pos_valid = true;
+
+ out:
++ if (pos_valid)
++ file->f_version = inode_query_iversion(dir);
+ if (fibh.sbh != fibh.ebh)
+ brelse(fibh.ebh);
+ brelse(fibh.sbh);
+diff --git a/fs/udf/ialloc.c b/fs/udf/ialloc.c
+index 2ecf0e87660e3..b5d611cee749c 100644
+--- a/fs/udf/ialloc.c
++++ b/fs/udf/ialloc.c
+@@ -77,6 +77,7 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode)
+ GFP_KERNEL);
+ }
+ if (!iinfo->i_data) {
++ make_bad_inode(inode);
+ iput(inode);
+ return ERR_PTR(-ENOMEM);
+ }
+@@ -86,6 +87,7 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode)
+ dinfo->i_location.partitionReferenceNum,
+ start, &err);
+ if (err) {
++ make_bad_inode(inode);
+ iput(inode);
+ return ERR_PTR(err);
+ }
+diff --git a/fs/udf/inode.c b/fs/udf/inode.c
+index 1d6b7a50736ba..ea8f6cd01f501 100644
+--- a/fs/udf/inode.c
++++ b/fs/udf/inode.c
+@@ -258,10 +258,6 @@ int udf_expand_file_adinicb(struct inode *inode)
+ char *kaddr;
+ struct udf_inode_info *iinfo = UDF_I(inode);
+ int err;
+- struct writeback_control udf_wbc = {
+- .sync_mode = WB_SYNC_NONE,
+- .nr_to_write = 1,
+- };
+
+ WARN_ON_ONCE(!inode_is_locked(inode));
+ if (!iinfo->i_lenAlloc) {
+@@ -305,8 +301,10 @@ int udf_expand_file_adinicb(struct inode *inode)
+ iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG;
+ /* from now on we have normal address_space methods */
+ inode->i_data.a_ops = &udf_aops;
++ set_page_dirty(page);
++ unlock_page(page);
+ up_write(&iinfo->i_data_sem);
+- err = inode->i_data.a_ops->writepage(page, &udf_wbc);
++ err = filemap_fdatawrite(inode->i_mapping);
+ if (err) {
+ /* Restore everything back so that we don't lose data... */
+ lock_page(page);
+@@ -317,6 +315,7 @@ int udf_expand_file_adinicb(struct inode *inode)
+ unlock_page(page);
+ iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB;
+ inode->i_data.a_ops = &udf_adinicb_aops;
++ iinfo->i_lenAlloc = inode->i_size;
+ up_write(&iinfo->i_data_sem);
+ }
+ put_page(page);
+diff --git a/fs/udf/namei.c b/fs/udf/namei.c
+index caeef08efed23..b3d5f97f16cdb 100644
+--- a/fs/udf/namei.c
++++ b/fs/udf/namei.c
+@@ -30,6 +30,7 @@
+ #include <linux/sched.h>
+ #include <linux/crc-itu-t.h>
+ #include <linux/exportfs.h>
++#include <linux/iversion.h>
+
+ static inline int udf_match(int len1, const unsigned char *name1, int len2,
+ const unsigned char *name2)
+@@ -74,11 +75,11 @@ int udf_write_fi(struct inode *inode, struct fileIdentDesc *cfi,
+
+ if (fileident) {
+ if (adinicb || (offset + lfi < 0)) {
+- memcpy(udf_get_fi_ident(sfi), fileident, lfi);
++ memcpy(sfi->impUse + liu, fileident, lfi);
+ } else if (offset >= 0) {
+ memcpy(fibh->ebh->b_data + offset, fileident, lfi);
+ } else {
+- memcpy(udf_get_fi_ident(sfi), fileident, -offset);
++ memcpy(sfi->impUse + liu, fileident, -offset);
+ memcpy(fibh->ebh->b_data, fileident - offset,
+ lfi + offset);
+ }
+@@ -87,11 +88,11 @@ int udf_write_fi(struct inode *inode, struct fileIdentDesc *cfi,
+ offset += lfi;
+
+ if (adinicb || (offset + padlen < 0)) {
+- memset(udf_get_fi_ident(sfi) + lfi, 0x00, padlen);
++ memset(sfi->impUse + liu + lfi, 0x00, padlen);
+ } else if (offset >= 0) {
+ memset(fibh->ebh->b_data + offset, 0x00, padlen);
+ } else {
+- memset(udf_get_fi_ident(sfi) + lfi, 0x00, -offset);
++ memset(sfi->impUse + liu + lfi, 0x00, -offset);
+ memset(fibh->ebh->b_data, 0x00, padlen + offset);
+ }
+
+@@ -134,6 +135,8 @@ int udf_write_fi(struct inode *inode, struct fileIdentDesc *cfi,
+ mark_buffer_dirty_inode(fibh->ebh, inode);
+ mark_buffer_dirty_inode(fibh->sbh, inode);
+ }
++ inode_inc_iversion(inode);
++
+ return 0;
+ }
+
+diff --git a/fs/udf/super.c b/fs/udf/super.c
+index b2d7c57d06881..aa2f6093d3f6f 100644
+--- a/fs/udf/super.c
++++ b/fs/udf/super.c
+@@ -57,6 +57,7 @@
+ #include <linux/crc-itu-t.h>
+ #include <linux/log2.h>
+ #include <asm/byteorder.h>
++#include <linux/iversion.h>
+
+ #include "udf_sb.h"
+ #include "udf_i.h"
+@@ -149,6 +150,7 @@ static struct inode *udf_alloc_inode(struct super_block *sb)
+ init_rwsem(&ei->i_data_sem);
+ ei->cached_extent.lstart = -1;
+ spin_lock_init(&ei->i_extent_cache_lock);
++ inode_set_iversion(&ei->vfs_inode, 1);
+
+ return &ei->vfs_inode;
+ }
+diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
+index 7aa943edfc02f..240eb932c014b 100644
+--- a/fs/xfs/xfs_file.c
++++ b/fs/xfs/xfs_file.c
+@@ -259,7 +259,7 @@ xfs_file_dio_read(
+ ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
+ if (ret)
+ return ret;
+- ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0);
++ ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0, 0);
+ xfs_iunlock(ip, XFS_IOLOCK_SHARED);
+
+ return ret;
+@@ -569,7 +569,7 @@ xfs_file_dio_write_aligned(
+ }
+ trace_xfs_file_direct_write(iocb, from);
+ ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
+- &xfs_dio_write_ops, 0);
++ &xfs_dio_write_ops, 0, 0);
+ out_unlock:
+ if (iolock)
+ xfs_iunlock(ip, iolock);
+@@ -647,7 +647,7 @@ retry_exclusive:
+
+ trace_xfs_file_direct_write(iocb, from);
+ ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
+- &xfs_dio_write_ops, flags);
++ &xfs_dio_write_ops, flags, 0);
+
+ /*
+ * Retry unaligned I/O with exclusive blocking semantics if the DIO
+diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
+index 0c795dc093efa..09269f478df9c 100644
+--- a/fs/xfs/xfs_ioctl.c
++++ b/fs/xfs/xfs_ioctl.c
+@@ -687,7 +687,8 @@ xfs_ioc_space(
+
+ if (bf->l_start > XFS_ISIZE(ip)) {
+ error = xfs_alloc_file_space(ip, XFS_ISIZE(ip),
+- bf->l_start - XFS_ISIZE(ip), 0);
++ bf->l_start - XFS_ISIZE(ip),
++ XFS_BMAPI_PREALLOC);
+ if (error)
+ goto out_unlock;
+ }
+diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
+index c4e0cd1c1c8ca..170fee98c45c7 100644
+--- a/fs/xfs/xfs_super.c
++++ b/fs/xfs/xfs_super.c
+@@ -729,6 +729,7 @@ xfs_fs_sync_fs(
+ int wait)
+ {
+ struct xfs_mount *mp = XFS_M(sb);
++ int error;
+
+ trace_xfs_fs_sync_fs(mp, __return_address);
+
+@@ -738,7 +739,10 @@ xfs_fs_sync_fs(
+ if (!wait)
+ return 0;
+
+- xfs_log_force(mp, XFS_LOG_SYNC);
++ error = xfs_log_force(mp, XFS_LOG_SYNC);
++ if (error)
++ return error;
++
+ if (laptop_mode) {
+ /*
+ * The disk must be active because we're syncing.
+diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
+index ddc346a9df9ba..b34ccfd71b0ff 100644
+--- a/fs/zonefs/super.c
++++ b/fs/zonefs/super.c
+@@ -35,6 +35,17 @@ static inline int zonefs_zone_mgmt(struct inode *inode,
+
+ lockdep_assert_held(&zi->i_truncate_mutex);
+
++ /*
++ * With ZNS drives, closing an explicitly open zone that has not been
++ * written will change the zone state to "closed", that is, the zone
++ * will remain active. Since this can then cause failure of explicit
++ * open operation on other zones if the drive active zone resources
++ * are exceeded, make sure that the zone does not remain active by
++ * resetting it.
++ */
++ if (op == REQ_OP_ZONE_CLOSE && !zi->i_wpoffset)
++ op = REQ_OP_ZONE_RESET;
++
+ trace_zonefs_zone_mgmt(inode, op);
+ ret = blkdev_zone_mgmt(inode->i_sb->s_bdev, op, zi->i_zsector,
+ zi->i_zone_size >> SECTOR_SHIFT, GFP_NOFS);
+@@ -852,7 +863,7 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from)
+ ret = zonefs_file_dio_append(iocb, from);
+ else
+ ret = iomap_dio_rw(iocb, from, &zonefs_iomap_ops,
+- &zonefs_write_dio_ops, 0);
++ &zonefs_write_dio_ops, 0, 0);
+ if (zi->i_ztype == ZONEFS_ZTYPE_SEQ &&
+ (ret > 0 || ret == -EIOCBQUEUED)) {
+ if (ret > 0)
+@@ -987,7 +998,7 @@ static ssize_t zonefs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+ }
+ file_accessed(iocb->ki_filp);
+ ret = iomap_dio_rw(iocb, to, &zonefs_iomap_ops,
+- &zonefs_read_dio_ops, 0);
++ &zonefs_read_dio_ops, 0, 0);
+ } else {
+ ret = generic_file_read_iter(iocb, to);
+ if (ret == -EIO)
+@@ -1144,6 +1155,7 @@ static struct inode *zonefs_alloc_inode(struct super_block *sb)
+ inode_init_once(&zi->i_vnode);
+ mutex_init(&zi->i_truncate_mutex);
+ zi->i_wr_refcnt = 0;
++ zi->i_flags = 0;
+
+ return &zi->i_vnode;
+ }
+@@ -1295,12 +1307,13 @@ static void zonefs_init_dir_inode(struct inode *parent, struct inode *inode,
+ inc_nlink(parent);
+ }
+
+-static void zonefs_init_file_inode(struct inode *inode, struct blk_zone *zone,
+- enum zonefs_ztype type)
++static int zonefs_init_file_inode(struct inode *inode, struct blk_zone *zone,
++ enum zonefs_ztype type)
+ {
+ struct super_block *sb = inode->i_sb;
+ struct zonefs_sb_info *sbi = ZONEFS_SB(sb);
+ struct zonefs_inode_info *zi = ZONEFS_I(inode);
++ int ret = 0;
+
+ inode->i_ino = zone->start >> sbi->s_zone_sectors_shift;
+ inode->i_mode = S_IFREG | sbi->s_perm;
+@@ -1325,6 +1338,22 @@ static void zonefs_init_file_inode(struct inode *inode, struct blk_zone *zone,
+ sb->s_maxbytes = max(zi->i_max_size, sb->s_maxbytes);
+ sbi->s_blocks += zi->i_max_size >> sb->s_blocksize_bits;
+ sbi->s_used_blocks += zi->i_wpoffset >> sb->s_blocksize_bits;
++
++ /*
++ * For sequential zones, make sure that any open zone is closed first
++ * to ensure that the initial number of open zones is 0, in sync with
++ * the open zone accounting done when the mount option
++ * ZONEFS_MNTOPT_EXPLICIT_OPEN is used.
++ */
++ if (type == ZONEFS_ZTYPE_SEQ &&
++ (zone->cond == BLK_ZONE_COND_IMP_OPEN ||
++ zone->cond == BLK_ZONE_COND_EXP_OPEN)) {
++ mutex_lock(&zi->i_truncate_mutex);
++ ret = zonefs_zone_mgmt(inode, REQ_OP_ZONE_CLOSE);
++ mutex_unlock(&zi->i_truncate_mutex);
++ }
++
++ return ret;
+ }
+
+ static struct dentry *zonefs_create_inode(struct dentry *parent,
+@@ -1334,6 +1363,7 @@ static struct dentry *zonefs_create_inode(struct dentry *parent,
+ struct inode *dir = d_inode(parent);
+ struct dentry *dentry;
+ struct inode *inode;
++ int ret;
+
+ dentry = d_alloc_name(parent, name);
+ if (!dentry)
+@@ -1344,10 +1374,16 @@ static struct dentry *zonefs_create_inode(struct dentry *parent,
+ goto dput;
+
+ inode->i_ctime = inode->i_mtime = inode->i_atime = dir->i_ctime;
+- if (zone)
+- zonefs_init_file_inode(inode, zone, type);
+- else
++ if (zone) {
++ ret = zonefs_init_file_inode(inode, zone, type);
++ if (ret) {
++ iput(inode);
++ goto dput;
++ }
++ } else {
+ zonefs_init_dir_inode(dir, inode, type);
++ }
++
+ d_add(dentry, inode);
+ dir->i_size++;
+
+@@ -1787,5 +1823,6 @@ static void __exit zonefs_exit(void)
+ MODULE_AUTHOR("Damien Le Moal");
+ MODULE_DESCRIPTION("Zone file system for zoned block devices");
+ MODULE_LICENSE("GPL");
++MODULE_ALIAS_FS("zonefs");
+ module_init(zonefs_init);
+ module_exit(zonefs_exit);
+diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
+index 13d93371790ec..e9c7d7b270e73 100644
+--- a/include/acpi/acpi_bus.h
++++ b/include/acpi/acpi_bus.h
+@@ -613,9 +613,10 @@ int acpi_enable_wakeup_device_power(struct acpi_device *dev, int state);
+ int acpi_disable_wakeup_device_power(struct acpi_device *dev);
+
+ #ifdef CONFIG_X86
+-bool acpi_device_always_present(struct acpi_device *adev);
++bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *status);
+ #else
+-static inline bool acpi_device_always_present(struct acpi_device *adev)
++static inline bool acpi_device_override_status(struct acpi_device *adev,
++ unsigned long long *status)
+ {
+ return false;
+ }
+diff --git a/include/acpi/actypes.h b/include/acpi/actypes.h
+index 92c71dfce0d5d..cefbb7ad253e0 100644
+--- a/include/acpi/actypes.h
++++ b/include/acpi/actypes.h
+@@ -536,8 +536,14 @@ typedef u64 acpi_integer;
+ * Can be used with access_width of struct acpi_generic_address and access_size of
+ * struct acpi_resource_generic_register.
+ */
+-#define ACPI_ACCESS_BIT_WIDTH(size) (1 << ((size) + 2))
+-#define ACPI_ACCESS_BYTE_WIDTH(size) (1 << ((size) - 1))
++#define ACPI_ACCESS_BIT_SHIFT 2
++#define ACPI_ACCESS_BYTE_SHIFT -1
++#define ACPI_ACCESS_BIT_MAX (31 - ACPI_ACCESS_BIT_SHIFT)
++#define ACPI_ACCESS_BYTE_MAX (31 - ACPI_ACCESS_BYTE_SHIFT)
++#define ACPI_ACCESS_BIT_DEFAULT (8 - ACPI_ACCESS_BIT_SHIFT)
++#define ACPI_ACCESS_BYTE_DEFAULT (8 - ACPI_ACCESS_BYTE_SHIFT)
++#define ACPI_ACCESS_BIT_WIDTH(size) (1 << ((size) + ACPI_ACCESS_BIT_SHIFT))
++#define ACPI_ACCESS_BYTE_WIDTH(size) (1 << ((size) + ACPI_ACCESS_BYTE_SHIFT))
+
+ /*******************************************************************************
+ *
+diff --git a/include/asm-generic/bitops/find.h b/include/asm-generic/bitops/find.h
+index 0d132ee2a2913..835f959a25f25 100644
+--- a/include/asm-generic/bitops/find.h
++++ b/include/asm-generic/bitops/find.h
+@@ -97,6 +97,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
+
+ #ifdef CONFIG_GENERIC_FIND_FIRST_BIT
+
++#ifndef find_first_bit
+ /**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+@@ -116,7 +117,9 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
+
+ return _find_first_bit(addr, size);
+ }
++#endif
+
++#ifndef find_first_zero_bit
+ /**
+ * find_first_zero_bit - find the first cleared bit in a memory region
+ * @addr: The address to start the search at
+@@ -136,6 +139,8 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
+
+ return _find_first_zero_bit(addr, size);
+ }
++#endif
++
+ #else /* CONFIG_GENERIC_FIND_FIRST_BIT */
+
+ #ifndef find_first_bit
+diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
+index 2c68a545ffa7d..71942a1c642d4 100644
+--- a/include/asm-generic/tlb.h
++++ b/include/asm-generic/tlb.h
+@@ -565,10 +565,14 @@ static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
+ #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
+ do { \
+ unsigned long _sz = huge_page_size(h); \
+- if (_sz == PMD_SIZE) \
+- tlb_flush_pmd_range(tlb, address, _sz); \
+- else if (_sz == PUD_SIZE) \
++ if (_sz >= P4D_SIZE) \
++ tlb_flush_p4d_range(tlb, address, _sz); \
++ else if (_sz >= PUD_SIZE) \
+ tlb_flush_pud_range(tlb, address, _sz); \
++ else if (_sz >= PMD_SIZE) \
++ tlb_flush_pmd_range(tlb, address, _sz); \
++ else \
++ tlb_flush_pte_range(tlb, address, _sz); \
+ __tlb_remove_tlb_entry(tlb, ptep, address); \
+ } while (0)
+
+diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h
+index bc3fb59442ce5..4e30e1799e614 100644
+--- a/include/crypto/blake2s.h
++++ b/include/crypto/blake2s.h
+@@ -101,7 +101,4 @@ static inline void blake2s(u8 *out, const u8 *in, const u8 *key,
+ blake2s_final(&state, out);
+ }
+
+-void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen,
+- const size_t keylen);
+-
+ #endif /* _CRYPTO_BLAKE2S_H */
+diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h
+index dabaee6987186..b3ea73b819443 100644
+--- a/include/crypto/chacha.h
++++ b/include/crypto/chacha.h
+@@ -47,12 +47,19 @@ static inline void hchacha_block(const u32 *state, u32 *out, int nrounds)
+ hchacha_block_generic(state, out, nrounds);
+ }
+
++enum chacha_constants { /* expand 32-byte k */
++ CHACHA_CONSTANT_EXPA = 0x61707865U,
++ CHACHA_CONSTANT_ND_3 = 0x3320646eU,
++ CHACHA_CONSTANT_2_BY = 0x79622d32U,
++ CHACHA_CONSTANT_TE_K = 0x6b206574U
++};
++
+ static inline void chacha_init_consts(u32 *state)
+ {
+- state[0] = 0x61707865; /* "expa" */
+- state[1] = 0x3320646e; /* "nd 3" */
+- state[2] = 0x79622d32; /* "2-by" */
+- state[3] = 0x6b206574; /* "te k" */
++ state[0] = CHACHA_CONSTANT_EXPA;
++ state[1] = CHACHA_CONSTANT_ND_3;
++ state[2] = CHACHA_CONSTANT_2_BY;
++ state[3] = CHACHA_CONSTANT_TE_K;
+ }
+
+ void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv);
+diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
+index c4165126937e4..88e4d145f7cda 100644
+--- a/include/crypto/drbg.h
++++ b/include/crypto/drbg.h
+@@ -136,7 +136,7 @@ struct drbg_state {
+ const struct drbg_state_ops *d_ops;
+ const struct drbg_core *core;
+ struct drbg_string test_data;
+- struct random_ready_callback random_ready;
++ struct notifier_block random_ready;
+ };
+
+ static inline __u8 drbg_statelen(struct drbg_state *drbg)
+diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h
+index 8e50d487500f2..52363eee2b20e 100644
+--- a/include/crypto/internal/blake2s.h
++++ b/include/crypto/internal/blake2s.h
+@@ -11,11 +11,11 @@
+ #include <crypto/internal/hash.h>
+ #include <linux/string.h>
+
+-void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
++void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
+ size_t nblocks, const u32 inc);
+
+-void blake2s_compress_arch(struct blake2s_state *state,const u8 *block,
+- size_t nblocks, const u32 inc);
++void blake2s_compress(struct blake2s_state *state, const u8 *block,
++ size_t nblocks, const u32 inc);
+
+ bool blake2s_selftest(void);
+
+@@ -24,14 +24,11 @@ static inline void blake2s_set_lastblock(struct blake2s_state *state)
+ state->f[0] = -1;
+ }
+
+-typedef void (*blake2s_compress_t)(struct blake2s_state *state,
+- const u8 *block, size_t nblocks, u32 inc);
+-
+ /* Helper functions for BLAKE2s shared by the library and shash APIs */
+
+-static inline void __blake2s_update(struct blake2s_state *state,
+- const u8 *in, size_t inlen,
+- blake2s_compress_t compress)
++static __always_inline void
++__blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen,
++ bool force_generic)
+ {
+ const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
+
+@@ -39,7 +36,12 @@ static inline void __blake2s_update(struct blake2s_state *state,
+ return;
+ if (inlen > fill) {
+ memcpy(state->buf + state->buflen, in, fill);
+- (*compress)(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
++ if (force_generic)
++ blake2s_compress_generic(state, state->buf, 1,
++ BLAKE2S_BLOCK_SIZE);
++ else
++ blake2s_compress(state, state->buf, 1,
++ BLAKE2S_BLOCK_SIZE);
+ state->buflen = 0;
+ in += fill;
+ inlen -= fill;
+@@ -47,7 +49,12 @@ static inline void __blake2s_update(struct blake2s_state *state,
+ if (inlen > BLAKE2S_BLOCK_SIZE) {
+ const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
+ /* Hash one less (full) block than strictly possible */
+- (*compress)(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
++ if (force_generic)
++ blake2s_compress_generic(state, in, nblocks - 1,
++ BLAKE2S_BLOCK_SIZE);
++ else
++ blake2s_compress(state, in, nblocks - 1,
++ BLAKE2S_BLOCK_SIZE);
+ in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
+ inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
+ }
+@@ -55,13 +62,16 @@ static inline void __blake2s_update(struct blake2s_state *state,
+ state->buflen += inlen;
+ }
+
+-static inline void __blake2s_final(struct blake2s_state *state, u8 *out,
+- blake2s_compress_t compress)
++static __always_inline void
++__blake2s_final(struct blake2s_state *state, u8 *out, bool force_generic)
+ {
+ blake2s_set_lastblock(state);
+ memset(state->buf + state->buflen, 0,
+ BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
+- (*compress)(state, state->buf, 1, state->buflen);
++ if (force_generic)
++ blake2s_compress_generic(state, state->buf, 1, state->buflen);
++ else
++ blake2s_compress(state, state->buf, 1, state->buflen);
+ cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
+ memcpy(out, state->h, state->outlen);
+ }
+@@ -99,20 +109,20 @@ static inline int crypto_blake2s_init(struct shash_desc *desc)
+
+ static inline int crypto_blake2s_update(struct shash_desc *desc,
+ const u8 *in, unsigned int inlen,
+- blake2s_compress_t compress)
++ bool force_generic)
+ {
+ struct blake2s_state *state = shash_desc_ctx(desc);
+
+- __blake2s_update(state, in, inlen, compress);
++ __blake2s_update(state, in, inlen, force_generic);
+ return 0;
+ }
+
+ static inline int crypto_blake2s_final(struct shash_desc *desc, u8 *out,
+- blake2s_compress_t compress)
++ bool force_generic)
+ {
+ struct blake2s_state *state = shash_desc_ctx(desc);
+
+- __blake2s_final(state, out, compress);
++ __blake2s_final(state, out, force_generic);
+ return 0;
+ }
+
+diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
+index 1647960c9e506..1f43d7c6724aa 100644
+--- a/include/drm/drm_connector.h
++++ b/include/drm/drm_connector.h
+@@ -566,10 +566,16 @@ struct drm_display_info {
+ bool rgb_quant_range_selectable;
+
+ /**
+- * @edid_hdmi_dc_modes: Mask of supported hdmi deep color modes. Even
+- * more stuff redundant with @bus_formats.
++ * @edid_hdmi_rgb444_dc_modes: Mask of supported hdmi deep color modes
++ * in RGB 4:4:4. Even more stuff redundant with @bus_formats.
+ */
+- u8 edid_hdmi_dc_modes;
++ u8 edid_hdmi_rgb444_dc_modes;
++
++ /**
++ * @edid_hdmi_ycbcr444_dc_modes: Mask of supported hdmi deep color
++ * modes in YCbCr 4:4:4. Even more stuff redundant with @bus_formats.
++ */
++ u8 edid_hdmi_ycbcr444_dc_modes;
+
+ /**
+ * @cea_rev: CEA revision of the HDMI sink.
+diff --git a/include/drm/drm_dp_helper.h b/include/drm/drm_dp_helper.h
+index 1d5b3dbb6e563..dfb46915015b1 100644
+--- a/include/drm/drm_dp_helper.h
++++ b/include/drm/drm_dp_helper.h
+@@ -455,7 +455,7 @@ struct drm_panel;
+ # define DP_FEC_BIT_ERROR_COUNT_CAP (1 << 3)
+
+ /* DP-HDMI2.1 PCON DSC ENCODER SUPPORT */
+-#define DP_PCON_DSC_ENCODER_CAP_SIZE 0xC /* 0x9E - 0x92 */
++#define DP_PCON_DSC_ENCODER_CAP_SIZE 0xD /* 0x92 through 0x9E */
+ #define DP_PCON_DSC_ENCODER 0x092
+ # define DP_PCON_DSC_ENCODER_SUPPORTED (1 << 0)
+ # define DP_PCON_DSC_PPS_ENC_OVERRIDE (1 << 1)
+diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
+index f681bbdbc6982..36f7eb9d06639 100644
+--- a/include/drm/ttm/ttm_bo_api.h
++++ b/include/drm/ttm/ttm_bo_api.h
+@@ -594,8 +594,7 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
+
+ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
+ pgprot_t prot,
+- pgoff_t num_prefault,
+- pgoff_t fault_page_size);
++ pgoff_t num_prefault);
+
+ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf);
+
+diff --git a/include/linux/acpi.h b/include/linux/acpi.h
+index 974d497a897dc..6224b1e32681c 100644
+--- a/include/linux/acpi.h
++++ b/include/linux/acpi.h
+@@ -976,6 +976,15 @@ static inline int acpi_get_local_address(acpi_handle handle, u32 *addr)
+ return -ENODEV;
+ }
+
++static inline int acpi_register_wakeup_handler(int wake_irq,
++ bool (*wakeup)(void *context), void *context)
++{
++ return -ENXIO;
++}
++
++static inline void acpi_unregister_wakeup_handler(
++ bool (*wakeup)(void *context), void *context) { }
++
+ #endif /* !CONFIG_ACPI */
+
+ #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
+diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
+index 63ccb52521902..220c8c60e021a 100644
+--- a/include/linux/arm-smccc.h
++++ b/include/linux/arm-smccc.h
+@@ -92,6 +92,11 @@
+ ARM_SMCCC_SMC_32, \
+ 0, 0x7fff)
+
++#define ARM_SMCCC_ARCH_WORKAROUND_3 \
++ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
++ ARM_SMCCC_SMC_32, \
++ 0, 0x3fff)
++
+ #define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h
+index a3dba31df01e9..6db58d1808665 100644
+--- a/include/linux/atomic/atomic-arch-fallback.h
++++ b/include/linux/atomic/atomic-arch-fallback.h
+@@ -151,7 +151,16 @@
+ static __always_inline int
+ arch_atomic_read_acquire(const atomic_t *v)
+ {
+- return smp_load_acquire(&(v)->counter);
++ int ret;
++
++ if (__native_word(atomic_t)) {
++ ret = smp_load_acquire(&(v)->counter);
++ } else {
++ ret = arch_atomic_read(v);
++ __atomic_acquire_fence();
++ }
++
++ return ret;
+ }
+ #define arch_atomic_read_acquire arch_atomic_read_acquire
+ #endif
+@@ -160,7 +169,12 @@ arch_atomic_read_acquire(const atomic_t *v)
+ static __always_inline void
+ arch_atomic_set_release(atomic_t *v, int i)
+ {
+- smp_store_release(&(v)->counter, i);
++ if (__native_word(atomic_t)) {
++ smp_store_release(&(v)->counter, i);
++ } else {
++ __atomic_release_fence();
++ arch_atomic_set(v, i);
++ }
+ }
+ #define arch_atomic_set_release arch_atomic_set_release
+ #endif
+@@ -1258,7 +1272,16 @@ arch_atomic_dec_if_positive(atomic_t *v)
+ static __always_inline s64
+ arch_atomic64_read_acquire(const atomic64_t *v)
+ {
+- return smp_load_acquire(&(v)->counter);
++ s64 ret;
++
++ if (__native_word(atomic64_t)) {
++ ret = smp_load_acquire(&(v)->counter);
++ } else {
++ ret = arch_atomic64_read(v);
++ __atomic_acquire_fence();
++ }
++
++ return ret;
+ }
+ #define arch_atomic64_read_acquire arch_atomic64_read_acquire
+ #endif
+@@ -1267,7 +1290,12 @@ arch_atomic64_read_acquire(const atomic64_t *v)
+ static __always_inline void
+ arch_atomic64_set_release(atomic64_t *v, s64 i)
+ {
+- smp_store_release(&(v)->counter, i);
++ if (__native_word(atomic64_t)) {
++ smp_store_release(&(v)->counter, i);
++ } else {
++ __atomic_release_fence();
++ arch_atomic64_set(v, i);
++ }
+ }
+ #define arch_atomic64_set_release arch_atomic64_set_release
+ #endif
+@@ -2358,4 +2386,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
+ #endif
+
+ #endif /* _LINUX_ATOMIC_FALLBACK_H */
+-// cca554917d7ea73d5e3e7397dd70c484cad9b2c4
++// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae
+diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
+index 049cf9421d831..f821b72433613 100644
+--- a/include/linux/binfmts.h
++++ b/include/linux/binfmts.h
+@@ -87,6 +87,9 @@ struct coredump_params {
+ loff_t written;
+ loff_t pos;
+ loff_t to_skip;
++ int vma_count;
++ size_t vma_data_size;
++ struct core_vma_metadata *vma_meta;
+ };
+
+ /*
+diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
+index b4de2010fba55..bc5c04d711bbc 100644
+--- a/include/linux/blk-cgroup.h
++++ b/include/linux/blk-cgroup.h
+@@ -24,6 +24,7 @@
+ #include <linux/atomic.h>
+ #include <linux/kthread.h>
+ #include <linux/fs.h>
++#include <linux/blk-mq.h>
+
+ /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
+ #define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
+@@ -604,6 +605,21 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg)
+ atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
+ }
+
++/**
++ * blk_cgroup_mergeable - Determine whether to allow or disallow merges
++ * @rq: request to merge into
++ * @bio: bio to merge
++ *
++ * @bio and @rq should belong to the same cgroup and their issue_as_root should
++ * match. The latter is necessary as we don't want to throttle e.g. a metadata
++ * update because it happens to be next to a regular IO.
++ */
++static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio)
++{
++ return rq->bio->bi_blkg == bio->bi_blkg &&
++ bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);
++}
++
+ void blk_cgroup_bio_start(struct bio *bio);
+ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
+ void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay);
+@@ -659,6 +675,7 @@ static inline void blkg_put(struct blkcg_gq *blkg) { }
+ static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; }
+ static inline void blkcg_bio_issue_init(struct bio *bio) { }
+ static inline void blk_cgroup_bio_start(struct bio *bio) { }
++static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }
+
+ #define blk_queue_for_each_rl(rl, q) \
+ for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
+diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h
+index b80c65aba2493..2580e05a8ab67 100644
+--- a/include/linux/blk-pm.h
++++ b/include/linux/blk-pm.h
+@@ -14,7 +14,7 @@ extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
+ extern int blk_pre_runtime_suspend(struct request_queue *q);
+ extern void blk_post_runtime_suspend(struct request_queue *q, int err);
+ extern void blk_pre_runtime_resume(struct request_queue *q);
+-extern void blk_post_runtime_resume(struct request_queue *q, int err);
++extern void blk_post_runtime_resume(struct request_queue *q);
+ extern void blk_set_runtime_active(struct request_queue *q);
+ #else
+ static inline void blk_pm_runtime_init(struct request_queue *q,
+diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
+index 12b9dbcc980ee..aebe67ed7a73e 100644
+--- a/include/linux/blkdev.h
++++ b/include/linux/blkdev.h
+@@ -235,6 +235,14 @@ struct request {
+ void *end_io_data;
+ };
+
++static inline int blk_validate_block_size(unsigned int bsize)
++{
++ if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
++ return -EINVAL;
++
++ return 0;
++}
++
+ static inline bool blk_op_is_passthrough(unsigned int op)
+ {
+ op &= REQ_OP_MASK;
+@@ -1176,7 +1184,8 @@ extern void blk_dump_rq_flags(struct request *, char *);
+
+ bool __must_check blk_get_queue(struct request_queue *);
+ extern void blk_put_queue(struct request_queue *);
+-extern void blk_set_queue_dying(struct request_queue *);
++
++void blk_mark_disk_dead(struct gendisk *disk);
+
+ #ifdef CONFIG_BLOCK
+ /*
+@@ -1198,8 +1207,6 @@ struct blk_plug {
+ bool multiple_queues;
+ bool nowait;
+ };
+-#define BLK_MAX_REQUEST_COUNT 16
+-#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
+
+ struct blk_plug_cb;
+ typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
+@@ -1941,6 +1948,7 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
+ void disk_end_io_acct(struct gendisk *disk, unsigned int op,
+ unsigned long start_time);
+
++void bio_start_io_acct_time(struct bio *bio, unsigned long start_time);
+ unsigned long bio_start_io_acct(struct bio *bio);
+ void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
+ struct block_device *orig_bdev);
+@@ -1991,6 +1999,8 @@ int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
+ #ifdef CONFIG_BLOCK
+ void invalidate_bdev(struct block_device *bdev);
+ int sync_blockdev(struct block_device *bdev);
++int sync_blockdev_nowait(struct block_device *bdev);
++void sync_bdevs(bool wait);
+ #else
+ static inline void invalidate_bdev(struct block_device *bdev)
+ {
+@@ -1999,6 +2009,13 @@ static inline int sync_blockdev(struct block_device *bdev)
+ {
+ return 0;
+ }
++static inline int sync_blockdev_nowait(struct block_device *bdev)
++{
++ return 0;
++}
++static inline void sync_bdevs(bool wait)
++{
++}
+ #endif
+ int fsync_bdev(struct block_device *bdev);
+
+diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
+index 2746fd8042162..3536ab432b30c 100644
+--- a/include/linux/bpf-cgroup.h
++++ b/include/linux/bpf-cgroup.h
+@@ -517,6 +517,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
+
+ #define cgroup_bpf_enabled(atype) (0)
+ #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, atype, t_ctx) ({ 0; })
++#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, atype) ({ 0; })
+ #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
+ #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
+ #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
+diff --git a/include/linux/bpf.h b/include/linux/bpf.h
+index 3db6f6c95489e..c5c4b6f09e230 100644
+--- a/include/linux/bpf.h
++++ b/include/linux/bpf.h
+@@ -190,7 +190,7 @@ struct bpf_map {
+ atomic64_t usercnt;
+ struct work_struct work;
+ struct mutex freeze_mutex;
+- u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */
++ atomic64_t writecnt;
+ };
+
+ static inline bool map_value_has_spin_lock(const struct bpf_map *map)
+@@ -206,11 +206,9 @@ static inline bool map_value_has_timer(const struct bpf_map *map)
+ static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
+ {
+ if (unlikely(map_value_has_spin_lock(map)))
+- *(struct bpf_spin_lock *)(dst + map->spin_lock_off) =
+- (struct bpf_spin_lock){};
++ memset(dst + map->spin_lock_off, 0, sizeof(struct bpf_spin_lock));
+ if (unlikely(map_value_has_timer(map)))
+- *(struct bpf_timer *)(dst + map->timer_off) =
+- (struct bpf_timer){};
++ memset(dst + map->timer_off, 0, sizeof(struct bpf_timer));
+ }
+
+ /* copy everything but bpf_spin_lock and bpf_timer. There could be one of each. */
+@@ -221,7 +219,8 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
+ if (unlikely(map_value_has_spin_lock(map))) {
+ s_off = map->spin_lock_off;
+ s_sz = sizeof(struct bpf_spin_lock);
+- } else if (unlikely(map_value_has_timer(map))) {
++ }
++ if (unlikely(map_value_has_timer(map))) {
+ t_off = map->timer_off;
+ t_sz = sizeof(struct bpf_timer);
+ }
+@@ -294,6 +293,34 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0,
+
+ extern const struct bpf_map_ops bpf_map_offload_ops;
+
++/* bpf_type_flag contains a set of flags that are applicable to the values of
++ * arg_type, ret_type and reg_type. For example, a pointer value may be null,
++ * or a memory is read-only. We classify types into two categories: base types
++ * and extended types. Extended types are base types combined with a type flag.
++ *
++ * Currently there are no more than 32 base types in arg_type, ret_type and
++ * reg_types.
++ */
++#define BPF_BASE_TYPE_BITS 8
++
++enum bpf_type_flag {
++ /* PTR may be NULL. */
++ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
++
++ /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
++ * compatible with both mutable and immutable memory.
++ */
++ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
++
++ __BPF_TYPE_LAST_FLAG = MEM_RDONLY,
++};
++
++/* Max number of base types. */
++#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
++
++/* Max number of all types. */
++#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
++
+ /* function argument constraints */
+ enum bpf_arg_type {
+ ARG_DONTCARE = 0, /* unused argument in helper function */
+@@ -305,13 +332,11 @@ enum bpf_arg_type {
+ ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
+ ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
+ ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
+- ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
+
+ /* the following constraints used to prototype bpf_memcmp() and other
+ * functions that access data on eBPF program stack
+ */
+ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
+- ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
+ ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
+ * helper function must fill all bytes or clear
+ * them in error case.
+@@ -321,42 +346,65 @@ enum bpf_arg_type {
+ ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
+
+ ARG_PTR_TO_CTX, /* pointer to context */
+- ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */
+ ARG_ANYTHING, /* any (initialized) argument is ok */
+ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
+ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
+ ARG_PTR_TO_INT, /* pointer to int */
+ ARG_PTR_TO_LONG, /* pointer to long */
+ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
+- ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */
+ ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
+ ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */
+- ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
+ ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
+ ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
+ ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
+ ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
+- ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */
++ ARG_PTR_TO_STACK, /* pointer to stack */
+ ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
+ ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
+ __BPF_ARG_TYPE_MAX,
++
++ /* Extended arg_types. */
++ ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
++ ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
++ ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
++ ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
++ ARG_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_ALLOC_MEM,
++ ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
++
++ /* This must be the last entry. Its purpose is to ensure the enum is
++ * wide enough to hold the higher bits reserved for bpf_type_flag.
++ */
++ __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
+ };
++static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
+
+ /* type of values returned from helper functions */
+ enum bpf_return_type {
+ RET_INTEGER, /* function returns integer */
+ RET_VOID, /* function doesn't return anything */
+ RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
+- RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */
+- RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */
+- RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */
+- RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */
+- RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
+- RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
+- RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
++ RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
++ RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
++ RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
++ RET_PTR_TO_ALLOC_MEM, /* returns a pointer to dynamically allocated memory */
+ RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
+ RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
++ __BPF_RET_TYPE_MAX,
++
++ /* Extended ret_types. */
++ RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
++ RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
++ RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
++ RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
++ RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM,
++ RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
++
++ /* This must be the last entry. Its purpose is to ensure the enum is
++ * wide enough to hold the higher bits reserved for bpf_type_flag.
++ */
++ __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
+ };
++static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
+
+ /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
+ * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
+@@ -418,18 +466,15 @@ enum bpf_reg_type {
+ PTR_TO_CTX, /* reg points to bpf_context */
+ CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
+ PTR_TO_MAP_VALUE, /* reg points to map element value */
+- PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */
++ PTR_TO_MAP_KEY, /* reg points to a map element key */
+ PTR_TO_STACK, /* reg == frame_pointer + offset */
+ PTR_TO_PACKET_META, /* skb->data - meta_len */
+ PTR_TO_PACKET, /* reg points to skb->data */
+ PTR_TO_PACKET_END, /* skb->data + headlen */
+ PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
+ PTR_TO_SOCKET, /* reg points to struct bpf_sock */
+- PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */
+ PTR_TO_SOCK_COMMON, /* reg points to sock_common */
+- PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */
+ PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
+- PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
+ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
+ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
+ /* PTR_TO_BTF_ID points to a kernel struct that does not need
+@@ -447,18 +492,25 @@ enum bpf_reg_type {
+ * been checked for null. Used primarily to inform the verifier
+ * an explicit null check is required for this struct.
+ */
+- PTR_TO_BTF_ID_OR_NULL,
+ PTR_TO_MEM, /* reg points to valid memory region */
+- PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */
+- PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */
+- PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
+- PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
+- PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
++ PTR_TO_BUF, /* reg points to a read/write buffer */
+ PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
+ PTR_TO_FUNC, /* reg points to a bpf program function */
+- PTR_TO_MAP_KEY, /* reg points to a map element key */
+ __BPF_REG_TYPE_MAX,
++
++ /* Extended reg_types. */
++ PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
++ PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
++ PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
++ PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
++ PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
++
++ /* This must be the last entry. Its purpose is to ensure the enum is
++ * wide enough to hold the higher bits reserved for bpf_type_flag.
++ */
++ __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
+ };
++static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
+
+ /* The information passed from prog-specific *_is_valid_access
+ * back to the verifier.
+@@ -723,6 +775,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+ struct bpf_trampoline *bpf_trampoline_get(u64 key,
+ struct bpf_attach_target_info *tgt_info);
+ void bpf_trampoline_put(struct bpf_trampoline *tr);
++int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
+ #define BPF_DISPATCHER_INIT(_name) { \
+ .mutex = __MUTEX_INITIALIZER(_name.mutex), \
+ .func = &_name##_func, \
+@@ -1320,28 +1373,16 @@ extern struct mutex bpf_stats_enabled_mutex;
+ * kprobes, tracepoints) to prevent deadlocks on map operations as any of
+ * these events can happen inside a region which holds a map bucket lock
+ * and can deadlock on it.
+- *
+- * Use the preemption safe inc/dec variants on RT because migrate disable
+- * is preemptible on RT and preemption in the middle of the RMW operation
+- * might lead to inconsistent state. Use the raw variants for non RT
+- * kernels as migrate_disable() maps to preempt_disable() so the slightly
+- * more expensive save operation can be avoided.
+ */
+ static inline void bpf_disable_instrumentation(void)
+ {
+ migrate_disable();
+- if (IS_ENABLED(CONFIG_PREEMPT_RT))
+- this_cpu_inc(bpf_prog_active);
+- else
+- __this_cpu_inc(bpf_prog_active);
++ this_cpu_inc(bpf_prog_active);
+ }
+
+ static inline void bpf_enable_instrumentation(void)
+ {
+- if (IS_ENABLED(CONFIG_PREEMPT_RT))
+- this_cpu_dec(bpf_prog_active);
+- else
+- __this_cpu_dec(bpf_prog_active);
++ this_cpu_dec(bpf_prog_active);
+ migrate_enable();
+ }
+
+@@ -1387,6 +1428,7 @@ void bpf_map_put(struct bpf_map *map);
+ void *bpf_map_area_alloc(u64 size, int numa_node);
+ void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
+ void bpf_map_area_free(void *base);
++bool bpf_map_write_active(const struct bpf_map *map);
+ void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
+ int generic_map_lookup_batch(struct bpf_map *map,
+ const union bpf_attr *attr,
+@@ -1677,6 +1719,12 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
+ const struct btf_func_model *
+ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
+ const struct bpf_insn *insn);
++
++static inline bool unprivileged_ebpf_enabled(void)
++{
++ return !sysctl_unprivileged_bpf_disabled;
++}
++
+ #else /* !CONFIG_BPF_SYSCALL */
+ static inline struct bpf_prog *bpf_prog_get(u32 ufd)
+ {
+@@ -1895,6 +1943,12 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
+ {
+ return NULL;
+ }
++
++static inline bool unprivileged_ebpf_enabled(void)
++{
++ return false;
++}
++
+ #endif /* CONFIG_BPF_SYSCALL */
+
+ void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
+diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
+index 5424124dbe365..bb1cc3fbc4bab 100644
+--- a/include/linux/bpf_verifier.h
++++ b/include/linux/bpf_verifier.h
+@@ -18,6 +18,8 @@
+ * that converting umax_value to int cannot overflow.
+ */
+ #define BPF_MAX_VAR_SIZ (1 << 29)
++/* size of type_str_buf in bpf_verifier. */
++#define TYPE_STR_BUF_LEN 64
+
+ /* Liveness marks, used for registers and spilled-regs (in stack slots).
+ * Read marks propagate upwards until they find a write mark; they record that
+@@ -396,6 +398,13 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
+ log->level == BPF_LOG_KERNEL);
+ }
+
++static inline bool
++bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
++{
++ return log->len_total >= 128 && log->len_total <= UINT_MAX >> 2 &&
++ log->level && log->ubuf && !(log->level & ~BPF_LOG_MASK);
++}
++
+ #define BPF_MAX_SUBPROGS 256
+
+ struct bpf_subprog_info {
+@@ -467,6 +476,8 @@ struct bpf_verifier_env {
+ /* longest register parentage chain walked for liveness marking */
+ u32 longest_mark_read_walk;
+ bpfptr_t fd_array;
++ /* buffer used in reg_type_str() to generate reg_type string */
++ char type_str_buf[TYPE_STR_BUF_LEN];
+ };
+
+ __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
+@@ -528,4 +539,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
+ u32 btf_id,
+ struct bpf_attach_target_info *tgt_info);
+
++#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
++
++/* extract base type from bpf_{arg, return, reg}_type. */
++static inline u32 base_type(u32 type)
++{
++ return type & BPF_BASE_TYPE_MASK;
++}
++
++/* extract flags from an extended type. See bpf_type_flag in bpf.h. */
++static inline u32 type_flag(u32 type)
++{
++ return type & ~BPF_BASE_TYPE_MASK;
++}
++
+ #endif /* _LINUX_BPF_VERIFIER_H */
+diff --git a/include/linux/cc_platform.h b/include/linux/cc_platform.h
+new file mode 100644
+index 0000000000000..a075b70b9a70c
+--- /dev/null
++++ b/include/linux/cc_platform.h
+@@ -0,0 +1,88 @@
++/* SPDX-License-Identifier: GPL-2.0-only */
++/*
++ * Confidential Computing Platform Capability checks
++ *
++ * Copyright (C) 2021 Advanced Micro Devices, Inc.
++ *
++ * Author: Tom Lendacky <thomas.lendacky@amd.com>
++ */
++
++#ifndef _LINUX_CC_PLATFORM_H
++#define _LINUX_CC_PLATFORM_H
++
++#include <linux/types.h>
++#include <linux/stddef.h>
++
++/**
++ * enum cc_attr - Confidential computing attributes
++ *
++ * These attributes represent confidential computing features that are
++ * currently active.
++ */
++enum cc_attr {
++ /**
++ * @CC_ATTR_MEM_ENCRYPT: Memory encryption is active
++ *
++ * The platform/OS is running with active memory encryption. This
++ * includes running either as a bare-metal system or a hypervisor
++ * and actively using memory encryption or as a guest/virtual machine
++ * and actively using memory encryption.
++ *
++ * Examples include SME, SEV and SEV-ES.
++ */
++ CC_ATTR_MEM_ENCRYPT,
++
++ /**
++ * @CC_ATTR_HOST_MEM_ENCRYPT: Host memory encryption is active
++ *
++ * The platform/OS is running as a bare-metal system or a hypervisor
++ * and actively using memory encryption.
++ *
++ * Examples include SME.
++ */
++ CC_ATTR_HOST_MEM_ENCRYPT,
++
++ /**
++ * @CC_ATTR_GUEST_MEM_ENCRYPT: Guest memory encryption is active
++ *
++ * The platform/OS is running as a guest/virtual machine and actively
++ * using memory encryption.
++ *
++ * Examples include SEV and SEV-ES.
++ */
++ CC_ATTR_GUEST_MEM_ENCRYPT,
++
++ /**
++ * @CC_ATTR_GUEST_STATE_ENCRYPT: Guest state encryption is active
++ *
++ * The platform/OS is running as a guest/virtual machine and actively
++ * using memory encryption and register state encryption.
++ *
++ * Examples include SEV-ES.
++ */
++ CC_ATTR_GUEST_STATE_ENCRYPT,
++};
++
++#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
++
++/**
++ * cc_platform_has() - Checks if the specified cc_attr attribute is active
++ * @attr: Confidential computing attribute to check
++ *
++ * The cc_platform_has() function will return an indicator as to whether the
++ * specified Confidential Computing attribute is currently active.
++ *
++ * Context: Any context
++ * Return:
++ * * TRUE - Specified Confidential Computing attribute is active
++ * * FALSE - Specified Confidential Computing attribute is not active
++ */
++bool cc_platform_has(enum cc_attr attr);
++
++#else /* !CONFIG_ARCH_HAS_CC_PLATFORM */
++
++static inline bool cc_platform_has(enum cc_attr attr) { return false; }
++
++#endif /* CONFIG_ARCH_HAS_CC_PLATFORM */
++
++#endif /* _LINUX_CC_PLATFORM_H */
+diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
+index 83fa08a065071..787fff5ec7f58 100644
+--- a/include/linux/ceph/osd_client.h
++++ b/include/linux/ceph/osd_client.h
+@@ -287,6 +287,9 @@ struct ceph_osd_linger_request {
+ rados_watcherrcb_t errcb;
+ void *data;
+
++ struct ceph_pagelist *request_pl;
++ struct page **notify_id_pages;
++
+ struct page ***preply_pages;
+ size_t *preply_len;
+ };
+diff --git a/include/linux/compiler.h b/include/linux/compiler.h
+index 3d5af56337bdb..0f7fd205ab7ea 100644
+--- a/include/linux/compiler.h
++++ b/include/linux/compiler.h
+@@ -117,40 +117,29 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
+ */
+ #define __stringify_label(n) #n
+
+-#define __annotate_reachable(c) ({ \
+- asm volatile(__stringify_label(c) ":\n\t" \
+- ".pushsection .discard.reachable\n\t" \
+- ".long " __stringify_label(c) "b - .\n\t" \
+- ".popsection\n\t"); \
+-})
+-#define annotate_reachable() __annotate_reachable(__COUNTER__)
+-
+ #define __annotate_unreachable(c) ({ \
+ asm volatile(__stringify_label(c) ":\n\t" \
+ ".pushsection .discard.unreachable\n\t" \
+ ".long " __stringify_label(c) "b - .\n\t" \
+- ".popsection\n\t"); \
++ ".popsection\n\t" : : "i" (c)); \
+ })
+ #define annotate_unreachable() __annotate_unreachable(__COUNTER__)
+
+-#define ASM_UNREACHABLE \
+- "999:\n\t" \
+- ".pushsection .discard.unreachable\n\t" \
+- ".long 999b - .\n\t" \
++#define ASM_REACHABLE \
++ "998:\n\t" \
++ ".pushsection .discard.reachable\n\t" \
++ ".long 998b - .\n\t" \
+ ".popsection\n\t"
+
+ /* Annotate a C jump table to allow objtool to follow the code flow */
+ #define __annotate_jump_table __section(".rodata..c_jump_table")
+
+ #else
+-#define annotate_reachable()
+ #define annotate_unreachable()
++# define ASM_REACHABLE
+ #define __annotate_jump_table
+ #endif
+
+-#ifndef ASM_UNREACHABLE
+-# define ASM_UNREACHABLE
+-#endif
+ #ifndef unreachable
+ # define unreachable() do { \
+ annotate_unreachable(); \
+diff --git a/include/linux/console.h b/include/linux/console.h
+index 20874db50bc8a..a97f277cfdfa3 100644
+--- a/include/linux/console.h
++++ b/include/linux/console.h
+@@ -149,6 +149,8 @@ struct console {
+ short flags;
+ short index;
+ int cflag;
++ uint ispeed;
++ uint ospeed;
+ void *data;
+ struct console *next;
+ };
+diff --git a/include/linux/coredump.h b/include/linux/coredump.h
+index 78fcd776b185a..4b95e46d215f1 100644
+--- a/include/linux/coredump.h
++++ b/include/linux/coredump.h
+@@ -12,6 +12,8 @@ struct core_vma_metadata {
+ unsigned long start, end;
+ unsigned long flags;
+ unsigned long dump_size;
++ unsigned long pgoff;
++ struct file *file;
+ };
+
+ extern int core_uses_pid;
+@@ -29,9 +31,6 @@ extern int dump_emit(struct coredump_params *cprm, const void *addr, int nr);
+ extern int dump_align(struct coredump_params *cprm, int align);
+ int dump_user_range(struct coredump_params *cprm, unsigned long start,
+ unsigned long len);
+-int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count,
+- struct core_vma_metadata **vma_meta,
+- size_t *vma_data_size_ptr);
+ extern void do_coredump(const kernel_siginfo_t *siginfo);
+ #else
+ static inline void do_coredump(const kernel_siginfo_t *siginfo) {}
+diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
+index ff88bb3e44fca..66a1f495f01a6 100644
+--- a/include/linux/cpufreq.h
++++ b/include/linux/cpufreq.h
+@@ -1041,7 +1041,7 @@ static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_
+ if (cpu == pcpu)
+ continue;
+
+- ret = parse_perf_domain(pcpu, list_name, cell_name);
++ ret = parse_perf_domain(cpu, list_name, cell_name);
+ if (ret < 0)
+ continue;
+
+diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
+index 991911048857a..c88ccc48877d6 100644
+--- a/include/linux/cpuhotplug.h
++++ b/include/linux/cpuhotplug.h
+@@ -99,6 +99,7 @@ enum cpuhp_state {
+ CPUHP_LUSTRE_CFS_DEAD,
+ CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
+ CPUHP_PADATA_DEAD,
++ CPUHP_RANDOM_PREPARE,
+ CPUHP_WORKQUEUE_PREP,
+ CPUHP_POWER_NUMA_PREPARE,
+ CPUHP_HRTIMERS_PREPARE,
+@@ -238,6 +239,7 @@ enum cpuhp_state {
+ CPUHP_AP_PERF_CSKY_ONLINE,
+ CPUHP_AP_WATCHDOG_ONLINE,
+ CPUHP_AP_WORKQUEUE_ONLINE,
++ CPUHP_AP_RANDOM_ONLINE,
+ CPUHP_AP_RCUTREE_ONLINE,
+ CPUHP_AP_BASE_CACHEINFO_ONLINE,
+ CPUHP_AP_ONLINE_DYN,
+diff --git a/include/linux/delay.h b/include/linux/delay.h
+index 1d0e2ce6b6d9f..e8607992c68a5 100644
+--- a/include/linux/delay.h
++++ b/include/linux/delay.h
+@@ -20,6 +20,7 @@
+ */
+
+ #include <linux/kernel.h>
++#include <linux/sched.h>
+
+ extern unsigned long loops_per_jiffy;
+
+@@ -58,7 +59,18 @@ void calibrate_delay(void);
+ void __attribute__((weak)) calibration_delay_done(void);
+ void msleep(unsigned int msecs);
+ unsigned long msleep_interruptible(unsigned int msecs);
+-void usleep_range(unsigned long min, unsigned long max);
++void usleep_range_state(unsigned long min, unsigned long max,
++ unsigned int state);
++
++static inline void usleep_range(unsigned long min, unsigned long max)
++{
++ usleep_range_state(min, max, TASK_UNINTERRUPTIBLE);
++}
++
++static inline void usleep_idle_range(unsigned long min, unsigned long max)
++{
++ usleep_range_state(min, max, TASK_IDLE);
++}
+
+ static inline void ssleep(unsigned int seconds)
+ {
+diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
+index 8b32b4bdd5908..3ad636a13b8e9 100644
+--- a/include/linux/dma-buf.h
++++ b/include/linux/dma-buf.h
+@@ -433,7 +433,7 @@ struct dma_buf {
+ wait_queue_head_t *poll;
+
+ __poll_t active;
+- } cb_excl, cb_shared;
++ } cb_in, cb_out;
+ #ifdef CONFIG_DMABUF_SYSFS_STATS
+ /**
+ * @sysfs_entry:
+diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
+index e5c2c9e71bf10..9000f3ffce8b3 100644
+--- a/include/linux/dmaengine.h
++++ b/include/linux/dmaengine.h
+@@ -944,10 +944,8 @@ struct dma_device {
+ void (*device_issue_pending)(struct dma_chan *chan);
+ void (*device_release)(struct dma_device *dev);
+ /* debugfs support */
+-#ifdef CONFIG_DEBUG_FS
+ void (*dbg_summary_show)(struct seq_file *s, struct dma_device *dev);
+ struct dentry *dbg_dev_root;
+-#endif
+ };
+
+ static inline int dmaengine_slave_config(struct dma_chan *chan,
+diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h
+index 8ae999f587c48..289064b51fa9a 100644
+--- a/include/linux/dsa/ocelot.h
++++ b/include/linux/dsa/ocelot.h
+@@ -12,6 +12,7 @@
+ struct ocelot_skb_cb {
+ struct sk_buff *clone;
+ unsigned int ptp_class; /* valid only for clones */
++ u32 tstamp_lo;
+ u8 ptp_cmd;
+ u8 ts_id;
+ };
+diff --git a/include/linux/efi.h b/include/linux/efi.h
+index 6b5d36babfcc4..3d8ddc5eca8ca 100644
+--- a/include/linux/efi.h
++++ b/include/linux/efi.h
+@@ -1282,4 +1282,10 @@ static inline struct efi_mokvar_table_entry *efi_mokvar_entry_find(
+ }
+ #endif
+
++#ifdef CONFIG_SYSFB
++extern void efifb_setup_from_dmi(struct screen_info *si, const char *opt);
++#else
++static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) { }
++#endif
++
+ #endif /* _LINUX_EFI_H */
+diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
+index c58d504514854..7f28fa702bb72 100644
+--- a/include/linux/etherdevice.h
++++ b/include/linux/etherdevice.h
+@@ -127,7 +127,7 @@ static inline bool is_multicast_ether_addr(const u8 *addr)
+ #endif
+ }
+
+-static inline bool is_multicast_ether_addr_64bits(const u8 addr[6+2])
++static inline bool is_multicast_ether_addr_64bits(const u8 *addr)
+ {
+ #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+ #ifdef __BIG_ENDIAN
+@@ -364,8 +364,7 @@ static inline bool ether_addr_equal(const u8 *addr1, const u8 *addr2)
+ * Please note that alignment of addr1 & addr2 are only guaranteed to be 16 bits.
+ */
+
+-static inline bool ether_addr_equal_64bits(const u8 addr1[6+2],
+- const u8 addr2[6+2])
++static inline bool ether_addr_equal_64bits(const u8 *addr1, const u8 *addr2)
+ {
+ #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
+ u64 fold = (*(const u64 *)addr1) ^ (*(const u64 *)addr2);
+diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h
+index 849524b55d89a..3fad741df53ef 100644
+--- a/include/linux/ethtool.h
++++ b/include/linux/ethtool.h
+@@ -94,7 +94,7 @@ struct ethtool_link_ext_state_info {
+ enum ethtool_link_ext_substate_link_logical_mismatch link_logical_mismatch;
+ enum ethtool_link_ext_substate_bad_signal_integrity bad_signal_integrity;
+ enum ethtool_link_ext_substate_cable_issue cable_issue;
+- u8 __link_ext_substate;
++ u32 __link_ext_substate;
+ };
+ };
+
+diff --git a/include/linux/ethtool_netlink.h b/include/linux/ethtool_netlink.h
+index 1e7bf78cb3829..aba348d58ff61 100644
+--- a/include/linux/ethtool_netlink.h
++++ b/include/linux/ethtool_netlink.h
+@@ -10,6 +10,9 @@
+ #define __ETHTOOL_LINK_MODE_MASK_NWORDS \
+ DIV_ROUND_UP(__ETHTOOL_LINK_MODE_MASK_NBITS, 32)
+
++#define ETHTOOL_PAUSE_STAT_CNT (__ETHTOOL_A_PAUSE_STAT_CNT - \
++ ETHTOOL_A_PAUSE_STAT_TX_FRAMES)
++
+ enum ethtool_multicast_groups {
+ ETHNL_MCGRP_MONITOR,
+ };
+diff --git a/include/linux/fb.h b/include/linux/fb.h
+index 5950f8f5dc74d..3d7306c9a7065 100644
+--- a/include/linux/fb.h
++++ b/include/linux/fb.h
+@@ -502,6 +502,7 @@ struct fb_info {
+ } *apertures;
+
+ bool skip_vt_switch; /* no VT switch on suspend/resume required */
++ bool forced_out; /* set when being removed by another driver */
+ };
+
+ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) {
+@@ -610,6 +611,7 @@ extern int remove_conflicting_pci_framebuffers(struct pci_dev *pdev,
+ const char *name);
+ extern int remove_conflicting_framebuffers(struct apertures_struct *a,
+ const char *name, bool primary);
++extern bool is_firmware_framebuffer(struct apertures_struct *a);
+ extern int fb_prepare_logo(struct fb_info *fb_info, int rotate);
+ extern int fb_show_logo(struct fb_info *fb_info, int rotate);
+ extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size);
+diff --git a/include/linux/filter.h b/include/linux/filter.h
+index ef03ff34234d8..a9956b681f090 100644
+--- a/include/linux/filter.h
++++ b/include/linux/filter.h
+@@ -554,9 +554,9 @@ struct bpf_binary_header {
+ };
+
+ struct bpf_prog_stats {
+- u64 cnt;
+- u64 nsecs;
+- u64 misses;
++ u64_stats_t cnt;
++ u64_stats_t nsecs;
++ u64_stats_t misses;
+ struct u64_stats_sync syncp;
+ } __aligned(2 * sizeof(u64));
+
+@@ -613,13 +613,14 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog,
+ if (static_branch_unlikely(&bpf_stats_enabled_key)) {
+ struct bpf_prog_stats *stats;
+ u64 start = sched_clock();
++ unsigned long flags;
+
+ ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
+ stats = this_cpu_ptr(prog->stats);
+- u64_stats_update_begin(&stats->syncp);
+- stats->cnt++;
+- stats->nsecs += sched_clock() - start;
+- u64_stats_update_end(&stats->syncp);
++ flags = u64_stats_update_begin_irqsave(&stats->syncp);
++ u64_stats_inc(&stats->cnt);
++ u64_stats_add(&stats->nsecs, sched_clock() - start);
++ u64_stats_update_end_irqrestore(&stats->syncp, flags);
+ } else {
+ ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
+ }
+@@ -638,9 +639,6 @@ static __always_inline u32 bpf_prog_run(const struct bpf_prog *prog, const void
+ * This uses migrate_disable/enable() explicitly to document that the
+ * invocation of a BPF program does not require reentrancy protection
+ * against a BPF program which is invoked from a preempting task.
+- *
+- * For non RT enabled kernels migrate_disable/enable() maps to
+- * preempt_disable/enable(), i.e. it disables also preemption.
+ */
+ static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog,
+ const void *ctx)
+diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h
+index c1be37437e778..0c70febd03e95 100644
+--- a/include/linux/fortify-string.h
++++ b/include/linux/fortify-string.h
+@@ -280,7 +280,10 @@ __FORTIFY_INLINE char *strcpy(char *p, const char *q)
+ if (p_size == (size_t)-1 && q_size == (size_t)-1)
+ return __underlying_strcpy(p, q);
+ size = strlen(q) + 1;
+- /* test here to use the more stringent object size */
++ /* Compile-time check for const size overflow. */
++ if (__builtin_constant_p(size) && p_size < size)
++ __write_overflow();
++ /* Run-time check for dynamic size overflow. */
+ if (p_size < size)
+ fortify_panic(__func__);
+ memcpy(p, q, size);
+diff --git a/include/linux/fs.h b/include/linux/fs.h
+index e7a633353fd20..56eba723477e3 100644
+--- a/include/linux/fs.h
++++ b/include/linux/fs.h
+@@ -2498,6 +2498,8 @@ enum file_time_flags {
+
+ extern bool atime_needs_update(const struct path *, struct inode *);
+ extern void touch_atime(const struct path *);
++int inode_update_time(struct inode *inode, struct timespec64 *time, int flags);
++
+ static inline void file_accessed(struct file *file)
+ {
+ if (!(file->f_flags & O_NOATIME))
+diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h
+index 6b54982fc5f37..13fa6f3df8e46 100644
+--- a/include/linux/fs_context.h
++++ b/include/linux/fs_context.h
+@@ -142,6 +142,8 @@ extern void put_fs_context(struct fs_context *fc);
+ extern int vfs_parse_fs_param_source(struct fs_context *fc,
+ struct fs_parameter *param);
+ extern void fc_drop_locked(struct fs_context *fc);
++int reconfigure_single(struct super_block *s,
++ int flags, void *data);
+
+ /*
+ * sget() wrappers to be called from the ->get_tree() op.
+diff --git a/include/linux/fscache.h b/include/linux/fscache.h
+index a4dab59986137..3b2282c157f79 100644
+--- a/include/linux/fscache.h
++++ b/include/linux/fscache.h
+@@ -167,7 +167,7 @@ struct fscache_cookie {
+
+ static inline bool fscache_cookie_enabled(struct fscache_cookie *cookie)
+ {
+- return test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
++ return fscache_cookie_valid(cookie) && test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
+ }
+
+ /*
+diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h
+index 12d3a7d308ab9..a9477c14fad5c 100644
+--- a/include/linux/fsnotify.h
++++ b/include/linux/fsnotify.h
+@@ -212,6 +212,42 @@ static inline void fsnotify_link(struct inode *dir, struct inode *inode,
+ fsnotify_name(dir, FS_CREATE, inode, &new_dentry->d_name, 0);
+ }
+
++/*
++ * fsnotify_delete - @dentry was unlinked and unhashed
++ *
++ * Caller must make sure that dentry->d_name is stable.
++ *
++ * Note: unlike fsnotify_unlink(), we have to pass also the unlinked inode
++ * as this may be called after d_delete() and old_dentry may be negative.
++ */
++static inline void fsnotify_delete(struct inode *dir, struct inode *inode,
++ struct dentry *dentry)
++{
++ __u32 mask = FS_DELETE;
++
++ if (S_ISDIR(inode->i_mode))
++ mask |= FS_ISDIR;
++
++ fsnotify_name(dir, mask, inode, &dentry->d_name, 0);
++}
++
++/**
++ * d_delete_notify - delete a dentry and call fsnotify_delete()
++ * @dentry: The dentry to delete
++ *
++ * This helper is used to guaranty that the unlinked inode cannot be found
++ * by lookup of this name after fsnotify_delete() event has been delivered.
++ */
++static inline void d_delete_notify(struct inode *dir, struct dentry *dentry)
++{
++ struct inode *inode = d_inode(dentry);
++
++ ihold(inode);
++ d_delete(dentry);
++ fsnotify_delete(dir, inode, dentry);
++ iput(inode);
++}
++
+ /*
+ * fsnotify_unlink - 'name' was unlinked
+ *
+@@ -219,10 +255,10 @@ static inline void fsnotify_link(struct inode *dir, struct inode *inode,
+ */
+ static inline void fsnotify_unlink(struct inode *dir, struct dentry *dentry)
+ {
+- /* Expected to be called before d_delete() */
+- WARN_ON_ONCE(d_is_negative(dentry));
++ if (WARN_ON_ONCE(d_is_negative(dentry)))
++ return;
+
+- fsnotify_dirent(dir, dentry, FS_DELETE);
++ fsnotify_delete(dir, d_inode(dentry), dentry);
+ }
+
+ /*
+@@ -242,10 +278,10 @@ static inline void fsnotify_mkdir(struct inode *inode, struct dentry *dentry)
+ */
+ static inline void fsnotify_rmdir(struct inode *dir, struct dentry *dentry)
+ {
+- /* Expected to be called before d_delete() */
+- WARN_ON_ONCE(d_is_negative(dentry));
++ if (WARN_ON_ONCE(d_is_negative(dentry)))
++ return;
+
+- fsnotify_dirent(dir, dentry, FS_DELETE | FS_ISDIR);
++ fsnotify_delete(dir, d_inode(dentry), dentry);
+ }
+
+ /*
+diff --git a/include/linux/genhd.h b/include/linux/genhd.h
+index 0f5315c2b5a34..0b48a0cf42624 100644
+--- a/include/linux/genhd.h
++++ b/include/linux/genhd.h
+@@ -12,12 +12,10 @@
+
+ #include <linux/types.h>
+ #include <linux/kdev_t.h>
+-#include <linux/rcupdate.h>
+-#include <linux/slab.h>
+-#include <linux/percpu-refcount.h>
+ #include <linux/uuid.h>
+ #include <linux/blk_types.h>
+-#include <asm/local.h>
++#include <linux/device.h>
++#include <linux/xarray.h>
+
+ extern const struct device_type disk_type;
+ extern struct device_type part_type;
+@@ -26,14 +24,6 @@ extern struct class block_class;
+ #define DISK_MAX_PARTS 256
+ #define DISK_NAME_LEN 32
+
+-#include <linux/major.h>
+-#include <linux/device.h>
+-#include <linux/smp.h>
+-#include <linux/string.h>
+-#include <linux/fs.h>
+-#include <linux/workqueue.h>
+-#include <linux/xarray.h>
+-
+ #define PARTITION_META_INFO_VOLNAMELTH 64
+ /*
+ * Enough for the string representation of any kind of UUID plus NULL.
+diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
+index a0f9901dcae60..e3c29d2e68263 100644
+--- a/include/linux/gpio/driver.h
++++ b/include/linux/gpio/driver.h
+@@ -224,6 +224,15 @@ struct gpio_irq_chip {
+ unsigned long *valid_mask,
+ unsigned int ngpios);
+
++ /**
++ * @initialized:
++ *
++ * Flag to track GPIO chip irq member's initialization.
++ * This flag will make sure GPIO chip irq members are not used
++ * before they are initialized.
++ */
++ bool initialized;
++
+ /**
+ * @valid_mask:
+ *
+diff --git a/include/linux/hid.h b/include/linux/hid.h
+index 9e067f937dbc2..26742ca14609a 100644
+--- a/include/linux/hid.h
++++ b/include/linux/hid.h
+@@ -349,6 +349,8 @@ struct hid_item {
+ /* BIT(9) reserved for backward compatibility, was NO_INIT_INPUT_REPORTS */
+ #define HID_QUIRK_ALWAYS_POLL BIT(10)
+ #define HID_QUIRK_INPUT_PER_APP BIT(11)
++#define HID_QUIRK_X_INVERT BIT(12)
++#define HID_QUIRK_Y_INVERT BIT(13)
+ #define HID_QUIRK_SKIP_OUTPUT_REPORTS BIT(16)
+ #define HID_QUIRK_SKIP_OUTPUT_REPORT_ID BIT(17)
+ #define HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP BIT(18)
+@@ -840,6 +842,11 @@ static inline bool hid_is_using_ll_driver(struct hid_device *hdev,
+ return hdev->ll_driver == driver;
+ }
+
++static inline bool hid_is_usb(struct hid_device *hdev)
++{
++ return hid_is_using_ll_driver(hdev, &usb_hid_driver);
++}
++
+ #define PM_HINT_FULLON 1<<5
+ #define PM_HINT_NORMAL 1<<1
+
+diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
+index 8e6dd908da216..aa1d4da03538b 100644
+--- a/include/linux/hw_random.h
++++ b/include/linux/hw_random.h
+@@ -60,7 +60,5 @@ extern int devm_hwrng_register(struct device *dev, struct hwrng *rng);
+ /** Unregister a Hardware Random Number Generator driver. */
+ extern void hwrng_unregister(struct hwrng *rng);
+ extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng);
+-/** Feed random bits into the pool. */
+-extern void add_hwgenerator_randomness(const char *buffer, size_t count, size_t entropy);
+
+ #endif /* LINUX_HWRANDOM_H_ */
+diff --git a/include/linux/if_arp.h b/include/linux/if_arp.h
+index b712217f70304..1ed52441972f9 100644
+--- a/include/linux/if_arp.h
++++ b/include/linux/if_arp.h
+@@ -52,6 +52,7 @@ static inline bool dev_is_mac_header_xmit(const struct net_device *dev)
+ case ARPHRD_VOID:
+ case ARPHRD_NONE:
+ case ARPHRD_RAWIP:
++ case ARPHRD_PIMREG:
+ return false;
+ default:
+ return true;
+diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h
+index 096f68dd2e0ca..4c69b144677b1 100644
+--- a/include/linux/iio/trigger.h
++++ b/include/linux/iio/trigger.h
+@@ -55,6 +55,7 @@ struct iio_trigger_ops {
+ * @attached_own_device:[INTERN] if we are using our own device as trigger,
+ * i.e. if we registered a poll function to the same
+ * device as the one providing the trigger.
++ * @reenable_work: [INTERN] work item used to ensure reenable can sleep.
+ **/
+ struct iio_trigger {
+ const struct iio_trigger_ops *ops;
+@@ -74,6 +75,7 @@ struct iio_trigger {
+ unsigned long pool[BITS_TO_LONGS(CONFIG_IIO_CONSUMERS_PER_TRIGGER)];
+ struct mutex pool_lock;
+ bool attached_own_device;
++ struct work_struct reenable_work;
+ };
+
+
+diff --git a/include/linux/instrumentation.h b/include/linux/instrumentation.h
+index fa2cd8c63dcc9..24359b4a96053 100644
+--- a/include/linux/instrumentation.h
++++ b/include/linux/instrumentation.h
+@@ -11,7 +11,7 @@
+ asm volatile(__stringify(c) ": nop\n\t" \
+ ".pushsection .discard.instr_begin\n\t" \
+ ".long " __stringify(c) "b - .\n\t" \
+- ".popsection\n\t"); \
++ ".popsection\n\t" : : "i" (c)); \
+ })
+ #define instrumentation_begin() __instrumentation_begin(__COUNTER__)
+
+@@ -50,7 +50,7 @@
+ asm volatile(__stringify(c) ": nop\n\t" \
+ ".pushsection .discard.instr_end\n\t" \
+ ".long " __stringify(c) "b - .\n\t" \
+- ".popsection\n\t"); \
++ ".popsection\n\t" : : "i" (c)); \
+ })
+ #define instrumentation_end() __instrumentation_end(__COUNTER__)
+ #else
+diff --git a/include/linux/iomap.h b/include/linux/iomap.h
+index 24f8489583ca7..829f2325ecbab 100644
+--- a/include/linux/iomap.h
++++ b/include/linux/iomap.h
+@@ -330,12 +330,19 @@ struct iomap_dio_ops {
+ */
+ #define IOMAP_DIO_OVERWRITE_ONLY (1 << 1)
+
++/*
++ * When a page fault occurs, return a partial synchronous result and allow
++ * the caller to retry the rest of the operation after dealing with the page
++ * fault.
++ */
++#define IOMAP_DIO_PARTIAL (1 << 2)
++
+ ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+- unsigned int dio_flags);
++ unsigned int dio_flags, size_t done_before);
+ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+- unsigned int dio_flags);
++ unsigned int dio_flags, size_t done_before);
+ ssize_t iomap_dio_complete(struct iomap_dio *dio);
+ int iomap_dio_iopoll(struct kiocb *kiocb, bool spin);
+
+diff --git a/include/linux/ioport.h b/include/linux/ioport.h
+index 8359c50f99884..ec5f71f7135b0 100644
+--- a/include/linux/ioport.h
++++ b/include/linux/ioport.h
+@@ -262,6 +262,8 @@ resource_union(struct resource *r1, struct resource *r2, struct resource *r)
+ #define request_muxed_region(start,n,name) __request_region(&ioport_resource, (start), (n), (name), IORESOURCE_MUXED)
+ #define __request_mem_region(start,n,name, excl) __request_region(&iomem_resource, (start), (n), (name), excl)
+ #define request_mem_region(start,n,name) __request_region(&iomem_resource, (start), (n), (name), 0)
++#define request_mem_region_muxed(start, n, name) \
++ __request_region(&iomem_resource, (start), (n), (name), IORESOURCE_MUXED)
+ #define request_mem_region_exclusive(start,n,name) \
+ __request_region(&iomem_resource, (start), (n), (name), IORESOURCE_EXCLUSIVE)
+ #define rename_region(region, newname) do { (region)->name = (newname); } while (0)
+diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h
+index 05e22770af517..b75395ec8d521 100644
+--- a/include/linux/ipc_namespace.h
++++ b/include/linux/ipc_namespace.h
+@@ -131,6 +131,16 @@ static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
+ return ns;
+ }
+
++static inline struct ipc_namespace *get_ipc_ns_not_zero(struct ipc_namespace *ns)
++{
++ if (ns) {
++ if (refcount_inc_not_zero(&ns->ns.count))
++ return ns;
++ }
++
++ return NULL;
++}
++
+ extern void put_ipc_ns(struct ipc_namespace *ns);
+ #else
+ static inline struct ipc_namespace *copy_ipcs(unsigned long flags,
+@@ -147,6 +157,11 @@ static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
+ return ns;
+ }
+
++static inline struct ipc_namespace *get_ipc_ns_not_zero(struct ipc_namespace *ns)
++{
++ return ns;
++}
++
+ static inline void put_ipc_ns(struct ipc_namespace *ns)
+ {
+ }
+diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
+index ef4a69865737c..d1f3864307959 100644
+--- a/include/linux/ipv6.h
++++ b/include/linux/ipv6.h
+@@ -51,7 +51,7 @@ struct ipv6_devconf {
+ __s32 use_optimistic;
+ #endif
+ #ifdef CONFIG_IPV6_MROUTE
+- __s32 mc_forwarding;
++ atomic_t mc_forwarding;
+ #endif
+ __s32 disable_ipv6;
+ __s32 drop_unicast_in_l2_multicast;
+@@ -132,6 +132,7 @@ struct inet6_skb_parm {
+ __u16 dsthao;
+ #endif
+ __u16 frag_max_size;
++ __u16 srhoff;
+
+ #define IP6SKB_XFRM_TRANSFORMED 1
+ #define IP6SKB_FORWARDED 2
+@@ -141,6 +142,7 @@ struct inet6_skb_parm {
+ #define IP6SKB_HOPBYHOP 32
+ #define IP6SKB_L3SLAVE 64
+ #define IP6SKB_JUMBOGRAM 128
++#define IP6SKB_SEG6 256
+ };
+
+ #if defined(CONFIG_NET_L3_MASTER_DEV)
+@@ -282,7 +284,6 @@ struct ipv6_pinfo {
+ __be32 rcv_flowinfo;
+
+ __u32 dst_cookie;
+- __u32 rx_dst_cookie;
+
+ struct ipv6_mc_socklist __rcu *ipv6_mc_list;
+ struct ipv6_ac_socklist *ipv6_ac_list;
+diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h
+index fd933c45281af..d63b8106796e2 100644
+--- a/include/linux/jbd2.h
++++ b/include/linux/jbd2.h
+@@ -1295,7 +1295,7 @@ struct journal_s
+ * Clean-up after fast commit or full commit. JBD2 calls this function
+ * after every commit operation.
+ */
+- void (*j_fc_cleanup_callback)(struct journal_s *journal, int);
++ void (*j_fc_cleanup_callback)(struct journal_s *journal, int full, tid_t tid);
+
+ /**
+ * @j_fc_replay_callback:
+diff --git a/include/linux/kasan.h b/include/linux/kasan.h
+index dd874a1ee862a..f407e937241af 100644
+--- a/include/linux/kasan.h
++++ b/include/linux/kasan.h
+@@ -461,12 +461,12 @@ static inline void kasan_release_vmalloc(unsigned long start,
+ * allocations with real shadow memory. With KASAN vmalloc, the special
+ * case is unnecessary, as the work is handled in the generic case.
+ */
+-int kasan_module_alloc(void *addr, size_t size);
++int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask);
+ void kasan_free_shadow(const struct vm_struct *vm);
+
+ #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
+
+-static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
++static inline int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
+ static inline void kasan_free_shadow(const struct vm_struct *vm) {}
+
+ #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
+diff --git a/include/linux/kernel.h b/include/linux/kernel.h
+index 2776423a587e4..f56cd8879a594 100644
+--- a/include/linux/kernel.h
++++ b/include/linux/kernel.h
+@@ -277,7 +277,7 @@ static inline char *hex_byte_pack_upper(char *buf, u8 byte)
+ return buf;
+ }
+
+-extern int hex_to_bin(char ch);
++extern int hex_to_bin(unsigned char ch);
+ extern int __must_check hex2bin(u8 *dst, const char *src, size_t count);
+ extern char *bin2hex(char *dst, const void *src, size_t count);
+
+diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
+index 44ae1a7eb9e39..69ae6b2784645 100644
+--- a/include/linux/kernel_stat.h
++++ b/include/linux/kernel_stat.h
+@@ -102,6 +102,7 @@ extern void account_system_index_time(struct task_struct *, u64,
+ enum cpu_usage_stat);
+ extern void account_steal_time(u64);
+ extern void account_idle_time(u64);
++extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu);
+
+ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+ static inline void account_process_tick(struct task_struct *tsk, int user)
+diff --git a/include/linux/kfence.h b/include/linux/kfence.h
+index 3fe6dd8a18c19..3c75209a545e1 100644
+--- a/include/linux/kfence.h
++++ b/include/linux/kfence.h
+@@ -14,6 +14,9 @@
+
+ #ifdef CONFIG_KFENCE
+
++#include <linux/atomic.h>
++#include <linux/static_key.h>
++
+ /*
+ * We allocate an even number of pages, as it simplifies calculations to map
+ * address to metadata indices; effectively, the very first page serves as an
+@@ -22,13 +25,8 @@
+ #define KFENCE_POOL_SIZE ((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 * PAGE_SIZE)
+ extern char *__kfence_pool;
+
+-#ifdef CONFIG_KFENCE_STATIC_KEYS
+-#include <linux/static_key.h>
+ DECLARE_STATIC_KEY_FALSE(kfence_allocation_key);
+-#else
+-#include <linux/atomic.h>
+ extern atomic_t kfence_allocation_gate;
+-#endif
+
+ /**
+ * is_kfence_address() - check if an address belongs to KFENCE pool
+@@ -116,13 +114,16 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags);
+ */
+ static __always_inline void *kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
+ {
+-#ifdef CONFIG_KFENCE_STATIC_KEYS
+- if (static_branch_unlikely(&kfence_allocation_key))
++#if defined(CONFIG_KFENCE_STATIC_KEYS) || CONFIG_KFENCE_SAMPLE_INTERVAL == 0
++ if (!static_branch_unlikely(&kfence_allocation_key))
++ return NULL;
+ #else
+- if (unlikely(!atomic_read(&kfence_allocation_gate)))
++ if (!static_branch_likely(&kfence_allocation_key))
++ return NULL;
+ #endif
+- return __kfence_alloc(s, size, flags);
+- return NULL;
++ if (likely(atomic_read(&kfence_allocation_gate)))
++ return NULL;
++ return __kfence_alloc(s, size, flags);
+ }
+
+ /**
+@@ -201,6 +202,22 @@ static __always_inline __must_check bool kfence_free(void *addr)
+ */
+ bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs);
+
++#ifdef CONFIG_PRINTK
++struct kmem_obj_info;
++/**
++ * __kfence_obj_info() - fill kmem_obj_info struct
++ * @kpp: kmem_obj_info to be filled
++ * @object: the object
++ *
++ * Return:
++ * * false - not a KFENCE object
++ * * true - a KFENCE object, filled @kpp
++ *
++ * Copies information to @kpp for KFENCE objects.
++ */
++bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page);
++#endif
++
+ #else /* CONFIG_KFENCE */
+
+ static inline bool is_kfence_address(const void *addr) { return false; }
+@@ -218,6 +235,14 @@ static inline bool __must_check kfence_handle_page_fault(unsigned long addr, boo
+ return false;
+ }
+
++#ifdef CONFIG_PRINTK
++struct kmem_obj_info;
++static inline bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++{
++ return false;
++}
++#endif
++
+ #endif
+
+ #endif /* _LINUX_KFENCE_H */
+diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
+index e4f3bfe087570..ef8c7accbc689 100644
+--- a/include/linux/kprobes.h
++++ b/include/linux/kprobes.h
+@@ -154,6 +154,8 @@ struct kretprobe {
+ struct kretprobe_holder *rph;
+ };
+
++#define KRETPROBE_MAX_DATA_SIZE 4096
++
+ struct kretprobe_instance {
+ union {
+ struct freelist_node freelist;
+diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
+index 0f18df7fe8749..926f60499c347 100644
+--- a/include/linux/kvm_host.h
++++ b/include/linux/kvm_host.h
+@@ -15,6 +15,8 @@
+ #include <linux/minmax.h>
+ #include <linux/mm.h>
+ #include <linux/mmu_notifier.h>
++#include <linux/ftrace.h>
++#include <linux/instrumentation.h>
+ #include <linux/preempt.h>
+ #include <linux/msi.h>
+ #include <linux/slab.h>
+@@ -363,8 +365,11 @@ struct kvm_vcpu {
+ int last_used_slot;
+ };
+
+-/* must be called with irqs disabled */
+-static __always_inline void guest_enter_irqoff(void)
++/*
++ * Start accounting time towards a guest.
++ * Must be called before entering guest context.
++ */
++static __always_inline void guest_timing_enter_irqoff(void)
+ {
+ /*
+ * This is running in ioctl context so its safe to assume that it's the
+@@ -373,7 +378,18 @@ static __always_inline void guest_enter_irqoff(void)
+ instrumentation_begin();
+ vtime_account_guest_enter();
+ instrumentation_end();
++}
+
++/*
++ * Enter guest context and enter an RCU extended quiescent state.
++ *
++ * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is
++ * unsafe to use any code which may directly or indirectly use RCU, tracing
++ * (including IRQ flag tracing), or lockdep. All code in this period must be
++ * non-instrumentable.
++ */
++static __always_inline void guest_context_enter_irqoff(void)
++{
+ /*
+ * KVM does not hold any references to rcu protected data when it
+ * switches CPU into a guest mode. In fact switching to a guest mode
+@@ -389,16 +405,79 @@ static __always_inline void guest_enter_irqoff(void)
+ }
+ }
+
+-static __always_inline void guest_exit_irqoff(void)
++/*
++ * Deprecated. Architectures should move to guest_timing_enter_irqoff() and
++ * guest_state_enter_irqoff().
++ */
++static __always_inline void guest_enter_irqoff(void)
++{
++ guest_timing_enter_irqoff();
++ guest_context_enter_irqoff();
++}
++
++/**
++ * guest_state_enter_irqoff - Fixup state when entering a guest
++ *
++ * Entry to a guest will enable interrupts, but the kernel state is interrupts
++ * disabled when this is invoked. Also tell RCU about it.
++ *
++ * 1) Trace interrupts on state
++ * 2) Invoke context tracking if enabled to adjust RCU state
++ * 3) Tell lockdep that interrupts are enabled
++ *
++ * Invoked from architecture specific code before entering a guest.
++ * Must be called with interrupts disabled and the caller must be
++ * non-instrumentable.
++ * The caller has to invoke guest_timing_enter_irqoff() before this.
++ *
++ * Note: this is analogous to exit_to_user_mode().
++ */
++static __always_inline void guest_state_enter_irqoff(void)
++{
++ instrumentation_begin();
++ trace_hardirqs_on_prepare();
++ lockdep_hardirqs_on_prepare(CALLER_ADDR0);
++ instrumentation_end();
++
++ guest_context_enter_irqoff();
++ lockdep_hardirqs_on(CALLER_ADDR0);
++}
++
++/*
++ * Exit guest context and exit an RCU extended quiescent state.
++ *
++ * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is
++ * unsafe to use any code which may directly or indirectly use RCU, tracing
++ * (including IRQ flag tracing), or lockdep. All code in this period must be
++ * non-instrumentable.
++ */
++static __always_inline void guest_context_exit_irqoff(void)
+ {
+ context_tracking_guest_exit();
++}
+
++/*
++ * Stop accounting time towards a guest.
++ * Must be called after exiting guest context.
++ */
++static __always_inline void guest_timing_exit_irqoff(void)
++{
+ instrumentation_begin();
+ /* Flush the guest cputime we spent on the guest */
+ vtime_account_guest_exit();
+ instrumentation_end();
+ }
+
++/*
++ * Deprecated. Architectures should move to guest_state_exit_irqoff() and
++ * guest_timing_exit_irqoff().
++ */
++static __always_inline void guest_exit_irqoff(void)
++{
++ guest_context_exit_irqoff();
++ guest_timing_exit_irqoff();
++}
++
+ static inline void guest_exit(void)
+ {
+ unsigned long flags;
+@@ -408,6 +487,33 @@ static inline void guest_exit(void)
+ local_irq_restore(flags);
+ }
+
++/**
++ * guest_state_exit_irqoff - Establish state when returning from guest mode
++ *
++ * Entry from a guest disables interrupts, but guest mode is traced as
++ * interrupts enabled. Also with NO_HZ_FULL RCU might be idle.
++ *
++ * 1) Tell lockdep that interrupts are disabled
++ * 2) Invoke context tracking if enabled to reactivate RCU
++ * 3) Trace interrupts off state
++ *
++ * Invoked from architecture specific code after exiting a guest.
++ * Must be invoked with interrupts disabled and the caller must be
++ * non-instrumentable.
++ * The caller has to invoke guest_timing_exit_irqoff() after this.
++ *
++ * Note: this is analogous to enter_from_user_mode().
++ */
++static __always_inline void guest_state_exit_irqoff(void)
++{
++ lockdep_hardirqs_off(CALLER_ADDR0);
++ guest_context_exit_irqoff();
++
++ instrumentation_begin();
++ trace_hardirqs_off_finish();
++ instrumentation_end();
++}
++
+ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
+ {
+ /*
+diff --git a/include/linux/libata.h b/include/linux/libata.h
+index c0c64f03e1074..a64e12605d31d 100644
+--- a/include/linux/libata.h
++++ b/include/linux/libata.h
+@@ -394,7 +394,7 @@ enum {
+ /* This should match the actual table size of
+ * ata_eh_cmd_timeout_table in libata-eh.c.
+ */
+- ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 6,
++ ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 7,
+
+ /* Horkage types. May be set by libata or controller on drives
+ (some horkage may be drive/controller pair dependent */
+diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
+index 2adeea44c0d53..61590c1f2d333 100644
+--- a/include/linux/lsm_hook_defs.h
++++ b/include/linux/lsm_hook_defs.h
+@@ -26,13 +26,13 @@
+ * #undef LSM_HOOK
+ * };
+ */
+-LSM_HOOK(int, 0, binder_set_context_mgr, struct task_struct *mgr)
+-LSM_HOOK(int, 0, binder_transaction, struct task_struct *from,
+- struct task_struct *to)
+-LSM_HOOK(int, 0, binder_transfer_binder, struct task_struct *from,
+- struct task_struct *to)
+-LSM_HOOK(int, 0, binder_transfer_file, struct task_struct *from,
+- struct task_struct *to, struct file *file)
++LSM_HOOK(int, 0, binder_set_context_mgr, const struct cred *mgr)
++LSM_HOOK(int, 0, binder_transaction, const struct cred *from,
++ const struct cred *to)
++LSM_HOOK(int, 0, binder_transfer_binder, const struct cred *from,
++ const struct cred *to)
++LSM_HOOK(int, 0, binder_transfer_file, const struct cred *from,
++ const struct cred *to, struct file *file)
+ LSM_HOOK(int, 0, ptrace_access_check, struct task_struct *child,
+ unsigned int mode)
+ LSM_HOOK(int, 0, ptrace_traceme, struct task_struct *parent)
+diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
+index 5c4c5c0602cb7..59024618554e2 100644
+--- a/include/linux/lsm_hooks.h
++++ b/include/linux/lsm_hooks.h
+@@ -1313,22 +1313,22 @@
+ *
+ * @binder_set_context_mgr:
+ * Check whether @mgr is allowed to be the binder context manager.
+- * @mgr contains the task_struct for the task being registered.
++ * @mgr contains the struct cred for the current binder process.
+ * Return 0 if permission is granted.
+ * @binder_transaction:
+ * Check whether @from is allowed to invoke a binder transaction call
+ * to @to.
+- * @from contains the task_struct for the sending task.
+- * @to contains the task_struct for the receiving task.
++ * @from contains the struct cred for the sending process.
++ * @to contains the struct cred for the receiving process.
+ * @binder_transfer_binder:
+ * Check whether @from is allowed to transfer a binder reference to @to.
+- * @from contains the task_struct for the sending task.
+- * @to contains the task_struct for the receiving task.
++ * @from contains the struct cred for the sending process.
++ * @to contains the struct cred for the receiving process.
+ * @binder_transfer_file:
+ * Check whether @from is allowed to transfer @file to @to.
+- * @from contains the task_struct for the sending task.
++ * @from contains the struct cred for the sending process.
+ * @file contains the struct file being transferred.
+- * @to contains the task_struct for the receiving task.
++ * @to contains the struct cred for the receiving process.
+ *
+ * @ptrace_access_check:
+ * Check permission before allowing the current process to trace the
+diff --git a/include/linux/mc146818rtc.h b/include/linux/mc146818rtc.h
+index 0661af17a7584..3038124c61154 100644
+--- a/include/linux/mc146818rtc.h
++++ b/include/linux/mc146818rtc.h
+@@ -86,6 +86,8 @@ struct cmos_rtc_board_info {
+ /* 2 values for divider stage reset, others for "testing purposes only" */
+ # define RTC_DIV_RESET1 0x60
+ # define RTC_DIV_RESET2 0x70
++ /* In AMD BKDG bit 5 and 6 are reserved, bit 4 is for select dv0 bank */
++# define RTC_AMD_BANK_SELECT 0x10
+ /* Periodic intr. / Square wave rate select. 0=none, 1=32.8kHz,... 15=2Hz */
+ # define RTC_RATE_SELECT 0x0F
+
+@@ -123,7 +125,8 @@ struct cmos_rtc_board_info {
+ #define RTC_IO_EXTENT_USED RTC_IO_EXTENT
+ #endif /* ARCH_RTC_LOCATION */
+
+-unsigned int mc146818_get_time(struct rtc_time *time);
++bool mc146818_does_rtc_work(void);
++int mc146818_get_time(struct rtc_time *time);
+ int mc146818_set_time(struct rtc_time *time);
+
+ #endif /* _MC146818RTC_H */
+diff --git a/include/linux/memblock.h b/include/linux/memblock.h
+index 34de69b3b8bad..5df38332e4139 100644
+--- a/include/linux/memblock.h
++++ b/include/linux/memblock.h
+@@ -388,8 +388,8 @@ phys_addr_t memblock_alloc_range_nid(phys_addr_t size,
+ phys_addr_t end, int nid, bool exact_nid);
+ phys_addr_t memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid);
+
+-static inline phys_addr_t memblock_phys_alloc(phys_addr_t size,
+- phys_addr_t align)
++static __always_inline phys_addr_t memblock_phys_alloc(phys_addr_t size,
++ phys_addr_t align)
+ {
+ return memblock_phys_alloc_range(size, align, 0,
+ MEMBLOCK_ALLOC_ACCESSIBLE);
+diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
+index 3096c9a0ee014..d35439db047cb 100644
+--- a/include/linux/memcontrol.h
++++ b/include/linux/memcontrol.h
+@@ -223,7 +223,7 @@ struct obj_cgroup {
+ struct mem_cgroup *memcg;
+ atomic_t nr_charged_bytes;
+ union {
+- struct list_head list;
++ struct list_head list; /* protected by objcg_lock */
+ struct rcu_head rcu;
+ };
+ };
+@@ -320,7 +320,8 @@ struct mem_cgroup {
+ int kmemcg_id;
+ enum memcg_kmem_state kmem_state;
+ struct obj_cgroup __rcu *objcg;
+- struct list_head objcg_list; /* list of inherited objcgs */
++ /* list of inherited objcgs, protected by objcg_lock */
++ struct list_head objcg_list;
+ #endif
+
+ MEMCG_PADDING(_pad2_);
+@@ -1001,6 +1002,7 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+ }
+
+ void mem_cgroup_flush_stats(void);
++void mem_cgroup_flush_stats_delayed(void);
+
+ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
+ int val);
+@@ -1421,6 +1423,10 @@ static inline void mem_cgroup_flush_stats(void)
+ {
+ }
+
++static inline void mem_cgroup_flush_stats_delayed(void)
++{
++}
++
+ static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec,
+ enum node_stat_item idx, int val)
+ {
+diff --git a/include/linux/mhi.h b/include/linux/mhi.h
+index 7239858790353..a5cc4cdf9cc86 100644
+--- a/include/linux/mhi.h
++++ b/include/linux/mhi.h
+@@ -663,6 +663,19 @@ int mhi_pm_suspend(struct mhi_controller *mhi_cntrl);
+ */
+ int mhi_pm_resume(struct mhi_controller *mhi_cntrl);
+
++/**
++ * mhi_pm_resume_force - Force resume MHI from suspended state
++ * @mhi_cntrl: MHI controller
++ *
++ * Resume the device irrespective of its MHI state. As per the MHI spec, devices
++ * has to be in M3 state during resume. But some devices seem to be in a
++ * different MHI state other than M3 but they continue working fine if allowed.
++ * This API is intented to be used for such devices.
++ *
++ * Return: 0 if the resume succeeds, a negative error code otherwise
++ */
++int mhi_pm_resume_force(struct mhi_controller *mhi_cntrl);
++
+ /**
+ * mhi_download_rddm_image - Download ramdump image from device for
+ * debugging purpose.
+diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h
+index 4ab5c1fc1270d..a09ed4c8361b6 100644
+--- a/include/linux/mlx5/eswitch.h
++++ b/include/linux/mlx5/eswitch.h
+@@ -136,13 +136,13 @@ u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
+ ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT)
+ #define ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK ESW_TUN_OPTS_MASK
+
+-u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev);
++u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev);
+ u16 mlx5_eswitch_get_total_vports(const struct mlx5_core_dev *dev);
+ struct mlx5_core_dev *mlx5_eswitch_get_core_dev(struct mlx5_eswitch *esw);
+
+ #else /* CONFIG_MLX5_ESWITCH */
+
+-static inline u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev)
++static inline u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev)
+ {
+ return MLX5_ESWITCH_NONE;
+ }
+diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
+index 993204a6c1a13..fdf4589ab4d48 100644
+--- a/include/linux/mlx5/mlx5_ifc.h
++++ b/include/linux/mlx5/mlx5_ifc.h
+@@ -3309,8 +3309,8 @@ enum {
+ };
+
+ enum {
+- MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO = 0x1,
+- MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO = 0x2,
++ MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO = BIT(0),
++ MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO = BIT(1),
+ };
+
+ enum {
+@@ -3335,7 +3335,7 @@ struct mlx5_ifc_tirc_bits {
+
+ u8 reserved_at_80[0x4];
+ u8 lro_timeout_period_usecs[0x10];
+- u8 lro_enable_mask[0x4];
++ u8 packet_merge_mask[0x4];
+ u8 lro_max_ip_payload_size[0x8];
+
+ u8 reserved_at_a0[0x40];
+@@ -6369,7 +6369,7 @@ struct mlx5_ifc_modify_tir_bitmask_bits {
+ u8 reserved_at_3c[0x1];
+ u8 hash[0x1];
+ u8 reserved_at_3e[0x1];
+- u8 lro[0x1];
++ u8 packet_merge[0x1];
+ };
+
+ struct mlx5_ifc_modify_tir_out_bits {
+@@ -9508,8 +9508,8 @@ struct mlx5_ifc_bufferx_reg_bits {
+ u8 reserved_at_0[0x6];
+ u8 lossy[0x1];
+ u8 epsb[0x1];
+- u8 reserved_at_8[0xc];
+- u8 size[0xc];
++ u8 reserved_at_8[0x8];
++ u8 size[0x10];
+
+ u8 xoff_threshold[0x10];
+ u8 xon_threshold[0x10];
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index 73a52aba448f9..85205adcdd0d1 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -1511,11 +1511,18 @@ static inline u8 page_kasan_tag(const struct page *page)
+
+ static inline void page_kasan_tag_set(struct page *page, u8 tag)
+ {
+- if (kasan_enabled()) {
+- tag ^= 0xff;
+- page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
+- page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
+- }
++ unsigned long old_flags, flags;
++
++ if (!kasan_enabled())
++ return;
++
++ tag ^= 0xff;
++ old_flags = READ_ONCE(page->flags);
++ do {
++ flags = old_flags;
++ flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
++ flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
++ } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));
+ }
+
+ static inline void page_kasan_tag_reset(struct page *page)
+@@ -2600,6 +2607,7 @@ extern int install_special_mapping(struct mm_struct *mm,
+ unsigned long flags, struct page **pages);
+
+ unsigned long randomize_stack_top(unsigned long stack_top);
++unsigned long randomize_page(unsigned long start, unsigned long range);
+
+ extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
+
+@@ -2851,7 +2859,8 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
+ #define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
+ #define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
+ * and return without waiting upon it */
+-#define FOLL_POPULATE 0x40 /* fault in page */
++#define FOLL_POPULATE 0x40 /* fault in pages (with FOLL_MLOCK) */
++#define FOLL_NOFAULT 0x80 /* do not fault in pages */
+ #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
+ #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
+ #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
+diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
+index 6a1d79d84675a..6ba1002165302 100644
+--- a/include/linux/mmzone.h
++++ b/include/linux/mmzone.h
+@@ -1031,6 +1031,15 @@ static inline int is_highmem_idx(enum zone_type idx)
+ #endif
+ }
+
++#ifdef CONFIG_ZONE_DMA
++bool has_managed_dma(void);
++#else
++static inline bool has_managed_dma(void)
++{
++ return false;
++}
++#endif
++
+ /**
+ * is_highmem - helper function to quickly check if a struct zone is a
+ * highmem zone or not. This is an attempt to keep references
+@@ -1342,13 +1351,16 @@ static inline unsigned long *section_to_usemap(struct mem_section *ms)
+
+ static inline struct mem_section *__nr_to_section(unsigned long nr)
+ {
++ unsigned long root = SECTION_NR_TO_ROOT(nr);
++
++ if (unlikely(root >= NR_SECTION_ROOTS))
++ return NULL;
++
+ #ifdef CONFIG_SPARSEMEM_EXTREME
+- if (!mem_section)
++ if (!mem_section || !mem_section[root])
+ return NULL;
+ #endif
+- if (!mem_section[SECTION_NR_TO_ROOT(nr)])
+- return NULL;
+- return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
++ return &mem_section[root][nr & SECTION_ROOT_MASK];
+ }
+ extern size_t mem_section_usage_size(void);
+
+diff --git a/include/linux/msi.h b/include/linux/msi.h
+index 49cf6eb222e76..e616f94c7c585 100644
+--- a/include/linux/msi.h
++++ b/include/linux/msi.h
+@@ -148,7 +148,7 @@ struct msi_desc {
+ u8 is_msix : 1;
+ u8 multiple : 3;
+ u8 multi_cap : 3;
+- u8 maskbit : 1;
++ u8 can_mask : 1;
+ u8 is_64 : 1;
+ u8 is_virtual : 1;
+ u16 entry_nr;
+diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h
+index 88227044fc86c..8a2c60235ebb8 100644
+--- a/include/linux/mtd/mtd.h
++++ b/include/linux/mtd/mtd.h
+@@ -394,10 +394,8 @@ struct mtd_info {
+ /* List of partitions attached to this MTD device */
+ struct list_head partitions;
+
+- union {
+- struct mtd_part part;
+- struct mtd_master master;
+- };
++ struct mtd_part part;
++ struct mtd_master master;
+ };
+
+ static inline struct mtd_info *mtd_get_master(struct mtd_info *mtd)
+diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h
+index b2f9dd3cbd695..dcf90144d70b7 100644
+--- a/include/linux/mtd/rawnand.h
++++ b/include/linux/mtd/rawnand.h
+@@ -1240,6 +1240,7 @@ struct nand_secure_region {
+ * @lock: Lock protecting the suspended field. Also used to serialize accesses
+ * to the NAND device
+ * @suspended: Set to 1 when the device is suspended, 0 when it's not
++ * @resume_wq: wait queue to sleep if rawnand is in suspended state.
+ * @cur_cs: Currently selected target. -1 means no target selected, otherwise we
+ * should always have cur_cs >= 0 && cur_cs < nanddev_ntargets().
+ * NAND Controller drivers should not modify this value, but they're
+@@ -1294,6 +1295,7 @@ struct nand_chip {
+ /* Internals */
+ struct mutex lock;
+ unsigned int suspended : 1;
++ wait_queue_head_t resume_wq;
+ int cur_cs;
+ int read_retries;
+ struct nand_secure_region *secure_regions;
+@@ -1539,6 +1541,8 @@ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
+ bool force_8bit, bool check_only);
+ int nand_write_data_op(struct nand_chip *chip, const void *buf,
+ unsigned int len, bool force_8bit);
++int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf,
++ int oob_required, int page);
+
+ /* Scan and identify a NAND device */
+ int nand_scan_with_ids(struct nand_chip *chip, unsigned int max_chips,
+diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
+index 2c6b9e4162254..7c2d77d75a888 100644
+--- a/include/linux/netdev_features.h
++++ b/include/linux/netdev_features.h
+@@ -169,7 +169,7 @@ enum {
+ #define NETIF_F_HW_HSR_FWD __NETIF_F(HW_HSR_FWD)
+ #define NETIF_F_HW_HSR_DUP __NETIF_F(HW_HSR_DUP)
+
+-/* Finds the next feature with the highest number of the range of start till 0.
++/* Finds the next feature with the highest number of the range of start-1 till 0.
+ */
+ static inline int find_next_netdev_feature(u64 feature, unsigned long start)
+ {
+@@ -188,7 +188,7 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start)
+ for ((bit) = find_next_netdev_feature((mask_addr), \
+ NETDEV_FEATURE_COUNT); \
+ (bit) >= 0; \
+- (bit) = find_next_netdev_feature((mask_addr), (bit) - 1))
++ (bit) = find_next_netdev_feature((mask_addr), (bit)))
+
+ /* Features valid for ethtool to change */
+ /* = all defined minus driver/device-class-related */
+diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
+index d79163208dfdb..39f1893ecac03 100644
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -887,7 +887,7 @@ struct net_device_path_stack {
+
+ struct net_device_path_ctx {
+ const struct net_device *dev;
+- const u8 *daddr;
++ u8 daddr[ETH_ALEN];
+
+ int num_vlans;
+ struct {
+@@ -2149,7 +2149,7 @@ struct net_device {
+ struct netdev_queue *_tx ____cacheline_aligned_in_smp;
+ unsigned int num_tx_queues;
+ unsigned int real_num_tx_queues;
+- struct Qdisc *qdisc;
++ struct Qdisc __rcu *qdisc;
+ unsigned int tx_queue_len;
+ spinlock_t tx_global_lock;
+
+@@ -2636,6 +2636,7 @@ struct packet_type {
+ struct net_device *);
+ bool (*id_match)(struct packet_type *ptype,
+ struct sock *sk);
++ struct net *af_packet_net;
+ void *af_packet_priv;
+ struct list_head list;
+ };
+@@ -4403,7 +4404,8 @@ static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
+ static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
+ {
+ spin_lock(&txq->_xmit_lock);
+- txq->xmit_lock_owner = cpu;
++ /* Pairs with READ_ONCE() in __dev_queue_xmit() */
++ WRITE_ONCE(txq->xmit_lock_owner, cpu);
+ }
+
+ static inline bool __netif_tx_acquire(struct netdev_queue *txq)
+@@ -4420,26 +4422,32 @@ static inline void __netif_tx_release(struct netdev_queue *txq)
+ static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
+ {
+ spin_lock_bh(&txq->_xmit_lock);
+- txq->xmit_lock_owner = smp_processor_id();
++ /* Pairs with READ_ONCE() in __dev_queue_xmit() */
++ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
+ }
+
+ static inline bool __netif_tx_trylock(struct netdev_queue *txq)
+ {
+ bool ok = spin_trylock(&txq->_xmit_lock);
+- if (likely(ok))
+- txq->xmit_lock_owner = smp_processor_id();
++
++ if (likely(ok)) {
++ /* Pairs with READ_ONCE() in __dev_queue_xmit() */
++ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
++ }
+ return ok;
+ }
+
+ static inline void __netif_tx_unlock(struct netdev_queue *txq)
+ {
+- txq->xmit_lock_owner = -1;
++ /* Pairs with READ_ONCE() in __dev_queue_xmit() */
++ WRITE_ONCE(txq->xmit_lock_owner, -1);
+ spin_unlock(&txq->_xmit_lock);
+ }
+
+ static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
+ {
+- txq->xmit_lock_owner = -1;
++ /* Pairs with READ_ONCE() in __dev_queue_xmit() */
++ WRITE_ONCE(txq->xmit_lock_owner, -1);
+ spin_unlock_bh(&txq->_xmit_lock);
+ }
+
+diff --git a/include/linux/netfilter/nf_conntrack_common.h b/include/linux/netfilter/nf_conntrack_common.h
+index 700ea077ce2d6..2770db2fa080d 100644
+--- a/include/linux/netfilter/nf_conntrack_common.h
++++ b/include/linux/netfilter/nf_conntrack_common.h
+@@ -2,7 +2,7 @@
+ #ifndef _NF_CONNTRACK_COMMON_H
+ #define _NF_CONNTRACK_COMMON_H
+
+-#include <linux/atomic.h>
++#include <linux/refcount.h>
+ #include <uapi/linux/netfilter/nf_conntrack_common.h>
+
+ struct ip_conntrack_stat {
+@@ -25,19 +25,21 @@ struct ip_conntrack_stat {
+ #define NFCT_PTRMASK ~(NFCT_INFOMASK)
+
+ struct nf_conntrack {
+- atomic_t use;
++ refcount_t use;
+ };
+
+ void nf_conntrack_destroy(struct nf_conntrack *nfct);
++
++/* like nf_ct_put, but without module dependency on nf_conntrack */
+ static inline void nf_conntrack_put(struct nf_conntrack *nfct)
+ {
+- if (nfct && atomic_dec_and_test(&nfct->use))
++ if (nfct && refcount_dec_and_test(&nfct->use))
+ nf_conntrack_destroy(nfct);
+ }
+ static inline void nf_conntrack_get(struct nf_conntrack *nfct)
+ {
+ if (nfct)
+- atomic_inc(&nfct->use);
++ refcount_inc(&nfct->use);
+ }
+
+ #endif /* _NF_CONNTRACK_COMMON_H */
+diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
+index b9a8b925db430..5ffcde9ac4130 100644
+--- a/include/linux/nfs_fs.h
++++ b/include/linux/nfs_fs.h
+@@ -494,10 +494,10 @@ static inline const struct cred *nfs_file_cred(struct file *file)
+ * linux/fs/nfs/direct.c
+ */
+ extern ssize_t nfs_direct_IO(struct kiocb *, struct iov_iter *);
+-extern ssize_t nfs_file_direct_read(struct kiocb *iocb,
+- struct iov_iter *iter);
+-extern ssize_t nfs_file_direct_write(struct kiocb *iocb,
+- struct iov_iter *iter);
++ssize_t nfs_file_direct_read(struct kiocb *iocb,
++ struct iov_iter *iter, bool swap);
++ssize_t nfs_file_direct_write(struct kiocb *iocb,
++ struct iov_iter *iter, bool swap);
+
+ /*
+ * linux/fs/nfs/dir.c
+@@ -517,8 +517,8 @@ extern int nfs_instantiate(struct dentry *dentry, struct nfs_fh *fh,
+ struct nfs_fattr *fattr, struct nfs4_label *label);
+ extern int nfs_may_open(struct inode *inode, const struct cred *cred, int openflags);
+ extern void nfs_access_zap_cache(struct inode *inode);
+-extern int nfs_access_get_cached(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res,
+- bool may_block);
++extern int nfs_access_get_cached(struct inode *inode, const struct cred *cred,
++ u32 *mask, bool may_block);
+
+ /*
+ * linux/fs/nfs/symlink.c
+@@ -567,8 +567,9 @@ extern int nfs_wb_all(struct inode *inode);
+ extern int nfs_wb_page(struct inode *inode, struct page *page);
+ extern int nfs_wb_page_cancel(struct inode *inode, struct page* page);
+ extern int nfs_commit_inode(struct inode *, int);
+-extern struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail);
++extern struct nfs_commit_data *nfs_commitdata_alloc(void);
+ extern void nfs_commit_free(struct nfs_commit_data *data);
++bool nfs_commit_end(struct nfs_mds_commit_info *cinfo);
+
+ static inline int
+ nfs_have_writebacks(struct inode *inode)
+diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
+index 2a9acbfe00f0f..9a6e70ccde56e 100644
+--- a/include/linux/nfs_fs_sb.h
++++ b/include/linux/nfs_fs_sb.h
+@@ -287,5 +287,5 @@ struct nfs_server {
+ #define NFS_CAP_COPY_NOTIFY (1U << 27)
+ #define NFS_CAP_XATTR (1U << 28)
+ #define NFS_CAP_READ_PLUS (1U << 29)
+-
++#define NFS_CAP_FS_LOCATIONS (1U << 30)
+ #endif
+diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
+index e9698b6278a52..ecd74cc347974 100644
+--- a/include/linux/nfs_xdr.h
++++ b/include/linux/nfs_xdr.h
+@@ -1805,6 +1805,7 @@ struct nfs_rpc_ops {
+ struct nfs_server *(*create_server)(struct fs_context *);
+ struct nfs_server *(*clone_server)(struct nfs_server *, struct nfs_fh *,
+ struct nfs_fattr *, rpc_authflavor_t);
++ int (*discover_trunking)(struct nfs_server *, struct nfs_fh *);
+ };
+
+ /*
+diff --git a/include/linux/nvme.h b/include/linux/nvme.h
+index b7c4c4130b65e..039f59ee8f435 100644
+--- a/include/linux/nvme.h
++++ b/include/linux/nvme.h
+@@ -322,6 +322,7 @@ enum {
+ NVME_CTRL_ONCS_TIMESTAMP = 1 << 6,
+ NVME_CTRL_VWC_PRESENT = 1 << 0,
+ NVME_CTRL_OACS_SEC_SUPP = 1 << 0,
++ NVME_CTRL_OACS_NS_MNGT_SUPP = 1 << 3,
+ NVME_CTRL_OACS_DIRECTIVES = 1 << 5,
+ NVME_CTRL_OACS_DBBUF_SUPP = 1 << 8,
+ NVME_CTRL_LPA_CMD_EFFECTS_LOG = 1 << 1,
+diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h
+index 104505e9028f7..87932bdb25d7b 100644
+--- a/include/linux/nvmem-provider.h
++++ b/include/linux/nvmem-provider.h
+@@ -66,7 +66,8 @@ struct nvmem_keepout {
+ * @word_size: Minimum read/write access granularity.
+ * @stride: Minimum read/write access stride.
+ * @priv: User context passed to read/write callbacks.
+- * @wp-gpio: Write protect pin
++ * @wp-gpio: Write protect pin
++ * @ignore_wp: Write Protect pin is managed by the provider.
+ *
+ * Note: A default "nvmem<id>" name will be assigned to the device if
+ * no name is specified in its configuration. In such case "<id>" is
+@@ -88,6 +89,7 @@ struct nvmem_config {
+ enum nvmem_type type;
+ bool read_only;
+ bool root_only;
++ bool ignore_wp;
+ struct device_node *of_node;
+ bool no_of_node;
+ nvmem_reg_read_t reg_read;
+diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h
+index cf6a65b94d40e..6508b97dbf1d2 100644
+--- a/include/linux/of_fdt.h
++++ b/include/linux/of_fdt.h
+@@ -62,6 +62,7 @@ extern int early_init_dt_scan_chosen(unsigned long node, const char *uname,
+ int depth, void *data);
+ extern int early_init_dt_scan_memory(unsigned long node, const char *uname,
+ int depth, void *data);
++extern void early_init_dt_check_for_usable_mem_range(void);
+ extern int early_init_dt_scan_chosen_stdout(void);
+ extern void early_init_fdt_scan_reserved_mem(void);
+ extern void early_init_fdt_reserve_self(void);
+@@ -87,6 +88,7 @@ extern void unflatten_and_copy_device_tree(void);
+ extern void early_init_devtree(void *);
+ extern void early_get_first_memblock_info(void *, phys_addr_t *);
+ #else /* CONFIG_OF_EARLY_FLATTREE */
++static inline void early_init_dt_check_for_usable_mem_range(void) {}
+ static inline int early_init_dt_scan_chosen_stdout(void) { return -ENODEV; }
+ static inline void early_init_fdt_scan_reserved_mem(void) {}
+ static inline void early_init_fdt_reserve_self(void) {}
+diff --git a/include/linux/of_net.h b/include/linux/of_net.h
+index daef3b0d9270d..55460ecfa50ad 100644
+--- a/include/linux/of_net.h
++++ b/include/linux/of_net.h
+@@ -8,7 +8,7 @@
+
+ #include <linux/phy.h>
+
+-#ifdef CONFIG_OF_NET
++#if defined(CONFIG_OF) && defined(CONFIG_NET)
+ #include <linux/of.h>
+
+ struct net_device;
+diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
+index 62db6b0176b95..2f7dd14083d94 100644
+--- a/include/linux/pagemap.h
++++ b/include/linux/pagemap.h
+@@ -733,61 +733,11 @@ int wait_on_page_private_2_killable(struct page *page);
+ extern void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter);
+
+ /*
+- * Fault everything in given userspace address range in.
++ * Fault in userspace address range.
+ */
+-static inline int fault_in_pages_writeable(char __user *uaddr, size_t size)
+-{
+- char __user *end = uaddr + size - 1;
+-
+- if (unlikely(size == 0))
+- return 0;
+-
+- if (unlikely(uaddr > end))
+- return -EFAULT;
+- /*
+- * Writing zeroes into userspace here is OK, because we know that if
+- * the zero gets there, we'll be overwriting it.
+- */
+- do {
+- if (unlikely(__put_user(0, uaddr) != 0))
+- return -EFAULT;
+- uaddr += PAGE_SIZE;
+- } while (uaddr <= end);
+-
+- /* Check whether the range spilled into the next page. */
+- if (((unsigned long)uaddr & PAGE_MASK) ==
+- ((unsigned long)end & PAGE_MASK))
+- return __put_user(0, end);
+-
+- return 0;
+-}
+-
+-static inline int fault_in_pages_readable(const char __user *uaddr, size_t size)
+-{
+- volatile char c;
+- const char __user *end = uaddr + size - 1;
+-
+- if (unlikely(size == 0))
+- return 0;
+-
+- if (unlikely(uaddr > end))
+- return -EFAULT;
+-
+- do {
+- if (unlikely(__get_user(c, uaddr) != 0))
+- return -EFAULT;
+- uaddr += PAGE_SIZE;
+- } while (uaddr <= end);
+-
+- /* Check whether the range spilled into the next page. */
+- if (((unsigned long)uaddr & PAGE_MASK) ==
+- ((unsigned long)end & PAGE_MASK)) {
+- return __get_user(c, end);
+- }
+-
+- (void)c;
+- return 0;
+-}
++size_t fault_in_writeable(char __user *uaddr, size_t size);
++size_t fault_in_safe_writeable(const char __user *uaddr, size_t size);
++size_t fault_in_readable(const char __user *uaddr, size_t size);
+
+ int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
+ pgoff_t index, gfp_t gfp_mask);
+diff --git a/include/linux/part_stat.h b/include/linux/part_stat.h
+index d2558121d48c0..6f7949b2fd8dc 100644
+--- a/include/linux/part_stat.h
++++ b/include/linux/part_stat.h
+@@ -3,6 +3,7 @@
+ #define _LINUX_PART_STAT_H
+
+ #include <linux/genhd.h>
++#include <asm/local.h>
+
+ struct disk_stats {
+ u64 nsecs[NR_STAT_GROUPS];
+diff --git a/include/linux/pci.h b/include/linux/pci.h
+index cd8aa6fce2041..9d6e75222868f 100644
+--- a/include/linux/pci.h
++++ b/include/linux/pci.h
+@@ -233,6 +233,8 @@ enum pci_dev_flags {
+ PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10),
+ /* Don't use Relaxed Ordering for TLPs directed at this device */
+ PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11),
++ /* Device does honor MSI masking despite saying otherwise */
++ PCI_DEV_FLAGS_HAS_MSI_MASKING = (__force pci_dev_flags_t) (1 << 12),
+ };
+
+ enum pci_irq_reroute_variant {
+@@ -654,6 +656,7 @@ struct pci_bus {
+ struct bin_attribute *legacy_io; /* Legacy I/O for this bus */
+ struct bin_attribute *legacy_mem; /* Legacy mem */
+ unsigned int is_added:1;
++ unsigned int unsafe_warn:1; /* warned about RW1C config write */
+ };
+
+ #define to_pci_bus(n) container_of(n, struct pci_bus, dev)
+diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
+index 9b60bb89d86ab..6cce33e7e7acc 100644
+--- a/include/linux/perf_event.h
++++ b/include/linux/perf_event.h
+@@ -680,18 +680,6 @@ struct perf_event {
+ u64 total_time_running;
+ u64 tstamp;
+
+- /*
+- * timestamp shadows the actual context timing but it can
+- * be safely used in NMI interrupt context. It reflects the
+- * context time as it was when the event was last scheduled in,
+- * or when ctx_sched_in failed to schedule the event because we
+- * run out of PMC.
+- *
+- * ctx_time already accounts for ctx->timestamp. Therefore to
+- * compute ctx_time for a sample, simply add perf_clock().
+- */
+- u64 shadow_ctx_time;
+-
+ struct perf_event_attr attr;
+ u16 header_size;
+ u16 id_header_size;
+@@ -838,6 +826,7 @@ struct perf_event_context {
+ */
+ u64 time;
+ u64 timestamp;
++ u64 timeoffset;
+
+ /*
+ * These fields let us detect when two contexts have both
+@@ -920,6 +909,8 @@ struct bpf_perf_event_data_kern {
+ struct perf_cgroup_info {
+ u64 time;
+ u64 timestamp;
++ u64 timeoffset;
++ int active;
+ };
+
+ struct perf_cgroup {
+@@ -1239,7 +1230,18 @@ extern void perf_event_bpf_event(struct bpf_prog *prog,
+ enum perf_bpf_event_type type,
+ u16 flags);
+
+-extern struct perf_guest_info_callbacks *perf_guest_cbs;
++extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
++static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void)
++{
++ /*
++ * Callbacks are RCU-protected and must be READ_ONCE to avoid reloading
++ * the callbacks between a !NULL check and dereferences, to ensure
++ * pending stores/changes to the callback pointers are visible before a
++ * non-NULL perf_guest_cbs is visible to readers, and to prevent a
++ * module from unloading callbacks while readers are active.
++ */
++ return rcu_dereference(perf_guest_cbs);
++}
+ extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
+ extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
+
+diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
+index e24d2c992b112..d468efcf48f45 100644
+--- a/include/linux/pgtable.h
++++ b/include/linux/pgtable.h
+@@ -62,6 +62,7 @@ static inline unsigned long pte_index(unsigned long address)
+ {
+ return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
+ }
++#define pte_index pte_index
+
+ #ifndef pmd_index
+ static inline unsigned long pmd_index(unsigned long address)
+diff --git a/include/linux/platform_data/ti-sysc.h b/include/linux/platform_data/ti-sysc.h
+index 9837fb011f2fb..989aa30c598dc 100644
+--- a/include/linux/platform_data/ti-sysc.h
++++ b/include/linux/platform_data/ti-sysc.h
+@@ -50,6 +50,7 @@ struct sysc_regbits {
+ s8 emufree_shift;
+ };
+
++#define SYSC_QUIRK_REINIT_ON_CTX_LOST BIT(28)
+ #define SYSC_QUIRK_REINIT_ON_RESUME BIT(27)
+ #define SYSC_QUIRK_GPMC_DEBUG BIT(26)
+ #define SYSC_MODULE_QUIRK_ENA_RESETDONE BIT(25)
+diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h
+index 222da43b7096d..016de5776b6db 100644
+--- a/include/linux/pm_runtime.h
++++ b/include/linux/pm_runtime.h
+@@ -58,6 +58,7 @@ extern void pm_runtime_get_suppliers(struct device *dev);
+ extern void pm_runtime_put_suppliers(struct device *dev);
+ extern void pm_runtime_new_link(struct device *dev);
+ extern void pm_runtime_drop_link(struct device_link *link);
++extern void pm_runtime_release_supplier(struct device_link *link, bool check_idle);
+
+ extern int devm_pm_runtime_enable(struct device *dev);
+
+@@ -129,7 +130,7 @@ static inline bool pm_runtime_suspended(struct device *dev)
+ * pm_runtime_active - Check whether or not a device is runtime-active.
+ * @dev: Target device.
+ *
+- * Return %true if runtime PM is enabled for @dev and its runtime PM status is
++ * Return %true if runtime PM is disabled for @dev or its runtime PM status is
+ * %RPM_ACTIVE, or %false otherwise.
+ *
+ * Note that the return value of this function can only be trusted if it is
+@@ -283,6 +284,8 @@ static inline void pm_runtime_get_suppliers(struct device *dev) {}
+ static inline void pm_runtime_put_suppliers(struct device *dev) {}
+ static inline void pm_runtime_new_link(struct device *dev) {}
+ static inline void pm_runtime_drop_link(struct device_link *link) {}
++static inline void pm_runtime_release_supplier(struct device_link *link,
++ bool check_idle) {}
+
+ #endif /* !CONFIG_PM */
+
+diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h
+index 00fef0064355f..5bbcd280bfd26 100644
+--- a/include/linux/posix-timers.h
++++ b/include/linux/posix-timers.h
+@@ -184,8 +184,10 @@ static inline void posix_cputimers_group_init(struct posix_cputimers *pct,
+ #endif
+
+ #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
++void clear_posix_cputimers_work(struct task_struct *p);
+ void posix_cputimers_init_work(void);
+ #else
++static inline void clear_posix_cputimers_work(struct task_struct *p) { }
+ static inline void posix_cputimers_init_work(void) { }
+ #endif
+
+diff --git a/include/linux/prandom.h b/include/linux/prandom.h
+index 056d31317e499..a4aadd2dc153e 100644
+--- a/include/linux/prandom.h
++++ b/include/linux/prandom.h
+@@ -10,6 +10,7 @@
+
+ #include <linux/types.h>
+ #include <linux/percpu.h>
++#include <linux/siphash.h>
+
+ u32 prandom_u32(void);
+ void prandom_bytes(void *buf, size_t nbytes);
+@@ -27,15 +28,10 @@ DECLARE_PER_CPU(unsigned long, net_rand_noise);
+ * The core SipHash round function. Each line can be executed in
+ * parallel given enough CPU resources.
+ */
+-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
+- v0 += v1, v1 = rol64(v1, 13), v2 += v3, v3 = rol64(v3, 16), \
+- v1 ^= v0, v0 = rol64(v0, 32), v3 ^= v2, \
+- v0 += v3, v3 = rol64(v3, 21), v2 += v1, v1 = rol64(v1, 17), \
+- v3 ^= v0, v1 ^= v2, v2 = rol64(v2, 32) \
+-)
++#define PRND_SIPROUND(v0, v1, v2, v3) SIPHASH_PERMUTATION(v0, v1, v2, v3)
+
+-#define PRND_K0 (0x736f6d6570736575 ^ 0x6c7967656e657261)
+-#define PRND_K1 (0x646f72616e646f6d ^ 0x7465646279746573)
++#define PRND_K0 (SIPHASH_CONST_0 ^ SIPHASH_CONST_2)
++#define PRND_K1 (SIPHASH_CONST_1 ^ SIPHASH_CONST_3)
+
+ #elif BITS_PER_LONG == 32
+ /*
+@@ -43,14 +39,9 @@ DECLARE_PER_CPU(unsigned long, net_rand_noise);
+ * This is weaker, but 32-bit machines are not used for high-traffic
+ * applications, so there is less output for an attacker to analyze.
+ */
+-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
+- v0 += v1, v1 = rol32(v1, 5), v2 += v3, v3 = rol32(v3, 8), \
+- v1 ^= v0, v0 = rol32(v0, 16), v3 ^= v2, \
+- v0 += v3, v3 = rol32(v3, 7), v2 += v1, v1 = rol32(v1, 13), \
+- v3 ^= v0, v1 ^= v2, v2 = rol32(v2, 16) \
+-)
+-#define PRND_K0 0x6c796765
+-#define PRND_K1 0x74656462
++#define PRND_SIPROUND(v0, v1, v2, v3) HSIPHASH_PERMUTATION(v0, v1, v2, v3)
++#define PRND_K0 (HSIPHASH_CONST_0 ^ HSIPHASH_CONST_2)
++#define PRND_K1 (HSIPHASH_CONST_1 ^ HSIPHASH_CONST_3)
+
+ #else
+ #error Unsupported BITS_PER_LONG
+diff --git a/include/linux/printk.h b/include/linux/printk.h
+index 85b656f82d752..9497f6b983399 100644
+--- a/include/linux/printk.h
++++ b/include/linux/printk.h
+@@ -198,6 +198,7 @@ void dump_stack_print_info(const char *log_lvl);
+ void show_regs_print_info(const char *log_lvl);
+ extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;
+ extern asmlinkage void dump_stack(void) __cold;
++void printk_trigger_flush(void);
+ #else
+ static inline __printf(1, 0)
+ int vprintk(const char *s, va_list args)
+@@ -274,6 +275,9 @@ static inline void dump_stack_lvl(const char *log_lvl)
+ static inline void dump_stack(void)
+ {
+ }
++static inline void printk_trigger_flush(void)
++{
++}
+ #endif
+
+ #ifdef CONFIG_SMP
+diff --git a/include/linux/psi.h b/include/linux/psi.h
+index 65eb1476ac705..57823b30c2d3d 100644
+--- a/include/linux/psi.h
++++ b/include/linux/psi.h
+@@ -24,18 +24,17 @@ void psi_memstall_enter(unsigned long *flags);
+ void psi_memstall_leave(unsigned long *flags);
+
+ int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res);
+-
+-#ifdef CONFIG_CGROUPS
+-int psi_cgroup_alloc(struct cgroup *cgrp);
+-void psi_cgroup_free(struct cgroup *cgrp);
+-void cgroup_move_task(struct task_struct *p, struct css_set *to);
+-
+ struct psi_trigger *psi_trigger_create(struct psi_group *group,
+ char *buf, size_t nbytes, enum psi_res res);
+-void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *t);
++void psi_trigger_destroy(struct psi_trigger *t);
+
+ __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file,
+ poll_table *wait);
++
++#ifdef CONFIG_CGROUPS
++int psi_cgroup_alloc(struct cgroup *cgrp);
++void psi_cgroup_free(struct cgroup *cgrp);
++void cgroup_move_task(struct task_struct *p, struct css_set *to);
+ #endif
+
+ #else /* CONFIG_PSI */
+diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h
+index 0a23300d49af7..6f190002a2022 100644
+--- a/include/linux/psi_types.h
++++ b/include/linux/psi_types.h
+@@ -21,7 +21,17 @@ enum psi_task_count {
+ * don't have to special case any state tracking for it.
+ */
+ NR_ONCPU,
+- NR_PSI_TASK_COUNTS = 4,
++ /*
++ * For IO and CPU stalls the presence of running/oncpu tasks
++ * in the domain means a partial rather than a full stall.
++ * For memory it's not so simple because of page reclaimers:
++ * they are running/oncpu while representing a stall. To tell
++ * whether a domain has productivity left or not, we need to
++ * distinguish between regular running (i.e. productive)
++ * threads and memstall ones.
++ */
++ NR_MEMSTALL_RUNNING,
++ NR_PSI_TASK_COUNTS = 5,
+ };
+
+ /* Task state bitmasks */
+@@ -29,6 +39,7 @@ enum psi_task_count {
+ #define TSK_MEMSTALL (1 << NR_MEMSTALL)
+ #define TSK_RUNNING (1 << NR_RUNNING)
+ #define TSK_ONCPU (1 << NR_ONCPU)
++#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING)
+
+ /* Resources that workloads could be stalled on */
+ enum psi_res {
+@@ -129,9 +140,6 @@ struct psi_trigger {
+ * events to one per window
+ */
+ u64 last_event_time;
+-
+- /* Refcounting to prevent premature destruction */
+- struct kref refcount;
+ };
+
+ struct psi_group {
+diff --git a/include/linux/pstore.h b/include/linux/pstore.h
+index eb93a54cff31f..e97a8188f0fd8 100644
+--- a/include/linux/pstore.h
++++ b/include/linux/pstore.h
+@@ -14,7 +14,7 @@
+ #include <linux/errno.h>
+ #include <linux/kmsg_dump.h>
+ #include <linux/mutex.h>
+-#include <linux/semaphore.h>
++#include <linux/spinlock.h>
+ #include <linux/time.h>
+ #include <linux/types.h>
+
+@@ -87,7 +87,7 @@ struct pstore_record {
+ * @owner: module which is responsible for this backend driver
+ * @name: name of the backend driver
+ *
+- * @buf_lock: semaphore to serialize access to @buf
++ * @buf_lock: spinlock to serialize access to @buf
+ * @buf: preallocated crash dump buffer
+ * @bufsize: size of @buf available for crash dump bytes (must match
+ * smallest number of bytes available for writing to a
+@@ -178,7 +178,7 @@ struct pstore_info {
+ struct module *owner;
+ const char *name;
+
+- struct semaphore buf_lock;
++ spinlock_t buf_lock;
+ char *buf;
+ size_t bufsize;
+
+diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h
+index 2e5565067355b..554454cb86931 100644
+--- a/include/linux/ptp_clock_kernel.h
++++ b/include/linux/ptp_clock_kernel.h
+@@ -351,15 +351,17 @@ int ptp_get_vclocks_index(int pclock_index, int **vclock_index);
+ *
+ * @hwtstamps: skb_shared_hwtstamps structure pointer
+ * @vclock_index: phc index of ptp vclock.
++ *
++ * Returns converted timestamp, or 0 on error.
+ */
+-void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps,
+- int vclock_index);
++ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps,
++ int vclock_index);
+ #else
+ static inline int ptp_get_vclocks_index(int pclock_index, int **vclock_index)
+ { return 0; }
+-static inline void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps,
+- int vclock_index)
+-{ }
++static inline ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps,
++ int vclock_index)
++{ return 0; }
+
+ #endif
+
+diff --git a/include/linux/random.h b/include/linux/random.h
+index f45b8be3e3c4e..917470c4490ac 100644
+--- a/include/linux/random.h
++++ b/include/linux/random.h
+@@ -1,9 +1,5 @@
+ /* SPDX-License-Identifier: GPL-2.0 */
+-/*
+- * include/linux/random.h
+- *
+- * Include file for the random number generator.
+- */
++
+ #ifndef _LINUX_RANDOM_H
+ #define _LINUX_RANDOM_H
+
+@@ -14,41 +10,26 @@
+
+ #include <uapi/linux/random.h>
+
+-struct random_ready_callback {
+- struct list_head list;
+- void (*func)(struct random_ready_callback *rdy);
+- struct module *owner;
+-};
++struct notifier_block;
+
+-extern void add_device_randomness(const void *, unsigned int);
+-extern void add_bootloader_randomness(const void *, unsigned int);
++void add_device_randomness(const void *buf, size_t len);
++void add_bootloader_randomness(const void *buf, size_t len);
++void add_input_randomness(unsigned int type, unsigned int code,
++ unsigned int value) __latent_entropy;
++void add_interrupt_randomness(int irq) __latent_entropy;
++void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
+
+ #if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
+ static inline void add_latent_entropy(void)
+ {
+- add_device_randomness((const void *)&latent_entropy,
+- sizeof(latent_entropy));
++ add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
+ }
+ #else
+-static inline void add_latent_entropy(void) {}
+-#endif
+-
+-extern void add_input_randomness(unsigned int type, unsigned int code,
+- unsigned int value) __latent_entropy;
+-extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy;
+-
+-extern void get_random_bytes(void *buf, int nbytes);
+-extern int wait_for_random_bytes(void);
+-extern int __init rand_initialize(void);
+-extern bool rng_is_initialized(void);
+-extern int add_random_ready_callback(struct random_ready_callback *rdy);
+-extern void del_random_ready_callback(struct random_ready_callback *rdy);
+-extern int __must_check get_random_bytes_arch(void *buf, int nbytes);
+-
+-#ifndef MODULE
+-extern const struct file_operations random_fops, urandom_fops;
++static inline void add_latent_entropy(void) { }
+ #endif
+
++void get_random_bytes(void *buf, size_t len);
++size_t __must_check get_random_bytes_arch(void *buf, size_t len);
+ u32 get_random_u32(void);
+ u64 get_random_u64(void);
+ static inline unsigned int get_random_int(void)
+@@ -80,36 +61,38 @@ static inline unsigned long get_random_long(void)
+
+ static inline unsigned long get_random_canary(void)
+ {
+- unsigned long val = get_random_long();
+-
+- return val & CANARY_MASK;
++ return get_random_long() & CANARY_MASK;
+ }
+
++int __init random_init(const char *command_line);
++bool rng_is_initialized(void);
++int wait_for_random_bytes(void);
++int register_random_ready_notifier(struct notifier_block *nb);
++int unregister_random_ready_notifier(struct notifier_block *nb);
++
+ /* Calls wait_for_random_bytes() and then calls get_random_bytes(buf, nbytes).
+ * Returns the result of the call to wait_for_random_bytes. */
+-static inline int get_random_bytes_wait(void *buf, int nbytes)
++static inline int get_random_bytes_wait(void *buf, size_t nbytes)
+ {
+ int ret = wait_for_random_bytes();
+ get_random_bytes(buf, nbytes);
+ return ret;
+ }
+
+-#define declare_get_random_var_wait(var) \
+- static inline int get_random_ ## var ## _wait(var *out) { \
++#define declare_get_random_var_wait(name, ret_type) \
++ static inline int get_random_ ## name ## _wait(ret_type *out) { \
+ int ret = wait_for_random_bytes(); \
+ if (unlikely(ret)) \
+ return ret; \
+- *out = get_random_ ## var(); \
++ *out = get_random_ ## name(); \
+ return 0; \
+ }
+-declare_get_random_var_wait(u32)
+-declare_get_random_var_wait(u64)
+-declare_get_random_var_wait(int)
+-declare_get_random_var_wait(long)
++declare_get_random_var_wait(u32, u32)
++declare_get_random_var_wait(u64, u32)
++declare_get_random_var_wait(int, unsigned int)
++declare_get_random_var_wait(long, unsigned long)
+ #undef declare_get_random_var
+
+-unsigned long randomize_page(unsigned long start, unsigned long range);
+-
+ /*
+ * This is designed to be standalone for just prandom
+ * users, but for now we include it from <linux/random.h>
+@@ -120,22 +103,10 @@ unsigned long randomize_page(unsigned long start, unsigned long range);
+ #ifdef CONFIG_ARCH_RANDOM
+ # include <asm/archrandom.h>
+ #else
+-static inline bool __must_check arch_get_random_long(unsigned long *v)
+-{
+- return false;
+-}
+-static inline bool __must_check arch_get_random_int(unsigned int *v)
+-{
+- return false;
+-}
+-static inline bool __must_check arch_get_random_seed_long(unsigned long *v)
+-{
+- return false;
+-}
+-static inline bool __must_check arch_get_random_seed_int(unsigned int *v)
+-{
+- return false;
+-}
++static inline bool __must_check arch_get_random_long(unsigned long *v) { return false; }
++static inline bool __must_check arch_get_random_int(unsigned int *v) { return false; }
++static inline bool __must_check arch_get_random_seed_long(unsigned long *v) { return false; }
++static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { return false; }
+ #endif
+
+ /*
+@@ -158,4 +129,13 @@ static inline bool __init arch_get_random_long_early(unsigned long *v)
+ }
+ #endif
+
++#ifdef CONFIG_SMP
++int random_prepare_cpu(unsigned int cpu);
++int random_online_cpu(unsigned int cpu);
++#endif
++
++#ifndef MODULE
++extern const struct file_operations random_fops, urandom_fops;
++#endif
++
+ #endif /* _LINUX_RANDOM_H */
+diff --git a/include/linux/randomize_kstack.h b/include/linux/randomize_kstack.h
+index bebc911161b6f..d373f1bcbf7ca 100644
+--- a/include/linux/randomize_kstack.h
++++ b/include/linux/randomize_kstack.h
+@@ -16,8 +16,20 @@ DECLARE_PER_CPU(u32, kstack_offset);
+ * alignment. Also, since this use is being explicitly masked to a max of
+ * 10 bits, stack-clash style attacks are unlikely. For more details see
+ * "VLAs" in Documentation/process/deprecated.rst
++ *
++ * The normal __builtin_alloca() is initialized with INIT_STACK_ALL (currently
++ * only with Clang and not GCC). Initializing the unused area on each syscall
++ * entry is expensive, and generating an implicit call to memset() may also be
++ * problematic (such as in noinstr functions). Therefore, if the compiler
++ * supports it (which it should if it initializes allocas), always use the
++ * "uninitialized" variant of the builtin.
+ */
+-void *__builtin_alloca(size_t size);
++#if __has_builtin(__builtin_alloca_uninitialized)
++#define __kstack_alloca __builtin_alloca_uninitialized
++#else
++#define __kstack_alloca __builtin_alloca
++#endif
++
+ /*
+ * Use, at most, 10 bits of entropy. We explicitly cap this to keep the
+ * "VLA" from being unbounded (see above). 10 bits leaves enough room for
+@@ -36,7 +48,7 @@ void *__builtin_alloca(size_t size);
+ if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \
+ &randomize_kstack_offset)) { \
+ u32 offset = raw_cpu_read(kstack_offset); \
+- u8 *ptr = __builtin_alloca(KSTACK_OFFSET_MAX(offset)); \
++ u8 *ptr = __kstack_alloca(KSTACK_OFFSET_MAX(offset)); \
+ /* Keep allocation even after "ptr" loses scope. */ \
+ asm volatile("" :: "r"(ptr) : "memory"); \
+ } \
+diff --git a/include/linux/rpmsg.h b/include/linux/rpmsg.h
+index d97dcd049f18f..a8dcf8a9ae885 100644
+--- a/include/linux/rpmsg.h
++++ b/include/linux/rpmsg.h
+@@ -231,7 +231,7 @@ static inline struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev
+ /* This shouldn't be possible */
+ WARN_ON(1);
+
+- return ERR_PTR(-ENXIO);
++ return NULL;
+ }
+
+ static inline int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index c1a927ddec646..ad7ff332a0ac8 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -1436,6 +1436,7 @@ struct task_struct {
+ int pagefault_disabled;
+ #ifdef CONFIG_MMU
+ struct task_struct *oom_reaper_list;
++ struct timer_list oom_reaper_timer;
+ #endif
+ #ifdef CONFIG_VMAP_STACK
+ struct vm_struct *stack_vm_area;
+@@ -1626,6 +1627,14 @@ static inline unsigned int task_state_index(struct task_struct *tsk)
+ if (tsk_state == TASK_IDLE)
+ state = TASK_REPORT_IDLE;
+
++ /*
++ * We're lying here, but rather than expose a completely new task state
++ * to userspace, we can make this appear as if the task has gone through
++ * a regular rt_mutex_lock() call.
++ */
++ if (tsk_state == TASK_RTLOCK_WAIT)
++ state = TASK_UNINTERRUPTIBLE;
++
+ return fls(state);
+ }
+
+@@ -1675,7 +1684,6 @@ extern struct pid *cad_pid;
+ #define PF_MEMALLOC 0x00000800 /* Allocating memory */
+ #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */
+ #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */
+-#define PF_USED_ASYNC 0x00004000 /* Used async_schedule*(), used by module init */
+ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */
+ #define PF_FROZEN 0x00010000 /* Frozen for system suspend */
+ #define PF_KSWAPD 0x00020000 /* I am kswapd */
+diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
+index 5561486fddef7..95fb7aaaec8de 100644
+--- a/include/linux/sched/mm.h
++++ b/include/linux/sched/mm.h
+@@ -106,6 +106,14 @@ static inline void mm_update_next_owner(struct mm_struct *mm)
+ #endif /* CONFIG_MEMCG */
+
+ #ifdef CONFIG_MMU
++#ifndef arch_get_mmap_end
++#define arch_get_mmap_end(addr) (TASK_SIZE)
++#endif
++
++#ifndef arch_get_mmap_base
++#define arch_get_mmap_base(addr, base) (base)
++#endif
++
+ extern void arch_pick_mmap_layout(struct mm_struct *mm,
+ struct rlimit *rlim_stack);
+ extern unsigned long
+diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
+index e5f4ce622ee61..9a707b555b0a0 100644
+--- a/include/linux/sched/signal.h
++++ b/include/linux/sched/signal.h
+@@ -338,6 +338,8 @@ extern int kill_pid(struct pid *pid, int sig, int priv);
+ extern __must_check bool do_notify_parent(struct task_struct *, int);
+ extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
+ extern void force_sig(int);
++extern void force_fatal_sig(int);
++extern void force_exit_sig(int);
+ extern int send_sig(int, struct task_struct *, int);
+ extern int zap_other_threads(struct task_struct *p);
+ extern struct sigqueue *sigqueue_alloc(void);
+diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
+index ef02be869cf28..8db25fcc1eba7 100644
+--- a/include/linux/sched/task.h
++++ b/include/linux/sched/task.h
+@@ -54,6 +54,7 @@ extern asmlinkage void schedule_tail(struct task_struct *prev);
+ extern void init_idle(struct task_struct *idle, int cpu);
+
+ extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
++extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
+ extern void sched_post_fork(struct task_struct *p);
+ extern void sched_dead(struct task_struct *p);
+
+@@ -157,7 +158,7 @@ static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
+ * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
+ * subscriptions and synchronises with wait4(). Also used in procfs. Also
+ * pins the final release of task.io_context. Also protects ->cpuset and
+- * ->cgroup.subsys[]. And ->vfork_done.
++ * ->cgroup.subsys[]. And ->vfork_done. And ->sysvshm.shm_clist.
+ *
+ * Nests both inside and outside of read_lock(&tasklist_lock).
+ * It must not be nested with write_lock_irq(&tasklist_lock),
+diff --git a/include/linux/sched/task_stack.h b/include/linux/sched/task_stack.h
+index 2413427e439c7..d10150587d819 100644
+--- a/include/linux/sched/task_stack.h
++++ b/include/linux/sched/task_stack.h
+@@ -25,7 +25,11 @@ static inline void *task_stack_page(const struct task_struct *task)
+
+ static inline unsigned long *end_of_stack(const struct task_struct *task)
+ {
++#ifdef CONFIG_STACK_GROWSUP
++ return (unsigned long *)((unsigned long)task->stack + THREAD_SIZE) - 1;
++#else
+ return task->stack;
++#endif
+ }
+
+ #elif !defined(__HAVE_THREAD_FUNCTIONS)
+diff --git a/include/linux/security.h b/include/linux/security.h
+index 5b7288521300b..da184e7b361f4 100644
+--- a/include/linux/security.h
++++ b/include/linux/security.h
+@@ -121,10 +121,12 @@ enum lockdown_reason {
+ LOCKDOWN_DEBUGFS,
+ LOCKDOWN_XMON_WR,
+ LOCKDOWN_BPF_WRITE_USER,
++ LOCKDOWN_DBG_WRITE_KERNEL,
+ LOCKDOWN_INTEGRITY_MAX,
+ LOCKDOWN_KCORE,
+ LOCKDOWN_KPROBES,
+ LOCKDOWN_BPF_READ_KERNEL,
++ LOCKDOWN_DBG_READ_KERNEL,
+ LOCKDOWN_PERF,
+ LOCKDOWN_TRACEFS,
+ LOCKDOWN_XMON_RW,
+@@ -258,13 +260,13 @@ extern int security_init(void);
+ extern int early_security_init(void);
+
+ /* Security operations */
+-int security_binder_set_context_mgr(struct task_struct *mgr);
+-int security_binder_transaction(struct task_struct *from,
+- struct task_struct *to);
+-int security_binder_transfer_binder(struct task_struct *from,
+- struct task_struct *to);
+-int security_binder_transfer_file(struct task_struct *from,
+- struct task_struct *to, struct file *file);
++int security_binder_set_context_mgr(const struct cred *mgr);
++int security_binder_transaction(const struct cred *from,
++ const struct cred *to);
++int security_binder_transfer_binder(const struct cred *from,
++ const struct cred *to);
++int security_binder_transfer_file(const struct cred *from,
++ const struct cred *to, struct file *file);
+ int security_ptrace_access_check(struct task_struct *child, unsigned int mode);
+ int security_ptrace_traceme(struct task_struct *parent);
+ int security_capget(struct task_struct *target,
+@@ -508,25 +510,25 @@ static inline int early_security_init(void)
+ return 0;
+ }
+
+-static inline int security_binder_set_context_mgr(struct task_struct *mgr)
++static inline int security_binder_set_context_mgr(const struct cred *mgr)
+ {
+ return 0;
+ }
+
+-static inline int security_binder_transaction(struct task_struct *from,
+- struct task_struct *to)
++static inline int security_binder_transaction(const struct cred *from,
++ const struct cred *to)
+ {
+ return 0;
+ }
+
+-static inline int security_binder_transfer_binder(struct task_struct *from,
+- struct task_struct *to)
++static inline int security_binder_transfer_binder(const struct cred *from,
++ const struct cred *to)
+ {
+ return 0;
+ }
+
+-static inline int security_binder_transfer_file(struct task_struct *from,
+- struct task_struct *to,
++static inline int security_binder_transfer_file(const struct cred *from,
++ const struct cred *to,
+ struct file *file)
+ {
+ return 0;
+@@ -1041,6 +1043,11 @@ static inline void security_transfer_creds(struct cred *new,
+ {
+ }
+
++static inline void security_cred_getsecid(const struct cred *c, u32 *secid)
++{
++ *secid = 0;
++}
++
+ static inline int security_kernel_act_as(struct cred *cred, u32 secid)
+ {
+ return 0;
+diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h
+index dd99569595fd3..5733890df64f5 100644
+--- a/include/linux/seq_file.h
++++ b/include/linux/seq_file.h
+@@ -194,7 +194,7 @@ static const struct file_operations __name ## _fops = { \
+ #define DEFINE_PROC_SHOW_ATTRIBUTE(__name) \
+ static int __name ## _open(struct inode *inode, struct file *file) \
+ { \
+- return single_open(file, __name ## _show, inode->i_private); \
++ return single_open(file, __name ## _show, PDE_DATA(inode)); \
+ } \
+ \
+ static const struct proc_ops __name ## _proc_ops = { \
+diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
+index c58cc142d23f4..8c32935e1059d 100644
+--- a/include/linux/serial_core.h
++++ b/include/linux/serial_core.h
+@@ -458,6 +458,8 @@ extern void uart_handle_cts_change(struct uart_port *uport,
+ extern void uart_insert_char(struct uart_port *port, unsigned int status,
+ unsigned int overrun, unsigned int ch, unsigned int flag);
+
++void uart_xchar_out(struct uart_port *uport, int offset);
++
+ #ifdef CONFIG_MAGIC_SYSRQ_SERIAL
+ #define SYSRQ_TIMEOUT (HZ * 5)
+
+diff --git a/include/linux/signal_types.h b/include/linux/signal_types.h
+index 34cb28b8f16ca..a70b2bdbf4d96 100644
+--- a/include/linux/signal_types.h
++++ b/include/linux/signal_types.h
+@@ -70,6 +70,9 @@ struct ksignal {
+ int sig;
+ };
+
++/* Used to kill the race between sigaction and forced signals */
++#define SA_IMMUTABLE 0x00800000
++
+ #ifndef __ARCH_UAPI_SA_FLAGS
+ #ifdef SA_RESTORER
+ #define __ARCH_UAPI_SA_FLAGS SA_RESTORER
+diff --git a/include/linux/siphash.h b/include/linux/siphash.h
+index bf21591a9e5e6..0bb5ecd507bef 100644
+--- a/include/linux/siphash.h
++++ b/include/linux/siphash.h
+@@ -27,9 +27,7 @@ static inline bool siphash_key_is_zero(const siphash_key_t *key)
+ }
+
+ u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key);
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key);
+-#endif
+
+ u64 siphash_1u64(const u64 a, const siphash_key_t *key);
+ u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key);
+@@ -82,10 +80,9 @@ static inline u64 ___siphash_aligned(const __le64 *data, size_t len,
+ static inline u64 siphash(const void *data, size_t len,
+ const siphash_key_t *key)
+ {
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+- if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
++ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
++ !IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
+ return __siphash_unaligned(data, len, key);
+-#endif
+ return ___siphash_aligned(data, len, key);
+ }
+
+@@ -96,10 +93,8 @@ typedef struct {
+
+ u32 __hsiphash_aligned(const void *data, size_t len,
+ const hsiphash_key_t *key);
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u32 __hsiphash_unaligned(const void *data, size_t len,
+ const hsiphash_key_t *key);
+-#endif
+
+ u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key);
+ u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key);
+@@ -135,11 +130,38 @@ static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len,
+ static inline u32 hsiphash(const void *data, size_t len,
+ const hsiphash_key_t *key)
+ {
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+- if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT))
++ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
++ !IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT))
+ return __hsiphash_unaligned(data, len, key);
+-#endif
+ return ___hsiphash_aligned(data, len, key);
+ }
+
++/*
++ * These macros expose the raw SipHash and HalfSipHash permutations.
++ * Do not use them directly! If you think you have a use for them,
++ * be sure to CC the maintainer of this file explaining why.
++ */
++
++#define SIPHASH_PERMUTATION(a, b, c, d) ( \
++ (a) += (b), (b) = rol64((b), 13), (b) ^= (a), (a) = rol64((a), 32), \
++ (c) += (d), (d) = rol64((d), 16), (d) ^= (c), \
++ (a) += (d), (d) = rol64((d), 21), (d) ^= (a), \
++ (c) += (b), (b) = rol64((b), 17), (b) ^= (c), (c) = rol64((c), 32))
++
++#define SIPHASH_CONST_0 0x736f6d6570736575ULL
++#define SIPHASH_CONST_1 0x646f72616e646f6dULL
++#define SIPHASH_CONST_2 0x6c7967656e657261ULL
++#define SIPHASH_CONST_3 0x7465646279746573ULL
++
++#define HSIPHASH_PERMUTATION(a, b, c, d) ( \
++ (a) += (b), (b) = rol32((b), 5), (b) ^= (a), (a) = rol32((a), 16), \
++ (c) += (d), (d) = rol32((d), 8), (d) ^= (c), \
++ (a) += (d), (d) = rol32((d), 7), (d) ^= (a), \
++ (c) += (b), (b) = rol32((b), 13), (b) ^= (c), (c) = rol32((c), 16))
++
++#define HSIPHASH_CONST_0 0U
++#define HSIPHASH_CONST_1 0U
++#define HSIPHASH_CONST_2 0x6c796765U
++#define HSIPHASH_CONST_3 0x74656462U
++
+ #endif /* _LINUX_SIPHASH_H */
+diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
+index 841e2f0f5240b..e213acaa91ecc 100644
+--- a/include/linux/skbuff.h
++++ b/include/linux/skbuff.h
+@@ -286,7 +286,10 @@ struct nf_bridge_info {
+ struct tc_skb_ext {
+ __u32 chain;
+ __u16 mru;
+- bool post_ct;
++ __u16 zone;
++ u8 post_ct:1;
++ u8 post_ct_snat:1;
++ u8 post_ct_dnat:1;
+ };
+ #endif
+
+@@ -1370,7 +1373,7 @@ skb_flow_dissect_ct(const struct sk_buff *skb,
+ struct flow_dissector *flow_dissector,
+ void *target_container,
+ u16 *ctinfo_map, size_t mapsize,
+- bool post_ct);
++ bool post_ct, u16 zone);
+ void
+ skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
+ struct flow_dissector *flow_dissector,
+@@ -1433,6 +1436,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb)
+ {
+ return skb->end;
+ }
++
++static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
++{
++ skb->end = offset;
++}
+ #else
+ static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
+ {
+@@ -1443,6 +1451,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb)
+ {
+ return skb->end - skb->head;
+ }
++
++static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
++{
++ skb->end = skb->head + offset;
++}
+ #endif
+
+ /* Internal */
+@@ -1671,6 +1684,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
+ return 0;
+ }
+
++/* This variant of skb_unclone() makes sure skb->truesize
++ * and skb_end_offset() are not changed, whenever a new skb->head is needed.
++ *
++ * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
++ * when various debugging features are in place.
++ */
++int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
++static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
++{
++ might_sleep_if(gfpflags_allow_blocking(pri));
++
++ if (skb_cloned(skb))
++ return __skb_unclone_keeptruesize(skb, pri);
++ return 0;
++}
++
+ /**
+ * skb_header_cloned - is the header a clone
+ * @skb: buffer to check
+diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
+index 1ce9a9eb223b6..73bedd128d529 100644
+--- a/include/linux/skmsg.h
++++ b/include/linux/skmsg.h
+@@ -310,21 +310,16 @@ static inline void sock_drop(struct sock *sk, struct sk_buff *skb)
+ kfree_skb(skb);
+ }
+
+-static inline void drop_sk_msg(struct sk_psock *psock, struct sk_msg *msg)
+-{
+- if (msg->skb)
+- sock_drop(psock->sk, msg->skb);
+- kfree(msg);
+-}
+-
+ static inline void sk_psock_queue_msg(struct sk_psock *psock,
+ struct sk_msg *msg)
+ {
+ spin_lock_bh(&psock->ingress_lock);
+ if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
+ list_add_tail(&msg->list, &psock->ingress_msg);
+- else
+- drop_sk_msg(psock, msg);
++ else {
++ sk_msg_free(psock->sk, msg);
++ kfree(msg);
++ }
+ spin_unlock_bh(&psock->ingress_lock);
+ }
+
+@@ -509,8 +504,22 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock)
+
+ #if IS_ENABLED(CONFIG_NET_SOCK_MSG)
+
+-/* We only have one bit so far. */
+-#define BPF_F_PTR_MASK ~(BPF_F_INGRESS)
++#define BPF_F_STRPARSER (1UL << 1)
++
++/* We only have two bits so far. */
++#define BPF_F_PTR_MASK ~(BPF_F_INGRESS | BPF_F_STRPARSER)
++
++static inline bool skb_bpf_strparser(const struct sk_buff *skb)
++{
++ unsigned long sk_redir = skb->_sk_redir;
++
++ return sk_redir & BPF_F_STRPARSER;
++}
++
++static inline void skb_bpf_set_strparser(struct sk_buff *skb)
++{
++ skb->_sk_redir |= BPF_F_STRPARSER;
++}
+
+ static inline bool skb_bpf_ingress(const struct sk_buff *skb)
+ {
+diff --git a/include/linux/soc/qcom/qcom_aoss.h b/include/linux/soc/qcom/qcom_aoss.h
+new file mode 100644
+index 0000000000000..3c2a82e606f81
+--- /dev/null
++++ b/include/linux/soc/qcom/qcom_aoss.h
+@@ -0,0 +1,38 @@
++/* SPDX-License-Identifier: GPL-2.0-only */
++/*
++ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
++ */
++
++#ifndef __QCOM_AOSS_H__
++#define __QCOM_AOSS_H__
++
++#include <linux/err.h>
++#include <linux/device.h>
++
++struct qmp;
++
++#if IS_ENABLED(CONFIG_QCOM_AOSS_QMP)
++
++int qmp_send(struct qmp *qmp, const void *data, size_t len);
++struct qmp *qmp_get(struct device *dev);
++void qmp_put(struct qmp *qmp);
++
++#else
++
++static inline int qmp_send(struct qmp *qmp, const void *data, size_t len)
++{
++ return -ENODEV;
++}
++
++static inline struct qmp *qmp_get(struct device *dev)
++{
++ return ERR_PTR(-ENODEV);
++}
++
++static inline void qmp_put(struct qmp *qmp)
++{
++}
++
++#endif
++
++#endif
+diff --git a/include/linux/soc/ti/ti_sci_protocol.h b/include/linux/soc/ti/ti_sci_protocol.h
+index 0aad7009b50e6..bd0d11af76c5e 100644
+--- a/include/linux/soc/ti/ti_sci_protocol.h
++++ b/include/linux/soc/ti/ti_sci_protocol.h
+@@ -645,7 +645,7 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
+
+ static inline struct ti_sci_resource *
+ devm_ti_sci_get_resource(const struct ti_sci_handle *handle, struct device *dev,
+- u32 dev_id, u32 sub_type);
++ u32 dev_id, u32 sub_type)
+ {
+ return ERR_PTR(-EINVAL);
+ }
+diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h
+index 6bb4bc1a5f545..22919a94ca19d 100644
+--- a/include/linux/stackdepot.h
++++ b/include/linux/stackdepot.h
+@@ -19,8 +19,6 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
+ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
+ unsigned long **entries);
+
+-unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries);
+-
+ #ifdef CONFIG_STACKDEPOT
+ int stack_depot_init(void);
+ #else
+diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h
+index 9edecb494e9e2..bef158815e83d 100644
+--- a/include/linux/stacktrace.h
++++ b/include/linux/stacktrace.h
+@@ -21,6 +21,7 @@ unsigned int stack_trace_save_tsk(struct task_struct *task,
+ unsigned int stack_trace_save_regs(struct pt_regs *regs, unsigned long *store,
+ unsigned int size, unsigned int skipnr);
+ unsigned int stack_trace_save_user(unsigned long *store, unsigned int size);
++unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries);
+
+ /* Internal interfaces. Do not use in generic code */
+ #ifdef CONFIG_ARCH_STACKWALK
+diff --git a/include/linux/static_call.h b/include/linux/static_call.h
+index 3e56a9751c062..fcc5b48989b3c 100644
+--- a/include/linux/static_call.h
++++ b/include/linux/static_call.h
+@@ -248,10 +248,7 @@ static inline int static_call_text_reserved(void *start, void *end)
+ return 0;
+ }
+
+-static inline long __static_call_return0(void)
+-{
+- return 0;
+-}
++extern long __static_call_return0(void);
+
+ #define EXPORT_STATIC_CALL(name) \
+ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
+diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
+index a6f03b36fc4f7..48d015ed21752 100644
+--- a/include/linux/stmmac.h
++++ b/include/linux/stmmac.h
+@@ -233,6 +233,7 @@ struct plat_stmmacenet_data {
+ int (*clks_config)(void *priv, bool enabled);
+ int (*crosststamp)(ktime_t *device, struct system_counterval_t *system,
+ void *ctx);
++ void (*dump_debug_regs)(void *priv);
+ void *bsp_priv;
+ struct clk *stmmac_clk;
+ struct clk *pclk;
+@@ -268,5 +269,6 @@ struct plat_stmmacenet_data {
+ int msi_rx_base_vec;
+ int msi_tx_base_vec;
+ bool use_phy_wol;
++ bool sph_disable;
+ };
+ #endif
+diff --git a/include/linux/string.h b/include/linux/string.h
+index 5e96d656be7ae..d68097b4f600b 100644
+--- a/include/linux/string.h
++++ b/include/linux/string.h
+@@ -262,23 +262,8 @@ void __write_overflow(void) __compiletime_error("detected write beyond size of o
+ #include <linux/fortify-string.h>
+ #endif
+
+-/**
+- * memcpy_and_pad - Copy one buffer to another with padding
+- * @dest: Where to copy to
+- * @dest_len: The destination buffer size
+- * @src: Where to copy from
+- * @count: The number of bytes to copy
+- * @pad: Character to use for padding if space is left in destination.
+- */
+-static inline void memcpy_and_pad(void *dest, size_t dest_len,
+- const void *src, size_t count, int pad)
+-{
+- if (dest_len > count) {
+- memcpy(dest, src, count);
+- memset(dest + count, pad, dest_len - count);
+- } else
+- memcpy(dest, src, dest_len);
+-}
++void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count,
++ int pad);
+
+ /**
+ * str_has_prefix - Test if a string has a given prefix
+diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
+index a4661646adc9c..9fcf5ffc4f9ad 100644
+--- a/include/linux/sunrpc/clnt.h
++++ b/include/linux/sunrpc/clnt.h
+@@ -159,6 +159,7 @@ struct rpc_add_xprt_test {
+ #define RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT (1UL << 9)
+ #define RPC_CLNT_CREATE_SOFTERR (1UL << 10)
+ #define RPC_CLNT_CREATE_REUSEPORT (1UL << 11)
++#define RPC_CLNT_CREATE_CONNECTED (1UL << 12)
+
+ struct rpc_clnt *rpc_create(struct rpc_create_args *args);
+ struct rpc_clnt *rpc_bind_new_program(struct rpc_clnt *,
+diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
+index 064c96157d1f0..01f09adccc63c 100644
+--- a/include/linux/sunrpc/svc.h
++++ b/include/linux/sunrpc/svc.h
+@@ -384,6 +384,7 @@ struct svc_deferred_req {
+ size_t addrlen;
+ struct sockaddr_storage daddr; /* where reply must come from */
+ size_t daddrlen;
++ void *xprt_ctxt;
+ struct cache_deferred_req handle;
+ size_t xprt_hlen;
+ int argslen;
+@@ -532,8 +533,7 @@ int svc_encode_result_payload(struct svc_rqst *rqstp,
+ unsigned int offset,
+ unsigned int length);
+ unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
+- struct page **pages,
+- struct kvec *first, size_t total);
++ struct xdr_buf *payload);
+ char *svc_fill_symlink_pathname(struct svc_rqst *rqstp,
+ struct kvec *first, void *p,
+ size_t total);
+diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h
+index b519609af1d02..4417f667c757e 100644
+--- a/include/linux/sunrpc/xdr.h
++++ b/include/linux/sunrpc/xdr.h
+@@ -731,6 +731,8 @@ xdr_stream_decode_uint32_array(struct xdr_stream *xdr,
+
+ if (unlikely(xdr_stream_decode_u32(xdr, &len) < 0))
+ return -EBADMSG;
++ if (len > SIZE_MAX / sizeof(*p))
++ return -EBADMSG;
+ p = xdr_inline_decode(xdr, len * sizeof(*p));
+ if (unlikely(!p))
+ return -EBADMSG;
+diff --git a/include/linux/sunrpc/xprtsock.h b/include/linux/sunrpc/xprtsock.h
+index 8c2a712cb2420..689062afdd610 100644
+--- a/include/linux/sunrpc/xprtsock.h
++++ b/include/linux/sunrpc/xprtsock.h
+@@ -89,5 +89,6 @@ struct sock_xprt {
+ #define XPRT_SOCK_WAKE_WRITE (5)
+ #define XPRT_SOCK_WAKE_PENDING (6)
+ #define XPRT_SOCK_WAKE_DISCONNECT (7)
++#define XPRT_SOCK_CONNECT_SENT (8)
+
+ #endif /* _LINUX_SUNRPC_XPRTSOCK_H */
+diff --git a/include/linux/surface_aggregator/controller.h b/include/linux/surface_aggregator/controller.h
+index 068e1982ad371..74bfdffaf7b0e 100644
+--- a/include/linux/surface_aggregator/controller.h
++++ b/include/linux/surface_aggregator/controller.h
+@@ -792,8 +792,8 @@ enum ssam_event_mask {
+ #define SSAM_EVENT_REGISTRY_KIP \
+ SSAM_EVENT_REGISTRY(SSAM_SSH_TC_KIP, 0x02, 0x27, 0x28)
+
+-#define SSAM_EVENT_REGISTRY_REG \
+- SSAM_EVENT_REGISTRY(SSAM_SSH_TC_REG, 0x02, 0x01, 0x02)
++#define SSAM_EVENT_REGISTRY_REG(tid)\
++ SSAM_EVENT_REGISTRY(SSAM_SSH_TC_REG, tid, 0x01, 0x02)
+
+ /**
+ * enum ssam_event_notifier_flags - Flags for event notifiers.
+diff --git a/include/linux/suspend.h b/include/linux/suspend.h
+index 8af13ba60c7e4..4bcd65679cee0 100644
+--- a/include/linux/suspend.h
++++ b/include/linux/suspend.h
+@@ -430,15 +430,7 @@ struct platform_hibernation_ops {
+
+ #ifdef CONFIG_HIBERNATION
+ /* kernel/power/snapshot.c */
+-extern void __register_nosave_region(unsigned long b, unsigned long e, int km);
+-static inline void __init register_nosave_region(unsigned long b, unsigned long e)
+-{
+- __register_nosave_region(b, e, 0);
+-}
+-static inline void __init register_nosave_region_late(unsigned long b, unsigned long e)
+-{
+- __register_nosave_region(b, e, 1);
+-}
++extern void register_nosave_region(unsigned long b, unsigned long e);
+ extern int swsusp_page_is_forbidden(struct page *);
+ extern void swsusp_set_page_free(struct page *);
+ extern void swsusp_unset_page_free(struct page *);
+@@ -457,7 +449,6 @@ int pfn_is_nosave(unsigned long pfn);
+ int hibernate_quiet_exec(int (*func)(void *data), void *data);
+ #else /* CONFIG_HIBERNATION */
+ static inline void register_nosave_region(unsigned long b, unsigned long e) {}
+-static inline void register_nosave_region_late(unsigned long b, unsigned long e) {}
+ static inline int swsusp_page_is_forbidden(struct page *p) { return 0; }
+ static inline void swsusp_set_page_free(struct page *p) {}
+ static inline void swsusp_unset_page_free(struct page *p) {}
+@@ -505,14 +496,14 @@ extern void ksys_sync_helper(void);
+
+ /* drivers/base/power/wakeup.c */
+ extern bool events_check_enabled;
+-extern unsigned int pm_wakeup_irq;
+ extern suspend_state_t pm_suspend_target_state;
+
+ extern bool pm_wakeup_pending(void);
+ extern void pm_system_wakeup(void);
+ extern void pm_system_cancel_wakeup(void);
+-extern void pm_wakeup_clear(bool reset);
++extern void pm_wakeup_clear(unsigned int irq_number);
+ extern void pm_system_irq_wakeup(unsigned int irq_number);
++extern unsigned int pm_wakeup_irq(void);
+ extern bool pm_get_wakeup_count(unsigned int *count, bool block);
+ extern bool pm_save_wakeup_count(unsigned int count);
+ extern void pm_wakep_autosleep_enabled(bool set);
+diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
+index b0cb2a9973f49..569272871375c 100644
+--- a/include/linux/swiotlb.h
++++ b/include/linux/swiotlb.h
+@@ -45,7 +45,8 @@ extern void __init swiotlb_update_mem_attributes(void);
+
+ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
+ size_t mapping_size, size_t alloc_size,
+- enum dma_data_direction dir, unsigned long attrs);
++ unsigned int alloc_aligned_mask, enum dma_data_direction dir,
++ unsigned long attrs);
+
+ extern void swiotlb_tbl_unmap_single(struct device *hwdev,
+ phys_addr_t tlb_addr,
+diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h
+index 3ebfea0781f10..38b701b7af4cf 100644
+--- a/include/linux/tee_drv.h
++++ b/include/linux/tee_drv.h
+@@ -195,7 +195,7 @@ int tee_session_calc_client_uuid(uuid_t *uuid, u32 connection_method,
+ * @offset: offset of buffer in user space
+ * @pages: locked pages from userspace
+ * @num_pages: number of locked pages
+- * @dmabuf: dmabuf used to for exporting to user space
++ * @refcount: reference counter
+ * @flags: defined by TEE_SHM_* in tee_drv.h
+ * @id: unique id of a shared memory object on this device
+ *
+@@ -210,7 +210,7 @@ struct tee_shm {
+ unsigned int offset;
+ struct page **pages;
+ size_t num_pages;
+- struct dma_buf *dmabuf;
++ refcount_t refcount;
+ u32 flags;
+ int id;
+ };
+@@ -582,4 +582,18 @@ struct tee_client_driver {
+ #define to_tee_client_driver(d) \
+ container_of(d, struct tee_client_driver, driver)
+
++/**
++ * teedev_open() - Open a struct tee_device
++ * @teedev: Device to open
++ *
++ * @return a pointer to struct tee_context on success or an ERR_PTR on failure.
++ */
++struct tee_context *teedev_open(struct tee_device *teedev);
++
++/**
++ * teedev_close_context() - closes a struct tee_context
++ * @ctx: The struct tee_context to close
++ */
++void teedev_close_context(struct tee_context *ctx);
++
+ #endif /*__TEE_DRV_H*/
+diff --git a/include/linux/timex.h b/include/linux/timex.h
+index 059b18eb1f1fa..3871b06bd302c 100644
+--- a/include/linux/timex.h
++++ b/include/linux/timex.h
+@@ -62,6 +62,8 @@
+ #include <linux/types.h>
+ #include <linux/param.h>
+
++unsigned long random_get_entropy_fallback(void);
++
+ #include <asm/timex.h>
+
+ #ifndef random_get_entropy
+@@ -74,8 +76,14 @@
+ *
+ * By default we use get_cycles() for this purpose, but individual
+ * architectures may override this in their asm/timex.h header file.
++ * If a given arch does not have get_cycles(), then we fallback to
++ * using random_get_entropy_fallback().
+ */
+-#define random_get_entropy() get_cycles()
++#ifdef get_cycles
++#define random_get_entropy() ((unsigned long)get_cycles())
++#else
++#define random_get_entropy() random_get_entropy_fallback()
++#endif
+ #endif
+
+ /*
+diff --git a/include/linux/tpm.h b/include/linux/tpm.h
+index aa11fe323c56b..12d827734686d 100644
+--- a/include/linux/tpm.h
++++ b/include/linux/tpm.h
+@@ -269,6 +269,7 @@ enum tpm2_cc_attrs {
+ #define TPM_VID_INTEL 0x8086
+ #define TPM_VID_WINBOND 0x1050
+ #define TPM_VID_STM 0x104A
++#define TPM_VID_ATML 0x1114
+
+ enum tpm_chip_flags {
+ TPM_CHIP_FLAG_TPM2 = BIT(1),
+diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
+index 3e475eeb5a995..57113190448c6 100644
+--- a/include/linux/trace_events.h
++++ b/include/linux/trace_events.h
+@@ -673,7 +673,7 @@ struct trace_event_file {
+
+ #define PERF_MAX_TRACE_SIZE 2048
+
+-#define MAX_FILTER_STR_VAL 256 /* Should handle KSYM_SYMBOL_LEN */
++#define MAX_FILTER_STR_VAL 256U /* Should handle KSYM_SYMBOL_LEN */
+
+ enum event_trigger_type {
+ ETT_NONE = (0),
+diff --git a/include/linux/uio.h b/include/linux/uio.h
+index 207101a9c5c32..6350354f97e90 100644
+--- a/include/linux/uio.h
++++ b/include/linux/uio.h
+@@ -35,6 +35,7 @@ struct iov_iter_state {
+
+ struct iov_iter {
+ u8 iter_type;
++ bool nofault;
+ bool data_source;
+ size_t iov_offset;
+ size_t count;
+@@ -133,7 +134,8 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
+ size_t bytes, struct iov_iter *i);
+ void iov_iter_advance(struct iov_iter *i, size_t bytes);
+ void iov_iter_revert(struct iov_iter *i, size_t bytes);
+-int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes);
++size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
++size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);
+ size_t iov_iter_single_seg_count(const struct iov_iter *i);
+ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i);
+diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h
+index 2c1fc9212cf28..548a028f2dabb 100644
+--- a/include/linux/usb/hcd.h
++++ b/include/linux/usb/hcd.h
+@@ -124,7 +124,6 @@ struct usb_hcd {
+ #define HCD_FLAG_RH_RUNNING 5 /* root hub is running? */
+ #define HCD_FLAG_DEAD 6 /* controller has died? */
+ #define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */
+-#define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */
+
+ /* The flags can be tested using these macros; they are likely to
+ * be slightly faster than test_bit().
+@@ -135,7 +134,6 @@ struct usb_hcd {
+ #define HCD_WAKEUP_PENDING(hcd) ((hcd)->flags & (1U << HCD_FLAG_WAKEUP_PENDING))
+ #define HCD_RH_RUNNING(hcd) ((hcd)->flags & (1U << HCD_FLAG_RH_RUNNING))
+ #define HCD_DEAD(hcd) ((hcd)->flags & (1U << HCD_FLAG_DEAD))
+-#define HCD_DEFER_RH_REGISTER(hcd) ((hcd)->flags & (1U << HCD_FLAG_DEFER_RH_REGISTER))
+
+ /*
+ * Specifies if interfaces are authorized by default
+diff --git a/include/linux/usb/role.h b/include/linux/usb/role.h
+index 031f148ab3734..b5deafd91f67b 100644
+--- a/include/linux/usb/role.h
++++ b/include/linux/usb/role.h
+@@ -91,6 +91,12 @@ fwnode_usb_role_switch_get(struct fwnode_handle *node)
+
+ static inline void usb_role_switch_put(struct usb_role_switch *sw) { }
+
++static inline struct usb_role_switch *
++usb_role_switch_find_by_fwnode(const struct fwnode_handle *fwnode)
++{
++ return NULL;
++}
++
+ static inline struct usb_role_switch *
+ usb_role_switch_register(struct device *parent,
+ const struct usb_role_switch_desc *desc)
+diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
+index ef9a44b6cf5d5..6e5db4edc3359 100644
+--- a/include/linux/vfio_pci_core.h
++++ b/include/linux/vfio_pci_core.h
+@@ -133,6 +133,8 @@ struct vfio_pci_core_device {
+ struct mutex ioeventfds_lock;
+ struct list_head ioeventfds_list;
+ struct vfio_pci_vf_token *vf_token;
++ struct list_head sriov_pfs_item;
++ struct vfio_pci_core_device *sriov_pf_core_dev;
+ struct notifier_block nb;
+ struct mutex vma_lock;
+ struct list_head vma_list;
+@@ -159,8 +161,17 @@ extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
+ extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite);
+
++#ifdef CONFIG_VFIO_PCI_VGA
+ extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite);
++#else
++static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev,
++ char __user *buf, size_t count,
++ loff_t *ppos, bool iswrite)
++{
++ return -EINVAL;
++}
++#endif
+
+ extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
+ uint64_t data, int count, int fd);
+diff --git a/include/linux/virtio.h b/include/linux/virtio.h
+index 41edbc01ffa40..1af8d65d4c8f7 100644
+--- a/include/linux/virtio.h
++++ b/include/linux/virtio.h
+@@ -133,7 +133,6 @@ bool is_virtio_device(struct device *dev);
+ void virtio_break_device(struct virtio_device *dev);
+
+ void virtio_config_changed(struct virtio_device *dev);
+-int virtio_finalize_features(struct virtio_device *dev);
+ #ifdef CONFIG_PM_SLEEP
+ int virtio_device_freeze(struct virtio_device *dev);
+ int virtio_device_restore(struct virtio_device *dev);
+diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
+index 8519b3ae5d52e..b341dd62aa4da 100644
+--- a/include/linux/virtio_config.h
++++ b/include/linux/virtio_config.h
+@@ -62,8 +62,9 @@ struct virtio_shm_region {
+ * Returns the first 64 feature bits (all we currently need).
+ * @finalize_features: confirm what device features we'll be using.
+ * vdev: the virtio_device
+- * This gives the final feature bits for the device: it can change
++ * This sends the driver feature bits to the device: it can change
+ * the dev->feature bits if it wants.
++ * Note: despite the name this can be called any number of times.
+ * Returns 0 on success or error status
+ * @bus_name: return the bus name associated with the device (optional)
+ * vdev: the virtio_device
+diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h
+index b465f8f3e554f..a960de68ac69e 100644
+--- a/include/linux/virtio_net.h
++++ b/include/linux/virtio_net.h
+@@ -7,9 +7,27 @@
+ #include <uapi/linux/udp.h>
+ #include <uapi/linux/virtio_net.h>
+
++static inline bool virtio_net_hdr_match_proto(__be16 protocol, __u8 gso_type)
++{
++ switch (gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
++ case VIRTIO_NET_HDR_GSO_TCPV4:
++ return protocol == cpu_to_be16(ETH_P_IP);
++ case VIRTIO_NET_HDR_GSO_TCPV6:
++ return protocol == cpu_to_be16(ETH_P_IPV6);
++ case VIRTIO_NET_HDR_GSO_UDP:
++ return protocol == cpu_to_be16(ETH_P_IP) ||
++ protocol == cpu_to_be16(ETH_P_IPV6);
++ default:
++ return false;
++ }
++}
++
+ static inline int virtio_net_hdr_set_proto(struct sk_buff *skb,
+ const struct virtio_net_hdr *hdr)
+ {
++ if (skb->protocol)
++ return 0;
++
+ switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+ case VIRTIO_NET_HDR_GSO_TCPV4:
+ case VIRTIO_NET_HDR_GSO_UDP:
+@@ -88,9 +106,12 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb,
+ if (!skb->protocol) {
+ __be16 protocol = dev_parse_header_protocol(skb);
+
+- virtio_net_hdr_set_proto(skb, hdr);
+- if (protocol && protocol != skb->protocol)
++ if (!protocol)
++ virtio_net_hdr_set_proto(skb, hdr);
++ else if (!virtio_net_hdr_match_proto(protocol, hdr->gso_type))
+ return -EINVAL;
++ else
++ skb->protocol = protocol;
+ }
+ retry:
+ if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
+@@ -120,10 +141,15 @@ retry:
+
+ if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+ u16 gso_size = __virtio16_to_cpu(little_endian, hdr->gso_size);
++ unsigned int nh_off = p_off;
+ struct skb_shared_info *shinfo = skb_shinfo(skb);
+
++ /* UFO may not include transport header in gso_size. */
++ if (gso_type & SKB_GSO_UDP)
++ nh_off -= thlen;
++
+ /* Too small packets are not really GSO ones. */
+- if (skb->len - p_off > gso_size) {
++ if (skb->len - nh_off > gso_size) {
+ shinfo->gso_size = gso_size;
+ shinfo->gso_type = gso_type;
+
+diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
+index 671d402c3778f..4fe9e885bbfac 100644
+--- a/include/linux/vmalloc.h
++++ b/include/linux/vmalloc.h
+@@ -28,6 +28,13 @@ struct notifier_block; /* in notifier.h */
+ #define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */
+ #define VM_NO_HUGE_VMAP 0x00000400 /* force PAGE_SIZE pte mapping */
+
++#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
++ !defined(CONFIG_KASAN_VMALLOC)
++#define VM_DEFER_KMEMLEAK 0x00000800 /* defer kmemleak object creation */
++#else
++#define VM_DEFER_KMEMLEAK 0
++#endif
++
+ /*
+ * VM_KASAN is used slightly differently depending on CONFIG_KASAN_VMALLOC.
+ *
+diff --git a/include/linux/wait.h b/include/linux/wait.h
+index 93dab0e9580f8..d22cf2985b8fd 100644
+--- a/include/linux/wait.h
++++ b/include/linux/wait.h
+@@ -217,6 +217,7 @@ void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void
+ void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
+ void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
+ void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
++void __wake_up_pollfree(struct wait_queue_head *wq_head);
+
+ #define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
+ #define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
+@@ -245,6 +246,31 @@ void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
+ #define wake_up_interruptible_sync_poll_locked(x, m) \
+ __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
+
++/**
++ * wake_up_pollfree - signal that a polled waitqueue is going away
++ * @wq_head: the wait queue head
++ *
++ * In the very rare cases where a ->poll() implementation uses a waitqueue whose
++ * lifetime is tied to a task rather than to the 'struct file' being polled,
++ * this function must be called before the waitqueue is freed so that
++ * non-blocking polls (e.g. epoll) are notified that the queue is going away.
++ *
++ * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
++ * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
++ */
++static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
++{
++ /*
++ * For performance reasons, we don't always take the queue lock here.
++ * Therefore, we might race with someone removing the last entry from
++ * the queue, and proceed while they still hold the queue lock.
++ * However, rcu_read_lock() is required to be held in such cases, so we
++ * can safely proceed with an RCU-delayed free.
++ */
++ if (waitqueue_active(wq_head))
++ __wake_up_pollfree(wq_head);
++}
++
+ #define ___wait_cond_timeout(condition) \
+ ({ \
+ bool __cond = (condition); \
+diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h
+index c994d1b2cdbaa..3b9a40ae8bdba 100644
+--- a/include/linux/watch_queue.h
++++ b/include/linux/watch_queue.h
+@@ -28,7 +28,8 @@ struct watch_type_filter {
+ struct watch_filter {
+ union {
+ struct rcu_head rcu;
+- unsigned long type_filter[2]; /* Bitmask of accepted types */
++ /* Bitmask of accepted types */
++ DECLARE_BITMAP(type_filter, WATCH_TYPE__NR);
+ };
+ u32 nr_filters; /* Number of filters */
+ struct watch_type_filter filters[];
+diff --git a/include/media/cec.h b/include/media/cec.h
+index 208c9613c07eb..77346f757036d 100644
+--- a/include/media/cec.h
++++ b/include/media/cec.h
+@@ -26,13 +26,17 @@
+ * @dev: cec device
+ * @cdev: cec character device
+ * @minor: device node minor number
++ * @lock: lock to serialize open/release and registration
+ * @registered: the device was correctly registered
+ * @unregistered: the device was unregistered
++ * @lock_fhs: lock to control access to @fhs
+ * @fhs: the list of open filehandles (cec_fh)
+- * @lock: lock to control access to this structure
+ *
+ * This structure represents a cec-related device node.
+ *
++ * To add or remove filehandles from @fhs the @lock must be taken first,
++ * followed by @lock_fhs. It is safe to access @fhs if either lock is held.
++ *
+ * The @parent is a physical device. It must be set by core or device drivers
+ * before registering the node.
+ */
+@@ -43,10 +47,13 @@ struct cec_devnode {
+
+ /* device info */
+ int minor;
++ /* serialize open/release and registration */
++ struct mutex lock;
+ bool registered;
+ bool unregistered;
++ /* protect access to fhs */
++ struct mutex lock_fhs;
+ struct list_head fhs;
+- struct mutex lock;
+ };
+
+ struct cec_adapter;
+diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
+index 12955cb460d23..3b5986cee0739 100644
+--- a/include/media/videobuf2-core.h
++++ b/include/media/videobuf2-core.h
+@@ -46,6 +46,7 @@ enum vb2_memory {
+
+ struct vb2_fileio_data;
+ struct vb2_threadio_data;
++struct vb2_buffer;
+
+ /**
+ * struct vb2_mem_ops - memory handling/memory allocator operations.
+@@ -53,10 +54,8 @@ struct vb2_threadio_data;
+ * return ERR_PTR() on failure or a pointer to allocator private,
+ * per-buffer data on success; the returned private structure
+ * will then be passed as @buf_priv argument to other ops in this
+- * structure. Additional gfp_flags to use when allocating the
+- * are also passed to this operation. These flags are from the
+- * gfp_flags field of vb2_queue. The size argument to this function
+- * shall be *page aligned*.
++ * structure. The size argument to this function shall be
++ * *page aligned*.
+ * @put: inform the allocator that the buffer will no longer be used;
+ * usually will result in the allocator freeing the buffer (if
+ * no other users of this buffer are present); the @buf_priv
+@@ -117,31 +116,33 @@ struct vb2_threadio_data;
+ * map_dmabuf, unmap_dmabuf.
+ */
+ struct vb2_mem_ops {
+- void *(*alloc)(struct device *dev, unsigned long attrs,
+- unsigned long size,
+- enum dma_data_direction dma_dir,
+- gfp_t gfp_flags);
++ void *(*alloc)(struct vb2_buffer *vb,
++ struct device *dev,
++ unsigned long size);
+ void (*put)(void *buf_priv);
+- struct dma_buf *(*get_dmabuf)(void *buf_priv, unsigned long flags);
+-
+- void *(*get_userptr)(struct device *dev, unsigned long vaddr,
+- unsigned long size,
+- enum dma_data_direction dma_dir);
++ struct dma_buf *(*get_dmabuf)(struct vb2_buffer *vb,
++ void *buf_priv,
++ unsigned long flags);
++
++ void *(*get_userptr)(struct vb2_buffer *vb,
++ struct device *dev,
++ unsigned long vaddr,
++ unsigned long size);
+ void (*put_userptr)(void *buf_priv);
+
+ void (*prepare)(void *buf_priv);
+ void (*finish)(void *buf_priv);
+
+- void *(*attach_dmabuf)(struct device *dev,
++ void *(*attach_dmabuf)(struct vb2_buffer *vb,
++ struct device *dev,
+ struct dma_buf *dbuf,
+- unsigned long size,
+- enum dma_data_direction dma_dir);
++ unsigned long size);
+ void (*detach_dmabuf)(void *buf_priv);
+ int (*map_dmabuf)(void *buf_priv);
+ void (*unmap_dmabuf)(void *buf_priv);
+
+- void *(*vaddr)(void *buf_priv);
+- void *(*cookie)(void *buf_priv);
++ void *(*vaddr)(struct vb2_buffer *vb, void *buf_priv);
++ void *(*cookie)(struct vb2_buffer *vb, void *buf_priv);
+
+ unsigned int (*num_users)(void *buf_priv);
+
+diff --git a/include/memory/renesas-rpc-if.h b/include/memory/renesas-rpc-if.h
+index e3e770f76f349..15dd0076c2936 100644
+--- a/include/memory/renesas-rpc-if.h
++++ b/include/memory/renesas-rpc-if.h
+@@ -59,12 +59,14 @@ struct rpcif_op {
+
+ struct rpcif {
+ struct device *dev;
++ void __iomem *base;
+ void __iomem *dirmap;
+ struct regmap *regmap;
+ struct reset_control *rstc;
+ size_t size;
+ enum rpcif_data_dir dir;
+ u8 bus_size;
++ u8 xfer_size;
+ void *buffer;
+ u32 xferlen;
+ u32 smcr;
+diff --git a/include/net/addrconf.h b/include/net/addrconf.h
+index 78ea3e332688f..59940e230b782 100644
+--- a/include/net/addrconf.h
++++ b/include/net/addrconf.h
+@@ -6,6 +6,8 @@
+ #define RTR_SOLICITATION_INTERVAL (4*HZ)
+ #define RTR_SOLICITATION_MAX_INTERVAL (3600*HZ) /* 1 hour */
+
++#define MIN_VALID_LIFETIME (2*3600) /* 2 hours */
++
+ #define TEMP_VALID_LIFETIME (7*86400)
+ #define TEMP_PREFERRED_LIFETIME (86400)
+ #define REGEN_MAX_RETRY (3)
+@@ -107,8 +109,6 @@ struct inet6_ifaddr *ipv6_get_ifaddr(struct net *net,
+ int ipv6_dev_get_saddr(struct net *net, const struct net_device *dev,
+ const struct in6_addr *daddr, unsigned int srcprefs,
+ struct in6_addr *saddr);
+-int __ipv6_get_lladdr(struct inet6_dev *idev, struct in6_addr *addr,
+- u32 banned_flags);
+ int ipv6_get_lladdr(struct net_device *dev, struct in6_addr *addr,
+ u32 banned_flags);
+ bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
+diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h
+index ab207677e0a8b..f742e50207fbd 100644
+--- a/include/net/af_vsock.h
++++ b/include/net/af_vsock.h
+@@ -205,7 +205,8 @@ struct sock *vsock_find_bound_socket(struct sockaddr_vm *addr);
+ struct sock *vsock_find_connected_socket(struct sockaddr_vm *src,
+ struct sockaddr_vm *dst);
+ void vsock_remove_sock(struct vsock_sock *vsk);
+-void vsock_for_each_connected_socket(void (*fn)(struct sock *sk));
++void vsock_for_each_connected_socket(struct vsock_transport *transport,
++ void (*fn)(struct sock *sk));
+ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk);
+ bool vsock_find_cid(unsigned int cid);
+
+diff --git a/include/net/arp.h b/include/net/arp.h
+index 4950191f6b2bf..4a23a97195f33 100644
+--- a/include/net/arp.h
++++ b/include/net/arp.h
+@@ -71,6 +71,7 @@ void arp_send(int type, int ptype, __be32 dest_ip,
+ const unsigned char *src_hw, const unsigned char *th);
+ int arp_mc_map(__be32 addr, u8 *haddr, struct net_device *dev, int dir);
+ void arp_ifdown(struct net_device *dev);
++int arp_invalidate(struct net_device *dev, __be32 ip, bool force);
+
+ struct sk_buff *arp_create(int type, int ptype, __be32 dest_ip,
+ struct net_device *dev, __be32 src_ip,
+diff --git a/include/net/ax25.h b/include/net/ax25.h
+index 8b7eb46ad72d8..aadff553e4b73 100644
+--- a/include/net/ax25.h
++++ b/include/net/ax25.h
+@@ -236,6 +236,7 @@ typedef struct ax25_dev {
+ #if defined(CONFIG_AX25_DAMA_SLAVE) || defined(CONFIG_AX25_DAMA_MASTER)
+ ax25_dama_info dama;
+ #endif
++ refcount_t refcount;
+ } ax25_dev;
+
+ typedef struct ax25_cb {
+@@ -290,6 +291,17 @@ static __inline__ void ax25_cb_put(ax25_cb *ax25)
+ }
+ }
+
++static inline void ax25_dev_hold(ax25_dev *ax25_dev)
++{
++ refcount_inc(&ax25_dev->refcount);
++}
++
++static inline void ax25_dev_put(ax25_dev *ax25_dev)
++{
++ if (refcount_dec_and_test(&ax25_dev->refcount)) {
++ kfree(ax25_dev);
++ }
++}
+ static inline __be16 ax25_type_trans(struct sk_buff *skb, struct net_device *dev)
+ {
+ skb->dev = dev;
+diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h
+index 9125effbf4483..3fecc4a411a13 100644
+--- a/include/net/bluetooth/bluetooth.h
++++ b/include/net/bluetooth/bluetooth.h
+@@ -180,19 +180,21 @@ void bt_err_ratelimited(const char *fmt, ...);
+ #define BT_DBG(fmt, ...) pr_debug(fmt "\n", ##__VA_ARGS__)
+ #endif
+
++#define bt_dev_name(hdev) ((hdev) ? (hdev)->name : "null")
++
+ #define bt_dev_info(hdev, fmt, ...) \
+- BT_INFO("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ BT_INFO("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+ #define bt_dev_warn(hdev, fmt, ...) \
+- BT_WARN("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ BT_WARN("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+ #define bt_dev_err(hdev, fmt, ...) \
+- BT_ERR("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ BT_ERR("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+ #define bt_dev_dbg(hdev, fmt, ...) \
+- BT_DBG("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ BT_DBG("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+
+ #define bt_dev_warn_ratelimited(hdev, fmt, ...) \
+- bt_warn_ratelimited("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ bt_warn_ratelimited("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+ #define bt_dev_err_ratelimited(hdev, fmt, ...) \
+- bt_err_ratelimited("%s: " fmt, (hdev)->name, ##__VA_ARGS__)
++ bt_err_ratelimited("%s: " fmt, bt_dev_name(hdev), ##__VA_ARGS__)
+
+ /* Connection and socket states */
+ enum {
+diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h
+index b80415011dcd5..9ce46cb8564d6 100644
+--- a/include/net/bluetooth/hci.h
++++ b/include/net/bluetooth/hci.h
+@@ -246,6 +246,15 @@ enum {
+ * HCI after resume.
+ */
+ HCI_QUIRK_NO_SUSPEND_NOTIFIER,
++
++ /*
++ * When this quirk is set, LE tx power is not queried on startup
++ * and the min/max tx power values default to HCI_TX_POWER_INVALID.
++ *
++ * This quirk can be set before hci_register_dev is called or
++ * during the hdev->setup vendor callback.
++ */
++ HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER,
+ };
+
+ /* HCI device flags */
+diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
+index a7360c8c72f82..3da5cfcf84c1d 100644
+--- a/include/net/bluetooth/hci_core.h
++++ b/include/net/bluetooth/hci_core.h
+@@ -35,6 +35,9 @@
+ /* HCI priority */
+ #define HCI_PRIO_MAX 7
+
++/* HCI maximum id value */
++#define HCI_MAX_ID 10000
++
+ /* HCI Core structures */
+ struct inquiry_data {
+ bdaddr_t bdaddr;
+diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h
+index 38785d48baff9..184105d682942 100644
+--- a/include/net/bond_3ad.h
++++ b/include/net/bond_3ad.h
+@@ -262,7 +262,7 @@ struct ad_system {
+ struct ad_bond_info {
+ struct ad_system system; /* 802.3ad system structure */
+ struct bond_3ad_stats stats;
+- u32 agg_select_timer; /* Timer to select aggregator after all adapter's hand shakes */
++ atomic_t agg_select_timer; /* Timer to select aggregator after all adapter's hand shakes */
+ u16 aggregator_identifier;
+ };
+
+diff --git a/include/net/bond_alb.h b/include/net/bond_alb.h
+index f6af76c87a6c3..191c36afa1f4a 100644
+--- a/include/net/bond_alb.h
++++ b/include/net/bond_alb.h
+@@ -126,7 +126,7 @@ struct tlb_slave_info {
+ struct alb_bond_info {
+ struct tlb_client_info *tx_hashtbl; /* Dynamically allocated */
+ u32 unbalanced_load;
+- int tx_rebalance_counter;
++ atomic_t tx_rebalance_counter;
+ int lp_counter;
+ /* -------- rlb parameters -------- */
+ int rlb_enabled;
+diff --git a/include/net/checksum.h b/include/net/checksum.h
+index 5b96d5bd6e545..d3b5d368a0caa 100644
+--- a/include/net/checksum.h
++++ b/include/net/checksum.h
+@@ -22,7 +22,7 @@
+ #include <asm/checksum.h>
+
+ #ifndef _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
+-static inline
++static __always_inline
+ __wsum csum_and_copy_from_user (const void __user *src, void *dst,
+ int len)
+ {
+@@ -33,7 +33,7 @@ __wsum csum_and_copy_from_user (const void __user *src, void *dst,
+ #endif
+
+ #ifndef HAVE_CSUM_COPY_USER
+-static __inline__ __wsum csum_and_copy_to_user
++static __always_inline __wsum csum_and_copy_to_user
+ (const void *src, void __user *dst, int len)
+ {
+ __wsum sum = csum_partial(src, len, ~0U);
+@@ -45,7 +45,7 @@ static __inline__ __wsum csum_and_copy_to_user
+ #endif
+
+ #ifndef _HAVE_ARCH_CSUM_AND_COPY
+-static inline __wsum
++static __always_inline __wsum
+ csum_partial_copy_nocheck(const void *src, void *dst, int len)
+ {
+ memcpy(dst, src, len);
+@@ -54,7 +54,7 @@ csum_partial_copy_nocheck(const void *src, void *dst, int len)
+ #endif
+
+ #ifndef HAVE_ARCH_CSUM_ADD
+-static inline __wsum csum_add(__wsum csum, __wsum addend)
++static __always_inline __wsum csum_add(__wsum csum, __wsum addend)
+ {
+ u32 res = (__force u32)csum;
+ res += (__force u32)addend;
+@@ -62,12 +62,12 @@ static inline __wsum csum_add(__wsum csum, __wsum addend)
+ }
+ #endif
+
+-static inline __wsum csum_sub(__wsum csum, __wsum addend)
++static __always_inline __wsum csum_sub(__wsum csum, __wsum addend)
+ {
+ return csum_add(csum, ~addend);
+ }
+
+-static inline __sum16 csum16_add(__sum16 csum, __be16 addend)
++static __always_inline __sum16 csum16_add(__sum16 csum, __be16 addend)
+ {
+ u16 res = (__force u16)csum;
+
+@@ -75,12 +75,12 @@ static inline __sum16 csum16_add(__sum16 csum, __be16 addend)
+ return (__force __sum16)(res + (res < (__force u16)addend));
+ }
+
+-static inline __sum16 csum16_sub(__sum16 csum, __be16 addend)
++static __always_inline __sum16 csum16_sub(__sum16 csum, __be16 addend)
+ {
+ return csum16_add(csum, ~addend);
+ }
+
+-static inline __wsum csum_shift(__wsum sum, int offset)
++static __always_inline __wsum csum_shift(__wsum sum, int offset)
+ {
+ /* rotate sum to align it with a 16b boundary */
+ if (offset & 1)
+@@ -88,42 +88,43 @@ static inline __wsum csum_shift(__wsum sum, int offset)
+ return sum;
+ }
+
+-static inline __wsum
++static __always_inline __wsum
+ csum_block_add(__wsum csum, __wsum csum2, int offset)
+ {
+ return csum_add(csum, csum_shift(csum2, offset));
+ }
+
+-static inline __wsum
++static __always_inline __wsum
+ csum_block_add_ext(__wsum csum, __wsum csum2, int offset, int len)
+ {
+ return csum_block_add(csum, csum2, offset);
+ }
+
+-static inline __wsum
++static __always_inline __wsum
+ csum_block_sub(__wsum csum, __wsum csum2, int offset)
+ {
+ return csum_block_add(csum, ~csum2, offset);
+ }
+
+-static inline __wsum csum_unfold(__sum16 n)
++static __always_inline __wsum csum_unfold(__sum16 n)
+ {
+ return (__force __wsum)n;
+ }
+
+-static inline __wsum csum_partial_ext(const void *buff, int len, __wsum sum)
++static __always_inline
++__wsum csum_partial_ext(const void *buff, int len, __wsum sum)
+ {
+ return csum_partial(buff, len, sum);
+ }
+
+ #define CSUM_MANGLED_0 ((__force __sum16)0xffff)
+
+-static inline void csum_replace_by_diff(__sum16 *sum, __wsum diff)
++static __always_inline void csum_replace_by_diff(__sum16 *sum, __wsum diff)
+ {
+ *sum = csum_fold(csum_add(diff, ~csum_unfold(*sum)));
+ }
+
+-static inline void csum_replace4(__sum16 *sum, __be32 from, __be32 to)
++static __always_inline void csum_replace4(__sum16 *sum, __be32 from, __be32 to)
+ {
+ __wsum tmp = csum_sub(~csum_unfold(*sum), (__force __wsum)from);
+
+@@ -136,11 +137,16 @@ static inline void csum_replace4(__sum16 *sum, __be32 from, __be32 to)
+ * m : old value of a 16bit field
+ * m' : new value of a 16bit field
+ */
+-static inline void csum_replace2(__sum16 *sum, __be16 old, __be16 new)
++static __always_inline void csum_replace2(__sum16 *sum, __be16 old, __be16 new)
+ {
+ *sum = ~csum16_add(csum16_sub(~(*sum), old), new);
+ }
+
++static inline void csum_replace(__wsum *csum, __wsum old, __wsum new)
++{
++ *csum = csum_add(csum_sub(*csum, old), new);
++}
++
+ struct sk_buff;
+ void inet_proto_csum_replace4(__sum16 *sum, struct sk_buff *skb,
+ __be32 from, __be32 to, bool pseudohdr);
+@@ -150,16 +156,16 @@ void inet_proto_csum_replace16(__sum16 *sum, struct sk_buff *skb,
+ void inet_proto_csum_replace_by_diff(__sum16 *sum, struct sk_buff *skb,
+ __wsum diff, bool pseudohdr);
+
+-static inline void inet_proto_csum_replace2(__sum16 *sum, struct sk_buff *skb,
+- __be16 from, __be16 to,
+- bool pseudohdr)
++static __always_inline
++void inet_proto_csum_replace2(__sum16 *sum, struct sk_buff *skb,
++ __be16 from, __be16 to, bool pseudohdr)
+ {
+ inet_proto_csum_replace4(sum, skb, (__force __be32)from,
+ (__force __be32)to, pseudohdr);
+ }
+
+-static inline __wsum remcsum_adjust(void *ptr, __wsum csum,
+- int start, int offset)
++static __always_inline __wsum remcsum_adjust(void *ptr, __wsum csum,
++ int start, int offset)
+ {
+ __sum16 *psum = (__sum16 *)(ptr + offset);
+ __wsum delta;
+@@ -175,7 +181,7 @@ static inline __wsum remcsum_adjust(void *ptr, __wsum csum,
+ return delta;
+ }
+
+-static inline void remcsum_unadjust(__sum16 *psum, __wsum delta)
++static __always_inline void remcsum_unadjust(__sum16 *psum, __wsum delta)
+ {
+ *psum = csum_fold(csum_sub(delta, (__force __wsum)*psum));
+ }
+diff --git a/include/net/dst_cache.h b/include/net/dst_cache.h
+index 67634675e9197..df6622a5fe98f 100644
+--- a/include/net/dst_cache.h
++++ b/include/net/dst_cache.h
+@@ -79,6 +79,17 @@ static inline void dst_cache_reset(struct dst_cache *dst_cache)
+ dst_cache->reset_ts = jiffies;
+ }
+
++/**
++ * dst_cache_reset_now - invalidate the cache contents immediately
++ * @dst_cache: the cache
++ *
++ * The caller must be sure there are no concurrent users, as this frees
++ * all dst_cache users immediately, rather than waiting for the next
++ * per-cpu usage like dst_cache_reset does. Most callers should use the
++ * higher speed lazily-freed dst_cache_reset function instead.
++ */
++void dst_cache_reset_now(struct dst_cache *dst_cache);
++
+ /**
+ * dst_cache_init - initialize the cache, allocating the required storage
+ * @dst_cache: the cache
+diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h
+index 14efa0ded75dd..adab27ba1ecbf 100644
+--- a/include/net/dst_metadata.h
++++ b/include/net/dst_metadata.h
+@@ -123,8 +123,20 @@ static inline struct metadata_dst *tun_dst_unclone(struct sk_buff *skb)
+
+ memcpy(&new_md->u.tun_info, &md_dst->u.tun_info,
+ sizeof(struct ip_tunnel_info) + md_size);
++#ifdef CONFIG_DST_CACHE
++ /* Unclone the dst cache if there is one */
++ if (new_md->u.tun_info.dst_cache.cache) {
++ int ret;
++
++ ret = dst_cache_init(&new_md->u.tun_info.dst_cache, GFP_ATOMIC);
++ if (ret) {
++ metadata_dst_free(new_md);
++ return ERR_PTR(ret);
++ }
++ }
++#endif
++
+ skb_dst_drop(skb);
+- dst_hold(&new_md->dst);
+ skb_dst_set(skb, &new_md->dst);
+ return new_md;
+ }
+diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h
+index 4b10676c69d19..bd07484ab9dd5 100644
+--- a/include/net/fib_rules.h
++++ b/include/net/fib_rules.h
+@@ -69,7 +69,7 @@ struct fib_rules_ops {
+ int (*action)(struct fib_rule *,
+ struct flowi *, int,
+ struct fib_lookup_arg *);
+- bool (*suppress)(struct fib_rule *,
++ bool (*suppress)(struct fib_rule *, int,
+ struct fib_lookup_arg *);
+ int (*match)(struct fib_rule *,
+ struct flowi *, int);
+@@ -218,7 +218,9 @@ INDIRECT_CALLABLE_DECLARE(int fib4_rule_action(struct fib_rule *rule,
+ struct fib_lookup_arg *arg));
+
+ INDIRECT_CALLABLE_DECLARE(bool fib6_rule_suppress(struct fib_rule *rule,
++ int flags,
+ struct fib_lookup_arg *arg));
+ INDIRECT_CALLABLE_DECLARE(bool fib4_rule_suppress(struct fib_rule *rule,
++ int flags,
+ struct fib_lookup_arg *arg));
+ #endif
+diff --git a/include/net/flow_dissector.h b/include/net/flow_dissector.h
+index ffd386ea0dbb3..c8d1c5e187e4b 100644
+--- a/include/net/flow_dissector.h
++++ b/include/net/flow_dissector.h
+@@ -59,6 +59,8 @@ struct flow_dissector_key_vlan {
+ __be16 vlan_tci;
+ };
+ __be16 vlan_tpid;
++ __be16 vlan_eth_type;
++ u16 padding;
+ };
+
+ struct flow_dissector_mpls_lse {
+diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
+index b06c2d02ec84e..fa6a87246a7b8 100644
+--- a/include/net/inet_connection_sock.h
++++ b/include/net/inet_connection_sock.h
+@@ -289,7 +289,7 @@ static inline void inet_csk_prepare_for_destroy_sock(struct sock *sk)
+ {
+ /* The below has to be done to allow calling inet_csk_destroy_sock */
+ sock_set_flag(sk, SOCK_DEAD);
+- percpu_counter_inc(sk->sk_prot->orphan_count);
++ this_cpu_inc(*sk->sk_prot->orphan_count);
+ }
+
+ void inet_csk_destroy_sock(struct sock *sk);
+diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h
+index 48cc5795ceda6..63540be0fc34a 100644
+--- a/include/net/inet_frag.h
++++ b/include/net/inet_frag.h
+@@ -117,8 +117,15 @@ int fqdir_init(struct fqdir **fqdirp, struct inet_frags *f, struct net *net);
+
+ static inline void fqdir_pre_exit(struct fqdir *fqdir)
+ {
+- fqdir->high_thresh = 0; /* prevent creation of new frags */
+- fqdir->dead = true;
++ /* Prevent creation of new frags.
++ * Pairs with READ_ONCE() in inet_frag_find().
++ */
++ WRITE_ONCE(fqdir->high_thresh, 0);
++
++ /* Pairs with READ_ONCE() in inet_frag_kill(), ip_expire()
++ * and ip6frag_expire_frag_queue().
++ */
++ WRITE_ONCE(fqdir->dead, true);
+ }
+ void fqdir_exit(struct fqdir *fqdir);
+
+diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
+index f72ec113ae568..98e1ec1a14f03 100644
+--- a/include/net/inet_hashtables.h
++++ b/include/net/inet_hashtables.h
+@@ -425,7 +425,7 @@ static inline void sk_rcv_saddr_set(struct sock *sk, __be32 addr)
+ }
+
+ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
+- struct sock *sk, u32 port_offset,
++ struct sock *sk, u64 port_offset,
+ int (*check_established)(struct inet_timewait_death_row *,
+ struct sock *, __u16,
+ struct inet_timewait_sock **));
+diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h
+index 89163ef8cf4be..9e1111f5915bd 100644
+--- a/include/net/inet_sock.h
++++ b/include/net/inet_sock.h
+@@ -207,11 +207,10 @@ struct inet_sock {
+ __be32 inet_saddr;
+ __s16 uc_ttl;
+ __u16 cmsg_flags;
++ struct ip_options_rcu __rcu *inet_opt;
+ __be16 inet_sport;
+ __u16 inet_id;
+
+- struct ip_options_rcu __rcu *inet_opt;
+- int rx_dst_ifindex;
+ __u8 tos;
+ __u8 min_ttl;
+ __u8 mc_ttl;
+diff --git a/include/net/ip.h b/include/net/ip.h
+index 9192444f2964e..a77a9e1c6c047 100644
+--- a/include/net/ip.h
++++ b/include/net/ip.h
+@@ -55,6 +55,7 @@ struct inet_skb_parm {
+ #define IPSKB_DOREDIRECT BIT(5)
+ #define IPSKB_FRAG_PMTU BIT(6)
+ #define IPSKB_L3SLAVE BIT(7)
++#define IPSKB_NOPOLICY BIT(8)
+
+ u16 frag_max_size;
+ };
+@@ -520,19 +521,18 @@ static inline void ip_select_ident_segs(struct net *net, struct sk_buff *skb,
+ {
+ struct iphdr *iph = ip_hdr(skb);
+
++ /* We had many attacks based on IPID, use the private
++ * generator as much as we can.
++ */
++ if (sk && inet_sk(sk)->inet_daddr) {
++ iph->id = htons(inet_sk(sk)->inet_id);
++ inet_sk(sk)->inet_id += segs;
++ return;
++ }
+ if ((iph->frag_off & htons(IP_DF)) && !skb->ignore_df) {
+- /* This is only to work around buggy Windows95/2000
+- * VJ compression implementations. If the ID field
+- * does not change, they drop every other packet in
+- * a TCP stream using header compression.
+- */
+- if (sk && inet_sk(sk)->inet_daddr) {
+- iph->id = htons(inet_sk(sk)->inet_id);
+- inet_sk(sk)->inet_id += segs;
+- } else {
+- iph->id = 0;
+- }
++ iph->id = 0;
+ } else {
++ /* Unfortunately we need the big hammer to get a suitable IPID */
+ __ip_select_ident(net, iph, segs);
+ }
+ }
+diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h
+index c412dde4d67dc..bbb27639f2933 100644
+--- a/include/net/ip6_fib.h
++++ b/include/net/ip6_fib.h
+@@ -189,14 +189,16 @@ struct fib6_info {
+ u32 fib6_metric;
+ u8 fib6_protocol;
+ u8 fib6_type;
++
++ u8 offload;
++ u8 trap;
++ u8 offload_failed;
++
+ u8 should_flush:1,
+ dst_nocount:1,
+ dst_nopolicy:1,
+ fib6_destroying:1,
+- offload:1,
+- trap:1,
+- offload_failed:1,
+- unused:1;
++ unused:4;
+
+ struct rcu_head rcu;
+ struct nexthop *nh;
+@@ -281,7 +283,7 @@ static inline bool fib6_get_cookie_safe(const struct fib6_info *f6i,
+ fn = rcu_dereference(f6i->fib6_node);
+
+ if (fn) {
+- *cookie = fn->fn_sernum;
++ *cookie = READ_ONCE(fn->fn_sernum);
+ /* pairs with smp_wmb() in __fib6_update_sernum_upto_root() */
+ smp_rmb();
+ status = true;
+@@ -485,6 +487,7 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
+ struct fib6_config *cfg, gfp_t gfp_flags,
+ struct netlink_ext_ack *extack);
+ void fib6_nh_release(struct fib6_nh *fib6_nh);
++void fib6_nh_release_dsts(struct fib6_nh *fib6_nh);
+
+ int call_fib6_entry_notifiers(struct net *net,
+ enum fib_event_type event_type,
+diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
+index 028eaea1c8544..42d50856fcf24 100644
+--- a/include/net/ip6_tunnel.h
++++ b/include/net/ip6_tunnel.h
+@@ -57,7 +57,7 @@ struct ip6_tnl {
+
+ /* These fields used only by GRE */
+ __u32 i_seqno; /* The last seen seqno */
+- __u32 o_seqno; /* The last output seqno */
++ atomic_t o_seqno; /* The last output seqno */
+ int hlen; /* tun_hlen + encap_hlen */
+ int tun_hlen; /* Precalculated header length */
+ int encap_hlen; /* Encap header length (FOU,GUE) */
+diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h
+index ab5348e57db1a..3417ba2d27ad6 100644
+--- a/include/net/ip_fib.h
++++ b/include/net/ip_fib.h
+@@ -438,7 +438,7 @@ int fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+ static inline int fib_num_tclassid_users(struct net *net)
+ {
+- return net->ipv4.fib_num_tclassid_users;
++ return atomic_read(&net->ipv4.fib_num_tclassid_users);
+ }
+ #else
+ static inline int fib_num_tclassid_users(struct net *net)
+diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
+index bc3b13ec93c9d..37d5d4968e20a 100644
+--- a/include/net/ip_tunnels.h
++++ b/include/net/ip_tunnels.h
+@@ -113,7 +113,7 @@ struct ip_tunnel {
+
+ /* These four fields used only by GRE */
+ u32 i_seqno; /* The last seen seqno */
+- u32 o_seqno; /* The last output seqno */
++ atomic_t o_seqno; /* The last output seqno */
+ int tun_hlen; /* Precalculated header length */
+
+ /* These four fields used only by ERSPAN */
+diff --git a/include/net/ipv6.h b/include/net/ipv6.h
+index f2d0ecc257bb2..359540dfc0339 100644
+--- a/include/net/ipv6.h
++++ b/include/net/ipv6.h
+@@ -391,17 +391,20 @@ static inline void txopt_put(struct ipv6_txoptions *opt)
+ kfree_rcu(opt, rcu);
+ }
+
++#if IS_ENABLED(CONFIG_IPV6)
+ struct ip6_flowlabel *__fl6_sock_lookup(struct sock *sk, __be32 label);
+
+ extern struct static_key_false_deferred ipv6_flowlabel_exclusive;
+ static inline struct ip6_flowlabel *fl6_sock_lookup(struct sock *sk,
+ __be32 label)
+ {
+- if (static_branch_unlikely(&ipv6_flowlabel_exclusive.key))
++ if (static_branch_unlikely(&ipv6_flowlabel_exclusive.key) &&
++ READ_ONCE(sock_net(sk)->ipv6.flowlabel_has_excl))
+ return __fl6_sock_lookup(sk, label) ? : ERR_PTR(-ENOENT);
+
+ return NULL;
+ }
++#endif
+
+ struct ipv6_txoptions *fl6_merge_options(struct ipv6_txoptions *opt_space,
+ struct ip6_flowlabel *fl,
+diff --git a/include/net/ipv6_frag.h b/include/net/ipv6_frag.h
+index 851029ecff13c..0a4779175a523 100644
+--- a/include/net/ipv6_frag.h
++++ b/include/net/ipv6_frag.h
+@@ -67,7 +67,8 @@ ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq)
+ struct sk_buff *head;
+
+ rcu_read_lock();
+- if (fq->q.fqdir->dead)
++ /* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */
++ if (READ_ONCE(fq->q.fqdir->dead))
+ goto out_rcu_unlock;
+ spin_lock(&fq->q.lock);
+
+diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h
+index afbce90c44808..45e0339be6fa4 100644
+--- a/include/net/ipv6_stubs.h
++++ b/include/net/ipv6_stubs.h
+@@ -47,6 +47,7 @@ struct ipv6_stub {
+ struct fib6_config *cfg, gfp_t gfp_flags,
+ struct netlink_ext_ack *extack);
+ void (*fib6_nh_release)(struct fib6_nh *fib6_nh);
++ void (*fib6_nh_release_dsts)(struct fib6_nh *fib6_nh);
+ void (*fib6_update_sernum)(struct net *net, struct fib6_info *rt);
+ int (*ip6_del_rt)(struct net *net, struct fib6_info *rt, bool skip_notify);
+ void (*fib6_rt_update)(struct net *net, struct fib6_info *rt,
+diff --git a/include/net/llc.h b/include/net/llc.h
+index df282d9b40170..9c10b121b49b0 100644
+--- a/include/net/llc.h
++++ b/include/net/llc.h
+@@ -72,7 +72,9 @@ struct llc_sap {
+ static inline
+ struct hlist_head *llc_sk_dev_hash(struct llc_sap *sap, int ifindex)
+ {
+- return &sap->sk_dev_hash[ifindex % LLC_SK_DEV_HASH_ENTRIES];
++ u32 bucket = hash_32(ifindex, LLC_SK_DEV_HASH_BITS);
++
++ return &sap->sk_dev_hash[bucket];
+ }
+
+ static inline
+diff --git a/include/net/ndisc.h b/include/net/ndisc.h
+index 38e4094960cee..e97ef508664f4 100644
+--- a/include/net/ndisc.h
++++ b/include/net/ndisc.h
+@@ -487,9 +487,9 @@ int igmp6_late_init(void);
+ void igmp6_cleanup(void);
+ void igmp6_late_cleanup(void);
+
+-int igmp6_event_query(struct sk_buff *skb);
++void igmp6_event_query(struct sk_buff *skb);
+
+-int igmp6_event_report(struct sk_buff *skb);
++void igmp6_event_report(struct sk_buff *skb);
+
+
+ #ifdef CONFIG_SYSCTL
+diff --git a/include/net/neighbour.h b/include/net/neighbour.h
+index 22ced1381ede5..d5767e25509cc 100644
+--- a/include/net/neighbour.h
++++ b/include/net/neighbour.h
+@@ -253,6 +253,7 @@ static inline void *neighbour_priv(const struct neighbour *n)
+ #define NEIGH_UPDATE_F_OVERRIDE 0x00000001
+ #define NEIGH_UPDATE_F_WEAK_OVERRIDE 0x00000002
+ #define NEIGH_UPDATE_F_OVERRIDE_ISROUTER 0x00000004
++#define NEIGH_UPDATE_F_USE 0x10000000
+ #define NEIGH_UPDATE_F_EXT_LEARNED 0x20000000
+ #define NEIGH_UPDATE_F_ISROUTER 0x40000000
+ #define NEIGH_UPDATE_F_ADMIN 0x80000000
+@@ -504,10 +505,15 @@ static inline int neigh_output(struct neighbour *n, struct sk_buff *skb,
+ {
+ const struct hh_cache *hh = &n->hh;
+
+- if ((n->nud_state & NUD_CONNECTED) && hh->hh_len && !skip_cache)
++ /* n->nud_state and hh->hh_len could be changed under us.
++ * neigh_hh_output() is taking care of the race later.
++ */
++ if (!skip_cache &&
++ (READ_ONCE(n->nud_state) & NUD_CONNECTED) &&
++ READ_ONCE(hh->hh_len))
+ return neigh_hh_output(hh, skb);
+- else
+- return n->output(n, skb);
++
++ return n->output(n, skb);
+ }
+
+ static inline struct neighbour *
+diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
+index bb5fa59140321..2ba326f9e004d 100644
+--- a/include/net/net_namespace.h
++++ b/include/net/net_namespace.h
+@@ -479,4 +479,10 @@ static inline void fnhe_genid_bump(struct net *net)
+ atomic_inc(&net->fnhe_genid);
+ }
+
++#ifdef CONFIG_NET
++void net_ns_init(void);
++#else
++static inline void net_ns_init(void) {}
++#endif
++
+ #endif /* __NET_NET_NAMESPACE_H */
+diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h
+index cc663c68ddc4b..34c266502a50e 100644
+--- a/include/net/netfilter/nf_conntrack.h
++++ b/include/net/netfilter/nf_conntrack.h
+@@ -76,6 +76,8 @@ struct nf_conn {
+ * Hint, SKB address this struct and refcnt via skb->_nfct and
+ * helpers nf_conntrack_get() and nf_conntrack_put().
+ * Helper nf_ct_put() equals nf_conntrack_put() by dec refcnt,
++ * except that the latter uses internal indirection and does not
++ * result in a conntrack module dependency.
+ * beware nf_ct_get() is different and don't inc refcnt.
+ */
+ struct nf_conntrack ct_general;
+@@ -169,11 +171,13 @@ nf_ct_get(const struct sk_buff *skb, enum ip_conntrack_info *ctinfo)
+ return (struct nf_conn *)(nfct & NFCT_PTRMASK);
+ }
+
++void nf_ct_destroy(struct nf_conntrack *nfct);
++
+ /* decrement reference count on a conntrack */
+ static inline void nf_ct_put(struct nf_conn *ct)
+ {
+- WARN_ON(!ct);
+- nf_conntrack_put(&ct->ct_general);
++ if (ct && refcount_dec_and_test(&ct->ct_general.use))
++ nf_ct_destroy(&ct->ct_general);
+ }
+
+ /* Protocol module loading */
+@@ -276,14 +280,14 @@ static inline bool nf_is_loopback_packet(const struct sk_buff *skb)
+ /* jiffies until ct expires, 0 if already expired */
+ static inline unsigned long nf_ct_expires(const struct nf_conn *ct)
+ {
+- s32 timeout = ct->timeout - nfct_time_stamp;
++ s32 timeout = READ_ONCE(ct->timeout) - nfct_time_stamp;
+
+ return timeout > 0 ? timeout : 0;
+ }
+
+ static inline bool nf_ct_is_expired(const struct nf_conn *ct)
+ {
+- return (__s32)(ct->timeout - nfct_time_stamp) <= 0;
++ return (__s32)(READ_ONCE(ct->timeout) - nfct_time_stamp) <= 0;
+ }
+
+ /* use after obtaining a reference count */
+@@ -302,7 +306,7 @@ static inline bool nf_ct_should_gc(const struct nf_conn *ct)
+ static inline void nf_ct_offload_timeout(struct nf_conn *ct)
+ {
+ if (nf_ct_expires(ct) < NF_CT_DAY / 2)
+- ct->timeout = nfct_time_stamp + NF_CT_DAY;
++ WRITE_ONCE(ct->timeout, nfct_time_stamp + NF_CT_DAY);
+ }
+
+ struct kernel_param;
+diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h
+index 37f0fbefb060f..9939c366f720d 100644
+--- a/include/net/netfilter/nf_conntrack_helper.h
++++ b/include/net/netfilter/nf_conntrack_helper.h
+@@ -177,4 +177,5 @@ void nf_nat_helper_unregister(struct nf_conntrack_nat_helper *nat);
+ int nf_nat_helper_try_module_get(const char *name, u16 l3num,
+ u8 protonum);
+ void nf_nat_helper_put(struct nf_conntrack_helper *helper);
++void nf_ct_set_auto_assign_helper_warned(struct net *net);
+ #endif /*_NF_CONNTRACK_HELPER_H*/
+diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
+index a3647fadf1ccb..9f927c44087de 100644
+--- a/include/net/netfilter/nf_flow_table.h
++++ b/include/net/netfilter/nf_flow_table.h
+@@ -10,6 +10,8 @@
+ #include <linux/netfilter/nf_conntrack_tuple_common.h>
+ #include <net/flow_offload.h>
+ #include <net/dst.h>
++#include <linux/if_pppox.h>
++#include <linux/ppp_defs.h>
+
+ struct nf_flowtable;
+ struct nf_flow_rule;
+@@ -313,4 +315,20 @@ int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow,
+ int nf_flow_table_offload_init(void);
+ void nf_flow_table_offload_exit(void);
+
++static inline __be16 nf_flow_pppoe_proto(const struct sk_buff *skb)
++{
++ __be16 proto;
++
++ proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
++ sizeof(struct pppoe_hdr)));
++ switch (proto) {
++ case htons(PPP_IP):
++ return htons(ETH_P_IP);
++ case htons(PPP_IPV6):
++ return htons(ETH_P_IPV6);
++ }
++
++ return 0;
++}
++
+ #endif /* _NF_FLOW_TABLE_H */
+diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h
+index 9eed51e920e87..980daa6e1e3aa 100644
+--- a/include/net/netfilter/nf_queue.h
++++ b/include/net/netfilter/nf_queue.h
+@@ -37,7 +37,7 @@ void nf_register_queue_handler(const struct nf_queue_handler *qh);
+ void nf_unregister_queue_handler(void);
+ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict);
+
+-void nf_queue_entry_get_refs(struct nf_queue_entry *entry);
++bool nf_queue_entry_get_refs(struct nf_queue_entry *entry);
+ void nf_queue_entry_free(struct nf_queue_entry *entry);
+
+ static inline void init_hashrandom(u32 *jhash_initval)
+diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
+index a16171c5fd9eb..d52a5d776e764 100644
+--- a/include/net/netfilter/nf_tables.h
++++ b/include/net/netfilter/nf_tables.h
+@@ -883,9 +883,9 @@ struct nft_expr_ops {
+ int (*offload)(struct nft_offload_ctx *ctx,
+ struct nft_flow_rule *flow,
+ const struct nft_expr *expr);
++ bool (*offload_action)(const struct nft_expr *expr);
+ void (*offload_stats)(struct nft_expr *expr,
+ const struct flow_stats *stats);
+- u32 offload_flags;
+ const struct nft_expr_type *type;
+ void *data;
+ };
+diff --git a/include/net/netfilter/nf_tables_offload.h b/include/net/netfilter/nf_tables_offload.h
+index f9d95ff82df83..7971478439580 100644
+--- a/include/net/netfilter/nf_tables_offload.h
++++ b/include/net/netfilter/nf_tables_offload.h
+@@ -67,8 +67,6 @@ struct nft_flow_rule {
+ struct flow_rule *rule;
+ };
+
+-#define NFT_OFFLOAD_F_ACTION (1 << 0)
+-
+ void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
+ enum flow_dissector_key_id addr_type);
+
+diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
+index 2f65701a43c95..6c5b2efc4f17d 100644
+--- a/include/net/netns/ipv4.h
++++ b/include/net/netns/ipv4.h
+@@ -65,7 +65,7 @@ struct netns_ipv4 {
+ bool fib_has_custom_local_routes;
+ bool fib_offload_disabled;
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+- int fib_num_tclassid_users;
++ atomic_t fib_num_tclassid_users;
+ #endif
+ struct hlist_head *fib_table_hash;
+ struct sock *fibnl;
+diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h
+index a4b5503803165..ff82983b7ab41 100644
+--- a/include/net/netns/ipv6.h
++++ b/include/net/netns/ipv6.h
+@@ -75,11 +75,12 @@ struct netns_ipv6 {
+ struct list_head fib6_walkers;
+ rwlock_t fib6_walker_lock;
+ spinlock_t fib6_gc_lock;
+- unsigned int ip6_rt_gc_expire;
+- unsigned long ip6_rt_last_gc;
++ atomic_t ip6_rt_gc_expire;
++ unsigned long ip6_rt_last_gc;
++ unsigned char flowlabel_has_excl;
+ #ifdef CONFIG_IPV6_MULTIPLE_TABLES
+- unsigned int fib6_rules_require_fldissect;
+ bool fib6_has_custom_rules;
++ unsigned int fib6_rules_require_fldissect;
+ #ifdef CONFIG_IPV6_SUBTREES
+ unsigned int fib6_routes_require_src;
+ #endif
+diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h
+index 947733a639a6f..bd7c3be4af5d7 100644
+--- a/include/net/netns/xfrm.h
++++ b/include/net/netns/xfrm.h
+@@ -66,11 +66,7 @@ struct netns_xfrm {
+ int sysctl_larval_drop;
+ u32 sysctl_acq_expires;
+
+- u8 policy_default;
+-#define XFRM_POL_DEFAULT_IN 1
+-#define XFRM_POL_DEFAULT_OUT 2
+-#define XFRM_POL_DEFAULT_FWD 4
+-#define XFRM_POL_DEFAULT_MASK 7
++ u8 policy_default[XFRM_POLICY_MAX];
+
+ #ifdef CONFIG_SYSCTL
+ struct ctl_table_header *sysctl_hdr;
+diff --git a/include/net/nfc/nci_core.h b/include/net/nfc/nci_core.h
+index a964daedc17b6..ea8595651c384 100644
+--- a/include/net/nfc/nci_core.h
++++ b/include/net/nfc/nci_core.h
+@@ -30,6 +30,7 @@ enum nci_flag {
+ NCI_UP,
+ NCI_DATA_EXCHANGE,
+ NCI_DATA_EXCHANGE_TO,
++ NCI_UNREG,
+ };
+
+ /* NCI device states */
+diff --git a/include/net/nl802154.h b/include/net/nl802154.h
+index ddcee128f5d9a..145acb8f25095 100644
+--- a/include/net/nl802154.h
++++ b/include/net/nl802154.h
+@@ -19,6 +19,8 @@
+ *
+ */
+
++#include <linux/types.h>
++
+ #define NL802154_GENL_NAME "nl802154"
+
+ enum nl802154_commands {
+@@ -150,10 +152,9 @@ enum nl802154_attrs {
+ };
+
+ enum nl802154_iftype {
+- /* for backwards compatibility TODO */
+- NL802154_IFTYPE_UNSPEC = -1,
++ NL802154_IFTYPE_UNSPEC = (~(__u32)0),
+
+- NL802154_IFTYPE_NODE,
++ NL802154_IFTYPE_NODE = 0,
+ NL802154_IFTYPE_MONITOR,
+ NL802154_IFTYPE_COORD,
+
+diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
+index bf79f3a890af2..9e7b21c0b3a6d 100644
+--- a/include/net/pkt_sched.h
++++ b/include/net/pkt_sched.h
+@@ -193,4 +193,22 @@ static inline void skb_txtime_consumed(struct sk_buff *skb)
+ skb->tstamp = ktime_set(0, 0);
+ }
+
++struct tc_skb_cb {
++ struct qdisc_skb_cb qdisc_cb;
++
++ u16 mru;
++ u8 post_ct:1;
++ u8 post_ct_snat:1;
++ u8 post_ct_dnat:1;
++ u16 zone; /* Only valid if post_ct = true */
++};
++
++static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
++{
++ struct tc_skb_cb *cb = (struct tc_skb_cb *)skb->cb;
++
++ BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb));
++ return cb;
++}
++
+ #endif
+diff --git a/include/net/route.h b/include/net/route.h
+index 2e6c0e153e3a5..2551f3f03b37e 100644
+--- a/include/net/route.h
++++ b/include/net/route.h
+@@ -369,7 +369,7 @@ static inline struct neighbour *ip_neigh_gw4(struct net_device *dev,
+ {
+ struct neighbour *neigh;
+
+- neigh = __ipv4_neigh_lookup_noref(dev, daddr);
++ neigh = __ipv4_neigh_lookup_noref(dev, (__force u32)daddr);
+ if (unlikely(!neigh))
+ neigh = __neigh_create(&arp_tbl, &daddr, dev, false);
+
+diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
+index c0069ac00e62d..4121ffd0faf8f 100644
+--- a/include/net/sch_generic.h
++++ b/include/net/sch_generic.h
+@@ -308,6 +308,8 @@ struct Qdisc_ops {
+ struct netlink_ext_ack *extack);
+ void (*attach)(struct Qdisc *sch);
+ int (*change_tx_queue_len)(struct Qdisc *, unsigned int);
++ void (*change_real_num_tx)(struct Qdisc *sch,
++ unsigned int new_real_tx);
+
+ int (*dump)(struct Qdisc *, struct sk_buff *);
+ int (*dump_stats)(struct Qdisc *, struct gnet_dump *);
+@@ -438,8 +440,6 @@ struct qdisc_skb_cb {
+ };
+ #define QDISC_CB_PRIV_LEN 20
+ unsigned char data[QDISC_CB_PRIV_LEN];
+- u16 mru;
+- bool post_ct;
+ };
+
+ typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
+@@ -684,6 +684,8 @@ void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
+ void qdisc_class_hash_destroy(struct Qdisc_class_hash *);
+
+ int dev_qdisc_change_tx_queue_len(struct net_device *dev);
++void dev_qdisc_change_real_num_tx(struct net_device *dev,
++ unsigned int new_real_tx);
+ void dev_init_scheduler(struct net_device *dev);
+ void dev_shutdown(struct net_device *dev);
+ void dev_activate(struct net_device *dev);
+@@ -1258,6 +1260,7 @@ struct psched_ratecfg {
+ u64 rate_bytes_ps; /* bytes per second */
+ u32 mult;
+ u16 overhead;
++ u16 mpu;
+ u8 linklayer;
+ u8 shift;
+ };
+@@ -1267,6 +1270,9 @@ static inline u64 psched_l2t_ns(const struct psched_ratecfg *r,
+ {
+ len += r->overhead;
+
++ if (len < r->mpu)
++ len = r->mpu;
++
+ if (unlikely(r->linklayer == TC_LINKLAYER_ATM))
+ return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift;
+
+@@ -1289,6 +1295,7 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res,
+ res->rate = min_t(u64, r->rate_bytes_ps, ~0U);
+
+ res->overhead = r->overhead;
++ res->mpu = r->mpu;
+ res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
+ }
+
+diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h
+index 69bab88ad66b1..3ae61ce2eabd0 100644
+--- a/include/net/sctp/sctp.h
++++ b/include/net/sctp/sctp.h
+@@ -105,19 +105,18 @@ extern struct percpu_counter sctp_sockets_allocated;
+ int sctp_asconf_mgmt(struct sctp_sock *, struct sctp_sockaddr_entry *);
+ struct sk_buff *sctp_skb_recv_datagram(struct sock *, int, int, int *);
+
++typedef int (*sctp_callback_t)(struct sctp_endpoint *, struct sctp_transport *, void *);
+ void sctp_transport_walk_start(struct rhashtable_iter *iter);
+ void sctp_transport_walk_stop(struct rhashtable_iter *iter);
+ struct sctp_transport *sctp_transport_get_next(struct net *net,
+ struct rhashtable_iter *iter);
+ struct sctp_transport *sctp_transport_get_idx(struct net *net,
+ struct rhashtable_iter *iter, int pos);
+-int sctp_transport_lookup_process(int (*cb)(struct sctp_transport *, void *),
+- struct net *net,
++int sctp_transport_lookup_process(sctp_callback_t cb, struct net *net,
+ const union sctp_addr *laddr,
+ const union sctp_addr *paddr, void *p);
+-int sctp_for_each_transport(int (*cb)(struct sctp_transport *, void *),
+- int (*cb_done)(struct sctp_transport *, void *),
+- struct net *net, int *pos, void *p);
++int sctp_transport_traverse_process(sctp_callback_t cb, sctp_callback_t cb_done,
++ struct net *net, int *pos, void *p);
+ int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), void *p);
+ int sctp_get_sctp_info(struct sock *sk, struct sctp_association *asoc,
+ struct sctp_info *info);
+@@ -626,7 +625,8 @@ static inline __u32 sctp_min_frag_point(struct sctp_sock *sp, __u16 datasize)
+
+ static inline int sctp_transport_pl_hlen(struct sctp_transport *t)
+ {
+- return __sctp_mtu_payload(sctp_sk(t->asoc->base.sk), t, 0, 0);
++ return __sctp_mtu_payload(sctp_sk(t->asoc->base.sk), t, 0, 0) -
++ sizeof(struct sctphdr);
+ }
+
+ static inline void sctp_transport_pl_reset(struct sctp_transport *t)
+@@ -653,12 +653,10 @@ static inline void sctp_transport_pl_update(struct sctp_transport *t)
+ if (t->pl.state == SCTP_PL_DISABLED)
+ return;
+
+- if (del_timer(&t->probe_timer))
+- sctp_transport_put(t);
+-
+ t->pl.state = SCTP_PL_BASE;
+ t->pl.pmtu = SCTP_BASE_PLPMTU;
+ t->pl.probe_size = SCTP_BASE_PLPMTU;
++ sctp_transport_reset_probe_timer(t);
+ }
+
+ static inline bool sctp_transport_pl_enabled(struct sctp_transport *t)
+diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
+index 651bba654d77d..8d2c3dd9f5953 100644
+--- a/include/net/sctp/structs.h
++++ b/include/net/sctp/structs.h
+@@ -1365,6 +1365,7 @@ struct sctp_endpoint {
+
+ u32 secid;
+ u32 peer_secid;
++ struct rcu_head rcu;
+ };
+
+ /* Recover the outter endpoint structure. */
+@@ -1380,7 +1381,7 @@ static inline struct sctp_endpoint *sctp_ep(struct sctp_ep_common *base)
+ struct sctp_endpoint *sctp_endpoint_new(struct sock *, gfp_t);
+ void sctp_endpoint_free(struct sctp_endpoint *);
+ void sctp_endpoint_put(struct sctp_endpoint *);
+-void sctp_endpoint_hold(struct sctp_endpoint *);
++int sctp_endpoint_hold(struct sctp_endpoint *ep);
+ void sctp_endpoint_add_asoc(struct sctp_endpoint *, struct sctp_association *);
+ struct sctp_association *sctp_endpoint_lookup_assoc(
+ const struct sctp_endpoint *ep,
+diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h
+index d7d2495f83c27..dac91aa38c5af 100644
+--- a/include/net/secure_seq.h
++++ b/include/net/secure_seq.h
+@@ -4,8 +4,8 @@
+
+ #include <linux/types.h>
+
+-u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport);
+-u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
++u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport);
++u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
+ __be16 dport);
+ u32 secure_tcp_seq(__be32 saddr, __be32 daddr,
+ __be16 sport, __be16 dport);
+diff --git a/include/net/seg6.h b/include/net/seg6.h
+index 9d19c15e8545c..af668f17b3988 100644
+--- a/include/net/seg6.h
++++ b/include/net/seg6.h
+@@ -58,9 +58,30 @@ extern int seg6_local_init(void);
+ extern void seg6_local_exit(void);
+
+ extern bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced);
++extern struct ipv6_sr_hdr *seg6_get_srh(struct sk_buff *skb, int flags);
++extern void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt);
+ extern int seg6_do_srh_encap(struct sk_buff *skb, struct ipv6_sr_hdr *osrh,
+ int proto);
+ extern int seg6_do_srh_inline(struct sk_buff *skb, struct ipv6_sr_hdr *osrh);
+ extern int seg6_lookup_nexthop(struct sk_buff *skb, struct in6_addr *nhaddr,
+ u32 tbl_id);
++
++/* If the packet which invoked an ICMP error contains an SRH return
++ * the true destination address from within the SRH, otherwise use the
++ * destination address in the IP header.
++ */
++static inline const struct in6_addr *seg6_get_daddr(struct sk_buff *skb,
++ struct inet6_skb_parm *opt)
++{
++ struct ipv6_sr_hdr *srh;
++
++ if (opt->flags & IP6SKB_SEG6) {
++ srh = (struct ipv6_sr_hdr *)(skb->data + opt->srhoff);
++ return &srh->segments[0];
++ }
++
++ return NULL;
++}
++
++
+ #endif
+diff --git a/include/net/sock.h b/include/net/sock.h
+index 463f390d90b3e..7d49196a3880e 100644
+--- a/include/net/sock.h
++++ b/include/net/sock.h
+@@ -259,6 +259,8 @@ struct bpf_local_storage;
+ * @sk_rcvbuf: size of receive buffer in bytes
+ * @sk_wq: sock wait queue and async head
+ * @sk_rx_dst: receive input route used by early demux
++ * @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
++ * @sk_rx_dst_cookie: cookie for @sk_rx_dst
+ * @sk_dst_cache: destination cache
+ * @sk_dst_pending_confirm: need to confirm neighbour
+ * @sk_policy: flow policy
+@@ -430,7 +432,10 @@ struct sock {
+ #ifdef CONFIG_XFRM
+ struct xfrm_policy __rcu *sk_policy[2];
+ #endif
+- struct dst_entry *sk_rx_dst;
++ struct dst_entry __rcu *sk_rx_dst;
++ int sk_rx_dst_ifindex;
++ u32 sk_rx_dst_cookie;
++
+ struct dst_entry __rcu *sk_dst_cache;
+ atomic_t sk_omem_alloc;
+ int sk_sndbuf;
+@@ -501,7 +506,7 @@ struct sock {
+ u16 sk_tsflags;
+ int sk_bind_phc;
+ u8 sk_shutdown;
+- u32 sk_tskey;
++ atomic_t sk_tskey;
+ atomic_t sk_zckey;
+
+ u8 sk_clockid;
+@@ -1237,7 +1242,7 @@ struct proto {
+ unsigned int useroffset; /* Usercopy region offset */
+ unsigned int usersize; /* Usercopy region size */
+
+- struct percpu_counter *orphan_count;
++ unsigned int __percpu *orphan_count;
+
+ struct request_sock_ops *rsk_prot;
+ struct timewait_sock_ops *twsk_prot;
+@@ -2400,19 +2405,22 @@ struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
+ * @sk: socket
+ *
+ * Use the per task page_frag instead of the per socket one for
+- * optimization when we know that we're in the normal context and owns
++ * optimization when we know that we're in process context and own
+ * everything that's associated with %current.
+ *
+- * gfpflags_allow_blocking() isn't enough here as direct reclaim may nest
+- * inside other socket operations and end up recursing into sk_page_frag()
+- * while it's already in use.
++ * Both direct reclaim and page faults can nest inside other
++ * socket operations and end up recursing into sk_page_frag()
++ * while it's already in use: explicitly avoid task page_frag
++ * usage if the caller is potentially doing any of them.
++ * This assumes that page fault handlers use the GFP_NOFS flags.
+ *
+ * Return: a per task page_frag if context allows that,
+ * otherwise a per socket one.
+ */
+ static inline struct page_frag *sk_page_frag(struct sock *sk)
+ {
+- if (gfpflags_normal_context(sk->sk_allocation))
++ if ((sk->sk_allocation & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC | __GFP_FS)) ==
++ (__GFP_DIRECT_RECLAIM | __GFP_FS))
+ return &current->task_frag;
+
+ return &sk->sk_frag;
+@@ -2590,7 +2598,7 @@ static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
+ __sock_tx_timestamp(tsflags, tx_flags);
+ if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
+ tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
+- *tskey = sk->sk_tskey++;
++ *tskey = atomic_inc_return(&sk->sk_tskey) - 1;
+ }
+ if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
+ *tx_flags |= SKBTX_WIFI_STATUS;
+diff --git a/include/net/strparser.h b/include/net/strparser.h
+index 1d20b98493a10..732b7097d78e4 100644
+--- a/include/net/strparser.h
++++ b/include/net/strparser.h
+@@ -54,10 +54,28 @@ struct strp_msg {
+ int offset;
+ };
+
++struct _strp_msg {
++ /* Internal cb structure. struct strp_msg must be first for passing
++ * to upper layer.
++ */
++ struct strp_msg strp;
++ int accum_len;
++};
++
++struct sk_skb_cb {
++#define SK_SKB_CB_PRIV_LEN 20
++ unsigned char data[SK_SKB_CB_PRIV_LEN];
++ struct _strp_msg strp;
++ /* temp_reg is a temporary register used for bpf_convert_data_end_access
++ * when dst_reg == src_reg.
++ */
++ u64 temp_reg;
++};
++
+ static inline struct strp_msg *strp_msg(struct sk_buff *skb)
+ {
+ return (struct strp_msg *)((void *)skb->cb +
+- offsetof(struct qdisc_skb_cb, data));
++ offsetof(struct sk_skb_cb, strp));
+ }
+
+ /* Structure for an attached lower socket */
+diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h
+index 748cf87a4d7ea..3e02709a1df65 100644
+--- a/include/net/tc_act/tc_pedit.h
++++ b/include/net/tc_act/tc_pedit.h
+@@ -14,6 +14,7 @@ struct tcf_pedit {
+ struct tc_action common;
+ unsigned char tcfp_nkeys;
+ unsigned char tcfp_flags;
++ u32 tcfp_off_max_hint;
+ struct tc_pedit_key *tcfp_keys;
+ struct tcf_pedit_key_ex *tcfp_keys_ex;
+ };
+diff --git a/include/net/tcp.h b/include/net/tcp.h
+index 60c384569e9cd..91ac329ca5789 100644
+--- a/include/net/tcp.h
++++ b/include/net/tcp.h
+@@ -48,7 +48,9 @@
+
+ extern struct inet_hashinfo tcp_hashinfo;
+
+-extern struct percpu_counter tcp_orphan_count;
++DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
++int tcp_orphan_count_sum(void);
++
+ void tcp_time_wait(struct sock *sk, int state, int timeo);
+
+ #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
+@@ -290,19 +292,6 @@ static inline bool tcp_out_of_memory(struct sock *sk)
+
+ void sk_forced_mem_schedule(struct sock *sk, int size);
+
+-static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
+-{
+- struct percpu_counter *ocp = sk->sk_prot->orphan_count;
+- int orphans = percpu_counter_read_positive(ocp);
+-
+- if (orphans << shift > sysctl_tcp_max_orphans) {
+- orphans = percpu_counter_sum_positive(ocp);
+- if (orphans << shift > sysctl_tcp_max_orphans)
+- return true;
+- }
+- return false;
+-}
+-
+ bool tcp_check_oom(struct sock *sk, int shift);
+
+
+@@ -481,6 +470,7 @@ int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
+ u32 cookie);
+ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
+ struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
++ const struct tcp_request_sock_ops *af_ops,
+ struct sock *sk, struct sk_buff *skb);
+ #ifdef CONFIG_SYN_COOKIES
+
+@@ -619,6 +609,7 @@ void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
+ void tcp_reset(struct sock *sk, struct sk_buff *skb);
+ void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
+ void tcp_fin(struct sock *sk);
++void tcp_check_space(struct sock *sk);
+
+ /* tcp_timer.c */
+ void tcp_init_xmit_timers(struct sock *);
+@@ -1037,6 +1028,7 @@ struct rate_sample {
+ int losses; /* number of packets marked lost upon ACK */
+ u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
+ u32 prior_in_flight; /* in flight before this ACK */
++ u32 last_end_seq; /* end_seq of most recently ACKed packet */
+ bool is_app_limited; /* is sample from packet with bubble in pipe? */
+ bool is_retrans; /* is sample from retransmission? */
+ bool is_ack_delayed; /* is this (likely) a delayed ACK? */
+@@ -1159,6 +1151,11 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
+ bool is_sack_reneg, struct rate_sample *rs);
+ void tcp_rate_check_app_limited(struct sock *sk);
+
++static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
++{
++ return t1 > t2 || (t1 == t2 && after(seq1, seq2));
++}
++
+ /* These functions determine how the current flow behaves in respect of SACK
+ * handling. SACK is negotiated with the peer, and therefore it can vary
+ * between different flows.
+diff --git a/include/net/xfrm.h b/include/net/xfrm.h
+index 2308210793a01..65242172e41c2 100644
+--- a/include/net/xfrm.h
++++ b/include/net/xfrm.h
+@@ -200,6 +200,11 @@ struct xfrm_state {
+ struct xfrm_algo_aead *aead;
+ const char *geniv;
+
++ /* mapping change rate limiting */
++ __be16 new_mapping_sport;
++ u32 new_mapping; /* seconds */
++ u32 mapping_maxage; /* seconds for input SA */
++
+ /* Data for encapsulator */
+ struct xfrm_encap_tmpl *encap;
+ struct sock __rcu *encap_sk;
+@@ -1075,24 +1080,29 @@ xfrm_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, un
+ }
+
+ #ifdef CONFIG_XFRM
+-static inline bool
+-xfrm_default_allow(struct net *net, int dir)
+-{
+- u8 def = net->xfrm.policy_default;
+-
+- switch (dir) {
+- case XFRM_POLICY_IN:
+- return def & XFRM_POL_DEFAULT_IN ? false : true;
+- case XFRM_POLICY_OUT:
+- return def & XFRM_POL_DEFAULT_OUT ? false : true;
+- case XFRM_POLICY_FWD:
+- return def & XFRM_POL_DEFAULT_FWD ? false : true;
+- }
++int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb,
++ unsigned short family);
++
++static inline bool __xfrm_check_nopolicy(struct net *net, struct sk_buff *skb,
++ int dir)
++{
++ if (!net->xfrm.policy_count[dir] && !secpath_exists(skb))
++ return net->xfrm.policy_default[dir] == XFRM_USERPOLICY_ACCEPT;
++
+ return false;
+ }
+
+-int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb,
+- unsigned short family);
++static inline bool __xfrm_check_dev_nopolicy(struct sk_buff *skb,
++ int dir, unsigned short family)
++{
++ if (dir != XFRM_POLICY_OUT && family == AF_INET) {
++ /* same dst may be used for traffic originating from
++ * devices with different policy settings.
++ */
++ return IPCB(skb)->flags & IPSKB_NOPOLICY;
++ }
++ return skb_dst(skb) && (skb_dst(skb)->flags & DST_NOPOLICY);
++}
+
+ static inline int __xfrm_policy_check2(struct sock *sk, int dir,
+ struct sk_buff *skb,
+@@ -1104,13 +1114,9 @@ static inline int __xfrm_policy_check2(struct sock *sk, int dir,
+ if (sk && sk->sk_policy[XFRM_POLICY_IN])
+ return __xfrm_policy_check(sk, ndir, skb, family);
+
+- if (xfrm_default_allow(net, dir))
+- return (!net->xfrm.policy_count[dir] && !secpath_exists(skb)) ||
+- (skb_dst(skb) && (skb_dst(skb)->flags & DST_NOPOLICY)) ||
+- __xfrm_policy_check(sk, ndir, skb, family);
+- else
+- return (skb_dst(skb) && (skb_dst(skb)->flags & DST_NOPOLICY)) ||
+- __xfrm_policy_check(sk, ndir, skb, family);
++ return __xfrm_check_nopolicy(net, skb, dir) ||
++ __xfrm_check_dev_nopolicy(skb, dir, family) ||
++ __xfrm_policy_check(sk, ndir, skb, family);
+ }
+
+ static inline int xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, unsigned short family)
+@@ -1162,13 +1168,12 @@ static inline int xfrm_route_forward(struct sk_buff *skb, unsigned short family)
+ {
+ struct net *net = dev_net(skb->dev);
+
+- if (xfrm_default_allow(net, XFRM_POLICY_FWD))
+- return !net->xfrm.policy_count[XFRM_POLICY_OUT] ||
+- (skb_dst(skb)->flags & DST_NOXFRM) ||
+- __xfrm_route_forward(skb, family);
+- else
+- return (skb_dst(skb)->flags & DST_NOXFRM) ||
+- __xfrm_route_forward(skb, family);
++ if (!net->xfrm.policy_count[XFRM_POLICY_OUT] &&
++ net->xfrm.policy_default[XFRM_POLICY_OUT] == XFRM_USERPOLICY_ACCEPT)
++ return true;
++
++ return (skb_dst(skb)->flags & DST_NOXFRM) ||
++ __xfrm_route_forward(skb, family);
+ }
+
+ static inline int xfrm4_route_forward(struct sk_buff *skb)
+@@ -1562,7 +1567,6 @@ void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si);
+ void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si);
+ u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq);
+ int xfrm_init_replay(struct xfrm_state *x);
+-u32 __xfrm_state_mtu(struct xfrm_state *x, int mtu);
+ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu);
+ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload);
+ int xfrm_init_state(struct xfrm_state *x);
+@@ -1675,14 +1679,15 @@ int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ const struct xfrm_migrate *m, int num_bundles,
+ const struct xfrm_kmaddress *k,
+ const struct xfrm_encap_tmpl *encap);
+-struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net);
++struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
++ u32 if_id);
+ struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
+ struct xfrm_migrate *m,
+ struct xfrm_encap_tmpl *encap);
+ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ struct xfrm_migrate *m, int num_bundles,
+ struct xfrm_kmaddress *k, struct net *net,
+- struct xfrm_encap_tmpl *encap);
++ struct xfrm_encap_tmpl *encap, u32 if_id);
+ #endif
+
+ int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport);
+diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
+index 4b50d9a3018a6..4ba642fc8a19a 100644
+--- a/include/rdma/ib_verbs.h
++++ b/include/rdma/ib_verbs.h
+@@ -4097,8 +4097,13 @@ static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,
+ enum dma_data_direction direction,
+ unsigned long dma_attrs)
+ {
++ int nents;
++
+ if (ib_uses_virt_dma(dev)) {
+- ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents);
++ nents = ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents);
++ if (!nents)
++ return -EIO;
++ sgt->nents = nents;
+ return 0;
+ }
+ return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs);
+diff --git a/include/rdma/rdma_netlink.h b/include/rdma/rdma_netlink.h
+index 2758d9df71ee9..c2a79aeee113c 100644
+--- a/include/rdma/rdma_netlink.h
++++ b/include/rdma/rdma_netlink.h
+@@ -30,7 +30,7 @@ enum rdma_nl_flags {
+ * constant as well and the compiler checks they are the same.
+ */
+ #define MODULE_ALIAS_RDMA_NETLINK(_index, _val) \
+- static inline void __chk_##_index(void) \
++ static inline void __maybe_unused __chk_##_index(void) \
+ { \
+ BUILD_BUG_ON(_index != _val); \
+ } \
+diff --git a/include/scsi/libiscsi.h b/include/scsi/libiscsi.h
+index 4ee233e5a6ffa..d1e282f0d6f18 100644
+--- a/include/scsi/libiscsi.h
++++ b/include/scsi/libiscsi.h
+@@ -52,8 +52,10 @@ enum {
+
+ #define ISID_SIZE 6
+
+-/* Connection suspend "bit" */
+-#define ISCSI_SUSPEND_BIT 1
++/* Connection flags */
++#define ISCSI_CONN_FLAG_SUSPEND_TX BIT(0)
++#define ISCSI_CONN_FLAG_SUSPEND_RX BIT(1)
++#define ISCSI_CONN_FLAG_BOUND BIT(2)
+
+ #define ISCSI_ITT_MASK 0x1fff
+ #define ISCSI_TOTAL_CMDS_MAX 4096
+@@ -199,8 +201,7 @@ struct iscsi_conn {
+ struct list_head cmdqueue; /* data-path cmd queue */
+ struct list_head requeue; /* tasks needing another run */
+ struct work_struct xmitwork; /* per-conn. xmit workqueue */
+- unsigned long suspend_tx; /* suspend Tx */
+- unsigned long suspend_rx; /* suspend Rx */
++ unsigned long flags; /* ISCSI_CONN_FLAGs */
+
+ /* negotiated params */
+ unsigned max_recv_dlength; /* initiator_max_recv_dsl*/
+diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h
+index eaf04c9a1dfcb..59afe8787cf7c 100644
+--- a/include/scsi/scsi_cmnd.h
++++ b/include/scsi/scsi_cmnd.h
+@@ -68,7 +68,7 @@ struct scsi_pointer {
+ struct scsi_cmnd {
+ struct scsi_request req;
+ struct scsi_device *device;
+- struct list_head eh_entry; /* entry for the host eh_cmd_q */
++ struct list_head eh_entry; /* entry for the host eh_abort_list/eh_cmd_q */
+ struct delayed_work abort_work;
+
+ struct rcu_head rcu;
+diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
+index b97e142a7ca92..3b3dbc37653da 100644
+--- a/include/scsi/scsi_device.h
++++ b/include/scsi/scsi_device.h
+@@ -206,6 +206,7 @@ struct scsi_device {
+ unsigned rpm_autosuspend:1; /* Enable runtime autosuspend at device
+ * creation time */
+ unsigned ignore_media_change:1; /* Ignore MEDIA CHANGE on resume */
++ unsigned silence_suspend:1; /* Do not print runtime PM related messages */
+
+ bool offline_already; /* Device offline message logged */
+
+diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
+index 75363707b73f9..1a02e58eb4e44 100644
+--- a/include/scsi/scsi_host.h
++++ b/include/scsi/scsi_host.h
+@@ -556,6 +556,7 @@ struct Scsi_Host {
+
+ struct mutex scan_mutex;/* serialize scanning activity */
+
++ struct list_head eh_abort_list;
+ struct list_head eh_cmd_q;
+ struct task_struct * ehandler; /* Error recovery thread. */
+ struct completion * eh_action; /* Wait for specific actions on the
+diff --git a/include/scsi/scsi_transport_iscsi.h b/include/scsi/scsi_transport_iscsi.h
+index c5d7810fd7926..3ecf9702287be 100644
+--- a/include/scsi/scsi_transport_iscsi.h
++++ b/include/scsi/scsi_transport_iscsi.h
+@@ -211,6 +211,8 @@ struct iscsi_cls_conn {
+ struct mutex ep_mutex;
+ struct iscsi_endpoint *ep;
+
++ /* Used when accessing flags and queueing work. */
++ spinlock_t lock;
+ unsigned long flags;
+ struct work_struct cleanup_work;
+
+@@ -294,7 +296,7 @@ extern void iscsi_host_for_each_session(struct Scsi_Host *shost,
+ struct iscsi_endpoint {
+ void *dd_data; /* LLD private data */
+ struct device dev;
+- uint64_t id;
++ int id;
+ struct iscsi_cls_conn *conn;
+ };
+
+diff --git a/include/sound/core.h b/include/sound/core.h
+index b7e9b58d3c788..6d4cc49584c63 100644
+--- a/include/sound/core.h
++++ b/include/sound/core.h
+@@ -284,6 +284,7 @@ int snd_card_disconnect(struct snd_card *card);
+ void snd_card_disconnect_sync(struct snd_card *card);
+ int snd_card_free(struct snd_card *card);
+ int snd_card_free_when_closed(struct snd_card *card);
++int snd_card_free_on_error(struct device *dev, int ret);
+ void snd_card_set_id(struct snd_card *card, const char *id);
+ int snd_card_register(struct snd_card *card);
+ int snd_card_info_init(void);
+diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h
+index 0e45963bb767f..82d9daa178517 100644
+--- a/include/sound/hda_codec.h
++++ b/include/sound/hda_codec.h
+@@ -8,7 +8,7 @@
+ #ifndef __SOUND_HDA_CODEC_H
+ #define __SOUND_HDA_CODEC_H
+
+-#include <linux/kref.h>
++#include <linux/refcount.h>
+ #include <linux/mod_devicetable.h>
+ #include <sound/info.h>
+ #include <sound/control.h>
+@@ -166,8 +166,8 @@ struct hda_pcm {
+ bool own_chmap; /* codec driver provides own channel maps */
+ /* private: */
+ struct hda_codec *codec;
+- struct kref kref;
+ struct list_head list;
++ unsigned int disconnected:1;
+ };
+
+ /* codec information */
+@@ -187,6 +187,8 @@ struct hda_codec {
+
+ /* PCM to create, set by patch_ops.build_pcms callback */
+ struct list_head pcm_list_head;
++ refcount_t pcm_ref;
++ wait_queue_head_t remove_sleep;
+
+ /* codec specific info */
+ void *spec;
+@@ -420,7 +422,7 @@ void snd_hda_codec_cleanup_for_unbind(struct hda_codec *codec);
+
+ static inline void snd_hda_codec_pcm_get(struct hda_pcm *pcm)
+ {
+- kref_get(&pcm->kref);
++ refcount_inc(&pcm->codec->pcm_ref);
+ }
+ void snd_hda_codec_pcm_put(struct hda_pcm *pcm);
+
+diff --git a/include/sound/hdaudio_ext.h b/include/sound/hdaudio_ext.h
+index 375581634143c..d4e31ea16aba3 100644
+--- a/include/sound/hdaudio_ext.h
++++ b/include/sound/hdaudio_ext.h
+@@ -88,6 +88,8 @@ struct hdac_ext_stream *snd_hdac_ext_stream_assign(struct hdac_bus *bus,
+ struct snd_pcm_substream *substream,
+ int type);
+ void snd_hdac_ext_stream_release(struct hdac_ext_stream *azx_dev, int type);
++void snd_hdac_ext_stream_decouple_locked(struct hdac_bus *bus,
++ struct hdac_ext_stream *azx_dev, bool decouple);
+ void snd_hdac_ext_stream_decouple(struct hdac_bus *bus,
+ struct hdac_ext_stream *azx_dev, bool decouple);
+ void snd_hdac_ext_stop_streams(struct hdac_bus *bus);
+diff --git a/include/sound/pcm.h b/include/sound/pcm.h
+index 33451f8ff755b..e08bf475d02d4 100644
+--- a/include/sound/pcm.h
++++ b/include/sound/pcm.h
+@@ -398,6 +398,8 @@ struct snd_pcm_runtime {
+ wait_queue_head_t tsleep; /* transfer sleep */
+ struct fasync_struct *fasync;
+ bool stop_operating; /* sync_stop will be called */
++ struct mutex buffer_mutex; /* protect for buffer changes */
++ atomic_t buffer_accessing; /* >0: in r/w operation, <0: blocked */
+
+ /* -- private section -- */
+ void *private_data;
+diff --git a/include/sound/soc-topology.h b/include/sound/soc-topology.h
+index 4afd667e124c2..3e8a85e1e8094 100644
+--- a/include/sound/soc-topology.h
++++ b/include/sound/soc-topology.h
+@@ -188,8 +188,7 @@ int snd_soc_tplg_widget_bind_event(struct snd_soc_dapm_widget *w,
+
+ #else
+
+-static inline int snd_soc_tplg_component_remove(struct snd_soc_component *comp,
+- u32 index)
++static inline int snd_soc_tplg_component_remove(struct snd_soc_component *comp)
+ {
+ return 0;
+ }
+diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
+index fb11c7693b257..2121a323fd6c3 100644
+--- a/include/target/target_core_base.h
++++ b/include/target/target_core_base.h
+@@ -812,8 +812,9 @@ struct se_device {
+ atomic_long_t read_bytes;
+ atomic_long_t write_bytes;
+ /* Active commands on this virtual SE device */
+- atomic_t simple_cmds;
+- atomic_t dev_ordered_sync;
++ atomic_t non_ordered;
++ bool ordered_sync_in_progress;
++ atomic_t delayed_cmd_count;
+ atomic_t dev_qf_count;
+ u32 export_count;
+ spinlock_t delayed_cmd_lock;
+@@ -834,6 +835,7 @@ struct se_device {
+ struct list_head dev_sep_list;
+ struct list_head dev_tmr_list;
+ struct work_struct qf_work_queue;
++ struct work_struct delayed_cmd_work;
+ struct list_head delayed_cmd_list;
+ struct list_head qf_cmd_list;
+ /* Pointer to associated SE HBA */
+diff --git a/include/trace/events/cgroup.h b/include/trace/events/cgroup.h
+index 7f42a3de59e6b..dd7d7c9efecdf 100644
+--- a/include/trace/events/cgroup.h
++++ b/include/trace/events/cgroup.h
+@@ -59,8 +59,8 @@ DECLARE_EVENT_CLASS(cgroup,
+
+ TP_STRUCT__entry(
+ __field( int, root )
+- __field( int, id )
+ __field( int, level )
++ __field( u64, id )
+ __string( path, path )
+ ),
+
+@@ -71,7 +71,7 @@ DECLARE_EVENT_CLASS(cgroup,
+ __assign_str(path, path);
+ ),
+
+- TP_printk("root=%d id=%d level=%d path=%s",
++ TP_printk("root=%d id=%llu level=%d path=%s",
+ __entry->root, __entry->id, __entry->level, __get_str(path))
+ );
+
+@@ -126,8 +126,8 @@ DECLARE_EVENT_CLASS(cgroup_migrate,
+
+ TP_STRUCT__entry(
+ __field( int, dst_root )
+- __field( int, dst_id )
+ __field( int, dst_level )
++ __field( u64, dst_id )
+ __field( int, pid )
+ __string( dst_path, path )
+ __string( comm, task->comm )
+@@ -142,7 +142,7 @@ DECLARE_EVENT_CLASS(cgroup_migrate,
+ __assign_str(comm, task->comm);
+ ),
+
+- TP_printk("dst_root=%d dst_id=%d dst_level=%d dst_path=%s pid=%d comm=%s",
++ TP_printk("dst_root=%d dst_id=%llu dst_level=%d dst_path=%s pid=%d comm=%s",
+ __entry->dst_root, __entry->dst_id, __entry->dst_level,
+ __get_str(dst_path), __entry->pid, __get_str(comm))
+ );
+@@ -171,8 +171,8 @@ DECLARE_EVENT_CLASS(cgroup_event,
+
+ TP_STRUCT__entry(
+ __field( int, root )
+- __field( int, id )
+ __field( int, level )
++ __field( u64, id )
+ __string( path, path )
+ __field( int, val )
+ ),
+@@ -185,7 +185,7 @@ DECLARE_EVENT_CLASS(cgroup_event,
+ __entry->val = val;
+ ),
+
+- TP_printk("root=%d id=%d level=%d path=%s val=%d",
++ TP_printk("root=%d id=%llu level=%d path=%s val=%d",
+ __entry->root, __entry->id, __entry->level, __get_str(path),
+ __entry->val)
+ );
+diff --git a/include/trace/events/ext4.h b/include/trace/events/ext4.h
+index 0ea36b2b0662a..61a64d1b2bb68 100644
+--- a/include/trace/events/ext4.h
++++ b/include/trace/events/ext4.h
+@@ -95,6 +95,17 @@ TRACE_DEFINE_ENUM(ES_REFERENCED_B);
+ { FALLOC_FL_COLLAPSE_RANGE, "COLLAPSE_RANGE"}, \
+ { FALLOC_FL_ZERO_RANGE, "ZERO_RANGE"})
+
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_XATTR);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_CROSS_RENAME);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_JOURNAL_FLAG_CHANGE);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_NOMEM);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_SWAP_BOOT);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_RESIZE);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_RENAME_DIR);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_FALLOC_RANGE);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_INODE_JOURNAL_DATA);
++TRACE_DEFINE_ENUM(EXT4_FC_REASON_MAX);
++
+ #define show_fc_reason(reason) \
+ __print_symbolic(reason, \
+ { EXT4_FC_REASON_XATTR, "XATTR"}, \
+@@ -2723,41 +2734,50 @@ TRACE_EVENT(ext4_fc_commit_stop,
+
+ #define FC_REASON_NAME_STAT(reason) \
+ show_fc_reason(reason), \
+- __entry->sbi->s_fc_stats.fc_ineligible_reason_count[reason]
++ __entry->fc_ineligible_rc[reason]
+
+ TRACE_EVENT(ext4_fc_stats,
+- TP_PROTO(struct super_block *sb),
+-
+- TP_ARGS(sb),
++ TP_PROTO(struct super_block *sb),
+
+- TP_STRUCT__entry(
+- __field(dev_t, dev)
+- __field(struct ext4_sb_info *, sbi)
+- __field(int, count)
+- ),
++ TP_ARGS(sb),
+
+- TP_fast_assign(
+- __entry->dev = sb->s_dev;
+- __entry->sbi = EXT4_SB(sb);
+- ),
++ TP_STRUCT__entry(
++ __field(dev_t, dev)
++ __array(unsigned int, fc_ineligible_rc, EXT4_FC_REASON_MAX)
++ __field(unsigned long, fc_commits)
++ __field(unsigned long, fc_ineligible_commits)
++ __field(unsigned long, fc_numblks)
++ ),
+
+- TP_printk("dev %d:%d fc ineligible reasons:\n"
+- "%s:%d, %s:%d, %s:%d, %s:%d, %s:%d, %s:%d, %s:%d, %s:%d, %s:%d; "
+- "num_commits:%ld, ineligible: %ld, numblks: %ld",
+- MAJOR(__entry->dev), MINOR(__entry->dev),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_XATTR),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_CROSS_RENAME),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_JOURNAL_FLAG_CHANGE),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_NOMEM),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_SWAP_BOOT),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_RESIZE),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_RENAME_DIR),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_FALLOC_RANGE),
+- FC_REASON_NAME_STAT(EXT4_FC_REASON_INODE_JOURNAL_DATA),
+- __entry->sbi->s_fc_stats.fc_num_commits,
+- __entry->sbi->s_fc_stats.fc_ineligible_commits,
+- __entry->sbi->s_fc_stats.fc_numblks)
++ TP_fast_assign(
++ int i;
+
++ __entry->dev = sb->s_dev;
++ for (i = 0; i < EXT4_FC_REASON_MAX; i++) {
++ __entry->fc_ineligible_rc[i] =
++ EXT4_SB(sb)->s_fc_stats.fc_ineligible_reason_count[i];
++ }
++ __entry->fc_commits = EXT4_SB(sb)->s_fc_stats.fc_num_commits;
++ __entry->fc_ineligible_commits =
++ EXT4_SB(sb)->s_fc_stats.fc_ineligible_commits;
++ __entry->fc_numblks = EXT4_SB(sb)->s_fc_stats.fc_numblks;
++ ),
++
++ TP_printk("dev %d,%d fc ineligible reasons:\n"
++ "%s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u "
++ "num_commits:%lu, ineligible: %lu, numblks: %lu",
++ MAJOR(__entry->dev), MINOR(__entry->dev),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_XATTR),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_CROSS_RENAME),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_JOURNAL_FLAG_CHANGE),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_NOMEM),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_SWAP_BOOT),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_RESIZE),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_RENAME_DIR),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_FALLOC_RANGE),
++ FC_REASON_NAME_STAT(EXT4_FC_REASON_INODE_JOURNAL_DATA),
++ __entry->fc_commits, __entry->fc_ineligible_commits,
++ __entry->fc_numblks)
+ );
+
+ #define DEFINE_TRACE_DENTRY_EVENT(__type) \
+diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
+index 4e881d91c8744..4cb055af1ec0b 100644
+--- a/include/trace/events/f2fs.h
++++ b/include/trace/events/f2fs.h
+@@ -807,20 +807,20 @@ TRACE_EVENT(f2fs_lookup_start,
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(ino_t, ino)
+- __field(const char *, name)
++ __string(name, dentry->d_name.name)
+ __field(unsigned int, flags)
+ ),
+
+ TP_fast_assign(
+ __entry->dev = dir->i_sb->s_dev;
+ __entry->ino = dir->i_ino;
+- __entry->name = dentry->d_name.name;
++ __assign_str(name, dentry->d_name.name);
+ __entry->flags = flags;
+ ),
+
+ TP_printk("dev = (%d,%d), pino = %lu, name:%s, flags:%u",
+ show_dev_ino(__entry),
+- __entry->name,
++ __get_str(name),
+ __entry->flags)
+ );
+
+@@ -834,7 +834,7 @@ TRACE_EVENT(f2fs_lookup_end,
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(ino_t, ino)
+- __field(const char *, name)
++ __string(name, dentry->d_name.name)
+ __field(nid_t, cino)
+ __field(int, err)
+ ),
+@@ -842,14 +842,14 @@ TRACE_EVENT(f2fs_lookup_end,
+ TP_fast_assign(
+ __entry->dev = dir->i_sb->s_dev;
+ __entry->ino = dir->i_ino;
+- __entry->name = dentry->d_name.name;
++ __assign_str(name, dentry->d_name.name);
+ __entry->cino = ino;
+ __entry->err = err;
+ ),
+
+ TP_printk("dev = (%d,%d), pino = %lu, name:%s, ino:%u, err:%d",
+ show_dev_ino(__entry),
+- __entry->name,
++ __get_str(name),
+ __entry->cino,
+ __entry->err)
+ );
+diff --git a/include/trace/events/random.h b/include/trace/events/random.h
+deleted file mode 100644
+index 3d7b432ca5f31..0000000000000
+--- a/include/trace/events/random.h
++++ /dev/null
+@@ -1,247 +0,0 @@
+-/* SPDX-License-Identifier: GPL-2.0 */
+-#undef TRACE_SYSTEM
+-#define TRACE_SYSTEM random
+-
+-#if !defined(_TRACE_RANDOM_H) || defined(TRACE_HEADER_MULTI_READ)
+-#define _TRACE_RANDOM_H
+-
+-#include <linux/writeback.h>
+-#include <linux/tracepoint.h>
+-
+-TRACE_EVENT(add_device_randomness,
+- TP_PROTO(int bytes, unsigned long IP),
+-
+- TP_ARGS(bytes, IP),
+-
+- TP_STRUCT__entry(
+- __field( int, bytes )
+- __field(unsigned long, IP )
+- ),
+-
+- TP_fast_assign(
+- __entry->bytes = bytes;
+- __entry->IP = IP;
+- ),
+-
+- TP_printk("bytes %d caller %pS",
+- __entry->bytes, (void *)__entry->IP)
+-);
+-
+-DECLARE_EVENT_CLASS(random__mix_pool_bytes,
+- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
+-
+- TP_ARGS(pool_name, bytes, IP),
+-
+- TP_STRUCT__entry(
+- __field( const char *, pool_name )
+- __field( int, bytes )
+- __field(unsigned long, IP )
+- ),
+-
+- TP_fast_assign(
+- __entry->pool_name = pool_name;
+- __entry->bytes = bytes;
+- __entry->IP = IP;
+- ),
+-
+- TP_printk("%s pool: bytes %d caller %pS",
+- __entry->pool_name, __entry->bytes, (void *)__entry->IP)
+-);
+-
+-DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes,
+- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
+-
+- TP_ARGS(pool_name, bytes, IP)
+-);
+-
+-DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes_nolock,
+- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
+-
+- TP_ARGS(pool_name, bytes, IP)
+-);
+-
+-TRACE_EVENT(credit_entropy_bits,
+- TP_PROTO(const char *pool_name, int bits, int entropy_count,
+- unsigned long IP),
+-
+- TP_ARGS(pool_name, bits, entropy_count, IP),
+-
+- TP_STRUCT__entry(
+- __field( const char *, pool_name )
+- __field( int, bits )
+- __field( int, entropy_count )
+- __field(unsigned long, IP )
+- ),
+-
+- TP_fast_assign(
+- __entry->pool_name = pool_name;
+- __entry->bits = bits;
+- __entry->entropy_count = entropy_count;
+- __entry->IP = IP;
+- ),
+-
+- TP_printk("%s pool: bits %d entropy_count %d caller %pS",
+- __entry->pool_name, __entry->bits,
+- __entry->entropy_count, (void *)__entry->IP)
+-);
+-
+-TRACE_EVENT(debit_entropy,
+- TP_PROTO(const char *pool_name, int debit_bits),
+-
+- TP_ARGS(pool_name, debit_bits),
+-
+- TP_STRUCT__entry(
+- __field( const char *, pool_name )
+- __field( int, debit_bits )
+- ),
+-
+- TP_fast_assign(
+- __entry->pool_name = pool_name;
+- __entry->debit_bits = debit_bits;
+- ),
+-
+- TP_printk("%s: debit_bits %d", __entry->pool_name,
+- __entry->debit_bits)
+-);
+-
+-TRACE_EVENT(add_input_randomness,
+- TP_PROTO(int input_bits),
+-
+- TP_ARGS(input_bits),
+-
+- TP_STRUCT__entry(
+- __field( int, input_bits )
+- ),
+-
+- TP_fast_assign(
+- __entry->input_bits = input_bits;
+- ),
+-
+- TP_printk("input_pool_bits %d", __entry->input_bits)
+-);
+-
+-TRACE_EVENT(add_disk_randomness,
+- TP_PROTO(dev_t dev, int input_bits),
+-
+- TP_ARGS(dev, input_bits),
+-
+- TP_STRUCT__entry(
+- __field( dev_t, dev )
+- __field( int, input_bits )
+- ),
+-
+- TP_fast_assign(
+- __entry->dev = dev;
+- __entry->input_bits = input_bits;
+- ),
+-
+- TP_printk("dev %d,%d input_pool_bits %d", MAJOR(__entry->dev),
+- MINOR(__entry->dev), __entry->input_bits)
+-);
+-
+-DECLARE_EVENT_CLASS(random__get_random_bytes,
+- TP_PROTO(int nbytes, unsigned long IP),
+-
+- TP_ARGS(nbytes, IP),
+-
+- TP_STRUCT__entry(
+- __field( int, nbytes )
+- __field(unsigned long, IP )
+- ),
+-
+- TP_fast_assign(
+- __entry->nbytes = nbytes;
+- __entry->IP = IP;
+- ),
+-
+- TP_printk("nbytes %d caller %pS", __entry->nbytes, (void *)__entry->IP)
+-);
+-
+-DEFINE_EVENT(random__get_random_bytes, get_random_bytes,
+- TP_PROTO(int nbytes, unsigned long IP),
+-
+- TP_ARGS(nbytes, IP)
+-);
+-
+-DEFINE_EVENT(random__get_random_bytes, get_random_bytes_arch,
+- TP_PROTO(int nbytes, unsigned long IP),
+-
+- TP_ARGS(nbytes, IP)
+-);
+-
+-DECLARE_EVENT_CLASS(random__extract_entropy,
+- TP_PROTO(const char *pool_name, int nbytes, int entropy_count,
+- unsigned long IP),
+-
+- TP_ARGS(pool_name, nbytes, entropy_count, IP),
+-
+- TP_STRUCT__entry(
+- __field( const char *, pool_name )
+- __field( int, nbytes )
+- __field( int, entropy_count )
+- __field(unsigned long, IP )
+- ),
+-
+- TP_fast_assign(
+- __entry->pool_name = pool_name;
+- __entry->nbytes = nbytes;
+- __entry->entropy_count = entropy_count;
+- __entry->IP = IP;
+- ),
+-
+- TP_printk("%s pool: nbytes %d entropy_count %d caller %pS",
+- __entry->pool_name, __entry->nbytes, __entry->entropy_count,
+- (void *)__entry->IP)
+-);
+-
+-
+-DEFINE_EVENT(random__extract_entropy, extract_entropy,
+- TP_PROTO(const char *pool_name, int nbytes, int entropy_count,
+- unsigned long IP),
+-
+- TP_ARGS(pool_name, nbytes, entropy_count, IP)
+-);
+-
+-TRACE_EVENT(urandom_read,
+- TP_PROTO(int got_bits, int pool_left, int input_left),
+-
+- TP_ARGS(got_bits, pool_left, input_left),
+-
+- TP_STRUCT__entry(
+- __field( int, got_bits )
+- __field( int, pool_left )
+- __field( int, input_left )
+- ),
+-
+- TP_fast_assign(
+- __entry->got_bits = got_bits;
+- __entry->pool_left = pool_left;
+- __entry->input_left = input_left;
+- ),
+-
+- TP_printk("got_bits %d nonblocking_pool_entropy_left %d "
+- "input_entropy_left %d", __entry->got_bits,
+- __entry->pool_left, __entry->input_left)
+-);
+-
+-TRACE_EVENT(prandom_u32,
+-
+- TP_PROTO(unsigned int ret),
+-
+- TP_ARGS(ret),
+-
+- TP_STRUCT__entry(
+- __field( unsigned int, ret)
+- ),
+-
+- TP_fast_assign(
+- __entry->ret = ret;
+- ),
+-
+- TP_printk("ret=%u" , __entry->ret)
+-);
+-
+-#endif /* _TRACE_RANDOM_H */
+-
+-/* This part must be outside protection */
+-#include <trace/define_trace.h>
+diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h
+index e70c90116edae..4a3ab0ed6e062 100644
+--- a/include/trace/events/rxrpc.h
++++ b/include/trace/events/rxrpc.h
+@@ -83,12 +83,15 @@ enum rxrpc_call_trace {
+ rxrpc_call_error,
+ rxrpc_call_got,
+ rxrpc_call_got_kernel,
++ rxrpc_call_got_timer,
+ rxrpc_call_got_userid,
+ rxrpc_call_new_client,
+ rxrpc_call_new_service,
+ rxrpc_call_put,
+ rxrpc_call_put_kernel,
+ rxrpc_call_put_noqueue,
++ rxrpc_call_put_notimer,
++ rxrpc_call_put_timer,
+ rxrpc_call_put_userid,
+ rxrpc_call_queued,
+ rxrpc_call_queued_ref,
+@@ -278,12 +281,15 @@ enum rxrpc_tx_point {
+ EM(rxrpc_call_error, "*E*") \
+ EM(rxrpc_call_got, "GOT") \
+ EM(rxrpc_call_got_kernel, "Gke") \
++ EM(rxrpc_call_got_timer, "GTM") \
+ EM(rxrpc_call_got_userid, "Gus") \
+ EM(rxrpc_call_new_client, "NWc") \
+ EM(rxrpc_call_new_service, "NWs") \
+ EM(rxrpc_call_put, "PUT") \
+ EM(rxrpc_call_put_kernel, "Pke") \
+- EM(rxrpc_call_put_noqueue, "PNQ") \
++ EM(rxrpc_call_put_noqueue, "PnQ") \
++ EM(rxrpc_call_put_notimer, "PnT") \
++ EM(rxrpc_call_put_timer, "PTM") \
+ EM(rxrpc_call_put_userid, "Pus") \
+ EM(rxrpc_call_queued, "QUE") \
+ EM(rxrpc_call_queued_ref, "QUR") \
+diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h
+index 2d04eb96d4183..2a598fb45bf4f 100644
+--- a/include/trace/events/sunrpc.h
++++ b/include/trace/events/sunrpc.h
+@@ -925,18 +925,19 @@ TRACE_EVENT(rpc_socket_nospace,
+
+ #define rpc_show_xprt_state(x) \
+ __print_flags(x, "|", \
+- { (1UL << XPRT_LOCKED), "LOCKED"}, \
+- { (1UL << XPRT_CONNECTED), "CONNECTED"}, \
+- { (1UL << XPRT_CONNECTING), "CONNECTING"}, \
+- { (1UL << XPRT_CLOSE_WAIT), "CLOSE_WAIT"}, \
+- { (1UL << XPRT_BOUND), "BOUND"}, \
+- { (1UL << XPRT_BINDING), "BINDING"}, \
+- { (1UL << XPRT_CLOSING), "CLOSING"}, \
+- { (1UL << XPRT_OFFLINE), "OFFLINE"}, \
+- { (1UL << XPRT_REMOVE), "REMOVE"}, \
+- { (1UL << XPRT_CONGESTED), "CONGESTED"}, \
+- { (1UL << XPRT_CWND_WAIT), "CWND_WAIT"}, \
+- { (1UL << XPRT_WRITE_SPACE), "WRITE_SPACE"})
++ { BIT(XPRT_LOCKED), "LOCKED" }, \
++ { BIT(XPRT_CONNECTED), "CONNECTED" }, \
++ { BIT(XPRT_CONNECTING), "CONNECTING" }, \
++ { BIT(XPRT_CLOSE_WAIT), "CLOSE_WAIT" }, \
++ { BIT(XPRT_BOUND), "BOUND" }, \
++ { BIT(XPRT_BINDING), "BINDING" }, \
++ { BIT(XPRT_CLOSING), "CLOSING" }, \
++ { BIT(XPRT_OFFLINE), "OFFLINE" }, \
++ { BIT(XPRT_REMOVE), "REMOVE" }, \
++ { BIT(XPRT_CONGESTED), "CONGESTED" }, \
++ { BIT(XPRT_CWND_WAIT), "CWND_WAIT" }, \
++ { BIT(XPRT_WRITE_SPACE), "WRITE_SPACE" }, \
++ { BIT(XPRT_SND_IS_COOKIE), "SND_IS_COOKIE" })
+
+ DECLARE_EVENT_CLASS(rpc_xprt_lifetime_class,
+ TP_PROTO(
+@@ -975,7 +976,6 @@ DEFINE_RPC_XPRT_LIFETIME_EVENT(connect);
+ DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_auto);
+ DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_done);
+ DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_force);
+-DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_cleanup);
+ DEFINE_RPC_XPRT_LIFETIME_EVENT(destroy);
+
+ DECLARE_EVENT_CLASS(rpc_xprt_event,
+@@ -1133,8 +1133,11 @@ DECLARE_EVENT_CLASS(xprt_writelock_event,
+ __entry->task_id = -1;
+ __entry->client_id = -1;
+ }
+- __entry->snd_task_id = xprt->snd_task ?
+- xprt->snd_task->tk_pid : -1;
++ if (xprt->snd_task &&
++ !test_bit(XPRT_SND_IS_COOKIE, &xprt->state))
++ __entry->snd_task_id = xprt->snd_task->tk_pid;
++ else
++ __entry->snd_task_id = -1;
+ ),
+
+ TP_printk("task:%u@%u snd_task:%u",
+@@ -1178,8 +1181,12 @@ DECLARE_EVENT_CLASS(xprt_cong_event,
+ __entry->task_id = -1;
+ __entry->client_id = -1;
+ }
+- __entry->snd_task_id = xprt->snd_task ?
+- xprt->snd_task->tk_pid : -1;
++ if (xprt->snd_task &&
++ !test_bit(XPRT_SND_IS_COOKIE, &xprt->state))
++ __entry->snd_task_id = xprt->snd_task->tk_pid;
++ else
++ __entry->snd_task_id = -1;
++
+ __entry->cong = xprt->cong;
+ __entry->cwnd = xprt->cwnd;
+ __entry->wait = test_bit(XPRT_CWND_WAIT, &xprt->state);
+@@ -1719,10 +1726,11 @@ TRACE_EVENT(svc_xprt_create_err,
+ const char *program,
+ const char *protocol,
+ struct sockaddr *sap,
++ size_t salen,
+ const struct svc_xprt *xprt
+ ),
+
+- TP_ARGS(program, protocol, sap, xprt),
++ TP_ARGS(program, protocol, sap, salen, xprt),
+
+ TP_STRUCT__entry(
+ __field(long, error)
+@@ -1735,7 +1743,7 @@ TRACE_EVENT(svc_xprt_create_err,
+ __entry->error = PTR_ERR(xprt);
+ __assign_str(program, program);
+ __assign_str(protocol, protocol);
+- memcpy(__entry->addr, sap, sizeof(__entry->addr));
++ memcpy(__entry->addr, sap, min(salen, sizeof(__entry->addr)));
+ ),
+
+ TP_printk("addr=%pISpc program=%s protocol=%s error=%ld",
+@@ -1915,17 +1923,18 @@ DECLARE_EVENT_CLASS(svc_deferred_event,
+ TP_STRUCT__entry(
+ __field(const void *, dr)
+ __field(u32, xid)
+- __string(addr, dr->xprt->xpt_remotebuf)
++ __array(__u8, addr, INET6_ADDRSTRLEN + 10)
+ ),
+
+ TP_fast_assign(
+ __entry->dr = dr;
+ __entry->xid = be32_to_cpu(*(__be32 *)(dr->args +
+ (dr->xprt_hlen>>2)));
+- __assign_str(addr, dr->xprt->xpt_remotebuf);
++ snprintf(__entry->addr, sizeof(__entry->addr) - 1,
++ "%pISpc", (struct sockaddr *)&dr->addr);
+ ),
+
+- TP_printk("addr=%s dr=%p xid=0x%08x", __get_str(addr), __entry->dr,
++ TP_printk("addr=%s dr=%p xid=0x%08x", __entry->addr, __entry->dr,
+ __entry->xid)
+ );
+
+@@ -2103,17 +2112,17 @@ DECLARE_EVENT_CLASS(svcsock_accept_class,
+ TP_STRUCT__entry(
+ __field(long, status)
+ __string(service, service)
+- __array(unsigned char, addr, sizeof(struct sockaddr_in6))
++ __field(unsigned int, netns_ino)
+ ),
+
+ TP_fast_assign(
+ __entry->status = status;
+ __assign_str(service, service);
+- memcpy(__entry->addr, &xprt->xpt_local, sizeof(__entry->addr));
++ __entry->netns_ino = xprt->xpt_net->ns.inum;
+ ),
+
+- TP_printk("listener=%pISpc service=%s status=%ld",
+- __entry->addr, __get_str(service), __entry->status
++ TP_printk("addr=listener service=%s status=%ld",
++ __get_str(service), __entry->status
+ )
+ );
+
+diff --git a/include/uapi/asm-generic/poll.h b/include/uapi/asm-generic/poll.h
+index 41b509f410bf9..f9c520ce4bf4e 100644
+--- a/include/uapi/asm-generic/poll.h
++++ b/include/uapi/asm-generic/poll.h
+@@ -29,7 +29,7 @@
+ #define POLLRDHUP 0x2000
+ #endif
+
+-#define POLLFREE (__force __poll_t)0x4000 /* currently only for epoll */
++#define POLLFREE (__force __poll_t)0x4000
+
+ #define POLL_BUSY_LOOP (__force __poll_t)0x8000
+
+diff --git a/include/uapi/asm-generic/signal-defs.h b/include/uapi/asm-generic/signal-defs.h
+index fe929e7b77ca1..7572f2f46ee89 100644
+--- a/include/uapi/asm-generic/signal-defs.h
++++ b/include/uapi/asm-generic/signal-defs.h
+@@ -45,6 +45,7 @@
+ #define SA_UNSUPPORTED 0x00000400
+ #define SA_EXPOSE_TAGBITS 0x00000800
+ /* 0x00010000 used on mips */
++/* 0x00800000 used for internal SA_IMMUTABLE */
+ /* 0x01000000 used on x86 */
+ /* 0x02000000 used on x86 */
+ /*
+diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
+index 791f31dd0abee..a887e582f0e78 100644
+--- a/include/uapi/linux/bpf.h
++++ b/include/uapi/linux/bpf.h
+@@ -2276,8 +2276,8 @@ union bpf_attr {
+ * Return
+ * The return value depends on the result of the test, and can be:
+ *
+- * * 0, if current task belongs to the cgroup2.
+- * * 1, if current task does not belong to the cgroup2.
++ * * 1, if current task belongs to the cgroup2.
++ * * 0, if current task does not belong to the cgroup2.
+ * * A negative error code, if an error occurred.
+ *
+ * long bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags)
+@@ -2965,8 +2965,8 @@ union bpf_attr {
+ *
+ * # sysctl kernel.perf_event_max_stack=<new value>
+ * Return
+- * A non-negative value equal to or less than *size* on success,
+- * or a negative error in case of failure.
++ * The non-negative copied *buf* length equal to or less than
++ * *size* on success, or a negative error in case of failure.
+ *
+ * long bpf_skb_load_bytes_relative(const void *skb, u32 offset, void *to, u32 len, u32 start_header)
+ * Description
+@@ -4269,8 +4269,8 @@ union bpf_attr {
+ *
+ * # sysctl kernel.perf_event_max_stack=<new value>
+ * Return
+- * A non-negative value equal to or less than *size* on success,
+- * or a negative error in case of failure.
++ * The non-negative copied *buf* length equal to or less than
++ * *size* on success, or a negative error in case of failure.
+ *
+ * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags)
+ * Description
+@@ -5347,7 +5347,8 @@ struct bpf_sock {
+ __u32 src_ip4;
+ __u32 src_ip6[4];
+ __u32 src_port; /* host byte order */
+- __u32 dst_port; /* network byte order */
++ __be16 dst_port; /* network byte order */
++ __u16 :16; /* zero padding */
+ __u32 dst_ip4;
+ __u32 dst_ip6[4];
+ __u32 state;
+@@ -6222,7 +6223,8 @@ struct bpf_sk_lookup {
+ __u32 protocol; /* IP protocol (IPPROTO_TCP, IPPROTO_UDP) */
+ __u32 remote_ip4; /* Network byte order */
+ __u32 remote_ip6[4]; /* Network byte order */
+- __u32 remote_port; /* Network byte order */
++ __be16 remote_port; /* Network byte order */
++ __u16 :16; /* Zero padding */
+ __u32 local_ip4; /* Network byte order */
+ __u32 local_ip6[4]; /* Network byte order */
+ __u32 local_port; /* Host byte order */
+diff --git a/include/uapi/linux/byteorder/big_endian.h b/include/uapi/linux/byteorder/big_endian.h
+index 2199adc6a6c20..80aa5c41a7636 100644
+--- a/include/uapi/linux/byteorder/big_endian.h
++++ b/include/uapi/linux/byteorder/big_endian.h
+@@ -9,6 +9,7 @@
+ #define __BIG_ENDIAN_BITFIELD
+ #endif
+
++#include <linux/stddef.h>
+ #include <linux/types.h>
+ #include <linux/swab.h>
+
+diff --git a/include/uapi/linux/byteorder/little_endian.h b/include/uapi/linux/byteorder/little_endian.h
+index 601c904fd5cd9..cd98982e7523e 100644
+--- a/include/uapi/linux/byteorder/little_endian.h
++++ b/include/uapi/linux/byteorder/little_endian.h
+@@ -9,6 +9,7 @@
+ #define __LITTLE_ENDIAN_BITFIELD
+ #endif
+
++#include <linux/stddef.h>
+ #include <linux/types.h>
+ #include <linux/swab.h>
+
+diff --git a/include/uapi/linux/can/isotp.h b/include/uapi/linux/can/isotp.h
+index c55935b64ccc8..590f8aea2b6d2 100644
+--- a/include/uapi/linux/can/isotp.h
++++ b/include/uapi/linux/can/isotp.h
+@@ -137,20 +137,16 @@ struct can_isotp_ll_options {
+ #define CAN_ISOTP_WAIT_TX_DONE 0x400 /* wait for tx completion */
+ #define CAN_ISOTP_SF_BROADCAST 0x800 /* 1-to-N functional addressing */
+
+-/* default values */
++/* protocol machine default values */
+
+ #define CAN_ISOTP_DEFAULT_FLAGS 0
+ #define CAN_ISOTP_DEFAULT_EXT_ADDRESS 0x00
+ #define CAN_ISOTP_DEFAULT_PAD_CONTENT 0xCC /* prevent bit-stuffing */
+-#define CAN_ISOTP_DEFAULT_FRAME_TXTIME 0
++#define CAN_ISOTP_DEFAULT_FRAME_TXTIME 50000 /* 50 micro seconds */
+ #define CAN_ISOTP_DEFAULT_RECV_BS 0
+ #define CAN_ISOTP_DEFAULT_RECV_STMIN 0x00
+ #define CAN_ISOTP_DEFAULT_RECV_WFTMAX 0
+
+-#define CAN_ISOTP_DEFAULT_LL_MTU CAN_MTU
+-#define CAN_ISOTP_DEFAULT_LL_TX_DL CAN_MAX_DLEN
+-#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS 0
+-
+ /*
+ * Remark on CAN_ISOTP_DEFAULT_RECV_* values:
+ *
+@@ -162,4 +158,24 @@ struct can_isotp_ll_options {
+ * consistency and copied directly into the flow control (FC) frame.
+ */
+
++/* link layer default values => make use of Classical CAN frames */
++
++#define CAN_ISOTP_DEFAULT_LL_MTU CAN_MTU
++#define CAN_ISOTP_DEFAULT_LL_TX_DL CAN_MAX_DLEN
++#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS 0
++
++/*
++ * The CAN_ISOTP_DEFAULT_FRAME_TXTIME has become a non-zero value as
++ * it only makes sense for isotp implementation tests to run without
++ * a N_As value. As user space applications usually do not set the
++ * frame_txtime element of struct can_isotp_options the new in-kernel
++ * default is very likely overwritten with zero when the sockopt()
++ * CAN_ISOTP_OPTS is invoked.
++ * To make sure that a N_As value of zero is only set intentional the
++ * value '0' is now interpreted as 'do not change the current value'.
++ * When a frame_txtime of zero is required for testing purposes this
++ * CAN_ISOTP_FRAME_TXTIME_ZERO u32 value has to be set in frame_txtime.
++ */
++#define CAN_ISOTP_FRAME_TXTIME_ZERO 0xFFFFFFFF
++
+ #endif /* !_UAPI_CAN_ISOTP_H */
+diff --git a/include/uapi/linux/cyclades.h b/include/uapi/linux/cyclades.h
+new file mode 100644
+index 0000000000000..6225c5aebe06a
+--- /dev/null
++++ b/include/uapi/linux/cyclades.h
+@@ -0,0 +1,35 @@
++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
++
++#ifndef _UAPI_LINUX_CYCLADES_H
++#define _UAPI_LINUX_CYCLADES_H
++
++#warning "Support for features provided by this header has been removed"
++#warning "Please consider updating your code"
++
++struct cyclades_monitor {
++ unsigned long int_count;
++ unsigned long char_count;
++ unsigned long char_max;
++ unsigned long char_last;
++};
++
++#define CYGETMON 0x435901
++#define CYGETTHRESH 0x435902
++#define CYSETTHRESH 0x435903
++#define CYGETDEFTHRESH 0x435904
++#define CYSETDEFTHRESH 0x435905
++#define CYGETTIMEOUT 0x435906
++#define CYSETTIMEOUT 0x435907
++#define CYGETDEFTIMEOUT 0x435908
++#define CYSETDEFTIMEOUT 0x435909
++#define CYSETRFLOW 0x43590a
++#define CYGETRFLOW 0x43590b
++#define CYSETRTSDTR_INV 0x43590c
++#define CYGETRTSDTR_INV 0x43590d
++#define CYZSETPOLLCYCLE 0x43590e
++#define CYZGETPOLLCYCLE 0x43590f
++#define CYGETCD1400VER 0x435910
++#define CYSETWAIT 0x435912
++#define CYGETWAIT 0x435913
++
++#endif /* _UAPI_LINUX_CYCLADES_H */
+diff --git a/include/uapi/linux/dma-buf.h b/include/uapi/linux/dma-buf.h
+index 8e4a2ca0bcbf7..b1523cb8ab307 100644
+--- a/include/uapi/linux/dma-buf.h
++++ b/include/uapi/linux/dma-buf.h
+@@ -92,7 +92,7 @@ struct dma_buf_sync {
+ * between them in actual uapi, they're just different numbers.
+ */
+ #define DMA_BUF_SET_NAME _IOW(DMA_BUF_BASE, 1, const char *)
+-#define DMA_BUF_SET_NAME_A _IOW(DMA_BUF_BASE, 1, u32)
+-#define DMA_BUF_SET_NAME_B _IOW(DMA_BUF_BASE, 1, u64)
++#define DMA_BUF_SET_NAME_A _IOW(DMA_BUF_BASE, 1, __u32)
++#define DMA_BUF_SET_NAME_B _IOW(DMA_BUF_BASE, 1, __u64)
+
+ #endif
+diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h
+index 5545f1ca9237c..f7204bdfe8db1 100644
+--- a/include/uapi/linux/ethtool_netlink.h
++++ b/include/uapi/linux/ethtool_netlink.h
+@@ -407,7 +407,9 @@ enum {
+ ETHTOOL_A_PAUSE_STAT_TX_FRAMES,
+ ETHTOOL_A_PAUSE_STAT_RX_FRAMES,
+
+- /* add new constants above here */
++ /* add new constants above here
++ * adjust ETHTOOL_PAUSE_STAT_CNT if adding non-stats!
++ */
+ __ETHTOOL_A_PAUSE_STAT_CNT,
+ ETHTOOL_A_PAUSE_STAT_MAX = (__ETHTOOL_A_PAUSE_STAT_CNT - 1)
+ };
+diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h
+index 225ec87d4f228..7989d9483ea75 100644
+--- a/include/uapi/linux/input-event-codes.h
++++ b/include/uapi/linux/input-event-codes.h
+@@ -278,7 +278,8 @@
+ #define KEY_PAUSECD 201
+ #define KEY_PROG3 202
+ #define KEY_PROG4 203
+-#define KEY_DASHBOARD 204 /* AL Dashboard */
++#define KEY_ALL_APPLICATIONS 204 /* AC Desktop Show All Applications */
++#define KEY_DASHBOARD KEY_ALL_APPLICATIONS
+ #define KEY_SUSPEND 205
+ #define KEY_CLOSE 206 /* AC Close */
+ #define KEY_PLAY 207
+@@ -612,6 +613,7 @@
+ #define KEY_ASSISTANT 0x247 /* AL Context-aware desktop assistant */
+ #define KEY_KBD_LAYOUT_NEXT 0x248 /* AC Next Keyboard Layout Select */
+ #define KEY_EMOJI_PICKER 0x249 /* Show/hide emoji picker (HUTRR101) */
++#define KEY_DICTATE 0x24a /* Start or Stop Voice Dictation Session (HUTRR99) */
+
+ #define KEY_BRIGHTNESS_MIN 0x250 /* Set Brightness to Minimum */
+ #define KEY_BRIGHTNESS_MAX 0x251 /* Set Brightness to Maximum */
+diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h
+index f66038b9551fa..80c40194e2977 100644
+--- a/include/uapi/linux/mptcp.h
++++ b/include/uapi/linux/mptcp.h
+@@ -129,19 +129,21 @@ struct mptcp_info {
+ * MPTCP_EVENT_REMOVED: token, rem_id
+ * An address has been lost by the peer.
+ *
+- * MPTCP_EVENT_SUB_ESTABLISHED: token, family, saddr4 | saddr6,
+- * daddr4 | daddr6, sport, dport, backup,
+- * if_idx [, error]
++ * MPTCP_EVENT_SUB_ESTABLISHED: token, family, loc_id, rem_id,
++ * saddr4 | saddr6, daddr4 | daddr6, sport,
++ * dport, backup, if_idx [, error]
+ * A new subflow has been established. 'error' should not be set.
+ *
+- * MPTCP_EVENT_SUB_CLOSED: token, family, saddr4 | saddr6, daddr4 | daddr6,
+- * sport, dport, backup, if_idx [, error]
++ * MPTCP_EVENT_SUB_CLOSED: token, family, loc_id, rem_id, saddr4 | saddr6,
++ * daddr4 | daddr6, sport, dport, backup, if_idx
++ * [, error]
+ * A subflow has been closed. An error (copy of sk_err) could be set if an
+ * error has been detected for this subflow.
+ *
+- * MPTCP_EVENT_SUB_PRIORITY: token, family, saddr4 | saddr6, daddr4 | daddr6,
+- * sport, dport, backup, if_idx [, error]
+- * The priority of a subflow has changed. 'error' should not be set.
++ * MPTCP_EVENT_SUB_PRIORITY: token, family, loc_id, rem_id, saddr4 | saddr6,
++ * daddr4 | daddr6, sport, dport, backup, if_idx
++ * [, error]
++ * The priority of a subflow has changed. 'error' should not be set.
+ */
+ enum mptcp_event_type {
+ MPTCP_EVENT_UNSPEC = 0,
+diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h
+index 4b3395082d15c..26071021e986f 100644
+--- a/include/uapi/linux/netfilter/nf_conntrack_common.h
++++ b/include/uapi/linux/netfilter/nf_conntrack_common.h
+@@ -106,7 +106,7 @@ enum ip_conntrack_status {
+ IPS_NAT_CLASH = IPS_UNTRACKED,
+ #endif
+
+- /* Conntrack got a helper explicitly attached via CT target. */
++ /* Conntrack got a helper explicitly attached (ruleset, ctnetlink). */
+ IPS_HELPER_BIT = 13,
+ IPS_HELPER = (1 << IPS_HELPER_BIT),
+
+diff --git a/include/uapi/linux/nfc.h b/include/uapi/linux/nfc.h
+index f6e3c8c9c7449..4fa4e979e948a 100644
+--- a/include/uapi/linux/nfc.h
++++ b/include/uapi/linux/nfc.h
+@@ -263,7 +263,7 @@ enum nfc_sdp_attr {
+ #define NFC_SE_ENABLED 0x1
+
+ struct sockaddr_nfc {
+- sa_family_t sa_family;
++ __kernel_sa_family_t sa_family;
+ __u32 dev_idx;
+ __u32 target_idx;
+ __u32 nfc_protocol;
+@@ -271,14 +271,14 @@ struct sockaddr_nfc {
+
+ #define NFC_LLCP_MAX_SERVICE_NAME 63
+ struct sockaddr_nfc_llcp {
+- sa_family_t sa_family;
++ __kernel_sa_family_t sa_family;
+ __u32 dev_idx;
+ __u32 target_idx;
+ __u32 nfc_protocol;
+ __u8 dsap; /* Destination SAP, if known */
+ __u8 ssap; /* Source SAP to be bound to */
+ char service_name[NFC_LLCP_MAX_SERVICE_NAME]; /* Service name URI */;
+- size_t service_name_len;
++ __kernel_size_t service_name_len;
+ };
+
+ /* NFC socket protocols */
+diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
+index e709ae8235e7f..ff6ccbc6efe96 100644
+--- a/include/uapi/linux/pci_regs.h
++++ b/include/uapi/linux/pci_regs.h
+@@ -504,6 +504,12 @@
+ #define PCI_EXP_DEVCTL_URRE 0x0008 /* Unsupported Request Reporting En. */
+ #define PCI_EXP_DEVCTL_RELAX_EN 0x0010 /* Enable relaxed ordering */
+ #define PCI_EXP_DEVCTL_PAYLOAD 0x00e0 /* Max_Payload_Size */
++#define PCI_EXP_DEVCTL_PAYLOAD_128B 0x0000 /* 128 Bytes */
++#define PCI_EXP_DEVCTL_PAYLOAD_256B 0x0020 /* 256 Bytes */
++#define PCI_EXP_DEVCTL_PAYLOAD_512B 0x0040 /* 512 Bytes */
++#define PCI_EXP_DEVCTL_PAYLOAD_1024B 0x0060 /* 1024 Bytes */
++#define PCI_EXP_DEVCTL_PAYLOAD_2048B 0x0080 /* 2048 Bytes */
++#define PCI_EXP_DEVCTL_PAYLOAD_4096B 0x00a0 /* 4096 Bytes */
+ #define PCI_EXP_DEVCTL_EXT_TAG 0x0100 /* Extended Tag Field Enable */
+ #define PCI_EXP_DEVCTL_PHANTOM 0x0200 /* Phantom Functions Enable */
+ #define PCI_EXP_DEVCTL_AUX_PME 0x0400 /* Auxiliary Power PM Enable */
+diff --git a/include/uapi/linux/rfkill.h b/include/uapi/linux/rfkill.h
+index 9b77cfc42efa3..db6c8588c1d0c 100644
+--- a/include/uapi/linux/rfkill.h
++++ b/include/uapi/linux/rfkill.h
+@@ -159,8 +159,16 @@ struct rfkill_event_ext {
+ * old behaviour for all userspace, unless it explicitly opts in to the
+ * rules outlined here by using the new &struct rfkill_event_ext.
+ *
+- * Userspace using &struct rfkill_event_ext must adhere to the following
+- * rules
++ * Additionally, some other userspace (bluez, g-s-d) was reading with a
++ * large size but as streaming reads rather than message-based, or with
++ * too strict checks for the returned size. So eventually, we completely
++ * reverted this, and extended messages need to be opted in to by using
++ * an ioctl:
++ *
++ * ioctl(fd, RFKILL_IOCTL_MAX_SIZE, sizeof(struct rfkill_event_ext));
++ *
++ * Userspace using &struct rfkill_event_ext and the ioctl must adhere to
++ * the following rules:
+ *
+ * 1. accept short writes, optionally using them to detect that it's
+ * running on an older kernel;
+@@ -175,6 +183,8 @@ struct rfkill_event_ext {
+ #define RFKILL_IOC_MAGIC 'R'
+ #define RFKILL_IOC_NOINPUT 1
+ #define RFKILL_IOCTL_NOINPUT _IO(RFKILL_IOC_MAGIC, RFKILL_IOC_NOINPUT)
++#define RFKILL_IOC_MAX_SIZE 2
++#define RFKILL_IOCTL_MAX_SIZE _IOW(RFKILL_IOC_MAGIC, RFKILL_IOC_MAX_SIZE, __u32)
+
+ /* and that's all userspace gets */
+
+diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h
+index 9a402fdb60e97..77ee207623a9b 100644
+--- a/include/uapi/linux/rseq.h
++++ b/include/uapi/linux/rseq.h
+@@ -105,23 +105,11 @@ struct rseq {
+ * Read and set by the kernel. Set by user-space with single-copy
+ * atomicity semantics. This field should only be updated by the
+ * thread which registered this data structure. Aligned on 64-bit.
++ *
++ * 32-bit architectures should update the low order bits of the
++ * rseq_cs field, leaving the high order bits initialized to 0.
+ */
+- union {
+- __u64 ptr64;
+-#ifdef __LP64__
+- __u64 ptr;
+-#else
+- struct {
+-#if (defined(__BYTE_ORDER) && (__BYTE_ORDER == __BIG_ENDIAN)) || defined(__BIG_ENDIAN)
+- __u32 padding; /* Initialized to zero. */
+- __u32 ptr32;
+-#else /* LITTLE */
+- __u32 ptr32;
+- __u32 padding; /* Initialized to zero. */
+-#endif /* ENDIAN */
+- } ptr;
+-#endif
+- } rseq_cs;
++ __u64 rseq_cs;
+
+ /*
+ * Restartable sequences flags field.
+diff --git a/include/uapi/linux/serial_core.h b/include/uapi/linux/serial_core.h
+index c4042dcfdc0c3..8885e69178bd7 100644
+--- a/include/uapi/linux/serial_core.h
++++ b/include/uapi/linux/serial_core.h
+@@ -68,6 +68,9 @@
+ /* NVIDIA Tegra Combined UART */
+ #define PORT_TEGRA_TCU 41
+
++/* ASPEED AST2x00 virtual UART */
++#define PORT_ASPEED_VUART 42
++
+ /* Intel EG20 */
+ #define PORT_PCH_8LINE 44
+ #define PORT_PCH_2LINE 45
+diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h
+index 80d76b75bccd9..7aa2eb7662050 100644
+--- a/include/uapi/linux/virtio_ids.h
++++ b/include/uapi/linux/virtio_ids.h
+@@ -73,12 +73,12 @@
+ * Virtio Transitional IDs
+ */
+
+-#define VIRTIO_TRANS_ID_NET 1000 /* transitional virtio net */
+-#define VIRTIO_TRANS_ID_BLOCK 1001 /* transitional virtio block */
+-#define VIRTIO_TRANS_ID_BALLOON 1002 /* transitional virtio balloon */
+-#define VIRTIO_TRANS_ID_CONSOLE 1003 /* transitional virtio console */
+-#define VIRTIO_TRANS_ID_SCSI 1004 /* transitional virtio SCSI */
+-#define VIRTIO_TRANS_ID_RNG 1005 /* transitional virtio rng */
+-#define VIRTIO_TRANS_ID_9P 1009 /* transitional virtio 9p console */
++#define VIRTIO_TRANS_ID_NET 0x1000 /* transitional virtio net */
++#define VIRTIO_TRANS_ID_BLOCK 0x1001 /* transitional virtio block */
++#define VIRTIO_TRANS_ID_BALLOON 0x1002 /* transitional virtio balloon */
++#define VIRTIO_TRANS_ID_CONSOLE 0x1003 /* transitional virtio console */
++#define VIRTIO_TRANS_ID_SCSI 0x1004 /* transitional virtio SCSI */
++#define VIRTIO_TRANS_ID_RNG 0x1005 /* transitional virtio rng */
++#define VIRTIO_TRANS_ID_9P 0x1009 /* transitional virtio 9p console */
+
+ #endif /* _LINUX_VIRTIO_IDS_H */
+diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h
+index eda0426ec4c2b..65e13a099b1a0 100644
+--- a/include/uapi/linux/xfrm.h
++++ b/include/uapi/linux/xfrm.h
+@@ -313,6 +313,7 @@ enum xfrm_attr_type_t {
+ XFRMA_SET_MARK, /* __u32 */
+ XFRMA_SET_MARK_MASK, /* __u32 */
+ XFRMA_IF_ID, /* __u32 */
++ XFRMA_MTIMER_THRESH, /* __u32 in seconds for input SA */
+ __XFRMA_MAX
+
+ #define XFRMA_OUTPUT_MARK XFRMA_SET_MARK /* Compatibility */
+@@ -510,6 +511,12 @@ struct xfrm_user_offload {
+ int ifindex;
+ __u8 flags;
+ };
++/* This flag was exposed without any kernel code that supporting it.
++ * Unfortunately, strongswan has the code that uses sets this flag,
++ * which makes impossible to reuse this bit.
++ *
++ * So leave it here to make sure that it won't be reused by mistake.
++ */
+ #define XFRM_OFFLOAD_IPV6 1
+ #define XFRM_OFFLOAD_INBOUND 2
+
+diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h
+index 5859ca0a1439b..93e40f91bd49a 100644
+--- a/include/uapi/sound/asound.h
++++ b/include/uapi/sound/asound.h
+@@ -56,8 +56,10 @@
+ * *
+ ****************************************************************************/
+
++#define AES_IEC958_STATUS_SIZE 24
++
+ struct snd_aes_iec958 {
+- unsigned char status[24]; /* AES/IEC958 channel status bits */
++ unsigned char status[AES_IEC958_STATUS_SIZE]; /* AES/IEC958 channel status bits */
+ unsigned char subcode[147]; /* AES/IEC958 subcode bits */
+ unsigned char pad; /* nothing */
+ unsigned char dig_subframe[4]; /* AES/IEC958 subframe bits */
+diff --git a/include/xen/events.h b/include/xen/events.h
+index c204262d9fc24..344081e71584b 100644
+--- a/include/xen/events.h
++++ b/include/xen/events.h
+@@ -17,6 +17,7 @@ struct xenbus_device;
+ unsigned xen_evtchn_nr_channels(void);
+
+ int bind_evtchn_to_irq(evtchn_port_t evtchn);
++int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn);
+ int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
+ irq_handler_t handler,
+ unsigned long irqflags, const char *devname,
+diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h
+index cb854df031ce0..c9fea9389ebec 100644
+--- a/include/xen/grant_table.h
++++ b/include/xen/grant_table.h
+@@ -104,17 +104,32 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly);
+ * access has been ended, free the given page too. Access will be ended
+ * immediately iff the grant entry is not in use, otherwise it will happen
+ * some time later. page may be 0, in which case no freeing will occur.
++ * Note that the granted page might still be accessed (read or write) by the
++ * other side after gnttab_end_foreign_access() returns, so even if page was
++ * specified as 0 it is not allowed to just reuse the page for other
++ * purposes immediately. gnttab_end_foreign_access() will take an additional
++ * reference to the granted page in this case, which is dropped only after
++ * the grant is no longer in use.
++ * This requires that multi page allocations for areas subject to
++ * gnttab_end_foreign_access() are done via alloc_pages_exact() (and freeing
++ * via free_pages_exact()) in order to avoid high order pages.
+ */
+ void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
+ unsigned long page);
+
++/*
++ * End access through the given grant reference, iff the grant entry is
++ * no longer in use. In case of success ending foreign access, the
++ * grant reference is deallocated.
++ * Return 1 if the grant entry was freed, 0 if it is still in use.
++ */
++int gnttab_try_end_foreign_access(grant_ref_t ref);
++
+ int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn);
+
+ unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref);
+ unsigned long gnttab_end_foreign_transfer(grant_ref_t ref);
+
+-int gnttab_query_foreign_access(grant_ref_t ref);
+-
+ /*
+ * operations on reserved batches of grant references
+ */
+diff --git a/init/main.c b/init/main.c
+index 3c4054a955458..cf79b5a766cb1 100644
+--- a/init/main.c
++++ b/init/main.c
+@@ -100,6 +100,7 @@
+ #include <linux/kcsan.h>
+ #include <linux/init_syscalls.h>
+ #include <linux/stackdepot.h>
++#include <net/net_namespace.h>
+
+ #include <asm/io.h>
+ #include <asm/bugs.h>
+@@ -924,7 +925,9 @@ static void __init print_unknown_bootoptions(void)
+ for (p = &envp_init[2]; *p; p++)
+ end += sprintf(end, " %s", *p);
+
+- pr_notice("Unknown command line parameters:%s\n", unknown_options);
++ /* Start at unknown_options[1] to skip the initial space */
++ pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
++ &unknown_options[1]);
+ memblock_free_ptr(unknown_options, len);
+ }
+
+@@ -1038,21 +1041,18 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
+ softirq_init();
+ timekeeping_init();
+ kfence_init();
++ time_init();
+
+ /*
+ * For best initial stack canary entropy, prepare it after:
+ * - setup_arch() for any UEFI RNG entropy and boot cmdline access
+- * - timekeeping_init() for ktime entropy used in rand_initialize()
+- * - rand_initialize() to get any arch-specific entropy like RDRAND
+- * - add_latent_entropy() to get any latent entropy
+- * - adding command line entropy
++ * - timekeeping_init() for ktime entropy used in random_init()
++ * - time_init() for making random_get_entropy() work on some platforms
++ * - random_init() to initialize the RNG from from early entropy sources
+ */
+- rand_initialize();
+- add_latent_entropy();
+- add_device_randomness(command_line, strlen(command_line));
++ random_init(command_line);
+ boot_init_stack_canary();
+
+- time_init();
+ perf_event_init();
+ profile_init();
+ call_function_init();
+@@ -1120,6 +1120,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
+ key_init();
+ security_init();
+ dbg_late_init();
++ net_ns_init();
+ vfs_caches_init();
+ pagecache_init();
+ signals_init();
+@@ -1194,7 +1195,7 @@ static int __init initcall_blacklist(char *str)
+ }
+ } while (str_entry);
+
+- return 0;
++ return 1;
+ }
+
+ static bool __init_or_module initcall_blacklisted(initcall_t fn)
+@@ -1456,7 +1457,9 @@ static noinline void __init kernel_init_freeable(void);
+ bool rodata_enabled __ro_after_init = true;
+ static int __init set_debug_rodata(char *str)
+ {
+- return strtobool(str, &rodata_enabled);
++ if (strtobool(str, &rodata_enabled))
++ pr_warn("Invalid option string for rodata: '%s'\n", str);
++ return 1;
+ }
+ __setup("rodata=", set_debug_rodata);
+ #endif
+diff --git a/ipc/sem.c b/ipc/sem.c
+index 6693daf4fe112..0dbdb98fdf2d9 100644
+--- a/ipc/sem.c
++++ b/ipc/sem.c
+@@ -1964,6 +1964,7 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
+ */
+ un = lookup_undo(ulp, semid);
+ if (un) {
++ spin_unlock(&ulp->lock);
+ kvfree(new);
+ goto success;
+ }
+@@ -1976,9 +1977,8 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
+ ipc_assert_locked_object(&sma->sem_perm);
+ list_add(&new->list_id, &sma->list_id);
+ un = new;
+-
+-success:
+ spin_unlock(&ulp->lock);
++success:
+ sem_unlock(sma, -1);
+ out:
+ return un;
+diff --git a/ipc/shm.c b/ipc/shm.c
+index ab749be6d8b71..048eb183b24b9 100644
+--- a/ipc/shm.c
++++ b/ipc/shm.c
+@@ -62,9 +62,18 @@ struct shmid_kernel /* private to the kernel */
+ struct pid *shm_lprid;
+ struct ucounts *mlock_ucounts;
+
+- /* The task created the shm object. NULL if the task is dead. */
++ /*
++ * The task created the shm object, for
++ * task_lock(shp->shm_creator)
++ */
+ struct task_struct *shm_creator;
+- struct list_head shm_clist; /* list by creator */
++
++ /*
++ * List by creator. task_lock(->shm_creator) required for read/write.
++ * If list_empty(), then the creator is dead already.
++ */
++ struct list_head shm_clist;
++ struct ipc_namespace *ns;
+ } __randomize_layout;
+
+ /* shm_mode upper byte flags */
+@@ -115,6 +124,7 @@ static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
+ struct shmid_kernel *shp;
+
+ shp = container_of(ipcp, struct shmid_kernel, shm_perm);
++ WARN_ON(ns != shp->ns);
+
+ if (shp->shm_nattch) {
+ shp->shm_perm.mode |= SHM_DEST;
+@@ -225,10 +235,43 @@ static void shm_rcu_free(struct rcu_head *head)
+ kfree(shp);
+ }
+
+-static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
++/*
++ * It has to be called with shp locked.
++ * It must be called before ipc_rmid()
++ */
++static inline void shm_clist_rm(struct shmid_kernel *shp)
+ {
+- list_del(&s->shm_clist);
+- ipc_rmid(&shm_ids(ns), &s->shm_perm);
++ struct task_struct *creator;
++
++ /* ensure that shm_creator does not disappear */
++ rcu_read_lock();
++
++ /*
++ * A concurrent exit_shm may do a list_del_init() as well.
++ * Just do nothing if exit_shm already did the work
++ */
++ if (!list_empty(&shp->shm_clist)) {
++ /*
++ * shp->shm_creator is guaranteed to be valid *only*
++ * if shp->shm_clist is not empty.
++ */
++ creator = shp->shm_creator;
++
++ task_lock(creator);
++ /*
++ * list_del_init() is a nop if the entry was already removed
++ * from the list.
++ */
++ list_del_init(&shp->shm_clist);
++ task_unlock(creator);
++ }
++ rcu_read_unlock();
++}
++
++static inline void shm_rmid(struct shmid_kernel *s)
++{
++ shm_clist_rm(s);
++ ipc_rmid(&shm_ids(s->ns), &s->shm_perm);
+ }
+
+
+@@ -283,7 +326,7 @@ static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
+ shm_file = shp->shm_file;
+ shp->shm_file = NULL;
+ ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
+- shm_rmid(ns, shp);
++ shm_rmid(shp);
+ shm_unlock(shp);
+ if (!is_file_hugepages(shm_file))
+ shmem_lock(shm_file, 0, shp->mlock_ucounts);
+@@ -306,10 +349,10 @@ static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
+ *
+ * 2) sysctl kernel.shm_rmid_forced is set to 1.
+ */
+-static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
++static bool shm_may_destroy(struct shmid_kernel *shp)
+ {
+ return (shp->shm_nattch == 0) &&
+- (ns->shm_rmid_forced ||
++ (shp->ns->shm_rmid_forced ||
+ (shp->shm_perm.mode & SHM_DEST));
+ }
+
+@@ -340,7 +383,7 @@ static void shm_close(struct vm_area_struct *vma)
+ ipc_update_pid(&shp->shm_lprid, task_tgid(current));
+ shp->shm_dtim = ktime_get_real_seconds();
+ shp->shm_nattch--;
+- if (shm_may_destroy(ns, shp))
++ if (shm_may_destroy(shp))
+ shm_destroy(ns, shp);
+ else
+ shm_unlock(shp);
+@@ -361,10 +404,10 @@ static int shm_try_destroy_orphaned(int id, void *p, void *data)
+ *
+ * As shp->* are changed under rwsem, it's safe to skip shp locking.
+ */
+- if (shp->shm_creator != NULL)
++ if (!list_empty(&shp->shm_clist))
+ return 0;
+
+- if (shm_may_destroy(ns, shp)) {
++ if (shm_may_destroy(shp)) {
+ shm_lock_by_ptr(shp);
+ shm_destroy(ns, shp);
+ }
+@@ -382,48 +425,97 @@ void shm_destroy_orphaned(struct ipc_namespace *ns)
+ /* Locking assumes this will only be called with task == current */
+ void exit_shm(struct task_struct *task)
+ {
+- struct ipc_namespace *ns = task->nsproxy->ipc_ns;
+- struct shmid_kernel *shp, *n;
++ for (;;) {
++ struct shmid_kernel *shp;
++ struct ipc_namespace *ns;
+
+- if (list_empty(&task->sysvshm.shm_clist))
+- return;
++ task_lock(task);
++
++ if (list_empty(&task->sysvshm.shm_clist)) {
++ task_unlock(task);
++ break;
++ }
++
++ shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,
++ shm_clist);
+
+- /*
+- * If kernel.shm_rmid_forced is not set then only keep track of
+- * which shmids are orphaned, so that a later set of the sysctl
+- * can clean them up.
+- */
+- if (!ns->shm_rmid_forced) {
+- down_read(&shm_ids(ns).rwsem);
+- list_for_each_entry(shp, &task->sysvshm.shm_clist, shm_clist)
+- shp->shm_creator = NULL;
+ /*
+- * Only under read lock but we are only called on current
+- * so no entry on the list will be shared.
++ * 1) Get pointer to the ipc namespace. It is worth to say
++ * that this pointer is guaranteed to be valid because
++ * shp lifetime is always shorter than namespace lifetime
++ * in which shp lives.
++ * We taken task_lock it means that shp won't be freed.
+ */
+- list_del(&task->sysvshm.shm_clist);
+- up_read(&shm_ids(ns).rwsem);
+- return;
+- }
++ ns = shp->ns;
+
+- /*
+- * Destroy all already created segments, that were not yet mapped,
+- * and mark any mapped as orphan to cover the sysctl toggling.
+- * Destroy is skipped if shm_may_destroy() returns false.
+- */
+- down_write(&shm_ids(ns).rwsem);
+- list_for_each_entry_safe(shp, n, &task->sysvshm.shm_clist, shm_clist) {
+- shp->shm_creator = NULL;
++ /*
++ * 2) If kernel.shm_rmid_forced is not set then only keep track of
++ * which shmids are orphaned, so that a later set of the sysctl
++ * can clean them up.
++ */
++ if (!ns->shm_rmid_forced)
++ goto unlink_continue;
+
+- if (shm_may_destroy(ns, shp)) {
+- shm_lock_by_ptr(shp);
+- shm_destroy(ns, shp);
++ /*
++ * 3) get a reference to the namespace.
++ * The refcount could be already 0. If it is 0, then
++ * the shm objects will be free by free_ipc_work().
++ */
++ ns = get_ipc_ns_not_zero(ns);
++ if (!ns) {
++unlink_continue:
++ list_del_init(&shp->shm_clist);
++ task_unlock(task);
++ continue;
+ }
+- }
+
+- /* Remove the list head from any segments still attached. */
+- list_del(&task->sysvshm.shm_clist);
+- up_write(&shm_ids(ns).rwsem);
++ /*
++ * 4) get a reference to shp.
++ * This cannot fail: shm_clist_rm() is called before
++ * ipc_rmid(), thus the refcount cannot be 0.
++ */
++ WARN_ON(!ipc_rcu_getref(&shp->shm_perm));
++
++ /*
++ * 5) unlink the shm segment from the list of segments
++ * created by current.
++ * This must be done last. After unlinking,
++ * only the refcounts obtained above prevent IPC_RMID
++ * from destroying the segment or the namespace.
++ */
++ list_del_init(&shp->shm_clist);
++
++ task_unlock(task);
++
++ /*
++ * 6) we have all references
++ * Thus lock & if needed destroy shp.
++ */
++ down_write(&shm_ids(ns).rwsem);
++ shm_lock_by_ptr(shp);
++ /*
++ * rcu_read_lock was implicitly taken in shm_lock_by_ptr, it's
++ * safe to call ipc_rcu_putref here
++ */
++ ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
++
++ if (ipc_valid_object(&shp->shm_perm)) {
++ if (shm_may_destroy(shp))
++ shm_destroy(ns, shp);
++ else
++ shm_unlock(shp);
++ } else {
++ /*
++ * Someone else deleted the shp from namespace
++ * idr/kht while we have waited.
++ * Just unlock and continue.
++ */
++ shm_unlock(shp);
++ }
++
++ up_write(&shm_ids(ns).rwsem);
++ put_ipc_ns(ns); /* paired with get_ipc_ns_not_zero */
++ }
+ }
+
+ static vm_fault_t shm_fault(struct vm_fault *vmf)
+@@ -680,7 +772,11 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
+ if (error < 0)
+ goto no_id;
+
++ shp->ns = ns;
++
++ task_lock(current);
+ list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
++ task_unlock(current);
+
+ /*
+ * shmid gets reported as "inode#" in /proc/pid/maps.
+@@ -1573,7 +1669,8 @@ out_nattch:
+ down_write(&shm_ids(ns).rwsem);
+ shp = shm_lock(ns, shmid);
+ shp->shm_nattch--;
+- if (shm_may_destroy(ns, shp))
++
++ if (shm_may_destroy(shp))
+ shm_destroy(ns, shp);
+ else
+ shm_unlock(shp);
+diff --git a/ipc/util.c b/ipc/util.c
+index d48d8cfa1f3fa..fa2d86ef3fb80 100644
+--- a/ipc/util.c
++++ b/ipc/util.c
+@@ -447,8 +447,8 @@ static int ipcget_public(struct ipc_namespace *ns, struct ipc_ids *ids,
+ static void ipc_kht_remove(struct ipc_ids *ids, struct kern_ipc_perm *ipcp)
+ {
+ if (ipcp->key != IPC_PRIVATE)
+- rhashtable_remove_fast(&ids->key_ht, &ipcp->khtnode,
+- ipc_kht_params);
++ WARN_ON_ONCE(rhashtable_remove_fast(&ids->key_ht, &ipcp->khtnode,
++ ipc_kht_params));
+ }
+
+ /**
+@@ -498,7 +498,7 @@ void ipc_rmid(struct ipc_ids *ids, struct kern_ipc_perm *ipcp)
+ {
+ int idx = ipcid_to_idx(ipcp->id);
+
+- idr_remove(&ids->ipcs_idr, idx);
++ WARN_ON_ONCE(idr_remove(&ids->ipcs_idr, idx) != ipcp);
+ ipc_kht_remove(ids, ipcp);
+ ids->in_use--;
+ ipcp->deleted = true;
+diff --git a/kernel/Makefile b/kernel/Makefile
+index 4df609be42d07..0e119c52a2cd6 100644
+--- a/kernel/Makefile
++++ b/kernel/Makefile
+@@ -113,7 +113,8 @@ obj-$(CONFIG_CPU_PM) += cpu_pm.o
+ obj-$(CONFIG_BPF) += bpf/
+ obj-$(CONFIG_KCSAN) += kcsan/
+ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
+-obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o
++obj-$(CONFIG_HAVE_STATIC_CALL) += static_call.o
++obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call_inline.o
+ obj-$(CONFIG_CFI_CLANG) += cfi.o
+
+ obj-$(CONFIG_PERF_EVENTS) += events/
+diff --git a/kernel/async.c b/kernel/async.c
+index b8d7a663497f9..b2c4ba5686ee4 100644
+--- a/kernel/async.c
++++ b/kernel/async.c
+@@ -205,9 +205,6 @@ async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
+ atomic_inc(&entry_count);
+ spin_unlock_irqrestore(&async_lock, flags);
+
+- /* mark that this task has queued an async job, used by module init */
+- current->flags |= PF_USED_ASYNC;
+-
+ /* schedule for execution */
+ queue_work_node(node, system_unbound_wq, &entry->work);
+
+diff --git a/kernel/audit.c b/kernel/audit.c
+index 121d37e700a62..94ded5de91317 100644
+--- a/kernel/audit.c
++++ b/kernel/audit.c
+@@ -541,20 +541,22 @@ static void kauditd_printk_skb(struct sk_buff *skb)
+ /**
+ * kauditd_rehold_skb - Handle a audit record send failure in the hold queue
+ * @skb: audit record
++ * @error: error code (unused)
+ *
+ * Description:
+ * This should only be used by the kauditd_thread when it fails to flush the
+ * hold queue.
+ */
+-static void kauditd_rehold_skb(struct sk_buff *skb)
++static void kauditd_rehold_skb(struct sk_buff *skb, __always_unused int error)
+ {
+- /* put the record back in the queue at the same place */
+- skb_queue_head(&audit_hold_queue, skb);
++ /* put the record back in the queue */
++ skb_queue_tail(&audit_hold_queue, skb);
+ }
+
+ /**
+ * kauditd_hold_skb - Queue an audit record, waiting for auditd
+ * @skb: audit record
++ * @error: error code
+ *
+ * Description:
+ * Queue the audit record, waiting for an instance of auditd. When this
+@@ -564,19 +566,31 @@ static void kauditd_rehold_skb(struct sk_buff *skb)
+ * and queue it, if we have room. If we want to hold on to the record, but we
+ * don't have room, record a record lost message.
+ */
+-static void kauditd_hold_skb(struct sk_buff *skb)
++static void kauditd_hold_skb(struct sk_buff *skb, int error)
+ {
+ /* at this point it is uncertain if we will ever send this to auditd so
+ * try to send the message via printk before we go any further */
+ kauditd_printk_skb(skb);
+
+ /* can we just silently drop the message? */
+- if (!audit_default) {
+- kfree_skb(skb);
+- return;
++ if (!audit_default)
++ goto drop;
++
++ /* the hold queue is only for when the daemon goes away completely,
++ * not -EAGAIN failures; if we are in a -EAGAIN state requeue the
++ * record on the retry queue unless it's full, in which case drop it
++ */
++ if (error == -EAGAIN) {
++ if (!audit_backlog_limit ||
++ skb_queue_len(&audit_retry_queue) < audit_backlog_limit) {
++ skb_queue_tail(&audit_retry_queue, skb);
++ return;
++ }
++ audit_log_lost("kauditd retry queue overflow");
++ goto drop;
+ }
+
+- /* if we have room, queue the message */
++ /* if we have room in the hold queue, queue the message */
+ if (!audit_backlog_limit ||
+ skb_queue_len(&audit_hold_queue) < audit_backlog_limit) {
+ skb_queue_tail(&audit_hold_queue, skb);
+@@ -585,24 +599,32 @@ static void kauditd_hold_skb(struct sk_buff *skb)
+
+ /* we have no other options - drop the message */
+ audit_log_lost("kauditd hold queue overflow");
++drop:
+ kfree_skb(skb);
+ }
+
+ /**
+ * kauditd_retry_skb - Queue an audit record, attempt to send again to auditd
+ * @skb: audit record
++ * @error: error code (unused)
+ *
+ * Description:
+ * Not as serious as kauditd_hold_skb() as we still have a connected auditd,
+ * but for some reason we are having problems sending it audit records so
+ * queue the given record and attempt to resend.
+ */
+-static void kauditd_retry_skb(struct sk_buff *skb)
++static void kauditd_retry_skb(struct sk_buff *skb, __always_unused int error)
+ {
+- /* NOTE: because records should only live in the retry queue for a
+- * short period of time, before either being sent or moved to the hold
+- * queue, we don't currently enforce a limit on this queue */
+- skb_queue_tail(&audit_retry_queue, skb);
++ if (!audit_backlog_limit ||
++ skb_queue_len(&audit_retry_queue) < audit_backlog_limit) {
++ skb_queue_tail(&audit_retry_queue, skb);
++ return;
++ }
++
++ /* we have to drop the record, send it via printk as a last effort */
++ kauditd_printk_skb(skb);
++ audit_log_lost("kauditd retry queue overflow");
++ kfree_skb(skb);
+ }
+
+ /**
+@@ -640,7 +662,7 @@ static void auditd_reset(const struct auditd_connection *ac)
+ /* flush the retry queue to the hold queue, but don't touch the main
+ * queue since we need to process that normally for multicast */
+ while ((skb = skb_dequeue(&audit_retry_queue)))
+- kauditd_hold_skb(skb);
++ kauditd_hold_skb(skb, -ECONNREFUSED);
+ }
+
+ /**
+@@ -714,16 +736,18 @@ static int kauditd_send_queue(struct sock *sk, u32 portid,
+ struct sk_buff_head *queue,
+ unsigned int retry_limit,
+ void (*skb_hook)(struct sk_buff *skb),
+- void (*err_hook)(struct sk_buff *skb))
++ void (*err_hook)(struct sk_buff *skb, int error))
+ {
+ int rc = 0;
+- struct sk_buff *skb;
+- static unsigned int failed = 0;
++ struct sk_buff *skb = NULL;
++ struct sk_buff *skb_tail;
++ unsigned int failed = 0;
+
+ /* NOTE: kauditd_thread takes care of all our locking, we just use
+ * the netlink info passed to us (e.g. sk and portid) */
+
+- while ((skb = skb_dequeue(queue))) {
++ skb_tail = skb_peek_tail(queue);
++ while ((skb != skb_tail) && (skb = skb_dequeue(queue))) {
+ /* call the skb_hook for each skb we touch */
+ if (skb_hook)
+ (*skb_hook)(skb);
+@@ -731,36 +755,34 @@ static int kauditd_send_queue(struct sock *sk, u32 portid,
+ /* can we send to anyone via unicast? */
+ if (!sk) {
+ if (err_hook)
+- (*err_hook)(skb);
++ (*err_hook)(skb, -ECONNREFUSED);
+ continue;
+ }
+
++retry:
+ /* grab an extra skb reference in case of error */
+ skb_get(skb);
+ rc = netlink_unicast(sk, skb, portid, 0);
+ if (rc < 0) {
+- /* fatal failure for our queue flush attempt? */
++ /* send failed - try a few times unless fatal error */
+ if (++failed >= retry_limit ||
+ rc == -ECONNREFUSED || rc == -EPERM) {
+- /* yes - error processing for the queue */
+ sk = NULL;
+ if (err_hook)
+- (*err_hook)(skb);
+- if (!skb_hook)
+- goto out;
+- /* keep processing with the skb_hook */
++ (*err_hook)(skb, rc);
++ if (rc == -EAGAIN)
++ rc = 0;
++ /* continue to drain the queue */
+ continue;
+ } else
+- /* no - requeue to preserve ordering */
+- skb_queue_head(queue, skb);
++ goto retry;
+ } else {
+- /* it worked - drop the extra reference and continue */
++ /* skb sent - drop the extra reference and continue */
+ consume_skb(skb);
+ failed = 0;
+ }
+ }
+
+-out:
+ return (rc >= 0 ? 0 : rc);
+ }
+
+@@ -1542,6 +1564,20 @@ static void audit_receive(struct sk_buff *skb)
+ nlh = nlmsg_next(nlh, &len);
+ }
+ audit_ctl_unlock();
++
++ /* can't block with the ctrl lock, so penalize the sender now */
++ if (audit_backlog_limit &&
++ (skb_queue_len(&audit_queue) > audit_backlog_limit)) {
++ DECLARE_WAITQUEUE(wait, current);
++
++ /* wake kauditd to try and flush the queue */
++ wake_up_interruptible(&kauditd_wait);
++
++ add_wait_queue_exclusive(&audit_backlog_wait, &wait);
++ set_current_state(TASK_UNINTERRUPTIBLE);
++ schedule_timeout(audit_backlog_wait_time);
++ remove_wait_queue(&audit_backlog_wait, &wait);
++ }
+ }
+
+ /* Log information about who is connecting to the audit multicast socket */
+@@ -1609,7 +1645,8 @@ static int __net_init audit_net_init(struct net *net)
+ audit_panic("cannot initialize netlink socket in namespace");
+ return -ENOMEM;
+ }
+- aunet->sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
++ /* limit the timeout in case auditd is blocked/stopped */
++ aunet->sk->sk_sndtimeo = HZ / 10;
+
+ return 0;
+ }
+@@ -1825,7 +1862,9 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
+ * task_tgid_vnr() since auditd_pid is set in audit_receive_msg()
+ * using a PID anchored in the caller's namespace
+ * 2. generator holding the audit_cmd_mutex - we don't want to block
+- * while holding the mutex */
++ * while holding the mutex, although we do penalize the sender
++ * later in audit_receive() when it is safe to block
++ */
+ if (!(auditd_test_task(current) || audit_ctl_owner_current())) {
+ long stime = audit_backlog_wait_time;
+
+diff --git a/kernel/audit.h b/kernel/audit.h
+index d6a2c899a8dbf..b2ef4c0d3ec03 100644
+--- a/kernel/audit.h
++++ b/kernel/audit.h
+@@ -194,6 +194,10 @@ struct audit_context {
+ struct {
+ char *name;
+ } module;
++ struct {
++ struct audit_ntp_data ntp_data;
++ struct timespec64 tk_injoffset;
++ } time;
+ };
+ int fds[2];
+ struct audit_proctitle proctitle;
+diff --git a/kernel/auditsc.c b/kernel/auditsc.c
+index b1cb1dbf7417f..2f036bab3c28f 100644
+--- a/kernel/auditsc.c
++++ b/kernel/auditsc.c
+@@ -1219,6 +1219,53 @@ static void audit_log_fcaps(struct audit_buffer *ab, struct audit_names *name)
+ from_kuid(&init_user_ns, name->fcap.rootid));
+ }
+
++static void audit_log_time(struct audit_context *context, struct audit_buffer **ab)
++{
++ const struct audit_ntp_data *ntp = &context->time.ntp_data;
++ const struct timespec64 *tk = &context->time.tk_injoffset;
++ static const char * const ntp_name[] = {
++ "offset",
++ "freq",
++ "status",
++ "tai",
++ "tick",
++ "adjust",
++ };
++ int type;
++
++ if (context->type == AUDIT_TIME_ADJNTPVAL) {
++ for (type = 0; type < AUDIT_NTP_NVALS; type++) {
++ if (ntp->vals[type].newval != ntp->vals[type].oldval) {
++ if (!*ab) {
++ *ab = audit_log_start(context,
++ GFP_KERNEL,
++ AUDIT_TIME_ADJNTPVAL);
++ if (!*ab)
++ return;
++ }
++ audit_log_format(*ab, "op=%s old=%lli new=%lli",
++ ntp_name[type],
++ ntp->vals[type].oldval,
++ ntp->vals[type].newval);
++ audit_log_end(*ab);
++ *ab = NULL;
++ }
++ }
++ }
++ if (tk->tv_sec != 0 || tk->tv_nsec != 0) {
++ if (!*ab) {
++ *ab = audit_log_start(context, GFP_KERNEL,
++ AUDIT_TIME_INJOFFSET);
++ if (!*ab)
++ return;
++ }
++ audit_log_format(*ab, "sec=%lli nsec=%li",
++ (long long)tk->tv_sec, tk->tv_nsec);
++ audit_log_end(*ab);
++ *ab = NULL;
++ }
++}
++
+ static void show_special(struct audit_context *context, int *call_panic)
+ {
+ struct audit_buffer *ab;
+@@ -1327,6 +1374,11 @@ static void show_special(struct audit_context *context, int *call_panic)
+ audit_log_format(ab, "(null)");
+
+ break;
++ case AUDIT_TIME_ADJNTPVAL:
++ case AUDIT_TIME_INJOFFSET:
++ /* this call deviates from the rest, eating the buffer */
++ audit_log_time(context, &ab);
++ break;
+ }
+ audit_log_end(ab);
+ }
+@@ -2564,31 +2616,26 @@ void __audit_fanotify(unsigned int response)
+
+ void __audit_tk_injoffset(struct timespec64 offset)
+ {
+- audit_log(audit_context(), GFP_KERNEL, AUDIT_TIME_INJOFFSET,
+- "sec=%lli nsec=%li",
+- (long long)offset.tv_sec, offset.tv_nsec);
+-}
+-
+-static void audit_log_ntp_val(const struct audit_ntp_data *ad,
+- const char *op, enum audit_ntp_type type)
+-{
+- const struct audit_ntp_val *val = &ad->vals[type];
+-
+- if (val->newval == val->oldval)
+- return;
++ struct audit_context *context = audit_context();
+
+- audit_log(audit_context(), GFP_KERNEL, AUDIT_TIME_ADJNTPVAL,
+- "op=%s old=%lli new=%lli", op, val->oldval, val->newval);
++ /* only set type if not already set by NTP */
++ if (!context->type)
++ context->type = AUDIT_TIME_INJOFFSET;
++ memcpy(&context->time.tk_injoffset, &offset, sizeof(offset));
+ }
+
+ void __audit_ntp_log(const struct audit_ntp_data *ad)
+ {
+- audit_log_ntp_val(ad, "offset", AUDIT_NTP_OFFSET);
+- audit_log_ntp_val(ad, "freq", AUDIT_NTP_FREQ);
+- audit_log_ntp_val(ad, "status", AUDIT_NTP_STATUS);
+- audit_log_ntp_val(ad, "tai", AUDIT_NTP_TAI);
+- audit_log_ntp_val(ad, "tick", AUDIT_NTP_TICK);
+- audit_log_ntp_val(ad, "adjust", AUDIT_NTP_ADJUST);
++ struct audit_context *context = audit_context();
++ int type;
++
++ for (type = 0; type < AUDIT_NTP_NVALS; type++)
++ if (ad->vals[type].newval != ad->vals[type].oldval) {
++ /* unconditionally set type, overwriting TK */
++ context->type = AUDIT_TIME_ADJNTPVAL;
++ memcpy(&context->time.ntp_data, ad, sizeof(*ad));
++ break;
++ }
+ }
+
+ void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries,
+diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
+index dfe61df4f974d..40df35088cdbd 100644
+--- a/kernel/bpf/btf.c
++++ b/kernel/bpf/btf.c
+@@ -4332,8 +4332,7 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
+ log->len_total = log_size;
+
+ /* log attributes have to be sane */
+- if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
+- !log->level || !log->ubuf) {
++ if (!bpf_verifier_log_attr_valid(log)) {
+ err = -EINVAL;
+ goto errout;
+ }
+@@ -4801,10 +4800,12 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
+ /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
+ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
+ const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
++ u32 type, flag;
+
+- if (ctx_arg_info->offset == off &&
+- (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL ||
+- ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
++ type = base_type(ctx_arg_info->reg_type);
++ flag = type_flag(ctx_arg_info->reg_type);
++ if (ctx_arg_info->offset == off && type == PTR_TO_BUF &&
++ (flag & PTR_MAYBE_NULL)) {
+ info->reg_type = ctx_arg_info->reg_type;
+ return true;
+ }
+@@ -5509,9 +5510,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
+ if (reg->type == PTR_TO_BTF_ID) {
+ reg_btf = reg->btf;
+ reg_ref_id = reg->btf_id;
+- } else if (reg2btf_ids[reg->type]) {
++ } else if (reg2btf_ids[base_type(reg->type)]) {
+ reg_btf = btf_vmlinux;
+- reg_ref_id = *reg2btf_ids[reg->type];
++ reg_ref_id = *reg2btf_ids[base_type(reg->type)];
+ } else {
+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
+ func_name, i,
+@@ -5718,7 +5719,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
+ return -EINVAL;
+ }
+
+- reg->type = PTR_TO_MEM_OR_NULL;
++ reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
+ reg->id = ++env->id_gen;
+
+ continue;
+@@ -6007,12 +6008,17 @@ bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
+ return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
+ }
+
++enum {
++ BTF_MODULE_F_LIVE = (1 << 0),
++};
++
+ #ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+ struct btf_module {
+ struct list_head list;
+ struct module *module;
+ struct btf *btf;
+ struct bin_attribute *sysfs_attr;
++ int flags;
+ };
+
+ static LIST_HEAD(btf_modules);
+@@ -6038,7 +6044,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
+ int err = 0;
+
+ if (mod->btf_data_size == 0 ||
+- (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
++ (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE &&
++ op != MODULE_STATE_GOING))
+ goto out;
+
+ switch (op) {
+@@ -6095,6 +6102,17 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
+ btf_mod->sysfs_attr = attr;
+ }
+
++ break;
++ case MODULE_STATE_LIVE:
++ mutex_lock(&btf_module_mutex);
++ list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
++ if (btf_mod->module != module)
++ continue;
++
++ btf_mod->flags |= BTF_MODULE_F_LIVE;
++ break;
++ }
++ mutex_unlock(&btf_module_mutex);
+ break;
+ case MODULE_STATE_GOING:
+ mutex_lock(&btf_module_mutex);
+@@ -6141,7 +6159,12 @@ struct module *btf_try_get_module(const struct btf *btf)
+ if (btf_mod->btf != btf)
+ continue;
+
+- if (try_module_get(btf_mod->module))
++ /* We must only consider module whose __init routine has
++ * finished, hence we must check for BTF_MODULE_F_LIVE flag,
++ * which is set from the notifier callback for
++ * MODULE_STATE_LIVE.
++ */
++ if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module))
+ res = btf_mod->module;
+
+ break;
+@@ -6208,7 +6231,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
+ .func = bpf_btf_find_by_name_kind,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+- .arg1_type = ARG_PTR_TO_MEM,
++ .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg2_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_ANYTHING,
+diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
+index 03145d45e3d5b..fe053ffd89329 100644
+--- a/kernel/bpf/cgroup.c
++++ b/kernel/bpf/cgroup.c
+@@ -1753,7 +1753,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ };
+
+@@ -1773,6 +1773,8 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+ return &bpf_sysctl_get_new_value_proto;
+ case BPF_FUNC_sysctl_set_new_value:
+ return &bpf_sysctl_set_new_value_proto;
++ case BPF_FUNC_ktime_get_coarse_ns:
++ return &bpf_ktime_get_coarse_ns_proto;
+ default:
+ return cgroup_base_func_proto(func_id, prog);
+ }
+diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
+index 9aabf84afd4b2..a711ffe238932 100644
+--- a/kernel/bpf/helpers.c
++++ b/kernel/bpf/helpers.c
+@@ -530,7 +530,7 @@ const struct bpf_func_proto bpf_strtol_proto = {
+ .func = bpf_strtol,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+- .arg1_type = ARG_PTR_TO_MEM,
++ .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg2_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_PTR_TO_LONG,
+@@ -558,7 +558,7 @@ const struct bpf_func_proto bpf_strtoul_proto = {
+ .func = bpf_strtoul,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+- .arg1_type = ARG_PTR_TO_MEM,
++ .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg2_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_PTR_TO_LONG,
+@@ -630,7 +630,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -667,7 +667,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
+ const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
+ .func = bpf_per_cpu_ptr,
+ .gpl_only = false,
+- .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
++ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
+ .arg2_type = ARG_ANYTHING,
+ };
+@@ -680,7 +680,7 @@ BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
+ const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
+ .func = bpf_this_cpu_ptr,
+ .gpl_only = false,
+- .ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
++ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID | MEM_RDONLY,
+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
+ };
+
+@@ -1013,7 +1013,7 @@ const struct bpf_func_proto bpf_snprintf_proto = {
+ .arg1_type = ARG_PTR_TO_MEM_OR_NULL,
+ .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_PTR_TO_CONST_STR,
+- .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
++ .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -1367,8 +1367,6 @@ bpf_base_func_proto(enum bpf_func_id func_id)
+ return &bpf_ktime_get_ns_proto;
+ case BPF_FUNC_ktime_get_boot_ns:
+ return &bpf_ktime_get_boot_ns_proto;
+- case BPF_FUNC_ktime_get_coarse_ns:
+- return &bpf_ktime_get_coarse_ns_proto;
+ case BPF_FUNC_ringbuf_output:
+ return &bpf_ringbuf_output_proto;
+ case BPF_FUNC_ringbuf_reserve:
+diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
+index 80da1db47c686..5a8d9f7467bf4 100644
+--- a/kernel/bpf/inode.c
++++ b/kernel/bpf/inode.c
+@@ -648,12 +648,22 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param)
+ int opt;
+
+ opt = fs_parse(fc, bpf_fs_parameters, param, &result);
+- if (opt < 0)
++ if (opt < 0) {
+ /* We might like to report bad mount options here, but
+ * traditionally we've ignored all mount options, so we'd
+ * better continue to ignore non-existing options for bpf.
+ */
+- return opt == -ENOPARAM ? 0 : opt;
++ if (opt == -ENOPARAM) {
++ opt = vfs_parse_fs_param_source(fc, param);
++ if (opt != -ENOPARAM)
++ return opt;
++
++ return 0;
++ }
++
++ if (opt < 0)
++ return opt;
++ }
+
+ switch (opt) {
+ case OPT_MODE:
+diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
+index 6a9542af4212a..b0fa190b09790 100644
+--- a/kernel/bpf/map_iter.c
++++ b/kernel/bpf/map_iter.c
+@@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = {
+ .ctx_arg_info_size = 2,
+ .ctx_arg_info = {
+ { offsetof(struct bpf_iter__bpf_map_elem, key),
+- PTR_TO_RDONLY_BUF_OR_NULL },
++ PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY },
+ { offsetof(struct bpf_iter__bpf_map_elem, value),
+- PTR_TO_RDWR_BUF_OR_NULL },
++ PTR_TO_BUF | PTR_MAYBE_NULL },
+ },
+ };
+
+diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c
+index 9e0c10c6892ad..710ba9de12ce4 100644
+--- a/kernel/bpf/ringbuf.c
++++ b/kernel/bpf/ringbuf.c
+@@ -104,7 +104,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node)
+ }
+
+ rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
+- VM_ALLOC | VM_USERMAP, PAGE_KERNEL);
++ VM_MAP | VM_USERMAP, PAGE_KERNEL);
+ if (rb) {
+ kmemleak_not_leak(pages);
+ rb->pages = pages;
+@@ -444,7 +444,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = {
+ .func = bpf_ringbuf_output,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_CONST_MAP_PTR,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_ANYTHING,
+ };
+diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
+index 6e75bbee39f0b..fc0f77f91224b 100644
+--- a/kernel/bpf/stackmap.c
++++ b/kernel/bpf/stackmap.c
+@@ -219,7 +219,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
+ }
+
+ static struct perf_callchain_entry *
+-get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
++get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
+ {
+ #ifdef CONFIG_STACKTRACE
+ struct perf_callchain_entry *entry;
+@@ -230,9 +230,8 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
+ if (!entry)
+ return NULL;
+
+- entry->nr = init_nr +
+- stack_trace_save_tsk(task, (unsigned long *)(entry->ip + init_nr),
+- sysctl_perf_event_max_stack - init_nr, 0);
++ entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
++ max_depth, 0);
+
+ /* stack_trace_save_tsk() works on unsigned long array, while
+ * perf_callchain_entry uses u64 array. For 32-bit systems, it is
+@@ -244,7 +243,7 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
+ int i;
+
+ /* copy data from the end to avoid using extra buffer */
+- for (i = entry->nr - 1; i >= (int)init_nr; i--)
++ for (i = entry->nr - 1; i >= 0; i--)
+ to[i] = (u64)(from[i]);
+ }
+
+@@ -261,27 +260,19 @@ static long __bpf_get_stackid(struct bpf_map *map,
+ {
+ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+ struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
+- u32 max_depth = map->value_size / stack_map_data_size(map);
+- /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
+- u32 init_nr = sysctl_perf_event_max_stack - max_depth;
+ u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ u32 hash, id, trace_nr, trace_len;
+ bool user = flags & BPF_F_USER_STACK;
+ u64 *ips;
+ bool hash_matches;
+
+- /* get_perf_callchain() guarantees that trace->nr >= init_nr
+- * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
+- */
+- trace_nr = trace->nr - init_nr;
+-
+- if (trace_nr <= skip)
++ if (trace->nr <= skip)
+ /* skipping more than usable stack trace */
+ return -EFAULT;
+
+- trace_nr -= skip;
++ trace_nr = trace->nr - skip;
+ trace_len = trace_nr * sizeof(u64);
+- ips = trace->ip + skip + init_nr;
++ ips = trace->ip + skip;
+ hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
+ id = hash & (smap->n_buckets - 1);
+ bucket = READ_ONCE(smap->buckets[id]);
+@@ -338,8 +329,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
+ u64, flags)
+ {
+ u32 max_depth = map->value_size / stack_map_data_size(map);
+- /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
+- u32 init_nr = sysctl_perf_event_max_stack - max_depth;
++ u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ bool user = flags & BPF_F_USER_STACK;
+ struct perf_callchain_entry *trace;
+ bool kernel = !user;
+@@ -348,8 +338,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
+ BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
+ return -EINVAL;
+
+- trace = get_perf_callchain(regs, init_nr, kernel, user,
+- sysctl_perf_event_max_stack, false, false);
++ max_depth += skip;
++ if (max_depth > sysctl_perf_event_max_stack)
++ max_depth = sysctl_perf_event_max_stack;
++
++ trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
++ false, false);
+
+ if (unlikely(!trace))
+ /* couldn't fetch the stack trace */
+@@ -440,7 +434,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
+ struct perf_callchain_entry *trace_in,
+ void *buf, u32 size, u64 flags)
+ {
+- u32 init_nr, trace_nr, copy_len, elem_size, num_elem;
++ u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ bool user = flags & BPF_F_USER_STACK;
+@@ -465,30 +459,28 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
+ goto err_fault;
+
+ num_elem = size / elem_size;
+- if (sysctl_perf_event_max_stack < num_elem)
+- init_nr = 0;
+- else
+- init_nr = sysctl_perf_event_max_stack - num_elem;
++ max_depth = num_elem + skip;
++ if (sysctl_perf_event_max_stack < max_depth)
++ max_depth = sysctl_perf_event_max_stack;
+
+ if (trace_in)
+ trace = trace_in;
+ else if (kernel && task)
+- trace = get_callchain_entry_for_task(task, init_nr);
++ trace = get_callchain_entry_for_task(task, max_depth);
+ else
+- trace = get_perf_callchain(regs, init_nr, kernel, user,
+- sysctl_perf_event_max_stack,
++ trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
+ false, false);
+ if (unlikely(!trace))
+ goto err_fault;
+
+- trace_nr = trace->nr - init_nr;
+- if (trace_nr < skip)
++ if (trace->nr < skip)
+ goto err_fault;
+
+- trace_nr -= skip;
++ trace_nr = trace->nr - skip;
+ trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
+ copy_len = trace_nr * elem_size;
+- ips = trace->ip + skip + init_nr;
++
++ ips = trace->ip + skip;
+ if (user && user_build_id)
+ stack_map_get_build_id_offset(buf, ips, trace_nr, user);
+ else
+@@ -525,13 +517,14 @@ BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
+ u32, size, u64, flags)
+ {
+ struct pt_regs *regs;
+- long res;
++ long res = -EINVAL;
+
+ if (!try_get_task_stack(task))
+ return -EFAULT;
+
+ regs = task_pt_regs(task);
+- res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
++ if (regs)
++ res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
+ put_task_stack(task);
+
+ return res;
+diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
+index 1cad6979a0d0f..48e02a725563f 100644
+--- a/kernel/bpf/syscall.c
++++ b/kernel/bpf/syscall.c
+@@ -132,6 +132,21 @@ static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
+ return map;
+ }
+
++static void bpf_map_write_active_inc(struct bpf_map *map)
++{
++ atomic64_inc(&map->writecnt);
++}
++
++static void bpf_map_write_active_dec(struct bpf_map *map)
++{
++ atomic64_dec(&map->writecnt);
++}
++
++bool bpf_map_write_active(const struct bpf_map *map)
++{
++ return atomic64_read(&map->writecnt) != 0;
++}
++
+ static u32 bpf_map_value_size(const struct bpf_map *map)
+ {
+ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+@@ -596,11 +611,8 @@ static void bpf_map_mmap_open(struct vm_area_struct *vma)
+ {
+ struct bpf_map *map = vma->vm_file->private_data;
+
+- if (vma->vm_flags & VM_MAYWRITE) {
+- mutex_lock(&map->freeze_mutex);
+- map->writecnt++;
+- mutex_unlock(&map->freeze_mutex);
+- }
++ if (vma->vm_flags & VM_MAYWRITE)
++ bpf_map_write_active_inc(map);
+ }
+
+ /* called for all unmapped memory region (including initial) */
+@@ -608,11 +620,8 @@ static void bpf_map_mmap_close(struct vm_area_struct *vma)
+ {
+ struct bpf_map *map = vma->vm_file->private_data;
+
+- if (vma->vm_flags & VM_MAYWRITE) {
+- mutex_lock(&map->freeze_mutex);
+- map->writecnt--;
+- mutex_unlock(&map->freeze_mutex);
+- }
++ if (vma->vm_flags & VM_MAYWRITE)
++ bpf_map_write_active_dec(map);
+ }
+
+ static const struct vm_operations_struct bpf_map_default_vmops = {
+@@ -663,7 +672,7 @@ static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
+ goto out;
+
+ if (vma->vm_flags & VM_MAYWRITE)
+- map->writecnt++;
++ bpf_map_write_active_inc(map);
+ out:
+ mutex_unlock(&map->freeze_mutex);
+ return err;
+@@ -1122,6 +1131,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
+ map = __bpf_map_get(f);
+ if (IS_ERR(map))
+ return PTR_ERR(map);
++ bpf_map_write_active_inc(map);
+ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
+ err = -EPERM;
+ goto err_put;
+@@ -1157,6 +1167,7 @@ free_value:
+ free_key:
+ kvfree(key);
+ err_put:
++ bpf_map_write_active_dec(map);
+ fdput(f);
+ return err;
+ }
+@@ -1179,6 +1190,7 @@ static int map_delete_elem(union bpf_attr *attr)
+ map = __bpf_map_get(f);
+ if (IS_ERR(map))
+ return PTR_ERR(map);
++ bpf_map_write_active_inc(map);
+ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
+ err = -EPERM;
+ goto err_put;
+@@ -1209,6 +1221,7 @@ static int map_delete_elem(union bpf_attr *attr)
+ out:
+ kvfree(key);
+ err_put:
++ bpf_map_write_active_dec(map);
+ fdput(f);
+ return err;
+ }
+@@ -1324,6 +1337,7 @@ int generic_map_delete_batch(struct bpf_map *map,
+ maybe_wait_bpf_programs(map);
+ if (err)
+ break;
++ cond_resched();
+ }
+ if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
+ err = -EFAULT;
+@@ -1381,6 +1395,7 @@ int generic_map_update_batch(struct bpf_map *map,
+
+ if (err)
+ break;
++ cond_resched();
+ }
+
+ if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
+@@ -1478,6 +1493,7 @@ int generic_map_lookup_batch(struct bpf_map *map,
+ swap(prev_key, key);
+ retry = MAP_LOOKUP_RETRIES;
+ cp++;
++ cond_resched();
+ }
+
+ if (err == -EFAULT)
+@@ -1516,6 +1532,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr)
+ map = __bpf_map_get(f);
+ if (IS_ERR(map))
+ return PTR_ERR(map);
++ bpf_map_write_active_inc(map);
+ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
+ !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
+ err = -EPERM;
+@@ -1580,6 +1597,7 @@ free_value:
+ free_key:
+ kvfree(key);
+ err_put:
++ bpf_map_write_active_dec(map);
+ fdput(f);
+ return err;
+ }
+@@ -1607,8 +1625,7 @@ static int map_freeze(const union bpf_attr *attr)
+ }
+
+ mutex_lock(&map->freeze_mutex);
+-
+- if (map->writecnt) {
++ if (bpf_map_write_active(map)) {
+ err = -EBUSY;
+ goto err_put;
+ }
+@@ -1807,8 +1824,14 @@ static int bpf_prog_release(struct inode *inode, struct file *filp)
+ return 0;
+ }
+
++struct bpf_prog_kstats {
++ u64 nsecs;
++ u64 cnt;
++ u64 misses;
++};
++
+ static void bpf_prog_get_stats(const struct bpf_prog *prog,
+- struct bpf_prog_stats *stats)
++ struct bpf_prog_kstats *stats)
+ {
+ u64 nsecs = 0, cnt = 0, misses = 0;
+ int cpu;
+@@ -1821,9 +1844,9 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog,
+ st = per_cpu_ptr(prog->stats, cpu);
+ do {
+ start = u64_stats_fetch_begin_irq(&st->syncp);
+- tnsecs = st->nsecs;
+- tcnt = st->cnt;
+- tmisses = st->misses;
++ tnsecs = u64_stats_read(&st->nsecs);
++ tcnt = u64_stats_read(&st->cnt);
++ tmisses = u64_stats_read(&st->misses);
+ } while (u64_stats_fetch_retry_irq(&st->syncp, start));
+ nsecs += tnsecs;
+ cnt += tcnt;
+@@ -1839,7 +1862,7 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
+ {
+ const struct bpf_prog *prog = filp->private_data;
+ char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
+- struct bpf_prog_stats stats;
++ struct bpf_prog_kstats stats;
+
+ bpf_prog_get_stats(prog, &stats);
+ bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
+@@ -3578,7 +3601,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,
+ struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
+ struct bpf_prog_info info;
+ u32 info_len = attr->info.info_len;
+- struct bpf_prog_stats stats;
++ struct bpf_prog_kstats stats;
+ char __user *uinsns;
+ u32 ulen;
+ int err;
+@@ -4143,6 +4166,9 @@ static int bpf_map_do_batch(const union bpf_attr *attr,
+ union bpf_attr __user *uattr,
+ int cmd)
+ {
++ bool has_read = cmd == BPF_MAP_LOOKUP_BATCH ||
++ cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
++ bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
+ struct bpf_map *map;
+ int err, ufd;
+ struct fd f;
+@@ -4155,16 +4181,13 @@ static int bpf_map_do_batch(const union bpf_attr *attr,
+ map = __bpf_map_get(f);
+ if (IS_ERR(map))
+ return PTR_ERR(map);
+-
+- if ((cmd == BPF_MAP_LOOKUP_BATCH ||
+- cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
+- !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
++ if (has_write)
++ bpf_map_write_active_inc(map);
++ if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
+ err = -EPERM;
+ goto err_put;
+ }
+-
+- if (cmd != BPF_MAP_LOOKUP_BATCH &&
+- !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
++ if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
+ err = -EPERM;
+ goto err_put;
+ }
+@@ -4177,8 +4200,9 @@ static int bpf_map_do_batch(const union bpf_attr *attr,
+ BPF_DO_BATCH(map->ops->map_update_batch);
+ else
+ BPF_DO_BATCH(map->ops->map_delete_batch);
+-
+ err_put:
++ if (has_write)
++ bpf_map_write_active_dec(map);
+ fdput(f);
+ return err;
+ }
+@@ -4729,7 +4753,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_ANYTHING,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ };
+
+diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
+index fe1e857324e66..2660fbced9ad4 100644
+--- a/kernel/bpf/trampoline.c
++++ b/kernel/bpf/trampoline.c
+@@ -541,11 +541,12 @@ static u64 notrace bpf_prog_start_time(void)
+ static void notrace inc_misses_counter(struct bpf_prog *prog)
+ {
+ struct bpf_prog_stats *stats;
++ unsigned int flags;
+
+ stats = this_cpu_ptr(prog->stats);
+- u64_stats_update_begin(&stats->syncp);
+- stats->misses++;
+- u64_stats_update_end(&stats->syncp);
++ flags = u64_stats_update_begin_irqsave(&stats->syncp);
++ u64_stats_inc(&stats->misses);
++ u64_stats_update_end_irqrestore(&stats->syncp, flags);
+ }
+
+ /* The logic is similar to bpf_prog_run(), but with an explicit
+@@ -585,11 +586,13 @@ static void notrace update_prog_stats(struct bpf_prog *prog,
+ * Hence check that 'start' is valid.
+ */
+ start > NO_START_TIME) {
++ unsigned long flags;
++
+ stats = this_cpu_ptr(prog->stats);
+- u64_stats_update_begin(&stats->syncp);
+- stats->cnt++;
+- stats->nsecs += sched_clock() - start;
+- u64_stats_update_end(&stats->syncp);
++ flags = u64_stats_update_begin_irqsave(&stats->syncp);
++ u64_stats_inc(&stats->cnt);
++ u64_stats_add(&stats->nsecs, sched_clock() - start);
++ u64_stats_update_end_irqrestore(&stats->syncp, flags);
+ }
+ }
+
+diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
+index e76b559179054..d2b119b4fbe74 100644
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -445,18 +445,6 @@ static bool reg_type_not_null(enum bpf_reg_type type)
+ type == PTR_TO_SOCK_COMMON;
+ }
+
+-static bool reg_type_may_be_null(enum bpf_reg_type type)
+-{
+- return type == PTR_TO_MAP_VALUE_OR_NULL ||
+- type == PTR_TO_SOCKET_OR_NULL ||
+- type == PTR_TO_SOCK_COMMON_OR_NULL ||
+- type == PTR_TO_TCP_SOCK_OR_NULL ||
+- type == PTR_TO_BTF_ID_OR_NULL ||
+- type == PTR_TO_MEM_OR_NULL ||
+- type == PTR_TO_RDONLY_BUF_OR_NULL ||
+- type == PTR_TO_RDWR_BUF_OR_NULL;
+-}
+-
+ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
+ {
+ return reg->type == PTR_TO_MAP_VALUE &&
+@@ -465,12 +453,14 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
+
+ static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
+ {
+- return type == PTR_TO_SOCKET ||
+- type == PTR_TO_SOCKET_OR_NULL ||
+- type == PTR_TO_TCP_SOCK ||
+- type == PTR_TO_TCP_SOCK_OR_NULL ||
+- type == PTR_TO_MEM ||
+- type == PTR_TO_MEM_OR_NULL;
++ return base_type(type) == PTR_TO_SOCKET ||
++ base_type(type) == PTR_TO_TCP_SOCK ||
++ base_type(type) == PTR_TO_MEM;
++}
++
++static bool type_is_rdonly_mem(u32 type)
++{
++ return type & MEM_RDONLY;
+ }
+
+ static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
+@@ -478,14 +468,9 @@ static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
+ return type == ARG_PTR_TO_SOCK_COMMON;
+ }
+
+-static bool arg_type_may_be_null(enum bpf_arg_type type)
++static bool type_may_be_null(u32 type)
+ {
+- return type == ARG_PTR_TO_MAP_VALUE_OR_NULL ||
+- type == ARG_PTR_TO_MEM_OR_NULL ||
+- type == ARG_PTR_TO_CTX_OR_NULL ||
+- type == ARG_PTR_TO_SOCKET_OR_NULL ||
+- type == ARG_PTR_TO_ALLOC_MEM_OR_NULL ||
+- type == ARG_PTR_TO_STACK_OR_NULL;
++ return type & PTR_MAYBE_NULL;
+ }
+
+ /* Determine whether the function releases some resources allocated by another
+@@ -545,39 +530,54 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn)
+ insn->imm == BPF_CMPXCHG;
+ }
+
+-/* string representation of 'enum bpf_reg_type' */
+-static const char * const reg_type_str[] = {
+- [NOT_INIT] = "?",
+- [SCALAR_VALUE] = "inv",
+- [PTR_TO_CTX] = "ctx",
+- [CONST_PTR_TO_MAP] = "map_ptr",
+- [PTR_TO_MAP_VALUE] = "map_value",
+- [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
+- [PTR_TO_STACK] = "fp",
+- [PTR_TO_PACKET] = "pkt",
+- [PTR_TO_PACKET_META] = "pkt_meta",
+- [PTR_TO_PACKET_END] = "pkt_end",
+- [PTR_TO_FLOW_KEYS] = "flow_keys",
+- [PTR_TO_SOCKET] = "sock",
+- [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
+- [PTR_TO_SOCK_COMMON] = "sock_common",
+- [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
+- [PTR_TO_TCP_SOCK] = "tcp_sock",
+- [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
+- [PTR_TO_TP_BUFFER] = "tp_buffer",
+- [PTR_TO_XDP_SOCK] = "xdp_sock",
+- [PTR_TO_BTF_ID] = "ptr_",
+- [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
+- [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
+- [PTR_TO_MEM] = "mem",
+- [PTR_TO_MEM_OR_NULL] = "mem_or_null",
+- [PTR_TO_RDONLY_BUF] = "rdonly_buf",
+- [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",
+- [PTR_TO_RDWR_BUF] = "rdwr_buf",
+- [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",
+- [PTR_TO_FUNC] = "func",
+- [PTR_TO_MAP_KEY] = "map_key",
+-};
++/* string representation of 'enum bpf_reg_type'
++ *
++ * Note that reg_type_str() can not appear more than once in a single verbose()
++ * statement.
++ */
++static const char *reg_type_str(struct bpf_verifier_env *env,
++ enum bpf_reg_type type)
++{
++ char postfix[16] = {0}, prefix[16] = {0};
++ static const char * const str[] = {
++ [NOT_INIT] = "?",
++ [SCALAR_VALUE] = "inv",
++ [PTR_TO_CTX] = "ctx",
++ [CONST_PTR_TO_MAP] = "map_ptr",
++ [PTR_TO_MAP_VALUE] = "map_value",
++ [PTR_TO_STACK] = "fp",
++ [PTR_TO_PACKET] = "pkt",
++ [PTR_TO_PACKET_META] = "pkt_meta",
++ [PTR_TO_PACKET_END] = "pkt_end",
++ [PTR_TO_FLOW_KEYS] = "flow_keys",
++ [PTR_TO_SOCKET] = "sock",
++ [PTR_TO_SOCK_COMMON] = "sock_common",
++ [PTR_TO_TCP_SOCK] = "tcp_sock",
++ [PTR_TO_TP_BUFFER] = "tp_buffer",
++ [PTR_TO_XDP_SOCK] = "xdp_sock",
++ [PTR_TO_BTF_ID] = "ptr_",
++ [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
++ [PTR_TO_MEM] = "mem",
++ [PTR_TO_BUF] = "buf",
++ [PTR_TO_FUNC] = "func",
++ [PTR_TO_MAP_KEY] = "map_key",
++ };
++
++ if (type & PTR_MAYBE_NULL) {
++ if (base_type(type) == PTR_TO_BTF_ID ||
++ base_type(type) == PTR_TO_PERCPU_BTF_ID)
++ strncpy(postfix, "or_null_", 16);
++ else
++ strncpy(postfix, "_or_null", 16);
++ }
++
++ if (type & MEM_RDONLY)
++ strncpy(prefix, "rdonly_", 16);
++
++ snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s",
++ prefix, str[base_type(type)], postfix);
++ return env->type_str_buf;
++}
+
+ static char slot_type_char[] = {
+ [STACK_INVALID] = '?',
+@@ -628,7 +628,7 @@ static void print_verifier_state(struct bpf_verifier_env *env,
+ continue;
+ verbose(env, " R%d", i);
+ print_liveness(env, reg->live);
+- verbose(env, "=%s", reg_type_str[t]);
++ verbose(env, "=%s", reg_type_str(env, t));
+ if (t == SCALAR_VALUE && reg->precise)
+ verbose(env, "P");
+ if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
+@@ -636,9 +636,8 @@ static void print_verifier_state(struct bpf_verifier_env *env,
+ /* reg->off should be 0 for SCALAR_VALUE */
+ verbose(env, "%lld", reg->var_off.value + reg->off);
+ } else {
+- if (t == PTR_TO_BTF_ID ||
+- t == PTR_TO_BTF_ID_OR_NULL ||
+- t == PTR_TO_PERCPU_BTF_ID)
++ if (base_type(t) == PTR_TO_BTF_ID ||
++ base_type(t) == PTR_TO_PERCPU_BTF_ID)
+ verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
+ verbose(env, "(id=%d", reg->id);
+ if (reg_type_may_be_refcounted_or_null(t))
+@@ -647,10 +646,9 @@ static void print_verifier_state(struct bpf_verifier_env *env,
+ verbose(env, ",off=%d", reg->off);
+ if (type_is_pkt_pointer(t))
+ verbose(env, ",r=%d", reg->range);
+- else if (t == CONST_PTR_TO_MAP ||
+- t == PTR_TO_MAP_KEY ||
+- t == PTR_TO_MAP_VALUE ||
+- t == PTR_TO_MAP_VALUE_OR_NULL)
++ else if (base_type(t) == CONST_PTR_TO_MAP ||
++ base_type(t) == PTR_TO_MAP_KEY ||
++ base_type(t) == PTR_TO_MAP_VALUE)
+ verbose(env, ",ks=%d,vs=%d",
+ reg->map_ptr->key_size,
+ reg->map_ptr->value_size);
+@@ -720,7 +718,7 @@ static void print_verifier_state(struct bpf_verifier_env *env,
+ if (state->stack[i].slot_type[0] == STACK_SPILL) {
+ reg = &state->stack[i].spilled_ptr;
+ t = reg->type;
+- verbose(env, "=%s", reg_type_str[t]);
++ verbose(env, "=%s", reg_type_str(env, t));
+ if (t == SCALAR_VALUE && reg->precise)
+ verbose(env, "P");
+ if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
+@@ -1133,8 +1131,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env,
+
+ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
+ {
+- switch (reg->type) {
+- case PTR_TO_MAP_VALUE_OR_NULL: {
++ if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
+ const struct bpf_map *map = reg->map_ptr;
+
+ if (map->inner_map_meta) {
+@@ -1143,7 +1140,8 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
+ /* transfer reg's id which is unique for every map_lookup_elem
+ * as UID of the inner map.
+ */
+- reg->map_uid = reg->id;
++ if (map_value_has_timer(map->inner_map_meta))
++ reg->map_uid = reg->id;
+ } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
+ reg->type = PTR_TO_XDP_SOCK;
+ } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
+@@ -1152,32 +1150,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
+ } else {
+ reg->type = PTR_TO_MAP_VALUE;
+ }
+- break;
+- }
+- case PTR_TO_SOCKET_OR_NULL:
+- reg->type = PTR_TO_SOCKET;
+- break;
+- case PTR_TO_SOCK_COMMON_OR_NULL:
+- reg->type = PTR_TO_SOCK_COMMON;
+- break;
+- case PTR_TO_TCP_SOCK_OR_NULL:
+- reg->type = PTR_TO_TCP_SOCK;
+- break;
+- case PTR_TO_BTF_ID_OR_NULL:
+- reg->type = PTR_TO_BTF_ID;
+- break;
+- case PTR_TO_MEM_OR_NULL:
+- reg->type = PTR_TO_MEM;
+- break;
+- case PTR_TO_RDONLY_BUF_OR_NULL:
+- reg->type = PTR_TO_RDONLY_BUF;
+- break;
+- case PTR_TO_RDWR_BUF_OR_NULL:
+- reg->type = PTR_TO_RDWR_BUF;
+- break;
+- default:
+- WARN_ONCE(1, "unknown nullable register type");
++ return;
+ }
++
++ reg->type &= ~PTR_MAYBE_NULL;
+ }
+
+ static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
+@@ -1357,22 +1333,28 @@ static void __reg_bound_offset(struct bpf_reg_state *reg)
+ reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
+ }
+
++static bool __reg32_bound_s64(s32 a)
++{
++ return a >= 0 && a <= S32_MAX;
++}
++
+ static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
+ {
+ reg->umin_value = reg->u32_min_value;
+ reg->umax_value = reg->u32_max_value;
+- /* Attempt to pull 32-bit signed bounds into 64-bit bounds
+- * but must be positive otherwise set to worse case bounds
+- * and refine later from tnum.
++
++ /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
++ * be positive otherwise set to worse case bounds and refine later
++ * from tnum.
+ */
+- if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0)
+- reg->smax_value = reg->s32_max_value;
+- else
+- reg->smax_value = U32_MAX;
+- if (reg->s32_min_value >= 0)
++ if (__reg32_bound_s64(reg->s32_min_value) &&
++ __reg32_bound_s64(reg->s32_max_value)) {
+ reg->smin_value = reg->s32_min_value;
+- else
++ reg->smax_value = reg->s32_max_value;
++ } else {
+ reg->smin_value = 0;
++ reg->smax_value = U32_MAX;
++ }
+ }
+
+ static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
+@@ -1406,12 +1388,12 @@ static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
+
+ static bool __reg64_bound_s32(s64 a)
+ {
+- return a > S32_MIN && a < S32_MAX;
++ return a >= S32_MIN && a <= S32_MAX;
+ }
+
+ static bool __reg64_bound_u32(u64 a)
+ {
+- return a > U32_MIN && a < U32_MAX;
++ return a >= U32_MIN && a <= U32_MAX;
+ }
+
+ static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
+@@ -1899,7 +1881,7 @@ static int mark_reg_read(struct bpf_verifier_env *env,
+ break;
+ if (parent->live & REG_LIVE_DONE) {
+ verbose(env, "verifier BUG type %s var_off %lld off %d\n",
+- reg_type_str[parent->type],
++ reg_type_str(env, parent->type),
+ parent->var_off.value, parent->off);
+ return -EFAULT;
+ }
+@@ -2557,9 +2539,8 @@ static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
+
+ static bool is_spillable_regtype(enum bpf_reg_type type)
+ {
+- switch (type) {
++ switch (base_type(type)) {
+ case PTR_TO_MAP_VALUE:
+- case PTR_TO_MAP_VALUE_OR_NULL:
+ case PTR_TO_STACK:
+ case PTR_TO_CTX:
+ case PTR_TO_PACKET:
+@@ -2568,21 +2549,13 @@ static bool is_spillable_regtype(enum bpf_reg_type type)
+ case PTR_TO_FLOW_KEYS:
+ case CONST_PTR_TO_MAP:
+ case PTR_TO_SOCKET:
+- case PTR_TO_SOCKET_OR_NULL:
+ case PTR_TO_SOCK_COMMON:
+- case PTR_TO_SOCK_COMMON_OR_NULL:
+ case PTR_TO_TCP_SOCK:
+- case PTR_TO_TCP_SOCK_OR_NULL:
+ case PTR_TO_XDP_SOCK:
+ case PTR_TO_BTF_ID:
+- case PTR_TO_BTF_ID_OR_NULL:
+- case PTR_TO_RDONLY_BUF:
+- case PTR_TO_RDONLY_BUF_OR_NULL:
+- case PTR_TO_RDWR_BUF:
+- case PTR_TO_RDWR_BUF_OR_NULL:
++ case PTR_TO_BUF:
+ case PTR_TO_PERCPU_BTF_ID:
+ case PTR_TO_MEM:
+- case PTR_TO_MEM_OR_NULL:
+ case PTR_TO_FUNC:
+ case PTR_TO_MAP_KEY:
+ return true;
+@@ -3398,7 +3371,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off,
+ */
+ *reg_type = info.reg_type;
+
+- if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) {
++ if (base_type(*reg_type) == PTR_TO_BTF_ID) {
+ *btf = info.btf;
+ *btf_id = info.btf_id;
+ } else {
+@@ -3466,7 +3439,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
+ }
+
+ verbose(env, "R%d invalid %s access off=%d size=%d\n",
+- regno, reg_type_str[reg->type], off, size);
++ regno, reg_type_str(env, reg->type), off, size);
+
+ return -EACCES;
+ }
+@@ -3884,7 +3857,22 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
+
+ static bool bpf_map_is_rdonly(const struct bpf_map *map)
+ {
+- return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen;
++ /* A map is considered read-only if the following condition are true:
++ *
++ * 1) BPF program side cannot change any of the map content. The
++ * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
++ * and was set at map creation time.
++ * 2) The map value(s) have been initialized from user space by a
++ * loader and then "frozen", such that no new map update/delete
++ * operations from syscall side are possible for the rest of
++ * the map's lifetime from that point onwards.
++ * 3) Any parallel/pending map update/delete operations from syscall
++ * side have been completed. Only after that point, it's safe to
++ * assume that map value(s) are immutable.
++ */
++ return (map->map_flags & BPF_F_RDONLY_PROG) &&
++ READ_ONCE(map->frozen) &&
++ !bpf_map_write_active(map);
+ }
+
+ static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val)
+@@ -4178,15 +4166,30 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
+ mark_reg_unknown(env, regs, value_regno);
+ }
+ }
+- } else if (reg->type == PTR_TO_MEM) {
++ } else if (base_type(reg->type) == PTR_TO_MEM) {
++ bool rdonly_mem = type_is_rdonly_mem(reg->type);
++
++ if (type_may_be_null(reg->type)) {
++ verbose(env, "R%d invalid mem access '%s'\n", regno,
++ reg_type_str(env, reg->type));
++ return -EACCES;
++ }
++
++ if (t == BPF_WRITE && rdonly_mem) {
++ verbose(env, "R%d cannot write into %s\n",
++ regno, reg_type_str(env, reg->type));
++ return -EACCES;
++ }
++
+ if (t == BPF_WRITE && value_regno >= 0 &&
+ is_pointer_value(env, value_regno)) {
+ verbose(env, "R%d leaks addr into mem\n", value_regno);
+ return -EACCES;
+ }
++
+ err = check_mem_region_access(env, regno, off, size,
+ reg->mem_size, false);
+- if (!err && t == BPF_READ && value_regno >= 0)
++ if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
+ mark_reg_unknown(env, regs, value_regno);
+ } else if (reg->type == PTR_TO_CTX) {
+ enum bpf_reg_type reg_type = SCALAR_VALUE;
+@@ -4216,7 +4219,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
+ } else {
+ mark_reg_known_zero(env, regs,
+ value_regno);
+- if (reg_type_may_be_null(reg_type))
++ if (type_may_be_null(reg_type))
+ regs[value_regno].id = ++env->id_gen;
+ /* A load of ctx field could have different
+ * actual load size with the one encoded in the
+@@ -4224,8 +4227,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
+ * a sub-register.
+ */
+ regs[value_regno].subreg_def = DEF_NOT_SUBREG;
+- if (reg_type == PTR_TO_BTF_ID ||
+- reg_type == PTR_TO_BTF_ID_OR_NULL) {
++ if (base_type(reg_type) == PTR_TO_BTF_ID) {
+ regs[value_regno].btf = btf;
+ regs[value_regno].btf_id = btf_id;
+ }
+@@ -4278,7 +4280,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
+ } else if (type_is_sk_pointer(reg->type)) {
+ if (t == BPF_WRITE) {
+ verbose(env, "R%d cannot write into %s\n",
+- regno, reg_type_str[reg->type]);
++ regno, reg_type_str(env, reg->type));
+ return -EACCES;
+ }
+ err = check_sock_access(env, insn_idx, regno, off, size, t);
+@@ -4294,26 +4296,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
+ } else if (reg->type == CONST_PTR_TO_MAP) {
+ err = check_ptr_to_map_access(env, regs, regno, off, size, t,
+ value_regno);
+- } else if (reg->type == PTR_TO_RDONLY_BUF) {
+- if (t == BPF_WRITE) {
+- verbose(env, "R%d cannot write into %s\n",
+- regno, reg_type_str[reg->type]);
+- return -EACCES;
++ } else if (base_type(reg->type) == PTR_TO_BUF) {
++ bool rdonly_mem = type_is_rdonly_mem(reg->type);
++ const char *buf_info;
++ u32 *max_access;
++
++ if (rdonly_mem) {
++ if (t == BPF_WRITE) {
++ verbose(env, "R%d cannot write into %s\n",
++ regno, reg_type_str(env, reg->type));
++ return -EACCES;
++ }
++ buf_info = "rdonly";
++ max_access = &env->prog->aux->max_rdonly_access;
++ } else {
++ buf_info = "rdwr";
++ max_access = &env->prog->aux->max_rdwr_access;
+ }
++
+ err = check_buffer_access(env, reg, regno, off, size, false,
+- "rdonly",
+- &env->prog->aux->max_rdonly_access);
+- if (!err && value_regno >= 0)
+- mark_reg_unknown(env, regs, value_regno);
+- } else if (reg->type == PTR_TO_RDWR_BUF) {
+- err = check_buffer_access(env, reg, regno, off, size, false,
+- "rdwr",
+- &env->prog->aux->max_rdwr_access);
+- if (!err && t == BPF_READ && value_regno >= 0)
++ buf_info, max_access);
++
++ if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
+ mark_reg_unknown(env, regs, value_regno);
+ } else {
+ verbose(env, "R%d invalid mem access '%s'\n", regno,
+- reg_type_str[reg->type]);
++ reg_type_str(env, reg->type));
+ return -EACCES;
+ }
+
+@@ -4364,9 +4372,16 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i
+
+ if (insn->imm == BPF_CMPXCHG) {
+ /* Check comparison of R0 with memory location */
+- err = check_reg_arg(env, BPF_REG_0, SRC_OP);
++ const u32 aux_reg = BPF_REG_0;
++
++ err = check_reg_arg(env, aux_reg, SRC_OP);
+ if (err)
+ return err;
++
++ if (is_pointer_value(env, aux_reg)) {
++ verbose(env, "R%d leaks addr into mem\n", aux_reg);
++ return -EACCES;
++ }
+ }
+
+ if (is_pointer_value(env, insn->src_reg)) {
+@@ -4380,7 +4395,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i
+ is_sk_reg(env, insn->dst_reg)) {
+ verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
+ insn->dst_reg,
+- reg_type_str[reg_state(env, insn->dst_reg)->type]);
++ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
+ return -EACCES;
+ }
+
+@@ -4401,13 +4416,19 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i
+ load_reg = -1;
+ }
+
+- /* check whether we can read the memory */
++ /* Check whether we can read the memory, with second call for fetch
++ * case to simulate the register fill.
++ */
+ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
+- BPF_SIZE(insn->code), BPF_READ, load_reg, true);
++ BPF_SIZE(insn->code), BPF_READ, -1, true);
++ if (!err && load_reg >= 0)
++ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
++ BPF_SIZE(insn->code), BPF_READ, load_reg,
++ true);
+ if (err)
+ return err;
+
+- /* check whether we can write into the same memory */
++ /* Check whether we can write into the same memory. */
+ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
+ BPF_SIZE(insn->code), BPF_WRITE, -1, true);
+ if (err)
+@@ -4557,8 +4578,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
+ struct bpf_call_arg_meta *meta)
+ {
+ struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
++ const char *buf_info;
++ u32 *max_access;
+
+- switch (reg->type) {
++ switch (base_type(reg->type)) {
+ case PTR_TO_PACKET:
+ case PTR_TO_PACKET_META:
+ return check_packet_access(env, regno, reg->off, access_size,
+@@ -4577,18 +4600,20 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
+ return check_mem_region_access(env, regno, reg->off,
+ access_size, reg->mem_size,
+ zero_size_allowed);
+- case PTR_TO_RDONLY_BUF:
+- if (meta && meta->raw_mode)
+- return -EACCES;
+- return check_buffer_access(env, reg, regno, reg->off,
+- access_size, zero_size_allowed,
+- "rdonly",
+- &env->prog->aux->max_rdonly_access);
+- case PTR_TO_RDWR_BUF:
++ case PTR_TO_BUF:
++ if (type_is_rdonly_mem(reg->type)) {
++ if (meta && meta->raw_mode)
++ return -EACCES;
++
++ buf_info = "rdonly";
++ max_access = &env->prog->aux->max_rdonly_access;
++ } else {
++ buf_info = "rdwr";
++ max_access = &env->prog->aux->max_rdwr_access;
++ }
+ return check_buffer_access(env, reg, regno, reg->off,
+ access_size, zero_size_allowed,
+- "rdwr",
+- &env->prog->aux->max_rdwr_access);
++ buf_info, max_access);
+ case PTR_TO_STACK:
+ return check_stack_range_initialized(
+ env,
+@@ -4600,9 +4625,9 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
+ register_is_null(reg))
+ return 0;
+
+- verbose(env, "R%d type=%s expected=%s\n", regno,
+- reg_type_str[reg->type],
+- reg_type_str[PTR_TO_STACK]);
++ verbose(env, "R%d type=%s ", regno,
++ reg_type_str(env, reg->type));
++ verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
+ return -EACCES;
+ }
+ }
+@@ -4613,7 +4638,7 @@ int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+ if (register_is_null(reg))
+ return 0;
+
+- if (reg_type_may_be_null(reg->type)) {
++ if (type_may_be_null(reg->type)) {
+ /* Assuming that the register contains a value check if the memory
+ * access is safe. Temporarily save and restore the register's state as
+ * the conversion shouldn't be visible to a caller.
+@@ -4761,9 +4786,8 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno,
+
+ static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
+ {
+- return type == ARG_PTR_TO_MEM ||
+- type == ARG_PTR_TO_MEM_OR_NULL ||
+- type == ARG_PTR_TO_UNINIT_MEM;
++ return base_type(type) == ARG_PTR_TO_MEM ||
++ base_type(type) == ARG_PTR_TO_UNINIT_MEM;
+ }
+
+ static bool arg_type_is_mem_size(enum bpf_arg_type type)
+@@ -4865,8 +4889,7 @@ static const struct bpf_reg_types mem_types = {
+ PTR_TO_MAP_KEY,
+ PTR_TO_MAP_VALUE,
+ PTR_TO_MEM,
+- PTR_TO_RDONLY_BUF,
+- PTR_TO_RDWR_BUF,
++ PTR_TO_BUF,
+ },
+ };
+
+@@ -4897,31 +4920,26 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
+ [ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
+ [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
+ [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,
+- [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types,
+ [ARG_CONST_SIZE] = &scalar_types,
+ [ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
+ [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
+ [ARG_CONST_MAP_PTR] = &const_map_ptr_types,
+ [ARG_PTR_TO_CTX] = &context_types,
+- [ARG_PTR_TO_CTX_OR_NULL] = &context_types,
+ [ARG_PTR_TO_SOCK_COMMON] = &sock_types,
+ #ifdef CONFIG_NET
+ [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
+ #endif
+ [ARG_PTR_TO_SOCKET] = &fullsock_types,
+- [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
+ [ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
+ [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
+ [ARG_PTR_TO_MEM] = &mem_types,
+- [ARG_PTR_TO_MEM_OR_NULL] = &mem_types,
+ [ARG_PTR_TO_UNINIT_MEM] = &mem_types,
+ [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
+- [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
+ [ARG_PTR_TO_INT] = &int_ptr_types,
+ [ARG_PTR_TO_LONG] = &int_ptr_types,
+ [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
+ [ARG_PTR_TO_FUNC] = &func_ptr_types,
+- [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types,
++ [ARG_PTR_TO_STACK] = &stack_ptr_types,
+ [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
+ [ARG_PTR_TO_TIMER] = &timer_types,
+ };
+@@ -4935,12 +4953,27 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
+ const struct bpf_reg_types *compatible;
+ int i, j;
+
+- compatible = compatible_reg_types[arg_type];
++ compatible = compatible_reg_types[base_type(arg_type)];
+ if (!compatible) {
+ verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
+ return -EFAULT;
+ }
+
++ /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
++ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
++ *
++ * Same for MAYBE_NULL:
++ *
++ * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
++ * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
++ *
++ * Therefore we fold these flags depending on the arg_type before comparison.
++ */
++ if (arg_type & MEM_RDONLY)
++ type &= ~MEM_RDONLY;
++ if (arg_type & PTR_MAYBE_NULL)
++ type &= ~PTR_MAYBE_NULL;
++
+ for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
+ expected = compatible->types[i];
+ if (expected == NOT_INIT)
+@@ -4950,14 +4983,14 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
+ goto found;
+ }
+
+- verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
++ verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
+ for (j = 0; j + 1 < i; j++)
+- verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
+- verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
++ verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
++ verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
+ return -EACCES;
+
+ found:
+- if (type == PTR_TO_BTF_ID) {
++ if (reg->type == PTR_TO_BTF_ID) {
+ if (!arg_btf_id) {
+ if (!compatible->btf_id) {
+ verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
+@@ -5016,15 +5049,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
+ return -EACCES;
+ }
+
+- if (arg_type == ARG_PTR_TO_MAP_VALUE ||
+- arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
+- arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
++ if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE ||
++ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
+ err = resolve_map_arg_type(env, meta, &arg_type);
+ if (err)
+ return err;
+ }
+
+- if (register_is_null(reg) && arg_type_may_be_null(arg_type))
++ if (register_is_null(reg) && type_may_be_null(arg_type))
+ /* A NULL register has a SCALAR_VALUE type, so skip
+ * type checking.
+ */
+@@ -5093,10 +5125,11 @@ skip_type_check:
+ err = check_helper_mem_access(env, regno,
+ meta->map_ptr->key_size, false,
+ NULL);
+- } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
+- (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
+- !register_is_null(reg)) ||
+- arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
++ } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE ||
++ base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) {
++ if (type_may_be_null(arg_type) && register_is_null(reg))
++ return 0;
++
+ /* bpf_map_xxx(..., map_ptr, ..., value) call:
+ * check [value, value + map->value_size) validity
+ */
+@@ -5750,6 +5783,7 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ }
+
+ if (insn->code == (BPF_JMP | BPF_CALL) &&
++ insn->src_reg == 0 &&
+ insn->imm == BPF_FUNC_timer_set_callback) {
+ struct bpf_verifier_state *async_cb;
+
+@@ -6170,6 +6204,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ int *insn_idx_p)
+ {
+ const struct bpf_func_proto *fn = NULL;
++ enum bpf_return_type ret_type;
++ enum bpf_type_flag ret_flag;
+ struct bpf_reg_state *regs;
+ struct bpf_call_arg_meta meta;
+ int insn_idx = *insn_idx_p;
+@@ -6303,13 +6339,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
+
+ /* update return register (already marked as written above) */
+- if (fn->ret_type == RET_INTEGER) {
++ ret_type = fn->ret_type;
++ ret_flag = type_flag(fn->ret_type);
++ if (ret_type == RET_INTEGER) {
+ /* sets type to SCALAR_VALUE */
+ mark_reg_unknown(env, regs, BPF_REG_0);
+- } else if (fn->ret_type == RET_VOID) {
++ } else if (ret_type == RET_VOID) {
+ regs[BPF_REG_0].type = NOT_INIT;
+- } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
+- fn->ret_type == RET_PTR_TO_MAP_VALUE) {
++ } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) {
+ /* There is no offset yet applied, variable or fixed */
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+ /* remember map_ptr, so that check_map_access()
+@@ -6323,28 +6360,25 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ }
+ regs[BPF_REG_0].map_ptr = meta.map_ptr;
+ regs[BPF_REG_0].map_uid = meta.map_uid;
+- if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
+- regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
+- if (map_value_has_spin_lock(meta.map_ptr))
+- regs[BPF_REG_0].id = ++env->id_gen;
+- } else {
+- regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
++ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
++ if (!type_may_be_null(ret_type) &&
++ map_value_has_spin_lock(meta.map_ptr)) {
++ regs[BPF_REG_0].id = ++env->id_gen;
+ }
+- } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
++ } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) {
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+- regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
+- } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
++ regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
++ } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) {
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+- regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
+- } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
++ regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
++ } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) {
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+- regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
+- } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {
++ regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
++ } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) {
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+- regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
++ regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
+ regs[BPF_REG_0].mem_size = meta.mem_size;
+- } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL ||
+- fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
++ } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) {
+ const struct btf_type *t;
+
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+@@ -6362,29 +6396,30 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ tname, PTR_ERR(ret));
+ return -EINVAL;
+ }
+- regs[BPF_REG_0].type =
+- fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
+- PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
++ regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
+ regs[BPF_REG_0].mem_size = tsize;
+ } else {
+- regs[BPF_REG_0].type =
+- fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
+- PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
++ /* MEM_RDONLY may be carried from ret_flag, but it
++ * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
++ * it will confuse the check of PTR_TO_BTF_ID in
++ * check_mem_access().
++ */
++ ret_flag &= ~MEM_RDONLY;
++
++ regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
+ regs[BPF_REG_0].btf = meta.ret_btf;
+ regs[BPF_REG_0].btf_id = meta.ret_btf_id;
+ }
+- } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL ||
+- fn->ret_type == RET_PTR_TO_BTF_ID) {
++ } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) {
+ int ret_btf_id;
+
+ mark_reg_known_zero(env, regs, BPF_REG_0);
+- regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
+- PTR_TO_BTF_ID :
+- PTR_TO_BTF_ID_OR_NULL;
++ regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
+ ret_btf_id = *fn->ret_btf_id;
+ if (ret_btf_id == 0) {
+- verbose(env, "invalid return type %d of func %s#%d\n",
+- fn->ret_type, func_id_name(func_id), func_id);
++ verbose(env, "invalid return type %u of func %s#%d\n",
++ base_type(ret_type), func_id_name(func_id),
++ func_id);
+ return -EINVAL;
+ }
+ /* current BPF helper definitions are only coming from
+@@ -6393,12 +6428,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
+ regs[BPF_REG_0].btf = btf_vmlinux;
+ regs[BPF_REG_0].btf_id = ret_btf_id;
+ } else {
+- verbose(env, "unknown return type %d of func %s#%d\n",
+- fn->ret_type, func_id_name(func_id), func_id);
++ verbose(env, "unknown return type %u of func %s#%d\n",
++ base_type(ret_type), func_id_name(func_id), func_id);
+ return -EINVAL;
+ }
+
+- if (reg_type_may_be_null(regs[BPF_REG_0].type))
++ if (type_may_be_null(regs[BPF_REG_0].type))
+ regs[BPF_REG_0].id = ++env->id_gen;
+
+ if (is_ptr_cast_function(func_id)) {
+@@ -6597,25 +6632,25 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env,
+
+ if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
+ verbose(env, "math between %s pointer and %lld is not allowed\n",
+- reg_type_str[type], val);
++ reg_type_str(env, type), val);
+ return false;
+ }
+
+ if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
+ verbose(env, "%s pointer offset %d is not allowed\n",
+- reg_type_str[type], reg->off);
++ reg_type_str(env, type), reg->off);
+ return false;
+ }
+
+ if (smin == S64_MIN) {
+ verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
+- reg_type_str[type]);
++ reg_type_str(env, type));
+ return false;
+ }
+
+ if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
+ verbose(env, "value %lld makes %s pointer be out of bounds\n",
+- smin, reg_type_str[type]);
++ smin, reg_type_str(env, type));
+ return false;
+ }
+
+@@ -6992,11 +7027,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
+ return -EACCES;
+ }
+
+- switch (ptr_reg->type) {
+- case PTR_TO_MAP_VALUE_OR_NULL:
++ if (ptr_reg->type & PTR_MAYBE_NULL) {
+ verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
+- dst, reg_type_str[ptr_reg->type]);
++ dst, reg_type_str(env, ptr_reg->type));
+ return -EACCES;
++ }
++
++ switch (base_type(ptr_reg->type)) {
+ case CONST_PTR_TO_MAP:
+ /* smin_val represents the known value */
+ if (known && smin_val == 0 && opcode == BPF_ADD)
+@@ -7004,16 +7041,16 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
+ fallthrough;
+ case PTR_TO_PACKET_END:
+ case PTR_TO_SOCKET:
+- case PTR_TO_SOCKET_OR_NULL:
+ case PTR_TO_SOCK_COMMON:
+- case PTR_TO_SOCK_COMMON_OR_NULL:
+ case PTR_TO_TCP_SOCK:
+- case PTR_TO_TCP_SOCK_OR_NULL:
+ case PTR_TO_XDP_SOCK:
++reject:
+ verbose(env, "R%d pointer arithmetic on %s prohibited\n",
+- dst, reg_type_str[ptr_reg->type]);
++ dst, reg_type_str(env, ptr_reg->type));
+ return -EACCES;
+ default:
++ if (type_may_be_null(ptr_reg->type))
++ goto reject;
+ break;
+ }
+
+@@ -8098,6 +8135,10 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
+ insn->dst_reg);
+ }
+ zext_32_to_64(dst_reg);
++
++ __update_reg_bounds(dst_reg);
++ __reg_deduce_bounds(dst_reg);
++ __reg_bound_offset(dst_reg);
+ }
+ } else {
+ /* case: R = imm
+@@ -8212,7 +8253,7 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
+
+ new_range = dst_reg->off;
+ if (range_right_open)
+- new_range--;
++ new_range++;
+
+ /* Examples for register markings:
+ *
+@@ -8730,17 +8771,17 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state,
+ struct bpf_reg_state *reg, u32 id,
+ bool is_null)
+ {
+- if (reg_type_may_be_null(reg->type) && reg->id == id &&
++ if (type_may_be_null(reg->type) && reg->id == id &&
+ !WARN_ON_ONCE(!reg->id)) {
+- /* Old offset (both fixed and variable parts) should
+- * have been known-zero, because we don't allow pointer
+- * arithmetic on pointers that might be NULL.
+- */
+ if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
+ !tnum_equals_const(reg->var_off, 0) ||
+ reg->off)) {
+- __mark_reg_known_zero(reg);
+- reg->off = 0;
++ /* Old offset (both fixed and variable parts) should
++ * have been known-zero, because we don't allow pointer
++ * arithmetic on pointers that might be NULL. If we
++ * see this happening, don't convert the register.
++ */
++ return;
+ }
+ if (is_null) {
+ reg->type = SCALAR_VALUE;
+@@ -9108,7 +9149,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
+ */
+ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
+ insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
+- reg_type_may_be_null(dst_reg->type)) {
++ type_may_be_null(dst_reg->type)) {
+ /* Mark all identical registers in each branch as either
+ * safe or unknown depending R == 0 or R != 0 conditional.
+ */
+@@ -9159,11 +9200,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
+ return 0;
+ }
+
+- if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
+- mark_reg_known_zero(env, regs, insn->dst_reg);
++ /* All special src_reg cases are listed below. From this point onwards
++ * we either succeed and assign a corresponding dst_reg->type after
++ * zeroing the offset, or fail and reject the program.
++ */
++ mark_reg_known_zero(env, regs, insn->dst_reg);
+
++ if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
+ dst_reg->type = aux->btf_var.reg_type;
+- switch (dst_reg->type) {
++ switch (base_type(dst_reg->type)) {
+ case PTR_TO_MEM:
+ dst_reg->mem_size = aux->btf_var.mem_size;
+ break;
+@@ -9198,7 +9243,6 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
+ }
+
+ map = env->used_maps[aux->map_index];
+- mark_reg_known_zero(env, regs, insn->dst_reg);
+ dst_reg->map_ptr = map;
+
+ if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
+@@ -9361,7 +9405,7 @@ static int check_return_code(struct bpf_verifier_env *env)
+ /* enforce return zero from async callbacks like timer */
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "In async callback the register R0 is not a known value (%s)\n",
+- reg_type_str[reg->type]);
++ reg_type_str(env, reg->type));
+ return -EINVAL;
+ }
+
+@@ -9375,7 +9419,7 @@ static int check_return_code(struct bpf_verifier_env *env)
+ if (is_subprog) {
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
+- reg_type_str[reg->type]);
++ reg_type_str(env, reg->type));
+ return -EINVAL;
+ }
+ return 0;
+@@ -9439,7 +9483,7 @@ static int check_return_code(struct bpf_verifier_env *env)
+
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "At program exit the register R0 is not a known value (%s)\n",
+- reg_type_str[reg->type]);
++ reg_type_str(env, reg->type));
+ return -EINVAL;
+ }
+
+@@ -10220,7 +10264,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
+ return true;
+ if (rcur->type == NOT_INIT)
+ return false;
+- switch (rold->type) {
++ switch (base_type(rold->type)) {
+ case SCALAR_VALUE:
+ if (env->explore_alu_limits)
+ return false;
+@@ -10242,6 +10286,22 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
+ }
+ case PTR_TO_MAP_KEY:
+ case PTR_TO_MAP_VALUE:
++ /* a PTR_TO_MAP_VALUE could be safe to use as a
++ * PTR_TO_MAP_VALUE_OR_NULL into the same map.
++ * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
++ * checked, doing so could have affected others with the same
++ * id, and we can't check for that because we lost the id when
++ * we converted to a PTR_TO_MAP_VALUE.
++ */
++ if (type_may_be_null(rold->type)) {
++ if (!type_may_be_null(rcur->type))
++ return false;
++ if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
++ return false;
++ /* Check our ids match any regs they're supposed to */
++ return check_ids(rold->id, rcur->id, idmap);
++ }
++
+ /* If the new min/max/var_off satisfy the old ones and
+ * everything else matches, we are OK.
+ * 'id' is not compared, since it's only used for maps with
+@@ -10253,20 +10313,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
+ return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
+ range_within(rold, rcur) &&
+ tnum_in(rold->var_off, rcur->var_off);
+- case PTR_TO_MAP_VALUE_OR_NULL:
+- /* a PTR_TO_MAP_VALUE could be safe to use as a
+- * PTR_TO_MAP_VALUE_OR_NULL into the same map.
+- * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
+- * checked, doing so could have affected others with the same
+- * id, and we can't check for that because we lost the id when
+- * we converted to a PTR_TO_MAP_VALUE.
+- */
+- if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
+- return false;
+- if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
+- return false;
+- /* Check our ids match any regs they're supposed to */
+- return check_ids(rold->id, rcur->id, idmap);
+ case PTR_TO_PACKET_META:
+ case PTR_TO_PACKET:
+ if (rcur->type != rold->type)
+@@ -10295,11 +10341,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
+ case PTR_TO_PACKET_END:
+ case PTR_TO_FLOW_KEYS:
+ case PTR_TO_SOCKET:
+- case PTR_TO_SOCKET_OR_NULL:
+ case PTR_TO_SOCK_COMMON:
+- case PTR_TO_SOCK_COMMON_OR_NULL:
+ case PTR_TO_TCP_SOCK:
+- case PTR_TO_TCP_SOCK_OR_NULL:
+ case PTR_TO_XDP_SOCK:
+ /* Only valid matches are exact, which memcmp() above
+ * would have accepted
+@@ -10825,17 +10868,13 @@ next:
+ /* Return true if it's OK to have the same insn return a different type. */
+ static bool reg_type_mismatch_ok(enum bpf_reg_type type)
+ {
+- switch (type) {
++ switch (base_type(type)) {
+ case PTR_TO_CTX:
+ case PTR_TO_SOCKET:
+- case PTR_TO_SOCKET_OR_NULL:
+ case PTR_TO_SOCK_COMMON:
+- case PTR_TO_SOCK_COMMON_OR_NULL:
+ case PTR_TO_TCP_SOCK:
+- case PTR_TO_TCP_SOCK_OR_NULL:
+ case PTR_TO_XDP_SOCK:
+ case PTR_TO_BTF_ID:
+- case PTR_TO_BTF_ID_OR_NULL:
+ return false;
+ default:
+ return true;
+@@ -11059,7 +11098,7 @@ static int do_check(struct bpf_verifier_env *env)
+ if (is_ctx_reg(env, insn->dst_reg)) {
+ verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
+ insn->dst_reg,
+- reg_type_str[reg_state(env, insn->dst_reg)->type]);
++ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
+ return -EACCES;
+ }
+
+@@ -11310,7 +11349,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env,
+ err = -EINVAL;
+ goto err_put;
+ }
+- aux->btf_var.reg_type = PTR_TO_MEM;
++ aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
+ aux->btf_var.mem_size = tsize;
+ } else {
+ aux->btf_var.reg_type = PTR_TO_BTF_ID;
+@@ -11435,6 +11474,13 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
+ }
+ }
+
++ if (map_value_has_timer(map)) {
++ if (is_tracing_prog_type(prog_type)) {
++ verbose(env, "tracing progs cannot use bpf_timer yet\n");
++ return -EINVAL;
++ }
++ }
++
+ if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
+ !bpf_offload_prog_map_match(prog, map)) {
+ verbose(env, "offload device mismatch between prog and map\n");
+@@ -13125,7 +13171,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
+ mark_reg_known_zero(env, regs, i);
+ else if (regs[i].type == SCALAR_VALUE)
+ mark_reg_unknown(env, regs, i);
+- else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
++ else if (base_type(regs[i].type) == PTR_TO_MEM) {
+ const u32 mem_size = regs[i].mem_size;
+
+ mark_reg_known_zero(env, regs, i);
+@@ -13713,11 +13759,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr)
+ log->ubuf = (char __user *) (unsigned long) attr->log_buf;
+ log->len_total = attr->log_size;
+
+- ret = -EINVAL;
+ /* log attributes have to be sane */
+- if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
+- !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
++ if (!bpf_verifier_log_attr_valid(log)) {
++ ret = -EINVAL;
+ goto err_unlock;
++ }
+ }
+
+ if (IS_ERR(btf_vmlinux)) {
+diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
+index bfbeabc17a9df..6e36e854b5124 100644
+--- a/kernel/cgroup/cgroup-internal.h
++++ b/kernel/cgroup/cgroup-internal.h
+@@ -65,6 +65,25 @@ static inline struct cgroup_fs_context *cgroup_fc2context(struct fs_context *fc)
+ return container_of(kfc, struct cgroup_fs_context, kfc);
+ }
+
++struct cgroup_pidlist;
++
++struct cgroup_file_ctx {
++ struct cgroup_namespace *ns;
++
++ struct {
++ void *trigger;
++ } psi;
++
++ struct {
++ bool started;
++ struct css_task_iter iter;
++ } procs;
++
++ struct {
++ struct cgroup_pidlist *pidlist;
++ } procs1;
++};
++
+ /*
+ * A cgroup can be associated with multiple css_sets as different tasks may
+ * belong to different cgroups on different hierarchies. In the other
+diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
+index 35b9203283447..58900dc92ac98 100644
+--- a/kernel/cgroup/cgroup-v1.c
++++ b/kernel/cgroup/cgroup-v1.c
+@@ -397,6 +397,7 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
+ * next pid to display, if any
+ */
+ struct kernfs_open_file *of = s->private;
++ struct cgroup_file_ctx *ctx = of->priv;
+ struct cgroup *cgrp = seq_css(s)->cgroup;
+ struct cgroup_pidlist *l;
+ enum cgroup_filetype type = seq_cft(s)->private;
+@@ -406,25 +407,24 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
+ mutex_lock(&cgrp->pidlist_mutex);
+
+ /*
+- * !NULL @of->priv indicates that this isn't the first start()
+- * after open. If the matching pidlist is around, we can use that.
+- * Look for it. Note that @of->priv can't be used directly. It
+- * could already have been destroyed.
++ * !NULL @ctx->procs1.pidlist indicates that this isn't the first
++ * start() after open. If the matching pidlist is around, we can use
++ * that. Look for it. Note that @ctx->procs1.pidlist can't be used
++ * directly. It could already have been destroyed.
+ */
+- if (of->priv)
+- of->priv = cgroup_pidlist_find(cgrp, type);
++ if (ctx->procs1.pidlist)
++ ctx->procs1.pidlist = cgroup_pidlist_find(cgrp, type);
+
+ /*
+ * Either this is the first start() after open or the matching
+ * pidlist has been destroyed inbetween. Create a new one.
+ */
+- if (!of->priv) {
+- ret = pidlist_array_load(cgrp, type,
+- (struct cgroup_pidlist **)&of->priv);
++ if (!ctx->procs1.pidlist) {
++ ret = pidlist_array_load(cgrp, type, &ctx->procs1.pidlist);
+ if (ret)
+ return ERR_PTR(ret);
+ }
+- l = of->priv;
++ l = ctx->procs1.pidlist;
+
+ if (pid) {
+ int end = l->length;
+@@ -452,7 +452,8 @@ static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
+ static void cgroup_pidlist_stop(struct seq_file *s, void *v)
+ {
+ struct kernfs_open_file *of = s->private;
+- struct cgroup_pidlist *l = of->priv;
++ struct cgroup_file_ctx *ctx = of->priv;
++ struct cgroup_pidlist *l = ctx->procs1.pidlist;
+
+ if (l)
+ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
+@@ -463,7 +464,8 @@ static void cgroup_pidlist_stop(struct seq_file *s, void *v)
+ static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
+ {
+ struct kernfs_open_file *of = s->private;
+- struct cgroup_pidlist *l = of->priv;
++ struct cgroup_file_ctx *ctx = of->priv;
++ struct cgroup_pidlist *l = ctx->procs1.pidlist;
+ pid_t *p = v;
+ pid_t *end = l->list + l->length;
+ /*
+@@ -507,10 +509,11 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of,
+ goto out_unlock;
+
+ /*
+- * Even if we're attaching all tasks in the thread group, we only
+- * need to check permissions on one of them.
++ * Even if we're attaching all tasks in the thread group, we only need
++ * to check permissions on one of them. Check permissions using the
++ * credentials from file open to protect against inherited fd attacks.
+ */
+- cred = current_cred();
++ cred = of->file->f_cred;
+ tcred = get_task_cred(task);
+ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
+ !uid_eq(cred->euid, tcred->uid) &&
+@@ -546,9 +549,19 @@ static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes, loff_t off)
+ {
+ struct cgroup *cgrp;
++ struct cgroup_file_ctx *ctx;
+
+ BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
+
++ /*
++ * Release agent gets called with all capabilities,
++ * require capabilities to set release agent.
++ */
++ ctx = of->priv;
++ if ((ctx->ns->user_ns != &init_user_ns) ||
++ !file_ns_capable(of->file, &init_user_ns, CAP_SYS_ADMIN))
++ return -EPERM;
++
+ cgrp = cgroup_kn_lock_live(of->kn, false);
+ if (!cgrp)
+ return -ENODEV;
+@@ -960,6 +973,12 @@ int cgroup1_parse_param(struct fs_context *fc, struct fs_parameter *param)
+ /* Specifying two release agents is forbidden */
+ if (ctx->release_agent)
+ return invalfc(fc, "release_agent respecified");
++ /*
++ * Release agent gets called with all capabilities,
++ * require capabilities to set release agent.
++ */
++ if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN))
++ return invalfc(fc, "Setting release_agent not allowed");
+ ctx->release_agent = param->string;
+ param->string = NULL;
+ break;
+diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
+index ea08f01d0111a..de8b4fa1e1fd6 100644
+--- a/kernel/cgroup/cgroup.c
++++ b/kernel/cgroup/cgroup.c
+@@ -1740,6 +1740,7 @@ int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
+ struct cgroup *dcgrp = &dst_root->cgrp;
+ struct cgroup_subsys *ss;
+ int ssid, i, ret;
++ u16 dfl_disable_ss_mask = 0;
+
+ lockdep_assert_held(&cgroup_mutex);
+
+@@ -1756,8 +1757,28 @@ int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
+ /* can't move between two non-dummy roots either */
+ if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
+ return -EBUSY;
++
++ /*
++ * Collect ssid's that need to be disabled from default
++ * hierarchy.
++ */
++ if (ss->root == &cgrp_dfl_root)
++ dfl_disable_ss_mask |= 1 << ssid;
++
+ } while_each_subsys_mask();
+
++ if (dfl_disable_ss_mask) {
++ struct cgroup *scgrp = &cgrp_dfl_root.cgrp;
++
++ /*
++ * Controllers from default hierarchy that need to be rebound
++ * are all disabled together in one go.
++ */
++ cgrp_dfl_root.subsys_mask &= ~dfl_disable_ss_mask;
++ WARN_ON(cgroup_apply_control(scgrp));
++ cgroup_finalize_control(scgrp, 0);
++ }
++
+ do_each_subsys_mask(ss, ssid, ss_mask) {
+ struct cgroup_root *src_root = ss->root;
+ struct cgroup *scgrp = &src_root->cgrp;
+@@ -1766,10 +1787,12 @@ int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
+
+ WARN_ON(!css || cgroup_css(dcgrp, ss));
+
+- /* disable from the source */
+- src_root->subsys_mask &= ~(1 << ssid);
+- WARN_ON(cgroup_apply_control(scgrp));
+- cgroup_finalize_control(scgrp, 0);
++ if (src_root != &cgrp_dfl_root) {
++ /* disable from the source */
++ src_root->subsys_mask &= ~(1 << ssid);
++ WARN_ON(cgroup_apply_control(scgrp));
++ cgroup_finalize_control(scgrp, 0);
++ }
+
+ /* rebind */
+ RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
+@@ -3607,6 +3630,7 @@ static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
+ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,
+ size_t nbytes, enum psi_res res)
+ {
++ struct cgroup_file_ctx *ctx = of->priv;
+ struct psi_trigger *new;
+ struct cgroup *cgrp;
+ struct psi_group *psi;
+@@ -3618,6 +3642,12 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,
+ cgroup_get(cgrp);
+ cgroup_kn_unlock(of->kn);
+
++ /* Allow only one trigger per file descriptor */
++ if (ctx->psi.trigger) {
++ cgroup_put(cgrp);
++ return -EBUSY;
++ }
++
+ psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi;
+ new = psi_trigger_create(psi, buf, nbytes, res);
+ if (IS_ERR(new)) {
+@@ -3625,8 +3655,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,
+ return PTR_ERR(new);
+ }
+
+- psi_trigger_replace(&of->priv, new);
+-
++ smp_store_release(&ctx->psi.trigger, new);
+ cgroup_put(cgrp);
+
+ return nbytes;
+@@ -3656,12 +3685,16 @@ static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
+ static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
+ poll_table *pt)
+ {
+- return psi_trigger_poll(&of->priv, of->file, pt);
++ struct cgroup_file_ctx *ctx = of->priv;
++
++ return psi_trigger_poll(&ctx->psi.trigger, of->file, pt);
+ }
+
+ static void cgroup_pressure_release(struct kernfs_open_file *of)
+ {
+- psi_trigger_replace(&of->priv, NULL);
++ struct cgroup_file_ctx *ctx = of->priv;
++
++ psi_trigger_destroy(ctx->psi.trigger);
+ }
+
+ bool cgroup_psi_enabled(void)
+@@ -3788,24 +3821,43 @@ static ssize_t cgroup_kill_write(struct kernfs_open_file *of, char *buf,
+ static int cgroup_file_open(struct kernfs_open_file *of)
+ {
+ struct cftype *cft = of_cft(of);
++ struct cgroup_file_ctx *ctx;
++ int ret;
+
+- if (cft->open)
+- return cft->open(of);
+- return 0;
++ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
++ if (!ctx)
++ return -ENOMEM;
++
++ ctx->ns = current->nsproxy->cgroup_ns;
++ get_cgroup_ns(ctx->ns);
++ of->priv = ctx;
++
++ if (!cft->open)
++ return 0;
++
++ ret = cft->open(of);
++ if (ret) {
++ put_cgroup_ns(ctx->ns);
++ kfree(ctx);
++ }
++ return ret;
+ }
+
+ static void cgroup_file_release(struct kernfs_open_file *of)
+ {
+ struct cftype *cft = of_cft(of);
++ struct cgroup_file_ctx *ctx = of->priv;
+
+ if (cft->release)
+ cft->release(of);
++ put_cgroup_ns(ctx->ns);
++ kfree(ctx);
+ }
+
+ static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
+ size_t nbytes, loff_t off)
+ {
+- struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
++ struct cgroup_file_ctx *ctx = of->priv;
+ struct cgroup *cgrp = of->kn->parent->priv;
+ struct cftype *cft = of_cft(of);
+ struct cgroup_subsys_state *css;
+@@ -3822,7 +3874,7 @@ static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
+ */
+ if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) &&
+ !(cft->flags & CFTYPE_NS_DELEGATABLE) &&
+- ns != &init_cgroup_ns && ns->root_cset->dfl_cgrp == cgrp)
++ ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp)
+ return -EPERM;
+
+ if (cft->write)
+@@ -4728,21 +4780,21 @@ void css_task_iter_end(struct css_task_iter *it)
+
+ static void cgroup_procs_release(struct kernfs_open_file *of)
+ {
+- if (of->priv) {
+- css_task_iter_end(of->priv);
+- kfree(of->priv);
+- }
++ struct cgroup_file_ctx *ctx = of->priv;
++
++ if (ctx->procs.started)
++ css_task_iter_end(&ctx->procs.iter);
+ }
+
+ static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos)
+ {
+ struct kernfs_open_file *of = s->private;
+- struct css_task_iter *it = of->priv;
++ struct cgroup_file_ctx *ctx = of->priv;
+
+ if (pos)
+ (*pos)++;
+
+- return css_task_iter_next(it);
++ return css_task_iter_next(&ctx->procs.iter);
+ }
+
+ static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos,
+@@ -4750,21 +4802,18 @@ static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos,
+ {
+ struct kernfs_open_file *of = s->private;
+ struct cgroup *cgrp = seq_css(s)->cgroup;
+- struct css_task_iter *it = of->priv;
++ struct cgroup_file_ctx *ctx = of->priv;
++ struct css_task_iter *it = &ctx->procs.iter;
+
+ /*
+ * When a seq_file is seeked, it's always traversed sequentially
+ * from position 0, so we can simply keep iterating on !0 *pos.
+ */
+- if (!it) {
++ if (!ctx->procs.started) {
+ if (WARN_ON_ONCE((*pos)))
+ return ERR_PTR(-EINVAL);
+-
+- it = kzalloc(sizeof(*it), GFP_KERNEL);
+- if (!it)
+- return ERR_PTR(-ENOMEM);
+- of->priv = it;
+ css_task_iter_start(&cgrp->self, iter_flags, it);
++ ctx->procs.started = true;
+ } else if (!(*pos)) {
+ css_task_iter_end(it);
+ css_task_iter_start(&cgrp->self, iter_flags, it);
+@@ -4815,9 +4864,9 @@ static int cgroup_may_write(const struct cgroup *cgrp, struct super_block *sb)
+
+ static int cgroup_procs_write_permission(struct cgroup *src_cgrp,
+ struct cgroup *dst_cgrp,
+- struct super_block *sb)
++ struct super_block *sb,
++ struct cgroup_namespace *ns)
+ {
+- struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
+ struct cgroup *com_cgrp = src_cgrp;
+ int ret;
+
+@@ -4846,11 +4895,12 @@ static int cgroup_procs_write_permission(struct cgroup *src_cgrp,
+
+ static int cgroup_attach_permissions(struct cgroup *src_cgrp,
+ struct cgroup *dst_cgrp,
+- struct super_block *sb, bool threadgroup)
++ struct super_block *sb, bool threadgroup,
++ struct cgroup_namespace *ns)
+ {
+ int ret = 0;
+
+- ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, sb);
++ ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, sb, ns);
+ if (ret)
+ return ret;
+
+@@ -4867,8 +4917,10 @@ static int cgroup_attach_permissions(struct cgroup *src_cgrp,
+ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
+ bool threadgroup)
+ {
++ struct cgroup_file_ctx *ctx = of->priv;
+ struct cgroup *src_cgrp, *dst_cgrp;
+ struct task_struct *task;
++ const struct cred *saved_cred;
+ ssize_t ret;
+ bool locked;
+
+@@ -4886,9 +4938,16 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
+ src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
+ spin_unlock_irq(&css_set_lock);
+
+- /* process and thread migrations follow same delegation rule */
++ /*
++ * Process and thread migrations follow same delegation rule. Check
++ * permissions using the credentials from file open to protect against
++ * inherited fd attacks.
++ */
++ saved_cred = override_creds(of->file->f_cred);
+ ret = cgroup_attach_permissions(src_cgrp, dst_cgrp,
+- of->file->f_path.dentry->d_sb, threadgroup);
++ of->file->f_path.dentry->d_sb,
++ threadgroup, ctx->ns);
++ revert_creds(saved_cred);
+ if (ret)
+ goto out_finish;
+
+@@ -6104,7 +6163,8 @@ static int cgroup_css_set_fork(struct kernel_clone_args *kargs)
+ goto err;
+
+ ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,
+- !(kargs->flags & CLONE_THREAD));
++ !(kargs->flags & CLONE_THREAD),
++ current->nsproxy->cgroup_ns);
+ if (ret)
+ goto err;
+
+diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
+index 2a9695ccb65f5..31f94c6ea0a55 100644
+--- a/kernel/cgroup/cpuset.c
++++ b/kernel/cgroup/cpuset.c
+@@ -1512,10 +1512,15 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
+ struct cpuset *sibling;
+ struct cgroup_subsys_state *pos_css;
+
++ percpu_rwsem_assert_held(&cpuset_rwsem);
++
+ /*
+ * Check all its siblings and call update_cpumasks_hier()
+ * if their use_parent_ecpus flag is set in order for them
+ * to use the right effective_cpus value.
++ *
++ * The update_cpumasks_hier() function may sleep. So we have to
++ * release the RCU read lock before calling it.
+ */
+ rcu_read_lock();
+ cpuset_for_each_child(sibling, pos_css, parent) {
+@@ -1523,8 +1528,13 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
+ continue;
+ if (!sibling->use_parent_ecpus)
+ continue;
++ if (!css_tryget_online(&sibling->css))
++ continue;
+
++ rcu_read_unlock();
+ update_cpumasks_hier(sibling, tmp);
++ rcu_read_lock();
++ css_put(&sibling->css);
+ }
+ rcu_read_unlock();
+ }
+@@ -1597,8 +1607,7 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
+ * Make sure that subparts_cpus is a subset of cpus_allowed.
+ */
+ if (cs->nr_subparts_cpus) {
+- cpumask_andnot(cs->subparts_cpus, cs->subparts_cpus,
+- cs->cpus_allowed);
++ cpumask_and(cs->subparts_cpus, cs->subparts_cpus, cs->cpus_allowed);
+ cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus);
+ }
+ spin_unlock_irq(&callback_lock);
+@@ -2240,6 +2249,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
+ cgroup_taskset_first(tset, &css);
+ cs = css_cs(css);
+
++ cpus_read_lock();
+ percpu_down_write(&cpuset_rwsem);
+
+ guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
+@@ -2293,6 +2303,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
+ wake_up(&cpuset_attach_wq);
+
+ percpu_up_write(&cpuset_rwsem);
++ cpus_read_unlock();
+ }
+
+ /* The various types of files and directories in a cpuset file system */
+@@ -3336,8 +3347,11 @@ static struct notifier_block cpuset_track_online_nodes_nb = {
+ */
+ void __init cpuset_init_smp(void)
+ {
+- cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
+- top_cpuset.mems_allowed = node_states[N_MEMORY];
++ /*
++ * cpus_allowd/mems_allowed set to v2 values in the initial
++ * cpuset_bind() call will be reset to v1 values in another
++ * cpuset_bind() call when v1 cpuset is mounted.
++ */
+ top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
+
+ cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
+diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
+index b264ab5652ba9..1486768f23185 100644
+--- a/kernel/cgroup/rstat.c
++++ b/kernel/cgroup/rstat.c
+@@ -433,8 +433,6 @@ static void root_cgroup_cputime(struct task_cputime *cputime)
+ cputime->sum_exec_runtime += user;
+ cputime->sum_exec_runtime += sys;
+ cputime->sum_exec_runtime += cpustat[CPUTIME_STEAL];
+- cputime->sum_exec_runtime += cpustat[CPUTIME_GUEST];
+- cputime->sum_exec_runtime += cpustat[CPUTIME_GUEST_NICE];
+ }
+ }
+
+diff --git a/kernel/cpu.c b/kernel/cpu.c
+index 192e43a874076..da871eb075662 100644
+--- a/kernel/cpu.c
++++ b/kernel/cpu.c
+@@ -31,8 +31,10 @@
+ #include <linux/smpboot.h>
+ #include <linux/relay.h>
+ #include <linux/slab.h>
++#include <linux/scs.h>
+ #include <linux/percpu-rwsem.h>
+ #include <linux/cpuset.h>
++#include <linux/random.h>
+
+ #include <trace/events/power.h>
+ #define CREATE_TRACE_POINTS
+@@ -69,7 +71,6 @@ struct cpuhp_cpu_state {
+ bool rollback;
+ bool single;
+ bool bringup;
+- int cpu;
+ struct hlist_node *node;
+ struct hlist_node *last;
+ enum cpuhp_state cb_state;
+@@ -473,7 +474,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
+ #endif
+
+ static inline enum cpuhp_state
+-cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
++cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
+ {
+ enum cpuhp_state prev_state = st->state;
+ bool bringup = st->state < target;
+@@ -484,14 +485,15 @@ cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
+ st->target = target;
+ st->single = false;
+ st->bringup = bringup;
+- if (cpu_dying(st->cpu) != !bringup)
+- set_cpu_dying(st->cpu, !bringup);
++ if (cpu_dying(cpu) != !bringup)
++ set_cpu_dying(cpu, !bringup);
+
+ return prev_state;
+ }
+
+ static inline void
+-cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
++cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
++ enum cpuhp_state prev_state)
+ {
+ bool bringup = !st->bringup;
+
+@@ -518,8 +520,8 @@ cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
+ }
+
+ st->bringup = bringup;
+- if (cpu_dying(st->cpu) != !bringup)
+- set_cpu_dying(st->cpu, !bringup);
++ if (cpu_dying(cpu) != !bringup)
++ set_cpu_dying(cpu, !bringup);
+ }
+
+ /* Regular hotplug invocation of the AP hotplug thread */
+@@ -539,15 +541,16 @@ static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
+ wait_for_ap_thread(st, st->bringup);
+ }
+
+-static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
++static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
++ enum cpuhp_state target)
+ {
+ enum cpuhp_state prev_state;
+ int ret;
+
+- prev_state = cpuhp_set_state(st, target);
++ prev_state = cpuhp_set_state(cpu, st, target);
+ __cpuhp_kick_ap(st);
+ if ((ret = st->result)) {
+- cpuhp_reset_state(st, prev_state);
++ cpuhp_reset_state(cpu, st, prev_state);
+ __cpuhp_kick_ap(st);
+ }
+
+@@ -579,7 +582,7 @@ static int bringup_wait_for_ap(unsigned int cpu)
+ if (st->target <= CPUHP_AP_ONLINE_IDLE)
+ return 0;
+
+- return cpuhp_kick_ap(st, st->target);
++ return cpuhp_kick_ap(cpu, st, st->target);
+ }
+
+ static int bringup_cpu(unsigned int cpu)
+@@ -587,6 +590,12 @@ static int bringup_cpu(unsigned int cpu)
+ struct task_struct *idle = idle_thread_get(cpu);
+ int ret;
+
++ /*
++ * Reset stale stack state from the last time this CPU was online.
++ */
++ scs_task_reset(idle);
++ kasan_unpoison_task_stack(idle);
++
+ /*
+ * Some architectures have to walk the irq descriptors to
+ * setup the vector space for the cpu which comes online.
+@@ -696,7 +705,7 @@ static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
+ ret, cpu, cpuhp_get_step(st->state)->name,
+ st->state);
+
+- cpuhp_reset_state(st, prev_state);
++ cpuhp_reset_state(cpu, st, prev_state);
+ if (can_rollback_cpu(st))
+ WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
+ prev_state));
+@@ -713,7 +722,6 @@ static void cpuhp_create(unsigned int cpu)
+
+ init_completion(&st->done_up);
+ init_completion(&st->done_down);
+- st->cpu = cpu;
+ }
+
+ static int cpuhp_should_run(unsigned int cpu)
+@@ -867,7 +875,7 @@ static int cpuhp_kick_ap_work(unsigned int cpu)
+ cpuhp_lock_release(true);
+
+ trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
+- ret = cpuhp_kick_ap(st, st->target);
++ ret = cpuhp_kick_ap(cpu, st, st->target);
+ trace_cpuhp_exit(cpu, st->state, prev_state, ret);
+
+ return ret;
+@@ -1099,7 +1107,7 @@ static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
+ ret, cpu, cpuhp_get_step(st->state)->name,
+ st->state);
+
+- cpuhp_reset_state(st, prev_state);
++ cpuhp_reset_state(cpu, st, prev_state);
+
+ if (st->state < prev_state)
+ WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
+@@ -1126,7 +1134,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+
+ cpuhp_tasks_frozen = tasks_frozen;
+
+- prev_state = cpuhp_set_state(st, target);
++ prev_state = cpuhp_set_state(cpu, st, target);
+ /*
+ * If the current CPU state is in the range of the AP hotplug thread,
+ * then we need to kick the thread.
+@@ -1157,7 +1165,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
+ ret = cpuhp_down_callbacks(cpu, st, target);
+ if (ret && st->state < prev_state) {
+ if (st->state == CPUHP_TEARDOWN_CPU) {
+- cpuhp_reset_state(st, prev_state);
++ cpuhp_reset_state(cpu, st, prev_state);
+ __cpuhp_kick_ap(st);
+ } else {
+ WARN(1, "DEAD callback error for CPU%d", cpu);
+@@ -1344,7 +1352,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
+
+ cpuhp_tasks_frozen = tasks_frozen;
+
+- cpuhp_set_state(st, target);
++ cpuhp_set_state(cpu, st, target);
+ /*
+ * If the current CPU state is in the range of the AP hotplug thread,
+ * then we need to kick the thread once more.
+@@ -1652,6 +1660,11 @@ static struct cpuhp_step cpuhp_hp_states[] = {
+ .startup.single = perf_event_init_cpu,
+ .teardown.single = perf_event_exit_cpu,
+ },
++ [CPUHP_RANDOM_PREPARE] = {
++ .name = "random:prepare",
++ .startup.single = random_prepare_cpu,
++ .teardown.single = NULL,
++ },
+ [CPUHP_WORKQUEUE_PREP] = {
+ .name = "workqueue:prepare",
+ .startup.single = workqueue_prepare_cpu,
+@@ -1775,6 +1788,11 @@ static struct cpuhp_step cpuhp_hp_states[] = {
+ .startup.single = workqueue_online_cpu,
+ .teardown.single = workqueue_offline_cpu,
+ },
++ [CPUHP_AP_RANDOM_ONLINE] = {
++ .name = "random:online",
++ .startup.single = random_online_cpu,
++ .teardown.single = NULL,
++ },
+ [CPUHP_AP_RCUTREE_ONLINE] = {
+ .name = "RCU/tree:online",
+ .startup.single = rcutree_online_cpu,
+diff --git a/kernel/crash_core.c b/kernel/crash_core.c
+index eb53f5ec62c90..256cf6db573cd 100644
+--- a/kernel/crash_core.c
++++ b/kernel/crash_core.c
+@@ -6,6 +6,7 @@
+
+ #include <linux/buildid.h>
+ #include <linux/crash_core.h>
++#include <linux/init.h>
+ #include <linux/utsname.h>
+ #include <linux/vmalloc.h>
+
+@@ -295,6 +296,16 @@ int __init parse_crashkernel_low(char *cmdline,
+ "crashkernel=", suffix_tbl[SUFFIX_LOW]);
+ }
+
++/*
++ * Add a dummy early_param handler to mark crashkernel= as a known command line
++ * parameter and suppress incorrect warnings in init/main.c.
++ */
++static int __init parse_crashkernel_dummy(char *arg)
++{
++ return 0;
++}
++early_param("crashkernel", parse_crashkernel_dummy);
++
+ Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
+ void *data, size_t data_len)
+ {
+diff --git a/kernel/cred.c b/kernel/cred.c
+index 1ae0b4948a5a8..933155c969227 100644
+--- a/kernel/cred.c
++++ b/kernel/cred.c
+@@ -665,26 +665,20 @@ EXPORT_SYMBOL(cred_fscmp);
+
+ int set_cred_ucounts(struct cred *new)
+ {
+- struct task_struct *task = current;
+- const struct cred *old = task->real_cred;
+ struct ucounts *new_ucounts, *old_ucounts = new->ucounts;
+
+- if (new->user == old->user && new->user_ns == old->user_ns)
+- return 0;
+-
+ /*
+ * This optimization is needed because alloc_ucounts() uses locks
+ * for table lookups.
+ */
+- if (old_ucounts && old_ucounts->ns == new->user_ns && uid_eq(old_ucounts->uid, new->euid))
++ if (old_ucounts->ns == new->user_ns && uid_eq(old_ucounts->uid, new->uid))
+ return 0;
+
+- if (!(new_ucounts = alloc_ucounts(new->user_ns, new->euid)))
++ if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid)))
+ return -EAGAIN;
+
+ new->ucounts = new_ucounts;
+- if (old_ucounts)
+- put_ucounts(old_ucounts);
++ put_ucounts(old_ucounts);
+
+ return 0;
+ }
+diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
+index da06a5553835b..7beceb447211d 100644
+--- a/kernel/debug/debug_core.c
++++ b/kernel/debug/debug_core.c
+@@ -53,6 +53,7 @@
+ #include <linux/vmacache.h>
+ #include <linux/rcupdate.h>
+ #include <linux/irq.h>
++#include <linux/security.h>
+
+ #include <asm/cacheflush.h>
+ #include <asm/byteorder.h>
+@@ -752,6 +753,29 @@ cpu_master_loop:
+ continue;
+ kgdb_connected = 0;
+ } else {
++ /*
++ * This is a brutal way to interfere with the debugger
++ * and prevent gdb being used to poke at kernel memory.
++ * This could cause trouble if lockdown is applied when
++ * there is already an active gdb session. For now the
++ * answer is simply "don't do that". Typically lockdown
++ * *will* be applied before the debug core gets started
++ * so only developers using kgdb for fairly advanced
++ * early kernel debug can be biten by this. Hopefully
++ * they are sophisticated enough to take care of
++ * themselves, especially with help from the lockdown
++ * message printed on the console!
++ */
++ if (security_locked_down(LOCKDOWN_DBG_WRITE_KERNEL)) {
++ if (IS_ENABLED(CONFIG_KGDB_KDB)) {
++ /* Switch back to kdb if possible... */
++ dbg_kdb_mode = 1;
++ continue;
++ } else {
++ /* ... otherwise just bail */
++ break;
++ }
++ }
+ error = gdb_serial_stub(ks);
+ }
+
+diff --git a/kernel/debug/kdb/kdb_bt.c b/kernel/debug/kdb/kdb_bt.c
+index 1f9f0e47aedaa..10b454554ab03 100644
+--- a/kernel/debug/kdb/kdb_bt.c
++++ b/kernel/debug/kdb/kdb_bt.c
+@@ -46,7 +46,7 @@ static void kdb_show_stack(struct task_struct *p, void *addr)
+ * btp <pid> Kernel stack for <pid>
+ * btt <address-expression> Kernel stack for task structure at
+ * <address-expression>
+- * bta [DRSTCZEUIMA] All useful processes, optionally
++ * bta [state_chars>|A] All useful processes, optionally
+ * filtered by state
+ * btc [<cpu>] The current process on one cpu,
+ * default is all cpus
+@@ -74,7 +74,7 @@ static void kdb_show_stack(struct task_struct *p, void *addr)
+ */
+
+ static int
+-kdb_bt1(struct task_struct *p, unsigned long mask, bool btaprompt)
++kdb_bt1(struct task_struct *p, const char *mask, bool btaprompt)
+ {
+ char ch;
+
+@@ -120,7 +120,7 @@ kdb_bt_cpu(unsigned long cpu)
+ return;
+ }
+
+- kdb_bt1(kdb_tsk, ~0UL, false);
++ kdb_bt1(kdb_tsk, "A", false);
+ }
+
+ int
+@@ -138,8 +138,8 @@ kdb_bt(int argc, const char **argv)
+ if (strcmp(argv[0], "bta") == 0) {
+ struct task_struct *g, *p;
+ unsigned long cpu;
+- unsigned long mask = kdb_task_state_string(argc ? argv[1] :
+- NULL);
++ const char *mask = argc ? argv[1] : kdbgetenv("PS");
++
+ if (argc == 0)
+ kdb_ps_suppressed();
+ /* Run the active tasks first */
+@@ -167,7 +167,7 @@ kdb_bt(int argc, const char **argv)
+ return diag;
+ p = find_task_by_pid_ns(pid, &init_pid_ns);
+ if (p)
+- return kdb_bt1(p, ~0UL, false);
++ return kdb_bt1(p, "A", false);
+ kdb_printf("No process with pid == %ld found\n", pid);
+ return 0;
+ } else if (strcmp(argv[0], "btt") == 0) {
+@@ -176,7 +176,7 @@ kdb_bt(int argc, const char **argv)
+ diag = kdbgetularg((char *)argv[1], &addr);
+ if (diag)
+ return diag;
+- return kdb_bt1((struct task_struct *)addr, ~0UL, false);
++ return kdb_bt1((struct task_struct *)addr, "A", false);
+ } else if (strcmp(argv[0], "btc") == 0) {
+ unsigned long cpu = ~0;
+ if (argc > 1)
+@@ -212,7 +212,7 @@ kdb_bt(int argc, const char **argv)
+ kdb_show_stack(kdb_current_task, (void *)addr);
+ return 0;
+ } else {
+- return kdb_bt1(kdb_current_task, ~0UL, false);
++ return kdb_bt1(kdb_current_task, "A", false);
+ }
+ }
+
+diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c
+index fa6deda894a17..ead4da9471270 100644
+--- a/kernel/debug/kdb/kdb_main.c
++++ b/kernel/debug/kdb/kdb_main.c
+@@ -45,6 +45,7 @@
+ #include <linux/proc_fs.h>
+ #include <linux/uaccess.h>
+ #include <linux/slab.h>
++#include <linux/security.h>
+ #include "kdb_private.h"
+
+ #undef MODULE_PARAM_PREFIX
+@@ -166,10 +167,62 @@ struct task_struct *kdb_curr_task(int cpu)
+ }
+
+ /*
+- * Check whether the flags of the current command and the permissions
+- * of the kdb console has allow a command to be run.
++ * Update the permissions flags (kdb_cmd_enabled) to match the
++ * current lockdown state.
++ *
++ * Within this function the calls to security_locked_down() are "lazy". We
++ * avoid calling them if the current value of kdb_cmd_enabled already excludes
++ * flags that might be subject to lockdown. Additionally we deliberately check
++ * the lockdown flags independently (even though read lockdown implies write
++ * lockdown) since that results in both simpler code and clearer messages to
++ * the user on first-time debugger entry.
++ *
++ * The permission masks during a read+write lockdown permits the following
++ * flags: INSPECT, SIGNAL, REBOOT (and ALWAYS_SAFE).
++ *
++ * The INSPECT commands are not blocked during lockdown because they are
++ * not arbitrary memory reads. INSPECT covers the backtrace family (sometimes
++ * forcing them to have no arguments) and lsmod. These commands do expose
++ * some kernel state but do not allow the developer seated at the console to
++ * choose what state is reported. SIGNAL and REBOOT should not be controversial,
++ * given these are allowed for root during lockdown already.
++ */
++static void kdb_check_for_lockdown(void)
++{
++ const int write_flags = KDB_ENABLE_MEM_WRITE |
++ KDB_ENABLE_REG_WRITE |
++ KDB_ENABLE_FLOW_CTRL;
++ const int read_flags = KDB_ENABLE_MEM_READ |
++ KDB_ENABLE_REG_READ;
++
++ bool need_to_lockdown_write = false;
++ bool need_to_lockdown_read = false;
++
++ if (kdb_cmd_enabled & (KDB_ENABLE_ALL | write_flags))
++ need_to_lockdown_write =
++ security_locked_down(LOCKDOWN_DBG_WRITE_KERNEL);
++
++ if (kdb_cmd_enabled & (KDB_ENABLE_ALL | read_flags))
++ need_to_lockdown_read =
++ security_locked_down(LOCKDOWN_DBG_READ_KERNEL);
++
++ /* De-compose KDB_ENABLE_ALL if required */
++ if (need_to_lockdown_write || need_to_lockdown_read)
++ if (kdb_cmd_enabled & KDB_ENABLE_ALL)
++ kdb_cmd_enabled = KDB_ENABLE_MASK & ~KDB_ENABLE_ALL;
++
++ if (need_to_lockdown_write)
++ kdb_cmd_enabled &= ~write_flags;
++
++ if (need_to_lockdown_read)
++ kdb_cmd_enabled &= ~read_flags;
++}
++
++/*
++ * Check whether the flags of the current command, the permissions of the kdb
++ * console and the lockdown state allow a command to be run.
+ */
+-static inline bool kdb_check_flags(kdb_cmdflags_t flags, int permissions,
++static bool kdb_check_flags(kdb_cmdflags_t flags, int permissions,
+ bool no_args)
+ {
+ /* permissions comes from userspace so needs massaging slightly */
+@@ -1180,6 +1233,9 @@ static int kdb_local(kdb_reason_t reason, int error, struct pt_regs *regs,
+ kdb_curr_task(raw_smp_processor_id());
+
+ KDB_DEBUG_STATE("kdb_local 1", reason);
++
++ kdb_check_for_lockdown();
++
+ kdb_go_count = 0;
+ if (reason == KDB_REASON_DEBUG) {
+ /* special case below */
+@@ -2203,8 +2259,8 @@ static void kdb_cpu_status(void)
+ state = 'D'; /* cpu is online but unresponsive */
+ } else {
+ state = ' '; /* cpu is responding to kdb */
+- if (kdb_task_state_char(KDB_TSK(i)) == 'I')
+- state = 'I'; /* idle task */
++ if (kdb_task_state_char(KDB_TSK(i)) == '-')
++ state = '-'; /* idle task */
+ }
+ if (state != prev_state) {
+ if (prev_state != '?') {
+@@ -2271,37 +2327,30 @@ static int kdb_cpu(int argc, const char **argv)
+ void kdb_ps_suppressed(void)
+ {
+ int idle = 0, daemon = 0;
+- unsigned long mask_I = kdb_task_state_string("I"),
+- mask_M = kdb_task_state_string("M");
+ unsigned long cpu;
+ const struct task_struct *p, *g;
+ for_each_online_cpu(cpu) {
+ p = kdb_curr_task(cpu);
+- if (kdb_task_state(p, mask_I))
++ if (kdb_task_state(p, "-"))
+ ++idle;
+ }
+ for_each_process_thread(g, p) {
+- if (kdb_task_state(p, mask_M))
++ if (kdb_task_state(p, "ims"))
+ ++daemon;
+ }
+ if (idle || daemon) {
+ if (idle)
+- kdb_printf("%d idle process%s (state I)%s\n",
++ kdb_printf("%d idle process%s (state -)%s\n",
+ idle, idle == 1 ? "" : "es",
+ daemon ? " and " : "");
+ if (daemon)
+- kdb_printf("%d sleeping system daemon (state M) "
++ kdb_printf("%d sleeping system daemon (state [ims]) "
+ "process%s", daemon,
+ daemon == 1 ? "" : "es");
+ kdb_printf(" suppressed,\nuse 'ps A' to see all.\n");
+ }
+ }
+
+-/*
+- * kdb_ps - This function implements the 'ps' command which shows a
+- * list of the active processes.
+- * ps [DRSTCZEUIMA] All processes, optionally filtered by state
+- */
+ void kdb_ps1(const struct task_struct *p)
+ {
+ int cpu;
+@@ -2330,17 +2379,25 @@ void kdb_ps1(const struct task_struct *p)
+ }
+ }
+
++/*
++ * kdb_ps - This function implements the 'ps' command which shows a
++ * list of the active processes.
++ *
++ * ps [<state_chars>] Show processes, optionally selecting only those whose
++ * state character is found in <state_chars>.
++ */
+ static int kdb_ps(int argc, const char **argv)
+ {
+ struct task_struct *g, *p;
+- unsigned long mask, cpu;
++ const char *mask;
++ unsigned long cpu;
+
+ if (argc == 0)
+ kdb_ps_suppressed();
+ kdb_printf("%-*s Pid Parent [*] cpu State %-*s Command\n",
+ (int)(2*sizeof(void *))+2, "Task Addr",
+ (int)(2*sizeof(void *))+2, "Thread");
+- mask = kdb_task_state_string(argc ? argv[1] : NULL);
++ mask = argc ? argv[1] : kdbgetenv("PS");
+ /* Run the active tasks first */
+ for_each_online_cpu(cpu) {
+ if (KDB_FLAG(CMD_INTERRUPT))
+@@ -2742,8 +2799,8 @@ static kdbtab_t maintab[] = {
+ },
+ { .name = "bta",
+ .func = kdb_bt,
+- .usage = "[D|R|S|T|C|Z|E|U|I|M|A]",
+- .help = "Backtrace all processes matching state flag",
++ .usage = "[<state_chars>|A]",
++ .help = "Backtrace all processes whose state matches",
+ .flags = KDB_ENABLE_INSPECT,
+ },
+ { .name = "btc",
+@@ -2797,7 +2854,7 @@ static kdbtab_t maintab[] = {
+ },
+ { .name = "ps",
+ .func = kdb_ps,
+- .usage = "[<flags>|A]",
++ .usage = "[<state_chars>|A]",
+ .help = "Display active task list",
+ .flags = KDB_ENABLE_INSPECT,
+ },
+diff --git a/kernel/debug/kdb/kdb_private.h b/kernel/debug/kdb/kdb_private.h
+index 629590084a0dc..0d2f9feea0a46 100644
+--- a/kernel/debug/kdb/kdb_private.h
++++ b/kernel/debug/kdb/kdb_private.h
+@@ -190,10 +190,8 @@ extern char kdb_grep_string[];
+ extern int kdb_grep_leading;
+ extern int kdb_grep_trailing;
+ extern char *kdb_cmds[];
+-extern unsigned long kdb_task_state_string(const char *);
+ extern char kdb_task_state_char (const struct task_struct *);
+-extern unsigned long kdb_task_state(const struct task_struct *p,
+- unsigned long mask);
++extern bool kdb_task_state(const struct task_struct *p, const char *mask);
+ extern void kdb_ps_suppressed(void);
+ extern void kdb_ps1(const struct task_struct *p);
+ extern void kdb_send_sig(struct task_struct *p, int sig);
+diff --git a/kernel/debug/kdb/kdb_support.c b/kernel/debug/kdb/kdb_support.c
+index 7507d9a8dc6ac..85cb51c4a17e6 100644
+--- a/kernel/debug/kdb/kdb_support.c
++++ b/kernel/debug/kdb/kdb_support.c
+@@ -24,6 +24,7 @@
+ #include <linux/uaccess.h>
+ #include <linux/kdb.h>
+ #include <linux/slab.h>
++#include <linux/ctype.h>
+ #include "kdb_private.h"
+
+ /*
+@@ -290,7 +291,7 @@ int kdb_getarea_size(void *res, unsigned long addr, size_t size)
+ */
+ int kdb_putarea_size(unsigned long addr, void *res, size_t size)
+ {
+- int ret = copy_from_kernel_nofault((char *)addr, (char *)res, size);
++ int ret = copy_to_kernel_nofault((char *)addr, (char *)res, size);
+ if (ret) {
+ if (!KDB_STATE(SUPPRESS)) {
+ kdb_func_printf("Bad address 0x%lx\n", addr);
+@@ -473,82 +474,7 @@ int kdb_putword(unsigned long addr, unsigned long word, size_t size)
+ return diag;
+ }
+
+-/*
+- * kdb_task_state_string - Convert a string containing any of the
+- * letters DRSTCZEUIMA to a mask for the process state field and
+- * return the value. If no argument is supplied, return the mask
+- * that corresponds to environment variable PS, DRSTCZEU by
+- * default.
+- * Inputs:
+- * s String to convert
+- * Returns:
+- * Mask for process state.
+- * Notes:
+- * The mask folds data from several sources into a single long value, so
+- * be careful not to overlap the bits. TASK_* bits are in the LSB,
+- * special cases like UNRUNNABLE are in the MSB. As of 2.6.10-rc1 there
+- * is no overlap between TASK_* and EXIT_* but that may not always be
+- * true, so EXIT_* bits are shifted left 16 bits before being stored in
+- * the mask.
+- */
+-
+-/* unrunnable is < 0 */
+-#define UNRUNNABLE (1UL << (8*sizeof(unsigned long) - 1))
+-#define RUNNING (1UL << (8*sizeof(unsigned long) - 2))
+-#define IDLE (1UL << (8*sizeof(unsigned long) - 3))
+-#define DAEMON (1UL << (8*sizeof(unsigned long) - 4))
+
+-unsigned long kdb_task_state_string(const char *s)
+-{
+- long res = 0;
+- if (!s) {
+- s = kdbgetenv("PS");
+- if (!s)
+- s = "DRSTCZEU"; /* default value for ps */
+- }
+- while (*s) {
+- switch (*s) {
+- case 'D':
+- res |= TASK_UNINTERRUPTIBLE;
+- break;
+- case 'R':
+- res |= RUNNING;
+- break;
+- case 'S':
+- res |= TASK_INTERRUPTIBLE;
+- break;
+- case 'T':
+- res |= TASK_STOPPED;
+- break;
+- case 'C':
+- res |= TASK_TRACED;
+- break;
+- case 'Z':
+- res |= EXIT_ZOMBIE << 16;
+- break;
+- case 'E':
+- res |= EXIT_DEAD << 16;
+- break;
+- case 'U':
+- res |= UNRUNNABLE;
+- break;
+- case 'I':
+- res |= IDLE;
+- break;
+- case 'M':
+- res |= DAEMON;
+- break;
+- case 'A':
+- res = ~0UL;
+- break;
+- default:
+- kdb_func_printf("unknown flag '%c' ignored\n", *s);
+- break;
+- }
+- ++s;
+- }
+- return res;
+-}
+
+ /*
+ * kdb_task_state_char - Return the character that represents the task state.
+@@ -559,7 +485,6 @@ unsigned long kdb_task_state_string(const char *s)
+ */
+ char kdb_task_state_char (const struct task_struct *p)
+ {
+- unsigned int p_state;
+ unsigned long tmp;
+ char state;
+ int cpu;
+@@ -568,25 +493,18 @@ char kdb_task_state_char (const struct task_struct *p)
+ copy_from_kernel_nofault(&tmp, (char *)p, sizeof(unsigned long)))
+ return 'E';
+
+- cpu = kdb_process_cpu(p);
+- p_state = READ_ONCE(p->__state);
+- state = (p_state == 0) ? 'R' :
+- (p_state < 0) ? 'U' :
+- (p_state & TASK_UNINTERRUPTIBLE) ? 'D' :
+- (p_state & TASK_STOPPED) ? 'T' :
+- (p_state & TASK_TRACED) ? 'C' :
+- (p->exit_state & EXIT_ZOMBIE) ? 'Z' :
+- (p->exit_state & EXIT_DEAD) ? 'E' :
+- (p_state & TASK_INTERRUPTIBLE) ? 'S' : '?';
++ state = task_state_to_char((struct task_struct *) p);
++
+ if (is_idle_task(p)) {
+ /* Idle task. Is it really idle, apart from the kdb
+ * interrupt? */
++ cpu = kdb_process_cpu(p);
+ if (!kdb_task_has_cpu(p) || kgdb_info[cpu].irq_depth == 1) {
+ if (cpu != kdb_initial_cpu)
+- state = 'I'; /* idle task */
++ state = '-'; /* idle task */
+ }
+- } else if (!p->mm && state == 'S') {
+- state = 'M'; /* sleeping system daemon */
++ } else if (!p->mm && strchr("IMS", state)) {
++ state = tolower(state); /* sleeping system daemon */
+ }
+ return state;
+ }
+@@ -596,14 +514,28 @@ char kdb_task_state_char (const struct task_struct *p)
+ * given by the mask.
+ * Inputs:
+ * p struct task for the process
+- * mask mask from kdb_task_state_string to select processes
++ * mask set of characters used to select processes; both NULL
++ * and the empty string mean adopt a default filter, which
++ * is to suppress sleeping system daemons and the idle tasks
+ * Returns:
+ * True if the process matches at least one criteria defined by the mask.
+ */
+-unsigned long kdb_task_state(const struct task_struct *p, unsigned long mask)
++bool kdb_task_state(const struct task_struct *p, const char *mask)
+ {
+- char state[] = { kdb_task_state_char(p), '\0' };
+- return (mask & kdb_task_state_string(state)) != 0;
++ char state = kdb_task_state_char(p);
++
++ /* If there is no mask, then we will filter code that runs when the
++ * scheduler is idling and any system daemons that are currently
++ * sleeping.
++ */
++ if (!mask || mask[0] == '\0')
++ return !strchr("-ims", state);
++
++ /* A is a special case that matches all states */
++ if (strchr(mask, 'A'))
++ return true;
++
++ return strchr(mask, state);
+ }
+
+ /* Maintain a small stack of kdb_flags to allow recursion without disturbing
+diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
+index 7a14ca29c3778..f8ff598596b85 100644
+--- a/kernel/dma/debug.c
++++ b/kernel/dma/debug.c
+@@ -927,7 +927,7 @@ static __init int dma_debug_cmdline(char *str)
+ global_disable = true;
+ }
+
+- return 0;
++ return 1;
+ }
+
+ static __init int dma_debug_entries_cmdline(char *str)
+@@ -936,7 +936,7 @@ static __init int dma_debug_entries_cmdline(char *str)
+ return -EINVAL;
+ if (!get_option(&str, &nr_prealloc_entries))
+ nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
+- return 0;
++ return 1;
+ }
+
+ __setup("dma_debug=", dma_debug_cmdline);
+diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
+index 4632b0f4f72eb..8a6cd53dbe8ce 100644
+--- a/kernel/dma/direct.h
++++ b/kernel/dma/direct.h
+@@ -114,6 +114,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
+ dma_direct_sync_single_for_cpu(dev, addr, size, dir);
+
+ if (unlikely(is_swiotlb_buffer(dev, phys)))
+- swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
++ swiotlb_tbl_unmap_single(dev, phys, size, dir,
++ attrs | DMA_ATTR_SKIP_CPU_SYNC);
+ }
+ #endif /* _KERNEL_DMA_DIRECT_H */
+diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
+index 8349a9f2c3453..9478eccd1c8e6 100644
+--- a/kernel/dma/mapping.c
++++ b/kernel/dma/mapping.c
+@@ -296,10 +296,6 @@ dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
+ if (WARN_ON_ONCE(!dev->dma_mask))
+ return DMA_MAPPING_ERROR;
+
+- /* Don't allow RAM to be mapped */
+- if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
+- return DMA_MAPPING_ERROR;
+-
+ if (dma_map_direct(dev, ops))
+ addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
+ else if (ops->map_resource)
+diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
+index 5f84e6cdb78ea..4d40dcce7604b 100644
+--- a/kernel/dma/pool.c
++++ b/kernel/dma/pool.c
+@@ -203,7 +203,7 @@ static int __init dma_atomic_pool_init(void)
+ GFP_KERNEL);
+ if (!atomic_pool_kernel)
+ ret = -ENOMEM;
+- if (IS_ENABLED(CONFIG_ZONE_DMA)) {
++ if (has_managed_dma()) {
+ atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
+ GFP_KERNEL | GFP_DMA);
+ if (!atomic_pool_dma)
+@@ -226,7 +226,7 @@ static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+ if (prev == NULL) {
+ if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32))
+ return atomic_pool_dma32;
+- if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA))
++ if (atomic_pool_dma && (gfp & GFP_DMA))
+ return atomic_pool_dma;
+ return atomic_pool_kernel;
+ }
+diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
+index 87c40517e8227..2ee5419649ed7 100644
+--- a/kernel/dma/swiotlb.c
++++ b/kernel/dma/swiotlb.c
+@@ -459,7 +459,7 @@ static unsigned int wrap_index(struct io_tlb_mem *mem, unsigned int index)
+ * allocate a buffer from that IO TLB pool.
+ */
+ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
+- size_t alloc_size)
++ size_t alloc_size, unsigned int alloc_align_mask)
+ {
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ unsigned long boundary_mask = dma_get_seg_boundary(dev);
+@@ -483,6 +483,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
+ stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
+ if (alloc_size >= PAGE_SIZE)
+ stride = max(stride, stride << (PAGE_SHIFT - IO_TLB_SHIFT));
++ stride = max(stride, (alloc_align_mask >> IO_TLB_SHIFT) + 1);
+
+ spin_lock_irqsave(&mem->lock, flags);
+ if (unlikely(nslots > mem->nslabs - mem->used))
+@@ -541,7 +542,8 @@ found:
+
+ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
+ size_t mapping_size, size_t alloc_size,
+- enum dma_data_direction dir, unsigned long attrs)
++ unsigned int alloc_align_mask, enum dma_data_direction dir,
++ unsigned long attrs)
+ {
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+ unsigned int offset = swiotlb_align_offset(dev, orig_addr);
+@@ -561,7 +563,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+ }
+
+- index = swiotlb_find_slots(dev, orig_addr, alloc_size + offset);
++ index = swiotlb_find_slots(dev, orig_addr,
++ alloc_size + offset, alloc_align_mask);
+ if (index == -1) {
+ if (!(attrs & DMA_ATTR_NO_WARN))
+ dev_warn_ratelimited(dev,
+@@ -578,9 +581,14 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
+ for (i = 0; i < nr_slots(alloc_size + offset); i++)
+ mem->slots[index + i].orig_addr = slot_addr(orig_addr, i);
+ tlb_addr = slot_addr(mem->start, index) + offset;
+- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+- (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
+- swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
++ /*
++ * When dir == DMA_FROM_DEVICE we could omit the copy from the orig
++ * to the tlb buffer, if we knew for sure the device will
++ * overwirte the entire current content. But we don't. Thus
++ * unconditional bounce may prevent leaking swiotlb content (i.e.
++ * kernel memory) to user-space.
++ */
++ swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
+ return tlb_addr;
+ }
+
+@@ -675,7 +683,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
+ trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size,
+ swiotlb_force);
+
+- swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, dir,
++ swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir,
+ attrs);
+ if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
+ return DMA_MAPPING_ERROR;
+@@ -759,7 +767,7 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
+ if (!mem)
+ return NULL;
+
+- index = swiotlb_find_slots(dev, 0, size);
++ index = swiotlb_find_slots(dev, 0, size, 0);
+ if (index == -1)
+ return NULL;
+
+diff --git a/kernel/entry/syscall_user_dispatch.c b/kernel/entry/syscall_user_dispatch.c
+index c240302f56e23..0b6379adff6bd 100644
+--- a/kernel/entry/syscall_user_dispatch.c
++++ b/kernel/entry/syscall_user_dispatch.c
+@@ -47,14 +47,18 @@ bool syscall_user_dispatch(struct pt_regs *regs)
+ * access_ok() is performed once, at prctl time, when
+ * the selector is loaded by userspace.
+ */
+- if (unlikely(__get_user(state, sd->selector)))
+- do_exit(SIGSEGV);
++ if (unlikely(__get_user(state, sd->selector))) {
++ force_exit_sig(SIGSEGV);
++ return true;
++ }
+
+ if (likely(state == SYSCALL_DISPATCH_FILTER_ALLOW))
+ return false;
+
+- if (state != SYSCALL_DISPATCH_FILTER_BLOCK)
+- do_exit(SIGSYS);
++ if (state != SYSCALL_DISPATCH_FILTER_BLOCK) {
++ force_exit_sig(SIGSYS);
++ return true;
++ }
+ }
+
+ sd->on_dispatch = true;
+diff --git a/kernel/events/core.c b/kernel/events/core.c
+index f23ca260307f0..565910de92e9b 100644
+--- a/kernel/events/core.c
++++ b/kernel/events/core.c
+@@ -674,6 +674,23 @@ perf_event_set_state(struct perf_event *event, enum perf_event_state state)
+ WRITE_ONCE(event->state, state);
+ }
+
++/*
++ * UP store-release, load-acquire
++ */
++
++#define __store_release(ptr, val) \
++do { \
++ barrier(); \
++ WRITE_ONCE(*(ptr), (val)); \
++} while (0)
++
++#define __load_acquire(ptr) \
++({ \
++ __unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \
++ barrier(); \
++ ___p; \
++})
++
+ #ifdef CONFIG_CGROUP_PERF
+
+ static inline bool
+@@ -719,34 +736,51 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event)
+ return t->time;
+ }
+
+-static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
++static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
+ {
+- struct perf_cgroup_info *info;
+- u64 now;
+-
+- now = perf_clock();
++ struct perf_cgroup_info *t;
+
+- info = this_cpu_ptr(cgrp->info);
++ t = per_cpu_ptr(event->cgrp->info, event->cpu);
++ if (!__load_acquire(&t->active))
++ return t->time;
++ now += READ_ONCE(t->timeoffset);
++ return now;
++}
+
+- info->time += now - info->timestamp;
++static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
++{
++ if (adv)
++ info->time += now - info->timestamp;
+ info->timestamp = now;
++ /*
++ * see update_context_time()
++ */
++ WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
+ }
+
+-static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
++static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
+ {
+ struct perf_cgroup *cgrp = cpuctx->cgrp;
+ struct cgroup_subsys_state *css;
++ struct perf_cgroup_info *info;
+
+ if (cgrp) {
++ u64 now = perf_clock();
++
+ for (css = &cgrp->css; css; css = css->parent) {
+ cgrp = container_of(css, struct perf_cgroup, css);
+- __update_cgrp_time(cgrp);
++ info = this_cpu_ptr(cgrp->info);
++
++ __update_cgrp_time(info, now, true);
++ if (final)
++ __store_release(&info->active, 0);
+ }
+ }
+ }
+
+ static inline void update_cgrp_time_from_event(struct perf_event *event)
+ {
++ struct perf_cgroup_info *info;
+ struct perf_cgroup *cgrp;
+
+ /*
+@@ -760,8 +794,10 @@ static inline void update_cgrp_time_from_event(struct perf_event *event)
+ /*
+ * Do not update time when cgroup is not active
+ */
+- if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
+- __update_cgrp_time(event->cgrp);
++ if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) {
++ info = this_cpu_ptr(event->cgrp->info);
++ __update_cgrp_time(info, perf_clock(), true);
++ }
+ }
+
+ static inline void
+@@ -785,7 +821,8 @@ perf_cgroup_set_timestamp(struct task_struct *task,
+ for (css = &cgrp->css; css; css = css->parent) {
+ cgrp = container_of(css, struct perf_cgroup, css);
+ info = this_cpu_ptr(cgrp->info);
+- info->timestamp = ctx->timestamp;
++ __update_cgrp_time(info, ctx->timestamp, false);
++ __store_release(&info->active, 1);
+ }
+ }
+
+@@ -802,7 +839,7 @@ static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list);
+ */
+ static void perf_cgroup_switch(struct task_struct *task, int mode)
+ {
+- struct perf_cpu_context *cpuctx;
++ struct perf_cpu_context *cpuctx, *tmp;
+ struct list_head *list;
+ unsigned long flags;
+
+@@ -813,7 +850,7 @@ static void perf_cgroup_switch(struct task_struct *task, int mode)
+ local_irq_save(flags);
+
+ list = this_cpu_ptr(&cgrp_cpuctx_list);
+- list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) {
++ list_for_each_entry_safe(cpuctx, tmp, list, cgrp_cpuctx_entry) {
+ WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
+
+ perf_ctx_lock(cpuctx, cpuctx->task_ctx);
+@@ -981,14 +1018,6 @@ out:
+ return ret;
+ }
+
+-static inline void
+-perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
+-{
+- struct perf_cgroup_info *t;
+- t = per_cpu_ptr(event->cgrp->info, event->cpu);
+- event->shadow_ctx_time = now - t->timestamp;
+-}
+-
+ static inline void
+ perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
+ {
+@@ -1066,7 +1095,8 @@ static inline void update_cgrp_time_from_event(struct perf_event *event)
+ {
+ }
+
+-static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
++static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
++ bool final)
+ {
+ }
+
+@@ -1098,12 +1128,12 @@ perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
+ {
+ }
+
+-static inline void
+-perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
++static inline u64 perf_cgroup_event_time(struct perf_event *event)
+ {
++ return 0;
+ }
+
+-static inline u64 perf_cgroup_event_time(struct perf_event *event)
++static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
+ {
+ return 0;
+ }
+@@ -1525,22 +1555,59 @@ static void perf_unpin_context(struct perf_event_context *ctx)
+ /*
+ * Update the record of the current time in a context.
+ */
+-static void update_context_time(struct perf_event_context *ctx)
++static void __update_context_time(struct perf_event_context *ctx, bool adv)
+ {
+ u64 now = perf_clock();
+
+- ctx->time += now - ctx->timestamp;
++ if (adv)
++ ctx->time += now - ctx->timestamp;
+ ctx->timestamp = now;
++
++ /*
++ * The above: time' = time + (now - timestamp), can be re-arranged
++ * into: time` = now + (time - timestamp), which gives a single value
++ * offset to compute future time without locks on.
++ *
++ * See perf_event_time_now(), which can be used from NMI context where
++ * it's (obviously) not possible to acquire ctx->lock in order to read
++ * both the above values in a consistent manner.
++ */
++ WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
++}
++
++static void update_context_time(struct perf_event_context *ctx)
++{
++ __update_context_time(ctx, true);
+ }
+
+ static u64 perf_event_time(struct perf_event *event)
+ {
+ struct perf_event_context *ctx = event->ctx;
+
++ if (unlikely(!ctx))
++ return 0;
++
+ if (is_cgroup_event(event))
+ return perf_cgroup_event_time(event);
+
+- return ctx ? ctx->time : 0;
++ return ctx->time;
++}
++
++static u64 perf_event_time_now(struct perf_event *event, u64 now)
++{
++ struct perf_event_context *ctx = event->ctx;
++
++ if (unlikely(!ctx))
++ return 0;
++
++ if (is_cgroup_event(event))
++ return perf_cgroup_event_time_now(event, now);
++
++ if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
++ return ctx->time;
++
++ now += READ_ONCE(ctx->timeoffset);
++ return now;
+ }
+
+ static enum event_type_t get_event_type(struct perf_event *event)
+@@ -2346,7 +2413,7 @@ __perf_remove_from_context(struct perf_event *event,
+
+ if (ctx->is_active & EVENT_TIME) {
+ update_context_time(ctx);
+- update_cgrp_time_from_cpuctx(cpuctx);
++ update_cgrp_time_from_cpuctx(cpuctx, false);
+ }
+
+ event_sched_out(event, cpuctx, ctx);
+@@ -2357,6 +2424,9 @@ __perf_remove_from_context(struct perf_event *event,
+ list_del_event(event, ctx);
+
+ if (!ctx->nr_events && ctx->is_active) {
++ if (ctx == &cpuctx->ctx)
++ update_cgrp_time_from_cpuctx(cpuctx, true);
++
+ ctx->is_active = 0;
+ ctx->rotate_necessary = 0;
+ if (ctx->task) {
+@@ -2388,7 +2458,11 @@ static void perf_remove_from_context(struct perf_event *event, unsigned long fla
+ * event_function_call() user.
+ */
+ raw_spin_lock_irq(&ctx->lock);
+- if (!ctx->is_active) {
++ /*
++ * Cgroup events are per-cpu events, and must IPI because of
++ * cgrp_cpuctx_list.
++ */
++ if (!ctx->is_active && !is_cgroup_event(event)) {
+ __perf_remove_from_context(event, __get_cpu_context(ctx),
+ ctx, (void *)flags);
+ raw_spin_unlock_irq(&ctx->lock);
+@@ -2478,40 +2552,6 @@ void perf_event_disable_inatomic(struct perf_event *event)
+ irq_work_queue(&event->pending);
+ }
+
+-static void perf_set_shadow_time(struct perf_event *event,
+- struct perf_event_context *ctx)
+-{
+- /*
+- * use the correct time source for the time snapshot
+- *
+- * We could get by without this by leveraging the
+- * fact that to get to this function, the caller
+- * has most likely already called update_context_time()
+- * and update_cgrp_time_xx() and thus both timestamp
+- * are identical (or very close). Given that tstamp is,
+- * already adjusted for cgroup, we could say that:
+- * tstamp - ctx->timestamp
+- * is equivalent to
+- * tstamp - cgrp->timestamp.
+- *
+- * Then, in perf_output_read(), the calculation would
+- * work with no changes because:
+- * - event is guaranteed scheduled in
+- * - no scheduled out in between
+- * - thus the timestamp would be the same
+- *
+- * But this is a bit hairy.
+- *
+- * So instead, we have an explicit cgroup call to remain
+- * within the time source all along. We believe it
+- * is cleaner and simpler to understand.
+- */
+- if (is_cgroup_event(event))
+- perf_cgroup_set_shadow_time(event, event->tstamp);
+- else
+- event->shadow_ctx_time = event->tstamp - ctx->timestamp;
+-}
+-
+ #define MAX_INTERRUPTS (~0ULL)
+
+ static void perf_log_throttle(struct perf_event *event, int enable);
+@@ -2552,8 +2592,6 @@ event_sched_in(struct perf_event *event,
+
+ perf_pmu_disable(event->pmu);
+
+- perf_set_shadow_time(event, ctx);
+-
+ perf_log_itrace_start(event);
+
+ if (event->pmu->add(event, PERF_EF_START)) {
+@@ -2857,11 +2895,14 @@ perf_install_in_context(struct perf_event_context *ctx,
+ * perf_event_attr::disabled events will not run and can be initialized
+ * without IPI. Except when this is the first event for the context, in
+ * that case we need the magic of the IPI to set ctx->is_active.
++ * Similarly, cgroup events for the context also needs the IPI to
++ * manipulate the cgrp_cpuctx_list.
+ *
+ * The IOC_ENABLE that is sure to follow the creation of a disabled
+ * event will issue the IPI and reprogram the hardware.
+ */
+- if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && ctx->nr_events) {
++ if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
++ ctx->nr_events && !is_cgroup_event(event)) {
+ raw_spin_lock_irq(&ctx->lock);
+ if (ctx->task == TASK_TOMBSTONE) {
+ raw_spin_unlock_irq(&ctx->lock);
+@@ -3193,6 +3234,15 @@ static int perf_event_modify_breakpoint(struct perf_event *bp,
+ return err;
+ }
+
++/*
++ * Copy event-type-independent attributes that may be modified.
++ */
++static void perf_event_modify_copy_attr(struct perf_event_attr *to,
++ const struct perf_event_attr *from)
++{
++ to->sig_data = from->sig_data;
++}
++
+ static int perf_event_modify_attr(struct perf_event *event,
+ struct perf_event_attr *attr)
+ {
+@@ -3215,10 +3265,17 @@ static int perf_event_modify_attr(struct perf_event *event,
+ WARN_ON_ONCE(event->ctx->parent_ctx);
+
+ mutex_lock(&event->child_mutex);
++ /*
++ * Event-type-independent attributes must be copied before event-type
++ * modification, which will validate that final attributes match the
++ * source attributes after all relevant attributes have been copied.
++ */
++ perf_event_modify_copy_attr(&event->attr, attr);
+ err = func(event, attr);
+ if (err)
+ goto out;
+ list_for_each_entry(child, &event->child_list, child_list) {
++ perf_event_modify_copy_attr(&child->attr, attr);
+ err = func(child, attr);
+ if (err)
+ goto out;
+@@ -3247,16 +3304,6 @@ static void ctx_sched_out(struct perf_event_context *ctx,
+ return;
+ }
+
+- ctx->is_active &= ~event_type;
+- if (!(ctx->is_active & EVENT_ALL))
+- ctx->is_active = 0;
+-
+- if (ctx->task) {
+- WARN_ON_ONCE(cpuctx->task_ctx != ctx);
+- if (!ctx->is_active)
+- cpuctx->task_ctx = NULL;
+- }
+-
+ /*
+ * Always update time if it was set; not only when it changes.
+ * Otherwise we can 'forget' to update time for any but the last
+@@ -3270,7 +3317,22 @@ static void ctx_sched_out(struct perf_event_context *ctx,
+ if (is_active & EVENT_TIME) {
+ /* update (and stop) ctx time */
+ update_context_time(ctx);
+- update_cgrp_time_from_cpuctx(cpuctx);
++ update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx);
++ /*
++ * CPU-release for the below ->is_active store,
++ * see __load_acquire() in perf_event_time_now()
++ */
++ barrier();
++ }
++
++ ctx->is_active &= ~event_type;
++ if (!(ctx->is_active & EVENT_ALL))
++ ctx->is_active = 0;
++
++ if (ctx->task) {
++ WARN_ON_ONCE(cpuctx->task_ctx != ctx);
++ if (!ctx->is_active)
++ cpuctx->task_ctx = NULL;
+ }
+
+ is_active ^= ctx->is_active; /* changed bits */
+@@ -3707,13 +3769,19 @@ static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx,
+ return 0;
+ }
+
++/*
++ * Because the userpage is strictly per-event (there is no concept of context,
++ * so there cannot be a context indirection), every userpage must be updated
++ * when context time starts :-(
++ *
++ * IOW, we must not miss EVENT_TIME edges.
++ */
+ static inline bool event_update_userpage(struct perf_event *event)
+ {
+ if (likely(!atomic_read(&event->mmap_count)))
+ return false;
+
+ perf_event_update_time(event);
+- perf_set_shadow_time(event, event->ctx);
+ perf_event_update_userpage(event);
+
+ return true;
+@@ -3797,13 +3865,23 @@ ctx_sched_in(struct perf_event_context *ctx,
+ struct task_struct *task)
+ {
+ int is_active = ctx->is_active;
+- u64 now;
+
+ lockdep_assert_held(&ctx->lock);
+
+ if (likely(!ctx->nr_events))
+ return;
+
++ if (is_active ^ EVENT_TIME) {
++ /* start ctx time */
++ __update_context_time(ctx, false);
++ perf_cgroup_set_timestamp(task, ctx);
++ /*
++ * CPU-release for the below ->is_active store,
++ * see __load_acquire() in perf_event_time_now()
++ */
++ barrier();
++ }
++
+ ctx->is_active |= (event_type | EVENT_TIME);
+ if (ctx->task) {
+ if (!is_active)
+@@ -3814,13 +3892,6 @@ ctx_sched_in(struct perf_event_context *ctx,
+
+ is_active ^= ctx->is_active; /* changed bits */
+
+- if (is_active & EVENT_TIME) {
+- /* start ctx time */
+- now = perf_clock();
+- ctx->timestamp = now;
+- perf_cgroup_set_timestamp(task, ctx);
+- }
+-
+ /*
+ * First go through the list and put on any pinned groups
+ * in order to give them the best chance of going on.
+@@ -4414,6 +4485,18 @@ static inline u64 perf_event_count(struct perf_event *event)
+ return local64_read(&event->count) + atomic64_read(&event->child_count);
+ }
+
++static void calc_timer_values(struct perf_event *event,
++ u64 *now,
++ u64 *enabled,
++ u64 *running)
++{
++ u64 ctx_time;
++
++ *now = perf_clock();
++ ctx_time = perf_event_time_now(event, *now);
++ __perf_update_times(event, ctx_time, enabled, running);
++}
++
+ /*
+ * NMI-safe method to read a local event, that is an event that
+ * is:
+@@ -4473,10 +4556,9 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
+
+ *value = local64_read(&event->count);
+ if (enabled || running) {
+- u64 now = event->shadow_ctx_time + perf_clock();
+- u64 __enabled, __running;
++ u64 __enabled, __running, __now;;
+
+- __perf_update_times(event, now, &__enabled, &__running);
++ calc_timer_values(event, &__now, &__enabled, &__running);
+ if (enabled)
+ *enabled = __enabled;
+ if (running)
+@@ -5798,18 +5880,6 @@ static int perf_event_index(struct perf_event *event)
+ return event->pmu->event_idx(event);
+ }
+
+-static void calc_timer_values(struct perf_event *event,
+- u64 *now,
+- u64 *enabled,
+- u64 *running)
+-{
+- u64 ctx_time;
+-
+- *now = perf_clock();
+- ctx_time = event->shadow_ctx_time + *now;
+- __perf_update_times(event, ctx_time, enabled, running);
+-}
+-
+ static void perf_event_init_userpage(struct perf_event *event)
+ {
+ struct perf_event_mmap_page *userpg;
+@@ -5934,6 +6004,8 @@ static void ring_buffer_attach(struct perf_event *event,
+ struct perf_buffer *old_rb = NULL;
+ unsigned long flags;
+
++ WARN_ON_ONCE(event->parent);
++
+ if (event->rb) {
+ /*
+ * Should be impossible, we set this when removing
+@@ -5991,6 +6063,9 @@ static void ring_buffer_wakeup(struct perf_event *event)
+ {
+ struct perf_buffer *rb;
+
++ if (event->parent)
++ event = event->parent;
++
+ rcu_read_lock();
+ rb = rcu_dereference(event->rb);
+ if (rb) {
+@@ -6004,6 +6079,9 @@ struct perf_buffer *ring_buffer_get(struct perf_event *event)
+ {
+ struct perf_buffer *rb;
+
++ if (event->parent)
++ event = event->parent;
++
+ rcu_read_lock();
+ rb = rcu_dereference(event->rb);
+ if (rb) {
+@@ -6270,7 +6348,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
+ again:
+ mutex_lock(&event->mmap_mutex);
+ if (event->rb) {
+- if (event->rb->nr_pages != nr_pages) {
++ if (data_page_nr(event->rb) != nr_pages) {
+ ret = -EINVAL;
+ goto unlock;
+ }
+@@ -6349,7 +6427,6 @@ accounting:
+ ring_buffer_attach(event, rb);
+
+ perf_event_update_time(event);
+- perf_set_shadow_time(event, event->ctx);
+ perf_event_init_userpage(event);
+ perf_event_update_userpage(event);
+ } else {
+@@ -6526,18 +6603,25 @@ static void perf_pending_event(struct irq_work *entry)
+ * Later on, we might change it to a list if there is
+ * another virtualization implementation supporting the callbacks.
+ */
+-struct perf_guest_info_callbacks *perf_guest_cbs;
++struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
+
+ int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
+ {
+- perf_guest_cbs = cbs;
++ if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
++ return -EBUSY;
++
++ rcu_assign_pointer(perf_guest_cbs, cbs);
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
+
+ int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
+ {
+- perf_guest_cbs = NULL;
++ if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
++ return -EINVAL;
++
++ rcu_assign_pointer(perf_guest_cbs, NULL);
++ synchronize_rcu();
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
+@@ -6696,7 +6780,7 @@ static unsigned long perf_prepare_sample_aux(struct perf_event *event,
+ if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
+ goto out;
+
+- rb = ring_buffer_get(sampler->parent ? sampler->parent : sampler);
++ rb = ring_buffer_get(sampler);
+ if (!rb)
+ goto out;
+
+@@ -6762,7 +6846,7 @@ static void perf_aux_sample_output(struct perf_event *event,
+ if (WARN_ON_ONCE(!sampler || !data->aux_size))
+ return;
+
+- rb = ring_buffer_get(sampler->parent ? sampler->parent : sampler);
++ rb = ring_buffer_get(sampler);
+ if (!rb)
+ return;
+
+@@ -7154,7 +7238,6 @@ void perf_output_sample(struct perf_output_handle *handle,
+ static u64 perf_virt_to_phys(u64 virt)
+ {
+ u64 phys_addr = 0;
+- struct page *p = NULL;
+
+ if (!virt)
+ return 0;
+@@ -7173,14 +7256,15 @@ static u64 perf_virt_to_phys(u64 virt)
+ * If failed, leave phys_addr as 0.
+ */
+ if (current->mm != NULL) {
++ struct page *p;
++
+ pagefault_disable();
+- if (get_user_page_fast_only(virt, 0, &p))
++ if (get_user_page_fast_only(virt, 0, &p)) {
+ phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
++ put_page(p);
++ }
+ pagefault_enable();
+ }
+-
+- if (p)
+- put_page(p);
+ }
+
+ return phys_addr;
+@@ -9729,6 +9813,9 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
+ continue;
+ if (event->attr.config != entry->type)
+ continue;
++ /* Cannot deliver synchronous signal to other task. */
++ if (event->attr.sigtrap)
++ continue;
+ if (perf_tp_event_match(event, &data, regs))
+ perf_swevent_event(event, count, &data, regs);
+ }
+@@ -10443,8 +10530,11 @@ perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
+ }
+
+ /* ready to consume more filters */
++ kfree(filename);
++ filename = NULL;
+ state = IF_STATE_ACTION;
+ filter = NULL;
++ kernel = 0;
+ }
+ }
+
+@@ -11506,6 +11596,9 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
+
+ event->state = PERF_EVENT_STATE_INACTIVE;
+
++ if (parent_event)
++ event->event_caps = parent_event->event_caps;
++
+ if (event->attr.sigtrap)
+ atomic_set(&event->event_limit, 1);
+
+@@ -12190,6 +12283,9 @@ SYSCALL_DEFINE5(perf_event_open,
+ * Do not allow to attach to a group in a different task
+ * or CPU context. If we're moving SW events, we'll fix
+ * this up later, so allow that.
++ *
++ * Racy, not holding group_leader->ctx->mutex, see comment with
++ * perf_event_ctx_lock().
+ */
+ if (!move_group && group_leader->ctx != ctx)
+ goto err_context;
+@@ -12255,6 +12351,7 @@ SYSCALL_DEFINE5(perf_event_open,
+ } else {
+ perf_event_ctx_unlock(group_leader, gctx);
+ move_group = 0;
++ goto not_move_group;
+ }
+ }
+
+@@ -12271,7 +12368,17 @@ SYSCALL_DEFINE5(perf_event_open,
+ }
+ } else {
+ mutex_lock(&ctx->mutex);
++
++ /*
++ * Now that we hold ctx->lock, (re)validate group_leader->ctx == ctx,
++ * see the group_leader && !move_group test earlier.
++ */
++ if (group_leader && group_leader->ctx != ctx) {
++ err = -EINVAL;
++ goto err_locked;
++ }
+ }
++not_move_group:
+
+ if (ctx->task == TASK_TOMBSTONE) {
+ err = -ESRCH;
+diff --git a/kernel/events/internal.h b/kernel/events/internal.h
+index 228801e207886..aa23ffdaf819f 100644
+--- a/kernel/events/internal.h
++++ b/kernel/events/internal.h
+@@ -116,6 +116,11 @@ static inline int page_order(struct perf_buffer *rb)
+ }
+ #endif
+
++static inline int data_page_nr(struct perf_buffer *rb)
++{
++ return rb->nr_pages << page_order(rb);
++}
++
+ static inline unsigned long perf_data_size(struct perf_buffer *rb)
+ {
+ return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
+diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
+index 52868716ec358..fb35b926024ca 100644
+--- a/kernel/events/ring_buffer.c
++++ b/kernel/events/ring_buffer.c
+@@ -859,11 +859,6 @@ void rb_free(struct perf_buffer *rb)
+ }
+
+ #else
+-static int data_page_nr(struct perf_buffer *rb)
+-{
+- return rb->nr_pages << page_order(rb);
+-}
+-
+ static struct page *
+ __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
+ {
+diff --git a/kernel/fork.c b/kernel/fork.c
+index 38681ad44c76b..89475c994ca91 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -2055,18 +2055,18 @@ static __latent_entropy struct task_struct *copy_process(
+ #ifdef CONFIG_PROVE_LOCKING
+ DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
+ #endif
++ retval = copy_creds(p, clone_flags);
++ if (retval < 0)
++ goto bad_fork_free;
++
+ retval = -EAGAIN;
+ if (is_ucounts_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) {
+ if (p->real_cred->user != INIT_USER &&
+ !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
+- goto bad_fork_free;
++ goto bad_fork_cleanup_count;
+ }
+ current->flags &= ~PF_NPROC_EXCEEDED;
+
+- retval = copy_creds(p, clone_flags);
+- if (retval < 0)
+- goto bad_fork_free;
+-
+ /*
+ * If multiple threads are within copy_process(), then this check
+ * triggers too late. This doesn't hurt, the check is only there
+@@ -2280,6 +2280,7 @@ static __latent_entropy struct task_struct *copy_process(
+ p->pdeath_signal = 0;
+ INIT_LIST_HEAD(&p->thread_group);
+ p->task_works = NULL;
++ clear_posix_cputimers_work(p);
+
+ #ifdef CONFIG_KRETPROBES
+ p->kretprobe_instances.first = NULL;
+@@ -2295,6 +2296,17 @@ static __latent_entropy struct task_struct *copy_process(
+ if (retval)
+ goto bad_fork_put_pidfd;
+
++ /*
++ * Now that the cgroups are pinned, re-clone the parent cgroup and put
++ * the new task on the correct runqueue. All this *before* the task
++ * becomes visible.
++ *
++ * This isn't part of ->can_fork() because while the re-cloning is
++ * cgroup specific, it unconditionally needs to place the task on a
++ * runqueue.
++ */
++ sched_cgroup_fork(p, args);
++
+ /*
+ * From this point on we must avoid any synchronous user-space
+ * communication until we take the tasklist-lock. In particular, we do
+@@ -2352,10 +2364,6 @@ static __latent_entropy struct task_struct *copy_process(
+ goto bad_fork_cancel_cgroup;
+ }
+
+- /* past the last point of failure */
+- if (pidfile)
+- fd_install(pidfd, pidfile);
+-
+ init_task_pid_links(p);
+ if (likely(p->pid)) {
+ ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
+@@ -2404,6 +2412,9 @@ static __latent_entropy struct task_struct *copy_process(
+ syscall_tracepoint_update(p);
+ write_unlock_irq(&tasklist_lock);
+
++ if (pidfile)
++ fd_install(pidfd, pidfile);
++
+ proc_fork_connector(p);
+ sched_post_fork(p);
+ cgroup_post_fork(p, args);
+diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
+index f7ff8919dc9bb..fdf170404650f 100644
+--- a/kernel/irq/affinity.c
++++ b/kernel/irq/affinity.c
+@@ -269,8 +269,9 @@ static int __irq_build_affinity_masks(unsigned int startvec,
+ */
+ if (numvecs <= nodes) {
+ for_each_node_mask(n, nodemsk) {
+- cpumask_or(&masks[curvec].mask, &masks[curvec].mask,
+- node_to_cpumask[n]);
++ /* Ensure that only CPUs which are in both masks are set */
++ cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
++ cpumask_or(&masks[curvec].mask, &masks[curvec].mask, nmsk);
+ if (++curvec == last_affv)
+ curvec = firstvec;
+ }
+diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
+index 221d80c31e94c..fca637d4da1a7 100644
+--- a/kernel/irq/handle.c
++++ b/kernel/irq/handle.c
+@@ -195,7 +195,7 @@ irqreturn_t handle_irq_event_percpu(struct irq_desc *desc)
+
+ retval = __handle_irq_event_percpu(desc, &flags);
+
+- add_interrupt_randomness(desc->irq_data.irq, flags);
++ add_interrupt_randomness(desc->irq_data.irq);
+
+ if (!irq_settings_no_debug(desc))
+ note_interrupt(desc, retval);
+diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
+index 54363527feea4..e58342ace11f2 100644
+--- a/kernel/irq/internals.h
++++ b/kernel/irq/internals.h
+@@ -29,12 +29,14 @@ extern struct irqaction chained_action;
+ * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
+ * IRQTF_AFFINITY - irq thread is requested to adjust affinity
+ * IRQTF_FORCED_THREAD - irq action is force threaded
++ * IRQTF_READY - signals that irq thread is ready
+ */
+ enum {
+ IRQTF_RUNTHREAD,
+ IRQTF_WARNED,
+ IRQTF_AFFINITY,
+ IRQTF_FORCED_THREAD,
++ IRQTF_READY,
+ };
+
+ /*
+diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
+index 4e3c29bb603c3..21b3ac2a29d20 100644
+--- a/kernel/irq/irqdesc.c
++++ b/kernel/irq/irqdesc.c
+@@ -407,6 +407,7 @@ static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
+ lockdep_set_class(&desc->lock, &irq_desc_lock_class);
+ mutex_init(&desc->request_mutex);
+ init_rcu_head(&desc->rcu);
++ init_waitqueue_head(&desc->wait_for_threads);
+
+ desc_set_defaults(irq, desc, node, affinity, owner);
+ irqd_set(&desc->irq_data, flags);
+@@ -575,6 +576,7 @@ int __init early_irq_init(void)
+ raw_spin_lock_init(&desc[i].lock);
+ lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
+ mutex_init(&desc[i].request_mutex);
++ init_waitqueue_head(&desc[i].wait_for_threads);
+ desc_set_defaults(i, &desc[i], node, NULL, NULL);
+ }
+ return arch_early_irq_init();
+diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
+index 27667e82ecc91..0c3c26fb054f7 100644
+--- a/kernel/irq/manage.c
++++ b/kernel/irq/manage.c
+@@ -1248,6 +1248,31 @@ static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
+ raw_spin_unlock_irq(&desc->lock);
+ }
+
++/*
++ * Internal function to notify that a interrupt thread is ready.
++ */
++static void irq_thread_set_ready(struct irq_desc *desc,
++ struct irqaction *action)
++{
++ set_bit(IRQTF_READY, &action->thread_flags);
++ wake_up(&desc->wait_for_threads);
++}
++
++/*
++ * Internal function to wake up a interrupt thread and wait until it is
++ * ready.
++ */
++static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc,
++ struct irqaction *action)
++{
++ if (!action || !action->thread)
++ return;
++
++ wake_up_process(action->thread);
++ wait_event(desc->wait_for_threads,
++ test_bit(IRQTF_READY, &action->thread_flags));
++}
++
+ /*
+ * Interrupt handler thread
+ */
+@@ -1259,6 +1284,8 @@ static int irq_thread(void *data)
+ irqreturn_t (*handler_fn)(struct irq_desc *desc,
+ struct irqaction *action);
+
++ irq_thread_set_ready(desc, action);
++
+ if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
+ &action->thread_flags))
+ handler_fn = irq_forced_thread_fn;
+@@ -1683,8 +1710,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
+ }
+
+ if (!shared) {
+- init_waitqueue_head(&desc->wait_for_threads);
+-
+ /* Setup the type (level, edge polarity) if configured: */
+ if (new->flags & IRQF_TRIGGER_MASK) {
+ ret = __irq_set_trigger(desc,
+@@ -1780,14 +1805,8 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
+
+ irq_setup_timings(desc, new);
+
+- /*
+- * Strictly no need to wake it up, but hung_task complains
+- * when no hard interrupt wakes the thread up.
+- */
+- if (new->thread)
+- wake_up_process(new->thread);
+- if (new->secondary)
+- wake_up_process(new->secondary->thread);
++ wake_up_and_wait_for_irq_thread_ready(desc, new);
++ wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
+
+ register_irq_proc(irq, desc);
+ new->dir = NULL;
+diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
+index 6a5ecee6e5674..7f350ae59c5fd 100644
+--- a/kernel/irq/msi.c
++++ b/kernel/irq/msi.c
+@@ -529,10 +529,10 @@ static bool msi_check_reservation_mode(struct irq_domain *domain,
+
+ /*
+ * Checking the first MSI descriptor is sufficient. MSIX supports
+- * masking and MSI does so when the maskbit is set.
++ * masking and MSI does so when the can_mask attribute is set.
+ */
+ desc = first_msi_entry(dev);
+- return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;
++ return desc->msi_attrib.is_msix || desc->msi_attrib.can_mask;
+ }
+
+ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
+diff --git a/kernel/kprobes.c b/kernel/kprobes.c
+index 790a573bbe00c..2ef90d15699fb 100644
+--- a/kernel/kprobes.c
++++ b/kernel/kprobes.c
+@@ -2006,6 +2006,9 @@ int register_kretprobe(struct kretprobe *rp)
+ }
+ }
+
++ if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
++ return -E2BIG;
++
+ rp->kp.pre_handler = pre_handler_kretprobe;
+ rp->kp.post_handler = NULL;
+
+@@ -2809,13 +2812,12 @@ static const struct file_operations fops_kp = {
+ static int __init debugfs_kprobe_init(void)
+ {
+ struct dentry *dir;
+- unsigned int value = 1;
+
+ dir = debugfs_create_dir("kprobes", NULL);
+
+ debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
+
+- debugfs_create_file("enabled", 0600, dir, &value, &fops_kp);
++ debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
+
+ debugfs_create_file("blacklist", 0400, dir, NULL,
+ &kprobe_blacklist_fops);
+diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c
+index 335d988bd8111..c0789383807b9 100644
+--- a/kernel/livepatch/core.c
++++ b/kernel/livepatch/core.c
+@@ -190,7 +190,7 @@ static int klp_find_object_symbol(const char *objname, const char *name,
+ return -EINVAL;
+ }
+
+-static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab,
++static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab,
+ unsigned int symndx, Elf_Shdr *relasec,
+ const char *sec_objname)
+ {
+@@ -218,7 +218,7 @@ static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab,
+ relas = (Elf_Rela *) relasec->sh_addr;
+ /* For each rela in this klp relocation section */
+ for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
+- sym = (Elf64_Sym *)sechdrs[symndx].sh_addr + ELF_R_SYM(relas[i].r_info);
++ sym = (Elf_Sym *)sechdrs[symndx].sh_addr + ELF_R_SYM(relas[i].r_info);
+ if (sym->st_shndx != SHN_LIVEPATCH) {
+ pr_err("symbol %s is not marked as a livepatch symbol\n",
+ strtab + sym->st_name);
+diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
+index bf1c00c881e48..a30702b847ba8 100644
+--- a/kernel/locking/lockdep.c
++++ b/kernel/locking/lockdep.c
+@@ -183,11 +183,9 @@ static DECLARE_BITMAP(list_entries_in_use, MAX_LOCKDEP_ENTRIES);
+ static struct hlist_head lock_keys_hash[KEYHASH_SIZE];
+ unsigned long nr_lock_classes;
+ unsigned long nr_zapped_classes;
+-#ifndef CONFIG_DEBUG_LOCKDEP
+-static
+-#endif
++unsigned long max_lock_class_idx;
+ struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
+-static DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);
++DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS);
+
+ static inline struct lock_class *hlock_class(struct held_lock *hlock)
+ {
+@@ -338,7 +336,7 @@ static inline void lock_release_holdtime(struct held_lock *hlock)
+ * elements. These elements are linked together by the lock_entry member in
+ * struct lock_class.
+ */
+-LIST_HEAD(all_lock_classes);
++static LIST_HEAD(all_lock_classes);
+ static LIST_HEAD(free_lock_classes);
+
+ /**
+@@ -888,7 +886,7 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
+ if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+ return NULL;
+
+- hlist_for_each_entry_rcu(class, hash_head, hash_entry) {
++ hlist_for_each_entry_rcu_notrace(class, hash_head, hash_entry) {
+ if (class->key == key) {
+ /*
+ * Huh! same key, different name? Did someone trample
+@@ -1240,6 +1238,7 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
+ struct lockdep_subclass_key *key;
+ struct hlist_head *hash_head;
+ struct lock_class *class;
++ int idx;
+
+ DEBUG_LOCKS_WARN_ON(!irqs_disabled());
+
+@@ -1305,6 +1304,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
+ * of classes.
+ */
+ list_move_tail(&class->lock_entry, &all_lock_classes);
++ idx = class - lock_classes;
++ if (idx > max_lock_class_idx)
++ max_lock_class_idx = idx;
+
+ if (verbose(class)) {
+ graph_unlock();
+@@ -3450,7 +3452,7 @@ struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
+ u16 chain_hlock = chain_hlocks[chain->base + i];
+ unsigned int class_idx = chain_hlock_class_idx(chain_hlock);
+
+- return lock_classes + class_idx - 1;
++ return lock_classes + class_idx;
+ }
+
+ /*
+@@ -3518,7 +3520,7 @@ static void print_chain_keys_chain(struct lock_chain *chain)
+ hlock_id = chain_hlocks[chain->base + i];
+ chain_key = print_chain_key_iteration(hlock_id, chain_key);
+
+- print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id) - 1);
++ print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id));
+ printk("\n");
+ }
+ }
+@@ -5366,7 +5368,7 @@ int __lock_is_held(const struct lockdep_map *lock, int read)
+ struct held_lock *hlock = curr->held_locks + i;
+
+ if (match_held_lock(hlock, lock)) {
+- if (read == -1 || hlock->read == read)
++ if (read == -1 || !!hlock->read == read)
+ return LOCK_STATE_HELD;
+
+ return LOCK_STATE_NOT_HELD;
+@@ -5986,6 +5988,8 @@ static void zap_class(struct pending_free *pf, struct lock_class *class)
+ WRITE_ONCE(class->name, NULL);
+ nr_lock_classes--;
+ __clear_bit(class - lock_classes, lock_classes_in_use);
++ if (class - lock_classes == max_lock_class_idx)
++ max_lock_class_idx--;
+ } else {
+ WARN_ONCE(true, "%s() failed for class %s\n", __func__,
+ class->name);
+@@ -6276,7 +6280,13 @@ void lockdep_reset_lock(struct lockdep_map *lock)
+ lockdep_reset_lock_reg(lock);
+ }
+
+-/* Unregister a dynamically allocated key. */
++/*
++ * Unregister a dynamically allocated key.
++ *
++ * Unlike lockdep_register_key(), a search is always done to find a matching
++ * key irrespective of debug_locks to avoid potential invalid access to freed
++ * memory in lock_class entry.
++ */
+ void lockdep_unregister_key(struct lock_class_key *key)
+ {
+ struct hlist_head *hash_head = keyhashentry(key);
+@@ -6291,10 +6301,8 @@ void lockdep_unregister_key(struct lock_class_key *key)
+ return;
+
+ raw_local_irq_save(flags);
+- if (!graph_lock())
+- goto out_irq;
++ lockdep_lock();
+
+- pf = get_pending_free();
+ hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
+ if (k == key) {
+ hlist_del_rcu(&k->hash_entry);
+@@ -6302,11 +6310,13 @@ void lockdep_unregister_key(struct lock_class_key *key)
+ break;
+ }
+ }
+- WARN_ON_ONCE(!found);
+- __lockdep_free_key_range(pf, key, 1);
+- call_rcu_zapped(pf);
+- graph_unlock();
+-out_irq:
++ WARN_ON_ONCE(!found && debug_locks);
++ if (found) {
++ pf = get_pending_free();
++ __lockdep_free_key_range(pf, key, 1);
++ call_rcu_zapped(pf);
++ }
++ lockdep_unlock();
+ raw_local_irq_restore(flags);
+
+ /* Wait until is_dynamic_key() has finished accessing k->hash_entry. */
+diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h
+index ecb8662e7a4ed..bbe9000260d02 100644
+--- a/kernel/locking/lockdep_internals.h
++++ b/kernel/locking/lockdep_internals.h
+@@ -121,7 +121,6 @@ static const unsigned long LOCKF_USED_IN_IRQ_READ =
+
+ #define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
+
+-extern struct list_head all_lock_classes;
+ extern struct lock_chain lock_chains[];
+
+ #define LOCK_USAGE_CHARS (2*XXX_LOCK_USAGE_STATES + 1)
+@@ -151,6 +150,10 @@ extern unsigned int nr_large_chain_blocks;
+
+ extern unsigned int max_lockdep_depth;
+ extern unsigned int max_bfs_queue_depth;
++extern unsigned long max_lock_class_idx;
++
++extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
++extern unsigned long lock_classes_in_use[];
+
+ #ifdef CONFIG_PROVE_LOCKING
+ extern unsigned long lockdep_count_forward_deps(struct lock_class *);
+@@ -205,7 +208,6 @@ struct lockdep_stats {
+ };
+
+ DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
+-extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
+
+ #define __debug_atomic_inc(ptr) \
+ this_cpu_inc(lockdep_stats.ptr);
+diff --git a/kernel/locking/lockdep_proc.c b/kernel/locking/lockdep_proc.c
+index b8d9a050c337a..15fdc7fa5c688 100644
+--- a/kernel/locking/lockdep_proc.c
++++ b/kernel/locking/lockdep_proc.c
+@@ -24,14 +24,33 @@
+
+ #include "lockdep_internals.h"
+
++/*
++ * Since iteration of lock_classes is done without holding the lockdep lock,
++ * it is not safe to iterate all_lock_classes list directly as the iteration
++ * may branch off to free_lock_classes or the zapped list. Iteration is done
++ * directly on the lock_classes array by checking the lock_classes_in_use
++ * bitmap and max_lock_class_idx.
++ */
++#define iterate_lock_classes(idx, class) \
++ for (idx = 0, class = lock_classes; idx <= max_lock_class_idx; \
++ idx++, class++)
++
+ static void *l_next(struct seq_file *m, void *v, loff_t *pos)
+ {
+- return seq_list_next(v, &all_lock_classes, pos);
++ struct lock_class *class = v;
++
++ ++class;
++ *pos = class - lock_classes;
++ return (*pos > max_lock_class_idx) ? NULL : class;
+ }
+
+ static void *l_start(struct seq_file *m, loff_t *pos)
+ {
+- return seq_list_start_head(&all_lock_classes, *pos);
++ unsigned long idx = *pos;
++
++ if (idx > max_lock_class_idx)
++ return NULL;
++ return lock_classes + idx;
+ }
+
+ static void l_stop(struct seq_file *m, void *v)
+@@ -57,14 +76,16 @@ static void print_name(struct seq_file *m, struct lock_class *class)
+
+ static int l_show(struct seq_file *m, void *v)
+ {
+- struct lock_class *class = list_entry(v, struct lock_class, lock_entry);
++ struct lock_class *class = v;
+ struct lock_list *entry;
+ char usage[LOCK_USAGE_CHARS];
++ int idx = class - lock_classes;
+
+- if (v == &all_lock_classes) {
++ if (v == lock_classes)
+ seq_printf(m, "all lock classes:\n");
++
++ if (!test_bit(idx, lock_classes_in_use))
+ return 0;
+- }
+
+ seq_printf(m, "%p", class->key);
+ #ifdef CONFIG_DEBUG_LOCKDEP
+@@ -220,8 +241,11 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
+
+ #ifdef CONFIG_PROVE_LOCKING
+ struct lock_class *class;
++ unsigned long idx;
+
+- list_for_each_entry(class, &all_lock_classes, lock_entry) {
++ iterate_lock_classes(idx, class) {
++ if (!test_bit(idx, lock_classes_in_use))
++ continue;
+
+ if (class->usage_mask == 0)
+ nr_unused++;
+@@ -254,6 +278,7 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
+
+ sum_forward_deps += lockdep_count_forward_deps(class);
+ }
++
+ #ifdef CONFIG_DEBUG_LOCKDEP
+ DEBUG_LOCKS_WARN_ON(debug_atomic_read(nr_unused_locks) != nr_unused);
+ #endif
+@@ -345,6 +370,8 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
+ seq_printf(m, " max bfs queue depth: %11u\n",
+ max_bfs_queue_depth);
+ #endif
++ seq_printf(m, " max lock class index: %11lu\n",
++ max_lock_class_idx);
+ lockdep_stats_debug_show(m);
+ seq_printf(m, " debug_locks: %11u\n",
+ debug_locks);
+@@ -622,12 +649,16 @@ static int lock_stat_open(struct inode *inode, struct file *file)
+ if (!res) {
+ struct lock_stat_data *iter = data->stats;
+ struct seq_file *m = file->private_data;
++ unsigned long idx;
+
+- list_for_each_entry(class, &all_lock_classes, lock_entry) {
++ iterate_lock_classes(idx, class) {
++ if (!test_bit(idx, lock_classes_in_use))
++ continue;
+ iter->class = class;
+ iter->stats = lock_stats(class);
+ iter++;
+ }
++
+ data->iter_end = iter;
+
+ sort(data->stats, data->iter_end - data->stats,
+@@ -645,6 +676,7 @@ static ssize_t lock_stat_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+ {
+ struct lock_class *class;
++ unsigned long idx;
+ char c;
+
+ if (count) {
+@@ -654,8 +686,11 @@ static ssize_t lock_stat_write(struct file *file, const char __user *buf,
+ if (c != '0')
+ return count;
+
+- list_for_each_entry(class, &all_lock_classes, lock_entry)
++ iterate_lock_classes(idx, class) {
++ if (!test_bit(idx, lock_classes_in_use))
++ continue;
+ clear_lock_stats(class);
++ }
+ }
+ return count;
+ }
+diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
+index 6bb116c559b4a..ea5a701ab2408 100644
+--- a/kernel/locking/rtmutex.c
++++ b/kernel/locking/rtmutex.c
+@@ -1373,7 +1373,7 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
+ * - the VCPU on which owner runs is preempted
+ */
+ if (!owner->on_cpu || need_resched() ||
+- rt_mutex_waiter_is_top_waiter(lock, waiter) ||
++ !rt_mutex_waiter_is_top_waiter(lock, waiter) ||
+ vcpu_is_preempted(task_cpu(owner))) {
+ res = false;
+ break;
+diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
+index 000e8d5a28841..e63f740c2cc84 100644
+--- a/kernel/locking/rwsem.c
++++ b/kernel/locking/rwsem.c
+@@ -106,9 +106,9 @@
+ * atomic_long_cmpxchg() will be used to obtain writer lock.
+ *
+ * There are three places where the lock handoff bit may be set or cleared.
+- * 1) rwsem_mark_wake() for readers.
+- * 2) rwsem_try_write_lock() for writers.
+- * 3) Error path of rwsem_down_write_slowpath().
++ * 1) rwsem_mark_wake() for readers -- set, clear
++ * 2) rwsem_try_write_lock() for writers -- set, clear
++ * 3) rwsem_del_waiter() -- clear
+ *
+ * For all the above cases, wait_lock will be held. A writer must also
+ * be the first one in the wait_list to be eligible for setting the handoff
+@@ -335,6 +335,9 @@ struct rwsem_waiter {
+ struct task_struct *task;
+ enum rwsem_waiter_type type;
+ unsigned long timeout;
++
++ /* Writer only, not initialized in reader */
++ bool handoff_set;
+ };
+ #define rwsem_first_waiter(sem) \
+ list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
+@@ -345,12 +348,6 @@ enum rwsem_wake_type {
+ RWSEM_WAKE_READ_OWNED /* Waker thread holds the read lock */
+ };
+
+-enum writer_wait_state {
+- WRITER_NOT_FIRST, /* Writer is not first in wait list */
+- WRITER_FIRST, /* Writer is first in wait list */
+- WRITER_HANDOFF /* Writer is first & handoff needed */
+-};
+-
+ /*
+ * The typical HZ value is either 250 or 1000. So set the minimum waiting
+ * time to at least 4ms or 1 jiffy (if it is higher than 4ms) in the wait
+@@ -366,6 +363,31 @@ enum writer_wait_state {
+ */
+ #define MAX_READERS_WAKEUP 0x100
+
++static inline void
++rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
++{
++ lockdep_assert_held(&sem->wait_lock);
++ list_add_tail(&waiter->list, &sem->wait_list);
++ /* caller will set RWSEM_FLAG_WAITERS */
++}
++
++/*
++ * Remove a waiter from the wait_list and clear flags.
++ *
++ * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of
++ * this function. Modify with care.
++ */
++static inline void
++rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
++{
++ lockdep_assert_held(&sem->wait_lock);
++ list_del(&waiter->list);
++ if (likely(!list_empty(&sem->wait_list)))
++ return;
++
++ atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
++}
++
+ /*
+ * handle the lock release when processes blocked on it that can now run
+ * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must
+@@ -377,6 +399,8 @@ enum writer_wait_state {
+ * preferably when the wait_lock is released
+ * - woken process blocks are discarded from the list after having task zeroed
+ * - writers are only marked woken if downgrading is false
++ *
++ * Implies rwsem_del_waiter() for all woken readers.
+ */
+ static void rwsem_mark_wake(struct rw_semaphore *sem,
+ enum rwsem_wake_type wake_type,
+@@ -491,18 +515,25 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
+
+ adjustment = woken * RWSEM_READER_BIAS - adjustment;
+ lockevent_cond_inc(rwsem_wake_reader, woken);
++
++ oldcount = atomic_long_read(&sem->count);
+ if (list_empty(&sem->wait_list)) {
+- /* hit end of list above */
++ /*
++ * Combined with list_move_tail() above, this implies
++ * rwsem_del_waiter().
++ */
+ adjustment -= RWSEM_FLAG_WAITERS;
++ if (oldcount & RWSEM_FLAG_HANDOFF)
++ adjustment -= RWSEM_FLAG_HANDOFF;
++ } else if (woken) {
++ /*
++ * When we've woken a reader, we no longer need to force
++ * writers to give up the lock and we can clear HANDOFF.
++ */
++ if (oldcount & RWSEM_FLAG_HANDOFF)
++ adjustment -= RWSEM_FLAG_HANDOFF;
+ }
+
+- /*
+- * When we've woken a reader, we no longer need to force writers
+- * to give up the lock and we can clear HANDOFF.
+- */
+- if (woken && (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF))
+- adjustment -= RWSEM_FLAG_HANDOFF;
+-
+ if (adjustment)
+ atomic_long_add(adjustment, &sem->count);
+
+@@ -533,12 +564,12 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
+ * race conditions between checking the rwsem wait list and setting the
+ * sem->count accordingly.
+ *
+- * If wstate is WRITER_HANDOFF, it will make sure that either the handoff
+- * bit is set or the lock is acquired with handoff bit cleared.
++ * Implies rwsem_del_waiter() on success.
+ */
+ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
+- enum writer_wait_state wstate)
++ struct rwsem_waiter *waiter)
+ {
++ bool first = rwsem_first_waiter(sem) == waiter;
+ long count, new;
+
+ lockdep_assert_held(&sem->wait_lock);
+@@ -547,13 +578,19 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
+ do {
+ bool has_handoff = !!(count & RWSEM_FLAG_HANDOFF);
+
+- if (has_handoff && wstate == WRITER_NOT_FIRST)
+- return false;
++ if (has_handoff) {
++ if (!first)
++ return false;
++
++ /* First waiter inherits a previously set handoff bit */
++ waiter->handoff_set = true;
++ }
+
+ new = count;
+
+ if (count & RWSEM_LOCK_MASK) {
+- if (has_handoff || (wstate != WRITER_HANDOFF))
++ if (has_handoff || (!rt_task(waiter->task) &&
++ !time_after(jiffies, waiter->timeout)))
+ return false;
+
+ new |= RWSEM_FLAG_HANDOFF;
+@@ -570,13 +607,39 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
+ * We have either acquired the lock with handoff bit cleared or
+ * set the handoff bit.
+ */
+- if (new & RWSEM_FLAG_HANDOFF)
++ if (new & RWSEM_FLAG_HANDOFF) {
++ waiter->handoff_set = true;
++ lockevent_inc(rwsem_wlock_handoff);
+ return false;
++ }
+
++ /*
++ * Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on
++ * success.
++ */
++ list_del(&waiter->list);
+ rwsem_set_owner(sem);
+ return true;
+ }
+
++/*
++ * The rwsem_spin_on_owner() function returns the following 4 values
++ * depending on the lock owner state.
++ * OWNER_NULL : owner is currently NULL
++ * OWNER_WRITER: when owner changes and is a writer
++ * OWNER_READER: when owner changes and the new owner may be a reader.
++ * OWNER_NONSPINNABLE:
++ * when optimistic spinning has to stop because either the
++ * owner stops running, is unknown, or its timeslice has
++ * been used up.
++ */
++enum owner_state {
++ OWNER_NULL = 1 << 0,
++ OWNER_WRITER = 1 << 1,
++ OWNER_READER = 1 << 2,
++ OWNER_NONSPINNABLE = 1 << 3,
++};
++
+ #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
+ /*
+ * Try to acquire write lock before the writer has been put on wait queue.
+@@ -632,23 +695,6 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
+ return ret;
+ }
+
+-/*
+- * The rwsem_spin_on_owner() function returns the following 4 values
+- * depending on the lock owner state.
+- * OWNER_NULL : owner is currently NULL
+- * OWNER_WRITER: when owner changes and is a writer
+- * OWNER_READER: when owner changes and the new owner may be a reader.
+- * OWNER_NONSPINNABLE:
+- * when optimistic spinning has to stop because either the
+- * owner stops running, is unknown, or its timeslice has
+- * been used up.
+- */
+-enum owner_state {
+- OWNER_NULL = 1 << 0,
+- OWNER_WRITER = 1 << 1,
+- OWNER_READER = 1 << 2,
+- OWNER_NONSPINNABLE = 1 << 3,
+-};
+ #define OWNER_SPINNABLE (OWNER_NULL | OWNER_WRITER | OWNER_READER)
+
+ static inline enum owner_state
+@@ -878,12 +924,11 @@ static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem)
+
+ static inline void clear_nonspinnable(struct rw_semaphore *sem) { }
+
+-static inline int
++static inline enum owner_state
+ rwsem_spin_on_owner(struct rw_semaphore *sem)
+ {
+- return 0;
++ return OWNER_NONSPINNABLE;
+ }
+-#define OWNER_NULL 1
+ #endif
+
+ /*
+@@ -953,7 +998,7 @@ queue:
+ }
+ adjustment += RWSEM_FLAG_WAITERS;
+ }
+- list_add_tail(&waiter.list, &sem->wait_list);
++ rwsem_add_waiter(sem, &waiter);
+
+ /* we're now waiting on the lock, but no longer actively locking */
+ count = atomic_long_add_return(adjustment, &sem->count);
+@@ -999,11 +1044,7 @@ queue:
+ return sem;
+
+ out_nolock:
+- list_del(&waiter.list);
+- if (list_empty(&sem->wait_list)) {
+- atomic_long_andnot(RWSEM_FLAG_WAITERS|RWSEM_FLAG_HANDOFF,
+- &sem->count);
+- }
++ rwsem_del_waiter(sem, &waiter);
+ raw_spin_unlock_irq(&sem->wait_lock);
+ __set_current_state(TASK_RUNNING);
+ lockevent_inc(rwsem_rlock_fail);
+@@ -1017,9 +1058,7 @@ static struct rw_semaphore *
+ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
+ {
+ long count;
+- enum writer_wait_state wstate;
+ struct rwsem_waiter waiter;
+- struct rw_semaphore *ret = sem;
+ DEFINE_WAKE_Q(wake_q);
+
+ /* do optimistic spinning and steal lock if possible */
+@@ -1035,16 +1074,13 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
+ waiter.task = current;
+ waiter.type = RWSEM_WAITING_FOR_WRITE;
+ waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;
++ waiter.handoff_set = false;
+
+ raw_spin_lock_irq(&sem->wait_lock);
+-
+- /* account for this before adding a new element to the list */
+- wstate = list_empty(&sem->wait_list) ? WRITER_FIRST : WRITER_NOT_FIRST;
+-
+- list_add_tail(&waiter.list, &sem->wait_list);
++ rwsem_add_waiter(sem, &waiter);
+
+ /* we're now waiting on the lock */
+- if (wstate == WRITER_NOT_FIRST) {
++ if (rwsem_first_waiter(sem) != &waiter) {
+ count = atomic_long_read(&sem->count);
+
+ /*
+@@ -1080,13 +1116,16 @@ wait:
+ /* wait until we successfully acquire the lock */
+ set_current_state(state);
+ for (;;) {
+- if (rwsem_try_write_lock(sem, wstate)) {
++ if (rwsem_try_write_lock(sem, &waiter)) {
+ /* rwsem_try_write_lock() implies ACQUIRE on success */
+ break;
+ }
+
+ raw_spin_unlock_irq(&sem->wait_lock);
+
++ if (signal_pending_state(state, current))
++ goto out_nolock;
++
+ /*
+ * After setting the handoff bit and failing to acquire
+ * the lock, attempt to spin on owner to accelerate lock
+@@ -1095,70 +1134,37 @@ wait:
+ * In this case, we attempt to acquire the lock again
+ * without sleeping.
+ */
+- if (wstate == WRITER_HANDOFF &&
+- rwsem_spin_on_owner(sem) == OWNER_NULL)
+- goto trylock_again;
+-
+- /* Block until there are no active lockers. */
+- for (;;) {
+- if (signal_pending_state(state, current))
+- goto out_nolock;
+-
+- schedule();
+- lockevent_inc(rwsem_sleep_writer);
+- set_current_state(state);
+- /*
+- * If HANDOFF bit is set, unconditionally do
+- * a trylock.
+- */
+- if (wstate == WRITER_HANDOFF)
+- break;
+-
+- if ((wstate == WRITER_NOT_FIRST) &&
+- (rwsem_first_waiter(sem) == &waiter))
+- wstate = WRITER_FIRST;
++ if (waiter.handoff_set) {
++ enum owner_state owner_state;
+
+- count = atomic_long_read(&sem->count);
+- if (!(count & RWSEM_LOCK_MASK))
+- break;
++ preempt_disable();
++ owner_state = rwsem_spin_on_owner(sem);
++ preempt_enable();
+
+- /*
+- * The setting of the handoff bit is deferred
+- * until rwsem_try_write_lock() is called.
+- */
+- if ((wstate == WRITER_FIRST) && (rt_task(current) ||
+- time_after(jiffies, waiter.timeout))) {
+- wstate = WRITER_HANDOFF;
+- lockevent_inc(rwsem_wlock_handoff);
+- break;
+- }
++ if (owner_state == OWNER_NULL)
++ goto trylock_again;
+ }
++
++ schedule();
++ lockevent_inc(rwsem_sleep_writer);
++ set_current_state(state);
+ trylock_again:
+ raw_spin_lock_irq(&sem->wait_lock);
+ }
+ __set_current_state(TASK_RUNNING);
+- list_del(&waiter.list);
+ raw_spin_unlock_irq(&sem->wait_lock);
+ lockevent_inc(rwsem_wlock);
+-
+- return ret;
++ return sem;
+
+ out_nolock:
+ __set_current_state(TASK_RUNNING);
+ raw_spin_lock_irq(&sem->wait_lock);
+- list_del(&waiter.list);
+-
+- if (unlikely(wstate == WRITER_HANDOFF))
+- atomic_long_add(-RWSEM_FLAG_HANDOFF, &sem->count);
+-
+- if (list_empty(&sem->wait_list))
+- atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count);
+- else
++ rwsem_del_waiter(sem, &waiter);
++ if (!list_empty(&sem->wait_list))
+ rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
+ raw_spin_unlock_irq(&sem->wait_lock);
+ wake_up_q(&wake_q);
+ lockevent_inc(rwsem_wlock_fail);
+-
+ return ERR_PTR(-EINTR);
+ }
+
+diff --git a/kernel/module.c b/kernel/module.c
+index 5c26a76e800b5..83991c2d5af9e 100644
+--- a/kernel/module.c
++++ b/kernel/module.c
+@@ -3683,12 +3683,6 @@ static noinline int do_init_module(struct module *mod)
+ }
+ freeinit->module_init = mod->init_layout.base;
+
+- /*
+- * We want to find out whether @mod uses async during init. Clear
+- * PF_USED_ASYNC. async_schedule*() will set it.
+- */
+- current->flags &= ~PF_USED_ASYNC;
+-
+ do_mod_ctors(mod);
+ /* Start the module */
+ if (mod->init != NULL)
+@@ -3714,22 +3708,13 @@ static noinline int do_init_module(struct module *mod)
+
+ /*
+ * We need to finish all async code before the module init sequence
+- * is done. This has potential to deadlock. For example, a newly
+- * detected block device can trigger request_module() of the
+- * default iosched from async probing task. Once userland helper
+- * reaches here, async_synchronize_full() will wait on the async
+- * task waiting on request_module() and deadlock.
+- *
+- * This deadlock is avoided by perfomring async_synchronize_full()
+- * iff module init queued any async jobs. This isn't a full
+- * solution as it will deadlock the same if module loading from
+- * async jobs nests more than once; however, due to the various
+- * constraints, this hack seems to be the best option for now.
+- * Please refer to the following thread for details.
++ * is done. This has potential to deadlock if synchronous module
++ * loading is requested from async (which is not allowed!).
+ *
+- * http://thread.gmane.org/gmane.linux.kernel/1420814
++ * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
++ * request_module() from async workers") for more details.
+ */
+- if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC))
++ if (!mod->async_probe_requested)
+ async_synchronize_full();
+
+ ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
+diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c
+index a332ccd829e24..97e62469a6b32 100644
+--- a/kernel/power/energy_model.c
++++ b/kernel/power/energy_model.c
+@@ -107,8 +107,7 @@ static void em_debug_remove_pd(struct device *dev) {}
+ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd,
+ int nr_states, struct em_data_callback *cb)
+ {
+- unsigned long opp_eff, prev_opp_eff = ULONG_MAX;
+- unsigned long power, freq, prev_freq = 0;
++ unsigned long power, freq, prev_freq = 0, prev_cost = ULONG_MAX;
+ struct em_perf_state *table;
+ int i, ret;
+ u64 fmax;
+@@ -153,27 +152,21 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd,
+
+ table[i].power = power;
+ table[i].frequency = prev_freq = freq;
+-
+- /*
+- * The hertz/watts efficiency ratio should decrease as the
+- * frequency grows on sane platforms. But this isn't always
+- * true in practice so warn the user if a higher OPP is more
+- * power efficient than a lower one.
+- */
+- opp_eff = freq / power;
+- if (opp_eff >= prev_opp_eff)
+- dev_dbg(dev, "EM: hertz/watts ratio non-monotonically decreasing: em_perf_state %d >= em_perf_state%d\n",
+- i, i - 1);
+- prev_opp_eff = opp_eff;
+ }
+
+ /* Compute the cost of each performance state. */
+ fmax = (u64) table[nr_states - 1].frequency;
+- for (i = 0; i < nr_states; i++) {
++ for (i = nr_states - 1; i >= 0; i--) {
+ unsigned long power_res = em_scale_power(table[i].power);
+
+ table[i].cost = div64_u64(fmax * power_res,
+ table[i].frequency);
++ if (table[i].cost >= prev_cost) {
++ dev_dbg(dev, "EM: OPP:%lu is inefficient\n",
++ table[i].frequency);
++ } else {
++ prev_cost = table[i].cost;
++ }
+ }
+
+ pd->table = table;
+diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
+index 559acef3fddb8..d926852f81191 100644
+--- a/kernel/power/hibernate.c
++++ b/kernel/power/hibernate.c
+@@ -691,7 +691,7 @@ static int load_image_and_restore(void)
+ goto Unlock;
+
+ error = swsusp_read(&flags);
+- swsusp_close(FMODE_READ);
++ swsusp_close(FMODE_READ | FMODE_EXCL);
+ if (!error)
+ error = hibernation_restore(flags & SF_PLATFORM_MODE);
+
+@@ -981,7 +981,7 @@ static int software_resume(void)
+ /* The snapshot device should not be opened while we're running */
+ if (!hibernate_acquire()) {
+ error = -EBUSY;
+- swsusp_close(FMODE_READ);
++ swsusp_close(FMODE_READ | FMODE_EXCL);
+ goto Unlock;
+ }
+
+@@ -1016,7 +1016,7 @@ static int software_resume(void)
+ pm_pr_dbg("Hibernation image not present or could not be loaded.\n");
+ return error;
+ Close_Finish:
+- swsusp_close(FMODE_READ);
++ swsusp_close(FMODE_READ | FMODE_EXCL);
+ goto Finish;
+ }
+
+@@ -1326,7 +1326,7 @@ static int __init resumedelay_setup(char *str)
+ int rc = kstrtouint(str, 0, &resume_delay);
+
+ if (rc)
+- return rc;
++ pr_warn("resumedelay: bad option string '%s'\n", str);
+ return 1;
+ }
+
+diff --git a/kernel/power/main.c b/kernel/power/main.c
+index 44169f3081fdc..7e646079fbeb2 100644
+--- a/kernel/power/main.c
++++ b/kernel/power/main.c
+@@ -504,7 +504,10 @@ static ssize_t pm_wakeup_irq_show(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+ {
+- return pm_wakeup_irq ? sprintf(buf, "%u\n", pm_wakeup_irq) : -ENODATA;
++ if (!pm_wakeup_irq())
++ return -ENODATA;
++
++ return sprintf(buf, "%u\n", pm_wakeup_irq());
+ }
+
+ power_attr_ro(pm_wakeup_irq);
+diff --git a/kernel/power/process.c b/kernel/power/process.c
+index 37401c99b7d7d..ee78a39463e63 100644
+--- a/kernel/power/process.c
++++ b/kernel/power/process.c
+@@ -134,7 +134,7 @@ int freeze_processes(void)
+ if (!pm_freezing)
+ atomic_inc(&system_freezing_cnt);
+
+- pm_wakeup_clear(true);
++ pm_wakeup_clear(0);
+ pr_info("Freezing user space processes ... ");
+ pm_freezing = true;
+ error = try_to_freeze_tasks(true);
+diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
+index f7a9860782135..330d499376924 100644
+--- a/kernel/power/snapshot.c
++++ b/kernel/power/snapshot.c
+@@ -978,8 +978,7 @@ static void memory_bm_recycle(struct memory_bitmap *bm)
+ * Register a range of page frames the contents of which should not be saved
+ * during hibernation (to be used in the early initialization code).
+ */
+-void __init __register_nosave_region(unsigned long start_pfn,
+- unsigned long end_pfn, int use_kmalloc)
++void __init register_nosave_region(unsigned long start_pfn, unsigned long end_pfn)
+ {
+ struct nosave_region *region;
+
+@@ -995,18 +994,12 @@ void __init __register_nosave_region(unsigned long start_pfn,
+ goto Report;
+ }
+ }
+- if (use_kmalloc) {
+- /* During init, this shouldn't fail */
+- region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
+- BUG_ON(!region);
+- } else {
+- /* This allocation cannot fail */
+- region = memblock_alloc(sizeof(struct nosave_region),
+- SMP_CACHE_BYTES);
+- if (!region)
+- panic("%s: Failed to allocate %zu bytes\n", __func__,
+- sizeof(struct nosave_region));
+- }
++ /* This allocation cannot fail */
++ region = memblock_alloc(sizeof(struct nosave_region),
++ SMP_CACHE_BYTES);
++ if (!region)
++ panic("%s: Failed to allocate %zu bytes\n", __func__,
++ sizeof(struct nosave_region));
+ region->start_pfn = start_pfn;
+ region->end_pfn = end_pfn;
+ list_add_tail(&region->list, &nosave_regions);
+diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
+index eb75f394a0590..13d905dd32675 100644
+--- a/kernel/power/suspend.c
++++ b/kernel/power/suspend.c
+@@ -138,8 +138,6 @@ static void s2idle_loop(void)
+ break;
+ }
+
+- pm_wakeup_clear(false);
+-
+ s2idle_enter();
+ }
+
+diff --git a/kernel/power/suspend_test.c b/kernel/power/suspend_test.c
+index d20526c5be15b..b663a97f5867a 100644
+--- a/kernel/power/suspend_test.c
++++ b/kernel/power/suspend_test.c
+@@ -157,22 +157,22 @@ static int __init setup_test_suspend(char *value)
+ value++;
+ suspend_type = strsep(&value, ",");
+ if (!suspend_type)
+- return 0;
++ return 1;
+
+ repeat = strsep(&value, ",");
+ if (repeat) {
+ if (kstrtou32(repeat, 0, &test_repeat_count_max))
+- return 0;
++ return 1;
+ }
+
+ for (i = PM_SUSPEND_MIN; i < PM_SUSPEND_MAX; i++)
+ if (!strcmp(pm_labels[i], suspend_type)) {
+ test_state_label = pm_labels[i];
+- return 0;
++ return 1;
+ }
+
+ printk(warn_bad_state, suspend_type);
+- return 0;
++ return 1;
+ }
+ __setup("test_suspend", setup_test_suspend);
+
+diff --git a/kernel/power/swap.c b/kernel/power/swap.c
+index 3cb89baebc796..f3a1086f7cdb2 100644
+--- a/kernel/power/swap.c
++++ b/kernel/power/swap.c
+@@ -299,7 +299,7 @@ static int hib_submit_io(int op, int op_flags, pgoff_t page_off, void *addr,
+ return error;
+ }
+
+-static blk_status_t hib_wait_io(struct hib_bio_batch *hb)
++static int hib_wait_io(struct hib_bio_batch *hb)
+ {
+ /*
+ * We are relying on the behavior of blk_plug that a thread with
+@@ -1521,9 +1521,10 @@ end:
+ int swsusp_check(void)
+ {
+ int error;
++ void *holder;
+
+ hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device,
+- FMODE_READ, NULL);
++ FMODE_READ | FMODE_EXCL, &holder);
+ if (!IS_ERR(hib_resume_bdev)) {
+ set_blocksize(hib_resume_bdev, PAGE_SIZE);
+ clear_page(swsusp_header);
+@@ -1545,7 +1546,7 @@ int swsusp_check(void)
+
+ put:
+ if (error)
+- blkdev_put(hib_resume_bdev, FMODE_READ);
++ blkdev_put(hib_resume_bdev, FMODE_READ | FMODE_EXCL);
+ else
+ pr_debug("Image signature found, resuming\n");
+ } else {
+diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c
+index 105df4dfc7839..52571dcad768b 100644
+--- a/kernel/power/wakelock.c
++++ b/kernel/power/wakelock.c
+@@ -39,23 +39,20 @@ ssize_t pm_show_wakelocks(char *buf, bool show_active)
+ {
+ struct rb_node *node;
+ struct wakelock *wl;
+- char *str = buf;
+- char *end = buf + PAGE_SIZE;
++ int len = 0;
+
+ mutex_lock(&wakelocks_lock);
+
+ for (node = rb_first(&wakelocks_tree); node; node = rb_next(node)) {
+ wl = rb_entry(node, struct wakelock, node);
+ if (wl->ws->active == show_active)
+- str += scnprintf(str, end - str, "%s ", wl->name);
++ len += sysfs_emit_at(buf, len, "%s ", wl->name);
+ }
+- if (str > buf)
+- str--;
+
+- str += scnprintf(str, end - str, "\n");
++ len += sysfs_emit_at(buf, len, "\n");
+
+ mutex_unlock(&wakelocks_lock);
+- return (str - buf);
++ return len;
+ }
+
+ #if CONFIG_PM_WAKELOCKS_LIMIT > 0
+diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
+index a8d0a58deebc7..7aeb13542ce7a 100644
+--- a/kernel/printk/printk.c
++++ b/kernel/printk/printk.c
+@@ -146,8 +146,10 @@ static int __control_devkmsg(char *str)
+
+ static int __init control_devkmsg(char *str)
+ {
+- if (__control_devkmsg(str) < 0)
++ if (__control_devkmsg(str) < 0) {
++ pr_warn("printk.devkmsg: bad option string '%s'\n", str);
+ return 1;
++ }
+
+ /*
+ * Set sysctl string accordingly:
+@@ -166,7 +168,7 @@ static int __init control_devkmsg(char *str)
+ */
+ devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
+
+- return 0;
++ return 1;
+ }
+ __setup("printk.devkmsg=", control_devkmsg);
+
+@@ -3252,6 +3254,11 @@ void defer_console_output(void)
+ preempt_enable();
+ }
+
++void printk_trigger_flush(void)
++{
++ defer_console_output();
++}
++
+ int vprintk_deferred(const char *fmt, va_list args)
+ {
+ int r;
+diff --git a/kernel/ptrace.c b/kernel/ptrace.c
+index f8589bf8d7dce..516ad5e65849f 100644
+--- a/kernel/ptrace.c
++++ b/kernel/ptrace.c
+@@ -371,6 +371,26 @@ bool ptrace_may_access(struct task_struct *task, unsigned int mode)
+ return !err;
+ }
+
++static int check_ptrace_options(unsigned long data)
++{
++ if (data & ~(unsigned long)PTRACE_O_MASK)
++ return -EINVAL;
++
++ if (unlikely(data & PTRACE_O_SUSPEND_SECCOMP)) {
++ if (!IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) ||
++ !IS_ENABLED(CONFIG_SECCOMP))
++ return -EINVAL;
++
++ if (!capable(CAP_SYS_ADMIN))
++ return -EPERM;
++
++ if (seccomp_mode(&current->seccomp) != SECCOMP_MODE_DISABLED ||
++ current->ptrace & PT_SUSPEND_SECCOMP)
++ return -EPERM;
++ }
++ return 0;
++}
++
+ static int ptrace_attach(struct task_struct *task, long request,
+ unsigned long addr,
+ unsigned long flags)
+@@ -382,8 +402,16 @@ static int ptrace_attach(struct task_struct *task, long request,
+ if (seize) {
+ if (addr != 0)
+ goto out;
++ /*
++ * This duplicates the check in check_ptrace_options() because
++ * ptrace_attach() and ptrace_setoptions() have historically
++ * used different error codes for unknown ptrace options.
++ */
+ if (flags & ~(unsigned long)PTRACE_O_MASK)
+ goto out;
++ retval = check_ptrace_options(flags);
++ if (retval)
++ return retval;
+ flags = PT_PTRACED | PT_SEIZED | (flags << PT_OPT_FLAG_SHIFT);
+ } else {
+ flags = PT_PTRACED;
+@@ -656,22 +684,11 @@ int ptrace_writedata(struct task_struct *tsk, char __user *src, unsigned long ds
+ static int ptrace_setoptions(struct task_struct *child, unsigned long data)
+ {
+ unsigned flags;
++ int ret;
+
+- if (data & ~(unsigned long)PTRACE_O_MASK)
+- return -EINVAL;
+-
+- if (unlikely(data & PTRACE_O_SUSPEND_SECCOMP)) {
+- if (!IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) ||
+- !IS_ENABLED(CONFIG_SECCOMP))
+- return -EINVAL;
+-
+- if (!capable(CAP_SYS_ADMIN))
+- return -EPERM;
+-
+- if (seccomp_mode(&current->seccomp) != SECCOMP_MODE_DISABLED ||
+- current->ptrace & PT_SUSPEND_SECCOMP)
+- return -EPERM;
+- }
++ ret = check_ptrace_options(data);
++ if (ret)
++ return ret;
+
+ /* Avoid intermediate state when all opts are cleared */
+ flags = child->ptrace;
+diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h
+index 9a19328ff2514..5d405943823ec 100644
+--- a/kernel/rcu/rcu_segcblist.h
++++ b/kernel/rcu/rcu_segcblist.h
+@@ -56,13 +56,13 @@ static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
+ static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp,
+ int flags)
+ {
+- rsclp->flags |= flags;
++ WRITE_ONCE(rsclp->flags, rsclp->flags | flags);
+ }
+
+ static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp,
+ int flags)
+ {
+- rsclp->flags &= ~flags;
++ WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags);
+ }
+
+ static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp,
+diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
+index ab4215266ebee..f922937eb39ad 100644
+--- a/kernel/rcu/rcutorture.c
++++ b/kernel/rcu/rcutorture.c
+@@ -46,6 +46,7 @@
+ #include <linux/oom.h>
+ #include <linux/tick.h>
+ #include <linux/rcupdate_trace.h>
++#include <linux/nmi.h>
+
+ #include "rcu.h"
+
+@@ -109,6 +110,8 @@ torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
+ torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
+ torture_param(int, stall_cpu_holdoff, 10,
+ "Time to wait before starting stall (s).");
++torture_param(bool, stall_no_softlockup, false,
++ "Avoid softlockup warning during cpu stall.");
+ torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
+ torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
+ torture_param(int, stall_gp_kthread, 0,
+@@ -1432,28 +1435,34 @@ static void rcutorture_one_extend(int *readstate, int newstate,
+ /* First, put new protection in place to avoid critical-section gap. */
+ if (statesnew & RCUTORTURE_RDR_BH)
+ local_bh_disable();
++ if (statesnew & RCUTORTURE_RDR_RBH)
++ rcu_read_lock_bh();
+ if (statesnew & RCUTORTURE_RDR_IRQ)
+ local_irq_disable();
+ if (statesnew & RCUTORTURE_RDR_PREEMPT)
+ preempt_disable();
+- if (statesnew & RCUTORTURE_RDR_RBH)
+- rcu_read_lock_bh();
+ if (statesnew & RCUTORTURE_RDR_SCHED)
+ rcu_read_lock_sched();
+ if (statesnew & RCUTORTURE_RDR_RCU)
+ idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT;
+
+- /* Next, remove old protection, irq first due to bh conflict. */
++ /*
++ * Next, remove old protection, in decreasing order of strength
++ * to avoid unlock paths that aren't safe in the stronger
++ * context. Namely: BH can not be enabled with disabled interrupts.
++ * Additionally PREEMPT_RT requires that BH is enabled in preemptible
++ * context.
++ */
+ if (statesold & RCUTORTURE_RDR_IRQ)
+ local_irq_enable();
+- if (statesold & RCUTORTURE_RDR_BH)
+- local_bh_enable();
+ if (statesold & RCUTORTURE_RDR_PREEMPT)
+ preempt_enable();
+- if (statesold & RCUTORTURE_RDR_RBH)
+- rcu_read_unlock_bh();
+ if (statesold & RCUTORTURE_RDR_SCHED)
+ rcu_read_unlock_sched();
++ if (statesold & RCUTORTURE_RDR_BH)
++ local_bh_enable();
++ if (statesold & RCUTORTURE_RDR_RBH)
++ rcu_read_unlock_bh();
+ if (statesold & RCUTORTURE_RDR_RCU) {
+ bool lockit = !statesnew && !(torture_random(trsp) & 0xffff);
+
+@@ -1496,6 +1505,9 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
+ int mask = rcutorture_extend_mask_max();
+ unsigned long randmask1 = torture_random(trsp) >> 8;
+ unsigned long randmask2 = randmask1 >> 3;
++ unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
++ unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;
++ unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
+
+ WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT);
+ /* Mostly only one bit (need preemption!), sometimes lots of bits. */
+@@ -1503,11 +1515,26 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
+ mask = mask & randmask2;
+ else
+ mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
+- /* Can't enable bh w/irq disabled. */
+- if ((mask & RCUTORTURE_RDR_IRQ) &&
+- ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) ||
+- (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH))))
+- mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
++
++ /*
++ * Can't enable bh w/irq disabled.
++ */
++ if (mask & RCUTORTURE_RDR_IRQ)
++ mask |= oldmask & bhs;
++
++ /*
++ * Ideally these sequences would be detected in debug builds
++ * (regardless of RT), but until then don't stop testing
++ * them on non-RT.
++ */
++ if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
++ /* Can't modify BH in atomic context */
++ if (oldmask & preempts_irq)
++ mask &= ~bhs;
++ if ((oldmask | mask) & preempts_irq)
++ mask |= oldmask & bhs;
++ }
++
+ return mask ?: RCUTORTURE_RDR_RCU;
+ }
+
+@@ -2028,6 +2055,8 @@ static int rcu_torture_stall(void *args)
+ #else
+ schedule_timeout_uninterruptible(HZ);
+ #endif
++ } else if (stall_no_softlockup) {
++ touch_softlockup_watchdog();
+ }
+ if (stall_cpu_irqsoff)
+ local_irq_enable();
+diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
+index 806160c44b172..6591914af4864 100644
+--- a/kernel/rcu/tasks.h
++++ b/kernel/rcu/tasks.h
+@@ -197,6 +197,7 @@ static int __noreturn rcu_tasks_kthread(void *arg)
+ * This loop is terminated by the system going down. ;-)
+ */
+ for (;;) {
++ set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
+
+ /* Pick up any new callbacks. */
+ raw_spin_lock_irqsave(&rtp->cbs_lock, flags);
+@@ -236,8 +237,6 @@ static int __noreturn rcu_tasks_kthread(void *arg)
+ }
+ /* Paranoid sleep to keep this from entering a tight loop */
+ schedule_timeout_idle(rtp->gp_sleep);
+-
+- set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
+ }
+ }
+
+diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
+index bce848e50512e..a4a9d68b1fdce 100644
+--- a/kernel/rcu/tree.c
++++ b/kernel/rcu/tree.c
+@@ -327,7 +327,7 @@ static void rcu_dynticks_eqs_online(void)
+ */
+ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
+ {
+- return !(atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1);
++ return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1);
+ }
+
+ /*
+@@ -1594,10 +1594,11 @@ static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
+ struct rcu_data *rdp)
+ {
+ rcu_lockdep_assert_cblist_protected(rdp);
+- if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) ||
+- !raw_spin_trylock_rcu_node(rnp))
++ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))
+ return;
+- WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
++ // The grace period cannot end while we hold the rcu_node lock.
++ if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
++ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
+ raw_spin_unlock_rcu_node(rnp);
+ }
+
+@@ -1907,7 +1908,7 @@ static void rcu_gp_fqs(bool first_time)
+ struct rcu_node *rnp = rcu_get_root();
+
+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
+- rcu_state.n_force_qs++;
++ WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
+ if (first_time) {
+ /* Collect dyntick-idle snapshots. */
+ force_qs_rnp(dyntick_save_progress_counter);
+@@ -2475,7 +2476,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
+ div = READ_ONCE(rcu_divisor);
+ div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
+ bl = max(rdp->blimit, pending >> div);
+- if (unlikely(bl > 100)) {
++ if (in_serving_softirq() && unlikely(bl > 100)) {
+ long rrn = READ_ONCE(rcu_resched_ns);
+
+ rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
+@@ -2512,18 +2513,23 @@ static void rcu_do_batch(struct rcu_data *rdp)
+ /*
+ * Stop only if limit reached and CPU has something to do.
+ */
+- if (count >= bl && !offloaded &&
+- (need_resched() ||
+- (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
+- break;
+- if (unlikely(tlimit)) {
+- /* only call local_clock() every 32 callbacks */
+- if (likely((count & 31) || local_clock() < tlimit))
+- continue;
+- /* Exceeded the time limit, so leave. */
+- break;
+- }
+- if (!in_serving_softirq()) {
++ if (in_serving_softirq()) {
++ if (count >= bl && (need_resched() ||
++ (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
++ break;
++
++ /*
++ * Make sure we don't spend too much time here and deprive other
++ * softirq vectors of CPU cycles.
++ */
++ if (unlikely(tlimit)) {
++ /* only call local_clock() every 32 callbacks */
++ if (likely((count & 31) || local_clock() < tlimit))
++ continue;
++ /* Exceeded the time limit, so leave. */
++ break;
++ }
++ } else {
+ local_bh_enable();
+ lockdep_assert_irqs_enabled();
+ cond_resched_tasks_rcu_qs();
+@@ -2550,7 +2556,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
+ /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
+ if (count == 0 && rdp->qlen_last_fqs_check != 0) {
+ rdp->qlen_last_fqs_check = 0;
+- rdp->n_force_qs_snap = rcu_state.n_force_qs;
++ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
+ } else if (count < rdp->qlen_last_fqs_check - qhimark)
+ rdp->qlen_last_fqs_check = count;
+
+@@ -2898,10 +2904,10 @@ static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head,
+ } else {
+ /* Give the grace period a kick. */
+ rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
+- if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&
++ if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
+ rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
+ rcu_force_quiescent_state();
+- rdp->n_force_qs_snap = rcu_state.n_force_qs;
++ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
+ }
+ }
+@@ -4128,7 +4134,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
+ /* Set up local state, ensuring consistent view of global state. */
+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
+ rdp->qlen_last_fqs_check = 0;
+- rdp->n_force_qs_snap = rcu_state.n_force_qs;
++ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
+ rdp->blimit = blimit;
+ rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
+ rcu_dynticks_eqs_online();
+diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
+index 2796084ef85a5..16f94118ca34b 100644
+--- a/kernel/rcu/tree_exp.h
++++ b/kernel/rcu/tree_exp.h
+@@ -387,6 +387,7 @@ retry_ipi:
+ continue;
+ }
+ if (get_cpu() == cpu) {
++ mask_ofl_test |= mask;
+ put_cpu();
+ continue;
+ }
+@@ -760,7 +761,7 @@ static void sync_sched_exp_online_cleanup(int cpu)
+ my_cpu = get_cpu();
+ /* Quiescent state either not needed or already requested, leave. */
+ if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
+- __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) {
++ rdp->cpu_no_qs.b.exp) {
+ put_cpu();
+ return;
+ }
+diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
+index d070059163d70..ef2dd131e9553 100644
+--- a/kernel/rcu/tree_plugin.h
++++ b/kernel/rcu/tree_plugin.h
+@@ -554,16 +554,16 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
+ }
+
+- /* Unboost if we were boosted. */
+- if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
+- rt_mutex_futex_unlock(&rnp->boost_mtx.rtmutex);
+-
+ /*
+ * If this was the last task on the expedited lists,
+ * then we need to report up the rcu_node hierarchy.
+ */
+ if (!empty_exp && empty_exp_now)
+ rcu_report_exp_rnp(rnp, true);
++
++ /* Unboost if we were boosted. */
++ if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
++ rt_mutex_futex_unlock(&rnp->boost_mtx.rtmutex);
+ } else {
+ local_irq_restore(flags);
+ }
+@@ -1480,7 +1480,7 @@ static void rcu_bind_gp_kthread(void)
+ }
+
+ /* Record the current task on dyntick-idle entry. */
+-static void noinstr rcu_dynticks_task_enter(void)
++static __always_inline void rcu_dynticks_task_enter(void)
+ {
+ #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
+ WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
+@@ -1488,7 +1488,7 @@ static void noinstr rcu_dynticks_task_enter(void)
+ }
+
+ /* Record no current task on dyntick-idle exit. */
+-static void noinstr rcu_dynticks_task_exit(void)
++static __always_inline void rcu_dynticks_task_exit(void)
+ {
+ #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
+ WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
+@@ -1496,7 +1496,7 @@ static void noinstr rcu_dynticks_task_exit(void)
+ }
+
+ /* Turn on heavyweight RCU tasks trace readers on idle/user entry. */
+-static void rcu_dynticks_task_trace_enter(void)
++static __always_inline void rcu_dynticks_task_trace_enter(void)
+ {
+ #ifdef CONFIG_TASKS_TRACE_RCU
+ if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
+@@ -1505,7 +1505,7 @@ static void rcu_dynticks_task_trace_enter(void)
+ }
+
+ /* Turn off heavyweight RCU tasks trace readers on idle/user exit. */
+-static void rcu_dynticks_task_trace_exit(void)
++static __always_inline void rcu_dynticks_task_trace_exit(void)
+ {
+ #ifdef CONFIG_TASKS_TRACE_RCU
+ if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
+diff --git a/kernel/resource.c b/kernel/resource.c
+index ca9f5198a01ff..20e10e48f0523 100644
+--- a/kernel/resource.c
++++ b/kernel/resource.c
+@@ -56,14 +56,6 @@ struct resource_constraint {
+
+ static DEFINE_RWLOCK(resource_lock);
+
+-/*
+- * For memory hotplug, there is no way to free resource entries allocated
+- * by boot mem after the system is up. So for reusing the resource entry
+- * we need to remember the resource.
+- */
+-static struct resource *bootmem_resource_free;
+-static DEFINE_SPINLOCK(bootmem_resource_lock);
+-
+ static struct resource *next_resource(struct resource *p)
+ {
+ if (p->child)
+@@ -148,36 +140,19 @@ __initcall(ioresources_init);
+
+ static void free_resource(struct resource *res)
+ {
+- if (!res)
+- return;
+-
+- if (!PageSlab(virt_to_head_page(res))) {
+- spin_lock(&bootmem_resource_lock);
+- res->sibling = bootmem_resource_free;
+- bootmem_resource_free = res;
+- spin_unlock(&bootmem_resource_lock);
+- } else {
++ /**
++ * If the resource was allocated using memblock early during boot
++ * we'll leak it here: we can only return full pages back to the
++ * buddy and trying to be smart and reusing them eventually in
++ * alloc_resource() overcomplicates resource handling.
++ */
++ if (res && PageSlab(virt_to_head_page(res)))
+ kfree(res);
+- }
+ }
+
+ static struct resource *alloc_resource(gfp_t flags)
+ {
+- struct resource *res = NULL;
+-
+- spin_lock(&bootmem_resource_lock);
+- if (bootmem_resource_free) {
+- res = bootmem_resource_free;
+- bootmem_resource_free = res->sibling;
+- }
+- spin_unlock(&bootmem_resource_lock);
+-
+- if (res)
+- memset(res, 0, sizeof(struct resource));
+- else
+- res = kzalloc(sizeof(struct resource), flags);
+-
+- return res;
++ return kzalloc(sizeof(struct resource), flags);
+ }
+
+ /* Return the conflict entry if you can't request it */
+diff --git a/kernel/rseq.c b/kernel/rseq.c
+index 6d45ac3dae7fb..97ac20b4f7387 100644
+--- a/kernel/rseq.c
++++ b/kernel/rseq.c
+@@ -128,10 +128,10 @@ static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs)
+ int ret;
+
+ #ifdef CONFIG_64BIT
+- if (get_user(ptr, &t->rseq->rseq_cs.ptr64))
++ if (get_user(ptr, &t->rseq->rseq_cs))
+ return -EFAULT;
+ #else
+- if (copy_from_user(&ptr, &t->rseq->rseq_cs.ptr64, sizeof(ptr)))
++ if (copy_from_user(&ptr, &t->rseq->rseq_cs, sizeof(ptr)))
+ return -EFAULT;
+ #endif
+ if (!ptr) {
+@@ -217,9 +217,9 @@ static int clear_rseq_cs(struct task_struct *t)
+ * Set rseq_cs to NULL.
+ */
+ #ifdef CONFIG_64BIT
+- return put_user(0UL, &t->rseq->rseq_cs.ptr64);
++ return put_user(0UL, &t->rseq->rseq_cs);
+ #else
+- if (clear_user(&t->rseq->rseq_cs.ptr64, sizeof(t->rseq->rseq_cs.ptr64)))
++ if (clear_user(&t->rseq->rseq_cs, sizeof(t->rseq->rseq_cs)))
+ return -EFAULT;
+ return 0;
+ #endif
+diff --git a/kernel/sched/autogroup.c b/kernel/sched/autogroup.c
+index 2067080bb2358..8629b37d118e7 100644
+--- a/kernel/sched/autogroup.c
++++ b/kernel/sched/autogroup.c
+@@ -31,7 +31,7 @@ static inline void autogroup_destroy(struct kref *kref)
+ ag->tg->rt_se = NULL;
+ ag->tg->rt_rq = NULL;
+ #endif
+- sched_offline_group(ag->tg);
++ sched_release_group(ag->tg);
+ sched_destroy_group(ag->tg);
+ }
+
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index f21714ea3db85..779f3198b17de 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -36,6 +36,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
++EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_thermal_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
+ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
+@@ -1914,7 +1915,7 @@ static void __init init_uclamp_rq(struct rq *rq)
+ };
+ }
+
+- rq->uclamp_flags = 0;
++ rq->uclamp_flags = UCLAMP_FLAG_IDLE;
+ }
+
+ static void __init init_uclamp(void)
+@@ -3707,6 +3708,9 @@ out:
+
+ bool cpus_share_cache(int this_cpu, int that_cpu)
+ {
++ if (this_cpu == that_cpu)
++ return true;
++
+ return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
+ }
+
+@@ -4328,8 +4332,6 @@ int sysctl_schedstats(struct ctl_table *table, int write, void *buffer,
+ */
+ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+ {
+- unsigned long flags;
+-
+ __sched_fork(clone_flags, p);
+ /*
+ * We mark the process as NEW here. This guarantees that
+@@ -4375,23 +4377,6 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+
+ init_entity_runnable_average(&p->se);
+
+- /*
+- * The child is not yet in the pid-hash so no cgroup attach races,
+- * and the cgroup is pinned to this child due to cgroup_fork()
+- * is ran before sched_fork().
+- *
+- * Silence PROVE_RCU.
+- */
+- raw_spin_lock_irqsave(&p->pi_lock, flags);
+- rseq_migrate(p);
+- /*
+- * We're setting the CPU for the first time, we don't migrate,
+- * so use __set_task_cpu().
+- */
+- __set_task_cpu(p, smp_processor_id());
+- if (p->sched_class->task_fork)
+- p->sched_class->task_fork(p);
+- raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+ #ifdef CONFIG_SCHED_INFO
+ if (likely(sched_info_on()))
+@@ -4408,6 +4393,35 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+ return 0;
+ }
+
++void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
++{
++ unsigned long flags;
++
++ /*
++ * Because we're not yet on the pid-hash, p->pi_lock isn't strictly
++ * required yet, but lockdep gets upset if rules are violated.
++ */
++ raw_spin_lock_irqsave(&p->pi_lock, flags);
++#ifdef CONFIG_CGROUP_SCHED
++ if (1) {
++ struct task_group *tg;
++ tg = container_of(kargs->cset->subsys[cpu_cgrp_id],
++ struct task_group, css);
++ tg = autogroup_task_group(p, tg);
++ p->sched_task_group = tg;
++ }
++#endif
++ rseq_migrate(p);
++ /*
++ * We're setting the CPU for the first time, we don't migrate,
++ * so use __set_task_cpu().
++ */
++ __set_task_cpu(p, smp_processor_id());
++ if (p->sched_class->task_fork)
++ p->sched_class->task_fork(p);
++ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++}
++
+ void sched_post_fork(struct task_struct *p)
+ {
+ uclamp_post_fork(p);
+@@ -5913,7 +5927,7 @@ static bool try_steal_cookie(int this, int that)
+ if (p == src->core_pick || p == src->curr)
+ goto next;
+
+- if (!cpumask_test_cpu(this, &p->cpus_mask))
++ if (!is_cpu_allowed(p, this))
+ goto next;
+
+ if (p->core_occupation > dst->idle->core_occupation)
+@@ -6656,11 +6670,11 @@ static int __init setup_preempt_mode(char *str)
+ int mode = sched_dynamic_mode(str);
+ if (mode < 0) {
+ pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
+- return 1;
++ return 0;
+ }
+
+ sched_dynamic_update(mode);
+- return 0;
++ return 1;
+ }
+ __setup("preempt=", setup_preempt_mode);
+
+@@ -8195,9 +8209,7 @@ int __cond_resched_lock(spinlock_t *lock)
+
+ if (spin_needbreak(lock) || resched) {
+ spin_unlock(lock);
+- if (resched)
+- preempt_schedule_common();
+- else
++ if (!_cond_resched())
+ cpu_relax();
+ ret = 1;
+ spin_lock(lock);
+@@ -8215,9 +8227,7 @@ int __cond_resched_rwlock_read(rwlock_t *lock)
+
+ if (rwlock_needbreak(lock) || resched) {
+ read_unlock(lock);
+- if (resched)
+- preempt_schedule_common();
+- else
++ if (!_cond_resched())
+ cpu_relax();
+ ret = 1;
+ read_lock(lock);
+@@ -8235,9 +8245,7 @@ int __cond_resched_rwlock_write(rwlock_t *lock)
+
+ if (rwlock_needbreak(lock) || resched) {
+ write_unlock(lock);
+- if (resched)
+- preempt_schedule_common();
+- else
++ if (!_cond_resched())
+ cpu_relax();
+ ret = 1;
+ write_lock(lock);
+@@ -8637,9 +8645,6 @@ void __init init_idle(struct task_struct *idle, int cpu)
+ idle->flags |= PF_IDLE | PF_KTHREAD | PF_NO_SETAFFINITY;
+ kthread_set_per_cpu(idle, cpu);
+
+- scs_task_reset(idle);
+- kasan_unpoison_task_stack(idle);
+-
+ #ifdef CONFIG_SMP
+ /*
+ * It's possible that init_idle() gets called multiple times on a task,
+@@ -8795,7 +8800,6 @@ void idle_task_exit(void)
+ finish_arch_post_lock_switch();
+ }
+
+- scs_task_reset(current);
+ /* finish_cpu(), as ran on the BP, will clean up the active_mm state */
+ }
+
+@@ -9716,6 +9720,22 @@ static void sched_free_group(struct task_group *tg)
+ kmem_cache_free(task_group_cache, tg);
+ }
+
++static void sched_free_group_rcu(struct rcu_head *rcu)
++{
++ sched_free_group(container_of(rcu, struct task_group, rcu));
++}
++
++static void sched_unregister_group(struct task_group *tg)
++{
++ unregister_fair_sched_group(tg);
++ unregister_rt_sched_group(tg);
++ /*
++ * We have to wait for yet another RCU grace period to expire, as
++ * print_cfs_stats() might run concurrently.
++ */
++ call_rcu(&tg->rcu, sched_free_group_rcu);
++}
++
+ /* allocate runqueue etc for a new task group */
+ struct task_group *sched_create_group(struct task_group *parent)
+ {
+@@ -9759,25 +9779,35 @@ void sched_online_group(struct task_group *tg, struct task_group *parent)
+ }
+
+ /* rcu callback to free various structures associated with a task group */
+-static void sched_free_group_rcu(struct rcu_head *rhp)
++static void sched_unregister_group_rcu(struct rcu_head *rhp)
+ {
+ /* Now it should be safe to free those cfs_rqs: */
+- sched_free_group(container_of(rhp, struct task_group, rcu));
++ sched_unregister_group(container_of(rhp, struct task_group, rcu));
+ }
+
+ void sched_destroy_group(struct task_group *tg)
+ {
+ /* Wait for possible concurrent references to cfs_rqs complete: */
+- call_rcu(&tg->rcu, sched_free_group_rcu);
++ call_rcu(&tg->rcu, sched_unregister_group_rcu);
+ }
+
+-void sched_offline_group(struct task_group *tg)
++void sched_release_group(struct task_group *tg)
+ {
+ unsigned long flags;
+
+- /* End participation in shares distribution: */
+- unregister_fair_sched_group(tg);
+-
++ /*
++ * Unlink first, to avoid walk_tg_tree_from() from finding us (via
++ * sched_cfs_period_timer()).
++ *
++ * For this to be effective, we have to wait for all pending users of
++ * this task group to leave their RCU critical section to ensure no new
++ * user will see our dying task group any more. Specifically ensure
++ * that tg_unthrottle_up() won't add decayed cfs_rq's to it.
++ *
++ * We therefore defer calling unregister_fair_sched_group() to
++ * sched_unregister_group() which is guarantied to get called only after the
++ * current RCU grace period has expired.
++ */
+ spin_lock_irqsave(&task_group_lock, flags);
+ list_del_rcu(&tg->list);
+ list_del_rcu(&tg->siblings);
+@@ -9896,7 +9926,7 @@ static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
+ {
+ struct task_group *tg = css_tg(css);
+
+- sched_offline_group(tg);
++ sched_release_group(tg);
+ }
+
+ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
+@@ -9906,7 +9936,7 @@ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
+ /*
+ * Relies on the RCU grace period between css_released() and this.
+ */
+- sched_free_group(tg);
++ sched_unregister_group(tg);
+ }
+
+ /*
+diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c
+index 893eece65bfda..cacc2076ad214 100644
+--- a/kernel/sched/cpuacct.c
++++ b/kernel/sched/cpuacct.c
+@@ -21,15 +21,11 @@ static const char * const cpuacct_stat_desc[] = {
+ [CPUACCT_STAT_SYSTEM] = "system",
+ };
+
+-struct cpuacct_usage {
+- u64 usages[CPUACCT_STAT_NSTATS];
+-};
+-
+ /* track CPU usage of a group of tasks and its child groups */
+ struct cpuacct {
+ struct cgroup_subsys_state css;
+ /* cpuusage holds pointer to a u64-type object on every CPU */
+- struct cpuacct_usage __percpu *cpuusage;
++ u64 __percpu *cpuusage;
+ struct kernel_cpustat __percpu *cpustat;
+ };
+
+@@ -49,7 +45,7 @@ static inline struct cpuacct *parent_ca(struct cpuacct *ca)
+ return css_ca(ca->css.parent);
+ }
+
+-static DEFINE_PER_CPU(struct cpuacct_usage, root_cpuacct_cpuusage);
++static DEFINE_PER_CPU(u64, root_cpuacct_cpuusage);
+ static struct cpuacct root_cpuacct = {
+ .cpustat = &kernel_cpustat,
+ .cpuusage = &root_cpuacct_cpuusage,
+@@ -68,7 +64,7 @@ cpuacct_css_alloc(struct cgroup_subsys_state *parent_css)
+ if (!ca)
+ goto out;
+
+- ca->cpuusage = alloc_percpu(struct cpuacct_usage);
++ ca->cpuusage = alloc_percpu(u64);
+ if (!ca->cpuusage)
+ goto out_free_ca;
+
+@@ -99,7 +95,8 @@ static void cpuacct_css_free(struct cgroup_subsys_state *css)
+ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu,
+ enum cpuacct_stat_index index)
+ {
+- struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
++ u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
++ u64 *cpustat = per_cpu_ptr(ca->cpustat, cpu)->cpustat;
+ u64 data;
+
+ /*
+@@ -115,14 +112,17 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu,
+ raw_spin_rq_lock_irq(cpu_rq(cpu));
+ #endif
+
+- if (index == CPUACCT_STAT_NSTATS) {
+- int i = 0;
+-
+- data = 0;
+- for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
+- data += cpuusage->usages[i];
+- } else {
+- data = cpuusage->usages[index];
++ switch (index) {
++ case CPUACCT_STAT_USER:
++ data = cpustat[CPUTIME_USER] + cpustat[CPUTIME_NICE];
++ break;
++ case CPUACCT_STAT_SYSTEM:
++ data = cpustat[CPUTIME_SYSTEM] + cpustat[CPUTIME_IRQ] +
++ cpustat[CPUTIME_SOFTIRQ];
++ break;
++ case CPUACCT_STAT_NSTATS:
++ data = *cpuusage;
++ break;
+ }
+
+ #ifndef CONFIG_64BIT
+@@ -132,10 +132,14 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu,
+ return data;
+ }
+
+-static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
++static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu)
+ {
+- struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
+- int i;
++ u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
++ u64 *cpustat = per_cpu_ptr(ca->cpustat, cpu)->cpustat;
++
++ /* Don't allow to reset global kernel_cpustat */
++ if (ca == &root_cpuacct)
++ return;
+
+ #ifndef CONFIG_64BIT
+ /*
+@@ -143,9 +147,10 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
+ */
+ raw_spin_rq_lock_irq(cpu_rq(cpu));
+ #endif
+-
+- for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
+- cpuusage->usages[i] = val;
++ *cpuusage = 0;
++ cpustat[CPUTIME_USER] = cpustat[CPUTIME_NICE] = 0;
++ cpustat[CPUTIME_SYSTEM] = cpustat[CPUTIME_IRQ] = 0;
++ cpustat[CPUTIME_SOFTIRQ] = 0;
+
+ #ifndef CONFIG_64BIT
+ raw_spin_rq_unlock_irq(cpu_rq(cpu));
+@@ -196,7 +201,7 @@ static int cpuusage_write(struct cgroup_subsys_state *css, struct cftype *cft,
+ return -EINVAL;
+
+ for_each_possible_cpu(cpu)
+- cpuacct_cpuusage_write(ca, cpu, 0);
++ cpuacct_cpuusage_write(ca, cpu);
+
+ return 0;
+ }
+@@ -243,25 +248,10 @@ static int cpuacct_all_seq_show(struct seq_file *m, void *V)
+ seq_puts(m, "\n");
+
+ for_each_possible_cpu(cpu) {
+- struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
+-
+ seq_printf(m, "%d", cpu);
+-
+- for (index = 0; index < CPUACCT_STAT_NSTATS; index++) {
+-#ifndef CONFIG_64BIT
+- /*
+- * Take rq->lock to make 64-bit read safe on 32-bit
+- * platforms.
+- */
+- raw_spin_rq_lock_irq(cpu_rq(cpu));
+-#endif
+-
+- seq_printf(m, " %llu", cpuusage->usages[index]);
+-
+-#ifndef CONFIG_64BIT
+- raw_spin_rq_unlock_irq(cpu_rq(cpu));
+-#endif
+- }
++ for (index = 0; index < CPUACCT_STAT_NSTATS; index++)
++ seq_printf(m, " %llu",
++ cpuacct_cpuusage_read(ca, cpu, index));
+ seq_puts(m, "\n");
+ }
+ return 0;
+@@ -338,17 +328,13 @@ static struct cftype files[] = {
+ */
+ void cpuacct_charge(struct task_struct *tsk, u64 cputime)
+ {
++ unsigned int cpu = task_cpu(tsk);
+ struct cpuacct *ca;
+- int index = CPUACCT_STAT_SYSTEM;
+- struct pt_regs *regs = get_irq_regs() ? : task_pt_regs(tsk);
+-
+- if (regs && user_mode(regs))
+- index = CPUACCT_STAT_USER;
+
+ rcu_read_lock();
+
+ for (ca = task_ca(tsk); ca; ca = parent_ca(ca))
+- __this_cpu_add(ca->cpuusage->usages[index], cputime);
++ *per_cpu_ptr(ca->cpuusage, cpu) += cputime;
+
+ rcu_read_unlock();
+ }
+diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
+index e7af18857371e..7f6bb37d3a2f7 100644
+--- a/kernel/sched/cpufreq_schedutil.c
++++ b/kernel/sched/cpufreq_schedutil.c
+@@ -289,6 +289,7 @@ static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time)
+ * into the same scale so we can compare.
+ */
+ boost = (sg_cpu->iowait_boost * sg_cpu->max) >> SCHED_CAPACITY_SHIFT;
++ boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL);
+ if (sg_cpu->util < boost)
+ sg_cpu->util = boost;
+ }
+diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
+index 872e481d5098c..042a6dbce8f32 100644
+--- a/kernel/sched/cputime.c
++++ b/kernel/sched/cputime.c
+@@ -148,10 +148,10 @@ void account_guest_time(struct task_struct *p, u64 cputime)
+
+ /* Add guest time to cpustat. */
+ if (task_nice(p) > 0) {
+- cpustat[CPUTIME_NICE] += cputime;
++ task_group_account_field(p, CPUTIME_NICE, cputime);
+ cpustat[CPUTIME_GUEST_NICE] += cputime;
+ } else {
+- cpustat[CPUTIME_USER] += cputime;
++ task_group_account_field(p, CPUTIME_USER, cputime);
+ cpustat[CPUTIME_GUEST] += cputime;
+ }
+ }
+diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
+index e94314633b39d..1f811b375bf0a 100644
+--- a/kernel/sched/deadline.c
++++ b/kernel/sched/deadline.c
+@@ -2145,12 +2145,6 @@ static int push_dl_task(struct rq *rq)
+ return 0;
+
+ retry:
+- if (is_migration_disabled(next_task))
+- return 0;
+-
+- if (WARN_ON(next_task == rq->curr))
+- return 0;
+-
+ /*
+ * If next_task preempts rq->curr, and rq->curr
+ * can move away, it makes sense to just reschedule
+@@ -2163,6 +2157,12 @@ retry:
+ return 0;
+ }
+
++ if (is_migration_disabled(next_task))
++ return 0;
++
++ if (WARN_ON(next_task == rq->curr))
++ return 0;
++
+ /* We might release rq lock */
+ get_task_struct(next_task);
+
+diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
+index 17a653b67006a..7a2d32d2025fe 100644
+--- a/kernel/sched/debug.c
++++ b/kernel/sched/debug.c
+@@ -921,25 +921,15 @@ void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
+ static void sched_show_numa(struct task_struct *p, struct seq_file *m)
+ {
+ #ifdef CONFIG_NUMA_BALANCING
+- struct mempolicy *pol;
+-
+ if (p->mm)
+ P(mm->numa_scan_seq);
+
+- task_lock(p);
+- pol = p->mempolicy;
+- if (pol && !(pol->flags & MPOL_F_MORON))
+- pol = NULL;
+- mpol_get(pol);
+- task_unlock(p);
+-
+ P(numa_pages_migrated);
+ P(numa_preferred_nid);
+ P(total_numa_faults);
+ SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
+ task_node(p), task_numa_group_id(p));
+ show_numa_stats(p, m);
+- mpol_put(pol);
+ #endif
+ }
+
+diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
+index f6a05d9b54436..9a4fa22a69edf 100644
+--- a/kernel/sched/fair.c
++++ b/kernel/sched/fair.c
+@@ -3422,7 +3422,6 @@ void set_task_rq_fair(struct sched_entity *se,
+ se->avg.last_update_time = n_last_update_time;
+ }
+
+-
+ /*
+ * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
+ * propagate its contribution. The key to this propagation is the invariant
+@@ -3490,7 +3489,6 @@ void set_task_rq_fair(struct sched_entity *se,
+ * XXX: only do this for the part of runnable > running ?
+ *
+ */
+-
+ static inline void
+ update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
+ {
+@@ -3722,7 +3720,19 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
+
+ r = removed_util;
+ sub_positive(&sa->util_avg, r);
+- sa->util_sum = sa->util_avg * divider;
++ sub_positive(&sa->util_sum, r * divider);
++ /*
++ * Because of rounding, se->util_sum might ends up being +1 more than
++ * cfs->util_sum. Although this is not a problem by itself, detaching
++ * a lot of tasks with the rounding problem between 2 updates of
++ * util_avg (~1ms) can make cfs->util_sum becoming null whereas
++ * cfs_util_avg is not.
++ * Check that util_sum is still above its lower bound for the new
++ * util_avg. Given that period_contrib might have moved since the last
++ * sync, we are only sure that util_sum must be above or equal to
++ * util_avg * minimum possible divider
++ */
++ sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
+
+ r = removed_runnable;
+ sub_positive(&sa->runnable_avg, r);
+@@ -3784,11 +3794,11 @@ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
+
+ se->avg.runnable_sum = se->avg.runnable_avg * divider;
+
+- se->avg.load_sum = divider;
+- if (se_weight(se)) {
+- se->avg.load_sum =
+- div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));
+- }
++ se->avg.load_sum = se->avg.load_avg * divider;
++ if (se_weight(se) < se->avg.load_sum)
++ se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se));
++ else
++ se->avg.load_sum = 1;
+
+ enqueue_load_avg(cfs_rq, se);
+ cfs_rq->avg.util_avg += se->avg.util_avg;
+@@ -6429,8 +6439,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
+ * pattern is IO completions.
+ */
+ if (is_per_cpu_kthread(current) &&
++ in_task() &&
+ prev == smp_processor_id() &&
+- this_rq()->nr_running <= 1) {
++ this_rq()->nr_running <= 1 &&
++ asym_fits_capacity(task_util, prev)) {
+ return prev;
+ }
+
+@@ -8993,9 +9005,10 @@ static bool update_pick_idlest(struct sched_group *idlest,
+ * This is an approximation as the number of running tasks may not be
+ * related to the number of busy CPUs due to sched_setaffinity.
+ */
+-static inline bool allow_numa_imbalance(int dst_running, int dst_weight)
++static inline bool
++allow_numa_imbalance(unsigned int running, unsigned int weight)
+ {
+- return (dst_running < (dst_weight >> 2));
++ return (running < (weight >> 2));
+ }
+
+ /*
+@@ -9129,12 +9142,13 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
+ return idlest;
+ #endif
+ /*
+- * Otherwise, keep the task on this node to stay close
+- * its wakeup source and improve locality. If there is
+- * a real need of migration, periodic load balance will
+- * take care of it.
++ * Otherwise, keep the task close to the wakeup source
++ * and improve locality if the number of running tasks
++ * would remain below threshold where an imbalance is
++ * allowed. If there is a real need of migration,
++ * periodic load balance will take care of it.
+ */
+- if (allow_numa_imbalance(local_sgs.sum_nr_running, sd->span_weight))
++ if (allow_numa_imbalance(local_sgs.sum_nr_running + 1, local_sgs.group_weight))
+ return NULL;
+ }
+
+@@ -9340,7 +9354,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
+ /* Consider allowing a small imbalance between NUMA groups */
+ if (env->sd->flags & SD_NUMA) {
+ env->imbalance = adjust_numa_imbalance(env->imbalance,
+- busiest->sum_nr_running, busiest->group_weight);
++ local->sum_nr_running + 1, local->group_weight);
+ }
+
+ return;
+@@ -11358,8 +11372,6 @@ void free_fair_sched_group(struct task_group *tg)
+ {
+ int i;
+
+- destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
+-
+ for_each_possible_cpu(i) {
+ if (tg->cfs_rq)
+ kfree(tg->cfs_rq[i]);
+@@ -11436,6 +11448,8 @@ void unregister_fair_sched_group(struct task_group *tg)
+ struct rq *rq;
+ int cpu;
+
++ destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
++
+ for_each_possible_cpu(cpu) {
+ if (tg->se[cpu])
+ remove_entity_load_avg(tg->se[cpu]);
+diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c
+index b5add64d9698c..3d2825408e3a2 100644
+--- a/kernel/sched/membarrier.c
++++ b/kernel/sched/membarrier.c
+@@ -147,11 +147,11 @@
+ #endif
+
+ #ifdef CONFIG_RSEQ
+-#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \
++#define MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK \
+ (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \
+- | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
++ | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ)
+ #else
+-#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
++#define MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
+ #endif
+
+ #define MEMBARRIER_CMD_BITMASK \
+@@ -159,7 +159,8 @@
+ | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
+ | MEMBARRIER_CMD_PRIVATE_EXPEDITED \
+ | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED \
+- | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK)
++ | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK \
++ | MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
+
+ static void ipi_mb(void *info)
+ {
+diff --git a/kernel/sched/pelt.h b/kernel/sched/pelt.h
+index e06071bf3472c..c336f5f481bca 100644
+--- a/kernel/sched/pelt.h
++++ b/kernel/sched/pelt.h
+@@ -37,9 +37,11 @@ update_irq_load_avg(struct rq *rq, u64 running)
+ }
+ #endif
+
++#define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
++
+ static inline u32 get_pelt_divider(struct sched_avg *avg)
+ {
+- return LOAD_AVG_MAX - 1024 + avg->period_contrib;
++ return PELT_MIN_DIVIDER + avg->period_contrib;
+ }
+
+ static inline void cfs_se_util_change(struct sched_avg *avg)
+diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
+index 1652f2bb54b79..422f3b0445cf1 100644
+--- a/kernel/sched/psi.c
++++ b/kernel/sched/psi.c
+@@ -34,13 +34,19 @@
+ * delayed on that resource such that nobody is advancing and the CPU
+ * goes idle. This leaves both workload and CPU unproductive.
+ *
+- * Naturally, the FULL state doesn't exist for the CPU resource at the
+- * system level, but exist at the cgroup level, means all non-idle tasks
+- * in a cgroup are delayed on the CPU resource which used by others outside
+- * of the cgroup or throttled by the cgroup cpu.max configuration.
+- *
+ * SOME = nr_delayed_tasks != 0
+- * FULL = nr_delayed_tasks != 0 && nr_running_tasks == 0
++ * FULL = nr_delayed_tasks != 0 && nr_productive_tasks == 0
++ *
++ * What it means for a task to be productive is defined differently
++ * for each resource. For IO, productive means a running task. For
++ * memory, productive means a running task that isn't a reclaimer. For
++ * CPU, productive means an oncpu task.
++ *
++ * Naturally, the FULL state doesn't exist for the CPU resource at the
++ * system level, but exist at the cgroup level. At the cgroup level,
++ * FULL means all non-idle tasks in the cgroup are delayed on the CPU
++ * resource which is being used by others outside of the cgroup or
++ * throttled by the cgroup cpu.max configuration.
+ *
+ * The percentage of wallclock time spent in those compound stall
+ * states gives pressure numbers between 0 and 100 for each resource,
+@@ -81,13 +87,13 @@
+ *
+ * threads = min(nr_nonidle_tasks, nr_cpus)
+ * SOME = min(nr_delayed_tasks / threads, 1)
+- * FULL = (threads - min(nr_running_tasks, threads)) / threads
++ * FULL = (threads - min(nr_productive_tasks, threads)) / threads
+ *
+ * For the 257 number crunchers on 256 CPUs, this yields:
+ *
+ * threads = min(257, 256)
+ * SOME = min(1 / 256, 1) = 0.4%
+- * FULL = (256 - min(257, 256)) / 256 = 0%
++ * FULL = (256 - min(256, 256)) / 256 = 0%
+ *
+ * For the 1 out of 4 memory-delayed tasks, this yields:
+ *
+@@ -112,7 +118,7 @@
+ * For each runqueue, we track:
+ *
+ * tSOME[cpu] = time(nr_delayed_tasks[cpu] != 0)
+- * tFULL[cpu] = time(nr_delayed_tasks[cpu] && !nr_running_tasks[cpu])
++ * tFULL[cpu] = time(nr_delayed_tasks[cpu] && !nr_productive_tasks[cpu])
+ * tNONIDLE[cpu] = time(nr_nonidle_tasks[cpu] != 0)
+ *
+ * and then periodically aggregate:
+@@ -233,7 +239,8 @@ static bool test_state(unsigned int *tasks, enum psi_states state)
+ case PSI_MEM_SOME:
+ return unlikely(tasks[NR_MEMSTALL]);
+ case PSI_MEM_FULL:
+- return unlikely(tasks[NR_MEMSTALL] && !tasks[NR_RUNNING]);
++ return unlikely(tasks[NR_MEMSTALL] &&
++ tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);
+ case PSI_CPU_SOME:
+ return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]);
+ case PSI_CPU_FULL:
+@@ -710,10 +717,11 @@ static void psi_group_change(struct psi_group *group, int cpu,
+ if (groupc->tasks[t]) {
+ groupc->tasks[t]--;
+ } else if (!psi_bug) {
+- printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n",
++ printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n",
+ cpu, t, groupc->tasks[0],
+ groupc->tasks[1], groupc->tasks[2],
+- groupc->tasks[3], clear, set);
++ groupc->tasks[3], groupc->tasks[4],
++ clear, set);
+ psi_bug = 1;
+ }
+ }
+@@ -854,12 +862,15 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next,
+ int clear = TSK_ONCPU, set = 0;
+
+ /*
+- * When we're going to sleep, psi_dequeue() lets us handle
+- * TSK_RUNNING and TSK_IOWAIT here, where we can combine it
+- * with TSK_ONCPU and save walking common ancestors twice.
++ * When we're going to sleep, psi_dequeue() lets us
++ * handle TSK_RUNNING, TSK_MEMSTALL_RUNNING and
++ * TSK_IOWAIT here, where we can combine it with
++ * TSK_ONCPU and save walking common ancestors twice.
+ */
+ if (sleep) {
+ clear |= TSK_RUNNING;
++ if (prev->in_memstall)
++ clear |= TSK_MEMSTALL_RUNNING;
+ if (prev->in_iowait)
+ set |= TSK_IOWAIT;
+ }
+@@ -908,7 +919,7 @@ void psi_memstall_enter(unsigned long *flags)
+ rq = this_rq_lock_irq(&rf);
+
+ current->in_memstall = 1;
+- psi_task_change(current, 0, TSK_MEMSTALL);
++ psi_task_change(current, 0, TSK_MEMSTALL | TSK_MEMSTALL_RUNNING);
+
+ rq_unlock_irq(rq, &rf);
+ }
+@@ -937,7 +948,7 @@ void psi_memstall_leave(unsigned long *flags)
+ rq = this_rq_lock_irq(&rf);
+
+ current->in_memstall = 0;
+- psi_task_change(current, TSK_MEMSTALL, 0);
++ psi_task_change(current, TSK_MEMSTALL | TSK_MEMSTALL_RUNNING, 0);
+
+ rq_unlock_irq(rq, &rf);
+ }
+@@ -1071,44 +1082,6 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
+ return 0;
+ }
+
+-static int psi_io_show(struct seq_file *m, void *v)
+-{
+- return psi_show(m, &psi_system, PSI_IO);
+-}
+-
+-static int psi_memory_show(struct seq_file *m, void *v)
+-{
+- return psi_show(m, &psi_system, PSI_MEM);
+-}
+-
+-static int psi_cpu_show(struct seq_file *m, void *v)
+-{
+- return psi_show(m, &psi_system, PSI_CPU);
+-}
+-
+-static int psi_open(struct file *file, int (*psi_show)(struct seq_file *, void *))
+-{
+- if (file->f_mode & FMODE_WRITE && !capable(CAP_SYS_RESOURCE))
+- return -EPERM;
+-
+- return single_open(file, psi_show, NULL);
+-}
+-
+-static int psi_io_open(struct inode *inode, struct file *file)
+-{
+- return psi_open(file, psi_io_show);
+-}
+-
+-static int psi_memory_open(struct inode *inode, struct file *file)
+-{
+- return psi_open(file, psi_memory_show);
+-}
+-
+-static int psi_cpu_open(struct inode *inode, struct file *file)
+-{
+- return psi_open(file, psi_cpu_show);
+-}
+-
+ struct psi_trigger *psi_trigger_create(struct psi_group *group,
+ char *buf, size_t nbytes, enum psi_res res)
+ {
+@@ -1151,7 +1124,6 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group,
+ t->event = 0;
+ t->last_event_time = 0;
+ init_waitqueue_head(&t->event_wait);
+- kref_init(&t->refcount);
+
+ mutex_lock(&group->trigger_lock);
+
+@@ -1180,15 +1152,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group,
+ return t;
+ }
+
+-static void psi_trigger_destroy(struct kref *ref)
++void psi_trigger_destroy(struct psi_trigger *t)
+ {
+- struct psi_trigger *t = container_of(ref, struct psi_trigger, refcount);
+- struct psi_group *group = t->group;
++ struct psi_group *group;
+ struct task_struct *task_to_destroy = NULL;
+
+- if (static_branch_likely(&psi_disabled))
++ /*
++ * We do not check psi_disabled since it might have been disabled after
++ * the trigger got created.
++ */
++ if (!t)
+ return;
+
++ group = t->group;
+ /*
+ * Wakeup waiters to stop polling. Can happen if cgroup is deleted
+ * from under a polling process.
+@@ -1224,9 +1200,9 @@ static void psi_trigger_destroy(struct kref *ref)
+ mutex_unlock(&group->trigger_lock);
+
+ /*
+- * Wait for both *trigger_ptr from psi_trigger_replace and
+- * poll_task RCUs to complete their read-side critical sections
+- * before destroying the trigger and optionally the poll_task
++ * Wait for psi_schedule_poll_work RCU to complete its read-side
++ * critical section before destroying the trigger and optionally the
++ * poll_task.
+ */
+ synchronize_rcu();
+ /*
+@@ -1243,18 +1219,6 @@ static void psi_trigger_destroy(struct kref *ref)
+ kfree(t);
+ }
+
+-void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *new)
+-{
+- struct psi_trigger *old = *trigger_ptr;
+-
+- if (static_branch_likely(&psi_disabled))
+- return;
+-
+- rcu_assign_pointer(*trigger_ptr, new);
+- if (old)
+- kref_put(&old->refcount, psi_trigger_destroy);
+-}
+-
+ __poll_t psi_trigger_poll(void **trigger_ptr,
+ struct file *file, poll_table *wait)
+ {
+@@ -1264,27 +1228,57 @@ __poll_t psi_trigger_poll(void **trigger_ptr,
+ if (static_branch_likely(&psi_disabled))
+ return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI;
+
+- rcu_read_lock();
+-
+- t = rcu_dereference(*(void __rcu __force **)trigger_ptr);
+- if (!t) {
+- rcu_read_unlock();
++ t = smp_load_acquire(trigger_ptr);
++ if (!t)
+ return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI;
+- }
+- kref_get(&t->refcount);
+-
+- rcu_read_unlock();
+
+ poll_wait(file, &t->event_wait, wait);
+
+ if (cmpxchg(&t->event, 1, 0) == 1)
+ ret |= EPOLLPRI;
+
+- kref_put(&t->refcount, psi_trigger_destroy);
+-
+ return ret;
+ }
+
++#ifdef CONFIG_PROC_FS
++static int psi_io_show(struct seq_file *m, void *v)
++{
++ return psi_show(m, &psi_system, PSI_IO);
++}
++
++static int psi_memory_show(struct seq_file *m, void *v)
++{
++ return psi_show(m, &psi_system, PSI_MEM);
++}
++
++static int psi_cpu_show(struct seq_file *m, void *v)
++{
++ return psi_show(m, &psi_system, PSI_CPU);
++}
++
++static int psi_open(struct file *file, int (*psi_show)(struct seq_file *, void *))
++{
++ if (file->f_mode & FMODE_WRITE && !capable(CAP_SYS_RESOURCE))
++ return -EPERM;
++
++ return single_open(file, psi_show, NULL);
++}
++
++static int psi_io_open(struct inode *inode, struct file *file)
++{
++ return psi_open(file, psi_io_show);
++}
++
++static int psi_memory_open(struct inode *inode, struct file *file)
++{
++ return psi_open(file, psi_memory_show);
++}
++
++static int psi_cpu_open(struct inode *inode, struct file *file)
++{
++ return psi_open(file, psi_cpu_show);
++}
++
+ static ssize_t psi_write(struct file *file, const char __user *user_buf,
+ size_t nbytes, enum psi_res res)
+ {
+@@ -1305,14 +1299,24 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
+
+ buf[buf_size - 1] = '\0';
+
+- new = psi_trigger_create(&psi_system, buf, nbytes, res);
+- if (IS_ERR(new))
+- return PTR_ERR(new);
+-
+ seq = file->private_data;
++
+ /* Take seq->lock to protect seq->private from concurrent writes */
+ mutex_lock(&seq->lock);
+- psi_trigger_replace(&seq->private, new);
++
++ /* Allow only one trigger per file descriptor */
++ if (seq->private) {
++ mutex_unlock(&seq->lock);
++ return -EBUSY;
++ }
++
++ new = psi_trigger_create(&psi_system, buf, nbytes, res);
++ if (IS_ERR(new)) {
++ mutex_unlock(&seq->lock);
++ return PTR_ERR(new);
++ }
++
++ smp_store_release(&seq->private, new);
+ mutex_unlock(&seq->lock);
+
+ return nbytes;
+@@ -1347,7 +1351,7 @@ static int psi_fop_release(struct inode *inode, struct file *file)
+ {
+ struct seq_file *seq = file->private_data;
+
+- psi_trigger_replace(&seq->private, NULL);
++ psi_trigger_destroy(seq->private);
+ return single_release(inode, file);
+ }
+
+@@ -1389,3 +1393,5 @@ static int __init psi_proc_init(void)
+ return 0;
+ }
+ module_init(psi_proc_init);
++
++#endif /* CONFIG_PROC_FS */
+diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
+index 3daf42a0f4623..2758cf5f7987f 100644
+--- a/kernel/sched/rt.c
++++ b/kernel/sched/rt.c
+@@ -52,11 +52,8 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
+ rt_b->rt_period_timer.function = sched_rt_period_timer;
+ }
+
+-static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
++static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)
+ {
+- if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
+- return;
+-
+ raw_spin_lock(&rt_b->rt_runtime_lock);
+ if (!rt_b->rt_period_active) {
+ rt_b->rt_period_active = 1;
+@@ -75,6 +72,14 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
+ raw_spin_unlock(&rt_b->rt_runtime_lock);
+ }
+
++static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
++{
++ if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
++ return;
++
++ do_start_rt_bandwidth(rt_b);
++}
++
+ void init_rt_rq(struct rt_rq *rt_rq)
+ {
+ struct rt_prio_array *array;
+@@ -137,13 +142,17 @@ static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
+ return rt_rq->rq;
+ }
+
+-void free_rt_sched_group(struct task_group *tg)
++void unregister_rt_sched_group(struct task_group *tg)
+ {
+- int i;
+-
+ if (tg->rt_se)
+ destroy_rt_bandwidth(&tg->rt_bandwidth);
+
++}
++
++void free_rt_sched_group(struct task_group *tg)
++{
++ int i;
++
+ for_each_possible_cpu(i) {
+ if (tg->rt_rq)
+ kfree(tg->rt_rq[i]);
+@@ -250,6 +259,8 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
+ return &rq->rt;
+ }
+
++void unregister_rt_sched_group(struct task_group *tg) { }
++
+ void free_rt_sched_group(struct task_group *tg) { }
+
+ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
+@@ -1023,13 +1034,17 @@ static void update_curr_rt(struct rq *rq)
+
+ for_each_sched_rt_entity(rt_se) {
+ struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
++ int exceeded;
+
+ if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
+ raw_spin_lock(&rt_rq->rt_runtime_lock);
+ rt_rq->rt_time += delta_exec;
+- if (sched_rt_runtime_exceeded(rt_rq))
++ exceeded = sched_rt_runtime_exceeded(rt_rq);
++ if (exceeded)
+ resched_curr(rq);
+ raw_spin_unlock(&rt_rq->rt_runtime_lock);
++ if (exceeded)
++ do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
+ }
+ }
+ }
+@@ -1885,6 +1900,16 @@ static int push_rt_task(struct rq *rq, bool pull)
+ return 0;
+
+ retry:
++ /*
++ * It's possible that the next_task slipped in of
++ * higher priority than current. If that's the case
++ * just reschedule current.
++ */
++ if (unlikely(next_task->prio < rq->curr->prio)) {
++ resched_curr(rq);
++ return 0;
++ }
++
+ if (is_migration_disabled(next_task)) {
+ struct task_struct *push_task = NULL;
+ int cpu;
+@@ -1892,6 +1917,18 @@ retry:
+ if (!pull || rq->push_busy)
+ return 0;
+
++ /*
++ * Invoking find_lowest_rq() on anything but an RT task doesn't
++ * make sense. Per the above priority check, curr has to
++ * be of higher priority than next_task, so no need to
++ * reschedule when bailing out.
++ *
++ * Note that the stoppers are masqueraded as SCHED_FIFO
++ * (cf. sched_set_stop_task()), so we can't rely on rt_task().
++ */
++ if (rq->curr->sched_class != &rt_sched_class)
++ return 0;
++
+ cpu = find_lowest_rq(rq->curr);
+ if (cpu == -1 || cpu == rq->cpu)
+ return 0;
+@@ -1916,16 +1953,6 @@ retry:
+ if (WARN_ON(next_task == rq->curr))
+ return 0;
+
+- /*
+- * It's possible that the next_task slipped in of
+- * higher priority than current. If that's the case
+- * just reschedule current.
+- */
+- if (unlikely(next_task->prio < rq->curr->prio)) {
+- resched_curr(rq);
+- return 0;
+- }
+-
+ /* We might release rq lock */
+ get_task_struct(next_task);
+
+@@ -2779,8 +2806,12 @@ static int sched_rt_global_validate(void)
+
+ static void sched_rt_do_global(void)
+ {
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
+ def_rt_bandwidth.rt_runtime = global_rt_runtime();
+ def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
++ raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
+ }
+
+ int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
+diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
+index 3d3e5793e1172..4f432826933da 100644
+--- a/kernel/sched/sched.h
++++ b/kernel/sched/sched.h
+@@ -486,6 +486,7 @@ extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b);
+ extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
+ extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
+
++extern void unregister_rt_sched_group(struct task_group *tg);
+ extern void free_rt_sched_group(struct task_group *tg);
+ extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
+ extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
+@@ -501,7 +502,7 @@ extern struct task_group *sched_create_group(struct task_group *parent);
+ extern void sched_online_group(struct task_group *tg,
+ struct task_group *parent);
+ extern void sched_destroy_group(struct task_group *tg);
+-extern void sched_offline_group(struct task_group *tg);
++extern void sched_release_group(struct task_group *tg);
+
+ extern void sched_move_task(struct task_struct *tsk);
+
+diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
+index d8f8eb0c655ba..606a3982d13a5 100644
+--- a/kernel/sched/stats.h
++++ b/kernel/sched/stats.h
+@@ -69,6 +69,9 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup)
+ if (static_branch_likely(&psi_disabled))
+ return;
+
++ if (p->in_memstall)
++ set |= TSK_MEMSTALL_RUNNING;
++
+ if (!wakeup || p->sched_psi_wake_requeue) {
+ if (p->in_memstall)
+ set |= TSK_MEMSTALL;
+@@ -99,7 +102,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep)
+ return;
+
+ if (p->in_memstall)
+- clear |= TSK_MEMSTALL;
++ clear |= (TSK_MEMSTALL | TSK_MEMSTALL_RUNNING);
+
+ psi_task_change(p, clear, 0);
+ }
+diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
+index 76577d1642a5d..eca38107b32f1 100644
+--- a/kernel/sched/wait.c
++++ b/kernel/sched/wait.c
+@@ -238,6 +238,13 @@ void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode)
+ }
+ EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
+
++void __wake_up_pollfree(struct wait_queue_head *wq_head)
++{
++ __wake_up(wq_head, TASK_NORMAL, 0, poll_to_key(EPOLLHUP | POLLFREE));
++ /* POLLFREE must have cleared the queue. */
++ WARN_ON_ONCE(waitqueue_active(wq_head));
++}
++
+ /*
+ * Note: we use "set_current_state()" _after_ the wait-queue add,
+ * because we need a memory barrier there on SMP, so that any
+diff --git a/kernel/scs.c b/kernel/scs.c
+index e2a71fc82fa06..579841be88646 100644
+--- a/kernel/scs.c
++++ b/kernel/scs.c
+@@ -78,6 +78,7 @@ void scs_free(void *s)
+ if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
+ return;
+
++ kasan_unpoison_vmalloc(s, SCS_SIZE);
+ vfree_atomic(s);
+ }
+
+diff --git a/kernel/seccomp.c b/kernel/seccomp.c
+index 4d8f44a177274..db10e73d06e02 100644
+--- a/kernel/seccomp.c
++++ b/kernel/seccomp.c
+@@ -29,6 +29,9 @@
+ #include <linux/syscalls.h>
+ #include <linux/sysctl.h>
+
++/* Not exposed in headers: strictly internal use only. */
++#define SECCOMP_MODE_DEAD (SECCOMP_MODE_FILTER + 1)
++
+ #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
+ #include <asm/syscall.h>
+ #endif
+@@ -1010,6 +1013,7 @@ static void __secure_computing_strict(int this_syscall)
+ #ifdef SECCOMP_DEBUG
+ dump_stack();
+ #endif
++ current->seccomp.mode = SECCOMP_MODE_DEAD;
+ seccomp_log(this_syscall, SIGKILL, SECCOMP_RET_KILL_THREAD, true);
+ do_exit(SIGKILL);
+ }
+@@ -1261,6 +1265,7 @@ static int __seccomp_filter(int this_syscall, const struct seccomp_data *sd,
+ case SECCOMP_RET_KILL_THREAD:
+ case SECCOMP_RET_KILL_PROCESS:
+ default:
++ current->seccomp.mode = SECCOMP_MODE_DEAD;
+ seccomp_log(this_syscall, SIGSYS, action, true);
+ /* Dump core only if this is the last remaining thread. */
+ if (action != SECCOMP_RET_KILL_THREAD ||
+@@ -1309,6 +1314,11 @@ int __secure_computing(const struct seccomp_data *sd)
+ return 0;
+ case SECCOMP_MODE_FILTER:
+ return __seccomp_filter(this_syscall, sd, false);
++ /* Surviving SECCOMP_RET_KILL_* must be proactively impossible. */
++ case SECCOMP_MODE_DEAD:
++ WARN_ON_ONCE(1);
++ do_exit(SIGKILL);
++ return -1;
+ default:
+ BUG();
+ }
+diff --git a/kernel/signal.c b/kernel/signal.c
+index 487bf4f5dadf4..6e3dbb3d12170 100644
+--- a/kernel/signal.c
++++ b/kernel/signal.c
+@@ -1298,6 +1298,12 @@ int do_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p
+ return ret;
+ }
+
++enum sig_handler {
++ HANDLER_CURRENT, /* If reachable use the current handler */
++ HANDLER_SIG_DFL, /* Always use SIG_DFL handler semantics */
++ HANDLER_EXIT, /* Only visible as the process exit code */
++};
++
+ /*
+ * Force a signal that the process can't ignore: if necessary
+ * we unblock the signal and change any SIG_IGN to SIG_DFL.
+@@ -1310,7 +1316,8 @@ int do_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p
+ * that is why we also clear SIGNAL_UNKILLABLE.
+ */
+ static int
+-force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, bool sigdfl)
++force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
++ enum sig_handler handler)
+ {
+ unsigned long int flags;
+ int ret, blocked, ignored;
+@@ -1321,8 +1328,10 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, bool
+ action = &t->sighand->action[sig-1];
+ ignored = action->sa.sa_handler == SIG_IGN;
+ blocked = sigismember(&t->blocked, sig);
+- if (blocked || ignored || sigdfl) {
++ if (blocked || ignored || (handler != HANDLER_CURRENT)) {
+ action->sa.sa_handler = SIG_DFL;
++ if (handler == HANDLER_EXIT)
++ action->sa.sa_flags |= SA_IMMUTABLE;
+ if (blocked) {
+ sigdelset(&t->blocked, sig);
+ recalc_sigpending_and_wake(t);
+@@ -1330,9 +1339,10 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, bool
+ }
+ /*
+ * Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
+- * debugging to leave init killable.
++ * debugging to leave init killable. But HANDLER_EXIT is always fatal.
+ */
+- if (action->sa.sa_handler == SIG_DFL && !t->ptrace)
++ if (action->sa.sa_handler == SIG_DFL &&
++ (!t->ptrace || (handler == HANDLER_EXIT)))
+ t->signal->flags &= ~SIGNAL_UNKILLABLE;
+ ret = send_signal(sig, info, t, PIDTYPE_PID);
+ spin_unlock_irqrestore(&t->sighand->siglock, flags);
+@@ -1342,7 +1352,7 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, bool
+
+ int force_sig_info(struct kernel_siginfo *info)
+ {
+- return force_sig_info_to_task(info, current, false);
++ return force_sig_info_to_task(info, current, HANDLER_CURRENT);
+ }
+
+ /*
+@@ -1649,6 +1659,32 @@ void force_sig(int sig)
+ }
+ EXPORT_SYMBOL(force_sig);
+
++void force_fatal_sig(int sig)
++{
++ struct kernel_siginfo info;
++
++ clear_siginfo(&info);
++ info.si_signo = sig;
++ info.si_errno = 0;
++ info.si_code = SI_KERNEL;
++ info.si_pid = 0;
++ info.si_uid = 0;
++ force_sig_info_to_task(&info, current, HANDLER_SIG_DFL);
++}
++
++void force_exit_sig(int sig)
++{
++ struct kernel_siginfo info;
++
++ clear_siginfo(&info);
++ info.si_signo = sig;
++ info.si_errno = 0;
++ info.si_code = SI_KERNEL;
++ info.si_pid = 0;
++ info.si_uid = 0;
++ force_sig_info_to_task(&info, current, HANDLER_EXIT);
++}
++
+ /*
+ * When things go south during signal handling, we
+ * will force a SIGSEGV. And if the signal that caused
+@@ -1657,15 +1693,10 @@ EXPORT_SYMBOL(force_sig);
+ */
+ void force_sigsegv(int sig)
+ {
+- struct task_struct *p = current;
+-
+- if (sig == SIGSEGV) {
+- unsigned long flags;
+- spin_lock_irqsave(&p->sighand->siglock, flags);
+- p->sighand->action[sig - 1].sa.sa_handler = SIG_DFL;
+- spin_unlock_irqrestore(&p->sighand->siglock, flags);
+- }
+- force_sig(SIGSEGV);
++ if (sig == SIGSEGV)
++ force_fatal_sig(SIGSEGV);
++ else
++ force_sig(SIGSEGV);
+ }
+
+ int force_sig_fault_to_task(int sig, int code, void __user *addr
+@@ -1684,7 +1715,7 @@ int force_sig_fault_to_task(int sig, int code, void __user *addr
+ info.si_flags = flags;
+ info.si_isr = isr;
+ #endif
+- return force_sig_info_to_task(&info, t, false);
++ return force_sig_info_to_task(&info, t, HANDLER_CURRENT);
+ }
+
+ int force_sig_fault(int sig, int code, void __user *addr
+@@ -1804,7 +1835,8 @@ int force_sig_seccomp(int syscall, int reason, bool force_coredump)
+ info.si_errno = reason;
+ info.si_arch = syscall_get_arch(current);
+ info.si_syscall = syscall;
+- return force_sig_info_to_task(&info, current, force_coredump);
++ return force_sig_info_to_task(&info, current,
++ force_coredump ? HANDLER_EXIT : HANDLER_CURRENT);
+ }
+
+ /* For the crazy architectures that include trap information in
+@@ -2169,15 +2201,6 @@ static inline bool may_ptrace_stop(void)
+ return true;
+ }
+
+-/*
+- * Return non-zero if there is a SIGKILL that should be waking us up.
+- * Called with the siglock held.
+- */
+-static bool sigkill_pending(struct task_struct *tsk)
+-{
+- return sigismember(&tsk->pending.signal, SIGKILL) ||
+- sigismember(&tsk->signal->shared_pending.signal, SIGKILL);
+-}
+
+ /*
+ * This must be called with current->sighand->siglock held.
+@@ -2204,17 +2227,16 @@ static void ptrace_stop(int exit_code, int why, int clear_code, kernel_siginfo_t
+ * calling arch_ptrace_stop, so we must release it now.
+ * To preserve proper semantics, we must do this before
+ * any signal bookkeeping like checking group_stop_count.
+- * Meanwhile, a SIGKILL could come in before we retake the
+- * siglock. That must prevent us from sleeping in TASK_TRACED.
+- * So after regaining the lock, we must check for SIGKILL.
+ */
+ spin_unlock_irq(&current->sighand->siglock);
+ arch_ptrace_stop(exit_code, info);
+ spin_lock_irq(&current->sighand->siglock);
+- if (sigkill_pending(current))
+- return;
+ }
+
++ /*
++ * schedule() will not sleep if there is a pending signal that
++ * can awaken the task.
++ */
+ set_special_state(TASK_TRACED);
+
+ /*
+@@ -2688,19 +2710,19 @@ relock:
+ goto relock;
+ }
+
+- /* Has this task already been marked for death? */
+- if (signal_group_exit(signal)) {
+- ksig->info.si_signo = signr = SIGKILL;
+- sigdelset(&current->pending.signal, SIGKILL);
+- trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
+- &sighand->action[SIGKILL - 1]);
+- recalc_sigpending();
+- goto fatal;
+- }
+-
+ for (;;) {
+ struct k_sigaction *ka;
+
++ /* Has this task already been marked for death? */
++ if (signal_group_exit(signal)) {
++ ksig->info.si_signo = signr = SIGKILL;
++ sigdelset(&current->pending.signal, SIGKILL);
++ trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
++ &sighand->action[SIGKILL - 1]);
++ recalc_sigpending();
++ goto fatal;
++ }
++
+ if (unlikely(current->jobctl & JOBCTL_STOP_PENDING) &&
+ do_signal_stop(0))
+ goto relock;
+@@ -2739,7 +2761,8 @@ relock:
+ if (!signr)
+ break; /* will return 0 */
+
+- if (unlikely(current->ptrace) && signr != SIGKILL) {
++ if (unlikely(current->ptrace) && (signr != SIGKILL) &&
++ !(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) {
+ signr = ptrace_signal(signr, &ksig->info);
+ if (!signr)
+ continue;
+@@ -4089,6 +4112,10 @@ int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
+ k = &p->sighand->action[sig-1];
+
+ spin_lock_irq(&p->sighand->siglock);
++ if (k->sa.sa_flags & SA_IMMUTABLE) {
++ spin_unlock_irq(&p->sighand->siglock);
++ return -EINVAL;
++ }
+ if (oact)
+ *oact = *k;
+
+diff --git a/kernel/smp.c b/kernel/smp.c
+index f43ede0ab183a..b68d63e965db6 100644
+--- a/kernel/smp.c
++++ b/kernel/smp.c
+@@ -579,7 +579,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline)
+
+ /* There shouldn't be any pending callbacks on an offline CPU. */
+ if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
+- !warned && !llist_empty(head))) {
++ !warned && entry != NULL)) {
+ warned = true;
+ WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
+
+diff --git a/kernel/stackleak.c b/kernel/stackleak.c
+index ce161a8e8d975..dd07239ddff9f 100644
+--- a/kernel/stackleak.c
++++ b/kernel/stackleak.c
+@@ -48,7 +48,7 @@ int stack_erasing_sysctl(struct ctl_table *table, int write,
+ #define skip_erasing() false
+ #endif /* CONFIG_STACKLEAK_RUNTIME_DISABLE */
+
+-asmlinkage void notrace stackleak_erase(void)
++asmlinkage void noinstr stackleak_erase(void)
+ {
+ /* It would be nice not to have 'kstack_ptr' and 'boundary' on stack */
+ unsigned long kstack_ptr = current->lowest_stack;
+@@ -102,9 +102,8 @@ asmlinkage void notrace stackleak_erase(void)
+ /* Reset the 'lowest_stack' value for the next syscall */
+ current->lowest_stack = current_top_of_stack() - THREAD_SIZE/64;
+ }
+-NOKPROBE_SYMBOL(stackleak_erase);
+
+-void __used __no_caller_saved_registers notrace stackleak_track_stack(void)
++void __used __no_caller_saved_registers noinstr stackleak_track_stack(void)
+ {
+ unsigned long sp = current_stack_pointer;
+
+diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
+index 9f8117c7cfdde..9c625257023d2 100644
+--- a/kernel/stacktrace.c
++++ b/kernel/stacktrace.c
+@@ -13,6 +13,7 @@
+ #include <linux/export.h>
+ #include <linux/kallsyms.h>
+ #include <linux/stacktrace.h>
++#include <linux/interrupt.h>
+
+ /**
+ * stack_trace_print - Print the entries in the stack trace
+@@ -373,3 +374,32 @@ unsigned int stack_trace_save_user(unsigned long *store, unsigned int size)
+ #endif /* CONFIG_USER_STACKTRACE_SUPPORT */
+
+ #endif /* !CONFIG_ARCH_STACKWALK */
++
++static inline bool in_irqentry_text(unsigned long ptr)
++{
++ return (ptr >= (unsigned long)&__irqentry_text_start &&
++ ptr < (unsigned long)&__irqentry_text_end) ||
++ (ptr >= (unsigned long)&__softirqentry_text_start &&
++ ptr < (unsigned long)&__softirqentry_text_end);
++}
++
++/**
++ * filter_irq_stacks - Find first IRQ stack entry in trace
++ * @entries: Pointer to stack trace array
++ * @nr_entries: Number of entries in the storage array
++ *
++ * Return: Number of trace entries until IRQ stack starts.
++ */
++unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries)
++{
++ unsigned int i;
++
++ for (i = 0; i < nr_entries; i++) {
++ if (in_irqentry_text(entries[i])) {
++ /* Include the irqentry function into the stack. */
++ return i + 1;
++ }
++ }
++ return nr_entries;
++}
++EXPORT_SYMBOL_GPL(filter_irq_stacks);
+diff --git a/kernel/static_call.c b/kernel/static_call.c
+index 43ba0b1e0edbb..e9c3e69f38379 100644
+--- a/kernel/static_call.c
++++ b/kernel/static_call.c
+@@ -1,548 +1,8 @@
+ // SPDX-License-Identifier: GPL-2.0
+-#include <linux/init.h>
+ #include <linux/static_call.h>
+-#include <linux/bug.h>
+-#include <linux/smp.h>
+-#include <linux/sort.h>
+-#include <linux/slab.h>
+-#include <linux/module.h>
+-#include <linux/cpu.h>
+-#include <linux/processor.h>
+-#include <asm/sections.h>
+-
+-extern struct static_call_site __start_static_call_sites[],
+- __stop_static_call_sites[];
+-extern struct static_call_tramp_key __start_static_call_tramp_key[],
+- __stop_static_call_tramp_key[];
+-
+-static bool static_call_initialized;
+-
+-/* mutex to protect key modules/sites */
+-static DEFINE_MUTEX(static_call_mutex);
+-
+-static void static_call_lock(void)
+-{
+- mutex_lock(&static_call_mutex);
+-}
+-
+-static void static_call_unlock(void)
+-{
+- mutex_unlock(&static_call_mutex);
+-}
+-
+-static inline void *static_call_addr(struct static_call_site *site)
+-{
+- return (void *)((long)site->addr + (long)&site->addr);
+-}
+-
+-static inline unsigned long __static_call_key(const struct static_call_site *site)
+-{
+- return (long)site->key + (long)&site->key;
+-}
+-
+-static inline struct static_call_key *static_call_key(const struct static_call_site *site)
+-{
+- return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
+-}
+-
+-/* These assume the key is word-aligned. */
+-static inline bool static_call_is_init(struct static_call_site *site)
+-{
+- return __static_call_key(site) & STATIC_CALL_SITE_INIT;
+-}
+-
+-static inline bool static_call_is_tail(struct static_call_site *site)
+-{
+- return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
+-}
+-
+-static inline void static_call_set_init(struct static_call_site *site)
+-{
+- site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
+- (long)&site->key;
+-}
+-
+-static int static_call_site_cmp(const void *_a, const void *_b)
+-{
+- const struct static_call_site *a = _a;
+- const struct static_call_site *b = _b;
+- const struct static_call_key *key_a = static_call_key(a);
+- const struct static_call_key *key_b = static_call_key(b);
+-
+- if (key_a < key_b)
+- return -1;
+-
+- if (key_a > key_b)
+- return 1;
+-
+- return 0;
+-}
+-
+-static void static_call_site_swap(void *_a, void *_b, int size)
+-{
+- long delta = (unsigned long)_a - (unsigned long)_b;
+- struct static_call_site *a = _a;
+- struct static_call_site *b = _b;
+- struct static_call_site tmp = *a;
+-
+- a->addr = b->addr - delta;
+- a->key = b->key - delta;
+-
+- b->addr = tmp.addr + delta;
+- b->key = tmp.key + delta;
+-}
+-
+-static inline void static_call_sort_entries(struct static_call_site *start,
+- struct static_call_site *stop)
+-{
+- sort(start, stop - start, sizeof(struct static_call_site),
+- static_call_site_cmp, static_call_site_swap);
+-}
+-
+-static inline bool static_call_key_has_mods(struct static_call_key *key)
+-{
+- return !(key->type & 1);
+-}
+-
+-static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
+-{
+- if (!static_call_key_has_mods(key))
+- return NULL;
+-
+- return key->mods;
+-}
+-
+-static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
+-{
+- if (static_call_key_has_mods(key))
+- return NULL;
+-
+- return (struct static_call_site *)(key->type & ~1);
+-}
+-
+-void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+-{
+- struct static_call_site *site, *stop;
+- struct static_call_mod *site_mod, first;
+-
+- cpus_read_lock();
+- static_call_lock();
+-
+- if (key->func == func)
+- goto done;
+-
+- key->func = func;
+-
+- arch_static_call_transform(NULL, tramp, func, false);
+-
+- /*
+- * If uninitialized, we'll not update the callsites, but they still
+- * point to the trampoline and we just patched that.
+- */
+- if (WARN_ON_ONCE(!static_call_initialized))
+- goto done;
+-
+- first = (struct static_call_mod){
+- .next = static_call_key_next(key),
+- .mod = NULL,
+- .sites = static_call_key_sites(key),
+- };
+-
+- for (site_mod = &first; site_mod; site_mod = site_mod->next) {
+- bool init = system_state < SYSTEM_RUNNING;
+- struct module *mod = site_mod->mod;
+-
+- if (!site_mod->sites) {
+- /*
+- * This can happen if the static call key is defined in
+- * a module which doesn't use it.
+- *
+- * It also happens in the has_mods case, where the
+- * 'first' entry has no sites associated with it.
+- */
+- continue;
+- }
+-
+- stop = __stop_static_call_sites;
+-
+- if (mod) {
+-#ifdef CONFIG_MODULES
+- stop = mod->static_call_sites +
+- mod->num_static_call_sites;
+- init = mod->state == MODULE_STATE_COMING;
+-#endif
+- }
+-
+- for (site = site_mod->sites;
+- site < stop && static_call_key(site) == key; site++) {
+- void *site_addr = static_call_addr(site);
+-
+- if (!init && static_call_is_init(site))
+- continue;
+-
+- if (!kernel_text_address((unsigned long)site_addr)) {
+- /*
+- * This skips patching built-in __exit, which
+- * is part of init_section_contains() but is
+- * not part of kernel_text_address().
+- *
+- * Skipping built-in __exit is fine since it
+- * will never be executed.
+- */
+- WARN_ONCE(!static_call_is_init(site),
+- "can't patch static call site at %pS",
+- site_addr);
+- continue;
+- }
+-
+- arch_static_call_transform(site_addr, NULL, func,
+- static_call_is_tail(site));
+- }
+- }
+-
+-done:
+- static_call_unlock();
+- cpus_read_unlock();
+-}
+-EXPORT_SYMBOL_GPL(__static_call_update);
+-
+-static int __static_call_init(struct module *mod,
+- struct static_call_site *start,
+- struct static_call_site *stop)
+-{
+- struct static_call_site *site;
+- struct static_call_key *key, *prev_key = NULL;
+- struct static_call_mod *site_mod;
+-
+- if (start == stop)
+- return 0;
+-
+- static_call_sort_entries(start, stop);
+-
+- for (site = start; site < stop; site++) {
+- void *site_addr = static_call_addr(site);
+-
+- if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
+- (!mod && init_section_contains(site_addr, 1)))
+- static_call_set_init(site);
+-
+- key = static_call_key(site);
+- if (key != prev_key) {
+- prev_key = key;
+-
+- /*
+- * For vmlinux (!mod) avoid the allocation by storing
+- * the sites pointer in the key itself. Also see
+- * __static_call_update()'s @first.
+- *
+- * This allows architectures (eg. x86) to call
+- * static_call_init() before memory allocation works.
+- */
+- if (!mod) {
+- key->sites = site;
+- key->type |= 1;
+- goto do_transform;
+- }
+-
+- site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
+- if (!site_mod)
+- return -ENOMEM;
+-
+- /*
+- * When the key has a direct sites pointer, extract
+- * that into an explicit struct static_call_mod, so we
+- * can have a list of modules.
+- */
+- if (static_call_key_sites(key)) {
+- site_mod->mod = NULL;
+- site_mod->next = NULL;
+- site_mod->sites = static_call_key_sites(key);
+-
+- key->mods = site_mod;
+-
+- site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
+- if (!site_mod)
+- return -ENOMEM;
+- }
+-
+- site_mod->mod = mod;
+- site_mod->sites = site;
+- site_mod->next = static_call_key_next(key);
+- key->mods = site_mod;
+- }
+-
+-do_transform:
+- arch_static_call_transform(site_addr, NULL, key->func,
+- static_call_is_tail(site));
+- }
+-
+- return 0;
+-}
+-
+-static int addr_conflict(struct static_call_site *site, void *start, void *end)
+-{
+- unsigned long addr = (unsigned long)static_call_addr(site);
+-
+- if (addr <= (unsigned long)end &&
+- addr + CALL_INSN_SIZE > (unsigned long)start)
+- return 1;
+-
+- return 0;
+-}
+-
+-static int __static_call_text_reserved(struct static_call_site *iter_start,
+- struct static_call_site *iter_stop,
+- void *start, void *end, bool init)
+-{
+- struct static_call_site *iter = iter_start;
+-
+- while (iter < iter_stop) {
+- if (init || !static_call_is_init(iter)) {
+- if (addr_conflict(iter, start, end))
+- return 1;
+- }
+- iter++;
+- }
+-
+- return 0;
+-}
+-
+-#ifdef CONFIG_MODULES
+-
+-static int __static_call_mod_text_reserved(void *start, void *end)
+-{
+- struct module *mod;
+- int ret;
+-
+- preempt_disable();
+- mod = __module_text_address((unsigned long)start);
+- WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
+- if (!try_module_get(mod))
+- mod = NULL;
+- preempt_enable();
+-
+- if (!mod)
+- return 0;
+-
+- ret = __static_call_text_reserved(mod->static_call_sites,
+- mod->static_call_sites + mod->num_static_call_sites,
+- start, end, mod->state == MODULE_STATE_COMING);
+-
+- module_put(mod);
+-
+- return ret;
+-}
+-
+-static unsigned long tramp_key_lookup(unsigned long addr)
+-{
+- struct static_call_tramp_key *start = __start_static_call_tramp_key;
+- struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
+- struct static_call_tramp_key *tramp_key;
+-
+- for (tramp_key = start; tramp_key != stop; tramp_key++) {
+- unsigned long tramp;
+-
+- tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
+- if (tramp == addr)
+- return (long)tramp_key->key + (long)&tramp_key->key;
+- }
+-
+- return 0;
+-}
+-
+-static int static_call_add_module(struct module *mod)
+-{
+- struct static_call_site *start = mod->static_call_sites;
+- struct static_call_site *stop = start + mod->num_static_call_sites;
+- struct static_call_site *site;
+-
+- for (site = start; site != stop; site++) {
+- unsigned long s_key = __static_call_key(site);
+- unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
+- unsigned long key;
+-
+- /*
+- * Is the key is exported, 'addr' points to the key, which
+- * means modules are allowed to call static_call_update() on
+- * it.
+- *
+- * Otherwise, the key isn't exported, and 'addr' points to the
+- * trampoline so we need to lookup the key.
+- *
+- * We go through this dance to prevent crazy modules from
+- * abusing sensitive static calls.
+- */
+- if (!kernel_text_address(addr))
+- continue;
+-
+- key = tramp_key_lookup(addr);
+- if (!key) {
+- pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
+- static_call_addr(site));
+- return -EINVAL;
+- }
+-
+- key |= s_key & STATIC_CALL_SITE_FLAGS;
+- site->key = key - (long)&site->key;
+- }
+-
+- return __static_call_init(mod, start, stop);
+-}
+-
+-static void static_call_del_module(struct module *mod)
+-{
+- struct static_call_site *start = mod->static_call_sites;
+- struct static_call_site *stop = mod->static_call_sites +
+- mod->num_static_call_sites;
+- struct static_call_key *key, *prev_key = NULL;
+- struct static_call_mod *site_mod, **prev;
+- struct static_call_site *site;
+-
+- for (site = start; site < stop; site++) {
+- key = static_call_key(site);
+- if (key == prev_key)
+- continue;
+-
+- prev_key = key;
+-
+- for (prev = &key->mods, site_mod = key->mods;
+- site_mod && site_mod->mod != mod;
+- prev = &site_mod->next, site_mod = site_mod->next)
+- ;
+-
+- if (!site_mod)
+- continue;
+-
+- *prev = site_mod->next;
+- kfree(site_mod);
+- }
+-}
+-
+-static int static_call_module_notify(struct notifier_block *nb,
+- unsigned long val, void *data)
+-{
+- struct module *mod = data;
+- int ret = 0;
+-
+- cpus_read_lock();
+- static_call_lock();
+-
+- switch (val) {
+- case MODULE_STATE_COMING:
+- ret = static_call_add_module(mod);
+- if (ret) {
+- WARN(1, "Failed to allocate memory for static calls");
+- static_call_del_module(mod);
+- }
+- break;
+- case MODULE_STATE_GOING:
+- static_call_del_module(mod);
+- break;
+- }
+-
+- static_call_unlock();
+- cpus_read_unlock();
+-
+- return notifier_from_errno(ret);
+-}
+-
+-static struct notifier_block static_call_module_nb = {
+- .notifier_call = static_call_module_notify,
+-};
+-
+-#else
+-
+-static inline int __static_call_mod_text_reserved(void *start, void *end)
+-{
+- return 0;
+-}
+-
+-#endif /* CONFIG_MODULES */
+-
+-int static_call_text_reserved(void *start, void *end)
+-{
+- bool init = system_state < SYSTEM_RUNNING;
+- int ret = __static_call_text_reserved(__start_static_call_sites,
+- __stop_static_call_sites, start, end, init);
+-
+- if (ret)
+- return ret;
+-
+- return __static_call_mod_text_reserved(start, end);
+-}
+-
+-int __init static_call_init(void)
+-{
+- int ret;
+-
+- if (static_call_initialized)
+- return 0;
+-
+- cpus_read_lock();
+- static_call_lock();
+- ret = __static_call_init(NULL, __start_static_call_sites,
+- __stop_static_call_sites);
+- static_call_unlock();
+- cpus_read_unlock();
+-
+- if (ret) {
+- pr_err("Failed to allocate memory for static_call!\n");
+- BUG();
+- }
+-
+- static_call_initialized = true;
+-
+-#ifdef CONFIG_MODULES
+- register_module_notifier(&static_call_module_nb);
+-#endif
+- return 0;
+-}
+-early_initcall(static_call_init);
+
+ long __static_call_return0(void)
+ {
+ return 0;
+ }
+-
+-#ifdef CONFIG_STATIC_CALL_SELFTEST
+-
+-static int func_a(int x)
+-{
+- return x+1;
+-}
+-
+-static int func_b(int x)
+-{
+- return x+2;
+-}
+-
+-DEFINE_STATIC_CALL(sc_selftest, func_a);
+-
+-static struct static_call_data {
+- int (*func)(int);
+- int val;
+- int expect;
+-} static_call_data [] __initdata = {
+- { NULL, 2, 3 },
+- { func_b, 2, 4 },
+- { func_a, 2, 3 }
+-};
+-
+-static int __init test_static_call_init(void)
+-{
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
+- struct static_call_data *scd = &static_call_data[i];
+-
+- if (scd->func)
+- static_call_update(sc_selftest, scd->func);
+-
+- WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
+- }
+-
+- return 0;
+-}
+-early_initcall(test_static_call_init);
+-
+-#endif /* CONFIG_STATIC_CALL_SELFTEST */
++EXPORT_SYMBOL_GPL(__static_call_return0);
+diff --git a/kernel/static_call_inline.c b/kernel/static_call_inline.c
+new file mode 100644
+index 0000000000000..dc5665b628140
+--- /dev/null
++++ b/kernel/static_call_inline.c
+@@ -0,0 +1,543 @@
++// SPDX-License-Identifier: GPL-2.0
++#include <linux/init.h>
++#include <linux/static_call.h>
++#include <linux/bug.h>
++#include <linux/smp.h>
++#include <linux/sort.h>
++#include <linux/slab.h>
++#include <linux/module.h>
++#include <linux/cpu.h>
++#include <linux/processor.h>
++#include <asm/sections.h>
++
++extern struct static_call_site __start_static_call_sites[],
++ __stop_static_call_sites[];
++extern struct static_call_tramp_key __start_static_call_tramp_key[],
++ __stop_static_call_tramp_key[];
++
++static bool static_call_initialized;
++
++/* mutex to protect key modules/sites */
++static DEFINE_MUTEX(static_call_mutex);
++
++static void static_call_lock(void)
++{
++ mutex_lock(&static_call_mutex);
++}
++
++static void static_call_unlock(void)
++{
++ mutex_unlock(&static_call_mutex);
++}
++
++static inline void *static_call_addr(struct static_call_site *site)
++{
++ return (void *)((long)site->addr + (long)&site->addr);
++}
++
++static inline unsigned long __static_call_key(const struct static_call_site *site)
++{
++ return (long)site->key + (long)&site->key;
++}
++
++static inline struct static_call_key *static_call_key(const struct static_call_site *site)
++{
++ return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
++}
++
++/* These assume the key is word-aligned. */
++static inline bool static_call_is_init(struct static_call_site *site)
++{
++ return __static_call_key(site) & STATIC_CALL_SITE_INIT;
++}
++
++static inline bool static_call_is_tail(struct static_call_site *site)
++{
++ return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
++}
++
++static inline void static_call_set_init(struct static_call_site *site)
++{
++ site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
++ (long)&site->key;
++}
++
++static int static_call_site_cmp(const void *_a, const void *_b)
++{
++ const struct static_call_site *a = _a;
++ const struct static_call_site *b = _b;
++ const struct static_call_key *key_a = static_call_key(a);
++ const struct static_call_key *key_b = static_call_key(b);
++
++ if (key_a < key_b)
++ return -1;
++
++ if (key_a > key_b)
++ return 1;
++
++ return 0;
++}
++
++static void static_call_site_swap(void *_a, void *_b, int size)
++{
++ long delta = (unsigned long)_a - (unsigned long)_b;
++ struct static_call_site *a = _a;
++ struct static_call_site *b = _b;
++ struct static_call_site tmp = *a;
++
++ a->addr = b->addr - delta;
++ a->key = b->key - delta;
++
++ b->addr = tmp.addr + delta;
++ b->key = tmp.key + delta;
++}
++
++static inline void static_call_sort_entries(struct static_call_site *start,
++ struct static_call_site *stop)
++{
++ sort(start, stop - start, sizeof(struct static_call_site),
++ static_call_site_cmp, static_call_site_swap);
++}
++
++static inline bool static_call_key_has_mods(struct static_call_key *key)
++{
++ return !(key->type & 1);
++}
++
++static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
++{
++ if (!static_call_key_has_mods(key))
++ return NULL;
++
++ return key->mods;
++}
++
++static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
++{
++ if (static_call_key_has_mods(key))
++ return NULL;
++
++ return (struct static_call_site *)(key->type & ~1);
++}
++
++void __static_call_update(struct static_call_key *key, void *tramp, void *func)
++{
++ struct static_call_site *site, *stop;
++ struct static_call_mod *site_mod, first;
++
++ cpus_read_lock();
++ static_call_lock();
++
++ if (key->func == func)
++ goto done;
++
++ key->func = func;
++
++ arch_static_call_transform(NULL, tramp, func, false);
++
++ /*
++ * If uninitialized, we'll not update the callsites, but they still
++ * point to the trampoline and we just patched that.
++ */
++ if (WARN_ON_ONCE(!static_call_initialized))
++ goto done;
++
++ first = (struct static_call_mod){
++ .next = static_call_key_next(key),
++ .mod = NULL,
++ .sites = static_call_key_sites(key),
++ };
++
++ for (site_mod = &first; site_mod; site_mod = site_mod->next) {
++ bool init = system_state < SYSTEM_RUNNING;
++ struct module *mod = site_mod->mod;
++
++ if (!site_mod->sites) {
++ /*
++ * This can happen if the static call key is defined in
++ * a module which doesn't use it.
++ *
++ * It also happens in the has_mods case, where the
++ * 'first' entry has no sites associated with it.
++ */
++ continue;
++ }
++
++ stop = __stop_static_call_sites;
++
++ if (mod) {
++#ifdef CONFIG_MODULES
++ stop = mod->static_call_sites +
++ mod->num_static_call_sites;
++ init = mod->state == MODULE_STATE_COMING;
++#endif
++ }
++
++ for (site = site_mod->sites;
++ site < stop && static_call_key(site) == key; site++) {
++ void *site_addr = static_call_addr(site);
++
++ if (!init && static_call_is_init(site))
++ continue;
++
++ if (!kernel_text_address((unsigned long)site_addr)) {
++ /*
++ * This skips patching built-in __exit, which
++ * is part of init_section_contains() but is
++ * not part of kernel_text_address().
++ *
++ * Skipping built-in __exit is fine since it
++ * will never be executed.
++ */
++ WARN_ONCE(!static_call_is_init(site),
++ "can't patch static call site at %pS",
++ site_addr);
++ continue;
++ }
++
++ arch_static_call_transform(site_addr, NULL, func,
++ static_call_is_tail(site));
++ }
++ }
++
++done:
++ static_call_unlock();
++ cpus_read_unlock();
++}
++EXPORT_SYMBOL_GPL(__static_call_update);
++
++static int __static_call_init(struct module *mod,
++ struct static_call_site *start,
++ struct static_call_site *stop)
++{
++ struct static_call_site *site;
++ struct static_call_key *key, *prev_key = NULL;
++ struct static_call_mod *site_mod;
++
++ if (start == stop)
++ return 0;
++
++ static_call_sort_entries(start, stop);
++
++ for (site = start; site < stop; site++) {
++ void *site_addr = static_call_addr(site);
++
++ if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
++ (!mod && init_section_contains(site_addr, 1)))
++ static_call_set_init(site);
++
++ key = static_call_key(site);
++ if (key != prev_key) {
++ prev_key = key;
++
++ /*
++ * For vmlinux (!mod) avoid the allocation by storing
++ * the sites pointer in the key itself. Also see
++ * __static_call_update()'s @first.
++ *
++ * This allows architectures (eg. x86) to call
++ * static_call_init() before memory allocation works.
++ */
++ if (!mod) {
++ key->sites = site;
++ key->type |= 1;
++ goto do_transform;
++ }
++
++ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
++ if (!site_mod)
++ return -ENOMEM;
++
++ /*
++ * When the key has a direct sites pointer, extract
++ * that into an explicit struct static_call_mod, so we
++ * can have a list of modules.
++ */
++ if (static_call_key_sites(key)) {
++ site_mod->mod = NULL;
++ site_mod->next = NULL;
++ site_mod->sites = static_call_key_sites(key);
++
++ key->mods = site_mod;
++
++ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
++ if (!site_mod)
++ return -ENOMEM;
++ }
++
++ site_mod->mod = mod;
++ site_mod->sites = site;
++ site_mod->next = static_call_key_next(key);
++ key->mods = site_mod;
++ }
++
++do_transform:
++ arch_static_call_transform(site_addr, NULL, key->func,
++ static_call_is_tail(site));
++ }
++
++ return 0;
++}
++
++static int addr_conflict(struct static_call_site *site, void *start, void *end)
++{
++ unsigned long addr = (unsigned long)static_call_addr(site);
++
++ if (addr <= (unsigned long)end &&
++ addr + CALL_INSN_SIZE > (unsigned long)start)
++ return 1;
++
++ return 0;
++}
++
++static int __static_call_text_reserved(struct static_call_site *iter_start,
++ struct static_call_site *iter_stop,
++ void *start, void *end, bool init)
++{
++ struct static_call_site *iter = iter_start;
++
++ while (iter < iter_stop) {
++ if (init || !static_call_is_init(iter)) {
++ if (addr_conflict(iter, start, end))
++ return 1;
++ }
++ iter++;
++ }
++
++ return 0;
++}
++
++#ifdef CONFIG_MODULES
++
++static int __static_call_mod_text_reserved(void *start, void *end)
++{
++ struct module *mod;
++ int ret;
++
++ preempt_disable();
++ mod = __module_text_address((unsigned long)start);
++ WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
++ if (!try_module_get(mod))
++ mod = NULL;
++ preempt_enable();
++
++ if (!mod)
++ return 0;
++
++ ret = __static_call_text_reserved(mod->static_call_sites,
++ mod->static_call_sites + mod->num_static_call_sites,
++ start, end, mod->state == MODULE_STATE_COMING);
++
++ module_put(mod);
++
++ return ret;
++}
++
++static unsigned long tramp_key_lookup(unsigned long addr)
++{
++ struct static_call_tramp_key *start = __start_static_call_tramp_key;
++ struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
++ struct static_call_tramp_key *tramp_key;
++
++ for (tramp_key = start; tramp_key != stop; tramp_key++) {
++ unsigned long tramp;
++
++ tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
++ if (tramp == addr)
++ return (long)tramp_key->key + (long)&tramp_key->key;
++ }
++
++ return 0;
++}
++
++static int static_call_add_module(struct module *mod)
++{
++ struct static_call_site *start = mod->static_call_sites;
++ struct static_call_site *stop = start + mod->num_static_call_sites;
++ struct static_call_site *site;
++
++ for (site = start; site != stop; site++) {
++ unsigned long s_key = __static_call_key(site);
++ unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
++ unsigned long key;
++
++ /*
++ * Is the key is exported, 'addr' points to the key, which
++ * means modules are allowed to call static_call_update() on
++ * it.
++ *
++ * Otherwise, the key isn't exported, and 'addr' points to the
++ * trampoline so we need to lookup the key.
++ *
++ * We go through this dance to prevent crazy modules from
++ * abusing sensitive static calls.
++ */
++ if (!kernel_text_address(addr))
++ continue;
++
++ key = tramp_key_lookup(addr);
++ if (!key) {
++ pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
++ static_call_addr(site));
++ return -EINVAL;
++ }
++
++ key |= s_key & STATIC_CALL_SITE_FLAGS;
++ site->key = key - (long)&site->key;
++ }
++
++ return __static_call_init(mod, start, stop);
++}
++
++static void static_call_del_module(struct module *mod)
++{
++ struct static_call_site *start = mod->static_call_sites;
++ struct static_call_site *stop = mod->static_call_sites +
++ mod->num_static_call_sites;
++ struct static_call_key *key, *prev_key = NULL;
++ struct static_call_mod *site_mod, **prev;
++ struct static_call_site *site;
++
++ for (site = start; site < stop; site++) {
++ key = static_call_key(site);
++ if (key == prev_key)
++ continue;
++
++ prev_key = key;
++
++ for (prev = &key->mods, site_mod = key->mods;
++ site_mod && site_mod->mod != mod;
++ prev = &site_mod->next, site_mod = site_mod->next)
++ ;
++
++ if (!site_mod)
++ continue;
++
++ *prev = site_mod->next;
++ kfree(site_mod);
++ }
++}
++
++static int static_call_module_notify(struct notifier_block *nb,
++ unsigned long val, void *data)
++{
++ struct module *mod = data;
++ int ret = 0;
++
++ cpus_read_lock();
++ static_call_lock();
++
++ switch (val) {
++ case MODULE_STATE_COMING:
++ ret = static_call_add_module(mod);
++ if (ret) {
++ WARN(1, "Failed to allocate memory for static calls");
++ static_call_del_module(mod);
++ }
++ break;
++ case MODULE_STATE_GOING:
++ static_call_del_module(mod);
++ break;
++ }
++
++ static_call_unlock();
++ cpus_read_unlock();
++
++ return notifier_from_errno(ret);
++}
++
++static struct notifier_block static_call_module_nb = {
++ .notifier_call = static_call_module_notify,
++};
++
++#else
++
++static inline int __static_call_mod_text_reserved(void *start, void *end)
++{
++ return 0;
++}
++
++#endif /* CONFIG_MODULES */
++
++int static_call_text_reserved(void *start, void *end)
++{
++ bool init = system_state < SYSTEM_RUNNING;
++ int ret = __static_call_text_reserved(__start_static_call_sites,
++ __stop_static_call_sites, start, end, init);
++
++ if (ret)
++ return ret;
++
++ return __static_call_mod_text_reserved(start, end);
++}
++
++int __init static_call_init(void)
++{
++ int ret;
++
++ if (static_call_initialized)
++ return 0;
++
++ cpus_read_lock();
++ static_call_lock();
++ ret = __static_call_init(NULL, __start_static_call_sites,
++ __stop_static_call_sites);
++ static_call_unlock();
++ cpus_read_unlock();
++
++ if (ret) {
++ pr_err("Failed to allocate memory for static_call!\n");
++ BUG();
++ }
++
++ static_call_initialized = true;
++
++#ifdef CONFIG_MODULES
++ register_module_notifier(&static_call_module_nb);
++#endif
++ return 0;
++}
++early_initcall(static_call_init);
++
++#ifdef CONFIG_STATIC_CALL_SELFTEST
++
++static int func_a(int x)
++{
++ return x+1;
++}
++
++static int func_b(int x)
++{
++ return x+2;
++}
++
++DEFINE_STATIC_CALL(sc_selftest, func_a);
++
++static struct static_call_data {
++ int (*func)(int);
++ int val;
++ int expect;
++} static_call_data [] __initdata = {
++ { NULL, 2, 3 },
++ { func_b, 2, 4 },
++ { func_a, 2, 3 }
++};
++
++static int __init test_static_call_init(void)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
++ struct static_call_data *scd = &static_call_data[i];
++
++ if (scd->func)
++ static_call_update(sc_selftest, scd->func);
++
++ WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
++ }
++
++ return 0;
++}
++early_initcall(test_static_call_init);
++
++#endif /* CONFIG_STATIC_CALL_SELFTEST */
+diff --git a/kernel/sys.c b/kernel/sys.c
+index 8fdac0d90504a..3e4e8930fafc6 100644
+--- a/kernel/sys.c
++++ b/kernel/sys.c
+@@ -472,6 +472,16 @@ static int set_user(struct cred *new)
+ if (!new_user)
+ return -EAGAIN;
+
++ free_uid(new->user);
++ new->user = new_user;
++ return 0;
++}
++
++static void flag_nproc_exceeded(struct cred *new)
++{
++ if (new->ucounts == current_ucounts())
++ return;
++
+ /*
+ * We don't fail in case of NPROC limit excess here because too many
+ * poorly written programs don't check set*uid() return code, assuming
+@@ -480,15 +490,10 @@ static int set_user(struct cred *new)
+ * failure to the execve() stage.
+ */
+ if (is_ucounts_overlimit(new->ucounts, UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC)) &&
+- new_user != INIT_USER &&
+- !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
++ new->user != INIT_USER)
+ current->flags |= PF_NPROC_EXCEEDED;
+ else
+ current->flags &= ~PF_NPROC_EXCEEDED;
+-
+- free_uid(new->user);
+- new->user = new_user;
+- return 0;
+ }
+
+ /*
+@@ -563,6 +568,7 @@ long __sys_setreuid(uid_t ruid, uid_t euid)
+ if (retval < 0)
+ goto error;
+
++ flag_nproc_exceeded(new);
+ return commit_creds(new);
+
+ error:
+@@ -625,6 +631,7 @@ long __sys_setuid(uid_t uid)
+ if (retval < 0)
+ goto error;
+
++ flag_nproc_exceeded(new);
+ return commit_creds(new);
+
+ error:
+@@ -704,6 +711,7 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid)
+ if (retval < 0)
+ goto error;
+
++ flag_nproc_exceeded(new);
+ return commit_creds(new);
+
+ error:
+diff --git a/kernel/sysctl.c b/kernel/sysctl.c
+index 083be6af29d70..0586047f73234 100644
+--- a/kernel/sysctl.c
++++ b/kernel/sysctl.c
+@@ -228,6 +228,10 @@ static int bpf_stats_handler(struct ctl_table *table, int write,
+ return ret;
+ }
+
++void __weak unpriv_ebpf_notify(int new_state)
++{
++}
++
+ static int bpf_unpriv_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *lenp, loff_t *ppos)
+ {
+@@ -245,6 +249,9 @@ static int bpf_unpriv_handler(struct ctl_table *table, int write,
+ return -EPERM;
+ *(int *)table->data = unpriv_enable;
+ }
++
++ unpriv_ebpf_notify(unpriv_enable);
++
+ return ret;
+ }
+ #endif /* CONFIG_BPF_SYSCALL && CONFIG_SYSCTL */
+diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
+index b8a14d2fb5ba6..bcad1a1e5dcf1 100644
+--- a/kernel/time/clocksource.c
++++ b/kernel/time/clocksource.c
+@@ -107,7 +107,7 @@ static u64 suspend_start;
+ * This delay could be due to SMIs, NMIs, or to VCPU preemptions. Used as
+ * a lower bound for cs->uncertainty_margin values when registering clocks.
+ */
+-#define WATCHDOG_MAX_SKEW (50 * NSEC_PER_USEC)
++#define WATCHDOG_MAX_SKEW (100 * NSEC_PER_USEC)
+
+ #ifdef CONFIG_CLOCKSOURCE_WATCHDOG
+ static void clocksource_watchdog_work(struct work_struct *work);
+@@ -205,17 +205,24 @@ EXPORT_SYMBOL_GPL(max_cswd_read_retries);
+ static int verify_n_cpus = 8;
+ module_param(verify_n_cpus, int, 0644);
+
+-static bool cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
++enum wd_read_status {
++ WD_READ_SUCCESS,
++ WD_READ_UNSTABLE,
++ WD_READ_SKIP
++};
++
++static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
+ {
+ unsigned int nretries;
+- u64 wd_end, wd_delta;
+- int64_t wd_delay;
++ u64 wd_end, wd_end2, wd_delta;
++ int64_t wd_delay, wd_seq_delay;
+
+ for (nretries = 0; nretries <= max_cswd_read_retries; nretries++) {
+ local_irq_disable();
+ *wdnow = watchdog->read(watchdog);
+ *csnow = cs->read(cs);
+ wd_end = watchdog->read(watchdog);
++ wd_end2 = watchdog->read(watchdog);
+ local_irq_enable();
+
+ wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask);
+@@ -226,13 +233,34 @@ static bool cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
+ pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n",
+ smp_processor_id(), watchdog->name, nretries);
+ }
+- return true;
++ return WD_READ_SUCCESS;
+ }
++
++ /*
++ * Now compute delay in consecutive watchdog read to see if
++ * there is too much external interferences that cause
++ * significant delay in reading both clocksource and watchdog.
++ *
++ * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2,
++ * report system busy, reinit the watchdog and skip the current
++ * watchdog test.
++ */
++ wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask);
++ wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift);
++ if (wd_seq_delay > WATCHDOG_MAX_SKEW/2)
++ goto skip_test;
+ }
+
+ pr_warn("timekeeping watchdog on CPU%d: %s read-back delay of %lldns, attempt %d, marking unstable\n",
+ smp_processor_id(), watchdog->name, wd_delay, nretries);
+- return false;
++ return WD_READ_UNSTABLE;
++
++skip_test:
++ pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n",
++ smp_processor_id(), watchdog->name, wd_seq_delay);
++ pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n",
++ cs->name, wd_delay);
++ return WD_READ_SKIP;
+ }
+
+ static u64 csnow_mid;
+@@ -356,6 +384,7 @@ static void clocksource_watchdog(struct timer_list *unused)
+ int next_cpu, reset_pending;
+ int64_t wd_nsec, cs_nsec;
+ struct clocksource *cs;
++ enum wd_read_status read_ret;
+ u32 md;
+
+ spin_lock(&watchdog_lock);
+@@ -373,9 +402,12 @@ static void clocksource_watchdog(struct timer_list *unused)
+ continue;
+ }
+
+- if (!cs_watchdog_read(cs, &csnow, &wdnow)) {
+- /* Clock readout unreliable, so give it up. */
+- __clocksource_unstable(cs);
++ read_ret = cs_watchdog_read(cs, &csnow, &wdnow);
++
++ if (read_ret != WD_READ_SUCCESS) {
++ if (read_ret == WD_READ_UNSTABLE)
++ /* Clock readout unreliable, so give it up. */
++ __clocksource_unstable(cs);
+ continue;
+ }
+
+diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
+index 643d412ac6235..96b4e78104266 100644
+--- a/kernel/time/posix-cpu-timers.c
++++ b/kernel/time/posix-cpu-timers.c
+@@ -1158,14 +1158,29 @@ static void posix_cpu_timers_work(struct callback_head *work)
+ handle_posix_cpu_timers(current);
+ }
+
++/*
++ * Clear existing posix CPU timers task work.
++ */
++void clear_posix_cputimers_work(struct task_struct *p)
++{
++ /*
++ * A copied work entry from the old task is not meaningful, clear it.
++ * N.B. init_task_work will not do this.
++ */
++ memset(&p->posix_cputimers_work.work, 0,
++ sizeof(p->posix_cputimers_work.work));
++ init_task_work(&p->posix_cputimers_work.work,
++ posix_cpu_timers_work);
++ p->posix_cputimers_work.scheduled = false;
++}
++
+ /*
+ * Initialize posix CPU timers task work in init task. Out of line to
+ * keep the callback static and to avoid header recursion hell.
+ */
+ void __init posix_cputimers_init_work(void)
+ {
+- init_task_work(&current->posix_cputimers_work.work,
+- posix_cpu_timers_work);
++ clear_posix_cputimers_work(current);
+ }
+
+ /*
+diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
+index 6bffe5af8cb11..19e6b861de97b 100644
+--- a/kernel/time/tick-sched.c
++++ b/kernel/time/tick-sched.c
+@@ -186,7 +186,7 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
+ */
+ if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
+ #ifdef CONFIG_NO_HZ_FULL
+- WARN_ON(tick_nohz_full_running);
++ WARN_ON_ONCE(tick_nohz_full_running);
+ #endif
+ tick_do_timer_cpu = cpu;
+ }
+diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
+index b348749a9fc62..871c912860ed5 100644
+--- a/kernel/time/timekeeping.c
++++ b/kernel/time/timekeeping.c
+@@ -17,6 +17,7 @@
+ #include <linux/clocksource.h>
+ #include <linux/jiffies.h>
+ #include <linux/time.h>
++#include <linux/timex.h>
+ #include <linux/tick.h>
+ #include <linux/stop_machine.h>
+ #include <linux/pvclock_gtod.h>
+@@ -482,7 +483,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
+ * of the following timestamps. Callers need to be aware of that and
+ * deal with it.
+ */
+-u64 ktime_get_mono_fast_ns(void)
++u64 notrace ktime_get_mono_fast_ns(void)
+ {
+ return __ktime_get_fast_ns(&tk_fast_mono);
+ }
+@@ -494,7 +495,7 @@ EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
+ * Contrary to ktime_get_mono_fast_ns() this is always correct because the
+ * conversion factor is not affected by NTP/PTP correction.
+ */
+-u64 ktime_get_raw_fast_ns(void)
++u64 notrace ktime_get_raw_fast_ns(void)
+ {
+ return __ktime_get_fast_ns(&tk_fast_raw);
+ }
+@@ -1306,8 +1307,7 @@ int do_settimeofday64(const struct timespec64 *ts)
+ timekeeping_forward_now(tk);
+
+ xt = tk_xtime(tk);
+- ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
+- ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
++ ts_delta = timespec64_sub(*ts, xt);
+
+ if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
+ ret = -EINVAL;
+@@ -2381,6 +2381,20 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc)
+ return 0;
+ }
+
++/**
++ * random_get_entropy_fallback - Returns the raw clock source value,
++ * used by random.c for platforms with no valid random_get_entropy().
++ */
++unsigned long random_get_entropy_fallback(void)
++{
++ struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
++ struct clocksource *clock = READ_ONCE(tkr->clock);
++
++ if (unlikely(timekeeping_suspended || !clock))
++ return 0;
++ return clock->read(clock);
++}
++EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
+
+ /**
+ * do_adjtimex() - Accessor function to NTP __do_adjtimex function
+diff --git a/kernel/time/timer.c b/kernel/time/timer.c
+index e3d2c23c413d4..9dd2a39cb3b00 100644
+--- a/kernel/time/timer.c
++++ b/kernel/time/timer.c
+@@ -1722,11 +1722,14 @@ static inline void __run_timers(struct timer_base *base)
+ time_after_eq(jiffies, base->next_expiry)) {
+ levels = collect_expired_timers(base, heads);
+ /*
+- * The only possible reason for not finding any expired
+- * timer at this clk is that all matching timers have been
+- * dequeued.
++ * The two possible reasons for not finding any expired
++ * timer at this clk are that all matching timers have been
++ * dequeued or no timer has been queued since
++ * base::next_expiry was set to base::clk +
++ * NEXT_TIMER_MAX_DELTA.
+ */
+- WARN_ON_ONCE(!levels && !base->next_expiry_recalc);
++ WARN_ON_ONCE(!levels && !base->next_expiry_recalc
++ && base->timers_pending);
+ base->clk++;
+ base->next_expiry = __next_timer_interrupt(base);
+
+@@ -2054,26 +2057,28 @@ unsigned long msleep_interruptible(unsigned int msecs)
+ EXPORT_SYMBOL(msleep_interruptible);
+
+ /**
+- * usleep_range - Sleep for an approximate time
+- * @min: Minimum time in usecs to sleep
+- * @max: Maximum time in usecs to sleep
++ * usleep_range_state - Sleep for an approximate time in a given state
++ * @min: Minimum time in usecs to sleep
++ * @max: Maximum time in usecs to sleep
++ * @state: State of the current task that will be while sleeping
+ *
+ * In non-atomic context where the exact wakeup time is flexible, use
+- * usleep_range() instead of udelay(). The sleep improves responsiveness
++ * usleep_range_state() instead of udelay(). The sleep improves responsiveness
+ * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces
+ * power usage by allowing hrtimers to take advantage of an already-
+ * scheduled interrupt instead of scheduling a new one just for this sleep.
+ */
+-void __sched usleep_range(unsigned long min, unsigned long max)
++void __sched usleep_range_state(unsigned long min, unsigned long max,
++ unsigned int state)
+ {
+ ktime_t exp = ktime_add_us(ktime_get(), min);
+ u64 delta = (u64)(max - min) * NSEC_PER_USEC;
+
+ for (;;) {
+- __set_current_state(TASK_UNINTERRUPTIBLE);
++ __set_current_state(state);
+ /* Do not return before the requested sleep time has elapsed */
+ if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
+ break;
+ }
+ }
+-EXPORT_SYMBOL(usleep_range);
++EXPORT_SYMBOL(usleep_range_state);
+diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
+index fa91f398f28b7..c42ff77eb6ccc 100644
+--- a/kernel/trace/blktrace.c
++++ b/kernel/trace/blktrace.c
+@@ -310,10 +310,20 @@ record_it:
+ local_irq_restore(flags);
+ }
+
+-static void blk_trace_free(struct blk_trace *bt)
++static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)
+ {
+ relay_close(bt->rchan);
+- debugfs_remove(bt->dir);
++
++ /*
++ * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created
++ * under 'q->debugfs_dir', thus lookup and remove them.
++ */
++ if (!bt->dir) {
++ debugfs_remove(debugfs_lookup("dropped", q->debugfs_dir));
++ debugfs_remove(debugfs_lookup("msg", q->debugfs_dir));
++ } else {
++ debugfs_remove(bt->dir);
++ }
+ free_percpu(bt->sequence);
+ free_percpu(bt->msg_data);
+ kfree(bt);
+@@ -335,10 +345,10 @@ static void put_probe_ref(void)
+ mutex_unlock(&blk_probe_mutex);
+ }
+
+-static void blk_trace_cleanup(struct blk_trace *bt)
++static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)
+ {
+ synchronize_rcu();
+- blk_trace_free(bt);
++ blk_trace_free(q, bt);
+ put_probe_ref();
+ }
+
+@@ -352,7 +362,7 @@ static int __blk_trace_remove(struct request_queue *q)
+ return -EINVAL;
+
+ if (bt->trace_state != Blktrace_running)
+- blk_trace_cleanup(bt);
++ blk_trace_cleanup(q, bt);
+
+ return 0;
+ }
+@@ -572,7 +582,7 @@ static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
+ ret = 0;
+ err:
+ if (ret)
+- blk_trace_free(bt);
++ blk_trace_free(q, bt);
+ return ret;
+ }
+
+@@ -1615,7 +1625,7 @@ static int blk_trace_remove_queue(struct request_queue *q)
+
+ put_probe_ref();
+ synchronize_rcu();
+- blk_trace_free(bt);
++ blk_trace_free(q, bt);
+ return 0;
+ }
+
+@@ -1646,7 +1656,7 @@ static int blk_trace_setup_queue(struct request_queue *q,
+ return 0;
+
+ free_bt:
+- blk_trace_free(bt);
++ blk_trace_free(q, bt);
+ return ret;
+ }
+
+diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
+index 8e2eb950aa829..c289010b0964e 100644
+--- a/kernel/trace/bpf_trace.c
++++ b/kernel/trace/bpf_trace.c
+@@ -345,7 +345,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = {
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_ANYTHING,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ };
+
+@@ -394,7 +394,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = {
+ .func = bpf_trace_printk,
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+- .arg1_type = ARG_PTR_TO_MEM,
++ .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg2_type = ARG_CONST_SIZE,
+ };
+
+@@ -446,9 +446,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID,
+ .arg1_btf_id = &btf_seq_file_ids[0],
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+- .arg4_type = ARG_PTR_TO_MEM_OR_NULL,
++ .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -463,7 +463,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID,
+ .arg1_btf_id = &btf_seq_file_ids[0],
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -487,7 +487,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID,
+ .arg1_btf_id = &btf_seq_file_ids[0],
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_ANYTHING,
+ };
+@@ -648,7 +648,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -958,7 +958,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_MEM,
+ .arg2_type = ARG_CONST_SIZE,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE,
+ .arg5_type = ARG_ANYTHING,
+ };
+@@ -1037,8 +1037,6 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+ return &bpf_ktime_get_ns_proto;
+ case BPF_FUNC_ktime_get_boot_ns:
+ return &bpf_ktime_get_boot_ns_proto;
+- case BPF_FUNC_ktime_get_coarse_ns:
+- return &bpf_ktime_get_coarse_ns_proto;
+ case BPF_FUNC_tail_call:
+ return &bpf_tail_call_proto;
+ case BPF_FUNC_get_current_pid_tgid:
+@@ -1209,7 +1207,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -1324,9 +1322,6 @@ static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
+ BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
+ void *, buf, u32, size, u64, flags)
+ {
+-#ifndef CONFIG_X86
+- return -ENOENT;
+-#else
+ static const u32 br_entry_size = sizeof(struct perf_branch_entry);
+ struct perf_branch_stack *br_stack = ctx->data->br_stack;
+ u32 to_copy;
+@@ -1335,7 +1330,7 @@ BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
+ return -EINVAL;
+
+ if (unlikely(!br_stack))
+- return -EINVAL;
++ return -ENOENT;
+
+ if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE)
+ return br_stack->nr * br_entry_size;
+@@ -1347,7 +1342,6 @@ BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
+ memcpy(buf, br_stack->entries, to_copy);
+
+ return to_copy;
+-#endif
+ }
+
+ static const struct bpf_func_proto bpf_read_branch_records_proto = {
+@@ -1435,7 +1429,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -1489,7 +1483,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_ANYTHING,
+ };
+diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
+index feebf57c64588..c672040142e98 100644
+--- a/kernel/trace/ftrace.c
++++ b/kernel/trace/ftrace.c
+@@ -988,8 +988,9 @@ static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
+ }
+ }
+
+- entry = tracefs_create_file("function_profile_enabled", 0644,
+- d_tracer, NULL, &ftrace_profile_fops);
++ entry = tracefs_create_file("function_profile_enabled",
++ TRACE_MODE_WRITE, d_tracer, NULL,
++ &ftrace_profile_fops);
+ if (!entry)
+ pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
+ }
+@@ -6109,10 +6110,10 @@ void ftrace_create_filter_files(struct ftrace_ops *ops,
+ struct dentry *parent)
+ {
+
+- trace_create_file("set_ftrace_filter", 0644, parent,
++ trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
+ ops, &ftrace_filter_fops);
+
+- trace_create_file("set_ftrace_notrace", 0644, parent,
++ trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
+ ops, &ftrace_notrace_fops);
+ }
+
+@@ -6139,19 +6140,19 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops)
+ static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
+ {
+
+- trace_create_file("available_filter_functions", 0444,
++ trace_create_file("available_filter_functions", TRACE_MODE_READ,
+ d_tracer, NULL, &ftrace_avail_fops);
+
+- trace_create_file("enabled_functions", 0444,
++ trace_create_file("enabled_functions", TRACE_MODE_READ,
+ d_tracer, NULL, &ftrace_enabled_fops);
+
+ ftrace_create_filter_files(&global_ops, d_tracer);
+
+ #ifdef CONFIG_FUNCTION_GRAPH_TRACER
+- trace_create_file("set_graph_function", 0644, d_tracer,
++ trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
+ NULL,
+ &ftrace_graph_fops);
+- trace_create_file("set_graph_notrace", 0644, d_tracer,
++ trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
+ NULL,
+ &ftrace_graph_notrace_fops);
+ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
+@@ -7494,10 +7495,10 @@ static const struct file_operations ftrace_no_pid_fops = {
+
+ void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
+ {
+- trace_create_file("set_ftrace_pid", 0644, d_tracer,
++ trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer,
+ tr, &ftrace_pid_fops);
+- trace_create_file("set_ftrace_notrace_pid", 0644, d_tracer,
+- tr, &ftrace_no_pid_fops);
++ trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE,
++ d_tracer, tr, &ftrace_no_pid_fops);
+ }
+
+ void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
+diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
+index c5a3fbf19617e..46ae72095c1e2 100644
+--- a/kernel/trace/ring_buffer.c
++++ b/kernel/trace/ring_buffer.c
+@@ -5233,6 +5233,9 @@ void ring_buffer_reset(struct trace_buffer *buffer)
+ struct ring_buffer_per_cpu *cpu_buffer;
+ int cpu;
+
++ /* prevent another thread from changing buffer sizes */
++ mutex_lock(&buffer->mutex);
++
+ for_each_buffer_cpu(buffer, cpu) {
+ cpu_buffer = buffer->buffers[cpu];
+
+@@ -5251,6 +5254,8 @@ void ring_buffer_reset(struct trace_buffer *buffer)
+ atomic_dec(&cpu_buffer->record_disabled);
+ atomic_dec(&cpu_buffer->resize_disabled);
+ }
++
++ mutex_unlock(&buffer->mutex);
+ }
+ EXPORT_SYMBOL_GPL(ring_buffer_reset);
+
+diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
+index bc677cd642240..b49756608f838 100644
+--- a/kernel/trace/trace.c
++++ b/kernel/trace/trace.c
+@@ -235,7 +235,7 @@ static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
+ static int __init set_trace_boot_options(char *str)
+ {
+ strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
+- return 0;
++ return 1;
+ }
+ __setup("trace_options=", set_trace_boot_options);
+
+@@ -246,12 +246,16 @@ static int __init set_trace_boot_clock(char *str)
+ {
+ strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
+ trace_boot_clock = trace_boot_clock_buf;
+- return 0;
++ return 1;
+ }
+ __setup("trace_clock=", set_trace_boot_clock);
+
+ static int __init set_tracepoint_printk(char *str)
+ {
++ /* Ignore the "tp_printk_stop_on_boot" param */
++ if (*str == '_')
++ return 0;
++
+ if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
+ tracepoint_printk = 1;
+ return 1;
+@@ -1492,10 +1496,12 @@ static int __init set_buf_size(char *str)
+ if (!str)
+ return 0;
+ buf_size = memparse(str, &str);
+- /* nr_entries can not be zero */
+- if (buf_size == 0)
+- return 0;
+- trace_buf_size = buf_size;
++ /*
++ * nr_entries can not be zero and the startup
++ * tests require some buffer space. Therefore
++ * ensure we have at least 4096 bytes of buffer.
++ */
++ trace_buf_size = max(4096UL, buf_size);
+ return 1;
+ }
+ __setup("trace_buf_size=", set_buf_size);
+@@ -1714,7 +1720,8 @@ static void trace_create_maxlat_file(struct trace_array *tr,
+ {
+ INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
+ init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
+- tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
++ tr->d_max_latency = trace_create_file("tracing_max_latency",
++ TRACE_MODE_WRITE,
+ d_tracer, &tr->max_latency,
+ &tracing_max_lat_fops);
+ }
+@@ -1748,8 +1755,8 @@ void latency_fsnotify(struct trace_array *tr)
+ || defined(CONFIG_OSNOISE_TRACER)
+
+ #define trace_create_maxlat_file(tr, d_tracer) \
+- trace_create_file("tracing_max_latency", 0644, d_tracer, \
+- &tr->max_latency, &tracing_max_lat_fops)
++ trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
++ d_tracer, &tr->max_latency, &tracing_max_lat_fops)
+
+ #else
+ #define trace_create_maxlat_file(tr, d_tracer) do { } while (0)
+@@ -3230,7 +3237,7 @@ struct trace_buffer_struct {
+ char buffer[4][TRACE_BUF_SIZE];
+ };
+
+-static struct trace_buffer_struct *trace_percpu_buffer;
++static struct trace_buffer_struct __percpu *trace_percpu_buffer;
+
+ /*
+ * This allows for lockless recording. If we're nested too deeply, then
+@@ -3240,7 +3247,7 @@ static char *get_trace_buf(void)
+ {
+ struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
+
+- if (!buffer || buffer->nesting >= 4)
++ if (!trace_percpu_buffer || buffer->nesting >= 4)
+ return NULL;
+
+ buffer->nesting++;
+@@ -3259,7 +3266,7 @@ static void put_trace_buf(void)
+
+ static int alloc_percpu_trace_buffer(void)
+ {
+- struct trace_buffer_struct *buffers;
++ struct trace_buffer_struct __percpu *buffers;
+
+ if (trace_percpu_buffer)
+ return 0;
+@@ -3671,12 +3678,17 @@ static char *trace_iter_expand_format(struct trace_iterator *iter)
+ }
+
+ /* Returns true if the string is safe to dereference from an event */
+-static bool trace_safe_str(struct trace_iterator *iter, const char *str)
++static bool trace_safe_str(struct trace_iterator *iter, const char *str,
++ bool star, int len)
+ {
+ unsigned long addr = (unsigned long)str;
+ struct trace_event *trace_event;
+ struct trace_event_call *event;
+
++ /* Ignore strings with no length */
++ if (star && !len)
++ return true;
++
+ /* OK if part of the event data */
+ if ((addr >= (unsigned long)iter->ent) &&
+ (addr < (unsigned long)iter->ent + iter->ent_size))
+@@ -3835,6 +3847,18 @@ void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
+ iter->fmt[i] = '\0';
+ trace_seq_vprintf(&iter->seq, iter->fmt, ap);
+
++ /*
++ * If iter->seq is full, the above call no longer guarantees
++ * that ap is in sync with fmt processing, and further calls
++ * to va_arg() can return wrong positional arguments.
++ *
++ * Ensure that ap is no longer used in this case.
++ */
++ if (iter->seq.full) {
++ p = "";
++ break;
++ }
++
+ if (star)
+ len = va_arg(ap, int);
+
+@@ -3850,7 +3874,7 @@ void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
+ * instead. See samples/trace_events/trace-events-sample.h
+ * for reference.
+ */
+- if (WARN_ONCE(!trace_safe_str(iter, str),
++ if (WARN_ONCE(!trace_safe_str(iter, str, star, len),
+ "fmt: '%s' current_buffer: '%s'",
+ fmt, show_buffer(&iter->seq))) {
+ int ret;
+@@ -6077,7 +6101,7 @@ trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
+
+ static void trace_create_eval_file(struct dentry *d_tracer)
+ {
+- trace_create_file("eval_map", 0444, d_tracer,
++ trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
+ NULL, &tracing_eval_map_fops);
+ }
+
+@@ -7736,7 +7760,8 @@ static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
+ err = kzalloc(sizeof(*err), GFP_KERNEL);
+ if (!err)
+ err = ERR_PTR(-ENOMEM);
+- tr->n_err_log_entries++;
++ else
++ tr->n_err_log_entries++;
+
+ return err;
+ }
+@@ -8590,27 +8615,27 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
+ }
+
+ /* per cpu trace_pipe */
+- trace_create_cpu_file("trace_pipe", 0444, d_cpu,
++ trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
+ tr, cpu, &tracing_pipe_fops);
+
+ /* per cpu trace */
+- trace_create_cpu_file("trace", 0644, d_cpu,
++ trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
+ tr, cpu, &tracing_fops);
+
+- trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
++ trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
+ tr, cpu, &tracing_buffers_fops);
+
+- trace_create_cpu_file("stats", 0444, d_cpu,
++ trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
+ tr, cpu, &tracing_stats_fops);
+
+- trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
++ trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
+ tr, cpu, &tracing_entries_fops);
+
+ #ifdef CONFIG_TRACER_SNAPSHOT
+- trace_create_cpu_file("snapshot", 0644, d_cpu,
++ trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
+ tr, cpu, &snapshot_fops);
+
+- trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
++ trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
+ tr, cpu, &snapshot_raw_fops);
+ #endif
+ }
+@@ -8816,8 +8841,8 @@ create_trace_option_file(struct trace_array *tr,
+ topt->opt = opt;
+ topt->tr = tr;
+
+- topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
+- &trace_options_fops);
++ topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
++ t_options, topt, &trace_options_fops);
+
+ }
+
+@@ -8892,7 +8917,7 @@ create_trace_option_core_file(struct trace_array *tr,
+ if (!t_options)
+ return NULL;
+
+- return trace_create_file(option, 0644, t_options,
++ return trace_create_file(option, TRACE_MODE_WRITE, t_options,
+ (void *)&tr->trace_flags_index[index],
+ &trace_options_core_fops);
+ }
+@@ -9417,28 +9442,28 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
+ struct trace_event_file *file;
+ int cpu;
+
+- trace_create_file("available_tracers", 0444, d_tracer,
++ trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
+ tr, &show_traces_fops);
+
+- trace_create_file("current_tracer", 0644, d_tracer,
++ trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
+ tr, &set_tracer_fops);
+
+- trace_create_file("tracing_cpumask", 0644, d_tracer,
++ trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
+ tr, &tracing_cpumask_fops);
+
+- trace_create_file("trace_options", 0644, d_tracer,
++ trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
+ tr, &tracing_iter_fops);
+
+- trace_create_file("trace", 0644, d_tracer,
++ trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
+ tr, &tracing_fops);
+
+- trace_create_file("trace_pipe", 0444, d_tracer,
++ trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
+ tr, &tracing_pipe_fops);
+
+- trace_create_file("buffer_size_kb", 0644, d_tracer,
++ trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
+ tr, &tracing_entries_fops);
+
+- trace_create_file("buffer_total_size_kb", 0444, d_tracer,
++ trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
+ tr, &tracing_total_entries_fops);
+
+ trace_create_file("free_buffer", 0200, d_tracer,
+@@ -9449,25 +9474,25 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
+
+ file = __find_event_file(tr, "ftrace", "print");
+ if (file && file->dir)
+- trace_create_file("trigger", 0644, file->dir, file,
+- &event_trigger_fops);
++ trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
++ file, &event_trigger_fops);
+ tr->trace_marker_file = file;
+
+ trace_create_file("trace_marker_raw", 0220, d_tracer,
+ tr, &tracing_mark_raw_fops);
+
+- trace_create_file("trace_clock", 0644, d_tracer, tr,
++ trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
+ &trace_clock_fops);
+
+- trace_create_file("tracing_on", 0644, d_tracer,
++ trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
+ tr, &rb_simple_fops);
+
+- trace_create_file("timestamp_mode", 0444, d_tracer, tr,
++ trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
+ &trace_time_stamp_mode_fops);
+
+ tr->buffer_percent = 50;
+
+- trace_create_file("buffer_percent", 0444, d_tracer,
++ trace_create_file("buffer_percent", TRACE_MODE_READ, d_tracer,
+ tr, &buffer_percent_fops);
+
+ create_trace_options_dir(tr);
+@@ -9478,11 +9503,11 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
+ MEM_FAIL(1, "Could not allocate function filter files");
+
+ #ifdef CONFIG_TRACER_SNAPSHOT
+- trace_create_file("snapshot", 0644, d_tracer,
++ trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
+ tr, &snapshot_fops);
+ #endif
+
+- trace_create_file("error_log", 0644, d_tracer,
++ trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
+ tr, &tracing_err_log_fops);
+
+ for_each_tracing_cpu(cpu)
+@@ -9675,19 +9700,19 @@ static __init int tracer_init_tracefs(void)
+ init_tracer_tracefs(&global_trace, NULL);
+ ftrace_init_tracefs_toplevel(&global_trace, NULL);
+
+- trace_create_file("tracing_thresh", 0644, NULL,
++ trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
+ &global_trace, &tracing_thresh_fops);
+
+- trace_create_file("README", 0444, NULL,
++ trace_create_file("README", TRACE_MODE_READ, NULL,
+ NULL, &tracing_readme_fops);
+
+- trace_create_file("saved_cmdlines", 0444, NULL,
++ trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
+ NULL, &tracing_saved_cmdlines_fops);
+
+- trace_create_file("saved_cmdlines_size", 0644, NULL,
++ trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
+ NULL, &tracing_saved_cmdlines_size_fops);
+
+- trace_create_file("saved_tgids", 0444, NULL,
++ trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
+ NULL, &tracing_saved_tgids_fops);
+
+ trace_eval_init();
+@@ -9699,7 +9724,7 @@ static __init int tracer_init_tracefs(void)
+ #endif
+
+ #ifdef CONFIG_DYNAMIC_FTRACE
+- trace_create_file("dyn_ftrace_total_info", 0444, NULL,
++ trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
+ NULL, &tracing_dyn_info_fops);
+ #endif
+
+diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
+index b7c0f8e160fb4..421374c304fc0 100644
+--- a/kernel/trace/trace.h
++++ b/kernel/trace/trace.h
+@@ -27,6 +27,9 @@
+ #include <asm/syscall.h> /* some archs define it here */
+ #endif
+
++#define TRACE_MODE_WRITE 0640
++#define TRACE_MODE_READ 0440
++
+ enum trace_type {
+ __TRACE_FIRST_TYPE = 0,
+
+@@ -1357,14 +1360,26 @@ __event_trigger_test_discard(struct trace_event_file *file,
+ if (eflags & EVENT_FILE_FL_TRIGGER_COND)
+ *tt = event_triggers_call(file, buffer, entry, event);
+
+- if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
+- (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
+- !filter_match_preds(file->filter, entry))) {
+- __trace_event_discard_commit(buffer, event);
+- return true;
+- }
++ if (likely(!(file->flags & (EVENT_FILE_FL_SOFT_DISABLED |
++ EVENT_FILE_FL_FILTERED |
++ EVENT_FILE_FL_PID_FILTER))))
++ return false;
++
++ if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
++ goto discard;
++
++ if (file->flags & EVENT_FILE_FL_FILTERED &&
++ !filter_match_preds(file->filter, entry))
++ goto discard;
++
++ if ((file->flags & EVENT_FILE_FL_PID_FILTER) &&
++ trace_event_ignore_this_pid(file))
++ goto discard;
+
+ return false;
++ discard:
++ __trace_event_discard_commit(buffer, event);
++ return true;
+ }
+
+ /**
+diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
+index 8d252f63cd784..0580287d7a0d1 100644
+--- a/kernel/trace/trace_boot.c
++++ b/kernel/trace/trace_boot.c
+@@ -430,6 +430,8 @@ trace_boot_init_histograms(struct trace_event_file *file,
+ /* All digit started node should be instances. */
+ if (trace_boot_compose_hist_cmd(node, buf, size) == 0) {
+ tmp = kstrdup(buf, GFP_KERNEL);
++ if (!tmp)
++ return;
+ if (trigger_process_regex(file, buf) < 0)
+ pr_err("Failed to apply hist trigger: %s\n", tmp);
+ kfree(tmp);
+@@ -439,6 +441,8 @@ trace_boot_init_histograms(struct trace_event_file *file,
+ if (xbc_node_find_subkey(hnode, "keys")) {
+ if (trace_boot_compose_hist_cmd(hnode, buf, size) == 0) {
+ tmp = kstrdup(buf, GFP_KERNEL);
++ if (!tmp)
++ return;
+ if (trigger_process_regex(file, buf) < 0)
+ pr_err("Failed to apply hist trigger: %s\n", tmp);
+ kfree(tmp);
+diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c
+index 1110112e55bd7..e34e8182ee4b5 100644
+--- a/kernel/trace/trace_dynevent.c
++++ b/kernel/trace/trace_dynevent.c
+@@ -262,7 +262,7 @@ static __init int init_dynamic_event(void)
+ if (ret)
+ return 0;
+
+- entry = tracefs_create_file("dynamic_events", 0644, NULL,
++ entry = tracefs_create_file("dynamic_events", TRACE_MODE_WRITE, NULL,
+ NULL, &dynamic_events_ops);
+
+ /* Event list interface */
+diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
+index 6aed10e2f7ce0..fba8cb77a73af 100644
+--- a/kernel/trace/trace_event_perf.c
++++ b/kernel/trace/trace_event_perf.c
+@@ -441,13 +441,13 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
+ if (!rcu_is_watching())
+ return;
+
+- if ((unsigned long)ops->private != smp_processor_id())
+- return;
+-
+ bit = ftrace_test_recursion_trylock(ip, parent_ip);
+ if (bit < 0)
+ return;
+
++ if ((unsigned long)ops->private != smp_processor_id())
++ goto out;
++
+ event = container_of(ops, struct perf_event, ftrace_ops);
+
+ /*
+diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
+index 830b3b9940f4c..c072e8b9849c1 100644
+--- a/kernel/trace/trace_events.c
++++ b/kernel/trace/trace_events.c
+@@ -40,6 +40,14 @@ static LIST_HEAD(ftrace_generic_fields);
+ static LIST_HEAD(ftrace_common_fields);
+ static bool eventdir_initialized;
+
++static LIST_HEAD(module_strings);
++
++struct module_string {
++ struct list_head next;
++ struct module *module;
++ char *str;
++};
++
+ #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
+
+ static struct kmem_cache *field_cachep;
+@@ -2312,7 +2320,8 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
+ /* the ftrace system is special, do not create enable or filter files */
+ if (strcmp(name, "ftrace") != 0) {
+
+- entry = tracefs_create_file("filter", 0644, dir->entry, dir,
++ entry = tracefs_create_file("filter", TRACE_MODE_WRITE,
++ dir->entry, dir,
+ &ftrace_subsystem_filter_fops);
+ if (!entry) {
+ kfree(system->filter);
+@@ -2320,7 +2329,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
+ pr_warn("Could not create tracefs '%s/filter' entry\n", name);
+ }
+
+- trace_create_file("enable", 0644, dir->entry, dir,
++ trace_create_file("enable", TRACE_MODE_WRITE, dir->entry, dir,
+ &ftrace_system_enable_fops);
+ }
+
+@@ -2402,12 +2411,12 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
+ }
+
+ if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
+- trace_create_file("enable", 0644, file->dir, file,
++ trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file,
+ &ftrace_enable_fops);
+
+ #ifdef CONFIG_PERF_EVENTS
+ if (call->event.type && call->class->reg)
+- trace_create_file("id", 0444, file->dir,
++ trace_create_file("id", TRACE_MODE_READ, file->dir,
+ (void *)(long)call->event.type,
+ &ftrace_event_id_fops);
+ #endif
+@@ -2423,22 +2432,22 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
+ * triggers or filters.
+ */
+ if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
+- trace_create_file("filter", 0644, file->dir, file,
+- &ftrace_event_filter_fops);
++ trace_create_file("filter", TRACE_MODE_WRITE, file->dir,
++ file, &ftrace_event_filter_fops);
+
+- trace_create_file("trigger", 0644, file->dir, file,
+- &event_trigger_fops);
++ trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
++ file, &event_trigger_fops);
+ }
+
+ #ifdef CONFIG_HIST_TRIGGERS
+- trace_create_file("hist", 0444, file->dir, file,
++ trace_create_file("hist", TRACE_MODE_READ, file->dir, file,
+ &event_hist_fops);
+ #endif
+ #ifdef CONFIG_HIST_TRIGGERS_DEBUG
+- trace_create_file("hist_debug", 0444, file->dir, file,
++ trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file,
+ &event_hist_debug_fops);
+ #endif
+- trace_create_file("format", 0444, file->dir, call,
++ trace_create_file("format", TRACE_MODE_READ, file->dir, call,
+ &ftrace_event_format_fops);
+
+ #ifdef CONFIG_TRACE_EVENT_INJECT
+@@ -2632,6 +2641,76 @@ static void update_event_printk(struct trace_event_call *call,
+ }
+ }
+
++static void add_str_to_module(struct module *module, char *str)
++{
++ struct module_string *modstr;
++
++ modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
++
++ /*
++ * If we failed to allocate memory here, then we'll just
++ * let the str memory leak when the module is removed.
++ * If this fails to allocate, there's worse problems than
++ * a leaked string on module removal.
++ */
++ if (WARN_ON_ONCE(!modstr))
++ return;
++
++ modstr->module = module;
++ modstr->str = str;
++
++ list_add(&modstr->next, &module_strings);
++}
++
++static void update_event_fields(struct trace_event_call *call,
++ struct trace_eval_map *map)
++{
++ struct ftrace_event_field *field;
++ struct list_head *head;
++ char *ptr;
++ char *str;
++ int len = strlen(map->eval_string);
++
++ /* Dynamic events should never have field maps */
++ if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
++ return;
++
++ head = trace_get_fields(call);
++ list_for_each_entry(field, head, link) {
++ ptr = strchr(field->type, '[');
++ if (!ptr)
++ continue;
++ ptr++;
++
++ if (!isalpha(*ptr) && *ptr != '_')
++ continue;
++
++ if (strncmp(map->eval_string, ptr, len) != 0)
++ continue;
++
++ str = kstrdup(field->type, GFP_KERNEL);
++ if (WARN_ON_ONCE(!str))
++ return;
++ ptr = str + (ptr - field->type);
++ ptr = eval_replace(ptr, map, len);
++ /* enum/sizeof string smaller than value */
++ if (WARN_ON_ONCE(!ptr)) {
++ kfree(str);
++ continue;
++ }
++
++ /*
++ * If the event is part of a module, then we need to free the string
++ * when the module is removed. Otherwise, it will stay allocated
++ * until a reboot.
++ */
++ if (call->module)
++ add_str_to_module(call->module, str);
++
++ field->type = str;
++ }
++}
++
+ void trace_event_eval_update(struct trace_eval_map **map, int len)
+ {
+ struct trace_event_call *call, *p;
+@@ -2667,6 +2746,7 @@ void trace_event_eval_update(struct trace_eval_map **map, int len)
+ first = false;
+ }
+ update_event_printk(call, map[i]);
++ update_event_fields(call, map[i]);
+ }
+ }
+ }
+@@ -2677,12 +2757,22 @@ static struct trace_event_file *
+ trace_create_new_event(struct trace_event_call *call,
+ struct trace_array *tr)
+ {
++ struct trace_pid_list *no_pid_list;
++ struct trace_pid_list *pid_list;
+ struct trace_event_file *file;
+
+ file = kmem_cache_alloc(file_cachep, GFP_TRACE);
+ if (!file)
+ return NULL;
+
++ pid_list = rcu_dereference_protected(tr->filtered_pids,
++ lockdep_is_held(&event_mutex));
++ no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
++ lockdep_is_held(&event_mutex));
++
++ if (pid_list || no_pid_list)
++ file->flags |= EVENT_FILE_FL_PID_FILTER;
++
+ file->event_call = call;
+ file->tr = tr;
+ atomic_set(&file->sm_ref, 0);
+@@ -2840,6 +2930,7 @@ static void trace_module_add_events(struct module *mod)
+ static void trace_module_remove_events(struct module *mod)
+ {
+ struct trace_event_call *call, *p;
++ struct module_string *modstr, *m;
+
+ down_write(&trace_event_sem);
+ list_for_each_entry_safe(call, p, &ftrace_events, list) {
+@@ -2848,6 +2939,14 @@ static void trace_module_remove_events(struct module *mod)
+ if (call->module == mod)
+ __trace_remove_event_call(call);
+ }
++ /* Check for any strings allocade for this module */
++ list_for_each_entry_safe(modstr, m, &module_strings, next) {
++ if (modstr->module != mod)
++ continue;
++ list_del(&modstr->next);
++ kfree(modstr->str);
++ kfree(modstr);
++ }
+ up_write(&trace_event_sem);
+
+ /*
+@@ -3433,7 +3532,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
+ struct dentry *d_events;
+ struct dentry *entry;
+
+- entry = tracefs_create_file("set_event", 0644, parent,
++ entry = tracefs_create_file("set_event", TRACE_MODE_WRITE, parent,
+ tr, &ftrace_set_event_fops);
+ if (!entry) {
+ pr_warn("Could not create tracefs 'set_event' entry\n");
+@@ -3446,7 +3545,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
+ return -ENOMEM;
+ }
+
+- entry = trace_create_file("enable", 0644, d_events,
++ entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events,
+ tr, &ftrace_tr_enable_fops);
+ if (!entry) {
+ pr_warn("Could not create tracefs 'enable' entry\n");
+@@ -3455,24 +3554,25 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
+
+ /* There are not as crucial, just warn if they are not created */
+
+- entry = tracefs_create_file("set_event_pid", 0644, parent,
++ entry = tracefs_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
+ tr, &ftrace_set_event_pid_fops);
+ if (!entry)
+ pr_warn("Could not create tracefs 'set_event_pid' entry\n");
+
+- entry = tracefs_create_file("set_event_notrace_pid", 0644, parent,
+- tr, &ftrace_set_event_notrace_pid_fops);
++ entry = tracefs_create_file("set_event_notrace_pid",
++ TRACE_MODE_WRITE, parent, tr,
++ &ftrace_set_event_notrace_pid_fops);
+ if (!entry)
+ pr_warn("Could not create tracefs 'set_event_notrace_pid' entry\n");
+
+ /* ring buffer internal formats */
+- entry = trace_create_file("header_page", 0444, d_events,
++ entry = trace_create_file("header_page", TRACE_MODE_READ, d_events,
+ ring_buffer_print_page_header,
+ &ftrace_show_header_fops);
+ if (!entry)
+ pr_warn("Could not create tracefs 'header_page' entry\n");
+
+- entry = trace_create_file("header_event", 0444, d_events,
++ entry = trace_create_file("header_event", TRACE_MODE_READ, d_events,
+ ring_buffer_print_entry_header,
+ &ftrace_show_header_fops);
+ if (!entry)
+@@ -3689,8 +3789,8 @@ __init int event_trace_init(void)
+ if (!tr)
+ return -ENODEV;
+
+- entry = tracefs_create_file("available_events", 0444, NULL,
+- tr, &ftrace_avail_fops);
++ entry = tracefs_create_file("available_events", TRACE_MODE_READ,
++ NULL, tr, &ftrace_avail_fops);
+ if (!entry)
+ pr_warn("Could not create tracefs 'available_events' entry\n");
+
+diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
+index c9124038b140f..06d6318ee5377 100644
+--- a/kernel/trace/trace_events_filter.c
++++ b/kernel/trace/trace_events_filter.c
+@@ -5,6 +5,7 @@
+ * Copyright (C) 2009 Tom Zanussi <tzanussi@gmail.com>
+ */
+
++#include <linux/uaccess.h>
+ #include <linux/module.h>
+ #include <linux/ctype.h>
+ #include <linux/mutex.h>
+@@ -654,6 +655,52 @@ DEFINE_EQUALITY_PRED(32);
+ DEFINE_EQUALITY_PRED(16);
+ DEFINE_EQUALITY_PRED(8);
+
++/* user space strings temp buffer */
++#define USTRING_BUF_SIZE 1024
++
++struct ustring_buffer {
++ char buffer[USTRING_BUF_SIZE];
++};
++
++static __percpu struct ustring_buffer *ustring_per_cpu;
++
++static __always_inline char *test_string(char *str)
++{
++ struct ustring_buffer *ubuf;
++ char *kstr;
++
++ if (!ustring_per_cpu)
++ return NULL;
++
++ ubuf = this_cpu_ptr(ustring_per_cpu);
++ kstr = ubuf->buffer;
++
++ /* For safety, do not trust the string pointer */
++ if (!strncpy_from_kernel_nofault(kstr, str, USTRING_BUF_SIZE))
++ return NULL;
++ return kstr;
++}
++
++static __always_inline char *test_ustring(char *str)
++{
++ struct ustring_buffer *ubuf;
++ char __user *ustr;
++ char *kstr;
++
++ if (!ustring_per_cpu)
++ return NULL;
++
++ ubuf = this_cpu_ptr(ustring_per_cpu);
++ kstr = ubuf->buffer;
++
++ /* user space address? */
++ ustr = (char __user *)str;
++ if (!strncpy_from_user_nofault(kstr, ustr, USTRING_BUF_SIZE))
++ return NULL;
++
++ return kstr;
++}
++
+ /* Filter predicate for fixed sized arrays of characters */
+ static int filter_pred_string(struct filter_pred *pred, void *event)
+ {
+@@ -667,19 +714,43 @@ static int filter_pred_string(struct filter_pred *pred, void *event)
+ return match;
+ }
+
+-/* Filter predicate for char * pointers */
+-static int filter_pred_pchar(struct filter_pred *pred, void *event)
++static __always_inline int filter_pchar(struct filter_pred *pred, char *str)
+ {
+- char **addr = (char **)(event + pred->offset);
+ int cmp, match;
+- int len = strlen(*addr) + 1; /* including tailing '\0' */
++ int len;
+
+- cmp = pred->regex.match(*addr, &pred->regex, len);
++ len = strlen(str) + 1; /* including tailing '\0' */
++ cmp = pred->regex.match(str, &pred->regex, len);
+
+ match = cmp ^ pred->not;
+
+ return match;
+ }
++/* Filter predicate for char * pointers */
++static int filter_pred_pchar(struct filter_pred *pred, void *event)
++{
++ char **addr = (char **)(event + pred->offset);
++ char *str;
++
++ str = test_string(*addr);
++ if (!str)
++ return 0;
++
++ return filter_pchar(pred, str);
++}
++
++/* Filter predicate for char * pointers in user space*/
++static int filter_pred_pchar_user(struct filter_pred *pred, void *event)
++{
++ char **addr = (char **)(event + pred->offset);
++ char *str;
++
++ str = test_ustring(*addr);
++ if (!str)
++ return 0;
++
++ return filter_pchar(pred, str);
++}
+
+ /*
+ * Filter predicate for dynamic sized arrays of characters.
+@@ -1158,6 +1229,7 @@ static int parse_pred(const char *str, void *data,
+ struct filter_pred *pred = NULL;
+ char num_buf[24]; /* Big enough to hold an address */
+ char *field_name;
++ bool ustring = false;
+ char q;
+ u64 val;
+ int len;
+@@ -1192,6 +1264,12 @@ static int parse_pred(const char *str, void *data,
+ return -EINVAL;
+ }
+
++ /* See if the field is a user space string */
++ if ((len = str_has_prefix(str + i, ".ustring"))) {
++ ustring = true;
++ i += len;
++ }
++
+ while (isspace(str[i]))
+ i++;
+
+@@ -1320,8 +1398,20 @@ static int parse_pred(const char *str, void *data,
+
+ } else if (field->filter_type == FILTER_DYN_STRING)
+ pred->fn = filter_pred_strloc;
+- else
+- pred->fn = filter_pred_pchar;
++ else {
++
++ if (!ustring_per_cpu) {
++ /* Once allocated, keep it around for good */
++ ustring_per_cpu = alloc_percpu(struct ustring_buffer);
++ if (!ustring_per_cpu)
++ goto err_mem;
++ }
++
++ if (ustring)
++ pred->fn = filter_pred_pchar_user;
++ else
++ pred->fn = filter_pred_pchar;
++ }
+ /* go past the last quote */
+ i++;
+
+@@ -1387,6 +1477,9 @@ static int parse_pred(const char *str, void *data,
+ err_free:
+ kfree(pred);
+ return -EINVAL;
++err_mem:
++ kfree(pred);
++ return -ENOMEM;
+ }
+
+ enum {
+diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
+index f01e442716e2f..ea168d42c8a2a 100644
+--- a/kernel/trace/trace_events_hist.c
++++ b/kernel/trace/trace_events_hist.c
+@@ -1733,9 +1733,10 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
+ if (!hist_field->type)
+ goto free;
+
+- if (field->filter_type == FILTER_STATIC_STRING)
++ if (field->filter_type == FILTER_STATIC_STRING) {
+ hist_field->fn = hist_field_string;
+- else if (field->filter_type == FILTER_DYN_STRING)
++ hist_field->size = field->size;
++ } else if (field->filter_type == FILTER_DYN_STRING)
+ hist_field->fn = hist_field_dynstring;
+ else
+ hist_field->fn = hist_field_pstring;
+@@ -2048,9 +2049,9 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
+ /*
+ * For backward compatibility, if field_name
+ * was "cpu", then we treat this the same as
+- * common_cpu.
++ * common_cpu. This also works for "CPU".
+ */
+- if (strcmp(field_name, "cpu") == 0) {
++ if (field && field->filter_type == FILTER_CPU) {
+ *flags |= HIST_FIELD_FL_CPU;
+ } else {
+ hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
+@@ -2219,6 +2220,8 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
+ (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
+ expr->fn = hist_field_unary_minus;
+ expr->operands[0] = operand1;
++ expr->size = operand1->size;
++ expr->is_signed = operand1->is_signed;
+ expr->operator = FIELD_OP_UNARY_MINUS;
+ expr->name = expr_str(expr, 0);
+ expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
+@@ -2358,6 +2361,7 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
+
+ /* The operand sizes should be the same, so just pick one */
+ expr->size = operand1->size;
++ expr->is_signed = operand1->is_signed;
+
+ expr->operator = field_op;
+ expr->name = expr_str(expr, 0);
+@@ -2690,8 +2694,10 @@ static inline void __update_field_vars(struct tracing_map_elt *elt,
+ if (val->flags & HIST_FIELD_FL_STRING) {
+ char *str = elt_data->field_var_str[j++];
+ char *val_str = (char *)(uintptr_t)var_val;
++ unsigned int size;
+
+- strscpy(str, val_str, STR_VAR_LEN_MAX);
++ size = min(val->size, STR_VAR_LEN_MAX);
++ strscpy(str, val_str, size);
+ var_val = (u64)(uintptr_t)str;
+ }
+ tracing_map_set_var(elt, var_idx, var_val);
+@@ -3416,7 +3422,7 @@ static int check_synth_field(struct synth_event *event,
+
+ if (strcmp(field->type, hist_field->type) != 0) {
+ if (field->size != hist_field->size ||
+- field->is_signed != hist_field->is_signed)
++ (!field->is_string && field->is_signed != hist_field->is_signed))
+ return -EINVAL;
+ }
+
+@@ -3578,6 +3584,7 @@ static int trace_action_create(struct hist_trigger_data *hist_data,
+
+ var_ref_idx = find_var_ref_idx(hist_data, var_ref);
+ if (WARN_ON(var_ref_idx < 0)) {
++ kfree(p);
+ ret = var_ref_idx;
+ goto err;
+ }
+@@ -4471,7 +4478,7 @@ static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
+
+ if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
+ cmp_fn = tracing_map_cmp_none;
+- else if (!field)
++ else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
+ cmp_fn = tracing_map_cmp_num(hist_field->size,
+ hist_field->is_signed);
+ else if (is_string_field(field))
+@@ -4578,6 +4585,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
+ if (hist_field->flags & HIST_FIELD_FL_STRING) {
+ unsigned int str_start, var_str_idx, idx;
+ char *str, *val_str;
++ unsigned int size;
+
+ str_start = hist_data->n_field_var_str +
+ hist_data->n_save_var_str;
+@@ -4586,7 +4594,9 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
+
+ str = elt_data->field_var_str[idx];
+ val_str = (char *)(uintptr_t)hist_val;
+- strscpy(str, val_str, STR_VAR_LEN_MAX);
++
++ size = min(hist_field->size, STR_VAR_LEN_MAX);
++ strscpy(str, val_str, size);
+
+ hist_val = (u64)(uintptr_t)str;
+ }
+diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c
+index d54094b7a9d75..8c26092db8dee 100644
+--- a/kernel/trace/trace_events_synth.c
++++ b/kernel/trace/trace_events_synth.c
+@@ -2053,6 +2053,13 @@ static int create_synth_event(const char *raw_command)
+
+ last_cmd_set(raw_command);
+
++ name = raw_command;
++
++ /* Don't try to process if not our system */
++ if (name[0] != 's' || name[1] != ':')
++ return -ECANCELED;
++ name += 2;
++
+ p = strpbrk(raw_command, " \t");
+ if (!p) {
+ synth_err(SYNTH_ERR_INVALID_CMD, 0);
+@@ -2061,12 +2068,6 @@ static int create_synth_event(const char *raw_command)
+
+ fields = skip_spaces(p);
+
+- name = raw_command;
+-
+- if (name[0] != 's' || name[1] != ':')
+- return -ECANCELED;
+- name += 2;
+-
+ /* This interface accepts group name prefix */
+ if (strchr(name, '/')) {
+ len = str_has_prefix(name, SYNTH_SYSTEM "/");
+@@ -2227,8 +2228,8 @@ static __init int trace_events_synth_init(void)
+ if (err)
+ goto err;
+
+- entry = tracefs_create_file("synthetic_events", 0644, NULL,
+- NULL, &synth_events_fops);
++ entry = tracefs_create_file("synthetic_events", TRACE_MODE_WRITE,
++ NULL, NULL, &synth_events_fops);
+ if (!entry) {
+ err = -ENODEV;
+ goto err;
+diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
+index 3d5c07239a2a8..67c7979c40c0b 100644
+--- a/kernel/trace/trace_events_trigger.c
++++ b/kernel/trace/trace_events_trigger.c
+@@ -955,6 +955,16 @@ traceon_trigger(struct event_trigger_data *data,
+ struct trace_buffer *buffer, void *rec,
+ struct ring_buffer_event *event)
+ {
++ struct trace_event_file *file = data->private_data;
++
++ if (file) {
++ if (tracer_tracing_is_on(file->tr))
++ return;
++
++ tracer_tracing_on(file->tr);
++ return;
++ }
++
+ if (tracing_is_on())
+ return;
+
+@@ -966,8 +976,15 @@ traceon_count_trigger(struct event_trigger_data *data,
+ struct trace_buffer *buffer, void *rec,
+ struct ring_buffer_event *event)
+ {
+- if (tracing_is_on())
+- return;
++ struct trace_event_file *file = data->private_data;
++
++ if (file) {
++ if (tracer_tracing_is_on(file->tr))
++ return;
++ } else {
++ if (tracing_is_on())
++ return;
++ }
+
+ if (!data->count)
+ return;
+@@ -975,7 +992,10 @@ traceon_count_trigger(struct event_trigger_data *data,
+ if (data->count != -1)
+ (data->count)--;
+
+- tracing_on();
++ if (file)
++ tracer_tracing_on(file->tr);
++ else
++ tracing_on();
+ }
+
+ static void
+@@ -983,6 +1003,16 @@ traceoff_trigger(struct event_trigger_data *data,
+ struct trace_buffer *buffer, void *rec,
+ struct ring_buffer_event *event)
+ {
++ struct trace_event_file *file = data->private_data;
++
++ if (file) {
++ if (!tracer_tracing_is_on(file->tr))
++ return;
++
++ tracer_tracing_off(file->tr);
++ return;
++ }
++
+ if (!tracing_is_on())
+ return;
+
+@@ -994,8 +1024,15 @@ traceoff_count_trigger(struct event_trigger_data *data,
+ struct trace_buffer *buffer, void *rec,
+ struct ring_buffer_event *event)
+ {
+- if (!tracing_is_on())
+- return;
++ struct trace_event_file *file = data->private_data;
++
++ if (file) {
++ if (!tracer_tracing_is_on(file->tr))
++ return;
++ } else {
++ if (!tracing_is_on())
++ return;
++ }
+
+ if (!data->count)
+ return;
+@@ -1003,7 +1040,10 @@ traceoff_count_trigger(struct event_trigger_data *data,
+ if (data->count != -1)
+ (data->count)--;
+
+- tracing_off();
++ if (file)
++ tracer_tracing_off(file->tr);
++ else
++ tracing_off();
+ }
+
+ static int
+@@ -1200,7 +1240,12 @@ stacktrace_trigger(struct event_trigger_data *data,
+ struct trace_buffer *buffer, void *rec,
+ struct ring_buffer_event *event)
+ {
+- trace_dump_stack(STACK_SKIP);
++ struct trace_event_file *file = data->private_data;
++
++ if (file)
++ __trace_stack(file->tr, tracing_gen_ctx(), STACK_SKIP);
++ else
++ trace_dump_stack(STACK_SKIP);
+ }
+
+ static void
+diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
+index 0de6837722da5..6b5ff3ba4251f 100644
+--- a/kernel/trace/trace_functions_graph.c
++++ b/kernel/trace/trace_functions_graph.c
+@@ -1340,7 +1340,7 @@ static __init int init_graph_tracefs(void)
+ if (ret)
+ return 0;
+
+- trace_create_file("max_graph_depth", 0644, NULL,
++ trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL,
+ NULL, &graph_depth_fops);
+
+ return 0;
+diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c
+index 1b83d75eb103b..d0a730d99a331 100644
+--- a/kernel/trace/trace_hwlat.c
++++ b/kernel/trace/trace_hwlat.c
+@@ -782,21 +782,21 @@ static int init_tracefs(void)
+ if (!top_dir)
+ return -ENOMEM;
+
+- hwlat_sample_window = tracefs_create_file("window", 0640,
++ hwlat_sample_window = tracefs_create_file("window", TRACE_MODE_WRITE,
+ top_dir,
+ &hwlat_window,
+ &trace_min_max_fops);
+ if (!hwlat_sample_window)
+ goto err;
+
+- hwlat_sample_width = tracefs_create_file("width", 0644,
++ hwlat_sample_width = tracefs_create_file("width", TRACE_MODE_WRITE,
+ top_dir,
+ &hwlat_width,
+ &trace_min_max_fops);
+ if (!hwlat_sample_width)
+ goto err;
+
+- hwlat_thread_mode = trace_create_file("mode", 0644,
++ hwlat_thread_mode = trace_create_file("mode", TRACE_MODE_WRITE,
+ top_dir,
+ NULL,
+ &thread_mode_fops);
+diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
+index 3a64ba4bbad6f..39ee60725519b 100644
+--- a/kernel/trace/trace_kprobe.c
++++ b/kernel/trace/trace_kprobe.c
+@@ -31,7 +31,7 @@ static int __init set_kprobe_boot_events(char *str)
+ strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
+ disable_tracing_selftest("running kprobe events");
+
+- return 0;
++ return 1;
+ }
+ __setup("kprobe_event=", set_kprobe_boot_events);
+
+@@ -1175,15 +1175,18 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
+ {
+ struct dyn_event *ev = v;
+ struct trace_kprobe *tk;
++ unsigned long nmissed;
+
+ if (!is_trace_kprobe(ev))
+ return 0;
+
+ tk = to_trace_kprobe(ev);
++ nmissed = trace_kprobe_is_return(tk) ?
++ tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
+ seq_printf(m, " %-44s %15lu %15lu\n",
+ trace_probe_name(&tk->tp),
+ trace_kprobe_nhit(tk),
+- tk->rp.kp.nmissed);
++ nmissed);
+
+ return 0;
+ }
+@@ -1925,16 +1928,16 @@ static __init int init_kprobe_trace(void)
+ if (ret)
+ return 0;
+
+- entry = tracefs_create_file("kprobe_events", 0644, NULL,
+- NULL, &kprobe_events_ops);
++ entry = tracefs_create_file("kprobe_events", TRACE_MODE_WRITE,
++ NULL, NULL, &kprobe_events_ops);
+
+ /* Event list interface */
+ if (!entry)
+ pr_warn("Could not create tracefs 'kprobe_events' entry\n");
+
+ /* Profile interface */
+- entry = tracefs_create_file("kprobe_profile", 0444, NULL,
+- NULL, &kprobe_profile_ops);
++ entry = tracefs_create_file("kprobe_profile", TRACE_MODE_READ,
++ NULL, NULL, &kprobe_profile_ops);
+
+ if (!entry)
+ pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
+diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c
+index ce053619f289e..93de784ee681c 100644
+--- a/kernel/trace/trace_osnoise.c
++++ b/kernel/trace/trace_osnoise.c
+@@ -1195,6 +1195,26 @@ static int run_osnoise(void)
+ osnoise_stop_tracing();
+ }
+
++ /*
++ * In some cases, notably when running on a nohz_full CPU with
++ * a stopped tick PREEMPT_RCU has no way to account for QSs.
++ * This will eventually cause unwarranted noise as PREEMPT_RCU
++ * will force preemption as the means of ending the current
++ * grace period. We avoid this problem by calling
++ * rcu_momentary_dyntick_idle(), which performs a zero duration
++ * EQS allowing PREEMPT_RCU to end the current grace period.
++ * This call shouldn't be wrapped inside an RCU critical
++ * section.
++ *
++ * Note that in non PREEMPT_RCU kernels QSs are handled through
++ * cond_resched()
++ */
++ if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
++ local_irq_disable();
++ rcu_momentary_dyntick_idle();
++ local_irq_enable();
++ }
++
+ /*
+ * For the non-preemptive kernel config: let threads runs, if
+ * they so wish.
+@@ -1249,6 +1269,37 @@ out:
+ static struct cpumask osnoise_cpumask;
+ static struct cpumask save_cpumask;
+
++/*
++ * osnoise_sleep - sleep until the next period
++ */
++static void osnoise_sleep(void)
++{
++ u64 interval;
++ ktime_t wake_time;
++
++ mutex_lock(&interface_lock);
++ interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
++ mutex_unlock(&interface_lock);
++
++ /*
++ * differently from hwlat_detector, the osnoise tracer can run
++ * without a pause because preemption is on.
++ */
++ if (!interval) {
++ /* Let synchronize_rcu_tasks() make progress */
++ cond_resched_tasks_rcu_qs();
++ return;
++ }
++
++ wake_time = ktime_add_us(ktime_get(), interval);
++ __set_current_state(TASK_INTERRUPTIBLE);
++
++ while (schedule_hrtimeout_range(&wake_time, 0, HRTIMER_MODE_ABS)) {
++ if (kthread_should_stop())
++ break;
++ }
++}
++
+ /*
+ * osnoise_main - The osnoise detection kernel thread
+ *
+@@ -1257,30 +1308,10 @@ static struct cpumask save_cpumask;
+ */
+ static int osnoise_main(void *data)
+ {
+- u64 interval;
+
+ while (!kthread_should_stop()) {
+-
+ run_osnoise();
+-
+- mutex_lock(&interface_lock);
+- interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
+- mutex_unlock(&interface_lock);
+-
+- do_div(interval, USEC_PER_MSEC);
+-
+- /*
+- * differently from hwlat_detector, the osnoise tracer can run
+- * without a pause because preemption is on.
+- */
+- if (interval < 1) {
+- /* Let synchronize_rcu_tasks() make progress */
+- cond_resched_tasks_rcu_qs();
+- continue;
+- }
+-
+- if (msleep_interruptible(interval))
+- break;
++ osnoise_sleep();
+ }
+
+ return 0;
+@@ -1856,38 +1887,38 @@ static int init_tracefs(void)
+ if (!top_dir)
+ return 0;
+
+- tmp = tracefs_create_file("period_us", 0640, top_dir,
++ tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
+ &osnoise_period, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+
+- tmp = tracefs_create_file("runtime_us", 0644, top_dir,
++ tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
+ &osnoise_runtime, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+
+- tmp = tracefs_create_file("stop_tracing_us", 0640, top_dir,
++ tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
+ &osnoise_stop_tracing_in, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+
+- tmp = tracefs_create_file("stop_tracing_total_us", 0640, top_dir,
++ tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
+ &osnoise_stop_tracing_total, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+
+- tmp = trace_create_file("cpus", 0644, top_dir, NULL, &cpus_fops);
++ tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
+ if (!tmp)
+ goto err;
+ #ifdef CONFIG_TIMERLAT_TRACER
+ #ifdef CONFIG_STACKTRACE
+- tmp = tracefs_create_file("print_stack", 0640, top_dir,
++ tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
+ &osnoise_print_stack, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+ #endif
+
+- tmp = tracefs_create_file("timerlat_period_us", 0640, top_dir,
++ tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
+ &timerlat_period, &trace_min_max_fops);
+ if (!tmp)
+ goto err;
+@@ -1932,6 +1963,13 @@ out_unhook_irq:
+ return -EINVAL;
+ }
+
++static void osnoise_unhook_events(void)
++{
++ unhook_thread_events();
++ unhook_softirq_events();
++ unhook_irq_events();
++}
++
+ static int __osnoise_tracer_start(struct trace_array *tr)
+ {
+ int retval;
+@@ -1949,7 +1987,14 @@ static int __osnoise_tracer_start(struct trace_array *tr)
+
+ retval = start_per_cpu_kthreads(tr);
+ if (retval) {
+- unhook_irq_events();
++ trace_osnoise_callback_enabled = false;
++ /*
++ * Make sure that ftrace_nmi_enter/exit() see
++ * trace_osnoise_callback_enabled as false before continuing.
++ */
++ barrier();
++
++ osnoise_unhook_events();
+ return retval;
+ }
+
+@@ -1981,9 +2026,7 @@ static void osnoise_tracer_stop(struct trace_array *tr)
+
+ stop_per_cpu_kthreads();
+
+- unhook_irq_events();
+- unhook_softirq_events();
+- unhook_thread_events();
++ osnoise_unhook_events();
+
+ osnoise_busy = false;
+ }
+diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
+index 4b320fe7df704..29f6e95439b67 100644
+--- a/kernel/trace/trace_printk.c
++++ b/kernel/trace/trace_printk.c
+@@ -384,7 +384,7 @@ static __init int init_trace_printk_function_export(void)
+ if (ret)
+ return 0;
+
+- trace_create_file("printk_formats", 0444, NULL,
++ trace_create_file("printk_formats", TRACE_MODE_READ, NULL,
+ NULL, &ftrace_formats_fops);
+
+ return 0;
+diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
+index 3ed2a3f372972..bb4605b60de79 100644
+--- a/kernel/trace/trace_probe.c
++++ b/kernel/trace/trace_probe.c
+@@ -356,6 +356,8 @@ static int __parse_imm_string(char *str, char **pbuf, int offs)
+ return -EINVAL;
+ }
+ *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
++ if (!*pbuf)
++ return -ENOMEM;
+ return 0;
+ }
+
+diff --git a/kernel/trace/trace_recursion_record.c b/kernel/trace/trace_recursion_record.c
+index b2edac1fe156e..4d4b78c8ca257 100644
+--- a/kernel/trace/trace_recursion_record.c
++++ b/kernel/trace/trace_recursion_record.c
+@@ -226,8 +226,8 @@ __init static int create_recursed_functions(void)
+ {
+ struct dentry *dentry;
+
+- dentry = trace_create_file("recursed_functions", 0644, NULL, NULL,
+- &recursed_functions_fops);
++ dentry = trace_create_file("recursed_functions", TRACE_MODE_WRITE,
++ NULL, NULL, &recursed_functions_fops);
+ if (!dentry)
+ pr_warn("WARNING: Failed to create recursed_functions\n");
+ return 0;
+diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
+index 63c2850420516..5a48dba912eae 100644
+--- a/kernel/trace/trace_stack.c
++++ b/kernel/trace/trace_stack.c
+@@ -559,14 +559,14 @@ static __init int stack_trace_init(void)
+ if (ret)
+ return 0;
+
+- trace_create_file("stack_max_size", 0644, NULL,
++ trace_create_file("stack_max_size", TRACE_MODE_WRITE, NULL,
+ &stack_trace_max_size, &stack_max_size_fops);
+
+- trace_create_file("stack_trace", 0444, NULL,
++ trace_create_file("stack_trace", TRACE_MODE_READ, NULL,
+ NULL, &stack_trace_fops);
+
+ #ifdef CONFIG_DYNAMIC_FTRACE
+- trace_create_file("stack_trace_filter", 0644, NULL,
++ trace_create_file("stack_trace_filter", TRACE_MODE_WRITE, NULL,
+ &trace_ops, &stack_trace_filter_fops);
+ #endif
+
+diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
+index 8d141c3825a94..bb247beec4470 100644
+--- a/kernel/trace/trace_stat.c
++++ b/kernel/trace/trace_stat.c
+@@ -297,9 +297,9 @@ static int init_stat_file(struct stat_session *session)
+ if (!stat_dir && (ret = tracing_stat_init()))
+ return ret;
+
+- session->file = tracefs_create_file(session->ts->name, 0644,
+- stat_dir,
+- session, &tracing_stat_fops);
++ session->file = tracefs_create_file(session->ts->name, TRACE_MODE_WRITE,
++ stat_dir, session,
++ &tracing_stat_fops);
+ if (!session->file)
+ return -ENOMEM;
+ return 0;
+diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
+index 8bfcd3b094226..f755bde42fd07 100644
+--- a/kernel/trace/trace_syscalls.c
++++ b/kernel/trace/trace_syscalls.c
+@@ -323,8 +323,7 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
+
+ trace_ctx = tracing_gen_ctx();
+
+- buffer = tr->array_buffer.buffer;
+- event = trace_buffer_lock_reserve(buffer,
++ event = trace_event_buffer_lock_reserve(&buffer, trace_file,
+ sys_data->enter_event->event.type, size, trace_ctx);
+ if (!event)
+ return;
+@@ -367,8 +366,7 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
+
+ trace_ctx = tracing_gen_ctx();
+
+- buffer = tr->array_buffer.buffer;
+- event = trace_buffer_lock_reserve(buffer,
++ event = trace_event_buffer_lock_reserve(&buffer, trace_file,
+ sys_data->exit_event->event.type, sizeof(*entry),
+ trace_ctx);
+ if (!event)
+diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
+index 225ce569bf8f8..78ec1c16ccf4b 100644
+--- a/kernel/trace/trace_uprobe.c
++++ b/kernel/trace/trace_uprobe.c
+@@ -1313,6 +1313,7 @@ static int uprobe_perf_open(struct trace_event_call *call,
+ return 0;
+
+ list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
++ tu = container_of(pos, struct trace_uprobe, tp);
+ err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
+ if (err) {
+ uprobe_perf_close(call, event);
+@@ -1618,6 +1619,11 @@ create_local_trace_uprobe(char *name, unsigned long offs,
+ tu->path = path;
+ tu->ref_ctr_offset = ref_ctr_offset;
+ tu->filename = kstrdup(name, GFP_KERNEL);
++ if (!tu->filename) {
++ ret = -ENOMEM;
++ goto error;
++ }
++
+ init_trace_event_call(tu);
+
+ ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
+@@ -1655,10 +1661,10 @@ static __init int init_uprobe_trace(void)
+ if (ret)
+ return 0;
+
+- trace_create_file("uprobe_events", 0644, NULL,
++ trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
+ NULL, &uprobe_events_ops);
+ /* Profile interface */
+- trace_create_file("uprobe_profile", 0444, NULL,
++ trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
+ NULL, &uprobe_profile_ops);
+ return 0;
+ }
+diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c
+index d6bddb157ef20..9628b55718468 100644
+--- a/kernel/trace/tracing_map.c
++++ b/kernel/trace/tracing_map.c
+@@ -15,6 +15,7 @@
+ #include <linux/jhash.h>
+ #include <linux/slab.h>
+ #include <linux/sort.h>
++#include <linux/kmemleak.h>
+
+ #include "tracing_map.h"
+ #include "trace.h"
+@@ -307,6 +308,7 @@ static void tracing_map_array_free(struct tracing_map_array *a)
+ for (i = 0; i < a->n_pages; i++) {
+ if (!a->pages[i])
+ break;
++ kmemleak_free(a->pages[i]);
+ free_page((unsigned long)a->pages[i]);
+ }
+
+@@ -342,6 +344,7 @@ static struct tracing_map_array *tracing_map_array_alloc(unsigned int n_elts,
+ a->pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
+ if (!a->pages[i])
+ goto free;
++ kmemleak_alloc(a->pages[i], PAGE_SIZE, 1, GFP_KERNEL);
+ }
+ out:
+ return a;
+@@ -834,29 +837,35 @@ int tracing_map_init(struct tracing_map *map)
+ return err;
+ }
+
+-static int cmp_entries_dup(const struct tracing_map_sort_entry **a,
+- const struct tracing_map_sort_entry **b)
++static int cmp_entries_dup(const void *A, const void *B)
+ {
++ const struct tracing_map_sort_entry *a, *b;
+ int ret = 0;
+
+- if (memcmp((*a)->key, (*b)->key, (*a)->elt->map->key_size))
++ a = *(const struct tracing_map_sort_entry **)A;
++ b = *(const struct tracing_map_sort_entry **)B;
++
++ if (memcmp(a->key, b->key, a->elt->map->key_size))
+ ret = 1;
+
+ return ret;
+ }
+
+-static int cmp_entries_sum(const struct tracing_map_sort_entry **a,
+- const struct tracing_map_sort_entry **b)
++static int cmp_entries_sum(const void *A, const void *B)
+ {
+ const struct tracing_map_elt *elt_a, *elt_b;
++ const struct tracing_map_sort_entry *a, *b;
+ struct tracing_map_sort_key *sort_key;
+ struct tracing_map_field *field;
+ tracing_map_cmp_fn_t cmp_fn;
+ void *val_a, *val_b;
+ int ret = 0;
+
+- elt_a = (*a)->elt;
+- elt_b = (*b)->elt;
++ a = *(const struct tracing_map_sort_entry **)A;
++ b = *(const struct tracing_map_sort_entry **)B;
++
++ elt_a = a->elt;
++ elt_b = b->elt;
+
+ sort_key = &elt_a->map->sort_key;
+
+@@ -873,18 +882,21 @@ static int cmp_entries_sum(const struct tracing_map_sort_entry **a,
+ return ret;
+ }
+
+-static int cmp_entries_key(const struct tracing_map_sort_entry **a,
+- const struct tracing_map_sort_entry **b)
++static int cmp_entries_key(const void *A, const void *B)
+ {
+ const struct tracing_map_elt *elt_a, *elt_b;
++ const struct tracing_map_sort_entry *a, *b;
+ struct tracing_map_sort_key *sort_key;
+ struct tracing_map_field *field;
+ tracing_map_cmp_fn_t cmp_fn;
+ void *val_a, *val_b;
+ int ret = 0;
+
+- elt_a = (*a)->elt;
+- elt_b = (*b)->elt;
++ a = *(const struct tracing_map_sort_entry **)A;
++ b = *(const struct tracing_map_sort_entry **)B;
++
++ elt_a = a->elt;
++ elt_b = b->elt;
+
+ sort_key = &elt_a->map->sort_key;
+
+@@ -989,10 +1001,8 @@ static void sort_secondary(struct tracing_map *map,
+ struct tracing_map_sort_key *primary_key,
+ struct tracing_map_sort_key *secondary_key)
+ {
+- int (*primary_fn)(const struct tracing_map_sort_entry **,
+- const struct tracing_map_sort_entry **);
+- int (*secondary_fn)(const struct tracing_map_sort_entry **,
+- const struct tracing_map_sort_entry **);
++ int (*primary_fn)(const void *, const void *);
++ int (*secondary_fn)(const void *, const void *);
+ unsigned i, start = 0, n_sub = 1;
+
+ if (is_key(map, primary_key->field_idx))
+@@ -1061,8 +1071,7 @@ int tracing_map_sort_entries(struct tracing_map *map,
+ unsigned int n_sort_keys,
+ struct tracing_map_sort_entry ***sort_entries)
+ {
+- int (*cmp_entries_fn)(const struct tracing_map_sort_entry **,
+- const struct tracing_map_sort_entry **);
++ int (*cmp_entries_fn)(const void *, const void *);
+ struct tracing_map_sort_entry *sort_entry, **entries;
+ int i, n_entries, ret;
+
+diff --git a/kernel/tsacct.c b/kernel/tsacct.c
+index 257ffb993ea23..fd2f7a052fdd9 100644
+--- a/kernel/tsacct.c
++++ b/kernel/tsacct.c
+@@ -38,11 +38,10 @@ void bacct_add_tsk(struct user_namespace *user_ns,
+ stats->ac_btime = clamp_t(time64_t, btime, 0, U32_MAX);
+ stats->ac_btime64 = btime;
+
+- if (thread_group_leader(tsk)) {
++ if (tsk->flags & PF_EXITING)
+ stats->ac_exitcode = tsk->exit_code;
+- if (tsk->flags & PF_FORKNOEXEC)
+- stats->ac_flag |= AFORK;
+- }
++ if (thread_group_leader(tsk) && (tsk->flags & PF_FORKNOEXEC))
++ stats->ac_flag |= AFORK;
+ if (tsk->flags & PF_SUPERPRIV)
+ stats->ac_flag |= ASU;
+ if (tsk->flags & PF_DUMPCORE)
+diff --git a/kernel/ucount.c b/kernel/ucount.c
+index eb03f3c68375d..a1d67261501a6 100644
+--- a/kernel/ucount.c
++++ b/kernel/ucount.c
+@@ -184,6 +184,7 @@ struct ucounts *alloc_ucounts(struct user_namespace *ns, kuid_t uid)
+ kfree(new);
+ } else {
+ hlist_add_head(&new->node, hashent);
++ get_user_ns(new->ns);
+ spin_unlock_irq(&ucounts_lock);
+ return new;
+ }
+@@ -204,6 +205,7 @@ void put_ucounts(struct ucounts *ucounts)
+ if (atomic_dec_and_lock_irqsave(&ucounts->count, &ucounts_lock, flags)) {
+ hlist_del_init(&ucounts->node);
+ spin_unlock_irqrestore(&ucounts_lock, flags);
++ put_user_ns(ucounts->ns);
+ kfree(ucounts);
+ }
+ }
+@@ -258,15 +260,16 @@ void dec_ucount(struct ucounts *ucounts, enum ucount_type type)
+ long inc_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v)
+ {
+ struct ucounts *iter;
++ long max = LONG_MAX;
+ long ret = 0;
+
+ for (iter = ucounts; iter; iter = iter->ns->ucounts) {
+- long max = READ_ONCE(iter->ns->ucount_max[type]);
+ long new = atomic_long_add_return(v, &iter->ucount[type]);
+ if (new < 0 || new > max)
+ ret = LONG_MAX;
+ else if (iter == ucounts)
+ ret = new;
++ max = READ_ONCE(iter->ns->ucount_max[type]);
+ }
+ return ret;
+ }
+@@ -306,15 +309,16 @@ long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum ucount_type type)
+ {
+ /* Caller must hold a reference to ucounts */
+ struct ucounts *iter;
++ long max = LONG_MAX;
+ long dec, ret = 0;
+
+ for (iter = ucounts; iter; iter = iter->ns->ucounts) {
+- long max = READ_ONCE(iter->ns->ucount_max[type]);
+ long new = atomic_long_add_return(1, &iter->ucount[type]);
+ if (new < 0 || new > max)
+ goto unwind;
+ if (iter == ucounts)
+ ret = new;
++ max = READ_ONCE(iter->ns->ucount_max[type]);
+ /*
+ * Grab an extra ucount reference for the caller when
+ * the rlimit count was previously 0.
+@@ -333,15 +337,17 @@ unwind:
+ return 0;
+ }
+
+-bool is_ucounts_overlimit(struct ucounts *ucounts, enum ucount_type type, unsigned long max)
++bool is_ucounts_overlimit(struct ucounts *ucounts, enum ucount_type type, unsigned long rlimit)
+ {
+ struct ucounts *iter;
+- if (get_ucounts_value(ucounts, type) > max)
+- return true;
++ long max = rlimit;
++ if (rlimit > LONG_MAX)
++ max = LONG_MAX;
+ for (iter = ucounts; iter; iter = iter->ns->ucounts) {
+- max = READ_ONCE(iter->ns->ucount_max[type]);
+- if (get_ucounts_value(iter, type) > max)
++ long val = get_ucounts_value(iter, type);
++ if (val < 0 || val > max)
+ return true;
++ max = READ_ONCE(iter->ns->ucount_max[type]);
+ }
+ return false;
+ }
+diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
+index 6b2e3ca7ee993..5481ba44a8d68 100644
+--- a/kernel/user_namespace.c
++++ b/kernel/user_namespace.c
+@@ -58,6 +58,18 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
+ cred->user_ns = user_ns;
+ }
+
++static unsigned long enforced_nproc_rlimit(void)
++{
++ unsigned long limit = RLIM_INFINITY;
++
++ /* Is RLIMIT_NPROC currently enforced? */
++ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID) ||
++ (current_user_ns() != &init_user_ns))
++ limit = rlimit(RLIMIT_NPROC);
++
++ return limit;
++}
++
+ /*
+ * Create a new user namespace, deriving the creator from the user in the
+ * passed credentials, and replacing that user with the new root user for the
+@@ -122,7 +134,7 @@ int create_user_ns(struct cred *new)
+ for (i = 0; i < MAX_PER_NAMESPACE_UCOUNTS; i++) {
+ ns->ucount_max[i] = INT_MAX;
+ }
+- set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC));
++ set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_NPROC, enforced_nproc_rlimit());
+ set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_MSGQUEUE, rlimit(RLIMIT_MSGQUEUE));
+ set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_SIGPENDING, rlimit(RLIMIT_SIGPENDING));
+ set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_MEMLOCK, rlimit(RLIMIT_MEMLOCK));
+diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c
+index 9c9eb20dd2c50..b1ae7c9c3b473 100644
+--- a/kernel/watch_queue.c
++++ b/kernel/watch_queue.c
+@@ -54,6 +54,7 @@ static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe,
+ bit += page->index;
+
+ set_bit(bit, wqueue->notes_bitmap);
++ generic_pipe_buf_release(pipe, buf);
+ }
+
+ // No try_steal function => no stealing
+@@ -112,7 +113,7 @@ static bool post_one_notification(struct watch_queue *wqueue,
+ buf->offset = offset;
+ buf->len = len;
+ buf->flags = PIPE_BUF_FLAG_WHOLE;
+- pipe->head = head + 1;
++ smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */
+
+ if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {
+ spin_unlock_irq(&pipe->rd_wait.lock);
+@@ -243,7 +244,8 @@ long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes)
+ goto error;
+ }
+
+- ret = pipe_resize_ring(pipe, nr_notes);
++ nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;
++ ret = pipe_resize_ring(pipe, roundup_pow_of_two(nr_notes));
+ if (ret < 0)
+ goto error;
+
+@@ -268,11 +270,11 @@ long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes)
+ wqueue->notes = pages;
+ wqueue->notes_bitmap = bitmap;
+ wqueue->nr_pages = nr_pages;
+- wqueue->nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;
++ wqueue->nr_notes = nr_notes;
+ return 0;
+
+ error_p:
+- for (i = 0; i < nr_pages; i++)
++ while (--i >= 0)
+ __free_page(pages[i]);
+ kfree(pages);
+ error:
+@@ -320,7 +322,7 @@ long watch_queue_set_filter(struct pipe_inode_info *pipe,
+ tf[i].info_mask & WATCH_INFO_LENGTH)
+ goto err_filter;
+ /* Ignore any unknown types */
+- if (tf[i].type >= sizeof(wfilter->type_filter) * 8)
++ if (tf[i].type >= WATCH_TYPE__NR)
+ continue;
+ nr_filter++;
+ }
+@@ -336,7 +338,7 @@ long watch_queue_set_filter(struct pipe_inode_info *pipe,
+
+ q = wfilter->filters;
+ for (i = 0; i < filter.nr_filters; i++) {
+- if (tf[i].type >= sizeof(wfilter->type_filter) * BITS_PER_LONG)
++ if (tf[i].type >= WATCH_TYPE__NR)
+ continue;
+
+ q->type = tf[i].type;
+@@ -371,6 +373,8 @@ static void __put_watch_queue(struct kref *kref)
+
+ for (i = 0; i < wqueue->nr_pages; i++)
+ __free_page(wqueue->notes[i]);
++ kfree(wqueue->notes);
++ bitmap_free(wqueue->notes_bitmap);
+
+ wfilter = rcu_access_pointer(wqueue->filter);
+ if (wfilter)
+@@ -395,6 +399,7 @@ static void free_watch(struct rcu_head *rcu)
+ put_watch_queue(rcu_access_pointer(watch->queue));
+ atomic_dec(&watch->cred->user->nr_watches);
+ put_cred(watch->cred);
++ kfree(watch);
+ }
+
+ static void __put_watch(struct kref *kref)
+@@ -566,7 +571,7 @@ void watch_queue_clear(struct watch_queue *wqueue)
+ rcu_read_lock();
+ spin_lock_bh(&wqueue->lock);
+
+- /* Prevent new additions and prevent notifications from happening */
++ /* Prevent new notifications from being stored. */
+ wqueue->defunct = true;
+
+ while (!hlist_empty(&wqueue->watches)) {
+diff --git a/kernel/workqueue.c b/kernel/workqueue.c
+index 1b3eb1e9531f4..3f4d276685768 100644
+--- a/kernel/workqueue.c
++++ b/kernel/workqueue.c
+@@ -867,8 +867,17 @@ void wq_worker_running(struct task_struct *task)
+
+ if (!worker->sleeping)
+ return;
++
++ /*
++ * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
++ * and the nr_running increment below, we may ruin the nr_running reset
++ * and leave with an unexpected pool->nr_running == 1 on the newly unbound
++ * pool. Protect against such race.
++ */
++ preempt_disable();
+ if (!(worker->flags & WORKER_NOT_RUNNING))
+ atomic_inc(&worker->pool->nr_running);
++ preempt_enable();
+ worker->sleeping = 0;
+ }
+
+@@ -5384,9 +5393,6 @@ int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
+ int ret = -EINVAL;
+ cpumask_var_t saved_cpumask;
+
+- if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL))
+- return -ENOMEM;
+-
+ /*
+ * Not excluding isolated cpus on purpose.
+ * If the user wishes to include them, we allow that.
+@@ -5394,6 +5400,15 @@ int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
+ cpumask_and(cpumask, cpumask, cpu_possible_mask);
+ if (!cpumask_empty(cpumask)) {
+ apply_wqattrs_lock();
++ if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
++ ret = 0;
++ goto out_unlock;
++ }
++
++ if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) {
++ ret = -ENOMEM;
++ goto out_unlock;
++ }
+
+ /* save the old wq_unbound_cpumask. */
+ cpumask_copy(saved_cpumask, wq_unbound_cpumask);
+@@ -5406,10 +5421,11 @@ int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
+ if (ret < 0)
+ cpumask_copy(wq_unbound_cpumask, saved_cpumask);
+
++ free_cpumask_var(saved_cpumask);
++out_unlock:
+ apply_wqattrs_unlock();
+ }
+
+- free_cpumask_var(saved_cpumask);
+ return ret;
+ }
+
+diff --git a/lib/Kconfig b/lib/Kconfig
+index 5e7165e6a346c..fa4b10322efcd 100644
+--- a/lib/Kconfig
++++ b/lib/Kconfig
+@@ -45,7 +45,6 @@ config BITREVERSE
+ config HAVE_ARCH_BITREVERSE
+ bool
+ default n
+- depends on BITREVERSE
+ help
+ This option enables the use of hardware bit-reversal instructions on
+ architectures which support such operations.
+diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
+index 2a9b6dcdac4ff..7fd3fa05379e2 100644
+--- a/lib/Kconfig.debug
++++ b/lib/Kconfig.debug
+@@ -414,7 +414,8 @@ config SECTION_MISMATCH_WARN_ONLY
+ If unsure, say Y.
+
+ config DEBUG_FORCE_FUNCTION_ALIGN_64B
+- bool "Force all function address 64B aligned" if EXPERT
++ bool "Force all function address 64B aligned"
++ depends on EXPERT && (X86_64 || ARM64 || PPC32 || PPC64 || ARC)
+ help
+ There are cases that a commit from one domain changes the function
+ address alignment of other domains, and cause magic performance
+@@ -1558,8 +1559,7 @@ config WARN_ALL_UNSEEDED_RANDOM
+ so architecture maintainers really need to do what they can
+ to get the CRNG seeded sooner after the system is booted.
+ However, since users cannot do anything actionable to
+- address this, by default the kernel will issue only a single
+- warning for the first use of unseeded randomness.
++ address this, by default this option is disabled.
+
+ Say Y here if you want to receive warnings for all uses of
+ unseeded randomness. This will be of use primarily for
+diff --git a/lib/Kconfig.kfence b/lib/Kconfig.kfence
+index e641add339475..912f252a41fc6 100644
+--- a/lib/Kconfig.kfence
++++ b/lib/Kconfig.kfence
+@@ -25,17 +25,6 @@ menuconfig KFENCE
+
+ if KFENCE
+
+-config KFENCE_STATIC_KEYS
+- bool "Use static keys to set up allocations"
+- default y
+- depends on JUMP_LABEL # To ensure performance, require jump labels
+- help
+- Use static keys (static branches) to set up KFENCE allocations. Using
+- static keys is normally recommended, because it avoids a dynamic
+- branch in the allocator's fast path. However, with very low sample
+- intervals, or on systems that do not support jump labels, a dynamic
+- branch may still be an acceptable performance trade-off.
+-
+ config KFENCE_SAMPLE_INTERVAL
+ int "Default sample interval in milliseconds"
+ default 100
+@@ -56,6 +45,21 @@ config KFENCE_NUM_OBJECTS
+ pages are required; with one containing the object and two adjacent
+ ones used as guard pages.
+
++config KFENCE_STATIC_KEYS
++ bool "Use static keys to set up allocations" if EXPERT
++ depends on JUMP_LABEL
++ help
++ Use static keys (static branches) to set up KFENCE allocations. This
++ option is only recommended when using very large sample intervals, or
++ performance has carefully been evaluated with this option.
++
++ Using static keys comes with trade-offs that need to be carefully
++ evaluated given target workloads and system architectures. Notably,
++ enabling and disabling static keys invoke IPI broadcasts, the latency
++ and impact of which is much harder to predict than a dynamic branch.
++
++ Say N if you are unsure.
++
+ config KFENCE_STRESS_TEST_FAULTS
+ int "Stress testing of fault handling and error reporting" if EXPERT
+ default 0
+diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan
+index e5372a13511df..236c5cefc4cc5 100644
+--- a/lib/Kconfig.ubsan
++++ b/lib/Kconfig.ubsan
+@@ -112,19 +112,6 @@ config UBSAN_UNREACHABLE
+ This option enables -fsanitize=unreachable which checks for control
+ flow reaching an expected-to-be-unreachable position.
+
+-config UBSAN_OBJECT_SIZE
+- bool "Perform checking for accesses beyond the end of objects"
+- default UBSAN
+- # gcc hugely expands stack usage with -fsanitize=object-size
+- # https://lore.kernel.org/lkml/CAHk-=wjPasyJrDuwDnpHJS2TuQfExwe=px-SzLeN8GFMAQJPmQ@mail.gmail.com/
+- depends on !CC_IS_GCC
+- depends on $(cc-option,-fsanitize=object-size)
+- help
+- This option enables -fsanitize=object-size which checks for accesses
+- beyond the end of objects where the optimizer can determine both the
+- object being operated on and its size, usually seen with bad downcasts,
+- or access to struct members from NULL pointers.
+-
+ config UBSAN_BOOL
+ bool "Perform checking for non-boolean values used as boolean"
+ default UBSAN
+diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
+index 545ccbddf6a1d..8620f38e117c0 100644
+--- a/lib/crypto/Kconfig
++++ b/lib/crypto/Kconfig
+@@ -1,7 +1,5 @@
+ # SPDX-License-Identifier: GPL-2.0
+
+-comment "Crypto library routines"
+-
+ config CRYPTO_LIB_AES
+ tristate
+
+@@ -9,14 +7,14 @@ config CRYPTO_LIB_ARC4
+ tristate
+
+ config CRYPTO_ARCH_HAVE_LIB_BLAKE2S
+- tristate
++ bool
+ help
+ Declares whether the architecture provides an arch-specific
+ accelerated implementation of the Blake2s library interface,
+ either builtin or as a module.
+
+ config CRYPTO_LIB_BLAKE2S_GENERIC
+- tristate
++ def_bool !CRYPTO_ARCH_HAVE_LIB_BLAKE2S
+ help
+ This symbol can be depended upon by arch implementations of the
+ Blake2s library interface that require the generic code as a
+@@ -24,15 +22,6 @@ config CRYPTO_LIB_BLAKE2S_GENERIC
+ implementation is enabled, this implementation serves the users
+ of CRYPTO_LIB_BLAKE2S.
+
+-config CRYPTO_LIB_BLAKE2S
+- tristate "BLAKE2s hash function library"
+- depends on CRYPTO_ARCH_HAVE_LIB_BLAKE2S || !CRYPTO_ARCH_HAVE_LIB_BLAKE2S
+- select CRYPTO_LIB_BLAKE2S_GENERIC if CRYPTO_ARCH_HAVE_LIB_BLAKE2S=n
+- help
+- Enable the Blake2s library interface. This interface may be fulfilled
+- by either the generic implementation or an arch-specific one, if one
+- is available and enabled.
+-
+ config CRYPTO_ARCH_HAVE_LIB_CHACHA
+ tristate
+ help
+@@ -51,7 +40,7 @@ config CRYPTO_LIB_CHACHA_GENERIC
+ of CRYPTO_LIB_CHACHA.
+
+ config CRYPTO_LIB_CHACHA
+- tristate "ChaCha library interface"
++ tristate
+ depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA
+ select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n
+ help
+@@ -76,7 +65,7 @@ config CRYPTO_LIB_CURVE25519_GENERIC
+ of CRYPTO_LIB_CURVE25519.
+
+ config CRYPTO_LIB_CURVE25519
+- tristate "Curve25519 scalar multiplication library"
++ tristate
+ depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519
+ select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n
+ help
+@@ -111,7 +100,7 @@ config CRYPTO_LIB_POLY1305_GENERIC
+ of CRYPTO_LIB_POLY1305.
+
+ config CRYPTO_LIB_POLY1305
+- tristate "Poly1305 library interface"
++ tristate
+ depends on CRYPTO_ARCH_HAVE_LIB_POLY1305 || !CRYPTO_ARCH_HAVE_LIB_POLY1305
+ select CRYPTO_LIB_POLY1305_GENERIC if CRYPTO_ARCH_HAVE_LIB_POLY1305=n
+ help
+@@ -120,7 +109,7 @@ config CRYPTO_LIB_POLY1305
+ is available and enabled.
+
+ config CRYPTO_LIB_CHACHA20POLY1305
+- tristate "ChaCha20-Poly1305 AEAD support (8-byte nonce library version)"
++ tristate
+ depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA
+ depends on CRYPTO_ARCH_HAVE_LIB_POLY1305 || !CRYPTO_ARCH_HAVE_LIB_POLY1305
+ select CRYPTO_LIB_CHACHA
+diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
+index 73205ed269bad..ed43a41f2dcc8 100644
+--- a/lib/crypto/Makefile
++++ b/lib/crypto/Makefile
+@@ -10,11 +10,10 @@ libaes-y := aes.o
+ obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o
+ libarc4-y := arc4.o
+
+-obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += libblake2s-generic.o
+-libblake2s-generic-y += blake2s-generic.o
+-
+-obj-$(CONFIG_CRYPTO_LIB_BLAKE2S) += libblake2s.o
+-libblake2s-y += blake2s.o
++# blake2s is used by the /dev/random driver which is always builtin
++obj-y += libblake2s.o
++libblake2s-y := blake2s.o
++libblake2s-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += blake2s-generic.o
+
+ obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o
+ libchacha20poly1305-y += chacha20poly1305.o
+diff --git a/lib/crypto/blake2s-generic.c b/lib/crypto/blake2s-generic.c
+index 04ff8df245136..75ccb3e633e65 100644
+--- a/lib/crypto/blake2s-generic.c
++++ b/lib/crypto/blake2s-generic.c
+@@ -37,7 +37,11 @@ static inline void blake2s_increment_counter(struct blake2s_state *state,
+ state->t[1] += (state->t[0] < inc);
+ }
+
+-void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
++void blake2s_compress(struct blake2s_state *state, const u8 *block,
++ size_t nblocks, const u32 inc)
++ __weak __alias(blake2s_compress_generic);
++
++void blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
+ size_t nblocks, const u32 inc)
+ {
+ u32 m[16];
+diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c
+index 5d9ea53be9736..409e4b7287704 100644
+--- a/lib/crypto/blake2s-selftest.c
++++ b/lib/crypto/blake2s-selftest.c
+@@ -15,7 +15,6 @@
+ * #include <stdio.h>
+ *
+ * #include <openssl/evp.h>
+- * #include <openssl/hmac.h>
+ *
+ * #define BLAKE2S_TESTVEC_COUNT 256
+ *
+@@ -58,16 +57,6 @@
+ * }
+ * printf("};\n\n");
+ *
+- * printf("static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n");
+- *
+- * HMAC(EVP_blake2s256(), key, sizeof(key), buf, sizeof(buf), hash, NULL);
+- * print_vec(hash, BLAKE2S_OUTBYTES);
+- *
+- * HMAC(EVP_blake2s256(), buf, sizeof(buf), key, sizeof(key), hash, NULL);
+- * print_vec(hash, BLAKE2S_OUTBYTES);
+- *
+- * printf("};\n");
+- *
+ * return 0;
+ *}
+ */
+@@ -554,15 +543,6 @@ static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
+ 0xd6, 0x98, 0x6b, 0x07, 0x10, 0x65, 0x52, 0x65, },
+ };
+
+-static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
+- { 0xce, 0xe1, 0x57, 0x69, 0x82, 0xdc, 0xbf, 0x43, 0xad, 0x56, 0x4c, 0x70,
+- 0xed, 0x68, 0x16, 0x96, 0xcf, 0xa4, 0x73, 0xe8, 0xe8, 0xfc, 0x32, 0x79,
+- 0x08, 0x0a, 0x75, 0x82, 0xda, 0x3f, 0x05, 0x11, },
+- { 0x77, 0x2f, 0x0c, 0x71, 0x41, 0xf4, 0x4b, 0x2b, 0xb3, 0xc6, 0xb6, 0xf9,
+- 0x60, 0xde, 0xe4, 0x52, 0x38, 0x66, 0xe8, 0xbf, 0x9b, 0x96, 0xc4, 0x9f,
+- 0x60, 0xd9, 0x24, 0x37, 0x99, 0xd6, 0xec, 0x31, },
+-};
+-
+ bool __init blake2s_selftest(void)
+ {
+ u8 key[BLAKE2S_KEY_SIZE];
+@@ -607,16 +587,5 @@ bool __init blake2s_selftest(void)
+ }
+ }
+
+- if (success) {
+- blake2s256_hmac(hash, buf, key, sizeof(buf), sizeof(key));
+- success &= !memcmp(hash, blake2s_hmac_testvecs[0], BLAKE2S_HASH_SIZE);
+-
+- blake2s256_hmac(hash, key, buf, sizeof(key), sizeof(buf));
+- success &= !memcmp(hash, blake2s_hmac_testvecs[1], BLAKE2S_HASH_SIZE);
+-
+- if (!success)
+- pr_err("blake2s256_hmac self-test: FAIL\n");
+- }
+-
+ return success;
+ }
+diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c
+index 4055aa593ec49..c71c09621c09c 100644
+--- a/lib/crypto/blake2s.c
++++ b/lib/crypto/blake2s.c
+@@ -16,63 +16,20 @@
+ #include <linux/init.h>
+ #include <linux/bug.h>
+
+-#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S)
+-# define blake2s_compress blake2s_compress_arch
+-#else
+-# define blake2s_compress blake2s_compress_generic
+-#endif
+-
+ void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
+ {
+- __blake2s_update(state, in, inlen, blake2s_compress);
++ __blake2s_update(state, in, inlen, false);
+ }
+ EXPORT_SYMBOL(blake2s_update);
+
+ void blake2s_final(struct blake2s_state *state, u8 *out)
+ {
+ WARN_ON(IS_ENABLED(DEBUG) && !out);
+- __blake2s_final(state, out, blake2s_compress);
++ __blake2s_final(state, out, false);
+ memzero_explicit(state, sizeof(*state));
+ }
+ EXPORT_SYMBOL(blake2s_final);
+
+-void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen,
+- const size_t keylen)
+-{
+- struct blake2s_state state;
+- u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 };
+- u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32));
+- int i;
+-
+- if (keylen > BLAKE2S_BLOCK_SIZE) {
+- blake2s_init(&state, BLAKE2S_HASH_SIZE);
+- blake2s_update(&state, key, keylen);
+- blake2s_final(&state, x_key);
+- } else
+- memcpy(x_key, key, keylen);
+-
+- for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
+- x_key[i] ^= 0x36;
+-
+- blake2s_init(&state, BLAKE2S_HASH_SIZE);
+- blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
+- blake2s_update(&state, in, inlen);
+- blake2s_final(&state, i_hash);
+-
+- for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
+- x_key[i] ^= 0x5c ^ 0x36;
+-
+- blake2s_init(&state, BLAKE2S_HASH_SIZE);
+- blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
+- blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE);
+- blake2s_final(&state, i_hash);
+-
+- memcpy(out, i_hash, BLAKE2S_HASH_SIZE);
+- memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE);
+- memzero_explicit(i_hash, BLAKE2S_HASH_SIZE);
+-}
+-EXPORT_SYMBOL(blake2s256_hmac);
+-
+ static int __init blake2s_mod_init(void)
+ {
+ if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
+diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c
+index 633b59fed9db8..284e62576d0c6 100644
+--- a/lib/crypto/sm4.c
++++ b/lib/crypto/sm4.c
+@@ -15,7 +15,7 @@ static const u32 fk[4] = {
+ 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc
+ };
+
+-static const u32 __cacheline_aligned ck[32] = {
++static const u32 ____cacheline_aligned ck[32] = {
+ 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
+ 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
+ 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
+@@ -26,7 +26,7 @@ static const u32 __cacheline_aligned ck[32] = {
+ 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
+ };
+
+-static const u8 __cacheline_aligned sbox[256] = {
++static const u8 ____cacheline_aligned sbox[256] = {
+ 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7,
+ 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05,
+ 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3,
+diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
+index a2f38e23004aa..f7a3dc13316a3 100644
+--- a/lib/decompress_unxz.c
++++ b/lib/decompress_unxz.c
+@@ -167,7 +167,7 @@
+ * memeq and memzero are not used much and any remotely sane implementation
+ * is fast enough. memcpy/memmove speed matters in multi-call mode, but
+ * the kernel image is decompressed in single-call mode, in which only
+- * memcpy speed can matter and only if there is a lot of uncompressible data
++ * memmove speed can matter and only if there is a lot of uncompressible data
+ * (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the
+ * functions below should just be kept small; it's probably not worth
+ * optimizing for speed.
+diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c
+index 06811d866775c..53f6b9c6e9366 100644
+--- a/lib/dim/net_dim.c
++++ b/lib/dim/net_dim.c
+@@ -12,41 +12,41 @@
+ * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES
+ */
+ #define NET_DIM_PARAMS_NUM_PROFILES 5
+-#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
+-#define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
++#define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256
++#define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128
+ #define NET_DIM_DEF_PROFILE_CQE 1
+ #define NET_DIM_DEF_PROFILE_EQE 1
+
+ #define NET_DIM_RX_EQE_PROFILES { \
+- {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
++ {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \
+ }
+
+ #define NET_DIM_RX_CQE_PROFILES { \
+- {2, 256}, \
+- {8, 128}, \
+- {16, 64}, \
+- {32, 64}, \
+- {64, 64} \
++ {.usec = 2, .pkts = 256,}, \
++ {.usec = 8, .pkts = 128,}, \
++ {.usec = 16, .pkts = 64,}, \
++ {.usec = 32, .pkts = 64,}, \
++ {.usec = 64, .pkts = 64,} \
+ }
+
+ #define NET_DIM_TX_EQE_PROFILES { \
+- {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
+- {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \
++ {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
++ {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \
+ }
+
+ #define NET_DIM_TX_CQE_PROFILES { \
+- {5, 128}, \
+- {8, 64}, \
+- {16, 32}, \
+- {32, 32}, \
+- {64, 32} \
++ {.usec = 5, .pkts = 128,}, \
++ {.usec = 8, .pkts = 64,}, \
++ {.usec = 16, .pkts = 32,}, \
++ {.usec = 32, .pkts = 32,}, \
++ {.usec = 64, .pkts = 32,} \
+ }
+
+ static const struct dim_cq_moder
+diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
+index cb5abb42c16a2..84c16309cc637 100644
+--- a/lib/dynamic_debug.c
++++ b/lib/dynamic_debug.c
+@@ -761,6 +761,18 @@ static __init int ddebug_setup_query(char *str)
+
+ __setup("ddebug_query=", ddebug_setup_query);
+
++/*
++ * Install a noop handler to make dyndbg look like a normal kernel cli param.
++ * This avoids warnings about dyndbg being an unknown cli param when supplied
++ * by a user.
++ */
++static __init int dyndbg_setup(char *str)
++{
++ return 1;
++}
++
++__setup("dyndbg=", dyndbg_setup);
++
+ /*
+ * File_ops->write method for <debugfs>/dynamic_debug/control. Gathers the
+ * command text from userspace, parses and executes it.
+diff --git a/lib/hexdump.c b/lib/hexdump.c
+index 9301578f98e8c..06833d404398d 100644
+--- a/lib/hexdump.c
++++ b/lib/hexdump.c
+@@ -22,15 +22,33 @@ EXPORT_SYMBOL(hex_asc_upper);
+ *
+ * hex_to_bin() converts one hex digit to its actual value or -1 in case of bad
+ * input.
++ *
++ * This function is used to load cryptographic keys, so it is coded in such a
++ * way that there are no conditions or memory accesses that depend on data.
++ *
++ * Explanation of the logic:
++ * (ch - '9' - 1) is negative if ch <= '9'
++ * ('0' - 1 - ch) is negative if ch >= '0'
++ * we "and" these two values, so the result is negative if ch is in the range
++ * '0' ... '9'
++ * we are only interested in the sign, so we do a shift ">> 8"; note that right
++ * shift of a negative value is implementation-defined, so we cast the
++ * value to (unsigned) before the shift --- we have 0xffffff if ch is in
++ * the range '0' ... '9', 0 otherwise
++ * we "and" this value with (ch - '0' + 1) --- we have a value 1 ... 10 if ch is
++ * in the range '0' ... '9', 0 otherwise
++ * we add this value to -1 --- we have a value 0 ... 9 if ch is in the range '0'
++ * ... '9', -1 otherwise
++ * the next line is similar to the previous one, but we need to decode both
++ * uppercase and lowercase letters, so we use (ch & 0xdf), which converts
++ * lowercase to uppercase
+ */
+-int hex_to_bin(char ch)
++int hex_to_bin(unsigned char ch)
+ {
+- if ((ch >= '0') && (ch <= '9'))
+- return ch - '0';
+- ch = tolower(ch);
+- if ((ch >= 'a') && (ch <= 'f'))
+- return ch - 'a' + 10;
+- return -1;
++ unsigned char cu = ch & 0xdf;
++ return -1 +
++ ((ch - '0' + 1) & (unsigned)((ch - '9' - 1) & ('0' - 1 - ch)) >> 8) +
++ ((cu - 'A' + 11) & (unsigned)((cu - 'F' - 1) & ('A' - 1 - cu)) >> 8);
+ }
+ EXPORT_SYMBOL(hex_to_bin);
+
+@@ -45,10 +63,13 @@ EXPORT_SYMBOL(hex_to_bin);
+ int hex2bin(u8 *dst, const char *src, size_t count)
+ {
+ while (count--) {
+- int hi = hex_to_bin(*src++);
+- int lo = hex_to_bin(*src++);
++ int hi, lo;
+
+- if ((hi < 0) || (lo < 0))
++ hi = hex_to_bin(*src++);
++ if (unlikely(hi < 0))
++ return -EINVAL;
++ lo = hex_to_bin(*src++);
++ if (unlikely(lo < 0))
+ return -EINVAL;
+
+ *dst++ = (hi << 4) | lo;
+diff --git a/lib/iov_iter.c b/lib/iov_iter.c
+index 755c10c5138cd..6d146f77601d7 100644
+--- a/lib/iov_iter.c
++++ b/lib/iov_iter.c
+@@ -191,7 +191,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
+ buf = iov->iov_base + skip;
+ copy = min(bytes, iov->iov_len - skip);
+
+- if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
++ if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_writeable(buf, copy)) {
+ kaddr = kmap_atomic(page);
+ from = kaddr + offset;
+
+@@ -275,7 +275,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
+ buf = iov->iov_base + skip;
+ copy = min(bytes, iov->iov_len - skip);
+
+- if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
++ if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_readable(buf, copy)) {
+ kaddr = kmap_atomic(page);
+ to = kaddr + offset;
+
+@@ -416,6 +416,7 @@ static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t by
+ return 0;
+
+ buf->ops = &page_cache_pipe_buf_ops;
++ buf->flags = 0;
+ get_page(page);
+ buf->page = page;
+ buf->offset = offset;
+@@ -430,35 +431,81 @@ out:
+ }
+
+ /*
++ * fault_in_iov_iter_readable - fault in iov iterator for reading
++ * @i: iterator
++ * @size: maximum length
++ *
+ * Fault in one or more iovecs of the given iov_iter, to a maximum length of
+- * bytes. For each iovec, fault in each page that constitutes the iovec.
++ * @size. For each iovec, fault in each page that constitutes the iovec.
++ *
++ * Returns the number of bytes not faulted in (like copy_to_user() and
++ * copy_from_user()).
+ *
+- * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
+- * because it is an invalid address).
++ * Always returns 0 for non-userspace iterators.
+ */
+-int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes)
++size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
+ {
+ if (iter_is_iovec(i)) {
++ size_t count = min(size, iov_iter_count(i));
+ const struct iovec *p;
+ size_t skip;
+
+- if (bytes > i->count)
+- bytes = i->count;
+- for (p = i->iov, skip = i->iov_offset; bytes; p++, skip = 0) {
+- size_t len = min(bytes, p->iov_len - skip);
+- int err;
++ size -= count;
++ for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
++ size_t len = min(count, p->iov_len - skip);
++ size_t ret;
+
+ if (unlikely(!len))
+ continue;
+- err = fault_in_pages_readable(p->iov_base + skip, len);
+- if (unlikely(err))
+- return err;
+- bytes -= len;
++ ret = fault_in_readable(p->iov_base + skip, len);
++ count -= len - ret;
++ if (ret)
++ break;
+ }
++ return count + size;
+ }
+ return 0;
+ }
+-EXPORT_SYMBOL(iov_iter_fault_in_readable);
++EXPORT_SYMBOL(fault_in_iov_iter_readable);
++
++/*
++ * fault_in_iov_iter_writeable - fault in iov iterator for writing
++ * @i: iterator
++ * @size: maximum length
++ *
++ * Faults in the iterator using get_user_pages(), i.e., without triggering
++ * hardware page faults. This is primarily useful when we already know that
++ * some or all of the pages in @i aren't in memory.
++ *
++ * Returns the number of bytes not faulted in, like copy_to_user() and
++ * copy_from_user().
++ *
++ * Always returns 0 for non-user-space iterators.
++ */
++size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
++{
++ if (iter_is_iovec(i)) {
++ size_t count = min(size, iov_iter_count(i));
++ const struct iovec *p;
++ size_t skip;
++
++ size -= count;
++ for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
++ size_t len = min(count, p->iov_len - skip);
++ size_t ret;
++
++ if (unlikely(!len))
++ continue;
++ ret = fault_in_safe_writeable(p->iov_base + skip, len);
++ count -= len - ret;
++ if (ret)
++ break;
++ }
++ return count + size;
++ }
++ return 0;
++}
++EXPORT_SYMBOL(fault_in_iov_iter_writeable);
+
+ void iov_iter_init(struct iov_iter *i, unsigned int direction,
+ const struct iovec *iov, unsigned long nr_segs,
+@@ -467,6 +514,7 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction,
+ WARN_ON(direction & ~(READ | WRITE));
+ *i = (struct iov_iter) {
+ .iter_type = ITER_IOVEC,
++ .nofault = false,
+ .data_source = direction,
+ .iov = iov,
+ .nr_segs = nr_segs,
+@@ -532,6 +580,7 @@ static size_t push_pipe(struct iov_iter *i, size_t size,
+ break;
+
+ buf->ops = &default_pipe_buf_ops;
++ buf->flags = 0;
+ buf->page = page;
+ buf->offset = 0;
+ buf->len = min_t(ssize_t, left, PAGE_SIZE);
+@@ -1481,14 +1530,18 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
+ return 0;
+
+ if (likely(iter_is_iovec(i))) {
++ unsigned int gup_flags = 0;
+ unsigned long addr;
+
++ if (iov_iter_rw(i) != WRITE)
++ gup_flags |= FOLL_WRITE;
++ if (i->nofault)
++ gup_flags |= FOLL_NOFAULT;
++
+ addr = first_iovec_segment(i, &len, start, maxsize, maxpages);
+ n = DIV_ROUND_UP(len, PAGE_SIZE);
+- res = get_user_pages_fast(addr, n,
+- iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0,
+- pages);
+- if (unlikely(res < 0))
++ res = get_user_pages_fast(addr, n, gup_flags, pages);
++ if (unlikely(res <= 0))
+ return res;
+ return (res == n ? len : res * PAGE_SIZE) - *start;
+ }
+@@ -1603,17 +1656,23 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
+ return 0;
+
+ if (likely(iter_is_iovec(i))) {
++ unsigned int gup_flags = 0;
+ unsigned long addr;
+
++ if (iov_iter_rw(i) != WRITE)
++ gup_flags |= FOLL_WRITE;
++ if (i->nofault)
++ gup_flags |= FOLL_NOFAULT;
++
+ addr = first_iovec_segment(i, &len, start, maxsize, ~0U);
+ n = DIV_ROUND_UP(len, PAGE_SIZE);
+ p = get_pages_array(n);
+ if (!p)
+ return -ENOMEM;
+- res = get_user_pages_fast(addr, n,
+- iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p);
+- if (unlikely(res < 0)) {
++ res = get_user_pages_fast(addr, n, gup_flags, p);
++ if (unlikely(res <= 0)) {
+ kvfree(p);
++ *pages = NULL;
+ return res;
+ }
+ *pages = p;
+diff --git a/lib/kunit/test.c b/lib/kunit/test.c
+index f246b847024e3..9aef816e573c1 100644
+--- a/lib/kunit/test.c
++++ b/lib/kunit/test.c
+@@ -504,16 +504,18 @@ int kunit_run_tests(struct kunit_suite *suite)
+ struct kunit_result_stats param_stats = { 0 };
+ test_case->status = KUNIT_SKIPPED;
+
+- if (test_case->generate_params) {
++ if (!test_case->generate_params) {
++ /* Non-parameterised test. */
++ kunit_run_case_catch_errors(suite, test_case, &test);
++ kunit_update_stats(&param_stats, test.status);
++ } else {
+ /* Get initial param. */
+ param_desc[0] = '\0';
+ test.param_value = test_case->generate_params(NULL, param_desc);
+- }
+
+- do {
+- kunit_run_case_catch_errors(suite, test_case, &test);
++ while (test.param_value) {
++ kunit_run_case_catch_errors(suite, test_case, &test);
+
+- if (test_case->generate_params) {
+ if (param_desc[0] == '\0') {
+ snprintf(param_desc, sizeof(param_desc),
+ "param-%d", test.param_index);
+@@ -530,11 +532,11 @@ int kunit_run_tests(struct kunit_suite *suite)
+ param_desc[0] = '\0';
+ test.param_value = test_case->generate_params(test.param_value, param_desc);
+ test.param_index++;
+- }
+
+- kunit_update_stats(&param_stats, test.status);
++ kunit_update_stats(&param_stats, test.status);
++ }
++ }
+
+- } while (test.param_value);
+
+ kunit_print_test_stats(&test, param_stats);
+
+diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c
+index 0dd434e40487c..71e5c58530996 100644
+--- a/lib/kunit/try-catch.c
++++ b/lib/kunit/try-catch.c
+@@ -52,7 +52,7 @@ static unsigned long kunit_test_timeout(void)
+ * If tests timeout due to exceeding sysctl_hung_task_timeout_secs,
+ * the task will be killed and an oops generated.
+ */
+- return 300 * MSEC_PER_SEC; /* 5 min */
++ return 300 * msecs_to_jiffies(MSEC_PER_SEC); /* 5 min */
+ }
+
+ void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context)
+diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c
+index 9bdfde0c0f86d..e7ea9b28d8db5 100644
+--- a/lib/logic_iomem.c
++++ b/lib/logic_iomem.c
+@@ -21,15 +21,15 @@ struct logic_iomem_area {
+
+ #define AREA_SHIFT 24
+ #define MAX_AREA_SIZE (1 << AREA_SHIFT)
+-#define MAX_AREAS ((1ULL<<32) / MAX_AREA_SIZE)
++#define MAX_AREAS ((1U << 31) / MAX_AREA_SIZE)
+ #define AREA_BITS ((MAX_AREAS - 1) << AREA_SHIFT)
+ #define AREA_MASK (MAX_AREA_SIZE - 1)
+ #ifdef CONFIG_64BIT
+ #define IOREMAP_BIAS 0xDEAD000000000000UL
+ #define IOREMAP_MASK 0xFFFFFFFF00000000UL
+ #else
+-#define IOREMAP_BIAS 0
+-#define IOREMAP_MASK 0
++#define IOREMAP_BIAS 0x80000000UL
++#define IOREMAP_MASK 0x80000000UL
+ #endif
+
+ static DEFINE_MUTEX(regions_mtx);
+@@ -68,7 +68,7 @@ int logic_iomem_add_region(struct resource *resource,
+ }
+ EXPORT_SYMBOL(logic_iomem_add_region);
+
+-#ifndef CONFIG_LOGIC_IOMEM_FALLBACK
++#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK
+ static void __iomem *real_ioremap(phys_addr_t offset, size_t size)
+ {
+ WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n",
+@@ -79,9 +79,9 @@ static void __iomem *real_ioremap(phys_addr_t offset, size_t size)
+ static void real_iounmap(void __iomem *addr)
+ {
+ WARN(1, "invalid iounmap for addr 0x%llx\n",
+- (unsigned long long __force)addr);
++ (unsigned long long)(uintptr_t __force)addr);
+ }
+-#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */
++#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */
+
+ void __iomem *ioremap(phys_addr_t offset, size_t size)
+ {
+@@ -168,12 +168,12 @@ void iounmap(void __iomem *addr)
+ }
+ EXPORT_SYMBOL(iounmap);
+
+-#ifndef CONFIG_LOGIC_IOMEM_FALLBACK
++#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK
+ #define MAKE_FALLBACK(op, sz) \
+ static u##sz real_raw_read ## op(const volatile void __iomem *addr) \
+ { \
+ WARN(1, "Invalid read" #op " at address %llx\n", \
+- (unsigned long long __force)addr); \
++ (unsigned long long)(uintptr_t __force)addr); \
+ return (u ## sz)~0ULL; \
+ } \
+ \
+@@ -181,7 +181,8 @@ static void real_raw_write ## op(u ## sz val, \
+ volatile void __iomem *addr) \
+ { \
+ WARN(1, "Invalid writeq" #op " of 0x%llx at address %llx\n", \
+- (unsigned long long)val, (unsigned long long __force)addr);\
++ (unsigned long long)val, \
++ (unsigned long long)(uintptr_t __force)addr);\
+ } \
+
+ MAKE_FALLBACK(b, 8);
+@@ -194,14 +195,14 @@ MAKE_FALLBACK(q, 64);
+ static void real_memset_io(volatile void __iomem *addr, int value, size_t size)
+ {
+ WARN(1, "Invalid memset_io at address 0x%llx\n",
+- (unsigned long long __force)addr);
++ (unsigned long long)(uintptr_t __force)addr);
+ }
+
+ static void real_memcpy_fromio(void *buffer, const volatile void __iomem *addr,
+ size_t size)
+ {
+ WARN(1, "Invalid memcpy_fromio at address 0x%llx\n",
+- (unsigned long long __force)addr);
++ (unsigned long long)(uintptr_t __force)addr);
+
+ memset(buffer, 0xff, size);
+ }
+@@ -210,9 +211,9 @@ static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer,
+ size_t size)
+ {
+ WARN(1, "Invalid memcpy_toio at address 0x%llx\n",
+- (unsigned long long __force)addr);
++ (unsigned long long)(uintptr_t __force)addr);
+ }
+-#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */
++#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */
+
+ #define MAKE_OP(op, sz) \
+ u##sz __raw_read ## op(const volatile void __iomem *addr) \
+diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c
+index 926f4823d5eac..fd1728d94babb 100644
+--- a/lib/lz4/lz4_decompress.c
++++ b/lib/lz4/lz4_decompress.c
+@@ -271,8 +271,12 @@ static FORCE_INLINE int LZ4_decompress_generic(
+ ip += length;
+ op += length;
+
+- /* Necessarily EOF, due to parsing restrictions */
+- if (!partialDecoding || (cpy == oend))
++ /* Necessarily EOF when !partialDecoding.
++ * When partialDecoding, it is EOF if we've either
++ * filled the output buffer or
++ * can't proceed with reading an offset for following match.
++ */
++ if (!partialDecoding || (cpy == oend) || (ip >= (iend - 2)))
+ break;
+ } else {
+ /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+diff --git a/lib/mpi/mpi-mod.c b/lib/mpi/mpi-mod.c
+index 47bc59edd4ff9..54fcc01564d9d 100644
+--- a/lib/mpi/mpi-mod.c
++++ b/lib/mpi/mpi-mod.c
+@@ -40,6 +40,8 @@ mpi_barrett_t mpi_barrett_init(MPI m, int copy)
+
+ mpi_normalize(m);
+ ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL);
++ if (!ctx)
++ return NULL;
+
+ if (copy) {
+ ctx->m = mpi_copy(m);
+diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c
+index f9e89001b52eb..199ab201d5019 100644
+--- a/lib/nmi_backtrace.c
++++ b/lib/nmi_backtrace.c
+@@ -75,6 +75,12 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask,
+ touch_softlockup_watchdog();
+ }
+
++ /*
++ * Force flush any remote buffers that might be stuck in IRQ context
++ * and therefore could not run their irq_work.
++ */
++ printk_trigger_flush();
++
+ clear_bit_unlock(0, &backtrace_flag);
+ put_cpu();
+ }
+diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
+index a4c7cd74cff58..4fb7700a741bd 100644
+--- a/lib/raid6/test/Makefile
++++ b/lib/raid6/test/Makefile
+@@ -4,6 +4,8 @@
+ # from userspace.
+ #
+
++pound := \#
++
+ CC = gcc
+ OPTFLAGS = -O2 # Adjust as desired
+ CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS)
+@@ -42,7 +44,7 @@ else ifeq ($(HAS_NEON),yes)
+ OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
+ CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1
+ else
+- HAS_ALTIVEC := $(shell printf '\#include <altivec.h>\nvector int a;\n' |\
++ HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\
+ gcc -c -x c - >/dev/null && rm ./-.o && echo yes)
+ ifeq ($(HAS_ALTIVEC),yes)
+ CFLAGS += -I../../../arch/powerpc/include
+diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c
+index a3cf071941ab4..841a55242abaa 100644
+--- a/lib/raid6/test/test.c
++++ b/lib/raid6/test/test.c
+@@ -19,7 +19,6 @@
+ #define NDISKS 16 /* Including P and Q */
+
+ const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+-struct raid6_calls raid6_call;
+
+ char *dataptrs[NDISKS];
+ char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+diff --git a/lib/random32.c b/lib/random32.c
+index 4d0e05e471d72..f0ab17c2244be 100644
+--- a/lib/random32.c
++++ b/lib/random32.c
+@@ -39,8 +39,9 @@
+ #include <linux/random.h>
+ #include <linux/sched.h>
+ #include <linux/bitops.h>
++#include <linux/slab.h>
++#include <linux/notifier.h>
+ #include <asm/unaligned.h>
+-#include <trace/events/random.h>
+
+ /**
+ * prandom_u32_state - seeded pseudo-random number generator.
+@@ -386,7 +387,6 @@ u32 prandom_u32(void)
+ struct siprand_state *state = get_cpu_ptr(&net_rand_state);
+ u32 res = siprand_u32(state);
+
+- trace_prandom_u32(res);
+ put_cpu_ptr(&net_rand_state);
+ return res;
+ }
+@@ -552,9 +552,11 @@ static void prandom_reseed(struct timer_list *unused)
+ * To avoid worrying about whether it's safe to delay that interrupt
+ * long enough to seed all CPUs, just schedule an immediate timer event.
+ */
+-static void prandom_timer_start(struct random_ready_callback *unused)
++static int prandom_timer_start(struct notifier_block *nb,
++ unsigned long action, void *data)
+ {
+ mod_timer(&seed_timer, jiffies);
++ return 0;
+ }
+
+ #ifdef CONFIG_RANDOM32_SELFTEST
+@@ -618,13 +620,13 @@ core_initcall(prandom32_state_selftest);
+ */
+ static int __init prandom_init_late(void)
+ {
+- static struct random_ready_callback random_ready = {
+- .func = prandom_timer_start
++ static struct notifier_block random_ready = {
++ .notifier_call = prandom_timer_start
+ };
+- int ret = add_random_ready_callback(&random_ready);
++ int ret = register_random_ready_notifier(&random_ready);
+
+ if (ret == -EALREADY) {
+- prandom_timer_start(&random_ready);
++ prandom_timer_start(&random_ready, 0, NULL);
+ ret = 0;
+ }
+ return ret;
+diff --git a/lib/sha1.c b/lib/sha1.c
+index 9bd1935a14727..0494766fc574e 100644
+--- a/lib/sha1.c
++++ b/lib/sha1.c
+@@ -9,6 +9,7 @@
+ #include <linux/kernel.h>
+ #include <linux/export.h>
+ #include <linux/bitops.h>
++#include <linux/string.h>
+ #include <crypto/sha1.h>
+ #include <asm/unaligned.h>
+
+@@ -55,7 +56,8 @@
+ #define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
+ __u32 TEMP = input(t); setW(t, TEMP); \
+ E += TEMP + rol32(A,5) + (fn) + (constant); \
+- B = ror32(B, 2); } while (0)
++ B = ror32(B, 2); \
++ TEMP = E; E = D; D = C; C = B; B = A; A = TEMP; } while (0)
+
+ #define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
+ #define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
+@@ -84,6 +86,7 @@
+ void sha1_transform(__u32 *digest, const char *data, __u32 *array)
+ {
+ __u32 A, B, C, D, E;
++ unsigned int i = 0;
+
+ A = digest[0];
+ B = digest[1];
+@@ -92,94 +95,24 @@ void sha1_transform(__u32 *digest, const char *data, __u32 *array)
+ E = digest[4];
+
+ /* Round 1 - iterations 0-16 take their input from 'data' */
+- T_0_15( 0, A, B, C, D, E);
+- T_0_15( 1, E, A, B, C, D);
+- T_0_15( 2, D, E, A, B, C);
+- T_0_15( 3, C, D, E, A, B);
+- T_0_15( 4, B, C, D, E, A);
+- T_0_15( 5, A, B, C, D, E);
+- T_0_15( 6, E, A, B, C, D);
+- T_0_15( 7, D, E, A, B, C);
+- T_0_15( 8, C, D, E, A, B);
+- T_0_15( 9, B, C, D, E, A);
+- T_0_15(10, A, B, C, D, E);
+- T_0_15(11, E, A, B, C, D);
+- T_0_15(12, D, E, A, B, C);
+- T_0_15(13, C, D, E, A, B);
+- T_0_15(14, B, C, D, E, A);
+- T_0_15(15, A, B, C, D, E);
++ for (; i < 16; ++i)
++ T_0_15(i, A, B, C, D, E);
+
+ /* Round 1 - tail. Input from 512-bit mixing array */
+- T_16_19(16, E, A, B, C, D);
+- T_16_19(17, D, E, A, B, C);
+- T_16_19(18, C, D, E, A, B);
+- T_16_19(19, B, C, D, E, A);
++ for (; i < 20; ++i)
++ T_16_19(i, A, B, C, D, E);
+
+ /* Round 2 */
+- T_20_39(20, A, B, C, D, E);
+- T_20_39(21, E, A, B, C, D);
+- T_20_39(22, D, E, A, B, C);
+- T_20_39(23, C, D, E, A, B);
+- T_20_39(24, B, C, D, E, A);
+- T_20_39(25, A, B, C, D, E);
+- T_20_39(26, E, A, B, C, D);
+- T_20_39(27, D, E, A, B, C);
+- T_20_39(28, C, D, E, A, B);
+- T_20_39(29, B, C, D, E, A);
+- T_20_39(30, A, B, C, D, E);
+- T_20_39(31, E, A, B, C, D);
+- T_20_39(32, D, E, A, B, C);
+- T_20_39(33, C, D, E, A, B);
+- T_20_39(34, B, C, D, E, A);
+- T_20_39(35, A, B, C, D, E);
+- T_20_39(36, E, A, B, C, D);
+- T_20_39(37, D, E, A, B, C);
+- T_20_39(38, C, D, E, A, B);
+- T_20_39(39, B, C, D, E, A);
++ for (; i < 40; ++i)
++ T_20_39(i, A, B, C, D, E);
+
+ /* Round 3 */
+- T_40_59(40, A, B, C, D, E);
+- T_40_59(41, E, A, B, C, D);
+- T_40_59(42, D, E, A, B, C);
+- T_40_59(43, C, D, E, A, B);
+- T_40_59(44, B, C, D, E, A);
+- T_40_59(45, A, B, C, D, E);
+- T_40_59(46, E, A, B, C, D);
+- T_40_59(47, D, E, A, B, C);
+- T_40_59(48, C, D, E, A, B);
+- T_40_59(49, B, C, D, E, A);
+- T_40_59(50, A, B, C, D, E);
+- T_40_59(51, E, A, B, C, D);
+- T_40_59(52, D, E, A, B, C);
+- T_40_59(53, C, D, E, A, B);
+- T_40_59(54, B, C, D, E, A);
+- T_40_59(55, A, B, C, D, E);
+- T_40_59(56, E, A, B, C, D);
+- T_40_59(57, D, E, A, B, C);
+- T_40_59(58, C, D, E, A, B);
+- T_40_59(59, B, C, D, E, A);
++ for (; i < 60; ++i)
++ T_40_59(i, A, B, C, D, E);
+
+ /* Round 4 */
+- T_60_79(60, A, B, C, D, E);
+- T_60_79(61, E, A, B, C, D);
+- T_60_79(62, D, E, A, B, C);
+- T_60_79(63, C, D, E, A, B);
+- T_60_79(64, B, C, D, E, A);
+- T_60_79(65, A, B, C, D, E);
+- T_60_79(66, E, A, B, C, D);
+- T_60_79(67, D, E, A, B, C);
+- T_60_79(68, C, D, E, A, B);
+- T_60_79(69, B, C, D, E, A);
+- T_60_79(70, A, B, C, D, E);
+- T_60_79(71, E, A, B, C, D);
+- T_60_79(72, D, E, A, B, C);
+- T_60_79(73, C, D, E, A, B);
+- T_60_79(74, B, C, D, E, A);
+- T_60_79(75, A, B, C, D, E);
+- T_60_79(76, E, A, B, C, D);
+- T_60_79(77, D, E, A, B, C);
+- T_60_79(78, C, D, E, A, B);
+- T_60_79(79, B, C, D, E, A);
++ for (; i < 80; ++i)
++ T_60_79(i, A, B, C, D, E);
+
+ digest[0] += A;
+ digest[1] += B;
+diff --git a/lib/siphash.c b/lib/siphash.c
+index a90112ee72a1f..71d315a6ad623 100644
+--- a/lib/siphash.c
++++ b/lib/siphash.c
+@@ -18,19 +18,13 @@
+ #include <asm/word-at-a-time.h>
+ #endif
+
+-#define SIPROUND \
+- do { \
+- v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
+- v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
+- v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
+- v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
+- } while (0)
++#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3)
+
+ #define PREAMBLE(len) \
+- u64 v0 = 0x736f6d6570736575ULL; \
+- u64 v1 = 0x646f72616e646f6dULL; \
+- u64 v2 = 0x6c7967656e657261ULL; \
+- u64 v3 = 0x7465646279746573ULL; \
++ u64 v0 = SIPHASH_CONST_0; \
++ u64 v1 = SIPHASH_CONST_1; \
++ u64 v2 = SIPHASH_CONST_2; \
++ u64 v3 = SIPHASH_CONST_3; \
+ u64 b = ((u64)(len)) << 56; \
+ v3 ^= key->key[1]; \
+ v2 ^= key->key[0]; \
+@@ -49,6 +43,7 @@
+ SIPROUND; \
+ return (v0 ^ v1) ^ (v2 ^ v3);
+
++#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
+ {
+ const u8 *end = data + len - (len % sizeof(u64));
+@@ -80,8 +75,8 @@ u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
+ POSTAMBLE
+ }
+ EXPORT_SYMBOL(__siphash_aligned);
++#endif
+
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
+ {
+ const u8 *end = data + len - (len % sizeof(u64));
+@@ -113,7 +108,6 @@ u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
+ POSTAMBLE
+ }
+ EXPORT_SYMBOL(__siphash_unaligned);
+-#endif
+
+ /**
+ * siphash_1u64 - compute 64-bit siphash PRF value of a u64
+@@ -250,6 +244,7 @@ EXPORT_SYMBOL(siphash_3u32);
+ HSIPROUND; \
+ return (v0 ^ v1) ^ (v2 ^ v3);
+
++#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+ {
+ const u8 *end = data + len - (len % sizeof(u64));
+@@ -280,8 +275,8 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+ HPOSTAMBLE
+ }
+ EXPORT_SYMBOL(__hsiphash_aligned);
++#endif
+
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u32 __hsiphash_unaligned(const void *data, size_t len,
+ const hsiphash_key_t *key)
+ {
+@@ -313,7 +308,6 @@ u32 __hsiphash_unaligned(const void *data, size_t len,
+ HPOSTAMBLE
+ }
+ EXPORT_SYMBOL(__hsiphash_unaligned);
+-#endif
+
+ /**
+ * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
+@@ -389,19 +383,13 @@ u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
+ }
+ EXPORT_SYMBOL(hsiphash_4u32);
+ #else
+-#define HSIPROUND \
+- do { \
+- v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
+- v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
+- v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
+- v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
+- } while (0)
++#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3)
+
+ #define HPREAMBLE(len) \
+- u32 v0 = 0; \
+- u32 v1 = 0; \
+- u32 v2 = 0x6c796765U; \
+- u32 v3 = 0x74656462U; \
++ u32 v0 = HSIPHASH_CONST_0; \
++ u32 v1 = HSIPHASH_CONST_1; \
++ u32 v2 = HSIPHASH_CONST_2; \
++ u32 v3 = HSIPHASH_CONST_3; \
+ u32 b = ((u32)(len)) << 24; \
+ v3 ^= key->key[1]; \
+ v2 ^= key->key[0]; \
+@@ -418,6 +406,7 @@ EXPORT_SYMBOL(hsiphash_4u32);
+ HSIPROUND; \
+ return v1 ^ v3;
+
++#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+ {
+ const u8 *end = data + len - (len % sizeof(u32));
+@@ -438,8 +427,8 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
+ HPOSTAMBLE
+ }
+ EXPORT_SYMBOL(__hsiphash_aligned);
++#endif
+
+-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ u32 __hsiphash_unaligned(const void *data, size_t len,
+ const hsiphash_key_t *key)
+ {
+@@ -461,7 +450,6 @@ u32 __hsiphash_unaligned(const void *data, size_t len,
+ HPOSTAMBLE
+ }
+ EXPORT_SYMBOL(__hsiphash_unaligned);
+-#endif
+
+ /**
+ * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32
+diff --git a/lib/stackdepot.c b/lib/stackdepot.c
+index 0a2e417f83cba..e90f0f19e77f9 100644
+--- a/lib/stackdepot.c
++++ b/lib/stackdepot.c
+@@ -20,7 +20,6 @@
+ */
+
+ #include <linux/gfp.h>
+-#include <linux/interrupt.h>
+ #include <linux/jhash.h>
+ #include <linux/kernel.h>
+ #include <linux/mm.h>
+@@ -341,26 +340,3 @@ fast_exit:
+ return retval;
+ }
+ EXPORT_SYMBOL_GPL(stack_depot_save);
+-
+-static inline int in_irqentry_text(unsigned long ptr)
+-{
+- return (ptr >= (unsigned long)&__irqentry_text_start &&
+- ptr < (unsigned long)&__irqentry_text_end) ||
+- (ptr >= (unsigned long)&__softirqentry_text_start &&
+- ptr < (unsigned long)&__softirqentry_text_end);
+-}
+-
+-unsigned int filter_irq_stacks(unsigned long *entries,
+- unsigned int nr_entries)
+-{
+- unsigned int i;
+-
+- for (i = 0; i < nr_entries; i++) {
+- if (in_irqentry_text(entries[i])) {
+- /* Include the irqentry function into the stack. */
+- return i + 1;
+- }
+- }
+- return nr_entries;
+-}
+-EXPORT_SYMBOL_GPL(filter_irq_stacks);
+diff --git a/lib/string_helpers.c b/lib/string_helpers.c
+index 3806a52ce697a..2ddc10bd9add6 100644
+--- a/lib/string_helpers.c
++++ b/lib/string_helpers.c
+@@ -696,3 +696,23 @@ void kfree_strarray(char **array, size_t n)
+ kfree(array);
+ }
+ EXPORT_SYMBOL_GPL(kfree_strarray);
++
++/**
++ * memcpy_and_pad - Copy one buffer to another with padding
++ * @dest: Where to copy to
++ * @dest_len: The destination buffer size
++ * @src: Where to copy from
++ * @count: The number of bytes to copy
++ * @pad: Character to use for padding if space is left in destination.
++ */
++void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count,
++ int pad)
++{
++ if (dest_len > count) {
++ memcpy(dest, src, count);
++ memset(dest + count, pad, dest_len - count);
++ } else {
++ memcpy(dest, src, dest_len);
++ }
++}
++EXPORT_SYMBOL(memcpy_and_pad);
+diff --git a/lib/test_bpf.c b/lib/test_bpf.c
+index 830a18ecffc88..68d125b409f20 100644
+--- a/lib/test_bpf.c
++++ b/lib/test_bpf.c
+@@ -8992,10 +8992,15 @@ static __init int test_bpf(void)
+ struct tail_call_test {
+ const char *descr;
+ struct bpf_insn insns[MAX_INSNS];
++ int flags;
+ int result;
+ int stack_depth;
+ };
+
++/* Flags that can be passed to tail call test cases */
++#define FLAG_NEED_STATE BIT(0)
++#define FLAG_RESULT_IN_STATE BIT(1)
++
+ /*
+ * Magic marker used in test snippets for tail calls below.
+ * BPF_LD/MOV to R2 and R2 with this immediate value is replaced
+@@ -9065,32 +9070,38 @@ static struct tail_call_test tail_call_tests[] = {
+ {
+ "Tail call error path, max count reached",
+ .insns = {
+- BPF_ALU64_IMM(BPF_ADD, R1, 1),
+- BPF_ALU64_REG(BPF_MOV, R0, R1),
++ BPF_LDX_MEM(BPF_W, R2, R1, 0),
++ BPF_ALU64_IMM(BPF_ADD, R2, 1),
++ BPF_STX_MEM(BPF_W, R1, R2, 0),
+ TAIL_CALL(0),
+ BPF_EXIT_INSN(),
+ },
+- .result = MAX_TAIL_CALL_CNT + 1,
++ .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE,
++ .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS,
+ },
+ {
+ "Tail call error path, NULL target",
+ .insns = {
+- BPF_ALU64_IMM(BPF_MOV, R0, -1),
++ BPF_LDX_MEM(BPF_W, R2, R1, 0),
++ BPF_ALU64_IMM(BPF_ADD, R2, 1),
++ BPF_STX_MEM(BPF_W, R1, R2, 0),
+ TAIL_CALL(TAIL_CALL_NULL),
+- BPF_ALU64_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+- .result = 1,
++ .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE,
++ .result = MAX_TESTRUNS,
+ },
+ {
+ "Tail call error path, index out of range",
+ .insns = {
+- BPF_ALU64_IMM(BPF_MOV, R0, -1),
++ BPF_LDX_MEM(BPF_W, R2, R1, 0),
++ BPF_ALU64_IMM(BPF_ADD, R2, 1),
++ BPF_STX_MEM(BPF_W, R1, R2, 0),
+ TAIL_CALL(TAIL_CALL_INVALID),
+- BPF_ALU64_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+- .result = 1,
++ .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE,
++ .result = MAX_TESTRUNS,
+ },
+ };
+
+@@ -9196,6 +9207,8 @@ static __init int test_tail_calls(struct bpf_array *progs)
+ for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) {
+ struct tail_call_test *test = &tail_call_tests[i];
+ struct bpf_prog *fp = progs->ptrs[i];
++ int *data = NULL;
++ int state = 0;
+ u64 duration;
+ int ret;
+
+@@ -9212,7 +9225,11 @@ static __init int test_tail_calls(struct bpf_array *progs)
+ if (fp->jited)
+ jit_cnt++;
+
+- ret = __run_one(fp, NULL, MAX_TESTRUNS, &duration);
++ if (test->flags & FLAG_NEED_STATE)
++ data = &state;
++ ret = __run_one(fp, data, MAX_TESTRUNS, &duration);
++ if (test->flags & FLAG_RESULT_IN_STATE)
++ ret = state;
+ if (ret == test->result) {
+ pr_cont("%lld PASS", duration);
+ pass_cnt++;
+diff --git a/lib/test_hmm.c b/lib/test_hmm.c
+index c259842f6d443..ac794e3540693 100644
+--- a/lib/test_hmm.c
++++ b/lib/test_hmm.c
+@@ -1087,9 +1087,33 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
+ return 0;
+ }
+
++static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
++{
++ unsigned long addr;
++
++ for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
++ struct page *page;
++ int ret;
++
++ page = alloc_page(GFP_KERNEL | __GFP_ZERO);
++ if (!page)
++ return -ENOMEM;
++
++ ret = vm_insert_page(vma, addr, page);
++ if (ret) {
++ __free_page(page);
++ return ret;
++ }
++ put_page(page);
++ }
++
++ return 0;
++}
++
+ static const struct file_operations dmirror_fops = {
+ .open = dmirror_fops_open,
+ .release = dmirror_fops_release,
++ .mmap = dmirror_fops_mmap,
+ .unlocked_ioctl = dmirror_fops_unlocked_ioctl,
+ .llseek = default_llseek,
+ .owner = THIS_MODULE,
+diff --git a/lib/test_kmod.c b/lib/test_kmod.c
+index ce15893914131..cb800b1d0d99c 100644
+--- a/lib/test_kmod.c
++++ b/lib/test_kmod.c
+@@ -1149,6 +1149,7 @@ static struct kmod_test_device *register_test_dev_kmod(void)
+ if (ret) {
+ pr_err("could not register misc device: %d\n", ret);
+ free_test_dev_kmod(test_dev);
++ test_dev = NULL;
+ goto out;
+ }
+
+diff --git a/lib/test_lockup.c b/lib/test_lockup.c
+index 906b598740a7b..c3fd87d6c2dd0 100644
+--- a/lib/test_lockup.c
++++ b/lib/test_lockup.c
+@@ -417,9 +417,14 @@ static bool test_kernel_ptr(unsigned long addr, int size)
+ return false;
+
+ /* should be at least readable kernel address */
+- if (access_ok(ptr, 1) ||
+- access_ok(ptr + size - 1, 1) ||
+- get_kernel_nofault(buf, ptr) ||
++ if (!IS_ENABLED(CONFIG_ALTERNATE_USER_ADDRESS_SPACE) &&
++ (access_ok((void __user *)ptr, 1) ||
++ access_ok((void __user *)ptr + size - 1, 1))) {
++ pr_err("user space ptr invalid in kernel: %#lx\n", addr);
++ return true;
++ }
++
++ if (get_kernel_nofault(buf, ptr) ||
+ get_kernel_nofault(buf, ptr + size - 1)) {
+ pr_err("invalid kernel ptr: %#lx\n", addr);
+ return true;
+diff --git a/lib/test_meminit.c b/lib/test_meminit.c
+index e4f706a404b3a..3ca717f113977 100644
+--- a/lib/test_meminit.c
++++ b/lib/test_meminit.c
+@@ -337,6 +337,7 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
+ if (num)
+ kmem_cache_free_bulk(c, num, objects);
+ }
++ kmem_cache_destroy(c);
+ *total_failures += fail;
+ return 1;
+ }
+diff --git a/lib/test_ubsan.c b/lib/test_ubsan.c
+index 7e7bbd0f3fd27..2062be1f2e80f 100644
+--- a/lib/test_ubsan.c
++++ b/lib/test_ubsan.c
+@@ -79,15 +79,6 @@ static void test_ubsan_load_invalid_value(void)
+ eval2 = eval;
+ }
+
+-static void test_ubsan_null_ptr_deref(void)
+-{
+- volatile int *ptr = NULL;
+- int val;
+-
+- UBSAN_TEST(CONFIG_UBSAN_OBJECT_SIZE);
+- val = *ptr;
+-}
+-
+ static void test_ubsan_misaligned_access(void)
+ {
+ volatile char arr[5] __aligned(4) = {1, 2, 3, 4, 5};
+@@ -98,29 +89,16 @@ static void test_ubsan_misaligned_access(void)
+ *ptr = val;
+ }
+
+-static void test_ubsan_object_size_mismatch(void)
+-{
+- /* "((aligned(8)))" helps this not into be misaligned for ptr-access. */
+- volatile int val __aligned(8) = 4;
+- volatile long long *ptr, val2;
+-
+- UBSAN_TEST(CONFIG_UBSAN_OBJECT_SIZE);
+- ptr = (long long *)&val;
+- val2 = *ptr;
+-}
+-
+ static const test_ubsan_fp test_ubsan_array[] = {
+ test_ubsan_shift_out_of_bounds,
+ test_ubsan_out_of_bounds,
+ test_ubsan_load_invalid_value,
+ test_ubsan_misaligned_access,
+- test_ubsan_object_size_mismatch,
+ };
+
+ /* Excluded because they Oops the module. */
+ static const test_ubsan_fp skip_ubsan_array[] = {
+ test_ubsan_divrem_overflow,
+- test_ubsan_null_ptr_deref,
+ };
+
+ static int __init test_ubsan_init(void)
+diff --git a/lib/test_xarray.c b/lib/test_xarray.c
+index 8b1c318189ce8..e77d4856442c3 100644
+--- a/lib/test_xarray.c
++++ b/lib/test_xarray.c
+@@ -1463,6 +1463,25 @@ unlock:
+ XA_BUG_ON(xa, !xa_empty(xa));
+ }
+
++static noinline void check_create_range_5(struct xarray *xa,
++ unsigned long index, unsigned int order)
++{
++ XA_STATE_ORDER(xas, xa, index, order);
++ unsigned int i;
++
++ xa_store_order(xa, index, order, xa_mk_index(index), GFP_KERNEL);
++
++ for (i = 0; i < order + 10; i++) {
++ do {
++ xas_lock(&xas);
++ xas_create_range(&xas);
++ xas_unlock(&xas);
++ } while (xas_nomem(&xas, GFP_KERNEL));
++ }
++
++ xa_destroy(xa);
++}
++
+ static noinline void check_create_range(struct xarray *xa)
+ {
+ unsigned int order;
+@@ -1490,6 +1509,9 @@ static noinline void check_create_range(struct xarray *xa)
+ check_create_range_4(xa, (3U << order) + 1, order);
+ check_create_range_4(xa, (3U << order) - 1, order);
+ check_create_range_4(xa, (1U << 24) + 1, order);
++
++ check_create_range_5(xa, 0, order);
++ check_create_range_5(xa, (1U << order), order);
+ }
+
+ check_create_range_3();
+diff --git a/lib/vsprintf.c b/lib/vsprintf.c
+index d7ad44f2c8f57..a60f0bb2ea902 100644
+--- a/lib/vsprintf.c
++++ b/lib/vsprintf.c
+@@ -49,12 +49,16 @@
+
+ #include <asm/page.h> /* for PAGE_SIZE */
+ #include <asm/byteorder.h> /* cpu_to_le16 */
++#include <asm/unaligned.h>
+
+ #include <linux/string_helpers.h>
+ #include "kstrtox.h"
+
+-static unsigned long long simple_strntoull(const char *startp, size_t max_chars,
+- char **endp, unsigned int base)
++/* Disable pointer hashing if requested */
++bool no_hash_pointers __ro_after_init;
++EXPORT_SYMBOL_GPL(no_hash_pointers);
++
++static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base)
+ {
+ const char *cp;
+ unsigned long long result = 0ULL;
+@@ -757,14 +761,16 @@ static void enable_ptr_key_workfn(struct work_struct *work)
+
+ static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn);
+
+-static void fill_random_ptr_key(struct random_ready_callback *unused)
++static int fill_random_ptr_key(struct notifier_block *nb,
++ unsigned long action, void *data)
+ {
+ /* This may be in an interrupt handler. */
+ queue_work(system_unbound_wq, &enable_ptr_key_work);
++ return 0;
+ }
+
+-static struct random_ready_callback random_ready = {
+- .func = fill_random_ptr_key
++static struct notifier_block random_ready = {
++ .notifier_call = fill_random_ptr_key
+ };
+
+ static int __init initialize_ptr_random(void)
+@@ -778,7 +784,7 @@ static int __init initialize_ptr_random(void)
+ return 0;
+ }
+
+- ret = add_random_ready_callback(&random_ready);
++ ret = register_random_ready_notifier(&random_ready);
+ if (!ret) {
+ return 0;
+ } else if (ret == -EALREADY) {
+@@ -848,6 +854,19 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr,
+ return pointer_string(buf, end, (const void *)hashval, spec);
+ }
+
++static char *default_pointer(char *buf, char *end, const void *ptr,
++ struct printf_spec spec)
++{
++ /*
++ * default is to _not_ leak addresses, so hash before printing,
++ * unless no_hash_pointers is specified on the command line.
++ */
++ if (unlikely(no_hash_pointers))
++ return pointer_string(buf, end, ptr, spec);
++
++ return ptr_to_id(buf, end, ptr, spec);
++}
++
+ int kptr_restrict __read_mostly;
+
+ static noinline_for_stack
+@@ -857,7 +876,7 @@ char *restricted_pointer(char *buf, char *end, const void *ptr,
+ switch (kptr_restrict) {
+ case 0:
+ /* Handle as %p, hash and do _not_ leak addresses. */
+- return ptr_to_id(buf, end, ptr, spec);
++ return default_pointer(buf, end, ptr, spec);
+ case 1: {
+ const struct cred *cred;
+
+@@ -1771,7 +1790,7 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc,
+ char output[sizeof("0123 little-endian (0x01234567)")];
+ char *p = output;
+ unsigned int i;
+- u32 val;
++ u32 orig, val;
+
+ if (fmt[1] != 'c' || fmt[2] != 'c')
+ return error_string(buf, end, "(%p4?)", spec);
+@@ -1779,21 +1798,22 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc,
+ if (check_pointer(&buf, end, fourcc, spec))
+ return buf;
+
+- val = *fourcc & ~BIT(31);
++ orig = get_unaligned(fourcc);
++ val = orig & ~BIT(31);
+
+- for (i = 0; i < sizeof(*fourcc); i++) {
++ for (i = 0; i < sizeof(u32); i++) {
+ unsigned char c = val >> (i * 8);
+
+ /* Print non-control ASCII characters as-is, dot otherwise */
+ *p++ = isascii(c) && isprint(c) ? c : '.';
+ }
+
+- strcpy(p, *fourcc & BIT(31) ? " big-endian" : " little-endian");
++ strcpy(p, orig & BIT(31) ? " big-endian" : " little-endian");
+ p += strlen(p);
+
+ *p++ = ' ';
+ *p++ = '(';
+- p = special_hex_number(p, output + sizeof(output) - 2, *fourcc, sizeof(u32));
++ p = special_hex_number(p, output + sizeof(output) - 2, orig, sizeof(u32));
+ *p++ = ')';
+ *p = '\0';
+
+@@ -2225,10 +2245,6 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode,
+ return widen_string(buf, buf - buf_start, end, spec);
+ }
+
+-/* Disable pointer hashing if requested */
+-bool no_hash_pointers __ro_after_init;
+-EXPORT_SYMBOL_GPL(no_hash_pointers);
+-
+ int __init no_hash_pointers_enable(char *str)
+ {
+ if (no_hash_pointers)
+@@ -2457,7 +2473,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
+ case 'e':
+ /* %pe with a non-ERR_PTR gets treated as plain %p */
+ if (!IS_ERR(ptr))
+- break;
++ return default_pointer(buf, end, ptr, spec);
+ return err_ptr(buf, end, ptr, spec);
+ case 'u':
+ case 'k':
+@@ -2467,16 +2483,9 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
+ default:
+ return error_string(buf, end, "(einval)", spec);
+ }
++ default:
++ return default_pointer(buf, end, ptr, spec);
+ }
+-
+- /*
+- * default is to _not_ leak addresses, so hash before printing,
+- * unless no_hash_pointers is specified on the command line.
+- */
+- if (unlikely(no_hash_pointers))
+- return pointer_string(buf, end, ptr, spec);
+- else
+- return ptr_to_id(buf, end, ptr, spec);
+ }
+
+ /*
+diff --git a/lib/xarray.c b/lib/xarray.c
+index f5d8f54907b4f..96e2d7748e5aa 100644
+--- a/lib/xarray.c
++++ b/lib/xarray.c
+@@ -722,6 +722,8 @@ void xas_create_range(struct xa_state *xas)
+
+ for (;;) {
+ struct xa_node *node = xas->xa_node;
++ if (node->shift >= shift)
++ break;
+ xas->xa_node = xa_parent_locked(xas->xa, node);
+ xas->xa_offset = node->offset - 1;
+ if (node->offset != 0)
+@@ -1079,6 +1081,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order)
+ xa_mk_node(child));
+ if (xa_is_value(curr))
+ values--;
++ xas_update(xas, child);
+ } else {
+ unsigned int canon = offset - xas->xa_sibs;
+
+@@ -1093,6 +1096,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order)
+ } while (offset-- > xas->xa_offset);
+
+ node->nr_values += values;
++ xas_update(xas, node);
+ }
+ EXPORT_SYMBOL_GPL(xas_split);
+ #endif
+diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c
+index 7a6781e3f47b6..d548cf0e59fe6 100644
+--- a/lib/xz/xz_dec_lzma2.c
++++ b/lib/xz/xz_dec_lzma2.c
+@@ -387,7 +387,14 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b,
+
+ *left -= copy_size;
+
+- memcpy(dict->buf + dict->pos, b->in + b->in_pos, copy_size);
++ /*
++ * If doing in-place decompression in single-call mode and the
++ * uncompressed size of the file is larger than the caller
++ * thought (i.e. it is invalid input!), the buffers below may
++ * overlap and cause undefined behavior with memcpy().
++ * With valid inputs memcpy() would be fine here.
++ */
++ memmove(dict->buf + dict->pos, b->in + b->in_pos, copy_size);
+ dict->pos += copy_size;
+
+ if (dict->full < dict->pos)
+@@ -397,7 +404,11 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b,
+ if (dict->pos == dict->end)
+ dict->pos = 0;
+
+- memcpy(b->out + b->out_pos, b->in + b->in_pos,
++ /*
++ * Like above but for multi-call mode: use memmove()
++ * to avoid undefined behavior with invalid input.
++ */
++ memmove(b->out + b->out_pos, b->in + b->in_pos,
+ copy_size);
+ }
+
+@@ -421,6 +432,12 @@ static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b)
+ if (dict->pos == dict->end)
+ dict->pos = 0;
+
++ /*
++ * These buffers cannot overlap even if doing in-place
++ * decompression because in multi-call mode dict->buf
++ * has been allocated by us in this file; it's not
++ * provided by the caller like in single-call mode.
++ */
+ memcpy(b->out + b->out_pos, dict->buf + dict->start,
+ copy_size);
+ }
+diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
+index fea86deaaa01d..683570b93a8c4 100644
+--- a/lib/xz/xz_dec_stream.c
++++ b/lib/xz/xz_dec_stream.c
+@@ -402,12 +402,12 @@ static enum xz_ret dec_stream_header(struct xz_dec *s)
+ * we will accept other check types too, but then the check won't
+ * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given.
+ */
++ if (s->temp.buf[HEADER_MAGIC_SIZE + 1] > XZ_CHECK_MAX)
++ return XZ_OPTIONS_ERROR;
++
+ s->check_type = s->temp.buf[HEADER_MAGIC_SIZE + 1];
+
+ #ifdef XZ_DEC_ANY_CHECK
+- if (s->check_type > XZ_CHECK_MAX)
+- return XZ_OPTIONS_ERROR;
+-
+ if (s->check_type > XZ_CHECK_CRC32)
+ return XZ_UNSUPPORTED_CHECK;
+ #else
+diff --git a/mm/Kconfig b/mm/Kconfig
+index d16ba9249bc53..c048dea7e3420 100644
+--- a/mm/Kconfig
++++ b/mm/Kconfig
+@@ -887,6 +887,9 @@ config MAPPING_DIRTY_HELPERS
+ config KMAP_LOCAL
+ bool
+
++config KMAP_LOCAL_NON_LINEAR_PTE_ARRAY
++ bool
++
+ # struct io_mapping based helper. Selected by drivers that need them
+ config IO_MAPPING
+ bool
+diff --git a/mm/backing-dev.c b/mm/backing-dev.c
+index 4a9d4e27d0d9b..02ff66f863587 100644
+--- a/mm/backing-dev.c
++++ b/mm/backing-dev.c
+@@ -947,6 +947,13 @@ void bdi_unregister(struct backing_dev_info *bdi)
+ wb_shutdown(&bdi->wb);
+ cgwb_bdi_unregister(bdi);
+
++ /*
++ * If this BDI's min ratio has been set, use bdi_set_min_ratio() to
++ * update the global bdi_min_ratio.
++ */
++ if (bdi->min_ratio)
++ bdi_set_min_ratio(bdi, 0);
++
+ if (bdi->dev) {
+ bdi_debug_unregister(bdi);
+ device_unregister(bdi->dev);
+diff --git a/mm/damon/core.c b/mm/damon/core.c
+index 30e9211f494a7..7a4912d6e65f2 100644
+--- a/mm/damon/core.c
++++ b/mm/damon/core.c
+@@ -357,6 +357,15 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs)
+ return err;
+ }
+
++static void kdamond_usleep(unsigned long usecs)
++{
++ /* See Documentation/timers/timers-howto.rst for the thresholds */
++ if (usecs > 20 * 1000)
++ schedule_timeout_idle(usecs_to_jiffies(usecs));
++ else
++ usleep_idle_range(usecs, usecs + 1);
++}
++
+ /*
+ * __damon_stop() - Stops monitoring of given context.
+ * @ctx: monitoring context
+@@ -370,8 +379,7 @@ static int __damon_stop(struct damon_ctx *ctx)
+ ctx->kdamond_stop = true;
+ mutex_unlock(&ctx->kdamond_lock);
+ while (damon_kdamond_running(ctx))
+- usleep_range(ctx->sample_interval,
+- ctx->sample_interval * 2);
++ kdamond_usleep(ctx->sample_interval);
+ return 0;
+ }
+ mutex_unlock(&ctx->kdamond_lock);
+@@ -670,7 +678,7 @@ static int kdamond_fn(void *data)
+ ctx->callback.after_sampling(ctx))
+ set_kdamond_stop(ctx);
+
+- usleep_range(ctx->sample_interval, ctx->sample_interval + 1);
++ kdamond_usleep(ctx->sample_interval);
+
+ if (ctx->primitive.check_accesses)
+ max_nr_accesses = ctx->primitive.check_accesses(ctx);
+diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c
+index faee070977d80..36624990b5777 100644
+--- a/mm/damon/dbgfs.c
++++ b/mm/damon/dbgfs.c
+@@ -32,7 +32,7 @@ static char *user_input_str(const char __user *buf, size_t count, loff_t *ppos)
+ if (*ppos)
+ return ERR_PTR(-EINVAL);
+
+- kbuf = kmalloc(count + 1, GFP_KERNEL);
++ kbuf = kmalloc(count + 1, GFP_KERNEL | __GFP_NOWARN);
+ if (!kbuf)
+ return ERR_PTR(-ENOMEM);
+
+@@ -185,6 +185,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file,
+ const char __user *buf, size_t count, loff_t *ppos)
+ {
+ struct damon_ctx *ctx = file->private_data;
++ struct damon_target *t, *next_t;
+ char *kbuf, *nrs;
+ unsigned long *targets;
+ ssize_t nr_targets;
+@@ -224,6 +225,13 @@ static ssize_t dbgfs_target_ids_write(struct file *file,
+ goto unlock_out;
+ }
+
++ /* remove previously set targets */
++ damon_for_each_target_safe(t, next_t, ctx) {
++ if (targetid_is_pid(ctx))
++ put_pid((struct pid *)t->id);
++ damon_destroy_target(t);
++ }
++
+ err = damon_set_targets(ctx, targets, nr_targets);
+ if (err) {
+ if (targetid_is_pid(ctx))
+@@ -247,7 +255,7 @@ static ssize_t dbgfs_kdamond_pid_read(struct file *file,
+ char *kbuf;
+ ssize_t len;
+
+- kbuf = kmalloc(count, GFP_KERNEL);
++ kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN);
+ if (!kbuf)
+ return -ENOMEM;
+
+@@ -309,10 +317,12 @@ static int dbgfs_before_terminate(struct damon_ctx *ctx)
+ if (!targetid_is_pid(ctx))
+ return 0;
+
++ mutex_lock(&ctx->kdamond_lock);
+ damon_for_each_target_safe(t, next, ctx) {
+ put_pid((struct pid *)t->id);
+ damon_destroy_target(t);
+ }
++ mutex_unlock(&ctx->kdamond_lock);
+ return 0;
+ }
+
+@@ -538,12 +548,14 @@ static ssize_t dbgfs_monitor_on_write(struct file *file,
+ return -EINVAL;
+ }
+
++ mutex_lock(&damon_dbgfs_lock);
+ if (!strncmp(kbuf, "on", count))
+ err = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs);
+ else if (!strncmp(kbuf, "off", count))
+ err = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs);
+ else
+ err = -EINVAL;
++ mutex_unlock(&damon_dbgfs_lock);
+
+ if (err)
+ ret = err;
+@@ -596,15 +608,16 @@ static int __init __damon_dbgfs_init(void)
+
+ static int __init damon_dbgfs_init(void)
+ {
+- int rc;
++ int rc = -ENOMEM;
+
++ mutex_lock(&damon_dbgfs_lock);
+ dbgfs_ctxs = kmalloc(sizeof(*dbgfs_ctxs), GFP_KERNEL);
+ if (!dbgfs_ctxs)
+- return -ENOMEM;
++ goto out;
+ dbgfs_ctxs[0] = dbgfs_new_ctx();
+ if (!dbgfs_ctxs[0]) {
+ kfree(dbgfs_ctxs);
+- return -ENOMEM;
++ goto out;
+ }
+ dbgfs_nr_ctxs = 1;
+
+@@ -615,6 +628,8 @@ static int __init damon_dbgfs_init(void)
+ pr_err("%s: dbgfs init failed\n", __func__);
+ }
+
++out:
++ mutex_unlock(&damon_dbgfs_lock);
+ return rc;
+ }
+
+diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c
+index 1403639302e48..718d0d3ad8c4e 100644
+--- a/mm/debug_vm_pgtable.c
++++ b/mm/debug_vm_pgtable.c
+@@ -171,6 +171,8 @@ static void __init pte_advanced_tests(struct pgtable_debug_args *args)
+ ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep);
+ pte = ptep_get(args->ptep);
+ WARN_ON(pte_young(pte));
++
++ ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);
+ }
+
+ static void __init pte_savedwrite_tests(struct pgtable_debug_args *args)
+diff --git a/mm/filemap.c b/mm/filemap.c
+index dae481293b5d9..00e391e758801 100644
+--- a/mm/filemap.c
++++ b/mm/filemap.c
+@@ -90,7 +90,7 @@
+ * ->lock_page (filemap_fault, access_process_vm)
+ *
+ * ->i_rwsem (generic_perform_write)
+- * ->mmap_lock (fault_in_pages_readable->do_page_fault)
++ * ->mmap_lock (fault_in_readable->do_page_fault)
+ *
+ * bdi->wb.list_lock
+ * sb_lock (fs/fs-writeback.c)
+@@ -2093,7 +2093,6 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
+ if (!xa_is_value(page)) {
+ if (page->index < start)
+ goto put;
+- VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
+ if (page->index + thp_nr_pages(page) - 1 > end)
+ goto put;
+ if (!trylock_page(page))
+@@ -2355,8 +2354,12 @@ static void filemap_get_read_batch(struct address_space *mapping,
+ break;
+ if (PageReadahead(head))
+ break;
+- xas.xa_index = head->index + thp_nr_pages(head) - 1;
+- xas.xa_offset = (xas.xa_index >> xas.xa_shift) & XA_CHUNK_MASK;
++ if (PageHead(head)) {
++ xas_set(&xas, head->index + thp_nr_pages(head));
++ /* Handle wrap correctly */
++ if (xas.xa_index - 1 >= max)
++ break;
++ }
+ continue;
+ put_page:
+ put_page(head);
+@@ -3757,7 +3760,7 @@ again:
+ * same page as we're writing to, without it being marked
+ * up-to-date.
+ */
+- if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
++ if (unlikely(fault_in_iov_iter_readable(i, bytes))) {
+ status = -EFAULT;
+ break;
+ }
+diff --git a/mm/gup.c b/mm/gup.c
+index 886d6148d3d03..05068d3d2557e 100644
+--- a/mm/gup.c
++++ b/mm/gup.c
+@@ -124,8 +124,8 @@ static inline struct page *try_get_compound_head(struct page *page, int refs)
+ * considered failure, and furthermore, a likely bug in the caller, so a warning
+ * is also emitted.
+ */
+-struct page *try_grab_compound_head(struct page *page,
+- int refs, unsigned int flags)
++__maybe_unused struct page *try_grab_compound_head(struct page *page,
++ int refs, unsigned int flags)
+ {
+ if (flags & FOLL_GET)
+ return try_get_compound_head(page, refs);
+@@ -208,10 +208,35 @@ static void put_compound_head(struct page *page, int refs, unsigned int flags)
+ */
+ bool __must_check try_grab_page(struct page *page, unsigned int flags)
+ {
+- if (!(flags & (FOLL_GET | FOLL_PIN)))
+- return true;
++ WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
+
+- return try_grab_compound_head(page, 1, flags);
++ if (flags & FOLL_GET)
++ return try_get_page(page);
++ else if (flags & FOLL_PIN) {
++ int refs = 1;
++
++ page = compound_head(page);
++
++ if (WARN_ON_ONCE(page_ref_count(page) <= 0))
++ return false;
++
++ if (hpage_pincount_available(page))
++ hpage_pincount_add(page, 1);
++ else
++ refs = GUP_PIN_COUNTING_BIAS;
++
++ /*
++ * Similar to try_grab_compound_head(): even if using the
++ * hpage_pincount_add/_sub() routines, be sure to
++ * *also* increment the normal page refcount field at least
++ * once, so that the page really is pinned.
++ */
++ page_ref_add(page, refs);
++
++ mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
++ }
++
++ return true;
+ }
+
+ /**
+@@ -440,7 +465,7 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
+ pte_t *pte, unsigned int flags)
+ {
+ /* No page to get reference */
+- if (flags & FOLL_GET)
++ if (flags & (FOLL_GET | FOLL_PIN))
+ return -EFAULT;
+
+ if (flags & FOLL_TOUCH) {
+@@ -918,6 +943,8 @@ static int faultin_page(struct vm_area_struct *vma,
+ /* mlock all present pages, but do not fault in new pages */
+ if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
+ return -ENOENT;
++ if (*flags & FOLL_NOFAULT)
++ return -EFAULT;
+ if (*flags & FOLL_WRITE)
+ fault_flags |= FAULT_FLAG_WRITE;
+ if (*flags & FOLL_REMOTE)
+@@ -1656,6 +1683,122 @@ finish_or_fault:
+ }
+ #endif /* !CONFIG_MMU */
+
++/**
++ * fault_in_writeable - fault in userspace address range for writing
++ * @uaddr: start of address range
++ * @size: size of address range
++ *
++ * Returns the number of bytes not faulted in (like copy_to_user() and
++ * copy_from_user()).
++ */
++size_t fault_in_writeable(char __user *uaddr, size_t size)
++{
++ char __user *start = uaddr, *end;
++
++ if (unlikely(size == 0))
++ return 0;
++ if (!PAGE_ALIGNED(uaddr)) {
++ if (unlikely(__put_user(0, uaddr) != 0))
++ return size;
++ uaddr = (char __user *)PAGE_ALIGN((unsigned long)uaddr);
++ }
++ end = (char __user *)PAGE_ALIGN((unsigned long)start + size);
++ if (unlikely(end < start))
++ end = NULL;
++ while (uaddr != end) {
++ if (unlikely(__put_user(0, uaddr) != 0))
++ goto out;
++ uaddr += PAGE_SIZE;
++ }
++
++out:
++ if (size > uaddr - start)
++ return size - (uaddr - start);
++ return 0;
++}
++EXPORT_SYMBOL(fault_in_writeable);
++
++/*
++ * fault_in_safe_writeable - fault in an address range for writing
++ * @uaddr: start of address range
++ * @size: length of address range
++ *
++ * Faults in an address range for writing. This is primarily useful when we
++ * already know that some or all of the pages in the address range aren't in
++ * memory.
++ *
++ * Unlike fault_in_writeable(), this function is non-destructive.
++ *
++ * Note that we don't pin or otherwise hold the pages referenced that we fault
++ * in. There's no guarantee that they'll stay in memory for any duration of
++ * time.
++ *
++ * Returns the number of bytes not faulted in, like copy_to_user() and
++ * copy_from_user().
++ */
++size_t fault_in_safe_writeable(const char __user *uaddr, size_t size)
++{
++ unsigned long start = (unsigned long)uaddr, end;
++ struct mm_struct *mm = current->mm;
++ bool unlocked = false;
++
++ if (unlikely(size == 0))
++ return 0;
++ end = PAGE_ALIGN(start + size);
++ if (end < start)
++ end = 0;
++
++ mmap_read_lock(mm);
++ do {
++ if (fixup_user_fault(mm, start, FAULT_FLAG_WRITE, &unlocked))
++ break;
++ start = (start + PAGE_SIZE) & PAGE_MASK;
++ } while (start != end);
++ mmap_read_unlock(mm);
++
++ if (size > (unsigned long)uaddr - start)
++ return size - ((unsigned long)uaddr - start);
++ return 0;
++}
++EXPORT_SYMBOL(fault_in_safe_writeable);
++
++/**
++ * fault_in_readable - fault in userspace address range for reading
++ * @uaddr: start of user address range
++ * @size: size of user address range
++ *
++ * Returns the number of bytes not faulted in (like copy_to_user() and
++ * copy_from_user()).
++ */
++size_t fault_in_readable(const char __user *uaddr, size_t size)
++{
++ const char __user *start = uaddr, *end;
++ volatile char c;
++
++ if (unlikely(size == 0))
++ return 0;
++ if (!PAGE_ALIGNED(uaddr)) {
++ if (unlikely(__get_user(c, uaddr) != 0))
++ return size;
++ uaddr = (const char __user *)PAGE_ALIGN((unsigned long)uaddr);
++ }
++ end = (const char __user *)PAGE_ALIGN((unsigned long)start + size);
++ if (unlikely(end < start))
++ end = NULL;
++ while (uaddr != end) {
++ if (unlikely(__get_user(c, uaddr) != 0))
++ goto out;
++ uaddr += PAGE_SIZE;
++ }
++
++out:
++ (void)c;
++ if (size > uaddr - start)
++ return size - (uaddr - start);
++ return 0;
++}
++EXPORT_SYMBOL(fault_in_readable);
++
+ /**
+ * get_dump_page() - pin user page in memory while writing it to core dump
+ * @addr: user address
+@@ -2708,7 +2851,7 @@ static int internal_get_user_pages_fast(unsigned long start,
+
+ if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
+ FOLL_FORCE | FOLL_PIN | FOLL_GET |
+- FOLL_FAST_ONLY)))
++ FOLL_FAST_ONLY | FOLL_NOFAULT)))
+ return -EINVAL;
+
+ if (gup_flags & FOLL_PIN)
+diff --git a/mm/highmem.c b/mm/highmem.c
+index 4212ad0e4a195..4f942678e9da2 100644
+--- a/mm/highmem.c
++++ b/mm/highmem.c
+@@ -504,16 +504,22 @@ static inline int kmap_local_calc_idx(int idx)
+
+ static pte_t *__kmap_pte;
+
+-static pte_t *kmap_get_pte(void)
++static pte_t *kmap_get_pte(unsigned long vaddr, int idx)
+ {
++ if (IS_ENABLED(CONFIG_KMAP_LOCAL_NON_LINEAR_PTE_ARRAY))
++ /*
++ * Set by the arch if __kmap_pte[-idx] does not produce
++ * the correct entry.
++ */
++ return virt_to_kpte(vaddr);
+ if (!__kmap_pte)
+ __kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
+- return __kmap_pte;
++ return &__kmap_pte[-idx];
+ }
+
+ void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot)
+ {
+- pte_t pteval, *kmap_pte = kmap_get_pte();
++ pte_t pteval, *kmap_pte;
+ unsigned long vaddr;
+ int idx;
+
+@@ -525,9 +531,10 @@ void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot)
+ preempt_disable();
+ idx = arch_kmap_local_map_idx(kmap_local_idx_push(), pfn);
+ vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+- BUG_ON(!pte_none(*(kmap_pte - idx)));
++ kmap_pte = kmap_get_pte(vaddr, idx);
++ BUG_ON(!pte_none(*kmap_pte));
+ pteval = pfn_pte(pfn, prot);
+- arch_kmap_local_set_pte(&init_mm, vaddr, kmap_pte - idx, pteval);
++ arch_kmap_local_set_pte(&init_mm, vaddr, kmap_pte, pteval);
+ arch_kmap_local_post_map(vaddr, pteval);
+ current->kmap_ctrl.pteval[kmap_local_idx()] = pteval;
+ preempt_enable();
+@@ -560,7 +567,7 @@ EXPORT_SYMBOL(__kmap_local_page_prot);
+ void kunmap_local_indexed(void *vaddr)
+ {
+ unsigned long addr = (unsigned long) vaddr & PAGE_MASK;
+- pte_t *kmap_pte = kmap_get_pte();
++ pte_t *kmap_pte;
+ int idx;
+
+ if (addr < __fix_to_virt(FIX_KMAP_END) ||
+@@ -585,8 +592,9 @@ void kunmap_local_indexed(void *vaddr)
+ idx = arch_kmap_local_unmap_idx(kmap_local_idx(), addr);
+ WARN_ON_ONCE(addr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
+
++ kmap_pte = kmap_get_pte(addr, idx);
+ arch_kmap_local_pre_unmap(addr);
+- pte_clear(&init_mm, addr, kmap_pte - idx);
++ pte_clear(&init_mm, addr, kmap_pte);
+ arch_kmap_local_post_unmap(addr);
+ current->kmap_ctrl.pteval[kmap_local_idx()] = __pte(0);
+ kmap_local_idx_pop();
+@@ -608,7 +616,7 @@ EXPORT_SYMBOL(kunmap_local_indexed);
+ void __kmap_local_sched_out(void)
+ {
+ struct task_struct *tsk = current;
+- pte_t *kmap_pte = kmap_get_pte();
++ pte_t *kmap_pte;
+ int i;
+
+ /* Clear kmaps */
+@@ -619,7 +627,7 @@ void __kmap_local_sched_out(void)
+
+ /* With debug all even slots are unmapped and act as guard */
+ if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL) && !(i & 0x01)) {
+- WARN_ON_ONCE(!pte_none(pteval));
++ WARN_ON_ONCE(pte_val(pteval) != 0);
+ continue;
+ }
+ if (WARN_ON_ONCE(pte_none(pteval)))
+@@ -635,8 +643,9 @@ void __kmap_local_sched_out(void)
+ idx = arch_kmap_local_map_idx(i, pte_pfn(pteval));
+
+ addr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
++ kmap_pte = kmap_get_pte(addr, idx);
+ arch_kmap_local_pre_unmap(addr);
+- pte_clear(&init_mm, addr, kmap_pte - idx);
++ pte_clear(&init_mm, addr, kmap_pte);
+ arch_kmap_local_post_unmap(addr);
+ }
+ }
+@@ -644,7 +653,7 @@ void __kmap_local_sched_out(void)
+ void __kmap_local_sched_in(void)
+ {
+ struct task_struct *tsk = current;
+- pte_t *kmap_pte = kmap_get_pte();
++ pte_t *kmap_pte;
+ int i;
+
+ /* Restore kmaps */
+@@ -655,7 +664,7 @@ void __kmap_local_sched_in(void)
+
+ /* With debug all even slots are unmapped and act as guard */
+ if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL) && !(i & 0x01)) {
+- WARN_ON_ONCE(!pte_none(pteval));
++ WARN_ON_ONCE(pte_val(pteval) != 0);
+ continue;
+ }
+ if (WARN_ON_ONCE(pte_none(pteval)))
+@@ -664,7 +673,8 @@ void __kmap_local_sched_in(void)
+ /* See comment in __kmap_local_sched_out() */
+ idx = arch_kmap_local_map_idx(i, pte_pfn(pteval));
+ addr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+- set_pte_at(&init_mm, addr, kmap_pte - idx, pteval);
++ kmap_pte = kmap_get_pte(addr, idx);
++ set_pte_at(&init_mm, addr, kmap_pte, pteval);
+ arch_kmap_local_post_map(addr, pteval);
+ }
+ }
+diff --git a/mm/hmm.c b/mm/hmm.c
+index 842e265992380..bd56641c79d4e 100644
+--- a/mm/hmm.c
++++ b/mm/hmm.c
+@@ -300,7 +300,8 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
+ * Since each architecture defines a struct page for the zero page, just
+ * fall through and treat it like a normal page.
+ */
+- if (pte_special(pte) && !pte_devmap(pte) &&
++ if (!vm_normal_page(walk->vma, addr, pte) &&
++ !pte_devmap(pte) &&
+ !is_zero_pfn(pte_pfn(pte))) {
+ if (hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0)) {
+ pte_unmap(ptep);
+@@ -518,7 +519,7 @@ static int hmm_vma_walk_test(unsigned long start, unsigned long end,
+ struct hmm_range *range = hmm_vma_walk->range;
+ struct vm_area_struct *vma = walk->vma;
+
+- if (!(vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP)) &&
++ if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) &&
+ vma->vm_flags & VM_READ)
+ return 0;
+
+diff --git a/mm/huge_memory.c b/mm/huge_memory.c
+index c5142d237e482..8cc150a883615 100644
+--- a/mm/huge_memory.c
++++ b/mm/huge_memory.c
+@@ -2617,11 +2617,16 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
+ struct address_space *mapping = NULL;
+ int extra_pins, ret;
+ pgoff_t end;
++ bool is_hzp;
+
+- VM_BUG_ON_PAGE(is_huge_zero_page(head), head);
+ VM_BUG_ON_PAGE(!PageLocked(head), head);
+ VM_BUG_ON_PAGE(!PageCompound(head), head);
+
++ is_hzp = is_huge_zero_page(head);
++ VM_WARN_ON_ONCE_PAGE(is_hzp, head);
++ if (is_hzp)
++ return -EBUSY;
++
+ if (PageWriteback(head))
+ return -EBUSY;
+
+diff --git a/mm/hugetlb.c b/mm/hugetlb.c
+index 95dc7b83381f9..f5ed98843557b 100644
+--- a/mm/hugetlb.c
++++ b/mm/hugetlb.c
+@@ -4439,6 +4439,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
+ struct hstate *h = hstate_vma(vma);
+ unsigned long sz = huge_page_size(h);
+ struct mmu_notifier_range range;
++ bool force_flush = false;
+
+ WARN_ON(!is_vm_hugetlb_page(vma));
+ BUG_ON(start & ~huge_page_mask(h));
+@@ -4467,10 +4468,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
+ ptl = huge_pte_lock(h, mm, ptep);
+ if (huge_pmd_unshare(mm, vma, &address, ptep)) {
+ spin_unlock(ptl);
+- /*
+- * We just unmapped a page of PMDs by clearing a PUD.
+- * The caller's TLB flush range should cover this area.
+- */
++ tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
++ force_flush = true;
+ continue;
+ }
+
+@@ -4527,6 +4526,22 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
+ }
+ mmu_notifier_invalidate_range_end(&range);
+ tlb_end_vma(tlb, vma);
++
++ /*
++ * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
++ * could defer the flush until now, since by holding i_mmap_rwsem we
++ * guaranteed that the last refernece would not be dropped. But we must
++ * do the flushing before we return, as otherwise i_mmap_rwsem will be
++ * dropped and the last reference to the shared PMDs page might be
++ * dropped as well.
++ *
++ * In theory we could defer the freeing of the PMD pages as well, but
++ * huge_pmd_unshare() relies on the exact page_count for the PMD page to
++ * detect sharing, so we cannot defer the release of the page either.
++ * Instead, do flush now.
++ */
++ if (force_flush)
++ tlb_flush_mmu_tlbonly(tlb);
+ }
+
+ void __unmap_hugepage_range_final(struct mmu_gather *tlb,
+@@ -5236,13 +5251,14 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
+ int ret = -ENOMEM;
+ struct page *page;
+ int writable;
+- bool new_pagecache_page = false;
++ bool page_in_pagecache = false;
+
+ if (is_continue) {
+ ret = -EFAULT;
+ page = find_lock_page(mapping, idx);
+ if (!page)
+ goto out;
++ page_in_pagecache = true;
+ } else if (!*pagep) {
+ /* If a page already exists, then it's UFFDIO_COPY for
+ * a non-missing case. Return -EEXIST.
+@@ -5330,7 +5346,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
+ ret = huge_add_to_page_cache(page, mapping, idx);
+ if (ret)
+ goto out_release_nounlock;
+- new_pagecache_page = true;
++ page_in_pagecache = true;
+ }
+
+ ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
+@@ -5394,7 +5410,7 @@ out_release_unlock:
+ if (vm_shared || is_continue)
+ unlock_page(page);
+ out_release_nounlock:
+- if (!new_pagecache_page)
++ if (!page_in_pagecache)
+ restore_reserve_on_error(h, dst_vma, dst_addr, page);
+ put_page(page);
+ goto out;
+diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c
+index d8ccff4c1275e..1bd6a3f13467b 100644
+--- a/mm/kasan/quarantine.c
++++ b/mm/kasan/quarantine.c
+@@ -132,11 +132,22 @@ static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache)
+ static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache)
+ {
+ void *object = qlink_to_object(qlink, cache);
++ struct kasan_free_meta *meta = kasan_get_free_meta(cache, object);
+ unsigned long flags;
+
+ if (IS_ENABLED(CONFIG_SLAB))
+ local_irq_save(flags);
+
++ /*
++ * If init_on_free is enabled and KASAN's free metadata is stored in
++ * the object, zero the metadata. Otherwise, the object's memory will
++ * not be properly zeroed, as KASAN saves the metadata after the slab
++ * allocator zeroes the object.
++ */
++ if (slab_want_init_on_free(cache) &&
++ cache->kasan_info.free_meta_offset == 0)
++ memzero_explicit(meta, sizeof(*meta));
++
+ /*
+ * As the object now gets freed from the quarantine, assume that its
+ * free track is no longer valid.
+@@ -304,6 +315,13 @@ static void per_cpu_remove_cache(void *arg)
+ struct qlist_head *q;
+
+ q = this_cpu_ptr(&cpu_quarantine);
++ /*
++ * Ensure the ordering between the writing to q->offline and
++ * per_cpu_remove_cache. Prevent cpu_quarantine from being corrupted
++ * by interrupt.
++ */
++ if (READ_ONCE(q->offline))
++ return;
+ qlist_move_cache(q, &to_free, cache);
+ qlist_free_all(&to_free, cache);
+ }
+diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c
+index 8d95ee52d0194..dd79840e60964 100644
+--- a/mm/kasan/shadow.c
++++ b/mm/kasan/shadow.c
+@@ -493,7 +493,7 @@ void kasan_release_vmalloc(unsigned long start, unsigned long end,
+
+ #else /* CONFIG_KASAN_VMALLOC */
+
+-int kasan_module_alloc(void *addr, size_t size)
++int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask)
+ {
+ void *ret;
+ size_t scaled_size;
+@@ -515,9 +515,14 @@ int kasan_module_alloc(void *addr, size_t size)
+ __builtin_return_address(0));
+
+ if (ret) {
++ struct vm_struct *vm = find_vm_area(addr);
+ __memset(ret, KASAN_SHADOW_INIT, shadow_size);
+- find_vm_area(addr)->flags |= VM_KASAN;
++ vm->flags |= VM_KASAN;
+ kmemleak_ignore(ret);
++
++ if (vm->flags & VM_DEFER_KMEMLEAK)
++ kmemleak_vmalloc(vm, size, gfp_mask);
++
+ return 0;
+ }
+
+diff --git a/mm/kfence/core.c b/mm/kfence/core.c
+index 7a97db8bc8e75..d25202766fbba 100644
+--- a/mm/kfence/core.c
++++ b/mm/kfence/core.c
+@@ -10,12 +10,15 @@
+ #include <linux/atomic.h>
+ #include <linux/bug.h>
+ #include <linux/debugfs.h>
++#include <linux/hash.h>
+ #include <linux/irq_work.h>
++#include <linux/jhash.h>
+ #include <linux/kcsan-checks.h>
+ #include <linux/kfence.h>
+ #include <linux/kmemleak.h>
+ #include <linux/list.h>
+ #include <linux/lockdep.h>
++#include <linux/log2.h>
+ #include <linux/memblock.h>
+ #include <linux/moduleparam.h>
+ #include <linux/random.h>
+@@ -82,6 +85,10 @@ static const struct kernel_param_ops sample_interval_param_ops = {
+ };
+ module_param_cb(sample_interval, &sample_interval_param_ops, &kfence_sample_interval, 0600);
+
++/* Pool usage% threshold when currently covered allocations are skipped. */
++static unsigned long kfence_skip_covered_thresh __read_mostly = 75;
++module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong, 0644);
++
+ /* The pool of pages used for guard pages and objects. */
+ char *__kfence_pool __ro_after_init;
+ EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */
+@@ -97,14 +104,41 @@ struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];
+ static struct list_head kfence_freelist = LIST_HEAD_INIT(kfence_freelist);
+ static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */
+
+-#ifdef CONFIG_KFENCE_STATIC_KEYS
+-/* The static key to set up a KFENCE allocation. */
++/*
++ * The static key to set up a KFENCE allocation; or if static keys are not used
++ * to gate allocations, to avoid a load and compare if KFENCE is disabled.
++ */
+ DEFINE_STATIC_KEY_FALSE(kfence_allocation_key);
+-#endif
+
+ /* Gates the allocation, ensuring only one succeeds in a given period. */
+ atomic_t kfence_allocation_gate = ATOMIC_INIT(1);
+
++/*
++ * A Counting Bloom filter of allocation coverage: limits currently covered
++ * allocations of the same source filling up the pool.
++ *
++ * Assuming a range of 15%-85% unique allocations in the pool at any point in
++ * time, the below parameters provide a probablity of 0.02-0.33 for false
++ * positive hits respectively:
++ *
++ * P(alloc_traces) = (1 - e^(-HNUM * (alloc_traces / SIZE)) ^ HNUM
++ */
++#define ALLOC_COVERED_HNUM 2
++#define ALLOC_COVERED_ORDER (const_ilog2(CONFIG_KFENCE_NUM_OBJECTS) + 2)
++#define ALLOC_COVERED_SIZE (1 << ALLOC_COVERED_ORDER)
++#define ALLOC_COVERED_HNEXT(h) hash_32(h, ALLOC_COVERED_ORDER)
++#define ALLOC_COVERED_MASK (ALLOC_COVERED_SIZE - 1)
++static atomic_t alloc_covered[ALLOC_COVERED_SIZE];
++
++/* Stack depth used to determine uniqueness of an allocation. */
++#define UNIQUE_ALLOC_STACK_DEPTH ((size_t)8)
++
++/*
++ * Randomness for stack hashes, making the same collisions across reboots and
++ * different machines less likely.
++ */
++static u32 stack_hash_seed __ro_after_init;
++
+ /* Statistics counters for debugfs. */
+ enum kfence_counter_id {
+ KFENCE_COUNTER_ALLOCATED,
+@@ -112,6 +146,9 @@ enum kfence_counter_id {
+ KFENCE_COUNTER_FREES,
+ KFENCE_COUNTER_ZOMBIES,
+ KFENCE_COUNTER_BUGS,
++ KFENCE_COUNTER_SKIP_INCOMPAT,
++ KFENCE_COUNTER_SKIP_CAPACITY,
++ KFENCE_COUNTER_SKIP_COVERED,
+ KFENCE_COUNTER_COUNT,
+ };
+ static atomic_long_t counters[KFENCE_COUNTER_COUNT];
+@@ -121,40 +158,67 @@ static const char *const counter_names[] = {
+ [KFENCE_COUNTER_FREES] = "total frees",
+ [KFENCE_COUNTER_ZOMBIES] = "zombie allocations",
+ [KFENCE_COUNTER_BUGS] = "total bugs",
++ [KFENCE_COUNTER_SKIP_INCOMPAT] = "skipped allocations (incompatible)",
++ [KFENCE_COUNTER_SKIP_CAPACITY] = "skipped allocations (capacity)",
++ [KFENCE_COUNTER_SKIP_COVERED] = "skipped allocations (covered)",
+ };
+ static_assert(ARRAY_SIZE(counter_names) == KFENCE_COUNTER_COUNT);
+
+ /* === Internals ============================================================ */
+
+-static bool kfence_protect(unsigned long addr)
++static inline bool should_skip_covered(void)
+ {
+- return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), true));
++ unsigned long thresh = (CONFIG_KFENCE_NUM_OBJECTS * kfence_skip_covered_thresh) / 100;
++
++ return atomic_long_read(&counters[KFENCE_COUNTER_ALLOCATED]) > thresh;
+ }
+
+-static bool kfence_unprotect(unsigned long addr)
++static u32 get_alloc_stack_hash(unsigned long *stack_entries, size_t num_entries)
+ {
+- return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), false));
++ num_entries = min(num_entries, UNIQUE_ALLOC_STACK_DEPTH);
++ num_entries = filter_irq_stacks(stack_entries, num_entries);
++ return jhash(stack_entries, num_entries * sizeof(stack_entries[0]), stack_hash_seed);
+ }
+
+-static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
++/*
++ * Adds (or subtracts) count @val for allocation stack trace hash
++ * @alloc_stack_hash from Counting Bloom filter.
++ */
++static void alloc_covered_add(u32 alloc_stack_hash, int val)
+ {
+- long index;
++ int i;
+
+- /* The checks do not affect performance; only called from slow-paths. */
++ for (i = 0; i < ALLOC_COVERED_HNUM; i++) {
++ atomic_add(val, &alloc_covered[alloc_stack_hash & ALLOC_COVERED_MASK]);
++ alloc_stack_hash = ALLOC_COVERED_HNEXT(alloc_stack_hash);
++ }
++}
+
+- if (!is_kfence_address((void *)addr))
+- return NULL;
++/*
++ * Returns true if the allocation stack trace hash @alloc_stack_hash is
++ * currently contained (non-zero count) in Counting Bloom filter.
++ */
++static bool alloc_covered_contains(u32 alloc_stack_hash)
++{
++ int i;
+
+- /*
+- * May be an invalid index if called with an address at the edge of
+- * __kfence_pool, in which case we would report an "invalid access"
+- * error.
+- */
+- index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
+- if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
+- return NULL;
++ for (i = 0; i < ALLOC_COVERED_HNUM; i++) {
++ if (!atomic_read(&alloc_covered[alloc_stack_hash & ALLOC_COVERED_MASK]))
++ return false;
++ alloc_stack_hash = ALLOC_COVERED_HNEXT(alloc_stack_hash);
++ }
+
+- return &kfence_metadata[index];
++ return true;
++}
++
++static bool kfence_protect(unsigned long addr)
++{
++ return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), true));
++}
++
++static bool kfence_unprotect(unsigned long addr)
++{
++ return !KFENCE_WARN_ON(!kfence_protect_page(ALIGN_DOWN(addr, PAGE_SIZE), false));
+ }
+
+ static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta)
+@@ -183,19 +247,26 @@ static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *m
+ * Update the object's metadata state, including updating the alloc/free stacks
+ * depending on the state transition.
+ */
+-static noinline void metadata_update_state(struct kfence_metadata *meta,
+- enum kfence_object_state next)
++static noinline void
++metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next,
++ unsigned long *stack_entries, size_t num_stack_entries)
+ {
+ struct kfence_track *track =
+ next == KFENCE_OBJECT_FREED ? &meta->free_track : &meta->alloc_track;
+
+ lockdep_assert_held(&meta->lock);
+
+- /*
+- * Skip over 1 (this) functions; noinline ensures we do not accidentally
+- * skip over the caller by never inlining.
+- */
+- track->num_stack_entries = stack_trace_save(track->stack_entries, KFENCE_STACK_DEPTH, 1);
++ if (stack_entries) {
++ memcpy(track->stack_entries, stack_entries,
++ num_stack_entries * sizeof(stack_entries[0]));
++ } else {
++ /*
++ * Skip over 1 (this) functions; noinline ensures we do not
++ * accidentally skip over the caller by never inlining.
++ */
++ num_stack_entries = stack_trace_save(track->stack_entries, KFENCE_STACK_DEPTH, 1);
++ }
++ track->num_stack_entries = num_stack_entries;
+ track->pid = task_pid_nr(current);
+ track->cpu = raw_smp_processor_id();
+ track->ts_nsec = local_clock(); /* Same source as printk timestamps. */
+@@ -257,7 +328,9 @@ static __always_inline void for_each_canary(const struct kfence_metadata *meta,
+ }
+ }
+
+-static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t gfp)
++static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t gfp,
++ unsigned long *stack_entries, size_t num_stack_entries,
++ u32 alloc_stack_hash)
+ {
+ struct kfence_metadata *meta = NULL;
+ unsigned long flags;
+@@ -271,8 +344,10 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
+ list_del_init(&meta->list);
+ }
+ raw_spin_unlock_irqrestore(&kfence_freelist_lock, flags);
+- if (!meta)
++ if (!meta) {
++ atomic_long_inc(&counters[KFENCE_COUNTER_SKIP_CAPACITY]);
+ return NULL;
++ }
+
+ if (unlikely(!raw_spin_trylock_irqsave(&meta->lock, flags))) {
+ /*
+@@ -314,10 +389,12 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
+ addr = (void *)meta->addr;
+
+ /* Update remaining metadata. */
+- metadata_update_state(meta, KFENCE_OBJECT_ALLOCATED);
++ metadata_update_state(meta, KFENCE_OBJECT_ALLOCATED, stack_entries, num_stack_entries);
+ /* Pairs with READ_ONCE() in kfence_shutdown_cache(). */
+ WRITE_ONCE(meta->cache, cache);
+ meta->size = size;
++ meta->alloc_stack_hash = alloc_stack_hash;
++
+ for_each_canary(meta, set_canary_byte);
+
+ /* Set required struct page fields. */
+@@ -330,6 +407,8 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
+
+ raw_spin_unlock_irqrestore(&meta->lock, flags);
+
++ alloc_covered_add(alloc_stack_hash, 1);
++
+ /* Memory initialization. */
+
+ /*
+@@ -394,10 +473,12 @@ static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool z
+ memzero_explicit(addr, meta->size);
+
+ /* Mark the object as freed. */
+- metadata_update_state(meta, KFENCE_OBJECT_FREED);
++ metadata_update_state(meta, KFENCE_OBJECT_FREED, NULL, 0);
+
+ raw_spin_unlock_irqrestore(&meta->lock, flags);
+
++ alloc_covered_add(meta->alloc_stack_hash, -1);
++
+ /* Protect to detect use-after-frees. */
+ kfence_protect((unsigned long)addr);
+
+@@ -429,6 +510,7 @@ static bool __init kfence_init_pool(void)
+ unsigned long addr = (unsigned long)__kfence_pool;
+ struct page *pages;
+ int i;
++ char *p;
+
+ if (!__kfence_pool)
+ return false;
+@@ -447,6 +529,8 @@ static bool __init kfence_init_pool(void)
+ * enters __slab_free() slow-path.
+ */
+ for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
++ struct page *page = &pages[i];
++
+ if (!i || (i % 2))
+ continue;
+
+@@ -454,7 +538,11 @@ static bool __init kfence_init_pool(void)
+ if (WARN_ON(compound_head(&pages[i]) != &pages[i]))
+ goto err;
+
+- __SetPageSlab(&pages[i]);
++ __SetPageSlab(page);
++#ifdef CONFIG_MEMCG
++ page->memcg_data = (unsigned long)&kfence_metadata[i / 2 - 1].objcg |
++ MEMCG_DATA_OBJCGS;
++#endif
+ }
+
+ /*
+@@ -505,6 +593,16 @@ err:
+ * fails for the first page, and therefore expect addr==__kfence_pool in
+ * most failure cases.
+ */
++ for (p = (char *)addr; p < __kfence_pool + KFENCE_POOL_SIZE; p += PAGE_SIZE) {
++ struct page *page = virt_to_page(p);
++
++ if (!PageSlab(page))
++ continue;
++#ifdef CONFIG_MEMCG
++ page->memcg_data = 0;
++#endif
++ __ClearPageSlab(page);
++ }
+ memblock_free_late(__pa(addr), KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool));
+ __kfence_pool = NULL;
+ return false;
+@@ -577,6 +675,7 @@ static const struct file_operations objects_fops = {
+ .open = open_objects,
+ .read = seq_read,
+ .llseek = seq_lseek,
++ .release = seq_release,
+ };
+
+ static int __init kfence_debugfs_init(void)
+@@ -663,11 +762,14 @@ void __init kfence_init(void)
+ if (!kfence_sample_interval)
+ return;
+
++ stack_hash_seed = (u32)random_get_entropy();
+ if (!kfence_init_pool()) {
+ pr_err("%s failed\n", __func__);
+ return;
+ }
+
++ if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS))
++ static_branch_enable(&kfence_allocation_key);
+ WRITE_ONCE(kfence_enabled, true);
+ queue_delayed_work(system_unbound_wq, &kfence_timer, 0);
+ pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE,
+@@ -736,12 +838,18 @@ void kfence_shutdown_cache(struct kmem_cache *s)
+
+ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
+ {
++ unsigned long stack_entries[KFENCE_STACK_DEPTH];
++ size_t num_stack_entries;
++ u32 alloc_stack_hash;
++
+ /*
+ * Perform size check before switching kfence_allocation_gate, so that
+ * we don't disable KFENCE without making an allocation.
+ */
+- if (size > PAGE_SIZE)
++ if (size > PAGE_SIZE) {
++ atomic_long_inc(&counters[KFENCE_COUNTER_SKIP_INCOMPAT]);
+ return NULL;
++ }
+
+ /*
+ * Skip allocations from non-default zones, including DMA. We cannot
+@@ -749,15 +857,12 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
+ * properties (e.g. reside in DMAable memory).
+ */
+ if ((flags & GFP_ZONEMASK) ||
+- (s->flags & (SLAB_CACHE_DMA | SLAB_CACHE_DMA32)))
++ (s->flags & (SLAB_CACHE_DMA | SLAB_CACHE_DMA32))) {
++ atomic_long_inc(&counters[KFENCE_COUNTER_SKIP_INCOMPAT]);
+ return NULL;
++ }
+
+- /*
+- * allocation_gate only needs to become non-zero, so it doesn't make
+- * sense to continue writing to it and pay the associated contention
+- * cost, in case we have a large number of concurrent allocations.
+- */
+- if (atomic_read(&kfence_allocation_gate) || atomic_inc_return(&kfence_allocation_gate) > 1)
++ if (atomic_inc_return(&kfence_allocation_gate) > 1)
+ return NULL;
+ #ifdef CONFIG_KFENCE_STATIC_KEYS
+ /*
+@@ -776,7 +881,25 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
+ if (!READ_ONCE(kfence_enabled))
+ return NULL;
+
+- return kfence_guarded_alloc(s, size, flags);
++ num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 0);
++
++ /*
++ * Do expensive check for coverage of allocation in slow-path after
++ * allocation_gate has already become non-zero, even though it might
++ * mean not making any allocation within a given sample interval.
++ *
++ * This ensures reasonable allocation coverage when the pool is almost
++ * full, including avoiding long-lived allocations of the same source
++ * filling up the pool (e.g. pagecache allocations).
++ */
++ alloc_stack_hash = get_alloc_stack_hash(stack_entries, num_stack_entries);
++ if (should_skip_covered() && alloc_covered_contains(alloc_stack_hash)) {
++ atomic_long_inc(&counters[KFENCE_COUNTER_SKIP_COVERED]);
++ return NULL;
++ }
++
++ return kfence_guarded_alloc(s, size, flags, stack_entries, num_stack_entries,
++ alloc_stack_hash);
+ }
+
+ size_t kfence_ksize(const void *addr)
+@@ -805,6 +928,9 @@ void __kfence_free(void *addr)
+ {
+ struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);
+
++#ifdef CONFIG_MEMCG
++ KFENCE_WARN_ON(meta->objcg);
++#endif
+ /*
+ * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing
+ * the object, as the object page may be recycled for other-typed
+diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h
+index c1f23c61e5f91..600f2e2431d6d 100644
+--- a/mm/kfence/kfence.h
++++ b/mm/kfence/kfence.h
+@@ -87,10 +87,36 @@ struct kfence_metadata {
+ /* Allocation and free stack information. */
+ struct kfence_track alloc_track;
+ struct kfence_track free_track;
++ /* For updating alloc_covered on frees. */
++ u32 alloc_stack_hash;
++#ifdef CONFIG_MEMCG
++ struct obj_cgroup *objcg;
++#endif
+ };
+
+ extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];
+
++static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
++{
++ long index;
++
++ /* The checks do not affect performance; only called from slow-paths. */
++
++ if (!is_kfence_address((void *)addr))
++ return NULL;
++
++ /*
++ * May be an invalid index if called with an address at the edge of
++ * __kfence_pool, in which case we would report an "invalid access"
++ * error.
++ */
++ index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
++ if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
++ return NULL;
++
++ return &kfence_metadata[index];
++}
++
+ /* KFENCE error types for report generation. */
+ enum kfence_error_type {
+ KFENCE_ERROR_OOB, /* Detected a out-of-bounds access. */
+diff --git a/mm/kfence/report.c b/mm/kfence/report.c
+index f93a7b2a338be..37e140e7f201e 100644
+--- a/mm/kfence/report.c
++++ b/mm/kfence/report.c
+@@ -273,3 +273,50 @@ void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *r
+ /* We encountered a memory safety error, taint the kernel! */
+ add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);
+ }
++
++#ifdef CONFIG_PRINTK
++static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)
++{
++ int i, j;
++
++ i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
++ for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)
++ kp_stack[j] = (void *)track->stack_entries[i];
++ if (j < KS_ADDRS_COUNT)
++ kp_stack[j] = NULL;
++}
++
++bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++{
++ struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);
++ unsigned long flags;
++
++ if (!meta)
++ return false;
++
++ /*
++ * If state is UNUSED at least show the pointer requested; the rest
++ * would be garbage data.
++ */
++ kpp->kp_ptr = object;
++
++ /* Requesting info an a never-used object is almost certainly a bug. */
++ if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))
++ return true;
++
++ raw_spin_lock_irqsave(&meta->lock, flags);
++
++ kpp->kp_page = page;
++ kpp->kp_slab_cache = meta->cache;
++ kpp->kp_objp = (void *)meta->addr;
++ kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);
++ if (meta->state == KFENCE_OBJECT_FREED)
++ kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);
++ /* get_stack_skipnr() ensures the first entry is outside allocator. */
++ kpp->kp_ret = kpp->kp_stack[0];
++
++ raw_spin_unlock_irqrestore(&meta->lock, flags);
++
++ return true;
++}
++#endif
+diff --git a/mm/kmemleak.c b/mm/kmemleak.c
+index b57383c17cf60..859303aae1809 100644
+--- a/mm/kmemleak.c
++++ b/mm/kmemleak.c
+@@ -789,6 +789,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
+ unsigned long flags;
+ struct kmemleak_object *object;
+ struct kmemleak_scan_area *area = NULL;
++ unsigned long untagged_ptr;
++ unsigned long untagged_objp;
+
+ object = find_and_get_object(ptr, 1);
+ if (!object) {
+@@ -797,6 +799,9 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
+ return;
+ }
+
++ untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
++ untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
++
+ if (scan_area_cache)
+ area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
+
+@@ -808,8 +813,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
+ goto out_unlock;
+ }
+ if (size == SIZE_MAX) {
+- size = object->pointer + object->size - ptr;
+- } else if (ptr + size > object->pointer + object->size) {
++ size = untagged_objp + object->size - untagged_ptr;
++ } else if (untagged_ptr + size > untagged_objp + object->size) {
+ kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
+ dump_object_info(object);
+ kmem_cache_free(scan_area_cache, area);
+@@ -1120,7 +1125,7 @@ EXPORT_SYMBOL(kmemleak_no_scan);
+ void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
+ gfp_t gfp)
+ {
+- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
++ if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
+ kmemleak_alloc(__va(phys), size, min_count, gfp);
+ }
+ EXPORT_SYMBOL(kmemleak_alloc_phys);
+@@ -1134,7 +1139,7 @@ EXPORT_SYMBOL(kmemleak_alloc_phys);
+ */
+ void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
+ {
+- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
++ if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
+ kmemleak_free_part(__va(phys), size);
+ }
+ EXPORT_SYMBOL(kmemleak_free_part_phys);
+@@ -1146,7 +1151,7 @@ EXPORT_SYMBOL(kmemleak_free_part_phys);
+ */
+ void __ref kmemleak_not_leak_phys(phys_addr_t phys)
+ {
+- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
++ if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
+ kmemleak_not_leak(__va(phys));
+ }
+ EXPORT_SYMBOL(kmemleak_not_leak_phys);
+@@ -1158,7 +1163,7 @@ EXPORT_SYMBOL(kmemleak_not_leak_phys);
+ */
+ void __ref kmemleak_ignore_phys(phys_addr_t phys)
+ {
+- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
++ if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
+ kmemleak_ignore(__va(phys));
+ }
+ EXPORT_SYMBOL(kmemleak_ignore_phys);
+@@ -1403,7 +1408,8 @@ static void kmemleak_scan(void)
+ {
+ unsigned long flags;
+ struct kmemleak_object *object;
+- int i;
++ struct zone *zone;
++ int __maybe_unused i;
+ int new_leaks = 0;
+
+ jiffies_last_scan = jiffies;
+@@ -1443,9 +1449,9 @@ static void kmemleak_scan(void)
+ * Struct page scanning for each node.
+ */
+ get_online_mems();
+- for_each_online_node(i) {
+- unsigned long start_pfn = node_start_pfn(i);
+- unsigned long end_pfn = node_end_pfn(i);
++ for_each_populated_zone(zone) {
++ unsigned long start_pfn = zone->zone_start_pfn;
++ unsigned long end_pfn = zone_end_pfn(zone);
+ unsigned long pfn;
+
+ for (pfn = start_pfn; pfn < end_pfn; pfn++) {
+@@ -1454,8 +1460,8 @@ static void kmemleak_scan(void)
+ if (!page)
+ continue;
+
+- /* only scan pages belonging to this node */
+- if (page_to_nid(page) != i)
++ /* only scan pages belonging to this zone */
++ if (page_zone(page) != zone)
+ continue;
+ /* only scan if page is in use */
+ if (page_count(page) == 0)
+diff --git a/mm/madvise.c b/mm/madvise.c
+index 0734db8d53a7a..8e5ca01a6cc07 100644
+--- a/mm/madvise.c
++++ b/mm/madvise.c
+@@ -1294,8 +1294,7 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
+ iov_iter_advance(&iter, iovec.iov_len);
+ }
+
+- if (ret == 0)
+- ret = total_len - iov_iter_count(&iter);
++ ret = (total_len - iov_iter_count(&iter)) ? : ret;
+
+ release_mm:
+ mmput(mm);
+diff --git a/mm/memblock.c b/mm/memblock.c
+index 5096500b26473..2b7397781c99a 100644
+--- a/mm/memblock.c
++++ b/mm/memblock.c
+@@ -366,14 +366,20 @@ void __init memblock_discard(void)
+ addr = __pa(memblock.reserved.regions);
+ size = PAGE_ALIGN(sizeof(struct memblock_region) *
+ memblock.reserved.max);
+- __memblock_free_late(addr, size);
++ if (memblock_reserved_in_slab)
++ kfree(memblock.reserved.regions);
++ else
++ __memblock_free_late(addr, size);
+ }
+
+ if (memblock.memory.regions != memblock_memory_init_regions) {
+ addr = __pa(memblock.memory.regions);
+ size = PAGE_ALIGN(sizeof(struct memblock_region) *
+ memblock.memory.max);
+- __memblock_free_late(addr, size);
++ if (memblock_memory_in_slab)
++ kfree(memblock.memory.regions);
++ else
++ __memblock_free_late(addr, size);
+ }
+
+ memblock_memory = NULL;
+diff --git a/mm/memcontrol.c b/mm/memcontrol.c
+index 6da5020a8656d..971546bb99e04 100644
+--- a/mm/memcontrol.c
++++ b/mm/memcontrol.c
+@@ -103,11 +103,6 @@ static bool do_memsw_account(void)
+ return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
+ }
+
+-/* memcg and lruvec stats flushing */
+-static void flush_memcg_stats_dwork(struct work_struct *w);
+-static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
+-static DEFINE_SPINLOCK(stats_flush_lock);
+-
+ #define THRESHOLDS_EVENTS_TARGET 128
+ #define SOFTLIMIT_EVENTS_TARGET 1024
+
+@@ -239,7 +234,7 @@ enum res_type {
+ iter != NULL; \
+ iter = mem_cgroup_iter(NULL, iter, NULL))
+
+-static inline bool should_force_charge(void)
++static inline bool task_is_dying(void)
+ {
+ return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
+ (current->flags & PF_EXITING);
+@@ -259,7 +254,7 @@ struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr)
+ }
+
+ #ifdef CONFIG_MEMCG_KMEM
+-extern spinlock_t css_set_lock;
++static DEFINE_SPINLOCK(objcg_lock);
+
+ bool mem_cgroup_kmem_disabled(void)
+ {
+@@ -303,9 +298,9 @@ static void obj_cgroup_release(struct percpu_ref *ref)
+ if (nr_pages)
+ obj_cgroup_uncharge_pages(objcg, nr_pages);
+
+- spin_lock_irqsave(&css_set_lock, flags);
++ spin_lock_irqsave(&objcg_lock, flags);
+ list_del(&objcg->list);
+- spin_unlock_irqrestore(&css_set_lock, flags);
++ spin_unlock_irqrestore(&objcg_lock, flags);
+
+ percpu_ref_exit(ref);
+ kfree_rcu(objcg, rcu);
+@@ -337,7 +332,7 @@ static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
+
+ objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
+
+- spin_lock_irq(&css_set_lock);
++ spin_lock_irq(&objcg_lock);
+
+ /* 1) Ready to reparent active objcg. */
+ list_add(&objcg->list, &memcg->objcg_list);
+@@ -347,7 +342,7 @@ static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
+ /* 3) Move already reparented objcgs to the parent's list */
+ list_splice(&memcg->objcg_list, &parent->objcg_list);
+
+- spin_unlock_irq(&css_set_lock);
++ spin_unlock_irq(&objcg_lock);
+
+ percpu_ref_kill(&objcg->refcnt);
+ }
+@@ -635,6 +630,74 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
+ return mz;
+ }
+
++/*
++ * memcg and lruvec stats flushing
++ *
++ * Many codepaths leading to stats update or read are performance sensitive and
++ * adding stats flushing in such codepaths is not desirable. So, to optimize the
++ * flushing the kernel does:
++ *
++ * 1) Periodically and asynchronously flush the stats every 2 seconds to not let
++ * rstat update tree grow unbounded.
++ *
++ * 2) Flush the stats synchronously on reader side only when there are more than
++ * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization
++ * will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but
++ * only for 2 seconds due to (1).
++ */
++static void flush_memcg_stats_dwork(struct work_struct *w);
++static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
++static DEFINE_SPINLOCK(stats_flush_lock);
++static DEFINE_PER_CPU(unsigned int, stats_updates);
++static atomic_t stats_flush_threshold = ATOMIC_INIT(0);
++static u64 flush_next_time;
++
++#define FLUSH_TIME (2UL*HZ)
++
++static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
++{
++ unsigned int x;
++
++ cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
++
++ x = __this_cpu_add_return(stats_updates, abs(val));
++ if (x > MEMCG_CHARGE_BATCH) {
++ atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold);
++ __this_cpu_write(stats_updates, 0);
++ }
++}
++
++static void __mem_cgroup_flush_stats(void)
++{
++ unsigned long flag;
++
++ if (!spin_trylock_irqsave(&stats_flush_lock, flag))
++ return;
++
++ flush_next_time = jiffies_64 + 2*FLUSH_TIME;
++ cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup);
++ atomic_set(&stats_flush_threshold, 0);
++ spin_unlock_irqrestore(&stats_flush_lock, flag);
++}
++
++void mem_cgroup_flush_stats(void)
++{
++ if (atomic_read(&stats_flush_threshold) > num_online_cpus())
++ __mem_cgroup_flush_stats();
++}
++
++void mem_cgroup_flush_stats_delayed(void)
++{
++ if (time_after64(jiffies_64, flush_next_time))
++ mem_cgroup_flush_stats();
++}
++
++static void flush_memcg_stats_dwork(struct work_struct *w)
++{
++ __mem_cgroup_flush_stats();
++ queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME);
++}
++
+ /**
+ * __mod_memcg_state - update cgroup memory statistics
+ * @memcg: the memory cgroup
+@@ -647,7 +710,7 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
+ return;
+
+ __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
+- cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
++ memcg_rstat_updated(memcg, val);
+ }
+
+ /* idx can be of type enum memcg_stat_item or node_stat_item. */
+@@ -675,10 +738,12 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
+ memcg = pn->memcg;
+
+ /* Update memcg */
+- __mod_memcg_state(memcg, idx, val);
++ __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
+
+ /* Update lruvec */
+ __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val);
++
++ memcg_rstat_updated(memcg, val);
+ }
+
+ /**
+@@ -780,7 +845,7 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
+ return;
+
+ __this_cpu_add(memcg->vmstats_percpu->events[idx], count);
+- cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
++ memcg_rstat_updated(memcg, count);
+ }
+
+ static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
+@@ -1414,7 +1479,7 @@ static char *memory_stat_format(struct mem_cgroup *memcg)
+ *
+ * Current memory state:
+ */
+- cgroup_rstat_flush(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+
+ for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
+ u64 size;
+@@ -1575,7 +1640,7 @@ static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
+ * A few threads which were not waiting at mutex_lock_killable() can
+ * fail to bail out. Therefore, check again after holding oom_lock.
+ */
+- ret = should_force_charge() || out_of_memory(&oc);
++ ret = task_is_dying() || out_of_memory(&oc);
+
+ unlock:
+ mutex_unlock(&oom_lock);
+@@ -2530,6 +2595,7 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
+ struct page_counter *counter;
+ enum oom_status oom_status;
+ unsigned long nr_reclaimed;
++ bool passed_oom = false;
+ bool may_swap = true;
+ bool drained = false;
+ unsigned long pflags;
+@@ -2564,15 +2630,6 @@ retry:
+ if (gfp_mask & __GFP_ATOMIC)
+ goto force;
+
+- /*
+- * Unlike in global OOM situations, memcg is not in a physical
+- * memory shortage. Allow dying and OOM-killed tasks to
+- * bypass the last charges so that they can exit quickly and
+- * free their memory.
+- */
+- if (unlikely(should_force_charge()))
+- goto force;
+-
+ /*
+ * Prevent unbounded recursion when reclaim operations need to
+ * allocate memory. This might exceed the limits temporarily,
+@@ -2630,8 +2687,9 @@ retry:
+ if (gfp_mask & __GFP_RETRY_MAYFAIL)
+ goto nomem;
+
+- if (fatal_signal_pending(current))
+- goto force;
++ /* Avoid endless loop for tasks bypassed by the oom killer */
++ if (passed_oom && task_is_dying())
++ goto nomem;
+
+ /*
+ * keep retrying as long as the memcg oom killer is able to make
+@@ -2640,14 +2698,10 @@ retry:
+ */
+ oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
+ get_order(nr_pages * PAGE_SIZE));
+- switch (oom_status) {
+- case OOM_SUCCESS:
++ if (oom_status == OOM_SUCCESS) {
++ passed_oom = true;
+ nr_retries = MAX_RECLAIM_RETRIES;
+ goto retry;
+- case OOM_FAILED:
+- goto force;
+- default:
+- goto nomem;
+ }
+ nomem:
+ if (!(gfp_mask & __GFP_NOFAIL))
+@@ -3518,8 +3572,7 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
+ unsigned long val;
+
+ if (mem_cgroup_is_root(memcg)) {
+- /* mem_cgroup_threshold() calls here from irqsafe context */
+- cgroup_rstat_flush_irqsafe(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+ val = memcg_page_state(memcg, NR_FILE_PAGES) +
+ memcg_page_state(memcg, NR_ANON_MAPPED);
+ if (swap)
+@@ -3900,7 +3953,7 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v)
+ int nid;
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+- cgroup_rstat_flush(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+
+ for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
+ seq_printf(m, "%s=%lu", stat->name,
+@@ -3972,7 +4025,7 @@ static int memcg_stat_show(struct seq_file *m, void *v)
+
+ BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
+
+- cgroup_rstat_flush(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+
+ for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
+ unsigned long nr;
+@@ -4475,7 +4528,7 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
+ struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
+ struct mem_cgroup *parent;
+
+- cgroup_rstat_flush_irqsafe(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+
+ *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
+ *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
+@@ -5341,21 +5394,6 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
+ memcg_wb_domain_size_changed(memcg);
+ }
+
+-void mem_cgroup_flush_stats(void)
+-{
+- if (!spin_trylock(&stats_flush_lock))
+- return;
+-
+- cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup);
+- spin_unlock(&stats_flush_lock);
+-}
+-
+-static void flush_memcg_stats_dwork(struct work_struct *w)
+-{
+- mem_cgroup_flush_stats();
+- queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 2UL*HZ);
+-}
+-
+ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
+ {
+ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
+@@ -6373,7 +6411,7 @@ static int memory_numa_stat_show(struct seq_file *m, void *v)
+ int i;
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+- cgroup_rstat_flush(memcg->css.cgroup);
++ mem_cgroup_flush_stats();
+
+ for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
+ int nid;
+@@ -7077,7 +7115,7 @@ static int __init cgroup_memory(char *s)
+ if (!strcmp(token, "nokmem"))
+ cgroup_memory_nokmem = true;
+ }
+- return 0;
++ return 1;
+ }
+ __setup("cgroup.memory=", cgroup_memory);
+
+diff --git a/mm/memfd.c b/mm/memfd.c
+index 081dd33e6a61b..475d095dd7f53 100644
+--- a/mm/memfd.c
++++ b/mm/memfd.c
+@@ -31,20 +31,28 @@
+ static void memfd_tag_pins(struct xa_state *xas)
+ {
+ struct page *page;
+- unsigned int tagged = 0;
++ int latency = 0;
++ int cache_count;
+
+ lru_add_drain();
+
+ xas_lock_irq(xas);
+ xas_for_each(xas, page, ULONG_MAX) {
+- if (xa_is_value(page))
+- continue;
+- page = find_subpage(page, xas->xa_index);
+- if (page_count(page) - page_mapcount(page) > 1)
++ cache_count = 1;
++ if (!xa_is_value(page) &&
++ PageTransHuge(page) && !PageHuge(page))
++ cache_count = HPAGE_PMD_NR;
++
++ if (!xa_is_value(page) &&
++ page_count(page) - total_mapcount(page) != cache_count)
+ xas_set_mark(xas, MEMFD_TAG_PINNED);
++ if (cache_count != 1)
++ xas_set(xas, page->index + cache_count);
+
+- if (++tagged % XA_CHECK_SCHED)
++ latency += cache_count;
++ if (latency < XA_CHECK_SCHED)
+ continue;
++ latency = 0;
+
+ xas_pause(xas);
+ xas_unlock_irq(xas);
+@@ -73,7 +81,8 @@ static int memfd_wait_for_pins(struct address_space *mapping)
+
+ error = 0;
+ for (scan = 0; scan <= LAST_SCAN; scan++) {
+- unsigned int tagged = 0;
++ int latency = 0;
++ int cache_count;
+
+ if (!xas_marked(&xas, MEMFD_TAG_PINNED))
+ break;
+@@ -87,10 +96,14 @@ static int memfd_wait_for_pins(struct address_space *mapping)
+ xas_lock_irq(&xas);
+ xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
+ bool clear = true;
+- if (xa_is_value(page))
+- continue;
+- page = find_subpage(page, xas.xa_index);
+- if (page_count(page) - page_mapcount(page) != 1) {
++
++ cache_count = 1;
++ if (!xa_is_value(page) &&
++ PageTransHuge(page) && !PageHuge(page))
++ cache_count = HPAGE_PMD_NR;
++
++ if (!xa_is_value(page) && cache_count !=
++ page_count(page) - total_mapcount(page)) {
+ /*
+ * On the last scan, we clean up all those tags
+ * we inserted; but make a note that we still
+@@ -103,8 +116,11 @@ static int memfd_wait_for_pins(struct address_space *mapping)
+ }
+ if (clear)
+ xas_clear_mark(&xas, MEMFD_TAG_PINNED);
+- if (++tagged % XA_CHECK_SCHED)
++
++ latency += cache_count;
++ if (latency < XA_CHECK_SCHED)
+ continue;
++ latency = 0;
+
+ xas_pause(&xas);
+ xas_unlock_irq(&xas);
+diff --git a/mm/memory-failure.c b/mm/memory-failure.c
+index bdbbb32211a58..c3ceb74369334 100644
+--- a/mm/memory-failure.c
++++ b/mm/memory-failure.c
+@@ -705,8 +705,10 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn,
+ (void *)&priv);
+ if (ret == 1 && priv.tk.addr)
+ kill_proc(&priv.tk, pfn, flags);
++ else
++ ret = 0;
+ mmap_read_unlock(p->mm);
+- return ret ? -EFAULT : -EHWPOISON;
++ return ret > 0 ? -EHWPOISON : -EFAULT;
+ }
+
+ static const char *action_name[] = {
+@@ -1215,7 +1217,7 @@ try_again:
+ }
+ out:
+ if (ret == -EIO)
+- dump_page(p, "hwpoison: unhandlable page");
++ pr_err("Memory failure: %#lx: unhandlable page.\n", page_to_pfn(p));
+
+ return ret;
+ }
+@@ -1437,17 +1439,12 @@ static int memory_failure_hugetlb(unsigned long pfn, int flags)
+ if (!(flags & MF_COUNT_INCREASED)) {
+ res = get_hwpoison_page(p, flags);
+ if (!res) {
+- /*
+- * Check "filter hit" and "race with other subpage."
+- */
+ lock_page(head);
+- if (PageHWPoison(head)) {
+- if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
+- || (p != head && TestSetPageHWPoison(head))) {
++ if (hwpoison_filter(p)) {
++ if (TestClearPageHWPoison(head))
+ num_poisoned_pages_dec();
+- unlock_page(head);
+- return 0;
+- }
++ unlock_page(head);
++ return 0;
+ }
+ unlock_page(head);
+ res = MF_FAILED;
+@@ -2206,6 +2203,7 @@ retry:
+ } else if (ret == 0) {
+ if (soft_offline_free_page(page) && try_again) {
+ try_again = false;
++ flags &= ~MF_COUNT_INCREASED;
+ goto retry;
+ }
+ }
+diff --git a/mm/memory.c b/mm/memory.c
+index c52be6d6b6055..26d115ded4ab7 100644
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -1301,6 +1301,17 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
+ return ret;
+ }
+
++/* Whether we should zap all COWed (private) pages too */
++static inline bool should_zap_cows(struct zap_details *details)
++{
++ /* By default, zap all pages */
++ if (!details)
++ return true;
++
++ /* Or, we zap COWed pages only if the caller wants to */
++ return !details->check_mapping;
++}
++
+ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+ struct vm_area_struct *vma, pmd_t *pmd,
+ unsigned long addr, unsigned long end,
+@@ -1396,16 +1407,18 @@ again:
+ continue;
+ }
+
+- /* If details->check_mapping, we leave swap entries. */
+- if (unlikely(details))
+- continue;
+-
+- if (!non_swap_entry(entry))
++ if (!non_swap_entry(entry)) {
++ /* Genuine swap entry, hence a private anon page */
++ if (!should_zap_cows(details))
++ continue;
+ rss[MM_SWAPENTS]--;
+- else if (is_migration_entry(entry)) {
++ } else if (is_migration_entry(entry)) {
+ struct page *page;
+
+ page = pfn_swap_entry_to_page(entry);
++ if (details && details->check_mapping &&
++ details->check_mapping != page_rmapping(page))
++ continue;
+ rss[mm_counter(page)]--;
+ }
+ if (unlikely(!free_swap_and_cache(entry)))
+@@ -3861,11 +3874,20 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
+ return ret;
+
+ if (unlikely(PageHWPoison(vmf->page))) {
+- if (ret & VM_FAULT_LOCKED)
+- unlock_page(vmf->page);
+- put_page(vmf->page);
++ struct page *page = vmf->page;
++ vm_fault_t poisonret = VM_FAULT_HWPOISON;
++ if (ret & VM_FAULT_LOCKED) {
++ if (page_mapped(page))
++ unmap_mapping_pages(page_mapping(page),
++ page->index, 1, false);
++ /* Retry if a clean page was removed from the cache. */
++ if (invalidate_inode_page(page))
++ poisonret = VM_FAULT_NOPAGE;
++ unlock_page(page);
++ }
++ put_page(page);
+ vmf->page = NULL;
+- return VM_FAULT_HWPOISON;
++ return poisonret;
+ }
+
+ if (unlikely(!(ret & VM_FAULT_LOCKED)))
+@@ -5445,6 +5467,8 @@ long copy_huge_page_from_user(struct page *dst_page,
+ if (rc)
+ break;
+
++ flush_dcache_page(subpage);
++
+ cond_resched();
+ }
+ return ret_val;
+diff --git a/mm/mempolicy.c b/mm/mempolicy.c
+index d12e0608fced2..e75872035c768 100644
+--- a/mm/mempolicy.c
++++ b/mm/mempolicy.c
+@@ -783,7 +783,6 @@ static int vma_replace_policy(struct vm_area_struct *vma,
+ static int mbind_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct mempolicy *new_pol)
+ {
+- struct vm_area_struct *next;
+ struct vm_area_struct *prev;
+ struct vm_area_struct *vma;
+ int err = 0;
+@@ -798,8 +797,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
+ if (start > vma->vm_start)
+ prev = vma;
+
+- for (; vma && vma->vm_start < end; prev = vma, vma = next) {
+- next = vma->vm_next;
++ for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) {
+ vmstart = max(start, vma->vm_start);
+ vmend = min(end, vma->vm_end);
+
+@@ -813,10 +811,6 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
+ new_pol, vma->vm_userfaultfd_ctx);
+ if (prev) {
+ vma = prev;
+- next = vma->vm_next;
+- if (mpol_equal(vma_policy(vma), new_pol))
+- continue;
+- /* vma_merge() joined vma && vma->next, case 8 */
+ goto replace;
+ }
+ if (vma->vm_start != vmstart) {
+@@ -2140,8 +2134,7 @@ struct page *alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
+ * memory with both reclaim and compact as well.
+ */
+ if (!page && (gfp & __GFP_DIRECT_RECLAIM))
+- page = __alloc_pages_node(hpage_node,
+- gfp, order);
++ page = __alloc_pages(gfp, order, hpage_node, nmask);
+
+ goto out;
+ }
+@@ -2568,6 +2561,7 @@ alloc_new:
+ mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
+ if (!mpol_new)
+ goto err_out;
++ atomic_set(&mpol_new->refcnt, 1);
+ goto restart;
+ }
+
+diff --git a/mm/migrate.c b/mm/migrate.c
+index 1852d787e6ab6..afb944b600fe6 100644
+--- a/mm/migrate.c
++++ b/mm/migrate.c
+@@ -948,9 +948,12 @@ static int move_to_new_page(struct page *newpage, struct page *page,
+ if (!PageMappingFlags(page))
+ page->mapping = NULL;
+
+- if (likely(!is_zone_device_page(newpage)))
+- flush_dcache_page(newpage);
++ if (likely(!is_zone_device_page(newpage))) {
++ int i, nr = compound_nr(newpage);
+
++ for (i = 0; i < nr; i++)
++ flush_dcache_page(newpage + i);
++ }
+ }
+ out:
+ return rc;
+diff --git a/mm/mlock.c b/mm/mlock.c
+index 16d2ee160d43c..0cee3f97d3df7 100644
+--- a/mm/mlock.c
++++ b/mm/mlock.c
+@@ -826,18 +826,18 @@ int user_shm_lock(size_t size, struct ucounts *ucounts)
+
+ locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ lock_limit = rlimit(RLIMIT_MEMLOCK);
+- if (lock_limit == RLIM_INFINITY)
+- allowed = 1;
+- lock_limit >>= PAGE_SHIFT;
++ if (lock_limit != RLIM_INFINITY)
++ lock_limit >>= PAGE_SHIFT;
+ spin_lock(&shmlock_user_lock);
+ memlock = inc_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
+
+- if (!allowed && (memlock == LONG_MAX || memlock > lock_limit) && !capable(CAP_IPC_LOCK)) {
++ if ((memlock == LONG_MAX || memlock > lock_limit) && !capable(CAP_IPC_LOCK)) {
+ dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
+ goto out;
+ }
+ if (!get_ucounts(ucounts)) {
+ dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
++ allowed = 0;
+ goto out;
+ }
+ allowed = 1;
+diff --git a/mm/mmap.c b/mm/mmap.c
+index 88dcc5c252255..6bb553ed5c557 100644
+--- a/mm/mmap.c
++++ b/mm/mmap.c
+@@ -2113,14 +2113,6 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
+ return addr;
+ }
+
+-#ifndef arch_get_mmap_end
+-#define arch_get_mmap_end(addr) (TASK_SIZE)
+-#endif
+-
+-#ifndef arch_get_mmap_base
+-#define arch_get_mmap_base(addr, base) (base)
+-#endif
+-
+ /* Get an address range which is currently unmapped.
+ * For shmat() with addr=0.
+ *
+@@ -2551,7 +2543,7 @@ static int __init cmdline_parse_stack_guard_gap(char *p)
+ if (!*endptr)
+ stack_guard_gap = val << PAGE_SHIFT;
+
+- return 0;
++ return 1;
+ }
+ __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
+
+diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
+index 459d195d2ff64..f45ff1b7626a6 100644
+--- a/mm/mmu_notifier.c
++++ b/mm/mmu_notifier.c
+@@ -1036,6 +1036,18 @@ int mmu_interval_notifier_insert_locked(
+ }
+ EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_locked);
+
++static bool
++mmu_interval_seq_released(struct mmu_notifier_subscriptions *subscriptions,
++ unsigned long seq)
++{
++ bool ret;
++
++ spin_lock(&subscriptions->lock);
++ ret = subscriptions->invalidate_seq != seq;
++ spin_unlock(&subscriptions->lock);
++ return ret;
++}
++
+ /**
+ * mmu_interval_notifier_remove - Remove a interval notifier
+ * @interval_sub: Interval subscription to unregister
+@@ -1083,7 +1095,7 @@ void mmu_interval_notifier_remove(struct mmu_interval_notifier *interval_sub)
+ lock_map_release(&__mmu_notifier_invalidate_range_start_map);
+ if (seq)
+ wait_event(subscriptions->wq,
+- READ_ONCE(subscriptions->invalidate_seq) != seq);
++ mmu_interval_seq_released(subscriptions, seq));
+
+ /* pairs with mmgrab in mmu_interval_notifier_insert() */
+ mmdrop(mm);
+diff --git a/mm/mprotect.c b/mm/mprotect.c
+index 883e2cc85cad8..ed18dc49533f6 100644
+--- a/mm/mprotect.c
++++ b/mm/mprotect.c
+@@ -94,7 +94,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
+
+ /* Also skip shared copy-on-write pages */
+ if (is_cow_mapping(vma->vm_flags) &&
+- page_mapcount(page) != 1)
++ page_count(page) != 1)
+ continue;
+
+ /*
+diff --git a/mm/mremap.c b/mm/mremap.c
+index badfe17ade1f0..3a3cf4cc2c632 100644
+--- a/mm/mremap.c
++++ b/mm/mremap.c
+@@ -486,6 +486,9 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
+ pmd_t *old_pmd, *new_pmd;
+ pud_t *old_pud, *new_pud;
+
++ if (!len)
++ return 0;
++
+ old_end = old_addr + len;
+ flush_cache_range(vma, old_addr, old_end);
+
+diff --git a/mm/oom_kill.c b/mm/oom_kill.c
+index 989f35a2bbb1d..262f752d3d516 100644
+--- a/mm/oom_kill.c
++++ b/mm/oom_kill.c
+@@ -635,7 +635,7 @@ done:
+ */
+ set_bit(MMF_OOM_SKIP, &mm->flags);
+
+- /* Drop a reference taken by wake_oom_reaper */
++ /* Drop a reference taken by queue_oom_reaper */
+ put_task_struct(tsk);
+ }
+
+@@ -645,12 +645,12 @@ static int oom_reaper(void *unused)
+ struct task_struct *tsk = NULL;
+
+ wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL);
+- spin_lock(&oom_reaper_lock);
++ spin_lock_irq(&oom_reaper_lock);
+ if (oom_reaper_list != NULL) {
+ tsk = oom_reaper_list;
+ oom_reaper_list = tsk->oom_reaper_list;
+ }
+- spin_unlock(&oom_reaper_lock);
++ spin_unlock_irq(&oom_reaper_lock);
+
+ if (tsk)
+ oom_reap_task(tsk);
+@@ -659,22 +659,48 @@ static int oom_reaper(void *unused)
+ return 0;
+ }
+
+-static void wake_oom_reaper(struct task_struct *tsk)
++static void wake_oom_reaper(struct timer_list *timer)
+ {
+- /* mm is already queued? */
+- if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags))
+- return;
++ struct task_struct *tsk = container_of(timer, struct task_struct,
++ oom_reaper_timer);
++ struct mm_struct *mm = tsk->signal->oom_mm;
++ unsigned long flags;
+
+- get_task_struct(tsk);
++ /* The victim managed to terminate on its own - see exit_mmap */
++ if (test_bit(MMF_OOM_SKIP, &mm->flags)) {
++ put_task_struct(tsk);
++ return;
++ }
+
+- spin_lock(&oom_reaper_lock);
++ spin_lock_irqsave(&oom_reaper_lock, flags);
+ tsk->oom_reaper_list = oom_reaper_list;
+ oom_reaper_list = tsk;
+- spin_unlock(&oom_reaper_lock);
++ spin_unlock_irqrestore(&oom_reaper_lock, flags);
+ trace_wake_reaper(tsk->pid);
+ wake_up(&oom_reaper_wait);
+ }
+
++/*
++ * Give the OOM victim time to exit naturally before invoking the oom_reaping.
++ * The timers timeout is arbitrary... the longer it is, the longer the worst
++ * case scenario for the OOM can take. If it is too small, the oom_reaper can
++ * get in the way and release resources needed by the process exit path.
++ * e.g. The futex robust list can sit in Anon|Private memory that gets reaped
++ * before the exit path is able to wake the futex waiters.
++ */
++#define OOM_REAPER_DELAY (2*HZ)
++static void queue_oom_reaper(struct task_struct *tsk)
++{
++ /* mm is already queued? */
++ if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags))
++ return;
++
++ get_task_struct(tsk);
++ timer_setup(&tsk->oom_reaper_timer, wake_oom_reaper, 0);
++ tsk->oom_reaper_timer.expires = jiffies + OOM_REAPER_DELAY;
++ add_timer(&tsk->oom_reaper_timer);
++}
++
+ static int __init oom_init(void)
+ {
+ oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper");
+@@ -682,7 +708,7 @@ static int __init oom_init(void)
+ }
+ subsys_initcall(oom_init)
+ #else
+-static inline void wake_oom_reaper(struct task_struct *tsk)
++static inline void queue_oom_reaper(struct task_struct *tsk)
+ {
+ }
+ #endif /* CONFIG_MMU */
+@@ -933,7 +959,7 @@ static void __oom_kill_process(struct task_struct *victim, const char *message)
+ rcu_read_unlock();
+
+ if (can_oom_reap)
+- wake_oom_reaper(victim);
++ queue_oom_reaper(victim);
+
+ mmdrop(mm);
+ put_task_struct(victim);
+@@ -969,7 +995,7 @@ static void oom_kill_process(struct oom_control *oc, const char *message)
+ task_lock(victim);
+ if (task_will_free_mem(victim)) {
+ mark_oom_victim(victim);
+- wake_oom_reaper(victim);
++ queue_oom_reaper(victim);
+ task_unlock(victim);
+ put_task_struct(victim);
+ return;
+@@ -1067,7 +1093,7 @@ bool out_of_memory(struct oom_control *oc)
+ */
+ if (task_will_free_mem(current)) {
+ mark_oom_victim(current);
+- wake_oom_reaper(current);
++ queue_oom_reaper(current);
+ return true;
+ }
+
+@@ -1120,27 +1146,24 @@ bool out_of_memory(struct oom_control *oc)
+ }
+
+ /*
+- * The pagefault handler calls here because it is out of memory, so kill a
+- * memory-hogging task. If oom_lock is held by somebody else, a parallel oom
+- * killing is already in progress so do nothing.
++ * The pagefault handler calls here because some allocation has failed. We have
++ * to take care of the memcg OOM here because this is the only safe context without
++ * any locks held but let the oom killer triggered from the allocation context care
++ * about the global OOM.
+ */
+ void pagefault_out_of_memory(void)
+ {
+- struct oom_control oc = {
+- .zonelist = NULL,
+- .nodemask = NULL,
+- .memcg = NULL,
+- .gfp_mask = 0,
+- .order = 0,
+- };
++ static DEFINE_RATELIMIT_STATE(pfoom_rs, DEFAULT_RATELIMIT_INTERVAL,
++ DEFAULT_RATELIMIT_BURST);
+
+ if (mem_cgroup_oom_synchronize(true))
+ return;
+
+- if (!mutex_trylock(&oom_lock))
++ if (fatal_signal_pending(current))
+ return;
+- out_of_memory(&oc);
+- mutex_unlock(&oom_lock);
++
++ if (__ratelimit(&pfoom_rs))
++ pr_warn("Huh VM_FAULT_OOM leaked out to the #PF handler. Retrying PF\n");
+ }
+
+ SYSCALL_DEFINE2(process_mrelease, int, pidfd, unsigned int, flags)
+diff --git a/mm/page_alloc.c b/mm/page_alloc.c
+index 23d3339ac4e8e..47c22810c3c5f 100644
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@ -4210,7 +4210,9 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
+ va_list args;
+ static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
+
+- if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
++ if ((gfp_mask & __GFP_NOWARN) ||
++ !__ratelimit(&nopage_rs) ||
++ ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
+ return;
+
+ va_start(args, fmt);
+@@ -6090,7 +6092,7 @@ static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
+ do {
+ zone_type--;
+ zone = pgdat->node_zones + zone_type;
+- if (managed_zone(zone)) {
++ if (populated_zone(zone)) {
+ zoneref_set_zone(zone, &zonerefs[nr_zones++]);
+ check_highest_zone(zone_type);
+ }
+@@ -7897,10 +7899,17 @@ restart:
+
+ out2:
+ /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
+- for (nid = 0; nid < MAX_NUMNODES; nid++)
++ for (nid = 0; nid < MAX_NUMNODES; nid++) {
++ unsigned long start_pfn, end_pfn;
++
+ zone_movable_pfn[nid] =
+ roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
+
++ get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
++ if (zone_movable_pfn[nid] >= end_pfn)
++ zone_movable_pfn[nid] = 0;
++ }
++
+ out:
+ /* restore the node_state */
+ node_states[N_MEMORY] = saved_node_state;
+@@ -8160,7 +8169,7 @@ void __init mem_init_print_info(void)
+ */
+ #define adj_init_size(start, end, size, pos, adj) \
+ do { \
+- if (start <= pos && pos < end && size > adj) \
++ if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
+ size -= adj; \
+ } while (0)
+
+@@ -9449,3 +9458,18 @@ bool take_page_off_buddy(struct page *page)
+ return ret;
+ }
+ #endif
++
++#ifdef CONFIG_ZONE_DMA
++bool has_managed_dma(void)
++{
++ struct pglist_data *pgdat;
++
++ for_each_online_pgdat(pgdat) {
++ struct zone *zone = &pgdat->node_zones[ZONE_DMA];
++
++ if (managed_zone(zone))
++ return true;
++ }
++ return false;
++}
++#endif /* CONFIG_ZONE_DMA */
+diff --git a/mm/page_io.c b/mm/page_io.c
+index c493ce9ebcf50..66c6fbb07bc4c 100644
+--- a/mm/page_io.c
++++ b/mm/page_io.c
+@@ -50,54 +50,6 @@ void end_swap_bio_write(struct bio *bio)
+ bio_put(bio);
+ }
+
+-static void swap_slot_free_notify(struct page *page)
+-{
+- struct swap_info_struct *sis;
+- struct gendisk *disk;
+- swp_entry_t entry;
+-
+- /*
+- * There is no guarantee that the page is in swap cache - the software
+- * suspend code (at least) uses end_swap_bio_read() against a non-
+- * swapcache page. So we must check PG_swapcache before proceeding with
+- * this optimization.
+- */
+- if (unlikely(!PageSwapCache(page)))
+- return;
+-
+- sis = page_swap_info(page);
+- if (data_race(!(sis->flags & SWP_BLKDEV)))
+- return;
+-
+- /*
+- * The swap subsystem performs lazy swap slot freeing,
+- * expecting that the page will be swapped out again.
+- * So we can avoid an unnecessary write if the page
+- * isn't redirtied.
+- * This is good for real swap storage because we can
+- * reduce unnecessary I/O and enhance wear-leveling
+- * if an SSD is used as the as swap device.
+- * But if in-memory swap device (eg zram) is used,
+- * this causes a duplicated copy between uncompressed
+- * data in VM-owned memory and compressed data in
+- * zram-owned memory. So let's free zram-owned memory
+- * and make the VM-owned decompressed page *dirty*,
+- * so the page should be swapped out somewhere again if
+- * we again wish to reclaim it.
+- */
+- disk = sis->bdev->bd_disk;
+- entry.val = page_private(page);
+- if (disk->fops->swap_slot_free_notify && __swap_count(entry) == 1) {
+- unsigned long offset;
+-
+- offset = swp_offset(entry);
+-
+- SetPageDirty(page);
+- disk->fops->swap_slot_free_notify(sis->bdev,
+- offset);
+- }
+-}
+-
+ static void end_swap_bio_read(struct bio *bio)
+ {
+ struct page *page = bio_first_page_all(bio);
+@@ -113,7 +65,6 @@ static void end_swap_bio_read(struct bio *bio)
+ }
+
+ SetPageUptodate(page);
+- swap_slot_free_notify(page);
+ out:
+ unlock_page(page);
+ WRITE_ONCE(bio->bi_private, NULL);
+@@ -392,11 +343,6 @@ int swap_readpage(struct page *page, bool synchronous)
+ if (sis->flags & SWP_SYNCHRONOUS_IO) {
+ ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
+ if (!ret) {
+- if (trylock_page(page)) {
+- swap_slot_free_notify(page);
+- unlock_page(page);
+- }
+-
+ count_vm_event(PSWPIN);
+ goto out;
+ }
+diff --git a/mm/rmap.c b/mm/rmap.c
+index 6aebd17472512..3e340ee380cb9 100644
+--- a/mm/rmap.c
++++ b/mm/rmap.c
+@@ -1570,7 +1570,30 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
+
+ /* MADV_FREE page check */
+ if (!PageSwapBacked(page)) {
+- if (!PageDirty(page)) {
++ int ref_count, map_count;
++
++ /*
++ * Synchronize with gup_pte_range():
++ * - clear PTE; barrier; read refcount
++ * - inc refcount; barrier; read PTE
++ */
++ smp_mb();
++
++ ref_count = page_ref_count(page);
++ map_count = page_mapcount(page);
++
++ /*
++ * Order reads for page refcount and dirty flag
++ * (see comments in __remove_mapping()).
++ */
++ smp_rmb();
++
++ /*
++ * The only page refs must be one from isolation
++ * plus the rmap(s) (dropped by discard:).
++ */
++ if (ref_count == 1 + map_count &&
++ !PageDirty(page)) {
+ /* Invalidate as we cleared the pte */
+ mmu_notifier_invalidate_range(mm,
+ address, address + PAGE_SIZE);
+diff --git a/mm/secretmem.c b/mm/secretmem.c
+index 22b310adb53d9..5a62ef3bcfcff 100644
+--- a/mm/secretmem.c
++++ b/mm/secretmem.c
+@@ -158,6 +158,22 @@ const struct address_space_operations secretmem_aops = {
+ .isolate_page = secretmem_isolate_page,
+ };
+
++static int secretmem_setattr(struct user_namespace *mnt_userns,
++ struct dentry *dentry, struct iattr *iattr)
++{
++ struct inode *inode = d_inode(dentry);
++ unsigned int ia_valid = iattr->ia_valid;
++
++ if ((ia_valid & ATTR_SIZE) && inode->i_size)
++ return -EINVAL;
++
++ return simple_setattr(mnt_userns, dentry, iattr);
++}
++
++static const struct inode_operations secretmem_iops = {
++ .setattr = secretmem_setattr,
++};
++
+ static struct vfsmount *secretmem_mnt;
+
+ static struct file *secretmem_file_create(unsigned long flags)
+@@ -177,6 +193,7 @@ static struct file *secretmem_file_create(unsigned long flags)
+ mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
+ mapping_set_unevictable(inode->i_mapping);
+
++ inode->i_op = &secretmem_iops;
+ inode->i_mapping->a_ops = &secretmem_aops;
+
+ /* pretend we are a normal file with zero size */
+diff --git a/mm/shmem.c b/mm/shmem.c
+index b5860f4a2738e..342d1bc728670 100644
+--- a/mm/shmem.c
++++ b/mm/shmem.c
+@@ -555,7 +555,7 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
+ struct shmem_inode_info *info;
+ struct page *page;
+ unsigned long batch = sc ? sc->nr_to_scan : 128;
+- int removed = 0, split = 0;
++ int split = 0;
+
+ if (list_empty(&sbinfo->shrinklist))
+ return SHRINK_STOP;
+@@ -570,7 +570,6 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
+ /* inode is about to be evicted */
+ if (!inode) {
+ list_del_init(&info->shrinklist);
+- removed++;
+ goto next;
+ }
+
+@@ -578,12 +577,12 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
+ if (round_up(inode->i_size, PAGE_SIZE) ==
+ round_up(inode->i_size, HPAGE_PMD_SIZE)) {
+ list_move(&info->shrinklist, &to_remove);
+- removed++;
+ goto next;
+ }
+
+ list_move(&info->shrinklist, &list);
+ next:
++ sbinfo->shrinklist_len--;
+ if (!--batch)
+ break;
+ }
+@@ -603,7 +602,7 @@ next:
+ inode = &info->vfs_inode;
+
+ if (nr_to_split && split >= nr_to_split)
+- goto leave;
++ goto move_back;
+
+ page = find_get_page(inode->i_mapping,
+ (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
+@@ -617,38 +616,44 @@ next:
+ }
+
+ /*
+- * Leave the inode on the list if we failed to lock
+- * the page at this time.
++ * Move the inode on the list back to shrinklist if we failed
++ * to lock the page at this time.
+ *
+ * Waiting for the lock may lead to deadlock in the
+ * reclaim path.
+ */
+ if (!trylock_page(page)) {
+ put_page(page);
+- goto leave;
++ goto move_back;
+ }
+
+ ret = split_huge_page(page);
+ unlock_page(page);
+ put_page(page);
+
+- /* If split failed leave the inode on the list */
++ /* If split failed move the inode on the list back to shrinklist */
+ if (ret)
+- goto leave;
++ goto move_back;
+
+ split++;
+ drop:
+ list_del_init(&info->shrinklist);
+- removed++;
+-leave:
++ goto put;
++move_back:
++ /*
++ * Make sure the inode is either on the global list or deleted
++ * from any local list before iput() since it could be deleted
++ * in another thread once we put the inode (then the local list
++ * is corrupted).
++ */
++ spin_lock(&sbinfo->shrinklist_lock);
++ list_move(&info->shrinklist, &sbinfo->shrinklist);
++ sbinfo->shrinklist_len++;
++ spin_unlock(&sbinfo->shrinklist_lock);
++put:
+ iput(inode);
+ }
+
+- spin_lock(&sbinfo->shrinklist_lock);
+- list_splice_tail(&list, &sbinfo->shrinklist);
+- sbinfo->shrinklist_len -= removed;
+- spin_unlock(&sbinfo->shrinklist_lock);
+-
+ return split;
+ }
+
+@@ -2389,8 +2394,10 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
+ /* don't free the page */
+ goto out_unacct_blocks;
+ }
++
++ flush_dcache_page(page);
+ } else { /* ZEROPAGE */
+- clear_highpage(page);
++ clear_user_highpage(page, dst_addr);
+ }
+ } else {
+ page = *pagep;
+diff --git a/mm/slab.c b/mm/slab.c
+index 874b3f8fe80da..1bd283e98c58c 100644
+--- a/mm/slab.c
++++ b/mm/slab.c
+@@ -3429,6 +3429,7 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
+
+ if (is_kfence_address(objp)) {
+ kmemleak_free_recursive(objp, cachep->flags);
++ memcg_slab_free_hook(cachep, &objp, 1);
+ __kfence_free(objp);
+ return;
+ }
+@@ -3657,7 +3658,7 @@ EXPORT_SYMBOL(__kmalloc_node_track_caller);
+ #endif /* CONFIG_NUMA */
+
+ #ifdef CONFIG_PRINTK
+-void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
+ {
+ struct kmem_cache *cachep;
+ unsigned int objnr;
+diff --git a/mm/slab.h b/mm/slab.h
+index 58c01a34e5b86..1ae1bdd485c17 100644
+--- a/mm/slab.h
++++ b/mm/slab.h
+@@ -147,7 +147,7 @@ static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
+ #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
+ SLAB_TEMPORARY | SLAB_ACCOUNT)
+ #else
+-#define SLAB_CACHE_FLAGS (0)
++#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE)
+ #endif
+
+ /* Common flags available with current configuration */
+@@ -643,7 +643,7 @@ struct kmem_obj_info {
+ void *kp_stack[KS_ADDRS_COUNT];
+ void *kp_free_stack[KS_ADDRS_COUNT];
+ };
+-void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page);
++void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page);
+ #endif
+
+ #endif /* MM_SLAB_H */
+diff --git a/mm/slab_common.c b/mm/slab_common.c
+index ec2bb0beed757..022319e7deaf7 100644
+--- a/mm/slab_common.c
++++ b/mm/slab_common.c
+@@ -568,6 +568,13 @@ bool kmem_valid_obj(void *object)
+ }
+ EXPORT_SYMBOL_GPL(kmem_valid_obj);
+
++static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++{
++ if (__kfence_obj_info(kpp, object, page))
++ return;
++ __kmem_obj_info(kpp, object, page);
++}
++
+ /**
+ * kmem_dump_obj - Print available slab provenance information
+ * @object: slab object for which to find provenance information.
+@@ -603,6 +610,8 @@ void kmem_dump_obj(void *object)
+ pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
+ else
+ pr_cont(" slab%s", cp);
++ if (is_kfence_address(object))
++ pr_cont(" (kfence)");
+ if (kp.kp_objp)
+ pr_cont(" start %px", kp.kp_objp);
+ if (kp.kp_data_offset)
+diff --git a/mm/slob.c b/mm/slob.c
+index 74d3f6e60666e..f3fc15df971af 100644
+--- a/mm/slob.c
++++ b/mm/slob.c
+@@ -462,7 +462,7 @@ out:
+ }
+
+ #ifdef CONFIG_PRINTK
+-void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
+ {
+ kpp->kp_ptr = object;
+ kpp->kp_page = page;
+diff --git a/mm/slub.c b/mm/slub.c
+index d8f77346376d8..b75eebc0350e7 100644
+--- a/mm/slub.c
++++ b/mm/slub.c
+@@ -4299,7 +4299,7 @@ int __kmem_cache_shutdown(struct kmem_cache *s)
+ }
+
+ #ifdef CONFIG_PRINTK
+-void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
++void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
+ {
+ void *base;
+ int __maybe_unused i;
+@@ -5072,6 +5072,7 @@ struct loc_track {
+ unsigned long max;
+ unsigned long count;
+ struct location *loc;
++ loff_t idx;
+ };
+
+ static struct dentry *slab_debugfs_root;
+@@ -6035,11 +6036,11 @@ __initcall(slab_sysfs_init);
+ #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
+ static int slab_debugfs_show(struct seq_file *seq, void *v)
+ {
+-
+- struct location *l;
+- unsigned int idx = *(unsigned int *)v;
+ struct loc_track *t = seq->private;
++ struct location *l;
++ unsigned long idx;
+
++ idx = (unsigned long) t->idx;
+ if (idx < t->count) {
+ l = &t->loc[idx];
+
+@@ -6088,16 +6089,18 @@ static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
+ {
+ struct loc_track *t = seq->private;
+
+- v = ppos;
+- ++*ppos;
++ t->idx = ++(*ppos);
+ if (*ppos <= t->count)
+- return v;
++ return ppos;
+
+ return NULL;
+ }
+
+ static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
+ {
++ struct loc_track *t = seq->private;
++
++ t->idx = *ppos;
+ return ppos;
+ }
+
+diff --git a/mm/swap_state.c b/mm/swap_state.c
+index bc7cee6b2ec54..122a37cbc081f 100644
+--- a/mm/swap_state.c
++++ b/mm/swap_state.c
+@@ -478,7 +478,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
+ * __read_swap_cache_async(), which has set SWAP_HAS_CACHE
+ * in swap_map, but not yet added its page to swap cache.
+ */
+- cond_resched();
++ schedule_timeout_uninterruptible(1);
+ }
+
+ /*
+diff --git a/mm/usercopy.c b/mm/usercopy.c
+index b3de3c4eefba7..540968b481e7e 100644
+--- a/mm/usercopy.c
++++ b/mm/usercopy.c
+@@ -294,7 +294,10 @@ static bool enable_checks __initdata = true;
+
+ static int __init parse_hardened_usercopy(char *str)
+ {
+- return strtobool(str, &enable_checks);
++ if (strtobool(str, &enable_checks))
++ pr_warn("Invalid option string for hardened_usercopy: '%s'\n",
++ str);
++ return 1;
+ }
+
+ __setup("hardened_usercopy=", parse_hardened_usercopy);
+diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
+index 7a90084155343..c9bab48856c65 100644
+--- a/mm/userfaultfd.c
++++ b/mm/userfaultfd.c
+@@ -151,6 +151,8 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm,
+ /* don't free the page */
+ goto out;
+ }
++
++ flush_dcache_page(page);
+ } else {
+ page = *pagep;
+ *pagep = NULL;
+@@ -621,6 +623,7 @@ retry:
+ err = -EFAULT;
+ goto out;
+ }
++ flush_dcache_page(page);
+ goto retry;
+ } else
+ BUG_ON(page);
+diff --git a/mm/util.c b/mm/util.c
+index bacabe4469065..3073de05c2bd4 100644
+--- a/mm/util.c
++++ b/mm/util.c
+@@ -343,6 +343,38 @@ unsigned long randomize_stack_top(unsigned long stack_top)
+ #endif
+ }
+
++/**
++ * randomize_page - Generate a random, page aligned address
++ * @start: The smallest acceptable address the caller will take.
++ * @range: The size of the area, starting at @start, within which the
++ * random address must fall.
++ *
++ * If @start + @range would overflow, @range is capped.
++ *
++ * NOTE: Historical use of randomize_range, which this replaces, presumed that
++ * @start was already page aligned. We now align it regardless.
++ *
++ * Return: A page aligned address within [start, start + range). On error,
++ * @start is returned.
++ */
++unsigned long randomize_page(unsigned long start, unsigned long range)
++{
++ if (!PAGE_ALIGNED(start)) {
++ range -= PAGE_ALIGN(start) - start;
++ start = PAGE_ALIGN(start);
++ }
++
++ if (start > ULONG_MAX - range)
++ range = ULONG_MAX - start;
++
++ range >>= PAGE_SHIFT;
++
++ if (range == 0)
++ return start;
++
++ return start + (get_random_long() % range << PAGE_SHIFT);
++}
++
+ #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
+ unsigned long arch_randomize_brk(struct mm_struct *mm)
+ {
+@@ -594,8 +626,10 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node)
+ return ret;
+
+ /* Don't even allow crazy sizes */
+- if (WARN_ON_ONCE(size > INT_MAX))
++ if (unlikely(size > INT_MAX)) {
++ WARN_ON_ONCE(!(flags & __GFP_NOWARN));
+ return NULL;
++ }
+
+ return __vmalloc_node(size, 1, flags, node,
+ __builtin_return_address(0));
+diff --git a/mm/vmalloc.c b/mm/vmalloc.c
+index e8a807c781107..8375eecc55de5 100644
+--- a/mm/vmalloc.c
++++ b/mm/vmalloc.c
+@@ -3032,7 +3032,8 @@ again:
+ clear_vm_uninitialized_flag(area);
+
+ size = PAGE_ALIGN(size);
+- kmemleak_vmalloc(area, size, gfp_mask);
++ if (!(vm_flags & VM_DEFER_KMEMLEAK))
++ kmemleak_vmalloc(area, size, gfp_mask);
+
+ return addr;
+
+diff --git a/mm/workingset.c b/mm/workingset.c
+index d5b81e4f4cbe8..880d882f3325f 100644
+--- a/mm/workingset.c
++++ b/mm/workingset.c
+@@ -352,7 +352,7 @@ void workingset_refault(struct page *page, void *shadow)
+
+ inc_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file);
+
+- mem_cgroup_flush_stats();
++ mem_cgroup_flush_stats_delayed();
+ /*
+ * Compare the distance to the existing workingset size. We
+ * don't activate pages that couldn't stay resident even if
+diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
+index 68e8831068f4b..b897ce3b399a1 100644
+--- a/mm/zsmalloc.c
++++ b/mm/zsmalloc.c
+@@ -1830,10 +1830,11 @@ static inline void zs_pool_dec_isolated(struct zs_pool *pool)
+ VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
+ atomic_long_dec(&pool->isolated_pages);
+ /*
+- * There's no possibility of racing, since wait_for_isolated_drain()
+- * checks the isolated count under &class->lock after enqueuing
+- * on migration_wait.
++ * Checking pool->destroying must happen after atomic_long_dec()
++ * for pool->isolated_pages above. Paired with the smp_mb() in
++ * zs_unregister_migration().
+ */
++ smp_mb__after_atomic();
+ if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
+ wake_up_all(&pool->migration_wait);
+ }
+diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c
+index 55275ef9a31a7..abaa5d96ded24 100644
+--- a/net/8021q/vlan.c
++++ b/net/8021q/vlan.c
+@@ -123,9 +123,6 @@ void unregister_vlan_dev(struct net_device *dev, struct list_head *head)
+ }
+
+ vlan_vid_del(real_dev, vlan->vlan_proto, vlan_id);
+-
+- /* Get rid of the vlan's reference to real_dev */
+- dev_put(real_dev);
+ }
+
+ int vlan_check_real_dev(struct net_device *real_dev,
+@@ -187,9 +184,6 @@ int register_vlan_dev(struct net_device *dev, struct netlink_ext_ack *extack)
+ if (err)
+ goto out_unregister_netdev;
+
+- /* Account for reference in struct vlan_dev_priv */
+- dev_hold(real_dev);
+-
+ vlan_stacked_transfer_operstate(real_dev, dev, vlan);
+ linkwatch_fire_event(dev); /* _MUST_ call rfc2863_policy() */
+
+diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
+index 0c21d1fec8522..8602885c8a8e0 100644
+--- a/net/8021q/vlan_dev.c
++++ b/net/8021q/vlan_dev.c
+@@ -615,6 +615,9 @@ static int vlan_dev_init(struct net_device *dev)
+ if (!vlan->vlan_pcpu_stats)
+ return -ENOMEM;
+
++ /* Get vlan's reference to real_dev */
++ dev_hold(real_dev);
++
+ return 0;
+ }
+
+@@ -843,6 +846,9 @@ static void vlan_dev_free(struct net_device *dev)
+
+ free_percpu(vlan->vlan_pcpu_stats);
+ vlan->vlan_pcpu_stats = NULL;
++
++ /* Get rid of the vlan's reference to real_dev */
++ dev_put(vlan->real_dev);
+ }
+
+ void vlan_setup(struct net_device *dev)
+diff --git a/net/9p/client.c b/net/9p/client.c
+index 213f12ed76cd8..7973267ec8464 100644
+--- a/net/9p/client.c
++++ b/net/9p/client.c
+@@ -541,6 +541,8 @@ static int p9_check_errors(struct p9_client *c, struct p9_req_t *req)
+ kfree(ename);
+ } else {
+ err = p9pdu_readf(&req->rc, c->proto_version, "d", &ecode);
++ if (err)
++ goto out_err;
+ err = -ecode;
+
+ p9_debug(P9_DEBUG_9P, "<<< RLERROR (%d)\n", -ecode);
+diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c
+index 3ec1a51a6944e..432ac5a16f2e0 100644
+--- a/net/9p/trans_xen.c
++++ b/net/9p/trans_xen.c
+@@ -304,9 +304,9 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv)
+ ref = priv->rings[i].intf->ref[j];
+ gnttab_end_foreign_access(ref, 0, 0);
+ }
+- free_pages((unsigned long)priv->rings[i].data.in,
+- priv->rings[i].intf->ring_order -
+- (PAGE_SHIFT - XEN_PAGE_SHIFT));
++ free_pages_exact(priv->rings[i].data.in,
++ 1UL << (priv->rings[i].intf->ring_order +
++ XEN_PAGE_SHIFT));
+ }
+ gnttab_end_foreign_access(priv->rings[i].ref, 0, 0);
+ free_page((unsigned long)priv->rings[i].intf);
+@@ -345,8 +345,8 @@ static int xen_9pfs_front_alloc_dataring(struct xenbus_device *dev,
+ if (ret < 0)
+ goto out;
+ ring->ref = ret;
+- bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+- order - (PAGE_SHIFT - XEN_PAGE_SHIFT));
++ bytes = alloc_pages_exact(1UL << (order + XEN_PAGE_SHIFT),
++ GFP_KERNEL | __GFP_ZERO);
+ if (!bytes) {
+ ret = -ENOMEM;
+ goto out;
+@@ -377,9 +377,7 @@ out:
+ if (bytes) {
+ for (i--; i >= 0; i--)
+ gnttab_end_foreign_access(ring->intf->ref[i], 0, 0);
+- free_pages((unsigned long)bytes,
+- ring->intf->ring_order -
+- (PAGE_SHIFT - XEN_PAGE_SHIFT));
++ free_pages_exact(bytes, 1UL << (order + XEN_PAGE_SHIFT));
+ }
+ gnttab_end_foreign_access(ring->ref, 0, 0);
+ free_page((unsigned long)ring->intf);
+diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c
+index 2631efc6e359f..7b69503dc46a2 100644
+--- a/net/ax25/af_ax25.c
++++ b/net/ax25/af_ax25.c
+@@ -77,6 +77,7 @@ static void ax25_kill_by_device(struct net_device *dev)
+ {
+ ax25_dev *ax25_dev;
+ ax25_cb *s;
++ struct sock *sk;
+
+ if ((ax25_dev = ax25_dev_ax25dev(dev)) == NULL)
+ return;
+@@ -85,11 +86,26 @@ static void ax25_kill_by_device(struct net_device *dev)
+ again:
+ ax25_for_each(s, &ax25_list) {
+ if (s->ax25_dev == ax25_dev) {
+- s->ax25_dev = NULL;
++ sk = s->sk;
++ if (!sk) {
++ spin_unlock_bh(&ax25_list_lock);
++ ax25_disconnect(s, ENETUNREACH);
++ s->ax25_dev = NULL;
++ spin_lock_bh(&ax25_list_lock);
++ goto again;
++ }
++ sock_hold(sk);
+ spin_unlock_bh(&ax25_list_lock);
++ lock_sock(sk);
+ ax25_disconnect(s, ENETUNREACH);
++ s->ax25_dev = NULL;
++ if (sk->sk_socket) {
++ dev_put(ax25_dev->dev);
++ ax25_dev_put(ax25_dev);
++ }
++ release_sock(sk);
+ spin_lock_bh(&ax25_list_lock);
+-
++ sock_put(sk);
+ /* The entry could have been deleted from the
+ * list meanwhile and thus the next pointer is
+ * no longer valid. Play it safe and restart
+@@ -353,21 +369,25 @@ static int ax25_ctl_ioctl(const unsigned int cmd, void __user *arg)
+ if (copy_from_user(&ax25_ctl, arg, sizeof(ax25_ctl)))
+ return -EFAULT;
+
+- if ((ax25_dev = ax25_addr_ax25dev(&ax25_ctl.port_addr)) == NULL)
+- return -ENODEV;
+-
+ if (ax25_ctl.digi_count > AX25_MAX_DIGIS)
+ return -EINVAL;
+
+ if (ax25_ctl.arg > ULONG_MAX / HZ && ax25_ctl.cmd != AX25_KILL)
+ return -EINVAL;
+
++ ax25_dev = ax25_addr_ax25dev(&ax25_ctl.port_addr);
++ if (!ax25_dev)
++ return -ENODEV;
++
+ digi.ndigi = ax25_ctl.digi_count;
+ for (k = 0; k < digi.ndigi; k++)
+ digi.calls[k] = ax25_ctl.digi_addr[k];
+
+- if ((ax25 = ax25_find_cb(&ax25_ctl.source_addr, &ax25_ctl.dest_addr, &digi, ax25_dev->dev)) == NULL)
++ ax25 = ax25_find_cb(&ax25_ctl.source_addr, &ax25_ctl.dest_addr, &digi, ax25_dev->dev);
++ if (!ax25) {
++ ax25_dev_put(ax25_dev);
+ return -ENOTCONN;
++ }
+
+ switch (ax25_ctl.cmd) {
+ case AX25_KILL:
+@@ -434,6 +454,7 @@ static int ax25_ctl_ioctl(const unsigned int cmd, void __user *arg)
+ }
+
+ out_put:
++ ax25_dev_put(ax25_dev);
+ ax25_cb_put(ax25);
+ return ret;
+
+@@ -534,7 +555,7 @@ static int ax25_setsockopt(struct socket *sock, int level, int optname,
+ ax25_cb *ax25;
+ struct net_device *dev;
+ char devname[IFNAMSIZ];
+- unsigned long opt;
++ unsigned int opt;
+ int res = 0;
+
+ if (level != SOL_AX25)
+@@ -566,7 +587,7 @@ static int ax25_setsockopt(struct socket *sock, int level, int optname,
+ break;
+
+ case AX25_T1:
+- if (opt < 1 || opt > ULONG_MAX / HZ) {
++ if (opt < 1 || opt > UINT_MAX / HZ) {
+ res = -EINVAL;
+ break;
+ }
+@@ -575,7 +596,7 @@ static int ax25_setsockopt(struct socket *sock, int level, int optname,
+ break;
+
+ case AX25_T2:
+- if (opt < 1 || opt > ULONG_MAX / HZ) {
++ if (opt < 1 || opt > UINT_MAX / HZ) {
+ res = -EINVAL;
+ break;
+ }
+@@ -591,7 +612,7 @@ static int ax25_setsockopt(struct socket *sock, int level, int optname,
+ break;
+
+ case AX25_T3:
+- if (opt < 1 || opt > ULONG_MAX / HZ) {
++ if (opt < 1 || opt > UINT_MAX / HZ) {
+ res = -EINVAL;
+ break;
+ }
+@@ -599,7 +620,7 @@ static int ax25_setsockopt(struct socket *sock, int level, int optname,
+ break;
+
+ case AX25_IDLE:
+- if (opt > ULONG_MAX / (60 * HZ)) {
++ if (opt > UINT_MAX / (60 * HZ)) {
+ res = -EINVAL;
+ break;
+ }
+@@ -960,14 +981,16 @@ static int ax25_release(struct socket *sock)
+ {
+ struct sock *sk = sock->sk;
+ ax25_cb *ax25;
++ ax25_dev *ax25_dev;
+
+ if (sk == NULL)
+ return 0;
+
+ sock_hold(sk);
+- sock_orphan(sk);
+ lock_sock(sk);
++ sock_orphan(sk);
+ ax25 = sk_to_ax25(sk);
++ ax25_dev = ax25->ax25_dev;
+
+ if (sk->sk_type == SOCK_SEQPACKET) {
+ switch (ax25->state) {
+@@ -1029,6 +1052,15 @@ static int ax25_release(struct socket *sock)
+ sk->sk_state_change(sk);
+ ax25_destroy_socket(ax25);
+ }
++ if (ax25_dev) {
++ del_timer_sync(&ax25->timer);
++ del_timer_sync(&ax25->t1timer);
++ del_timer_sync(&ax25->t2timer);
++ del_timer_sync(&ax25->t3timer);
++ del_timer_sync(&ax25->idletimer);
++ dev_put(ax25_dev->dev);
++ ax25_dev_put(ax25_dev);
++ }
+
+ sock->sk = NULL;
+ release_sock(sk);
+@@ -1105,8 +1137,10 @@ static int ax25_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
+ }
+ }
+
+- if (ax25_dev != NULL)
++ if (ax25_dev) {
+ ax25_fillin_cb(ax25, ax25_dev);
++ dev_hold(ax25_dev->dev);
++ }
+
+ done:
+ ax25_cb_add(ax25);
+diff --git a/net/ax25/ax25_dev.c b/net/ax25/ax25_dev.c
+index 4ac2e0847652a..d2e0cc67d91a7 100644
+--- a/net/ax25/ax25_dev.c
++++ b/net/ax25/ax25_dev.c
+@@ -37,6 +37,7 @@ ax25_dev *ax25_addr_ax25dev(ax25_address *addr)
+ for (ax25_dev = ax25_dev_list; ax25_dev != NULL; ax25_dev = ax25_dev->next)
+ if (ax25cmp(addr, (ax25_address *)ax25_dev->dev->dev_addr) == 0) {
+ res = ax25_dev;
++ ax25_dev_hold(ax25_dev);
+ }
+ spin_unlock_bh(&ax25_dev_lock);
+
+@@ -56,6 +57,7 @@ void ax25_dev_device_up(struct net_device *dev)
+ return;
+ }
+
++ refcount_set(&ax25_dev->refcount, 1);
+ dev->ax25_ptr = ax25_dev;
+ ax25_dev->dev = dev;
+ dev_hold(dev);
+@@ -84,6 +86,7 @@ void ax25_dev_device_up(struct net_device *dev)
+ ax25_dev->next = ax25_dev_list;
+ ax25_dev_list = ax25_dev;
+ spin_unlock_bh(&ax25_dev_lock);
++ ax25_dev_hold(ax25_dev);
+
+ ax25_register_dev_sysctl(ax25_dev);
+ }
+@@ -113,9 +116,10 @@ void ax25_dev_device_down(struct net_device *dev)
+ if ((s = ax25_dev_list) == ax25_dev) {
+ ax25_dev_list = s->next;
+ spin_unlock_bh(&ax25_dev_lock);
++ ax25_dev_put(ax25_dev);
+ dev->ax25_ptr = NULL;
+ dev_put(dev);
+- kfree(ax25_dev);
++ ax25_dev_put(ax25_dev);
+ return;
+ }
+
+@@ -123,9 +127,10 @@ void ax25_dev_device_down(struct net_device *dev)
+ if (s->next == ax25_dev) {
+ s->next = ax25_dev->next;
+ spin_unlock_bh(&ax25_dev_lock);
++ ax25_dev_put(ax25_dev);
+ dev->ax25_ptr = NULL;
+ dev_put(dev);
+- kfree(ax25_dev);
++ ax25_dev_put(ax25_dev);
+ return;
+ }
+
+@@ -133,6 +138,7 @@ void ax25_dev_device_down(struct net_device *dev)
+ }
+ spin_unlock_bh(&ax25_dev_lock);
+ dev->ax25_ptr = NULL;
++ ax25_dev_put(ax25_dev);
+ }
+
+ int ax25_fwd_ioctl(unsigned int cmd, struct ax25_fwd_struct *fwd)
+@@ -144,20 +150,32 @@ int ax25_fwd_ioctl(unsigned int cmd, struct ax25_fwd_struct *fwd)
+
+ switch (cmd) {
+ case SIOCAX25ADDFWD:
+- if ((fwd_dev = ax25_addr_ax25dev(&fwd->port_to)) == NULL)
++ fwd_dev = ax25_addr_ax25dev(&fwd->port_to);
++ if (!fwd_dev) {
++ ax25_dev_put(ax25_dev);
+ return -EINVAL;
+- if (ax25_dev->forward != NULL)
++ }
++ if (ax25_dev->forward) {
++ ax25_dev_put(fwd_dev);
++ ax25_dev_put(ax25_dev);
+ return -EINVAL;
++ }
+ ax25_dev->forward = fwd_dev->dev;
++ ax25_dev_put(fwd_dev);
++ ax25_dev_put(ax25_dev);
+ break;
+
+ case SIOCAX25DELFWD:
+- if (ax25_dev->forward == NULL)
++ if (!ax25_dev->forward) {
++ ax25_dev_put(ax25_dev);
+ return -EINVAL;
++ }
+ ax25_dev->forward = NULL;
++ ax25_dev_put(ax25_dev);
+ break;
+
+ default:
++ ax25_dev_put(ax25_dev);
+ return -EINVAL;
+ }
+
+diff --git a/net/ax25/ax25_route.c b/net/ax25/ax25_route.c
+index d0b2e094bd552..9751207f77572 100644
+--- a/net/ax25/ax25_route.c
++++ b/net/ax25/ax25_route.c
+@@ -75,11 +75,13 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+ ax25_dev *ax25_dev;
+ int i;
+
+- if ((ax25_dev = ax25_addr_ax25dev(&route->port_addr)) == NULL)
+- return -EINVAL;
+ if (route->digi_count > AX25_MAX_DIGIS)
+ return -EINVAL;
+
++ ax25_dev = ax25_addr_ax25dev(&route->port_addr);
++ if (!ax25_dev)
++ return -EINVAL;
++
+ write_lock_bh(&ax25_route_lock);
+
+ ax25_rt = ax25_route_list;
+@@ -91,6 +93,7 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+ if (route->digi_count != 0) {
+ if ((ax25_rt->digipeat = kmalloc(sizeof(ax25_digi), GFP_ATOMIC)) == NULL) {
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+ return -ENOMEM;
+ }
+ ax25_rt->digipeat->lastrepeat = -1;
+@@ -101,6 +104,7 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+ }
+ }
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+ return 0;
+ }
+ ax25_rt = ax25_rt->next;
+@@ -108,6 +112,7 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+
+ if ((ax25_rt = kmalloc(sizeof(ax25_route), GFP_ATOMIC)) == NULL) {
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+ return -ENOMEM;
+ }
+
+@@ -120,6 +125,7 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+ if ((ax25_rt->digipeat = kmalloc(sizeof(ax25_digi), GFP_ATOMIC)) == NULL) {
+ write_unlock_bh(&ax25_route_lock);
+ kfree(ax25_rt);
++ ax25_dev_put(ax25_dev);
+ return -ENOMEM;
+ }
+ ax25_rt->digipeat->lastrepeat = -1;
+@@ -132,6 +138,7 @@ static int __must_check ax25_rt_add(struct ax25_routes_struct *route)
+ ax25_rt->next = ax25_route_list;
+ ax25_route_list = ax25_rt;
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+
+ return 0;
+ }
+@@ -173,6 +180,7 @@ static int ax25_rt_del(struct ax25_routes_struct *route)
+ }
+ }
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+
+ return 0;
+ }
+@@ -215,6 +223,7 @@ static int ax25_rt_opt(struct ax25_route_opt_struct *rt_option)
+
+ out:
+ write_unlock_bh(&ax25_route_lock);
++ ax25_dev_put(ax25_dev);
+ return err;
+ }
+
+diff --git a/net/ax25/ax25_subr.c b/net/ax25/ax25_subr.c
+index 15ab812c4fe4b..3a476e4f6cd0b 100644
+--- a/net/ax25/ax25_subr.c
++++ b/net/ax25/ax25_subr.c
+@@ -261,12 +261,20 @@ void ax25_disconnect(ax25_cb *ax25, int reason)
+ {
+ ax25_clear_queues(ax25);
+
+- if (!ax25->sk || !sock_flag(ax25->sk, SOCK_DESTROY))
+- ax25_stop_heartbeat(ax25);
+- ax25_stop_t1timer(ax25);
+- ax25_stop_t2timer(ax25);
+- ax25_stop_t3timer(ax25);
+- ax25_stop_idletimer(ax25);
++ if (reason == ENETUNREACH) {
++ del_timer_sync(&ax25->timer);
++ del_timer_sync(&ax25->t1timer);
++ del_timer_sync(&ax25->t2timer);
++ del_timer_sync(&ax25->t3timer);
++ del_timer_sync(&ax25->idletimer);
++ } else {
++ if (!ax25->sk || !sock_flag(ax25->sk, SOCK_DESTROY))
++ ax25_stop_heartbeat(ax25);
++ ax25_stop_t1timer(ax25);
++ ax25_stop_t2timer(ax25);
++ ax25_stop_t3timer(ax25);
++ ax25_stop_idletimer(ax25);
++ }
+
+ ax25->state = AX25_STATE_0;
+
+diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c
+index 0899a729a23f4..c120c7c6d25fc 100644
+--- a/net/batman-adv/fragmentation.c
++++ b/net/batman-adv/fragmentation.c
+@@ -475,6 +475,17 @@ int batadv_frag_send_packet(struct sk_buff *skb,
+ goto free_skb;
+ }
+
++ /* GRO might have added fragments to the fragment list instead of
++ * frags[]. But this is not handled by skb_split and must be
++ * linearized to avoid incorrect length information after all
++ * batman-adv fragments were created and submitted to the
++ * hard-interface
++ */
++ if (skb_has_frag_list(skb) && __skb_linearize(skb)) {
++ ret = -ENOMEM;
++ goto free_skb;
++ }
++
+ /* Create one header to be copied to all fragments */
+ frag_header.packet_type = BATADV_UNICAST_FRAG;
+ frag_header.version = BATADV_COMPAT_VERSION;
+diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c
+index 8a2b78f9c4b2c..35fadb9248498 100644
+--- a/net/batman-adv/hard-interface.c
++++ b/net/batman-adv/hard-interface.c
+@@ -149,22 +149,25 @@ static bool batadv_is_on_batman_iface(const struct net_device *net_dev)
+ struct net *net = dev_net(net_dev);
+ struct net_device *parent_dev;
+ struct net *parent_net;
++ int iflink;
+ bool ret;
+
+ /* check if this is a batman-adv mesh interface */
+ if (batadv_softif_is_valid(net_dev))
+ return true;
+
+- /* no more parents..stop recursion */
+- if (dev_get_iflink(net_dev) == 0 ||
+- dev_get_iflink(net_dev) == net_dev->ifindex)
++ iflink = dev_get_iflink(net_dev);
++ if (iflink == 0)
+ return false;
+
+ parent_net = batadv_getlink_net(net_dev, net);
+
++ /* iflink to itself, most likely physical device */
++ if (net == parent_net && iflink == net_dev->ifindex)
++ return false;
++
+ /* recurse over the parent device */
+- parent_dev = __dev_get_by_index((struct net *)parent_net,
+- dev_get_iflink(net_dev));
++ parent_dev = __dev_get_by_index((struct net *)parent_net, iflink);
+ /* if we got a NULL parent_dev there is something broken.. */
+ if (!parent_dev) {
+ pr_err("Cannot find parent device\n");
+@@ -214,14 +217,15 @@ static struct net_device *batadv_get_real_netdevice(struct net_device *netdev)
+ struct net_device *real_netdev = NULL;
+ struct net *real_net;
+ struct net *net;
+- int ifindex;
++ int iflink;
+
+ ASSERT_RTNL();
+
+ if (!netdev)
+ return NULL;
+
+- if (netdev->ifindex == dev_get_iflink(netdev)) {
++ iflink = dev_get_iflink(netdev);
++ if (iflink == 0) {
+ dev_hold(netdev);
+ return netdev;
+ }
+@@ -231,9 +235,16 @@ static struct net_device *batadv_get_real_netdevice(struct net_device *netdev)
+ goto out;
+
+ net = dev_net(hard_iface->soft_iface);
+- ifindex = dev_get_iflink(netdev);
+ real_net = batadv_getlink_net(netdev, net);
+- real_netdev = dev_get_by_index(real_net, ifindex);
++
++ /* iflink to itself, most likely physical device */
++ if (net == real_net && netdev->ifindex == iflink) {
++ real_netdev = netdev;
++ dev_hold(real_netdev);
++ goto out;
++ }
++
++ real_netdev = dev_get_by_index(real_net, iflink);
+
+ out:
+ batadv_hardif_put(hard_iface);
+diff --git a/net/batman-adv/multicast.c b/net/batman-adv/multicast.c
+index a3b6658ed7899..2853634a39790 100644
+--- a/net/batman-adv/multicast.c
++++ b/net/batman-adv/multicast.c
+@@ -134,7 +134,7 @@ static u8 batadv_mcast_mla_rtr_flags_softif_get_ipv6(struct net_device *dev)
+ {
+ struct inet6_dev *in6_dev = __in6_dev_get(dev);
+
+- if (in6_dev && in6_dev->cnf.mc_forwarding)
++ if (in6_dev && atomic_read(&in6_dev->cnf.mc_forwarding))
+ return BATADV_NO_FLAGS;
+ else
+ return BATADV_MCAST_WANT_NO_RTR6;
+@@ -1339,6 +1339,7 @@ batadv_mcast_forw_rtr_node_get(struct batadv_priv *bat_priv,
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: The multicast packet to check
+ * @orig: an originator to be set to forward the skb to
++ * @is_routable: stores whether the destination is routable
+ *
+ * Return: the forwarding mode as enum batadv_forw_mode and in case of
+ * BATADV_FORW_SINGLE set the orig to the single originator the skb
+@@ -1346,17 +1347,16 @@ batadv_mcast_forw_rtr_node_get(struct batadv_priv *bat_priv,
+ */
+ enum batadv_forw_mode
+ batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- struct batadv_orig_node **orig)
++ struct batadv_orig_node **orig, int *is_routable)
+ {
+ int ret, tt_count, ip_count, unsnoop_count, total_count;
+ bool is_unsnoopable = false;
+ unsigned int mcast_fanout;
+ struct ethhdr *ethhdr;
+- int is_routable = 0;
+ int rtr_count = 0;
+
+ ret = batadv_mcast_forw_mode_check(bat_priv, skb, &is_unsnoopable,
+- &is_routable);
++ is_routable);
+ if (ret == -ENOMEM)
+ return BATADV_FORW_NONE;
+ else if (ret < 0)
+@@ -1369,7 +1369,7 @@ batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
+ ip_count = batadv_mcast_forw_want_all_ip_count(bat_priv, ethhdr);
+ unsnoop_count = !is_unsnoopable ? 0 :
+ atomic_read(&bat_priv->mcast.num_want_all_unsnoopables);
+- rtr_count = batadv_mcast_forw_rtr_count(bat_priv, is_routable);
++ rtr_count = batadv_mcast_forw_rtr_count(bat_priv, *is_routable);
+
+ total_count = tt_count + ip_count + unsnoop_count + rtr_count;
+
+@@ -1689,6 +1689,7 @@ batadv_mcast_forw_want_rtr(struct batadv_priv *bat_priv,
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the multicast packet to transmit
+ * @vid: the vlan identifier
++ * @is_routable: stores whether the destination is routable
+ *
+ * Sends copies of a frame with multicast destination to any node that signaled
+ * interest in it, that is either via the translation table or the according
+@@ -1701,7 +1702,7 @@ batadv_mcast_forw_want_rtr(struct batadv_priv *bat_priv,
+ * is neither IPv4 nor IPv6. NET_XMIT_SUCCESS otherwise.
+ */
+ int batadv_mcast_forw_send(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- unsigned short vid)
++ unsigned short vid, int is_routable)
+ {
+ int ret;
+
+@@ -1717,12 +1718,16 @@ int batadv_mcast_forw_send(struct batadv_priv *bat_priv, struct sk_buff *skb,
+ return ret;
+ }
+
++ if (!is_routable)
++ goto skip_mc_router;
++
+ ret = batadv_mcast_forw_want_rtr(bat_priv, skb, vid);
+ if (ret != NET_XMIT_SUCCESS) {
+ kfree_skb(skb);
+ return ret;
+ }
+
++skip_mc_router:
+ consume_skb(skb);
+ return ret;
+ }
+diff --git a/net/batman-adv/multicast.h b/net/batman-adv/multicast.h
+index 9fee5da083113..8aec818d0bf63 100644
+--- a/net/batman-adv/multicast.h
++++ b/net/batman-adv/multicast.h
+@@ -43,7 +43,8 @@ enum batadv_forw_mode {
+
+ enum batadv_forw_mode
+ batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- struct batadv_orig_node **mcast_single_orig);
++ struct batadv_orig_node **mcast_single_orig,
++ int *is_routable);
+
+ int batadv_mcast_forw_send_orig(struct batadv_priv *bat_priv,
+ struct sk_buff *skb,
+@@ -51,7 +52,7 @@ int batadv_mcast_forw_send_orig(struct batadv_priv *bat_priv,
+ struct batadv_orig_node *orig_node);
+
+ int batadv_mcast_forw_send(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- unsigned short vid);
++ unsigned short vid, int is_routable);
+
+ void batadv_mcast_init(struct batadv_priv *bat_priv);
+
+@@ -68,7 +69,8 @@ void batadv_mcast_purge_orig(struct batadv_orig_node *orig_node);
+
+ static inline enum batadv_forw_mode
+ batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- struct batadv_orig_node **mcast_single_orig)
++ struct batadv_orig_node **mcast_single_orig,
++ int *is_routable)
+ {
+ return BATADV_FORW_ALL;
+ }
+@@ -85,7 +87,7 @@ batadv_mcast_forw_send_orig(struct batadv_priv *bat_priv,
+
+ static inline int
+ batadv_mcast_forw_send(struct batadv_priv *bat_priv, struct sk_buff *skb,
+- unsigned short vid)
++ unsigned short vid, int is_routable)
+ {
+ kfree_skb(skb);
+ return NET_XMIT_DROP;
+diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c
+index 29276284d281c..00875e1d8c44c 100644
+--- a/net/batman-adv/netlink.c
++++ b/net/batman-adv/netlink.c
+@@ -1368,21 +1368,21 @@ static const struct genl_small_ops batadv_netlink_ops[] = {
+ {
+ .cmd = BATADV_CMD_TP_METER,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .doit = batadv_netlink_tp_meter_start,
+ .internal_flags = BATADV_FLAG_NEED_MESH,
+ },
+ {
+ .cmd = BATADV_CMD_TP_METER_CANCEL,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .doit = batadv_netlink_tp_meter_cancel,
+ .internal_flags = BATADV_FLAG_NEED_MESH,
+ },
+ {
+ .cmd = BATADV_CMD_GET_ROUTING_ALGOS,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_algo_dump,
+ },
+ {
+@@ -1397,68 +1397,68 @@ static const struct genl_small_ops batadv_netlink_ops[] = {
+ {
+ .cmd = BATADV_CMD_GET_TRANSTABLE_LOCAL,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_tt_local_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_TRANSTABLE_GLOBAL,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_tt_global_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_ORIGINATORS,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_orig_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_NEIGHBORS,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_hardif_neigh_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_GATEWAYS,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_gw_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_BLA_CLAIM,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_bla_claim_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_BLA_BACKBONE,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_bla_backbone_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_DAT_CACHE,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_dat_cache_dump,
+ },
+ {
+ .cmd = BATADV_CMD_GET_MCAST_FLAGS,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .dumpit = batadv_mcast_flags_dump,
+ },
+ {
+ .cmd = BATADV_CMD_SET_MESH,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .doit = batadv_netlink_set_mesh,
+ .internal_flags = BATADV_FLAG_NEED_MESH,
+ },
+ {
+ .cmd = BATADV_CMD_SET_HARDIF,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .doit = batadv_netlink_set_hardif,
+ .internal_flags = BATADV_FLAG_NEED_MESH |
+ BATADV_FLAG_NEED_HARDIF,
+@@ -1474,7 +1474,7 @@ static const struct genl_small_ops batadv_netlink_ops[] = {
+ {
+ .cmd = BATADV_CMD_SET_VLAN,
+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+- .flags = GENL_ADMIN_PERM,
++ .flags = GENL_UNS_ADMIN_PERM,
+ .doit = batadv_netlink_set_vlan,
+ .internal_flags = BATADV_FLAG_NEED_MESH |
+ BATADV_FLAG_NEED_VLAN,
+diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c
+index 0604b02795731..6ab28b509d4bb 100644
+--- a/net/batman-adv/soft-interface.c
++++ b/net/batman-adv/soft-interface.c
+@@ -198,6 +198,7 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
+ int gw_mode;
+ enum batadv_forw_mode forw_mode = BATADV_FORW_SINGLE;
+ struct batadv_orig_node *mcast_single_orig = NULL;
++ int mcast_is_routable = 0;
+ int network_offset = ETH_HLEN;
+ __be16 proto;
+
+@@ -300,7 +301,8 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
+ send:
+ if (do_bcast && !is_broadcast_ether_addr(ethhdr->h_dest)) {
+ forw_mode = batadv_mcast_forw_mode(bat_priv, skb,
+- &mcast_single_orig);
++ &mcast_single_orig,
++ &mcast_is_routable);
+ if (forw_mode == BATADV_FORW_NONE)
+ goto dropped;
+
+@@ -359,7 +361,8 @@ send:
+ ret = batadv_mcast_forw_send_orig(bat_priv, skb, vid,
+ mcast_single_orig);
+ } else if (forw_mode == BATADV_FORW_SOME) {
+- ret = batadv_mcast_forw_send(bat_priv, skb, vid);
++ ret = batadv_mcast_forw_send(bat_priv, skb, vid,
++ mcast_is_routable);
+ } else {
+ if (batadv_dat_snoop_outgoing_arp_request(bat_priv,
+ skb))
+diff --git a/net/bluetooth/cmtp/core.c b/net/bluetooth/cmtp/core.c
+index 0a2d78e811cf5..83eb84e8e688f 100644
+--- a/net/bluetooth/cmtp/core.c
++++ b/net/bluetooth/cmtp/core.c
+@@ -501,9 +501,7 @@ static int __init cmtp_init(void)
+ {
+ BT_INFO("CMTP (CAPI Emulation) ver %s", VERSION);
+
+- cmtp_init_sockets();
+-
+- return 0;
++ return cmtp_init_sockets();
+ }
+
+ static void __exit cmtp_exit(void)
+diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
+index 2b5059a56cdaa..7a7e92be1652c 100644
+--- a/net/bluetooth/hci_conn.c
++++ b/net/bluetooth/hci_conn.c
+@@ -541,7 +541,9 @@ static void le_conn_timeout(struct work_struct *work)
+ if (conn->role == HCI_ROLE_SLAVE) {
+ /* Disable LE Advertising */
+ le_disable_advertising(hdev);
++ hci_dev_lock(hdev);
+ hci_le_conn_failed(conn, HCI_ERROR_ADVERTISING_TIMEOUT);
++ hci_dev_unlock(hdev);
+ return;
+ }
+
+diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
+index 8a47a3017d61d..cdca53732304c 100644
+--- a/net/bluetooth/hci_core.c
++++ b/net/bluetooth/hci_core.c
+@@ -742,7 +742,8 @@ static int hci_init3_req(struct hci_request *req, unsigned long opt)
+ hci_req_add(req, HCI_OP_LE_READ_ADV_TX_POWER, 0, NULL);
+ }
+
+- if (hdev->commands[38] & 0x80) {
++ if ((hdev->commands[38] & 0x80) &&
++ !test_bit(HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER, &hdev->quirks)) {
+ /* Read LE Min/Max Tx Power*/
+ hci_req_add(req, HCI_OP_LE_READ_TRANSMIT_POWER,
+ 0, NULL);
+@@ -3906,10 +3907,10 @@ int hci_register_dev(struct hci_dev *hdev)
+ */
+ switch (hdev->dev_type) {
+ case HCI_PRIMARY:
+- id = ida_simple_get(&hci_index_ida, 0, 0, GFP_KERNEL);
++ id = ida_simple_get(&hci_index_ida, 0, HCI_MAX_ID, GFP_KERNEL);
+ break;
+ case HCI_AMP:
+- id = ida_simple_get(&hci_index_ida, 1, 0, GFP_KERNEL);
++ id = ida_simple_get(&hci_index_ida, 1, HCI_MAX_ID, GFP_KERNEL);
+ break;
+ default:
+ return -EINVAL;
+@@ -3918,7 +3919,7 @@ int hci_register_dev(struct hci_dev *hdev)
+ if (id < 0)
+ return id;
+
+- sprintf(hdev->name, "hci%d", id);
++ snprintf(hdev->name, sizeof(hdev->name), "hci%d", id);
+ hdev->id = id;
+
+ BT_DBG("%p name %s bus %d", hdev, hdev->name, hdev->bus);
+@@ -3998,6 +3999,7 @@ int hci_register_dev(struct hci_dev *hdev)
+ return id;
+
+ err_wqueue:
++ debugfs_remove_recursive(hdev->debugfs);
+ destroy_workqueue(hdev->workqueue);
+ destroy_workqueue(hdev->req_workqueue);
+ err:
+@@ -4081,6 +4083,7 @@ void hci_release_dev(struct hci_dev *hdev)
+ hci_dev_unlock(hdev);
+
+ ida_simple_remove(&hci_index_ida, hdev->id);
++ kfree_skb(hdev->sent_cmd);
+ kfree(hdev);
+ }
+ EXPORT_SYMBOL(hci_release_dev);
+diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
+index 0bca035bf2dcc..e984a8b4b9143 100644
+--- a/net/bluetooth/hci_event.c
++++ b/net/bluetooth/hci_event.c
+@@ -1325,8 +1325,10 @@ static void hci_cc_le_set_ext_adv_enable(struct hci_dev *hdev,
+ &conn->le_conn_timeout,
+ conn->conn_timeout);
+ } else {
+- if (adv) {
+- adv->enabled = false;
++ if (cp->num_of_sets) {
++ if (adv)
++ adv->enabled = false;
++
+ /* If just one instance was disabled check if there are
+ * any other instance enabled before clearing HCI_LE_ADV
+ */
+@@ -5151,8 +5153,9 @@ static void hci_disconn_phylink_complete_evt(struct hci_dev *hdev,
+ hci_dev_lock(hdev);
+
+ hcon = hci_conn_hash_lookup_handle(hdev, ev->phy_handle);
+- if (hcon) {
++ if (hcon && hcon->type == AMP_LINK) {
+ hcon->state = BT_CLOSED;
++ hci_disconn_cfm(hcon, ev->reason);
+ hci_conn_del(hcon);
+ }
+
+@@ -5780,7 +5783,13 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, struct sk_buff *skb)
+ struct hci_ev_le_advertising_info *ev = ptr;
+ s8 rssi;
+
+- if (ev->length <= HCI_MAX_AD_LENGTH) {
++ if (ptr > (void *)skb_tail_pointer(skb) - sizeof(*ev)) {
++ bt_dev_err(hdev, "Malicious advertising data.");
++ break;
++ }
++
++ if (ev->length <= HCI_MAX_AD_LENGTH &&
++ ev->data + ev->length <= skb_tail_pointer(skb)) {
+ rssi = ev->data[ev->length];
+ process_adv_report(hdev, ev->evt_type, &ev->bdaddr,
+ ev->bdaddr_type, NULL, 0, rssi,
+diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c
+index f15626607b2d6..1d34d330afd34 100644
+--- a/net/bluetooth/hci_request.c
++++ b/net/bluetooth/hci_request.c
+@@ -2318,7 +2318,7 @@ int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance)
+ /* Set duration per instance since controller is responsible for
+ * scheduling it.
+ */
+- if (adv_instance && adv_instance->duration) {
++ if (adv_instance && adv_instance->timeout) {
+ u16 duration = adv_instance->timeout * MSEC_PER_SEC;
+
+ /* Time = N * 10 ms */
+diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c
+index 7827639ecf5c3..4e3e0451b08c1 100644
+--- a/net/bluetooth/hci_sysfs.c
++++ b/net/bluetooth/hci_sysfs.c
+@@ -86,6 +86,8 @@ static void bt_host_release(struct device *dev)
+
+ if (hci_dev_test_flag(hdev, HCI_UNREGISTER))
+ hci_release_dev(hdev);
++ else
++ kfree(hdev);
+ module_put(THIS_MODULE);
+ }
+
+diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
+index 77ba68209dbd8..c57a45df7a267 100644
+--- a/net/bluetooth/l2cap_core.c
++++ b/net/bluetooth/l2cap_core.c
+@@ -1436,6 +1436,7 @@ static void l2cap_ecred_connect(struct l2cap_chan *chan)
+
+ l2cap_ecred_init(chan, 0);
+
++ memset(&data, 0, sizeof(data));
+ data.pdu.req.psm = chan->psm;
+ data.pdu.req.mtu = cpu_to_le16(chan->imtu);
+ data.pdu.req.mps = cpu_to_le16(chan->mps);
+diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
+index c99d65ef13b1e..d2c6785205992 100644
+--- a/net/bluetooth/l2cap_sock.c
++++ b/net/bluetooth/l2cap_sock.c
+@@ -161,7 +161,11 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr *addr, int alen)
+ break;
+ }
+
+- if (chan->psm && bdaddr_type_is_le(chan->src_type))
++ /* Use L2CAP_MODE_LE_FLOWCTL (CoC) in case of LE address and
++ * L2CAP_MODE_EXT_FLOWCTL (ECRED) has not been set.
++ */
++ if (chan->psm && bdaddr_type_is_le(chan->src_type) &&
++ chan->mode != L2CAP_MODE_EXT_FLOWCTL)
+ chan->mode = L2CAP_MODE_LE_FLOWCTL;
+
+ chan->state = BT_BOUND;
+@@ -172,6 +176,21 @@ done:
+ return err;
+ }
+
++static void l2cap_sock_init_pid(struct sock *sk)
++{
++ struct l2cap_chan *chan = l2cap_pi(sk)->chan;
++
++ /* Only L2CAP_MODE_EXT_FLOWCTL ever need to access the PID in order to
++ * group the channels being requested.
++ */
++ if (chan->mode != L2CAP_MODE_EXT_FLOWCTL)
++ return;
++
++ spin_lock(&sk->sk_peer_lock);
++ sk->sk_peer_pid = get_pid(task_tgid(current));
++ spin_unlock(&sk->sk_peer_lock);
++}
++
+ static int l2cap_sock_connect(struct socket *sock, struct sockaddr *addr,
+ int alen, int flags)
+ {
+@@ -240,9 +259,15 @@ static int l2cap_sock_connect(struct socket *sock, struct sockaddr *addr,
+ return -EINVAL;
+ }
+
+- if (chan->psm && bdaddr_type_is_le(chan->src_type) && !chan->mode)
++ /* Use L2CAP_MODE_LE_FLOWCTL (CoC) in case of LE address and
++ * L2CAP_MODE_EXT_FLOWCTL (ECRED) has not been set.
++ */
++ if (chan->psm && bdaddr_type_is_le(chan->src_type) &&
++ chan->mode != L2CAP_MODE_EXT_FLOWCTL)
+ chan->mode = L2CAP_MODE_LE_FLOWCTL;
+
++ l2cap_sock_init_pid(sk);
++
+ err = l2cap_chan_connect(chan, la.l2_psm, __le16_to_cpu(la.l2_cid),
+ &la.l2_bdaddr, la.l2_bdaddr_type);
+ if (err)
+@@ -298,6 +323,8 @@ static int l2cap_sock_listen(struct socket *sock, int backlog)
+ goto done;
+ }
+
++ l2cap_sock_init_pid(sk);
++
+ sk->sk_max_ack_backlog = backlog;
+ sk->sk_ack_backlog = 0;
+
+@@ -876,6 +903,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
+ struct l2cap_conn *conn;
+ int len, err = 0;
+ u32 opt;
++ u16 mtu;
++ u8 mode;
+
+ BT_DBG("sk %p", sk);
+
+@@ -1058,16 +1087,16 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
+ break;
+ }
+
+- if (copy_from_sockptr(&opt, optval, sizeof(u16))) {
++ if (copy_from_sockptr(&mtu, optval, sizeof(u16))) {
+ err = -EFAULT;
+ break;
+ }
+
+ if (chan->mode == L2CAP_MODE_EXT_FLOWCTL &&
+ sk->sk_state == BT_CONNECTED)
+- err = l2cap_chan_reconfigure(chan, opt);
++ err = l2cap_chan_reconfigure(chan, mtu);
+ else
+- chan->imtu = opt;
++ chan->imtu = mtu;
+
+ break;
+
+@@ -1089,14 +1118,14 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
+ break;
+ }
+
+- if (copy_from_sockptr(&opt, optval, sizeof(u8))) {
++ if (copy_from_sockptr(&mode, optval, sizeof(u8))) {
+ err = -EFAULT;
+ break;
+ }
+
+- BT_DBG("opt %u", opt);
++ BT_DBG("mode %u", mode);
+
+- err = l2cap_set_mode(chan, opt);
++ err = l2cap_set_mode(chan, mode);
+ if (err)
+ break;
+
+@@ -1508,6 +1537,9 @@ static void l2cap_sock_close_cb(struct l2cap_chan *chan)
+ {
+ struct sock *sk = chan->data;
+
++ if (!sk)
++ return;
++
+ l2cap_sock_kill(sk);
+ }
+
+@@ -1516,6 +1548,9 @@ static void l2cap_sock_teardown_cb(struct l2cap_chan *chan, int err)
+ struct sock *sk = chan->data;
+ struct sock *parent;
+
++ if (!sk)
++ return;
++
+ BT_DBG("chan %p state %s", chan, state_to_string(chan->state));
+
+ /* This callback can be called both for server (BT_LISTEN)
+@@ -1707,8 +1742,10 @@ static void l2cap_sock_destruct(struct sock *sk)
+ {
+ BT_DBG("sk %p", sk);
+
+- if (l2cap_pi(sk)->chan)
++ if (l2cap_pi(sk)->chan) {
++ l2cap_pi(sk)->chan->data = NULL;
+ l2cap_chan_put(l2cap_pi(sk)->chan);
++ }
+
+ if (l2cap_pi(sk)->rx_busy_skb) {
+ kfree_skb(l2cap_pi(sk)->rx_busy_skb);
+diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
+index cea01e275f1ea..f09f0a78eb7be 100644
+--- a/net/bluetooth/mgmt.c
++++ b/net/bluetooth/mgmt.c
+@@ -3806,7 +3806,7 @@ static const u8 rpa_resolution_uuid[16] = {
+ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev,
+ void *data, u16 data_len)
+ {
+- char buf[62]; /* Enough space for 3 features */
++ char buf[62]; /* Enough space for 3 features */
+ struct mgmt_rp_read_exp_features_info *rp = (void *)buf;
+ u16 idx = 0;
+ u32 flags;
+@@ -3892,150 +3892,186 @@ static int exp_debug_feature_changed(bool enabled, struct sock *skip)
+ }
+ #endif
+
+-static int set_exp_feature(struct sock *sk, struct hci_dev *hdev,
+- void *data, u16 data_len)
++#define EXP_FEAT(_uuid, _set_func) \
++{ \
++ .uuid = _uuid, \
++ .set_func = _set_func, \
++}
++
++/* The zero key uuid is special. Multiple exp features are set through it. */
++static int set_zero_key_func(struct sock *sk, struct hci_dev *hdev,
++ struct mgmt_cp_set_exp_feature *cp, u16 data_len)
+ {
+- struct mgmt_cp_set_exp_feature *cp = data;
+ struct mgmt_rp_set_exp_feature rp;
+
+- bt_dev_dbg(hdev, "sock %p", sk);
+-
+- if (!memcmp(cp->uuid, ZERO_KEY, 16)) {
+- memset(rp.uuid, 0, 16);
+- rp.flags = cpu_to_le32(0);
++ memset(rp.uuid, 0, 16);
++ rp.flags = cpu_to_le32(0);
+
+ #ifdef CONFIG_BT_FEATURE_DEBUG
+- if (!hdev) {
+- bool changed = bt_dbg_get();
++ if (!hdev) {
++ bool changed = bt_dbg_get();
+
+- bt_dbg_set(false);
++ bt_dbg_set(false);
+
+- if (changed)
+- exp_debug_feature_changed(false, sk);
+- }
++ if (changed)
++ exp_debug_feature_changed(false, sk);
++ }
+ #endif
+
+- if (hdev && use_ll_privacy(hdev) && !hdev_is_powered(hdev)) {
+- bool changed = hci_dev_test_flag(hdev,
+- HCI_ENABLE_LL_PRIVACY);
+-
+- hci_dev_clear_flag(hdev, HCI_ENABLE_LL_PRIVACY);
++ if (hdev && use_ll_privacy(hdev) && !hdev_is_powered(hdev)) {
++ bool changed;
+
+- if (changed)
+- exp_ll_privacy_feature_changed(false, hdev, sk);
+- }
++ changed = hci_dev_test_and_clear_flag(hdev,
++ HCI_ENABLE_LL_PRIVACY);
++ if (changed)
++ exp_ll_privacy_feature_changed(false, hdev, sk);
++ }
+
+- hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
++ hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
+
+- return mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE,
+- MGMT_OP_SET_EXP_FEATURE, 0,
+- &rp, sizeof(rp));
+- }
++ return mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE,
++ MGMT_OP_SET_EXP_FEATURE, 0,
++ &rp, sizeof(rp));
++}
+
+ #ifdef CONFIG_BT_FEATURE_DEBUG
+- if (!memcmp(cp->uuid, debug_uuid, 16)) {
+- bool val, changed;
+- int err;
++static int set_debug_func(struct sock *sk, struct hci_dev *hdev,
++ struct mgmt_cp_set_exp_feature *cp, u16 data_len)
++{
++ struct mgmt_rp_set_exp_feature rp;
+
+- /* Command requires to use the non-controller index */
+- if (hdev)
+- return mgmt_cmd_status(sk, hdev->id,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_INDEX);
++ bool val, changed;
++ int err;
+
+- /* Parameters are limited to a single octet */
+- if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1)
+- return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_PARAMS);
++ /* Command requires to use the non-controller index */
++ if (hdev)
++ return mgmt_cmd_status(sk, hdev->id,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_INDEX);
+
+- /* Only boolean on/off is supported */
+- if (cp->param[0] != 0x00 && cp->param[0] != 0x01)
+- return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_PARAMS);
++ /* Parameters are limited to a single octet */
++ if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1)
++ return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_PARAMS);
+
+- val = !!cp->param[0];
+- changed = val ? !bt_dbg_get() : bt_dbg_get();
+- bt_dbg_set(val);
++ /* Only boolean on/off is supported */
++ if (cp->param[0] != 0x00 && cp->param[0] != 0x01)
++ return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_PARAMS);
+
+- memcpy(rp.uuid, debug_uuid, 16);
+- rp.flags = cpu_to_le32(val ? BIT(0) : 0);
++ val = !!cp->param[0];
++ changed = val ? !bt_dbg_get() : bt_dbg_get();
++ bt_dbg_set(val);
+
+- hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
++ memcpy(rp.uuid, debug_uuid, 16);
++ rp.flags = cpu_to_le32(val ? BIT(0) : 0);
+
+- err = mgmt_cmd_complete(sk, MGMT_INDEX_NONE,
+- MGMT_OP_SET_EXP_FEATURE, 0,
+- &rp, sizeof(rp));
++ hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
+
+- if (changed)
+- exp_debug_feature_changed(val, sk);
++ err = mgmt_cmd_complete(sk, MGMT_INDEX_NONE,
++ MGMT_OP_SET_EXP_FEATURE, 0,
++ &rp, sizeof(rp));
+
+- return err;
+- }
++ if (changed)
++ exp_debug_feature_changed(val, sk);
++
++ return err;
++}
+ #endif
+
+- if (!memcmp(cp->uuid, rpa_resolution_uuid, 16)) {
+- bool val, changed;
+- int err;
+- u32 flags;
++static int set_rpa_resolution_func(struct sock *sk, struct hci_dev *hdev,
++ struct mgmt_cp_set_exp_feature *cp,
++ u16 data_len)
++{
++ struct mgmt_rp_set_exp_feature rp;
++ bool val, changed;
++ int err;
++ u32 flags;
+
+- /* Command requires to use the controller index */
+- if (!hdev)
+- return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_INDEX);
++ /* Command requires to use the controller index */
++ if (!hdev)
++ return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_INDEX);
+
+- /* Changes can only be made when controller is powered down */
+- if (hdev_is_powered(hdev))
+- return mgmt_cmd_status(sk, hdev->id,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_REJECTED);
++ /* Changes can only be made when controller is powered down */
++ if (hdev_is_powered(hdev))
++ return mgmt_cmd_status(sk, hdev->id,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_REJECTED);
+
+- /* Parameters are limited to a single octet */
+- if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1)
+- return mgmt_cmd_status(sk, hdev->id,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_PARAMS);
++ /* Parameters are limited to a single octet */
++ if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1)
++ return mgmt_cmd_status(sk, hdev->id,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_PARAMS);
+
+- /* Only boolean on/off is supported */
+- if (cp->param[0] != 0x00 && cp->param[0] != 0x01)
+- return mgmt_cmd_status(sk, hdev->id,
+- MGMT_OP_SET_EXP_FEATURE,
+- MGMT_STATUS_INVALID_PARAMS);
++ /* Only boolean on/off is supported */
++ if (cp->param[0] != 0x00 && cp->param[0] != 0x01)
++ return mgmt_cmd_status(sk, hdev->id,
++ MGMT_OP_SET_EXP_FEATURE,
++ MGMT_STATUS_INVALID_PARAMS);
+
+- val = !!cp->param[0];
++ val = !!cp->param[0];
+
+- if (val) {
+- changed = !hci_dev_test_flag(hdev,
++ if (val) {
++ changed = !hci_dev_test_and_set_flag(hdev,
+ HCI_ENABLE_LL_PRIVACY);
+- hci_dev_set_flag(hdev, HCI_ENABLE_LL_PRIVACY);
+- hci_dev_clear_flag(hdev, HCI_ADVERTISING);
++ hci_dev_clear_flag(hdev, HCI_ADVERTISING);
+
+- /* Enable LL privacy + supported settings changed */
+- flags = BIT(0) | BIT(1);
+- } else {
+- changed = hci_dev_test_flag(hdev,
+- HCI_ENABLE_LL_PRIVACY);
+- hci_dev_clear_flag(hdev, HCI_ENABLE_LL_PRIVACY);
++ /* Enable LL privacy + supported settings changed */
++ flags = BIT(0) | BIT(1);
++ } else {
++ changed = hci_dev_test_and_clear_flag(hdev,
++ HCI_ENABLE_LL_PRIVACY);
+
+- /* Disable LL privacy + supported settings changed */
+- flags = BIT(1);
+- }
++ /* Disable LL privacy + supported settings changed */
++ flags = BIT(1);
++ }
+
+- memcpy(rp.uuid, rpa_resolution_uuid, 16);
+- rp.flags = cpu_to_le32(flags);
++ memcpy(rp.uuid, rpa_resolution_uuid, 16);
++ rp.flags = cpu_to_le32(flags);
+
+- hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
++ hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
+
+- err = mgmt_cmd_complete(sk, hdev->id,
+- MGMT_OP_SET_EXP_FEATURE, 0,
+- &rp, sizeof(rp));
++ err = mgmt_cmd_complete(sk, hdev->id,
++ MGMT_OP_SET_EXP_FEATURE, 0,
++ &rp, sizeof(rp));
+
+- if (changed)
+- exp_ll_privacy_feature_changed(val, hdev, sk);
++ if (changed)
++ exp_ll_privacy_feature_changed(val, hdev, sk);
+
+- return err;
++ return err;
++}
++
++static const struct mgmt_exp_feature {
++ const u8 *uuid;
++ int (*set_func)(struct sock *sk, struct hci_dev *hdev,
++ struct mgmt_cp_set_exp_feature *cp, u16 data_len);
++} exp_features[] = {
++ EXP_FEAT(ZERO_KEY, set_zero_key_func),
++#ifdef CONFIG_BT_FEATURE_DEBUG
++ EXP_FEAT(debug_uuid, set_debug_func),
++#endif
++ EXP_FEAT(rpa_resolution_uuid, set_rpa_resolution_func),
++
++ /* end with a null feature */
++ EXP_FEAT(NULL, NULL)
++};
++
++static int set_exp_feature(struct sock *sk, struct hci_dev *hdev,
++ void *data, u16 data_len)
++{
++ struct mgmt_cp_set_exp_feature *cp = data;
++ size_t i = 0;
++
++ bt_dev_dbg(hdev, "sock %p", sk);
++
++ for (i = 0; exp_features[i].uuid; i++) {
++ if (!memcmp(cp->uuid, exp_features[i].uuid, 16))
++ return exp_features[i].set_func(sk, hdev, cp, data_len);
+ }
+
+ return mgmt_cmd_status(sk, hdev ? hdev->id : MGMT_INDEX_NONE,
+diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c
+index 98a8815865128..6e047e178c0ab 100644
+--- a/net/bluetooth/sco.c
++++ b/net/bluetooth/sco.c
+@@ -133,6 +133,7 @@ static struct sco_conn *sco_conn_add(struct hci_conn *hcon)
+ return NULL;
+
+ spin_lock_init(&conn->lock);
++ INIT_DELAYED_WORK(&conn->timeout_work, sco_sock_timeout);
+
+ hcon->sco_data = conn;
+ conn->hcon = hcon;
+@@ -187,20 +188,21 @@ static void sco_conn_del(struct hci_conn *hcon, int err)
+ /* Kill socket */
+ sco_conn_lock(conn);
+ sk = conn->sk;
++ if (sk)
++ sock_hold(sk);
+ sco_conn_unlock(conn);
+
+ if (sk) {
+- sock_hold(sk);
+ lock_sock(sk);
+ sco_sock_clear_timer(sk);
+ sco_chan_del(sk, err);
+ release_sock(sk);
+ sock_put(sk);
+-
+- /* Ensure no more work items will run before freeing conn. */
+- cancel_delayed_work_sync(&conn->timeout_work);
+ }
+
++ /* Ensure no more work items will run before freeing conn. */
++ cancel_delayed_work_sync(&conn->timeout_work);
++
+ hcon->sco_data = NULL;
+ kfree(conn);
+ }
+@@ -213,8 +215,6 @@ static void __sco_chan_add(struct sco_conn *conn, struct sock *sk,
+ sco_pi(sk)->conn = conn;
+ conn->sk = sk;
+
+- INIT_DELAYED_WORK(&conn->timeout_work, sco_sock_timeout);
+-
+ if (parent)
+ bt_accept_enqueue(parent, sk, true);
+ }
+@@ -280,7 +280,8 @@ static int sco_connect(struct hci_dev *hdev, struct sock *sk)
+ return err;
+ }
+
+-static int sco_send_frame(struct sock *sk, struct msghdr *msg, int len)
++static int sco_send_frame(struct sock *sk, void *buf, int len,
++ unsigned int msg_flags)
+ {
+ struct sco_conn *conn = sco_pi(sk)->conn;
+ struct sk_buff *skb;
+@@ -292,15 +293,11 @@ static int sco_send_frame(struct sock *sk, struct msghdr *msg, int len)
+
+ BT_DBG("sk %p len %d", sk, len);
+
+- skb = bt_skb_send_alloc(sk, len, msg->msg_flags & MSG_DONTWAIT, &err);
++ skb = bt_skb_send_alloc(sk, len, msg_flags & MSG_DONTWAIT, &err);
+ if (!skb)
+ return err;
+
+- if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
+- kfree_skb(skb);
+- return -EFAULT;
+- }
+-
++ memcpy(skb_put(skb, len), buf, len);
+ hci_send_sco(conn->hcon, skb);
+
+ return len;
+@@ -725,6 +722,7 @@ static int sco_sock_sendmsg(struct socket *sock, struct msghdr *msg,
+ size_t len)
+ {
+ struct sock *sk = sock->sk;
++ void *buf;
+ int err;
+
+ BT_DBG("sock %p, sk %p", sock, sk);
+@@ -736,14 +734,24 @@ static int sco_sock_sendmsg(struct socket *sock, struct msghdr *msg,
+ if (msg->msg_flags & MSG_OOB)
+ return -EOPNOTSUPP;
+
++ buf = kmalloc(len, GFP_KERNEL);
++ if (!buf)
++ return -ENOMEM;
++
++ if (memcpy_from_msg(buf, msg, len)) {
++ kfree(buf);
++ return -EFAULT;
++ }
++
+ lock_sock(sk);
+
+ if (sk->sk_state == BT_CONNECTED)
+- err = sco_send_frame(sk, msg, len);
++ err = sco_send_frame(sk, buf, len, msg->msg_flags);
+ else
+ err = -ENOTCONN;
+
+ release_sock(sk);
++ kfree(buf);
+ return err;
+ }
+
+diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
+index b5f4ef35357c8..655ee0e2de86d 100644
+--- a/net/bpf/test_run.c
++++ b/net/bpf/test_run.c
+@@ -954,7 +954,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat
+ if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
+ goto out;
+
+- if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) {
++ if (user_ctx->local_port > U16_MAX) {
+ ret = -ERANGE;
+ goto out;
+ }
+@@ -962,7 +962,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat
+ ctx.family = (u16)user_ctx->family;
+ ctx.protocol = (u16)user_ctx->protocol;
+ ctx.dport = (u16)user_ctx->local_port;
+- ctx.sport = (__force __be16)user_ctx->remote_port;
++ ctx.sport = user_ctx->remote_port;
+
+ switch (ctx.family) {
+ case AF_INET:
+diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c
+index b50382f957c12..6743c8a0fe8e1 100644
+--- a/net/bridge/br_input.c
++++ b/net/bridge/br_input.c
+@@ -39,6 +39,13 @@ static int br_pass_frame_up(struct sk_buff *skb)
+ dev_sw_netstats_rx_add(brdev, skb->len);
+
+ vg = br_vlan_group_rcu(br);
++
++ /* Reset the offload_fwd_mark because there could be a stacked
++ * bridge above, and it should not think this bridge it doing
++ * that bridge's work forwarding out its ports.
++ */
++ br_switchdev_frame_unmark(skb);
++
+ /* Bridge is just like any other port. Make sure the
+ * packet is allowed except in promisc mode when someone
+ * may be running packet capture.
+diff --git a/net/bridge/br_ioctl.c b/net/bridge/br_ioctl.c
+index 793b0db9d9a36..9922497e59f8c 100644
+--- a/net/bridge/br_ioctl.c
++++ b/net/bridge/br_ioctl.c
+@@ -71,7 +71,8 @@ static int get_fdb_entries(struct net_bridge *br, void __user *userbuf,
+
+ num = br_fdb_fillbuf(br, buf, maxnum, offset);
+ if (num > 0) {
+- if (copy_to_user(userbuf, buf, num*sizeof(struct __fdb_entry)))
++ if (copy_to_user(userbuf, buf,
++ array_size(num, sizeof(struct __fdb_entry))))
+ num = -EFAULT;
+ }
+ kfree(buf);
+@@ -188,7 +189,7 @@ int br_dev_siocdevprivate(struct net_device *dev, struct ifreq *rq, void __user
+ return -ENOMEM;
+
+ get_port_ifindices(br, indices, num);
+- if (copy_to_user(argp, indices, num * sizeof(int)))
++ if (copy_to_user(argp, indices, array_size(num, sizeof(int))))
+ num = -EFAULT;
+ kfree(indices);
+ return num;
+@@ -336,7 +337,8 @@ static int old_deviceless(struct net *net, void __user *uarg)
+
+ args[2] = get_bridge_ifindices(net, indices, args[2]);
+
+- ret = copy_to_user(uarg, indices, args[2]*sizeof(int))
++ ret = copy_to_user((void __user *)args[1], indices,
++ array_size(args[2], sizeof(int)))
+ ? -EFAULT : args[2];
+
+ kfree(indices);
+diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c
+index f3d751105343c..db4f2641d1cd1 100644
+--- a/net/bridge/br_multicast.c
++++ b/net/bridge/br_multicast.c
+@@ -82,6 +82,9 @@ static void br_multicast_find_del_pg(struct net_bridge *br,
+ struct net_bridge_port_group *pg);
+ static void __br_multicast_stop(struct net_bridge_mcast *brmctx);
+
++static int br_mc_disabled_update(struct net_device *dev, bool value,
++ struct netlink_ext_ack *extack);
++
+ static struct net_bridge_port_group *
+ br_sg_port_find(struct net_bridge *br,
+ struct net_bridge_port_group_sg_key *sg_p)
+@@ -1156,6 +1159,7 @@ struct net_bridge_mdb_entry *br_multicast_new_group(struct net_bridge *br,
+ return mp;
+
+ if (atomic_read(&br->mdb_hash_tbl.nelems) >= br->hash_max) {
++ br_mc_disabled_update(br->dev, false, NULL);
+ br_opt_toggle(br, BROPT_MULTICAST_ENABLED, false);
+ return ERR_PTR(-E2BIG);
+ }
+@@ -4522,6 +4526,38 @@ int br_multicast_set_mld_version(struct net_bridge_mcast *brmctx,
+ }
+ #endif
+
++void br_multicast_set_query_intvl(struct net_bridge_mcast *brmctx,
++ unsigned long val)
++{
++ unsigned long intvl_jiffies = clock_t_to_jiffies(val);
++
++ if (intvl_jiffies < BR_MULTICAST_QUERY_INTVL_MIN) {
++ br_info(brmctx->br,
++ "trying to set multicast query interval below minimum, setting to %lu (%ums)\n",
++ jiffies_to_clock_t(BR_MULTICAST_QUERY_INTVL_MIN),
++ jiffies_to_msecs(BR_MULTICAST_QUERY_INTVL_MIN));
++ intvl_jiffies = BR_MULTICAST_QUERY_INTVL_MIN;
++ }
++
++ brmctx->multicast_query_interval = intvl_jiffies;
++}
++
++void br_multicast_set_startup_query_intvl(struct net_bridge_mcast *brmctx,
++ unsigned long val)
++{
++ unsigned long intvl_jiffies = clock_t_to_jiffies(val);
++
++ if (intvl_jiffies < BR_MULTICAST_STARTUP_QUERY_INTVL_MIN) {
++ br_info(brmctx->br,
++ "trying to set multicast startup query interval below minimum, setting to %lu (%ums)\n",
++ jiffies_to_clock_t(BR_MULTICAST_STARTUP_QUERY_INTVL_MIN),
++ jiffies_to_msecs(BR_MULTICAST_STARTUP_QUERY_INTVL_MIN));
++ intvl_jiffies = BR_MULTICAST_STARTUP_QUERY_INTVL_MIN;
++ }
++
++ brmctx->multicast_startup_query_interval = intvl_jiffies;
++}
++
+ /**
+ * br_multicast_list_adjacent - Returns snooped multicast addresses
+ * @dev: The bridge port adjacent to which to retrieve addresses
+diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c
+index 8edfb98ae1d58..68c0d0f928908 100644
+--- a/net/bridge/br_netfilter_hooks.c
++++ b/net/bridge/br_netfilter_hooks.c
+@@ -743,6 +743,9 @@ static int br_nf_dev_queue_xmit(struct net *net, struct sock *sk, struct sk_buff
+ if (nf_bridge->frag_max_size && nf_bridge->frag_max_size < mtu)
+ mtu = nf_bridge->frag_max_size;
+
++ nf_bridge_update_protocol(skb);
++ nf_bridge_push_encap_header(skb);
++
+ if (skb_is_gso(skb) || skb->len + mtu_reserved <= mtu) {
+ nf_bridge_info_free(skb);
+ return br_dev_queue_push_xmit(net, sk, skb);
+@@ -760,8 +763,6 @@ static int br_nf_dev_queue_xmit(struct net *net, struct sock *sk, struct sk_buff
+
+ IPCB(skb)->frag_max_size = nf_bridge->frag_max_size;
+
+- nf_bridge_update_protocol(skb);
+-
+ data = this_cpu_ptr(&brnf_frag_data_storage);
+
+ if (skb_vlan_tag_present(skb)) {
+@@ -789,8 +790,6 @@ static int br_nf_dev_queue_xmit(struct net *net, struct sock *sk, struct sk_buff
+
+ IP6CB(skb)->frag_max_size = nf_bridge->frag_max_size;
+
+- nf_bridge_update_protocol(skb);
+-
+ data = this_cpu_ptr(&brnf_frag_data_storage);
+ data->encap_size = nf_bridge_encap_header_len(skb);
+ data->size = ETH_HLEN + data->encap_size;
+diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c
+index 5c6c4305ed235..e365cf82f0615 100644
+--- a/net/bridge/br_netlink.c
++++ b/net/bridge/br_netlink.c
+@@ -1357,7 +1357,7 @@ static int br_changelink(struct net_device *brdev, struct nlattr *tb[],
+ if (data[IFLA_BR_MCAST_QUERY_INTVL]) {
+ u64 val = nla_get_u64(data[IFLA_BR_MCAST_QUERY_INTVL]);
+
+- br->multicast_ctx.multicast_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_query_intvl(&br->multicast_ctx, val);
+ }
+
+ if (data[IFLA_BR_MCAST_QUERY_RESPONSE_INTVL]) {
+@@ -1369,7 +1369,7 @@ static int br_changelink(struct net_device *brdev, struct nlattr *tb[],
+ if (data[IFLA_BR_MCAST_STARTUP_QUERY_INTVL]) {
+ u64 val = nla_get_u64(data[IFLA_BR_MCAST_STARTUP_QUERY_INTVL]);
+
+- br->multicast_ctx.multicast_startup_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_startup_query_intvl(&br->multicast_ctx, val);
+ }
+
+ if (data[IFLA_BR_MCAST_STATS_ENABLED]) {
+diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h
+index 37ca76406f1e8..bd218c2b2cd97 100644
+--- a/net/bridge/br_private.h
++++ b/net/bridge/br_private.h
+@@ -28,6 +28,8 @@
+ #define BR_MAX_PORTS (1<<BR_PORT_BITS)
+
+ #define BR_MULTICAST_DEFAULT_HASH_MAX 4096
++#define BR_MULTICAST_QUERY_INTVL_MIN msecs_to_jiffies(1000)
++#define BR_MULTICAST_STARTUP_QUERY_INTVL_MIN BR_MULTICAST_QUERY_INTVL_MIN
+
+ #define BR_HWDOM_MAX BITS_PER_LONG
+
+@@ -968,6 +970,10 @@ int br_multicast_dump_querier_state(struct sk_buff *skb,
+ int nest_attr);
+ size_t br_multicast_querier_state_size(void);
+ size_t br_rports_size(const struct net_bridge_mcast *brmctx);
++void br_multicast_set_query_intvl(struct net_bridge_mcast *brmctx,
++ unsigned long val);
++void br_multicast_set_startup_query_intvl(struct net_bridge_mcast *brmctx,
++ unsigned long val);
+
+ static inline bool br_group_is_l2(const struct br_ip *group)
+ {
+@@ -1152,9 +1158,9 @@ br_multicast_port_ctx_get_global(const struct net_bridge_mcast_port *pmctx)
+ static inline bool
+ br_multicast_ctx_vlan_global_disabled(const struct net_bridge_mcast *brmctx)
+ {
+- return br_opt_get(brmctx->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) &&
+- br_multicast_ctx_is_vlan(brmctx) &&
+- !(brmctx->vlan->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED);
++ return br_multicast_ctx_is_vlan(brmctx) &&
++ (!br_opt_get(brmctx->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) ||
++ !(brmctx->vlan->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED));
+ }
+
+ static inline bool
+@@ -1911,11 +1917,13 @@ static inline int br_cfm_status_fill_info(struct sk_buff *skb,
+
+ static inline int br_cfm_mep_count(struct net_bridge *br, u32 *count)
+ {
++ *count = 0;
+ return -EOPNOTSUPP;
+ }
+
+ static inline int br_cfm_peer_mep_count(struct net_bridge *br, u32 *count)
+ {
++ *count = 0;
+ return -EOPNOTSUPP;
+ }
+ #endif
+diff --git a/net/bridge/br_sysfs_br.c b/net/bridge/br_sysfs_br.c
+index d9a89ddd03310..7b0c19772111c 100644
+--- a/net/bridge/br_sysfs_br.c
++++ b/net/bridge/br_sysfs_br.c
+@@ -658,7 +658,7 @@ static ssize_t multicast_query_interval_show(struct device *d,
+ static int set_query_interval(struct net_bridge *br, unsigned long val,
+ struct netlink_ext_ack *extack)
+ {
+- br->multicast_ctx.multicast_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_query_intvl(&br->multicast_ctx, val);
+ return 0;
+ }
+
+@@ -706,7 +706,7 @@ static ssize_t multicast_startup_query_interval_show(
+ static int set_startup_query_interval(struct net_bridge *br, unsigned long val,
+ struct netlink_ext_ack *extack)
+ {
+- br->multicast_ctx.multicast_startup_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_startup_query_intvl(&br->multicast_ctx, val);
+ return 0;
+ }
+
+diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c
+index 19f65ab91a027..10e63ea6a13e1 100644
+--- a/net/bridge/br_vlan.c
++++ b/net/bridge/br_vlan.c
+@@ -560,10 +560,10 @@ static bool __allowed_ingress(const struct net_bridge *br,
+ !br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) {
+ if (*state == BR_STATE_FORWARDING) {
+ *state = br_vlan_get_pvid_state(vg);
+- return br_vlan_state_allowed(*state, true);
+- } else {
+- return true;
++ if (!br_vlan_state_allowed(*state, true))
++ goto drop;
+ }
++ return true;
+ }
+ }
+ v = br_vlan_find(vg, *vid);
+@@ -2105,7 +2105,8 @@ static int br_vlan_rtm_dump(struct sk_buff *skb, struct netlink_callback *cb)
+ goto out_err;
+ }
+ err = br_vlan_dump_dev(dev, skb, cb, dump_flags);
+- if (err && err != -EMSGSIZE)
++ /* if the dump completed without an error we return 0 here */
++ if (err != -EMSGSIZE)
+ goto out_err;
+ } else {
+ for_each_netdev_rcu(net, dev) {
+diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c
+index 8ffd4ed2563c6..a6382973b3e70 100644
+--- a/net/bridge/br_vlan_options.c
++++ b/net/bridge/br_vlan_options.c
+@@ -521,7 +521,7 @@ static int br_vlan_process_global_one_opts(const struct net_bridge *br,
+ u64 val;
+
+ val = nla_get_u64(tb[BRIDGE_VLANDB_GOPTS_MCAST_QUERY_INTVL]);
+- v->br_mcast_ctx.multicast_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_query_intvl(&v->br_mcast_ctx, val);
+ *changed = true;
+ }
+ if (tb[BRIDGE_VLANDB_GOPTS_MCAST_QUERY_RESPONSE_INTVL]) {
+@@ -535,7 +535,7 @@ static int br_vlan_process_global_one_opts(const struct net_bridge *br,
+ u64 val;
+
+ val = nla_get_u64(tb[BRIDGE_VLANDB_GOPTS_MCAST_STARTUP_QUERY_INTVL]);
+- v->br_mcast_ctx.multicast_startup_query_interval = clock_t_to_jiffies(val);
++ br_multicast_set_startup_query_intvl(&v->br_mcast_ctx, val);
+ *changed = true;
+ }
+ if (tb[BRIDGE_VLANDB_GOPTS_MCAST_QUERIER]) {
+diff --git a/net/bridge/netfilter/nft_reject_bridge.c b/net/bridge/netfilter/nft_reject_bridge.c
+index eba0efe64d05a..fbf858ddec352 100644
+--- a/net/bridge/netfilter/nft_reject_bridge.c
++++ b/net/bridge/netfilter/nft_reject_bridge.c
+@@ -49,7 +49,7 @@ static void nft_reject_br_send_v4_tcp_reset(struct net *net,
+ {
+ struct sk_buff *nskb;
+
+- nskb = nf_reject_skb_v4_tcp_reset(net, oldskb, dev, hook);
++ nskb = nf_reject_skb_v4_tcp_reset(net, oldskb, NULL, hook);
+ if (!nskb)
+ return;
+
+@@ -65,7 +65,7 @@ static void nft_reject_br_send_v4_unreach(struct net *net,
+ {
+ struct sk_buff *nskb;
+
+- nskb = nf_reject_skb_v4_unreach(net, oldskb, dev, hook, code);
++ nskb = nf_reject_skb_v4_unreach(net, oldskb, NULL, hook, code);
+ if (!nskb)
+ return;
+
+@@ -81,7 +81,7 @@ static void nft_reject_br_send_v6_tcp_reset(struct net *net,
+ {
+ struct sk_buff *nskb;
+
+- nskb = nf_reject_skb_v6_tcp_reset(net, oldskb, dev, hook);
++ nskb = nf_reject_skb_v6_tcp_reset(net, oldskb, NULL, hook);
+ if (!nskb)
+ return;
+
+@@ -98,7 +98,7 @@ static void nft_reject_br_send_v6_unreach(struct net *net,
+ {
+ struct sk_buff *nskb;
+
+- nskb = nf_reject_skb_v6_unreach(net, oldskb, dev, hook, code);
++ nskb = nf_reject_skb_v6_unreach(net, oldskb, NULL, hook, code);
+ if (!nskb)
+ return;
+
+diff --git a/net/can/isotp.c b/net/can/isotp.c
+index df6968b28bf41..26821487a0573 100644
+--- a/net/can/isotp.c
++++ b/net/can/isotp.c
+@@ -56,6 +56,7 @@
+ #include <linux/module.h>
+ #include <linux/init.h>
+ #include <linux/interrupt.h>
++#include <linux/spinlock.h>
+ #include <linux/hrtimer.h>
+ #include <linux/wait.h>
+ #include <linux/uio.h>
+@@ -119,8 +120,8 @@ enum {
+ };
+
+ struct tpcon {
+- int idx;
+- int len;
++ unsigned int idx;
++ unsigned int len;
+ u32 state;
+ u8 bs;
+ u8 sn;
+@@ -140,11 +141,13 @@ struct isotp_sock {
+ struct can_isotp_options opt;
+ struct can_isotp_fc_options rxfc, txfc;
+ struct can_isotp_ll_options ll;
++ u32 frame_txtime;
+ u32 force_tx_stmin;
+ u32 force_rx_stmin;
+ struct tpcon rx, tx;
+ struct list_head notifier;
+ wait_queue_head_t wait;
++ spinlock_t rx_lock; /* protect single thread state machine */
+ };
+
+ static LIST_HEAD(isotp_notifier_list);
+@@ -358,7 +361,7 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae)
+
+ so->tx_gap = ktime_set(0, 0);
+ /* add transmission time for CAN frame N_As */
+- so->tx_gap = ktime_add_ns(so->tx_gap, so->opt.frame_txtime);
++ so->tx_gap = ktime_add_ns(so->tx_gap, so->frame_txtime);
+ /* add waiting time for consecutive frames N_Cs */
+ if (so->opt.flags & CAN_ISOTP_FORCE_TXSTMIN)
+ so->tx_gap = ktime_add_ns(so->tx_gap,
+@@ -615,11 +618,17 @@ static void isotp_rcv(struct sk_buff *skb, void *data)
+
+ n_pci_type = cf->data[ae] & 0xF0;
+
++ /* Make sure the state changes and data structures stay consistent at
++ * CAN frame reception time. This locking is not needed in real world
++ * use cases but the inconsistency can be triggered with syzkaller.
++ */
++ spin_lock(&so->rx_lock);
++
+ if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) {
+ /* check rx/tx path half duplex expectations */
+ if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) ||
+ (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC))
+- return;
++ goto out_unlock;
+ }
+
+ switch (n_pci_type) {
+@@ -668,6 +677,9 @@ static void isotp_rcv(struct sk_buff *skb, void *data)
+ isotp_rcv_cf(sk, cf, ae, skb);
+ break;
+ }
++
++out_unlock:
++ spin_unlock(&so->rx_lock);
+ }
+
+ static void isotp_fill_dataframe(struct canfd_frame *cf, struct isotp_sock *so,
+@@ -854,6 +866,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+ struct canfd_frame *cf;
+ int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0;
+ int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0;
++ s64 hrtimer_sec = 0;
+ int off;
+ int err;
+
+@@ -876,7 +889,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+
+ if (!size || size > MAX_MSG_LENGTH) {
+ err = -EINVAL;
+- goto err_out;
++ goto err_out_drop;
+ }
+
+ /* take care of a potential SF_DL ESC offset for TX_DL > 8 */
+@@ -886,24 +899,24 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+ if ((so->opt.flags & CAN_ISOTP_SF_BROADCAST) &&
+ (size > so->tx.ll_dl - SF_PCI_SZ4 - ae - off)) {
+ err = -EINVAL;
+- goto err_out;
++ goto err_out_drop;
+ }
+
+ err = memcpy_from_msg(so->tx.buf, msg, size);
+ if (err < 0)
+- goto err_out;
++ goto err_out_drop;
+
+ dev = dev_get_by_index(sock_net(sk), so->ifindex);
+ if (!dev) {
+ err = -ENXIO;
+- goto err_out;
++ goto err_out_drop;
+ }
+
+ skb = sock_alloc_send_skb(sk, so->ll.mtu + sizeof(struct can_skb_priv),
+ msg->msg_flags & MSG_DONTWAIT, &err);
+ if (!skb) {
+ dev_put(dev);
+- goto err_out;
++ goto err_out_drop;
+ }
+
+ can_skb_reserve(skb);
+@@ -952,7 +965,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+ isotp_create_fframe(cf, so, ae);
+
+ /* start timeout for FC */
+- hrtimer_start(&so->txtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
++ hrtimer_sec = 1;
++ hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0),
++ HRTIMER_MODE_REL_SOFT);
+ }
+
+ /* send the first or only CAN frame */
+@@ -965,7 +980,12 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+ if (err) {
+ pr_notice_once("can-isotp: %s: can_send_ret %pe\n",
+ __func__, ERR_PTR(err));
+- goto err_out;
++
++ /* no transmission -> no timeout monitoring */
++ if (hrtimer_sec)
++ hrtimer_cancel(&so->txtimer);
++
++ goto err_out_drop;
+ }
+
+ if (wait_tx_done) {
+@@ -978,6 +998,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
+
+ return size;
+
++err_out_drop:
++ /* drop this PDU and unlock a potential wait queue */
++ old_state = ISOTP_IDLE;
+ err_out:
+ so->tx.state = old_state;
+ if (so->tx.state == ISOTP_IDLE)
+@@ -991,26 +1014,29 @@ static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
+ {
+ struct sock *sk = sock->sk;
+ struct sk_buff *skb;
+- int err = 0;
+- int noblock;
++ struct isotp_sock *so = isotp_sk(sk);
++ int noblock = flags & MSG_DONTWAIT;
++ int ret = 0;
+
+- noblock = flags & MSG_DONTWAIT;
+- flags &= ~MSG_DONTWAIT;
++ if (flags & ~(MSG_DONTWAIT | MSG_TRUNC | MSG_PEEK))
++ return -EINVAL;
+
+- skb = skb_recv_datagram(sk, flags, noblock, &err);
++ if (!so->bound)
++ return -EADDRNOTAVAIL;
++
++ flags &= ~MSG_DONTWAIT;
++ skb = skb_recv_datagram(sk, flags, noblock, &ret);
+ if (!skb)
+- return err;
++ return ret;
+
+ if (size < skb->len)
+ msg->msg_flags |= MSG_TRUNC;
+ else
+ size = skb->len;
+
+- err = memcpy_to_msg(msg, skb->data, size);
+- if (err < 0) {
+- skb_free_datagram(sk, skb);
+- return err;
+- }
++ ret = memcpy_to_msg(msg, skb->data, size);
++ if (ret < 0)
++ goto out_err;
+
+ sock_recv_timestamp(msg, sk, skb);
+
+@@ -1020,9 +1046,13 @@ static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
+ memcpy(msg->msg_name, skb->cb, msg->msg_namelen);
+ }
+
++ /* set length of return value */
++ ret = (flags & MSG_TRUNC) ? skb->len : size;
++
++out_err:
+ skb_free_datagram(sk, skb);
+
+- return size;
++ return ret;
+ }
+
+ static int isotp_release(struct socket *sock)
+@@ -1090,6 +1120,7 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ struct net *net = sock_net(sk);
+ int ifindex;
+ struct net_device *dev;
++ canid_t tx_id, rx_id;
+ int err = 0;
+ int notify_enetdown = 0;
+ int do_rx_reg = 1;
+@@ -1097,35 +1128,38 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ if (len < ISOTP_MIN_NAMELEN)
+ return -EINVAL;
+
+- if (addr->can_addr.tp.tx_id & (CAN_ERR_FLAG | CAN_RTR_FLAG))
+- return -EADDRNOTAVAIL;
++ /* sanitize tx/rx CAN identifiers */
++ tx_id = addr->can_addr.tp.tx_id;
++ if (tx_id & CAN_EFF_FLAG)
++ tx_id &= (CAN_EFF_FLAG | CAN_EFF_MASK);
++ else
++ tx_id &= CAN_SFF_MASK;
++
++ rx_id = addr->can_addr.tp.rx_id;
++ if (rx_id & CAN_EFF_FLAG)
++ rx_id &= (CAN_EFF_FLAG | CAN_EFF_MASK);
++ else
++ rx_id &= CAN_SFF_MASK;
+
+ if (!addr->can_ifindex)
+ return -ENODEV;
+
+ lock_sock(sk);
+
++ if (so->bound) {
++ err = -EINVAL;
++ goto out;
++ }
++
+ /* do not register frame reception for functional addressing */
+ if (so->opt.flags & CAN_ISOTP_SF_BROADCAST)
+ do_rx_reg = 0;
+
+ /* do not validate rx address for functional addressing */
+- if (do_rx_reg) {
+- if (addr->can_addr.tp.rx_id == addr->can_addr.tp.tx_id) {
+- err = -EADDRNOTAVAIL;
+- goto out;
+- }
+-
+- if (addr->can_addr.tp.rx_id & (CAN_ERR_FLAG | CAN_RTR_FLAG)) {
+- err = -EADDRNOTAVAIL;
+- goto out;
+- }
+- }
+-
+- if (so->bound && addr->can_ifindex == so->ifindex &&
+- addr->can_addr.tp.rx_id == so->rxid &&
+- addr->can_addr.tp.tx_id == so->txid)
++ if (do_rx_reg && rx_id == tx_id) {
++ err = -EADDRNOTAVAIL;
+ goto out;
++ }
+
+ dev = dev_get_by_index(net, addr->can_ifindex);
+ if (!dev) {
+@@ -1148,29 +1182,15 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ ifindex = dev->ifindex;
+
+ if (do_rx_reg)
+- can_rx_register(net, dev, addr->can_addr.tp.rx_id,
+- SINGLE_MASK(addr->can_addr.tp.rx_id),
++ can_rx_register(net, dev, rx_id, SINGLE_MASK(rx_id),
+ isotp_rcv, sk, "isotp", sk);
+
+ dev_put(dev);
+
+- if (so->bound && do_rx_reg) {
+- /* unregister old filter */
+- if (so->ifindex) {
+- dev = dev_get_by_index(net, so->ifindex);
+- if (dev) {
+- can_rx_unregister(net, dev, so->rxid,
+- SINGLE_MASK(so->rxid),
+- isotp_rcv, sk);
+- dev_put(dev);
+- }
+- }
+- }
+-
+ /* switch to new settings */
+ so->ifindex = ifindex;
+- so->rxid = addr->can_addr.tp.rx_id;
+- so->txid = addr->can_addr.tp.tx_id;
++ so->rxid = rx_id;
++ so->txid = tx_id;
+ so->bound = 1;
+
+ out:
+@@ -1224,6 +1244,14 @@ static int isotp_setsockopt_locked(struct socket *sock, int level, int optname,
+ /* no separate rx_ext_address is given => use ext_address */
+ if (!(so->opt.flags & CAN_ISOTP_RX_EXT_ADDR))
+ so->opt.rx_ext_address = so->opt.ext_address;
++
++ /* check for frame_txtime changes (0 => no changes) */
++ if (so->opt.frame_txtime) {
++ if (so->opt.frame_txtime == CAN_ISOTP_FRAME_TXTIME_ZERO)
++ so->frame_txtime = 0;
++ else
++ so->frame_txtime = so->opt.frame_txtime;
++ }
+ break;
+
+ case CAN_ISOTP_RECV_FC:
+@@ -1425,6 +1453,7 @@ static int isotp_init(struct sock *sk)
+ so->opt.rxpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT;
+ so->opt.txpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT;
+ so->opt.frame_txtime = CAN_ISOTP_DEFAULT_FRAME_TXTIME;
++ so->frame_txtime = CAN_ISOTP_DEFAULT_FRAME_TXTIME;
+ so->rxfc.bs = CAN_ISOTP_DEFAULT_RECV_BS;
+ so->rxfc.stmin = CAN_ISOTP_DEFAULT_RECV_STMIN;
+ so->rxfc.wftmax = CAN_ISOTP_DEFAULT_RECV_WFTMAX;
+@@ -1444,6 +1473,7 @@ static int isotp_init(struct sock *sk)
+ so->txtimer.function = isotp_tx_timer_handler;
+
+ init_waitqueue_head(&so->wait);
++ spin_lock_init(&so->rx_lock);
+
+ spin_lock(&isotp_notifier_lock);
+ list_add_tail(&so->notifier, &isotp_notifier_list);
+diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c
+index 9bc55ecb37f9f..8452b0fbb78c9 100644
+--- a/net/can/j1939/main.c
++++ b/net/can/j1939/main.c
+@@ -75,6 +75,13 @@ static void j1939_can_recv(struct sk_buff *iskb, void *data)
+ skcb->addr.pgn = (cf->can_id >> 8) & J1939_PGN_MAX;
+ /* set default message type */
+ skcb->addr.type = J1939_TP;
++
++ if (!j1939_address_is_valid(skcb->addr.sa)) {
++ netdev_err_once(priv->ndev, "%s: sa is broadcast address, ignoring!\n",
++ __func__);
++ goto done;
++ }
++
+ if (j1939_pgn_is_pdu1(skcb->addr.pgn)) {
+ /* Type 1: with destination address */
+ skcb->addr.da = skcb->addr.pgn;
+diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c
+index 6c0a0ebdd024c..307ee1174a6e2 100644
+--- a/net/can/j1939/transport.c
++++ b/net/can/j1939/transport.c
+@@ -2006,7 +2006,7 @@ struct j1939_session *j1939_tp_send(struct j1939_priv *priv,
+ /* set the end-packet for broadcast */
+ session->pkt.last = session->pkt.total;
+
+- skcb->tskey = session->sk->sk_tskey++;
++ skcb->tskey = atomic_inc_return(&session->sk->sk_tskey) - 1;
+ session->tskey = skcb->tskey;
+
+ return session;
+@@ -2023,6 +2023,11 @@ static void j1939_tp_cmd_recv(struct j1939_priv *priv, struct sk_buff *skb)
+ extd = J1939_ETP;
+ fallthrough;
+ case J1939_TP_CMD_BAM:
++ if (cmd == J1939_TP_CMD_BAM && !j1939_cb_is_broadcast(skcb)) {
++ netdev_err_once(priv->ndev, "%s: BAM to unicast (%02x), ignoring!\n",
++ __func__, skcb->addr.sa);
++ return;
++ }
+ fallthrough;
+ case J1939_TP_CMD_RTS:
+ if (skcb->addr.type != extd)
+@@ -2085,6 +2090,12 @@ static void j1939_tp_cmd_recv(struct j1939_priv *priv, struct sk_buff *skb)
+ break;
+
+ case J1939_ETP_CMD_ABORT: /* && J1939_TP_CMD_ABORT */
++ if (j1939_cb_is_broadcast(skcb)) {
++ netdev_err_once(priv->ndev, "%s: abort to broadcast (%02x), ignoring!\n",
++ __func__, skcb->addr.sa);
++ return;
++ }
++
+ if (j1939_tp_im_transmitter(skcb))
+ j1939_xtp_rx_abort(priv, skb, true);
+
+diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
+index ff8624a7c9643..f6b7436458aeb 100644
+--- a/net/ceph/osd_client.c
++++ b/net/ceph/osd_client.c
+@@ -537,43 +537,6 @@ static void request_init(struct ceph_osd_request *req)
+ target_init(&req->r_t);
+ }
+
+-/*
+- * This is ugly, but it allows us to reuse linger registration and ping
+- * requests, keeping the structure of the code around send_linger{_ping}()
+- * reasonable. Setting up a min_nr=2 mempool for each linger request
+- * and dealing with copying ops (this blasts req only, watch op remains
+- * intact) isn't any better.
+- */
+-static void request_reinit(struct ceph_osd_request *req)
+-{
+- struct ceph_osd_client *osdc = req->r_osdc;
+- bool mempool = req->r_mempool;
+- unsigned int num_ops = req->r_num_ops;
+- u64 snapid = req->r_snapid;
+- struct ceph_snap_context *snapc = req->r_snapc;
+- bool linger = req->r_linger;
+- struct ceph_msg *request_msg = req->r_request;
+- struct ceph_msg *reply_msg = req->r_reply;
+-
+- dout("%s req %p\n", __func__, req);
+- WARN_ON(kref_read(&req->r_kref) != 1);
+- request_release_checks(req);
+-
+- WARN_ON(kref_read(&request_msg->kref) != 1);
+- WARN_ON(kref_read(&reply_msg->kref) != 1);
+- target_destroy(&req->r_t);
+-
+- request_init(req);
+- req->r_osdc = osdc;
+- req->r_mempool = mempool;
+- req->r_num_ops = num_ops;
+- req->r_snapid = snapid;
+- req->r_snapc = snapc;
+- req->r_linger = linger;
+- req->r_request = request_msg;
+- req->r_reply = reply_msg;
+-}
+-
+ struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc,
+ struct ceph_snap_context *snapc,
+ unsigned int num_ops,
+@@ -918,14 +881,30 @@ EXPORT_SYMBOL(osd_req_op_xattr_init);
+ * @watch_opcode: CEPH_OSD_WATCH_OP_*
+ */
+ static void osd_req_op_watch_init(struct ceph_osd_request *req, int which,
+- u64 cookie, u8 watch_opcode)
++ u8 watch_opcode, u64 cookie, u32 gen)
+ {
+ struct ceph_osd_req_op *op;
+
+ op = osd_req_op_init(req, which, CEPH_OSD_OP_WATCH, 0);
+ op->watch.cookie = cookie;
+ op->watch.op = watch_opcode;
+- op->watch.gen = 0;
++ op->watch.gen = gen;
++}
++
++/*
++ * prot_ver, timeout and notify payload (may be empty) should already be
++ * encoded in @request_pl
++ */
++static void osd_req_op_notify_init(struct ceph_osd_request *req, int which,
++ u64 cookie, struct ceph_pagelist *request_pl)
++{
++ struct ceph_osd_req_op *op;
++
++ op = osd_req_op_init(req, which, CEPH_OSD_OP_NOTIFY, 0);
++ op->notify.cookie = cookie;
++
++ ceph_osd_data_pagelist_init(&op->notify.request_data, request_pl);
++ op->indata_len = request_pl->length;
+ }
+
+ /*
+@@ -2727,10 +2706,13 @@ static void linger_release(struct kref *kref)
+ WARN_ON(!list_empty(&lreq->pending_lworks));
+ WARN_ON(lreq->osd);
+
+- if (lreq->reg_req)
+- ceph_osdc_put_request(lreq->reg_req);
+- if (lreq->ping_req)
+- ceph_osdc_put_request(lreq->ping_req);
++ if (lreq->request_pl)
++ ceph_pagelist_release(lreq->request_pl);
++ if (lreq->notify_id_pages)
++ ceph_release_page_vector(lreq->notify_id_pages, 1);
++
++ ceph_osdc_put_request(lreq->reg_req);
++ ceph_osdc_put_request(lreq->ping_req);
+ target_destroy(&lreq->t);
+ kfree(lreq);
+ }
+@@ -2999,6 +2981,12 @@ static void linger_commit_cb(struct ceph_osd_request *req)
+ struct ceph_osd_linger_request *lreq = req->r_priv;
+
+ mutex_lock(&lreq->lock);
++ if (req != lreq->reg_req) {
++ dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n",
++ __func__, lreq, lreq->linger_id, req, lreq->reg_req);
++ goto out;
++ }
++
+ dout("%s lreq %p linger_id %llu result %d\n", __func__, lreq,
+ lreq->linger_id, req->r_result);
+ linger_reg_commit_complete(lreq, req->r_result);
+@@ -3022,6 +3010,7 @@ static void linger_commit_cb(struct ceph_osd_request *req)
+ }
+ }
+
++out:
+ mutex_unlock(&lreq->lock);
+ linger_put(lreq);
+ }
+@@ -3044,6 +3033,12 @@ static void linger_reconnect_cb(struct ceph_osd_request *req)
+ struct ceph_osd_linger_request *lreq = req->r_priv;
+
+ mutex_lock(&lreq->lock);
++ if (req != lreq->reg_req) {
++ dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n",
++ __func__, lreq, lreq->linger_id, req, lreq->reg_req);
++ goto out;
++ }
++
+ dout("%s lreq %p linger_id %llu result %d last_error %d\n", __func__,
+ lreq, lreq->linger_id, req->r_result, lreq->last_error);
+ if (req->r_result < 0) {
+@@ -3053,46 +3048,64 @@ static void linger_reconnect_cb(struct ceph_osd_request *req)
+ }
+ }
+
++out:
+ mutex_unlock(&lreq->lock);
+ linger_put(lreq);
+ }
+
+ static void send_linger(struct ceph_osd_linger_request *lreq)
+ {
+- struct ceph_osd_request *req = lreq->reg_req;
+- struct ceph_osd_req_op *op = &req->r_ops[0];
++ struct ceph_osd_client *osdc = lreq->osdc;
++ struct ceph_osd_request *req;
++ int ret;
+
+- verify_osdc_wrlocked(req->r_osdc);
++ verify_osdc_wrlocked(osdc);
++ mutex_lock(&lreq->lock);
+ dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id);
+
+- if (req->r_osd)
+- cancel_linger_request(req);
++ if (lreq->reg_req) {
++ if (lreq->reg_req->r_osd)
++ cancel_linger_request(lreq->reg_req);
++ ceph_osdc_put_request(lreq->reg_req);
++ }
++
++ req = ceph_osdc_alloc_request(osdc, NULL, 1, true, GFP_NOIO);
++ BUG_ON(!req);
+
+- request_reinit(req);
+ target_copy(&req->r_t, &lreq->t);
+ req->r_mtime = lreq->mtime;
+
+- mutex_lock(&lreq->lock);
+ if (lreq->is_watch && lreq->committed) {
+- WARN_ON(op->op != CEPH_OSD_OP_WATCH ||
+- op->watch.cookie != lreq->linger_id);
+- op->watch.op = CEPH_OSD_WATCH_OP_RECONNECT;
+- op->watch.gen = ++lreq->register_gen;
++ osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_RECONNECT,
++ lreq->linger_id, ++lreq->register_gen);
+ dout("lreq %p reconnect register_gen %u\n", lreq,
+- op->watch.gen);
++ req->r_ops[0].watch.gen);
+ req->r_callback = linger_reconnect_cb;
+ } else {
+- if (!lreq->is_watch)
++ if (lreq->is_watch) {
++ osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_WATCH,
++ lreq->linger_id, 0);
++ } else {
+ lreq->notify_id = 0;
+- else
+- WARN_ON(op->watch.op != CEPH_OSD_WATCH_OP_WATCH);
++
++ refcount_inc(&lreq->request_pl->refcnt);
++ osd_req_op_notify_init(req, 0, lreq->linger_id,
++ lreq->request_pl);
++ ceph_osd_data_pages_init(
++ osd_req_op_data(req, 0, notify, response_data),
++ lreq->notify_id_pages, PAGE_SIZE, 0, false, false);
++ }
+ dout("lreq %p register\n", lreq);
+ req->r_callback = linger_commit_cb;
+ }
+- mutex_unlock(&lreq->lock);
++
++ ret = ceph_osdc_alloc_messages(req, GFP_NOIO);
++ BUG_ON(ret);
+
+ req->r_priv = linger_get(lreq);
+ req->r_linger = true;
++ lreq->reg_req = req;
++ mutex_unlock(&lreq->lock);
+
+ submit_request(req, true);
+ }
+@@ -3102,6 +3115,12 @@ static void linger_ping_cb(struct ceph_osd_request *req)
+ struct ceph_osd_linger_request *lreq = req->r_priv;
+
+ mutex_lock(&lreq->lock);
++ if (req != lreq->ping_req) {
++ dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n",
++ __func__, lreq, lreq->linger_id, req, lreq->ping_req);
++ goto out;
++ }
++
+ dout("%s lreq %p linger_id %llu result %d ping_sent %lu last_error %d\n",
+ __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent,
+ lreq->last_error);
+@@ -3117,6 +3136,7 @@ static void linger_ping_cb(struct ceph_osd_request *req)
+ lreq->register_gen, req->r_ops[0].watch.gen);
+ }
+
++out:
+ mutex_unlock(&lreq->lock);
+ linger_put(lreq);
+ }
+@@ -3124,8 +3144,8 @@ static void linger_ping_cb(struct ceph_osd_request *req)
+ static void send_linger_ping(struct ceph_osd_linger_request *lreq)
+ {
+ struct ceph_osd_client *osdc = lreq->osdc;
+- struct ceph_osd_request *req = lreq->ping_req;
+- struct ceph_osd_req_op *op = &req->r_ops[0];
++ struct ceph_osd_request *req;
++ int ret;
+
+ if (ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD)) {
+ dout("%s PAUSERD\n", __func__);
+@@ -3137,19 +3157,26 @@ static void send_linger_ping(struct ceph_osd_linger_request *lreq)
+ __func__, lreq, lreq->linger_id, lreq->ping_sent,
+ lreq->register_gen);
+
+- if (req->r_osd)
+- cancel_linger_request(req);
++ if (lreq->ping_req) {
++ if (lreq->ping_req->r_osd)
++ cancel_linger_request(lreq->ping_req);
++ ceph_osdc_put_request(lreq->ping_req);
++ }
+
+- request_reinit(req);
+- target_copy(&req->r_t, &lreq->t);
++ req = ceph_osdc_alloc_request(osdc, NULL, 1, true, GFP_NOIO);
++ BUG_ON(!req);
+
+- WARN_ON(op->op != CEPH_OSD_OP_WATCH ||
+- op->watch.cookie != lreq->linger_id ||
+- op->watch.op != CEPH_OSD_WATCH_OP_PING);
+- op->watch.gen = lreq->register_gen;
++ target_copy(&req->r_t, &lreq->t);
++ osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_PING, lreq->linger_id,
++ lreq->register_gen);
+ req->r_callback = linger_ping_cb;
++
++ ret = ceph_osdc_alloc_messages(req, GFP_NOIO);
++ BUG_ON(ret);
++
+ req->r_priv = linger_get(lreq);
+ req->r_linger = true;
++ lreq->ping_req = req;
+
+ ceph_osdc_get_request(req);
+ account_request(req);
+@@ -3165,12 +3192,6 @@ static void linger_submit(struct ceph_osd_linger_request *lreq)
+
+ down_write(&osdc->lock);
+ linger_register(lreq);
+- if (lreq->is_watch) {
+- lreq->reg_req->r_ops[0].watch.cookie = lreq->linger_id;
+- lreq->ping_req->r_ops[0].watch.cookie = lreq->linger_id;
+- } else {
+- lreq->reg_req->r_ops[0].notify.cookie = lreq->linger_id;
+- }
+
+ calc_target(osdc, &lreq->t, false);
+ osd = lookup_create_osd(osdc, lreq->t.osd, true);
+@@ -3202,9 +3223,9 @@ static void cancel_linger_map_check(struct ceph_osd_linger_request *lreq)
+ */
+ static void __linger_cancel(struct ceph_osd_linger_request *lreq)
+ {
+- if (lreq->is_watch && lreq->ping_req->r_osd)
++ if (lreq->ping_req && lreq->ping_req->r_osd)
+ cancel_linger_request(lreq->ping_req);
+- if (lreq->reg_req->r_osd)
++ if (lreq->reg_req && lreq->reg_req->r_osd)
+ cancel_linger_request(lreq->reg_req);
+ cancel_linger_map_check(lreq);
+ unlink_linger(lreq->osd, lreq);
+@@ -4653,43 +4674,6 @@ again:
+ }
+ EXPORT_SYMBOL(ceph_osdc_sync);
+
+-static struct ceph_osd_request *
+-alloc_linger_request(struct ceph_osd_linger_request *lreq)
+-{
+- struct ceph_osd_request *req;
+-
+- req = ceph_osdc_alloc_request(lreq->osdc, NULL, 1, false, GFP_NOIO);
+- if (!req)
+- return NULL;
+-
+- ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid);
+- ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc);
+- return req;
+-}
+-
+-static struct ceph_osd_request *
+-alloc_watch_request(struct ceph_osd_linger_request *lreq, u8 watch_opcode)
+-{
+- struct ceph_osd_request *req;
+-
+- req = alloc_linger_request(lreq);
+- if (!req)
+- return NULL;
+-
+- /*
+- * Pass 0 for cookie because we don't know it yet, it will be
+- * filled in by linger_submit().
+- */
+- osd_req_op_watch_init(req, 0, 0, watch_opcode);
+-
+- if (ceph_osdc_alloc_messages(req, GFP_NOIO)) {
+- ceph_osdc_put_request(req);
+- return NULL;
+- }
+-
+- return req;
+-}
+-
+ /*
+ * Returns a handle, caller owns a ref.
+ */
+@@ -4719,18 +4703,6 @@ ceph_osdc_watch(struct ceph_osd_client *osdc,
+ lreq->t.flags = CEPH_OSD_FLAG_WRITE;
+ ktime_get_real_ts64(&lreq->mtime);
+
+- lreq->reg_req = alloc_watch_request(lreq, CEPH_OSD_WATCH_OP_WATCH);
+- if (!lreq->reg_req) {
+- ret = -ENOMEM;
+- goto err_put_lreq;
+- }
+-
+- lreq->ping_req = alloc_watch_request(lreq, CEPH_OSD_WATCH_OP_PING);
+- if (!lreq->ping_req) {
+- ret = -ENOMEM;
+- goto err_put_lreq;
+- }
+-
+ linger_submit(lreq);
+ ret = linger_reg_commit_wait(lreq);
+ if (ret) {
+@@ -4768,8 +4740,8 @@ int ceph_osdc_unwatch(struct ceph_osd_client *osdc,
+ ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc);
+ req->r_flags = CEPH_OSD_FLAG_WRITE;
+ ktime_get_real_ts64(&req->r_mtime);
+- osd_req_op_watch_init(req, 0, lreq->linger_id,
+- CEPH_OSD_WATCH_OP_UNWATCH);
++ osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_UNWATCH,
++ lreq->linger_id, 0);
+
+ ret = ceph_osdc_alloc_messages(req, GFP_NOIO);
+ if (ret)
+@@ -4855,35 +4827,6 @@ out_put_req:
+ }
+ EXPORT_SYMBOL(ceph_osdc_notify_ack);
+
+-static int osd_req_op_notify_init(struct ceph_osd_request *req, int which,
+- u64 cookie, u32 prot_ver, u32 timeout,
+- void *payload, u32 payload_len)
+-{
+- struct ceph_osd_req_op *op;
+- struct ceph_pagelist *pl;
+- int ret;
+-
+- op = osd_req_op_init(req, which, CEPH_OSD_OP_NOTIFY, 0);
+- op->notify.cookie = cookie;
+-
+- pl = ceph_pagelist_alloc(GFP_NOIO);
+- if (!pl)
+- return -ENOMEM;
+-
+- ret = ceph_pagelist_encode_32(pl, 1); /* prot_ver */
+- ret |= ceph_pagelist_encode_32(pl, timeout);
+- ret |= ceph_pagelist_encode_32(pl, payload_len);
+- ret |= ceph_pagelist_append(pl, payload, payload_len);
+- if (ret) {
+- ceph_pagelist_release(pl);
+- return -ENOMEM;
+- }
+-
+- ceph_osd_data_pagelist_init(&op->notify.request_data, pl);
+- op->indata_len = pl->length;
+- return 0;
+-}
+-
+ /*
+ * @timeout: in seconds
+ *
+@@ -4902,7 +4845,6 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,
+ size_t *preply_len)
+ {
+ struct ceph_osd_linger_request *lreq;
+- struct page **pages;
+ int ret;
+
+ WARN_ON(!timeout);
+@@ -4915,41 +4857,35 @@ int ceph_osdc_notify(struct ceph_osd_client *osdc,
+ if (!lreq)
+ return -ENOMEM;
+
+- lreq->preply_pages = preply_pages;
+- lreq->preply_len = preply_len;
+-
+- ceph_oid_copy(&lreq->t.base_oid, oid);
+- ceph_oloc_copy(&lreq->t.base_oloc, oloc);
+- lreq->t.flags = CEPH_OSD_FLAG_READ;
+-
+- lreq->reg_req = alloc_linger_request(lreq);
+- if (!lreq->reg_req) {
++ lreq->request_pl = ceph_pagelist_alloc(GFP_NOIO);
++ if (!lreq->request_pl) {
+ ret = -ENOMEM;
+ goto out_put_lreq;
+ }
+
+- /*
+- * Pass 0 for cookie because we don't know it yet, it will be
+- * filled in by linger_submit().
+- */
+- ret = osd_req_op_notify_init(lreq->reg_req, 0, 0, 1, timeout,
+- payload, payload_len);
+- if (ret)
++ ret = ceph_pagelist_encode_32(lreq->request_pl, 1); /* prot_ver */
++ ret |= ceph_pagelist_encode_32(lreq->request_pl, timeout);
++ ret |= ceph_pagelist_encode_32(lreq->request_pl, payload_len);
++ ret |= ceph_pagelist_append(lreq->request_pl, payload, payload_len);
++ if (ret) {
++ ret = -ENOMEM;
+ goto out_put_lreq;
++ }
+
+ /* for notify_id */
+- pages = ceph_alloc_page_vector(1, GFP_NOIO);
+- if (IS_ERR(pages)) {
+- ret = PTR_ERR(pages);
++ lreq->notify_id_pages = ceph_alloc_page_vector(1, GFP_NOIO);
++ if (IS_ERR(lreq->notify_id_pages)) {
++ ret = PTR_ERR(lreq->notify_id_pages);
++ lreq->notify_id_pages = NULL;
+ goto out_put_lreq;
+ }
+- ceph_osd_data_pages_init(osd_req_op_data(lreq->reg_req, 0, notify,
+- response_data),
+- pages, PAGE_SIZE, 0, false, true);
+
+- ret = ceph_osdc_alloc_messages(lreq->reg_req, GFP_NOIO);
+- if (ret)
+- goto out_put_lreq;
++ lreq->preply_pages = preply_pages;
++ lreq->preply_len = preply_len;
++
++ ceph_oid_copy(&lreq->t.base_oid, oid);
++ ceph_oloc_copy(&lreq->t.base_oloc, oloc);
++ lreq->t.flags = CEPH_OSD_FLAG_READ;
+
+ linger_submit(lreq);
+ ret = linger_reg_commit_wait(lreq);
+diff --git a/net/core/Makefile b/net/core/Makefile
+index 35ced6201814c..4268846f2f475 100644
+--- a/net/core/Makefile
++++ b/net/core/Makefile
+@@ -36,3 +36,4 @@ obj-$(CONFIG_FAILOVER) += failover.o
+ obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o
+ obj-$(CONFIG_BPF_SYSCALL) += sock_map.o
+ obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o
++obj-$(CONFIG_OF) += of_net.o
+diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
+index 68d2cbf8331ac..ea61dfe19c869 100644
+--- a/net/core/bpf_sk_storage.c
++++ b/net/core/bpf_sk_storage.c
+@@ -929,7 +929,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
+ { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
+ PTR_TO_BTF_ID_OR_NULL },
+ { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
+- PTR_TO_RDWR_BUF_OR_NULL },
++ PTR_TO_BUF | PTR_MAYBE_NULL },
+ },
+ .seq_info = &iter_seq_info,
+ };
+diff --git a/net/core/dev.c b/net/core/dev.c
+index eb3a366bf212c..5907212c00f37 100644
+--- a/net/core/dev.c
++++ b/net/core/dev.c
+@@ -741,11 +741,11 @@ int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
+ const struct net_device *last_dev;
+ struct net_device_path_ctx ctx = {
+ .dev = dev,
+- .daddr = daddr,
+ };
+ struct net_device_path *path;
+ int ret = 0;
+
++ memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
+ stack->num_paths = 0;
+ while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
+ last_dev = ctx.dev;
+@@ -2921,6 +2921,8 @@ int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
+ if (dev->num_tc)
+ netif_setup_tc(dev, txq);
+
++ dev_qdisc_change_real_num_tx(dev, txq);
++
+ dev->real_num_tx_queues = txq;
+
+ if (disabling) {
+@@ -3932,8 +3934,8 @@ sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
+ return skb;
+
+ /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
+- qdisc_skb_cb(skb)->mru = 0;
+- qdisc_skb_cb(skb)->post_ct = false;
++ tc_skb_cb(skb)->mru = 0;
++ tc_skb_cb(skb)->post_ct = false;
+ mini_qdisc_bstats_cpu_update(miniq, skb);
+
+ switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
+@@ -4193,7 +4195,10 @@ static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
+ if (dev->flags & IFF_UP) {
+ int cpu = smp_processor_id(); /* ok because BHs are off */
+
+- if (txq->xmit_lock_owner != cpu) {
++ /* Other cpus might concurrently change txq->xmit_lock_owner
++ * to -1 or to their cpu id, but not to our id.
++ */
++ if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
+ if (dev_xmit_recursion())
+ goto recursion_alert;
+
+@@ -5083,8 +5088,8 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
+ }
+
+ qdisc_skb_cb(skb)->pkt_len = skb->len;
+- qdisc_skb_cb(skb)->mru = 0;
+- qdisc_skb_cb(skb)->post_ct = false;
++ tc_skb_cb(skb)->mru = 0;
++ tc_skb_cb(skb)->post_ct = false;
+ skb->tc_at_ingress = 1;
+ mini_qdisc_bstats_cpu_update(miniq, skb);
+
+@@ -9631,6 +9636,12 @@ static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
+ goto out_unlock;
+ }
+ old_prog = link->prog;
++ if (old_prog->type != new_prog->type ||
++ old_prog->expected_attach_type != new_prog->expected_attach_type) {
++ err = -EINVAL;
++ goto out_unlock;
++ }
++
+ if (old_prog == new_prog) {
+ /* no-op, don't disturb drivers */
+ bpf_prog_put(new_prog);
+@@ -11367,8 +11378,7 @@ static int __net_init netdev_init(struct net *net)
+ BUILD_BUG_ON(GRO_HASH_BUCKETS >
+ 8 * sizeof_field(struct napi_struct, gro_bitmask));
+
+- if (net != &init_net)
+- INIT_LIST_HEAD(&net->dev_base_head);
++ INIT_LIST_HEAD(&net->dev_base_head);
+
+ net->dev_name_head = netdev_create_hash();
+ if (net->dev_name_head == NULL)
+diff --git a/net/core/devlink.c b/net/core/devlink.c
+index a856ae401ea5c..db76c55e1a6d7 100644
+--- a/net/core/devlink.c
++++ b/net/core/devlink.c
+@@ -4031,14 +4031,6 @@ static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
+ return err;
+ }
+
+- if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
+- info->attrs[DEVLINK_ATTR_NETNS_FD] ||
+- info->attrs[DEVLINK_ATTR_NETNS_ID]) {
+- dest_net = devlink_netns_get(skb, info);
+- if (IS_ERR(dest_net))
+- return PTR_ERR(dest_net);
+- }
+-
+ if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
+ action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
+ else
+@@ -4081,6 +4073,14 @@ static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
+ return -EINVAL;
+ }
+ }
++ if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
++ info->attrs[DEVLINK_ATTR_NETNS_FD] ||
++ info->attrs[DEVLINK_ATTR_NETNS_ID]) {
++ dest_net = devlink_netns_get(skb, info);
++ if (IS_ERR(dest_net))
++ return PTR_ERR(dest_net);
++ }
++
+ err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
+
+ if (dest_net)
+@@ -8795,8 +8795,6 @@ static const struct genl_small_ops devlink_nl_ops[] = {
+ GENL_DONT_VALIDATE_DUMP_STRICT,
+ .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
+ .flags = GENL_ADMIN_PERM,
+- .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
+- DEVLINK_NL_FLAG_NO_LOCK,
+ },
+ {
+ .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
+diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c
+index 49442cae6f69d..1d99b731e5b21 100644
+--- a/net/core/drop_monitor.c
++++ b/net/core/drop_monitor.c
+@@ -280,13 +280,17 @@ static void trace_napi_poll_hit(void *ignore, struct napi_struct *napi,
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(new_stat, &hw_stats_list, list) {
++ struct net_device *dev;
++
+ /*
+ * only add a note to our monitor buffer if:
+ * 1) this is the dev we received on
+ * 2) its after the last_rx delta
+ * 3) our rx_dropped count has gone up
+ */
+- if ((new_stat->dev == napi->dev) &&
++ /* Paired with WRITE_ONCE() in dropmon_net_event() */
++ dev = READ_ONCE(new_stat->dev);
++ if ((dev == napi->dev) &&
+ (time_after(jiffies, new_stat->last_rx + dm_hw_check_delta)) &&
+ (napi->dev->stats.rx_dropped != new_stat->last_drop_val)) {
+ trace_drop_common(NULL, NULL);
+@@ -1572,7 +1576,10 @@ static int dropmon_net_event(struct notifier_block *ev_block,
+ mutex_lock(&net_dm_mutex);
+ list_for_each_entry_safe(new_stat, tmp, &hw_stats_list, list) {
+ if (new_stat->dev == dev) {
+- new_stat->dev = NULL;
++
++ /* Paired with READ_ONCE() in trace_napi_poll_hit() */
++ WRITE_ONCE(new_stat->dev, NULL);
++
+ if (trace_state == TRACE_OFF) {
+ list_del_rcu(&new_stat->list);
+ kfree_rcu(new_stat, rcu);
+diff --git a/net/core/dst_cache.c b/net/core/dst_cache.c
+index be74ab4551c20..0ccfd5fa5cb9b 100644
+--- a/net/core/dst_cache.c
++++ b/net/core/dst_cache.c
+@@ -162,3 +162,22 @@ void dst_cache_destroy(struct dst_cache *dst_cache)
+ free_percpu(dst_cache->cache);
+ }
+ EXPORT_SYMBOL_GPL(dst_cache_destroy);
++
++void dst_cache_reset_now(struct dst_cache *dst_cache)
++{
++ int i;
++
++ if (!dst_cache->cache)
++ return;
++
++ dst_cache->reset_ts = jiffies;
++ for_each_possible_cpu(i) {
++ struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
++ struct dst_entry *dst = idst->dst;
++
++ idst->cookie = 0;
++ idst->dst = NULL;
++ dst_release(dst);
++ }
++}
++EXPORT_SYMBOL_GPL(dst_cache_reset_now);
+diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
+index 79df7cd9dbc16..1bb567a3b329c 100644
+--- a/net/core/fib_rules.c
++++ b/net/core/fib_rules.c
+@@ -323,7 +323,7 @@ jumped:
+ if (!err && ops->suppress && INDIRECT_CALL_MT(ops->suppress,
+ fib6_rule_suppress,
+ fib4_rule_suppress,
+- rule, arg))
++ rule, flags, arg))
+ continue;
+
+ if (err != -EAGAIN) {
+diff --git a/net/core/filter.c b/net/core/filter.c
+index 2e32cee2c4690..821278b906b71 100644
+--- a/net/core/filter.c
++++ b/net/core/filter.c
+@@ -1713,7 +1713,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE,
+ .arg5_type = ARG_ANYTHING,
+ };
+@@ -2018,9 +2018,9 @@ static const struct bpf_func_proto bpf_csum_diff_proto = {
+ .gpl_only = false,
+ .pkt_access = true,
+ .ret_type = RET_INTEGER,
+- .arg1_type = ARG_PTR_TO_MEM_OR_NULL,
++ .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+- .arg3_type = ARG_PTR_TO_MEM_OR_NULL,
++ .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_ANYTHING,
+ };
+@@ -2541,7 +2541,7 @@ static const struct bpf_func_proto bpf_redirect_neigh_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_ANYTHING,
+- .arg2_type = ARG_PTR_TO_MEM_OR_NULL,
++ .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_ANYTHING,
+ };
+@@ -2711,6 +2711,9 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
+ if (unlikely(flags))
+ return -EINVAL;
+
++ if (unlikely(len == 0))
++ return 0;
++
+ /* First find the starting scatterlist element */
+ i = msg->sg.start;
+ do {
+@@ -4174,7 +4177,7 @@ static const struct bpf_func_proto bpf_skb_event_output_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -4188,7 +4191,7 @@ const struct bpf_func_proto bpf_skb_output_proto = {
+ .arg1_btf_id = &bpf_skb_output_btf_ids[0],
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -4371,7 +4374,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ };
+@@ -4397,7 +4400,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ };
+
+@@ -4567,7 +4570,7 @@ static const struct bpf_func_proto bpf_xdp_event_output_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -4581,7 +4584,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = {
+ .arg1_btf_id = &bpf_xdp_output_btf_ids[0],
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ };
+
+@@ -4742,12 +4745,14 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname,
+ switch (optname) {
+ case SO_RCVBUF:
+ val = min_t(u32, val, sysctl_rmem_max);
++ val = min_t(int, val, INT_MAX / 2);
+ sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
+ WRITE_ONCE(sk->sk_rcvbuf,
+ max_t(int, val * 2, SOCK_MIN_RCVBUF));
+ break;
+ case SO_SNDBUF:
+ val = min_t(u32, val, sysctl_wmem_max);
++ val = min_t(int, val, INT_MAX / 2);
+ sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
+ WRITE_ONCE(sk->sk_sndbuf,
+ max_t(int, val * 2, SOCK_MIN_SNDBUF));
+@@ -5067,7 +5072,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_proto = {
+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+ };
+
+@@ -5101,7 +5106,7 @@ static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+ };
+
+@@ -5135,7 +5140,7 @@ static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = {
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+ };
+
+@@ -5310,7 +5315,7 @@ static const struct bpf_func_proto bpf_bind_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ };
+
+@@ -5898,7 +5903,7 @@ static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE
+ };
+
+@@ -5908,7 +5913,7 @@ static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE
+ };
+
+@@ -5951,7 +5956,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE
+ };
+
+@@ -6039,7 +6044,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_action_proto = {
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+- .arg3_type = ARG_PTR_TO_MEM,
++ .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg4_type = ARG_CONST_SIZE
+ };
+
+@@ -6264,7 +6269,7 @@ static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6283,7 +6288,7 @@ static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6302,7 +6307,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6339,7 +6344,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6362,7 +6367,7 @@ static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6385,7 +6390,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {
+ .pkt_access = true,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6404,7 +6409,7 @@ static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = {
+ .gpl_only = false,
+ .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6423,7 +6428,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {
+ .gpl_only = false,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6442,7 +6447,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {
+ .gpl_only = false,
+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ .arg5_type = ARG_ANYTHING,
+@@ -6714,24 +6719,33 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len
+ if (!th->ack || th->rst || th->syn)
+ return -ENOENT;
+
++ if (unlikely(iph_len < sizeof(struct iphdr)))
++ return -EINVAL;
++
+ if (tcp_synq_no_recent_overflow(sk))
+ return -ENOENT;
+
+ cookie = ntohl(th->ack_seq) - 1;
+
+- switch (sk->sk_family) {
+- case AF_INET:
+- if (unlikely(iph_len < sizeof(struct iphdr)))
++ /* Both struct iphdr and struct ipv6hdr have the version field at the
++ * same offset so we can cast to the shorter header (struct iphdr).
++ */
++ switch (((struct iphdr *)iph)->version) {
++ case 4:
++ if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
+ return -EINVAL;
+
+ ret = __cookie_v4_check((struct iphdr *)iph, th, cookie);
+ break;
+
+ #if IS_BUILTIN(CONFIG_IPV6)
+- case AF_INET6:
++ case 6:
+ if (unlikely(iph_len < sizeof(struct ipv6hdr)))
+ return -EINVAL;
+
++ if (sk->sk_family != AF_INET6)
++ return -EINVAL;
++
+ ret = __cookie_v6_check((struct ipv6hdr *)iph, th, cookie);
+ break;
+ #endif /* CONFIG_IPV6 */
+@@ -6755,9 +6769,9 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = {
+ .pkt_access = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+ };
+
+@@ -6824,9 +6838,9 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = {
+ .pkt_access = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+- .arg4_type = ARG_PTR_TO_MEM,
++ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+ };
+
+@@ -7055,7 +7069,7 @@ static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+- .arg2_type = ARG_PTR_TO_MEM,
++ .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg3_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_ANYTHING,
+ };
+@@ -7162,6 +7176,8 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+ #endif
+ case BPF_FUNC_sk_storage_get:
+ return &bpf_sk_storage_get_cg_sock_proto;
++ case BPF_FUNC_ktime_get_coarse_ns:
++ return &bpf_ktime_get_coarse_ns_proto;
+ default:
+ return bpf_base_func_proto(func_id);
+ }
+@@ -7959,6 +7975,7 @@ bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
+ struct bpf_insn_access_aux *info)
+ {
+ const int size_default = sizeof(__u32);
++ int field_size;
+
+ if (off < 0 || off >= sizeof(struct bpf_sock))
+ return false;
+@@ -7970,7 +7987,6 @@ bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
+ case offsetof(struct bpf_sock, family):
+ case offsetof(struct bpf_sock, type):
+ case offsetof(struct bpf_sock, protocol):
+- case offsetof(struct bpf_sock, dst_port):
+ case offsetof(struct bpf_sock, src_port):
+ case offsetof(struct bpf_sock, rx_queue_mapping):
+ case bpf_ctx_range(struct bpf_sock, src_ip4):
+@@ -7979,6 +7995,14 @@ bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
+ case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
+ bpf_ctx_record_field_size(info, size_default);
+ return bpf_ctx_narrow_access_ok(off, size, size_default);
++ case bpf_ctx_range(struct bpf_sock, dst_port):
++ field_size = size == size_default ?
++ size_default : sizeof_field(struct bpf_sock, dst_port);
++ bpf_ctx_record_field_size(info, field_size);
++ return bpf_ctx_narrow_access_ok(off, size, field_size);
++ case offsetofend(struct bpf_sock, dst_port) ...
++ offsetof(struct bpf_sock, dst_ip4) - 1:
++ return false;
+ }
+
+ return size == size_default;
+@@ -8174,9 +8198,9 @@ void bpf_warn_invalid_xdp_action(u32 act)
+ {
+ const u32 act_max = XDP_REDIRECT;
+
+- WARN_ONCE(1, "%s XDP return value %u, expect packet loss!\n",
+- act > act_max ? "Illegal" : "Driver unsupported",
+- act);
++ pr_warn_once("%s XDP return value %u, expect packet loss!\n",
++ act > act_max ? "Illegal" : "Driver unsupported",
++ act);
+ }
+ EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
+
+@@ -9735,22 +9759,46 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
+ static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si,
+ struct bpf_insn *insn)
+ {
+- /* si->dst_reg = skb->data */
++ int reg;
++ int temp_reg_off = offsetof(struct sk_buff, cb) +
++ offsetof(struct sk_skb_cb, temp_reg);
++
++ if (si->src_reg == si->dst_reg) {
++ /* We need an extra register, choose and save a register. */
++ reg = BPF_REG_9;
++ if (si->src_reg == reg || si->dst_reg == reg)
++ reg--;
++ if (si->src_reg == reg || si->dst_reg == reg)
++ reg--;
++ *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, temp_reg_off);
++ } else {
++ reg = si->dst_reg;
++ }
++
++ /* reg = skb->data */
+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
+- si->dst_reg, si->src_reg,
++ reg, si->src_reg,
+ offsetof(struct sk_buff, data));
+ /* AX = skb->len */
+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
+ BPF_REG_AX, si->src_reg,
+ offsetof(struct sk_buff, len));
+- /* si->dst_reg = skb->data + skb->len */
+- *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX);
++ /* reg = skb->data + skb->len */
++ *insn++ = BPF_ALU64_REG(BPF_ADD, reg, BPF_REG_AX);
+ /* AX = skb->data_len */
+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len),
+ BPF_REG_AX, si->src_reg,
+ offsetof(struct sk_buff, data_len));
+- /* si->dst_reg = skb->data + skb->len - skb->data_len */
+- *insn++ = BPF_ALU64_REG(BPF_SUB, si->dst_reg, BPF_REG_AX);
++
++ /* reg = skb->data + skb->len - skb->data_len */
++ *insn++ = BPF_ALU64_REG(BPF_SUB, reg, BPF_REG_AX);
++
++ if (si->src_reg == si->dst_reg) {
++ /* Restore the saved register */
++ *insn++ = BPF_MOV64_REG(BPF_REG_AX, si->src_reg);
++ *insn++ = BPF_MOV64_REG(si->dst_reg, reg);
++ *insn++ = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_AX, temp_reg_off);
++ }
+
+ return insn;
+ }
+@@ -9761,11 +9809,33 @@ static u32 sk_skb_convert_ctx_access(enum bpf_access_type type,
+ struct bpf_prog *prog, u32 *target_size)
+ {
+ struct bpf_insn *insn = insn_buf;
++ int off;
+
+ switch (si->off) {
+ case offsetof(struct __sk_buff, data_end):
+ insn = bpf_convert_data_end_access(si, insn);
+ break;
++ case offsetof(struct __sk_buff, cb[0]) ...
++ offsetofend(struct __sk_buff, cb[4]) - 1:
++ BUILD_BUG_ON(sizeof_field(struct sk_skb_cb, data) < 20);
++ BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
++ offsetof(struct sk_skb_cb, data)) %
++ sizeof(__u64));
++
++ prog->cb_access = 1;
++ off = si->off;
++ off -= offsetof(struct __sk_buff, cb[0]);
++ off += offsetof(struct sk_buff, cb);
++ off += offsetof(struct sk_skb_cb, data);
++ if (type == BPF_WRITE)
++ *insn++ = BPF_STX_MEM(BPF_SIZE(si->code), si->dst_reg,
++ si->src_reg, off);
++ else
++ *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
++ si->src_reg, off);
++ break;
++
++
+ default:
+ return bpf_convert_ctx_access(type, si, insn_buf, prog,
+ target_size);
+@@ -10260,6 +10330,8 @@ sk_reuseport_func_proto(enum bpf_func_id func_id,
+ return &sk_reuseport_load_bytes_relative_proto;
+ case BPF_FUNC_get_socket_cookie:
+ return &bpf_get_socket_ptr_cookie_proto;
++ case BPF_FUNC_ktime_get_coarse_ns:
++ return &bpf_ktime_get_coarse_ns_proto;
+ default:
+ return bpf_base_func_proto(func_id);
+ }
+@@ -10468,7 +10540,8 @@ static bool sk_lookup_is_valid_access(int off, int size,
+ case bpf_ctx_range(struct bpf_sk_lookup, local_ip4):
+ case bpf_ctx_range_till(struct bpf_sk_lookup, remote_ip6[0], remote_ip6[3]):
+ case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]):
+- case bpf_ctx_range(struct bpf_sk_lookup, remote_port):
++ case offsetof(struct bpf_sk_lookup, remote_port) ...
++ offsetof(struct bpf_sk_lookup, local_ip4) - 1:
+ case bpf_ctx_range(struct bpf_sk_lookup, local_port):
+ bpf_ctx_record_field_size(info, sizeof(__u32));
+ return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32));
+@@ -10741,6 +10814,8 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
+ case BPF_FUNC_skc_to_udp6_sock:
+ func = &bpf_skc_to_udp6_sock_proto;
+ break;
++ case BPF_FUNC_ktime_get_coarse_ns:
++ return &bpf_ktime_get_coarse_ns_proto;
+ default:
+ return bpf_base_func_proto(func_id);
+ }
+diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
+index bac0184cf3de7..bc50bd331d5b6 100644
+--- a/net/core/flow_dissector.c
++++ b/net/core/flow_dissector.c
+@@ -238,7 +238,7 @@ void
+ skb_flow_dissect_ct(const struct sk_buff *skb,
+ struct flow_dissector *flow_dissector,
+ void *target_container, u16 *ctinfo_map,
+- size_t mapsize, bool post_ct)
++ size_t mapsize, bool post_ct, u16 zone)
+ {
+ #if IS_ENABLED(CONFIG_NF_CONNTRACK)
+ struct flow_dissector_key_ct *key;
+@@ -260,6 +260,7 @@ skb_flow_dissect_ct(const struct sk_buff *skb,
+ if (!ct) {
+ key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
+ TCA_FLOWER_KEY_CT_FLAGS_INVALID;
++ key->ct_zone = zone;
+ return;
+ }
+
+@@ -1180,6 +1181,7 @@ proto_again:
+ VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ }
+ key_vlan->vlan_tpid = saved_vlan_tpid;
++ key_vlan->vlan_eth_type = proto;
+ }
+
+ fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
+diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c
+index 2f7940bcf7151..3fd207fe1284a 100644
+--- a/net/core/lwt_bpf.c
++++ b/net/core/lwt_bpf.c
+@@ -158,10 +158,8 @@ static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb)
+ return dst->lwtstate->orig_output(net, sk, skb);
+ }
+
+-static int xmit_check_hhlen(struct sk_buff *skb)
++static int xmit_check_hhlen(struct sk_buff *skb, int hh_len)
+ {
+- int hh_len = skb_dst(skb)->dev->hard_header_len;
+-
+ if (skb_headroom(skb) < hh_len) {
+ int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
+
+@@ -273,6 +271,7 @@ static int bpf_xmit(struct sk_buff *skb)
+
+ bpf = bpf_lwt_lwtunnel(dst->lwtstate);
+ if (bpf->xmit.prog) {
++ int hh_len = dst->dev->hard_header_len;
+ __be16 proto = skb->protocol;
+ int ret;
+
+@@ -290,7 +289,7 @@ static int bpf_xmit(struct sk_buff *skb)
+ /* If the header was expanded, headroom might be too
+ * small for L2 header to come, expand as needed.
+ */
+- ret = xmit_check_hhlen(skb);
++ ret = xmit_check_hhlen(skb, hh_len);
+ if (unlikely(ret))
+ return ret;
+
+diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c
+index 2820aca2173a8..9ccd64e8a666a 100644
+--- a/net/core/lwtunnel.c
++++ b/net/core/lwtunnel.c
+@@ -197,6 +197,10 @@ int lwtunnel_valid_encap_type_attr(struct nlattr *attr, int remaining,
+ nla_entype = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
+
+ if (nla_entype) {
++ if (nla_len(nla_entype) < sizeof(u16)) {
++ NL_SET_ERR_MSG(extack, "Invalid RTA_ENCAP_TYPE");
++ return -EINVAL;
++ }
+ encap_type = nla_get_u16(nla_entype);
+
+ if (lwtunnel_valid_encap_type(encap_type,
+diff --git a/net/core/neighbour.c b/net/core/neighbour.c
+index 2d5bc3a75faec..ff049733cceeb 100644
+--- a/net/core/neighbour.c
++++ b/net/core/neighbour.c
+@@ -379,7 +379,7 @@ EXPORT_SYMBOL(neigh_ifdown);
+
+ static struct neighbour *neigh_alloc(struct neigh_table *tbl,
+ struct net_device *dev,
+- bool exempt_from_gc)
++ u8 flags, bool exempt_from_gc)
+ {
+ struct neighbour *n = NULL;
+ unsigned long now = jiffies;
+@@ -412,6 +412,7 @@ do_alloc:
+ n->updated = n->used = now;
+ n->nud_state = NUD_NONE;
+ n->output = neigh_blackhole;
++ n->flags = flags;
+ seqlock_init(&n->hh.hh_lock);
+ n->parms = neigh_parms_clone(&tbl->parms);
+ timer_setup(&n->timer, neigh_timer_handler, 0);
+@@ -575,19 +576,18 @@ struct neighbour *neigh_lookup_nodev(struct neigh_table *tbl, struct net *net,
+ }
+ EXPORT_SYMBOL(neigh_lookup_nodev);
+
+-static struct neighbour *___neigh_create(struct neigh_table *tbl,
+- const void *pkey,
+- struct net_device *dev,
+- bool exempt_from_gc, bool want_ref)
++static struct neighbour *
++___neigh_create(struct neigh_table *tbl, const void *pkey,
++ struct net_device *dev, u8 flags,
++ bool exempt_from_gc, bool want_ref)
+ {
+- struct neighbour *n1, *rc, *n = neigh_alloc(tbl, dev, exempt_from_gc);
+- u32 hash_val;
+- unsigned int key_len = tbl->key_len;
+- int error;
++ u32 hash_val, key_len = tbl->key_len;
++ struct neighbour *n1, *rc, *n;
+ struct neigh_hash_table *nht;
++ int error;
+
++ n = neigh_alloc(tbl, dev, flags, exempt_from_gc);
+ trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);
+-
+ if (!n) {
+ rc = ERR_PTR(-ENOBUFS);
+ goto out;
+@@ -674,7 +674,7 @@ out_neigh_release:
+ struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
+ struct net_device *dev, bool want_ref)
+ {
+- return ___neigh_create(tbl, pkey, dev, false, want_ref);
++ return ___neigh_create(tbl, pkey, dev, 0, false, want_ref);
+ }
+ EXPORT_SYMBOL(__neigh_create);
+
+@@ -733,11 +733,10 @@ struct pneigh_entry * pneigh_lookup(struct neigh_table *tbl,
+
+ ASSERT_RTNL();
+
+- n = kmalloc(sizeof(*n) + key_len, GFP_KERNEL);
++ n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL);
+ if (!n)
+ goto out;
+
+- n->protocol = 0;
+ write_pnet(&n->net, net);
+ memcpy(n->key, pkey, key_len);
+ n->dev = dev;
+@@ -1217,7 +1216,7 @@ static void neigh_update_hhs(struct neighbour *neigh)
+ lladdr instead of overriding it
+ if it is different.
+ NEIGH_UPDATE_F_ADMIN means that the change is administrative.
+-
++ NEIGH_UPDATE_F_USE means that the entry is user triggered.
+ NEIGH_UPDATE_F_OVERRIDE_ISROUTER allows to override existing
+ NTF_ROUTER flag.
+ NEIGH_UPDATE_F_ISROUTER indicates if the neighbour is known as
+@@ -1255,6 +1254,12 @@ static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
+ goto out;
+
+ ext_learn_change = neigh_update_ext_learned(neigh, flags, &notify);
++ if (flags & NEIGH_UPDATE_F_USE) {
++ new = old & ~NUD_PERMANENT;
++ neigh->nud_state = new;
++ err = 0;
++ goto out;
++ }
+
+ if (!(new & NUD_VALID)) {
+ neigh_del_timer(neigh);
+@@ -1942,7 +1947,9 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
+
+ exempt_from_gc = ndm->ndm_state & NUD_PERMANENT ||
+ ndm->ndm_flags & NTF_EXT_LEARNED;
+- neigh = ___neigh_create(tbl, dst, dev, exempt_from_gc, true);
++ neigh = ___neigh_create(tbl, dst, dev,
++ ndm->ndm_flags & NTF_EXT_LEARNED,
++ exempt_from_gc, true);
+ if (IS_ERR(neigh)) {
+ err = PTR_ERR(neigh);
+ goto out;
+@@ -1961,22 +1968,20 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
+
+ if (protocol)
+ neigh->protocol = protocol;
+-
+ if (ndm->ndm_flags & NTF_EXT_LEARNED)
+ flags |= NEIGH_UPDATE_F_EXT_LEARNED;
+-
+ if (ndm->ndm_flags & NTF_ROUTER)
+ flags |= NEIGH_UPDATE_F_ISROUTER;
++ if (ndm->ndm_flags & NTF_USE)
++ flags |= NEIGH_UPDATE_F_USE;
+
+- if (ndm->ndm_flags & NTF_USE) {
++ err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,
++ NETLINK_CB(skb).portid, extack);
++ if (!err && ndm->ndm_flags & NTF_USE) {
+ neigh_event_send(neigh, NULL);
+ err = 0;
+- } else
+- err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,
+- NETLINK_CB(skb).portid, extack);
+-
++ }
+ neigh_release(neigh);
+-
+ out:
+ return err;
+ }
+diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c
+index d8b9dbabd4a43..88cc0ad7d386e 100644
+--- a/net/core/net-procfs.c
++++ b/net/core/net-procfs.c
+@@ -190,12 +190,23 @@ static const struct seq_operations softnet_seq_ops = {
+ .show = softnet_seq_show,
+ };
+
+-static void *ptype_get_idx(loff_t pos)
++static void *ptype_get_idx(struct seq_file *seq, loff_t pos)
+ {
++ struct list_head *ptype_list = NULL;
+ struct packet_type *pt = NULL;
++ struct net_device *dev;
+ loff_t i = 0;
+ int t;
+
++ for_each_netdev_rcu(seq_file_net(seq), dev) {
++ ptype_list = &dev->ptype_all;
++ list_for_each_entry_rcu(pt, ptype_list, list) {
++ if (i == pos)
++ return pt;
++ ++i;
++ }
++ }
++
+ list_for_each_entry_rcu(pt, &ptype_all, list) {
+ if (i == pos)
+ return pt;
+@@ -216,22 +227,40 @@ static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
+ __acquires(RCU)
+ {
+ rcu_read_lock();
+- return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
++ return *pos ? ptype_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
+ }
+
+ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+ {
++ struct net_device *dev;
+ struct packet_type *pt;
+ struct list_head *nxt;
+ int hash;
+
+ ++*pos;
+ if (v == SEQ_START_TOKEN)
+- return ptype_get_idx(0);
++ return ptype_get_idx(seq, 0);
+
+ pt = v;
+ nxt = pt->list.next;
++ if (pt->dev) {
++ if (nxt != &pt->dev->ptype_all)
++ goto found;
++
++ dev = pt->dev;
++ for_each_netdev_continue_rcu(seq_file_net(seq), dev) {
++ if (!list_empty(&dev->ptype_all)) {
++ nxt = dev->ptype_all.next;
++ goto found;
++ }
++ }
++
++ nxt = ptype_all.next;
++ goto ptype_all;
++ }
++
+ if (pt->type == htons(ETH_P_ALL)) {
++ptype_all:
+ if (nxt != &ptype_all)
+ goto found;
+ hash = 0;
+@@ -260,7 +289,8 @@ static int ptype_seq_show(struct seq_file *seq, void *v)
+
+ if (v == SEQ_START_TOKEN)
+ seq_puts(seq, "Type Device Function\n");
+- else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
++ else if ((!pt->af_packet_net || net_eq(pt->af_packet_net, seq_file_net(seq))) &&
++ (!pt->dev || net_eq(dev_net(pt->dev), seq_file_net(seq)))) {
+ if (pt->type == htons(ETH_P_ALL))
+ seq_puts(seq, "ALL ");
+ else
+diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
+index b2e49eb7001d6..9e5657f632453 100644
+--- a/net/core/net-sysfs.c
++++ b/net/core/net-sysfs.c
+@@ -175,6 +175,14 @@ static int change_carrier(struct net_device *dev, unsigned long new_carrier)
+ static ssize_t carrier_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t len)
+ {
++ struct net_device *netdev = to_net_dev(dev);
++
++ /* The check is also done in change_carrier; this helps returning early
++ * without hitting the trylock/restart in netdev_store.
++ */
++ if (!netdev->netdev_ops->ndo_change_carrier)
++ return -EOPNOTSUPP;
++
+ return netdev_store(dev, attr, buf, len, change_carrier);
+ }
+
+@@ -196,10 +204,16 @@ static ssize_t speed_show(struct device *dev,
+ struct net_device *netdev = to_net_dev(dev);
+ int ret = -EINVAL;
+
++ /* The check is also done in __ethtool_get_link_ksettings; this helps
++ * returning early without hitting the trylock/restart below.
++ */
++ if (!netdev->ethtool_ops->get_link_ksettings)
++ return ret;
++
+ if (!rtnl_trylock())
+ return restart_syscall();
+
+- if (netif_running(netdev)) {
++ if (netif_running(netdev) && netif_device_present(netdev)) {
+ struct ethtool_link_ksettings cmd;
+
+ if (!__ethtool_get_link_ksettings(netdev, &cmd))
+@@ -216,6 +230,12 @@ static ssize_t duplex_show(struct device *dev,
+ struct net_device *netdev = to_net_dev(dev);
+ int ret = -EINVAL;
+
++ /* The check is also done in __ethtool_get_link_ksettings; this helps
++ * returning early without hitting the trylock/restart below.
++ */
++ if (!netdev->ethtool_ops->get_link_ksettings)
++ return ret;
++
+ if (!rtnl_trylock())
+ return restart_syscall();
+
+@@ -468,6 +488,14 @@ static ssize_t proto_down_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+ {
++ struct net_device *netdev = to_net_dev(dev);
++
++ /* The check is also done in change_proto_down; this helps returning
++ * early without hitting the trylock/restart in netdev_store.
++ */
++ if (!netdev->netdev_ops->ndo_change_proto_down)
++ return -EOPNOTSUPP;
++
+ return netdev_store(dev, attr, buf, len, change_proto_down);
+ }
+ NETDEVICE_SHOW_RW(proto_down, fmt_dec);
+@@ -478,6 +506,12 @@ static ssize_t phys_port_id_show(struct device *dev,
+ struct net_device *netdev = to_net_dev(dev);
+ ssize_t ret = -EINVAL;
+
++ /* The check is also done in dev_get_phys_port_id; this helps returning
++ * early without hitting the trylock/restart below.
++ */
++ if (!netdev->netdev_ops->ndo_get_phys_port_id)
++ return -EOPNOTSUPP;
++
+ if (!rtnl_trylock())
+ return restart_syscall();
+
+@@ -500,6 +534,13 @@ static ssize_t phys_port_name_show(struct device *dev,
+ struct net_device *netdev = to_net_dev(dev);
+ ssize_t ret = -EINVAL;
+
++ /* The checks are also done in dev_get_phys_port_name; this helps
++ * returning early without hitting the trylock/restart below.
++ */
++ if (!netdev->netdev_ops->ndo_get_phys_port_name &&
++ !netdev->netdev_ops->ndo_get_devlink_port)
++ return -EOPNOTSUPP;
++
+ if (!rtnl_trylock())
+ return restart_syscall();
+
+@@ -522,6 +563,14 @@ static ssize_t phys_switch_id_show(struct device *dev,
+ struct net_device *netdev = to_net_dev(dev);
+ ssize_t ret = -EINVAL;
+
++ /* The checks are also done in dev_get_phys_port_name; this helps
++ * returning early without hitting the trylock/restart below. This works
++ * because recurse is false when calling dev_get_port_parent_id.
++ */
++ if (!netdev->netdev_ops->ndo_get_port_parent_id &&
++ !netdev->netdev_ops->ndo_get_devlink_port)
++ return -EOPNOTSUPP;
++
+ if (!rtnl_trylock())
+ return restart_syscall();
+
+@@ -1226,6 +1275,12 @@ static ssize_t tx_maxrate_store(struct netdev_queue *queue,
+ if (!capable(CAP_NET_ADMIN))
+ return -EPERM;
+
++ /* The check is also done later; this helps returning early without
++ * hitting the trylock/restart below.
++ */
++ if (!dev->netdev_ops->ndo_set_tx_maxrate)
++ return -EOPNOTSUPP;
++
+ err = kstrtou32(buf, 10, &rate);
+ if (err < 0)
+ return err;
+@@ -1765,6 +1820,9 @@ static void remove_queue_kobjects(struct net_device *dev)
+
+ net_rx_queue_update_kobjects(dev, real_rx, 0);
+ netdev_queue_update_kobjects(dev, real_tx, 0);
++
++ dev->real_num_rx_queues = 0;
++ dev->real_num_tx_queues = 0;
+ #ifdef CONFIG_SYSFS
+ kset_unregister(dev->queues_kset);
+ #endif
+@@ -1869,7 +1927,7 @@ static struct class net_class __ro_after_init = {
+ .get_ownership = net_get_ownership,
+ };
+
+-#ifdef CONFIG_OF_NET
++#ifdef CONFIG_OF
+ static int of_dev_node_match(struct device *dev, const void *data)
+ {
+ for (; dev; dev = dev->parent) {
+diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
+index a448a9b5bb2d6..9745cb6fdf516 100644
+--- a/net/core/net_namespace.c
++++ b/net/core/net_namespace.c
+@@ -44,13 +44,7 @@ EXPORT_SYMBOL_GPL(net_rwsem);
+ static struct key_tag init_net_key_domain = { .usage = REFCOUNT_INIT(1) };
+ #endif
+
+-struct net init_net = {
+- .ns.count = REFCOUNT_INIT(1),
+- .dev_base_head = LIST_HEAD_INIT(init_net.dev_base_head),
+-#ifdef CONFIG_KEYS
+- .key_domain = &init_net_key_domain,
+-#endif
+-};
++struct net init_net;
+ EXPORT_SYMBOL(init_net);
+
+ static bool init_net_initialized;
+@@ -164,8 +158,10 @@ static void ops_exit_list(const struct pernet_operations *ops,
+ {
+ struct net *net;
+ if (ops->exit) {
+- list_for_each_entry(net, net_exit_list, exit_list)
++ list_for_each_entry(net, net_exit_list, exit_list) {
+ ops->exit(net);
++ cond_resched();
++ }
+ }
+ if (ops->exit_batch)
+ ops->exit_batch(net_exit_list);
+@@ -473,7 +469,9 @@ struct net *copy_net_ns(unsigned long flags,
+
+ if (rv < 0) {
+ put_userns:
++#ifdef CONFIG_KEYS
+ key_remove_domain(net->key_domain);
++#endif
+ put_user_ns(user_ns);
+ net_free(net);
+ dec_ucounts:
+@@ -605,7 +603,9 @@ static void cleanup_net(struct work_struct *work)
+ list_for_each_entry_safe(net, tmp, &net_exit_list, exit_list) {
+ list_del_init(&net->exit_list);
+ dec_net_namespaces(net->ucounts);
++#ifdef CONFIG_KEYS
+ key_remove_domain(net->key_domain);
++#endif
+ put_user_ns(net->user_ns);
+ net_free(net);
+ }
+@@ -1075,7 +1075,7 @@ out:
+ rtnl_set_sk_err(net, RTNLGRP_NSID, err);
+ }
+
+-static int __init net_ns_init(void)
++void __init net_ns_init(void)
+ {
+ struct net_generic *ng;
+
+@@ -1096,6 +1096,9 @@ static int __init net_ns_init(void)
+
+ rcu_assign_pointer(init_net.gen, ng);
+
++#ifdef CONFIG_KEYS
++ init_net.key_domain = &init_net_key_domain;
++#endif
+ down_write(&pernet_ops_rwsem);
+ if (setup_net(&init_net, &init_user_ns))
+ panic("Could not setup the initial network namespace");
+@@ -1110,12 +1113,8 @@ static int __init net_ns_init(void)
+ RTNL_FLAG_DOIT_UNLOCKED);
+ rtnl_register(PF_UNSPEC, RTM_GETNSID, rtnl_net_getid, rtnl_net_dumpid,
+ RTNL_FLAG_DOIT_UNLOCKED);
+-
+- return 0;
+ }
+
+-pure_initcall(net_ns_init);
+-
+ static void free_exit_list(struct pernet_operations *ops, struct list_head *net_exit_list)
+ {
+ ops_pre_exit_list(ops, net_exit_list);
+diff --git a/net/core/of_net.c b/net/core/of_net.c
+new file mode 100644
+index 0000000000000..dbac3a172a11e
+--- /dev/null
++++ b/net/core/of_net.c
+@@ -0,0 +1,145 @@
++// SPDX-License-Identifier: GPL-2.0-only
++/*
++ * OF helpers for network devices.
++ *
++ * Initially copied out of arch/powerpc/kernel/prom_parse.c
++ */
++#include <linux/etherdevice.h>
++#include <linux/kernel.h>
++#include <linux/of_net.h>
++#include <linux/of_platform.h>
++#include <linux/phy.h>
++#include <linux/export.h>
++#include <linux/device.h>
++#include <linux/nvmem-consumer.h>
++
++/**
++ * of_get_phy_mode - Get phy mode for given device_node
++ * @np: Pointer to the given device_node
++ * @interface: Pointer to the result
++ *
++ * The function gets phy interface string from property 'phy-mode' or
++ * 'phy-connection-type'. The index in phy_modes table is set in
++ * interface and 0 returned. In case of error interface is set to
++ * PHY_INTERFACE_MODE_NA and an errno is returned, e.g. -ENODEV.
++ */
++int of_get_phy_mode(struct device_node *np, phy_interface_t *interface)
++{
++ const char *pm;
++ int err, i;
++
++ *interface = PHY_INTERFACE_MODE_NA;
++
++ err = of_property_read_string(np, "phy-mode", &pm);
++ if (err < 0)
++ err = of_property_read_string(np, "phy-connection-type", &pm);
++ if (err < 0)
++ return err;
++
++ for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++)
++ if (!strcasecmp(pm, phy_modes(i))) {
++ *interface = i;
++ return 0;
++ }
++
++ return -ENODEV;
++}
++EXPORT_SYMBOL_GPL(of_get_phy_mode);
++
++static int of_get_mac_addr(struct device_node *np, const char *name, u8 *addr)
++{
++ struct property *pp = of_find_property(np, name, NULL);
++
++ if (pp && pp->length == ETH_ALEN && is_valid_ether_addr(pp->value)) {
++ memcpy(addr, pp->value, ETH_ALEN);
++ return 0;
++ }
++ return -ENODEV;
++}
++
++static int of_get_mac_addr_nvmem(struct device_node *np, u8 *addr)
++{
++ struct platform_device *pdev = of_find_device_by_node(np);
++ struct nvmem_cell *cell;
++ const void *mac;
++ size_t len;
++ int ret;
++
++ /* Try lookup by device first, there might be a nvmem_cell_lookup
++ * associated with a given device.
++ */
++ if (pdev) {
++ ret = nvmem_get_mac_address(&pdev->dev, addr);
++ put_device(&pdev->dev);
++ return ret;
++ }
++
++ cell = of_nvmem_cell_get(np, "mac-address");
++ if (IS_ERR(cell))
++ return PTR_ERR(cell);
++
++ mac = nvmem_cell_read(cell, &len);
++ nvmem_cell_put(cell);
++
++ if (IS_ERR(mac))
++ return PTR_ERR(mac);
++
++ if (len != ETH_ALEN || !is_valid_ether_addr(mac)) {
++ kfree(mac);
++ return -EINVAL;
++ }
++
++ memcpy(addr, mac, ETH_ALEN);
++ kfree(mac);
++
++ return 0;
++}
++
++/**
++ * of_get_mac_address()
++ * @np: Caller's Device Node
++ * @addr: Pointer to a six-byte array for the result
++ *
++ * Search the device tree for the best MAC address to use. 'mac-address' is
++ * checked first, because that is supposed to contain to "most recent" MAC
++ * address. If that isn't set, then 'local-mac-address' is checked next,
++ * because that is the default address. If that isn't set, then the obsolete
++ * 'address' is checked, just in case we're using an old device tree. If any
++ * of the above isn't set, then try to get MAC address from nvmem cell named
++ * 'mac-address'.
++ *
++ * Note that the 'address' property is supposed to contain a virtual address of
++ * the register set, but some DTS files have redefined that property to be the
++ * MAC address.
++ *
++ * All-zero MAC addresses are rejected, because those could be properties that
++ * exist in the device tree, but were not set by U-Boot. For example, the
++ * DTS could define 'mac-address' and 'local-mac-address', with zero MAC
++ * addresses. Some older U-Boots only initialized 'local-mac-address'. In
++ * this case, the real MAC is in 'local-mac-address', and 'mac-address' exists
++ * but is all zeros.
++ *
++ * Return: 0 on success and errno in case of error.
++*/
++int of_get_mac_address(struct device_node *np, u8 *addr)
++{
++ int ret;
++
++ if (!np)
++ return -ENODEV;
++
++ ret = of_get_mac_addr(np, "mac-address", addr);
++ if (!ret)
++ return 0;
++
++ ret = of_get_mac_addr(np, "local-mac-address", addr);
++ if (!ret)
++ return 0;
++
++ ret = of_get_mac_addr(np, "address", addr);
++ if (!ret)
++ return 0;
++
++ return of_get_mac_addr_nvmem(np, addr);
++}
++EXPORT_SYMBOL(of_get_mac_address);
+diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
+index 8ccce85562a1d..9c0e8ccf9bc58 100644
+--- a/net/core/rtnetlink.c
++++ b/net/core/rtnetlink.c
+@@ -1698,6 +1698,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
+ {
+ struct ifinfomsg *ifm;
+ struct nlmsghdr *nlh;
++ struct Qdisc *qdisc;
+
+ ASSERT_RTNL();
+ nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifm), flags);
+@@ -1715,6 +1716,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
+ if (tgt_netnsid >= 0 && nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid))
+ goto nla_put_failure;
+
++ qdisc = rtnl_dereference(dev->qdisc);
+ if (nla_put_string(skb, IFLA_IFNAME, dev->name) ||
+ nla_put_u32(skb, IFLA_TXQLEN, dev->tx_queue_len) ||
+ nla_put_u8(skb, IFLA_OPERSTATE,
+@@ -1733,8 +1735,8 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
+ #endif
+ put_master_ifindex(skb, dev) ||
+ nla_put_u8(skb, IFLA_CARRIER, netif_carrier_ok(dev)) ||
+- (dev->qdisc &&
+- nla_put_string(skb, IFLA_QDISC, dev->qdisc->ops->id)) ||
++ (qdisc &&
++ nla_put_string(skb, IFLA_QDISC, qdisc->ops->id)) ||
+ nla_put_ifalias(skb, dev) ||
+ nla_put_u32(skb, IFLA_CARRIER_CHANGES,
+ atomic_read(&dev->carrier_up_count) +
+@@ -3254,8 +3256,8 @@ static int __rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr *slave_attr[RTNL_SLAVE_MAX_TYPE + 1];
+ unsigned char name_assign_type = NET_NAME_USER;
+ struct nlattr *linkinfo[IFLA_INFO_MAX + 1];
+- const struct rtnl_link_ops *m_ops = NULL;
+- struct net_device *master_dev = NULL;
++ const struct rtnl_link_ops *m_ops;
++ struct net_device *master_dev;
+ struct net *net = sock_net(skb->sk);
+ const struct rtnl_link_ops *ops;
+ struct nlattr *tb[IFLA_MAX + 1];
+@@ -3293,6 +3295,8 @@ replay:
+ else
+ dev = NULL;
+
++ master_dev = NULL;
++ m_ops = NULL;
+ if (dev) {
+ master_dev = netdev_master_upper_dev_get(dev);
+ if (master_dev)
+@@ -3627,13 +3631,24 @@ static int rtnl_alt_ifname(int cmd, struct net_device *dev, struct nlattr *attr,
+ bool *changed, struct netlink_ext_ack *extack)
+ {
+ char *alt_ifname;
++ size_t size;
+ int err;
+
+ err = nla_validate(attr, attr->nla_len, IFLA_MAX, ifla_policy, extack);
+ if (err)
+ return err;
+
+- alt_ifname = nla_strdup(attr, GFP_KERNEL);
++ if (cmd == RTM_NEWLINKPROP) {
++ size = rtnl_prop_list_size(dev);
++ size += nla_total_size(ALTIFNAMSIZ);
++ if (size >= U16_MAX) {
++ NL_SET_ERR_MSG(extack,
++ "effective property list too long");
++ return -EINVAL;
++ }
++ }
++
++ alt_ifname = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
+ if (!alt_ifname)
+ return -ENOMEM;
+
+diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c
+index b5bc680d47553..7131cd1fb2ad5 100644
+--- a/net/core/secure_seq.c
++++ b/net/core/secure_seq.c
+@@ -22,6 +22,8 @@
+ static siphash_key_t net_secret __read_mostly;
+ static siphash_key_t ts_secret __read_mostly;
+
++#define EPHEMERAL_PORT_SHUFFLE_PERIOD (10 * HZ)
++
+ static __always_inline void net_secret_init(void)
+ {
+ net_get_random_once(&net_secret, sizeof(net_secret));
+@@ -94,17 +96,19 @@ u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr,
+ }
+ EXPORT_SYMBOL(secure_tcpv6_seq);
+
+-u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
++u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr,
+ __be16 dport)
+ {
+ const struct {
+ struct in6_addr saddr;
+ struct in6_addr daddr;
++ unsigned int timeseed;
+ __be16 dport;
+ } __aligned(SIPHASH_ALIGNMENT) combined = {
+ .saddr = *(struct in6_addr *)saddr,
+ .daddr = *(struct in6_addr *)daddr,
+- .dport = dport
++ .timeseed = jiffies / EPHEMERAL_PORT_SHUFFLE_PERIOD,
++ .dport = dport,
+ };
+ net_secret_init();
+ return siphash(&combined, offsetofend(typeof(combined), dport),
+@@ -142,11 +146,13 @@ u32 secure_tcp_seq(__be32 saddr, __be32 daddr,
+ }
+ EXPORT_SYMBOL_GPL(secure_tcp_seq);
+
+-u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport)
++u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport)
+ {
+ net_secret_init();
+- return siphash_3u32((__force u32)saddr, (__force u32)daddr,
+- (__force u16)dport, &net_secret);
++ return siphash_4u32((__force u32)saddr, (__force u32)daddr,
++ (__force u16)dport,
++ jiffies / EPHEMERAL_PORT_SHUFFLE_PERIOD,
++ &net_secret);
+ }
+ EXPORT_SYMBOL_GPL(secure_ipv4_port_ephemeral);
+ #endif
+diff --git a/net/core/skbuff.c b/net/core/skbuff.c
+index fe9358437380c..7ef0f5a8ab036 100644
+--- a/net/core/skbuff.c
++++ b/net/core/skbuff.c
+@@ -203,7 +203,7 @@ static void __build_skb_around(struct sk_buff *skb, void *data,
+ skb->head = data;
+ skb->data = data;
+ skb_reset_tail_pointer(skb);
+- skb->end = skb->tail + size;
++ skb_set_end_offset(skb, size);
+ skb->mac_header = (typeof(skb->mac_header))~0U;
+ skb->transport_header = (typeof(skb->transport_header))~0U;
+
+@@ -832,7 +832,7 @@ void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
+ ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
+
+ if (dev)
+- printk("%sdev name=%s feat=0x%pNF\n",
++ printk("%sdev name=%s feat=%pNF\n",
+ level, dev->name, &dev->features);
+ if (sk)
+ printk("%ssk family=%hu type=%u proto=%u\n",
+@@ -1738,11 +1738,10 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
+ skb->head = data;
+ skb->head_frag = 0;
+ skb->data += off;
++
++ skb_set_end_offset(skb, size);
+ #ifdef NET_SKBUFF_DATA_USES_OFFSET
+- skb->end = size;
+ off = nhead;
+-#else
+- skb->end = skb->head + size;
+ #endif
+ skb->tail += off;
+ skb_headers_offset_update(skb, nhead);
+@@ -1790,6 +1789,38 @@ struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
+ }
+ EXPORT_SYMBOL(skb_realloc_headroom);
+
++int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
++{
++ unsigned int saved_end_offset, saved_truesize;
++ struct skb_shared_info *shinfo;
++ int res;
++
++ saved_end_offset = skb_end_offset(skb);
++ saved_truesize = skb->truesize;
++
++ res = pskb_expand_head(skb, 0, 0, pri);
++ if (res)
++ return res;
++
++ skb->truesize = saved_truesize;
++
++ if (likely(skb_end_offset(skb) == saved_end_offset))
++ return 0;
++
++ shinfo = skb_shinfo(skb);
++
++ /* We are about to change back skb->end,
++ * we need to move skb_shinfo() to its new location.
++ */
++ memmove(skb->head + saved_end_offset,
++ shinfo,
++ offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
++
++ skb_set_end_offset(skb, saved_end_offset);
++
++ return 0;
++}
++
+ /**
+ * skb_expand_head - reallocate header of &sk_buff
+ * @skb: buffer to reallocate
+@@ -2254,7 +2285,7 @@ void *__pskb_pull_tail(struct sk_buff *skb, int delta)
+ /* Free pulled out fragments. */
+ while ((list = skb_shinfo(skb)->frag_list) != insp) {
+ skb_shinfo(skb)->frag_list = list->next;
+- kfree_skb(list);
++ consume_skb(list);
+ }
+ /* And insert new clone at head. */
+ if (clone) {
+@@ -3449,19 +3480,7 @@ EXPORT_SYMBOL(skb_split);
+ */
+ static int skb_prepare_for_shift(struct sk_buff *skb)
+ {
+- int ret = 0;
+-
+- if (skb_cloned(skb)) {
+- /* Save and restore truesize: pskb_expand_head() may reallocate
+- * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
+- * cannot change truesize at this point.
+- */
+- unsigned int save_truesize = skb->truesize;
+-
+- ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
+- skb->truesize = save_truesize;
+- }
+- return ret;
++ return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
+ }
+
+ /**
+@@ -3854,7 +3873,7 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb,
+ unsigned int delta_len = 0;
+ struct sk_buff *tail = NULL;
+ struct sk_buff *nskb, *tmp;
+- int err;
++ int len_diff, err;
+
+ skb_push(skb, -skb_network_offset(skb) + offset);
+
+@@ -3865,6 +3884,7 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb,
+ list_skb = list_skb->next;
+
+ err = 0;
++ delta_truesize += nskb->truesize;
+ if (skb_shared(nskb)) {
+ tmp = skb_clone(nskb, GFP_ATOMIC);
+ if (tmp) {
+@@ -3889,14 +3909,15 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb,
+ tail = nskb;
+
+ delta_len += nskb->len;
+- delta_truesize += nskb->truesize;
+
+ skb_push(nskb, -skb_network_offset(nskb) + offset);
+
+ skb_release_head_state(nskb);
++ len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
+ __copy_skb_header(nskb, skb);
+
+ skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
++ nskb->transport_header += len_diff;
+ skb_copy_from_linear_data_offset(skb, -tnl_hlen,
+ nskb->data - tnl_hlen,
+ offset + tnl_hlen);
+@@ -4856,7 +4877,7 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb,
+ serr->ee.ee_data = skb_shinfo(skb)->tskey;
+ if (sk->sk_protocol == IPPROTO_TCP &&
+ sk->sk_type == SOCK_STREAM)
+- serr->ee.ee_data -= sk->sk_tskey;
++ serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
+ }
+
+ err = sock_queue_err_skb(sk, skb);
+@@ -5371,11 +5392,18 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
+ if (skb_cloned(to))
+ return false;
+
+- /* The page pool signature of struct page will eventually figure out
+- * which pages can be recycled or not but for now let's prohibit slab
+- * allocated and page_pool allocated SKBs from being coalesced.
++ /* In general, avoid mixing slab allocated and page_pool allocated
++ * pages within the same SKB. However when @to is not pp_recycle and
++ * @from is cloned, we can transition frag pages from page_pool to
++ * reference counted.
++ *
++ * On the other hand, don't allow coalescing two pp_recycle SKBs if
++ * @from is cloned, in case the SKB is using page_pool fragment
++ * references (PP_FLAG_PAGE_FRAG). Since we only take full page
++ * references for cloned SKBs at the moment that would result in
++ * inconsistent reference counts.
+ */
+- if (to->pp_recycle != from->pp_recycle)
++ if (to->pp_recycle != (from->pp_recycle && !skb_cloned(from)))
+ return false;
+
+ if (len <= skb_tailroom(to)) {
+@@ -6171,11 +6199,7 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
+ skb->head = data;
+ skb->data = data;
+ skb->head_frag = 0;
+-#ifdef NET_SKBUFF_DATA_USES_OFFSET
+- skb->end = size;
+-#else
+- skb->end = skb->head + size;
+-#endif
++ skb_set_end_offset(skb, size);
+ skb_set_tail_pointer(skb, skb_headlen(skb));
+ skb_headers_offset_update(skb, 0);
+ skb->cloned = 0;
+@@ -6232,7 +6256,7 @@ static int pskb_carve_frag_list(struct sk_buff *skb,
+ /* Free pulled out fragments. */
+ while ((list = shinfo->frag_list) != insp) {
+ shinfo->frag_list = list->next;
+- kfree_skb(list);
++ consume_skb(list);
+ }
+ /* And insert new clone at head. */
+ if (clone) {
+@@ -6313,11 +6337,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
+ skb->head = data;
+ skb->head_frag = 0;
+ skb->data = data;
+-#ifdef NET_SKBUFF_DATA_USES_OFFSET
+- skb->end = size;
+-#else
+- skb->end = skb->head + size;
+-#endif
++ skb_set_end_offset(skb, size);
+ skb_reset_tail_pointer(skb);
+ skb_headers_offset_update(skb, 0);
+ skb->cloned = 0;
+diff --git a/net/core/skmsg.c b/net/core/skmsg.c
+index a86ef7e844f8c..cc381165ea080 100644
+--- a/net/core/skmsg.c
++++ b/net/core/skmsg.c
+@@ -27,6 +27,7 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
+ int elem_first_coalesce)
+ {
+ struct page_frag *pfrag = sk_page_frag(sk);
++ u32 osize = msg->sg.size;
+ int ret = 0;
+
+ len -= msg->sg.size;
+@@ -35,13 +36,17 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
+ u32 orig_offset;
+ int use, i;
+
+- if (!sk_page_frag_refill(sk, pfrag))
+- return -ENOMEM;
++ if (!sk_page_frag_refill(sk, pfrag)) {
++ ret = -ENOMEM;
++ goto msg_trim;
++ }
+
+ orig_offset = pfrag->offset;
+ use = min_t(int, len, pfrag->size - orig_offset);
+- if (!sk_wmem_schedule(sk, use))
+- return -ENOMEM;
++ if (!sk_wmem_schedule(sk, use)) {
++ ret = -ENOMEM;
++ goto msg_trim;
++ }
+
+ i = msg->sg.end;
+ sk_msg_iter_var_prev(i);
+@@ -71,6 +76,10 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
+ }
+
+ return ret;
++
++msg_trim:
++ sk_msg_trim(sk, msg, osize);
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(sk_msg_alloc);
+
+@@ -508,6 +517,7 @@ static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
+ }
+
+ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
++ u32 off, u32 len,
+ struct sk_psock *psock,
+ struct sock *sk,
+ struct sk_msg *msg)
+@@ -521,11 +531,11 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
+ */
+ if (skb_linearize(skb))
+ return -EAGAIN;
+- num_sge = skb_to_sgvec(skb, msg->sg.data, 0, skb->len);
++ num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
+ if (unlikely(num_sge < 0))
+ return num_sge;
+
+- copied = skb->len;
++ copied = len;
+ msg->sg.start = 0;
+ msg->sg.size = copied;
+ msg->sg.end = num_sge;
+@@ -536,9 +546,11 @@ static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
+ return copied;
+ }
+
+-static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
++static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
++ u32 off, u32 len);
+
+-static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
++static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
++ u32 off, u32 len)
+ {
+ struct sock *sk = psock->sk;
+ struct sk_msg *msg;
+@@ -549,7 +561,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
+ * correctly.
+ */
+ if (unlikely(skb->sk == sk))
+- return sk_psock_skb_ingress_self(psock, skb);
++ return sk_psock_skb_ingress_self(psock, skb, off, len);
+ msg = sk_psock_create_ingress_msg(sk, skb);
+ if (!msg)
+ return -EAGAIN;
+@@ -561,7 +573,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
+ * into user buffers.
+ */
+ skb_set_owner_r(skb, sk);
+- err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);
++ err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
+ if (err < 0)
+ kfree(msg);
+ return err;
+@@ -571,7 +583,8 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
+ * skb. In this case we do not need to check memory limits or skb_set_owner_r
+ * because the skb is already accounted for here.
+ */
+-static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb)
++static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
++ u32 off, u32 len)
+ {
+ struct sk_msg *msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_ATOMIC);
+ struct sock *sk = psock->sk;
+@@ -581,7 +594,7 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb
+ return -EAGAIN;
+ sk_msg_init(msg);
+ skb_set_owner_r(skb, sk);
+- err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);
++ err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
+ if (err < 0)
+ kfree(msg);
+ return err;
+@@ -595,7 +608,7 @@ static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
+ return -EAGAIN;
+ return skb_send_sock(psock->sk, skb, off, len);
+ }
+- return sk_psock_skb_ingress(psock, skb);
++ return sk_psock_skb_ingress(psock, skb, off, len);
+ }
+
+ static void sk_psock_skb_state(struct sk_psock *psock,
+@@ -638,6 +651,12 @@ static void sk_psock_backlog(struct work_struct *work)
+ while ((skb = skb_dequeue(&psock->ingress_skb))) {
+ len = skb->len;
+ off = 0;
++ if (skb_bpf_strparser(skb)) {
++ struct strp_msg *stm = strp_msg(skb);
++
++ off = stm->offset;
++ len = stm->full_len;
++ }
+ start:
+ ingress = skb_bpf_ingress(skb);
+ skb_bpf_redirect_clear(skb);
+@@ -877,6 +896,7 @@ static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb)
+ * return code, but then didn't set a redirect interface.
+ */
+ if (unlikely(!sk_other)) {
++ skb_bpf_redirect_clear(skb);
+ sock_drop(from->sk, skb);
+ return -EIO;
+ }
+@@ -944,6 +964,7 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
+ {
+ struct sock *sk_other;
+ int err = 0;
++ u32 len, off;
+
+ switch (verdict) {
+ case __SK_PASS:
+@@ -951,6 +972,7 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
+ sk_other = psock->sk;
+ if (sock_flag(sk_other, SOCK_DEAD) ||
+ !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
++ skb_bpf_redirect_clear(skb);
+ goto out_free;
+ }
+
+@@ -963,7 +985,15 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
+ * retrying later from workqueue.
+ */
+ if (skb_queue_empty(&psock->ingress_skb)) {
+- err = sk_psock_skb_ingress_self(psock, skb);
++ len = skb->len;
++ off = 0;
++ if (skb_bpf_strparser(skb)) {
++ struct strp_msg *stm = strp_msg(skb);
++
++ off = stm->offset;
++ len = stm->full_len;
++ }
++ err = sk_psock_skb_ingress_self(psock, skb, off, len);
+ }
+ if (err < 0) {
+ spin_lock_bh(&psock->ingress_lock);
+@@ -1029,6 +1059,8 @@ static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
+ skb_dst_drop(skb);
+ skb_bpf_redirect_clear(skb);
+ ret = bpf_prog_run_pin_on_cpu(prog, skb);
++ if (ret == SK_PASS)
++ skb_bpf_set_strparser(skb);
+ ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
+ skb->sk = NULL;
+ }
+@@ -1101,6 +1133,8 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
+
+ void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
+ {
++ psock_set_prog(&psock->progs.stream_parser, NULL);
++
+ if (!psock->saved_data_ready)
+ return;
+
+@@ -1128,7 +1162,7 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
+ struct sk_psock *psock;
+ struct bpf_prog *prog;
+ int ret = __SK_DROP;
+- int len = skb->len;
++ int len = orig_len;
+
+ /* clone here so sk_eat_skb() in tcp_read_sock does not drop our data */
+ skb = skb_clone(skb, GFP_ATOMIC);
+@@ -1189,6 +1223,9 @@ void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
+
+ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
+ {
++ psock_set_prog(&psock->progs.stream_verdict, NULL);
++ psock_set_prog(&psock->progs.skb_verdict, NULL);
++
+ if (!psock->saved_data_ready)
+ return;
+
+diff --git a/net/core/sock.c b/net/core/sock.c
+index c1601f75ec4b3..deaed1b206823 100644
+--- a/net/core/sock.c
++++ b/net/core/sock.c
+@@ -830,6 +830,8 @@ static int sock_timestamping_bind_phc(struct sock *sk, int phc_index)
+ }
+
+ num = ethtool_get_phc_vclocks(dev, &vclock_index);
++ dev_put(dev);
++
+ for (i = 0; i < num; i++) {
+ if (*(vclock_index + i) == phc_index) {
+ match = true;
+@@ -864,9 +866,9 @@ int sock_set_timestamping(struct sock *sk, int optname,
+ if ((1 << sk->sk_state) &
+ (TCPF_CLOSE | TCPF_LISTEN))
+ return -EINVAL;
+- sk->sk_tskey = tcp_sk(sk)->snd_una;
++ atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una);
+ } else {
+- sk->sk_tskey = 0;
++ atomic_set(&sk->sk_tskey, 0);
+ }
+ }
+
+@@ -2043,8 +2045,10 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
+ newsk->sk_prot_creator = prot;
+
+ /* SANITY */
+- if (likely(newsk->sk_net_refcnt))
++ if (likely(newsk->sk_net_refcnt)) {
+ get_net(sock_net(newsk));
++ sock_inuse_add(sock_net(newsk), 1);
++ }
+ sk_node_init(&newsk->sk_node);
+ sock_lock_init(newsk);
+ bh_lock_sock(newsk);
+@@ -2115,8 +2119,6 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
+ newsk->sk_err_soft = 0;
+ newsk->sk_priority = 0;
+ newsk->sk_incoming_cpu = raw_smp_processor_id();
+- if (likely(newsk->sk_net_refcnt))
+- sock_inuse_add(sock_net(newsk), 1);
+
+ /* Before updating sk_refcnt, we must commit prior changes to memory
+ * (Documentation/RCU/rculist_nulls.rst for details)
+diff --git a/net/core/sock_map.c b/net/core/sock_map.c
+index e252b8ec2b85e..6351b6af7aca9 100644
+--- a/net/core/sock_map.c
++++ b/net/core/sock_map.c
+@@ -167,8 +167,11 @@ static void sock_map_del_link(struct sock *sk,
+ write_lock_bh(&sk->sk_callback_lock);
+ if (strp_stop)
+ sk_psock_stop_strp(sk, psock);
+- else
++ if (verdict_stop)
+ sk_psock_stop_verdict(sk, psock);
++
++ if (psock->psock_update_sk_prot)
++ psock->psock_update_sk_prot(sk, psock, false);
+ write_unlock_bh(&sk->sk_callback_lock);
+ }
+ }
+@@ -282,32 +285,38 @@ static int sock_map_link(struct bpf_map *map, struct sock *sk)
+
+ if (msg_parser)
+ psock_set_prog(&psock->progs.msg_parser, msg_parser);
++ if (stream_parser)
++ psock_set_prog(&psock->progs.stream_parser, stream_parser);
++ if (stream_verdict)
++ psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
++ if (skb_verdict)
++ psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
+
++ /* msg_* and stream_* programs references tracked in psock after this
++ * point. Reference dec and cleanup will occur through psock destructor
++ */
+ ret = sock_map_init_proto(sk, psock);
+- if (ret < 0)
+- goto out_drop;
++ if (ret < 0) {
++ sk_psock_put(sk, psock);
++ goto out;
++ }
+
+ write_lock_bh(&sk->sk_callback_lock);
+ if (stream_parser && stream_verdict && !psock->saved_data_ready) {
+ ret = sk_psock_init_strp(sk, psock);
+- if (ret)
+- goto out_unlock_drop;
+- psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
+- psock_set_prog(&psock->progs.stream_parser, stream_parser);
++ if (ret) {
++ write_unlock_bh(&sk->sk_callback_lock);
++ sk_psock_put(sk, psock);
++ goto out;
++ }
+ sk_psock_start_strp(sk, psock);
+ } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
+- psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
+ sk_psock_start_verdict(sk,psock);
+ } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
+- psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
+ sk_psock_start_verdict(sk, psock);
+ }
+ write_unlock_bh(&sk->sk_callback_lock);
+ return 0;
+-out_unlock_drop:
+- write_unlock_bh(&sk->sk_callback_lock);
+-out_drop:
+- sk_psock_put(sk, psock);
+ out_progs:
+ if (skb_verdict)
+ bpf_prog_put(skb_verdict);
+@@ -320,6 +329,7 @@ out_put_stream_parser:
+ out_put_stream_verdict:
+ if (stream_verdict)
+ bpf_prog_put(stream_verdict);
++out:
+ return ret;
+ }
+
+@@ -1565,7 +1575,7 @@ static struct bpf_iter_reg sock_map_iter_reg = {
+ .ctx_arg_info_size = 2,
+ .ctx_arg_info = {
+ { offsetof(struct bpf_iter__sockmap, key),
+- PTR_TO_RDONLY_BUF_OR_NULL },
++ PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY },
+ { offsetof(struct bpf_iter__sockmap, sk),
+ PTR_TO_BTF_ID_OR_NULL },
+ },
+diff --git a/net/core/stream.c b/net/core/stream.c
+index 4f1d4aa5fb38d..a166a32b411fa 100644
+--- a/net/core/stream.c
++++ b/net/core/stream.c
+@@ -195,9 +195,6 @@ void sk_stream_kill_queues(struct sock *sk)
+ /* First the read buffer. */
+ __skb_queue_purge(&sk->sk_receive_queue);
+
+- /* Next, the error queue. */
+- __skb_queue_purge(&sk->sk_error_queue);
+-
+ /* Next, the write queue. */
+ WARN_ON(!skb_queue_empty(&sk->sk_write_queue));
+
+diff --git a/net/dcb/dcbnl.c b/net/dcb/dcbnl.c
+index b441ab330fd34..dc4fb699b56c3 100644
+--- a/net/dcb/dcbnl.c
++++ b/net/dcb/dcbnl.c
+@@ -2073,8 +2073,52 @@ u8 dcb_ieee_getapp_default_prio_mask(const struct net_device *dev)
+ }
+ EXPORT_SYMBOL(dcb_ieee_getapp_default_prio_mask);
+
++static void dcbnl_flush_dev(struct net_device *dev)
++{
++ struct dcb_app_type *itr, *tmp;
++
++ spin_lock_bh(&dcb_lock);
++
++ list_for_each_entry_safe(itr, tmp, &dcb_app_list, list) {
++ if (itr->ifindex == dev->ifindex) {
++ list_del(&itr->list);
++ kfree(itr);
++ }
++ }
++
++ spin_unlock_bh(&dcb_lock);
++}
++
++static int dcbnl_netdevice_event(struct notifier_block *nb,
++ unsigned long event, void *ptr)
++{
++ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
++
++ switch (event) {
++ case NETDEV_UNREGISTER:
++ if (!dev->dcbnl_ops)
++ return NOTIFY_DONE;
++
++ dcbnl_flush_dev(dev);
++
++ return NOTIFY_OK;
++ default:
++ return NOTIFY_DONE;
++ }
++}
++
++static struct notifier_block dcbnl_nb __read_mostly = {
++ .notifier_call = dcbnl_netdevice_event,
++};
++
+ static int __init dcbnl_init(void)
+ {
++ int err;
++
++ err = register_netdevice_notifier(&dcbnl_nb);
++ if (err)
++ return err;
++
+ rtnl_register(PF_UNSPEC, RTM_GETDCB, dcb_doit, NULL, 0);
+ rtnl_register(PF_UNSPEC, RTM_SETDCB, dcb_doit, NULL, 0);
+
+diff --git a/net/dccp/dccp.h b/net/dccp/dccp.h
+index c5c1d2b8045e8..5183e627468d8 100644
+--- a/net/dccp/dccp.h
++++ b/net/dccp/dccp.h
+@@ -48,7 +48,7 @@ extern bool dccp_debug;
+
+ extern struct inet_hashinfo dccp_hashinfo;
+
+-extern struct percpu_counter dccp_orphan_count;
++DECLARE_PER_CPU(unsigned int, dccp_orphan_count);
+
+ void dccp_time_wait(struct sock *sk, int state, int timeo);
+
+diff --git a/net/dccp/proto.c b/net/dccp/proto.c
+index abb5c596a8176..fc44dadc778bb 100644
+--- a/net/dccp/proto.c
++++ b/net/dccp/proto.c
+@@ -42,8 +42,8 @@ DEFINE_SNMP_STAT(struct dccp_mib, dccp_statistics) __read_mostly;
+
+ EXPORT_SYMBOL_GPL(dccp_statistics);
+
+-struct percpu_counter dccp_orphan_count;
+-EXPORT_SYMBOL_GPL(dccp_orphan_count);
++DEFINE_PER_CPU(unsigned int, dccp_orphan_count);
++EXPORT_PER_CPU_SYMBOL_GPL(dccp_orphan_count);
+
+ struct inet_hashinfo dccp_hashinfo;
+ EXPORT_SYMBOL_GPL(dccp_hashinfo);
+@@ -1055,7 +1055,7 @@ adjudge_to_death:
+ bh_lock_sock(sk);
+ WARN_ON(sock_owned_by_user(sk));
+
+- percpu_counter_inc(sk->sk_prot->orphan_count);
++ this_cpu_inc(dccp_orphan_count);
+
+ /* Have we already been destroyed by a softirq or backlog? */
+ if (state != DCCP_CLOSED && sk->sk_state == DCCP_CLOSED)
+@@ -1115,13 +1115,10 @@ static int __init dccp_init(void)
+
+ BUILD_BUG_ON(sizeof(struct dccp_skb_cb) >
+ sizeof_field(struct sk_buff, cb));
+- rc = percpu_counter_init(&dccp_orphan_count, 0, GFP_KERNEL);
+- if (rc)
+- goto out_fail;
+ inet_hashinfo_init(&dccp_hashinfo);
+ rc = inet_hashinfo2_init_mod(&dccp_hashinfo);
+ if (rc)
+- goto out_free_percpu;
++ goto out_fail;
+ rc = -ENOBUFS;
+ dccp_hashinfo.bind_bucket_cachep =
+ kmem_cache_create("dccp_bind_bucket",
+@@ -1226,8 +1223,6 @@ out_free_bind_bucket_cachep:
+ kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);
+ out_free_hashinfo2:
+ inet_hashinfo2_free_mod(&dccp_hashinfo);
+-out_free_percpu:
+- percpu_counter_destroy(&dccp_orphan_count);
+ out_fail:
+ dccp_hashinfo.bhash = NULL;
+ dccp_hashinfo.ehash = NULL;
+@@ -1250,7 +1245,6 @@ static void __exit dccp_fini(void)
+ dccp_ackvec_exit();
+ dccp_sysctl_exit();
+ inet_hashinfo2_free_mod(&dccp_hashinfo);
+- percpu_counter_destroy(&dccp_orphan_count);
+ }
+
+ module_init(dccp_init);
+diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c
+index e9911b18bdbfa..64a56db3de586 100644
+--- a/net/dsa/dsa2.c
++++ b/net/dsa/dsa2.c
+@@ -1341,6 +1341,7 @@ static int dsa_port_parse_of(struct dsa_port *dp, struct device_node *dn)
+ const char *user_protocol;
+
+ master = of_find_net_device_by_node(ethernet);
++ of_node_put(ethernet);
+ if (!master)
+ return -EPROBE_DEFER;
+
+@@ -1630,6 +1631,10 @@ void dsa_switch_shutdown(struct dsa_switch *ds)
+ struct dsa_port *dp;
+
+ mutex_lock(&dsa2_mutex);
++
++ if (!ds->setup)
++ goto out;
++
+ rtnl_lock();
+
+ list_for_each_entry(dp, &ds->dst->ports, list) {
+@@ -1664,6 +1669,7 @@ void dsa_switch_shutdown(struct dsa_switch *ds)
+ unregister_netdevice_many(&unregister_list);
+
+ rtnl_unlock();
++out:
+ mutex_unlock(&dsa2_mutex);
+ }
+ EXPORT_SYMBOL_GPL(dsa_switch_shutdown);
+diff --git a/net/dsa/port.c b/net/dsa/port.c
+index 616330a16d319..63e88de963936 100644
+--- a/net/dsa/port.c
++++ b/net/dsa/port.c
+@@ -1201,8 +1201,10 @@ int dsa_port_link_register_of(struct dsa_port *dp)
+ if (ds->ops->phylink_mac_link_down)
+ ds->ops->phylink_mac_link_down(ds, port,
+ MLO_AN_FIXED, PHY_INTERFACE_MODE_NA);
++ of_node_put(phy_np);
+ return dsa_port_phylink_register(dp);
+ }
++ of_node_put(phy_np);
+ return 0;
+ }
+
+diff --git a/net/dsa/switch.c b/net/dsa/switch.c
+index 6466d0539af9f..fb69f2f14234e 100644
+--- a/net/dsa/switch.c
++++ b/net/dsa/switch.c
+@@ -264,7 +264,7 @@ static int dsa_switch_do_mdb_del(struct dsa_switch *ds, int port,
+
+ err = ds->ops->port_mdb_del(ds, port, mdb);
+ if (err) {
+- refcount_inc(&a->refcount);
++ refcount_set(&a->refcount, 1);
+ return err;
+ }
+
+@@ -329,7 +329,7 @@ static int dsa_switch_do_fdb_del(struct dsa_switch *ds, int port,
+
+ err = ds->ops->port_fdb_del(ds, port, addr, vid);
+ if (err) {
+- refcount_inc(&a->refcount);
++ refcount_set(&a->refcount, 1);
+ return err;
+ }
+
+@@ -644,7 +644,7 @@ static int
+ dsa_switch_mrp_add_ring_role(struct dsa_switch *ds,
+ struct dsa_notifier_mrp_ring_role_info *info)
+ {
+- if (!ds->ops->port_mrp_add)
++ if (!ds->ops->port_mrp_add_ring_role)
+ return -EOPNOTSUPP;
+
+ if (ds->index == info->sw_index)
+@@ -658,7 +658,7 @@ static int
+ dsa_switch_mrp_del_ring_role(struct dsa_switch *ds,
+ struct dsa_notifier_mrp_ring_role_info *info)
+ {
+- if (!ds->ops->port_mrp_del)
++ if (!ds->ops->port_mrp_del_ring_role)
+ return -EOPNOTSUPP;
+
+ if (ds->index == info->sw_index)
+diff --git a/net/dsa/tag_hellcreek.c b/net/dsa/tag_hellcreek.c
+index f64b805303cd7..eb204ad36eeec 100644
+--- a/net/dsa/tag_hellcreek.c
++++ b/net/dsa/tag_hellcreek.c
+@@ -21,6 +21,14 @@ static struct sk_buff *hellcreek_xmit(struct sk_buff *skb,
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+ u8 *tag;
+
++ /* Calculate checksums (if required) before adding the trailer tag to
++ * avoid including it in calculations. That would lead to wrong
++ * checksums after the switch strips the tag.
++ */
++ if (skb->ip_summed == CHECKSUM_PARTIAL &&
++ skb_checksum_help(skb))
++ return NULL;
++
+ /* Tag encoding */
+ tag = skb_put(skb, HELLCREEK_TAG_LEN);
+ *tag = BIT(dp->index);
+diff --git a/net/dsa/tag_lan9303.c b/net/dsa/tag_lan9303.c
+index cb548188f8134..98d7d7120bab2 100644
+--- a/net/dsa/tag_lan9303.c
++++ b/net/dsa/tag_lan9303.c
+@@ -77,7 +77,6 @@ static struct sk_buff *lan9303_xmit(struct sk_buff *skb, struct net_device *dev)
+
+ static struct sk_buff *lan9303_rcv(struct sk_buff *skb, struct net_device *dev)
+ {
+- __be16 *lan9303_tag;
+ u16 lan9303_tag1;
+ unsigned int source_port;
+
+@@ -87,14 +86,15 @@ static struct sk_buff *lan9303_rcv(struct sk_buff *skb, struct net_device *dev)
+ return NULL;
+ }
+
+- lan9303_tag = dsa_etype_header_pos_rx(skb);
+-
+- if (lan9303_tag[0] != htons(ETH_P_8021Q)) {
+- dev_warn_ratelimited(&dev->dev, "Dropping packet due to invalid VLAN marker\n");
+- return NULL;
++ if (skb_vlan_tag_present(skb)) {
++ lan9303_tag1 = skb_vlan_tag_get(skb);
++ __vlan_hwaccel_clear_tag(skb);
++ } else {
++ skb_push_rcsum(skb, ETH_HLEN);
++ __skb_vlan_pop(skb, &lan9303_tag1);
++ skb_pull_rcsum(skb, ETH_HLEN);
+ }
+
+- lan9303_tag1 = ntohs(lan9303_tag[1]);
+ source_port = lan9303_tag1 & 0x3;
+
+ skb->dev = dsa_master_find_slave(dev, 0, source_port);
+@@ -103,13 +103,6 @@ static struct sk_buff *lan9303_rcv(struct sk_buff *skb, struct net_device *dev)
+ return NULL;
+ }
+
+- /* remove the special VLAN tag between the MAC addresses
+- * and the current ethertype field.
+- */
+- skb_pull_rcsum(skb, 2 + 2);
+-
+- dsa_strip_etype_header(skb, LAN9303_TAG_LEN);
+-
+ if (!(lan9303_tag1 & LAN9303_TAG_RX_TRAPPED_TO_CPU))
+ dsa_default_offload_fwd_mark(skb);
+
+diff --git a/net/dsa/tag_ocelot.c b/net/dsa/tag_ocelot.c
+index 605b51ca69210..6e0518aa3a4d2 100644
+--- a/net/dsa/tag_ocelot.c
++++ b/net/dsa/tag_ocelot.c
+@@ -62,6 +62,7 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb,
+ struct dsa_port *dp;
+ u8 *extraction;
+ u16 vlan_tpid;
++ u64 rew_val;
+
+ /* Revert skb->data by the amount consumed by the DSA master,
+ * so it points to the beginning of the frame.
+@@ -91,6 +92,7 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb,
+ ocelot_xfh_get_qos_class(extraction, &qos_class);
+ ocelot_xfh_get_tag_type(extraction, &tag_type);
+ ocelot_xfh_get_vlan_tci(extraction, &vlan_tci);
++ ocelot_xfh_get_rew_val(extraction, &rew_val);
+
+ skb->dev = dsa_master_find_slave(netdev, 0, src_port);
+ if (!skb->dev)
+@@ -104,6 +106,7 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb,
+
+ dsa_default_offload_fwd_mark(skb);
+ skb->priority = qos_class;
++ OCELOT_SKB_CB(skb)->tstamp_lo = rew_val;
+
+ /* Ocelot switches copy frames unmodified to the CPU. However, it is
+ * possible for the user to request a VLAN modification through
+diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c
+index f2abc31528883..e4983f473a3c5 100644
+--- a/net/ethtool/ioctl.c
++++ b/net/ethtool/ioctl.c
+@@ -1697,7 +1697,7 @@ static noinline_for_stack int ethtool_set_coalesce(struct net_device *dev,
+ struct ethtool_coalesce coalesce;
+ int ret;
+
+- if (!dev->ethtool_ops->set_coalesce && !dev->ethtool_ops->get_coalesce)
++ if (!dev->ethtool_ops->set_coalesce || !dev->ethtool_ops->get_coalesce)
+ return -EOPNOTSUPP;
+
+ ret = dev->ethtool_ops->get_coalesce(dev, &coalesce, &kernel_coalesce,
+diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c
+index 1797a0a900195..b3729bdafb602 100644
+--- a/net/ethtool/netlink.c
++++ b/net/ethtool/netlink.c
+@@ -40,7 +40,8 @@ int ethnl_ops_begin(struct net_device *dev)
+ if (dev->dev.parent)
+ pm_runtime_get_sync(dev->dev.parent);
+
+- if (!netif_device_present(dev)) {
++ if (!netif_device_present(dev) ||
++ dev->reg_state == NETREG_UNREGISTERING) {
+ ret = -ENODEV;
+ goto err;
+ }
+diff --git a/net/ethtool/pause.c b/net/ethtool/pause.c
+index 9009f412151e7..ee1e5806bc93a 100644
+--- a/net/ethtool/pause.c
++++ b/net/ethtool/pause.c
+@@ -56,8 +56,7 @@ static int pause_reply_size(const struct ethnl_req_info *req_base,
+
+ if (req_base->flags & ETHTOOL_FLAG_STATS)
+ n += nla_total_size(0) + /* _PAUSE_STATS */
+- nla_total_size_64bit(sizeof(u64)) *
+- (ETHTOOL_A_PAUSE_STAT_MAX - 2);
++ nla_total_size_64bit(sizeof(u64)) * ETHTOOL_PAUSE_STAT_CNT;
+ return n;
+ }
+
+diff --git a/net/ieee802154/nl802154.c b/net/ieee802154/nl802154.c
+index 277124f206e06..e0b072aecf0f3 100644
+--- a/net/ieee802154/nl802154.c
++++ b/net/ieee802154/nl802154.c
+@@ -1441,7 +1441,7 @@ static int nl802154_send_key(struct sk_buff *msg, u32 cmd, u32 portid,
+
+ hdr = nl802154hdr_put(msg, portid, seq, flags, cmd);
+ if (!hdr)
+- return -1;
++ return -ENOBUFS;
+
+ if (nla_put_u32(msg, NL802154_ATTR_IFINDEX, dev->ifindex))
+ goto nla_put_failure;
+@@ -1634,7 +1634,7 @@ static int nl802154_send_device(struct sk_buff *msg, u32 cmd, u32 portid,
+
+ hdr = nl802154hdr_put(msg, portid, seq, flags, cmd);
+ if (!hdr)
+- return -1;
++ return -ENOBUFS;
+
+ if (nla_put_u32(msg, NL802154_ATTR_IFINDEX, dev->ifindex))
+ goto nla_put_failure;
+@@ -1812,7 +1812,7 @@ static int nl802154_send_devkey(struct sk_buff *msg, u32 cmd, u32 portid,
+
+ hdr = nl802154hdr_put(msg, portid, seq, flags, cmd);
+ if (!hdr)
+- return -1;
++ return -ENOBUFS;
+
+ if (nla_put_u32(msg, NL802154_ATTR_IFINDEX, dev->ifindex))
+ goto nla_put_failure;
+@@ -1988,7 +1988,7 @@ static int nl802154_send_seclevel(struct sk_buff *msg, u32 cmd, u32 portid,
+
+ hdr = nl802154hdr_put(msg, portid, seq, flags, cmd);
+ if (!hdr)
+- return -1;
++ return -ENOBUFS;
+
+ if (nla_put_u32(msg, NL802154_ATTR_IFINDEX, dev->ifindex))
+ goto nla_put_failure;
+diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
+index 1d816a5fd3eb9..77534b44b8c7c 100644
+--- a/net/ipv4/af_inet.c
++++ b/net/ipv4/af_inet.c
+@@ -158,7 +158,7 @@ void inet_sock_destruct(struct sock *sk)
+
+ kfree(rcu_dereference_protected(inet->inet_opt, 1));
+ dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1));
+- dst_release(sk->sk_rx_dst);
++ dst_release(rcu_dereference_protected(sk->sk_rx_dst, 1));
+ sk_refcnt_debug_dec(sk);
+ }
+ EXPORT_SYMBOL(inet_sock_destruct);
+@@ -773,26 +773,28 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr,
+ DECLARE_SOCKADDR(struct sockaddr_in *, sin, uaddr);
+
+ sin->sin_family = AF_INET;
++ lock_sock(sk);
+ if (peer) {
+ if (!inet->inet_dport ||
+ (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_SYN_SENT)) &&
+- peer == 1))
++ peer == 1)) {
++ release_sock(sk);
+ return -ENOTCONN;
++ }
+ sin->sin_port = inet->inet_dport;
+ sin->sin_addr.s_addr = inet->inet_daddr;
+- BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
+- CGROUP_INET4_GETPEERNAME,
+- NULL);
++ BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin,
++ CGROUP_INET4_GETPEERNAME);
+ } else {
+ __be32 addr = inet->inet_rcv_saddr;
+ if (!addr)
+ addr = inet->inet_saddr;
+ sin->sin_port = inet->inet_sport;
+ sin->sin_addr.s_addr = addr;
+- BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
+- CGROUP_INET4_GETSOCKNAME,
+- NULL);
++ BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin,
++ CGROUP_INET4_GETSOCKNAME);
+ }
++ release_sock(sk);
+ memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
+ return sizeof(*sin);
+ }
+@@ -1378,8 +1380,11 @@ struct sk_buff *inet_gso_segment(struct sk_buff *skb,
+ }
+
+ ops = rcu_dereference(inet_offloads[proto]);
+- if (likely(ops && ops->callbacks.gso_segment))
++ if (likely(ops && ops->callbacks.gso_segment)) {
+ segs = ops->callbacks.gso_segment(skb, features);
++ if (!segs)
++ skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
++ }
+
+ if (IS_ERR_OR_NULL(segs))
+ goto out;
+@@ -2002,6 +2007,10 @@ static int __init inet_init(void)
+
+ ip_init();
+
++ /* Initialise per-cpu ipv4 mibs */
++ if (init_ipv4_mibs())
++ panic("%s: Cannot init ipv4 mibs\n", __func__);
++
+ /* Setup TCP slab cache for open requests. */
+ tcp_init();
+
+@@ -2032,12 +2041,6 @@ static int __init inet_init(void)
+
+ if (init_inet_pernet_ops())
+ pr_crit("%s: Cannot init ipv4 inet pernet ops\n", __func__);
+- /*
+- * Initialise per-cpu ipv4 mibs
+- */
+-
+- if (init_ipv4_mibs())
+- pr_crit("%s: Cannot init ipv4 mibs\n", __func__);
+
+ ipv4_proc_init();
+
+diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c
+index 922dd73e57406..83a47998c4b18 100644
+--- a/net/ipv4/arp.c
++++ b/net/ipv4/arp.c
+@@ -1116,13 +1116,18 @@ static int arp_req_get(struct arpreq *r, struct net_device *dev)
+ return err;
+ }
+
+-static int arp_invalidate(struct net_device *dev, __be32 ip)
++int arp_invalidate(struct net_device *dev, __be32 ip, bool force)
+ {
+ struct neighbour *neigh = neigh_lookup(&arp_tbl, &ip, dev);
+ int err = -ENXIO;
+ struct neigh_table *tbl = &arp_tbl;
+
+ if (neigh) {
++ if ((neigh->nud_state & NUD_VALID) && !force) {
++ neigh_release(neigh);
++ return 0;
++ }
++
+ if (neigh->nud_state & ~NUD_NOARP)
+ err = neigh_update(neigh, NULL, NUD_FAILED,
+ NEIGH_UPDATE_F_OVERRIDE|
+@@ -1169,7 +1174,7 @@ static int arp_req_delete(struct net *net, struct arpreq *r,
+ if (!dev)
+ return -EINVAL;
+ }
+- return arp_invalidate(dev, ip);
++ return arp_invalidate(dev, ip, true);
+ }
+
+ /*
+diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c
+index 0dcee9df13268..d3a2dbd13ea6b 100644
+--- a/net/ipv4/bpf_tcp_ca.c
++++ b/net/ipv4/bpf_tcp_ca.c
+@@ -212,6 +212,8 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
+ offsetof(struct tcp_congestion_ops, release))
+ return &bpf_sk_getsockopt_proto;
+ return NULL;
++ case BPF_FUNC_ktime_get_coarse_ns:
++ return &bpf_ktime_get_coarse_ns_proto;
+ default:
+ return bpf_base_func_proto(func_id);
+ }
+diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c
+index f4468980b6757..4744c7839de53 100644
+--- a/net/ipv4/devinet.c
++++ b/net/ipv4/devinet.c
+@@ -2587,7 +2587,7 @@ static int __devinet_sysctl_register(struct net *net, char *dev_name,
+ free:
+ kfree(t);
+ out:
+- return -ENOBUFS;
++ return -ENOMEM;
+ }
+
+ static void __devinet_sysctl_unregister(struct net *net,
+diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
+index 851f542928a33..d747166bb291c 100644
+--- a/net/ipv4/esp4.c
++++ b/net/ipv4/esp4.c
+@@ -455,6 +455,10 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
+ return err;
+ }
+
++ if (ALIGN(tailen, L1_CACHE_BYTES) > PAGE_SIZE ||
++ ALIGN(skb->data_len, L1_CACHE_BYTES) > PAGE_SIZE)
++ goto cow;
++
+ if (!skb_cloned(skb)) {
+ if (tailen <= skb_tailroom(skb)) {
+ nfrags = 1;
+@@ -671,7 +675,7 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
+ struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
+ u32 padto;
+
+- padto = min(x->tfcpad, __xfrm_state_mtu(x, dst->child_mtu_cached));
++ padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached));
+ if (skb->len < padto)
+ esp.tfclen = padto - skb->len;
+ }
+diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c
+index 8e4e9aa12130d..dad5d29a6a8db 100644
+--- a/net/ipv4/esp4_offload.c
++++ b/net/ipv4/esp4_offload.c
+@@ -159,6 +159,9 @@ static struct sk_buff *xfrm4_beet_gso_segment(struct xfrm_state *x,
+ skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
+ }
+
++ if (proto == IPPROTO_IPV6)
++ skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
++
+ __skb_pull(skb, skb_transport_offset(skb));
+ ops = rcu_dereference(inet_offloads[proto]);
+ if (likely(ops && ops->callbacks.gso_segment))
+diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
+index 9fe13e4f5d08a..1eb7795edb9dc 100644
+--- a/net/ipv4/fib_frontend.c
++++ b/net/ipv4/fib_frontend.c
+@@ -1112,9 +1112,11 @@ void fib_add_ifaddr(struct in_ifaddr *ifa)
+ return;
+
+ /* Add broadcast address, if it is explicitly assigned. */
+- if (ifa->ifa_broadcast && ifa->ifa_broadcast != htonl(0xFFFFFFFF))
++ if (ifa->ifa_broadcast && ifa->ifa_broadcast != htonl(0xFFFFFFFF)) {
+ fib_magic(RTM_NEWROUTE, RTN_BROADCAST, ifa->ifa_broadcast, 32,
+ prim, 0);
++ arp_invalidate(dev, ifa->ifa_broadcast, false);
++ }
+
+ if (!ipv4_is_zeronet(prefix) && !(ifa->ifa_flags & IFA_F_SECONDARY) &&
+ (prefix != addr || ifa->ifa_prefixlen < 32)) {
+@@ -1128,6 +1130,7 @@ void fib_add_ifaddr(struct in_ifaddr *ifa)
+ if (ifa->ifa_prefixlen < 31) {
+ fib_magic(RTM_NEWROUTE, RTN_BROADCAST, prefix | ~mask,
+ 32, prim, 0);
++ arp_invalidate(dev, prefix | ~mask, false);
+ }
+ }
+ }
+@@ -1582,7 +1585,7 @@ static int __net_init fib_net_init(struct net *net)
+ int error;
+
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+- net->ipv4.fib_num_tclassid_users = 0;
++ atomic_set(&net->ipv4.fib_num_tclassid_users, 0);
+ #endif
+ error = ip_fib_net_init(net);
+ if (error < 0)
+diff --git a/net/ipv4/fib_lookup.h b/net/ipv4/fib_lookup.h
+index e184bcb199434..78e40ea42e58d 100644
+--- a/net/ipv4/fib_lookup.h
++++ b/net/ipv4/fib_lookup.h
+@@ -16,10 +16,9 @@ struct fib_alias {
+ u8 fa_slen;
+ u32 tb_id;
+ s16 fa_default;
+- u8 offload:1,
+- trap:1,
+- offload_failed:1,
+- unused:5;
++ u8 offload;
++ u8 trap;
++ u8 offload_failed;
+ struct rcu_head rcu;
+ };
+
+diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
+index ce54a30c2ef1e..d279cb8ac1584 100644
+--- a/net/ipv4/fib_rules.c
++++ b/net/ipv4/fib_rules.c
+@@ -141,6 +141,7 @@ INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
+ }
+
+ INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
++ int flags,
+ struct fib_lookup_arg *arg)
+ {
+ struct fib_result *result = (struct fib_result *) arg->result;
+@@ -263,7 +264,7 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
+ if (tb[FRA_FLOW]) {
+ rule4->tclassid = nla_get_u32(tb[FRA_FLOW]);
+ if (rule4->tclassid)
+- net->ipv4.fib_num_tclassid_users++;
++ atomic_inc(&net->ipv4.fib_num_tclassid_users);
+ }
+ #endif
+
+@@ -295,7 +296,7 @@ static int fib4_rule_delete(struct fib_rule *rule)
+
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+ if (((struct fib4_rule *)rule)->tclassid)
+- net->ipv4.fib_num_tclassid_users--;
++ atomic_dec(&net->ipv4.fib_num_tclassid_users);
+ #endif
+ net->ipv4.fib_has_custom_rules = true;
+
+diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
+index 3364cb9c67e01..b5563f5ff1765 100644
+--- a/net/ipv4/fib_semantics.c
++++ b/net/ipv4/fib_semantics.c
+@@ -29,6 +29,7 @@
+ #include <linux/init.h>
+ #include <linux/slab.h>
+ #include <linux/netlink.h>
++#include <linux/hash.h>
+
+ #include <net/arp.h>
+ #include <net/ip.h>
+@@ -220,7 +221,7 @@ void fib_nh_release(struct net *net, struct fib_nh *fib_nh)
+ {
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+ if (fib_nh->nh_tclassid)
+- net->ipv4.fib_num_tclassid_users--;
++ atomic_dec(&net->ipv4.fib_num_tclassid_users);
+ #endif
+ fib_nh_common_release(&fib_nh->nh_common);
+ }
+@@ -249,7 +250,6 @@ void free_fib_info(struct fib_info *fi)
+ pr_warn("Freeing alive fib_info %p\n", fi);
+ return;
+ }
+- fib_info_cnt--;
+
+ call_rcu(&fi->rcu, free_fib_info_rcu);
+ }
+@@ -260,6 +260,10 @@ void fib_release_info(struct fib_info *fi)
+ spin_lock_bh(&fib_info_lock);
+ if (fi && refcount_dec_and_test(&fi->fib_treeref)) {
+ hlist_del(&fi->fib_hash);
++
++ /* Paired with READ_ONCE() in fib_create_info(). */
++ WRITE_ONCE(fib_info_cnt, fib_info_cnt - 1);
++
+ if (fi->fib_prefsrc)
+ hlist_del(&fi->fib_lhash);
+ if (fi->nh) {
+@@ -316,11 +320,15 @@ static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)
+
+ static inline unsigned int fib_devindex_hashfn(unsigned int val)
+ {
+- unsigned int mask = DEVINDEX_HASHSIZE - 1;
++ return hash_32(val, DEVINDEX_HASHBITS);
++}
++
++static struct hlist_head *
++fib_info_devhash_bucket(const struct net_device *dev)
++{
++ u32 val = net_hash_mix(dev_net(dev)) ^ dev->ifindex;
+
+- return (val ^
+- (val >> DEVINDEX_HASHBITS) ^
+- (val >> (DEVINDEX_HASHBITS * 2))) & mask;
++ return &fib_info_devhash[fib_devindex_hashfn(val)];
+ }
+
+ static unsigned int fib_info_hashfn_1(int init_val, u8 protocol, u8 scope,
+@@ -430,12 +438,11 @@ int ip_fib_check_default(__be32 gw, struct net_device *dev)
+ {
+ struct hlist_head *head;
+ struct fib_nh *nh;
+- unsigned int hash;
+
+ spin_lock(&fib_info_lock);
+
+- hash = fib_devindex_hashfn(dev->ifindex);
+- head = &fib_info_devhash[hash];
++ head = fib_info_devhash_bucket(dev);
++
+ hlist_for_each_entry(nh, head, nh_hash) {
+ if (nh->fib_nh_dev == dev &&
+ nh->fib_nh_gw4 == gw &&
+@@ -517,9 +524,9 @@ void rtmsg_fib(int event, __be32 key, struct fib_alias *fa,
+ fri.dst_len = dst_len;
+ fri.tos = fa->fa_tos;
+ fri.type = fa->fa_type;
+- fri.offload = fa->offload;
+- fri.trap = fa->trap;
+- fri.offload_failed = fa->offload_failed;
++ fri.offload = READ_ONCE(fa->offload);
++ fri.trap = READ_ONCE(fa->trap);
++ fri.offload_failed = READ_ONCE(fa->offload_failed);
+ err = fib_dump_info(skb, info->portid, seq, event, &fri, nlm_flags);
+ if (err < 0) {
+ /* -EMSGSIZE implies BUG in fib_nlmsg_size() */
+@@ -632,7 +639,7 @@ int fib_nh_init(struct net *net, struct fib_nh *nh,
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+ nh->nh_tclassid = cfg->fc_flow;
+ if (nh->nh_tclassid)
+- net->ipv4.fib_num_tclassid_users++;
++ atomic_inc(&net->ipv4.fib_num_tclassid_users);
+ #endif
+ #ifdef CONFIG_IP_ROUTE_MULTIPATH
+ nh->fib_nh_weight = nh_weight;
+@@ -662,6 +669,19 @@ static int fib_count_nexthops(struct rtnexthop *rtnh, int remaining,
+ return nhs;
+ }
+
++static int fib_gw_from_attr(__be32 *gw, struct nlattr *nla,
++ struct netlink_ext_ack *extack)
++{
++ if (nla_len(nla) < sizeof(*gw)) {
++ NL_SET_ERR_MSG(extack, "Invalid IPv4 address in RTA_GATEWAY");
++ return -EINVAL;
++ }
++
++ *gw = nla_get_in_addr(nla);
++
++ return 0;
++}
++
+ /* only called when fib_nh is integrated into fib_info */
+ static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
+ int remaining, struct fib_config *cfg,
+@@ -704,7 +724,11 @@ static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
+ return -EINVAL;
+ }
+ if (nla) {
+- fib_cfg.fc_gw4 = nla_get_in_addr(nla);
++ ret = fib_gw_from_attr(&fib_cfg.fc_gw4, nla,
++ extack);
++ if (ret)
++ goto errout;
++
+ if (fib_cfg.fc_gw4)
+ fib_cfg.fc_gw_family = AF_INET;
+ } else if (nlav) {
+@@ -714,10 +738,18 @@ static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
+ }
+
+ nla = nla_find(attrs, attrlen, RTA_FLOW);
+- if (nla)
++ if (nla) {
++ if (nla_len(nla) < sizeof(u32)) {
++ NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW");
++ return -EINVAL;
++ }
+ fib_cfg.fc_flow = nla_get_u32(nla);
++ }
+
+ fib_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
++ /* RTA_ENCAP_TYPE length checked in
++ * lwtunnel_valid_encap_type_attr
++ */
+ nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
+ if (nla)
+ fib_cfg.fc_encap_type = nla_get_u16(nla);
+@@ -855,8 +887,13 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,
+ }
+
+ if (cfg->fc_oif || cfg->fc_gw_family) {
+- struct fib_nh *nh = fib_info_nh(fi, 0);
++ struct fib_nh *nh;
++
++ /* cannot match on nexthop object attributes */
++ if (fi->nh)
++ return 1;
+
++ nh = fib_info_nh(fi, 0);
+ if (cfg->fc_encap) {
+ if (fib_encap_match(net, cfg->fc_encap_type,
+ cfg->fc_encap, nh, cfg, extack))
+@@ -902,6 +939,7 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,
+ attrlen = rtnh_attrlen(rtnh);
+ if (attrlen > 0) {
+ struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh);
++ int err;
+
+ nla = nla_find(attrs, attrlen, RTA_GATEWAY);
+ nlav = nla_find(attrs, attrlen, RTA_VIA);
+@@ -912,12 +950,17 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,
+ }
+
+ if (nla) {
++ __be32 gw;
++
++ err = fib_gw_from_attr(&gw, nla, extack);
++ if (err)
++ return err;
++
+ if (nh->fib_nh_gw_family != AF_INET ||
+- nla_get_in_addr(nla) != nh->fib_nh_gw4)
++ gw != nh->fib_nh_gw4)
+ return 1;
+ } else if (nlav) {
+ struct fib_config cfg2;
+- int err;
+
+ err = fib_gw_from_via(&cfg2, nlav, extack);
+ if (err)
+@@ -940,8 +983,14 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,
+
+ #ifdef CONFIG_IP_ROUTE_CLASSID
+ nla = nla_find(attrs, attrlen, RTA_FLOW);
+- if (nla && nla_get_u32(nla) != nh->nh_tclassid)
+- return 1;
++ if (nla) {
++ if (nla_len(nla) < sizeof(u32)) {
++ NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW");
++ return -EINVAL;
++ }
++ if (nla_get_u32(nla) != nh->nh_tclassid)
++ return 1;
++ }
+ #endif
+ }
+
+@@ -1393,7 +1442,9 @@ struct fib_info *fib_create_info(struct fib_config *cfg,
+ #endif
+
+ err = -ENOBUFS;
+- if (fib_info_cnt >= fib_info_hash_size) {
++
++ /* Paired with WRITE_ONCE() in fib_release_info() */
++ if (READ_ONCE(fib_info_cnt) >= fib_info_hash_size) {
+ unsigned int new_size = fib_info_hash_size << 1;
+ struct hlist_head *new_info_hash;
+ struct hlist_head *new_laddrhash;
+@@ -1425,7 +1476,6 @@ struct fib_info *fib_create_info(struct fib_config *cfg,
+ return ERR_PTR(err);
+ }
+
+- fib_info_cnt++;
+ fi->fib_net = net;
+ fi->fib_protocol = cfg->fc_protocol;
+ fi->fib_scope = cfg->fc_scope;
+@@ -1552,6 +1602,7 @@ link_it:
+ refcount_set(&fi->fib_treeref, 1);
+ refcount_set(&fi->fib_clntref, 1);
+ spin_lock_bh(&fib_info_lock);
++ fib_info_cnt++;
+ hlist_add_head(&fi->fib_hash,
+ &fib_info_hash[fib_info_hashfn(fi)]);
+ if (fi->fib_prefsrc) {
+@@ -1565,12 +1616,10 @@ link_it:
+ } else {
+ change_nexthops(fi) {
+ struct hlist_head *head;
+- unsigned int hash;
+
+ if (!nexthop_nh->fib_nh_dev)
+ continue;
+- hash = fib_devindex_hashfn(nexthop_nh->fib_nh_dev->ifindex);
+- head = &fib_info_devhash[hash];
++ head = fib_info_devhash_bucket(nexthop_nh->fib_nh_dev);
+ hlist_add_head(&nexthop_nh->nh_hash, head);
+ } endfor_nexthops(fi)
+ }
+@@ -1922,8 +1971,7 @@ void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig)
+
+ void fib_sync_mtu(struct net_device *dev, u32 orig_mtu)
+ {
+- unsigned int hash = fib_devindex_hashfn(dev->ifindex);
+- struct hlist_head *head = &fib_info_devhash[hash];
++ struct hlist_head *head = fib_info_devhash_bucket(dev);
+ struct fib_nh *nh;
+
+ hlist_for_each_entry(nh, head, nh_hash) {
+@@ -1942,12 +1990,11 @@ void fib_sync_mtu(struct net_device *dev, u32 orig_mtu)
+ */
+ int fib_sync_down_dev(struct net_device *dev, unsigned long event, bool force)
+ {
+- int ret = 0;
+- int scope = RT_SCOPE_NOWHERE;
++ struct hlist_head *head = fib_info_devhash_bucket(dev);
+ struct fib_info *prev_fi = NULL;
+- unsigned int hash = fib_devindex_hashfn(dev->ifindex);
+- struct hlist_head *head = &fib_info_devhash[hash];
++ int scope = RT_SCOPE_NOWHERE;
+ struct fib_nh *nh;
++ int ret = 0;
+
+ if (force)
+ scope = -1;
+@@ -2092,7 +2139,6 @@ out:
+ int fib_sync_up(struct net_device *dev, unsigned char nh_flags)
+ {
+ struct fib_info *prev_fi;
+- unsigned int hash;
+ struct hlist_head *head;
+ struct fib_nh *nh;
+ int ret;
+@@ -2108,8 +2154,7 @@ int fib_sync_up(struct net_device *dev, unsigned char nh_flags)
+ }
+
+ prev_fi = NULL;
+- hash = fib_devindex_hashfn(dev->ifindex);
+- head = &fib_info_devhash[hash];
++ head = fib_info_devhash_bucket(dev);
+ ret = 0;
+
+ hlist_for_each_entry(nh, head, nh_hash) {
+diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
+index 8060524f42566..f7f74d5c14da6 100644
+--- a/net/ipv4/fib_trie.c
++++ b/net/ipv4/fib_trie.c
+@@ -1047,19 +1047,23 @@ void fib_alias_hw_flags_set(struct net *net, const struct fib_rt_info *fri)
+ if (!fa_match)
+ goto out;
+
+- if (fa_match->offload == fri->offload && fa_match->trap == fri->trap &&
+- fa_match->offload_failed == fri->offload_failed)
++ /* These are paired with the WRITE_ONCE() happening in this function.
++ * The reason is that we are only protected by RCU at this point.
++ */
++ if (READ_ONCE(fa_match->offload) == fri->offload &&
++ READ_ONCE(fa_match->trap) == fri->trap &&
++ READ_ONCE(fa_match->offload_failed) == fri->offload_failed)
+ goto out;
+
+- fa_match->offload = fri->offload;
+- fa_match->trap = fri->trap;
++ WRITE_ONCE(fa_match->offload, fri->offload);
++ WRITE_ONCE(fa_match->trap, fri->trap);
+
+ /* 2 means send notifications only if offload_failed was changed. */
+ if (net->ipv4.sysctl_fib_notify_on_flag_change == 2 &&
+- fa_match->offload_failed == fri->offload_failed)
++ READ_ONCE(fa_match->offload_failed) == fri->offload_failed)
+ goto out;
+
+- fa_match->offload_failed = fri->offload_failed;
++ WRITE_ONCE(fa_match->offload_failed, fri->offload_failed);
+
+ if (!net->ipv4.sysctl_fib_notify_on_flag_change)
+ goto out;
+@@ -2297,9 +2301,9 @@ static int fn_trie_dump_leaf(struct key_vector *l, struct fib_table *tb,
+ fri.dst_len = KEYLENGTH - fa->fa_slen;
+ fri.tos = fa->fa_tos;
+ fri.type = fa->fa_type;
+- fri.offload = fa->offload;
+- fri.trap = fa->trap;
+- fri.offload_failed = fa->offload_failed;
++ fri.offload = READ_ONCE(fa->offload);
++ fri.trap = READ_ONCE(fa->trap);
++ fri.offload_failed = READ_ONCE(fa->offload_failed);
+ err = fib_dump_info(skb,
+ NETLINK_CB(cb->skb).portid,
+ cb->nlh->nlmsg_seq,
+diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
+index d2e2b3d18c668..930f6c41f519c 100644
+--- a/net/ipv4/igmp.c
++++ b/net/ipv4/igmp.c
+@@ -2403,9 +2403,10 @@ int ip_mc_source(int add, int omode, struct sock *sk, struct
+ /* decrease mem now to avoid the memleak warning */
+ atomic_sub(struct_size(psl, sl_addr, psl->sl_max),
+ &sk->sk_omem_alloc);
+- kfree_rcu(psl, rcu);
+ }
+ rcu_assign_pointer(pmc->sflist, newpsl);
++ if (psl)
++ kfree_rcu(psl, rcu);
+ psl = newpsl;
+ }
+ rv = 1; /* > 0 for insert logic below if sl_count is 0 */
+@@ -2507,11 +2508,13 @@ int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf, int ifindex)
+ /* decrease mem now to avoid the memleak warning */
+ atomic_sub(struct_size(psl, sl_addr, psl->sl_max),
+ &sk->sk_omem_alloc);
+- kfree_rcu(psl, rcu);
+- } else
++ } else {
+ (void) ip_mc_del_src(in_dev, &msf->imsf_multiaddr, pmc->sfmode,
+ 0, NULL, 0);
++ }
+ rcu_assign_pointer(pmc->sflist, newpsl);
++ if (psl)
++ kfree_rcu(psl, rcu);
+ pmc->sfmode = msf->imsf_fmode;
+ err = 0;
+ done:
+diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
+index f25d02ad4a8af..62a67fdc344cd 100644
+--- a/net/ipv4/inet_connection_sock.c
++++ b/net/ipv4/inet_connection_sock.c
+@@ -721,7 +721,7 @@ static struct request_sock *inet_reqsk_clone(struct request_sock *req,
+
+ sk_node_init(&nreq_sk->sk_node);
+ nreq_sk->sk_tx_queue_mapping = req_sk->sk_tx_queue_mapping;
+-#ifdef CONFIG_XPS
++#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
+ nreq_sk->sk_rx_queue_mapping = req_sk->sk_rx_queue_mapping;
+ #endif
+ nreq_sk->sk_incoming_cpu = req_sk->sk_incoming_cpu;
+@@ -1015,7 +1015,7 @@ void inet_csk_destroy_sock(struct sock *sk)
+
+ sk_refcnt_debug_release(sk);
+
+- percpu_counter_dec(sk->sk_prot->orphan_count);
++ this_cpu_dec(*sk->sk_prot->orphan_count);
+
+ sock_put(sk);
+ }
+@@ -1074,7 +1074,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req,
+
+ sock_orphan(child);
+
+- percpu_counter_inc(sk->sk_prot->orphan_count);
++ this_cpu_inc(*sk->sk_prot->orphan_count);
+
+ if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
+ BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req);
+diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c
+index ef7897226f08e..ae70e07c52445 100644
+--- a/net/ipv4/inet_diag.c
++++ b/net/ipv4/inet_diag.c
+@@ -261,6 +261,7 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
+ r->idiag_state = sk->sk_state;
+ r->idiag_timer = 0;
+ r->idiag_retrans = 0;
++ r->idiag_expires = 0;
+
+ if (inet_diag_msg_attrs_fill(sk, skb, r, ext,
+ sk_user_ns(NETLINK_CB(cb->skb).sk),
+@@ -314,9 +315,6 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
+ r->idiag_retrans = icsk->icsk_probes_out;
+ r->idiag_expires =
+ jiffies_delta_to_msecs(sk->sk_timer.expires - jiffies);
+- } else {
+- r->idiag_timer = 0;
+- r->idiag_expires = 0;
+ }
+
+ if ((ext & (1 << (INET_DIAG_INFO - 1))) && handler->idiag_info_size) {
+diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
+index 05cd198d7a6ba..341096807100c 100644
+--- a/net/ipv4/inet_fragment.c
++++ b/net/ipv4/inet_fragment.c
+@@ -235,9 +235,9 @@ void inet_frag_kill(struct inet_frag_queue *fq)
+ /* The RCU read lock provides a memory barrier
+ * guaranteeing that if fqdir->dead is false then
+ * the hash table destruction will not start until
+- * after we unlock. Paired with inet_frags_exit_net().
++ * after we unlock. Paired with fqdir_pre_exit().
+ */
+- if (!fqdir->dead) {
++ if (!READ_ONCE(fqdir->dead)) {
+ rhashtable_remove_fast(&fqdir->rhashtable, &fq->node,
+ fqdir->f->rhash_params);
+ refcount_dec(&fq->refcnt);
+@@ -352,9 +352,11 @@ static struct inet_frag_queue *inet_frag_create(struct fqdir *fqdir,
+ /* TODO : call from rcu_read_lock() and no longer use refcount_inc_not_zero() */
+ struct inet_frag_queue *inet_frag_find(struct fqdir *fqdir, void *key)
+ {
++ /* This pairs with WRITE_ONCE() in fqdir_pre_exit(). */
++ long high_thresh = READ_ONCE(fqdir->high_thresh);
+ struct inet_frag_queue *fq = NULL, *prev;
+
+- if (!fqdir->high_thresh || frag_mem_limit(fqdir) > fqdir->high_thresh)
++ if (!high_thresh || frag_mem_limit(fqdir) > high_thresh)
+ return NULL;
+
+ rcu_read_lock();
+diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
+index bfb522e513461..ee9c587031b4b 100644
+--- a/net/ipv4/inet_hashtables.c
++++ b/net/ipv4/inet_hashtables.c
+@@ -504,7 +504,7 @@ not_unique:
+ return -EADDRNOTAVAIL;
+ }
+
+-static u32 inet_sk_port_offset(const struct sock *sk)
++static u64 inet_sk_port_offset(const struct sock *sk)
+ {
+ const struct inet_sock *inet = inet_sk(sk);
+
+@@ -598,7 +598,7 @@ bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk)
+ if (ok) {
+ sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
+ } else {
+- percpu_counter_inc(sk->sk_prot->orphan_count);
++ this_cpu_inc(*sk->sk_prot->orphan_count);
+ inet_sk_set_state(sk, TCP_CLOSE);
+ sock_set_flag(sk, SOCK_DEAD);
+ inet_csk_destroy_sock(sk);
+@@ -637,7 +637,9 @@ int __inet_hash(struct sock *sk, struct sock *osk)
+ int err = 0;
+
+ if (sk->sk_state != TCP_LISTEN) {
++ local_bh_disable();
+ inet_ehash_nolisten(sk, osk, NULL);
++ local_bh_enable();
+ return 0;
+ }
+ WARN_ON(!sk_unhashed(sk));
+@@ -669,45 +671,54 @@ int inet_hash(struct sock *sk)
+ {
+ int err = 0;
+
+- if (sk->sk_state != TCP_CLOSE) {
+- local_bh_disable();
++ if (sk->sk_state != TCP_CLOSE)
+ err = __inet_hash(sk, NULL);
+- local_bh_enable();
+- }
+
+ return err;
+ }
+ EXPORT_SYMBOL_GPL(inet_hash);
+
+-void inet_unhash(struct sock *sk)
++static void __inet_unhash(struct sock *sk, struct inet_listen_hashbucket *ilb)
+ {
+- struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
+- struct inet_listen_hashbucket *ilb = NULL;
+- spinlock_t *lock;
+-
+ if (sk_unhashed(sk))
+ return;
+
+- if (sk->sk_state == TCP_LISTEN) {
+- ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
+- lock = &ilb->lock;
+- } else {
+- lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
+- }
+- spin_lock_bh(lock);
+- if (sk_unhashed(sk))
+- goto unlock;
+-
+ if (rcu_access_pointer(sk->sk_reuseport_cb))
+ reuseport_stop_listen_sock(sk);
+ if (ilb) {
++ struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
++
+ inet_unhash2(hashinfo, sk);
+ ilb->count--;
+ }
+ __sk_nulls_del_node_init_rcu(sk);
+ sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
+-unlock:
+- spin_unlock_bh(lock);
++}
++
++void inet_unhash(struct sock *sk)
++{
++ struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
++
++ if (sk_unhashed(sk))
++ return;
++
++ if (sk->sk_state == TCP_LISTEN) {
++ struct inet_listen_hashbucket *ilb;
++
++ ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
++ /* Don't disable bottom halves while acquiring the lock to
++ * avoid circular locking dependency on PREEMPT_RT.
++ */
++ spin_lock(&ilb->lock);
++ __inet_unhash(sk, ilb);
++ spin_unlock(&ilb->lock);
++ } else {
++ spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
++
++ spin_lock_bh(lock);
++ __inet_unhash(sk, NULL);
++ spin_unlock_bh(lock);
++ }
+ }
+ EXPORT_SYMBOL_GPL(inet_unhash);
+
+@@ -715,15 +726,17 @@ EXPORT_SYMBOL_GPL(inet_unhash);
+ * Note that we use 32bit integers (vs RFC 'short integers')
+ * because 2^16 is not a multiple of num_ephemeral and this
+ * property might be used by clever attacker.
+- * RFC claims using TABLE_LENGTH=10 buckets gives an improvement,
+- * we use 256 instead to really give more isolation and
+- * privacy, this only consumes 1 KB of kernel memory.
++ * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
++ * attacks were since demonstrated, thus we use 65536 instead to really
++ * give more isolation and privacy, at the expense of 256kB of kernel
++ * memory.
+ */
+-#define INET_TABLE_PERTURB_SHIFT 8
+-static u32 table_perturb[1 << INET_TABLE_PERTURB_SHIFT];
++#define INET_TABLE_PERTURB_SHIFT 16
++#define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT)
++static u32 *table_perturb;
+
+ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
+- struct sock *sk, u32 port_offset,
++ struct sock *sk, u64 port_offset,
+ int (*check_established)(struct inet_timewait_death_row *,
+ struct sock *, __u16, struct inet_timewait_sock **))
+ {
+@@ -763,10 +776,13 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
+ if (likely(remaining > 1))
+ remaining &= ~1U;
+
+- net_get_random_once(table_perturb, sizeof(table_perturb));
+- index = hash_32(port_offset, INET_TABLE_PERTURB_SHIFT);
++ net_get_random_once(table_perturb,
++ INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
++ index = port_offset & (INET_TABLE_PERTURB_SIZE - 1);
++
++ offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32);
++ offset %= remaining;
+
+- offset = (READ_ONCE(table_perturb[index]) + port_offset) % remaining;
+ /* In first pass we try ports of @low parity.
+ * inet_csk_get_port() does the opposite choice.
+ */
+@@ -820,11 +836,12 @@ next_port:
+ return -EADDRNOTAVAIL;
+
+ ok:
+- /* If our first attempt found a candidate, skip next candidate
+- * in 1/16 of cases to add some noise.
++ /* Here we want to add a little bit of randomness to the next source
++ * port that will be chosen. We use a max() with a random here so that
++ * on low contention the randomness is maximal and on high contention
++ * it may be inexistent.
+ */
+- if (!i && !(prandom_u32() % 16))
+- i = 2;
++ i = max_t(int, i, (prandom_u32() & 7) * 2);
+ WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
+
+ /* Head lock still held and bh's disabled */
+@@ -848,7 +865,7 @@ ok:
+ int inet_hash_connect(struct inet_timewait_death_row *death_row,
+ struct sock *sk)
+ {
+- u32 port_offset = 0;
++ u64 port_offset = 0;
+
+ if (!inet_sk(sk)->inet_num)
+ port_offset = inet_sk_port_offset(sk);
+@@ -898,6 +915,12 @@ void __init inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,
+ low_limit,
+ high_limit);
+ init_hashinfo_lhash2(h);
++
++ /* this one is used for source ports of outgoing connections */
++ table_perturb = kmalloc_array(INET_TABLE_PERTURB_SIZE,
++ sizeof(*table_perturb), GFP_KERNEL);
++ if (!table_perturb)
++ panic("TCP: failed to alloc table_perturb");
+ }
+
+ int inet_hashinfo2_init_mod(struct inet_hashinfo *h)
+diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
+index cfeb8890f94ee..fad803d2d711e 100644
+--- a/net/ipv4/ip_fragment.c
++++ b/net/ipv4/ip_fragment.c
+@@ -144,7 +144,8 @@ static void ip_expire(struct timer_list *t)
+
+ rcu_read_lock();
+
+- if (qp->q.fqdir->dead)
++ /* Paired with WRITE_ONCE() in fqdir_pre_exit(). */
++ if (READ_ONCE(qp->q.fqdir->dead))
+ goto out_rcu_unlock;
+
+ spin_lock(&qp->q.lock);
+diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
+index 0fe6c936dc54a..276a3b7b0e9c1 100644
+--- a/net/ipv4/ip_gre.c
++++ b/net/ipv4/ip_gre.c
+@@ -459,14 +459,12 @@ static void __gre_xmit(struct sk_buff *skb, struct net_device *dev,
+ __be16 proto)
+ {
+ struct ip_tunnel *tunnel = netdev_priv(dev);
+-
+- if (tunnel->parms.o_flags & TUNNEL_SEQ)
+- tunnel->o_seqno++;
++ __be16 flags = tunnel->parms.o_flags;
+
+ /* Push GRE header. */
+ gre_build_header(skb, tunnel->tun_hlen,
+- tunnel->parms.o_flags, proto, tunnel->parms.o_key,
+- htonl(tunnel->o_seqno));
++ flags, proto, tunnel->parms.o_key,
++ (flags & TUNNEL_SEQ) ? htonl(atomic_fetch_inc(&tunnel->o_seqno)) : 0);
+
+ ip_tunnel_xmit(skb, dev, tnl_params, tnl_params->protocol);
+ }
+@@ -504,7 +502,7 @@ static void gre_fb_xmit(struct sk_buff *skb, struct net_device *dev,
+ (TUNNEL_CSUM | TUNNEL_KEY | TUNNEL_SEQ);
+ gre_build_header(skb, tunnel_hlen, flags, proto,
+ tunnel_id_to_key32(tun_info->key.tun_id),
+- (flags & TUNNEL_SEQ) ? htonl(tunnel->o_seqno++) : 0);
++ (flags & TUNNEL_SEQ) ? htonl(atomic_fetch_inc(&tunnel->o_seqno)) : 0);
+
+ ip_md_tunnel_xmit(skb, dev, IPPROTO_GRE, tunnel_hlen);
+
+@@ -581,7 +579,7 @@ static void erspan_fb_xmit(struct sk_buff *skb, struct net_device *dev)
+ }
+
+ gre_build_header(skb, 8, TUNNEL_SEQ,
+- proto, 0, htonl(tunnel->o_seqno++));
++ proto, 0, htonl(atomic_fetch_inc(&tunnel->o_seqno)));
+
+ ip_md_tunnel_xmit(skb, dev, IPPROTO_GRE, tunnel_hlen);
+
+@@ -604,8 +602,9 @@ static int gre_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
+
+ key = &info->key;
+ ip_tunnel_init_flow(&fl4, IPPROTO_GRE, key->u.ipv4.dst, key->u.ipv4.src,
+- tunnel_id_to_key32(key->tun_id), key->tos, 0,
+- skb->mark, skb_get_hash(skb));
++ tunnel_id_to_key32(key->tun_id),
++ key->tos & ~INET_ECN_MASK, 0, skb->mark,
++ skb_get_hash(skb));
+ rt = ip_route_output_key(dev_net(dev), &fl4);
+ if (IS_ERR(rt))
+ return PTR_ERR(rt);
+diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
+index 9bca57ef8b838..131066d0319a2 100644
+--- a/net/ipv4/ip_output.c
++++ b/net/ipv4/ip_output.c
+@@ -162,12 +162,19 @@ int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
+ iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
+ iph->saddr = saddr;
+ iph->protocol = sk->sk_protocol;
+- if (ip_dont_fragment(sk, &rt->dst)) {
++ /* Do not bother generating IPID for small packets (eg SYNACK) */
++ if (skb->len <= IPV4_MIN_MTU || ip_dont_fragment(sk, &rt->dst)) {
+ iph->frag_off = htons(IP_DF);
+ iph->id = 0;
+ } else {
+ iph->frag_off = 0;
+- __ip_select_ident(net, iph, 1);
++ /* TCP packets here are SYNACK with fat IPv4/TCP options.
++ * Avoid using the hashed IP ident generator.
++ */
++ if (sk->sk_protocol == IPPROTO_TCP)
++ iph->id = (__force __be16)prandom_u32();
++ else
++ __ip_select_ident(net, iph, 1);
+ }
+
+ if (opt && opt->opt.optlen) {
+@@ -826,15 +833,24 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
+ /* Everything is OK. Generate! */
+ ip_fraglist_init(skb, iph, hlen, &iter);
+
+- if (iter.frag)
+- ip_options_fragment(iter.frag);
+-
+ for (;;) {
+ /* Prepare header of the next frame,
+ * before previous one went down. */
+ if (iter.frag) {
++ bool first_frag = (iter.offset == 0);
++
+ IPCB(iter.frag)->flags = IPCB(skb)->flags;
+ ip_fraglist_prepare(skb, &iter);
++ if (first_frag && IPCB(skb)->opt.optlen) {
++ /* ipcb->opt is not populated for frags
++ * coming from __ip_make_skb(),
++ * ip_options_fragment() needs optlen
++ */
++ IPCB(iter.frag)->opt.optlen =
++ IPCB(skb)->opt.optlen;
++ ip_options_fragment(iter.frag);
++ ip_send_check(iter.iph);
++ }
+ }
+
+ skb->tstamp = tstamp;
+@@ -976,7 +992,7 @@ static int __ip_append_data(struct sock *sk,
+
+ if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
+ sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
+- tskey = sk->sk_tskey++;
++ tskey = atomic_inc_return(&sk->sk_tskey) - 1;
+
+ hh_len = LL_RESERVED_SPACE(rt->dst.dev);
+
+diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
+index 2dda856ca2602..aea29d97f8dfa 100644
+--- a/net/ipv4/ipmr.c
++++ b/net/ipv4/ipmr.c
+@@ -261,7 +261,9 @@ static int __net_init ipmr_rules_init(struct net *net)
+ return 0;
+
+ err2:
++ rtnl_lock();
+ ipmr_free_table(mrt);
++ rtnl_unlock();
+ err1:
+ fib_rules_unregister(ops);
+ return err;
+diff --git a/net/ipv4/netfilter/ipt_CLUSTERIP.c b/net/ipv4/netfilter/ipt_CLUSTERIP.c
+index 8fd1aba8af31c..b518f20c9a244 100644
+--- a/net/ipv4/netfilter/ipt_CLUSTERIP.c
++++ b/net/ipv4/netfilter/ipt_CLUSTERIP.c
+@@ -520,8 +520,11 @@ static int clusterip_tg_check(const struct xt_tgchk_param *par)
+ if (IS_ERR(config))
+ return PTR_ERR(config);
+ }
+- } else if (memcmp(&config->clustermac, &cipinfo->clustermac, ETH_ALEN))
++ } else if (memcmp(&config->clustermac, &cipinfo->clustermac, ETH_ALEN)) {
++ clusterip_config_entry_put(config);
++ clusterip_config_put(config);
+ return -EINVAL;
++ }
+
+ ret = nf_ct_netns_get(par->net, par->family);
+ if (ret < 0) {
+diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c
+index 9e8100728d464..5dbd4b5505eba 100644
+--- a/net/ipv4/nexthop.c
++++ b/net/ipv4/nexthop.c
+@@ -1899,15 +1899,36 @@ static void remove_nexthop(struct net *net, struct nexthop *nh,
+ /* if any FIB entries reference this nexthop, any dst entries
+ * need to be regenerated
+ */
+-static void nh_rt_cache_flush(struct net *net, struct nexthop *nh)
++static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
++ struct nexthop *replaced_nh)
+ {
+ struct fib6_info *f6i;
++ struct nh_group *nhg;
++ int i;
+
+ if (!list_empty(&nh->fi_list))
+ rt_cache_flush(net);
+
+ list_for_each_entry(f6i, &nh->f6i_list, nh_list)
+ ipv6_stub->fib6_update_sernum(net, f6i);
++
++ /* if an IPv6 group was replaced, we have to release all old
++ * dsts to make sure all refcounts are released
++ */
++ if (!replaced_nh->is_group)
++ return;
++
++ /* new dsts must use only the new nexthop group */
++ synchronize_net();
++
++ nhg = rtnl_dereference(replaced_nh->nh_grp);
++ for (i = 0; i < nhg->num_nh; i++) {
++ struct nh_grp_entry *nhge = &nhg->nh_entries[i];
++ struct nh_info *nhi = rtnl_dereference(nhge->nh->nh_info);
++
++ if (nhi->family == AF_INET6)
++ ipv6_stub->fib6_nh_release_dsts(&nhi->fib6_nh);
++ }
+ }
+
+ static int replace_nexthop_grp(struct net *net, struct nexthop *old,
+@@ -2247,7 +2268,7 @@ static int replace_nexthop(struct net *net, struct nexthop *old,
+ err = replace_nexthop_single(net, old, new, extack);
+
+ if (!err) {
+- nh_rt_cache_flush(net, old);
++ nh_rt_cache_flush(net, old, new);
+
+ __remove_nexthop(net, new, NULL);
+ nexthop_put(new);
+@@ -2544,11 +2565,15 @@ static int nh_create_ipv6(struct net *net, struct nexthop *nh,
+ /* sets nh_dev if successful */
+ err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
+ extack);
+- if (err)
++ if (err) {
++ /* IPv6 is not enabled, don't call fib6_nh_release */
++ if (err == -EAFNOSUPPORT)
++ goto out;
+ ipv6_stub->fib6_nh_release(fib6_nh);
+- else
++ } else {
+ nh->nh_flags = fib6_nh->fib_nh_flags;
+-
++ }
++out:
+ return err;
+ }
+
+diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c
+index 1e44a43acfe2d..c4a2565da2806 100644
+--- a/net/ipv4/ping.c
++++ b/net/ipv4/ping.c
+@@ -172,16 +172,22 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident)
+ struct sock *sk = NULL;
+ struct inet_sock *isk;
+ struct hlist_nulls_node *hnode;
+- int dif = skb->dev->ifindex;
++ int dif, sdif;
+
+ if (skb->protocol == htons(ETH_P_IP)) {
++ dif = inet_iif(skb);
++ sdif = inet_sdif(skb);
+ pr_debug("try to find: num = %d, daddr = %pI4, dif = %d\n",
+ (int)ident, &ip_hdr(skb)->daddr, dif);
+ #if IS_ENABLED(CONFIG_IPV6)
+ } else if (skb->protocol == htons(ETH_P_IPV6)) {
++ dif = inet6_iif(skb);
++ sdif = inet6_sdif(skb);
+ pr_debug("try to find: num = %d, daddr = %pI6c, dif = %d\n",
+ (int)ident, &ipv6_hdr(skb)->daddr, dif);
+ #endif
++ } else {
++ return NULL;
+ }
+
+ read_lock_bh(&ping_table.lock);
+@@ -220,7 +226,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident)
+ continue;
+ }
+
+- if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif)
++ if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif &&
++ sk->sk_bound_dev_if != sdif)
+ continue;
+
+ sock_hold(sk);
+@@ -298,6 +305,7 @@ static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk,
+ struct net *net = sock_net(sk);
+ if (sk->sk_family == AF_INET) {
+ struct sockaddr_in *addr = (struct sockaddr_in *) uaddr;
++ u32 tb_id = RT_TABLE_LOCAL;
+ int chk_addr_ret;
+
+ if (addr_len < sizeof(*addr))
+@@ -313,8 +321,10 @@ static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk,
+
+ if (addr->sin_addr.s_addr == htonl(INADDR_ANY))
+ chk_addr_ret = RTN_LOCAL;
+- else
+- chk_addr_ret = inet_addr_type(net, addr->sin_addr.s_addr);
++ else {
++ tb_id = l3mdev_fib_table_by_index(net, sk->sk_bound_dev_if) ? : tb_id;
++ chk_addr_ret = inet_addr_type_table(net, addr->sin_addr.s_addr, tb_id);
++ }
+
+ if ((!inet_can_nonlocal_bind(net, isk) &&
+ chk_addr_ret != RTN_LOCAL) ||
+@@ -352,6 +362,14 @@ static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk,
+ return -ENODEV;
+ }
+ }
++
++ if (!dev && sk->sk_bound_dev_if) {
++ dev = dev_get_by_index_rcu(net, sk->sk_bound_dev_if);
++ if (!dev) {
++ rcu_read_unlock();
++ return -ENODEV;
++ }
++ }
+ has_addr = pingv6_ops.ipv6_chk_addr(net, &addr->sin6_addr, dev,
+ scoped);
+ rcu_read_unlock();
+diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
+index b0d3a09dc84e7..f30273afb5399 100644
+--- a/net/ipv4/proc.c
++++ b/net/ipv4/proc.c
+@@ -53,7 +53,7 @@ static int sockstat_seq_show(struct seq_file *seq, void *v)
+ struct net *net = seq->private;
+ int orphans, sockets;
+
+- orphans = percpu_counter_sum_positive(&tcp_orphan_count);
++ orphans = tcp_orphan_count_sum();
+ sockets = proto_sockets_allocated_sum_positive(&tcp_prot);
+
+ socket_seq_show(seq);
+diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
+index bb446e60cf580..b8689052079cd 100644
+--- a/net/ipv4/raw.c
++++ b/net/ipv4/raw.c
+@@ -721,6 +721,7 @@ static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+ int ret = -EINVAL;
+ int chk_addr_ret;
+
++ lock_sock(sk);
+ if (sk->sk_state != TCP_CLOSE || addr_len < sizeof(struct sockaddr_in))
+ goto out;
+
+@@ -740,7 +741,9 @@ static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+ inet->inet_saddr = 0; /* Use device */
+ sk_dst_reset(sk);
+ ret = 0;
+-out: return ret;
++out:
++ release_sock(sk);
++ return ret;
+ }
+
+ /*
+diff --git a/net/ipv4/route.c b/net/ipv4/route.c
+index d6899ab5fb39b..1db2fda228306 100644
+--- a/net/ipv4/route.c
++++ b/net/ipv4/route.c
+@@ -506,6 +506,15 @@ void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
+ }
+ EXPORT_SYMBOL(__ip_select_ident);
+
++static void ip_rt_fix_tos(struct flowi4 *fl4)
++{
++ __u8 tos = RT_FL_TOS(fl4);
++
++ fl4->flowi4_tos = tos & IPTOS_RT_MASK;
++ fl4->flowi4_scope = tos & RTO_ONLINK ?
++ RT_SCOPE_LINK : RT_SCOPE_UNIVERSE;
++}
++
+ static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
+ const struct sock *sk,
+ const struct iphdr *iph,
+@@ -831,6 +840,7 @@ static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buf
+ rt = (struct rtable *) dst;
+
+ __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
++ ip_rt_fix_tos(&fl4);
+ __ip_do_redirect(rt, skb, &fl4, true);
+ }
+
+@@ -1055,6 +1065,7 @@ static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
+ struct flowi4 fl4;
+
+ ip_rt_build_flow_key(&fl4, sk, skb);
++ ip_rt_fix_tos(&fl4);
+
+ /* Don't make lookup fail for bridged encapsulations */
+ if (skb && netif_is_any_bridge_port(skb->dev))
+@@ -1129,6 +1140,8 @@ void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
+ goto out;
+
+ new = true;
++ } else {
++ ip_rt_fix_tos(&fl4);
+ }
+
+ __ip_rt_update_pmtu((struct rtable *)xfrm_dst_path(&rt->dst), &fl4, mtu);
+@@ -1714,6 +1727,7 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
+ struct in_device *in_dev = __in_dev_get_rcu(dev);
+ unsigned int flags = RTCF_MULTICAST;
+ struct rtable *rth;
++ bool no_policy;
+ u32 itag = 0;
+ int err;
+
+@@ -1724,8 +1738,12 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
+ if (our)
+ flags |= RTCF_LOCAL;
+
++ no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY);
++ if (no_policy)
++ IPCB(skb)->flags |= IPSKB_NOPOLICY;
++
+ rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
+- IN_DEV_ORCONF(in_dev, NOPOLICY), false);
++ no_policy, false);
+ if (!rth)
+ return -ENOBUFS;
+
+@@ -1741,6 +1759,7 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
+ #endif
+ RT_CACHE_STAT_INC(in_slow_mc);
+
++ skb_dst_drop(skb);
+ skb_dst_set(skb, &rth->dst);
+ return 0;
+ }
+@@ -1783,7 +1802,7 @@ static int __mkroute_input(struct sk_buff *skb,
+ struct rtable *rth;
+ int err;
+ struct in_device *out_dev;
+- bool do_cache;
++ bool do_cache, no_policy;
+ u32 itag = 0;
+
+ /* get a working reference to the output device */
+@@ -1828,6 +1847,10 @@ static int __mkroute_input(struct sk_buff *skb,
+ }
+ }
+
++ no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY);
++ if (no_policy)
++ IPCB(skb)->flags |= IPSKB_NOPOLICY;
++
+ fnhe = find_exception(nhc, daddr);
+ if (do_cache) {
+ if (fnhe)
+@@ -1840,8 +1863,7 @@ static int __mkroute_input(struct sk_buff *skb,
+ }
+ }
+
+- rth = rt_dst_alloc(out_dev->dev, 0, res->type,
+- IN_DEV_ORCONF(in_dev, NOPOLICY),
++ rth = rt_dst_alloc(out_dev->dev, 0, res->type, no_policy,
+ IN_DEV_ORCONF(out_dev, NOXFRM));
+ if (!rth) {
+ err = -ENOBUFS;
+@@ -2216,6 +2238,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
+ struct rtable *rth;
+ struct flowi4 fl4;
+ bool do_cache = true;
++ bool no_policy;
+
+ /* IP on this device is disabled. */
+
+@@ -2333,6 +2356,10 @@ brd_input:
+ RT_CACHE_STAT_INC(in_brd);
+
+ local_input:
++ no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY);
++ if (no_policy)
++ IPCB(skb)->flags |= IPSKB_NOPOLICY;
++
+ do_cache &= res->fi && !itag;
+ if (do_cache) {
+ struct fib_nh_common *nhc = FIB_RES_NHC(*res);
+@@ -2347,7 +2374,7 @@ local_input:
+
+ rth = rt_dst_alloc(ip_rt_get_dev(net, res),
+ flags | RTCF_LOCAL, res->type,
+- IN_DEV_ORCONF(in_dev, NOPOLICY), false);
++ no_policy, false);
+ if (!rth)
+ goto e_nobufs;
+
+@@ -2609,7 +2636,6 @@ add:
+ struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
+ const struct sk_buff *skb)
+ {
+- __u8 tos = RT_FL_TOS(fl4);
+ struct fib_result res = {
+ .type = RTN_UNSPEC,
+ .fi = NULL,
+@@ -2619,9 +2645,7 @@ struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
+ struct rtable *rth;
+
+ fl4->flowi4_iif = LOOPBACK_IFINDEX;
+- fl4->flowi4_tos = tos & IPTOS_RT_MASK;
+- fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
+- RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
++ ip_rt_fix_tos(fl4);
+
+ rcu_read_lock();
+ rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
+@@ -3401,8 +3425,8 @@ static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
+ fa->fa_tos == fri.tos &&
+ fa->fa_info == res.fi &&
+ fa->fa_type == fri.type) {
+- fri.offload = fa->offload;
+- fri.trap = fa->trap;
++ fri.offload = READ_ONCE(fa->offload);
++ fri.trap = READ_ONCE(fa->trap);
+ break;
+ }
+ }
+diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c
+index 33792cf55a793..10b469aee4920 100644
+--- a/net/ipv4/syncookies.c
++++ b/net/ipv4/syncookies.c
+@@ -283,6 +283,7 @@ bool cookie_ecn_ok(const struct tcp_options_received *tcp_opt,
+ EXPORT_SYMBOL(cookie_ecn_ok);
+
+ struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
++ const struct tcp_request_sock_ops *af_ops,
+ struct sock *sk,
+ struct sk_buff *skb)
+ {
+@@ -299,6 +300,10 @@ struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
+ return NULL;
+
+ treq = tcp_rsk(req);
++
++ /* treq->af_specific might be used to perform TCP_MD5 lookup */
++ treq->af_specific = af_ops;
++
+ treq->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
+ #if IS_ENABLED(CONFIG_MPTCP)
+ treq->is_mptcp = sk_is_mptcp(sk);
+@@ -366,7 +371,8 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb)
+ goto out;
+
+ ret = NULL;
+- req = cookie_tcp_reqsk_alloc(&tcp_request_sock_ops, sk, skb);
++ req = cookie_tcp_reqsk_alloc(&tcp_request_sock_ops,
++ &tcp_request_sock_ipv4_ops, sk, skb);
+ if (!req)
+ goto out;
+
+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
+index f5c336f8b0c8e..ef68d55e09448 100644
+--- a/net/ipv4/tcp.c
++++ b/net/ipv4/tcp.c
+@@ -287,8 +287,8 @@ enum {
+ TCP_CMSG_TS = 2
+ };
+
+-struct percpu_counter tcp_orphan_count;
+-EXPORT_SYMBOL_GPL(tcp_orphan_count);
++DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
++EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
+
+ long sysctl_tcp_mem[3] __read_mostly;
+ EXPORT_SYMBOL(sysctl_tcp_mem);
+@@ -952,7 +952,7 @@ int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
+ */
+ void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb)
+ {
+- if (skb && !skb->len) {
++ if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
+ tcp_unlink_write_queue(skb, sk);
+ if (tcp_write_queue_empty(sk))
+ tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
+@@ -1663,11 +1663,13 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
+ if (!copied)
+ copied = used;
+ break;
+- } else if (used <= len) {
+- seq += used;
+- copied += used;
+- offset += used;
+ }
++ if (WARN_ON_ONCE(used > len))
++ used = len;
++ seq += used;
++ copied += used;
++ offset += used;
++
+ /* If recv_actor drops the lock (e.g. TCP splice
+ * receive) the skb pointer might be invalid when
+ * getting here: tcp_collapse might have deleted it
+@@ -1776,6 +1778,9 @@ static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
+ {
+ skb_frag_t *frag;
+
++ if (unlikely(offset_skb >= skb->len))
++ return NULL;
++
+ offset_skb -= skb_headlen(skb);
+ if ((int)offset_skb < 0 || skb_has_frag_list(skb))
+ return NULL;
+@@ -2687,11 +2692,36 @@ void tcp_shutdown(struct sock *sk, int how)
+ }
+ EXPORT_SYMBOL(tcp_shutdown);
+
++int tcp_orphan_count_sum(void)
++{
++ int i, total = 0;
++
++ for_each_possible_cpu(i)
++ total += per_cpu(tcp_orphan_count, i);
++
++ return max(total, 0);
++}
++
++static int tcp_orphan_cache;
++static struct timer_list tcp_orphan_timer;
++#define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
++
++static void tcp_orphan_update(struct timer_list *unused)
++{
++ WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
++ mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
++}
++
++static bool tcp_too_many_orphans(int shift)
++{
++ return READ_ONCE(tcp_orphan_cache) << shift > sysctl_tcp_max_orphans;
++}
++
+ bool tcp_check_oom(struct sock *sk, int shift)
+ {
+ bool too_many_orphans, out_of_socket_memory;
+
+- too_many_orphans = tcp_too_many_orphans(sk, shift);
++ too_many_orphans = tcp_too_many_orphans(shift);
+ out_of_socket_memory = tcp_out_of_memory(sk);
+
+ if (too_many_orphans)
+@@ -2800,7 +2830,7 @@ adjudge_to_death:
+ /* remove backlog if any, without releasing ownership. */
+ __release_sock(sk);
+
+- percpu_counter_inc(sk->sk_prot->orphan_count);
++ this_cpu_inc(tcp_orphan_count);
+
+ /* Have we already been destroyed by a softirq or backlog? */
+ if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
+@@ -3011,8 +3041,7 @@ int tcp_disconnect(struct sock *sk, int flags)
+ icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
+ memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
+ __sk_dst_reset(sk);
+- dst_release(sk->sk_rx_dst);
+- sk->sk_rx_dst = NULL;
++ dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL));
+ tcp_saved_syn_free(tp);
+ tp->compressed_ack = 0;
+ tp->segs_in = 0;
+@@ -4502,7 +4531,10 @@ void __init tcp_init(void)
+ sizeof_field(struct sk_buff, cb));
+
+ percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
+- percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
++
++ timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
++ mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
++
+ inet_hashinfo_init(&tcp_hashinfo);
+ inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
+ thash_entries, 21, /* one slot per 2 MB*/
+diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
+index 5f4d6f45d87f7..1cdcb4df0eb7e 100644
+--- a/net/ipv4/tcp_bpf.c
++++ b/net/ipv4/tcp_bpf.c
+@@ -138,10 +138,9 @@ int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
+ struct sk_psock *psock = sk_psock_get(sk);
+ int ret;
+
+- if (unlikely(!psock)) {
+- sk_msg_free(sk, msg);
+- return 0;
+- }
++ if (unlikely(!psock))
++ return -EPIPE;
++
+ ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
+ tcp_bpf_push_locked(sk, msg, bytes, flags, false);
+ sk_psock_put(sk, psock);
+@@ -172,6 +171,68 @@ static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
+ return ret;
+ }
+
++static int tcp_bpf_recvmsg_parser(struct sock *sk,
++ struct msghdr *msg,
++ size_t len,
++ int nonblock,
++ int flags,
++ int *addr_len)
++{
++ struct sk_psock *psock;
++ int copied;
++
++ if (unlikely(flags & MSG_ERRQUEUE))
++ return inet_recv_error(sk, msg, len, addr_len);
++
++ psock = sk_psock_get(sk);
++ if (unlikely(!psock))
++ return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
++
++ lock_sock(sk);
++msg_bytes_ready:
++ copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
++ if (!copied) {
++ long timeo;
++ int data;
++
++ if (sock_flag(sk, SOCK_DONE))
++ goto out;
++
++ if (sk->sk_err) {
++ copied = sock_error(sk);
++ goto out;
++ }
++
++ if (sk->sk_shutdown & RCV_SHUTDOWN)
++ goto out;
++
++ if (sk->sk_state == TCP_CLOSE) {
++ copied = -ENOTCONN;
++ goto out;
++ }
++
++ timeo = sock_rcvtimeo(sk, nonblock);
++ if (!timeo) {
++ copied = -EAGAIN;
++ goto out;
++ }
++
++ if (signal_pending(current)) {
++ copied = sock_intr_errno(timeo);
++ goto out;
++ }
++
++ data = tcp_msg_wait_data(sk, psock, timeo);
++ if (data && !sk_psock_queue_empty(psock))
++ goto msg_bytes_ready;
++ copied = -EAGAIN;
++ }
++out:
++ release_sock(sk);
++ sk_psock_put(sk, psock);
++ return copied;
++}
++
+ static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
+ int nonblock, int flags, int *addr_len)
+ {
+@@ -273,7 +334,7 @@ more_data:
+ cork = true;
+ psock->cork = NULL;
+ }
+- sk_msg_return(sk, msg, tosend);
++ sk_msg_return(sk, msg, msg->sg.size);
+ release_sock(sk);
+
+ ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
+@@ -313,8 +374,11 @@ more_data:
+ }
+ if (msg &&
+ msg->sg.data[msg->sg.start].page_link &&
+- msg->sg.data[msg->sg.start].length)
++ msg->sg.data[msg->sg.start].length) {
++ if (eval == __SK_REDIRECT)
++ sk_mem_charge(sk, msg->sg.size);
+ goto more_data;
++ }
+ }
+ return ret;
+ }
+@@ -464,6 +528,8 @@ enum {
+ enum {
+ TCP_BPF_BASE,
+ TCP_BPF_TX,
++ TCP_BPF_RX,
++ TCP_BPF_TXRX,
+ TCP_BPF_NUM_CFGS,
+ };
+
+@@ -475,7 +541,6 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
+ struct proto *base)
+ {
+ prot[TCP_BPF_BASE] = *base;
+- prot[TCP_BPF_BASE].unhash = sock_map_unhash;
+ prot[TCP_BPF_BASE].close = sock_map_close;
+ prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;
+ prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable;
+@@ -483,6 +548,12 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
+ prot[TCP_BPF_TX] = prot[TCP_BPF_BASE];
+ prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg;
+ prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage;
++
++ prot[TCP_BPF_RX] = prot[TCP_BPF_BASE];
++ prot[TCP_BPF_RX].recvmsg = tcp_bpf_recvmsg_parser;
++
++ prot[TCP_BPF_TXRX] = prot[TCP_BPF_TX];
++ prot[TCP_BPF_TXRX].recvmsg = tcp_bpf_recvmsg_parser;
+ }
+
+ static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
+@@ -520,6 +591,10 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
+ int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
+ int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
+
++ if (psock->progs.stream_verdict || psock->progs.skb_verdict) {
++ config = (config == TCP_BPF_TX) ? TCP_BPF_TXRX : TCP_BPF_RX;
++ }
++
+ if (restore) {
+ if (inet_csk_has_ulp(sk)) {
+ /* TLS does not have an unhash proto in SW cases,
+diff --git a/net/ipv4/tcp_cubic.c b/net/ipv4/tcp_cubic.c
+index 4a30deaa9a37f..8d2d4d652f6d4 100644
+--- a/net/ipv4/tcp_cubic.c
++++ b/net/ipv4/tcp_cubic.c
+@@ -328,8 +328,6 @@ static void cubictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
+ return;
+
+ if (tcp_in_slow_start(tp)) {
+- if (hystart && after(ack, ca->end_seq))
+- bictcp_hystart_reset(sk);
+ acked = tcp_slow_start(tp, acked);
+ if (!acked)
+ return;
+@@ -389,6 +387,9 @@ static void hystart_update(struct sock *sk, u32 delay)
+ struct bictcp *ca = inet_csk_ca(sk);
+ u32 threshold;
+
++ if (after(tp->snd_una, ca->end_seq))
++ bictcp_hystart_reset(sk);
++
+ if (hystart_detect & HYSTART_ACK_TRAIN) {
+ u32 now = bictcp_clock_us(sk);
+
+diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
+index 141e85e6422b1..dfd32cd3b95e7 100644
+--- a/net/ipv4/tcp_input.c
++++ b/net/ipv4/tcp_input.c
+@@ -1652,6 +1652,8 @@ static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
+ (mss != tcp_skb_seglen(skb)))
+ goto out;
+
++ if (!tcp_skb_can_collapse(prev, skb))
++ goto out;
+ len = skb->len;
+ pcount = tcp_skb_pcount(skb);
+ if (tcp_skb_shift(prev, skb, pcount, len))
+@@ -3858,7 +3860,8 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
+ tcp_process_tlp_ack(sk, ack, flag);
+
+ if (tcp_ack_is_dubious(sk, flag)) {
+- if (!(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP))) {
++ if (!(flag & (FLAG_SND_UNA_ADVANCED |
++ FLAG_NOT_DUP | FLAG_DSACKING_ACK))) {
+ num_dupack = 1;
+ /* Consider if pure acks were aggregated in tcp_add_backlog() */
+ if (!(flag & FLAG_DATA))
+@@ -5418,7 +5421,17 @@ static void tcp_new_space(struct sock *sk)
+ INDIRECT_CALL_1(sk->sk_write_space, sk_stream_write_space, sk);
+ }
+
+-static void tcp_check_space(struct sock *sk)
++/* Caller made space either from:
++ * 1) Freeing skbs in rtx queues (after tp->snd_una has advanced)
++ * 2) Sent skbs from output queue (and thus advancing tp->snd_nxt)
++ *
++ * We might be able to generate EPOLLOUT to the application if:
++ * 1) Space consumed in output/rtx queues is below sk->sk_sndbuf/2
++ * 2) notsent amount (tp->write_seq - tp->snd_nxt) became
++ * small enough that tcp_stream_memory_free() decides it
++ * is time to generate EPOLLOUT.
++ */
++void tcp_check_space(struct sock *sk)
+ {
+ /* pairs with tcp_poll() */
+ smp_mb();
+@@ -5770,7 +5783,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
+ trace_tcp_probe(sk, skb);
+
+ tcp_mstamp_refresh(tp);
+- if (unlikely(!sk->sk_rx_dst))
++ if (unlikely(!rcu_access_pointer(sk->sk_rx_dst)))
+ inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
+ /*
+ * Header prediction.
+diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
+index 5b8ce65dfc067..0fe9461647da5 100644
+--- a/net/ipv4/tcp_ipv4.c
++++ b/net/ipv4/tcp_ipv4.c
+@@ -1698,16 +1698,19 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
+ struct sock *rsk;
+
+ if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
+- struct dst_entry *dst = sk->sk_rx_dst;
++ struct dst_entry *dst;
++
++ dst = rcu_dereference_protected(sk->sk_rx_dst,
++ lockdep_sock_is_held(sk));
+
+ sock_rps_save_rxhash(sk, skb);
+ sk_mark_napi_id(sk, skb);
+ if (dst) {
+- if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
++ if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
+ !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
+ dst, 0)) {
++ RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
+ dst_release(dst);
+- sk->sk_rx_dst = NULL;
+ }
+ }
+ tcp_rcv_established(sk, skb);
+@@ -1783,12 +1786,12 @@ int tcp_v4_early_demux(struct sk_buff *skb)
+ skb->sk = sk;
+ skb->destructor = sock_edemux;
+ if (sk_fullsock(sk)) {
+- struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
++ struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
+
+ if (dst)
+ dst = dst_check(dst, 0);
+ if (dst &&
+- inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
++ sk->sk_rx_dst_ifindex == skb->skb_iif)
+ skb_dst_set_noref(skb, dst);
+ }
+ }
+@@ -2200,8 +2203,8 @@ void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
+ struct dst_entry *dst = skb_dst(skb);
+
+ if (dst && dst_hold_safe(dst)) {
+- sk->sk_rx_dst = dst;
+- inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
++ rcu_assign_pointer(sk->sk_rx_dst, dst);
++ sk->sk_rx_dst_ifindex = skb->skb_iif;
+ }
+ }
+ EXPORT_SYMBOL(inet_sk_rx_dst_set);
+diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
+index 0a4f3f16140ad..13783fc58e030 100644
+--- a/net/ipv4/tcp_minisocks.c
++++ b/net/ipv4/tcp_minisocks.c
+@@ -538,7 +538,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
+ newtp->tsoffset = treq->ts_off;
+ #ifdef CONFIG_TCP_MD5SIG
+ newtp->md5sig_info = NULL; /*XXX*/
+- if (newtp->af_specific->md5_lookup(sk, newsk))
++ if (treq->af_specific->req_md5_lookup(sk, req_to_sk(req)))
+ newtp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
+ #endif
+ if (skb->len >= TCP_MSS_DEFAULT + newtp->tcp_header_len)
+diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
+index 6d72f3ea48c4e..df413282fa2e5 100644
+--- a/net/ipv4/tcp_output.c
++++ b/net/ipv4/tcp_output.c
+@@ -82,6 +82,7 @@ static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
+
+ NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
+ tcp_skb_pcount(skb));
++ tcp_check_space(sk);
+ }
+
+ /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
+@@ -1562,7 +1563,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
+ return -ENOMEM;
+ }
+
+- if (skb_unclone(skb, gfp))
++ if (skb_unclone_keeptruesize(skb, gfp))
+ return -ENOMEM;
+
+ /* Get a new skb... force flag on. */
+@@ -1672,7 +1673,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
+ {
+ u32 delta_truesize;
+
+- if (skb_unclone(skb, GFP_ATOMIC))
++ if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
+ return -ENOMEM;
+
+ delta_truesize = __pskb_trim_head(skb, len);
+@@ -3184,7 +3185,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
+ cur_mss, GFP_ATOMIC))
+ return -ENOMEM; /* We'll try again later. */
+ } else {
+- if (skb_unclone(skb, GFP_ATOMIC))
++ if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
+ return -ENOMEM;
+
+ diff = tcp_skb_pcount(skb);
+@@ -3734,6 +3735,7 @@ static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
+ */
+ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
+ {
++ struct inet_connection_sock *icsk = inet_csk(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ struct tcp_fastopen_request *fo = tp->fastopen_req;
+ int space, err = 0;
+@@ -3748,8 +3750,10 @@ static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
+ * private TCP options. The cost is reduced data space in SYN :(
+ */
+ tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
++ /* Sync mss_cache after updating the mss_clamp */
++ tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
+
+- space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
++ space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) -
+ MAX_TCP_OPTION_SPACE;
+
+ space = min_t(size_t, space, fo->size);
+diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c
+index 0de6935659635..6ab197928abbc 100644
+--- a/net/ipv4/tcp_rate.c
++++ b/net/ipv4/tcp_rate.c
+@@ -73,26 +73,31 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb)
+ *
+ * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is
+ * called multiple times. We favor the information from the most recently
+- * sent skb, i.e., the skb with the highest prior_delivered count.
++ * sent skb, i.e., the skb with the most recently sent time and the highest
++ * sequence.
+ */
+ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
+ struct rate_sample *rs)
+ {
+ struct tcp_sock *tp = tcp_sk(sk);
+ struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
++ u64 tx_tstamp;
+
+ if (!scb->tx.delivered_mstamp)
+ return;
+
++ tx_tstamp = tcp_skb_timestamp_us(skb);
+ if (!rs->prior_delivered ||
+- after(scb->tx.delivered, rs->prior_delivered)) {
++ tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp,
++ scb->end_seq, rs->last_end_seq)) {
+ rs->prior_delivered = scb->tx.delivered;
+ rs->prior_mstamp = scb->tx.delivered_mstamp;
+ rs->is_app_limited = scb->tx.is_app_limited;
+ rs->is_retrans = scb->sacked & TCPCB_RETRANS;
++ rs->last_end_seq = scb->end_seq;
+
+ /* Record send time of most recently ACKed packet: */
+- tp->first_tx_mstamp = tcp_skb_timestamp_us(skb);
++ tp->first_tx_mstamp = tx_tstamp;
+ /* Find the duration of the "send phase" of this window: */
+ rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,
+ scb->tx.first_tx_mstamp);
+diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
+index 2fffcf2b54f3f..835b9d6e4e686 100644
+--- a/net/ipv4/udp.c
++++ b/net/ipv4/udp.c
+@@ -917,7 +917,7 @@ static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+- if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) {
++ if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+@@ -1808,6 +1808,17 @@ int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
+ skb = skb_recv_udp(sk, 0, 1, &err);
+ if (!skb)
+ return err;
++
++ if (udp_lib_checksum_complete(skb)) {
++ __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
++ IS_UDPLITE(sk));
++ __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
++ IS_UDPLITE(sk));
++ atomic_inc(&sk->sk_drops);
++ kfree_skb(skb);
++ continue;
++ }
++
+ used = recv_actor(desc, skb, 0, skb->len);
+ if (used <= 0) {
+ if (!copied)
+@@ -2240,7 +2251,7 @@ bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
+ struct dst_entry *old;
+
+ if (dst_hold_safe(dst)) {
+- old = xchg(&sk->sk_rx_dst, dst);
++ old = xchg((__force struct dst_entry **)&sk->sk_rx_dst, dst);
+ dst_release(old);
+ return old != dst;
+ }
+@@ -2430,7 +2441,7 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
+ struct dst_entry *dst = skb_dst(skb);
+ int ret;
+
+- if (unlikely(sk->sk_rx_dst != dst))
++ if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
+ udp_sk_rx_dst_set(sk, dst);
+
+ ret = udp_unicast_rcv_skb(sk, skb, uh);
+@@ -2589,7 +2600,7 @@ int udp_v4_early_demux(struct sk_buff *skb)
+
+ skb->sk = sk;
+ skb->destructor = sock_efree;
+- dst = READ_ONCE(sk->sk_rx_dst);
++ dst = rcu_dereference(sk->sk_rx_dst);
+
+ if (dst)
+ dst = dst_check(dst, 0);
+@@ -3065,7 +3076,7 @@ int udp4_seq_show(struct seq_file *seq, void *v)
+ {
+ seq_setwidth(seq, 127);
+ if (v == SEQ_START_TOKEN)
+- seq_puts(seq, " sl local_address rem_address st tx_queue "
++ seq_puts(seq, " sl local_address rem_address st tx_queue "
+ "rx_queue tr tm->when retrnsmt uid timeout "
+ "inode ref pointer drops");
+ else {
+diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c
+index b91003538d87a..bc3a043a5d5c7 100644
+--- a/net/ipv4/udp_tunnel_nic.c
++++ b/net/ipv4/udp_tunnel_nic.c
+@@ -846,7 +846,7 @@ udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
+ list_for_each_entry(node, &info->shared->devices, list)
+ if (node->dev == dev)
+ break;
+- if (node->dev != dev)
++ if (list_entry_is_head(node, &info->shared->devices, list))
+ return;
+
+ list_del(&node->list);
+diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
+index c6a90b7bbb70e..3a8838b79bb69 100644
+--- a/net/ipv6/addrconf.c
++++ b/net/ipv6/addrconf.c
+@@ -552,7 +552,7 @@ static int inet6_netconf_fill_devconf(struct sk_buff *skb, int ifindex,
+ #ifdef CONFIG_IPV6_MROUTE
+ if ((all || type == NETCONFA_MC_FORWARDING) &&
+ nla_put_s32(skb, NETCONFA_MC_FORWARDING,
+- devconf->mc_forwarding) < 0)
++ atomic_read(&devconf->mc_forwarding)) < 0)
+ goto nla_put_failure;
+ #endif
+ if ((all || type == NETCONFA_PROXY_NEIGH) &&
+@@ -1837,8 +1837,8 @@ out:
+ }
+ EXPORT_SYMBOL(ipv6_dev_get_saddr);
+
+-int __ipv6_get_lladdr(struct inet6_dev *idev, struct in6_addr *addr,
+- u32 banned_flags)
++static int __ipv6_get_lladdr(struct inet6_dev *idev, struct in6_addr *addr,
++ u32 banned_flags)
+ {
+ struct inet6_ifaddr *ifp;
+ int err = -EADDRNOTAVAIL;
+@@ -2587,7 +2587,7 @@ int addrconf_prefix_rcv_add_addr(struct net *net, struct net_device *dev,
+ __u32 valid_lft, u32 prefered_lft)
+ {
+ struct inet6_ifaddr *ifp = ipv6_get_ifaddr(net, addr, dev, 1);
+- int create = 0;
++ int create = 0, update_lft = 0;
+
+ if (!ifp && valid_lft) {
+ int max_addresses = in6_dev->cnf.max_addresses;
+@@ -2631,19 +2631,32 @@ int addrconf_prefix_rcv_add_addr(struct net *net, struct net_device *dev,
+ unsigned long now;
+ u32 stored_lft;
+
+- /* Update lifetime (RFC4862 5.5.3 e)
+- * We deviate from RFC4862 by honoring all Valid Lifetimes to
+- * improve the reaction of SLAAC to renumbering events
+- * (draft-gont-6man-slaac-renum-06, Section 4.2)
+- */
++ /* update lifetime (RFC2462 5.5.3 e) */
+ spin_lock_bh(&ifp->lock);
+ now = jiffies;
+ if (ifp->valid_lft > (now - ifp->tstamp) / HZ)
+ stored_lft = ifp->valid_lft - (now - ifp->tstamp) / HZ;
+ else
+ stored_lft = 0;
+-
+ if (!create && stored_lft) {
++ const u32 minimum_lft = min_t(u32,
++ stored_lft, MIN_VALID_LIFETIME);
++ valid_lft = max(valid_lft, minimum_lft);
++
++ /* RFC4862 Section 5.5.3e:
++ * "Note that the preferred lifetime of the
++ * corresponding address is always reset to
++ * the Preferred Lifetime in the received
++ * Prefix Information option, regardless of
++ * whether the valid lifetime is also reset or
++ * ignored."
++ *
++ * So we should always update prefered_lft here.
++ */
++ update_lft = 1;
++ }
++
++ if (update_lft) {
+ ifp->valid_lft = valid_lft;
+ ifp->prefered_lft = prefered_lft;
+ ifp->tstamp = now;
+@@ -3110,6 +3123,9 @@ static void add_v4_addrs(struct inet6_dev *idev)
+ memcpy(&addr.s6_addr32[3], idev->dev->dev_addr + offset, 4);
+
+ if (idev->dev->flags&IFF_POINTOPOINT) {
++ if (idev->cnf.addr_gen_mode == IN6_ADDR_GEN_MODE_NONE)
++ return;
++
+ addr.s6_addr32[0] = htonl(0xfe800000);
+ scope = IFA_LINK;
+ plen = 64;
+@@ -3714,6 +3730,7 @@ static int addrconf_ifdown(struct net_device *dev, bool unregister)
+ struct inet6_dev *idev;
+ struct inet6_ifaddr *ifa, *tmp;
+ bool keep_addr = false;
++ bool was_ready;
+ int state, i;
+
+ ASSERT_RTNL();
+@@ -3779,7 +3796,10 @@ restart:
+
+ addrconf_del_rs_timer(idev);
+
+- /* Step 2: clear flags for stateless addrconf */
++ /* Step 2: clear flags for stateless addrconf, repeated down
++ * detection
++ */
++ was_ready = idev->if_flags & IF_READY;
+ if (!unregister)
+ idev->if_flags &= ~(IF_RS_SENT|IF_RA_RCVD|IF_READY);
+
+@@ -3853,7 +3873,7 @@ restart:
+ if (unregister) {
+ ipv6_ac_destroy_dev(idev);
+ ipv6_mc_destroy_dev(idev);
+- } else {
++ } else if (was_ready) {
+ ipv6_mc_down(idev);
+ }
+
+@@ -4980,6 +5000,7 @@ static int inet6_fill_ifaddr(struct sk_buff *skb, struct inet6_ifaddr *ifa,
+ nla_put_s32(skb, IFA_TARGET_NETNSID, args->netnsid))
+ goto error;
+
++ spin_lock_bh(&ifa->lock);
+ if (!((ifa->flags&IFA_F_PERMANENT) &&
+ (ifa->prefered_lft == INFINITY_LIFE_TIME))) {
+ preferred = ifa->prefered_lft;
+@@ -5001,6 +5022,7 @@ static int inet6_fill_ifaddr(struct sk_buff *skb, struct inet6_ifaddr *ifa,
+ preferred = INFINITY_LIFE_TIME;
+ valid = INFINITY_LIFE_TIME;
+ }
++ spin_unlock_bh(&ifa->lock);
+
+ if (!ipv6_addr_any(&ifa->peer_addr)) {
+ if (nla_put_in6_addr(skb, IFA_LOCAL, &ifa->addr) < 0 ||
+@@ -5515,7 +5537,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf,
+ array[DEVCONF_USE_OPTIMISTIC] = cnf->use_optimistic;
+ #endif
+ #ifdef CONFIG_IPV6_MROUTE
+- array[DEVCONF_MC_FORWARDING] = cnf->mc_forwarding;
++ array[DEVCONF_MC_FORWARDING] = atomic_read(&cnf->mc_forwarding);
+ #endif
+ array[DEVCONF_DISABLE_IPV6] = cnf->disable_ipv6;
+ array[DEVCONF_ACCEPT_DAD] = cnf->accept_dad;
+diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
+index b5878bb8e419d..dab4a047590b7 100644
+--- a/net/ipv6/af_inet6.c
++++ b/net/ipv6/af_inet6.c
+@@ -521,31 +521,32 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr,
+ sin->sin6_family = AF_INET6;
+ sin->sin6_flowinfo = 0;
+ sin->sin6_scope_id = 0;
++ lock_sock(sk);
+ if (peer) {
+- if (!inet->inet_dport)
+- return -ENOTCONN;
+- if (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_SYN_SENT)) &&
+- peer == 1)
++ if (!inet->inet_dport ||
++ (((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_SYN_SENT)) &&
++ peer == 1)) {
++ release_sock(sk);
+ return -ENOTCONN;
++ }
+ sin->sin6_port = inet->inet_dport;
+ sin->sin6_addr = sk->sk_v6_daddr;
+ if (np->sndflow)
+ sin->sin6_flowinfo = np->flow_label;
+- BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
+- CGROUP_INET6_GETPEERNAME,
+- NULL);
++ BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin,
++ CGROUP_INET6_GETPEERNAME);
+ } else {
+ if (ipv6_addr_any(&sk->sk_v6_rcv_saddr))
+ sin->sin6_addr = np->saddr;
+ else
+ sin->sin6_addr = sk->sk_v6_rcv_saddr;
+ sin->sin6_port = inet->inet_sport;
+- BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin,
+- CGROUP_INET6_GETSOCKNAME,
+- NULL);
++ BPF_CGROUP_RUN_SA_PROG(sk, (struct sockaddr *)sin,
++ CGROUP_INET6_GETSOCKNAME);
+ }
+ sin->sin6_scope_id = ipv6_iface_scope_id(&sin->sin6_addr,
+ sk->sk_bound_dev_if);
++ release_sock(sk);
+ return sizeof(*sin);
+ }
+ EXPORT_SYMBOL(inet6_getname);
+@@ -1025,6 +1026,7 @@ static const struct ipv6_stub ipv6_stub_impl = {
+ .ip6_mtu_from_fib6 = ip6_mtu_from_fib6,
+ .fib6_nh_init = fib6_nh_init,
+ .fib6_nh_release = fib6_nh_release,
++ .fib6_nh_release_dsts = fib6_nh_release_dsts,
+ .fib6_update_sernum = fib6_update_sernum_stub,
+ .fib6_rt_update = fib6_rt_update,
+ .ip6_del_rt = ip6_del_rt,
+diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
+index ed2f061b87685..6219d97cac7a3 100644
+--- a/net/ipv6/esp6.c
++++ b/net/ipv6/esp6.c
+@@ -491,6 +491,10 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
+ return err;
+ }
+
++ if (ALIGN(tailen, L1_CACHE_BYTES) > PAGE_SIZE ||
++ ALIGN(skb->data_len, L1_CACHE_BYTES) > PAGE_SIZE)
++ goto cow;
++
+ if (!skb_cloned(skb)) {
+ if (tailen <= skb_tailroom(skb)) {
+ nfrags = 1;
+@@ -708,7 +712,7 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
+ struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
+ u32 padto;
+
+- padto = min(x->tfcpad, __xfrm_state_mtu(x, dst->child_mtu_cached));
++ padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached));
+ if (skb->len < padto)
+ esp.tfclen = padto - skb->len;
+ }
+@@ -808,6 +812,11 @@ int esp6_input_done2(struct sk_buff *skb, int err)
+ struct tcphdr *th;
+
+ offset = ipv6_skip_exthdr(skb, offset, &nexthdr, &frag_off);
++ if (offset == -1) {
++ err = -EINVAL;
++ goto out;
++ }
++
+ uh = (void *)(skb->data + offset);
+ th = (void *)(skb->data + offset);
+ hdr_len += offset;
+diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c
+index a349d47980776..302170882382a 100644
+--- a/net/ipv6/esp6_offload.c
++++ b/net/ipv6/esp6_offload.c
+@@ -198,6 +198,9 @@ static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
+ ipv6_skip_exthdr(skb, 0, &proto, &frag);
+ }
+
++ if (proto == IPPROTO_IPIP)
++ skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
++
+ __skb_pull(skb, skb_transport_offset(skb));
+ ops = rcu_dereference(inet6_offloads[proto]);
+ if (likely(ops && ops->callbacks.gso_segment))
+diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
+index 40f3e4f9f33a2..dcedfe29d9d93 100644
+--- a/net/ipv6/fib6_rules.c
++++ b/net/ipv6/fib6_rules.c
+@@ -267,6 +267,7 @@ INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
+ }
+
+ INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
++ int flags,
+ struct fib_lookup_arg *arg)
+ {
+ struct fib6_result *res = arg->result;
+@@ -294,8 +295,7 @@ INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
+ return false;
+
+ suppress_route:
+- if (!(arg->flags & FIB_LOOKUP_NOREF))
+- ip6_rt_put(rt);
++ ip6_rt_put_flags(rt, flags);
+ return true;
+ }
+
+diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
+index a7c31ab67c5d6..96c5cc0f30ceb 100644
+--- a/net/ipv6/icmp.c
++++ b/net/ipv6/icmp.c
+@@ -57,6 +57,7 @@
+ #include <net/protocol.h>
+ #include <net/raw.h>
+ #include <net/rawv6.h>
++#include <net/seg6.h>
+ #include <net/transp_v6.h>
+ #include <net/ip6_route.h>
+ #include <net/addrconf.h>
+@@ -820,6 +821,7 @@ out_bh_enable:
+
+ void icmpv6_notify(struct sk_buff *skb, u8 type, u8 code, __be32 info)
+ {
++ struct inet6_skb_parm *opt = IP6CB(skb);
+ const struct inet6_protocol *ipprot;
+ int inner_offset;
+ __be16 frag_off;
+@@ -829,6 +831,8 @@ void icmpv6_notify(struct sk_buff *skb, u8 type, u8 code, __be32 info)
+ if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
+ goto out;
+
++ seg6_icmp_srh(skb, opt);
++
+ nexthdr = ((struct ipv6hdr *)skb->data)->nexthdr;
+ if (ipv6_ext_hdr(nexthdr)) {
+ /* now skip over extension headers */
+@@ -853,7 +857,7 @@ void icmpv6_notify(struct sk_buff *skb, u8 type, u8 code, __be32 info)
+
+ ipprot = rcu_dereference(inet6_protos[nexthdr]);
+ if (ipprot && ipprot->err_handler)
+- ipprot->err_handler(skb, NULL, type, code, inner_offset, info);
++ ipprot->err_handler(skb, opt, type, code, inner_offset, info);
+
+ raw6_icmp_error(skb, nexthdr, type, code, inner_offset, info);
+ return;
+diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
+index 67c9114835c84..40203255ed88b 100644
+--- a/net/ipv6/inet6_hashtables.c
++++ b/net/ipv6/inet6_hashtables.c
+@@ -308,7 +308,7 @@ not_unique:
+ return -EADDRNOTAVAIL;
+ }
+
+-static u32 inet6_sk_port_offset(const struct sock *sk)
++static u64 inet6_sk_port_offset(const struct sock *sk)
+ {
+ const struct inet_sock *inet = inet_sk(sk);
+
+@@ -320,7 +320,7 @@ static u32 inet6_sk_port_offset(const struct sock *sk)
+ int inet6_hash_connect(struct inet_timewait_death_row *death_row,
+ struct sock *sk)
+ {
+- u32 port_offset = 0;
++ u64 port_offset = 0;
+
+ if (!inet_sk(sk)->inet_num)
+ port_offset = inet6_sk_port_offset(sk);
+@@ -333,11 +333,8 @@ int inet6_hash(struct sock *sk)
+ {
+ int err = 0;
+
+- if (sk->sk_state != TCP_CLOSE) {
+- local_bh_disable();
++ if (sk->sk_state != TCP_CLOSE)
+ err = __inet_hash(sk, NULL);
+- local_bh_enable();
+- }
+
+ return err;
+ }
+diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
+index 0371d2c141455..a506e57c4032a 100644
+--- a/net/ipv6/ip6_fib.c
++++ b/net/ipv6/ip6_fib.c
+@@ -111,7 +111,7 @@ void fib6_update_sernum(struct net *net, struct fib6_info *f6i)
+ fn = rcu_dereference_protected(f6i->fib6_node,
+ lockdep_is_held(&f6i->fib6_table->tb6_lock));
+ if (fn)
+- fn->fn_sernum = fib6_new_sernum(net);
++ WRITE_ONCE(fn->fn_sernum, fib6_new_sernum(net));
+ }
+
+ /*
+@@ -589,12 +589,13 @@ static int fib6_dump_table(struct fib6_table *table, struct sk_buff *skb,
+ spin_unlock_bh(&table->tb6_lock);
+ if (res > 0) {
+ cb->args[4] = 1;
+- cb->args[5] = w->root->fn_sernum;
++ cb->args[5] = READ_ONCE(w->root->fn_sernum);
+ }
+ } else {
+- if (cb->args[5] != w->root->fn_sernum) {
++ int sernum = READ_ONCE(w->root->fn_sernum);
++ if (cb->args[5] != sernum) {
+ /* Begin at the root if the tree changed */
+- cb->args[5] = w->root->fn_sernum;
++ cb->args[5] = sernum;
+ w->state = FWS_INIT;
+ w->node = w->root;
+ w->skip = w->count;
+@@ -1344,7 +1345,7 @@ static void __fib6_update_sernum_upto_root(struct fib6_info *rt,
+ /* paired with smp_rmb() in fib6_get_cookie_safe() */
+ smp_wmb();
+ while (fn) {
+- fn->fn_sernum = sernum;
++ WRITE_ONCE(fn->fn_sernum, sernum);
+ fn = rcu_dereference_protected(fn->parent,
+ lockdep_is_held(&rt->fib6_table->tb6_lock));
+ }
+@@ -2173,8 +2174,8 @@ static int fib6_clean_node(struct fib6_walker *w)
+ };
+
+ if (c->sernum != FIB6_NO_SERNUM_CHANGE &&
+- w->node->fn_sernum != c->sernum)
+- w->node->fn_sernum = c->sernum;
++ READ_ONCE(w->node->fn_sernum) != c->sernum)
++ WRITE_ONCE(w->node->fn_sernum, c->sernum);
+
+ if (!c->func) {
+ WARN_ON_ONCE(c->sernum == FIB6_NO_SERNUM_CHANGE);
+@@ -2542,7 +2543,7 @@ static void ipv6_route_seq_setup_walk(struct ipv6_route_iter *iter,
+ iter->w.state = FWS_INIT;
+ iter->w.node = iter->w.root;
+ iter->w.args = iter;
+- iter->sernum = iter->w.root->fn_sernum;
++ iter->sernum = READ_ONCE(iter->w.root->fn_sernum);
+ INIT_LIST_HEAD(&iter->w.lh);
+ fib6_walker_link(net, &iter->w);
+ }
+@@ -2570,8 +2571,10 @@ static struct fib6_table *ipv6_route_seq_next_table(struct fib6_table *tbl,
+
+ static void ipv6_route_check_sernum(struct ipv6_route_iter *iter)
+ {
+- if (iter->sernum != iter->w.root->fn_sernum) {
+- iter->sernum = iter->w.root->fn_sernum;
++ int sernum = READ_ONCE(iter->w.root->fn_sernum);
++
++ if (iter->sernum != sernum) {
++ iter->sernum = sernum;
+ iter->w.state = FWS_INIT;
+ iter->w.node = iter->w.root;
+ WARN_ON(iter->w.skip);
+diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c
+index aa673a6a7e432..ceb85c67ce395 100644
+--- a/net/ipv6/ip6_flowlabel.c
++++ b/net/ipv6/ip6_flowlabel.c
+@@ -450,8 +450,10 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq,
+ err = -EINVAL;
+ goto done;
+ }
+- if (fl_shared_exclusive(fl) || fl->opt)
++ if (fl_shared_exclusive(fl) || fl->opt) {
++ WRITE_ONCE(sock_net(sk)->ipv6.flowlabel_has_excl, 1);
+ static_branch_deferred_inc(&ipv6_flowlabel_exclusive);
++ }
+ return fl;
+
+ done:
+diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
+index 3ad201d372d88..a817ac6d97598 100644
+--- a/net/ipv6/ip6_gre.c
++++ b/net/ipv6/ip6_gre.c
+@@ -724,6 +724,7 @@ static netdev_tx_t __gre6_xmit(struct sk_buff *skb,
+ {
+ struct ip6_tnl *tunnel = netdev_priv(dev);
+ __be16 protocol;
++ __be16 flags;
+
+ if (dev->type == ARPHRD_ETHER)
+ IPCB(skb)->flags = 0;
+@@ -733,16 +734,13 @@ static netdev_tx_t __gre6_xmit(struct sk_buff *skb,
+ else
+ fl6->daddr = tunnel->parms.raddr;
+
+- if (skb_cow_head(skb, dev->needed_headroom ?: tunnel->hlen))
+- return -ENOMEM;
+-
+ /* Push GRE header. */
+ protocol = (dev->type == ARPHRD_ETHER) ? htons(ETH_P_TEB) : proto;
+
+ if (tunnel->parms.collect_md) {
+ struct ip_tunnel_info *tun_info;
+ const struct ip_tunnel_key *key;
+- __be16 flags;
++ int tun_hlen;
+
+ tun_info = skb_tunnel_info_txcheck(skb);
+ if (IS_ERR(tun_info) ||
+@@ -755,25 +753,32 @@ static netdev_tx_t __gre6_xmit(struct sk_buff *skb,
+ fl6->daddr = key->u.ipv6.dst;
+ fl6->flowlabel = key->label;
+ fl6->flowi6_uid = sock_net_uid(dev_net(dev), NULL);
++ fl6->fl6_gre_key = tunnel_id_to_key32(key->tun_id);
+
+ dsfield = key->tos;
+ flags = key->tun_flags &
+ (TUNNEL_CSUM | TUNNEL_KEY | TUNNEL_SEQ);
+- tunnel->tun_hlen = gre_calc_hlen(flags);
++ tun_hlen = gre_calc_hlen(flags);
+
+- gre_build_header(skb, tunnel->tun_hlen,
++ if (skb_cow_head(skb, dev->needed_headroom ?: tun_hlen + tunnel->encap_hlen))
++ return -ENOMEM;
++
++ gre_build_header(skb, tun_hlen,
+ flags, protocol,
+ tunnel_id_to_key32(tun_info->key.tun_id),
+- (flags & TUNNEL_SEQ) ? htonl(tunnel->o_seqno++)
++ (flags & TUNNEL_SEQ) ? htonl(atomic_fetch_inc(&tunnel->o_seqno))
+ : 0);
+
+ } else {
+- if (tunnel->parms.o_flags & TUNNEL_SEQ)
+- tunnel->o_seqno++;
++ if (skb_cow_head(skb, dev->needed_headroom ?: tunnel->hlen))
++ return -ENOMEM;
++
++ flags = tunnel->parms.o_flags;
+
+- gre_build_header(skb, tunnel->tun_hlen, tunnel->parms.o_flags,
++ gre_build_header(skb, tunnel->tun_hlen, flags,
+ protocol, tunnel->parms.o_key,
+- htonl(tunnel->o_seqno));
++ (flags & TUNNEL_SEQ) ? htonl(atomic_fetch_inc(&tunnel->o_seqno))
++ : 0);
+ }
+
+ return ip6_tnl_xmit(skb, dev, dsfield, fl6, encap_limit, pmtu,
+@@ -990,6 +995,7 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb,
+ fl6.daddr = key->u.ipv6.dst;
+ fl6.flowlabel = key->label;
+ fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL);
++ fl6.fl6_gre_key = tunnel_id_to_key32(key->tun_id);
+
+ dsfield = key->tos;
+ if (!(tun_info->key.tun_flags & TUNNEL_ERSPAN_OPT))
+@@ -1050,7 +1056,7 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb,
+ /* Push GRE header. */
+ proto = (t->parms.erspan_ver == 1) ? htons(ETH_P_ERSPAN)
+ : htons(ETH_P_ERSPAN2);
+- gre_build_header(skb, 8, TUNNEL_SEQ, proto, 0, htonl(t->o_seqno++));
++ gre_build_header(skb, 8, TUNNEL_SEQ, proto, 0, htonl(atomic_fetch_inc(&t->o_seqno)));
+
+ /* TooBig packet may have updated dst->dev's mtu */
+ if (!t->parms.collect_md && dst && dst_mtu(dst) > dst->dev->mtu)
+@@ -1098,6 +1104,7 @@ static void ip6gre_tnl_link_config_common(struct ip6_tnl *t)
+ fl6->flowi6_oif = p->link;
+ fl6->flowlabel = 0;
+ fl6->flowi6_proto = IPPROTO_GRE;
++ fl6->fl6_gre_key = t->parms.o_key;
+
+ if (!(p->flags&IP6_TNL_F_USE_ORIG_TCLASS))
+ fl6->flowlabel |= IPV6_TCLASS_MASK & p->flowinfo;
+@@ -1544,7 +1551,7 @@ static void ip6gre_fb_tunnel_init(struct net_device *dev)
+ static struct inet6_protocol ip6gre_protocol __read_mostly = {
+ .handler = gre_rcv,
+ .err_handler = ip6gre_err,
+- .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
++ .flags = INET6_PROTO_FINAL,
+ };
+
+ static void ip6gre_destroy_tunnels(struct net *net, struct list_head *head)
+diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c
+index 80256717868e6..d4b1e2c5aa76d 100644
+--- a/net/ipv6/ip6_input.c
++++ b/net/ipv6/ip6_input.c
+@@ -508,7 +508,7 @@ int ip6_mc_input(struct sk_buff *skb)
+ /*
+ * IPv6 multicast router mode is now supported ;)
+ */
+- if (dev_net(skb->dev)->ipv6.devconf_all->mc_forwarding &&
++ if (atomic_read(&dev_net(skb->dev)->ipv6.devconf_all->mc_forwarding) &&
+ !(ipv6_addr_type(&hdr->daddr) &
+ (IPV6_ADDR_LOOPBACK|IPV6_ADDR_LINKLOCAL)) &&
+ likely(!(IP6CB(skb)->flags & IP6SKB_FORWARDED))) {
+diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
+index 1b9827ff8ccf4..172565d125704 100644
+--- a/net/ipv6/ip6_offload.c
++++ b/net/ipv6/ip6_offload.c
+@@ -114,6 +114,8 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
+ if (likely(ops && ops->callbacks.gso_segment)) {
+ skb_reset_transport_header(skb);
+ segs = ops->callbacks.gso_segment(skb, features);
++ if (!segs)
++ skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
+ }
+
+ if (IS_ERR_OR_NULL(segs))
+diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
+index 2f044a49afa8c..04c3cb4c5fec4 100644
+--- a/net/ipv6/ip6_output.c
++++ b/net/ipv6/ip6_output.c
+@@ -174,7 +174,7 @@ static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff
+ #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
+ /* Policy lookup after SNAT yielded a new policy */
+ if (skb_dst(skb)->xfrm) {
+- IPCB(skb)->flags |= IPSKB_REROUTED;
++ IP6CB(skb)->flags |= IP6SKB_REROUTED;
+ return dst_output(net, sk, skb);
+ }
+ #endif
+@@ -485,7 +485,7 @@ int ip6_forward(struct sk_buff *skb)
+ goto drop;
+
+ if (!net->ipv6.devconf_all->disable_policy &&
+- !idev->cnf.disable_policy &&
++ (!idev || !idev->cnf.disable_policy) &&
+ !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
+ __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
+ goto drop;
+@@ -1408,8 +1408,6 @@ static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
+ if (np->frag_size)
+ mtu = np->frag_size;
+ }
+- if (mtu < IPV6_MIN_MTU)
+- return -EINVAL;
+ cork->base.fragsize = mtu;
+ cork->base.gso_size = ipc6->gso_size;
+ cork->base.tx_flags = 0;
+@@ -1465,14 +1463,12 @@ static int __ip6_append_data(struct sock *sk,
+
+ if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
+ sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
+- tskey = sk->sk_tskey++;
++ tskey = atomic_inc_return(&sk->sk_tskey) - 1;
+
+ hh_len = LL_RESERVED_SPACE(rt->dst.dev);
+
+ fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
+ (opt ? opt->opt_nflen : 0);
+- maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
+- sizeof(struct frag_hdr);
+
+ headersize = sizeof(struct ipv6hdr) +
+ (opt ? opt->opt_flen + opt->opt_nflen : 0) +
+@@ -1480,6 +1476,13 @@ static int __ip6_append_data(struct sock *sk,
+ sizeof(struct frag_hdr) : 0) +
+ rt->rt6i_nfheader_len;
+
++ if (mtu <= fragheaderlen ||
++ ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr))
++ goto emsgsize;
++
++ maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
++ sizeof(struct frag_hdr);
++
+ /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
+ * the first fragment
+ */
+diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
+index 20a67efda47f5..fa8da8ff35b42 100644
+--- a/net/ipv6/ip6_tunnel.c
++++ b/net/ipv6/ip6_tunnel.c
+@@ -1036,14 +1036,14 @@ int ip6_tnl_xmit_ctl(struct ip6_tnl *t,
+
+ if (unlikely(!ipv6_chk_addr_and_flags(net, laddr, ldev, false,
+ 0, IFA_F_TENTATIVE)))
+- pr_warn("%s xmit: Local address not yet configured!\n",
+- p->name);
++ pr_warn_ratelimited("%s xmit: Local address not yet configured!\n",
++ p->name);
+ else if (!(p->flags & IP6_TNL_F_ALLOW_LOCAL_REMOTE) &&
+ !ipv6_addr_is_multicast(raddr) &&
+ unlikely(ipv6_chk_addr_and_flags(net, raddr, ldev,
+ true, 0, IFA_F_TENTATIVE)))
+- pr_warn("%s xmit: Routing loop! Remote address found on this node!\n",
+- p->name);
++ pr_warn_ratelimited("%s xmit: Routing loop! Remote address found on this node!\n",
++ p->name);
+ else
+ ret = 1;
+ rcu_read_unlock();
+diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c
+index 1d8e3ffa225d8..42c37ec832f15 100644
+--- a/net/ipv6/ip6_vti.c
++++ b/net/ipv6/ip6_vti.c
+@@ -808,6 +808,8 @@ vti6_siocdevprivate(struct net_device *dev, struct ifreq *ifr, void __user *data
+ struct net *net = dev_net(dev);
+ struct vti6_net *ip6n = net_generic(net, vti6_net_id);
+
++ memset(&p1, 0, sizeof(p1));
++
+ switch (cmd) {
+ case SIOCGETTUNNEL:
+ if (dev == ip6n->fb_tnl_dev) {
+diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c
+index 36ed9efb88254..91f1c5f56d5fa 100644
+--- a/net/ipv6/ip6mr.c
++++ b/net/ipv6/ip6mr.c
+@@ -248,7 +248,9 @@ static int __net_init ip6mr_rules_init(struct net *net)
+ return 0;
+
+ err2:
++ rtnl_lock();
+ ip6mr_free_table(mrt);
++ rtnl_unlock();
+ err1:
+ fib_rules_unregister(ops);
+ return err;
+@@ -737,7 +739,7 @@ static int mif6_delete(struct mr_table *mrt, int vifi, int notify,
+
+ in6_dev = __in6_dev_get(dev);
+ if (in6_dev) {
+- in6_dev->cnf.mc_forwarding--;
++ atomic_dec(&in6_dev->cnf.mc_forwarding);
+ inet6_netconf_notify_devconf(dev_net(dev), RTM_NEWNETCONF,
+ NETCONFA_MC_FORWARDING,
+ dev->ifindex, &in6_dev->cnf);
+@@ -905,7 +907,7 @@ static int mif6_add(struct net *net, struct mr_table *mrt,
+
+ in6_dev = __in6_dev_get(dev);
+ if (in6_dev) {
+- in6_dev->cnf.mc_forwarding++;
++ atomic_inc(&in6_dev->cnf.mc_forwarding);
+ inet6_netconf_notify_devconf(dev_net(dev), RTM_NEWNETCONF,
+ NETCONFA_MC_FORWARDING,
+ dev->ifindex, &in6_dev->cnf);
+@@ -1555,7 +1557,7 @@ static int ip6mr_sk_init(struct mr_table *mrt, struct sock *sk)
+ } else {
+ rcu_assign_pointer(mrt->mroute_sk, sk);
+ sock_set_flag(sk, SOCK_RCU_FREE);
+- net->ipv6.devconf_all->mc_forwarding++;
++ atomic_inc(&net->ipv6.devconf_all->mc_forwarding);
+ }
+ write_unlock_bh(&mrt_lock);
+
+@@ -1588,7 +1590,7 @@ int ip6mr_sk_done(struct sock *sk)
+ * so the RCU grace period before sk freeing
+ * is guaranteed by sk_destruct()
+ */
+- net->ipv6.devconf_all->mc_forwarding--;
++ atomic_dec(&net->ipv6.devconf_all->mc_forwarding);
+ write_unlock_bh(&mrt_lock);
+ inet6_netconf_notify_devconf(net, RTM_NEWNETCONF,
+ NETCONFA_MC_FORWARDING,
+diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c
+index bed8155508c85..7f695c39d9a8c 100644
+--- a/net/ipv6/mcast.c
++++ b/net/ipv6/mcast.c
+@@ -460,10 +460,10 @@ int ip6_mc_source(int add, int omode, struct sock *sk,
+ newpsl->sl_addr[i] = psl->sl_addr[i];
+ atomic_sub(struct_size(psl, sl_addr, psl->sl_max),
+ &sk->sk_omem_alloc);
+- kfree_rcu(psl, rcu);
+ }
++ rcu_assign_pointer(pmc->sflist, newpsl);
++ kfree_rcu(psl, rcu);
+ psl = newpsl;
+- rcu_assign_pointer(pmc->sflist, psl);
+ }
+ rv = 1; /* > 0 for insert logic below if sl_count is 0 */
+ for (i = 0; i < psl->sl_count; i++) {
+@@ -565,12 +565,12 @@ int ip6_mc_msfilter(struct sock *sk, struct group_filter *gsf,
+ psl->sl_count, psl->sl_addr, 0);
+ atomic_sub(struct_size(psl, sl_addr, psl->sl_max),
+ &sk->sk_omem_alloc);
+- kfree_rcu(psl, rcu);
+ } else {
+ ip6_mc_del_src(idev, group, pmc->sfmode, 0, NULL, 0);
+ }
+- mutex_unlock(&idev->mc_lock);
+ rcu_assign_pointer(pmc->sflist, newpsl);
++ mutex_unlock(&idev->mc_lock);
++ kfree_rcu(psl, rcu);
+ pmc->sfmode = gsf->gf_fmode;
+ err = 0;
+ done:
+@@ -1371,27 +1371,23 @@ static void mld_process_v2(struct inet6_dev *idev, struct mld2_query *mld,
+ }
+
+ /* called with rcu_read_lock() */
+-int igmp6_event_query(struct sk_buff *skb)
++void igmp6_event_query(struct sk_buff *skb)
+ {
+ struct inet6_dev *idev = __in6_dev_get(skb->dev);
+
+- if (!idev)
+- return -EINVAL;
+-
+- if (idev->dead) {
+- kfree_skb(skb);
+- return -ENODEV;
+- }
++ if (!idev || idev->dead)
++ goto out;
+
+ spin_lock_bh(&idev->mc_query_lock);
+ if (skb_queue_len(&idev->mc_query_queue) < MLD_MAX_SKBS) {
+ __skb_queue_tail(&idev->mc_query_queue, skb);
+ if (!mod_delayed_work(mld_wq, &idev->mc_query_work, 0))
+ in6_dev_hold(idev);
++ skb = NULL;
+ }
+ spin_unlock_bh(&idev->mc_query_lock);
+-
+- return 0;
++out:
++ kfree_skb(skb);
+ }
+
+ static void __mld_query_work(struct sk_buff *skb)
+@@ -1542,27 +1538,23 @@ static void mld_query_work(struct work_struct *work)
+ }
+
+ /* called with rcu_read_lock() */
+-int igmp6_event_report(struct sk_buff *skb)
++void igmp6_event_report(struct sk_buff *skb)
+ {
+ struct inet6_dev *idev = __in6_dev_get(skb->dev);
+
+- if (!idev)
+- return -EINVAL;
+-
+- if (idev->dead) {
+- kfree_skb(skb);
+- return -ENODEV;
+- }
++ if (!idev || idev->dead)
++ goto out;
+
+ spin_lock_bh(&idev->mc_report_lock);
+ if (skb_queue_len(&idev->mc_report_queue) < MLD_MAX_SKBS) {
+ __skb_queue_tail(&idev->mc_report_queue, skb);
+ if (!mod_delayed_work(mld_wq, &idev->mc_report_work, 0))
+ in6_dev_hold(idev);
++ skb = NULL;
+ }
+ spin_unlock_bh(&idev->mc_report_lock);
+-
+- return 0;
++out:
++ kfree_skb(skb);
+ }
+
+ static void __mld_report_work(struct sk_buff *skb)
+@@ -1759,7 +1751,7 @@ static struct sk_buff *mld_newpack(struct inet6_dev *idev, unsigned int mtu)
+ skb_reserve(skb, hlen);
+ skb_tailroom_reserve(skb, mtu, tlen);
+
+- if (__ipv6_get_lladdr(idev, &addr_buf, IFA_F_TENTATIVE)) {
++ if (ipv6_get_lladdr(dev, &addr_buf, IFA_F_TENTATIVE)) {
+ /* <draft-ietf-magma-mld-source-05.txt>:
+ * use unspecified address as the source address
+ * when a valid link-local address is not available.
+diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c
+index 6ab710b5a1a82..118e834e91902 100644
+--- a/net/ipv6/netfilter.c
++++ b/net/ipv6/netfilter.c
+@@ -24,14 +24,13 @@ int ip6_route_me_harder(struct net *net, struct sock *sk_partial, struct sk_buff
+ {
+ const struct ipv6hdr *iph = ipv6_hdr(skb);
+ struct sock *sk = sk_to_full_sk(sk_partial);
++ struct net_device *dev = skb_dst(skb)->dev;
+ struct flow_keys flkeys;
+ unsigned int hh_len;
+ struct dst_entry *dst;
+ int strict = (ipv6_addr_type(&iph->daddr) &
+ (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL));
+ struct flowi6 fl6 = {
+- .flowi6_oif = sk && sk->sk_bound_dev_if ? sk->sk_bound_dev_if :
+- strict ? skb_dst(skb)->dev->ifindex : 0,
+ .flowi6_mark = skb->mark,
+ .flowi6_uid = sock_net_uid(net, sk),
+ .daddr = iph->daddr,
+@@ -39,6 +38,13 @@ int ip6_route_me_harder(struct net *net, struct sock *sk_partial, struct sk_buff
+ };
+ int err;
+
++ if (sk && sk->sk_bound_dev_if)
++ fl6.flowi6_oif = sk->sk_bound_dev_if;
++ else if (strict)
++ fl6.flowi6_oif = dev->ifindex;
++ else
++ fl6.flowi6_oif = l3mdev_master_ifindex(dev);
++
+ fib6_rules_early_flow_dissect(net, skb, &fl6, &flkeys);
+ dst = ip6_route_output(net, sk, &fl6);
+ err = dst->error;
+diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c
+index 60f1e4f5be5aa..c51d5ce3711c2 100644
+--- a/net/ipv6/raw.c
++++ b/net/ipv6/raw.c
+@@ -1020,6 +1020,9 @@ static int do_rawv6_setsockopt(struct sock *sk, int level, int optname,
+ struct raw6_sock *rp = raw6_sk(sk);
+ int val;
+
++ if (optlen < sizeof(val))
++ return -EINVAL;
++
+ if (copy_from_sockptr(&val, optval, sizeof(val)))
+ return -EFAULT;
+
+diff --git a/net/ipv6/route.c b/net/ipv6/route.c
+index 9b9ef09382ab9..0ca7c780d97a2 100644
+--- a/net/ipv6/route.c
++++ b/net/ipv6/route.c
+@@ -2802,7 +2802,7 @@ static void ip6_link_failure(struct sk_buff *skb)
+ if (from) {
+ fn = rcu_dereference(from->fib6_node);
+ if (fn && (rt->rt6i_flags & RTF_DEFAULT))
+- fn->fn_sernum = -1;
++ WRITE_ONCE(fn->fn_sernum, -1);
+ }
+ }
+ rcu_read_unlock();
+@@ -3303,6 +3303,7 @@ static int ip6_dst_gc(struct dst_ops *ops)
+ int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
+ int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
+ unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
++ unsigned int val;
+ int entries;
+
+ entries = dst_entries_get_fast(ops);
+@@ -3313,13 +3314,13 @@ static int ip6_dst_gc(struct dst_ops *ops)
+ entries <= rt_max_size)
+ goto out;
+
+- net->ipv6.ip6_rt_gc_expire++;
+- fib6_run_gc(net->ipv6.ip6_rt_gc_expire, net, true);
++ fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
+ entries = dst_entries_get_slow(ops);
+ if (entries < ops->gc_thresh)
+- net->ipv6.ip6_rt_gc_expire = rt_gc_timeout>>1;
++ atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1);
+ out:
+- net->ipv6.ip6_rt_gc_expire -= net->ipv6.ip6_rt_gc_expire>>rt_elasticity;
++ val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
++ atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
+ return entries > rt_max_size;
+ }
+
+@@ -3680,6 +3681,25 @@ void fib6_nh_release(struct fib6_nh *fib6_nh)
+ fib_nh_common_release(&fib6_nh->nh_common);
+ }
+
++void fib6_nh_release_dsts(struct fib6_nh *fib6_nh)
++{
++ int cpu;
++
++ if (!fib6_nh->rt6i_pcpu)
++ return;
++
++ for_each_possible_cpu(cpu) {
++ struct rt6_info *pcpu_rt, **ppcpu_rt;
++
++ ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
++ pcpu_rt = xchg(ppcpu_rt, NULL);
++ if (pcpu_rt) {
++ dst_dev_put(&pcpu_rt->dst);
++ dst_release(&pcpu_rt->dst);
++ }
++ }
++}
++
+ static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
+ gfp_t gfp_flags,
+ struct netlink_ext_ack *extack)
+@@ -4490,7 +4510,7 @@ static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
+ struct inet6_dev *idev;
+ int type;
+
+- if (netif_is_l3_master(skb->dev) &&
++ if (netif_is_l3_master(skb->dev) ||
+ dst->dev == net->loopback_dev)
+ idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
+ else
+@@ -5205,6 +5225,19 @@ out:
+ return should_notify;
+ }
+
++static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla,
++ struct netlink_ext_ack *extack)
++{
++ if (nla_len(nla) < sizeof(*gw)) {
++ NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY");
++ return -EINVAL;
++ }
++
++ *gw = nla_get_in6_addr(nla);
++
++ return 0;
++}
++
+ static int ip6_route_multipath_add(struct fib6_config *cfg,
+ struct netlink_ext_ack *extack)
+ {
+@@ -5245,10 +5278,18 @@ static int ip6_route_multipath_add(struct fib6_config *cfg,
+
+ nla = nla_find(attrs, attrlen, RTA_GATEWAY);
+ if (nla) {
+- r_cfg.fc_gateway = nla_get_in6_addr(nla);
++ err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
++ extack);
++ if (err)
++ goto cleanup;
++
+ r_cfg.fc_flags |= RTF_GATEWAY;
+ }
+ r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
++
++ /* RTA_ENCAP_TYPE length checked in
++ * lwtunnel_valid_encap_type_attr
++ */
+ nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
+ if (nla)
+ r_cfg.fc_encap_type = nla_get_u16(nla);
+@@ -5415,7 +5456,13 @@ static int ip6_route_multipath_del(struct fib6_config *cfg,
+
+ nla = nla_find(attrs, attrlen, RTA_GATEWAY);
+ if (nla) {
+- nla_memcpy(&r_cfg.fc_gateway, nla, 16);
++ err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
++ extack);
++ if (err) {
++ last_err = err;
++ goto next_rtnh;
++ }
++
+ r_cfg.fc_flags |= RTF_GATEWAY;
+ }
+ }
+@@ -5423,6 +5470,7 @@ static int ip6_route_multipath_del(struct fib6_config *cfg,
+ if (err)
+ last_err = err;
+
++next_rtnh:
+ rtnh = rtnh_next(rtnh, &remaining);
+ }
+
+@@ -5720,11 +5768,11 @@ static int rt6_fill_node(struct net *net, struct sk_buff *skb,
+ }
+
+ if (!dst) {
+- if (rt->offload)
++ if (READ_ONCE(rt->offload))
+ rtm->rtm_flags |= RTM_F_OFFLOAD;
+- if (rt->trap)
++ if (READ_ONCE(rt->trap))
+ rtm->rtm_flags |= RTM_F_TRAP;
+- if (rt->offload_failed)
++ if (READ_ONCE(rt->offload_failed))
+ rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;
+ }
+
+@@ -6182,19 +6230,20 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
+ struct sk_buff *skb;
+ int err;
+
+- if (f6i->offload == offload && f6i->trap == trap &&
+- f6i->offload_failed == offload_failed)
++ if (READ_ONCE(f6i->offload) == offload &&
++ READ_ONCE(f6i->trap) == trap &&
++ READ_ONCE(f6i->offload_failed) == offload_failed)
+ return;
+
+- f6i->offload = offload;
+- f6i->trap = trap;
++ WRITE_ONCE(f6i->offload, offload);
++ WRITE_ONCE(f6i->trap, trap);
+
+ /* 2 means send notifications only if offload_failed was changed. */
+ if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 &&
+- f6i->offload_failed == offload_failed)
++ READ_ONCE(f6i->offload_failed) == offload_failed)
+ return;
+
+- f6i->offload_failed = offload_failed;
++ WRITE_ONCE(f6i->offload_failed, offload_failed);
+
+ if (!rcu_access_pointer(f6i->fib6_node))
+ /* The route was removed from the tree, do not send
+@@ -6480,7 +6529,7 @@ static int __net_init ip6_route_net_init(struct net *net)
+ net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
+ net->ipv6.sysctl.skip_notify_on_dev_down = 0;
+
+- net->ipv6.ip6_rt_gc_expire = 30*HZ;
++ atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ);
+
+ ret = 0;
+ out:
+diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c
+index e412817fba2f3..fa6b64c95d3ae 100644
+--- a/net/ipv6/seg6.c
++++ b/net/ipv6/seg6.c
+@@ -75,6 +75,65 @@ bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced)
+ return true;
+ }
+
++struct ipv6_sr_hdr *seg6_get_srh(struct sk_buff *skb, int flags)
++{
++ struct ipv6_sr_hdr *srh;
++ int len, srhoff = 0;
++
++ if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, &flags) < 0)
++ return NULL;
++
++ if (!pskb_may_pull(skb, srhoff + sizeof(*srh)))
++ return NULL;
++
++ srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
++
++ len = (srh->hdrlen + 1) << 3;
++
++ if (!pskb_may_pull(skb, srhoff + len))
++ return NULL;
++
++ /* note that pskb_may_pull may change pointers in header;
++ * for this reason it is necessary to reload them when needed.
++ */
++ srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
++
++ if (!seg6_validate_srh(srh, len, true))
++ return NULL;
++
++ return srh;
++}
++
++/* Determine if an ICMP invoking packet contains a segment routing
++ * header. If it does, extract the offset to the true destination
++ * address, which is in the first segment address.
++ */
++void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt)
++{
++ __u16 network_header = skb->network_header;
++ struct ipv6_sr_hdr *srh;
++
++ /* Update network header to point to the invoking packet
++ * inside the ICMP packet, so we can use the seg6_get_srh()
++ * helper.
++ */
++ skb_reset_network_header(skb);
++
++ srh = seg6_get_srh(skb, 0);
++ if (!srh)
++ goto out;
++
++ if (srh->type != IPV6_SRCRT_TYPE_4)
++ goto out;
++
++ opt->flags |= IP6SKB_SEG6;
++ opt->srhoff = (unsigned char *)srh - skb->data;
++
++out:
++ /* Restore the network header back to the ICMP packet */
++ skb->network_header = network_header;
++}
++
+ static struct genl_family seg6_genl_family;
+
+ static const struct nla_policy seg6_genl_policy[SEG6_ATTR_MAX + 1] = {
+diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c
+index 3adc5d9211ad6..d64855010948d 100644
+--- a/net/ipv6/seg6_iptunnel.c
++++ b/net/ipv6/seg6_iptunnel.c
+@@ -161,6 +161,14 @@ int seg6_do_srh_encap(struct sk_buff *skb, struct ipv6_sr_hdr *osrh, int proto)
+ hdr->hop_limit = ip6_dst_hoplimit(skb_dst(skb));
+
+ memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
++
++ /* the control block has been erased, so we have to set the
++ * iif once again.
++ * We read the receiving interface index directly from the
++ * skb->skb_iif as it is done in the IPv4 receiving path (i.e.:
++ * ip_rcv_core(...)).
++ */
++ IP6CB(skb)->iif = skb->skb_iif;
+ }
+
+ hdr->nexthdr = NEXTHDR_ROUTING;
+diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c
+index 2dc40b3f373ef..ef88489c71f52 100644
+--- a/net/ipv6/seg6_local.c
++++ b/net/ipv6/seg6_local.c
+@@ -150,40 +150,11 @@ static struct seg6_local_lwt *seg6_local_lwtunnel(struct lwtunnel_state *lwt)
+ return (struct seg6_local_lwt *)lwt->data;
+ }
+
+-static struct ipv6_sr_hdr *get_srh(struct sk_buff *skb, int flags)
+-{
+- struct ipv6_sr_hdr *srh;
+- int len, srhoff = 0;
+-
+- if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, &flags) < 0)
+- return NULL;
+-
+- if (!pskb_may_pull(skb, srhoff + sizeof(*srh)))
+- return NULL;
+-
+- srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
+-
+- len = (srh->hdrlen + 1) << 3;
+-
+- if (!pskb_may_pull(skb, srhoff + len))
+- return NULL;
+-
+- /* note that pskb_may_pull may change pointers in header;
+- * for this reason it is necessary to reload them when needed.
+- */
+- srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
+-
+- if (!seg6_validate_srh(srh, len, true))
+- return NULL;
+-
+- return srh;
+-}
+-
+ static struct ipv6_sr_hdr *get_and_validate_srh(struct sk_buff *skb)
+ {
+ struct ipv6_sr_hdr *srh;
+
+- srh = get_srh(skb, IP6_FH_F_SKIP_RH);
++ srh = seg6_get_srh(skb, IP6_FH_F_SKIP_RH);
+ if (!srh)
+ return NULL;
+
+@@ -200,7 +171,7 @@ static bool decap_and_validate(struct sk_buff *skb, int proto)
+ struct ipv6_sr_hdr *srh;
+ unsigned int off = 0;
+
+- srh = get_srh(skb, 0);
++ srh = seg6_get_srh(skb, 0);
+ if (srh && srh->segments_left > 0)
+ return false;
+
+diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
+index ef0c7a7c18e23..626cb53aa57ab 100644
+--- a/net/ipv6/sit.c
++++ b/net/ipv6/sit.c
+@@ -1933,7 +1933,6 @@ static int __net_init sit_init_net(struct net *net)
+ return 0;
+
+ err_reg_dev:
+- ipip6_dev_free(sitn->fb_tunnel_dev);
+ free_netdev(sitn->fb_tunnel_dev);
+ err_alloc_dev:
+ return err;
+diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c
+index e8cfb9e997bf0..ca92dd6981dea 100644
+--- a/net/ipv6/syncookies.c
++++ b/net/ipv6/syncookies.c
+@@ -170,7 +170,8 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb)
+ goto out;
+
+ ret = NULL;
+- req = cookie_tcp_reqsk_alloc(&tcp6_request_sock_ops, sk, skb);
++ req = cookie_tcp_reqsk_alloc(&tcp6_request_sock_ops,
++ &tcp_request_sock_ipv6_ops, sk, skb);
+ if (!req)
+ goto out;
+
+diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
+index b03dd02c9f13c..8eedf59e9cf25 100644
+--- a/net/ipv6/tcp_ipv6.c
++++ b/net/ipv6/tcp_ipv6.c
+@@ -107,9 +107,9 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
+ if (dst && dst_hold_safe(dst)) {
+ const struct rt6_info *rt = (const struct rt6_info *)dst;
+
+- sk->sk_rx_dst = dst;
+- inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
+- tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
++ rcu_assign_pointer(sk->sk_rx_dst, dst);
++ sk->sk_rx_dst_ifindex = skb->skb_iif;
++ sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
+ }
+ }
+
+@@ -1504,16 +1504,19 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
+ opt_skb = skb_clone(skb, sk_gfp_mask(sk, GFP_ATOMIC));
+
+ if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
+- struct dst_entry *dst = sk->sk_rx_dst;
++ struct dst_entry *dst;
++
++ dst = rcu_dereference_protected(sk->sk_rx_dst,
++ lockdep_sock_is_held(sk));
+
+ sock_rps_save_rxhash(sk, skb);
+ sk_mark_napi_id(sk, skb);
+ if (dst) {
+- if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
++ if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
+ INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
+- dst, np->rx_dst_cookie) == NULL) {
++ dst, sk->sk_rx_dst_cookie) == NULL) {
++ RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
+ dst_release(dst);
+- sk->sk_rx_dst = NULL;
+ }
+ }
+
+@@ -1875,12 +1878,12 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
+ skb->sk = sk;
+ skb->destructor = sock_edemux;
+ if (sk_fullsock(sk)) {
+- struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
++ struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
+
+ if (dst)
+- dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
++ dst = dst_check(dst, sk->sk_rx_dst_cookie);
+ if (dst &&
+- inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
++ sk->sk_rx_dst_ifindex == skb->skb_iif)
+ skb_dst_set_noref(skb, dst);
+ }
+ }
+diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
+index 8d785232b4796..932c6f2a54946 100644
+--- a/net/ipv6/udp.c
++++ b/net/ipv6/udp.c
+@@ -40,6 +40,7 @@
+ #include <net/transp_v6.h>
+ #include <net/ip6_route.h>
+ #include <net/raw.h>
++#include <net/seg6.h>
+ #include <net/tcp_states.h>
+ #include <net/ip6_checksum.h>
+ #include <net/ip6_tunnel.h>
+@@ -561,7 +562,7 @@ int __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
+ struct ipv6_pinfo *np;
+ const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
+ const struct in6_addr *saddr = &hdr->saddr;
+- const struct in6_addr *daddr = &hdr->daddr;
++ const struct in6_addr *daddr = seg6_get_daddr(skb, opt) ? : &hdr->daddr;
+ struct udphdr *uh = (struct udphdr *)(skb->data+offset);
+ bool tunnel = false;
+ struct sock *sk;
+@@ -884,7 +885,7 @@ static void udp6_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
+ if (udp_sk_rx_dst_set(sk, dst)) {
+ const struct rt6_info *rt = (const struct rt6_info *)dst;
+
+- inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
++ sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
+ }
+ }
+
+@@ -956,7 +957,7 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
+ struct dst_entry *dst = skb_dst(skb);
+ int ret;
+
+- if (unlikely(sk->sk_rx_dst != dst))
++ if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
+ udp6_sk_rx_dst_set(sk, dst);
+
+ if (!uh->check && !udp_sk(sk)->no_check6_rx) {
+@@ -1070,10 +1071,10 @@ INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb)
+
+ skb->sk = sk;
+ skb->destructor = sock_efree;
+- dst = READ_ONCE(sk->sk_rx_dst);
++ dst = rcu_dereference(sk->sk_rx_dst);
+
+ if (dst)
+- dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie);
++ dst = dst_check(dst, sk->sk_rx_dst_cookie);
+ if (dst) {
+ /* set noref for now.
+ * any place which wants to hold dst has to call
+@@ -1204,7 +1205,7 @@ static int udp_v6_send_skb(struct sk_buff *skb, struct flowi6 *fl6,
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+- if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) {
++ if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+@@ -1435,7 +1436,6 @@ do_udp_sendmsg:
+ if (!fl6.flowi6_oif)
+ fl6.flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
+
+- fl6.flowi6_mark = ipc6.sockc.mark;
+ fl6.flowi6_uid = sk->sk_uid;
+
+ if (msg->msg_controllen) {
+@@ -1471,6 +1471,7 @@ do_udp_sendmsg:
+ ipc6.opt = opt;
+
+ fl6.flowi6_proto = sk->sk_protocol;
++ fl6.flowi6_mark = ipc6.sockc.mark;
+ fl6.daddr = *daddr;
+ if (ipv6_addr_any(&fl6.saddr) && !ipv6_addr_any(&np->saddr))
+ fl6.saddr = np->saddr;
+diff --git a/net/ipv6/xfrm6_output.c b/net/ipv6/xfrm6_output.c
+index d0d280077721b..ad07904642cad 100644
+--- a/net/ipv6/xfrm6_output.c
++++ b/net/ipv6/xfrm6_output.c
+@@ -45,6 +45,19 @@ static int __xfrm6_output_finish(struct net *net, struct sock *sk, struct sk_buf
+ return xfrm_output(sk, skb);
+ }
+
++static int xfrm6_noneed_fragment(struct sk_buff *skb)
++{
++ struct frag_hdr *fh;
++ u8 prevhdr = ipv6_hdr(skb)->nexthdr;
++
++ if (prevhdr != NEXTHDR_FRAGMENT)
++ return 0;
++ fh = (struct frag_hdr *)(skb->data + sizeof(struct ipv6hdr));
++ if (fh->nexthdr == NEXTHDR_ESP || fh->nexthdr == NEXTHDR_AUTH)
++ return 1;
++ return 0;
++}
++
+ static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
+ {
+ struct dst_entry *dst = skb_dst(skb);
+@@ -73,6 +86,9 @@ static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
+ xfrm6_local_rxpmtu(skb, mtu);
+ kfree_skb(skb);
+ return -EMSGSIZE;
++ } else if (toobig && xfrm6_noneed_fragment(skb)) {
++ skb->ignore_df = 1;
++ goto skip_frag;
+ } else if (!skb->ignore_df && toobig && skb->sk) {
+ xfrm_local_error(skb, mtu);
+ kfree_skb(skb);
+diff --git a/net/key/af_key.c b/net/key/af_key.c
+index de24a7d474dfd..92e9d75dba2f4 100644
+--- a/net/key/af_key.c
++++ b/net/key/af_key.c
+@@ -1699,7 +1699,7 @@ static int pfkey_register(struct sock *sk, struct sk_buff *skb, const struct sad
+
+ xfrm_probe_algs();
+
+- supp_skb = compose_sadb_supported(hdr, GFP_KERNEL);
++ supp_skb = compose_sadb_supported(hdr, GFP_KERNEL | __GFP_ZERO);
+ if (!supp_skb) {
+ if (hdr->sadb_msg_satype != SADB_SATYPE_UNSPEC)
+ pfk->registered &= ~(1<<hdr->sadb_msg_satype);
+@@ -2623,7 +2623,7 @@ static int pfkey_migrate(struct sock *sk, struct sk_buff *skb,
+ }
+
+ return xfrm_migrate(&sel, dir, XFRM_POLICY_TYPE_MAIN, m, i,
+- kma ? &k : NULL, net, NULL);
++ kma ? &k : NULL, net, NULL, 0);
+
+ out:
+ return err;
+@@ -2826,8 +2826,10 @@ static int pfkey_process(struct sock *sk, struct sk_buff *skb, const struct sadb
+ void *ext_hdrs[SADB_EXT_MAX];
+ int err;
+
+- pfkey_broadcast(skb_clone(skb, GFP_KERNEL), GFP_KERNEL,
+- BROADCAST_PROMISC_ONLY, NULL, sock_net(sk));
++ err = pfkey_broadcast(skb_clone(skb, GFP_KERNEL), GFP_KERNEL,
++ BROADCAST_PROMISC_ONLY, NULL, sock_net(sk));
++ if (err)
++ return err;
+
+ memset(ext_hdrs, 0, sizeof(ext_hdrs));
+ err = parse_exthdrs(skb, hdr, ext_hdrs);
+diff --git a/net/l3mdev/l3mdev.c b/net/l3mdev/l3mdev.c
+index 17927966abb33..8b14a24f10404 100644
+--- a/net/l3mdev/l3mdev.c
++++ b/net/l3mdev/l3mdev.c
+@@ -147,7 +147,7 @@ int l3mdev_master_upper_ifindex_by_index_rcu(struct net *net, int ifindex)
+
+ dev = dev_get_by_index_rcu(net, ifindex);
+ while (dev && !netif_is_l3_master(dev))
+- dev = netdev_master_upper_dev_get(dev);
++ dev = netdev_master_upper_dev_get_rcu(dev);
+
+ return dev ? dev->ifindex : 0;
+ }
+diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c
+index 3086f4a6ae683..99305aadaa087 100644
+--- a/net/llc/af_llc.c
++++ b/net/llc/af_llc.c
+@@ -275,6 +275,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr)
+ {
+ struct sock *sk = sock->sk;
+ struct llc_sock *llc = llc_sk(sk);
++ struct net_device *dev = NULL;
+ struct llc_sap *sap;
+ int rc = -EINVAL;
+
+@@ -286,14 +287,14 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr)
+ goto out;
+ rc = -ENODEV;
+ if (sk->sk_bound_dev_if) {
+- llc->dev = dev_get_by_index(&init_net, sk->sk_bound_dev_if);
+- if (llc->dev && addr->sllc_arphrd != llc->dev->type) {
+- dev_put(llc->dev);
+- llc->dev = NULL;
++ dev = dev_get_by_index(&init_net, sk->sk_bound_dev_if);
++ if (dev && addr->sllc_arphrd != dev->type) {
++ dev_put(dev);
++ dev = NULL;
+ }
+ } else
+- llc->dev = dev_getfirstbyhwtype(&init_net, addr->sllc_arphrd);
+- if (!llc->dev)
++ dev = dev_getfirstbyhwtype(&init_net, addr->sllc_arphrd);
++ if (!dev)
+ goto out;
+ rc = -EUSERS;
+ llc->laddr.lsap = llc_ui_autoport();
+@@ -303,6 +304,11 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr)
+ sap = llc_sap_open(llc->laddr.lsap, NULL);
+ if (!sap)
+ goto out;
++
++ /* Note: We do not expect errors from this point. */
++ llc->dev = dev;
++ dev = NULL;
++
+ memcpy(llc->laddr.mac, llc->dev->dev_addr, IFHWADDRLEN);
+ memcpy(&llc->addr, addr, sizeof(llc->addr));
+ /* assign new connection to its SAP */
+@@ -310,6 +316,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr)
+ sock_reset_flag(sk, SOCK_ZAPPED);
+ rc = 0;
+ out:
++ dev_put(dev);
+ return rc;
+ }
+
+@@ -332,6 +339,7 @@ static int llc_ui_bind(struct socket *sock, struct sockaddr *uaddr, int addrlen)
+ struct sockaddr_llc *addr = (struct sockaddr_llc *)uaddr;
+ struct sock *sk = sock->sk;
+ struct llc_sock *llc = llc_sk(sk);
++ struct net_device *dev = NULL;
+ struct llc_sap *sap;
+ int rc = -EINVAL;
+
+@@ -347,25 +355,27 @@ static int llc_ui_bind(struct socket *sock, struct sockaddr *uaddr, int addrlen)
+ rc = -ENODEV;
+ rcu_read_lock();
+ if (sk->sk_bound_dev_if) {
+- llc->dev = dev_get_by_index_rcu(&init_net, sk->sk_bound_dev_if);
+- if (llc->dev) {
++ dev = dev_get_by_index_rcu(&init_net, sk->sk_bound_dev_if);
++ if (dev) {
+ if (is_zero_ether_addr(addr->sllc_mac))
+- memcpy(addr->sllc_mac, llc->dev->dev_addr,
++ memcpy(addr->sllc_mac, dev->dev_addr,
+ IFHWADDRLEN);
+- if (addr->sllc_arphrd != llc->dev->type ||
++ if (addr->sllc_arphrd != dev->type ||
+ !ether_addr_equal(addr->sllc_mac,
+- llc->dev->dev_addr)) {
++ dev->dev_addr)) {
+ rc = -EINVAL;
+- llc->dev = NULL;
++ dev = NULL;
+ }
+ }
+- } else
+- llc->dev = dev_getbyhwaddr_rcu(&init_net, addr->sllc_arphrd,
++ } else {
++ dev = dev_getbyhwaddr_rcu(&init_net, addr->sllc_arphrd,
+ addr->sllc_mac);
+- dev_hold(llc->dev);
++ }
++ dev_hold(dev);
+ rcu_read_unlock();
+- if (!llc->dev)
++ if (!dev)
+ goto out;
++
+ if (!addr->sllc_sap) {
+ rc = -EUSERS;
+ addr->sllc_sap = llc_ui_autoport();
+@@ -397,6 +407,11 @@ static int llc_ui_bind(struct socket *sock, struct sockaddr *uaddr, int addrlen)
+ goto out_put;
+ }
+ }
++
++ /* Note: We do not expect errors from this point. */
++ llc->dev = dev;
++ dev = NULL;
++
+ llc->laddr.lsap = addr->sllc_sap;
+ memcpy(llc->laddr.mac, addr->sllc_mac, IFHWADDRLEN);
+ memcpy(&llc->addr, addr, sizeof(llc->addr));
+@@ -407,6 +422,7 @@ static int llc_ui_bind(struct socket *sock, struct sockaddr *uaddr, int addrlen)
+ out_put:
+ llc_sap_put(sap);
+ out:
++ dev_put(dev);
+ release_sock(sk);
+ return rc;
+ }
+diff --git a/net/mac80211/agg-rx.c b/net/mac80211/agg-rx.c
+index cce28e3b22323..ef729b1e39ea3 100644
+--- a/net/mac80211/agg-rx.c
++++ b/net/mac80211/agg-rx.c
+@@ -9,7 +9,7 @@
+ * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
+ * Copyright 2007-2010, Intel Corporation
+ * Copyright(c) 2015-2017 Intel Deutschland GmbH
+- * Copyright (C) 2018-2020 Intel Corporation
++ * Copyright (C) 2018-2021 Intel Corporation
+ */
+
+ /**
+@@ -191,7 +191,8 @@ static void ieee80211_add_addbaext(struct ieee80211_sub_if_data *sdata,
+ sband = ieee80211_get_sband(sdata);
+ if (!sband)
+ return;
+- he_cap = ieee80211_get_he_iftype_cap(sband, sdata->vif.type);
++ he_cap = ieee80211_get_he_iftype_cap(sband,
++ ieee80211_vif_type_p2p(&sdata->vif));
+ if (!he_cap)
+ return;
+
+diff --git a/net/mac80211/agg-tx.c b/net/mac80211/agg-tx.c
+index 430a585875388..1deb3d874a4b9 100644
+--- a/net/mac80211/agg-tx.c
++++ b/net/mac80211/agg-tx.c
+@@ -9,7 +9,7 @@
+ * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
+ * Copyright 2007-2010, Intel Corporation
+ * Copyright(c) 2015-2017 Intel Deutschland GmbH
+- * Copyright (C) 2018 - 2020 Intel Corporation
++ * Copyright (C) 2018 - 2022 Intel Corporation
+ */
+
+ #include <linux/ieee80211.h>
+@@ -106,7 +106,7 @@ static void ieee80211_send_addba_request(struct ieee80211_sub_if_data *sdata,
+ mgmt->u.action.u.addba_req.start_seq_num =
+ cpu_to_le16(start_seq_num << 4);
+
+- ieee80211_tx_skb(sdata, skb);
++ ieee80211_tx_skb_tid(sdata, skb, tid);
+ }
+
+ void ieee80211_send_bar(struct ieee80211_vif *vif, u8 *ra, u16 tid, u16 ssn)
+@@ -213,6 +213,8 @@ ieee80211_agg_start_txq(struct sta_info *sta, int tid, bool enable)
+ struct ieee80211_txq *txq = sta->sta.txq[tid];
+ struct txq_info *txqi;
+
++ lockdep_assert_held(&sta->ampdu_mlme.mtx);
++
+ if (!txq)
+ return;
+
+@@ -290,7 +292,6 @@ static void ieee80211_remove_tid_tx(struct sta_info *sta, int tid)
+ ieee80211_assign_tid_tx(sta, tid, NULL);
+
+ ieee80211_agg_splice_finish(sta->sdata, tid);
+- ieee80211_agg_start_txq(sta, tid, false);
+
+ kfree_rcu(tid_tx, rcu_head);
+ }
+@@ -480,8 +481,7 @@ static void ieee80211_send_addba_with_timeout(struct sta_info *sta,
+
+ /* send AddBA request */
+ ieee80211_send_addba_request(sdata, sta->sta.addr, tid,
+- tid_tx->dialog_token,
+- sta->tid_seq[tid] >> 4,
++ tid_tx->dialog_token, tid_tx->ssn,
+ buf_size, tid_tx->timeout);
+
+ WARN_ON(test_and_set_bit(HT_AGG_STATE_SENT_ADDBA, &tid_tx->state));
+@@ -523,6 +523,7 @@ void ieee80211_tx_ba_session_handle_start(struct sta_info *sta, int tid)
+
+ params.ssn = sta->tid_seq[tid] >> 4;
+ ret = drv_ampdu_action(local, sdata, &params);
++ tid_tx->ssn = params.ssn;
+ if (ret == IEEE80211_AMPDU_TX_START_DELAY_ADDBA) {
+ return;
+ } else if (ret == IEEE80211_AMPDU_TX_START_IMMEDIATE) {
+@@ -625,6 +626,14 @@ int ieee80211_start_tx_ba_session(struct ieee80211_sta *pubsta, u16 tid,
+ return -EINVAL;
+ }
+
++ if (test_sta_flag(sta, WLAN_STA_MFP) &&
++ !test_sta_flag(sta, WLAN_STA_AUTHORIZED)) {
++ ht_dbg(sdata,
++ "MFP STA not authorized - deny BA session request %pM tid %d\n",
++ sta->sta.addr, tid);
++ return -EINVAL;
++ }
++
+ /*
+ * 802.11n-2009 11.5.1.1: If the initiating STA is an HT STA, is a
+ * member of an IBSS, and has no other existing Block Ack agreement
+@@ -889,6 +898,7 @@ void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid,
+ {
+ struct ieee80211_sub_if_data *sdata = sta->sdata;
+ bool send_delba = false;
++ bool start_txq = false;
+
+ ht_dbg(sdata, "Stopping Tx BA session for %pM tid %d\n",
+ sta->sta.addr, tid);
+@@ -906,10 +916,14 @@ void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid,
+ send_delba = true;
+
+ ieee80211_remove_tid_tx(sta, tid);
++ start_txq = true;
+
+ unlock_sta:
+ spin_unlock_bh(&sta->lock);
+
++ if (start_txq)
++ ieee80211_agg_start_txq(sta, tid, false);
++
+ if (send_delba)
+ ieee80211_send_delba(sdata, sta->sta.addr, tid,
+ WLAN_BACK_INITIATOR, WLAN_REASON_QSTA_NOT_USE);
+diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
+index d69b31c20fe28..3f625e836a03f 100644
+--- a/net/mac80211/cfg.c
++++ b/net/mac80211/cfg.c
+@@ -80,7 +80,8 @@ static int ieee80211_set_mon_options(struct ieee80211_sub_if_data *sdata,
+ }
+
+ /* also validate MU-MIMO change */
+- monitor_sdata = rtnl_dereference(local->monitor_sdata);
++ monitor_sdata = wiphy_dereference(local->hw.wiphy,
++ local->monitor_sdata);
+
+ if (!monitor_sdata &&
+ (params->vht_mumimo_groups || params->vht_mumimo_follow_addr))
+@@ -810,7 +811,8 @@ static int ieee80211_set_monitor_channel(struct wiphy *wiphy,
+
+ mutex_lock(&local->mtx);
+ if (local->use_chanctx) {
+- sdata = rtnl_dereference(local->monitor_sdata);
++ sdata = wiphy_dereference(local->hw.wiphy,
++ local->monitor_sdata);
+ if (sdata) {
+ ieee80211_vif_release_channel(sdata);
+ ret = ieee80211_vif_use_channel(sdata, chandef,
+@@ -1224,7 +1226,10 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev,
+ return 0;
+
+ error:
++ mutex_lock(&local->mtx);
+ ieee80211_vif_release_channel(sdata);
++ mutex_unlock(&local->mtx);
++
+ return err;
+ }
+
+@@ -2105,14 +2110,12 @@ static int copy_mesh_setup(struct ieee80211_if_mesh *ifmsh,
+ const struct mesh_setup *setup)
+ {
+ u8 *new_ie;
+- const u8 *old_ie;
+ struct ieee80211_sub_if_data *sdata = container_of(ifmsh,
+ struct ieee80211_sub_if_data, u.mesh);
+ int i;
+
+ /* allocate information elements */
+ new_ie = NULL;
+- old_ie = ifmsh->ie;
+
+ if (setup->ie_len) {
+ new_ie = kmemdup(setup->ie, setup->ie_len,
+@@ -2122,7 +2125,6 @@ static int copy_mesh_setup(struct ieee80211_if_mesh *ifmsh,
+ }
+ ifmsh->ie_len = setup->ie_len;
+ ifmsh->ie = new_ie;
+- kfree(old_ie);
+
+ /* now copy the rest of the setup parameters */
+ ifmsh->mesh_id_len = setup->mesh_id_len;
+@@ -2669,7 +2671,8 @@ static int ieee80211_set_tx_power(struct wiphy *wiphy,
+ sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
+
+ if (sdata->vif.type == NL80211_IFTYPE_MONITOR) {
+- sdata = rtnl_dereference(local->monitor_sdata);
++ sdata = wiphy_dereference(local->hw.wiphy,
++ local->monitor_sdata);
+ if (!sdata)
+ return -EOPNOTSUPP;
+ }
+@@ -2729,7 +2732,8 @@ static int ieee80211_set_tx_power(struct wiphy *wiphy,
+ mutex_unlock(&local->iflist_mtx);
+
+ if (has_monitor) {
+- sdata = rtnl_dereference(local->monitor_sdata);
++ sdata = wiphy_dereference(local->hw.wiphy,
++ local->monitor_sdata);
+ if (sdata) {
+ sdata->user_power_level = local->user_power_level;
+ if (txp_type != sdata->vif.bss_conf.txpower_type)
+diff --git a/net/mac80211/driver-ops.h b/net/mac80211/driver-ops.h
+index cd3731cbf6c68..c336267f4599c 100644
+--- a/net/mac80211/driver-ops.h
++++ b/net/mac80211/driver-ops.h
+@@ -1219,8 +1219,11 @@ static inline void drv_wake_tx_queue(struct ieee80211_local *local,
+ {
+ struct ieee80211_sub_if_data *sdata = vif_to_sdata(txq->txq.vif);
+
+- if (local->in_reconfig)
++ /* In reconfig don't transmit now, but mark for waking later */
++ if (local->in_reconfig) {
++ set_bit(IEEE80211_TXQ_STOP_NETIF_TX, &txq->flags);
+ return;
++ }
+
+ if (!check_sdata_in_driver(sdata))
+ return;
+diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
+index 159af6c3ffb05..d30bd21697a3b 100644
+--- a/net/mac80211/ieee80211_i.h
++++ b/net/mac80211/ieee80211_i.h
+@@ -376,7 +376,7 @@ struct ieee80211_mgd_auth_data {
+
+ u8 key[WLAN_KEY_LEN_WEP104];
+ u8 key_len, key_idx;
+- bool done;
++ bool done, waiting;
+ bool peer_confirmed;
+ bool timeout_started;
+
+@@ -648,6 +648,26 @@ struct mesh_csa_settings {
+ struct cfg80211_csa_settings settings;
+ };
+
++/**
++ * struct mesh_table
++ *
++ * @known_gates: list of known mesh gates and their mpaths by the station. The
++ * gate's mpath may or may not be resolved and active.
++ * @gates_lock: protects updates to known_gates
++ * @rhead: the rhashtable containing struct mesh_paths, keyed by dest addr
++ * @walk_head: linked list containing all mesh_path objects
++ * @walk_lock: lock protecting walk_head
++ * @entries: number of entries in the table
++ */
++struct mesh_table {
++ struct hlist_head known_gates;
++ spinlock_t gates_lock;
++ struct rhashtable rhead;
++ struct hlist_head walk_head;
++ spinlock_t walk_lock;
++ atomic_t entries; /* Up to MAX_MESH_NEIGHBOURS */
++};
++
+ struct ieee80211_if_mesh {
+ struct timer_list housekeeping_timer;
+ struct timer_list mesh_path_timer;
+@@ -722,8 +742,8 @@ struct ieee80211_if_mesh {
+ /* offset from skb->data while building IE */
+ int meshconf_offset;
+
+- struct mesh_table *mesh_paths;
+- struct mesh_table *mpp_paths; /* Store paths for MPP&MAP */
++ struct mesh_table mesh_paths;
++ struct mesh_table mpp_paths; /* Store paths for MPP&MAP */
+ int mesh_paths_generation;
+ int mpp_paths_generation;
+ };
+@@ -2359,7 +2379,7 @@ u8 *ieee80211_ie_build_vht_cap(u8 *pos, struct ieee80211_sta_vht_cap *vht_cap,
+ u8 *ieee80211_ie_build_vht_oper(u8 *pos, struct ieee80211_sta_vht_cap *vht_cap,
+ const struct cfg80211_chan_def *chandef);
+ u8 ieee80211_ie_len_he_cap(struct ieee80211_sub_if_data *sdata, u8 iftype);
+-u8 *ieee80211_ie_build_he_cap(u8 *pos,
++u8 *ieee80211_ie_build_he_cap(u32 disable_flags, u8 *pos,
+ const struct ieee80211_sta_he_cap *he_cap,
+ u8 *end);
+ void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata,
+diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
+index 62c95597704b4..041859b5b71d0 100644
+--- a/net/mac80211/iface.c
++++ b/net/mac80211/iface.c
+@@ -588,7 +588,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do
+ */
+ if (local->suspended) {
+ WARN_ON(local->wowlan);
+- WARN_ON(rtnl_dereference(local->monitor_sdata));
++ WARN_ON(rcu_access_pointer(local->monitor_sdata));
+ return;
+ }
+
+@@ -932,6 +932,7 @@ int ieee80211_add_virtual_monitor(struct ieee80211_local *local)
+ return 0;
+
+ ASSERT_RTNL();
++ lockdep_assert_wiphy(local->hw.wiphy);
+
+ if (local->monitor_sdata)
+ return 0;
+@@ -999,6 +1000,7 @@ void ieee80211_del_virtual_monitor(struct ieee80211_local *local)
+ return;
+
+ ASSERT_RTNL();
++ lockdep_assert_wiphy(local->hw.wiphy);
+
+ mutex_lock(&local->iflist_mtx);
+
+diff --git a/net/mac80211/led.h b/net/mac80211/led.h
+index fb3aaa3c56069..b71a1428d883c 100644
+--- a/net/mac80211/led.h
++++ b/net/mac80211/led.h
+@@ -72,19 +72,19 @@ static inline void ieee80211_mod_tpt_led_trig(struct ieee80211_local *local,
+ #endif
+
+ static inline void
+-ieee80211_tpt_led_trig_tx(struct ieee80211_local *local, __le16 fc, int bytes)
++ieee80211_tpt_led_trig_tx(struct ieee80211_local *local, int bytes)
+ {
+ #ifdef CONFIG_MAC80211_LEDS
+- if (ieee80211_is_data(fc) && atomic_read(&local->tpt_led_active))
++ if (atomic_read(&local->tpt_led_active))
+ local->tpt_led_trigger->tx_bytes += bytes;
+ #endif
+ }
+
+ static inline void
+-ieee80211_tpt_led_trig_rx(struct ieee80211_local *local, __le16 fc, int bytes)
++ieee80211_tpt_led_trig_rx(struct ieee80211_local *local, int bytes)
+ {
+ #ifdef CONFIG_MAC80211_LEDS
+- if (ieee80211_is_data(fc) && atomic_read(&local->tpt_led_active))
++ if (atomic_read(&local->tpt_led_active))
+ local->tpt_led_trigger->rx_bytes += bytes;
+ #endif
+ }
+diff --git a/net/mac80211/main.c b/net/mac80211/main.c
+index 45fb517591ee9..5311c3cd3050d 100644
+--- a/net/mac80211/main.c
++++ b/net/mac80211/main.c
+@@ -1131,17 +1131,14 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
+ local->scan_ies_len +=
+ 2 + sizeof(struct ieee80211_vht_cap);
+
+- /* HE cap element is variable in size - set len to allow max size */
+ /*
+- * TODO: 1 is added at the end of the calculation to accommodate for
+- * the temporary placing of the HE capabilities IE under EXT.
+- * Remove it once it is placed in the final place.
+- */
+- if (supp_he)
++ * HE cap element is variable in size - set len to allow max size */
++ if (supp_he) {
+ local->scan_ies_len +=
+- 2 + sizeof(struct ieee80211_he_cap_elem) +
++ 3 + sizeof(struct ieee80211_he_cap_elem) +
+ sizeof(struct ieee80211_he_mcs_nss_supp) +
+- IEEE80211_HE_PPE_THRES_MAX_LEN + 1;
++ IEEE80211_HE_PPE_THRES_MAX_LEN;
++ }
+
+ if (!local->ops->hw_scan) {
+ /* For hw_scan, driver needs to set these up. */
+diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c
+index 5dcfd53a4ab6c..42bd81a303101 100644
+--- a/net/mac80211/mesh.c
++++ b/net/mac80211/mesh.c
+@@ -580,7 +580,7 @@ int mesh_add_he_cap_ie(struct ieee80211_sub_if_data *sdata,
+ return -ENOMEM;
+
+ pos = skb_put(skb, ie_len);
+- ieee80211_ie_build_he_cap(pos, he_cap, pos + ie_len);
++ ieee80211_ie_build_he_cap(0, pos, he_cap, pos + ie_len);
+
+ return 0;
+ }
+diff --git a/net/mac80211/mesh.h b/net/mac80211/mesh.h
+index 77080b4f87b8a..b2b717a78114f 100644
+--- a/net/mac80211/mesh.h
++++ b/net/mac80211/mesh.h
+@@ -127,26 +127,6 @@ struct mesh_path {
+ u32 path_change_count;
+ };
+
+-/**
+- * struct mesh_table
+- *
+- * @known_gates: list of known mesh gates and their mpaths by the station. The
+- * gate's mpath may or may not be resolved and active.
+- * @gates_lock: protects updates to known_gates
+- * @rhead: the rhashtable containing struct mesh_paths, keyed by dest addr
+- * @walk_head: linked list containing all mesh_path objects
+- * @walk_lock: lock protecting walk_head
+- * @entries: number of entries in the table
+- */
+-struct mesh_table {
+- struct hlist_head known_gates;
+- spinlock_t gates_lock;
+- struct rhashtable rhead;
+- struct hlist_head walk_head;
+- spinlock_t walk_lock;
+- atomic_t entries; /* Up to MAX_MESH_NEIGHBOURS */
+-};
+-
+ /* Recent multicast cache */
+ /* RMC_BUCKETS must be a power of 2, maximum 256 */
+ #define RMC_BUCKETS 256
+@@ -308,7 +288,7 @@ int mesh_path_error_tx(struct ieee80211_sub_if_data *sdata,
+ void mesh_path_assign_nexthop(struct mesh_path *mpath, struct sta_info *sta);
+ void mesh_path_flush_pending(struct mesh_path *mpath);
+ void mesh_path_tx_pending(struct mesh_path *mpath);
+-int mesh_pathtbl_init(struct ieee80211_sub_if_data *sdata);
++void mesh_pathtbl_init(struct ieee80211_sub_if_data *sdata);
+ void mesh_pathtbl_unregister(struct ieee80211_sub_if_data *sdata);
+ int mesh_path_del(struct ieee80211_sub_if_data *sdata, const u8 *addr);
+ void mesh_path_timer(struct timer_list *t);
+diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c
+index 7cab1cf09bf1a..acc1c299f1ae5 100644
+--- a/net/mac80211/mesh_pathtbl.c
++++ b/net/mac80211/mesh_pathtbl.c
+@@ -47,32 +47,24 @@ static void mesh_path_rht_free(void *ptr, void *tblptr)
+ mesh_path_free_rcu(tbl, mpath);
+ }
+
+-static struct mesh_table *mesh_table_alloc(void)
++static void mesh_table_init(struct mesh_table *tbl)
+ {
+- struct mesh_table *newtbl;
++ INIT_HLIST_HEAD(&tbl->known_gates);
++ INIT_HLIST_HEAD(&tbl->walk_head);
++ atomic_set(&tbl->entries, 0);
++ spin_lock_init(&tbl->gates_lock);
++ spin_lock_init(&tbl->walk_lock);
+
+- newtbl = kmalloc(sizeof(struct mesh_table), GFP_ATOMIC);
+- if (!newtbl)
+- return NULL;
+-
+- INIT_HLIST_HEAD(&newtbl->known_gates);
+- INIT_HLIST_HEAD(&newtbl->walk_head);
+- atomic_set(&newtbl->entries, 0);
+- spin_lock_init(&newtbl->gates_lock);
+- spin_lock_init(&newtbl->walk_lock);
+- if (rhashtable_init(&newtbl->rhead, &mesh_rht_params)) {
+- kfree(newtbl);
+- return NULL;
+- }
+-
+- return newtbl;
++ /* rhashtable_init() may fail only in case of wrong
++ * mesh_rht_params
++ */
++ WARN_ON(rhashtable_init(&tbl->rhead, &mesh_rht_params));
+ }
+
+ static void mesh_table_free(struct mesh_table *tbl)
+ {
+ rhashtable_free_and_destroy(&tbl->rhead,
+ mesh_path_rht_free, tbl);
+- kfree(tbl);
+ }
+
+ /**
+@@ -238,13 +230,13 @@ static struct mesh_path *mpath_lookup(struct mesh_table *tbl, const u8 *dst,
+ struct mesh_path *
+ mesh_path_lookup(struct ieee80211_sub_if_data *sdata, const u8 *dst)
+ {
+- return mpath_lookup(sdata->u.mesh.mesh_paths, dst, sdata);
++ return mpath_lookup(&sdata->u.mesh.mesh_paths, dst, sdata);
+ }
+
+ struct mesh_path *
+ mpp_path_lookup(struct ieee80211_sub_if_data *sdata, const u8 *dst)
+ {
+- return mpath_lookup(sdata->u.mesh.mpp_paths, dst, sdata);
++ return mpath_lookup(&sdata->u.mesh.mpp_paths, dst, sdata);
+ }
+
+ static struct mesh_path *
+@@ -281,7 +273,7 @@ __mesh_path_lookup_by_idx(struct mesh_table *tbl, int idx)
+ struct mesh_path *
+ mesh_path_lookup_by_idx(struct ieee80211_sub_if_data *sdata, int idx)
+ {
+- return __mesh_path_lookup_by_idx(sdata->u.mesh.mesh_paths, idx);
++ return __mesh_path_lookup_by_idx(&sdata->u.mesh.mesh_paths, idx);
+ }
+
+ /**
+@@ -296,7 +288,7 @@ mesh_path_lookup_by_idx(struct ieee80211_sub_if_data *sdata, int idx)
+ struct mesh_path *
+ mpp_path_lookup_by_idx(struct ieee80211_sub_if_data *sdata, int idx)
+ {
+- return __mesh_path_lookup_by_idx(sdata->u.mesh.mpp_paths, idx);
++ return __mesh_path_lookup_by_idx(&sdata->u.mesh.mpp_paths, idx);
+ }
+
+ /**
+@@ -309,7 +301,7 @@ int mesh_path_add_gate(struct mesh_path *mpath)
+ int err;
+
+ rcu_read_lock();
+- tbl = mpath->sdata->u.mesh.mesh_paths;
++ tbl = &mpath->sdata->u.mesh.mesh_paths;
+
+ spin_lock_bh(&mpath->state_lock);
+ if (mpath->is_gate) {
+@@ -418,7 +410,7 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata,
+ if (!new_mpath)
+ return ERR_PTR(-ENOMEM);
+
+- tbl = sdata->u.mesh.mesh_paths;
++ tbl = &sdata->u.mesh.mesh_paths;
+ spin_lock_bh(&tbl->walk_lock);
+ mpath = rhashtable_lookup_get_insert_fast(&tbl->rhead,
+ &new_mpath->rhash,
+@@ -460,7 +452,7 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata,
+ return -ENOMEM;
+
+ memcpy(new_mpath->mpp, mpp, ETH_ALEN);
+- tbl = sdata->u.mesh.mpp_paths;
++ tbl = &sdata->u.mesh.mpp_paths;
+
+ spin_lock_bh(&tbl->walk_lock);
+ ret = rhashtable_lookup_insert_fast(&tbl->rhead,
+@@ -489,7 +481,7 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata,
+ void mesh_plink_broken(struct sta_info *sta)
+ {
+ struct ieee80211_sub_if_data *sdata = sta->sdata;
+- struct mesh_table *tbl = sdata->u.mesh.mesh_paths;
++ struct mesh_table *tbl = &sdata->u.mesh.mesh_paths;
+ static const u8 bcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+ struct mesh_path *mpath;
+
+@@ -548,7 +540,7 @@ static void __mesh_path_del(struct mesh_table *tbl, struct mesh_path *mpath)
+ void mesh_path_flush_by_nexthop(struct sta_info *sta)
+ {
+ struct ieee80211_sub_if_data *sdata = sta->sdata;
+- struct mesh_table *tbl = sdata->u.mesh.mesh_paths;
++ struct mesh_table *tbl = &sdata->u.mesh.mesh_paths;
+ struct mesh_path *mpath;
+ struct hlist_node *n;
+
+@@ -563,7 +555,7 @@ void mesh_path_flush_by_nexthop(struct sta_info *sta)
+ static void mpp_flush_by_proxy(struct ieee80211_sub_if_data *sdata,
+ const u8 *proxy)
+ {
+- struct mesh_table *tbl = sdata->u.mesh.mpp_paths;
++ struct mesh_table *tbl = &sdata->u.mesh.mpp_paths;
+ struct mesh_path *mpath;
+ struct hlist_node *n;
+
+@@ -597,8 +589,8 @@ static void table_flush_by_iface(struct mesh_table *tbl)
+ */
+ void mesh_path_flush_by_iface(struct ieee80211_sub_if_data *sdata)
+ {
+- table_flush_by_iface(sdata->u.mesh.mesh_paths);
+- table_flush_by_iface(sdata->u.mesh.mpp_paths);
++ table_flush_by_iface(&sdata->u.mesh.mesh_paths);
++ table_flush_by_iface(&sdata->u.mesh.mpp_paths);
+ }
+
+ /**
+@@ -644,7 +636,7 @@ int mesh_path_del(struct ieee80211_sub_if_data *sdata, const u8 *addr)
+ /* flush relevant mpp entries first */
+ mpp_flush_by_proxy(sdata, addr);
+
+- err = table_path_del(sdata->u.mesh.mesh_paths, sdata, addr);
++ err = table_path_del(&sdata->u.mesh.mesh_paths, sdata, addr);
+ sdata->u.mesh.mesh_paths_generation++;
+ return err;
+ }
+@@ -682,7 +674,7 @@ int mesh_path_send_to_gates(struct mesh_path *mpath)
+ struct mesh_path *gate;
+ bool copy = false;
+
+- tbl = sdata->u.mesh.mesh_paths;
++ tbl = &sdata->u.mesh.mesh_paths;
+
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(gate, &tbl->known_gates, gate_list) {
+@@ -762,29 +754,10 @@ void mesh_path_fix_nexthop(struct mesh_path *mpath, struct sta_info *next_hop)
+ mesh_path_tx_pending(mpath);
+ }
+
+-int mesh_pathtbl_init(struct ieee80211_sub_if_data *sdata)
++void mesh_pathtbl_init(struct ieee80211_sub_if_data *sdata)
+ {
+- struct mesh_table *tbl_path, *tbl_mpp;
+- int ret;
+-
+- tbl_path = mesh_table_alloc();
+- if (!tbl_path)
+- return -ENOMEM;
+-
+- tbl_mpp = mesh_table_alloc();
+- if (!tbl_mpp) {
+- ret = -ENOMEM;
+- goto free_path;
+- }
+-
+- sdata->u.mesh.mesh_paths = tbl_path;
+- sdata->u.mesh.mpp_paths = tbl_mpp;
+-
+- return 0;
+-
+-free_path:
+- mesh_table_free(tbl_path);
+- return ret;
++ mesh_table_init(&sdata->u.mesh.mesh_paths);
++ mesh_table_init(&sdata->u.mesh.mpp_paths);
+ }
+
+ static
+@@ -806,12 +779,12 @@ void mesh_path_tbl_expire(struct ieee80211_sub_if_data *sdata,
+
+ void mesh_path_expire(struct ieee80211_sub_if_data *sdata)
+ {
+- mesh_path_tbl_expire(sdata, sdata->u.mesh.mesh_paths);
+- mesh_path_tbl_expire(sdata, sdata->u.mesh.mpp_paths);
++ mesh_path_tbl_expire(sdata, &sdata->u.mesh.mesh_paths);
++ mesh_path_tbl_expire(sdata, &sdata->u.mesh.mpp_paths);
+ }
+
+ void mesh_pathtbl_unregister(struct ieee80211_sub_if_data *sdata)
+ {
+- mesh_table_free(sdata->u.mesh.mesh_paths);
+- mesh_table_free(sdata->u.mesh.mpp_paths);
++ mesh_table_free(&sdata->u.mesh.mesh_paths);
++ mesh_table_free(&sdata->u.mesh.mpp_paths);
+ }
+diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
+index c0ea3b1aa9e1c..1548f532dc1ad 100644
+--- a/net/mac80211/mlme.c
++++ b/net/mac80211/mlme.c
+@@ -37,6 +37,7 @@
+ #define IEEE80211_AUTH_TIMEOUT_SAE (HZ * 2)
+ #define IEEE80211_AUTH_MAX_TRIES 3
+ #define IEEE80211_AUTH_WAIT_ASSOC (HZ * 5)
++#define IEEE80211_AUTH_WAIT_SAE_RETRY (HZ * 2)
+ #define IEEE80211_ASSOC_TIMEOUT (HZ / 5)
+ #define IEEE80211_ASSOC_TIMEOUT_LONG (HZ / 2)
+ #define IEEE80211_ASSOC_TIMEOUT_SHORT (HZ / 10)
+@@ -629,7 +630,7 @@ static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata,
+ struct sk_buff *skb,
+ struct ieee80211_supported_band *sband)
+ {
+- u8 *pos;
++ u8 *pos, *pre_he_pos;
+ const struct ieee80211_sta_he_cap *he_cap = NULL;
+ struct ieee80211_chanctx_conf *chanctx_conf;
+ u8 he_cap_size;
+@@ -646,25 +647,26 @@ static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata,
+
+ he_cap = ieee80211_get_he_iftype_cap(sband,
+ ieee80211_vif_type_p2p(&sdata->vif));
+- if (!he_cap || !reg_cap)
++ if (!he_cap || !chanctx_conf || !reg_cap)
+ return;
+
+- /*
+- * TODO: the 1 added is because this temporarily is under the EXTENSION
+- * IE. Get rid of it when it moves.
+- */
++ /* get a max size estimate */
+ he_cap_size =
+ 2 + 1 + sizeof(he_cap->he_cap_elem) +
+ ieee80211_he_mcs_nss_size(&he_cap->he_cap_elem) +
+ ieee80211_he_ppe_size(he_cap->ppe_thres[0],
+ he_cap->he_cap_elem.phy_cap_info);
+ pos = skb_put(skb, he_cap_size);
+- ieee80211_ie_build_he_cap(pos, he_cap, pos + he_cap_size);
++ pre_he_pos = pos;
++ pos = ieee80211_ie_build_he_cap(sdata->u.mgd.flags,
++ pos, he_cap, pos + he_cap_size);
++ /* trim excess if any */
++ skb_trim(skb, skb->len - (pre_he_pos + he_cap_size - pos));
+
+ ieee80211_ie_build_he_6ghz_cap(sdata, skb);
+ }
+
+-static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
++static int ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
+ {
+ struct ieee80211_local *local = sdata->local;
+ struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
+@@ -684,6 +686,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
+ enum nl80211_iftype iftype = ieee80211_vif_type_p2p(&sdata->vif);
+ const struct ieee80211_sband_iftype_data *iftd;
+ struct ieee80211_prep_tx_info info = {};
++ int ret;
+
+ /* we know it's writable, cast away the const */
+ if (assoc_data->ie_len)
+@@ -697,7 +700,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
+ chanctx_conf = rcu_dereference(sdata->vif.chanctx_conf);
+ if (WARN_ON(!chanctx_conf)) {
+ rcu_read_unlock();
+- return;
++ return -EINVAL;
+ }
+ chan = chanctx_conf->def.chan;
+ rcu_read_unlock();
+@@ -748,7 +751,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
+ (iftd ? iftd->vendor_elems.len : 0),
+ GFP_KERNEL);
+ if (!skb)
+- return;
++ return -ENOMEM;
+
+ skb_reserve(skb, local->hw.extra_tx_headroom);
+
+@@ -1029,15 +1032,22 @@ skip_rates:
+ skb_put_data(skb, assoc_data->ie + offset, noffset - offset);
+ }
+
+- if (assoc_data->fils_kek_len &&
+- fils_encrypt_assoc_req(skb, assoc_data) < 0) {
+- dev_kfree_skb(skb);
+- return;
++ if (assoc_data->fils_kek_len) {
++ ret = fils_encrypt_assoc_req(skb, assoc_data);
++ if (ret < 0) {
++ dev_kfree_skb(skb);
++ return ret;
++ }
+ }
+
+ pos = skb_tail_pointer(skb);
+ kfree(ifmgd->assoc_req_ies);
+ ifmgd->assoc_req_ies = kmemdup(ie_start, pos - ie_start, GFP_ATOMIC);
++ if (!ifmgd->assoc_req_ies) {
++ dev_kfree_skb(skb);
++ return -ENOMEM;
++ }
++
+ ifmgd->assoc_req_ies_len = pos - ie_start;
+
+ drv_mgd_prepare_tx(local, sdata, &info);
+@@ -1047,6 +1057,8 @@ skip_rates:
+ IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS |
+ IEEE80211_TX_INTFL_MLME_CONN_TX;
+ ieee80211_tx_skb(sdata, skb);
++
++ return 0;
+ }
+
+ void ieee80211_send_pspoll(struct ieee80211_local *local,
+@@ -2446,11 +2458,18 @@ static void ieee80211_sta_tx_wmm_ac_notify(struct ieee80211_sub_if_data *sdata,
+ u16 tx_time)
+ {
+ struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
+- u16 tid = ieee80211_get_tid(hdr);
+- int ac = ieee80211_ac_from_tid(tid);
+- struct ieee80211_sta_tx_tspec *tx_tspec = &ifmgd->tx_tspec[ac];
++ u16 tid;
++ int ac;
++ struct ieee80211_sta_tx_tspec *tx_tspec;
+ unsigned long now = jiffies;
+
++ if (!ieee80211_is_data_qos(hdr->frame_control))
++ return;
++
++ tid = ieee80211_get_tid(hdr);
++ ac = ieee80211_ac_from_tid(tid);
++ tx_tspec = &ifmgd->tx_tspec[ac];
++
+ if (likely(!tx_tspec->admitted_time))
+ return;
+
+@@ -2977,8 +2996,15 @@ static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata,
+ (status_code == WLAN_STATUS_ANTI_CLOG_REQUIRED ||
+ (auth_transaction == 1 &&
+ (status_code == WLAN_STATUS_SAE_HASH_TO_ELEMENT ||
+- status_code == WLAN_STATUS_SAE_PK))))
++ status_code == WLAN_STATUS_SAE_PK)))) {
++ /* waiting for userspace now */
++ ifmgd->auth_data->waiting = true;
++ ifmgd->auth_data->timeout =
++ jiffies + IEEE80211_AUTH_WAIT_SAE_RETRY;
++ ifmgd->auth_data->timeout_started = true;
++ run_again(sdata, ifmgd->auth_data->timeout);
+ goto notify_driver;
++ }
+
+ sdata_info(sdata, "%pM denied authentication (status %d)\n",
+ mgmt->sa, status_code);
+@@ -3515,6 +3541,12 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
+ cbss->transmitted_bss->bssid);
+ bss_conf->bssid_indicator = cbss->max_bssid_indicator;
+ bss_conf->bssid_index = cbss->bssid_index;
++ } else {
++ bss_conf->nontransmitted = false;
++ memset(bss_conf->transmitter_bssid, 0,
++ sizeof(bss_conf->transmitter_bssid));
++ bss_conf->bssid_indicator = 0;
++ bss_conf->bssid_index = 0;
+ }
+
+ /*
+@@ -4444,6 +4476,7 @@ static int ieee80211_do_assoc(struct ieee80211_sub_if_data *sdata)
+ {
+ struct ieee80211_mgd_assoc_data *assoc_data = sdata->u.mgd.assoc_data;
+ struct ieee80211_local *local = sdata->local;
++ int ret;
+
+ sdata_assert_lock(sdata);
+
+@@ -4464,7 +4497,9 @@ static int ieee80211_do_assoc(struct ieee80211_sub_if_data *sdata)
+ sdata_info(sdata, "associate with %pM (try %d/%d)\n",
+ assoc_data->bss->bssid, assoc_data->tries,
+ IEEE80211_ASSOC_MAX_TRIES);
+- ieee80211_send_assoc(sdata);
++ ret = ieee80211_send_assoc(sdata);
++ if (ret)
++ return ret;
+
+ if (!ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
+ assoc_data->timeout = jiffies + IEEE80211_ASSOC_TIMEOUT;
+@@ -4537,10 +4572,10 @@ void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata)
+
+ if (ifmgd->auth_data && ifmgd->auth_data->timeout_started &&
+ time_after(jiffies, ifmgd->auth_data->timeout)) {
+- if (ifmgd->auth_data->done) {
++ if (ifmgd->auth_data->done || ifmgd->auth_data->waiting) {
+ /*
+- * ok ... we waited for assoc but userspace didn't,
+- * so let's just kill the auth data
++ * ok ... we waited for assoc or continuation but
++ * userspace didn't do it, so kill the auth data
+ */
+ ieee80211_destroy_auth_data(sdata, false);
+ } else if (ieee80211_auth(sdata)) {
+@@ -5209,7 +5244,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
+ */
+ if (new_sta) {
+ u32 rates = 0, basic_rates = 0;
+- bool have_higher_than_11mbit;
++ bool have_higher_than_11mbit = false;
+ int min_rate = INT_MAX, min_rate_index = -1;
+ const struct cfg80211_bss_ies *ies;
+ int shift = ieee80211_vif_get_shift(&sdata->vif);
+diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
+index c4071b015c188..743e97ba352c8 100644
+--- a/net/mac80211/rx.c
++++ b/net/mac80211/rx.c
+@@ -364,7 +364,7 @@ ieee80211_add_rx_radiotap_header(struct ieee80211_local *local,
+ * the compiler to think we have walked past the end of the
+ * struct member.
+ */
+- pos = (void *)&rthdr->it_optional[it_present - rthdr->it_optional];
++ pos = (void *)&rthdr->it_optional[it_present + 1 - rthdr->it_optional];
+
+ /* the order of the following fields is important */
+
+@@ -1400,8 +1400,7 @@ static void ieee80211_rx_reorder_ampdu(struct ieee80211_rx_data *rx,
+ goto dont_reorder;
+
+ /* not part of a BA session */
+- if (ack_policy != IEEE80211_QOS_CTL_ACK_POLICY_BLOCKACK &&
+- ack_policy != IEEE80211_QOS_CTL_ACK_POLICY_NORMAL)
++ if (ack_policy == IEEE80211_QOS_CTL_ACK_POLICY_NOACK)
+ goto dont_reorder;
+
+ /* new, potentially un-ordered, ampdu frame - process it */
+@@ -1952,7 +1951,8 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx)
+ int keyid = rx->sta->ptk_idx;
+ sta_ptk = rcu_dereference(rx->sta->ptk[keyid]);
+
+- if (ieee80211_has_protected(fc)) {
++ if (ieee80211_has_protected(fc) &&
++ !(status->flag & RX_FLAG_IV_STRIPPED)) {
+ cs = rx->sta->cipher_scheme;
+ keyid = ieee80211_get_keyid(rx->skb, cs);
+
+@@ -2601,7 +2601,8 @@ static void ieee80211_deliver_skb_to_local_stack(struct sk_buff *skb,
+ * address, so that the authenticator (e.g. hostapd) will see
+ * the frame, but bridge won't forward it anywhere else. Note
+ * that due to earlier filtering, the only other address can
+- * be the PAE group address.
++ * be the PAE group address, unless the hardware allowed them
++ * through in 802.3 offloaded mode.
+ */
+ if (unlikely(skb->protocol == sdata->control_port_protocol &&
+ !ether_addr_equal(ehdr->h_dest, sdata->vif.addr)))
+@@ -2916,13 +2917,13 @@ ieee80211_rx_h_mesh_fwding(struct ieee80211_rx_data *rx)
+ ether_addr_equal(sdata->vif.addr, hdr->addr3))
+ return RX_CONTINUE;
+
+- ac = ieee80211_select_queue_80211(sdata, skb, hdr);
++ ac = ieee802_1d_to_ac[skb->priority];
+ q = sdata->vif.hw_queue[ac];
+ if (ieee80211_queue_stopped(&local->hw, q)) {
+ IEEE80211_IFSTA_MESH_CTR_INC(ifmsh, dropped_frames_congestion);
+ return RX_DROP_MONITOR;
+ }
+- skb_set_queue_mapping(skb, q);
++ skb_set_queue_mapping(skb, ac);
+
+ if (!--mesh_hdr->ttl) {
+ if (!is_multicast_ether_addr(hdr->addr1))
+@@ -4517,12 +4518,7 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx,
+
+ /* deliver to local stack */
+ skb->protocol = eth_type_trans(skb, fast_rx->dev);
+- memset(skb->cb, 0, sizeof(skb->cb));
+- if (rx->list)
+- list_add_tail(&skb->list, rx->list);
+- else
+- netif_receive_skb(skb);
+-
++ ieee80211_deliver_skb_to_local_stack(skb, rx);
+ }
+
+ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
+@@ -4873,6 +4869,7 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
+ struct ieee80211_rate *rate = NULL;
+ struct ieee80211_supported_band *sband;
+ struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
++ struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+
+ WARN_ON_ONCE(softirq_count() == 0);
+
+@@ -4931,7 +4928,7 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
+ goto drop;
+ break;
+ case RX_ENC_VHT:
+- if (WARN_ONCE(status->rate_idx > 9 ||
++ if (WARN_ONCE(status->rate_idx > 11 ||
+ !status->nss ||
+ status->nss > 8,
+ "Rate marked as a VHT rate but data is invalid: MCS: %d, NSS: %d\n",
+@@ -4969,9 +4966,9 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
+ if (!(status->flag & RX_FLAG_8023))
+ skb = ieee80211_rx_monitor(local, skb, rate);
+ if (skb) {
+- ieee80211_tpt_led_trig_rx(local,
+- ((struct ieee80211_hdr *)skb->data)->frame_control,
+- skb->len);
++ if ((status->flag & RX_FLAG_8023) ||
++ ieee80211_is_data_present(hdr->frame_control))
++ ieee80211_tpt_led_trig_rx(local, skb->len);
+
+ if (status->flag & RX_FLAG_8023)
+ __ieee80211_rx_handle_8023(hw, pubsta, skb, list);
+diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c
+index 7e35ab5b61664..4141bc80cdfd6 100644
+--- a/net/mac80211/s1g.c
++++ b/net/mac80211/s1g.c
+@@ -104,9 +104,11 @@ ieee80211_s1g_rx_twt_setup(struct ieee80211_sub_if_data *sdata,
+
+ /* broadcast TWT not supported yet */
+ if (twt->control & IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST) {
+- le16p_replace_bits(&twt_agrt->req_type,
+- TWT_SETUP_CMD_REJECT,
+- IEEE80211_TWT_REQTYPE_SETUP_CMD);
++ twt_agrt->req_type &=
++ ~cpu_to_le16(IEEE80211_TWT_REQTYPE_SETUP_CMD);
++ twt_agrt->req_type |=
++ le16_encode_bits(TWT_SETUP_CMD_REJECT,
++ IEEE80211_TWT_REQTYPE_SETUP_CMD);
+ goto out;
+ }
+
+diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h
+index ba27967820084..e7443fc4669c8 100644
+--- a/net/mac80211/sta_info.h
++++ b/net/mac80211/sta_info.h
+@@ -199,6 +199,7 @@ struct tid_ampdu_tx {
+ u8 stop_initiator;
+ bool tx_stop;
+ u16 buf_size;
++ u16 ssn;
+
+ u16 failed_bar_ssn;
+ bool bar_pending;
+diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
+index 8921088a5df65..a499b07fee336 100644
+--- a/net/mac80211/tx.c
++++ b/net/mac80211/tx.c
+@@ -1720,21 +1720,19 @@ static bool ieee80211_tx_frags(struct ieee80211_local *local,
+ * Returns false if the frame couldn't be transmitted but was queued instead.
+ */
+ static bool __ieee80211_tx(struct ieee80211_local *local,
+- struct sk_buff_head *skbs, int led_len,
+- struct sta_info *sta, bool txpending)
++ struct sk_buff_head *skbs, struct sta_info *sta,
++ bool txpending)
+ {
+ struct ieee80211_tx_info *info;
+ struct ieee80211_sub_if_data *sdata;
+ struct ieee80211_vif *vif;
+ struct sk_buff *skb;
+ bool result;
+- __le16 fc;
+
+ if (WARN_ON(skb_queue_empty(skbs)))
+ return true;
+
+ skb = skb_peek(skbs);
+- fc = ((struct ieee80211_hdr *)skb->data)->frame_control;
+ info = IEEE80211_SKB_CB(skb);
+ sdata = vif_to_sdata(info->control.vif);
+ if (sta && !sta->uploaded)
+@@ -1768,8 +1766,6 @@ static bool __ieee80211_tx(struct ieee80211_local *local,
+
+ result = ieee80211_tx_frags(local, vif, sta, skbs, txpending);
+
+- ieee80211_tpt_led_trig_tx(local, fc, led_len);
+-
+ WARN_ON_ONCE(!skb_queue_empty(skbs));
+
+ return result;
+@@ -1825,15 +1821,15 @@ static int invoke_tx_handlers_late(struct ieee80211_tx_data *tx)
+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
+ ieee80211_tx_result res = TX_CONTINUE;
+
++ if (!ieee80211_hw_check(&tx->local->hw, HAS_RATE_CONTROL))
++ CALL_TXH(ieee80211_tx_h_rate_ctrl);
++
+ if (unlikely(info->flags & IEEE80211_TX_INTFL_RETRANSMISSION)) {
+ __skb_queue_tail(&tx->skbs, tx->skb);
+ tx->skb = NULL;
+ goto txh_done;
+ }
+
+- if (!ieee80211_hw_check(&tx->local->hw, HAS_RATE_CONTROL))
+- CALL_TXH(ieee80211_tx_h_rate_ctrl);
+-
+ CALL_TXH(ieee80211_tx_h_michael_mic_add);
+ CALL_TXH(ieee80211_tx_h_sequence);
+ CALL_TXH(ieee80211_tx_h_fragment);
+@@ -1919,7 +1915,6 @@ static bool ieee80211_tx(struct ieee80211_sub_if_data *sdata,
+ ieee80211_tx_result res_prepare;
+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+ bool result = true;
+- int led_len;
+
+ if (unlikely(skb->len < 10)) {
+ dev_kfree_skb(skb);
+@@ -1927,7 +1922,6 @@ static bool ieee80211_tx(struct ieee80211_sub_if_data *sdata,
+ }
+
+ /* initialises tx */
+- led_len = skb->len;
+ res_prepare = ieee80211_tx_prepare(sdata, &tx, sta, skb);
+
+ if (unlikely(res_prepare == TX_DROP)) {
+@@ -1950,8 +1944,7 @@ static bool ieee80211_tx(struct ieee80211_sub_if_data *sdata,
+ return true;
+
+ if (!invoke_tx_handlers_late(&tx))
+- result = __ieee80211_tx(local, &tx.skbs, led_len,
+- tx.sta, txpending);
++ result = __ieee80211_tx(local, &tx.skbs, tx.sta, txpending);
+
+ return result;
+ }
+@@ -4174,6 +4167,7 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb,
+ struct ieee80211_local *local = sdata->local;
+ struct sta_info *sta;
+ struct sk_buff *next;
++ int len = skb->len;
+
+ if (unlikely(skb->len < ETH_HLEN)) {
+ kfree_skb(skb);
+@@ -4220,10 +4214,8 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb,
+ }
+ } else {
+ /* we cannot process non-linear frames on this path */
+- if (skb_linearize(skb)) {
+- kfree_skb(skb);
+- goto out;
+- }
++ if (skb_linearize(skb))
++ goto out_free;
+
+ /* the frame could be fragmented, software-encrypted, and other
+ * things so we cannot really handle checksum offload with it -
+@@ -4257,7 +4249,10 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb,
+ goto out;
+ out_free:
+ kfree_skb(skb);
++ len = 0;
+ out:
++ if (len)
++ ieee80211_tpt_led_trig_tx(local, len);
+ rcu_read_unlock();
+ }
+
+@@ -4395,8 +4390,7 @@ netdev_tx_t ieee80211_subif_start_xmit(struct sk_buff *skb,
+ }
+
+ static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata,
+- struct sk_buff *skb, int led_len,
+- struct sta_info *sta,
++ struct sk_buff *skb, struct sta_info *sta,
+ bool txpending)
+ {
+ struct ieee80211_local *local = sdata->local;
+@@ -4409,6 +4403,8 @@ static bool ieee80211_tx_8023(struct ieee80211_sub_if_data *sdata,
+ if (sta)
+ sk_pacing_shift_update(skb->sk, local->hw.tx_sk_pacing_shift);
+
++ ieee80211_tpt_led_trig_tx(local, skb->len);
++
+ if (ieee80211_queue_skb(local, sdata, sta, skb))
+ return true;
+
+@@ -4497,7 +4493,7 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
+ if (key)
+ info->control.hw_key = &key->conf;
+
+- ieee80211_tx_8023(sdata, skb, skb->len, sta, false);
++ ieee80211_tx_8023(sdata, skb, sta, false);
+
+ return;
+
+@@ -4636,7 +4632,7 @@ static bool ieee80211_tx_pending_skb(struct ieee80211_local *local,
+ if (IS_ERR(sta) || (sta && !sta->uploaded))
+ sta = NULL;
+
+- result = ieee80211_tx_8023(sdata, skb, skb->len, sta, true);
++ result = ieee80211_tx_8023(sdata, skb, sta, true);
+ } else {
+ struct sk_buff_head skbs;
+
+@@ -4646,7 +4642,7 @@ static bool ieee80211_tx_pending_skb(struct ieee80211_local *local,
+ hdr = (struct ieee80211_hdr *)skb->data;
+ sta = sta_info_get(sdata, hdr->addr1);
+
+- result = __ieee80211_tx(local, &skbs, skb->len, sta, true);
++ result = __ieee80211_tx(local, &skbs, sta, true);
+ }
+
+ return result;
+diff --git a/net/mac80211/util.c b/net/mac80211/util.c
+index 49cb96d251695..be1911d8089fa 100644
+--- a/net/mac80211/util.c
++++ b/net/mac80211/util.c
+@@ -796,7 +796,7 @@ static void __iterate_interfaces(struct ieee80211_local *local,
+
+ sdata = rcu_dereference_check(local->monitor_sdata,
+ lockdep_is_held(&local->iflist_mtx) ||
+- lockdep_rtnl_is_held());
++ lockdep_is_held(&local->hw.wiphy->mtx));
+ if (sdata &&
+ (iter_flags & IEEE80211_IFACE_ITER_RESUME_ALL || !active_only ||
+ sdata->flags & IEEE80211_SDATA_IN_DRIVER))
+@@ -943,7 +943,12 @@ static void ieee80211_parse_extension_element(u32 *crc,
+ struct ieee802_11_elems *elems)
+ {
+ const void *data = elem->data + 1;
+- u8 len = elem->datalen - 1;
++ u8 len;
++
++ if (!elem->datalen)
++ return;
++
++ len = elem->datalen - 1;
+
+ switch (elem->data[0]) {
+ case WLAN_EID_EXT_HE_MU_EDCA:
+@@ -1954,7 +1959,7 @@ static int ieee80211_build_preq_ies_band(struct ieee80211_sub_if_data *sdata,
+ if (he_cap &&
+ cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band),
+ IEEE80211_CHAN_NO_HE)) {
+- pos = ieee80211_ie_build_he_cap(pos, he_cap, end);
++ pos = ieee80211_ie_build_he_cap(0, pos, he_cap, end);
+ if (!pos)
+ goto out_err;
+ }
+@@ -2379,7 +2384,7 @@ int ieee80211_reconfig(struct ieee80211_local *local)
+ IEEE80211_TPT_LEDTRIG_FL_RADIO, 0);
+
+ /* add interfaces */
+- sdata = rtnl_dereference(local->monitor_sdata);
++ sdata = wiphy_dereference(local->hw.wiphy, local->monitor_sdata);
+ if (sdata) {
+ /* in HW restart it exists already */
+ WARN_ON(local->resuming);
+@@ -2424,7 +2429,8 @@ int ieee80211_reconfig(struct ieee80211_local *local)
+ WARN_ON(drv_add_chanctx(local, ctx));
+ mutex_unlock(&local->chanctx_mtx);
+
+- sdata = rtnl_dereference(local->monitor_sdata);
++ sdata = wiphy_dereference(local->hw.wiphy,
++ local->monitor_sdata);
+ if (sdata && ieee80211_sdata_running(sdata))
+ ieee80211_assign_chanctx(local, sdata);
+ }
+@@ -2897,10 +2903,11 @@ u8 ieee80211_ie_len_he_cap(struct ieee80211_sub_if_data *sdata, u8 iftype)
+ he_cap->he_cap_elem.phy_cap_info);
+ }
+
+-u8 *ieee80211_ie_build_he_cap(u8 *pos,
++u8 *ieee80211_ie_build_he_cap(u32 disable_flags, u8 *pos,
+ const struct ieee80211_sta_he_cap *he_cap,
+ u8 *end)
+ {
++ struct ieee80211_he_cap_elem elem;
+ u8 n;
+ u8 ie_len;
+ u8 *orig_pos = pos;
+@@ -2913,7 +2920,23 @@ u8 *ieee80211_ie_build_he_cap(u8 *pos,
+ if (!he_cap)
+ return orig_pos;
+
+- n = ieee80211_he_mcs_nss_size(&he_cap->he_cap_elem);
++ /* modify on stack first to calculate 'n' and 'ie_len' correctly */
++ elem = he_cap->he_cap_elem;
++
++ if (disable_flags & IEEE80211_STA_DISABLE_40MHZ)
++ elem.phy_cap_info[0] &=
++ ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
++ IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G);
++
++ if (disable_flags & IEEE80211_STA_DISABLE_160MHZ)
++ elem.phy_cap_info[0] &=
++ ~IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
++
++ if (disable_flags & IEEE80211_STA_DISABLE_80P80MHZ)
++ elem.phy_cap_info[0] &=
++ ~IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G;
++
++ n = ieee80211_he_mcs_nss_size(&elem);
+ ie_len = 2 + 1 +
+ sizeof(he_cap->he_cap_elem) + n +
+ ieee80211_he_ppe_size(he_cap->ppe_thres[0],
+@@ -2927,8 +2950,8 @@ u8 *ieee80211_ie_build_he_cap(u8 *pos,
+ *pos++ = WLAN_EID_EXT_HE_CAPABILITY;
+
+ /* Fixed data */
+- memcpy(pos, &he_cap->he_cap_elem, sizeof(he_cap->he_cap_elem));
+- pos += sizeof(he_cap->he_cap_elem);
++ memcpy(pos, &elem, sizeof(elem));
++ pos += sizeof(elem);
+
+ memcpy(pos, &he_cap->he_mcs_nss_supp, n);
+ pos += n;
+diff --git a/net/mac80211/wme.c b/net/mac80211/wme.c
+index 9ea6004abe1be..62c6733e07923 100644
+--- a/net/mac80211/wme.c
++++ b/net/mac80211/wme.c
+@@ -143,7 +143,6 @@ u16 ieee80211_select_queue_80211(struct ieee80211_sub_if_data *sdata,
+ u16 __ieee80211_select_queue(struct ieee80211_sub_if_data *sdata,
+ struct sta_info *sta, struct sk_buff *skb)
+ {
+- struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+ struct mac80211_qos_map *qos_map;
+ bool qos;
+
+@@ -156,7 +155,7 @@ u16 __ieee80211_select_queue(struct ieee80211_sub_if_data *sdata,
+ else
+ qos = false;
+
+- if (!qos || (info->control.flags & IEEE80211_TX_CTRL_DONT_REORDER)) {
++ if (!qos) {
+ skb->priority = 0; /* required for correct WPA/11i MIC */
+ return IEEE80211_AC_BE;
+ }
+diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c
+index a9526ac29dffe..85cc1a28cbe9f 100644
+--- a/net/mctp/af_mctp.c
++++ b/net/mctp/af_mctp.c
+@@ -30,6 +30,12 @@ static int mctp_release(struct socket *sock)
+ return 0;
+ }
+
++/* Generic sockaddr checks, padding checks only so far */
++static bool mctp_sockaddr_is_ok(const struct sockaddr_mctp *addr)
++{
++ return !addr->__smctp_pad0 && !addr->__smctp_pad1;
++}
++
+ static int mctp_bind(struct socket *sock, struct sockaddr *addr, int addrlen)
+ {
+ struct sock *sk = sock->sk;
+@@ -49,6 +55,9 @@ static int mctp_bind(struct socket *sock, struct sockaddr *addr, int addrlen)
+ /* it's a valid sockaddr for MCTP, cast and do protocol checks */
+ smctp = (struct sockaddr_mctp *)addr;
+
++ if (!mctp_sockaddr_is_ok(smctp))
++ return -EINVAL;
++
+ lock_sock(sk);
+
+ /* TODO: allow rebind */
+@@ -83,6 +92,8 @@ static int mctp_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
+ return -EINVAL;
+ if (addr->smctp_family != AF_MCTP)
+ return -EINVAL;
++ if (!mctp_sockaddr_is_ok(addr))
++ return -EINVAL;
+ if (addr->smctp_tag & ~(MCTP_TAG_MASK | MCTP_TAG_OWNER))
+ return -EINVAL;
+
+@@ -172,11 +183,13 @@ static int mctp_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+
+ addr = msg->msg_name;
+ addr->smctp_family = AF_MCTP;
++ addr->__smctp_pad0 = 0;
+ addr->smctp_network = cb->net;
+ addr->smctp_addr.s_addr = hdr->src;
+ addr->smctp_type = type;
+ addr->smctp_tag = hdr->flags_seq_tag &
+ (MCTP_HDR_TAG_MASK | MCTP_HDR_FLAG_TO);
++ addr->__smctp_pad1 = 0;
+ msg->msg_namelen = sizeof(*addr);
+ }
+
+diff --git a/net/mctp/route.c b/net/mctp/route.c
+index 5ca186d53cb0f..bbb13dbc9227d 100644
+--- a/net/mctp/route.c
++++ b/net/mctp/route.c
+@@ -396,7 +396,7 @@ static int mctp_route_output(struct mctp_route *route, struct sk_buff *skb)
+
+ rc = dev_hard_header(skb, skb->dev, ntohs(skb->protocol),
+ daddr, skb->dev->dev_addr, skb->len);
+- if (rc) {
++ if (rc < 0) {
+ kfree_skb(skb);
+ return -EHOSTUNREACH;
+ }
+@@ -760,7 +760,7 @@ static int mctp_route_add(struct mctp_dev *mdev, mctp_eid_t daddr_start,
+ }
+
+ static int mctp_route_remove(struct mctp_dev *mdev, mctp_eid_t daddr_start,
+- unsigned int daddr_extent)
++ unsigned int daddr_extent, unsigned char type)
+ {
+ struct net *net = dev_net(mdev->dev);
+ struct mctp_route *rt, *tmp;
+@@ -777,7 +777,8 @@ static int mctp_route_remove(struct mctp_dev *mdev, mctp_eid_t daddr_start,
+
+ list_for_each_entry_safe(rt, tmp, &net->mctp.routes, list) {
+ if (rt->dev == mdev &&
+- rt->min == daddr_start && rt->max == daddr_end) {
++ rt->min == daddr_start && rt->max == daddr_end &&
++ rt->type == type) {
+ list_del_rcu(&rt->list);
+ /* TODO: immediate RTM_DELROUTE */
+ mctp_route_release(rt);
+@@ -795,7 +796,7 @@ int mctp_route_add_local(struct mctp_dev *mdev, mctp_eid_t addr)
+
+ int mctp_route_remove_local(struct mctp_dev *mdev, mctp_eid_t addr)
+ {
+- return mctp_route_remove(mdev, addr, 0);
++ return mctp_route_remove(mdev, addr, 0, RTN_LOCAL);
+ }
+
+ /* removes all entries for a given device */
+@@ -975,7 +976,7 @@ static int mctp_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
+ if (rtm->rtm_type != RTN_UNICAST)
+ return -EINVAL;
+
+- rc = mctp_route_remove(mdev, daddr_start, rtm->rtm_dst_len);
++ rc = mctp_route_remove(mdev, daddr_start, rtm->rtm_dst_len, RTN_UNICAST);
+ return rc;
+ }
+
+diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c
+index ffeb2df8be7ae..6e587feb705c4 100644
+--- a/net/mpls/af_mpls.c
++++ b/net/mpls/af_mpls.c
+@@ -1491,22 +1491,52 @@ static void mpls_dev_destroy_rcu(struct rcu_head *head)
+ kfree(mdev);
+ }
+
+-static void mpls_ifdown(struct net_device *dev, int event)
++static int mpls_ifdown(struct net_device *dev, int event)
+ {
+ struct mpls_route __rcu **platform_label;
+ struct net *net = dev_net(dev);
+- u8 alive, deleted;
+ unsigned index;
+
+ platform_label = rtnl_dereference(net->mpls.platform_label);
+ for (index = 0; index < net->mpls.platform_labels; index++) {
+ struct mpls_route *rt = rtnl_dereference(platform_label[index]);
++ bool nh_del = false;
++ u8 alive = 0;
+
+ if (!rt)
+ continue;
+
+- alive = 0;
+- deleted = 0;
++ if (event == NETDEV_UNREGISTER) {
++ u8 deleted = 0;
++
++ for_nexthops(rt) {
++ struct net_device *nh_dev =
++ rtnl_dereference(nh->nh_dev);
++
++ if (!nh_dev || nh_dev == dev)
++ deleted++;
++ if (nh_dev == dev)
++ nh_del = true;
++ } endfor_nexthops(rt);
++
++ /* if there are no more nexthops, delete the route */
++ if (deleted == rt->rt_nhn) {
++ mpls_route_update(net, index, NULL, NULL);
++ continue;
++ }
++
++ if (nh_del) {
++ size_t size = sizeof(*rt) + rt->rt_nhn *
++ rt->rt_nh_size;
++ struct mpls_route *orig = rt;
++
++ rt = kmalloc(size, GFP_KERNEL);
++ if (!rt)
++ return -ENOMEM;
++ memcpy(rt, orig, size);
++ }
++ }
++
+ change_nexthops(rt) {
+ unsigned int nh_flags = nh->nh_flags;
+
+@@ -1530,16 +1560,15 @@ static void mpls_ifdown(struct net_device *dev, int event)
+ next:
+ if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
+ alive++;
+- if (!rtnl_dereference(nh->nh_dev))
+- deleted++;
+ } endfor_nexthops(rt);
+
+ WRITE_ONCE(rt->rt_nhn_alive, alive);
+
+- /* if there are no more nexthops, delete the route */
+- if (event == NETDEV_UNREGISTER && deleted == rt->rt_nhn)
+- mpls_route_update(net, index, NULL, NULL);
++ if (nh_del)
++ mpls_route_update(net, index, rt, NULL);
+ }
++
++ return 0;
+ }
+
+ static void mpls_ifup(struct net_device *dev, unsigned int flags)
+@@ -1597,8 +1626,12 @@ static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
+ return NOTIFY_OK;
+
+ switch (event) {
++ int err;
++
+ case NETDEV_DOWN:
+- mpls_ifdown(dev, event);
++ err = mpls_ifdown(dev, event);
++ if (err)
++ return notifier_from_errno(err);
+ break;
+ case NETDEV_UP:
+ flags = dev_get_flags(dev);
+@@ -1609,13 +1642,18 @@ static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
+ break;
+ case NETDEV_CHANGE:
+ flags = dev_get_flags(dev);
+- if (flags & (IFF_RUNNING | IFF_LOWER_UP))
++ if (flags & (IFF_RUNNING | IFF_LOWER_UP)) {
+ mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
+- else
+- mpls_ifdown(dev, event);
++ } else {
++ err = mpls_ifdown(dev, event);
++ if (err)
++ return notifier_from_errno(err);
++ }
+ break;
+ case NETDEV_UNREGISTER:
+- mpls_ifdown(dev, event);
++ err = mpls_ifdown(dev, event);
++ if (err)
++ return notifier_from_errno(err);
+ mdev = mpls_dev_get(dev);
+ if (mdev) {
+ mpls_dev_sysctl_unregister(dev, mdev);
+@@ -1626,8 +1664,6 @@ static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
+ case NETDEV_CHANGENAME:
+ mdev = mpls_dev_get(dev);
+ if (mdev) {
+- int err;
+-
+ mpls_dev_sysctl_unregister(dev, mdev);
+ err = mpls_dev_sysctl_register(dev, mdev);
+ if (err)
+diff --git a/net/mptcp/mib.c b/net/mptcp/mib.c
+index b21ff9be04c61..8d1c67b935911 100644
+--- a/net/mptcp/mib.c
++++ b/net/mptcp/mib.c
+@@ -35,12 +35,14 @@ static const struct snmp_mib mptcp_snmp_list[] = {
+ SNMP_MIB_ITEM("AddAddr", MPTCP_MIB_ADDADDR),
+ SNMP_MIB_ITEM("EchoAdd", MPTCP_MIB_ECHOADD),
+ SNMP_MIB_ITEM("PortAdd", MPTCP_MIB_PORTADD),
++ SNMP_MIB_ITEM("AddAddrDrop", MPTCP_MIB_ADDADDRDROP),
+ SNMP_MIB_ITEM("MPJoinPortSynRx", MPTCP_MIB_JOINPORTSYNRX),
+ SNMP_MIB_ITEM("MPJoinPortSynAckRx", MPTCP_MIB_JOINPORTSYNACKRX),
+ SNMP_MIB_ITEM("MPJoinPortAckRx", MPTCP_MIB_JOINPORTACKRX),
+ SNMP_MIB_ITEM("MismatchPortSynRx", MPTCP_MIB_MISMATCHPORTSYNRX),
+ SNMP_MIB_ITEM("MismatchPortAckRx", MPTCP_MIB_MISMATCHPORTACKRX),
+ SNMP_MIB_ITEM("RmAddr", MPTCP_MIB_RMADDR),
++ SNMP_MIB_ITEM("RmAddrDrop", MPTCP_MIB_RMADDRDROP),
+ SNMP_MIB_ITEM("RmSubflow", MPTCP_MIB_RMSUBFLOW),
+ SNMP_MIB_ITEM("MPPrioTx", MPTCP_MIB_MPPRIOTX),
+ SNMP_MIB_ITEM("MPPrioRx", MPTCP_MIB_MPPRIORX),
+diff --git a/net/mptcp/mib.h b/net/mptcp/mib.h
+index ecd3d8b117e0b..2966fcb6548ba 100644
+--- a/net/mptcp/mib.h
++++ b/net/mptcp/mib.h
+@@ -28,12 +28,14 @@ enum linux_mptcp_mib_field {
+ MPTCP_MIB_ADDADDR, /* Received ADD_ADDR with echo-flag=0 */
+ MPTCP_MIB_ECHOADD, /* Received ADD_ADDR with echo-flag=1 */
+ MPTCP_MIB_PORTADD, /* Received ADD_ADDR with a port-number */
++ MPTCP_MIB_ADDADDRDROP, /* Dropped incoming ADD_ADDR */
+ MPTCP_MIB_JOINPORTSYNRX, /* Received a SYN MP_JOIN with a different port-number */
+ MPTCP_MIB_JOINPORTSYNACKRX, /* Received a SYNACK MP_JOIN with a different port-number */
+ MPTCP_MIB_JOINPORTACKRX, /* Received an ACK MP_JOIN with a different port-number */
+ MPTCP_MIB_MISMATCHPORTSYNRX, /* Received a SYN MP_JOIN with a mismatched port-number */
+ MPTCP_MIB_MISMATCHPORTACKRX, /* Received an ACK MP_JOIN with a mismatched port-number */
+ MPTCP_MIB_RMADDR, /* Received RM_ADDR */
++ MPTCP_MIB_RMADDRDROP, /* Dropped incoming RM_ADDR */
+ MPTCP_MIB_RMSUBFLOW, /* Remove a subflow */
+ MPTCP_MIB_MPPRIOTX, /* Transmit a MP_PRIO */
+ MPTCP_MIB_MPPRIORX, /* Received a MP_PRIO */
+diff --git a/net/mptcp/options.c b/net/mptcp/options.c
+index f0f22eb4fd5f7..193f0fcce8d85 100644
+--- a/net/mptcp/options.c
++++ b/net/mptcp/options.c
+@@ -107,7 +107,7 @@ static void mptcp_parse_option(const struct sk_buff *skb,
+ ptr += 2;
+ }
+ if (opsize == TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM) {
+- mp_opt->csum = (__force __sum16)get_unaligned_be16(ptr);
++ mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
+ mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
+ ptr += 2;
+ }
+@@ -221,7 +221,7 @@ static void mptcp_parse_option(const struct sk_buff *skb,
+
+ if (opsize == expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) {
+ mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
+- mp_opt->csum = (__force __sum16)get_unaligned_be16(ptr);
++ mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
+ ptr += 2;
+ }
+
+@@ -422,28 +422,6 @@ bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
+ return false;
+ }
+
+-/* MP_JOIN client subflow must wait for 4th ack before sending any data:
+- * TCP can't schedule delack timer before the subflow is fully established.
+- * MPTCP uses the delack timer to do 3rd ack retransmissions
+- */
+-static void schedule_3rdack_retransmission(struct sock *sk)
+-{
+- struct inet_connection_sock *icsk = inet_csk(sk);
+- struct tcp_sock *tp = tcp_sk(sk);
+- unsigned long timeout;
+-
+- /* reschedule with a timeout above RTT, as we must look only for drop */
+- if (tp->srtt_us)
+- timeout = tp->srtt_us << 1;
+- else
+- timeout = TCP_TIMEOUT_INIT;
+-
+- WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER);
+- icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
+- icsk->icsk_ack.timeout = timeout;
+- sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
+-}
+-
+ static void clear_3rdack_retransmission(struct sock *sk)
+ {
+ struct inet_connection_sock *icsk = inet_csk(sk);
+@@ -526,7 +504,15 @@ static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb,
+ *size = TCPOLEN_MPTCP_MPJ_ACK;
+ pr_debug("subflow=%p", subflow);
+
+- schedule_3rdack_retransmission(sk);
++ /* we can use the full delegate action helper only from BH context
++ * If we are in process context - sk is flushing the backlog at
++ * socket lock release time - just set the appropriate flag, will
++ * be handled by the release callback
++ */
++ if (sock_owned_by_user(sk))
++ set_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status);
++ else
++ mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_ACK);
+ return true;
+ }
+ return false;
+@@ -837,10 +823,13 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
+ if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, remaining, opts))
+ ret = true;
+ else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts)) {
++ unsigned int mp_fail_size;
++
+ ret = true;
+- if (mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
+- *size += opt_size;
+- remaining -= opt_size;
++ if (mptcp_established_options_mp_fail(sk, &mp_fail_size,
++ remaining - opt_size, opts)) {
++ *size += opt_size + mp_fail_size;
++ remaining -= opt_size - mp_fail_size;
+ return true;
+ }
+ }
+@@ -1019,11 +1008,9 @@ static void ack_update_msk(struct mptcp_sock *msk,
+ old_snd_una = msk->snd_una;
+ new_snd_una = mptcp_expand_seq(old_snd_una, mp_opt->data_ack, mp_opt->ack64);
+
+- /* ACK for data not even sent yet and even above recovery bound? Ignore.*/
+- if (unlikely(after64(new_snd_una, snd_nxt))) {
+- if (!msk->recovery || after64(new_snd_una, msk->recovery_snd_nxt))
+- new_snd_una = old_snd_una;
+- }
++ /* ACK for data not even sent yet? Ignore.*/
++ if (unlikely(after64(new_snd_una, snd_nxt)))
++ new_snd_una = old_snd_una;
+
+ new_wnd_end = new_snd_una + tcp_sk(ssk)->snd_wnd;
+
+@@ -1227,7 +1214,7 @@ static void mptcp_set_rwin(const struct tcp_sock *tp)
+ WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
+ }
+
+-static u16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __sum16 sum)
++__sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum)
+ {
+ struct csum_pseudo_header header;
+ __wsum csum;
+@@ -1242,14 +1229,24 @@ static u16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __sum1
+ header.data_len = htons(data_len);
+ header.csum = 0;
+
+- csum = csum_partial(&header, sizeof(header), ~csum_unfold(sum));
+- return (__force u16)csum_fold(csum);
++ csum = csum_partial(&header, sizeof(header), sum);
++ return csum_fold(csum);
+ }
+
+-static u16 mptcp_make_csum(const struct mptcp_ext *mpext)
++static __sum16 mptcp_make_csum(const struct mptcp_ext *mpext)
+ {
+ return __mptcp_make_csum(mpext->data_seq, mpext->subflow_seq, mpext->data_len,
+- mpext->csum);
++ ~csum_unfold(mpext->csum));
++}
++
++static void put_len_csum(u16 len, __sum16 csum, void *data)
++{
++ __sum16 *sumptr = data + 2;
++ __be16 *ptr = data;
++
++ put_unaligned_be16(len, ptr);
++
++ put_unaligned(csum, sumptr);
+ }
+
+ void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+@@ -1328,12 +1325,14 @@ void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+ put_unaligned_be32(mpext->subflow_seq, ptr);
+ ptr += 1;
+ if (opts->csum_reqd) {
+- put_unaligned_be32(mpext->data_len << 16 |
+- mptcp_make_csum(mpext), ptr);
++ put_len_csum(mpext->data_len,
++ mptcp_make_csum(mpext),
++ ptr);
+ } else {
+ put_unaligned_be32(mpext->data_len << 16 |
+ TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
+ }
++ ptr += 1;
+ }
+ } else if ((OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK |
+ OPTION_MPTCP_MPC_ACK) & opts->suboptions) {
+@@ -1376,11 +1375,12 @@ void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
+ goto mp_capable_done;
+
+ if (opts->csum_reqd) {
+- put_unaligned_be32(opts->data_len << 16 |
+- __mptcp_make_csum(opts->data_seq,
+- opts->subflow_seq,
+- opts->data_len,
+- opts->csum), ptr);
++ put_len_csum(opts->data_len,
++ __mptcp_make_csum(opts->data_seq,
++ opts->subflow_seq,
++ opts->data_len,
++ ~csum_unfold(opts->csum)),
++ ptr);
+ } else {
+ put_unaligned_be32(opts->data_len << 16 |
+ TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
+diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
+index 6ab386ff32944..d9790d6fbce9c 100644
+--- a/net/mptcp/pm.c
++++ b/net/mptcp/pm.c
+@@ -194,6 +194,8 @@ void mptcp_pm_add_addr_received(struct mptcp_sock *msk,
+ mptcp_pm_add_addr_send_ack(msk);
+ } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) {
+ pm->remote = *addr;
++ } else {
++ __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
+ }
+
+ spin_unlock_bh(&pm->lock);
+@@ -234,8 +236,10 @@ void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
+ mptcp_event_addr_removed(msk, rm_list->ids[i]);
+
+ spin_lock_bh(&pm->lock);
+- mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED);
+- pm->rm_list_rx = *rm_list;
++ if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED))
++ pm->rm_list_rx = *rm_list;
++ else
++ __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP);
+ spin_unlock_bh(&pm->lock);
+ }
+
+diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
+index 050eea231528b..cf0f700f46dd9 100644
+--- a/net/mptcp/pm_netlink.c
++++ b/net/mptcp/pm_netlink.c
+@@ -459,6 +459,18 @@ static unsigned int fill_remote_addresses_vec(struct mptcp_sock *msk, bool fullm
+ return i;
+ }
+
++static struct mptcp_pm_addr_entry *
++__lookup_addr(struct pm_nl_pernet *pernet, struct mptcp_addr_info *info)
++{
++ struct mptcp_pm_addr_entry *entry;
++
++ list_for_each_entry(entry, &pernet->local_addr_list, list) {
++ if (addresses_equal(&entry->addr, info, true))
++ return entry;
++ }
++ return NULL;
++}
++
+ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
+ {
+ struct sock *sk = (struct sock *)msk;
+@@ -594,6 +606,7 @@ static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
+ unsigned int add_addr_accept_max;
+ struct mptcp_addr_info remote;
+ unsigned int subflows_max;
++ bool reset_port = false;
+ int i, nr;
+
+ add_addr_accept_max = mptcp_pm_get_add_addr_accept_max(msk);
+@@ -603,15 +616,19 @@ static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
+ msk->pm.add_addr_accepted, add_addr_accept_max,
+ msk->pm.remote.family);
+
+- if (lookup_subflow_by_daddr(&msk->conn_list, &msk->pm.remote))
++ remote = msk->pm.remote;
++ if (lookup_subflow_by_daddr(&msk->conn_list, &remote))
+ goto add_addr_echo;
+
++ /* pick id 0 port, if none is provided the remote address */
++ if (!remote.port) {
++ reset_port = true;
++ remote.port = sk->sk_dport;
++ }
++
+ /* connect to the specified remote address, using whatever
+ * local address the routing configuration will pick.
+ */
+- remote = msk->pm.remote;
+- if (!remote.port)
+- remote.port = sk->sk_dport;
+ nr = fill_local_addresses_vec(msk, addrs);
+
+ msk->pm.add_addr_accepted++;
+@@ -624,8 +641,12 @@ static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
+ __mptcp_subflow_connect(sk, &addrs[i], &remote);
+ spin_lock_bh(&msk->pm.lock);
+
++ /* be sure to echo exactly the received address */
++ if (reset_port)
++ remote.port = 0;
++
+ add_addr_echo:
+- mptcp_pm_announce_addr(msk, &msk->pm.remote, true);
++ mptcp_pm_announce_addr(msk, &remote, true);
+ mptcp_pm_nl_addr_send_ack(msk);
+ }
+
+@@ -700,6 +721,9 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk,
+
+ msk_owned_by_me(msk);
+
++ if (sk->sk_state == TCP_LISTEN)
++ return;
++
+ if (!rm_list->nr)
+ return;
+
+@@ -707,6 +731,8 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk,
+ return;
+
+ for (i = 0; i < rm_list->nr; i++) {
++ bool removed = false;
++
+ list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
+ struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
+ int how = RCV_SHUTDOWN | SEND_SHUTDOWN;
+@@ -726,15 +752,19 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk,
+ mptcp_close_ssk(sk, ssk, subflow);
+ spin_lock_bh(&msk->pm.lock);
+
+- if (rm_type == MPTCP_MIB_RMADDR) {
+- msk->pm.add_addr_accepted--;
+- WRITE_ONCE(msk->pm.accept_addr, true);
+- } else if (rm_type == MPTCP_MIB_RMSUBFLOW) {
+- msk->pm.local_addr_used--;
+- }
++ removed = true;
+ msk->pm.subflows--;
+ __MPTCP_INC_STATS(sock_net(sk), rm_type);
+ }
++ if (!removed)
++ continue;
++
++ if (rm_type == MPTCP_MIB_RMADDR) {
++ msk->pm.add_addr_accepted--;
++ WRITE_ONCE(msk->pm.accept_addr, true);
++ } else if (rm_type == MPTCP_MIB_RMSUBFLOW) {
++ msk->pm.local_addr_used--;
++ }
+ }
+ }
+
+@@ -857,6 +887,7 @@ out:
+ static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
+ struct mptcp_pm_addr_entry *entry)
+ {
++ int addrlen = sizeof(struct sockaddr_in);
+ struct sockaddr_storage addr;
+ struct mptcp_sock *msk;
+ struct socket *ssock;
+@@ -881,8 +912,11 @@ static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
+ }
+
+ mptcp_info2sockaddr(&entry->addr, &addr, entry->addr.family);
+- err = kernel_bind(ssock, (struct sockaddr *)&addr,
+- sizeof(struct sockaddr_in));
++#if IS_ENABLED(CONFIG_MPTCP_IPV6)
++ if (entry->addr.family == AF_INET6)
++ addrlen = sizeof(struct sockaddr_in6);
++#endif
++ err = kernel_bind(ssock, (struct sockaddr *)&addr, addrlen);
+ if (err) {
+ pr_warn("kernel_bind error, err=%d", err);
+ goto out;
+@@ -1716,17 +1750,21 @@ static int mptcp_nl_cmd_set_flags(struct sk_buff *skb, struct genl_info *info)
+ if (addr.flags & MPTCP_PM_ADDR_FLAG_BACKUP)
+ bkup = 1;
+
+- list_for_each_entry(entry, &pernet->local_addr_list, list) {
+- if (addresses_equal(&entry->addr, &addr.addr, true)) {
+- mptcp_nl_addr_backup(net, &entry->addr, bkup);
+-
+- if (bkup)
+- entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
+- else
+- entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
+- }
++ spin_lock_bh(&pernet->lock);
++ entry = __lookup_addr(pernet, &addr.addr);
++ if (!entry) {
++ spin_unlock_bh(&pernet->lock);
++ return -EINVAL;
+ }
+
++ if (bkup)
++ entry->flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
++ else
++ entry->flags &= ~MPTCP_PM_ADDR_FLAG_BACKUP;
++ addr = *entry;
++ spin_unlock_bh(&pernet->lock);
++
++ mptcp_nl_addr_backup(net, &addr.addr, bkup);
+ return 0;
+ }
+
+diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
+index d073b21113828..d6def23b8cba0 100644
+--- a/net/mptcp/protocol.c
++++ b/net/mptcp/protocol.c
+@@ -406,9 +406,12 @@ static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
+ static void mptcp_set_datafin_timeout(const struct sock *sk)
+ {
+ struct inet_connection_sock *icsk = inet_csk(sk);
++ u32 retransmits;
+
+- mptcp_sk(sk)->timer_ival = min(TCP_RTO_MAX,
+- TCP_RTO_MIN << icsk->icsk_retransmits);
++ retransmits = min_t(u32, icsk->icsk_retransmits,
++ ilog2(TCP_RTO_MAX / TCP_RTO_MIN));
++
++ mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits;
+ }
+
+ static void __mptcp_set_timeout(struct sock *sk, long tout)
+@@ -1505,6 +1508,32 @@ static void mptcp_push_release(struct sock *sk, struct sock *ssk,
+ release_sock(ssk);
+ }
+
++static void mptcp_update_post_push(struct mptcp_sock *msk,
++ struct mptcp_data_frag *dfrag,
++ u32 sent)
++{
++ u64 snd_nxt_new = dfrag->data_seq;
++
++ dfrag->already_sent += sent;
++
++ msk->snd_burst -= sent;
++ msk->tx_pending_data -= sent;
++
++ snd_nxt_new += dfrag->already_sent;
++
++ /* snd_nxt_new can be smaller than snd_nxt in case mptcp
++ * is recovering after a failover. In that event, this re-sends
++ * old segments.
++ *
++ * Thus compute snd_nxt_new candidate based on
++ * the dfrag->data_seq that was sent and the data
++ * that has been handed to the subflow for transmission
++ * and skip update in case it was old dfrag.
++ */
++ if (likely(after64(snd_nxt_new, msk->snd_nxt)))
++ msk->snd_nxt = snd_nxt_new;
++}
++
+ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
+ {
+ struct sock *prev_ssk = NULL, *ssk = NULL;
+@@ -1523,7 +1552,7 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
+ int ret = 0;
+
+ prev_ssk = ssk;
+- mptcp_flush_join_list(msk);
++ __mptcp_flush_join_list(msk);
+ ssk = mptcp_subflow_get_send(msk);
+
+ /* First check. If the ssk has changed since
+@@ -1548,12 +1577,10 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
+ }
+
+ info.sent += ret;
+- dfrag->already_sent += ret;
+- msk->snd_nxt += ret;
+- msk->snd_burst -= ret;
+- msk->tx_pending_data -= ret;
+ copied += ret;
+ len -= ret;
++
++ mptcp_update_post_push(msk, dfrag, ret);
+ }
+ WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
+ }
+@@ -1597,7 +1624,8 @@ static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk)
+ if (!xmit_ssk)
+ goto out;
+ if (xmit_ssk != ssk) {
+- mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk));
++ mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk),
++ MPTCP_DELEGATE_SEND);
+ goto out;
+ }
+
+@@ -1606,13 +1634,11 @@ static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk)
+ goto out;
+
+ info.sent += ret;
+- dfrag->already_sent += ret;
+- msk->snd_nxt += ret;
+- msk->snd_burst -= ret;
+- msk->tx_pending_data -= ret;
+ copied += ret;
+ len -= ret;
+ first = false;
++
++ mptcp_update_post_push(msk, dfrag, ret);
+ }
+ WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
+ }
+@@ -2183,15 +2209,12 @@ bool __mptcp_retransmit_pending_data(struct sock *sk)
+ return false;
+ }
+
+- /* will accept ack for reijected data before re-sending them */
+- if (!msk->recovery || after64(msk->snd_nxt, msk->recovery_snd_nxt))
+- msk->recovery_snd_nxt = msk->snd_nxt;
++ msk->recovery_snd_nxt = msk->snd_nxt;
+ msk->recovery = true;
+ mptcp_data_unlock(sk);
+
+ msk->first_pending = rtx_head;
+ msk->tx_pending_data += msk->snd_nxt - rtx_head->data_seq;
+- msk->snd_nxt = rtx_head->data_seq;
+ msk->snd_burst = 0;
+
+ /* be sure to clear the "sent status" on all re-injected fragments */
+@@ -2881,7 +2904,7 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
+ */
+ if (WARN_ON_ONCE(!new_mptcp_sock)) {
+ tcp_sk(newsk)->is_mptcp = 0;
+- return newsk;
++ goto out;
+ }
+
+ /* acquire the 2nd reference for the owning socket */
+@@ -2893,6 +2916,8 @@ static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
+ MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
+ }
+
++out:
++ newsk->sk_kern_sock = kern;
+ return newsk;
+ }
+
+@@ -2940,7 +2965,7 @@ void __mptcp_check_push(struct sock *sk, struct sock *ssk)
+ if (xmit_ssk == ssk)
+ __mptcp_subflow_push_pending(sk, ssk);
+ else if (xmit_ssk)
+- mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk));
++ mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk), MPTCP_DELEGATE_SEND);
+ } else {
+ set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
+ }
+@@ -2994,18 +3019,50 @@ static void mptcp_release_cb(struct sock *sk)
+ __mptcp_update_rmem(sk);
+ }
+
++/* MP_JOIN client subflow must wait for 4th ack before sending any data:
++ * TCP can't schedule delack timer before the subflow is fully established.
++ * MPTCP uses the delack timer to do 3rd ack retransmissions
++ */
++static void schedule_3rdack_retransmission(struct sock *ssk)
++{
++ struct inet_connection_sock *icsk = inet_csk(ssk);
++ struct tcp_sock *tp = tcp_sk(ssk);
++ unsigned long timeout;
++
++ if (mptcp_subflow_ctx(ssk)->fully_established)
++ return;
++
++ /* reschedule with a timeout above RTT, as we must look only for drop */
++ if (tp->srtt_us)
++ timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1));
++ else
++ timeout = TCP_TIMEOUT_INIT;
++ timeout += jiffies;
++
++ WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER);
++ icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
++ icsk->icsk_ack.timeout = timeout;
++ sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout);
++}
++
+ void mptcp_subflow_process_delegated(struct sock *ssk)
+ {
+ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
+ struct sock *sk = subflow->conn;
+
+- mptcp_data_lock(sk);
+- if (!sock_owned_by_user(sk))
+- __mptcp_subflow_push_pending(sk, ssk);
+- else
+- set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
+- mptcp_data_unlock(sk);
+- mptcp_subflow_delegated_done(subflow);
++ if (test_bit(MPTCP_DELEGATE_SEND, &subflow->delegated_status)) {
++ mptcp_data_lock(sk);
++ if (!sock_owned_by_user(sk))
++ __mptcp_subflow_push_pending(sk, ssk);
++ else
++ set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
++ mptcp_data_unlock(sk);
++ mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_SEND);
++ }
++ if (test_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status)) {
++ schedule_3rdack_retransmission(ssk);
++ mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_ACK);
++ }
+ }
+
+ static int mptcp_hash(struct sock *sk)
+diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
+index dc984676c5eb1..e193b710b471a 100644
+--- a/net/mptcp/protocol.h
++++ b/net/mptcp/protocol.h
+@@ -401,6 +401,7 @@ struct mptcp_delegated_action {
+ DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions);
+
+ #define MPTCP_DELEGATE_SEND 0
++#define MPTCP_DELEGATE_ACK 1
+
+ /* MPTCP subflow context */
+ struct mptcp_subflow_context {
+@@ -435,7 +436,8 @@ struct mptcp_subflow_context {
+ rx_eof : 1,
+ can_ack : 1, /* only after processing the remote a key */
+ disposable : 1, /* ctx can be free at ulp release time */
+- stale : 1; /* unable to snd/rcv data, do not use for xmit */
++ stale : 1, /* unable to snd/rcv data, do not use for xmit */
++ valid_csum_seen : 1; /* at least one csum validated */
+ enum mptcp_data_avail data_avail;
+ u32 remote_nonce;
+ u64 thmac;
+@@ -506,23 +508,23 @@ static inline void mptcp_add_pending_subflow(struct mptcp_sock *msk,
+
+ void mptcp_subflow_process_delegated(struct sock *ssk);
+
+-static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow)
++static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow, int action)
+ {
+ struct mptcp_delegated_action *delegated;
+ bool schedule;
+
++ /* the caller held the subflow bh socket lock */
++ lockdep_assert_in_softirq();
++
+ /* The implied barrier pairs with mptcp_subflow_delegated_done(), and
+ * ensures the below list check sees list updates done prior to status
+ * bit changes
+ */
+- if (!test_and_set_bit(MPTCP_DELEGATE_SEND, &subflow->delegated_status)) {
++ if (!test_and_set_bit(action, &subflow->delegated_status)) {
+ /* still on delegated list from previous scheduling */
+ if (!list_empty(&subflow->delegated_node))
+ return;
+
+- /* the caller held the subflow bh socket lock */
+- lockdep_assert_in_softirq();
+-
+ delegated = this_cpu_ptr(&mptcp_delegated_actions);
+ schedule = list_empty(&delegated->head);
+ list_add_tail(&subflow->delegated_node, &delegated->head);
+@@ -547,16 +549,16 @@ mptcp_subflow_delegated_next(struct mptcp_delegated_action *delegated)
+
+ static inline bool mptcp_subflow_has_delegated_action(const struct mptcp_subflow_context *subflow)
+ {
+- return test_bit(MPTCP_DELEGATE_SEND, &subflow->delegated_status);
++ return !!READ_ONCE(subflow->delegated_status);
+ }
+
+-static inline void mptcp_subflow_delegated_done(struct mptcp_subflow_context *subflow)
++static inline void mptcp_subflow_delegated_done(struct mptcp_subflow_context *subflow, int action)
+ {
+ /* pairs with mptcp_subflow_delegate, ensures delegate_node is updated before
+ * touching the status bit
+ */
+ smp_wmb();
+- clear_bit(MPTCP_DELEGATE_SEND, &subflow->delegated_status);
++ clear_bit(action, &subflow->delegated_status);
+ }
+
+ int mptcp_is_enabled(const struct net *net);
+@@ -717,6 +719,7 @@ void mptcp_token_destroy(struct mptcp_sock *msk);
+ void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn);
+
+ void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac);
++__sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum);
+
+ void __init mptcp_pm_init(void);
+ void mptcp_pm_data_init(struct mptcp_sock *msk);
+diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c
+index 8c03afac5ca03..4bb305342fcc7 100644
+--- a/net/mptcp/sockopt.c
++++ b/net/mptcp/sockopt.c
+@@ -523,7 +523,6 @@ static bool mptcp_supported_sockopt(int level, int optname)
+ case TCP_NODELAY:
+ case TCP_THIN_LINEAR_TIMEOUTS:
+ case TCP_CONGESTION:
+- case TCP_ULP:
+ case TCP_CORK:
+ case TCP_KEEPIDLE:
+ case TCP_KEEPINTVL:
+diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
+index 6172f380dfb76..5ef9013b94c74 100644
+--- a/net/mptcp/subflow.c
++++ b/net/mptcp/subflow.c
+@@ -845,9 +845,8 @@ static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *
+ bool csum_reqd)
+ {
+ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
+- struct csum_pseudo_header header;
+ u32 offset, seq, delta;
+- __wsum csum;
++ __sum16 csum;
+ int len;
+
+ if (!csum_reqd)
+@@ -908,19 +907,20 @@ static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *
+ * while the pseudo header requires the original DSS data len,
+ * including that
+ */
+- header.data_seq = cpu_to_be64(subflow->map_seq);
+- header.subflow_seq = htonl(subflow->map_subflow_seq);
+- header.data_len = htons(subflow->map_data_len + subflow->map_data_fin);
+- header.csum = 0;
+-
+- csum = csum_partial(&header, sizeof(header), subflow->map_data_csum);
+- if (unlikely(csum_fold(csum))) {
++ csum = __mptcp_make_csum(subflow->map_seq,
++ subflow->map_subflow_seq,
++ subflow->map_data_len + subflow->map_data_fin,
++ subflow->map_data_csum);
++ if (unlikely(csum)) {
+ MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
+- subflow->send_mp_fail = 1;
+- MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPFAILTX);
++ if (subflow->mp_join || subflow->valid_csum_seen) {
++ subflow->send_mp_fail = 1;
++ MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPFAILTX);
++ }
+ return subflow->mp_join ? MAPPING_INVALID : MAPPING_DUMMY;
+ }
+
++ subflow->valid_csum_seen = 1;
+ return MAPPING_OK;
+ }
+
+@@ -1102,6 +1102,18 @@ static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ss
+ }
+ }
+
++static bool subflow_can_fallback(struct mptcp_subflow_context *subflow)
++{
++ struct mptcp_sock *msk = mptcp_sk(subflow->conn);
++
++ if (subflow->mp_join)
++ return false;
++ else if (READ_ONCE(msk->csum_enabled))
++ return !subflow->valid_csum_seen;
++ else
++ return !subflow->fully_established;
++}
++
+ static bool subflow_check_data_avail(struct sock *ssk)
+ {
+ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
+@@ -1179,7 +1191,7 @@ fallback:
+ return true;
+ }
+
+- if (subflow->mp_join || subflow->fully_established) {
++ if (!subflow_can_fallback(subflow)) {
+ /* fatal protocol error, close the socket.
+ * subflow_error_report() will introduce the appropriate barriers
+ */
+diff --git a/net/ncsi/ncsi-cmd.c b/net/ncsi/ncsi-cmd.c
+index ba9ae482141b0..dda8b76b77988 100644
+--- a/net/ncsi/ncsi-cmd.c
++++ b/net/ncsi/ncsi-cmd.c
+@@ -18,6 +18,8 @@
+ #include "internal.h"
+ #include "ncsi-pkt.h"
+
++static const int padding_bytes = 26;
++
+ u32 ncsi_calculate_checksum(unsigned char *data, int len)
+ {
+ u32 checksum = 0;
+@@ -213,12 +215,17 @@ static int ncsi_cmd_handler_oem(struct sk_buff *skb,
+ {
+ struct ncsi_cmd_oem_pkt *cmd;
+ unsigned int len;
++ int payload;
++ /* NC-SI spec DSP_0222_1.2.0, section 8.2.2.2
++ * requires payload to be padded with 0 to
++ * 32-bit boundary before the checksum field.
++ * Ensure the padding bytes are accounted for in
++ * skb allocation
++ */
+
++ payload = ALIGN(nca->payload, 4);
+ len = sizeof(struct ncsi_cmd_pkt_hdr) + 4;
+- if (nca->payload < 26)
+- len += 26;
+- else
+- len += nca->payload;
++ len += max(payload, padding_bytes);
+
+ cmd = skb_put_zero(skb, len);
+ memcpy(&cmd->mfr_id, nca->data, nca->payload);
+@@ -272,6 +279,7 @@ static struct ncsi_request *ncsi_alloc_command(struct ncsi_cmd_arg *nca)
+ struct net_device *dev = nd->dev;
+ int hlen = LL_RESERVED_SPACE(dev);
+ int tlen = dev->needed_tailroom;
++ int payload;
+ int len = hlen + tlen;
+ struct sk_buff *skb;
+ struct ncsi_request *nr;
+@@ -281,14 +289,14 @@ static struct ncsi_request *ncsi_alloc_command(struct ncsi_cmd_arg *nca)
+ return NULL;
+
+ /* NCSI command packet has 16-bytes header, payload, 4 bytes checksum.
++ * Payload needs padding so that the checksum field following payload is
++ * aligned to 32-bit boundary.
+ * The packet needs padding if its payload is less than 26 bytes to
+ * meet 64 bytes minimal ethernet frame length.
+ */
+ len += sizeof(struct ncsi_cmd_pkt_hdr) + 4;
+- if (nca->payload < 26)
+- len += 26;
+- else
+- len += nca->payload;
++ payload = ALIGN(nca->payload, 4);
++ len += max(payload, padding_bytes);
+
+ /* Allocate skb */
+ skb = alloc_skb(len, GFP_ATOMIC);
+diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c
+index bb5f1650f11cb..c189b4c8a1823 100644
+--- a/net/ncsi/ncsi-netlink.c
++++ b/net/ncsi/ncsi-netlink.c
+@@ -112,7 +112,11 @@ static int ncsi_write_package_info(struct sk_buff *skb,
+ pnest = nla_nest_start_noflag(skb, NCSI_PKG_ATTR);
+ if (!pnest)
+ return -ENOMEM;
+- nla_put_u32(skb, NCSI_PKG_ATTR_ID, np->id);
++ rc = nla_put_u32(skb, NCSI_PKG_ATTR_ID, np->id);
++ if (rc) {
++ nla_nest_cancel(skb, pnest);
++ return rc;
++ }
+ if ((0x1 << np->id) == ndp->package_whitelist)
+ nla_put_flag(skb, NCSI_PKG_ATTR_FORCED);
+ cnest = nla_nest_start_noflag(skb, NCSI_PKG_ATTR_CHANNEL_LIST);
+diff --git a/net/netfilter/core.c b/net/netfilter/core.c
+index 63d032191e626..60332fdb6dd44 100644
+--- a/net/netfilter/core.c
++++ b/net/netfilter/core.c
+@@ -406,14 +406,15 @@ static int __nf_register_net_hook(struct net *net, int pf,
+ p = nf_entry_dereference(*pp);
+ new_hooks = nf_hook_entries_grow(p, reg);
+
+- if (!IS_ERR(new_hooks))
++ if (!IS_ERR(new_hooks)) {
++ hooks_validate(new_hooks);
+ rcu_assign_pointer(*pp, new_hooks);
++ }
+
+ mutex_unlock(&nf_hook_mutex);
+ if (IS_ERR(new_hooks))
+ return PTR_ERR(new_hooks);
+
+- hooks_validate(new_hooks);
+ #ifdef CONFIG_NETFILTER_INGRESS
+ if (nf_ingress_hook(reg, pf))
+ net_inc_ingress_queue();
+diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c
+index 2c467c422dc63..fb67f1ca2495b 100644
+--- a/net/netfilter/ipvs/ip_vs_conn.c
++++ b/net/netfilter/ipvs/ip_vs_conn.c
+@@ -1495,7 +1495,7 @@ int __init ip_vs_conn_init(void)
+ pr_info("Connection hash table configured "
+ "(size=%d, memory=%ldKbytes)\n",
+ ip_vs_conn_tab_size,
+- (long)(ip_vs_conn_tab_size*sizeof(struct list_head))/1024);
++ (long)(ip_vs_conn_tab_size*sizeof(*ip_vs_conn_tab))/1024);
+ IP_VS_DBG(0, "Each connection entry needs %zd bytes at least\n",
+ sizeof(struct ip_vs_conn));
+
+diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c
+index 128690c512dff..393058a43aa73 100644
+--- a/net/netfilter/ipvs/ip_vs_core.c
++++ b/net/netfilter/ipvs/ip_vs_core.c
+@@ -1964,7 +1964,6 @@ ip_vs_in(struct netns_ipvs *ipvs, unsigned int hooknum, struct sk_buff *skb, int
+ struct ip_vs_proto_data *pd;
+ struct ip_vs_conn *cp;
+ int ret, pkts;
+- int conn_reuse_mode;
+ struct sock *sk;
+
+ /* Already marked as IPVS request or reply? */
+@@ -2041,15 +2040,16 @@ ip_vs_in(struct netns_ipvs *ipvs, unsigned int hooknum, struct sk_buff *skb, int
+ cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
+ ipvs, af, skb, &iph);
+
+- conn_reuse_mode = sysctl_conn_reuse_mode(ipvs);
+- if (conn_reuse_mode && !iph.fragoffs && is_new_conn(skb, &iph) && cp) {
++ if (!iph.fragoffs && is_new_conn(skb, &iph) && cp) {
++ int conn_reuse_mode = sysctl_conn_reuse_mode(ipvs);
+ bool old_ct = false, resched = false;
+
+ if (unlikely(sysctl_expire_nodest_conn(ipvs)) && cp->dest &&
+ unlikely(!atomic_read(&cp->dest->weight))) {
+ resched = true;
+ old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
+- } else if (is_new_conn_expected(cp, conn_reuse_mode)) {
++ } else if (conn_reuse_mode &&
++ is_new_conn_expected(cp, conn_reuse_mode)) {
+ old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
+ if (!atomic_read(&cp->n_control)) {
+ resched = true;
+diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
+index 770a63103c7a4..31399c53dfb13 100644
+--- a/net/netfilter/nf_conntrack_core.c
++++ b/net/netfilter/nf_conntrack_core.c
+@@ -66,6 +66,8 @@ EXPORT_SYMBOL_GPL(nf_conntrack_hash);
+ struct conntrack_gc_work {
+ struct delayed_work dwork;
+ u32 next_bucket;
++ u32 avg_timeout;
++ u32 start_time;
+ bool exiting;
+ bool early_drop;
+ };
+@@ -77,8 +79,19 @@ static __read_mostly bool nf_conntrack_locks_all;
+ /* serialize hash resizes and nf_ct_iterate_cleanup */
+ static DEFINE_MUTEX(nf_conntrack_mutex);
+
+-#define GC_SCAN_INTERVAL (120u * HZ)
++#define GC_SCAN_INTERVAL_MAX (60ul * HZ)
++#define GC_SCAN_INTERVAL_MIN (1ul * HZ)
++
++/* clamp timeouts to this value (TCP unacked) */
++#define GC_SCAN_INTERVAL_CLAMP (300ul * HZ)
++
++/* large initial bias so that we don't scan often just because we have
++ * three entries with a 1s timeout.
++ */
++#define GC_SCAN_INTERVAL_INIT INT_MAX
++
+ #define GC_SCAN_MAX_DURATION msecs_to_jiffies(10)
++#define GC_SCAN_EXPIRED_MAX (64000u / HZ)
+
+ #define MIN_CHAINLEN 8u
+ #define MAX_CHAINLEN (32u - MIN_CHAINLEN)
+@@ -558,7 +571,7 @@ static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
+
+ #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK)
+
+-/* Released via destroy_conntrack() */
++/* Released via nf_ct_destroy() */
+ struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
+ const struct nf_conntrack_zone *zone,
+ gfp_t flags)
+@@ -585,7 +598,7 @@ struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
+ tmpl->status = IPS_TEMPLATE;
+ write_pnet(&tmpl->ct_net, net);
+ nf_ct_zone_add(tmpl, zone);
+- atomic_set(&tmpl->ct_general.use, 0);
++ refcount_set(&tmpl->ct_general.use, 1);
+
+ return tmpl;
+ }
+@@ -612,13 +625,12 @@ static void destroy_gre_conntrack(struct nf_conn *ct)
+ #endif
+ }
+
+-static void
+-destroy_conntrack(struct nf_conntrack *nfct)
++void nf_ct_destroy(struct nf_conntrack *nfct)
+ {
+ struct nf_conn *ct = (struct nf_conn *)nfct;
+
+- pr_debug("destroy_conntrack(%p)\n", ct);
+- WARN_ON(atomic_read(&nfct->use) != 0);
++ pr_debug("%s(%p)\n", __func__, ct);
++ WARN_ON(refcount_read(&nfct->use) != 0);
+
+ if (unlikely(nf_ct_is_template(ct))) {
+ nf_ct_tmpl_free(ct);
+@@ -643,9 +655,10 @@ destroy_conntrack(struct nf_conntrack *nfct)
+ if (ct->master)
+ nf_ct_put(ct->master);
+
+- pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
++ pr_debug("%s: returning ct=%p to slab\n", __func__, ct);
+ nf_conntrack_free(ct);
+ }
++EXPORT_SYMBOL(nf_ct_destroy);
+
+ static void nf_ct_delete_from_lists(struct nf_conn *ct)
+ {
+@@ -684,7 +697,7 @@ bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
+
+ tstamp = nf_conn_tstamp_find(ct);
+ if (tstamp) {
+- s32 timeout = ct->timeout - nfct_time_stamp;
++ s32 timeout = READ_ONCE(ct->timeout) - nfct_time_stamp;
+
+ tstamp->stop = ktime_get_real_ns();
+ if (timeout < 0)
+@@ -742,7 +755,7 @@ nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2)
+ /* caller must hold rcu readlock and none of the nf_conntrack_locks */
+ static void nf_ct_gc_expired(struct nf_conn *ct)
+ {
+- if (!atomic_inc_not_zero(&ct->ct_general.use))
++ if (!refcount_inc_not_zero(&ct->ct_general.use))
+ return;
+
+ if (nf_ct_should_gc(ct))
+@@ -810,7 +823,7 @@ __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
+ * in, try to obtain a reference and re-check tuple
+ */
+ ct = nf_ct_tuplehash_to_ctrack(h);
+- if (likely(atomic_inc_not_zero(&ct->ct_general.use))) {
++ if (likely(refcount_inc_not_zero(&ct->ct_general.use))) {
+ if (likely(nf_ct_key_equal(h, tuple, zone, net)))
+ goto found;
+
+@@ -907,7 +920,7 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct)
+
+ smp_wmb();
+ /* The caller holds a reference to this object */
+- atomic_set(&ct->ct_general.use, 2);
++ refcount_set(&ct->ct_general.use, 2);
+ __nf_conntrack_hash_insert(ct, hash, reply_hash);
+ nf_conntrack_double_unlock(hash, reply_hash);
+ NF_CT_STAT_INC(net, insert);
+@@ -958,7 +971,7 @@ static void __nf_conntrack_insert_prepare(struct nf_conn *ct)
+ {
+ struct nf_conn_tstamp *tstamp;
+
+- atomic_inc(&ct->ct_general.use);
++ refcount_inc(&ct->ct_general.use);
+ ct->status |= IPS_CONFIRMED;
+
+ /* set conntrack timestamp, if enabled. */
+@@ -989,7 +1002,7 @@ static int __nf_ct_resolve_clash(struct sk_buff *skb,
+
+ nf_ct_acct_merge(ct, ctinfo, loser_ct);
+ nf_ct_add_to_dying_list(loser_ct);
+- nf_conntrack_put(&loser_ct->ct_general);
++ nf_ct_put(loser_ct);
+ nf_ct_set(skb, ct, ctinfo);
+
+ NF_CT_STAT_INC(net, clash_resolve);
+@@ -1036,7 +1049,7 @@ static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx)
+ }
+
+ /* We want the clashing entry to go away real soon: 1 second timeout. */
+- loser_ct->timeout = nfct_time_stamp + HZ;
++ WRITE_ONCE(loser_ct->timeout, nfct_time_stamp + HZ);
+
+ /* IPS_NAT_CLASH removes the entry automatically on the first
+ * reply. Also prevents UDP tracker from moving the entry to
+@@ -1351,7 +1364,7 @@ static unsigned int early_drop_list(struct net *net,
+ nf_ct_is_dying(tmp))
+ continue;
+
+- if (!atomic_inc_not_zero(&tmp->ct_general.use))
++ if (!refcount_inc_not_zero(&tmp->ct_general.use))
+ continue;
+
+ /* kill only if still in same netns -- might have moved due to
+@@ -1420,16 +1433,28 @@ static bool gc_worker_can_early_drop(const struct nf_conn *ct)
+
+ static void gc_worker(struct work_struct *work)
+ {
+- unsigned long end_time = jiffies + GC_SCAN_MAX_DURATION;
+ unsigned int i, hashsz, nf_conntrack_max95 = 0;
+- unsigned long next_run = GC_SCAN_INTERVAL;
++ u32 end_time, start_time = nfct_time_stamp;
+ struct conntrack_gc_work *gc_work;
++ unsigned int expired_count = 0;
++ unsigned long next_run;
++ s32 delta_time;
++
+ gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
+
+ i = gc_work->next_bucket;
+ if (gc_work->early_drop)
+ nf_conntrack_max95 = nf_conntrack_max / 100u * 95u;
+
++ if (i == 0) {
++ gc_work->avg_timeout = GC_SCAN_INTERVAL_INIT;
++ gc_work->start_time = start_time;
++ }
++
++ next_run = gc_work->avg_timeout;
++
++ end_time = start_time + GC_SCAN_MAX_DURATION;
++
+ do {
+ struct nf_conntrack_tuple_hash *h;
+ struct hlist_nulls_head *ct_hash;
+@@ -1446,6 +1471,7 @@ static void gc_worker(struct work_struct *work)
+
+ hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
+ struct nf_conntrack_net *cnet;
++ unsigned long expires;
+ struct net *net;
+
+ tmp = nf_ct_tuplehash_to_ctrack(h);
+@@ -1455,11 +1481,29 @@ static void gc_worker(struct work_struct *work)
+ continue;
+ }
+
++ if (expired_count > GC_SCAN_EXPIRED_MAX) {
++ rcu_read_unlock();
++
++ gc_work->next_bucket = i;
++ gc_work->avg_timeout = next_run;
++
++ delta_time = nfct_time_stamp - gc_work->start_time;
++
++ /* re-sched immediately if total cycle time is exceeded */
++ next_run = delta_time < (s32)GC_SCAN_INTERVAL_MAX;
++ goto early_exit;
++ }
++
+ if (nf_ct_is_expired(tmp)) {
+ nf_ct_gc_expired(tmp);
++ expired_count++;
+ continue;
+ }
+
++ expires = clamp(nf_ct_expires(tmp), GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_CLAMP);
++ next_run += expires;
++ next_run /= 2u;
++
+ if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
+ continue;
+
+@@ -1469,7 +1513,7 @@ static void gc_worker(struct work_struct *work)
+ continue;
+
+ /* need to take reference to avoid possible races */
+- if (!atomic_inc_not_zero(&tmp->ct_general.use))
++ if (!refcount_inc_not_zero(&tmp->ct_general.use))
+ continue;
+
+ if (gc_worker_skip_ct(tmp)) {
+@@ -1477,8 +1521,10 @@ static void gc_worker(struct work_struct *work)
+ continue;
+ }
+
+- if (gc_worker_can_early_drop(tmp))
++ if (gc_worker_can_early_drop(tmp)) {
+ nf_ct_kill(tmp);
++ expired_count++;
++ }
+
+ nf_ct_put(tmp);
+ }
+@@ -1491,33 +1537,38 @@ static void gc_worker(struct work_struct *work)
+ cond_resched();
+ i++;
+
+- if (time_after(jiffies, end_time) && i < hashsz) {
++ delta_time = nfct_time_stamp - end_time;
++ if (delta_time > 0 && i < hashsz) {
++ gc_work->avg_timeout = next_run;
+ gc_work->next_bucket = i;
+ next_run = 0;
+- break;
++ goto early_exit;
+ }
+ } while (i < hashsz);
+
++ gc_work->next_bucket = 0;
++
++ next_run = clamp(next_run, GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_MAX);
++
++ delta_time = max_t(s32, nfct_time_stamp - gc_work->start_time, 1);
++ if (next_run > (unsigned long)delta_time)
++ next_run -= delta_time;
++ else
++ next_run = 1;
++
++early_exit:
+ if (gc_work->exiting)
+ return;
+
+- /*
+- * Eviction will normally happen from the packet path, and not
+- * from this gc worker.
+- *
+- * This worker is only here to reap expired entries when system went
+- * idle after a busy period.
+- */
+- if (next_run) {
++ if (next_run)
+ gc_work->early_drop = false;
+- gc_work->next_bucket = 0;
+- }
++
+ queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run);
+ }
+
+ static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
+ {
+- INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker);
++ INIT_DELAYED_WORK(&gc_work->dwork, gc_worker);
+ gc_work->exiting = false;
+ }
+
+@@ -1560,7 +1611,7 @@ __nf_conntrack_alloc(struct net *net,
+ /* save hash for reusing when confirming */
+ *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
+ ct->status = 0;
+- ct->timeout = 0;
++ WRITE_ONCE(ct->timeout, 0);
+ write_pnet(&ct->ct_net, net);
+ memset(&ct->__nfct_init_offset, 0,
+ offsetof(struct nf_conn, proto) -
+@@ -1571,7 +1622,7 @@ __nf_conntrack_alloc(struct net *net,
+ /* Because we use RCU lookups, we set ct_general.use to zero before
+ * this is inserted in any list.
+ */
+- atomic_set(&ct->ct_general.use, 0);
++ refcount_set(&ct->ct_general.use, 0);
+ return ct;
+ out:
+ atomic_dec(&cnet->count);
+@@ -1596,7 +1647,7 @@ void nf_conntrack_free(struct nf_conn *ct)
+ /* A freed object has refcnt == 0, that's
+ * the golden rule for SLAB_TYPESAFE_BY_RCU
+ */
+- WARN_ON(atomic_read(&ct->ct_general.use) != 0);
++ WARN_ON(refcount_read(&ct->ct_general.use) != 0);
+
+ nf_ct_ext_destroy(ct);
+ kmem_cache_free(nf_conntrack_cachep, ct);
+@@ -1688,8 +1739,8 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,
+ if (!exp)
+ __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
+
+- /* Now it is inserted into the unconfirmed list, bump refcount */
+- nf_conntrack_get(&ct->ct_general);
++ /* Now it is inserted into the unconfirmed list, set refcount to 1. */
++ refcount_set(&ct->ct_general.use, 1);
+ nf_ct_add_to_unconfirmed_list(ct);
+
+ local_bh_enable();
+@@ -1920,17 +1971,19 @@ repeat:
+ /* Invalid: inverse of the return code tells
+ * the netfilter core what to do */
+ pr_debug("nf_conntrack_in: Can't track with proto module\n");
+- nf_conntrack_put(&ct->ct_general);
++ nf_ct_put(ct);
+ skb->_nfct = 0;
+- NF_CT_STAT_INC_ATOMIC(state->net, invalid);
+- if (ret == -NF_DROP)
+- NF_CT_STAT_INC_ATOMIC(state->net, drop);
+ /* Special case: TCP tracker reports an attempt to reopen a
+ * closed/aborted connection. We have to go back and create a
+ * fresh conntrack.
+ */
+ if (ret == -NF_REPEAT)
+ goto repeat;
++
++ NF_CT_STAT_INC_ATOMIC(state->net, invalid);
++ if (ret == -NF_DROP)
++ NF_CT_STAT_INC_ATOMIC(state->net, drop);
++
+ ret = -ret;
+ goto out;
+ }
+@@ -2299,7 +2352,7 @@ get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
+
+ return NULL;
+ found:
+- atomic_inc(&ct->ct_general.use);
++ refcount_inc(&ct->ct_general.use);
+ spin_unlock(lockp);
+ local_bh_enable();
+ return ct;
+@@ -2772,7 +2825,7 @@ err_cachep:
+
+ static struct nf_ct_hook nf_conntrack_hook = {
+ .update = nf_conntrack_update,
+- .destroy = destroy_conntrack,
++ .destroy = nf_ct_destroy,
+ .get_tuple_skb = nf_conntrack_get_tuple_skb,
+ };
+
+diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c
+index f562eeef42349..6d056ebba57c6 100644
+--- a/net/netfilter/nf_conntrack_expect.c
++++ b/net/netfilter/nf_conntrack_expect.c
+@@ -203,12 +203,12 @@ nf_ct_find_expectation(struct net *net,
+ * about to invoke ->destroy(), or nf_ct_delete() via timeout
+ * or early_drop().
+ *
+- * The atomic_inc_not_zero() check tells: If that fails, we
++ * The refcount_inc_not_zero() check tells: If that fails, we
+ * know that the ct is being destroyed. If it succeeds, we
+ * can be sure the ct cannot disappear underneath.
+ */
+ if (unlikely(nf_ct_is_dying(exp->master) ||
+- !atomic_inc_not_zero(&exp->master->ct_general.use)))
++ !refcount_inc_not_zero(&exp->master->ct_general.use)))
+ return NULL;
+
+ if (exp->flags & NF_CT_EXPECT_PERMANENT) {
+diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c
+index ae4488a13c70c..ceb38a7b37cb7 100644
+--- a/net/netfilter/nf_conntrack_helper.c
++++ b/net/netfilter/nf_conntrack_helper.c
+@@ -556,6 +556,12 @@ static const struct nf_ct_ext_type helper_extend = {
+ .id = NF_CT_EXT_HELPER,
+ };
+
++void nf_ct_set_auto_assign_helper_warned(struct net *net)
++{
++ nf_ct_pernet(net)->auto_assign_helper_warned = true;
++}
++EXPORT_SYMBOL_GPL(nf_ct_set_auto_assign_helper_warned);
++
+ void nf_conntrack_helper_pernet_init(struct net *net)
+ {
+ struct nf_conntrack_net *cnet = nf_ct_pernet(net);
+diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c
+index f1e5443fe7c74..ef0a78aa9ba9e 100644
+--- a/net/netfilter/nf_conntrack_netlink.c
++++ b/net/netfilter/nf_conntrack_netlink.c
+@@ -508,7 +508,7 @@ nla_put_failure:
+
+ static int ctnetlink_dump_use(struct sk_buff *skb, const struct nf_conn *ct)
+ {
+- if (nla_put_be32(skb, CTA_USE, htonl(atomic_read(&ct->ct_general.use))))
++ if (nla_put_be32(skb, CTA_USE, htonl(refcount_read(&ct->ct_general.use))))
+ goto nla_put_failure;
+ return 0;
+
+@@ -1011,11 +1011,9 @@ ctnetlink_alloc_filter(const struct nlattr * const cda[], u8 family)
+ CTA_TUPLE_REPLY,
+ filter->family,
+ &filter->zone,
+- filter->orig_flags);
+- if (err < 0) {
+- err = -EINVAL;
++ filter->reply_flags);
++ if (err < 0)
+ goto err_filter;
+- }
+ }
+
+ return filter;
+@@ -1202,7 +1200,7 @@ restart:
+ ct = nf_ct_tuplehash_to_ctrack(h);
+ if (nf_ct_is_expired(ct)) {
+ if (i < ARRAY_SIZE(nf_ct_evict) &&
+- atomic_inc_not_zero(&ct->ct_general.use))
++ refcount_inc_not_zero(&ct->ct_general.use))
+ nf_ct_evict[i++] = ct;
+ continue;
+ }
+@@ -1750,7 +1748,7 @@ restart:
+ NFNL_MSG_TYPE(cb->nlh->nlmsg_type),
+ ct, dying ? true : false, 0);
+ if (res < 0) {
+- if (!atomic_inc_not_zero(&ct->ct_general.use))
++ if (!refcount_inc_not_zero(&ct->ct_general.use))
+ continue;
+ cb->args[0] = cpu;
+ cb->args[1] = (unsigned long)ct;
+@@ -2000,7 +1998,7 @@ static int ctnetlink_change_timeout(struct nf_conn *ct,
+
+ if (timeout > INT_MAX)
+ timeout = INT_MAX;
+- ct->timeout = nfct_time_stamp + (u32)timeout;
++ WRITE_ONCE(ct->timeout, nfct_time_stamp + (u32)timeout);
+
+ if (test_bit(IPS_DYING_BIT, &ct->status))
+ return -ETIME;
+@@ -2312,7 +2310,8 @@ ctnetlink_create_conntrack(struct net *net,
+ if (helper->from_nlattr)
+ helper->from_nlattr(helpinfo, ct);
+
+- /* not in hash table yet so not strictly necessary */
++ /* disable helper auto-assignment for this entry */
++ ct->status |= IPS_HELPER;
+ RCU_INIT_POINTER(help->helper, helper);
+ }
+ } else {
+diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c
+index 2394238d01c91..5a936334b517a 100644
+--- a/net/netfilter/nf_conntrack_proto_sctp.c
++++ b/net/netfilter/nf_conntrack_proto_sctp.c
+@@ -489,6 +489,15 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct,
+ pr_debug("Setting vtag %x for dir %d\n",
+ ih->init_tag, !dir);
+ ct->proto.sctp.vtag[!dir] = ih->init_tag;
++
++ /* don't renew timeout on init retransmit so
++ * port reuse by client or NAT middlebox cannot
++ * keep entry alive indefinitely (incl. nat info).
++ */
++ if (new_state == SCTP_CONNTRACK_CLOSED &&
++ old_state == SCTP_CONNTRACK_CLOSED &&
++ nf_ct_is_confirmed(ct))
++ ignore = true;
+ }
+
+ ct->proto.sctp.state = new_state;
+diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c
+index af5115e127cfd..3cee5d8ee7027 100644
+--- a/net/netfilter/nf_conntrack_proto_tcp.c
++++ b/net/netfilter/nf_conntrack_proto_tcp.c
+@@ -341,8 +341,8 @@ static void tcp_options(const struct sk_buff *skb,
+ if (!ptr)
+ return;
+
+- state->td_scale =
+- state->flags = 0;
++ state->td_scale = 0;
++ state->flags &= IP_CT_TCP_FLAG_BE_LIBERAL;
+
+ while (length > 0) {
+ int opcode=*ptr++;
+@@ -839,6 +839,16 @@ static bool tcp_can_early_drop(const struct nf_conn *ct)
+ return false;
+ }
+
++static void nf_ct_tcp_state_reset(struct ip_ct_tcp_state *state)
++{
++ state->td_end = 0;
++ state->td_maxend = 0;
++ state->td_maxwin = 0;
++ state->td_maxack = 0;
++ state->td_scale = 0;
++ state->flags &= IP_CT_TCP_FLAG_BE_LIBERAL;
++}
++
+ /* Returns verdict for packet, or -1 for invalid. */
+ int nf_conntrack_tcp_packet(struct nf_conn *ct,
+ struct sk_buff *skb,
+@@ -945,8 +955,7 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct,
+ ct->proto.tcp.last_flags &= ~IP_CT_EXP_CHALLENGE_ACK;
+ ct->proto.tcp.seen[ct->proto.tcp.last_dir].flags =
+ ct->proto.tcp.last_flags;
+- memset(&ct->proto.tcp.seen[dir], 0,
+- sizeof(struct ip_ct_tcp_state));
++ nf_ct_tcp_state_reset(&ct->proto.tcp.seen[dir]);
+ break;
+ }
+ ct->proto.tcp.last_index = index;
+diff --git a/net/netfilter/nf_conntrack_proto_udp.c b/net/netfilter/nf_conntrack_proto_udp.c
+index f8e3c0d2602f6..3b516cffc779b 100644
+--- a/net/netfilter/nf_conntrack_proto_udp.c
++++ b/net/netfilter/nf_conntrack_proto_udp.c
+@@ -104,10 +104,13 @@ int nf_conntrack_udp_packet(struct nf_conn *ct,
+ */
+ if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
+ unsigned long extra = timeouts[UDP_CT_UNREPLIED];
++ bool stream = false;
+
+ /* Still active after two seconds? Extend timeout. */
+- if (time_after(jiffies, ct->proto.udp.stream_ts))
++ if (time_after(jiffies, ct->proto.udp.stream_ts)) {
+ extra = timeouts[UDP_CT_REPLIED];
++ stream = true;
++ }
+
+ nf_ct_refresh_acct(ct, ctinfo, skb, extra);
+
+@@ -116,7 +119,7 @@ int nf_conntrack_udp_packet(struct nf_conn *ct,
+ return NF_ACCEPT;
+
+ /* Also, more likely to be important, and not a probe */
+- if (!test_and_set_bit(IPS_ASSURED_BIT, &ct->status))
++ if (stream && !test_and_set_bit(IPS_ASSURED_BIT, &ct->status))
+ nf_conntrack_event_cache(IPCT_ASSURED, ct);
+ } else {
+ nf_ct_refresh_acct(ct, ctinfo, skb, timeouts[UDP_CT_UNREPLIED]);
+diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c
+index 80f675d884b26..55aa55b252b20 100644
+--- a/net/netfilter/nf_conntrack_standalone.c
++++ b/net/netfilter/nf_conntrack_standalone.c
+@@ -303,7 +303,7 @@ static int ct_seq_show(struct seq_file *s, void *v)
+ int ret = 0;
+
+ WARN_ON(!ct);
+- if (unlikely(!atomic_inc_not_zero(&ct->ct_general.use)))
++ if (unlikely(!refcount_inc_not_zero(&ct->ct_general.use)))
+ return 0;
+
+ if (nf_ct_should_gc(ct)) {
+@@ -370,7 +370,7 @@ static int ct_seq_show(struct seq_file *s, void *v)
+ ct_show_zone(s, ct, NF_CT_DEFAULT_ZONE_DIR);
+ ct_show_delta_time(s, ct);
+
+- seq_printf(s, "use=%u\n", atomic_read(&ct->ct_general.use));
++ seq_printf(s, "use=%u\n", refcount_read(&ct->ct_general.use));
+
+ if (seq_has_overflowed(s))
+ goto release;
+@@ -823,7 +823,7 @@ static struct ctl_table nf_ct_sysctl_table[] = {
+ .mode = 0644,
+ .proc_handler = proc_dointvec_jiffies,
+ },
+-#if IS_ENABLED(CONFIG_NFT_FLOW_OFFLOAD)
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
+ [NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD] = {
+ .procname = "nf_flowtable_udp_timeout",
+ .maxlen = sizeof(unsigned int),
+diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
+index 87a7388b6c894..9fb407084c506 100644
+--- a/net/netfilter/nf_flow_table_core.c
++++ b/net/netfilter/nf_flow_table_core.c
+@@ -48,7 +48,7 @@ struct flow_offload *flow_offload_alloc(struct nf_conn *ct)
+ struct flow_offload *flow;
+
+ if (unlikely(nf_ct_is_dying(ct) ||
+- !atomic_inc_not_zero(&ct->ct_general.use)))
++ !refcount_inc_not_zero(&ct->ct_general.use)))
+ return NULL;
+
+ flow = kzalloc(sizeof(*flow), GFP_ATOMIC);
+@@ -173,12 +173,11 @@ EXPORT_SYMBOL_GPL(flow_offload_route_init);
+
+ static void flow_offload_fixup_tcp(struct ip_ct_tcp *tcp)
+ {
+- tcp->state = TCP_CONNTRACK_ESTABLISHED;
+ tcp->seen[0].td_maxwin = 0;
+ tcp->seen[1].td_maxwin = 0;
+ }
+
+-static void flow_offload_fixup_ct_timeout(struct nf_conn *ct)
++static void flow_offload_fixup_ct(struct nf_conn *ct)
+ {
+ struct net *net = nf_ct_net(ct);
+ int l4num = nf_ct_protonum(ct);
+@@ -187,7 +186,9 @@ static void flow_offload_fixup_ct_timeout(struct nf_conn *ct)
+ if (l4num == IPPROTO_TCP) {
+ struct nf_tcp_net *tn = nf_tcp_pernet(net);
+
+- timeout = tn->timeouts[TCP_CONNTRACK_ESTABLISHED];
++ flow_offload_fixup_tcp(&ct->proto.tcp);
++
++ timeout = tn->timeouts[ct->proto.tcp.state];
+ timeout -= tn->offload_timeout;
+ } else if (l4num == IPPROTO_UDP) {
+ struct nf_udp_net *tn = nf_udp_pernet(net);
+@@ -201,20 +202,8 @@ static void flow_offload_fixup_ct_timeout(struct nf_conn *ct)
+ if (timeout < 0)
+ timeout = 0;
+
+- if (nf_flow_timeout_delta(ct->timeout) > (__s32)timeout)
+- ct->timeout = nfct_time_stamp + timeout;
+-}
+-
+-static void flow_offload_fixup_ct_state(struct nf_conn *ct)
+-{
+- if (nf_ct_protonum(ct) == IPPROTO_TCP)
+- flow_offload_fixup_tcp(&ct->proto.tcp);
+-}
+-
+-static void flow_offload_fixup_ct(struct nf_conn *ct)
+-{
+- flow_offload_fixup_ct_state(ct);
+- flow_offload_fixup_ct_timeout(ct);
++ if (nf_flow_timeout_delta(READ_ONCE(ct->timeout)) > (__s32)timeout)
++ WRITE_ONCE(ct->timeout, nfct_time_stamp + timeout);
+ }
+
+ static void flow_offload_route_release(struct flow_offload *flow)
+@@ -329,8 +318,10 @@ void flow_offload_refresh(struct nf_flowtable *flow_table,
+ u32 timeout;
+
+ timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
+- if (READ_ONCE(flow->timeout) != timeout)
++ if (timeout - READ_ONCE(flow->timeout) > HZ)
+ WRITE_ONCE(flow->timeout, timeout);
++ else
++ return;
+
+ if (likely(!nf_flowtable_hw_offload(flow_table)))
+ return;
+@@ -353,22 +344,14 @@ static void flow_offload_del(struct nf_flowtable *flow_table,
+ rhashtable_remove_fast(&flow_table->rhashtable,
+ &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
+ nf_flow_offload_rhash_params);
+-
+- clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
+-
+- if (nf_flow_has_expired(flow))
+- flow_offload_fixup_ct(flow->ct);
+- else
+- flow_offload_fixup_ct_timeout(flow->ct);
+-
+ flow_offload_free(flow);
+ }
+
+ void flow_offload_teardown(struct flow_offload *flow)
+ {
++ clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
+ set_bit(NF_FLOW_TEARDOWN, &flow->flags);
+-
+- flow_offload_fixup_ct_state(flow->ct);
++ flow_offload_fixup_ct(flow->ct);
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_teardown);
+
+@@ -399,7 +382,8 @@ EXPORT_SYMBOL_GPL(flow_offload_lookup);
+
+ static int
+ nf_flow_table_iterate(struct nf_flowtable *flow_table,
+- void (*iter)(struct flow_offload *flow, void *data),
++ void (*iter)(struct nf_flowtable *flowtable,
++ struct flow_offload *flow, void *data),
+ void *data)
+ {
+ struct flow_offload_tuple_rhash *tuplehash;
+@@ -423,7 +407,7 @@ nf_flow_table_iterate(struct nf_flowtable *flow_table,
+
+ flow = container_of(tuplehash, struct flow_offload, tuplehash[0]);
+
+- iter(flow, data);
++ iter(flow_table, flow, data);
+ }
+ rhashtable_walk_stop(&hti);
+ rhashtable_walk_exit(&hti);
+@@ -431,34 +415,12 @@ nf_flow_table_iterate(struct nf_flowtable *flow_table,
+ return err;
+ }
+
+-static bool flow_offload_stale_dst(struct flow_offload_tuple *tuple)
++static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table,
++ struct flow_offload *flow, void *data)
+ {
+- struct dst_entry *dst;
+-
+- if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH ||
+- tuple->xmit_type == FLOW_OFFLOAD_XMIT_XFRM) {
+- dst = tuple->dst_cache;
+- if (!dst_check(dst, tuple->dst_cookie))
+- return true;
+- }
+-
+- return false;
+-}
+-
+-static bool nf_flow_has_stale_dst(struct flow_offload *flow)
+-{
+- return flow_offload_stale_dst(&flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple) ||
+- flow_offload_stale_dst(&flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple);
+-}
+-
+-static void nf_flow_offload_gc_step(struct flow_offload *flow, void *data)
+-{
+- struct nf_flowtable *flow_table = data;
+-
+ if (nf_flow_has_expired(flow) ||
+- nf_ct_is_dying(flow->ct) ||
+- nf_flow_has_stale_dst(flow))
+- set_bit(NF_FLOW_TEARDOWN, &flow->flags);
++ nf_ct_is_dying(flow->ct))
++ flow_offload_teardown(flow);
+
+ if (test_bit(NF_FLOW_TEARDOWN, &flow->flags)) {
+ if (test_bit(NF_FLOW_HW, &flow->flags)) {
+@@ -479,7 +441,7 @@ static void nf_flow_offload_work_gc(struct work_struct *work)
+ struct nf_flowtable *flow_table;
+
+ flow_table = container_of(work, struct nf_flowtable, gc_work.work);
+- nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, flow_table);
++ nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL);
+ queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ);
+ }
+
+@@ -595,7 +557,8 @@ int nf_flow_table_init(struct nf_flowtable *flowtable)
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_table_init);
+
+-static void nf_flow_table_do_cleanup(struct flow_offload *flow, void *data)
++static void nf_flow_table_do_cleanup(struct nf_flowtable *flow_table,
++ struct flow_offload *flow, void *data)
+ {
+ struct net_device *dev = data;
+
+@@ -637,11 +600,10 @@ void nf_flow_table_free(struct nf_flowtable *flow_table)
+
+ cancel_delayed_work_sync(&flow_table->gc_work);
+ nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL);
+- nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, flow_table);
++ nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL);
+ nf_flow_table_offload_flush(flow_table);
+ if (nf_flowtable_hw_offload(flow_table))
+- nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step,
+- flow_table);
++ nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL);
+ rhashtable_destroy(&flow_table->rhashtable);
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_table_free);
+diff --git a/net/netfilter/nf_flow_table_inet.c b/net/netfilter/nf_flow_table_inet.c
+index bc4126d8ef65f..280fdd32965f6 100644
+--- a/net/netfilter/nf_flow_table_inet.c
++++ b/net/netfilter/nf_flow_table_inet.c
+@@ -6,12 +6,29 @@
+ #include <linux/rhashtable.h>
+ #include <net/netfilter/nf_flow_table.h>
+ #include <net/netfilter/nf_tables.h>
++#include <linux/if_vlan.h>
+
+ static unsigned int
+ nf_flow_offload_inet_hook(void *priv, struct sk_buff *skb,
+ const struct nf_hook_state *state)
+ {
++ struct vlan_ethhdr *veth;
++ __be16 proto;
++
+ switch (skb->protocol) {
++ case htons(ETH_P_8021Q):
++ veth = (struct vlan_ethhdr *)skb_mac_header(skb);
++ proto = veth->h_vlan_encapsulated_proto;
++ break;
++ case htons(ETH_P_PPP_SES):
++ proto = nf_flow_pppoe_proto(skb);
++ break;
++ default:
++ proto = skb->protocol;
++ break;
++ }
++
++ switch (proto) {
+ case htons(ETH_P_IP):
+ return nf_flow_offload_ip_hook(priv, skb, state);
+ case htons(ETH_P_IPV6):
+diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c
+index 889cf88d3dba6..28026467b54cd 100644
+--- a/net/netfilter/nf_flow_table_ip.c
++++ b/net/netfilter/nf_flow_table_ip.c
+@@ -8,8 +8,6 @@
+ #include <linux/ipv6.h>
+ #include <linux/netdevice.h>
+ #include <linux/if_ether.h>
+-#include <linux/if_pppox.h>
+-#include <linux/ppp_defs.h>
+ #include <net/ip.h>
+ #include <net/ipv6.h>
+ #include <net/ip6_route.h>
+@@ -229,6 +227,15 @@ static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
+ return true;
+ }
+
++static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple)
++{
++ if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_NEIGH &&
++ tuple->xmit_type != FLOW_OFFLOAD_XMIT_XFRM)
++ return true;
++
++ return dst_check(tuple->dst_cache, tuple->dst_cookie);
++}
++
+ static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
+ const struct nf_hook_state *state,
+ struct dst_entry *dst)
+@@ -239,22 +246,6 @@ static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
+ return NF_STOLEN;
+ }
+
+-static inline __be16 nf_flow_pppoe_proto(const struct sk_buff *skb)
+-{
+- __be16 proto;
+-
+- proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
+- sizeof(struct pppoe_hdr)));
+- switch (proto) {
+- case htons(PPP_IP):
+- return htons(ETH_P_IP);
+- case htons(PPP_IPV6):
+- return htons(ETH_P_IPV6);
+- }
+-
+- return 0;
+-}
+-
+ static bool nf_flow_skb_encap_protocol(const struct sk_buff *skb, __be16 proto,
+ u32 *offset)
+ {
+@@ -364,6 +355,11 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
+ if (nf_flow_state_check(flow, iph->protocol, skb, thoff))
+ return NF_ACCEPT;
+
++ if (!nf_flow_dst_check(&tuplehash->tuple)) {
++ flow_offload_teardown(flow);
++ return NF_ACCEPT;
++ }
++
+ if (skb_try_make_writable(skb, thoff + hdrsize))
+ return NF_DROP;
+
+@@ -600,6 +596,11 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
+ if (nf_flow_state_check(flow, ip6h->nexthdr, skb, thoff))
+ return NF_ACCEPT;
+
++ if (!nf_flow_dst_check(&tuplehash->tuple)) {
++ flow_offload_teardown(flow);
++ return NF_ACCEPT;
++ }
++
+ if (skb_try_make_writable(skb, thoff + hdrsize))
+ return NF_DROP;
+
+diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c
+index d6bf1b2cd541b..b561e0a44a45f 100644
+--- a/net/netfilter/nf_flow_table_offload.c
++++ b/net/netfilter/nf_flow_table_offload.c
+@@ -65,11 +65,11 @@ static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
+ sizeof(struct in6_addr));
+ if (memcmp(&key->enc_ipv6.src, &in6addr_any,
+ sizeof(struct in6_addr)))
+- memset(&key->enc_ipv6.src, 0xff,
++ memset(&mask->enc_ipv6.src, 0xff,
+ sizeof(struct in6_addr));
+ if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
+ sizeof(struct in6_addr)))
+- memset(&key->enc_ipv6.dst, 0xff,
++ memset(&mask->enc_ipv6.dst, 0xff,
+ sizeof(struct in6_addr));
+ enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
+ key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
+diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c
+index 6d12afabfe8a3..63d1516816b1f 100644
+--- a/net/netfilter/nf_queue.c
++++ b/net/netfilter/nf_queue.c
+@@ -46,6 +46,15 @@ void nf_unregister_queue_handler(void)
+ }
+ EXPORT_SYMBOL(nf_unregister_queue_handler);
+
++static void nf_queue_sock_put(struct sock *sk)
++{
++#ifdef CONFIG_INET
++ sock_gen_put(sk);
++#else
++ sock_put(sk);
++#endif
++}
++
+ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry)
+ {
+ struct nf_hook_state *state = &entry->state;
+@@ -54,7 +63,7 @@ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry)
+ dev_put(state->in);
+ dev_put(state->out);
+ if (state->sk)
+- sock_put(state->sk);
++ nf_queue_sock_put(state->sk);
+
+ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
+ dev_put(entry->physin);
+@@ -87,19 +96,21 @@ static void __nf_queue_entry_init_physdevs(struct nf_queue_entry *entry)
+ }
+
+ /* Bump dev refs so they don't vanish while packet is out */
+-void nf_queue_entry_get_refs(struct nf_queue_entry *entry)
++bool nf_queue_entry_get_refs(struct nf_queue_entry *entry)
+ {
+ struct nf_hook_state *state = &entry->state;
+
++ if (state->sk && !refcount_inc_not_zero(&state->sk->sk_refcnt))
++ return false;
++
+ dev_hold(state->in);
+ dev_hold(state->out);
+- if (state->sk)
+- sock_hold(state->sk);
+
+ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
+ dev_hold(entry->physin);
+ dev_hold(entry->physout);
+ #endif
++ return true;
+ }
+ EXPORT_SYMBOL_GPL(nf_queue_entry_get_refs);
+
+@@ -169,6 +180,18 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
+ break;
+ }
+
++ if (skb_sk_is_prefetched(skb)) {
++ struct sock *sk = skb->sk;
++
++ if (!sk_is_refcounted(sk)) {
++ if (!refcount_inc_not_zero(&sk->sk_refcnt))
++ return -ENOTCONN;
++
++ /* drop refcount on skb_orphan */
++ skb->destructor = sock_edemux;
++ }
++ }
++
+ entry = kmalloc(sizeof(*entry) + route_key_size, GFP_ATOMIC);
+ if (!entry)
+ return -ENOMEM;
+@@ -187,7 +210,10 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
+
+ __nf_queue_entry_init_physdevs(entry);
+
+- nf_queue_entry_get_refs(entry);
++ if (!nf_queue_entry_get_refs(entry)) {
++ kfree(entry);
++ return -ENOTCONN;
++ }
+
+ switch (entry->state.pf) {
+ case AF_INET:
+diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c
+index 3d6d49420db8b..2dfc5dae06563 100644
+--- a/net/netfilter/nf_synproxy_core.c
++++ b/net/netfilter/nf_synproxy_core.c
+@@ -349,7 +349,6 @@ static int __net_init synproxy_net_init(struct net *net)
+ goto err2;
+
+ __set_bit(IPS_CONFIRMED_BIT, &ct->status);
+- nf_conntrack_get(&ct->ct_general);
+ snet->tmpl = ct;
+
+ snet->stats = alloc_percpu(struct synproxy_stats);
+diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
+index c0851fec11d46..2feb88ffcd81f 100644
+--- a/net/netfilter/nf_tables_api.c
++++ b/net/netfilter/nf_tables_api.c
+@@ -4481,12 +4481,12 @@ struct nft_set_elem_catchall {
+ static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
+ struct nft_set *set)
+ {
+- struct nft_set_elem_catchall *catchall;
++ struct nft_set_elem_catchall *next, *catchall;
+
+- list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
++ list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
+ list_del_rcu(&catchall->list);
+ nft_set_elem_destroy(set, catchall->elem, true);
+- kfree_rcu(catchall);
++ kfree_rcu(catchall, rcu);
+ }
+ }
+
+@@ -5653,7 +5653,7 @@ static void nft_setelem_catchall_remove(const struct net *net,
+ list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
+ if (catchall->elem == elem->priv) {
+ list_del_rcu(&catchall->list);
+- kfree_rcu(catchall);
++ kfree_rcu(catchall, rcu);
+ break;
+ }
+ }
+@@ -6535,12 +6535,15 @@ static int nf_tables_updobj(const struct nft_ctx *ctx,
+ {
+ struct nft_object *newobj;
+ struct nft_trans *trans;
+- int err;
++ int err = -ENOMEM;
++
++ if (!try_module_get(type->owner))
++ return -ENOENT;
+
+ trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
+ sizeof(struct nft_trans_obj));
+ if (!trans)
+- return -ENOMEM;
++ goto err_trans;
+
+ newobj = nft_obj_init(ctx, type, attr);
+ if (IS_ERR(newobj)) {
+@@ -6557,6 +6560,8 @@ static int nf_tables_updobj(const struct nft_ctx *ctx,
+
+ err_free_trans:
+ kfree(trans);
++err_trans:
++ module_put(type->owner);
+ return err;
+ }
+
+@@ -8169,7 +8174,7 @@ static void nft_obj_commit_update(struct nft_trans *trans)
+ if (obj->ops->update)
+ obj->ops->update(obj, newobj);
+
+- kfree(newobj);
++ nft_obj_destroy(&trans->ctx, newobj);
+ }
+
+ static void nft_commit_release(struct nft_trans *trans)
+@@ -8914,7 +8919,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
+ break;
+ case NFT_MSG_NEWOBJ:
+ if (nft_trans_obj_update(trans)) {
+- kfree(nft_trans_obj_newobj(trans));
++ nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
+ nft_trans_destroy(trans);
+ } else {
+ trans->ctx.table->use--;
+@@ -9203,17 +9208,23 @@ int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
+ }
+ EXPORT_SYMBOL_GPL(nft_parse_u32_check);
+
+-static unsigned int nft_parse_register(const struct nlattr *attr)
++static int nft_parse_register(const struct nlattr *attr, u32 *preg)
+ {
+ unsigned int reg;
+
+ reg = ntohl(nla_get_be32(attr));
+ switch (reg) {
+ case NFT_REG_VERDICT...NFT_REG_4:
+- return reg * NFT_REG_SIZE / NFT_REG32_SIZE;
++ *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
++ break;
++ case NFT_REG32_00...NFT_REG32_15:
++ *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
++ break;
+ default:
+- return reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
++ return -ERANGE;
+ }
++
++ return 0;
+ }
+
+ /**
+@@ -9255,7 +9266,10 @@ int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
+ u32 reg;
+ int err;
+
+- reg = nft_parse_register(attr);
++ err = nft_parse_register(attr, &reg);
++ if (err < 0)
++ return err;
++
+ err = nft_validate_register_load(reg, len);
+ if (err < 0)
+ return err;
+@@ -9310,7 +9324,10 @@ int nft_parse_register_store(const struct nft_ctx *ctx,
+ int err;
+ u32 reg;
+
+- reg = nft_parse_register(attr);
++ err = nft_parse_register(attr, &reg);
++ if (err < 0)
++ return err;
++
+ err = nft_validate_register_store(ctx, reg, data, type, len);
+ if (err < 0)
+ return err;
+@@ -9574,10 +9591,13 @@ EXPORT_SYMBOL_GPL(__nft_release_basechain);
+
+ static void __nft_release_hook(struct net *net, struct nft_table *table)
+ {
++ struct nft_flowtable *flowtable;
+ struct nft_chain *chain;
+
+ list_for_each_entry(chain, &table->chains, list)
+ nf_tables_unregister_hook(net, table, chain);
++ list_for_each_entry(flowtable, &table->flowtables, list)
++ nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list);
+ }
+
+ static void __nft_release_hooks(struct net *net)
+diff --git a/net/netfilter/nf_tables_core.c b/net/netfilter/nf_tables_core.c
+index 866cfba04d6c0..907e848dbc178 100644
+--- a/net/netfilter/nf_tables_core.c
++++ b/net/netfilter/nf_tables_core.c
+@@ -162,7 +162,7 @@ nft_do_chain(struct nft_pktinfo *pkt, void *priv)
+ struct nft_rule *const *rules;
+ const struct nft_rule *rule;
+ const struct nft_expr *expr, *last;
+- struct nft_regs regs;
++ struct nft_regs regs = {};
+ unsigned int stackptr = 0;
+ struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
+ bool genbit = READ_ONCE(net->nft.gencursor);
+diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c
+index 9656c16462222..2d36952b13920 100644
+--- a/net/netfilter/nf_tables_offload.c
++++ b/net/netfilter/nf_tables_offload.c
+@@ -94,7 +94,8 @@ struct nft_flow_rule *nft_flow_rule_create(struct net *net,
+
+ expr = nft_expr_first(rule);
+ while (nft_expr_more(rule, expr)) {
+- if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
++ if (expr->ops->offload_action &&
++ expr->ops->offload_action(expr))
+ num_actions++;
+
+ expr = nft_expr_next(expr);
+diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c
+index 691ef4cffdd90..7f83f9697fc14 100644
+--- a/net/netfilter/nfnetlink_log.c
++++ b/net/netfilter/nfnetlink_log.c
+@@ -556,7 +556,8 @@ __build_packet_message(struct nfnl_log_net *log,
+ goto nla_put_failure;
+
+ if (indev && skb->dev &&
+- skb->mac_header != skb->network_header) {
++ skb_mac_header_was_set(skb) &&
++ skb_mac_header_len(skb) != 0) {
+ struct nfulnl_msg_packet_hw phw;
+ int len;
+
+diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c
+index 4c3fbaaeb1030..8787d0613ad83 100644
+--- a/net/netfilter/nfnetlink_queue.c
++++ b/net/netfilter/nfnetlink_queue.c
+@@ -560,7 +560,8 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue,
+ goto nla_put_failure;
+
+ if (indev && entskb->dev &&
+- entskb->mac_header != entskb->network_header) {
++ skb_mac_header_was_set(entskb) &&
++ skb_mac_header_len(entskb) != 0) {
+ struct nfqnl_msg_packet_hw phw;
+ int len;
+
+@@ -709,9 +710,15 @@ static struct nf_queue_entry *
+ nf_queue_entry_dup(struct nf_queue_entry *e)
+ {
+ struct nf_queue_entry *entry = kmemdup(e, e->size, GFP_ATOMIC);
+- if (entry)
+- nf_queue_entry_get_refs(entry);
+- return entry;
++
++ if (!entry)
++ return NULL;
++
++ if (nf_queue_entry_get_refs(entry))
++ return entry;
++
++ kfree(entry);
++ return NULL;
+ }
+
+ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
+diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c
+index 99b1de14ff7ee..9c7472af9e4a1 100644
+--- a/net/netfilter/nft_ct.c
++++ b/net/netfilter/nft_ct.c
+@@ -259,10 +259,13 @@ static void nft_ct_set_zone_eval(const struct nft_expr *expr,
+
+ ct = this_cpu_read(nft_ct_pcpu_template);
+
+- if (likely(atomic_read(&ct->ct_general.use) == 1)) {
++ if (likely(refcount_read(&ct->ct_general.use) == 1)) {
++ refcount_inc(&ct->ct_general.use);
+ nf_ct_zone_add(ct, &zone);
+ } else {
+- /* previous skb got queued to userspace */
++ /* previous skb got queued to userspace, allocate temporary
++ * one until percpu template can be reused.
++ */
+ ct = nf_ct_tmpl_alloc(nft_net(pkt), &zone, GFP_ATOMIC);
+ if (!ct) {
+ regs->verdict.code = NF_DROP;
+@@ -270,7 +273,6 @@ static void nft_ct_set_zone_eval(const struct nft_expr *expr,
+ }
+ }
+
+- atomic_inc(&ct->ct_general.use);
+ nf_ct_set(skb, ct, IP_CT_NEW);
+ }
+ #endif
+@@ -375,7 +377,6 @@ static bool nft_ct_tmpl_alloc_pcpu(void)
+ return false;
+ }
+
+- atomic_set(&tmp->ct_general.use, 1);
+ per_cpu(nft_ct_pcpu_template, cpu) = tmp;
+ }
+
+@@ -1040,6 +1041,9 @@ static int nft_ct_helper_obj_init(const struct nft_ctx *ctx,
+ if (err < 0)
+ goto err_put_helper;
+
++ /* Avoid the bogus warning, helper will be assigned after CT init */
++ nf_ct_set_auto_assign_helper_warned(ctx->net);
++
+ return 0;
+
+ err_put_helper:
+diff --git a/net/netfilter/nft_dup_netdev.c b/net/netfilter/nft_dup_netdev.c
+index bbf3fcba3df40..5b5c607fbf83f 100644
+--- a/net/netfilter/nft_dup_netdev.c
++++ b/net/netfilter/nft_dup_netdev.c
+@@ -67,6 +67,11 @@ static int nft_dup_netdev_offload(struct nft_offload_ctx *ctx,
+ return nft_fwd_dup_netdev_offload(ctx, flow, FLOW_ACTION_MIRRED, oif);
+ }
+
++static bool nft_dup_netdev_offload_action(const struct nft_expr *expr)
++{
++ return true;
++}
++
+ static struct nft_expr_type nft_dup_netdev_type;
+ static const struct nft_expr_ops nft_dup_netdev_ops = {
+ .type = &nft_dup_netdev_type,
+@@ -75,6 +80,7 @@ static const struct nft_expr_ops nft_dup_netdev_ops = {
+ .init = nft_dup_netdev_init,
+ .dump = nft_dup_netdev_dump,
+ .offload = nft_dup_netdev_offload,
++ .offload_action = nft_dup_netdev_offload_action,
+ };
+
+ static struct nft_expr_type nft_dup_netdev_type __read_mostly = {
+diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c
+index 6ba3256fa8449..87f3af4645d9c 100644
+--- a/net/netfilter/nft_dynset.c
++++ b/net/netfilter/nft_dynset.c
+@@ -198,17 +198,8 @@ static int nft_dynset_init(const struct nft_ctx *ctx,
+ return -EBUSY;
+
+ priv->op = ntohl(nla_get_be32(tb[NFTA_DYNSET_OP]));
+- switch (priv->op) {
+- case NFT_DYNSET_OP_ADD:
+- case NFT_DYNSET_OP_DELETE:
+- break;
+- case NFT_DYNSET_OP_UPDATE:
+- if (!(set->flags & NFT_SET_TIMEOUT))
+- return -EOPNOTSUPP;
+- break;
+- default:
++ if (priv->op > NFT_DYNSET_OP_DELETE)
+ return -EOPNOTSUPP;
+- }
+
+ timeout = 0;
+ if (tb[NFTA_DYNSET_TIMEOUT] != NULL) {
+diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c
+index af4ee874a067c..dbe1f2e7dd9ed 100644
+--- a/net/netfilter/nft_exthdr.c
++++ b/net/netfilter/nft_exthdr.c
+@@ -236,7 +236,7 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr,
+
+ tcph = nft_tcp_header_pointer(pkt, sizeof(buff), buff, &tcphdr_len);
+ if (!tcph)
+- return;
++ goto err;
+
+ opt = (u8 *)tcph;
+ for (i = sizeof(*tcph); i < tcphdr_len - 1; i += optl) {
+@@ -251,16 +251,16 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr,
+ continue;
+
+ if (i + optl > tcphdr_len || priv->len + priv->offset > optl)
+- return;
++ goto err;
+
+ if (skb_ensure_writable(pkt->skb,
+ nft_thoff(pkt) + i + priv->len))
+- return;
++ goto err;
+
+ tcph = nft_tcp_header_pointer(pkt, sizeof(buff), buff,
+ &tcphdr_len);
+ if (!tcph)
+- return;
++ goto err;
+
+ offset = i + priv->offset;
+
+@@ -303,6 +303,9 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr,
+
+ return;
+ }
++ return;
++err:
++ regs->verdict.code = NFT_BREAK;
+ }
+
+ static void nft_exthdr_sctp_eval(const struct nft_expr *expr,
+diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c
+index 0af34ad414796..aac6db8680d47 100644
+--- a/net/netfilter/nft_flow_offload.c
++++ b/net/netfilter/nft_flow_offload.c
+@@ -36,6 +36,15 @@ static void nft_default_forward_path(struct nf_flow_route *route,
+ route->tuple[dir].xmit_type = nft_xmit_type(dst_cache);
+ }
+
++static bool nft_is_valid_ether_device(const struct net_device *dev)
++{
++ if (!dev || (dev->flags & IFF_LOOPBACK) || dev->type != ARPHRD_ETHER ||
++ dev->addr_len != ETH_ALEN || !is_valid_ether_addr(dev->dev_addr))
++ return false;
++
++ return true;
++}
++
+ static int nft_dev_fill_forward_path(const struct nf_flow_route *route,
+ const struct dst_entry *dst_cache,
+ const struct nf_conn *ct,
+@@ -47,6 +56,9 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route,
+ struct neighbour *n;
+ u8 nud_state;
+
++ if (!nft_is_valid_ether_device(dev))
++ goto out;
++
+ n = dst_neigh_lookup(dst_cache, daddr);
+ if (!n)
+ return -1;
+@@ -60,6 +72,7 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route,
+ if (!(nud_state & NUD_VALID))
+ return -1;
+
++out:
+ return dev_fill_forward_path(dev, ha, stack);
+ }
+
+@@ -78,15 +91,6 @@ struct nft_forward_info {
+ enum flow_offload_xmit_type xmit_type;
+ };
+
+-static bool nft_is_valid_ether_device(const struct net_device *dev)
+-{
+- if (!dev || (dev->flags & IFF_LOOPBACK) || dev->type != ARPHRD_ETHER ||
+- dev->addr_len != ETH_ALEN || !is_valid_ether_addr(dev->dev_addr))
+- return false;
+-
+- return true;
+-}
+-
+ static void nft_dev_path_info(const struct net_device_path_stack *stack,
+ struct nft_forward_info *info,
+ unsigned char *ha, struct nf_flowtable *flowtable)
+@@ -119,7 +123,8 @@ static void nft_dev_path_info(const struct net_device_path_stack *stack,
+ info->indev = NULL;
+ break;
+ }
+- info->outdev = path->dev;
++ if (!info->outdev)
++ info->outdev = path->dev;
+ info->encap[info->num_encaps].id = path->encap.id;
+ info->encap[info->num_encaps].proto = path->encap.proto;
+ info->num_encaps++;
+@@ -293,7 +298,8 @@ static void nft_flow_offload_eval(const struct nft_expr *expr,
+ case IPPROTO_TCP:
+ tcph = skb_header_pointer(pkt->skb, nft_thoff(pkt),
+ sizeof(_tcph), &_tcph);
+- if (unlikely(!tcph || tcph->fin || tcph->rst))
++ if (unlikely(!tcph || tcph->fin || tcph->rst ||
++ !nf_conntrack_tcp_established(ct)))
+ goto out;
+ break;
+ case IPPROTO_UDP:
+diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c
+index cd59afde5b2f8..7730409f6f091 100644
+--- a/net/netfilter/nft_fwd_netdev.c
++++ b/net/netfilter/nft_fwd_netdev.c
+@@ -77,6 +77,11 @@ static int nft_fwd_netdev_offload(struct nft_offload_ctx *ctx,
+ return nft_fwd_dup_netdev_offload(ctx, flow, FLOW_ACTION_REDIRECT, oif);
+ }
+
++static bool nft_fwd_netdev_offload_action(const struct nft_expr *expr)
++{
++ return true;
++}
++
+ struct nft_fwd_neigh {
+ u8 sreg_dev;
+ u8 sreg_addr;
+@@ -219,6 +224,7 @@ static const struct nft_expr_ops nft_fwd_netdev_ops = {
+ .dump = nft_fwd_netdev_dump,
+ .validate = nft_fwd_validate,
+ .offload = nft_fwd_netdev_offload,
++ .offload_action = nft_fwd_netdev_offload_action,
+ };
+
+ static const struct nft_expr_ops *
+diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c
+index 90c64d27ae532..d0f67d325bdfd 100644
+--- a/net/netfilter/nft_immediate.c
++++ b/net/netfilter/nft_immediate.c
+@@ -213,6 +213,16 @@ static int nft_immediate_offload(struct nft_offload_ctx *ctx,
+ return 0;
+ }
+
++static bool nft_immediate_offload_action(const struct nft_expr *expr)
++{
++ const struct nft_immediate_expr *priv = nft_expr_priv(expr);
++
++ if (priv->dreg == NFT_REG_VERDICT)
++ return true;
++
++ return false;
++}
++
+ static const struct nft_expr_ops nft_imm_ops = {
+ .type = &nft_imm_type,
+ .size = NFT_EXPR_SIZE(sizeof(struct nft_immediate_expr)),
+@@ -224,7 +234,7 @@ static const struct nft_expr_ops nft_imm_ops = {
+ .dump = nft_immediate_dump,
+ .validate = nft_immediate_validate,
+ .offload = nft_immediate_offload,
+- .offload_flags = NFT_OFFLOAD_F_ACTION,
++ .offload_action = nft_immediate_offload_action,
+ };
+
+ struct nft_expr_type nft_imm_type __read_mostly = {
+diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c
+index a44b14f6c0dc0..132875cd7fff2 100644
+--- a/net/netfilter/nft_payload.c
++++ b/net/netfilter/nft_payload.c
+@@ -502,6 +502,9 @@ static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
+ struct sk_buff *skb,
+ unsigned int *l4csum_offset)
+ {
++ if (pkt->fragoff)
++ return -1;
++
+ switch (pkt->tprot) {
+ case IPPROTO_TCP:
+ *l4csum_offset = offsetof(struct tcphdr, check);
+diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c
+index dce866d93feed..2c8051d8cca69 100644
+--- a/net/netfilter/nft_set_pipapo.c
++++ b/net/netfilter/nft_set_pipapo.c
+@@ -1290,6 +1290,11 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old)
+ if (!new->scratch_aligned)
+ goto out_scratch;
+ #endif
++ for_each_possible_cpu(i)
++ *per_cpu_ptr(new->scratch, i) = NULL;
++
++ if (pipapo_realloc_scratch(new, old->bsize_max))
++ goto out_scratch_realloc;
+
+ rcu_head_init(&new->rcu);
+
+@@ -1334,6 +1339,9 @@ out_lt:
+ kvfree(dst->lt);
+ dst--;
+ }
++out_scratch_realloc:
++ for_each_possible_cpu(i)
++ kfree(*per_cpu_ptr(new->scratch, i));
+ #ifdef NFT_PIPAPO_ALIGN
+ free_percpu(new->scratch_aligned);
+ #endif
+diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c
+index e517663e0cd17..6f4116e729581 100644
+--- a/net/netfilter/nft_set_pipapo_avx2.c
++++ b/net/netfilter/nft_set_pipapo_avx2.c
+@@ -886,7 +886,7 @@ static int nft_pipapo_avx2_lookup_8b_6(unsigned long *map, unsigned long *fill,
+ NFT_PIPAPO_AVX2_BUCKET_LOAD8(4, lt, 4, pkt[4], bsize);
+
+ NFT_PIPAPO_AVX2_AND(5, 0, 1);
+- NFT_PIPAPO_AVX2_BUCKET_LOAD8(6, lt, 6, pkt[5], bsize);
++ NFT_PIPAPO_AVX2_BUCKET_LOAD8(6, lt, 5, pkt[5], bsize);
+ NFT_PIPAPO_AVX2_AND(7, 2, 3);
+
+ /* Stall */
+diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c
+index d600a566da324..7325bee7d1442 100644
+--- a/net/netfilter/nft_set_rbtree.c
++++ b/net/netfilter/nft_set_rbtree.c
+@@ -349,7 +349,11 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set,
+ *ext = &rbe->ext;
+ return -EEXIST;
+ } else {
+- p = &parent->rb_left;
++ overlap = false;
++ if (nft_rbtree_interval_end(rbe))
++ p = &parent->rb_left;
++ else
++ p = &parent->rb_right;
+ }
+ }
+
+diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c
+index d601974c9d2e0..9ad9cc0d1d27c 100644
+--- a/net/netfilter/nft_socket.c
++++ b/net/netfilter/nft_socket.c
+@@ -36,12 +36,11 @@ static void nft_socket_wildcard(const struct nft_pktinfo *pkt,
+
+ #ifdef CONFIG_SOCK_CGROUP_DATA
+ static noinline bool
+-nft_sock_get_eval_cgroupv2(u32 *dest, const struct nft_pktinfo *pkt, u32 level)
++nft_sock_get_eval_cgroupv2(u32 *dest, struct sock *sk, const struct nft_pktinfo *pkt, u32 level)
+ {
+- struct sock *sk = skb_to_full_sk(pkt->skb);
+ struct cgroup *cgrp;
+
+- if (!sk || !sk_fullsock(sk) || !net_eq(nft_net(pkt), sock_net(sk)))
++ if (!sk_fullsock(sk))
+ return false;
+
+ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
+@@ -54,6 +53,32 @@ nft_sock_get_eval_cgroupv2(u32 *dest, const struct nft_pktinfo *pkt, u32 level)
+ }
+ #endif
+
++static struct sock *nft_socket_do_lookup(const struct nft_pktinfo *pkt)
++{
++ const struct net_device *indev = nft_in(pkt);
++ const struct sk_buff *skb = pkt->skb;
++ struct sock *sk = NULL;
++
++ if (!indev)
++ return NULL;
++
++ switch (nft_pf(pkt)) {
++ case NFPROTO_IPV4:
++ sk = nf_sk_lookup_slow_v4(nft_net(pkt), skb, indev);
++ break;
++#if IS_ENABLED(CONFIG_NF_TABLES_IPV6)
++ case NFPROTO_IPV6:
++ sk = nf_sk_lookup_slow_v6(nft_net(pkt), skb, indev);
++ break;
++#endif
++ default:
++ WARN_ON_ONCE(1);
++ break;
++ }
++
++ return sk;
++}
++
+ static void nft_socket_eval(const struct nft_expr *expr,
+ struct nft_regs *regs,
+ const struct nft_pktinfo *pkt)
+@@ -67,20 +92,7 @@ static void nft_socket_eval(const struct nft_expr *expr,
+ sk = NULL;
+
+ if (!sk)
+- switch(nft_pf(pkt)) {
+- case NFPROTO_IPV4:
+- sk = nf_sk_lookup_slow_v4(nft_net(pkt), skb, nft_in(pkt));
+- break;
+-#if IS_ENABLED(CONFIG_NF_TABLES_IPV6)
+- case NFPROTO_IPV6:
+- sk = nf_sk_lookup_slow_v6(nft_net(pkt), skb, nft_in(pkt));
+- break;
+-#endif
+- default:
+- WARN_ON_ONCE(1);
+- regs->verdict.code = NFT_BREAK;
+- return;
+- }
++ sk = nft_socket_do_lookup(pkt);
+
+ if (!sk) {
+ regs->verdict.code = NFT_BREAK;
+@@ -108,7 +120,7 @@ static void nft_socket_eval(const struct nft_expr *expr,
+ break;
+ #ifdef CONFIG_SOCK_CGROUP_DATA
+ case NFT_SOCKET_CGROUPV2:
+- if (!nft_sock_get_eval_cgroupv2(dest, pkt, priv->level)) {
++ if (!nft_sock_get_eval_cgroupv2(dest, sk, pkt, priv->level)) {
+ regs->verdict.code = NFT_BREAK;
+ return;
+ }
+@@ -198,6 +210,16 @@ static int nft_socket_dump(struct sk_buff *skb,
+ return 0;
+ }
+
++static int nft_socket_validate(const struct nft_ctx *ctx,
++ const struct nft_expr *expr,
++ const struct nft_data **data)
++{
++ return nft_chain_validate_hooks(ctx->chain,
++ (1 << NF_INET_PRE_ROUTING) |
++ (1 << NF_INET_LOCAL_IN) |
++ (1 << NF_INET_LOCAL_OUT));
++}
++
+ static struct nft_expr_type nft_socket_type;
+ static const struct nft_expr_ops nft_socket_ops = {
+ .type = &nft_socket_type,
+@@ -205,6 +227,7 @@ static const struct nft_expr_ops nft_socket_ops = {
+ .eval = nft_socket_eval,
+ .init = nft_socket_init,
+ .dump = nft_socket_dump,
++ .validate = nft_socket_validate,
+ };
+
+ static struct nft_expr_type nft_socket_type __read_mostly = {
+diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c
+index a0109fa1e92d0..1133e06f3c40e 100644
+--- a/net/netfilter/nft_synproxy.c
++++ b/net/netfilter/nft_synproxy.c
+@@ -191,8 +191,10 @@ static int nft_synproxy_do_init(const struct nft_ctx *ctx,
+ if (err)
+ goto nf_ct_failure;
+ err = nf_synproxy_ipv6_init(snet, ctx->net);
+- if (err)
++ if (err) {
++ nf_synproxy_ipv4_fini(snet, ctx->net);
+ goto nf_ct_failure;
++ }
+ break;
+ }
+
+diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c
+index 0a913ce07425a..267757b0392a6 100644
+--- a/net/netfilter/xt_CT.c
++++ b/net/netfilter/xt_CT.c
+@@ -24,7 +24,7 @@ static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct)
+ return XT_CONTINUE;
+
+ if (ct) {
+- atomic_inc(&ct->ct_general.use);
++ refcount_inc(&ct->ct_general.use);
+ nf_ct_set(skb, ct, IP_CT_NEW);
+ } else {
+ nf_ct_set(skb, ct, IP_CT_UNTRACKED);
+@@ -201,7 +201,6 @@ static int xt_ct_tg_check(const struct xt_tgchk_param *par,
+ goto err4;
+ }
+ __set_bit(IPS_CONFIRMED_BIT, &ct->status);
+- nf_conntrack_get(&ct->ct_general);
+ out:
+ info->ct = ct;
+ return 0;
+diff --git a/net/netfilter/xt_socket.c b/net/netfilter/xt_socket.c
+index 5e6459e116055..7013f55f05d1e 100644
+--- a/net/netfilter/xt_socket.c
++++ b/net/netfilter/xt_socket.c
+@@ -220,8 +220,10 @@ static void socket_mt_destroy(const struct xt_mtdtor_param *par)
+ {
+ if (par->family == NFPROTO_IPV4)
+ nf_defrag_ipv4_disable(par->net);
++#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
+ else if (par->family == NFPROTO_IPV6)
+- nf_defrag_ipv4_disable(par->net);
++ nf_defrag_ipv6_disable(par->net);
++#endif
+ }
+
+ static struct xt_match socket_mt_reg[] __read_mostly = {
+diff --git a/net/netlabel/netlabel_kapi.c b/net/netlabel/netlabel_kapi.c
+index beb0e573266d0..54c0830039470 100644
+--- a/net/netlabel/netlabel_kapi.c
++++ b/net/netlabel/netlabel_kapi.c
+@@ -885,6 +885,8 @@ int netlbl_bitmap_walk(const unsigned char *bitmap, u32 bitmap_len,
+ unsigned char bitmask;
+ unsigned char byte;
+
++ if (offset >= bitmap_len)
++ return -1;
+ byte_offset = offset / 8;
+ byte = bitmap[byte_offset];
+ bit_spot = offset;
+diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
+index ada47e59647a0..974d32632ef41 100644
+--- a/net/netlink/af_netlink.c
++++ b/net/netlink/af_netlink.c
+@@ -157,6 +157,8 @@ EXPORT_SYMBOL(do_trace_netlink_extack);
+
+ static inline u32 netlink_group_mask(u32 group)
+ {
++ if (group > 32)
++ return 0;
+ return group ? 1 << (group - 1) : 0;
+ }
+
+@@ -1871,6 +1873,11 @@ static int netlink_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
+ if (msg->msg_flags & MSG_OOB)
+ return -EOPNOTSUPP;
+
++ if (len == 0) {
++ pr_warn_once("Zero length message leads to an empty skb\n");
++ return -ENODATA;
++ }
++
+ err = scm_send(sock, msg, &scm, true);
+ if (err < 0)
+ return err;
+@@ -1989,7 +1996,6 @@ static int netlink_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+ copied = len;
+ }
+
+- skb_reset_transport_header(data_skb);
+ err = skb_copy_datagram_msg(data_skb, 0, msg, copied);
+
+ if (msg->msg_name) {
+@@ -2277,6 +2283,13 @@ static int netlink_dump(struct sock *sk)
+ * single netdev. The outcome is MSG_TRUNC error.
+ */
+ skb_reserve(skb, skb_tailroom(skb) - alloc_size);
++
++ /* Make sure malicious BPF programs can not read unitialized memory
++ * from skb->head -> skb->data
++ */
++ skb_reset_network_header(skb);
++ skb_reset_mac_header(skb);
++
+ netlink_skb_set_owner_r(skb, sk);
+
+ if (nlk->dump_done_errno > 0) {
+diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c
+index 6d16e1ab1a8ab..e5c8a295e6406 100644
+--- a/net/netrom/af_netrom.c
++++ b/net/netrom/af_netrom.c
+@@ -298,7 +298,7 @@ static int nr_setsockopt(struct socket *sock, int level, int optname,
+ {
+ struct sock *sk = sock->sk;
+ struct nr_sock *nr = nr_sk(sk);
+- unsigned long opt;
++ unsigned int opt;
+
+ if (level != SOL_NETROM)
+ return -ENOPROTOOPT;
+@@ -306,18 +306,18 @@ static int nr_setsockopt(struct socket *sock, int level, int optname,
+ if (optlen < sizeof(unsigned int))
+ return -EINVAL;
+
+- if (copy_from_sockptr(&opt, optval, sizeof(unsigned int)))
++ if (copy_from_sockptr(&opt, optval, sizeof(opt)))
+ return -EFAULT;
+
+ switch (optname) {
+ case NETROM_T1:
+- if (opt < 1 || opt > ULONG_MAX / HZ)
++ if (opt < 1 || opt > UINT_MAX / HZ)
+ return -EINVAL;
+ nr->t1 = opt * HZ;
+ return 0;
+
+ case NETROM_T2:
+- if (opt < 1 || opt > ULONG_MAX / HZ)
++ if (opt < 1 || opt > UINT_MAX / HZ)
+ return -EINVAL;
+ nr->t2 = opt * HZ;
+ return 0;
+@@ -329,13 +329,13 @@ static int nr_setsockopt(struct socket *sock, int level, int optname,
+ return 0;
+
+ case NETROM_T4:
+- if (opt < 1 || opt > ULONG_MAX / HZ)
++ if (opt < 1 || opt > UINT_MAX / HZ)
+ return -EINVAL;
+ nr->t4 = opt * HZ;
+ return 0;
+
+ case NETROM_IDLE:
+- if (opt > ULONG_MAX / (60 * HZ))
++ if (opt > UINT_MAX / (60 * HZ))
+ return -EINVAL;
+ nr->idle = opt * 60 * HZ;
+ return 0;
+diff --git a/net/nfc/core.c b/net/nfc/core.c
+index 3c645c1d99c9b..5b286e1e0a6ff 100644
+--- a/net/nfc/core.c
++++ b/net/nfc/core.c
+@@ -38,7 +38,7 @@ int nfc_fw_download(struct nfc_dev *dev, const char *firmware_name)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -94,13 +94,13 @@ int nfc_dev_up(struct nfc_dev *dev)
+
+ device_lock(&dev->dev);
+
+- if (dev->rfkill && rfkill_blocked(dev->rfkill)) {
+- rc = -ERFKILL;
++ if (dev->shutting_down) {
++ rc = -ENODEV;
+ goto error;
+ }
+
+- if (!device_is_registered(&dev->dev)) {
+- rc = -ENODEV;
++ if (dev->rfkill && rfkill_blocked(dev->rfkill)) {
++ rc = -ERFKILL;
+ goto error;
+ }
+
+@@ -142,7 +142,7 @@ int nfc_dev_down(struct nfc_dev *dev)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -207,7 +207,7 @@ int nfc_start_poll(struct nfc_dev *dev, u32 im_protocols, u32 tm_protocols)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -246,7 +246,7 @@ int nfc_stop_poll(struct nfc_dev *dev)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -291,7 +291,7 @@ int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -335,7 +335,7 @@ int nfc_dep_link_down(struct nfc_dev *dev)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -401,7 +401,7 @@ int nfc_activate_target(struct nfc_dev *dev, u32 target_idx, u32 protocol)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -448,7 +448,7 @@ int nfc_deactivate_target(struct nfc_dev *dev, u32 target_idx, u8 mode)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -495,7 +495,7 @@ int nfc_data_exchange(struct nfc_dev *dev, u32 target_idx, struct sk_buff *skb,
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ kfree_skb(skb);
+ goto error;
+@@ -552,7 +552,7 @@ int nfc_enable_se(struct nfc_dev *dev, u32 se_idx)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -601,7 +601,7 @@ int nfc_disable_se(struct nfc_dev *dev, u32 se_idx)
+
+ device_lock(&dev->dev);
+
+- if (!device_is_registered(&dev->dev)) {
++ if (dev->shutting_down) {
+ rc = -ENODEV;
+ goto error;
+ }
+@@ -1125,11 +1125,7 @@ int nfc_register_device(struct nfc_dev *dev)
+ if (rc)
+ pr_err("Could not register llcp device\n");
+
+- rc = nfc_genl_device_added(dev);
+- if (rc)
+- pr_debug("The userspace won't be notified that the device %s was added\n",
+- dev_name(&dev->dev));
+-
++ device_lock(&dev->dev);
+ dev->rfkill = rfkill_alloc(dev_name(&dev->dev), &dev->dev,
+ RFKILL_TYPE_NFC, &nfc_rfkill_ops, dev);
+ if (dev->rfkill) {
+@@ -1138,6 +1134,13 @@ int nfc_register_device(struct nfc_dev *dev)
+ dev->rfkill = NULL;
+ }
+ }
++ dev->shutting_down = false;
++ device_unlock(&dev->dev);
++
++ rc = nfc_genl_device_added(dev);
++ if (rc)
++ pr_debug("The userspace won't be notified that the device %s was added\n",
++ dev_name(&dev->dev));
+
+ return 0;
+ }
+@@ -1154,24 +1157,24 @@ void nfc_unregister_device(struct nfc_dev *dev)
+
+ pr_debug("dev_name=%s\n", dev_name(&dev->dev));
+
++ rc = nfc_genl_device_removed(dev);
++ if (rc)
++ pr_debug("The userspace won't be notified that the device %s "
++ "was removed\n", dev_name(&dev->dev));
++
++ device_lock(&dev->dev);
+ if (dev->rfkill) {
+ rfkill_unregister(dev->rfkill);
+ rfkill_destroy(dev->rfkill);
+ }
++ dev->shutting_down = true;
++ device_unlock(&dev->dev);
+
+ if (dev->ops->check_presence) {
+- device_lock(&dev->dev);
+- dev->shutting_down = true;
+- device_unlock(&dev->dev);
+ del_timer_sync(&dev->check_pres_timer);
+ cancel_work_sync(&dev->check_pres_work);
+ }
+
+- rc = nfc_genl_device_removed(dev);
+- if (rc)
+- pr_debug("The userspace won't be notified that the device %s "
+- "was removed\n", dev_name(&dev->dev));
+-
+ nfc_llcp_unregister_device(dev);
+
+ mutex_lock(&nfc_devlist_mutex);
+diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c
+index 6cfd30fc07985..0b93a17b9f11f 100644
+--- a/net/nfc/llcp_sock.c
++++ b/net/nfc/llcp_sock.c
+@@ -789,6 +789,11 @@ static int llcp_sock_sendmsg(struct socket *sock, struct msghdr *msg,
+
+ lock_sock(sk);
+
++ if (!llcp_sock->local) {
++ release_sock(sk);
++ return -ENODEV;
++ }
++
+ if (sk->sk_type == SOCK_DGRAM) {
+ DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, addr,
+ msg->msg_name);
+diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c
+index 82ab39d80726e..189c9f428a3c2 100644
+--- a/net/nfc/nci/core.c
++++ b/net/nfc/nci/core.c
+@@ -144,12 +144,15 @@ inline int nci_request(struct nci_dev *ndev,
+ {
+ int rc;
+
+- if (!test_bit(NCI_UP, &ndev->flags))
+- return -ENETDOWN;
+-
+ /* Serialize all requests */
+ mutex_lock(&ndev->req_lock);
+- rc = __nci_request(ndev, req, opt, timeout);
++ /* check the state after obtaing the lock against any races
++ * from nci_close_device when the device gets removed.
++ */
++ if (test_bit(NCI_UP, &ndev->flags))
++ rc = __nci_request(ndev, req, opt, timeout);
++ else
++ rc = -ENETDOWN;
+ mutex_unlock(&ndev->req_lock);
+
+ return rc;
+@@ -473,6 +476,11 @@ static int nci_open_device(struct nci_dev *ndev)
+
+ mutex_lock(&ndev->req_lock);
+
++ if (test_bit(NCI_UNREG, &ndev->flags)) {
++ rc = -ENODEV;
++ goto done;
++ }
++
+ if (test_bit(NCI_UP, &ndev->flags)) {
+ rc = -EALREADY;
+ goto done;
+@@ -545,9 +553,17 @@ done:
+ static int nci_close_device(struct nci_dev *ndev)
+ {
+ nci_req_cancel(ndev, ENODEV);
++
++ /* This mutex needs to be held as a barrier for
++ * caller nci_unregister_device
++ */
+ mutex_lock(&ndev->req_lock);
+
+ if (!test_and_clear_bit(NCI_UP, &ndev->flags)) {
++ /* Need to flush the cmd wq in case
++ * there is a queued/running cmd_work
++ */
++ flush_workqueue(ndev->cmd_wq);
+ del_timer_sync(&ndev->cmd_timer);
+ del_timer_sync(&ndev->data_timer);
+ mutex_unlock(&ndev->req_lock);
+@@ -582,8 +598,8 @@ static int nci_close_device(struct nci_dev *ndev)
+
+ del_timer_sync(&ndev->cmd_timer);
+
+- /* Clear flags */
+- ndev->flags = 0;
++ /* Clear flags except NCI_UNREG */
++ ndev->flags &= BIT(NCI_UNREG);
+
+ mutex_unlock(&ndev->req_lock);
+
+@@ -1270,6 +1286,12 @@ void nci_unregister_device(struct nci_dev *ndev)
+ {
+ struct nci_conn_info *conn_info, *n;
+
++ /* This set_bit is not protected with specialized barrier,
++ * However, it is fine because the mutex_lock(&ndev->req_lock);
++ * in nci_close_device() will help to emit one.
++ */
++ set_bit(NCI_UNREG, &ndev->flags);
++
+ nci_close_device(ndev);
+
+ destroy_workqueue(ndev->cmd_wq);
+diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c
+index 6055dc9a82aa0..aa5e712adf078 100644
+--- a/net/nfc/nci/data.c
++++ b/net/nfc/nci/data.c
+@@ -118,7 +118,7 @@ static int nci_queue_tx_data_frags(struct nci_dev *ndev,
+
+ skb_frag = nci_skb_alloc(ndev,
+ (NCI_DATA_HDR_SIZE + frag_len),
+- GFP_KERNEL);
++ GFP_ATOMIC);
+ if (skb_frag == NULL) {
+ rc = -ENOMEM;
+ goto free_exit;
+diff --git a/net/nfc/nci/hci.c b/net/nfc/nci/hci.c
+index e199912ee1e59..85b808fdcbc3a 100644
+--- a/net/nfc/nci/hci.c
++++ b/net/nfc/nci/hci.c
+@@ -153,7 +153,7 @@ static int nci_hci_send_data(struct nci_dev *ndev, u8 pipe,
+
+ i = 0;
+ skb = nci_skb_alloc(ndev, conn_info->max_pkt_payload_len +
+- NCI_DATA_HDR_SIZE, GFP_KERNEL);
++ NCI_DATA_HDR_SIZE, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+@@ -184,7 +184,7 @@ static int nci_hci_send_data(struct nci_dev *ndev, u8 pipe,
+ if (i < data_len) {
+ skb = nci_skb_alloc(ndev,
+ conn_info->max_pkt_payload_len +
+- NCI_DATA_HDR_SIZE, GFP_KERNEL);
++ NCI_DATA_HDR_SIZE, GFP_ATOMIC);
+ if (!skb)
+ return -ENOMEM;
+
+diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c
+index 49089c50872e6..a207f0b8137b0 100644
+--- a/net/nfc/netlink.c
++++ b/net/nfc/netlink.c
+@@ -636,8 +636,10 @@ static int nfc_genl_dump_devices_done(struct netlink_callback *cb)
+ {
+ struct class_dev_iter *iter = (struct class_dev_iter *) cb->args[0];
+
+- nfc_device_iter_exit(iter);
+- kfree(iter);
++ if (iter) {
++ nfc_device_iter_exit(iter);
++ kfree(iter);
++ }
+
+ return 0;
+ }
+@@ -1242,7 +1244,7 @@ int nfc_genl_fw_download_done(struct nfc_dev *dev, const char *firmware_name,
+ struct sk_buff *msg;
+ void *hdr;
+
+- msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+ if (!msg)
+ return -ENOMEM;
+
+@@ -1258,7 +1260,7 @@ int nfc_genl_fw_download_done(struct nfc_dev *dev, const char *firmware_name,
+
+ genlmsg_end(msg, hdr);
+
+- genlmsg_multicast(&nfc_genl_family, msg, 0, 0, GFP_KERNEL);
++ genlmsg_multicast(&nfc_genl_family, msg, 0, 0, GFP_ATOMIC);
+
+ return 0;
+
+@@ -1392,8 +1394,10 @@ static int nfc_genl_dump_ses_done(struct netlink_callback *cb)
+ {
+ struct class_dev_iter *iter = (struct class_dev_iter *) cb->args[0];
+
+- nfc_device_iter_exit(iter);
+- kfree(iter);
++ if (iter) {
++ nfc_device_iter_exit(iter);
++ kfree(iter);
++ }
+
+ return 0;
+ }
+diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c
+index 076774034bb96..8955f31fa47e9 100644
+--- a/net/openvswitch/actions.c
++++ b/net/openvswitch/actions.c
+@@ -423,12 +423,43 @@ static void set_ipv6_addr(struct sk_buff *skb, u8 l4_proto,
+ memcpy(addr, new_addr, sizeof(__be32[4]));
+ }
+
+-static void set_ipv6_fl(struct ipv6hdr *nh, u32 fl, u32 mask)
++static void set_ipv6_dsfield(struct sk_buff *skb, struct ipv6hdr *nh, u8 ipv6_tclass, u8 mask)
+ {
++ u8 old_ipv6_tclass = ipv6_get_dsfield(nh);
++
++ ipv6_tclass = OVS_MASKED(old_ipv6_tclass, ipv6_tclass, mask);
++
++ if (skb->ip_summed == CHECKSUM_COMPLETE)
++ csum_replace(&skb->csum, (__force __wsum)(old_ipv6_tclass << 12),
++ (__force __wsum)(ipv6_tclass << 12));
++
++ ipv6_change_dsfield(nh, ~mask, ipv6_tclass);
++}
++
++static void set_ipv6_fl(struct sk_buff *skb, struct ipv6hdr *nh, u32 fl, u32 mask)
++{
++ u32 ofl;
++
++ ofl = nh->flow_lbl[0] << 16 | nh->flow_lbl[1] << 8 | nh->flow_lbl[2];
++ fl = OVS_MASKED(ofl, fl, mask);
++
+ /* Bits 21-24 are always unmasked, so this retains their values. */
+- OVS_SET_MASKED(nh->flow_lbl[0], (u8)(fl >> 16), (u8)(mask >> 16));
+- OVS_SET_MASKED(nh->flow_lbl[1], (u8)(fl >> 8), (u8)(mask >> 8));
+- OVS_SET_MASKED(nh->flow_lbl[2], (u8)fl, (u8)mask);
++ nh->flow_lbl[0] = (u8)(fl >> 16);
++ nh->flow_lbl[1] = (u8)(fl >> 8);
++ nh->flow_lbl[2] = (u8)fl;
++
++ if (skb->ip_summed == CHECKSUM_COMPLETE)
++ csum_replace(&skb->csum, (__force __wsum)htonl(ofl), (__force __wsum)htonl(fl));
++}
++
++static void set_ipv6_ttl(struct sk_buff *skb, struct ipv6hdr *nh, u8 new_ttl, u8 mask)
++{
++ new_ttl = OVS_MASKED(nh->hop_limit, new_ttl, mask);
++
++ if (skb->ip_summed == CHECKSUM_COMPLETE)
++ csum_replace(&skb->csum, (__force __wsum)(nh->hop_limit << 8),
++ (__force __wsum)(new_ttl << 8));
++ nh->hop_limit = new_ttl;
+ }
+
+ static void set_ip_ttl(struct sk_buff *skb, struct iphdr *nh, u8 new_ttl,
+@@ -546,18 +577,17 @@ static int set_ipv6(struct sk_buff *skb, struct sw_flow_key *flow_key,
+ }
+ }
+ if (mask->ipv6_tclass) {
+- ipv6_change_dsfield(nh, ~mask->ipv6_tclass, key->ipv6_tclass);
++ set_ipv6_dsfield(skb, nh, key->ipv6_tclass, mask->ipv6_tclass);
+ flow_key->ip.tos = ipv6_get_dsfield(nh);
+ }
+ if (mask->ipv6_label) {
+- set_ipv6_fl(nh, ntohl(key->ipv6_label),
++ set_ipv6_fl(skb, nh, ntohl(key->ipv6_label),
+ ntohl(mask->ipv6_label));
+ flow_key->ipv6.label =
+ *(__be32 *)nh & htonl(IPV6_FLOWINFO_FLOWLABEL);
+ }
+ if (mask->ipv6_hlimit) {
+- OVS_SET_MASKED(nh->hop_limit, key->ipv6_hlimit,
+- mask->ipv6_hlimit);
++ set_ipv6_ttl(skb, nh, key->ipv6_hlimit, mask->ipv6_hlimit);
+ flow_key->ip.ttl = nh->hop_limit;
+ }
+ return 0;
+@@ -1021,7 +1051,7 @@ static int clone(struct datapath *dp, struct sk_buff *skb,
+ int rem = nla_len(attr);
+ bool dont_clone_flow_key;
+
+- /* The first action is always 'OVS_CLONE_ATTR_ARG'. */
++ /* The first action is always 'OVS_CLONE_ATTR_EXEC'. */
+ clone_arg = nla_data(attr);
+ dont_clone_flow_key = nla_get_u32(clone_arg);
+ actions = nla_next(clone_arg, &rem);
+diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c
+index 1b5eae57bc900..815916056e0de 100644
+--- a/net/openvswitch/conntrack.c
++++ b/net/openvswitch/conntrack.c
+@@ -574,7 +574,7 @@ ovs_ct_expect_find(struct net *net, const struct nf_conntrack_zone *zone,
+ struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
+
+ nf_ct_delete(ct, 0, 0);
+- nf_conntrack_put(&ct->ct_general);
++ nf_ct_put(ct);
+ }
+ }
+
+@@ -723,7 +723,7 @@ static bool skb_nfct_cached(struct net *net,
+ if (nf_ct_is_confirmed(ct))
+ nf_ct_delete(ct, 0, 0);
+
+- nf_conntrack_put(&ct->ct_general);
++ nf_ct_put(ct);
+ nf_ct_set(skb, NULL, 0);
+ return false;
+ }
+@@ -732,6 +732,57 @@ static bool skb_nfct_cached(struct net *net,
+ }
+
+ #if IS_ENABLED(CONFIG_NF_NAT)
++static void ovs_nat_update_key(struct sw_flow_key *key,
++ const struct sk_buff *skb,
++ enum nf_nat_manip_type maniptype)
++{
++ if (maniptype == NF_NAT_MANIP_SRC) {
++ __be16 src;
++
++ key->ct_state |= OVS_CS_F_SRC_NAT;
++ if (key->eth.type == htons(ETH_P_IP))
++ key->ipv4.addr.src = ip_hdr(skb)->saddr;
++ else if (key->eth.type == htons(ETH_P_IPV6))
++ memcpy(&key->ipv6.addr.src, &ipv6_hdr(skb)->saddr,
++ sizeof(key->ipv6.addr.src));
++ else
++ return;
++
++ if (key->ip.proto == IPPROTO_UDP)
++ src = udp_hdr(skb)->source;
++ else if (key->ip.proto == IPPROTO_TCP)
++ src = tcp_hdr(skb)->source;
++ else if (key->ip.proto == IPPROTO_SCTP)
++ src = sctp_hdr(skb)->source;
++ else
++ return;
++
++ key->tp.src = src;
++ } else {
++ __be16 dst;
++
++ key->ct_state |= OVS_CS_F_DST_NAT;
++ if (key->eth.type == htons(ETH_P_IP))
++ key->ipv4.addr.dst = ip_hdr(skb)->daddr;
++ else if (key->eth.type == htons(ETH_P_IPV6))
++ memcpy(&key->ipv6.addr.dst, &ipv6_hdr(skb)->daddr,
++ sizeof(key->ipv6.addr.dst));
++ else
++ return;
++
++ if (key->ip.proto == IPPROTO_UDP)
++ dst = udp_hdr(skb)->dest;
++ else if (key->ip.proto == IPPROTO_TCP)
++ dst = tcp_hdr(skb)->dest;
++ else if (key->ip.proto == IPPROTO_SCTP)
++ dst = sctp_hdr(skb)->dest;
++ else
++ return;
++
++ key->tp.dst = dst;
++ }
++}
++
+ /* Modelled after nf_nat_ipv[46]_fn().
+ * range is only used for new, uninitialized NAT state.
+ * Returns either NF_ACCEPT or NF_DROP.
+@@ -739,7 +790,7 @@ static bool skb_nfct_cached(struct net *net,
+ static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct,
+ enum ip_conntrack_info ctinfo,
+ const struct nf_nat_range2 *range,
+- enum nf_nat_manip_type maniptype)
++ enum nf_nat_manip_type maniptype, struct sw_flow_key *key)
+ {
+ int hooknum, nh_off, err = NF_ACCEPT;
+
+@@ -811,58 +862,11 @@ static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct,
+ push:
+ skb_push_rcsum(skb, nh_off);
+
+- return err;
+-}
+-
+-static void ovs_nat_update_key(struct sw_flow_key *key,
+- const struct sk_buff *skb,
+- enum nf_nat_manip_type maniptype)
+-{
+- if (maniptype == NF_NAT_MANIP_SRC) {
+- __be16 src;
+-
+- key->ct_state |= OVS_CS_F_SRC_NAT;
+- if (key->eth.type == htons(ETH_P_IP))
+- key->ipv4.addr.src = ip_hdr(skb)->saddr;
+- else if (key->eth.type == htons(ETH_P_IPV6))
+- memcpy(&key->ipv6.addr.src, &ipv6_hdr(skb)->saddr,
+- sizeof(key->ipv6.addr.src));
+- else
+- return;
+-
+- if (key->ip.proto == IPPROTO_UDP)
+- src = udp_hdr(skb)->source;
+- else if (key->ip.proto == IPPROTO_TCP)
+- src = tcp_hdr(skb)->source;
+- else if (key->ip.proto == IPPROTO_SCTP)
+- src = sctp_hdr(skb)->source;
+- else
+- return;
+-
+- key->tp.src = src;
+- } else {
+- __be16 dst;
+-
+- key->ct_state |= OVS_CS_F_DST_NAT;
+- if (key->eth.type == htons(ETH_P_IP))
+- key->ipv4.addr.dst = ip_hdr(skb)->daddr;
+- else if (key->eth.type == htons(ETH_P_IPV6))
+- memcpy(&key->ipv6.addr.dst, &ipv6_hdr(skb)->daddr,
+- sizeof(key->ipv6.addr.dst));
+- else
+- return;
+-
+- if (key->ip.proto == IPPROTO_UDP)
+- dst = udp_hdr(skb)->dest;
+- else if (key->ip.proto == IPPROTO_TCP)
+- dst = tcp_hdr(skb)->dest;
+- else if (key->ip.proto == IPPROTO_SCTP)
+- dst = sctp_hdr(skb)->dest;
+- else
+- return;
++ /* Update the flow key if NAT successful. */
++ if (err == NF_ACCEPT)
++ ovs_nat_update_key(key, skb, maniptype);
+
+- key->tp.dst = dst;
+- }
++ return err;
+ }
+
+ /* Returns NF_DROP if the packet should be dropped, NF_ACCEPT otherwise. */
+@@ -904,7 +908,7 @@ static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
+ } else {
+ return NF_ACCEPT; /* Connection is not NATed. */
+ }
+- err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, maniptype);
++ err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, maniptype, key);
+
+ if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) {
+ if (ct->status & IPS_SRC_NAT) {
+@@ -914,17 +918,13 @@ static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
+ maniptype = NF_NAT_MANIP_SRC;
+
+ err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range,
+- maniptype);
++ maniptype, key);
+ } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) {
+ err = ovs_ct_nat_execute(skb, ct, ctinfo, NULL,
+- NF_NAT_MANIP_SRC);
++ NF_NAT_MANIP_SRC, key);
+ }
+ }
+
+- /* Mark NAT done if successful and update the flow key. */
+- if (err == NF_ACCEPT)
+- ovs_nat_update_key(key, skb, maniptype);
+-
+ return err;
+ }
+ #else /* !CONFIG_NF_NAT */
+@@ -967,7 +967,8 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
+
+ /* Associate skb with specified zone. */
+ if (tmpl) {
+- nf_conntrack_put(skb_nfct(skb));
++ ct = nf_ct_get(skb, &ctinfo);
++ nf_ct_put(ct);
+ nf_conntrack_get(&tmpl->ct_general);
+ nf_ct_set(skb, tmpl, IP_CT_NEW);
+ }
+@@ -1328,7 +1329,12 @@ int ovs_ct_execute(struct net *net, struct sk_buff *skb,
+
+ int ovs_ct_clear(struct sk_buff *skb, struct sw_flow_key *key)
+ {
+- nf_conntrack_put(skb_nfct(skb));
++ enum ip_conntrack_info ctinfo;
++ struct nf_conn *ct;
++
++ ct = nf_ct_get(skb, &ctinfo);
++
++ nf_ct_put(ct);
+ nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
+ ovs_ct_fill_key(skb, key, false);
+
+@@ -1716,7 +1722,6 @@ int ovs_ct_copy_action(struct net *net, const struct nlattr *attr,
+ goto err_free_ct;
+
+ __set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status);
+- nf_conntrack_get(&ct_info.ct->ct_general);
+ return 0;
+ err_free_ct:
+ __ovs_ct_free_action(&ct_info);
+diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c
+index 9713035b89e3a..02096f2ec6784 100644
+--- a/net/openvswitch/flow.c
++++ b/net/openvswitch/flow.c
+@@ -34,6 +34,7 @@
+ #include <net/mpls.h>
+ #include <net/ndisc.h>
+ #include <net/nsh.h>
++#include <net/netfilter/nf_conntrack_zones.h>
+
+ #include "conntrack.h"
+ #include "datapath.h"
+@@ -858,8 +859,9 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
+ #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
+ struct tc_skb_ext *tc_ext;
+ #endif
+- bool post_ct = false;
++ bool post_ct = false, post_ct_snat = false, post_ct_dnat = false;
+ int res, err;
++ u16 zone = 0;
+
+ /* Extract metadata from packet. */
+ if (tun_info) {
+@@ -898,6 +900,9 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
+ key->recirc_id = tc_ext ? tc_ext->chain : 0;
+ OVS_CB(skb)->mru = tc_ext ? tc_ext->mru : 0;
+ post_ct = tc_ext ? tc_ext->post_ct : false;
++ post_ct_snat = post_ct ? tc_ext->post_ct_snat : false;
++ post_ct_dnat = post_ct ? tc_ext->post_ct_dnat : false;
++ zone = post_ct ? tc_ext->zone : 0;
+ } else {
+ key->recirc_id = 0;
+ }
+@@ -906,8 +911,19 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
+ #endif
+
+ err = key_extract(skb, key);
+- if (!err)
++ if (!err) {
+ ovs_ct_fill_key(skb, key, post_ct); /* Must be after key_extract(). */
++ if (post_ct) {
++ if (!skb_get_nfct(skb)) {
++ key->ct_zone = zone;
++ } else {
++ if (!post_ct_dnat)
++ key->ct_state &= ~OVS_CS_F_DST_NAT;
++ if (!post_ct_snat)
++ key->ct_state &= ~OVS_CS_F_SRC_NAT;
++ }
++ }
++ }
+ return err;
+ }
+
+diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c
+index fd1f809e9bc1b..d77c21ff066c9 100644
+--- a/net/openvswitch/flow_netlink.c
++++ b/net/openvswitch/flow_netlink.c
+@@ -2201,8 +2201,8 @@ static int __ovs_nla_put_key(const struct sw_flow_key *swkey,
+ icmpv6_key->icmpv6_type = ntohs(output->tp.src);
+ icmpv6_key->icmpv6_code = ntohs(output->tp.dst);
+
+- if (icmpv6_key->icmpv6_type == NDISC_NEIGHBOUR_SOLICITATION ||
+- icmpv6_key->icmpv6_type == NDISC_NEIGHBOUR_ADVERTISEMENT) {
++ if (swkey->tp.src == htons(NDISC_NEIGHBOUR_SOLICITATION) ||
++ swkey->tp.src == htons(NDISC_NEIGHBOUR_ADVERTISEMENT)) {
+ struct ovs_key_nd *nd_key;
+
+ nla = nla_reserve(skb, OVS_KEY_ATTR_ND, sizeof(*nd_key));
+@@ -2288,6 +2288,62 @@ static struct sw_flow_actions *nla_alloc_flow_actions(int size)
+ return sfa;
+ }
+
++static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len);
++
++static void ovs_nla_free_check_pkt_len_action(const struct nlattr *action)
++{
++ const struct nlattr *a;
++ int rem;
++
++ nla_for_each_nested(a, action, rem) {
++ switch (nla_type(a)) {
++ case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL:
++ case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER:
++ ovs_nla_free_nested_actions(nla_data(a), nla_len(a));
++ break;
++ }
++ }
++}
++
++static void ovs_nla_free_clone_action(const struct nlattr *action)
++{
++ const struct nlattr *a = nla_data(action);
++ int rem = nla_len(action);
++
++ switch (nla_type(a)) {
++ case OVS_CLONE_ATTR_EXEC:
++ /* The real list of actions follows this attribute. */
++ a = nla_next(a, &rem);
++ ovs_nla_free_nested_actions(a, rem);
++ break;
++ }
++}
++
++static void ovs_nla_free_dec_ttl_action(const struct nlattr *action)
++{
++ const struct nlattr *a = nla_data(action);
++
++ switch (nla_type(a)) {
++ case OVS_DEC_TTL_ATTR_ACTION:
++ ovs_nla_free_nested_actions(nla_data(a), nla_len(a));
++ break;
++ }
++}
++
++static void ovs_nla_free_sample_action(const struct nlattr *action)
++{
++ const struct nlattr *a = nla_data(action);
++ int rem = nla_len(action);
++
++ switch (nla_type(a)) {
++ case OVS_SAMPLE_ATTR_ARG:
++ /* The real list of actions follows this attribute. */
++ a = nla_next(a, &rem);
++ ovs_nla_free_nested_actions(a, rem);
++ break;
++ }
++}
++
+ static void ovs_nla_free_set_action(const struct nlattr *a)
+ {
+ const struct nlattr *ovs_key = nla_data(a);
+@@ -2301,25 +2357,54 @@ static void ovs_nla_free_set_action(const struct nlattr *a)
+ }
+ }
+
+-void ovs_nla_free_flow_actions(struct sw_flow_actions *sf_acts)
++static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len)
+ {
+ const struct nlattr *a;
+ int rem;
+
+- if (!sf_acts)
++ /* Whenever new actions are added, the need to update this
++ * function should be considered.
++ */
++ BUILD_BUG_ON(OVS_ACTION_ATTR_MAX != 23);
++
++ if (!actions)
+ return;
+
+- nla_for_each_attr(a, sf_acts->actions, sf_acts->actions_len, rem) {
++ nla_for_each_attr(a, actions, len, rem) {
+ switch (nla_type(a)) {
+- case OVS_ACTION_ATTR_SET:
+- ovs_nla_free_set_action(a);
++ case OVS_ACTION_ATTR_CHECK_PKT_LEN:
++ ovs_nla_free_check_pkt_len_action(a);
++ break;
++
++ case OVS_ACTION_ATTR_CLONE:
++ ovs_nla_free_clone_action(a);
+ break;
++
+ case OVS_ACTION_ATTR_CT:
+ ovs_ct_free_action(a);
+ break;
++
++ case OVS_ACTION_ATTR_DEC_TTL:
++ ovs_nla_free_dec_ttl_action(a);
++ break;
++
++ case OVS_ACTION_ATTR_SAMPLE:
++ ovs_nla_free_sample_action(a);
++ break;
++
++ case OVS_ACTION_ATTR_SET:
++ ovs_nla_free_set_action(a);
++ break;
+ }
+ }
++}
++
++void ovs_nla_free_flow_actions(struct sw_flow_actions *sf_acts)
++{
++ if (!sf_acts)
++ return;
+
++ ovs_nla_free_nested_actions(sf_acts->actions, sf_acts->actions_len);
+ kfree(sf_acts);
+ }
+
+@@ -2351,7 +2436,7 @@ static struct nlattr *reserve_sfa_size(struct sw_flow_actions **sfa,
+ new_acts_size = max(next_offset + req_size, ksize(*sfa) * 2);
+
+ if (new_acts_size > MAX_ACTIONS_BUFSIZE) {
+- if ((MAX_ACTIONS_BUFSIZE - next_offset) < req_size) {
++ if ((next_offset + req_size) > MAX_ACTIONS_BUFSIZE) {
+ OVS_NLERR(log, "Flow action size exceeds max %u",
+ MAX_ACTIONS_BUFSIZE);
+ return ERR_PTR(-EMSGSIZE);
+@@ -3429,7 +3514,9 @@ static int clone_action_to_attr(const struct nlattr *attr,
+ if (!start)
+ return -EMSGSIZE;
+
+- err = ovs_nla_put_actions(nla_data(attr), rem, skb);
++ /* Skipping the OVS_CLONE_ATTR_EXEC that is always the first attribute. */
++ attr = nla_next(nla_data(attr), &rem);
++ err = ovs_nla_put_actions(attr, rem, skb);
+
+ if (err)
+ nla_nest_cancel(skb, start);
+diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
+index 2a2bc64f75cfd..88c3b5cf8d94c 100644
+--- a/net/packet/af_packet.c
++++ b/net/packet/af_packet.c
+@@ -1738,6 +1738,7 @@ static int fanout_add(struct sock *sk, struct fanout_args *args)
+ match->prot_hook.dev = po->prot_hook.dev;
+ match->prot_hook.func = packet_rcv_fanout;
+ match->prot_hook.af_packet_priv = match;
++ match->prot_hook.af_packet_net = read_pnet(&match->net);
+ match->prot_hook.id_match = match_fanout_group;
+ match->max_num_members = args->max_num_members;
+ list_add(&match->list, &fanout_list);
+@@ -1752,7 +1753,10 @@ static int fanout_add(struct sock *sk, struct fanout_args *args)
+ err = -ENOSPC;
+ if (refcount_read(&match->sk_ref) < match->max_num_members) {
+ __dev_remove_pack(&po->prot_hook);
+- po->fanout = match;
++
++ /* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
++ WRITE_ONCE(po->fanout, match);
++
+ po->rollover = rollover;
+ rollover = NULL;
+ refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
+@@ -2277,8 +2281,11 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
+ copy_skb = skb_get(skb);
+ skb_head = skb->data;
+ }
+- if (copy_skb)
++ if (copy_skb) {
++ memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
++ sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
+ skb_set_owner_r(copy_skb, sk);
++ }
+ }
+ snaplen = po->rx_ring.frame_size - macoff;
+ if ((int)snaplen < 0) {
+@@ -2813,8 +2820,9 @@ tpacket_error:
+
+ status = TP_STATUS_SEND_REQUEST;
+ err = po->xmit(skb);
+- if (unlikely(err > 0)) {
+- err = net_xmit_errno(err);
++ if (unlikely(err != 0)) {
++ if (err > 0)
++ err = net_xmit_errno(err);
+ if (err && __packet_get_status(po, ph) ==
+ TP_STATUS_AVAILABLE) {
+ /* skb was destructed already */
+@@ -3015,8 +3023,12 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
+ skb->no_fcs = 1;
+
+ err = po->xmit(skb);
+- if (err > 0 && (err = net_xmit_errno(err)) != 0)
+- goto out_unlock;
++ if (unlikely(err != 0)) {
++ if (err > 0)
++ err = net_xmit_errno(err);
++ if (err)
++ goto out_unlock;
++ }
+
+ dev_put(dev);
+
+@@ -3323,6 +3335,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol,
+ po->prot_hook.func = packet_rcv_spkt;
+
+ po->prot_hook.af_packet_priv = sk;
++ po->prot_hook.af_packet_net = sock_net(sk);
+
+ if (proto) {
+ po->prot_hook.type = proto;
+@@ -3429,6 +3442,8 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+ sock_recv_ts_and_drops(msg, sk, skb);
+
+ if (msg->msg_name) {
++ const size_t max_len = min(sizeof(skb->cb),
++ sizeof(struct sockaddr_storage));
+ int copy_len;
+
+ /* If the address length field is there to be filled
+@@ -3451,6 +3466,10 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+ msg->msg_namelen = sizeof(struct sockaddr_ll);
+ }
+ }
++ if (WARN_ON_ONCE(copy_len > max_len)) {
++ copy_len = max_len;
++ msg->msg_namelen = copy_len;
++ }
+ memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
+ }
+
+@@ -3904,7 +3923,8 @@ packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
+ }
+ case PACKET_FANOUT_DATA:
+ {
+- if (!po->fanout)
++ /* Paired with the WRITE_ONCE() in fanout_add() */
++ if (!READ_ONCE(po->fanout))
+ return -EINVAL;
+
+ return fanout_set_data(po, optval, optlen);
+@@ -4457,9 +4477,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
+ }
+
+ out_free_pg_vec:
+- bitmap_free(rx_owner_map);
+- if (pg_vec)
++ if (pg_vec) {
++ bitmap_free(rx_owner_map);
+ free_pg_vec(pg_vec, order, req->tp_block_nr);
++ }
+ out:
+ return err;
+ }
+diff --git a/net/phonet/pep.c b/net/phonet/pep.c
+index a1525916885ae..65d463ad87707 100644
+--- a/net/phonet/pep.c
++++ b/net/phonet/pep.c
+@@ -868,6 +868,7 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp,
+
+ err = pep_accept_conn(newsk, skb);
+ if (err) {
++ __sock_put(sk);
+ sock_put(newsk);
+ newsk = NULL;
+ goto drop;
+@@ -946,6 +947,8 @@ static int pep_ioctl(struct sock *sk, int cmd, unsigned long arg)
+ ret = -EBUSY;
+ else if (sk->sk_state == TCP_ESTABLISHED)
+ ret = -EISCONN;
++ else if (!pn->pn_sk.sobject)
++ ret = -EADDRNOTAVAIL;
+ else
+ ret = pep_sock_enable(sk, NULL, 0);
+ release_sock(sk);
+diff --git a/net/rds/connection.c b/net/rds/connection.c
+index a3bc4b54d4910..b4cc699c5fad3 100644
+--- a/net/rds/connection.c
++++ b/net/rds/connection.c
+@@ -253,6 +253,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
+ * should end up here, but if it
+ * does, reset/destroy the connection.
+ */
++ kfree(conn->c_path);
+ kmem_cache_free(rds_conn_slab, conn);
+ conn = ERR_PTR(-EOPNOTSUPP);
+ goto out;
+diff --git a/net/rds/tcp.c b/net/rds/tcp.c
+index abf19c0e3ba0b..5327d130c4b56 100644
+--- a/net/rds/tcp.c
++++ b/net/rds/tcp.c
+@@ -500,7 +500,7 @@ void rds_tcp_tune(struct socket *sock)
+ sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
+ }
+ if (rtn->rcvbuf_size > 0) {
+- sk->sk_sndbuf = rtn->rcvbuf_size;
++ sk->sk_rcvbuf = rtn->rcvbuf_size;
+ sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
+ }
+ release_sock(sk);
+diff --git a/net/rfkill/core.c b/net/rfkill/core.c
+index ac15a944573f7..068c7bcd30c94 100644
+--- a/net/rfkill/core.c
++++ b/net/rfkill/core.c
+@@ -78,6 +78,7 @@ struct rfkill_data {
+ struct mutex mtx;
+ wait_queue_head_t read_wait;
+ bool input_handler;
++ u8 max_size;
+ };
+
+
+@@ -1141,6 +1142,8 @@ static int rfkill_fop_open(struct inode *inode, struct file *file)
+ if (!data)
+ return -ENOMEM;
+
++ data->max_size = RFKILL_EVENT_SIZE_V1;
++
+ INIT_LIST_HEAD(&data->events);
+ mutex_init(&data->mtx);
+ init_waitqueue_head(&data->read_wait);
+@@ -1223,6 +1226,7 @@ static ssize_t rfkill_fop_read(struct file *file, char __user *buf,
+ list);
+
+ sz = min_t(unsigned long, sizeof(ev->ev), count);
++ sz = min_t(unsigned long, sz, data->max_size);
+ ret = sz;
+ if (copy_to_user(buf, &ev->ev, sz))
+ ret = -EFAULT;
+@@ -1237,6 +1241,7 @@ static ssize_t rfkill_fop_read(struct file *file, char __user *buf,
+ static ssize_t rfkill_fop_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *pos)
+ {
++ struct rfkill_data *data = file->private_data;
+ struct rfkill *rfkill;
+ struct rfkill_event_ext ev;
+ int ret;
+@@ -1251,6 +1256,7 @@ static ssize_t rfkill_fop_write(struct file *file, const char __user *buf,
+ * our API version even in a write() call, if it cares.
+ */
+ count = min(count, sizeof(ev));
++ count = min_t(size_t, count, data->max_size);
+ if (copy_from_user(&ev, buf, count))
+ return -EFAULT;
+
+@@ -1310,31 +1316,47 @@ static int rfkill_fop_release(struct inode *inode, struct file *file)
+ return 0;
+ }
+
+-#ifdef CONFIG_RFKILL_INPUT
+ static long rfkill_fop_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+ {
+ struct rfkill_data *data = file->private_data;
++ int ret = -ENOSYS;
++ u32 size;
+
+ if (_IOC_TYPE(cmd) != RFKILL_IOC_MAGIC)
+ return -ENOSYS;
+
+- if (_IOC_NR(cmd) != RFKILL_IOC_NOINPUT)
+- return -ENOSYS;
+-
+ mutex_lock(&data->mtx);
+-
+- if (!data->input_handler) {
+- if (atomic_inc_return(&rfkill_input_disabled) == 1)
+- printk(KERN_DEBUG "rfkill: input handler disabled\n");
+- data->input_handler = true;
++ switch (_IOC_NR(cmd)) {
++#ifdef CONFIG_RFKILL_INPUT
++ case RFKILL_IOC_NOINPUT:
++ if (!data->input_handler) {
++ if (atomic_inc_return(&rfkill_input_disabled) == 1)
++ printk(KERN_DEBUG "rfkill: input handler disabled\n");
++ data->input_handler = true;
++ }
++ ret = 0;
++ break;
++#endif
++ case RFKILL_IOC_MAX_SIZE:
++ if (get_user(size, (__u32 __user *)arg)) {
++ ret = -EFAULT;
++ break;
++ }
++ if (size < RFKILL_EVENT_SIZE_V1 || size > U8_MAX) {
++ ret = -EINVAL;
++ break;
++ }
++ data->max_size = size;
++ ret = 0;
++ break;
++ default:
++ break;
+ }
+-
+ mutex_unlock(&data->mtx);
+
+- return 0;
++ return ret;
+ }
+-#endif
+
+ static const struct file_operations rfkill_fops = {
+ .owner = THIS_MODULE,
+@@ -1343,10 +1365,8 @@ static const struct file_operations rfkill_fops = {
+ .write = rfkill_fop_write,
+ .poll = rfkill_fop_poll,
+ .release = rfkill_fop_release,
+-#ifdef CONFIG_RFKILL_INPUT
+ .unlocked_ioctl = rfkill_fop_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+-#endif
+ .llseek = no_llseek,
+ };
+
+diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
+index 7bd6f8a66a3ef..969e532f77a90 100644
+--- a/net/rxrpc/ar-internal.h
++++ b/net/rxrpc/ar-internal.h
+@@ -777,14 +777,12 @@ void rxrpc_propose_ACK(struct rxrpc_call *, u8, u32, bool, bool,
+ enum rxrpc_propose_ack_trace);
+ void rxrpc_process_call(struct work_struct *);
+
+-static inline void rxrpc_reduce_call_timer(struct rxrpc_call *call,
+- unsigned long expire_at,
+- unsigned long now,
+- enum rxrpc_timer_trace why)
+-{
+- trace_rxrpc_timer(call, why, now);
+- timer_reduce(&call->timer, expire_at);
+-}
++void rxrpc_reduce_call_timer(struct rxrpc_call *call,
++ unsigned long expire_at,
++ unsigned long now,
++ enum rxrpc_timer_trace why);
++
++void rxrpc_delete_call_timer(struct rxrpc_call *call);
+
+ /*
+ * call_object.c
+@@ -808,6 +806,7 @@ void rxrpc_release_calls_on_socket(struct rxrpc_sock *);
+ bool __rxrpc_queue_call(struct rxrpc_call *);
+ bool rxrpc_queue_call(struct rxrpc_call *);
+ void rxrpc_see_call(struct rxrpc_call *);
++bool rxrpc_try_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op);
+ void rxrpc_get_call(struct rxrpc_call *, enum rxrpc_call_trace);
+ void rxrpc_put_call(struct rxrpc_call *, enum rxrpc_call_trace);
+ void rxrpc_cleanup_call(struct rxrpc_call *);
+diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c
+index 6be2672a65eab..22e05de5d1ca9 100644
+--- a/net/rxrpc/call_event.c
++++ b/net/rxrpc/call_event.c
+@@ -157,7 +157,7 @@ static void rxrpc_congestion_timeout(struct rxrpc_call *call)
+ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
+ {
+ struct sk_buff *skb;
+- unsigned long resend_at, rto_j;
++ unsigned long resend_at;
+ rxrpc_seq_t cursor, seq, top;
+ ktime_t now, max_age, oldest, ack_ts;
+ int ix;
+@@ -165,10 +165,8 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
+
+ _enter("{%d,%d}", call->tx_hard_ack, call->tx_top);
+
+- rto_j = call->peer->rto_j;
+-
+ now = ktime_get_real();
+- max_age = ktime_sub(now, jiffies_to_usecs(rto_j));
++ max_age = ktime_sub(now, jiffies_to_usecs(call->peer->rto_j));
+
+ spin_lock_bh(&call->lock);
+
+@@ -213,7 +211,7 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j)
+ }
+
+ resend_at = nsecs_to_jiffies(ktime_to_ns(ktime_sub(now, oldest)));
+- resend_at += jiffies + rto_j;
++ resend_at += jiffies + rxrpc_get_rto_backoff(call->peer, retrans);
+ WRITE_ONCE(call->resend_at, resend_at);
+
+ if (unacked)
+@@ -312,7 +310,7 @@ recheck_state:
+ }
+
+ if (call->state == RXRPC_CALL_COMPLETE) {
+- del_timer_sync(&call->timer);
++ rxrpc_delete_call_timer(call);
+ goto out_put;
+ }
+
+diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c
+index 4eb91d958a48d..043508fd8d8a5 100644
+--- a/net/rxrpc/call_object.c
++++ b/net/rxrpc/call_object.c
+@@ -53,10 +53,30 @@ static void rxrpc_call_timer_expired(struct timer_list *t)
+
+ if (call->state < RXRPC_CALL_COMPLETE) {
+ trace_rxrpc_timer(call, rxrpc_timer_expired, jiffies);
+- rxrpc_queue_call(call);
++ __rxrpc_queue_call(call);
++ } else {
++ rxrpc_put_call(call, rxrpc_call_put);
++ }
++}
++
++void rxrpc_reduce_call_timer(struct rxrpc_call *call,
++ unsigned long expire_at,
++ unsigned long now,
++ enum rxrpc_timer_trace why)
++{
++ if (rxrpc_try_get_call(call, rxrpc_call_got_timer)) {
++ trace_rxrpc_timer(call, why, now);
++ if (timer_reduce(&call->timer, expire_at))
++ rxrpc_put_call(call, rxrpc_call_put_notimer);
+ }
+ }
+
++void rxrpc_delete_call_timer(struct rxrpc_call *call)
++{
++ if (del_timer_sync(&call->timer))
++ rxrpc_put_call(call, rxrpc_call_put_timer);
++}
++
+ static struct lock_class_key rxrpc_call_user_mutex_lock_class_key;
+
+ /*
+@@ -463,6 +483,17 @@ void rxrpc_see_call(struct rxrpc_call *call)
+ }
+ }
+
++bool rxrpc_try_get_call(struct rxrpc_call *call, enum rxrpc_call_trace op)
++{
++ const void *here = __builtin_return_address(0);
++ int n = atomic_fetch_add_unless(&call->usage, 1, 0);
++
++ if (n == 0)
++ return false;
++ trace_rxrpc_call(call->debug_id, op, n, here, NULL);
++ return true;
++}
++
+ /*
+ * Note the addition of a ref on a call.
+ */
+@@ -510,8 +541,7 @@ void rxrpc_release_call(struct rxrpc_sock *rx, struct rxrpc_call *call)
+ spin_unlock_bh(&call->lock);
+
+ rxrpc_put_call_slot(call);
+-
+- del_timer_sync(&call->timer);
++ rxrpc_delete_call_timer(call);
+
+ /* Make sure we don't get any more notifications */
+ write_lock_bh(&rx->recvmsg_lock);
+@@ -618,6 +648,8 @@ static void rxrpc_destroy_call(struct work_struct *work)
+ struct rxrpc_call *call = container_of(work, struct rxrpc_call, processor);
+ struct rxrpc_net *rxnet = call->rxnet;
+
++ rxrpc_delete_call_timer(call);
++
+ rxrpc_put_connection(call->conn);
+ rxrpc_put_peer(call->peer);
+ kfree(call->rxtx_buffer);
+@@ -652,8 +684,6 @@ void rxrpc_cleanup_call(struct rxrpc_call *call)
+
+ memset(&call->sock_node, 0xcd, sizeof(call->sock_node));
+
+- del_timer_sync(&call->timer);
+-
+ ASSERTCMP(call->state, ==, RXRPC_CALL_COMPLETE);
+ ASSERT(test_bit(RXRPC_CALL_RELEASED, &call->flags));
+
+diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c
+index dbea0bfee48e9..8120138dac018 100644
+--- a/net/rxrpc/conn_client.c
++++ b/net/rxrpc/conn_client.c
+@@ -135,16 +135,20 @@ struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
+ return bundle;
+ }
+
++static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
++{
++ rxrpc_put_peer(bundle->params.peer);
++ kfree(bundle);
++}
++
+ void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
+ {
+ unsigned int d = bundle->debug_id;
+ unsigned int u = atomic_dec_return(&bundle->usage);
+
+ _debug("PUT B=%x %u", d, u);
+- if (u == 0) {
+- rxrpc_put_peer(bundle->params.peer);
+- kfree(bundle);
+- }
++ if (u == 0)
++ rxrpc_free_bundle(bundle);
+ }
+
+ /*
+@@ -328,7 +332,7 @@ static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *c
+ return candidate;
+
+ found_bundle_free:
+- kfree(candidate);
++ rxrpc_free_bundle(candidate);
+ found_bundle:
+ rxrpc_get_bundle(bundle);
+ spin_unlock(&local->client_bundles_lock);
+diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c
+index a4111408ffd0c..6a1611b0e3037 100644
+--- a/net/rxrpc/local_object.c
++++ b/net/rxrpc/local_object.c
+@@ -117,6 +117,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net)
+ local, srx->transport_type, srx->transport.family);
+
+ udp_conf.family = srx->transport.family;
++ udp_conf.use_udp_checksums = true;
+ if (udp_conf.family == AF_INET) {
+ udp_conf.local_ip = srx->transport.sin.sin_addr;
+ udp_conf.local_udp_port = srx->transport.sin.sin_port;
+@@ -124,6 +125,8 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net)
+ } else {
+ udp_conf.local_ip6 = srx->transport.sin6.sin6_addr;
+ udp_conf.local_udp_port = srx->transport.sin6.sin6_port;
++ udp_conf.use_udp6_tx_checksums = true;
++ udp_conf.use_udp6_rx_checksums = true;
+ #endif
+ }
+ ret = udp_sock_create(net, &udp_conf, &local->socket);
+diff --git a/net/rxrpc/net_ns.c b/net/rxrpc/net_ns.c
+index 25bbc4cc8b135..cc7e30733feb0 100644
+--- a/net/rxrpc/net_ns.c
++++ b/net/rxrpc/net_ns.c
+@@ -115,6 +115,8 @@ static __net_exit void rxrpc_exit_net(struct net *net)
+ rxnet->live = false;
+ del_timer_sync(&rxnet->peer_keepalive_timer);
+ cancel_work_sync(&rxnet->peer_keepalive_work);
++ /* Remove the timer again as the worker may have restarted it. */
++ del_timer_sync(&rxnet->peer_keepalive_timer);
+ rxrpc_destroy_all_calls(rxnet);
+ rxrpc_destroy_all_connections(rxnet);
+ rxrpc_destroy_all_peers(rxnet);
+diff --git a/net/rxrpc/output.c b/net/rxrpc/output.c
+index 10f2bf2e9068a..a45c83f22236e 100644
+--- a/net/rxrpc/output.c
++++ b/net/rxrpc/output.c
+@@ -468,7 +468,7 @@ done:
+ if (call->peer->rtt_count > 1) {
+ unsigned long nowj = jiffies, ack_lost_at;
+
+- ack_lost_at = rxrpc_get_rto_backoff(call->peer, retrans);
++ ack_lost_at = rxrpc_get_rto_backoff(call->peer, false);
+ ack_lost_at += nowj;
+ WRITE_ONCE(call->ack_lost_at, ack_lost_at);
+ rxrpc_reduce_call_timer(call, ack_lost_at, nowj,
+diff --git a/net/rxrpc/peer_object.c b/net/rxrpc/peer_object.c
+index 68396d0520525..0298fe2ad6d32 100644
+--- a/net/rxrpc/peer_object.c
++++ b/net/rxrpc/peer_object.c
+@@ -299,6 +299,12 @@ static struct rxrpc_peer *rxrpc_create_peer(struct rxrpc_sock *rx,
+ return peer;
+ }
+
++static void rxrpc_free_peer(struct rxrpc_peer *peer)
++{
++ rxrpc_put_local(peer->local);
++ kfree_rcu(peer, rcu);
++}
++
+ /*
+ * Set up a new incoming peer. There shouldn't be any other matching peers
+ * since we've already done a search in the list from the non-reentrant context
+@@ -365,7 +371,7 @@ struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_sock *rx,
+ spin_unlock_bh(&rxnet->peer_hash_lock);
+
+ if (peer)
+- kfree(candidate);
++ rxrpc_free_peer(candidate);
+ else
+ peer = candidate;
+ }
+@@ -420,8 +426,7 @@ static void __rxrpc_put_peer(struct rxrpc_peer *peer)
+ list_del_init(&peer->keepalive_link);
+ spin_unlock_bh(&rxnet->peer_hash_lock);
+
+- rxrpc_put_local(peer->local);
+- kfree_rcu(peer, rcu);
++ rxrpc_free_peer(peer);
+ }
+
+ /*
+@@ -457,8 +462,7 @@ void rxrpc_put_peer_locked(struct rxrpc_peer *peer)
+ if (n == 0) {
+ hash_del_rcu(&peer->hash_link);
+ list_del_init(&peer->keepalive_link);
+- rxrpc_put_local(peer->local);
+- kfree_rcu(peer, rcu);
++ rxrpc_free_peer(peer);
+ }
+ }
+
+diff --git a/net/rxrpc/rtt.c b/net/rxrpc/rtt.c
+index 4e565eeab4260..be61d6f5be8d1 100644
+--- a/net/rxrpc/rtt.c
++++ b/net/rxrpc/rtt.c
+@@ -22,7 +22,7 @@ static u32 rxrpc_rto_min_us(struct rxrpc_peer *peer)
+
+ static u32 __rxrpc_set_rto(const struct rxrpc_peer *peer)
+ {
+- return _usecs_to_jiffies((peer->srtt_us >> 3) + peer->rttvar_us);
++ return usecs_to_jiffies((peer->srtt_us >> 3) + peer->rttvar_us);
+ }
+
+ static u32 rxrpc_bound_rto(u32 rto)
+diff --git a/net/rxrpc/server_key.c b/net/rxrpc/server_key.c
+index ead3471307ee5..ee269e0e6ee87 100644
+--- a/net/rxrpc/server_key.c
++++ b/net/rxrpc/server_key.c
+@@ -84,6 +84,9 @@ static int rxrpc_preparse_s(struct key_preparsed_payload *prep)
+
+ prep->payload.data[1] = (struct rxrpc_security *)sec;
+
++ if (!sec->preparse_server_key)
++ return -EINVAL;
++
+ return sec->preparse_server_key(prep);
+ }
+
+@@ -91,7 +94,7 @@ static void rxrpc_free_preparse_s(struct key_preparsed_payload *prep)
+ {
+ const struct rxrpc_security *sec = prep->payload.data[1];
+
+- if (sec)
++ if (sec && sec->free_preparse_server_key)
+ sec->free_preparse_server_key(prep);
+ }
+
+@@ -99,7 +102,7 @@ static void rxrpc_destroy_s(struct key *key)
+ {
+ const struct rxrpc_security *sec = key->payload.data[1];
+
+- if (sec)
++ if (sec && sec->destroy_server_key)
+ sec->destroy_server_key(key);
+ }
+
+diff --git a/net/sched/act_api.c b/net/sched/act_api.c
+index 7dd3a2dc5fa40..7d53272727bfa 100644
+--- a/net/sched/act_api.c
++++ b/net/sched/act_api.c
+@@ -728,15 +728,24 @@ int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions,
+ restart_act_graph:
+ for (i = 0; i < nr_actions; i++) {
+ const struct tc_action *a = actions[i];
++ int repeat_ttl;
+
+ if (jmp_prgcnt > 0) {
+ jmp_prgcnt -= 1;
+ continue;
+ }
++
++ repeat_ttl = 32;
+ repeat:
+ ret = a->ops->act(skb, a, res);
+- if (ret == TC_ACT_REPEAT)
+- goto repeat; /* we need a ttl - JHS */
++
++ if (unlikely(ret == TC_ACT_REPEAT)) {
++ if (--repeat_ttl != 0)
++ goto repeat;
++ /* suspicious opcode, stop pipeline */
++ net_warn_ratelimited("TC_ACT_REPEAT abuse ?\n");
++ return TC_ACT_OK;
++ }
+
+ if (TC_ACT_EXT_CMP(ret, TC_ACT_JUMP)) {
+ jmp_prgcnt = ret & TCA_ACT_MAX_PRIO_MASK;
+diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c
+index 90866ae45573a..f4fd584fba08a 100644
+--- a/net/sched/act_ct.c
++++ b/net/sched/act_ct.c
+@@ -516,11 +516,6 @@ static bool tcf_ct_flow_table_lookup(struct tcf_ct_params *p,
+ struct nf_conn *ct;
+ u8 dir;
+
+- /* Previously seen or loopback */
+- ct = nf_ct_get(skb, &ctinfo);
+- if ((ct && !nf_ct_is_template(ct)) || ctinfo == IP_CT_UNTRACKED)
+- return false;
+-
+ switch (family) {
+ case NFPROTO_IPV4:
+ if (!tcf_ct_flow_table_fill_tuple_ipv4(skb, &tuple, &tcph))
+@@ -588,22 +583,25 @@ static bool tcf_ct_skb_nfct_cached(struct net *net, struct sk_buff *skb,
+ if (!ct)
+ return false;
+ if (!net_eq(net, read_pnet(&ct->ct_net)))
+- return false;
++ goto drop_ct;
+ if (nf_ct_zone(ct)->id != zone_id)
+- return false;
++ goto drop_ct;
+
+ /* Force conntrack entry direction. */
+ if (force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
+ if (nf_ct_is_confirmed(ct))
+ nf_ct_kill(ct);
+
+- nf_conntrack_put(&ct->ct_general);
+- nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
+-
+- return false;
++ goto drop_ct;
+ }
+
+ return true;
++
++drop_ct:
++ nf_ct_put(ct);
++ nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
++
++ return false;
+ }
+
+ /* Trim the skb to the length specified by the IP/IPv6 header,
+@@ -690,10 +688,10 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
+ u8 family, u16 zone, bool *defrag)
+ {
+ enum ip_conntrack_info ctinfo;
+- struct qdisc_skb_cb cb;
+ struct nf_conn *ct;
+ int err = 0;
+ bool frag;
++ u16 mru;
+
+ /* Previously seen (loopback)? Ignore. */
+ ct = nf_ct_get(skb, &ctinfo);
+@@ -708,7 +706,7 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
+ return err;
+
+ skb_get(skb);
+- cb = *qdisc_skb_cb(skb);
++ mru = tc_skb_cb(skb)->mru;
+
+ if (family == NFPROTO_IPV4) {
+ enum ip_defrag_users user = IP_DEFRAG_CONNTRACK_IN + zone;
+@@ -722,7 +720,7 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
+
+ if (!err) {
+ *defrag = true;
+- cb.mru = IPCB(skb)->frag_max_size;
++ mru = IPCB(skb)->frag_max_size;
+ }
+ } else { /* NFPROTO_IPV6 */
+ #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
+@@ -735,7 +733,7 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
+
+ if (!err) {
+ *defrag = true;
+- cb.mru = IP6CB(skb)->frag_max_size;
++ mru = IP6CB(skb)->frag_max_size;
+ }
+ #else
+ err = -EOPNOTSUPP;
+@@ -744,7 +742,7 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
+ }
+
+ if (err != -EINPROGRESS)
+- *qdisc_skb_cb(skb) = cb;
++ tc_skb_cb(skb)->mru = mru;
+ skb_clear_hash(skb);
+ skb->ignore_df = 1;
+ return err;
+@@ -762,7 +760,7 @@ static void tcf_ct_params_free(struct rcu_head *head)
+ tcf_ct_flow_table_put(params);
+
+ if (params->tmpl)
+- nf_conntrack_put(&params->tmpl->ct_general);
++ nf_ct_put(params->tmpl);
+ kfree(params);
+ }
+
+@@ -839,6 +837,12 @@ static int ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct,
+ }
+
+ err = nf_nat_packet(ct, ctinfo, hooknum, skb);
++ if (err == NF_ACCEPT) {
++ if (maniptype == NF_NAT_MANIP_SRC)
++ tc_skb_cb(skb)->post_ct_snat = 1;
++ if (maniptype == NF_NAT_MANIP_DST)
++ tc_skb_cb(skb)->post_ct_dnat = 1;
++ }
+ out:
+ return err;
+ }
+@@ -963,10 +967,10 @@ static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
+ tcf_action_update_bstats(&c->common, skb);
+
+ if (clear) {
+- qdisc_skb_cb(skb)->post_ct = false;
++ tc_skb_cb(skb)->post_ct = false;
+ ct = nf_ct_get(skb, &ctinfo);
+ if (ct) {
+- nf_conntrack_put(&ct->ct_general);
++ nf_ct_put(ct);
+ nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
+ }
+
+@@ -1048,7 +1052,8 @@ do_nat:
+ out_push:
+ skb_push_rcsum(skb, nh_ofs);
+
+- qdisc_skb_cb(skb)->post_ct = true;
++ tc_skb_cb(skb)->post_ct = true;
++ tc_skb_cb(skb)->zone = p->zone;
+ out_clear:
+ if (defrag)
+ qdisc_skb_cb(skb)->pkt_len = skb->len;
+@@ -1227,7 +1232,6 @@ static int tcf_ct_fill_params(struct net *net,
+ return -ENOMEM;
+ }
+ __set_bit(IPS_CONFIRMED_BIT, &tmpl->status);
+- nf_conntrack_get(&tmpl->ct_general);
+ p->tmpl = tmpl;
+
+ return 0;
+diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
+index d64b0eeccbe4d..efc963ab995a3 100644
+--- a/net/sched/act_mirred.c
++++ b/net/sched/act_mirred.c
+@@ -19,6 +19,7 @@
+ #include <linux/if_arp.h>
+ #include <net/net_namespace.h>
+ #include <net/netlink.h>
++#include <net/dst.h>
+ #include <net/pkt_sched.h>
+ #include <net/pkt_cls.h>
+ #include <linux/tc_act/tc_mirred.h>
+@@ -228,6 +229,7 @@ static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a,
+ bool want_ingress;
+ bool is_redirect;
+ bool expects_nh;
++ bool at_ingress;
+ int m_eaction;
+ int mac_len;
+ bool at_nh;
+@@ -263,7 +265,8 @@ static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a,
+ * ingress - that covers the TC S/W datapath.
+ */
+ is_redirect = tcf_mirred_is_act_redirect(m_eaction);
+- use_reinsert = skb_at_tc_ingress(skb) && is_redirect &&
++ at_ingress = skb_at_tc_ingress(skb);
++ use_reinsert = at_ingress && is_redirect &&
+ tcf_mirred_can_reinsert(retval);
+ if (!use_reinsert) {
+ skb2 = skb_clone(skb, GFP_ATOMIC);
+@@ -271,10 +274,12 @@ static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a,
+ goto out;
+ }
+
++ want_ingress = tcf_mirred_act_wants_ingress(m_eaction);
++
+ /* All mirred/redirected skbs should clear previous ct info */
+ nf_reset_ct(skb2);
+-
+- want_ingress = tcf_mirred_act_wants_ingress(m_eaction);
++ if (want_ingress && !at_ingress) /* drop dst for egress -> ingress */
++ skb_dst_drop(skb2);
+
+ expects_nh = want_ingress || !m_mac_header_xmit;
+ at_nh = skb->data == skb_network_header(skb);
+diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c
+index c6c862c459cc3..1262a84b725fc 100644
+--- a/net/sched/act_pedit.c
++++ b/net/sched/act_pedit.c
+@@ -149,7 +149,7 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla,
+ struct nlattr *pattr;
+ struct tcf_pedit *p;
+ int ret = 0, err;
+- int ksize;
++ int i, ksize;
+ u32 index;
+
+ if (!nla) {
+@@ -228,6 +228,22 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla,
+ p->tcfp_nkeys = parm->nkeys;
+ }
+ memcpy(p->tcfp_keys, parm->keys, ksize);
++ p->tcfp_off_max_hint = 0;
++ for (i = 0; i < p->tcfp_nkeys; ++i) {
++ u32 cur = p->tcfp_keys[i].off;
++
++ /* sanitize the shift value for any later use */
++ p->tcfp_keys[i].shift = min_t(size_t, BITS_PER_TYPE(int) - 1,
++ p->tcfp_keys[i].shift);
++
++ /* The AT option can read a single byte, we can bound the actual
++ * value with uchar max.
++ */
++ cur += (0xff & p->tcfp_keys[i].offmask) >> p->tcfp_keys[i].shift;
++
++ /* Each key touches 4 bytes starting from the computed offset */
++ p->tcfp_off_max_hint = max(p->tcfp_off_max_hint, cur + 4);
++ }
+
+ p->tcfp_flags = parm->flags;
+ goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
+@@ -308,13 +324,18 @@ static int tcf_pedit_act(struct sk_buff *skb, const struct tc_action *a,
+ struct tcf_result *res)
+ {
+ struct tcf_pedit *p = to_pedit(a);
++ u32 max_offset;
+ int i;
+
+- if (skb_unclone(skb, GFP_ATOMIC))
+- return p->tcf_action;
+-
+ spin_lock(&p->tcf_lock);
+
++ max_offset = (skb_transport_header_was_set(skb) ?
++ skb_transport_offset(skb) :
++ skb_network_offset(skb)) +
++ p->tcfp_off_max_hint;
++ if (skb_ensure_writable(skb, min(skb->len, max_offset)))
++ goto unlock;
++
+ tcf_lastuse_update(&p->tcf_tm);
+
+ if (p->tcfp_nkeys > 0) {
+@@ -403,6 +424,7 @@ bad:
+ p->tcf_qstats.overlimits++;
+ done:
+ bstats_update(&p->tcf_bstats, skb);
++unlock:
+ spin_unlock(&p->tcf_lock);
+ return p->tcf_action;
+ }
+diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
+index 2ef8f5a6205a9..4b552c10e7b98 100644
+--- a/net/sched/cls_api.c
++++ b/net/sched/cls_api.c
+@@ -1044,7 +1044,7 @@ static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
+
+ /* Find qdisc */
+ if (!*parent) {
+- *q = dev->qdisc;
++ *q = rcu_dereference(dev->qdisc);
+ *parent = (*q)->handle;
+ } else {
+ *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
+@@ -1617,12 +1617,17 @@ int tcf_classify(struct sk_buff *skb,
+
+ /* If we missed on some chain */
+ if (ret == TC_ACT_UNSPEC && last_executed_chain) {
++ struct tc_skb_cb *cb = tc_skb_cb(skb);
++
+ ext = tc_skb_ext_alloc(skb);
+ if (WARN_ON_ONCE(!ext))
+ return TC_ACT_SHOT;
+ ext->chain = last_executed_chain;
+- ext->mru = qdisc_skb_cb(skb)->mru;
+- ext->post_ct = qdisc_skb_cb(skb)->post_ct;
++ ext->mru = cb->mru;
++ ext->post_ct = cb->post_ct;
++ ext->post_ct_snat = cb->post_ct_snat;
++ ext->post_ct_dnat = cb->post_ct_dnat;
++ ext->zone = cb->zone;
+ }
+
+ return ret;
+@@ -1648,10 +1653,10 @@ static int tcf_chain_tp_insert(struct tcf_chain *chain,
+ if (chain->flushing)
+ return -EAGAIN;
+
++ RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
+ if (*chain_info->pprev == chain->filter_chain)
+ tcf_chain0_head_change(chain, tp);
+ tcf_proto_get(tp);
+- RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
+ rcu_assign_pointer(*chain_info->pprev, tp);
+
+ return 0;
+@@ -1940,9 +1945,9 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
+ bool prio_allocate;
+ u32 parent;
+ u32 chain_index;
+- struct Qdisc *q = NULL;
++ struct Qdisc *q;
+ struct tcf_chain_info chain_info;
+- struct tcf_chain *chain = NULL;
++ struct tcf_chain *chain;
+ struct tcf_block *block;
+ struct tcf_proto *tp;
+ unsigned long cl;
+@@ -1971,6 +1976,8 @@ replay:
+ tp = NULL;
+ cl = 0;
+ block = NULL;
++ q = NULL;
++ chain = NULL;
+ flags = 0;
+
+ if (prio == 0) {
+@@ -2580,7 +2587,7 @@ static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
+
+ parent = tcm->tcm_parent;
+ if (!parent)
+- q = dev->qdisc;
++ q = rtnl_dereference(dev->qdisc);
+ else
+ q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
+ if (!q)
+@@ -2793,8 +2800,8 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
+ struct tcmsg *t;
+ u32 parent;
+ u32 chain_index;
+- struct Qdisc *q = NULL;
+- struct tcf_chain *chain = NULL;
++ struct Qdisc *q;
++ struct tcf_chain *chain;
+ struct tcf_block *block;
+ unsigned long cl;
+ int err;
+@@ -2804,6 +2811,7 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
+ return -EPERM;
+
+ replay:
++ q = NULL;
+ err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
+ rtm_tca_policy, extack);
+ if (err < 0)
+@@ -2954,7 +2962,7 @@ static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
+ return skb->len;
+
+ if (!tcm->tcm_parent)
+- q = dev->qdisc;
++ q = rtnl_dereference(dev->qdisc);
+ else
+ q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
+
+@@ -3687,6 +3695,7 @@ int tc_setup_flow_action(struct flow_action *flow_action,
+ entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
+ break;
+ default:
++ err = -EOPNOTSUPP;
+ goto err_out_locked;
+ }
+ } else if (is_tcf_skbedit_ptype(act)) {
+diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
+index eb6345a027e13..32b03a13f9b25 100644
+--- a/net/sched/cls_flower.c
++++ b/net/sched/cls_flower.c
+@@ -19,6 +19,7 @@
+
+ #include <net/sch_generic.h>
+ #include <net/pkt_cls.h>
++#include <net/pkt_sched.h>
+ #include <net/ip.h>
+ #include <net/flow_dissector.h>
+ #include <net/geneve.h>
+@@ -309,7 +310,8 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
+ struct tcf_result *res)
+ {
+ struct cls_fl_head *head = rcu_dereference_bh(tp->root);
+- bool post_ct = qdisc_skb_cb(skb)->post_ct;
++ bool post_ct = tc_skb_cb(skb)->post_ct;
++ u16 zone = tc_skb_cb(skb)->zone;
+ struct fl_flow_key skb_key;
+ struct fl_flow_mask *mask;
+ struct cls_fl_filter *f;
+@@ -327,7 +329,7 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
+ skb_flow_dissect_ct(skb, &mask->dissector, &skb_key,
+ fl_ct_info_to_flower_map,
+ ARRAY_SIZE(fl_ct_info_to_flower_map),
+- post_ct);
++ post_ct, zone);
+ skb_flow_dissect_hash(skb, &mask->dissector, &skb_key);
+ skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
+
+@@ -1002,6 +1004,7 @@ static int fl_set_key_mpls(struct nlattr **tb,
+ static void fl_set_key_vlan(struct nlattr **tb,
+ __be16 ethertype,
+ int vlan_id_key, int vlan_prio_key,
++ int vlan_next_eth_type_key,
+ struct flow_dissector_key_vlan *key_val,
+ struct flow_dissector_key_vlan *key_mask)
+ {
+@@ -1020,6 +1023,11 @@ static void fl_set_key_vlan(struct nlattr **tb,
+ }
+ key_val->vlan_tpid = ethertype;
+ key_mask->vlan_tpid = cpu_to_be16(~0);
++ if (tb[vlan_next_eth_type_key]) {
++ key_val->vlan_eth_type =
++ nla_get_be16(tb[vlan_next_eth_type_key]);
++ key_mask->vlan_eth_type = cpu_to_be16(~0);
++ }
+ }
+
+ static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
+@@ -1516,8 +1524,9 @@ static int fl_set_key(struct net *net, struct nlattr **tb,
+
+ if (eth_type_vlan(ethertype)) {
+ fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID,
+- TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan,
+- &mask->vlan);
++ TCA_FLOWER_KEY_VLAN_PRIO,
++ TCA_FLOWER_KEY_VLAN_ETH_TYPE,
++ &key->vlan, &mask->vlan);
+
+ if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) {
+ ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]);
+@@ -1525,6 +1534,7 @@ static int fl_set_key(struct net *net, struct nlattr **tb,
+ fl_set_key_vlan(tb, ethertype,
+ TCA_FLOWER_KEY_CVLAN_ID,
+ TCA_FLOWER_KEY_CVLAN_PRIO,
++ TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
+ &key->cvlan, &mask->cvlan);
+ fl_set_key_val(tb, &key->basic.n_proto,
+ TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
+@@ -2880,13 +2890,13 @@ static int fl_dump_key(struct sk_buff *skb, struct net *net,
+ goto nla_put_failure;
+
+ if (mask->basic.n_proto) {
+- if (mask->cvlan.vlan_tpid) {
++ if (mask->cvlan.vlan_eth_type) {
+ if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
+ key->basic.n_proto))
+ goto nla_put_failure;
+- } else if (mask->vlan.vlan_tpid) {
++ } else if (mask->vlan.vlan_eth_type) {
+ if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
+- key->basic.n_proto))
++ key->vlan.vlan_eth_type))
+ goto nla_put_failure;
+ }
+ }
+diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
+index 4272814487f09..5d30db0d71574 100644
+--- a/net/sched/cls_u32.c
++++ b/net/sched/cls_u32.c
+@@ -386,14 +386,19 @@ static int u32_init(struct tcf_proto *tp)
+ return 0;
+ }
+
+-static int u32_destroy_key(struct tc_u_knode *n, bool free_pf)
++static void __u32_destroy_key(struct tc_u_knode *n)
+ {
+ struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
+
+ tcf_exts_destroy(&n->exts);
+- tcf_exts_put_net(&n->exts);
+ if (ht && --ht->refcnt == 0)
+ kfree(ht);
++ kfree(n);
++}
++
++static void u32_destroy_key(struct tc_u_knode *n, bool free_pf)
++{
++ tcf_exts_put_net(&n->exts);
+ #ifdef CONFIG_CLS_U32_PERF
+ if (free_pf)
+ free_percpu(n->pf);
+@@ -402,8 +407,7 @@ static int u32_destroy_key(struct tc_u_knode *n, bool free_pf)
+ if (free_pf)
+ free_percpu(n->pcpu_success);
+ #endif
+- kfree(n);
+- return 0;
++ __u32_destroy_key(n);
+ }
+
+ /* u32_delete_key_rcu should be called when free'ing a copied
+@@ -810,10 +814,6 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp,
+ new->flags = n->flags;
+ RCU_INIT_POINTER(new->ht_down, ht);
+
+- /* bump reference count as long as we hold pointer to structure */
+- if (ht)
+- ht->refcnt++;
+-
+ #ifdef CONFIG_CLS_U32_PERF
+ /* Statistics may be incremented by readers during update
+ * so we must keep them in tact. When the node is later destroyed
+@@ -835,6 +835,10 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp,
+ return NULL;
+ }
+
++ /* bump reference count as long as we hold pointer to structure */
++ if (ht)
++ ht->refcnt++;
++
+ return new;
+ }
+
+@@ -898,13 +902,13 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
+ tca[TCA_RATE], flags, extack);
+
+ if (err) {
+- u32_destroy_key(new, false);
++ __u32_destroy_key(new);
+ return err;
+ }
+
+ err = u32_replace_hw_knode(tp, new, flags, extack);
+ if (err) {
+- u32_destroy_key(new, false);
++ __u32_destroy_key(new);
+ return err;
+ }
+
+diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
+index 12f39a2dffd47..0fb387c9d706d 100644
+--- a/net/sched/sch_api.c
++++ b/net/sched/sch_api.c
+@@ -301,7 +301,7 @@ struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
+
+ if (!handle)
+ return NULL;
+- q = qdisc_match_from_root(dev->qdisc, handle);
++ q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);
+ if (q)
+ goto out;
+
+@@ -320,7 +320,7 @@ struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
+
+ if (!handle)
+ return NULL;
+- q = qdisc_match_from_root(dev->qdisc, handle);
++ q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);
+ if (q)
+ goto out;
+
+@@ -1062,7 +1062,7 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
+
+ qdisc_offload_graft_root(dev, new, old, extack);
+
+- if (new && new->ops->attach)
++ if (new && new->ops->attach && !ingress)
+ goto skip;
+
+ for (i = 0; i < num_q; i++) {
+@@ -1082,10 +1082,10 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
+ skip:
+ if (!ingress) {
+ notify_and_destroy(net, skb, n, classid,
+- dev->qdisc, new);
++ rtnl_dereference(dev->qdisc), new);
+ if (new && !new->ops->attach)
+ qdisc_refcount_inc(new);
+- dev->qdisc = new ? : &noop_qdisc;
++ rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc);
+
+ if (new && new->ops->attach)
+ new->ops->attach(new);
+@@ -1204,7 +1204,7 @@ static struct Qdisc *qdisc_create(struct net_device *dev,
+
+ err = -ENOENT;
+ if (!ops) {
+- NL_SET_ERR_MSG(extack, "Specified qdisc not found");
++ NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown");
+ goto err_out;
+ }
+
+@@ -1460,7 +1460,7 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
+ q = dev_ingress_queue(dev)->qdisc_sleeping;
+ }
+ } else {
+- q = dev->qdisc;
++ q = rtnl_dereference(dev->qdisc);
+ }
+ if (!q) {
+ NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
+@@ -1549,7 +1549,7 @@ replay:
+ q = dev_ingress_queue(dev)->qdisc_sleeping;
+ }
+ } else {
+- q = dev->qdisc;
++ q = rtnl_dereference(dev->qdisc);
+ }
+
+ /* It may be default qdisc, ignore it */
+@@ -1771,7 +1771,8 @@ static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
+ s_q_idx = 0;
+ q_idx = 0;
+
+- if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx,
++ if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
++ skb, cb, &q_idx, s_q_idx,
+ true, tca[TCA_DUMP_INVISIBLE]) < 0)
+ goto done;
+
+@@ -2042,7 +2043,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
+ } else if (qid1) {
+ qid = qid1;
+ } else if (qid == 0)
+- qid = dev->qdisc->handle;
++ qid = rtnl_dereference(dev->qdisc)->handle;
+
+ /* Now qid is genuine qdisc handle consistent
+ * both with parent and child.
+@@ -2053,7 +2054,7 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
+ portid = TC_H_MAKE(qid, portid);
+ } else {
+ if (qid == 0)
+- qid = dev->qdisc->handle;
++ qid = rtnl_dereference(dev->qdisc)->handle;
+ }
+
+ /* OK. Locate qdisc */
+@@ -2214,7 +2215,8 @@ static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
+ s_t = cb->args[0];
+ t = 0;
+
+- if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t, true) < 0)
++ if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc),
++ skb, tcm, cb, &t, s_t, true) < 0)
+ goto done;
+
+ dev_queue = dev_ingress_queue(dev);
+diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
+index 3c2300d144681..857aaebd49f43 100644
+--- a/net/sched/sch_cake.c
++++ b/net/sched/sch_cake.c
+@@ -2736,7 +2736,7 @@ static int cake_init(struct Qdisc *sch, struct nlattr *opt,
+ q->tins = kvcalloc(CAKE_MAX_TINS, sizeof(struct cake_tin_data),
+ GFP_KERNEL);
+ if (!q->tins)
+- goto nomem;
++ return -ENOMEM;
+
+ for (i = 0; i < CAKE_MAX_TINS; i++) {
+ struct cake_tin_data *b = q->tins + i;
+@@ -2766,10 +2766,6 @@ static int cake_init(struct Qdisc *sch, struct nlattr *opt,
+ q->min_netlen = ~0;
+ q->min_adjlen = ~0;
+ return 0;
+-
+-nomem:
+- cake_destroy(sch);
+- return -ENOMEM;
+ }
+
+ static int cake_dump(struct Qdisc *sch, struct sk_buff *skb)
+diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c
+index 1f857ffd1ac23..44fa2532a87c0 100644
+--- a/net/sched/sch_ets.c
++++ b/net/sched/sch_ets.c
+@@ -667,12 +667,14 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
+ q->classes[i].deficit = quanta[i];
+ }
+ }
++ for (i = q->nbands; i < oldbands; i++) {
++ if (i >= q->nstrict && q->classes[i].qdisc->q.qlen)
++ list_del(&q->classes[i].alist);
++ qdisc_tree_flush_backlog(q->classes[i].qdisc);
++ }
+ q->nstrict = nstrict;
+ memcpy(q->prio2band, priomap, sizeof(priomap));
+
+- for (i = q->nbands; i < oldbands; i++)
+- qdisc_tree_flush_backlog(q->classes[i].qdisc);
+-
+ for (i = 0; i < q->nbands; i++)
+ q->classes[i].quantum = quanta[i];
+
+diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
+index 830f3559f727a..d6aba6edd16e5 100644
+--- a/net/sched/sch_fq_pie.c
++++ b/net/sched/sch_fq_pie.c
+@@ -531,6 +531,7 @@ static void fq_pie_destroy(struct Qdisc *sch)
+ struct fq_pie_sched_data *q = qdisc_priv(sch);
+
+ tcf_block_put(q->block);
++ q->p_params.tupdate = 0;
+ del_timer_sync(&q->adapt_timer);
+ kvfree(q->flows);
+ }
+diff --git a/net/sched/sch_frag.c b/net/sched/sch_frag.c
+index 8c06381391d6f..5ded4c8672a64 100644
+--- a/net/sched/sch_frag.c
++++ b/net/sched/sch_frag.c
+@@ -1,6 +1,7 @@
+ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+ #include <net/netlink.h>
+ #include <net/sch_generic.h>
++#include <net/pkt_sched.h>
+ #include <net/dst.h>
+ #include <net/ip.h>
+ #include <net/ip6_fib.h>
+@@ -137,7 +138,7 @@ err:
+
+ int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb))
+ {
+- u16 mru = qdisc_skb_cb(skb)->mru;
++ u16 mru = tc_skb_cb(skb)->mru;
+ int err;
+
+ if (mru && skb->len > mru + skb->dev->hard_header_len)
+diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
+index a8dd06c74e318..30c29a9a2efd2 100644
+--- a/net/sched/sch_generic.c
++++ b/net/sched/sch_generic.c
+@@ -1114,30 +1114,33 @@ static void attach_default_qdiscs(struct net_device *dev)
+ if (!netif_is_multiqueue(dev) ||
+ dev->priv_flags & IFF_NO_QUEUE) {
+ netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
+- dev->qdisc = txq->qdisc_sleeping;
+- qdisc_refcount_inc(dev->qdisc);
++ qdisc = txq->qdisc_sleeping;
++ rcu_assign_pointer(dev->qdisc, qdisc);
++ qdisc_refcount_inc(qdisc);
+ } else {
+ qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
+ if (qdisc) {
+- dev->qdisc = qdisc;
++ rcu_assign_pointer(dev->qdisc, qdisc);
+ qdisc->ops->attach(qdisc);
+ }
+ }
++ qdisc = rtnl_dereference(dev->qdisc);
+
+ /* Detect default qdisc setup/init failed and fallback to "noqueue" */
+- if (dev->qdisc == &noop_qdisc) {
++ if (qdisc == &noop_qdisc) {
+ netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
+ default_qdisc_ops->id, noqueue_qdisc_ops.id);
+ dev->priv_flags |= IFF_NO_QUEUE;
+ netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
+- dev->qdisc = txq->qdisc_sleeping;
+- qdisc_refcount_inc(dev->qdisc);
++ qdisc = txq->qdisc_sleeping;
++ rcu_assign_pointer(dev->qdisc, qdisc);
++ qdisc_refcount_inc(qdisc);
+ dev->priv_flags ^= IFF_NO_QUEUE;
+ }
+
+ #ifdef CONFIG_NET_SCHED
+- if (dev->qdisc != &noop_qdisc)
+- qdisc_hash_add(dev->qdisc, false);
++ if (qdisc != &noop_qdisc)
++ qdisc_hash_add(qdisc, false);
+ #endif
+ }
+
+@@ -1167,7 +1170,7 @@ void dev_activate(struct net_device *dev)
+ * and noqueue_qdisc for virtual interfaces
+ */
+
+- if (dev->qdisc == &noop_qdisc)
++ if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
+ attach_default_qdiscs(dev);
+
+ if (!netif_carrier_ok(dev))
+@@ -1330,6 +1333,15 @@ static int qdisc_change_tx_queue_len(struct net_device *dev,
+ return 0;
+ }
+
++void dev_qdisc_change_real_num_tx(struct net_device *dev,
++ unsigned int new_real_tx)
++{
++ struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
++
++ if (qdisc->ops->change_real_num_tx)
++ qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
++}
++
+ int dev_qdisc_change_tx_queue_len(struct net_device *dev)
+ {
+ bool up = dev->flags & IFF_UP;
+@@ -1364,7 +1376,7 @@ static void dev_init_scheduler_queue(struct net_device *dev,
+
+ void dev_init_scheduler(struct net_device *dev)
+ {
+- dev->qdisc = &noop_qdisc;
++ rcu_assign_pointer(dev->qdisc, &noop_qdisc);
+ netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
+ if (dev_ingress_queue(dev))
+ dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
+@@ -1392,8 +1404,8 @@ void dev_shutdown(struct net_device *dev)
+ netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
+ if (dev_ingress_queue(dev))
+ shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
+- qdisc_put(dev->qdisc);
+- dev->qdisc = &noop_qdisc;
++ qdisc_put(rtnl_dereference(dev->qdisc));
++ rcu_assign_pointer(dev->qdisc, &noop_qdisc);
+
+ WARN_ON(timer_pending(&dev->watchdog_timer));
+ }
+@@ -1446,6 +1458,7 @@ void psched_ratecfg_precompute(struct psched_ratecfg *r,
+ {
+ memset(r, 0, sizeof(*r));
+ r->overhead = conf->overhead;
++ r->mpu = conf->mpu;
+ r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
+ r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
+ psched_ratecfg_precompute__(r->rate_bytes_ps, &r->mult, &r->shift);
+diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
+index 5067a6e5d4fde..5cbc32fee8674 100644
+--- a/net/sched/sch_htb.c
++++ b/net/sched/sch_htb.c
+@@ -1803,6 +1803,26 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
+ if (!hopt->rate.rate || !hopt->ceil.rate)
+ goto failure;
+
++ if (q->offload) {
++ /* Options not supported by the offload. */
++ if (hopt->rate.overhead || hopt->ceil.overhead) {
++ NL_SET_ERR_MSG(extack, "HTB offload doesn't support the overhead parameter");
++ goto failure;
++ }
++ if (hopt->rate.mpu || hopt->ceil.mpu) {
++ NL_SET_ERR_MSG(extack, "HTB offload doesn't support the mpu parameter");
++ goto failure;
++ }
++ if (hopt->quantum) {
++ NL_SET_ERR_MSG(extack, "HTB offload doesn't support the quantum parameter");
++ goto failure;
++ }
++ if (hopt->prio) {
++ NL_SET_ERR_MSG(extack, "HTB offload doesn't support the prio parameter");
++ goto failure;
++ }
++ }
++
+ /* Keeping backward compatible with rate_table based iproute2 tc */
+ if (hopt->rate.linklayer == TC_LINKLAYER_UNAWARE)
+ qdisc_put_rtab(qdisc_get_rtab(&hopt->rate, tb[TCA_HTB_RTAB],
+diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
+index e79f1afe0cfd6..db18d8a860f9c 100644
+--- a/net/sched/sch_mq.c
++++ b/net/sched/sch_mq.c
+@@ -125,6 +125,29 @@ static void mq_attach(struct Qdisc *sch)
+ priv->qdiscs = NULL;
+ }
+
++static void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx)
++{
++#ifdef CONFIG_NET_SCHED
++ struct net_device *dev = qdisc_dev(sch);
++ struct Qdisc *qdisc;
++ unsigned int i;
++
++ for (i = new_real_tx; i < dev->real_num_tx_queues; i++) {
++ qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping;
++ /* Only update the default qdiscs we created,
++ * qdiscs with handles are always hashed.
++ */
++ if (qdisc != &noop_qdisc && !qdisc->handle)
++ qdisc_hash_del(qdisc);
++ }
++ for (i = dev->real_num_tx_queues; i < new_real_tx; i++) {
++ qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping;
++ if (qdisc != &noop_qdisc && !qdisc->handle)
++ qdisc_hash_add(qdisc, false);
++ }
++#endif
++}
++
+ static int mq_dump(struct Qdisc *sch, struct sk_buff *skb)
+ {
+ struct net_device *dev = qdisc_dev(sch);
+@@ -288,6 +311,7 @@ struct Qdisc_ops mq_qdisc_ops __read_mostly = {
+ .init = mq_init,
+ .destroy = mq_destroy,
+ .attach = mq_attach,
++ .change_real_num_tx = mq_change_real_num_tx,
+ .dump = mq_dump,
+ .owner = THIS_MODULE,
+ };
+diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
+index 5eb3b1b7ae5e7..50e15add6068f 100644
+--- a/net/sched/sch_mqprio.c
++++ b/net/sched/sch_mqprio.c
+@@ -306,6 +306,28 @@ static void mqprio_attach(struct Qdisc *sch)
+ priv->qdiscs = NULL;
+ }
+
++static void mqprio_change_real_num_tx(struct Qdisc *sch,
++ unsigned int new_real_tx)
++{
++ struct net_device *dev = qdisc_dev(sch);
++ struct Qdisc *qdisc;
++ unsigned int i;
++
++ for (i = new_real_tx; i < dev->real_num_tx_queues; i++) {
++ qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping;
++ /* Only update the default qdiscs we created,
++ * qdiscs with handles are always hashed.
++ */
++ if (qdisc != &noop_qdisc && !qdisc->handle)
++ qdisc_hash_del(qdisc);
++ }
++ for (i = dev->real_num_tx_queues; i < new_real_tx; i++) {
++ qdisc = netdev_get_tx_queue(dev, i)->qdisc_sleeping;
++ if (qdisc != &noop_qdisc && !qdisc->handle)
++ qdisc_hash_add(qdisc, false);
++ }
++}
++
+ static struct netdev_queue *mqprio_queue_get(struct Qdisc *sch,
+ unsigned long cl)
+ {
+@@ -629,6 +651,7 @@ static struct Qdisc_ops mqprio_qdisc_ops __read_mostly = {
+ .init = mqprio_init,
+ .destroy = mqprio_destroy,
+ .attach = mqprio_attach,
++ .change_real_num_tx = mqprio_change_real_num_tx,
+ .dump = mqprio_dump,
+ .owner = THIS_MODULE,
+ };
+diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
+index 58a9d42b52b8f..aea435b0aeb34 100644
+--- a/net/sched/sch_qfq.c
++++ b/net/sched/sch_qfq.c
+@@ -1422,10 +1422,8 @@ static int qfq_init_qdisc(struct Qdisc *sch, struct nlattr *opt,
+ if (err < 0)
+ return err;
+
+- if (qdisc_dev(sch)->tx_queue_len + 1 > QFQ_MAX_AGG_CLASSES)
+- max_classes = QFQ_MAX_AGG_CLASSES;
+- else
+- max_classes = qdisc_dev(sch)->tx_queue_len + 1;
++ max_classes = min_t(u64, (u64)qdisc_dev(sch)->tx_queue_len + 1,
++ QFQ_MAX_AGG_CLASSES);
+ /* max_cl_shift = floor(log_2(max_classes)) */
+ max_cl_shift = __fls(max_classes);
+ q->max_agg_classes = 1<<max_cl_shift;
+diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
+index b9fd18d986464..474ba4db5de2d 100644
+--- a/net/sched/sch_taprio.c
++++ b/net/sched/sch_taprio.c
+@@ -95,18 +95,22 @@ static ktime_t sched_base_time(const struct sched_gate_list *sched)
+ return ns_to_ktime(sched->base_time);
+ }
+
+-static ktime_t taprio_get_time(struct taprio_sched *q)
++static ktime_t taprio_mono_to_any(const struct taprio_sched *q, ktime_t mono)
+ {
+- ktime_t mono = ktime_get();
++ /* This pairs with WRITE_ONCE() in taprio_parse_clockid() */
++ enum tk_offsets tk_offset = READ_ONCE(q->tk_offset);
+
+- switch (q->tk_offset) {
++ switch (tk_offset) {
+ case TK_OFFS_MAX:
+ return mono;
+ default:
+- return ktime_mono_to_any(mono, q->tk_offset);
++ return ktime_mono_to_any(mono, tk_offset);
+ }
++}
+
+- return KTIME_MAX;
++static ktime_t taprio_get_time(const struct taprio_sched *q)
++{
++ return taprio_mono_to_any(q, ktime_get());
+ }
+
+ static void taprio_free_sched_cb(struct rcu_head *head)
+@@ -319,7 +323,7 @@ static ktime_t get_tcp_tstamp(struct taprio_sched *q, struct sk_buff *skb)
+ return 0;
+ }
+
+- return ktime_mono_to_any(skb->skb_mstamp_ns, q->tk_offset);
++ return taprio_mono_to_any(q, skb->skb_mstamp_ns);
+ }
+
+ /* There are a few scenarios where we will have to modify the txtime from
+@@ -413,7 +417,8 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
+ {
+ struct taprio_sched *q = qdisc_priv(sch);
+
+- if (skb->sk && sock_flag(skb->sk, SOCK_TXTIME)) {
++ /* sk_flags are only safe to use on full sockets. */
++ if (skb->sk && sk_fullsock(skb->sk) && sock_flag(skb->sk, SOCK_TXTIME)) {
+ if (!is_valid_interval(skb, sch))
+ return qdisc_drop(skb, sch, to_free);
+ } else if (TXTIME_ASSIST_IS_ENABLED(q->flags)) {
+@@ -1352,6 +1357,7 @@ static int taprio_parse_clockid(struct Qdisc *sch, struct nlattr **tb,
+ }
+ } else if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) {
+ int clockid = nla_get_s32(tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]);
++ enum tk_offsets tk_offset;
+
+ /* We only support static clockids and we don't allow
+ * for it to be modified after the first init.
+@@ -1366,22 +1372,24 @@ static int taprio_parse_clockid(struct Qdisc *sch, struct nlattr **tb,
+
+ switch (clockid) {
+ case CLOCK_REALTIME:
+- q->tk_offset = TK_OFFS_REAL;
++ tk_offset = TK_OFFS_REAL;
+ break;
+ case CLOCK_MONOTONIC:
+- q->tk_offset = TK_OFFS_MAX;
++ tk_offset = TK_OFFS_MAX;
+ break;
+ case CLOCK_BOOTTIME:
+- q->tk_offset = TK_OFFS_BOOT;
++ tk_offset = TK_OFFS_BOOT;
+ break;
+ case CLOCK_TAI:
+- q->tk_offset = TK_OFFS_TAI;
++ tk_offset = TK_OFFS_TAI;
+ break;
+ default:
+ NL_SET_ERR_MSG(extack, "Invalid 'clockid'");
+ err = -EINVAL;
+ goto out;
+ }
++ /* This pairs with READ_ONCE() in taprio_mono_to_any */
++ WRITE_ONCE(q->tk_offset, tk_offset);
+
+ q->clockid = clockid;
+ } else {
+diff --git a/net/sctp/diag.c b/net/sctp/diag.c
+index 760b367644c12..d9c6d8f30f093 100644
+--- a/net/sctp/diag.c
++++ b/net/sctp/diag.c
+@@ -61,10 +61,6 @@ static void inet_diag_msg_sctpasoc_fill(struct inet_diag_msg *r,
+ r->idiag_timer = SCTP_EVENT_TIMEOUT_T3_RTX;
+ r->idiag_retrans = asoc->rtx_data_chunks;
+ r->idiag_expires = jiffies_to_msecs(t3_rtx->expires - jiffies);
+- } else {
+- r->idiag_timer = 0;
+- r->idiag_retrans = 0;
+- r->idiag_expires = 0;
+ }
+ }
+
+@@ -144,13 +140,14 @@ static int inet_sctp_diag_fill(struct sock *sk, struct sctp_association *asoc,
+ r = nlmsg_data(nlh);
+ BUG_ON(!sk_fullsock(sk));
+
++ r->idiag_timer = 0;
++ r->idiag_retrans = 0;
++ r->idiag_expires = 0;
+ if (asoc) {
+ inet_diag_msg_sctpasoc_fill(r, sk, asoc);
+ } else {
+ inet_diag_msg_common_fill(r, sk);
+ r->idiag_state = sk->sk_state;
+- r->idiag_timer = 0;
+- r->idiag_retrans = 0;
+ }
+
+ if (inet_diag_msg_attrs_fill(sk, skb, r, ext, user_ns, net_admin))
+@@ -245,54 +242,49 @@ static size_t inet_assoc_attr_size(struct sctp_association *asoc)
+ + 64;
+ }
+
+-static int sctp_tsp_dump_one(struct sctp_transport *tsp, void *p)
++static int sctp_sock_dump_one(struct sctp_endpoint *ep, struct sctp_transport *tsp, void *p)
+ {
+ struct sctp_association *assoc = tsp->asoc;
+- struct sock *sk = tsp->asoc->base.sk;
+ struct sctp_comm_param *commp = p;
+- struct sk_buff *in_skb = commp->skb;
++ struct sock *sk = ep->base.sk;
+ const struct inet_diag_req_v2 *req = commp->r;
+- const struct nlmsghdr *nlh = commp->nlh;
+- struct net *net = sock_net(in_skb->sk);
++ struct sk_buff *skb = commp->skb;
+ struct sk_buff *rep;
+ int err;
+
+ err = sock_diag_check_cookie(sk, req->id.idiag_cookie);
+ if (err)
+- goto out;
++ return err;
+
+- err = -ENOMEM;
+ rep = nlmsg_new(inet_assoc_attr_size(assoc), GFP_KERNEL);
+ if (!rep)
+- goto out;
++ return -ENOMEM;
+
+ lock_sock(sk);
+- if (sk != assoc->base.sk) {
+- release_sock(sk);
+- sk = assoc->base.sk;
+- lock_sock(sk);
++ if (ep != assoc->ep) {
++ err = -EAGAIN;
++ goto out;
+ }
+- err = inet_sctp_diag_fill(sk, assoc, rep, req,
+- sk_user_ns(NETLINK_CB(in_skb).sk),
+- NETLINK_CB(in_skb).portid,
+- nlh->nlmsg_seq, 0, nlh,
+- commp->net_admin);
+- release_sock(sk);
++
++ err = inet_sctp_diag_fill(sk, assoc, rep, req, sk_user_ns(NETLINK_CB(skb).sk),
++ NETLINK_CB(skb).portid, commp->nlh->nlmsg_seq, 0,
++ commp->nlh, commp->net_admin);
+ if (err < 0) {
+ WARN_ON(err == -EMSGSIZE);
+- kfree_skb(rep);
+ goto out;
+ }
++ release_sock(sk);
+
+- err = nlmsg_unicast(net->diag_nlsk, rep, NETLINK_CB(in_skb).portid);
++ return nlmsg_unicast(sock_net(skb->sk)->diag_nlsk, rep, NETLINK_CB(skb).portid);
+
+ out:
++ release_sock(sk);
++ kfree_skb(rep);
+ return err;
+ }
+
+-static int sctp_sock_dump(struct sctp_transport *tsp, void *p)
++static int sctp_sock_dump(struct sctp_endpoint *ep, struct sctp_transport *tsp, void *p)
+ {
+- struct sctp_endpoint *ep = tsp->asoc->ep;
+ struct sctp_comm_param *commp = p;
+ struct sock *sk = ep->base.sk;
+ struct sk_buff *skb = commp->skb;
+@@ -302,6 +294,8 @@ static int sctp_sock_dump(struct sctp_transport *tsp, void *p)
+ int err = 0;
+
+ lock_sock(sk);
++ if (ep != tsp->asoc->ep)
++ goto release;
+ list_for_each_entry(assoc, &ep->asocs, asocs) {
+ if (cb->args[4] < cb->args[1])
+ goto next;
+@@ -344,9 +338,8 @@ release:
+ return err;
+ }
+
+-static int sctp_sock_filter(struct sctp_transport *tsp, void *p)
++static int sctp_sock_filter(struct sctp_endpoint *ep, struct sctp_transport *tsp, void *p)
+ {
+- struct sctp_endpoint *ep = tsp->asoc->ep;
+ struct sctp_comm_param *commp = p;
+ struct sock *sk = ep->base.sk;
+ const struct inet_diag_req_v2 *r = commp->r;
+@@ -429,15 +422,15 @@ static void sctp_diag_get_info(struct sock *sk, struct inet_diag_msg *r,
+ static int sctp_diag_dump_one(struct netlink_callback *cb,
+ const struct inet_diag_req_v2 *req)
+ {
+- struct sk_buff *in_skb = cb->skb;
+- struct net *net = sock_net(in_skb->sk);
++ struct sk_buff *skb = cb->skb;
++ struct net *net = sock_net(skb->sk);
+ const struct nlmsghdr *nlh = cb->nlh;
+ union sctp_addr laddr, paddr;
+ struct sctp_comm_param commp = {
+- .skb = in_skb,
++ .skb = skb,
+ .r = req,
+ .nlh = nlh,
+- .net_admin = netlink_net_capable(in_skb, CAP_NET_ADMIN),
++ .net_admin = netlink_net_capable(skb, CAP_NET_ADMIN),
+ };
+
+ if (req->sdiag_family == AF_INET) {
+@@ -460,7 +453,7 @@ static int sctp_diag_dump_one(struct netlink_callback *cb,
+ paddr.v6.sin6_family = AF_INET6;
+ }
+
+- return sctp_transport_lookup_process(sctp_tsp_dump_one,
++ return sctp_transport_lookup_process(sctp_sock_dump_one,
+ net, &laddr, &paddr, &commp);
+ }
+
+@@ -505,8 +498,8 @@ skip:
+ if (!(idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE)))
+ goto done;
+
+- sctp_for_each_transport(sctp_sock_filter, sctp_sock_dump,
+- net, &pos, &commp);
++ sctp_transport_traverse_process(sctp_sock_filter, sctp_sock_dump,
++ net, &pos, &commp);
+ cb->args[2] = pos;
+
+ done:
+diff --git a/net/sctp/endpointola.c b/net/sctp/endpointola.c
+index 48c9c2c7602f7..efffde7f2328e 100644
+--- a/net/sctp/endpointola.c
++++ b/net/sctp/endpointola.c
+@@ -184,6 +184,18 @@ void sctp_endpoint_free(struct sctp_endpoint *ep)
+ }
+
+ /* Final destructor for endpoint. */
++static void sctp_endpoint_destroy_rcu(struct rcu_head *head)
++{
++ struct sctp_endpoint *ep = container_of(head, struct sctp_endpoint, rcu);
++ struct sock *sk = ep->base.sk;
++
++ sctp_sk(sk)->ep = NULL;
++ sock_put(sk);
++
++ kfree(ep);
++ SCTP_DBG_OBJCNT_DEC(ep);
++}
++
+ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
+ {
+ struct sock *sk;
+@@ -213,18 +225,13 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
+ if (sctp_sk(sk)->bind_hash)
+ sctp_put_port(sk);
+
+- sctp_sk(sk)->ep = NULL;
+- /* Give up our hold on the sock */
+- sock_put(sk);
+-
+- kfree(ep);
+- SCTP_DBG_OBJCNT_DEC(ep);
++ call_rcu(&ep->rcu, sctp_endpoint_destroy_rcu);
+ }
+
+ /* Hold a reference to an endpoint. */
+-void sctp_endpoint_hold(struct sctp_endpoint *ep)
++int sctp_endpoint_hold(struct sctp_endpoint *ep)
+ {
+- refcount_inc(&ep->base.refcnt);
++ return refcount_inc_not_zero(&ep->base.refcnt);
+ }
+
+ /* Release a reference to an endpoint and clean up if there are
+diff --git a/net/sctp/output.c b/net/sctp/output.c
+index 4dfb5ea82b05b..cdfdbd353c678 100644
+--- a/net/sctp/output.c
++++ b/net/sctp/output.c
+@@ -581,13 +581,16 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp)
+ chunk = list_entry(packet->chunk_list.next, struct sctp_chunk, list);
+ sk = chunk->skb->sk;
+
+- /* check gso */
+ if (packet->size > tp->pathmtu && !packet->ipfragok && !chunk->pmtu_probe) {
+- if (!sk_can_gso(sk)) {
+- pr_err_once("Trying to GSO but underlying device doesn't support it.");
+- goto out;
++ if (tp->pl.state == SCTP_PL_ERROR) { /* do IP fragmentation if in Error state */
++ packet->ipfragok = 1;
++ } else {
++ if (!sk_can_gso(sk)) { /* check gso */
++ pr_err_once("Trying to GSO but underlying device doesn't support it.");
++ goto out;
++ }
++ gso = 1;
+ }
+- gso = 1;
+ }
+
+ /* alloc head skb */
+diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c
+index ff47091c385e7..b3950963fc8f0 100644
+--- a/net/sctp/outqueue.c
++++ b/net/sctp/outqueue.c
+@@ -911,6 +911,7 @@ static void sctp_outq_flush_ctrl(struct sctp_flush_ctx *ctx)
+ ctx->asoc->base.sk->sk_err = -error;
+ return;
+ }
++ ctx->asoc->stats.octrlchunks++;
+ break;
+
+ case SCTP_CID_ABORT:
+@@ -935,7 +936,10 @@ static void sctp_outq_flush_ctrl(struct sctp_flush_ctx *ctx)
+
+ case SCTP_CID_HEARTBEAT:
+ if (chunk->pmtu_probe) {
+- sctp_packet_singleton(ctx->transport, chunk, ctx->gfp);
++ error = sctp_packet_singleton(ctx->transport,
++ chunk, ctx->gfp);
++ if (!error)
++ ctx->asoc->stats.octrlchunks++;
+ break;
+ }
+ fallthrough;
+diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c
+index b3815b568e8e5..463c4a58d2c36 100644
+--- a/net/sctp/sm_sideeffect.c
++++ b/net/sctp/sm_sideeffect.c
+@@ -458,6 +458,10 @@ void sctp_generate_reconf_event(struct timer_list *t)
+ goto out_unlock;
+ }
+
++ /* This happens when the response arrives after the timer is triggered. */
++ if (!asoc->strreset_chunk)
++ goto out_unlock;
++
+ error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
+ SCTP_ST_TIMEOUT(SCTP_EVENT_TIMEOUT_RECONF),
+ asoc->state, asoc->ep, asoc,
+diff --git a/net/sctp/socket.c b/net/sctp/socket.c
+index 6b937bfd47515..5f6e6a6e91b3d 100644
+--- a/net/sctp/socket.c
++++ b/net/sctp/socket.c
+@@ -5317,32 +5317,41 @@ int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *),
+ }
+ EXPORT_SYMBOL_GPL(sctp_for_each_endpoint);
+
+-int sctp_transport_lookup_process(int (*cb)(struct sctp_transport *, void *),
+- struct net *net,
++int sctp_transport_lookup_process(sctp_callback_t cb, struct net *net,
+ const union sctp_addr *laddr,
+ const union sctp_addr *paddr, void *p)
+ {
+ struct sctp_transport *transport;
+- int err;
++ struct sctp_endpoint *ep;
++ int err = -ENOENT;
+
+ rcu_read_lock();
+ transport = sctp_addrs_lookup_transport(net, laddr, paddr);
++ if (!transport) {
++ rcu_read_unlock();
++ return err;
++ }
++ ep = transport->asoc->ep;
++ if (!sctp_endpoint_hold(ep)) { /* asoc can be peeled off */
++ sctp_transport_put(transport);
++ rcu_read_unlock();
++ return err;
++ }
+ rcu_read_unlock();
+- if (!transport)
+- return -ENOENT;
+
+- err = cb(transport, p);
++ err = cb(ep, transport, p);
++ sctp_endpoint_put(ep);
+ sctp_transport_put(transport);
+-
+ return err;
+ }
+ EXPORT_SYMBOL_GPL(sctp_transport_lookup_process);
+
+-int sctp_for_each_transport(int (*cb)(struct sctp_transport *, void *),
+- int (*cb_done)(struct sctp_transport *, void *),
+- struct net *net, int *pos, void *p) {
++int sctp_transport_traverse_process(sctp_callback_t cb, sctp_callback_t cb_done,
++ struct net *net, int *pos, void *p)
++{
+ struct rhashtable_iter hti;
+ struct sctp_transport *tsp;
++ struct sctp_endpoint *ep;
+ int ret;
+
+ again:
+@@ -5351,26 +5360,32 @@ again:
+
+ tsp = sctp_transport_get_idx(net, &hti, *pos + 1);
+ for (; !IS_ERR_OR_NULL(tsp); tsp = sctp_transport_get_next(net, &hti)) {
+- ret = cb(tsp, p);
+- if (ret)
+- break;
++ ep = tsp->asoc->ep;
++ if (sctp_endpoint_hold(ep)) { /* asoc can be peeled off */
++ ret = cb(ep, tsp, p);
++ if (ret)
++ break;
++ sctp_endpoint_put(ep);
++ }
+ (*pos)++;
+ sctp_transport_put(tsp);
+ }
+ sctp_transport_walk_stop(&hti);
+
+ if (ret) {
+- if (cb_done && !cb_done(tsp, p)) {
++ if (cb_done && !cb_done(ep, tsp, p)) {
+ (*pos)++;
++ sctp_endpoint_put(ep);
+ sctp_transport_put(tsp);
+ goto again;
+ }
++ sctp_endpoint_put(ep);
+ sctp_transport_put(tsp);
+ }
+
+ return ret;
+ }
+-EXPORT_SYMBOL_GPL(sctp_for_each_transport);
++EXPORT_SYMBOL_GPL(sctp_transport_traverse_process);
+
+ /* 7.2.1 Association Status (SCTP_STATUS)
+
+@@ -5626,7 +5641,7 @@ int sctp_do_peeloff(struct sock *sk, sctp_assoc_t id, struct socket **sockp)
+ * Set the daddr and initialize id to something more random and also
+ * copy over any ip options.
+ */
+- sp->pf->to_sk_daddr(&asoc->peer.primary_addr, sk);
++ sp->pf->to_sk_daddr(&asoc->peer.primary_addr, sock->sk);
+ sp->pf->copy_ip_options(sk, sock->sk);
+
+ /* Populate the fields of the newsk from the oldsk and migrate the
+diff --git a/net/sctp/transport.c b/net/sctp/transport.c
+index a3d3ca6dd63dd..133f1719bf1b7 100644
+--- a/net/sctp/transport.c
++++ b/net/sctp/transport.c
+@@ -269,7 +269,7 @@ bool sctp_transport_pl_send(struct sctp_transport *t)
+ if (t->pl.probe_size == SCTP_BASE_PLPMTU) { /* BASE_PLPMTU Confirmation Failed */
+ t->pl.state = SCTP_PL_ERROR; /* Base -> Error */
+
+- t->pl.pmtu = SCTP_MIN_PLPMTU;
++ t->pl.pmtu = SCTP_BASE_PLPMTU;
+ t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t);
+ sctp_assoc_sync_pmtu(t->asoc);
+ }
+@@ -366,8 +366,9 @@ static bool sctp_transport_pl_toobig(struct sctp_transport *t, u32 pmtu)
+ if (pmtu >= SCTP_MIN_PLPMTU && pmtu < SCTP_BASE_PLPMTU) {
+ t->pl.state = SCTP_PL_ERROR; /* Base -> Error */
+
+- t->pl.pmtu = SCTP_MIN_PLPMTU;
++ t->pl.pmtu = SCTP_BASE_PLPMTU;
+ t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t);
++ return true;
+ }
+ } else if (t->pl.state == SCTP_PL_SEARCH) {
+ if (pmtu >= SCTP_BASE_PLPMTU && pmtu < t->pl.pmtu) {
+@@ -378,11 +379,10 @@ static bool sctp_transport_pl_toobig(struct sctp_transport *t, u32 pmtu)
+ t->pl.probe_high = 0;
+ t->pl.pmtu = SCTP_BASE_PLPMTU;
+ t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t);
++ return true;
+ } else if (pmtu > t->pl.pmtu && pmtu < t->pl.probe_size) {
+ t->pl.probe_size = pmtu;
+ t->pl.probe_count = 0;
+-
+- return false;
+ }
+ } else if (t->pl.state == SCTP_PL_COMPLETE) {
+ if (pmtu >= SCTP_BASE_PLPMTU && pmtu < t->pl.pmtu) {
+@@ -393,10 +393,11 @@ static bool sctp_transport_pl_toobig(struct sctp_transport *t, u32 pmtu)
+ t->pl.probe_high = 0;
+ t->pl.pmtu = SCTP_BASE_PLPMTU;
+ t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t);
++ return true;
+ }
+ }
+
+- return true;
++ return false;
+ }
+
+ bool sctp_transport_update_pmtu(struct sctp_transport *t, u32 pmtu)
+diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
+index 78b663dbfa1f9..fb801c249d923 100644
+--- a/net/smc/af_smc.c
++++ b/net/smc/af_smc.c
+@@ -148,14 +148,18 @@ static int __smc_release(struct smc_sock *smc)
+ sock_set_flag(sk, SOCK_DEAD);
+ sk->sk_shutdown |= SHUTDOWN_MASK;
+ } else {
+- if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
+- sock_put(sk); /* passive closing */
+- if (sk->sk_state == SMC_LISTEN) {
+- /* wake up clcsock accept */
+- rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
++ if (sk->sk_state != SMC_CLOSED) {
++ if (sk->sk_state != SMC_LISTEN &&
++ sk->sk_state != SMC_INIT)
++ sock_put(sk); /* passive closing */
++ if (sk->sk_state == SMC_LISTEN) {
++ /* wake up clcsock accept */
++ rc = kernel_sock_shutdown(smc->clcsock,
++ SHUT_RDWR);
++ }
++ sk->sk_state = SMC_CLOSED;
++ sk->sk_state_change(sk);
+ }
+- sk->sk_state = SMC_CLOSED;
+- sk->sk_state_change(sk);
+ smc_restore_fallback_changes(smc);
+ }
+
+@@ -178,7 +182,7 @@ static int smc_release(struct socket *sock)
+ {
+ struct sock *sk = sock->sk;
+ struct smc_sock *smc;
+- int rc = 0;
++ int old_state, rc = 0;
+
+ if (!sk)
+ goto out;
+@@ -186,10 +190,14 @@ static int smc_release(struct socket *sock)
+ sock_hold(sk); /* sock_put below */
+ smc = smc_sk(sk);
+
++ old_state = sk->sk_state;
++
+ /* cleanup for a dangling non-blocking connect */
+- if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
++ if (smc->connect_nonblock && old_state == SMC_INIT)
+ tcp_abort(smc->clcsock->sk, ECONNABORTED);
+- flush_work(&smc->connect_work);
++
++ if (cancel_work_sync(&smc->connect_work))
++ sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
+
+ if (sk->sk_state == SMC_LISTEN)
+ /* smc_close_non_accepted() is called and acquires
+@@ -199,6 +207,10 @@ static int smc_release(struct socket *sock)
+ else
+ lock_sock(sk);
+
++ if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
++ !smc->use_fallback)
++ smc_close_active_abort(smc);
++
+ rc = __smc_release(smc);
+
+ /* detach socket */
+@@ -542,8 +554,118 @@ static void smc_stat_fallback(struct smc_sock *smc)
+ mutex_unlock(&net->smc.mutex_fback_rsn);
+ }
+
+-static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
++/* must be called under rcu read lock */
++static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
++{
++ struct socket_wq *wq;
++ __poll_t flags;
++
++ wq = rcu_dereference(smc->sk.sk_wq);
++ if (!skwq_has_sleeper(wq))
++ return;
++
++ /* wake up smc sk->sk_wq */
++ if (!key) {
++ /* sk_state_change */
++ wake_up_interruptible_all(&wq->wait);
++ } else {
++ flags = key_to_poll(key);
++ if (flags & (EPOLLIN | EPOLLOUT))
++ /* sk_data_ready or sk_write_space */
++ wake_up_interruptible_sync_poll(&wq->wait, flags);
++ else if (flags & EPOLLERR)
++ /* sk_error_report */
++ wake_up_interruptible_poll(&wq->wait, flags);
++ }
++}
++
++static int smc_fback_mark_woken(wait_queue_entry_t *wait,
++ unsigned int mode, int sync, void *key)
++{
++ struct smc_mark_woken *mark =
++ container_of(wait, struct smc_mark_woken, wait_entry);
++
++ mark->woken = true;
++ mark->key = key;
++ return 0;
++}
++
++static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
++ void (*clcsock_callback)(struct sock *sk))
++{
++ struct smc_mark_woken mark = { .woken = false };
++ struct socket_wq *wq;
++
++ init_waitqueue_func_entry(&mark.wait_entry,
++ smc_fback_mark_woken);
++ rcu_read_lock();
++ wq = rcu_dereference(clcsk->sk_wq);
++ if (!wq)
++ goto out;
++ add_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
++ clcsock_callback(clcsk);
++ remove_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
++
++ if (mark.woken)
++ smc_fback_wakeup_waitqueue(smc, mark.key);
++out:
++ rcu_read_unlock();
++}
++
++static void smc_fback_state_change(struct sock *clcsk)
++{
++ struct smc_sock *smc =
++ smc_clcsock_user_data(clcsk);
++
++ if (!smc)
++ return;
++ smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_state_change);
++}
++
++static void smc_fback_data_ready(struct sock *clcsk)
++{
++ struct smc_sock *smc =
++ smc_clcsock_user_data(clcsk);
++
++ if (!smc)
++ return;
++ smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_data_ready);
++}
++
++static void smc_fback_write_space(struct sock *clcsk)
++{
++ struct smc_sock *smc =
++ smc_clcsock_user_data(clcsk);
++
++ if (!smc)
++ return;
++ smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_write_space);
++}
++
++static void smc_fback_error_report(struct sock *clcsk)
+ {
++ struct smc_sock *smc =
++ smc_clcsock_user_data(clcsk);
++
++ if (!smc)
++ return;
++ smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_error_report);
++}
++
++static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
++{
++ struct sock *clcsk;
++ int rc = 0;
++
++ mutex_lock(&smc->clcsock_release_lock);
++ if (!smc->clcsock) {
++ rc = -EBADF;
++ goto out;
++ }
++ clcsk = smc->clcsock->sk;
++
++ if (smc->use_fallback)
++ goto out;
+ smc->use_fallback = true;
+ smc->fallback_rsn = reason_code;
+ smc_stat_fallback(smc);
+@@ -552,13 +674,42 @@ static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
+ smc->clcsock->file->private_data = smc->clcsock;
+ smc->clcsock->wq.fasync_list =
+ smc->sk.sk_socket->wq.fasync_list;
++
++ /* There might be some wait entries remaining
++ * in smc sk->sk_wq and they should be woken up
++ * as clcsock's wait queue is woken up.
++ */
++ smc->clcsk_state_change = clcsk->sk_state_change;
++ smc->clcsk_data_ready = clcsk->sk_data_ready;
++ smc->clcsk_write_space = clcsk->sk_write_space;
++ smc->clcsk_error_report = clcsk->sk_error_report;
++
++ clcsk->sk_state_change = smc_fback_state_change;
++ clcsk->sk_data_ready = smc_fback_data_ready;
++ clcsk->sk_write_space = smc_fback_write_space;
++ clcsk->sk_error_report = smc_fback_error_report;
++
++ smc->clcsock->sk->sk_user_data =
++ (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
+ }
++out:
++ mutex_unlock(&smc->clcsock_release_lock);
++ return rc;
+ }
+
+ /* fall back during connect */
+ static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
+ {
+- smc_switch_to_fallback(smc, reason_code);
++ struct net *net = sock_net(&smc->sk);
++ int rc = 0;
++
++ rc = smc_switch_to_fallback(smc, reason_code);
++ if (rc) { /* fallback fails */
++ this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
++ if (smc->sk.sk_state == SMC_INIT)
++ sock_put(&smc->sk); /* passive closing */
++ return rc;
++ }
+ smc_copy_sock_settings_to_clc(smc);
+ smc->connect_nonblock = 0;
+ if (smc->sk.sk_state == SMC_INIT)
+@@ -593,10 +744,12 @@ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
+
+ static void smc_conn_abort(struct smc_sock *smc, int local_first)
+ {
++ struct smc_connection *conn = &smc->conn;
++ struct smc_link_group *lgr = conn->lgr;
++
++ smc_conn_free(conn);
+ if (local_first)
+- smc_lgr_cleanup_early(&smc->conn);
+- else
+- smc_conn_free(&smc->conn);
++ smc_lgr_cleanup_early(lgr);
+ }
+
+ /* check if there is a rdma device available for this connection. */
+@@ -1070,6 +1223,8 @@ static void smc_connect_work(struct work_struct *work)
+ smc->sk.sk_state = SMC_CLOSED;
+ if (rc == -EPIPE || rc == -EAGAIN)
+ smc->sk.sk_err = EPIPE;
++ else if (rc == -ECONNREFUSED)
++ smc->sk.sk_err = ECONNREFUSED;
+ else if (signal_pending(current))
+ smc->sk.sk_err = -sock_intr_errno(timeo);
+ sock_put(&smc->sk); /* passive closing */
+@@ -1362,11 +1517,12 @@ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
+ {
+ /* RDMA setup failed, switch back to TCP */
+ smc_conn_abort(new_smc, local_first);
+- if (reason_code < 0) { /* error, no fallback possible */
++ if (reason_code < 0 ||
++ smc_switch_to_fallback(new_smc, reason_code)) {
++ /* error, no fallback possible */
+ smc_listen_out_err(new_smc);
+ return;
+ }
+- smc_switch_to_fallback(new_smc, reason_code);
+ if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
+ if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
+ smc_listen_out_err(new_smc);
+@@ -1739,8 +1895,11 @@ static void smc_listen_work(struct work_struct *work)
+
+ /* check if peer is smc capable */
+ if (!tcp_sk(newclcsock->sk)->syn_smc) {
+- smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
+- smc_listen_out_connected(new_smc);
++ rc = smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
++ if (rc)
++ smc_listen_out_err(new_smc);
++ else
++ smc_listen_out_connected(new_smc);
+ return;
+ }
+
+@@ -1866,10 +2025,9 @@ out:
+
+ static void smc_clcsock_data_ready(struct sock *listen_clcsock)
+ {
+- struct smc_sock *lsmc;
++ struct smc_sock *lsmc =
++ smc_clcsock_user_data(listen_clcsock);
+
+- lsmc = (struct smc_sock *)
+- ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
+ if (!lsmc)
+ return;
+ lsmc->clcsk_data_ready(listen_clcsock);
+@@ -1914,8 +2072,10 @@ static int smc_listen(struct socket *sock, int backlog)
+ smc->clcsock->sk->sk_user_data =
+ (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
+ rc = kernel_listen(smc->clcsock, backlog);
+- if (rc)
++ if (rc) {
++ smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
+ goto out;
++ }
+ sk->sk_max_ack_backlog = backlog;
+ sk->sk_ack_backlog = 0;
+ sk->sk_state = SMC_LISTEN;
+@@ -2024,7 +2184,9 @@ static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
+
+ if (msg->msg_flags & MSG_FASTOPEN) {
+ if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
+- smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
++ rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
++ if (rc)
++ goto out;
+ } else {
+ rc = -EINVAL;
+ goto out;
+@@ -2148,8 +2310,10 @@ static __poll_t smc_poll(struct file *file, struct socket *sock,
+ static int smc_shutdown(struct socket *sock, int how)
+ {
+ struct sock *sk = sock->sk;
++ bool do_shutdown = true;
+ struct smc_sock *smc;
+ int rc = -EINVAL;
++ int old_state;
+ int rc1 = 0;
+
+ smc = smc_sk(sk);
+@@ -2170,13 +2334,19 @@ static int smc_shutdown(struct socket *sock, int how)
+ if (smc->use_fallback) {
+ rc = kernel_sock_shutdown(smc->clcsock, how);
+ sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
+- if (sk->sk_shutdown == SHUTDOWN_MASK)
++ if (sk->sk_shutdown == SHUTDOWN_MASK) {
+ sk->sk_state = SMC_CLOSED;
++ sock_put(sk);
++ }
+ goto out;
+ }
+ switch (how) {
+ case SHUT_RDWR: /* shutdown in both directions */
++ old_state = sk->sk_state;
+ rc = smc_close_active(smc);
++ if (old_state == SMC_ACTIVE &&
++ sk->sk_state == SMC_PEERCLOSEWAIT1)
++ do_shutdown = false;
+ break;
+ case SHUT_WR:
+ rc = smc_close_shutdown_write(smc);
+@@ -2186,7 +2356,7 @@ static int smc_shutdown(struct socket *sock, int how)
+ /* nothing more to do because peer is not involved */
+ break;
+ }
+- if (smc->clcsock)
++ if (do_shutdown && smc->clcsock)
+ rc1 = kernel_sock_shutdown(smc->clcsock, how);
+ /* map sock_shutdown_cmd constants to sk_shutdown value range */
+ sk->sk_shutdown |= how + 1;
+@@ -2211,6 +2381,11 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
+ /* generic setsockopts reaching us here always apply to the
+ * CLC socket
+ */
++ mutex_lock(&smc->clcsock_release_lock);
++ if (!smc->clcsock) {
++ mutex_unlock(&smc->clcsock_release_lock);
++ return -EBADF;
++ }
+ if (unlikely(!smc->clcsock->ops->setsockopt))
+ rc = -EOPNOTSUPP;
+ else
+@@ -2220,6 +2395,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
+ sk->sk_err = smc->clcsock->sk->sk_err;
+ sk_error_report(sk);
+ }
++ mutex_unlock(&smc->clcsock_release_lock);
+
+ if (optlen < sizeof(int))
+ return -EINVAL;
+@@ -2236,7 +2412,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
+ case TCP_FASTOPEN_NO_COOKIE:
+ /* option not supported by SMC */
+ if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
+- smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
++ rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
+ } else {
+ rc = -EINVAL;
+ }
+@@ -2247,8 +2423,8 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
+ sk->sk_state != SMC_CLOSED) {
+ if (val) {
+ SMC_STAT_INC(smc, ndly_cnt);
+- mod_delayed_work(smc->conn.lgr->tx_wq,
+- &smc->conn.tx_work, 0);
++ smc_tx_pending(&smc->conn);
++ cancel_delayed_work(&smc->conn.tx_work);
+ }
+ }
+ break;
+@@ -2258,8 +2434,8 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
+ sk->sk_state != SMC_CLOSED) {
+ if (!val) {
+ SMC_STAT_INC(smc, cork_cnt);
+- mod_delayed_work(smc->conn.lgr->tx_wq,
+- &smc->conn.tx_work, 0);
++ smc_tx_pending(&smc->conn);
++ cancel_delayed_work(&smc->conn.tx_work);
+ }
+ }
+ break;
+@@ -2279,13 +2455,23 @@ static int smc_getsockopt(struct socket *sock, int level, int optname,
+ char __user *optval, int __user *optlen)
+ {
+ struct smc_sock *smc;
++ int rc;
+
+ smc = smc_sk(sock->sk);
++ mutex_lock(&smc->clcsock_release_lock);
++ if (!smc->clcsock) {
++ mutex_unlock(&smc->clcsock_release_lock);
++ return -EBADF;
++ }
+ /* socket options apply to the CLC socket */
+- if (unlikely(!smc->clcsock->ops->getsockopt))
++ if (unlikely(!smc->clcsock->ops->getsockopt)) {
++ mutex_unlock(&smc->clcsock_release_lock);
+ return -EOPNOTSUPP;
+- return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
+- optval, optlen);
++ }
++ rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
++ optval, optlen);
++ mutex_unlock(&smc->clcsock_release_lock);
++ return rc;
+ }
+
+ static int smc_ioctl(struct socket *sock, unsigned int cmd,
+diff --git a/net/smc/smc.h b/net/smc/smc.h
+index d65e15f0c944c..930544f7b2e2c 100644
+--- a/net/smc/smc.h
++++ b/net/smc/smc.h
+@@ -129,6 +129,12 @@ enum smc_urg_state {
+ SMC_URG_READ = 3, /* data was already read */
+ };
+
++struct smc_mark_woken {
++ bool woken;
++ void *key;
++ wait_queue_entry_t wait_entry;
++};
++
+ struct smc_connection {
+ struct rb_node alert_node;
+ struct smc_link_group *lgr; /* link group of connection */
+@@ -170,6 +176,11 @@ struct smc_connection {
+ u16 tx_cdc_seq; /* sequence # for CDC send */
+ u16 tx_cdc_seq_fin; /* sequence # - tx completed */
+ spinlock_t send_lock; /* protect wr_sends */
++ atomic_t cdc_pend_tx_wr; /* number of pending tx CDC wqe
++ * - inc when post wqe,
++ * - dec on polled tx cqe
++ */
++ wait_queue_head_t cdc_pend_tx_wq; /* wakeup on no cdc_pend_tx_wr*/
+ struct delayed_work tx_work; /* retry of smc_cdc_msg_send */
+ u32 tx_off; /* base offset in peer rmb */
+
+@@ -212,8 +223,14 @@ struct smc_connection {
+ struct smc_sock { /* smc sock container */
+ struct sock sk;
+ struct socket *clcsock; /* internal tcp socket */
++ void (*clcsk_state_change)(struct sock *sk);
++ /* original stat_change fct. */
+ void (*clcsk_data_ready)(struct sock *sk);
+- /* original data_ready fct. **/
++ /* original data_ready fct. */
++ void (*clcsk_write_space)(struct sock *sk);
++ /* original write_space fct. */
++ void (*clcsk_error_report)(struct sock *sk);
++ /* original error_report fct. */
+ struct smc_connection conn; /* smc connection */
+ struct smc_sock *listen_smc; /* listen parent */
+ struct work_struct connect_work; /* handle non-blocking connect*/
+@@ -248,6 +265,12 @@ static inline struct smc_sock *smc_sk(const struct sock *sk)
+ return (struct smc_sock *)sk;
+ }
+
++static inline struct smc_sock *smc_clcsock_user_data(struct sock *clcsk)
++{
++ return (struct smc_sock *)
++ ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY);
++}
++
+ extern struct workqueue_struct *smc_hs_wq; /* wq for handshake work */
+ extern struct workqueue_struct *smc_close_wq; /* wq for close work */
+
+diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c
+index 99acd337ba90d..84c8a4374fddd 100644
+--- a/net/smc/smc_cdc.c
++++ b/net/smc/smc_cdc.c
+@@ -31,10 +31,6 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
+ struct smc_sock *smc;
+ int diff;
+
+- if (!conn)
+- /* already dismissed */
+- return;
+-
+ smc = container_of(conn, struct smc_sock, conn);
+ bh_lock_sock(&smc->sk);
+ if (!wc_status) {
+@@ -51,6 +47,12 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
+ conn);
+ conn->tx_cdc_seq_fin = cdcpend->ctrl_seq;
+ }
++
++ if (atomic_dec_and_test(&conn->cdc_pend_tx_wr) &&
++ unlikely(wq_has_sleeper(&conn->cdc_pend_tx_wq)))
++ wake_up(&conn->cdc_pend_tx_wq);
++ WARN_ON(atomic_read(&conn->cdc_pend_tx_wr) < 0);
++
+ smc_tx_sndbuf_nonfull(smc);
+ bh_unlock_sock(&smc->sk);
+ }
+@@ -107,6 +109,10 @@ int smc_cdc_msg_send(struct smc_connection *conn,
+ conn->tx_cdc_seq++;
+ conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;
+ smc_host_msg_to_cdc((struct smc_cdc_msg *)wr_buf, conn, &cfed);
++
++ atomic_inc(&conn->cdc_pend_tx_wr);
++ smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */
++
+ rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend);
+ if (!rc) {
+ smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn);
+@@ -114,6 +120,7 @@ int smc_cdc_msg_send(struct smc_connection *conn,
+ } else {
+ conn->tx_cdc_seq--;
+ conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;
++ atomic_dec(&conn->cdc_pend_tx_wr);
+ }
+
+ return rc;
+@@ -136,7 +143,18 @@ int smcr_cdc_msg_send_validation(struct smc_connection *conn,
+ peer->token = htonl(local->token);
+ peer->prod_flags.failover_validation = 1;
+
++ /* We need to set pend->conn here to make sure smc_cdc_tx_handler()
++ * can handle properly
++ */
++ smc_cdc_add_pending_send(conn, pend);
++
++ atomic_inc(&conn->cdc_pend_tx_wr);
++ smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */
++
+ rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend);
++ if (unlikely(rc))
++ atomic_dec(&conn->cdc_pend_tx_wr);
++
+ return rc;
+ }
+
+@@ -193,31 +211,9 @@ int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn)
+ return rc;
+ }
+
+-static bool smc_cdc_tx_filter(struct smc_wr_tx_pend_priv *tx_pend,
+- unsigned long data)
++void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn)
+ {
+- struct smc_connection *conn = (struct smc_connection *)data;
+- struct smc_cdc_tx_pend *cdc_pend =
+- (struct smc_cdc_tx_pend *)tx_pend;
+-
+- return cdc_pend->conn == conn;
+-}
+-
+-static void smc_cdc_tx_dismisser(struct smc_wr_tx_pend_priv *tx_pend)
+-{
+- struct smc_cdc_tx_pend *cdc_pend =
+- (struct smc_cdc_tx_pend *)tx_pend;
+-
+- cdc_pend->conn = NULL;
+-}
+-
+-void smc_cdc_tx_dismiss_slots(struct smc_connection *conn)
+-{
+- struct smc_link *link = conn->lnk;
+-
+- smc_wr_tx_dismiss_slots(link, SMC_CDC_MSG_TYPE,
+- smc_cdc_tx_filter, smc_cdc_tx_dismisser,
+- (unsigned long)conn);
++ wait_event(conn->cdc_pend_tx_wq, !atomic_read(&conn->cdc_pend_tx_wr));
+ }
+
+ /* Send a SMC-D CDC header.
+diff --git a/net/smc/smc_cdc.h b/net/smc/smc_cdc.h
+index 0a0a89abd38b2..696cc11f2303b 100644
+--- a/net/smc/smc_cdc.h
++++ b/net/smc/smc_cdc.h
+@@ -291,7 +291,7 @@ int smc_cdc_get_free_slot(struct smc_connection *conn,
+ struct smc_wr_buf **wr_buf,
+ struct smc_rdma_wr **wr_rdma_buf,
+ struct smc_cdc_tx_pend **pend);
+-void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
++void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
+ int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
+ struct smc_cdc_tx_pend *pend);
+ int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
+diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c
+index 0f9ffba07d268..84102db5bb314 100644
+--- a/net/smc/smc_close.c
++++ b/net/smc/smc_close.c
+@@ -195,6 +195,7 @@ int smc_close_active(struct smc_sock *smc)
+ int old_state;
+ long timeout;
+ int rc = 0;
++ int rc1 = 0;
+
+ timeout = current->flags & PF_EXITING ?
+ 0 : sock_flag(sk, SOCK_LINGER) ?
+@@ -228,6 +229,15 @@ again:
+ /* send close request */
+ rc = smc_close_final(conn);
+ sk->sk_state = SMC_PEERCLOSEWAIT1;
++
++ /* actively shutdown clcsock before peer close it,
++ * prevent peer from entering TIME_WAIT state.
++ */
++ if (smc->clcsock && smc->clcsock->sk) {
++ rc1 = kernel_sock_shutdown(smc->clcsock,
++ SHUT_RDWR);
++ rc = rc ? rc : rc1;
++ }
+ } else {
+ /* peer event has changed the state */
+ goto again;
+diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
+index d2206743dc714..7401ec67ebcf9 100644
+--- a/net/smc/smc_core.c
++++ b/net/smc/smc_core.c
+@@ -170,8 +170,10 @@ static int smc_lgr_register_conn(struct smc_connection *conn, bool first)
+
+ if (!conn->lgr->is_smcd) {
+ rc = smcr_lgr_conn_assign_link(conn, first);
+- if (rc)
++ if (rc) {
++ conn->lgr = NULL;
+ return rc;
++ }
+ }
+ /* find a new alert_token_local value not yet used by some connection
+ * in this link group
+@@ -579,21 +581,18 @@ int smcd_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb)
+ return skb->len;
+ }
+
+-void smc_lgr_cleanup_early(struct smc_connection *conn)
++void smc_lgr_cleanup_early(struct smc_link_group *lgr)
+ {
+- struct smc_link_group *lgr = conn->lgr;
+- struct list_head *lgr_list;
+ spinlock_t *lgr_lock;
+
+ if (!lgr)
+ return;
+
+- smc_conn_free(conn);
+- lgr_list = smc_lgr_list_head(lgr, &lgr_lock);
++ smc_lgr_list_head(lgr, &lgr_lock);
+ spin_lock_bh(lgr_lock);
+ /* do not use this link group for new connections */
+- if (!list_empty(lgr_list))
+- list_del_init(lgr_list);
++ if (!list_empty(&lgr->list))
++ list_del_init(&lgr->list);
+ spin_unlock_bh(lgr_lock);
+ __smc_lgr_terminate(lgr, true);
+ }
+@@ -605,7 +604,7 @@ static void smcr_lgr_link_deactivate_all(struct smc_link_group *lgr)
+ for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
+ struct smc_link *lnk = &lgr->lnk[i];
+
+- if (smc_link_usable(lnk))
++ if (smc_link_sendable(lnk))
+ lnk->state = SMC_LNK_INACTIVE;
+ }
+ wake_up_all(&lgr->llc_msg_waiter);
+@@ -665,13 +664,14 @@ static u8 smcr_next_link_id(struct smc_link_group *lgr)
+ int i;
+
+ while (1) {
++again:
+ link_id = ++lgr->next_link_id;
+ if (!link_id) /* skip zero as link_id */
+ link_id = ++lgr->next_link_id;
+ for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
+ if (smc_link_usable(&lgr->lnk[i]) &&
+ lgr->lnk[i].link_id == link_id)
+- continue;
++ goto again;
+ }
+ break;
+ }
+@@ -1056,13 +1056,13 @@ void smc_conn_free(struct smc_connection *conn)
+ smc_ism_unset_conn(conn);
+ tasklet_kill(&conn->rx_tsklet);
+ } else {
+- smc_cdc_tx_dismiss_slots(conn);
++ smc_cdc_wait_pend_tx_wr(conn);
+ if (current_work() != &conn->abort_work)
+ cancel_work_sync(&conn->abort_work);
+ }
+ if (!list_empty(&lgr->list)) {
+- smc_lgr_unregister_conn(conn);
+ smc_buf_unuse(conn, lgr); /* allow buffer reuse */
++ smc_lgr_unregister_conn(conn);
+ }
+
+ if (!lgr->conns_num)
+@@ -1133,7 +1133,7 @@ void smcr_link_clear(struct smc_link *lnk, bool log)
+ smc_llc_link_clear(lnk, log);
+ smcr_buf_unmap_lgr(lnk);
+ smcr_rtoken_clear_link(lnk);
+- smc_ib_modify_qp_reset(lnk);
++ smc_ib_modify_qp_error(lnk);
+ smc_wr_free_link(lnk);
+ smc_ib_destroy_queue_pair(lnk);
+ smc_ib_dealloc_protection_domain(lnk);
+@@ -1264,7 +1264,7 @@ static void smc_conn_kill(struct smc_connection *conn, bool soft)
+ else
+ tasklet_unlock_wait(&conn->rx_tsklet);
+ } else {
+- smc_cdc_tx_dismiss_slots(conn);
++ smc_cdc_wait_pend_tx_wr(conn);
+ }
+ smc_lgr_unregister_conn(conn);
+ smc_close_active_abort(smc);
+@@ -1514,7 +1514,6 @@ static void smcr_link_down(struct smc_link *lnk)
+ if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))
+ return;
+
+- smc_ib_modify_qp_reset(lnk);
+ to_lnk = smc_switch_conns(lgr, lnk, true);
+ if (!to_lnk) { /* no backup link available */
+ smcr_link_clear(lnk, true);
+@@ -1595,14 +1594,26 @@ static void smc_link_down_work(struct work_struct *work)
+ mutex_unlock(&lgr->llc_conf_mutex);
+ }
+
+-/* Determine vlan of internal TCP socket.
+- * @vlan_id: address to store the determined vlan id into
+- */
++static int smc_vlan_by_tcpsk_walk(struct net_device *lower_dev,
++ struct netdev_nested_priv *priv)
++{
++ unsigned short *vlan_id = (unsigned short *)priv->data;
++
++ if (is_vlan_dev(lower_dev)) {
++ *vlan_id = vlan_dev_vlan_id(lower_dev);
++ return 1;
++ }
++
++ return 0;
++}
++
++/* Determine vlan of internal TCP socket. */
+ int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini)
+ {
+ struct dst_entry *dst = sk_dst_get(clcsock->sk);
++ struct netdev_nested_priv priv;
+ struct net_device *ndev;
+- int i, nest_lvl, rc = 0;
++ int rc = 0;
+
+ ini->vlan_id = 0;
+ if (!dst) {
+@@ -1620,20 +1631,9 @@ int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini)
+ goto out_rel;
+ }
+
++ priv.data = (void *)&ini->vlan_id;
+ rtnl_lock();
+- nest_lvl = ndev->lower_level;
+- for (i = 0; i < nest_lvl; i++) {
+- struct list_head *lower = &ndev->adj_list.lower;
+-
+- if (list_empty(lower))
+- break;
+- lower = lower->next;
+- ndev = (struct net_device *)netdev_lower_get_next(ndev, &lower);
+- if (is_vlan_dev(ndev)) {
+- ini->vlan_id = vlan_dev_vlan_id(ndev);
+- break;
+- }
+- }
++ netdev_walk_all_lower_dev(ndev, smc_vlan_by_tcpsk_walk, &priv);
+ rtnl_unlock();
+
+ out_rel:
+@@ -1701,7 +1701,8 @@ int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini)
+ (ini->smcd_version == SMC_V2 ||
+ lgr->vlan_id == ini->vlan_id) &&
+ (role == SMC_CLNT || ini->is_smcd ||
+- lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) {
++ (lgr->conns_num < SMC_RMBS_PER_LGR_MAX &&
++ !bitmap_full(lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX)))) {
+ /* link group found */
+ ini->first_contact_local = 0;
+ conn->lgr = lgr;
+@@ -1735,12 +1736,15 @@ create:
+ write_lock_bh(&lgr->conns_lock);
+ rc = smc_lgr_register_conn(conn, true);
+ write_unlock_bh(&lgr->conns_lock);
+- if (rc)
++ if (rc) {
++ smc_lgr_cleanup_early(lgr);
+ goto out;
++ }
+ }
+ conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
+ conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
+ conn->urg_state = SMC_URG_READ;
++ init_waitqueue_head(&conn->cdc_pend_tx_wq);
+ INIT_WORK(&smc->conn.abort_work, smc_conn_abort_work);
+ if (ini->is_smcd) {
+ conn->rx_off = sizeof(struct smcd_cdc_msg);
+@@ -1818,7 +1822,7 @@ static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize,
+ */
+ static inline int smc_rmb_wnd_update_limit(int rmbe_size)
+ {
+- return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
++ return max_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
+ }
+
+ /* map an rmb buf to a link */
+diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
+index c043ecdca5c44..9a0523f4c7ba6 100644
+--- a/net/smc/smc_core.h
++++ b/net/smc/smc_core.h
+@@ -366,6 +366,12 @@ static inline bool smc_link_usable(struct smc_link *lnk)
+ return true;
+ }
+
++static inline bool smc_link_sendable(struct smc_link *lnk)
++{
++ return smc_link_usable(lnk) &&
++ lnk->qp_attr.cur_qp_state == IB_QPS_RTS;
++}
++
+ static inline bool smc_link_active(struct smc_link *lnk)
+ {
+ return lnk->state == SMC_LNK_ACTIVE;
+@@ -413,7 +419,7 @@ static inline void smc_set_pci_values(struct pci_dev *pci_dev,
+ struct smc_sock;
+ struct smc_clc_msg_accept_confirm;
+
+-void smc_lgr_cleanup_early(struct smc_connection *conn);
++void smc_lgr_cleanup_early(struct smc_link_group *lgr);
+ void smc_lgr_terminate_sched(struct smc_link_group *lgr);
+ void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport);
+ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport);
+diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
+index a8845343d183e..f0ec1f1d50fac 100644
+--- a/net/smc/smc_ib.c
++++ b/net/smc/smc_ib.c
+@@ -101,12 +101,12 @@ int smc_ib_modify_qp_rts(struct smc_link *lnk)
+ IB_QP_MAX_QP_RD_ATOMIC);
+ }
+
+-int smc_ib_modify_qp_reset(struct smc_link *lnk)
++int smc_ib_modify_qp_error(struct smc_link *lnk)
+ {
+ struct ib_qp_attr qp_attr;
+
+ memset(&qp_attr, 0, sizeof(qp_attr));
+- qp_attr.qp_state = IB_QPS_RESET;
++ qp_attr.qp_state = IB_QPS_ERR;
+ return ib_modify_qp(lnk->roce_qp, &qp_attr, IB_QP_STATE);
+ }
+
+diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h
+index 3085f5180da79..6967c3d52b03e 100644
+--- a/net/smc/smc_ib.h
++++ b/net/smc/smc_ib.h
+@@ -79,6 +79,7 @@ int smc_ib_create_queue_pair(struct smc_link *lnk);
+ int smc_ib_ready_link(struct smc_link *lnk);
+ int smc_ib_modify_qp_rts(struct smc_link *lnk);
+ int smc_ib_modify_qp_reset(struct smc_link *lnk);
++int smc_ib_modify_qp_error(struct smc_link *lnk);
+ long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev);
+ int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags,
+ struct smc_buf_desc *buf_slot, u8 link_idx);
+diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
+index f1d323439a2af..ee1f0fdba0855 100644
+--- a/net/smc/smc_llc.c
++++ b/net/smc/smc_llc.c
+@@ -1358,7 +1358,7 @@ void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
+ delllc.reason = htonl(rsn);
+
+ for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
+- if (!smc_link_usable(&lgr->lnk[i]))
++ if (!smc_link_sendable(&lgr->lnk[i]))
+ continue;
+ if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
+ break;
+diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c
+index 4a964e9190b02..79ee0618d919b 100644
+--- a/net/smc/smc_pnet.c
++++ b/net/smc/smc_pnet.c
+@@ -112,7 +112,7 @@ static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name)
+ pnettable = &sn->pnettable;
+
+ /* remove table entry */
+- write_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist,
+ list) {
+ if (!pnet_name ||
+@@ -130,7 +130,7 @@ static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name)
+ rc = 0;
+ }
+ }
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+
+ /* if this is not the initial namespace, stop here */
+ if (net != &init_net)
+@@ -191,7 +191,7 @@ static int smc_pnet_add_by_ndev(struct net_device *ndev)
+ sn = net_generic(net, smc_net_id);
+ pnettable = &sn->pnettable;
+
+- write_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
+ if (pnetelem->type == SMC_PNET_ETH && !pnetelem->ndev &&
+ !strncmp(pnetelem->eth_name, ndev->name, IFNAMSIZ)) {
+@@ -205,7 +205,7 @@ static int smc_pnet_add_by_ndev(struct net_device *ndev)
+ break;
+ }
+ }
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ return rc;
+ }
+
+@@ -223,7 +223,7 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev)
+ sn = net_generic(net, smc_net_id);
+ pnettable = &sn->pnettable;
+
+- write_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
+ if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev == ndev) {
+ dev_put(pnetelem->ndev);
+@@ -236,7 +236,7 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev)
+ break;
+ }
+ }
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ return rc;
+ }
+
+@@ -310,8 +310,9 @@ static struct smc_ib_device *smc_pnet_find_ib(char *ib_name)
+ list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
+ if (!strncmp(ibdev->ibdev->name, ib_name,
+ sizeof(ibdev->ibdev->name)) ||
+- !strncmp(dev_name(ibdev->ibdev->dev.parent), ib_name,
+- IB_DEVICE_NAME_MAX - 1)) {
++ (ibdev->ibdev->dev.parent &&
++ !strncmp(dev_name(ibdev->ibdev->dev.parent), ib_name,
++ IB_DEVICE_NAME_MAX - 1))) {
+ goto out;
+ }
+ }
+@@ -371,7 +372,7 @@ static int smc_pnet_add_eth(struct smc_pnettable *pnettable, struct net *net,
+
+ rc = -EEXIST;
+ new_netdev = true;
+- write_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
+ if (tmp_pe->type == SMC_PNET_ETH &&
+ !strncmp(tmp_pe->eth_name, eth_name, IFNAMSIZ)) {
+@@ -381,9 +382,9 @@ static int smc_pnet_add_eth(struct smc_pnettable *pnettable, struct net *net,
+ }
+ if (new_netdev) {
+ list_add_tail(&new_pe->list, &pnettable->pnetlist);
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ } else {
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ kfree(new_pe);
+ goto out_put;
+ }
+@@ -444,7 +445,7 @@ static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
+ new_pe->ib_port = ib_port;
+
+ new_ibdev = true;
+- write_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
+ if (tmp_pe->type == SMC_PNET_IB &&
+ !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
+@@ -454,9 +455,9 @@ static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
+ }
+ if (new_ibdev) {
+ list_add_tail(&new_pe->list, &pnettable->pnetlist);
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ } else {
+- write_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ kfree(new_pe);
+ }
+ return (new_ibdev) ? 0 : -EEXIST;
+@@ -601,7 +602,7 @@ static int _smc_pnet_dump(struct net *net, struct sk_buff *skb, u32 portid,
+ pnettable = &sn->pnettable;
+
+ /* dump pnettable entries */
+- read_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
+ if (pnetid && !smc_pnet_match(pnetelem->pnet_name, pnetid))
+ continue;
+@@ -616,7 +617,7 @@ static int _smc_pnet_dump(struct net *net, struct sk_buff *skb, u32 portid,
+ break;
+ }
+ }
+- read_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ return idx;
+ }
+
+@@ -860,7 +861,7 @@ int smc_pnet_net_init(struct net *net)
+ struct smc_pnetids_ndev *pnetids_ndev = &sn->pnetids_ndev;
+
+ INIT_LIST_HEAD(&pnettable->pnetlist);
+- rwlock_init(&pnettable->lock);
++ mutex_init(&pnettable->lock);
+ INIT_LIST_HEAD(&pnetids_ndev->list);
+ rwlock_init(&pnetids_ndev->lock);
+
+@@ -940,7 +941,7 @@ static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *ndev,
+ sn = net_generic(net, smc_net_id);
+ pnettable = &sn->pnettable;
+
+- read_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
+ if (pnetelem->type == SMC_PNET_ETH && ndev == pnetelem->ndev) {
+ /* get pnetid of netdev device */
+@@ -949,7 +950,7 @@ static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *ndev,
+ break;
+ }
+ }
+- read_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+ return rc;
+ }
+
+@@ -1130,7 +1131,7 @@ int smc_pnetid_by_table_ib(struct smc_ib_device *smcibdev, u8 ib_port)
+ sn = net_generic(&init_net, smc_net_id);
+ pnettable = &sn->pnettable;
+
+- read_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
+ if (tmp_pe->type == SMC_PNET_IB &&
+ !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX) &&
+@@ -1140,7 +1141,7 @@ int smc_pnetid_by_table_ib(struct smc_ib_device *smcibdev, u8 ib_port)
+ break;
+ }
+ }
+- read_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+
+ return rc;
+ }
+@@ -1159,7 +1160,7 @@ int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
+ sn = net_generic(&init_net, smc_net_id);
+ pnettable = &sn->pnettable;
+
+- read_lock(&pnettable->lock);
++ mutex_lock(&pnettable->lock);
+ list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
+ if (tmp_pe->type == SMC_PNET_IB &&
+ !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
+@@ -1168,7 +1169,7 @@ int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
+ break;
+ }
+ }
+- read_unlock(&pnettable->lock);
++ mutex_unlock(&pnettable->lock);
+
+ return rc;
+ }
+diff --git a/net/smc/smc_pnet.h b/net/smc/smc_pnet.h
+index 14039272f7e42..80a88eea49491 100644
+--- a/net/smc/smc_pnet.h
++++ b/net/smc/smc_pnet.h
+@@ -29,7 +29,7 @@ struct smc_link_group;
+ * @pnetlist: List of PNETIDs
+ */
+ struct smc_pnettable {
+- rwlock_t lock;
++ struct mutex lock;
+ struct list_head pnetlist;
+ };
+
+diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
+index 170b733bc7367..45b0575520da4 100644
+--- a/net/smc/smc_rx.c
++++ b/net/smc/smc_rx.c
+@@ -354,12 +354,12 @@ int smc_rx_recvmsg(struct smc_sock *smc, struct msghdr *msg,
+ }
+ break;
+ }
++ if (!timeo)
++ return -EAGAIN;
+ if (signal_pending(current)) {
+ read_done = sock_intr_errno(timeo);
+ break;
+ }
+- if (!timeo)
+- return -EAGAIN;
+ }
+
+ if (!smc_rx_data_available(conn)) {
+diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c
+index 738a4a99c8279..31ee76131a79e 100644
+--- a/net/smc/smc_tx.c
++++ b/net/smc/smc_tx.c
+@@ -594,27 +594,32 @@ int smc_tx_sndbuf_nonempty(struct smc_connection *conn)
+ return rc;
+ }
+
+-/* Wakeup sndbuf consumers from process context
+- * since there is more data to transmit
+- */
+-void smc_tx_work(struct work_struct *work)
++void smc_tx_pending(struct smc_connection *conn)
+ {
+- struct smc_connection *conn = container_of(to_delayed_work(work),
+- struct smc_connection,
+- tx_work);
+ struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
+ int rc;
+
+- lock_sock(&smc->sk);
+ if (smc->sk.sk_err)
+- goto out;
++ return;
+
+ rc = smc_tx_sndbuf_nonempty(conn);
+ if (!rc && conn->local_rx_ctrl.prod_flags.write_blocked &&
+ !atomic_read(&conn->bytes_to_rcv))
+ conn->local_rx_ctrl.prod_flags.write_blocked = 0;
++}
++
++/* Wakeup sndbuf consumers from process context
++ * since there is more data to transmit
++ */
++void smc_tx_work(struct work_struct *work)
++{
++ struct smc_connection *conn = container_of(to_delayed_work(work),
++ struct smc_connection,
++ tx_work);
++ struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
+
+-out:
++ lock_sock(&smc->sk);
++ smc_tx_pending(conn);
+ release_sock(&smc->sk);
+ }
+
+diff --git a/net/smc/smc_tx.h b/net/smc/smc_tx.h
+index 07e6ad76224a0..a59f370b8b432 100644
+--- a/net/smc/smc_tx.h
++++ b/net/smc/smc_tx.h
+@@ -27,6 +27,7 @@ static inline int smc_tx_prepared_sends(struct smc_connection *conn)
+ return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
+ }
+
++void smc_tx_pending(struct smc_connection *conn);
+ void smc_tx_work(struct work_struct *work);
+ void smc_tx_init(struct smc_sock *smc);
+ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len);
+diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c
+index a419e9af36b98..59ca1a2d5c650 100644
+--- a/net/smc/smc_wr.c
++++ b/net/smc/smc_wr.c
+@@ -62,13 +62,9 @@ static inline bool smc_wr_is_tx_pend(struct smc_link *link)
+ }
+
+ /* wait till all pending tx work requests on the given link are completed */
+-int smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
++void smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
+ {
+- if (wait_event_timeout(link->wr_tx_wait, !smc_wr_is_tx_pend(link),
+- SMC_WR_TX_WAIT_PENDING_TIME))
+- return 0;
+- else /* timeout */
+- return -EPIPE;
++ wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link));
+ }
+
+ static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
+@@ -87,7 +83,6 @@ static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
+ struct smc_wr_tx_pend pnd_snd;
+ struct smc_link *link;
+ u32 pnd_snd_idx;
+- int i;
+
+ link = wc->qp->qp_context;
+
+@@ -115,14 +110,6 @@ static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
+ if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
+ return;
+ if (wc->status) {
+- for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
+- /* clear full struct smc_wr_tx_pend including .priv */
+- memset(&link->wr_tx_pends[i], 0,
+- sizeof(link->wr_tx_pends[i]));
+- memset(&link->wr_tx_bufs[i], 0,
+- sizeof(link->wr_tx_bufs[i]));
+- clear_bit(i, link->wr_tx_mask);
+- }
+ /* terminate link */
+ smcr_link_down_cond_sched(link);
+ }
+@@ -169,7 +156,7 @@ void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
+ static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
+ {
+ *idx = link->wr_tx_cnt;
+- if (!smc_link_usable(link))
++ if (!smc_link_sendable(link))
+ return -ENOLINK;
+ for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
+ if (!test_and_set_bit(*idx, link->wr_tx_mask))
+@@ -212,7 +199,7 @@ int smc_wr_tx_get_free_slot(struct smc_link *link,
+ } else {
+ rc = wait_event_interruptible_timeout(
+ link->wr_tx_wait,
+- !smc_link_usable(link) ||
++ !smc_link_sendable(link) ||
+ lgr->terminating ||
+ (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
+ SMC_WR_TX_WAIT_FREE_SLOT_TIME);
+@@ -288,18 +275,20 @@ int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
+ unsigned long timeout)
+ {
+ struct smc_wr_tx_pend *pend;
++ u32 pnd_idx;
+ int rc;
+
+ pend = container_of(priv, struct smc_wr_tx_pend, priv);
+ pend->compl_requested = 1;
+- init_completion(&link->wr_tx_compl[pend->idx]);
++ pnd_idx = pend->idx;
++ init_completion(&link->wr_tx_compl[pnd_idx]);
+
+ rc = smc_wr_tx_send(link, priv);
+ if (rc)
+ return rc;
+ /* wait for completion by smc_wr_tx_process_cqe() */
+ rc = wait_for_completion_interruptible_timeout(
+- &link->wr_tx_compl[pend->idx], timeout);
++ &link->wr_tx_compl[pnd_idx], timeout);
+ if (rc <= 0)
+ rc = -ENODATA;
+ if (rc > 0)
+@@ -349,25 +338,6 @@ int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
+ return rc;
+ }
+
+-void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
+- smc_wr_tx_filter filter,
+- smc_wr_tx_dismisser dismisser,
+- unsigned long data)
+-{
+- struct smc_wr_tx_pend_priv *tx_pend;
+- struct smc_wr_rx_hdr *wr_tx;
+- int i;
+-
+- for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
+- wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
+- if (wr_tx->type != wr_tx_hdr_type)
+- continue;
+- tx_pend = &link->wr_tx_pends[i].priv;
+- if (filter(tx_pend, data))
+- dismisser(tx_pend);
+- }
+-}
+-
+ /****************************** receive queue ********************************/
+
+ int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
+@@ -572,10 +542,7 @@ void smc_wr_free_link(struct smc_link *lnk)
+ smc_wr_wakeup_reg_wait(lnk);
+ smc_wr_wakeup_tx_wait(lnk);
+
+- if (smc_wr_tx_wait_no_pending_sends(lnk))
+- memset(lnk->wr_tx_mask, 0,
+- BITS_TO_LONGS(SMC_WR_BUF_CNT) *
+- sizeof(*lnk->wr_tx_mask));
++ smc_wr_tx_wait_no_pending_sends(lnk);
+ wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt)));
+ wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt)));
+
+diff --git a/net/smc/smc_wr.h b/net/smc/smc_wr.h
+index 2bc626f230a56..cb58e60078f57 100644
+--- a/net/smc/smc_wr.h
++++ b/net/smc/smc_wr.h
+@@ -22,7 +22,6 @@
+ #define SMC_WR_BUF_CNT 16 /* # of ctrl buffers per link */
+
+ #define SMC_WR_TX_WAIT_FREE_SLOT_TIME (10 * HZ)
+-#define SMC_WR_TX_WAIT_PENDING_TIME (5 * HZ)
+
+ #define SMC_WR_TX_SIZE 44 /* actual size of wr_send data (<=SMC_WR_BUF_SIZE) */
+
+@@ -62,7 +61,7 @@ static inline void smc_wr_tx_set_wr_id(atomic_long_t *wr_tx_id, long val)
+
+ static inline bool smc_wr_tx_link_hold(struct smc_link *link)
+ {
+- if (!smc_link_usable(link))
++ if (!smc_link_sendable(link))
+ return false;
+ atomic_inc(&link->wr_tx_refcnt);
+ return true;
+@@ -122,7 +121,7 @@ void smc_wr_tx_dismiss_slots(struct smc_link *lnk, u8 wr_rx_hdr_type,
+ smc_wr_tx_filter filter,
+ smc_wr_tx_dismisser dismisser,
+ unsigned long data);
+-int smc_wr_tx_wait_no_pending_sends(struct smc_link *link);
++void smc_wr_tx_wait_no_pending_sends(struct smc_link *link);
+
+ int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler);
+ int smc_wr_rx_post_init(struct smc_link *link);
+diff --git a/net/socket.c b/net/socket.c
+index 7f64a6eccf63f..5053eb0100e48 100644
+--- a/net/socket.c
++++ b/net/socket.c
+@@ -829,6 +829,7 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
+ int empty = 1, false_tstamp = 0;
+ struct skb_shared_hwtstamps *shhwtstamps =
+ skb_hwtstamps(skb);
++ ktime_t hwtstamp;
+
+ /* Race occurred between timestamp enabling and packet
+ receiving. Fill in the current time for now. */
+@@ -877,10 +878,12 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
+ (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
+ !skb_is_swtx_tstamp(skb, false_tstamp)) {
+ if (sk->sk_tsflags & SOF_TIMESTAMPING_BIND_PHC)
+- ptp_convert_timestamp(shhwtstamps, sk->sk_bind_phc);
++ hwtstamp = ptp_convert_timestamp(shhwtstamps,
++ sk->sk_bind_phc);
++ else
++ hwtstamp = shhwtstamps->hwtstamp;
+
+- if (ktime_to_timespec64_cond(shhwtstamps->hwtstamp,
+- tss.ts + 2)) {
++ if (ktime_to_timespec64_cond(hwtstamp, tss.ts + 2)) {
+ empty = 0;
+
+ if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_PKTINFO) &&
+diff --git a/net/strparser/strparser.c b/net/strparser/strparser.c
+index 9c0343568d2a0..1a72c67afed5e 100644
+--- a/net/strparser/strparser.c
++++ b/net/strparser/strparser.c
+@@ -27,18 +27,10 @@
+
+ static struct workqueue_struct *strp_wq;
+
+-struct _strp_msg {
+- /* Internal cb structure. struct strp_msg must be first for passing
+- * to upper layer.
+- */
+- struct strp_msg strp;
+- int accum_len;
+-};
+-
+ static inline struct _strp_msg *_strp_msg(struct sk_buff *skb)
+ {
+ return (struct _strp_msg *)((void *)skb->cb +
+- offsetof(struct qdisc_skb_cb, data));
++ offsetof(struct sk_skb_cb, strp));
+ }
+
+ /* Lower lock held */
+diff --git a/net/sunrpc/addr.c b/net/sunrpc/addr.c
+index 6e4dbd577a39f..d435bffc61999 100644
+--- a/net/sunrpc/addr.c
++++ b/net/sunrpc/addr.c
+@@ -162,8 +162,10 @@ static int rpc_parse_scope_id(struct net *net, const char *buf,
+ const size_t buflen, const char *delim,
+ struct sockaddr_in6 *sin6)
+ {
+- char *p;
++ char p[IPV6_SCOPE_ID_LEN + 1];
+ size_t len;
++ u32 scope_id = 0;
++ struct net_device *dev;
+
+ if ((buf + buflen) == delim)
+ return 1;
+@@ -175,29 +177,23 @@ static int rpc_parse_scope_id(struct net *net, const char *buf,
+ return 0;
+
+ len = (buf + buflen) - delim - 1;
+- p = kmemdup_nul(delim + 1, len, GFP_KERNEL);
+- if (p) {
+- u32 scope_id = 0;
+- struct net_device *dev;
+-
+- dev = dev_get_by_name(net, p);
+- if (dev != NULL) {
+- scope_id = dev->ifindex;
+- dev_put(dev);
+- } else {
+- if (kstrtou32(p, 10, &scope_id) != 0) {
+- kfree(p);
+- return 0;
+- }
+- }
+-
+- kfree(p);
+-
+- sin6->sin6_scope_id = scope_id;
+- return 1;
++ if (len > IPV6_SCOPE_ID_LEN)
++ return 0;
++
++ memcpy(p, delim + 1, len);
++ p[len] = 0;
++
++ dev = dev_get_by_name(net, p);
++ if (dev != NULL) {
++ scope_id = dev->ifindex;
++ dev_put(dev);
++ } else {
++ if (kstrtou32(p, 10, &scope_id) != 0)
++ return 0;
+ }
+
+- return 0;
++ sin6->sin6_scope_id = scope_id;
++ return 1;
+ }
+
+ static size_t rpc_pton6(struct net *net, const char *buf, const size_t buflen,
+diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c
+index 61c276bddaf25..f549e4c05defc 100644
+--- a/net/sunrpc/auth_gss/gss_rpc_upcall.c
++++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c
+@@ -98,6 +98,7 @@ static int gssp_rpc_create(struct net *net, struct rpc_clnt **_clnt)
+ * done without the correct namespace:
+ */
+ .flags = RPC_CLNT_CREATE_NOPING |
++ RPC_CLNT_CREATE_CONNECTED |
+ RPC_CLNT_CREATE_NO_IDLE_TIMEOUT
+ };
+ struct rpc_clnt *clnt;
+diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
+index f056ff9314442..0a0818e55879a 100644
+--- a/net/sunrpc/clnt.c
++++ b/net/sunrpc/clnt.c
+@@ -76,6 +76,7 @@ static int rpc_encode_header(struct rpc_task *task,
+ static int rpc_decode_header(struct rpc_task *task,
+ struct xdr_stream *xdr);
+ static int rpc_ping(struct rpc_clnt *clnt);
++static int rpc_ping_noreply(struct rpc_clnt *clnt);
+ static void rpc_check_timeout(struct rpc_task *task);
+
+ static void rpc_register_client(struct rpc_clnt *clnt)
+@@ -483,6 +484,12 @@ static struct rpc_clnt *rpc_create_xprt(struct rpc_create_args *args,
+ rpc_shutdown_client(clnt);
+ return ERR_PTR(err);
+ }
++ } else if (args->flags & RPC_CLNT_CREATE_CONNECTED) {
++ int err = rpc_ping_noreply(clnt);
++ if (err != 0) {
++ rpc_shutdown_client(clnt);
++ return ERR_PTR(err);
++ }
+ }
+
+ clnt->cl_softrtry = 1;
+@@ -1065,8 +1072,13 @@ rpc_task_get_next_xprt(struct rpc_clnt *clnt)
+ static
+ void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt)
+ {
+- if (task->tk_xprt)
+- return;
++ if (task->tk_xprt) {
++ if (!(test_bit(XPRT_OFFLINE, &task->tk_xprt->state) &&
++ (task->tk_flags & RPC_TASK_MOVEABLE)))
++ return;
++ xprt_release(task);
++ xprt_put(task->tk_xprt);
++ }
+ if (task->tk_flags & RPC_TASK_NO_ROUND_ROBIN)
+ task->tk_xprt = rpc_task_get_first_xprt(clnt);
+ else
+@@ -2200,6 +2212,7 @@ call_transmit_status(struct rpc_task *task)
+ * socket just returned a connection error,
+ * then hold onto the transport lock.
+ */
++ case -ENOMEM:
+ case -ENOBUFS:
+ rpc_delay(task, HZ>>2);
+ fallthrough;
+@@ -2283,6 +2296,7 @@ call_bc_transmit_status(struct rpc_task *task)
+ case -ENOTCONN:
+ case -EPIPE:
+ break;
++ case -ENOMEM:
+ case -ENOBUFS:
+ rpc_delay(task, HZ>>2);
+ fallthrough;
+@@ -2365,6 +2379,11 @@ call_status(struct rpc_task *task)
+ case -EPIPE:
+ case -EAGAIN:
+ break;
++ case -ENFILE:
++ case -ENOBUFS:
++ case -ENOMEM:
++ rpc_delay(task, HZ>>2);
++ break;
+ case -EIO:
+ /* shutdown or soft timeout */
+ goto out_exit;
+@@ -2692,6 +2711,10 @@ static const struct rpc_procinfo rpcproc_null = {
+ .p_decode = rpcproc_decode_null,
+ };
+
++static const struct rpc_procinfo rpcproc_null_noreply = {
++ .p_encode = rpcproc_encode_null,
++};
++
+ static void
+ rpc_null_call_prepare(struct rpc_task *task, void *data)
+ {
+@@ -2745,6 +2768,28 @@ static int rpc_ping(struct rpc_clnt *clnt)
+ return status;
+ }
+
++static int rpc_ping_noreply(struct rpc_clnt *clnt)
++{
++ struct rpc_message msg = {
++ .rpc_proc = &rpcproc_null_noreply,
++ };
++ struct rpc_task_setup task_setup_data = {
++ .rpc_client = clnt,
++ .rpc_message = &msg,
++ .callback_ops = &rpc_null_ops,
++ .flags = RPC_TASK_SOFT | RPC_TASK_SOFTCONN | RPC_TASK_NULLCREDS,
++ };
++ struct rpc_task *task;
++ int status;
++
++ task = rpc_run_task(&task_setup_data);
++ if (IS_ERR(task))
++ return PTR_ERR(task);
++ status = task->tk_status;
++ rpc_put_task(task);
++ return status;
++}
++
+ struct rpc_cb_add_xprt_calldata {
+ struct rpc_xprt_switch *xps;
+ struct rpc_xprt *xprt;
+@@ -2903,7 +2948,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
+ unsigned long connect_timeout;
+ unsigned long reconnect_timeout;
+ unsigned char resvport, reuseport;
+- int ret = 0;
++ int ret = 0, ident;
+
+ rcu_read_lock();
+ xps = xprt_switch_get(rcu_dereference(clnt->cl_xpi.xpi_xpswitch));
+@@ -2917,8 +2962,11 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
+ reuseport = xprt->reuseport;
+ connect_timeout = xprt->connect_timeout;
+ reconnect_timeout = xprt->max_reconnect_timeout;
++ ident = xprt->xprt_class->ident;
+ rcu_read_unlock();
+
++ if (!xprtargs->ident)
++ xprtargs->ident = ident;
+ xprt = xprt_create_transport(xprtargs);
+ if (IS_ERR(xprt)) {
+ ret = PTR_ERR(xprt);
+diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c
+index ee5336d73fddc..35588f0afa864 100644
+--- a/net/sunrpc/rpc_pipe.c
++++ b/net/sunrpc/rpc_pipe.c
+@@ -600,9 +600,9 @@ static int __rpc_rmdir(struct inode *dir, struct dentry *dentry)
+
+ dget(dentry);
+ ret = simple_rmdir(dir, dentry);
++ d_drop(dentry);
+ if (!ret)
+ fsnotify_rmdir(dir, dentry);
+- d_delete(dentry);
+ dput(dentry);
+ return ret;
+ }
+@@ -613,9 +613,9 @@ static int __rpc_unlink(struct inode *dir, struct dentry *dentry)
+
+ dget(dentry);
+ ret = simple_unlink(dir, dentry);
++ d_drop(dentry);
+ if (!ret)
+ fsnotify_unlink(dir, dentry);
+- d_delete(dentry);
+ dput(dentry);
+ return ret;
+ }
+diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
+index c045f63d11fa6..f0f55fbd13752 100644
+--- a/net/sunrpc/sched.c
++++ b/net/sunrpc/sched.c
+@@ -186,11 +186,6 @@ static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,
+
+ /*
+ * Add new request to wait queue.
+- *
+- * Swapper tasks always get inserted at the head of the queue.
+- * This should avoid many nasty memory deadlocks and hopefully
+- * improve overall performance.
+- * Everyone else gets appended to the queue to ensure proper FIFO behavior.
+ */
+ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
+ struct rpc_task *task,
+@@ -199,8 +194,6 @@ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
+ INIT_LIST_HEAD(&task->u.tk_wait.timer_list);
+ if (RPC_IS_PRIORITY(queue))
+ __rpc_add_wait_queue_priority(queue, task, queue_priority);
+- else if (RPC_IS_SWAPPER(task))
+- list_add(&task->u.tk_wait.list, &queue->tasks[0]);
+ else
+ list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
+ task->tk_waitqueue = queue;
+@@ -1012,8 +1005,10 @@ int rpc_malloc(struct rpc_task *task)
+ struct rpc_buffer *buf;
+ gfp_t gfp = GFP_NOFS;
+
++ if (RPC_IS_ASYNC(task))
++ gfp = GFP_NOWAIT | __GFP_NOWARN;
+ if (RPC_IS_SWAPPER(task))
+- gfp = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
++ gfp |= __GFP_MEMALLOC;
+
+ size += sizeof(struct rpc_buffer);
+ if (size <= RPC_BUFFER_MAXSIZE)
+diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
+index a3bbe5ce4570f..08ca797bb8a46 100644
+--- a/net/sunrpc/svc.c
++++ b/net/sunrpc/svc.c
+@@ -1676,16 +1676,17 @@ EXPORT_SYMBOL_GPL(svc_encode_result_payload);
+ /**
+ * svc_fill_write_vector - Construct data argument for VFS write call
+ * @rqstp: svc_rqst to operate on
+- * @pages: list of pages containing data payload
+- * @first: buffer containing first section of write payload
+- * @total: total number of bytes of write payload
++ * @payload: xdr_buf containing only the write data payload
+ *
+ * Fills in rqstp::rq_vec, and returns the number of elements.
+ */
+-unsigned int svc_fill_write_vector(struct svc_rqst *rqstp, struct page **pages,
+- struct kvec *first, size_t total)
++unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
++ struct xdr_buf *payload)
+ {
++ struct page **pages = payload->pages;
++ struct kvec *first = payload->head;
+ struct kvec *vec = rqstp->rq_vec;
++ size_t total = payload->len;
+ unsigned int i;
+
+ /* Some types of transport can present the write payload
+diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
+index 6316bd2b8f37b..935bba065636c 100644
+--- a/net/sunrpc/svc_xprt.c
++++ b/net/sunrpc/svc_xprt.c
+@@ -243,7 +243,7 @@ static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl,
+ xprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags);
+ if (IS_ERR(xprt))
+ trace_svc_xprt_create_err(serv->sv_program->pg_name,
+- xcl->xcl_name, sap, xprt);
++ xcl->xcl_name, sap, len, xprt);
+ return xprt;
+ }
+
+@@ -1213,6 +1213,8 @@ static struct cache_deferred_req *svc_defer(struct cache_req *req)
+ dr->daddr = rqstp->rq_daddr;
+ dr->argslen = rqstp->rq_arg.len >> 2;
+ dr->xprt_hlen = rqstp->rq_xprt_hlen;
++ dr->xprt_ctxt = rqstp->rq_xprt_ctxt;
++ rqstp->rq_xprt_ctxt = NULL;
+
+ /* back up head to the start of the buffer and copy */
+ skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;
+@@ -1251,6 +1253,7 @@ static noinline int svc_deferred_recv(struct svc_rqst *rqstp)
+ rqstp->rq_xprt_hlen = dr->xprt_hlen;
+ rqstp->rq_daddr = dr->daddr;
+ rqstp->rq_respages = rqstp->rq_pages;
++ rqstp->rq_xprt_ctxt = dr->xprt_ctxt;
+ svc_xprt_received(rqstp->rq_xprt);
+ return (dr->argslen<<2) - dr->xprt_hlen;
+ }
+diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
+index 478f857cdaed4..6ea3d87e11475 100644
+--- a/net/sunrpc/svcsock.c
++++ b/net/sunrpc/svcsock.c
+@@ -1096,7 +1096,9 @@ static int svc_tcp_sendmsg(struct socket *sock, struct xdr_buf *xdr,
+ int ret;
+
+ *sentp = 0;
+- xdr_alloc_bvec(xdr, GFP_KERNEL);
++ ret = xdr_alloc_bvec(xdr, GFP_KERNEL);
++ if (ret < 0)
++ return ret;
+
+ ret = kernel_sendmsg(sock, &msg, &rm, 1, rm.iov_len);
+ if (ret < 0)
+diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c
+index 9a6f17e18f73b..326a31422a3c1 100644
+--- a/net/sunrpc/sysfs.c
++++ b/net/sunrpc/sysfs.c
+@@ -291,8 +291,10 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj,
+ online = 1;
+ else if (!strncmp(buf, "remove", 6))
+ remove = 1;
+- else
+- return -EINVAL;
++ else {
++ count = -EINVAL;
++ goto out_put;
++ }
+
+ if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) {
+ count = -EINTR;
+@@ -303,25 +305,28 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj,
+ goto release_tasks;
+ }
+ if (offline) {
+- set_bit(XPRT_OFFLINE, &xprt->state);
+- spin_lock(&xps->xps_lock);
+- xps->xps_nactive--;
+- spin_unlock(&xps->xps_lock);
++ if (!test_and_set_bit(XPRT_OFFLINE, &xprt->state)) {
++ spin_lock(&xps->xps_lock);
++ xps->xps_nactive--;
++ spin_unlock(&xps->xps_lock);
++ }
+ } else if (online) {
+- clear_bit(XPRT_OFFLINE, &xprt->state);
+- spin_lock(&xps->xps_lock);
+- xps->xps_nactive++;
+- spin_unlock(&xps->xps_lock);
++ if (test_and_clear_bit(XPRT_OFFLINE, &xprt->state)) {
++ spin_lock(&xps->xps_lock);
++ xps->xps_nactive++;
++ spin_unlock(&xps->xps_lock);
++ }
+ } else if (remove) {
+ if (test_bit(XPRT_OFFLINE, &xprt->state)) {
+- set_bit(XPRT_REMOVE, &xprt->state);
+- xprt_force_disconnect(xprt);
+- if (test_bit(XPRT_CONNECTED, &xprt->state)) {
+- if (!xprt->sending.qlen &&
+- !xprt->pending.qlen &&
+- !xprt->backlog.qlen &&
+- !atomic_long_read(&xprt->queuelen))
+- rpc_xprt_switch_remove_xprt(xps, xprt);
++ if (!test_and_set_bit(XPRT_REMOVE, &xprt->state)) {
++ xprt_force_disconnect(xprt);
++ if (test_bit(XPRT_CONNECTED, &xprt->state)) {
++ if (!xprt->sending.qlen &&
++ !xprt->pending.qlen &&
++ !xprt->backlog.qlen &&
++ !atomic_long_read(&xprt->queuelen))
++ rpc_xprt_switch_remove_xprt(xps, xprt);
++ }
+ }
+ } else {
+ count = -EINVAL;
+diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
+index cfd681700d1a1..2db834318d141 100644
+--- a/net/sunrpc/xprt.c
++++ b/net/sunrpc/xprt.c
+@@ -767,7 +767,8 @@ EXPORT_SYMBOL_GPL(xprt_disconnect_done);
+ */
+ static void xprt_schedule_autoclose_locked(struct rpc_xprt *xprt)
+ {
+- set_bit(XPRT_CLOSE_WAIT, &xprt->state);
++ if (test_and_set_bit(XPRT_CLOSE_WAIT, &xprt->state))
++ return;
+ if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
+ queue_work(xprtiod_workqueue, &xprt->task_cleanup);
+ else if (xprt->snd_task && !test_bit(XPRT_SND_IS_COOKIE, &xprt->state))
+@@ -928,12 +929,7 @@ void xprt_connect(struct rpc_task *task)
+ if (!xprt_lock_write(xprt, task))
+ return;
+
+- if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
+- trace_xprt_disconnect_cleanup(xprt);
+- xprt->ops->close(xprt);
+- }
+-
+- if (!xprt_connected(xprt)) {
++ if (!xprt_connected(xprt) && !test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
+ task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;
+ rpc_sleep_on_timeout(&xprt->pending, task, NULL,
+ xprt_request_timeout(task->tk_rqstp));
+@@ -1353,17 +1349,6 @@ xprt_request_enqueue_transmit(struct rpc_task *task)
+ INIT_LIST_HEAD(&req->rq_xmit2);
+ goto out;
+ }
+- } else if (RPC_IS_SWAPPER(task)) {
+- list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) {
+- if (pos->rq_cong || pos->rq_bytes_sent)
+- continue;
+- if (RPC_IS_SWAPPER(pos->rq_task))
+- continue;
+- /* Note: req is added _before_ pos */
+- list_add_tail(&req->rq_xmit, &pos->rq_xmit);
+- INIT_LIST_HEAD(&req->rq_xmit2);
+- goto out;
+- }
+ } else if (!req->rq_seqno) {
+ list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) {
+ if (pos->rq_task->tk_owner != task->tk_owner)
+@@ -1603,15 +1588,14 @@ xprt_transmit(struct rpc_task *task)
+ {
+ struct rpc_rqst *next, *req = task->tk_rqstp;
+ struct rpc_xprt *xprt = req->rq_xprt;
+- int counter, status;
++ int status;
+
+ spin_lock(&xprt->queue_lock);
+- counter = 0;
+- while (!list_empty(&xprt->xmit_queue)) {
+- if (++counter == 20)
++ for (;;) {
++ next = list_first_entry_or_null(&xprt->xmit_queue,
++ struct rpc_rqst, rq_xmit);
++ if (!next)
+ break;
+- next = list_first_entry(&xprt->xmit_queue,
+- struct rpc_rqst, rq_xmit);
+ xprt_pin_rqst(next);
+ spin_unlock(&xprt->queue_lock);
+ status = xprt_request_transmit(next, task);
+@@ -1619,13 +1603,16 @@ xprt_transmit(struct rpc_task *task)
+ status = 0;
+ spin_lock(&xprt->queue_lock);
+ xprt_unpin_rqst(next);
+- if (status == 0) {
+- if (!xprt_request_data_received(task) ||
+- test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
+- continue;
+- } else if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
+- task->tk_status = status;
+- break;
++ if (status < 0) {
++ if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
++ task->tk_status = status;
++ break;
++ }
++ /* Was @task transmitted, and has it received a reply? */
++ if (xprt_request_data_received(task) &&
++ !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
++ break;
++ cond_resched_lock(&xprt->queue_lock);
+ }
+ spin_unlock(&xprt->queue_lock);
+ }
+@@ -1684,12 +1671,15 @@ out:
+ static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt)
+ {
+ struct rpc_rqst *req = ERR_PTR(-EAGAIN);
++ gfp_t gfp_mask = GFP_KERNEL;
+
+ if (xprt->num_reqs >= xprt->max_reqs)
+ goto out;
+ ++xprt->num_reqs;
+ spin_unlock(&xprt->reserve_lock);
+- req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS);
++ if (current->flags & PF_WQ_WORKER)
++ gfp_mask |= __GFP_NORETRY | __GFP_NOWARN;
++ req = kzalloc(sizeof(*req), gfp_mask);
+ spin_lock(&xprt->reserve_lock);
+ if (req != NULL)
+ goto out;
+@@ -2109,7 +2099,14 @@ static void xprt_destroy(struct rpc_xprt *xprt)
+ */
+ wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
+
++ /*
++ * xprt_schedule_autodisconnect() can run after XPRT_LOCKED
++ * is cleared. We use ->transport_lock to ensure the mod_timer()
++ * can only run *before* del_time_sync(), never after.
++ */
++ spin_lock(&xprt->transport_lock);
+ del_timer_sync(&xprt->timer);
++ spin_unlock(&xprt->transport_lock);
+
+ /*
+ * Destroy sockets etc from the system workqueue so they can
+diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+index 6be23ce7a93d2..387a5da09dafb 100644
+--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
++++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+@@ -826,7 +826,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
+ goto out_err;
+ if (ret == 0)
+ goto out_drop;
+- rqstp->rq_xprt_hlen = ret;
++ rqstp->rq_xprt_hlen = 0;
+
+ if (svc_rdma_is_reverse_direction_reply(xprt, ctxt))
+ goto out_backchannel;
+diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
+index 16e5696314a4f..32df237967472 100644
+--- a/net/sunrpc/xprtrdma/transport.c
++++ b/net/sunrpc/xprtrdma/transport.c
+@@ -521,7 +521,7 @@ xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
+ return;
+
+ out_sleep:
+- task->tk_status = -EAGAIN;
++ task->tk_status = -ENOMEM;
+ xprt_add_backlog(xprt, task);
+ }
+
+@@ -574,8 +574,10 @@ xprt_rdma_allocate(struct rpc_task *task)
+ gfp_t flags;
+
+ flags = RPCRDMA_DEF_GFP;
++ if (RPC_IS_ASYNC(task))
++ flags = GFP_NOWAIT | __GFP_NOWARN;
+ if (RPC_IS_SWAPPER(task))
+- flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
++ flags |= __GFP_MEMALLOC;
+
+ if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize,
+ flags))
+diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
+index aaec3c9be8db6..1295f9ab839fd 100644
+--- a/net/sunrpc/xprtrdma/verbs.c
++++ b/net/sunrpc/xprtrdma/verbs.c
+@@ -438,6 +438,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
+ IB_POLL_WORKQUEUE);
+ if (IS_ERR(ep->re_attr.send_cq)) {
+ rc = PTR_ERR(ep->re_attr.send_cq);
++ ep->re_attr.send_cq = NULL;
+ goto out_destroy;
+ }
+
+@@ -446,6 +447,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
+ IB_POLL_WORKQUEUE);
+ if (IS_ERR(ep->re_attr.recv_cq)) {
+ rc = PTR_ERR(ep->re_attr.recv_cq);
++ ep->re_attr.recv_cq = NULL;
+ goto out_destroy;
+ }
+ ep->re_receive_count = 0;
+@@ -484,6 +486,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
+ ep->re_pd = ib_alloc_pd(device, 0);
+ if (IS_ERR(ep->re_pd)) {
+ rc = PTR_ERR(ep->re_pd);
++ ep->re_pd = NULL;
+ goto out_destroy;
+ }
+
+diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
+index 04f1b78bcbca3..897dfce7dd271 100644
+--- a/net/sunrpc/xprtsock.c
++++ b/net/sunrpc/xprtsock.c
+@@ -763,12 +763,12 @@ xs_stream_start_connect(struct sock_xprt *transport)
+ /**
+ * xs_nospace - handle transmit was incomplete
+ * @req: pointer to RPC request
++ * @transport: pointer to struct sock_xprt
+ *
+ */
+-static int xs_nospace(struct rpc_rqst *req)
++static int xs_nospace(struct rpc_rqst *req, struct sock_xprt *transport)
+ {
+- struct rpc_xprt *xprt = req->rq_xprt;
+- struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
++ struct rpc_xprt *xprt = &transport->xprt;
+ struct sock *sk = transport->inet;
+ int ret = -EAGAIN;
+
+@@ -779,25 +779,49 @@ static int xs_nospace(struct rpc_rqst *req)
+
+ /* Don't race with disconnect */
+ if (xprt_connected(xprt)) {
++ struct socket_wq *wq;
++
++ rcu_read_lock();
++ wq = rcu_dereference(sk->sk_wq);
++ set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags);
++ rcu_read_unlock();
++
+ /* wait for more buffer space */
++ set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+ sk->sk_write_pending++;
+ xprt_wait_for_buffer_space(xprt);
+ } else
+ ret = -ENOTCONN;
+
+ spin_unlock(&xprt->transport_lock);
++ return ret;
++}
+
+- /* Race breaker in case memory is freed before above code is called */
+- if (ret == -EAGAIN) {
+- struct socket_wq *wq;
++static int xs_sock_nospace(struct rpc_rqst *req)
++{
++ struct sock_xprt *transport =
++ container_of(req->rq_xprt, struct sock_xprt, xprt);
++ struct sock *sk = transport->inet;
++ int ret = -EAGAIN;
+
+- rcu_read_lock();
+- wq = rcu_dereference(sk->sk_wq);
+- set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags);
+- rcu_read_unlock();
++ lock_sock(sk);
++ if (!sock_writeable(sk))
++ ret = xs_nospace(req, transport);
++ release_sock(sk);
++ return ret;
++}
+
+- sk->sk_write_space(sk);
+- }
++static int xs_stream_nospace(struct rpc_rqst *req)
++{
++ struct sock_xprt *transport =
++ container_of(req->rq_xprt, struct sock_xprt, xprt);
++ struct sock *sk = transport->inet;
++ int ret = -EAGAIN;
++
++ lock_sock(sk);
++ if (!sk_stream_memory_free(sk))
++ ret = xs_nospace(req, transport);
++ release_sock(sk);
+ return ret;
+ }
+
+@@ -856,7 +880,7 @@ static int xs_local_send_request(struct rpc_rqst *req)
+
+ /* Close the stream if the previous transmission was incomplete */
+ if (xs_send_request_was_aborted(transport, req)) {
+- xs_close(xprt);
++ xprt_force_disconnect(xprt);
+ return -ENOTCONN;
+ }
+
+@@ -887,14 +911,14 @@ static int xs_local_send_request(struct rpc_rqst *req)
+ case -ENOBUFS:
+ break;
+ case -EAGAIN:
+- status = xs_nospace(req);
++ status = xs_stream_nospace(req);
+ break;
+ default:
+ dprintk("RPC: sendmsg returned unrecognized error %d\n",
+ -status);
+ fallthrough;
+ case -EPIPE:
+- xs_close(xprt);
++ xprt_force_disconnect(xprt);
+ status = -ENOTCONN;
+ }
+
+@@ -963,7 +987,7 @@ process_status:
+ /* Should we call xs_close() here? */
+ break;
+ case -EAGAIN:
+- status = xs_nospace(req);
++ status = xs_sock_nospace(req);
+ break;
+ case -ENETUNREACH:
+ case -ENOBUFS:
+@@ -1083,7 +1107,7 @@ static int xs_tcp_send_request(struct rpc_rqst *req)
+ /* Should we call xs_close() here? */
+ break;
+ case -EAGAIN:
+- status = xs_nospace(req);
++ status = xs_stream_nospace(req);
+ break;
+ case -ECONNRESET:
+ case -ECONNREFUSED:
+@@ -1181,6 +1205,16 @@ static void xs_reset_transport(struct sock_xprt *transport)
+
+ if (sk == NULL)
+ return;
++ /*
++ * Make sure we're calling this in a context from which it is safe
++ * to call __fput_sync(). In practice that means rpciod and the
++ * system workqueue.
++ */
++ if (!(current->flags & PF_WQ_WORKER)) {
++ WARN_ON_ONCE(1);
++ set_bit(XPRT_CLOSE_WAIT, &xprt->state);
++ return;
++ }
+
+ if (atomic_read(&transport->xprt.swapper))
+ sk_clear_memalloc(sk);
+@@ -1204,7 +1238,7 @@ static void xs_reset_transport(struct sock_xprt *transport)
+ mutex_unlock(&transport->recv_mutex);
+
+ trace_rpc_socket_close(xprt, sock);
+- fput(filp);
++ __fput_sync(filp);
+
+ xprt_disconnect_done(xprt);
+ }
+@@ -2233,6 +2267,7 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
+ fallthrough;
+ case -EINPROGRESS:
+ /* SYN_SENT! */
++ set_bit(XPRT_SOCK_CONNECT_SENT, &transport->sock_state);
+ if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
+ xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
+ break;
+@@ -2258,10 +2293,14 @@ static void xs_tcp_setup_socket(struct work_struct *work)
+ struct rpc_xprt *xprt = &transport->xprt;
+ int status = -EIO;
+
+- if (!sock) {
+- sock = xs_create_sock(xprt, transport,
+- xs_addr(xprt)->sa_family, SOCK_STREAM,
+- IPPROTO_TCP, true);
++ if (xprt_connected(xprt))
++ goto out;
++ if (test_and_clear_bit(XPRT_SOCK_CONNECT_SENT,
++ &transport->sock_state) ||
++ !sock) {
++ xs_reset_transport(transport);
++ sock = xs_create_sock(xprt, transport, xs_addr(xprt)->sa_family,
++ SOCK_STREAM, IPPROTO_TCP, true);
+ if (IS_ERR(sock)) {
+ status = PTR_ERR(sock);
+ goto out;
+@@ -2343,11 +2382,7 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
+
+ if (transport->sock != NULL) {
+ dprintk("RPC: xs_connect delayed xprt %p for %lu "
+- "seconds\n",
+- xprt, xprt->reestablish_timeout / HZ);
+-
+- /* Start by resetting any existing state */
+- xs_reset_transport(transport);
++ "seconds\n", xprt, xprt->reestablish_timeout / HZ);
+
+ delay = xprt_reconnect_delay(xprt);
+ xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO);
+@@ -2823,9 +2858,6 @@ static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
+ }
+ xprt_set_bound(xprt);
+ xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
+- ret = ERR_PTR(xs_local_setup_socket(transport));
+- if (ret)
+- goto out_err;
+ break;
+ default:
+ ret = ERR_PTR(-EAFNOSUPPORT);
+diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c
+index 443f8e5b94777..36b466cfd9e11 100644
+--- a/net/tipc/bearer.c
++++ b/net/tipc/bearer.c
+@@ -352,16 +352,18 @@ static int tipc_enable_bearer(struct net *net, const char *name,
+ goto rejected;
+ }
+
+- test_and_set_bit_lock(0, &b->up);
+- rcu_assign_pointer(tn->bearer_list[bearer_id], b);
+- if (skb)
+- tipc_bearer_xmit_skb(net, bearer_id, skb, &b->bcast_addr);
+-
++ /* Create monitoring data before accepting activate messages */
+ if (tipc_mon_create(net, bearer_id)) {
+ bearer_disable(net, b);
++ kfree_skb(skb);
+ return -ENOMEM;
+ }
+
++ test_and_set_bit_lock(0, &b->up);
++ rcu_assign_pointer(tn->bearer_list[bearer_id], b);
++ if (skb)
++ tipc_bearer_xmit_skb(net, bearer_id, skb, &b->bcast_addr);
++
+ pr_info("Enabled bearer <%s>, priority %u\n", name, prio);
+
+ return res;
+diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c
+index dc60c32bb70df..b5074957e8812 100644
+--- a/net/tipc/crypto.c
++++ b/net/tipc/crypto.c
+@@ -597,6 +597,10 @@ static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
+ tmp->cloned = NULL;
+ tmp->authsize = TIPC_AES_GCM_TAG_SIZE;
+ tmp->key = kmemdup(ukey, tipc_aead_key_size(ukey), GFP_KERNEL);
++ if (!tmp->key) {
++ tipc_aead_free(&tmp->rcu);
++ return -ENOMEM;
++ }
+ memcpy(&tmp->salt, ukey->key + keylen, TIPC_AES_GCM_SALT_SIZE);
+ atomic_set(&tmp->users, 0);
+ atomic64_set(&tmp->seqno, 0);
+@@ -2283,7 +2287,7 @@ static bool tipc_crypto_key_rcv(struct tipc_crypto *rx, struct tipc_msg *hdr)
+ struct tipc_crypto *tx = tipc_net(rx->net)->crypto_tx;
+ struct tipc_aead_key *skey = NULL;
+ u16 key_gen = msg_key_gen(hdr);
+- u16 size = msg_data_sz(hdr);
++ u32 size = msg_data_sz(hdr);
+ u8 *data = msg_data(hdr);
+ unsigned int keylen;
+
+diff --git a/net/tipc/link.c b/net/tipc/link.c
+index 1b7a487c88419..115a4a7950f50 100644
+--- a/net/tipc/link.c
++++ b/net/tipc/link.c
+@@ -1298,8 +1298,11 @@ static bool tipc_data_input(struct tipc_link *l, struct sk_buff *skb,
+ return false;
+ #ifdef CONFIG_TIPC_CRYPTO
+ case MSG_CRYPTO:
+- tipc_crypto_msg_rcv(l->net, skb);
+- return true;
++ if (TIPC_SKB_CB(skb)->decrypted) {
++ tipc_crypto_msg_rcv(l->net, skb);
++ return true;
++ }
++ fallthrough;
+ #endif
+ default:
+ pr_warn("Dropping received illegal msg type\n");
+@@ -2196,7 +2199,7 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
+ struct tipc_msg *hdr = buf_msg(skb);
+ struct tipc_gap_ack_blks *ga = NULL;
+ bool reply = msg_probe(hdr), retransmitted = false;
+- u16 dlen = msg_data_sz(hdr), glen = 0;
++ u32 dlen = msg_data_sz(hdr), glen = 0;
+ u16 peers_snd_nxt = msg_next_sent(hdr);
+ u16 peers_tol = msg_link_tolerance(hdr);
+ u16 peers_prio = msg_linkprio(hdr);
+@@ -2210,6 +2213,10 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
+ void *data;
+
+ trace_tipc_proto_rcv(skb, false, l->name);
++
++ if (dlen > U16_MAX)
++ goto exit;
++
+ if (tipc_link_is_blocked(l) || !xmitq)
+ goto exit;
+
+@@ -2278,6 +2285,11 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
+ break;
+
+ case STATE_MSG:
++ /* Validate Gap ACK blocks, drop if invalid */
++ glen = tipc_get_gap_ack_blks(&ga, l, hdr, true);
++ if (glen > dlen)
++ break;
++
+ l->rcv_nxt_state = msg_seqno(hdr) + 1;
+
+ /* Update own tolerance if peer indicates a non-zero value */
+@@ -2303,9 +2315,6 @@ static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
+ break;
+ }
+
+- /* Receive Gap ACK blocks from peer if any */
+- glen = tipc_get_gap_ack_blks(&ga, l, hdr, true);
+-
+ tipc_mon_rcv(l->net, data + glen, dlen - glen, l->addr,
+ &l->mon_state, l->bearer_id);
+
+diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c
+index 407619697292f..2f4d23238a7e3 100644
+--- a/net/tipc/monitor.c
++++ b/net/tipc/monitor.c
+@@ -496,6 +496,8 @@ void tipc_mon_rcv(struct net *net, void *data, u16 dlen, u32 addr,
+ state->probing = false;
+
+ /* Sanity check received domain record */
++ if (new_member_cnt > MAX_MON_DOMAIN)
++ return;
+ if (dlen < dom_rec_len(arrv_dom, 0))
+ return;
+ if (dlen != dom_rec_len(arrv_dom, new_member_cnt))
+diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
+index bda902caa8147..8267b751a526a 100644
+--- a/net/tipc/name_distr.c
++++ b/net/tipc/name_distr.c
+@@ -313,7 +313,7 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
+ pr_warn_ratelimited("Failed to remove binding %u,%u from %u\n",
+ ua.sr.type, ua.sr.lower, node);
+ } else {
+- pr_warn("Unrecognized name table message received\n");
++ pr_warn_ratelimited("Unknown name table message received\n");
+ }
+ return false;
+ }
+diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
+index 01396dd1c899b..1d8ba233d0474 100644
+--- a/net/tipc/name_table.c
++++ b/net/tipc/name_table.c
+@@ -967,7 +967,7 @@ static int __tipc_nl_add_nametable_publ(struct tipc_nl_msg *msg,
+ list_for_each_entry(p, &sr->all_publ, all_publ)
+ if (p->key == *last_key)
+ break;
+- if (p->key != *last_key)
++ if (list_entry_is_head(p, &sr->all_publ, all_publ))
+ return -EPIPE;
+ } else {
+ p = list_first_entry(&sr->all_publ,
+diff --git a/net/tipc/node.c b/net/tipc/node.c
+index 9947b7dfe1d2d..6ef95ce565bd3 100644
+--- a/net/tipc/node.c
++++ b/net/tipc/node.c
+@@ -403,7 +403,7 @@ static void tipc_node_write_unlock(struct tipc_node *n)
+ u32 flags = n->action_flags;
+ struct list_head *publ_list;
+ struct tipc_uaddr ua;
+- u32 bearer_id;
++ u32 bearer_id, node;
+
+ if (likely(!flags)) {
+ write_unlock_bh(&n->lock);
+@@ -413,7 +413,8 @@ static void tipc_node_write_unlock(struct tipc_node *n)
+ tipc_uaddr(&ua, TIPC_SERVICE_RANGE, TIPC_NODE_SCOPE,
+ TIPC_LINK_STATE, n->addr, n->addr);
+ sk.ref = n->link_id;
+- sk.node = n->addr;
++ sk.node = tipc_own_addr(net);
++ node = n->addr;
+ bearer_id = n->link_id & 0xffff;
+ publ_list = &n->publ_list;
+
+@@ -423,17 +424,17 @@ static void tipc_node_write_unlock(struct tipc_node *n)
+ write_unlock_bh(&n->lock);
+
+ if (flags & TIPC_NOTIFY_NODE_DOWN)
+- tipc_publ_notify(net, publ_list, sk.node, n->capabilities);
++ tipc_publ_notify(net, publ_list, node, n->capabilities);
+
+ if (flags & TIPC_NOTIFY_NODE_UP)
+- tipc_named_node_up(net, sk.node, n->capabilities);
++ tipc_named_node_up(net, node, n->capabilities);
+
+ if (flags & TIPC_NOTIFY_LINK_UP) {
+- tipc_mon_peer_up(net, sk.node, bearer_id);
++ tipc_mon_peer_up(net, node, bearer_id);
+ tipc_nametbl_publish(net, &ua, &sk, sk.ref);
+ }
+ if (flags & TIPC_NOTIFY_LINK_DOWN) {
+- tipc_mon_peer_down(net, sk.node, bearer_id);
++ tipc_mon_peer_down(net, node, bearer_id);
+ tipc_nametbl_withdraw(net, &ua, &sk, sk.ref);
+ }
+ }
+diff --git a/net/tipc/socket.c b/net/tipc/socket.c
+index ad570c2450be8..17f8c523e33b0 100644
+--- a/net/tipc/socket.c
++++ b/net/tipc/socket.c
+@@ -1461,6 +1461,8 @@ static int __tipc_sendmsg(struct socket *sock, struct msghdr *m, size_t dlen)
+ msg_set_syn(hdr, 1);
+ }
+
++ memset(&skaddr, 0, sizeof(skaddr));
++
+ /* Determine destination */
+ if (atype == TIPC_SERVICE_RANGE) {
+ return tipc_sendmcast(sock, ua, m, dlen, timeout);
+@@ -2850,7 +2852,8 @@ static void tipc_sk_retry_connect(struct sock *sk, struct sk_buff_head *list)
+
+ /* Try again later if dest link is congested */
+ if (tsk->cong_link_cnt) {
+- sk_reset_timer(sk, &sk->sk_timer, msecs_to_jiffies(100));
++ sk_reset_timer(sk, &sk->sk_timer,
++ jiffies + msecs_to_jiffies(100));
+ return;
+ }
+ /* Prepare SYN for retransmit */
+@@ -3747,7 +3750,7 @@ static int __tipc_nl_list_sk_publ(struct sk_buff *skb,
+ if (p->key == *last_publ)
+ break;
+ }
+- if (p->key != *last_publ) {
++ if (list_entry_is_head(p, &tsk->publications, binding_sock)) {
+ /* We never set seq or call nl_dump_check_consistent()
+ * this means that setting prev_seq here will cause the
+ * consistence check to fail in the netlink callback
+diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
+index b932469ee69cc..f3e3d009cf1cf 100644
+--- a/net/tls/tls_device.c
++++ b/net/tls/tls_device.c
+@@ -483,11 +483,13 @@ handle_error:
+ copy = min_t(size_t, size, (pfrag->size - pfrag->offset));
+ copy = min_t(size_t, copy, (max_open_record_len - record->len));
+
+- rc = tls_device_copy_data(page_address(pfrag->page) +
+- pfrag->offset, copy, msg_iter);
+- if (rc)
+- goto handle_error;
+- tls_append_frag(record, pfrag, copy);
++ if (copy) {
++ rc = tls_device_copy_data(page_address(pfrag->page) +
++ pfrag->offset, copy, msg_iter);
++ if (rc)
++ goto handle_error;
++ tls_append_frag(record, pfrag, copy);
++ }
+
+ size -= copy;
+ if (!size) {
+@@ -1345,7 +1347,10 @@ static int tls_device_down(struct net_device *netdev)
+
+ /* Device contexts for RX and TX will be freed in on sk_destruct
+ * by tls_device_free_ctx. rx_conf and tx_conf stay in TLS_HW.
++ * Now release the ref taken above.
+ */
++ if (refcount_dec_and_test(&ctx->refcount))
++ tls_device_free_ctx(ctx);
+ }
+
+ up_write(&device_offload_lock);
+diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
+index 9ab81db8a6545..9aac9c60d786d 100644
+--- a/net/tls/tls_main.c
++++ b/net/tls/tls_main.c
+@@ -61,7 +61,7 @@ static DEFINE_MUTEX(tcpv6_prot_mutex);
+ static const struct proto *saved_tcpv4_prot;
+ static DEFINE_MUTEX(tcpv4_prot_mutex);
+ static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
+-static struct proto_ops tls_sw_proto_ops;
++static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
+ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
+ const struct proto *base);
+
+@@ -71,6 +71,8 @@ void update_sk_prot(struct sock *sk, struct tls_context *ctx)
+
+ WRITE_ONCE(sk->sk_prot,
+ &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]);
++ WRITE_ONCE(sk->sk_socket->ops,
++ &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]);
+ }
+
+ int wait_on_pending_writer(struct sock *sk, long *timeo)
+@@ -581,8 +583,6 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
+ if (tx) {
+ ctx->sk_write_space = sk->sk_write_space;
+ sk->sk_write_space = tls_write_space;
+- } else {
+- sk->sk_socket->ops = &tls_sw_proto_ops;
+ }
+ goto out;
+
+@@ -640,6 +640,39 @@ struct tls_context *tls_ctx_create(struct sock *sk)
+ return ctx;
+ }
+
++static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
++ const struct proto_ops *base)
++{
++ ops[TLS_BASE][TLS_BASE] = *base;
++
++ ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
++ ops[TLS_SW ][TLS_BASE].sendpage_locked = tls_sw_sendpage_locked;
++
++ ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE];
++ ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read;
++
++ ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE];
++ ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read;
++
++#ifdef CONFIG_TLS_DEVICE
++ ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
++ ops[TLS_HW ][TLS_BASE].sendpage_locked = NULL;
++
++ ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ];
++ ops[TLS_HW ][TLS_SW ].sendpage_locked = NULL;
++
++ ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ];
++
++ ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ];
++
++ ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ];
++ ops[TLS_HW ][TLS_HW ].sendpage_locked = NULL;
++#endif
++#ifdef CONFIG_TLS_TOE
++ ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
++#endif
++}
++
+ static void tls_build_proto(struct sock *sk)
+ {
+ int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
+@@ -651,6 +684,8 @@ static void tls_build_proto(struct sock *sk)
+ mutex_lock(&tcpv6_prot_mutex);
+ if (likely(prot != saved_tcpv6_prot)) {
+ build_protos(tls_prots[TLSV6], prot);
++ build_proto_ops(tls_proto_ops[TLSV6],
++ sk->sk_socket->ops);
+ smp_store_release(&saved_tcpv6_prot, prot);
+ }
+ mutex_unlock(&tcpv6_prot_mutex);
+@@ -661,6 +696,8 @@ static void tls_build_proto(struct sock *sk)
+ mutex_lock(&tcpv4_prot_mutex);
+ if (likely(prot != saved_tcpv4_prot)) {
+ build_protos(tls_prots[TLSV4], prot);
++ build_proto_ops(tls_proto_ops[TLSV4],
++ sk->sk_socket->ops);
+ smp_store_release(&saved_tcpv4_prot, prot);
+ }
+ mutex_unlock(&tcpv4_prot_mutex);
+@@ -871,10 +908,6 @@ static int __init tls_register(void)
+ if (err)
+ return err;
+
+- tls_sw_proto_ops = inet_stream_ops;
+- tls_sw_proto_ops.splice_read = tls_sw_splice_read;
+- tls_sw_proto_ops.sendpage_locked = tls_sw_sendpage_locked;
+-
+ tls_device_init();
+ tcp_register_ulp(&tcp_tls_ulp_ops);
+
+diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
+index 1b08b877a8900..794ef3b3d7d4b 100644
+--- a/net/tls/tls_sw.c
++++ b/net/tls/tls_sw.c
+@@ -515,7 +515,7 @@ static int tls_do_encryption(struct sock *sk,
+ memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
+ prot->iv_size + prot->salt_size);
+
+- xor_iv_with_seq(prot, rec->iv_data, tls_ctx->tx.rec_seq);
++ xor_iv_with_seq(prot, rec->iv_data + iv_offset, tls_ctx->tx.rec_seq);
+
+ sge->offset += prot->prepend_size;
+ sge->length -= prot->prepend_size;
+@@ -1483,11 +1483,11 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
+ if (prot->version == TLS_1_3_VERSION ||
+ prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305)
+ memcpy(iv + iv_offset, tls_ctx->rx.iv,
+- crypto_aead_ivsize(ctx->aead_recv));
++ prot->iv_size + prot->salt_size);
+ else
+ memcpy(iv + iv_offset, tls_ctx->rx.iv, prot->salt_size);
+
+- xor_iv_with_seq(prot, iv, tls_ctx->rx.rec_seq);
++ xor_iv_with_seq(prot, iv + iv_offset, tls_ctx->rx.rec_seq);
+
+ /* Prepare AAD */
+ tls_make_aad(aad, rxm->full_len - prot->overhead_size +
+@@ -1993,6 +1993,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
+ struct sock *sk = sock->sk;
+ struct sk_buff *skb;
+ ssize_t copied = 0;
++ bool from_queue;
+ int err = 0;
+ long timeo;
+ int chunk;
+@@ -2002,25 +2003,28 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
+
+ timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK);
+
+- skb = tls_wait_data(sk, NULL, flags & SPLICE_F_NONBLOCK, timeo, &err);
+- if (!skb)
+- goto splice_read_end;
+-
+- if (!ctx->decrypted) {
+- err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc, false);
+-
+- /* splice does not support reading control messages */
+- if (ctx->control != TLS_RECORD_TYPE_DATA) {
+- err = -EINVAL;
++ from_queue = !skb_queue_empty(&ctx->rx_list);
++ if (from_queue) {
++ skb = __skb_dequeue(&ctx->rx_list);
++ } else {
++ skb = tls_wait_data(sk, NULL, flags & SPLICE_F_NONBLOCK, timeo,
++ &err);
++ if (!skb)
+ goto splice_read_end;
+- }
+
++ err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc, false);
+ if (err < 0) {
+ tls_err_abort(sk, -EBADMSG);
+ goto splice_read_end;
+ }
+- ctx->decrypted = 1;
+ }
++
++ /* splice does not support reading control messages */
++ if (ctx->control != TLS_RECORD_TYPE_DATA) {
++ err = -EINVAL;
++ goto splice_read_end;
++ }
++
+ rxm = strp_msg(skb);
+
+ chunk = min_t(unsigned int, rxm->full_len, len);
+@@ -2028,7 +2032,17 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
+ if (copied < 0)
+ goto splice_read_end;
+
+- tls_sw_advance_skb(sk, skb, copied);
++ if (!from_queue) {
++ ctx->recv_pkt = NULL;
++ __strp_unpause(&ctx->strp);
++ }
++ if (chunk < rxm->full_len) {
++ __skb_queue_head(&ctx->rx_list, skb);
++ rxm->offset += len;
++ rxm->full_len -= len;
++ } else {
++ consume_skb(skb);
++ }
+
+ splice_read_end:
+ release_sock(sk);
+diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
+index 78e08e82c08c4..62f47821d783f 100644
+--- a/net/unix/af_unix.c
++++ b/net/unix/af_unix.c
+@@ -1996,7 +1996,7 @@ static int queue_oob(struct socket *sock, struct msghdr *msg, struct sock *other
+ if (ousk->oob_skb)
+ consume_skb(ousk->oob_skb);
+
+- ousk->oob_skb = skb;
++ WRITE_ONCE(ousk->oob_skb, skb);
+
+ scm_stat_add(other, skb);
+ skb_queue_tail(&other->sk_receive_queue, skb);
+@@ -2514,9 +2514,8 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state)
+
+ oob_skb = u->oob_skb;
+
+- if (!(state->flags & MSG_PEEK)) {
+- u->oob_skb = NULL;
+- }
++ if (!(state->flags & MSG_PEEK))
++ WRITE_ONCE(u->oob_skb, NULL);
+
+ unix_state_unlock(sk);
+
+@@ -2551,7 +2550,7 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk,
+ skb = NULL;
+ } else if (sock_flag(sk, SOCK_URGINLINE)) {
+ if (!(flags & MSG_PEEK)) {
+- u->oob_skb = NULL;
++ WRITE_ONCE(u->oob_skb, NULL);
+ consume_skb(skb);
+ }
+ } else if (!(flags & MSG_PEEK)) {
+@@ -2882,9 +2881,6 @@ static int unix_shutdown(struct socket *sock, int mode)
+
+ unix_state_lock(sk);
+ sk->sk_shutdown |= mode;
+- if ((sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) &&
+- mode == SHUTDOWN_MASK)
+- sk->sk_state = TCP_CLOSE;
+ other = unix_peer(sk);
+ if (other)
+ sock_hold(other);
+@@ -3009,11 +3005,10 @@ static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
+ case SIOCATMARK:
+ {
+ struct sk_buff *skb;
+- struct unix_sock *u = unix_sk(sk);
+ int answ = 0;
+
+ skb = skb_peek(&sk->sk_receive_queue);
+- if (skb && skb == u->oob_skb)
++ if (skb && skb == READ_ONCE(unix_sk(sk)->oob_skb))
+ answ = 1;
+ err = put_user(answ, (int __user *)arg);
+ }
+@@ -3054,6 +3049,10 @@ static __poll_t unix_poll(struct file *file, struct socket *sock, poll_table *wa
+ mask |= EPOLLIN | EPOLLRDNORM;
+ if (sk_is_readable(sk))
+ mask |= EPOLLIN | EPOLLRDNORM;
++#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
++ if (READ_ONCE(unix_sk(sk)->oob_skb))
++ mask |= EPOLLPRI;
++#endif
+
+ /* Connection-based need to check for termination and startup */
+ if ((sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) &&
+diff --git a/net/unix/garbage.c b/net/unix/garbage.c
+index 12e2ddaf887f2..d45d5366115a7 100644
+--- a/net/unix/garbage.c
++++ b/net/unix/garbage.c
+@@ -192,8 +192,11 @@ void wait_for_unix_gc(void)
+ {
+ /* If number of inflight sockets is insane,
+ * force a garbage collect right now.
++ * Paired with the WRITE_ONCE() in unix_inflight(),
++ * unix_notinflight() and gc_in_progress().
+ */
+- if (unix_tot_inflight > UNIX_INFLIGHT_TRIGGER_GC && !gc_in_progress)
++ if (READ_ONCE(unix_tot_inflight) > UNIX_INFLIGHT_TRIGGER_GC &&
++ !READ_ONCE(gc_in_progress))
+ unix_gc();
+ wait_event(unix_gc_wait, gc_in_progress == false);
+ }
+@@ -213,7 +216,9 @@ void unix_gc(void)
+ if (gc_in_progress)
+ goto out;
+
+- gc_in_progress = true;
++ /* Paired with READ_ONCE() in wait_for_unix_gc(). */
++ WRITE_ONCE(gc_in_progress, true);
++
+ /* First, select candidates for garbage collection. Only
+ * in-flight sockets are considered, and from those only ones
+ * which don't have any external reference.
+@@ -299,7 +304,10 @@ void unix_gc(void)
+
+ /* All candidates should have been detached by now. */
+ BUG_ON(!list_empty(&gc_candidates));
+- gc_in_progress = false;
++
++ /* Paired with READ_ONCE() in wait_for_unix_gc(). */
++ WRITE_ONCE(gc_in_progress, false);
++
+ wake_up(&unix_gc_wait);
+
+ out:
+diff --git a/net/unix/scm.c b/net/unix/scm.c
+index 052ae709ce289..aa27a02478dc1 100644
+--- a/net/unix/scm.c
++++ b/net/unix/scm.c
+@@ -60,7 +60,8 @@ void unix_inflight(struct user_struct *user, struct file *fp)
+ } else {
+ BUG_ON(list_empty(&u->link));
+ }
+- unix_tot_inflight++;
++ /* Paired with READ_ONCE() in wait_for_unix_gc() */
++ WRITE_ONCE(unix_tot_inflight, unix_tot_inflight + 1);
+ }
+ user->unix_inflight++;
+ spin_unlock(&unix_gc_lock);
+@@ -80,7 +81,8 @@ void unix_notinflight(struct user_struct *user, struct file *fp)
+
+ if (atomic_long_dec_and_test(&u->inflight))
+ list_del_init(&u->link);
+- unix_tot_inflight--;
++ /* Paired with READ_ONCE() in wait_for_unix_gc() */
++ WRITE_ONCE(unix_tot_inflight, unix_tot_inflight - 1);
+ }
+ user->unix_inflight--;
+ spin_unlock(&unix_gc_lock);
+diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
+index e2c0cfb334d20..5df530e89e5ac 100644
+--- a/net/vmw_vsock/af_vsock.c
++++ b/net/vmw_vsock/af_vsock.c
+@@ -333,7 +333,8 @@ void vsock_remove_sock(struct vsock_sock *vsk)
+ }
+ EXPORT_SYMBOL_GPL(vsock_remove_sock);
+
+-void vsock_for_each_connected_socket(void (*fn)(struct sock *sk))
++void vsock_for_each_connected_socket(struct vsock_transport *transport,
++ void (*fn)(struct sock *sk))
+ {
+ int i;
+
+@@ -342,8 +343,12 @@ void vsock_for_each_connected_socket(void (*fn)(struct sock *sk))
+ for (i = 0; i < ARRAY_SIZE(vsock_connected_table); i++) {
+ struct vsock_sock *vsk;
+ list_for_each_entry(vsk, &vsock_connected_table[i],
+- connected_table)
++ connected_table) {
++ if (vsk->transport != transport)
++ continue;
++
+ fn(sk_vsock(vsk));
++ }
+ }
+
+ spin_unlock_bh(&vsock_table_lock);
+@@ -1322,6 +1327,8 @@ static int vsock_connect(struct socket *sock, struct sockaddr *addr,
+ * non-blocking call.
+ */
+ err = -EALREADY;
++ if (flags & O_NONBLOCK)
++ goto out;
+ break;
+ default:
+ if ((sk->sk_state == TCP_LISTEN) ||
+@@ -1398,6 +1405,7 @@ static int vsock_connect(struct socket *sock, struct sockaddr *addr,
+ sk->sk_state = sk->sk_state == TCP_ESTABLISHED ? TCP_CLOSING : TCP_CLOSE;
+ sock->state = SS_UNCONNECTED;
+ vsock_transport_cancel_pkt(vsk);
++ vsock_remove_connected(vsk);
+ goto out_wait;
+ } else if (timeout == 0) {
+ err = -ETIMEDOUT;
+diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c
+index 4f7c99dfd16cf..c5f936fbf876d 100644
+--- a/net/vmw_vsock/virtio_transport.c
++++ b/net/vmw_vsock/virtio_transport.c
+@@ -24,6 +24,7 @@
+ static struct workqueue_struct *virtio_vsock_workqueue;
+ static struct virtio_vsock __rcu *the_virtio_vsock;
+ static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */
++static struct virtio_transport virtio_transport; /* forward declaration */
+
+ struct virtio_vsock {
+ struct virtio_device *vdev;
+@@ -384,7 +385,8 @@ static void virtio_vsock_event_handle(struct virtio_vsock *vsock,
+ switch (le32_to_cpu(event->id)) {
+ case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET:
+ virtio_vsock_update_guest_cid(vsock);
+- vsock_for_each_connected_socket(virtio_vsock_reset_sock);
++ vsock_for_each_connected_socket(&virtio_transport.transport,
++ virtio_vsock_reset_sock);
+ break;
+ }
+ }
+@@ -620,6 +622,13 @@ static int virtio_vsock_probe(struct virtio_device *vdev)
+ INIT_WORK(&vsock->event_work, virtio_transport_event_work);
+ INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);
+
++ if (virtio_has_feature(vdev, VIRTIO_VSOCK_F_SEQPACKET))
++ vsock->seqpacket_allow = true;
++
++ vdev->priv = vsock;
++
++ virtio_device_ready(vdev);
++
+ mutex_lock(&vsock->tx_lock);
+ vsock->tx_run = true;
+ mutex_unlock(&vsock->tx_lock);
+@@ -634,10 +643,6 @@ static int virtio_vsock_probe(struct virtio_device *vdev)
+ vsock->event_run = true;
+ mutex_unlock(&vsock->event_lock);
+
+- if (virtio_has_feature(vdev, VIRTIO_VSOCK_F_SEQPACKET))
+- vsock->seqpacket_allow = true;
+-
+- vdev->priv = vsock;
+ rcu_assign_pointer(the_virtio_vsock, vsock);
+
+ mutex_unlock(&the_virtio_vsock_mutex);
+@@ -662,7 +667,8 @@ static void virtio_vsock_remove(struct virtio_device *vdev)
+ synchronize_rcu();
+
+ /* Reset all connected sockets when the device disappear */
+- vsock_for_each_connected_socket(virtio_vsock_reset_sock);
++ vsock_for_each_connected_socket(&virtio_transport.transport,
++ virtio_vsock_reset_sock);
+
+ /* Stop all work handlers to make sure no one is accessing the device,
+ * so we can safely call vdev->config->reset().
+diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
+index 59ee1be5a6dd3..ec2c2afbf0d06 100644
+--- a/net/vmw_vsock/virtio_transport_common.c
++++ b/net/vmw_vsock/virtio_transport_common.c
+@@ -1299,7 +1299,8 @@ void virtio_transport_recv_pkt(struct virtio_transport *t,
+ space_available = virtio_transport_space_update(sk, pkt);
+
+ /* Update CID in case it has changed after a transport reset event */
+- vsk->local_addr.svm_cid = dst.svm_cid;
++ if (vsk->local_addr.svm_cid != VMADDR_CID_ANY)
++ vsk->local_addr.svm_cid = dst.svm_cid;
+
+ if (space_available)
+ sk->sk_write_space(sk);
+diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c
+index 7aef34e32bdf8..b17dc9745188e 100644
+--- a/net/vmw_vsock/vmci_transport.c
++++ b/net/vmw_vsock/vmci_transport.c
+@@ -75,6 +75,8 @@ static u32 vmci_transport_qp_resumed_sub_id = VMCI_INVALID_ID;
+
+ static int PROTOCOL_OVERRIDE = -1;
+
++static struct vsock_transport vmci_transport; /* forward declaration */
++
+ /* Helper function to convert from a VMCI error code to a VSock error code. */
+
+ static s32 vmci_transport_error_to_vsock_error(s32 vmci_error)
+@@ -882,7 +884,8 @@ static void vmci_transport_qp_resumed_cb(u32 sub_id,
+ const struct vmci_event_data *e_data,
+ void *client_data)
+ {
+- vsock_for_each_connected_socket(vmci_transport_handle_detach);
++ vsock_for_each_connected_socket(&vmci_transport,
++ vmci_transport_handle_detach);
+ }
+
+ static void vmci_transport_recv_pkt_work(struct work_struct *work)
+diff --git a/net/wireless/core.c b/net/wireless/core.c
+index aaba847d79eb2..441136646f89a 100644
+--- a/net/wireless/core.c
++++ b/net/wireless/core.c
+@@ -5,7 +5,7 @@
+ * Copyright 2006-2010 Johannes Berg <johannes@sipsolutions.net>
+ * Copyright 2013-2014 Intel Mobile Communications GmbH
+ * Copyright 2015-2017 Intel Deutschland GmbH
+- * Copyright (C) 2018-2021 Intel Corporation
++ * Copyright (C) 2018-2022 Intel Corporation
+ */
+
+ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+@@ -332,29 +332,20 @@ static void cfg80211_event_work(struct work_struct *work)
+ void cfg80211_destroy_ifaces(struct cfg80211_registered_device *rdev)
+ {
+ struct wireless_dev *wdev, *tmp;
+- bool found = false;
+
+ ASSERT_RTNL();
+
+- list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
++ list_for_each_entry_safe(wdev, tmp, &rdev->wiphy.wdev_list, list) {
+ if (wdev->nl_owner_dead) {
+ if (wdev->netdev)
+ dev_close(wdev->netdev);
+- found = true;
+- }
+- }
+
+- if (!found)
+- return;
+-
+- wiphy_lock(&rdev->wiphy);
+- list_for_each_entry_safe(wdev, tmp, &rdev->wiphy.wdev_list, list) {
+- if (wdev->nl_owner_dead) {
++ wiphy_lock(&rdev->wiphy);
+ cfg80211_leave(rdev, wdev);
+ rdev_del_virtual_intf(rdev, wdev);
++ wiphy_unlock(&rdev->wiphy);
+ }
+ }
+- wiphy_unlock(&rdev->wiphy);
+ }
+
+ static void cfg80211_destroy_iface_wk(struct work_struct *work)
+@@ -1081,6 +1072,16 @@ void cfg80211_dev_free(struct cfg80211_registered_device *rdev)
+ list_for_each_entry_safe(scan, tmp, &rdev->bss_list, list)
+ cfg80211_put_bss(&rdev->wiphy, &scan->pub);
+ mutex_destroy(&rdev->wiphy.mtx);
++
++ /*
++ * The 'regd' can only be non-NULL if we never finished
++ * initializing the wiphy and thus never went through the
++ * unregister path - e.g. in failure scenarios. Thus, it
++ * cannot have been visible to anyone if non-NULL, so we
++ * can just free it here.
++ */
++ kfree(rcu_dereference_raw(rdev->wiphy.regd));
++
+ kfree(rdev);
+ }
+
+diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
+index bf7cd47525472..7c65ad17bf50a 100644
+--- a/net/wireless/nl80211.c
++++ b/net/wireless/nl80211.c
+@@ -509,7 +509,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
+ .len = IEEE80211_MAX_MESH_ID_LEN },
+ [NL80211_ATTR_MPATH_NEXT_HOP] = NLA_POLICY_ETH_ADDR_COMPAT,
+
+- [NL80211_ATTR_REG_ALPHA2] = { .type = NLA_STRING, .len = 2 },
++ /* allow 3 for NUL-termination, we used to declare this NLA_STRING */
++ [NL80211_ATTR_REG_ALPHA2] = NLA_POLICY_RANGE(NLA_BINARY, 2, 3),
+ [NL80211_ATTR_REG_RULES] = { .type = NLA_NESTED },
+
+ [NL80211_ATTR_BSS_CTS_PROT] = { .type = NLA_U8 },
+@@ -922,33 +923,37 @@ nl80211_packet_pattern_policy[MAX_NL80211_PKTPAT + 1] = {
+ [NL80211_PKTPAT_OFFSET] = { .type = NLA_U32 },
+ };
+
+-int nl80211_prepare_wdev_dump(struct netlink_callback *cb,
+- struct cfg80211_registered_device **rdev,
+- struct wireless_dev **wdev)
++static int nl80211_prepare_wdev_dump(struct netlink_callback *cb,
++ struct cfg80211_registered_device **rdev,
++ struct wireless_dev **wdev,
++ struct nlattr **attrbuf)
+ {
+ int err;
+
+ if (!cb->args[0]) {
+- struct nlattr **attrbuf;
++ struct nlattr **attrbuf_free = NULL;
+
+- attrbuf = kcalloc(NUM_NL80211_ATTR, sizeof(*attrbuf),
+- GFP_KERNEL);
+- if (!attrbuf)
+- return -ENOMEM;
++ if (!attrbuf) {
++ attrbuf = kcalloc(NUM_NL80211_ATTR, sizeof(*attrbuf),
++ GFP_KERNEL);
++ if (!attrbuf)
++ return -ENOMEM;
++ attrbuf_free = attrbuf;
++ }
+
+ err = nlmsg_parse_deprecated(cb->nlh,
+ GENL_HDRLEN + nl80211_fam.hdrsize,
+ attrbuf, nl80211_fam.maxattr,
+ nl80211_policy, NULL);
+ if (err) {
+- kfree(attrbuf);
++ kfree(attrbuf_free);
+ return err;
+ }
+
+ rtnl_lock();
+ *wdev = __cfg80211_wdev_from_attrs(NULL, sock_net(cb->skb->sk),
+ attrbuf);
+- kfree(attrbuf);
++ kfree(attrbuf_free);
+ if (IS_ERR(*wdev)) {
+ rtnl_unlock();
+ return PTR_ERR(*wdev);
+@@ -3075,6 +3080,15 @@ int nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
+ } else if (attrs[NL80211_ATTR_CHANNEL_WIDTH]) {
+ chandef->width =
+ nla_get_u32(attrs[NL80211_ATTR_CHANNEL_WIDTH]);
++ if (chandef->chan->band == NL80211_BAND_S1GHZ) {
++ /* User input error for channel width doesn't match channel */
++ if (chandef->width != ieee80211_s1g_channel_width(chandef->chan)) {
++ NL_SET_ERR_MSG_ATTR(extack,
++ attrs[NL80211_ATTR_CHANNEL_WIDTH],
++ "bad channel width");
++ return -EINVAL;
++ }
++ }
+ if (attrs[NL80211_ATTR_CENTER_FREQ1]) {
+ chandef->center_freq1 =
+ nla_get_u32(attrs[NL80211_ATTR_CENTER_FREQ1]);
+@@ -6001,7 +6015,7 @@ static int nl80211_dump_station(struct sk_buff *skb,
+ int sta_idx = cb->args[2];
+ int err;
+
+- err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev);
++ err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL);
+ if (err)
+ return err;
+ /* nl80211_prepare_wdev_dump acquired it in the successful case */
+@@ -6896,7 +6910,7 @@ static int nl80211_dump_mpath(struct sk_buff *skb,
+ int path_idx = cb->args[2];
+ int err;
+
+- err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev);
++ err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL);
+ if (err)
+ return err;
+ /* nl80211_prepare_wdev_dump acquired it in the successful case */
+@@ -7096,7 +7110,7 @@ static int nl80211_dump_mpp(struct sk_buff *skb,
+ int path_idx = cb->args[2];
+ int err;
+
+- err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev);
++ err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL);
+ if (err)
+ return err;
+ /* nl80211_prepare_wdev_dump acquired it in the successful case */
+@@ -9518,7 +9532,7 @@ static int nl80211_dump_scan(struct sk_buff *skb, struct netlink_callback *cb)
+ int start = cb->args[2], idx = 0;
+ int err;
+
+- err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev);
++ err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL);
+ if (err)
+ return err;
+ /* nl80211_prepare_wdev_dump acquired it in the successful case */
+@@ -9651,7 +9665,7 @@ static int nl80211_dump_survey(struct sk_buff *skb, struct netlink_callback *cb)
+ if (!attrbuf)
+ return -ENOMEM;
+
+- res = nl80211_prepare_wdev_dump(cb, &rdev, &wdev);
++ res = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, attrbuf);
+ if (res) {
+ kfree(attrbuf);
+ return res;
+@@ -11327,18 +11341,23 @@ static int nl80211_set_tx_bitrate_mask(struct sk_buff *skb,
+ struct cfg80211_bitrate_mask mask;
+ struct cfg80211_registered_device *rdev = info->user_ptr[0];
+ struct net_device *dev = info->user_ptr[1];
++ struct wireless_dev *wdev = dev->ieee80211_ptr;
+ int err;
+
+ if (!rdev->ops->set_bitrate_mask)
+ return -EOPNOTSUPP;
+
++ wdev_lock(wdev);
+ err = nl80211_parse_tx_bitrate_mask(info, info->attrs,
+ NL80211_ATTR_TX_RATES, &mask,
+ dev, true);
+ if (err)
+- return err;
++ goto out;
+
+- return rdev_set_bitrate_mask(rdev, dev, NULL, &mask);
++ err = rdev_set_bitrate_mask(rdev, dev, NULL, &mask);
++out:
++ wdev_unlock(wdev);
++ return err;
+ }
+
+ static int nl80211_register_mgmt(struct sk_buff *skb, struct genl_info *info)
+@@ -13173,6 +13192,9 @@ static int handle_nan_filter(struct nlattr *attr_filter,
+ i = 0;
+ nla_for_each_nested(attr, attr_filter, rem) {
+ filter[i].filter = nla_memdup(attr, GFP_KERNEL);
++ if (!filter[i].filter)
++ goto err;
++
+ filter[i].len = nla_len(attr);
+ i++;
+ }
+@@ -13185,6 +13207,15 @@ static int handle_nan_filter(struct nlattr *attr_filter,
+ }
+
+ return 0;
++
++err:
++ i = 0;
++ nla_for_each_nested(attr, attr_filter, rem) {
++ kfree(filter[i].filter);
++ i++;
++ }
++ kfree(filter);
++ return -ENOMEM;
+ }
+
+ static int nl80211_nan_add_func(struct sk_buff *skb,
+@@ -17509,7 +17540,8 @@ void cfg80211_ch_switch_notify(struct net_device *dev,
+ wdev->chandef = *chandef;
+ wdev->preset_chandef = *chandef;
+
+- if (wdev->iftype == NL80211_IFTYPE_STATION &&
++ if ((wdev->iftype == NL80211_IFTYPE_STATION ||
++ wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) &&
+ !WARN_ON(!wdev->current_bss))
+ cfg80211_update_assoc_bss_entry(wdev, chandef->chan);
+
+diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h
+index a3f387770f1bf..d642e3be4ee78 100644
+--- a/net/wireless/nl80211.h
++++ b/net/wireless/nl80211.h
+@@ -1,7 +1,7 @@
+ /* SPDX-License-Identifier: GPL-2.0 */
+ /*
+ * Portions of this file
+- * Copyright (C) 2018, 2020 Intel Corporation
++ * Copyright (C) 2018, 2020-2021 Intel Corporation
+ */
+ #ifndef __NET_WIRELESS_NL80211_H
+ #define __NET_WIRELESS_NL80211_H
+@@ -22,10 +22,6 @@ static inline u64 wdev_id(struct wireless_dev *wdev)
+ ((u64)wiphy_to_rdev(wdev->wiphy)->wiphy_idx << 32);
+ }
+
+-int nl80211_prepare_wdev_dump(struct netlink_callback *cb,
+- struct cfg80211_registered_device **rdev,
+- struct wireless_dev **wdev);
+-
+ int nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
+ struct genl_info *info,
+ struct cfg80211_chan_def *chandef);
+diff --git a/net/wireless/reg.c b/net/wireless/reg.c
+index df87c7f3a0492..795e86b371bba 100644
+--- a/net/wireless/reg.c
++++ b/net/wireless/reg.c
+@@ -2338,6 +2338,7 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
+ struct cfg80211_chan_def chandef = {};
+ struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
+ enum nl80211_iftype iftype;
++ bool ret;
+
+ wdev_lock(wdev);
+ iftype = wdev->iftype;
+@@ -2387,7 +2388,11 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
+ case NL80211_IFTYPE_AP:
+ case NL80211_IFTYPE_P2P_GO:
+ case NL80211_IFTYPE_ADHOC:
+- return cfg80211_reg_can_beacon_relax(wiphy, &chandef, iftype);
++ wiphy_lock(wiphy);
++ ret = cfg80211_reg_can_beacon_relax(wiphy, &chandef, iftype);
++ wiphy_unlock(wiphy);
++
++ return ret;
+ case NL80211_IFTYPE_STATION:
+ case NL80211_IFTYPE_P2P_CLIENT:
+ return cfg80211_chandef_usable(wiphy, &chandef,
+diff --git a/net/wireless/scan.c b/net/wireless/scan.c
+index adc0d14cfd860..1a8b76c9dd567 100644
+--- a/net/wireless/scan.c
++++ b/net/wireless/scan.c
+@@ -702,8 +702,12 @@ static bool cfg80211_find_ssid_match(struct cfg80211_colocated_ap *ap,
+
+ for (i = 0; i < request->n_ssids; i++) {
+ /* wildcard ssid in the scan request */
+- if (!request->ssids[i].ssid_len)
++ if (!request->ssids[i].ssid_len) {
++ if (ap->multi_bss && !ap->transmitted_bssid)
++ continue;
++
+ return true;
++ }
+
+ if (ap->ssid_len &&
+ ap->ssid_len == request->ssids[i].ssid_len) {
+@@ -829,6 +833,9 @@ static int cfg80211_scan_6ghz(struct cfg80211_registered_device *rdev)
+ !cfg80211_find_ssid_match(ap, request))
+ continue;
+
++ if (!request->n_ssids && ap->multi_bss && !ap->transmitted_bssid)
++ continue;
++
+ cfg80211_scan_req_add_chan(request, chan, true);
+ memcpy(scan_6ghz_params->bssid, ap->bssid, ETH_ALEN);
+ scan_6ghz_params->short_ssid = ap->short_ssid;
+@@ -1971,11 +1978,13 @@ cfg80211_inform_single_bss_data(struct wiphy *wiphy,
+ /* this is a nontransmitting bss, we need to add it to
+ * transmitting bss' list if it is not there
+ */
++ spin_lock_bh(&rdev->bss_lock);
+ if (cfg80211_add_nontrans_list(non_tx_data->tx_bss,
+ &res->pub)) {
+ if (__cfg80211_unlink_bss(rdev, res))
+ rdev->bss_generation++;
+ }
++ spin_unlock_bh(&rdev->bss_lock);
+ }
+
+ trace_cfg80211_return_bss(&res->pub);
+diff --git a/net/wireless/util.c b/net/wireless/util.c
+index a1a99a5749844..4ddc269164f86 100644
+--- a/net/wireless/util.c
++++ b/net/wireless/util.c
+@@ -1044,6 +1044,7 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev,
+
+ switch (otype) {
+ case NL80211_IFTYPE_AP:
++ case NL80211_IFTYPE_P2P_GO:
+ cfg80211_stop_ap(rdev, dev, true);
+ break;
+ case NL80211_IFTYPE_ADHOC:
+diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c
+index 3583354a7d7fe..3a171828638b1 100644
+--- a/net/x25/af_x25.c
++++ b/net/x25/af_x25.c
+@@ -1765,10 +1765,15 @@ void x25_kill_by_neigh(struct x25_neigh *nb)
+
+ write_lock_bh(&x25_list_lock);
+
+- sk_for_each(s, &x25_list)
+- if (x25_sk(s)->neighbour == nb)
++ sk_for_each(s, &x25_list) {
++ if (x25_sk(s)->neighbour == nb) {
++ write_unlock_bh(&x25_list_lock);
++ lock_sock(s);
+ x25_disconnect(s, ENETUNREACH, 0, 0);
+-
++ release_sock(s);
++ write_lock_bh(&x25_list_lock);
++ }
++ }
+ write_unlock_bh(&x25_list_lock);
+
+ /* Remove any related forwards */
+diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
+index d6b500dc42084..444ad0bc09083 100644
+--- a/net/xdp/xsk.c
++++ b/net/xdp/xsk.c
+@@ -418,18 +418,8 @@ EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch);
+ static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
+ {
+ struct net_device *dev = xs->dev;
+- int err;
+-
+- rcu_read_lock();
+- err = dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
+- rcu_read_unlock();
+-
+- return err;
+-}
+
+-static int xsk_zc_xmit(struct xdp_sock *xs)
+-{
+- return xsk_wakeup(xs, XDP_WAKEUP_TX);
++ return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
+ }
+
+ static void xsk_destruct_skb(struct sk_buff *skb)
+@@ -548,6 +538,12 @@ static int xsk_generic_xmit(struct sock *sk)
+
+ mutex_lock(&xs->mutex);
+
++ /* Since we dropped the RCU read lock, the socket state might have changed. */
++ if (unlikely(!xsk_is_bound(xs))) {
++ err = -ENXIO;
++ goto out;
++ }
++
+ if (xs->queue_id >= xs->dev->real_num_tx_queues)
+ goto out;
+
+@@ -611,16 +607,26 @@ out:
+ return err;
+ }
+
+-static int __xsk_sendmsg(struct sock *sk)
++static int xsk_xmit(struct sock *sk)
+ {
+ struct xdp_sock *xs = xdp_sk(sk);
++ int ret;
+
+ if (unlikely(!(xs->dev->flags & IFF_UP)))
+ return -ENETDOWN;
+ if (unlikely(!xs->tx))
+ return -ENOBUFS;
+
+- return xs->zc ? xsk_zc_xmit(xs) : xsk_generic_xmit(sk);
++ if (xs->zc)
++ return xsk_wakeup(xs, XDP_WAKEUP_TX);
++
++ /* Drop the RCU lock since the SKB path might sleep. */
++ rcu_read_unlock();
++ ret = xsk_generic_xmit(sk);
++ /* Reaquire RCU lock before going into common code. */
++ rcu_read_lock();
++
++ return ret;
+ }
+
+ static bool xsk_no_wakeup(struct sock *sk)
+@@ -634,7 +640,7 @@ static bool xsk_no_wakeup(struct sock *sk)
+ #endif
+ }
+
+-static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
++static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
+ {
+ bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
+ struct sock *sk = sock->sk;
+@@ -649,16 +655,27 @@ static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
+ if (sk_can_busy_loop(sk))
+ sk_busy_loop(sk, 1); /* only support non-blocking sockets */
+
+- if (xsk_no_wakeup(sk))
++ if (xs->zc && xsk_no_wakeup(sk))
+ return 0;
+
+ pool = xs->pool;
+ if (pool->cached_need_wakeup & XDP_WAKEUP_TX)
+- return __xsk_sendmsg(sk);
++ return xsk_xmit(sk);
+ return 0;
+ }
+
+-static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
++static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
++{
++ int ret;
++
++ rcu_read_lock();
++ ret = __xsk_sendmsg(sock, m, total_len);
++ rcu_read_unlock();
++
++ return ret;
++}
++
++static int __xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
+ {
+ bool need_wait = !(flags & MSG_DONTWAIT);
+ struct sock *sk = sock->sk;
+@@ -684,6 +701,17 @@ static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int fl
+ return 0;
+ }
+
++static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
++{
++ int ret;
++
++ rcu_read_lock();
++ ret = __xsk_recvmsg(sock, m, len, flags);
++ rcu_read_unlock();
++
++ return ret;
++}
++
+ static __poll_t xsk_poll(struct file *file, struct socket *sock,
+ struct poll_table_struct *wait)
+ {
+@@ -694,8 +722,11 @@ static __poll_t xsk_poll(struct file *file, struct socket *sock,
+
+ sock_poll_wait(file, sock, wait);
+
+- if (unlikely(!xsk_is_bound(xs)))
++ rcu_read_lock();
++ if (unlikely(!xsk_is_bound(xs))) {
++ rcu_read_unlock();
+ return mask;
++ }
+
+ pool = xs->pool;
+
+@@ -704,7 +735,7 @@ static __poll_t xsk_poll(struct file *file, struct socket *sock,
+ xsk_wakeup(xs, pool->cached_need_wakeup);
+ else
+ /* Poll needs to drive Tx also in copy mode */
+- __xsk_sendmsg(sk);
++ xsk_xmit(sk);
+ }
+
+ if (xs->rx && !xskq_prod_is_empty(xs->rx))
+@@ -712,6 +743,7 @@ static __poll_t xsk_poll(struct file *file, struct socket *sock,
+ if (xs->tx && xsk_tx_writeable(xs))
+ mask |= EPOLLOUT | EPOLLWRNORM;
+
++ rcu_read_unlock();
+ return mask;
+ }
+
+@@ -743,7 +775,6 @@ static void xsk_unbind_dev(struct xdp_sock *xs)
+
+ /* Wait for driver to stop using the xdp socket. */
+ xp_del_xsk(xs->pool, xs);
+- xs->dev = NULL;
+ synchronize_net();
+ dev_put(dev);
+ }
+diff --git a/net/xfrm/xfrm_compat.c b/net/xfrm/xfrm_compat.c
+index 2bf2693901631..a0f62fa02e06e 100644
+--- a/net/xfrm/xfrm_compat.c
++++ b/net/xfrm/xfrm_compat.c
+@@ -127,6 +127,7 @@ static const struct nla_policy compat_policy[XFRMA_MAX+1] = {
+ [XFRMA_SET_MARK] = { .type = NLA_U32 },
+ [XFRMA_SET_MARK_MASK] = { .type = NLA_U32 },
+ [XFRMA_IF_ID] = { .type = NLA_U32 },
++ [XFRMA_MTIMER_THRESH] = { .type = NLA_U32 },
+ };
+
+ static struct nlmsghdr *xfrm_nlmsg_put_compat(struct sk_buff *skb,
+@@ -274,9 +275,10 @@ static int xfrm_xlate64_attr(struct sk_buff *dst, const struct nlattr *src)
+ case XFRMA_SET_MARK:
+ case XFRMA_SET_MARK_MASK:
+ case XFRMA_IF_ID:
++ case XFRMA_MTIMER_THRESH:
+ return xfrm_nla_cpy(dst, src, nla_len(src));
+ default:
+- BUILD_BUG_ON(XFRMA_MAX != XFRMA_IF_ID);
++ BUILD_BUG_ON(XFRMA_MAX != XFRMA_MTIMER_THRESH);
+ pr_warn_once("unsupported nla_type %d\n", src->nla_type);
+ return -EOPNOTSUPP;
+ }
+@@ -431,7 +433,7 @@ static int xfrm_xlate32_attr(void *dst, const struct nlattr *nla,
+ int err;
+
+ if (type > XFRMA_MAX) {
+- BUILD_BUG_ON(XFRMA_MAX != XFRMA_IF_ID);
++ BUILD_BUG_ON(XFRMA_MAX != XFRMA_MTIMER_THRESH);
+ NL_SET_ERR_MSG(extack, "Bad attribute");
+ return -EOPNOTSUPP;
+ }
+diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
+index e843b0d9e2a61..c255aac6b816b 100644
+--- a/net/xfrm/xfrm_device.c
++++ b/net/xfrm/xfrm_device.c
+@@ -223,6 +223,9 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
+ if (x->encap || x->tfcpad)
+ return -EINVAL;
+
++ if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND))
++ return -EINVAL;
++
+ dev = dev_get_by_index(net, xuo->ifindex);
+ if (!dev) {
+ if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
+@@ -261,7 +264,8 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
+ xso->dev = dev;
+ xso->real_dev = dev;
+ xso->num_exthdrs = 1;
+- xso->flags = xuo->flags;
++ /* Don't forward bit that is not implemented */
++ xso->flags = xuo->flags & ~XFRM_OFFLOAD_IPV6;
+
+ err = dev->xfrmdev_ops->xdo_dev_state_add(x);
+ if (err) {
+diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c
+index 41de46b5ffa94..1e8b26eecb3f8 100644
+--- a/net/xfrm/xfrm_interface.c
++++ b/net/xfrm/xfrm_interface.c
+@@ -304,7 +304,10 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
+ if (mtu < IPV6_MIN_MTU)
+ mtu = IPV6_MIN_MTU;
+
+- icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
++ if (skb->len > 1280)
++ icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
++ else
++ goto xmit;
+ } else {
+ if (!(ip_hdr(skb)->frag_off & htons(IP_DF)))
+ goto xmit;
+@@ -637,11 +640,16 @@ static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
+ struct netlink_ext_ack *extack)
+ {
+ struct net *net = dev_net(dev);
+- struct xfrm_if_parms p;
++ struct xfrm_if_parms p = {};
+ struct xfrm_if *xi;
+ int err;
+
+ xfrmi_netlink_parms(data, &p);
++ if (!p.if_id) {
++ NL_SET_ERR_MSG(extack, "if_id must be non zero");
++ return -EINVAL;
++ }
++
+ xi = xfrmi_locate(net, &p);
+ if (xi)
+ return -EEXIST;
+@@ -666,9 +674,14 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
+ {
+ struct xfrm_if *xi = netdev_priv(dev);
+ struct net *net = xi->net;
+- struct xfrm_if_parms p;
++ struct xfrm_if_parms p = {};
+
+ xfrmi_netlink_parms(data, &p);
++ if (!p.if_id) {
++ NL_SET_ERR_MSG(extack, "if_id must be non zero");
++ return -EINVAL;
++ }
++
+ xi = xfrmi_locate(net, &p);
+ if (!xi) {
+ xi = netdev_priv(dev);
+diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
+index 229544bc70c21..4dc4a7bbe51cf 100644
+--- a/net/xfrm/xfrm_output.c
++++ b/net/xfrm/xfrm_output.c
+@@ -647,10 +647,12 @@ static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb
+ * This requires hardware to know the inner packet type to calculate
+ * the inner header checksum. Save inner ip protocol here to avoid
+ * traversing the packet in the vendor's xmit code.
+- * If the encap type is IPIP, just save skb->inner_ipproto. Otherwise,
+- * get the ip protocol from the IP header.
++ * For IPsec tunnel mode save the ip protocol from the IP header of the
++ * plain text packet. Otherwise If the encap type is IPIP, just save
++ * skb->inner_ipproto in any other case get the ip protocol from the IP
++ * header.
+ */
+-static void xfrm_get_inner_ipproto(struct sk_buff *skb)
++static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x)
+ {
+ struct xfrm_offload *xo = xfrm_offload(skb);
+ const struct ethhdr *eth;
+@@ -658,6 +660,25 @@ static void xfrm_get_inner_ipproto(struct sk_buff *skb)
+ if (!xo)
+ return;
+
++ if (x->outer_mode.encap == XFRM_MODE_TUNNEL) {
++ switch (x->outer_mode.family) {
++ case AF_INET:
++ xo->inner_ipproto = ip_hdr(skb)->protocol;
++ break;
++ case AF_INET6:
++ xo->inner_ipproto = ipv6_hdr(skb)->nexthdr;
++ break;
++ default:
++ break;
++ }
++
++ return;
++ }
++
++ /* non-Tunnel Mode */
++ if (!skb->encapsulation)
++ return;
++
+ if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) {
+ xo->inner_ipproto = skb->inner_ipproto;
+ return;
+@@ -712,8 +733,7 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb)
+ sp->xvec[sp->len++] = x;
+ xfrm_state_hold(x);
+
+- if (skb->encapsulation)
+- xfrm_get_inner_ipproto(skb);
++ xfrm_get_inner_ipproto(skb, x);
+ skb->encapsulation = 1;
+
+ if (skb_is_gso(skb)) {
+diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
+index 37d17a79617c9..a6271b955e11d 100644
+--- a/net/xfrm/xfrm_policy.c
++++ b/net/xfrm/xfrm_policy.c
+@@ -31,8 +31,10 @@
+ #include <linux/if_tunnel.h>
+ #include <net/dst.h>
+ #include <net/flow.h>
++#include <net/inet_ecn.h>
+ #include <net/xfrm.h>
+ #include <net/ip.h>
++#include <net/gre.h>
+ #if IS_ENABLED(CONFIG_IPV6_MIP6)
+ #include <net/mip6.h>
+ #endif
+@@ -3158,7 +3160,7 @@ ok:
+
+ nopol:
+ if (!(dst_orig->dev->flags & IFF_LOOPBACK) &&
+- !xfrm_default_allow(net, dir)) {
++ net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
+ err = -EPERM;
+ goto error;
+ }
+@@ -3296,7 +3298,7 @@ decode_session4(struct sk_buff *skb, struct flowi *fl, bool reverse)
+ fl4->flowi4_proto = iph->protocol;
+ fl4->daddr = reverse ? iph->saddr : iph->daddr;
+ fl4->saddr = reverse ? iph->daddr : iph->saddr;
+- fl4->flowi4_tos = iph->tos;
++ fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK;
+
+ if (!ip_is_fragment(iph)) {
+ switch (iph->protocol) {
+@@ -3424,6 +3426,26 @@ decode_session6(struct sk_buff *skb, struct flowi *fl, bool reverse)
+ }
+ fl6->flowi6_proto = nexthdr;
+ return;
++ case IPPROTO_GRE:
++ if (!onlyproto &&
++ (nh + offset + 12 < skb->data ||
++ pskb_may_pull(skb, nh + offset + 12 - skb->data))) {
++ struct gre_base_hdr *gre_hdr;
++ __be32 *gre_key;
++
++ nh = skb_network_header(skb);
++ gre_hdr = (struct gre_base_hdr *)(nh + offset);
++ gre_key = (__be32 *)(gre_hdr + 1);
++
++ if (gre_hdr->flags & GRE_KEY) {
++ if (gre_hdr->flags & GRE_CSUM)
++ gre_key++;
++ fl6->fl6_gre_key = *gre_key;
++ }
++ }
++ fl6->flowi6_proto = nexthdr;
++ return;
++
+ #if IS_ENABLED(CONFIG_IPV6_MIP6)
+ case IPPROTO_MH:
+ offset += ipv6_optlen(exthdr);
+@@ -3550,7 +3572,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
+ }
+
+ if (!pol) {
+- if (!xfrm_default_allow(net, dir)) {
++ if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
+ return 0;
+ }
+@@ -3610,7 +3632,8 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
+ }
+ xfrm_nr = ti;
+
+- if (!xfrm_default_allow(net, dir) && !xfrm_nr) {
++ if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK &&
++ !xfrm_nr) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
+ goto reject;
+ }
+@@ -4099,6 +4122,9 @@ static int __net_init xfrm_net_init(struct net *net)
+ spin_lock_init(&net->xfrm.xfrm_policy_lock);
+ seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
+ mutex_init(&net->xfrm.xfrm_cfg_mutex);
++ net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
++ net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
++ net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
+
+ rv = xfrm_statistics_init(net);
+ if (rv < 0)
+@@ -4237,7 +4263,7 @@ static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
+ }
+
+ static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
+- u8 dir, u8 type, struct net *net)
++ u8 dir, u8 type, struct net *net, u32 if_id)
+ {
+ struct xfrm_policy *pol, *ret = NULL;
+ struct hlist_head *chain;
+@@ -4246,7 +4272,8 @@ static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *
+ spin_lock_bh(&net->xfrm.xfrm_policy_lock);
+ chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
+ hlist_for_each_entry(pol, chain, bydst) {
+- if (xfrm_migrate_selector_match(sel, &pol->selector) &&
++ if ((if_id == 0 || pol->if_id == if_id) &&
++ xfrm_migrate_selector_match(sel, &pol->selector) &&
+ pol->type == type) {
+ ret = pol;
+ priority = ret->priority;
+@@ -4258,7 +4285,8 @@ static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *
+ if ((pol->priority >= priority) && ret)
+ break;
+
+- if (xfrm_migrate_selector_match(sel, &pol->selector) &&
++ if ((if_id == 0 || pol->if_id == if_id) &&
++ xfrm_migrate_selector_match(sel, &pol->selector) &&
+ pol->type == type) {
+ ret = pol;
+ break;
+@@ -4374,7 +4402,7 @@ static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
+ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ struct xfrm_migrate *m, int num_migrate,
+ struct xfrm_kmaddress *k, struct net *net,
+- struct xfrm_encap_tmpl *encap)
++ struct xfrm_encap_tmpl *encap, u32 if_id)
+ {
+ int i, err, nx_cur = 0, nx_new = 0;
+ struct xfrm_policy *pol = NULL;
+@@ -4393,14 +4421,14 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
+ }
+
+ /* Stage 1 - find policy */
+- if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {
++ if ((pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id)) == NULL) {
+ err = -ENOENT;
+ goto out;
+ }
+
+ /* Stage 2 - find and update state(s) */
+ for (i = 0, mp = m; i < num_migrate; i++, mp++) {
+- if ((x = xfrm_migrate_state_find(mp, net))) {
++ if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
+ x_cur[nx_cur] = x;
+ nx_cur++;
+ xc = xfrm_state_migrate(x, mp, encap);
+diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
+index a2f4001221d16..f7bfa19169688 100644
+--- a/net/xfrm/xfrm_state.c
++++ b/net/xfrm/xfrm_state.c
+@@ -1578,9 +1578,6 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
+ memcpy(&x->mark, &orig->mark, sizeof(x->mark));
+ memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
+
+- if (xfrm_init_state(x) < 0)
+- goto error;
+-
+ x->props.flags = orig->props.flags;
+ x->props.extra_flags = orig->props.extra_flags;
+
+@@ -1593,6 +1590,9 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
+ x->km.seq = orig->km.seq;
+ x->replay = orig->replay;
+ x->preplay = orig->preplay;
++ x->mapping_maxage = orig->mapping_maxage;
++ x->new_mapping = 0;
++ x->new_mapping_sport = 0;
+
+ return x;
+
+@@ -1602,7 +1602,8 @@ out:
+ return NULL;
+ }
+
+-struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net)
++struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
++ u32 if_id)
+ {
+ unsigned int h;
+ struct xfrm_state *x = NULL;
+@@ -1618,6 +1619,8 @@ struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *n
+ continue;
+ if (m->reqid && x->props.reqid != m->reqid)
+ continue;
++ if (if_id != 0 && x->if_id != if_id)
++ continue;
+ if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
+ m->old_family) ||
+ !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
+@@ -1633,6 +1636,8 @@ struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *n
+ if (x->props.mode != m->mode ||
+ x->id.proto != m->proto)
+ continue;
++ if (if_id != 0 && x->if_id != if_id)
++ continue;
+ if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
+ m->old_family) ||
+ !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
+@@ -1659,6 +1664,11 @@ struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
+ if (!xc)
+ return NULL;
+
++ xc->props.family = m->new_family;
++
++ if (xfrm_init_state(xc) < 0)
++ goto error;
++
+ memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
+ memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
+
+@@ -2242,7 +2252,7 @@ int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
+ }
+ EXPORT_SYMBOL(km_query);
+
+-int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
++static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
+ {
+ int err = -EINVAL;
+ struct xfrm_mgr *km;
+@@ -2257,6 +2267,24 @@ int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
+ rcu_read_unlock();
+ return err;
+ }
++
++int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
++{
++ int ret = 0;
++
++ if (x->mapping_maxage) {
++ if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
++ x->new_mapping_sport != sport) {
++ x->new_mapping_sport = sport;
++ x->new_mapping = jiffies / HZ;
++ ret = __km_new_mapping(x, ipaddr, sport);
++ }
++ } else {
++ ret = __km_new_mapping(x, ipaddr, sport);
++ }
++
++ return ret;
++}
+ EXPORT_SYMBOL(km_new_mapping);
+
+ void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
+@@ -2550,7 +2578,7 @@ void xfrm_state_delete_tunnel(struct xfrm_state *x)
+ }
+ EXPORT_SYMBOL(xfrm_state_delete_tunnel);
+
+-u32 __xfrm_state_mtu(struct xfrm_state *x, int mtu)
++u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
+ {
+ const struct xfrm_type *type = READ_ONCE(x->type);
+ struct crypto_aead *aead;
+@@ -2581,17 +2609,7 @@ u32 __xfrm_state_mtu(struct xfrm_state *x, int mtu)
+ return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
+ net_adj) & ~(blksize - 1)) + net_adj - 2;
+ }
+-EXPORT_SYMBOL_GPL(__xfrm_state_mtu);
+-
+-u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
+-{
+- mtu = __xfrm_state_mtu(x, mtu);
+-
+- if (x->props.family == AF_INET6 && mtu < IPV6_MIN_MTU)
+- return IPV6_MIN_MTU;
+-
+- return mtu;
+-}
++EXPORT_SYMBOL_GPL(xfrm_state_mtu);
+
+ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
+ {
+diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
+index 3a3cb09eec122..5fba82757ce5e 100644
+--- a/net/xfrm/xfrm_user.c
++++ b/net/xfrm/xfrm_user.c
+@@ -282,6 +282,10 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
+
+ err = 0;
+
++ if (attrs[XFRMA_MTIMER_THRESH])
++ if (!attrs[XFRMA_ENCAP])
++ err = -EINVAL;
++
+ out:
+ return err;
+ }
+@@ -521,6 +525,7 @@ static void xfrm_update_ae_params(struct xfrm_state *x, struct nlattr **attrs,
+ struct nlattr *lt = attrs[XFRMA_LTIME_VAL];
+ struct nlattr *et = attrs[XFRMA_ETIMER_THRESH];
+ struct nlattr *rt = attrs[XFRMA_REPLAY_THRESH];
++ struct nlattr *mt = attrs[XFRMA_MTIMER_THRESH];
+
+ if (re) {
+ struct xfrm_replay_state_esn *replay_esn;
+@@ -552,6 +557,9 @@ static void xfrm_update_ae_params(struct xfrm_state *x, struct nlattr **attrs,
+
+ if (rt)
+ x->replay_maxdiff = nla_get_u32(rt);
++
++ if (mt)
++ x->mapping_maxage = nla_get_u32(mt);
+ }
+
+ static void xfrm_smark_init(struct nlattr **attrs, struct xfrm_mark *m)
+@@ -1024,8 +1032,13 @@ static int copy_to_user_state_extra(struct xfrm_state *x,
+ if (ret)
+ goto out;
+ }
+- if (x->security)
++ if (x->security) {
+ ret = copy_sec_ctx(x->security, skb);
++ if (ret)
++ goto out;
++ }
++ if (x->mapping_maxage)
++ ret = nla_put_u32(skb, XFRMA_MTIMER_THRESH, x->mapping_maxage);
+ out:
+ return ret;
+ }
+@@ -1980,12 +1993,9 @@ static int xfrm_notify_userpolicy(struct net *net)
+ }
+
+ up = nlmsg_data(nlh);
+- up->in = net->xfrm.policy_default & XFRM_POL_DEFAULT_IN ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
+- up->fwd = net->xfrm.policy_default & XFRM_POL_DEFAULT_FWD ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
+- up->out = net->xfrm.policy_default & XFRM_POL_DEFAULT_OUT ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
++ up->in = net->xfrm.policy_default[XFRM_POLICY_IN];
++ up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD];
++ up->out = net->xfrm.policy_default[XFRM_POLICY_OUT];
+
+ nlmsg_end(skb, nlh);
+
+@@ -1996,26 +2006,26 @@ static int xfrm_notify_userpolicy(struct net *net)
+ return err;
+ }
+
++static bool xfrm_userpolicy_is_valid(__u8 policy)
++{
++ return policy == XFRM_USERPOLICY_BLOCK ||
++ policy == XFRM_USERPOLICY_ACCEPT;
++}
++
+ static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct nlattr **attrs)
+ {
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_userpolicy_default *up = nlmsg_data(nlh);
+
+- if (up->in == XFRM_USERPOLICY_BLOCK)
+- net->xfrm.policy_default |= XFRM_POL_DEFAULT_IN;
+- else if (up->in == XFRM_USERPOLICY_ACCEPT)
+- net->xfrm.policy_default &= ~XFRM_POL_DEFAULT_IN;
++ if (xfrm_userpolicy_is_valid(up->in))
++ net->xfrm.policy_default[XFRM_POLICY_IN] = up->in;
+
+- if (up->fwd == XFRM_USERPOLICY_BLOCK)
+- net->xfrm.policy_default |= XFRM_POL_DEFAULT_FWD;
+- else if (up->fwd == XFRM_USERPOLICY_ACCEPT)
+- net->xfrm.policy_default &= ~XFRM_POL_DEFAULT_FWD;
++ if (xfrm_userpolicy_is_valid(up->fwd))
++ net->xfrm.policy_default[XFRM_POLICY_FWD] = up->fwd;
+
+- if (up->out == XFRM_USERPOLICY_BLOCK)
+- net->xfrm.policy_default |= XFRM_POL_DEFAULT_OUT;
+- else if (up->out == XFRM_USERPOLICY_ACCEPT)
+- net->xfrm.policy_default &= ~XFRM_POL_DEFAULT_OUT;
++ if (xfrm_userpolicy_is_valid(up->out))
++ net->xfrm.policy_default[XFRM_POLICY_OUT] = up->out;
+
+ rt_genid_bump_all(net);
+
+@@ -2045,13 +2055,9 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh,
+ }
+
+ r_up = nlmsg_data(r_nlh);
+-
+- r_up->in = net->xfrm.policy_default & XFRM_POL_DEFAULT_IN ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
+- r_up->fwd = net->xfrm.policy_default & XFRM_POL_DEFAULT_FWD ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
+- r_up->out = net->xfrm.policy_default & XFRM_POL_DEFAULT_OUT ?
+- XFRM_USERPOLICY_BLOCK : XFRM_USERPOLICY_ACCEPT;
++ r_up->in = net->xfrm.policy_default[XFRM_POLICY_IN];
++ r_up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD];
++ r_up->out = net->xfrm.policy_default[XFRM_POLICY_OUT];
+ nlmsg_end(r_skb, r_nlh);
+
+ return nlmsg_unicast(net->xfrm.nlsk, r_skb, portid);
+@@ -2579,6 +2585,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
+ int n = 0;
+ struct net *net = sock_net(skb->sk);
+ struct xfrm_encap_tmpl *encap = NULL;
++ u32 if_id = 0;
+
+ if (attrs[XFRMA_MIGRATE] == NULL)
+ return -EINVAL;
+@@ -2603,7 +2610,10 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
+ return -ENOMEM;
+ }
+
+- err = xfrm_migrate(&pi->sel, pi->dir, type, m, n, kmp, net, encap);
++ if (attrs[XFRMA_IF_ID])
++ if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
++
++ err = xfrm_migrate(&pi->sel, pi->dir, type, m, n, kmp, net, encap, if_id);
+
+ kfree(encap);
+
+@@ -3058,7 +3068,7 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x)
+ if (x->props.extra_flags)
+ l += nla_total_size(sizeof(x->props.extra_flags));
+ if (x->xso.dev)
+- l += nla_total_size(sizeof(x->xso));
++ l += nla_total_size(sizeof(struct xfrm_user_offload));
+ if (x->props.smark.v | x->props.smark.m) {
+ l += nla_total_size(sizeof(x->props.smark.v));
+ l += nla_total_size(sizeof(x->props.smark.m));
+@@ -3069,6 +3079,9 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x)
+ /* Must count x->lastused as it may become non-zero behind our back. */
+ l += nla_total_size_64bit(sizeof(u64));
+
++ if (x->mapping_maxage)
++ l += nla_total_size(sizeof(x->mapping_maxage));
++
+ return l;
+ }
+
+diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
+index 5fd48a8d4f10a..c6e38e43c3fd0 100644
+--- a/samples/bpf/Makefile
++++ b/samples/bpf/Makefile
+@@ -59,7 +59,11 @@ tprogs-y += xdp_redirect
+ tprogs-y += xdp_monitor
+
+ # Libbpf dependencies
+-LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a
++LIBBPF_SRC = $(TOOLS_PATH)/lib/bpf
++LIBBPF_OUTPUT = $(abspath $(BPF_SAMPLES_PATH))/libbpf
++LIBBPF_DESTDIR = $(LIBBPF_OUTPUT)
++LIBBPF_INCLUDE = $(LIBBPF_DESTDIR)/include
++LIBBPF = $(LIBBPF_OUTPUT)/libbpf.a
+
+ CGROUP_HELPERS := ../../tools/testing/selftests/bpf/cgroup_helpers.o
+ TRACE_HELPERS := ../../tools/testing/selftests/bpf/trace_helpers.o
+@@ -198,7 +202,7 @@ TPROGS_CFLAGS += -Wstrict-prototypes
+
+ TPROGS_CFLAGS += -I$(objtree)/usr/include
+ TPROGS_CFLAGS += -I$(srctree)/tools/testing/selftests/bpf/
+-TPROGS_CFLAGS += -I$(srctree)/tools/lib/
++TPROGS_CFLAGS += -I$(LIBBPF_INCLUDE)
+ TPROGS_CFLAGS += -I$(srctree)/tools/include
+ TPROGS_CFLAGS += -I$(srctree)/tools/perf
+ TPROGS_CFLAGS += -DHAVE_ATTR_TEST=0
+@@ -209,6 +213,11 @@ TPROGS_LDFLAGS := -L$(SYSROOT)/usr/lib
+ endif
+
+ TPROGS_LDLIBS += $(LIBBPF) -lelf -lz
++TPROGLDLIBS_xdp_monitor += -lm
++TPROGLDLIBS_xdp_redirect += -lm
++TPROGLDLIBS_xdp_redirect_cpu += -lm
++TPROGLDLIBS_xdp_redirect_map += -lm
++TPROGLDLIBS_xdp_redirect_map_multi += -lm
+ TPROGLDLIBS_tracex4 += -lrt
+ TPROGLDLIBS_trace_output += -lrt
+ TPROGLDLIBS_map_perf_test += -lrt
+@@ -268,16 +277,27 @@ all:
+ clean:
+ $(MAKE) -C ../../ M=$(CURDIR) clean
+ @find $(CURDIR) -type f -name '*~' -delete
++ @$(RM) -r $(CURDIR)/libbpf $(CURDIR)/bpftool
+
+-$(LIBBPF): FORCE
++$(LIBBPF): FORCE | $(LIBBPF_OUTPUT)
+ # Fix up variables inherited from Kbuild that tools/ build system won't like
+- $(MAKE) -C $(dir $@) RM='rm -rf' EXTRA_CFLAGS="$(TPROGS_CFLAGS)" \
+- LDFLAGS=$(TPROGS_LDFLAGS) srctree=$(BPF_SAMPLES_PATH)/../../ O=
++ $(MAKE) -C $(LIBBPF_SRC) RM='rm -rf' EXTRA_CFLAGS="$(TPROGS_CFLAGS)" \
++ LDFLAGS=$(TPROGS_LDFLAGS) srctree=$(BPF_SAMPLES_PATH)/../../ \
++ O= OUTPUT=$(LIBBPF_OUTPUT)/ DESTDIR=$(LIBBPF_DESTDIR) prefix= \
++ $@ install_headers
+
+ BPFTOOLDIR := $(TOOLS_PATH)/bpf/bpftool
+-BPFTOOL := $(BPFTOOLDIR)/bpftool
+-$(BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile)
+- $(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../
++BPFTOOL_OUTPUT := $(abspath $(BPF_SAMPLES_PATH))/bpftool
++BPFTOOL := $(BPFTOOL_OUTPUT)/bpftool
++$(BPFTOOL): $(LIBBPF) $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) | $(BPFTOOL_OUTPUT)
++ $(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../ \
++ OUTPUT=$(BPFTOOL_OUTPUT)/ \
++ LIBBPF_OUTPUT=$(LIBBPF_OUTPUT)/ \
++ LIBBPF_DESTDIR=$(LIBBPF_DESTDIR)/
++
++$(LIBBPF_OUTPUT) $(BPFTOOL_OUTPUT):
++ $(call msg,MKDIR,$@)
++ $(Q)mkdir -p $@
+
+ $(obj)/syscall_nrs.h: $(obj)/syscall_nrs.s FORCE
+ $(call filechk,offsets,__SYSCALL_NRS_H__)
+@@ -309,6 +329,11 @@ verify_target_bpf: verify_cmds
+ $(BPF_SAMPLES_PATH)/*.c: verify_target_bpf $(LIBBPF)
+ $(src)/*.c: verify_target_bpf $(LIBBPF)
+
++libbpf_hdrs: $(LIBBPF)
++$(obj)/$(TRACE_HELPERS) $(obj)/$(CGROUP_HELPERS) $(obj)/$(XDP_SAMPLE): | libbpf_hdrs
++
++.PHONY: libbpf_hdrs
++
+ $(obj)/xdp_redirect_cpu_user.o: $(obj)/xdp_redirect_cpu.skel.h
+ $(obj)/xdp_redirect_map_multi_user.o: $(obj)/xdp_redirect_map_multi.skel.h
+ $(obj)/xdp_redirect_map_user.o: $(obj)/xdp_redirect_map.skel.h
+@@ -320,6 +345,17 @@ $(obj)/hbm_out_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
+ $(obj)/hbm.o: $(src)/hbm.h
+ $(obj)/hbm_edt_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
+
++# Override includes for xdp_sample_user.o because $(srctree)/usr/include in
++# TPROGS_CFLAGS causes conflicts
++XDP_SAMPLE_CFLAGS += -Wall -O2 \
++ -I$(src)/../../tools/include \
++ -I$(src)/../../tools/include/uapi \
++ -I$(LIBBPF_INCLUDE) \
++ -I$(src)/../../tools/testing/selftests/bpf
++
++$(obj)/$(XDP_SAMPLE): TPROGS_CFLAGS = $(XDP_SAMPLE_CFLAGS)
++$(obj)/$(XDP_SAMPLE): $(src)/xdp_sample_user.h $(src)/xdp_sample_shared.h
++
+ -include $(BPF_SAMPLES_PATH)/Makefile.target
+
+ VMLINUX_BTF_PATHS ?= $(abspath $(if $(O),$(O)/vmlinux)) \
+@@ -366,7 +402,7 @@ $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/x
+ $(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(SRCARCH) \
+ -Wno-compare-distinct-pointer-types -I$(srctree)/include \
+ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \
+- -I$(srctree)/tools/lib $(CLANG_SYS_INCLUDES) \
++ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \
+ -c $(filter %.bpf.c,$^) -o $@
+
+ LINKED_SKELS := xdp_redirect_cpu.skel.h xdp_redirect_map_multi.skel.h \
+@@ -403,7 +439,7 @@ $(obj)/%.o: $(src)/%.c
+ @echo " CLANG-bpf " $@
+ $(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \
+ -I$(obj) -I$(srctree)/tools/testing/selftests/bpf/ \
+- -I$(srctree)/tools/lib/ \
++ -I$(LIBBPF_INCLUDE) \
+ -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
+ -D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \
+ -Wno-gnu-variable-sized-type-not-at-end \
+diff --git a/samples/bpf/Makefile.target b/samples/bpf/Makefile.target
+index 5a368affa0386..7621f55e2947d 100644
+--- a/samples/bpf/Makefile.target
++++ b/samples/bpf/Makefile.target
+@@ -73,14 +73,3 @@ quiet_cmd_tprog-cobjs = CC $@
+ cmd_tprog-cobjs = $(CC) $(tprogc_flags) -c -o $@ $<
+ $(tprog-cobjs): $(obj)/%.o: $(src)/%.c FORCE
+ $(call if_changed_dep,tprog-cobjs)
+-
+-# Override includes for xdp_sample_user.o because $(srctree)/usr/include in
+-# TPROGS_CFLAGS causes conflicts
+-XDP_SAMPLE_CFLAGS += -Wall -O2 -lm \
+- -I./tools/include \
+- -I./tools/include/uapi \
+- -I./tools/lib \
+- -I./tools/testing/selftests/bpf
+-$(obj)/xdp_sample_user.o: $(src)/xdp_sample_user.c \
+- $(src)/xdp_sample_user.h $(src)/xdp_sample_shared.h
+- $(CC) $(XDP_SAMPLE_CFLAGS) -c -o $@ $<
+diff --git a/samples/bpf/hbm_kern.h b/samples/bpf/hbm_kern.h
+index 722b3fadb4675..1752a46a2b056 100644
+--- a/samples/bpf/hbm_kern.h
++++ b/samples/bpf/hbm_kern.h
+@@ -9,8 +9,6 @@
+ * Include file for sample Host Bandwidth Manager (HBM) BPF programs
+ */
+ #define KBUILD_MODNAME "foo"
+-#include <stddef.h>
+-#include <stdbool.h>
+ #include <uapi/linux/bpf.h>
+ #include <uapi/linux/if_ether.h>
+ #include <uapi/linux/if_packet.h>
+diff --git a/samples/bpf/lwt_len_hist_kern.c b/samples/bpf/lwt_len_hist_kern.c
+index 9ed63e10e1709..1fa14c54963a1 100644
+--- a/samples/bpf/lwt_len_hist_kern.c
++++ b/samples/bpf/lwt_len_hist_kern.c
+@@ -16,13 +16,6 @@
+ #include <uapi/linux/in.h>
+ #include <bpf/bpf_helpers.h>
+
+-# define printk(fmt, ...) \
+- ({ \
+- char ____fmt[] = fmt; \
+- bpf_trace_printk(____fmt, sizeof(____fmt), \
+- ##__VA_ARGS__); \
+- })
+-
+ struct bpf_elf_map {
+ __u32 type;
+ __u32 size_key;
+diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c
+index 6e25fba64c72b..a81704d3317ba 100644
+--- a/samples/bpf/xdp_redirect_cpu_user.c
++++ b/samples/bpf/xdp_redirect_cpu_user.c
+@@ -309,7 +309,6 @@ int main(int argc, char **argv)
+ const char *mprog_filename = NULL, *mprog_name = NULL;
+ struct xdp_redirect_cpu *skel;
+ struct bpf_map_info info = {};
+- char ifname_buf[IF_NAMESIZE];
+ struct bpf_cpumap_val value;
+ __u32 infosz = sizeof(info);
+ int ret = EXIT_FAIL_OPTION;
+@@ -325,7 +324,6 @@ int main(int argc, char **argv)
+ int add_cpu = -1;
+ int ifindex = -1;
+ int *cpu, i, opt;
+- char *ifname;
+ __u32 qsize;
+ int n_cpus;
+
+@@ -391,11 +389,10 @@ int main(int argc, char **argv)
+ case 'd':
+ if (strlen(optarg) >= IF_NAMESIZE) {
+ fprintf(stderr, "-d/--dev name too long\n");
++ usage(argv, long_options, __doc__, mask, true, skel->obj);
+ goto end_cpu;
+ }
+- ifname = (char *)&ifname_buf;
+- safe_strncpy(ifname, optarg, sizeof(ifname));
+- ifindex = if_nametoindex(ifname);
++ ifindex = if_nametoindex(optarg);
+ if (!ifindex)
+ ifindex = strtoul(optarg, NULL, 0);
+ if (!ifindex) {
+diff --git a/samples/bpf/xdp_sample_user.c b/samples/bpf/xdp_sample_user.c
+index b32d821781990..8740838e77679 100644
+--- a/samples/bpf/xdp_sample_user.c
++++ b/samples/bpf/xdp_sample_user.c
+@@ -120,7 +120,10 @@ struct sample_output {
+ __u64 xmit;
+ } totals;
+ struct {
+- __u64 pps;
++ union {
++ __u64 pps;
++ __u64 num;
++ };
+ __u64 drop;
+ __u64 err;
+ } rx_cnt;
+@@ -1322,7 +1325,7 @@ int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
+
+ static void sample_summary_print(void)
+ {
+- double period = sample_out.rx_cnt.pps;
++ double num = sample_out.rx_cnt.num;
+
+ if (sample_out.totals.rx) {
+ double pkts = sample_out.totals.rx;
+@@ -1330,7 +1333,7 @@ static void sample_summary_print(void)
+ print_always(" Packets received : %'-10llu\n",
+ sample_out.totals.rx);
+ print_always(" Average packets/s : %'-10.0f\n",
+- sample_round(pkts / period));
++ sample_round(pkts / num));
+ }
+ if (sample_out.totals.redir) {
+ double pkts = sample_out.totals.redir;
+@@ -1338,7 +1341,7 @@ static void sample_summary_print(void)
+ print_always(" Packets redirected : %'-10llu\n",
+ sample_out.totals.redir);
+ print_always(" Average redir/s : %'-10.0f\n",
+- sample_round(pkts / period));
++ sample_round(pkts / num));
+ }
+ if (sample_out.totals.drop)
+ print_always(" Rx dropped : %'-10llu\n",
+@@ -1355,7 +1358,7 @@ static void sample_summary_print(void)
+ print_always(" Packets transmitted : %'-10llu\n",
+ sample_out.totals.xmit);
+ print_always(" Average transmit/s : %'-10.0f\n",
+- sample_round(pkts / period));
++ sample_round(pkts / num));
+ }
+ }
+
+@@ -1422,7 +1425,7 @@ static int sample_stats_collect(struct stats_record *rec)
+ return 0;
+ }
+
+-static void sample_summary_update(struct sample_output *out, int interval)
++static void sample_summary_update(struct sample_output *out)
+ {
+ sample_out.totals.rx += out->totals.rx;
+ sample_out.totals.redir += out->totals.redir;
+@@ -1430,12 +1433,11 @@ static void sample_summary_update(struct sample_output *out, int interval)
+ sample_out.totals.drop_xmit += out->totals.drop_xmit;
+ sample_out.totals.err += out->totals.err;
+ sample_out.totals.xmit += out->totals.xmit;
+- sample_out.rx_cnt.pps += interval;
++ sample_out.rx_cnt.num++;
+ }
+
+ static void sample_stats_print(int mask, struct stats_record *cur,
+- struct stats_record *prev, char *prog_name,
+- int interval)
++ struct stats_record *prev, char *prog_name)
+ {
+ struct sample_output out = {};
+
+@@ -1452,7 +1454,7 @@ static void sample_stats_print(int mask, struct stats_record *cur,
+ else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
+ stats_get_devmap_xmit_multi(cur, prev, 0, &out,
+ mask & SAMPLE_DEVMAP_XMIT_CNT);
+- sample_summary_update(&out, interval);
++ sample_summary_update(&out);
+
+ stats_print(prog_name, mask, cur, prev, &out);
+ }
+@@ -1495,7 +1497,7 @@ static void swap(struct stats_record **a, struct stats_record **b)
+ }
+
+ static int sample_timer_cb(int timerfd, struct stats_record **rec,
+- struct stats_record **prev, int interval)
++ struct stats_record **prev)
+ {
+ char line[64] = "Summary";
+ int ret;
+@@ -1524,7 +1526,7 @@ static int sample_timer_cb(int timerfd, struct stats_record **rec,
+ snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");
+ }
+
+- sample_stats_print(sample_mask, *rec, *prev, line, interval);
++ sample_stats_print(sample_mask, *rec, *prev, line);
+ return 0;
+ }
+
+@@ -1579,7 +1581,7 @@ int sample_run(int interval, void (*post_cb)(void *), void *ctx)
+ if (pfd[0].revents & POLLIN)
+ ret = sample_signal_cb();
+ else if (pfd[1].revents & POLLIN)
+- ret = sample_timer_cb(timerfd, &rec, &prev, interval);
++ ret = sample_timer_cb(timerfd, &rec, &prev);
+
+ if (ret)
+ break;
+diff --git a/samples/bpf/xdp_sample_user.h b/samples/bpf/xdp_sample_user.h
+index d97465ff8c62c..5f44b877ecf5f 100644
+--- a/samples/bpf/xdp_sample_user.h
++++ b/samples/bpf/xdp_sample_user.h
+@@ -45,7 +45,9 @@ const char *get_driver_name(int ifindex);
+ int get_mac_addr(int ifindex, void *mac_addr);
+
+ #pragma GCC diagnostic push
++#ifndef __clang__
+ #pragma GCC diagnostic ignored "-Wstringop-truncation"
++#endif
+ __attribute__((unused))
+ static inline char *safe_strncpy(char *dst, const char *src, size_t size)
+ {
+diff --git a/samples/bpf/xdpsock_user.c b/samples/bpf/xdpsock_user.c
+index 49d7a6ad7e397..1fb79b3ecdd51 100644
+--- a/samples/bpf/xdpsock_user.c
++++ b/samples/bpf/xdpsock_user.c
+@@ -1673,14 +1673,15 @@ int main(int argc, char **argv)
+
+ setlocale(LC_ALL, "");
+
++ prev_time = get_nsecs();
++ start_time = prev_time;
++
+ if (!opt_quiet) {
+ ret = pthread_create(&pt, NULL, poller, NULL);
+ if (ret)
+ exit_with_error(ret);
+ }
+
+- prev_time = get_nsecs();
+- start_time = prev_time;
+
+ if (opt_bench == BENCH_RXDROP)
+ rx_drop_all();
+diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c
+index 5b9a09957c6e0..d620f3da086f7 100644
+--- a/samples/ftrace/ftrace-direct-modify.c
++++ b/samples/ftrace/ftrace-direct-modify.c
+@@ -3,6 +3,9 @@
+ #include <linux/kthread.h>
+ #include <linux/ftrace.h>
+
++extern void my_direct_func1(void);
++extern void my_direct_func2(void);
++
+ void my_direct_func1(void)
+ {
+ trace_printk("my direct func1\n");
+@@ -28,7 +31,7 @@ asm (
+ " call my_direct_func1\n"
+ " leave\n"
+ " .size my_tramp1, .-my_tramp1\n"
+-" ret\n"
++ ASM_RET
+ " .type my_tramp2, @function\n"
+ " .globl my_tramp2\n"
+ " my_tramp2:"
+@@ -36,7 +39,7 @@ asm (
+ " movq %rsp, %rbp\n"
+ " call my_direct_func2\n"
+ " leave\n"
+-" ret\n"
++ ASM_RET
+ " .size my_tramp2, .-my_tramp2\n"
+ " .popsection\n"
+ );
+diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c
+index 3f0079c9bd6fa..3927cb880d1ab 100644
+--- a/samples/ftrace/ftrace-direct-too.c
++++ b/samples/ftrace/ftrace-direct-too.c
+@@ -4,6 +4,9 @@
+ #include <linux/mm.h> /* for handle_mm_fault() */
+ #include <linux/ftrace.h>
+
++extern void my_direct_func(struct vm_area_struct *vma,
++ unsigned long address, unsigned int flags);
++
+ void my_direct_func(struct vm_area_struct *vma,
+ unsigned long address, unsigned int flags)
+ {
+@@ -28,7 +31,7 @@ asm (
+ " popq %rsi\n"
+ " popq %rdi\n"
+ " leave\n"
+-" ret\n"
++ ASM_RET
+ " .size my_tramp, .-my_tramp\n"
+ " .popsection\n"
+ );
+diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
+index a2729d1ef17f5..1e901bb8d7293 100644
+--- a/samples/ftrace/ftrace-direct.c
++++ b/samples/ftrace/ftrace-direct.c
+@@ -4,6 +4,8 @@
+ #include <linux/sched.h> /* for wake_up_process() */
+ #include <linux/ftrace.h>
+
++extern void my_direct_func(struct task_struct *p);
++
+ void my_direct_func(struct task_struct *p)
+ {
+ trace_printk("waking up %s-%d\n", p->comm, p->pid);
+@@ -22,7 +24,7 @@ asm (
+ " call my_direct_func\n"
+ " popq %rdi\n"
+ " leave\n"
+-" ret\n"
++ ASM_RET
+ " .size my_tramp, .-my_tramp\n"
+ " .popsection\n"
+ );
+diff --git a/samples/kprobes/kretprobe_example.c b/samples/kprobes/kretprobe_example.c
+index 5dc1bf3baa98b..228321ecb1616 100644
+--- a/samples/kprobes/kretprobe_example.c
++++ b/samples/kprobes/kretprobe_example.c
+@@ -86,7 +86,7 @@ static int __init kretprobe_init(void)
+ ret = register_kretprobe(&my_kretprobe);
+ if (ret < 0) {
+ pr_err("register_kretprobe failed, returned %d\n", ret);
+- return -1;
++ return ret;
+ }
+ pr_info("Planted return probe at %s: %p\n",
+ my_kretprobe.kp.symbol_name, my_kretprobe.kp.addr);
+diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c
+index 7a15910d21718..8859fc1935428 100644
+--- a/samples/landlock/sandboxer.c
++++ b/samples/landlock/sandboxer.c
+@@ -134,6 +134,7 @@ static int populate_ruleset(
+ ret = 0;
+
+ out_free_name:
++ free(path_list);
+ free(env_path_name);
+ return ret;
+ }
+diff --git a/scripts/Makefile.build b/scripts/Makefile.build
+index 3efc984d4c690..2b988b6ccacb6 100644
+--- a/scripts/Makefile.build
++++ b/scripts/Makefile.build
+@@ -224,6 +224,17 @@ cmd_record_mcount = $(if $(findstring $(strip $(CC_FLAGS_FTRACE)),$(_c_flags)),
+ endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT
+
+ ifdef CONFIG_STACK_VALIDATION
++
++objtool_args = \
++ $(if $(CONFIG_UNWINDER_ORC),orc generate,check) \
++ $(if $(part-of-module), --module) \
++ $(if $(CONFIG_FRAME_POINTER),, --no-fp) \
++ $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
++ $(if $(CONFIG_RETPOLINE), --retpoline) \
++ $(if $(CONFIG_X86_SMAP), --uaccess) \
++ $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \
++ $(if $(CONFIG_SLS), --sls)
++
+ ifndef CONFIG_LTO_CLANG
+
+ __objtool_obj := $(objtree)/tools/objtool/objtool
+diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn
+index d538255038747..8be892887d716 100644
+--- a/scripts/Makefile.extrawarn
++++ b/scripts/Makefile.extrawarn
+@@ -51,6 +51,7 @@ KBUILD_CFLAGS += -Wno-sign-compare
+ KBUILD_CFLAGS += -Wno-format-zero-length
+ KBUILD_CFLAGS += $(call cc-disable-warning, pointer-to-enum-cast)
+ KBUILD_CFLAGS += -Wno-tautological-constant-out-of-range-compare
++KBUILD_CFLAGS += $(call cc-disable-warning, unaligned-access)
+ endif
+
+ endif
+diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
+index 54582673fc1a0..0a8a4689c3ebb 100644
+--- a/scripts/Makefile.lib
++++ b/scripts/Makefile.lib
+@@ -232,17 +232,6 @@ ifeq ($(CONFIG_LTO_CLANG),y)
+ mod-prelink-ext := .lto
+ endif
+
+-# Objtool arguments are also needed for modfinal with LTO, so we define
+-# then here to avoid duplication.
+-objtool_args = \
+- $(if $(CONFIG_UNWINDER_ORC),orc generate,check) \
+- $(if $(part-of-module), --module) \
+- $(if $(CONFIG_FRAME_POINTER),, --no-fp) \
+- $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
+- $(if $(CONFIG_RETPOLINE), --retpoline) \
+- $(if $(CONFIG_X86_SMAP), --uaccess) \
+- $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount)
+-
+ # Useful for describing the dependency of composite objects
+ # Usage:
+ # $(call multi_depend, multi_used_targets, suffix_to_remove, suffix_to_add)
+diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan
+index 9e2092fd5206c..7099c603ff0ad 100644
+--- a/scripts/Makefile.ubsan
++++ b/scripts/Makefile.ubsan
+@@ -8,7 +8,6 @@ ubsan-cflags-$(CONFIG_UBSAN_LOCAL_BOUNDS) += -fsanitize=local-bounds
+ ubsan-cflags-$(CONFIG_UBSAN_SHIFT) += -fsanitize=shift
+ ubsan-cflags-$(CONFIG_UBSAN_DIV_ZERO) += -fsanitize=integer-divide-by-zero
+ ubsan-cflags-$(CONFIG_UBSAN_UNREACHABLE) += -fsanitize=unreachable
+-ubsan-cflags-$(CONFIG_UBSAN_OBJECT_SIZE) += -fsanitize=object-size
+ ubsan-cflags-$(CONFIG_UBSAN_BOOL) += -fsanitize=bool
+ ubsan-cflags-$(CONFIG_UBSAN_ENUM) += -fsanitize=enum
+ ubsan-cflags-$(CONFIG_UBSAN_TRAP) += -fsanitize-undefined-trap-on-error
+diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire
+index 803ba75610766..a0ea1d26e6b2e 100755
+--- a/scripts/atomic/fallbacks/read_acquire
++++ b/scripts/atomic/fallbacks/read_acquire
+@@ -2,6 +2,15 @@ cat <<EOF
+ static __always_inline ${ret}
+ arch_${atomic}_read_acquire(const ${atomic}_t *v)
+ {
+- return smp_load_acquire(&(v)->counter);
++ ${int} ret;
++
++ if (__native_word(${atomic}_t)) {
++ ret = smp_load_acquire(&(v)->counter);
++ } else {
++ ret = arch_${atomic}_read(v);
++ __atomic_acquire_fence();
++ }
++
++ return ret;
+ }
+ EOF
+diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release
+index 86ede759f24ea..05cdb7f42477a 100755
+--- a/scripts/atomic/fallbacks/set_release
++++ b/scripts/atomic/fallbacks/set_release
+@@ -2,6 +2,11 @@ cat <<EOF
+ static __always_inline void
+ arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
+ {
+- smp_store_release(&(v)->counter, i);
++ if (__native_word(${atomic}_t)) {
++ smp_store_release(&(v)->counter, i);
++ } else {
++ __atomic_release_fence();
++ arch_${atomic}_set(v, i);
++ }
+ }
+ EOF
+diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile
+index 95aaf7431bffa..1cba78e1dce68 100644
+--- a/scripts/dtc/Makefile
++++ b/scripts/dtc/Makefile
+@@ -29,7 +29,7 @@ dtc-objs += yamltree.o
+ # To include <yaml.h> installed in a non-default path
+ HOSTCFLAGS_yamltree.o := $(shell pkg-config --cflags yaml-0.1)
+ # To link libyaml installed in a non-default path
+-HOSTLDLIBS_dtc := $(shell pkg-config yaml-0.1 --libs)
++HOSTLDLIBS_dtc := $(shell pkg-config --libs yaml-0.1)
+ endif
+
+ # Generated files need one more search path to include headers in source tree
+diff --git a/scripts/dtc/dtx_diff b/scripts/dtc/dtx_diff
+index d3422ee15e300..f2bbde4bba86b 100755
+--- a/scripts/dtc/dtx_diff
++++ b/scripts/dtc/dtx_diff
+@@ -59,12 +59,8 @@ Otherwise DTx is treated as a dts source file (aka .dts).
+ or '/include/' to be processed.
+
+ If DTx_1 and DTx_2 are in different architectures, then this script
+- may not work since \${ARCH} is part of the include path. Two possible
+- workarounds:
+-
+- `basename $0` \\
+- <(ARCH=arch_of_dtx_1 `basename $0` DTx_1) \\
+- <(ARCH=arch_of_dtx_2 `basename $0` DTx_2)
++ may not work since \${ARCH} is part of the include path. The following
++ workaround can be used:
+
+ `basename $0` ARCH=arch_of_dtx_1 DTx_1 >tmp_dtx_1.dts
+ `basename $0` ARCH=arch_of_dtx_2 DTx_2 >tmp_dtx_2.dts
+diff --git a/scripts/gcc-plugins/latent_entropy_plugin.c b/scripts/gcc-plugins/latent_entropy_plugin.c
+index 589454bce9301..8425da41de0da 100644
+--- a/scripts/gcc-plugins/latent_entropy_plugin.c
++++ b/scripts/gcc-plugins/latent_entropy_plugin.c
+@@ -86,25 +86,31 @@ static struct plugin_info latent_entropy_plugin_info = {
+ .help = "disable\tturn off latent entropy instrumentation\n",
+ };
+
+-static unsigned HOST_WIDE_INT seed;
+-/*
+- * get_random_seed() (this is a GCC function) generates the seed.
+- * This is a simple random generator without any cryptographic security because
+- * the entropy doesn't come from here.
+- */
++static unsigned HOST_WIDE_INT deterministic_seed;
++static unsigned HOST_WIDE_INT rnd_buf[32];
++static size_t rnd_idx = ARRAY_SIZE(rnd_buf);
++static int urandom_fd = -1;
++
+ static unsigned HOST_WIDE_INT get_random_const(void)
+ {
+- unsigned int i;
+- unsigned HOST_WIDE_INT ret = 0;
+-
+- for (i = 0; i < 8 * sizeof(ret); i++) {
+- ret = (ret << 1) | (seed & 1);
+- seed >>= 1;
+- if (ret & 1)
+- seed ^= 0xD800000000000000ULL;
++ if (deterministic_seed) {
++ unsigned HOST_WIDE_INT w = deterministic_seed;
++ w ^= w << 13;
++ w ^= w >> 7;
++ w ^= w << 17;
++ deterministic_seed = w;
++ return deterministic_seed;
+ }
+
+- return ret;
++ if (urandom_fd < 0) {
++ urandom_fd = open("/dev/urandom", O_RDONLY);
++ gcc_assert(urandom_fd >= 0);
++ }
++ if (rnd_idx >= ARRAY_SIZE(rnd_buf)) {
++ gcc_assert(read(urandom_fd, rnd_buf, sizeof(rnd_buf)) == sizeof(rnd_buf));
++ rnd_idx = 0;
++ }
++ return rnd_buf[rnd_idx++];
+ }
+
+ static tree tree_get_random_const(tree type)
+@@ -537,8 +543,6 @@ static void latent_entropy_start_unit(void *gcc_data __unused,
+ tree type, id;
+ int quals;
+
+- seed = get_random_seed(false);
+-
+ if (in_lto_p)
+ return;
+
+@@ -573,6 +577,12 @@ __visible int plugin_init(struct plugin_name_args *plugin_info,
+ const struct plugin_argument * const argv = plugin_info->argv;
+ int i;
+
++ /*
++ * Call get_random_seed() with noinit=true, so that this returns
++ * 0 in the case where no seed has been passed via -frandom-seed.
++ */
++ deterministic_seed = get_random_seed(true);
++
+ static const struct ggc_root_tab gt_ggc_r_gt_latent_entropy[] = {
+ {
+ .base = &latent_entropy_decl,
+diff --git a/scripts/gcc-plugins/stackleak_plugin.c b/scripts/gcc-plugins/stackleak_plugin.c
+index e9db7dcb3e5f4..b04aa8e91a41f 100644
+--- a/scripts/gcc-plugins/stackleak_plugin.c
++++ b/scripts/gcc-plugins/stackleak_plugin.c
+@@ -429,6 +429,23 @@ static unsigned int stackleak_cleanup_execute(void)
+ return 0;
+ }
+
++/*
++ * STRING_CST may or may not be NUL terminated:
++ * https://gcc.gnu.org/onlinedocs/gccint/Constant-expressions.html
++ */
++static inline bool string_equal(tree node, const char *string, int length)
++{
++ if (TREE_STRING_LENGTH(node) < length)
++ return false;
++ if (TREE_STRING_LENGTH(node) > length + 1)
++ return false;
++ if (TREE_STRING_LENGTH(node) == length + 1 &&
++ TREE_STRING_POINTER(node)[length] != '\0')
++ return false;
++ return !memcmp(TREE_STRING_POINTER(node), string, length);
++}
++#define STRING_EQUAL(node, str) string_equal(node, str, strlen(str))
++
+ static bool stackleak_gate(void)
+ {
+ tree section;
+@@ -438,13 +455,13 @@ static bool stackleak_gate(void)
+ if (section && TREE_VALUE(section)) {
+ section = TREE_VALUE(TREE_VALUE(section));
+
+- if (!strncmp(TREE_STRING_POINTER(section), ".init.text", 10))
++ if (STRING_EQUAL(section, ".init.text"))
+ return false;
+- if (!strncmp(TREE_STRING_POINTER(section), ".devinit.text", 13))
++ if (STRING_EQUAL(section, ".devinit.text"))
+ return false;
+- if (!strncmp(TREE_STRING_POINTER(section), ".cpuinit.text", 13))
++ if (STRING_EQUAL(section, ".cpuinit.text"))
+ return false;
+- if (!strncmp(TREE_STRING_POINTER(section), ".meminit.text", 13))
++ if (STRING_EQUAL(section, ".meminit.text"))
+ return false;
+ }
+
+diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c
+index cf72680cd7692..4a828bca071e8 100644
+--- a/scripts/kconfig/confdata.c
++++ b/scripts/kconfig/confdata.c
+@@ -983,14 +983,19 @@ static int conf_write_dep(const char *name)
+
+ static int conf_touch_deps(void)
+ {
+- const char *name;
++ const char *name, *tmp;
+ struct symbol *sym;
+ int res, i;
+
+- strcpy(depfile_path, "include/config/");
+- depfile_prefix_len = strlen(depfile_path);
+-
+ name = conf_get_autoconfig_name();
++ tmp = strrchr(name, '/');
++ depfile_prefix_len = tmp ? tmp - name + 1 : 0;
++ if (depfile_prefix_len + 1 > sizeof(depfile_path))
++ return -1;
++
++ strncpy(depfile_path, name, depfile_prefix_len);
++ depfile_path[depfile_prefix_len] = 0;
++
+ conf_read_simple(name, S_DEF_AUTO);
+ sym_calc_value(modules_sym);
+
+diff --git a/scripts/kconfig/preprocess.c b/scripts/kconfig/preprocess.c
+index 0590f86df6e40..748da578b418c 100644
+--- a/scripts/kconfig/preprocess.c
++++ b/scripts/kconfig/preprocess.c
+@@ -141,7 +141,7 @@ static char *do_lineno(int argc, char *argv[])
+ static char *do_shell(int argc, char *argv[])
+ {
+ FILE *p;
+- char buf[256];
++ char buf[4096];
+ char *cmd;
+ size_t nread;
+ int i;
+diff --git a/scripts/leaking_addresses.pl b/scripts/leaking_addresses.pl
+index b2d8b8aa2d99e..8f636a23bc3f2 100755
+--- a/scripts/leaking_addresses.pl
++++ b/scripts/leaking_addresses.pl
+@@ -455,8 +455,9 @@ sub parse_file
+
+ open my $fh, "<", $file or return;
+ while ( <$fh> ) {
++ chomp;
+ if (may_leak_address($_)) {
+- print $file . ': ' . $_;
++ printf("$file: $_\n");
+ }
+ }
+ close $fh;
+diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
+index d74cee5c4326a..59a3df87907ef 100755
+--- a/scripts/link-vmlinux.sh
++++ b/scripts/link-vmlinux.sh
+@@ -139,6 +139,9 @@ objtool_link()
+ if [ -n "${CONFIG_X86_SMAP}" ]; then
+ objtoolopt="${objtoolopt} --uaccess"
+ fi
++ if [ -n "${CONFIG_SLS}" ]; then
++ objtoolopt="${objtoolopt} --sls"
++ fi
+ info OBJTOOL ${1}
+ tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1}
+ fi
+diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c
+index cb8ab7d91d307..ca491aa2b3762 100644
+--- a/scripts/mod/modpost.c
++++ b/scripts/mod/modpost.c
+@@ -669,7 +669,7 @@ static void handle_modversion(const struct module *mod,
+ unsigned int crc;
+
+ if (sym->st_shndx == SHN_UNDEF) {
+- warn("EXPORT symbol \"%s\" [%s%s] version ...\n"
++ warn("EXPORT symbol \"%s\" [%s%s] version generation failed, symbol will not be versioned.\n"
+ "Is \"%s\" prototyped in <asm/asm-prototypes.h>?\n",
+ symname, mod->name, mod->is_vmlinux ? "" : ".ko",
+ symname);
+diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
+index 7d631aaa0ae11..3ccb2c70add4d 100755
+--- a/scripts/recordmcount.pl
++++ b/scripts/recordmcount.pl
+@@ -219,7 +219,7 @@ if ($arch eq "x86_64") {
+
+ } elsif ($arch eq "s390" && $bits == 64) {
+ if ($cc =~ /-DCC_USING_HOTPATCH/) {
+- $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*c0 04 00 00 00 00\\s*brcl\\s*0,[0-9a-f]+ <([^\+]*)>\$";
++ $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*c0 04 00 00 00 00\\s*(brcl\\s*0,|jgnop\\s*)[0-9a-f]+ <([^\+]*)>\$";
+ $mcount_adjust = 0;
+ }
+ $alignment = 8;
+diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install
+index 288e86a9d1e58..f126ecbb0494d 100755
+--- a/scripts/sphinx-pre-install
++++ b/scripts/sphinx-pre-install
+@@ -78,6 +78,7 @@ my %texlive = (
+ 'ucs.sty' => 'texlive-ucs',
+ 'upquote.sty' => 'texlive-upquote',
+ 'wrapfig.sty' => 'texlive-wrapfig',
++ 'ctexhook.sty' => 'texlive-ctex',
+ );
+
+ #
+@@ -369,6 +370,9 @@ sub give_debian_hints()
+ );
+
+ if ($pdf) {
++ check_missing_file(["/usr/share/texlive/texmf-dist/tex/latex/ctex/ctexhook.sty"],
++ "texlive-lang-chinese", 2);
++
+ check_missing_file(["/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"],
+ "fonts-dejavu", 2);
+
+diff --git a/security/Kconfig b/security/Kconfig
+index 0ced7fd33e4d0..fe6c0395fa025 100644
+--- a/security/Kconfig
++++ b/security/Kconfig
+@@ -191,6 +191,9 @@ config HARDENED_USERCOPY_PAGESPAN
+ config FORTIFY_SOURCE
+ bool "Harden common str/mem functions against buffer overflows"
+ depends on ARCH_HAS_FORTIFY_SOURCE
++ # https://bugs.llvm.org/show_bug.cgi?id=50322
++ # https://bugs.llvm.org/show_bug.cgi?id=41459
++ depends on !CC_IS_CLANG
+ help
+ Detect overflows of buffers in common string and memory functions
+ where the compiler can determine and validate the buffer sizes.
+diff --git a/security/apparmor/label.c b/security/apparmor/label.c
+index e68bcedca976b..6222fdfebe4e5 100644
+--- a/security/apparmor/label.c
++++ b/security/apparmor/label.c
+@@ -1454,7 +1454,7 @@ bool aa_update_label_name(struct aa_ns *ns, struct aa_label *label, gfp_t gfp)
+ if (label->hname || labels_ns(label) != ns)
+ return res;
+
+- if (aa_label_acntsxprint(&name, ns, label, FLAGS_NONE, gfp) == -1)
++ if (aa_label_acntsxprint(&name, ns, label, FLAGS_NONE, gfp) < 0)
+ return res;
+
+ ls = labels_set(label);
+@@ -1704,7 +1704,7 @@ int aa_label_asxprint(char **strp, struct aa_ns *ns, struct aa_label *label,
+
+ /**
+ * aa_label_acntsxprint - allocate a __counted string buffer and print label
+- * @strp: buffer to write to. (MAY BE NULL if @size == 0)
++ * @strp: buffer to write to.
+ * @ns: namespace profile is being viewed from
+ * @label: label to view (NOT NULL)
+ * @flags: flags controlling what label info is printed
+diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c
+index 23240d793b074..895f4b9ce8c6b 100644
+--- a/security/integrity/digsig_asymmetric.c
++++ b/security/integrity/digsig_asymmetric.c
+@@ -109,22 +109,25 @@ int asymmetric_verify(struct key *keyring, const char *sig,
+
+ pk = asymmetric_key_public_key(key);
+ pks.pkey_algo = pk->pkey_algo;
+- if (!strcmp(pk->pkey_algo, "rsa"))
++ if (!strcmp(pk->pkey_algo, "rsa")) {
+ pks.encoding = "pkcs1";
+- else if (!strncmp(pk->pkey_algo, "ecdsa-", 6))
++ } else if (!strncmp(pk->pkey_algo, "ecdsa-", 6)) {
+ /* edcsa-nist-p192 etc. */
+ pks.encoding = "x962";
+- else if (!strcmp(pk->pkey_algo, "ecrdsa") ||
+- !strcmp(pk->pkey_algo, "sm2"))
++ } else if (!strcmp(pk->pkey_algo, "ecrdsa") ||
++ !strcmp(pk->pkey_algo, "sm2")) {
+ pks.encoding = "raw";
+- else
+- return -ENOPKG;
++ } else {
++ ret = -ENOPKG;
++ goto out;
++ }
+
+ pks.digest = (u8 *)data;
+ pks.digest_size = datalen;
+ pks.s = hdr->sig;
+ pks.s_size = siglen;
+ ret = verify_signature(key, &pks);
++out:
+ key_put(key);
+ pr_debug("%s() = %d\n", __func__, ret);
+ return ret;
+diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c
+index 1c8435dfabeea..7d87772f0ce68 100644
+--- a/security/integrity/evm/evm_main.c
++++ b/security/integrity/evm/evm_main.c
+@@ -78,7 +78,7 @@ static struct xattr_list evm_config_default_xattrnames[] = {
+
+ LIST_HEAD(evm_config_xattrnames);
+
+-static int evm_fixmode;
++static int evm_fixmode __ro_after_init;
+ static int __init evm_set_fixmode(char *str)
+ {
+ if (strncmp(str, "fix", 3) == 0)
+@@ -86,7 +86,7 @@ static int __init evm_set_fixmode(char *str)
+ else
+ pr_err("invalid \"%s\" mode", str);
+
+- return 0;
++ return 1;
+ }
+ __setup("evm=", evm_set_fixmode);
+
+diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
+index 3d8e9d5db5aa5..3ad8f7734208b 100644
+--- a/security/integrity/ima/ima_fs.c
++++ b/security/integrity/ima/ima_fs.c
+@@ -496,12 +496,12 @@ int __init ima_fs_init(void)
+
+ return 0;
+ out:
++ securityfs_remove(ima_policy);
+ securityfs_remove(violations);
+ securityfs_remove(runtime_measurements_count);
+ securityfs_remove(ascii_runtime_measurements);
+ securityfs_remove(binary_runtime_measurements);
+ securityfs_remove(ima_symlink);
+ securityfs_remove(ima_dir);
+- securityfs_remove(ima_policy);
+ return -1;
+ }
+diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c
+index 87b9b71cb8201..fa5a93dbe5d26 100644
+--- a/security/integrity/ima/ima_policy.c
++++ b/security/integrity/ima/ima_policy.c
+@@ -228,7 +228,7 @@ static struct ima_rule_entry *arch_policy_entry __ro_after_init;
+ static LIST_HEAD(ima_default_rules);
+ static LIST_HEAD(ima_policy_rules);
+ static LIST_HEAD(ima_temp_rules);
+-static struct list_head *ima_rules = &ima_default_rules;
++static struct list_head __rcu *ima_rules = (struct list_head __rcu *)(&ima_default_rules);
+
+ static int ima_policy __initdata;
+
+@@ -675,12 +675,14 @@ int ima_match_policy(struct user_namespace *mnt_userns, struct inode *inode,
+ {
+ struct ima_rule_entry *entry;
+ int action = 0, actmask = flags | (flags << 1);
++ struct list_head *ima_rules_tmp;
+
+ if (template_desc && !*template_desc)
+ *template_desc = ima_template_desc_current();
+
+ rcu_read_lock();
+- list_for_each_entry_rcu(entry, ima_rules, list) {
++ ima_rules_tmp = rcu_dereference(ima_rules);
++ list_for_each_entry_rcu(entry, ima_rules_tmp, list) {
+
+ if (!(entry->action & actmask))
+ continue;
+@@ -741,9 +743,11 @@ void ima_update_policy_flags(void)
+ {
+ struct ima_rule_entry *entry;
+ int new_policy_flag = 0;
++ struct list_head *ima_rules_tmp;
+
+ rcu_read_lock();
+- list_for_each_entry(entry, ima_rules, list) {
++ ima_rules_tmp = rcu_dereference(ima_rules);
++ list_for_each_entry_rcu(entry, ima_rules_tmp, list) {
+ /*
+ * SETXATTR_CHECK rules do not implement a full policy check
+ * because rule checking would probably have an important
+@@ -968,10 +972,10 @@ void ima_update_policy(void)
+
+ list_splice_tail_init_rcu(&ima_temp_rules, policy, synchronize_rcu);
+
+- if (ima_rules != policy) {
++ if (ima_rules != (struct list_head __rcu *)policy) {
+ ima_policy_flag = 0;
+- ima_rules = policy;
+
++ rcu_assign_pointer(ima_rules, policy);
+ /*
+ * IMA architecture specific policy rules are specified
+ * as strings and converted to an array of ima_entry_rules
+@@ -1061,7 +1065,7 @@ static int ima_lsm_rule_init(struct ima_rule_entry *entry,
+ pr_warn("rule for LSM \'%s\' is undefined\n",
+ entry->lsm[lsm_rule].args_p);
+
+- if (ima_rules == &ima_default_rules) {
++ if (ima_rules == (struct list_head __rcu *)(&ima_default_rules)) {
+ kfree(entry->lsm[lsm_rule].args_p);
+ entry->lsm[lsm_rule].args_p = NULL;
+ result = -EINVAL;
+@@ -1768,9 +1772,11 @@ void *ima_policy_start(struct seq_file *m, loff_t *pos)
+ {
+ loff_t l = *pos;
+ struct ima_rule_entry *entry;
++ struct list_head *ima_rules_tmp;
+
+ rcu_read_lock();
+- list_for_each_entry_rcu(entry, ima_rules, list) {
++ ima_rules_tmp = rcu_dereference(ima_rules);
++ list_for_each_entry_rcu(entry, ima_rules_tmp, list) {
+ if (!l--) {
+ rcu_read_unlock();
+ return entry;
+@@ -1789,7 +1795,8 @@ void *ima_policy_next(struct seq_file *m, void *v, loff_t *pos)
+ rcu_read_unlock();
+ (*pos)++;
+
+- return (&entry->list == ima_rules) ? NULL : entry;
++ return (&entry->list == &ima_default_rules ||
++ &entry->list == &ima_policy_rules) ? NULL : entry;
+ }
+
+ void ima_policy_stop(struct seq_file *m, void *v)
+@@ -1845,6 +1852,14 @@ int ima_policy_show(struct seq_file *m, void *v)
+
+ rcu_read_lock();
+
++ /* Do not print rules with inactive LSM labels */
++ for (i = 0; i < MAX_LSM_RULES; i++) {
++ if (entry->lsm[i].args_p && !entry->lsm[i].rule) {
++ rcu_read_unlock();
++ return 0;
++ }
++ }
++
+ if (entry->action & MEASURE)
+ seq_puts(m, pt(Opt_measure));
+ if (entry->action & DONT_MEASURE)
+@@ -2014,6 +2029,7 @@ bool ima_appraise_signature(enum kernel_read_file_id id)
+ struct ima_rule_entry *entry;
+ bool found = false;
+ enum ima_hooks func;
++ struct list_head *ima_rules_tmp;
+
+ if (id >= READING_MAX_ID)
+ return false;
+@@ -2021,7 +2037,8 @@ bool ima_appraise_signature(enum kernel_read_file_id id)
+ func = read_idmap[id] ?: FILE_CHECK;
+
+ rcu_read_lock();
+- list_for_each_entry_rcu(entry, ima_rules, list) {
++ ima_rules_tmp = rcu_dereference(ima_rules);
++ list_for_each_entry_rcu(entry, ima_rules_tmp, list) {
+ if (entry->action != APPRAISE)
+ continue;
+
+diff --git a/security/integrity/ima/ima_template.c b/security/integrity/ima/ima_template.c
+index 694560396be05..db1ad6d7a57fb 100644
+--- a/security/integrity/ima/ima_template.c
++++ b/security/integrity/ima/ima_template.c
+@@ -29,6 +29,7 @@ static struct ima_template_desc builtin_templates[] = {
+
+ static LIST_HEAD(defined_templates);
+ static DEFINE_SPINLOCK(template_list);
++static int template_setup_done;
+
+ static const struct ima_template_field supported_fields[] = {
+ {.field_id = "d", .field_init = ima_eventdigest_init,
+@@ -101,10 +102,11 @@ static int __init ima_template_setup(char *str)
+ struct ima_template_desc *template_desc;
+ int template_len = strlen(str);
+
+- if (ima_template)
++ if (template_setup_done)
+ return 1;
+
+- ima_init_template_list();
++ if (!ima_template)
++ ima_init_template_list();
+
+ /*
+ * Verify that a template with the supplied name exists.
+@@ -128,6 +130,7 @@ static int __init ima_template_setup(char *str)
+ }
+
+ ima_template = template_desc;
++ template_setup_done = 1;
+ return 1;
+ }
+ __setup("ima_template=", ima_template_setup);
+@@ -136,7 +139,7 @@ static int __init ima_template_fmt_setup(char *str)
+ {
+ int num_templates = ARRAY_SIZE(builtin_templates);
+
+- if (ima_template)
++ if (template_setup_done)
+ return 1;
+
+ if (template_desc_init_fields(str, NULL, NULL) < 0) {
+@@ -147,6 +150,7 @@ static int __init ima_template_fmt_setup(char *str)
+
+ builtin_templates[num_templates - 1].fmt = str;
+ ima_template = builtin_templates + num_templates - 1;
++ template_setup_done = 1;
+
+ return 1;
+ }
+diff --git a/security/integrity/integrity_audit.c b/security/integrity/integrity_audit.c
+index 29220056207f4..0ec5e4c22cb2a 100644
+--- a/security/integrity/integrity_audit.c
++++ b/security/integrity/integrity_audit.c
+@@ -45,6 +45,8 @@ void integrity_audit_message(int audit_msgno, struct inode *inode,
+ return;
+
+ ab = audit_log_start(audit_context(), GFP_KERNEL, audit_msgno);
++ if (!ab)
++ return;
+ audit_log_format(ab, "pid=%d uid=%u auid=%u ses=%u",
+ task_pid_nr(current),
+ from_kuid(&init_user_ns, current_uid()),
+diff --git a/security/keys/keyctl_pkey.c b/security/keys/keyctl_pkey.c
+index 5de0d599a2748..97bc27bbf0797 100644
+--- a/security/keys/keyctl_pkey.c
++++ b/security/keys/keyctl_pkey.c
+@@ -135,15 +135,23 @@ static int keyctl_pkey_params_get_2(const struct keyctl_pkey_params __user *_par
+
+ switch (op) {
+ case KEYCTL_PKEY_ENCRYPT:
++ if (uparams.in_len > info.max_dec_size ||
++ uparams.out_len > info.max_enc_size)
++ return -EINVAL;
++ break;
+ case KEYCTL_PKEY_DECRYPT:
+ if (uparams.in_len > info.max_enc_size ||
+ uparams.out_len > info.max_dec_size)
+ return -EINVAL;
+ break;
+ case KEYCTL_PKEY_SIGN:
++ if (uparams.in_len > info.max_data_size ||
++ uparams.out_len > info.max_sig_size)
++ return -EINVAL;
++ break;
+ case KEYCTL_PKEY_VERIFY:
+- if (uparams.in_len > info.max_sig_size ||
+- uparams.out_len > info.max_data_size)
++ if (uparams.in_len > info.max_data_size ||
++ uparams.in2_len > info.max_sig_size)
+ return -EINVAL;
+ break;
+ default:
+@@ -151,7 +159,7 @@ static int keyctl_pkey_params_get_2(const struct keyctl_pkey_params __user *_par
+ }
+
+ params->in_len = uparams.in_len;
+- params->out_len = uparams.out_len;
++ params->out_len = uparams.out_len; /* Note: same as in2_len */
+ return 0;
+ }
+
+diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c
+index d5c891d8d3534..9b9d3ef79cbe3 100644
+--- a/security/keys/trusted-keys/trusted_core.c
++++ b/security/keys/trusted-keys/trusted_core.c
+@@ -27,10 +27,10 @@ module_param_named(source, trusted_key_source, charp, 0);
+ MODULE_PARM_DESC(source, "Select trusted keys source (tpm or tee)");
+
+ static const struct trusted_key_source trusted_key_sources[] = {
+-#if defined(CONFIG_TCG_TPM)
++#if IS_REACHABLE(CONFIG_TCG_TPM)
+ { "tpm", &trusted_key_tpm_ops },
+ #endif
+-#if defined(CONFIG_TEE)
++#if IS_REACHABLE(CONFIG_TEE)
+ { "tee", &trusted_key_tee_ops },
+ #endif
+ };
+@@ -351,7 +351,7 @@ static int __init init_trusted(void)
+
+ static void __exit cleanup_trusted(void)
+ {
+- static_call(trusted_key_exit)();
++ static_call_cond(trusted_key_exit)();
+ }
+
+ late_initcall(init_trusted);
+diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c
+index 32396962f04d6..7e27ce394020d 100644
+--- a/security/landlock/syscalls.c
++++ b/security/landlock/syscalls.c
+@@ -192,7 +192,7 @@ SYSCALL_DEFINE3(landlock_create_ruleset,
+ return PTR_ERR(ruleset);
+
+ /* Creates anonymous FD referring to the ruleset. */
+- ruleset_fd = anon_inode_getfd("landlock-ruleset", &ruleset_fops,
++ ruleset_fd = anon_inode_getfd("[landlock-ruleset]", &ruleset_fops,
+ ruleset, O_RDWR | O_CLOEXEC);
+ if (ruleset_fd < 0)
+ landlock_put_ruleset(ruleset);
+diff --git a/security/security.c b/security/security.c
+index 9ffa9e9c5c554..7b9f9d3fffe52 100644
+--- a/security/security.c
++++ b/security/security.c
+@@ -59,10 +59,12 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX+1] = {
+ [LOCKDOWN_DEBUGFS] = "debugfs access",
+ [LOCKDOWN_XMON_WR] = "xmon write access",
+ [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM",
++ [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM",
+ [LOCKDOWN_INTEGRITY_MAX] = "integrity",
+ [LOCKDOWN_KCORE] = "/proc/kcore access",
+ [LOCKDOWN_KPROBES] = "use of kprobes",
+ [LOCKDOWN_BPF_READ_KERNEL] = "use of bpf to read kernel RAM",
++ [LOCKDOWN_DBG_READ_KERNEL] = "use of kgdb/kdb to read kernel RAM",
+ [LOCKDOWN_PERF] = "unsafe use of perf",
+ [LOCKDOWN_TRACEFS] = "use of tracefs",
+ [LOCKDOWN_XMON_RW] = "xmon read and write access",
+@@ -747,25 +749,25 @@ static int lsm_superblock_alloc(struct super_block *sb)
+
+ /* Security operations */
+
+-int security_binder_set_context_mgr(struct task_struct *mgr)
++int security_binder_set_context_mgr(const struct cred *mgr)
+ {
+ return call_int_hook(binder_set_context_mgr, 0, mgr);
+ }
+
+-int security_binder_transaction(struct task_struct *from,
+- struct task_struct *to)
++int security_binder_transaction(const struct cred *from,
++ const struct cred *to)
+ {
+ return call_int_hook(binder_transaction, 0, from, to);
+ }
+
+-int security_binder_transfer_binder(struct task_struct *from,
+- struct task_struct *to)
++int security_binder_transfer_binder(const struct cred *from,
++ const struct cred *to)
+ {
+ return call_int_hook(binder_transfer_binder, 0, from, to);
+ }
+
+-int security_binder_transfer_file(struct task_struct *from,
+- struct task_struct *to, struct file *file)
++int security_binder_transfer_file(const struct cred *from,
++ const struct cred *to, struct file *file)
+ {
+ return call_int_hook(binder_transfer_file, 0, from, to, file);
+ }
+@@ -884,9 +886,22 @@ int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc)
+ return call_int_hook(fs_context_dup, 0, fc, src_fc);
+ }
+
+-int security_fs_context_parse_param(struct fs_context *fc, struct fs_parameter *param)
++int security_fs_context_parse_param(struct fs_context *fc,
++ struct fs_parameter *param)
+ {
+- return call_int_hook(fs_context_parse_param, -ENOPARAM, fc, param);
++ struct security_hook_list *hp;
++ int trc;
++ int rc = -ENOPARAM;
++
++ hlist_for_each_entry(hp, &security_hook_heads.fs_context_parse_param,
++ list) {
++ trc = hp->hook.fs_context_parse_param(fc, param);
++ if (trc == 0)
++ rc = 0;
++ else if (trc != -ENOPARAM)
++ return trc;
++ }
++ return rc;
+ }
+
+ int security_sb_alloc(struct super_block *sb)
+diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
+index e7ebd45ca3457..9ce029b2f2267 100644
+--- a/security/selinux/hooks.c
++++ b/security/selinux/hooks.c
+@@ -255,29 +255,6 @@ static inline u32 task_sid_obj(const struct task_struct *task)
+ return sid;
+ }
+
+-/*
+- * get the security ID of a task for use with binder
+- */
+-static inline u32 task_sid_binder(const struct task_struct *task)
+-{
+- /*
+- * In many case where this function is used we should be using the
+- * task's subjective SID, but we can't reliably access the subjective
+- * creds of a task other than our own so we must use the objective
+- * creds/SID, which are safe to access. The downside is that if a task
+- * is temporarily overriding it's creds it will not be reflected here;
+- * however, it isn't clear that binder would handle that case well
+- * anyway.
+- *
+- * If this ever changes and we can safely reference the subjective
+- * creds/SID of another task, this function will make it easier to
+- * identify the various places where we make use of the task SIDs in
+- * the binder code. It is also likely that we will need to adjust
+- * the main drivers/android binder code as well.
+- */
+- return task_sid_obj(task);
+-}
+-
+ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dentry);
+
+ /*
+@@ -378,6 +355,10 @@ static void inode_free_security(struct inode *inode)
+
+ struct selinux_mnt_opts {
+ const char *fscontext, *context, *rootcontext, *defcontext;
++ u32 fscontext_sid;
++ u32 context_sid;
++ u32 rootcontext_sid;
++ u32 defcontext_sid;
+ };
+
+ static void selinux_free_mnt_opts(void *mnt_opts)
+@@ -515,7 +496,7 @@ static int selinux_is_sblabel_mnt(struct super_block *sb)
+
+ static int sb_check_xattr_support(struct super_block *sb)
+ {
+- struct superblock_security_struct *sbsec = sb->s_security;
++ struct superblock_security_struct *sbsec = selinux_superblock(sb);
+ struct dentry *root = sb->s_root;
+ struct inode *root_inode = d_backing_inode(root);
+ u32 sid;
+@@ -641,7 +622,7 @@ static int parse_sid(struct super_block *sb, const char *s, u32 *sid)
+ if (rc)
+ pr_warn("SELinux: security_context_str_to_sid"
+ "(%s) failed for (dev %s, type %s) errno=%d\n",
+- s, sb->s_id, sb->s_type->name, rc);
++ s, sb ? sb->s_id : "?", sb ? sb->s_type->name : "?", rc);
+ return rc;
+ }
+
+@@ -1005,42 +986,58 @@ out:
+ static int selinux_add_opt(int token, const char *s, void **mnt_opts)
+ {
+ struct selinux_mnt_opts *opts = *mnt_opts;
++ bool is_alloc_opts = false;
+
+ if (token == Opt_seclabel) /* eaten and completely ignored */
+ return 0;
+
++ if (!s)
++ return -ENOMEM;
++
+ if (!opts) {
+ opts = kzalloc(sizeof(struct selinux_mnt_opts), GFP_KERNEL);
+ if (!opts)
+ return -ENOMEM;
+ *mnt_opts = opts;
++ is_alloc_opts = true;
+ }
+- if (!s)
+- return -ENOMEM;
++
+ switch (token) {
+ case Opt_context:
+ if (opts->context || opts->defcontext)
+ goto Einval;
+ opts->context = s;
++ if (selinux_initialized(&selinux_state))
++ parse_sid(NULL, s, &opts->context_sid);
+ break;
+ case Opt_fscontext:
+ if (opts->fscontext)
+ goto Einval;
+ opts->fscontext = s;
++ if (selinux_initialized(&selinux_state))
++ parse_sid(NULL, s, &opts->fscontext_sid);
+ break;
+ case Opt_rootcontext:
+ if (opts->rootcontext)
+ goto Einval;
+ opts->rootcontext = s;
++ if (selinux_initialized(&selinux_state))
++ parse_sid(NULL, s, &opts->rootcontext_sid);
+ break;
+ case Opt_defcontext:
+ if (opts->context || opts->defcontext)
+ goto Einval;
+ opts->defcontext = s;
++ if (selinux_initialized(&selinux_state))
++ parse_sid(NULL, s, &opts->defcontext_sid);
+ break;
+ }
+ return 0;
+ Einval:
++ if (is_alloc_opts) {
++ kfree(opts);
++ *mnt_opts = NULL;
++ }
+ pr_warn(SEL_MOUNT_FAIL_MSG);
+ return -EINVAL;
+ }
+@@ -2066,18 +2063,19 @@ static inline u32 open_file_to_av(struct file *file)
+
+ /* Hook functions begin here. */
+
+-static int selinux_binder_set_context_mgr(struct task_struct *mgr)
++static int selinux_binder_set_context_mgr(const struct cred *mgr)
+ {
+ return avc_has_perm(&selinux_state,
+- current_sid(), task_sid_binder(mgr), SECCLASS_BINDER,
++ current_sid(), cred_sid(mgr), SECCLASS_BINDER,
+ BINDER__SET_CONTEXT_MGR, NULL);
+ }
+
+-static int selinux_binder_transaction(struct task_struct *from,
+- struct task_struct *to)
++static int selinux_binder_transaction(const struct cred *from,
++ const struct cred *to)
+ {
+ u32 mysid = current_sid();
+- u32 fromsid = task_sid_binder(from);
++ u32 fromsid = cred_sid(from);
++ u32 tosid = cred_sid(to);
+ int rc;
+
+ if (mysid != fromsid) {
+@@ -2088,24 +2086,24 @@ static int selinux_binder_transaction(struct task_struct *from,
+ return rc;
+ }
+
+- return avc_has_perm(&selinux_state, fromsid, task_sid_binder(to),
++ return avc_has_perm(&selinux_state, fromsid, tosid,
+ SECCLASS_BINDER, BINDER__CALL, NULL);
+ }
+
+-static int selinux_binder_transfer_binder(struct task_struct *from,
+- struct task_struct *to)
++static int selinux_binder_transfer_binder(const struct cred *from,
++ const struct cred *to)
+ {
+ return avc_has_perm(&selinux_state,
+- task_sid_binder(from), task_sid_binder(to),
++ cred_sid(from), cred_sid(to),
+ SECCLASS_BINDER, BINDER__TRANSFER,
+ NULL);
+ }
+
+-static int selinux_binder_transfer_file(struct task_struct *from,
+- struct task_struct *to,
++static int selinux_binder_transfer_file(const struct cred *from,
++ const struct cred *to,
+ struct file *file)
+ {
+- u32 sid = task_sid_binder(to);
++ u32 sid = cred_sid(to);
+ struct file_security_struct *fsec = selinux_file(file);
+ struct dentry *dentry = file->f_path.dentry;
+ struct inode_security_struct *isec;
+@@ -2704,9 +2702,7 @@ free_opt:
+ static int selinux_sb_mnt_opts_compat(struct super_block *sb, void *mnt_opts)
+ {
+ struct selinux_mnt_opts *opts = mnt_opts;
+- struct superblock_security_struct *sbsec = sb->s_security;
+- u32 sid;
+- int rc;
++ struct superblock_security_struct *sbsec = selinux_superblock(sb);
+
+ /*
+ * Superblock not initialized (i.e. no options) - reject if any
+@@ -2723,34 +2719,36 @@ static int selinux_sb_mnt_opts_compat(struct super_block *sb, void *mnt_opts)
+ return (sbsec->flags & SE_MNTMASK) ? 1 : 0;
+
+ if (opts->fscontext) {
+- rc = parse_sid(sb, opts->fscontext, &sid);
+- if (rc)
++ if (opts->fscontext_sid == SECSID_NULL)
+ return 1;
+- if (bad_option(sbsec, FSCONTEXT_MNT, sbsec->sid, sid))
++ else if (bad_option(sbsec, FSCONTEXT_MNT, sbsec->sid,
++ opts->fscontext_sid))
+ return 1;
+ }
+ if (opts->context) {
+- rc = parse_sid(sb, opts->context, &sid);
+- if (rc)
++ if (opts->context_sid == SECSID_NULL)
+ return 1;
+- if (bad_option(sbsec, CONTEXT_MNT, sbsec->mntpoint_sid, sid))
++ else if (bad_option(sbsec, CONTEXT_MNT, sbsec->mntpoint_sid,
++ opts->context_sid))
+ return 1;
+ }
+ if (opts->rootcontext) {
+- struct inode_security_struct *root_isec;
+-
+- root_isec = backing_inode_security(sb->s_root);
+- rc = parse_sid(sb, opts->rootcontext, &sid);
+- if (rc)
+- return 1;
+- if (bad_option(sbsec, ROOTCONTEXT_MNT, root_isec->sid, sid))
++ if (opts->rootcontext_sid == SECSID_NULL)
+ return 1;
++ else {
++ struct inode_security_struct *root_isec;
++
++ root_isec = backing_inode_security(sb->s_root);
++ if (bad_option(sbsec, ROOTCONTEXT_MNT, root_isec->sid,
++ opts->rootcontext_sid))
++ return 1;
++ }
+ }
+ if (opts->defcontext) {
+- rc = parse_sid(sb, opts->defcontext, &sid);
+- if (rc)
++ if (opts->defcontext_sid == SECSID_NULL)
+ return 1;
+- if (bad_option(sbsec, DEFCONTEXT_MNT, sbsec->def_sid, sid))
++ else if (bad_option(sbsec, DEFCONTEXT_MNT, sbsec->def_sid,
++ opts->defcontext_sid))
+ return 1;
+ }
+ return 0;
+@@ -2917,10 +2915,9 @@ static int selinux_fs_context_parse_param(struct fs_context *fc,
+ return opt;
+
+ rc = selinux_add_opt(opt, param->string, &fc->security);
+- if (!rc) {
++ if (!rc)
+ param->string = NULL;
+- rc = 1;
+- }
++
+ return rc;
+ }
+
+@@ -3798,6 +3795,12 @@ static int selinux_file_ioctl(struct file *file, unsigned int cmd,
+ CAP_OPT_NONE, true);
+ break;
+
++ case FIOCLEX:
++ case FIONCLEX:
++ if (!selinux_policycap_ioctl_skip_cloexec())
++ error = ioctl_has_perm(cred, file, FILE__IOCTL, (u16) cmd);
++ break;
++
+ /* default case assumes that the command will go
+ * to the file's ioctl() function.
+ */
+@@ -5829,7 +5832,7 @@ static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb,
+ struct common_audit_data ad;
+ struct lsm_network_audit net = {0,};
+ char *addrp;
+- u8 proto;
++ u8 proto = 0;
+
+ if (sk == NULL)
+ return NF_ACCEPT;
+diff --git a/security/selinux/ima.c b/security/selinux/ima.c
+index 727c4e43219d7..ff7aea6b3774a 100644
+--- a/security/selinux/ima.c
++++ b/security/selinux/ima.c
+@@ -77,7 +77,7 @@ void selinux_ima_measure_state_locked(struct selinux_state *state)
+ size_t policy_len;
+ int rc = 0;
+
+- WARN_ON(!mutex_is_locked(&state->policy_mutex));
++ lockdep_assert_held(&state->policy_mutex);
+
+ state_str = selinux_ima_collect_state(state);
+ if (!state_str) {
+@@ -117,7 +117,7 @@ void selinux_ima_measure_state_locked(struct selinux_state *state)
+ */
+ void selinux_ima_measure_state(struct selinux_state *state)
+ {
+- WARN_ON(mutex_is_locked(&state->policy_mutex));
++ lockdep_assert_not_held(&state->policy_mutex);
+
+ mutex_lock(&state->policy_mutex);
+ selinux_ima_measure_state_locked(state);
+diff --git a/security/selinux/include/policycap.h b/security/selinux/include/policycap.h
+index 2ec038efbb03c..a9e572ca4fd96 100644
+--- a/security/selinux/include/policycap.h
++++ b/security/selinux/include/policycap.h
+@@ -11,6 +11,7 @@ enum {
+ POLICYDB_CAPABILITY_CGROUPSECLABEL,
+ POLICYDB_CAPABILITY_NNP_NOSUID_TRANSITION,
+ POLICYDB_CAPABILITY_GENFS_SECLABEL_SYMLINKS,
++ POLICYDB_CAPABILITY_IOCTL_SKIP_CLOEXEC,
+ __POLICYDB_CAPABILITY_MAX
+ };
+ #define POLICYDB_CAPABILITY_MAX (__POLICYDB_CAPABILITY_MAX - 1)
+diff --git a/security/selinux/include/policycap_names.h b/security/selinux/include/policycap_names.h
+index b89289f092c93..ebd64afe1defd 100644
+--- a/security/selinux/include/policycap_names.h
++++ b/security/selinux/include/policycap_names.h
+@@ -12,7 +12,8 @@ const char *selinux_policycap_names[__POLICYDB_CAPABILITY_MAX] = {
+ "always_check_network",
+ "cgroup_seclabel",
+ "nnp_nosuid_transition",
+- "genfs_seclabel_symlinks"
++ "genfs_seclabel_symlinks",
++ "ioctl_skip_cloexec"
+ };
+
+ #endif /* _SELINUX_POLICYCAP_NAMES_H_ */
+diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h
+index ac0ece01305a6..c0d966020ebdd 100644
+--- a/security/selinux/include/security.h
++++ b/security/selinux/include/security.h
+@@ -219,6 +219,13 @@ static inline bool selinux_policycap_genfs_seclabel_symlinks(void)
+ return READ_ONCE(state->policycap[POLICYDB_CAPABILITY_GENFS_SECLABEL_SYMLINKS]);
+ }
+
++static inline bool selinux_policycap_ioctl_skip_cloexec(void)
++{
++ struct selinux_state *state = &selinux_state;
++
++ return READ_ONCE(state->policycap[POLICYDB_CAPABILITY_IOCTL_SKIP_CLOEXEC]);
++}
++
+ struct selinux_policy_convert_data;
+
+ struct selinux_load_state {
+diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
+index e4cd7cb856f37..f2f6203e0fff5 100644
+--- a/security/selinux/selinuxfs.c
++++ b/security/selinux/selinuxfs.c
+@@ -2127,6 +2127,8 @@ static int sel_fill_super(struct super_block *sb, struct fs_context *fc)
+ }
+
+ ret = sel_make_avc_files(dentry);
++ if (ret)
++ goto err;
+
+ dentry = sel_make_dir(sb->s_root, "ss", &fsi->last_ino);
+ if (IS_ERR(dentry)) {
+diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c
+index 2ec6e5cd25d9b..feb206f3acb4a 100644
+--- a/security/selinux/ss/conditional.c
++++ b/security/selinux/ss/conditional.c
+@@ -152,6 +152,8 @@ static void cond_list_destroy(struct policydb *p)
+ for (i = 0; i < p->cond_list_len; i++)
+ cond_node_destroy(&p->cond_list[i]);
+ kfree(p->cond_list);
++ p->cond_list = NULL;
++ p->cond_list_len = 0;
+ }
+
+ void cond_policydb_destroy(struct policydb *p)
+@@ -441,7 +443,6 @@ int cond_read_list(struct policydb *p, void *fp)
+ return 0;
+ err:
+ cond_list_destroy(p);
+- p->cond_list = NULL;
+ return rc;
+ }
+
+diff --git a/security/selinux/ss/hashtab.c b/security/selinux/ss/hashtab.c
+index b8f6b3e0a9219..298098bb9c06d 100644
+--- a/security/selinux/ss/hashtab.c
++++ b/security/selinux/ss/hashtab.c
+@@ -30,13 +30,20 @@ static u32 hashtab_compute_size(u32 nel)
+
+ int hashtab_init(struct hashtab *h, u32 nel_hint)
+ {
+- h->size = hashtab_compute_size(nel_hint);
++ u32 size = hashtab_compute_size(nel_hint);
++
++ /* should already be zeroed, but better be safe */
+ h->nel = 0;
+- if (!h->size)
+- return 0;
++ h->size = 0;
++ h->htable = NULL;
+
+- h->htable = kcalloc(h->size, sizeof(*h->htable), GFP_KERNEL);
+- return h->htable ? 0 : -ENOMEM;
++ if (size) {
++ h->htable = kcalloc(size, sizeof(*h->htable), GFP_KERNEL);
++ if (!h->htable)
++ return -ENOMEM;
++ h->size = size;
++ }
++ return 0;
+ }
+
+ int __hashtab_insert(struct hashtab *h, struct hashtab_node **dst,
+@@ -171,7 +178,8 @@ int hashtab_duplicate(struct hashtab *new, struct hashtab *orig,
+ kmem_cache_free(hashtab_node_cachep, cur);
+ }
+ }
+- kmem_cache_free(hashtab_node_cachep, new);
++ kfree(new->htable);
++ memset(new, 0, sizeof(*new));
+ return -ENOMEM;
+ }
+
+diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
+index e5f1b2757a83a..c4931bf6f92a4 100644
+--- a/security/selinux/ss/services.c
++++ b/security/selinux/ss/services.c
+@@ -2376,6 +2376,43 @@ err_policy:
+ return rc;
+ }
+
++/**
++ * ocontext_to_sid - Helper to safely get sid for an ocontext
++ * @sidtab: SID table
++ * @c: ocontext structure
++ * @index: index of the context entry (0 or 1)
++ * @out_sid: pointer to the resulting SID value
++ *
++ * For all ocontexts except OCON_ISID the SID fields are populated
++ * on-demand when needed. Since updating the SID value is an SMP-sensitive
++ * operation, this helper must be used to do that safely.
++ *
++ * WARNING: This function may return -ESTALE, indicating that the caller
++ * must retry the operation after re-acquiring the policy pointer!
++ */
++static int ocontext_to_sid(struct sidtab *sidtab, struct ocontext *c,
++ size_t index, u32 *out_sid)
++{
++ int rc;
++ u32 sid;
++
++ /* Ensure the associated sidtab entry is visible to this thread. */
++ sid = smp_load_acquire(&c->sid[index]);
++ if (!sid) {
++ rc = sidtab_context_to_sid(sidtab, &c->context[index], &sid);
++ if (rc)
++ return rc;
++
++ /*
++ * Ensure the new sidtab entry is visible to other threads
++ * when they see the SID.
++ */
++ smp_store_release(&c->sid[index], sid);
++ }
++ *out_sid = sid;
++ return 0;
++}
++
+ /**
+ * security_port_sid - Obtain the SID for a port.
+ * @state: SELinux state
+@@ -2414,17 +2451,13 @@ retry:
+ }
+
+ if (c) {
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab, &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, out_sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- *out_sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else {
+ *out_sid = SECINITSID_PORT;
+ }
+@@ -2473,18 +2506,13 @@ retry:
+ }
+
+ if (c) {
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab,
+- &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, out_sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- *out_sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else
+ *out_sid = SECINITSID_UNLABELED;
+
+@@ -2533,17 +2561,13 @@ retry:
+ }
+
+ if (c) {
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab, &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, out_sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- *out_sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else
+ *out_sid = SECINITSID_UNLABELED;
+
+@@ -2587,25 +2611,13 @@ retry:
+ }
+
+ if (c) {
+- if (!c->sid[0] || !c->sid[1]) {
+- rc = sidtab_context_to_sid(sidtab, &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
+- rc = sidtab_context_to_sid(sidtab, &c->context[1],
+- &c->sid[1]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, if_sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- *if_sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else
+ *if_sid = SECINITSID_NETIF;
+
+@@ -2697,18 +2709,13 @@ retry:
+ }
+
+ if (c) {
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab,
+- &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, out_sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- *out_sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else {
+ *out_sid = SECINITSID_NODE;
+ }
+@@ -2873,7 +2880,7 @@ static inline int __security_genfs_sid(struct selinux_policy *policy,
+ u16 sclass;
+ struct genfs *genfs;
+ struct ocontext *c;
+- int rc, cmp = 0;
++ int cmp = 0;
+
+ while (path[0] == '/' && path[1] == '/')
+ path++;
+@@ -2887,9 +2894,8 @@ static inline int __security_genfs_sid(struct selinux_policy *policy,
+ break;
+ }
+
+- rc = -ENOENT;
+ if (!genfs || cmp)
+- goto out;
++ return -ENOENT;
+
+ for (c = genfs->head; c; c = c->next) {
+ len = strlen(c->u.name);
+@@ -2898,20 +2904,10 @@ static inline int __security_genfs_sid(struct selinux_policy *policy,
+ break;
+ }
+
+- rc = -ENOENT;
+ if (!c)
+- goto out;
+-
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]);
+- if (rc)
+- goto out;
+- }
++ return -ENOENT;
+
+- *sid = c->sid[0];
+- rc = 0;
+-out:
+- return rc;
++ return ocontext_to_sid(sidtab, c, 0, sid);
+ }
+
+ /**
+@@ -2996,17 +2992,13 @@ retry:
+
+ if (c) {
+ sbsec->behavior = c->v.behavior;
+- if (!c->sid[0]) {
+- rc = sidtab_context_to_sid(sidtab, &c->context[0],
+- &c->sid[0]);
+- if (rc == -ESTALE) {
+- rcu_read_unlock();
+- goto retry;
+- }
+- if (rc)
+- goto out;
++ rc = ocontext_to_sid(sidtab, c, 0, &sbsec->sid);
++ if (rc == -ESTALE) {
++ rcu_read_unlock();
++ goto retry;
+ }
+- sbsec->sid = c->sid[0];
++ if (rc)
++ goto out;
+ } else {
+ rc = __security_genfs_sid(policy, fstype, "/",
+ SECCLASS_DIR, &sbsec->sid);
+diff --git a/security/selinux/xfrm.c b/security/selinux/xfrm.c
+index be83e5ce4469c..debe15207d2bf 100644
+--- a/security/selinux/xfrm.c
++++ b/security/selinux/xfrm.c
+@@ -347,7 +347,7 @@ int selinux_xfrm_state_alloc_acquire(struct xfrm_state *x,
+ int rc;
+ struct xfrm_sec_ctx *ctx;
+ char *ctx_str = NULL;
+- int str_len;
++ u32 str_len;
+
+ if (!polsec)
+ return 0;
+diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
+index 21a0e7c3b8dee..3f3f56f6be4da 100644
+--- a/security/smack/smack_lsm.c
++++ b/security/smack/smack_lsm.c
+@@ -2505,7 +2505,7 @@ static int smk_ipv6_check(struct smack_known *subject,
+ #ifdef CONFIG_AUDIT
+ smk_ad_init_net(&ad, __func__, LSM_AUDIT_DATA_NET, &net);
+ ad.a.u.net->family = PF_INET6;
+- ad.a.u.net->dport = ntohs(address->sin6_port);
++ ad.a.u.net->dport = address->sin6_port;
+ if (act == SMK_RECEIVING)
+ ad.a.u.net->v6info.saddr = address->sin6_addr;
+ else
+diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
+index 3a75d2a8f5178..658eab05599e6 100644
+--- a/security/smack/smackfs.c
++++ b/security/smack/smackfs.c
+@@ -693,9 +693,7 @@ static void smk_cipso_doi(void)
+ printk(KERN_WARNING "%s:%d remove rc = %d\n",
+ __func__, __LINE__, rc);
+
+- doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL);
+- if (doip == NULL)
+- panic("smack: Failed to initialize cipso DOI.\n");
++ doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL | __GFP_NOFAIL);
+ doip->map.std = NULL;
+ doip->doi = smk_cipso_doi_value;
+ doip->type = CIPSO_V4_MAP_PASS;
+@@ -714,7 +712,7 @@ static void smk_cipso_doi(void)
+ if (rc != 0) {
+ printk(KERN_WARNING "%s:%d map add rc = %d\n",
+ __func__, __LINE__, rc);
+- kfree(doip);
++ netlbl_cfg_cipsov4_del(doip->doi, &nai);
+ return;
+ }
+ }
+@@ -831,6 +829,7 @@ static int smk_open_cipso(struct inode *inode, struct file *file)
+ static ssize_t smk_set_cipso(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos, int format)
+ {
++ struct netlbl_lsm_catmap *old_cat;
+ struct smack_known *skp;
+ struct netlbl_lsm_secattr ncats;
+ char mapcatset[SMK_CIPSOLEN];
+@@ -920,9 +919,11 @@ static ssize_t smk_set_cipso(struct file *file, const char __user *buf,
+
+ rc = smk_netlbl_mls(maplevel, mapcatset, &ncats, SMK_CIPSOLEN);
+ if (rc >= 0) {
+- netlbl_catmap_free(skp->smk_netlabel.attr.mls.cat);
++ old_cat = skp->smk_netlabel.attr.mls.cat;
+ skp->smk_netlabel.attr.mls.cat = ncats.attr.mls.cat;
+ skp->smk_netlabel.attr.mls.lvl = ncats.attr.mls.lvl;
++ synchronize_rcu();
++ netlbl_catmap_free(old_cat);
+ rc = count;
+ /*
+ * This mapping may have been cached, so clear the cache.
+diff --git a/security/tomoyo/load_policy.c b/security/tomoyo/load_policy.c
+index 3445ae6fd4794..363b65be87ab7 100644
+--- a/security/tomoyo/load_policy.c
++++ b/security/tomoyo/load_policy.c
+@@ -24,7 +24,7 @@ static const char *tomoyo_loader;
+ static int __init tomoyo_loader_setup(char *str)
+ {
+ tomoyo_loader = str;
+- return 0;
++ return 1;
+ }
+
+ __setup("TOMOYO_loader=", tomoyo_loader_setup);
+@@ -64,7 +64,7 @@ static const char *tomoyo_trigger;
+ static int __init tomoyo_trigger_setup(char *str)
+ {
+ tomoyo_trigger = str;
+- return 0;
++ return 1;
+ }
+
+ __setup("TOMOYO_trigger=", tomoyo_trigger_setup);
+diff --git a/security/tomoyo/util.c b/security/tomoyo/util.c
+index 1da2e3722b126..6799b1122c9d8 100644
+--- a/security/tomoyo/util.c
++++ b/security/tomoyo/util.c
+@@ -1051,10 +1051,11 @@ bool tomoyo_domain_quota_is_ok(struct tomoyo_request_info *r)
+ return false;
+ if (!domain)
+ return true;
++ if (READ_ONCE(domain->flags[TOMOYO_DIF_QUOTA_WARNED]))
++ return false;
+ list_for_each_entry_rcu(ptr, &domain->acl_info_list, list,
+ srcu_read_lock_held(&tomoyo_ss)) {
+ u16 perm;
+- u8 i;
+
+ if (ptr->is_deleted)
+ continue;
+@@ -1065,23 +1066,23 @@ bool tomoyo_domain_quota_is_ok(struct tomoyo_request_info *r)
+ */
+ switch (ptr->type) {
+ case TOMOYO_TYPE_PATH_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_path_acl, head)->perm);
++ perm = data_race(container_of(ptr, struct tomoyo_path_acl, head)->perm);
+ break;
+ case TOMOYO_TYPE_PATH2_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_path2_acl, head)->perm);
++ perm = data_race(container_of(ptr, struct tomoyo_path2_acl, head)->perm);
+ break;
+ case TOMOYO_TYPE_PATH_NUMBER_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_path_number_acl, head)
++ perm = data_race(container_of(ptr, struct tomoyo_path_number_acl, head)
+ ->perm);
+ break;
+ case TOMOYO_TYPE_MKDEV_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_mkdev_acl, head)->perm);
++ perm = data_race(container_of(ptr, struct tomoyo_mkdev_acl, head)->perm);
+ break;
+ case TOMOYO_TYPE_INET_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_inet_acl, head)->perm);
++ perm = data_race(container_of(ptr, struct tomoyo_inet_acl, head)->perm);
+ break;
+ case TOMOYO_TYPE_UNIX_ACL:
+- data_race(perm = container_of(ptr, struct tomoyo_unix_acl, head)->perm);
++ perm = data_race(container_of(ptr, struct tomoyo_unix_acl, head)->perm);
+ break;
+ case TOMOYO_TYPE_MANUAL_TASK_ACL:
+ perm = 0;
+@@ -1089,21 +1090,17 @@ bool tomoyo_domain_quota_is_ok(struct tomoyo_request_info *r)
+ default:
+ perm = 1;
+ }
+- for (i = 0; i < 16; i++)
+- if (perm & (1 << i))
+- count++;
++ count += hweight16(perm);
+ }
+ if (count < tomoyo_profile(domain->ns, domain->profile)->
+ pref[TOMOYO_PREF_MAX_LEARNING_ENTRY])
+ return true;
+- if (!domain->flags[TOMOYO_DIF_QUOTA_WARNED]) {
+- domain->flags[TOMOYO_DIF_QUOTA_WARNED] = true;
+- /* r->granted = false; */
+- tomoyo_write_log(r, "%s", tomoyo_dif[TOMOYO_DIF_QUOTA_WARNED]);
++ WRITE_ONCE(domain->flags[TOMOYO_DIF_QUOTA_WARNED], true);
++ /* r->granted = false; */
++ tomoyo_write_log(r, "%s", tomoyo_dif[TOMOYO_DIF_QUOTA_WARNED]);
+ #ifndef CONFIG_SECURITY_TOMOYO_INSECURE_BUILTIN_SETTING
+- pr_warn("WARNING: Domain '%s' has too many ACLs to hold. Stopped learning mode.\n",
+- domain->domainname->name);
++ pr_warn("WARNING: Domain '%s' has too many ACLs to hold. Stopped learning mode.\n",
++ domain->domainname->name);
+ #endif
+- }
+ return false;
+ }
+diff --git a/sound/core/Makefile b/sound/core/Makefile
+index d774792850f31..79e1407cd0de7 100644
+--- a/sound/core/Makefile
++++ b/sound/core/Makefile
+@@ -9,7 +9,9 @@ ifneq ($(CONFIG_SND_PROC_FS),)
+ snd-y += info.o
+ snd-$(CONFIG_SND_OSSEMUL) += info_oss.o
+ endif
++ifneq ($(CONFIG_M68K),y)
+ snd-$(CONFIG_ISA_DMA_API) += isadma.o
++endif
+ snd-$(CONFIG_SND_OSSEMUL) += sound_oss.o
+ snd-$(CONFIG_SND_VMASTER) += vmaster.o
+ snd-$(CONFIG_SND_JACK) += ctljack.o jack.o
+diff --git a/sound/core/control_compat.c b/sound/core/control_compat.c
+index 470dabc60aa0e..edff063e088d2 100644
+--- a/sound/core/control_compat.c
++++ b/sound/core/control_compat.c
+@@ -264,6 +264,7 @@ static int copy_ctl_value_to_user(void __user *userdata,
+ struct snd_ctl_elem_value *data,
+ int type, int count)
+ {
++ struct snd_ctl_elem_value32 __user *data32 = userdata;
+ int i, size;
+
+ if (type == SNDRV_CTL_ELEM_TYPE_BOOLEAN ||
+@@ -280,6 +281,8 @@ static int copy_ctl_value_to_user(void __user *userdata,
+ if (copy_to_user(valuep, data->value.bytes.data, size))
+ return -EFAULT;
+ }
++ if (copy_to_user(&data32->id, &data->id, sizeof(data32->id)))
++ return -EFAULT;
+ return 0;
+ }
+
+diff --git a/sound/core/init.c b/sound/core/init.c
+index ac335f5906c6b..362588e3a275b 100644
+--- a/sound/core/init.c
++++ b/sound/core/init.c
+@@ -209,6 +209,12 @@ static void __snd_card_release(struct device *dev, void *data)
+ * snd_card_register(), the very first devres action to call snd_card_free()
+ * is added automatically. In that way, the resource disconnection is assured
+ * at first, then released in the expected order.
++ *
++ * If an error happens at the probe before snd_card_register() is called and
++ * there have been other devres resources, you'd need to free the card manually
++ * via snd_card_free() call in the error; otherwise it may lead to UAF due to
++ * devres call orders. You can use snd_card_free_on_error() helper for
++ * handling it more easily.
+ */
+ int snd_devm_card_new(struct device *parent, int idx, const char *xid,
+ struct module *module, size_t extra_size,
+@@ -235,6 +241,28 @@ int snd_devm_card_new(struct device *parent, int idx, const char *xid,
+ }
+ EXPORT_SYMBOL_GPL(snd_devm_card_new);
+
++/**
++ * snd_card_free_on_error - a small helper for handling devm probe errors
++ * @dev: the managed device object
++ * @ret: the return code from the probe callback
++ *
++ * This function handles the explicit snd_card_free() call at the error from
++ * the probe callback. It's just a small helper for simplifying the error
++ * handling with the managed devices.
++ */
++int snd_card_free_on_error(struct device *dev, int ret)
++{
++ struct snd_card *card;
++
++ if (!ret)
++ return 0;
++ card = devres_find(dev, __snd_card_release, NULL, NULL);
++ if (card)
++ snd_card_free(card);
++ return ret;
++}
++EXPORT_SYMBOL_GPL(snd_card_free_on_error);
++
+ static int snd_card_init(struct snd_card *card, struct device *parent,
+ int idx, const char *xid, struct module *module,
+ size_t extra_size)
+diff --git a/sound/core/jack.c b/sound/core/jack.c
+index 32350c6aba849..d1e3055f2b6a5 100644
+--- a/sound/core/jack.c
++++ b/sound/core/jack.c
+@@ -62,10 +62,13 @@ static int snd_jack_dev_free(struct snd_device *device)
+ struct snd_card *card = device->card;
+ struct snd_jack_kctl *jack_kctl, *tmp_jack_kctl;
+
++ down_write(&card->controls_rwsem);
+ list_for_each_entry_safe(jack_kctl, tmp_jack_kctl, &jack->kctl_list, list) {
+ list_del_init(&jack_kctl->list);
+ snd_ctl_remove(card, jack_kctl->kctl);
+ }
++ up_write(&card->controls_rwsem);
++
+ if (jack->private_free)
+ jack->private_free(jack);
+
+@@ -509,6 +512,10 @@ int snd_jack_new(struct snd_card *card, const char *id, int type,
+ return -ENOMEM;
+
+ jack->id = kstrdup(id, GFP_KERNEL);
++ if (jack->id == NULL) {
++ kfree(jack);
++ return -ENOMEM;
++ }
+
+ /* don't creat input device for phantom jack */
+ if (!phantom_jack) {
+diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c
+index c7c943c661e63..2d842982576bb 100644
+--- a/sound/core/memalloc.c
++++ b/sound/core/memalloc.c
+@@ -176,8 +176,11 @@ EXPORT_SYMBOL_GPL(snd_devm_alloc_pages);
+ int snd_dma_buffer_mmap(struct snd_dma_buffer *dmab,
+ struct vm_area_struct *area)
+ {
+- const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
++ const struct snd_malloc_ops *ops;
+
++ if (!dmab)
++ return -ENOENT;
++ ops = snd_dma_get_ops(dmab);
+ if (ops && ops->mmap)
+ return ops->mmap(dmab, area);
+ else
+@@ -491,6 +494,8 @@ static const struct snd_malloc_ops *dma_ops[] = {
+
+ static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab)
+ {
++ if (WARN_ON_ONCE(!dmab))
++ return NULL;
+ if (WARN_ON_ONCE(dmab->dev.type <= SNDRV_DMA_TYPE_UNKNOWN ||
+ dmab->dev.type >= ARRAY_SIZE(dma_ops)))
+ return NULL;
+diff --git a/sound/core/misc.c b/sound/core/misc.c
+index 3579dd7a161f7..50e4aaa6270d1 100644
+--- a/sound/core/misc.c
++++ b/sound/core/misc.c
+@@ -112,7 +112,7 @@ snd_pci_quirk_lookup_id(u16 vendor, u16 device,
+ {
+ const struct snd_pci_quirk *q;
+
+- for (q = list; q->subvendor; q++) {
++ for (q = list; q->subvendor || q->subdevice; q++) {
+ if (q->subvendor != vendor)
+ continue;
+ if (!q->subdevice ||
+diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c
+index 6a5abdd4271ba..9620115cfdc09 100644
+--- a/sound/core/oss/mixer_oss.c
++++ b/sound/core/oss/mixer_oss.c
+@@ -130,11 +130,13 @@ static int snd_mixer_oss_devmask(struct snd_mixer_oss_file *fmixer)
+
+ if (mixer == NULL)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ for (chn = 0; chn < 31; chn++) {
+ pslot = &mixer->slots[chn];
+ if (pslot->put_volume || pslot->put_recsrc)
+ result |= 1 << chn;
+ }
++ mutex_unlock(&mixer->reg_mutex);
+ return result;
+ }
+
+@@ -146,11 +148,13 @@ static int snd_mixer_oss_stereodevs(struct snd_mixer_oss_file *fmixer)
+
+ if (mixer == NULL)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ for (chn = 0; chn < 31; chn++) {
+ pslot = &mixer->slots[chn];
+ if (pslot->put_volume && pslot->stereo)
+ result |= 1 << chn;
+ }
++ mutex_unlock(&mixer->reg_mutex);
+ return result;
+ }
+
+@@ -161,6 +165,7 @@ static int snd_mixer_oss_recmask(struct snd_mixer_oss_file *fmixer)
+
+ if (mixer == NULL)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ if (mixer->put_recsrc && mixer->get_recsrc) { /* exclusive */
+ result = mixer->mask_recsrc;
+ } else {
+@@ -172,6 +177,7 @@ static int snd_mixer_oss_recmask(struct snd_mixer_oss_file *fmixer)
+ result |= 1 << chn;
+ }
+ }
++ mutex_unlock(&mixer->reg_mutex);
+ return result;
+ }
+
+@@ -182,12 +188,12 @@ static int snd_mixer_oss_get_recsrc(struct snd_mixer_oss_file *fmixer)
+
+ if (mixer == NULL)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ if (mixer->put_recsrc && mixer->get_recsrc) { /* exclusive */
+- int err;
+ unsigned int index;
+- err = mixer->get_recsrc(fmixer, &index);
+- if (err < 0)
+- return err;
++ result = mixer->get_recsrc(fmixer, &index);
++ if (result < 0)
++ goto unlock;
+ result = 1 << index;
+ } else {
+ struct snd_mixer_oss_slot *pslot;
+@@ -202,7 +208,10 @@ static int snd_mixer_oss_get_recsrc(struct snd_mixer_oss_file *fmixer)
+ }
+ }
+ }
+- return mixer->oss_recsrc = result;
++ mixer->oss_recsrc = result;
++ unlock:
++ mutex_unlock(&mixer->reg_mutex);
++ return result;
+ }
+
+ static int snd_mixer_oss_set_recsrc(struct snd_mixer_oss_file *fmixer, int recsrc)
+@@ -215,6 +224,7 @@ static int snd_mixer_oss_set_recsrc(struct snd_mixer_oss_file *fmixer, int recsr
+
+ if (mixer == NULL)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ if (mixer->get_recsrc && mixer->put_recsrc) { /* exclusive input */
+ if (recsrc & ~mixer->oss_recsrc)
+ recsrc &= ~mixer->oss_recsrc;
+@@ -240,6 +250,7 @@ static int snd_mixer_oss_set_recsrc(struct snd_mixer_oss_file *fmixer, int recsr
+ }
+ }
+ }
++ mutex_unlock(&mixer->reg_mutex);
+ return result;
+ }
+
+@@ -251,6 +262,7 @@ static int snd_mixer_oss_get_volume(struct snd_mixer_oss_file *fmixer, int slot)
+
+ if (mixer == NULL || slot > 30)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ pslot = &mixer->slots[slot];
+ left = pslot->volume[0];
+ right = pslot->volume[1];
+@@ -258,15 +270,21 @@ static int snd_mixer_oss_get_volume(struct snd_mixer_oss_file *fmixer, int slot)
+ result = pslot->get_volume(fmixer, pslot, &left, &right);
+ if (!pslot->stereo)
+ right = left;
+- if (snd_BUG_ON(left < 0 || left > 100))
+- return -EIO;
+- if (snd_BUG_ON(right < 0 || right > 100))
+- return -EIO;
++ if (snd_BUG_ON(left < 0 || left > 100)) {
++ result = -EIO;
++ goto unlock;
++ }
++ if (snd_BUG_ON(right < 0 || right > 100)) {
++ result = -EIO;
++ goto unlock;
++ }
+ if (result >= 0) {
+ pslot->volume[0] = left;
+ pslot->volume[1] = right;
+ result = (left & 0xff) | ((right & 0xff) << 8);
+ }
++ unlock:
++ mutex_unlock(&mixer->reg_mutex);
+ return result;
+ }
+
+@@ -279,6 +297,7 @@ static int snd_mixer_oss_set_volume(struct snd_mixer_oss_file *fmixer,
+
+ if (mixer == NULL || slot > 30)
+ return -EIO;
++ mutex_lock(&mixer->reg_mutex);
+ pslot = &mixer->slots[slot];
+ if (left > 100)
+ left = 100;
+@@ -289,10 +308,13 @@ static int snd_mixer_oss_set_volume(struct snd_mixer_oss_file *fmixer,
+ if (pslot->put_volume)
+ result = pslot->put_volume(fmixer, pslot, left, right);
+ if (result < 0)
+- return result;
++ goto unlock;
+ pslot->volume[0] = left;
+ pslot->volume[1] = right;
+- return (left & 0xff) | ((right & 0xff) << 8);
++ result = (left & 0xff) | ((right & 0xff) << 8);
++ unlock:
++ mutex_unlock(&mixer->reg_mutex);
++ return result;
+ }
+
+ static int snd_mixer_oss_ioctl1(struct snd_mixer_oss_file *fmixer, unsigned int cmd, unsigned long arg)
+diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
+index 82a818734a5f7..f158f0abd25d8 100644
+--- a/sound/core/oss/pcm_oss.c
++++ b/sound/core/oss/pcm_oss.c
+@@ -147,7 +147,7 @@ snd_pcm_hw_param_value_min(const struct snd_pcm_hw_params *params,
+ *
+ * Return the maximum value for field PAR.
+ */
+-static unsigned int
++static int
+ snd_pcm_hw_param_value_max(const struct snd_pcm_hw_params *params,
+ snd_pcm_hw_param_t var, int *dir)
+ {
+@@ -682,18 +682,24 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
+ struct snd_pcm_hw_params *oss_params,
+ struct snd_pcm_hw_params *slave_params)
+ {
+- size_t s;
+- size_t oss_buffer_size, oss_period_size, oss_periods;
+- size_t min_period_size, max_period_size;
++ ssize_t s;
++ ssize_t oss_buffer_size;
++ ssize_t oss_period_size, oss_periods;
++ ssize_t min_period_size, max_period_size;
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ size_t oss_frame_size;
+
+ oss_frame_size = snd_pcm_format_physical_width(params_format(oss_params)) *
+ params_channels(oss_params) / 8;
+
++ oss_buffer_size = snd_pcm_hw_param_value_max(slave_params,
++ SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
++ NULL);
++ if (oss_buffer_size <= 0)
++ return -EINVAL;
+ oss_buffer_size = snd_pcm_plug_client_size(substream,
+- snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_BUFFER_SIZE, NULL)) * oss_frame_size;
+- if (!oss_buffer_size)
++ oss_buffer_size * oss_frame_size);
++ if (oss_buffer_size <= 0)
+ return -EINVAL;
+ oss_buffer_size = rounddown_pow_of_two(oss_buffer_size);
+ if (atomic_read(&substream->mmap_count)) {
+@@ -730,7 +736,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
+
+ min_period_size = snd_pcm_plug_client_size(substream,
+ snd_pcm_hw_param_value_min(slave_params, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, NULL));
+- if (min_period_size) {
++ if (min_period_size > 0) {
+ min_period_size *= oss_frame_size;
+ min_period_size = roundup_pow_of_two(min_period_size);
+ if (oss_period_size < min_period_size)
+@@ -739,7 +745,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
+
+ max_period_size = snd_pcm_plug_client_size(substream,
+ snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, NULL));
+- if (max_period_size) {
++ if (max_period_size > 0) {
+ max_period_size *= oss_frame_size;
+ max_period_size = rounddown_pow_of_two(max_period_size);
+ if (oss_period_size > max_period_size)
+@@ -752,7 +758,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
+ oss_periods = substream->oss.setup.periods;
+
+ s = snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_PERIODS, NULL);
+- if (runtime->oss.maxfrags && s > runtime->oss.maxfrags)
++ if (s > 0 && runtime->oss.maxfrags && s > runtime->oss.maxfrags)
+ s = runtime->oss.maxfrags;
+ if (oss_periods > s)
+ oss_periods = s;
+@@ -768,6 +774,11 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
+
+ if (oss_period_size < 16)
+ return -EINVAL;
++
++ /* don't allocate too large period; 1MB period must be enough */
++ if (oss_period_size > 1024 * 1024)
++ return -ENOMEM;
++
+ runtime->oss.period_bytes = oss_period_size;
+ runtime->oss.period_frames = 1;
+ runtime->oss.periods = oss_periods;
+@@ -878,8 +889,15 @@ static int snd_pcm_oss_change_params_locked(struct snd_pcm_substream *substream)
+ err = -EINVAL;
+ goto failure;
+ }
+- choose_rate(substream, sparams, runtime->oss.rate);
+- snd_pcm_hw_param_near(substream, sparams, SNDRV_PCM_HW_PARAM_CHANNELS, runtime->oss.channels, NULL);
++
++ err = choose_rate(substream, sparams, runtime->oss.rate);
++ if (err < 0)
++ goto failure;
++ err = snd_pcm_hw_param_near(substream, sparams,
++ SNDRV_PCM_HW_PARAM_CHANNELS,
++ runtime->oss.channels, NULL);
++ if (err < 0)
++ goto failure;
+
+ format = snd_pcm_oss_format_from(runtime->oss.format);
+
+@@ -1030,10 +1048,9 @@ static int snd_pcm_oss_change_params_locked(struct snd_pcm_substream *substream)
+ goto failure;
+ }
+ #endif
+- oss_period_size *= oss_frame_size;
+-
+- oss_buffer_size = oss_period_size * runtime->oss.periods;
+- if (oss_buffer_size < 0) {
++ oss_period_size = array_size(oss_period_size, oss_frame_size);
++ oss_buffer_size = array_size(oss_period_size, runtime->oss.periods);
++ if (oss_buffer_size <= 0) {
+ err = -EINVAL;
+ goto failure;
+ }
+@@ -1956,7 +1973,7 @@ static int snd_pcm_oss_set_fragment1(struct snd_pcm_substream *substream, unsign
+ if (runtime->oss.subdivision || runtime->oss.fragshift)
+ return -EINVAL;
+ fragshift = val & 0xffff;
+- if (fragshift >= 31)
++ if (fragshift >= 25) /* should be large enough */
+ return -EINVAL;
+ runtime->oss.fragshift = fragshift;
+ runtime->oss.maxfrags = (val >> 16) & 0xffff;
+@@ -2052,7 +2069,7 @@ static int snd_pcm_oss_set_trigger(struct snd_pcm_oss_file *pcm_oss_file, int tr
+ int err, cmd;
+
+ #ifdef OSS_DEBUG
+- pcm_dbg(substream->pcm, "pcm_oss: trigger = 0x%x\n", trigger);
++ pr_debug("pcm_oss: trigger = 0x%x\n", trigger);
+ #endif
+
+ psubstream = pcm_oss_file->streams[SNDRV_PCM_STREAM_PLAYBACK];
+diff --git a/sound/core/oss/pcm_plugin.c b/sound/core/oss/pcm_plugin.c
+index 061ba06bc9262..82e180c776ae1 100644
+--- a/sound/core/oss/pcm_plugin.c
++++ b/sound/core/oss/pcm_plugin.c
+@@ -62,7 +62,10 @@ static int snd_pcm_plugin_alloc(struct snd_pcm_plugin *plugin, snd_pcm_uframes_t
+ width = snd_pcm_format_physical_width(format->format);
+ if (width < 0)
+ return width;
+- size = frames * format->channels * width;
++ size = array3_size(frames, format->channels, width);
++ /* check for too large period size once again */
++ if (size > 1024 * 1024)
++ return -ENOMEM;
+ if (snd_BUG_ON(size % 8))
+ return -ENXIO;
+ size /= 8;
+diff --git a/sound/core/pcm.c b/sound/core/pcm.c
+index 6fd3677685d70..977d54320a5ca 100644
+--- a/sound/core/pcm.c
++++ b/sound/core/pcm.c
+@@ -810,7 +810,11 @@ EXPORT_SYMBOL(snd_pcm_new_internal);
+ static void free_chmap(struct snd_pcm_str *pstr)
+ {
+ if (pstr->chmap_kctl) {
+- snd_ctl_remove(pstr->pcm->card, pstr->chmap_kctl);
++ struct snd_card *card = pstr->pcm->card;
++
++ down_write(&card->controls_rwsem);
++ snd_ctl_remove(card, pstr->chmap_kctl);
++ up_write(&card->controls_rwsem);
+ pstr->chmap_kctl = NULL;
+ }
+ }
+@@ -965,6 +969,8 @@ int snd_pcm_attach_substream(struct snd_pcm *pcm, int stream,
+ init_waitqueue_head(&runtime->tsleep);
+
+ runtime->status->state = SNDRV_PCM_STATE_OPEN;
++ mutex_init(&runtime->buffer_mutex);
++ atomic_set(&runtime->buffer_accessing, 0);
+
+ substream->runtime = runtime;
+ substream->private_data = pcm->private_data;
+@@ -998,6 +1004,7 @@ void snd_pcm_detach_substream(struct snd_pcm_substream *substream)
+ } else {
+ substream->runtime = NULL;
+ }
++ mutex_destroy(&runtime->buffer_mutex);
+ kfree(runtime);
+ put_pid(substream->pid);
+ substream->pid = NULL;
+diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c
+index a144a3f68e9eb..491064f55515b 100644
+--- a/sound/core/pcm_lib.c
++++ b/sound/core/pcm_lib.c
+@@ -2255,10 +2255,15 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
+ err = -EINVAL;
+ goto _end_unlock;
+ }
++ if (!atomic_inc_unless_negative(&runtime->buffer_accessing)) {
++ err = -EBUSY;
++ goto _end_unlock;
++ }
+ snd_pcm_stream_unlock_irq(substream);
+ err = writer(substream, appl_ofs, data, offset, frames,
+ transfer);
+ snd_pcm_stream_lock_irq(substream);
++ atomic_dec(&runtime->buffer_accessing);
+ if (err < 0)
+ goto _end_unlock;
+ err = pcm_accessible_state(runtime);
+diff --git a/sound/core/pcm_memory.c b/sound/core/pcm_memory.c
+index 7fbd1ccbb5b01..f1fb856ff4ac7 100644
+--- a/sound/core/pcm_memory.c
++++ b/sound/core/pcm_memory.c
+@@ -158,19 +158,20 @@ static void snd_pcm_lib_preallocate_proc_write(struct snd_info_entry *entry,
+ size_t size;
+ struct snd_dma_buffer new_dmab;
+
++ mutex_lock(&substream->pcm->open_mutex);
+ if (substream->runtime) {
+ buffer->error = -EBUSY;
+- return;
++ goto unlock;
+ }
+ if (!snd_info_get_line(buffer, line, sizeof(line))) {
+ snd_info_get_str(str, line, sizeof(str));
+ size = simple_strtoul(str, NULL, 10) * 1024;
+ if ((size != 0 && size < 8192) || size > substream->dma_max) {
+ buffer->error = -EINVAL;
+- return;
++ goto unlock;
+ }
+ if (substream->dma_buffer.bytes == size)
+- return;
++ goto unlock;
+ memset(&new_dmab, 0, sizeof(new_dmab));
+ new_dmab.dev = substream->dma_buffer.dev;
+ if (size > 0) {
+@@ -183,7 +184,7 @@ static void snd_pcm_lib_preallocate_proc_write(struct snd_info_entry *entry,
+ substream->pcm->card->number, substream->pcm->device,
+ substream->stream ? 'c' : 'p', substream->number,
+ substream->pcm->name, size);
+- return;
++ goto unlock;
+ }
+ substream->buffer_bytes_max = size;
+ } else {
+@@ -195,6 +196,8 @@ static void snd_pcm_lib_preallocate_proc_write(struct snd_info_entry *entry,
+ } else {
+ buffer->error = -EINVAL;
+ }
++ unlock:
++ mutex_unlock(&substream->pcm->open_mutex);
+ }
+
+ static inline void preallocate_info_init(struct snd_pcm_substream *substream)
+diff --git a/sound/core/pcm_misc.c b/sound/core/pcm_misc.c
+index 4866aed97aacc..5588b6a1ee8bd 100644
+--- a/sound/core/pcm_misc.c
++++ b/sound/core/pcm_misc.c
+@@ -433,7 +433,7 @@ int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int
+ return 0;
+ width = pcm_formats[(INT)format].phys; /* physical width */
+ pat = pcm_formats[(INT)format].silence;
+- if (! width)
++ if (!width || !pat)
+ return -EINVAL;
+ /* signed or 1 byte data */
+ if (pcm_formats[(INT)format].signd == 1 || width <= 8) {
+diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
+index d233cb3b41d8b..f38c2e5e9a297 100644
+--- a/sound/core/pcm_native.c
++++ b/sound/core/pcm_native.c
+@@ -672,6 +672,30 @@ static int snd_pcm_hw_params_choose(struct snd_pcm_substream *pcm,
+ return 0;
+ }
+
++/* acquire buffer_mutex; if it's in r/w operation, return -EBUSY, otherwise
++ * block the further r/w operations
++ */
++static int snd_pcm_buffer_access_lock(struct snd_pcm_runtime *runtime)
++{
++ if (!atomic_dec_unless_positive(&runtime->buffer_accessing))
++ return -EBUSY;
++ mutex_lock(&runtime->buffer_mutex);
++ return 0; /* keep buffer_mutex, unlocked by below */
++}
++
++/* release buffer_mutex and clear r/w access flag */
++static void snd_pcm_buffer_access_unlock(struct snd_pcm_runtime *runtime)
++{
++ mutex_unlock(&runtime->buffer_mutex);
++ atomic_inc(&runtime->buffer_accessing);
++}
++
++#if IS_ENABLED(CONFIG_SND_PCM_OSS)
++#define is_oss_stream(substream) ((substream)->oss.oss)
++#else
++#define is_oss_stream(substream) false
++#endif
++
+ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
+ struct snd_pcm_hw_params *params)
+ {
+@@ -683,22 +707,25 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
+ if (PCM_RUNTIME_CHECK(substream))
+ return -ENXIO;
+ runtime = substream->runtime;
++ err = snd_pcm_buffer_access_lock(runtime);
++ if (err < 0)
++ return err;
+ snd_pcm_stream_lock_irq(substream);
+ switch (runtime->status->state) {
+ case SNDRV_PCM_STATE_OPEN:
+ case SNDRV_PCM_STATE_SETUP:
+ case SNDRV_PCM_STATE_PREPARED:
++ if (!is_oss_stream(substream) &&
++ atomic_read(&substream->mmap_count))
++ err = -EBADFD;
+ break;
+ default:
+- snd_pcm_stream_unlock_irq(substream);
+- return -EBADFD;
++ err = -EBADFD;
++ break;
+ }
+ snd_pcm_stream_unlock_irq(substream);
+-#if IS_ENABLED(CONFIG_SND_PCM_OSS)
+- if (!substream->oss.oss)
+-#endif
+- if (atomic_read(&substream->mmap_count))
+- return -EBADFD;
++ if (err)
++ goto unlock;
+
+ snd_pcm_sync_stop(substream, true);
+
+@@ -786,16 +813,21 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
+ if (usecs >= 0)
+ cpu_latency_qos_add_request(&substream->latency_pm_qos_req,
+ usecs);
+- return 0;
++ err = 0;
+ _error:
+- /* hardware might be unusable from this time,
+- so we force application to retry to set
+- the correct hardware parameter settings */
+- snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN);
+- if (substream->ops->hw_free != NULL)
+- substream->ops->hw_free(substream);
+- if (substream->managed_buffer_alloc)
+- snd_pcm_lib_free_pages(substream);
++ if (err) {
++ /* hardware might be unusable from this time,
++ * so we force application to retry to set
++ * the correct hardware parameter settings
++ */
++ snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN);
++ if (substream->ops->hw_free != NULL)
++ substream->ops->hw_free(substream);
++ if (substream->managed_buffer_alloc)
++ snd_pcm_lib_free_pages(substream);
++ }
++ unlock:
++ snd_pcm_buffer_access_unlock(runtime);
+ return err;
+ }
+
+@@ -835,26 +867,33 @@ static int do_hw_free(struct snd_pcm_substream *substream)
+ static int snd_pcm_hw_free(struct snd_pcm_substream *substream)
+ {
+ struct snd_pcm_runtime *runtime;
+- int result;
++ int result = 0;
+
+ if (PCM_RUNTIME_CHECK(substream))
+ return -ENXIO;
+ runtime = substream->runtime;
++ result = snd_pcm_buffer_access_lock(runtime);
++ if (result < 0)
++ return result;
+ snd_pcm_stream_lock_irq(substream);
+ switch (runtime->status->state) {
+ case SNDRV_PCM_STATE_SETUP:
+ case SNDRV_PCM_STATE_PREPARED:
++ if (atomic_read(&substream->mmap_count))
++ result = -EBADFD;
+ break;
+ default:
+- snd_pcm_stream_unlock_irq(substream);
+- return -EBADFD;
++ result = -EBADFD;
++ break;
+ }
+ snd_pcm_stream_unlock_irq(substream);
+- if (atomic_read(&substream->mmap_count))
+- return -EBADFD;
++ if (result)
++ goto unlock;
+ result = do_hw_free(substream);
+ snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN);
+ cpu_latency_qos_remove_request(&substream->latency_pm_qos_req);
++ unlock:
++ snd_pcm_buffer_access_unlock(runtime);
+ return result;
+ }
+
+@@ -1160,15 +1199,17 @@ struct action_ops {
+ static int snd_pcm_action_group(const struct action_ops *ops,
+ struct snd_pcm_substream *substream,
+ snd_pcm_state_t state,
+- bool do_lock)
++ bool stream_lock)
+ {
+ struct snd_pcm_substream *s = NULL;
+ struct snd_pcm_substream *s1;
+ int res = 0, depth = 1;
+
+ snd_pcm_group_for_each_entry(s, substream) {
+- if (do_lock && s != substream) {
+- if (s->pcm->nonatomic)
++ if (s != substream) {
++ if (!stream_lock)
++ mutex_lock_nested(&s->runtime->buffer_mutex, depth);
++ else if (s->pcm->nonatomic)
+ mutex_lock_nested(&s->self_group.mutex, depth);
+ else
+ spin_lock_nested(&s->self_group.lock, depth);
+@@ -1196,18 +1237,18 @@ static int snd_pcm_action_group(const struct action_ops *ops,
+ ops->post_action(s, state);
+ }
+ _unlock:
+- if (do_lock) {
+- /* unlock streams */
+- snd_pcm_group_for_each_entry(s1, substream) {
+- if (s1 != substream) {
+- if (s1->pcm->nonatomic)
+- mutex_unlock(&s1->self_group.mutex);
+- else
+- spin_unlock(&s1->self_group.lock);
+- }
+- if (s1 == s) /* end */
+- break;
++ /* unlock streams */
++ snd_pcm_group_for_each_entry(s1, substream) {
++ if (s1 != substream) {
++ if (!stream_lock)
++ mutex_unlock(&s1->runtime->buffer_mutex);
++ else if (s1->pcm->nonatomic)
++ mutex_unlock(&s1->self_group.mutex);
++ else
++ spin_unlock(&s1->self_group.lock);
+ }
++ if (s1 == s) /* end */
++ break;
+ }
+ return res;
+ }
+@@ -1337,10 +1378,15 @@ static int snd_pcm_action_nonatomic(const struct action_ops *ops,
+
+ /* Guarantee the group members won't change during non-atomic action */
+ down_read(&snd_pcm_link_rwsem);
++ res = snd_pcm_buffer_access_lock(substream->runtime);
++ if (res < 0)
++ goto unlock;
+ if (snd_pcm_stream_linked(substream))
+ res = snd_pcm_action_group(ops, substream, state, false);
+ else
+ res = snd_pcm_action_single(ops, substream, state);
++ snd_pcm_buffer_access_unlock(substream->runtime);
++ unlock:
+ up_read(&snd_pcm_link_rwsem);
+ return res;
+ }
+@@ -1830,11 +1876,13 @@ static int snd_pcm_do_reset(struct snd_pcm_substream *substream,
+ int err = snd_pcm_ops_ioctl(substream, SNDRV_PCM_IOCTL1_RESET, NULL);
+ if (err < 0)
+ return err;
++ snd_pcm_stream_lock_irq(substream);
+ runtime->hw_ptr_base = 0;
+ runtime->hw_ptr_interrupt = runtime->status->hw_ptr -
+ runtime->status->hw_ptr % runtime->period_size;
+ runtime->silence_start = runtime->status->hw_ptr;
+ runtime->silence_filled = 0;
++ snd_pcm_stream_unlock_irq(substream);
+ return 0;
+ }
+
+@@ -1842,10 +1890,12 @@ static void snd_pcm_post_reset(struct snd_pcm_substream *substream,
+ snd_pcm_state_t state)
+ {
+ struct snd_pcm_runtime *runtime = substream->runtime;
++ snd_pcm_stream_lock_irq(substream);
+ runtime->control->appl_ptr = runtime->status->hw_ptr;
+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK &&
+ runtime->silence_size > 0)
+ snd_pcm_playback_silence(substream, ULONG_MAX);
++ snd_pcm_stream_unlock_irq(substream);
+ }
+
+ static const struct action_ops snd_pcm_action_reset = {
+diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c
+index 6f30231bdb884..befa9809ff001 100644
+--- a/sound/core/rawmidi.c
++++ b/sound/core/rawmidi.c
+@@ -447,6 +447,7 @@ static int snd_rawmidi_open(struct inode *inode, struct file *file)
+ err = -ENOMEM;
+ goto __error;
+ }
++ rawmidi_file->user_pversion = 0;
+ init_waitqueue_entry(&wait, current);
+ add_wait_queue(&rmidi->open_wait, &wait);
+ while (1) {
+diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c
+index d6c02dea976c8..bc933104c3eea 100644
+--- a/sound/core/seq/seq_queue.c
++++ b/sound/core/seq/seq_queue.c
+@@ -235,12 +235,15 @@ struct snd_seq_queue *snd_seq_queue_find_name(char *name)
+
+ /* -------------------------------------------------------- */
+
++#define MAX_CELL_PROCESSES_IN_QUEUE 1000
++
+ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop)
+ {
+ unsigned long flags;
+ struct snd_seq_event_cell *cell;
+ snd_seq_tick_time_t cur_tick;
+ snd_seq_real_time_t cur_time;
++ int processed = 0;
+
+ if (q == NULL)
+ return;
+@@ -263,6 +266,8 @@ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop)
+ if (!cell)
+ break;
+ snd_seq_dispatch_event(cell, atomic, hop);
++ if (++processed >= MAX_CELL_PROCESSES_IN_QUEUE)
++ goto out; /* the rest processed at the next batch */
+ }
+
+ /* Process time queue... */
+@@ -272,14 +277,19 @@ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop)
+ if (!cell)
+ break;
+ snd_seq_dispatch_event(cell, atomic, hop);
++ if (++processed >= MAX_CELL_PROCESSES_IN_QUEUE)
++ goto out; /* the rest processed at the next batch */
+ }
+
++ out:
+ /* free lock */
+ spin_lock_irqsave(&q->check_lock, flags);
+ if (q->check_again) {
+ q->check_again = 0;
+- spin_unlock_irqrestore(&q->check_lock, flags);
+- goto __again;
++ if (processed < MAX_CELL_PROCESSES_IN_QUEUE) {
++ spin_unlock_irqrestore(&q->check_lock, flags);
++ goto __again;
++ }
+ }
+ q->check_blocked = 0;
+ spin_unlock_irqrestore(&q->check_lock, flags);
+diff --git a/sound/core/timer.c b/sound/core/timer.c
+index 92b7008fcdb86..b3214baa89193 100644
+--- a/sound/core/timer.c
++++ b/sound/core/timer.c
+@@ -624,13 +624,13 @@ static int snd_timer_stop1(struct snd_timer_instance *timeri, bool stop)
+ if (!timer)
+ return -EINVAL;
+ spin_lock_irqsave(&timer->lock, flags);
++ list_del_init(&timeri->ack_list);
++ list_del_init(&timeri->active_list);
+ if (!(timeri->flags & (SNDRV_TIMER_IFLG_RUNNING |
+ SNDRV_TIMER_IFLG_START))) {
+ result = -EBUSY;
+ goto unlock;
+ }
+- list_del_init(&timeri->ack_list);
+- list_del_init(&timeri->active_list);
+ if (timer->card && timer->card->shutdown)
+ goto unlock;
+ if (stop) {
+@@ -665,23 +665,22 @@ static int snd_timer_stop1(struct snd_timer_instance *timeri, bool stop)
+ static int snd_timer_stop_slave(struct snd_timer_instance *timeri, bool stop)
+ {
+ unsigned long flags;
++ bool running;
+
+ spin_lock_irqsave(&slave_active_lock, flags);
+- if (!(timeri->flags & SNDRV_TIMER_IFLG_RUNNING)) {
+- spin_unlock_irqrestore(&slave_active_lock, flags);
+- return -EBUSY;
+- }
++ running = timeri->flags & SNDRV_TIMER_IFLG_RUNNING;
+ timeri->flags &= ~SNDRV_TIMER_IFLG_RUNNING;
+ if (timeri->timer) {
+ spin_lock(&timeri->timer->lock);
+ list_del_init(&timeri->ack_list);
+ list_del_init(&timeri->active_list);
+- snd_timer_notify1(timeri, stop ? SNDRV_TIMER_EVENT_STOP :
+- SNDRV_TIMER_EVENT_PAUSE);
++ if (running)
++ snd_timer_notify1(timeri, stop ? SNDRV_TIMER_EVENT_STOP :
++ SNDRV_TIMER_EVENT_PAUSE);
+ spin_unlock(&timeri->timer->lock);
+ }
+ spin_unlock_irqrestore(&slave_active_lock, flags);
+- return 0;
++ return running ? 0 : -EBUSY;
+ }
+
+ /*
+diff --git a/sound/drivers/mtpav.c b/sound/drivers/mtpav.c
+index 11235baaf6fa5..f212f233ea618 100644
+--- a/sound/drivers/mtpav.c
++++ b/sound/drivers/mtpav.c
+@@ -693,8 +693,6 @@ static int snd_mtpav_probe(struct platform_device *dev)
+ mtp_card->outmidihwport = 0xffffffff;
+ timer_setup(&mtp_card->timer, snd_mtpav_output_timer, 0);
+
+- card->private_free = snd_mtpav_free;
+-
+ err = snd_mtpav_get_RAWMIDI(mtp_card);
+ if (err < 0)
+ return err;
+@@ -716,6 +714,8 @@ static int snd_mtpav_probe(struct platform_device *dev)
+ if (err < 0)
+ return err;
+
++ card->private_free = snd_mtpav_free;
++
+ platform_set_drvdata(dev, card);
+ printk(KERN_INFO "Motu MidiTimePiece on parallel port irq: %d ioport: 0x%lx\n", irq, port);
+ return 0;
+diff --git a/sound/drivers/opl3/opl3_midi.c b/sound/drivers/opl3/opl3_midi.c
+index e1b69c65c3c88..e2b7be67f0e30 100644
+--- a/sound/drivers/opl3/opl3_midi.c
++++ b/sound/drivers/opl3/opl3_midi.c
+@@ -397,7 +397,7 @@ void snd_opl3_note_on(void *p, int note, int vel, struct snd_midi_channel *chan)
+ }
+ if (instr_4op) {
+ vp2 = &opl3->voices[voice + 3];
+- if (vp->state > 0) {
++ if (vp2->state > 0) {
+ opl3_reg = reg_side | (OPL3_REG_KEYON_BLOCK +
+ voice_offset + 3);
+ reg_val = vp->keyon_reg & ~OPL3_KEYON_BIT;
+diff --git a/sound/firewire/fcp.c b/sound/firewire/fcp.c
+index bbfbebf4affbc..df44dd5dc4b22 100644
+--- a/sound/firewire/fcp.c
++++ b/sound/firewire/fcp.c
+@@ -240,9 +240,7 @@ int fcp_avc_transaction(struct fw_unit *unit,
+ t.response_match_bytes = response_match_bytes;
+ t.state = STATE_PENDING;
+ init_waitqueue_head(&t.wait);
+-
+- if (*(const u8 *)command == 0x00 || *(const u8 *)command == 0x03)
+- t.deferrable = true;
++ t.deferrable = (*(const u8 *)command == 0x00 || *(const u8 *)command == 0x03);
+
+ spin_lock_irq(&transactions_lock);
+ list_add_tail(&t.list, &transactions);
+diff --git a/sound/firewire/fireworks/fireworks_hwdep.c b/sound/firewire/fireworks/fireworks_hwdep.c
+index 626c0c34b0b66..3a53914277d35 100644
+--- a/sound/firewire/fireworks/fireworks_hwdep.c
++++ b/sound/firewire/fireworks/fireworks_hwdep.c
+@@ -34,6 +34,7 @@ hwdep_read_resp_buf(struct snd_efw *efw, char __user *buf, long remained,
+ type = SNDRV_FIREWIRE_EVENT_EFW_RESPONSE;
+ if (copy_to_user(buf, &type, sizeof(type)))
+ return -EFAULT;
++ count += sizeof(type);
+ remained -= sizeof(type);
+ buf += sizeof(type);
+
+diff --git a/sound/firewire/oxfw/oxfw-stream.c b/sound/firewire/oxfw/oxfw-stream.c
+index fff18b5d4e052..f4a702def3979 100644
+--- a/sound/firewire/oxfw/oxfw-stream.c
++++ b/sound/firewire/oxfw/oxfw-stream.c
+@@ -9,7 +9,7 @@
+ #include <linux/delay.h>
+
+ #define AVC_GENERIC_FRAME_MAXIMUM_BYTES 512
+-#define READY_TIMEOUT_MS 200
++#define READY_TIMEOUT_MS 600
+
+ /*
+ * According to datasheet of Oxford Semiconductor:
+@@ -367,6 +367,11 @@ int snd_oxfw_stream_start_duplex(struct snd_oxfw *oxfw)
+ // Just after changing sampling transfer frequency, many cycles are
+ // skipped for packet transmission.
+ tx_init_skip_cycles = 400;
++ } else if (oxfw->quirks & SND_OXFW_QUIRK_VOLUNTARY_RECOVERY) {
++ // It takes a bit time for target device to adjust event frequency
++ // according to nominal event frequency in isochronous packets from
++ // ALSA oxfw driver.
++ tx_init_skip_cycles = 4000;
+ } else {
+ replay_seq = true;
+ }
+diff --git a/sound/firewire/oxfw/oxfw.c b/sound/firewire/oxfw/oxfw.c
+index daf731364695b..b496f87841aec 100644
+--- a/sound/firewire/oxfw/oxfw.c
++++ b/sound/firewire/oxfw/oxfw.c
+@@ -25,6 +25,7 @@
+ #define MODEL_SATELLITE 0x00200f
+ #define MODEL_SCS1M 0x001000
+ #define MODEL_DUET_FW 0x01dddd
++#define MODEL_ONYX_1640I 0x001640
+
+ #define SPECIFIER_1394TA 0x00a02d
+ #define VERSION_AVC 0x010001
+@@ -192,6 +193,13 @@ static int detect_quirks(struct snd_oxfw *oxfw, const struct ieee1394_device_id
+ // OXFW971-based models may transfer events by blocking method.
+ if (!(oxfw->quirks & SND_OXFW_QUIRK_JUMBO_PAYLOAD))
+ oxfw->quirks |= SND_OXFW_QUIRK_BLOCKING_TRANSMISSION;
++
++ if (model == MODEL_ONYX_1640I) {
++ //Unless receiving packets without NOINFO packet, the device transfers
++ //mostly half of events in packets than expected.
++ oxfw->quirks |= SND_OXFW_QUIRK_IGNORE_NO_INFO_PACKET |
++ SND_OXFW_QUIRK_VOLUNTARY_RECOVERY;
++ }
+ }
+
+ return 0;
+diff --git a/sound/firewire/oxfw/oxfw.h b/sound/firewire/oxfw/oxfw.h
+index c13034f6c2ca5..d728e451a25c6 100644
+--- a/sound/firewire/oxfw/oxfw.h
++++ b/sound/firewire/oxfw/oxfw.h
+@@ -47,6 +47,11 @@ enum snd_oxfw_quirk {
+ // the device to process audio data even if the value is invalid in a point of
+ // IEC 61883-1/6.
+ SND_OXFW_QUIRK_IGNORE_NO_INFO_PACKET = 0x10,
++ // Loud Technologies Mackie Onyx 1640i seems to configure OXFW971 ASIC so that it decides
++ // event frequency according to events in received isochronous packets. The device looks to
++ // performs media clock recovery voluntarily. In the recovery, the packets with NO_INFO
++ // are ignored, thus driver should transfer packets with timestamp.
++ SND_OXFW_QUIRK_VOLUNTARY_RECOVERY = 0x20,
+ };
+
+ /* This is an arbitrary number for convinience. */
+diff --git a/sound/hda/ext/hdac_ext_stream.c b/sound/hda/ext/hdac_ext_stream.c
+index 0c005d67fa891..37154ed43bd53 100644
+--- a/sound/hda/ext/hdac_ext_stream.c
++++ b/sound/hda/ext/hdac_ext_stream.c
+@@ -106,20 +106,14 @@ void snd_hdac_stream_free_all(struct hdac_bus *bus)
+ }
+ EXPORT_SYMBOL_GPL(snd_hdac_stream_free_all);
+
+-/**
+- * snd_hdac_ext_stream_decouple - decouple the hdac stream
+- * @bus: HD-audio core bus
+- * @stream: HD-audio ext core stream object to initialize
+- * @decouple: flag to decouple
+- */
+-void snd_hdac_ext_stream_decouple(struct hdac_bus *bus,
+- struct hdac_ext_stream *stream, bool decouple)
++void snd_hdac_ext_stream_decouple_locked(struct hdac_bus *bus,
++ struct hdac_ext_stream *stream,
++ bool decouple)
+ {
+ struct hdac_stream *hstream = &stream->hstream;
+ u32 val;
+ int mask = AZX_PPCTL_PROCEN(hstream->index);
+
+- spin_lock_irq(&bus->reg_lock);
+ val = readw(bus->ppcap + AZX_REG_PP_PPCTL) & mask;
+
+ if (decouple && !val)
+@@ -128,6 +122,20 @@ void snd_hdac_ext_stream_decouple(struct hdac_bus *bus,
+ snd_hdac_updatel(bus->ppcap, AZX_REG_PP_PPCTL, mask, 0);
+
+ stream->decoupled = decouple;
++}
++EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_decouple_locked);
++
++/**
++ * snd_hdac_ext_stream_decouple - decouple the hdac stream
++ * @bus: HD-audio core bus
++ * @stream: HD-audio ext core stream object to initialize
++ * @decouple: flag to decouple
++ */
++void snd_hdac_ext_stream_decouple(struct hdac_bus *bus,
++ struct hdac_ext_stream *stream, bool decouple)
++{
++ spin_lock_irq(&bus->reg_lock);
++ snd_hdac_ext_stream_decouple_locked(bus, stream, decouple);
+ spin_unlock_irq(&bus->reg_lock);
+ }
+ EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_decouple);
+@@ -252,6 +260,7 @@ hdac_ext_link_stream_assign(struct hdac_bus *bus,
+ return NULL;
+ }
+
++ spin_lock_irq(&bus->reg_lock);
+ list_for_each_entry(stream, &bus->stream_list, list) {
+ struct hdac_ext_stream *hstream = container_of(stream,
+ struct hdac_ext_stream,
+@@ -266,17 +275,16 @@ hdac_ext_link_stream_assign(struct hdac_bus *bus,
+ }
+
+ if (!hstream->link_locked) {
+- snd_hdac_ext_stream_decouple(bus, hstream, true);
++ snd_hdac_ext_stream_decouple_locked(bus, hstream, true);
+ res = hstream;
+ break;
+ }
+ }
+ if (res) {
+- spin_lock_irq(&bus->reg_lock);
+ res->link_locked = 1;
+ res->link_substream = substream;
+- spin_unlock_irq(&bus->reg_lock);
+ }
++ spin_unlock_irq(&bus->reg_lock);
+ return res;
+ }
+
+@@ -292,6 +300,7 @@ hdac_ext_host_stream_assign(struct hdac_bus *bus,
+ return NULL;
+ }
+
++ spin_lock_irq(&bus->reg_lock);
+ list_for_each_entry(stream, &bus->stream_list, list) {
+ struct hdac_ext_stream *hstream = container_of(stream,
+ struct hdac_ext_stream,
+@@ -301,18 +310,17 @@ hdac_ext_host_stream_assign(struct hdac_bus *bus,
+
+ if (!stream->opened) {
+ if (!hstream->decoupled)
+- snd_hdac_ext_stream_decouple(bus, hstream, true);
++ snd_hdac_ext_stream_decouple_locked(bus, hstream, true);
+ res = hstream;
+ break;
+ }
+ }
+ if (res) {
+- spin_lock_irq(&bus->reg_lock);
+ res->hstream.opened = 1;
+ res->hstream.running = 0;
+ res->hstream.substream = substream;
+- spin_unlock_irq(&bus->reg_lock);
+ }
++ spin_unlock_irq(&bus->reg_lock);
+
+ return res;
+ }
+@@ -378,15 +386,17 @@ void snd_hdac_ext_stream_release(struct hdac_ext_stream *stream, int type)
+ break;
+
+ case HDAC_EXT_STREAM_TYPE_HOST:
++ spin_lock_irq(&bus->reg_lock);
+ if (stream->decoupled && !stream->link_locked)
+- snd_hdac_ext_stream_decouple(bus, stream, false);
++ snd_hdac_ext_stream_decouple_locked(bus, stream, false);
++ spin_unlock_irq(&bus->reg_lock);
+ snd_hdac_stream_release(&stream->hstream);
+ break;
+
+ case HDAC_EXT_STREAM_TYPE_LINK:
+- if (stream->decoupled && !stream->hstream.opened)
+- snd_hdac_ext_stream_decouple(bus, stream, false);
+ spin_lock_irq(&bus->reg_lock);
++ if (stream->decoupled && !stream->hstream.opened)
++ snd_hdac_ext_stream_decouple_locked(bus, stream, false);
+ stream->link_locked = 0;
+ stream->link_substream = NULL;
+ spin_unlock_irq(&bus->reg_lock);
+diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c
+index 1eb8563db2dff..aa7955fdf68a0 100644
+--- a/sound/hda/hdac_stream.c
++++ b/sound/hda/hdac_stream.c
+@@ -296,6 +296,7 @@ struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus,
+ int key = (substream->pcm->device << 16) | (substream->number << 2) |
+ (substream->stream + 1);
+
++ spin_lock_irq(&bus->reg_lock);
+ list_for_each_entry(azx_dev, &bus->stream_list, list) {
+ if (azx_dev->direction != substream->stream)
+ continue;
+@@ -309,13 +310,12 @@ struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus,
+ res = azx_dev;
+ }
+ if (res) {
+- spin_lock_irq(&bus->reg_lock);
+ res->opened = 1;
+ res->running = 0;
+ res->assigned_key = key;
+ res->substream = substream;
+- spin_unlock_irq(&bus->reg_lock);
+ }
++ spin_unlock_irq(&bus->reg_lock);
+ return res;
+ }
+ EXPORT_SYMBOL_GPL(snd_hdac_stream_assign);
+@@ -534,17 +534,11 @@ static void azx_timecounter_init(struct hdac_stream *azx_dev,
+ cc->mask = CLOCKSOURCE_MASK(32);
+
+ /*
+- * Converting from 24 MHz to ns means applying a 125/3 factor.
+- * To avoid any saturation issues in intermediate operations,
+- * the 125 factor is applied first. The division is applied
+- * last after reading the timecounter value.
+- * Applying the 1/3 factor as part of the multiplication
+- * requires at least 20 bits for a decent precision, however
+- * overflows occur after about 4 hours or less, not a option.
++ * Calculate the optimal mult/shift values. The counter wraps
++ * around after ~178.9 seconds.
+ */
+-
+- cc->mult = 125; /* saturation after 195 years */
+- cc->shift = 0;
++ clocks_calc_mult_shift(&cc->mult, &cc->shift, 24000000,
++ NSEC_PER_SEC, 178);
+
+ nsec = 0; /* audio time is elapsed time since trigger */
+ timecounter_init(tc, cc, nsec);
+diff --git a/sound/hda/intel-dsp-config.c b/sound/hda/intel-dsp-config.c
+index c9d0ba353463b..4208fa8a4db5b 100644
+--- a/sound/hda/intel-dsp-config.c
++++ b/sound/hda/intel-dsp-config.c
+@@ -31,6 +31,7 @@ struct config_entry {
+ u16 device;
+ u8 acpi_hid[ACPI_ID_LEN];
+ const struct dmi_system_id *dmi_table;
++ u8 codec_hid[ACPI_ID_LEN];
+ };
+
+ /*
+@@ -56,7 +57,7 @@ static const struct config_entry config_table[] = {
+ /*
+ * Apollolake (Broxton-P)
+ * the legacy HDAudio driver is used except on Up Squared (SOF) and
+- * Chromebooks (SST)
++ * Chromebooks (SST), as well as devices based on the ES8336 codec
+ */
+ #if IS_ENABLED(CONFIG_SND_SOC_SOF_APOLLOLAKE)
+ {
+@@ -73,6 +74,11 @@ static const struct config_entry config_table[] = {
+ {}
+ }
+ },
++ {
++ .flags = FLAG_SOF,
++ .device = 0x5a98,
++ .codec_hid = "ESSX8336",
++ },
+ #endif
+ #if IS_ENABLED(CONFIG_SND_SOC_INTEL_APL)
+ {
+@@ -137,7 +143,7 @@ static const struct config_entry config_table[] = {
+
+ /*
+ * Geminilake uses legacy HDAudio driver except for Google
+- * Chromebooks
++ * Chromebooks and devices based on the ES8336 codec
+ */
+ /* Geminilake */
+ #if IS_ENABLED(CONFIG_SND_SOC_SOF_GEMINILAKE)
+@@ -154,6 +160,11 @@ static const struct config_entry config_table[] = {
+ {}
+ }
+ },
++ {
++ .flags = FLAG_SOF,
++ .device = 0x3198,
++ .codec_hid = "ESSX8336",
++ },
+ #endif
+
+ /*
+@@ -241,6 +252,11 @@ static const struct config_entry config_table[] = {
+ .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
+ .device = 0x02c8,
+ },
++ {
++ .flags = FLAG_SOF,
++ .device = 0x02c8,
++ .codec_hid = "ESSX8336",
++ },
+ /* Cometlake-H */
+ {
+ .flags = FLAG_SOF,
+@@ -265,6 +281,11 @@ static const struct config_entry config_table[] = {
+ .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
+ .device = 0x06c8,
+ },
++ {
++ .flags = FLAG_SOF,
++ .device = 0x06c8,
++ .codec_hid = "ESSX8336",
++ },
+ #endif
+
+ /* Icelake */
+@@ -288,6 +309,15 @@ static const struct config_entry config_table[] = {
+ },
+ #endif
+
++/* JasperLake */
++#if IS_ENABLED(CONFIG_SND_SOC_SOF_JASPERLAKE)
++ {
++ .flags = FLAG_SOF,
++ .device = 0x4dc8,
++ .codec_hid = "ESSX8336",
++ },
++#endif
++
+ /* Tigerlake */
+ #if IS_ENABLED(CONFIG_SND_SOC_SOF_TIGERLAKE)
+ {
+@@ -311,6 +341,11 @@ static const struct config_entry config_table[] = {
+ .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
+ .device = 0x43c8,
+ },
++ {
++ .flags = FLAG_SOF,
++ .device = 0xa0c8,
++ .codec_hid = "ESSX8336",
++ },
+ #endif
+
+ /* Elkhart Lake */
+@@ -354,6 +389,8 @@ static const struct config_entry *snd_intel_dsp_find_config
+ continue;
+ if (table->dmi_table && !dmi_check_system(table->dmi_table))
+ continue;
++ if (table->codec_hid[0] && !acpi_dev_present(table->codec_hid, NULL, -1))
++ continue;
+ return table;
+ }
+ return NULL;
+diff --git a/sound/hda/intel-sdw-acpi.c b/sound/hda/intel-sdw-acpi.c
+index c0123bc31c0dd..b7758dbe23714 100644
+--- a/sound/hda/intel-sdw-acpi.c
++++ b/sound/hda/intel-sdw-acpi.c
+@@ -132,8 +132,6 @@ static acpi_status sdw_intel_acpi_cb(acpi_handle handle, u32 level,
+ return AE_NOT_FOUND;
+ }
+
+- info->handle = handle;
+-
+ /*
+ * On some Intel platforms, multiple children of the HDAS
+ * device can be found, but only one of them is the SoundWire
+@@ -144,6 +142,9 @@ static acpi_status sdw_intel_acpi_cb(acpi_handle handle, u32 level,
+ if (FIELD_GET(GENMASK(31, 28), adr) != SDW_LINK_TYPE)
+ return AE_OK; /* keep going */
+
++ /* found the correct SoundWire controller */
++ info->handle = handle;
++
+ /* device found, stop namespace walk */
+ return AE_CTRL_TERMINATE;
+ }
+@@ -164,8 +165,14 @@ int sdw_intel_acpi_scan(acpi_handle *parent_handle,
+ acpi_status status;
+
+ info->handle = NULL;
++ /*
++ * In the HDAS ACPI scope, 'SNDW' may be either the child of
++ * 'HDAS' or the grandchild of 'HDAS'. So let's go through
++ * the ACPI from 'HDAS' at max depth of 2 to find the 'SNDW'
++ * device.
++ */
+ status = acpi_walk_namespace(ACPI_TYPE_DEVICE,
+- parent_handle, 1,
++ parent_handle, 2,
+ sdw_intel_acpi_cb,
+ NULL, info, NULL);
+ if (ACPI_FAILURE(status) || info->handle == NULL)
+diff --git a/sound/isa/Kconfig b/sound/isa/Kconfig
+index 6ffa48dd59830..570b88e0b2018 100644
+--- a/sound/isa/Kconfig
++++ b/sound/isa/Kconfig
+@@ -22,7 +22,7 @@ config SND_SB16_DSP
+ menuconfig SND_ISA
+ bool "ISA sound devices"
+ depends on ISA || COMPILE_TEST
+- depends on ISA_DMA_API
++ depends on ISA_DMA_API && !M68K
+ default y
+ help
+ Support for sound devices connected via the ISA bus.
+diff --git a/sound/isa/cs423x/cs4236.c b/sound/isa/cs423x/cs4236.c
+index b6bdebd9ef275..10112e1bb25dc 100644
+--- a/sound/isa/cs423x/cs4236.c
++++ b/sound/isa/cs423x/cs4236.c
+@@ -494,7 +494,7 @@ static int snd_cs423x_pnpbios_detect(struct pnp_dev *pdev,
+ static int dev;
+ int err;
+ struct snd_card *card;
+- struct pnp_dev *cdev;
++ struct pnp_dev *cdev, *iter;
+ char cid[PNP_ID_LEN];
+
+ if (pnp_device_is_isapnp(pdev))
+@@ -510,9 +510,11 @@ static int snd_cs423x_pnpbios_detect(struct pnp_dev *pdev,
+ strcpy(cid, pdev->id[0].id);
+ cid[5] = '1';
+ cdev = NULL;
+- list_for_each_entry(cdev, &(pdev->protocol->devices), protocol_list) {
+- if (!strcmp(cdev->id[0].id, cid))
++ list_for_each_entry(iter, &(pdev->protocol->devices), protocol_list) {
++ if (!strcmp(iter->id[0].id, cid)) {
++ cdev = iter;
+ break;
++ }
+ }
+ err = snd_cs423x_card_new(&pdev->dev, dev, &card);
+ if (err < 0)
+diff --git a/sound/isa/galaxy/galaxy.c b/sound/isa/galaxy/galaxy.c
+index ea001c80149dd..3164eb8510fa4 100644
+--- a/sound/isa/galaxy/galaxy.c
++++ b/sound/isa/galaxy/galaxy.c
+@@ -478,7 +478,7 @@ static void snd_galaxy_free(struct snd_card *card)
+ galaxy_set_config(galaxy, galaxy->config);
+ }
+
+-static int snd_galaxy_probe(struct device *dev, unsigned int n)
++static int __snd_galaxy_probe(struct device *dev, unsigned int n)
+ {
+ struct snd_galaxy *galaxy;
+ struct snd_wss *chip;
+@@ -598,6 +598,11 @@ static int snd_galaxy_probe(struct device *dev, unsigned int n)
+ return 0;
+ }
+
++static int snd_galaxy_probe(struct device *dev, unsigned int n)
++{
++ return snd_card_free_on_error(dev, __snd_galaxy_probe(dev, n));
++}
++
+ static struct isa_driver snd_galaxy_driver = {
+ .match = snd_galaxy_match,
+ .probe = snd_galaxy_probe,
+diff --git a/sound/isa/gus/gus_dma.c b/sound/isa/gus/gus_dma.c
+index a1c770d826dda..6d664dd8dde0b 100644
+--- a/sound/isa/gus/gus_dma.c
++++ b/sound/isa/gus/gus_dma.c
+@@ -126,6 +126,8 @@ static void snd_gf1_dma_interrupt(struct snd_gus_card * gus)
+ }
+ block = snd_gf1_dma_next_block(gus);
+ spin_unlock(&gus->dma_lock);
++ if (!block)
++ return;
+ snd_gf1_dma_program(gus, block->addr, block->buf_addr, block->count, (unsigned short) block->cmd);
+ kfree(block);
+ #if 0
+diff --git a/sound/isa/sc6000.c b/sound/isa/sc6000.c
+index 26ab7ff807684..60398fced046b 100644
+--- a/sound/isa/sc6000.c
++++ b/sound/isa/sc6000.c
+@@ -537,7 +537,7 @@ static void snd_sc6000_free(struct snd_card *card)
+ sc6000_setup_board(vport, 0);
+ }
+
+-static int snd_sc6000_probe(struct device *devptr, unsigned int dev)
++static int __snd_sc6000_probe(struct device *devptr, unsigned int dev)
+ {
+ static const int possible_irqs[] = { 5, 7, 9, 10, 11, -1 };
+ static const int possible_dmas[] = { 1, 3, 0, -1 };
+@@ -662,6 +662,11 @@ static int snd_sc6000_probe(struct device *devptr, unsigned int dev)
+ return 0;
+ }
+
++static int snd_sc6000_probe(struct device *devptr, unsigned int dev)
++{
++ return snd_card_free_on_error(devptr, __snd_sc6000_probe(devptr, dev));
++}
++
+ static struct isa_driver snd_sc6000_driver = {
+ .match = snd_sc6000_match,
+ .probe = snd_sc6000_probe,
+diff --git a/sound/isa/wavefront/wavefront_synth.c b/sound/isa/wavefront/wavefront_synth.c
+index 69cbc79fbb716..2aaaa68071744 100644
+--- a/sound/isa/wavefront/wavefront_synth.c
++++ b/sound/isa/wavefront/wavefront_synth.c
+@@ -1094,7 +1094,8 @@ wavefront_send_sample (snd_wavefront_t *dev,
+
+ if (dataptr < data_end) {
+
+- __get_user (sample_short, dataptr);
++ if (get_user(sample_short, dataptr))
++ return -EFAULT;
+ dataptr += skip;
+
+ if (data_is_unsigned) { /* GUS ? */
+diff --git a/sound/pci/Kconfig b/sound/pci/Kconfig
+index 93bc9bef7641f..41ce125971777 100644
+--- a/sound/pci/Kconfig
++++ b/sound/pci/Kconfig
+@@ -279,6 +279,7 @@ config SND_CS46XX_NEW_DSP
+ config SND_CS5530
+ tristate "CS5530 Audio"
+ depends on ISA_DMA_API && (X86_32 || COMPILE_TEST)
++ depends on !M68K
+ select SND_SB16_DSP
+ help
+ Say Y here to include support for audio on Cyrix/NatSemi CS5530 chips.
+diff --git a/sound/pci/ac97/ac97_codec.c b/sound/pci/ac97/ac97_codec.c
+index 01f296d524ce6..cb60a07d39a8e 100644
+--- a/sound/pci/ac97/ac97_codec.c
++++ b/sound/pci/ac97/ac97_codec.c
+@@ -938,8 +938,8 @@ static int snd_ac97_ad18xx_pcm_get_volume(struct snd_kcontrol *kcontrol, struct
+ int codec = kcontrol->private_value & 3;
+
+ mutex_lock(&ac97->page_mutex);
+- ucontrol->value.integer.value[0] = 31 - ((ac97->spec.ad18xx.pcmreg[codec] >> 0) & 31);
+- ucontrol->value.integer.value[1] = 31 - ((ac97->spec.ad18xx.pcmreg[codec] >> 8) & 31);
++ ucontrol->value.integer.value[0] = 31 - ((ac97->spec.ad18xx.pcmreg[codec] >> 8) & 31);
++ ucontrol->value.integer.value[1] = 31 - ((ac97->spec.ad18xx.pcmreg[codec] >> 0) & 31);
+ mutex_unlock(&ac97->page_mutex);
+ return 0;
+ }
+diff --git a/sound/pci/ad1889.c b/sound/pci/ad1889.c
+index bba4dae8dcc70..50e30704bf6f9 100644
+--- a/sound/pci/ad1889.c
++++ b/sound/pci/ad1889.c
+@@ -844,8 +844,8 @@ snd_ad1889_create(struct snd_card *card, struct pci_dev *pci)
+ }
+
+ static int
+-snd_ad1889_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++__snd_ad1889_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ int err;
+ static int devno;
+@@ -904,6 +904,12 @@ snd_ad1889_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_ad1889_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_ad1889_probe(pci, pci_id));
++}
++
+ static const struct pci_device_id snd_ad1889_ids[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_ANALOG_DEVICES, PCI_DEVICE_ID_AD1889JS) },
+ { 0, },
+diff --git a/sound/pci/ali5451/ali5451.c b/sound/pci/ali5451/ali5451.c
+index 92eb59db106de..2378a39abaebe 100644
+--- a/sound/pci/ali5451/ali5451.c
++++ b/sound/pci/ali5451/ali5451.c
+@@ -2124,8 +2124,8 @@ static int snd_ali_create(struct snd_card *card,
+ return 0;
+ }
+
+-static int snd_ali_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_ali_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct snd_ali *codec;
+@@ -2170,6 +2170,12 @@ static int snd_ali_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_ali_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_ali_probe(pci, pci_id));
++}
++
+ static struct pci_driver ali5451_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_ali_ids,
+diff --git a/sound/pci/als300.c b/sound/pci/als300.c
+index b86565dcdbe41..c70aff0601205 100644
+--- a/sound/pci/als300.c
++++ b/sound/pci/als300.c
+@@ -708,7 +708,7 @@ static int snd_als300_probe(struct pci_dev *pci,
+
+ err = snd_als300_create(card, pci, chip_type);
+ if (err < 0)
+- return err;
++ goto error;
+
+ strcpy(card->driver, "ALS300");
+ if (chip->chip_type == DEVICE_ALS300_PLUS)
+@@ -723,11 +723,15 @@ static int snd_als300_probe(struct pci_dev *pci,
+
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver als300_driver = {
+diff --git a/sound/pci/als4000.c b/sound/pci/als4000.c
+index 535eccd124bee..f33aeb692a112 100644
+--- a/sound/pci/als4000.c
++++ b/sound/pci/als4000.c
+@@ -806,8 +806,8 @@ static void snd_card_als4000_free( struct snd_card *card )
+ snd_als4000_free_gameport(acard);
+ }
+
+-static int snd_card_als4000_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_card_als4000_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -930,6 +930,12 @@ static int snd_card_als4000_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_card_als4000_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_card_als4000_probe(pci, pci_id));
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static int snd_als4000_suspend(struct device *dev)
+ {
+diff --git a/sound/pci/atiixp.c b/sound/pci/atiixp.c
+index b8e035d5930d2..43d01f1847ed7 100644
+--- a/sound/pci/atiixp.c
++++ b/sound/pci/atiixp.c
+@@ -1572,8 +1572,8 @@ static int snd_atiixp_init(struct snd_card *card, struct pci_dev *pci)
+ }
+
+
+-static int snd_atiixp_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_atiixp_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct atiixp *chip;
+@@ -1623,6 +1623,12 @@ static int snd_atiixp_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_atiixp_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_atiixp_probe(pci, pci_id));
++}
++
+ static struct pci_driver atiixp_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_atiixp_ids,
+diff --git a/sound/pci/atiixp_modem.c b/sound/pci/atiixp_modem.c
+index 178dce8ef1e99..8864c4c3c7e13 100644
+--- a/sound/pci/atiixp_modem.c
++++ b/sound/pci/atiixp_modem.c
+@@ -1201,8 +1201,8 @@ static int snd_atiixp_init(struct snd_card *card, struct pci_dev *pci)
+ }
+
+
+-static int snd_atiixp_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_atiixp_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct atiixp_modem *chip;
+@@ -1247,6 +1247,12 @@ static int snd_atiixp_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_atiixp_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_atiixp_probe(pci, pci_id));
++}
++
+ static struct pci_driver atiixp_modem_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_atiixp_ids,
+diff --git a/sound/pci/au88x0/au88x0.c b/sound/pci/au88x0/au88x0.c
+index 342ef2a6655e3..eb234153691bc 100644
+--- a/sound/pci/au88x0/au88x0.c
++++ b/sound/pci/au88x0/au88x0.c
+@@ -193,7 +193,7 @@ snd_vortex_create(struct snd_card *card, struct pci_dev *pci)
+
+ // constructor -- see "Constructor" sub-section
+ static int
+-snd_vortex_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++__snd_vortex_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -310,6 +310,12 @@ snd_vortex_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ return 0;
+ }
+
++static int
++snd_vortex_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_vortex_probe(pci, pci_id));
++}
++
+ // pci_driver definition
+ static struct pci_driver vortex_driver = {
+ .name = KBUILD_MODNAME,
+diff --git a/sound/pci/aw2/aw2-alsa.c b/sound/pci/aw2/aw2-alsa.c
+index d56f126d6fdd9..29a4bcdec237a 100644
+--- a/sound/pci/aw2/aw2-alsa.c
++++ b/sound/pci/aw2/aw2-alsa.c
+@@ -275,7 +275,7 @@ static int snd_aw2_probe(struct pci_dev *pci,
+ /* (3) Create main component */
+ err = snd_aw2_create(card, pci);
+ if (err < 0)
+- return err;
++ goto error;
+
+ /* initialize mutex */
+ mutex_init(&chip->mtx);
+@@ -294,13 +294,17 @@ static int snd_aw2_probe(struct pci_dev *pci,
+ /* (6) Register card instance */
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+
+ /* (7) Set PCI driver data */
+ pci_set_drvdata(pci, card);
+
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ /* open callback */
+diff --git a/sound/pci/azt3328.c b/sound/pci/azt3328.c
+index 089050470ff27..7f329dfc5404a 100644
+--- a/sound/pci/azt3328.c
++++ b/sound/pci/azt3328.c
+@@ -2427,7 +2427,7 @@ snd_azf3328_create(struct snd_card *card,
+ }
+
+ static int
+-snd_azf3328_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++__snd_azf3328_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2520,6 +2520,12 @@ snd_azf3328_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ return 0;
+ }
+
++static int
++snd_azf3328_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_azf3328_probe(pci, pci_id));
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static inline void
+ snd_azf3328_suspend_regs(const struct snd_azf3328 *chip,
+diff --git a/sound/pci/bt87x.c b/sound/pci/bt87x.c
+index d23f931638410..621985bfee5d7 100644
+--- a/sound/pci/bt87x.c
++++ b/sound/pci/bt87x.c
+@@ -805,8 +805,8 @@ static int snd_bt87x_detect_card(struct pci_dev *pci)
+ return SND_BT87X_BOARD_UNKNOWN;
+ }
+
+-static int snd_bt87x_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_bt87x_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -889,6 +889,12 @@ static int snd_bt87x_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_bt87x_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_bt87x_probe(pci, pci_id));
++}
++
+ /* default entries for all Bt87x cards - it's not exported */
+ /* driver_data is set to 0 to call detection */
+ static const struct pci_device_id snd_bt87x_default_ids[] = {
+diff --git a/sound/pci/ca0106/ca0106_main.c b/sound/pci/ca0106/ca0106_main.c
+index 36fb150b72fb5..f4cc112bddf3e 100644
+--- a/sound/pci/ca0106/ca0106_main.c
++++ b/sound/pci/ca0106/ca0106_main.c
+@@ -1725,8 +1725,8 @@ static int snd_ca0106_midi(struct snd_ca0106 *chip, unsigned int channel)
+ }
+
+
+-static int snd_ca0106_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_ca0106_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1786,6 +1786,12 @@ static int snd_ca0106_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_ca0106_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_ca0106_probe(pci, pci_id));
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static int snd_ca0106_suspend(struct device *dev)
+ {
+diff --git a/sound/pci/cmipci.c b/sound/pci/cmipci.c
+index ea20236f35dbc..42fcbed9220c1 100644
+--- a/sound/pci/cmipci.c
++++ b/sound/pci/cmipci.c
+@@ -298,7 +298,6 @@ MODULE_PARM_DESC(joystick_port, "Joystick port address.");
+ #define CM_MICGAINZ 0x01 /* mic boost */
+ #define CM_MICGAINZ_SHIFT 0
+
+-#define CM_REG_MIXER3 0x24
+ #define CM_REG_AUX_VOL 0x26
+ #define CM_VAUXL_MASK 0xf0
+ #define CM_VAUXR_MASK 0x0f
+@@ -3250,15 +3249,19 @@ static int snd_cmipci_probe(struct pci_dev *pci,
+
+ err = snd_cmipci_create(card, pci, dev);
+ if (err < 0)
+- return err;
++ goto error;
+
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ #ifdef CONFIG_PM_SLEEP
+@@ -3267,7 +3270,7 @@ static int snd_cmipci_probe(struct pci_dev *pci,
+ */
+ static const unsigned char saved_regs[] = {
+ CM_REG_FUNCTRL1, CM_REG_CHFORMAT, CM_REG_LEGACY_CTRL, CM_REG_MISC_CTRL,
+- CM_REG_MIXER0, CM_REG_MIXER1, CM_REG_MIXER2, CM_REG_MIXER3, CM_REG_PLL,
++ CM_REG_MIXER0, CM_REG_MIXER1, CM_REG_MIXER2, CM_REG_AUX_VOL, CM_REG_PLL,
+ CM_REG_CH0_FRAME1, CM_REG_CH0_FRAME2,
+ CM_REG_CH1_FRAME1, CM_REG_CH1_FRAME2, CM_REG_EXT_MISC,
+ CM_REG_INT_STATUS, CM_REG_INT_HLDCLR, CM_REG_FUNCTRL0,
+diff --git a/sound/pci/cs4281.c b/sound/pci/cs4281.c
+index e7367402b84a3..0c9cadf7b3b80 100644
+--- a/sound/pci/cs4281.c
++++ b/sound/pci/cs4281.c
+@@ -1827,8 +1827,8 @@ static void snd_cs4281_opl3_command(struct snd_opl3 *opl3, unsigned short cmd,
+ spin_unlock_irqrestore(&opl3->reg_lock, flags);
+ }
+
+-static int snd_cs4281_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_cs4281_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1888,6 +1888,12 @@ static int snd_cs4281_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_cs4281_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_cs4281_probe(pci, pci_id));
++}
++
+ /*
+ * Power Management
+ */
+diff --git a/sound/pci/cs5535audio/cs5535audio.c b/sound/pci/cs5535audio/cs5535audio.c
+index 499fa0148f9a4..440b8f9b40c96 100644
+--- a/sound/pci/cs5535audio/cs5535audio.c
++++ b/sound/pci/cs5535audio/cs5535audio.c
+@@ -281,8 +281,8 @@ static int snd_cs5535audio_create(struct snd_card *card,
+ return 0;
+ }
+
+-static int snd_cs5535audio_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_cs5535audio_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -331,6 +331,12 @@ static int snd_cs5535audio_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_cs5535audio_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_cs5535audio_probe(pci, pci_id));
++}
++
+ static struct pci_driver cs5535audio_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_cs5535audio_ids,
+diff --git a/sound/pci/ctxfi/ctamixer.c b/sound/pci/ctxfi/ctamixer.c
+index da6e6350ceafa..d074727c3e21d 100644
+--- a/sound/pci/ctxfi/ctamixer.c
++++ b/sound/pci/ctxfi/ctamixer.c
+@@ -23,16 +23,15 @@
+
+ #define BLANK_SLOT 4094
+
+-static int amixer_master(struct rsc *rsc)
++static void amixer_master(struct rsc *rsc)
+ {
+ rsc->conj = 0;
+- return rsc->idx = container_of(rsc, struct amixer, rsc)->idx[0];
++ rsc->idx = container_of(rsc, struct amixer, rsc)->idx[0];
+ }
+
+-static int amixer_next_conj(struct rsc *rsc)
++static void amixer_next_conj(struct rsc *rsc)
+ {
+ rsc->conj++;
+- return container_of(rsc, struct amixer, rsc)->idx[rsc->conj];
+ }
+
+ static int amixer_index(const struct rsc *rsc)
+@@ -331,16 +330,15 @@ int amixer_mgr_destroy(struct amixer_mgr *amixer_mgr)
+
+ /* SUM resource management */
+
+-static int sum_master(struct rsc *rsc)
++static void sum_master(struct rsc *rsc)
+ {
+ rsc->conj = 0;
+- return rsc->idx = container_of(rsc, struct sum, rsc)->idx[0];
++ rsc->idx = container_of(rsc, struct sum, rsc)->idx[0];
+ }
+
+-static int sum_next_conj(struct rsc *rsc)
++static void sum_next_conj(struct rsc *rsc)
+ {
+ rsc->conj++;
+- return container_of(rsc, struct sum, rsc)->idx[rsc->conj];
+ }
+
+ static int sum_index(const struct rsc *rsc)
+diff --git a/sound/pci/ctxfi/ctatc.c b/sound/pci/ctxfi/ctatc.c
+index 78f35e88aed6b..fbdb8a3d5b8e5 100644
+--- a/sound/pci/ctxfi/ctatc.c
++++ b/sound/pci/ctxfi/ctatc.c
+@@ -36,6 +36,7 @@
+ | ((IEC958_AES3_CON_FS_48000) << 24))
+
+ static const struct snd_pci_quirk subsys_20k1_list[] = {
++ SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, 0x0021, "SB046x", CTSB046X),
+ SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, 0x0022, "SB055x", CTSB055X),
+ SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, 0x002f, "SB055x", CTSB055X),
+ SND_PCI_QUIRK(PCI_VENDOR_ID_CREATIVE, 0x0029, "SB073x", CTSB073X),
+@@ -64,6 +65,7 @@ static const struct snd_pci_quirk subsys_20k2_list[] = {
+
+ static const char *ct_subsys_name[NUM_CTCARDS] = {
+ /* 20k1 models */
++ [CTSB046X] = "SB046x",
+ [CTSB055X] = "SB055x",
+ [CTSB073X] = "SB073x",
+ [CTUAA] = "UAA",
+diff --git a/sound/pci/ctxfi/ctdaio.c b/sound/pci/ctxfi/ctdaio.c
+index f589da0453424..7fc720046ce29 100644
+--- a/sound/pci/ctxfi/ctdaio.c
++++ b/sound/pci/ctxfi/ctdaio.c
+@@ -51,12 +51,12 @@ static const struct daio_rsc_idx idx_20k2[NUM_DAIOTYP] = {
+ [SPDIFIO] = {.left = 0x05, .right = 0x85},
+ };
+
+-static int daio_master(struct rsc *rsc)
++static void daio_master(struct rsc *rsc)
+ {
+ /* Actually, this is not the resource index of DAIO.
+ * For DAO, it is the input mapper index. And, for DAI,
+ * it is the output time-slot index. */
+- return rsc->conj = rsc->idx;
++ rsc->conj = rsc->idx;
+ }
+
+ static int daio_index(const struct rsc *rsc)
+@@ -64,19 +64,19 @@ static int daio_index(const struct rsc *rsc)
+ return rsc->conj;
+ }
+
+-static int daio_out_next_conj(struct rsc *rsc)
++static void daio_out_next_conj(struct rsc *rsc)
+ {
+- return rsc->conj += 2;
++ rsc->conj += 2;
+ }
+
+-static int daio_in_next_conj_20k1(struct rsc *rsc)
++static void daio_in_next_conj_20k1(struct rsc *rsc)
+ {
+- return rsc->conj += 0x200;
++ rsc->conj += 0x200;
+ }
+
+-static int daio_in_next_conj_20k2(struct rsc *rsc)
++static void daio_in_next_conj_20k2(struct rsc *rsc)
+ {
+- return rsc->conj += 0x100;
++ rsc->conj += 0x100;
+ }
+
+ static const struct rsc_ops daio_out_rsc_ops = {
+diff --git a/sound/pci/ctxfi/cthardware.h b/sound/pci/ctxfi/cthardware.h
+index f406b626a28c4..2875cec83b8f2 100644
+--- a/sound/pci/ctxfi/cthardware.h
++++ b/sound/pci/ctxfi/cthardware.h
+@@ -26,8 +26,9 @@ enum CHIPTYP {
+
+ enum CTCARDS {
+ /* 20k1 models */
++ CTSB046X,
++ CT20K1_MODEL_FIRST = CTSB046X,
+ CTSB055X,
+- CT20K1_MODEL_FIRST = CTSB055X,
+ CTSB073X,
+ CTUAA,
+ CT20K1_UNKNOWN,
+diff --git a/sound/pci/ctxfi/ctresource.c b/sound/pci/ctxfi/ctresource.c
+index 81ad269345182..be1d3e61309ce 100644
+--- a/sound/pci/ctxfi/ctresource.c
++++ b/sound/pci/ctxfi/ctresource.c
+@@ -109,18 +109,17 @@ static int audio_ring_slot(const struct rsc *rsc)
+ return (rsc->conj << 4) + offset_in_audio_slot_block[rsc->type];
+ }
+
+-static int rsc_next_conj(struct rsc *rsc)
++static void rsc_next_conj(struct rsc *rsc)
+ {
+ unsigned int i;
+ for (i = 0; (i < 8) && (!(rsc->msr & (0x1 << i))); )
+ i++;
+ rsc->conj += (AUDIO_SLOT_BLOCK_NUM >> i);
+- return rsc->conj;
+ }
+
+-static int rsc_master(struct rsc *rsc)
++static void rsc_master(struct rsc *rsc)
+ {
+- return rsc->conj = rsc->idx;
++ rsc->conj = rsc->idx;
+ }
+
+ static const struct rsc_ops rsc_generic_ops = {
+diff --git a/sound/pci/ctxfi/ctresource.h b/sound/pci/ctxfi/ctresource.h
+index fdbfd808816d3..58553bda44f43 100644
+--- a/sound/pci/ctxfi/ctresource.h
++++ b/sound/pci/ctxfi/ctresource.h
+@@ -39,8 +39,8 @@ struct rsc {
+ };
+
+ struct rsc_ops {
+- int (*master)(struct rsc *rsc); /* Move to master resource */
+- int (*next_conj)(struct rsc *rsc); /* Move to next conjugate resource */
++ void (*master)(struct rsc *rsc); /* Move to master resource */
++ void (*next_conj)(struct rsc *rsc); /* Move to next conjugate resource */
+ int (*index)(const struct rsc *rsc); /* Return the index of resource */
+ /* Return the output slot number */
+ int (*output_slot)(const struct rsc *rsc);
+diff --git a/sound/pci/ctxfi/ctsrc.c b/sound/pci/ctxfi/ctsrc.c
+index bd4697b442334..4a94b4708a77e 100644
+--- a/sound/pci/ctxfi/ctsrc.c
++++ b/sound/pci/ctxfi/ctsrc.c
+@@ -590,16 +590,15 @@ int src_mgr_destroy(struct src_mgr *src_mgr)
+
+ /* SRCIMP resource manager operations */
+
+-static int srcimp_master(struct rsc *rsc)
++static void srcimp_master(struct rsc *rsc)
+ {
+ rsc->conj = 0;
+- return rsc->idx = container_of(rsc, struct srcimp, rsc)->idx[0];
++ rsc->idx = container_of(rsc, struct srcimp, rsc)->idx[0];
+ }
+
+-static int srcimp_next_conj(struct rsc *rsc)
++static void srcimp_next_conj(struct rsc *rsc)
+ {
+ rsc->conj++;
+- return container_of(rsc, struct srcimp, rsc)->idx[rsc->conj];
+ }
+
+ static int srcimp_index(const struct rsc *rsc)
+diff --git a/sound/pci/echoaudio/echoaudio.c b/sound/pci/echoaudio/echoaudio.c
+index 25b012ef5c3e6..c70c3ac4e99a5 100644
+--- a/sound/pci/echoaudio/echoaudio.c
++++ b/sound/pci/echoaudio/echoaudio.c
+@@ -1970,8 +1970,8 @@ static int snd_echo_create(struct snd_card *card,
+ }
+
+ /* constructor */
+-static int snd_echo_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_echo_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2139,6 +2139,11 @@ static int snd_echo_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_echo_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_echo_probe(pci, pci_id));
++}
+
+
+ #if defined(CONFIG_PM_SLEEP)
+diff --git a/sound/pci/emu10k1/emu10k1x.c b/sound/pci/emu10k1/emu10k1x.c
+index c49c44dc10820..89043392f3ec7 100644
+--- a/sound/pci/emu10k1/emu10k1x.c
++++ b/sound/pci/emu10k1/emu10k1x.c
+@@ -1491,8 +1491,8 @@ static int snd_emu10k1x_midi(struct emu10k1x *emu)
+ return 0;
+ }
+
+-static int snd_emu10k1x_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_emu10k1x_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1554,6 +1554,12 @@ static int snd_emu10k1x_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_emu10k1x_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_emu10k1x_probe(pci, pci_id));
++}
++
+ // PCI IDs
+ static const struct pci_device_id snd_emu10k1x_ids[] = {
+ { PCI_VDEVICE(CREATIVE, 0x0006), 0 }, /* Dell OEM version (EMU10K1) */
+diff --git a/sound/pci/ens1370.c b/sound/pci/ens1370.c
+index 2651f0c64c062..94efe347a97a9 100644
+--- a/sound/pci/ens1370.c
++++ b/sound/pci/ens1370.c
+@@ -2304,8 +2304,8 @@ static irqreturn_t snd_audiopci_interrupt(int irq, void *dev_id)
+ return IRQ_HANDLED;
+ }
+
+-static int snd_audiopci_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_audiopci_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2369,6 +2369,12 @@ static int snd_audiopci_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_audiopci_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_audiopci_probe(pci, pci_id));
++}
++
+ static struct pci_driver ens137x_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_audiopci_ids,
+diff --git a/sound/pci/es1938.c b/sound/pci/es1938.c
+index 00b976f42a3db..e34ec6f89e7e0 100644
+--- a/sound/pci/es1938.c
++++ b/sound/pci/es1938.c
+@@ -1716,8 +1716,8 @@ static int snd_es1938_mixer(struct es1938 *chip)
+ }
+
+
+-static int snd_es1938_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_es1938_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1796,6 +1796,12 @@ static int snd_es1938_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_es1938_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_es1938_probe(pci, pci_id));
++}
++
+ static struct pci_driver es1938_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_es1938_ids,
+diff --git a/sound/pci/es1968.c b/sound/pci/es1968.c
+index 6a8a02a9ecf41..4a7e20bb11bca 100644
+--- a/sound/pci/es1968.c
++++ b/sound/pci/es1968.c
+@@ -2741,8 +2741,8 @@ static int snd_es1968_create(struct snd_card *card,
+
+ /*
+ */
+-static int snd_es1968_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_es1968_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2848,6 +2848,12 @@ static int snd_es1968_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_es1968_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_es1968_probe(pci, pci_id));
++}
++
+ static struct pci_driver es1968_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_es1968_ids,
+diff --git a/sound/pci/fm801.c b/sound/pci/fm801.c
+index 9c22ff19e56d2..62b3cb126c6d0 100644
+--- a/sound/pci/fm801.c
++++ b/sound/pci/fm801.c
+@@ -1268,8 +1268,8 @@ static int snd_fm801_create(struct snd_card *card,
+ return 0;
+ }
+
+-static int snd_card_fm801_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_card_fm801_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1333,6 +1333,12 @@ static int snd_card_fm801_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_card_fm801_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_card_fm801_probe(pci, pci_id));
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static const unsigned char saved_regs[] = {
+ FM801_PCM_VOL, FM801_I2S_VOL, FM801_FM_VOL, FM801_REC_SRC,
+diff --git a/sound/pci/hda/hda_auto_parser.c b/sound/pci/hda/hda_auto_parser.c
+index 4a854475a0e60..500d0d474d27b 100644
+--- a/sound/pci/hda/hda_auto_parser.c
++++ b/sound/pci/hda/hda_auto_parser.c
+@@ -985,7 +985,7 @@ void snd_hda_pick_fixup(struct hda_codec *codec,
+ int id = HDA_FIXUP_ID_NOT_SET;
+ const char *name = NULL;
+ const char *type = NULL;
+- int vendor, device;
++ unsigned int vendor, device;
+
+ if (codec->fixup_id != HDA_FIXUP_ID_NOT_SET)
+ return;
+diff --git a/sound/pci/hda/hda_bind.c b/sound/pci/hda/hda_bind.c
+index 1c8bffc3eec6e..7153bd53e1893 100644
+--- a/sound/pci/hda/hda_bind.c
++++ b/sound/pci/hda/hda_bind.c
+@@ -156,6 +156,11 @@ static int hda_codec_driver_remove(struct device *dev)
+ return codec->bus->core.ext_ops->hdev_detach(&codec->core);
+ }
+
++ refcount_dec(&codec->pcm_ref);
++ snd_hda_codec_disconnect_pcms(codec);
++ wait_event(codec->remove_sleep, !refcount_read(&codec->pcm_ref));
++ snd_power_sync_ref(codec->bus->card);
++
+ if (codec->patch_ops.free)
+ codec->patch_ops.free(codec);
+ snd_hda_codec_cleanup_for_unbind(codec);
+diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c
+index 0c4a337c9fc0d..f552785d301e0 100644
+--- a/sound/pci/hda/hda_codec.c
++++ b/sound/pci/hda/hda_codec.c
+@@ -703,20 +703,10 @@ get_hda_cvt_setup(struct hda_codec *codec, hda_nid_t nid)
+ /*
+ * PCM device
+ */
+-static void release_pcm(struct kref *kref)
+-{
+- struct hda_pcm *pcm = container_of(kref, struct hda_pcm, kref);
+-
+- if (pcm->pcm)
+- snd_device_free(pcm->codec->card, pcm->pcm);
+- clear_bit(pcm->device, pcm->codec->bus->pcm_dev_bits);
+- kfree(pcm->name);
+- kfree(pcm);
+-}
+-
+ void snd_hda_codec_pcm_put(struct hda_pcm *pcm)
+ {
+- kref_put(&pcm->kref, release_pcm);
++ if (refcount_dec_and_test(&pcm->codec->pcm_ref))
++ wake_up(&pcm->codec->remove_sleep);
+ }
+ EXPORT_SYMBOL_GPL(snd_hda_codec_pcm_put);
+
+@@ -731,7 +721,6 @@ struct hda_pcm *snd_hda_codec_pcm_new(struct hda_codec *codec,
+ return NULL;
+
+ pcm->codec = codec;
+- kref_init(&pcm->kref);
+ va_start(args, fmt);
+ pcm->name = kvasprintf(GFP_KERNEL, fmt, args);
+ va_end(args);
+@@ -741,6 +730,7 @@ struct hda_pcm *snd_hda_codec_pcm_new(struct hda_codec *codec,
+ }
+
+ list_add_tail(&pcm->list, &codec->pcm_list_head);
++ refcount_inc(&codec->pcm_ref);
+ return pcm;
+ }
+ EXPORT_SYMBOL_GPL(snd_hda_codec_pcm_new);
+@@ -748,15 +738,31 @@ EXPORT_SYMBOL_GPL(snd_hda_codec_pcm_new);
+ /*
+ * codec destructor
+ */
++void snd_hda_codec_disconnect_pcms(struct hda_codec *codec)
++{
++ struct hda_pcm *pcm;
++
++ list_for_each_entry(pcm, &codec->pcm_list_head, list) {
++ if (pcm->disconnected)
++ continue;
++ if (pcm->pcm)
++ snd_device_disconnect(codec->card, pcm->pcm);
++ snd_hda_codec_pcm_put(pcm);
++ pcm->disconnected = 1;
++ }
++}
++
+ static void codec_release_pcms(struct hda_codec *codec)
+ {
+ struct hda_pcm *pcm, *n;
+
+ list_for_each_entry_safe(pcm, n, &codec->pcm_list_head, list) {
+- list_del_init(&pcm->list);
++ list_del(&pcm->list);
+ if (pcm->pcm)
+- snd_device_disconnect(codec->card, pcm->pcm);
+- snd_hda_codec_pcm_put(pcm);
++ snd_device_free(pcm->codec->card, pcm->pcm);
++ clear_bit(pcm->device, pcm->codec->bus->pcm_dev_bits);
++ kfree(pcm->name);
++ kfree(pcm);
+ }
+ }
+
+@@ -769,6 +775,7 @@ void snd_hda_codec_cleanup_for_unbind(struct hda_codec *codec)
+ codec->registered = 0;
+ }
+
++ snd_hda_codec_disconnect_pcms(codec);
+ cancel_delayed_work_sync(&codec->jackpoll_work);
+ if (!codec->in_freeing)
+ snd_hda_ctls_clear(codec);
+@@ -792,6 +799,7 @@ void snd_hda_codec_cleanup_for_unbind(struct hda_codec *codec)
+ remove_conn_list(codec);
+ snd_hdac_regmap_exit(&codec->core);
+ codec->configured = 0;
++ refcount_set(&codec->pcm_ref, 1); /* reset refcount */
+ }
+ EXPORT_SYMBOL_GPL(snd_hda_codec_cleanup_for_unbind);
+
+@@ -958,6 +966,8 @@ int snd_hda_codec_device_new(struct hda_bus *bus, struct snd_card *card,
+ snd_array_init(&codec->verbs, sizeof(struct hda_verb *), 8);
+ INIT_LIST_HEAD(&codec->conn_list);
+ INIT_LIST_HEAD(&codec->pcm_list_head);
++ refcount_set(&codec->pcm_ref, 1);
++ init_waitqueue_head(&codec->remove_sleep);
+
+ INIT_DELAYED_WORK(&codec->jackpoll_work, hda_jackpoll_work);
+ codec->depop_delay = -1;
+@@ -1727,8 +1737,11 @@ void snd_hda_ctls_clear(struct hda_codec *codec)
+ {
+ int i;
+ struct hda_nid_item *items = codec->mixers.list;
++
++ down_write(&codec->card->controls_rwsem);
+ for (i = 0; i < codec->mixers.used; i++)
+ snd_ctl_remove(codec->card, items[i].kctl);
++ up_write(&codec->card->controls_rwsem);
+ snd_array_free(&codec->mixers);
+ snd_array_free(&codec->nids);
+ }
+@@ -2987,6 +3000,10 @@ void snd_hda_codec_shutdown(struct hda_codec *codec)
+ {
+ struct hda_pcm *cpcm;
+
++ /* Skip the shutdown if codec is not registered */
++ if (!codec->registered)
++ return;
++
+ list_for_each_entry(cpcm, &codec->pcm_list_head, list)
+ snd_pcm_suspend_all(cpcm->pcm);
+
+diff --git a/sound/pci/hda/hda_controller.c b/sound/pci/hda/hda_controller.c
+index 930ae4002a818..75dcb14ff20ad 100644
+--- a/sound/pci/hda/hda_controller.c
++++ b/sound/pci/hda/hda_controller.c
+@@ -504,7 +504,6 @@ static int azx_get_time_info(struct snd_pcm_substream *substream,
+ snd_pcm_gettime(substream->runtime, system_ts);
+
+ nsec = timecounter_read(&azx_dev->core.tc);
+- nsec = div_u64(nsec, 3); /* can be optimized */
+ if (audio_tstamp_config->report_delay)
+ nsec = azx_adjust_codec_delay(substream, nsec);
+
+diff --git a/sound/pci/hda/hda_generic.c b/sound/pci/hda/hda_generic.c
+index 3bf5e34107038..fc114e5224806 100644
+--- a/sound/pci/hda/hda_generic.c
++++ b/sound/pci/hda/hda_generic.c
+@@ -91,6 +91,12 @@ static void snd_hda_gen_spec_free(struct hda_gen_spec *spec)
+ free_kctls(spec);
+ snd_array_free(&spec->paths);
+ snd_array_free(&spec->loopback_list);
++#ifdef CONFIG_SND_HDA_GENERIC_LEDS
++ if (spec->led_cdevs[LED_AUDIO_MUTE])
++ led_classdev_unregister(spec->led_cdevs[LED_AUDIO_MUTE]);
++ if (spec->led_cdevs[LED_AUDIO_MICMUTE])
++ led_classdev_unregister(spec->led_cdevs[LED_AUDIO_MICMUTE]);
++#endif
+ }
+
+ /*
+@@ -3922,7 +3928,10 @@ static int create_mute_led_cdev(struct hda_codec *codec,
+ enum led_brightness),
+ bool micmute)
+ {
++ struct hda_gen_spec *spec = codec->spec;
+ struct led_classdev *cdev;
++ int idx = micmute ? LED_AUDIO_MICMUTE : LED_AUDIO_MUTE;
++ int err;
+
+ cdev = devm_kzalloc(&codec->core.dev, sizeof(*cdev), GFP_KERNEL);
+ if (!cdev)
+@@ -3932,10 +3941,14 @@ static int create_mute_led_cdev(struct hda_codec *codec,
+ cdev->max_brightness = 1;
+ cdev->default_trigger = micmute ? "audio-micmute" : "audio-mute";
+ cdev->brightness_set_blocking = callback;
+- cdev->brightness = ledtrig_audio_get(micmute ? LED_AUDIO_MICMUTE : LED_AUDIO_MUTE);
++ cdev->brightness = ledtrig_audio_get(idx);
+ cdev->flags = LED_CORE_SUSPENDRESUME;
+
+- return devm_led_classdev_register(&codec->core.dev, cdev);
++ err = led_classdev_register(&codec->core.dev, cdev);
++ if (err < 0)
++ return err;
++ spec->led_cdevs[idx] = cdev;
++ return 0;
+ }
+
+ /**
+diff --git a/sound/pci/hda/hda_generic.h b/sound/pci/hda/hda_generic.h
+index c43bd0f0338ea..362ddcaea15b3 100644
+--- a/sound/pci/hda/hda_generic.h
++++ b/sound/pci/hda/hda_generic.h
+@@ -294,6 +294,9 @@ struct hda_gen_spec {
+ struct hda_jack_callback *cb);
+ void (*mic_autoswitch_hook)(struct hda_codec *codec,
+ struct hda_jack_callback *cb);
++
++ /* leds */
++ struct led_classdev *led_cdevs[NUM_AUDIO_LEDS];
+ };
+
+ /* values for add_stereo_mix_input flag */
+diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
+index 4d22e7adeee8e..9e36f992605ab 100644
+--- a/sound/pci/hda/hda_intel.c
++++ b/sound/pci/hda/hda_intel.c
+@@ -335,7 +335,10 @@ enum {
+ ((pci)->device == 0x0c0c) || \
+ ((pci)->device == 0x0d0c) || \
+ ((pci)->device == 0x160c) || \
+- ((pci)->device == 0x490d))
++ ((pci)->device == 0x490d) || \
++ ((pci)->device == 0x4f90) || \
++ ((pci)->device == 0x4f91) || \
++ ((pci)->device == 0x4f92))
+
+ #define IS_BXT(pci) ((pci)->vendor == 0x8086 && (pci)->device == 0x5a98)
+
+@@ -638,13 +641,17 @@ static int azx_position_check(struct azx *chip, struct azx_dev *azx_dev)
+ * the update-IRQ timing. The IRQ is issued before actually the
+ * data is processed. So, we need to process it afterwords in a
+ * workqueue.
++ *
++ * Returns 1 if OK to proceed, 0 for delay handling, -1 for skipping update
+ */
+ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev)
+ {
+ struct snd_pcm_substream *substream = azx_dev->core.substream;
++ struct snd_pcm_runtime *runtime = substream->runtime;
+ int stream = substream->stream;
+ u32 wallclk;
+ unsigned int pos;
++ snd_pcm_uframes_t hwptr, target;
+
+ wallclk = azx_readl(chip, WALLCLK) - azx_dev->core.start_wallclk;
+ if (wallclk < (azx_dev->core.period_wallclk * 2) / 3)
+@@ -681,6 +688,24 @@ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev)
+ /* NG - it's below the first next period boundary */
+ return chip->bdl_pos_adj ? 0 : -1;
+ azx_dev->core.start_wallclk += wallclk;
++
++ if (azx_dev->core.no_period_wakeup)
++ return 1; /* OK, no need to check period boundary */
++
++ if (runtime->hw_ptr_base != runtime->hw_ptr_interrupt)
++ return 1; /* OK, already in hwptr updating process */
++
++ /* check whether the period gets really elapsed */
++ pos = bytes_to_frames(runtime, pos);
++ hwptr = runtime->hw_ptr_base + pos;
++ if (hwptr < runtime->status->hw_ptr)
++ hwptr += runtime->buffer_size;
++ target = runtime->hw_ptr_interrupt + runtime->period_size;
++ if (hwptr < target) {
++ /* too early wakeup, process it later */
++ return chip->bdl_pos_adj ? 0 : -1;
++ }
++
+ return 1; /* OK, it's fine */
+ }
+
+@@ -859,31 +884,6 @@ static int azx_get_delay_from_fifo(struct azx *chip, struct azx_dev *azx_dev,
+ return substream->runtime->delay;
+ }
+
+-static unsigned int azx_skl_get_dpib_pos(struct azx *chip,
+- struct azx_dev *azx_dev)
+-{
+- return _snd_hdac_chip_readl(azx_bus(chip),
+- AZX_REG_VS_SDXDPIB_XBASE +
+- (AZX_REG_VS_SDXDPIB_XINTERVAL *
+- azx_dev->core.index));
+-}
+-
+-/* get the current DMA position with correction on SKL+ chips */
+-static unsigned int azx_get_pos_skl(struct azx *chip, struct azx_dev *azx_dev)
+-{
+- /* DPIB register gives a more accurate position for playback */
+- if (azx_dev->core.substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+- return azx_skl_get_dpib_pos(chip, azx_dev);
+-
+- /* For capture, we need to read posbuf, but it requires a delay
+- * for the possible boundary overlap; the read of DPIB fetches the
+- * actual posbuf
+- */
+- udelay(20);
+- azx_skl_get_dpib_pos(chip, azx_dev);
+- return azx_get_pos_posbuf(chip, azx_dev);
+-}
+-
+ static void __azx_shutdown_chip(struct azx *chip, bool skip_link_reset)
+ {
+ azx_stop_chip(chip);
+@@ -1573,7 +1573,7 @@ static void assign_position_fix(struct azx *chip, int fix)
+ [POS_FIX_POSBUF] = azx_get_pos_posbuf,
+ [POS_FIX_VIACOMBO] = azx_via_get_position,
+ [POS_FIX_COMBO] = azx_get_pos_lpib,
+- [POS_FIX_SKL] = azx_get_pos_skl,
++ [POS_FIX_SKL] = azx_get_pos_posbuf,
+ [POS_FIX_FIFO] = azx_get_pos_fifo,
+ };
+
+@@ -1611,6 +1611,7 @@ static const struct snd_pci_quirk probe_mask_list[] = {
+ /* forced codec slots */
+ SND_PCI_QUIRK(0x1043, 0x1262, "ASUS W5Fm", 0x103),
+ SND_PCI_QUIRK(0x1046, 0x1262, "ASUS W5F", 0x103),
++ SND_PCI_QUIRK(0x1558, 0x0351, "Schenker Dock 15", 0x105),
+ /* WinFast VP200 H (Teradici) user reported broken communication */
+ SND_PCI_QUIRK(0x3a21, 0x040d, "WinFast VP200 H", 0x101),
+ {}
+@@ -1794,8 +1795,6 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
+
+ assign_position_fix(chip, check_position_fix(chip, position_fix[dev]));
+
+- check_probe_mask(chip, dev);
+-
+ if (single_cmd < 0) /* allow fallback to single_cmd at errors */
+ chip->fallback_to_single_cmd = 1;
+ else /* explicitly set to single_cmd or not */
+@@ -1821,6 +1820,8 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
+ chip->bus.core.needs_damn_long_delay = 1;
+ }
+
++ check_probe_mask(chip, dev);
++
+ err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, chip, &ops);
+ if (err < 0) {
+ dev_err(card->dev, "Error creating device [card]!\n");
+@@ -2330,7 +2331,8 @@ static int azx_probe_continue(struct azx *chip)
+
+ out_free:
+ if (err < 0) {
+- azx_free(chip);
++ pci_set_drvdata(pci, NULL);
++ snd_card_free(chip->card);
+ return err;
+ }
+
+@@ -2474,6 +2476,13 @@ static const struct pci_device_id azx_ids[] = {
+ /* DG1 */
+ { PCI_DEVICE(0x8086, 0x490d),
+ .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
++ /* DG2 */
++ { PCI_DEVICE(0x8086, 0x4f90),
++ .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
++ { PCI_DEVICE(0x8086, 0x4f91),
++ .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
++ { PCI_DEVICE(0x8086, 0x4f92),
++ .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
+ /* Alderlake-S */
+ { PCI_DEVICE(0x8086, 0x7ad0),
+ .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
+diff --git a/sound/pci/hda/hda_local.h b/sound/pci/hda/hda_local.h
+index ea8ab8b433378..8621f576446b8 100644
+--- a/sound/pci/hda/hda_local.h
++++ b/sound/pci/hda/hda_local.h
+@@ -137,6 +137,7 @@ int __snd_hda_add_vmaster(struct hda_codec *codec, char *name,
+ int snd_hda_codec_reset(struct hda_codec *codec);
+ void snd_hda_codec_register(struct hda_codec *codec);
+ void snd_hda_codec_cleanup_for_unbind(struct hda_codec *codec);
++void snd_hda_codec_disconnect_pcms(struct hda_codec *codec);
+
+ #define snd_hda_regmap_sync(codec) snd_hdac_regmap_sync(&(codec)->core)
+
+@@ -438,6 +439,15 @@ int snd_hda_codec_set_pin_target(struct hda_codec *codec, hda_nid_t nid,
+ #define for_each_hda_codec_node(nid, codec) \
+ for ((nid) = (codec)->core.start_nid; (nid) < (codec)->core.end_nid; (nid)++)
+
++/* Set the codec power_state flag to indicate to allow unsol event handling;
++ * see hda_codec_unsol_event() in hda_bind.c. Calling this might confuse the
++ * state tracking, so use with care.
++ */
++static inline void snd_hda_codec_allow_unsol_events(struct hda_codec *codec)
++{
++ codec->core.dev.power.power_state = PMSG_ON;
++}
++
+ /*
+ * get widget capabilities
+ */
+diff --git a/sound/pci/hda/hda_tegra.c b/sound/pci/hda/hda_tegra.c
+index ea700395bef40..773f4903550a0 100644
+--- a/sound/pci/hda/hda_tegra.c
++++ b/sound/pci/hda/hda_tegra.c
+@@ -68,14 +68,20 @@
+ */
+ #define TEGRA194_NUM_SDO_LINES 4
+
++struct hda_tegra_soc {
++ bool has_hda2codec_2x_reset;
++};
++
+ struct hda_tegra {
+ struct azx chip;
+ struct device *dev;
+- struct reset_control *reset;
++ struct reset_control_bulk_data resets[3];
+ struct clk_bulk_data clocks[3];
++ unsigned int nresets;
+ unsigned int nclocks;
+ void __iomem *regs;
+ struct work_struct probe_work;
++ const struct hda_tegra_soc *soc;
+ };
+
+ #ifdef CONFIG_PM
+@@ -170,7 +176,7 @@ static int __maybe_unused hda_tegra_runtime_resume(struct device *dev)
+ int rc;
+
+ if (!chip->running) {
+- rc = reset_control_assert(hda->reset);
++ rc = reset_control_bulk_assert(hda->nresets, hda->resets);
+ if (rc)
+ return rc;
+ }
+@@ -187,7 +193,7 @@ static int __maybe_unused hda_tegra_runtime_resume(struct device *dev)
+ } else {
+ usleep_range(10, 100);
+
+- rc = reset_control_deassert(hda->reset);
++ rc = reset_control_bulk_deassert(hda->nresets, hda->resets);
+ if (rc)
+ return rc;
+ }
+@@ -427,9 +433,17 @@ static int hda_tegra_create(struct snd_card *card,
+ return 0;
+ }
+
++static const struct hda_tegra_soc tegra30_data = {
++ .has_hda2codec_2x_reset = true,
++};
++
++static const struct hda_tegra_soc tegra194_data = {
++ .has_hda2codec_2x_reset = false,
++};
++
+ static const struct of_device_id hda_tegra_match[] = {
+- { .compatible = "nvidia,tegra30-hda" },
+- { .compatible = "nvidia,tegra194-hda" },
++ { .compatible = "nvidia,tegra30-hda", .data = &tegra30_data },
++ { .compatible = "nvidia,tegra194-hda", .data = &tegra194_data },
+ {},
+ };
+ MODULE_DEVICE_TABLE(of, hda_tegra_match);
+@@ -449,6 +463,8 @@ static int hda_tegra_probe(struct platform_device *pdev)
+ hda->dev = &pdev->dev;
+ chip = &hda->chip;
+
++ hda->soc = of_device_get_match_data(&pdev->dev);
++
+ err = snd_card_new(&pdev->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1,
+ THIS_MODULE, 0, &card);
+ if (err < 0) {
+@@ -456,11 +472,20 @@ static int hda_tegra_probe(struct platform_device *pdev)
+ return err;
+ }
+
+- hda->reset = devm_reset_control_array_get_exclusive(&pdev->dev);
+- if (IS_ERR(hda->reset)) {
+- err = PTR_ERR(hda->reset);
++ hda->resets[hda->nresets++].id = "hda";
++ hda->resets[hda->nresets++].id = "hda2hdmi";
++ /*
++ * "hda2codec_2x" reset is not present on Tegra194. Though DT would
++ * be updated to reflect this, but to have backward compatibility
++ * below is necessary.
++ */
++ if (hda->soc->has_hda2codec_2x_reset)
++ hda->resets[hda->nresets++].id = "hda2codec_2x";
++
++ err = devm_reset_control_bulk_get_exclusive(&pdev->dev, hda->nresets,
++ hda->resets);
++ if (err)
+ goto out_free;
+- }
+
+ hda->clocks[hda->nclocks++].id = "hda";
+ hda->clocks[hda->nclocks++].id = "hda2hdmi";
+diff --git a/sound/pci/hda/patch_cs8409-tables.c b/sound/pci/hda/patch_cs8409-tables.c
+index 0fb0a428428b4..df0b4522babf7 100644
+--- a/sound/pci/hda/patch_cs8409-tables.c
++++ b/sound/pci/hda/patch_cs8409-tables.c
+@@ -252,6 +252,7 @@ struct sub_codec cs8409_cs42l42_codec = {
+ .init_seq_num = ARRAY_SIZE(cs42l42_init_reg_seq),
+ .hp_jack_in = 0,
+ .mic_jack_in = 0,
++ .force_status_change = 1,
+ .paged = 1,
+ .suspended = 1,
+ .no_type_dect = 0,
+@@ -443,6 +444,7 @@ struct sub_codec dolphin_cs42l42_0 = {
+ .init_seq_num = ARRAY_SIZE(dolphin_c0_init_reg_seq),
+ .hp_jack_in = 0,
+ .mic_jack_in = 0,
++ .force_status_change = 1,
+ .paged = 1,
+ .suspended = 1,
+ .no_type_dect = 0,
+@@ -456,6 +458,7 @@ struct sub_codec dolphin_cs42l42_1 = {
+ .init_seq_num = ARRAY_SIZE(dolphin_c1_init_reg_seq),
+ .hp_jack_in = 0,
+ .mic_jack_in = 0,
++ .force_status_change = 1,
+ .paged = 1,
+ .suspended = 1,
+ .no_type_dect = 1,
+diff --git a/sound/pci/hda/patch_cs8409.c b/sound/pci/hda/patch_cs8409.c
+index 31ff11ab868e1..aff2b5abb81ea 100644
+--- a/sound/pci/hda/patch_cs8409.c
++++ b/sound/pci/hda/patch_cs8409.c
+@@ -628,15 +628,17 @@ static void cs42l42_run_jack_detect(struct sub_codec *cs42l42)
+ cs8409_i2c_write(cs42l42, 0x1b74, 0x07);
+ cs8409_i2c_write(cs42l42, 0x131b, 0xFD);
+ cs8409_i2c_write(cs42l42, 0x1120, 0x80);
+- /* Wait ~100us*/
+- usleep_range(100, 200);
++ /* Wait ~20ms*/
++ usleep_range(20000, 25000);
+ cs8409_i2c_write(cs42l42, 0x111f, 0x77);
+ cs8409_i2c_write(cs42l42, 0x1120, 0xc0);
+ }
+
+ static int cs42l42_handle_tip_sense(struct sub_codec *cs42l42, unsigned int reg_ts_status)
+ {
+- int status_changed = 0;
++ int status_changed = cs42l42->force_status_change;
++
++ cs42l42->force_status_change = 0;
+
+ /* TIP_SENSE INSERT/REMOVE */
+ switch (reg_ts_status) {
+@@ -750,6 +752,11 @@ static void cs42l42_resume(struct sub_codec *cs42l42)
+ if (cs42l42->full_scale_vol)
+ cs8409_i2c_write(cs42l42, 0x2001, 0x01);
+
++ /* we have to explicitly allow unsol event handling even during the
++ * resume phase so that the jack event is processed properly
++ */
++ snd_hda_codec_allow_unsol_events(cs42l42->codec);
++
+ cs42l42_enable_jack_detect(cs42l42);
+ }
+
+@@ -786,6 +793,7 @@ static void cs42l42_suspend(struct sub_codec *cs42l42)
+ cs42l42->last_page = 0;
+ cs42l42->hp_jack_in = 0;
+ cs42l42->mic_jack_in = 0;
++ cs42l42->force_status_change = 1;
+
+ /* Put CS42L42 into Reset */
+ gpio_data = snd_hda_codec_read(codec, CS8409_PIN_AFG, 0, AC_VERB_GET_GPIO_DATA, 0);
+diff --git a/sound/pci/hda/patch_cs8409.h b/sound/pci/hda/patch_cs8409.h
+index ade2b838590cf..d0b725c7285b6 100644
+--- a/sound/pci/hda/patch_cs8409.h
++++ b/sound/pci/hda/patch_cs8409.h
+@@ -305,6 +305,7 @@ struct sub_codec {
+
+ unsigned int hp_jack_in:1;
+ unsigned int mic_jack_in:1;
++ unsigned int force_status_change:1;
+ unsigned int suspended:1;
+ unsigned int paged:1;
+ unsigned int last_page;
+diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c
+index 65d2c55399195..24da843f39a11 100644
+--- a/sound/pci/hda/patch_hdmi.c
++++ b/sound/pci/hda/patch_hdmi.c
+@@ -1387,7 +1387,7 @@ static int hdmi_find_pcm_slot(struct hdmi_spec *spec,
+
+ last_try:
+ /* the last try; check the empty slots in pins */
+- for (i = 0; i < spec->num_nids; i++) {
++ for (i = 0; i < spec->pcm_used; i++) {
+ if (!test_bit(i, &spec->pcm_bitmap))
+ return i;
+ }
+@@ -1617,6 +1617,7 @@ static void hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin,
+ struct hda_codec *codec = per_pin->codec;
+ struct hdmi_spec *spec = codec->spec;
+ struct hdmi_eld *eld = &spec->temp_eld;
++ struct device *dev = hda_codec_dev(codec);
+ hda_nid_t pin_nid = per_pin->pin_nid;
+ int dev_id = per_pin->dev_id;
+ /*
+@@ -1630,8 +1631,13 @@ static void hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin,
+ int present;
+ int ret;
+
++#ifdef CONFIG_PM
++ if (dev->power.runtime_status == RPM_SUSPENDING)
++ return;
++#endif
++
+ ret = snd_hda_power_up_pm(codec);
+- if (ret < 0 && pm_runtime_suspended(hda_codec_dev(codec)))
++ if (ret < 0 && pm_runtime_suspended(dev))
+ goto out;
+
+ present = snd_hda_jack_pin_sense(codec, pin_nid, dev_id);
+@@ -2257,7 +2263,9 @@ static int generic_hdmi_build_pcms(struct hda_codec *codec)
+ * dev_num is the device entry number in a pin
+ */
+
+- if (codec->mst_no_extra_pcms)
++ if (spec->dyn_pcm_no_legacy && codec->mst_no_extra_pcms)
++ pcm_num = spec->num_cvts;
++ else if (codec->mst_no_extra_pcms)
+ pcm_num = spec->num_nids;
+ else
+ pcm_num = spec->num_nids + spec->dev_num - 1;
+@@ -2947,7 +2955,8 @@ static int parse_intel_hdmi(struct hda_codec *codec)
+
+ /* Intel Haswell and onwards; audio component with eld notifier */
+ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid,
+- const int *port_map, int port_num, int dev_num)
++ const int *port_map, int port_num, int dev_num,
++ bool send_silent_stream)
+ {
+ struct hdmi_spec *spec;
+ int err;
+@@ -2980,7 +2989,7 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid,
+ * Enable silent stream feature, if it is enabled via
+ * module param or Kconfig option
+ */
+- if (enable_silent_stream)
++ if (send_silent_stream)
+ spec->send_silent_stream = true;
+
+ return parse_intel_hdmi(codec);
+@@ -2988,12 +2997,18 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid,
+
+ static int patch_i915_hsw_hdmi(struct hda_codec *codec)
+ {
+- return intel_hsw_common_init(codec, 0x08, NULL, 0, 3);
++ return intel_hsw_common_init(codec, 0x08, NULL, 0, 3,
++ enable_silent_stream);
+ }
+
+ static int patch_i915_glk_hdmi(struct hda_codec *codec)
+ {
+- return intel_hsw_common_init(codec, 0x0b, NULL, 0, 3);
++ /*
++ * Silent stream calls audio component .get_power() from
++ * .pin_eld_notify(). On GLK this will deadlock in i915 due
++ * to the audio vs. CDCLK workaround.
++ */
++ return intel_hsw_common_init(codec, 0x0b, NULL, 0, 3, false);
+ }
+
+ static int patch_i915_icl_hdmi(struct hda_codec *codec)
+@@ -3004,7 +3019,8 @@ static int patch_i915_icl_hdmi(struct hda_codec *codec)
+ */
+ static const int map[] = {0x0, 0x4, 0x6, 0x8, 0xa, 0xb};
+
+- return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 3);
++ return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 3,
++ enable_silent_stream);
+ }
+
+ static int patch_i915_tgl_hdmi(struct hda_codec *codec)
+@@ -3016,7 +3032,8 @@ static int patch_i915_tgl_hdmi(struct hda_codec *codec)
+ static const int map[] = {0x4, 0x6, 0x8, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf};
+ int ret;
+
+- ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4);
++ ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4,
++ enable_silent_stream);
+ if (!ret) {
+ struct hdmi_spec *spec = codec->spec;
+
+@@ -4380,10 +4397,11 @@ HDA_CODEC_ENTRY(0x8086280f, "Icelake HDMI", patch_i915_icl_hdmi),
+ HDA_CODEC_ENTRY(0x80862812, "Tigerlake HDMI", patch_i915_tgl_hdmi),
+ HDA_CODEC_ENTRY(0x80862814, "DG1 HDMI", patch_i915_tgl_hdmi),
+ HDA_CODEC_ENTRY(0x80862815, "Alderlake HDMI", patch_i915_tgl_hdmi),
+-HDA_CODEC_ENTRY(0x8086281c, "Alderlake-P HDMI", patch_i915_tgl_hdmi),
+ HDA_CODEC_ENTRY(0x80862816, "Rocketlake HDMI", patch_i915_tgl_hdmi),
++HDA_CODEC_ENTRY(0x80862819, "DG2 HDMI", patch_i915_tgl_hdmi),
+ HDA_CODEC_ENTRY(0x8086281a, "Jasperlake HDMI", patch_i915_icl_hdmi),
+ HDA_CODEC_ENTRY(0x8086281b, "Elkhartlake HDMI", patch_i915_icl_hdmi),
++HDA_CODEC_ENTRY(0x8086281c, "Alderlake-P HDMI", patch_i915_tgl_hdmi),
+ HDA_CODEC_ENTRY(0x80862880, "CedarTrail HDMI", patch_generic_hdmi),
+ HDA_CODEC_ENTRY(0x80862882, "Valleyview2 HDMI", patch_i915_byt_hdmi),
+ HDA_CODEC_ENTRY(0x80862883, "Braswell HDMI", patch_i915_byt_hdmi),
+diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
+index 965b096f416f6..040825ea9a089 100644
+--- a/sound/pci/hda/patch_realtek.c
++++ b/sound/pci/hda/patch_realtek.c
+@@ -97,6 +97,7 @@ struct alc_spec {
+ unsigned int gpio_mic_led_mask;
+ struct alc_coef_led mute_led_coef;
+ struct alc_coef_led mic_led_coef;
++ struct mutex coef_mutex;
+
+ hda_nid_t headset_mic_pin;
+ hda_nid_t headphone_mic_pin;
+@@ -132,8 +133,24 @@ struct alc_spec {
+ * COEF access helper functions
+ */
+
+-static int alc_read_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
+- unsigned int coef_idx)
++static void coef_mutex_lock(struct hda_codec *codec)
++{
++ struct alc_spec *spec = codec->spec;
++
++ snd_hda_power_up_pm(codec);
++ mutex_lock(&spec->coef_mutex);
++}
++
++static void coef_mutex_unlock(struct hda_codec *codec)
++{
++ struct alc_spec *spec = codec->spec;
++
++ mutex_unlock(&spec->coef_mutex);
++ snd_hda_power_down_pm(codec);
++}
++
++static int __alc_read_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
++ unsigned int coef_idx)
+ {
+ unsigned int val;
+
+@@ -142,28 +159,56 @@ static int alc_read_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
+ return val;
+ }
+
++static int alc_read_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
++ unsigned int coef_idx)
++{
++ unsigned int val;
++
++ coef_mutex_lock(codec);
++ val = __alc_read_coefex_idx(codec, nid, coef_idx);
++ coef_mutex_unlock(codec);
++ return val;
++}
++
+ #define alc_read_coef_idx(codec, coef_idx) \
+ alc_read_coefex_idx(codec, 0x20, coef_idx)
+
+-static void alc_write_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
+- unsigned int coef_idx, unsigned int coef_val)
++static void __alc_write_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
++ unsigned int coef_idx, unsigned int coef_val)
+ {
+ snd_hda_codec_write(codec, nid, 0, AC_VERB_SET_COEF_INDEX, coef_idx);
+ snd_hda_codec_write(codec, nid, 0, AC_VERB_SET_PROC_COEF, coef_val);
+ }
+
++static void alc_write_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
++ unsigned int coef_idx, unsigned int coef_val)
++{
++ coef_mutex_lock(codec);
++ __alc_write_coefex_idx(codec, nid, coef_idx, coef_val);
++ coef_mutex_unlock(codec);
++}
++
+ #define alc_write_coef_idx(codec, coef_idx, coef_val) \
+ alc_write_coefex_idx(codec, 0x20, coef_idx, coef_val)
+
++static void __alc_update_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
++ unsigned int coef_idx, unsigned int mask,
++ unsigned int bits_set)
++{
++ unsigned int val = __alc_read_coefex_idx(codec, nid, coef_idx);
++
++ if (val != -1)
++ __alc_write_coefex_idx(codec, nid, coef_idx,
++ (val & ~mask) | bits_set);
++}
++
+ static void alc_update_coefex_idx(struct hda_codec *codec, hda_nid_t nid,
+ unsigned int coef_idx, unsigned int mask,
+ unsigned int bits_set)
+ {
+- unsigned int val = alc_read_coefex_idx(codec, nid, coef_idx);
+-
+- if (val != -1)
+- alc_write_coefex_idx(codec, nid, coef_idx,
+- (val & ~mask) | bits_set);
++ coef_mutex_lock(codec);
++ __alc_update_coefex_idx(codec, nid, coef_idx, mask, bits_set);
++ coef_mutex_unlock(codec);
+ }
+
+ #define alc_update_coef_idx(codec, coef_idx, mask, bits_set) \
+@@ -196,13 +241,15 @@ struct coef_fw {
+ static void alc_process_coef_fw(struct hda_codec *codec,
+ const struct coef_fw *fw)
+ {
++ coef_mutex_lock(codec);
+ for (; fw->nid; fw++) {
+ if (fw->mask == (unsigned short)-1)
+- alc_write_coefex_idx(codec, fw->nid, fw->idx, fw->val);
++ __alc_write_coefex_idx(codec, fw->nid, fw->idx, fw->val);
+ else
+- alc_update_coefex_idx(codec, fw->nid, fw->idx,
+- fw->mask, fw->val);
++ __alc_update_coefex_idx(codec, fw->nid, fw->idx,
++ fw->mask, fw->val);
+ }
++ coef_mutex_unlock(codec);
+ }
+
+ /*
+@@ -885,6 +932,9 @@ static int alc_init(struct hda_codec *codec)
+ return 0;
+ }
+
++#define alc_free snd_hda_gen_free
++
++#ifdef CONFIG_PM
+ static inline void alc_shutup(struct hda_codec *codec)
+ {
+ struct alc_spec *spec = codec->spec;
+@@ -898,9 +948,6 @@ static inline void alc_shutup(struct hda_codec *codec)
+ alc_shutup_pins(codec);
+ }
+
+-#define alc_free snd_hda_gen_free
+-
+-#ifdef CONFIG_PM
+ static void alc_power_eapd(struct hda_codec *codec)
+ {
+ alc_auto_setup_eapd(codec, false);
+@@ -914,9 +961,7 @@ static int alc_suspend(struct hda_codec *codec)
+ spec->power_hook(codec);
+ return 0;
+ }
+-#endif
+
+-#ifdef CONFIG_PM
+ static int alc_resume(struct hda_codec *codec)
+ {
+ struct alc_spec *spec = codec->spec;
+@@ -1148,6 +1193,7 @@ static int alc_alloc_spec(struct hda_codec *codec, hda_nid_t mixer_nid)
+ codec->spdif_status_reset = 1;
+ codec->forced_resume = 1;
+ codec->patch_ops = alc_patch_ops;
++ mutex_init(&spec->coef_mutex);
+
+ err = alc_codec_rename_from_preset(codec);
+ if (err < 0) {
+@@ -1924,6 +1970,7 @@ enum {
+ ALC887_FIXUP_ASUS_BASS,
+ ALC887_FIXUP_BASS_CHMAP,
+ ALC1220_FIXUP_GB_DUAL_CODECS,
++ ALC1220_FIXUP_GB_X570,
+ ALC1220_FIXUP_CLEVO_P950,
+ ALC1220_FIXUP_CLEVO_PB51ED,
+ ALC1220_FIXUP_CLEVO_PB51ED_PINS,
+@@ -2113,6 +2160,30 @@ static void alc1220_fixup_gb_dual_codecs(struct hda_codec *codec,
+ }
+ }
+
++static void alc1220_fixup_gb_x570(struct hda_codec *codec,
++ const struct hda_fixup *fix,
++ int action)
++{
++ static const hda_nid_t conn1[] = { 0x0c };
++ static const struct coef_fw gb_x570_coefs[] = {
++ WRITE_COEF(0x07, 0x03c0),
++ WRITE_COEF(0x1a, 0x01c1),
++ WRITE_COEF(0x1b, 0x0202),
++ WRITE_COEF(0x43, 0x3005),
++ {}
++ };
++
++ switch (action) {
++ case HDA_FIXUP_ACT_PRE_PROBE:
++ snd_hda_override_conn_list(codec, 0x14, ARRAY_SIZE(conn1), conn1);
++ snd_hda_override_conn_list(codec, 0x1b, ARRAY_SIZE(conn1), conn1);
++ break;
++ case HDA_FIXUP_ACT_INIT:
++ alc_process_coef_fw(codec, gb_x570_coefs);
++ break;
++ }
++}
++
+ static void alc1220_fixup_clevo_p950(struct hda_codec *codec,
+ const struct hda_fixup *fix,
+ int action)
+@@ -2415,6 +2486,10 @@ static const struct hda_fixup alc882_fixups[] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc1220_fixup_gb_dual_codecs,
+ },
++ [ALC1220_FIXUP_GB_X570] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc1220_fixup_gb_x570,
++ },
+ [ALC1220_FIXUP_CLEVO_P950] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc1220_fixup_clevo_p950,
+@@ -2517,8 +2592,9 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x13fe, 0x1009, "Advantech MIT-W101", ALC886_FIXUP_EAPD),
+ SND_PCI_QUIRK(0x1458, 0xa002, "Gigabyte EP45-DS3/Z87X-UD3H", ALC889_FIXUP_FRONT_HP_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1458, 0xa0b8, "Gigabyte AZ370-Gaming", ALC1220_FIXUP_GB_DUAL_CODECS),
+- SND_PCI_QUIRK(0x1458, 0xa0cd, "Gigabyte X570 Aorus Master", ALC1220_FIXUP_CLEVO_P950),
+- SND_PCI_QUIRK(0x1458, 0xa0ce, "Gigabyte X570 Aorus Xtreme", ALC1220_FIXUP_CLEVO_P950),
++ SND_PCI_QUIRK(0x1458, 0xa0cd, "Gigabyte X570 Aorus Master", ALC1220_FIXUP_GB_X570),
++ SND_PCI_QUIRK(0x1458, 0xa0ce, "Gigabyte X570 Aorus Xtreme", ALC1220_FIXUP_GB_X570),
++ SND_PCI_QUIRK(0x1458, 0xa0d5, "Gigabyte X570S Aorus Master", ALC1220_FIXUP_GB_X570),
+ SND_PCI_QUIRK(0x1462, 0x11f7, "MSI-GE63", ALC1220_FIXUP_CLEVO_P950),
+ SND_PCI_QUIRK(0x1462, 0x1228, "MSI-GP63", ALC1220_FIXUP_CLEVO_P950),
+ SND_PCI_QUIRK(0x1462, 0x1229, "MSI-GP73", ALC1220_FIXUP_CLEVO_P950),
+@@ -2536,9 +2612,11 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1558, 0x65e1, "Clevo PB51[ED][DF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x65e5, "Clevo PC50D[PRS](?:-D|-G)?", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x65f1, "Clevo PC50HS", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
++ SND_PCI_QUIRK(0x1558, 0x65f5, "Clevo PD50PN[NRT]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x67d1, "Clevo PB71[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x67e1, "Clevo PB71[DE][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x67e5, "Clevo PC70D[PRS](?:-D|-G)?", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
++ SND_PCI_QUIRK(0x1558, 0x67f1, "Clevo PC70H[PRS]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x70d1, "Clevo PC70[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x7714, "Clevo X170SM", ALC1220_FIXUP_CLEVO_PB51ED_PINS),
+ SND_PCI_QUIRK(0x1558, 0x7715, "Clevo X170KM-G", ALC1220_FIXUP_CLEVO_PB51ED),
+@@ -2592,6 +2670,7 @@ static const struct hda_model_fixup alc882_fixup_models[] = {
+ {.id = ALC882_FIXUP_NO_PRIMARY_HP, .name = "no-primary-hp"},
+ {.id = ALC887_FIXUP_ASUS_BASS, .name = "asus-bass"},
+ {.id = ALC1220_FIXUP_GB_DUAL_CODECS, .name = "dual-codecs"},
++ {.id = ALC1220_FIXUP_GB_X570, .name = "gb-x570"},
+ {.id = ALC1220_FIXUP_CLEVO_P950, .name = "clevo-p950"},
+ {}
+ };
+@@ -3532,8 +3611,8 @@ static void alc256_shutup(struct hda_codec *codec)
+ /* If disable 3k pulldown control for alc257, the Mic detection will not work correctly
+ * when booting with headset plugged. So skip setting it for the codec alc257
+ */
+- if (spec->codec_variant != ALC269_TYPE_ALC257 &&
+- spec->codec_variant != ALC269_TYPE_ALC256)
++ if (codec->core.vendor_id != 0x10ec0236 &&
++ codec->core.vendor_id != 0x10ec0257)
+ alc_update_coef_idx(codec, 0x46, 0, 3 << 12);
+
+ if (!spec->no_shutup_pins)
+@@ -4355,6 +4434,16 @@ static void alc287_fixup_hp_gpio_led(struct hda_codec *codec,
+ alc_fixup_hp_gpio_led(codec, action, 0x10, 0);
+ }
+
++static void alc245_fixup_hp_gpio_led(struct hda_codec *codec,
++ const struct hda_fixup *fix, int action)
++{
++ struct alc_spec *spec = codec->spec;
++
++ if (action == HDA_FIXUP_ACT_PRE_PROBE)
++ spec->micmute_led_polarity = 1;
++ alc_fixup_hp_gpio_led(codec, action, 0, 0x04);
++}
++
+ /* turn on/off mic-mute LED per capture hook via VREF change */
+ static int vref_micmute_led_set(struct led_classdev *led_cdev,
+ enum led_brightness brightness)
+@@ -6492,22 +6581,64 @@ static void alc287_fixup_legion_15imhg05_speakers(struct hda_codec *codec,
+ /* for alc285_fixup_ideapad_s740_coef() */
+ #include "ideapad_s740_helper.c"
+
+-static void alc256_fixup_tongfang_reset_persistent_settings(struct hda_codec *codec,
+- const struct hda_fixup *fix,
+- int action)
++static const struct coef_fw alc256_fixup_set_coef_defaults_coefs[] = {
++ WRITE_COEF(0x10, 0x0020), WRITE_COEF(0x24, 0x0000),
++ WRITE_COEF(0x26, 0x0000), WRITE_COEF(0x29, 0x3000),
++ WRITE_COEF(0x37, 0xfe05), WRITE_COEF(0x45, 0x5089),
++ {}
++};
++
++static void alc256_fixup_set_coef_defaults(struct hda_codec *codec,
++ const struct hda_fixup *fix,
++ int action)
+ {
+ /*
+- * A certain other OS sets these coeffs to different values. On at least one TongFang
+- * barebone these settings might survive even a cold reboot. So to restore a clean slate the
+- * values are explicitly reset to default here. Without this, the external microphone is
+- * always in a plugged-in state, while the internal microphone is always in an unplugged
+- * state, breaking the ability to use the internal microphone.
+- */
+- alc_write_coef_idx(codec, 0x24, 0x0000);
+- alc_write_coef_idx(codec, 0x26, 0x0000);
+- alc_write_coef_idx(codec, 0x29, 0x3000);
+- alc_write_coef_idx(codec, 0x37, 0xfe05);
+- alc_write_coef_idx(codec, 0x45, 0x5089);
++ * A certain other OS sets these coeffs to different values. On at least
++ * one TongFang barebone these settings might survive even a cold
++ * reboot. So to restore a clean slate the values are explicitly reset
++ * to default here. Without this, the external microphone is always in a
++ * plugged-in state, while the internal microphone is always in an
++ * unplugged state, breaking the ability to use the internal microphone.
++ */
++ alc_process_coef_fw(codec, alc256_fixup_set_coef_defaults_coefs);
++}
++
++static const struct coef_fw alc233_fixup_no_audio_jack_coefs[] = {
++ WRITE_COEF(0x1a, 0x9003), WRITE_COEF(0x1b, 0x0e2b), WRITE_COEF(0x37, 0xfe06),
++ WRITE_COEF(0x38, 0x4981), WRITE_COEF(0x45, 0xd489), WRITE_COEF(0x46, 0x0074),
++ WRITE_COEF(0x49, 0x0149),
++ {}
++};
++
++static void alc233_fixup_no_audio_jack(struct hda_codec *codec,
++ const struct hda_fixup *fix,
++ int action)
++{
++ /*
++ * The audio jack input and output is not detected on the ASRock NUC Box
++ * 1100 series when cold booting without this fix. Warm rebooting from a
++ * certain other OS makes the audio functional, as COEF settings are
++ * preserved in this case. This fix sets these altered COEF values as
++ * the default.
++ */
++ alc_process_coef_fw(codec, alc233_fixup_no_audio_jack_coefs);
++}
++
++static void alc256_fixup_mic_no_presence_and_resume(struct hda_codec *codec,
++ const struct hda_fixup *fix,
++ int action)
++{
++ /*
++ * The Clevo NJ51CU comes either with the ALC293 or the ALC256 codec,
++ * but uses the 0x8686 subproduct id in both cases. The ALC256 codec
++ * needs an additional quirk for sound working after suspend and resume.
++ */
++ if (codec->core.vendor_id == 0x10ec0256) {
++ alc_update_coef_idx(codec, 0x10, 1<<9, 0);
++ snd_hda_codec_set_pincfg(codec, 0x19, 0x04a11120);
++ } else {
++ snd_hda_codec_set_pincfg(codec, 0x1a, 0x04a1113c);
++ }
+ }
+
+ enum {
+@@ -6684,6 +6815,7 @@ enum {
+ ALC236_FIXUP_HP_MUTE_LED,
+ ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF,
+ ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET,
++ ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET,
+ ALC295_FIXUP_ASUS_MIC_NO_PRESENCE,
+ ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS,
+ ALC269VC_FIXUP_ACER_HEADSET_MIC,
+@@ -6709,6 +6841,7 @@ enum {
+ ALC285_FIXUP_THINKPAD_NO_BASS_SPK_HEADSET_JACK,
+ ALC287_FIXUP_HP_GPIO_LED,
+ ALC256_FIXUP_HP_HEADSET_MIC,
++ ALC245_FIXUP_HP_GPIO_LED,
+ ALC236_FIXUP_DELL_AIO_HEADSET_MIC,
+ ALC282_FIXUP_ACER_DISABLE_LINEOUT,
+ ALC255_FIXUP_ACER_LIMIT_INT_MIC_BOOST,
+@@ -6726,7 +6859,12 @@ enum {
+ ALC287_FIXUP_LEGION_15IMHG05_AUTOMUTE,
+ ALC287_FIXUP_YOGA7_14ITL_SPEAKERS,
+ ALC287_FIXUP_13S_GEN2_SPEAKERS,
+- ALC256_FIXUP_TONGFANG_RESET_PERSISTENT_SETTINGS,
++ ALC256_FIXUP_SET_COEF_DEFAULTS,
++ ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE,
++ ALC233_FIXUP_NO_AUDIO_JACK,
++ ALC256_FIXUP_MIC_NO_PRESENCE_AND_RESUME,
++ ALC285_FIXUP_LEGION_Y9000X_SPEAKERS,
++ ALC285_FIXUP_LEGION_Y9000X_AUTOMUTE,
+ };
+
+ static const struct hda_fixup alc269_fixups[] = {
+@@ -7333,6 +7471,8 @@ static const struct hda_fixup alc269_fixups[] = {
+ [ALC245_FIXUP_HP_X360_AMP] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc245_fixup_hp_x360_amp,
++ .chained = true,
++ .chain_id = ALC245_FIXUP_HP_GPIO_LED
+ },
+ [ALC288_FIXUP_DELL_HEADSET_MODE] = {
+ .type = HDA_FIXUP_FUNC,
+@@ -7998,6 +8138,14 @@ static const struct hda_fixup alc269_fixups[] = {
+ { }
+ },
+ },
++ [ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = {
++ .type = HDA_FIXUP_VERBS,
++ .v.verbs = (const struct hda_verb[]) {
++ { 0x20, AC_VERB_SET_COEF_INDEX, 0x08},
++ { 0x20, AC_VERB_SET_PROC_COEF, 0x2fcf},
++ { }
++ },
++ },
+ [ALC295_FIXUP_ASUS_MIC_NO_PRESENCE] = {
+ .type = HDA_FIXUP_PINS,
+ .v.pins = (const struct hda_pintbl[]) {
+@@ -8321,6 +8469,18 @@ static const struct hda_fixup alc269_fixups[] = {
+ .chained = true,
+ .chain_id = ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF,
+ },
++ [ALC285_FIXUP_LEGION_Y9000X_SPEAKERS] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc285_fixup_ideapad_s740_coef,
++ .chained = true,
++ .chain_id = ALC285_FIXUP_LEGION_Y9000X_AUTOMUTE,
++ },
++ [ALC285_FIXUP_LEGION_Y9000X_AUTOMUTE] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc287_fixup_legion_15imhg05_speakers,
++ .chained = true,
++ .chain_id = ALC269_FIXUP_THINKPAD_ACPI,
++ },
+ [ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS] = {
+ .type = HDA_FIXUP_VERBS,
+ //.v.verbs = legion_15imhg05_coefs,
+@@ -8428,9 +8588,32 @@ static const struct hda_fixup alc269_fixups[] = {
+ .chained = true,
+ .chain_id = ALC269_FIXUP_HEADSET_MODE,
+ },
+- [ALC256_FIXUP_TONGFANG_RESET_PERSISTENT_SETTINGS] = {
++ [ALC256_FIXUP_SET_COEF_DEFAULTS] = {
+ .type = HDA_FIXUP_FUNC,
+- .v.func = alc256_fixup_tongfang_reset_persistent_settings,
++ .v.func = alc256_fixup_set_coef_defaults,
++ },
++ [ALC245_FIXUP_HP_GPIO_LED] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc245_fixup_hp_gpio_led,
++ },
++ [ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE] = {
++ .type = HDA_FIXUP_PINS,
++ .v.pins = (const struct hda_pintbl[]) {
++ { 0x19, 0x03a11120 }, /* use as headset mic, without its own jack detect */
++ { }
++ },
++ .chained = true,
++ .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC,
++ },
++ [ALC233_FIXUP_NO_AUDIO_JACK] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc233_fixup_no_audio_jack,
++ },
++ [ALC256_FIXUP_MIC_NO_PRESENCE_AND_RESUME] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc256_fixup_mic_no_presence_and_resume,
++ .chained = true,
++ .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC
+ },
+ };
+
+@@ -8468,6 +8651,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1025, 0x1308, "Acer Aspire Z24-890", ALC286_FIXUP_ACER_AIO_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1025, 0x132a, "Acer TravelMate B114-21", ALC233_FIXUP_ACER_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1025, 0x1330, "Acer TravelMate X514-51T", ALC255_FIXUP_ACER_HEADSET_MIC),
++ SND_PCI_QUIRK(0x1025, 0x141f, "Acer Spin SP513-54N", ALC255_FIXUP_ACER_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1025, 0x142b, "Acer Swift SF314-42", ALC255_FIXUP_ACER_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1025, 0x1430, "Acer TravelMate B311R-31", ALC256_FIXUP_ACER_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC),
+@@ -8601,14 +8785,17 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x103c, 0x84da, "HP OMEN dc0019-ur", ALC295_FIXUP_HP_OMEN),
+ SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3),
+ SND_PCI_QUIRK(0x103c, 0x8519, "HP Spectre x360 15-df0xxx", ALC285_FIXUP_HP_SPECTRE_X360),
++ SND_PCI_QUIRK(0x103c, 0x860f, "HP ZBook 15 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x861f, "HP Elite Dragonfly G1", ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x869d, "HP", ALC236_FIXUP_HP_MUTE_LED),
+ SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO),
+ SND_PCI_QUIRK(0x103c, 0x8716, "HP Elite Dragonfly G2 Notebook PC", ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x8720, "HP EliteBook x360 1040 G8 Notebook PC", ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x8724, "HP EliteBook 850 G7", ALC285_FIXUP_HP_GPIO_LED),
++ SND_PCI_QUIRK(0x103c, 0x8728, "HP EliteBook 840 G7", ALC285_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x8729, "HP", ALC285_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
++ SND_PCI_QUIRK(0x103c, 0x8735, "HP ProBook 435 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
+ SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x8760, "HP", ALC285_FIXUP_HP_MUTE_LED),
+ SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED),
+@@ -8617,6 +8804,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ ALC285_FIXUP_HP_GPIO_AMP_INIT),
+ SND_PCI_QUIRK(0x103c, 0x8783, "HP ZBook Fury 15 G7 Mobile Workstation",
+ ALC285_FIXUP_HP_GPIO_AMP_INIT),
++ SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED),
+ SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x87e5, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x87e7, "HP ProBook 450 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED),
+@@ -8643,6 +8831,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x103c, 0x8896, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_MUTE_LED),
+ SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED),
++ SND_PCI_QUIRK(0x103c, 0x89c3, "HP", ALC285_FIXUP_HP_GPIO_LED),
++ SND_PCI_QUIRK(0x103c, 0x89ca, "HP", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
+ SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC),
+ SND_PCI_QUIRK(0x1043, 0x103f, "ASUS TX300", ALC282_FIXUP_ASUS_TX300),
+ SND_PCI_QUIRK(0x1043, 0x106d, "Asus K53BE", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+@@ -8668,6 +8858,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1043, 0x18b1, "Asus MJ401TA", ALC256_FIXUP_ASUS_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1043, 0x18f1, "Asus FX505DT", ALC256_FIXUP_ASUS_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1043, 0x194e, "ASUS UX563FD", ALC294_FIXUP_ASUS_HPE),
++ SND_PCI_QUIRK(0x1043, 0x1970, "ASUS UX550VE", ALC289_FIXUP_ASUS_GA401),
+ SND_PCI_QUIRK(0x1043, 0x1982, "ASUS B1400CEPE", ALC256_FIXUP_ASUS_HPE),
+ SND_PCI_QUIRK(0x1043, 0x19ce, "ASUS B9450FA", ALC294_FIXUP_ASUS_HPE),
+ SND_PCI_QUIRK(0x1043, 0x19e1, "ASUS UX581LV", ALC295_FIXUP_ASUS_MIC_NO_PRESENCE),
+@@ -8683,6 +8874,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1043, 0x1e51, "ASUS Zephyrus M15", ALC294_FIXUP_ASUS_GU502_PINS),
+ SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401),
+ SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401),
++ SND_PCI_QUIRK(0x1043, 0x1d42, "ASUS Zephyrus G14 2022", ALC289_FIXUP_ASUS_GA401),
++ SND_PCI_QUIRK(0x1043, 0x16b2, "ASUS GU603", ALC289_FIXUP_ASUS_GA401),
+ SND_PCI_QUIRK(0x1043, 0x3030, "ASUS ZN270IE", ALC256_FIXUP_ASUS_AIO_GPIO2),
+ SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC),
+ SND_PCI_QUIRK(0x1043, 0x834a, "ASUS S101", ALC269_FIXUP_STEREO_DMIC),
+@@ -8715,6 +8908,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x144d, 0xc740, "Samsung Ativ book 8 (NP870Z5G)", ALC269_FIXUP_ATIV_BOOK_8),
+ SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET),
+ SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET),
++ SND_PCI_QUIRK(0x144d, 0xc832, "Samsung Galaxy Book Flex Alpha (NP730QCJ)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET),
+ SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC),
+@@ -8731,11 +8925,15 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1558, 0x40a1, "Clevo NL40GU", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x40c1, "Clevo NL40[CZ]U", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x40d1, "Clevo NL41DU", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x5015, "Clevo NH5[58]H[HJK]Q", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x5017, "Clevo NH7[79]H[HJK]Q", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50a3, "Clevo NJ51GU", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50b3, "Clevo NK50S[BEZ]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50b6, "Clevo NK50S5", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50b8, "Clevo NK50SZ", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50d5, "Clevo NP50D5", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x50e1, "Clevo NH5[58]HPQ", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x50e2, "Clevo NH7[79]HPQ", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50f0, "Clevo NH50A[CDF]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50f2, "Clevo NH50E[PR]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x50f3, "Clevo NH58DPQ", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+@@ -8761,8 +8959,11 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1558, 0x8561, "Clevo NH[57][0-9][ER][ACDH]Q", ALC269_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1558, 0x8562, "Clevo NH[57][0-9]RZ[Q]", ALC269_FIXUP_DMIC),
+ SND_PCI_QUIRK(0x1558, 0x8668, "Clevo NP50B[BE]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x866d, "Clevo NP5[05]PN[HJK]", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x867c, "Clevo NP7[01]PNP", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x867d, "Clevo NP7[01]PN[HJK]", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x8680, "Clevo NJ50LU", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+- SND_PCI_QUIRK(0x1558, 0x8686, "Clevo NH50[CZ]U", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
++ SND_PCI_QUIRK(0x1558, 0x8686, "Clevo NH50[CZ]U", ALC256_FIXUP_MIC_NO_PRESENCE_AND_RESUME),
+ SND_PCI_QUIRK(0x1558, 0x8a20, "Clevo NH55DCQ-Y", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x8a51, "Clevo NH70RCQ-Y", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1558, 0x8d50, "Clevo NH55RCQ-M", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+@@ -8828,13 +9029,18 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x17aa, 0x3176, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x17aa, 0x3178, "ThinkCentre Station", ALC283_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x17aa, 0x31af, "ThinkCentre Station", ALC623_FIXUP_LENOVO_THINKSTATION_P340),
++ SND_PCI_QUIRK(0x17aa, 0x3813, "Legion 7i 15IMHG05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940", ALC298_FIXUP_LENOVO_SPK_VOLUME),
++ SND_PCI_QUIRK(0x17aa, 0x3819, "Lenovo 13s Gen2 ITL", ALC287_FIXUP_13S_GEN2_SPEAKERS),
++ SND_PCI_QUIRK(0x17aa, 0x3820, "Yoga Duet 7 13ITL6", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
++ SND_PCI_QUIRK(0x17aa, 0x3824, "Legion Y9000X 2020", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3827, "Ideapad S740", ALC285_FIXUP_IDEAPAD_S740_COEF),
++ SND_PCI_QUIRK(0x17aa, 0x3834, "Lenovo IdeaPad Slim 9i 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
++ SND_PCI_QUIRK(0x17aa, 0x383d, "Legion Y9000X 2019", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3843, "Yoga 9i", ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP),
+- SND_PCI_QUIRK(0x17aa, 0x3813, "Legion 7i 15IMHG05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS),
++ SND_PCI_QUIRK(0x17aa, 0x384a, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3852, "Lenovo Yoga 7 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3853, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
+- SND_PCI_QUIRK(0x17aa, 0x3819, "Lenovo 13s Gen2 ITL", ALC287_FIXUP_13S_GEN2_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
+ SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC),
+ SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
+@@ -8854,10 +9060,12 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x17aa, 0x505d, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
+ SND_PCI_QUIRK(0x17aa, 0x505f, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
+ SND_PCI_QUIRK(0x17aa, 0x5062, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
++ SND_PCI_QUIRK(0x17aa, 0x508b, "Thinkpad X12 Gen 1", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS),
+ SND_PCI_QUIRK(0x17aa, 0x5109, "Thinkpad", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0x17aa, 0x511e, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
+ SND_PCI_QUIRK(0x17aa, 0x511f, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
+ SND_PCI_QUIRK(0x17aa, 0x9e54, "LENOVO NB", ALC269_FIXUP_LENOVO_EAPD),
++ SND_PCI_QUIRK(0x1849, 0x1233, "ASRock NUC Box 1100", ALC233_FIXUP_NO_AUDIO_JACK),
+ SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS),
+ SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20),
+ SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI),
+@@ -8865,7 +9073,15 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1b7d, 0xa831, "Ordissimo EVE2 ", ALC269VB_FIXUP_ORDISSIMO_EVE2), /* Also known as Malata PC-B1303 */
+ SND_PCI_QUIRK(0x1c06, 0x2013, "Lemote A1802", ALC269_FIXUP_LEMOTE_A1802),
+ SND_PCI_QUIRK(0x1c06, 0x2015, "Lemote A190X", ALC269_FIXUP_LEMOTE_A190X),
+- SND_PCI_QUIRK(0x1d05, 0x1132, "TongFang PHxTxX1", ALC256_FIXUP_TONGFANG_RESET_PERSISTENT_SETTINGS),
++ SND_PCI_QUIRK(0x1d05, 0x1132, "TongFang PHxTxX1", ALC256_FIXUP_SET_COEF_DEFAULTS),
++ SND_PCI_QUIRK(0x1d05, 0x1096, "TongFang GMxMRxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x1100, "TongFang GKxNRxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x1111, "TongFang GMxZGxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x1119, "TongFang GMxZGxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x1129, "TongFang GMxZGxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x1147, "TongFang GMxTGxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x115c, "TongFang GMxTGxx", ALC269_FIXUP_NO_SHUTUP),
++ SND_PCI_QUIRK(0x1d05, 0x121b, "TongFang GMxAGxx", ALC269_FIXUP_NO_SHUTUP),
+ SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC),
+@@ -9046,6 +9262,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = {
+ {.id = ALC298_FIXUP_HUAWEI_MBX_STEREO, .name = "huawei-mbx-stereo"},
+ {.id = ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE, .name = "alc256-medion-headset"},
+ {.id = ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc298-samsung-headphone"},
++ {.id = ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc256-samsung-headphone"},
+ {.id = ALC255_FIXUP_XIAOMI_HEADSET_MIC, .name = "alc255-xiaomi-headset"},
+ {.id = ALC274_FIXUP_HP_MIC, .name = "alc274-hp-mic-detect"},
+ {.id = ALC245_FIXUP_HP_X360_AMP, .name = "alc245-hp-x360-amp"},
+@@ -9055,6 +9272,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = {
+ {.id = ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP, .name = "alc287-ideapad-bass-spk-amp"},
+ {.id = ALC623_FIXUP_LENOVO_THINKSTATION_P340, .name = "alc623-lenovo-thinkstation-p340"},
+ {.id = ALC255_FIXUP_ACER_HEADPHONE_AND_MIC, .name = "alc255-acer-headphone-and-mic"},
++ {.id = ALC285_FIXUP_HP_GPIO_AMP_INIT, .name = "alc285-hp-amp-init"},
+ {}
+ };
+ #define ALC225_STANDARD_PINS \
+@@ -10167,6 +10385,27 @@ static void alc671_fixup_hp_headset_mic2(struct hda_codec *codec,
+ }
+ }
+
++static void alc897_hp_automute_hook(struct hda_codec *codec,
++ struct hda_jack_callback *jack)
++{
++ struct alc_spec *spec = codec->spec;
++ int vref;
++
++ snd_hda_gen_hp_automute(codec, jack);
++ vref = spec->gen.hp_jack_present ? (PIN_HP | AC_PINCTL_VREF_100) : PIN_HP;
++ snd_hda_codec_write(codec, 0x1b, 0, AC_VERB_SET_PIN_WIDGET_CONTROL,
++ vref);
++}
++
++static void alc897_fixup_lenovo_headset_mic(struct hda_codec *codec,
++ const struct hda_fixup *fix, int action)
++{
++ struct alc_spec *spec = codec->spec;
++ if (action == HDA_FIXUP_ACT_PRE_PROBE) {
++ spec->gen.hp_automute_hook = alc897_hp_automute_hook;
++ }
++}
++
+ static const struct coef_fw alc668_coefs[] = {
+ WRITE_COEF(0x01, 0xbebe), WRITE_COEF(0x02, 0xaaaa), WRITE_COEF(0x03, 0x0),
+ WRITE_COEF(0x04, 0x0180), WRITE_COEF(0x06, 0x0), WRITE_COEF(0x07, 0x0f80),
+@@ -10247,6 +10486,8 @@ enum {
+ ALC668_FIXUP_ASUS_NO_HEADSET_MIC,
+ ALC668_FIXUP_HEADSET_MIC,
+ ALC668_FIXUP_MIC_DET_COEF,
++ ALC897_FIXUP_LENOVO_HEADSET_MIC,
++ ALC897_FIXUP_HEADSET_MIC_PIN,
+ };
+
+ static const struct hda_fixup alc662_fixups[] = {
+@@ -10653,6 +10894,19 @@ static const struct hda_fixup alc662_fixups[] = {
+ {}
+ },
+ },
++ [ALC897_FIXUP_LENOVO_HEADSET_MIC] = {
++ .type = HDA_FIXUP_FUNC,
++ .v.func = alc897_fixup_lenovo_headset_mic,
++ },
++ [ALC897_FIXUP_HEADSET_MIC_PIN] = {
++ .type = HDA_FIXUP_PINS,
++ .v.pins = (const struct hda_pintbl[]) {
++ { 0x1a, 0x03a11050 },
++ { }
++ },
++ .chained = true,
++ .chain_id = ALC897_FIXUP_LENOVO_HEADSET_MIC
++ },
+ };
+
+ static const struct snd_pci_quirk alc662_fixup_tbl[] = {
+@@ -10679,6 +10933,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x1028, 0x069f, "Dell", ALC668_FIXUP_DELL_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0x103c, 0x1632, "HP RP5800", ALC662_FIXUP_HP_RP5800),
+ SND_PCI_QUIRK(0x103c, 0x873e, "HP", ALC671_FIXUP_HP_HEADSET_MIC2),
++ SND_PCI_QUIRK(0x103c, 0x885f, "HP 288 Pro G8", ALC671_FIXUP_HP_HEADSET_MIC2),
+ SND_PCI_QUIRK(0x1043, 0x1080, "Asus UX501VW", ALC668_FIXUP_HEADSET_MODE),
+ SND_PCI_QUIRK(0x1043, 0x11cd, "Asus N550", ALC662_FIXUP_ASUS_Nx50),
+ SND_PCI_QUIRK(0x1043, 0x129d, "Asus N750", ALC662_FIXUP_ASUS_Nx50),
+@@ -10697,6 +10952,11 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = {
+ SND_PCI_QUIRK(0x144d, 0xc051, "Samsung R720", ALC662_FIXUP_IDEAPAD),
+ SND_PCI_QUIRK(0x14cd, 0x5003, "USI", ALC662_FIXUP_USI_HEADSET_MODE),
+ SND_PCI_QUIRK(0x17aa, 0x1036, "Lenovo P520", ALC662_FIXUP_LENOVO_MULTI_CODECS),
++ SND_PCI_QUIRK(0x17aa, 0x1057, "Lenovo P360", ALC897_FIXUP_HEADSET_MIC_PIN),
++ SND_PCI_QUIRK(0x17aa, 0x32ca, "Lenovo ThinkCentre M80", ALC897_FIXUP_HEADSET_MIC_PIN),
++ SND_PCI_QUIRK(0x17aa, 0x32cb, "Lenovo ThinkCentre M70", ALC897_FIXUP_HEADSET_MIC_PIN),
++ SND_PCI_QUIRK(0x17aa, 0x32cf, "Lenovo ThinkCentre M950", ALC897_FIXUP_HEADSET_MIC_PIN),
++ SND_PCI_QUIRK(0x17aa, 0x32f7, "Lenovo ThinkCentre M90", ALC897_FIXUP_HEADSET_MIC_PIN),
+ SND_PCI_QUIRK(0x17aa, 0x38af, "Lenovo Ideapad Y550P", ALC662_FIXUP_IDEAPAD),
+ SND_PCI_QUIRK(0x17aa, 0x3a0d, "Lenovo Ideapad Y550", ALC662_FIXUP_IDEAPAD),
+ SND_PCI_QUIRK(0x1849, 0x5892, "ASRock B150M", ALC892_FIXUP_ASROCK_MOBO),
+diff --git a/sound/pci/ice1712/ice1724.c b/sound/pci/ice1712/ice1724.c
+index f6275868877a7..6fab2ad85bbec 100644
+--- a/sound/pci/ice1712/ice1724.c
++++ b/sound/pci/ice1712/ice1724.c
+@@ -2519,8 +2519,8 @@ static int snd_vt1724_create(struct snd_card *card,
+ *
+ */
+
+-static int snd_vt1724_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_vt1724_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2662,6 +2662,12 @@ static int snd_vt1724_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_vt1724_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_vt1724_probe(pci, pci_id));
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static int snd_vt1724_suspend(struct device *dev)
+ {
+diff --git a/sound/pci/intel8x0.c b/sound/pci/intel8x0.c
+index a51032b3ac4d8..ae285c0a629c8 100644
+--- a/sound/pci/intel8x0.c
++++ b/sound/pci/intel8x0.c
+@@ -3109,8 +3109,8 @@ static int check_default_spdif_aclink(struct pci_dev *pci)
+ return 0;
+ }
+
+-static int snd_intel8x0_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_intel8x0_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct intel8x0 *chip;
+@@ -3189,6 +3189,12 @@ static int snd_intel8x0_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_intel8x0_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_intel8x0_probe(pci, pci_id));
++}
++
+ static struct pci_driver intel8x0_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_intel8x0_ids,
+diff --git a/sound/pci/intel8x0m.c b/sound/pci/intel8x0m.c
+index 7de3cb2f17b52..2845cc006d0cf 100644
+--- a/sound/pci/intel8x0m.c
++++ b/sound/pci/intel8x0m.c
+@@ -1178,8 +1178,8 @@ static struct shortname_table {
+ { 0 },
+ };
+
+-static int snd_intel8x0m_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_intel8x0m_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct intel8x0m *chip;
+@@ -1225,6 +1225,12 @@ static int snd_intel8x0m_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_intel8x0m_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_intel8x0m_probe(pci, pci_id));
++}
++
+ static struct pci_driver intel8x0m_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_intel8x0m_ids,
+diff --git a/sound/pci/korg1212/korg1212.c b/sound/pci/korg1212/korg1212.c
+index 5c9e240ff6a9c..33b4f95d65b3f 100644
+--- a/sound/pci/korg1212/korg1212.c
++++ b/sound/pci/korg1212/korg1212.c
+@@ -2355,7 +2355,7 @@ snd_korg1212_probe(struct pci_dev *pci,
+
+ err = snd_korg1212_create(card, pci);
+ if (err < 0)
+- return err;
++ goto error;
+
+ strcpy(card->driver, "korg1212");
+ strcpy(card->shortname, "korg1212");
+@@ -2366,10 +2366,14 @@ snd_korg1212_probe(struct pci_dev *pci,
+
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver korg1212_driver = {
+diff --git a/sound/pci/lola/lola.c b/sound/pci/lola/lola.c
+index 5269a1d396a5b..1aa30e90b86a7 100644
+--- a/sound/pci/lola/lola.c
++++ b/sound/pci/lola/lola.c
+@@ -637,8 +637,8 @@ static int lola_create(struct snd_card *card, struct pci_dev *pci, int dev)
+ return 0;
+ }
+
+-static int lola_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __lola_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -687,6 +687,12 @@ static int lola_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int lola_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __lola_probe(pci, pci_id));
++}
++
+ /* PCI IDs */
+ static const struct pci_device_id lola_ids[] = {
+ { PCI_VDEVICE(DIGIGRAM, 0x0001) },
+diff --git a/sound/pci/lx6464es/lx6464es.c b/sound/pci/lx6464es/lx6464es.c
+index 168a1084f7303..bd9b6148dd6fb 100644
+--- a/sound/pci/lx6464es/lx6464es.c
++++ b/sound/pci/lx6464es/lx6464es.c
+@@ -1019,7 +1019,7 @@ static int snd_lx6464es_probe(struct pci_dev *pci,
+ err = snd_lx6464es_create(card, pci);
+ if (err < 0) {
+ dev_err(card->dev, "error during snd_lx6464es_create\n");
+- return err;
++ goto error;
+ }
+
+ strcpy(card->driver, "LX6464ES");
+@@ -1036,12 +1036,16 @@ static int snd_lx6464es_probe(struct pci_dev *pci,
+
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+
+ dev_dbg(chip->card->dev, "initialization successful\n");
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver lx6464es_driver = {
+diff --git a/sound/pci/maestro3.c b/sound/pci/maestro3.c
+index 056838ead21d6..261850775c807 100644
+--- a/sound/pci/maestro3.c
++++ b/sound/pci/maestro3.c
+@@ -2637,7 +2637,7 @@ snd_m3_create(struct snd_card *card, struct pci_dev *pci,
+ /*
+ */
+ static int
+-snd_m3_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++__snd_m3_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2702,6 +2702,12 @@ snd_m3_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ return 0;
+ }
+
++static int
++snd_m3_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_m3_probe(pci, pci_id));
++}
++
+ static struct pci_driver m3_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_m3_ids,
+diff --git a/sound/pci/nm256/nm256.c b/sound/pci/nm256/nm256.c
+index c9c178504959e..f99a1e96e9231 100644
+--- a/sound/pci/nm256/nm256.c
++++ b/sound/pci/nm256/nm256.c
+@@ -1573,7 +1573,6 @@ snd_nm256_create(struct snd_card *card, struct pci_dev *pci)
+ chip->coeffs_current = 0;
+
+ snd_nm256_init_chip(chip);
+- card->private_free = snd_nm256_free;
+
+ // pci_set_master(pci); /* needed? */
+ return 0;
+@@ -1680,6 +1679,7 @@ static int snd_nm256_probe(struct pci_dev *pci,
+ err = snd_card_register(card);
+ if (err < 0)
+ return err;
++ card->private_free = snd_nm256_free;
+
+ pci_set_drvdata(pci, card);
+ return 0;
+diff --git a/sound/pci/oxygen/oxygen_lib.c b/sound/pci/oxygen/oxygen_lib.c
+index 4fb3f2484fdba..92ffe9dc20c55 100644
+--- a/sound/pci/oxygen/oxygen_lib.c
++++ b/sound/pci/oxygen/oxygen_lib.c
+@@ -576,7 +576,7 @@ static void oxygen_card_free(struct snd_card *card)
+ mutex_destroy(&chip->mutex);
+ }
+
+-int oxygen_pci_probe(struct pci_dev *pci, int index, char *id,
++static int __oxygen_pci_probe(struct pci_dev *pci, int index, char *id,
+ struct module *owner,
+ const struct pci_device_id *ids,
+ int (*get_model)(struct oxygen *chip,
+@@ -701,6 +701,16 @@ int oxygen_pci_probe(struct pci_dev *pci, int index, char *id,
+ pci_set_drvdata(pci, card);
+ return 0;
+ }
++
++int oxygen_pci_probe(struct pci_dev *pci, int index, char *id,
++ struct module *owner,
++ const struct pci_device_id *ids,
++ int (*get_model)(struct oxygen *chip,
++ const struct pci_device_id *id))
++{
++ return snd_card_free_on_error(&pci->dev,
++ __oxygen_pci_probe(pci, index, id, owner, ids, get_model));
++}
+ EXPORT_SYMBOL(oxygen_pci_probe);
+
+ #ifdef CONFIG_PM_SLEEP
+diff --git a/sound/pci/riptide/riptide.c b/sound/pci/riptide/riptide.c
+index 5a987c683c41c..b37c877c2c160 100644
+--- a/sound/pci/riptide/riptide.c
++++ b/sound/pci/riptide/riptide.c
+@@ -2023,7 +2023,7 @@ static void snd_riptide_joystick_remove(struct pci_dev *pci)
+ #endif
+
+ static int
+-snd_card_riptide_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++__snd_card_riptide_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -2124,6 +2124,12 @@ snd_card_riptide_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ return 0;
+ }
+
++static int
++snd_card_riptide_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_card_riptide_probe(pci, pci_id));
++}
++
+ static struct pci_driver driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_riptide_ids,
+diff --git a/sound/pci/rme32.c b/sound/pci/rme32.c
+index 5b6bd9f0b2f77..9c0ac025e1432 100644
+--- a/sound/pci/rme32.c
++++ b/sound/pci/rme32.c
+@@ -1875,7 +1875,7 @@ static void snd_rme32_card_free(struct snd_card *card)
+ }
+
+ static int
+-snd_rme32_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++__snd_rme32_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct rme32 *rme32;
+@@ -1927,6 +1927,12 @@ snd_rme32_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+ return 0;
+ }
+
++static int
++snd_rme32_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_rme32_probe(pci, pci_id));
++}
++
+ static struct pci_driver rme32_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_rme32_ids,
+diff --git a/sound/pci/rme96.c b/sound/pci/rme96.c
+index 8fc8115049203..bccb7e0d3d116 100644
+--- a/sound/pci/rme96.c
++++ b/sound/pci/rme96.c
+@@ -2430,8 +2430,8 @@ static void snd_rme96_card_free(struct snd_card *card)
+ }
+
+ static int
+-snd_rme96_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++__snd_rme96_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct rme96 *rme96;
+@@ -2498,6 +2498,12 @@ snd_rme96_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_rme96_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_rme96_probe(pci, pci_id));
++}
++
+ static struct pci_driver rme96_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_rme96_ids,
+diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c
+index 75aa2ea733a59..3db641318d3ae 100644
+--- a/sound/pci/rme9652/hdsp.c
++++ b/sound/pci/rme9652/hdsp.c
+@@ -468,8 +468,11 @@ struct hdsp {
+ unsigned char ss_out_channels;
+ u32 io_loopback; /* output loopback channel states*/
+
+- struct snd_dma_buffer *capture_dma_buf;
+- struct snd_dma_buffer *playback_dma_buf;
++ /* DMA buffers; those are copied instances from the original snd_dma_buf
++ * objects (which are managed via devres) for the address alignments
++ */
++ struct snd_dma_buffer capture_dma_buf;
++ struct snd_dma_buffer playback_dma_buf;
+ unsigned char *capture_buffer; /* suitably aligned address */
+ unsigned char *playback_buffer; /* suitably aligned address */
+
+@@ -3764,30 +3767,32 @@ static void snd_hdsp_proc_init(struct hdsp *hdsp)
+
+ static int snd_hdsp_initialize_memory(struct hdsp *hdsp)
+ {
+- unsigned long pb_bus, cb_bus;
++ struct snd_dma_buffer *capture_dma, *playback_dma;
+
+- hdsp->capture_dma_buf =
+- snd_hammerfall_get_buffer(hdsp->pci, HDSP_DMA_AREA_BYTES);
+- hdsp->playback_dma_buf =
+- snd_hammerfall_get_buffer(hdsp->pci, HDSP_DMA_AREA_BYTES);
+- if (!hdsp->capture_dma_buf || !hdsp->playback_dma_buf) {
++ capture_dma = snd_hammerfall_get_buffer(hdsp->pci, HDSP_DMA_AREA_BYTES);
++ playback_dma = snd_hammerfall_get_buffer(hdsp->pci, HDSP_DMA_AREA_BYTES);
++ if (!capture_dma || !playback_dma) {
+ dev_err(hdsp->card->dev,
+ "%s: no buffers available\n", hdsp->card_name);
+ return -ENOMEM;
+ }
+
+- /* Align to bus-space 64K boundary */
++ /* copy to the own data for alignment */
++ hdsp->capture_dma_buf = *capture_dma;
++ hdsp->playback_dma_buf = *playback_dma;
+
+- cb_bus = ALIGN(hdsp->capture_dma_buf->addr, 0x10000ul);
+- pb_bus = ALIGN(hdsp->playback_dma_buf->addr, 0x10000ul);
++ /* Align to bus-space 64K boundary */
++ hdsp->capture_dma_buf.addr = ALIGN(capture_dma->addr, 0x10000ul);
++ hdsp->playback_dma_buf.addr = ALIGN(playback_dma->addr, 0x10000ul);
+
+ /* Tell the card where it is */
++ hdsp_write(hdsp, HDSP_inputBufferAddress, hdsp->capture_dma_buf.addr);
++ hdsp_write(hdsp, HDSP_outputBufferAddress, hdsp->playback_dma_buf.addr);
+
+- hdsp_write(hdsp, HDSP_inputBufferAddress, cb_bus);
+- hdsp_write(hdsp, HDSP_outputBufferAddress, pb_bus);
+-
+- hdsp->capture_buffer = hdsp->capture_dma_buf->area + (cb_bus - hdsp->capture_dma_buf->addr);
+- hdsp->playback_buffer = hdsp->playback_dma_buf->area + (pb_bus - hdsp->playback_dma_buf->addr);
++ hdsp->capture_dma_buf.area += hdsp->capture_dma_buf.addr - capture_dma->addr;
++ hdsp->playback_dma_buf.area += hdsp->playback_dma_buf.addr - playback_dma->addr;
++ hdsp->capture_buffer = hdsp->capture_dma_buf.area;
++ hdsp->playback_buffer = hdsp->playback_dma_buf.area;
+
+ return 0;
+ }
+@@ -4507,7 +4512,7 @@ static int snd_hdsp_playback_open(struct snd_pcm_substream *substream)
+ snd_pcm_set_sync(substream);
+
+ runtime->hw = snd_hdsp_playback_subinfo;
+- snd_pcm_set_runtime_buffer(substream, hdsp->playback_dma_buf);
++ snd_pcm_set_runtime_buffer(substream, &hdsp->playback_dma_buf);
+
+ hdsp->playback_pid = current->pid;
+ hdsp->playback_substream = substream;
+@@ -4583,7 +4588,7 @@ static int snd_hdsp_capture_open(struct snd_pcm_substream *substream)
+ snd_pcm_set_sync(substream);
+
+ runtime->hw = snd_hdsp_capture_subinfo;
+- snd_pcm_set_runtime_buffer(substream, hdsp->capture_dma_buf);
++ snd_pcm_set_runtime_buffer(substream, &hdsp->capture_dma_buf);
+
+ hdsp->capture_pid = current->pid;
+ hdsp->capture_substream = substream;
+@@ -5439,17 +5444,21 @@ static int snd_hdsp_probe(struct pci_dev *pci,
+ hdsp->pci = pci;
+ err = snd_hdsp_create(card, hdsp);
+ if (err)
+- return err;
++ goto error;
+
+ strcpy(card->shortname, "Hammerfall DSP");
+ sprintf(card->longname, "%s at 0x%lx, irq %d", hdsp->card_name,
+ hdsp->port, hdsp->irq);
+ err = snd_card_register(card);
+ if (err)
+- return err;
++ goto error;
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver hdsp_driver = {
+diff --git a/sound/pci/rme9652/hdspm.c b/sound/pci/rme9652/hdspm.c
+index ff06ee82607cf..fa1812e7a49dc 100644
+--- a/sound/pci/rme9652/hdspm.c
++++ b/sound/pci/rme9652/hdspm.c
+@@ -6895,7 +6895,7 @@ static int snd_hdspm_probe(struct pci_dev *pci,
+
+ err = snd_hdspm_create(card, hdspm);
+ if (err < 0)
+- return err;
++ goto error;
+
+ if (hdspm->io_type != MADIface) {
+ snprintf(card->shortname, sizeof(card->shortname), "%s_%x",
+@@ -6914,12 +6914,16 @@ static int snd_hdspm_probe(struct pci_dev *pci,
+
+ err = snd_card_register(card);
+ if (err < 0)
+- return err;
++ goto error;
+
+ pci_set_drvdata(pci, card);
+
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver hdspm_driver = {
+diff --git a/sound/pci/rme9652/rme9652.c b/sound/pci/rme9652/rme9652.c
+index e76f737ac9e8e..1d614fe89a6ae 100644
+--- a/sound/pci/rme9652/rme9652.c
++++ b/sound/pci/rme9652/rme9652.c
+@@ -208,8 +208,11 @@ struct snd_rme9652 {
+ unsigned char ds_channels;
+ unsigned char ss_channels; /* different for hammerfall/hammerfall-light */
+
+- struct snd_dma_buffer *playback_dma_buf;
+- struct snd_dma_buffer *capture_dma_buf;
++ /* DMA buffers; those are copied instances from the original snd_dma_buf
++ * objects (which are managed via devres) for the address alignments
++ */
++ struct snd_dma_buffer playback_dma_buf;
++ struct snd_dma_buffer capture_dma_buf;
+
+ unsigned char *capture_buffer; /* suitably aligned address */
+ unsigned char *playback_buffer; /* suitably aligned address */
+@@ -1719,30 +1722,32 @@ static void snd_rme9652_card_free(struct snd_card *card)
+
+ static int snd_rme9652_initialize_memory(struct snd_rme9652 *rme9652)
+ {
+- unsigned long pb_bus, cb_bus;
++ struct snd_dma_buffer *capture_dma, *playback_dma;
+
+- rme9652->capture_dma_buf =
+- snd_hammerfall_get_buffer(rme9652->pci, RME9652_DMA_AREA_BYTES);
+- rme9652->playback_dma_buf =
+- snd_hammerfall_get_buffer(rme9652->pci, RME9652_DMA_AREA_BYTES);
+- if (!rme9652->capture_dma_buf || !rme9652->playback_dma_buf) {
++ capture_dma = snd_hammerfall_get_buffer(rme9652->pci, RME9652_DMA_AREA_BYTES);
++ playback_dma = snd_hammerfall_get_buffer(rme9652->pci, RME9652_DMA_AREA_BYTES);
++ if (!capture_dma || !playback_dma) {
+ dev_err(rme9652->card->dev,
+ "%s: no buffers available\n", rme9652->card_name);
+ return -ENOMEM;
+ }
+
+- /* Align to bus-space 64K boundary */
++ /* copy to the own data for alignment */
++ rme9652->capture_dma_buf = *capture_dma;
++ rme9652->playback_dma_buf = *playback_dma;
+
+- cb_bus = ALIGN(rme9652->capture_dma_buf->addr, 0x10000ul);
+- pb_bus = ALIGN(rme9652->playback_dma_buf->addr, 0x10000ul);
++ /* Align to bus-space 64K boundary */
++ rme9652->capture_dma_buf.addr = ALIGN(capture_dma->addr, 0x10000ul);
++ rme9652->playback_dma_buf.addr = ALIGN(playback_dma->addr, 0x10000ul);
+
+ /* Tell the card where it is */
++ rme9652_write(rme9652, RME9652_rec_buffer, rme9652->capture_dma_buf.addr);
++ rme9652_write(rme9652, RME9652_play_buffer, rme9652->playback_dma_buf.addr);
+
+- rme9652_write(rme9652, RME9652_rec_buffer, cb_bus);
+- rme9652_write(rme9652, RME9652_play_buffer, pb_bus);
+-
+- rme9652->capture_buffer = rme9652->capture_dma_buf->area + (cb_bus - rme9652->capture_dma_buf->addr);
+- rme9652->playback_buffer = rme9652->playback_dma_buf->area + (pb_bus - rme9652->playback_dma_buf->addr);
++ rme9652->capture_dma_buf.area += rme9652->capture_dma_buf.addr - capture_dma->addr;
++ rme9652->playback_dma_buf.area += rme9652->playback_dma_buf.addr - playback_dma->addr;
++ rme9652->capture_buffer = rme9652->capture_dma_buf.area;
++ rme9652->playback_buffer = rme9652->playback_dma_buf.area;
+
+ return 0;
+ }
+@@ -2259,7 +2264,7 @@ static int snd_rme9652_playback_open(struct snd_pcm_substream *substream)
+ snd_pcm_set_sync(substream);
+
+ runtime->hw = snd_rme9652_playback_subinfo;
+- snd_pcm_set_runtime_buffer(substream, rme9652->playback_dma_buf);
++ snd_pcm_set_runtime_buffer(substream, &rme9652->playback_dma_buf);
+
+ if (rme9652->capture_substream == NULL) {
+ rme9652_stop(rme9652);
+@@ -2318,7 +2323,7 @@ static int snd_rme9652_capture_open(struct snd_pcm_substream *substream)
+ snd_pcm_set_sync(substream);
+
+ runtime->hw = snd_rme9652_capture_subinfo;
+- snd_pcm_set_runtime_buffer(substream, rme9652->capture_dma_buf);
++ snd_pcm_set_runtime_buffer(substream, &rme9652->capture_dma_buf);
+
+ if (rme9652->playback_substream == NULL) {
+ rme9652_stop(rme9652);
+@@ -2567,7 +2572,7 @@ static int snd_rme9652_probe(struct pci_dev *pci,
+ rme9652->pci = pci;
+ err = snd_rme9652_create(card, rme9652, precise_ptr[dev]);
+ if (err)
+- return err;
++ goto error;
+
+ strcpy(card->shortname, rme9652->card_name);
+
+@@ -2575,10 +2580,14 @@ static int snd_rme9652_probe(struct pci_dev *pci,
+ card->shortname, rme9652->port, rme9652->irq);
+ err = snd_card_register(card);
+ if (err)
+- return err;
++ goto error;
+ pci_set_drvdata(pci, card);
+ dev++;
+ return 0;
++
++ error:
++ snd_card_free(card);
++ return err;
+ }
+
+ static struct pci_driver rme9652_driver = {
+diff --git a/sound/pci/sis7019.c b/sound/pci/sis7019.c
+index 0b722b0e0604b..fabe393607f8f 100644
+--- a/sound/pci/sis7019.c
++++ b/sound/pci/sis7019.c
+@@ -1331,8 +1331,8 @@ static int sis_chip_create(struct snd_card *card,
+ return 0;
+ }
+
+-static int snd_sis7019_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_sis7019_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct sis7019 *sis;
+@@ -1352,8 +1352,8 @@ static int snd_sis7019_probe(struct pci_dev *pci,
+ if (!codecs)
+ codecs = SIS_PRIMARY_CODEC_PRESENT;
+
+- rc = snd_card_new(&pci->dev, index, id, THIS_MODULE,
+- sizeof(*sis), &card);
++ rc = snd_devm_card_new(&pci->dev, index, id, THIS_MODULE,
++ sizeof(*sis), &card);
+ if (rc < 0)
+ return rc;
+
+@@ -1386,6 +1386,12 @@ static int snd_sis7019_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_sis7019_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_sis7019_probe(pci, pci_id));
++}
++
+ static struct pci_driver sis7019_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_sis7019_ids,
+diff --git a/sound/pci/sonicvibes.c b/sound/pci/sonicvibes.c
+index c8c49881008fd..f91cbf6eeca0f 100644
+--- a/sound/pci/sonicvibes.c
++++ b/sound/pci/sonicvibes.c
+@@ -1387,8 +1387,8 @@ static int snd_sonicvibes_midi(struct sonicvibes *sonic,
+ return 0;
+ }
+
+-static int snd_sonic_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_sonic_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ static int dev;
+ struct snd_card *card;
+@@ -1459,6 +1459,12 @@ static int snd_sonic_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_sonic_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_sonic_probe(pci, pci_id));
++}
++
+ static struct pci_driver sonicvibes_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_sonic_ids,
+diff --git a/sound/pci/via82xx.c b/sound/pci/via82xx.c
+index 65514f7e42d7d..361b83fd721e6 100644
+--- a/sound/pci/via82xx.c
++++ b/sound/pci/via82xx.c
+@@ -2458,8 +2458,8 @@ static int check_dxs_list(struct pci_dev *pci, int revision)
+ return VIA_DXS_48K;
+ };
+
+-static int snd_via82xx_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_via82xx_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct via82xx *chip;
+@@ -2569,6 +2569,12 @@ static int snd_via82xx_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_via82xx_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_via82xx_probe(pci, pci_id));
++}
++
+ static struct pci_driver via82xx_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_via82xx_ids,
+diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c
+index 234f7fbed2364..ca7f024bf8ec6 100644
+--- a/sound/pci/via82xx_modem.c
++++ b/sound/pci/via82xx_modem.c
+@@ -1103,8 +1103,8 @@ static int snd_via82xx_create(struct snd_card *card,
+ }
+
+
+-static int snd_via82xx_probe(struct pci_dev *pci,
+- const struct pci_device_id *pci_id)
++static int __snd_via82xx_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
+ {
+ struct snd_card *card;
+ struct via82xx_modem *chip;
+@@ -1157,6 +1157,12 @@ static int snd_via82xx_probe(struct pci_dev *pci,
+ return 0;
+ }
+
++static int snd_via82xx_probe(struct pci_dev *pci,
++ const struct pci_device_id *pci_id)
++{
++ return snd_card_free_on_error(&pci->dev, __snd_via82xx_probe(pci, pci_id));
++}
++
+ static struct pci_driver via82xx_modem_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = snd_via82xx_modem_ids,
+diff --git a/sound/soc/amd/vangogh/acp5x-pcm-dma.c b/sound/soc/amd/vangogh/acp5x-pcm-dma.c
+index f10de38976cb5..bfca4cf423cf1 100644
+--- a/sound/soc/amd/vangogh/acp5x-pcm-dma.c
++++ b/sound/soc/amd/vangogh/acp5x-pcm-dma.c
+@@ -281,7 +281,7 @@ static int acp5x_dma_hw_params(struct snd_soc_component *component,
+ return -EINVAL;
+ }
+ size = params_buffer_bytes(params);
+- rtd->dma_addr = substream->dma_buffer.addr;
++ rtd->dma_addr = substream->runtime->dma_addr;
+ rtd->num_pages = (PAGE_ALIGN(size) >> PAGE_SHIFT);
+ config_acp5x_dma(rtd, substream->stream);
+ return 0;
+@@ -426,51 +426,51 @@ static int acp5x_audio_remove(struct platform_device *pdev)
+ static int __maybe_unused acp5x_pcm_resume(struct device *dev)
+ {
+ struct i2s_dev_data *adata;
+- u32 val, reg_val, frmt_val;
++ struct i2s_stream_instance *rtd;
++ u32 val;
+
+- reg_val = 0;
+- frmt_val = 0;
+ adata = dev_get_drvdata(dev);
+
+ if (adata->play_stream && adata->play_stream->runtime) {
+- struct i2s_stream_instance *rtd =
+- adata->play_stream->runtime->private_data;
++ rtd = adata->play_stream->runtime->private_data;
+ config_acp5x_dma(rtd, SNDRV_PCM_STREAM_PLAYBACK);
+- switch (rtd->i2s_instance) {
+- case I2S_HS_INSTANCE:
+- reg_val = ACP_HSTDM_ITER;
+- frmt_val = ACP_HSTDM_TXFRMT;
+- break;
+- case I2S_SP_INSTANCE:
+- default:
+- reg_val = ACP_I2STDM_ITER;
+- frmt_val = ACP_I2STDM_TXFRMT;
++ acp_writel((rtd->xfer_resolution << 3), rtd->acp5x_base + ACP_HSTDM_ITER);
++ if (adata->tdm_mode == TDM_ENABLE) {
++ acp_writel(adata->tdm_fmt, adata->acp5x_base + ACP_HSTDM_TXFRMT);
++ val = acp_readl(adata->acp5x_base + ACP_HSTDM_ITER);
++ acp_writel(val | 0x2, adata->acp5x_base + ACP_HSTDM_ITER);
++ }
++ }
++ if (adata->i2ssp_play_stream && adata->i2ssp_play_stream->runtime) {
++ rtd = adata->i2ssp_play_stream->runtime->private_data;
++ config_acp5x_dma(rtd, SNDRV_PCM_STREAM_PLAYBACK);
++ acp_writel((rtd->xfer_resolution << 3), rtd->acp5x_base + ACP_I2STDM_ITER);
++ if (adata->tdm_mode == TDM_ENABLE) {
++ acp_writel(adata->tdm_fmt, adata->acp5x_base + ACP_I2STDM_TXFRMT);
++ val = acp_readl(adata->acp5x_base + ACP_I2STDM_ITER);
++ acp_writel(val | 0x2, adata->acp5x_base + ACP_I2STDM_ITER);
+ }
+- acp_writel((rtd->xfer_resolution << 3),
+- rtd->acp5x_base + reg_val);
+ }
+
+ if (adata->capture_stream && adata->capture_stream->runtime) {
+- struct i2s_stream_instance *rtd =
+- adata->capture_stream->runtime->private_data;
++ rtd = adata->capture_stream->runtime->private_data;
+ config_acp5x_dma(rtd, SNDRV_PCM_STREAM_CAPTURE);
+- switch (rtd->i2s_instance) {
+- case I2S_HS_INSTANCE:
+- reg_val = ACP_HSTDM_IRER;
+- frmt_val = ACP_HSTDM_RXFRMT;
+- break;
+- case I2S_SP_INSTANCE:
+- default:
+- reg_val = ACP_I2STDM_IRER;
+- frmt_val = ACP_I2STDM_RXFRMT;
++ acp_writel((rtd->xfer_resolution << 3), rtd->acp5x_base + ACP_HSTDM_IRER);
++ if (adata->tdm_mode == TDM_ENABLE) {
++ acp_writel(adata->tdm_fmt, adata->acp5x_base + ACP_HSTDM_RXFRMT);
++ val = acp_readl(adata->acp5x_base + ACP_HSTDM_IRER);
++ acp_writel(val | 0x2, adata->acp5x_base + ACP_HSTDM_IRER);
+ }
+- acp_writel((rtd->xfer_resolution << 3),
+- rtd->acp5x_base + reg_val);
+ }
+- if (adata->tdm_mode == TDM_ENABLE) {
+- acp_writel(adata->tdm_fmt, adata->acp5x_base + frmt_val);
+- val = acp_readl(adata->acp5x_base + reg_val);
+- acp_writel(val | 0x2, adata->acp5x_base + reg_val);
++ if (adata->i2ssp_capture_stream && adata->i2ssp_capture_stream->runtime) {
++ rtd = adata->i2ssp_capture_stream->runtime->private_data;
++ config_acp5x_dma(rtd, SNDRV_PCM_STREAM_CAPTURE);
++ acp_writel((rtd->xfer_resolution << 3), rtd->acp5x_base + ACP_I2STDM_IRER);
++ if (adata->tdm_mode == TDM_ENABLE) {
++ acp_writel(adata->tdm_fmt, adata->acp5x_base + ACP_I2STDM_RXFRMT);
++ val = acp_readl(adata->acp5x_base + ACP_I2STDM_IRER);
++ acp_writel(val | 0x2, adata->acp5x_base + ACP_I2STDM_IRER);
++ }
+ }
+ acp_writel(1, adata->acp5x_base + ACP_EXTERNAL_INTR_ENB);
+ return 0;
+diff --git a/sound/soc/atmel/atmel_ssc_dai.c b/sound/soc/atmel/atmel_ssc_dai.c
+index 6a63e8797a0b6..97533412ce11e 100644
+--- a/sound/soc/atmel/atmel_ssc_dai.c
++++ b/sound/soc/atmel/atmel_ssc_dai.c
+@@ -280,7 +280,10 @@ static int atmel_ssc_startup(struct snd_pcm_substream *substream,
+
+ /* Enable PMC peripheral clock for this SSC */
+ pr_debug("atmel_ssc_dai: Starting clock\n");
+- clk_enable(ssc_p->ssc->clk);
++ ret = clk_enable(ssc_p->ssc->clk);
++ if (ret)
++ return ret;
++
+ ssc_p->mck_rate = clk_get_rate(ssc_p->ssc->clk);
+
+ /* Reset the SSC unless initialized to keep it in a clean state */
+diff --git a/sound/soc/atmel/mikroe-proto.c b/sound/soc/atmel/mikroe-proto.c
+index 0be7b4221c146..93d114f5b9e6a 100644
+--- a/sound/soc/atmel/mikroe-proto.c
++++ b/sound/soc/atmel/mikroe-proto.c
+@@ -115,7 +115,8 @@ static int snd_proto_probe(struct platform_device *pdev)
+ cpu_np = of_parse_phandle(np, "i2s-controller", 0);
+ if (!cpu_np) {
+ dev_err(&pdev->dev, "i2s-controller missing\n");
+- return -EINVAL;
++ ret = -EINVAL;
++ goto put_codec_node;
+ }
+ dai->cpus->of_node = cpu_np;
+ dai->platforms->of_node = cpu_np;
+@@ -125,7 +126,8 @@ static int snd_proto_probe(struct platform_device *pdev)
+ &bitclkmaster, &framemaster);
+ if (bitclkmaster != framemaster) {
+ dev_err(&pdev->dev, "Must be the same bitclock and frame master\n");
+- return -EINVAL;
++ ret = -EINVAL;
++ goto put_cpu_node;
+ }
+ if (bitclkmaster) {
+ if (codec_np == bitclkmaster)
+@@ -136,18 +138,20 @@ static int snd_proto_probe(struct platform_device *pdev)
+ dai_fmt |= snd_soc_daifmt_parse_clock_provider_as_flag(np, NULL);
+ }
+
+- of_node_put(bitclkmaster);
+- of_node_put(framemaster);
+- dai->dai_fmt = dai_fmt;
+-
+- of_node_put(codec_np);
+- of_node_put(cpu_np);
+
++ dai->dai_fmt = dai_fmt;
+ ret = snd_soc_register_card(&snd_proto);
+ if (ret && ret != -EPROBE_DEFER)
+ dev_err(&pdev->dev,
+ "snd_soc_register_card() failed: %d\n", ret);
+
++
++put_cpu_node:
++ of_node_put(bitclkmaster);
++ of_node_put(framemaster);
++ of_node_put(cpu_np);
++put_codec_node:
++ of_node_put(codec_np);
+ return ret;
+ }
+
+diff --git a/sound/soc/atmel/sam9g20_wm8731.c b/sound/soc/atmel/sam9g20_wm8731.c
+index ed1f69b570244..d243de5f23dc1 100644
+--- a/sound/soc/atmel/sam9g20_wm8731.c
++++ b/sound/soc/atmel/sam9g20_wm8731.c
+@@ -46,35 +46,6 @@
+ */
+ #undef ENABLE_MIC_INPUT
+
+-static struct clk *mclk;
+-
+-static int at91sam9g20ek_set_bias_level(struct snd_soc_card *card,
+- struct snd_soc_dapm_context *dapm,
+- enum snd_soc_bias_level level)
+-{
+- static int mclk_on;
+- int ret = 0;
+-
+- switch (level) {
+- case SND_SOC_BIAS_ON:
+- case SND_SOC_BIAS_PREPARE:
+- if (!mclk_on)
+- ret = clk_enable(mclk);
+- if (ret == 0)
+- mclk_on = 1;
+- break;
+-
+- case SND_SOC_BIAS_OFF:
+- case SND_SOC_BIAS_STANDBY:
+- if (mclk_on)
+- clk_disable(mclk);
+- mclk_on = 0;
+- break;
+- }
+-
+- return ret;
+-}
+-
+ static const struct snd_soc_dapm_widget at91sam9g20ek_dapm_widgets[] = {
+ SND_SOC_DAPM_MIC("Int Mic", NULL),
+ SND_SOC_DAPM_SPK("Ext Spk", NULL),
+@@ -135,7 +106,6 @@ static struct snd_soc_card snd_soc_at91sam9g20ek = {
+ .owner = THIS_MODULE,
+ .dai_link = &at91sam9g20ek_dai,
+ .num_links = 1,
+- .set_bias_level = at91sam9g20ek_set_bias_level,
+
+ .dapm_widgets = at91sam9g20ek_dapm_widgets,
+ .num_dapm_widgets = ARRAY_SIZE(at91sam9g20ek_dapm_widgets),
+@@ -148,7 +118,6 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev)
+ {
+ struct device_node *np = pdev->dev.of_node;
+ struct device_node *codec_np, *cpu_np;
+- struct clk *pllb;
+ struct snd_soc_card *card = &snd_soc_at91sam9g20ek;
+ int ret;
+
+@@ -162,31 +131,6 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev)
+ return -EINVAL;
+ }
+
+- /*
+- * Codec MCLK is supplied by PCK0 - set it up.
+- */
+- mclk = clk_get(NULL, "pck0");
+- if (IS_ERR(mclk)) {
+- dev_err(&pdev->dev, "Failed to get MCLK\n");
+- ret = PTR_ERR(mclk);
+- goto err;
+- }
+-
+- pllb = clk_get(NULL, "pllb");
+- if (IS_ERR(pllb)) {
+- dev_err(&pdev->dev, "Failed to get PLLB\n");
+- ret = PTR_ERR(pllb);
+- goto err_mclk;
+- }
+- ret = clk_set_parent(mclk, pllb);
+- clk_put(pllb);
+- if (ret != 0) {
+- dev_err(&pdev->dev, "Failed to set MCLK parent\n");
+- goto err_mclk;
+- }
+-
+- clk_set_rate(mclk, MCLK_RATE);
+-
+ card->dev = &pdev->dev;
+
+ /* Parse device node info */
+@@ -214,6 +158,7 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev)
+ cpu_np = of_parse_phandle(np, "atmel,ssc-controller", 0);
+ if (!cpu_np) {
+ dev_err(&pdev->dev, "dai and pcm info missing\n");
++ of_node_put(codec_np);
+ return -EINVAL;
+ }
+ at91sam9g20ek_dai.cpus->of_node = cpu_np;
+@@ -229,9 +174,6 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev)
+
+ return ret;
+
+-err_mclk:
+- clk_put(mclk);
+- mclk = NULL;
+ err:
+ atmel_ssc_put_audio(0);
+ return ret;
+@@ -241,8 +183,6 @@ static int at91sam9g20ek_audio_remove(struct platform_device *pdev)
+ {
+ struct snd_soc_card *card = platform_get_drvdata(pdev);
+
+- clk_disable(mclk);
+- mclk = NULL;
+ snd_soc_unregister_card(card);
+ atmel_ssc_put_audio(0);
+
+diff --git a/sound/soc/atmel/sam9x5_wm8731.c b/sound/soc/atmel/sam9x5_wm8731.c
+index 7745250fd7438..529604a06c532 100644
+--- a/sound/soc/atmel/sam9x5_wm8731.c
++++ b/sound/soc/atmel/sam9x5_wm8731.c
+@@ -142,7 +142,7 @@ static int sam9x5_wm8731_driver_probe(struct platform_device *pdev)
+ if (!cpu_np) {
+ dev_err(&pdev->dev, "atmel,ssc-controller node missing\n");
+ ret = -EINVAL;
+- goto out;
++ goto out_put_codec_np;
+ }
+ dai->cpus->of_node = cpu_np;
+ dai->platforms->of_node = cpu_np;
+@@ -153,12 +153,9 @@ static int sam9x5_wm8731_driver_probe(struct platform_device *pdev)
+ if (ret != 0) {
+ dev_err(&pdev->dev, "Failed to set SSC %d for audio: %d\n",
+ ret, priv->ssc_id);
+- goto out;
++ goto out_put_cpu_np;
+ }
+
+- of_node_put(codec_np);
+- of_node_put(cpu_np);
+-
+ ret = devm_snd_soc_register_card(&pdev->dev, card);
+ if (ret) {
+ dev_err(&pdev->dev, "Platform device allocation failed\n");
+@@ -167,10 +164,14 @@ static int sam9x5_wm8731_driver_probe(struct platform_device *pdev)
+
+ dev_dbg(&pdev->dev, "%s ok\n", __func__);
+
+- return ret;
++ goto out_put_cpu_np;
+
+ out_put_audio:
+ atmel_ssc_put_audio(priv->ssc_id);
++out_put_cpu_np:
++ of_node_put(cpu_np);
++out_put_codec_np:
++ of_node_put(codec_np);
+ out:
+ return ret;
+ }
+diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
+index 216cea04ad704..47e675e8bd006 100644
+--- a/sound/soc/codecs/Kconfig
++++ b/sound/soc/codecs/Kconfig
+@@ -235,8 +235,7 @@ config SND_SOC_ALL_CODECS
+ imply SND_SOC_UDA1380
+ imply SND_SOC_WCD9335
+ imply SND_SOC_WCD934X
+- imply SND_SOC_WCD937X
+- imply SND_SOC_WCD938X
++ imply SND_SOC_WCD938X_SDW
+ imply SND_SOC_LPASS_RX_MACRO
+ imply SND_SOC_LPASS_TX_MACRO
+ imply SND_SOC_WL1273
+@@ -691,6 +690,7 @@ config SND_SOC_CS4349
+
+ config SND_SOC_CS47L15
+ tristate
++ depends on MFD_CS47L15
+
+ config SND_SOC_CS47L24
+ tristate
+@@ -698,15 +698,19 @@ config SND_SOC_CS47L24
+
+ config SND_SOC_CS47L35
+ tristate
++ depends on MFD_CS47L35
+
+ config SND_SOC_CS47L85
+ tristate
++ depends on MFD_CS47L85
+
+ config SND_SOC_CS47L90
+ tristate
++ depends on MFD_CS47L90
+
+ config SND_SOC_CS47L92
+ tristate
++ depends on MFD_CS47L92
+
+ # Cirrus Logic Quad-Channel ADC
+ config SND_SOC_CS53L30
+diff --git a/sound/soc/codecs/cpcap.c b/sound/soc/codecs/cpcap.c
+index 05bbacd0d174d..f1c13f42e1c14 100644
+--- a/sound/soc/codecs/cpcap.c
++++ b/sound/soc/codecs/cpcap.c
+@@ -1667,6 +1667,8 @@ static int cpcap_codec_probe(struct platform_device *pdev)
+ {
+ struct device_node *codec_node =
+ of_get_child_by_name(pdev->dev.parent->of_node, "audio-codec");
++ if (!codec_node)
++ return -ENODEV;
+
+ pdev->dev.of_node = codec_node;
+
+diff --git a/sound/soc/codecs/cs4265.c b/sound/soc/codecs/cs4265.c
+index cffd6111afaca..b49cb92d7b9e8 100644
+--- a/sound/soc/codecs/cs4265.c
++++ b/sound/soc/codecs/cs4265.c
+@@ -150,7 +150,6 @@ static const struct snd_kcontrol_new cs4265_snd_controls[] = {
+ SOC_SINGLE("E to F Buffer Disable Switch", CS4265_SPDIF_CTL1,
+ 6, 1, 0),
+ SOC_ENUM("C Data Access", cam_mode_enum),
+- SOC_SINGLE("SPDIF Switch", CS4265_SPDIF_CTL2, 5, 1, 1),
+ SOC_SINGLE("Validity Bit Control Switch", CS4265_SPDIF_CTL2,
+ 3, 1, 0),
+ SOC_ENUM("SPDIF Mono/Stereo", spdif_mono_stereo_enum),
+@@ -186,7 +185,7 @@ static const struct snd_soc_dapm_widget cs4265_dapm_widgets[] = {
+
+ SND_SOC_DAPM_SWITCH("Loopback", SND_SOC_NOPM, 0, 0,
+ &loopback_ctl),
+- SND_SOC_DAPM_SWITCH("SPDIF", SND_SOC_NOPM, 0, 0,
++ SND_SOC_DAPM_SWITCH("SPDIF", CS4265_SPDIF_CTL2, 5, 1,
+ &spdif_switch),
+ SND_SOC_DAPM_SWITCH("DAC", CS4265_PWRCTL, 1, 1,
+ &dac_switch),
+diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c
+index 9a463ab54bddc..762d9de73dbc2 100644
+--- a/sound/soc/codecs/cs42l42.c
++++ b/sound/soc/codecs/cs42l42.c
+@@ -93,7 +93,7 @@ static const struct reg_default cs42l42_reg_defaults[] = {
+ { CS42L42_ASP_RX_INT_MASK, 0x1F },
+ { CS42L42_ASP_TX_INT_MASK, 0x0F },
+ { CS42L42_CODEC_INT_MASK, 0x03 },
+- { CS42L42_SRCPL_INT_MASK, 0xFF },
++ { CS42L42_SRCPL_INT_MASK, 0x7F },
+ { CS42L42_VPMON_INT_MASK, 0x01 },
+ { CS42L42_PLL_LOCK_INT_MASK, 0x01 },
+ { CS42L42_TSRS_PLUG_INT_MASK, 0x0F },
+@@ -130,7 +130,7 @@ static const struct reg_default cs42l42_reg_defaults[] = {
+ { CS42L42_MIXER_CHA_VOL, 0x3F },
+ { CS42L42_MIXER_ADC_VOL, 0x3F },
+ { CS42L42_MIXER_CHB_VOL, 0x3F },
+- { CS42L42_EQ_COEF_IN0, 0x22 },
++ { CS42L42_EQ_COEF_IN0, 0x00 },
+ { CS42L42_EQ_COEF_IN1, 0x00 },
+ { CS42L42_EQ_COEF_IN2, 0x00 },
+ { CS42L42_EQ_COEF_IN3, 0x00 },
+@@ -853,11 +853,10 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream,
+
+ switch(substream->stream) {
+ case SNDRV_PCM_STREAM_CAPTURE:
+- if (channels == 2) {
+- val |= CS42L42_ASP_TX_CH2_AP_MASK;
+- val |= width << CS42L42_ASP_TX_CH2_RES_SHIFT;
+- }
+- val |= width << CS42L42_ASP_TX_CH1_RES_SHIFT;
++ /* channel 2 on high LRCLK */
++ val = CS42L42_ASP_TX_CH2_AP_MASK |
++ (width << CS42L42_ASP_TX_CH2_RES_SHIFT) |
++ (width << CS42L42_ASP_TX_CH1_RES_SHIFT);
+
+ snd_soc_component_update_bits(component, CS42L42_ASP_TX_CH_AP_RES,
+ CS42L42_ASP_TX_CH1_AP_MASK | CS42L42_ASP_TX_CH2_AP_MASK |
+@@ -1685,12 +1684,15 @@ static void cs42l42_setup_hs_type_detect(struct cs42l42_private *cs42l42)
+ (1 << CS42L42_HS_CLAMP_DISABLE_SHIFT));
+
+ /* Enable the tip sense circuit */
++ regmap_update_bits(cs42l42->regmap, CS42L42_TSENSE_CTL,
++ CS42L42_TS_INV_MASK, CS42L42_TS_INV_MASK);
++
+ regmap_update_bits(cs42l42->regmap, CS42L42_TIPSENSE_CTL,
+ CS42L42_TIP_SENSE_CTRL_MASK |
+ CS42L42_TIP_SENSE_INV_MASK |
+ CS42L42_TIP_SENSE_DEBOUNCE_MASK,
+ (3 << CS42L42_TIP_SENSE_CTRL_SHIFT) |
+- (0 << CS42L42_TIP_SENSE_INV_SHIFT) |
++ (!cs42l42->ts_inv << CS42L42_TIP_SENSE_INV_SHIFT) |
+ (2 << CS42L42_TIP_SENSE_DEBOUNCE_SHIFT));
+
+ /* Save the initial status of the tip sense */
+@@ -1734,10 +1736,6 @@ static int cs42l42_handle_device_data(struct device *dev,
+ cs42l42->ts_inv = CS42L42_TS_INV_DIS;
+ }
+
+- regmap_update_bits(cs42l42->regmap, CS42L42_TSENSE_CTL,
+- CS42L42_TS_INV_MASK,
+- (cs42l42->ts_inv << CS42L42_TS_INV_SHIFT));
+-
+ ret = device_property_read_u32(dev, "cirrus,ts-dbnc-rise", &val);
+ if (!ret) {
+ switch (val) {
+@@ -1948,8 +1946,9 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client,
+ NULL, cs42l42_irq_thread,
+ IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+ "cs42l42", cs42l42);
+-
+- if (ret != 0)
++ if (ret == -EPROBE_DEFER)
++ goto err_disable;
++ else if (ret != 0)
+ dev_err(&i2c_client->dev,
+ "Failed to request IRQ: %d\n", ret);
+
+diff --git a/sound/soc/codecs/da7219.c b/sound/soc/codecs/da7219.c
+index 13009d08b09ac..c7493549a9a50 100644
+--- a/sound/soc/codecs/da7219.c
++++ b/sound/soc/codecs/da7219.c
+@@ -446,7 +446,7 @@ static int da7219_tonegen_freq_put(struct snd_kcontrol *kcontrol,
+ struct soc_mixer_control *mixer_ctrl =
+ (struct soc_mixer_control *) kcontrol->private_value;
+ unsigned int reg = mixer_ctrl->reg;
+- __le16 val;
++ __le16 val_new, val_old;
+ int ret;
+
+ /*
+@@ -454,13 +454,19 @@ static int da7219_tonegen_freq_put(struct snd_kcontrol *kcontrol,
+ * Therefore we need to convert to little endian here to align with
+ * HW registers.
+ */
+- val = cpu_to_le16(ucontrol->value.integer.value[0]);
++ val_new = cpu_to_le16(ucontrol->value.integer.value[0]);
+
+ mutex_lock(&da7219->ctrl_lock);
+- ret = regmap_raw_write(da7219->regmap, reg, &val, sizeof(val));
++ ret = regmap_raw_read(da7219->regmap, reg, &val_old, sizeof(val_old));
++ if (ret == 0 && (val_old != val_new))
++ ret = regmap_raw_write(da7219->regmap, reg,
++ &val_new, sizeof(val_new));
+ mutex_unlock(&da7219->ctrl_lock);
+
+- return ret;
++ if (ret < 0)
++ return ret;
++
++ return val_old != val_new;
+ }
+
+
+diff --git a/sound/soc/codecs/es8316.c b/sound/soc/codecs/es8316.c
+index 067757d1d70a3..5fb02635c1406 100644
+--- a/sound/soc/codecs/es8316.c
++++ b/sound/soc/codecs/es8316.c
+@@ -811,12 +811,9 @@ static int es8316_i2c_probe(struct i2c_client *i2c_client,
+ mutex_init(&es8316->lock);
+
+ ret = devm_request_threaded_irq(dev, es8316->irq, NULL, es8316_irq,
+- IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
++ IRQF_TRIGGER_HIGH | IRQF_ONESHOT | IRQF_NO_AUTOEN,
+ "es8316", es8316);
+- if (ret == 0) {
+- /* Gets re-enabled by es8316_set_jack() */
+- disable_irq(es8316->irq);
+- } else {
++ if (ret) {
+ dev_warn(dev, "Failed to get IRQ %d: %d\n", es8316->irq, ret);
+ es8316->irq = -ENXIO;
+ }
+diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c
+index b61f980cabdc0..b07607a9ecea4 100644
+--- a/sound/soc/codecs/hdmi-codec.c
++++ b/sound/soc/codecs/hdmi-codec.c
+@@ -277,7 +277,7 @@ struct hdmi_codec_priv {
+ bool busy;
+ struct snd_soc_jack *jack;
+ unsigned int jack_status;
+- u8 iec_status[5];
++ u8 iec_status[AES_IEC958_STATUS_SIZE];
+ };
+
+ static const struct snd_soc_dapm_widget hdmi_widgets[] = {
+diff --git a/sound/soc/codecs/lpass-rx-macro.c b/sound/soc/codecs/lpass-rx-macro.c
+index 196b06898eeb2..23452900b9ae1 100644
+--- a/sound/soc/codecs/lpass-rx-macro.c
++++ b/sound/soc/codecs/lpass-rx-macro.c
+@@ -2039,6 +2039,10 @@ static int rx_macro_load_compander_coeff(struct snd_soc_component *component,
+ int i;
+ int hph_pwr_mode;
+
++ /* AUX does not have compander */
++ if (comp == INTERP_AUX)
++ return 0;
++
+ if (!rx->comp_enabled[comp])
+ return 0;
+
+@@ -2188,7 +2192,7 @@ static int rx_macro_config_classh(struct snd_soc_component *component,
+ snd_soc_component_update_bits(component,
+ CDC_RX_CLSH_DECAY_CTRL,
+ CDC_RX_CLSH_DECAY_RATE_MASK, 0x0);
+- snd_soc_component_update_bits(component,
++ snd_soc_component_write_field(component,
+ CDC_RX_RX1_RX_PATH_CFG0,
+ CDC_RX_RXn_CLSH_EN_MASK, 0x1);
+ break;
+@@ -2268,7 +2272,7 @@ static int rx_macro_mux_get(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *component = snd_soc_dapm_to_component(widget->dapm);
+ struct rx_macro *rx = snd_soc_component_get_drvdata(component);
+
+- ucontrol->value.integer.value[0] =
++ ucontrol->value.enumerated.item[0] =
+ rx->rx_port_value[widget->shift];
+ return 0;
+ }
+@@ -2280,7 +2284,7 @@ static int rx_macro_mux_put(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *component = snd_soc_dapm_to_component(widget->dapm);
+ struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+ struct snd_soc_dapm_update *update = NULL;
+- u32 rx_port_value = ucontrol->value.integer.value[0];
++ u32 rx_port_value = ucontrol->value.enumerated.item[0];
+ u32 aif_rst;
+ struct rx_macro *rx = snd_soc_component_get_drvdata(component);
+
+@@ -2392,7 +2396,7 @@ static int rx_macro_get_hph_pwr_mode(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+ struct rx_macro *rx = snd_soc_component_get_drvdata(component);
+
+- ucontrol->value.integer.value[0] = rx->hph_pwr_mode;
++ ucontrol->value.enumerated.item[0] = rx->hph_pwr_mode;
+ return 0;
+ }
+
+@@ -2402,7 +2406,7 @@ static int rx_macro_put_hph_pwr_mode(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+ struct rx_macro *rx = snd_soc_component_get_drvdata(component);
+
+- rx->hph_pwr_mode = ucontrol->value.integer.value[0];
++ rx->hph_pwr_mode = ucontrol->value.enumerated.item[0];
+ return 0;
+ }
+
+@@ -2688,8 +2692,8 @@ static uint32_t get_iir_band_coeff(struct snd_soc_component *component,
+ int reg, b2_reg;
+
+ /* Address does not automatically update if reading */
+- reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B1_CTL + 16 * iir_idx;
+- b2_reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B2_CTL + 16 * iir_idx;
++ reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B1_CTL + 0x80 * iir_idx;
++ b2_reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B2_CTL + 0x80 * iir_idx;
+
+ snd_soc_component_write(component, reg,
+ ((band_idx * BAND_MAX + coeff_idx) *
+@@ -2718,7 +2722,7 @@ static uint32_t get_iir_band_coeff(struct snd_soc_component *component,
+ static void set_iir_band_coeff(struct snd_soc_component *component,
+ int iir_idx, int band_idx, uint32_t value)
+ {
+- int reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B2_CTL + 16 * iir_idx;
++ int reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B2_CTL + 0x80 * iir_idx;
+
+ snd_soc_component_write(component, reg, (value & 0xFF));
+ snd_soc_component_write(component, reg, (value >> 8) & 0xFF);
+@@ -2739,7 +2743,7 @@ static int rx_macro_put_iir_band_audio_mixer(
+ int iir_idx = ctl->iir_idx;
+ int band_idx = ctl->band_idx;
+ u32 coeff[BAND_MAX];
+- int reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B1_CTL + 16 * iir_idx;
++ int reg = CDC_RX_SIDETONE_IIR0_IIR_COEF_B1_CTL + 0x80 * iir_idx;
+
+ memcpy(&coeff[0], ucontrol->value.bytes.data, params->max);
+
+@@ -3542,6 +3546,8 @@ static int rx_macro_probe(struct platform_device *pdev)
+ return PTR_ERR(base);
+
+ rx->regmap = devm_regmap_init_mmio(dev, base, &rx_regmap_config);
++ if (IS_ERR(rx->regmap))
++ return PTR_ERR(rx->regmap);
+
+ dev_set_drvdata(dev, rx);
+
+diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c
+index 27a0d5defd273..e4bbc6bd49252 100644
+--- a/sound/soc/codecs/lpass-tx-macro.c
++++ b/sound/soc/codecs/lpass-tx-macro.c
+@@ -1803,6 +1803,8 @@ static int tx_macro_probe(struct platform_device *pdev)
+ return PTR_ERR(base);
+
+ tx->regmap = devm_regmap_init_mmio(dev, base, &tx_regmap_config);
++ if (IS_ERR(tx->regmap))
++ return PTR_ERR(tx->regmap);
+
+ dev_set_drvdata(dev, tx);
+
+diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c
+index 56c93f4465c9c..08702a21212c1 100644
+--- a/sound/soc/codecs/lpass-va-macro.c
++++ b/sound/soc/codecs/lpass-va-macro.c
+@@ -780,7 +780,7 @@ static int va_macro_dec_mode_get(struct snd_kcontrol *kcontrol,
+ struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+ int path = e->shift_l;
+
+- ucontrol->value.integer.value[0] = va->dec_mode[path];
++ ucontrol->value.enumerated.item[0] = va->dec_mode[path];
+
+ return 0;
+ }
+@@ -789,7 +789,7 @@ static int va_macro_dec_mode_put(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
+- int value = ucontrol->value.integer.value[0];
++ int value = ucontrol->value.enumerated.item[0];
+ struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+ int path = e->shift_l;
+ struct va_macro *va = snd_soc_component_get_drvdata(comp);
+diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c
+index d3ac318fd6b6a..dd1a8b7bc794c 100644
+--- a/sound/soc/codecs/lpass-wsa-macro.c
++++ b/sound/soc/codecs/lpass-wsa-macro.c
+@@ -2405,6 +2405,8 @@ static int wsa_macro_probe(struct platform_device *pdev)
+ return PTR_ERR(base);
+
+ wsa->regmap = devm_regmap_init_mmio(dev, base, &wsa_regmap_config);
++ if (IS_ERR(wsa->regmap))
++ return PTR_ERR(wsa->regmap);
+
+ dev_set_drvdata(dev, wsa);
+
+diff --git a/sound/soc/codecs/max9759.c b/sound/soc/codecs/max9759.c
+index 00e9d4fd1651f..0c261335c8a16 100644
+--- a/sound/soc/codecs/max9759.c
++++ b/sound/soc/codecs/max9759.c
+@@ -64,7 +64,8 @@ static int speaker_gain_control_put(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *c = snd_soc_kcontrol_component(kcontrol);
+ struct max9759 *priv = snd_soc_component_get_drvdata(c);
+
+- if (ucontrol->value.integer.value[0] > 3)
++ if (ucontrol->value.integer.value[0] < 0 ||
++ ucontrol->value.integer.value[0] > 3)
+ return -EINVAL;
+
+ priv->gain = ucontrol->value.integer.value[0];
+diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c
+index b45ec35cd63c3..62b41ca050a20 100644
+--- a/sound/soc/codecs/max98090.c
++++ b/sound/soc/codecs/max98090.c
+@@ -413,6 +413,9 @@ static int max98090_put_enab_tlv(struct snd_kcontrol *kcontrol,
+
+ val = (val >> mc->shift) & mask;
+
++ if (sel < 0 || sel > mc->max)
++ return -EINVAL;
++
+ *select = sel;
+
+ /* Setting a volume is only valid if it is already On */
+@@ -427,7 +430,7 @@ static int max98090_put_enab_tlv(struct snd_kcontrol *kcontrol,
+ mask << mc->shift,
+ sel << mc->shift);
+
+- return 0;
++ return *select != val;
+ }
+
+ static const char *max98090_perf_pwr_text[] =
+diff --git a/sound/soc/codecs/msm8916-wcd-analog.c b/sound/soc/codecs/msm8916-wcd-analog.c
+index 3ddd822240e3a..971b8360b5b1b 100644
+--- a/sound/soc/codecs/msm8916-wcd-analog.c
++++ b/sound/soc/codecs/msm8916-wcd-analog.c
+@@ -1221,8 +1221,10 @@ static int pm8916_wcd_analog_spmi_probe(struct platform_device *pdev)
+ }
+
+ irq = platform_get_irq_byname(pdev, "mbhc_switch_int");
+- if (irq < 0)
+- return irq;
++ if (irq < 0) {
++ ret = irq;
++ goto err_disable_clk;
++ }
+
+ ret = devm_request_threaded_irq(dev, irq, NULL,
+ pm8916_mbhc_switch_irq_handler,
+@@ -1234,8 +1236,10 @@ static int pm8916_wcd_analog_spmi_probe(struct platform_device *pdev)
+
+ if (priv->mbhc_btn_enabled) {
+ irq = platform_get_irq_byname(pdev, "mbhc_but_press_det");
+- if (irq < 0)
+- return irq;
++ if (irq < 0) {
++ ret = irq;
++ goto err_disable_clk;
++ }
+
+ ret = devm_request_threaded_irq(dev, irq, NULL,
+ mbhc_btn_press_irq_handler,
+@@ -1246,8 +1250,10 @@ static int pm8916_wcd_analog_spmi_probe(struct platform_device *pdev)
+ dev_err(dev, "cannot request mbhc button press irq\n");
+
+ irq = platform_get_irq_byname(pdev, "mbhc_but_rel_det");
+- if (irq < 0)
+- return irq;
++ if (irq < 0) {
++ ret = irq;
++ goto err_disable_clk;
++ }
+
+ ret = devm_request_threaded_irq(dev, irq, NULL,
+ mbhc_btn_release_irq_handler,
+@@ -1264,6 +1270,10 @@ static int pm8916_wcd_analog_spmi_probe(struct platform_device *pdev)
+ return devm_snd_soc_register_component(dev, &pm8916_wcd_analog,
+ pm8916_wcd_analog_dai,
+ ARRAY_SIZE(pm8916_wcd_analog_dai));
++
++err_disable_clk:
++ clk_disable_unprepare(priv->mclk);
++ return ret;
+ }
+
+ static int pm8916_wcd_analog_spmi_remove(struct platform_device *pdev)
+diff --git a/sound/soc/codecs/msm8916-wcd-digital.c b/sound/soc/codecs/msm8916-wcd-digital.c
+index fcc10c8bc6259..20a07c92b2fc2 100644
+--- a/sound/soc/codecs/msm8916-wcd-digital.c
++++ b/sound/soc/codecs/msm8916-wcd-digital.c
+@@ -1201,14 +1201,24 @@ static int msm8916_wcd_digital_probe(struct platform_device *pdev)
+ ret = clk_prepare_enable(priv->mclk);
+ if (ret < 0) {
+ dev_err(dev, "failed to enable mclk %d\n", ret);
+- return ret;
++ goto err_clk;
+ }
+
+ dev_set_drvdata(dev, priv);
+
+- return devm_snd_soc_register_component(dev, &msm8916_wcd_digital,
++ ret = devm_snd_soc_register_component(dev, &msm8916_wcd_digital,
+ msm8916_wcd_digital_dai,
+ ARRAY_SIZE(msm8916_wcd_digital_dai));
++ if (ret)
++ goto err_mclk;
++
++ return 0;
++
++err_mclk:
++ clk_disable_unprepare(priv->mclk);
++err_clk:
++ clk_disable_unprepare(priv->ahbclk);
++ return ret;
+ }
+
+ static int msm8916_wcd_digital_remove(struct platform_device *pdev)
+diff --git a/sound/soc/codecs/mt6358.c b/sound/soc/codecs/mt6358.c
+index 9b263a9a669dc..4c7b5d940799b 100644
+--- a/sound/soc/codecs/mt6358.c
++++ b/sound/soc/codecs/mt6358.c
+@@ -107,6 +107,7 @@ int mt6358_set_mtkaif_protocol(struct snd_soc_component *cmpnt,
+ priv->mtkaif_protocol = mtkaif_protocol;
+ return 0;
+ }
++EXPORT_SYMBOL_GPL(mt6358_set_mtkaif_protocol);
+
+ static void playback_gpio_set(struct mt6358_priv *priv)
+ {
+@@ -273,6 +274,7 @@ int mt6358_mtkaif_calibration_enable(struct snd_soc_component *cmpnt)
+ 1 << RG_AUD_PAD_TOP_DAT_MISO_LOOPBACK_SFT);
+ return 0;
+ }
++EXPORT_SYMBOL_GPL(mt6358_mtkaif_calibration_enable);
+
+ int mt6358_mtkaif_calibration_disable(struct snd_soc_component *cmpnt)
+ {
+@@ -296,6 +298,7 @@ int mt6358_mtkaif_calibration_disable(struct snd_soc_component *cmpnt)
+ capture_gpio_reset(priv);
+ return 0;
+ }
++EXPORT_SYMBOL_GPL(mt6358_mtkaif_calibration_disable);
+
+ int mt6358_set_mtkaif_calibration_phase(struct snd_soc_component *cmpnt,
+ int phase_1, int phase_2)
+@@ -310,6 +313,7 @@ int mt6358_set_mtkaif_calibration_phase(struct snd_soc_component *cmpnt,
+ phase_2 << RG_AUD_PAD_TOP_PHASE_MODE2_SFT);
+ return 0;
+ }
++EXPORT_SYMBOL_GPL(mt6358_set_mtkaif_calibration_phase);
+
+ /* dl pga gain */
+ enum {
+diff --git a/sound/soc/codecs/nau8824.c b/sound/soc/codecs/nau8824.c
+index f946ef65a4c19..f7018f2dd21fd 100644
+--- a/sound/soc/codecs/nau8824.c
++++ b/sound/soc/codecs/nau8824.c
+@@ -8,6 +8,7 @@
+
+ #include <linux/module.h>
+ #include <linux/delay.h>
++#include <linux/dmi.h>
+ #include <linux/init.h>
+ #include <linux/i2c.h>
+ #include <linux/regmap.h>
+@@ -27,6 +28,12 @@
+
+ #include "nau8824.h"
+
++#define NAU8824_JD_ACTIVE_HIGH BIT(0)
++
++static int nau8824_quirk;
++static int quirk_override = -1;
++module_param_named(quirk, quirk_override, uint, 0444);
++MODULE_PARM_DESC(quirk, "Board-specific quirk override");
+
+ static int nau8824_config_sysclk(struct nau8824 *nau8824,
+ int clk_id, unsigned int freq);
+@@ -1845,6 +1852,34 @@ static int nau8824_read_device_properties(struct device *dev,
+ return 0;
+ }
+
++/* Please keep this list alphabetically sorted */
++static const struct dmi_system_id nau8824_quirk_table[] = {
++ {
++ /* Cyberbook T116 rugged tablet */
++ .matches = {
++ DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Default string"),
++ DMI_EXACT_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "20170531"),
++ },
++ .driver_data = (void *)(NAU8824_JD_ACTIVE_HIGH),
++ },
++ {}
++};
++
++static void nau8824_check_quirks(void)
++{
++ const struct dmi_system_id *dmi_id;
++
++ if (quirk_override != -1) {
++ nau8824_quirk = quirk_override;
++ return;
++ }
++
++ dmi_id = dmi_first_match(nau8824_quirk_table);
++ if (dmi_id)
++ nau8824_quirk = (unsigned long)dmi_id->driver_data;
++}
++
+ static int nau8824_i2c_probe(struct i2c_client *i2c,
+ const struct i2c_device_id *id)
+ {
+@@ -1869,6 +1904,11 @@ static int nau8824_i2c_probe(struct i2c_client *i2c,
+ nau8824->irq = i2c->irq;
+ sema_init(&nau8824->jd_sem, 1);
+
++ nau8824_check_quirks();
++
++ if (nau8824_quirk & NAU8824_JD_ACTIVE_HIGH)
++ nau8824->jkdet_polarity = 0;
++
+ nau8824_print_device_properties(nau8824);
+
+ ret = regmap_read(nau8824->regmap, NAU8824_REG_I2C_DEVICE_ID, &value);
+diff --git a/sound/soc/codecs/rk817_codec.c b/sound/soc/codecs/rk817_codec.c
+index 943d7d933e81b..cce6f4e7992f5 100644
+--- a/sound/soc/codecs/rk817_codec.c
++++ b/sound/soc/codecs/rk817_codec.c
+@@ -489,7 +489,7 @@ static int rk817_platform_probe(struct platform_device *pdev)
+
+ rk817_codec_parse_dt_property(&pdev->dev, rk817_codec_data);
+
+- rk817_codec_data->mclk = clk_get(pdev->dev.parent, "mclk");
++ rk817_codec_data->mclk = devm_clk_get(pdev->dev.parent, "mclk");
+ if (IS_ERR(rk817_codec_data->mclk)) {
+ dev_dbg(&pdev->dev, "Unable to get mclk\n");
+ ret = -ENXIO;
+@@ -508,12 +508,14 @@ static int rk817_platform_probe(struct platform_device *pdev)
+ if (ret < 0) {
+ dev_err(&pdev->dev, "%s() register codec error %d\n",
+ __func__, ret);
+- goto err_;
++ goto err_clk;
+ }
+
+ return 0;
+-err_:
+
++err_clk:
++ clk_disable_unprepare(rk817_codec_data->mclk);
++err_:
+ return ret;
+ }
+
+@@ -539,3 +541,4 @@ module_platform_driver(rk817_codec_driver);
+ MODULE_DESCRIPTION("ASoC RK817 codec driver");
+ MODULE_AUTHOR("binyuan <kevan.lan@rock-chips.com>");
+ MODULE_LICENSE("GPL v2");
++MODULE_ALIAS("platform:rk817-codec");
+diff --git a/sound/soc/codecs/rt5651.c b/sound/soc/codecs/rt5651.c
+index fc0c83b73f099..93820561b9f5d 100644
+--- a/sound/soc/codecs/rt5651.c
++++ b/sound/soc/codecs/rt5651.c
+@@ -2261,11 +2261,8 @@ static int rt5651_i2c_probe(struct i2c_client *i2c,
+
+ ret = devm_request_irq(&i2c->dev, rt5651->irq, rt5651_irq,
+ IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING
+- | IRQF_ONESHOT, "rt5651", rt5651);
+- if (ret == 0) {
+- /* Gets re-enabled by rt5651_set_jack() */
+- disable_irq(rt5651->irq);
+- } else {
++ | IRQF_ONESHOT | IRQF_NO_AUTOEN, "rt5651", rt5651);
++ if (ret) {
+ dev_warn(&i2c->dev, "Failed to reguest IRQ %d: %d\n",
+ rt5651->irq, ret);
+ rt5651->irq = -ENXIO;
+diff --git a/sound/soc/codecs/rt5663.c b/sound/soc/codecs/rt5663.c
+index be9fc58ff6812..4aba6e106ee46 100644
+--- a/sound/soc/codecs/rt5663.c
++++ b/sound/soc/codecs/rt5663.c
+@@ -3461,6 +3461,7 @@ static void rt5663_calibrate(struct rt5663_priv *rt5663)
+ static int rt5663_parse_dp(struct rt5663_priv *rt5663, struct device *dev)
+ {
+ int table_size;
++ int ret;
+
+ device_property_read_u32(dev, "realtek,dc_offset_l_manual",
+ &rt5663->pdata.dc_offset_l_manual);
+@@ -3477,9 +3478,13 @@ static int rt5663_parse_dp(struct rt5663_priv *rt5663, struct device *dev)
+ table_size = sizeof(struct impedance_mapping_table) *
+ rt5663->pdata.impedance_sensing_num;
+ rt5663->imp_table = devm_kzalloc(dev, table_size, GFP_KERNEL);
+- device_property_read_u32_array(dev,
++ if (!rt5663->imp_table)
++ return -ENOMEM;
++ ret = device_property_read_u32_array(dev,
+ "realtek,impedance_sensing_table",
+ (u32 *)rt5663->imp_table, table_size);
++ if (ret)
++ return ret;
+ }
+
+ return 0;
+@@ -3504,8 +3509,11 @@ static int rt5663_i2c_probe(struct i2c_client *i2c,
+
+ if (pdata)
+ rt5663->pdata = *pdata;
+- else
+- rt5663_parse_dp(rt5663, &i2c->dev);
++ else {
++ ret = rt5663_parse_dp(rt5663, &i2c->dev);
++ if (ret)
++ return ret;
++ }
+
+ for (i = 0; i < ARRAY_SIZE(rt5663->supplies); i++)
+ rt5663->supplies[i].supply = rt5663_supply_names[i];
+diff --git a/sound/soc/codecs/rt5668.c b/sound/soc/codecs/rt5668.c
+index 6ab1a8bc37352..1186ceb5a978e 100644
+--- a/sound/soc/codecs/rt5668.c
++++ b/sound/soc/codecs/rt5668.c
+@@ -1022,11 +1022,13 @@ static void rt5668_jack_detect_handler(struct work_struct *work)
+ container_of(work, struct rt5668_priv, jack_detect_work.work);
+ int val, btn_type;
+
+- while (!rt5668->component)
+- usleep_range(10000, 15000);
+-
+- while (!rt5668->component->card->instantiated)
+- usleep_range(10000, 15000);
++ if (!rt5668->component || !rt5668->component->card ||
++ !rt5668->component->card->instantiated) {
++ /* card not yet ready, try later */
++ mod_delayed_work(system_power_efficient_wq,
++ &rt5668->jack_detect_work, msecs_to_jiffies(15));
++ return;
++ }
+
+ mutex_lock(&rt5668->calibrate_mutex);
+
+diff --git a/sound/soc/codecs/rt5682.c b/sound/soc/codecs/rt5682.c
+index 4a64cab99c55b..6ad3159eceb98 100644
+--- a/sound/soc/codecs/rt5682.c
++++ b/sound/soc/codecs/rt5682.c
+@@ -46,6 +46,8 @@ static const struct reg_sequence patch_list[] = {
+ {RT5682_SAR_IL_CMD_1, 0x22b7},
+ {RT5682_SAR_IL_CMD_3, 0x0365},
+ {RT5682_SAR_IL_CMD_6, 0x0110},
++ {RT5682_CHARGE_PUMP_1, 0x0210},
++ {RT5682_HP_LOGIC_CTRL_2, 0x0007},
+ };
+
+ void rt5682_apply_patch_list(struct rt5682_priv *rt5682, struct device *dev)
+@@ -925,6 +927,8 @@ int rt5682_headset_detect(struct snd_soc_component *component, int jack_insert)
+ unsigned int val, count;
+
+ if (jack_insert) {
++ snd_soc_dapm_mutex_lock(dapm);
++
+ snd_soc_component_update_bits(component, RT5682_PWR_ANLG_1,
+ RT5682_PWR_VREF2 | RT5682_PWR_MB,
+ RT5682_PWR_VREF2 | RT5682_PWR_MB);
+@@ -971,6 +975,8 @@ int rt5682_headset_detect(struct snd_soc_component *component, int jack_insert)
+ snd_soc_component_update_bits(component, RT5682_MICBIAS_2,
+ RT5682_PWR_CLK25M_MASK | RT5682_PWR_CLK1M_MASK,
+ RT5682_PWR_CLK25M_PU | RT5682_PWR_CLK1M_PU);
++
++ snd_soc_dapm_mutex_unlock(dapm);
+ } else {
+ rt5682_enable_push_button_irq(component, false);
+ snd_soc_component_update_bits(component, RT5682_CBJ_CTRL_1,
+@@ -1084,11 +1090,13 @@ void rt5682_jack_detect_handler(struct work_struct *work)
+ container_of(work, struct rt5682_priv, jack_detect_work.work);
+ int val, btn_type;
+
+- while (!rt5682->component)
+- usleep_range(10000, 15000);
+-
+- while (!rt5682->component->card->instantiated)
+- usleep_range(10000, 15000);
++ if (!rt5682->component || !rt5682->component->card ||
++ !rt5682->component->card->instantiated) {
++ /* card not yet ready, try later */
++ mod_delayed_work(system_power_efficient_wq,
++ &rt5682->jack_detect_work, msecs_to_jiffies(15));
++ return;
++ }
+
+ mutex_lock(&rt5682->calibrate_mutex);
+
+@@ -1515,21 +1523,29 @@ static int rt5682_hp_event(struct snd_soc_dapm_widget *w,
+
+ switch (event) {
+ case SND_SOC_DAPM_PRE_PMU:
+- snd_soc_component_write(component,
+- RT5682_HP_LOGIC_CTRL_2, 0x0012);
+- snd_soc_component_write(component,
+- RT5682_HP_CTRL_2, 0x6000);
++ snd_soc_component_update_bits(component, RT5682_HP_CTRL_2,
++ RT5682_HP_C2_DAC_AMP_MUTE, 0);
++ snd_soc_component_update_bits(component, RT5682_HP_LOGIC_CTRL_2,
++ RT5682_HP_LC2_SIG_SOUR2_MASK, RT5682_HP_LC2_SIG_SOUR2_REG);
+ snd_soc_component_update_bits(component,
+ RT5682_DEPOP_1, 0x60, 0x60);
+ snd_soc_component_update_bits(component,
+ RT5682_DAC_ADC_DIG_VOL1, 0x00c0, 0x0080);
++ snd_soc_component_update_bits(component, RT5682_HP_CTRL_2,
++ RT5682_HP_C2_DAC_L_EN | RT5682_HP_C2_DAC_R_EN,
++ RT5682_HP_C2_DAC_L_EN | RT5682_HP_C2_DAC_R_EN);
++ usleep_range(5000, 10000);
++ snd_soc_component_update_bits(component, RT5682_CHARGE_PUMP_1,
++ RT5682_CP_SW_SIZE_MASK, RT5682_CP_SW_SIZE_L);
+ break;
+
+ case SND_SOC_DAPM_POST_PMD:
++ snd_soc_component_update_bits(component, RT5682_HP_CTRL_2,
++ RT5682_HP_C2_DAC_L_EN | RT5682_HP_C2_DAC_R_EN, 0);
++ snd_soc_component_update_bits(component, RT5682_CHARGE_PUMP_1,
++ RT5682_CP_SW_SIZE_MASK, RT5682_CP_SW_SIZE_M);
+ snd_soc_component_update_bits(component,
+ RT5682_DEPOP_1, 0x60, 0x0);
+- snd_soc_component_write(component,
+- RT5682_HP_CTRL_2, 0x0000);
+ snd_soc_component_update_bits(component,
+ RT5682_DAC_ADC_DIG_VOL1, 0x00c0, 0x0000);
+ break;
+@@ -1637,6 +1653,23 @@ static SOC_VALUE_ENUM_SINGLE_DECL(rt5682_adcdat_pin_enum,
+ static const struct snd_kcontrol_new rt5682_adcdat_pin_ctrl =
+ SOC_DAPM_ENUM("ADCDAT", rt5682_adcdat_pin_enum);
+
++static const unsigned int rt5682_hpo_sig_out_values[] = {
++ 2,
++ 7,
++};
++
++static const char * const rt5682_hpo_sig_out_mode[] = {
++ "Legacy",
++ "OneBit",
++};
++
++static SOC_VALUE_ENUM_SINGLE_DECL(rt5682_hpo_sig_out_enum,
++ RT5682_HP_LOGIC_CTRL_2, 0, RT5682_HP_LC2_SIG_SOUR1_MASK,
++ rt5682_hpo_sig_out_mode, rt5682_hpo_sig_out_values);
++
++static const struct snd_kcontrol_new rt5682_hpo_sig_demux =
++ SOC_DAPM_ENUM("HPO Signal Demux", rt5682_hpo_sig_out_enum);
++
+ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = {
+ SND_SOC_DAPM_SUPPLY("LDO2", RT5682_PWR_ANLG_3, RT5682_PWR_LDO2_BIT,
+ 0, NULL, 0),
+@@ -1820,6 +1853,10 @@ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = {
+ SND_SOC_DAPM_SWITCH("HPOR Playback", SND_SOC_NOPM, 0, 0,
+ &hpor_switch),
+
++ SND_SOC_DAPM_OUT_DRV("HPO Legacy", SND_SOC_NOPM, 0, 0, NULL, 0),
++ SND_SOC_DAPM_OUT_DRV("HPO OneBit", SND_SOC_NOPM, 0, 0, NULL, 0),
++ SND_SOC_DAPM_DEMUX("HPO Signal Demux", SND_SOC_NOPM, 0, 0, &rt5682_hpo_sig_demux),
++
+ /* CLK DET */
+ SND_SOC_DAPM_SUPPLY("CLKDET SYS", RT5682_CLK_DET,
+ RT5682_SYS_CLK_DET_SFT, 0, NULL, 0),
+@@ -1987,10 +2024,19 @@ static const struct snd_soc_dapm_route rt5682_dapm_routes[] = {
+ {"HP Amp", NULL, "Charge Pump"},
+ {"HP Amp", NULL, "CLKDET SYS"},
+ {"HP Amp", NULL, "Vref1"},
+- {"HPOL Playback", "Switch", "HP Amp"},
+- {"HPOR Playback", "Switch", "HP Amp"},
++
++ {"HPO Signal Demux", NULL, "HP Amp"},
++
++ {"HPO Legacy", "Legacy", "HPO Signal Demux"},
++ {"HPO OneBit", "OneBit", "HPO Signal Demux"},
++
++ {"HPOL Playback", "Switch", "HPO Legacy"},
++ {"HPOR Playback", "Switch", "HPO Legacy"},
++
+ {"HPOL", NULL, "HPOL Playback"},
+ {"HPOR", NULL, "HPOR Playback"},
++ {"HPOL", NULL, "HPO OneBit"},
++ {"HPOR", NULL, "HPO OneBit"},
+ };
+
+ static int rt5682_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+@@ -2800,6 +2846,8 @@ static int rt5682_register_dai_clks(struct snd_soc_component *component)
+
+ for (i = 0; i < RT5682_DAI_NUM_CLKS; ++i) {
+ struct clk_init_data init = { };
++ struct clk_parent_data parent_data;
++ const struct clk_hw *parent;
+
+ dai_clk_hw = &rt5682->dai_clks_hw[i];
+
+@@ -2807,17 +2855,17 @@ static int rt5682_register_dai_clks(struct snd_soc_component *component)
+ case RT5682_DAI_WCLK_IDX:
+ /* Make MCLK the parent of WCLK */
+ if (rt5682->mclk) {
+- init.parent_data = &(struct clk_parent_data){
++ parent_data = (struct clk_parent_data){
+ .fw_name = "mclk",
+ };
++ init.parent_data = &parent_data;
+ init.num_parents = 1;
+ }
+ break;
+ case RT5682_DAI_BCLK_IDX:
+ /* Make WCLK the parent of BCLK */
+- init.parent_hws = &(const struct clk_hw *){
+- &rt5682->dai_clks_hw[RT5682_DAI_WCLK_IDX]
+- };
++ parent = &rt5682->dai_clks_hw[RT5682_DAI_WCLK_IDX];
++ init.parent_hws = &parent;
+ init.num_parents = 1;
+ break;
+ default:
+diff --git a/sound/soc/codecs/rt5682.h b/sound/soc/codecs/rt5682.h
+index b59221048ebf9..8e3244a62c160 100644
+--- a/sound/soc/codecs/rt5682.h
++++ b/sound/soc/codecs/rt5682.h
+@@ -375,6 +375,14 @@
+ #define RT5682_R_VOL_MASK (0x3f)
+ #define RT5682_R_VOL_SFT 0
+
++/* Headphone Amp Control 2 (0x0003) */
++#define RT5682_HP_C2_DAC_AMP_MUTE_SFT 15
++#define RT5682_HP_C2_DAC_AMP_MUTE (0x1 << 15)
++#define RT5682_HP_C2_DAC_L_EN_SFT 14
++#define RT5682_HP_C2_DAC_L_EN (0x1 << 14)
++#define RT5682_HP_C2_DAC_R_EN_SFT 13
++#define RT5682_HP_C2_DAC_R_EN (0x1 << 13)
++
+ /*Headphone Amp L/R Analog Gain and Digital NG2 Gain Control (0x0005 0x0006)*/
+ #define RT5682_G_HP (0xf << 8)
+ #define RT5682_G_HP_SFT 8
+@@ -1265,6 +1273,10 @@
+ #define RT5682_HPA_CP_BIAS_6UA (0x3 << 2)
+
+ /* Charge Pump Internal Register1 (0x0125) */
++#define RT5682_CP_SW_SIZE_MASK (0x7 << 8)
++#define RT5682_CP_SW_SIZE_L (0x4 << 8)
++#define RT5682_CP_SW_SIZE_M (0x2 << 8)
++#define RT5682_CP_SW_SIZE_S (0x1 << 8)
+ #define RT5682_CP_CLK_HP_MASK (0x3 << 4)
+ #define RT5682_CP_CLK_HP_100KHZ (0x0 << 4)
+ #define RT5682_CP_CLK_HP_200KHZ (0x1 << 4)
+@@ -1315,6 +1327,14 @@
+ #define RT5682_DEB_STO_DAC_MASK (0x7 << 4)
+ #define RT5682_DEB_80_MS (0x0 << 4)
+
++/* HP Behavior Logic Control 2 (0x01db) */
++#define RT5682_HP_LC2_SIG_SOUR2_MASK (0x1 << 4)
++#define RT5682_HP_LC2_SIG_SOUR2_REG (0x1 << 4)
++#define RT5682_HP_LC2_SIG_SOUR2_DC_CAL (0x0 << 4)
++#define RT5682_HP_LC2_SIG_SOUR1_MASK (0x7)
++#define RT5682_HP_LC2_SIG_SOUR1_1BIT (0x7)
++#define RT5682_HP_LC2_SIG_SOUR1_LEGA (0x2)
++
+ /* SAR ADC Inline Command Control 1 (0x0210) */
+ #define RT5682_SAR_BUTT_DET_MASK (0x1 << 15)
+ #define RT5682_SAR_BUTT_DET_EN (0x1 << 15)
+diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c
+index 172e79cbe0daf..c5ea3b115966b 100644
+--- a/sound/soc/codecs/tas2770.c
++++ b/sound/soc/codecs/tas2770.c
+@@ -38,10 +38,12 @@ static void tas2770_reset(struct tas2770_priv *tas2770)
+ gpiod_set_value_cansleep(tas2770->reset_gpio, 0);
+ msleep(20);
+ gpiod_set_value_cansleep(tas2770->reset_gpio, 1);
++ usleep_range(1000, 2000);
+ }
+
+ snd_soc_component_write(tas2770->component, TAS2770_SW_RST,
+ TAS2770_RST);
++ usleep_range(1000, 2000);
+ }
+
+ static int tas2770_set_bias_level(struct snd_soc_component *component,
+@@ -110,6 +112,7 @@ static int tas2770_codec_resume(struct snd_soc_component *component)
+
+ if (tas2770->sdz_gpio) {
+ gpiod_set_value_cansleep(tas2770->sdz_gpio, 1);
++ usleep_range(1000, 2000);
+ } else {
+ ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL,
+ TAS2770_PWR_CTRL_MASK,
+@@ -291,11 +294,11 @@ static int tas2770_set_samplerate(struct tas2770_priv *tas2770, int samplerate)
+ ramp_rate_val = TAS2770_TDM_CFG_REG0_SMP_44_1KHZ |
+ TAS2770_TDM_CFG_REG0_31_88_2_96KHZ;
+ break;
+- case 19200:
++ case 192000:
+ ramp_rate_val = TAS2770_TDM_CFG_REG0_SMP_48KHZ |
+ TAS2770_TDM_CFG_REG0_31_176_4_192KHZ;
+ break;
+- case 17640:
++ case 176400:
+ ramp_rate_val = TAS2770_TDM_CFG_REG0_SMP_44_1KHZ |
+ TAS2770_TDM_CFG_REG0_31_176_4_192KHZ;
+ break;
+@@ -510,8 +513,10 @@ static int tas2770_codec_probe(struct snd_soc_component *component)
+
+ tas2770->component = component;
+
+- if (tas2770->sdz_gpio)
++ if (tas2770->sdz_gpio) {
+ gpiod_set_value_cansleep(tas2770->sdz_gpio, 1);
++ usleep_range(1000, 2000);
++ }
+
+ tas2770_reset(tas2770);
+
+diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c
+index d885ced34f606..bc5d68c53e5ab 100644
+--- a/sound/soc/codecs/wcd9335.c
++++ b/sound/soc/codecs/wcd9335.c
+@@ -4859,7 +4859,7 @@ static int wcd9335_codec_probe(struct snd_soc_component *component)
+
+ snd_soc_component_init_regmap(component, wcd->regmap);
+ /* Class-H Init*/
+- wcd->clsh_ctrl = wcd_clsh_ctrl_alloc(component, wcd->version);
++ wcd->clsh_ctrl = wcd_clsh_ctrl_alloc(component, WCD9335);
+ if (IS_ERR(wcd->clsh_ctrl))
+ return PTR_ERR(wcd->clsh_ctrl);
+
+diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c
+index c496b359f2f40..144046864d157 100644
+--- a/sound/soc/codecs/wcd934x.c
++++ b/sound/soc/codecs/wcd934x.c
+@@ -1274,29 +1274,7 @@ static int wcd934x_set_sido_input_src(struct wcd934x_codec *wcd, int sido_src)
+ if (sido_src == wcd->sido_input_src)
+ return 0;
+
+- if (sido_src == SIDO_SOURCE_INTERNAL) {
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL,
+- WCD934X_ANA_BUCK_HI_ACCU_EN_MASK, 0);
+- usleep_range(100, 110);
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL,
+- WCD934X_ANA_BUCK_HI_ACCU_PRE_ENX_MASK, 0x0);
+- usleep_range(100, 110);
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_RCO,
+- WCD934X_ANA_RCO_BG_EN_MASK, 0);
+- usleep_range(100, 110);
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL,
+- WCD934X_ANA_BUCK_PRE_EN1_MASK,
+- WCD934X_ANA_BUCK_PRE_EN1_ENABLE);
+- usleep_range(100, 110);
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL,
+- WCD934X_ANA_BUCK_PRE_EN2_MASK,
+- WCD934X_ANA_BUCK_PRE_EN2_ENABLE);
+- usleep_range(100, 110);
+- regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL,
+- WCD934X_ANA_BUCK_HI_ACCU_EN_MASK,
+- WCD934X_ANA_BUCK_HI_ACCU_ENABLE);
+- usleep_range(100, 110);
+- } else if (sido_src == SIDO_SOURCE_RCO_BG) {
++ if (sido_src == SIDO_SOURCE_RCO_BG) {
+ regmap_update_bits(wcd->regmap, WCD934X_ANA_RCO,
+ WCD934X_ANA_RCO_BG_EN_MASK,
+ WCD934X_ANA_RCO_BG_ENABLE);
+@@ -1382,8 +1360,6 @@ static int wcd934x_disable_ana_bias_and_syclk(struct wcd934x_codec *wcd)
+ regmap_update_bits(wcd->regmap, WCD934X_CLK_SYS_MCLK_PRG,
+ WCD934X_EXT_CLK_BUF_EN_MASK |
+ WCD934X_MCLK_EN_MASK, 0x0);
+- wcd934x_set_sido_input_src(wcd, SIDO_SOURCE_INTERNAL);
+-
+ regmap_update_bits(wcd->regmap, WCD934X_ANA_BIAS,
+ WCD934X_ANA_BIAS_EN_MASK, 0);
+ regmap_update_bits(wcd->regmap, WCD934X_ANA_BIAS,
+@@ -1896,9 +1872,8 @@ static int wcd934x_hw_params(struct snd_pcm_substream *substream,
+ }
+
+ wcd->dai[dai->id].sconfig.rate = params_rate(params);
+- wcd934x_slim_set_hw_params(wcd, &wcd->dai[dai->id], substream->stream);
+
+- return 0;
++ return wcd934x_slim_set_hw_params(wcd, &wcd->dai[dai->id], substream->stream);
+ }
+
+ static int wcd934x_hw_free(struct snd_pcm_substream *substream,
+@@ -3024,14 +2999,14 @@ static int wcd934x_hph_impedance_get(struct snd_kcontrol *kcontrol,
+ return 0;
+ }
+ static const struct snd_kcontrol_new hph_type_detect_controls[] = {
+- SOC_SINGLE_EXT("HPH Type", 0, 0, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPH Type", 0, 0, WCD_MBHC_HPH_STEREO, 0,
+ wcd934x_get_hph_type, NULL),
+ };
+
+ static const struct snd_kcontrol_new impedance_detect_controls[] = {
+- SOC_SINGLE_EXT("HPHL Impedance", 0, 0, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPHL Impedance", 0, 0, INT_MAX, 0,
+ wcd934x_hph_impedance_get, NULL),
+- SOC_SINGLE_EXT("HPHR Impedance", 0, 1, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPHR Impedance", 0, 1, INT_MAX, 0,
+ wcd934x_hph_impedance_get, NULL),
+ };
+
+@@ -3257,6 +3232,9 @@ static int wcd934x_compander_set(struct snd_kcontrol *kc,
+ int value = ucontrol->value.integer.value[0];
+ int sel;
+
++ if (wcd->comp_enabled[comp] == value)
++ return 0;
++
+ wcd->comp_enabled[comp] = value;
+ sel = value ? WCD934X_HPH_GAIN_SRC_SEL_COMPANDER :
+ WCD934X_HPH_GAIN_SRC_SEL_REGISTER;
+@@ -3280,10 +3258,10 @@ static int wcd934x_compander_set(struct snd_kcontrol *kc,
+ case COMPANDER_8:
+ break;
+ default:
+- break;
++ return 0;
+ }
+
+- return 0;
++ return 1;
+ }
+
+ static int wcd934x_rx_hph_mode_get(struct snd_kcontrol *kc,
+@@ -3306,13 +3284,16 @@ static int wcd934x_rx_hph_mode_put(struct snd_kcontrol *kc,
+
+ mode_val = ucontrol->value.enumerated.item[0];
+
++ if (mode_val == wcd->hph_mode)
++ return 0;
++
+ if (mode_val == 0) {
+ dev_err(wcd->dev, "Invalid HPH Mode, default to ClSH HiFi\n");
+ mode_val = CLS_H_LOHIFI;
+ }
+ wcd->hph_mode = mode_val;
+
+- return 0;
++ return 1;
+ }
+
+ static int slim_rx_mux_get(struct snd_kcontrol *kc,
+@@ -3327,6 +3308,31 @@ static int slim_rx_mux_get(struct snd_kcontrol *kc,
+ return 0;
+ }
+
++static int slim_rx_mux_to_dai_id(int mux)
++{
++ int aif_id;
++
++ switch (mux) {
++ case 1:
++ aif_id = AIF1_PB;
++ break;
++ case 2:
++ aif_id = AIF2_PB;
++ break;
++ case 3:
++ aif_id = AIF3_PB;
++ break;
++ case 4:
++ aif_id = AIF4_PB;
++ break;
++ default:
++ aif_id = -1;
++ break;
++ }
++
++ return aif_id;
++}
++
+ static int slim_rx_mux_put(struct snd_kcontrol *kc,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+@@ -3334,43 +3340,59 @@ static int slim_rx_mux_put(struct snd_kcontrol *kc,
+ struct wcd934x_codec *wcd = dev_get_drvdata(w->dapm->dev);
+ struct soc_enum *e = (struct soc_enum *)kc->private_value;
+ struct snd_soc_dapm_update *update = NULL;
++ struct wcd934x_slim_ch *ch, *c;
+ u32 port_id = w->shift;
++ bool found = false;
++ int mux_idx;
++ int prev_mux_idx = wcd->rx_port_value[port_id];
++ int aif_id;
+
+- if (wcd->rx_port_value[port_id] == ucontrol->value.enumerated.item[0])
+- return 0;
++ mux_idx = ucontrol->value.enumerated.item[0];
+
+- wcd->rx_port_value[port_id] = ucontrol->value.enumerated.item[0];
++ if (mux_idx == prev_mux_idx)
++ return 0;
+
+- switch (wcd->rx_port_value[port_id]) {
++ switch(mux_idx) {
+ case 0:
+- list_del_init(&wcd->rx_chs[port_id].list);
+- break;
+- case 1:
+- list_add_tail(&wcd->rx_chs[port_id].list,
+- &wcd->dai[AIF1_PB].slim_ch_list);
+- break;
+- case 2:
+- list_add_tail(&wcd->rx_chs[port_id].list,
+- &wcd->dai[AIF2_PB].slim_ch_list);
+- break;
+- case 3:
+- list_add_tail(&wcd->rx_chs[port_id].list,
+- &wcd->dai[AIF3_PB].slim_ch_list);
++ aif_id = slim_rx_mux_to_dai_id(prev_mux_idx);
++ if (aif_id < 0)
++ return 0;
++
++ list_for_each_entry_safe(ch, c, &wcd->dai[aif_id].slim_ch_list, list) {
++ if (ch->port == port_id + WCD934X_RX_START) {
++ found = true;
++ list_del_init(&ch->list);
++ break;
++ }
++ }
++ if (!found)
++ return 0;
++
+ break;
+- case 4:
+- list_add_tail(&wcd->rx_chs[port_id].list,
+- &wcd->dai[AIF4_PB].slim_ch_list);
++ case 1 ... 4:
++ aif_id = slim_rx_mux_to_dai_id(mux_idx);
++ if (aif_id < 0)
++ return 0;
++
++ if (list_empty(&wcd->rx_chs[port_id].list)) {
++ list_add_tail(&wcd->rx_chs[port_id].list,
++ &wcd->dai[aif_id].slim_ch_list);
++ } else {
++ dev_err(wcd->dev ,"SLIM_RX%d PORT is busy\n", port_id);
++ return 0;
++ }
+ break;
++
+ default:
+- dev_err(wcd->dev, "Unknown AIF %d\n",
+- wcd->rx_port_value[port_id]);
++ dev_err(wcd->dev, "Unknown AIF %d\n", mux_idx);
+ goto err;
+ }
+
++ wcd->rx_port_value[port_id] = mux_idx;
+ snd_soc_dapm_mux_update_power(w->dapm, kc, wcd->rx_port_value[port_id],
+ e, update);
+
+- return 0;
++ return 1;
+ err:
+ return -EINVAL;
+ }
+@@ -3816,6 +3838,7 @@ static int slim_tx_mixer_put(struct snd_kcontrol *kc,
+ struct soc_mixer_control *mixer =
+ (struct soc_mixer_control *)kc->private_value;
+ int enable = ucontrol->value.integer.value[0];
++ struct wcd934x_slim_ch *ch, *c;
+ int dai_id = widget->shift;
+ int port_id = mixer->shift;
+
+@@ -3823,17 +3846,32 @@ static int slim_tx_mixer_put(struct snd_kcontrol *kc,
+ if (enable == wcd->tx_port_value[port_id])
+ return 0;
+
+- wcd->tx_port_value[port_id] = enable;
+-
+- if (enable)
+- list_add_tail(&wcd->tx_chs[port_id].list,
+- &wcd->dai[dai_id].slim_ch_list);
+- else
+- list_del_init(&wcd->tx_chs[port_id].list);
++ if (enable) {
++ if (list_empty(&wcd->tx_chs[port_id].list)) {
++ list_add_tail(&wcd->tx_chs[port_id].list,
++ &wcd->dai[dai_id].slim_ch_list);
++ } else {
++ dev_err(wcd->dev ,"SLIM_TX%d PORT is busy\n", port_id);
++ return 0;
++ }
++ } else {
++ bool found = false;
++
++ list_for_each_entry_safe(ch, c, &wcd->dai[dai_id].slim_ch_list, list) {
++ if (ch->port == port_id) {
++ found = true;
++ list_del_init(&wcd->tx_chs[port_id].list);
++ break;
++ }
++ }
++ if (!found)
++ return 0;
++ }
+
++ wcd->tx_port_value[port_id] = enable;
+ snd_soc_dapm_mixer_update_power(widget->dapm, kc, enable, update);
+
+- return 0;
++ return 1;
+ }
+
+ static const struct snd_kcontrol_new aif1_slim_cap_mixer[] = {
+@@ -5826,6 +5864,7 @@ static int wcd934x_codec_parse_data(struct wcd934x_codec *wcd)
+ }
+
+ wcd->sidev = of_slim_get_device(wcd->sdev->ctrl, ifc_dev_np);
++ of_node_put(ifc_dev_np);
+ if (!wcd->sidev) {
+ dev_err(dev, "Unable to get SLIM Interface device\n");
+ return -EINVAL;
+diff --git a/sound/soc/codecs/wcd938x.c b/sound/soc/codecs/wcd938x.c
+index 52de7d14b1398..4480c118ed5d9 100644
+--- a/sound/soc/codecs/wcd938x.c
++++ b/sound/soc/codecs/wcd938x.c
+@@ -1174,6 +1174,9 @@ static bool wcd938x_readonly_register(struct device *dev, unsigned int reg)
+ case WCD938X_DIGITAL_INTR_STATUS_0:
+ case WCD938X_DIGITAL_INTR_STATUS_1:
+ case WCD938X_DIGITAL_INTR_STATUS_2:
++ case WCD938X_DIGITAL_INTR_CLEAR_0:
++ case WCD938X_DIGITAL_INTR_CLEAR_1:
++ case WCD938X_DIGITAL_INTR_CLEAR_2:
+ case WCD938X_DIGITAL_SWR_HM_TEST_0:
+ case WCD938X_DIGITAL_SWR_HM_TEST_1:
+ case WCD938X_DIGITAL_EFUSE_T_DATA_0:
+@@ -1429,14 +1432,10 @@ static int wcd938x_sdw_connect_port(struct wcd938x_sdw_ch_info *ch_info,
+ return 0;
+ }
+
+-static int wcd938x_connect_port(struct wcd938x_sdw_priv *wcd, u8 ch_id, u8 enable)
++static int wcd938x_connect_port(struct wcd938x_sdw_priv *wcd, u8 port_num, u8 ch_id, u8 enable)
+ {
+- u8 port_num;
+-
+- port_num = wcd->ch_info[ch_id].port_num;
+-
+ return wcd938x_sdw_connect_port(&wcd->ch_info[ch_id],
+- &wcd->port_config[port_num],
++ &wcd->port_config[port_num - 1],
+ enable);
+ }
+
+@@ -2505,7 +2504,7 @@ static int wcd938x_tx_mode_get(struct snd_kcontrol *kcontrol,
+ struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+ int path = e->shift_l;
+
+- ucontrol->value.integer.value[0] = wcd938x->tx_mode[path];
++ ucontrol->value.enumerated.item[0] = wcd938x->tx_mode[path];
+
+ return 0;
+ }
+@@ -2529,7 +2528,7 @@ static int wcd938x_rx_hph_mode_get(struct snd_kcontrol *kcontrol,
+ struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+ struct wcd938x_priv *wcd938x = snd_soc_component_get_drvdata(component);
+
+- ucontrol->value.integer.value[0] = wcd938x->hph_mode;
++ ucontrol->value.enumerated.item[0] = wcd938x->hph_mode;
+
+ return 0;
+ }
+@@ -2560,7 +2559,7 @@ static int wcd938x_ear_pa_put_gain(struct snd_kcontrol *kcontrol,
+ WCD938X_EAR_GAIN_MASK,
+ ucontrol->value.integer.value[0]);
+
+- return 0;
++ return 1;
+ }
+
+ static int wcd938x_get_compander(struct snd_kcontrol *kcontrol,
+@@ -2590,6 +2589,7 @@ static int wcd938x_set_compander(struct snd_kcontrol *kcontrol,
+ struct wcd938x_priv *wcd938x = snd_soc_component_get_drvdata(component);
+ struct wcd938x_sdw_priv *wcd;
+ int value = ucontrol->value.integer.value[0];
++ int portidx;
+ struct soc_mixer_control *mc;
+ bool hphr;
+
+@@ -2603,12 +2603,14 @@ static int wcd938x_set_compander(struct snd_kcontrol *kcontrol,
+ else
+ wcd938x->comp1_enable = value;
+
++ portidx = wcd->ch_info[mc->reg].port_num;
++
+ if (value)
+- wcd938x_connect_port(wcd, mc->reg, true);
++ wcd938x_connect_port(wcd, portidx, mc->reg, true);
+ else
+- wcd938x_connect_port(wcd, mc->reg, false);
++ wcd938x_connect_port(wcd, portidx, mc->reg, false);
+
+- return 0;
++ return 1;
+ }
+
+ static int wcd938x_ldoh_get(struct snd_kcontrol *kcontrol,
+@@ -2879,9 +2881,11 @@ static int wcd938x_get_swr_port(struct snd_kcontrol *kcontrol,
+ struct wcd938x_sdw_priv *wcd;
+ struct soc_mixer_control *mixer = (struct soc_mixer_control *)kcontrol->private_value;
+ int dai_id = mixer->shift;
+- int portidx = mixer->reg;
++ int portidx, ch_idx = mixer->reg;
++
+
+ wcd = wcd938x->sdw_priv[dai_id];
++ portidx = wcd->ch_info[ch_idx].port_num;
+
+ ucontrol->value.integer.value[0] = wcd->port_enable[portidx];
+
+@@ -2896,12 +2900,14 @@ static int wcd938x_set_swr_port(struct snd_kcontrol *kcontrol,
+ struct wcd938x_sdw_priv *wcd;
+ struct soc_mixer_control *mixer =
+ (struct soc_mixer_control *)kcontrol->private_value;
+- int portidx = mixer->reg;
++ int ch_idx = mixer->reg;
++ int portidx;
+ int dai_id = mixer->shift;
+ bool enable;
+
+ wcd = wcd938x->sdw_priv[dai_id];
+
++ portidx = wcd->ch_info[ch_idx].port_num;
+ if (ucontrol->value.integer.value[0])
+ enable = true;
+ else
+@@ -2909,9 +2915,9 @@ static int wcd938x_set_swr_port(struct snd_kcontrol *kcontrol,
+
+ wcd->port_enable[portidx] = enable;
+
+- wcd938x_connect_port(wcd, portidx, enable);
++ wcd938x_connect_port(wcd, portidx, ch_idx, enable);
+
+- return 0;
++ return 1;
+
+ }
+
+@@ -3571,14 +3577,14 @@ static int wcd938x_hph_impedance_get(struct snd_kcontrol *kcontrol,
+ }
+
+ static const struct snd_kcontrol_new hph_type_detect_controls[] = {
+- SOC_SINGLE_EXT("HPH Type", 0, 0, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPH Type", 0, 0, WCD_MBHC_HPH_STEREO, 0,
+ wcd938x_get_hph_type, NULL),
+ };
+
+ static const struct snd_kcontrol_new impedance_detect_controls[] = {
+- SOC_SINGLE_EXT("HPHL Impedance", 0, 0, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPHL Impedance", 0, 0, INT_MAX, 0,
+ wcd938x_hph_impedance_get, NULL),
+- SOC_SINGLE_EXT("HPHR Impedance", 0, 1, UINT_MAX, 0,
++ SOC_SINGLE_EXT("HPHR Impedance", 0, 1, INT_MAX, 0,
+ wcd938x_hph_impedance_get, NULL),
+ };
+
+diff --git a/sound/soc/codecs/wm8350.c b/sound/soc/codecs/wm8350.c
+index 15d42ce3b21d6..41504ce2a682f 100644
+--- a/sound/soc/codecs/wm8350.c
++++ b/sound/soc/codecs/wm8350.c
+@@ -1537,18 +1537,38 @@ static int wm8350_component_probe(struct snd_soc_component *component)
+ wm8350_clear_bits(wm8350, WM8350_JACK_DETECT,
+ WM8350_JDL_ENA | WM8350_JDR_ENA);
+
+- wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_L,
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_L,
+ wm8350_hpl_jack_handler, 0, "Left jack detect",
+ priv);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_R,
++ if (ret != 0)
++ goto err;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_R,
+ wm8350_hpr_jack_handler, 0, "Right jack detect",
+ priv);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_MICSCD,
++ if (ret != 0)
++ goto free_jck_det_l;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_MICSCD,
+ wm8350_mic_handler, 0, "Microphone short", priv);
+- wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_MICD,
++ if (ret != 0)
++ goto free_jck_det_r;
++
++ ret = wm8350_register_irq(wm8350, WM8350_IRQ_CODEC_MICD,
+ wm8350_mic_handler, 0, "Microphone detect", priv);
++ if (ret != 0)
++ goto free_micscd;
+
+ return 0;
++
++free_micscd:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CODEC_MICSCD, priv);
++free_jck_det_r:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_R, priv);
++free_jck_det_l:
++ wm8350_free_irq(wm8350, WM8350_IRQ_CODEC_JCK_DET_L, priv);
++err:
++ return ret;
+ }
+
+ static void wm8350_component_remove(struct snd_soc_component *component)
+diff --git a/sound/soc/codecs/wm8731.c b/sound/soc/codecs/wm8731.c
+index dcee7b2bd3d79..859ebcec83838 100644
+--- a/sound/soc/codecs/wm8731.c
++++ b/sound/soc/codecs/wm8731.c
+@@ -602,7 +602,7 @@ static int wm8731_hw_init(struct device *dev, struct wm8731_priv *wm8731)
+ ret = wm8731_reset(wm8731->regmap);
+ if (ret < 0) {
+ dev_err(dev, "Failed to issue reset: %d\n", ret);
+- goto err_regulator_enable;
++ goto err;
+ }
+
+ /* Clear POWEROFF, keep everything else disabled */
+@@ -619,10 +619,7 @@ static int wm8731_hw_init(struct device *dev, struct wm8731_priv *wm8731)
+
+ regcache_mark_dirty(wm8731->regmap);
+
+-err_regulator_enable:
+- /* Regulators will be enabled by bias management */
+- regulator_bulk_disable(ARRAY_SIZE(wm8731->supplies), wm8731->supplies);
+-
++err:
+ return ret;
+ }
+
+@@ -766,21 +763,27 @@ static int wm8731_i2c_probe(struct i2c_client *i2c,
+ ret = PTR_ERR(wm8731->regmap);
+ dev_err(&i2c->dev, "Failed to allocate register map: %d\n",
+ ret);
+- return ret;
++ goto err_regulator_enable;
+ }
+
+ ret = wm8731_hw_init(&i2c->dev, wm8731);
+ if (ret != 0)
+- return ret;
++ goto err_regulator_enable;
+
+ ret = devm_snd_soc_register_component(&i2c->dev,
+ &soc_component_dev_wm8731, &wm8731_dai, 1);
+ if (ret != 0) {
+ dev_err(&i2c->dev, "Failed to register CODEC: %d\n", ret);
+- return ret;
++ goto err_regulator_enable;
+ }
+
+ return 0;
++
++err_regulator_enable:
++ /* Regulators will be enabled by bias management */
++ regulator_bulk_disable(ARRAY_SIZE(wm8731->supplies), wm8731->supplies);
++
++ return ret;
+ }
+
+ static int wm8731_i2c_remove(struct i2c_client *client)
+diff --git a/sound/soc/codecs/wm8958-dsp2.c b/sound/soc/codecs/wm8958-dsp2.c
+index e4018ba3b19a2..7878c7a58ff10 100644
+--- a/sound/soc/codecs/wm8958-dsp2.c
++++ b/sound/soc/codecs/wm8958-dsp2.c
+@@ -530,7 +530,7 @@ static int wm8958_mbc_put(struct snd_kcontrol *kcontrol,
+
+ wm8958_dsp_apply(component, mbc, wm8994->mbc_ena[mbc]);
+
+- return 0;
++ return 1;
+ }
+
+ #define WM8958_MBC_SWITCH(xname, xval) {\
+@@ -656,7 +656,7 @@ static int wm8958_vss_put(struct snd_kcontrol *kcontrol,
+
+ wm8958_dsp_apply(component, vss, wm8994->vss_ena[vss]);
+
+- return 0;
++ return 1;
+ }
+
+
+@@ -730,7 +730,7 @@ static int wm8958_hpf_put(struct snd_kcontrol *kcontrol,
+
+ wm8958_dsp_apply(component, hpf % 3, ucontrol->value.integer.value[0]);
+
+- return 0;
++ return 1;
+ }
+
+ #define WM8958_HPF_SWITCH(xname, xval) {\
+@@ -824,7 +824,7 @@ static int wm8958_enh_eq_put(struct snd_kcontrol *kcontrol,
+
+ wm8958_dsp_apply(component, eq, ucontrol->value.integer.value[0]);
+
+- return 0;
++ return 1;
+ }
+
+ #define WM8958_ENH_EQ_SWITCH(xname, xval) {\
+diff --git a/sound/soc/codecs/wsa881x.c b/sound/soc/codecs/wsa881x.c
+index 2da4a5fa7a18d..564b78f3cdd0a 100644
+--- a/sound/soc/codecs/wsa881x.c
++++ b/sound/soc/codecs/wsa881x.c
+@@ -772,7 +772,8 @@ static int wsa881x_put_pa_gain(struct snd_kcontrol *kc,
+
+ usleep_range(1000, 1010);
+ }
+- return 0;
++
++ return 1;
+ }
+
+ static int wsa881x_get_port(struct snd_kcontrol *kcontrol,
+@@ -816,15 +817,22 @@ static int wsa881x_set_port(struct snd_kcontrol *kcontrol,
+ (struct soc_mixer_control *)kcontrol->private_value;
+ int portidx = mixer->reg;
+
+- if (ucontrol->value.integer.value[0])
++ if (ucontrol->value.integer.value[0]) {
++ if (data->port_enable[portidx])
++ return 0;
++
+ data->port_enable[portidx] = true;
+- else
++ } else {
++ if (!data->port_enable[portidx])
++ return 0;
++
+ data->port_enable[portidx] = false;
++ }
+
+ if (portidx == WSA881X_PORT_BOOST) /* Boost Switch */
+ wsa881x_boost_ctrl(comp, data->port_enable[portidx]);
+
+- return 0;
++ return 1;
+ }
+
+ static const char * const smart_boost_lvl_text[] = {
+diff --git a/sound/soc/dwc/dwc-i2s.c b/sound/soc/dwc/dwc-i2s.c
+index 33ce257ae1986..315ca5c4b057a 100644
+--- a/sound/soc/dwc/dwc-i2s.c
++++ b/sound/soc/dwc/dwc-i2s.c
+@@ -403,9 +403,13 @@ static int dw_i2s_runtime_suspend(struct device *dev)
+ static int dw_i2s_runtime_resume(struct device *dev)
+ {
+ struct dw_i2s_dev *dw_dev = dev_get_drvdata(dev);
++ int ret;
+
+- if (dw_dev->capability & DW_I2S_MASTER)
+- clk_enable(dw_dev->clk);
++ if (dw_dev->capability & DW_I2S_MASTER) {
++ ret = clk_enable(dw_dev->clk);
++ if (ret)
++ return ret;
++ }
+ return 0;
+ }
+
+@@ -422,10 +426,13 @@ static int dw_i2s_resume(struct snd_soc_component *component)
+ {
+ struct dw_i2s_dev *dev = snd_soc_component_get_drvdata(component);
+ struct snd_soc_dai *dai;
+- int stream;
++ int stream, ret;
+
+- if (dev->capability & DW_I2S_MASTER)
+- clk_enable(dev->clk);
++ if (dev->capability & DW_I2S_MASTER) {
++ ret = clk_enable(dev->clk);
++ if (ret)
++ return ret;
++ }
+
+ for_each_component_dais(component, dai) {
+ for_each_pcm_streams(stream)
+diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c
+index 24b41881a68f8..d7d1536a4f377 100644
+--- a/sound/soc/fsl/fsl_asrc.c
++++ b/sound/soc/fsl/fsl_asrc.c
+@@ -19,6 +19,7 @@
+ #include "fsl_asrc.h"
+
+ #define IDEAL_RATIO_DECIMAL_DEPTH 26
++#define DIVIDER_NUM 64
+
+ #define pair_err(fmt, ...) \
+ dev_err(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__)
+@@ -101,6 +102,55 @@ static unsigned char clk_map_imx8qxp[2][ASRC_CLK_MAP_LEN] = {
+ },
+ };
+
++/*
++ * According to RM, the divider range is 1 ~ 8,
++ * prescaler is power of 2 from 1 ~ 128.
++ */
++static int asrc_clk_divider[DIVIDER_NUM] = {
++ 1, 2, 4, 8, 16, 32, 64, 128, /* divider = 1 */
++ 2, 4, 8, 16, 32, 64, 128, 256, /* divider = 2 */
++ 3, 6, 12, 24, 48, 96, 192, 384, /* divider = 3 */
++ 4, 8, 16, 32, 64, 128, 256, 512, /* divider = 4 */
++ 5, 10, 20, 40, 80, 160, 320, 640, /* divider = 5 */
++ 6, 12, 24, 48, 96, 192, 384, 768, /* divider = 6 */
++ 7, 14, 28, 56, 112, 224, 448, 896, /* divider = 7 */
++ 8, 16, 32, 64, 128, 256, 512, 1024, /* divider = 8 */
++};
++
++/*
++ * Check if the divider is available for internal ratio mode
++ */
++static bool fsl_asrc_divider_avail(int clk_rate, int rate, int *div)
++{
++ u32 rem, i;
++ u64 n;
++
++ if (div)
++ *div = 0;
++
++ if (clk_rate == 0 || rate == 0)
++ return false;
++
++ n = clk_rate;
++ rem = do_div(n, rate);
++
++ if (div)
++ *div = n;
++
++ if (rem != 0)
++ return false;
++
++ for (i = 0; i < DIVIDER_NUM; i++) {
++ if (n == asrc_clk_divider[i])
++ break;
++ }
++
++ if (i == DIVIDER_NUM)
++ return false;
++
++ return true;
++}
++
+ /**
+ * fsl_asrc_sel_proc - Select the pre-processing and post-processing options
+ * @inrate: input sample rate
+@@ -330,12 +380,12 @@ static int fsl_asrc_config_pair(struct fsl_asrc_pair *pair, bool use_ideal_rate)
+ enum asrc_word_width input_word_width;
+ enum asrc_word_width output_word_width;
+ u32 inrate, outrate, indiv, outdiv;
+- u32 clk_index[2], div[2], rem[2];
++ u32 clk_index[2], div[2];
+ u64 clk_rate;
+ int in, out, channels;
+ int pre_proc, post_proc;
+ struct clk *clk;
+- bool ideal;
++ bool ideal, div_avail;
+
+ if (!config) {
+ pair_err("invalid pair config\n");
+@@ -415,8 +465,7 @@ static int fsl_asrc_config_pair(struct fsl_asrc_pair *pair, bool use_ideal_rate)
+ clk = asrc_priv->asrck_clk[clk_index[ideal ? OUT : IN]];
+
+ clk_rate = clk_get_rate(clk);
+- rem[IN] = do_div(clk_rate, inrate);
+- div[IN] = (u32)clk_rate;
++ div_avail = fsl_asrc_divider_avail(clk_rate, inrate, &div[IN]);
+
+ /*
+ * The divider range is [1, 1024], defined by the hardware. For non-
+@@ -425,7 +474,7 @@ static int fsl_asrc_config_pair(struct fsl_asrc_pair *pair, bool use_ideal_rate)
+ * only result in different converting speeds. So remainder does not
+ * matter, as long as we keep the divider within its valid range.
+ */
+- if (div[IN] == 0 || (!ideal && (div[IN] > 1024 || rem[IN] != 0))) {
++ if (div[IN] == 0 || (!ideal && !div_avail)) {
+ pair_err("failed to support input sample rate %dHz by asrck_%x\n",
+ inrate, clk_index[ideal ? OUT : IN]);
+ return -EINVAL;
+@@ -436,13 +485,12 @@ static int fsl_asrc_config_pair(struct fsl_asrc_pair *pair, bool use_ideal_rate)
+ clk = asrc_priv->asrck_clk[clk_index[OUT]];
+ clk_rate = clk_get_rate(clk);
+ if (ideal && use_ideal_rate)
+- rem[OUT] = do_div(clk_rate, IDEAL_RATIO_RATE);
++ div_avail = fsl_asrc_divider_avail(clk_rate, IDEAL_RATIO_RATE, &div[OUT]);
+ else
+- rem[OUT] = do_div(clk_rate, outrate);
+- div[OUT] = clk_rate;
++ div_avail = fsl_asrc_divider_avail(clk_rate, outrate, &div[OUT]);
+
+ /* Output divider has the same limitation as the input one */
+- if (div[OUT] == 0 || (!ideal && (div[OUT] > 1024 || rem[OUT] != 0))) {
++ if (div[OUT] == 0 || (!ideal && !div_avail)) {
+ pair_err("failed to support output sample rate %dHz by asrck_%x\n",
+ outrate, clk_index[OUT]);
+ return -EINVAL;
+@@ -621,8 +669,7 @@ static void fsl_asrc_select_clk(struct fsl_asrc_priv *asrc_priv,
+ clk_index = asrc_priv->clk_map[j][i];
+ clk_rate = clk_get_rate(asrc_priv->asrck_clk[clk_index]);
+ /* Only match a perfect clock source with no remainder */
+- if (clk_rate != 0 && (clk_rate / rate[j]) <= 1024 &&
+- (clk_rate % rate[j]) == 0)
++ if (fsl_asrc_divider_avail(clk_rate, rate[j], NULL))
+ break;
+ }
+
+diff --git a/sound/soc/fsl/fsl_mqs.c b/sound/soc/fsl/fsl_mqs.c
+index 69aeb0e71844d..0d4efbed41dab 100644
+--- a/sound/soc/fsl/fsl_mqs.c
++++ b/sound/soc/fsl/fsl_mqs.c
+@@ -337,4 +337,4 @@ module_platform_driver(fsl_mqs_driver);
+ MODULE_AUTHOR("Shengjiu Wang <Shengjiu.Wang@nxp.com>");
+ MODULE_DESCRIPTION("MQS codec driver");
+ MODULE_LICENSE("GPL v2");
+-MODULE_ALIAS("platform: fsl-mqs");
++MODULE_ALIAS("platform:fsl-mqs");
+diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c
+index 1c53719bb61e2..8b5c3ba48516c 100644
+--- a/sound/soc/fsl/fsl_spdif.c
++++ b/sound/soc/fsl/fsl_spdif.c
+@@ -598,6 +598,8 @@ static void fsl_spdif_shutdown(struct snd_pcm_substream *substream,
+ mask = SCR_TXFIFO_AUTOSYNC_MASK | SCR_TXFIFO_CTRL_MASK |
+ SCR_TXSEL_MASK | SCR_USRC_SEL_MASK |
+ SCR_TXFIFO_FSEL_MASK;
++ /* Disable TX clock */
++ regmap_update_bits(regmap, REG_SPDIF_STC, STC_TXCLK_ALL_EN_MASK, 0);
+ } else {
+ scr = SCR_RXFIFO_OFF | SCR_RXFIFO_CTL_ZERO;
+ mask = SCR_RXFIFO_FSEL_MASK | SCR_RXFIFO_AUTOSYNC_MASK|
+diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c
+index 58fd0639a0698..db947180617a6 100644
+--- a/sound/soc/fsl/imx-card.c
++++ b/sound/soc/fsl/imx-card.c
+@@ -120,7 +120,7 @@ struct imx_card_data {
+
+ static struct imx_akcodec_fs_mul ak4458_fs_mul[] = {
+ /* Normal, < 32kHz */
+- { .rmin = 8000, .rmax = 24000, .wmin = 1024, .wmax = 1024, },
++ { .rmin = 8000, .rmax = 24000, .wmin = 256, .wmax = 1024, },
+ /* Normal, 32kHz */
+ { .rmin = 32000, .rmax = 32000, .wmin = 256, .wmax = 1024, },
+ /* Normal */
+@@ -151,8 +151,8 @@ static struct imx_akcodec_fs_mul ak4497_fs_mul[] = {
+ * Table 7 - mapping multiplier and speed mode
+ * Tables 8 & 9 - mapping speed mode and LRCK fs
+ */
+- { .rmin = 8000, .rmax = 32000, .wmin = 1024, .wmax = 1024, }, /* Normal, <= 32kHz */
+- { .rmin = 44100, .rmax = 48000, .wmin = 512, .wmax = 512, }, /* Normal */
++ { .rmin = 8000, .rmax = 32000, .wmin = 256, .wmax = 1024, }, /* Normal, <= 32kHz */
++ { .rmin = 44100, .rmax = 48000, .wmin = 256, .wmax = 512, }, /* Normal */
+ { .rmin = 88200, .rmax = 96000, .wmin = 256, .wmax = 256, }, /* Double */
+ { .rmin = 176400, .rmax = 192000, .wmin = 128, .wmax = 128, }, /* Quad */
+ { .rmin = 352800, .rmax = 384000, .wmin = 128, .wmax = 128, }, /* Oct */
+@@ -164,7 +164,7 @@ static struct imx_akcodec_fs_mul ak4497_fs_mul[] = {
+ * (Table 4 from datasheet)
+ */
+ static struct imx_akcodec_fs_mul ak5558_fs_mul[] = {
+- { .rmin = 8000, .rmax = 32000, .wmin = 1024, .wmax = 1024, },
++ { .rmin = 8000, .rmax = 32000, .wmin = 512, .wmax = 1024, },
+ { .rmin = 44100, .rmax = 48000, .wmin = 512, .wmax = 512, },
+ { .rmin = 88200, .rmax = 96000, .wmin = 256, .wmax = 256, },
+ { .rmin = 176400, .rmax = 192000, .wmin = 128, .wmax = 128, },
+@@ -247,13 +247,14 @@ static bool codec_is_akcodec(unsigned int type)
+ }
+
+ static unsigned long akcodec_get_mclk_rate(struct snd_pcm_substream *substream,
+- struct snd_pcm_hw_params *params)
++ struct snd_pcm_hw_params *params,
++ int slots, int slot_width)
+ {
+ struct snd_soc_pcm_runtime *rtd = substream->private_data;
+ struct imx_card_data *data = snd_soc_card_get_drvdata(rtd->card);
+ const struct imx_card_plat_data *plat_data = data->plat_data;
+ struct dai_link_data *link_data = &data->link_data[rtd->num];
+- unsigned int width = link_data->slots * link_data->slot_width;
++ unsigned int width = slots * slot_width;
+ unsigned int rate = params_rate(params);
+ int i;
+
+@@ -349,7 +350,7 @@ static int imx_aif_hw_params(struct snd_pcm_substream *substream,
+
+ /* Set MCLK freq */
+ if (codec_is_akcodec(plat_data->type))
+- mclk_freq = akcodec_get_mclk_rate(substream, params);
++ mclk_freq = akcodec_get_mclk_rate(substream, params, slots, slot_width);
+ else
+ mclk_freq = params_rate(params) * slots * slot_width;
+ /* Use the maximum freq from DSD512 (512*44100 = 22579200) */
+@@ -553,8 +554,23 @@ static int imx_card_parse_of(struct imx_card_data *data)
+ link_data->cpu_sysclk_id = FSL_SAI_CLK_MAST1;
+
+ /* sai may support mclk/bclk = 1 */
+- if (of_find_property(np, "fsl,mclk-equal-bclk", NULL))
++ if (of_find_property(np, "fsl,mclk-equal-bclk", NULL)) {
+ link_data->one2one_ratio = true;
++ } else {
++ int i;
++
++ /*
++ * i.MX8MQ don't support one2one ratio, then
++ * with ak4497 only 16bit case is supported.
++ */
++ for (i = 0; i < ARRAY_SIZE(ak4497_fs_mul); i++) {
++ if (ak4497_fs_mul[i].rmin == 705600 &&
++ ak4497_fs_mul[i].rmax == 768000) {
++ ak4497_fs_mul[i].wmin = 32;
++ ak4497_fs_mul[i].wmax = 32;
++ }
++ }
++ }
+ }
+
+ link->cpus->of_node = args.np;
+diff --git a/sound/soc/fsl/imx-es8328.c b/sound/soc/fsl/imx-es8328.c
+index 1981dcd7e9305..a7fb53e0f96f8 100644
+--- a/sound/soc/fsl/imx-es8328.c
++++ b/sound/soc/fsl/imx-es8328.c
+@@ -87,6 +87,7 @@ static int imx_es8328_probe(struct platform_device *pdev)
+ if (int_port > MUX_PORT_MAX || int_port == 0) {
+ dev_err(dev, "mux-int-port: hardware only has %d mux ports\n",
+ MUX_PORT_MAX);
++ ret = -EINVAL;
+ goto fail;
+ }
+
+diff --git a/sound/soc/fsl/imx-hdmi.c b/sound/soc/fsl/imx-hdmi.c
+index 34a0dceae6216..ef8d7a65ebc61 100644
+--- a/sound/soc/fsl/imx-hdmi.c
++++ b/sound/soc/fsl/imx-hdmi.c
+@@ -145,6 +145,8 @@ static int imx_hdmi_probe(struct platform_device *pdev)
+ data->dai.capture_only = false;
+ data->dai.init = imx_hdmi_init;
+
++ put_device(&cpu_pdev->dev);
++
+ if (of_node_name_eq(cpu_np, "sai")) {
+ data->cpu_priv.sysclk_id[1] = FSL_SAI_CLK_MAST1;
+ data->cpu_priv.sysclk_id[0] = FSL_SAI_CLK_MAST1;
+diff --git a/sound/soc/fsl/pcm030-audio-fabric.c b/sound/soc/fsl/pcm030-audio-fabric.c
+index af3c3b90c0aca..83b4a22bf15ac 100644
+--- a/sound/soc/fsl/pcm030-audio-fabric.c
++++ b/sound/soc/fsl/pcm030-audio-fabric.c
+@@ -93,16 +93,21 @@ static int pcm030_fabric_probe(struct platform_device *op)
+ dev_err(&op->dev, "platform_device_alloc() failed\n");
+
+ ret = platform_device_add(pdata->codec_device);
+- if (ret)
++ if (ret) {
+ dev_err(&op->dev, "platform_device_add() failed: %d\n", ret);
++ platform_device_put(pdata->codec_device);
++ }
+
+ ret = snd_soc_register_card(card);
+- if (ret)
++ if (ret) {
+ dev_err(&op->dev, "snd_soc_register_card() failed: %d\n", ret);
++ platform_device_del(pdata->codec_device);
++ platform_device_put(pdata->codec_device);
++ }
+
+ platform_set_drvdata(op, pdata);
+-
+ return ret;
++
+ }
+
+ static int pcm030_fabric_remove(struct platform_device *op)
+diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
+index 10c63b73900c6..ffda8a38de3ed 100644
+--- a/sound/soc/generic/simple-card-utils.c
++++ b/sound/soc/generic/simple-card-utils.c
+@@ -275,6 +275,7 @@ int asoc_simple_hw_params(struct snd_pcm_substream *substream,
+ mclk_fs = props->mclk_fs;
+
+ if (mclk_fs) {
++ struct snd_soc_component *component;
+ mclk = params_rate(params) * mclk_fs;
+
+ for_each_prop_dai_codec(props, i, pdai) {
+@@ -282,16 +283,30 @@ int asoc_simple_hw_params(struct snd_pcm_substream *substream,
+ if (ret < 0)
+ return ret;
+ }
++
+ for_each_prop_dai_cpu(props, i, pdai) {
+ ret = asoc_simple_set_clk_rate(pdai, mclk);
+ if (ret < 0)
+ return ret;
+ }
++
++ /* Ensure sysclk is set on all components in case any
++ * (such as platform components) are missed by calls to
++ * snd_soc_dai_set_sysclk.
++ */
++ for_each_rtd_components(rtd, i, component) {
++ ret = snd_soc_component_set_sysclk(component, 0, 0,
++ mclk, SND_SOC_CLOCK_IN);
++ if (ret && ret != -ENOTSUPP)
++ return ret;
++ }
++
+ for_each_rtd_codec_dais(rtd, i, sdai) {
+ ret = snd_soc_dai_set_sysclk(sdai, 0, mclk, SND_SOC_CLOCK_IN);
+ if (ret && ret != -ENOTSUPP)
+ return ret;
+ }
++
+ for_each_rtd_cpu_dais(rtd, i, sdai) {
+ ret = snd_soc_dai_set_sysclk(sdai, 0, mclk, SND_SOC_CLOCK_OUT);
+ if (ret && ret != -ENOTSUPP)
+diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c
+index a3a7990b5cb66..bc3e24c6a28a8 100644
+--- a/sound/soc/generic/simple-card.c
++++ b/sound/soc/generic/simple-card.c
+@@ -28,6 +28,30 @@ static const struct snd_soc_ops simple_ops = {
+ .hw_params = asoc_simple_hw_params,
+ };
+
++static int asoc_simple_parse_platform(struct device_node *node,
++ struct snd_soc_dai_link_component *dlc)
++{
++ struct of_phandle_args args;
++ int ret;
++
++ if (!node)
++ return 0;
++
++ /*
++ * Get node via "sound-dai = <&phandle port>"
++ * it will be used as xxx_of_node on soc_bind_dai_link()
++ */
++ ret = of_parse_phandle_with_args(node, DAI, CELL, 0, &args);
++ if (ret)
++ return ret;
++
++ /* dai_name is not required and may not exist for plat component */
++
++ dlc->of_node = args.np;
++
++ return 0;
++}
++
+ static int asoc_simple_parse_dai(struct device_node *node,
+ struct snd_soc_dai_link_component *dlc,
+ int *is_single_link)
+@@ -289,7 +313,7 @@ static int simple_dai_link_of(struct asoc_simple_priv *priv,
+ if (ret < 0)
+ goto dai_link_of_err;
+
+- ret = asoc_simple_parse_dai(plat, platforms, NULL);
++ ret = asoc_simple_parse_platform(plat, platforms);
+ if (ret < 0)
+ goto dai_link_of_err;
+
+diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c
+index 6b06248a9327a..0bf3e56e1d580 100644
+--- a/sound/soc/intel/boards/sof_sdw.c
++++ b/sound/soc/intel/boards/sof_sdw.c
+@@ -184,11 +184,11 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = {
+ .callback = sof_sdw_quirk_cb,
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "HP"),
+- DMI_MATCH(DMI_PRODUCT_NAME, "HP Spectre x360 Convertible"),
++ DMI_MATCH(DMI_PRODUCT_NAME, "HP Spectre x360 Conv"),
+ },
+ .driver_data = (void *)(SOF_SDW_TGL_HDMI |
+ SOF_SDW_PCH_DMIC |
+- RT711_JD2),
++ RT711_JD1),
+ },
+ {
+ /* NUC15 'Bishop County' LAPBC510 and LAPBC710 skews */
+@@ -213,6 +213,16 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = {
+ SOF_RT715_DAI_ID_FIX |
+ SOF_SDW_FOUR_SPK),
+ },
++ {
++ .callback = sof_sdw_quirk_cb,
++ .matches = {
++ DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"),
++ DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0A45")
++ },
++ .driver_data = (void *)(SOF_SDW_TGL_HDMI |
++ RT711_JD2 |
++ SOF_RT715_DAI_ID_FIX),
++ },
+ /* AlderLake devices */
+ {
+ .callback = sof_sdw_quirk_cb,
+diff --git a/sound/soc/intel/catpt/dsp.c b/sound/soc/intel/catpt/dsp.c
+index 9c5fd18f2600f..346bec0003066 100644
+--- a/sound/soc/intel/catpt/dsp.c
++++ b/sound/soc/intel/catpt/dsp.c
+@@ -65,6 +65,7 @@ static int catpt_dma_memcpy(struct catpt_dev *cdev, struct dma_chan *chan,
+ {
+ struct dma_async_tx_descriptor *desc;
+ enum dma_status status;
++ int ret;
+
+ desc = dmaengine_prep_dma_memcpy(chan, dst_addr, src_addr, size,
+ DMA_CTRL_ACK);
+@@ -77,13 +78,22 @@ static int catpt_dma_memcpy(struct catpt_dev *cdev, struct dma_chan *chan,
+ catpt_updatel_shim(cdev, HMDC,
+ CATPT_HMDC_HDDA(CATPT_DMA_DEVID, chan->chan_id),
+ CATPT_HMDC_HDDA(CATPT_DMA_DEVID, chan->chan_id));
+- dmaengine_submit(desc);
++
++ ret = dma_submit_error(dmaengine_submit(desc));
++ if (ret) {
++ dev_err(cdev->dev, "submit tx failed: %d\n", ret);
++ goto clear_hdda;
++ }
++
+ status = dma_wait_for_async_tx(desc);
++ ret = (status == DMA_COMPLETE) ? 0 : -EPROTO;
++
++clear_hdda:
+ /* regardless of status, disable access to HOST memory in demand mode */
+ catpt_updatel_shim(cdev, HMDC,
+ CATPT_HMDC_HDDA(CATPT_DMA_DEVID, chan->chan_id), 0);
+
+- return (status == DMA_COMPLETE) ? 0 : -EPROTO;
++ return ret;
+ }
+
+ int catpt_dma_memcpy_todsp(struct catpt_dev *cdev, struct dma_chan *chan,
+diff --git a/sound/soc/intel/common/soc-acpi-intel-tgl-match.c b/sound/soc/intel/common/soc-acpi-intel-tgl-match.c
+index 785d5f5f8a9c9..c93d8019b0e55 100644
+--- a/sound/soc/intel/common/soc-acpi-intel-tgl-match.c
++++ b/sound/soc/intel/common/soc-acpi-intel-tgl-match.c
+@@ -127,13 +127,13 @@ static const struct snd_soc_acpi_adr_device mx8373_1_adr[] = {
+ {
+ .adr = 0x000123019F837300ull,
+ .num_endpoints = 1,
+- .endpoints = &spk_l_endpoint,
++ .endpoints = &spk_r_endpoint,
+ .name_prefix = "Right"
+ },
+ {
+ .adr = 0x000127019F837300ull,
+ .num_endpoints = 1,
+- .endpoints = &spk_r_endpoint,
++ .endpoints = &spk_l_endpoint,
+ .name_prefix = "Left"
+ }
+ };
+@@ -156,6 +156,15 @@ static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = {
+ }
+ };
+
++static const struct snd_soc_acpi_adr_device rt1316_1_single_adr[] = {
++ {
++ .adr = 0x000131025D131601ull,
++ .num_endpoints = 1,
++ .endpoints = &single_endpoint,
++ .name_prefix = "rt1316-1"
++ }
++};
++
+ static const struct snd_soc_acpi_adr_device rt1316_1_group1_adr[] = {
+ {
+ .adr = 0x000131025D131601ull, /* unique ID is set for some reason */
+@@ -320,6 +329,25 @@ static const struct snd_soc_acpi_link_adr tgl_3_in_1_sdca[] = {
+ {}
+ };
+
++static const struct snd_soc_acpi_link_adr tgl_3_in_1_sdca_mono[] = {
++ {
++ .mask = BIT(0),
++ .num_adr = ARRAY_SIZE(rt711_sdca_0_adr),
++ .adr_d = rt711_sdca_0_adr,
++ },
++ {
++ .mask = BIT(1),
++ .num_adr = ARRAY_SIZE(rt1316_1_single_adr),
++ .adr_d = rt1316_1_single_adr,
++ },
++ {
++ .mask = BIT(3),
++ .num_adr = ARRAY_SIZE(rt714_3_adr),
++ .adr_d = rt714_3_adr,
++ },
++ {}
++};
++
+ static const struct snd_soc_acpi_codecs tgl_max98373_amp = {
+ .num_codecs = 1,
+ .codecs = {"MX98373"}
+@@ -412,6 +440,19 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_sdw_machines[] = {
+ .drv_name = "sof_sdw",
+ .sof_tplg_filename = "sof-tgl-rt711-rt1316-rt714.tplg",
+ },
++ {
++ /*
++ * link_mask should be 0xB, but all links are enabled by BIOS.
++ * This entry will be selected if there is no rt1316 amplifier exposed
++ * on link2 since it will fail to match the above entry.
++ */
++
++ .link_mask = 0xF, /* 4 active links required */
++ .links = tgl_3_in_1_sdca_mono,
++ .drv_name = "sof_sdw",
++ .sof_tplg_filename = "sof-tgl-rt711-l0-rt1316-l1-mono-rt714-l3.tplg",
++ },
++
+ {
+ .link_mask = 0x3, /* rt711 on link 0 and 1 rt1308 on link 1 */
+ .links = tgl_hp,
+diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c
+index 9ecaf6a1e8475..e4aa366d356eb 100644
+--- a/sound/soc/intel/skylake/skl-pcm.c
++++ b/sound/soc/intel/skylake/skl-pcm.c
+@@ -1251,7 +1251,6 @@ static int skl_platform_soc_get_time_info(
+ snd_pcm_gettime(substream->runtime, system_ts);
+
+ nsec = timecounter_read(&hstr->tc);
+- nsec = div_u64(nsec, 3); /* can be optimized */
+ if (audio_tstamp_config->report_delay)
+ nsec = skl_adjust_codec_delay(substream, nsec);
+
+diff --git a/sound/soc/mediatek/mt8173/mt8173-max98090.c b/sound/soc/mediatek/mt8173/mt8173-max98090.c
+index fc94314bfc02f..3bdd4931316cd 100644
+--- a/sound/soc/mediatek/mt8173/mt8173-max98090.c
++++ b/sound/soc/mediatek/mt8173/mt8173-max98090.c
+@@ -180,6 +180,9 @@ static int mt8173_max98090_dev_probe(struct platform_device *pdev)
+ if (ret)
+ dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+ __func__, ret);
++
++ of_node_put(codec_node);
++ of_node_put(platform_node);
+ return ret;
+ }
+
+diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c
+index 0f28dc2217c09..390da5bf727eb 100644
+--- a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c
++++ b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c
+@@ -218,6 +218,8 @@ static int mt8173_rt5650_rt5514_dev_probe(struct platform_device *pdev)
+ if (ret)
+ dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+ __func__, ret);
++
++ of_node_put(platform_node);
+ return ret;
+ }
+
+diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c
+index 077c6ee067806..c8e4e85e10575 100644
+--- a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c
++++ b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c
+@@ -285,6 +285,8 @@ static int mt8173_rt5650_rt5676_dev_probe(struct platform_device *pdev)
+ if (ret)
+ dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+ __func__, ret);
++
++ of_node_put(platform_node);
+ return ret;
+ }
+
+diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650.c b/sound/soc/mediatek/mt8173/mt8173-rt5650.c
+index c28ebf891cb05..e168d31f44459 100644
+--- a/sound/soc/mediatek/mt8173/mt8173-rt5650.c
++++ b/sound/soc/mediatek/mt8173/mt8173-rt5650.c
+@@ -323,6 +323,8 @@ static int mt8173_rt5650_dev_probe(struct platform_device *pdev)
+ if (ret)
+ dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+ __func__, ret);
++
++ of_node_put(platform_node);
+ return ret;
+ }
+
+diff --git a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
+index a4d26a6fc8492..0ab8b050b305f 100644
+--- a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
++++ b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
+@@ -685,7 +685,6 @@ static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev)
+ struct snd_soc_dai_link *dai_link;
+ struct mt8183_da7219_max98357_priv *priv;
+ struct pinctrl *pinctrl;
+- const struct of_device_id *match;
+ int ret, i;
+
+ platform_node = of_parse_phandle(pdev->dev.of_node,
+@@ -695,11 +694,9 @@ static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev)
+ return -EINVAL;
+ }
+
+- match = of_match_device(pdev->dev.driver->of_match_table, &pdev->dev);
+- if (!match || !match->data)
++ card = (struct snd_soc_card *)of_device_get_match_data(&pdev->dev);
++ if (!card)
+ return -EINVAL;
+-
+- card = (struct snd_soc_card *)match->data;
+ card->dev = &pdev->dev;
+
+ hdmi_codec = of_parse_phandle(pdev->dev.of_node,
+@@ -781,7 +778,11 @@ static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev)
+ return ret;
+ }
+
+- return devm_snd_soc_register_card(&pdev->dev, card);
++ ret = devm_snd_soc_register_card(&pdev->dev, card);
++
++ of_node_put(platform_node);
++ of_node_put(hdmi_codec);
++ return ret;
+ }
+
+ #ifdef CONFIG_OF
+diff --git a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
+index 94dcbd36c8697..a56c1e87d5642 100644
+--- a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
++++ b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
+@@ -637,7 +637,6 @@ mt8183_mt6358_ts3a227_max98357_dev_probe(struct platform_device *pdev)
+ struct device_node *platform_node, *ec_codec, *hdmi_codec;
+ struct snd_soc_dai_link *dai_link;
+ struct mt8183_mt6358_ts3a227_max98357_priv *priv;
+- const struct of_device_id *match;
+ int ret, i;
+
+ platform_node = of_parse_phandle(pdev->dev.of_node,
+@@ -647,11 +646,9 @@ mt8183_mt6358_ts3a227_max98357_dev_probe(struct platform_device *pdev)
+ return -EINVAL;
+ }
+
+- match = of_match_device(pdev->dev.driver->of_match_table, &pdev->dev);
+- if (!match || !match->data)
++ card = (struct snd_soc_card *)of_device_get_match_data(&pdev->dev);
++ if (!card)
+ return -EINVAL;
+-
+- card = (struct snd_soc_card *)match->data;
+ card->dev = &pdev->dev;
+
+ ec_codec = of_parse_phandle(pdev->dev.of_node, "mediatek,ec-codec", 0);
+@@ -780,7 +777,12 @@ mt8183_mt6358_ts3a227_max98357_dev_probe(struct platform_device *pdev)
+ __func__, ret);
+ }
+
+- return devm_snd_soc_register_card(&pdev->dev, card);
++ ret = devm_snd_soc_register_card(&pdev->dev, card);
++
++ of_node_put(platform_node);
++ of_node_put(ec_codec);
++ of_node_put(hdmi_codec);
++ return ret;
+ }
+
+ #ifdef CONFIG_OF
+diff --git a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
+index a606133951b70..c1d225b498513 100644
+--- a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
++++ b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
+@@ -1106,7 +1106,6 @@ static int mt8192_mt6359_dev_probe(struct platform_device *pdev)
+ struct device_node *platform_node, *hdmi_codec;
+ int ret, i;
+ struct snd_soc_dai_link *dai_link;
+- const struct of_device_id *match;
+ struct mt8192_mt6359_priv *priv;
+
+ platform_node = of_parse_phandle(pdev->dev.of_node,
+@@ -1116,11 +1115,11 @@ static int mt8192_mt6359_dev_probe(struct platform_device *pdev)
+ return -EINVAL;
+ }
+
+- match = of_match_device(pdev->dev.driver->of_match_table, &pdev->dev);
+- if (!match || !match->data)
+- return -EINVAL;
+-
+- card = (struct snd_soc_card *)match->data;
++ card = (struct snd_soc_card *)of_device_get_match_data(&pdev->dev);
++ if (!card) {
++ ret = -EINVAL;
++ goto put_platform_node;
++ }
+ card->dev = &pdev->dev;
+
+ hdmi_codec = of_parse_phandle(pdev->dev.of_node,
+@@ -1162,17 +1161,25 @@ static int mt8192_mt6359_dev_probe(struct platform_device *pdev)
+ }
+
+ priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+- if (!priv)
+- return -ENOMEM;
++ if (!priv) {
++ ret = -ENOMEM;
++ goto put_hdmi_codec;
++ }
+ snd_soc_card_set_drvdata(card, priv);
+
+ ret = mt8192_afe_gpio_init(&pdev->dev);
+ if (ret) {
+ dev_err(&pdev->dev, "init gpio error %d\n", ret);
+- return ret;
++ goto put_hdmi_codec;
+ }
+
+- return devm_snd_soc_register_card(&pdev->dev, card);
++ ret = devm_snd_soc_register_card(&pdev->dev, card);
++
++put_hdmi_codec:
++ of_node_put(hdmi_codec);
++put_platform_node:
++ of_node_put(platform_node);
++ return ret;
+ }
+
+ #ifdef CONFIG_OF
+diff --git a/sound/soc/mediatek/mt8195/mt8195-afe-pcm.c b/sound/soc/mediatek/mt8195/mt8195-afe-pcm.c
+index 6635c3f72eccc..2edb40fe27ccb 100644
+--- a/sound/soc/mediatek/mt8195/mt8195-afe-pcm.c
++++ b/sound/soc/mediatek/mt8195/mt8195-afe-pcm.c
+@@ -3028,7 +3028,7 @@ static const struct reg_sequence mt8195_afe_reg_defaults[] = {
+
+ static const struct reg_sequence mt8195_cg_patch[] = {
+ { AUDIO_TOP_CON0, 0xfffffffb },
+- { AUDIO_TOP_CON1, 0xfffffffa },
++ { AUDIO_TOP_CON1, 0xfffffff8 },
+ };
+
+ static int mt8195_afe_init_registers(struct mtk_base_afe *afe)
+diff --git a/sound/soc/mediatek/mt8195/mt8195-dai-pcm.c b/sound/soc/mediatek/mt8195/mt8195-dai-pcm.c
+index 5d10d2c4c991c..151914c873acd 100644
+--- a/sound/soc/mediatek/mt8195/mt8195-dai-pcm.c
++++ b/sound/soc/mediatek/mt8195/mt8195-dai-pcm.c
+@@ -80,8 +80,15 @@ static const struct snd_soc_dapm_widget mtk_dai_pcm_widgets[] = {
+ mtk_dai_pcm_o001_mix,
+ ARRAY_SIZE(mtk_dai_pcm_o001_mix)),
+
++ SND_SOC_DAPM_SUPPLY("PCM_EN", PCM_INTF_CON1,
++ PCM_INTF_CON1_PCM_EN_SHIFT, 0, NULL, 0),
++
+ SND_SOC_DAPM_INPUT("PCM1_INPUT"),
+ SND_SOC_DAPM_OUTPUT("PCM1_OUTPUT"),
++
++ SND_SOC_DAPM_CLOCK_SUPPLY("aud_asrc11"),
++ SND_SOC_DAPM_CLOCK_SUPPLY("aud_asrc12"),
++ SND_SOC_DAPM_CLOCK_SUPPLY("aud_pcmif"),
+ };
+
+ static const struct snd_soc_dapm_route mtk_dai_pcm_routes[] = {
+@@ -97,22 +104,18 @@ static const struct snd_soc_dapm_route mtk_dai_pcm_routes[] = {
+ {"PCM1 Playback", NULL, "O000"},
+ {"PCM1 Playback", NULL, "O001"},
+
++ {"PCM1 Playback", NULL, "PCM_EN"},
++ {"PCM1 Playback", NULL, "aud_asrc12"},
++ {"PCM1 Playback", NULL, "aud_pcmif"},
++
++ {"PCM1 Capture", NULL, "PCM_EN"},
++ {"PCM1 Capture", NULL, "aud_asrc11"},
++ {"PCM1 Capture", NULL, "aud_pcmif"},
++
+ {"PCM1_OUTPUT", NULL, "PCM1 Playback"},
+ {"PCM1 Capture", NULL, "PCM1_INPUT"},
+ };
+
+-static void mtk_dai_pcm_enable(struct mtk_base_afe *afe)
+-{
+- regmap_update_bits(afe->regmap, PCM_INTF_CON1,
+- PCM_INTF_CON1_PCM_EN, PCM_INTF_CON1_PCM_EN);
+-}
+-
+-static void mtk_dai_pcm_disable(struct mtk_base_afe *afe)
+-{
+- regmap_update_bits(afe->regmap, PCM_INTF_CON1,
+- PCM_INTF_CON1_PCM_EN, 0x0);
+-}
+-
+ static int mtk_dai_pcm_configure(struct snd_pcm_substream *substream,
+ struct snd_soc_dai *dai)
+ {
+@@ -207,54 +210,22 @@ static int mtk_dai_pcm_configure(struct snd_pcm_substream *substream,
+ }
+
+ /* dai ops */
+-static int mtk_dai_pcm_startup(struct snd_pcm_substream *substream,
+- struct snd_soc_dai *dai)
+-{
+- struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+- struct mt8195_afe_private *afe_priv = afe->platform_priv;
+-
+- if (dai->component->active)
+- return 0;
+-
+- mt8195_afe_enable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_ASRC11]);
+- mt8195_afe_enable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_ASRC12]);
+- mt8195_afe_enable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_PCMIF]);
+-
+- return 0;
+-}
+-
+-static void mtk_dai_pcm_shutdown(struct snd_pcm_substream *substream,
+- struct snd_soc_dai *dai)
+-{
+- struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+- struct mt8195_afe_private *afe_priv = afe->platform_priv;
+-
+- if (dai->component->active)
+- return;
+-
+- mtk_dai_pcm_disable(afe);
+-
+- mt8195_afe_disable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_PCMIF]);
+- mt8195_afe_disable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_ASRC12]);
+- mt8195_afe_disable_clk(afe, afe_priv->clk[MT8195_CLK_AUD_ASRC11]);
+-}
+-
+ static int mtk_dai_pcm_prepare(struct snd_pcm_substream *substream,
+ struct snd_soc_dai *dai)
+ {
+- struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+- int ret = 0;
++ int ret;
+
+- if (snd_soc_dai_stream_active(dai, SNDRV_PCM_STREAM_PLAYBACK) &&
+- snd_soc_dai_stream_active(dai, SNDRV_PCM_STREAM_CAPTURE))
++ dev_dbg(dai->dev, "%s(), id %d, stream %d, widget active p %d, c %d\n",
++ __func__, dai->id, substream->stream,
++ dai->playback_widget->active, dai->capture_widget->active);
++
++ if (dai->playback_widget->active || dai->capture_widget->active)
+ return 0;
+
+ ret = mtk_dai_pcm_configure(substream, dai);
+ if (ret)
+ return ret;
+
+- mtk_dai_pcm_enable(afe);
+-
+ return 0;
+ }
+
+@@ -316,8 +287,6 @@ static int mtk_dai_pcm_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+ }
+
+ static const struct snd_soc_dai_ops mtk_dai_pcm_ops = {
+- .startup = mtk_dai_pcm_startup,
+- .shutdown = mtk_dai_pcm_shutdown,
+ .prepare = mtk_dai_pcm_prepare,
+ .set_fmt = mtk_dai_pcm_set_fmt,
+ };
+diff --git a/sound/soc/mediatek/mt8195/mt8195-mt6359-rt1019-rt5682.c b/sound/soc/mediatek/mt8195/mt8195-mt6359-rt1019-rt5682.c
+index de09f67c04502..a3fa8efc8f81c 100644
+--- a/sound/soc/mediatek/mt8195/mt8195-mt6359-rt1019-rt5682.c
++++ b/sound/soc/mediatek/mt8195/mt8195-mt6359-rt1019-rt5682.c
+@@ -1040,8 +1040,10 @@ static int mt8195_mt6359_rt1019_rt5682_dev_probe(struct platform_device *pdev)
+ }
+
+ priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+- if (!priv)
++ if (!priv) {
++ of_node_put(platform_node);
+ return -ENOMEM;
++ }
+
+ snd_soc_card_set_drvdata(card, priv);
+
+@@ -1049,6 +1051,8 @@ static int mt8195_mt6359_rt1019_rt5682_dev_probe(struct platform_device *pdev)
+ if (ret)
+ dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+ __func__, ret);
++
++ of_node_put(platform_node);
+ return ret;
+ }
+
+diff --git a/sound/soc/mediatek/mt8195/mt8195-reg.h b/sound/soc/mediatek/mt8195/mt8195-reg.h
+index d06f9cf85a4ec..d3871353db415 100644
+--- a/sound/soc/mediatek/mt8195/mt8195-reg.h
++++ b/sound/soc/mediatek/mt8195/mt8195-reg.h
+@@ -2550,6 +2550,7 @@
+ #define PCM_INTF_CON1_PCM_FMT(x) (((x) & 0x3) << 1)
+ #define PCM_INTF_CON1_PCM_FMT_MASK (0x3 << 1)
+ #define PCM_INTF_CON1_PCM_EN BIT(0)
++#define PCM_INTF_CON1_PCM_EN_SHIFT 0
+
+ /* PCM_INTF_CON2 */
+ #define PCM_INTF_CON2_CLK_DOMAIN_SEL(x) (((x) & 0x3) << 23)
+diff --git a/sound/soc/meson/aiu-acodec-ctrl.c b/sound/soc/meson/aiu-acodec-ctrl.c
+index 27a6d3259c50a..442c215936d97 100644
+--- a/sound/soc/meson/aiu-acodec-ctrl.c
++++ b/sound/soc/meson/aiu-acodec-ctrl.c
+@@ -58,7 +58,7 @@ static int aiu_acodec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol,
+
+ snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL);
+
+- return 0;
++ return 1;
+ }
+
+ static SOC_ENUM_SINGLE_DECL(aiu_acodec_ctrl_mux_enum, AIU_ACODEC_CTRL,
+diff --git a/sound/soc/meson/aiu-codec-ctrl.c b/sound/soc/meson/aiu-codec-ctrl.c
+index c3ea733fce91f..c966fc60dc733 100644
+--- a/sound/soc/meson/aiu-codec-ctrl.c
++++ b/sound/soc/meson/aiu-codec-ctrl.c
+@@ -57,7 +57,7 @@ static int aiu_codec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol,
+
+ snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL);
+
+- return 0;
++ return 1;
+ }
+
+ static SOC_ENUM_SINGLE_DECL(aiu_hdmi_ctrl_mux_enum, AIU_HDMI_CLK_DATA_CTRL,
+diff --git a/sound/soc/meson/aiu-encoder-i2s.c b/sound/soc/meson/aiu-encoder-i2s.c
+index 9322245521463..67729de41a73e 100644
+--- a/sound/soc/meson/aiu-encoder-i2s.c
++++ b/sound/soc/meson/aiu-encoder-i2s.c
+@@ -18,7 +18,6 @@
+ #define AIU_RST_SOFT_I2S_FAST BIT(0)
+
+ #define AIU_I2S_DAC_CFG_MSB_FIRST BIT(2)
+-#define AIU_I2S_MISC_HOLD_EN BIT(2)
+ #define AIU_CLK_CTRL_I2S_DIV_EN BIT(0)
+ #define AIU_CLK_CTRL_I2S_DIV GENMASK(3, 2)
+ #define AIU_CLK_CTRL_AOCLK_INVERT BIT(6)
+@@ -36,37 +35,6 @@ static void aiu_encoder_i2s_divider_enable(struct snd_soc_component *component,
+ enable ? AIU_CLK_CTRL_I2S_DIV_EN : 0);
+ }
+
+-static void aiu_encoder_i2s_hold(struct snd_soc_component *component,
+- bool enable)
+-{
+- snd_soc_component_update_bits(component, AIU_I2S_MISC,
+- AIU_I2S_MISC_HOLD_EN,
+- enable ? AIU_I2S_MISC_HOLD_EN : 0);
+-}
+-
+-static int aiu_encoder_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
+- struct snd_soc_dai *dai)
+-{
+- struct snd_soc_component *component = dai->component;
+-
+- switch (cmd) {
+- case SNDRV_PCM_TRIGGER_START:
+- case SNDRV_PCM_TRIGGER_RESUME:
+- case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+- aiu_encoder_i2s_hold(component, false);
+- return 0;
+-
+- case SNDRV_PCM_TRIGGER_STOP:
+- case SNDRV_PCM_TRIGGER_SUSPEND:
+- case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+- aiu_encoder_i2s_hold(component, true);
+- return 0;
+-
+- default:
+- return -EINVAL;
+- }
+-}
+-
+ static int aiu_encoder_i2s_setup_desc(struct snd_soc_component *component,
+ struct snd_pcm_hw_params *params)
+ {
+@@ -353,7 +321,6 @@ static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream,
+ }
+
+ const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops = {
+- .trigger = aiu_encoder_i2s_trigger,
+ .hw_params = aiu_encoder_i2s_hw_params,
+ .hw_free = aiu_encoder_i2s_hw_free,
+ .set_fmt = aiu_encoder_i2s_set_fmt,
+diff --git a/sound/soc/meson/aiu-fifo-i2s.c b/sound/soc/meson/aiu-fifo-i2s.c
+index 2388a2d0b3a6c..57e6e7160d2f2 100644
+--- a/sound/soc/meson/aiu-fifo-i2s.c
++++ b/sound/soc/meson/aiu-fifo-i2s.c
+@@ -20,6 +20,8 @@
+ #define AIU_MEM_I2S_CONTROL_MODE_16BIT BIT(6)
+ #define AIU_MEM_I2S_BUF_CNTL_INIT BIT(0)
+ #define AIU_RST_SOFT_I2S_FAST BIT(0)
++#define AIU_I2S_MISC_HOLD_EN BIT(2)
++#define AIU_I2S_MISC_FORCE_LEFT_RIGHT BIT(4)
+
+ #define AIU_FIFO_I2S_BLOCK 256
+
+@@ -90,6 +92,10 @@ static int aiu_fifo_i2s_hw_params(struct snd_pcm_substream *substream,
+ unsigned int val;
+ int ret;
+
++ snd_soc_component_update_bits(component, AIU_I2S_MISC,
++ AIU_I2S_MISC_HOLD_EN,
++ AIU_I2S_MISC_HOLD_EN);
++
+ ret = aiu_fifo_hw_params(substream, params, dai);
+ if (ret)
+ return ret;
+@@ -117,6 +123,19 @@ static int aiu_fifo_i2s_hw_params(struct snd_pcm_substream *substream,
+ snd_soc_component_update_bits(component, AIU_MEM_I2S_MASKS,
+ AIU_MEM_I2S_MASKS_IRQ_BLOCK, val);
+
++ /*
++ * Most (all?) supported SoCs have this bit set by default. The vendor
++ * driver however sets it manually (depending on the version either
++ * while un-setting AIU_I2S_MISC_HOLD_EN or right before that). Follow
++ * the same approach for consistency with the vendor driver.
++ */
++ snd_soc_component_update_bits(component, AIU_I2S_MISC,
++ AIU_I2S_MISC_FORCE_LEFT_RIGHT,
++ AIU_I2S_MISC_FORCE_LEFT_RIGHT);
++
++ snd_soc_component_update_bits(component, AIU_I2S_MISC,
++ AIU_I2S_MISC_HOLD_EN, 0);
++
+ return 0;
+ }
+
+diff --git a/sound/soc/meson/aiu-fifo.c b/sound/soc/meson/aiu-fifo.c
+index 4ad23267cace5..d67ff4cdabd5a 100644
+--- a/sound/soc/meson/aiu-fifo.c
++++ b/sound/soc/meson/aiu-fifo.c
+@@ -5,6 +5,7 @@
+
+ #include <linux/bitfield.h>
+ #include <linux/clk.h>
++#include <linux/dma-mapping.h>
+ #include <sound/pcm_params.h>
+ #include <sound/soc.h>
+ #include <sound/soc-dai.h>
+@@ -179,6 +180,11 @@ int aiu_fifo_pcm_new(struct snd_soc_pcm_runtime *rtd,
+ struct snd_card *card = rtd->card->snd_card;
+ struct aiu_fifo *fifo = dai->playback_dma_data;
+ size_t size = fifo->pcm->buffer_bytes_max;
++ int ret;
++
++ ret = dma_coerce_mask_and_coherent(card->dev, DMA_BIT_MASK(32));
++ if (ret)
++ return ret;
+
+ snd_pcm_set_managed_buffer_all(rtd->pcm, SNDRV_DMA_TYPE_DEV,
+ card->dev, size, size);
+diff --git a/sound/soc/meson/g12a-tohdmitx.c b/sound/soc/meson/g12a-tohdmitx.c
+index 9b2b59536ced0..6c99052feafd8 100644
+--- a/sound/soc/meson/g12a-tohdmitx.c
++++ b/sound/soc/meson/g12a-tohdmitx.c
+@@ -67,7 +67,7 @@ static int g12a_tohdmitx_i2s_mux_put_enum(struct snd_kcontrol *kcontrol,
+
+ snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL);
+
+- return 0;
++ return 1;
+ }
+
+ static SOC_ENUM_SINGLE_DECL(g12a_tohdmitx_i2s_mux_enum, TOHDMITX_CTRL0,
+diff --git a/sound/soc/mxs/mxs-saif.c b/sound/soc/mxs/mxs-saif.c
+index 6a2d24d489647..879c1221a809b 100644
+--- a/sound/soc/mxs/mxs-saif.c
++++ b/sound/soc/mxs/mxs-saif.c
+@@ -455,7 +455,10 @@ static int mxs_saif_hw_params(struct snd_pcm_substream *substream,
+ * basic clock which should be fast enough for the internal
+ * logic.
+ */
+- clk_enable(saif->clk);
++ ret = clk_enable(saif->clk);
++ if (ret)
++ return ret;
++
+ ret = clk_set_rate(saif->clk, 24000000);
+ clk_disable(saif->clk);
+ if (ret)
+diff --git a/sound/soc/mxs/mxs-sgtl5000.c b/sound/soc/mxs/mxs-sgtl5000.c
+index a6407f4388de7..fb721bc499496 100644
+--- a/sound/soc/mxs/mxs-sgtl5000.c
++++ b/sound/soc/mxs/mxs-sgtl5000.c
+@@ -118,6 +118,9 @@ static int mxs_sgtl5000_probe(struct platform_device *pdev)
+ codec_np = of_parse_phandle(np, "audio-codec", 0);
+ if (!saif_np[0] || !saif_np[1] || !codec_np) {
+ dev_err(&pdev->dev, "phandle missing or invalid\n");
++ of_node_put(codec_np);
++ of_node_put(saif_np[0]);
++ of_node_put(saif_np[1]);
+ return -EINVAL;
+ }
+
+diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c
+index a59e9d20cb46b..4b1773c1fb95f 100644
+--- a/sound/soc/qcom/lpass-platform.c
++++ b/sound/soc/qcom/lpass-platform.c
+@@ -524,7 +524,7 @@ static int lpass_platform_pcmops_trigger(struct snd_soc_component *component,
+ return -EINVAL;
+ }
+
+- ret = regmap_update_bits(map, reg_irqclr, val_irqclr, val_irqclr);
++ ret = regmap_write_bits(map, reg_irqclr, val_irqclr, val_irqclr);
+ if (ret) {
+ dev_err(soc_runtime->dev, "error writing to irqclear reg: %d\n", ret);
+ return ret;
+@@ -665,7 +665,7 @@ static irqreturn_t lpass_dma_interrupt_handler(
+ return -EINVAL;
+ }
+ if (interrupts & LPAIF_IRQ_PER(chan)) {
+- rv = regmap_update_bits(map, reg, mask, (LPAIF_IRQ_PER(chan) | val));
++ rv = regmap_write_bits(map, reg, mask, (LPAIF_IRQ_PER(chan) | val));
+ if (rv) {
+ dev_err(soc_runtime->dev,
+ "error writing to irqclear reg: %d\n", rv);
+@@ -676,7 +676,7 @@ static irqreturn_t lpass_dma_interrupt_handler(
+ }
+
+ if (interrupts & LPAIF_IRQ_XRUN(chan)) {
+- rv = regmap_update_bits(map, reg, mask, (LPAIF_IRQ_XRUN(chan) | val));
++ rv = regmap_write_bits(map, reg, mask, (LPAIF_IRQ_XRUN(chan) | val));
+ if (rv) {
+ dev_err(soc_runtime->dev,
+ "error writing to irqclear reg: %d\n", rv);
+@@ -688,7 +688,7 @@ static irqreturn_t lpass_dma_interrupt_handler(
+ }
+
+ if (interrupts & LPAIF_IRQ_ERR(chan)) {
+- rv = regmap_update_bits(map, reg, mask, (LPAIF_IRQ_ERR(chan) | val));
++ rv = regmap_write_bits(map, reg, mask, (LPAIF_IRQ_ERR(chan) | val));
+ if (rv) {
+ dev_err(soc_runtime->dev,
+ "error writing to irqclear reg: %d\n", rv);
+diff --git a/sound/soc/qcom/qdsp6/q6asm-dai.c b/sound/soc/qcom/qdsp6/q6asm-dai.c
+index 46f365528d501..b74b67720ef43 100644
+--- a/sound/soc/qcom/qdsp6/q6asm-dai.c
++++ b/sound/soc/qcom/qdsp6/q6asm-dai.c
+@@ -269,9 +269,7 @@ static int q6asm_dai_prepare(struct snd_soc_component *component,
+
+ if (ret < 0) {
+ dev_err(dev, "%s: q6asm_open_write failed\n", __func__);
+- q6asm_audio_client_free(prtd->audio_client);
+- prtd->audio_client = NULL;
+- return -ENOMEM;
++ goto open_err;
+ }
+
+ prtd->session_id = q6asm_get_session_id(prtd->audio_client);
+@@ -279,7 +277,7 @@ static int q6asm_dai_prepare(struct snd_soc_component *component,
+ prtd->session_id, substream->stream);
+ if (ret) {
+ dev_err(dev, "%s: stream reg failed ret:%d\n", __func__, ret);
+- return ret;
++ goto routing_err;
+ }
+
+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+@@ -301,10 +299,19 @@ static int q6asm_dai_prepare(struct snd_soc_component *component,
+ }
+ if (ret < 0)
+ dev_info(dev, "%s: CMD Format block failed\n", __func__);
++ else
++ prtd->state = Q6ASM_STREAM_RUNNING;
+
+- prtd->state = Q6ASM_STREAM_RUNNING;
++ return ret;
+
+- return 0;
++routing_err:
++ q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE);
++open_err:
++ q6asm_unmap_memory_regions(substream->stream, prtd->audio_client);
++ q6asm_audio_client_free(prtd->audio_client);
++ prtd->audio_client = NULL;
++
++ return ret;
+ }
+
+ static int q6asm_dai_trigger(struct snd_soc_component *component,
+diff --git a/sound/soc/qcom/qdsp6/q6routing.c b/sound/soc/qcom/qdsp6/q6routing.c
+index 3390ebef9549d..18c90bb4922be 100644
+--- a/sound/soc/qcom/qdsp6/q6routing.c
++++ b/sound/soc/qcom/qdsp6/q6routing.c
+@@ -492,9 +492,15 @@ static int msm_routing_put_audio_mixer(struct snd_kcontrol *kcontrol,
+ struct session_data *session = &data->sessions[session_id];
+
+ if (ucontrol->value.integer.value[0]) {
++ if (session->port_id == be_id)
++ return 0;
++
+ session->port_id = be_id;
+ snd_soc_dapm_mixer_update_power(dapm, kcontrol, 1, update);
+ } else {
++ if (session->port_id == -1 || session->port_id != be_id)
++ return 0;
++
+ session->port_id = -1;
+ snd_soc_dapm_mixer_update_power(dapm, kcontrol, 0, update);
+ }
+diff --git a/sound/soc/rockchip/rockchip_i2s.c b/sound/soc/rockchip/rockchip_i2s.c
+index 7e89f5b0c237f..2880a05376469 100644
+--- a/sound/soc/rockchip/rockchip_i2s.c
++++ b/sound/soc/rockchip/rockchip_i2s.c
+@@ -717,19 +717,23 @@ static int rockchip_i2s_probe(struct platform_device *pdev)
+ i2s->mclk = devm_clk_get(&pdev->dev, "i2s_clk");
+ if (IS_ERR(i2s->mclk)) {
+ dev_err(&pdev->dev, "Can't retrieve i2s master clock\n");
+- return PTR_ERR(i2s->mclk);
++ ret = PTR_ERR(i2s->mclk);
++ goto err_clk;
+ }
+
+ regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
+- if (IS_ERR(regs))
+- return PTR_ERR(regs);
++ if (IS_ERR(regs)) {
++ ret = PTR_ERR(regs);
++ goto err_clk;
++ }
+
+ i2s->regmap = devm_regmap_init_mmio(&pdev->dev, regs,
+ &rockchip_i2s_regmap_config);
+ if (IS_ERR(i2s->regmap)) {
+ dev_err(&pdev->dev,
+ "Failed to initialise managed register map\n");
+- return PTR_ERR(i2s->regmap);
++ ret = PTR_ERR(i2s->regmap);
++ goto err_clk;
+ }
+
+ i2s->bclk_ratio = 64;
+@@ -769,7 +773,8 @@ err_suspend:
+ i2s_runtime_suspend(&pdev->dev);
+ err_pm_disable:
+ pm_runtime_disable(&pdev->dev);
+-
++err_clk:
++ clk_disable_unprepare(i2s->hclk);
+ return ret;
+ }
+
+diff --git a/sound/soc/samsung/idma.c b/sound/soc/samsung/idma.c
+index 66bcc2f97544b..c3f1b054e2389 100644
+--- a/sound/soc/samsung/idma.c
++++ b/sound/soc/samsung/idma.c
+@@ -360,6 +360,8 @@ static int preallocate_idma_buffer(struct snd_pcm *pcm, int stream)
+ buf->addr = idma.lp_tx_addr;
+ buf->bytes = idma_hardware.buffer_bytes_max;
+ buf->area = (unsigned char * __force)ioremap(buf->addr, buf->bytes);
++ if (!buf->area)
++ return -ENOMEM;
+
+ return 0;
+ }
+diff --git a/sound/soc/sh/fsi.c b/sound/soc/sh/fsi.c
+index cdf3b7f69ba70..e9a1eb6bdf66a 100644
+--- a/sound/soc/sh/fsi.c
++++ b/sound/soc/sh/fsi.c
+@@ -816,14 +816,27 @@ static int fsi_clk_enable(struct device *dev,
+ return ret;
+ }
+
+- clk_enable(clock->xck);
+- clk_enable(clock->ick);
+- clk_enable(clock->div);
++ ret = clk_enable(clock->xck);
++ if (ret)
++ goto err;
++ ret = clk_enable(clock->ick);
++ if (ret)
++ goto disable_xck;
++ ret = clk_enable(clock->div);
++ if (ret)
++ goto disable_ick;
+
+ clock->count++;
+ }
+
+ return ret;
++
++disable_ick:
++ clk_disable(clock->ick);
++disable_xck:
++ clk_disable(clock->xck);
++err:
++ return ret;
+ }
+
+ static int fsi_clk_disable(struct device *dev,
+diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c
+index 978bd0406729a..6a8fe0da7670b 100644
+--- a/sound/soc/sh/rcar/core.c
++++ b/sound/soc/sh/rcar/core.c
+@@ -1225,6 +1225,7 @@ int rsnd_node_count(struct rsnd_priv *priv, struct device_node *node, char *name
+ if (i < 0) {
+ dev_err(dev, "strange node numbering (%s)",
+ of_node_full_name(node));
++ of_node_put(np);
+ return 0;
+ }
+ i++;
+diff --git a/sound/soc/sh/rcar/dma.c b/sound/soc/sh/rcar/dma.c
+index 16c6e0265749b..03e0d4eca7815 100644
+--- a/sound/soc/sh/rcar/dma.c
++++ b/sound/soc/sh/rcar/dma.c
+@@ -102,7 +102,7 @@ static int rsnd_dmaen_stop(struct rsnd_mod *mod,
+ struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
+
+ if (dmaen->chan)
+- dmaengine_terminate_sync(dmaen->chan);
++ dmaengine_terminate_async(dmaen->chan);
+
+ return 0;
+ }
+diff --git a/sound/soc/sh/rz-ssi.c b/sound/soc/sh/rz-ssi.c
+index fa0cc08f70ec4..16de2633a8736 100644
+--- a/sound/soc/sh/rz-ssi.c
++++ b/sound/soc/sh/rz-ssi.c
+@@ -411,54 +411,56 @@ static int rz_ssi_pio_recv(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
+ {
+ struct snd_pcm_substream *substream = strm->substream;
+ struct snd_pcm_runtime *runtime;
++ bool done = false;
+ u16 *buf;
+ int fifo_samples;
+ int frames_left;
+- int samples = 0;
++ int samples;
+ int i;
+
+ if (!rz_ssi_stream_is_valid(ssi, strm))
+ return -EINVAL;
+
+ runtime = substream->runtime;
+- /* frames left in this period */
+- frames_left = runtime->period_size - (strm->buffer_pos %
+- runtime->period_size);
+- if (frames_left == 0)
+- frames_left = runtime->period_size;
+
+- /* Samples in RX FIFO */
+- fifo_samples = (rz_ssi_reg_readl(ssi, SSIFSR) >>
+- SSIFSR_RDC_SHIFT) & SSIFSR_RDC_MASK;
++ while (!done) {
++ /* frames left in this period */
++ frames_left = runtime->period_size -
++ (strm->buffer_pos % runtime->period_size);
++ if (!frames_left)
++ frames_left = runtime->period_size;
++
++ /* Samples in RX FIFO */
++ fifo_samples = (rz_ssi_reg_readl(ssi, SSIFSR) >>
++ SSIFSR_RDC_SHIFT) & SSIFSR_RDC_MASK;
++
++ /* Only read full frames at a time */
++ samples = 0;
++ while (frames_left && (fifo_samples >= runtime->channels)) {
++ samples += runtime->channels;
++ fifo_samples -= runtime->channels;
++ frames_left--;
++ }
+
+- /* Only read full frames at a time */
+- while (frames_left && (fifo_samples >= runtime->channels)) {
+- samples += runtime->channels;
+- fifo_samples -= runtime->channels;
+- frames_left--;
+- }
++ /* not enough samples yet */
++ if (!samples)
++ break;
+
+- /* not enough samples yet */
+- if (samples == 0)
+- return 0;
++ /* calculate new buffer index */
++ buf = (u16 *)(runtime->dma_area);
++ buf += strm->buffer_pos * runtime->channels;
+
+- /* calculate new buffer index */
+- buf = (u16 *)(runtime->dma_area);
+- buf += strm->buffer_pos * runtime->channels;
+-
+- /* Note, only supports 16-bit samples */
+- for (i = 0; i < samples; i++)
+- *buf++ = (u16)(rz_ssi_reg_readl(ssi, SSIFRDR) >> 16);
++ /* Note, only supports 16-bit samples */
++ for (i = 0; i < samples; i++)
++ *buf++ = (u16)(rz_ssi_reg_readl(ssi, SSIFRDR) >> 16);
+
+- rz_ssi_reg_mask_setl(ssi, SSIFSR, SSIFSR_RDF, 0);
+- rz_ssi_pointer_update(strm, samples / runtime->channels);
++ rz_ssi_reg_mask_setl(ssi, SSIFSR, SSIFSR_RDF, 0);
++ rz_ssi_pointer_update(strm, samples / runtime->channels);
+
+- /*
+- * If we finished this period, but there are more samples in
+- * the RX FIFO, call this function again
+- */
+- if (frames_left == 0 && fifo_samples >= runtime->channels)
+- rz_ssi_pio_recv(ssi, strm);
++ /* check if there are no more samples in the RX FIFO */
++ if (!(!frames_left && fifo_samples >= runtime->channels))
++ done = true;
++ }
+
+ return 0;
+ }
+@@ -975,6 +977,9 @@ static int rz_ssi_probe(struct platform_device *pdev)
+ ssi->playback.priv = ssi;
+ ssi->capture.priv = ssi;
+
++ spin_lock_init(&ssi->lock);
++ dev_set_drvdata(&pdev->dev, ssi);
++
+ /* Error Interrupt */
+ ssi->irq_int = platform_get_irq_byname(pdev, "int_req");
+ if (ssi->irq_int < 0)
+@@ -1022,8 +1027,6 @@ static int rz_ssi_probe(struct platform_device *pdev)
+ pm_runtime_enable(&pdev->dev);
+ pm_runtime_resume_and_get(&pdev->dev);
+
+- spin_lock_init(&ssi->lock);
+- dev_set_drvdata(&pdev->dev, ssi);
+ ret = devm_snd_soc_register_component(&pdev->dev, &rz_ssi_soc_component,
+ rz_ssi_soc_dai,
+ ARRAY_SIZE(rz_ssi_soc_dai));
+diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c
+index 36060800e9bd9..2050728063a15 100644
+--- a/sound/soc/soc-compress.c
++++ b/sound/soc/soc-compress.c
+@@ -534,6 +534,11 @@ int snd_soc_new_compress(struct snd_soc_pcm_runtime *rtd, int num)
+ return -EINVAL;
+ }
+
++ if (!codec_dai) {
++ dev_err(rtd->card->dev, "Missing codec\n");
++ return -EINVAL;
++ }
++
+ /* check client and interface hw capabilities */
+ if (snd_soc_dai_stream_valid(codec_dai, SNDRV_PCM_STREAM_PLAYBACK) &&
+ snd_soc_dai_stream_valid(cpu_dai, SNDRV_PCM_STREAM_PLAYBACK))
+diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
+index 80ca260595fda..5da762807824d 100644
+--- a/sound/soc/soc-core.c
++++ b/sound/soc/soc-core.c
+@@ -3208,7 +3208,7 @@ int snd_soc_get_dai_name(const struct of_phandle_args *args,
+ for_each_component(pos) {
+ struct device_node *component_of_node = soc_component_to_node(pos);
+
+- if (component_of_node != args->np)
++ if (component_of_node != args->np || !pos->num_dai)
+ continue;
+
+ ret = snd_soc_component_of_xlate_dai_name(pos, args, dai_name);
+diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
+index 59d07648a7e7f..0b166e074457f 100644
+--- a/sound/soc/soc-dapm.c
++++ b/sound/soc/soc-dapm.c
+@@ -1685,8 +1685,7 @@ static void dapm_seq_run(struct snd_soc_card *card,
+ switch (w->id) {
+ case snd_soc_dapm_pre:
+ if (!w->event)
+- list_for_each_entry_safe_continue(w, n, list,
+- power_list);
++ continue;
+
+ if (event == SND_SOC_DAPM_STREAM_START)
+ ret = w->event(w,
+@@ -1698,8 +1697,7 @@ static void dapm_seq_run(struct snd_soc_card *card,
+
+ case snd_soc_dapm_post:
+ if (!w->event)
+- list_for_each_entry_safe_continue(w, n, list,
+- power_list);
++ continue;
+
+ if (event == SND_SOC_DAPM_STREAM_START)
+ ret = w->event(w,
+@@ -2557,8 +2555,13 @@ static struct snd_soc_dapm_widget *dapm_find_widget(
+ return NULL;
+ }
+
+-static int snd_soc_dapm_set_pin(struct snd_soc_dapm_context *dapm,
+- const char *pin, int status)
++/*
++ * set the DAPM pin status:
++ * returns 1 when the value has been updated, 0 when unchanged, or a negative
++ * error code; called from kcontrol put callback
++ */
++static int __snd_soc_dapm_set_pin(struct snd_soc_dapm_context *dapm,
++ const char *pin, int status)
+ {
+ struct snd_soc_dapm_widget *w = dapm_find_widget(dapm, pin, true);
+ int ret = 0;
+@@ -2584,6 +2587,18 @@ static int snd_soc_dapm_set_pin(struct snd_soc_dapm_context *dapm,
+ return ret;
+ }
+
++/*
++ * similar as __snd_soc_dapm_set_pin(), but returns 0 when successful;
++ * called from several API functions below
++ */
++static int snd_soc_dapm_set_pin(struct snd_soc_dapm_context *dapm,
++ const char *pin, int status)
++{
++ int ret = __snd_soc_dapm_set_pin(dapm, pin, status);
++
++ return ret < 0 ? ret : 0;
++}
++
+ /**
+ * snd_soc_dapm_sync_unlocked - scan and power dapm paths
+ * @dapm: DAPM context
+@@ -3587,10 +3602,10 @@ int snd_soc_dapm_put_pin_switch(struct snd_kcontrol *kcontrol,
+ const char *pin = (const char *)kcontrol->private_value;
+ int ret;
+
+- if (ucontrol->value.integer.value[0])
+- ret = snd_soc_dapm_enable_pin(&card->dapm, pin);
+- else
+- ret = snd_soc_dapm_disable_pin(&card->dapm, pin);
++ mutex_lock_nested(&card->dapm_mutex, SND_SOC_DAPM_CLASS_RUNTIME);
++ ret = __snd_soc_dapm_set_pin(&card->dapm, pin,
++ !!ucontrol->value.integer.value[0]);
++ mutex_unlock(&card->dapm_mutex);
+
+ snd_soc_dapm_sync(&card->dapm);
+ return ret;
+diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c
+index 08eaa9ddf191e..f32ba64c5ddab 100644
+--- a/sound/soc/soc-ops.c
++++ b/sound/soc/soc-ops.c
+@@ -308,7 +308,7 @@ int snd_soc_put_volsw(struct snd_kcontrol *kcontrol,
+ unsigned int sign_bit = mc->sign_bit;
+ unsigned int mask = (1 << fls(max)) - 1;
+ unsigned int invert = mc->invert;
+- int err;
++ int err, ret;
+ bool type_2r = false;
+ unsigned int val2 = 0;
+ unsigned int val, val_mask;
+@@ -316,13 +316,27 @@ int snd_soc_put_volsw(struct snd_kcontrol *kcontrol,
+ if (sign_bit)
+ mask = BIT(sign_bit + 1) - 1;
+
+- val = ((ucontrol->value.integer.value[0] + min) & mask);
++ val = ucontrol->value.integer.value[0];
++ if (mc->platform_max && ((int)val + min) > mc->platform_max)
++ return -EINVAL;
++ if (val > max - min)
++ return -EINVAL;
++ if (val < 0)
++ return -EINVAL;
++ val = (val + min) & mask;
+ if (invert)
+ val = max - val;
+ val_mask = mask << shift;
+ val = val << shift;
+ if (snd_soc_volsw_is_stereo(mc)) {
+- val2 = ((ucontrol->value.integer.value[1] + min) & mask);
++ val2 = ucontrol->value.integer.value[1];
++ if (mc->platform_max && ((int)val2 + min) > mc->platform_max)
++ return -EINVAL;
++ if (val2 > max - min)
++ return -EINVAL;
++ if (val2 < 0)
++ return -EINVAL;
++ val2 = (val2 + min) & mask;
+ if (invert)
+ val2 = max - val2;
+ if (reg == reg2) {
+@@ -336,12 +350,18 @@ int snd_soc_put_volsw(struct snd_kcontrol *kcontrol,
+ err = snd_soc_component_update_bits(component, reg, val_mask, val);
+ if (err < 0)
+ return err;
++ ret = err;
+
+- if (type_2r)
++ if (type_2r) {
+ err = snd_soc_component_update_bits(component, reg2, val_mask,
+- val2);
++ val2);
++ /* Don't discard any error code or drop change flag */
++ if (ret == 0 || err < 0) {
++ ret = err;
++ }
++ }
+
+- return err;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(snd_soc_put_volsw);
+
+@@ -407,15 +427,24 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol,
+ int min = mc->min;
+ unsigned int mask = (1U << (fls(min + max) - 1)) - 1;
+ int err = 0;
++ int ret;
+ unsigned int val, val_mask;
+
++ val = ucontrol->value.integer.value[0];
++ if (mc->platform_max && val > mc->platform_max)
++ return -EINVAL;
++ if (val > max - min)
++ return -EINVAL;
++ if (val < 0)
++ return -EINVAL;
+ val_mask = mask << shift;
+- val = (ucontrol->value.integer.value[0] + min) & mask;
++ val = (val + min) & mask;
+ val = val << shift;
+
+ err = snd_soc_component_update_bits(component, reg, val_mask, val);
+ if (err < 0)
+ return err;
++ ret = err;
+
+ if (snd_soc_volsw_is_stereo(mc)) {
+ unsigned int val2;
+@@ -426,8 +455,13 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol,
+
+ err = snd_soc_component_update_bits(component, reg2, val_mask,
+ val2);
++
++ /* Don't discard any error code or drop change flag */
++ if (ret == 0 || err < 0) {
++ ret = err;
++ }
+ }
+- return err;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(snd_soc_put_volsw_sx);
+
+@@ -485,7 +519,15 @@ int snd_soc_put_volsw_range(struct snd_kcontrol *kcontrol,
+ unsigned int mask = (1 << fls(max)) - 1;
+ unsigned int invert = mc->invert;
+ unsigned int val, val_mask;
+- int ret;
++ int err, ret, tmp;
++
++ tmp = ucontrol->value.integer.value[0];
++ if (tmp < 0)
++ return -EINVAL;
++ if (mc->platform_max && tmp > mc->platform_max)
++ return -EINVAL;
++ if (tmp > mc->max - mc->min + 1)
++ return -EINVAL;
+
+ if (invert)
+ val = (max - ucontrol->value.integer.value[0]) & mask;
+@@ -494,11 +536,20 @@ int snd_soc_put_volsw_range(struct snd_kcontrol *kcontrol,
+ val_mask = mask << shift;
+ val = val << shift;
+
+- ret = snd_soc_component_update_bits(component, reg, val_mask, val);
+- if (ret < 0)
+- return ret;
++ err = snd_soc_component_update_bits(component, reg, val_mask, val);
++ if (err < 0)
++ return err;
++ ret = err;
+
+ if (snd_soc_volsw_is_stereo(mc)) {
++ tmp = ucontrol->value.integer.value[1];
++ if (tmp < 0)
++ return -EINVAL;
++ if (mc->platform_max && tmp > mc->platform_max)
++ return -EINVAL;
++ if (tmp > mc->max - mc->min + 1)
++ return -EINVAL;
++
+ if (invert)
+ val = (max - ucontrol->value.integer.value[1]) & mask;
+ else
+@@ -506,8 +557,12 @@ int snd_soc_put_volsw_range(struct snd_kcontrol *kcontrol,
+ val_mask = mask << shift;
+ val = val << shift;
+
+- ret = snd_soc_component_update_bits(component, rreg, val_mask,
++ err = snd_soc_component_update_bits(component, rreg, val_mask,
+ val);
++ /* Don't discard any error code or drop change flag */
++ if (ret == 0 || err < 0) {
++ ret = err;
++ }
+ }
+
+ return ret;
+@@ -856,8 +911,11 @@ int snd_soc_put_xr_sx(struct snd_kcontrol *kcontrol,
+ unsigned long mask = (1UL<<mc->nbits)-1;
+ long max = mc->max;
+ long val = ucontrol->value.integer.value[0];
++ int ret = 0;
+ unsigned int i;
+
++ if (val < mc->min || val > mc->max)
++ return -EINVAL;
+ if (invert)
+ val = max - val;
+ val &= mask;
+@@ -868,9 +926,11 @@ int snd_soc_put_xr_sx(struct snd_kcontrol *kcontrol,
+ regmask, regval);
+ if (err < 0)
+ return err;
++ if (err > 0)
++ ret = err;
+ }
+
+- return 0;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(snd_soc_put_xr_sx);
+
+diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c
+index f6e5ac3e03140..eff8d4f715611 100644
+--- a/sound/soc/soc-topology.c
++++ b/sound/soc/soc-topology.c
+@@ -510,7 +510,8 @@ static int soc_tplg_kcontrol_bind_io(struct snd_soc_tplg_ctl_hdr *hdr,
+
+ if (le32_to_cpu(hdr->ops.info) == SND_SOC_TPLG_CTL_BYTES
+ && k->iface & SNDRV_CTL_ELEM_IFACE_MIXER
+- && k->access & SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE
++ && (k->access & SNDRV_CTL_ELEM_ACCESS_TLV_READ
++ || k->access & SNDRV_CTL_ELEM_ACCESS_TLV_WRITE)
+ && k->access & SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK) {
+ struct soc_bytes_ext *sbe;
+ struct snd_soc_tplg_bytes_control *be;
+@@ -1480,12 +1481,12 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
+ template.num_kcontrols = le32_to_cpu(w->num_kcontrols);
+ kc = devm_kcalloc(tplg->dev, le32_to_cpu(w->num_kcontrols), sizeof(*kc), GFP_KERNEL);
+ if (!kc)
+- goto err;
++ goto hdr_err;
+
+ kcontrol_type = devm_kcalloc(tplg->dev, le32_to_cpu(w->num_kcontrols), sizeof(unsigned int),
+ GFP_KERNEL);
+ if (!kcontrol_type)
+- goto err;
++ goto hdr_err;
+
+ for (i = 0; i < w->num_kcontrols; i++) {
+ control_hdr = (struct snd_soc_tplg_ctl_hdr *)tplg->pos;
+@@ -2674,6 +2675,7 @@ EXPORT_SYMBOL_GPL(snd_soc_tplg_component_load);
+ /* remove dynamic controls from the component driver */
+ int snd_soc_tplg_component_remove(struct snd_soc_component *comp)
+ {
++ struct snd_card *card = comp->card->snd_card;
+ struct snd_soc_dobj *dobj, *next_dobj;
+ int pass = SOC_TPLG_PASS_END;
+
+@@ -2681,6 +2683,7 @@ int snd_soc_tplg_component_remove(struct snd_soc_component *comp)
+ while (pass >= SOC_TPLG_PASS_START) {
+
+ /* remove mixer controls */
++ down_write(&card->controls_rwsem);
+ list_for_each_entry_safe(dobj, next_dobj, &comp->dobj_list,
+ list) {
+
+@@ -2719,6 +2722,7 @@ int snd_soc_tplg_component_remove(struct snd_soc_component *comp)
+ break;
+ }
+ }
++ up_write(&card->controls_rwsem);
+ pass--;
+ }
+
+diff --git a/sound/soc/sof/imx/imx8m.c b/sound/soc/sof/imx/imx8m.c
+index 892e1482f97fa..b3d3edc36bb00 100644
+--- a/sound/soc/sof/imx/imx8m.c
++++ b/sound/soc/sof/imx/imx8m.c
+@@ -191,6 +191,7 @@ static int imx8m_probe(struct snd_sof_dev *sdev)
+ }
+
+ ret = of_address_to_resource(res_node, 0, &res);
++ of_node_put(res_node);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to get reserved region address\n");
+ goto exit_pdev_unregister;
+diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig
+index 88b6176af021c..d83e1a36707af 100644
+--- a/sound/soc/sof/intel/Kconfig
++++ b/sound/soc/sof/intel/Kconfig
+@@ -84,6 +84,7 @@ if SND_SOC_SOF_PCI
+ config SND_SOC_SOF_MERRIFIELD
+ tristate "SOF support for Tangier/Merrifield"
+ default SND_SOC_SOF_PCI
++ select SND_SOC_SOF_PCI_DEV
+ select SND_SOC_SOF_INTEL_ATOM_HIFI_EP
+ help
+ This adds support for Sound Open Firmware for Intel(R) platforms
+diff --git a/sound/soc/sof/intel/hda-bus.c b/sound/soc/sof/intel/hda-bus.c
+index 30025d3c16b6e..0862ff8b66273 100644
+--- a/sound/soc/sof/intel/hda-bus.c
++++ b/sound/soc/sof/intel/hda-bus.c
+@@ -10,6 +10,8 @@
+ #include <linux/io.h>
+ #include <sound/hdaudio.h>
+ #include <sound/hda_i915.h>
++#include <sound/hda_codec.h>
++#include <sound/hda_register.h>
+ #include "../sof-priv.h"
+ #include "hda.h"
+
+@@ -21,6 +23,18 @@
+ #endif
+
+ #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
++static void update_codec_wake_enable(struct hdac_bus *bus, unsigned int addr, bool link_power)
++{
++ unsigned int mask = snd_hdac_chip_readw(bus, WAKEEN);
++
++ if (link_power)
++ mask &= ~BIT(addr);
++ else
++ mask |= BIT(addr);
++
++ snd_hdac_chip_updatew(bus, WAKEEN, STATESTS_INT_MASK, mask);
++}
++
+ static void sof_hda_bus_link_power(struct hdac_device *codec, bool enable)
+ {
+ struct hdac_bus *bus = codec->bus;
+@@ -41,6 +55,9 @@ static void sof_hda_bus_link_power(struct hdac_device *codec, bool enable)
+ */
+ if (codec->addr == HDA_IDISP_ADDR && !enable)
+ snd_hdac_display_power(bus, HDA_CODEC_IDX_CONTROLLER, false);
++
++ /* WAKEEN needs to be set for disabled links */
++ update_codec_wake_enable(bus, codec->addr, enable);
+ }
+
+ static const struct hdac_bus_ops bus_core_ops = {
+diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c
+index c1f9f0f584647..6704dbcd101cd 100644
+--- a/sound/soc/sof/intel/hda-dai.c
++++ b/sound/soc/sof/intel/hda-dai.c
+@@ -68,6 +68,7 @@ static struct hdac_ext_stream *
+ return NULL;
+ }
+
++ spin_lock_irq(&bus->reg_lock);
+ list_for_each_entry(stream, &bus->stream_list, list) {
+ struct hdac_ext_stream *hstream =
+ stream_to_hdac_ext_stream(stream);
+@@ -107,12 +108,12 @@ static struct hdac_ext_stream *
+ * is updated in snd_hdac_ext_stream_decouple().
+ */
+ if (!res->decoupled)
+- snd_hdac_ext_stream_decouple(bus, res, true);
+- spin_lock_irq(&bus->reg_lock);
++ snd_hdac_ext_stream_decouple_locked(bus, res, true);
++
+ res->link_locked = 1;
+ res->link_substream = substream;
+- spin_unlock_irq(&bus->reg_lock);
+ }
++ spin_unlock_irq(&bus->reg_lock);
+
+ return res;
+ }
+diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c
+index 623cf291e2074..262a70791a8f8 100644
+--- a/sound/soc/sof/intel/hda-dsp.c
++++ b/sound/soc/sof/intel/hda-dsp.c
+@@ -623,8 +623,7 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend)
+ hda_dsp_ipc_int_disable(sdev);
+
+ #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+- if (runtime_suspend)
+- hda_codec_jack_wake_enable(sdev, true);
++ hda_codec_jack_wake_enable(sdev, runtime_suspend);
+
+ /* power down all hda link */
+ snd_hdac_ext_bus_link_power_down_all(bus);
+diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c
+index 6f4771bf9de34..14469e087b00d 100644
+--- a/sound/soc/sof/intel/hda-loader.c
++++ b/sound/soc/sof/intel/hda-loader.c
+@@ -48,7 +48,7 @@ static struct hdac_ext_stream *cl_stream_prepare(struct snd_sof_dev *sdev, unsig
+ ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV_SG, &pci->dev, size, dmab);
+ if (ret < 0) {
+ dev_err(sdev->dev, "error: memory alloc failed: %d\n", ret);
+- goto error;
++ goto out_put;
+ }
+
+ hstream->period_bytes = 0;/* initialize period_bytes */
+@@ -59,22 +59,23 @@ static struct hdac_ext_stream *cl_stream_prepare(struct snd_sof_dev *sdev, unsig
+ ret = hda_dsp_iccmax_stream_hw_params(sdev, dsp_stream, dmab, NULL);
+ if (ret < 0) {
+ dev_err(sdev->dev, "error: iccmax stream prepare failed: %d\n", ret);
+- goto error;
++ goto out_free;
+ }
+ } else {
+ ret = hda_dsp_stream_hw_params(sdev, dsp_stream, dmab, NULL);
+ if (ret < 0) {
+ dev_err(sdev->dev, "error: hdac prepare failed: %d\n", ret);
+- goto error;
++ goto out_free;
+ }
+ hda_dsp_stream_spib_config(sdev, dsp_stream, HDA_DSP_SPIB_ENABLE, size);
+ }
+
+ return dsp_stream;
+
+-error:
+- hda_dsp_stream_put(sdev, direction, hstream->stream_tag);
++out_free:
+ snd_dma_free_pages(dmab);
++out_put:
++ hda_dsp_stream_put(sdev, direction, hstream->stream_tag);
+ return ERR_PTR(ret);
+ }
+
+diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c
+index cc8ddef37f37b..d81e87e304e42 100644
+--- a/sound/soc/sof/intel/hda-pcm.c
++++ b/sound/soc/sof/intel/hda-pcm.c
+@@ -242,6 +242,7 @@ int hda_dsp_pcm_open(struct snd_sof_dev *sdev,
+ runtime->hw.info &= ~SNDRV_PCM_INFO_PAUSE;
+
+ if (hda_always_enable_dmi_l1 ||
++ direction == SNDRV_PCM_STREAM_PLAYBACK ||
+ spcm->stream[substream->stream].d0i3_compatible)
+ flags |= SOF_HDA_STREAM_DMI_L1_COMPATIBLE;
+
+diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c
+index f60e2c57d3d0c..ddf70902e53c4 100644
+--- a/sound/soc/sof/intel/hda.c
++++ b/sound/soc/sof/intel/hda.c
+@@ -696,6 +696,20 @@ skip_soundwire:
+ return 0;
+ }
+
++static void hda_check_for_state_change(struct snd_sof_dev *sdev)
++{
++#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
++ struct hdac_bus *bus = sof_to_bus(sdev);
++ unsigned int codec_mask;
++
++ codec_mask = snd_hdac_chip_readw(bus, STATESTS);
++ if (codec_mask) {
++ hda_codec_jack_check(sdev);
++ snd_hdac_chip_writew(bus, STATESTS, codec_mask);
++ }
++#endif
++}
++
+ static irqreturn_t hda_dsp_interrupt_handler(int irq, void *context)
+ {
+ struct snd_sof_dev *sdev = context;
+@@ -737,6 +751,8 @@ static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context)
+ if (hda_sdw_check_wakeen_irq(sdev))
+ hda_sdw_process_wakeen(sdev);
+
++ hda_check_for_state_change(sdev);
++
+ /* enable GIE interrupt */
+ snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+ SOF_HDA_INTCTL,
+@@ -1056,7 +1072,7 @@ static bool link_slaves_found(struct snd_sof_dev *sdev,
+ struct hdac_bus *bus = sof_to_bus(sdev);
+ struct sdw_intel_slave_id *ids = sdw->ids;
+ int num_slaves = sdw->num_slaves;
+- unsigned int part_id, link_id, unique_id, mfg_id;
++ unsigned int part_id, link_id, unique_id, mfg_id, version;
+ int i, j, k;
+
+ for (i = 0; i < link->num_adr; i++) {
+@@ -1066,12 +1082,14 @@ static bool link_slaves_found(struct snd_sof_dev *sdev,
+ mfg_id = SDW_MFG_ID(adr);
+ part_id = SDW_PART_ID(adr);
+ link_id = SDW_DISCO_LINK_ID(adr);
++ version = SDW_VERSION(adr);
+
+ for (j = 0; j < num_slaves; j++) {
+ /* find out how many identical parts were reported on that link */
+ if (ids[j].link_id == link_id &&
+ ids[j].id.part_id == part_id &&
+- ids[j].id.mfg_id == mfg_id)
++ ids[j].id.mfg_id == mfg_id &&
++ ids[j].id.sdw_version == version)
+ reported_part_count++;
+ }
+
+@@ -1080,21 +1098,24 @@ static bool link_slaves_found(struct snd_sof_dev *sdev,
+
+ if (ids[j].link_id != link_id ||
+ ids[j].id.part_id != part_id ||
+- ids[j].id.mfg_id != mfg_id)
++ ids[j].id.mfg_id != mfg_id ||
++ ids[j].id.sdw_version != version)
+ continue;
+
+ /* find out how many identical parts are expected */
+ for (k = 0; k < link->num_adr; k++) {
+ u64 adr2 = link->adr_d[k].adr;
+- unsigned int part_id2, link_id2, mfg_id2;
++ unsigned int part_id2, link_id2, mfg_id2, version2;
+
+ mfg_id2 = SDW_MFG_ID(adr2);
+ part_id2 = SDW_PART_ID(adr2);
+ link_id2 = SDW_DISCO_LINK_ID(adr2);
++ version2 = SDW_VERSION(adr2);
+
+ if (link_id2 == link_id &&
+ part_id2 == part_id &&
+- mfg_id2 == mfg_id)
++ mfg_id2 == mfg_id &&
++ version2 == version)
+ expected_part_count++;
+ }
+
+diff --git a/sound/soc/sof/intel/pci-tgl.c b/sound/soc/sof/intel/pci-tgl.c
+index d04ce84fe7cc2..beb2fb3cd0141 100644
+--- a/sound/soc/sof/intel/pci-tgl.c
++++ b/sound/soc/sof/intel/pci-tgl.c
+@@ -117,8 +117,12 @@ static const struct pci_device_id sof_pci_ids[] = {
+ .driver_data = (unsigned long)&adls_desc},
+ { PCI_DEVICE(0x8086, 0x51c8), /* ADL-P */
+ .driver_data = (unsigned long)&adl_desc},
++ { PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */
++ .driver_data = (unsigned long)&adl_desc},
+ { PCI_DEVICE(0x8086, 0x51cc), /* ADL-M */
+ .driver_data = (unsigned long)&adl_desc},
++ { PCI_DEVICE(0x8086, 0x54c8), /* ADL-N */
++ .driver_data = (unsigned long)&adl_desc},
+ { 0, }
+ };
+ MODULE_DEVICE_TABLE(pci, sof_pci_ids);
+diff --git a/sound/soc/sof/sof-pci-dev.c b/sound/soc/sof/sof-pci-dev.c
+index bc9e707656789..b773289c928d4 100644
+--- a/sound/soc/sof/sof-pci-dev.c
++++ b/sound/soc/sof/sof-pci-dev.c
+@@ -129,6 +129,11 @@ int sof_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+
+ dev_dbg(&pci->dev, "PCI DSP detected");
+
++ if (!desc) {
++ dev_err(dev, "error: no matching PCI descriptor\n");
++ return -ENODEV;
++ }
++
+ if (!desc->ops) {
+ dev_err(dev, "error: no matching PCI descriptor ops\n");
+ return -ENODEV;
+diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
+index cc9585bfa4e9f..1bb2dcf37ffe9 100644
+--- a/sound/soc/sof/topology.c
++++ b/sound/soc/sof/topology.c
+@@ -2598,6 +2598,15 @@ static int sof_widget_unload(struct snd_soc_component *scomp,
+
+ /* power down the pipeline schedule core */
+ pipeline = swidget->private;
++
++ /*
++ * Runtime PM should still function normally if topology loading fails and
++ * it's components are unloaded. Do not power down the primary core so that the
++ * CTX_SAVE IPC can succeed during runtime suspend.
++ */
++ if (pipeline->core == SOF_DSP_PRIMARY_CORE)
++ break;
++
+ ret = snd_sof_dsp_core_power_down(sdev, 1 << pipeline->core);
+ if (ret < 0)
+ dev_err(scomp->dev, "error: powering down pipeline schedule core %d\n",
+diff --git a/sound/soc/sti/uniperif_player.c b/sound/soc/sti/uniperif_player.c
+index 2ed92c990b97c..dd9013c476649 100644
+--- a/sound/soc/sti/uniperif_player.c
++++ b/sound/soc/sti/uniperif_player.c
+@@ -91,7 +91,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
+ SET_UNIPERIF_ITM_BCLR_FIFO_ERROR(player);
+
+ /* Stop the player */
+- snd_pcm_stop_xrun(player->substream);
++ snd_pcm_stop(player->substream, SNDRV_PCM_STATE_XRUN);
+ }
+
+ ret = IRQ_HANDLED;
+@@ -105,7 +105,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
+ SET_UNIPERIF_ITM_BCLR_DMA_ERROR(player);
+
+ /* Stop the player */
+- snd_pcm_stop_xrun(player->substream);
++ snd_pcm_stop(player->substream, SNDRV_PCM_STATE_XRUN);
+
+ ret = IRQ_HANDLED;
+ }
+@@ -138,7 +138,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
+ dev_err(player->dev, "Underflow recovery failed\n");
+
+ /* Stop the player */
+- snd_pcm_stop_xrun(player->substream);
++ snd_pcm_stop(player->substream, SNDRV_PCM_STATE_XRUN);
+
+ ret = IRQ_HANDLED;
+ }
+diff --git a/sound/soc/sti/uniperif_reader.c b/sound/soc/sti/uniperif_reader.c
+index 136059331211d..065c5f0d1f5f0 100644
+--- a/sound/soc/sti/uniperif_reader.c
++++ b/sound/soc/sti/uniperif_reader.c
+@@ -65,7 +65,7 @@ static irqreturn_t uni_reader_irq_handler(int irq, void *dev_id)
+ if (unlikely(status & UNIPERIF_ITS_FIFO_ERROR_MASK(reader))) {
+ dev_err(reader->dev, "FIFO error detected\n");
+
+- snd_pcm_stop_xrun(reader->substream);
++ snd_pcm_stop(reader->substream, SNDRV_PCM_STATE_XRUN);
+
+ ret = IRQ_HANDLED;
+ }
+diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c
+index 6254bacad6eb7..717f45a83445c 100644
+--- a/sound/soc/stm/stm32_i2s.c
++++ b/sound/soc/stm/stm32_i2s.c
+@@ -700,7 +700,7 @@ static int stm32_i2s_configure_clock(struct snd_soc_dai *cpu_dai,
+ if (ret < 0)
+ return ret;
+
+- nb_bits = frame_len * ((cgfr & I2S_CGFR_CHLEN) + 1);
++ nb_bits = frame_len * (FIELD_GET(I2S_CGFR_CHLEN, cgfr) + 1);
+ ret = stm32_i2s_calc_clk_div(i2s, i2s_clock_rate,
+ (nb_bits * rate));
+ if (ret)
+diff --git a/sound/soc/tegra/tegra186_dspk.c b/sound/soc/tegra/tegra186_dspk.c
+index 8ee9a77bd83d3..a74c980ee7753 100644
+--- a/sound/soc/tegra/tegra186_dspk.c
++++ b/sound/soc/tegra/tegra186_dspk.c
+@@ -26,51 +26,162 @@ static const struct reg_default tegra186_dspk_reg_defaults[] = {
+ { TEGRA186_DSPK_CODEC_CTRL, 0x03000000 },
+ };
+
+-static int tegra186_dspk_get_control(struct snd_kcontrol *kcontrol,
++static int tegra186_dspk_get_fifo_th(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
+ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
+
+- if (strstr(kcontrol->id.name, "FIFO Threshold"))
+- ucontrol->value.integer.value[0] = dspk->rx_fifo_th;
+- else if (strstr(kcontrol->id.name, "OSR Value"))
+- ucontrol->value.integer.value[0] = dspk->osr_val;
+- else if (strstr(kcontrol->id.name, "LR Polarity Select"))
+- ucontrol->value.integer.value[0] = dspk->lrsel;
+- else if (strstr(kcontrol->id.name, "Channel Select"))
+- ucontrol->value.integer.value[0] = dspk->ch_sel;
+- else if (strstr(kcontrol->id.name, "Mono To Stereo"))
+- ucontrol->value.integer.value[0] = dspk->mono_to_stereo;
+- else if (strstr(kcontrol->id.name, "Stereo To Mono"))
+- ucontrol->value.integer.value[0] = dspk->stereo_to_mono;
++ ucontrol->value.integer.value[0] = dspk->rx_fifo_th;
+
+ return 0;
+ }
+
+-static int tegra186_dspk_put_control(struct snd_kcontrol *kcontrol,
++static int tegra186_dspk_put_fifo_th(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
+ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
+- int val = ucontrol->value.integer.value[0];
+-
+- if (strstr(kcontrol->id.name, "FIFO Threshold"))
+- dspk->rx_fifo_th = val;
+- else if (strstr(kcontrol->id.name, "OSR Value"))
+- dspk->osr_val = val;
+- else if (strstr(kcontrol->id.name, "LR Polarity Select"))
+- dspk->lrsel = val;
+- else if (strstr(kcontrol->id.name, "Channel Select"))
+- dspk->ch_sel = val;
+- else if (strstr(kcontrol->id.name, "Mono To Stereo"))
+- dspk->mono_to_stereo = val;
+- else if (strstr(kcontrol->id.name, "Stereo To Mono"))
+- dspk->stereo_to_mono = val;
++ int value = ucontrol->value.integer.value[0];
++
++ if (value == dspk->rx_fifo_th)
++ return 0;
++
++ dspk->rx_fifo_th = value;
++
++ return 1;
++}
++
++static int tegra186_dspk_get_osr_val(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++
++ ucontrol->value.enumerated.item[0] = dspk->osr_val;
+
+ return 0;
+ }
+
++static int tegra186_dspk_put_osr_val(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dspk->osr_val)
++ return 0;
++
++ dspk->osr_val = value;
++
++ return 1;
++}
++
++static int tegra186_dspk_get_pol_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++
++ ucontrol->value.enumerated.item[0] = dspk->lrsel;
++
++ return 0;
++}
++
++static int tegra186_dspk_put_pol_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dspk->lrsel)
++ return 0;
++
++ dspk->lrsel = value;
++
++ return 1;
++}
++
++static int tegra186_dspk_get_ch_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++
++ ucontrol->value.enumerated.item[0] = dspk->ch_sel;
++
++ return 0;
++}
++
++static int tegra186_dspk_put_ch_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dspk->ch_sel)
++ return 0;
++
++ dspk->ch_sel = value;
++
++ return 1;
++}
++
++static int tegra186_dspk_get_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++
++ ucontrol->value.enumerated.item[0] = dspk->mono_to_stereo;
++
++ return 0;
++}
++
++static int tegra186_dspk_put_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dspk->mono_to_stereo)
++ return 0;
++
++ dspk->mono_to_stereo = value;
++
++ return 1;
++}
++
++static int tegra186_dspk_get_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++
++ ucontrol->value.enumerated.item[0] = dspk->stereo_to_mono;
++
++ return 0;
++}
++
++static int tegra186_dspk_put_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *codec = snd_soc_kcontrol_component(kcontrol);
++ struct tegra186_dspk *dspk = snd_soc_component_get_drvdata(codec);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dspk->stereo_to_mono)
++ return 0;
++
++ dspk->stereo_to_mono = value;
++
++ return 1;
++}
++
+ static int __maybe_unused tegra186_dspk_runtime_suspend(struct device *dev)
+ {
+ struct tegra186_dspk *dspk = dev_get_drvdata(dev);
+@@ -279,17 +390,19 @@ static const struct soc_enum tegra186_dspk_lrsel_enum =
+ static const struct snd_kcontrol_new tegrat186_dspk_controls[] = {
+ SOC_SINGLE_EXT("FIFO Threshold", SND_SOC_NOPM, 0,
+ TEGRA186_DSPK_RX_FIFO_DEPTH - 1, 0,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_fifo_th, tegra186_dspk_put_fifo_th),
+ SOC_ENUM_EXT("OSR Value", tegra186_dspk_osr_enum,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_osr_val, tegra186_dspk_put_osr_val),
+ SOC_ENUM_EXT("LR Polarity Select", tegra186_dspk_lrsel_enum,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_pol_sel, tegra186_dspk_put_pol_sel),
+ SOC_ENUM_EXT("Channel Select", tegra186_dspk_ch_sel_enum,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_ch_sel, tegra186_dspk_put_ch_sel),
+ SOC_ENUM_EXT("Mono To Stereo", tegra186_dspk_mono_conv_enum,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_mono_to_stereo,
++ tegra186_dspk_put_mono_to_stereo),
+ SOC_ENUM_EXT("Stereo To Mono", tegra186_dspk_stereo_conv_enum,
+- tegra186_dspk_get_control, tegra186_dspk_put_control),
++ tegra186_dspk_get_stereo_to_mono,
++ tegra186_dspk_put_stereo_to_mono),
+ };
+
+ static const struct snd_soc_component_driver tegra186_dspk_cmpnt = {
+diff --git a/sound/soc/tegra/tegra210_admaif.c b/sound/soc/tegra/tegra210_admaif.c
+index bcccdf3ddc528..1a2e868a62209 100644
+--- a/sound/soc/tegra/tegra210_admaif.c
++++ b/sound/soc/tegra/tegra210_admaif.c
+@@ -424,46 +424,122 @@ static const struct snd_soc_dai_ops tegra_admaif_dai_ops = {
+ .trigger = tegra_admaif_trigger,
+ };
+
+-static int tegra_admaif_get_control(struct snd_kcontrol *kcontrol,
+- struct snd_ctl_elem_value *ucontrol)
++static int tegra210_admaif_pget_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++
++ ucontrol->value.enumerated.item[0] =
++ admaif->mono_to_stereo[ADMAIF_TX_PATH][ec->reg];
++
++ return 0;
++}
++
++static int tegra210_admaif_pput_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == admaif->mono_to_stereo[ADMAIF_TX_PATH][ec->reg])
++ return 0;
++
++ admaif->mono_to_stereo[ADMAIF_TX_PATH][ec->reg] = value;
++
++ return 1;
++}
++
++static int tegra210_admaif_cget_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++
++ ucontrol->value.enumerated.item[0] =
++ admaif->mono_to_stereo[ADMAIF_RX_PATH][ec->reg];
++
++ return 0;
++}
++
++static int tegra210_admaif_cput_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
+ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == admaif->mono_to_stereo[ADMAIF_RX_PATH][ec->reg])
++ return 0;
++
++ admaif->mono_to_stereo[ADMAIF_RX_PATH][ec->reg] = value;
++
++ return 1;
++}
++
++static int tegra210_admaif_pget_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
+ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
+- long *uctl_val = &ucontrol->value.integer.value[0];
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
+
+- if (strstr(kcontrol->id.name, "Playback Mono To Stereo"))
+- *uctl_val = admaif->mono_to_stereo[ADMAIF_TX_PATH][ec->reg];
+- else if (strstr(kcontrol->id.name, "Capture Mono To Stereo"))
+- *uctl_val = admaif->mono_to_stereo[ADMAIF_RX_PATH][ec->reg];
+- else if (strstr(kcontrol->id.name, "Playback Stereo To Mono"))
+- *uctl_val = admaif->stereo_to_mono[ADMAIF_TX_PATH][ec->reg];
+- else if (strstr(kcontrol->id.name, "Capture Stereo To Mono"))
+- *uctl_val = admaif->stereo_to_mono[ADMAIF_RX_PATH][ec->reg];
++ ucontrol->value.enumerated.item[0] =
++ admaif->stereo_to_mono[ADMAIF_TX_PATH][ec->reg];
+
+ return 0;
+ }
+
+-static int tegra_admaif_put_control(struct snd_kcontrol *kcontrol,
+- struct snd_ctl_elem_value *ucontrol)
++static int tegra210_admaif_pput_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
+ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == admaif->stereo_to_mono[ADMAIF_TX_PATH][ec->reg])
++ return 0;
++
++ admaif->stereo_to_mono[ADMAIF_TX_PATH][ec->reg] = value;
++
++ return 1;
++}
++
++static int tegra210_admaif_cget_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
+ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
+- int value = ucontrol->value.integer.value[0];
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
+
+- if (strstr(kcontrol->id.name, "Playback Mono To Stereo"))
+- admaif->mono_to_stereo[ADMAIF_TX_PATH][ec->reg] = value;
+- else if (strstr(kcontrol->id.name, "Capture Mono To Stereo"))
+- admaif->mono_to_stereo[ADMAIF_RX_PATH][ec->reg] = value;
+- else if (strstr(kcontrol->id.name, "Playback Stereo To Mono"))
+- admaif->stereo_to_mono[ADMAIF_TX_PATH][ec->reg] = value;
+- else if (strstr(kcontrol->id.name, "Capture Stereo To Mono"))
+- admaif->stereo_to_mono[ADMAIF_RX_PATH][ec->reg] = value;
++ ucontrol->value.enumerated.item[0] =
++ admaif->stereo_to_mono[ADMAIF_RX_PATH][ec->reg];
+
+ return 0;
+ }
+
++static int tegra210_admaif_cput_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *cmpnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra_admaif *admaif = snd_soc_component_get_drvdata(cmpnt);
++ struct soc_enum *ec = (struct soc_enum *)kcontrol->private_value;
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == admaif->stereo_to_mono[ADMAIF_RX_PATH][ec->reg])
++ return 0;
++
++ admaif->stereo_to_mono[ADMAIF_RX_PATH][ec->reg] = value;
++
++ return 1;
++}
++
+ static int tegra_admaif_dai_probe(struct snd_soc_dai *dai)
+ {
+ struct tegra_admaif *admaif = snd_soc_dai_get_drvdata(dai);
+@@ -559,17 +635,21 @@ static const char * const tegra_admaif_mono_conv_text[] = {
+ }
+
+ #define TEGRA_ADMAIF_CIF_CTRL(reg) \
+- NV_SOC_ENUM_EXT("ADMAIF" #reg " Playback Mono To Stereo", reg - 1,\
+- tegra_admaif_get_control, tegra_admaif_put_control, \
++ NV_SOC_ENUM_EXT("ADMAIF" #reg " Playback Mono To Stereo", reg - 1, \
++ tegra210_admaif_pget_mono_to_stereo, \
++ tegra210_admaif_pput_mono_to_stereo, \
+ tegra_admaif_mono_conv_text), \
+- NV_SOC_ENUM_EXT("ADMAIF" #reg " Playback Stereo To Mono", reg - 1,\
+- tegra_admaif_get_control, tegra_admaif_put_control, \
++ NV_SOC_ENUM_EXT("ADMAIF" #reg " Playback Stereo To Mono", reg - 1, \
++ tegra210_admaif_pget_stereo_to_mono, \
++ tegra210_admaif_pput_stereo_to_mono, \
+ tegra_admaif_stereo_conv_text), \
+- NV_SOC_ENUM_EXT("ADMAIF" #reg " Capture Mono To Stereo", reg - 1, \
+- tegra_admaif_get_control, tegra_admaif_put_control, \
++ NV_SOC_ENUM_EXT("ADMAIF" #reg " Capture Mono To Stereo", reg - 1, \
++ tegra210_admaif_cget_mono_to_stereo, \
++ tegra210_admaif_cput_mono_to_stereo, \
+ tegra_admaif_mono_conv_text), \
+- NV_SOC_ENUM_EXT("ADMAIF" #reg " Capture Stereo To Mono", reg - 1, \
+- tegra_admaif_get_control, tegra_admaif_put_control, \
++ NV_SOC_ENUM_EXT("ADMAIF" #reg " Capture Stereo To Mono", reg - 1, \
++ tegra210_admaif_cget_stereo_to_mono, \
++ tegra210_admaif_cput_stereo_to_mono, \
+ tegra_admaif_stereo_conv_text)
+
+ static struct snd_kcontrol_new tegra210_admaif_controls[] = {
+diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c
+index 66287a7c9865d..1b2f7cb8c6adc 100644
+--- a/sound/soc/tegra/tegra210_ahub.c
++++ b/sound/soc/tegra/tegra210_ahub.c
+@@ -62,6 +62,7 @@ static int tegra_ahub_put_value_enum(struct snd_kcontrol *kctl,
+ unsigned int *item = uctl->value.enumerated.item;
+ unsigned int value = e->values[item[0]];
+ unsigned int i, bit_pos, reg_idx = 0, reg_val = 0;
++ int change = 0;
+
+ if (item[0] >= e->items)
+ return -EINVAL;
+@@ -86,12 +87,14 @@ static int tegra_ahub_put_value_enum(struct snd_kcontrol *kctl,
+
+ /* Update widget power if state has changed */
+ if (snd_soc_component_test_bits(cmpnt, update[i].reg,
+- update[i].mask, update[i].val))
+- snd_soc_dapm_mux_update_power(dapm, kctl, item[0], e,
+- &update[i]);
++ update[i].mask,
++ update[i].val))
++ change |= snd_soc_dapm_mux_update_power(dapm, kctl,
++ item[0], e,
++ &update[i]);
+ }
+
+- return 0;
++ return change;
+ }
+
+ static struct snd_soc_dai_driver tegra210_ahub_dais[] = {
+diff --git a/sound/soc/tegra/tegra210_dmic.c b/sound/soc/tegra/tegra210_dmic.c
+index b096478cd2ef0..db95794530f46 100644
+--- a/sound/soc/tegra/tegra210_dmic.c
++++ b/sound/soc/tegra/tegra210_dmic.c
+@@ -156,51 +156,162 @@ static int tegra210_dmic_hw_params(struct snd_pcm_substream *substream,
+ return 0;
+ }
+
+-static int tegra210_dmic_get_control(struct snd_kcontrol *kcontrol,
++static int tegra210_dmic_get_boost_gain(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++
++ ucontrol->value.integer.value[0] = dmic->boost_gain;
++
++ return 0;
++}
++
++static int tegra210_dmic_put_boost_gain(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++ int value = ucontrol->value.integer.value[0];
++
++ if (value == dmic->boost_gain)
++ return 0;
++
++ dmic->boost_gain = value;
++
++ return 1;
++}
++
++static int tegra210_dmic_get_ch_select(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++
++ ucontrol->value.enumerated.item[0] = dmic->ch_select;
++
++ return 0;
++}
++
++static int tegra210_dmic_put_ch_select(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dmic->ch_select)
++ return 0;
++
++ dmic->ch_select = value;
++
++ return 1;
++}
++
++static int tegra210_dmic_get_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++
++ ucontrol->value.enumerated.item[0] = dmic->mono_to_stereo;
++
++ return 0;
++}
++
++static int tegra210_dmic_put_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dmic->mono_to_stereo)
++ return 0;
++
++ dmic->mono_to_stereo = value;
++
++ return 1;
++}
++
++static int tegra210_dmic_get_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++
++ ucontrol->value.enumerated.item[0] = dmic->stereo_to_mono;
++
++ return 0;
++}
++
++static int tegra210_dmic_put_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dmic->stereo_to_mono)
++ return 0;
++
++ dmic->stereo_to_mono = value;
++
++ return 1;
++}
++
++static int tegra210_dmic_get_osr_val(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
+ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
+
+- if (strstr(kcontrol->id.name, "Boost Gain Volume"))
+- ucontrol->value.integer.value[0] = dmic->boost_gain;
+- else if (strstr(kcontrol->id.name, "Channel Select"))
+- ucontrol->value.integer.value[0] = dmic->ch_select;
+- else if (strstr(kcontrol->id.name, "Mono To Stereo"))
+- ucontrol->value.integer.value[0] = dmic->mono_to_stereo;
+- else if (strstr(kcontrol->id.name, "Stereo To Mono"))
+- ucontrol->value.integer.value[0] = dmic->stereo_to_mono;
+- else if (strstr(kcontrol->id.name, "OSR Value"))
+- ucontrol->value.integer.value[0] = dmic->osr_val;
+- else if (strstr(kcontrol->id.name, "LR Polarity Select"))
+- ucontrol->value.integer.value[0] = dmic->lrsel;
++ ucontrol->value.enumerated.item[0] = dmic->osr_val;
+
+ return 0;
+ }
+
+-static int tegra210_dmic_put_control(struct snd_kcontrol *kcontrol,
++static int tegra210_dmic_put_osr_val(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
+ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
+- int value = ucontrol->value.integer.value[0];
++ unsigned int value = ucontrol->value.enumerated.item[0];
+
+- if (strstr(kcontrol->id.name, "Boost Gain Volume"))
+- dmic->boost_gain = value;
+- else if (strstr(kcontrol->id.name, "Channel Select"))
+- dmic->ch_select = ucontrol->value.integer.value[0];
+- else if (strstr(kcontrol->id.name, "Mono To Stereo"))
+- dmic->mono_to_stereo = value;
+- else if (strstr(kcontrol->id.name, "Stereo To Mono"))
+- dmic->stereo_to_mono = value;
+- else if (strstr(kcontrol->id.name, "OSR Value"))
+- dmic->osr_val = value;
+- else if (strstr(kcontrol->id.name, "LR Polarity Select"))
+- dmic->lrsel = value;
++ if (value == dmic->osr_val)
++ return 0;
++
++ dmic->osr_val = value;
++
++ return 1;
++}
++
++static int tegra210_dmic_get_pol_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++
++ ucontrol->value.enumerated.item[0] = dmic->lrsel;
+
+ return 0;
+ }
+
++static int tegra210_dmic_put_pol_sel(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *comp = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_dmic *dmic = snd_soc_component_get_drvdata(comp);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == dmic->lrsel)
++ return 0;
++
++ dmic->lrsel = value;
++
++ return 1;
++}
++
+ static const struct snd_soc_dai_ops tegra210_dmic_dai_ops = {
+ .hw_params = tegra210_dmic_hw_params,
+ };
+@@ -287,19 +398,22 @@ static const struct soc_enum tegra210_dmic_lrsel_enum =
+
+ static const struct snd_kcontrol_new tegra210_dmic_controls[] = {
+ SOC_SINGLE_EXT("Boost Gain Volume", 0, 0, MAX_BOOST_GAIN, 0,
+- tegra210_dmic_get_control, tegra210_dmic_put_control),
++ tegra210_dmic_get_boost_gain,
++ tegra210_dmic_put_boost_gain),
+ SOC_ENUM_EXT("Channel Select", tegra210_dmic_ch_enum,
+- tegra210_dmic_get_control, tegra210_dmic_put_control),
++ tegra210_dmic_get_ch_select, tegra210_dmic_put_ch_select),
+ SOC_ENUM_EXT("Mono To Stereo",
+- tegra210_dmic_mono_conv_enum, tegra210_dmic_get_control,
+- tegra210_dmic_put_control),
++ tegra210_dmic_mono_conv_enum,
++ tegra210_dmic_get_mono_to_stereo,
++ tegra210_dmic_put_mono_to_stereo),
+ SOC_ENUM_EXT("Stereo To Mono",
+- tegra210_dmic_stereo_conv_enum, tegra210_dmic_get_control,
+- tegra210_dmic_put_control),
++ tegra210_dmic_stereo_conv_enum,
++ tegra210_dmic_get_stereo_to_mono,
++ tegra210_dmic_put_stereo_to_mono),
+ SOC_ENUM_EXT("OSR Value", tegra210_dmic_osr_enum,
+- tegra210_dmic_get_control, tegra210_dmic_put_control),
++ tegra210_dmic_get_osr_val, tegra210_dmic_put_osr_val),
+ SOC_ENUM_EXT("LR Polarity Select", tegra210_dmic_lrsel_enum,
+- tegra210_dmic_get_control, tegra210_dmic_put_control),
++ tegra210_dmic_get_pol_sel, tegra210_dmic_put_pol_sel),
+ };
+
+ static const struct snd_soc_component_driver tegra210_dmic_compnt = {
+diff --git a/sound/soc/tegra/tegra210_i2s.c b/sound/soc/tegra/tegra210_i2s.c
+index 45f31ccb49d89..9552bbb939dd1 100644
+--- a/sound/soc/tegra/tegra210_i2s.c
++++ b/sound/soc/tegra/tegra210_i2s.c
+@@ -302,85 +302,235 @@ static int tegra210_i2s_set_tdm_slot(struct snd_soc_dai *dai,
+ return 0;
+ }
+
+-static int tegra210_i2s_set_dai_bclk_ratio(struct snd_soc_dai *dai,
+- unsigned int ratio)
++static int tegra210_i2s_get_loopback(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
+ {
+- struct tegra210_i2s *i2s = snd_soc_dai_get_drvdata(dai);
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
+
+- i2s->bclk_ratio = ratio;
++ ucontrol->value.integer.value[0] = i2s->loopback;
+
+ return 0;
+ }
+
+-static int tegra210_i2s_get_control(struct snd_kcontrol *kcontrol,
+- struct snd_ctl_elem_value *ucontrol)
++static int tegra210_i2s_put_loopback(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ int value = ucontrol->value.integer.value[0];
++
++ if (value == i2s->loopback)
++ return 0;
++
++ i2s->loopback = value;
++
++ regmap_update_bits(i2s->regmap, TEGRA210_I2S_CTRL, I2S_CTRL_LPBK_MASK,
++ i2s->loopback << I2S_CTRL_LPBK_SHIFT);
++
++ return 1;
++}
++
++static int tegra210_i2s_get_fsync_width(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
+ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
+- long *uctl_val = &ucontrol->value.integer.value[0];
+-
+- if (strstr(kcontrol->id.name, "Loopback"))
+- *uctl_val = i2s->loopback;
+- else if (strstr(kcontrol->id.name, "FSYNC Width"))
+- *uctl_val = i2s->fsync_width;
+- else if (strstr(kcontrol->id.name, "Capture Stereo To Mono"))
+- *uctl_val = i2s->stereo_to_mono[I2S_TX_PATH];
+- else if (strstr(kcontrol->id.name, "Capture Mono To Stereo"))
+- *uctl_val = i2s->mono_to_stereo[I2S_TX_PATH];
+- else if (strstr(kcontrol->id.name, "Playback Stereo To Mono"))
+- *uctl_val = i2s->stereo_to_mono[I2S_RX_PATH];
+- else if (strstr(kcontrol->id.name, "Playback Mono To Stereo"))
+- *uctl_val = i2s->mono_to_stereo[I2S_RX_PATH];
+- else if (strstr(kcontrol->id.name, "Playback FIFO Threshold"))
+- *uctl_val = i2s->rx_fifo_th;
+- else if (strstr(kcontrol->id.name, "BCLK Ratio"))
+- *uctl_val = i2s->bclk_ratio;
++
++ ucontrol->value.integer.value[0] = i2s->fsync_width;
+
+ return 0;
+ }
+
+-static int tegra210_i2s_put_control(struct snd_kcontrol *kcontrol,
+- struct snd_ctl_elem_value *ucontrol)
++static int tegra210_i2s_put_fsync_width(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
+ {
+ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
+ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
+ int value = ucontrol->value.integer.value[0];
+
+- if (strstr(kcontrol->id.name, "Loopback")) {
+- i2s->loopback = value;
++ if (value == i2s->fsync_width)
++ return 0;
+
+- regmap_update_bits(i2s->regmap, TEGRA210_I2S_CTRL,
+- I2S_CTRL_LPBK_MASK,
+- i2s->loopback << I2S_CTRL_LPBK_SHIFT);
++ i2s->fsync_width = value;
+
+- } else if (strstr(kcontrol->id.name, "FSYNC Width")) {
+- /*
+- * Frame sync width is used only for FSYNC modes and not
+- * applicable for LRCK modes. Reset value for this field is "0",
+- * which means the width is one bit clock wide.
+- * The width requirement may depend on the codec and in such
+- * cases mixer control is used to update custom values. A value
+- * of "N" here means, width is "N + 1" bit clock wide.
+- */
+- i2s->fsync_width = value;
+-
+- regmap_update_bits(i2s->regmap, TEGRA210_I2S_CTRL,
+- I2S_CTRL_FSYNC_WIDTH_MASK,
+- i2s->fsync_width << I2S_FSYNC_WIDTH_SHIFT);
+-
+- } else if (strstr(kcontrol->id.name, "Capture Stereo To Mono")) {
+- i2s->stereo_to_mono[I2S_TX_PATH] = value;
+- } else if (strstr(kcontrol->id.name, "Capture Mono To Stereo")) {
+- i2s->mono_to_stereo[I2S_TX_PATH] = value;
+- } else if (strstr(kcontrol->id.name, "Playback Stereo To Mono")) {
+- i2s->stereo_to_mono[I2S_RX_PATH] = value;
+- } else if (strstr(kcontrol->id.name, "Playback Mono To Stereo")) {
+- i2s->mono_to_stereo[I2S_RX_PATH] = value;
+- } else if (strstr(kcontrol->id.name, "Playback FIFO Threshold")) {
+- i2s->rx_fifo_th = value;
+- } else if (strstr(kcontrol->id.name, "BCLK Ratio")) {
+- i2s->bclk_ratio = value;
+- }
++ /*
++ * Frame sync width is used only for FSYNC modes and not
++ * applicable for LRCK modes. Reset value for this field is "0",
++ * which means the width is one bit clock wide.
++ * The width requirement may depend on the codec and in such
++ * cases mixer control is used to update custom values. A value
++ * of "N" here means, width is "N + 1" bit clock wide.
++ */
++ regmap_update_bits(i2s->regmap, TEGRA210_I2S_CTRL,
++ I2S_CTRL_FSYNC_WIDTH_MASK,
++ i2s->fsync_width << I2S_FSYNC_WIDTH_SHIFT);
++
++ return 1;
++}
++
++static int tegra210_i2s_cget_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.enumerated.item[0] = i2s->stereo_to_mono[I2S_TX_PATH];
++
++ return 0;
++}
++
++static int tegra210_i2s_cput_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == i2s->stereo_to_mono[I2S_TX_PATH])
++ return 0;
++
++ i2s->stereo_to_mono[I2S_TX_PATH] = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_cget_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.enumerated.item[0] = i2s->mono_to_stereo[I2S_TX_PATH];
++
++ return 0;
++}
++
++static int tegra210_i2s_cput_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == i2s->mono_to_stereo[I2S_TX_PATH])
++ return 0;
++
++ i2s->mono_to_stereo[I2S_TX_PATH] = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_pget_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.enumerated.item[0] = i2s->stereo_to_mono[I2S_RX_PATH];
++
++ return 0;
++}
++
++static int tegra210_i2s_pput_stereo_to_mono(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == i2s->stereo_to_mono[I2S_RX_PATH])
++ return 0;
++
++ i2s->stereo_to_mono[I2S_RX_PATH] = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_pget_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.enumerated.item[0] = i2s->mono_to_stereo[I2S_RX_PATH];
++
++ return 0;
++}
++
++static int tegra210_i2s_pput_mono_to_stereo(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ unsigned int value = ucontrol->value.enumerated.item[0];
++
++ if (value == i2s->mono_to_stereo[I2S_RX_PATH])
++ return 0;
++
++ i2s->mono_to_stereo[I2S_RX_PATH] = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_pget_fifo_th(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.integer.value[0] = i2s->rx_fifo_th;
++
++ return 0;
++}
++
++static int tegra210_i2s_pput_fifo_th(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ int value = ucontrol->value.integer.value[0];
++
++ if (value == i2s->rx_fifo_th)
++ return 0;
++
++ i2s->rx_fifo_th = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_get_bclk_ratio(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++
++ ucontrol->value.integer.value[0] = i2s->bclk_ratio;
++
++ return 0;
++}
++
++static int tegra210_i2s_put_bclk_ratio(struct snd_kcontrol *kcontrol,
++ struct snd_ctl_elem_value *ucontrol)
++{
++ struct snd_soc_component *compnt = snd_soc_kcontrol_component(kcontrol);
++ struct tegra210_i2s *i2s = snd_soc_component_get_drvdata(compnt);
++ int value = ucontrol->value.integer.value[0];
++
++ if (value == i2s->bclk_ratio)
++ return 0;
++
++ i2s->bclk_ratio = value;
++
++ return 1;
++}
++
++static int tegra210_i2s_set_dai_bclk_ratio(struct snd_soc_dai *dai,
++ unsigned int ratio)
++{
++ struct tegra210_i2s *i2s = snd_soc_dai_get_drvdata(dai);
++
++ i2s->bclk_ratio = ratio;
+
+ return 0;
+ }
+@@ -598,22 +748,28 @@ static const struct soc_enum tegra210_i2s_stereo_conv_enum =
+ tegra210_i2s_stereo_conv_text);
+
+ static const struct snd_kcontrol_new tegra210_i2s_controls[] = {
+- SOC_SINGLE_EXT("Loopback", 0, 0, 1, 0, tegra210_i2s_get_control,
+- tegra210_i2s_put_control),
+- SOC_SINGLE_EXT("FSYNC Width", 0, 0, 255, 0, tegra210_i2s_get_control,
+- tegra210_i2s_put_control),
++ SOC_SINGLE_EXT("Loopback", 0, 0, 1, 0, tegra210_i2s_get_loopback,
++ tegra210_i2s_put_loopback),
++ SOC_SINGLE_EXT("FSYNC Width", 0, 0, 255, 0,
++ tegra210_i2s_get_fsync_width,
++ tegra210_i2s_put_fsync_width),
+ SOC_ENUM_EXT("Capture Stereo To Mono", tegra210_i2s_stereo_conv_enum,
+- tegra210_i2s_get_control, tegra210_i2s_put_control),
++ tegra210_i2s_cget_stereo_to_mono,
++ tegra210_i2s_cput_stereo_to_mono),
+ SOC_ENUM_EXT("Capture Mono To Stereo", tegra210_i2s_mono_conv_enum,
+- tegra210_i2s_get_control, tegra210_i2s_put_control),
++ tegra210_i2s_cget_mono_to_stereo,
++ tegra210_i2s_cput_mono_to_stereo),
+ SOC_ENUM_EXT("Playback Stereo To Mono", tegra210_i2s_stereo_conv_enum,
+- tegra210_i2s_get_control, tegra210_i2s_put_control),
++ tegra210_i2s_pget_mono_to_stereo,
++ tegra210_i2s_pput_mono_to_stereo),
+ SOC_ENUM_EXT("Playback Mono To Stereo", tegra210_i2s_mono_conv_enum,
+- tegra210_i2s_get_control, tegra210_i2s_put_control),
++ tegra210_i2s_pget_stereo_to_mono,
++ tegra210_i2s_pput_stereo_to_mono),
+ SOC_SINGLE_EXT("Playback FIFO Threshold", 0, 0, I2S_RX_FIFO_DEPTH - 1,
+- 0, tegra210_i2s_get_control, tegra210_i2s_put_control),
+- SOC_SINGLE_EXT("BCLK Ratio", 0, 0, INT_MAX, 0, tegra210_i2s_get_control,
+- tegra210_i2s_put_control),
++ 0, tegra210_i2s_pget_fifo_th, tegra210_i2s_pput_fifo_th),
++ SOC_SINGLE_EXT("BCLK Ratio", 0, 0, INT_MAX, 0,
++ tegra210_i2s_get_bclk_ratio,
++ tegra210_i2s_put_bclk_ratio),
+ };
+
+ static const struct snd_soc_dapm_widget tegra210_i2s_widgets[] = {
+diff --git a/sound/soc/tegra/tegra_asoc_machine.c b/sound/soc/tegra/tegra_asoc_machine.c
+index 735909310a262..2e549b69061ca 100644
+--- a/sound/soc/tegra/tegra_asoc_machine.c
++++ b/sound/soc/tegra/tegra_asoc_machine.c
+@@ -116,16 +116,24 @@ static const struct snd_kcontrol_new tegra_machine_controls[] = {
+ SOC_DAPM_PIN_SWITCH("Headset Mic"),
+ SOC_DAPM_PIN_SWITCH("Internal Mic 1"),
+ SOC_DAPM_PIN_SWITCH("Internal Mic 2"),
++ SOC_DAPM_PIN_SWITCH("Headphones"),
++ SOC_DAPM_PIN_SWITCH("Mic Jack"),
+ };
+
+ int tegra_asoc_machine_init(struct snd_soc_pcm_runtime *rtd)
+ {
+ struct snd_soc_card *card = rtd->card;
+ struct tegra_machine *machine = snd_soc_card_get_drvdata(card);
++ const char *jack_name;
+ int err;
+
+ if (machine->gpiod_hp_det && machine->asoc->add_hp_jack) {
+- err = snd_soc_card_jack_new(card, "Headphones Jack",
++ if (machine->asoc->hp_jack_name)
++ jack_name = machine->asoc->hp_jack_name;
++ else
++ jack_name = "Headphones Jack";
++
++ err = snd_soc_card_jack_new(card, jack_name,
+ SND_JACK_HEADPHONE,
+ &tegra_machine_hp_jack,
+ tegra_machine_hp_jack_pins,
+@@ -341,9 +349,34 @@ tegra_machine_parse_phandle(struct device *dev, const char *name)
+ return np;
+ }
+
++static void tegra_machine_unregister_codec(void *pdev)
++{
++ platform_device_unregister(pdev);
++}
++
++static int tegra_machine_register_codec(struct device *dev, const char *name)
++{
++ struct platform_device *pdev;
++ int err;
++
++ if (!name)
++ return 0;
++
++ pdev = platform_device_register_simple(name, -1, NULL, 0);
++ if (IS_ERR(pdev))
++ return PTR_ERR(pdev);
++
++ err = devm_add_action_or_reset(dev, tegra_machine_unregister_codec,
++ pdev);
++ if (err)
++ return err;
++
++ return 0;
++}
++
+ int tegra_asoc_machine_probe(struct platform_device *pdev)
+ {
+- struct device_node *np_codec, *np_i2s;
++ struct device_node *np_codec, *np_i2s, *np_ac97;
+ const struct tegra_asoc_data *asoc;
+ struct device *dev = &pdev->dev;
+ struct tegra_machine *machine;
+@@ -404,17 +437,30 @@ int tegra_asoc_machine_probe(struct platform_device *pdev)
+ return err;
+ }
+
+- np_codec = tegra_machine_parse_phandle(dev, "nvidia,audio-codec");
+- if (IS_ERR(np_codec))
+- return PTR_ERR(np_codec);
++ if (asoc->set_ac97) {
++ err = tegra_machine_register_codec(dev, asoc->codec_dev_name);
++ if (err)
++ return err;
++
++ np_ac97 = tegra_machine_parse_phandle(dev, "nvidia,ac97-controller");
++ if (IS_ERR(np_ac97))
++ return PTR_ERR(np_ac97);
+
+- np_i2s = tegra_machine_parse_phandle(dev, "nvidia,i2s-controller");
+- if (IS_ERR(np_i2s))
+- return PTR_ERR(np_i2s);
++ card->dai_link->cpus->of_node = np_ac97;
++ card->dai_link->platforms->of_node = np_ac97;
++ } else {
++ np_codec = tegra_machine_parse_phandle(dev, "nvidia,audio-codec");
++ if (IS_ERR(np_codec))
++ return PTR_ERR(np_codec);
+
+- card->dai_link->cpus->of_node = np_i2s;
+- card->dai_link->codecs->of_node = np_codec;
+- card->dai_link->platforms->of_node = np_i2s;
++ np_i2s = tegra_machine_parse_phandle(dev, "nvidia,i2s-controller");
++ if (IS_ERR(np_i2s))
++ return PTR_ERR(np_i2s);
++
++ card->dai_link->cpus->of_node = np_i2s;
++ card->dai_link->codecs->of_node = np_codec;
++ card->dai_link->platforms->of_node = np_i2s;
++ }
+
+ if (asoc->add_common_controls) {
+ card->controls = tegra_machine_controls;
+@@ -589,6 +635,7 @@ static struct snd_soc_card snd_soc_tegra_wm9712 = {
+ static const struct tegra_asoc_data tegra_wm9712_data = {
+ .card = &snd_soc_tegra_wm9712,
+ .add_common_dapm_widgets = true,
++ .codec_dev_name = "wm9712-codec",
+ .set_ac97 = true,
+ };
+
+@@ -619,6 +666,7 @@ static struct snd_soc_card snd_soc_tegra_max98090 = {
+ static const struct tegra_asoc_data tegra_max98090_data = {
+ .mclk_rate = tegra_machine_mclk_rate_12mhz,
+ .card = &snd_soc_tegra_max98090,
++ .hp_jack_name = "Headphones",
+ .add_common_dapm_widgets = true,
+ .add_common_controls = true,
+ .add_common_snd_ops = true,
+@@ -686,6 +734,7 @@ static struct snd_soc_dai_link tegra_tlv320aic23_dai = {
+ };
+
+ static struct snd_soc_card snd_soc_tegra_trimslice = {
++ .name = "tegra-trimslice",
+ .components = "codec:tlv320aic23",
+ .dai_link = &tegra_tlv320aic23_dai,
+ .num_links = 1,
+diff --git a/sound/soc/tegra/tegra_asoc_machine.h b/sound/soc/tegra/tegra_asoc_machine.h
+index 8ee0ec814f67c..6f795d7dff7c1 100644
+--- a/sound/soc/tegra/tegra_asoc_machine.h
++++ b/sound/soc/tegra/tegra_asoc_machine.h
+@@ -13,6 +13,8 @@ struct snd_soc_pcm_runtime;
+
+ struct tegra_asoc_data {
+ unsigned int (*mclk_rate)(unsigned int srate);
++ const char *codec_dev_name;
++ const char *hp_jack_name;
+ struct snd_soc_card *card;
+ unsigned int mclk_id;
+ bool hp_jack_gpio_active_low;
+diff --git a/sound/soc/ti/davinci-i2s.c b/sound/soc/ti/davinci-i2s.c
+index 6dca51862dd76..0363a088d2e00 100644
+--- a/sound/soc/ti/davinci-i2s.c
++++ b/sound/soc/ti/davinci-i2s.c
+@@ -708,7 +708,9 @@ static int davinci_i2s_probe(struct platform_device *pdev)
+ dev->clk = clk_get(&pdev->dev, NULL);
+ if (IS_ERR(dev->clk))
+ return -ENODEV;
+- clk_enable(dev->clk);
++ ret = clk_enable(dev->clk);
++ if (ret)
++ goto err_put_clk;
+
+ dev->dev = &pdev->dev;
+ dev_set_drvdata(&pdev->dev, dev);
+@@ -730,6 +732,7 @@ err_unregister_component:
+ snd_soc_unregister_component(&pdev->dev);
+ err_release_clk:
+ clk_disable(dev->clk);
++err_put_clk:
+ clk_put(dev->clk);
+ return ret;
+ }
+diff --git a/sound/soc/uniphier/Kconfig b/sound/soc/uniphier/Kconfig
+index aa3592ee1358b..ddfa6424c656b 100644
+--- a/sound/soc/uniphier/Kconfig
++++ b/sound/soc/uniphier/Kconfig
+@@ -23,7 +23,6 @@ config SND_SOC_UNIPHIER_LD11
+ tristate "UniPhier LD11/LD20 Device Driver"
+ depends on SND_SOC_UNIPHIER
+ select SND_SOC_UNIPHIER_AIO
+- select SND_SOC_UNIPHIER_AIO_DMA
+ help
+ This adds ASoC driver for Socionext UniPhier LD11/LD20
+ input and output that can be used with other codecs.
+@@ -34,7 +33,6 @@ config SND_SOC_UNIPHIER_PXS2
+ tristate "UniPhier PXs2 Device Driver"
+ depends on SND_SOC_UNIPHIER
+ select SND_SOC_UNIPHIER_AIO
+- select SND_SOC_UNIPHIER_AIO_DMA
+ help
+ This adds ASoC driver for Socionext UniPhier PXs2
+ input and output that can be used with other codecs.
+diff --git a/sound/soc/xilinx/xlnx_formatter_pcm.c b/sound/soc/xilinx/xlnx_formatter_pcm.c
+index 91afea9d5de67..5c4158069a5a8 100644
+--- a/sound/soc/xilinx/xlnx_formatter_pcm.c
++++ b/sound/soc/xilinx/xlnx_formatter_pcm.c
+@@ -37,6 +37,7 @@
+ #define XLNX_AUD_XFER_COUNT 0x28
+ #define XLNX_AUD_CH_STS_START 0x2C
+ #define XLNX_BYTES_PER_CH 0x44
++#define XLNX_AUD_ALIGN_BYTES 64
+
+ #define AUD_STS_IOC_IRQ_MASK BIT(31)
+ #define AUD_STS_CH_STS_MASK BIT(29)
+@@ -83,6 +84,7 @@ struct xlnx_pcm_drv_data {
+ struct snd_pcm_substream *play_stream;
+ struct snd_pcm_substream *capture_stream;
+ struct clk *axi_clk;
++ unsigned int sysclk;
+ };
+
+ /*
+@@ -313,6 +315,15 @@ static irqreturn_t xlnx_s2mm_irq_handler(int irq, void *arg)
+ return IRQ_NONE;
+ }
+
++static int xlnx_formatter_set_sysclk(struct snd_soc_component *component,
++ int clk_id, int source, unsigned int freq, int dir)
++{
++ struct xlnx_pcm_drv_data *adata = dev_get_drvdata(component->dev);
++
++ adata->sysclk = freq;
++ return 0;
++}
++
+ static int xlnx_formatter_pcm_open(struct snd_soc_component *component,
+ struct snd_pcm_substream *substream)
+ {
+@@ -368,12 +379,32 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component,
+ snd_soc_set_runtime_hwparams(substream, &xlnx_pcm_hardware);
+ runtime->private_data = stream_data;
+
+- /* Resize the period size divisible by 64 */
++ /* Resize the period bytes as divisible by 64 */
++ err = snd_pcm_hw_constraint_step(runtime, 0,
++ SNDRV_PCM_HW_PARAM_PERIOD_BYTES,
++ XLNX_AUD_ALIGN_BYTES);
++ if (err) {
++ dev_err(component->dev,
++ "Unable to set constraint on period bytes\n");
++ return err;
++ }
++
++ /* Resize the buffer bytes as divisible by 64 */
+ err = snd_pcm_hw_constraint_step(runtime, 0,
+- SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 64);
++ SNDRV_PCM_HW_PARAM_BUFFER_BYTES,
++ XLNX_AUD_ALIGN_BYTES);
+ if (err) {
+ dev_err(component->dev,
+- "unable to set constraint on period bytes\n");
++ "Unable to set constraint on buffer bytes\n");
++ return err;
++ }
++
++ /* Set periods as integer multiple */
++ err = snd_pcm_hw_constraint_integer(runtime,
++ SNDRV_PCM_HW_PARAM_PERIODS);
++ if (err < 0) {
++ dev_err(component->dev,
++ "Unable to set constraint on periods to be integer\n");
+ return err;
+ }
+
+@@ -429,11 +460,25 @@ static int xlnx_formatter_pcm_hw_params(struct snd_soc_component *component,
+ u64 size;
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct xlnx_pcm_stream_param *stream_data = runtime->private_data;
++ struct xlnx_pcm_drv_data *adata = dev_get_drvdata(component->dev);
+
+ active_ch = params_channels(params);
+ if (active_ch > stream_data->ch_limit)
+ return -EINVAL;
+
++ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK &&
++ adata->sysclk) {
++ unsigned int mclk_fs = adata->sysclk / params_rate(params);
++
++ if (adata->sysclk % params_rate(params) != 0) {
++ dev_warn(component->dev, "sysclk %u not divisible by rate %u\n",
++ adata->sysclk, params_rate(params));
++ return -EINVAL;
++ }
++
++ writel(mclk_fs, stream_data->mmio + XLNX_AUD_FS_MULTIPLIER);
++ }
++
+ if (substream->stream == SNDRV_PCM_STREAM_CAPTURE &&
+ stream_data->xfer_mode == AES_TO_PCM) {
+ val = readl(stream_data->mmio + XLNX_AUD_STS);
+@@ -531,6 +576,7 @@ static int xlnx_formatter_pcm_new(struct snd_soc_component *component,
+
+ static const struct snd_soc_component_driver xlnx_asoc_component = {
+ .name = DRV_NAME,
++ .set_sysclk = xlnx_formatter_set_sysclk,
+ .open = xlnx_formatter_pcm_open,
+ .close = xlnx_formatter_pcm_close,
+ .hw_params = xlnx_formatter_pcm_hw_params,
+diff --git a/sound/spi/at73c213.c b/sound/spi/at73c213.c
+index 76c0e37a838cf..8a2da6b1012eb 100644
+--- a/sound/spi/at73c213.c
++++ b/sound/spi/at73c213.c
+@@ -218,7 +218,9 @@ static int snd_at73c213_pcm_open(struct snd_pcm_substream *substream)
+ runtime->hw = snd_at73c213_playback_hw;
+ chip->substream = substream;
+
+- clk_enable(chip->ssc->clk);
++ err = clk_enable(chip->ssc->clk);
++ if (err)
++ return err;
+
+ return 0;
+ }
+@@ -776,7 +778,9 @@ static int snd_at73c213_chip_init(struct snd_at73c213 *chip)
+ goto out;
+
+ /* Enable DAC master clock. */
+- clk_enable(chip->board->dac_clk);
++ retval = clk_enable(chip->board->dac_clk);
++ if (retval)
++ goto out;
+
+ /* Initialize at73c213 on SPI bus. */
+ retval = snd_at73c213_write_reg(chip, DAC_RST, 0x04);
+@@ -889,7 +893,9 @@ static int snd_at73c213_dev_init(struct snd_card *card,
+ chip->card = card;
+ chip->irq = -1;
+
+- clk_enable(chip->ssc->clk);
++ retval = clk_enable(chip->ssc->clk);
++ if (retval)
++ return retval;
+
+ retval = request_irq(irq, snd_at73c213_interrupt, 0, "at73c213", chip);
+ if (retval) {
+@@ -1008,7 +1014,9 @@ static int snd_at73c213_remove(struct spi_device *spi)
+ int retval;
+
+ /* Stop playback. */
+- clk_enable(chip->ssc->clk);
++ retval = clk_enable(chip->ssc->clk);
++ if (retval)
++ goto out;
+ ssc_writel(chip->ssc->regs, CR, SSC_BIT(CR_TXDIS));
+ clk_disable(chip->ssc->clk);
+
+@@ -1088,9 +1096,16 @@ static int snd_at73c213_resume(struct device *dev)
+ {
+ struct snd_card *card = dev_get_drvdata(dev);
+ struct snd_at73c213 *chip = card->private_data;
++ int retval;
+
+- clk_enable(chip->board->dac_clk);
+- clk_enable(chip->ssc->clk);
++ retval = clk_enable(chip->board->dac_clk);
++ if (retval)
++ return retval;
++ retval = clk_enable(chip->ssc->clk);
++ if (retval) {
++ clk_disable(chip->board->dac_clk);
++ return retval;
++ }
+ ssc_writel(chip->ssc->regs, CR, SSC_BIT(CR_TXEN));
+
+ return 0;
+diff --git a/sound/synth/emux/emux.c b/sound/synth/emux/emux.c
+index 49d1976a132c0..5ed8e36d2e043 100644
+--- a/sound/synth/emux/emux.c
++++ b/sound/synth/emux/emux.c
+@@ -88,7 +88,7 @@ int snd_emux_register(struct snd_emux *emu, struct snd_card *card, int index, ch
+ emu->name = kstrdup(name, GFP_KERNEL);
+ emu->voices = kcalloc(emu->max_voices, sizeof(struct snd_emux_voice),
+ GFP_KERNEL);
+- if (emu->voices == NULL)
++ if (emu->name == NULL || emu->voices == NULL)
+ return -ENOMEM;
+
+ /* create soundfont list */
+diff --git a/sound/usb/6fire/comm.c b/sound/usb/6fire/comm.c
+index 43a2a62d66f7e..49629d4bb327a 100644
+--- a/sound/usb/6fire/comm.c
++++ b/sound/usb/6fire/comm.c
+@@ -95,7 +95,7 @@ static int usb6fire_comm_send_buffer(u8 *buffer, struct usb_device *dev)
+ int actual_len;
+
+ ret = usb_interrupt_msg(dev, usb_sndintpipe(dev, COMM_EP),
+- buffer, buffer[1] + 2, &actual_len, HZ);
++ buffer, buffer[1] + 2, &actual_len, 1000);
+ if (ret < 0)
+ return ret;
+ else if (actual_len != buffer[1] + 2)
+diff --git a/sound/usb/6fire/firmware.c b/sound/usb/6fire/firmware.c
+index 8981e61f2da4a..c51abc54d2f84 100644
+--- a/sound/usb/6fire/firmware.c
++++ b/sound/usb/6fire/firmware.c
+@@ -160,7 +160,7 @@ static int usb6fire_fw_ezusb_write(struct usb_device *device,
+ {
+ return usb_control_msg_send(device, 0, type,
+ USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- value, 0, data, len, HZ, GFP_KERNEL);
++ value, 0, data, len, 1000, GFP_KERNEL);
+ }
+
+ static int usb6fire_fw_ezusb_read(struct usb_device *device,
+@@ -168,7 +168,7 @@ static int usb6fire_fw_ezusb_read(struct usb_device *device,
+ {
+ return usb_control_msg_recv(device, 0, type,
+ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+- value, 0, data, len, HZ, GFP_KERNEL);
++ value, 0, data, len, 1000, GFP_KERNEL);
+ }
+
+ static int usb6fire_fw_fpga_write(struct usb_device *device,
+@@ -178,7 +178,7 @@ static int usb6fire_fw_fpga_write(struct usb_device *device,
+ int ret;
+
+ ret = usb_bulk_msg(device, usb_sndbulkpipe(device, FPGA_EP), data, len,
+- &actual_len, HZ);
++ &actual_len, 1000);
+ if (ret < 0)
+ return ret;
+ else if (actual_len != len)
+diff --git a/sound/usb/card.h b/sound/usb/card.h
+index 5b19901f305a3..87f042d06ce08 100644
+--- a/sound/usb/card.h
++++ b/sound/usb/card.h
+@@ -74,8 +74,9 @@ struct snd_usb_endpoint {
+
+ atomic_t state; /* running state */
+
+- void (*prepare_data_urb) (struct snd_usb_substream *subs,
+- struct urb *urb);
++ int (*prepare_data_urb) (struct snd_usb_substream *subs,
++ struct urb *urb,
++ bool in_stream_lock);
+ void (*retire_data_urb) (struct snd_usb_substream *subs,
+ struct urb *urb);
+
+@@ -94,9 +95,9 @@ struct snd_usb_endpoint {
+ struct list_head ready_playback_urbs; /* playback URB FIFO for implicit fb */
+
+ unsigned int nurbs; /* # urbs */
+- unsigned int nominal_queue_size; /* total buffer sizes in URBs */
+ unsigned long active_mask; /* bitmask of active urbs */
+ unsigned long unlink_mask; /* bitmask of unlinked urbs */
++ atomic_t submitted_urbs; /* currently submitted urbs */
+ char *syncbuf; /* sync buffer for all sync URBs */
+ dma_addr_t sync_dma; /* DMA address of syncbuf */
+
+@@ -125,6 +126,7 @@ struct snd_usb_endpoint {
+ int skip_packets; /* quirks for devices to ignore the first n packets
+ in a stream */
+ bool implicit_fb_sync; /* syncs with implicit feedback */
++ bool lowlatency_playback; /* low-latency playback mode */
+ bool need_setup; /* (re-)need for configure? */
+
+ /* for hw constraints */
+@@ -136,6 +138,7 @@ struct snd_usb_endpoint {
+ unsigned int cur_period_frames;
+ unsigned int cur_period_bytes;
+ unsigned int cur_buffer_periods;
++ unsigned char cur_clock;
+
+ spinlock_t lock;
+ struct list_head list;
+@@ -188,7 +191,7 @@ struct snd_usb_substream {
+ } dsd_dop;
+
+ bool trigger_tstamp_pending_update; /* trigger timestamp being updated from initial estimate */
+- bool early_playback_start; /* early start needed for playback? */
++ bool lowlatency_playback; /* low-latency playback mode */
+ struct media_ctl *media_ctl;
+ };
+
+diff --git a/sound/usb/clock.c b/sound/usb/clock.c
+index 81d5ce07d548b..98345a695dccb 100644
+--- a/sound/usb/clock.c
++++ b/sound/usb/clock.c
+@@ -496,6 +496,10 @@ int snd_usb_set_sample_rate_v2v3(struct snd_usb_audio *chip,
+ union uac23_clock_source_desc *cs_desc;
+
+ cs_desc = snd_usb_find_clock_source(chip, clock, fmt->protocol);
++
++ if (!cs_desc)
++ return 0;
++
+ if (fmt->protocol == UAC_VERSION_3)
+ bmControls = le32_to_cpu(cs_desc->v3.bmControls);
+ else
+diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c
+index 533919a28856f..743b8287cfcdd 100644
+--- a/sound/usb/endpoint.c
++++ b/sound/usb/endpoint.c
+@@ -148,18 +148,23 @@ int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep)
+ * This won't be used for implicit feedback which takes the packet size
+ * returned from the sync source
+ */
+-static int slave_next_packet_size(struct snd_usb_endpoint *ep)
++static int slave_next_packet_size(struct snd_usb_endpoint *ep,
++ unsigned int avail)
+ {
+ unsigned long flags;
++ unsigned int phase;
+ int ret;
+
+ if (ep->fill_max)
+ return ep->maxframesize;
+
+ spin_lock_irqsave(&ep->lock, flags);
+- ep->phase = (ep->phase & 0xffff)
+- + (ep->freqm << ep->datainterval);
+- ret = min(ep->phase >> 16, ep->maxframesize);
++ phase = (ep->phase & 0xffff) + (ep->freqm << ep->datainterval);
++ ret = min(phase >> 16, ep->maxframesize);
++ if (avail && ret >= avail)
++ ret = -EAGAIN;
++ else
++ ep->phase = phase;
+ spin_unlock_irqrestore(&ep->lock, flags);
+
+ return ret;
+@@ -169,20 +174,25 @@ static int slave_next_packet_size(struct snd_usb_endpoint *ep)
+ * Return the number of samples to be sent in the next packet
+ * for adaptive and synchronous endpoints
+ */
+-static int next_packet_size(struct snd_usb_endpoint *ep)
++static int next_packet_size(struct snd_usb_endpoint *ep, unsigned int avail)
+ {
++ unsigned int sample_accum;
+ int ret;
+
+ if (ep->fill_max)
+ return ep->maxframesize;
+
+- ep->sample_accum += ep->sample_rem;
+- if (ep->sample_accum >= ep->pps) {
+- ep->sample_accum -= ep->pps;
++ sample_accum = ep->sample_accum + ep->sample_rem;
++ if (sample_accum >= ep->pps) {
++ sample_accum -= ep->pps;
+ ret = ep->packsize[1];
+ } else {
+ ret = ep->packsize[0];
+ }
++ if (avail && ret >= avail)
++ ret = -EAGAIN;
++ else
++ ep->sample_accum = sample_accum;
+
+ return ret;
+ }
+@@ -190,16 +200,27 @@ static int next_packet_size(struct snd_usb_endpoint *ep)
+ /*
+ * snd_usb_endpoint_next_packet_size: Return the number of samples to be sent
+ * in the next packet
++ *
++ * If the size is equal or exceeds @avail, don't proceed but return -EAGAIN
++ * Exception: @avail = 0 for skipping the check.
+ */
+ int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep,
+- struct snd_urb_ctx *ctx, int idx)
++ struct snd_urb_ctx *ctx, int idx,
++ unsigned int avail)
+ {
+- if (ctx->packet_size[idx])
+- return ctx->packet_size[idx];
+- else if (ep->sync_source)
+- return slave_next_packet_size(ep);
++ unsigned int packet;
++
++ packet = ctx->packet_size[idx];
++ if (packet) {
++ if (avail && packet >= avail)
++ return -EAGAIN;
++ return packet;
++ }
++
++ if (ep->sync_source)
++ return slave_next_packet_size(ep, avail);
+ else
+- return next_packet_size(ep);
++ return next_packet_size(ep, avail);
+ }
+
+ static void call_retire_callback(struct snd_usb_endpoint *ep,
+@@ -263,7 +284,7 @@ static void prepare_silent_urb(struct snd_usb_endpoint *ep,
+ unsigned int length;
+ int counts;
+
+- counts = snd_usb_endpoint_next_packet_size(ep, ctx, i);
++ counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0);
+ length = counts * ep->stride; /* number of silent bytes */
+ offset = offs * ep->stride + extra * i;
+ urb->iso_frame_desc[i].offset = offset;
+@@ -286,8 +307,9 @@ static void prepare_silent_urb(struct snd_usb_endpoint *ep,
+ /*
+ * Prepare a PLAYBACK urb for submission to the bus.
+ */
+-static void prepare_outbound_urb(struct snd_usb_endpoint *ep,
+- struct snd_urb_ctx *ctx)
++static int prepare_outbound_urb(struct snd_usb_endpoint *ep,
++ struct snd_urb_ctx *ctx,
++ bool in_stream_lock)
+ {
+ struct urb *urb = ctx->urb;
+ unsigned char *cp = urb->transfer_buffer;
+@@ -299,9 +321,9 @@ static void prepare_outbound_urb(struct snd_usb_endpoint *ep,
+ case SND_USB_ENDPOINT_TYPE_DATA:
+ data_subs = READ_ONCE(ep->data_subs);
+ if (data_subs && ep->prepare_data_urb)
+- ep->prepare_data_urb(data_subs, urb);
+- else /* no data provider, so send silence */
+- prepare_silent_urb(ep, ctx);
++ return ep->prepare_data_urb(data_subs, urb, in_stream_lock);
++ /* no data provider, so send silence */
++ prepare_silent_urb(ep, ctx);
+ break;
+
+ case SND_USB_ENDPOINT_TYPE_SYNC:
+@@ -330,13 +352,14 @@ static void prepare_outbound_urb(struct snd_usb_endpoint *ep,
+
+ break;
+ }
++ return 0;
+ }
+
+ /*
+ * Prepare a CAPTURE or SYNC urb for submission to the bus.
+ */
+-static inline void prepare_inbound_urb(struct snd_usb_endpoint *ep,
+- struct snd_urb_ctx *urb_ctx)
++static int prepare_inbound_urb(struct snd_usb_endpoint *ep,
++ struct snd_urb_ctx *urb_ctx)
+ {
+ int i, offs;
+ struct urb *urb = urb_ctx->urb;
+@@ -361,6 +384,7 @@ static inline void prepare_inbound_urb(struct snd_usb_endpoint *ep,
+ urb->iso_frame_desc[0].offset = 0;
+ break;
+ }
++ return 0;
+ }
+
+ /* notify an error as XRUN to the assigned PCM data substream */
+@@ -396,6 +420,16 @@ next_packet_fifo_dequeue(struct snd_usb_endpoint *ep)
+ return p;
+ }
+
++static void push_back_to_ready_list(struct snd_usb_endpoint *ep,
++ struct snd_urb_ctx *ctx)
++{
++ unsigned long flags;
++
++ spin_lock_irqsave(&ep->lock, flags);
++ list_add_tail(&ctx->ready_list, &ep->ready_playback_urbs);
++ spin_unlock_irqrestore(&ep->lock, flags);
++}
++
+ /*
+ * Send output urbs that have been prepared previously. URBs are dequeued
+ * from ep->ready_playback_urbs and in case there aren't any available
+@@ -406,12 +440,14 @@ next_packet_fifo_dequeue(struct snd_usb_endpoint *ep)
+ * is that host controllers don't guarantee the order in which they return
+ * inbound and outbound packets to their submitters.
+ *
+- * This function is only used for implicit feedback endpoints. For endpoints
+- * driven by dedicated sync endpoints, URBs are immediately re-submitted
+- * from their completion handler.
++ * This function is used both for implicit feedback endpoints and in low-
++ * latency playback mode.
+ */
+-static void queue_pending_output_urbs(struct snd_usb_endpoint *ep)
++void snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep,
++ bool in_stream_lock)
+ {
++ bool implicit_fb = snd_usb_endpoint_implicit_feedback_sink(ep);
++
+ while (ep_state_running(ep)) {
+
+ unsigned long flags;
+@@ -420,14 +456,14 @@ static void queue_pending_output_urbs(struct snd_usb_endpoint *ep)
+ int err, i;
+
+ spin_lock_irqsave(&ep->lock, flags);
+- if (ep->next_packet_queued > 0 &&
++ if ((!implicit_fb || ep->next_packet_queued > 0) &&
+ !list_empty(&ep->ready_playback_urbs)) {
+ /* take URB out of FIFO */
+ ctx = list_first_entry(&ep->ready_playback_urbs,
+ struct snd_urb_ctx, ready_list);
+ list_del_init(&ctx->ready_list);
+-
+- packet = next_packet_fifo_dequeue(ep);
++ if (implicit_fb)
++ packet = next_packet_fifo_dequeue(ep);
+ }
+ spin_unlock_irqrestore(&ep->lock, flags);
+
+@@ -435,11 +471,24 @@ static void queue_pending_output_urbs(struct snd_usb_endpoint *ep)
+ return;
+
+ /* copy over the length information */
+- for (i = 0; i < packet->packets; i++)
+- ctx->packet_size[i] = packet->packet_size[i];
++ if (implicit_fb) {
++ for (i = 0; i < packet->packets; i++)
++ ctx->packet_size[i] = packet->packet_size[i];
++ }
+
+ /* call the data handler to fill in playback data */
+- prepare_outbound_urb(ep, ctx);
++ err = prepare_outbound_urb(ep, ctx, in_stream_lock);
++ /* can be stopped during prepare callback */
++ if (unlikely(!ep_state_running(ep)))
++ break;
++ if (err < 0) {
++ /* push back to ready list again for -EAGAIN */
++ if (err == -EAGAIN)
++ push_back_to_ready_list(ep, ctx);
++ else
++ notify_xrun(ep);
++ return;
++ }
+
+ err = usb_submit_urb(ctx->urb, GFP_ATOMIC);
+ if (err < 0) {
+@@ -451,6 +500,7 @@ static void queue_pending_output_urbs(struct snd_usb_endpoint *ep)
+ }
+
+ set_bit(ctx->index, &ep->active_mask);
++ atomic_inc(&ep->submitted_urbs);
+ }
+ }
+
+@@ -461,7 +511,6 @@ static void snd_complete_urb(struct urb *urb)
+ {
+ struct snd_urb_ctx *ctx = urb->context;
+ struct snd_usb_endpoint *ep = ctx->ep;
+- unsigned long flags;
+ int err;
+
+ if (unlikely(urb->status == -ENOENT || /* unlinked */
+@@ -482,16 +531,20 @@ static void snd_complete_urb(struct urb *urb)
+ if (unlikely(!ep_state_running(ep)))
+ goto exit_clear;
+
+- if (snd_usb_endpoint_implicit_feedback_sink(ep)) {
+- spin_lock_irqsave(&ep->lock, flags);
+- list_add_tail(&ctx->ready_list, &ep->ready_playback_urbs);
++ /* in low-latency and implicit-feedback modes, push back the
++ * URB to ready list at first, then process as much as possible
++ */
++ if (ep->lowlatency_playback ||
++ snd_usb_endpoint_implicit_feedback_sink(ep)) {
++ push_back_to_ready_list(ep, ctx);
+ clear_bit(ctx->index, &ep->active_mask);
+- spin_unlock_irqrestore(&ep->lock, flags);
+- queue_pending_output_urbs(ep);
++ snd_usb_queue_pending_output_urbs(ep, false);
++ atomic_dec(&ep->submitted_urbs); /* decrement at last */
+ return;
+ }
+
+- prepare_outbound_urb(ep, ctx);
++ /* in non-lowlatency mode, no error handling for prepare */
++ prepare_outbound_urb(ep, ctx, false);
+ /* can be stopped during prepare callback */
+ if (unlikely(!ep_state_running(ep)))
+ goto exit_clear;
+@@ -513,6 +566,7 @@ static void snd_complete_urb(struct urb *urb)
+
+ exit_clear:
+ clear_bit(ctx->index, &ep->active_mask);
++ atomic_dec(&ep->submitted_urbs);
+ }
+
+ /*
+@@ -596,6 +650,7 @@ int snd_usb_add_endpoint(struct snd_usb_audio *chip, int ep_num, int type)
+ ep->type = type;
+ ep->ep_num = ep_num;
+ INIT_LIST_HEAD(&ep->ready_playback_urbs);
++ atomic_set(&ep->submitted_urbs, 0);
+
+ is_playback = ((ep_num & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT);
+ ep_num &= USB_ENDPOINT_NUMBER_MASK;
+@@ -722,6 +777,7 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip,
+ ep->cur_period_frames = params_period_size(params);
+ ep->cur_period_bytes = ep->cur_period_frames * ep->cur_frame_bytes;
+ ep->cur_buffer_periods = params_periods(params);
++ ep->cur_clock = fp->clock;
+
+ if (ep->type == SND_USB_ENDPOINT_TYPE_SYNC)
+ endpoint_set_syncinterval(chip, ep);
+@@ -781,14 +837,19 @@ void snd_usb_endpoint_set_sync(struct snd_usb_audio *chip,
+ * Pass NULL to deactivate each callback.
+ */
+ void snd_usb_endpoint_set_callback(struct snd_usb_endpoint *ep,
+- void (*prepare)(struct snd_usb_substream *subs,
+- struct urb *urb),
++ int (*prepare)(struct snd_usb_substream *subs,
++ struct urb *urb,
++ bool in_stream_lock),
+ void (*retire)(struct snd_usb_substream *subs,
+ struct urb *urb),
+ struct snd_usb_substream *data_subs)
+ {
+ ep->prepare_data_urb = prepare;
+ ep->retire_data_urb = retire;
++ if (data_subs)
++ ep->lowlatency_playback = data_subs->lowlatency_playback;
++ else
++ ep->lowlatency_playback = false;
+ WRITE_ONCE(ep->data_subs, data_subs);
+ }
+
+@@ -833,6 +894,7 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip,
+ ep->altsetting = 0;
+ ep->cur_audiofmt = NULL;
+ ep->cur_rate = 0;
++ ep->cur_clock = 0;
+ ep->iface_ref = NULL;
+ usb_audio_dbg(chip, "EP 0x%x closed\n", ep->ep_num);
+ }
+@@ -859,7 +921,7 @@ static int wait_clear_urbs(struct snd_usb_endpoint *ep)
+ return 0;
+
+ do {
+- alive = bitmap_weight(&ep->active_mask, ep->nurbs);
++ alive = atomic_read(&ep->submitted_urbs);
+ if (!alive)
+ break;
+
+@@ -893,9 +955,10 @@ void snd_usb_endpoint_sync_pending_stop(struct snd_usb_endpoint *ep)
+ *
+ * This function moves the EP to STOPPING state if it's being RUNNING.
+ */
+-static int stop_urbs(struct snd_usb_endpoint *ep, bool force)
++static int stop_urbs(struct snd_usb_endpoint *ep, bool force, bool keep_pending)
+ {
+ unsigned int i;
++ unsigned long flags;
+
+ if (!force && atomic_read(&ep->running))
+ return -EBUSY;
+@@ -903,9 +966,14 @@ static int stop_urbs(struct snd_usb_endpoint *ep, bool force)
+ if (!ep_state_update(ep, EP_STATE_RUNNING, EP_STATE_STOPPING))
+ return 0;
+
++ spin_lock_irqsave(&ep->lock, flags);
+ INIT_LIST_HEAD(&ep->ready_playback_urbs);
+ ep->next_packet_head = 0;
+ ep->next_packet_queued = 0;
++ spin_unlock_irqrestore(&ep->lock, flags);
++
++ if (keep_pending)
++ return 0;
+
+ for (i = 0; i < ep->nurbs; i++) {
+ if (test_bit(i, &ep->active_mask)) {
+@@ -930,7 +998,7 @@ static int release_urbs(struct snd_usb_endpoint *ep, bool force)
+ snd_usb_endpoint_set_callback(ep, NULL, NULL, NULL);
+
+ /* stop and unlink urbs */
+- err = stop_urbs(ep, force);
++ err = stop_urbs(ep, force, false);
+ if (err)
+ return err;
+
+@@ -1132,10 +1200,6 @@ static int data_ep_set_params(struct snd_usb_endpoint *ep)
+ INIT_LIST_HEAD(&u->ready_list);
+ }
+
+- /* total buffer bytes of all URBs plus the next queue;
+- * referred in pcm.c
+- */
+- ep->nominal_queue_size = maxsize * urb_packs * (ep->nurbs + 1);
+ return 0;
+
+ out_of_memory:
+@@ -1340,6 +1404,25 @@ unlock:
+ return err;
+ }
+
++/* get the current rate set to the given clock by any endpoint */
++int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock)
++{
++ struct snd_usb_endpoint *ep;
++ int rate = 0;
++
++ if (!clock)
++ return 0;
++ mutex_lock(&chip->mutex);
++ list_for_each_entry(ep, &chip->ep_list, list) {
++ if (ep->cur_clock == clock && ep->cur_rate) {
++ rate = ep->cur_rate;
++ break;
++ }
++ }
++ mutex_unlock(&chip->mutex);
++ return rate;
++}
++
+ /**
+ * snd_usb_endpoint_start: start an snd_usb_endpoint
+ *
+@@ -1355,6 +1438,7 @@ unlock:
+ */
+ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
+ {
++ bool is_playback = usb_pipeout(ep->pipe);
+ int err;
+ unsigned int i;
+
+@@ -1391,13 +1475,9 @@ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
+
+ if (snd_usb_endpoint_implicit_feedback_sink(ep) &&
+ !(ep->chip->quirk_flags & QUIRK_FLAG_PLAYBACK_FIRST)) {
+- for (i = 0; i < ep->nurbs; i++) {
+- struct snd_urb_ctx *ctx = ep->urb + i;
+- list_add_tail(&ctx->ready_list, &ep->ready_playback_urbs);
+- }
+-
+ usb_audio_dbg(ep->chip, "No URB submission due to implicit fb sync\n");
+- return 0;
++ i = 0;
++ goto fill_rest;
+ }
+
+ for (i = 0; i < ep->nurbs; i++) {
+@@ -1406,10 +1486,18 @@ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
+ if (snd_BUG_ON(!urb))
+ goto __error;
+
+- if (usb_pipeout(ep->pipe)) {
+- prepare_outbound_urb(ep, urb->context);
+- } else {
+- prepare_inbound_urb(ep, urb->context);
++ if (is_playback)
++ err = prepare_outbound_urb(ep, urb->context, true);
++ else
++ err = prepare_inbound_urb(ep, urb->context);
++ if (err < 0) {
++ /* stop filling at applptr */
++ if (err == -EAGAIN)
++ break;
++ usb_audio_dbg(ep->chip,
++ "EP 0x%x: failed to prepare urb: %d\n",
++ ep->ep_num, err);
++ goto __error;
+ }
+
+ err = usb_submit_urb(urb, GFP_ATOMIC);
+@@ -1420,14 +1508,29 @@ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
+ goto __error;
+ }
+ set_bit(i, &ep->active_mask);
++ atomic_inc(&ep->submitted_urbs);
++ }
++
++ if (!i) {
++ usb_audio_dbg(ep->chip, "XRUN at starting EP 0x%x\n",
++ ep->ep_num);
++ goto __error;
+ }
+
+ usb_audio_dbg(ep->chip, "%d URBs submitted for EP 0x%x\n",
+- ep->nurbs, ep->ep_num);
++ i, ep->ep_num);
++
++ fill_rest:
++ /* put the remaining URBs to ready list */
++ if (is_playback) {
++ for (; i < ep->nurbs; i++)
++ push_back_to_ready_list(ep, ep->urb + i);
++ }
++
+ return 0;
+
+ __error:
+- snd_usb_endpoint_stop(ep);
++ snd_usb_endpoint_stop(ep, false);
+ return -EPIPE;
+ }
+
+@@ -1435,6 +1538,7 @@ __error:
+ * snd_usb_endpoint_stop: stop an snd_usb_endpoint
+ *
+ * @ep: the endpoint to stop (may be NULL)
++ * @keep_pending: keep in-flight URBs
+ *
+ * A call to this function will decrement the running count of the endpoint.
+ * In case the last user has requested the endpoint stop, the URBs will
+@@ -1445,7 +1549,7 @@ __error:
+ * The caller needs to synchronize the pending stop operation via
+ * snd_usb_endpoint_sync_pending_stop().
+ */
+-void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep)
++void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending)
+ {
+ if (!ep)
+ return;
+@@ -1460,7 +1564,7 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep)
+ if (!atomic_dec_return(&ep->running)) {
+ if (ep->sync_source)
+ WRITE_ONCE(ep->sync_source->sync_sink, NULL);
+- stop_urbs(ep, false);
++ stop_urbs(ep, false, keep_pending);
+ }
+ }
+
+@@ -1575,7 +1679,7 @@ static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep,
+ }
+
+ spin_unlock_irqrestore(&ep->lock, flags);
+- queue_pending_output_urbs(ep);
++ snd_usb_queue_pending_output_urbs(ep, false);
+
+ return;
+ }
+diff --git a/sound/usb/endpoint.h b/sound/usb/endpoint.h
+index a668f675b52b0..6a9af04cf175a 100644
+--- a/sound/usb/endpoint.h
++++ b/sound/usb/endpoint.h
+@@ -19,6 +19,7 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip,
+ struct snd_usb_endpoint *ep);
+ int snd_usb_endpoint_configure(struct snd_usb_audio *chip,
+ struct snd_usb_endpoint *ep);
++int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock);
+
+ bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip,
+ struct snd_usb_endpoint *ep,
+@@ -29,14 +30,15 @@ void snd_usb_endpoint_set_sync(struct snd_usb_audio *chip,
+ struct snd_usb_endpoint *data_ep,
+ struct snd_usb_endpoint *sync_ep);
+ void snd_usb_endpoint_set_callback(struct snd_usb_endpoint *ep,
+- void (*prepare)(struct snd_usb_substream *subs,
+- struct urb *urb),
++ int (*prepare)(struct snd_usb_substream *subs,
++ struct urb *urb,
++ bool in_stream_lock),
+ void (*retire)(struct snd_usb_substream *subs,
+ struct urb *urb),
+ struct snd_usb_substream *data_subs);
+
+ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep);
+-void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep);
++void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending);
+ void snd_usb_endpoint_sync_pending_stop(struct snd_usb_endpoint *ep);
+ void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep);
+ int snd_usb_endpoint_activate(struct snd_usb_endpoint *ep);
+@@ -45,6 +47,9 @@ void snd_usb_endpoint_free_all(struct snd_usb_audio *chip);
+
+ int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep);
+ int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep,
+- struct snd_urb_ctx *ctx, int idx);
++ struct snd_urb_ctx *ctx, int idx,
++ unsigned int avail);
++void snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep,
++ bool in_stream_lock);
+
+ #endif /* __USBAUDIO_ENDPOINT_H */
+diff --git a/sound/usb/format.c b/sound/usb/format.c
+index 50efccbffb8a7..405dc0bf6678c 100644
+--- a/sound/usb/format.c
++++ b/sound/usb/format.c
+@@ -375,7 +375,7 @@ static int parse_uac2_sample_rate_range(struct snd_usb_audio *chip,
+ for (rate = min; rate <= max; rate += res) {
+
+ /* Filter out invalid rates on Presonus Studio 1810c */
+- if (chip->usb_id == USB_ID(0x0194f, 0x010c) &&
++ if (chip->usb_id == USB_ID(0x194f, 0x010c) &&
+ !s1810c_valid_sample_rate(fp, rate))
+ goto skip_rate;
+
+@@ -414,6 +414,7 @@ static int line6_parse_audio_format_rates_quirk(struct snd_usb_audio *chip,
+ case USB_ID(0x0e41, 0x4242): /* Line6 Helix Rack */
+ case USB_ID(0x0e41, 0x4244): /* Line6 Helix LT */
+ case USB_ID(0x0e41, 0x4246): /* Line6 HX-Stomp */
++ case USB_ID(0x0e41, 0x4253): /* Line6 HX-Stomp XL */
+ case USB_ID(0x0e41, 0x4247): /* Line6 Pod Go */
+ case USB_ID(0x0e41, 0x4248): /* Line6 Helix >= fw 2.82 */
+ case USB_ID(0x0e41, 0x4249): /* Line6 Helix Rack >= fw 2.82 */
+diff --git a/sound/usb/implicit.c b/sound/usb/implicit.c
+index 23767a14d1266..2d444ec742029 100644
+--- a/sound/usb/implicit.c
++++ b/sound/usb/implicit.c
+@@ -47,15 +47,13 @@ struct snd_usb_implicit_fb_match {
+ static const struct snd_usb_implicit_fb_match playback_implicit_fb_quirks[] = {
+ /* Generic matching */
+ IMPLICIT_FB_GENERIC_DEV(0x0499, 0x1509), /* Steinberg UR22 */
+- IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2080), /* M-Audio FastTrack Ultra */
+- IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2081), /* M-Audio FastTrack Ultra */
+ IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2030), /* M-Audio Fast Track C400 */
+ IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2031), /* M-Audio Fast Track C600 */
+
+ /* Fixed EP */
+ /* FIXME: check the availability of generic matching */
+- IMPLICIT_FB_FIXED_DEV(0x1397, 0x0001, 0x81, 1), /* Behringer UFX1604 */
+- IMPLICIT_FB_FIXED_DEV(0x1397, 0x0002, 0x81, 1), /* Behringer UFX1204 */
++ IMPLICIT_FB_FIXED_DEV(0x0763, 0x2080, 0x81, 2), /* M-Audio FastTrack Ultra */
++ IMPLICIT_FB_FIXED_DEV(0x0763, 0x2081, 0x81, 2), /* M-Audio FastTrack Ultra */
+ IMPLICIT_FB_FIXED_DEV(0x2466, 0x8010, 0x81, 2), /* Fractal Audio Axe-Fx III */
+ IMPLICIT_FB_FIXED_DEV(0x31e9, 0x0001, 0x81, 2), /* Solid State Logic SSL2 */
+ IMPLICIT_FB_FIXED_DEV(0x31e9, 0x0002, 0x81, 2), /* Solid State Logic SSL2+ */
+diff --git a/sound/usb/line6/driver.c b/sound/usb/line6/driver.c
+index 9602929b7de90..59faa5a9a7141 100644
+--- a/sound/usb/line6/driver.c
++++ b/sound/usb/line6/driver.c
+@@ -113,12 +113,12 @@ int line6_send_raw_message(struct usb_line6 *line6, const char *buffer,
+ retval = usb_interrupt_msg(line6->usbdev,
+ usb_sndintpipe(line6->usbdev, properties->ep_ctrl_w),
+ (char *)frag_buf, frag_size,
+- &partial, LINE6_TIMEOUT * HZ);
++ &partial, LINE6_TIMEOUT);
+ } else {
+ retval = usb_bulk_msg(line6->usbdev,
+ usb_sndbulkpipe(line6->usbdev, properties->ep_ctrl_w),
+ (char *)frag_buf, frag_size,
+- &partial, LINE6_TIMEOUT * HZ);
++ &partial, LINE6_TIMEOUT);
+ }
+
+ if (retval) {
+@@ -347,7 +347,7 @@ int line6_read_data(struct usb_line6 *line6, unsigned address, void *data,
+ ret = usb_control_msg_send(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+ (datalen << 8) | 0x21, address, NULL, 0,
+- LINE6_TIMEOUT * HZ, GFP_KERNEL);
++ LINE6_TIMEOUT, GFP_KERNEL);
+ if (ret) {
+ dev_err(line6->ifcdev, "read request failed (error %d)\n", ret);
+ goto exit;
+@@ -360,7 +360,7 @@ int line6_read_data(struct usb_line6 *line6, unsigned address, void *data,
+ ret = usb_control_msg_recv(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+ 0x0012, 0x0000, &len, 1,
+- LINE6_TIMEOUT * HZ, GFP_KERNEL);
++ LINE6_TIMEOUT, GFP_KERNEL);
+ if (ret) {
+ dev_err(line6->ifcdev,
+ "receive length failed (error %d)\n", ret);
+@@ -387,7 +387,7 @@ int line6_read_data(struct usb_line6 *line6, unsigned address, void *data,
+ /* receive the result: */
+ ret = usb_control_msg_recv(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- 0x0013, 0x0000, data, datalen, LINE6_TIMEOUT * HZ,
++ 0x0013, 0x0000, data, datalen, LINE6_TIMEOUT,
+ GFP_KERNEL);
+ if (ret)
+ dev_err(line6->ifcdev, "read failed (error %d)\n", ret);
+@@ -417,7 +417,7 @@ int line6_write_data(struct usb_line6 *line6, unsigned address, void *data,
+
+ ret = usb_control_msg_send(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- 0x0022, address, data, datalen, LINE6_TIMEOUT * HZ,
++ 0x0022, address, data, datalen, LINE6_TIMEOUT,
+ GFP_KERNEL);
+ if (ret) {
+ dev_err(line6->ifcdev,
+@@ -430,7 +430,7 @@ int line6_write_data(struct usb_line6 *line6, unsigned address, void *data,
+
+ ret = usb_control_msg_recv(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+- 0x0012, 0x0000, status, 1, LINE6_TIMEOUT * HZ,
++ 0x0012, 0x0000, status, 1, LINE6_TIMEOUT,
+ GFP_KERNEL);
+ if (ret) {
+ dev_err(line6->ifcdev,
+diff --git a/sound/usb/line6/driver.h b/sound/usb/line6/driver.h
+index 71d3da1db8c81..ecf3a2b39c7eb 100644
+--- a/sound/usb/line6/driver.h
++++ b/sound/usb/line6/driver.h
+@@ -27,7 +27,7 @@
+ #define LINE6_FALLBACK_INTERVAL 10
+ #define LINE6_FALLBACK_MAXPACKETSIZE 16
+
+-#define LINE6_TIMEOUT 1
++#define LINE6_TIMEOUT 1000
+ #define LINE6_BUFSIZE_LISTEN 64
+ #define LINE6_MIDI_MESSAGE_MAXLEN 256
+
+diff --git a/sound/usb/line6/podhd.c b/sound/usb/line6/podhd.c
+index 28794a35949d4..b24bc82f89e37 100644
+--- a/sound/usb/line6/podhd.c
++++ b/sound/usb/line6/podhd.c
+@@ -190,7 +190,7 @@ static int podhd_dev_start(struct usb_line6_podhd *pod)
+ ret = usb_control_msg_send(usbdev, 0,
+ 0x67, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+ 0x11, 0,
+- NULL, 0, LINE6_TIMEOUT * HZ, GFP_KERNEL);
++ NULL, 0, LINE6_TIMEOUT, GFP_KERNEL);
+ if (ret) {
+ dev_err(pod->line6.ifcdev, "read request failed (error %d)\n", ret);
+ goto exit;
+@@ -200,7 +200,7 @@ static int podhd_dev_start(struct usb_line6_podhd *pod)
+ ret = usb_control_msg_recv(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+ 0x11, 0x0,
+- init_bytes, 3, LINE6_TIMEOUT * HZ, GFP_KERNEL);
++ init_bytes, 3, LINE6_TIMEOUT, GFP_KERNEL);
+ if (ret) {
+ dev_err(pod->line6.ifcdev,
+ "receive length failed (error %d)\n", ret);
+@@ -220,7 +220,7 @@ static int podhd_dev_start(struct usb_line6_podhd *pod)
+ USB_REQ_SET_FEATURE,
+ USB_TYPE_STANDARD | USB_RECIP_DEVICE | USB_DIR_OUT,
+ 1, 0,
+- NULL, 0, LINE6_TIMEOUT * HZ, GFP_KERNEL);
++ NULL, 0, LINE6_TIMEOUT, GFP_KERNEL);
+ exit:
+ return ret;
+ }
+diff --git a/sound/usb/line6/toneport.c b/sound/usb/line6/toneport.c
+index 4e5693c97aa42..e33df58740a91 100644
+--- a/sound/usb/line6/toneport.c
++++ b/sound/usb/line6/toneport.c
+@@ -128,7 +128,7 @@ static int toneport_send_cmd(struct usb_device *usbdev, int cmd1, int cmd2)
+
+ ret = usb_control_msg_send(usbdev, 0, 0x67,
+ USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+- cmd1, cmd2, NULL, 0, LINE6_TIMEOUT * HZ,
++ cmd1, cmd2, NULL, 0, LINE6_TIMEOUT,
+ GFP_KERNEL);
+
+ if (ret) {
+diff --git a/sound/usb/midi.c b/sound/usb/midi.c
+index 2c01649c70f61..7c6ca2b433a53 100644
+--- a/sound/usb/midi.c
++++ b/sound/usb/midi.c
+@@ -1194,6 +1194,7 @@ static void snd_usbmidi_output_drain(struct snd_rawmidi_substream *substream)
+ } while (drain_urbs && timeout);
+ finish_wait(&ep->drain_wait, &wait);
+ }
++ port->active = 0;
+ spin_unlock_irq(&ep->buffer_lock);
+ }
+
+diff --git a/sound/usb/misc/ua101.c b/sound/usb/misc/ua101.c
+index 5834d1dc317ef..4f6b20ed29dd7 100644
+--- a/sound/usb/misc/ua101.c
++++ b/sound/usb/misc/ua101.c
+@@ -1000,7 +1000,7 @@ static int detect_usb_format(struct ua101 *ua)
+ fmt_playback->bSubframeSize * ua->playback.channels;
+
+ epd = &ua->intf[INTF_CAPTURE]->altsetting[1].endpoint[0].desc;
+- if (!usb_endpoint_is_isoc_in(epd)) {
++ if (!usb_endpoint_is_isoc_in(epd) || usb_endpoint_maxp(epd) == 0) {
+ dev_err(&ua->dev->dev, "invalid capture endpoint\n");
+ return -ENXIO;
+ }
+@@ -1008,7 +1008,7 @@ static int detect_usb_format(struct ua101 *ua)
+ ua->capture.max_packet_bytes = usb_endpoint_maxp(epd);
+
+ epd = &ua->intf[INTF_PLAYBACK]->altsetting[1].endpoint[0].desc;
+- if (!usb_endpoint_is_isoc_out(epd)) {
++ if (!usb_endpoint_is_isoc_out(epd) || usb_endpoint_maxp(epd) == 0) {
+ dev_err(&ua->dev->dev, "invalid playback endpoint\n");
+ return -ENXIO;
+ }
+diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c
+index 8e030b1c061ab..567514832b0df 100644
+--- a/sound/usb/mixer.c
++++ b/sound/usb/mixer.c
+@@ -1496,6 +1496,10 @@ error:
+ usb_audio_err(chip,
+ "cannot get connectors status: req = %#x, wValue = %#x, wIndex = %#x, type = %d\n",
+ UAC_GET_CUR, validx, idx, cval->val_type);
++
++ if (val)
++ *val = 0;
++
+ return filter_error(cval, ret);
+ }
+
+diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c
+index 55eea90ee993f..6ffd23f2ee65d 100644
+--- a/sound/usb/mixer_maps.c
++++ b/sound/usb/mixer_maps.c
+@@ -536,6 +536,16 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = {
+ .id = USB_ID(0x1b1c, 0x0a41),
+ .map = corsair_virtuoso_map,
+ },
++ {
++ /* Corsair Virtuoso SE Latest (wired mode) */
++ .id = USB_ID(0x1b1c, 0x0a3f),
++ .map = corsair_virtuoso_map,
++ },
++ {
++ /* Corsair Virtuoso SE Latest (wireless mode) */
++ .id = USB_ID(0x1b1c, 0x0a40),
++ .map = corsair_virtuoso_map,
++ },
+ {
+ /* Corsair Virtuoso (wireless mode) */
+ .id = USB_ID(0x1b1c, 0x0a42),
+diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c
+index 46082dc57be09..d12b87e52d22a 100644
+--- a/sound/usb/mixer_quirks.c
++++ b/sound/usb/mixer_quirks.c
+@@ -2795,6 +2795,7 @@ static int snd_bbfpro_controls_create(struct usb_mixer_interface *mixer)
+ #define SND_DJM_750_IDX 0x1
+ #define SND_DJM_850_IDX 0x2
+ #define SND_DJM_900NXS2_IDX 0x3
++#define SND_DJM_750MK2_IDX 0x4
+
+
+ #define SND_DJM_CTL(_name, suffix, _default_value, _windex) { \
+@@ -2984,12 +2985,42 @@ static const struct snd_djm_ctl snd_djm_ctls_900nxs2[] = {
+ SND_DJM_CTL("Ch5 Input", 900nxs2_cap5, 3, SND_DJM_WINDEX_CAP)
+ };
+
++// DJM-750MK2
++static const u16 snd_djm_opts_750mk2_cap1[] = {
++ 0x0100, 0x0102, 0x0103, 0x0106, 0x0107, 0x0108, 0x0109, 0x010a };
++static const u16 snd_djm_opts_750mk2_cap2[] = {
++ 0x0200, 0x0202, 0x0203, 0x0206, 0x0207, 0x0208, 0x0209, 0x020a };
++static const u16 snd_djm_opts_750mk2_cap3[] = {
++ 0x0300, 0x0302, 0x0303, 0x0306, 0x0307, 0x0308, 0x0309, 0x030a };
++static const u16 snd_djm_opts_750mk2_cap4[] = {
++ 0x0400, 0x0402, 0x0403, 0x0406, 0x0407, 0x0408, 0x0409, 0x040a };
++static const u16 snd_djm_opts_750mk2_cap5[] = {
++ 0x0507, 0x0508, 0x0509, 0x050a, 0x0511, 0x0512, 0x0513, 0x0514 };
++
++static const u16 snd_djm_opts_750mk2_pb1[] = { 0x0100, 0x0101, 0x0104 };
++static const u16 snd_djm_opts_750mk2_pb2[] = { 0x0200, 0x0201, 0x0204 };
++static const u16 snd_djm_opts_750mk2_pb3[] = { 0x0300, 0x0301, 0x0304 };
++
++
++static const struct snd_djm_ctl snd_djm_ctls_750mk2[] = {
++ SND_DJM_CTL("Capture Level", cap_level, 0, SND_DJM_WINDEX_CAPLVL),
++ SND_DJM_CTL("Ch1 Input", 750mk2_cap1, 2, SND_DJM_WINDEX_CAP),
++ SND_DJM_CTL("Ch2 Input", 750mk2_cap2, 2, SND_DJM_WINDEX_CAP),
++ SND_DJM_CTL("Ch3 Input", 750mk2_cap3, 2, SND_DJM_WINDEX_CAP),
++ SND_DJM_CTL("Ch4 Input", 750mk2_cap4, 2, SND_DJM_WINDEX_CAP),
++ SND_DJM_CTL("Ch5 Input", 750mk2_cap5, 3, SND_DJM_WINDEX_CAP),
++ SND_DJM_CTL("Ch1 Output", 750mk2_pb1, 0, SND_DJM_WINDEX_PB),
++ SND_DJM_CTL("Ch2 Output", 750mk2_pb2, 1, SND_DJM_WINDEX_PB),
++ SND_DJM_CTL("Ch3 Output", 750mk2_pb3, 2, SND_DJM_WINDEX_PB)
++};
++
+
+ static const struct snd_djm_device snd_djm_devices[] = {
+- SND_DJM_DEVICE(250mk2),
+- SND_DJM_DEVICE(750),
+- SND_DJM_DEVICE(850),
+- SND_DJM_DEVICE(900nxs2)
++ [SND_DJM_250MK2_IDX] = SND_DJM_DEVICE(250mk2),
++ [SND_DJM_750_IDX] = SND_DJM_DEVICE(750),
++ [SND_DJM_850_IDX] = SND_DJM_DEVICE(850),
++ [SND_DJM_900NXS2_IDX] = SND_DJM_DEVICE(900nxs2),
++ [SND_DJM_750MK2_IDX] = SND_DJM_DEVICE(750mk2),
+ };
+
+
+@@ -3223,7 +3254,7 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer)
+ err = snd_rme_controls_create(mixer);
+ break;
+
+- case USB_ID(0x0194f, 0x010c): /* Presonus Studio 1810c */
++ case USB_ID(0x194f, 0x010c): /* Presonus Studio 1810c */
+ err = snd_sc1810_init_mixer(mixer);
+ break;
+ case USB_ID(0x2a39, 0x3fb0): /* RME Babyface Pro FS */
+@@ -3235,6 +3266,9 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer)
+ case USB_ID(0x08e4, 0x017f): /* Pioneer DJ DJM-750 */
+ err = snd_djm_controls_create(mixer, SND_DJM_750_IDX);
+ break;
++ case USB_ID(0x2b73, 0x001b): /* Pioneer DJ DJM-750MK2 */
++ err = snd_djm_controls_create(mixer, SND_DJM_750MK2_IDX);
++ break;
+ case USB_ID(0x08e4, 0x0163): /* Pioneer DJ DJM-850 */
+ err = snd_djm_controls_create(mixer, SND_DJM_850_IDX);
+ break;
+@@ -3328,9 +3362,10 @@ void snd_usb_mixer_fu_apply_quirk(struct usb_mixer_interface *mixer,
+ if (unitid == 7 && cval->control == UAC_FU_VOLUME)
+ snd_dragonfly_quirk_db_scale(mixer, cval, kctl);
+ break;
+- /* lowest playback value is muted on C-Media devices */
+- case USB_ID(0x0d8c, 0x000c):
+- case USB_ID(0x0d8c, 0x0014):
++ /* lowest playback value is muted on some devices */
++ case USB_ID(0x0d8c, 0x000c): /* C-Media */
++ case USB_ID(0x0d8c, 0x0014): /* C-Media */
++ case USB_ID(0x19f7, 0x0003): /* RODE NT-USB */
+ if (strstr(kctl->id.name, "Playback"))
+ cval->min_mute = 1;
+ break;
+diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c
+index 5dc9266180e37..729e26f5ac4c7 100644
+--- a/sound/usb/pcm.c
++++ b/sound/usb/pcm.c
+@@ -219,16 +219,16 @@ int snd_usb_init_pitch(struct snd_usb_audio *chip,
+ return 0;
+ }
+
+-static bool stop_endpoints(struct snd_usb_substream *subs)
++static bool stop_endpoints(struct snd_usb_substream *subs, bool keep_pending)
+ {
+ bool stopped = 0;
+
+ if (test_and_clear_bit(SUBSTREAM_FLAG_SYNC_EP_STARTED, &subs->flags)) {
+- snd_usb_endpoint_stop(subs->sync_endpoint);
++ snd_usb_endpoint_stop(subs->sync_endpoint, keep_pending);
+ stopped = true;
+ }
+ if (test_and_clear_bit(SUBSTREAM_FLAG_DATA_EP_STARTED, &subs->flags)) {
+- snd_usb_endpoint_stop(subs->data_endpoint);
++ snd_usb_endpoint_stop(subs->data_endpoint, keep_pending);
+ stopped = true;
+ }
+ return stopped;
+@@ -261,7 +261,7 @@ static int start_endpoints(struct snd_usb_substream *subs)
+ return 0;
+
+ error:
+- stop_endpoints(subs);
++ stop_endpoints(subs, false);
+ return err;
+ }
+
+@@ -437,7 +437,7 @@ static int configure_endpoints(struct snd_usb_audio *chip,
+
+ if (subs->data_endpoint->need_setup) {
+ /* stop any running stream beforehand */
+- if (stop_endpoints(subs))
++ if (stop_endpoints(subs, false))
+ sync_pending_stops(subs);
+ err = snd_usb_endpoint_configure(chip, subs->data_endpoint);
+ if (err < 0)
+@@ -572,7 +572,7 @@ static int snd_usb_hw_free(struct snd_pcm_substream *substream)
+ subs->cur_audiofmt = NULL;
+ mutex_unlock(&chip->mutex);
+ if (!snd_usb_lock_shutdown(chip)) {
+- if (stop_endpoints(subs))
++ if (stop_endpoints(subs, false))
+ sync_pending_stops(subs);
+ close_endpoints(chip, subs);
+ snd_usb_unlock_shutdown(chip);
+@@ -581,6 +581,31 @@ static int snd_usb_hw_free(struct snd_pcm_substream *substream)
+ return 0;
+ }
+
++/* free-wheeling mode? (e.g. dmix) */
++static int in_free_wheeling_mode(struct snd_pcm_runtime *runtime)
++{
++ return runtime->stop_threshold > runtime->buffer_size;
++}
++
++/* check whether early start is needed for playback stream */
++static int lowlatency_playback_available(struct snd_pcm_runtime *runtime,
++ struct snd_usb_substream *subs)
++{
++ struct snd_usb_audio *chip = subs->stream->chip;
++
++ if (subs->direction == SNDRV_PCM_STREAM_CAPTURE)
++ return false;
++ /* disabled via module option? */
++ if (!chip->lowlatency)
++ return false;
++ if (in_free_wheeling_mode(runtime))
++ return false;
++ /* implicit feedback mode has own operation mode */
++ if (snd_usb_endpoint_implicit_feedback_sink(subs->data_endpoint))
++ return false;
++ return true;
++}
++
+ /*
+ * prepare callback
+ *
+@@ -614,13 +639,9 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream)
+ subs->period_elapsed_pending = 0;
+ runtime->delay = 0;
+
+- /* check whether early start is needed for playback stream */
+- subs->early_playback_start =
+- subs->direction == SNDRV_PCM_STREAM_PLAYBACK &&
+- (!chip->lowlatency ||
+- (subs->data_endpoint->nominal_queue_size >= subs->buffer_bytes));
+-
+- if (subs->early_playback_start)
++ subs->lowlatency_playback = lowlatency_playback_available(runtime, subs);
++ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK &&
++ !subs->lowlatency_playback)
+ ret = start_endpoints(subs);
+
+ unlock:
+@@ -648,9 +669,9 @@ static const struct snd_pcm_hardware snd_usb_hardware =
+ SNDRV_PCM_INFO_PAUSE,
+ .channels_min = 1,
+ .channels_max = 256,
+- .buffer_bytes_max = 1024 * 1024,
++ .buffer_bytes_max = INT_MAX, /* limited by BUFFER_TIME later */
+ .period_bytes_min = 64,
+- .period_bytes_max = 512 * 1024,
++ .period_bytes_max = INT_MAX, /* limited by PERIOD_TIME later */
+ .periods_min = 2,
+ .periods_max = 1024,
+ };
+@@ -734,6 +755,7 @@ static int hw_rule_rate(struct snd_pcm_hw_params *params,
+ struct snd_pcm_hw_rule *rule)
+ {
+ struct snd_usb_substream *subs = rule->private;
++ struct snd_usb_audio *chip = subs->stream->chip;
+ const struct audioformat *fp;
+ struct snd_interval *it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
+ unsigned int rmin, rmax, r;
+@@ -745,6 +767,14 @@ static int hw_rule_rate(struct snd_pcm_hw_params *params,
+ list_for_each_entry(fp, &subs->fmt_list, list) {
+ if (!hw_check_valid_format(subs, params, fp))
+ continue;
++ r = snd_usb_endpoint_get_clock_rate(chip, fp->clock);
++ if (r > 0) {
++ if (!snd_interval_test(it, r))
++ continue;
++ rmin = min(rmin, r);
++ rmax = max(rmax, r);
++ continue;
++ }
+ if (fp->rate_table && fp->nr_rates) {
+ for (i = 0; i < fp->nr_rates; i++) {
+ r = fp->rate_table[i];
+@@ -1034,6 +1064,18 @@ static int setup_hw_info(struct snd_pcm_runtime *runtime, struct snd_usb_substre
+ return err;
+ }
+
++ /* set max period and buffer sizes for 1 and 2 seconds, respectively */
++ err = snd_pcm_hw_constraint_minmax(runtime,
++ SNDRV_PCM_HW_PARAM_PERIOD_TIME,
++ 0, 1000000);
++ if (err < 0)
++ return err;
++ err = snd_pcm_hw_constraint_minmax(runtime,
++ SNDRV_PCM_HW_PARAM_BUFFER_TIME,
++ 0, 2000000);
++ if (err < 0)
++ return err;
++
+ /* additional hw constraints for implicit fb */
+ err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT,
+ hw_rule_format_implicit_fb, subs,
+@@ -1068,6 +1110,10 @@ static int snd_usb_pcm_open(struct snd_pcm_substream *substream)
+ int ret;
+
+ runtime->hw = snd_usb_hardware;
++ /* need an explicit sync to catch applptr update in low-latency mode */
++ if (direction == SNDRV_PCM_STREAM_PLAYBACK &&
++ as->chip->lowlatency)
++ runtime->hw.info |= SNDRV_PCM_INFO_SYNC_APPLPTR;
+ runtime->private_data = subs;
+ subs->pcm_substream = substream;
+ /* runtime PM is also done there */
+@@ -1320,44 +1366,66 @@ static unsigned int copy_to_urb_quirk(struct snd_usb_substream *subs,
+ return bytes;
+ }
+
+-static void prepare_playback_urb(struct snd_usb_substream *subs,
+- struct urb *urb)
++static int prepare_playback_urb(struct snd_usb_substream *subs,
++ struct urb *urb,
++ bool in_stream_lock)
+ {
+ struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime;
+ struct snd_usb_endpoint *ep = subs->data_endpoint;
+ struct snd_urb_ctx *ctx = urb->context;
+- unsigned int counts, frames, bytes;
++ unsigned int frames, bytes;
++ int counts;
++ unsigned int transfer_done, frame_limit, avail = 0;
+ int i, stride, period_elapsed = 0;
+ unsigned long flags;
++ int err = 0;
+
+ stride = ep->stride;
+
+ frames = 0;
+ ctx->queued = 0;
+ urb->number_of_packets = 0;
++
+ spin_lock_irqsave(&subs->lock, flags);
+- subs->frame_limit += ep->max_urb_frames;
++ frame_limit = subs->frame_limit + ep->max_urb_frames;
++ transfer_done = subs->transfer_done;
++
++ if (subs->lowlatency_playback &&
++ runtime->status->state != SNDRV_PCM_STATE_DRAINING) {
++ unsigned int hwptr = subs->hwptr_done / stride;
++
++ /* calculate the byte offset-in-buffer of the appl_ptr */
++ avail = (runtime->control->appl_ptr - runtime->hw_ptr_base)
++ % runtime->buffer_size;
++ if (avail <= hwptr)
++ avail += runtime->buffer_size;
++ avail -= hwptr;
++ }
++
+ for (i = 0; i < ctx->packets; i++) {
+- counts = snd_usb_endpoint_next_packet_size(ep, ctx, i);
++ counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, avail);
++ if (counts < 0)
++ break;
+ /* set up descriptor */
+ urb->iso_frame_desc[i].offset = frames * stride;
+ urb->iso_frame_desc[i].length = counts * stride;
+ frames += counts;
++ avail -= counts;
+ urb->number_of_packets++;
+- subs->transfer_done += counts;
+- if (subs->transfer_done >= runtime->period_size) {
+- subs->transfer_done -= runtime->period_size;
+- subs->frame_limit = 0;
++ transfer_done += counts;
++ if (transfer_done >= runtime->period_size) {
++ transfer_done -= runtime->period_size;
++ frame_limit = 0;
+ period_elapsed = 1;
+ if (subs->fmt_type == UAC_FORMAT_TYPE_II) {
+- if (subs->transfer_done > 0) {
++ if (transfer_done > 0) {
+ /* FIXME: fill-max mode is not
+ * supported yet */
+- frames -= subs->transfer_done;
+- counts -= subs->transfer_done;
++ frames -= transfer_done;
++ counts -= transfer_done;
+ urb->iso_frame_desc[i].length =
+ counts * stride;
+- subs->transfer_done = 0;
++ transfer_done = 0;
+ }
+ i++;
+ if (i < ctx->packets) {
+@@ -1371,13 +1439,19 @@ static void prepare_playback_urb(struct snd_usb_substream *subs,
+ }
+ }
+ /* finish at the period boundary or after enough frames */
+- if ((period_elapsed ||
+- subs->transfer_done >= subs->frame_limit) &&
++ if ((period_elapsed || transfer_done >= frame_limit) &&
+ !snd_usb_endpoint_implicit_feedback_sink(ep))
+ break;
+ }
+- bytes = frames * stride;
+
++ if (!frames) {
++ err = -EAGAIN;
++ goto unlock;
++ }
++
++ bytes = frames * stride;
++ subs->transfer_done = transfer_done;
++ subs->frame_limit = frame_limit;
+ if (unlikely(ep->cur_format == SNDRV_PCM_FORMAT_DSD_U16_LE &&
+ subs->cur_audiofmt->dsd_dop)) {
+ fill_playback_urb_dsd_dop(subs, urb, bytes);
+@@ -1403,14 +1477,23 @@ static void prepare_playback_urb(struct snd_usb_substream *subs,
+ subs->trigger_tstamp_pending_update = false;
+ }
+
+- if (period_elapsed && !subs->running && !subs->early_playback_start) {
++ if (period_elapsed && !subs->running && subs->lowlatency_playback) {
+ subs->period_elapsed_pending = 1;
+ period_elapsed = 0;
+ }
++
++ unlock:
+ spin_unlock_irqrestore(&subs->lock, flags);
++ if (err < 0)
++ return err;
+ urb->transfer_buffer_length = bytes;
+- if (period_elapsed)
+- snd_pcm_period_elapsed(subs->pcm_substream);
++ if (period_elapsed) {
++ if (in_stream_lock)
++ snd_pcm_period_elapsed_under_stream_lock(subs->pcm_substream);
++ else
++ snd_pcm_period_elapsed(subs->pcm_substream);
++ }
++ return 0;
+ }
+
+ /*
+@@ -1442,6 +1525,27 @@ static void retire_playback_urb(struct snd_usb_substream *subs,
+ snd_pcm_period_elapsed(subs->pcm_substream);
+ }
+
++/* PCM ack callback for the playback stream;
++ * this plays a role only when the stream is running in low-latency mode.
++ */
++static int snd_usb_pcm_playback_ack(struct snd_pcm_substream *substream)
++{
++ struct snd_usb_substream *subs = substream->runtime->private_data;
++ struct snd_usb_endpoint *ep;
++
++ if (!subs->lowlatency_playback || !subs->running)
++ return 0;
++ ep = subs->data_endpoint;
++ if (!ep)
++ return 0;
++ /* When no more in-flight URBs available, try to process the pending
++ * outputs here
++ */
++ if (!ep->active_mask)
++ snd_usb_queue_pending_output_urbs(ep, true);
++ return 0;
++}
++
+ static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substream,
+ int cmd)
+ {
+@@ -1457,8 +1561,10 @@ static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substrea
+ prepare_playback_urb,
+ retire_playback_urb,
+ subs);
+- if (!subs->early_playback_start &&
++ if (subs->lowlatency_playback &&
+ cmd == SNDRV_PCM_TRIGGER_START) {
++ if (in_free_wheeling_mode(substream->runtime))
++ subs->lowlatency_playback = false;
+ err = start_endpoints(subs);
+ if (err < 0) {
+ snd_usb_endpoint_set_callback(subs->data_endpoint,
+@@ -1473,7 +1579,7 @@ static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substrea
+ return 0;
+ case SNDRV_PCM_TRIGGER_SUSPEND:
+ case SNDRV_PCM_TRIGGER_STOP:
+- stop_endpoints(subs);
++ stop_endpoints(subs, substream->runtime->status->state == SNDRV_PCM_STATE_DRAINING);
+ snd_usb_endpoint_set_callback(subs->data_endpoint,
+ NULL, NULL, NULL);
+ subs->running = 0;
+@@ -1521,7 +1627,7 @@ static int snd_usb_substream_capture_trigger(struct snd_pcm_substream *substream
+ return 0;
+ case SNDRV_PCM_TRIGGER_SUSPEND:
+ case SNDRV_PCM_TRIGGER_STOP:
+- stop_endpoints(subs);
++ stop_endpoints(subs, false);
+ fallthrough;
+ case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+ snd_usb_endpoint_set_callback(subs->data_endpoint,
+@@ -1545,6 +1651,7 @@ static const struct snd_pcm_ops snd_usb_playback_ops = {
+ .trigger = snd_usb_substream_playback_trigger,
+ .sync_stop = snd_usb_pcm_sync_stop,
+ .pointer = snd_usb_pcm_pointer,
++ .ack = snd_usb_pcm_playback_ack,
+ };
+
+ static const struct snd_pcm_ops snd_usb_capture_ops = {
+diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h
+index 2af8c68fac275..40a5e3eb4ef26 100644
+--- a/sound/usb/quirks-table.h
++++ b/sound/usb/quirks-table.h
+@@ -84,7 +84,7 @@
+ * combination.
+ */
+ {
+- USB_DEVICE(0x041e, 0x4095),
++ USB_AUDIO_DEVICE(0x041e, 0x4095),
+ .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) {
+ .ifnum = QUIRK_ANY_INTERFACE,
+ .type = QUIRK_COMPOSITE,
+@@ -3235,6 +3235,15 @@ YAMAHA_DEVICE(0x7010, "UB99"),
+ }
+ },
+
++/* Rane SL-1 */
++{
++ USB_DEVICE(0x13e5, 0x0001),
++ .driver_info = (unsigned long) & (const struct snd_usb_audio_quirk) {
++ .ifnum = QUIRK_ANY_INTERFACE,
++ .type = QUIRK_AUDIO_STANDARD_INTERFACE
++ }
++},
++
+ /* disabled due to regression for other devices;
+ * see https://bugzilla.kernel.org/show_bug.cgi?id=199905
+ */
+@@ -3892,6 +3901,64 @@ YAMAHA_DEVICE(0x7010, "UB99"),
+ }
+ }
+ },
++{
++ /*
++ * Pioneer DJ DJM-750MK2
++ * 10 channels playback & 12 channels capture @ 48kHz S24LE
++ */
++ USB_DEVICE_VENDOR_SPEC(0x2b73, 0x001b),
++ .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) {
++ .ifnum = QUIRK_ANY_INTERFACE,
++ .type = QUIRK_COMPOSITE,
++ .data = (const struct snd_usb_audio_quirk[]) {
++ {
++ .ifnum = 0,
++ .type = QUIRK_AUDIO_FIXED_ENDPOINT,
++ .data = &(const struct audioformat) {
++ .formats = SNDRV_PCM_FMTBIT_S24_3LE,
++ .channels = 10,
++ .iface = 0,
++ .altsetting = 1,
++ .altset_idx = 1,
++ .endpoint = 0x01,
++ .ep_attr = USB_ENDPOINT_XFER_ISOC|
++ USB_ENDPOINT_SYNC_ASYNC,
++ .rates = SNDRV_PCM_RATE_48000,
++ .rate_min = 48000,
++ .rate_max = 48000,
++ .nr_rates = 1,
++ .rate_table = (unsigned int[]) {
++ 48000
++ }
++ }
++ },
++ {
++ .ifnum = 0,
++ .type = QUIRK_AUDIO_FIXED_ENDPOINT,
++ .data = &(const struct audioformat) {
++ .formats = SNDRV_PCM_FMTBIT_S24_3LE,
++ .channels = 12,
++ .iface = 0,
++ .altsetting = 1,
++ .altset_idx = 1,
++ .endpoint = 0x82,
++ .ep_idx = 1,
++ .ep_attr = USB_ENDPOINT_XFER_ISOC|
++ USB_ENDPOINT_SYNC_ASYNC|
++ USB_ENDPOINT_USAGE_IMPLICIT_FB,
++ .rates = SNDRV_PCM_RATE_48000,
++ .rate_min = 48000,
++ .rate_max = 48000,
++ .nr_rates = 1,
++ .rate_table = (unsigned int[]) { 48000 }
++ }
++ },
++ {
++ .ifnum = -1
++ }
++ }
++ }
++},
+ {
+ /*
+ * Pioneer DJ DJM-850
+diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
+index 8929d9abe8aa8..ab9f3da49941f 100644
+--- a/sound/usb/quirks.c
++++ b/sound/usb/quirks.c
+@@ -1290,7 +1290,7 @@ int snd_usb_apply_interface_quirk(struct snd_usb_audio *chip,
+ if (chip->usb_id == USB_ID(0x0763, 0x2012))
+ return fasttrackpro_skip_setting_quirk(chip, iface, altno);
+ /* presonus studio 1810c: skip altsets incompatible with device_setup */
+- if (chip->usb_id == USB_ID(0x0194f, 0x010c))
++ if (chip->usb_id == USB_ID(0x194f, 0x010c))
+ return s1810c_skip_setting_quirk(chip, iface, altno);
+
+
+@@ -1749,6 +1749,7 @@ static const struct registration_quirk registration_quirks[] = {
+ REG_QUIRK_ENTRY(0x0951, 0x16ea, 2), /* Kingston HyperX Cloud Flight S */
+ REG_QUIRK_ENTRY(0x0ecb, 0x1f46, 2), /* JBL Quantum 600 */
+ REG_QUIRK_ENTRY(0x0ecb, 0x1f47, 2), /* JBL Quantum 800 */
++ REG_QUIRK_ENTRY(0x0ecb, 0x1f4c, 2), /* JBL Quantum 400 */
+ REG_QUIRK_ENTRY(0x0ecb, 0x2039, 2), /* JBL Quantum 400 */
+ REG_QUIRK_ENTRY(0x0ecb, 0x203c, 2), /* JBL Quantum 600 */
+ REG_QUIRK_ENTRY(0x0ecb, 0x203e, 2), /* JBL Quantum 800 */
+@@ -1887,6 +1888,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
+ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER),
+ DEVICE_FLG(0x21b4, 0x0081, /* AudioQuest DragonFly */
+ QUIRK_FLAG_GET_SAMPLE_RATE),
++ DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */
++ QUIRK_FLAG_IGNORE_CTL_ERROR),
+ DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */
+ QUIRK_FLAG_GET_SAMPLE_RATE),
+ DEVICE_FLG(0x30be, 0x0101, /* Schiit Hel */
+diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h
+index 167834133b9bc..b8359a0aa008a 100644
+--- a/sound/usb/usbaudio.h
++++ b/sound/usb/usbaudio.h
+@@ -8,7 +8,7 @@
+ */
+
+ /* handling of USB vendor/product ID pairs as 32-bit numbers */
+-#define USB_ID(vendor, product) (((vendor) << 16) | (product))
++#define USB_ID(vendor, product) (((unsigned int)(vendor) << 16) | (product))
+ #define USB_ID_VENDOR(id) ((id) >> 16)
+ #define USB_ID_PRODUCT(id) ((u16)(id))
+
+diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c
+index 378826312abe6..42add5df37fda 100644
+--- a/sound/x86/intel_hdmi_audio.c
++++ b/sound/x86/intel_hdmi_audio.c
+@@ -1261,7 +1261,7 @@ static int had_pcm_mmap(struct snd_pcm_substream *substream,
+ {
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start,
+- substream->dma_buffer.addr >> PAGE_SHIFT,
++ substream->runtime->dma_addr >> PAGE_SHIFT,
+ vma->vm_end - vma->vm_start, vma->vm_page_prot);
+ }
+
+@@ -1665,7 +1665,7 @@ static void hdmi_lpe_audio_free(struct snd_card *card)
+ * This function is called when the i915 driver creates the
+ * hdmi-lpe-audio platform device.
+ */
+-static int hdmi_lpe_audio_probe(struct platform_device *pdev)
++static int __hdmi_lpe_audio_probe(struct platform_device *pdev)
+ {
+ struct snd_card *card;
+ struct snd_intelhad_card *card_ctx;
+@@ -1826,6 +1826,11 @@ static int hdmi_lpe_audio_probe(struct platform_device *pdev)
+ return 0;
+ }
+
++static int hdmi_lpe_audio_probe(struct platform_device *pdev)
++{
++ return snd_card_free_on_error(&pdev->dev, __hdmi_lpe_audio_probe(pdev));
++}
++
+ static const struct dev_pm_ops hdmi_lpe_audio_pm = {
+ SET_SYSTEM_SLEEP_PM_OPS(hdmi_lpe_audio_suspend, hdmi_lpe_audio_resume)
+ };
+diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h
+index d0ce5cfd3ac14..63d30dde20f94 100644
+--- a/tools/arch/x86/include/asm/cpufeatures.h
++++ b/tools/arch/x86/include/asm/cpufeatures.h
+@@ -204,7 +204,7 @@
+ /* FREE! ( 7*32+10) */
+ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */
+ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
+-#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */
++#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCEs for Spectre variant 2 */
+ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */
+ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */
+ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */
+diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h
+index a7c413432b33d..66ae309840fef 100644
+--- a/tools/arch/x86/include/asm/msr-index.h
++++ b/tools/arch/x86/include/asm/msr-index.h
+@@ -128,9 +128,9 @@
+ #define TSX_CTRL_RTM_DISABLE BIT(0) /* Disable RTM feature */
+ #define TSX_CTRL_CPUID_CLEAR BIT(1) /* Disable TSX enumeration */
+
+-/* SRBDS support */
+ #define MSR_IA32_MCU_OPT_CTRL 0x00000123
+-#define RNGDS_MITG_DIS BIT(0)
++#define RNGDS_MITG_DIS BIT(0) /* SRBDS support */
++#define RTM_ALLOW BIT(1) /* TSX development mode */
+
+ #define MSR_IA32_SYSENTER_CS 0x00000174
+ #define MSR_IA32_SYSENTER_ESP 0x00000175
+diff --git a/tools/arch/x86/lib/insn.c b/tools/arch/x86/lib/insn.c
+index 797699462cd8e..8fd63a067308a 100644
+--- a/tools/arch/x86/lib/insn.c
++++ b/tools/arch/x86/lib/insn.c
+@@ -13,6 +13,7 @@
+ #endif
+ #include "../include/asm/inat.h" /* __ignore_sync_check__ */
+ #include "../include/asm/insn.h" /* __ignore_sync_check__ */
++#include "../include/asm-generic/unaligned.h" /* __ignore_sync_check__ */
+
+ #include <linux/errno.h>
+ #include <linux/kconfig.h>
+@@ -37,10 +38,10 @@
+ ((insn)->next_byte + sizeof(t) + n <= (insn)->end_kaddr)
+
+ #define __get_next(t, insn) \
+- ({ t r; memcpy(&r, insn->next_byte, sizeof(t)); insn->next_byte += sizeof(t); leXX_to_cpu(t, r); })
++ ({ t r = get_unaligned((t *)(insn)->next_byte); (insn)->next_byte += sizeof(t); leXX_to_cpu(t, r); })
+
+ #define __peek_nbyte_next(t, insn, n) \
+- ({ t r; memcpy(&r, (insn)->next_byte + n, sizeof(t)); leXX_to_cpu(t, r); })
++ ({ t r = get_unaligned((t *)(insn)->next_byte + n); leXX_to_cpu(t, r); })
+
+ #define get_next(t, insn) \
+ ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); })
+diff --git a/tools/arch/x86/lib/memcpy_64.S b/tools/arch/x86/lib/memcpy_64.S
+index 1cc9da6e29c79..59cf2343f3d90 100644
+--- a/tools/arch/x86/lib/memcpy_64.S
++++ b/tools/arch/x86/lib/memcpy_64.S
+@@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy)
+ rep movsq
+ movl %edx, %ecx
+ rep movsb
+- ret
++ RET
+ SYM_FUNC_END(memcpy)
+ SYM_FUNC_END_ALIAS(__memcpy)
+ EXPORT_SYMBOL(memcpy)
+@@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms)
+ movq %rdi, %rax
+ movq %rdx, %rcx
+ rep movsb
+- ret
++ RET
+ SYM_FUNC_END(memcpy_erms)
+
+ SYM_FUNC_START_LOCAL(memcpy_orig)
+@@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movq %r9, 1*8(%rdi)
+ movq %r10, -2*8(%rdi, %rdx)
+ movq %r11, -1*8(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_16bytes:
+ cmpl $8, %edx
+@@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movq -1*8(%rsi, %rdx), %r9
+ movq %r8, 0*8(%rdi)
+ movq %r9, -1*8(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_8bytes:
+ cmpl $4, %edx
+@@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movl -4(%rsi, %rdx), %r8d
+ movl %ecx, (%rdi)
+ movl %r8d, -4(%rdi, %rdx)
+- retq
++ RET
+ .p2align 4
+ .Lless_3bytes:
+ subl $1, %edx
+@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig)
+ movb %cl, (%rdi)
+
+ .Lend:
+- retq
++ RET
+ SYM_FUNC_END(memcpy_orig)
+
+ .popsection
+diff --git a/tools/arch/x86/lib/memset_64.S b/tools/arch/x86/lib/memset_64.S
+index 9827ae267f96e..d624f2bc42f16 100644
+--- a/tools/arch/x86/lib/memset_64.S
++++ b/tools/arch/x86/lib/memset_64.S
+@@ -40,7 +40,7 @@ SYM_FUNC_START(__memset)
+ movl %edx,%ecx
+ rep stosb
+ movq %r9,%rax
+- ret
++ RET
+ SYM_FUNC_END(__memset)
+ SYM_FUNC_END_ALIAS(memset)
+ EXPORT_SYMBOL(memset)
+@@ -63,7 +63,7 @@ SYM_FUNC_START_LOCAL(memset_erms)
+ movq %rdx,%rcx
+ rep stosb
+ movq %r9,%rax
+- ret
++ RET
+ SYM_FUNC_END(memset_erms)
+
+ SYM_FUNC_START_LOCAL(memset_orig)
+@@ -125,7 +125,7 @@ SYM_FUNC_START_LOCAL(memset_orig)
+
+ .Lende:
+ movq %r10,%rax
+- ret
++ RET
+
+ .Lbad_alignment:
+ cmpq $7,%rdx
+diff --git a/tools/bpf/bpftool/Documentation/Makefile b/tools/bpf/bpftool/Documentation/Makefile
+index c49487905cebe..f89929c7038d5 100644
+--- a/tools/bpf/bpftool/Documentation/Makefile
++++ b/tools/bpf/bpftool/Documentation/Makefile
+@@ -1,6 +1,5 @@
+ # SPDX-License-Identifier: GPL-2.0-only
+ include ../../../scripts/Makefile.include
+-include ../../../scripts/utilities.mak
+
+ INSTALL ?= install
+ RM ?= rm -f
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-btf.rst b/tools/bpf/bpftool/Documentation/bpftool-btf.rst
+index 88b28aa7431f6..4425d942dd39a 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-btf.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-btf.rst
+@@ -13,7 +13,7 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **btf** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | {**-d** | **--debug** } |
+- { **-B** | **--base-btf** } }
++ { **-B** | **--base-btf** } }
+
+ *COMMANDS* := { **dump** | **help** }
+
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
+index 3e4395eede4f7..13a217a2503d8 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
+@@ -13,7 +13,7 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **cgroup** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } |
+- { **-f** | **--bpffs** } }
++ { **-f** | **--bpffs** } }
+
+ *COMMANDS* :=
+ { **show** | **list** | **tree** | **attach** | **detach** | **help** }
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-gen.rst b/tools/bpf/bpftool/Documentation/bpftool-gen.rst
+index 2ef2f2df02799..2a137f8a4cea0 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-gen.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-gen.rst
+@@ -13,7 +13,7 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **gen** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } |
+- { **-L** | **--use-loader** } }
++ { **-L** | **--use-loader** } }
+
+ *COMMAND* := { **object** | **skeleton** | **help** }
+
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-link.rst b/tools/bpf/bpftool/Documentation/bpftool-link.rst
+index 0de90f086238c..9434349636a5e 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-link.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-link.rst
+@@ -13,7 +13,7 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **link** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } |
+- { **-f** | **--bpffs** } | { **-n** | **--nomount** } }
++ { **-f** | **--bpffs** } | { **-n** | **--nomount** } }
+
+ *COMMANDS* := { **show** | **list** | **pin** | **help** }
+
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst
+index d0c4abe08abab..1445cadc15d4c 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-map.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst
+@@ -13,11 +13,11 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **map** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } |
+- { **-f** | **--bpffs** } | { **-n** | **--nomount** } }
++ { **-f** | **--bpffs** } | { **-n** | **--nomount** } }
+
+ *COMMANDS* :=
+- { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext**
+- | **delete** | **pin** | **help** }
++ { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext** |
++ **delete** | **pin** | **help** }
+
+ MAP COMMANDS
+ =============
+diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
+index 91608cb7e44a0..f27265bd589b4 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
+@@ -13,12 +13,12 @@ SYNOPSIS
+ **bpftool** [*OPTIONS*] **prog** *COMMAND*
+
+ *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } |
+- { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } |
+- { **-L** | **--use-loader** } }
++ { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } |
++ { **-L** | **--use-loader** } }
+
+ *COMMANDS* :=
+- { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load**
+- | **loadall** | **help** }
++ { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** |
++ **loadall** | **help** }
+
+ PROG COMMANDS
+ =============
+diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst
+index bb23f55bb05ad..8ac86565c501e 100644
+--- a/tools/bpf/bpftool/Documentation/bpftool.rst
++++ b/tools/bpf/bpftool/Documentation/bpftool.rst
+@@ -19,14 +19,14 @@ SYNOPSIS
+ *OBJECT* := { **map** | **program** | **cgroup** | **perf** | **net** | **feature** }
+
+ *OPTIONS* := { { **-V** | **--version** } |
+- { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } }
++ { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } }
+
+ *MAP-COMMANDS* :=
+ { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext** |
+- **delete** | **pin** | **event_pipe** | **help** }
++ **delete** | **pin** | **event_pipe** | **help** }
+
+ *PROG-COMMANDS* := { **show** | **list** | **dump jited** | **dump xlated** | **pin** |
+- **load** | **attach** | **detach** | **help** }
++ **load** | **attach** | **detach** | **help** }
+
+ *CGROUP-COMMANDS* := { **show** | **list** | **attach** | **detach** | **help** }
+
+diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
+index d73232be1e991..cce52df3be064 100644
+--- a/tools/bpf/bpftool/Makefile
++++ b/tools/bpf/bpftool/Makefile
+@@ -1,6 +1,5 @@
+ # SPDX-License-Identifier: GPL-2.0-only
+ include ../../scripts/Makefile.include
+-include ../../scripts/utilities.mak
+
+ ifeq ($(srctree),)
+ srctree := $(patsubst %/,%,$(dir $(CURDIR)))
+diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
+index d40d92bbf0e48..07fa502a4ac15 100644
+--- a/tools/bpf/bpftool/gen.c
++++ b/tools/bpf/bpftool/gen.c
+@@ -870,7 +870,6 @@ static int do_skeleton(int argc, char **argv)
+ s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s));\n\
+ if (!s) \n\
+ goto err; \n\
+- obj->skeleton = s; \n\
+ \n\
+ s->sz = sizeof(*s); \n\
+ s->name = \"%1$s\"; \n\
+@@ -955,6 +954,7 @@ static int do_skeleton(int argc, char **argv)
+ \n\
+ \"; \n\
+ \n\
++ obj->skeleton = s; \n\
+ return 0; \n\
+ err: \n\
+ bpf_object__destroy_skeleton(s); \n\
+diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c
+index 02eaaf065f651..d27ec4f852bbb 100644
+--- a/tools/bpf/bpftool/main.c
++++ b/tools/bpf/bpftool/main.c
+@@ -402,6 +402,8 @@ int main(int argc, char **argv)
+ };
+ int opt, ret;
+
++ setlinebuf(stdout);
++
+ last_do_help = do_help;
+ pretty_output = false;
+ json_output = false;
+diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
+index 407071d54ab1c..72ef9ddae2609 100644
+--- a/tools/bpf/bpftool/map.c
++++ b/tools/bpf/bpftool/map.c
+@@ -1042,11 +1042,9 @@ static void print_key_value(struct bpf_map_info *info, void *key,
+ json_writer_t *btf_wtr;
+ struct btf *btf;
+
+- btf = btf__load_from_kernel_by_id(info->btf_id);
+- if (libbpf_get_error(btf)) {
+- p_err("failed to get btf");
++ btf = get_map_kv_btf(info);
++ if (libbpf_get_error(btf))
+ return;
+- }
+
+ if (json_output) {
+ print_entry_json(info, key, value, btf);
+diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
+index 9c3e343b7d872..f8755beb3d9eb 100644
+--- a/tools/bpf/bpftool/prog.c
++++ b/tools/bpf/bpftool/prog.c
+@@ -308,18 +308,12 @@ static void show_prog_metadata(int fd, __u32 num_maps)
+ if (printed_header)
+ jsonw_end_object(json_wtr);
+ } else {
+- json_writer_t *btf_wtr = jsonw_new(stdout);
++ json_writer_t *btf_wtr;
+ struct btf_dumper d = {
+ .btf = btf,
+- .jw = btf_wtr,
+ .is_plain_text = true,
+ };
+
+- if (!btf_wtr) {
+- p_err("jsonw alloc failed");
+- goto out_free;
+- }
+-
+ for (i = 0; i < vlen; i++, vsi++) {
+ t_var = btf__type_by_id(btf, vsi->type);
+ name = btf__name_by_offset(btf, t_var->name_off);
+@@ -329,6 +323,14 @@ static void show_prog_metadata(int fd, __u32 num_maps)
+
+ if (!printed_header) {
+ printf("\tmetadata:");
++
++ btf_wtr = jsonw_new(stdout);
++ if (!btf_wtr) {
++ p_err("jsonw alloc failed");
++ goto out_free;
++ }
++ d.jw = btf_wtr,
++
+ printed_header = true;
+ }
+
+@@ -627,8 +629,8 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
+ char func_sig[1024];
+ unsigned char *buf;
+ __u32 member_len;
++ int fd, err = -1;
+ ssize_t n;
+- int fd;
+
+ if (mode == DUMP_JITED) {
+ if (info->jited_prog_len == 0 || !info->jited_prog_insns) {
+@@ -667,7 +669,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
+ if (fd < 0) {
+ p_err("can't open file %s: %s", filepath,
+ strerror(errno));
+- return -1;
++ goto exit_free;
+ }
+
+ n = write(fd, buf, member_len);
+@@ -675,7 +677,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
+ if (n != (ssize_t)member_len) {
+ p_err("error writing output file: %s",
+ n < 0 ? strerror(errno) : "short write");
+- return -1;
++ goto exit_free;
+ }
+
+ if (json_output)
+@@ -689,7 +691,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
+ info->netns_ino,
+ &disasm_opt);
+ if (!name)
+- return -1;
++ goto exit_free;
+ }
+
+ if (info->nr_jited_func_lens && info->jited_func_lens) {
+@@ -784,9 +786,12 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
+ kernel_syms_destroy(&dd);
+ }
+
+- btf__free(btf);
++ err = 0;
+
+- return 0;
++exit_free:
++ btf__free(btf);
++ bpf_prog_linfo__free(prog_linfo);
++ return err;
+ }
+
+ static int do_dump(int argc, char **argv)
+diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile
+index bb9fa8de7e625..af9f9d3534c96 100644
+--- a/tools/bpf/resolve_btfids/Makefile
++++ b/tools/bpf/resolve_btfids/Makefile
+@@ -9,7 +9,11 @@ ifeq ($(V),1)
+ msg =
+ else
+ Q = @
+- msg = @printf ' %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
++ ifeq ($(silent),1)
++ msg =
++ else
++ msg = @printf ' %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
++ endif
+ MAKEFLAGS=--no-print-directory
+ endif
+
+diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c
+index de6365b53c9ca..45e0d640618ac 100644
+--- a/tools/bpf/resolve_btfids/main.c
++++ b/tools/bpf/resolve_btfids/main.c
+@@ -166,7 +166,7 @@ static struct btf_id *btf_id__find(struct rb_root *root, const char *name)
+ return NULL;
+ }
+
+-static struct btf_id*
++static struct btf_id *
+ btf_id__add(struct rb_root *root, char *name, bool unique)
+ {
+ struct rb_node **p = &root->rb_node;
+@@ -720,7 +720,8 @@ int main(int argc, const char **argv)
+ if (no_fail)
+ return 0;
+ pr_err("FAILED to find needed sections\n");
+- return -1;
++ err = 0;
++ goto out;
+ }
+
+ if (symbols_collect(&obj))
+diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature
+index 3dd2f68366f95..6abde487bba19 100644
+--- a/tools/build/Makefile.feature
++++ b/tools/build/Makefile.feature
+@@ -48,7 +48,6 @@ FEATURE_TESTS_BASIC := \
+ numa_num_possible_cpus \
+ libperl \
+ libpython \
+- libpython-version \
+ libslang \
+ libslang-include-subdir \
+ libtraceevent \
+@@ -98,6 +97,7 @@ FEATURE_TESTS_EXTRA := \
+ llvm-version \
+ clang \
+ libbpf \
++ libbpf-btf__load_from_kernel_by_id \
+ libpfm4 \
+ libdebuginfod \
+ clang-bpf-co-re
+diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile
+index eff55d287db1f..69a43d9ea331f 100644
+--- a/tools/build/feature/Makefile
++++ b/tools/build/feature/Makefile
+@@ -32,7 +32,6 @@ FILES= \
+ test-numa_num_possible_cpus.bin \
+ test-libperl.bin \
+ test-libpython.bin \
+- test-libpython-version.bin \
+ test-libslang.bin \
+ test-libslang-include-subdir.bin \
+ test-libtraceevent.bin \
+@@ -57,6 +56,7 @@ FILES= \
+ test-lzma.bin \
+ test-bpf.bin \
+ test-libbpf.bin \
++ test-libbpf-btf__load_from_kernel_by_id.bin \
+ test-get_cpuid.bin \
+ test-sdt.bin \
+ test-cxx.bin \
+@@ -214,18 +214,22 @@ strip-libs = $(filter-out -l%,$(1))
+ PERL_EMBED_LDOPTS = $(shell perl -MExtUtils::Embed -e ldopts 2>/dev/null)
+ PERL_EMBED_LDFLAGS = $(call strip-libs,$(PERL_EMBED_LDOPTS))
+ PERL_EMBED_LIBADD = $(call grep-libs,$(PERL_EMBED_LDOPTS))
+-PERL_EMBED_CCOPTS = `perl -MExtUtils::Embed -e ccopts 2>/dev/null`
++PERL_EMBED_CCOPTS = $(shell perl -MExtUtils::Embed -e ccopts 2>/dev/null)
+ FLAGS_PERL_EMBED=$(PERL_EMBED_CCOPTS) $(PERL_EMBED_LDOPTS)
+
++ifeq ($(CC_NO_CLANG), 0)
++ PERL_EMBED_LDOPTS := $(filter-out -specs=%,$(PERL_EMBED_LDOPTS))
++ PERL_EMBED_CCOPTS := $(filter-out -flto=auto -ffat-lto-objects, $(PERL_EMBED_CCOPTS))
++ PERL_EMBED_CCOPTS := $(filter-out -specs=%,$(PERL_EMBED_CCOPTS))
++ FLAGS_PERL_EMBED += -Wno-compound-token-split-by-macro
++endif
++
+ $(OUTPUT)test-libperl.bin:
+ $(BUILD) $(FLAGS_PERL_EMBED)
+
+ $(OUTPUT)test-libpython.bin:
+ $(BUILD) $(FLAGS_PYTHON_EMBED)
+
+-$(OUTPUT)test-libpython-version.bin:
+- $(BUILD)
+-
+ $(OUTPUT)test-libbfd.bin:
+ $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl
+
+@@ -280,6 +284,9 @@ $(OUTPUT)test-bpf.bin:
+ $(OUTPUT)test-libbpf.bin:
+ $(BUILD) -lbpf
+
++$(OUTPUT)test-libbpf-btf__load_from_kernel_by_id.bin:
++ $(BUILD) -lbpf
++
+ $(OUTPUT)test-sdt.bin:
+ $(BUILD)
+
+diff --git a/tools/build/feature/test-all.c b/tools/build/feature/test-all.c
+index 9204395272912..5ffafb967b6e4 100644
+--- a/tools/build/feature/test-all.c
++++ b/tools/build/feature/test-all.c
+@@ -14,10 +14,6 @@
+ # include "test-libpython.c"
+ #undef main
+
+-#define main main_test_libpython_version
+-# include "test-libpython-version.c"
+-#undef main
+-
+ #define main main_test_libperl
+ # include "test-libperl.c"
+ #undef main
+@@ -177,7 +173,6 @@
+ int main(int argc, char *argv[])
+ {
+ main_test_libpython();
+- main_test_libpython_version();
+ main_test_libperl();
+ main_test_hello();
+ main_test_libelf();
+@@ -200,7 +195,6 @@ int main(int argc, char *argv[])
+ main_test_timerfd();
+ main_test_stackprotector_all();
+ main_test_libdw_dwarf_unwind();
+- main_test_sync_compare_and_swap(argc, argv);
+ main_test_zlib();
+ main_test_pthread_attr_setaffinity_np();
+ main_test_pthread_barrier();
+diff --git a/tools/build/feature/test-libbpf-btf__load_from_kernel_by_id.c b/tools/build/feature/test-libbpf-btf__load_from_kernel_by_id.c
+new file mode 100644
+index 0000000000000..f7c084428735a
+--- /dev/null
++++ b/tools/build/feature/test-libbpf-btf__load_from_kernel_by_id.c
+@@ -0,0 +1,7 @@
++// SPDX-License-Identifier: GPL-2.0
++#include <bpf/libbpf.h>
++
++int main(void)
++{
++ return btf__load_from_kernel_by_id(20151128, NULL);
++}
+diff --git a/tools/build/feature/test-libpython-version.c b/tools/build/feature/test-libpython-version.c
+deleted file mode 100644
+index 47714b942d4d3..0000000000000
+--- a/tools/build/feature/test-libpython-version.c
++++ /dev/null
+@@ -1,11 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0
+-#include <Python.h>
+-
+-#if PY_VERSION_HEX >= 0x03000000
+- #error
+-#endif
+-
+-int main(void)
+-{
+- return 0;
+-}
+diff --git a/tools/include/asm-generic/unaligned.h b/tools/include/asm-generic/unaligned.h
+new file mode 100644
+index 0000000000000..47387c607035e
+--- /dev/null
++++ b/tools/include/asm-generic/unaligned.h
+@@ -0,0 +1,23 @@
++/* SPDX-License-Identifier: GPL-2.0-or-later */
++/*
++ * Copied from the kernel sources to tools/perf/:
++ */
++
++#ifndef __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H
++#define __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H
++
++#define __get_unaligned_t(type, ptr) ({ \
++ const struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
++ __pptr->x; \
++})
++
++#define __put_unaligned_t(type, val, ptr) do { \
++ struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
++ __pptr->x = (val); \
++} while (0)
++
++#define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
++#define put_unaligned(val, ptr) __put_unaligned_t(typeof(*(ptr)), (val), (ptr))
++
++#endif /* __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H */
++
+diff --git a/tools/include/nolibc/nolibc.h b/tools/include/nolibc/nolibc.h
+index 3430667b0d241..3e2c6f2ed587f 100644
+--- a/tools/include/nolibc/nolibc.h
++++ b/tools/include/nolibc/nolibc.h
+@@ -399,16 +399,22 @@ struct stat {
+ })
+
+ /* startup code */
++/*
++ * x86-64 System V ABI mandates:
++ * 1) %rsp must be 16-byte aligned right before the function call.
++ * 2) The deepest stack frame should be zero (the %rbp).
++ *
++ */
+ asm(".section .text\n"
+ ".global _start\n"
+ "_start:\n"
+ "pop %rdi\n" // argc (first arg, %rdi)
+ "mov %rsp, %rsi\n" // argv[] (second arg, %rsi)
+ "lea 8(%rsi,%rdi,8),%rdx\n" // then a NULL then envp (third arg, %rdx)
+- "and $-16, %rsp\n" // x86 ABI : esp must be 16-byte aligned when
+- "sub $8, %rsp\n" // entering the callee
++ "xor %ebp, %ebp\n" // zero the stack frame
++ "and $-16, %rsp\n" // x86 ABI : esp must be 16-byte aligned before call
+ "call main\n" // main() returns the status code, we'll exit with it.
+- "movzb %al, %rdi\n" // retrieve exit code from 8 lower bits
++ "mov %eax, %edi\n" // retrieve exit code (32 bit)
+ "mov $60, %rax\n" // NR_exit == 60
+ "syscall\n" // really exit
+ "hlt\n" // ensure it does not return
+@@ -577,20 +583,28 @@ struct sys_stat_struct {
+ })
+
+ /* startup code */
++/*
++ * i386 System V ABI mandates:
++ * 1) last pushed argument must be 16-byte aligned.
++ * 2) The deepest stack frame should be set to zero
++ *
++ */
+ asm(".section .text\n"
+ ".global _start\n"
+ "_start:\n"
+ "pop %eax\n" // argc (first arg, %eax)
+ "mov %esp, %ebx\n" // argv[] (second arg, %ebx)
+ "lea 4(%ebx,%eax,4),%ecx\n" // then a NULL then envp (third arg, %ecx)
+- "and $-16, %esp\n" // x86 ABI : esp must be 16-byte aligned when
++ "xor %ebp, %ebp\n" // zero the stack frame
++ "and $-16, %esp\n" // x86 ABI : esp must be 16-byte aligned before
++ "sub $4, %esp\n" // the call instruction (args are aligned)
+ "push %ecx\n" // push all registers on the stack so that we
+ "push %ebx\n" // support both regparm and plain stack modes
+ "push %eax\n"
+ "call main\n" // main() returns the status code in %eax
+- "movzbl %al, %ebx\n" // retrieve exit code from lower 8 bits
+- "movl $1, %eax\n" // NR_exit == 1
+- "int $0x80\n" // exit now
++ "mov %eax, %ebx\n" // retrieve exit code (32-bit int)
++ "movl $1, %eax\n" // NR_exit == 1
++ "int $0x80\n" // exit now
+ "hlt\n" // ensure it does not
+ "");
+
+@@ -774,7 +788,6 @@ asm(".section .text\n"
+ "and %r3, %r1, $-8\n" // AAPCS : sp must be 8-byte aligned in the
+ "mov %sp, %r3\n" // callee, an bl doesn't push (lr=pc)
+ "bl main\n" // main() returns the status code, we'll exit with it.
+- "and %r0, %r0, $0xff\n" // limit exit code to 8 bits
+ "movs r7, $1\n" // NR_exit == 1
+ "svc $0x00\n"
+ "");
+@@ -971,7 +984,6 @@ asm(".section .text\n"
+ "add x2, x2, x1\n" // + argv
+ "and sp, x1, -16\n" // sp must be 16-byte aligned in the callee
+ "bl main\n" // main() returns the status code, we'll exit with it.
+- "and x0, x0, 0xff\n" // limit exit code to 8 bits
+ "mov x8, 93\n" // NR_exit == 93
+ "svc #0\n"
+ "");
+@@ -1176,7 +1188,7 @@ asm(".section .text\n"
+ "addiu $sp,$sp,-16\n" // the callee expects to save a0..a3 there!
+ "jal main\n" // main() returns the status code, we'll exit with it.
+ "nop\n" // delayed slot
+- "and $a0, $v0, 0xff\n" // limit exit code to 8 bits
++ "move $a0, $v0\n" // retrieve 32-bit exit code from v0
+ "li $v0, 4001\n" // NR_exit == 4001
+ "syscall\n"
+ ".end __start\n"
+@@ -1374,7 +1386,6 @@ asm(".section .text\n"
+ "add a2,a2,a1\n" // + argv
+ "andi sp,a1,-16\n" // sp must be 16-byte aligned
+ "call main\n" // main() returns the status code, we'll exit with it.
+- "andi a0, a0, 0xff\n" // limit exit code to 8 bits
+ "li a7, 93\n" // NR_exit == 93
+ "ecall\n"
+ "");
+diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
+index 791f31dd0abee..e2c8f946c5416 100644
+--- a/tools/include/uapi/linux/bpf.h
++++ b/tools/include/uapi/linux/bpf.h
+@@ -2276,8 +2276,8 @@ union bpf_attr {
+ * Return
+ * The return value depends on the result of the test, and can be:
+ *
+- * * 0, if current task belongs to the cgroup2.
+- * * 1, if current task does not belong to the cgroup2.
++ * * 1, if current task belongs to the cgroup2.
++ * * 0, if current task does not belong to the cgroup2.
+ * * A negative error code, if an error occurred.
+ *
+ * long bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags)
+diff --git a/tools/include/uapi/sound/asound.h b/tools/include/uapi/sound/asound.h
+index 5859ca0a1439b..93e40f91bd49a 100644
+--- a/tools/include/uapi/sound/asound.h
++++ b/tools/include/uapi/sound/asound.h
+@@ -56,8 +56,10 @@
+ * *
+ ****************************************************************************/
+
++#define AES_IEC958_STATUS_SIZE 24
++
+ struct snd_aes_iec958 {
+- unsigned char status[24]; /* AES/IEC958 channel status bits */
++ unsigned char status[AES_IEC958_STATUS_SIZE]; /* AES/IEC958 channel status bits */
+ unsigned char subcode[147]; /* AES/IEC958 subcode bits */
+ unsigned char pad; /* nothing */
+ unsigned char dig_subframe[4]; /* AES/IEC958 subframe bits */
+diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
+index 74c3b73a5fbe8..089b73b3cb379 100644
+--- a/tools/lib/bpf/Makefile
++++ b/tools/lib/bpf/Makefile
+@@ -126,7 +126,7 @@ GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \
+ sort -u | wc -l)
+ VERSIONED_SYM_COUNT = $(shell readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \
+ sed 's/\[.*\]//' | \
+- awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \
++ awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}' | \
+ grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l)
+
+ CMD_TARGETS = $(LIB_TARGET) $(PC_FILE)
+@@ -195,7 +195,7 @@ check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT)
+ sort -u > $(OUTPUT)libbpf_global_syms.tmp; \
+ readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \
+ sed 's/\[.*\]//' | \
+- awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \
++ awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}'| \
+ grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | \
+ sort -u > $(OUTPUT)libbpf_versioned_syms.tmp; \
+ diff -u $(OUTPUT)libbpf_global_syms.tmp \
+diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
+index 2401fad090c52..bfd1ce9fe2110 100644
+--- a/tools/lib/bpf/bpf.c
++++ b/tools/lib/bpf/bpf.c
+@@ -480,6 +480,7 @@ int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value)
+ int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags)
+ {
+ union bpf_attr attr;
++ int ret;
+
+ memset(&attr, 0, sizeof(attr));
+ attr.map_fd = fd;
+@@ -487,7 +488,8 @@ int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, _
+ attr.value = ptr_to_u64(value);
+ attr.flags = flags;
+
+- return sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr));
++ ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr));
++ return libbpf_err_errno(ret);
+ }
+
+ int bpf_map_delete_elem(int fd, const void *key)
+diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h
+index 09ebe3db5f2f8..e4aa9996a5501 100644
+--- a/tools/lib/bpf/bpf_core_read.h
++++ b/tools/lib/bpf/bpf_core_read.h
+@@ -40,7 +40,7 @@ enum bpf_enum_value_kind {
+ #define __CORE_RELO(src, field, info) \
+ __builtin_preserve_field_info((src)->field, BPF_FIELD_##info)
+
+-#if __BYTE_ORDER == __LITTLE_ENDIAN
++#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ #define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \
+ bpf_probe_read_kernel( \
+ (void *)dst, \
+diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
+index 77dc24d58302d..5f3d20ae66d56 100644
+--- a/tools/lib/bpf/btf.c
++++ b/tools/lib/bpf/btf.c
+@@ -231,17 +231,23 @@ static int btf_parse_hdr(struct btf *btf)
+ }
+ btf_bswap_hdr(hdr);
+ } else if (hdr->magic != BTF_MAGIC) {
+- pr_debug("Invalid BTF magic:%x\n", hdr->magic);
++ pr_debug("Invalid BTF magic: %x\n", hdr->magic);
+ return -EINVAL;
+ }
+
+- meta_left = btf->raw_size - sizeof(*hdr);
+- if (meta_left < hdr->str_off + hdr->str_len) {
+- pr_debug("Invalid BTF total size:%u\n", btf->raw_size);
++ if (btf->raw_size < hdr->hdr_len) {
++ pr_debug("BTF header len %u larger than data size %u\n",
++ hdr->hdr_len, btf->raw_size);
+ return -EINVAL;
+ }
+
+- if (hdr->type_off + hdr->type_len > hdr->str_off) {
++ meta_left = btf->raw_size - hdr->hdr_len;
++ if (meta_left < (long long)hdr->str_off + hdr->str_len) {
++ pr_debug("Invalid BTF total size: %u\n", btf->raw_size);
++ return -EINVAL;
++ }
++
++ if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) {
+ pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
+ hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
+ return -EINVAL;
+@@ -2620,15 +2626,11 @@ void btf_ext__free(struct btf_ext *btf_ext)
+ free(btf_ext);
+ }
+
+-struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
++struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
+ {
+ struct btf_ext *btf_ext;
+ int err;
+
+- err = btf_ext_parse_hdr(data, size);
+- if (err)
+- return libbpf_err_ptr(err);
+-
+ btf_ext = calloc(1, sizeof(struct btf_ext));
+ if (!btf_ext)
+ return libbpf_err_ptr(-ENOMEM);
+@@ -2641,6 +2643,10 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
+ }
+ memcpy(btf_ext->data, data, size);
+
++ err = btf_ext_parse_hdr(btf_ext->data, size);
++ if (err)
++ goto done;
++
+ if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
+ err = -EINVAL;
+ goto done;
+@@ -2914,8 +2920,10 @@ int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
+ return libbpf_err(-EINVAL);
+ }
+
+- if (btf_ensure_modifiable(btf))
+- return libbpf_err(-ENOMEM);
++ if (btf_ensure_modifiable(btf)) {
++ err = -ENOMEM;
++ goto done;
++ }
+
+ err = btf_dedup_prep(d);
+ if (err) {
+@@ -3350,8 +3358,8 @@ static long btf_hash_struct(struct btf_type *t)
+ }
+
+ /*
+- * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
+- * IDs. This check is performed during type graph equivalence check and
++ * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced
++ * type IDs. This check is performed during type graph equivalence check and
+ * referenced types equivalence is checked separately.
+ */
+ static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
+@@ -3722,6 +3730,31 @@ static int btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
+ return btf_equal_array(t1, t2);
+ }
+
++/* Check if given two types are identical STRUCT/UNION definitions */
++static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2)
++{
++ const struct btf_member *m1, *m2;
++ struct btf_type *t1, *t2;
++ int n, i;
++
++ t1 = btf_type_by_id(d->btf, id1);
++ t2 = btf_type_by_id(d->btf, id2);
++
++ if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2))
++ return false;
++
++ if (!btf_shallow_equal_struct(t1, t2))
++ return false;
++
++ m1 = btf_members(t1);
++ m2 = btf_members(t2);
++ for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
++ if (m1->type != m2->type)
++ return false;
++ }
++ return true;
++}
++
+ /*
+ * Check equivalence of BTF type graph formed by candidate struct/union (we'll
+ * call it "candidate graph" in this description for brevity) to a type graph
+@@ -3833,6 +3866,8 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
+
+ hypot_type_id = d->hypot_map[canon_id];
+ if (hypot_type_id <= BTF_MAX_NR_TYPES) {
++ if (hypot_type_id == cand_id)
++ return 1;
+ /* In some cases compiler will generate different DWARF types
+ * for *identical* array type definitions and use them for
+ * different fields within the *same* struct. This breaks type
+@@ -3841,8 +3876,18 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
+ * types within a single CU. So work around that by explicitly
+ * allowing identical array types here.
+ */
+- return hypot_type_id == cand_id ||
+- btf_dedup_identical_arrays(d, hypot_type_id, cand_id);
++ if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
++ return 1;
++ /* It turns out that similar situation can happen with
++ * struct/union sometimes, sigh... Handle the case where
++ * structs/unions are exactly the same, down to the referenced
++ * type IDs. Anything more complicated (e.g., if referenced
++ * types are different, but equivalent) is *way more*
++ * complicated and requires a many-to-many equivalence mapping.
++ */
++ if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
++ return 1;
++ return 0;
+ }
+
+ if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
+diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
+index 4a711f990904b..b0ee338a0cc87 100644
+--- a/tools/lib/bpf/btf.h
++++ b/tools/lib/bpf/btf.h
+@@ -80,7 +80,7 @@ LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
+ __u32 expected_value_size,
+ __u32 *key_type_id, __u32 *value_type_id);
+
+-LIBBPF_API struct btf_ext *btf_ext__new(__u8 *data, __u32 size);
++LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);
+ LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);
+ LIBBPF_API const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext,
+ __u32 *size);
+diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c
+index e4b483f15fb99..841cc68e3f427 100644
+--- a/tools/lib/bpf/btf_dump.c
++++ b/tools/lib/bpf/btf_dump.c
+@@ -1481,6 +1481,11 @@ static const char *btf_dump_resolve_name(struct btf_dump *d, __u32 id,
+ if (s->name_resolved)
+ return *cached_name ? *cached_name : orig_name;
+
++ if (btf_is_fwd(t) || (btf_is_enum(t) && btf_vlen(t) == 0)) {
++ s->name_resolved = 1;
++ return orig_name;
++ }
++
+ dup_cnt = btf_dump_name_dups(d, name_map, orig_name);
+ if (dup_cnt > 1) {
+ const size_t max_len = 256;
+@@ -1829,14 +1834,16 @@ static int btf_dump_array_data(struct btf_dump *d,
+ {
+ const struct btf_array *array = btf_array(t);
+ const struct btf_type *elem_type;
+- __u32 i, elem_size = 0, elem_type_id;
++ __u32 i, elem_type_id;
++ __s64 elem_size;
+ bool is_array_member;
+
+ elem_type_id = array->type;
+ elem_type = skip_mods_and_typedefs(d->btf, elem_type_id, NULL);
+ elem_size = btf__resolve_size(d->btf, elem_type_id);
+ if (elem_size <= 0) {
+- pr_warn("unexpected elem size %d for array type [%u]\n", elem_size, id);
++ pr_warn("unexpected elem size %zd for array type [%u]\n",
++ (ssize_t)elem_size, id);
+ return -EINVAL;
+ }
+
+@@ -2186,7 +2193,7 @@ static int btf_dump_dump_type_data(struct btf_dump *d,
+ __u8 bits_offset,
+ __u8 bit_sz)
+ {
+- int size, err;
++ int size, err = 0;
+
+ size = btf_dump_type_data_check_overflow(d, t, id, data, bits_offset);
+ if (size < 0)
+diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c
+index 8df718a6b142d..33c19590ee434 100644
+--- a/tools/lib/bpf/gen_loader.c
++++ b/tools/lib/bpf/gen_loader.c
+@@ -663,9 +663,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
+ debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
+ /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
+ cleanup_relos(gen, insns);
+- if (gen->attach_kind)
++ if (gen->attach_kind) {
+ emit_sys_close_blob(gen,
+ attr_field(prog_load_attr, attach_btf_obj_fd));
++ gen->attach_kind = 0;
++ }
+ emit_check_err(gen);
+ /* remember prog_fd in the stack, if successful */
+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
+diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
+index e4f83c304ec92..693e14799fb96 100644
+--- a/tools/lib/bpf/libbpf.c
++++ b/tools/lib/bpf/libbpf.c
+@@ -2993,6 +2993,12 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
+ }
+ }
+
++ if (!obj->efile.symbols) {
++ pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
++ obj->path);
++ return -ENOENT;
++ }
++
+ scn = NULL;
+ while ((scn = elf_nextscn(elf, scn)) != NULL) {
+ idx++;
+@@ -5132,7 +5138,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
+ * relocated, so it's enough to just subtract in-section offset
+ */
+ insn_idx = insn_idx - prog->sec_insn_off;
+- if (insn_idx > prog->insns_cnt)
++ if (insn_idx >= prog->insns_cnt)
+ return -EINVAL;
+ insn = &prog->insns[insn_idx];
+
+@@ -8670,7 +8676,10 @@ int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
+ pr_warn("error: inner_map_fd already specified\n");
+ return libbpf_err(-EINVAL);
+ }
+- zfree(&map->inner_map);
++ if (map->inner_map) {
++ bpf_map__destroy(map->inner_map);
++ zfree(&map->inner_map);
++ }
+ map->inner_map_fd = fd;
+ return 0;
+ }
+@@ -10800,6 +10809,9 @@ void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
+
+ void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
+ {
++ if (!s)
++ return;
++
+ if (s->progs)
+ bpf_object__detach_skeleton(s);
+ if (s->obj)
+diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c
+index 2df880cefdaee..6b2f59ddb6918 100644
+--- a/tools/lib/bpf/linker.c
++++ b/tools/lib/bpf/linker.c
+@@ -211,6 +211,7 @@ void bpf_linker__free(struct bpf_linker *linker)
+ }
+ free(linker->secs);
+
++ free(linker->glob_syms);
+ free(linker);
+ }
+
+@@ -2000,7 +2001,7 @@ add_sym:
+ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
+ {
+ struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
+- struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
++ struct dst_sec *dst_symtab;
+ int i, err;
+
+ for (i = 1; i < obj->sec_cnt; i++) {
+@@ -2033,6 +2034,9 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob
+ return -1;
+ }
+
++ /* add_dst_sec() above could have invalidated linker->secs */
++ dst_symtab = &linker->secs[linker->symtab_sec_idx];
++
+ /* shdr->sh_link points to SYMTAB */
+ dst_sec->shdr->sh_link = linker->symtab_sec_idx;
+
+diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c
+index 39f25e09b51e2..fadde7d80a51c 100644
+--- a/tools/lib/bpf/netlink.c
++++ b/tools/lib/bpf/netlink.c
+@@ -87,29 +87,75 @@ enum {
+ NL_DONE,
+ };
+
++static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags)
++{
++ int len;
++
++ do {
++ len = recvmsg(sock, mhdr, flags);
++ } while (len < 0 && (errno == EINTR || errno == EAGAIN));
++
++ if (len < 0)
++ return -errno;
++ return len;
++}
++
++static int alloc_iov(struct iovec *iov, int len)
++{
++ void *nbuf;
++
++ nbuf = realloc(iov->iov_base, len);
++ if (!nbuf)
++ return -ENOMEM;
++
++ iov->iov_base = nbuf;
++ iov->iov_len = len;
++ return 0;
++}
++
+ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq,
+ __dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn,
+ void *cookie)
+ {
++ struct iovec iov = {};
++ struct msghdr mhdr = {
++ .msg_iov = &iov,
++ .msg_iovlen = 1,
++ };
+ bool multipart = true;
+ struct nlmsgerr *err;
+ struct nlmsghdr *nh;
+- char buf[4096];
+ int len, ret;
+
++ ret = alloc_iov(&iov, 4096);
++ if (ret)
++ goto done;
++
+ while (multipart) {
+ start:
+ multipart = false;
+- len = recv(sock, buf, sizeof(buf), 0);
++ len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC);
++ if (len < 0) {
++ ret = len;
++ goto done;
++ }
++
++ if (len > iov.iov_len) {
++ ret = alloc_iov(&iov, len);
++ if (ret)
++ goto done;
++ }
++
++ len = netlink_recvmsg(sock, &mhdr, 0);
+ if (len < 0) {
+- ret = -errno;
++ ret = len;
+ goto done;
+ }
+
+ if (len == 0)
+ break;
+
+- for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len);
++ for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len);
+ nh = NLMSG_NEXT(nh, len)) {
+ if (nh->nlmsg_pid != nl_pid) {
+ ret = -LIBBPF_ERRNO__WRNGPID;
+@@ -130,7 +176,8 @@ start:
+ libbpf_nla_dump_errormsg(nh);
+ goto done;
+ case NLMSG_DONE:
+- return 0;
++ ret = 0;
++ goto done;
+ default:
+ break;
+ }
+@@ -142,15 +189,17 @@ start:
+ case NL_NEXT:
+ goto start;
+ case NL_DONE:
+- return 0;
++ ret = 0;
++ goto done;
+ default:
+- return ret;
++ goto done;
+ }
+ }
+ }
+ }
+ ret = 0;
+ done:
++ free(iov.iov_base);
+ return ret;
+ }
+
+diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h
+index b22b50c1b173e..9cf66702fa8dd 100644
+--- a/tools/lib/bpf/skel_internal.h
++++ b/tools/lib/bpf/skel_internal.h
+@@ -105,10 +105,12 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
+ err = skel_sys_bpf(BPF_PROG_RUN, &attr, sizeof(attr));
+ if (err < 0 || (int)attr.test.retval < 0) {
+ opts->errstr = "failed to execute loader prog";
+- if (err < 0)
++ if (err < 0) {
+ err = -errno;
+- else
++ } else {
+ err = (int)attr.test.retval;
++ errno = -err;
++ }
+ goto out;
+ }
+ err = 0;
+diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c
+index e9b619aa0cdf3..a27b3141463ad 100644
+--- a/tools/lib/bpf/xsk.c
++++ b/tools/lib/bpf/xsk.c
+@@ -1210,12 +1210,23 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
+
+ int xsk_umem__delete(struct xsk_umem *umem)
+ {
++ struct xdp_mmap_offsets off;
++ int err;
++
+ if (!umem)
+ return 0;
+
+ if (umem->refcount)
+ return -EBUSY;
+
++ err = xsk_get_mmap_offsets(umem->fd, &off);
++ if (!err && umem->fill_save && umem->comp_save) {
++ munmap(umem->fill_save->ring - off.fr.desc,
++ off.fr.desc + umem->config.fill_size * sizeof(__u64));
++ munmap(umem->comp_save->ring - off.cr.desc,
++ off.cr.desc + umem->config.comp_size * sizeof(__u64));
++ }
++
+ close(umem->fd);
+ free(umem);
+
+diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c
+index e37dfad31383f..5146ff0fa078c 100644
+--- a/tools/lib/perf/evlist.c
++++ b/tools/lib/perf/evlist.c
+@@ -577,7 +577,6 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist,
+ {
+ struct perf_evsel *evsel;
+ const struct perf_cpu_map *cpus = evlist->cpus;
+- const struct perf_thread_map *threads = evlist->threads;
+
+ if (!ops || !ops->get || !ops->mmap)
+ return -EINVAL;
+@@ -589,7 +588,7 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist,
+ perf_evlist__for_each_entry(evlist, evsel) {
+ if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
+ evsel->sample_id == NULL &&
+- perf_evsel__alloc_id(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
++ perf_evsel__alloc_id(evsel, evsel->fd->max_x, evsel->fd->max_y) < 0)
+ return -ENOMEM;
+ }
+
+diff --git a/tools/lib/subcmd/subcmd-util.h b/tools/lib/subcmd/subcmd-util.h
+index 794a375dad360..b2aec04fce8f6 100644
+--- a/tools/lib/subcmd/subcmd-util.h
++++ b/tools/lib/subcmd/subcmd-util.h
+@@ -50,15 +50,8 @@ static NORETURN inline void die(const char *err, ...)
+ static inline void *xrealloc(void *ptr, size_t size)
+ {
+ void *ret = realloc(ptr, size);
+- if (!ret && !size)
+- ret = realloc(ptr, 1);
+- if (!ret) {
+- ret = realloc(ptr, size);
+- if (!ret && !size)
+- ret = realloc(ptr, 1);
+- if (!ret)
+- die("Out of memory, realloc failed");
+- }
++ if (!ret)
++ die("Out of memory, realloc failed");
+ return ret;
+ }
+
+diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
+index 0893436cc09f8..63ffbc36dacc2 100644
+--- a/tools/objtool/arch/x86/decode.c
++++ b/tools/objtool/arch/x86/decode.c
+@@ -529,6 +529,11 @@ int arch_decode_instruction(const struct elf *elf, const struct section *sec,
+ }
+ break;
+
++ case 0xcc:
++ /* int3 */
++ *type = INSN_TRAP;
++ break;
++
+ case 0xe3:
+ /* jecxz/jrcxz */
+ *type = INSN_JUMP_CONDITIONAL;
+@@ -659,6 +664,26 @@ const char *arch_nop_insn(int len)
+ return nops[len-1];
+ }
+
++#define BYTE_RET 0xC3
++
++const char *arch_ret_insn(int len)
++{
++ static const char ret[5][5] = {
++ { BYTE_RET },
++ { BYTE_RET, 0xcc },
++ { BYTE_RET, 0xcc, BYTES_NOP1 },
++ { BYTE_RET, 0xcc, BYTES_NOP2 },
++ { BYTE_RET, 0xcc, BYTES_NOP3 },
++ };
++
++ if (len < 1 || len > 5) {
++ WARN("invalid RET size: %d\n", len);
++ return NULL;
++ }
++
++ return ret[len-1];
++}
++
+ /* asm/alternative.h ? */
+
+ #define ALTINSTR_FLAG_INV (1 << 15)
+diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c
+index 8b38b5d6fec7b..38070f26105bd 100644
+--- a/tools/objtool/builtin-check.c
++++ b/tools/objtool/builtin-check.c
+@@ -20,7 +20,7 @@
+ #include <objtool/objtool.h>
+
+ bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
+- validate_dup, vmlinux, mcount, noinstr, backup;
++ validate_dup, vmlinux, mcount, noinstr, backup, sls;
+
+ static const char * const check_usage[] = {
+ "objtool check [<options>] file.o",
+@@ -45,6 +45,7 @@ const struct option check_options[] = {
+ OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"),
+ OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
+ OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"),
++ OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"),
+ OPT_END(),
+ };
+
+diff --git a/tools/objtool/check.c b/tools/objtool/check.c
+index 06b5c164ae931..66c7c13098b31 100644
+--- a/tools/objtool/check.c
++++ b/tools/objtool/check.c
+@@ -173,6 +173,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
+ "rewind_stack_do_exit",
+ "kunit_try_catch_throw",
+ "xen_start_kernel",
++ "cpu_bringup_and_idle",
+ };
+
+ if (!func)
+@@ -392,12 +393,12 @@ static int add_dead_ends(struct objtool_file *file)
+ else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
+ insn = find_last_insn(file, reloc->sym->sec);
+ if (!insn) {
+- WARN("can't find unreachable insn at %s+0x%x",
++ WARN("can't find unreachable insn at %s+0x%lx",
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+ } else {
+- WARN("can't find unreachable insn at %s+0x%x",
++ WARN("can't find unreachable insn at %s+0x%lx",
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+@@ -427,12 +428,12 @@ reachable:
+ else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
+ insn = find_last_insn(file, reloc->sym->sec);
+ if (!insn) {
+- WARN("can't find reachable insn at %s+0x%x",
++ WARN("can't find reachable insn at %s+0x%lx",
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+ } else {
+- WARN("can't find reachable insn at %s+0x%x",
++ WARN("can't find reachable insn at %s+0x%lx",
+ reloc->sym->sec->name, reloc->addend);
+ return -1;
+ }
+@@ -828,6 +829,89 @@ static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *i
+ return insn->reloc;
+ }
+
++static void remove_insn_ops(struct instruction *insn)
++{
++ struct stack_op *op, *tmp;
++
++ list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
++ list_del(&op->list);
++ free(op);
++ }
++}
++
++static void add_call_dest(struct objtool_file *file, struct instruction *insn,
++ struct symbol *dest, bool sibling)
++{
++ struct reloc *reloc = insn_reloc(file, insn);
++
++ insn->call_dest = dest;
++ if (!dest)
++ return;
++
++ if (insn->call_dest->static_call_tramp) {
++ list_add_tail(&insn->call_node,
++ &file->static_call_list);
++ }
++
++ /*
++ * Many compilers cannot disable KCOV with a function attribute
++ * so they need a little help, NOP out any KCOV calls from noinstr
++ * text.
++ */
++ if (insn->sec->noinstr &&
++ !strncmp(insn->call_dest->name, "__sanitizer_cov_", 16)) {
++ if (reloc) {
++ reloc->type = R_NONE;
++ elf_write_reloc(file->elf, reloc);
++ }
++
++ elf_write_insn(file->elf, insn->sec,
++ insn->offset, insn->len,
++ sibling ? arch_ret_insn(insn->len)
++ : arch_nop_insn(insn->len));
++
++ insn->type = sibling ? INSN_RETURN : INSN_NOP;
++
++ if (sibling) {
++ /*
++ * We've replaced the tail-call JMP insn by two new
++ * insn: RET; INT3, except we only have a single struct
++ * insn here. Mark it retpoline_safe to avoid the SLS
++ * warning, instead of adding another insn.
++ */
++ insn->retpoline_safe = true;
++ }
++ }
++
++ if (mcount && !strcmp(insn->call_dest->name, "__fentry__")) {
++ if (sibling)
++ WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
++
++ if (reloc) {
++ reloc->type = R_NONE;
++ elf_write_reloc(file->elf, reloc);
++ }
++
++ elf_write_insn(file->elf, insn->sec,
++ insn->offset, insn->len,
++ arch_nop_insn(insn->len));
++
++ insn->type = INSN_NOP;
++
++ list_add_tail(&insn->mcount_loc_node,
++ &file->mcount_loc_list);
++ }
++
++ /*
++ * Whatever stack impact regular CALLs have, should be undone
++ * by the RETURN of the called function.
++ *
++ * Annotated intra-function calls retain the stack_ops but
++ * are converted to JUMP, see read_intra_function_calls().
++ */
++ remove_insn_ops(insn);
++}
++
+ /*
+ * Find the destination instructions for all jumps.
+ */
+@@ -866,11 +950,7 @@ static int add_jump_destinations(struct objtool_file *file)
+ continue;
+ } else if (insn->func) {
+ /* internal or external sibling call (with reloc) */
+- insn->call_dest = reloc->sym;
+- if (insn->call_dest->static_call_tramp) {
+- list_add_tail(&insn->call_node,
+- &file->static_call_list);
+- }
++ add_call_dest(file, insn, reloc->sym, true);
+ continue;
+ } else if (reloc->sym->sec->idx) {
+ dest_sec = reloc->sym->sec;
+@@ -926,13 +1006,8 @@ static int add_jump_destinations(struct objtool_file *file)
+
+ } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
+ insn->jump_dest->offset == insn->jump_dest->func->offset) {
+-
+ /* internal sibling call (without reloc) */
+- insn->call_dest = insn->jump_dest->func;
+- if (insn->call_dest->static_call_tramp) {
+- list_add_tail(&insn->call_node,
+- &file->static_call_list);
+- }
++ add_call_dest(file, insn, insn->jump_dest->func, true);
+ }
+ }
+ }
+@@ -940,16 +1015,6 @@ static int add_jump_destinations(struct objtool_file *file)
+ return 0;
+ }
+
+-static void remove_insn_ops(struct instruction *insn)
+-{
+- struct stack_op *op, *tmp;
+-
+- list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
+- list_del(&op->list);
+- free(op);
+- }
+-}
+-
+ static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
+ {
+ struct symbol *call_dest;
+@@ -968,6 +1033,7 @@ static int add_call_destinations(struct objtool_file *file)
+ {
+ struct instruction *insn;
+ unsigned long dest_off;
++ struct symbol *dest;
+ struct reloc *reloc;
+
+ for_each_insn(file, insn) {
+@@ -977,7 +1043,9 @@ static int add_call_destinations(struct objtool_file *file)
+ reloc = insn_reloc(file, insn);
+ if (!reloc) {
+ dest_off = arch_jump_destination(insn);
+- insn->call_dest = find_call_destination(insn->sec, dest_off);
++ dest = find_call_destination(insn->sec, dest_off);
++
++ add_call_dest(file, insn, dest, false);
+
+ if (insn->ignore)
+ continue;
+@@ -995,9 +1063,8 @@ static int add_call_destinations(struct objtool_file *file)
+
+ } else if (reloc->sym->type == STT_SECTION) {
+ dest_off = arch_dest_reloc_offset(reloc->addend);
+- insn->call_dest = find_call_destination(reloc->sym->sec,
+- dest_off);
+- if (!insn->call_dest) {
++ dest = find_call_destination(reloc->sym->sec, dest_off);
++ if (!dest) {
+ WARN_FUNC("can't find call dest symbol at %s+0x%lx",
+ insn->sec, insn->offset,
+ reloc->sym->sec->name,
+@@ -1005,6 +1072,8 @@ static int add_call_destinations(struct objtool_file *file)
+ return -1;
+ }
+
++ add_call_dest(file, insn, dest, false);
++
+ } else if (arch_is_retpoline(reloc->sym)) {
+ /*
+ * Retpoline calls are really dynamic calls in
+@@ -1020,55 +1089,7 @@ static int add_call_destinations(struct objtool_file *file)
+ continue;
+
+ } else
+- insn->call_dest = reloc->sym;
+-
+- if (insn->call_dest && insn->call_dest->static_call_tramp) {
+- list_add_tail(&insn->call_node,
+- &file->static_call_list);
+- }
+-
+- /*
+- * Many compilers cannot disable KCOV with a function attribute
+- * so they need a little help, NOP out any KCOV calls from noinstr
+- * text.
+- */
+- if (insn->sec->noinstr &&
+- !strncmp(insn->call_dest->name, "__sanitizer_cov_", 16)) {
+- if (reloc) {
+- reloc->type = R_NONE;
+- elf_write_reloc(file->elf, reloc);
+- }
+-
+- elf_write_insn(file->elf, insn->sec,
+- insn->offset, insn->len,
+- arch_nop_insn(insn->len));
+- insn->type = INSN_NOP;
+- }
+-
+- if (mcount && !strcmp(insn->call_dest->name, "__fentry__")) {
+- if (reloc) {
+- reloc->type = R_NONE;
+- elf_write_reloc(file->elf, reloc);
+- }
+-
+- elf_write_insn(file->elf, insn->sec,
+- insn->offset, insn->len,
+- arch_nop_insn(insn->len));
+-
+- insn->type = INSN_NOP;
+-
+- list_add_tail(&insn->mcount_loc_node,
+- &file->mcount_loc_list);
+- }
+-
+- /*
+- * Whatever stack impact regular CALLs have, should be undone
+- * by the RETURN of the called function.
+- *
+- * Annotated intra-function calls retain the stack_ops but
+- * are converted to JUMP, see read_intra_function_calls().
+- */
+- remove_insn_ops(insn);
++ add_call_dest(file, insn, reloc->sym, false);
+ }
+
+ return 0;
+@@ -2765,6 +2786,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
+ switch (insn->type) {
+
+ case INSN_RETURN:
++ if (next_insn && next_insn->type == INSN_TRAP) {
++ next_insn->ignore = true;
++ } else if (sls && !insn->retpoline_safe) {
++ WARN_FUNC("missing int3 after ret",
++ insn->sec, insn->offset);
++ }
+ return validate_return(func, insn, &state);
+
+ case INSN_CALL:
+@@ -2808,6 +2835,14 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
+ break;
+
+ case INSN_JUMP_DYNAMIC:
++ if (next_insn && next_insn->type == INSN_TRAP) {
++ next_insn->ignore = true;
++ } else if (sls && !insn->retpoline_safe) {
++ WARN_FUNC("missing int3 after indirect jump",
++ insn->sec, insn->offset);
++ }
++
++ /* fallthrough */
+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
+ if (is_sibling_call(insn)) {
+ ret = validate_sibling_call(insn, &state);
+diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
+index fee03b744a6ea..a3395467c3161 100644
+--- a/tools/objtool/elf.c
++++ b/tools/objtool/elf.c
+@@ -485,7 +485,7 @@ static struct section *elf_create_reloc_section(struct elf *elf,
+ int reltype);
+
+ int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
+- unsigned int type, struct symbol *sym, int addend)
++ unsigned int type, struct symbol *sym, long addend)
+ {
+ struct reloc *reloc;
+
+@@ -514,37 +514,180 @@ int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
+ return 0;
+ }
+
+-int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
+- unsigned long offset, unsigned int type,
+- struct section *insn_sec, unsigned long insn_off)
++/*
++ * Ensure that any reloc section containing references to @sym is marked
++ * changed such that it will get re-generated in elf_rebuild_reloc_sections()
++ * with the new symbol index.
++ */
++static void elf_dirty_reloc_sym(struct elf *elf, struct symbol *sym)
++{
++ struct section *sec;
++
++ list_for_each_entry(sec, &elf->sections, list) {
++ struct reloc *reloc;
++
++ if (sec->changed)
++ continue;
++
++ list_for_each_entry(reloc, &sec->reloc_list, list) {
++ if (reloc->sym == sym) {
++ sec->changed = true;
++ break;
++ }
++ }
++ }
++}
++
++/*
++ * Move the first global symbol, as per sh_info, into a new, higher symbol
++ * index. This fees up the shndx for a new local symbol.
++ */
++static int elf_move_global_symbol(struct elf *elf, struct section *symtab,
++ struct section *symtab_shndx)
+ {
++ Elf_Data *data, *shndx_data = NULL;
++ Elf32_Word first_non_local;
+ struct symbol *sym;
+- int addend;
++ Elf_Scn *s;
+
+- if (insn_sec->sym) {
+- sym = insn_sec->sym;
+- addend = insn_off;
++ first_non_local = symtab->sh.sh_info;
+
+- } else {
+- /*
+- * The Clang assembler strips section symbols, so we have to
+- * reference the function symbol instead:
+- */
+- sym = find_symbol_containing(insn_sec, insn_off);
+- if (!sym) {
+- /*
+- * Hack alert. This happens when we need to reference
+- * the NOP pad insn immediately after the function.
+- */
+- sym = find_symbol_containing(insn_sec, insn_off - 1);
++ sym = find_symbol_by_index(elf, first_non_local);
++ if (!sym) {
++ WARN("no non-local symbols !?");
++ return first_non_local;
++ }
++
++ s = elf_getscn(elf->elf, symtab->idx);
++ if (!s) {
++ WARN_ELF("elf_getscn");
++ return -1;
++ }
++
++ data = elf_newdata(s);
++ if (!data) {
++ WARN_ELF("elf_newdata");
++ return -1;
++ }
++
++ data->d_buf = &sym->sym;
++ data->d_size = sizeof(sym->sym);
++ data->d_align = 1;
++ data->d_type = ELF_T_SYM;
++
++ sym->idx = symtab->sh.sh_size / sizeof(sym->sym);
++ elf_dirty_reloc_sym(elf, sym);
++
++ symtab->sh.sh_info += 1;
++ symtab->sh.sh_size += data->d_size;
++ symtab->changed = true;
++
++ if (symtab_shndx) {
++ s = elf_getscn(elf->elf, symtab_shndx->idx);
++ if (!s) {
++ WARN_ELF("elf_getscn");
++ return -1;
+ }
+
+- if (!sym) {
+- WARN("can't find symbol containing %s+0x%lx", insn_sec->name, insn_off);
++ shndx_data = elf_newdata(s);
++ if (!shndx_data) {
++ WARN_ELF("elf_newshndx_data");
+ return -1;
+ }
+
+- addend = insn_off - sym->offset;
++ shndx_data->d_buf = &sym->sec->idx;
++ shndx_data->d_size = sizeof(Elf32_Word);
++ shndx_data->d_align = 4;
++ shndx_data->d_type = ELF_T_WORD;
++
++ symtab_shndx->sh.sh_size += 4;
++ symtab_shndx->changed = true;
++ }
++
++ return first_non_local;
++}
++
++static struct symbol *
++elf_create_section_symbol(struct elf *elf, struct section *sec)
++{
++ struct section *symtab, *symtab_shndx;
++ Elf_Data *shndx_data = NULL;
++ struct symbol *sym;
++ Elf32_Word shndx;
++
++ symtab = find_section_by_name(elf, ".symtab");
++ if (symtab) {
++ symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
++ if (symtab_shndx)
++ shndx_data = symtab_shndx->data;
++ } else {
++ WARN("no .symtab");
++ return NULL;
++ }
++
++ sym = malloc(sizeof(*sym));
++ if (!sym) {
++ perror("malloc");
++ return NULL;
++ }
++ memset(sym, 0, sizeof(*sym));
++
++ sym->idx = elf_move_global_symbol(elf, symtab, symtab_shndx);
++ if (sym->idx < 0) {
++ WARN("elf_move_global_symbol");
++ return NULL;
++ }
++
++ sym->name = sec->name;
++ sym->sec = sec;
++
++ // st_name 0
++ sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
++ // st_other 0
++ // st_value 0
++ // st_size 0
++ shndx = sec->idx;
++ if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) {
++ sym->sym.st_shndx = shndx;
++ if (!shndx_data)
++ shndx = 0;
++ } else {
++ sym->sym.st_shndx = SHN_XINDEX;
++ if (!shndx_data) {
++ WARN("no .symtab_shndx");
++ return NULL;
++ }
++ }
++
++ if (!gelf_update_symshndx(symtab->data, shndx_data, sym->idx, &sym->sym, shndx)) {
++ WARN_ELF("gelf_update_symshndx");
++ return NULL;
++ }
++
++ elf_add_symbol(elf, sym);
++
++ return sym;
++}
++
++int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
++ unsigned long offset, unsigned int type,
++ struct section *insn_sec, unsigned long insn_off)
++{
++ struct symbol *sym = insn_sec->sym;
++ int addend = insn_off;
++
++ if (!sym) {
++ /*
++ * Due to how weak functions work, we must use section based
++ * relocations. Symbol based relocations would result in the
++ * weak and non-weak function annotations being overlaid on the
++ * non-weak function after linking.
++ */
++ sym = elf_create_section_symbol(elf, insn_sec);
++ if (!sym)
++ return -1;
++
++ insn_sec->sym = sym;
+ }
+
+ return elf_add_reloc(elf, sec, offset, type, sym, addend);
+diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h
+index 062bb6e9b8658..9ca08d95e78ee 100644
+--- a/tools/objtool/include/objtool/arch.h
++++ b/tools/objtool/include/objtool/arch.h
+@@ -26,6 +26,7 @@ enum insn_type {
+ INSN_CLAC,
+ INSN_STD,
+ INSN_CLD,
++ INSN_TRAP,
+ INSN_OTHER,
+ };
+
+@@ -82,6 +83,7 @@ unsigned long arch_jump_destination(struct instruction *insn);
+ unsigned long arch_dest_reloc_offset(int addend);
+
+ const char *arch_nop_insn(int len);
++const char *arch_ret_insn(int len);
+
+ int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg);
+
+diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h
+index 15ac0b7d3d6a3..89ba869ed08fe 100644
+--- a/tools/objtool/include/objtool/builtin.h
++++ b/tools/objtool/include/objtool/builtin.h
+@@ -9,7 +9,7 @@
+
+ extern const struct option check_options[];
+ extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
+- validate_dup, vmlinux, mcount, noinstr, backup;
++ validate_dup, vmlinux, mcount, noinstr, backup, sls;
+
+ extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]);
+
+diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h
+index 075d8291b8546..b4d01f8fd09b8 100644
+--- a/tools/objtool/include/objtool/elf.h
++++ b/tools/objtool/include/objtool/elf.h
+@@ -69,7 +69,7 @@ struct reloc {
+ struct symbol *sym;
+ unsigned long offset;
+ unsigned int type;
+- int addend;
++ long addend;
+ int idx;
+ bool jump_table_start;
+ };
+@@ -131,7 +131,7 @@ struct elf *elf_open_read(const char *name, int flags);
+ struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
+
+ int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
+- unsigned int type, struct symbol *sym, int addend);
++ unsigned int type, struct symbol *sym, long addend);
+ int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
+ unsigned long offset, unsigned int type,
+ struct section *insn_sec, unsigned long insn_off);
+diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config
+index 14e3e8d702a02..e0660bc76b7b4 100644
+--- a/tools/perf/Makefile.config
++++ b/tools/perf/Makefile.config
+@@ -143,7 +143,10 @@ FEATURE_CHECK_LDFLAGS-libcrypto = -lcrypto
+ ifdef CSINCLUDES
+ LIBOPENCSD_CFLAGS := -I$(CSINCLUDES)
+ endif
+-OPENCSDLIBS := -lopencsd_c_api -lopencsd -lstdc++
++OPENCSDLIBS := -lopencsd_c_api
++ifeq ($(findstring -static,${LDFLAGS}),-static)
++ OPENCSDLIBS += -lopencsd -lstdc++
++endif
+ ifdef CSLIBS
+ LIBOPENCSD_LDFLAGS := -L$(CSLIBS)
+ endif
+@@ -267,12 +270,13 @@ ifdef PYTHON_CONFIG
+ PYTHON_EMBED_LIBADD := $(call grep-libs,$(PYTHON_EMBED_LDOPTS)) -lutil
+ PYTHON_EMBED_CCOPTS := $(shell $(PYTHON_CONFIG_SQ) --includes 2>/dev/null)
+ FLAGS_PYTHON_EMBED := $(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS)
++ ifeq ($(CC_NO_CLANG), 0)
++ PYTHON_EMBED_CCOPTS := $(filter-out -ffat-lto-objects, $(PYTHON_EMBED_CCOPTS))
++ endif
+ endif
+
+ FEATURE_CHECK_CFLAGS-libpython := $(PYTHON_EMBED_CCOPTS)
+ FEATURE_CHECK_LDFLAGS-libpython := $(PYTHON_EMBED_LDOPTS)
+-FEATURE_CHECK_CFLAGS-libpython-version := $(PYTHON_EMBED_CCOPTS)
+-FEATURE_CHECK_LDFLAGS-libpython-version := $(PYTHON_EMBED_LDOPTS)
+
+ FEATURE_CHECK_LDFLAGS-libaio = -lrt
+
+@@ -544,9 +548,16 @@ ifndef NO_LIBELF
+ ifeq ($(feature-libbpf), 1)
+ EXTLIBS += -lbpf
+ $(call detected,CONFIG_LIBBPF_DYNAMIC)
++
++ $(call feature_check,libbpf-btf__load_from_kernel_by_id)
++ ifeq ($(feature-libbpf-btf__load_from_kernel_by_id), 1)
++ CFLAGS += -DHAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
++ endif
+ else
+ dummy := $(error Error: No libbpf devel library found, please install libbpf-devel);
+ endif
++ else
++ CFLAGS += -DHAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
+ endif
+ endif
+
+@@ -784,6 +795,9 @@ else
+ LDFLAGS += $(PERL_EMBED_LDFLAGS)
+ EXTLIBS += $(PERL_EMBED_LIBADD)
+ CFLAGS += -DHAVE_LIBPERL_SUPPORT
++ ifeq ($(CC_NO_CLANG), 0)
++ CFLAGS += -Wno-compound-token-split-by-macro
++ endif
+ $(call detected,CONFIG_LIBPERL)
+ endif
+ endif
+diff --git a/tools/perf/arch/arm64/util/Build b/tools/perf/arch/arm64/util/Build
+index 9fcb4e68add93..78dfc282e5e2b 100644
+--- a/tools/perf/arch/arm64/util/Build
++++ b/tools/perf/arch/arm64/util/Build
+@@ -1,5 +1,4 @@
+ perf-y += header.o
+-perf-y += machine.o
+ perf-y += perf_regs.o
+ perf-y += tsc.o
+ perf-y += pmu.o
+diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c
+index a4420d4df5033..7d589a705fc89 100644
+--- a/tools/perf/arch/arm64/util/arm-spe.c
++++ b/tools/perf/arch/arm64/util/arm-spe.c
+@@ -154,6 +154,12 @@ static int arm_spe_recording_options(struct auxtrace_record *itr,
+ arm_spe_set_timestamp(itr, arm_spe_evsel);
+ }
+
++ /*
++ * Set this only so that perf report knows that SPE generates memory info. It has no effect
++ * on the opening of the event or the SPE data produced.
++ */
++ evsel__set_sample_bit(arm_spe_evsel, DATA_SRC);
++
+ /* Add dummy event to keep tracking */
+ err = parse_events(evlist, "dummy:u", NULL);
+ if (err)
+diff --git a/tools/perf/arch/arm64/util/machine.c b/tools/perf/arch/arm64/util/machine.c
+deleted file mode 100644
+index 7e7714290a873..0000000000000
+--- a/tools/perf/arch/arm64/util/machine.c
++++ /dev/null
+@@ -1,28 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0
+-
+-#include <inttypes.h>
+-#include <stdio.h>
+-#include <string.h>
+-#include "debug.h"
+-#include "symbol.h"
+-
+-/* On arm64, kernel text segment starts at high memory address,
+- * for example 0xffff 0000 8xxx xxxx. Modules start at a low memory
+- * address, like 0xffff 0000 00ax xxxx. When only small amount of
+- * memory is used by modules, gap between end of module's text segment
+- * and start of kernel text segment may reach 2G.
+- * Therefore do not fill this gap and do not assign it to the kernel dso map.
+- */
+-
+-#define SYMBOL_LIMIT (1 << 12) /* 4K */
+-
+-void arch__symbols__fixup_end(struct symbol *p, struct symbol *c)
+-{
+- if ((strchr(p->name, '[') && strchr(c->name, '[') == NULL) ||
+- (strchr(p->name, '[') == NULL && strchr(c->name, '[')))
+- /* Limit range of last symbol in module and kernel */
+- p->end += SYMBOL_LIMIT;
+- else
+- p->end = c->start;
+- pr_debug4("%s sym:%s end:%#" PRIx64 "\n", __func__, p->name, p->end);
+-}
+diff --git a/tools/perf/arch/powerpc/util/Build b/tools/perf/arch/powerpc/util/Build
+index 8a79c4126e5b4..0115f31665684 100644
+--- a/tools/perf/arch/powerpc/util/Build
++++ b/tools/perf/arch/powerpc/util/Build
+@@ -1,5 +1,4 @@
+ perf-y += header.o
+-perf-y += machine.o
+ perf-y += kvm-stat.o
+ perf-y += perf_regs.o
+ perf-y += mem-events.o
+diff --git a/tools/perf/arch/powerpc/util/machine.c b/tools/perf/arch/powerpc/util/machine.c
+deleted file mode 100644
+index e652a1aa81322..0000000000000
+--- a/tools/perf/arch/powerpc/util/machine.c
++++ /dev/null
+@@ -1,25 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0
+-
+-#include <inttypes.h>
+-#include <stdio.h>
+-#include <string.h>
+-#include <internal/lib.h> // page_size
+-#include "debug.h"
+-#include "symbol.h"
+-
+-/* On powerpc kernel text segment start at memory addresses, 0xc000000000000000
+- * whereas the modules are located at very high memory addresses,
+- * for example 0xc00800000xxxxxxx. The gap between end of kernel text segment
+- * and beginning of first module's text segment is very high.
+- * Therefore do not fill this gap and do not assign it to the kernel dso map.
+- */
+-
+-void arch__symbols__fixup_end(struct symbol *p, struct symbol *c)
+-{
+- if (strchr(p->name, '[') == NULL && strchr(c->name, '['))
+- /* Limit the range of last kernel symbol */
+- p->end += page_size;
+- else
+- p->end = c->start;
+- pr_debug4("%s sym:%s end:%#" PRIx64 "\n", __func__, p->name, p->end);
+-}
+diff --git a/tools/perf/arch/s390/util/machine.c b/tools/perf/arch/s390/util/machine.c
+index 7644a4f6d4a40..98bc3f39d5f35 100644
+--- a/tools/perf/arch/s390/util/machine.c
++++ b/tools/perf/arch/s390/util/machine.c
+@@ -35,19 +35,3 @@ int arch__fix_module_text_start(u64 *start, u64 *size, const char *name)
+
+ return 0;
+ }
+-
+-/* On s390 kernel text segment start is located at very low memory addresses,
+- * for example 0x10000. Modules are located at very high memory addresses,
+- * for example 0x3ff xxxx xxxx. The gap between end of kernel text segment
+- * and beginning of first module's text segment is very big.
+- * Therefore do not fill this gap and do not assign it to the kernel dso map.
+- */
+-void arch__symbols__fixup_end(struct symbol *p, struct symbol *c)
+-{
+- if (strchr(p->name, '[') == NULL && strchr(c->name, '['))
+- /* Last kernel symbol mapped to end of page */
+- p->end = roundup(p->end, page_size);
+- else
+- p->end = c->start;
+- pr_debug4("%s sym:%s end:%#" PRIx64 "\n", __func__, p->name, p->end);
+-}
+diff --git a/tools/perf/arch/x86/util/perf_regs.c b/tools/perf/arch/x86/util/perf_regs.c
+index 207c56805c551..0ed177991ad05 100644
+--- a/tools/perf/arch/x86/util/perf_regs.c
++++ b/tools/perf/arch/x86/util/perf_regs.c
+@@ -9,6 +9,8 @@
+ #include "../../../util/perf_regs.h"
+ #include "../../../util/debug.h"
+ #include "../../../util/event.h"
++#include "../../../util/pmu.h"
++#include "../../../util/pmu-hybrid.h"
+
+ const struct sample_reg sample_reg_masks[] = {
+ SMPL_REG(AX, PERF_REG_X86_AX),
+@@ -284,12 +286,22 @@ uint64_t arch__intr_reg_mask(void)
+ .disabled = 1,
+ .exclude_kernel = 1,
+ };
++ struct perf_pmu *pmu;
+ int fd;
+ /*
+ * In an unnamed union, init it here to build on older gcc versions
+ */
+ attr.sample_period = 1;
+
++ if (perf_pmu__has_hybrid()) {
++ /*
++ * The same register set is supported among different hybrid PMUs.
++ * Only check the first available one.
++ */
++ pmu = list_first_entry(&perf_pmu__hybrid_pmus, typeof(*pmu), hybrid_list);
++ attr.config |= (__u64)pmu->type << PERF_PMU_TYPE_SHIFT;
++ }
++
+ event_attr_init(&attr);
+
+ fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
+diff --git a/tools/perf/bench/futex-lock-pi.c b/tools/perf/bench/futex-lock-pi.c
+index 5d1fe9c35807a..137890f78e17a 100644
+--- a/tools/perf/bench/futex-lock-pi.c
++++ b/tools/perf/bench/futex-lock-pi.c
+@@ -233,6 +233,7 @@ int bench_futex_lock_pi(int argc, const char **argv)
+ print_summary();
+
+ free(worker);
++ perf_cpu_map__put(cpu);
+ return ret;
+ err:
+ usage_with_options(bench_futex_lock_pi_usage, options);
+diff --git a/tools/perf/bench/futex-requeue.c b/tools/perf/bench/futex-requeue.c
+index 97fe31fd3a236..f7a5ffebb9408 100644
+--- a/tools/perf/bench/futex-requeue.c
++++ b/tools/perf/bench/futex-requeue.c
+@@ -294,6 +294,7 @@ int bench_futex_requeue(int argc, const char **argv)
+ print_summary();
+
+ free(worker);
++ perf_cpu_map__put(cpu);
+ return ret;
+ err:
+ usage_with_options(bench_futex_requeue_usage, options);
+diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c
+index e970e6b9ad535..0983f40b4b408 100644
+--- a/tools/perf/bench/futex-wake-parallel.c
++++ b/tools/perf/bench/futex-wake-parallel.c
+@@ -329,6 +329,7 @@ int bench_futex_wake_parallel(int argc, const char **argv)
+ print_summary();
+
+ free(blocked_worker);
++ perf_cpu_map__put(cpu);
+ return ret;
+ }
+ #endif /* HAVE_PTHREAD_BARRIER */
+diff --git a/tools/perf/bench/futex-wake.c b/tools/perf/bench/futex-wake.c
+index 77f058a477903..2226a475e782b 100644
+--- a/tools/perf/bench/futex-wake.c
++++ b/tools/perf/bench/futex-wake.c
+@@ -222,5 +222,6 @@ int bench_futex_wake(int argc, const char **argv)
+ print_summary();
+
+ free(worker);
++ perf_cpu_map__put(cpu);
+ return ret;
+ }
+diff --git a/tools/perf/bench/numa.c b/tools/perf/bench/numa.c
+index f2640179ada9e..c2c81567afa50 100644
+--- a/tools/perf/bench/numa.c
++++ b/tools/perf/bench/numa.c
+@@ -1672,7 +1672,7 @@ static int __bench_numa(const char *name)
+ "GB/sec,", "total-speed", "GB/sec total speed");
+
+ if (g->p.show_details >= 2) {
+- char tname[14 + 2 * 10 + 1];
++ char tname[14 + 2 * 11 + 1];
+ struct thread_data *td;
+ for (p = 0; p < g->p.nr_proc; p++) {
+ for (t = 0; t < g->p.nr_threads; t++) {
+diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
+index 6ad191e731fc9..50c2e6892b3e9 100644
+--- a/tools/perf/builtin-inject.c
++++ b/tools/perf/builtin-inject.c
+@@ -755,12 +755,16 @@ static int parse_vm_time_correlation(const struct option *opt, const char *str,
+ return inject->itrace_synth_opts.vm_tm_corr_args ? 0 : -ENOMEM;
+ }
+
++static int output_fd(struct perf_inject *inject)
++{
++ return inject->in_place_update ? -1 : perf_data__fd(&inject->output);
++}
++
+ static int __cmd_inject(struct perf_inject *inject)
+ {
+ int ret = -EINVAL;
+ struct perf_session *session = inject->session;
+- struct perf_data *data_out = &inject->output;
+- int fd = inject->in_place_update ? -1 : perf_data__fd(data_out);
++ int fd = output_fd(inject);
+ u64 output_data_offset;
+
+ signal(SIGINT, sig_handler);
+@@ -819,7 +823,7 @@ static int __cmd_inject(struct perf_inject *inject)
+ inject->tool.ordered_events = true;
+ inject->tool.ordering_requires_timestamps = true;
+ /* Allow space in the header for new attributes */
+- output_data_offset = 4096;
++ output_data_offset = roundup(8192 + session->header.data_offset, 4096);
+ if (inject->strip)
+ strip_init(inject);
+ }
+@@ -1006,7 +1010,7 @@ int cmd_inject(int argc, const char **argv)
+ }
+
+ inject.session = __perf_session__new(&data, repipe,
+- perf_data__fd(&inject.output),
++ output_fd(&inject),
+ &inject.tool);
+ if (IS_ERR(inject.session)) {
+ ret = PTR_ERR(inject.session);
+@@ -1069,7 +1073,8 @@ out_delete:
+ zstd_fini(&(inject.session->zstd_data));
+ perf_session__delete(inject.session);
+ out_close_output:
+- perf_data__close(&inject.output);
++ if (!inject.in_place_update)
++ perf_data__close(&inject.output);
+ free(inject.itrace_synth_opts.vm_tm_corr_args);
+ return ret;
+ }
+diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c
+index a0316ce910db6..6583ad9cc7deb 100644
+--- a/tools/perf/builtin-report.c
++++ b/tools/perf/builtin-report.c
+@@ -349,6 +349,7 @@ static int report__setup_sample_type(struct report *rep)
+ struct perf_session *session = rep->session;
+ u64 sample_type = evlist__combined_sample_type(session->evlist);
+ bool is_pipe = perf_data__is_pipe(session->data);
++ struct evsel *evsel;
+
+ if (session->itrace_synth_opts->callchain ||
+ session->itrace_synth_opts->add_callchain ||
+@@ -403,6 +404,19 @@ static int report__setup_sample_type(struct report *rep)
+ }
+
+ if (sort__mode == SORT_MODE__MEMORY) {
++ /*
++ * FIXUP: prior to kernel 5.18, Arm SPE missed to set
++ * PERF_SAMPLE_DATA_SRC bit in sample type. For backward
++ * compatibility, set the bit if it's an old perf data file.
++ */
++ evlist__for_each_entry(session->evlist, evsel) {
++ if (strstr(evsel->name, "arm_spe") &&
++ !(sample_type & PERF_SAMPLE_DATA_SRC)) {
++ evsel->core.attr.sample_type |= PERF_SAMPLE_DATA_SRC;
++ sample_type |= PERF_SAMPLE_DATA_SRC;
++ }
++ }
++
+ if (!is_pipe && !(sample_type & PERF_SAMPLE_DATA_SRC)) {
+ ui__error("Selected --mem-mode but no mem data. "
+ "Did you call perf record without -d?\n");
+@@ -619,14 +633,17 @@ static int report__browse_hists(struct report *rep)
+ int ret;
+ struct perf_session *session = rep->session;
+ struct evlist *evlist = session->evlist;
+- const char *help = perf_tip(system_path(TIPDIR));
++ char *help = NULL, *path = NULL;
+
+- if (help == NULL) {
++ path = system_path(TIPDIR);
++ if (perf_tip(&help, path) || help == NULL) {
+ /* fallback for people who don't install perf ;-) */
+- help = perf_tip(DOCDIR);
+- if (help == NULL)
+- help = "Cannot load tips.txt file, please install perf!";
++ free(path);
++ path = system_path(DOCDIR);
++ if (perf_tip(&help, path) || help == NULL)
++ help = strdup("Cannot load tips.txt file, please install perf!");
+ }
++ free(path);
+
+ switch (use_browser) {
+ case 1:
+@@ -651,7 +668,7 @@ static int report__browse_hists(struct report *rep)
+ ret = evlist__tty_browse_hists(evlist, rep, help);
+ break;
+ }
+-
++ free(help);
+ return ret;
+ }
+
+diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c
+index c32c2eb16d7df..cb3d81adf5ca8 100644
+--- a/tools/perf/builtin-script.c
++++ b/tools/perf/builtin-script.c
+@@ -455,7 +455,7 @@ static int evsel__check_attr(struct evsel *evsel, struct perf_session *session)
+ return -EINVAL;
+
+ if (PRINT_FIELD(DATA_SRC) &&
+- evsel__check_stype(evsel, PERF_SAMPLE_DATA_SRC, "DATA_SRC", PERF_OUTPUT_DATA_SRC))
++ evsel__do_check_stype(evsel, PERF_SAMPLE_DATA_SRC, "DATA_SRC", PERF_OUTPUT_DATA_SRC, allow_user_set))
+ return -EINVAL;
+
+ if (PRINT_FIELD(WEIGHT) &&
+@@ -2463,7 +2463,7 @@ static int process_switch_event(struct perf_tool *tool,
+ if (perf_event__process_switch(tool, event, sample, machine) < 0)
+ return -1;
+
+- if (scripting_ops && scripting_ops->process_switch)
++ if (scripting_ops && scripting_ops->process_switch && !filter_cpu(sample))
+ scripting_ops->process_switch(event, sample, machine);
+
+ if (!script->show_switch_events)
+diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
+index f0ecfda34eceb..1a194edb54520 100644
+--- a/tools/perf/builtin-stat.c
++++ b/tools/perf/builtin-stat.c
+@@ -956,10 +956,10 @@ try_again_reset:
+ * Enable counters and exec the command:
+ */
+ if (forks) {
+- evlist__start_workload(evsel_list);
+ err = enable_counters();
+ if (err)
+ return -1;
++ evlist__start_workload(evsel_list);
+
+ t0 = rdclock();
+ clock_gettime(CLOCK_MONOTONIC, &ref_time);
+diff --git a/tools/perf/perf.c b/tools/perf/perf.c
+index 2f6b67189b426..6aae7b6c376b4 100644
+--- a/tools/perf/perf.c
++++ b/tools/perf/perf.c
+@@ -434,7 +434,7 @@ void pthread__unblock_sigwinch(void)
+ static int libperf_print(enum libperf_print_level level,
+ const char *fmt, va_list ap)
+ {
+- return eprintf(level, verbose, fmt, ap);
++ return veprintf(level, verbose, fmt, ap);
+ }
+
+ int main(int argc, const char **argv)
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/cache.json b/tools/perf/pmu-events/arch/x86/skylakex/cache.json
+index 9ff67206ade4e..821d2f2a8f251 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/cache.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/cache.json
+@@ -314,6 +314,19 @@
+ "SampleAfterValue": "2000003",
+ "UMask": "0x82"
+ },
++ {
++ "BriefDescription": "All retired memory instructions.",
++ "Counter": "0,1,2,3",
++ "CounterHTOff": "0,1,2,3",
++ "Data_LA": "1",
++ "EventCode": "0xD0",
++ "EventName": "MEM_INST_RETIRED.ANY",
++ "L1_Hit_Indication": "1",
++ "PEBS": "1",
++ "PublicDescription": "Counts all retired memory instructions - loads and stores.",
++ "SampleAfterValue": "2000003",
++ "UMask": "0x83"
++ },
+ {
+ "BriefDescription": "Retired load instructions with locked access.",
+ "Counter": "0,1,2,3",
+@@ -358,6 +371,7 @@
+ "EventCode": "0xD0",
+ "EventName": "MEM_INST_RETIRED.STLB_MISS_LOADS",
+ "PEBS": "1",
++ "PublicDescription": "Number of retired load instructions that (start a) miss in the 2nd-level TLB (STLB).",
+ "SampleAfterValue": "100003",
+ "UMask": "0x11"
+ },
+@@ -370,6 +384,7 @@
+ "EventName": "MEM_INST_RETIRED.STLB_MISS_STORES",
+ "L1_Hit_Indication": "1",
+ "PEBS": "1",
++ "PublicDescription": "Number of retired store instructions that (start a) miss in the 2nd-level TLB (STLB).",
+ "SampleAfterValue": "100003",
+ "UMask": "0x12"
+ },
+@@ -733,7 +748,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010491",
++ "MSRValue": "0x10491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -772,7 +787,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0491",
++ "MSRValue": "0x4003C0491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -785,7 +800,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0491",
++ "MSRValue": "0x1003C0491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -798,7 +813,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0491",
++ "MSRValue": "0x8003C0491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -811,7 +826,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010490",
++ "MSRValue": "0x10490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -850,7 +865,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0490",
++ "MSRValue": "0x4003C0490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -863,7 +878,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0490",
++ "MSRValue": "0x1003C0490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -876,7 +891,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0490",
++ "MSRValue": "0x8003C0490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -889,7 +904,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010120",
++ "MSRValue": "0x10120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -928,7 +943,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0120",
++ "MSRValue": "0x4003C0120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -941,7 +956,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0120",
++ "MSRValue": "0x1003C0120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -954,7 +969,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0120",
++ "MSRValue": "0x8003C0120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -967,7 +982,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010122",
++ "MSRValue": "0x10122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1006,7 +1021,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0122",
++ "MSRValue": "0x4003C0122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1019,7 +1034,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0122",
++ "MSRValue": "0x1003C0122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1032,7 +1047,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0122",
++ "MSRValue": "0x8003C0122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1045,7 +1060,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010004",
++ "MSRValue": "0x10004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1084,7 +1099,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0004",
++ "MSRValue": "0x4003C0004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1097,7 +1112,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0004",
++ "MSRValue": "0x1003C0004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1110,7 +1125,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0004",
++ "MSRValue": "0x8003C0004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1123,7 +1138,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010001",
++ "MSRValue": "0x10001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1162,7 +1177,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0001",
++ "MSRValue": "0x4003C0001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1175,7 +1190,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0001",
++ "MSRValue": "0x1003C0001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1188,7 +1203,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0001",
++ "MSRValue": "0x8003C0001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1201,7 +1216,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010002",
++ "MSRValue": "0x10002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1240,7 +1255,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0002",
++ "MSRValue": "0x4003C0002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1253,7 +1268,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0002",
++ "MSRValue": "0x1003C0002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1266,7 +1281,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0002",
++ "MSRValue": "0x8003C0002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1279,7 +1294,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010400",
++ "MSRValue": "0x10400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1318,7 +1333,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0400",
++ "MSRValue": "0x4003C0400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1331,7 +1346,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0400",
++ "MSRValue": "0x1003C0400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1344,7 +1359,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0400",
++ "MSRValue": "0x8003C0400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1357,7 +1372,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010010",
++ "MSRValue": "0x10010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1396,7 +1411,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0010",
++ "MSRValue": "0x4003C0010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1409,7 +1424,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0010",
++ "MSRValue": "0x1003C0010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1422,7 +1437,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0010",
++ "MSRValue": "0x8003C0010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1435,7 +1450,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010020",
++ "MSRValue": "0x10020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1474,7 +1489,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0020",
++ "MSRValue": "0x4003C0020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1487,7 +1502,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0020",
++ "MSRValue": "0x1003C0020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1500,7 +1515,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0020",
++ "MSRValue": "0x8003C0020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1513,7 +1528,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010080",
++ "MSRValue": "0x10080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1552,7 +1567,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0080",
++ "MSRValue": "0x4003C0080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1565,7 +1580,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0080",
++ "MSRValue": "0x1003C0080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1578,7 +1593,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0080",
++ "MSRValue": "0x8003C0080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1591,7 +1606,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0000010100",
++ "MSRValue": "0x10100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1630,7 +1645,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_HIT.HIT_OTHER_CORE_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x04003C0100",
++ "MSRValue": "0x4003C0100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1643,7 +1658,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_HIT.NO_SNOOP_NEEDED",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x01003C0100",
++ "MSRValue": "0x1003C0100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1656,7 +1671,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x08003C0100",
++ "MSRValue": "0x8003C0100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/floating-point.json b/tools/perf/pmu-events/arch/x86/skylakex/floating-point.json
+index 503737ed3a83c..9e873ab224502 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/floating-point.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/floating-point.json
+@@ -1,73 +1,81 @@
+ [
+ {
+- "BriefDescription": "Number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 2 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT14 RCP14 DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD 128-bit packed computational double precision floating-point instructions retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE",
++ "PublicDescription": "Counts once for most SIMD 128-bit packed computational double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 2 computation operations, one for each element. Applies to packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x4"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD 128-bit packed computational single precision floating-point instruction retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE",
++ "PublicDescription": "Counts once for most SIMD 128-bit packed computational single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x8"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD 256-bit packed double computational precision floating-point instructions retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE",
++ "PublicDescription": "Counts once for most SIMD 256-bit packed double computational precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x10"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD 256-bit packed single computational precision floating-point instructions retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE",
++ "PublicDescription": "Counts once for most SIMD 256-bit packed single computational precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x20"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 8 calculations per element.",
++ "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE",
++ "PublicDescription": "Number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x40"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 16 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 16 calculations per element.",
++ "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 16 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE",
++ "PublicDescription": "Number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 16 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x80"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD scalar computational double precision floating-point instructions retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.SCALAR_DOUBLE",
++ "PublicDescription": "Counts once for most SIMD scalar computational double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SIMD scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x1"
+ },
+ {
+- "BriefDescription": "Number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
++ "BriefDescription": "Counts once for most SIMD scalar computational single precision floating-point instructions retired. Counts twice for DPP and FM(N)ADD/SUB instructions retired.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3,4,5,6,7",
+ "EventCode": "0xC7",
+ "EventName": "FP_ARITH_INST_RETIRED.SCALAR_SINGLE",
++ "PublicDescription": "Counts once for most SIMD scalar computational single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SIMD scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+ "SampleAfterValue": "2000003",
+ "UMask": "0x2"
+ },
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/frontend.json b/tools/perf/pmu-events/arch/x86/skylakex/frontend.json
+index 078706a500919..ecce4273ae52c 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/frontend.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/frontend.json
+@@ -30,7 +30,21 @@
+ "UMask": "0x2"
+ },
+ {
+- "BriefDescription": "Retired Instructions who experienced decode stream buffer (DSB - the decoded instruction-cache) miss.",
++ "BriefDescription": "Retired Instructions who experienced DSB miss.",
++ "Counter": "0,1,2,3",
++ "CounterHTOff": "0,1,2,3",
++ "EventCode": "0xC6",
++ "EventName": "FRONTEND_RETIRED.ANY_DSB_MISS",
++ "MSRIndex": "0x3F7",
++ "MSRValue": "0x1",
++ "PEBS": "1",
++ "PublicDescription": "Counts retired Instructions that experienced DSB (Decode stream buffer i.e. the decoded instruction-cache) miss.",
++ "SampleAfterValue": "100007",
++ "TakenAlone": "1",
++ "UMask": "0x1"
++ },
++ {
++ "BriefDescription": "Retired Instructions who experienced a critical DSB miss.",
+ "Counter": "0,1,2,3",
+ "CounterHTOff": "0,1,2,3",
+ "EventCode": "0xC6",
+@@ -38,7 +52,7 @@
+ "MSRIndex": "0x3F7",
+ "MSRValue": "0x11",
+ "PEBS": "1",
+- "PublicDescription": "Counts retired Instructions that experienced DSB (Decode stream buffer i.e. the decoded instruction-cache) miss.",
++ "PublicDescription": "Number of retired Instructions that experienced a critical DSB (Decode stream buffer i.e. the decoded instruction-cache) miss. Critical means stalls were exposed to the back-end as a result of the DSB miss.",
+ "SampleAfterValue": "100007",
+ "TakenAlone": "1",
+ "UMask": "0x1"
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/memory.json b/tools/perf/pmu-events/arch/x86/skylakex/memory.json
+index 6f29b02fa320c..60c286b4fe54c 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/memory.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/memory.json
+@@ -299,7 +299,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00491",
++ "MSRValue": "0x83FC00491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -312,7 +312,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00491",
++ "MSRValue": "0x63FC00491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -325,7 +325,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000491",
++ "MSRValue": "0x604000491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -338,7 +338,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_DATA_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800491",
++ "MSRValue": "0x63B800491",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -377,7 +377,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00490",
++ "MSRValue": "0x83FC00490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -390,7 +390,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00490",
++ "MSRValue": "0x63FC00490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -403,7 +403,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000490",
++ "MSRValue": "0x604000490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -416,7 +416,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_DATA_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800490",
++ "MSRValue": "0x63B800490",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -455,7 +455,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00120",
++ "MSRValue": "0x83FC00120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -468,7 +468,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00120",
++ "MSRValue": "0x63FC00120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -481,7 +481,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000120",
++ "MSRValue": "0x604000120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -494,7 +494,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_PF_RFO.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800120",
++ "MSRValue": "0x63B800120",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -533,7 +533,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00122",
++ "MSRValue": "0x83FC00122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -546,7 +546,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00122",
++ "MSRValue": "0x63FC00122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -559,7 +559,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000122",
++ "MSRValue": "0x604000122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -572,7 +572,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.ALL_RFO.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800122",
++ "MSRValue": "0x63B800122",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -611,7 +611,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00004",
++ "MSRValue": "0x83FC00004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -624,7 +624,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00004",
++ "MSRValue": "0x63FC00004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -637,7 +637,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000004",
++ "MSRValue": "0x604000004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -650,7 +650,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_CODE_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800004",
++ "MSRValue": "0x63B800004",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -689,7 +689,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00001",
++ "MSRValue": "0x83FC00001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -702,7 +702,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00001",
++ "MSRValue": "0x63FC00001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -715,7 +715,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000001",
++ "MSRValue": "0x604000001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -728,7 +728,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800001",
++ "MSRValue": "0x63B800001",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -767,7 +767,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00002",
++ "MSRValue": "0x83FC00002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -780,7 +780,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00002",
++ "MSRValue": "0x63FC00002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -793,7 +793,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000002",
++ "MSRValue": "0x604000002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -806,7 +806,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800002",
++ "MSRValue": "0x63B800002",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -845,7 +845,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00400",
++ "MSRValue": "0x83FC00400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -858,7 +858,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00400",
++ "MSRValue": "0x63FC00400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -871,7 +871,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000400",
++ "MSRValue": "0x604000400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -884,7 +884,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L1D_AND_SW.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800400",
++ "MSRValue": "0x63B800400",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -923,7 +923,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00010",
++ "MSRValue": "0x83FC00010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -936,7 +936,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00010",
++ "MSRValue": "0x63FC00010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -949,7 +949,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000010",
++ "MSRValue": "0x604000010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -962,7 +962,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_DATA_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800010",
++ "MSRValue": "0x63B800010",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1001,7 +1001,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00020",
++ "MSRValue": "0x83FC00020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1014,7 +1014,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00020",
++ "MSRValue": "0x63FC00020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1027,7 +1027,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000020",
++ "MSRValue": "0x604000020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1040,7 +1040,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800020",
++ "MSRValue": "0x63B800020",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1079,7 +1079,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00080",
++ "MSRValue": "0x83FC00080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1092,7 +1092,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00080",
++ "MSRValue": "0x63FC00080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1105,7 +1105,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000080",
++ "MSRValue": "0x604000080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1118,7 +1118,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_DATA_RD.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800080",
++ "MSRValue": "0x63B800080",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1157,7 +1157,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_MISS.REMOTE_HIT_FORWARD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x083FC00100",
++ "MSRValue": "0x83FC00100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1170,7 +1170,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_MISS.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063FC00100",
++ "MSRValue": "0x63FC00100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1183,7 +1183,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_MISS_LOCAL_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x0604000100",
++ "MSRValue": "0x604000100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+@@ -1196,7 +1196,7 @@
+ "EventCode": "0xB7, 0xBB",
+ "EventName": "OFFCORE_RESPONSE.PF_L3_RFO.L3_MISS_REMOTE_DRAM.SNOOP_MISS_OR_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+- "MSRValue": "0x063B800100",
++ "MSRValue": "0x63B800100",
+ "Offcore": "1",
+ "PublicDescription": "Offcore response can be programmed only with a specific pair of event select and counter MSR, and with specific event codes and predefine mask bit value in a dedicated MSR to specify attributes of the offcore transaction.",
+ "SampleAfterValue": "100003",
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/pipeline.json b/tools/perf/pmu-events/arch/x86/skylakex/pipeline.json
+index ca57481206660..12eabae3e2242 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/pipeline.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/pipeline.json
+@@ -435,6 +435,17 @@
+ "PublicDescription": "Counts the number of instructions (EOMs) retired. Counting covers macro-fused instructions individually (that is, increments by two).",
+ "SampleAfterValue": "2000003"
+ },
++ {
++ "BriefDescription": "Number of all retired NOP instructions.",
++ "Counter": "0,1,2,3",
++ "CounterHTOff": "0,1,2,3,4,5,6,7",
++ "Errata": "SKL091, SKL044",
++ "EventCode": "0xC0",
++ "EventName": "INST_RETIRED.NOP",
++ "PEBS": "1",
++ "SampleAfterValue": "2000003",
++ "UMask": "0x2"
++ },
+ {
+ "BriefDescription": "Precise instruction retired event with HW to reduce effect of PEBS shadow in IP distribution",
+ "Counter": "1",
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json
+index 863c9e103969e..b016f7d1ff3de 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json
+@@ -1,26 +1,167 @@
+ [
++ {
++ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
++ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
++ "MetricGroup": "TopdownL1",
++ "MetricName": "Frontend_Bound",
++ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
++ "MetricGroup": "TopdownL1_SMT",
++ "MetricName": "Frontend_Bound_SMT",
++ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
++ "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
++ "MetricGroup": "TopdownL1",
++ "MetricName": "Bad_Speculation",
++ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
++ "MetricGroup": "TopdownL1_SMT",
++ "MetricName": "Bad_Speculation_SMT",
++ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
++ "MetricConstraint": "NO_NMI_WATCHDOG",
++ "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
++ "MetricGroup": "TopdownL1",
++ "MetricName": "Backend_Bound",
++ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
++ "MetricGroup": "TopdownL1_SMT",
++ "MetricName": "Backend_Bound_SMT",
++ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
++ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
++ "MetricGroup": "TopdownL1",
++ "MetricName": "Retiring",
++ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
++ },
++ {
++ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
++ "MetricGroup": "TopdownL1_SMT",
++ "MetricName": "Retiring_SMT",
++ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
++ "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )",
++ "MetricGroup": "Bad;BadSpec;BrMispredicts",
++ "MetricName": "Mispredictions"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
++ "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
++ "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT",
++ "MetricName": "Mispredictions_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
++ "MetricGroup": "Mem;MemoryBW;Offcore",
++ "MetricName": "Memory_Bandwidth"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
++ "MetricGroup": "Mem;MemoryBW;Offcore_SMT",
++ "MetricName": "Memory_Bandwidth_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )",
++ "MetricGroup": "Mem;MemoryLat;Offcore",
++ "MetricName": "Memory_Latency"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )",
++ "MetricGroup": "Mem;MemoryLat;Offcore_SMT",
++ "MetricName": "Memory_Latency_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
++ "MetricGroup": "Mem;MemoryTLB",
++ "MetricName": "Memory_Data_TLBs"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
++ "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
++ "MetricGroup": "Mem;MemoryTLB;_SMT",
++ "MetricName": "Memory_Data_TLBs_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
++ "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))",
++ "MetricGroup": "Ret",
++ "MetricName": "Branching_Overhead"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
++ "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
++ "MetricGroup": "Ret_SMT",
++ "MetricName": "Branching_Overhead_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
++ "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))",
++ "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
++ "MetricName": "Big_Code"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
++ "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
++ "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT",
++ "MetricName": "Big_Code_SMT"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
++ "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))",
++ "MetricGroup": "Fed;FetchBW;Frontend",
++ "MetricName": "Instruction_Fetch_BW"
++ },
++ {
++ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
++ "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))",
++ "MetricGroup": "Fed;FetchBW;Frontend_SMT",
++ "MetricName": "Instruction_Fetch_BW_SMT"
++ },
+ {
+ "BriefDescription": "Instructions Per Cycle (per Logical Processor)",
+ "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+- "MetricGroup": "Summary",
++ "MetricGroup": "Ret;Summary",
+ "MetricName": "IPC"
+ },
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / INST_RETIRED.ANY",
+- "MetricGroup": "Pipeline;Retire",
++ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+- "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.NEAR_TAKEN",
+- "MetricGroup": "Branches;FetchBW;PGO",
+- "MetricName": "IpTB"
++ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / BR_INST_RETIRED.NEAR_TAKEN",
++ "MetricGroup": "Branches;Fed;FetchBW",
++ "MetricName": "UpTB"
+ },
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
+- "MetricGroup": "Pipeline",
++ "MetricGroup": "Pipeline;Mem",
+ "MetricName": "CPI"
+ },
+ {
+@@ -30,39 +171,84 @@
+ "MetricName": "CLKS"
+ },
+ {
+- "BriefDescription": "Instructions Per Cycle (per physical core)",
++ "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
++ "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
++ "MetricGroup": "TmaL1",
++ "MetricName": "SLOTS"
++ },
++ {
++ "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
++ "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
++ "MetricGroup": "TmaL1_SMT",
++ "MetricName": "SLOTS_SMT"
++ },
++ {
++ "BriefDescription": "The ratio of Executed- by Issued-Uops",
++ "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
++ "MetricGroup": "Cor;Pipeline",
++ "MetricName": "Execute_per_Issue",
++ "PublicDescription": "The ratio of Executed- by Issued-Uops. Ratio > 1 suggests high rate of uop micro-fusions. Ratio < 1 suggest high rate of \"execute\" at rename stage."
++ },
++ {
++ "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
+ "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+- "MetricGroup": "SMT;TmaL1",
++ "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricName": "CoreIPC"
+ },
+ {
+- "BriefDescription": "Instructions Per Cycle (per physical core)",
++ "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
+ "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+- "MetricGroup": "SMT;TmaL1",
++ "MetricGroup": "Ret;SMT;TmaL1_SMT",
+ "MetricName": "CoreIPC_SMT"
+ },
+ {
+ "BriefDescription": "Floating Point Operations Per Cycle",
+ "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
+- "MetricGroup": "Flops",
++ "MetricGroup": "Ret;Flops",
+ "MetricName": "FLOPc"
+ },
+ {
+ "BriefDescription": "Floating Point Operations Per Cycle",
+ "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+- "MetricGroup": "Flops_SMT",
++ "MetricGroup": "Ret;Flops_SMT",
+ "MetricName": "FLOPc_SMT"
+ },
++ {
++ "BriefDescription": "Actual per-core usage of the Floating Point execution units (regardless of the vector width)",
++ "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
++ "MetricGroup": "Cor;Flops;HPC",
++ "MetricName": "FP_Arith_Utilization",
++ "PublicDescription": "Actual per-core usage of the Floating Point execution units (regardless of the vector width). Values > 1 are possible due to Fused-Multiply Add (FMA) counting."
++ },
++ {
++ "BriefDescription": "Actual per-core usage of the Floating Point execution units (regardless of the vector width). SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
++ "MetricGroup": "Cor;Flops;HPC_SMT",
++ "MetricName": "FP_Arith_Utilization_SMT",
++ "PublicDescription": "Actual per-core usage of the Floating Point execution units (regardless of the vector width). Values > 1 are possible due to Fused-Multiply Add (FMA) counting. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
+ {
+ "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is at least 1 uop executed)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+- "MetricGroup": "Pipeline;PortsUtil",
++ "MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
+ "MetricName": "ILP"
+ },
++ {
++ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
++ "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;BrMispredicts",
++ "MetricName": "Branch_Misprediction_Cost"
++ },
++ {
++ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
++ "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;BrMispredicts_SMT",
++ "MetricName": "Branch_Misprediction_Cost_SMT"
++ },
+ {
+ "BriefDescription": "Number of Instructions per non-speculative Branch Misprediction (JEClear)",
+ "MetricExpr": "INST_RETIRED.ANY / BR_MISP_RETIRED.ALL_BRANCHES",
+- "MetricGroup": "BrMispredicts",
++ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "IpMispredict"
+ },
+ {
+@@ -86,122 +272,249 @@
+ {
+ "BriefDescription": "Instructions per Branch (lower number means higher occurrence rate)",
+ "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.ALL_BRANCHES",
+- "MetricGroup": "Branches;InsType",
++ "MetricGroup": "Branches;Fed;InsType",
+ "MetricName": "IpBranch"
+ },
+ {
+ "BriefDescription": "Instructions per (near) call (lower number means higher occurrence rate)",
+ "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.NEAR_CALL",
+- "MetricGroup": "Branches",
++ "MetricGroup": "Branches;Fed;PGO",
+ "MetricName": "IpCall"
+ },
++ {
++ "BriefDescription": "Instruction per taken branch",
++ "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.NEAR_TAKEN",
++ "MetricGroup": "Branches;Fed;FetchBW;Frontend;PGO",
++ "MetricName": "IpTB"
++ },
+ {
+ "BriefDescription": "Branch instructions per taken branch. ",
+ "MetricExpr": "BR_INST_RETIRED.ALL_BRANCHES / BR_INST_RETIRED.NEAR_TAKEN",
+- "MetricGroup": "Branches;PGO",
++ "MetricGroup": "Branches;Fed;PGO",
+ "MetricName": "BpTkBranch"
+ },
+ {
+ "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
+ "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+- "MetricGroup": "Flops;FpArith;InsType",
++ "MetricGroup": "Flops;InsType",
+ "MetricName": "IpFLOP"
+ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
++ "MetricGroup": "Flops;InsType",
++ "MetricName": "IpArith",
++ "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
++ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic Scalar Single-Precision instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / FP_ARITH_INST_RETIRED.SCALAR_SINGLE",
++ "MetricGroup": "Flops;FpScalar;InsType",
++ "MetricName": "IpArith_Scalar_SP",
++ "PublicDescription": "Instructions per FP Arithmetic Scalar Single-Precision instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
++ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic Scalar Double-Precision instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / FP_ARITH_INST_RETIRED.SCALAR_DOUBLE",
++ "MetricGroup": "Flops;FpScalar;InsType",
++ "MetricName": "IpArith_Scalar_DP",
++ "PublicDescription": "Instructions per FP Arithmetic Scalar Double-Precision instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
++ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
++ "MetricGroup": "Flops;FpVector;InsType",
++ "MetricName": "IpArith_AVX128",
++ "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
++ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
++ "MetricGroup": "Flops;FpVector;InsType",
++ "MetricName": "IpArith_AVX256",
++ "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
++ },
++ {
++ "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
++ "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
++ "MetricGroup": "Flops;FpVector;InsType",
++ "MetricName": "IpArith_AVX512",
++ "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
++ },
+ {
+ "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "MetricExpr": "INST_RETIRED.ANY",
+ "MetricGroup": "Summary;TmaL1",
+ "MetricName": "Instructions"
+ },
++ {
++ "BriefDescription": "Average number of Uops issued by front-end when it issued something",
++ "MetricExpr": "UOPS_ISSUED.ANY / cpu@UOPS_ISSUED.ANY\\,cmask\\=1@",
++ "MetricGroup": "Fed;FetchBW",
++ "MetricName": "Fetch_UpC"
++ },
+ {
+ "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
+ "MetricExpr": "IDQ.DSB_UOPS / (IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS)",
+- "MetricGroup": "DSB;FetchBW",
++ "MetricGroup": "DSB;Fed;FetchBW",
+ "MetricName": "DSB_Coverage"
+ },
+ {
+- "BriefDescription": "Actual Average Latency for L1 data-cache miss demand loads (in core cycles)",
++ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset/see of/the Instruction_Fetch_BW Bottleneck.",
++ "MetricExpr": "(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))",
++ "MetricGroup": "DSBmiss;Fed",
++ "MetricName": "DSB_Misses_Cost"
++ },
++ {
++ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset/see of/the Instruction_Fetch_BW Bottleneck.",
++ "MetricExpr": "(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))",
++ "MetricGroup": "DSBmiss;Fed_SMT",
++ "MetricName": "DSB_Misses_Cost_SMT"
++ },
++ {
++ "BriefDescription": "Number of Instructions per non-speculative DSB miss",
++ "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
++ "MetricGroup": "DSBmiss;Fed",
++ "MetricName": "IpDSB_Miss_Ret"
++ },
++ {
++ "BriefDescription": "Fraction of branches that are non-taken conditionals",
++ "MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;Branches;CodeGen;PGO",
++ "MetricName": "Cond_NT"
++ },
++ {
++ "BriefDescription": "Fraction of branches that are taken conditionals",
++ "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;Branches;CodeGen;PGO",
++ "MetricName": "Cond_TK"
++ },
++ {
++ "BriefDescription": "Fraction of branches that are CALL or RET",
++ "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;Branches",
++ "MetricName": "CallRet"
++ },
++ {
++ "BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps",
++ "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
++ "MetricGroup": "Bad;Branches",
++ "MetricName": "Jump"
++ },
++ {
++ "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load instructions (in core cycles)",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+- "MetricGroup": "MemoryBound;MemoryLat",
+- "MetricName": "Load_Miss_Real_Latency"
++ "MetricGroup": "Mem;MemoryBound;MemoryLat",
++ "MetricName": "Load_Miss_Real_Latency",
++ "PublicDescription": "Actual Average Latency for L1 data-cache miss demand load instructions (in core cycles). Latency may be overestimated for multi-load instructions - e.g. repeat strings."
+ },
+ {
+ "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
+- "MetricGroup": "MemoryBound;MemoryBW",
++ "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricName": "MLP"
+ },
+- {
+- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
+- "MetricConstraint": "NO_NMI_WATCHDOG",
+- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CORE_CLKS )",
+- "MetricGroup": "MemoryTLB",
+- "MetricName": "Page_Walks_Utilization"
+- },
+ {
+ "BriefDescription": "Average data fill bandwidth to the L1 data cache [GB / sec]",
+ "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
+- "MetricGroup": "MemoryBW",
++ "MetricGroup": "Mem;MemoryBW",
+ "MetricName": "L1D_Cache_Fill_BW"
+ },
+ {
+ "BriefDescription": "Average data fill bandwidth to the L2 cache [GB / sec]",
+ "MetricExpr": "64 * L2_LINES_IN.ALL / 1000000000 / duration_time",
+- "MetricGroup": "MemoryBW",
++ "MetricGroup": "Mem;MemoryBW",
+ "MetricName": "L2_Cache_Fill_BW"
+ },
+ {
+ "BriefDescription": "Average per-core data fill bandwidth to the L3 cache [GB / sec]",
+ "MetricExpr": "64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time",
+- "MetricGroup": "MemoryBW",
++ "MetricGroup": "Mem;MemoryBW",
+ "MetricName": "L3_Cache_Fill_BW"
+ },
+ {
+ "BriefDescription": "Average per-core data access bandwidth to the L3 cache [GB / sec]",
+ "MetricExpr": "64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time",
+- "MetricGroup": "MemoryBW;Offcore",
++ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "L3_Cache_Access_BW"
+ },
+ {
+ "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
+ "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
+- "MetricGroup": "CacheMisses",
++ "MetricGroup": "Mem;CacheMisses",
+ "MetricName": "L1MPKI"
+ },
++ {
++ "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
++ "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
++ "MetricGroup": "Mem;CacheMisses",
++ "MetricName": "L1MPKI_Load"
++ },
+ {
+ "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
+ "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
+- "MetricGroup": "CacheMisses",
++ "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricName": "L2MPKI"
+ },
+ {
+ "BriefDescription": "L2 cache misses per kilo instruction for all request types (including speculative)",
+ "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
+- "MetricGroup": "CacheMisses;Offcore",
++ "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricName": "L2MPKI_All"
+ },
++ {
++ "BriefDescription": "L2 cache misses per kilo instruction for all demand loads (including speculative)",
++ "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
++ "MetricGroup": "Mem;CacheMisses",
++ "MetricName": "L2MPKI_Load"
++ },
+ {
+ "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
+ "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
+- "MetricGroup": "CacheMisses",
++ "MetricGroup": "Mem;CacheMisses",
+ "MetricName": "L2HPKI_All"
+ },
++ {
++ "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
++ "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
++ "MetricGroup": "Mem;CacheMisses",
++ "MetricName": "L2HPKI_Load"
++ },
+ {
+ "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
+ "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
+- "MetricGroup": "CacheMisses",
++ "MetricGroup": "Mem;CacheMisses",
+ "MetricName": "L3MPKI"
+ },
++ {
++ "BriefDescription": "Fill Buffer (FB) true hits per kilo instructions for retired demand loads",
++ "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
++ "MetricGroup": "Mem;CacheMisses",
++ "MetricName": "FB_HPKI"
++ },
++ {
++ "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
++ "MetricConstraint": "NO_NMI_WATCHDOG",
++ "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
++ "MetricGroup": "Mem;MemoryTLB",
++ "MetricName": "Page_Walks_Utilization"
++ },
++ {
++ "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
++ "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
++ "MetricGroup": "Mem;MemoryTLB_SMT",
++ "MetricName": "Page_Walks_Utilization_SMT"
++ },
+ {
+ "BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)",
+ "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY",
+- "MetricGroup": "L2Evicts;Server",
++ "MetricGroup": "L2Evicts;Mem;Server",
+ "MetricName": "L2_Evictions_Silent_PKI"
+ },
+ {
+ "BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction",
+ "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY",
+- "MetricGroup": "L2Evicts;Server",
++ "MetricGroup": "L2Evicts;Mem;Server",
+ "MetricName": "L2_Evictions_NonSilent_PKI"
+ },
+ {
+@@ -219,7 +532,7 @@
+ {
+ "BriefDescription": "Giga Floating Point Operations Per Second",
+ "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+- "MetricGroup": "Flops;HPC",
++ "MetricGroup": "Cor;Flops;HPC",
+ "MetricName": "GFLOPs"
+ },
+ {
+@@ -228,6 +541,48 @@
+ "MetricGroup": "Power",
+ "MetricName": "Turbo_Utilization"
+ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
++ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
++ "MetricGroup": "Power",
++ "MetricName": "Power_License0_Utilization",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
++ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
++ "MetricGroup": "Power_SMT",
++ "MetricName": "Power_License0_Utilization_SMT",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
++ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
++ "MetricGroup": "Power",
++ "MetricName": "Power_License1_Utilization",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
++ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
++ "MetricGroup": "Power_SMT",
++ "MetricName": "Power_License1_Utilization_SMT",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
++ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
++ "MetricGroup": "Power",
++ "MetricName": "Power_License2_Utilization",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
++ },
++ {
++ "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). SMT version; use when SMT is enabled and measuring per logical CPU.",
++ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
++ "MetricGroup": "Power_SMT",
++ "MetricName": "Power_License2_Utilization_SMT",
++ "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
++ },
+ {
+ "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+@@ -240,34 +595,46 @@
+ "MetricGroup": "OS",
+ "MetricName": "Kernel_Utilization"
+ },
++ {
++ "BriefDescription": "Cycles Per Instruction for the Operating System (OS) Kernel mode",
++ "MetricExpr": "CPU_CLK_UNHALTED.THREAD_P:k / INST_RETIRED.ANY_P:k",
++ "MetricGroup": "OS",
++ "MetricName": "Kernel_CPI"
++ },
+ {
+ "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
+ "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+- "MetricGroup": "HPC;MemoryBW;SoC",
++ "MetricGroup": "HPC;Mem;MemoryBW;SoC",
+ "MetricName": "DRAM_BW_Use"
+ },
+ {
+ "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
+ "MetricExpr": "1000000000 * ( cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@ ) / ( cha_0@event\\=0x0@ / duration_time )",
+- "MetricGroup": "MemoryLat;SoC",
++ "MetricGroup": "Mem;MemoryLat;SoC",
+ "MetricName": "MEM_Read_Latency"
+ },
+ {
+ "BriefDescription": "Average number of parallel data read requests to external memory. Accounts for demand loads and L1/L2 prefetches",
+ "MetricExpr": "cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433\\,thresh\\=1@",
+- "MetricGroup": "MemoryBW;SoC",
++ "MetricGroup": "Mem;MemoryBW;SoC",
+ "MetricName": "MEM_Parallel_Reads"
+ },
++ {
++ "BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
++ "MetricExpr": "1000000000 * ( UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS ) / imc_0@event\\=0x0@",
++ "MetricGroup": "Mem;MemoryLat;SoC;Server",
++ "MetricName": "MEM_DRAM_Read_Latency"
++ },
+ {
+ "BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]",
+ "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000000 / duration_time",
+- "MetricGroup": "IoBW;SoC;Server",
++ "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricName": "IO_Write_BW"
+ },
+ {
+ "BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]",
+ "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3 ) * 4 / 1000000000 / duration_time",
+- "MetricGroup": "IoBW;SoC;Server",
++ "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricName": "IO_Read_BW"
+ },
+ {
+diff --git a/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json b/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json
+index 6ed92bc5c129b..06c5ca26ca3f3 100644
+--- a/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json
++++ b/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json
+@@ -537,6 +537,18 @@
+ "PublicDescription": "Counts clockticks of the 1GHz trafiic controller clock in the IIO unit.",
+ "Unit": "IIO"
+ },
++ {
++ "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 0-3",
++ "Counter": "0,1,2,3",
++ "EventCode": "0xC2",
++ "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.ALL_PARTS",
++ "FCMask": "0x4",
++ "PerPkg": "1",
++ "PortMask": "0x0f",
++ "PublicDescription": "PCIe Completion Buffer Inserts of completions with data: Part 0-3",
++ "UMask": "0x03",
++ "Unit": "IIO"
++ },
+ {
+ "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 0",
+ "Counter": "0,1,2,3",
+@@ -585,6 +597,17 @@
+ "UMask": "0x03",
+ "Unit": "IIO"
+ },
++ {
++ "BriefDescription": "PCIe Completion Buffer occupancy of completions with data: Part 0-3",
++ "Counter": "2,3",
++ "EventCode": "0xD5",
++ "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.ALL_PARTS",
++ "FCMask": "0x04",
++ "PerPkg": "1",
++ "PublicDescription": "PCIe Completion Buffer occupancy of completions with data: Part 0-3",
++ "UMask": "0x0f",
++ "Unit": "IIO"
++ },
+ {
+ "BriefDescription": "PCIe Completion Buffer occupancy of completions with data: Part 0",
+ "Counter": "2,3",
+diff --git a/tools/perf/scripts/python/intel-pt-events.py b/tools/perf/scripts/python/intel-pt-events.py
+index 1d3a189a9a547..66452a8ec3586 100644
+--- a/tools/perf/scripts/python/intel-pt-events.py
++++ b/tools/perf/scripts/python/intel-pt-events.py
+@@ -32,8 +32,7 @@ try:
+ except:
+ broken_pipe_exception = IOError
+
+-glb_switch_str = None
+-glb_switch_printed = True
++glb_switch_str = {}
+ glb_insn = False
+ glb_disassembler = None
+ glb_src = False
+@@ -70,6 +69,7 @@ def trace_begin():
+ ap = argparse.ArgumentParser(usage = "", add_help = False)
+ ap.add_argument("--insn-trace", action='store_true')
+ ap.add_argument("--src-trace", action='store_true')
++ ap.add_argument("--all-switch-events", action='store_true')
+ global glb_args
+ global glb_insn
+ global glb_src
+@@ -256,10 +256,6 @@ def print_srccode(comm, param_dict, sample, symbol, dso, with_insn):
+ print(start_str, src_str)
+
+ def do_process_event(param_dict):
+- global glb_switch_printed
+- if not glb_switch_printed:
+- print(glb_switch_str)
+- glb_switch_printed = True
+ event_attr = param_dict["attr"]
+ sample = param_dict["sample"]
+ raw_buf = param_dict["raw_buf"]
+@@ -274,6 +270,11 @@ def do_process_event(param_dict):
+ dso = get_optional(param_dict, "dso")
+ symbol = get_optional(param_dict, "symbol")
+
++ cpu = sample["cpu"]
++ if cpu in glb_switch_str:
++ print(glb_switch_str[cpu])
++ del glb_switch_str[cpu]
++
+ if name[0:12] == "instructions":
+ if glb_src:
+ print_srccode(comm, param_dict, sample, symbol, dso, True)
+@@ -336,8 +337,6 @@ def auxtrace_error(typ, code, cpu, pid, tid, ip, ts, msg, cpumode, *x):
+ sys.exit(1)
+
+ def context_switch(ts, cpu, pid, tid, np_pid, np_tid, machine_pid, out, out_preempt, *x):
+- global glb_switch_printed
+- global glb_switch_str
+ if out:
+ out_str = "Switch out "
+ else:
+@@ -350,6 +349,10 @@ def context_switch(ts, cpu, pid, tid, np_pid, np_tid, machine_pid, out, out_pree
+ machine_str = ""
+ else:
+ machine_str = "machine PID %d" % machine_pid
+- glb_switch_str = "%16s %5d/%-5d [%03u] %9u.%09u %5d/%-5d %s %s" % \
++ switch_str = "%16s %5d/%-5d [%03u] %9u.%09u %5d/%-5d %s %s" % \
+ (out_str, pid, tid, cpu, ts / 1000000000, ts %1000000000, np_pid, np_tid, machine_str, preempt_str)
+- glb_switch_printed = False
++ if glb_args.all_switch_events:
++ print(switch_str);
++ else:
++ global glb_switch_str
++ glb_switch_str[cpu] = switch_str
+diff --git a/tools/perf/tests/shell/record+zstd_comp_decomp.sh b/tools/perf/tests/shell/record+zstd_comp_decomp.sh
+index 8a168cf8bacca..49bd875d51227 100755
+--- a/tools/perf/tests/shell/record+zstd_comp_decomp.sh
++++ b/tools/perf/tests/shell/record+zstd_comp_decomp.sh
+@@ -12,7 +12,7 @@ skip_if_no_z_record() {
+
+ collect_z_record() {
+ echo "Collecting compressed record file:"
+- [[ "$(uname -m)" != s390x ]] && gflag='-g'
++ [ "$(uname -m)" != s390x ] && gflag='-g'
+ $perf_tool record -o $trace_file $gflag -z -F 5000 -- \
+ dd count=500 if=/dev/urandom of=/dev/null
+ }
+diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c
+index e81c2493efdf9..44ba900828f6c 100644
+--- a/tools/perf/ui/browsers/annotate.c
++++ b/tools/perf/ui/browsers/annotate.c
+@@ -966,6 +966,7 @@ int symbol__tui_annotate(struct map_symbol *ms, struct evsel *evsel,
+ .opts = opts,
+ };
+ int ret = -1, err;
++ int not_annotated = list_empty(&notes->src->source);
+
+ if (sym == NULL)
+ return -1;
+@@ -973,13 +974,15 @@ int symbol__tui_annotate(struct map_symbol *ms, struct evsel *evsel,
+ if (ms->map->dso->annotate_warned)
+ return -1;
+
+- err = symbol__annotate2(ms, evsel, opts, &browser.arch);
+- if (err) {
+- char msg[BUFSIZ];
+- ms->map->dso->annotate_warned = true;
+- symbol__strerror_disassemble(ms, err, msg, sizeof(msg));
+- ui__error("Couldn't annotate %s:\n%s", sym->name, msg);
+- goto out_free_offsets;
++ if (not_annotated) {
++ err = symbol__annotate2(ms, evsel, opts, &browser.arch);
++ if (err) {
++ char msg[BUFSIZ];
++ ms->map->dso->annotate_warned = true;
++ symbol__strerror_disassemble(ms, err, msg, sizeof(msg));
++ ui__error("Couldn't annotate %s:\n%s", sym->name, msg);
++ goto out_free_offsets;
++ }
+ }
+
+ ui_helpline__push("Press ESC to exit");
+@@ -994,9 +997,11 @@ int symbol__tui_annotate(struct map_symbol *ms, struct evsel *evsel,
+
+ ret = annotate_browser__run(&browser, evsel, hbt);
+
+- annotated_source__purge(notes->src);
++ if(not_annotated)
++ annotated_source__purge(notes->src);
+
+ out_free_offsets:
+- zfree(&notes->offsets);
++ if(not_annotated)
++ zfree(&notes->offsets);
+ return ret;
+ }
+diff --git a/tools/perf/ui/hist.c b/tools/perf/ui/hist.c
+index c1f24d0048527..5075ecead5f3d 100644
+--- a/tools/perf/ui/hist.c
++++ b/tools/perf/ui/hist.c
+@@ -535,6 +535,18 @@ struct perf_hpp_list perf_hpp_list = {
+ #undef __HPP_SORT_ACC_FN
+ #undef __HPP_SORT_RAW_FN
+
++static void fmt_free(struct perf_hpp_fmt *fmt)
++{
++ /*
++ * At this point fmt should be completely
++ * unhooked, if not it's a bug.
++ */
++ BUG_ON(!list_empty(&fmt->list));
++ BUG_ON(!list_empty(&fmt->sort_list));
++
++ if (fmt->free)
++ fmt->free(fmt);
++}
+
+ void perf_hpp__init(void)
+ {
+@@ -598,9 +610,10 @@ void perf_hpp_list__prepend_sort_field(struct perf_hpp_list *list,
+ list_add(&format->sort_list, &list->sorts);
+ }
+
+-void perf_hpp__column_unregister(struct perf_hpp_fmt *format)
++static void perf_hpp__column_unregister(struct perf_hpp_fmt *format)
+ {
+ list_del_init(&format->list);
++ fmt_free(format);
+ }
+
+ void perf_hpp__cancel_cumulate(void)
+@@ -672,19 +685,6 @@ next:
+ }
+
+
+-static void fmt_free(struct perf_hpp_fmt *fmt)
+-{
+- /*
+- * At this point fmt should be completely
+- * unhooked, if not it's a bug.
+- */
+- BUG_ON(!list_empty(&fmt->list));
+- BUG_ON(!list_empty(&fmt->sort_list));
+-
+- if (fmt->free)
+- fmt->free(fmt);
+-}
+-
+ void perf_hpp__reset_output_field(struct perf_hpp_list *list)
+ {
+ struct perf_hpp_fmt *fmt, *tmp;
+diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c
+index 58b7069c5a5f8..235549bb28b94 100644
+--- a/tools/perf/util/arm-spe.c
++++ b/tools/perf/util/arm-spe.c
+@@ -51,6 +51,7 @@ struct arm_spe {
+ u8 timeless_decoding;
+ u8 data_queued;
+
++ u64 sample_type;
+ u8 sample_flc;
+ u8 sample_llc;
+ u8 sample_tlb;
+@@ -248,6 +249,12 @@ static void arm_spe_prep_sample(struct arm_spe *spe,
+ event->sample.header.size = sizeof(struct perf_event_header);
+ }
+
++static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
++{
++ event->header.size = perf_event__sample_event_size(sample, type, 0);
++ return perf_event__synthesize_sample(event, type, 0, sample);
++}
++
+ static inline int
+ arm_spe_deliver_synth_event(struct arm_spe *spe,
+ struct arm_spe_queue *speq __maybe_unused,
+@@ -256,6 +263,12 @@ arm_spe_deliver_synth_event(struct arm_spe *spe,
+ {
+ int ret;
+
++ if (spe->synth_opts.inject) {
++ ret = arm_spe__inject_event(event, sample, spe->sample_type);
++ if (ret)
++ return ret;
++ }
++
+ ret = perf_session__deliver_synth_event(spe->session, event, sample);
+ if (ret)
+ pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
+@@ -914,12 +927,15 @@ arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
+ attr.type = PERF_TYPE_HARDWARE;
+ attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
+ attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
+- PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC;
++ PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC |
++ PERF_SAMPLE_ADDR;
+ if (spe->timeless_decoding)
+ attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
+ else
+ attr.sample_type |= PERF_SAMPLE_TIME;
+
++ spe->sample_type = attr.sample_type;
++
+ attr.exclude_user = evsel->core.attr.exclude_user;
+ attr.exclude_kernel = evsel->core.attr.exclude_kernel;
+ attr.exclude_hv = evsel->core.attr.exclude_hv;
+diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c
+index 1a7112a87736a..cf1b9f6ec0dbe 100644
+--- a/tools/perf/util/bpf-event.c
++++ b/tools/perf/util/bpf-event.c
+@@ -21,7 +21,8 @@
+ #include "record.h"
+ #include "util/synthetic-events.h"
+
+-struct btf * __weak btf__load_from_kernel_by_id(__u32 id)
++#ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
++struct btf *btf__load_from_kernel_by_id(__u32 id)
+ {
+ struct btf *btf;
+ #pragma GCC diagnostic push
+@@ -31,6 +32,7 @@ struct btf * __weak btf__load_from_kernel_by_id(__u32 id)
+
+ return err ? ERR_PTR(err) : btf;
+ }
++#endif
+
+ #define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr))
+
+@@ -120,7 +122,11 @@ static int perf_env__fetch_btf(struct perf_env *env,
+ node->data_size = data_size;
+ memcpy(node->data, data, data_size);
+
+- perf_env__insert_btf(env, node);
++ if (!perf_env__insert_btf(env, node)) {
++ /* Insertion failed because of a duplicate. */
++ free(node);
++ return -1;
++ }
+ return 0;
+ }
+
+@@ -576,7 +582,7 @@ void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
+ synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
+ fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
+ info->id, name, prog_addrs[0], prog_lens[0]);
+- return;
++ goto out;
+ }
+
+ fprintf(fp, "# bpf_prog_info %u:\n", info->id);
+@@ -586,4 +592,6 @@ void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
+ fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
+ i, name, prog_addrs[i], prog_lens[i]);
+ }
++out:
++ btf__free(btf);
+ }
+diff --git a/tools/perf/util/bpf-loader.c b/tools/perf/util/bpf-loader.c
+index fbb3c4057c302..71710a1da4472 100644
+--- a/tools/perf/util/bpf-loader.c
++++ b/tools/perf/util/bpf-loader.c
+@@ -1214,9 +1214,10 @@ bpf__obj_config_map(struct bpf_object *obj,
+ pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
+ err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
+ out:
+- free(map_name);
+ if (!err)
+ *key_scan_pos += strlen(map_opt);
++
++ free(map_name);
+ return err;
+ }
+
+diff --git a/tools/perf/util/bpf_skel/bperf.h b/tools/perf/util/bpf_skel/bperf.h
+deleted file mode 100644
+index 186a5551ddb9d..0000000000000
+--- a/tools/perf/util/bpf_skel/bperf.h
++++ /dev/null
+@@ -1,14 +0,0 @@
+-// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+-// Copyright (c) 2021 Facebook
+-
+-#ifndef __BPERF_STAT_H
+-#define __BPERF_STAT_H
+-
+-typedef struct {
+- __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+- __uint(key_size, sizeof(__u32));
+- __uint(value_size, sizeof(struct bpf_perf_event_value));
+- __uint(max_entries, 1);
+-} reading_map;
+-
+-#endif /* __BPERF_STAT_H */
+diff --git a/tools/perf/util/bpf_skel/bperf_follower.bpf.c b/tools/perf/util/bpf_skel/bperf_follower.bpf.c
+index b8fa3cb2da230..6d2ea67b161ac 100644
+--- a/tools/perf/util/bpf_skel/bperf_follower.bpf.c
++++ b/tools/perf/util/bpf_skel/bperf_follower.bpf.c
+@@ -4,11 +4,21 @@
+ #include <linux/perf_event.h>
+ #include <bpf/bpf_helpers.h>
+ #include <bpf/bpf_tracing.h>
+-#include "bperf.h"
+ #include "bperf_u.h"
+
+-reading_map diff_readings SEC(".maps");
+-reading_map accum_readings SEC(".maps");
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(key_size, sizeof(__u32));
++ __uint(value_size, sizeof(struct bpf_perf_event_value));
++ __uint(max_entries, 1);
++} diff_readings SEC(".maps");
++
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(key_size, sizeof(__u32));
++ __uint(value_size, sizeof(struct bpf_perf_event_value));
++ __uint(max_entries, 1);
++} accum_readings SEC(".maps");
+
+ struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+diff --git a/tools/perf/util/bpf_skel/bperf_leader.bpf.c b/tools/perf/util/bpf_skel/bperf_leader.bpf.c
+index 4f70d1459e86c..d82e1633a2e0a 100644
+--- a/tools/perf/util/bpf_skel/bperf_leader.bpf.c
++++ b/tools/perf/util/bpf_skel/bperf_leader.bpf.c
+@@ -4,7 +4,6 @@
+ #include <linux/perf_event.h>
+ #include <bpf/bpf_helpers.h>
+ #include <bpf/bpf_tracing.h>
+-#include "bperf.h"
+
+ struct {
+ __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+@@ -13,8 +12,19 @@ struct {
+ __uint(map_flags, BPF_F_PRESERVE_ELEMS);
+ } events SEC(".maps");
+
+-reading_map prev_readings SEC(".maps");
+-reading_map diff_readings SEC(".maps");
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(key_size, sizeof(__u32));
++ __uint(value_size, sizeof(struct bpf_perf_event_value));
++ __uint(max_entries, 1);
++} prev_readings SEC(".maps");
++
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(key_size, sizeof(__u32));
++ __uint(value_size, sizeof(struct bpf_perf_event_value));
++ __uint(max_entries, 1);
++} diff_readings SEC(".maps");
+
+ SEC("raw_tp/sched_switch")
+ int BPF_PROG(on_switch)
+diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c
+index f5d260b1df4d1..15a4547d608ec 100644
+--- a/tools/perf/util/data.c
++++ b/tools/perf/util/data.c
+@@ -44,10 +44,6 @@ int perf_data__create_dir(struct perf_data *data, int nr)
+ if (!files)
+ return -ENOMEM;
+
+- data->dir.version = PERF_DIR_VERSION;
+- data->dir.files = files;
+- data->dir.nr = nr;
+-
+ for (i = 0; i < nr; i++) {
+ struct perf_data_file *file = &files[i];
+
+@@ -62,6 +58,9 @@ int perf_data__create_dir(struct perf_data *data, int nr)
+ file->fd = ret;
+ }
+
++ data->dir.version = PERF_DIR_VERSION;
++ data->dir.files = files;
++ data->dir.nr = nr;
+ return 0;
+
+ out_err:
+diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c
+index 2c06abf6dcd26..65e6c22f38e4f 100644
+--- a/tools/perf/util/debug.c
++++ b/tools/perf/util/debug.c
+@@ -179,7 +179,7 @@ static int trace_event_printer(enum binary_printer_ops op,
+ break;
+ case BINARY_PRINT_CHAR_DATA:
+ printed += color_fprintf(fp, color, "%c",
+- isprint(ch) ? ch : '.');
++ isprint(ch) && isascii(ch) ? ch : '.');
+ break;
+ case BINARY_PRINT_CHAR_PAD:
+ printed += color_fprintf(fp, color, " ");
+diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c
+index cf773f0dec384..5b24eb010336c 100644
+--- a/tools/perf/util/env.c
++++ b/tools/perf/util/env.c
+@@ -74,12 +74,13 @@ out:
+ return node;
+ }
+
+-void perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)
++bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)
+ {
+ struct rb_node *parent = NULL;
+ __u32 btf_id = btf_node->id;
+ struct btf_node *node;
+ struct rb_node **p;
++ bool ret = true;
+
+ down_write(&env->bpf_progs.lock);
+ p = &env->bpf_progs.btfs.rb_node;
+@@ -93,6 +94,7 @@ void perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)
+ p = &(*p)->rb_right;
+ } else {
+ pr_debug("duplicated btf %u\n", btf_id);
++ ret = false;
+ goto out;
+ }
+ }
+@@ -102,6 +104,7 @@ void perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)
+ env->bpf_progs.btfs_cnt++;
+ out:
+ up_write(&env->bpf_progs.lock);
++ return ret;
+ }
+
+ struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id)
+diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h
+index 1383876f72b37..163e5ec503a26 100644
+--- a/tools/perf/util/env.h
++++ b/tools/perf/util/env.h
+@@ -167,7 +167,7 @@ void perf_env__insert_bpf_prog_info(struct perf_env *env,
+ struct bpf_prog_info_node *info_node);
+ struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env,
+ __u32 prog_id);
+-void perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);
++bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);
+ struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id);
+
+ int perf_env__numa_node(struct perf_env *env, int cpu);
+diff --git a/tools/perf/util/evlist-hybrid.c b/tools/perf/util/evlist-hybrid.c
+index 7c554234b43d4..f39c8ffc5a111 100644
+--- a/tools/perf/util/evlist-hybrid.c
++++ b/tools/perf/util/evlist-hybrid.c
+@@ -153,8 +153,8 @@ int evlist__fix_hybrid_cpus(struct evlist *evlist, const char *cpu_list)
+ perf_cpu_map__put(matched_cpus);
+ perf_cpu_map__put(unmatched_cpus);
+ }
+-
+- ret = (unmatched_count == events_nr) ? -1 : 0;
++ if (events_nr)
++ ret = (unmatched_count == events_nr) ? -1 : 0;
+ out:
+ perf_cpu_map__put(cpus);
+ return ret;
+diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
+index dbfeceb2546c6..c87f9974c0c10 100644
+--- a/tools/perf/util/evsel.c
++++ b/tools/perf/util/evsel.c
+@@ -1047,6 +1047,17 @@ void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
+ evsel__set_sample_bit(evsel, WEIGHT);
+ }
+
++static void evsel__set_default_freq_period(struct record_opts *opts,
++ struct perf_event_attr *attr)
++{
++ if (opts->freq) {
++ attr->freq = 1;
++ attr->sample_freq = opts->freq;
++ } else {
++ attr->sample_period = opts->default_interval;
++ }
++}
++
+ /*
+ * The enable_on_exec/disabled value strategy:
+ *
+@@ -1113,14 +1124,12 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts,
+ * We default some events to have a default interval. But keep
+ * it a weak assumption overridable by the user.
+ */
+- if (!attr->sample_period) {
+- if (opts->freq) {
+- attr->freq = 1;
+- attr->sample_freq = opts->freq;
+- } else {
+- attr->sample_period = opts->default_interval;
+- }
+- }
++ if ((evsel->is_libpfm_event && !attr->sample_period) ||
++ (!evsel->is_libpfm_event && (!attr->sample_period ||
++ opts->user_freq != UINT_MAX ||
++ opts->user_interval != ULLONG_MAX)))
++ evsel__set_default_freq_period(opts, attr);
++
+ /*
+ * If attr->freq was set (here or earlier), ask for period
+ * to be sampled.
+diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
+index 65fe65ba03c25..b776465e04ef3 100644
+--- a/tools/perf/util/hist.c
++++ b/tools/perf/util/hist.c
+@@ -289,15 +289,10 @@ static long hist_time(unsigned long htime)
+ return htime;
+ }
+
+-static void he_stat__add_period(struct he_stat *he_stat, u64 period,
+- u64 weight, u64 ins_lat, u64 p_stage_cyc)
++static void he_stat__add_period(struct he_stat *he_stat, u64 period)
+ {
+-
+ he_stat->period += period;
+- he_stat->weight += weight;
+ he_stat->nr_events += 1;
+- he_stat->ins_lat += ins_lat;
+- he_stat->p_stage_cyc += p_stage_cyc;
+ }
+
+ static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src)
+@@ -308,9 +303,6 @@ static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src)
+ dest->period_guest_sys += src->period_guest_sys;
+ dest->period_guest_us += src->period_guest_us;
+ dest->nr_events += src->nr_events;
+- dest->weight += src->weight;
+- dest->ins_lat += src->ins_lat;
+- dest->p_stage_cyc += src->p_stage_cyc;
+ }
+
+ static void he_stat__decay(struct he_stat *he_stat)
+@@ -598,9 +590,6 @@ static struct hist_entry *hists__findnew_entry(struct hists *hists,
+ struct hist_entry *he;
+ int64_t cmp;
+ u64 period = entry->stat.period;
+- u64 weight = entry->stat.weight;
+- u64 ins_lat = entry->stat.ins_lat;
+- u64 p_stage_cyc = entry->stat.p_stage_cyc;
+ bool leftmost = true;
+
+ p = &hists->entries_in->rb_root.rb_node;
+@@ -619,11 +608,11 @@ static struct hist_entry *hists__findnew_entry(struct hists *hists,
+
+ if (!cmp) {
+ if (sample_self) {
+- he_stat__add_period(&he->stat, period, weight, ins_lat, p_stage_cyc);
++ he_stat__add_period(&he->stat, period);
+ hist_entry__add_callchain_period(he, period);
+ }
+ if (symbol_conf.cumulate_callchain)
+- he_stat__add_period(he->stat_acc, period, weight, ins_lat, p_stage_cyc);
++ he_stat__add_period(he->stat_acc, period);
+
+ /*
+ * This mem info was allocated from sample__resolve_mem
+@@ -733,9 +722,6 @@ __hists__add_entry(struct hists *hists,
+ .stat = {
+ .nr_events = 1,
+ .period = sample->period,
+- .weight = sample->weight,
+- .ins_lat = sample->ins_lat,
+- .p_stage_cyc = sample->p_stage_cyc,
+ },
+ .parent = sym_parent,
+ .filtered = symbol__parent_filter(sym_parent) | al->filtered,
+@@ -748,6 +734,9 @@ __hists__add_entry(struct hists *hists,
+ .raw_size = sample->raw_size,
+ .ops = ops,
+ .time = hist_time(sample->time),
++ .weight = sample->weight,
++ .ins_lat = sample->ins_lat,
++ .p_stage_cyc = sample->p_stage_cyc,
+ }, *he = hists__findnew_entry(hists, &entry, al, sample_self);
+
+ if (!hists->has_callchains && he && he->callchain_size != 0)
+diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h
+index 5343b62476e60..621f35ae1efa5 100644
+--- a/tools/perf/util/hist.h
++++ b/tools/perf/util/hist.h
+@@ -369,7 +369,6 @@ enum {
+ };
+
+ void perf_hpp__init(void);
+-void perf_hpp__column_unregister(struct perf_hpp_fmt *format);
+ void perf_hpp__cancel_cumulate(void);
+ void perf_hpp__setup_output_field(struct perf_hpp_list *list);
+ void perf_hpp__reset_output_field(struct perf_hpp_list *list);
+diff --git a/tools/perf/util/intel-pt-decoder/Build b/tools/perf/util/intel-pt-decoder/Build
+index bc629359826fb..b41c2e9c6f887 100644
+--- a/tools/perf/util/intel-pt-decoder/Build
++++ b/tools/perf/util/intel-pt-decoder/Build
+@@ -18,3 +18,5 @@ CFLAGS_intel-pt-insn-decoder.o += -I$(OUTPUT)util/intel-pt-decoder
+ ifeq ($(CC_NO_CLANG), 1)
+ CFLAGS_intel-pt-insn-decoder.o += -Wno-override-init
+ endif
++
++CFLAGS_intel-pt-insn-decoder.o += -Wno-packed
+diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
+index 5ab631702769b..b0034ee4bba50 100644
+--- a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
++++ b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
+@@ -1204,61 +1204,69 @@ out_no_progress:
+
+ static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
+ {
++ enum intel_pt_sample_type type = decoder->state.type;
+ bool ret = false;
+
++ decoder->state.type &= ~INTEL_PT_BRANCH;
++
+ if (decoder->set_fup_tx_flags) {
+ decoder->set_fup_tx_flags = false;
+ decoder->tx_flags = decoder->fup_tx_flags;
+- decoder->state.type = INTEL_PT_TRANSACTION;
++ decoder->state.type |= INTEL_PT_TRANSACTION;
+ if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
+ decoder->state.type |= INTEL_PT_BRANCH;
+- decoder->state.from_ip = decoder->ip;
+- decoder->state.to_ip = 0;
+ decoder->state.flags = decoder->fup_tx_flags;
+- return true;
++ ret = true;
+ }
+ if (decoder->set_fup_ptw) {
+ decoder->set_fup_ptw = false;
+- decoder->state.type = INTEL_PT_PTW;
++ decoder->state.type |= INTEL_PT_PTW;
+ decoder->state.flags |= INTEL_PT_FUP_IP;
+- decoder->state.from_ip = decoder->ip;
+- decoder->state.to_ip = 0;
+ decoder->state.ptw_payload = decoder->fup_ptw_payload;
+- return true;
++ ret = true;
+ }
+ if (decoder->set_fup_mwait) {
+ decoder->set_fup_mwait = false;
+- decoder->state.type = INTEL_PT_MWAIT_OP;
+- decoder->state.from_ip = decoder->ip;
+- decoder->state.to_ip = 0;
++ decoder->state.type |= INTEL_PT_MWAIT_OP;
+ decoder->state.mwait_payload = decoder->fup_mwait_payload;
+ ret = true;
+ }
+ if (decoder->set_fup_pwre) {
+ decoder->set_fup_pwre = false;
+ decoder->state.type |= INTEL_PT_PWR_ENTRY;
+- decoder->state.type &= ~INTEL_PT_BRANCH;
+- decoder->state.from_ip = decoder->ip;
+- decoder->state.to_ip = 0;
+ decoder->state.pwre_payload = decoder->fup_pwre_payload;
+ ret = true;
+ }
+ if (decoder->set_fup_exstop) {
+ decoder->set_fup_exstop = false;
+ decoder->state.type |= INTEL_PT_EX_STOP;
+- decoder->state.type &= ~INTEL_PT_BRANCH;
+ decoder->state.flags |= INTEL_PT_FUP_IP;
+- decoder->state.from_ip = decoder->ip;
+- decoder->state.to_ip = 0;
+ ret = true;
+ }
+ if (decoder->set_fup_bep) {
+ decoder->set_fup_bep = false;
+ decoder->state.type |= INTEL_PT_BLK_ITEMS;
+- decoder->state.type &= ~INTEL_PT_BRANCH;
++ ret = true;
++ }
++ if (decoder->overflow) {
++ decoder->overflow = false;
++ if (!ret && !decoder->pge) {
++ if (decoder->hop) {
++ decoder->state.type = 0;
++ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
++ }
++ decoder->pge = true;
++ decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
++ decoder->state.from_ip = 0;
++ decoder->state.to_ip = decoder->ip;
++ return true;
++ }
++ }
++ if (ret) {
+ decoder->state.from_ip = decoder->ip;
+ decoder->state.to_ip = 0;
+- ret = true;
++ } else {
++ decoder->state.type = type;
+ }
+ return ret;
+ }
+@@ -1607,7 +1615,16 @@ static int intel_pt_overflow(struct intel_pt_decoder *decoder)
+ intel_pt_clear_tx_flags(decoder);
+ intel_pt_set_nr(decoder);
+ decoder->timestamp_insn_cnt = 0;
+- decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
++ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
++ decoder->state.from_ip = decoder->ip;
++ decoder->ip = 0;
++ decoder->pge = false;
++ decoder->set_fup_tx_flags = false;
++ decoder->set_fup_ptw = false;
++ decoder->set_fup_mwait = false;
++ decoder->set_fup_pwre = false;
++ decoder->set_fup_exstop = false;
++ decoder->set_fup_bep = false;
+ decoder->overflow = true;
+ return -EOVERFLOW;
+ }
+@@ -2665,6 +2682,8 @@ static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
+ /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
+ static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
+ {
++ *err = 0;
++
+ /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
+ if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
+ *err = intel_pt_scan_for_psb(decoder);
+@@ -2677,6 +2696,7 @@ static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, in
+ return HOP_IGNORE;
+
+ case INTEL_PT_TIP_PGD:
++ decoder->pge = false;
+ if (!decoder->packet.count) {
+ intel_pt_set_nr(decoder);
+ return HOP_IGNORE;
+@@ -2704,18 +2724,21 @@ static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, in
+ if (!decoder->packet.count)
+ return HOP_IGNORE;
+ intel_pt_set_ip(decoder);
+- if (intel_pt_fup_event(decoder))
+- return HOP_RETURN;
+- if (!decoder->branch_enable)
++ if (decoder->set_fup_mwait || decoder->set_fup_pwre)
++ *no_tip = true;
++ if (!decoder->branch_enable || !decoder->pge)
+ *no_tip = true;
+ if (*no_tip) {
+ decoder->state.type = INTEL_PT_INSTRUCTION;
+ decoder->state.from_ip = decoder->ip;
+ decoder->state.to_ip = 0;
++ intel_pt_fup_event(decoder);
+ return HOP_RETURN;
+ }
++ intel_pt_fup_event(decoder);
++ decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;
+ *err = intel_pt_walk_fup_tip(decoder);
+- if (!*err)
++ if (!*err && decoder->state.to_ip)
+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
+ return HOP_RETURN;
+
+@@ -2896,7 +2919,7 @@ static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
+ {
+ struct intel_pt_psb_info data = { .fup = false };
+
+- if (!decoder->branch_enable || !decoder->pge)
++ if (!decoder->branch_enable)
+ return false;
+
+ intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
+@@ -2923,6 +2946,7 @@ static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
+ if (err)
+ return err;
+ next:
++ err = 0;
+ if (decoder->cyc_threshold) {
+ if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
+ decoder->sample_cyc = false;
+@@ -2961,6 +2985,7 @@ next:
+
+ case INTEL_PT_TIP_PGE: {
+ decoder->pge = true;
++ decoder->overflow = false;
+ intel_pt_mtc_cyc_cnt_pge(decoder);
+ intel_pt_set_nr(decoder);
+ if (decoder->packet.count == 0) {
+@@ -2998,7 +3023,7 @@ next:
+ break;
+ }
+ intel_pt_set_last_ip(decoder);
+- if (!decoder->branch_enable) {
++ if (!decoder->branch_enable || !decoder->pge) {
+ decoder->ip = decoder->last_ip;
+ if (intel_pt_fup_event(decoder))
+ return 0;
+@@ -3466,10 +3491,10 @@ static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
+ decoder->set_fup_pwre = false;
+ decoder->set_fup_exstop = false;
+ decoder->set_fup_bep = false;
++ decoder->overflow = false;
+
+ if (!decoder->branch_enable) {
+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
+- decoder->overflow = false;
+ decoder->state.type = 0; /* Do not have a sample */
+ return 0;
+ }
+@@ -3484,7 +3509,6 @@ static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
+ else
+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
+- decoder->overflow = false;
+
+ decoder->state.from_ip = 0;
+ decoder->state.to_ip = decoder->ip;
+@@ -3606,7 +3630,7 @@ static int intel_pt_sync(struct intel_pt_decoder *decoder)
+ }
+
+ decoder->have_last_ip = true;
+- decoder->pkt_state = INTEL_PT_STATE_NO_IP;
++ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
+
+ err = intel_pt_walk_psb(decoder);
+ if (err)
+@@ -3703,7 +3727,8 @@ const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
+
+ if (err) {
+ decoder->state.err = intel_pt_ext_err(err);
+- decoder->state.from_ip = decoder->ip;
++ if (err != -EOVERFLOW)
++ decoder->state.from_ip = decoder->ip;
+ intel_pt_update_sample_time(decoder);
+ decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
+ intel_pt_set_nr(decoder);
+diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c
+index 6f852b305e92b..c3ceac1388106 100644
+--- a/tools/perf/util/intel-pt.c
++++ b/tools/perf/util/intel-pt.c
+@@ -2510,6 +2510,7 @@ static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
+ ptq->sync_switch = false;
+ intel_pt_next_tid(pt, ptq);
+ }
++ ptq->timestamp = state->est_timestamp;
+ if (pt->synth_opts.errors) {
+ err = intel_ptq_synth_error(ptq, state);
+ if (err)
+@@ -3539,6 +3540,7 @@ static int intel_pt_parse_vm_tm_corr_arg(struct intel_pt *pt, char **args)
+ *args = p;
+ return 0;
+ }
++ p += 1;
+ while (1) {
+ vmcs = strtoull(p, &p, 0);
+ if (errno)
+diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
+index 51a2219df6015..3bfe099d86438 100644
+--- a/tools/perf/util/parse-events.c
++++ b/tools/perf/util/parse-events.c
+@@ -1529,7 +1529,9 @@ int parse_events_add_pmu(struct parse_events_state *parse_state,
+ bool use_uncore_alias;
+ LIST_HEAD(config_terms);
+
+- if (verbose > 1) {
++ pmu = parse_state->fake_pmu ?: perf_pmu__find(name);
++
++ if (verbose > 1 && !(pmu && pmu->selectable)) {
+ fprintf(stderr, "Attempting to add event pmu '%s' with '",
+ name);
+ if (head_config) {
+@@ -1542,7 +1544,6 @@ int parse_events_add_pmu(struct parse_events_state *parse_state,
+ fprintf(stderr, "' that may result in non-fatal errors\n");
+ }
+
+- pmu = parse_state->fake_pmu ?: perf_pmu__find(name);
+ if (!pmu) {
+ char *err_str;
+
+diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
+index b2a02c9ab8ea9..a834918a0a0d3 100644
+--- a/tools/perf/util/probe-event.c
++++ b/tools/perf/util/probe-event.c
+@@ -3083,6 +3083,9 @@ static int find_probe_trace_events_from_map(struct perf_probe_event *pev,
+ for (j = 0; j < num_matched_functions; j++) {
+ sym = syms[j];
+
++ if (sym->type != STT_FUNC)
++ continue;
++
+ /* There can be duplicated symbols in the map */
+ for (i = 0; i < j; i++)
+ if (sym->start == syms[i]->start) {
+diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
+index 352f16076e01f..562e9b8080272 100644
+--- a/tools/perf/util/session.c
++++ b/tools/perf/util/session.c
+@@ -2076,6 +2076,7 @@ prefetch_event(char *buf, u64 head, size_t mmap_size,
+ bool needs_swap, union perf_event *error)
+ {
+ union perf_event *event;
++ u16 event_size;
+
+ /*
+ * Ensure we have enough space remaining to read
+@@ -2088,15 +2089,23 @@ prefetch_event(char *buf, u64 head, size_t mmap_size,
+ if (needs_swap)
+ perf_event_header__bswap(&event->header);
+
+- if (head + event->header.size <= mmap_size)
++ event_size = event->header.size;
++ if (head + event_size <= mmap_size)
+ return event;
+
+ /* We're not fetching the event so swap back again */
+ if (needs_swap)
+ perf_event_header__bswap(&event->header);
+
+- pr_debug("%s: head=%#" PRIx64 " event->header_size=%#x, mmap_size=%#zx:"
+- " fuzzed or compressed perf.data?\n",__func__, head, event->header.size, mmap_size);
++ /* Check if the event fits into the next mmapped buf. */
++ if (event_size <= mmap_size - head % page_size) {
++ /* Remap buf and fetch again. */
++ return NULL;
++ }
++
++ /* Invalid input. Event size should never exceed mmap_size. */
++ pr_debug("%s: head=%#" PRIx64 " event->header.size=%#x, mmap_size=%#zx:"
++ " fuzzed or compressed perf.data?\n", __func__, head, event_size, mmap_size);
+
+ return error;
+ }
+diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py
+index 483f05004e682..c255a2c90cd67 100644
+--- a/tools/perf/util/setup.py
++++ b/tools/perf/util/setup.py
+@@ -1,12 +1,14 @@
+-from os import getenv
++from os import getenv, path
+ from subprocess import Popen, PIPE
+ from re import sub
+
+ cc = getenv("CC")
+ cc_is_clang = b"clang version" in Popen([cc.split()[0], "-v"], stderr=PIPE).stderr.readline()
++src_feature_tests = getenv('srctree') + '/tools/build/feature'
+
+ def clang_has_option(option):
+- return [o for o in Popen([cc, option], stderr=PIPE).stderr.readlines() if b"unknown argument" in o] == [ ]
++ cc_output = Popen([cc, option, path.join(src_feature_tests, "test-hello.c") ], stderr=PIPE).stderr.readlines()
++ return [o for o in cc_output if ((b"unknown argument" in o) or (b"is not supported" in o))] == [ ]
+
+ if cc_is_clang:
+ from distutils.sysconfig import get_config_vars
+@@ -23,6 +25,8 @@ if cc_is_clang:
+ vars[var] = sub("-fstack-protector-strong", "", vars[var])
+ if not clang_has_option("-fno-semantic-interposition"):
+ vars[var] = sub("-fno-semantic-interposition", "", vars[var])
++ if not clang_has_option("-ffat-lto-objects"):
++ vars[var] = sub("-ffat-lto-objects", "", vars[var])
+
+ from distutils.core import setup, Extension
+
+diff --git a/tools/perf/util/smt.c b/tools/perf/util/smt.c
+index 20bacd5972ade..34f1b1b1176c7 100644
+--- a/tools/perf/util/smt.c
++++ b/tools/perf/util/smt.c
+@@ -15,7 +15,7 @@ int smt_on(void)
+ if (cached)
+ return cached_result;
+
+- if (sysfs__read_int("devices/system/cpu/smt/active", &cached_result) > 0)
++ if (sysfs__read_int("devices/system/cpu/smt/active", &cached_result) >= 0)
+ goto done;
+
+ ncpu = sysconf(_SC_NPROCESSORS_CONF);
+diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c
+index 568a88c001c6c..a111065b484ef 100644
+--- a/tools/perf/util/sort.c
++++ b/tools/perf/util/sort.c
+@@ -1325,88 +1325,68 @@ struct sort_entry sort_mispredict = {
+ .se_width_idx = HISTC_MISPREDICT,
+ };
+
+-static u64 he_weight(struct hist_entry *he)
+-{
+- return he->stat.nr_events ? he->stat.weight / he->stat.nr_events : 0;
+-}
+-
+ static int64_t
+-sort__local_weight_cmp(struct hist_entry *left, struct hist_entry *right)
++sort__weight_cmp(struct hist_entry *left, struct hist_entry *right)
+ {
+- return he_weight(left) - he_weight(right);
++ return left->weight - right->weight;
+ }
+
+ static int hist_entry__local_weight_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+ {
+- return repsep_snprintf(bf, size, "%-*llu", width, he_weight(he));
++ return repsep_snprintf(bf, size, "%-*llu", width, he->weight);
+ }
+
+ struct sort_entry sort_local_weight = {
+ .se_header = "Local Weight",
+- .se_cmp = sort__local_weight_cmp,
++ .se_cmp = sort__weight_cmp,
+ .se_snprintf = hist_entry__local_weight_snprintf,
+ .se_width_idx = HISTC_LOCAL_WEIGHT,
+ };
+
+-static int64_t
+-sort__global_weight_cmp(struct hist_entry *left, struct hist_entry *right)
+-{
+- return left->stat.weight - right->stat.weight;
+-}
+-
+ static int hist_entry__global_weight_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+ {
+- return repsep_snprintf(bf, size, "%-*llu", width, he->stat.weight);
++ return repsep_snprintf(bf, size, "%-*llu", width,
++ he->weight * he->stat.nr_events);
+ }
+
+ struct sort_entry sort_global_weight = {
+ .se_header = "Weight",
+- .se_cmp = sort__global_weight_cmp,
++ .se_cmp = sort__weight_cmp,
+ .se_snprintf = hist_entry__global_weight_snprintf,
+ .se_width_idx = HISTC_GLOBAL_WEIGHT,
+ };
+
+-static u64 he_ins_lat(struct hist_entry *he)
+-{
+- return he->stat.nr_events ? he->stat.ins_lat / he->stat.nr_events : 0;
+-}
+-
+ static int64_t
+-sort__local_ins_lat_cmp(struct hist_entry *left, struct hist_entry *right)
++sort__ins_lat_cmp(struct hist_entry *left, struct hist_entry *right)
+ {
+- return he_ins_lat(left) - he_ins_lat(right);
++ return left->ins_lat - right->ins_lat;
+ }
+
+ static int hist_entry__local_ins_lat_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+ {
+- return repsep_snprintf(bf, size, "%-*u", width, he_ins_lat(he));
++ return repsep_snprintf(bf, size, "%-*u", width, he->ins_lat);
+ }
+
+ struct sort_entry sort_local_ins_lat = {
+ .se_header = "Local INSTR Latency",
+- .se_cmp = sort__local_ins_lat_cmp,
++ .se_cmp = sort__ins_lat_cmp,
+ .se_snprintf = hist_entry__local_ins_lat_snprintf,
+ .se_width_idx = HISTC_LOCAL_INS_LAT,
+ };
+
+-static int64_t
+-sort__global_ins_lat_cmp(struct hist_entry *left, struct hist_entry *right)
+-{
+- return left->stat.ins_lat - right->stat.ins_lat;
+-}
+-
+ static int hist_entry__global_ins_lat_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+ {
+- return repsep_snprintf(bf, size, "%-*u", width, he->stat.ins_lat);
++ return repsep_snprintf(bf, size, "%-*u", width,
++ he->ins_lat * he->stat.nr_events);
+ }
+
+ struct sort_entry sort_global_ins_lat = {
+ .se_header = "INSTR Latency",
+- .se_cmp = sort__global_ins_lat_cmp,
++ .se_cmp = sort__ins_lat_cmp,
+ .se_snprintf = hist_entry__global_ins_lat_snprintf,
+ .se_width_idx = HISTC_GLOBAL_INS_LAT,
+ };
+@@ -1414,13 +1394,13 @@ struct sort_entry sort_global_ins_lat = {
+ static int64_t
+ sort__global_p_stage_cyc_cmp(struct hist_entry *left, struct hist_entry *right)
+ {
+- return left->stat.p_stage_cyc - right->stat.p_stage_cyc;
++ return left->p_stage_cyc - right->p_stage_cyc;
+ }
+
+ static int hist_entry__p_stage_cyc_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+ {
+- return repsep_snprintf(bf, size, "%-*u", width, he->stat.p_stage_cyc);
++ return repsep_snprintf(bf, size, "%-*u", width, he->p_stage_cyc);
+ }
+
+ struct sort_entry sort_p_stage_cyc = {
+diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h
+index b67c469aba795..7b7145501933f 100644
+--- a/tools/perf/util/sort.h
++++ b/tools/perf/util/sort.h
+@@ -49,9 +49,6 @@ struct he_stat {
+ u64 period_us;
+ u64 period_guest_sys;
+ u64 period_guest_us;
+- u64 weight;
+- u64 ins_lat;
+- u64 p_stage_cyc;
+ u32 nr_events;
+ };
+
+@@ -109,6 +106,9 @@ struct hist_entry {
+ s32 socket;
+ s32 cpu;
+ u64 code_page_size;
++ u64 weight;
++ u64 ins_lat;
++ u64 p_stage_cyc;
+ u8 cpumode;
+ u8 depth;
+
+diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c
+index 588601000f3f9..db00ca6a67deb 100644
+--- a/tools/perf/util/stat-display.c
++++ b/tools/perf/util/stat-display.c
+@@ -584,15 +584,16 @@ static void collect_all_aliases(struct perf_stat_config *config, struct evsel *c
+
+ alias = list_prepare_entry(counter, &(evlist->core.entries), core.node);
+ list_for_each_entry_continue (alias, &evlist->core.entries, core.node) {
+- if (strcmp(evsel__name(alias), evsel__name(counter)) ||
+- alias->scale != counter->scale ||
+- alias->cgrp != counter->cgrp ||
+- strcmp(alias->unit, counter->unit) ||
+- evsel__is_clock(alias) != evsel__is_clock(counter) ||
+- !strcmp(alias->pmu_name, counter->pmu_name))
+- break;
+- alias->merged_stat = true;
+- cb(config, alias, data, false);
++ /* Merge events with the same name, etc. but on different PMUs. */
++ if (!strcmp(evsel__name(alias), evsel__name(counter)) &&
++ alias->scale == counter->scale &&
++ alias->cgrp == counter->cgrp &&
++ !strcmp(alias->unit, counter->unit) &&
++ evsel__is_clock(alias) == evsel__is_clock(counter) &&
++ strcmp(alias->pmu_name, counter->pmu_name)) {
++ alias->merged_stat = true;
++ cb(config, alias, data, false);
++ }
+ }
+ }
+
+diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c
+index 31cd59a2b66e6..ecd377938eea8 100644
+--- a/tools/perf/util/symbol-elf.c
++++ b/tools/perf/util/symbol-elf.c
+@@ -1290,7 +1290,7 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss,
+ * For misannotated, zeroed, ASM function sizes.
+ */
+ if (nr > 0) {
+- symbols__fixup_end(&dso->symbols);
++ symbols__fixup_end(&dso->symbols, false);
+ symbols__fixup_duplicate(&dso->symbols);
+ if (kmap) {
+ /*
+diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
+index 0fc9a54107399..b1e5fd99e38af 100644
+--- a/tools/perf/util/symbol.c
++++ b/tools/perf/util/symbol.c
+@@ -101,11 +101,6 @@ static int prefix_underscores_count(const char *str)
+ return tail - str;
+ }
+
+-void __weak arch__symbols__fixup_end(struct symbol *p, struct symbol *c)
+-{
+- p->end = c->start;
+-}
+-
+ const char * __weak arch__normalize_symbol_name(const char *name)
+ {
+ return name;
+@@ -217,7 +212,8 @@ again:
+ }
+ }
+
+-void symbols__fixup_end(struct rb_root_cached *symbols)
++/* Update zero-sized symbols using the address of the next symbol */
++void symbols__fixup_end(struct rb_root_cached *symbols, bool is_kallsyms)
+ {
+ struct rb_node *nd, *prevnd = rb_first_cached(symbols);
+ struct symbol *curr, *prev;
+@@ -231,8 +227,29 @@ void symbols__fixup_end(struct rb_root_cached *symbols)
+ prev = curr;
+ curr = rb_entry(nd, struct symbol, rb_node);
+
+- if (prev->end == prev->start && prev->end != curr->start)
+- arch__symbols__fixup_end(prev, curr);
++ /*
++ * On some architecture kernel text segment start is located at
++ * some low memory address, while modules are located at high
++ * memory addresses (or vice versa). The gap between end of
++ * kernel text segment and beginning of first module's text
++ * segment is very big. Therefore do not fill this gap and do
++ * not assign it to the kernel dso map (kallsyms).
++ *
++ * In kallsyms, it determines module symbols using '[' character
++ * like in:
++ * ffffffffc1937000 T hdmi_driver_init [snd_hda_codec_hdmi]
++ */
++ if (prev->end == prev->start) {
++ /* Last kernel/module symbol mapped to end of page */
++ if (is_kallsyms && (!strchr(prev->name, '[') !=
++ !strchr(curr->name, '[')))
++ prev->end = roundup(prev->end + 4096, 4096);
++ else
++ prev->end = curr->start;
++
++ pr_debug4("%s sym:%s end:%#" PRIx64 "\n",
++ __func__, prev->name, prev->end);
++ }
+ }
+
+ /* Last entry */
+@@ -1456,7 +1473,7 @@ int __dso__load_kallsyms(struct dso *dso, const char *filename,
+ if (kallsyms__delta(kmap, filename, &delta))
+ return -1;
+
+- symbols__fixup_end(&dso->symbols);
++ symbols__fixup_end(&dso->symbols, true);
+ symbols__fixup_duplicate(&dso->symbols);
+
+ if (dso->kernel == DSO_SPACE__KERNEL_GUEST)
+@@ -1648,7 +1665,7 @@ int dso__load_bfd_symbols(struct dso *dso, const char *debugfile)
+ #undef bfd_asymbol_section
+ #endif
+
+- symbols__fixup_end(&dso->symbols);
++ symbols__fixup_end(&dso->symbols, false);
+ symbols__fixup_duplicate(&dso->symbols);
+ dso->adjust_symbols = 1;
+
+diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
+index 954d6a049ee23..28721d761d91e 100644
+--- a/tools/perf/util/symbol.h
++++ b/tools/perf/util/symbol.h
+@@ -192,7 +192,7 @@ void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym,
+ bool kernel);
+ void symbols__insert(struct rb_root_cached *symbols, struct symbol *sym);
+ void symbols__fixup_duplicate(struct rb_root_cached *symbols);
+-void symbols__fixup_end(struct rb_root_cached *symbols);
++void symbols__fixup_end(struct rb_root_cached *symbols, bool is_kallsyms);
+ void maps__fixup_end(struct maps *maps);
+
+ typedef int (*mapfn_t)(u64 start, u64 len, u64 pgoff, void *data);
+@@ -230,7 +230,6 @@ const char *arch__normalize_symbol_name(const char *name);
+ #define SYMBOL_A 0
+ #define SYMBOL_B 1
+
+-void arch__symbols__fixup_end(struct symbol *p, struct symbol *c);
+ int arch__compare_symbol_names(const char *namea, const char *nameb);
+ int arch__compare_symbol_names_n(const char *namea, const char *nameb,
+ unsigned int n);
+diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
+index 37a9492edb3eb..df3c4671be72a 100644
+--- a/tools/perf/util/util.c
++++ b/tools/perf/util/util.c
+@@ -379,32 +379,32 @@ fetch_kernel_version(unsigned int *puint, char *str,
+ return 0;
+ }
+
+-const char *perf_tip(const char *dirpath)
++int perf_tip(char **strp, const char *dirpath)
+ {
+ struct strlist *tips;
+ struct str_node *node;
+- char *tip = NULL;
+ struct strlist_config conf = {
+ .dirname = dirpath,
+ .file_only = true,
+ };
++ int ret = 0;
+
++ *strp = NULL;
+ tips = strlist__new("tips.txt", &conf);
+ if (tips == NULL)
+- return errno == ENOENT ? NULL :
+- "Tip: check path of tips.txt or get more memory! ;-p";
++ return -errno;
+
+ if (strlist__nr_entries(tips) == 0)
+ goto out;
+
+ node = strlist__entry(tips, random() % strlist__nr_entries(tips));
+- if (asprintf(&tip, "Tip: %s", node->s) < 0)
+- tip = (char *)"Tip: get more memory! ;-)";
++ if (asprintf(strp, "Tip: %s", node->s) < 0)
++ ret = -ENOMEM;
+
+ out:
+ strlist__delete(tips);
+
+- return tip;
++ return ret;
+ }
+
+ char *perf_exe(char *buf, int len)
+diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
+index ad737052e5977..9f0d36ba77f2d 100644
+--- a/tools/perf/util/util.h
++++ b/tools/perf/util/util.h
+@@ -39,7 +39,7 @@ int fetch_kernel_version(unsigned int *puint,
+ #define KVER_FMT "%d.%d.%d"
+ #define KVER_PARAM(x) KVER_VERSION(x), KVER_PATCHLEVEL(x), KVER_SUBLEVEL(x)
+
+-const char *perf_tip(const char *dirpath);
++int perf_tip(char **strp, const char *dirpath);
+
+ #ifndef HAVE_SCHED_GETCPU_SUPPORT
+ int sched_getcpu(void);
+diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
+index 2c6f916ccbafa..0874e512d109b 100644
+--- a/tools/testing/kunit/kunit_kernel.py
++++ b/tools/testing/kunit/kunit_kernel.py
+@@ -6,6 +6,7 @@
+ # Author: Felix Guo <felixguoxiuping@gmail.com>
+ # Author: Brendan Higgins <brendanhiggins@google.com>
+
++import importlib.abc
+ import importlib.util
+ import logging
+ import subprocess
+diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c
+index b692e6ead9b55..0a4ad7cb2c200 100644
+--- a/tools/testing/selftests/bpf/btf_helpers.c
++++ b/tools/testing/selftests/bpf/btf_helpers.c
+@@ -246,18 +246,23 @@ const char *btf_type_c_dump(const struct btf *btf)
+ d = btf_dump__new(btf, NULL, &opts, btf_dump_printf);
+ if (libbpf_get_error(d)) {
+ fprintf(stderr, "Failed to create btf_dump instance: %ld\n", libbpf_get_error(d));
+- return NULL;
++ goto err_out;
+ }
+
+ for (i = 1; i <= btf__get_nr_types(btf); i++) {
+ err = btf_dump__dump_type(d, i);
+ if (err) {
+ fprintf(stderr, "Failed to dump type [%d]: %d\n", i, err);
+- return NULL;
++ goto err_out;
+ }
+ }
+
++ btf_dump__free(d);
+ fflush(buf_file);
+ fclose(buf_file);
+ return buf;
++err_out:
++ btf_dump__free(d);
++ fclose(buf_file);
++ return NULL;
+ }
+diff --git a/tools/testing/selftests/bpf/prog_tests/bind_perm.c b/tools/testing/selftests/bpf/prog_tests/bind_perm.c
+index d0f06e40c16d0..eac71fbb24ce2 100644
+--- a/tools/testing/selftests/bpf/prog_tests/bind_perm.c
++++ b/tools/testing/selftests/bpf/prog_tests/bind_perm.c
+@@ -1,13 +1,24 @@
+ // SPDX-License-Identifier: GPL-2.0
+-#include <test_progs.h>
+-#include "bind_perm.skel.h"
+-
++#define _GNU_SOURCE
++#include <sched.h>
++#include <stdlib.h>
+ #include <sys/types.h>
+ #include <sys/socket.h>
+ #include <sys/capability.h>
+
++#include "test_progs.h"
++#include "bind_perm.skel.h"
++
+ static int duration;
+
++static int create_netns(void)
++{
++ if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns"))
++ return -1;
++
++ return 0;
++}
++
+ void try_bind(int family, int port, int expected_errno)
+ {
+ struct sockaddr_storage addr = {};
+@@ -75,6 +86,9 @@ void test_bind_perm(void)
+ struct bind_perm *skel;
+ int cgroup_fd;
+
++ if (create_netns())
++ return;
++
+ cgroup_fd = test__join_cgroup("/bind_perm");
+ if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno))
+ return;
+diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
+index 77ac24b191d4c..dc18e5ae0febc 100644
+--- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
++++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
+@@ -1208,13 +1208,14 @@ static void test_task_vma(void)
+ goto out;
+
+ /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
+- * to trigger seq_file corner cases. The expected output is much
+- * longer than 1kB, so the while loop will terminate.
++ * to trigger seq_file corner cases.
+ */
+ len = 0;
+ while (len < CMP_BUFFER_SIZE) {
+ err = read_fd_into_buffer(iter_fd, task_vma_output + len,
+ min(read_size, CMP_BUFFER_SIZE - len));
++ if (!err)
++ break;
+ if (CHECK(err < 0, "read_iter_fd", "read_iter_fd failed\n"))
+ goto out;
+ len += err;
+diff --git a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
+index 762f6a9da8b5e..664ffc0364f4f 100644
+--- a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
++++ b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
+@@ -90,7 +90,7 @@ static void print_err_line(void)
+
+ static void test_conn(void)
+ {
+- int listen_fd = -1, cli_fd = -1, err;
++ int listen_fd = -1, cli_fd = -1, srv_fd = -1, err;
+ socklen_t addrlen = sizeof(srv_sa6);
+ int srv_port;
+
+@@ -112,6 +112,10 @@ static void test_conn(void)
+ if (CHECK_FAIL(cli_fd == -1))
+ goto done;
+
++ srv_fd = accept(listen_fd, NULL, NULL);
++ if (CHECK_FAIL(srv_fd == -1))
++ goto done;
++
+ if (CHECK(skel->bss->listen_tp_sport != srv_port ||
+ skel->bss->req_sk_sport != srv_port,
+ "Unexpected sk src port",
+@@ -134,11 +138,13 @@ done:
+ close(listen_fd);
+ if (cli_fd != -1)
+ close(cli_fd);
++ if (srv_fd != -1)
++ close(srv_fd);
+ }
+
+ static void test_syncookie(void)
+ {
+- int listen_fd = -1, cli_fd = -1, err;
++ int listen_fd = -1, cli_fd = -1, srv_fd = -1, err;
+ socklen_t addrlen = sizeof(srv_sa6);
+ int srv_port;
+
+@@ -161,6 +167,10 @@ static void test_syncookie(void)
+ if (CHECK_FAIL(cli_fd == -1))
+ goto done;
+
++ srv_fd = accept(listen_fd, NULL, NULL);
++ if (CHECK_FAIL(srv_fd == -1))
++ goto done;
++
+ if (CHECK(skel->bss->listen_tp_sport != srv_port,
+ "Unexpected tp src port",
+ "listen_tp_sport:%u expected:%u\n",
+@@ -188,6 +198,8 @@ done:
+ close(listen_fd);
+ if (cli_fd != -1)
+ close(cli_fd);
++ if (srv_fd != -1)
++ close(srv_fd);
+ }
+
+ struct test {
+diff --git a/tools/testing/selftests/bpf/prog_tests/ksyms_btf.c b/tools/testing/selftests/bpf/prog_tests/ksyms_btf.c
+index cf3acfa5a91d5..69455fe90ac3e 100644
+--- a/tools/testing/selftests/bpf/prog_tests/ksyms_btf.c
++++ b/tools/testing/selftests/bpf/prog_tests/ksyms_btf.c
+@@ -7,6 +7,7 @@
+ #include "test_ksyms_btf.skel.h"
+ #include "test_ksyms_btf_null_check.skel.h"
+ #include "test_ksyms_weak.skel.h"
++#include "test_ksyms_btf_write_check.skel.h"
+
+ static int duration;
+
+@@ -109,6 +110,16 @@ cleanup:
+ test_ksyms_weak__destroy(skel);
+ }
+
++static void test_write_check(void)
++{
++ struct test_ksyms_btf_write_check *skel;
++
++ skel = test_ksyms_btf_write_check__open_and_load();
++ ASSERT_ERR_PTR(skel, "unexpected load of a prog writing to ksym memory\n");
++
++ test_ksyms_btf_write_check__destroy(skel);
++}
++
+ void test_ksyms_btf(void)
+ {
+ int percpu_datasec;
+@@ -136,4 +147,7 @@ void test_ksyms_btf(void)
+
+ if (test__start_subtest("weak_ksyms"))
+ test_weak_syms();
++
++ if (test__start_subtest("write_check"))
++ test_write_check();
+ }
+diff --git a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c
+index 59adb4715394f..3c85247f96f95 100644
+--- a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c
++++ b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c
+@@ -204,8 +204,8 @@ static int pass_ack(struct migrate_reuseport_test_case *test_case)
+ {
+ int err;
+
+- err = bpf_link__detach(test_case->link);
+- if (!ASSERT_OK(err, "bpf_link__detach"))
++ err = bpf_link__destroy(test_case->link);
++ if (!ASSERT_OK(err, "bpf_link__destroy"))
+ return -1;
+
+ test_case->link = NULL;
+diff --git a/tools/testing/selftests/bpf/prog_tests/perf_buffer.c b/tools/testing/selftests/bpf/prog_tests/perf_buffer.c
+index 6490e9673002f..7daaaab13681b 100644
+--- a/tools/testing/selftests/bpf/prog_tests/perf_buffer.c
++++ b/tools/testing/selftests/bpf/prog_tests/perf_buffer.c
+@@ -107,8 +107,8 @@ void test_perf_buffer(void)
+ "expect %d, seen %d\n", nr_on_cpus, CPU_COUNT(&cpu_seen)))
+ goto out_free_pb;
+
+- if (CHECK(perf_buffer__buffer_cnt(pb) != nr_cpus, "buf_cnt",
+- "got %zu, expected %d\n", perf_buffer__buffer_cnt(pb), nr_cpus))
++ if (CHECK(perf_buffer__buffer_cnt(pb) != nr_on_cpus, "buf_cnt",
++ "got %zu, expected %d\n", perf_buffer__buffer_cnt(pb), nr_on_cpus))
+ goto out_close;
+
+ for (i = 0; i < nr_cpus; i++) {
+diff --git a/tools/testing/selftests/bpf/prog_tests/sk_lookup.c b/tools/testing/selftests/bpf/prog_tests/sk_lookup.c
+index aee41547e7f45..6db07401bc493 100644
+--- a/tools/testing/selftests/bpf/prog_tests/sk_lookup.c
++++ b/tools/testing/selftests/bpf/prog_tests/sk_lookup.c
+@@ -598,7 +598,7 @@ close:
+
+ static void run_lookup_prog(const struct test *t)
+ {
+- int server_fds[MAX_SERVERS] = { -1 };
++ int server_fds[] = { [0 ... MAX_SERVERS - 1] = -1 };
+ int client_fd, reuse_conn_fd = -1;
+ struct bpf_link *lookup_link;
+ int i, err;
+@@ -1053,7 +1053,7 @@ static void run_sk_assign(struct test_sk_lookup *skel,
+ struct bpf_program *lookup_prog,
+ const char *remote_ip, const char *local_ip)
+ {
+- int server_fds[MAX_SERVERS] = { -1 };
++ int server_fds[] = { [0 ... MAX_SERVERS - 1] = -1 };
+ struct bpf_sk_lookup ctx;
+ __u64 server_cookie;
+ int i, err;
+diff --git a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
+index fafeddaad6a99..23915be6172d6 100644
+--- a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
++++ b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
+@@ -105,4 +105,6 @@ void test_skb_ctx(void)
+ "ctx_out_mark",
+ "skb->mark == %u, expected %d\n",
+ skb.mark, 10);
++
++ bpf_object__close(obj);
+ }
+diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
+index e7201ba29ccd6..47e3159729d21 100644
+--- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
++++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
+@@ -105,6 +105,13 @@ static int setns_by_fd(int nsfd)
+ if (!ASSERT_OK(err, "unshare"))
+ return err;
+
++ /* Make our /sys mount private, so the following umount won't
++ * trigger the global umount in case it's shared.
++ */
++ err = mount("none", "/sys", NULL, MS_PRIVATE, NULL);
++ if (!ASSERT_OK(err, "remount private /sys"))
++ return err;
++
+ err = umount2("/sys", MNT_DETACH);
+ if (!ASSERT_OK(err, "umount2 /sys"))
+ return err;
+diff --git a/tools/testing/selftests/bpf/prog_tests/test_ima.c b/tools/testing/selftests/bpf/prog_tests/test_ima.c
+index 0252f61d611a9..97d8a6f84f4ab 100644
+--- a/tools/testing/selftests/bpf/prog_tests/test_ima.c
++++ b/tools/testing/selftests/bpf/prog_tests/test_ima.c
+@@ -43,7 +43,7 @@ static int process_sample(void *ctx, void *data, size_t len)
+ void test_test_ima(void)
+ {
+ char measured_dir_template[] = "/tmp/ima_measuredXXXXXX";
+- struct ring_buffer *ringbuf;
++ struct ring_buffer *ringbuf = NULL;
+ const char *measured_dir;
+ char cmd[256];
+
+@@ -85,5 +85,6 @@ close_clean:
+ err = system(cmd);
+ CHECK(err, "failed to run command", "%s, errno = %d\n", cmd, errno);
+ close_prog:
++ ring_buffer__free(ringbuf);
+ ima__destroy(skel);
+ }
+diff --git a/tools/testing/selftests/bpf/prog_tests/timer_crash.c b/tools/testing/selftests/bpf/prog_tests/timer_crash.c
+new file mode 100644
+index 0000000000000..f74b82305da8c
+--- /dev/null
++++ b/tools/testing/selftests/bpf/prog_tests/timer_crash.c
+@@ -0,0 +1,32 @@
++// SPDX-License-Identifier: GPL-2.0
++#include <test_progs.h>
++#include "timer_crash.skel.h"
++
++enum {
++ MODE_ARRAY,
++ MODE_HASH,
++};
++
++static void test_timer_crash_mode(int mode)
++{
++ struct timer_crash *skel;
++
++ skel = timer_crash__open_and_load();
++ if (!ASSERT_OK_PTR(skel, "timer_crash__open_and_load"))
++ return;
++ skel->bss->pid = getpid();
++ skel->bss->crash_map = mode;
++ if (!ASSERT_OK(timer_crash__attach(skel), "timer_crash__attach"))
++ goto end;
++ usleep(1);
++end:
++ timer_crash__destroy(skel);
++}
++
++void test_timer_crash(void)
++{
++ if (test__start_subtest("array"))
++ test_timer_crash_mode(MODE_ARRAY);
++ if (test__start_subtest("hash"))
++ test_timer_crash_mode(MODE_HASH);
++}
+diff --git a/tools/testing/selftests/bpf/progs/strobemeta.h b/tools/testing/selftests/bpf/progs/strobemeta.h
+index 7de534f38c3f1..60c93aee2f4ad 100644
+--- a/tools/testing/selftests/bpf/progs/strobemeta.h
++++ b/tools/testing/selftests/bpf/progs/strobemeta.h
+@@ -358,7 +358,7 @@ static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
+ void *payload)
+ {
+ void *location;
+- uint32_t len;
++ uint64_t len;
+
+ data->str_lens[idx] = 0;
+ location = calc_location(&cfg->str_locs[idx], tls_base);
+@@ -390,7 +390,7 @@ static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
+ struct strobe_map_descr* descr = &data->map_descrs[idx];
+ struct strobe_map_raw map;
+ void *location;
+- uint32_t len;
++ uint64_t len;
+ int i;
+
+ descr->tag_len = 0; /* presume no tag is set */
+diff --git a/tools/testing/selftests/bpf/progs/test_ksyms_btf_write_check.c b/tools/testing/selftests/bpf/progs/test_ksyms_btf_write_check.c
+new file mode 100644
+index 0000000000000..2180c41cd890f
+--- /dev/null
++++ b/tools/testing/selftests/bpf/progs/test_ksyms_btf_write_check.c
+@@ -0,0 +1,29 @@
++// SPDX-License-Identifier: GPL-2.0
++/* Copyright (c) 2021 Google */
++
++#include "vmlinux.h"
++
++#include <bpf/bpf_helpers.h>
++
++extern const int bpf_prog_active __ksym; /* int type global var. */
++
++SEC("raw_tp/sys_enter")
++int handler(const void *ctx)
++{
++ int *active;
++ __u32 cpu;
++
++ cpu = bpf_get_smp_processor_id();
++ active = (int *)bpf_per_cpu_ptr(&bpf_prog_active, cpu);
++ if (active) {
++ /* Kernel memory obtained from bpf_{per,this}_cpu_ptr
++ * is read-only, should _not_ pass verification.
++ */
++ /* WRITE_ONCE */
++ *(volatile int *)active = -1;
++ }
++
++ return 0;
++}
++
++char _license[] SEC("license") = "GPL";
+diff --git a/tools/testing/selftests/bpf/progs/test_map_in_map.c b/tools/testing/selftests/bpf/progs/test_map_in_map.c
+index 1cfeb940cf9fb..5f0e0bfc151e5 100644
+--- a/tools/testing/selftests/bpf/progs/test_map_in_map.c
++++ b/tools/testing/selftests/bpf/progs/test_map_in_map.c
+@@ -23,7 +23,7 @@ struct {
+ __uint(value_size, sizeof(__u32));
+ } mim_hash SEC(".maps");
+
+-SEC("xdp_mimtest")
++SEC("xdp")
+ int xdp_mimtest0(struct xdp_md *ctx)
+ {
+ int value = 123;
+diff --git a/tools/testing/selftests/bpf/progs/test_sk_lookup.c b/tools/testing/selftests/bpf/progs/test_sk_lookup.c
+index ac6f7f205e25d..cb0aa46b20d19 100644
+--- a/tools/testing/selftests/bpf/progs/test_sk_lookup.c
++++ b/tools/testing/selftests/bpf/progs/test_sk_lookup.c
+@@ -404,8 +404,7 @@ int ctx_narrow_access(struct bpf_sk_lookup *ctx)
+
+ /* Narrow loads from remote_port field. Expect SRC_PORT. */
+ if (LSB(ctx->remote_port, 0) != ((SRC_PORT >> 0) & 0xff) ||
+- LSB(ctx->remote_port, 1) != ((SRC_PORT >> 8) & 0xff) ||
+- LSB(ctx->remote_port, 2) != 0 || LSB(ctx->remote_port, 3) != 0)
++ LSB(ctx->remote_port, 1) != ((SRC_PORT >> 8) & 0xff))
+ return SK_DROP;
+ if (LSW(ctx->remote_port, 0) != SRC_PORT)
+ return SK_DROP;
+diff --git a/tools/testing/selftests/bpf/progs/test_sock_fields.c b/tools/testing/selftests/bpf/progs/test_sock_fields.c
+index 81b57b9aaaeae..7967348b11af6 100644
+--- a/tools/testing/selftests/bpf/progs/test_sock_fields.c
++++ b/tools/testing/selftests/bpf/progs/test_sock_fields.c
+@@ -113,7 +113,7 @@ static void tpcpy(struct bpf_tcp_sock *dst,
+
+ #define RET_LOG() ({ \
+ linum = __LINE__; \
+- bpf_map_update_elem(&linum_map, &linum_idx, &linum, BPF_NOEXIST); \
++ bpf_map_update_elem(&linum_map, &linum_idx, &linum, BPF_ANY); \
+ return CG_OK; \
+ })
+
+diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_kern.h b/tools/testing/selftests/bpf/progs/test_sockmap_kern.h
+index 1858435de7aaf..5cb90ca292186 100644
+--- a/tools/testing/selftests/bpf/progs/test_sockmap_kern.h
++++ b/tools/testing/selftests/bpf/progs/test_sockmap_kern.h
+@@ -235,7 +235,7 @@ SEC("sk_msg1")
+ int bpf_prog4(struct sk_msg_md *msg)
+ {
+ int *bytes, zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
+- int *start, *end, *start_push, *end_push, *start_pop, *pop;
++ int *start, *end, *start_push, *end_push, *start_pop, *pop, err = 0;
+
+ bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
+ if (bytes)
+@@ -249,8 +249,11 @@ int bpf_prog4(struct sk_msg_md *msg)
+ bpf_msg_pull_data(msg, *start, *end, 0);
+ start_push = bpf_map_lookup_elem(&sock_bytes, &two);
+ end_push = bpf_map_lookup_elem(&sock_bytes, &three);
+- if (start_push && end_push)
+- bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (start_push && end_push) {
++ err = bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (err)
++ return SK_DROP;
++ }
+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
+ if (start_pop && pop)
+@@ -263,6 +266,7 @@ int bpf_prog6(struct sk_msg_md *msg)
+ {
+ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, key = 0;
+ int *bytes, *start, *end, *start_push, *end_push, *start_pop, *pop, *f;
++ int err = 0;
+ __u64 flags = 0;
+
+ bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
+@@ -279,8 +283,11 @@ int bpf_prog6(struct sk_msg_md *msg)
+
+ start_push = bpf_map_lookup_elem(&sock_bytes, &two);
+ end_push = bpf_map_lookup_elem(&sock_bytes, &three);
+- if (start_push && end_push)
+- bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (start_push && end_push) {
++ err = bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (err)
++ return SK_DROP;
++ }
+
+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
+@@ -338,7 +345,7 @@ SEC("sk_msg5")
+ int bpf_prog10(struct sk_msg_md *msg)
+ {
+ int *bytes, *start, *end, *start_push, *end_push, *start_pop, *pop;
+- int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
++ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, err = 0;
+
+ bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
+ if (bytes)
+@@ -352,8 +359,11 @@ int bpf_prog10(struct sk_msg_md *msg)
+ bpf_msg_pull_data(msg, *start, *end, 0);
+ start_push = bpf_map_lookup_elem(&sock_bytes, &two);
+ end_push = bpf_map_lookup_elem(&sock_bytes, &three);
+- if (start_push && end_push)
+- bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (start_push && end_push) {
++ err = bpf_msg_push_data(msg, *start_push, *end_push, 0);
++ if (err)
++ return SK_PASS;
++ }
+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
+ if (start_pop && pop)
+diff --git a/tools/testing/selftests/bpf/progs/test_tcp_check_syncookie_kern.c b/tools/testing/selftests/bpf/progs/test_tcp_check_syncookie_kern.c
+index 47cbe2eeae431..fac7ef99f9a67 100644
+--- a/tools/testing/selftests/bpf/progs/test_tcp_check_syncookie_kern.c
++++ b/tools/testing/selftests/bpf/progs/test_tcp_check_syncookie_kern.c
+@@ -156,7 +156,7 @@ int check_syncookie_clsact(struct __sk_buff *skb)
+ return TC_ACT_OK;
+ }
+
+-SEC("xdp/check_syncookie")
++SEC("xdp")
+ int check_syncookie_xdp(struct xdp_md *ctx)
+ {
+ check_syncookie(ctx, (void *)(long)ctx->data,
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp.c b/tools/testing/selftests/bpf/progs/test_xdp.c
+index 31f9bce37491a..e6aa2fc6ce6bd 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp.c
+@@ -210,7 +210,7 @@ static __always_inline int handle_ipv6(struct xdp_md *xdp)
+ return XDP_TX;
+ }
+
+-SEC("xdp_tx_iptunnel")
++SEC("xdp")
+ int _xdp_tx_iptunnel(struct xdp_md *xdp)
+ {
+ void *data_end = (void *)(long)xdp->data_end;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_grow.c b/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_grow.c
+index 3d66599eee2ec..199c61b7d0628 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_grow.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_grow.c
+@@ -2,7 +2,7 @@
+ #include <linux/bpf.h>
+ #include <bpf/bpf_helpers.h>
+
+-SEC("xdp_adjust_tail_grow")
++SEC("xdp")
+ int _xdp_adjust_tail_grow(struct xdp_md *xdp)
+ {
+ void *data_end = (void *)(long)xdp->data_end;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_shrink.c b/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_shrink.c
+index 22065a9cfb254..b7448253d1359 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_shrink.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_adjust_tail_shrink.c
+@@ -9,9 +9,7 @@
+ #include <linux/if_ether.h>
+ #include <bpf/bpf_helpers.h>
+
+-int _version SEC("version") = 1;
+-
+-SEC("xdp_adjust_tail_shrink")
++SEC("xdp")
+ int _xdp_adjust_tail_shrink(struct xdp_md *xdp)
+ {
+ void *data_end = (void *)(long)xdp->data_end;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_devmap_helpers.c b/tools/testing/selftests/bpf/progs/test_xdp_devmap_helpers.c
+index b360ba2bd4411..807bf895f42ca 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_devmap_helpers.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_devmap_helpers.c
+@@ -5,7 +5,7 @@
+ #include <linux/bpf.h>
+ #include <bpf/bpf_helpers.h>
+
+-SEC("xdp_dm_log")
++SEC("xdp")
+ int xdpdm_devlog(struct xdp_md *ctx)
+ {
+ char fmt[] = "devmap redirect: dev %u -> dev %u len %u\n";
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_link.c b/tools/testing/selftests/bpf/progs/test_xdp_link.c
+index eb93ea95d1d8f..ee7d6ac0f6151 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_link.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_link.c
+@@ -5,7 +5,7 @@
+
+ char LICENSE[] SEC("license") = "GPL";
+
+-SEC("xdp/handler")
++SEC("xdp")
+ int xdp_handler(struct xdp_md *xdp)
+ {
+ return 0;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_loop.c b/tools/testing/selftests/bpf/progs/test_xdp_loop.c
+index fcabcda30ba32..27eb52dda92c2 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_loop.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_loop.c
+@@ -206,7 +206,7 @@ static __always_inline int handle_ipv6(struct xdp_md *xdp)
+ return XDP_TX;
+ }
+
+-SEC("xdp_tx_iptunnel")
++SEC("xdp")
+ int _xdp_tx_iptunnel(struct xdp_md *xdp)
+ {
+ void *data_end = (void *)(long)xdp->data_end;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_noinline.c b/tools/testing/selftests/bpf/progs/test_xdp_noinline.c
+index 3a67921f62b52..596c4e71bf3ac 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_noinline.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_noinline.c
+@@ -797,7 +797,7 @@ out:
+ return XDP_DROP;
+ }
+
+-SEC("xdp-test-v4")
++SEC("xdp")
+ int balancer_ingress_v4(struct xdp_md *ctx)
+ {
+ void *data = (void *)(long)ctx->data;
+@@ -816,7 +816,7 @@ int balancer_ingress_v4(struct xdp_md *ctx)
+ return XDP_DROP;
+ }
+
+-SEC("xdp-test-v6")
++SEC("xdp")
+ int balancer_ingress_v6(struct xdp_md *ctx)
+ {
+ void *data = (void *)(long)ctx->data;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_with_cpumap_helpers.c b/tools/testing/selftests/bpf/progs/test_xdp_with_cpumap_helpers.c
+index 59ee4f182ff80..532025057711e 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_with_cpumap_helpers.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_with_cpumap_helpers.c
+@@ -12,13 +12,13 @@ struct {
+ __uint(max_entries, 4);
+ } cpu_map SEC(".maps");
+
+-SEC("xdp_redir")
++SEC("xdp")
+ int xdp_redir_prog(struct xdp_md *ctx)
+ {
+ return bpf_redirect_map(&cpu_map, 1, 0);
+ }
+
+-SEC("xdp_dummy")
++SEC("xdp")
+ int xdp_dummy_prog(struct xdp_md *ctx)
+ {
+ return XDP_PASS;
+diff --git a/tools/testing/selftests/bpf/progs/test_xdp_with_devmap_helpers.c b/tools/testing/selftests/bpf/progs/test_xdp_with_devmap_helpers.c
+index 0ac0864977222..1e6b9c38ea6d9 100644
+--- a/tools/testing/selftests/bpf/progs/test_xdp_with_devmap_helpers.c
++++ b/tools/testing/selftests/bpf/progs/test_xdp_with_devmap_helpers.c
+@@ -9,7 +9,7 @@ struct {
+ __uint(max_entries, 4);
+ } dm_ports SEC(".maps");
+
+-SEC("xdp_redir")
++SEC("xdp")
+ int xdp_redir_prog(struct xdp_md *ctx)
+ {
+ return bpf_redirect_map(&dm_ports, 1, 0);
+@@ -18,7 +18,7 @@ int xdp_redir_prog(struct xdp_md *ctx)
+ /* invalid program on DEVMAP entry;
+ * SEC name means expected attach type not set
+ */
+-SEC("xdp_dummy")
++SEC("xdp")
+ int xdp_dummy_prog(struct xdp_md *ctx)
+ {
+ return XDP_PASS;
+diff --git a/tools/testing/selftests/bpf/progs/timer_crash.c b/tools/testing/selftests/bpf/progs/timer_crash.c
+new file mode 100644
+index 0000000000000..f8f7944e70dae
+--- /dev/null
++++ b/tools/testing/selftests/bpf/progs/timer_crash.c
+@@ -0,0 +1,54 @@
++// SPDX-License-Identifier: GPL-2.0
++
++#include <vmlinux.h>
++#include <bpf/bpf_tracing.h>
++#include <bpf/bpf_helpers.h>
++
++struct map_elem {
++ struct bpf_timer timer;
++ struct bpf_spin_lock lock;
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_ARRAY);
++ __uint(max_entries, 1);
++ __type(key, int);
++ __type(value, struct map_elem);
++} amap SEC(".maps");
++
++struct {
++ __uint(type, BPF_MAP_TYPE_HASH);
++ __uint(max_entries, 1);
++ __type(key, int);
++ __type(value, struct map_elem);
++} hmap SEC(".maps");
++
++int pid = 0;
++int crash_map = 0; /* 0 for amap, 1 for hmap */
++
++SEC("fentry/do_nanosleep")
++int sys_enter(void *ctx)
++{
++ struct map_elem *e, value = {};
++ void *map = crash_map ? (void *)&hmap : (void *)&amap;
++
++ if (bpf_get_current_task_btf()->tgid != pid)
++ return 0;
++
++ *(void **)&value = (void *)0xdeadcaf3;
++
++ bpf_map_update_elem(map, &(int){0}, &value, 0);
++ /* For array map, doing bpf_map_update_elem will do a
++ * check_and_free_timer_in_array, which will trigger the crash if timer
++ * pointer was overwritten, for hmap we need to use bpf_timer_cancel.
++ */
++ if (crash_map == 1) {
++ e = bpf_map_lookup_elem(map, &(int){0});
++ if (!e)
++ return 0;
++ bpf_timer_cancel(&e->timer);
++ }
++ return 0;
++}
++
++char _license[] SEC("license") = "GPL";
+diff --git a/tools/testing/selftests/bpf/progs/xdp_dummy.c b/tools/testing/selftests/bpf/progs/xdp_dummy.c
+index ea25e88819928..d988b2e0cee84 100644
+--- a/tools/testing/selftests/bpf/progs/xdp_dummy.c
++++ b/tools/testing/selftests/bpf/progs/xdp_dummy.c
+@@ -4,7 +4,7 @@
+ #include <linux/bpf.h>
+ #include <bpf/bpf_helpers.h>
+
+-SEC("xdp_dummy")
++SEC("xdp")
+ int xdp_dummy_prog(struct xdp_md *ctx)
+ {
+ return XDP_PASS;
+diff --git a/tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c b/tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c
+index 880debcbcd65d..8395782b6e0a3 100644
+--- a/tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c
++++ b/tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c
+@@ -34,7 +34,7 @@ struct {
+ __uint(max_entries, 128);
+ } mac_map SEC(".maps");
+
+-SEC("xdp_redirect_map_multi")
++SEC("xdp")
+ int xdp_redirect_map_multi_prog(struct xdp_md *ctx)
+ {
+ void *data_end = (void *)(long)ctx->data_end;
+@@ -63,7 +63,7 @@ int xdp_redirect_map_multi_prog(struct xdp_md *ctx)
+ }
+
+ /* The following 2 progs are for 2nd devmap prog testing */
+-SEC("xdp_redirect_map_ingress")
++SEC("xdp")
+ int xdp_redirect_map_all_prog(struct xdp_md *ctx)
+ {
+ return bpf_redirect_map(&map_egress, 0,
+diff --git a/tools/testing/selftests/bpf/progs/xdping_kern.c b/tools/testing/selftests/bpf/progs/xdping_kern.c
+index 6b9ca40bd1f4f..4ad73847b8a5d 100644
+--- a/tools/testing/selftests/bpf/progs/xdping_kern.c
++++ b/tools/testing/selftests/bpf/progs/xdping_kern.c
+@@ -86,7 +86,7 @@ static __always_inline int icmp_check(struct xdp_md *ctx, int type)
+ return XDP_TX;
+ }
+
+-SEC("xdpclient")
++SEC("xdp")
+ int xdping_client(struct xdp_md *ctx)
+ {
+ void *data_end = (void *)(long)ctx->data_end;
+@@ -150,7 +150,7 @@ int xdping_client(struct xdp_md *ctx)
+ return XDP_TX;
+ }
+
+-SEC("xdpserver")
++SEC("xdp")
+ int xdping_server(struct xdp_md *ctx)
+ {
+ void *data_end = (void *)(long)ctx->data_end;
+diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh
+index ec4e15948e406..5252b91f48a18 100755
+--- a/tools/testing/selftests/bpf/test_lirc_mode2.sh
++++ b/tools/testing/selftests/bpf/test_lirc_mode2.sh
+@@ -3,6 +3,7 @@
+
+ # Kselftest framework requirement - SKIP code is 4.
+ ksft_skip=4
++ret=$ksft_skip
+
+ msg="skip all tests:"
+ if [ $UID != 0 ]; then
+@@ -25,7 +26,7 @@ do
+ fi
+ done
+
+-if [ -n $LIRCDEV ];
++if [ -n "$LIRCDEV" ];
+ then
+ TYPE=lirc_mode2
+ ./test_lirc_mode2_user $LIRCDEV $INPUTDEV
+@@ -36,3 +37,5 @@ then
+ echo -e ${GREEN}"PASS: $TYPE"${NC}
+ fi
+ fi
++
++exit $ret
+diff --git a/tools/testing/selftests/bpf/test_lwt_ip_encap.sh b/tools/testing/selftests/bpf/test_lwt_ip_encap.sh
+index b497bb85b667f..6c69c42b1d607 100755
+--- a/tools/testing/selftests/bpf/test_lwt_ip_encap.sh
++++ b/tools/testing/selftests/bpf/test_lwt_ip_encap.sh
+@@ -120,6 +120,14 @@ setup()
+ ip netns exec ${NS2} sysctl -wq net.ipv4.conf.default.rp_filter=0
+ ip netns exec ${NS3} sysctl -wq net.ipv4.conf.default.rp_filter=0
+
++ # disable IPv6 DAD because it sometimes takes too long and fails tests
++ ip netns exec ${NS1} sysctl -wq net.ipv6.conf.all.accept_dad=0
++ ip netns exec ${NS2} sysctl -wq net.ipv6.conf.all.accept_dad=0
++ ip netns exec ${NS3} sysctl -wq net.ipv6.conf.all.accept_dad=0
++ ip netns exec ${NS1} sysctl -wq net.ipv6.conf.default.accept_dad=0
++ ip netns exec ${NS2} sysctl -wq net.ipv6.conf.default.accept_dad=0
++ ip netns exec ${NS3} sysctl -wq net.ipv6.conf.default.accept_dad=0
++
+ ip link add veth1 type veth peer name veth2
+ ip link add veth3 type veth peer name veth4
+ ip link add veth5 type veth peer name veth6
+@@ -289,7 +297,7 @@ test_ping()
+ ip netns exec ${NS1} ping -c 1 -W 1 -I veth1 ${IPv4_DST} 2>&1 > /dev/null
+ RET=$?
+ elif [ "${PROTO}" == "IPv6" ] ; then
+- ip netns exec ${NS1} ping6 -c 1 -W 6 -I veth1 ${IPv6_DST} 2>&1 > /dev/null
++ ip netns exec ${NS1} ping6 -c 1 -W 1 -I veth1 ${IPv6_DST} 2>&1 > /dev/null
+ RET=$?
+ else
+ echo " test_ping: unknown PROTO: ${PROTO}"
+diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
+index cc1cd240445d2..e3fea6f281e4b 100644
+--- a/tools/testing/selftests/bpf/test_progs.c
++++ b/tools/testing/selftests/bpf/test_progs.c
+@@ -370,7 +370,7 @@ int extract_build_id(char *build_id, size_t size)
+
+ if (getline(&line, &len, fp) == -1)
+ goto err;
+- fclose(fp);
++ pclose(fp);
+
+ if (len > size)
+ len = size;
+@@ -379,7 +379,7 @@ int extract_build_id(char *build_id, size_t size)
+ free(line);
+ return 0;
+ err:
+- fclose(fp);
++ pclose(fp);
+ return -1;
+ }
+
+diff --git a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
+index 9b3617d770a52..fed765157c53c 100755
+--- a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
++++ b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
+@@ -77,7 +77,7 @@ TEST_IF=lo
+ MAX_PING_TRIES=5
+ BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.o"
+ CLSACT_SECTION="clsact/check_syncookie"
+-XDP_SECTION="xdp/check_syncookie"
++XDP_SECTION="xdp"
+ BPF_PROG_ID=0
+ PROG="${DIR}/test_tcp_check_syncookie_user"
+
+diff --git a/tools/testing/selftests/bpf/test_xdp_redirect.sh b/tools/testing/selftests/bpf/test_xdp_redirect.sh
+index c033850886f44..57c8db9972a65 100755
+--- a/tools/testing/selftests/bpf/test_xdp_redirect.sh
++++ b/tools/testing/selftests/bpf/test_xdp_redirect.sh
+@@ -52,8 +52,8 @@ test_xdp_redirect()
+ return 0
+ fi
+
+- ip -n ns1 link set veth11 $xdpmode obj xdp_dummy.o sec xdp_dummy &> /dev/null
+- ip -n ns2 link set veth22 $xdpmode obj xdp_dummy.o sec xdp_dummy &> /dev/null
++ ip -n ns1 link set veth11 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null
++ ip -n ns2 link set veth22 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null
+ ip link set dev veth1 $xdpmode obj test_xdp_redirect.o sec redirect_to_222 &> /dev/null
+ ip link set dev veth2 $xdpmode obj test_xdp_redirect.o sec redirect_to_111 &> /dev/null
+
+diff --git a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
+index 1538373157e3c..cc57cb87e65f6 100755
+--- a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
++++ b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
+@@ -2,11 +2,11 @@
+ # SPDX-License-Identifier: GPL-2.0
+ #
+ # Test topology:
+-# - - - - - - - - - - - - - - - - - - - - - - - - -
+-# | veth1 veth2 veth3 | ... init net
++# - - - - - - - - - - - - - - - - - - -
++# | veth1 veth2 veth3 | ns0
+ # - -| - - - - - - | - - - - - - | - -
+ # --------- --------- ---------
+-# | veth0 | | veth0 | | veth0 | ...
++# | veth0 | | veth0 | | veth0 |
+ # --------- --------- ---------
+ # ns1 ns2 ns3
+ #
+@@ -31,6 +31,12 @@ IFACES=""
+ DRV_MODE="xdpgeneric xdpdrv xdpegress"
+ PASS=0
+ FAIL=0
++LOG_DIR=$(mktemp -d)
++declare -a NS
++NS[0]="ns0-$(mktemp -u XXXXXX)"
++NS[1]="ns1-$(mktemp -u XXXXXX)"
++NS[2]="ns2-$(mktemp -u XXXXXX)"
++NS[3]="ns3-$(mktemp -u XXXXXX)"
+
+ test_pass()
+ {
+@@ -46,9 +52,8 @@ test_fail()
+
+ clean_up()
+ {
+- for i in $(seq $NUM); do
+- ip link del veth$i 2> /dev/null
+- ip netns del ns$i 2> /dev/null
++ for i in $(seq 0 $NUM); do
++ ip netns del ${NS[$i]} 2> /dev/null
+ done
+ }
+
+@@ -77,21 +82,22 @@ setup_ns()
+ mode="xdpdrv"
+ fi
+
++ ip netns add ${NS[0]}
+ for i in $(seq $NUM); do
+- ip netns add ns$i
+- ip link add veth$i type veth peer name veth0 netns ns$i
+- ip link set veth$i up
+- ip -n ns$i link set veth0 up
++ ip netns add ${NS[$i]}
++ ip -n ${NS[$i]} link add veth0 type veth peer name veth$i netns ${NS[0]}
++ ip -n ${NS[$i]} link set veth0 up
++ ip -n ${NS[0]} link set veth$i up
+
+- ip -n ns$i addr add 192.0.2.$i/24 dev veth0
+- ip -n ns$i addr add 2001:db8::$i/64 dev veth0
++ ip -n ${NS[$i]} addr add 192.0.2.$i/24 dev veth0
++ ip -n ${NS[$i]} addr add 2001:db8::$i/64 dev veth0
+ # Add a neigh entry for IPv4 ping test
+- ip -n ns$i neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
+- ip -n ns$i link set veth0 $mode obj \
+- xdp_dummy.o sec xdp_dummy &> /dev/null || \
++ ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
++ ip -n ${NS[$i]} link set veth0 $mode obj \
++ xdp_dummy.o sec xdp &> /dev/null || \
+ { test_fail "Unable to load dummy xdp" && exit 1; }
+ IFACES="$IFACES veth$i"
+- veth_mac[$i]=$(ip link show veth$i | awk '/link\/ether/ {print $2}')
++ veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i | awk '/link\/ether/ {print $2}')
+ done
+ }
+
+@@ -100,17 +106,17 @@ do_egress_tests()
+ local mode=$1
+
+ # mac test
+- ip netns exec ns2 tcpdump -e -i veth0 -nn -l -e &> mac_ns1-2_${mode}.log &
+- ip netns exec ns3 tcpdump -e -i veth0 -nn -l -e &> mac_ns1-3_${mode}.log &
++ ip netns exec ${NS[2]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-2_${mode}.log &
++ ip netns exec ${NS[3]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-3_${mode}.log &
+ sleep 0.5
+- ip netns exec ns1 ping 192.0.2.254 -i 0.1 -c 4 &> /dev/null
++ ip netns exec ${NS[1]} ping 192.0.2.254 -i 0.1 -c 4 &> /dev/null
+ sleep 0.5
+- pkill -9 tcpdump
++ pkill tcpdump
+
+ # mac check
+- grep -q "${veth_mac[2]} > ff:ff:ff:ff:ff:ff" mac_ns1-2_${mode}.log && \
++ grep -q "${veth_mac[2]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-2_${mode}.log && \
+ test_pass "$mode mac ns1-2" || test_fail "$mode mac ns1-2"
+- grep -q "${veth_mac[3]} > ff:ff:ff:ff:ff:ff" mac_ns1-3_${mode}.log && \
++ grep -q "${veth_mac[3]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-3_${mode}.log && \
+ test_pass "$mode mac ns1-3" || test_fail "$mode mac ns1-3"
+ }
+
+@@ -119,48 +125,48 @@ do_ping_tests()
+ local mode=$1
+
+ # ping6 test: echo request should be redirect back to itself, not others
+- ip netns exec ns1 ip neigh add 2001:db8::2 dev veth0 lladdr 00:00:00:00:00:02
++ ip netns exec ${NS[1]} ip neigh add 2001:db8::2 dev veth0 lladdr 00:00:00:00:00:02
+
+- ip netns exec ns1 tcpdump -i veth0 -nn -l -e &> ns1-1_${mode}.log &
+- ip netns exec ns2 tcpdump -i veth0 -nn -l -e &> ns1-2_${mode}.log &
+- ip netns exec ns3 tcpdump -i veth0 -nn -l -e &> ns1-3_${mode}.log &
++ ip netns exec ${NS[1]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-1_${mode}.log &
++ ip netns exec ${NS[2]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-2_${mode}.log &
++ ip netns exec ${NS[3]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-3_${mode}.log &
+ sleep 0.5
+ # ARP test
+- ip netns exec ns1 ping 192.0.2.254 -i 0.1 -c 4 &> /dev/null
++ ip netns exec ${NS[1]} arping -q -c 2 -I veth0 192.0.2.254
+ # IPv4 test
+- ip netns exec ns1 ping 192.0.2.253 -i 0.1 -c 4 &> /dev/null
++ ip netns exec ${NS[1]} ping 192.0.2.253 -i 0.1 -c 4 &> /dev/null
+ # IPv6 test
+- ip netns exec ns1 ping6 2001:db8::2 -i 0.1 -c 2 &> /dev/null
++ ip netns exec ${NS[1]} ping6 2001:db8::2 -i 0.1 -c 2 &> /dev/null
+ sleep 0.5
+- pkill -9 tcpdump
++ pkill tcpdump
+
+ # All netns should receive the redirect arp requests
+- [ $(grep -c "who-has 192.0.2.254" ns1-1_${mode}.log) -gt 4 ] && \
++ [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
+ test_pass "$mode arp(F_BROADCAST) ns1-1" || \
+ test_fail "$mode arp(F_BROADCAST) ns1-1"
+- [ $(grep -c "who-has 192.0.2.254" ns1-2_${mode}.log) -le 4 ] && \
++ [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-2_${mode}.log) -eq 2 ] && \
+ test_pass "$mode arp(F_BROADCAST) ns1-2" || \
+ test_fail "$mode arp(F_BROADCAST) ns1-2"
+- [ $(grep -c "who-has 192.0.2.254" ns1-3_${mode}.log) -le 4 ] && \
++ [ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-3_${mode}.log) -eq 2 ] && \
+ test_pass "$mode arp(F_BROADCAST) ns1-3" || \
+ test_fail "$mode arp(F_BROADCAST) ns1-3"
+
+ # ns1 should not receive the redirect echo request, others should
+- [ $(grep -c "ICMP echo request" ns1-1_${mode}.log) -eq 4 ] && \
++ [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
+ test_pass "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-1" || \
+ test_fail "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-1"
+- [ $(grep -c "ICMP echo request" ns1-2_${mode}.log) -eq 4 ] && \
++ [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 4 ] && \
+ test_pass "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-2" || \
+ test_fail "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-2"
+- [ $(grep -c "ICMP echo request" ns1-3_${mode}.log) -eq 4 ] && \
++ [ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-3_${mode}.log) -eq 4 ] && \
+ test_pass "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-3" || \
+ test_fail "$mode IPv4 (F_BROADCAST|F_EXCLUDE_INGRESS) ns1-3"
+
+ # ns1 should receive the echo request, ns2 should not
+- [ $(grep -c "ICMP6, echo request" ns1-1_${mode}.log) -eq 4 ] && \
++ [ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
+ test_pass "$mode IPv6 (no flags) ns1-1" || \
+ test_fail "$mode IPv6 (no flags) ns1-1"
+- [ $(grep -c "ICMP6, echo request" ns1-2_${mode}.log) -eq 0 ] && \
++ [ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 0 ] && \
+ test_pass "$mode IPv6 (no flags) ns1-2" || \
+ test_fail "$mode IPv6 (no flags) ns1-2"
+ }
+@@ -176,9 +182,13 @@ do_tests()
+ xdpgeneric) drv_p="-S";;
+ esac
+
+- ./xdp_redirect_multi $drv_p $IFACES &> xdp_redirect_${mode}.log &
++ ip netns exec ${NS[0]} ./xdp_redirect_multi $drv_p $IFACES &> ${LOG_DIR}/xdp_redirect_${mode}.log &
+ xdp_pid=$!
+ sleep 1
++ if ! ps -p $xdp_pid > /dev/null; then
++ test_fail "$mode xdp_redirect_multi start failed"
++ return 1
++ fi
+
+ if [ "$mode" = "xdpegress" ]; then
+ do_egress_tests $mode
+@@ -189,16 +199,16 @@ do_tests()
+ kill $xdp_pid
+ }
+
+-trap clean_up 0 2 3 6 9
+-
+ check_env
+-rm -f xdp_redirect_*.log ns*.log mac_ns*.log
++
++trap clean_up EXIT
+
+ for mode in ${DRV_MODE}; do
+ setup_ns $mode
+ do_tests $mode
+ clean_up
+ done
++rm -rf ${LOG_DIR}
+
+ echo "Summary: PASS $PASS, FAIL $FAIL"
+ [ $FAIL -eq 0 ] && exit 0 || exit 1
+diff --git a/tools/testing/selftests/bpf/test_xdp_veth.sh b/tools/testing/selftests/bpf/test_xdp_veth.sh
+index 995278e684b6e..a3a1eaee26ea6 100755
+--- a/tools/testing/selftests/bpf/test_xdp_veth.sh
++++ b/tools/testing/selftests/bpf/test_xdp_veth.sh
+@@ -107,9 +107,9 @@ ip link set dev veth1 xdp pinned $BPF_DIR/progs/redirect_map_0
+ ip link set dev veth2 xdp pinned $BPF_DIR/progs/redirect_map_1
+ ip link set dev veth3 xdp pinned $BPF_DIR/progs/redirect_map_2
+
+-ip -n ns1 link set dev veth11 xdp obj xdp_dummy.o sec xdp_dummy
++ip -n ns1 link set dev veth11 xdp obj xdp_dummy.o sec xdp
+ ip -n ns2 link set dev veth22 xdp obj xdp_tx.o sec xdp
+-ip -n ns3 link set dev veth33 xdp obj xdp_dummy.o sec xdp_dummy
++ip -n ns3 link set dev veth33 xdp obj xdp_dummy.o sec xdp
+
+ trap cleanup EXIT
+
+diff --git a/tools/testing/selftests/bpf/verifier/array_access.c b/tools/testing/selftests/bpf/verifier/array_access.c
+index 1b1c798e92489..1b138cd2b187d 100644
+--- a/tools/testing/selftests/bpf/verifier/array_access.c
++++ b/tools/testing/selftests/bpf/verifier/array_access.c
+@@ -186,7 +186,7 @@
+ },
+ .fixup_map_hash_48b = { 3 },
+ .errstr_unpriv = "R0 leaks addr",
+- .errstr = "R0 unbounded memory access",
++ .errstr = "invalid access to map value, value_size=48 off=44 size=8",
+ .result_unpriv = REJECT,
+ .result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+diff --git a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c
+index 6e52dfc644153..6fb52d8cfd889 100644
+--- a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c
++++ b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c
+@@ -71,6 +71,8 @@
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
++ .result_unpriv = REJECT,
++ .errstr_unpriv = "R0 leaks addr into mem",
+ },
+ {
+ "Can't use cmpxchg on uninit src reg",
+@@ -118,4 +120,88 @@
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
++ .result_unpriv = REJECT,
++ .errstr_unpriv = "R0 leaks addr into mem",
++},
++{
++ "Dest pointer in r0 - succeed, check 2",
++ .insns = {
++ /* r0 = &val */
++ BPF_MOV64_REG(BPF_REG_0, BPF_REG_10),
++ /* val = r0; */
++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
++ /* r5 = &val */
++ BPF_MOV64_REG(BPF_REG_5, BPF_REG_10),
++ /* r0 = atomic_cmpxchg(&val, r0, r5); */
++ BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_5, -8),
++ /* r1 = *r0 */
++ BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, -8),
++ /* exit(0); */
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .result_unpriv = REJECT,
++ .errstr_unpriv = "R0 leaks addr into mem",
++},
++{
++ "Dest pointer in r0 - succeed, check 3",
++ .insns = {
++ /* r0 = &val */
++ BPF_MOV64_REG(BPF_REG_0, BPF_REG_10),
++ /* val = r0; */
++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
++ /* r5 = &val */
++ BPF_MOV64_REG(BPF_REG_5, BPF_REG_10),
++ /* r0 = atomic_cmpxchg(&val, r0, r5); */
++ BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_5, -8),
++ /* exit(0); */
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = REJECT,
++ .errstr = "invalid size of register fill",
++ .errstr_unpriv = "R0 leaks addr into mem",
++},
++{
++ "Dest pointer in r0 - succeed, check 4",
++ .insns = {
++ /* r0 = &val */
++ BPF_MOV32_REG(BPF_REG_0, BPF_REG_10),
++ /* val = r0; */
++ BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -8),
++ /* r5 = &val */
++ BPF_MOV32_REG(BPF_REG_5, BPF_REG_10),
++ /* r0 = atomic_cmpxchg(&val, r0, r5); */
++ BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_5, -8),
++ /* r1 = *r10 */
++ BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -8),
++ /* exit(0); */
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .result_unpriv = REJECT,
++ .errstr_unpriv = "R10 partial copy of pointer",
++},
++{
++ "Dest pointer in r0 - succeed, check 5",
++ .insns = {
++ /* r0 = &val */
++ BPF_MOV32_REG(BPF_REG_0, BPF_REG_10),
++ /* val = r0; */
++ BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -8),
++ /* r5 = &val */
++ BPF_MOV32_REG(BPF_REG_5, BPF_REG_10),
++ /* r0 = atomic_cmpxchg(&val, r0, r5); */
++ BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_5, -8),
++ /* r1 = *r0 */
++ BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, -8),
++ /* exit(0); */
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = REJECT,
++ .errstr = "R0 invalid mem access",
++ .errstr_unpriv = "R10 partial copy of pointer",
+ },
+diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c
+index 336a749673d19..2e701e7f69680 100644
+--- a/tools/testing/selftests/bpf/verifier/calls.c
++++ b/tools/testing/selftests/bpf/verifier/calls.c
+@@ -107,6 +107,25 @@
+ .result = REJECT,
+ .errstr = "R0 min value is outside of the allowed memory range",
+ },
++{
++ "calls: trigger reg2btf_ids[reg->type] for reg->type > __BPF_REG_TYPE_MAX",
++ .insns = {
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
++ BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0),
++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
++ .result = REJECT,
++ .errstr = "arg#0 pointer type STRUCT prog_test_ref_kfunc must point",
++ .fixup_kfunc_btf_id = {
++ { "bpf_kfunc_call_test_acquire", 3 },
++ { "bpf_kfunc_call_test_release", 5 },
++ },
++},
+ {
+ "calls: overlapping caller/callee",
+ .insns = {
+diff --git a/tools/testing/selftests/bpf/verifier/value_ptr_arith.c b/tools/testing/selftests/bpf/verifier/value_ptr_arith.c
+index 2debba4e8a3a8..4d347bc53aa28 100644
+--- a/tools/testing/selftests/bpf/verifier/value_ptr_arith.c
++++ b/tools/testing/selftests/bpf/verifier/value_ptr_arith.c
+@@ -1077,6 +1077,29 @@
+ .errstr = "R0 invalid mem access 'inv'",
+ .errstr_unpriv = "R0 pointer -= pointer prohibited",
+ },
++{
++ "map access: trying to leak tained dst reg",
++ .insns = {
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
++ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
++ BPF_LD_MAP_FD(BPF_REG_1, 0),
++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
++ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
++ BPF_EXIT_INSN(),
++ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
++ BPF_MOV32_IMM(BPF_REG_1, 0xFFFFFFFF),
++ BPF_MOV32_REG(BPF_REG_1, BPF_REG_1),
++ BPF_ALU64_REG(BPF_SUB, BPF_REG_2, BPF_REG_1),
++ BPF_STX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .fixup_map_array_48b = { 4 },
++ .result = REJECT,
++ .errstr = "math between map_value pointer and 4294967295 is not allowed",
++},
+ {
+ "32bit pkt_ptr -= scalar",
+ .insns = {
+diff --git a/tools/testing/selftests/bpf/verifier/xdp_direct_packet_access.c b/tools/testing/selftests/bpf/verifier/xdp_direct_packet_access.c
+index bfb97383e6b5a..b4ec228eb95d0 100644
+--- a/tools/testing/selftests/bpf/verifier/xdp_direct_packet_access.c
++++ b/tools/testing/selftests/bpf/verifier/xdp_direct_packet_access.c
+@@ -35,7 +35,7 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ },
+ {
+- "XDP pkt read, pkt_data' > pkt_end, good access",
++ "XDP pkt read, pkt_data' > pkt_end, corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+@@ -87,6 +87,41 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_data' > pkt_end, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data' > pkt_end, corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_end > pkt_data', good access",
+ .insns = {
+@@ -106,16 +141,16 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_end > pkt_data', bad access 1",
++ "XDP pkt read, pkt_end > pkt_data', corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+ offsetof(struct xdp_md, data_end)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
+ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -142,6 +177,42 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_end > pkt_data', corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end > pkt_data', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_data' < pkt_end, good access",
+ .insns = {
+@@ -161,16 +232,16 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data' < pkt_end, bad access 1",
++ "XDP pkt read, pkt_data' < pkt_end, corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+ offsetof(struct xdp_md, data_end)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
+ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -198,7 +269,43 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_end < pkt_data', good access",
++ "XDP pkt read, pkt_data' < pkt_end, corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data' < pkt_end, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end < pkt_data', corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+@@ -250,6 +357,41 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_end < pkt_data', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end < pkt_data', corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_data' >= pkt_end, good access",
+ .insns = {
+@@ -268,15 +410,15 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data' >= pkt_end, bad access 1",
++ "XDP pkt read, pkt_data' >= pkt_end, corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+ offsetof(struct xdp_md, data_end)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -304,7 +446,41 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_end >= pkt_data', good access",
++ "XDP pkt read, pkt_data' >= pkt_end, corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data' >= pkt_end, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end >= pkt_data', corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+@@ -359,7 +535,44 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data' <= pkt_end, good access",
++ "XDP pkt read, pkt_end >= pkt_data', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end >= pkt_data', corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data' <= pkt_end, corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+@@ -413,6 +626,43 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_data' <= pkt_end, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data' <= pkt_end, corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_end <= pkt_data', good access",
+ .insns = {
+@@ -431,15 +681,15 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_end <= pkt_data', bad access 1",
++ "XDP pkt read, pkt_end <= pkt_data', corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
+ offsetof(struct xdp_md, data_end)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -467,7 +717,41 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_meta' > pkt_data, good access",
++ "XDP pkt read, pkt_end <= pkt_data', corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_end <= pkt_data', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
++ offsetof(struct xdp_md, data_end)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' > pkt_data, corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+@@ -519,6 +803,41 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_meta' > pkt_data, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' > pkt_data, corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_data > pkt_meta', good access",
+ .insns = {
+@@ -538,16 +857,16 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data > pkt_meta', bad access 1",
++ "XDP pkt read, pkt_data > pkt_meta', corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
+ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -574,6 +893,42 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_data > pkt_meta', corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data > pkt_meta', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_meta' < pkt_data, good access",
+ .insns = {
+@@ -593,16 +948,16 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_meta' < pkt_data, bad access 1",
++ "XDP pkt read, pkt_meta' < pkt_data, corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
+ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -630,7 +985,43 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data < pkt_meta', good access",
++ "XDP pkt read, pkt_meta' < pkt_data, corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' < pkt_data, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data < pkt_meta', corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+@@ -682,6 +1073,41 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_data < pkt_meta', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data < pkt_meta', corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLT, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_meta' >= pkt_data, good access",
+ .insns = {
+@@ -700,15 +1126,15 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_meta' >= pkt_data, bad access 1",
++ "XDP pkt read, pkt_meta' >= pkt_data, corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -736,7 +1162,41 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data >= pkt_meta', good access",
++ "XDP pkt read, pkt_meta' >= pkt_data, corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' >= pkt_data, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data >= pkt_meta', corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+@@ -791,7 +1251,44 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_meta' <= pkt_data, good access",
++ "XDP pkt read, pkt_data >= pkt_meta', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data >= pkt_meta', corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' <= pkt_data, corner case, good access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+@@ -845,6 +1342,43 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_meta' <= pkt_data, corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 9),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -9),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_meta' <= pkt_data, corner case -1, bad access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_1, BPF_REG_3, 1),
++ BPF_JMP_IMM(BPF_JA, 0, 0, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .errstr = "R1 offset is outside of the packet",
++ .result = REJECT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+ {
+ "XDP pkt read, pkt_data <= pkt_meta', good access",
+ .insns = {
+@@ -863,15 +1397,15 @@
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
+ {
+- "XDP pkt read, pkt_data <= pkt_meta', bad access 1",
++ "XDP pkt read, pkt_data <= pkt_meta', corner case -1, bad access",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
+ offsetof(struct xdp_md, data_meta)),
+ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+- BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 6),
+ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
+- BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -6),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+@@ -898,3 +1432,37 @@
+ .prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ },
++{
++ "XDP pkt read, pkt_data <= pkt_meta', corner case, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 7),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -7),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
++{
++ "XDP pkt read, pkt_data <= pkt_meta', corner case +1, good access",
++ .insns = {
++ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
++ offsetof(struct xdp_md, data_meta)),
++ BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, offsetof(struct xdp_md, data)),
++ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
++ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 8),
++ BPF_JMP_REG(BPF_JLE, BPF_REG_3, BPF_REG_1, 1),
++ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8),
++ BPF_MOV64_IMM(BPF_REG_0, 0),
++ BPF_EXIT_INSN(),
++ },
++ .result = ACCEPT,
++ .prog_type = BPF_PROG_TYPE_XDP,
++ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
++},
+diff --git a/tools/testing/selftests/bpf/xdp_redirect_multi.c b/tools/testing/selftests/bpf/xdp_redirect_multi.c
+index 3696a8f32c235..f5ffba341c174 100644
+--- a/tools/testing/selftests/bpf/xdp_redirect_multi.c
++++ b/tools/testing/selftests/bpf/xdp_redirect_multi.c
+@@ -129,7 +129,7 @@ int main(int argc, char **argv)
+ goto err_out;
+ }
+
+- printf("Get interfaces");
++ printf("Get interfaces:");
+ for (i = 0; i < MAX_IFACE_NUM && argv[optind + i]; i++) {
+ ifaces[i] = if_nametoindex(argv[optind + i]);
+ if (!ifaces[i])
+@@ -139,7 +139,7 @@ int main(int argc, char **argv)
+ goto err_out;
+ }
+ if (ifaces[i] > MAX_INDEX_NUM) {
+- printf("Interface index to large\n");
++ printf(" interface index too large\n");
+ goto err_out;
+ }
+ printf(" %d", ifaces[i]);
+diff --git a/tools/testing/selftests/bpf/xdping.c b/tools/testing/selftests/bpf/xdping.c
+index 842d9155d36c5..79a3453dab255 100644
+--- a/tools/testing/selftests/bpf/xdping.c
++++ b/tools/testing/selftests/bpf/xdping.c
+@@ -178,9 +178,8 @@ int main(int argc, char **argv)
+ return 1;
+ }
+
+- main_prog = bpf_object__find_program_by_title(obj,
+- server ? "xdpserver" :
+- "xdpclient");
++ main_prog = bpf_object__find_program_by_name(obj,
++ server ? "xdping_server" : "xdping_client");
+ if (main_prog)
+ prog_fd = bpf_program__fd(main_prog);
+ if (!main_prog || prog_fd < 0) {
+diff --git a/tools/testing/selftests/cgroup/cgroup_util.c b/tools/testing/selftests/cgroup/cgroup_util.c
+index 623cec04ad422..0cf7e90c0052e 100644
+--- a/tools/testing/selftests/cgroup/cgroup_util.c
++++ b/tools/testing/selftests/cgroup/cgroup_util.c
+@@ -221,7 +221,7 @@ int cg_find_unified_root(char *root, size_t len)
+
+ int cg_create(const char *cgroup)
+ {
+- return mkdir(cgroup, 0644);
++ return mkdir(cgroup, 0755);
+ }
+
+ int cg_wait_for_proc_count(const char *cgroup, int count)
+diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c
+index 3df648c378765..6001235030631 100644
+--- a/tools/testing/selftests/cgroup/test_core.c
++++ b/tools/testing/selftests/cgroup/test_core.c
+@@ -1,11 +1,14 @@
+ /* SPDX-License-Identifier: GPL-2.0 */
+
++#define _GNU_SOURCE
+ #include <linux/limits.h>
++#include <linux/sched.h>
+ #include <sys/types.h>
+ #include <sys/mman.h>
+ #include <sys/wait.h>
+ #include <unistd.h>
+ #include <fcntl.h>
++#include <sched.h>
+ #include <stdio.h>
+ #include <errno.h>
+ #include <signal.h>
+@@ -674,6 +677,166 @@ cleanup:
+ return ret;
+ }
+
++/*
++ * cgroup migration permission check should be performed based on the
++ * credentials at the time of open instead of write.
++ */
++static int test_cgcore_lesser_euid_open(const char *root)
++{
++ const uid_t test_euid = 65534; /* usually nobody, any !root is fine */
++ int ret = KSFT_FAIL;
++ char *cg_test_a = NULL, *cg_test_b = NULL;
++ char *cg_test_a_procs = NULL, *cg_test_b_procs = NULL;
++ int cg_test_b_procs_fd = -1;
++ uid_t saved_uid;
++
++ cg_test_a = cg_name(root, "cg_test_a");
++ cg_test_b = cg_name(root, "cg_test_b");
++
++ if (!cg_test_a || !cg_test_b)
++ goto cleanup;
++
++ cg_test_a_procs = cg_name(cg_test_a, "cgroup.procs");
++ cg_test_b_procs = cg_name(cg_test_b, "cgroup.procs");
++
++ if (!cg_test_a_procs || !cg_test_b_procs)
++ goto cleanup;
++
++ if (cg_create(cg_test_a) || cg_create(cg_test_b))
++ goto cleanup;
++
++ if (cg_enter_current(cg_test_a))
++ goto cleanup;
++
++ if (chown(cg_test_a_procs, test_euid, -1) ||
++ chown(cg_test_b_procs, test_euid, -1))
++ goto cleanup;
++
++ saved_uid = geteuid();
++ if (seteuid(test_euid))
++ goto cleanup;
++
++ cg_test_b_procs_fd = open(cg_test_b_procs, O_RDWR);
++
++ if (seteuid(saved_uid))
++ goto cleanup;
++
++ if (cg_test_b_procs_fd < 0)
++ goto cleanup;
++
++ if (write(cg_test_b_procs_fd, "0", 1) >= 0 || errno != EACCES)
++ goto cleanup;
++
++ ret = KSFT_PASS;
++
++cleanup:
++ cg_enter_current(root);
++ if (cg_test_b_procs_fd >= 0)
++ close(cg_test_b_procs_fd);
++ if (cg_test_b)
++ cg_destroy(cg_test_b);
++ if (cg_test_a)
++ cg_destroy(cg_test_a);
++ free(cg_test_b_procs);
++ free(cg_test_a_procs);
++ free(cg_test_b);
++ free(cg_test_a);
++ return ret;
++}
++
++struct lesser_ns_open_thread_arg {
++ const char *path;
++ int fd;
++ int err;
++};
++
++static int lesser_ns_open_thread_fn(void *arg)
++{
++ struct lesser_ns_open_thread_arg *targ = arg;
++
++ targ->fd = open(targ->path, O_RDWR);
++ targ->err = errno;
++ return 0;
++}
++
++/*
++ * cgroup migration permission check should be performed based on the cgroup
++ * namespace at the time of open instead of write.
++ */
++static int test_cgcore_lesser_ns_open(const char *root)
++{
++ static char stack[65536];
++ const uid_t test_euid = 65534; /* usually nobody, any !root is fine */
++ int ret = KSFT_FAIL;
++ char *cg_test_a = NULL, *cg_test_b = NULL;
++ char *cg_test_a_procs = NULL, *cg_test_b_procs = NULL;
++ int cg_test_b_procs_fd = -1;
++ struct lesser_ns_open_thread_arg targ = { .fd = -1 };
++ pid_t pid;
++ int status;
++
++ cg_test_a = cg_name(root, "cg_test_a");
++ cg_test_b = cg_name(root, "cg_test_b");
++
++ if (!cg_test_a || !cg_test_b)
++ goto cleanup;
++
++ cg_test_a_procs = cg_name(cg_test_a, "cgroup.procs");
++ cg_test_b_procs = cg_name(cg_test_b, "cgroup.procs");
++
++ if (!cg_test_a_procs || !cg_test_b_procs)
++ goto cleanup;
++
++ if (cg_create(cg_test_a) || cg_create(cg_test_b))
++ goto cleanup;
++
++ if (cg_enter_current(cg_test_b))
++ goto cleanup;
++
++ if (chown(cg_test_a_procs, test_euid, -1) ||
++ chown(cg_test_b_procs, test_euid, -1))
++ goto cleanup;
++
++ targ.path = cg_test_b_procs;
++ pid = clone(lesser_ns_open_thread_fn, stack + sizeof(stack),
++ CLONE_NEWCGROUP | CLONE_FILES | CLONE_VM | SIGCHLD,
++ &targ);
++ if (pid < 0)
++ goto cleanup;
++
++ if (waitpid(pid, &status, 0) < 0)
++ goto cleanup;
++
++ if (!WIFEXITED(status))
++ goto cleanup;
++
++ cg_test_b_procs_fd = targ.fd;
++ if (cg_test_b_procs_fd < 0)
++ goto cleanup;
++
++ if (cg_enter_current(cg_test_a))
++ goto cleanup;
++
++ if ((status = write(cg_test_b_procs_fd, "0", 1)) >= 0 || errno != ENOENT)
++ goto cleanup;
++
++ ret = KSFT_PASS;
++
++cleanup:
++ cg_enter_current(root);
++ if (cg_test_b_procs_fd >= 0)
++ close(cg_test_b_procs_fd);
++ if (cg_test_b)
++ cg_destroy(cg_test_b);
++ if (cg_test_a)
++ cg_destroy(cg_test_a);
++ free(cg_test_b_procs);
++ free(cg_test_a_procs);
++ free(cg_test_b);
++ free(cg_test_a);
++ return ret;
++}
++
+ #define T(x) { x, #x }
+ struct corecg_test {
+ int (*fn)(const char *root);
+@@ -689,6 +852,8 @@ struct corecg_test {
+ T(test_cgcore_proc_migration),
+ T(test_cgcore_thread_migration),
+ T(test_cgcore_destroy),
++ T(test_cgcore_lesser_euid_open),
++ T(test_cgcore_lesser_ns_open),
+ };
+ #undef T
+
+diff --git a/tools/testing/selftests/clone3/clone3.c b/tools/testing/selftests/clone3/clone3.c
+index 42be3b9258301..cd4582129c7d6 100644
+--- a/tools/testing/selftests/clone3/clone3.c
++++ b/tools/testing/selftests/clone3/clone3.c
+@@ -52,6 +52,12 @@ static int call_clone3(uint64_t flags, size_t size, enum test_mode test_mode)
+ size = sizeof(struct __clone_args);
+
+ switch (test_mode) {
++ case CLONE3_ARGS_NO_TEST:
++ /*
++ * Uses default 'flags' and 'SIGCHLD'
++ * assignment.
++ */
++ break;
+ case CLONE3_ARGS_ALL_0:
+ args.flags = 0;
+ args.exit_signal = 0;
+@@ -120,8 +126,6 @@ static void test_clone3(uint64_t flags, size_t size, int expected,
+
+ int main(int argc, char *argv[])
+ {
+- pid_t pid;
+-
+ uid_t uid = getuid();
+
+ ksft_print_header();
+diff --git a/tools/testing/selftests/core/close_range_test.c b/tools/testing/selftests/core/close_range_test.c
+index 73eb29c916d1b..aa7d13d91963f 100644
+--- a/tools/testing/selftests/core/close_range_test.c
++++ b/tools/testing/selftests/core/close_range_test.c
+@@ -54,7 +54,7 @@ static inline int sys_close_range(unsigned int fd, unsigned int max_fd,
+ #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+ #endif
+
+-TEST(close_range)
++TEST(core_close_range)
+ {
+ int i, ret;
+ int open_fds[101];
+diff --git a/tools/testing/selftests/damon/.gitignore b/tools/testing/selftests/damon/.gitignore
+new file mode 100644
+index 0000000000000..c6c2965a66075
+--- /dev/null
++++ b/tools/testing/selftests/damon/.gitignore
+@@ -0,0 +1,2 @@
++# SPDX-License-Identifier: GPL-2.0-only
++huge_count_read_write
+diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile
+index 8a3f2cd9fec0c..f0aa954b5d135 100644
+--- a/tools/testing/selftests/damon/Makefile
++++ b/tools/testing/selftests/damon/Makefile
+@@ -1,6 +1,8 @@
+ # SPDX-License-Identifier: GPL-2.0
+ # Makefile for damon selftests
+
++TEST_GEN_FILES += huge_count_read_write
++
+ TEST_FILES = _chk_dependency.sh
+ TEST_PROGS = debugfs_attrs.sh
+
+diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh
+index bfabb19dc0d3d..ecda972e87775 100644
+--- a/tools/testing/selftests/damon/debugfs_attrs.sh
++++ b/tools/testing/selftests/damon/debugfs_attrs.sh
+@@ -72,4 +72,22 @@ test_write_succ "$file" "" "$orig_content" "empty input"
+ test_content "$file" "$orig_content" "" "empty input written"
+ echo "$orig_content" > "$file"
+
++# Test huge count read write
++# ==========================
++
++dmesg -C
++
++for file in "$DBGFS/"*
++do
++ ./huge_count_read_write "$file"
++done
++
++if dmesg | grep -q WARNING
++then
++ dmesg
++ exit 1
++else
++ exit 0
++fi
++
+ echo "PASS"
+diff --git a/tools/testing/selftests/damon/huge_count_read_write.c b/tools/testing/selftests/damon/huge_count_read_write.c
+new file mode 100644
+index 0000000000000..ad7a6b4cf3387
+--- /dev/null
++++ b/tools/testing/selftests/damon/huge_count_read_write.c
+@@ -0,0 +1,39 @@
++// SPDX-License-Identifier: GPL-2.0
++/*
++ * Author: SeongJae Park <sj@kernel.org>
++ */
++
++#include <fcntl.h>
++#include <stdlib.h>
++#include <unistd.h>
++#include <stdio.h>
++
++void write_read_with_huge_count(char *file)
++{
++ int filedesc = open(file, O_RDWR);
++ char buf[25];
++ int ret;
++
++ printf("%s %s\n", __func__, file);
++ if (filedesc < 0) {
++ fprintf(stderr, "failed opening %s\n", file);
++ exit(1);
++ }
++
++ write(filedesc, "", 0xfffffffful);
++ perror("after write: ");
++ ret = read(filedesc, buf, 0xfffffffful);
++ perror("after read: ");
++ close(filedesc);
++}
++
++int main(int argc, char *argv[])
++{
++ if (argc != 2) {
++ fprintf(stderr, "Usage: %s <file>\n", argv[0]);
++ exit(1);
++ }
++ write_read_with_huge_count(argv[1]);
++
++ return 0;
++}
+diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh
+index 685dfb3478b3e..b9b8274643de1 100755
+--- a/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh
++++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh
+@@ -50,8 +50,8 @@ for current_test in ${TESTS:-$ALL_TESTS}; do
+ else
+ log_test "'$current_test' [$profile] overflow $target"
+ fi
++ RET_FIN=$(( RET_FIN || RET ))
+ done
+- RET_FIN=$(( RET_FIN || RET ))
+ done
+ done
+ current_test=""
+diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_police_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_police_scale.sh
+index 3e3e06ea5703c..86e787895f78b 100644
+--- a/tools/testing/selftests/drivers/net/mlxsw/tc_police_scale.sh
++++ b/tools/testing/selftests/drivers/net/mlxsw/tc_police_scale.sh
+@@ -60,7 +60,8 @@ __tc_police_test()
+
+ tc_police_rules_create $count $should_fail
+
+- offload_count=$(tc filter show dev $swp1 ingress | grep in_hw | wc -l)
++ offload_count=$(tc -j filter show dev $swp1 ingress |
++ jq "[.[] | select(.options.in_hw == true)] | length")
+ ((offload_count == count))
+ check_err_fail $should_fail $? "tc police offload count"
+ }
+diff --git a/tools/testing/selftests/drivers/net/mlxsw/vxlan_flooding.sh b/tools/testing/selftests/drivers/net/mlxsw/vxlan_flooding.sh
+index fedcb7b35af9f..af5ea50ed5c0e 100755
+--- a/tools/testing/selftests/drivers/net/mlxsw/vxlan_flooding.sh
++++ b/tools/testing/selftests/drivers/net/mlxsw/vxlan_flooding.sh
+@@ -172,6 +172,17 @@ flooding_filters_add()
+ local lsb
+ local i
+
++ # Prevent unwanted packets from entering the bridge and interfering
++ # with the test.
++ tc qdisc add dev br0 clsact
++ tc filter add dev br0 egress protocol all pref 1 handle 1 \
++ matchall skip_hw action drop
++ tc qdisc add dev $h1 clsact
++ tc filter add dev $h1 egress protocol all pref 1 handle 1 \
++ flower skip_hw dst_mac de:ad:be:ef:13:37 action pass
++ tc filter add dev $h1 egress protocol all pref 2 handle 2 \
++ matchall skip_hw action drop
++
+ tc qdisc add dev $rp2 clsact
+
+ for i in $(eval echo {1..$num_remotes}); do
+@@ -194,6 +205,12 @@ flooding_filters_del()
+ done
+
+ tc qdisc del dev $rp2 clsact
++
++ tc filter del dev $h1 egress protocol all pref 2 handle 2 matchall
++ tc filter del dev $h1 egress protocol all pref 1 handle 1 flower
++ tc qdisc del dev $h1 clsact
++ tc filter del dev br0 egress protocol all pref 1 handle 1 matchall
++ tc qdisc del dev br0 clsact
+ }
+
+ flooding_check_packets()
+diff --git a/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh
+index f7d84549cc3e3..79f751259098d 100755
+--- a/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh
++++ b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh
+@@ -185,7 +185,7 @@ setup_prepare()
+
+ tc filter add dev $eth0 ingress chain $(IS2 0 0) pref 1 \
+ protocol ipv4 flower skip_sw ip_proto udp dst_port 5201 \
+- action police rate 50mbit burst 64k \
++ action police rate 50mbit burst 64k conform-exceed drop/pipe \
+ action goto chain $(IS2 1 0)
+ }
+
+diff --git a/tools/testing/selftests/exec/Makefile b/tools/testing/selftests/exec/Makefile
+index dd61118df66ed..2d7fca446c7f7 100644
+--- a/tools/testing/selftests/exec/Makefile
++++ b/tools/testing/selftests/exec/Makefile
+@@ -3,9 +3,9 @@ CFLAGS = -Wall
+ CFLAGS += -Wno-nonnull
+ CFLAGS += -D_GNU_SOURCE
+
+-TEST_PROGS := binfmt_script non-regular
+-TEST_GEN_PROGS := execveat load_address_4096 load_address_2097152 load_address_16777216
+-TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir pipe
++TEST_PROGS := binfmt_script
++TEST_GEN_PROGS := execveat load_address_4096 load_address_2097152 load_address_16777216 non-regular
++TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir
+ # Makefile is a run-time dependency, since it's accessed by the execveat test
+ TEST_FILES := Makefile
+
+diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc
+index e96e279e0533a..25432b8cd5bd2 100644
+--- a/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc
++++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc
+@@ -19,7 +19,7 @@ fail() { # mesg
+
+ FILTER=set_ftrace_filter
+ FUNC1="schedule"
+-FUNC2="do_softirq"
++FUNC2="scheduler_tick"
+
+ ALL_FUNCS="#### all functions enabled ####"
+
+diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc b/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc
+index 98166fa3eb91c..34fb89b0c61fa 100644
+--- a/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc
++++ b/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc
+@@ -1,6 +1,6 @@
+ #!/bin/sh
+ # SPDX-License-Identifier: GPL-2.0
+-# description: Kprobe dynamic event - adding and removing
++# description: Kprobe profile
+ # requires: kprobe_events
+
+ ! grep -q 'myevent' kprobe_profile
+diff --git a/tools/testing/selftests/futex/Makefile b/tools/testing/selftests/futex/Makefile
+index 12631f0076a10..11e157d7533b8 100644
+--- a/tools/testing/selftests/futex/Makefile
++++ b/tools/testing/selftests/futex/Makefile
+@@ -11,7 +11,7 @@ all:
+ @for DIR in $(SUBDIRS); do \
+ BUILD_TARGET=$(OUTPUT)/$$DIR; \
+ mkdir $$BUILD_TARGET -p; \
+- make OUTPUT=$$BUILD_TARGET -C $$DIR $@;\
++ $(MAKE) OUTPUT=$$BUILD_TARGET -C $$DIR $@;\
+ if [ -e $$DIR/$(TEST_PROGS) ]; then \
+ rsync -a $$DIR/$(TEST_PROGS) $$BUILD_TARGET/; \
+ fi \
+@@ -32,6 +32,6 @@ override define CLEAN
+ @for DIR in $(SUBDIRS); do \
+ BUILD_TARGET=$(OUTPUT)/$$DIR; \
+ mkdir $$BUILD_TARGET -p; \
+- make OUTPUT=$$BUILD_TARGET -C $$DIR $@;\
++ $(MAKE) OUTPUT=$$BUILD_TARGET -C $$DIR $@;\
+ done
+ endef
+diff --git a/tools/testing/selftests/gpio/Makefile b/tools/testing/selftests/gpio/Makefile
+index 39f2bbe8dd3df..42ea7d2aa8440 100644
+--- a/tools/testing/selftests/gpio/Makefile
++++ b/tools/testing/selftests/gpio/Makefile
+@@ -3,5 +3,6 @@
+ TEST_PROGS := gpio-mockup.sh
+ TEST_FILES := gpio-mockup-sysfs.sh
+ TEST_GEN_PROGS_EXTENDED := gpio-mockup-cdev
++CFLAGS += -I../../../../usr/include
+
+ include ../lib.mk
+diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h
+index ae0f0f33b2a6e..78e59620d28de 100644
+--- a/tools/testing/selftests/kselftest_harness.h
++++ b/tools/testing/selftests/kselftest_harness.h
+@@ -875,7 +875,8 @@ static void __timeout_handler(int sig, siginfo_t *info, void *ucontext)
+ }
+
+ t->timed_out = true;
+- kill(t->pid, SIGKILL);
++ // signal process group
++ kill(-(t->pid), SIGKILL);
+ }
+
+ void __wait_for_test(struct __test_metadata *t)
+@@ -969,7 +970,7 @@ void __run_test(struct __fixture_metadata *f,
+ t->passed = 1;
+ t->skip = 0;
+ t->trigger = 0;
+- t->step = 0;
++ t->step = 1;
+ t->no_print = 0;
+ memset(t->results->reason, 0, sizeof(t->results->reason));
+
+@@ -985,6 +986,7 @@ void __run_test(struct __fixture_metadata *f,
+ ksft_print_msg("ERROR SPAWNING TEST CHILD\n");
+ t->passed = 0;
+ } else if (t->pid == 0) {
++ setpgrp();
+ t->fn(t, variant);
+ if (t->skip)
+ _exit(255);
+diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
+index 010b59b139176..1876d148ea0ae 100644
+--- a/tools/testing/selftests/kvm/include/kvm_util.h
++++ b/tools/testing/selftests/kvm/include/kvm_util.h
+@@ -309,6 +309,7 @@ bool vm_is_unrestricted_guest(struct kvm_vm *vm);
+
+ unsigned int vm_get_page_size(struct kvm_vm *vm);
+ unsigned int vm_get_page_shift(struct kvm_vm *vm);
++unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
+ uint64_t vm_get_max_gfn(struct kvm_vm *vm);
+ int vm_get_fd(struct kvm_vm *vm);
+
+diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h
+index 05e65ca1c30cd..23861c8faa61c 100644
+--- a/tools/testing/selftests/kvm/include/x86_64/processor.h
++++ b/tools/testing/selftests/kvm/include/x86_64/processor.h
+@@ -58,6 +58,21 @@
+ /* CPUID.0x8000_0001.EDX */
+ #define CPUID_GBPAGES (1ul << 26)
+
++/* Page table bitfield declarations */
++#define PTE_PRESENT_MASK BIT_ULL(0)
++#define PTE_WRITABLE_MASK BIT_ULL(1)
++#define PTE_USER_MASK BIT_ULL(2)
++#define PTE_ACCESSED_MASK BIT_ULL(5)
++#define PTE_DIRTY_MASK BIT_ULL(6)
++#define PTE_LARGE_MASK BIT_ULL(7)
++#define PTE_GLOBAL_MASK BIT_ULL(8)
++#define PTE_NX_MASK BIT_ULL(63)
++
++#define PAGE_SHIFT 12
++
++#define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12)
++#define PTE_GET_PFN(pte) (((pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
++
+ /* General Registers in 64-Bit Mode */
+ struct gpr64_regs {
+ u64 rax;
+diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
+index 0299cd81b8ba2..aa3795cd7bd3d 100644
+--- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
++++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
+@@ -12,6 +12,7 @@
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
++#include <sys/resource.h>
+
+ #include "test_util.h"
+
+@@ -40,10 +41,39 @@ int main(int argc, char *argv[])
+ {
+ int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID);
+ int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
++ /*
++ * Number of file descriptors reqired, KVM_CAP_MAX_VCPUS for vCPU fds +
++ * an arbitrary number for everything else.
++ */
++ int nr_fds_wanted = kvm_max_vcpus + 100;
++ struct rlimit rl;
+
+ pr_info("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id);
+ pr_info("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus);
+
++ /*
++ * Check that we're allowed to open nr_fds_wanted file descriptors and
++ * try raising the limits if needed.
++ */
++ TEST_ASSERT(!getrlimit(RLIMIT_NOFILE, &rl), "getrlimit() failed!");
++
++ if (rl.rlim_cur < nr_fds_wanted) {
++ rl.rlim_cur = nr_fds_wanted;
++ if (rl.rlim_max < nr_fds_wanted) {
++ int old_rlim_max = rl.rlim_max;
++ rl.rlim_max = nr_fds_wanted;
++
++ int r = setrlimit(RLIMIT_NOFILE, &rl);
++ if (r < 0) {
++ printf("RLIMIT_NOFILE hard limit is too low (%d, wanted %d)\n",
++ old_rlim_max, nr_fds_wanted);
++ exit(KSFT_SKIP);
++ }
++ } else {
++ TEST_ASSERT(!setrlimit(RLIMIT_NOFILE, &rl), "setrlimit() failed!");
++ }
++ }
++
+ /*
+ * Upstream KVM prior to 4.8 does not support KVM_CAP_MAX_VCPU_ID.
+ * Userspace is supposed to use KVM_CAP_MAX_VCPUS as the maximum ID
+diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c
+index 36407cb0ec85d..f1ddfe4c4a031 100644
+--- a/tools/testing/selftests/kvm/kvm_page_table_test.c
++++ b/tools/testing/selftests/kvm/kvm_page_table_test.c
+@@ -278,7 +278,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
+ else
+ guest_test_phys_mem = p->phys_offset;
+ #ifdef __s390x__
+- alignment = max(0x100000, alignment);
++ alignment = max(0x100000UL, alignment);
+ #endif
+ guest_test_phys_mem &= ~(alignment - 1);
+
+diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
+index 10a8ed691c669..e9d0ab9567fbf 100644
+--- a/tools/testing/selftests/kvm/lib/kvm_util.c
++++ b/tools/testing/selftests/kvm/lib/kvm_util.c
+@@ -307,7 +307,7 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
+ (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
+
+ /* Limit physical addresses to PA-bits. */
+- vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
++ vm->max_gfn = vm_compute_max_gfn(vm);
+
+ /* Allocate and setup memory for guest. */
+ vm->vpages_mapped = sparsebit_alloc();
+@@ -2282,6 +2282,11 @@ unsigned int vm_get_page_shift(struct kvm_vm *vm)
+ return vm->page_shift;
+ }
+
++unsigned long __attribute__((weak)) vm_compute_max_gfn(struct kvm_vm *vm)
++{
++ return ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
++}
++
+ uint64_t vm_get_max_gfn(struct kvm_vm *vm)
+ {
+ return vm->max_gfn;
+diff --git a/tools/testing/selftests/kvm/lib/x86_64/processor.c b/tools/testing/selftests/kvm/lib/x86_64/processor.c
+index 28cb881f440d0..46057079d8bba 100644
+--- a/tools/testing/selftests/kvm/lib/x86_64/processor.c
++++ b/tools/testing/selftests/kvm/lib/x86_64/processor.c
+@@ -19,38 +19,6 @@
+
+ vm_vaddr_t exception_handlers;
+
+-/* Virtual translation table structure declarations */
+-struct pageUpperEntry {
+- uint64_t present:1;
+- uint64_t writable:1;
+- uint64_t user:1;
+- uint64_t write_through:1;
+- uint64_t cache_disable:1;
+- uint64_t accessed:1;
+- uint64_t ignored_06:1;
+- uint64_t page_size:1;
+- uint64_t ignored_11_08:4;
+- uint64_t pfn:40;
+- uint64_t ignored_62_52:11;
+- uint64_t execute_disable:1;
+-};
+-
+-struct pageTableEntry {
+- uint64_t present:1;
+- uint64_t writable:1;
+- uint64_t user:1;
+- uint64_t write_through:1;
+- uint64_t cache_disable:1;
+- uint64_t accessed:1;
+- uint64_t dirty:1;
+- uint64_t reserved_07:1;
+- uint64_t global:1;
+- uint64_t ignored_11_09:3;
+- uint64_t pfn:40;
+- uint64_t ignored_62_52:11;
+- uint64_t execute_disable:1;
+-};
+-
+ void regs_dump(FILE *stream, struct kvm_regs *regs,
+ uint8_t indent)
+ {
+@@ -195,23 +163,21 @@ static void *virt_get_pte(struct kvm_vm *vm, uint64_t pt_pfn, uint64_t vaddr,
+ return &page_table[index];
+ }
+
+-static struct pageUpperEntry *virt_create_upper_pte(struct kvm_vm *vm,
+- uint64_t pt_pfn,
+- uint64_t vaddr,
+- uint64_t paddr,
+- int level,
+- enum x86_page_size page_size)
++static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
++ uint64_t pt_pfn,
++ uint64_t vaddr,
++ uint64_t paddr,
++ int level,
++ enum x86_page_size page_size)
+ {
+- struct pageUpperEntry *pte = virt_get_pte(vm, pt_pfn, vaddr, level);
+-
+- if (!pte->present) {
+- pte->writable = true;
+- pte->present = true;
+- pte->page_size = (level == page_size);
+- if (pte->page_size)
+- pte->pfn = paddr >> vm->page_shift;
++ uint64_t *pte = virt_get_pte(vm, pt_pfn, vaddr, level);
++
++ if (!(*pte & PTE_PRESENT_MASK)) {
++ *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK;
++ if (level == page_size)
++ *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK);
+ else
+- pte->pfn = vm_alloc_page_table(vm) >> vm->page_shift;
++ *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
+ } else {
+ /*
+ * Entry already present. Assert that the caller doesn't want
+@@ -221,7 +187,7 @@ static struct pageUpperEntry *virt_create_upper_pte(struct kvm_vm *vm,
+ TEST_ASSERT(level != page_size,
+ "Cannot create hugepage at level: %u, vaddr: 0x%lx\n",
+ page_size, vaddr);
+- TEST_ASSERT(!pte->page_size,
++ TEST_ASSERT(!(*pte & PTE_LARGE_MASK),
+ "Cannot create page table at level: %u, vaddr: 0x%lx\n",
+ level, vaddr);
+ }
+@@ -232,8 +198,8 @@ void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
+ enum x86_page_size page_size)
+ {
+ const uint64_t pg_size = 1ull << ((page_size * 9) + 12);
+- struct pageUpperEntry *pml4e, *pdpe, *pde;
+- struct pageTableEntry *pte;
++ uint64_t *pml4e, *pdpe, *pde;
++ uint64_t *pte;
+
+ TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,
+ "Unknown or unsupported guest mode, mode: 0x%x", vm->mode);
+@@ -257,24 +223,22 @@ void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
+ */
+ pml4e = virt_create_upper_pte(vm, vm->pgd >> vm->page_shift,
+ vaddr, paddr, 3, page_size);
+- if (pml4e->page_size)
++ if (*pml4e & PTE_LARGE_MASK)
+ return;
+
+- pdpe = virt_create_upper_pte(vm, pml4e->pfn, vaddr, paddr, 2, page_size);
+- if (pdpe->page_size)
++ pdpe = virt_create_upper_pte(vm, PTE_GET_PFN(*pml4e), vaddr, paddr, 2, page_size);
++ if (*pdpe & PTE_LARGE_MASK)
+ return;
+
+- pde = virt_create_upper_pte(vm, pdpe->pfn, vaddr, paddr, 1, page_size);
+- if (pde->page_size)
++ pde = virt_create_upper_pte(vm, PTE_GET_PFN(*pdpe), vaddr, paddr, 1, page_size);
++ if (*pde & PTE_LARGE_MASK)
+ return;
+
+ /* Fill in page table entry. */
+- pte = virt_get_pte(vm, pde->pfn, vaddr, 0);
+- TEST_ASSERT(!pte->present,
++ pte = virt_get_pte(vm, PTE_GET_PFN(*pde), vaddr, 0);
++ TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),
+ "PTE already present for 4k page at vaddr: 0x%lx\n", vaddr);
+- pte->pfn = paddr >> vm->page_shift;
+- pte->writable = true;
+- pte->present = 1;
++ *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK);
+ }
+
+ void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+@@ -282,12 +246,12 @@ void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+ __virt_pg_map(vm, vaddr, paddr, X86_PAGE_SIZE_4K);
+ }
+
+-static struct pageTableEntry *_vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid,
++static uint64_t *_vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid,
+ uint64_t vaddr)
+ {
+ uint16_t index[4];
+- struct pageUpperEntry *pml4e, *pdpe, *pde;
+- struct pageTableEntry *pte;
++ uint64_t *pml4e, *pdpe, *pde;
++ uint64_t *pte;
+ struct kvm_cpuid_entry2 *entry;
+ struct kvm_sregs sregs;
+ int max_phy_addr;
+@@ -329,30 +293,29 @@ static struct pageTableEntry *_vm_get_page_table_entry(struct kvm_vm *vm, int vc
+ index[3] = (vaddr >> 39) & 0x1ffu;
+
+ pml4e = addr_gpa2hva(vm, vm->pgd);
+- TEST_ASSERT(pml4e[index[3]].present,
++ TEST_ASSERT(pml4e[index[3]] & PTE_PRESENT_MASK,
+ "Expected pml4e to be present for gva: 0x%08lx", vaddr);
+- TEST_ASSERT((*(uint64_t*)(&pml4e[index[3]]) &
+- (rsvd_mask | (1ull << 7))) == 0,
++ TEST_ASSERT((pml4e[index[3]] & (rsvd_mask | PTE_LARGE_MASK)) == 0,
+ "Unexpected reserved bits set.");
+
+- pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size);
+- TEST_ASSERT(pdpe[index[2]].present,
++ pdpe = addr_gpa2hva(vm, PTE_GET_PFN(pml4e[index[3]]) * vm->page_size);
++ TEST_ASSERT(pdpe[index[2]] & PTE_PRESENT_MASK,
+ "Expected pdpe to be present for gva: 0x%08lx", vaddr);
+- TEST_ASSERT(pdpe[index[2]].page_size == 0,
++ TEST_ASSERT(!(pdpe[index[2]] & PTE_LARGE_MASK),
+ "Expected pdpe to map a pde not a 1-GByte page.");
+- TEST_ASSERT((*(uint64_t*)(&pdpe[index[2]]) & rsvd_mask) == 0,
++ TEST_ASSERT((pdpe[index[2]] & rsvd_mask) == 0,
+ "Unexpected reserved bits set.");
+
+- pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size);
+- TEST_ASSERT(pde[index[1]].present,
++ pde = addr_gpa2hva(vm, PTE_GET_PFN(pdpe[index[2]]) * vm->page_size);
++ TEST_ASSERT(pde[index[1]] & PTE_PRESENT_MASK,
+ "Expected pde to be present for gva: 0x%08lx", vaddr);
+- TEST_ASSERT(pde[index[1]].page_size == 0,
++ TEST_ASSERT(!(pde[index[1]] & PTE_LARGE_MASK),
+ "Expected pde to map a pte not a 2-MByte page.");
+- TEST_ASSERT((*(uint64_t*)(&pde[index[1]]) & rsvd_mask) == 0,
++ TEST_ASSERT((pde[index[1]] & rsvd_mask) == 0,
+ "Unexpected reserved bits set.");
+
+- pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size);
+- TEST_ASSERT(pte[index[0]].present,
++ pte = addr_gpa2hva(vm, PTE_GET_PFN(pde[index[1]]) * vm->page_size);
++ TEST_ASSERT(pte[index[0]] & PTE_PRESENT_MASK,
+ "Expected pte to be present for gva: 0x%08lx", vaddr);
+
+ return &pte[index[0]];
+@@ -360,7 +323,7 @@ static struct pageTableEntry *_vm_get_page_table_entry(struct kvm_vm *vm, int vc
+
+ uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr)
+ {
+- struct pageTableEntry *pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
++ uint64_t *pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
+
+ return *(uint64_t *)pte;
+ }
+@@ -368,18 +331,17 @@ uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr)
+ void vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr,
+ uint64_t pte)
+ {
+- struct pageTableEntry *new_pte = _vm_get_page_table_entry(vm, vcpuid,
+- vaddr);
++ uint64_t *new_pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
+
+ *(uint64_t *)new_pte = pte;
+ }
+
+ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+ {
+- struct pageUpperEntry *pml4e, *pml4e_start;
+- struct pageUpperEntry *pdpe, *pdpe_start;
+- struct pageUpperEntry *pde, *pde_start;
+- struct pageTableEntry *pte, *pte_start;
++ uint64_t *pml4e, *pml4e_start;
++ uint64_t *pdpe, *pdpe_start;
++ uint64_t *pde, *pde_start;
++ uint64_t *pte, *pte_start;
+
+ if (!vm->pgd_created)
+ return;
+@@ -389,58 +351,58 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+ fprintf(stream, "%*s index hvaddr gpaddr "
+ "addr w exec dirty\n",
+ indent, "");
+- pml4e_start = (struct pageUpperEntry *) addr_gpa2hva(vm, vm->pgd);
++ pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd);
+ for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
+ pml4e = &pml4e_start[n1];
+- if (!pml4e->present)
++ if (!(*pml4e & PTE_PRESENT_MASK))
+ continue;
+- fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10lx %u "
++ fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "
+ " %u\n",
+ indent, "",
+ pml4e - pml4e_start, pml4e,
+- addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->pfn,
+- pml4e->writable, pml4e->execute_disable);
++ addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),
++ !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK));
+
+- pdpe_start = addr_gpa2hva(vm, pml4e->pfn * vm->page_size);
++ pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
+ for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
+ pdpe = &pdpe_start[n2];
+- if (!pdpe->present)
++ if (!(*pdpe & PTE_PRESENT_MASK))
+ continue;
+- fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10lx "
++ fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10llx "
+ "%u %u\n",
+ indent, "",
+ pdpe - pdpe_start, pdpe,
+ addr_hva2gpa(vm, pdpe),
+- (uint64_t) pdpe->pfn, pdpe->writable,
+- pdpe->execute_disable);
++ PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK),
++ !!(*pdpe & PTE_NX_MASK));
+
+- pde_start = addr_gpa2hva(vm, pdpe->pfn * vm->page_size);
++ pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
+ for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
+ pde = &pde_start[n3];
+- if (!pde->present)
++ if (!(*pde & PTE_PRESENT_MASK))
+ continue;
+ fprintf(stream, "%*spde 0x%-3zx %p "
+- "0x%-12lx 0x%-10lx %u %u\n",
++ "0x%-12lx 0x%-10llx %u %u\n",
+ indent, "", pde - pde_start, pde,
+ addr_hva2gpa(vm, pde),
+- (uint64_t) pde->pfn, pde->writable,
+- pde->execute_disable);
++ PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK),
++ !!(*pde & PTE_NX_MASK));
+
+- pte_start = addr_gpa2hva(vm, pde->pfn * vm->page_size);
++ pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
+ for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
+ pte = &pte_start[n4];
+- if (!pte->present)
++ if (!(*pte & PTE_PRESENT_MASK))
+ continue;
+ fprintf(stream, "%*spte 0x%-3zx %p "
+- "0x%-12lx 0x%-10lx %u %u "
++ "0x%-12lx 0x%-10llx %u %u "
+ " %u 0x%-10lx\n",
+ indent, "",
+ pte - pte_start, pte,
+ addr_hva2gpa(vm, pte),
+- (uint64_t) pte->pfn,
+- pte->writable,
+- pte->execute_disable,
+- pte->dirty,
++ PTE_GET_PFN(*pte),
++ !!(*pte & PTE_WRITABLE_MASK),
++ !!(*pte & PTE_NX_MASK),
++ !!(*pte & PTE_DIRTY_MASK),
+ ((uint64_t) n1 << 27)
+ | ((uint64_t) n2 << 18)
+ | ((uint64_t) n3 << 9)
+@@ -558,8 +520,8 @@ static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
+ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+ {
+ uint16_t index[4];
+- struct pageUpperEntry *pml4e, *pdpe, *pde;
+- struct pageTableEntry *pte;
++ uint64_t *pml4e, *pdpe, *pde;
++ uint64_t *pte;
+
+ TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
+ "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
+@@ -572,22 +534,22 @@ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+ if (!vm->pgd_created)
+ goto unmapped_gva;
+ pml4e = addr_gpa2hva(vm, vm->pgd);
+- if (!pml4e[index[3]].present)
++ if (!(pml4e[index[3]] & PTE_PRESENT_MASK))
+ goto unmapped_gva;
+
+- pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size);
+- if (!pdpe[index[2]].present)
++ pdpe = addr_gpa2hva(vm, PTE_GET_PFN(pml4e[index[3]]) * vm->page_size);
++ if (!(pdpe[index[2]] & PTE_PRESENT_MASK))
+ goto unmapped_gva;
+
+- pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size);
+- if (!pde[index[1]].present)
++ pde = addr_gpa2hva(vm, PTE_GET_PFN(pdpe[index[2]]) * vm->page_size);
++ if (!(pde[index[1]] & PTE_PRESENT_MASK))
+ goto unmapped_gva;
+
+- pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size);
+- if (!pte[index[0]].present)
++ pte = addr_gpa2hva(vm, PTE_GET_PFN(pde[index[1]]) * vm->page_size);
++ if (!(pte[index[0]] & PTE_PRESENT_MASK))
+ goto unmapped_gva;
+
+- return (pte[index[0]].pfn * vm->page_size) + (gva & 0xfffu);
++ return (PTE_GET_PFN(pte[index[0]]) * vm->page_size) + (gva & 0xfffu);
+
+ unmapped_gva:
+ TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
+@@ -1433,3 +1395,71 @@ struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpui
+
+ return cpuid;
+ }
++
++#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541
++#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163
++#define X86EMUL_CPUID_VENDOR_AuthenticAMD_edx 0x69746e65
++
++static inline unsigned x86_family(unsigned int eax)
++{
++ unsigned int x86;
++
++ x86 = (eax >> 8) & 0xf;
++
++ if (x86 == 0xf)
++ x86 += (eax >> 20) & 0xff;
++
++ return x86;
++}
++
++unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
++{
++ const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
++ unsigned long ht_gfn, max_gfn, max_pfn;
++ uint32_t eax, ebx, ecx, edx, max_ext_leaf;
++
++ max_gfn = (1ULL << (vm->pa_bits - vm->page_shift)) - 1;
++
++ /* Avoid reserved HyperTransport region on AMD processors. */
++ eax = ecx = 0;
++ cpuid(&eax, &ebx, &ecx, &edx);
++ if (ebx != X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx ||
++ ecx != X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx ||
++ edx != X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
++ return max_gfn;
++
++ /* On parts with <40 physical address bits, the area is fully hidden */
++ if (vm->pa_bits < 40)
++ return max_gfn;
++
++ /* Before family 17h, the HyperTransport area is just below 1T. */
++ ht_gfn = (1 << 28) - num_ht_pages;
++ eax = 1;
++ cpuid(&eax, &ebx, &ecx, &edx);
++ if (x86_family(eax) < 0x17)
++ goto done;
++
++ /*
++ * Otherwise it's at the top of the physical address space, possibly
++ * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX. Use
++ * the old conservative value if MAXPHYADDR is not enumerated.
++ */
++ eax = 0x80000000;
++ cpuid(&eax, &ebx, &ecx, &edx);
++ max_ext_leaf = eax;
++ if (max_ext_leaf < 0x80000008)
++ goto done;
++
++ eax = 0x80000008;
++ cpuid(&eax, &ebx, &ecx, &edx);
++ max_pfn = (1ULL << ((eax & 0xff) - vm->page_shift)) - 1;
++ if (max_ext_leaf >= 0x8000001f) {
++ eax = 0x8000001f;
++ cpuid(&eax, &ebx, &ecx, &edx);
++ max_pfn >>= (ebx >> 6) & 0x3f;
++ }
++
++ ht_gfn = max_pfn - num_ht_pages;
++done:
++ return min(max_gfn, ht_gfn - 1);
++}
+diff --git a/tools/testing/selftests/kvm/lib/x86_64/svm.c b/tools/testing/selftests/kvm/lib/x86_64/svm.c
+index 2ac98d70d02bd..161eba7cd1289 100644
+--- a/tools/testing/selftests/kvm/lib/x86_64/svm.c
++++ b/tools/testing/selftests/kvm/lib/x86_64/svm.c
+@@ -54,6 +54,18 @@ static void vmcb_set_seg(struct vmcb_seg *seg, u16 selector,
+ seg->base = base;
+ }
+
++/*
++ * Avoid using memset to clear the vmcb, since libc may not be
++ * available in L1 (and, even if it is, features that libc memset may
++ * want to use, like AVX, may not be enabled).
++ */
++static void clear_vmcb(struct vmcb *vmcb)
++{
++ int n = sizeof(*vmcb) / sizeof(u32);
++
++ asm volatile ("rep stosl" : "+c"(n), "+D"(vmcb) : "a"(0) : "memory");
++}
++
+ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp)
+ {
+ struct vmcb *vmcb = svm->vmcb;
+@@ -70,7 +82,7 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r
+ wrmsr(MSR_EFER, efer | EFER_SVME);
+ wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa);
+
+- memset(vmcb, 0, sizeof(*vmcb));
++ clear_vmcb(vmcb);
+ asm volatile ("vmsave %0\n\t" : : "a" (vmcb_gpa) : "memory");
+ vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr);
+ vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr);
+diff --git a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c b/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c
+index 8039e1eff9388..9f55ccd169a13 100644
+--- a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c
++++ b/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c
+@@ -84,7 +84,7 @@ int get_warnings_count(void)
+ f = popen("dmesg | grep \"WARNING:\" | wc -l", "r");
+ if (fscanf(f, "%d", &warnings) < 1)
+ warnings = 0;
+- fclose(f);
++ pclose(f);
+
+ return warnings;
+ }
+diff --git a/tools/testing/selftests/kvm/x86_64/smm_test.c b/tools/testing/selftests/kvm/x86_64/smm_test.c
+index d0fe2fdce58c4..db2a17559c3d5 100644
+--- a/tools/testing/selftests/kvm/x86_64/smm_test.c
++++ b/tools/testing/selftests/kvm/x86_64/smm_test.c
+@@ -105,7 +105,6 @@ static void guest_code(void *arg)
+
+ if (cpu_has_svm()) {
+ run_guest(svm->vmcb, svm->vmcb_gpa);
+- svm->vmcb->save.rip += 3;
+ run_guest(svm->vmcb, svm->vmcb_gpa);
+ } else {
+ vmlaunch();
+diff --git a/tools/testing/selftests/lkdtm/config b/tools/testing/selftests/lkdtm/config
+index 38edea25631bc..b642411ceb6c3 100644
+--- a/tools/testing/selftests/lkdtm/config
++++ b/tools/testing/selftests/lkdtm/config
+@@ -6,5 +6,6 @@ CONFIG_HARDENED_USERCOPY=y
+ # CONFIG_HARDENED_USERCOPY_FALLBACK is not set
+ CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=y
+ CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y
++CONFIG_UBSAN=y
+ CONFIG_UBSAN_BOUNDS=y
+ CONFIG_UBSAN_TRAP=y
+diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c
+index 192a2899bae8f..94df2692e6e4a 100644
+--- a/tools/testing/selftests/memfd/memfd_test.c
++++ b/tools/testing/selftests/memfd/memfd_test.c
+@@ -455,6 +455,7 @@ static void mfd_fail_write(int fd)
+ printf("mmap()+mprotect() didn't fail as expected\n");
+ abort();
+ }
++ munmap(p, mfd_def_size);
+ }
+
+ /* verify PUNCH_HOLE fails */
+diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c
+index e54106643337b..4c88238fc8f05 100644
+--- a/tools/testing/selftests/mincore/mincore_selftest.c
++++ b/tools/testing/selftests/mincore/mincore_selftest.c
+@@ -207,15 +207,21 @@ TEST(check_file_mmap)
+
+ errno = 0;
+ fd = open(".", O_TMPFILE | O_RDWR, 0600);
+- ASSERT_NE(-1, fd) {
+- TH_LOG("Can't create temporary file: %s",
+- strerror(errno));
++ if (fd < 0) {
++ ASSERT_EQ(errno, EOPNOTSUPP) {
++ TH_LOG("Can't create temporary file: %s",
++ strerror(errno));
++ }
++ SKIP(goto out_free, "O_TMPFILE not supported by filesystem.");
+ }
+ errno = 0;
+ retval = fallocate(fd, 0, 0, FILE_SIZE);
+- ASSERT_EQ(0, retval) {
+- TH_LOG("Error allocating space for the temporary file: %s",
+- strerror(errno));
++ if (retval) {
++ ASSERT_EQ(errno, EOPNOTSUPP) {
++ TH_LOG("Error allocating space for the temporary file: %s",
++ strerror(errno));
++ }
++ SKIP(goto out_close, "fallocate not supported by filesystem.");
+ }
+
+ /*
+@@ -271,7 +277,9 @@ TEST(check_file_mmap)
+ }
+
+ munmap(addr, FILE_SIZE);
++out_close:
+ close(fd);
++out_free:
+ free(vec);
+ }
+
+diff --git a/tools/testing/selftests/mount_setattr/mount_setattr_test.c b/tools/testing/selftests/mount_setattr/mount_setattr_test.c
+index f31205f04ee05..8c5fea68ae677 100644
+--- a/tools/testing/selftests/mount_setattr/mount_setattr_test.c
++++ b/tools/testing/selftests/mount_setattr/mount_setattr_test.c
+@@ -1236,7 +1236,7 @@ static int get_userns_fd(unsigned long nsid, unsigned long hostid, unsigned long
+ }
+
+ /**
+- * Validate that an attached mount in our mount namespace can be idmapped.
++ * Validate that an attached mount in our mount namespace cannot be idmapped.
+ * (The kernel enforces that the mount's mount namespace and the caller's mount
+ * namespace match.)
+ */
+@@ -1259,7 +1259,7 @@ TEST_F(mount_setattr_idmapped, attached_mount_inside_current_mount_namespace)
+
+ attr.userns_fd = get_userns_fd(0, 10000, 10000);
+ ASSERT_GE(attr.userns_fd, 0);
+- ASSERT_EQ(sys_mount_setattr(open_tree_fd, "", AT_EMPTY_PATH, &attr, sizeof(attr)), 0);
++ ASSERT_NE(sys_mount_setattr(open_tree_fd, "", AT_EMPTY_PATH, &attr, sizeof(attr)), 0);
+ ASSERT_EQ(close(attr.userns_fd), 0);
+ ASSERT_EQ(close(open_tree_fd), 0);
+ }
+diff --git a/tools/testing/selftests/mqueue/mq_perf_tests.c b/tools/testing/selftests/mqueue/mq_perf_tests.c
+index b019e0b8221c7..84fda3b490735 100644
+--- a/tools/testing/selftests/mqueue/mq_perf_tests.c
++++ b/tools/testing/selftests/mqueue/mq_perf_tests.c
+@@ -180,6 +180,9 @@ void shutdown(int exit_val, char *err_cause, int line_no)
+ if (in_shutdown++)
+ return;
+
++ /* Free the cpu_set allocated using CPU_ALLOC in main function */
++ CPU_FREE(cpu_set);
++
+ for (i = 0; i < num_cpus_to_pin; i++)
+ if (cpu_threads[i]) {
+ pthread_kill(cpu_threads[i], SIGUSR1);
+@@ -551,6 +554,12 @@ int main(int argc, char *argv[])
+ perror("sysconf(_SC_NPROCESSORS_ONLN)");
+ exit(1);
+ }
++
++ if (getuid() != 0)
++ ksft_exit_skip("Not running as root, but almost all tests "
++ "require root in order to modify\nsystem settings. "
++ "Exiting.\n");
++
+ cpus_online = min(MAX_CPUS, sysconf(_SC_NPROCESSORS_ONLN));
+ cpu_set = CPU_ALLOC(cpus_online);
+ if (cpu_set == NULL) {
+@@ -589,7 +598,7 @@ int main(int argc, char *argv[])
+ cpu_set)) {
+ fprintf(stderr, "Any given CPU may "
+ "only be given once.\n");
+- exit(1);
++ goto err_code;
+ } else
+ CPU_SET_S(cpus_to_pin[cpu],
+ cpu_set_size, cpu_set);
+@@ -607,7 +616,7 @@ int main(int argc, char *argv[])
+ queue_path = malloc(strlen(option) + 2);
+ if (!queue_path) {
+ perror("malloc()");
+- exit(1);
++ goto err_code;
+ }
+ queue_path[0] = '/';
+ queue_path[1] = 0;
+@@ -622,17 +631,12 @@ int main(int argc, char *argv[])
+ fprintf(stderr, "Must pass at least one CPU to continuous "
+ "mode.\n");
+ poptPrintUsage(popt_context, stderr, 0);
+- exit(1);
++ goto err_code;
+ } else if (!continuous_mode) {
+ num_cpus_to_pin = 1;
+ cpus_to_pin[0] = cpus_online - 1;
+ }
+
+- if (getuid() != 0)
+- ksft_exit_skip("Not running as root, but almost all tests "
+- "require root in order to modify\nsystem settings. "
+- "Exiting.\n");
+-
+ max_msgs = fopen(MAX_MSGS, "r+");
+ max_msgsize = fopen(MAX_MSGSIZE, "r+");
+ if (!max_msgs)
+@@ -740,4 +744,9 @@ int main(int argc, char *argv[])
+ sleep(1);
+ }
+ shutdown(0, "", 0);
++
++err_code:
++ CPU_FREE(cpu_set);
++ exit(1);
++
+ }
+diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
+index 492b273743b4e..6a953ec793ced 100644
+--- a/tools/testing/selftests/net/Makefile
++++ b/tools/testing/selftests/net/Makefile
+@@ -12,7 +12,7 @@ TEST_PROGS += udpgro_bench.sh udpgro.sh test_vxlan_under_vrf.sh reuseport_addr_a
+ TEST_PROGS += test_vxlan_fdb_changelink.sh so_txtime.sh ipv6_flowlabel.sh
+ TEST_PROGS += tcp_fastopen_backup_key.sh fcnal-test.sh l2tp.sh traceroute.sh
+ TEST_PROGS += fin_ack_lat.sh fib_nexthop_multiprefix.sh fib_nexthops.sh
+-TEST_PROGS += altnames.sh icmp_redirect.sh ip6_gre_headroom.sh
++TEST_PROGS += altnames.sh icmp.sh icmp_redirect.sh ip6_gre_headroom.sh
+ TEST_PROGS += route_localnet.sh
+ TEST_PROGS += reuseaddr_ports_exhausted.sh
+ TEST_PROGS += txtimestamp.sh
+@@ -28,7 +28,12 @@ TEST_PROGS += veth.sh
+ TEST_PROGS += ioam6.sh
+ TEST_PROGS += gro.sh
+ TEST_PROGS += gre_gso.sh
+-TEST_PROGS_EXTENDED := in_netns.sh
++TEST_PROGS += srv6_end_dt46_l3vpn_test.sh
++TEST_PROGS += srv6_end_dt4_l3vpn_test.sh
++TEST_PROGS += srv6_end_dt6_l3vpn_test.sh
++TEST_PROGS += vrf_strict_mode_test.sh
++TEST_PROGS_EXTENDED := in_netns.sh setup_loopback.sh setup_veth.sh
++TEST_PROGS_EXTENDED += toeplitz_client.sh toeplitz.sh
+ TEST_GEN_FILES = socket nettest
+ TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy reuseport_addr_any
+ TEST_GEN_FILES += tcp_mmap tcp_inq psock_snd txring_overwrite
+diff --git a/tools/testing/selftests/net/af_unix/test_unix_oob.c b/tools/testing/selftests/net/af_unix/test_unix_oob.c
+index 3dece8b292536..b57e91e1c3f28 100644
+--- a/tools/testing/selftests/net/af_unix/test_unix_oob.c
++++ b/tools/testing/selftests/net/af_unix/test_unix_oob.c
+@@ -218,10 +218,10 @@ main(int argc, char **argv)
+
+ /* Test 1:
+ * veriyf that SIGURG is
+- * delivered and 63 bytes are
+- * read and oob is '@'
++ * delivered, 63 bytes are
++ * read, oob is '@', and POLLPRI works.
+ */
+- wait_for_data(pfd, POLLIN | POLLPRI);
++ wait_for_data(pfd, POLLPRI);
+ read_oob(pfd, &oob);
+ len = read_data(pfd, buf, 1024);
+ if (!signal_recvd || len != 63 || oob != '@') {
+diff --git a/tools/testing/selftests/net/fcnal-test.sh b/tools/testing/selftests/net/fcnal-test.sh
+index 3313566ce9062..91f54112167f1 100755
+--- a/tools/testing/selftests/net/fcnal-test.sh
++++ b/tools/testing/selftests/net/fcnal-test.sh
+@@ -455,6 +455,22 @@ cleanup()
+ ip netns del ${NSC} >/dev/null 2>&1
+ }
+
++cleanup_vrf_dup()
++{
++ ip link del ${NSA_DEV2} >/dev/null 2>&1
++ ip netns pids ${NSC} | xargs kill 2>/dev/null
++ ip netns del ${NSC} >/dev/null 2>&1
++}
++
++setup_vrf_dup()
++{
++ # some VRF tests use ns-C which has the same config as
++ # ns-B but for a device NOT in the VRF
++ create_ns ${NSC} "-" "-"
++ connect_ns ${NSA} ${NSA_DEV2} ${NSA_IP}/24 ${NSA_IP6}/64 \
++ ${NSC} ${NSC_DEV} ${NSB_IP}/24 ${NSB_IP6}/64
++}
++
+ setup()
+ {
+ local with_vrf=${1}
+@@ -484,12 +500,6 @@ setup()
+
+ ip -netns ${NSB} ro add ${VRF_IP}/32 via ${NSA_IP} dev ${NSB_DEV}
+ ip -netns ${NSB} -6 ro add ${VRF_IP6}/128 via ${NSA_IP6} dev ${NSB_DEV}
+-
+- # some VRF tests use ns-C which has the same config as
+- # ns-B but for a device NOT in the VRF
+- create_ns ${NSC} "-" "-"
+- connect_ns ${NSA} ${NSA_DEV2} ${NSA_IP}/24 ${NSA_IP6}/64 \
+- ${NSC} ${NSC_DEV} ${NSB_IP}/24 ${NSB_IP6}/64
+ else
+ ip -netns ${NSA} ro add ${NSB_LO_IP}/32 via ${NSB_IP} dev ${NSA_DEV}
+ ip -netns ${NSA} ro add ${NSB_LO_IP6}/128 via ${NSB_IP6} dev ${NSA_DEV}
+@@ -793,10 +803,16 @@ ipv4_ping()
+ setup
+ set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null
+ ipv4_ping_novrf
++ setup
++ set_sysctl net.ipv4.ping_group_range='0 2147483647' 2>/dev/null
++ ipv4_ping_novrf
+
+ log_subsection "With VRF"
+ setup "yes"
+ ipv4_ping_vrf
++ setup "yes"
++ set_sysctl net.ipv4.ping_group_range='0 2147483647' 2>/dev/null
++ ipv4_ping_vrf
+ }
+
+ ################################################################################
+@@ -1240,7 +1256,9 @@ ipv4_tcp_vrf()
+ log_test_addr ${a} $? 1 "Global server, local connection"
+
+ # run MD5 tests
++ setup_vrf_dup
+ ipv4_tcp_md5
++ cleanup_vrf_dup
+
+ #
+ # enable VRF global server
+@@ -1798,8 +1816,9 @@ ipv4_addr_bind_vrf()
+ for a in ${NSA_IP} ${VRF_IP}
+ do
+ log_start
++ show_hint "Socket not bound to VRF, but address is in VRF"
+ run_cmd nettest -s -R -P icmp -l ${a} -b
+- log_test_addr ${a} $? 0 "Raw socket bind to local address"
++ log_test_addr ${a} $? 1 "Raw socket bind to local address"
+
+ log_start
+ run_cmd nettest -s -R -P icmp -l ${a} -I ${NSA_DEV} -b
+@@ -2191,7 +2210,7 @@ ipv6_ping_vrf()
+ log_start
+ show_hint "Fails since VRF device does not support linklocal or multicast"
+ run_cmd ${ping6} -c1 -w1 ${a}
+- log_test_addr ${a} $? 2 "ping out, VRF bind"
++ log_test_addr ${a} $? 1 "ping out, VRF bind"
+ done
+
+ for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
+@@ -2311,10 +2330,16 @@ ipv6_ping()
+ log_subsection "No VRF"
+ setup
+ ipv6_ping_novrf
++ setup
++ set_sysctl net.ipv4.ping_group_range='0 2147483647' 2>/dev/null
++ ipv6_ping_novrf
+
+ log_subsection "With VRF"
+ setup "yes"
+ ipv6_ping_vrf
++ setup "yes"
++ set_sysctl net.ipv4.ping_group_range='0 2147483647' 2>/dev/null
++ ipv6_ping_vrf
+ }
+
+ ################################################################################
+@@ -2719,7 +2744,9 @@ ipv6_tcp_vrf()
+ log_test_addr ${a} $? 1 "Global server, local connection"
+
+ # run MD5 tests
++ setup_vrf_dup
+ ipv6_tcp_md5
++ cleanup_vrf_dup
+
+ #
+ # enable VRF global server
+@@ -3414,11 +3441,14 @@ ipv6_addr_bind_novrf()
+ run_cmd nettest -6 -s -l ${a} -I ${NSA_DEV} -t1 -b
+ log_test_addr ${a} $? 0 "TCP socket bind to local address after device bind"
+
++ # Sadly, the kernel allows binding a socket to a device and then
++ # binding to an address not on the device. So this test passes
++ # when it really should not
+ a=${NSA_LO_IP6}
+ log_start
+- show_hint "Should fail with 'Cannot assign requested address'"
++ show_hint "Tecnically should fail since address is not on device but kernel allows"
+ run_cmd nettest -6 -s -l ${a} -I ${NSA_DEV} -t1 -b
+- log_test_addr ${a} $? 1 "TCP socket bind to out of scope local address"
++ log_test_addr ${a} $? 0 "TCP socket bind to out of scope local address"
+ }
+
+ ipv6_addr_bind_vrf()
+@@ -3459,10 +3489,15 @@ ipv6_addr_bind_vrf()
+ run_cmd nettest -6 -s -l ${a} -I ${NSA_DEV} -t1 -b
+ log_test_addr ${a} $? 0 "TCP socket bind to local address with device bind"
+
++ # Sadly, the kernel allows binding a socket to a device and then
++ # binding to an address not on the device. The only restriction
++ # is that the address is valid in the L3 domain. So this test
++ # passes when it really should not
+ a=${VRF_IP6}
+ log_start
++ show_hint "Tecnically should fail since address is not on device but kernel allows"
+ run_cmd nettest -6 -s -l ${a} -I ${NSA_DEV} -t1 -b
+- log_test_addr ${a} $? 1 "TCP socket bind to VRF address with device bind"
++ log_test_addr ${a} $? 0 "TCP socket bind to VRF address with device bind"
+
+ a=${NSA_LO_IP6}
+ log_start
+@@ -4002,8 +4037,8 @@ EOF
+ ################################################################################
+ # main
+
+-TESTS_IPV4="ipv4_ping ipv4_tcp ipv4_udp ipv4_addr_bind ipv4_runtime ipv4_netfilter"
+-TESTS_IPV6="ipv6_ping ipv6_tcp ipv6_udp ipv6_addr_bind ipv6_runtime ipv6_netfilter"
++TESTS_IPV4="ipv4_ping ipv4_tcp ipv4_udp ipv4_bind ipv4_runtime ipv4_netfilter"
++TESTS_IPV6="ipv6_ping ipv6_tcp ipv6_udp ipv6_bind ipv6_runtime ipv6_netfilter"
+ TESTS_OTHER="use_cases"
+
+ PAUSE_ON_FAIL=no
+diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh
+index 0d293391e9a44..b5a69ad191b07 100755
+--- a/tools/testing/selftests/net/fib_nexthops.sh
++++ b/tools/testing/selftests/net/fib_nexthops.sh
+@@ -2078,6 +2078,7 @@ basic_res()
+ "id 101 index 0 nhid 2 id 101 index 1 nhid 2 id 101 index 2 nhid 1 id 101 index 3 nhid 1"
+ log_test $? 0 "Dump all nexthop buckets in a group"
+
++ sleep 0.1
+ (( $($IP -j nexthop bucket list id 101 |
+ jq '[.[] | select(.bucket.idle_time > 0 and
+ .bucket.idle_time < 2)] | length') == 4 ))
+diff --git a/tools/testing/selftests/net/fib_tests.sh b/tools/testing/selftests/net/fib_tests.sh
+index 5abe92d55b696..996af1ae3d3dd 100755
+--- a/tools/testing/selftests/net/fib_tests.sh
++++ b/tools/testing/selftests/net/fib_tests.sh
+@@ -444,24 +444,63 @@ fib_rp_filter_test()
+ setup
+
+ set -e
++ ip netns add ns2
++ ip netns set ns2 auto
++
++ ip -netns ns2 link set dev lo up
++
++ $IP link add name veth1 type veth peer name veth2
++ $IP link set dev veth2 netns ns2
++ $IP address add 192.0.2.1/24 dev veth1
++ ip -netns ns2 address add 192.0.2.1/24 dev veth2
++ $IP link set dev veth1 up
++ ip -netns ns2 link set dev veth2 up
++
+ $IP link set dev lo address 52:54:00:6a:c7:5e
+- $IP link set dummy0 address 52:54:00:6a:c7:5e
+- $IP link add dummy1 type dummy
+- $IP link set dummy1 address 52:54:00:6a:c7:5e
+- $IP link set dev dummy1 up
++ $IP link set dev veth1 address 52:54:00:6a:c7:5e
++ ip -netns ns2 link set dev lo address 52:54:00:6a:c7:5e
++ ip -netns ns2 link set dev veth2 address 52:54:00:6a:c7:5e
++
++ # 1. (ns2) redirect lo's egress to veth2's egress
++ ip netns exec ns2 tc qdisc add dev lo parent root handle 1: fq_codel
++ ip netns exec ns2 tc filter add dev lo parent 1: protocol arp basic \
++ action mirred egress redirect dev veth2
++ ip netns exec ns2 tc filter add dev lo parent 1: protocol ip basic \
++ action mirred egress redirect dev veth2
++
++ # 2. (ns1) redirect veth1's ingress to lo's ingress
++ $NS_EXEC tc qdisc add dev veth1 ingress
++ $NS_EXEC tc filter add dev veth1 ingress protocol arp basic \
++ action mirred ingress redirect dev lo
++ $NS_EXEC tc filter add dev veth1 ingress protocol ip basic \
++ action mirred ingress redirect dev lo
++
++ # 3. (ns1) redirect lo's egress to veth1's egress
++ $NS_EXEC tc qdisc add dev lo parent root handle 1: fq_codel
++ $NS_EXEC tc filter add dev lo parent 1: protocol arp basic \
++ action mirred egress redirect dev veth1
++ $NS_EXEC tc filter add dev lo parent 1: protocol ip basic \
++ action mirred egress redirect dev veth1
++
++ # 4. (ns2) redirect veth2's ingress to lo's ingress
++ ip netns exec ns2 tc qdisc add dev veth2 ingress
++ ip netns exec ns2 tc filter add dev veth2 ingress protocol arp basic \
++ action mirred ingress redirect dev lo
++ ip netns exec ns2 tc filter add dev veth2 ingress protocol ip basic \
++ action mirred ingress redirect dev lo
++
+ $NS_EXEC sysctl -qw net.ipv4.conf.all.rp_filter=1
+ $NS_EXEC sysctl -qw net.ipv4.conf.all.accept_local=1
+ $NS_EXEC sysctl -qw net.ipv4.conf.all.route_localnet=1
+-
+- $NS_EXEC tc qd add dev dummy1 parent root handle 1: fq_codel
+- $NS_EXEC tc filter add dev dummy1 parent 1: protocol arp basic action mirred egress redirect dev lo
+- $NS_EXEC tc filter add dev dummy1 parent 1: protocol ip basic action mirred egress redirect dev lo
++ ip netns exec ns2 sysctl -qw net.ipv4.conf.all.rp_filter=1
++ ip netns exec ns2 sysctl -qw net.ipv4.conf.all.accept_local=1
++ ip netns exec ns2 sysctl -qw net.ipv4.conf.all.route_localnet=1
+ set +e
+
+- run_cmd "ip netns exec ns1 ping -I dummy1 -w1 -c1 198.51.100.1"
++ run_cmd "ip netns exec ns2 ping -w1 -c1 192.0.2.1"
+ log_test $? 0 "rp_filter passes local packets"
+
+- run_cmd "ip netns exec ns1 ping -I dummy1 -w1 -c1 127.0.0.1"
++ run_cmd "ip netns exec ns2 ping -w1 -c1 127.0.0.1"
+ log_test $? 0 "rp_filter passes loopback packets"
+
+ cleanup
+diff --git a/tools/testing/selftests/net/forwarding/bridge_igmp.sh b/tools/testing/selftests/net/forwarding/bridge_igmp.sh
+index 675eff45b0371..1162836f8f329 100755
+--- a/tools/testing/selftests/net/forwarding/bridge_igmp.sh
++++ b/tools/testing/selftests/net/forwarding/bridge_igmp.sh
+@@ -482,10 +482,15 @@ v3exc_timeout_test()
+ local X=("192.0.2.20" "192.0.2.30")
+
+ # GMI should be 3 seconds
+- ip link set dev br0 type bridge mcast_query_interval 100 mcast_query_response_interval 100
++ ip link set dev br0 type bridge mcast_query_interval 100 \
++ mcast_query_response_interval 100 \
++ mcast_membership_interval 300
+
+ v3exclude_prepare $h1 $ALL_MAC $ALL_GROUP
+- ip link set dev br0 type bridge mcast_query_interval 500 mcast_query_response_interval 500
++ ip link set dev br0 type bridge mcast_query_interval 500 \
++ mcast_query_response_interval 500 \
++ mcast_membership_interval 1500
++
+ $MZ $h1 -c 1 -b $ALL_MAC -B $ALL_GROUP -t ip "proto=2,p=$MZPKT_ALLOW2" -q
+ sleep 3
+ bridge -j -d -s mdb show dev br0 \
+@@ -517,7 +522,8 @@ v3exc_timeout_test()
+ log_test "IGMPv3 group $TEST_GROUP exclude timeout"
+
+ ip link set dev br0 type bridge mcast_query_interval 12500 \
+- mcast_query_response_interval 1000
++ mcast_query_response_interval 1000 \
++ mcast_membership_interval 26000
+
+ v3cleanup $swp1 $TEST_GROUP
+ }
+diff --git a/tools/testing/selftests/net/forwarding/bridge_mld.sh b/tools/testing/selftests/net/forwarding/bridge_mld.sh
+index ffdcfa87ca2ba..e2b9ff773c6b6 100755
+--- a/tools/testing/selftests/net/forwarding/bridge_mld.sh
++++ b/tools/testing/selftests/net/forwarding/bridge_mld.sh
+@@ -479,10 +479,15 @@ mldv2exc_timeout_test()
+ local X=("2001:db8:1::20" "2001:db8:1::30")
+
+ # GMI should be 3 seconds
+- ip link set dev br0 type bridge mcast_query_interval 100 mcast_query_response_interval 100
++ ip link set dev br0 type bridge mcast_query_interval 100 \
++ mcast_query_response_interval 100 \
++ mcast_membership_interval 300
+
+ mldv2exclude_prepare $h1
+- ip link set dev br0 type bridge mcast_query_interval 500 mcast_query_response_interval 500
++ ip link set dev br0 type bridge mcast_query_interval 500 \
++ mcast_query_response_interval 500 \
++ mcast_membership_interval 1500
++
+ $MZ $h1 -c 1 $MZPKT_ALLOW2 -q
+ sleep 3
+ bridge -j -d -s mdb show dev br0 \
+@@ -514,7 +519,8 @@ mldv2exc_timeout_test()
+ log_test "MLDv2 group $TEST_GROUP exclude timeout"
+
+ ip link set dev br0 type bridge mcast_query_interval 12500 \
+- mcast_query_response_interval 1000
++ mcast_query_response_interval 1000 \
++ mcast_membership_interval 26000
+
+ mldv2cleanup $swp1
+ }
+diff --git a/tools/testing/selftests/net/forwarding/forwarding.config.sample b/tools/testing/selftests/net/forwarding/forwarding.config.sample
+index e5e2fbeca22ec..e51def39fd801 100644
+--- a/tools/testing/selftests/net/forwarding/forwarding.config.sample
++++ b/tools/testing/selftests/net/forwarding/forwarding.config.sample
+@@ -13,6 +13,8 @@ NETIFS[p5]=veth4
+ NETIFS[p6]=veth5
+ NETIFS[p7]=veth6
+ NETIFS[p8]=veth7
++NETIFS[p9]=veth8
++NETIFS[p10]=veth9
+
+ # Port that does not have a cable connected.
+ NETIF_NO_CABLE=eth8
+diff --git a/tools/testing/selftests/net/forwarding/mirror_gre_bridge_1q.sh b/tools/testing/selftests/net/forwarding/mirror_gre_bridge_1q.sh
+index a3402cd8d5b68..9ff22f28032dd 100755
+--- a/tools/testing/selftests/net/forwarding/mirror_gre_bridge_1q.sh
++++ b/tools/testing/selftests/net/forwarding/mirror_gre_bridge_1q.sh
+@@ -61,9 +61,12 @@ setup_prepare()
+
+ vrf_prepare
+ mirror_gre_topo_create
++ # Avoid changing br1's PVID while it is operational as a L3 interface.
++ ip link set dev br1 down
+
+ ip link set dev $swp3 master br1
+ bridge vlan add dev br1 vid 555 pvid untagged self
++ ip link set dev br1 up
+ ip address add dev br1 192.0.2.129/28
+ ip address add dev br1 2001:db8:2::1/64
+
+diff --git a/tools/testing/selftests/net/gre_gso.sh b/tools/testing/selftests/net/gre_gso.sh
+index facbb0c804439..3224651db97b8 100755
+--- a/tools/testing/selftests/net/gre_gso.sh
++++ b/tools/testing/selftests/net/gre_gso.sh
+@@ -116,17 +116,20 @@ gre_gst_test_checks()
+ {
+ local name=$1
+ local addr=$2
++ local proto=$3
+
+- $NS_EXEC nc -kl $port >/dev/null &
++ [ "$proto" == 6 ] && addr="[$addr]"
++
++ $NS_EXEC socat - tcp${proto}-listen:$port,reuseaddr,fork >/dev/null &
+ PID=$!
+ while ! $NS_EXEC ss -ltn | grep -q $port; do ((i++)); sleep 0.01; done
+
+- cat $TMPFILE | timeout 1 nc $addr $port
++ cat $TMPFILE | timeout 1 socat -u STDIN TCP:$addr:$port
+ log_test $? 0 "$name - copy file w/ TSO"
+
+ ethtool -K veth0 tso off
+
+- cat $TMPFILE | timeout 1 nc $addr $port
++ cat $TMPFILE | timeout 1 socat -u STDIN TCP:$addr:$port
+ log_test $? 0 "$name - copy file w/ GSO"
+
+ ethtool -K veth0 tso on
+@@ -154,8 +157,8 @@ gre6_gso_test()
+
+ sleep 2
+
+- gre_gst_test_checks GREv6/v4 172.16.2.2
+- gre_gst_test_checks GREv6/v6 2001:db8:1::2
++ gre_gst_test_checks GREv6/v4 172.16.2.2 4
++ gre_gst_test_checks GREv6/v6 2001:db8:1::2 6
+
+ cleanup
+ }
+@@ -211,8 +214,8 @@ if [ ! -x "$(command -v ip)" ]; then
+ exit $ksft_skip
+ fi
+
+-if [ ! -x "$(command -v nc)" ]; then
+- echo "SKIP: Could not run test without nc tool"
++if [ ! -x "$(command -v socat)" ]; then
++ echo "SKIP: Could not run test without socat tool"
+ exit $ksft_skip
+ fi
+
+diff --git a/tools/testing/selftests/net/icmp_redirect.sh b/tools/testing/selftests/net/icmp_redirect.sh
+index ecbf57f264ed9..7b9d6e31b8e7d 100755
+--- a/tools/testing/selftests/net/icmp_redirect.sh
++++ b/tools/testing/selftests/net/icmp_redirect.sh
+@@ -311,7 +311,7 @@ check_exception()
+ ip -netns h1 ro get ${H1_VRF_ARG} ${H2_N2_IP} | \
+ grep -E -v 'mtu|redirected' | grep -q "cache"
+ fi
+- log_test $? 0 "IPv4: ${desc}"
++ log_test $? 0 "IPv4: ${desc}" 0
+
+ # No PMTU info for test "redirect" and "mtu exception plus redirect"
+ if [ "$with_redirect" = "yes" ] && [ "$desc" != "redirect exception plus mtu" ]; then
+diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh
+index 2674ba20d5249..ff821025d3096 100755
+--- a/tools/testing/selftests/net/mptcp/diag.sh
++++ b/tools/testing/selftests/net/mptcp/diag.sh
+@@ -71,6 +71,36 @@ chk_msk_remote_key_nr()
+ __chk_nr "grep -c remote_key" $*
+ }
+
++# $1: ns, $2: port
++wait_local_port_listen()
++{
++ local listener_ns="${1}"
++ local port="${2}"
++
++ local port_hex i
++
++ port_hex="$(printf "%04X" "${port}")"
++ for i in $(seq 10); do
++ ip netns exec "${listener_ns}" cat /proc/net/tcp | \
++ awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
++ break
++ sleep 0.1
++ done
++}
++
++wait_connected()
++{
++ local listener_ns="${1}"
++ local port="${2}"
++
++ local port_hex i
++
++ port_hex="$(printf "%04X" "${port}")"
++ for i in $(seq 10); do
++ ip netns exec ${listener_ns} grep -q " 0100007F:${port_hex} " /proc/net/tcp && break
++ sleep 0.1
++ done
++}
+
+ trap cleanup EXIT
+ ip netns add $ns
+@@ -81,15 +111,15 @@ echo "a" | \
+ ip netns exec $ns \
+ ./mptcp_connect -p 10000 -l -t ${timeout_poll} \
+ 0.0.0.0 >/dev/null &
+-sleep 0.1
++wait_local_port_listen $ns 10000
+ chk_msk_nr 0 "no msk on netns creation"
+
+ echo "b" | \
+ timeout ${timeout_test} \
+ ip netns exec $ns \
+- ./mptcp_connect -p 10000 -j -t ${timeout_poll} \
++ ./mptcp_connect -p 10000 -r 0 -t ${timeout_poll} \
+ 127.0.0.1 >/dev/null &
+-sleep 0.1
++wait_connected $ns 10000
+ chk_msk_nr 2 "after MPC handshake "
+ chk_msk_remote_key_nr 2 "....chk remote_key"
+ chk_msk_fallback_nr 0 "....chk no fallback"
+@@ -101,13 +131,13 @@ echo "a" | \
+ ip netns exec $ns \
+ ./mptcp_connect -p 10001 -l -s TCP -t ${timeout_poll} \
+ 0.0.0.0 >/dev/null &
+-sleep 0.1
++wait_local_port_listen $ns 10001
+ echo "b" | \
+ timeout ${timeout_test} \
+ ip netns exec $ns \
+- ./mptcp_connect -p 10001 -j -t ${timeout_poll} \
++ ./mptcp_connect -p 10001 -r 0 -t ${timeout_poll} \
+ 127.0.0.1 >/dev/null &
+-sleep 0.1
++wait_connected $ns 10001
+ chk_msk_fallback_nr 1 "check fallback"
+ flush_pids
+
+@@ -119,7 +149,7 @@ for I in `seq 1 $NR_CLIENTS`; do
+ ./mptcp_connect -p $((I+10001)) -l -w 10 \
+ -t ${timeout_poll} 0.0.0.0 >/dev/null &
+ done
+-sleep 0.1
++wait_local_port_listen $ns $((NR_CLIENTS + 10001))
+
+ for I in `seq 1 $NR_CLIENTS`; do
+ echo "b" | \
+diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
+index 559173a8e387b..d75fa97609c15 100755
+--- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh
++++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
+@@ -445,6 +445,8 @@ do_transfer()
+ local stat_ackrx_last_l=$(get_mib_counter "${listener_ns}" "MPTcpExtMPCapableACKRX")
+ local stat_cookietx_last=$(get_mib_counter "${listener_ns}" "TcpExtSyncookiesSent")
+ local stat_cookierx_last=$(get_mib_counter "${listener_ns}" "TcpExtSyncookiesRecv")
++ local stat_csum_err_s=$(get_mib_counter "${listener_ns}" "MPTcpExtDataCsumErr")
++ local stat_csum_err_c=$(get_mib_counter "${connector_ns}" "MPTcpExtDataCsumErr")
+
+ timeout ${timeout_test} \
+ ip netns exec ${listener_ns} \
+@@ -537,6 +539,23 @@ do_transfer()
+ fi
+ fi
+
++ if $checksum; then
++ local csum_err_s=$(get_mib_counter "${listener_ns}" "MPTcpExtDataCsumErr")
++ local csum_err_c=$(get_mib_counter "${connector_ns}" "MPTcpExtDataCsumErr")
++
++ local csum_err_s_nr=$((csum_err_s - stat_csum_err_s))
++ if [ $csum_err_s_nr -gt 0 ]; then
++ printf "[ FAIL ]\nserver got $csum_err_s_nr data checksum error[s]"
++ rets=1
++ fi
++
++ local csum_err_c_nr=$((csum_err_c - stat_csum_err_c))
++ if [ $csum_err_c_nr -gt 0 ]; then
++ printf "[ FAIL ]\nclient got $csum_err_c_nr data checksum error[s]"
++ retc=1
++ fi
++ fi
++
+ if [ $retc -eq 0 ] && [ $rets -eq 0 ]; then
+ printf "[ OK ]"
+ fi
+diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh
+index 255793c5ac4ff..3be615ab1588b 100755
+--- a/tools/testing/selftests/net/mptcp/mptcp_join.sh
++++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh
+@@ -75,6 +75,7 @@ init()
+
+ # let $ns2 reach any $ns1 address from any interface
+ ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
++ ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
+ done
+ }
+
+@@ -297,7 +298,7 @@ do_transfer()
+ if [ "$test_link_fail" -eq 2 ];then
+ timeout ${timeout_test} \
+ ip netns exec ${listener_ns} \
+- $mptcp_connect -t ${timeout_poll} -l -p $port -s ${cl_proto} \
++ $mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
+ ${local_addr} < "$sinfail" > "$sout" &
+ else
+ timeout ${timeout_test} \
+@@ -623,6 +624,7 @@ chk_join_nr()
+ local ack_nr=$4
+ local count
+ local dump_stats
++ local with_cookie
+
+ printf "%02u %-36s %s" "$TEST_COUNT" "$msg" "syn"
+ count=`ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}'`
+@@ -636,12 +638,20 @@ chk_join_nr()
+ fi
+
+ echo -n " - synack"
++ with_cookie=`ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies`
+ count=`ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}'`
+ [ -z "$count" ] && count=0
+ if [ "$count" != "$syn_ack_nr" ]; then
+- echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
+- ret=1
+- dump_stats=1
++ # simult connections exceeding the limit with cookie enabled could go up to
++ # synack validation as the conn limit can be enforced reliably only after
++ # the subflow creation
++ if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
++ echo -n "[ ok ]"
++ else
++ echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
++ ret=1
++ dump_stats=1
++ fi
+ else
+ echo -n "[ ok ]"
+ fi
+@@ -1383,7 +1393,7 @@ ipv6_tests()
+ reset
+ ip netns exec $ns1 ./pm_nl_ctl limits 0 1
+ ip netns exec $ns2 ./pm_nl_ctl limits 0 1
+- ip netns exec $ns2 ./pm_nl_ctl add dead:beef:3::2 flags subflow
++ ip netns exec $ns2 ./pm_nl_ctl add dead:beef:3::2 dev ns2eth3 flags subflow
+ run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
+ chk_join_nr "single subflow IPv6" 1 1 1
+
+@@ -1418,7 +1428,7 @@ ipv6_tests()
+ ip netns exec $ns1 ./pm_nl_ctl limits 0 2
+ ip netns exec $ns1 ./pm_nl_ctl add dead:beef:2::1 flags signal
+ ip netns exec $ns2 ./pm_nl_ctl limits 1 2
+- ip netns exec $ns2 ./pm_nl_ctl add dead:beef:3::2 flags subflow
++ ip netns exec $ns2 ./pm_nl_ctl add dead:beef:3::2 dev ns2eth3 flags subflow
+ run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
+ chk_join_nr "remove subflow and signal IPv6" 2 2 2
+ chk_add_nr 1 1
+diff --git a/tools/testing/selftests/net/pmtu.sh b/tools/testing/selftests/net/pmtu.sh
+index 543ad7513a8e9..694732e4b3448 100755
+--- a/tools/testing/selftests/net/pmtu.sh
++++ b/tools/testing/selftests/net/pmtu.sh
+@@ -374,6 +374,16 @@ run_cmd() {
+ return $rc
+ }
+
++run_cmd_bg() {
++ cmd="$*"
++
++ if [ "$VERBOSE" = "1" ]; then
++ printf " COMMAND: %s &\n" "${cmd}"
++ fi
++
++ $cmd 2>&1 &
++}
++
+ # Find the auto-generated name for this namespace
+ nsname() {
+ eval echo \$NS_$1
+@@ -670,10 +680,10 @@ setup_nettest_xfrm() {
+ [ ${1} -eq 6 ] && proto="-6" || proto=""
+ port=${2}
+
+- run_cmd ${ns_a} nettest ${proto} -q -D -s -x -p ${port} -t 5 &
++ run_cmd_bg "${ns_a}" nettest "${proto}" -q -D -s -x -p "${port}" -t 5
+ nettest_pids="${nettest_pids} $!"
+
+- run_cmd ${ns_b} nettest ${proto} -q -D -s -x -p ${port} -t 5 &
++ run_cmd_bg "${ns_b}" nettest "${proto}" -q -D -s -x -p "${port}" -t 5
+ nettest_pids="${nettest_pids} $!"
+ }
+
+@@ -865,7 +875,6 @@ setup_ovs_bridge() {
+ setup() {
+ [ "$(id -u)" -ne 0 ] && echo " need to run as root" && return $ksft_skip
+
+- cleanup
+ for arg do
+ eval setup_${arg} || { echo " ${arg} not supported"; return 1; }
+ done
+@@ -876,7 +885,7 @@ trace() {
+
+ for arg do
+ [ "${ns_cmd}" = "" ] && ns_cmd="${arg}" && continue
+- ${ns_cmd} tcpdump -s 0 -i "${arg}" -w "${name}_${arg}.pcap" 2> /dev/null &
++ ${ns_cmd} tcpdump --immediate-mode -s 0 -i "${arg}" -w "${name}_${arg}.pcap" 2> /dev/null &
+ tcpdump_pids="${tcpdump_pids} $!"
+ ns_cmd=
+ done
+@@ -1836,6 +1845,10 @@ run_test() {
+
+ unset IFS
+
++ # Since cleanup() relies on variables modified by this subshell, it
++ # has to run in this context.
++ trap cleanup EXIT
++
+ if [ "$VERBOSE" = "1" ]; then
+ printf "\n##########################################################################\n\n"
+ fi
+diff --git a/tools/testing/selftests/net/so_txtime.c b/tools/testing/selftests/net/so_txtime.c
+index 59067f64b7753..2672ac0b6d1f3 100644
+--- a/tools/testing/selftests/net/so_txtime.c
++++ b/tools/testing/selftests/net/so_txtime.c
+@@ -421,7 +421,7 @@ static void usage(const char *progname)
+ "Options:\n"
+ " -4 only IPv4\n"
+ " -6 only IPv6\n"
+- " -c <clock> monotonic (default) or tai\n"
++ " -c <clock> monotonic or tai (default)\n"
+ " -D <addr> destination IP address (server)\n"
+ " -S <addr> source IP address (client)\n"
+ " -r run rx mode\n"
+@@ -475,7 +475,7 @@ static void parse_opts(int argc, char **argv)
+ cfg_rx = true;
+ break;
+ case 't':
+- cfg_start_time_ns = strtol(optarg, NULL, 0);
++ cfg_start_time_ns = strtoll(optarg, NULL, 0);
+ break;
+ case 'm':
+ cfg_mark = strtol(optarg, NULL, 0);
+diff --git a/tools/testing/selftests/net/test_vxlan_under_vrf.sh b/tools/testing/selftests/net/test_vxlan_under_vrf.sh
+index 534c8b7699ab9..6fadc8e2f116a 100755
+--- a/tools/testing/selftests/net/test_vxlan_under_vrf.sh
++++ b/tools/testing/selftests/net/test_vxlan_under_vrf.sh
+@@ -118,11 +118,11 @@ echo "[ OK ]"
+
+ # Move the underlay to a non-default VRF
+ ip -netns hv-1 link set veth0 vrf vrf-underlay
+-ip -netns hv-1 link set veth0 down
+-ip -netns hv-1 link set veth0 up
++ip -netns hv-1 link set vxlan0 down
++ip -netns hv-1 link set vxlan0 up
+ ip -netns hv-2 link set veth0 vrf vrf-underlay
+-ip -netns hv-2 link set veth0 down
+-ip -netns hv-2 link set veth0 up
++ip -netns hv-2 link set vxlan0 down
++ip -netns hv-2 link set vxlan0 up
+
+ echo -n "Check VM connectivity through VXLAN (underlay in a VRF) "
+ ip netns exec vm-1 ping -c 1 -W 1 10.0.0.2 &> /dev/null || (echo "[FAIL]"; false)
+diff --git a/tools/testing/selftests/net/timestamping.c b/tools/testing/selftests/net/timestamping.c
+index aee631c5284eb..044bc0e9ed81a 100644
+--- a/tools/testing/selftests/net/timestamping.c
++++ b/tools/testing/selftests/net/timestamping.c
+@@ -325,8 +325,8 @@ int main(int argc, char **argv)
+ struct ifreq device;
+ struct ifreq hwtstamp;
+ struct hwtstamp_config hwconfig, hwconfig_requested;
+- struct so_timestamping so_timestamping_get = { 0, -1 };
+- struct so_timestamping so_timestamping = { 0, -1 };
++ struct so_timestamping so_timestamping_get = { 0, 0 };
++ struct so_timestamping so_timestamping = { 0, 0 };
+ struct sockaddr_in addr;
+ struct ip_mreq imr;
+ struct in_addr iaddr;
+diff --git a/tools/testing/selftests/net/toeplitz.c b/tools/testing/selftests/net/toeplitz.c
+index 710ac956bdb33..c5489341cfb80 100644
+--- a/tools/testing/selftests/net/toeplitz.c
++++ b/tools/testing/selftests/net/toeplitz.c
+@@ -498,7 +498,7 @@ static void parse_opts(int argc, char **argv)
+ bool have_toeplitz = false;
+ int index, c;
+
+- while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:u:v", long_options, &index)) != -1) {
++ while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) {
+ switch (c) {
+ case '4':
+ cfg_family = AF_INET;
+diff --git a/tools/testing/selftests/net/udpgro_fwd.sh b/tools/testing/selftests/net/udpgro_fwd.sh
+index 7f26591f236b9..6f05e06f67613 100755
+--- a/tools/testing/selftests/net/udpgro_fwd.sh
++++ b/tools/testing/selftests/net/udpgro_fwd.sh
+@@ -132,7 +132,7 @@ run_test() {
+ local rcv=`ip netns exec $NS_DST $ipt"-save" -c | grep 'dport 8000' | \
+ sed -e 's/\[//' -e 's/:.*//'`
+ if [ $rcv != $pkts ]; then
+- echo " fail - received $rvs packets, expected $pkts"
++ echo " fail - received $rcv packets, expected $pkts"
+ ret=1
+ return
+ fi
+@@ -185,6 +185,7 @@ for family in 4 6; do
+ IPT=iptables
+ SUFFIX=24
+ VXDEV=vxlan
++ PING=ping
+
+ if [ $family = 6 ]; then
+ BM_NET=$BM_NET_V6
+@@ -192,6 +193,8 @@ for family in 4 6; do
+ SUFFIX="64 nodad"
+ VXDEV=vxlan6
+ IPT=ip6tables
++ # Use ping6 on systems where ping doesn't handle IPv6
++ ping -w 1 -c 1 ::1 > /dev/null 2>&1 || PING="ping6"
+ fi
+
+ echo "IPv$family"
+@@ -237,7 +240,7 @@ for family in 4 6; do
+
+ # load arp cache before running the test to reduce the amount of
+ # stray traffic on top of the UDP tunnel
+- ip netns exec $NS_SRC ping -q -c 1 $OL_NET$DST_NAT >/dev/null
++ ip netns exec $NS_SRC $PING -q -c 1 $OL_NET$DST_NAT >/dev/null
+ run_test "GRO fwd over UDP tunnel" $OL_NET$DST_NAT 1 1 $OL_NET$DST
+ cleanup
+
+diff --git a/tools/testing/selftests/net/udpgso.c b/tools/testing/selftests/net/udpgso.c
+index c66da6ffd6d8d..7badaf215de28 100644
+--- a/tools/testing/selftests/net/udpgso.c
++++ b/tools/testing/selftests/net/udpgso.c
+@@ -156,13 +156,13 @@ struct testcase testcases_v4[] = {
+ },
+ {
+ /* send max number of min sized segments */
+- .tlen = UDP_MAX_SEGMENTS - CONST_HDRLEN_V4,
++ .tlen = UDP_MAX_SEGMENTS,
+ .gso_len = 1,
+- .r_num_mss = UDP_MAX_SEGMENTS - CONST_HDRLEN_V4,
++ .r_num_mss = UDP_MAX_SEGMENTS,
+ },
+ {
+ /* send max number + 1 of min sized segments: fail */
+- .tlen = UDP_MAX_SEGMENTS - CONST_HDRLEN_V4 + 1,
++ .tlen = UDP_MAX_SEGMENTS + 1,
+ .gso_len = 1,
+ .tfail = true,
+ },
+@@ -259,13 +259,13 @@ struct testcase testcases_v6[] = {
+ },
+ {
+ /* send max number of min sized segments */
+- .tlen = UDP_MAX_SEGMENTS - CONST_HDRLEN_V6,
++ .tlen = UDP_MAX_SEGMENTS,
+ .gso_len = 1,
+- .r_num_mss = UDP_MAX_SEGMENTS - CONST_HDRLEN_V6,
++ .r_num_mss = UDP_MAX_SEGMENTS,
+ },
+ {
+ /* send max number + 1 of min sized segments: fail */
+- .tlen = UDP_MAX_SEGMENTS - CONST_HDRLEN_V6 + 1,
++ .tlen = UDP_MAX_SEGMENTS + 1,
+ .gso_len = 1,
+ .tfail = true,
+ },
+diff --git a/tools/testing/selftests/net/udpgso_bench_rx.c b/tools/testing/selftests/net/udpgso_bench_rx.c
+index 76a24052f4b47..6a193425c367f 100644
+--- a/tools/testing/selftests/net/udpgso_bench_rx.c
++++ b/tools/testing/selftests/net/udpgso_bench_rx.c
+@@ -293,19 +293,17 @@ static void usage(const char *filepath)
+
+ static void parse_opts(int argc, char **argv)
+ {
++ const char *bind_addr = NULL;
+ int c;
+
+- /* bind to any by default */
+- setup_sockaddr(PF_INET6, "::", &cfg_bind_addr);
+ while ((c = getopt(argc, argv, "4b:C:Gl:n:p:rR:S:tv")) != -1) {
+ switch (c) {
+ case '4':
+ cfg_family = PF_INET;
+ cfg_alen = sizeof(struct sockaddr_in);
+- setup_sockaddr(PF_INET, "0.0.0.0", &cfg_bind_addr);
+ break;
+ case 'b':
+- setup_sockaddr(cfg_family, optarg, &cfg_bind_addr);
++ bind_addr = optarg;
+ break;
+ case 'C':
+ cfg_connect_timeout_ms = strtoul(optarg, NULL, 0);
+@@ -341,6 +339,11 @@ static void parse_opts(int argc, char **argv)
+ }
+ }
+
++ if (!bind_addr)
++ bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0";
++
++ setup_sockaddr(cfg_family, bind_addr, &cfg_bind_addr);
++
+ if (optind != argc)
+ usage(argv[0]);
+
+diff --git a/tools/testing/selftests/net/udpgso_bench_tx.c b/tools/testing/selftests/net/udpgso_bench_tx.c
+index 17512a43885e7..f1fdaa2702913 100644
+--- a/tools/testing/selftests/net/udpgso_bench_tx.c
++++ b/tools/testing/selftests/net/udpgso_bench_tx.c
+@@ -419,6 +419,7 @@ static void usage(const char *filepath)
+
+ static void parse_opts(int argc, char **argv)
+ {
++ const char *bind_addr = NULL;
+ int max_len, hdrlen;
+ int c;
+
+@@ -446,7 +447,7 @@ static void parse_opts(int argc, char **argv)
+ cfg_cpu = strtol(optarg, NULL, 0);
+ break;
+ case 'D':
+- setup_sockaddr(cfg_family, optarg, &cfg_dst_addr);
++ bind_addr = optarg;
+ break;
+ case 'l':
+ cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000;
+@@ -492,6 +493,11 @@ static void parse_opts(int argc, char **argv)
+ }
+ }
+
++ if (!bind_addr)
++ bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0";
++
++ setup_sockaddr(cfg_family, bind_addr, &cfg_dst_addr);
++
+ if (optind != argc)
+ usage(argv[0]);
+
+diff --git a/tools/testing/selftests/netfilter/Makefile b/tools/testing/selftests/netfilter/Makefile
+index 8748199ac1098..ffca314897c4c 100644
+--- a/tools/testing/selftests/netfilter/Makefile
++++ b/tools/testing/selftests/netfilter/Makefile
+@@ -5,7 +5,8 @@ TEST_PROGS := nft_trans_stress.sh nft_fib.sh nft_nat.sh bridge_brouter.sh \
+ conntrack_icmp_related.sh nft_flowtable.sh ipvs.sh \
+ nft_concat_range.sh nft_conntrack_helper.sh \
+ nft_queue.sh nft_meta.sh nf_nat_edemux.sh \
+- ipip-conntrack-mtu.sh conntrack_tcp_unreplied.sh
++ ipip-conntrack-mtu.sh conntrack_tcp_unreplied.sh \
++ conntrack_vrf.sh
+
+ LDLIBS = -lmnl
+ TEST_GEN_FILES = nf-queue
+diff --git a/tools/testing/selftests/netfilter/conntrack_vrf.sh b/tools/testing/selftests/netfilter/conntrack_vrf.sh
+new file mode 100755
+index 0000000000000..8b5ea92345882
+--- /dev/null
++++ b/tools/testing/selftests/netfilter/conntrack_vrf.sh
+@@ -0,0 +1,241 @@
++#!/bin/sh
++
++# This script demonstrates interaction of conntrack and vrf.
++# The vrf driver calls the netfilter hooks again, with oif/iif
++# pointing at the VRF device.
++#
++# For ingress, this means first iteration has iifname of lower/real
++# device. In this script, thats veth0.
++# Second iteration is iifname set to vrf device, tvrf in this script.
++#
++# For egress, this is reversed: first iteration has the vrf device,
++# second iteration is done with the lower/real/veth0 device.
++#
++# test_ct_zone_in demonstrates unexpected change of nftables
++# behavior # caused by commit 09e856d54bda5f28 "vrf: Reset skb conntrack
++# connection on VRF rcv"
++#
++# It was possible to assign conntrack zone to a packet (or mark it for
++# `notracking`) in the prerouting chain before conntrack, based on real iif.
++#
++# After the change, the zone assignment is lost and the zone is assigned based
++# on the VRF master interface (in case such a rule exists).
++# assignment is lost. Instead, assignment based on the `iif` matching
++# Thus it is impossible to distinguish packets based on the original
++# interface.
++#
++# test_masquerade_vrf and test_masquerade_veth0 demonstrate the problem
++# that was supposed to be fixed by the commit mentioned above to make sure
++# that any fix to test case 1 won't break masquerade again.
++
++ksft_skip=4
++
++IP0=172.30.30.1
++IP1=172.30.30.2
++PFXL=30
++ret=0
++
++sfx=$(mktemp -u "XXXXXXXX")
++ns0="ns0-$sfx"
++ns1="ns1-$sfx"
++
++cleanup()
++{
++ ip netns pids $ns0 | xargs kill 2>/dev/null
++ ip netns pids $ns1 | xargs kill 2>/dev/null
++
++ ip netns del $ns0 $ns1
++}
++
++nft --version > /dev/null 2>&1
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not run test without nft tool"
++ exit $ksft_skip
++fi
++
++ip -Version > /dev/null 2>&1
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not run test without ip tool"
++ exit $ksft_skip
++fi
++
++ip netns add "$ns0"
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not create net namespace $ns0"
++ exit $ksft_skip
++fi
++ip netns add "$ns1"
++
++trap cleanup EXIT
++
++ip netns exec $ns0 sysctl -q -w net.ipv4.conf.default.rp_filter=0
++ip netns exec $ns0 sysctl -q -w net.ipv4.conf.all.rp_filter=0
++ip netns exec $ns0 sysctl -q -w net.ipv4.conf.all.rp_filter=0
++
++ip link add veth0 netns "$ns0" type veth peer name veth0 netns "$ns1" > /dev/null 2>&1
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not add veth device"
++ exit $ksft_skip
++fi
++
++ip -net $ns0 li add tvrf type vrf table 9876
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not add vrf device"
++ exit $ksft_skip
++fi
++
++ip -net $ns0 li set lo up
++
++ip -net $ns0 li set veth0 master tvrf
++ip -net $ns0 li set tvrf up
++ip -net $ns0 li set veth0 up
++ip -net $ns1 li set veth0 up
++
++ip -net $ns0 addr add $IP0/$PFXL dev veth0
++ip -net $ns1 addr add $IP1/$PFXL dev veth0
++
++ip netns exec $ns1 iperf3 -s > /dev/null 2>&1&
++if [ $? -ne 0 ];then
++ echo "SKIP: Could not start iperf3"
++ exit $ksft_skip
++fi
++
++# test vrf ingress handling.
++# The incoming connection should be placed in conntrack zone 1,
++# as decided by the first iteration of the ruleset.
++test_ct_zone_in()
++{
++ip netns exec $ns0 nft -f - <<EOF
++table testct {
++ chain rawpre {
++ type filter hook prerouting priority raw;
++
++ iif { veth0, tvrf } counter meta nftrace set 1
++ iif veth0 counter ct zone set 1 counter return
++ iif tvrf counter ct zone set 2 counter return
++ ip protocol icmp counter
++ notrack counter
++ }
++
++ chain rawout {
++ type filter hook output priority raw;
++
++ oif veth0 counter ct zone set 1 counter return
++ oif tvrf counter ct zone set 2 counter return
++ notrack counter
++ }
++}
++EOF
++ ip netns exec $ns1 ping -W 1 -c 1 -I veth0 $IP0 > /dev/null
++
++ # should be in zone 1, not zone 2
++ count=$(ip netns exec $ns0 conntrack -L -s $IP1 -d $IP0 -p icmp --zone 1 2>/dev/null | wc -l)
++ if [ $count -eq 1 ]; then
++ echo "PASS: entry found in conntrack zone 1"
++ else
++ echo "FAIL: entry not found in conntrack zone 1"
++ count=$(ip netns exec $ns0 conntrack -L -s $IP1 -d $IP0 -p icmp --zone 2 2> /dev/null | wc -l)
++ if [ $count -eq 1 ]; then
++ echo "FAIL: entry found in zone 2 instead"
++ else
++ echo "FAIL: entry not in zone 1 or 2, dumping table"
++ ip netns exec $ns0 conntrack -L
++ ip netns exec $ns0 nft list ruleset
++ fi
++ fi
++}
++
++# add masq rule that gets evaluated w. outif set to vrf device.
++# This tests the first iteration of the packet through conntrack,
++# oifname is the vrf device.
++test_masquerade_vrf()
++{
++ local qdisc=$1
++
++ if [ "$qdisc" != "default" ]; then
++ tc -net $ns0 qdisc add dev tvrf root $qdisc
++ fi
++
++ ip netns exec $ns0 conntrack -F 2>/dev/null
++
++ip netns exec $ns0 nft -f - <<EOF
++flush ruleset
++table ip nat {
++ chain rawout {
++ type filter hook output priority raw;
++
++ oif tvrf ct state untracked counter
++ }
++ chain postrouting2 {
++ type filter hook postrouting priority mangle;
++
++ oif tvrf ct state untracked counter
++ }
++ chain postrouting {
++ type nat hook postrouting priority 0;
++ # NB: masquerade should always be combined with 'oif(name) bla',
++ # lack of this is intentional here, we want to exercise double-snat.
++ ip saddr 172.30.30.0/30 counter masquerade random
++ }
++}
++EOF
++ ip netns exec $ns0 ip vrf exec tvrf iperf3 -t 1 -c $IP1 >/dev/null
++ if [ $? -ne 0 ]; then
++ echo "FAIL: iperf3 connect failure with masquerade + sport rewrite on vrf device"
++ ret=1
++ return
++ fi
++
++ # must also check that nat table was evaluated on second (lower device) iteration.
++ ip netns exec $ns0 nft list table ip nat |grep -q 'counter packets 2' &&
++ ip netns exec $ns0 nft list table ip nat |grep -q 'untracked counter packets [1-9]'
++ if [ $? -eq 0 ]; then
++ echo "PASS: iperf3 connect with masquerade + sport rewrite on vrf device ($qdisc qdisc)"
++ else
++ echo "FAIL: vrf rules have unexpected counter value"
++ ret=1
++ fi
++
++ if [ "$qdisc" != "default" ]; then
++ tc -net $ns0 qdisc del dev tvrf root
++ fi
++}
++
++# add masq rule that gets evaluated w. outif set to veth device.
++# This tests the 2nd iteration of the packet through conntrack,
++# oifname is the lower device (veth0 in this case).
++test_masquerade_veth()
++{
++ ip netns exec $ns0 conntrack -F 2>/dev/null
++ip netns exec $ns0 nft -f - <<EOF
++flush ruleset
++table ip nat {
++ chain postrouting {
++ type nat hook postrouting priority 0;
++ meta oif veth0 ip saddr 172.30.30.0/30 counter masquerade random
++ }
++}
++EOF
++ ip netns exec $ns0 ip vrf exec tvrf iperf3 -t 1 -c $IP1 > /dev/null
++ if [ $? -ne 0 ]; then
++ echo "FAIL: iperf3 connect failure with masquerade + sport rewrite on veth device"
++ ret=1
++ return
++ fi
++
++ # must also check that nat table was evaluated on second (lower device) iteration.
++ ip netns exec $ns0 nft list table ip nat |grep -q 'counter packets 2'
++ if [ $? -eq 0 ]; then
++ echo "PASS: iperf3 connect with masquerade + sport rewrite on veth device"
++ else
++ echo "FAIL: vrf masq rule has unexpected counter value"
++ ret=1
++ fi
++}
++
++test_ct_zone_in
++test_masquerade_vrf "default"
++test_masquerade_vrf "pfifo"
++test_masquerade_veth
++
++exit $ret
+diff --git a/tools/testing/selftests/netfilter/nft_concat_range.sh b/tools/testing/selftests/netfilter/nft_concat_range.sh
+index 5a4938d6dcf25..b5eef5ffb58e5 100755
+--- a/tools/testing/selftests/netfilter/nft_concat_range.sh
++++ b/tools/testing/selftests/netfilter/nft_concat_range.sh
+@@ -27,7 +27,7 @@ TYPES="net_port port_net net6_port port_proto net6_port_mac net6_port_mac_proto
+ net_port_mac_proto_net"
+
+ # Reported bugs, also described by TYPE_ variables below
+-BUGS="flush_remove_add"
++BUGS="flush_remove_add reload"
+
+ # List of possible paths to pktgen script from kernel tree for performance tests
+ PKTGEN_SCRIPT_PATHS="
+@@ -337,6 +337,23 @@ TYPE_flush_remove_add="
+ display Add two elements, flush, re-add
+ "
+
++TYPE_reload="
++display net,mac with reload
++type_spec ipv4_addr . ether_addr
++chain_spec ip daddr . ether saddr
++dst addr4
++src mac
++start 1
++count 1
++src_delta 2000
++tools sendip nc bash
++proto udp
++
++race_repeat 0
++
++perf_duration 0
++"
++
+ # Set template for all tests, types and rules are filled in depending on test
+ set_template='
+ flush ruleset
+@@ -1455,6 +1472,59 @@ test_bug_flush_remove_add() {
+ nft flush ruleset
+ }
+
++# - add ranged element, check that packets match it
++# - reload the set, check packets still match
++test_bug_reload() {
++ setup veth send_"${proto}" set || return ${KSELFTEST_SKIP}
++ rstart=${start}
++
++ range_size=1
++ for i in $(seq "${start}" $((start + count))); do
++ end=$((start + range_size))
++
++ # Avoid negative or zero-sized port ranges
++ if [ $((end / 65534)) -gt $((start / 65534)) ]; then
++ start=${end}
++ end=$((end + 1))
++ fi
++ srcstart=$((start + src_delta))
++ srcend=$((end + src_delta))
++
++ add "$(format)" || return 1
++ range_size=$((range_size + 1))
++ start=$((end + range_size))
++ done
++
++ # check kernel does allocate pcpu sctrach map
++ # for reload with no elemet add/delete
++ ( echo flush set inet filter test ;
++ nft list set inet filter test ) | nft -f -
++
++ start=${rstart}
++ range_size=1
++
++ for i in $(seq "${start}" $((start + count))); do
++ end=$((start + range_size))
++
++ # Avoid negative or zero-sized port ranges
++ if [ $((end / 65534)) -gt $((start / 65534)) ]; then
++ start=${end}
++ end=$((end + 1))
++ fi
++ srcstart=$((start + src_delta))
++ srcend=$((end + src_delta))
++
++ for j in $(seq ${start} $((range_size / 2 + 1)) ${end}); do
++ send_match "${j}" $((j + src_delta)) || return 1
++ done
++
++ range_size=$((range_size + 1))
++ start=$((end + range_size))
++ done
++
++ nft flush ruleset
++}
++
+ test_reported_issues() {
+ eval test_bug_"${subtest}"
+ }
+@@ -1513,4 +1583,4 @@ for name in ${TESTS}; do
+ done
+ done
+
+-[ ${passed} -eq 0 ] && exit ${KSELFTEST_SKIP}
++[ ${passed} -eq 0 ] && exit ${KSELFTEST_SKIP} || exit 0
+diff --git a/tools/testing/selftests/netfilter/nft_fib.sh b/tools/testing/selftests/netfilter/nft_fib.sh
+index 6caf6ac8c285f..695a1958723f5 100755
+--- a/tools/testing/selftests/netfilter/nft_fib.sh
++++ b/tools/testing/selftests/netfilter/nft_fib.sh
+@@ -174,6 +174,7 @@ test_ping() {
+ ip netns exec ${nsrouter} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
+ ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.forwarding=1 > /dev/null
+ ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth1.forwarding=1 > /dev/null
++ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.rp_filter=0 > /dev/null
+
+ sleep 3
+
+diff --git a/tools/testing/selftests/netfilter/nft_nat.sh b/tools/testing/selftests/netfilter/nft_nat.sh
+index da1c1e4b6c86b..781fa2d9ea9d5 100755
+--- a/tools/testing/selftests/netfilter/nft_nat.sh
++++ b/tools/testing/selftests/netfilter/nft_nat.sh
+@@ -885,6 +885,144 @@ EOF
+ ip netns exec "$ns0" nft delete table $family nat
+ }
+
++test_stateless_nat_ip()
++{
++ local lret=0
++
++ ip netns exec "$ns0" sysctl net.ipv4.conf.veth0.forwarding=1 > /dev/null
++ ip netns exec "$ns0" sysctl net.ipv4.conf.veth1.forwarding=1 > /dev/null
++
++ ip netns exec "$ns2" ping -q -c 1 10.0.1.99 > /dev/null # ping ns2->ns1
++ if [ $? -ne 0 ] ; then
++ echo "ERROR: cannot ping $ns1 from $ns2 before loading stateless rules"
++ return 1
++ fi
++
++ip netns exec "$ns0" nft -f /dev/stdin <<EOF
++table ip stateless {
++ map xlate_in {
++ typeof meta iifname . ip saddr . ip daddr : ip daddr
++ elements = {
++ "veth1" . 10.0.2.99 . 10.0.1.99 : 10.0.2.2,
++ }
++ }
++ map xlate_out {
++ typeof meta iifname . ip saddr . ip daddr : ip daddr
++ elements = {
++ "veth0" . 10.0.1.99 . 10.0.2.2 : 10.0.2.99
++ }
++ }
++
++ chain prerouting {
++ type filter hook prerouting priority -400; policy accept;
++ ip saddr set meta iifname . ip saddr . ip daddr map @xlate_in
++ ip daddr set meta iifname . ip saddr . ip daddr map @xlate_out
++ }
++}
++EOF
++ if [ $? -ne 0 ]; then
++ echo "SKIP: Could not add ip statless rules"
++ return $ksft_skip
++ fi
++
++ reset_counters
++
++ ip netns exec "$ns2" ping -q -c 1 10.0.1.99 > /dev/null # ping ns2->ns1
++ if [ $? -ne 0 ] ; then
++ echo "ERROR: cannot ping $ns1 from $ns2 with stateless rules"
++ lret=1
++ fi
++
++ # ns1 should have seen packets from .2.2, due to stateless rewrite.
++ expect="packets 1 bytes 84"
++ cnt=$(ip netns exec "$ns1" nft list counter inet filter ns0insl | grep -q "$expect")
++ if [ $? -ne 0 ]; then
++ bad_counter "$ns1" ns0insl "$expect" "test_stateless 1"
++ lret=1
++ fi
++
++ for dir in "in" "out" ; do
++ cnt=$(ip netns exec "$ns2" nft list counter inet filter ns1${dir} | grep -q "$expect")
++ if [ $? -ne 0 ]; then
++ bad_counter "$ns2" ns1$dir "$expect" "test_stateless 2"
++ lret=1
++ fi
++ done
++
++ # ns1 should not have seen packets from ns2, due to masquerade
++ expect="packets 0 bytes 0"
++ for dir in "in" "out" ; do
++ cnt=$(ip netns exec "$ns1" nft list counter inet filter ns2${dir} | grep -q "$expect")
++ if [ $? -ne 0 ]; then
++ bad_counter "$ns1" ns0$dir "$expect" "test_stateless 3"
++ lret=1
++ fi
++
++ cnt=$(ip netns exec "$ns0" nft list counter inet filter ns1${dir} | grep -q "$expect")
++ if [ $? -ne 0 ]; then
++ bad_counter "$ns0" ns1$dir "$expect" "test_stateless 4"
++ lret=1
++ fi
++ done
++
++ reset_counters
++
++ socat -h > /dev/null 2>&1
++ if [ $? -ne 0 ];then
++ echo "SKIP: Could not run stateless nat frag test without socat tool"
++ if [ $lret -eq 0 ]; then
++ return $ksft_skip
++ fi
++
++ ip netns exec "$ns0" nft delete table ip stateless
++ return $lret
++ fi
++
++ local tmpfile=$(mktemp)
++ dd if=/dev/urandom of=$tmpfile bs=4096 count=1 2>/dev/null
++
++ local outfile=$(mktemp)
++ ip netns exec "$ns1" timeout 3 socat -u UDP4-RECV:4233 OPEN:$outfile < /dev/null &
++ sc_r=$!
++
++ sleep 1
++ # re-do with large ping -> ip fragmentation
++ ip netns exec "$ns2" timeout 3 socat - UDP4-SENDTO:"10.0.1.99:4233" < "$tmpfile" > /dev/null
++ if [ $? -ne 0 ] ; then
++ echo "ERROR: failed to test udp $ns1 to $ns2 with stateless ip nat" 1>&2
++ lret=1
++ fi
++
++ wait
++
++ cmp "$tmpfile" "$outfile"
++ if [ $? -ne 0 ]; then
++ ls -l "$tmpfile" "$outfile"
++ echo "ERROR: in and output file mismatch when checking udp with stateless nat" 1>&2
++ lret=1
++ fi
++
++ rm -f "$tmpfile" "$outfile"
++
++ # ns1 should have seen packets from 2.2, due to stateless rewrite.
++ expect="packets 3 bytes 4164"
++ cnt=$(ip netns exec "$ns1" nft list counter inet filter ns0insl | grep -q "$expect")
++ if [ $? -ne 0 ]; then
++ bad_counter "$ns1" ns0insl "$expect" "test_stateless 5"
++ lret=1
++ fi
++
++ ip netns exec "$ns0" nft delete table ip stateless
++ if [ $? -ne 0 ]; then
++ echo "ERROR: Could not delete table ip stateless" 1>&2
++ lret=1
++ fi
++
++ test $lret -eq 0 && echo "PASS: IP statless for $ns2"
++
++ return $lret
++}
++
+ # ip netns exec "$ns0" ping -c 1 -q 10.0.$i.99
+ for i in 0 1 2; do
+ ip netns exec ns$i-$sfx nft -f /dev/stdin <<EOF
+@@ -951,6 +1089,19 @@ table inet filter {
+ EOF
+ done
+
++# special case for stateless nat check, counter needs to
++# be done before (input) ip defragmentation
++ip netns exec ns1-$sfx nft -f /dev/stdin <<EOF
++table inet filter {
++ counter ns0insl {}
++
++ chain pre {
++ type filter hook prerouting priority -400; policy accept;
++ ip saddr 10.0.2.2 counter name "ns0insl"
++ }
++}
++EOF
++
+ sleep 3
+ # test basic connectivity
+ for i in 1 2; do
+@@ -1005,6 +1156,7 @@ $test_inet_nat && test_redirect inet
+ $test_inet_nat && test_redirect6 inet
+
+ test_port_shadowing
++test_stateless_nat_ip
+
+ if [ $ret -ne 0 ];then
+ echo -n "FAIL: "
+diff --git a/tools/testing/selftests/openat2/Makefile b/tools/testing/selftests/openat2/Makefile
+index 4b93b1417b862..843ba56d8e49e 100644
+--- a/tools/testing/selftests/openat2/Makefile
++++ b/tools/testing/selftests/openat2/Makefile
+@@ -5,4 +5,4 @@ TEST_GEN_PROGS := openat2_test resolve_test rename_attack_test
+
+ include ../lib.mk
+
+-$(TEST_GEN_PROGS): helpers.c
++$(TEST_GEN_PROGS): helpers.c helpers.h
+diff --git a/tools/testing/selftests/openat2/helpers.h b/tools/testing/selftests/openat2/helpers.h
+index a6ea27344db2d..7056340b9339e 100644
+--- a/tools/testing/selftests/openat2/helpers.h
++++ b/tools/testing/selftests/openat2/helpers.h
+@@ -9,6 +9,7 @@
+
+ #define _GNU_SOURCE
+ #include <stdint.h>
++#include <stdbool.h>
+ #include <errno.h>
+ #include <linux/types.h>
+ #include "../kselftest.h"
+@@ -62,11 +63,12 @@ bool needs_openat2(const struct open_how *how);
+ (similar to chroot(2)). */
+ #endif /* RESOLVE_IN_ROOT */
+
+-#define E_func(func, ...) \
+- do { \
+- if (func(__VA_ARGS__) < 0) \
+- ksft_exit_fail_msg("%s:%d %s failed\n", \
+- __FILE__, __LINE__, #func);\
++#define E_func(func, ...) \
++ do { \
++ errno = 0; \
++ if (func(__VA_ARGS__) < 0) \
++ ksft_exit_fail_msg("%s:%d %s failed - errno:%d\n", \
++ __FILE__, __LINE__, #func, errno); \
+ } while (0)
+
+ #define E_asprintf(...) E_func(asprintf, __VA_ARGS__)
+diff --git a/tools/testing/selftests/openat2/openat2_test.c b/tools/testing/selftests/openat2/openat2_test.c
+index 1bddbe934204c..7fb902099de45 100644
+--- a/tools/testing/selftests/openat2/openat2_test.c
++++ b/tools/testing/selftests/openat2/openat2_test.c
+@@ -259,6 +259,16 @@ void test_openat2_flags(void)
+ unlink(path);
+
+ fd = sys_openat2(AT_FDCWD, path, &test->how);
++ if (fd < 0 && fd == -EOPNOTSUPP) {
++ /*
++ * Skip the testcase if it failed because not supported
++ * by FS. (e.g. a valid O_TMPFILE combination on NFS)
++ */
++ ksft_test_result_skip("openat2 with %s fails with %d (%s)\n",
++ test->name, fd, strerror(-fd));
++ goto next;
++ }
++
+ if (test->err >= 0)
+ failed = (fd < 0);
+ else
+@@ -303,7 +313,7 @@ skip:
+ else
+ resultfn("openat2 with %s fails with %d (%s)\n",
+ test->name, test->err, strerror(-test->err));
+-
++next:
+ free(fdpath);
+ fflush(stdout);
+ }
+diff --git a/tools/testing/selftests/pidfd/pidfd.h b/tools/testing/selftests/pidfd/pidfd.h
+index 01f8d3c0cf2cb..6922d6417e1cf 100644
+--- a/tools/testing/selftests/pidfd/pidfd.h
++++ b/tools/testing/selftests/pidfd/pidfd.h
+@@ -68,7 +68,7 @@
+ #define PIDFD_SKIP 3
+ #define PIDFD_XFAIL 4
+
+-int wait_for_pid(pid_t pid)
++static inline int wait_for_pid(pid_t pid)
+ {
+ int status, ret;
+
+@@ -78,13 +78,20 @@ again:
+ if (errno == EINTR)
+ goto again;
+
++ ksft_print_msg("waitpid returned -1, errno=%d\n", errno);
+ return -1;
+ }
+
+- if (!WIFEXITED(status))
++ if (!WIFEXITED(status)) {
++ ksft_print_msg(
++ "waitpid !WIFEXITED, WIFSIGNALED=%d, WTERMSIG=%d\n",
++ WIFSIGNALED(status), WTERMSIG(status));
+ return -1;
++ }
+
+- return WEXITSTATUS(status);
++ ret = WEXITSTATUS(status);
++ ksft_print_msg("waitpid WEXITSTATUS=%d\n", ret);
++ return ret;
+ }
+
+ static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
+diff --git a/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c b/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c
+index 22558524f71c3..3fd8e903118f5 100644
+--- a/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c
++++ b/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c
+@@ -12,6 +12,7 @@
+ #include <string.h>
+ #include <syscall.h>
+ #include <sys/wait.h>
++#include <sys/mman.h>
+
+ #include "pidfd.h"
+ #include "../kselftest.h"
+@@ -80,7 +81,10 @@ static inline int error_check(struct error *err, const char *test_name)
+ return err->code;
+ }
+
++#define CHILD_STACK_SIZE 8192
++
+ struct child {
++ char *stack;
+ pid_t pid;
+ int fd;
+ };
+@@ -89,17 +93,22 @@ static struct child clone_newns(int (*fn)(void *), void *args,
+ struct error *err)
+ {
+ static int flags = CLONE_PIDFD | CLONE_NEWPID | CLONE_NEWNS | SIGCHLD;
+- size_t stack_size = 1024;
+- char *stack[1024] = { 0 };
+ struct child ret;
+
+ if (!(flags & CLONE_NEWUSER) && geteuid() != 0)
+ flags |= CLONE_NEWUSER;
+
++ ret.stack = mmap(NULL, CHILD_STACK_SIZE, PROT_READ | PROT_WRITE,
++ MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
++ if (ret.stack == MAP_FAILED) {
++ error_set(err, -1, "mmap of stack failed (errno %d)", errno);
++ return ret;
++ }
++
+ #ifdef __ia64__
+- ret.pid = __clone2(fn, stack, stack_size, flags, args, &ret.fd);
++ ret.pid = __clone2(fn, ret.stack, CHILD_STACK_SIZE, flags, args, &ret.fd);
+ #else
+- ret.pid = clone(fn, stack + stack_size, flags, args, &ret.fd);
++ ret.pid = clone(fn, ret.stack + CHILD_STACK_SIZE, flags, args, &ret.fd);
+ #endif
+
+ if (ret.pid < 0) {
+@@ -129,6 +138,11 @@ static inline int child_join(struct child *child, struct error *err)
+ else if (r > 0)
+ error_set(err, r, "child %d reported: %d", child->pid, r);
+
++ if (munmap(child->stack, CHILD_STACK_SIZE)) {
++ error_set(err, -1, "munmap of child stack failed (errno %d)", errno);
++ r = -1;
++ }
++
+ return r;
+ }
+
+diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c
+index 529eb700ac26a..9a2d64901d591 100644
+--- a/tools/testing/selftests/pidfd/pidfd_test.c
++++ b/tools/testing/selftests/pidfd/pidfd_test.c
+@@ -441,7 +441,6 @@ static void test_pidfd_poll_exec(int use_waitpid)
+ {
+ int pid, pidfd = 0;
+ int status, ret;
+- pthread_t t1;
+ time_t prog_start = time(NULL);
+ const char *test_name = "pidfd_poll check for premature notification on child thread exec";
+
+@@ -500,13 +499,14 @@ static int child_poll_leader_exit_test(void *args)
+ */
+ *child_exit_secs = time(NULL);
+ syscall(SYS_exit, 0);
++ /* Never reached, but appeases compiler thinking we should return. */
++ exit(0);
+ }
+
+ static void test_pidfd_poll_leader_exit(int use_waitpid)
+ {
+ int pid, pidfd = 0;
+- int status, ret;
+- time_t prog_start = time(NULL);
++ int status, ret = 0;
+ const char *test_name = "pidfd_poll check for premature notification on non-empty"
+ "group leader exit";
+
+diff --git a/tools/testing/selftests/pidfd/pidfd_wait.c b/tools/testing/selftests/pidfd/pidfd_wait.c
+index be2943f072f60..17999e082aa71 100644
+--- a/tools/testing/selftests/pidfd/pidfd_wait.c
++++ b/tools/testing/selftests/pidfd/pidfd_wait.c
+@@ -39,7 +39,7 @@ static int sys_waitid(int which, pid_t pid, siginfo_t *info, int options,
+
+ TEST(wait_simple)
+ {
+- int pidfd = -1, status = 0;
++ int pidfd = -1;
+ pid_t parent_tid = -1;
+ struct clone_args args = {
+ .parent_tid = ptr_to_u64(&parent_tid),
+@@ -47,7 +47,6 @@ TEST(wait_simple)
+ .flags = CLONE_PIDFD | CLONE_PARENT_SETTID,
+ .exit_signal = SIGCHLD,
+ };
+- int ret;
+ pid_t pid;
+ siginfo_t info = {
+ .si_signo = 0,
+@@ -88,7 +87,7 @@ TEST(wait_simple)
+
+ TEST(wait_states)
+ {
+- int pidfd = -1, status = 0;
++ int pidfd = -1;
+ pid_t parent_tid = -1;
+ struct clone_args args = {
+ .parent_tid = ptr_to_u64(&parent_tid),
+diff --git a/tools/testing/selftests/powerpc/security/spectre_v2.c b/tools/testing/selftests/powerpc/security/spectre_v2.c
+index adc2b7294e5fd..83647b8277e7d 100644
+--- a/tools/testing/selftests/powerpc/security/spectre_v2.c
++++ b/tools/testing/selftests/powerpc/security/spectre_v2.c
+@@ -193,7 +193,7 @@ int spectre_v2_test(void)
+ * We are not vulnerable and reporting otherwise, so
+ * missing such a mismatch is safe.
+ */
+- if (state == VULNERABLE)
++ if (miss_percent > 95)
+ return 4;
+
+ return 1;
+diff --git a/tools/testing/selftests/powerpc/signal/.gitignore b/tools/testing/selftests/powerpc/signal/.gitignore
+index ce3375cd8e73e..8f6c816099a48 100644
+--- a/tools/testing/selftests/powerpc/signal/.gitignore
++++ b/tools/testing/selftests/powerpc/signal/.gitignore
+@@ -4,3 +4,4 @@ signal_tm
+ sigfuz
+ sigreturn_vdso
+ sig_sc_double_restart
++sigreturn_kernel
+diff --git a/tools/testing/selftests/powerpc/signal/Makefile b/tools/testing/selftests/powerpc/signal/Makefile
+index d6ae54663aed7..84e201572466d 100644
+--- a/tools/testing/selftests/powerpc/signal/Makefile
++++ b/tools/testing/selftests/powerpc/signal/Makefile
+@@ -1,5 +1,6 @@
+ # SPDX-License-Identifier: GPL-2.0
+ TEST_GEN_PROGS := signal signal_tm sigfuz sigreturn_vdso sig_sc_double_restart
++TEST_GEN_PROGS += sigreturn_kernel
+
+ CFLAGS += -maltivec
+ $(OUTPUT)/signal_tm: CFLAGS += -mhtm
+diff --git a/tools/testing/selftests/powerpc/signal/sigreturn_kernel.c b/tools/testing/selftests/powerpc/signal/sigreturn_kernel.c
+new file mode 100644
+index 0000000000000..0a1b6e591eeed
+--- /dev/null
++++ b/tools/testing/selftests/powerpc/signal/sigreturn_kernel.c
+@@ -0,0 +1,132 @@
++// SPDX-License-Identifier: GPL-2.0
++/*
++ * Test that we can't sigreturn to kernel addresses, or to kernel mode.
++ */
++
++#define _GNU_SOURCE
++
++#include <stdio.h>
++#include <signal.h>
++#include <stdlib.h>
++#include <sys/types.h>
++#include <sys/wait.h>
++#include <unistd.h>
++
++#include "utils.h"
++
++#define MSR_PR (1ul << 14)
++
++static volatile unsigned long long sigreturn_addr;
++static volatile unsigned long long sigreturn_msr_mask;
++
++static void sigusr1_handler(int signo, siginfo_t *si, void *uc_ptr)
++{
++ ucontext_t *uc = (ucontext_t *)uc_ptr;
++
++ if (sigreturn_addr)
++ UCONTEXT_NIA(uc) = sigreturn_addr;
++
++ if (sigreturn_msr_mask)
++ UCONTEXT_MSR(uc) &= sigreturn_msr_mask;
++}
++
++static pid_t fork_child(void)
++{
++ pid_t pid;
++
++ pid = fork();
++ if (pid == 0) {
++ raise(SIGUSR1);
++ exit(0);
++ }
++
++ return pid;
++}
++
++static int expect_segv(pid_t pid)
++{
++ int child_ret;
++
++ waitpid(pid, &child_ret, 0);
++ FAIL_IF(WIFEXITED(child_ret));
++ FAIL_IF(!WIFSIGNALED(child_ret));
++ FAIL_IF(WTERMSIG(child_ret) != 11);
++
++ return 0;
++}
++
++int test_sigreturn_kernel(void)
++{
++ struct sigaction act;
++ int child_ret, i;
++ pid_t pid;
++
++ act.sa_sigaction = sigusr1_handler;
++ act.sa_flags = SA_SIGINFO;
++ sigemptyset(&act.sa_mask);
++
++ FAIL_IF(sigaction(SIGUSR1, &act, NULL));
++
++ for (i = 0; i < 2; i++) {
++ // Return to kernel
++ sigreturn_addr = 0xcull << 60;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to kernel virtual
++ sigreturn_addr = 0xc008ull << 48;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return out of range
++ sigreturn_addr = 0xc010ull << 48;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to no-man's land, just below PAGE_OFFSET
++ sigreturn_addr = (0xcull << 60) - (64 * 1024);
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to no-man's land, above TASK_SIZE_4PB
++ sigreturn_addr = 0x1ull << 52;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to 0xd space
++ sigreturn_addr = 0xdull << 60;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to 0xe space
++ sigreturn_addr = 0xeull << 60;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Return to 0xf space
++ sigreturn_addr = 0xfull << 60;
++ pid = fork_child();
++ expect_segv(pid);
++
++ // Attempt to set PR=0 for 2nd loop (should be blocked by kernel)
++ sigreturn_msr_mask = ~MSR_PR;
++ }
++
++ printf("All children killed as expected\n");
++
++ // Don't change address, just MSR, should return to user as normal
++ sigreturn_addr = 0;
++ sigreturn_msr_mask = ~MSR_PR;
++ pid = fork_child();
++ waitpid(pid, &child_ret, 0);
++ FAIL_IF(!WIFEXITED(child_ret));
++ FAIL_IF(WIFSIGNALED(child_ret));
++ FAIL_IF(WEXITSTATUS(child_ret) != 0);
++
++ return 0;
++}
++
++int main(void)
++{
++ return test_harness(test_sigreturn_kernel, "sigreturn_kernel");
++}
+diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh
+index 363f56081eff3..66f0f724a1a6d 100755
+--- a/tools/testing/selftests/rcutorture/bin/torture.sh
++++ b/tools/testing/selftests/rcutorture/bin/torture.sh
+@@ -71,8 +71,8 @@ usage () {
+ echo " --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
+ echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
+ echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
+- echo " --doall"
+- echo " --doallmodconfig / --do-no-allmodconfig"
++ echo " --do-all"
++ echo " --do-allmodconfig / --do-no-allmodconfig"
+ echo " --do-clocksourcewd / --do-no-clocksourcewd"
+ echo " --do-kasan / --do-no-kasan"
+ echo " --do-kcsan / --do-no-kcsan"
+diff --git a/tools/testing/selftests/rtc/settings b/tools/testing/selftests/rtc/settings
+index ba4d85f74cd6b..a953c96aa16e1 100644
+--- a/tools/testing/selftests/rtc/settings
++++ b/tools/testing/selftests/rtc/settings
+@@ -1 +1 @@
+-timeout=90
++timeout=180
+diff --git a/tools/testing/selftests/sched/cs_prctl_test.c b/tools/testing/selftests/sched/cs_prctl_test.c
+index 7db9cf822dc75..8109b17dc764c 100644
+--- a/tools/testing/selftests/sched/cs_prctl_test.c
++++ b/tools/testing/selftests/sched/cs_prctl_test.c
+@@ -62,6 +62,17 @@ enum pid_type {PIDTYPE_PID = 0, PIDTYPE_TGID, PIDTYPE_PGID};
+
+ const int THREAD_CLONE_FLAGS = CLONE_THREAD | CLONE_SIGHAND | CLONE_FS | CLONE_VM | CLONE_FILES;
+
++struct child_args {
++ int num_threads;
++ int pfd[2];
++ int cpid;
++ int thr_tids[MAX_THREADS];
++};
++
++static struct child_args procs[MAX_PROCESSES];
++static int num_processes = 2;
++static int need_cleanup = 0;
++
+ static int _prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4,
+ unsigned long arg5)
+ {
+@@ -78,8 +89,14 @@ static int _prctl(int option, unsigned long arg2, unsigned long arg3, unsigned l
+ #define handle_error(msg) __handle_error(__FILE__, __LINE__, msg)
+ static void __handle_error(char *fn, int ln, char *msg)
+ {
++ int pidx;
+ printf("(%s:%d) - ", fn, ln);
+ perror(msg);
++ if (need_cleanup) {
++ for (pidx = 0; pidx < num_processes; ++pidx)
++ kill(procs[pidx].cpid, 15);
++ need_cleanup = 0;
++ }
+ exit(EXIT_FAILURE);
+ }
+
+@@ -106,13 +123,6 @@ static unsigned long get_cs_cookie(int pid)
+ return cookie;
+ }
+
+-struct child_args {
+- int num_threads;
+- int pfd[2];
+- int cpid;
+- int thr_tids[MAX_THREADS];
+-};
+-
+ static int child_func_thread(void __attribute__((unused))*arg)
+ {
+ while (1)
+@@ -212,10 +222,7 @@ void _validate(int line, int val, char *msg)
+
+ int main(int argc, char *argv[])
+ {
+- struct child_args procs[MAX_PROCESSES];
+-
+ int keypress = 0;
+- int num_processes = 2;
+ int num_threads = 3;
+ int delay = 0;
+ int res = 0;
+@@ -262,6 +269,7 @@ int main(int argc, char *argv[])
+
+ printf("\n## Create a thread/process/process group hiearchy\n");
+ create_processes(num_processes, num_threads, procs);
++ need_cleanup = 1;
+ disp_processes(num_processes, procs);
+ validate(get_cs_cookie(0) == 0);
+
+diff --git a/tools/testing/selftests/seccomp/Makefile b/tools/testing/selftests/seccomp/Makefile
+index 0ebfe8b0e147f..585f7a0c10cbe 100644
+--- a/tools/testing/selftests/seccomp/Makefile
++++ b/tools/testing/selftests/seccomp/Makefile
+@@ -1,5 +1,5 @@
+ # SPDX-License-Identifier: GPL-2.0
+-CFLAGS += -Wl,-no-as-needed -Wall
++CFLAGS += -Wl,-no-as-needed -Wall -isystem ../../../../usr/include/
+ LDFLAGS += -lpthread
+
+ TEST_GEN_PROGS := seccomp_bpf seccomp_benchmark
+diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c
+index 1d64891e64923..34ebd1fe5eed2 100644
+--- a/tools/testing/selftests/seccomp/seccomp_bpf.c
++++ b/tools/testing/selftests/seccomp/seccomp_bpf.c
+@@ -955,7 +955,7 @@ TEST(ERRNO_valid)
+ ASSERT_EQ(0, ret);
+
+ EXPECT_EQ(parent, syscall(__NR_getppid));
+- EXPECT_EQ(-1, read(0, NULL, 0));
++ EXPECT_EQ(-1, read(-1, NULL, 0));
+ EXPECT_EQ(E2BIG, errno);
+ }
+
+@@ -974,7 +974,7 @@ TEST(ERRNO_zero)
+
+ EXPECT_EQ(parent, syscall(__NR_getppid));
+ /* "errno" of 0 is ok. */
+- EXPECT_EQ(0, read(0, NULL, 0));
++ EXPECT_EQ(0, read(-1, NULL, 0));
+ }
+
+ /*
+@@ -995,7 +995,7 @@ TEST(ERRNO_capped)
+ ASSERT_EQ(0, ret);
+
+ EXPECT_EQ(parent, syscall(__NR_getppid));
+- EXPECT_EQ(-1, read(0, NULL, 0));
++ EXPECT_EQ(-1, read(-1, NULL, 0));
+ EXPECT_EQ(4095, errno);
+ }
+
+@@ -1026,7 +1026,7 @@ TEST(ERRNO_order)
+ ASSERT_EQ(0, ret);
+
+ EXPECT_EQ(parent, syscall(__NR_getppid));
+- EXPECT_EQ(-1, read(0, NULL, 0));
++ EXPECT_EQ(-1, read(-1, NULL, 0));
+ EXPECT_EQ(12, errno);
+ }
+
+@@ -2579,7 +2579,7 @@ void *tsync_sibling(void *data)
+ ret = prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0);
+ if (!ret)
+ return (void *)SIBLING_EXIT_NEWPRIVS;
+- read(0, NULL, 0);
++ read(-1, NULL, 0);
+ return (void *)SIBLING_EXIT_UNKILLED;
+ }
+
+diff --git a/tools/testing/selftests/sgx/Makefile b/tools/testing/selftests/sgx/Makefile
+index 7f12d55b97f86..472b27ccd7dcb 100644
+--- a/tools/testing/selftests/sgx/Makefile
++++ b/tools/testing/selftests/sgx/Makefile
+@@ -4,7 +4,7 @@ include ../lib.mk
+
+ .PHONY: all clean
+
+-CAN_BUILD_X86_64 := $(shell ../x86/check_cc.sh $(CC) \
++CAN_BUILD_X86_64 := $(shell ../x86/check_cc.sh "$(CC)" \
+ ../x86/trivial_64bit_program.c)
+
+ ifndef OBJCOPY
+diff --git a/tools/testing/selftests/vDSO/vdso_test_abi.c b/tools/testing/selftests/vDSO/vdso_test_abi.c
+index 3d603f1394af4..883ca85424bc5 100644
+--- a/tools/testing/selftests/vDSO/vdso_test_abi.c
++++ b/tools/testing/selftests/vDSO/vdso_test_abi.c
+@@ -33,110 +33,114 @@ typedef long (*vdso_clock_gettime_t)(clockid_t clk_id, struct timespec *ts);
+ typedef long (*vdso_clock_getres_t)(clockid_t clk_id, struct timespec *ts);
+ typedef time_t (*vdso_time_t)(time_t *t);
+
+-static int vdso_test_gettimeofday(void)
++#define VDSO_TEST_PASS_MSG() "\n%s(): PASS\n", __func__
++#define VDSO_TEST_FAIL_MSG(x) "\n%s(): %s FAIL\n", __func__, x
++#define VDSO_TEST_SKIP_MSG(x) "\n%s(): SKIP: Could not find %s\n", __func__, x
++
++static void vdso_test_gettimeofday(void)
+ {
+ /* Find gettimeofday. */
+ vdso_gettimeofday_t vdso_gettimeofday =
+ (vdso_gettimeofday_t)vdso_sym(version, name[0]);
+
+ if (!vdso_gettimeofday) {
+- printf("Could not find %s\n", name[0]);
+- return KSFT_SKIP;
++ ksft_test_result_skip(VDSO_TEST_SKIP_MSG(name[0]));
++ return;
+ }
+
+ struct timeval tv;
+ long ret = vdso_gettimeofday(&tv, 0);
+
+ if (ret == 0) {
+- printf("The time is %lld.%06lld\n",
+- (long long)tv.tv_sec, (long long)tv.tv_usec);
++ ksft_print_msg("The time is %lld.%06lld\n",
++ (long long)tv.tv_sec, (long long)tv.tv_usec);
++ ksft_test_result_pass(VDSO_TEST_PASS_MSG());
+ } else {
+- printf("%s failed\n", name[0]);
+- return KSFT_FAIL;
++ ksft_test_result_fail(VDSO_TEST_FAIL_MSG(name[0]));
+ }
+-
+- return KSFT_PASS;
+ }
+
+-static int vdso_test_clock_gettime(clockid_t clk_id)
++static void vdso_test_clock_gettime(clockid_t clk_id)
+ {
+ /* Find clock_gettime. */
+ vdso_clock_gettime_t vdso_clock_gettime =
+ (vdso_clock_gettime_t)vdso_sym(version, name[1]);
+
+ if (!vdso_clock_gettime) {
+- printf("Could not find %s\n", name[1]);
+- return KSFT_SKIP;
++ ksft_test_result_skip(VDSO_TEST_SKIP_MSG(name[1]));
++ return;
+ }
+
+ struct timespec ts;
+ long ret = vdso_clock_gettime(clk_id, &ts);
+
+ if (ret == 0) {
+- printf("The time is %lld.%06lld\n",
+- (long long)ts.tv_sec, (long long)ts.tv_nsec);
++ ksft_print_msg("The time is %lld.%06lld\n",
++ (long long)ts.tv_sec, (long long)ts.tv_nsec);
++ ksft_test_result_pass(VDSO_TEST_PASS_MSG());
+ } else {
+- printf("%s failed\n", name[1]);
+- return KSFT_FAIL;
++ ksft_test_result_fail(VDSO_TEST_FAIL_MSG(name[1]));
+ }
+-
+- return KSFT_PASS;
+ }
+
+-static int vdso_test_time(void)
++static void vdso_test_time(void)
+ {
+ /* Find time. */
+ vdso_time_t vdso_time =
+ (vdso_time_t)vdso_sym(version, name[2]);
+
+ if (!vdso_time) {
+- printf("Could not find %s\n", name[2]);
+- return KSFT_SKIP;
++ ksft_test_result_skip(VDSO_TEST_SKIP_MSG(name[2]));
++ return;
+ }
+
+ long ret = vdso_time(NULL);
+
+ if (ret > 0) {
+- printf("The time in hours since January 1, 1970 is %lld\n",
++ ksft_print_msg("The time in hours since January 1, 1970 is %lld\n",
+ (long long)(ret / 3600));
++ ksft_test_result_pass(VDSO_TEST_PASS_MSG());
+ } else {
+- printf("%s failed\n", name[2]);
+- return KSFT_FAIL;
++ ksft_test_result_fail(VDSO_TEST_FAIL_MSG(name[2]));
+ }
+-
+- return KSFT_PASS;
+ }
+
+-static int vdso_test_clock_getres(clockid_t clk_id)
++static void vdso_test_clock_getres(clockid_t clk_id)
+ {
++ int clock_getres_fail = 0;
++
+ /* Find clock_getres. */
+ vdso_clock_getres_t vdso_clock_getres =
+ (vdso_clock_getres_t)vdso_sym(version, name[3]);
+
+ if (!vdso_clock_getres) {
+- printf("Could not find %s\n", name[3]);
+- return KSFT_SKIP;
++ ksft_test_result_skip(VDSO_TEST_SKIP_MSG(name[3]));
++ return;
+ }
+
+ struct timespec ts, sys_ts;
+ long ret = vdso_clock_getres(clk_id, &ts);
+
+ if (ret == 0) {
+- printf("The resolution is %lld %lld\n",
+- (long long)ts.tv_sec, (long long)ts.tv_nsec);
++ ksft_print_msg("The vdso resolution is %lld %lld\n",
++ (long long)ts.tv_sec, (long long)ts.tv_nsec);
+ } else {
+- printf("%s failed\n", name[3]);
+- return KSFT_FAIL;
++ clock_getres_fail++;
+ }
+
+ ret = syscall(SYS_clock_getres, clk_id, &sys_ts);
+
+- if ((sys_ts.tv_sec != ts.tv_sec) || (sys_ts.tv_nsec != ts.tv_nsec)) {
+- printf("%s failed\n", name[3]);
+- return KSFT_FAIL;
+- }
++ ksft_print_msg("The syscall resolution is %lld %lld\n",
++ (long long)sys_ts.tv_sec, (long long)sys_ts.tv_nsec);
+
+- return KSFT_PASS;
++ if ((sys_ts.tv_sec != ts.tv_sec) || (sys_ts.tv_nsec != ts.tv_nsec))
++ clock_getres_fail++;
++
++ if (clock_getres_fail > 0) {
++ ksft_test_result_fail(VDSO_TEST_FAIL_MSG(name[3]));
++ } else {
++ ksft_test_result_pass(VDSO_TEST_PASS_MSG());
++ }
+ }
+
+ const char *vdso_clock_name[12] = {
+@@ -158,36 +162,23 @@ const char *vdso_clock_name[12] = {
+ * This function calls vdso_test_clock_gettime and vdso_test_clock_getres
+ * with different values for clock_id.
+ */
+-static inline int vdso_test_clock(clockid_t clock_id)
++static inline void vdso_test_clock(clockid_t clock_id)
+ {
+- int ret0, ret1;
+-
+- ret0 = vdso_test_clock_gettime(clock_id);
+- /* A skipped test is considered passed */
+- if (ret0 == KSFT_SKIP)
+- ret0 = KSFT_PASS;
+-
+- ret1 = vdso_test_clock_getres(clock_id);
+- /* A skipped test is considered passed */
+- if (ret1 == KSFT_SKIP)
+- ret1 = KSFT_PASS;
++ ksft_print_msg("\nclock_id: %s\n", vdso_clock_name[clock_id]);
+
+- ret0 += ret1;
++ vdso_test_clock_gettime(clock_id);
+
+- printf("clock_id: %s", vdso_clock_name[clock_id]);
+-
+- if (ret0 > 0)
+- printf(" [FAIL]\n");
+- else
+- printf(" [PASS]\n");
+-
+- return ret0;
++ vdso_test_clock_getres(clock_id);
+ }
+
++#define VDSO_TEST_PLAN 16
++
+ int main(int argc, char **argv)
+ {
+ unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
+- int ret;
++
++ ksft_print_header();
++ ksft_set_plan(VDSO_TEST_PLAN);
+
+ if (!sysinfo_ehdr) {
+ printf("AT_SYSINFO_EHDR is not present!\n");
+@@ -201,44 +192,42 @@ int main(int argc, char **argv)
+
+ vdso_init_from_sysinfo_ehdr(getauxval(AT_SYSINFO_EHDR));
+
+- ret = vdso_test_gettimeofday();
++ vdso_test_gettimeofday();
+
+ #if _POSIX_TIMERS > 0
+
+ #ifdef CLOCK_REALTIME
+- ret += vdso_test_clock(CLOCK_REALTIME);
++ vdso_test_clock(CLOCK_REALTIME);
+ #endif
+
+ #ifdef CLOCK_BOOTTIME
+- ret += vdso_test_clock(CLOCK_BOOTTIME);
++ vdso_test_clock(CLOCK_BOOTTIME);
+ #endif
+
+ #ifdef CLOCK_TAI
+- ret += vdso_test_clock(CLOCK_TAI);
++ vdso_test_clock(CLOCK_TAI);
+ #endif
+
+ #ifdef CLOCK_REALTIME_COARSE
+- ret += vdso_test_clock(CLOCK_REALTIME_COARSE);
++ vdso_test_clock(CLOCK_REALTIME_COARSE);
+ #endif
+
+ #ifdef CLOCK_MONOTONIC
+- ret += vdso_test_clock(CLOCK_MONOTONIC);
++ vdso_test_clock(CLOCK_MONOTONIC);
+ #endif
+
+ #ifdef CLOCK_MONOTONIC_RAW
+- ret += vdso_test_clock(CLOCK_MONOTONIC_RAW);
++ vdso_test_clock(CLOCK_MONOTONIC_RAW);
+ #endif
+
+ #ifdef CLOCK_MONOTONIC_COARSE
+- ret += vdso_test_clock(CLOCK_MONOTONIC_COARSE);
++ vdso_test_clock(CLOCK_MONOTONIC_COARSE);
+ #endif
+
+ #endif
+
+- ret += vdso_test_time();
+-
+- if (ret > 0)
+- return KSFT_FAIL;
++ vdso_test_time();
+
+- return KSFT_PASS;
++ ksft_print_cnts();
++ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+ }
+diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile
+index d9605bd10f2de..d8ae7cc012749 100644
+--- a/tools/testing/selftests/vm/Makefile
++++ b/tools/testing/selftests/vm/Makefile
+@@ -1,6 +1,8 @@
+ # SPDX-License-Identifier: GPL-2.0
+ # Makefile for vm selftests
+
++LOCAL_HDRS += $(selfdir)/vm/local_config.h $(top_srcdir)/mm/gup_test.h
++
+ include local_config.mk
+
+ uname_M := $(shell uname -m 2>/dev/null || echo not)
+@@ -48,13 +50,13 @@ TEST_GEN_FILES += split_huge_page_test
+ TEST_GEN_FILES += ksm_tests
+
+ ifeq ($(MACHINE),x86_64)
+-CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh $(CC) ../x86/trivial_32bit_program.c -m32)
+-CAN_BUILD_X86_64 := $(shell ./../x86/check_cc.sh $(CC) ../x86/trivial_64bit_program.c)
+-CAN_BUILD_WITH_NOPIE := $(shell ./../x86/check_cc.sh $(CC) ../x86/trivial_program.c -no-pie)
++CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32)
++CAN_BUILD_X86_64 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_64bit_program.c)
++CAN_BUILD_WITH_NOPIE := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_program.c -no-pie)
+
+-TARGETS := protection_keys
+-BINARIES_32 := $(TARGETS:%=%_32)
+-BINARIES_64 := $(TARGETS:%=%_64)
++VMTARGETS := protection_keys
++BINARIES_32 := $(VMTARGETS:%=%_32)
++BINARIES_64 := $(VMTARGETS:%=%_64)
+
+ ifeq ($(CAN_BUILD_WITH_NOPIE),1)
+ CFLAGS += -no-pie
+@@ -107,7 +109,7 @@ $(BINARIES_32): CFLAGS += -m32 -mxsave
+ $(BINARIES_32): LDLIBS += -lrt -ldl -lm
+ $(BINARIES_32): $(OUTPUT)/%_32: %.c
+ $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@
+-$(foreach t,$(TARGETS),$(eval $(call gen-target-rule-32,$(t))))
++$(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-32,$(t))))
+ endif
+
+ ifeq ($(CAN_BUILD_X86_64),1)
+@@ -115,7 +117,7 @@ $(BINARIES_64): CFLAGS += -m64 -mxsave
+ $(BINARIES_64): LDLIBS += -lrt -ldl
+ $(BINARIES_64): $(OUTPUT)/%_64: %.c
+ $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@
+-$(foreach t,$(TARGETS),$(eval $(call gen-target-rule-64,$(t))))
++$(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-64,$(t))))
+ endif
+
+ # x86_64 users should be encouraged to install 32-bit libraries
+@@ -139,10 +141,6 @@ endif
+
+ $(OUTPUT)/mlock-random-test $(OUTPUT)/memfd_secret: LDLIBS += -lcap
+
+-$(OUTPUT)/gup_test: ../../../../mm/gup_test.h
+-
+-$(OUTPUT)/hmm-tests: local_config.h
+-
+ # HMM_EXTRA_LIBS may get set in local_config.mk, or it may be left empty.
+ $(OUTPUT)/hmm-tests: LDLIBS += $(HMM_EXTRA_LIBS)
+
+diff --git a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
+index fe8fcfb334e06..a5cb4b09a46c4 100644
+--- a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
++++ b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh
+@@ -24,19 +24,23 @@ if [[ "$1" == "-cgroup-v2" ]]; then
+ reservation_usage_file=rsvd.current
+ fi
+
+-cgroup_path=/dev/cgroup/memory
+-if [[ ! -e $cgroup_path ]]; then
+- mkdir -p $cgroup_path
+- if [[ $cgroup2 ]]; then
++if [[ $cgroup2 ]]; then
++ cgroup_path=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
++ if [[ -z "$cgroup_path" ]]; then
++ cgroup_path=/dev/cgroup/memory
+ mount -t cgroup2 none $cgroup_path
+- else
++ do_umount=1
++ fi
++ echo "+hugetlb" >$cgroup_path/cgroup.subtree_control
++else
++ cgroup_path=$(mount -t cgroup | grep ",hugetlb" | awk -e '{print $3}')
++ if [[ -z "$cgroup_path" ]]; then
++ cgroup_path=/dev/cgroup/memory
+ mount -t cgroup memory,hugetlb $cgroup_path
++ do_umount=1
+ fi
+ fi
+-
+-if [[ $cgroup2 ]]; then
+- echo "+hugetlb" >/dev/cgroup/memory/cgroup.subtree_control
+-fi
++export cgroup_path
+
+ function cleanup() {
+ if [[ $cgroup2 ]]; then
+@@ -108,7 +112,7 @@ function setup_cgroup() {
+
+ function wait_for_hugetlb_memory_to_get_depleted() {
+ local cgroup="$1"
+- local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$reservation_usage_file"
++ local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file"
+ # Wait for hugetlbfs memory to get depleted.
+ while [ $(cat $path) != 0 ]; do
+ echo Waiting for hugetlb memory to get depleted.
+@@ -121,7 +125,7 @@ function wait_for_hugetlb_memory_to_get_reserved() {
+ local cgroup="$1"
+ local size="$2"
+
+- local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$reservation_usage_file"
++ local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file"
+ # Wait for hugetlbfs memory to get written.
+ while [ $(cat $path) != $size ]; do
+ echo Waiting for hugetlb memory reservation to reach size $size.
+@@ -134,7 +138,7 @@ function wait_for_hugetlb_memory_to_get_written() {
+ local cgroup="$1"
+ local size="$2"
+
+- local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$fault_usage_file"
++ local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file"
+ # Wait for hugetlbfs memory to get written.
+ while [ $(cat $path) != $size ]; do
+ echo Waiting for hugetlb memory to reach size $size.
+@@ -574,5 +578,7 @@ for populate in "" "-o"; do
+ done # populate
+ done # method
+
+-umount $cgroup_path
+-rmdir $cgroup_path
++if [[ $do_umount ]]; then
++ umount $cgroup_path
++ rmdir $cgroup_path
++fi
+diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c
+index 864f126ffd78f..203323967b507 100644
+--- a/tools/testing/selftests/vm/hmm-tests.c
++++ b/tools/testing/selftests/vm/hmm-tests.c
+@@ -1248,6 +1248,48 @@ TEST_F(hmm, anon_teardown)
+ }
+ }
+
++/*
++ * Test memory snapshot without faulting in pages accessed by the device.
++ */
++TEST_F(hmm, mixedmap)
++{
++ struct hmm_buffer *buffer;
++ unsigned long npages;
++ unsigned long size;
++ unsigned char *m;
++ int ret;
++
++ npages = 1;
++ size = npages << self->page_shift;
++
++ buffer = malloc(sizeof(*buffer));
++ ASSERT_NE(buffer, NULL);
++
++ buffer->fd = -1;
++ buffer->size = size;
++ buffer->mirror = malloc(npages);
++ ASSERT_NE(buffer->mirror, NULL);
++
++
++ /* Reserve a range of addresses. */
++ buffer->ptr = mmap(NULL, size,
++ PROT_READ | PROT_WRITE,
++ MAP_PRIVATE,
++ self->fd, 0);
++ ASSERT_NE(buffer->ptr, MAP_FAILED);
++
++ /* Simulate a device snapshotting CPU pagetables. */
++ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
++ ASSERT_EQ(ret, 0);
++ ASSERT_EQ(buffer->cpages, npages);
++
++ /* Check what the device saw. */
++ m = buffer->mirror;
++ ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
++
++ hmm_buffer_free(buffer);
++}
++
+ /*
+ * Test memory snapshot without faulting in pages accessed by the device.
+ */
+diff --git a/tools/testing/selftests/vm/hugetlb_reparenting_test.sh b/tools/testing/selftests/vm/hugetlb_reparenting_test.sh
+index 4a9a3afe9fd4d..bf2d2a684edfd 100644
+--- a/tools/testing/selftests/vm/hugetlb_reparenting_test.sh
++++ b/tools/testing/selftests/vm/hugetlb_reparenting_test.sh
+@@ -18,19 +18,24 @@ if [[ "$1" == "-cgroup-v2" ]]; then
+ usage_file=current
+ fi
+
+-CGROUP_ROOT='/dev/cgroup/memory'
+-MNT='/mnt/huge/'
+
+-if [[ ! -e $CGROUP_ROOT ]]; then
+- mkdir -p $CGROUP_ROOT
+- if [[ $cgroup2 ]]; then
++if [[ $cgroup2 ]]; then
++ CGROUP_ROOT=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
++ if [[ -z "$CGROUP_ROOT" ]]; then
++ CGROUP_ROOT=/dev/cgroup/memory
+ mount -t cgroup2 none $CGROUP_ROOT
+- sleep 1
+- echo "+hugetlb +memory" >$CGROUP_ROOT/cgroup.subtree_control
+- else
++ do_umount=1
++ fi
++ echo "+hugetlb +memory" >$CGROUP_ROOT/cgroup.subtree_control
++else
++ CGROUP_ROOT=$(mount -t cgroup | grep ",hugetlb" | awk -e '{print $3}')
++ if [[ -z "$CGROUP_ROOT" ]]; then
++ CGROUP_ROOT=/dev/cgroup/memory
+ mount -t cgroup memory,hugetlb $CGROUP_ROOT
++ do_umount=1
+ fi
+ fi
++MNT='/mnt/huge/'
+
+ function get_machine_hugepage_size() {
+ hpz=$(grep -i hugepagesize /proc/meminfo)
+diff --git a/tools/testing/selftests/vm/map_fixed_noreplace.c b/tools/testing/selftests/vm/map_fixed_noreplace.c
+index d91bde5112686..eed44322d1a63 100644
+--- a/tools/testing/selftests/vm/map_fixed_noreplace.c
++++ b/tools/testing/selftests/vm/map_fixed_noreplace.c
+@@ -17,9 +17,6 @@
+ #define MAP_FIXED_NOREPLACE 0x100000
+ #endif
+
+-#define BASE_ADDRESS (256ul * 1024 * 1024)
+-
+-
+ static void dump_maps(void)
+ {
+ char cmd[32];
+@@ -28,18 +25,46 @@ static void dump_maps(void)
+ system(cmd);
+ }
+
++static unsigned long find_base_addr(unsigned long size)
++{
++ void *addr;
++ unsigned long flags;
++
++ flags = MAP_PRIVATE | MAP_ANONYMOUS;
++ addr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
++ if (addr == MAP_FAILED) {
++ printf("Error: couldn't map the space we need for the test\n");
++ return 0;
++ }
++
++ if (munmap(addr, size) != 0) {
++ printf("Error: couldn't map the space we need for the test\n");
++ return 0;
++ }
++ return (unsigned long)addr;
++}
++
+ int main(void)
+ {
++ unsigned long base_addr;
+ unsigned long flags, addr, size, page_size;
+ char *p;
+
+ page_size = sysconf(_SC_PAGE_SIZE);
+
++ //let's find a base addr that is free before we start the tests
++ size = 5 * page_size;
++ base_addr = find_base_addr(size);
++ if (!base_addr) {
++ printf("Error: couldn't map the space we need for the test\n");
++ return 1;
++ }
++
+ flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE;
+
+ // Check we can map all the areas we need below
+ errno = 0;
+- addr = BASE_ADDRESS;
++ addr = base_addr;
+ size = 5 * page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+
+@@ -60,7 +85,7 @@ int main(void)
+ printf("unmap() successful\n");
+
+ errno = 0;
+- addr = BASE_ADDRESS + page_size;
++ addr = base_addr + page_size;
+ size = 3 * page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -80,7 +105,7 @@ int main(void)
+ * +4 | free | new
+ */
+ errno = 0;
+- addr = BASE_ADDRESS;
++ addr = base_addr;
+ size = 5 * page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -101,7 +126,7 @@ int main(void)
+ * +4 | free |
+ */
+ errno = 0;
+- addr = BASE_ADDRESS + (2 * page_size);
++ addr = base_addr + (2 * page_size);
+ size = page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -121,7 +146,7 @@ int main(void)
+ * +4 | free | new
+ */
+ errno = 0;
+- addr = BASE_ADDRESS + (3 * page_size);
++ addr = base_addr + (3 * page_size);
+ size = 2 * page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -141,7 +166,7 @@ int main(void)
+ * +4 | free |
+ */
+ errno = 0;
+- addr = BASE_ADDRESS;
++ addr = base_addr;
+ size = 2 * page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -161,7 +186,7 @@ int main(void)
+ * +4 | free |
+ */
+ errno = 0;
+- addr = BASE_ADDRESS;
++ addr = base_addr;
+ size = page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -181,7 +206,7 @@ int main(void)
+ * +4 | free | new
+ */
+ errno = 0;
+- addr = BASE_ADDRESS + (4 * page_size);
++ addr = base_addr + (4 * page_size);
+ size = page_size;
+ p = mmap((void *)addr, size, PROT_NONE, flags, -1, 0);
+ printf("mmap() @ 0x%lx-0x%lx p=%p result=%m\n", addr, addr + size, p);
+@@ -192,7 +217,7 @@ int main(void)
+ return 1;
+ }
+
+- addr = BASE_ADDRESS;
++ addr = base_addr;
+ size = 5 * page_size;
+ if (munmap((void *)addr, size) != 0) {
+ dump_maps();
+diff --git a/tools/testing/selftests/vm/mremap_test.c b/tools/testing/selftests/vm/mremap_test.c
+index 0624d1bd71b53..8f4dbbd60c09c 100644
+--- a/tools/testing/selftests/vm/mremap_test.c
++++ b/tools/testing/selftests/vm/mremap_test.c
+@@ -6,9 +6,11 @@
+
+ #include <errno.h>
+ #include <stdlib.h>
++#include <stdio.h>
+ #include <string.h>
+ #include <sys/mman.h>
+ #include <time.h>
++#include <stdbool.h>
+
+ #include "../kselftest.h"
+
+@@ -64,6 +66,112 @@ enum {
+ .expect_failure = should_fail \
+ }
+
++/*
++ * Returns false if the requested remap region overlaps with an
++ * existing mapping (e.g text, stack) else returns true.
++ */
++static bool is_remap_region_valid(void *addr, unsigned long long size)
++{
++ void *remap_addr = NULL;
++ bool ret = true;
++
++ /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
++ remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
++ MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
++ -1, 0);
++
++ if (remap_addr == MAP_FAILED) {
++ if (errno == EEXIST)
++ ret = false;
++ } else {
++ munmap(remap_addr, size);
++ }
++
++ return ret;
++}
++
++/* Returns mmap_min_addr sysctl tunable from procfs */
++static unsigned long long get_mmap_min_addr(void)
++{
++ FILE *fp;
++ int n_matched;
++ static unsigned long long addr;
++
++ if (addr)
++ return addr;
++
++ fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
++ if (fp == NULL) {
++ ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
++ strerror(errno));
++ exit(KSFT_SKIP);
++ }
++
++ n_matched = fscanf(fp, "%llu", &addr);
++ if (n_matched != 1) {
++ ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
++ strerror(errno));
++ fclose(fp);
++ exit(KSFT_SKIP);
++ }
++
++ fclose(fp);
++ return addr;
++}
++
++/*
++ * Returns false if the requested remap region overlaps with an
++ * existing mapping (e.g text, stack) else returns true.
++ */
++static bool is_remap_region_valid(void *addr, unsigned long long size)
++{
++ void *remap_addr = NULL;
++ bool ret = true;
++
++ /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
++ remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
++ MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
++ -1, 0);
++
++ if (remap_addr == MAP_FAILED) {
++ if (errno == EEXIST)
++ ret = false;
++ } else {
++ munmap(remap_addr, size);
++ }
++
++ return ret;
++}
++
++/* Returns mmap_min_addr sysctl tunable from procfs */
++static unsigned long long get_mmap_min_addr(void)
++{
++ FILE *fp;
++ int n_matched;
++ static unsigned long long addr;
++
++ if (addr)
++ return addr;
++
++ fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
++ if (fp == NULL) {
++ ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
++ strerror(errno));
++ exit(KSFT_SKIP);
++ }
++
++ n_matched = fscanf(fp, "%llu", &addr);
++ if (n_matched != 1) {
++ ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
++ strerror(errno));
++ fclose(fp);
++ exit(KSFT_SKIP);
++ }
++
++ fclose(fp);
++ return addr;
++}
++
+ /*
+ * Returns the start address of the mapping on success, else returns
+ * NULL on failure.
+@@ -72,11 +180,18 @@ static void *get_source_mapping(struct config c)
+ {
+ unsigned long long addr = 0ULL;
+ void *src_addr = NULL;
++ unsigned long long mmap_min_addr;
++
++ mmap_min_addr = get_mmap_min_addr();
++
+ retry:
+ addr += c.src_alignment;
++ if (addr < mmap_min_addr)
++ goto retry;
++
+ src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
+- MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
+- -1, 0);
++ MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
++ -1, 0);
+ if (src_addr == MAP_FAILED) {
+ if (errno == EPERM || errno == EEXIST)
+ goto retry;
+@@ -91,8 +206,10 @@ retry:
+ * alignment in the tests.
+ */
+ if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
+- !((unsigned long long) src_addr & c.src_alignment))
++ !((unsigned long long) src_addr & c.src_alignment)) {
++ munmap(src_addr, c.region_size);
+ goto retry;
++ }
+
+ if (!src_addr)
+ goto error;
+@@ -141,9 +258,20 @@ static long long remap_region(struct config c, unsigned int threshold_mb,
+ if (!((unsigned long long) addr & c.dest_alignment))
+ addr = (void *) ((unsigned long long) addr | c.dest_alignment);
+
++ /* Don't destroy existing mappings unless expected to overlap */
++ while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
++ /* Check for unsigned overflow */
++ if (addr + c.dest_alignment < addr) {
++ ksft_print_msg("Couldn't find a valid region to remap to\n");
++ ret = -1;
++ goto out;
++ }
++ addr += c.dest_alignment;
++ }
++
+ clock_gettime(CLOCK_MONOTONIC, &t_start);
+ dest_addr = mremap(src_addr, c.region_size, c.region_size,
+- MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
++ MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
+ clock_gettime(CLOCK_MONOTONIC, &t_end);
+
+ if (dest_addr == MAP_FAILED) {
+diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c
+index 60aa1a4fc69b6..138b011c667e2 100644
+--- a/tools/testing/selftests/vm/userfaultfd.c
++++ b/tools/testing/selftests/vm/userfaultfd.c
+@@ -46,6 +46,7 @@
+ #include <signal.h>
+ #include <poll.h>
+ #include <string.h>
++#include <linux/mman.h>
+ #include <sys/mman.h>
+ #include <sys/syscall.h>
+ #include <sys/ioctl.h>
+@@ -86,7 +87,7 @@ static bool test_uffdio_minor = false;
+
+ static bool map_shared;
+ static int shm_fd;
+-static int huge_fd;
++static int huge_fd = -1; /* only used for hugetlb_shared test */
+ static char *huge_fd_off0;
+ static unsigned long long *count_verify;
+ static int uffd = -1;
+@@ -222,6 +223,9 @@ static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
+
+ static void hugetlb_release_pages(char *rel_area)
+ {
++ if (huge_fd == -1)
++ return;
++
+ if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+ rel_area == huge_fd_off0 ? 0 : nr_pages * page_size,
+ nr_pages * page_size))
+@@ -234,16 +238,17 @@ static void hugetlb_allocate_area(void **alloc_area)
+ char **alloc_area_alias;
+
+ *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
+- (map_shared ? MAP_SHARED : MAP_PRIVATE) |
+- MAP_HUGETLB,
+- huge_fd, *alloc_area == area_src ? 0 :
+- nr_pages * page_size);
++ map_shared ? MAP_SHARED :
++ MAP_PRIVATE | MAP_HUGETLB |
++ (*alloc_area == area_src ? 0 : MAP_NORESERVE),
++ huge_fd,
++ *alloc_area == area_src ? 0 : nr_pages * page_size);
+ if (*alloc_area == MAP_FAILED)
+ err("mmap of hugetlbfs file failed");
+
+ if (map_shared) {
+ area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
+- MAP_SHARED | MAP_HUGETLB,
++ MAP_SHARED,
+ huge_fd, *alloc_area == area_src ? 0 :
+ nr_pages * page_size);
+ if (area_alias == MAP_FAILED)
+diff --git a/tools/testing/selftests/vm/write_hugetlb_memory.sh b/tools/testing/selftests/vm/write_hugetlb_memory.sh
+index d3d0d108924d4..70a02301f4c27 100644
+--- a/tools/testing/selftests/vm/write_hugetlb_memory.sh
++++ b/tools/testing/selftests/vm/write_hugetlb_memory.sh
+@@ -14,7 +14,7 @@ want_sleep=$8
+ reserve=$9
+
+ echo "Putting task in cgroup '$cgroup'"
+-echo $$ > /dev/cgroup/memory/"$cgroup"/cgroup.procs
++echo $$ > ${cgroup_path:-/dev/cgroup/memory}/"$cgroup"/cgroup.procs
+
+ echo "Method is $method"
+
+diff --git a/tools/testing/selftests/wireguard/netns.sh b/tools/testing/selftests/wireguard/netns.sh
+index ebc4ee0fe179f..8a9461aa0878a 100755
+--- a/tools/testing/selftests/wireguard/netns.sh
++++ b/tools/testing/selftests/wireguard/netns.sh
+@@ -276,7 +276,11 @@ n0 ping -W 1 -c 1 192.168.241.2
+ n1 wg set wg0 peer "$pub2" endpoint 192.168.241.2:7
+ ip2 link del wg0
+ ip2 link del wg1
+-! n0 ping -W 1 -c 10 -f 192.168.241.2 || false # Should not crash kernel
++read _ _ tx_bytes_before < <(n0 wg show wg1 transfer)
++! n0 ping -W 1 -c 10 -f 192.168.241.2 || false
++sleep 1
++read _ _ tx_bytes_after < <(n0 wg show wg1 transfer)
++(( tx_bytes_after - tx_bytes_before < 70000 ))
+
+ ip0 link del wg1
+ ip1 link del wg0
+@@ -609,6 +613,28 @@ ip0 link set wg0 up
+ kill $ncat_pid
+ ip0 link del wg0
+
++# Ensure that dst_cache references don't outlive netns lifetime
++ip1 link add dev wg0 type wireguard
++ip2 link add dev wg0 type wireguard
++configure_peers
++ip1 link add veth1 type veth peer name veth2
++ip1 link set veth2 netns $netns2
++ip1 addr add fd00:aa::1/64 dev veth1
++ip2 addr add fd00:aa::2/64 dev veth2
++ip1 link set veth1 up
++ip2 link set veth2 up
++waitiface $netns1 veth1
++waitiface $netns2 veth2
++ip1 -6 route add default dev veth1 via fd00:aa::2
++ip2 -6 route add default dev veth2 via fd00:aa::1
++n1 wg set wg0 peer "$pub2" endpoint [fd00:aa::2]:2
++n2 wg set wg0 peer "$pub1" endpoint [fd00:aa::1]:1
++n1 ping6 -c 1 fd00::2
++pp ip netns delete $netns1
++pp ip netns delete $netns2
++pp ip netns add $netns1
++pp ip netns add $netns2
++
+ # Ensure there aren't circular reference loops
+ ip1 link add wg1 type wireguard
+ ip2 link add wg2 type wireguard
+@@ -627,7 +653,7 @@ while read -t 0.1 -r line 2>/dev/null || [[ $? -ne 142 ]]; do
+ done < /dev/kmsg
+ alldeleted=1
+ for object in "${!objects[@]}"; do
+- if [[ ${objects["$object"]} != *createddestroyed ]]; then
++ if [[ ${objects["$object"]} != *createddestroyed && ${objects["$object"]} != *createdcreateddestroyeddestroyed ]]; then
+ echo "Error: $object: merely ${objects["$object"]}" >&3
+ alldeleted=0
+ fi
+diff --git a/tools/testing/selftests/wireguard/qemu/debug.config b/tools/testing/selftests/wireguard/qemu/debug.config
+index fe07d97df9fa8..2b321b8a96cf3 100644
+--- a/tools/testing/selftests/wireguard/qemu/debug.config
++++ b/tools/testing/selftests/wireguard/qemu/debug.config
+@@ -47,7 +47,7 @@ CONFIG_DEBUG_ATOMIC_SLEEP=y
+ CONFIG_TRACE_IRQFLAGS=y
+ CONFIG_DEBUG_BUGVERBOSE=y
+ CONFIG_DEBUG_LIST=y
+-CONFIG_DEBUG_PI_LIST=y
++CONFIG_DEBUG_PLIST=y
+ CONFIG_PROVE_RCU=y
+ CONFIG_SPARSE_RCU_POINTER=y
+ CONFIG_RCU_CPU_STALL_TIMEOUT=21
+diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config
+index 74db83a0aedd8..a9b5a520a1d22 100644
+--- a/tools/testing/selftests/wireguard/qemu/kernel.config
++++ b/tools/testing/selftests/wireguard/qemu/kernel.config
+@@ -66,6 +66,7 @@ CONFIG_PROC_SYSCTL=y
+ CONFIG_SYSFS=y
+ CONFIG_TMPFS=y
+ CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
++CONFIG_LOG_BUF_SHIFT=18
+ CONFIG_PRINTK_TIME=y
+ CONFIG_BLK_DEV_INITRD=y
+ CONFIG_LEGACY_VSYSCALL_NONE=y
+diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile
+index b4142cd1c5c23..02a77056bca3f 100644
+--- a/tools/testing/selftests/x86/Makefile
++++ b/tools/testing/selftests/x86/Makefile
+@@ -6,9 +6,9 @@ include ../lib.mk
+ .PHONY: all all_32 all_64 warn_32bit_failure clean
+
+ UNAME_M := $(shell uname -m)
+-CAN_BUILD_I386 := $(shell ./check_cc.sh $(CC) trivial_32bit_program.c -m32)
+-CAN_BUILD_X86_64 := $(shell ./check_cc.sh $(CC) trivial_64bit_program.c)
+-CAN_BUILD_WITH_NOPIE := $(shell ./check_cc.sh $(CC) trivial_program.c -no-pie)
++CAN_BUILD_I386 := $(shell ./check_cc.sh "$(CC)" trivial_32bit_program.c -m32)
++CAN_BUILD_X86_64 := $(shell ./check_cc.sh "$(CC)" trivial_64bit_program.c)
++CAN_BUILD_WITH_NOPIE := $(shell ./check_cc.sh "$(CC)" trivial_program.c -no-pie)
+
+ TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt test_mremap_vdso \
+ check_initial_reg_state sigreturn iopl ioperm \
+diff --git a/tools/testing/selftests/x86/check_cc.sh b/tools/testing/selftests/x86/check_cc.sh
+index 3e2089c8cf549..8c669c0d662ee 100755
+--- a/tools/testing/selftests/x86/check_cc.sh
++++ b/tools/testing/selftests/x86/check_cc.sh
+@@ -7,7 +7,7 @@ CC="$1"
+ TESTPROG="$2"
+ shift 2
+
+-if "$CC" -o /dev/null "$TESTPROG" -O0 "$@" 2>/dev/null; then
++if [ -n "$CC" ] && $CC -o /dev/null "$TESTPROG" -O0 "$@" 2>/dev/null; then
+ echo 1
+ else
+ echo 0
+diff --git a/tools/testing/selftests/x86/iopl.c b/tools/testing/selftests/x86/iopl.c
+index bab2f6e06b63d..7e3e09c1abac6 100644
+--- a/tools/testing/selftests/x86/iopl.c
++++ b/tools/testing/selftests/x86/iopl.c
+@@ -85,48 +85,88 @@ static void expect_gp_outb(unsigned short port)
+ printf("[OK]\toutb to 0x%02hx failed\n", port);
+ }
+
+-static bool try_cli(void)
++#define RET_FAULTED 0
++#define RET_FAIL 1
++#define RET_EMUL 2
++
++static int try_cli(void)
+ {
++ unsigned long flags;
++
+ sethandler(SIGSEGV, sigsegv, SA_RESETHAND);
+ if (sigsetjmp(jmpbuf, 1) != 0) {
+- return false;
++ return RET_FAULTED;
+ } else {
+- asm volatile ("cli");
+- return true;
++ asm volatile("cli; pushf; pop %[flags]"
++ : [flags] "=rm" (flags));
++
++ /* X86_FLAGS_IF */
++ if (!(flags & (1 << 9)))
++ return RET_FAIL;
++ else
++ return RET_EMUL;
+ }
+ clearhandler(SIGSEGV);
+ }
+
+-static bool try_sti(void)
++static int try_sti(bool irqs_off)
+ {
++ unsigned long flags;
++
+ sethandler(SIGSEGV, sigsegv, SA_RESETHAND);
+ if (sigsetjmp(jmpbuf, 1) != 0) {
+- return false;
++ return RET_FAULTED;
+ } else {
+- asm volatile ("sti");
+- return true;
++ asm volatile("sti; pushf; pop %[flags]"
++ : [flags] "=rm" (flags));
++
++ /* X86_FLAGS_IF */
++ if (irqs_off && (flags & (1 << 9)))
++ return RET_FAIL;
++ else
++ return RET_EMUL;
+ }
+ clearhandler(SIGSEGV);
+ }
+
+-static void expect_gp_sti(void)
++static void expect_gp_sti(bool irqs_off)
+ {
+- if (try_sti()) {
++ int ret = try_sti(irqs_off);
++
++ switch (ret) {
++ case RET_FAULTED:
++ printf("[OK]\tSTI faulted\n");
++ break;
++ case RET_EMUL:
++ printf("[OK]\tSTI NOPped\n");
++ break;
++ default:
+ printf("[FAIL]\tSTI worked\n");
+ nerrs++;
+- } else {
+- printf("[OK]\tSTI faulted\n");
+ }
+ }
+
+-static void expect_gp_cli(void)
++/*
++ * Returns whether it managed to disable interrupts.
++ */
++static bool test_cli(void)
+ {
+- if (try_cli()) {
++ int ret = try_cli();
++
++ switch (ret) {
++ case RET_FAULTED:
++ printf("[OK]\tCLI faulted\n");
++ break;
++ case RET_EMUL:
++ printf("[OK]\tCLI NOPped\n");
++ break;
++ default:
+ printf("[FAIL]\tCLI worked\n");
+ nerrs++;
+- } else {
+- printf("[OK]\tCLI faulted\n");
++ return true;
+ }
++
++ return false;
+ }
+
+ int main(void)
+@@ -152,8 +192,7 @@ int main(void)
+ }
+
+ /* Make sure that CLI/STI are blocked even with IOPL level 3 */
+- expect_gp_cli();
+- expect_gp_sti();
++ expect_gp_sti(test_cli());
+ expect_ok_outb(0x80);
+
+ /* Establish an I/O bitmap to test the restore */
+@@ -204,8 +243,7 @@ int main(void)
+ printf("[RUN]\tparent: write to 0x80 (should fail)\n");
+
+ expect_gp_outb(0x80);
+- expect_gp_cli();
+- expect_gp_sti();
++ expect_gp_sti(test_cli());
+
+ /* Test the capability checks. */
+ printf("\tiopl(3)\n");
+diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c
+index 65c141ebfbbde..5b45e6986aeab 100644
+--- a/tools/testing/selftests/x86/test_vsyscall.c
++++ b/tools/testing/selftests/x86/test_vsyscall.c
+@@ -497,7 +497,7 @@ static int test_process_vm_readv(void)
+ }
+
+ if (vsyscall_map_r) {
+- if (!memcmp(buf, (const void *)0xffffffffff600000, 4096)) {
++ if (!memcmp(buf, remote.iov_base, sizeof(buf))) {
+ printf("[OK]\tIt worked and read correct data\n");
+ } else {
+ printf("[FAIL]\tIt worked but returned incorrect data\n");
+diff --git a/tools/testing/selftests/zram/zram.sh b/tools/testing/selftests/zram/zram.sh
+index 232e958ec4547..b0b91d9b0dc21 100755
+--- a/tools/testing/selftests/zram/zram.sh
++++ b/tools/testing/selftests/zram/zram.sh
+@@ -2,9 +2,6 @@
+ # SPDX-License-Identifier: GPL-2.0
+ TCID="zram.sh"
+
+-# Kselftest framework requirement - SKIP code is 4.
+-ksft_skip=4
+-
+ . ./zram_lib.sh
+
+ run_zram () {
+@@ -18,14 +15,4 @@ echo ""
+
+ check_prereqs
+
+-# check zram module exists
+-MODULE_PATH=/lib/modules/`uname -r`/kernel/drivers/block/zram/zram.ko
+-if [ -f $MODULE_PATH ]; then
+- run_zram
+-elif [ -b /dev/zram0 ]; then
+- run_zram
+-else
+- echo "$TCID : No zram.ko module or /dev/zram0 device file not found"
+- echo "$TCID : CONFIG_ZRAM is not set"
+- exit $ksft_skip
+-fi
++run_zram
+diff --git a/tools/testing/selftests/zram/zram01.sh b/tools/testing/selftests/zram/zram01.sh
+index 114863d9fb876..8f4affe34f3e4 100755
+--- a/tools/testing/selftests/zram/zram01.sh
++++ b/tools/testing/selftests/zram/zram01.sh
+@@ -33,9 +33,7 @@ zram_algs="lzo"
+
+ zram_fill_fs()
+ {
+- local mem_free0=$(free -m | awk 'NR==2 {print $4}')
+-
+- for i in $(seq 0 $(($dev_num - 1))); do
++ for i in $(seq $dev_start $dev_end); do
+ echo "fill zram$i..."
+ local b=0
+ while [ true ]; do
+@@ -45,29 +43,17 @@ zram_fill_fs()
+ b=$(($b + 1))
+ done
+ echo "zram$i can be filled with '$b' KB"
+- done
+
+- local mem_free1=$(free -m | awk 'NR==2 {print $4}')
+- local used_mem=$(($mem_free0 - $mem_free1))
++ local mem_used_total=`awk '{print $3}' "/sys/block/zram$i/mm_stat"`
++ local v=$((100 * 1024 * $b / $mem_used_total))
++ if [ "$v" -lt 100 ]; then
++ echo "FAIL compression ratio: 0.$v:1"
++ ERR_CODE=-1
++ return
++ fi
+
+- local total_size=0
+- for sm in $zram_sizes; do
+- local s=$(echo $sm | sed 's/M//')
+- total_size=$(($total_size + $s))
++ echo "zram compression ratio: $(echo "scale=2; $v / 100 " | bc):1: OK"
+ done
+-
+- echo "zram used ${used_mem}M, zram disk sizes ${total_size}M"
+-
+- local v=$((100 * $total_size / $used_mem))
+-
+- if [ "$v" -lt 100 ]; then
+- echo "FAIL compression ratio: 0.$v:1"
+- ERR_CODE=-1
+- zram_cleanup
+- return
+- fi
+-
+- echo "zram compression ratio: $(echo "scale=2; $v / 100 " | bc):1: OK"
+ }
+
+ check_prereqs
+@@ -81,7 +67,6 @@ zram_mount
+
+ zram_fill_fs
+ zram_cleanup
+-zram_unload
+
+ if [ $ERR_CODE -ne 0 ]; then
+ echo "$TCID : [FAIL]"
+diff --git a/tools/testing/selftests/zram/zram02.sh b/tools/testing/selftests/zram/zram02.sh
+index e83b404807c09..2418b0c4ed136 100755
+--- a/tools/testing/selftests/zram/zram02.sh
++++ b/tools/testing/selftests/zram/zram02.sh
+@@ -36,7 +36,6 @@ zram_set_memlimit
+ zram_makeswap
+ zram_swapoff
+ zram_cleanup
+-zram_unload
+
+ if [ $ERR_CODE -ne 0 ]; then
+ echo "$TCID : [FAIL]"
+diff --git a/tools/testing/selftests/zram/zram_lib.sh b/tools/testing/selftests/zram/zram_lib.sh
+index 6f872f266fd11..21ec1966de76c 100755
+--- a/tools/testing/selftests/zram/zram_lib.sh
++++ b/tools/testing/selftests/zram/zram_lib.sh
+@@ -5,12 +5,17 @@
+ # Author: Alexey Kodanev <alexey.kodanev@oracle.com>
+ # Modified: Naresh Kamboju <naresh.kamboju@linaro.org>
+
+-MODULE=0
+ dev_makeswap=-1
+ dev_mounted=-1
+-
++dev_start=0
++dev_end=-1
++module_load=-1
++sys_control=-1
+ # Kselftest framework requirement - SKIP code is 4.
+ ksft_skip=4
++kernel_version=`uname -r | cut -d'.' -f1,2`
++kernel_major=${kernel_version%.*}
++kernel_minor=${kernel_version#*.}
+
+ trap INT
+
+@@ -25,68 +30,104 @@ check_prereqs()
+ fi
+ }
+
++kernel_gte()
++{
++ major=${1%.*}
++ minor=${1#*.}
++
++ if [ $kernel_major -gt $major ]; then
++ return 0
++ elif [[ $kernel_major -eq $major && $kernel_minor -ge $minor ]]; then
++ return 0
++ fi
++
++ return 1
++}
++
+ zram_cleanup()
+ {
+ echo "zram cleanup"
+ local i=
+- for i in $(seq 0 $dev_makeswap); do
++ for i in $(seq $dev_start $dev_makeswap); do
+ swapoff /dev/zram$i
+ done
+
+- for i in $(seq 0 $dev_mounted); do
++ for i in $(seq $dev_start $dev_mounted); do
+ umount /dev/zram$i
+ done
+
+- for i in $(seq 0 $(($dev_num - 1))); do
++ for i in $(seq $dev_start $dev_end); do
+ echo 1 > /sys/block/zram${i}/reset
+ rm -rf zram$i
+ done
+
+-}
++ if [ $sys_control -eq 1 ]; then
++ for i in $(seq $dev_start $dev_end); do
++ echo $i > /sys/class/zram-control/hot_remove
++ done
++ fi
+
+-zram_unload()
+-{
+- if [ $MODULE -ne 0 ] ; then
+- echo "zram rmmod zram"
++ if [ $module_load -eq 1 ]; then
+ rmmod zram > /dev/null 2>&1
+ fi
+ }
+
+ zram_load()
+ {
+- # check zram module exists
+- MODULE_PATH=/lib/modules/`uname -r`/kernel/drivers/block/zram/zram.ko
+- if [ -f $MODULE_PATH ]; then
+- MODULE=1
+- echo "create '$dev_num' zram device(s)"
+- modprobe zram num_devices=$dev_num
+- if [ $? -ne 0 ]; then
+- echo "failed to insert zram module"
+- exit 1
+- fi
+-
+- dev_num_created=$(ls /dev/zram* | wc -w)
++ echo "create '$dev_num' zram device(s)"
++
++ # zram module loaded, new kernel
++ if [ -d "/sys/class/zram-control" ]; then
++ echo "zram modules already loaded, kernel supports" \
++ "zram-control interface"
++ dev_start=$(ls /dev/zram* | wc -w)
++ dev_end=$(($dev_start + $dev_num - 1))
++ sys_control=1
++
++ for i in $(seq $dev_start $dev_end); do
++ cat /sys/class/zram-control/hot_add > /dev/null
++ done
++
++ echo "all zram devices (/dev/zram$dev_start~$dev_end" \
++ "successfully created"
++ return 0
++ fi
+
+- if [ "$dev_num_created" -ne "$dev_num" ]; then
+- echo "unexpected num of devices: $dev_num_created"
+- ERR_CODE=-1
++ # detect old kernel or built-in
++ modprobe zram num_devices=$dev_num
++ if [ ! -d "/sys/class/zram-control" ]; then
++ if grep -q '^zram' /proc/modules; then
++ rmmod zram > /dev/null 2>&1
++ if [ $? -ne 0 ]; then
++ echo "zram module is being used on old kernel" \
++ "without zram-control interface"
++ exit $ksft_skip
++ fi
+ else
+- echo "zram load module successful"
++ echo "test needs CONFIG_ZRAM=m on old kernel without" \
++ "zram-control interface"
++ exit $ksft_skip
+ fi
+- elif [ -b /dev/zram0 ]; then
+- echo "/dev/zram0 device file found: OK"
+- else
+- echo "ERROR: No zram.ko module or no /dev/zram0 device found"
+- echo "$TCID : CONFIG_ZRAM is not set"
+- exit 1
++ modprobe zram num_devices=$dev_num
+ fi
++
++ module_load=1
++ dev_end=$(($dev_num - 1))
++ echo "all zram devices (/dev/zram0~$dev_end) successfully created"
+ }
+
+ zram_max_streams()
+ {
+ echo "set max_comp_streams to zram device(s)"
+
+- local i=0
++ kernel_gte 4.7
++ if [ $? -eq 0 ]; then
++ echo "The device attribute max_comp_streams was"\
++ "deprecated in 4.7"
++ return 0
++ fi
++
++ local i=$dev_start
+ for max_s in $zram_max_streams; do
+ local sys_path="/sys/block/zram${i}/max_comp_streams"
+ echo $max_s > $sys_path || \
+@@ -98,7 +139,7 @@ zram_max_streams()
+ echo "FAIL can't set max_streams '$max_s', get $max_stream"
+
+ i=$(($i + 1))
+- echo "$sys_path = '$max_streams' ($i/$dev_num)"
++ echo "$sys_path = '$max_streams'"
+ done
+
+ echo "zram max streams: OK"
+@@ -108,15 +149,16 @@ zram_compress_alg()
+ {
+ echo "test that we can set compression algorithm"
+
+- local algs=$(cat /sys/block/zram0/comp_algorithm)
++ local i=$dev_start
++ local algs=$(cat /sys/block/zram${i}/comp_algorithm)
+ echo "supported algs: $algs"
+- local i=0
++
+ for alg in $zram_algs; do
+ local sys_path="/sys/block/zram${i}/comp_algorithm"
+ echo "$alg" > $sys_path || \
+ echo "FAIL can't set '$alg' to $sys_path"
+ i=$(($i + 1))
+- echo "$sys_path = '$alg' ($i/$dev_num)"
++ echo "$sys_path = '$alg'"
+ done
+
+ echo "zram set compression algorithm: OK"
+@@ -125,14 +167,14 @@ zram_compress_alg()
+ zram_set_disksizes()
+ {
+ echo "set disk size to zram device(s)"
+- local i=0
++ local i=$dev_start
+ for ds in $zram_sizes; do
+ local sys_path="/sys/block/zram${i}/disksize"
+ echo "$ds" > $sys_path || \
+ echo "FAIL can't set '$ds' to $sys_path"
+
+ i=$(($i + 1))
+- echo "$sys_path = '$ds' ($i/$dev_num)"
++ echo "$sys_path = '$ds'"
+ done
+
+ echo "zram set disksizes: OK"
+@@ -142,14 +184,14 @@ zram_set_memlimit()
+ {
+ echo "set memory limit to zram device(s)"
+
+- local i=0
++ local i=$dev_start
+ for ds in $zram_mem_limits; do
+ local sys_path="/sys/block/zram${i}/mem_limit"
+ echo "$ds" > $sys_path || \
+ echo "FAIL can't set '$ds' to $sys_path"
+
+ i=$(($i + 1))
+- echo "$sys_path = '$ds' ($i/$dev_num)"
++ echo "$sys_path = '$ds'"
+ done
+
+ echo "zram set memory limit: OK"
+@@ -158,8 +200,8 @@ zram_set_memlimit()
+ zram_makeswap()
+ {
+ echo "make swap with zram device(s)"
+- local i=0
+- for i in $(seq 0 $(($dev_num - 1))); do
++ local i=$dev_start
++ for i in $(seq $dev_start $dev_end); do
+ mkswap /dev/zram$i > err.log 2>&1
+ if [ $? -ne 0 ]; then
+ cat err.log
+@@ -182,7 +224,7 @@ zram_makeswap()
+ zram_swapoff()
+ {
+ local i=
+- for i in $(seq 0 $dev_makeswap); do
++ for i in $(seq $dev_start $dev_end); do
+ swapoff /dev/zram$i > err.log 2>&1
+ if [ $? -ne 0 ]; then
+ cat err.log
+@@ -196,7 +238,7 @@ zram_swapoff()
+
+ zram_makefs()
+ {
+- local i=0
++ local i=$dev_start
+ for fs in $zram_filesystems; do
+ # if requested fs not supported default it to ext2
+ which mkfs.$fs > /dev/null 2>&1 || fs=ext2
+@@ -215,7 +257,7 @@ zram_makefs()
+ zram_mount()
+ {
+ local i=0
+- for i in $(seq 0 $(($dev_num - 1))); do
++ for i in $(seq $dev_start $dev_end); do
+ echo "mount /dev/zram$i"
+ mkdir zram$i
+ mount /dev/zram$i zram$i > /dev/null || \
+diff --git a/tools/tracing/latency/latency-collector.c b/tools/tracing/latency/latency-collector.c
+index 3a2e6bb781a8c..59a7f2346eab4 100644
+--- a/tools/tracing/latency/latency-collector.c
++++ b/tools/tracing/latency/latency-collector.c
+@@ -1538,7 +1538,7 @@ static void tracing_loop(void)
+ mutex_lock(&print_mtx);
+ check_signals();
+ write_or_die(fd_stdout, queue_full_warning,
+- sizeof(queue_full_warning));
++ strlen(queue_full_warning));
+ mutex_unlock(&print_mtx);
+ }
+ modified--;
+diff --git a/tools/virtio/Makefile b/tools/virtio/Makefile
+index 0d7bbe49359d8..1b25cc7c64bbd 100644
+--- a/tools/virtio/Makefile
++++ b/tools/virtio/Makefile
+@@ -5,7 +5,8 @@ virtio_test: virtio_ring.o virtio_test.o
+ vringh_test: vringh_test.o vringh.o virtio_ring.o
+
+ CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h
+-LDFLAGS += -lpthread
++CFLAGS += -pthread
++LDFLAGS += -pthread
+ vpath %.c ../../drivers/virtio ../../drivers/vhost
+ mod:
+ ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}
+diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
+index cb3f29c09aff3..23f142af544ad 100644
+--- a/tools/virtio/virtio_test.c
++++ b/tools/virtio/virtio_test.c
+@@ -130,6 +130,7 @@ static void vdev_info_init(struct vdev_info* dev, unsigned long long features)
+ memset(dev, 0, sizeof *dev);
+ dev->vdev.features = features;
+ INIT_LIST_HEAD(&dev->vdev.vqs);
++ spin_lock_init(&dev->vdev.vqs_list_lock);
+ dev->buf_size = 1024;
+ dev->buf = malloc(dev->buf_size);
+ assert(dev->buf);
+diff --git a/usr/include/Makefile b/usr/include/Makefile
+index 1c2ae1368079d..adc6cb2587369 100644
+--- a/usr/include/Makefile
++++ b/usr/include/Makefile
+@@ -28,13 +28,13 @@ no-header-test += linux/am437x-vpfe.h
+ no-header-test += linux/android/binder.h
+ no-header-test += linux/android/binderfs.h
+ no-header-test += linux/coda.h
++no-header-test += linux/cyclades.h
+ no-header-test += linux/errqueue.h
+ no-header-test += linux/fsmap.h
+ no-header-test += linux/hdlc/ioctl.h
+ no-header-test += linux/ivtv.h
+ no-header-test += linux/kexec.h
+ no-header-test += linux/matroxfb.h
+-no-header-test += linux/nfc.h
+ no-header-test += linux/omap3isp.h
+ no-header-test += linux/omapfb.h
+ no-header-test += linux/patchkey.h
+diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c
+index e996989cd580e..5b874e7ba36fd 100644
+--- a/virt/kvm/eventfd.c
++++ b/virt/kvm/eventfd.c
+@@ -456,8 +456,8 @@ bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
+ idx = srcu_read_lock(&kvm->irq_srcu);
+ gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
+ if (gsi != -1)
+- hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
+- link)
++ hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
++ link, srcu_read_lock_held(&kvm->irq_srcu))
+ if (kian->gsi == gsi) {
+ srcu_read_unlock(&kvm->irq_srcu, idx);
+ return true;
+@@ -473,8 +473,8 @@ void kvm_notify_acked_gsi(struct kvm *kvm, int gsi)
+ {
+ struct kvm_irq_ack_notifier *kian;
+
+- hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
+- link)
++ hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
++ link, srcu_read_lock_held(&kvm->irq_srcu))
+ if (kian->gsi == gsi)
+ kian->irq_acked(kian);
+ }
+diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
+index 7851f3a1b5f7c..fefdf3a6dae35 100644
+--- a/virt/kvm/kvm_main.c
++++ b/virt/kvm/kvm_main.c
+@@ -117,6 +117,8 @@ EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
+
+ static const struct file_operations stat_fops_per_vm;
+
++static struct file_operations kvm_chardev_ops;
++
+ static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
+ unsigned long arg);
+ #ifdef CONFIG_KVM_COMPAT
+@@ -429,8 +431,8 @@ static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
+
+ void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
+ {
+- kvm_dirty_ring_free(&vcpu->dirty_ring);
+ kvm_arch_vcpu_destroy(vcpu);
++ kvm_dirty_ring_free(&vcpu->dirty_ring);
+
+ /*
+ * No need for rcu_read_lock as VCPU_RUN is the only place that changes
+@@ -1099,6 +1101,16 @@ static struct kvm *kvm_create_vm(unsigned long type)
+ preempt_notifier_inc();
+ kvm_init_pm_notifier(kvm);
+
++ /*
++ * When the fd passed to this ioctl() is opened it pins the module,
++ * but try_module_get() also prevents getting a reference if the module
++ * is in MODULE_STATE_GOING (e.g. if someone ran "rmmod --wait").
++ */
++ if (!try_module_get(kvm_chardev_ops.owner)) {
++ r = -ENODEV;
++ goto out_err;
++ }
++
+ return kvm;
+
+ out_err:
+@@ -1188,6 +1200,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
+ preempt_notifier_dec();
+ hardware_disable_all();
+ mmdrop(mm);
++ module_put(kvm_chardev_ops.owner);
+ }
+
+ void kvm_get_kvm(struct kvm *kvm)
+@@ -1523,11 +1536,10 @@ static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old,
+
+ static int kvm_set_memslot(struct kvm *kvm,
+ const struct kvm_userspace_memory_region *mem,
+- struct kvm_memory_slot *old,
+ struct kvm_memory_slot *new, int as_id,
+ enum kvm_mr_change change)
+ {
+- struct kvm_memory_slot *slot;
++ struct kvm_memory_slot *slot, old;
+ struct kvm_memslots *slots;
+ int r;
+
+@@ -1558,7 +1570,7 @@ static int kvm_set_memslot(struct kvm *kvm,
+ * Note, the INVALID flag needs to be in the appropriate entry
+ * in the freshly allocated memslots, not in @old or @new.
+ */
+- slot = id_to_memslot(slots, old->id);
++ slot = id_to_memslot(slots, new->id);
+ slot->flags |= KVM_MEMSLOT_INVALID;
+
+ /*
+@@ -1589,6 +1601,26 @@ static int kvm_set_memslot(struct kvm *kvm,
+ kvm_copy_memslots(slots, __kvm_memslots(kvm, as_id));
+ }
+
++ /*
++ * Make a full copy of the old memslot, the pointer will become stale
++ * when the memslots are re-sorted by update_memslots(), and the old
++ * memslot needs to be referenced after calling update_memslots(), e.g.
++ * to free its resources and for arch specific behavior. This needs to
++ * happen *after* (re)acquiring slots_arch_lock.
++ */
++ slot = id_to_memslot(slots, new->id);
++ if (slot) {
++ old = *slot;
++ } else {
++ WARN_ON_ONCE(change != KVM_MR_CREATE);
++ memset(&old, 0, sizeof(old));
++ old.id = new->id;
++ old.as_id = as_id;
++ }
++
++ /* Copy the arch-specific data, again after (re)acquiring slots_arch_lock. */
++ memcpy(&new->arch, &old.arch, sizeof(old.arch));
++
+ r = kvm_arch_prepare_memory_region(kvm, new, mem, change);
+ if (r)
+ goto out_slots;
+@@ -1596,14 +1628,18 @@ static int kvm_set_memslot(struct kvm *kvm,
+ update_memslots(slots, new, change);
+ slots = install_new_memslots(kvm, as_id, slots);
+
+- kvm_arch_commit_memory_region(kvm, mem, old, new, change);
++ kvm_arch_commit_memory_region(kvm, mem, &old, new, change);
++
++ /* Free the old memslot's metadata. Note, this is the full copy!!! */
++ if (change == KVM_MR_DELETE)
++ kvm_free_memslot(kvm, &old);
+
+ kvfree(slots);
+ return 0;
+
+ out_slots:
+ if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
+- slot = id_to_memslot(slots, old->id);
++ slot = id_to_memslot(slots, new->id);
+ slot->flags &= ~KVM_MEMSLOT_INVALID;
+ slots = install_new_memslots(kvm, as_id, slots);
+ } else {
+@@ -1618,7 +1654,6 @@ static int kvm_delete_memslot(struct kvm *kvm,
+ struct kvm_memory_slot *old, int as_id)
+ {
+ struct kvm_memory_slot new;
+- int r;
+
+ if (!old->npages)
+ return -EINVAL;
+@@ -1631,12 +1666,7 @@ static int kvm_delete_memslot(struct kvm *kvm,
+ */
+ new.as_id = as_id;
+
+- r = kvm_set_memslot(kvm, mem, old, &new, as_id, KVM_MR_DELETE);
+- if (r)
+- return r;
+-
+- kvm_free_memslot(kvm, old);
+- return 0;
++ return kvm_set_memslot(kvm, mem, &new, as_id, KVM_MR_DELETE);
+ }
+
+ /*
+@@ -1664,7 +1694,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
+ id = (u16)mem->slot;
+
+ /* General sanity checks */
+- if (mem->memory_size & (PAGE_SIZE - 1))
++ if ((mem->memory_size & (PAGE_SIZE - 1)) ||
++ (mem->memory_size != (unsigned long)mem->memory_size))
+ return -EINVAL;
+ if (mem->guest_phys_addr & (PAGE_SIZE - 1))
+ return -EINVAL;
+@@ -1710,7 +1741,6 @@ int __kvm_set_memory_region(struct kvm *kvm,
+ if (!old.npages) {
+ change = KVM_MR_CREATE;
+ new.dirty_bitmap = NULL;
+- memset(&new.arch, 0, sizeof(new.arch));
+ } else { /* Modify an existing slot. */
+ if ((new.userspace_addr != old.userspace_addr) ||
+ (new.npages != old.npages) ||
+@@ -1724,9 +1754,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
+ else /* Nothing to change. */
+ return 0;
+
+- /* Copy dirty_bitmap and arch from the current memslot. */
++ /* Copy dirty_bitmap from the current memslot. */
+ new.dirty_bitmap = old.dirty_bitmap;
+- memcpy(&new.arch, &old.arch, sizeof(new.arch));
+ }
+
+ if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
+@@ -1752,7 +1781,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
+ bitmap_set(new.dirty_bitmap, 0, new.npages);
+ }
+
+- r = kvm_set_memslot(kvm, mem, &old, &new, as_id, change);
++ r = kvm_set_memslot(kvm, mem, &new, as_id, change);
+ if (r)
+ goto out_bitmap;
+
+@@ -2088,7 +2117,6 @@ struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn
+
+ return NULL;
+ }
+-EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot);
+
+ bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
+ {
+@@ -2985,7 +3013,8 @@ int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
+ int r;
+ gpa_t gpa = ghc->gpa + offset;
+
+- BUG_ON(len + offset > ghc->len);
++ if (WARN_ON_ONCE(len + offset > ghc->len))
++ return -EINVAL;
+
+ if (slots->generation != ghc->generation) {
+ if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
+@@ -3022,7 +3051,8 @@ int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
+ int r;
+ gpa_t gpa = ghc->gpa + offset;
+
+- BUG_ON(len + offset > ghc->len);
++ if (WARN_ON_ONCE(len + offset > ghc->len))
++ return -EINVAL;
+
+ if (slots->generation != ghc->generation) {
+ if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
+@@ -3205,6 +3235,7 @@ update_halt_poll_stats(struct kvm_vcpu *vcpu, u64 poll_ns, bool waited)
+ */
+ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
+ {
++ bool halt_poll_allowed = !kvm_arch_no_poll(vcpu);
+ ktime_t start, cur, poll_end;
+ bool waited = false;
+ u64 block_ns;
+@@ -3212,7 +3243,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
+ kvm_arch_vcpu_blocking(vcpu);
+
+ start = cur = poll_end = ktime_get();
+- if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
++ if (vcpu->halt_poll_ns && halt_poll_allowed) {
+ ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
+
+ ++vcpu->stat.generic.halt_attempted_poll;
+@@ -3267,7 +3298,7 @@ out:
+ update_halt_poll_stats(
+ vcpu, ktime_to_ns(ktime_sub(poll_end, start)), waited);
+
+- if (!kvm_arch_no_poll(vcpu)) {
++ if (halt_poll_allowed) {
+ if (!vcpu_valid_wakeup(vcpu)) {
+ shrink_halt_poll_ns(vcpu);
+ } else if (vcpu->kvm->max_halt_poll_ns) {
+@@ -5399,9 +5430,7 @@ static int kvm_suspend(void)
+ static void kvm_resume(void)
+ {
+ if (kvm_usage_count) {
+-#ifdef CONFIG_LOCKDEP
+- WARN_ON(lockdep_is_held(&kvm_count_lock));
+-#endif
++ lockdep_assert_not_held(&kvm_count_lock);
+ hardware_enable_nolock(NULL);
+ }
+ }
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-10 10:33:14 +0000